3670 lines · cpp
1//===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "AMDGPUBaseInfo.h"10#include "AMDGPU.h"11#include "AMDGPUAsmUtils.h"12#include "AMDKernelCodeT.h"13#include "MCTargetDesc/AMDGPUMCTargetDesc.h"14#include "Utils/AMDKernelCodeTUtils.h"15#include "llvm/ADT/StringExtras.h"16#include "llvm/BinaryFormat/ELF.h"17#include "llvm/IR/Attributes.h"18#include "llvm/IR/Constants.h"19#include "llvm/IR/Function.h"20#include "llvm/IR/GlobalValue.h"21#include "llvm/IR/IntrinsicsAMDGPU.h"22#include "llvm/IR/IntrinsicsR600.h"23#include "llvm/IR/LLVMContext.h"24#include "llvm/IR/Metadata.h"25#include "llvm/MC/MCInstrInfo.h"26#include "llvm/MC/MCRegisterInfo.h"27#include "llvm/MC/MCSubtargetInfo.h"28#include "llvm/Support/CommandLine.h"29#include "llvm/TargetParser/TargetParser.h"30#include <optional>31 32#define GET_INSTRINFO_NAMED_OPS33#define GET_INSTRMAP_INFO34#include "AMDGPUGenInstrInfo.inc"35 36static llvm::cl::opt<unsigned> DefaultAMDHSACodeObjectVersion(37 "amdhsa-code-object-version", llvm::cl::Hidden,38 llvm::cl::init(llvm::AMDGPU::AMDHSA_COV6),39 llvm::cl::desc("Set default AMDHSA Code Object Version (module flag "40 "or asm directive still take priority if present)"));41 42namespace {43 44/// \returns Bit mask for given bit \p Shift and bit \p Width.45unsigned getBitMask(unsigned Shift, unsigned Width) {46 return ((1 << Width) - 1) << Shift;47}48 49/// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width.50///51/// \returns Packed \p Dst.52unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) {53 unsigned Mask = getBitMask(Shift, Width);54 return ((Src << Shift) & Mask) | (Dst & ~Mask);55}56 57/// Unpacks bits from \p Src for given bit \p Shift and bit \p Width.58///59/// \returns Unpacked bits.60unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) {61 return (Src & getBitMask(Shift, Width)) >> Shift;62}63 64/// \returns Vmcnt bit shift (lower bits).65unsigned getVmcntBitShiftLo(unsigned VersionMajor) {66 return VersionMajor >= 11 ? 10 : 0;67}68 69/// \returns Vmcnt bit width (lower bits).70unsigned getVmcntBitWidthLo(unsigned VersionMajor) {71 return VersionMajor >= 11 ? 6 : 4;72}73 74/// \returns Expcnt bit shift.75unsigned getExpcntBitShift(unsigned VersionMajor) {76 return VersionMajor >= 11 ? 0 : 4;77}78 79/// \returns Expcnt bit width.80unsigned getExpcntBitWidth(unsigned VersionMajor) { return 3; }81 82/// \returns Lgkmcnt bit shift.83unsigned getLgkmcntBitShift(unsigned VersionMajor) {84 return VersionMajor >= 11 ? 4 : 8;85}86 87/// \returns Lgkmcnt bit width.88unsigned getLgkmcntBitWidth(unsigned VersionMajor) {89 return VersionMajor >= 10 ? 6 : 4;90}91 92/// \returns Vmcnt bit shift (higher bits).93unsigned getVmcntBitShiftHi(unsigned VersionMajor) { return 14; }94 95/// \returns Vmcnt bit width (higher bits).96unsigned getVmcntBitWidthHi(unsigned VersionMajor) {97 return (VersionMajor == 9 || VersionMajor == 10) ? 2 : 0;98}99 100/// \returns Loadcnt bit width101unsigned getLoadcntBitWidth(unsigned VersionMajor) {102 return VersionMajor >= 12 ? 6 : 0;103}104 105/// \returns Samplecnt bit width.106unsigned getSamplecntBitWidth(unsigned VersionMajor) {107 return VersionMajor >= 12 ? 6 : 0;108}109 110/// \returns Bvhcnt bit width.111unsigned getBvhcntBitWidth(unsigned VersionMajor) {112 return VersionMajor >= 12 ? 3 : 0;113}114 115/// \returns Dscnt bit width.116unsigned getDscntBitWidth(unsigned VersionMajor) {117 return VersionMajor >= 12 ? 6 : 0;118}119 120/// \returns Dscnt bit shift in combined S_WAIT instructions.121unsigned getDscntBitShift(unsigned VersionMajor) { return 0; }122 123/// \returns Storecnt or Vscnt bit width, depending on VersionMajor.124unsigned getStorecntBitWidth(unsigned VersionMajor) {125 return VersionMajor >= 10 ? 6 : 0;126}127 128/// \returns Kmcnt bit width.129unsigned getKmcntBitWidth(unsigned VersionMajor) {130 return VersionMajor >= 12 ? 5 : 0;131}132 133/// \returns Xcnt bit width.134unsigned getXcntBitWidth(unsigned VersionMajor, unsigned VersionMinor) {135 return VersionMajor == 12 && VersionMinor == 5 ? 6 : 0;136}137 138/// \returns shift for Loadcnt/Storecnt in combined S_WAIT instructions.139unsigned getLoadcntStorecntBitShift(unsigned VersionMajor) {140 return VersionMajor >= 12 ? 8 : 0;141}142 143/// \returns VaSdst bit width144inline unsigned getVaSdstBitWidth() { return 3; }145 146/// \returns VaSdst bit shift147inline unsigned getVaSdstBitShift() { return 9; }148 149/// \returns VmVsrc bit width150inline unsigned getVmVsrcBitWidth() { return 3; }151 152/// \returns VmVsrc bit shift153inline unsigned getVmVsrcBitShift() { return 2; }154 155/// \returns VaVdst bit width156inline unsigned getVaVdstBitWidth() { return 4; }157 158/// \returns VaVdst bit shift159inline unsigned getVaVdstBitShift() { return 12; }160 161/// \returns VaVcc bit width162inline unsigned getVaVccBitWidth() { return 1; }163 164/// \returns VaVcc bit shift165inline unsigned getVaVccBitShift() { return 1; }166 167/// \returns SaSdst bit width168inline unsigned getSaSdstBitWidth() { return 1; }169 170/// \returns SaSdst bit shift171inline unsigned getSaSdstBitShift() { return 0; }172 173/// \returns VaSsrc width174inline unsigned getVaSsrcBitWidth() { return 1; }175 176/// \returns VaSsrc bit shift177inline unsigned getVaSsrcBitShift() { return 8; }178 179/// \returns HoldCnt bit shift180inline unsigned getHoldCntWidth() { return 1; }181 182/// \returns HoldCnt bit shift183inline unsigned getHoldCntBitShift() { return 7; }184 185} // end anonymous namespace186 187namespace llvm {188 189namespace AMDGPU {190 191/// \returns true if the target supports signed immediate offset for SMRD192/// instructions.193bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST) {194 return isGFX9Plus(ST);195}196 197/// \returns True if \p STI is AMDHSA.198bool isHsaAbi(const MCSubtargetInfo &STI) {199 return STI.getTargetTriple().getOS() == Triple::AMDHSA;200}201 202unsigned getAMDHSACodeObjectVersion(const Module &M) {203 if (auto *Ver = mdconst::extract_or_null<ConstantInt>(204 M.getModuleFlag("amdhsa_code_object_version"))) {205 return (unsigned)Ver->getZExtValue() / 100;206 }207 208 return getDefaultAMDHSACodeObjectVersion();209}210 211unsigned getDefaultAMDHSACodeObjectVersion() {212 return DefaultAMDHSACodeObjectVersion;213}214 215unsigned getAMDHSACodeObjectVersion(unsigned ABIVersion) {216 switch (ABIVersion) {217 case ELF::ELFABIVERSION_AMDGPU_HSA_V4:218 return 4;219 case ELF::ELFABIVERSION_AMDGPU_HSA_V5:220 return 5;221 case ELF::ELFABIVERSION_AMDGPU_HSA_V6:222 return 6;223 default:224 return getDefaultAMDHSACodeObjectVersion();225 }226}227 228uint8_t getELFABIVersion(const Triple &T, unsigned CodeObjectVersion) {229 if (T.getOS() != Triple::AMDHSA)230 return 0;231 232 switch (CodeObjectVersion) {233 case 4:234 return ELF::ELFABIVERSION_AMDGPU_HSA_V4;235 case 5:236 return ELF::ELFABIVERSION_AMDGPU_HSA_V5;237 case 6:238 return ELF::ELFABIVERSION_AMDGPU_HSA_V6;239 default:240 report_fatal_error("Unsupported AMDHSA Code Object Version " +241 Twine(CodeObjectVersion));242 }243}244 245unsigned getMultigridSyncArgImplicitArgPosition(unsigned CodeObjectVersion) {246 switch (CodeObjectVersion) {247 case AMDHSA_COV4:248 return 48;249 case AMDHSA_COV5:250 case AMDHSA_COV6:251 default:252 return AMDGPU::ImplicitArg::MULTIGRID_SYNC_ARG_OFFSET;253 }254}255 256// FIXME: All such magic numbers about the ABI should be in a257// central TD file.258unsigned getHostcallImplicitArgPosition(unsigned CodeObjectVersion) {259 switch (CodeObjectVersion) {260 case AMDHSA_COV4:261 return 24;262 case AMDHSA_COV5:263 case AMDHSA_COV6:264 default:265 return AMDGPU::ImplicitArg::HOSTCALL_PTR_OFFSET;266 }267}268 269unsigned getDefaultQueueImplicitArgPosition(unsigned CodeObjectVersion) {270 switch (CodeObjectVersion) {271 case AMDHSA_COV4:272 return 32;273 case AMDHSA_COV5:274 case AMDHSA_COV6:275 default:276 return AMDGPU::ImplicitArg::DEFAULT_QUEUE_OFFSET;277 }278}279 280unsigned getCompletionActionImplicitArgPosition(unsigned CodeObjectVersion) {281 switch (CodeObjectVersion) {282 case AMDHSA_COV4:283 return 40;284 case AMDHSA_COV5:285 case AMDHSA_COV6:286 default:287 return AMDGPU::ImplicitArg::COMPLETION_ACTION_OFFSET;288 }289}290 291#define GET_MIMGBaseOpcodesTable_IMPL292#define GET_MIMGDimInfoTable_IMPL293#define GET_MIMGInfoTable_IMPL294#define GET_MIMGLZMappingTable_IMPL295#define GET_MIMGMIPMappingTable_IMPL296#define GET_MIMGBiasMappingTable_IMPL297#define GET_MIMGOffsetMappingTable_IMPL298#define GET_MIMGG16MappingTable_IMPL299#define GET_MAIInstInfoTable_IMPL300#define GET_WMMAInstInfoTable_IMPL301#include "AMDGPUGenSearchableTables.inc"302 303int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding,304 unsigned VDataDwords, unsigned VAddrDwords) {305 const MIMGInfo *Info =306 getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding, VDataDwords, VAddrDwords);307 return Info ? Info->Opcode : -1;308}309 310const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc) {311 const MIMGInfo *Info = getMIMGInfo(Opc);312 return Info ? getMIMGBaseOpcodeInfo(Info->BaseOpcode) : nullptr;313}314 315int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) {316 const MIMGInfo *OrigInfo = getMIMGInfo(Opc);317 const MIMGInfo *NewInfo =318 getMIMGOpcodeHelper(OrigInfo->BaseOpcode, OrigInfo->MIMGEncoding,319 NewChannels, OrigInfo->VAddrDwords);320 return NewInfo ? NewInfo->Opcode : -1;321}322 323unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode,324 const MIMGDimInfo *Dim, bool IsA16,325 bool IsG16Supported) {326 unsigned AddrWords = BaseOpcode->NumExtraArgs;327 unsigned AddrComponents = (BaseOpcode->Coordinates ? Dim->NumCoords : 0) +328 (BaseOpcode->LodOrClampOrMip ? 1 : 0);329 if (IsA16)330 AddrWords += divideCeil(AddrComponents, 2);331 else332 AddrWords += AddrComponents;333 334 // Note: For subtargets that support A16 but not G16, enabling A16 also335 // enables 16 bit gradients.336 // For subtargets that support A16 (operand) and G16 (done with a different337 // instruction encoding), they are independent.338 339 if (BaseOpcode->Gradients) {340 if ((IsA16 && !IsG16Supported) || BaseOpcode->G16)341 // There are two gradients per coordinate, we pack them separately.342 // For the 3d case,343 // we get (dy/du, dx/du) (-, dz/du) (dy/dv, dx/dv) (-, dz/dv)344 AddrWords += alignTo<2>(Dim->NumGradients / 2);345 else346 AddrWords += Dim->NumGradients;347 }348 return AddrWords;349}350 351struct MUBUFInfo {352 uint16_t Opcode;353 uint16_t BaseOpcode;354 uint8_t elements;355 bool has_vaddr;356 bool has_srsrc;357 bool has_soffset;358 bool IsBufferInv;359 bool tfe;360};361 362struct MTBUFInfo {363 uint16_t Opcode;364 uint16_t BaseOpcode;365 uint8_t elements;366 bool has_vaddr;367 bool has_srsrc;368 bool has_soffset;369};370 371struct SMInfo {372 uint16_t Opcode;373 bool IsBuffer;374};375 376struct VOPInfo {377 uint16_t Opcode;378 bool IsSingle;379};380 381struct VOPC64DPPInfo {382 uint16_t Opcode;383};384 385struct VOPCDPPAsmOnlyInfo {386 uint16_t Opcode;387};388 389struct VOP3CDPPAsmOnlyInfo {390 uint16_t Opcode;391};392 393struct VOPDComponentInfo {394 uint16_t BaseVOP;395 uint16_t VOPDOp;396 bool CanBeVOPDX;397 bool CanBeVOPD3X;398};399 400struct VOPDInfo {401 uint16_t Opcode;402 uint16_t OpX;403 uint16_t OpY;404 uint16_t Subtarget;405 bool VOPD3;406};407 408struct VOPTrue16Info {409 uint16_t Opcode;410 bool IsTrue16;411};412 413#define GET_FP4FP8DstByteSelTable_DECL414#define GET_FP4FP8DstByteSelTable_IMPL415 416struct DPMACCInstructionInfo {417 uint16_t Opcode;418 bool IsDPMACCInstruction;419};420 421struct FP4FP8DstByteSelInfo {422 uint16_t Opcode;423 bool HasFP8DstByteSel;424 bool HasFP4DstByteSel;425};426 427#define GET_MTBUFInfoTable_DECL428#define GET_MTBUFInfoTable_IMPL429#define GET_MUBUFInfoTable_DECL430#define GET_MUBUFInfoTable_IMPL431#define GET_SMInfoTable_DECL432#define GET_SMInfoTable_IMPL433#define GET_VOP1InfoTable_DECL434#define GET_VOP1InfoTable_IMPL435#define GET_VOP2InfoTable_DECL436#define GET_VOP2InfoTable_IMPL437#define GET_VOP3InfoTable_DECL438#define GET_VOP3InfoTable_IMPL439#define GET_VOPC64DPPTable_DECL440#define GET_VOPC64DPPTable_IMPL441#define GET_VOPC64DPP8Table_DECL442#define GET_VOPC64DPP8Table_IMPL443#define GET_VOPCAsmOnlyInfoTable_DECL444#define GET_VOPCAsmOnlyInfoTable_IMPL445#define GET_VOP3CAsmOnlyInfoTable_DECL446#define GET_VOP3CAsmOnlyInfoTable_IMPL447#define GET_VOPDComponentTable_DECL448#define GET_VOPDComponentTable_IMPL449#define GET_VOPDPairs_DECL450#define GET_VOPDPairs_IMPL451#define GET_VOPTrue16Table_DECL452#define GET_VOPTrue16Table_IMPL453#define GET_True16D16Table_IMPL454#define GET_WMMAOpcode2AddrMappingTable_DECL455#define GET_WMMAOpcode2AddrMappingTable_IMPL456#define GET_WMMAOpcode3AddrMappingTable_DECL457#define GET_WMMAOpcode3AddrMappingTable_IMPL458#define GET_getMFMA_F8F6F4_WithSize_DECL459#define GET_getMFMA_F8F6F4_WithSize_IMPL460#define GET_isMFMA_F8F6F4Table_IMPL461#define GET_isCvtScaleF32_F32F16ToF8F4Table_IMPL462 463#include "AMDGPUGenSearchableTables.inc"464 465int getMTBUFBaseOpcode(unsigned Opc) {466 const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opc);467 return Info ? Info->BaseOpcode : -1;468}469 470int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) {471 const MTBUFInfo *Info =472 getMTBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements);473 return Info ? Info->Opcode : -1;474}475 476int getMTBUFElements(unsigned Opc) {477 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);478 return Info ? Info->elements : 0;479}480 481bool getMTBUFHasVAddr(unsigned Opc) {482 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);483 return Info && Info->has_vaddr;484}485 486bool getMTBUFHasSrsrc(unsigned Opc) {487 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);488 return Info && Info->has_srsrc;489}490 491bool getMTBUFHasSoffset(unsigned Opc) {492 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);493 return Info && Info->has_soffset;494}495 496int getMUBUFBaseOpcode(unsigned Opc) {497 const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opc);498 return Info ? Info->BaseOpcode : -1;499}500 501int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements) {502 const MUBUFInfo *Info =503 getMUBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements);504 return Info ? Info->Opcode : -1;505}506 507int getMUBUFElements(unsigned Opc) {508 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);509 return Info ? Info->elements : 0;510}511 512bool getMUBUFHasVAddr(unsigned Opc) {513 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);514 return Info && Info->has_vaddr;515}516 517bool getMUBUFHasSrsrc(unsigned Opc) {518 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);519 return Info && Info->has_srsrc;520}521 522bool getMUBUFHasSoffset(unsigned Opc) {523 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);524 return Info && Info->has_soffset;525}526 527bool getMUBUFIsBufferInv(unsigned Opc) {528 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);529 return Info && Info->IsBufferInv;530}531 532bool getMUBUFTfe(unsigned Opc) {533 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);534 return Info && Info->tfe;535}536 537bool getSMEMIsBuffer(unsigned Opc) {538 const SMInfo *Info = getSMEMOpcodeHelper(Opc);539 return Info && Info->IsBuffer;540}541 542bool getVOP1IsSingle(unsigned Opc) {543 const VOPInfo *Info = getVOP1OpcodeHelper(Opc);544 return !Info || Info->IsSingle;545}546 547bool getVOP2IsSingle(unsigned Opc) {548 const VOPInfo *Info = getVOP2OpcodeHelper(Opc);549 return !Info || Info->IsSingle;550}551 552bool getVOP3IsSingle(unsigned Opc) {553 const VOPInfo *Info = getVOP3OpcodeHelper(Opc);554 return !Info || Info->IsSingle;555}556 557bool isVOPC64DPP(unsigned Opc) {558 return isVOPC64DPPOpcodeHelper(Opc) || isVOPC64DPP8OpcodeHelper(Opc);559}560 561bool isVOPCAsmOnly(unsigned Opc) { return isVOPCAsmOnlyOpcodeHelper(Opc); }562 563bool getMAIIsDGEMM(unsigned Opc) {564 const MAIInstInfo *Info = getMAIInstInfoHelper(Opc);565 return Info && Info->is_dgemm;566}567 568bool getMAIIsGFX940XDL(unsigned Opc) {569 const MAIInstInfo *Info = getMAIInstInfoHelper(Opc);570 return Info && Info->is_gfx940_xdl;571}572 573bool getWMMAIsXDL(unsigned Opc) {574 const WMMAInstInfo *Info = getWMMAInstInfoHelper(Opc);575 return Info ? Info->is_wmma_xdl : false;576}577 578uint8_t mfmaScaleF8F6F4FormatToNumRegs(unsigned EncodingVal) {579 switch (EncodingVal) {580 case MFMAScaleFormats::FP6_E2M3:581 case MFMAScaleFormats::FP6_E3M2:582 return 6;583 case MFMAScaleFormats::FP4_E2M1:584 return 4;585 case MFMAScaleFormats::FP8_E4M3:586 case MFMAScaleFormats::FP8_E5M2:587 default:588 return 8;589 }590 591 llvm_unreachable("covered switch over mfma scale formats");592}593 594const MFMA_F8F6F4_Info *getMFMA_F8F6F4_WithFormatArgs(unsigned CBSZ,595 unsigned BLGP,596 unsigned F8F8Opcode) {597 uint8_t SrcANumRegs = mfmaScaleF8F6F4FormatToNumRegs(CBSZ);598 uint8_t SrcBNumRegs = mfmaScaleF8F6F4FormatToNumRegs(BLGP);599 return getMFMA_F8F6F4_InstWithNumRegs(SrcANumRegs, SrcBNumRegs, F8F8Opcode);600}601 602uint8_t wmmaScaleF8F6F4FormatToNumRegs(unsigned Fmt) {603 switch (Fmt) {604 case WMMA::MATRIX_FMT_FP8:605 case WMMA::MATRIX_FMT_BF8:606 return 16;607 case WMMA::MATRIX_FMT_FP6:608 case WMMA::MATRIX_FMT_BF6:609 return 12;610 case WMMA::MATRIX_FMT_FP4:611 return 8;612 }613 614 llvm_unreachable("covered switch over wmma scale formats");615}616 617const MFMA_F8F6F4_Info *getWMMA_F8F6F4_WithFormatArgs(unsigned FmtA,618 unsigned FmtB,619 unsigned F8F8Opcode) {620 uint8_t SrcANumRegs = wmmaScaleF8F6F4FormatToNumRegs(FmtA);621 uint8_t SrcBNumRegs = wmmaScaleF8F6F4FormatToNumRegs(FmtB);622 return getMFMA_F8F6F4_InstWithNumRegs(SrcANumRegs, SrcBNumRegs, F8F8Opcode);623}624 625unsigned getVOPDEncodingFamily(const MCSubtargetInfo &ST) {626 if (ST.hasFeature(AMDGPU::FeatureGFX1250Insts))627 return SIEncodingFamily::GFX1250;628 if (ST.hasFeature(AMDGPU::FeatureGFX12Insts))629 return SIEncodingFamily::GFX12;630 if (ST.hasFeature(AMDGPU::FeatureGFX11Insts))631 return SIEncodingFamily::GFX11;632 llvm_unreachable("Subtarget generation does not support VOPD!");633}634 635CanBeVOPD getCanBeVOPD(unsigned Opc, unsigned EncodingFamily, bool VOPD3) {636 bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(Opc) : 0;637 Opc = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : Opc;638 const VOPDComponentInfo *Info = getVOPDComponentHelper(Opc);639 if (Info) {640 // Check that Opc can be used as VOPDY for this encoding. V_MOV_B32 as a641 // VOPDX is just a placeholder here, it is supported on all encodings.642 // TODO: This can be optimized by creating tables of supported VOPDY643 // opcodes per encoding.644 unsigned VOPDMov = AMDGPU::getVOPDOpcode(AMDGPU::V_MOV_B32_e32, VOPD3);645 bool CanBeVOPDY = getVOPDFull(VOPDMov, AMDGPU::getVOPDOpcode(Opc, VOPD3),646 EncodingFamily, VOPD3) != -1;647 return {VOPD3 ? Info->CanBeVOPD3X : Info->CanBeVOPDX, CanBeVOPDY};648 }649 650 return {false, false};651}652 653unsigned getVOPDOpcode(unsigned Opc, bool VOPD3) {654 bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(Opc) : 0;655 Opc = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : Opc;656 const VOPDComponentInfo *Info = getVOPDComponentHelper(Opc);657 return Info ? Info->VOPDOp : ~0u;658}659 660bool isVOPD(unsigned Opc) {661 return AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::src0X);662}663 664bool isMAC(unsigned Opc) {665 return Opc == AMDGPU::V_MAC_F32_e64_gfx6_gfx7 ||666 Opc == AMDGPU::V_MAC_F32_e64_gfx10 ||667 Opc == AMDGPU::V_MAC_F32_e64_vi ||668 Opc == AMDGPU::V_MAC_LEGACY_F32_e64_gfx6_gfx7 ||669 Opc == AMDGPU::V_MAC_LEGACY_F32_e64_gfx10 ||670 Opc == AMDGPU::V_MAC_F16_e64_vi ||671 Opc == AMDGPU::V_FMAC_F64_e64_gfx90a ||672 Opc == AMDGPU::V_FMAC_F64_e64_gfx12 ||673 Opc == AMDGPU::V_FMAC_F32_e64_gfx10 ||674 Opc == AMDGPU::V_FMAC_F32_e64_gfx11 ||675 Opc == AMDGPU::V_FMAC_F32_e64_gfx12 ||676 Opc == AMDGPU::V_FMAC_F32_e64_vi ||677 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64_gfx10 ||678 Opc == AMDGPU::V_FMAC_DX9_ZERO_F32_e64_gfx11 ||679 Opc == AMDGPU::V_FMAC_F16_e64_gfx10 ||680 Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx11 ||681 Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx11 ||682 Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx12 ||683 Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx12 ||684 Opc == AMDGPU::V_DOT2C_F32_F16_e64_vi ||685 Opc == AMDGPU::V_DOT2C_F32_BF16_e64_vi ||686 Opc == AMDGPU::V_DOT2C_I32_I16_e64_vi ||687 Opc == AMDGPU::V_DOT4C_I32_I8_e64_vi ||688 Opc == AMDGPU::V_DOT8C_I32_I4_e64_vi;689}690 691bool isPermlane16(unsigned Opc) {692 return Opc == AMDGPU::V_PERMLANE16_B32_gfx10 ||693 Opc == AMDGPU::V_PERMLANEX16_B32_gfx10 ||694 Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx11 ||695 Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx11 ||696 Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx12 ||697 Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx12 ||698 Opc == AMDGPU::V_PERMLANE16_VAR_B32_e64_gfx12 ||699 Opc == AMDGPU::V_PERMLANEX16_VAR_B32_e64_gfx12;700}701 702bool isCvt_F32_Fp8_Bf8_e64(unsigned Opc) {703 return Opc == AMDGPU::V_CVT_F32_BF8_e64_gfx12 ||704 Opc == AMDGPU::V_CVT_F32_FP8_e64_gfx12 ||705 Opc == AMDGPU::V_CVT_F32_BF8_e64_dpp_gfx12 ||706 Opc == AMDGPU::V_CVT_F32_FP8_e64_dpp_gfx12 ||707 Opc == AMDGPU::V_CVT_F32_BF8_e64_dpp8_gfx12 ||708 Opc == AMDGPU::V_CVT_F32_FP8_e64_dpp8_gfx12 ||709 Opc == AMDGPU::V_CVT_PK_F32_BF8_fake16_e64_gfx12 ||710 Opc == AMDGPU::V_CVT_PK_F32_FP8_fake16_e64_gfx12 ||711 Opc == AMDGPU::V_CVT_PK_F32_BF8_t16_e64_gfx12 ||712 Opc == AMDGPU::V_CVT_PK_F32_FP8_t16_e64_gfx12;713}714 715bool isGenericAtomic(unsigned Opc) {716 return Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP ||717 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD ||718 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB ||719 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN ||720 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN ||721 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX ||722 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX ||723 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND ||724 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR ||725 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR ||726 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC ||727 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC ||728 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD ||729 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN ||730 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX ||731 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP ||732 Opc == AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG;733}734 735bool isAsyncStore(unsigned Opc) {736 return Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B8_gfx1250 ||737 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B32_gfx1250 ||738 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B64_gfx1250 ||739 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B128_gfx1250 ||740 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B8_SADDR_gfx1250 ||741 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B32_SADDR_gfx1250 ||742 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B64_SADDR_gfx1250 ||743 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR_gfx1250;744}745 746bool isTensorStore(unsigned Opc) {747 return Opc == TENSOR_STORE_FROM_LDS_gfx1250 ||748 Opc == TENSOR_STORE_FROM_LDS_D2_gfx1250;749}750 751unsigned getTemporalHintType(const MCInstrDesc TID) {752 if (TID.TSFlags & (SIInstrFlags::IsAtomicNoRet | SIInstrFlags::IsAtomicRet))753 return CPol::TH_TYPE_ATOMIC;754 unsigned Opc = TID.getOpcode();755 // Async and Tensor store should have the temporal hint type of TH_TYPE_STORE756 if (TID.mayStore() &&757 (isAsyncStore(Opc) || isTensorStore(Opc) || !TID.mayLoad()))758 return CPol::TH_TYPE_STORE;759 760 // This will default to returning TH_TYPE_LOAD when neither MayStore nor761 // MayLoad flag is present which is the case with instructions like762 // image_get_resinfo.763 return CPol::TH_TYPE_LOAD;764}765 766bool isTrue16Inst(unsigned Opc) {767 const VOPTrue16Info *Info = getTrue16OpcodeHelper(Opc);768 return Info && Info->IsTrue16;769}770 771FPType getFPDstSelType(unsigned Opc) {772 const FP4FP8DstByteSelInfo *Info = getFP4FP8DstByteSelHelper(Opc);773 if (!Info)774 return FPType::None;775 if (Info->HasFP8DstByteSel)776 return FPType::FP8;777 if (Info->HasFP4DstByteSel)778 return FPType::FP4;779 780 return FPType::None;781}782 783unsigned mapWMMA2AddrTo3AddrOpcode(unsigned Opc) {784 const WMMAOpcodeMappingInfo *Info = getWMMAMappingInfoFrom2AddrOpcode(Opc);785 return Info ? Info->Opcode3Addr : ~0u;786}787 788unsigned mapWMMA3AddrTo2AddrOpcode(unsigned Opc) {789 const WMMAOpcodeMappingInfo *Info = getWMMAMappingInfoFrom3AddrOpcode(Opc);790 return Info ? Info->Opcode2Addr : ~0u;791}792 793// Wrapper for Tablegen'd function. enum Subtarget is not defined in any794// header files, so we need to wrap it in a function that takes unsigned795// instead.796int getMCOpcode(uint16_t Opcode, unsigned Gen) {797 return getMCOpcodeGen(Opcode, static_cast<Subtarget>(Gen));798}799 800unsigned getBitOp2(unsigned Opc) {801 switch (Opc) {802 default:803 return 0;804 case AMDGPU::V_AND_B32_e32:805 return 0x40;806 case AMDGPU::V_OR_B32_e32:807 return 0x54;808 case AMDGPU::V_XOR_B32_e32:809 return 0x14;810 case AMDGPU::V_XNOR_B32_e32:811 return 0x41;812 }813}814 815int getVOPDFull(unsigned OpX, unsigned OpY, unsigned EncodingFamily,816 bool VOPD3) {817 bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(OpY) : 0;818 OpY = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : OpY;819 const VOPDInfo *Info =820 getVOPDInfoFromComponentOpcodes(OpX, OpY, EncodingFamily, VOPD3);821 return Info ? Info->Opcode : -1;822}823 824std::pair<unsigned, unsigned> getVOPDComponents(unsigned VOPDOpcode) {825 const VOPDInfo *Info = getVOPDOpcodeHelper(VOPDOpcode);826 assert(Info);827 const auto *OpX = getVOPDBaseFromComponent(Info->OpX);828 const auto *OpY = getVOPDBaseFromComponent(Info->OpY);829 assert(OpX && OpY);830 return {OpX->BaseVOP, OpY->BaseVOP};831}832 833namespace VOPD {834 835ComponentProps::ComponentProps(const MCInstrDesc &OpDesc, bool VOP3Layout) {836 assert(OpDesc.getNumDefs() == Component::DST_NUM);837 838 assert(OpDesc.getOperandConstraint(Component::SRC0, MCOI::TIED_TO) == -1);839 assert(OpDesc.getOperandConstraint(Component::SRC1, MCOI::TIED_TO) == -1);840 auto TiedIdx = OpDesc.getOperandConstraint(Component::SRC2, MCOI::TIED_TO);841 assert(TiedIdx == -1 || TiedIdx == Component::DST);842 HasSrc2Acc = TiedIdx != -1;843 Opcode = OpDesc.getOpcode();844 845 IsVOP3 = VOP3Layout || (OpDesc.TSFlags & SIInstrFlags::VOP3);846 SrcOperandsNum = AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src2) ? 3847 : AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::imm) ? 3848 : AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src1) ? 2849 : 1;850 assert(SrcOperandsNum <= Component::MAX_SRC_NUM);851 852 if (Opcode == AMDGPU::V_CNDMASK_B32_e32 ||853 Opcode == AMDGPU::V_CNDMASK_B32_e64) {854 // CNDMASK is an awkward exception, it has FP modifiers, but not FP855 // operands.856 NumVOPD3Mods = 2;857 if (IsVOP3)858 SrcOperandsNum = 3;859 } else if (isSISrcFPOperand(OpDesc,860 getNamedOperandIdx(Opcode, OpName::src0))) {861 // All FP VOPD instructions have Neg modifiers for all operands except862 // for tied src2.863 NumVOPD3Mods = SrcOperandsNum;864 if (HasSrc2Acc)865 --NumVOPD3Mods;866 }867 868 if (OpDesc.TSFlags & SIInstrFlags::VOP3)869 return;870 871 auto OperandsNum = OpDesc.getNumOperands();872 unsigned CompOprIdx;873 for (CompOprIdx = Component::SRC1; CompOprIdx < OperandsNum; ++CompOprIdx) {874 if (OpDesc.operands()[CompOprIdx].OperandType == AMDGPU::OPERAND_KIMM32) {875 MandatoryLiteralIdx = CompOprIdx;876 break;877 }878 }879}880 881int ComponentProps::getBitOp3OperandIdx() const {882 return getNamedOperandIdx(Opcode, OpName::bitop3);883}884 885unsigned ComponentInfo::getIndexInParsedOperands(unsigned CompOprIdx) const {886 assert(CompOprIdx < Component::MAX_OPR_NUM);887 888 if (CompOprIdx == Component::DST)889 return getIndexOfDstInParsedOperands();890 891 auto CompSrcIdx = CompOprIdx - Component::DST_NUM;892 if (CompSrcIdx < getCompParsedSrcOperandsNum())893 return getIndexOfSrcInParsedOperands(CompSrcIdx);894 895 // The specified operand does not exist.896 return 0;897}898 899std::optional<unsigned> InstInfo::getInvalidCompOperandIndex(900 std::function<MCRegister(unsigned, unsigned)> GetRegIdx,901 const MCRegisterInfo &MRI, bool SkipSrc, bool AllowSameVGPR,902 bool VOPD3) const {903 904 auto OpXRegs = getRegIndices(ComponentIndex::X, GetRegIdx,905 CompInfo[ComponentIndex::X].isVOP3());906 auto OpYRegs = getRegIndices(ComponentIndex::Y, GetRegIdx,907 CompInfo[ComponentIndex::Y].isVOP3());908 909 const auto banksOverlap = [&MRI](MCRegister X, MCRegister Y,910 unsigned BanksMask) -> bool {911 MCRegister BaseX = MRI.getSubReg(X, AMDGPU::sub0);912 MCRegister BaseY = MRI.getSubReg(Y, AMDGPU::sub0);913 if (!BaseX)914 BaseX = X;915 if (!BaseY)916 BaseY = Y;917 if ((BaseX.id() & BanksMask) == (BaseY.id() & BanksMask))918 return true;919 if (BaseX != X /* This is 64-bit register */ &&920 ((BaseX.id() + 1) & BanksMask) == (BaseY.id() & BanksMask))921 return true;922 if (BaseY != Y &&923 (BaseX.id() & BanksMask) == ((BaseY.id() + 1) & BanksMask))924 return true;925 926 // If both are 64-bit bank conflict will be detected yet while checking927 // the first subreg.928 return false;929 };930 931 unsigned CompOprIdx;932 for (CompOprIdx = 0; CompOprIdx < Component::MAX_OPR_NUM; ++CompOprIdx) {933 unsigned BanksMasks = VOPD3 ? VOPD3_VGPR_BANK_MASKS[CompOprIdx]934 : VOPD_VGPR_BANK_MASKS[CompOprIdx];935 if (!OpXRegs[CompOprIdx] || !OpYRegs[CompOprIdx])936 continue;937 938 if (getVGPREncodingMSBs(OpXRegs[CompOprIdx], MRI) !=939 getVGPREncodingMSBs(OpYRegs[CompOprIdx], MRI))940 return CompOprIdx;941 942 if (SkipSrc && CompOprIdx >= Component::DST_NUM)943 continue;944 945 if (CompOprIdx < Component::DST_NUM) {946 // Even if we do not check vdst parity, vdst operands still shall not947 // overlap.948 if (MRI.regsOverlap(OpXRegs[CompOprIdx], OpYRegs[CompOprIdx]))949 return CompOprIdx;950 if (VOPD3) // No need to check dst parity.951 continue;952 }953 954 if (banksOverlap(OpXRegs[CompOprIdx], OpYRegs[CompOprIdx], BanksMasks) &&955 (!AllowSameVGPR || CompOprIdx < Component::DST_NUM ||956 OpXRegs[CompOprIdx] != OpYRegs[CompOprIdx]))957 return CompOprIdx;958 }959 960 return {};961}962 963// Return an array of VGPR registers [DST,SRC0,SRC1,SRC2] used964// by the specified component. If an operand is unused965// or is not a VGPR, the corresponding value is 0.966//967// GetRegIdx(Component, MCOperandIdx) must return a VGPR register index968// for the specified component and MC operand. The callback must return 0969// if the operand is not a register or not a VGPR.970InstInfo::RegIndices971InstInfo::getRegIndices(unsigned CompIdx,972 std::function<MCRegister(unsigned, unsigned)> GetRegIdx,973 bool VOPD3) const {974 assert(CompIdx < COMPONENTS_NUM);975 976 const auto &Comp = CompInfo[CompIdx];977 InstInfo::RegIndices RegIndices;978 979 RegIndices[DST] = GetRegIdx(CompIdx, Comp.getIndexOfDstInMCOperands());980 981 for (unsigned CompOprIdx : {SRC0, SRC1, SRC2}) {982 unsigned CompSrcIdx = CompOprIdx - DST_NUM;983 RegIndices[CompOprIdx] =984 Comp.hasRegSrcOperand(CompSrcIdx)985 ? GetRegIdx(CompIdx,986 Comp.getIndexOfSrcInMCOperands(CompSrcIdx, VOPD3))987 : MCRegister();988 }989 return RegIndices;990}991 992} // namespace VOPD993 994VOPD::InstInfo getVOPDInstInfo(const MCInstrDesc &OpX, const MCInstrDesc &OpY) {995 return VOPD::InstInfo(OpX, OpY);996}997 998VOPD::InstInfo getVOPDInstInfo(unsigned VOPDOpcode,999 const MCInstrInfo *InstrInfo) {1000 auto [OpX, OpY] = getVOPDComponents(VOPDOpcode);1001 const auto &OpXDesc = InstrInfo->get(OpX);1002 const auto &OpYDesc = InstrInfo->get(OpY);1003 bool VOPD3 = InstrInfo->get(VOPDOpcode).TSFlags & SIInstrFlags::VOPD3;1004 VOPD::ComponentInfo OpXInfo(OpXDesc, VOPD::ComponentKind::COMPONENT_X, VOPD3);1005 VOPD::ComponentInfo OpYInfo(OpYDesc, OpXInfo, VOPD3);1006 return VOPD::InstInfo(OpXInfo, OpYInfo);1007}1008 1009namespace IsaInfo {1010 1011AMDGPUTargetID::AMDGPUTargetID(const MCSubtargetInfo &STI)1012 : STI(STI), XnackSetting(TargetIDSetting::Any),1013 SramEccSetting(TargetIDSetting::Any) {1014 if (!STI.getFeatureBits().test(FeatureSupportsXNACK))1015 XnackSetting = TargetIDSetting::Unsupported;1016 if (!STI.getFeatureBits().test(FeatureSupportsSRAMECC))1017 SramEccSetting = TargetIDSetting::Unsupported;1018}1019 1020void AMDGPUTargetID::setTargetIDFromFeaturesString(StringRef FS) {1021 // Check if xnack or sramecc is explicitly enabled or disabled. In the1022 // absence of the target features we assume we must generate code that can run1023 // in any environment.1024 SubtargetFeatures Features(FS);1025 std::optional<bool> XnackRequested;1026 std::optional<bool> SramEccRequested;1027 1028 for (const std::string &Feature : Features.getFeatures()) {1029 if (Feature == "+xnack")1030 XnackRequested = true;1031 else if (Feature == "-xnack")1032 XnackRequested = false;1033 else if (Feature == "+sramecc")1034 SramEccRequested = true;1035 else if (Feature == "-sramecc")1036 SramEccRequested = false;1037 }1038 1039 bool XnackSupported = isXnackSupported();1040 bool SramEccSupported = isSramEccSupported();1041 1042 if (XnackRequested) {1043 if (XnackSupported) {1044 XnackSetting =1045 *XnackRequested ? TargetIDSetting::On : TargetIDSetting::Off;1046 } else {1047 // If a specific xnack setting was requested and this GPU does not support1048 // xnack emit a warning. Setting will remain set to "Unsupported".1049 if (*XnackRequested) {1050 errs() << "warning: xnack 'On' was requested for a processor that does "1051 "not support it!\n";1052 } else {1053 errs() << "warning: xnack 'Off' was requested for a processor that "1054 "does not support it!\n";1055 }1056 }1057 }1058 1059 if (SramEccRequested) {1060 if (SramEccSupported) {1061 SramEccSetting =1062 *SramEccRequested ? TargetIDSetting::On : TargetIDSetting::Off;1063 } else {1064 // If a specific sramecc setting was requested and this GPU does not1065 // support sramecc emit a warning. Setting will remain set to1066 // "Unsupported".1067 if (*SramEccRequested) {1068 errs() << "warning: sramecc 'On' was requested for a processor that "1069 "does not support it!\n";1070 } else {1071 errs() << "warning: sramecc 'Off' was requested for a processor that "1072 "does not support it!\n";1073 }1074 }1075 }1076}1077 1078static TargetIDSetting1079getTargetIDSettingFromFeatureString(StringRef FeatureString) {1080 if (FeatureString.ends_with("-"))1081 return TargetIDSetting::Off;1082 if (FeatureString.ends_with("+"))1083 return TargetIDSetting::On;1084 1085 llvm_unreachable("Malformed feature string");1086}1087 1088void AMDGPUTargetID::setTargetIDFromTargetIDStream(StringRef TargetID) {1089 SmallVector<StringRef, 3> TargetIDSplit;1090 TargetID.split(TargetIDSplit, ':');1091 1092 for (const auto &FeatureString : TargetIDSplit) {1093 if (FeatureString.starts_with("xnack"))1094 XnackSetting = getTargetIDSettingFromFeatureString(FeatureString);1095 if (FeatureString.starts_with("sramecc"))1096 SramEccSetting = getTargetIDSettingFromFeatureString(FeatureString);1097 }1098}1099 1100std::string AMDGPUTargetID::toString() const {1101 std::string StringRep;1102 raw_string_ostream StreamRep(StringRep);1103 1104 auto TargetTriple = STI.getTargetTriple();1105 auto Version = getIsaVersion(STI.getCPU());1106 1107 StreamRep << TargetTriple.getArchName() << '-' << TargetTriple.getVendorName()1108 << '-' << TargetTriple.getOSName() << '-'1109 << TargetTriple.getEnvironmentName() << '-';1110 1111 std::string Processor;1112 // TODO: Following else statement is present here because we used various1113 // alias names for GPUs up until GFX9 (e.g. 'fiji' is same as 'gfx803').1114 // Remove once all aliases are removed from GCNProcessors.td.1115 if (Version.Major >= 9)1116 Processor = STI.getCPU().str();1117 else1118 Processor = (Twine("gfx") + Twine(Version.Major) + Twine(Version.Minor) +1119 Twine(Version.Stepping))1120 .str();1121 1122 std::string Features;1123 if (STI.getTargetTriple().getOS() == Triple::AMDHSA) {1124 // sramecc.1125 if (getSramEccSetting() == TargetIDSetting::Off)1126 Features += ":sramecc-";1127 else if (getSramEccSetting() == TargetIDSetting::On)1128 Features += ":sramecc+";1129 // xnack.1130 if (getXnackSetting() == TargetIDSetting::Off)1131 Features += ":xnack-";1132 else if (getXnackSetting() == TargetIDSetting::On)1133 Features += ":xnack+";1134 }1135 1136 StreamRep << Processor << Features;1137 1138 return StringRep;1139}1140 1141unsigned getWavefrontSize(const MCSubtargetInfo *STI) {1142 if (STI->getFeatureBits().test(FeatureWavefrontSize16))1143 return 16;1144 if (STI->getFeatureBits().test(FeatureWavefrontSize32))1145 return 32;1146 1147 return 64;1148}1149 1150unsigned getLocalMemorySize(const MCSubtargetInfo *STI) {1151 unsigned BytesPerCU = getAddressableLocalMemorySize(STI);1152 1153 // "Per CU" really means "per whatever functional block the waves of a1154 // workgroup must share". So the effective local memory size is doubled in1155 // WGP mode on gfx10.1156 if (isGFX10Plus(*STI) && !STI->getFeatureBits().test(FeatureCuMode))1157 BytesPerCU *= 2;1158 1159 return BytesPerCU;1160}1161 1162unsigned getAddressableLocalMemorySize(const MCSubtargetInfo *STI) {1163 if (STI->getFeatureBits().test(FeatureAddressableLocalMemorySize32768))1164 return 32768;1165 if (STI->getFeatureBits().test(FeatureAddressableLocalMemorySize65536))1166 return 65536;1167 if (STI->getFeatureBits().test(FeatureAddressableLocalMemorySize163840))1168 return 163840;1169 if (STI->getFeatureBits().test(FeatureAddressableLocalMemorySize327680))1170 return 327680;1171 return 32768;1172}1173 1174unsigned getEUsPerCU(const MCSubtargetInfo *STI) {1175 // "Per CU" really means "per whatever functional block the waves of a1176 // workgroup must share".1177 1178 // GFX12.5 only supports CU mode, which contains four SIMDs.1179 if (isGFX1250(*STI)) {1180 assert(STI->getFeatureBits().test(FeatureCuMode));1181 return 4;1182 }1183 1184 // For gfx10 in CU mode the functional block is the CU, which contains1185 // two SIMDs.1186 if (isGFX10Plus(*STI) && STI->getFeatureBits().test(FeatureCuMode))1187 return 2;1188 1189 // Pre-gfx10 a CU contains four SIMDs. For gfx10 in WGP mode the WGP1190 // contains two CUs, so a total of four SIMDs.1191 return 4;1192}1193 1194unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo *STI,1195 unsigned FlatWorkGroupSize) {1196 assert(FlatWorkGroupSize != 0);1197 if (!STI->getTargetTriple().isAMDGCN())1198 return 8;1199 unsigned MaxWaves = getMaxWavesPerEU(STI) * getEUsPerCU(STI);1200 unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize);1201 if (N == 1) {1202 // Single-wave workgroups don't consume barrier resources.1203 return MaxWaves;1204 }1205 1206 unsigned MaxBarriers = 16;1207 if (isGFX10Plus(*STI) && !STI->getFeatureBits().test(FeatureCuMode))1208 MaxBarriers = 32;1209 1210 return std::min(MaxWaves / N, MaxBarriers);1211}1212 1213unsigned getMinWavesPerEU(const MCSubtargetInfo *STI) { return 1; }1214 1215unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI) {1216 // FIXME: Need to take scratch memory into account.1217 if (isGFX90A(*STI))1218 return 8;1219 if (!isGFX10Plus(*STI))1220 return 10;1221 return hasGFX10_3Insts(*STI) ? 16 : 20;1222}1223 1224unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo *STI,1225 unsigned FlatWorkGroupSize) {1226 return divideCeil(getWavesPerWorkGroup(STI, FlatWorkGroupSize),1227 getEUsPerCU(STI));1228}1229 1230unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo *STI) { return 1; }1231 1232unsigned getMaxFlatWorkGroupSize(const MCSubtargetInfo *STI) {1233 // Some subtargets allow encoding 2048, but this isn't tested or supported.1234 return 1024;1235}1236 1237unsigned getWavesPerWorkGroup(const MCSubtargetInfo *STI,1238 unsigned FlatWorkGroupSize) {1239 return divideCeil(FlatWorkGroupSize, getWavefrontSize(STI));1240}1241 1242unsigned getSGPRAllocGranule(const MCSubtargetInfo *STI) {1243 IsaVersion Version = getIsaVersion(STI->getCPU());1244 if (Version.Major >= 10)1245 return getAddressableNumSGPRs(STI);1246 if (Version.Major >= 8)1247 return 16;1248 return 8;1249}1250 1251unsigned getSGPREncodingGranule(const MCSubtargetInfo *STI) { return 8; }1252 1253unsigned getTotalNumSGPRs(const MCSubtargetInfo *STI) {1254 IsaVersion Version = getIsaVersion(STI->getCPU());1255 if (Version.Major >= 8)1256 return 800;1257 return 512;1258}1259 1260unsigned getAddressableNumSGPRs(const MCSubtargetInfo *STI) {1261 if (STI->getFeatureBits().test(FeatureSGPRInitBug))1262 return FIXED_NUM_SGPRS_FOR_INIT_BUG;1263 1264 IsaVersion Version = getIsaVersion(STI->getCPU());1265 if (Version.Major >= 10)1266 return 106;1267 if (Version.Major >= 8)1268 return 102;1269 return 104;1270}1271 1272unsigned getMinNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {1273 assert(WavesPerEU != 0);1274 1275 IsaVersion Version = getIsaVersion(STI->getCPU());1276 if (Version.Major >= 10)1277 return 0;1278 1279 if (WavesPerEU >= getMaxWavesPerEU(STI))1280 return 0;1281 1282 unsigned MinNumSGPRs = getTotalNumSGPRs(STI) / (WavesPerEU + 1);1283 if (STI->getFeatureBits().test(FeatureTrapHandler))1284 MinNumSGPRs -= std::min(MinNumSGPRs, (unsigned)TRAP_NUM_SGPRS);1285 MinNumSGPRs = alignDown(MinNumSGPRs, getSGPRAllocGranule(STI)) + 1;1286 return std::min(MinNumSGPRs, getAddressableNumSGPRs(STI));1287}1288 1289unsigned getMaxNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU,1290 bool Addressable) {1291 assert(WavesPerEU != 0);1292 1293 unsigned AddressableNumSGPRs = getAddressableNumSGPRs(STI);1294 IsaVersion Version = getIsaVersion(STI->getCPU());1295 if (Version.Major >= 10)1296 return Addressable ? AddressableNumSGPRs : 108;1297 if (Version.Major >= 8 && !Addressable)1298 AddressableNumSGPRs = 112;1299 unsigned MaxNumSGPRs = getTotalNumSGPRs(STI) / WavesPerEU;1300 if (STI->getFeatureBits().test(FeatureTrapHandler))1301 MaxNumSGPRs -= std::min(MaxNumSGPRs, (unsigned)TRAP_NUM_SGPRS);1302 MaxNumSGPRs = alignDown(MaxNumSGPRs, getSGPRAllocGranule(STI));1303 return std::min(MaxNumSGPRs, AddressableNumSGPRs);1304}1305 1306unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed,1307 bool FlatScrUsed, bool XNACKUsed) {1308 unsigned ExtraSGPRs = 0;1309 if (VCCUsed)1310 ExtraSGPRs = 2;1311 1312 IsaVersion Version = getIsaVersion(STI->getCPU());1313 if (Version.Major >= 10)1314 return ExtraSGPRs;1315 1316 if (Version.Major < 8) {1317 if (FlatScrUsed)1318 ExtraSGPRs = 4;1319 } else {1320 if (XNACKUsed)1321 ExtraSGPRs = 4;1322 1323 if (FlatScrUsed ||1324 STI->getFeatureBits().test(AMDGPU::FeatureArchitectedFlatScratch))1325 ExtraSGPRs = 6;1326 }1327 1328 return ExtraSGPRs;1329}1330 1331unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed,1332 bool FlatScrUsed) {1333 return getNumExtraSGPRs(STI, VCCUsed, FlatScrUsed,1334 STI->getFeatureBits().test(AMDGPU::FeatureXNACK));1335}1336 1337static unsigned getGranulatedNumRegisterBlocks(unsigned NumRegs,1338 unsigned Granule) {1339 return divideCeil(std::max(1u, NumRegs), Granule);1340}1341 1342unsigned getNumSGPRBlocks(const MCSubtargetInfo *STI, unsigned NumSGPRs) {1343 // SGPRBlocks is actual number of SGPR blocks minus 1.1344 return getGranulatedNumRegisterBlocks(NumSGPRs, getSGPREncodingGranule(STI)) -1345 1;1346}1347 1348unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI,1349 unsigned DynamicVGPRBlockSize,1350 std::optional<bool> EnableWavefrontSize32) {1351 if (STI->getFeatureBits().test(FeatureGFX90AInsts))1352 return 8;1353 1354 if (DynamicVGPRBlockSize != 0)1355 return DynamicVGPRBlockSize;1356 1357 bool IsWave32 = EnableWavefrontSize321358 ? *EnableWavefrontSize321359 : STI->getFeatureBits().test(FeatureWavefrontSize32);1360 1361 if (STI->getFeatureBits().test(Feature1_5xVGPRs))1362 return IsWave32 ? 24 : 12;1363 1364 if (hasGFX10_3Insts(*STI))1365 return IsWave32 ? 16 : 8;1366 1367 return IsWave32 ? 8 : 4;1368}1369 1370unsigned getVGPREncodingGranule(const MCSubtargetInfo *STI,1371 std::optional<bool> EnableWavefrontSize32) {1372 if (STI->getFeatureBits().test(FeatureGFX90AInsts))1373 return 8;1374 1375 bool IsWave32 = EnableWavefrontSize321376 ? *EnableWavefrontSize321377 : STI->getFeatureBits().test(FeatureWavefrontSize32);1378 1379 if (STI->getFeatureBits().test(Feature1024AddressableVGPRs))1380 return IsWave32 ? 16 : 8;1381 1382 return IsWave32 ? 8 : 4;1383}1384 1385unsigned getArchVGPRAllocGranule() { return 4; }1386 1387unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) {1388 if (STI->getFeatureBits().test(FeatureGFX90AInsts))1389 return 512;1390 if (!isGFX10Plus(*STI))1391 return 256;1392 bool IsWave32 = STI->getFeatureBits().test(FeatureWavefrontSize32);1393 if (STI->getFeatureBits().test(Feature1_5xVGPRs))1394 return IsWave32 ? 1536 : 768;1395 return IsWave32 ? 1024 : 512;1396}1397 1398unsigned getAddressableNumArchVGPRs(const MCSubtargetInfo *STI) {1399 const auto &Features = STI->getFeatureBits();1400 if (Features.test(Feature1024AddressableVGPRs))1401 return Features.test(FeatureWavefrontSize32) ? 1024 : 512;1402 return 256;1403}1404 1405unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI,1406 unsigned DynamicVGPRBlockSize) {1407 const auto &Features = STI->getFeatureBits();1408 if (Features.test(FeatureGFX90AInsts))1409 return 512;1410 1411 if (DynamicVGPRBlockSize != 0)1412 // On GFX12 we can allocate at most 8 blocks of VGPRs.1413 return 8 * getVGPRAllocGranule(STI, DynamicVGPRBlockSize);1414 return getAddressableNumArchVGPRs(STI);1415}1416 1417unsigned getNumWavesPerEUWithNumVGPRs(const MCSubtargetInfo *STI,1418 unsigned NumVGPRs,1419 unsigned DynamicVGPRBlockSize) {1420 return getNumWavesPerEUWithNumVGPRs(1421 NumVGPRs, getVGPRAllocGranule(STI, DynamicVGPRBlockSize),1422 getMaxWavesPerEU(STI), getTotalNumVGPRs(STI));1423}1424 1425unsigned getNumWavesPerEUWithNumVGPRs(unsigned NumVGPRs, unsigned Granule,1426 unsigned MaxWaves,1427 unsigned TotalNumVGPRs) {1428 if (NumVGPRs < Granule)1429 return MaxWaves;1430 unsigned RoundedRegs = alignTo(NumVGPRs, Granule);1431 return std::min(std::max(TotalNumVGPRs / RoundedRegs, 1u), MaxWaves);1432}1433 1434unsigned getOccupancyWithNumSGPRs(unsigned SGPRs, unsigned MaxWaves,1435 AMDGPUSubtarget::Generation Gen) {1436 if (Gen >= AMDGPUSubtarget::GFX10)1437 return MaxWaves;1438 1439 if (Gen >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {1440 if (SGPRs <= 80)1441 return 10;1442 if (SGPRs <= 88)1443 return 9;1444 if (SGPRs <= 100)1445 return 8;1446 return 7;1447 }1448 if (SGPRs <= 48)1449 return 10;1450 if (SGPRs <= 56)1451 return 9;1452 if (SGPRs <= 64)1453 return 8;1454 if (SGPRs <= 72)1455 return 7;1456 if (SGPRs <= 80)1457 return 6;1458 return 5;1459}1460 1461unsigned getMinNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU,1462 unsigned DynamicVGPRBlockSize) {1463 assert(WavesPerEU != 0);1464 1465 unsigned MaxWavesPerEU = getMaxWavesPerEU(STI);1466 if (WavesPerEU >= MaxWavesPerEU)1467 return 0;1468 1469 unsigned TotNumVGPRs = getTotalNumVGPRs(STI);1470 unsigned AddrsableNumVGPRs =1471 getAddressableNumVGPRs(STI, DynamicVGPRBlockSize);1472 unsigned Granule = getVGPRAllocGranule(STI, DynamicVGPRBlockSize);1473 unsigned MaxNumVGPRs = alignDown(TotNumVGPRs / WavesPerEU, Granule);1474 1475 if (MaxNumVGPRs == alignDown(TotNumVGPRs / MaxWavesPerEU, Granule))1476 return 0;1477 1478 unsigned MinWavesPerEU = getNumWavesPerEUWithNumVGPRs(STI, AddrsableNumVGPRs,1479 DynamicVGPRBlockSize);1480 if (WavesPerEU < MinWavesPerEU)1481 return getMinNumVGPRs(STI, MinWavesPerEU, DynamicVGPRBlockSize);1482 1483 unsigned MaxNumVGPRsNext = alignDown(TotNumVGPRs / (WavesPerEU + 1), Granule);1484 unsigned MinNumVGPRs = 1 + std::min(MaxNumVGPRs - Granule, MaxNumVGPRsNext);1485 return std::min(MinNumVGPRs, AddrsableNumVGPRs);1486}1487 1488unsigned getMaxNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU,1489 unsigned DynamicVGPRBlockSize) {1490 assert(WavesPerEU != 0);1491 1492 unsigned MaxNumVGPRs =1493 alignDown(getTotalNumVGPRs(STI) / WavesPerEU,1494 getVGPRAllocGranule(STI, DynamicVGPRBlockSize));1495 unsigned AddressableNumVGPRs =1496 getAddressableNumVGPRs(STI, DynamicVGPRBlockSize);1497 return std::min(MaxNumVGPRs, AddressableNumVGPRs);1498}1499 1500unsigned getEncodedNumVGPRBlocks(const MCSubtargetInfo *STI, unsigned NumVGPRs,1501 std::optional<bool> EnableWavefrontSize32) {1502 return getGranulatedNumRegisterBlocks(1503 NumVGPRs, getVGPREncodingGranule(STI, EnableWavefrontSize32)) -1504 1;1505}1506 1507unsigned getAllocatedNumVGPRBlocks(const MCSubtargetInfo *STI,1508 unsigned NumVGPRs,1509 unsigned DynamicVGPRBlockSize,1510 std::optional<bool> EnableWavefrontSize32) {1511 return getGranulatedNumRegisterBlocks(1512 NumVGPRs,1513 getVGPRAllocGranule(STI, DynamicVGPRBlockSize, EnableWavefrontSize32));1514}1515} // end namespace IsaInfo1516 1517void initDefaultAMDKernelCodeT(AMDGPUMCKernelCodeT &KernelCode,1518 const MCSubtargetInfo *STI) {1519 IsaVersion Version = getIsaVersion(STI->getCPU());1520 KernelCode.amd_kernel_code_version_major = 1;1521 KernelCode.amd_kernel_code_version_minor = 2;1522 KernelCode.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU1523 KernelCode.amd_machine_version_major = Version.Major;1524 KernelCode.amd_machine_version_minor = Version.Minor;1525 KernelCode.amd_machine_version_stepping = Version.Stepping;1526 KernelCode.kernel_code_entry_byte_offset = sizeof(amd_kernel_code_t);1527 if (STI->getFeatureBits().test(FeatureWavefrontSize32)) {1528 KernelCode.wavefront_size = 5;1529 KernelCode.code_properties |= AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32;1530 } else {1531 KernelCode.wavefront_size = 6;1532 }1533 1534 // If the code object does not support indirect functions, then the value must1535 // be 0xffffffff.1536 KernelCode.call_convention = -1;1537 1538 // These alignment values are specified in powers of two, so alignment =1539 // 2^n. The minimum alignment is 2^4 = 16.1540 KernelCode.kernarg_segment_alignment = 4;1541 KernelCode.group_segment_alignment = 4;1542 KernelCode.private_segment_alignment = 4;1543 1544 if (Version.Major >= 10) {1545 KernelCode.compute_pgm_resource_registers |=1546 S_00B848_WGP_MODE(STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1) |1547 S_00B848_MEM_ORDERED(1) | S_00B848_FWD_PROGRESS(1);1548 }1549}1550 1551bool isGroupSegment(const GlobalValue *GV) {1552 return GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS;1553}1554 1555bool isGlobalSegment(const GlobalValue *GV) {1556 return GV->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS;1557}1558 1559bool isReadOnlySegment(const GlobalValue *GV) {1560 unsigned AS = GV->getAddressSpace();1561 return AS == AMDGPUAS::CONSTANT_ADDRESS ||1562 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;1563}1564 1565bool shouldEmitConstantsToTextSection(const Triple &TT) {1566 return TT.getArch() == Triple::r600;1567}1568 1569static bool isValidRegPrefix(char C) {1570 return C == 'v' || C == 's' || C == 'a';1571}1572 1573std::tuple<char, unsigned, unsigned> parseAsmPhysRegName(StringRef RegName) {1574 char Kind = RegName.front();1575 if (!isValidRegPrefix(Kind))1576 return {};1577 1578 RegName = RegName.drop_front();1579 if (RegName.consume_front("[")) {1580 unsigned Idx, End;1581 bool Failed = RegName.consumeInteger(10, Idx);1582 Failed |= !RegName.consume_front(":");1583 Failed |= RegName.consumeInteger(10, End);1584 Failed |= !RegName.consume_back("]");1585 if (!Failed) {1586 unsigned NumRegs = End - Idx + 1;1587 if (NumRegs > 1)1588 return {Kind, Idx, NumRegs};1589 }1590 } else {1591 unsigned Idx;1592 bool Failed = RegName.getAsInteger(10, Idx);1593 if (!Failed)1594 return {Kind, Idx, 1};1595 }1596 1597 return {};1598}1599 1600std::tuple<char, unsigned, unsigned>1601parseAsmConstraintPhysReg(StringRef Constraint) {1602 StringRef RegName = Constraint;1603 if (!RegName.consume_front("{") || !RegName.consume_back("}"))1604 return {};1605 return parseAsmPhysRegName(RegName);1606}1607 1608std::pair<unsigned, unsigned>1609getIntegerPairAttribute(const Function &F, StringRef Name,1610 std::pair<unsigned, unsigned> Default,1611 bool OnlyFirstRequired) {1612 if (auto Attr = getIntegerPairAttribute(F, Name, OnlyFirstRequired))1613 return {Attr->first, Attr->second.value_or(Default.second)};1614 return Default;1615}1616 1617std::optional<std::pair<unsigned, std::optional<unsigned>>>1618getIntegerPairAttribute(const Function &F, StringRef Name,1619 bool OnlyFirstRequired) {1620 Attribute A = F.getFnAttribute(Name);1621 if (!A.isStringAttribute())1622 return std::nullopt;1623 1624 LLVMContext &Ctx = F.getContext();1625 std::pair<unsigned, std::optional<unsigned>> Ints;1626 std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(',');1627 if (Strs.first.trim().getAsInteger(0, Ints.first)) {1628 Ctx.emitError("can't parse first integer attribute " + Name);1629 return std::nullopt;1630 }1631 unsigned Second = 0;1632 if (Strs.second.trim().getAsInteger(0, Second)) {1633 if (!OnlyFirstRequired || !Strs.second.trim().empty()) {1634 Ctx.emitError("can't parse second integer attribute " + Name);1635 return std::nullopt;1636 }1637 } else {1638 Ints.second = Second;1639 }1640 1641 return Ints;1642}1643 1644SmallVector<unsigned> getIntegerVecAttribute(const Function &F, StringRef Name,1645 unsigned Size,1646 unsigned DefaultVal) {1647 std::optional<SmallVector<unsigned>> R =1648 getIntegerVecAttribute(F, Name, Size);1649 return R.has_value() ? *R : SmallVector<unsigned>(Size, DefaultVal);1650}1651 1652std::optional<SmallVector<unsigned>>1653getIntegerVecAttribute(const Function &F, StringRef Name, unsigned Size) {1654 assert(Size > 2);1655 LLVMContext &Ctx = F.getContext();1656 1657 Attribute A = F.getFnAttribute(Name);1658 if (!A.isValid())1659 return std::nullopt;1660 if (!A.isStringAttribute()) {1661 Ctx.emitError(Name + " is not a string attribute");1662 return std::nullopt;1663 }1664 1665 SmallVector<unsigned> Vals(Size);1666 1667 StringRef S = A.getValueAsString();1668 unsigned i = 0;1669 for (; !S.empty() && i < Size; i++) {1670 std::pair<StringRef, StringRef> Strs = S.split(',');1671 unsigned IntVal;1672 if (Strs.first.trim().getAsInteger(0, IntVal)) {1673 Ctx.emitError("can't parse integer attribute " + Strs.first + " in " +1674 Name);1675 return std::nullopt;1676 }1677 Vals[i] = IntVal;1678 S = Strs.second;1679 }1680 1681 if (!S.empty() || i < Size) {1682 Ctx.emitError("attribute " + Name +1683 " has incorrect number of integers; expected " +1684 llvm::utostr(Size));1685 return std::nullopt;1686 }1687 return Vals;1688}1689 1690bool hasValueInRangeLikeMetadata(const MDNode &MD, int64_t Val) {1691 assert((MD.getNumOperands() % 2 == 0) && "invalid number of operands!");1692 for (unsigned I = 0, E = MD.getNumOperands() / 2; I != E; ++I) {1693 auto Low =1694 mdconst::extract<ConstantInt>(MD.getOperand(2 * I + 0))->getValue();1695 auto High =1696 mdconst::extract<ConstantInt>(MD.getOperand(2 * I + 1))->getValue();1697 // There are two types of [A; B) ranges:1698 // A < B, e.g. [4; 5) which is a range that only includes 4.1699 // A > B, e.g. [5; 4) which is a range that wraps around and includes1700 // everything except 4.1701 if (Low.ult(High)) {1702 if (Low.ule(Val) && High.ugt(Val))1703 return true;1704 } else {1705 if (Low.uge(Val) && High.ult(Val))1706 return true;1707 }1708 }1709 1710 return false;1711}1712 1713unsigned getVmcntBitMask(const IsaVersion &Version) {1714 return (1 << (getVmcntBitWidthLo(Version.Major) +1715 getVmcntBitWidthHi(Version.Major))) -1716 1;1717}1718 1719unsigned getLoadcntBitMask(const IsaVersion &Version) {1720 return (1 << getLoadcntBitWidth(Version.Major)) - 1;1721}1722 1723unsigned getSamplecntBitMask(const IsaVersion &Version) {1724 return (1 << getSamplecntBitWidth(Version.Major)) - 1;1725}1726 1727unsigned getBvhcntBitMask(const IsaVersion &Version) {1728 return (1 << getBvhcntBitWidth(Version.Major)) - 1;1729}1730 1731unsigned getExpcntBitMask(const IsaVersion &Version) {1732 return (1 << getExpcntBitWidth(Version.Major)) - 1;1733}1734 1735unsigned getLgkmcntBitMask(const IsaVersion &Version) {1736 return (1 << getLgkmcntBitWidth(Version.Major)) - 1;1737}1738 1739unsigned getDscntBitMask(const IsaVersion &Version) {1740 return (1 << getDscntBitWidth(Version.Major)) - 1;1741}1742 1743unsigned getKmcntBitMask(const IsaVersion &Version) {1744 return (1 << getKmcntBitWidth(Version.Major)) - 1;1745}1746 1747unsigned getXcntBitMask(const IsaVersion &Version) {1748 return (1 << getXcntBitWidth(Version.Major, Version.Minor)) - 1;1749}1750 1751unsigned getStorecntBitMask(const IsaVersion &Version) {1752 return (1 << getStorecntBitWidth(Version.Major)) - 1;1753}1754 1755unsigned getWaitcntBitMask(const IsaVersion &Version) {1756 unsigned VmcntLo = getBitMask(getVmcntBitShiftLo(Version.Major),1757 getVmcntBitWidthLo(Version.Major));1758 unsigned Expcnt = getBitMask(getExpcntBitShift(Version.Major),1759 getExpcntBitWidth(Version.Major));1760 unsigned Lgkmcnt = getBitMask(getLgkmcntBitShift(Version.Major),1761 getLgkmcntBitWidth(Version.Major));1762 unsigned VmcntHi = getBitMask(getVmcntBitShiftHi(Version.Major),1763 getVmcntBitWidthHi(Version.Major));1764 return VmcntLo | Expcnt | Lgkmcnt | VmcntHi;1765}1766 1767unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) {1768 unsigned VmcntLo = unpackBits(Waitcnt, getVmcntBitShiftLo(Version.Major),1769 getVmcntBitWidthLo(Version.Major));1770 unsigned VmcntHi = unpackBits(Waitcnt, getVmcntBitShiftHi(Version.Major),1771 getVmcntBitWidthHi(Version.Major));1772 return VmcntLo | VmcntHi << getVmcntBitWidthLo(Version.Major);1773}1774 1775unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) {1776 return unpackBits(Waitcnt, getExpcntBitShift(Version.Major),1777 getExpcntBitWidth(Version.Major));1778}1779 1780unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) {1781 return unpackBits(Waitcnt, getLgkmcntBitShift(Version.Major),1782 getLgkmcntBitWidth(Version.Major));1783}1784 1785void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned &Vmcnt,1786 unsigned &Expcnt, unsigned &Lgkmcnt) {1787 Vmcnt = decodeVmcnt(Version, Waitcnt);1788 Expcnt = decodeExpcnt(Version, Waitcnt);1789 Lgkmcnt = decodeLgkmcnt(Version, Waitcnt);1790}1791 1792Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded) {1793 Waitcnt Decoded;1794 Decoded.LoadCnt = decodeVmcnt(Version, Encoded);1795 Decoded.ExpCnt = decodeExpcnt(Version, Encoded);1796 Decoded.DsCnt = decodeLgkmcnt(Version, Encoded);1797 return Decoded;1798}1799 1800unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt,1801 unsigned Vmcnt) {1802 Waitcnt = packBits(Vmcnt, Waitcnt, getVmcntBitShiftLo(Version.Major),1803 getVmcntBitWidthLo(Version.Major));1804 return packBits(Vmcnt >> getVmcntBitWidthLo(Version.Major), Waitcnt,1805 getVmcntBitShiftHi(Version.Major),1806 getVmcntBitWidthHi(Version.Major));1807}1808 1809unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt,1810 unsigned Expcnt) {1811 return packBits(Expcnt, Waitcnt, getExpcntBitShift(Version.Major),1812 getExpcntBitWidth(Version.Major));1813}1814 1815unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt,1816 unsigned Lgkmcnt) {1817 return packBits(Lgkmcnt, Waitcnt, getLgkmcntBitShift(Version.Major),1818 getLgkmcntBitWidth(Version.Major));1819}1820 1821unsigned encodeWaitcnt(const IsaVersion &Version, unsigned Vmcnt,1822 unsigned Expcnt, unsigned Lgkmcnt) {1823 unsigned Waitcnt = getWaitcntBitMask(Version);1824 Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt);1825 Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt);1826 Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt);1827 return Waitcnt;1828}1829 1830unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded) {1831 return encodeWaitcnt(Version, Decoded.LoadCnt, Decoded.ExpCnt, Decoded.DsCnt);1832}1833 1834static unsigned getCombinedCountBitMask(const IsaVersion &Version,1835 bool IsStore) {1836 unsigned Dscnt = getBitMask(getDscntBitShift(Version.Major),1837 getDscntBitWidth(Version.Major));1838 if (IsStore) {1839 unsigned Storecnt = getBitMask(getLoadcntStorecntBitShift(Version.Major),1840 getStorecntBitWidth(Version.Major));1841 return Dscnt | Storecnt;1842 }1843 unsigned Loadcnt = getBitMask(getLoadcntStorecntBitShift(Version.Major),1844 getLoadcntBitWidth(Version.Major));1845 return Dscnt | Loadcnt;1846}1847 1848Waitcnt decodeLoadcntDscnt(const IsaVersion &Version, unsigned LoadcntDscnt) {1849 Waitcnt Decoded;1850 Decoded.LoadCnt =1851 unpackBits(LoadcntDscnt, getLoadcntStorecntBitShift(Version.Major),1852 getLoadcntBitWidth(Version.Major));1853 Decoded.DsCnt = unpackBits(LoadcntDscnt, getDscntBitShift(Version.Major),1854 getDscntBitWidth(Version.Major));1855 return Decoded;1856}1857 1858Waitcnt decodeStorecntDscnt(const IsaVersion &Version, unsigned StorecntDscnt) {1859 Waitcnt Decoded;1860 Decoded.StoreCnt =1861 unpackBits(StorecntDscnt, getLoadcntStorecntBitShift(Version.Major),1862 getStorecntBitWidth(Version.Major));1863 Decoded.DsCnt = unpackBits(StorecntDscnt, getDscntBitShift(Version.Major),1864 getDscntBitWidth(Version.Major));1865 return Decoded;1866}1867 1868static unsigned encodeLoadcnt(const IsaVersion &Version, unsigned Waitcnt,1869 unsigned Loadcnt) {1870 return packBits(Loadcnt, Waitcnt, getLoadcntStorecntBitShift(Version.Major),1871 getLoadcntBitWidth(Version.Major));1872}1873 1874static unsigned encodeStorecnt(const IsaVersion &Version, unsigned Waitcnt,1875 unsigned Storecnt) {1876 return packBits(Storecnt, Waitcnt, getLoadcntStorecntBitShift(Version.Major),1877 getStorecntBitWidth(Version.Major));1878}1879 1880static unsigned encodeDscnt(const IsaVersion &Version, unsigned Waitcnt,1881 unsigned Dscnt) {1882 return packBits(Dscnt, Waitcnt, getDscntBitShift(Version.Major),1883 getDscntBitWidth(Version.Major));1884}1885 1886static unsigned encodeLoadcntDscnt(const IsaVersion &Version, unsigned Loadcnt,1887 unsigned Dscnt) {1888 unsigned Waitcnt = getCombinedCountBitMask(Version, false);1889 Waitcnt = encodeLoadcnt(Version, Waitcnt, Loadcnt);1890 Waitcnt = encodeDscnt(Version, Waitcnt, Dscnt);1891 return Waitcnt;1892}1893 1894unsigned encodeLoadcntDscnt(const IsaVersion &Version, const Waitcnt &Decoded) {1895 return encodeLoadcntDscnt(Version, Decoded.LoadCnt, Decoded.DsCnt);1896}1897 1898static unsigned encodeStorecntDscnt(const IsaVersion &Version,1899 unsigned Storecnt, unsigned Dscnt) {1900 unsigned Waitcnt = getCombinedCountBitMask(Version, true);1901 Waitcnt = encodeStorecnt(Version, Waitcnt, Storecnt);1902 Waitcnt = encodeDscnt(Version, Waitcnt, Dscnt);1903 return Waitcnt;1904}1905 1906unsigned encodeStorecntDscnt(const IsaVersion &Version,1907 const Waitcnt &Decoded) {1908 return encodeStorecntDscnt(Version, Decoded.StoreCnt, Decoded.DsCnt);1909}1910 1911//===----------------------------------------------------------------------===//1912// Custom Operand Values1913//===----------------------------------------------------------------------===//1914 1915static unsigned getDefaultCustomOperandEncoding(const CustomOperandVal *Opr,1916 int Size,1917 const MCSubtargetInfo &STI) {1918 unsigned Enc = 0;1919 for (int Idx = 0; Idx < Size; ++Idx) {1920 const auto &Op = Opr[Idx];1921 if (Op.isSupported(STI))1922 Enc |= Op.encode(Op.Default);1923 }1924 return Enc;1925}1926 1927static bool isSymbolicCustomOperandEncoding(const CustomOperandVal *Opr,1928 int Size, unsigned Code,1929 bool &HasNonDefaultVal,1930 const MCSubtargetInfo &STI) {1931 unsigned UsedOprMask = 0;1932 HasNonDefaultVal = false;1933 for (int Idx = 0; Idx < Size; ++Idx) {1934 const auto &Op = Opr[Idx];1935 if (!Op.isSupported(STI))1936 continue;1937 UsedOprMask |= Op.getMask();1938 unsigned Val = Op.decode(Code);1939 if (!Op.isValid(Val))1940 return false;1941 HasNonDefaultVal |= (Val != Op.Default);1942 }1943 return (Code & ~UsedOprMask) == 0;1944}1945 1946static bool decodeCustomOperand(const CustomOperandVal *Opr, int Size,1947 unsigned Code, int &Idx, StringRef &Name,1948 unsigned &Val, bool &IsDefault,1949 const MCSubtargetInfo &STI) {1950 while (Idx < Size) {1951 const auto &Op = Opr[Idx++];1952 if (Op.isSupported(STI)) {1953 Name = Op.Name;1954 Val = Op.decode(Code);1955 IsDefault = (Val == Op.Default);1956 return true;1957 }1958 }1959 1960 return false;1961}1962 1963static int encodeCustomOperandVal(const CustomOperandVal &Op,1964 int64_t InputVal) {1965 if (InputVal < 0 || InputVal > Op.Max)1966 return OPR_VAL_INVALID;1967 return Op.encode(InputVal);1968}1969 1970static int encodeCustomOperand(const CustomOperandVal *Opr, int Size,1971 const StringRef Name, int64_t InputVal,1972 unsigned &UsedOprMask,1973 const MCSubtargetInfo &STI) {1974 int InvalidId = OPR_ID_UNKNOWN;1975 for (int Idx = 0; Idx < Size; ++Idx) {1976 const auto &Op = Opr[Idx];1977 if (Op.Name == Name) {1978 if (!Op.isSupported(STI)) {1979 InvalidId = OPR_ID_UNSUPPORTED;1980 continue;1981 }1982 auto OprMask = Op.getMask();1983 if (OprMask & UsedOprMask)1984 return OPR_ID_DUPLICATE;1985 UsedOprMask |= OprMask;1986 return encodeCustomOperandVal(Op, InputVal);1987 }1988 }1989 return InvalidId;1990}1991 1992//===----------------------------------------------------------------------===//1993// DepCtr1994//===----------------------------------------------------------------------===//1995 1996namespace DepCtr {1997 1998int getDefaultDepCtrEncoding(const MCSubtargetInfo &STI) {1999 static int Default = -1;2000 if (Default == -1)2001 Default = getDefaultCustomOperandEncoding(DepCtrInfo, DEP_CTR_SIZE, STI);2002 return Default;2003}2004 2005bool isSymbolicDepCtrEncoding(unsigned Code, bool &HasNonDefaultVal,2006 const MCSubtargetInfo &STI) {2007 return isSymbolicCustomOperandEncoding(DepCtrInfo, DEP_CTR_SIZE, Code,2008 HasNonDefaultVal, STI);2009}2010 2011bool decodeDepCtr(unsigned Code, int &Id, StringRef &Name, unsigned &Val,2012 bool &IsDefault, const MCSubtargetInfo &STI) {2013 return decodeCustomOperand(DepCtrInfo, DEP_CTR_SIZE, Code, Id, Name, Val,2014 IsDefault, STI);2015}2016 2017int encodeDepCtr(const StringRef Name, int64_t Val, unsigned &UsedOprMask,2018 const MCSubtargetInfo &STI) {2019 return encodeCustomOperand(DepCtrInfo, DEP_CTR_SIZE, Name, Val, UsedOprMask,2020 STI);2021}2022 2023unsigned decodeFieldVmVsrc(unsigned Encoded) {2024 return unpackBits(Encoded, getVmVsrcBitShift(), getVmVsrcBitWidth());2025}2026 2027unsigned decodeFieldVaVdst(unsigned Encoded) {2028 return unpackBits(Encoded, getVaVdstBitShift(), getVaVdstBitWidth());2029}2030 2031unsigned decodeFieldSaSdst(unsigned Encoded) {2032 return unpackBits(Encoded, getSaSdstBitShift(), getSaSdstBitWidth());2033}2034 2035unsigned decodeFieldVaSdst(unsigned Encoded) {2036 return unpackBits(Encoded, getVaSdstBitShift(), getVaSdstBitWidth());2037}2038 2039unsigned decodeFieldVaVcc(unsigned Encoded) {2040 return unpackBits(Encoded, getVaVccBitShift(), getVaVccBitWidth());2041}2042 2043unsigned decodeFieldVaSsrc(unsigned Encoded) {2044 return unpackBits(Encoded, getVaSsrcBitShift(), getVaSsrcBitWidth());2045}2046 2047unsigned decodeFieldHoldCnt(unsigned Encoded) {2048 return unpackBits(Encoded, getHoldCntBitShift(), getHoldCntWidth());2049}2050 2051unsigned encodeFieldVmVsrc(unsigned Encoded, unsigned VmVsrc) {2052 return packBits(VmVsrc, Encoded, getVmVsrcBitShift(), getVmVsrcBitWidth());2053}2054 2055unsigned encodeFieldVmVsrc(unsigned VmVsrc, const MCSubtargetInfo &STI) {2056 unsigned Encoded = getDefaultDepCtrEncoding(STI);2057 return encodeFieldVmVsrc(Encoded, VmVsrc);2058}2059 2060unsigned encodeFieldVaVdst(unsigned Encoded, unsigned VaVdst) {2061 return packBits(VaVdst, Encoded, getVaVdstBitShift(), getVaVdstBitWidth());2062}2063 2064unsigned encodeFieldVaVdst(unsigned VaVdst, const MCSubtargetInfo &STI) {2065 unsigned Encoded = getDefaultDepCtrEncoding(STI);2066 return encodeFieldVaVdst(Encoded, VaVdst);2067}2068 2069unsigned encodeFieldSaSdst(unsigned Encoded, unsigned SaSdst) {2070 return packBits(SaSdst, Encoded, getSaSdstBitShift(), getSaSdstBitWidth());2071}2072 2073unsigned encodeFieldSaSdst(unsigned SaSdst, const MCSubtargetInfo &STI) {2074 unsigned Encoded = getDefaultDepCtrEncoding(STI);2075 return encodeFieldSaSdst(Encoded, SaSdst);2076}2077 2078unsigned encodeFieldVaSdst(unsigned Encoded, unsigned VaSdst) {2079 return packBits(VaSdst, Encoded, getVaSdstBitShift(), getVaSdstBitWidth());2080}2081 2082unsigned encodeFieldVaSdst(unsigned VaSdst, const MCSubtargetInfo &STI) {2083 unsigned Encoded = getDefaultDepCtrEncoding(STI);2084 return encodeFieldVaSdst(Encoded, VaSdst);2085}2086 2087unsigned encodeFieldVaVcc(unsigned Encoded, unsigned VaVcc) {2088 return packBits(VaVcc, Encoded, getVaVccBitShift(), getVaVccBitWidth());2089}2090 2091unsigned encodeFieldVaVcc(unsigned VaVcc, const MCSubtargetInfo &STI) {2092 unsigned Encoded = getDefaultDepCtrEncoding(STI);2093 return encodeFieldVaVcc(Encoded, VaVcc);2094}2095 2096unsigned encodeFieldVaSsrc(unsigned Encoded, unsigned VaSsrc) {2097 return packBits(VaSsrc, Encoded, getVaSsrcBitShift(), getVaSsrcBitWidth());2098}2099 2100unsigned encodeFieldVaSsrc(unsigned VaSsrc, const MCSubtargetInfo &STI) {2101 unsigned Encoded = getDefaultDepCtrEncoding(STI);2102 return encodeFieldVaSsrc(Encoded, VaSsrc);2103}2104 2105unsigned encodeFieldHoldCnt(unsigned Encoded, unsigned HoldCnt) {2106 return packBits(HoldCnt, Encoded, getHoldCntBitShift(), getHoldCntWidth());2107}2108 2109unsigned encodeFieldHoldCnt(unsigned HoldCnt, const MCSubtargetInfo &STI) {2110 unsigned Encoded = getDefaultDepCtrEncoding(STI);2111 return encodeFieldHoldCnt(Encoded, HoldCnt);2112}2113 2114} // namespace DepCtr2115 2116//===----------------------------------------------------------------------===//2117// exp tgt2118//===----------------------------------------------------------------------===//2119 2120namespace Exp {2121 2122struct ExpTgt {2123 StringLiteral Name;2124 unsigned Tgt;2125 unsigned MaxIndex;2126};2127 2128// clang-format off2129static constexpr ExpTgt ExpTgtInfo[] = {2130 {{"null"}, ET_NULL, ET_NULL_MAX_IDX},2131 {{"mrtz"}, ET_MRTZ, ET_MRTZ_MAX_IDX},2132 {{"prim"}, ET_PRIM, ET_PRIM_MAX_IDX},2133 {{"mrt"}, ET_MRT0, ET_MRT_MAX_IDX},2134 {{"pos"}, ET_POS0, ET_POS_MAX_IDX},2135 {{"dual_src_blend"},ET_DUAL_SRC_BLEND0, ET_DUAL_SRC_BLEND_MAX_IDX},2136 {{"param"}, ET_PARAM0, ET_PARAM_MAX_IDX},2137};2138// clang-format on2139 2140bool getTgtName(unsigned Id, StringRef &Name, int &Index) {2141 for (const ExpTgt &Val : ExpTgtInfo) {2142 if (Val.Tgt <= Id && Id <= Val.Tgt + Val.MaxIndex) {2143 Index = (Val.MaxIndex == 0) ? -1 : (Id - Val.Tgt);2144 Name = Val.Name;2145 return true;2146 }2147 }2148 return false;2149}2150 2151unsigned getTgtId(const StringRef Name) {2152 2153 for (const ExpTgt &Val : ExpTgtInfo) {2154 if (Val.MaxIndex == 0 && Name == Val.Name)2155 return Val.Tgt;2156 2157 if (Val.MaxIndex > 0 && Name.starts_with(Val.Name)) {2158 StringRef Suffix = Name.drop_front(Val.Name.size());2159 2160 unsigned Id;2161 if (Suffix.getAsInteger(10, Id) || Id > Val.MaxIndex)2162 return ET_INVALID;2163 2164 // Disable leading zeroes2165 if (Suffix.size() > 1 && Suffix[0] == '0')2166 return ET_INVALID;2167 2168 return Val.Tgt + Id;2169 }2170 }2171 return ET_INVALID;2172}2173 2174bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI) {2175 switch (Id) {2176 case ET_NULL:2177 return !isGFX11Plus(STI);2178 case ET_POS4:2179 case ET_PRIM:2180 return isGFX10Plus(STI);2181 case ET_DUAL_SRC_BLEND0:2182 case ET_DUAL_SRC_BLEND1:2183 return isGFX11Plus(STI);2184 default:2185 if (Id >= ET_PARAM0 && Id <= ET_PARAM31)2186 return !isGFX11Plus(STI);2187 return true;2188 }2189}2190 2191} // namespace Exp2192 2193//===----------------------------------------------------------------------===//2194// MTBUF Format2195//===----------------------------------------------------------------------===//2196 2197namespace MTBUFFormat {2198 2199int64_t getDfmt(const StringRef Name) {2200 for (int Id = DFMT_MIN; Id <= DFMT_MAX; ++Id) {2201 if (Name == DfmtSymbolic[Id])2202 return Id;2203 }2204 return DFMT_UNDEF;2205}2206 2207StringRef getDfmtName(unsigned Id) {2208 assert(Id <= DFMT_MAX);2209 return DfmtSymbolic[Id];2210}2211 2212static StringLiteral const *getNfmtLookupTable(const MCSubtargetInfo &STI) {2213 if (isSI(STI) || isCI(STI))2214 return NfmtSymbolicSICI;2215 if (isVI(STI) || isGFX9(STI))2216 return NfmtSymbolicVI;2217 return NfmtSymbolicGFX10;2218}2219 2220int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI) {2221 const auto *lookupTable = getNfmtLookupTable(STI);2222 for (int Id = NFMT_MIN; Id <= NFMT_MAX; ++Id) {2223 if (Name == lookupTable[Id])2224 return Id;2225 }2226 return NFMT_UNDEF;2227}2228 2229StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI) {2230 assert(Id <= NFMT_MAX);2231 return getNfmtLookupTable(STI)[Id];2232}2233 2234bool isValidDfmtNfmt(unsigned Id, const MCSubtargetInfo &STI) {2235 unsigned Dfmt;2236 unsigned Nfmt;2237 decodeDfmtNfmt(Id, Dfmt, Nfmt);2238 return isValidNfmt(Nfmt, STI);2239}2240 2241bool isValidNfmt(unsigned Id, const MCSubtargetInfo &STI) {2242 return !getNfmtName(Id, STI).empty();2243}2244 2245int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt) {2246 return (Dfmt << DFMT_SHIFT) | (Nfmt << NFMT_SHIFT);2247}2248 2249void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt) {2250 Dfmt = (Format >> DFMT_SHIFT) & DFMT_MASK;2251 Nfmt = (Format >> NFMT_SHIFT) & NFMT_MASK;2252}2253 2254int64_t getUnifiedFormat(const StringRef Name, const MCSubtargetInfo &STI) {2255 if (isGFX11Plus(STI)) {2256 for (int Id = UfmtGFX11::UFMT_FIRST; Id <= UfmtGFX11::UFMT_LAST; ++Id) {2257 if (Name == UfmtSymbolicGFX11[Id])2258 return Id;2259 }2260 } else {2261 for (int Id = UfmtGFX10::UFMT_FIRST; Id <= UfmtGFX10::UFMT_LAST; ++Id) {2262 if (Name == UfmtSymbolicGFX10[Id])2263 return Id;2264 }2265 }2266 return UFMT_UNDEF;2267}2268 2269StringRef getUnifiedFormatName(unsigned Id, const MCSubtargetInfo &STI) {2270 if (isValidUnifiedFormat(Id, STI))2271 return isGFX10(STI) ? UfmtSymbolicGFX10[Id] : UfmtSymbolicGFX11[Id];2272 return "";2273}2274 2275bool isValidUnifiedFormat(unsigned Id, const MCSubtargetInfo &STI) {2276 return isGFX10(STI) ? Id <= UfmtGFX10::UFMT_LAST : Id <= UfmtGFX11::UFMT_LAST;2277}2278 2279int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt,2280 const MCSubtargetInfo &STI) {2281 int64_t Fmt = encodeDfmtNfmt(Dfmt, Nfmt);2282 if (isGFX11Plus(STI)) {2283 for (int Id = UfmtGFX11::UFMT_FIRST; Id <= UfmtGFX11::UFMT_LAST; ++Id) {2284 if (Fmt == DfmtNfmt2UFmtGFX11[Id])2285 return Id;2286 }2287 } else {2288 for (int Id = UfmtGFX10::UFMT_FIRST; Id <= UfmtGFX10::UFMT_LAST; ++Id) {2289 if (Fmt == DfmtNfmt2UFmtGFX10[Id])2290 return Id;2291 }2292 }2293 return UFMT_UNDEF;2294}2295 2296bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI) {2297 return isGFX10Plus(STI) ? (Val <= UFMT_MAX) : (Val <= DFMT_NFMT_MAX);2298}2299 2300unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI) {2301 if (isGFX10Plus(STI))2302 return UFMT_DEFAULT;2303 return DFMT_NFMT_DEFAULT;2304}2305 2306} // namespace MTBUFFormat2307 2308//===----------------------------------------------------------------------===//2309// SendMsg2310//===----------------------------------------------------------------------===//2311 2312namespace SendMsg {2313 2314static uint64_t getMsgIdMask(const MCSubtargetInfo &STI) {2315 return isGFX11Plus(STI) ? ID_MASK_GFX11Plus_ : ID_MASK_PreGFX11_;2316}2317 2318bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI) {2319 return (MsgId & ~(getMsgIdMask(STI))) == 0;2320}2321 2322bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI,2323 bool Strict) {2324 assert(isValidMsgId(MsgId, STI));2325 2326 if (!Strict)2327 return 0 <= OpId && isUInt<OP_WIDTH_>(OpId);2328 2329 if (msgRequiresOp(MsgId, STI)) {2330 if (MsgId == ID_GS_PreGFX11 && OpId == OP_GS_NOP)2331 return false;2332 2333 return !getMsgOpName(MsgId, OpId, STI).empty();2334 }2335 2336 return OpId == OP_NONE_;2337}2338 2339bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId,2340 const MCSubtargetInfo &STI, bool Strict) {2341 assert(isValidMsgOp(MsgId, OpId, STI, Strict));2342 2343 if (!Strict)2344 return 0 <= StreamId && isUInt<STREAM_ID_WIDTH_>(StreamId);2345 2346 if (!isGFX11Plus(STI)) {2347 switch (MsgId) {2348 case ID_GS_PreGFX11:2349 return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_;2350 case ID_GS_DONE_PreGFX11:2351 return (OpId == OP_GS_NOP)2352 ? (StreamId == STREAM_ID_NONE_)2353 : (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_);2354 }2355 }2356 return StreamId == STREAM_ID_NONE_;2357}2358 2359bool msgRequiresOp(int64_t MsgId, const MCSubtargetInfo &STI) {2360 return MsgId == ID_SYSMSG ||2361 (!isGFX11Plus(STI) &&2362 (MsgId == ID_GS_PreGFX11 || MsgId == ID_GS_DONE_PreGFX11));2363}2364 2365bool msgSupportsStream(int64_t MsgId, int64_t OpId,2366 const MCSubtargetInfo &STI) {2367 return !isGFX11Plus(STI) &&2368 (MsgId == ID_GS_PreGFX11 || MsgId == ID_GS_DONE_PreGFX11) &&2369 OpId != OP_GS_NOP;2370}2371 2372void decodeMsg(unsigned Val, uint16_t &MsgId, uint16_t &OpId,2373 uint16_t &StreamId, const MCSubtargetInfo &STI) {2374 MsgId = Val & getMsgIdMask(STI);2375 if (isGFX11Plus(STI)) {2376 OpId = 0;2377 StreamId = 0;2378 } else {2379 OpId = (Val & OP_MASK_) >> OP_SHIFT_;2380 StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_;2381 }2382}2383 2384uint64_t encodeMsg(uint64_t MsgId, uint64_t OpId, uint64_t StreamId) {2385 return MsgId | (OpId << OP_SHIFT_) | (StreamId << STREAM_ID_SHIFT_);2386}2387 2388} // namespace SendMsg2389 2390//===----------------------------------------------------------------------===//2391//2392//===----------------------------------------------------------------------===//2393 2394unsigned getInitialPSInputAddr(const Function &F) {2395 return F.getFnAttributeAsParsedInteger("InitialPSInputAddr", 0);2396}2397 2398bool getHasColorExport(const Function &F) {2399 // As a safe default always respond as if PS has color exports.2400 return F.getFnAttributeAsParsedInteger(2401 "amdgpu-color-export",2402 F.getCallingConv() == CallingConv::AMDGPU_PS ? 1 : 0) != 0;2403}2404 2405bool getHasDepthExport(const Function &F) {2406 return F.getFnAttributeAsParsedInteger("amdgpu-depth-export", 0) != 0;2407}2408 2409unsigned getDynamicVGPRBlockSize(const Function &F) {2410 unsigned BlockSize =2411 F.getFnAttributeAsParsedInteger("amdgpu-dynamic-vgpr-block-size", 0);2412 2413 if (BlockSize == 16 || BlockSize == 32)2414 return BlockSize;2415 2416 return 0;2417}2418 2419bool hasXNACK(const MCSubtargetInfo &STI) {2420 return STI.hasFeature(AMDGPU::FeatureXNACK);2421}2422 2423bool hasSRAMECC(const MCSubtargetInfo &STI) {2424 return STI.hasFeature(AMDGPU::FeatureSRAMECC);2425}2426 2427bool hasMIMG_R128(const MCSubtargetInfo &STI) {2428 return STI.hasFeature(AMDGPU::FeatureMIMG_R128) &&2429 !STI.hasFeature(AMDGPU::FeatureR128A16);2430}2431 2432bool hasA16(const MCSubtargetInfo &STI) {2433 return STI.hasFeature(AMDGPU::FeatureA16);2434}2435 2436bool hasG16(const MCSubtargetInfo &STI) {2437 return STI.hasFeature(AMDGPU::FeatureG16);2438}2439 2440bool hasPackedD16(const MCSubtargetInfo &STI) {2441 return !STI.hasFeature(AMDGPU::FeatureUnpackedD16VMem) && !isCI(STI) &&2442 !isSI(STI);2443}2444 2445bool hasGDS(const MCSubtargetInfo &STI) {2446 return STI.hasFeature(AMDGPU::FeatureGDS);2447}2448 2449unsigned getNSAMaxSize(const MCSubtargetInfo &STI, bool HasSampler) {2450 auto Version = getIsaVersion(STI.getCPU());2451 if (Version.Major == 10)2452 return Version.Minor >= 3 ? 13 : 5;2453 if (Version.Major == 11)2454 return 5;2455 if (Version.Major >= 12)2456 return HasSampler ? 4 : 5;2457 return 0;2458}2459 2460unsigned getMaxNumUserSGPRs(const MCSubtargetInfo &STI) {2461 if (isGFX1250(STI))2462 return 32;2463 return 16;2464}2465 2466bool isSI(const MCSubtargetInfo &STI) {2467 return STI.hasFeature(AMDGPU::FeatureSouthernIslands);2468}2469 2470bool isCI(const MCSubtargetInfo &STI) {2471 return STI.hasFeature(AMDGPU::FeatureSeaIslands);2472}2473 2474bool isVI(const MCSubtargetInfo &STI) {2475 return STI.hasFeature(AMDGPU::FeatureVolcanicIslands);2476}2477 2478bool isGFX9(const MCSubtargetInfo &STI) {2479 return STI.hasFeature(AMDGPU::FeatureGFX9);2480}2481 2482bool isGFX9_GFX10(const MCSubtargetInfo &STI) {2483 return isGFX9(STI) || isGFX10(STI);2484}2485 2486bool isGFX9_GFX10_GFX11(const MCSubtargetInfo &STI) {2487 return isGFX9(STI) || isGFX10(STI) || isGFX11(STI);2488}2489 2490bool isGFX8_GFX9_GFX10(const MCSubtargetInfo &STI) {2491 return isVI(STI) || isGFX9(STI) || isGFX10(STI);2492}2493 2494bool isGFX8Plus(const MCSubtargetInfo &STI) {2495 return isVI(STI) || isGFX9Plus(STI);2496}2497 2498bool isGFX9Plus(const MCSubtargetInfo &STI) {2499 return isGFX9(STI) || isGFX10Plus(STI);2500}2501 2502bool isNotGFX9Plus(const MCSubtargetInfo &STI) { return !isGFX9Plus(STI); }2503 2504bool isGFX10(const MCSubtargetInfo &STI) {2505 return STI.hasFeature(AMDGPU::FeatureGFX10);2506}2507 2508bool isGFX10_GFX11(const MCSubtargetInfo &STI) {2509 return isGFX10(STI) || isGFX11(STI);2510}2511 2512bool isGFX10Plus(const MCSubtargetInfo &STI) {2513 return isGFX10(STI) || isGFX11Plus(STI);2514}2515 2516bool isGFX11(const MCSubtargetInfo &STI) {2517 return STI.hasFeature(AMDGPU::FeatureGFX11);2518}2519 2520bool isGFX11Plus(const MCSubtargetInfo &STI) {2521 return isGFX11(STI) || isGFX12Plus(STI);2522}2523 2524bool isGFX12(const MCSubtargetInfo &STI) {2525 return STI.getFeatureBits()[AMDGPU::FeatureGFX12];2526}2527 2528bool isGFX12Plus(const MCSubtargetInfo &STI) { return isGFX12(STI); }2529 2530bool isNotGFX12Plus(const MCSubtargetInfo &STI) { return !isGFX12Plus(STI); }2531 2532bool isGFX1250(const MCSubtargetInfo &STI) {2533 return STI.getFeatureBits()[AMDGPU::FeatureGFX1250Insts];2534}2535 2536bool supportsWGP(const MCSubtargetInfo &STI) {2537 if (isGFX1250(STI))2538 return false;2539 return isGFX10Plus(STI);2540}2541 2542bool isNotGFX11Plus(const MCSubtargetInfo &STI) { return !isGFX11Plus(STI); }2543 2544bool isNotGFX10Plus(const MCSubtargetInfo &STI) {2545 return isSI(STI) || isCI(STI) || isVI(STI) || isGFX9(STI);2546}2547 2548bool isGFX10Before1030(const MCSubtargetInfo &STI) {2549 return isGFX10(STI) && !AMDGPU::isGFX10_BEncoding(STI);2550}2551 2552bool isGCN3Encoding(const MCSubtargetInfo &STI) {2553 return STI.hasFeature(AMDGPU::FeatureGCN3Encoding);2554}2555 2556bool isGFX10_AEncoding(const MCSubtargetInfo &STI) {2557 return STI.hasFeature(AMDGPU::FeatureGFX10_AEncoding);2558}2559 2560bool isGFX10_BEncoding(const MCSubtargetInfo &STI) {2561 return STI.hasFeature(AMDGPU::FeatureGFX10_BEncoding);2562}2563 2564bool hasGFX10_3Insts(const MCSubtargetInfo &STI) {2565 return STI.hasFeature(AMDGPU::FeatureGFX10_3Insts);2566}2567 2568bool isGFX10_3_GFX11(const MCSubtargetInfo &STI) {2569 return isGFX10_BEncoding(STI) && !isGFX12Plus(STI);2570}2571 2572bool isGFX90A(const MCSubtargetInfo &STI) {2573 return STI.hasFeature(AMDGPU::FeatureGFX90AInsts);2574}2575 2576bool isGFX940(const MCSubtargetInfo &STI) {2577 return STI.hasFeature(AMDGPU::FeatureGFX940Insts);2578}2579 2580bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI) {2581 return STI.hasFeature(AMDGPU::FeatureArchitectedFlatScratch);2582}2583 2584bool hasMAIInsts(const MCSubtargetInfo &STI) {2585 return STI.hasFeature(AMDGPU::FeatureMAIInsts);2586}2587 2588bool hasVOPD(const MCSubtargetInfo &STI) {2589 return STI.hasFeature(AMDGPU::FeatureVOPD);2590}2591 2592bool hasDPPSrc1SGPR(const MCSubtargetInfo &STI) {2593 return STI.hasFeature(AMDGPU::FeatureDPPSrc1SGPR);2594}2595 2596unsigned hasKernargPreload(const MCSubtargetInfo &STI) {2597 return STI.hasFeature(AMDGPU::FeatureKernargPreload);2598}2599 2600int32_t getTotalNumVGPRs(bool has90AInsts, int32_t ArgNumAGPR,2601 int32_t ArgNumVGPR) {2602 if (has90AInsts && ArgNumAGPR)2603 return alignTo(ArgNumVGPR, 4) + ArgNumAGPR;2604 return std::max(ArgNumVGPR, ArgNumAGPR);2605}2606 2607bool isSGPR(MCRegister Reg, const MCRegisterInfo *TRI) {2608 const MCRegisterClass SGPRClass = TRI->getRegClass(AMDGPU::SReg_32RegClassID);2609 const MCRegister FirstSubReg = TRI->getSubReg(Reg, AMDGPU::sub0);2610 return SGPRClass.contains(FirstSubReg != 0 ? FirstSubReg : Reg) ||2611 Reg == AMDGPU::SCC;2612}2613 2614bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI) {2615 return MRI.getEncodingValue(Reg) & AMDGPU::HWEncoding::IS_HI16;2616}2617 2618#define MAP_REG2REG \2619 using namespace AMDGPU; \2620 switch (Reg.id()) { \2621 default: \2622 return Reg; \2623 CASE_CI_VI(FLAT_SCR) \2624 CASE_CI_VI(FLAT_SCR_LO) \2625 CASE_CI_VI(FLAT_SCR_HI) \2626 CASE_VI_GFX9PLUS(TTMP0) \2627 CASE_VI_GFX9PLUS(TTMP1) \2628 CASE_VI_GFX9PLUS(TTMP2) \2629 CASE_VI_GFX9PLUS(TTMP3) \2630 CASE_VI_GFX9PLUS(TTMP4) \2631 CASE_VI_GFX9PLUS(TTMP5) \2632 CASE_VI_GFX9PLUS(TTMP6) \2633 CASE_VI_GFX9PLUS(TTMP7) \2634 CASE_VI_GFX9PLUS(TTMP8) \2635 CASE_VI_GFX9PLUS(TTMP9) \2636 CASE_VI_GFX9PLUS(TTMP10) \2637 CASE_VI_GFX9PLUS(TTMP11) \2638 CASE_VI_GFX9PLUS(TTMP12) \2639 CASE_VI_GFX9PLUS(TTMP13) \2640 CASE_VI_GFX9PLUS(TTMP14) \2641 CASE_VI_GFX9PLUS(TTMP15) \2642 CASE_VI_GFX9PLUS(TTMP0_TTMP1) \2643 CASE_VI_GFX9PLUS(TTMP2_TTMP3) \2644 CASE_VI_GFX9PLUS(TTMP4_TTMP5) \2645 CASE_VI_GFX9PLUS(TTMP6_TTMP7) \2646 CASE_VI_GFX9PLUS(TTMP8_TTMP9) \2647 CASE_VI_GFX9PLUS(TTMP10_TTMP11) \2648 CASE_VI_GFX9PLUS(TTMP12_TTMP13) \2649 CASE_VI_GFX9PLUS(TTMP14_TTMP15) \2650 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3) \2651 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7) \2652 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11) \2653 CASE_VI_GFX9PLUS(TTMP12_TTMP13_TTMP14_TTMP15) \2654 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \2655 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \2656 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \2657 CASE_VI_GFX9PLUS( \2658 TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \2659 CASE_GFXPRE11_GFX11PLUS(M0) \2660 CASE_GFXPRE11_GFX11PLUS(SGPR_NULL) \2661 CASE_GFXPRE11_GFX11PLUS_TO(SGPR_NULL64, SGPR_NULL) \2662 }2663 2664#define CASE_CI_VI(node) \2665 assert(!isSI(STI)); \2666 case node: \2667 return isCI(STI) ? node##_ci : node##_vi;2668 2669#define CASE_VI_GFX9PLUS(node) \2670 case node: \2671 return isGFX9Plus(STI) ? node##_gfx9plus : node##_vi;2672 2673#define CASE_GFXPRE11_GFX11PLUS(node) \2674 case node: \2675 return isGFX11Plus(STI) ? node##_gfx11plus : node##_gfxpre11;2676 2677#define CASE_GFXPRE11_GFX11PLUS_TO(node, result) \2678 case node: \2679 return isGFX11Plus(STI) ? result##_gfx11plus : result##_gfxpre11;2680 2681MCRegister getMCReg(MCRegister Reg, const MCSubtargetInfo &STI) {2682 if (STI.getTargetTriple().getArch() == Triple::r600)2683 return Reg;2684 MAP_REG2REG2685}2686 2687#undef CASE_CI_VI2688#undef CASE_VI_GFX9PLUS2689#undef CASE_GFXPRE11_GFX11PLUS2690#undef CASE_GFXPRE11_GFX11PLUS_TO2691 2692#define CASE_CI_VI(node) \2693 case node##_ci: \2694 case node##_vi: \2695 return node;2696#define CASE_VI_GFX9PLUS(node) \2697 case node##_vi: \2698 case node##_gfx9plus: \2699 return node;2700#define CASE_GFXPRE11_GFX11PLUS(node) \2701 case node##_gfx11plus: \2702 case node##_gfxpre11: \2703 return node;2704#define CASE_GFXPRE11_GFX11PLUS_TO(node, result)2705 2706MCRegister mc2PseudoReg(MCRegister Reg) { MAP_REG2REG }2707 2708bool isInlineValue(MCRegister Reg) {2709 switch (Reg.id()) {2710 case AMDGPU::SRC_SHARED_BASE_LO:2711 case AMDGPU::SRC_SHARED_BASE:2712 case AMDGPU::SRC_SHARED_LIMIT_LO:2713 case AMDGPU::SRC_SHARED_LIMIT:2714 case AMDGPU::SRC_PRIVATE_BASE_LO:2715 case AMDGPU::SRC_PRIVATE_BASE:2716 case AMDGPU::SRC_PRIVATE_LIMIT_LO:2717 case AMDGPU::SRC_PRIVATE_LIMIT:2718 case AMDGPU::SRC_FLAT_SCRATCH_BASE_LO:2719 case AMDGPU::SRC_FLAT_SCRATCH_BASE_HI:2720 case AMDGPU::SRC_POPS_EXITING_WAVE_ID:2721 return true;2722 case AMDGPU::SRC_VCCZ:2723 case AMDGPU::SRC_EXECZ:2724 case AMDGPU::SRC_SCC:2725 return true;2726 case AMDGPU::SGPR_NULL:2727 return true;2728 default:2729 return false;2730 }2731}2732 2733#undef CASE_CI_VI2734#undef CASE_VI_GFX9PLUS2735#undef CASE_GFXPRE11_GFX11PLUS2736#undef CASE_GFXPRE11_GFX11PLUS_TO2737#undef MAP_REG2REG2738 2739bool isKImmOperand(const MCInstrDesc &Desc, unsigned OpNo) {2740 assert(OpNo < Desc.NumOperands);2741 unsigned OpType = Desc.operands()[OpNo].OperandType;2742 return OpType >= AMDGPU::OPERAND_KIMM_FIRST &&2743 OpType <= AMDGPU::OPERAND_KIMM_LAST;2744}2745 2746bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) {2747 assert(OpNo < Desc.NumOperands);2748 unsigned OpType = Desc.operands()[OpNo].OperandType;2749 switch (OpType) {2750 case AMDGPU::OPERAND_REG_IMM_FP32:2751 case AMDGPU::OPERAND_REG_IMM_FP64:2752 case AMDGPU::OPERAND_REG_IMM_FP16:2753 case AMDGPU::OPERAND_REG_IMM_V2FP16:2754 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:2755 case AMDGPU::OPERAND_REG_INLINE_C_FP32:2756 case AMDGPU::OPERAND_REG_INLINE_C_FP64:2757 case AMDGPU::OPERAND_REG_INLINE_C_FP16:2758 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:2759 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:2760 case AMDGPU::OPERAND_REG_IMM_V2FP32:2761 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:2762 return true;2763 default:2764 return false;2765 }2766}2767 2768bool isSISrcInlinableOperand(const MCInstrDesc &Desc, unsigned OpNo) {2769 assert(OpNo < Desc.NumOperands);2770 unsigned OpType = Desc.operands()[OpNo].OperandType;2771 return (OpType >= AMDGPU::OPERAND_REG_INLINE_C_FIRST &&2772 OpType <= AMDGPU::OPERAND_REG_INLINE_C_LAST) ||2773 (OpType >= AMDGPU::OPERAND_REG_INLINE_AC_FIRST &&2774 OpType <= AMDGPU::OPERAND_REG_INLINE_AC_LAST);2775}2776 2777// Avoid using MCRegisterClass::getSize, since that function will go away2778// (move from MC* level to Target* level). Return size in bits.2779unsigned getRegBitWidth(unsigned RCID) {2780 switch (RCID) {2781 case AMDGPU::VGPR_16RegClassID:2782 case AMDGPU::VGPR_16_Lo128RegClassID:2783 case AMDGPU::SGPR_LO16RegClassID:2784 case AMDGPU::AGPR_LO16RegClassID:2785 return 16;2786 case AMDGPU::SGPR_32RegClassID:2787 case AMDGPU::VGPR_32RegClassID:2788 case AMDGPU::VGPR_32_Lo256RegClassID:2789 case AMDGPU::VRegOrLds_32RegClassID:2790 case AMDGPU::AGPR_32RegClassID:2791 case AMDGPU::VS_32RegClassID:2792 case AMDGPU::AV_32RegClassID:2793 case AMDGPU::SReg_32RegClassID:2794 case AMDGPU::SReg_32_XM0RegClassID:2795 case AMDGPU::SRegOrLds_32RegClassID:2796 return 32;2797 case AMDGPU::SGPR_64RegClassID:2798 case AMDGPU::VS_64RegClassID:2799 case AMDGPU::SReg_64RegClassID:2800 case AMDGPU::VReg_64RegClassID:2801 case AMDGPU::AReg_64RegClassID:2802 case AMDGPU::SReg_64_XEXECRegClassID:2803 case AMDGPU::VReg_64_Align2RegClassID:2804 case AMDGPU::AReg_64_Align2RegClassID:2805 case AMDGPU::AV_64RegClassID:2806 case AMDGPU::AV_64_Align2RegClassID:2807 case AMDGPU::VReg_64_Lo256_Align2RegClassID:2808 case AMDGPU::VS_64_Lo256RegClassID:2809 return 64;2810 case AMDGPU::SGPR_96RegClassID:2811 case AMDGPU::SReg_96RegClassID:2812 case AMDGPU::VReg_96RegClassID:2813 case AMDGPU::AReg_96RegClassID:2814 case AMDGPU::VReg_96_Align2RegClassID:2815 case AMDGPU::AReg_96_Align2RegClassID:2816 case AMDGPU::AV_96RegClassID:2817 case AMDGPU::AV_96_Align2RegClassID:2818 case AMDGPU::VReg_96_Lo256_Align2RegClassID:2819 return 96;2820 case AMDGPU::SGPR_128RegClassID:2821 case AMDGPU::SReg_128RegClassID:2822 case AMDGPU::VReg_128RegClassID:2823 case AMDGPU::AReg_128RegClassID:2824 case AMDGPU::VReg_128_Align2RegClassID:2825 case AMDGPU::AReg_128_Align2RegClassID:2826 case AMDGPU::AV_128RegClassID:2827 case AMDGPU::AV_128_Align2RegClassID:2828 case AMDGPU::SReg_128_XNULLRegClassID:2829 case AMDGPU::VReg_128_Lo256_Align2RegClassID:2830 return 128;2831 case AMDGPU::SGPR_160RegClassID:2832 case AMDGPU::SReg_160RegClassID:2833 case AMDGPU::VReg_160RegClassID:2834 case AMDGPU::AReg_160RegClassID:2835 case AMDGPU::VReg_160_Align2RegClassID:2836 case AMDGPU::AReg_160_Align2RegClassID:2837 case AMDGPU::AV_160RegClassID:2838 case AMDGPU::AV_160_Align2RegClassID:2839 case AMDGPU::VReg_160_Lo256_Align2RegClassID:2840 return 160;2841 case AMDGPU::SGPR_192RegClassID:2842 case AMDGPU::SReg_192RegClassID:2843 case AMDGPU::VReg_192RegClassID:2844 case AMDGPU::AReg_192RegClassID:2845 case AMDGPU::VReg_192_Align2RegClassID:2846 case AMDGPU::AReg_192_Align2RegClassID:2847 case AMDGPU::AV_192RegClassID:2848 case AMDGPU::AV_192_Align2RegClassID:2849 case AMDGPU::VReg_192_Lo256_Align2RegClassID:2850 return 192;2851 case AMDGPU::SGPR_224RegClassID:2852 case AMDGPU::SReg_224RegClassID:2853 case AMDGPU::VReg_224RegClassID:2854 case AMDGPU::AReg_224RegClassID:2855 case AMDGPU::VReg_224_Align2RegClassID:2856 case AMDGPU::AReg_224_Align2RegClassID:2857 case AMDGPU::AV_224RegClassID:2858 case AMDGPU::AV_224_Align2RegClassID:2859 case AMDGPU::VReg_224_Lo256_Align2RegClassID:2860 return 224;2861 case AMDGPU::SGPR_256RegClassID:2862 case AMDGPU::SReg_256RegClassID:2863 case AMDGPU::VReg_256RegClassID:2864 case AMDGPU::AReg_256RegClassID:2865 case AMDGPU::VReg_256_Align2RegClassID:2866 case AMDGPU::AReg_256_Align2RegClassID:2867 case AMDGPU::AV_256RegClassID:2868 case AMDGPU::AV_256_Align2RegClassID:2869 case AMDGPU::SReg_256_XNULLRegClassID:2870 case AMDGPU::VReg_256_Lo256_Align2RegClassID:2871 return 256;2872 case AMDGPU::SGPR_288RegClassID:2873 case AMDGPU::SReg_288RegClassID:2874 case AMDGPU::VReg_288RegClassID:2875 case AMDGPU::AReg_288RegClassID:2876 case AMDGPU::VReg_288_Align2RegClassID:2877 case AMDGPU::AReg_288_Align2RegClassID:2878 case AMDGPU::AV_288RegClassID:2879 case AMDGPU::AV_288_Align2RegClassID:2880 case AMDGPU::VReg_288_Lo256_Align2RegClassID:2881 return 288;2882 case AMDGPU::SGPR_320RegClassID:2883 case AMDGPU::SReg_320RegClassID:2884 case AMDGPU::VReg_320RegClassID:2885 case AMDGPU::AReg_320RegClassID:2886 case AMDGPU::VReg_320_Align2RegClassID:2887 case AMDGPU::AReg_320_Align2RegClassID:2888 case AMDGPU::AV_320RegClassID:2889 case AMDGPU::AV_320_Align2RegClassID:2890 case AMDGPU::VReg_320_Lo256_Align2RegClassID:2891 return 320;2892 case AMDGPU::SGPR_352RegClassID:2893 case AMDGPU::SReg_352RegClassID:2894 case AMDGPU::VReg_352RegClassID:2895 case AMDGPU::AReg_352RegClassID:2896 case AMDGPU::VReg_352_Align2RegClassID:2897 case AMDGPU::AReg_352_Align2RegClassID:2898 case AMDGPU::AV_352RegClassID:2899 case AMDGPU::AV_352_Align2RegClassID:2900 case AMDGPU::VReg_352_Lo256_Align2RegClassID:2901 return 352;2902 case AMDGPU::SGPR_384RegClassID:2903 case AMDGPU::SReg_384RegClassID:2904 case AMDGPU::VReg_384RegClassID:2905 case AMDGPU::AReg_384RegClassID:2906 case AMDGPU::VReg_384_Align2RegClassID:2907 case AMDGPU::AReg_384_Align2RegClassID:2908 case AMDGPU::AV_384RegClassID:2909 case AMDGPU::AV_384_Align2RegClassID:2910 case AMDGPU::VReg_384_Lo256_Align2RegClassID:2911 return 384;2912 case AMDGPU::SGPR_512RegClassID:2913 case AMDGPU::SReg_512RegClassID:2914 case AMDGPU::VReg_512RegClassID:2915 case AMDGPU::AReg_512RegClassID:2916 case AMDGPU::VReg_512_Align2RegClassID:2917 case AMDGPU::AReg_512_Align2RegClassID:2918 case AMDGPU::AV_512RegClassID:2919 case AMDGPU::AV_512_Align2RegClassID:2920 case AMDGPU::VReg_512_Lo256_Align2RegClassID:2921 return 512;2922 case AMDGPU::SGPR_1024RegClassID:2923 case AMDGPU::SReg_1024RegClassID:2924 case AMDGPU::VReg_1024RegClassID:2925 case AMDGPU::AReg_1024RegClassID:2926 case AMDGPU::VReg_1024_Align2RegClassID:2927 case AMDGPU::AReg_1024_Align2RegClassID:2928 case AMDGPU::AV_1024RegClassID:2929 case AMDGPU::AV_1024_Align2RegClassID:2930 case AMDGPU::VReg_1024_Lo256_Align2RegClassID:2931 return 1024;2932 default:2933 llvm_unreachable("Unexpected register class");2934 }2935}2936 2937unsigned getRegBitWidth(const MCRegisterClass &RC) {2938 return getRegBitWidth(RC.getID());2939}2940 2941bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) {2942 if (isInlinableIntLiteral(Literal))2943 return true;2944 2945 uint64_t Val = static_cast<uint64_t>(Literal);2946 return (Val == llvm::bit_cast<uint64_t>(0.0)) ||2947 (Val == llvm::bit_cast<uint64_t>(1.0)) ||2948 (Val == llvm::bit_cast<uint64_t>(-1.0)) ||2949 (Val == llvm::bit_cast<uint64_t>(0.5)) ||2950 (Val == llvm::bit_cast<uint64_t>(-0.5)) ||2951 (Val == llvm::bit_cast<uint64_t>(2.0)) ||2952 (Val == llvm::bit_cast<uint64_t>(-2.0)) ||2953 (Val == llvm::bit_cast<uint64_t>(4.0)) ||2954 (Val == llvm::bit_cast<uint64_t>(-4.0)) ||2955 (Val == 0x3fc45f306dc9c882 && HasInv2Pi);2956}2957 2958bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) {2959 if (isInlinableIntLiteral(Literal))2960 return true;2961 2962 // The actual type of the operand does not seem to matter as long2963 // as the bits match one of the inline immediate values. For example:2964 //2965 // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal,2966 // so it is a legal inline immediate.2967 //2968 // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in2969 // floating-point, so it is a legal inline immediate.2970 2971 uint32_t Val = static_cast<uint32_t>(Literal);2972 return (Val == llvm::bit_cast<uint32_t>(0.0f)) ||2973 (Val == llvm::bit_cast<uint32_t>(1.0f)) ||2974 (Val == llvm::bit_cast<uint32_t>(-1.0f)) ||2975 (Val == llvm::bit_cast<uint32_t>(0.5f)) ||2976 (Val == llvm::bit_cast<uint32_t>(-0.5f)) ||2977 (Val == llvm::bit_cast<uint32_t>(2.0f)) ||2978 (Val == llvm::bit_cast<uint32_t>(-2.0f)) ||2979 (Val == llvm::bit_cast<uint32_t>(4.0f)) ||2980 (Val == llvm::bit_cast<uint32_t>(-4.0f)) ||2981 (Val == 0x3e22f983 && HasInv2Pi);2982}2983 2984bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi) {2985 if (!HasInv2Pi)2986 return false;2987 if (isInlinableIntLiteral(Literal))2988 return true;2989 uint16_t Val = static_cast<uint16_t>(Literal);2990 return Val == 0x3F00 || // 0.52991 Val == 0xBF00 || // -0.52992 Val == 0x3F80 || // 1.02993 Val == 0xBF80 || // -1.02994 Val == 0x4000 || // 2.02995 Val == 0xC000 || // -2.02996 Val == 0x4080 || // 4.02997 Val == 0xC080 || // -4.02998 Val == 0x3E22; // 1.0 / (2.0 * pi)2999}3000 3001bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi) {3002 return isInlinableLiteral32(Literal, HasInv2Pi);3003}3004 3005bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi) {3006 if (!HasInv2Pi)3007 return false;3008 if (isInlinableIntLiteral(Literal))3009 return true;3010 uint16_t Val = static_cast<uint16_t>(Literal);3011 return Val == 0x3C00 || // 1.03012 Val == 0xBC00 || // -1.03013 Val == 0x3800 || // 0.53014 Val == 0xB800 || // -0.53015 Val == 0x4000 || // 2.03016 Val == 0xC000 || // -2.03017 Val == 0x4400 || // 4.03018 Val == 0xC400 || // -4.03019 Val == 0x3118; // 1/2pi3020}3021 3022std::optional<unsigned> getInlineEncodingV216(bool IsFloat, uint32_t Literal) {3023 // Unfortunately, the Instruction Set Architecture Reference Guide is3024 // misleading about how the inline operands work for (packed) 16-bit3025 // instructions. In a nutshell, the actual HW behavior is:3026 //3027 // - integer encodings (-16 .. 64) are always produced as sign-extended3028 // 32-bit values3029 // - float encodings are produced as:3030 // - for F16 instructions: corresponding half-precision float values in3031 // the LSBs, 0 in the MSBs3032 // - for UI16 instructions: corresponding single-precision float value3033 int32_t Signed = static_cast<int32_t>(Literal);3034 if (Signed >= 0 && Signed <= 64)3035 return 128 + Signed;3036 3037 if (Signed >= -16 && Signed <= -1)3038 return 192 + std::abs(Signed);3039 3040 if (IsFloat) {3041 // clang-format off3042 switch (Literal) {3043 case 0x3800: return 240; // 0.53044 case 0xB800: return 241; // -0.53045 case 0x3C00: return 242; // 1.03046 case 0xBC00: return 243; // -1.03047 case 0x4000: return 244; // 2.03048 case 0xC000: return 245; // -2.03049 case 0x4400: return 246; // 4.03050 case 0xC400: return 247; // -4.03051 case 0x3118: return 248; // 1.0 / (2.0 * pi)3052 default: break;3053 }3054 // clang-format on3055 } else {3056 // clang-format off3057 switch (Literal) {3058 case 0x3F000000: return 240; // 0.53059 case 0xBF000000: return 241; // -0.53060 case 0x3F800000: return 242; // 1.03061 case 0xBF800000: return 243; // -1.03062 case 0x40000000: return 244; // 2.03063 case 0xC0000000: return 245; // -2.03064 case 0x40800000: return 246; // 4.03065 case 0xC0800000: return 247; // -4.03066 case 0x3E22F983: return 248; // 1.0 / (2.0 * pi)3067 default: break;3068 }3069 // clang-format on3070 }3071 3072 return {};3073}3074 3075// Encoding of the literal as an inline constant for a V_PK_*_IU16 instruction3076// or nullopt.3077std::optional<unsigned> getInlineEncodingV2I16(uint32_t Literal) {3078 return getInlineEncodingV216(false, Literal);3079}3080 3081// Encoding of the literal as an inline constant for a V_PK_*_BF16 instruction3082// or nullopt.3083std::optional<unsigned> getInlineEncodingV2BF16(uint32_t Literal) {3084 int32_t Signed = static_cast<int32_t>(Literal);3085 if (Signed >= 0 && Signed <= 64)3086 return 128 + Signed;3087 3088 if (Signed >= -16 && Signed <= -1)3089 return 192 + std::abs(Signed);3090 3091 // clang-format off3092 switch (Literal) {3093 case 0x3F00: return 240; // 0.53094 case 0xBF00: return 241; // -0.53095 case 0x3F80: return 242; // 1.03096 case 0xBF80: return 243; // -1.03097 case 0x4000: return 244; // 2.03098 case 0xC000: return 245; // -2.03099 case 0x4080: return 246; // 4.03100 case 0xC080: return 247; // -4.03101 case 0x3E22: return 248; // 1.0 / (2.0 * pi)3102 default: break;3103 }3104 // clang-format on3105 3106 return std::nullopt;3107}3108 3109// Encoding of the literal as an inline constant for a V_PK_*_F16 instruction3110// or nullopt.3111std::optional<unsigned> getInlineEncodingV2F16(uint32_t Literal) {3112 return getInlineEncodingV216(true, Literal);3113}3114 3115// Whether the given literal can be inlined for a V_PK_* instruction.3116bool isInlinableLiteralV216(uint32_t Literal, uint8_t OpType) {3117 switch (OpType) {3118 case AMDGPU::OPERAND_REG_IMM_V2INT16:3119 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:3120 return getInlineEncodingV216(false, Literal).has_value();3121 case AMDGPU::OPERAND_REG_IMM_V2FP16:3122 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:3123 return getInlineEncodingV216(true, Literal).has_value();3124 case AMDGPU::OPERAND_REG_IMM_V2BF16:3125 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:3126 return isInlinableLiteralV2BF16(Literal);3127 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:3128 return false;3129 default:3130 llvm_unreachable("bad packed operand type");3131 }3132}3133 3134// Whether the given literal can be inlined for a V_PK_*_IU16 instruction.3135bool isInlinableLiteralV2I16(uint32_t Literal) {3136 return getInlineEncodingV2I16(Literal).has_value();3137}3138 3139// Whether the given literal can be inlined for a V_PK_*_BF16 instruction.3140bool isInlinableLiteralV2BF16(uint32_t Literal) {3141 return getInlineEncodingV2BF16(Literal).has_value();3142}3143 3144// Whether the given literal can be inlined for a V_PK_*_F16 instruction.3145bool isInlinableLiteralV2F16(uint32_t Literal) {3146 return getInlineEncodingV2F16(Literal).has_value();3147}3148 3149bool isValid32BitLiteral(uint64_t Val, bool IsFP64) {3150 if (IsFP64)3151 return !Lo_32(Val);3152 3153 return isUInt<32>(Val) || isInt<32>(Val);3154}3155 3156int64_t encode32BitLiteral(int64_t Imm, OperandType Type, bool IsLit) {3157 switch (Type) {3158 default:3159 break;3160 case OPERAND_REG_IMM_BF16:3161 case OPERAND_REG_IMM_FP16:3162 case OPERAND_REG_INLINE_C_BF16:3163 case OPERAND_REG_INLINE_C_FP16:3164 return Imm & 0xffff;3165 case OPERAND_INLINE_SPLIT_BARRIER_INT32:3166 case OPERAND_REG_IMM_FP32:3167 case OPERAND_REG_IMM_INT32:3168 case OPERAND_REG_IMM_V2BF16:3169 case OPERAND_REG_IMM_V2FP16:3170 case OPERAND_REG_IMM_V2FP32:3171 case OPERAND_REG_IMM_V2INT16:3172 case OPERAND_REG_IMM_V2INT32:3173 case OPERAND_REG_INLINE_AC_FP32:3174 case OPERAND_REG_INLINE_AC_INT32:3175 case OPERAND_REG_INLINE_C_FP32:3176 case OPERAND_REG_INLINE_C_INT32:3177 return Lo_32(Imm);3178 case OPERAND_REG_IMM_FP64:3179 return IsLit ? Imm : Hi_32(Imm);3180 }3181 return Imm;3182}3183 3184bool isArgPassedInSGPR(const Argument *A) {3185 const Function *F = A->getParent();3186 3187 // Arguments to compute shaders are never a source of divergence.3188 CallingConv::ID CC = F->getCallingConv();3189 switch (CC) {3190 case CallingConv::AMDGPU_KERNEL:3191 case CallingConv::SPIR_KERNEL:3192 return true;3193 case CallingConv::AMDGPU_VS:3194 case CallingConv::AMDGPU_LS:3195 case CallingConv::AMDGPU_HS:3196 case CallingConv::AMDGPU_ES:3197 case CallingConv::AMDGPU_GS:3198 case CallingConv::AMDGPU_PS:3199 case CallingConv::AMDGPU_CS:3200 case CallingConv::AMDGPU_Gfx:3201 case CallingConv::AMDGPU_CS_Chain:3202 case CallingConv::AMDGPU_CS_ChainPreserve:3203 // For non-compute shaders, SGPR inputs are marked with either inreg or3204 // byval. Everything else is in VGPRs.3205 return A->hasAttribute(Attribute::InReg) ||3206 A->hasAttribute(Attribute::ByVal);3207 default:3208 // TODO: treat i1 as divergent?3209 return A->hasAttribute(Attribute::InReg);3210 }3211}3212 3213bool isArgPassedInSGPR(const CallBase *CB, unsigned ArgNo) {3214 // Arguments to compute shaders are never a source of divergence.3215 CallingConv::ID CC = CB->getCallingConv();3216 switch (CC) {3217 case CallingConv::AMDGPU_KERNEL:3218 case CallingConv::SPIR_KERNEL:3219 return true;3220 case CallingConv::AMDGPU_VS:3221 case CallingConv::AMDGPU_LS:3222 case CallingConv::AMDGPU_HS:3223 case CallingConv::AMDGPU_ES:3224 case CallingConv::AMDGPU_GS:3225 case CallingConv::AMDGPU_PS:3226 case CallingConv::AMDGPU_CS:3227 case CallingConv::AMDGPU_Gfx:3228 case CallingConv::AMDGPU_CS_Chain:3229 case CallingConv::AMDGPU_CS_ChainPreserve:3230 // For non-compute shaders, SGPR inputs are marked with either inreg or3231 // byval. Everything else is in VGPRs.3232 return CB->paramHasAttr(ArgNo, Attribute::InReg) ||3233 CB->paramHasAttr(ArgNo, Attribute::ByVal);3234 default:3235 return CB->paramHasAttr(ArgNo, Attribute::InReg);3236 }3237}3238 3239static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) {3240 return isGCN3Encoding(ST) || isGFX10Plus(ST);3241}3242 3243bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST,3244 int64_t EncodedOffset) {3245 if (isGFX12Plus(ST))3246 return isUInt<23>(EncodedOffset);3247 3248 return hasSMEMByteOffset(ST) ? isUInt<20>(EncodedOffset)3249 : isUInt<8>(EncodedOffset);3250}3251 3252bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST,3253 int64_t EncodedOffset, bool IsBuffer) {3254 if (isGFX12Plus(ST)) {3255 if (IsBuffer && EncodedOffset < 0)3256 return false;3257 return isInt<24>(EncodedOffset);3258 }3259 3260 return !IsBuffer && hasSMRDSignedImmOffset(ST) && isInt<21>(EncodedOffset);3261}3262 3263static bool isDwordAligned(uint64_t ByteOffset) {3264 return (ByteOffset & 3) == 0;3265}3266 3267uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST,3268 uint64_t ByteOffset) {3269 if (hasSMEMByteOffset(ST))3270 return ByteOffset;3271 3272 assert(isDwordAligned(ByteOffset));3273 return ByteOffset >> 2;3274}3275 3276std::optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST,3277 int64_t ByteOffset, bool IsBuffer,3278 bool HasSOffset) {3279 // For unbuffered smem loads, it is illegal for the Immediate Offset to be3280 // negative if the resulting (Offset + (M0 or SOffset or zero) is negative.3281 // Handle case where SOffset is not present.3282 if (!IsBuffer && !HasSOffset && ByteOffset < 0 && hasSMRDSignedImmOffset(ST))3283 return std::nullopt;3284 3285 if (isGFX12Plus(ST)) // 24 bit signed offsets3286 return isInt<24>(ByteOffset) ? std::optional<int64_t>(ByteOffset)3287 : std::nullopt;3288 3289 // The signed version is always a byte offset.3290 if (!IsBuffer && hasSMRDSignedImmOffset(ST)) {3291 assert(hasSMEMByteOffset(ST));3292 return isInt<20>(ByteOffset) ? std::optional<int64_t>(ByteOffset)3293 : std::nullopt;3294 }3295 3296 if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST))3297 return std::nullopt;3298 3299 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);3300 return isLegalSMRDEncodedUnsignedOffset(ST, EncodedOffset)3301 ? std::optional<int64_t>(EncodedOffset)3302 : std::nullopt;3303}3304 3305std::optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST,3306 int64_t ByteOffset) {3307 if (!isCI(ST) || !isDwordAligned(ByteOffset))3308 return std::nullopt;3309 3310 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);3311 return isUInt<32>(EncodedOffset) ? std::optional<int64_t>(EncodedOffset)3312 : std::nullopt;3313}3314 3315unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST) {3316 if (AMDGPU::isGFX10(ST))3317 return 12;3318 3319 if (AMDGPU::isGFX12(ST))3320 return 24;3321 return 13;3322}3323 3324namespace {3325 3326struct SourceOfDivergence {3327 unsigned Intr;3328};3329const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr);3330 3331struct AlwaysUniform {3332 unsigned Intr;3333};3334const AlwaysUniform *lookupAlwaysUniform(unsigned Intr);3335 3336#define GET_SourcesOfDivergence_IMPL3337#define GET_UniformIntrinsics_IMPL3338#define GET_Gfx9BufferFormat_IMPL3339#define GET_Gfx10BufferFormat_IMPL3340#define GET_Gfx11PlusBufferFormat_IMPL3341 3342#include "AMDGPUGenSearchableTables.inc"3343 3344} // end anonymous namespace3345 3346bool isIntrinsicSourceOfDivergence(unsigned IntrID) {3347 return lookupSourceOfDivergence(IntrID);3348}3349 3350bool isIntrinsicAlwaysUniform(unsigned IntrID) {3351 return lookupAlwaysUniform(IntrID);3352}3353 3354const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp,3355 uint8_t NumComponents,3356 uint8_t NumFormat,3357 const MCSubtargetInfo &STI) {3358 return isGFX11Plus(STI) ? getGfx11PlusBufferFormatInfo(3359 BitsPerComp, NumComponents, NumFormat)3360 : isGFX10(STI)3361 ? getGfx10BufferFormatInfo(BitsPerComp, NumComponents, NumFormat)3362 : getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat);3363}3364 3365const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format,3366 const MCSubtargetInfo &STI) {3367 return isGFX11Plus(STI) ? getGfx11PlusBufferFormatInfo(Format)3368 : isGFX10(STI) ? getGfx10BufferFormatInfo(Format)3369 : getGfx9BufferFormatInfo(Format);3370}3371 3372const MCRegisterClass *getVGPRPhysRegClass(MCRegister Reg,3373 const MCRegisterInfo &MRI) {3374 const unsigned VGPRClasses[] = {3375 AMDGPU::VGPR_16RegClassID, AMDGPU::VGPR_32RegClassID,3376 AMDGPU::VReg_64RegClassID, AMDGPU::VReg_96RegClassID,3377 AMDGPU::VReg_128RegClassID, AMDGPU::VReg_160RegClassID,3378 AMDGPU::VReg_192RegClassID, AMDGPU::VReg_224RegClassID,3379 AMDGPU::VReg_256RegClassID, AMDGPU::VReg_288RegClassID,3380 AMDGPU::VReg_320RegClassID, AMDGPU::VReg_352RegClassID,3381 AMDGPU::VReg_384RegClassID, AMDGPU::VReg_512RegClassID,3382 AMDGPU::VReg_1024RegClassID};3383 3384 for (unsigned RCID : VGPRClasses) {3385 const MCRegisterClass &RC = MRI.getRegClass(RCID);3386 if (RC.contains(Reg))3387 return &RC;3388 }3389 3390 return nullptr;3391}3392 3393unsigned getVGPREncodingMSBs(MCRegister Reg, const MCRegisterInfo &MRI) {3394 unsigned Enc = MRI.getEncodingValue(Reg);3395 unsigned Idx = Enc & AMDGPU::HWEncoding::REG_IDX_MASK;3396 return Idx >> 8;3397}3398 3399MCRegister getVGPRWithMSBs(MCRegister Reg, unsigned MSBs,3400 const MCRegisterInfo &MRI) {3401 unsigned Enc = MRI.getEncodingValue(Reg);3402 unsigned Idx = Enc & AMDGPU::HWEncoding::REG_IDX_MASK;3403 if (Idx >= 0x100)3404 return MCRegister();3405 3406 const MCRegisterClass *RC = getVGPRPhysRegClass(Reg, MRI);3407 if (!RC)3408 return MCRegister();3409 3410 Idx |= MSBs << 8;3411 if (RC->getID() == AMDGPU::VGPR_16RegClassID) {3412 // This class has 2048 registers with interleaved lo16 and hi16.3413 Idx *= 2;3414 if (Enc & AMDGPU::HWEncoding::IS_HI16)3415 ++Idx;3416 }3417 3418 return RC->getRegister(Idx);3419}3420 3421std::pair<const AMDGPU::OpName *, const AMDGPU::OpName *>3422getVGPRLoweringOperandTables(const MCInstrDesc &Desc) {3423 static const AMDGPU::OpName VOPOps[4] = {3424 AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2,3425 AMDGPU::OpName::vdst};3426 static const AMDGPU::OpName VDSOps[4] = {3427 AMDGPU::OpName::addr, AMDGPU::OpName::data0, AMDGPU::OpName::data1,3428 AMDGPU::OpName::vdst};3429 static const AMDGPU::OpName FLATOps[4] = {3430 AMDGPU::OpName::vaddr, AMDGPU::OpName::vdata,3431 AMDGPU::OpName::NUM_OPERAND_NAMES, AMDGPU::OpName::vdst};3432 static const AMDGPU::OpName BUFOps[4] = {3433 AMDGPU::OpName::vaddr, AMDGPU::OpName::NUM_OPERAND_NAMES,3434 AMDGPU::OpName::NUM_OPERAND_NAMES, AMDGPU::OpName::vdata};3435 static const AMDGPU::OpName VIMGOps[4] = {3436 AMDGPU::OpName::vaddr0, AMDGPU::OpName::vaddr1, AMDGPU::OpName::vaddr2,3437 AMDGPU::OpName::vdata};3438 3439 // For VOPD instructions MSB of a corresponding Y component operand VGPR3440 // address is supposed to match X operand, otherwise VOPD shall not be3441 // combined.3442 static const AMDGPU::OpName VOPDOpsX[4] = {3443 AMDGPU::OpName::src0X, AMDGPU::OpName::vsrc1X, AMDGPU::OpName::vsrc2X,3444 AMDGPU::OpName::vdstX};3445 static const AMDGPU::OpName VOPDOpsY[4] = {3446 AMDGPU::OpName::src0Y, AMDGPU::OpName::vsrc1Y, AMDGPU::OpName::vsrc2Y,3447 AMDGPU::OpName::vdstY};3448 3449 // VOP2 MADMK instructions use src0, imm, src1 scheme.3450 static const AMDGPU::OpName VOP2MADMKOps[4] = {3451 AMDGPU::OpName::src0, AMDGPU::OpName::NUM_OPERAND_NAMES,3452 AMDGPU::OpName::src1, AMDGPU::OpName::vdst};3453 3454 unsigned TSFlags = Desc.TSFlags;3455 3456 if (TSFlags &3457 (SIInstrFlags::VOP1 | SIInstrFlags::VOP2 | SIInstrFlags::VOP3 |3458 SIInstrFlags::VOP3P | SIInstrFlags::VOPC | SIInstrFlags::DPP)) {3459 switch (Desc.getOpcode()) {3460 // LD_SCALE operands ignore MSB.3461 case AMDGPU::V_WMMA_LD_SCALE_PAIRED_B32:3462 case AMDGPU::V_WMMA_LD_SCALE_PAIRED_B32_gfx1250:3463 case AMDGPU::V_WMMA_LD_SCALE16_PAIRED_B64:3464 case AMDGPU::V_WMMA_LD_SCALE16_PAIRED_B64_gfx1250:3465 return {};3466 case AMDGPU::V_FMAMK_F16:3467 case AMDGPU::V_FMAMK_F16_t16:3468 case AMDGPU::V_FMAMK_F16_t16_gfx12:3469 case AMDGPU::V_FMAMK_F16_fake16:3470 case AMDGPU::V_FMAMK_F16_fake16_gfx12:3471 case AMDGPU::V_FMAMK_F32:3472 case AMDGPU::V_FMAMK_F32_gfx12:3473 case AMDGPU::V_FMAMK_F64:3474 case AMDGPU::V_FMAMK_F64_gfx1250:3475 return {VOP2MADMKOps, nullptr};3476 default:3477 break;3478 }3479 return {VOPOps, nullptr};3480 }3481 3482 if (TSFlags & SIInstrFlags::DS)3483 return {VDSOps, nullptr};3484 3485 if (TSFlags & SIInstrFlags::FLAT)3486 return {FLATOps, nullptr};3487 3488 if (TSFlags & (SIInstrFlags::MUBUF | SIInstrFlags::MTBUF))3489 return {BUFOps, nullptr};3490 3491 if (TSFlags & SIInstrFlags::VIMAGE)3492 return {VIMGOps, nullptr};3493 3494 if (AMDGPU::isVOPD(Desc.getOpcode()))3495 return {VOPDOpsX, VOPDOpsY};3496 3497 assert(!(TSFlags & SIInstrFlags::MIMG));3498 3499 if (TSFlags & (SIInstrFlags::VSAMPLE | SIInstrFlags::EXP))3500 llvm_unreachable("Sample and export VGPR lowering is not implemented and"3501 " these instructions are not expected on gfx1250");3502 3503 return {};3504}3505 3506bool supportsScaleOffset(const MCInstrInfo &MII, unsigned Opcode) {3507 uint64_t TSFlags = MII.get(Opcode).TSFlags;3508 3509 if (TSFlags & SIInstrFlags::SMRD)3510 return !getSMEMIsBuffer(Opcode);3511 if (!(TSFlags & SIInstrFlags::FLAT))3512 return false;3513 3514 // Only SV and SVS modes are supported.3515 if (TSFlags & SIInstrFlags::FlatScratch)3516 return hasNamedOperand(Opcode, OpName::vaddr);3517 3518 // Only GVS mode is supported.3519 return hasNamedOperand(Opcode, OpName::vaddr) &&3520 hasNamedOperand(Opcode, OpName::saddr);3521 3522 return false;3523}3524 3525bool hasAny64BitVGPROperands(const MCInstrDesc &OpDesc, const MCInstrInfo &MII,3526 const MCSubtargetInfo &ST) {3527 for (auto OpName : {OpName::vdst, OpName::src0, OpName::src1, OpName::src2}) {3528 int Idx = getNamedOperandIdx(OpDesc.getOpcode(), OpName);3529 if (Idx == -1)3530 continue;3531 3532 const MCOperandInfo &OpInfo = OpDesc.operands()[Idx];3533 int16_t RegClass = MII.getOpRegClassID(3534 OpInfo, ST.getHwMode(MCSubtargetInfo::HwMode_RegInfo));3535 if (RegClass == AMDGPU::VReg_64RegClassID ||3536 RegClass == AMDGPU::VReg_64_Align2RegClassID)3537 return true;3538 }3539 3540 return false;3541}3542 3543bool isDPALU_DPP32BitOpc(unsigned Opc) {3544 switch (Opc) {3545 case AMDGPU::V_MUL_LO_U32_e64:3546 case AMDGPU::V_MUL_LO_U32_e64_dpp:3547 case AMDGPU::V_MUL_LO_U32_e64_dpp_gfx1250:3548 case AMDGPU::V_MUL_HI_U32_e64:3549 case AMDGPU::V_MUL_HI_U32_e64_dpp:3550 case AMDGPU::V_MUL_HI_U32_e64_dpp_gfx1250:3551 case AMDGPU::V_MUL_HI_I32_e64:3552 case AMDGPU::V_MUL_HI_I32_e64_dpp:3553 case AMDGPU::V_MUL_HI_I32_e64_dpp_gfx1250:3554 case AMDGPU::V_MAD_U32_e64:3555 case AMDGPU::V_MAD_U32_e64_dpp:3556 case AMDGPU::V_MAD_U32_e64_dpp_gfx1250:3557 return true;3558 default:3559 return false;3560 }3561}3562 3563bool isDPALU_DPP(const MCInstrDesc &OpDesc, const MCInstrInfo &MII,3564 const MCSubtargetInfo &ST) {3565 if (!ST.hasFeature(AMDGPU::FeatureDPALU_DPP))3566 return false;3567 3568 if (isDPALU_DPP32BitOpc(OpDesc.getOpcode()))3569 return ST.hasFeature(AMDGPU::FeatureGFX1250Insts);3570 3571 return hasAny64BitVGPROperands(OpDesc, MII, ST);3572}3573 3574unsigned getLdsDwGranularity(const MCSubtargetInfo &ST) {3575 if (ST.getFeatureBits().test(FeatureAddressableLocalMemorySize32768))3576 return 64;3577 if (ST.getFeatureBits().test(FeatureAddressableLocalMemorySize65536))3578 return 128;3579 if (ST.getFeatureBits().test(FeatureAddressableLocalMemorySize163840))3580 return 320;3581 if (ST.getFeatureBits().test(FeatureAddressableLocalMemorySize327680))3582 return 512;3583 return 64; // In sync with getAddressableLocalMemorySize3584}3585 3586bool isPackedFP32Inst(unsigned Opc) {3587 switch (Opc) {3588 case AMDGPU::V_PK_ADD_F32:3589 case AMDGPU::V_PK_ADD_F32_gfx12:3590 case AMDGPU::V_PK_MUL_F32:3591 case AMDGPU::V_PK_MUL_F32_gfx12:3592 case AMDGPU::V_PK_FMA_F32:3593 case AMDGPU::V_PK_FMA_F32_gfx12:3594 return true;3595 default:3596 return false;3597 }3598}3599 3600const std::array<unsigned, 3> &ClusterDimsAttr::getDims() const {3601 assert(isFixedDims() && "expect kind to be FixedDims");3602 return Dims;3603}3604 3605std::string ClusterDimsAttr::to_string() const {3606 SmallString<10> Buffer;3607 raw_svector_ostream OS(Buffer);3608 3609 switch (getKind()) {3610 case Kind::Unknown:3611 return "";3612 case Kind::NoCluster: {3613 OS << EncoNoCluster << ',' << EncoNoCluster << ',' << EncoNoCluster;3614 return Buffer.c_str();3615 }3616 case Kind::VariableDims: {3617 OS << EncoVariableDims << ',' << EncoVariableDims << ','3618 << EncoVariableDims;3619 return Buffer.c_str();3620 }3621 case Kind::FixedDims: {3622 OS << Dims[0] << ',' << Dims[1] << ',' << Dims[2];3623 return Buffer.c_str();3624 }3625 }3626 llvm_unreachable("Unknown ClusterDimsAttr kind");3627}3628 3629ClusterDimsAttr ClusterDimsAttr::get(const Function &F) {3630 std::optional<SmallVector<unsigned>> Attr =3631 getIntegerVecAttribute(F, "amdgpu-cluster-dims", /*Size=*/3);3632 ClusterDimsAttr::Kind AttrKind = Kind::FixedDims;3633 3634 if (!Attr.has_value())3635 AttrKind = Kind::Unknown;3636 else if (all_of(*Attr, [](unsigned V) { return V == EncoNoCluster; }))3637 AttrKind = Kind::NoCluster;3638 else if (all_of(*Attr, [](unsigned V) { return V == EncoVariableDims; }))3639 AttrKind = Kind::VariableDims;3640 3641 ClusterDimsAttr A(AttrKind);3642 if (AttrKind == Kind::FixedDims)3643 A.Dims = {(*Attr)[0], (*Attr)[1], (*Attr)[2]};3644 3645 return A;3646}3647 3648} // namespace AMDGPU3649 3650raw_ostream &operator<<(raw_ostream &OS,3651 const AMDGPU::IsaInfo::TargetIDSetting S) {3652 switch (S) {3653 case (AMDGPU::IsaInfo::TargetIDSetting::Unsupported):3654 OS << "Unsupported";3655 break;3656 case (AMDGPU::IsaInfo::TargetIDSetting::Any):3657 OS << "Any";3658 break;3659 case (AMDGPU::IsaInfo::TargetIDSetting::Off):3660 OS << "Off";3661 break;3662 case (AMDGPU::IsaInfo::TargetIDSetting::On):3663 OS << "On";3664 break;3665 }3666 return OS;3667}3668 3669} // namespace llvm3670