1830 lines · c
1//===- SIInstrInfo.h - SI Instruction Info Interface ------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9/// \file10/// Interface definition for SIInstrInfo.11//12//===----------------------------------------------------------------------===//13 14#ifndef LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H15#define LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H16 17#include "AMDGPUMIRFormatter.h"18#include "MCTargetDesc/AMDGPUMCTargetDesc.h"19#include "SIRegisterInfo.h"20#include "Utils/AMDGPUBaseInfo.h"21#include "llvm/ADT/SetVector.h"22#include "llvm/CodeGen/TargetInstrInfo.h"23#include "llvm/CodeGen/TargetSchedule.h"24 25#define GET_INSTRINFO_HEADER26#include "AMDGPUGenInstrInfo.inc"27 28namespace llvm {29 30class APInt;31class GCNSubtarget;32class LiveVariables;33class MachineDominatorTree;34class MachineRegisterInfo;35class RegScavenger;36class SIMachineFunctionInfo;37class TargetRegisterClass;38class ScheduleHazardRecognizer;39 40constexpr unsigned DefaultMemoryClusterDWordsLimit = 8;41 42/// Mark the MMO of a uniform load if there are no potentially clobbering stores43/// on any path from the start of an entry function to this load.44static const MachineMemOperand::Flags MONoClobber =45 MachineMemOperand::MOTargetFlag1;46 47/// Mark the MMO of a load as the last use.48static const MachineMemOperand::Flags MOLastUse =49 MachineMemOperand::MOTargetFlag2;50 51/// Mark the MMO of cooperative load/store atomics.52static const MachineMemOperand::Flags MOCooperative =53 MachineMemOperand::MOTargetFlag3;54 55/// Utility to store machine instructions worklist.56struct SIInstrWorklist {57 SIInstrWorklist() = default;58 59 void insert(MachineInstr *MI);60 61 MachineInstr *top() const {62 const auto *iter = InstrList.begin();63 return *iter;64 }65 66 void erase_top() {67 const auto *iter = InstrList.begin();68 InstrList.erase(iter);69 }70 71 bool empty() const { return InstrList.empty(); }72 73 void clear() {74 InstrList.clear();75 DeferredList.clear();76 }77 78 bool isDeferred(MachineInstr *MI);79 80 SetVector<MachineInstr *> &getDeferredList() { return DeferredList; }81 82private:83 /// InstrList contains the MachineInstrs.84 SetVector<MachineInstr *> InstrList;85 /// Deferred instructions are specific MachineInstr86 /// that will be added by insert method.87 SetVector<MachineInstr *> DeferredList;88};89 90class SIInstrInfo final : public AMDGPUGenInstrInfo {91 struct ThreeAddressUpdates;92 93private:94 const SIRegisterInfo RI;95 const GCNSubtarget &ST;96 TargetSchedModel SchedModel;97 mutable std::unique_ptr<AMDGPUMIRFormatter> Formatter;98 99 // The inverse predicate should have the negative value.100 enum BranchPredicate {101 INVALID_BR = 0,102 SCC_TRUE = 1,103 SCC_FALSE = -1,104 VCCNZ = 2,105 VCCZ = -2,106 EXECNZ = -3,107 EXECZ = 3108 };109 110 using SetVectorType = SmallSetVector<MachineInstr *, 32>;111 112 static unsigned getBranchOpcode(BranchPredicate Cond);113 static BranchPredicate getBranchPredicate(unsigned Opcode);114 115public:116 unsigned buildExtractSubReg(MachineBasicBlock::iterator MI,117 MachineRegisterInfo &MRI,118 const MachineOperand &SuperReg,119 const TargetRegisterClass *SuperRC,120 unsigned SubIdx,121 const TargetRegisterClass *SubRC) const;122 MachineOperand buildExtractSubRegOrImm(123 MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI,124 const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC,125 unsigned SubIdx, const TargetRegisterClass *SubRC) const;126 127private:128 void swapOperands(MachineInstr &Inst) const;129 130 std::pair<bool, MachineBasicBlock *>131 moveScalarAddSub(SIInstrWorklist &Worklist, MachineInstr &Inst,132 MachineDominatorTree *MDT = nullptr) const;133 134 void lowerSelect(SIInstrWorklist &Worklist, MachineInstr &Inst,135 MachineDominatorTree *MDT = nullptr) const;136 137 void lowerScalarAbs(SIInstrWorklist &Worklist, MachineInstr &Inst) const;138 139 void lowerScalarAbsDiff(SIInstrWorklist &Worklist, MachineInstr &Inst) const;140 141 void lowerScalarXnor(SIInstrWorklist &Worklist, MachineInstr &Inst) const;142 143 void splitScalarNotBinop(SIInstrWorklist &Worklist, MachineInstr &Inst,144 unsigned Opcode) const;145 146 void splitScalarBinOpN2(SIInstrWorklist &Worklist, MachineInstr &Inst,147 unsigned Opcode) const;148 149 void splitScalar64BitUnaryOp(SIInstrWorklist &Worklist, MachineInstr &Inst,150 unsigned Opcode, bool Swap = false) const;151 152 void splitScalar64BitBinaryOp(SIInstrWorklist &Worklist, MachineInstr &Inst,153 unsigned Opcode,154 MachineDominatorTree *MDT = nullptr) const;155 156 void splitScalarSMulU64(SIInstrWorklist &Worklist, MachineInstr &Inst,157 MachineDominatorTree *MDT) const;158 159 void splitScalarSMulPseudo(SIInstrWorklist &Worklist, MachineInstr &Inst,160 MachineDominatorTree *MDT) const;161 162 void splitScalar64BitXnor(SIInstrWorklist &Worklist, MachineInstr &Inst,163 MachineDominatorTree *MDT = nullptr) const;164 165 void splitScalar64BitBCNT(SIInstrWorklist &Worklist,166 MachineInstr &Inst) const;167 void splitScalar64BitBFE(SIInstrWorklist &Worklist, MachineInstr &Inst) const;168 void splitScalar64BitCountOp(SIInstrWorklist &Worklist, MachineInstr &Inst,169 unsigned Opcode,170 MachineDominatorTree *MDT = nullptr) const;171 void movePackToVALU(SIInstrWorklist &Worklist, MachineRegisterInfo &MRI,172 MachineInstr &Inst) const;173 174 void addUsersToMoveToVALUWorklist(Register Reg, MachineRegisterInfo &MRI,175 SIInstrWorklist &Worklist) const;176 177 void addSCCDefUsersToVALUWorklist(const MachineOperand &Op,178 MachineInstr &SCCDefInst,179 SIInstrWorklist &Worklist,180 Register NewCond = Register()) const;181 void addSCCDefsToVALUWorklist(MachineInstr *SCCUseInst,182 SIInstrWorklist &Worklist) const;183 184 const TargetRegisterClass *185 getDestEquivalentVGPRClass(const MachineInstr &Inst) const;186 187 bool checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,188 const MachineInstr &MIb) const;189 190 Register findUsedSGPR(const MachineInstr &MI, int OpIndices[3]) const;191 192 bool verifyCopy(const MachineInstr &MI, const MachineRegisterInfo &MRI,193 StringRef &ErrInfo) const;194 195 bool resultDependsOnExec(const MachineInstr &MI) const;196 197 MachineInstr *convertToThreeAddressImpl(MachineInstr &MI,198 ThreeAddressUpdates &Updates) const;199 200protected:201 /// If the specific machine instruction is a instruction that moves/copies202 /// value from one register to another register return destination and source203 /// registers as machine operands.204 std::optional<DestSourcePair>205 isCopyInstrImpl(const MachineInstr &MI) const override;206 207 bool swapSourceModifiers(MachineInstr &MI, MachineOperand &Src0,208 AMDGPU::OpName Src0OpName, MachineOperand &Src1,209 AMDGPU::OpName Src1OpName) const;210 bool isLegalToSwap(const MachineInstr &MI, unsigned fromIdx,211 unsigned toIdx) const;212 MachineInstr *commuteInstructionImpl(MachineInstr &MI, bool NewMI,213 unsigned OpIdx0,214 unsigned OpIdx1) const override;215 216public:217 enum TargetOperandFlags {218 MO_MASK = 0xf,219 220 MO_NONE = 0,221 // MO_GOTPCREL -> symbol@GOTPCREL -> R_AMDGPU_GOTPCREL.222 MO_GOTPCREL = 1,223 // MO_GOTPCREL32_LO -> symbol@gotpcrel32@lo -> R_AMDGPU_GOTPCREL32_LO.224 MO_GOTPCREL32 = 2,225 MO_GOTPCREL32_LO = 2,226 // MO_GOTPCREL32_HI -> symbol@gotpcrel32@hi -> R_AMDGPU_GOTPCREL32_HI.227 MO_GOTPCREL32_HI = 3,228 // MO_GOTPCREL64 -> symbol@GOTPCREL -> R_AMDGPU_GOTPCREL.229 MO_GOTPCREL64 = 4,230 // MO_REL32_LO -> symbol@rel32@lo -> R_AMDGPU_REL32_LO.231 MO_REL32 = 5,232 MO_REL32_LO = 5,233 // MO_REL32_HI -> symbol@rel32@hi -> R_AMDGPU_REL32_HI.234 MO_REL32_HI = 6,235 MO_REL64 = 7,236 237 MO_FAR_BRANCH_OFFSET = 8,238 239 MO_ABS32_LO = 9,240 MO_ABS32_HI = 10,241 MO_ABS64 = 11,242 };243 244 explicit SIInstrInfo(const GCNSubtarget &ST);245 246 const SIRegisterInfo &getRegisterInfo() const {247 return RI;248 }249 250 const GCNSubtarget &getSubtarget() const {251 return ST;252 }253 254 bool isReMaterializableImpl(const MachineInstr &MI) const override;255 256 bool isIgnorableUse(const MachineOperand &MO) const override;257 258 bool isSafeToSink(MachineInstr &MI, MachineBasicBlock *SuccToSinkTo,259 MachineCycleInfo *CI) const override;260 261 bool areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1, int64_t &Offset0,262 int64_t &Offset1) const override;263 264 bool isGlobalMemoryObject(const MachineInstr *MI) const override;265 266 bool getMemOperandsWithOffsetWidth(267 const MachineInstr &LdSt,268 SmallVectorImpl<const MachineOperand *> &BaseOps, int64_t &Offset,269 bool &OffsetIsScalable, LocationSize &Width,270 const TargetRegisterInfo *TRI) const final;271 272 bool shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,273 int64_t Offset1, bool OffsetIsScalable1,274 ArrayRef<const MachineOperand *> BaseOps2,275 int64_t Offset2, bool OffsetIsScalable2,276 unsigned ClusterSize,277 unsigned NumBytes) const override;278 279 bool shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1, int64_t Offset0,280 int64_t Offset1, unsigned NumLoads) const override;281 282 void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,283 const DebugLoc &DL, Register DestReg, Register SrcReg,284 bool KillSrc, bool RenamableDest = false,285 bool RenamableSrc = false) const override;286 287 const TargetRegisterClass *getPreferredSelectRegClass(288 unsigned Size) const;289 290 Register insertNE(MachineBasicBlock *MBB,291 MachineBasicBlock::iterator I, const DebugLoc &DL,292 Register SrcReg, int Value) const;293 294 Register insertEQ(MachineBasicBlock *MBB,295 MachineBasicBlock::iterator I, const DebugLoc &DL,296 Register SrcReg, int Value) const;297 298 bool getConstValDefinedInReg(const MachineInstr &MI, const Register Reg,299 int64_t &ImmVal) const override;300 301 unsigned getVectorRegSpillSaveOpcode(Register Reg,302 const TargetRegisterClass *RC,303 unsigned Size,304 const SIMachineFunctionInfo &MFI) const;305 unsigned306 getVectorRegSpillRestoreOpcode(Register Reg, const TargetRegisterClass *RC,307 unsigned Size,308 const SIMachineFunctionInfo &MFI) const;309 310 void storeRegToStackSlot(311 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,312 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,313 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;314 315 void loadRegFromStackSlot(316 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg,317 int FrameIndex, const TargetRegisterClass *RC, Register VReg,318 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;319 320 bool expandPostRAPseudo(MachineInstr &MI) const override;321 322 void reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,323 Register DestReg, unsigned SubIdx,324 const MachineInstr &Orig) const override;325 326 // Splits a V_MOV_B64_DPP_PSEUDO opcode into a pair of v_mov_b32_dpp327 // instructions. Returns a pair of generated instructions.328 // Can split either post-RA with physical registers or pre-RA with329 // virtual registers. In latter case IR needs to be in SSA form and330 // and a REG_SEQUENCE is produced to define original register.331 std::pair<MachineInstr*, MachineInstr*>332 expandMovDPP64(MachineInstr &MI) const;333 334 // Returns an opcode that can be used to move a value to a \p DstRC335 // register. If there is no hardware instruction that can store to \p336 // DstRC, then AMDGPU::COPY is returned.337 unsigned getMovOpcode(const TargetRegisterClass *DstRC) const;338 339 const MCInstrDesc &getIndirectRegWriteMovRelPseudo(unsigned VecSize,340 unsigned EltSize,341 bool IsSGPR) const;342 343 const MCInstrDesc &getIndirectGPRIDXPseudo(unsigned VecSize,344 bool IsIndirectSrc) const;345 LLVM_READONLY346 int commuteOpcode(unsigned Opc) const;347 348 LLVM_READONLY349 inline int commuteOpcode(const MachineInstr &MI) const {350 return commuteOpcode(MI.getOpcode());351 }352 353 bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx0,354 unsigned &SrcOpIdx1) const override;355 356 bool findCommutedOpIndices(const MCInstrDesc &Desc, unsigned &SrcOpIdx0,357 unsigned &SrcOpIdx1) const;358 359 bool isBranchOffsetInRange(unsigned BranchOpc,360 int64_t BrOffset) const override;361 362 MachineBasicBlock *getBranchDestBlock(const MachineInstr &MI) const override;363 364 /// Return whether the block terminate with divergent branch.365 /// Note this only work before lowering the pseudo control flow instructions.366 bool hasDivergentBranch(const MachineBasicBlock *MBB) const;367 368 void insertIndirectBranch(MachineBasicBlock &MBB,369 MachineBasicBlock &NewDestBB,370 MachineBasicBlock &RestoreBB, const DebugLoc &DL,371 int64_t BrOffset, RegScavenger *RS) const override;372 373 bool analyzeBranchImpl(MachineBasicBlock &MBB,374 MachineBasicBlock::iterator I,375 MachineBasicBlock *&TBB,376 MachineBasicBlock *&FBB,377 SmallVectorImpl<MachineOperand> &Cond,378 bool AllowModify) const;379 380 bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,381 MachineBasicBlock *&FBB,382 SmallVectorImpl<MachineOperand> &Cond,383 bool AllowModify = false) const override;384 385 unsigned removeBranch(MachineBasicBlock &MBB,386 int *BytesRemoved = nullptr) const override;387 388 unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,389 MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,390 const DebugLoc &DL,391 int *BytesAdded = nullptr) const override;392 393 bool reverseBranchCondition(394 SmallVectorImpl<MachineOperand> &Cond) const override;395 396 bool canInsertSelect(const MachineBasicBlock &MBB,397 ArrayRef<MachineOperand> Cond, Register DstReg,398 Register TrueReg, Register FalseReg, int &CondCycles,399 int &TrueCycles, int &FalseCycles) const override;400 401 void insertSelect(MachineBasicBlock &MBB,402 MachineBasicBlock::iterator I, const DebugLoc &DL,403 Register DstReg, ArrayRef<MachineOperand> Cond,404 Register TrueReg, Register FalseReg) const override;405 406 void insertVectorSelect(MachineBasicBlock &MBB,407 MachineBasicBlock::iterator I, const DebugLoc &DL,408 Register DstReg, ArrayRef<MachineOperand> Cond,409 Register TrueReg, Register FalseReg) const;410 411 bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,412 Register &SrcReg2, int64_t &CmpMask,413 int64_t &CmpValue) const override;414 415 bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,416 Register SrcReg2, int64_t CmpMask, int64_t CmpValue,417 const MachineRegisterInfo *MRI) const override;418 419 bool420 areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,421 const MachineInstr &MIb) const override;422 423 static bool isFoldableCopy(const MachineInstr &MI);424 static unsigned getFoldableCopySrcIdx(const MachineInstr &MI);425 426 void removeModOperands(MachineInstr &MI) const;427 428 void mutateAndCleanupImplicit(MachineInstr &MI,429 const MCInstrDesc &NewDesc) const;430 431 /// Return the extracted immediate value in a subregister use from a constant432 /// materialized in a super register.433 ///434 /// e.g. %imm = S_MOV_B64 K[0:63]435 /// USE %imm.sub1436 /// This will return K[32:63]437 static std::optional<int64_t> extractSubregFromImm(int64_t ImmVal,438 unsigned SubRegIndex);439 440 bool foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg,441 MachineRegisterInfo *MRI) const final;442 443 unsigned getMachineCSELookAheadLimit() const override { return 500; }444 445 MachineInstr *convertToThreeAddress(MachineInstr &MI, LiveVariables *LV,446 LiveIntervals *LIS) const override;447 448 bool isSchedulingBoundary(const MachineInstr &MI,449 const MachineBasicBlock *MBB,450 const MachineFunction &MF) const override;451 452 static bool isSALU(const MachineInstr &MI) {453 return MI.getDesc().TSFlags & SIInstrFlags::SALU;454 }455 456 bool isSALU(uint16_t Opcode) const {457 return get(Opcode).TSFlags & SIInstrFlags::SALU;458 }459 460 static bool isVALU(const MachineInstr &MI) {461 return MI.getDesc().TSFlags & SIInstrFlags::VALU;462 }463 464 bool isVALU(uint16_t Opcode) const {465 return get(Opcode).TSFlags & SIInstrFlags::VALU;466 }467 468 static bool isImage(const MachineInstr &MI) {469 return isMIMG(MI) || isVSAMPLE(MI) || isVIMAGE(MI);470 }471 472 bool isImage(uint16_t Opcode) const {473 return isMIMG(Opcode) || isVSAMPLE(Opcode) || isVIMAGE(Opcode);474 }475 476 static bool isVMEM(const MachineInstr &MI) {477 return isMUBUF(MI) || isMTBUF(MI) || isImage(MI) || isFLAT(MI);478 }479 480 bool isVMEM(uint16_t Opcode) const {481 return isMUBUF(Opcode) || isMTBUF(Opcode) || isImage(Opcode);482 }483 484 static bool isSOP1(const MachineInstr &MI) {485 return MI.getDesc().TSFlags & SIInstrFlags::SOP1;486 }487 488 bool isSOP1(uint16_t Opcode) const {489 return get(Opcode).TSFlags & SIInstrFlags::SOP1;490 }491 492 static bool isSOP2(const MachineInstr &MI) {493 return MI.getDesc().TSFlags & SIInstrFlags::SOP2;494 }495 496 bool isSOP2(uint16_t Opcode) const {497 return get(Opcode).TSFlags & SIInstrFlags::SOP2;498 }499 500 static bool isSOPC(const MachineInstr &MI) {501 return MI.getDesc().TSFlags & SIInstrFlags::SOPC;502 }503 504 bool isSOPC(uint16_t Opcode) const {505 return get(Opcode).TSFlags & SIInstrFlags::SOPC;506 }507 508 static bool isSOPK(const MachineInstr &MI) {509 return MI.getDesc().TSFlags & SIInstrFlags::SOPK;510 }511 512 bool isSOPK(uint16_t Opcode) const {513 return get(Opcode).TSFlags & SIInstrFlags::SOPK;514 }515 516 static bool isSOPP(const MachineInstr &MI) {517 return MI.getDesc().TSFlags & SIInstrFlags::SOPP;518 }519 520 bool isSOPP(uint16_t Opcode) const {521 return get(Opcode).TSFlags & SIInstrFlags::SOPP;522 }523 524 static bool isPacked(const MachineInstr &MI) {525 return MI.getDesc().TSFlags & SIInstrFlags::IsPacked;526 }527 528 bool isPacked(uint16_t Opcode) const {529 return get(Opcode).TSFlags & SIInstrFlags::IsPacked;530 }531 532 static bool isVOP1(const MachineInstr &MI) {533 return MI.getDesc().TSFlags & SIInstrFlags::VOP1;534 }535 536 bool isVOP1(uint16_t Opcode) const {537 return get(Opcode).TSFlags & SIInstrFlags::VOP1;538 }539 540 static bool isVOP2(const MachineInstr &MI) {541 return MI.getDesc().TSFlags & SIInstrFlags::VOP2;542 }543 544 bool isVOP2(uint16_t Opcode) const {545 return get(Opcode).TSFlags & SIInstrFlags::VOP2;546 }547 548 static bool isVOP3(const MCInstrDesc &Desc) {549 return Desc.TSFlags & SIInstrFlags::VOP3;550 }551 552 static bool isVOP3(const MachineInstr &MI) { return isVOP3(MI.getDesc()); }553 554 bool isVOP3(uint16_t Opcode) const { return isVOP3(get(Opcode)); }555 556 static bool isSDWA(const MachineInstr &MI) {557 return MI.getDesc().TSFlags & SIInstrFlags::SDWA;558 }559 560 bool isSDWA(uint16_t Opcode) const {561 return get(Opcode).TSFlags & SIInstrFlags::SDWA;562 }563 564 static bool isVOPC(const MachineInstr &MI) {565 return MI.getDesc().TSFlags & SIInstrFlags::VOPC;566 }567 568 bool isVOPC(uint16_t Opcode) const {569 return get(Opcode).TSFlags & SIInstrFlags::VOPC;570 }571 572 static bool isMUBUF(const MachineInstr &MI) {573 return MI.getDesc().TSFlags & SIInstrFlags::MUBUF;574 }575 576 bool isMUBUF(uint16_t Opcode) const {577 return get(Opcode).TSFlags & SIInstrFlags::MUBUF;578 }579 580 static bool isMTBUF(const MachineInstr &MI) {581 return MI.getDesc().TSFlags & SIInstrFlags::MTBUF;582 }583 584 bool isMTBUF(uint16_t Opcode) const {585 return get(Opcode).TSFlags & SIInstrFlags::MTBUF;586 }587 588 static bool isBUF(const MachineInstr &MI) {589 return isMUBUF(MI) || isMTBUF(MI);590 }591 592 static bool isSMRD(const MachineInstr &MI) {593 return MI.getDesc().TSFlags & SIInstrFlags::SMRD;594 }595 596 bool isSMRD(uint16_t Opcode) const {597 return get(Opcode).TSFlags & SIInstrFlags::SMRD;598 }599 600 bool isBufferSMRD(const MachineInstr &MI) const;601 602 static bool isDS(const MachineInstr &MI) {603 return MI.getDesc().TSFlags & SIInstrFlags::DS;604 }605 606 bool isDS(uint16_t Opcode) const {607 return get(Opcode).TSFlags & SIInstrFlags::DS;608 }609 610 static bool isLDSDMA(const MachineInstr &MI) {611 return isVALU(MI) && (isMUBUF(MI) || isFLAT(MI));612 }613 614 bool isLDSDMA(uint16_t Opcode) {615 return isVALU(Opcode) && (isMUBUF(Opcode) || isFLAT(Opcode));616 }617 618 static bool isGWS(const MachineInstr &MI) {619 return MI.getDesc().TSFlags & SIInstrFlags::GWS;620 }621 622 bool isGWS(uint16_t Opcode) const {623 return get(Opcode).TSFlags & SIInstrFlags::GWS;624 }625 626 bool isAlwaysGDS(uint16_t Opcode) const;627 628 static bool isMIMG(const MachineInstr &MI) {629 return MI.getDesc().TSFlags & SIInstrFlags::MIMG;630 }631 632 bool isMIMG(uint16_t Opcode) const {633 return get(Opcode).TSFlags & SIInstrFlags::MIMG;634 }635 636 static bool isVIMAGE(const MachineInstr &MI) {637 return MI.getDesc().TSFlags & SIInstrFlags::VIMAGE;638 }639 640 bool isVIMAGE(uint16_t Opcode) const {641 return get(Opcode).TSFlags & SIInstrFlags::VIMAGE;642 }643 644 static bool isVSAMPLE(const MachineInstr &MI) {645 return MI.getDesc().TSFlags & SIInstrFlags::VSAMPLE;646 }647 648 bool isVSAMPLE(uint16_t Opcode) const {649 return get(Opcode).TSFlags & SIInstrFlags::VSAMPLE;650 }651 652 static bool isGather4(const MachineInstr &MI) {653 return MI.getDesc().TSFlags & SIInstrFlags::Gather4;654 }655 656 bool isGather4(uint16_t Opcode) const {657 return get(Opcode).TSFlags & SIInstrFlags::Gather4;658 }659 660 static bool isFLAT(const MachineInstr &MI) {661 return MI.getDesc().TSFlags & SIInstrFlags::FLAT;662 }663 664 // Is a FLAT encoded instruction which accesses a specific segment,665 // i.e. global_* or scratch_*.666 static bool isSegmentSpecificFLAT(const MachineInstr &MI) {667 auto Flags = MI.getDesc().TSFlags;668 return Flags & (SIInstrFlags::FlatGlobal | SIInstrFlags::FlatScratch);669 }670 671 bool isSegmentSpecificFLAT(uint16_t Opcode) const {672 auto Flags = get(Opcode).TSFlags;673 return Flags & (SIInstrFlags::FlatGlobal | SIInstrFlags::FlatScratch);674 }675 676 static bool isFLATGlobal(const MachineInstr &MI) {677 return MI.getDesc().TSFlags & SIInstrFlags::FlatGlobal;678 }679 680 bool isFLATGlobal(uint16_t Opcode) const {681 return get(Opcode).TSFlags & SIInstrFlags::FlatGlobal;682 }683 684 static bool isFLATScratch(const MachineInstr &MI) {685 return MI.getDesc().TSFlags & SIInstrFlags::FlatScratch;686 }687 688 bool isFLATScratch(uint16_t Opcode) const {689 return get(Opcode).TSFlags & SIInstrFlags::FlatScratch;690 }691 692 // Any FLAT encoded instruction, including global_* and scratch_*.693 bool isFLAT(uint16_t Opcode) const {694 return get(Opcode).TSFlags & SIInstrFlags::FLAT;695 }696 697 /// \returns true for SCRATCH_ instructions, or FLAT_ instructions with698 /// SCRATCH_ memory operands.699 /// Conservatively correct; will return true if \p MI cannot be proven700 /// to not hit scratch.701 bool mayAccessScratchThroughFlat(const MachineInstr &MI) const;702 703 /// \returns true for FLAT instructions that can access VMEM.704 bool mayAccessVMEMThroughFlat(const MachineInstr &MI) const;705 706 /// \returns true for FLAT instructions that can access LDS.707 bool mayAccessLDSThroughFlat(const MachineInstr &MI) const;708 709 static bool isBlockLoadStore(uint16_t Opcode) {710 switch (Opcode) {711 case AMDGPU::SI_BLOCK_SPILL_V1024_SAVE:712 case AMDGPU::SI_BLOCK_SPILL_V1024_RESTORE:713 case AMDGPU::SCRATCH_STORE_BLOCK_SADDR:714 case AMDGPU::SCRATCH_LOAD_BLOCK_SADDR:715 case AMDGPU::SCRATCH_STORE_BLOCK_SVS:716 case AMDGPU::SCRATCH_LOAD_BLOCK_SVS:717 return true;718 default:719 return false;720 }721 }722 723 static bool setsSCCifResultIsNonZero(const MachineInstr &MI) {724 switch (MI.getOpcode()) {725 case AMDGPU::S_ABSDIFF_I32:726 case AMDGPU::S_ABS_I32:727 case AMDGPU::S_AND_B32:728 case AMDGPU::S_AND_B64:729 case AMDGPU::S_ANDN2_B32:730 case AMDGPU::S_ANDN2_B64:731 case AMDGPU::S_ASHR_I32:732 case AMDGPU::S_ASHR_I64:733 case AMDGPU::S_BCNT0_I32_B32:734 case AMDGPU::S_BCNT0_I32_B64:735 case AMDGPU::S_BCNT1_I32_B32:736 case AMDGPU::S_BCNT1_I32_B64:737 case AMDGPU::S_BFE_I32:738 case AMDGPU::S_BFE_I64:739 case AMDGPU::S_BFE_U32:740 case AMDGPU::S_BFE_U64:741 case AMDGPU::S_LSHL_B32:742 case AMDGPU::S_LSHL_B64:743 case AMDGPU::S_LSHR_B32:744 case AMDGPU::S_LSHR_B64:745 case AMDGPU::S_NAND_B32:746 case AMDGPU::S_NAND_B64:747 case AMDGPU::S_NOR_B32:748 case AMDGPU::S_NOR_B64:749 case AMDGPU::S_NOT_B32:750 case AMDGPU::S_NOT_B64:751 case AMDGPU::S_OR_B32:752 case AMDGPU::S_OR_B64:753 case AMDGPU::S_ORN2_B32:754 case AMDGPU::S_ORN2_B64:755 case AMDGPU::S_QUADMASK_B32:756 case AMDGPU::S_QUADMASK_B64:757 case AMDGPU::S_WQM_B32:758 case AMDGPU::S_WQM_B64:759 case AMDGPU::S_XNOR_B32:760 case AMDGPU::S_XNOR_B64:761 case AMDGPU::S_XOR_B32:762 case AMDGPU::S_XOR_B64:763 return true;764 default:765 return false;766 }767 }768 769 static bool isEXP(const MachineInstr &MI) {770 return MI.getDesc().TSFlags & SIInstrFlags::EXP;771 }772 773 static bool isDualSourceBlendEXP(const MachineInstr &MI) {774 if (!isEXP(MI))775 return false;776 unsigned Target = MI.getOperand(0).getImm();777 return Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND0 ||778 Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND1;779 }780 781 bool isEXP(uint16_t Opcode) const {782 return get(Opcode).TSFlags & SIInstrFlags::EXP;783 }784 785 static bool isAtomicNoRet(const MachineInstr &MI) {786 return MI.getDesc().TSFlags & SIInstrFlags::IsAtomicNoRet;787 }788 789 bool isAtomicNoRet(uint16_t Opcode) const {790 return get(Opcode).TSFlags & SIInstrFlags::IsAtomicNoRet;791 }792 793 static bool isAtomicRet(const MachineInstr &MI) {794 return MI.getDesc().TSFlags & SIInstrFlags::IsAtomicRet;795 }796 797 bool isAtomicRet(uint16_t Opcode) const {798 return get(Opcode).TSFlags & SIInstrFlags::IsAtomicRet;799 }800 801 static bool isAtomic(const MachineInstr &MI) {802 return MI.getDesc().TSFlags & (SIInstrFlags::IsAtomicRet |803 SIInstrFlags::IsAtomicNoRet);804 }805 806 bool isAtomic(uint16_t Opcode) const {807 return get(Opcode).TSFlags & (SIInstrFlags::IsAtomicRet |808 SIInstrFlags::IsAtomicNoRet);809 }810 811 static bool mayWriteLDSThroughDMA(const MachineInstr &MI) {812 return isLDSDMA(MI) && MI.getOpcode() != AMDGPU::BUFFER_STORE_LDS_DWORD;813 }814 815 static bool isSBarrierSCCWrite(unsigned Opcode) {816 return Opcode == AMDGPU::S_BARRIER_LEAVE ||817 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM ||818 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0;819 }820 821 static bool isCBranchVCCZRead(const MachineInstr &MI) {822 unsigned Opc = MI.getOpcode();823 return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) &&824 !MI.getOperand(1).isUndef();825 }826 827 static bool isWQM(const MachineInstr &MI) {828 return MI.getDesc().TSFlags & SIInstrFlags::WQM;829 }830 831 bool isWQM(uint16_t Opcode) const {832 return get(Opcode).TSFlags & SIInstrFlags::WQM;833 }834 835 static bool isDisableWQM(const MachineInstr &MI) {836 return MI.getDesc().TSFlags & SIInstrFlags::DisableWQM;837 }838 839 bool isDisableWQM(uint16_t Opcode) const {840 return get(Opcode).TSFlags & SIInstrFlags::DisableWQM;841 }842 843 // SI_SPILL_S32_TO_VGPR and SI_RESTORE_S32_FROM_VGPR form a special case of844 // SGPRs spilling to VGPRs which are SGPR spills but from VALU instructions845 // therefore we need an explicit check for them since just checking if the846 // Spill bit is set and what instruction type it came from misclassifies847 // them.848 static bool isVGPRSpill(const MachineInstr &MI) {849 return MI.getOpcode() != AMDGPU::SI_SPILL_S32_TO_VGPR &&850 MI.getOpcode() != AMDGPU::SI_RESTORE_S32_FROM_VGPR &&851 (isSpill(MI) && isVALU(MI));852 }853 854 bool isVGPRSpill(uint16_t Opcode) const {855 return Opcode != AMDGPU::SI_SPILL_S32_TO_VGPR &&856 Opcode != AMDGPU::SI_RESTORE_S32_FROM_VGPR &&857 (isSpill(Opcode) && isVALU(Opcode));858 }859 860 static bool isSGPRSpill(const MachineInstr &MI) {861 return MI.getOpcode() == AMDGPU::SI_SPILL_S32_TO_VGPR ||862 MI.getOpcode() == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||863 (isSpill(MI) && isSALU(MI));864 }865 866 bool isSGPRSpill(uint16_t Opcode) const {867 return Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR ||868 Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||869 (isSpill(Opcode) && isSALU(Opcode));870 }871 872 bool isSpill(uint16_t Opcode) const {873 return get(Opcode).TSFlags & SIInstrFlags::Spill;874 }875 876 static bool isSpill(const MCInstrDesc &Desc) {877 return Desc.TSFlags & SIInstrFlags::Spill;878 }879 880 static bool isSpill(const MachineInstr &MI) { return isSpill(MI.getDesc()); }881 882 static bool isWWMRegSpillOpcode(uint16_t Opcode) {883 return Opcode == AMDGPU::SI_SPILL_WWM_V32_SAVE ||884 Opcode == AMDGPU::SI_SPILL_WWM_AV32_SAVE ||885 Opcode == AMDGPU::SI_SPILL_WWM_V32_RESTORE ||886 Opcode == AMDGPU::SI_SPILL_WWM_AV32_RESTORE;887 }888 889 static bool isChainCallOpcode(uint64_t Opcode) {890 return Opcode == AMDGPU::SI_CS_CHAIN_TC_W32 ||891 Opcode == AMDGPU::SI_CS_CHAIN_TC_W64;892 }893 894 static bool isDPP(const MachineInstr &MI) {895 return MI.getDesc().TSFlags & SIInstrFlags::DPP;896 }897 898 bool isDPP(uint16_t Opcode) const {899 return get(Opcode).TSFlags & SIInstrFlags::DPP;900 }901 902 static bool isTRANS(const MachineInstr &MI) {903 return MI.getDesc().TSFlags & SIInstrFlags::TRANS;904 }905 906 bool isTRANS(uint16_t Opcode) const {907 return get(Opcode).TSFlags & SIInstrFlags::TRANS;908 }909 910 static bool isVOP3P(const MachineInstr &MI) {911 return MI.getDesc().TSFlags & SIInstrFlags::VOP3P;912 }913 914 bool isVOP3P(uint16_t Opcode) const {915 return get(Opcode).TSFlags & SIInstrFlags::VOP3P;916 }917 918 static bool isVINTRP(const MachineInstr &MI) {919 return MI.getDesc().TSFlags & SIInstrFlags::VINTRP;920 }921 922 bool isVINTRP(uint16_t Opcode) const {923 return get(Opcode).TSFlags & SIInstrFlags::VINTRP;924 }925 926 static bool isMAI(const MCInstrDesc &Desc) {927 return Desc.TSFlags & SIInstrFlags::IsMAI;928 }929 930 static bool isMAI(const MachineInstr &MI) { return isMAI(MI.getDesc()); }931 932 bool isMAI(uint16_t Opcode) const { return isMAI(get(Opcode)); }933 934 static bool isMFMA(const MachineInstr &MI) {935 return isMAI(MI) && MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&936 MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64;937 }938 939 bool isMFMA(uint16_t Opcode) const {940 return isMAI(Opcode) && Opcode != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&941 Opcode != AMDGPU::V_ACCVGPR_READ_B32_e64;942 }943 944 static bool isDOT(const MachineInstr &MI) {945 return MI.getDesc().TSFlags & SIInstrFlags::IsDOT;946 }947 948 static bool isWMMA(const MachineInstr &MI) {949 return MI.getDesc().TSFlags & SIInstrFlags::IsWMMA;950 }951 952 bool isWMMA(uint16_t Opcode) const {953 return get(Opcode).TSFlags & SIInstrFlags::IsWMMA;954 }955 956 static bool isMFMAorWMMA(const MachineInstr &MI) {957 return isMFMA(MI) || isWMMA(MI) || isSWMMAC(MI);958 }959 960 bool isMFMAorWMMA(uint16_t Opcode) const {961 return isMFMA(Opcode) || isWMMA(Opcode) || isSWMMAC(Opcode);962 }963 964 static bool isSWMMAC(const MachineInstr &MI) {965 return MI.getDesc().TSFlags & SIInstrFlags::IsSWMMAC;966 }967 968 bool isSWMMAC(uint16_t Opcode) const {969 return get(Opcode).TSFlags & SIInstrFlags::IsSWMMAC;970 }971 972 bool isDOT(uint16_t Opcode) const {973 return get(Opcode).TSFlags & SIInstrFlags::IsDOT;974 }975 976 bool isXDLWMMA(const MachineInstr &MI) const;977 978 bool isXDL(const MachineInstr &MI) const;979 980 static bool isDGEMM(unsigned Opcode) { return AMDGPU::getMAIIsDGEMM(Opcode); }981 982 static bool isLDSDIR(const MachineInstr &MI) {983 return MI.getDesc().TSFlags & SIInstrFlags::LDSDIR;984 }985 986 bool isLDSDIR(uint16_t Opcode) const {987 return get(Opcode).TSFlags & SIInstrFlags::LDSDIR;988 }989 990 static bool isVINTERP(const MachineInstr &MI) {991 return MI.getDesc().TSFlags & SIInstrFlags::VINTERP;992 }993 994 bool isVINTERP(uint16_t Opcode) const {995 return get(Opcode).TSFlags & SIInstrFlags::VINTERP;996 }997 998 static bool isScalarUnit(const MachineInstr &MI) {999 return MI.getDesc().TSFlags & (SIInstrFlags::SALU | SIInstrFlags::SMRD);1000 }1001 1002 static bool usesVM_CNT(const MachineInstr &MI) {1003 return MI.getDesc().TSFlags & SIInstrFlags::VM_CNT;1004 }1005 1006 static bool usesLGKM_CNT(const MachineInstr &MI) {1007 return MI.getDesc().TSFlags & SIInstrFlags::LGKM_CNT;1008 }1009 1010 // Most sopk treat the immediate as a signed 16-bit, however some1011 // use it as unsigned.1012 static bool sopkIsZext(unsigned Opcode) {1013 return Opcode == AMDGPU::S_CMPK_EQ_U32 || Opcode == AMDGPU::S_CMPK_LG_U32 ||1014 Opcode == AMDGPU::S_CMPK_GT_U32 || Opcode == AMDGPU::S_CMPK_GE_U32 ||1015 Opcode == AMDGPU::S_CMPK_LT_U32 || Opcode == AMDGPU::S_CMPK_LE_U32 ||1016 Opcode == AMDGPU::S_GETREG_B32 ||1017 Opcode == AMDGPU::S_GETREG_B32_const;1018 }1019 1020 /// \returns true if this is an s_store_dword* instruction. This is more1021 /// specific than isSMEM && mayStore.1022 static bool isScalarStore(const MachineInstr &MI) {1023 return MI.getDesc().TSFlags & SIInstrFlags::SCALAR_STORE;1024 }1025 1026 bool isScalarStore(uint16_t Opcode) const {1027 return get(Opcode).TSFlags & SIInstrFlags::SCALAR_STORE;1028 }1029 1030 static bool isFixedSize(const MachineInstr &MI) {1031 return MI.getDesc().TSFlags & SIInstrFlags::FIXED_SIZE;1032 }1033 1034 bool isFixedSize(uint16_t Opcode) const {1035 return get(Opcode).TSFlags & SIInstrFlags::FIXED_SIZE;1036 }1037 1038 static bool hasFPClamp(const MachineInstr &MI) {1039 return MI.getDesc().TSFlags & SIInstrFlags::FPClamp;1040 }1041 1042 bool hasFPClamp(uint16_t Opcode) const {1043 return get(Opcode).TSFlags & SIInstrFlags::FPClamp;1044 }1045 1046 static bool hasIntClamp(const MachineInstr &MI) {1047 return MI.getDesc().TSFlags & SIInstrFlags::IntClamp;1048 }1049 1050 uint64_t getClampMask(const MachineInstr &MI) const {1051 const uint64_t ClampFlags = SIInstrFlags::FPClamp |1052 SIInstrFlags::IntClamp |1053 SIInstrFlags::ClampLo |1054 SIInstrFlags::ClampHi;1055 return MI.getDesc().TSFlags & ClampFlags;1056 }1057 1058 static bool usesFPDPRounding(const MachineInstr &MI) {1059 return MI.getDesc().TSFlags & SIInstrFlags::FPDPRounding;1060 }1061 1062 bool usesFPDPRounding(uint16_t Opcode) const {1063 return get(Opcode).TSFlags & SIInstrFlags::FPDPRounding;1064 }1065 1066 static bool isFPAtomic(const MachineInstr &MI) {1067 return MI.getDesc().TSFlags & SIInstrFlags::FPAtomic;1068 }1069 1070 bool isFPAtomic(uint16_t Opcode) const {1071 return get(Opcode).TSFlags & SIInstrFlags::FPAtomic;1072 }1073 1074 static bool isNeverUniform(const MachineInstr &MI) {1075 return MI.getDesc().TSFlags & SIInstrFlags::IsNeverUniform;1076 }1077 1078 // Check to see if opcode is for a barrier start. Pre gfx12 this is just the1079 // S_BARRIER, but after support for S_BARRIER_SIGNAL* / S_BARRIER_WAIT we want1080 // to check for the barrier start (S_BARRIER_SIGNAL*)1081 bool isBarrierStart(unsigned Opcode) const {1082 return Opcode == AMDGPU::S_BARRIER ||1083 Opcode == AMDGPU::S_BARRIER_SIGNAL_M0 ||1084 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0 ||1085 Opcode == AMDGPU::S_BARRIER_SIGNAL_IMM ||1086 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM;1087 }1088 1089 bool isBarrier(unsigned Opcode) const {1090 return isBarrierStart(Opcode) || Opcode == AMDGPU::S_BARRIER_WAIT ||1091 Opcode == AMDGPU::S_BARRIER_INIT_M0 ||1092 Opcode == AMDGPU::S_BARRIER_INIT_IMM ||1093 Opcode == AMDGPU::S_BARRIER_JOIN_IMM ||1094 Opcode == AMDGPU::S_BARRIER_LEAVE || Opcode == AMDGPU::DS_GWS_INIT ||1095 Opcode == AMDGPU::DS_GWS_BARRIER;1096 }1097 1098 static bool isGFX12CacheInvOrWBInst(unsigned Opc) {1099 return Opc == AMDGPU::GLOBAL_INV || Opc == AMDGPU::GLOBAL_WB ||1100 Opc == AMDGPU::GLOBAL_WBINV;1101 }1102 1103 static bool isF16PseudoScalarTrans(unsigned Opcode) {1104 return Opcode == AMDGPU::V_S_EXP_F16_e64 ||1105 Opcode == AMDGPU::V_S_LOG_F16_e64 ||1106 Opcode == AMDGPU::V_S_RCP_F16_e64 ||1107 Opcode == AMDGPU::V_S_RSQ_F16_e64 ||1108 Opcode == AMDGPU::V_S_SQRT_F16_e64;1109 }1110 1111 static bool doesNotReadTiedSource(const MachineInstr &MI) {1112 return MI.getDesc().TSFlags & SIInstrFlags::TiedSourceNotRead;1113 }1114 1115 bool doesNotReadTiedSource(uint16_t Opcode) const {1116 return get(Opcode).TSFlags & SIInstrFlags::TiedSourceNotRead;1117 }1118 1119 bool isIGLP(unsigned Opcode) const {1120 return Opcode == AMDGPU::SCHED_BARRIER ||1121 Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT;1122 }1123 1124 bool isIGLP(const MachineInstr &MI) const { return isIGLP(MI.getOpcode()); }1125 1126 // Return true if the instruction is mutually exclusive with all non-IGLP DAG1127 // mutations, requiring all other mutations to be disabled.1128 bool isIGLPMutationOnly(unsigned Opcode) const {1129 return Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT;1130 }1131 1132 static unsigned getNonSoftWaitcntOpcode(unsigned Opcode) {1133 switch (Opcode) {1134 case AMDGPU::S_WAITCNT_soft:1135 return AMDGPU::S_WAITCNT;1136 case AMDGPU::S_WAITCNT_VSCNT_soft:1137 return AMDGPU::S_WAITCNT_VSCNT;1138 case AMDGPU::S_WAIT_LOADCNT_soft:1139 return AMDGPU::S_WAIT_LOADCNT;1140 case AMDGPU::S_WAIT_STORECNT_soft:1141 return AMDGPU::S_WAIT_STORECNT;1142 case AMDGPU::S_WAIT_SAMPLECNT_soft:1143 return AMDGPU::S_WAIT_SAMPLECNT;1144 case AMDGPU::S_WAIT_BVHCNT_soft:1145 return AMDGPU::S_WAIT_BVHCNT;1146 case AMDGPU::S_WAIT_DSCNT_soft:1147 return AMDGPU::S_WAIT_DSCNT;1148 case AMDGPU::S_WAIT_KMCNT_soft:1149 return AMDGPU::S_WAIT_KMCNT;1150 case AMDGPU::S_WAIT_XCNT_soft:1151 return AMDGPU::S_WAIT_XCNT;1152 default:1153 return Opcode;1154 }1155 }1156 1157 static bool isWaitcnt(unsigned Opcode) {1158 switch (getNonSoftWaitcntOpcode(Opcode)) {1159 case AMDGPU::S_WAITCNT:1160 case AMDGPU::S_WAITCNT_VSCNT:1161 case AMDGPU::S_WAITCNT_VMCNT:1162 case AMDGPU::S_WAITCNT_EXPCNT:1163 case AMDGPU::S_WAITCNT_LGKMCNT:1164 case AMDGPU::S_WAIT_LOADCNT:1165 case AMDGPU::S_WAIT_LOADCNT_DSCNT:1166 case AMDGPU::S_WAIT_STORECNT:1167 case AMDGPU::S_WAIT_STORECNT_DSCNT:1168 case AMDGPU::S_WAIT_SAMPLECNT:1169 case AMDGPU::S_WAIT_BVHCNT:1170 case AMDGPU::S_WAIT_EXPCNT:1171 case AMDGPU::S_WAIT_DSCNT:1172 case AMDGPU::S_WAIT_KMCNT:1173 case AMDGPU::S_WAIT_IDLE:1174 return true;1175 default:1176 return false;1177 }1178 }1179 1180 bool isVGPRCopy(const MachineInstr &MI) const {1181 assert(isCopyInstr(MI));1182 Register Dest = MI.getOperand(0).getReg();1183 const MachineFunction &MF = *MI.getMF();1184 const MachineRegisterInfo &MRI = MF.getRegInfo();1185 return !RI.isSGPRReg(MRI, Dest);1186 }1187 1188 bool hasVGPRUses(const MachineInstr &MI) const {1189 const MachineFunction &MF = *MI.getMF();1190 const MachineRegisterInfo &MRI = MF.getRegInfo();1191 return llvm::any_of(MI.explicit_uses(),1192 [&MRI, this](const MachineOperand &MO) {1193 return MO.isReg() && RI.isVGPR(MRI, MO.getReg());});1194 }1195 1196 /// Return true if the instruction modifies the mode register.q1197 static bool modifiesModeRegister(const MachineInstr &MI);1198 1199 /// This function is used to determine if an instruction can be safely1200 /// executed under EXEC = 0 without hardware error, indeterminate results,1201 /// and/or visible effects on future vector execution or outside the shader.1202 /// Note: as of 2024 the only use of this is SIPreEmitPeephole where it is1203 /// used in removing branches over short EXEC = 0 sequences.1204 /// As such it embeds certain assumptions which may not apply to every case1205 /// of EXEC = 0 execution.1206 bool hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const;1207 1208 /// Returns true if the instruction could potentially depend on the value of1209 /// exec. If false, exec dependencies may safely be ignored.1210 bool mayReadEXEC(const MachineRegisterInfo &MRI, const MachineInstr &MI) const;1211 1212 bool isInlineConstant(const APInt &Imm) const;1213 1214 bool isInlineConstant(const APFloat &Imm) const;1215 1216 // Returns true if this non-register operand definitely does not need to be1217 // encoded as a 32-bit literal. Note that this function handles all kinds of1218 // operands, not just immediates.1219 //1220 // Some operands like FrameIndexes could resolve to an inline immediate value1221 // that will not require an additional 4-bytes; this function assumes that it1222 // will.1223 bool isInlineConstant(const MachineOperand &MO, uint8_t OperandType) const {1224 if (!MO.isImm())1225 return false;1226 return isInlineConstant(MO.getImm(), OperandType);1227 }1228 bool isInlineConstant(int64_t ImmVal, uint8_t OperandType) const;1229 1230 bool isInlineConstant(const MachineOperand &MO,1231 const MCOperandInfo &OpInfo) const {1232 return isInlineConstant(MO, OpInfo.OperandType);1233 }1234 1235 /// \p returns true if \p UseMO is substituted with \p DefMO in \p MI it would1236 /// be an inline immediate.1237 bool isInlineConstant(const MachineInstr &MI,1238 const MachineOperand &UseMO,1239 const MachineOperand &DefMO) const {1240 assert(UseMO.getParent() == &MI);1241 int OpIdx = UseMO.getOperandNo();1242 if (OpIdx >= MI.getDesc().NumOperands)1243 return false;1244 1245 return isInlineConstant(DefMO, MI.getDesc().operands()[OpIdx]);1246 }1247 1248 /// \p returns true if the operand \p OpIdx in \p MI is a valid inline1249 /// immediate.1250 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx) const {1251 const MachineOperand &MO = MI.getOperand(OpIdx);1252 return isInlineConstant(MO, MI.getDesc().operands()[OpIdx].OperandType);1253 }1254 1255 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx,1256 int64_t ImmVal) const {1257 if (OpIdx >= MI.getDesc().NumOperands)1258 return false;1259 1260 if (isCopyInstr(MI)) {1261 unsigned Size = getOpSize(MI, OpIdx);1262 assert(Size == 8 || Size == 4);1263 1264 uint8_t OpType = (Size == 8) ?1265 AMDGPU::OPERAND_REG_IMM_INT64 : AMDGPU::OPERAND_REG_IMM_INT32;1266 return isInlineConstant(ImmVal, OpType);1267 }1268 1269 return isInlineConstant(ImmVal, MI.getDesc().operands()[OpIdx].OperandType);1270 }1271 1272 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx,1273 const MachineOperand &MO) const {1274 return isInlineConstant(MI, OpIdx, MO.getImm());1275 }1276 1277 bool isInlineConstant(const MachineOperand &MO) const {1278 return isInlineConstant(*MO.getParent(), MO.getOperandNo());1279 }1280 1281 bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,1282 const MachineOperand &MO) const;1283 1284 bool isLiteralOperandLegal(const MCInstrDesc &InstDesc,1285 const MCOperandInfo &OpInfo) const;1286 1287 bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,1288 int64_t ImmVal) const;1289 1290 bool isImmOperandLegal(const MachineInstr &MI, unsigned OpNo,1291 const MachineOperand &MO) const {1292 return isImmOperandLegal(MI.getDesc(), OpNo, MO);1293 }1294 1295 bool isNeverCoissue(MachineInstr &MI) const;1296 1297 /// Check if this immediate value can be used for AV_MOV_B64_IMM_PSEUDO.1298 bool isLegalAV64PseudoImm(uint64_t Imm) const;1299 1300 /// Return true if this 64-bit VALU instruction has a 32-bit encoding.1301 /// This function will return false if you pass it a 32-bit instruction.1302 bool hasVALU32BitEncoding(unsigned Opcode) const;1303 1304 bool physRegUsesConstantBus(const MachineOperand &Reg) const;1305 bool regUsesConstantBus(const MachineOperand &Reg,1306 const MachineRegisterInfo &MRI) const;1307 1308 /// Returns true if this operand uses the constant bus.1309 bool usesConstantBus(const MachineRegisterInfo &MRI,1310 const MachineOperand &MO,1311 const MCOperandInfo &OpInfo) const;1312 1313 bool usesConstantBus(const MachineRegisterInfo &MRI, const MachineInstr &MI,1314 int OpIdx) const {1315 return usesConstantBus(MRI, MI.getOperand(OpIdx),1316 MI.getDesc().operands()[OpIdx]);1317 }1318 1319 /// Return true if this instruction has any modifiers.1320 /// e.g. src[012]_mod, omod, clamp.1321 bool hasModifiers(unsigned Opcode) const;1322 1323 bool hasModifiersSet(const MachineInstr &MI, AMDGPU::OpName OpName) const;1324 bool hasAnyModifiersSet(const MachineInstr &MI) const;1325 1326 bool canShrink(const MachineInstr &MI,1327 const MachineRegisterInfo &MRI) const;1328 1329 MachineInstr *buildShrunkInst(MachineInstr &MI,1330 unsigned NewOpcode) const;1331 1332 bool verifyInstruction(const MachineInstr &MI,1333 StringRef &ErrInfo) const override;1334 1335 unsigned getVALUOp(const MachineInstr &MI) const;1336 1337 void insertScratchExecCopy(MachineFunction &MF, MachineBasicBlock &MBB,1338 MachineBasicBlock::iterator MBBI,1339 const DebugLoc &DL, Register Reg, bool IsSCCLive,1340 SlotIndexes *Indexes = nullptr) const;1341 1342 void restoreExec(MachineFunction &MF, MachineBasicBlock &MBB,1343 MachineBasicBlock::iterator MBBI, const DebugLoc &DL,1344 Register Reg, SlotIndexes *Indexes = nullptr) const;1345 1346 MachineInstr *getWholeWaveFunctionSetup(MachineFunction &MF) const;1347 1348 /// Return the correct register class for \p OpNo. For target-specific1349 /// instructions, this will return the register class that has been defined1350 /// in tablegen. For generic instructions, like REG_SEQUENCE it will return1351 /// the register class of its machine operand.1352 /// to infer the correct register class base on the other operands.1353 const TargetRegisterClass *getOpRegClass(const MachineInstr &MI,1354 unsigned OpNo) const;1355 1356 /// Return the size in bytes of the operand OpNo on the given1357 // instruction opcode.1358 unsigned getOpSize(uint16_t Opcode, unsigned OpNo) const {1359 const MCOperandInfo &OpInfo = get(Opcode).operands()[OpNo];1360 1361 if (OpInfo.RegClass == -1) {1362 // If this is an immediate operand, this must be a 32-bit literal.1363 assert(OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE);1364 return 4;1365 }1366 1367 return RI.getRegSizeInBits(*RI.getRegClass(getOpRegClassID(OpInfo))) / 8;1368 }1369 1370 /// This form should usually be preferred since it handles operands1371 /// with unknown register classes.1372 unsigned getOpSize(const MachineInstr &MI, unsigned OpNo) const {1373 const MachineOperand &MO = MI.getOperand(OpNo);1374 if (MO.isReg()) {1375 if (unsigned SubReg = MO.getSubReg()) {1376 return RI.getSubRegIdxSize(SubReg) / 8;1377 }1378 }1379 return RI.getRegSizeInBits(*getOpRegClass(MI, OpNo)) / 8;1380 }1381 1382 /// Legalize the \p OpIndex operand of this instruction by inserting1383 /// a MOV. For example:1384 /// ADD_I32_e32 VGPR0, 151385 /// to1386 /// MOV VGPR1, 151387 /// ADD_I32_e32 VGPR0, VGPR11388 ///1389 /// If the operand being legalized is a register, then a COPY will be used1390 /// instead of MOV.1391 void legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const;1392 1393 /// Check if \p MO is a legal operand if it was the \p OpIdx Operand1394 /// for \p MI.1395 bool isOperandLegal(const MachineInstr &MI, unsigned OpIdx,1396 const MachineOperand *MO = nullptr) const;1397 1398 /// Check if \p MO would be a valid operand for the given operand1399 /// definition \p OpInfo. Note this does not attempt to validate constant bus1400 /// restrictions (e.g. literal constant usage).1401 bool isLegalVSrcOperand(const MachineRegisterInfo &MRI,1402 const MCOperandInfo &OpInfo,1403 const MachineOperand &MO) const;1404 1405 /// Check if \p MO (a register operand) is a legal register for the1406 /// given operand description or operand index.1407 /// The operand index version provide more legality checks1408 bool isLegalRegOperand(const MachineRegisterInfo &MRI,1409 const MCOperandInfo &OpInfo,1410 const MachineOperand &MO) const;1411 bool isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx,1412 const MachineOperand &MO) const;1413 1414 /// Check if \p MO would be a legal operand for gfx12+ packed math FP321415 /// instructions. Packed math FP32 instructions typically accept SGPRs or1416 /// VGPRs as source operands. On gfx12+, if a source operand uses SGPRs, the1417 /// HW can only read the first SGPR and use it for both the low and high1418 /// operations.1419 /// \p SrcN can be 0, 1, or 2, representing src0, src1, and src2,1420 /// respectively. If \p MO is nullptr, the operand corresponding to SrcN will1421 /// be used.1422 bool isLegalGFX12PlusPackedMathFP32Operand(1423 const MachineRegisterInfo &MRI, const MachineInstr &MI, unsigned SrcN,1424 const MachineOperand *MO = nullptr) const;1425 1426 /// Legalize operands in \p MI by either commuting it or inserting a1427 /// copy of src1.1428 void legalizeOperandsVOP2(MachineRegisterInfo &MRI, MachineInstr &MI) const;1429 1430 /// Fix operands in \p MI to satisfy constant bus requirements.1431 void legalizeOperandsVOP3(MachineRegisterInfo &MRI, MachineInstr &MI) const;1432 1433 /// Copy a value from a VGPR (\p SrcReg) to SGPR. The desired register class1434 /// for the dst register (\p DstRC) can be optionally supplied. This function1435 /// can only be used when it is know that the value in SrcReg is same across1436 /// all threads in the wave.1437 /// \returns The SGPR register that \p SrcReg was copied to.1438 Register readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI,1439 MachineRegisterInfo &MRI,1440 const TargetRegisterClass *DstRC = nullptr) const;1441 1442 void legalizeOperandsSMRD(MachineRegisterInfo &MRI, MachineInstr &MI) const;1443 void legalizeOperandsFLAT(MachineRegisterInfo &MRI, MachineInstr &MI) const;1444 1445 void legalizeGenericOperand(MachineBasicBlock &InsertMBB,1446 MachineBasicBlock::iterator I,1447 const TargetRegisterClass *DstRC,1448 MachineOperand &Op, MachineRegisterInfo &MRI,1449 const DebugLoc &DL) const;1450 1451 /// Legalize all operands in this instruction. This function may create new1452 /// instructions and control-flow around \p MI. If present, \p MDT is1453 /// updated.1454 /// \returns A new basic block that contains \p MI if new blocks were created.1455 MachineBasicBlock *1456 legalizeOperands(MachineInstr &MI, MachineDominatorTree *MDT = nullptr) const;1457 1458 /// Change SADDR form of a FLAT \p Inst to its VADDR form if saddr operand1459 /// was moved to VGPR. \returns true if succeeded.1460 bool moveFlatAddrToVGPR(MachineInstr &Inst) const;1461 1462 /// Fix operands in Inst to fix 16bit SALU to VALU lowering.1463 void legalizeOperandsVALUt16(MachineInstr &Inst,1464 MachineRegisterInfo &MRI) const;1465 void legalizeOperandsVALUt16(MachineInstr &Inst, unsigned OpIdx,1466 MachineRegisterInfo &MRI) const;1467 1468 /// Replace the instructions opcode with the equivalent VALU1469 /// opcode. This function will also move the users of MachineInstruntions1470 /// in the \p WorkList to the VALU if necessary. If present, \p MDT is1471 /// updated.1472 void moveToVALU(SIInstrWorklist &Worklist, MachineDominatorTree *MDT) const;1473 1474 void moveToVALUImpl(SIInstrWorklist &Worklist, MachineDominatorTree *MDT,1475 MachineInstr &Inst) const;1476 1477 void insertNoop(MachineBasicBlock &MBB,1478 MachineBasicBlock::iterator MI) const override;1479 1480 void insertNoops(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,1481 unsigned Quantity) const override;1482 1483 void insertReturn(MachineBasicBlock &MBB) const;1484 1485 /// Build instructions that simulate the behavior of a `s_trap 2` instructions1486 /// for hardware (namely, gfx11) that runs in PRIV=1 mode. There, s_trap is1487 /// interpreted as a nop.1488 MachineBasicBlock *insertSimulatedTrap(MachineRegisterInfo &MRI,1489 MachineBasicBlock &MBB,1490 MachineInstr &MI,1491 const DebugLoc &DL) const;1492 1493 /// Return the number of wait states that result from executing this1494 /// instruction.1495 static unsigned getNumWaitStates(const MachineInstr &MI);1496 1497 /// Returns the operand named \p Op. If \p MI does not have an1498 /// operand named \c Op, this function returns nullptr.1499 LLVM_READONLY1500 MachineOperand *getNamedOperand(MachineInstr &MI,1501 AMDGPU::OpName OperandName) const;1502 1503 LLVM_READONLY1504 const MachineOperand *getNamedOperand(const MachineInstr &MI,1505 AMDGPU::OpName OperandName) const {1506 return getNamedOperand(const_cast<MachineInstr &>(MI), OperandName);1507 }1508 1509 /// Get required immediate operand1510 int64_t getNamedImmOperand(const MachineInstr &MI,1511 AMDGPU::OpName OperandName) const {1512 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName);1513 return MI.getOperand(Idx).getImm();1514 }1515 1516 uint64_t getDefaultRsrcDataFormat() const;1517 uint64_t getScratchRsrcWords23() const;1518 1519 bool isLowLatencyInstruction(const MachineInstr &MI) const;1520 bool isHighLatencyDef(int Opc) const override;1521 1522 /// Return the descriptor of the target-specific machine instruction1523 /// that corresponds to the specified pseudo or native opcode.1524 const MCInstrDesc &getMCOpcodeFromPseudo(unsigned Opcode) const {1525 return get(pseudoToMCOpcode(Opcode));1526 }1527 1528 Register isStackAccess(const MachineInstr &MI, int &FrameIndex) const;1529 Register isSGPRStackAccess(const MachineInstr &MI, int &FrameIndex) const;1530 1531 Register isLoadFromStackSlot(const MachineInstr &MI,1532 int &FrameIndex) const override;1533 Register isStoreToStackSlot(const MachineInstr &MI,1534 int &FrameIndex) const override;1535 1536 unsigned getInstBundleSize(const MachineInstr &MI) const;1537 unsigned getInstSizeInBytes(const MachineInstr &MI) const override;1538 1539 bool mayAccessFlatAddressSpace(const MachineInstr &MI) const;1540 1541 std::pair<unsigned, unsigned>1542 decomposeMachineOperandsTargetFlags(unsigned TF) const override;1543 1544 ArrayRef<std::pair<int, const char *>>1545 getSerializableTargetIndices() const override;1546 1547 ArrayRef<std::pair<unsigned, const char *>>1548 getSerializableDirectMachineOperandTargetFlags() const override;1549 1550 ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>1551 getSerializableMachineMemOperandTargetFlags() const override;1552 1553 ScheduleHazardRecognizer *1554 CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,1555 const ScheduleDAG *DAG) const override;1556 1557 ScheduleHazardRecognizer *1558 CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const override;1559 1560 ScheduleHazardRecognizer *1561 CreateTargetMIHazardRecognizer(const InstrItineraryData *II,1562 const ScheduleDAGMI *DAG) const override;1563 1564 unsigned getLiveRangeSplitOpcode(Register Reg,1565 const MachineFunction &MF) const override;1566 1567 bool isBasicBlockPrologue(const MachineInstr &MI,1568 Register Reg = Register()) const override;1569 1570 MachineInstr *createPHIDestinationCopy(MachineBasicBlock &MBB,1571 MachineBasicBlock::iterator InsPt,1572 const DebugLoc &DL, Register Src,1573 Register Dst) const override;1574 1575 MachineInstr *createPHISourceCopy(MachineBasicBlock &MBB,1576 MachineBasicBlock::iterator InsPt,1577 const DebugLoc &DL, Register Src,1578 unsigned SrcSubReg,1579 Register Dst) const override;1580 1581 bool isWave32() const;1582 1583 /// Return a partially built integer add instruction without carry.1584 /// Caller must add source operands.1585 /// For pre-GFX9 it will generate unused carry destination operand.1586 /// TODO: After GFX9 it should return a no-carry operation.1587 MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,1588 MachineBasicBlock::iterator I,1589 const DebugLoc &DL,1590 Register DestReg) const;1591 1592 MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,1593 MachineBasicBlock::iterator I,1594 const DebugLoc &DL,1595 Register DestReg,1596 RegScavenger &RS) const;1597 1598 static bool isKillTerminator(unsigned Opcode);1599 const MCInstrDesc &getKillTerminatorFromPseudo(unsigned Opcode) const;1600 1601 bool isLegalMUBUFImmOffset(unsigned Imm) const;1602 1603 static unsigned getMaxMUBUFImmOffset(const GCNSubtarget &ST);1604 1605 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset,1606 Align Alignment = Align(4)) const;1607 1608 /// Returns if \p Offset is legal for the subtarget as the offset to a FLAT1609 /// encoded instruction with the given \p FlatVariant.1610 bool isLegalFLATOffset(int64_t Offset, unsigned AddrSpace,1611 uint64_t FlatVariant) const;1612 1613 /// Split \p COffsetVal into {immediate offset field, remainder offset}1614 /// values.1615 std::pair<int64_t, int64_t> splitFlatOffset(int64_t COffsetVal,1616 unsigned AddrSpace,1617 uint64_t FlatVariant) const;1618 1619 /// Returns true if negative offsets are allowed for the given \p FlatVariant.1620 bool allowNegativeFlatOffset(uint64_t FlatVariant) const;1621 1622 /// \brief Return a target-specific opcode if Opcode is a pseudo instruction.1623 /// Return -1 if the target-specific opcode for the pseudo instruction does1624 /// not exist. If Opcode is not a pseudo instruction, this is identity.1625 int pseudoToMCOpcode(int Opcode) const;1626 1627 /// \brief Check if this instruction should only be used by assembler.1628 /// Return true if this opcode should not be used by codegen.1629 bool isAsmOnlyOpcode(int MCOp) const;1630 1631 void fixImplicitOperands(MachineInstr &MI) const;1632 1633 MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,1634 ArrayRef<unsigned> Ops,1635 MachineBasicBlock::iterator InsertPt,1636 int FrameIndex,1637 LiveIntervals *LIS = nullptr,1638 VirtRegMap *VRM = nullptr) const override;1639 1640 unsigned getInstrLatency(const InstrItineraryData *ItinData,1641 const MachineInstr &MI,1642 unsigned *PredCost = nullptr) const override;1643 1644 InstructionUniformity1645 getInstructionUniformity(const MachineInstr &MI) const final;1646 1647 InstructionUniformity1648 getGenericInstructionUniformity(const MachineInstr &MI) const;1649 1650 const MIRFormatter *getMIRFormatter() const override {1651 if (!Formatter)1652 Formatter = std::make_unique<AMDGPUMIRFormatter>();1653 return Formatter.get();1654 }1655 1656 static unsigned getDSShaderTypeValue(const MachineFunction &MF);1657 1658 const TargetSchedModel &getSchedModel() const { return SchedModel; }1659 1660 // FIXME: This should be removed1661 // Enforce operand's \p OpName even alignment if required by target.1662 // This is used if an operand is a 32 bit register but needs to be aligned1663 // regardless.1664 void enforceOperandRCAlignment(MachineInstr &MI, AMDGPU::OpName OpName) const;1665};1666 1667/// \brief Returns true if a reg:subreg pair P has a TRC class1668inline bool isOfRegClass(const TargetInstrInfo::RegSubRegPair &P,1669 const TargetRegisterClass &TRC,1670 MachineRegisterInfo &MRI) {1671 auto *RC = MRI.getRegClass(P.Reg);1672 if (!P.SubReg)1673 return RC == &TRC;1674 auto *TRI = MRI.getTargetRegisterInfo();1675 return RC == TRI->getMatchingSuperRegClass(RC, &TRC, P.SubReg);1676}1677 1678/// \brief Create RegSubRegPair from a register MachineOperand1679inline1680TargetInstrInfo::RegSubRegPair getRegSubRegPair(const MachineOperand &O) {1681 assert(O.isReg());1682 return TargetInstrInfo::RegSubRegPair(O.getReg(), O.getSubReg());1683}1684 1685/// \brief Return the SubReg component from REG_SEQUENCE1686TargetInstrInfo::RegSubRegPair getRegSequenceSubReg(MachineInstr &MI,1687 unsigned SubReg);1688 1689/// \brief Return the defining instruction for a given reg:subreg pair1690/// skipping copy like instructions and subreg-manipulation pseudos.1691/// Following another subreg of a reg:subreg isn't supported.1692MachineInstr *getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,1693 const MachineRegisterInfo &MRI);1694 1695/// \brief Return false if EXEC is not changed between the def of \p VReg at \p1696/// DefMI and the use at \p UseMI. Should be run on SSA. Currently does not1697/// attempt to track between blocks.1698bool execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI,1699 Register VReg,1700 const MachineInstr &DefMI,1701 const MachineInstr &UseMI);1702 1703/// \brief Return false if EXEC is not changed between the def of \p VReg at \p1704/// DefMI and all its uses. Should be run on SSA. Currently does not attempt to1705/// track between blocks.1706bool execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI,1707 Register VReg,1708 const MachineInstr &DefMI);1709 1710namespace AMDGPU {1711 1712 LLVM_READONLY1713 int getVOPe64(uint16_t Opcode);1714 1715 LLVM_READONLY1716 int getVOPe32(uint16_t Opcode);1717 1718 LLVM_READONLY1719 int getSDWAOp(uint16_t Opcode);1720 1721 LLVM_READONLY1722 int getDPPOp32(uint16_t Opcode);1723 1724 LLVM_READONLY1725 int getDPPOp64(uint16_t Opcode);1726 1727 LLVM_READONLY1728 int getBasicFromSDWAOp(uint16_t Opcode);1729 1730 LLVM_READONLY1731 int getCommuteRev(uint16_t Opcode);1732 1733 LLVM_READONLY1734 int getCommuteOrig(uint16_t Opcode);1735 1736 LLVM_READONLY1737 int getAddr64Inst(uint16_t Opcode);1738 1739 /// Check if \p Opcode is an Addr64 opcode.1740 ///1741 /// \returns \p Opcode if it is an Addr64 opcode, otherwise -1.1742 LLVM_READONLY1743 int getIfAddr64Inst(uint16_t Opcode);1744 1745 LLVM_READONLY1746 int getSOPKOp(uint16_t Opcode);1747 1748 /// \returns SADDR form of a FLAT Global instruction given an \p Opcode1749 /// of a VADDR form.1750 LLVM_READONLY1751 int getGlobalSaddrOp(uint16_t Opcode);1752 1753 /// \returns VADDR form of a FLAT Global instruction given an \p Opcode1754 /// of a SADDR form.1755 LLVM_READONLY1756 int getGlobalVaddrOp(uint16_t Opcode);1757 1758 LLVM_READONLY1759 int getVCMPXNoSDstOp(uint16_t Opcode);1760 1761 /// \returns ST form with only immediate offset of a FLAT Scratch instruction1762 /// given an \p Opcode of an SS (SADDR) form.1763 LLVM_READONLY1764 int getFlatScratchInstSTfromSS(uint16_t Opcode);1765 1766 /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode1767 /// of an SVS (SADDR + VADDR) form.1768 LLVM_READONLY1769 int getFlatScratchInstSVfromSVS(uint16_t Opcode);1770 1771 /// \returns SS (SADDR) form of a FLAT Scratch instruction given an \p Opcode1772 /// of an SV (VADDR) form.1773 LLVM_READONLY1774 int getFlatScratchInstSSfromSV(uint16_t Opcode);1775 1776 /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode1777 /// of an SS (SADDR) form.1778 LLVM_READONLY1779 int getFlatScratchInstSVfromSS(uint16_t Opcode);1780 1781 /// \returns earlyclobber version of a MAC MFMA is exists.1782 LLVM_READONLY1783 int getMFMAEarlyClobberOp(uint16_t Opcode);1784 1785 /// \returns Version of an MFMA instruction which uses AGPRs for srcC and1786 /// vdst, given an \p Opcode of an MFMA which uses VGPRs for srcC/vdst.1787 LLVM_READONLY1788 int getMFMASrcCVDstAGPROp(uint16_t Opcode);1789 1790 /// \returns v_cmpx version of a v_cmp instruction.1791 LLVM_READONLY1792 int getVCMPXOpFromVCMP(uint16_t Opcode);1793 1794 const uint64_t RSRC_DATA_FORMAT = 0xf00000000000LL;1795 const uint64_t RSRC_ELEMENT_SIZE_SHIFT = (32 + 19);1796 const uint64_t RSRC_INDEX_STRIDE_SHIFT = (32 + 21);1797 const uint64_t RSRC_TID_ENABLE = UINT64_C(1) << (32 + 23);1798 1799} // end namespace AMDGPU1800 1801namespace AMDGPU {1802enum AsmComments {1803 // For sgpr to vgpr spill instructions1804 SGPR_SPILL = MachineInstr::TAsmComments1805};1806} // namespace AMDGPU1807 1808namespace SI {1809namespace KernelInputOffsets {1810 1811/// Offsets in bytes from the start of the input buffer1812enum Offsets {1813 NGROUPS_X = 0,1814 NGROUPS_Y = 4,1815 NGROUPS_Z = 8,1816 GLOBAL_SIZE_X = 12,1817 GLOBAL_SIZE_Y = 16,1818 GLOBAL_SIZE_Z = 20,1819 LOCAL_SIZE_X = 24,1820 LOCAL_SIZE_Y = 28,1821 LOCAL_SIZE_Z = 321822};1823 1824} // end namespace KernelInputOffsets1825} // end namespace SI1826 1827} // end namespace llvm1828 1829#endif // LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H1830