11046 lines · cpp
1//===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9/// \file10/// SI Implementation of TargetInstrInfo.11//12//===----------------------------------------------------------------------===//13 14#include "SIInstrInfo.h"15#include "AMDGPU.h"16#include "AMDGPUInstrInfo.h"17#include "AMDGPULaneMaskUtils.h"18#include "GCNHazardRecognizer.h"19#include "GCNSubtarget.h"20#include "SIMachineFunctionInfo.h"21#include "Utils/AMDGPUBaseInfo.h"22#include "llvm/ADT/STLExtras.h"23#include "llvm/Analysis/ValueTracking.h"24#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"25#include "llvm/CodeGen/LiveIntervals.h"26#include "llvm/CodeGen/LiveVariables.h"27#include "llvm/CodeGen/MachineDominators.h"28#include "llvm/CodeGen/MachineFrameInfo.h"29#include "llvm/CodeGen/MachineScheduler.h"30#include "llvm/CodeGen/RegisterScavenging.h"31#include "llvm/CodeGen/ScheduleDAG.h"32#include "llvm/IR/DiagnosticInfo.h"33#include "llvm/IR/IntrinsicsAMDGPU.h"34#include "llvm/MC/MCContext.h"35#include "llvm/Support/CommandLine.h"36#include "llvm/Target/TargetMachine.h"37 38using namespace llvm;39 40#define DEBUG_TYPE "si-instr-info"41 42#define GET_INSTRINFO_CTOR_DTOR43#include "AMDGPUGenInstrInfo.inc"44 45namespace llvm::AMDGPU {46#define GET_D16ImageDimIntrinsics_IMPL47#define GET_ImageDimIntrinsicTable_IMPL48#define GET_RsrcIntrinsics_IMPL49#include "AMDGPUGenSearchableTables.inc"50} // namespace llvm::AMDGPU51 52// Must be at least 4 to be able to branch over minimum unconditional branch53// code. This is only for making it possible to write reasonably small tests for54// long branches.55static cl::opt<unsigned>56BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16),57 cl::desc("Restrict range of branch instructions (DEBUG)"));58 59static cl::opt<bool> Fix16BitCopies(60 "amdgpu-fix-16-bit-physreg-copies",61 cl::desc("Fix copies between 32 and 16 bit registers by extending to 32 bit"),62 cl::init(true),63 cl::ReallyHidden);64 65SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST)66 : AMDGPUGenInstrInfo(ST, RI, AMDGPU::ADJCALLSTACKUP,67 AMDGPU::ADJCALLSTACKDOWN),68 RI(ST), ST(ST) {69 SchedModel.init(&ST);70}71 72//===----------------------------------------------------------------------===//73// TargetInstrInfo callbacks74//===----------------------------------------------------------------------===//75 76static unsigned getNumOperandsNoGlue(SDNode *Node) {77 unsigned N = Node->getNumOperands();78 while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue)79 --N;80 return N;81}82 83/// Returns true if both nodes have the same value for the given84/// operand \p Op, or if both nodes do not have this operand.85static bool nodesHaveSameOperandValue(SDNode *N0, SDNode *N1,86 AMDGPU::OpName OpName) {87 unsigned Opc0 = N0->getMachineOpcode();88 unsigned Opc1 = N1->getMachineOpcode();89 90 int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName);91 int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName);92 93 if (Op0Idx == -1 && Op1Idx == -1)94 return true;95 96 97 if ((Op0Idx == -1 && Op1Idx != -1) ||98 (Op1Idx == -1 && Op0Idx != -1))99 return false;100 101 // getNamedOperandIdx returns the index for the MachineInstr's operands,102 // which includes the result as the first operand. We are indexing into the103 // MachineSDNode's operands, so we need to skip the result operand to get104 // the real index.105 --Op0Idx;106 --Op1Idx;107 108 return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx);109}110 111static bool canRemat(const MachineInstr &MI) {112 113 if (SIInstrInfo::isVOP1(MI) || SIInstrInfo::isVOP2(MI) ||114 SIInstrInfo::isVOP3(MI) || SIInstrInfo::isSDWA(MI) ||115 SIInstrInfo::isSALU(MI))116 return true;117 118 if (SIInstrInfo::isSMRD(MI)) {119 return !MI.memoperands_empty() &&120 llvm::all_of(MI.memoperands(), [](const MachineMemOperand *MMO) {121 return MMO->isLoad() && MMO->isInvariant();122 });123 }124 125 return false;126}127 128bool SIInstrInfo::isReMaterializableImpl(129 const MachineInstr &MI) const {130 131 if (canRemat(MI)) {132 // Normally VALU use of exec would block the rematerialization, but that133 // is OK in this case to have an implicit exec read as all VALU do.134 // We really want all of the generic logic for this except for this.135 136 // Another potential implicit use is mode register. The core logic of137 // the RA will not attempt rematerialization if mode is set anywhere138 // in the function, otherwise it is safe since mode is not changed.139 140 // There is difference to generic method which does not allow141 // rematerialization if there are virtual register uses. We allow this,142 // therefore this method includes SOP instructions as well.143 if (!MI.hasImplicitDef() &&144 MI.getNumImplicitOperands() == MI.getDesc().implicit_uses().size() &&145 !MI.mayRaiseFPException())146 return true;147 }148 149 return TargetInstrInfo::isReMaterializableImpl(MI);150}151 152// Returns true if the scalar result of a VALU instruction depends on exec.153bool SIInstrInfo::resultDependsOnExec(const MachineInstr &MI) const {154 // Ignore comparisons which are only used masked with exec.155 // This allows some hoisting/sinking of VALU comparisons.156 if (MI.isCompare()) {157 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::sdst);158 if (!Dst)159 return true;160 161 Register DstReg = Dst->getReg();162 if (!DstReg.isVirtual())163 return true;164 165 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();166 for (MachineInstr &Use : MRI.use_nodbg_instructions(DstReg)) {167 switch (Use.getOpcode()) {168 case AMDGPU::S_AND_SAVEEXEC_B32:169 case AMDGPU::S_AND_SAVEEXEC_B64:170 break;171 case AMDGPU::S_AND_B32:172 case AMDGPU::S_AND_B64:173 if (!Use.readsRegister(AMDGPU::EXEC, /*TRI=*/nullptr))174 return true;175 break;176 default:177 return true;178 }179 }180 return false;181 }182 183 switch (MI.getOpcode()) {184 default:185 break;186 case AMDGPU::V_READFIRSTLANE_B32:187 return true;188 }189 190 return false;191}192 193bool SIInstrInfo::isIgnorableUse(const MachineOperand &MO) const {194 // Any implicit use of exec by VALU is not a real register read.195 return MO.getReg() == AMDGPU::EXEC && MO.isImplicit() &&196 isVALU(*MO.getParent()) && !resultDependsOnExec(*MO.getParent());197}198 199bool SIInstrInfo::isSafeToSink(MachineInstr &MI,200 MachineBasicBlock *SuccToSinkTo,201 MachineCycleInfo *CI) const {202 // Allow sinking if MI edits lane mask (divergent i1 in sgpr).203 if (MI.getOpcode() == AMDGPU::SI_IF_BREAK)204 return true;205 206 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();207 // Check if sinking of MI would create temporal divergent use.208 for (auto Op : MI.uses()) {209 if (Op.isReg() && Op.getReg().isVirtual() &&210 RI.isSGPRClass(MRI.getRegClass(Op.getReg()))) {211 MachineInstr *SgprDef = MRI.getVRegDef(Op.getReg());212 213 // SgprDef defined inside cycle214 MachineCycle *FromCycle = CI->getCycle(SgprDef->getParent());215 if (FromCycle == nullptr)216 continue;217 218 MachineCycle *ToCycle = CI->getCycle(SuccToSinkTo);219 // Check if there is a FromCycle that contains SgprDef's basic block but220 // does not contain SuccToSinkTo and also has divergent exit condition.221 while (FromCycle && !FromCycle->contains(ToCycle)) {222 SmallVector<MachineBasicBlock *, 1> ExitingBlocks;223 FromCycle->getExitingBlocks(ExitingBlocks);224 225 // FromCycle has divergent exit condition.226 for (MachineBasicBlock *ExitingBlock : ExitingBlocks) {227 if (hasDivergentBranch(ExitingBlock))228 return false;229 }230 231 FromCycle = FromCycle->getParentCycle();232 }233 }234 }235 236 return true;237}238 239bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1,240 int64_t &Offset0,241 int64_t &Offset1) const {242 if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())243 return false;244 245 unsigned Opc0 = Load0->getMachineOpcode();246 unsigned Opc1 = Load1->getMachineOpcode();247 248 // Make sure both are actually loads.249 if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad())250 return false;251 252 // A mayLoad instruction without a def is not a load. Likely a prefetch.253 if (!get(Opc0).getNumDefs() || !get(Opc1).getNumDefs())254 return false;255 256 if (isDS(Opc0) && isDS(Opc1)) {257 258 // FIXME: Handle this case:259 if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1))260 return false;261 262 // Check base reg.263 if (Load0->getOperand(0) != Load1->getOperand(0))264 return false;265 266 // Skip read2 / write2 variants for simplicity.267 // TODO: We should report true if the used offsets are adjacent (excluded268 // st64 versions).269 int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);270 int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);271 if (Offset0Idx == -1 || Offset1Idx == -1)272 return false;273 274 // XXX - be careful of dataless loads275 // getNamedOperandIdx returns the index for MachineInstrs. Since they276 // include the output in the operand list, but SDNodes don't, we need to277 // subtract the index by one.278 Offset0Idx -= get(Opc0).NumDefs;279 Offset1Idx -= get(Opc1).NumDefs;280 Offset0 = Load0->getConstantOperandVal(Offset0Idx);281 Offset1 = Load1->getConstantOperandVal(Offset1Idx);282 return true;283 }284 285 if (isSMRD(Opc0) && isSMRD(Opc1)) {286 // Skip time and cache invalidation instructions.287 if (!AMDGPU::hasNamedOperand(Opc0, AMDGPU::OpName::sbase) ||288 !AMDGPU::hasNamedOperand(Opc1, AMDGPU::OpName::sbase))289 return false;290 291 unsigned NumOps = getNumOperandsNoGlue(Load0);292 if (NumOps != getNumOperandsNoGlue(Load1))293 return false;294 295 // Check base reg.296 if (Load0->getOperand(0) != Load1->getOperand(0))297 return false;298 299 // Match register offsets, if both register and immediate offsets present.300 assert(NumOps == 4 || NumOps == 5);301 if (NumOps == 5 && Load0->getOperand(1) != Load1->getOperand(1))302 return false;303 304 const ConstantSDNode *Load0Offset =305 dyn_cast<ConstantSDNode>(Load0->getOperand(NumOps - 3));306 const ConstantSDNode *Load1Offset =307 dyn_cast<ConstantSDNode>(Load1->getOperand(NumOps - 3));308 309 if (!Load0Offset || !Load1Offset)310 return false;311 312 Offset0 = Load0Offset->getZExtValue();313 Offset1 = Load1Offset->getZExtValue();314 return true;315 }316 317 // MUBUF and MTBUF can access the same addresses.318 if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) {319 320 // MUBUF and MTBUF have vaddr at different indices.321 if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) ||322 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) ||323 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc))324 return false;325 326 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);327 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);328 329 if (OffIdx0 == -1 || OffIdx1 == -1)330 return false;331 332 // getNamedOperandIdx returns the index for MachineInstrs. Since they333 // include the output in the operand list, but SDNodes don't, we need to334 // subtract the index by one.335 OffIdx0 -= get(Opc0).NumDefs;336 OffIdx1 -= get(Opc1).NumDefs;337 338 SDValue Off0 = Load0->getOperand(OffIdx0);339 SDValue Off1 = Load1->getOperand(OffIdx1);340 341 // The offset might be a FrameIndexSDNode.342 if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1))343 return false;344 345 Offset0 = Off0->getAsZExtVal();346 Offset1 = Off1->getAsZExtVal();347 return true;348 }349 350 return false;351}352 353static bool isStride64(unsigned Opc) {354 switch (Opc) {355 case AMDGPU::DS_READ2ST64_B32:356 case AMDGPU::DS_READ2ST64_B64:357 case AMDGPU::DS_WRITE2ST64_B32:358 case AMDGPU::DS_WRITE2ST64_B64:359 return true;360 default:361 return false;362 }363}364 365bool SIInstrInfo::getMemOperandsWithOffsetWidth(366 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,367 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,368 const TargetRegisterInfo *TRI) const {369 if (!LdSt.mayLoadOrStore())370 return false;371 372 unsigned Opc = LdSt.getOpcode();373 OffsetIsScalable = false;374 const MachineOperand *BaseOp, *OffsetOp;375 int DataOpIdx;376 377 if (isDS(LdSt)) {378 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);379 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset);380 if (OffsetOp) {381 // Normal, single offset LDS instruction.382 if (!BaseOp) {383 // DS_CONSUME/DS_APPEND use M0 for the base address.384 // TODO: find the implicit use operand for M0 and use that as BaseOp?385 return false;386 }387 BaseOps.push_back(BaseOp);388 Offset = OffsetOp->getImm();389 // Get appropriate operand, and compute width accordingly.390 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);391 if (DataOpIdx == -1)392 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);393 if (Opc == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)394 Width = LocationSize::precise(64);395 else396 Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));397 } else {398 // The 2 offset instructions use offset0 and offset1 instead. We can treat399 // these as a load with a single offset if the 2 offsets are consecutive.400 // We will use this for some partially aligned loads.401 const MachineOperand *Offset0Op =402 getNamedOperand(LdSt, AMDGPU::OpName::offset0);403 const MachineOperand *Offset1Op =404 getNamedOperand(LdSt, AMDGPU::OpName::offset1);405 406 unsigned Offset0 = Offset0Op->getImm() & 0xff;407 unsigned Offset1 = Offset1Op->getImm() & 0xff;408 if (Offset0 + 1 != Offset1)409 return false;410 411 // Each of these offsets is in element sized units, so we need to convert412 // to bytes of the individual reads.413 414 unsigned EltSize;415 if (LdSt.mayLoad())416 EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16;417 else {418 assert(LdSt.mayStore());419 int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);420 EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8;421 }422 423 if (isStride64(Opc))424 EltSize *= 64;425 426 BaseOps.push_back(BaseOp);427 Offset = EltSize * Offset0;428 // Get appropriate operand(s), and compute width accordingly.429 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);430 if (DataOpIdx == -1) {431 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);432 Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));433 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1);434 Width = LocationSize::precise(435 Width.getValue() + TypeSize::getFixed(getOpSize(LdSt, DataOpIdx)));436 } else {437 Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));438 }439 }440 return true;441 }442 443 if (isMUBUF(LdSt) || isMTBUF(LdSt)) {444 const MachineOperand *RSrc = getNamedOperand(LdSt, AMDGPU::OpName::srsrc);445 if (!RSrc) // e.g. BUFFER_WBINVL1_VOL446 return false;447 BaseOps.push_back(RSrc);448 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);449 if (BaseOp && !BaseOp->isFI())450 BaseOps.push_back(BaseOp);451 const MachineOperand *OffsetImm =452 getNamedOperand(LdSt, AMDGPU::OpName::offset);453 Offset = OffsetImm->getImm();454 const MachineOperand *SOffset =455 getNamedOperand(LdSt, AMDGPU::OpName::soffset);456 if (SOffset) {457 if (SOffset->isReg())458 BaseOps.push_back(SOffset);459 else460 Offset += SOffset->getImm();461 }462 // Get appropriate operand, and compute width accordingly.463 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);464 if (DataOpIdx == -1)465 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);466 if (DataOpIdx == -1) // LDS DMA467 return false;468 Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));469 return true;470 }471 472 if (isImage(LdSt)) {473 auto RsrcOpName =474 isMIMG(LdSt) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;475 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opc, RsrcOpName);476 BaseOps.push_back(&LdSt.getOperand(SRsrcIdx));477 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0);478 if (VAddr0Idx >= 0) {479 // GFX10 possible NSA encoding.480 for (int I = VAddr0Idx; I < SRsrcIdx; ++I)481 BaseOps.push_back(&LdSt.getOperand(I));482 } else {483 BaseOps.push_back(getNamedOperand(LdSt, AMDGPU::OpName::vaddr));484 }485 Offset = 0;486 // Get appropriate operand, and compute width accordingly.487 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);488 if (DataOpIdx == -1)489 return false; // no return sampler490 Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));491 return true;492 }493 494 if (isSMRD(LdSt)) {495 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::sbase);496 if (!BaseOp) // e.g. S_MEMTIME497 return false;498 BaseOps.push_back(BaseOp);499 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset);500 Offset = OffsetOp ? OffsetOp->getImm() : 0;501 // Get appropriate operand, and compute width accordingly.502 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sdst);503 if (DataOpIdx == -1)504 return false;505 Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));506 return true;507 }508 509 if (isFLAT(LdSt)) {510 // Instructions have either vaddr or saddr or both or none.511 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);512 if (BaseOp)513 BaseOps.push_back(BaseOp);514 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr);515 if (BaseOp)516 BaseOps.push_back(BaseOp);517 Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm();518 // Get appropriate operand, and compute width accordingly.519 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);520 if (DataOpIdx == -1)521 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);522 if (DataOpIdx == -1) // LDS DMA523 return false;524 Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));525 return true;526 }527 528 return false;529}530 531static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,532 ArrayRef<const MachineOperand *> BaseOps1,533 const MachineInstr &MI2,534 ArrayRef<const MachineOperand *> BaseOps2) {535 // Only examine the first "base" operand of each instruction, on the536 // assumption that it represents the real base address of the memory access.537 // Other operands are typically offsets or indices from this base address.538 if (BaseOps1.front()->isIdenticalTo(*BaseOps2.front()))539 return true;540 541 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())542 return false;543 544 auto *MO1 = *MI1.memoperands_begin();545 auto *MO2 = *MI2.memoperands_begin();546 if (MO1->getAddrSpace() != MO2->getAddrSpace())547 return false;548 549 const auto *Base1 = MO1->getValue();550 const auto *Base2 = MO2->getValue();551 if (!Base1 || !Base2)552 return false;553 Base1 = getUnderlyingObject(Base1);554 Base2 = getUnderlyingObject(Base2);555 556 if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2))557 return false;558 559 return Base1 == Base2;560}561 562bool SIInstrInfo::shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,563 int64_t Offset1, bool OffsetIsScalable1,564 ArrayRef<const MachineOperand *> BaseOps2,565 int64_t Offset2, bool OffsetIsScalable2,566 unsigned ClusterSize,567 unsigned NumBytes) const {568 // If the mem ops (to be clustered) do not have the same base ptr, then they569 // should not be clustered570 unsigned MaxMemoryClusterDWords = DefaultMemoryClusterDWordsLimit;571 if (!BaseOps1.empty() && !BaseOps2.empty()) {572 const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent();573 const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();574 if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOps1, SecondLdSt, BaseOps2))575 return false;576 577 const SIMachineFunctionInfo *MFI =578 FirstLdSt.getMF()->getInfo<SIMachineFunctionInfo>();579 MaxMemoryClusterDWords = MFI->getMaxMemoryClusterDWords();580 } else if (!BaseOps1.empty() || !BaseOps2.empty()) {581 // If only one base op is empty, they do not have the same base ptr582 return false;583 }584 585 // In order to avoid register pressure, on an average, the number of DWORDS586 // loaded together by all clustered mem ops should not exceed587 // MaxMemoryClusterDWords. This is an empirical value based on certain588 // observations and performance related experiments.589 // The good thing about this heuristic is - it avoids clustering of too many590 // sub-word loads, and also avoids clustering of wide loads. Below is the591 // brief summary of how the heuristic behaves for various `LoadSize` when592 // MaxMemoryClusterDWords is 8.593 //594 // (1) 1 <= LoadSize <= 4: cluster at max 8 mem ops595 // (2) 5 <= LoadSize <= 8: cluster at max 4 mem ops596 // (3) 9 <= LoadSize <= 12: cluster at max 2 mem ops597 // (4) 13 <= LoadSize <= 16: cluster at max 2 mem ops598 // (5) LoadSize >= 17: do not cluster599 const unsigned LoadSize = NumBytes / ClusterSize;600 const unsigned NumDWords = ((LoadSize + 3) / 4) * ClusterSize;601 return NumDWords <= MaxMemoryClusterDWords;602}603 604// FIXME: This behaves strangely. If, for example, you have 32 load + stores,605// the first 16 loads will be interleaved with the stores, and the next 16 will606// be clustered as expected. It should really split into 2 16 store batches.607//608// Loads are clustered until this returns false, rather than trying to schedule609// groups of stores. This also means we have to deal with saying different610// address space loads should be clustered, and ones which might cause bank611// conflicts.612//613// This might be deprecated so it might not be worth that much effort to fix.614bool SIInstrInfo::shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1,615 int64_t Offset0, int64_t Offset1,616 unsigned NumLoads) const {617 assert(Offset1 > Offset0 &&618 "Second offset should be larger than first offset!");619 // If we have less than 16 loads in a row, and the offsets are within 64620 // bytes, then schedule together.621 622 // A cacheline is 64 bytes (for global memory).623 return (NumLoads <= 16 && (Offset1 - Offset0) < 64);624}625 626static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB,627 MachineBasicBlock::iterator MI,628 const DebugLoc &DL, MCRegister DestReg,629 MCRegister SrcReg, bool KillSrc,630 const char *Msg = "illegal VGPR to SGPR copy") {631 MachineFunction *MF = MBB.getParent();632 633 LLVMContext &C = MF->getFunction().getContext();634 C.diagnose(DiagnosticInfoUnsupported(MF->getFunction(), Msg, DL, DS_Error));635 636 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg)637 .addReg(SrcReg, getKillRegState(KillSrc));638}639 640/// Handle copying from SGPR to AGPR, or from AGPR to AGPR on GFX908. It is not641/// possible to have a direct copy in these cases on GFX908, so an intermediate642/// VGPR copy is required.643static void indirectCopyToAGPR(const SIInstrInfo &TII,644 MachineBasicBlock &MBB,645 MachineBasicBlock::iterator MI,646 const DebugLoc &DL, MCRegister DestReg,647 MCRegister SrcReg, bool KillSrc,648 RegScavenger &RS, bool RegsOverlap,649 Register ImpDefSuperReg = Register(),650 Register ImpUseSuperReg = Register()) {651 assert((TII.getSubtarget().hasMAIInsts() &&652 !TII.getSubtarget().hasGFX90AInsts()) &&653 "Expected GFX908 subtarget.");654 655 assert((AMDGPU::SReg_32RegClass.contains(SrcReg) ||656 AMDGPU::AGPR_32RegClass.contains(SrcReg)) &&657 "Source register of the copy should be either an SGPR or an AGPR.");658 659 assert(AMDGPU::AGPR_32RegClass.contains(DestReg) &&660 "Destination register of the copy should be an AGPR.");661 662 const SIRegisterInfo &RI = TII.getRegisterInfo();663 664 // First try to find defining accvgpr_write to avoid temporary registers.665 // In the case of copies of overlapping AGPRs, we conservatively do not666 // reuse previous accvgpr_writes. Otherwise, we may incorrectly pick up667 // an accvgpr_write used for this same copy due to implicit-defs668 if (!RegsOverlap) {669 for (auto Def = MI, E = MBB.begin(); Def != E; ) {670 --Def;671 672 if (!Def->modifiesRegister(SrcReg, &RI))673 continue;674 675 if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||676 Def->getOperand(0).getReg() != SrcReg)677 break;678 679 MachineOperand &DefOp = Def->getOperand(1);680 assert(DefOp.isReg() || DefOp.isImm());681 682 if (DefOp.isReg()) {683 bool SafeToPropagate = true;684 // Check that register source operand is not clobbered before MI.685 // Immediate operands are always safe to propagate.686 for (auto I = Def; I != MI && SafeToPropagate; ++I)687 if (I->modifiesRegister(DefOp.getReg(), &RI))688 SafeToPropagate = false;689 690 if (!SafeToPropagate)691 break;692 693 for (auto I = Def; I != MI; ++I)694 I->clearRegisterKills(DefOp.getReg(), &RI);695 }696 697 MachineInstrBuilder Builder =698 BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)699 .add(DefOp);700 if (ImpDefSuperReg)701 Builder.addReg(ImpDefSuperReg, RegState::Define | RegState::Implicit);702 703 if (ImpUseSuperReg) {704 Builder.addReg(ImpUseSuperReg,705 getKillRegState(KillSrc) | RegState::Implicit);706 }707 708 return;709 }710 }711 712 RS.enterBasicBlockEnd(MBB);713 RS.backward(std::next(MI));714 715 // Ideally we want to have three registers for a long reg_sequence copy716 // to hide 2 waitstates between v_mov_b32 and accvgpr_write.717 unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass,718 *MBB.getParent());719 720 // Registers in the sequence are allocated contiguously so we can just721 // use register number to pick one of three round-robin temps.722 unsigned RegNo = (DestReg - AMDGPU::AGPR0) % 3;723 Register Tmp =724 MBB.getParent()->getInfo<SIMachineFunctionInfo>()->getVGPRForAGPRCopy();725 assert(MBB.getParent()->getRegInfo().isReserved(Tmp) &&726 "VGPR used for an intermediate copy should have been reserved.");727 728 // Only loop through if there are any free registers left. We don't want to729 // spill.730 while (RegNo--) {731 Register Tmp2 = RS.scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass, MI,732 /* RestoreAfter */ false, 0,733 /* AllowSpill */ false);734 if (!Tmp2 || RI.getHWRegIndex(Tmp2) >= MaxVGPRs)735 break;736 Tmp = Tmp2;737 RS.setRegUsed(Tmp);738 }739 740 // Insert copy to temporary VGPR.741 unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32;742 if (AMDGPU::AGPR_32RegClass.contains(SrcReg)) {743 TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32_e64;744 } else {745 assert(AMDGPU::SReg_32RegClass.contains(SrcReg));746 }747 748 MachineInstrBuilder UseBuilder = BuildMI(MBB, MI, DL, TII.get(TmpCopyOp), Tmp)749 .addReg(SrcReg, getKillRegState(KillSrc));750 if (ImpUseSuperReg) {751 UseBuilder.addReg(ImpUseSuperReg,752 getKillRegState(KillSrc) | RegState::Implicit);753 }754 755 MachineInstrBuilder DefBuilder756 = BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)757 .addReg(Tmp, RegState::Kill);758 759 if (ImpDefSuperReg)760 DefBuilder.addReg(ImpDefSuperReg, RegState::Define | RegState::Implicit);761}762 763static void expandSGPRCopy(const SIInstrInfo &TII, MachineBasicBlock &MBB,764 MachineBasicBlock::iterator MI, const DebugLoc &DL,765 MCRegister DestReg, MCRegister SrcReg, bool KillSrc,766 const TargetRegisterClass *RC, bool Forward) {767 const SIRegisterInfo &RI = TII.getRegisterInfo();768 ArrayRef<int16_t> BaseIndices = RI.getRegSplitParts(RC, 4);769 MachineBasicBlock::iterator I = MI;770 MachineInstr *FirstMI = nullptr, *LastMI = nullptr;771 772 for (unsigned Idx = 0; Idx < BaseIndices.size(); ++Idx) {773 int16_t SubIdx = BaseIndices[Idx];774 Register DestSubReg = RI.getSubReg(DestReg, SubIdx);775 Register SrcSubReg = RI.getSubReg(SrcReg, SubIdx);776 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");777 unsigned Opcode = AMDGPU::S_MOV_B32;778 779 // Is SGPR aligned? If so try to combine with next.780 bool AlignedDest = ((DestSubReg - AMDGPU::SGPR0) % 2) == 0;781 bool AlignedSrc = ((SrcSubReg - AMDGPU::SGPR0) % 2) == 0;782 if (AlignedDest && AlignedSrc && (Idx + 1 < BaseIndices.size())) {783 // Can use SGPR64 copy784 unsigned Channel = RI.getChannelFromSubReg(SubIdx);785 SubIdx = RI.getSubRegFromChannel(Channel, 2);786 DestSubReg = RI.getSubReg(DestReg, SubIdx);787 SrcSubReg = RI.getSubReg(SrcReg, SubIdx);788 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");789 Opcode = AMDGPU::S_MOV_B64;790 Idx++;791 }792 793 LastMI = BuildMI(MBB, I, DL, TII.get(Opcode), DestSubReg)794 .addReg(SrcSubReg)795 .addReg(SrcReg, RegState::Implicit);796 797 if (!FirstMI)798 FirstMI = LastMI;799 800 if (!Forward)801 I--;802 }803 804 assert(FirstMI && LastMI);805 if (!Forward)806 std::swap(FirstMI, LastMI);807 808 FirstMI->addOperand(809 MachineOperand::CreateReg(DestReg, true /*IsDef*/, true /*IsImp*/));810 811 if (KillSrc)812 LastMI->addRegisterKilled(SrcReg, &RI);813}814 815void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,816 MachineBasicBlock::iterator MI,817 const DebugLoc &DL, Register DestReg,818 Register SrcReg, bool KillSrc, bool RenamableDest,819 bool RenamableSrc) const {820 const TargetRegisterClass *RC = RI.getPhysRegBaseClass(DestReg);821 unsigned Size = RI.getRegSizeInBits(*RC);822 const TargetRegisterClass *SrcRC = RI.getPhysRegBaseClass(SrcReg);823 unsigned SrcSize = RI.getRegSizeInBits(*SrcRC);824 825 // The rest of copyPhysReg assumes Src and Dst size are the same size.826 // TODO-GFX11_16BIT If all true 16 bit instruction patterns are completed can827 // we remove Fix16BitCopies and this code block?828 if (Fix16BitCopies) {829 if (((Size == 16) != (SrcSize == 16))) {830 // Non-VGPR Src and Dst will later be expanded back to 32 bits.831 assert(ST.useRealTrue16Insts());832 Register &RegToFix = (Size == 32) ? DestReg : SrcReg;833 MCRegister SubReg = RI.getSubReg(RegToFix, AMDGPU::lo16);834 RegToFix = SubReg;835 836 if (DestReg == SrcReg) {837 // Identity copy. Insert empty bundle since ExpandPostRA expects an838 // instruction here.839 BuildMI(MBB, MI, DL, get(AMDGPU::BUNDLE));840 return;841 }842 RC = RI.getPhysRegBaseClass(DestReg);843 Size = RI.getRegSizeInBits(*RC);844 SrcRC = RI.getPhysRegBaseClass(SrcReg);845 SrcSize = RI.getRegSizeInBits(*SrcRC);846 }847 }848 849 if (RC == &AMDGPU::VGPR_32RegClass) {850 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||851 AMDGPU::SReg_32RegClass.contains(SrcReg) ||852 AMDGPU::AGPR_32RegClass.contains(SrcReg));853 unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ?854 AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32;855 BuildMI(MBB, MI, DL, get(Opc), DestReg)856 .addReg(SrcReg, getKillRegState(KillSrc));857 return;858 }859 860 if (RC == &AMDGPU::SReg_32_XM0RegClass ||861 RC == &AMDGPU::SReg_32RegClass) {862 if (SrcReg == AMDGPU::SCC) {863 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg)864 .addImm(1)865 .addImm(0);866 return;867 }868 869 if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) {870 if (DestReg == AMDGPU::VCC_LO) {871 // FIXME: Hack until VReg_1 removed.872 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));873 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))874 .addImm(0)875 .addReg(SrcReg, getKillRegState(KillSrc));876 return;877 }878 879 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);880 return;881 }882 883 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)884 .addReg(SrcReg, getKillRegState(KillSrc));885 return;886 }887 888 if (RC == &AMDGPU::SReg_64RegClass) {889 if (SrcReg == AMDGPU::SCC) {890 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B64), DestReg)891 .addImm(1)892 .addImm(0);893 return;894 }895 896 if (!AMDGPU::SReg_64_EncodableRegClass.contains(SrcReg)) {897 if (DestReg == AMDGPU::VCC) {898 // FIXME: Hack until VReg_1 removed.899 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));900 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))901 .addImm(0)902 .addReg(SrcReg, getKillRegState(KillSrc));903 return;904 }905 906 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);907 return;908 }909 910 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)911 .addReg(SrcReg, getKillRegState(KillSrc));912 return;913 }914 915 if (DestReg == AMDGPU::SCC) {916 // Copying 64-bit or 32-bit sources to SCC barely makes sense,917 // but SelectionDAG emits such copies for i1 sources.918 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {919 // This copy can only be produced by patterns920 // with explicit SCC, which are known to be enabled921 // only for subtargets with S_CMP_LG_U64 present.922 assert(ST.hasScalarCompareEq64());923 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U64))924 .addReg(SrcReg, getKillRegState(KillSrc))925 .addImm(0);926 } else {927 assert(AMDGPU::SReg_32RegClass.contains(SrcReg));928 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32))929 .addReg(SrcReg, getKillRegState(KillSrc))930 .addImm(0);931 }932 933 return;934 }935 936 if (RC == &AMDGPU::AGPR_32RegClass) {937 if (AMDGPU::VGPR_32RegClass.contains(SrcReg) ||938 (ST.hasGFX90AInsts() && AMDGPU::SReg_32RegClass.contains(SrcReg))) {939 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)940 .addReg(SrcReg, getKillRegState(KillSrc));941 return;942 }943 944 if (AMDGPU::AGPR_32RegClass.contains(SrcReg) && ST.hasGFX90AInsts()) {945 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_MOV_B32), DestReg)946 .addReg(SrcReg, getKillRegState(KillSrc));947 return;948 }949 950 // FIXME: Pass should maintain scavenger to avoid scan through the block on951 // every AGPR spill.952 RegScavenger RS;953 const bool Overlap = RI.regsOverlap(SrcReg, DestReg);954 indirectCopyToAGPR(*this, MBB, MI, DL, DestReg, SrcReg, KillSrc, RS, Overlap);955 return;956 }957 958 if (Size == 16) {959 assert(AMDGPU::VGPR_16RegClass.contains(SrcReg) ||960 AMDGPU::SReg_LO16RegClass.contains(SrcReg) ||961 AMDGPU::AGPR_LO16RegClass.contains(SrcReg));962 963 bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(DestReg);964 bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(SrcReg);965 bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(DestReg);966 bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(SrcReg);967 bool DstLow = !AMDGPU::isHi16Reg(DestReg, RI);968 bool SrcLow = !AMDGPU::isHi16Reg(SrcReg, RI);969 MCRegister NewDestReg = RI.get32BitRegister(DestReg);970 MCRegister NewSrcReg = RI.get32BitRegister(SrcReg);971 972 if (IsSGPRDst) {973 if (!IsSGPRSrc) {974 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);975 return;976 }977 978 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), NewDestReg)979 .addReg(NewSrcReg, getKillRegState(KillSrc));980 return;981 }982 983 if (IsAGPRDst || IsAGPRSrc) {984 if (!DstLow || !SrcLow) {985 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc,986 "Cannot use hi16 subreg with an AGPR!");987 }988 989 copyPhysReg(MBB, MI, DL, NewDestReg, NewSrcReg, KillSrc);990 return;991 }992 993 if (ST.useRealTrue16Insts()) {994 if (IsSGPRSrc) {995 assert(SrcLow);996 SrcReg = NewSrcReg;997 }998 // Use the smaller instruction encoding if possible.999 if (AMDGPU::VGPR_16_Lo128RegClass.contains(DestReg) &&1000 (IsSGPRSrc || AMDGPU::VGPR_16_Lo128RegClass.contains(SrcReg))) {1001 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B16_t16_e32), DestReg)1002 .addReg(SrcReg);1003 } else {1004 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B16_t16_e64), DestReg)1005 .addImm(0) // src0_modifiers1006 .addReg(SrcReg)1007 .addImm(0); // op_sel1008 }1009 return;1010 }1011 1012 if (IsSGPRSrc && !ST.hasSDWAScalar()) {1013 if (!DstLow || !SrcLow) {1014 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc,1015 "Cannot use hi16 subreg on VI!");1016 }1017 1018 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), NewDestReg)1019 .addReg(NewSrcReg, getKillRegState(KillSrc));1020 return;1021 }1022 1023 auto MIB = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_sdwa), NewDestReg)1024 .addImm(0) // src0_modifiers1025 .addReg(NewSrcReg)1026 .addImm(0) // clamp1027 .addImm(DstLow ? AMDGPU::SDWA::SdwaSel::WORD_01028 : AMDGPU::SDWA::SdwaSel::WORD_1)1029 .addImm(AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE)1030 .addImm(SrcLow ? AMDGPU::SDWA::SdwaSel::WORD_01031 : AMDGPU::SDWA::SdwaSel::WORD_1)1032 .addReg(NewDestReg, RegState::Implicit | RegState::Undef);1033 // First implicit operand is $exec.1034 MIB->tieOperands(0, MIB->getNumOperands() - 1);1035 return;1036 }1037 1038 if (RC == RI.getVGPR64Class() && (SrcRC == RC || RI.isSGPRClass(SrcRC))) {1039 if (ST.hasMovB64()) {1040 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_e32), DestReg)1041 .addReg(SrcReg, getKillRegState(KillSrc));1042 return;1043 }1044 if (ST.hasPkMovB32()) {1045 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DestReg)1046 .addImm(SISrcMods::OP_SEL_1)1047 .addReg(SrcReg)1048 .addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1)1049 .addReg(SrcReg)1050 .addImm(0) // op_sel_lo1051 .addImm(0) // op_sel_hi1052 .addImm(0) // neg_lo1053 .addImm(0) // neg_hi1054 .addImm(0) // clamp1055 .addReg(SrcReg, getKillRegState(KillSrc) | RegState::Implicit);1056 return;1057 }1058 }1059 1060 const bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg);1061 if (RI.isSGPRClass(RC)) {1062 if (!RI.isSGPRClass(SrcRC)) {1063 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);1064 return;1065 }1066 const bool CanKillSuperReg = KillSrc && !RI.regsOverlap(SrcReg, DestReg);1067 expandSGPRCopy(*this, MBB, MI, DL, DestReg, SrcReg, CanKillSuperReg, RC,1068 Forward);1069 return;1070 }1071 1072 unsigned EltSize = 4;1073 unsigned Opcode = AMDGPU::V_MOV_B32_e32;1074 if (RI.isAGPRClass(RC)) {1075 if (ST.hasGFX90AInsts() && RI.isAGPRClass(SrcRC))1076 Opcode = AMDGPU::V_ACCVGPR_MOV_B32;1077 else if (RI.hasVGPRs(SrcRC) ||1078 (ST.hasGFX90AInsts() && RI.isSGPRClass(SrcRC)))1079 Opcode = AMDGPU::V_ACCVGPR_WRITE_B32_e64;1080 else1081 Opcode = AMDGPU::INSTRUCTION_LIST_END;1082 } else if (RI.hasVGPRs(RC) && RI.isAGPRClass(SrcRC)) {1083 Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64;1084 } else if ((Size % 64 == 0) && RI.hasVGPRs(RC) &&1085 (RI.isProperlyAlignedRC(*RC) &&1086 (SrcRC == RC || RI.isSGPRClass(SrcRC)))) {1087 // TODO: In 96-bit case, could do a 64-bit mov and then a 32-bit mov.1088 if (ST.hasMovB64()) {1089 Opcode = AMDGPU::V_MOV_B64_e32;1090 EltSize = 8;1091 } else if (ST.hasPkMovB32()) {1092 Opcode = AMDGPU::V_PK_MOV_B32;1093 EltSize = 8;1094 }1095 }1096 1097 // For the cases where we need an intermediate instruction/temporary register1098 // (destination is an AGPR), we need a scavenger.1099 //1100 // FIXME: The pass should maintain this for us so we don't have to re-scan the1101 // whole block for every handled copy.1102 std::unique_ptr<RegScavenger> RS;1103 if (Opcode == AMDGPU::INSTRUCTION_LIST_END)1104 RS = std::make_unique<RegScavenger>();1105 1106 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize);1107 1108 // If there is an overlap, we can't kill the super-register on the last1109 // instruction, since it will also kill the components made live by this def.1110 const bool Overlap = RI.regsOverlap(SrcReg, DestReg);1111 const bool CanKillSuperReg = KillSrc && !Overlap;1112 1113 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {1114 unsigned SubIdx;1115 if (Forward)1116 SubIdx = SubIndices[Idx];1117 else1118 SubIdx = SubIndices[SubIndices.size() - Idx - 1];1119 Register DestSubReg = RI.getSubReg(DestReg, SubIdx);1120 Register SrcSubReg = RI.getSubReg(SrcReg, SubIdx);1121 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");1122 1123 bool IsFirstSubreg = Idx == 0;1124 bool UseKill = CanKillSuperReg && Idx == SubIndices.size() - 1;1125 1126 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) {1127 Register ImpDefSuper = IsFirstSubreg ? Register(DestReg) : Register();1128 Register ImpUseSuper = SrcReg;1129 indirectCopyToAGPR(*this, MBB, MI, DL, DestSubReg, SrcSubReg, UseKill,1130 *RS, Overlap, ImpDefSuper, ImpUseSuper);1131 } else if (Opcode == AMDGPU::V_PK_MOV_B32) {1132 MachineInstrBuilder MIB =1133 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DestSubReg)1134 .addImm(SISrcMods::OP_SEL_1)1135 .addReg(SrcSubReg)1136 .addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1)1137 .addReg(SrcSubReg)1138 .addImm(0) // op_sel_lo1139 .addImm(0) // op_sel_hi1140 .addImm(0) // neg_lo1141 .addImm(0) // neg_hi1142 .addImm(0) // clamp1143 .addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);1144 if (IsFirstSubreg)1145 MIB.addReg(DestReg, RegState::Define | RegState::Implicit);1146 } else {1147 MachineInstrBuilder Builder =1148 BuildMI(MBB, MI, DL, get(Opcode), DestSubReg).addReg(SrcSubReg);1149 if (IsFirstSubreg)1150 Builder.addReg(DestReg, RegState::Define | RegState::Implicit);1151 1152 Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);1153 }1154 }1155}1156 1157int SIInstrInfo::commuteOpcode(unsigned Opcode) const {1158 int NewOpc;1159 1160 // Try to map original to commuted opcode1161 NewOpc = AMDGPU::getCommuteRev(Opcode);1162 if (NewOpc != -1)1163 // Check if the commuted (REV) opcode exists on the target.1164 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;1165 1166 // Try to map commuted to original opcode1167 NewOpc = AMDGPU::getCommuteOrig(Opcode);1168 if (NewOpc != -1)1169 // Check if the original (non-REV) opcode exists on the target.1170 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;1171 1172 return Opcode;1173}1174 1175const TargetRegisterClass *1176SIInstrInfo::getPreferredSelectRegClass(unsigned Size) const {1177 return &AMDGPU::VGPR_32RegClass;1178}1179 1180void SIInstrInfo::insertVectorSelect(MachineBasicBlock &MBB,1181 MachineBasicBlock::iterator I,1182 const DebugLoc &DL, Register DstReg,1183 ArrayRef<MachineOperand> Cond,1184 Register TrueReg,1185 Register FalseReg) const {1186 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();1187 const TargetRegisterClass *BoolXExecRC = RI.getWaveMaskRegClass();1188 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);1189 assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass &&1190 "Not a VGPR32 reg");1191 1192 if (Cond.size() == 1) {1193 Register SReg = MRI.createVirtualRegister(BoolXExecRC);1194 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)1195 .add(Cond[0]);1196 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)1197 .addImm(0)1198 .addReg(FalseReg)1199 .addImm(0)1200 .addReg(TrueReg)1201 .addReg(SReg);1202 } else if (Cond.size() == 2) {1203 assert(Cond[0].isImm() && "Cond[0] is not an immediate");1204 switch (Cond[0].getImm()) {1205 case SIInstrInfo::SCC_TRUE: {1206 Register SReg = MRI.createVirtualRegister(BoolXExecRC);1207 BuildMI(MBB, I, DL, get(LMC.CSelectOpc), SReg).addImm(1).addImm(0);1208 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)1209 .addImm(0)1210 .addReg(FalseReg)1211 .addImm(0)1212 .addReg(TrueReg)1213 .addReg(SReg);1214 break;1215 }1216 case SIInstrInfo::SCC_FALSE: {1217 Register SReg = MRI.createVirtualRegister(BoolXExecRC);1218 BuildMI(MBB, I, DL, get(LMC.CSelectOpc), SReg).addImm(0).addImm(1);1219 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)1220 .addImm(0)1221 .addReg(FalseReg)1222 .addImm(0)1223 .addReg(TrueReg)1224 .addReg(SReg);1225 break;1226 }1227 case SIInstrInfo::VCCNZ: {1228 MachineOperand RegOp = Cond[1];1229 RegOp.setImplicit(false);1230 Register SReg = MRI.createVirtualRegister(BoolXExecRC);1231 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)1232 .add(RegOp);1233 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)1234 .addImm(0)1235 .addReg(FalseReg)1236 .addImm(0)1237 .addReg(TrueReg)1238 .addReg(SReg);1239 break;1240 }1241 case SIInstrInfo::VCCZ: {1242 MachineOperand RegOp = Cond[1];1243 RegOp.setImplicit(false);1244 Register SReg = MRI.createVirtualRegister(BoolXExecRC);1245 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)1246 .add(RegOp);1247 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)1248 .addImm(0)1249 .addReg(TrueReg)1250 .addImm(0)1251 .addReg(FalseReg)1252 .addReg(SReg);1253 break;1254 }1255 case SIInstrInfo::EXECNZ: {1256 Register SReg = MRI.createVirtualRegister(BoolXExecRC);1257 Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC());1258 BuildMI(MBB, I, DL, get(LMC.OrSaveExecOpc), SReg2).addImm(0);1259 BuildMI(MBB, I, DL, get(LMC.CSelectOpc), SReg).addImm(1).addImm(0);1260 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)1261 .addImm(0)1262 .addReg(FalseReg)1263 .addImm(0)1264 .addReg(TrueReg)1265 .addReg(SReg);1266 break;1267 }1268 case SIInstrInfo::EXECZ: {1269 Register SReg = MRI.createVirtualRegister(BoolXExecRC);1270 Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC());1271 BuildMI(MBB, I, DL, get(LMC.OrSaveExecOpc), SReg2).addImm(0);1272 BuildMI(MBB, I, DL, get(LMC.CSelectOpc), SReg).addImm(0).addImm(1);1273 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)1274 .addImm(0)1275 .addReg(FalseReg)1276 .addImm(0)1277 .addReg(TrueReg)1278 .addReg(SReg);1279 llvm_unreachable("Unhandled branch predicate EXECZ");1280 break;1281 }1282 default:1283 llvm_unreachable("invalid branch predicate");1284 }1285 } else {1286 llvm_unreachable("Can only handle Cond size 1 or 2");1287 }1288}1289 1290Register SIInstrInfo::insertEQ(MachineBasicBlock *MBB,1291 MachineBasicBlock::iterator I,1292 const DebugLoc &DL,1293 Register SrcReg, int Value) const {1294 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();1295 Register Reg = MRI.createVirtualRegister(RI.getBoolRC());1296 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg)1297 .addImm(Value)1298 .addReg(SrcReg);1299 1300 return Reg;1301}1302 1303Register SIInstrInfo::insertNE(MachineBasicBlock *MBB,1304 MachineBasicBlock::iterator I,1305 const DebugLoc &DL,1306 Register SrcReg, int Value) const {1307 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();1308 Register Reg = MRI.createVirtualRegister(RI.getBoolRC());1309 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg)1310 .addImm(Value)1311 .addReg(SrcReg);1312 1313 return Reg;1314}1315 1316bool SIInstrInfo::getConstValDefinedInReg(const MachineInstr &MI,1317 const Register Reg,1318 int64_t &ImmVal) const {1319 switch (MI.getOpcode()) {1320 case AMDGPU::V_MOV_B32_e32:1321 case AMDGPU::S_MOV_B32:1322 case AMDGPU::S_MOVK_I32:1323 case AMDGPU::S_MOV_B64:1324 case AMDGPU::V_MOV_B64_e32:1325 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:1326 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:1327 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:1328 case AMDGPU::S_MOV_B64_IMM_PSEUDO:1329 case AMDGPU::V_MOV_B64_PSEUDO: {1330 const MachineOperand &Src0 = MI.getOperand(1);1331 if (Src0.isImm()) {1332 ImmVal = Src0.getImm();1333 return MI.getOperand(0).getReg() == Reg;1334 }1335 1336 return false;1337 }1338 case AMDGPU::S_BREV_B32:1339 case AMDGPU::V_BFREV_B32_e32:1340 case AMDGPU::V_BFREV_B32_e64: {1341 const MachineOperand &Src0 = MI.getOperand(1);1342 if (Src0.isImm()) {1343 ImmVal = static_cast<int64_t>(reverseBits<int32_t>(Src0.getImm()));1344 return MI.getOperand(0).getReg() == Reg;1345 }1346 1347 return false;1348 }1349 case AMDGPU::S_NOT_B32:1350 case AMDGPU::V_NOT_B32_e32:1351 case AMDGPU::V_NOT_B32_e64: {1352 const MachineOperand &Src0 = MI.getOperand(1);1353 if (Src0.isImm()) {1354 ImmVal = static_cast<int64_t>(~static_cast<int32_t>(Src0.getImm()));1355 return MI.getOperand(0).getReg() == Reg;1356 }1357 1358 return false;1359 }1360 default:1361 return false;1362 }1363}1364 1365unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {1366 1367 if (RI.isAGPRClass(DstRC))1368 return AMDGPU::COPY;1369 if (RI.getRegSizeInBits(*DstRC) == 16) {1370 // Assume hi bits are unneeded. Only _e64 true16 instructions are legal1371 // before RA.1372 return RI.isSGPRClass(DstRC) ? AMDGPU::COPY : AMDGPU::V_MOV_B16_t16_e64;1373 }1374 if (RI.getRegSizeInBits(*DstRC) == 32)1375 return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;1376 if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC))1377 return AMDGPU::S_MOV_B64;1378 if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC))1379 return AMDGPU::V_MOV_B64_PSEUDO;1380 return AMDGPU::COPY;1381}1382 1383const MCInstrDesc &1384SIInstrInfo::getIndirectGPRIDXPseudo(unsigned VecSize,1385 bool IsIndirectSrc) const {1386 if (IsIndirectSrc) {1387 if (VecSize <= 32) // 4 bytes1388 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1);1389 if (VecSize <= 64) // 8 bytes1390 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2);1391 if (VecSize <= 96) // 12 bytes1392 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3);1393 if (VecSize <= 128) // 16 bytes1394 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4);1395 if (VecSize <= 160) // 20 bytes1396 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5);1397 if (VecSize <= 256) // 32 bytes1398 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8);1399 if (VecSize <= 288) // 36 bytes1400 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9);1401 if (VecSize <= 320) // 40 bytes1402 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10);1403 if (VecSize <= 352) // 44 bytes1404 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11);1405 if (VecSize <= 384) // 48 bytes1406 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12);1407 if (VecSize <= 512) // 64 bytes1408 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16);1409 if (VecSize <= 1024) // 128 bytes1410 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32);1411 1412 llvm_unreachable("unsupported size for IndirectRegReadGPRIDX pseudos");1413 }1414 1415 if (VecSize <= 32) // 4 bytes1416 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1);1417 if (VecSize <= 64) // 8 bytes1418 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2);1419 if (VecSize <= 96) // 12 bytes1420 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3);1421 if (VecSize <= 128) // 16 bytes1422 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4);1423 if (VecSize <= 160) // 20 bytes1424 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5);1425 if (VecSize <= 256) // 32 bytes1426 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8);1427 if (VecSize <= 288) // 36 bytes1428 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9);1429 if (VecSize <= 320) // 40 bytes1430 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10);1431 if (VecSize <= 352) // 44 bytes1432 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11);1433 if (VecSize <= 384) // 48 bytes1434 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12);1435 if (VecSize <= 512) // 64 bytes1436 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16);1437 if (VecSize <= 1024) // 128 bytes1438 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32);1439 1440 llvm_unreachable("unsupported size for IndirectRegWriteGPRIDX pseudos");1441}1442 1443static unsigned getIndirectVGPRWriteMovRelPseudoOpc(unsigned VecSize) {1444 if (VecSize <= 32) // 4 bytes1445 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1;1446 if (VecSize <= 64) // 8 bytes1447 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2;1448 if (VecSize <= 96) // 12 bytes1449 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3;1450 if (VecSize <= 128) // 16 bytes1451 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4;1452 if (VecSize <= 160) // 20 bytes1453 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5;1454 if (VecSize <= 256) // 32 bytes1455 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8;1456 if (VecSize <= 288) // 36 bytes1457 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9;1458 if (VecSize <= 320) // 40 bytes1459 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10;1460 if (VecSize <= 352) // 44 bytes1461 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11;1462 if (VecSize <= 384) // 48 bytes1463 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12;1464 if (VecSize <= 512) // 64 bytes1465 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16;1466 if (VecSize <= 1024) // 128 bytes1467 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32;1468 1469 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");1470}1471 1472static unsigned getIndirectSGPRWriteMovRelPseudo32(unsigned VecSize) {1473 if (VecSize <= 32) // 4 bytes1474 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1;1475 if (VecSize <= 64) // 8 bytes1476 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2;1477 if (VecSize <= 96) // 12 bytes1478 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3;1479 if (VecSize <= 128) // 16 bytes1480 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4;1481 if (VecSize <= 160) // 20 bytes1482 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5;1483 if (VecSize <= 256) // 32 bytes1484 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8;1485 if (VecSize <= 288) // 36 bytes1486 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9;1487 if (VecSize <= 320) // 40 bytes1488 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10;1489 if (VecSize <= 352) // 44 bytes1490 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11;1491 if (VecSize <= 384) // 48 bytes1492 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12;1493 if (VecSize <= 512) // 64 bytes1494 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16;1495 if (VecSize <= 1024) // 128 bytes1496 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32;1497 1498 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");1499}1500 1501static unsigned getIndirectSGPRWriteMovRelPseudo64(unsigned VecSize) {1502 if (VecSize <= 64) // 8 bytes1503 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1;1504 if (VecSize <= 128) // 16 bytes1505 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2;1506 if (VecSize <= 256) // 32 bytes1507 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4;1508 if (VecSize <= 512) // 64 bytes1509 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8;1510 if (VecSize <= 1024) // 128 bytes1511 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16;1512 1513 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");1514}1515 1516const MCInstrDesc &1517SIInstrInfo::getIndirectRegWriteMovRelPseudo(unsigned VecSize, unsigned EltSize,1518 bool IsSGPR) const {1519 if (IsSGPR) {1520 switch (EltSize) {1521 case 32:1522 return get(getIndirectSGPRWriteMovRelPseudo32(VecSize));1523 case 64:1524 return get(getIndirectSGPRWriteMovRelPseudo64(VecSize));1525 default:1526 llvm_unreachable("invalid reg indexing elt size");1527 }1528 }1529 1530 assert(EltSize == 32 && "invalid reg indexing elt size");1531 return get(getIndirectVGPRWriteMovRelPseudoOpc(VecSize));1532}1533 1534static unsigned getSGPRSpillSaveOpcode(unsigned Size) {1535 switch (Size) {1536 case 4:1537 return AMDGPU::SI_SPILL_S32_SAVE;1538 case 8:1539 return AMDGPU::SI_SPILL_S64_SAVE;1540 case 12:1541 return AMDGPU::SI_SPILL_S96_SAVE;1542 case 16:1543 return AMDGPU::SI_SPILL_S128_SAVE;1544 case 20:1545 return AMDGPU::SI_SPILL_S160_SAVE;1546 case 24:1547 return AMDGPU::SI_SPILL_S192_SAVE;1548 case 28:1549 return AMDGPU::SI_SPILL_S224_SAVE;1550 case 32:1551 return AMDGPU::SI_SPILL_S256_SAVE;1552 case 36:1553 return AMDGPU::SI_SPILL_S288_SAVE;1554 case 40:1555 return AMDGPU::SI_SPILL_S320_SAVE;1556 case 44:1557 return AMDGPU::SI_SPILL_S352_SAVE;1558 case 48:1559 return AMDGPU::SI_SPILL_S384_SAVE;1560 case 64:1561 return AMDGPU::SI_SPILL_S512_SAVE;1562 case 128:1563 return AMDGPU::SI_SPILL_S1024_SAVE;1564 default:1565 llvm_unreachable("unknown register size");1566 }1567}1568 1569static unsigned getVGPRSpillSaveOpcode(unsigned Size) {1570 switch (Size) {1571 case 2:1572 return AMDGPU::SI_SPILL_V16_SAVE;1573 case 4:1574 return AMDGPU::SI_SPILL_V32_SAVE;1575 case 8:1576 return AMDGPU::SI_SPILL_V64_SAVE;1577 case 12:1578 return AMDGPU::SI_SPILL_V96_SAVE;1579 case 16:1580 return AMDGPU::SI_SPILL_V128_SAVE;1581 case 20:1582 return AMDGPU::SI_SPILL_V160_SAVE;1583 case 24:1584 return AMDGPU::SI_SPILL_V192_SAVE;1585 case 28:1586 return AMDGPU::SI_SPILL_V224_SAVE;1587 case 32:1588 return AMDGPU::SI_SPILL_V256_SAVE;1589 case 36:1590 return AMDGPU::SI_SPILL_V288_SAVE;1591 case 40:1592 return AMDGPU::SI_SPILL_V320_SAVE;1593 case 44:1594 return AMDGPU::SI_SPILL_V352_SAVE;1595 case 48:1596 return AMDGPU::SI_SPILL_V384_SAVE;1597 case 64:1598 return AMDGPU::SI_SPILL_V512_SAVE;1599 case 128:1600 return AMDGPU::SI_SPILL_V1024_SAVE;1601 default:1602 llvm_unreachable("unknown register size");1603 }1604}1605 1606static unsigned getAVSpillSaveOpcode(unsigned Size) {1607 switch (Size) {1608 case 4:1609 return AMDGPU::SI_SPILL_AV32_SAVE;1610 case 8:1611 return AMDGPU::SI_SPILL_AV64_SAVE;1612 case 12:1613 return AMDGPU::SI_SPILL_AV96_SAVE;1614 case 16:1615 return AMDGPU::SI_SPILL_AV128_SAVE;1616 case 20:1617 return AMDGPU::SI_SPILL_AV160_SAVE;1618 case 24:1619 return AMDGPU::SI_SPILL_AV192_SAVE;1620 case 28:1621 return AMDGPU::SI_SPILL_AV224_SAVE;1622 case 32:1623 return AMDGPU::SI_SPILL_AV256_SAVE;1624 case 36:1625 return AMDGPU::SI_SPILL_AV288_SAVE;1626 case 40:1627 return AMDGPU::SI_SPILL_AV320_SAVE;1628 case 44:1629 return AMDGPU::SI_SPILL_AV352_SAVE;1630 case 48:1631 return AMDGPU::SI_SPILL_AV384_SAVE;1632 case 64:1633 return AMDGPU::SI_SPILL_AV512_SAVE;1634 case 128:1635 return AMDGPU::SI_SPILL_AV1024_SAVE;1636 default:1637 llvm_unreachable("unknown register size");1638 }1639}1640 1641static unsigned getWWMRegSpillSaveOpcode(unsigned Size,1642 bool IsVectorSuperClass) {1643 // Currently, there is only 32-bit WWM register spills needed.1644 if (Size != 4)1645 llvm_unreachable("unknown wwm register spill size");1646 1647 if (IsVectorSuperClass)1648 return AMDGPU::SI_SPILL_WWM_AV32_SAVE;1649 1650 return AMDGPU::SI_SPILL_WWM_V32_SAVE;1651}1652 1653unsigned SIInstrInfo::getVectorRegSpillSaveOpcode(1654 Register Reg, const TargetRegisterClass *RC, unsigned Size,1655 const SIMachineFunctionInfo &MFI) const {1656 bool IsVectorSuperClass = RI.isVectorSuperClass(RC);1657 1658 // Choose the right opcode if spilling a WWM register.1659 if (MFI.checkFlag(Reg, AMDGPU::VirtRegFlag::WWM_REG))1660 return getWWMRegSpillSaveOpcode(Size, IsVectorSuperClass);1661 1662 // TODO: Check if AGPRs are available1663 if (ST.hasMAIInsts())1664 return getAVSpillSaveOpcode(Size);1665 1666 return getVGPRSpillSaveOpcode(Size);1667}1668 1669void SIInstrInfo::storeRegToStackSlot(1670 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,1671 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,1672 MachineInstr::MIFlag Flags) const {1673 MachineFunction *MF = MBB.getParent();1674 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();1675 MachineFrameInfo &FrameInfo = MF->getFrameInfo();1676 const DebugLoc &DL = MBB.findDebugLoc(MI);1677 1678 MachinePointerInfo PtrInfo1679 = MachinePointerInfo::getFixedStack(*MF, FrameIndex);1680 MachineMemOperand *MMO = MF->getMachineMemOperand(1681 PtrInfo, MachineMemOperand::MOStore, FrameInfo.getObjectSize(FrameIndex),1682 FrameInfo.getObjectAlign(FrameIndex));1683 unsigned SpillSize = RI.getSpillSize(*RC);1684 1685 MachineRegisterInfo &MRI = MF->getRegInfo();1686 if (RI.isSGPRClass(RC)) {1687 MFI->setHasSpilledSGPRs();1688 assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled");1689 assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI &&1690 SrcReg != AMDGPU::EXEC && "exec should not be spilled");1691 1692 // We are only allowed to create one new instruction when spilling1693 // registers, so we need to use pseudo instruction for spilling SGPRs.1694 const MCInstrDesc &OpDesc = get(getSGPRSpillSaveOpcode(SpillSize));1695 1696 // The SGPR spill/restore instructions only work on number sgprs, so we need1697 // to make sure we are using the correct register class.1698 if (SrcReg.isVirtual() && SpillSize == 4) {1699 MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0_XEXECRegClass);1700 }1701 1702 BuildMI(MBB, MI, DL, OpDesc)1703 .addReg(SrcReg, getKillRegState(isKill)) // data1704 .addFrameIndex(FrameIndex) // addr1705 .addMemOperand(MMO)1706 .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);1707 1708 if (RI.spillSGPRToVGPR())1709 FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);1710 return;1711 }1712 1713 unsigned Opcode =1714 getVectorRegSpillSaveOpcode(VReg ? VReg : SrcReg, RC, SpillSize, *MFI);1715 MFI->setHasSpilledVGPRs();1716 1717 BuildMI(MBB, MI, DL, get(Opcode))1718 .addReg(SrcReg, getKillRegState(isKill)) // data1719 .addFrameIndex(FrameIndex) // addr1720 .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset1721 .addImm(0) // offset1722 .addMemOperand(MMO);1723}1724 1725static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {1726 switch (Size) {1727 case 4:1728 return AMDGPU::SI_SPILL_S32_RESTORE;1729 case 8:1730 return AMDGPU::SI_SPILL_S64_RESTORE;1731 case 12:1732 return AMDGPU::SI_SPILL_S96_RESTORE;1733 case 16:1734 return AMDGPU::SI_SPILL_S128_RESTORE;1735 case 20:1736 return AMDGPU::SI_SPILL_S160_RESTORE;1737 case 24:1738 return AMDGPU::SI_SPILL_S192_RESTORE;1739 case 28:1740 return AMDGPU::SI_SPILL_S224_RESTORE;1741 case 32:1742 return AMDGPU::SI_SPILL_S256_RESTORE;1743 case 36:1744 return AMDGPU::SI_SPILL_S288_RESTORE;1745 case 40:1746 return AMDGPU::SI_SPILL_S320_RESTORE;1747 case 44:1748 return AMDGPU::SI_SPILL_S352_RESTORE;1749 case 48:1750 return AMDGPU::SI_SPILL_S384_RESTORE;1751 case 64:1752 return AMDGPU::SI_SPILL_S512_RESTORE;1753 case 128:1754 return AMDGPU::SI_SPILL_S1024_RESTORE;1755 default:1756 llvm_unreachable("unknown register size");1757 }1758}1759 1760static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {1761 switch (Size) {1762 case 2:1763 return AMDGPU::SI_SPILL_V16_RESTORE;1764 case 4:1765 return AMDGPU::SI_SPILL_V32_RESTORE;1766 case 8:1767 return AMDGPU::SI_SPILL_V64_RESTORE;1768 case 12:1769 return AMDGPU::SI_SPILL_V96_RESTORE;1770 case 16:1771 return AMDGPU::SI_SPILL_V128_RESTORE;1772 case 20:1773 return AMDGPU::SI_SPILL_V160_RESTORE;1774 case 24:1775 return AMDGPU::SI_SPILL_V192_RESTORE;1776 case 28:1777 return AMDGPU::SI_SPILL_V224_RESTORE;1778 case 32:1779 return AMDGPU::SI_SPILL_V256_RESTORE;1780 case 36:1781 return AMDGPU::SI_SPILL_V288_RESTORE;1782 case 40:1783 return AMDGPU::SI_SPILL_V320_RESTORE;1784 case 44:1785 return AMDGPU::SI_SPILL_V352_RESTORE;1786 case 48:1787 return AMDGPU::SI_SPILL_V384_RESTORE;1788 case 64:1789 return AMDGPU::SI_SPILL_V512_RESTORE;1790 case 128:1791 return AMDGPU::SI_SPILL_V1024_RESTORE;1792 default:1793 llvm_unreachable("unknown register size");1794 }1795}1796 1797static unsigned getAVSpillRestoreOpcode(unsigned Size) {1798 switch (Size) {1799 case 4:1800 return AMDGPU::SI_SPILL_AV32_RESTORE;1801 case 8:1802 return AMDGPU::SI_SPILL_AV64_RESTORE;1803 case 12:1804 return AMDGPU::SI_SPILL_AV96_RESTORE;1805 case 16:1806 return AMDGPU::SI_SPILL_AV128_RESTORE;1807 case 20:1808 return AMDGPU::SI_SPILL_AV160_RESTORE;1809 case 24:1810 return AMDGPU::SI_SPILL_AV192_RESTORE;1811 case 28:1812 return AMDGPU::SI_SPILL_AV224_RESTORE;1813 case 32:1814 return AMDGPU::SI_SPILL_AV256_RESTORE;1815 case 36:1816 return AMDGPU::SI_SPILL_AV288_RESTORE;1817 case 40:1818 return AMDGPU::SI_SPILL_AV320_RESTORE;1819 case 44:1820 return AMDGPU::SI_SPILL_AV352_RESTORE;1821 case 48:1822 return AMDGPU::SI_SPILL_AV384_RESTORE;1823 case 64:1824 return AMDGPU::SI_SPILL_AV512_RESTORE;1825 case 128:1826 return AMDGPU::SI_SPILL_AV1024_RESTORE;1827 default:1828 llvm_unreachable("unknown register size");1829 }1830}1831 1832static unsigned getWWMRegSpillRestoreOpcode(unsigned Size,1833 bool IsVectorSuperClass) {1834 // Currently, there is only 32-bit WWM register spills needed.1835 if (Size != 4)1836 llvm_unreachable("unknown wwm register spill size");1837 1838 if (IsVectorSuperClass) // TODO: Always use this if there are AGPRs1839 return AMDGPU::SI_SPILL_WWM_AV32_RESTORE;1840 1841 return AMDGPU::SI_SPILL_WWM_V32_RESTORE;1842}1843 1844unsigned SIInstrInfo::getVectorRegSpillRestoreOpcode(1845 Register Reg, const TargetRegisterClass *RC, unsigned Size,1846 const SIMachineFunctionInfo &MFI) const {1847 bool IsVectorSuperClass = RI.isVectorSuperClass(RC);1848 1849 // Choose the right opcode if restoring a WWM register.1850 if (MFI.checkFlag(Reg, AMDGPU::VirtRegFlag::WWM_REG))1851 return getWWMRegSpillRestoreOpcode(Size, IsVectorSuperClass);1852 1853 // TODO: Check if AGPRs are available1854 if (ST.hasMAIInsts())1855 return getAVSpillRestoreOpcode(Size);1856 1857 assert(!RI.isAGPRClass(RC));1858 return getVGPRSpillRestoreOpcode(Size);1859}1860 1861void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,1862 MachineBasicBlock::iterator MI,1863 Register DestReg, int FrameIndex,1864 const TargetRegisterClass *RC,1865 Register VReg,1866 MachineInstr::MIFlag Flags) const {1867 MachineFunction *MF = MBB.getParent();1868 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();1869 MachineFrameInfo &FrameInfo = MF->getFrameInfo();1870 const DebugLoc &DL = MBB.findDebugLoc(MI);1871 unsigned SpillSize = RI.getSpillSize(*RC);1872 1873 MachinePointerInfo PtrInfo1874 = MachinePointerInfo::getFixedStack(*MF, FrameIndex);1875 1876 MachineMemOperand *MMO = MF->getMachineMemOperand(1877 PtrInfo, MachineMemOperand::MOLoad, FrameInfo.getObjectSize(FrameIndex),1878 FrameInfo.getObjectAlign(FrameIndex));1879 1880 if (RI.isSGPRClass(RC)) {1881 MFI->setHasSpilledSGPRs();1882 assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into");1883 assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI &&1884 DestReg != AMDGPU::EXEC && "exec should not be spilled");1885 1886 // FIXME: Maybe this should not include a memoperand because it will be1887 // lowered to non-memory instructions.1888 const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize));1889 if (DestReg.isVirtual() && SpillSize == 4) {1890 MachineRegisterInfo &MRI = MF->getRegInfo();1891 MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0_XEXECRegClass);1892 }1893 1894 if (RI.spillSGPRToVGPR())1895 FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);1896 BuildMI(MBB, MI, DL, OpDesc, DestReg)1897 .addFrameIndex(FrameIndex) // addr1898 .addMemOperand(MMO)1899 .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);1900 1901 return;1902 }1903 1904 unsigned Opcode = getVectorRegSpillRestoreOpcode(VReg ? VReg : DestReg, RC,1905 SpillSize, *MFI);1906 BuildMI(MBB, MI, DL, get(Opcode), DestReg)1907 .addFrameIndex(FrameIndex) // vaddr1908 .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset1909 .addImm(0) // offset1910 .addMemOperand(MMO);1911}1912 1913void SIInstrInfo::insertNoop(MachineBasicBlock &MBB,1914 MachineBasicBlock::iterator MI) const {1915 insertNoops(MBB, MI, 1);1916}1917 1918void SIInstrInfo::insertNoops(MachineBasicBlock &MBB,1919 MachineBasicBlock::iterator MI,1920 unsigned Quantity) const {1921 DebugLoc DL = MBB.findDebugLoc(MI);1922 unsigned MaxSNopCount = 1u << ST.getSNopBits();1923 while (Quantity > 0) {1924 unsigned Arg = std::min(Quantity, MaxSNopCount);1925 Quantity -= Arg;1926 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP)).addImm(Arg - 1);1927 }1928}1929 1930void SIInstrInfo::insertReturn(MachineBasicBlock &MBB) const {1931 auto *MF = MBB.getParent();1932 SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();1933 1934 assert(Info->isEntryFunction());1935 1936 if (MBB.succ_empty()) {1937 bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end();1938 if (HasNoTerminator) {1939 if (Info->returnsVoid()) {1940 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0);1941 } else {1942 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG));1943 }1944 }1945 }1946}1947 1948MachineBasicBlock *SIInstrInfo::insertSimulatedTrap(MachineRegisterInfo &MRI,1949 MachineBasicBlock &MBB,1950 MachineInstr &MI,1951 const DebugLoc &DL) const {1952 MachineFunction *MF = MBB.getParent();1953 constexpr unsigned DoorbellIDMask = 0x3ff;1954 constexpr unsigned ECQueueWaveAbort = 0x400;1955 1956 MachineBasicBlock *TrapBB = &MBB;1957 MachineBasicBlock *ContBB = &MBB;1958 MachineBasicBlock *HaltLoopBB = MF->CreateMachineBasicBlock();1959 1960 if (!MBB.succ_empty() || std::next(MI.getIterator()) != MBB.end()) {1961 ContBB = MBB.splitAt(MI, /*UpdateLiveIns=*/false);1962 TrapBB = MF->CreateMachineBasicBlock();1963 BuildMI(MBB, MI, DL, get(AMDGPU::S_CBRANCH_EXECNZ)).addMBB(TrapBB);1964 MF->push_back(TrapBB);1965 MBB.addSuccessor(TrapBB);1966 } else {1967 // Since we're adding HaltLoopBB and modifying the CFG, we must return a1968 // different block to signal the change.1969 ContBB = HaltLoopBB;1970 }1971 1972 // Start with a `s_trap 2`, if we're in PRIV=1 and we need the workaround this1973 // will be a nop.1974 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_TRAP))1975 .addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap));1976 Register DoorbellReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);1977 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_SENDMSG_RTN_B32),1978 DoorbellReg)1979 .addImm(AMDGPU::SendMsg::ID_RTN_GET_DOORBELL);1980 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_MOV_B32), AMDGPU::TTMP2)1981 .addUse(AMDGPU::M0);1982 Register DoorbellRegMasked =1983 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);1984 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_AND_B32), DoorbellRegMasked)1985 .addUse(DoorbellReg)1986 .addImm(DoorbellIDMask);1987 Register SetWaveAbortBit =1988 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);1989 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_OR_B32), SetWaveAbortBit)1990 .addUse(DoorbellRegMasked)1991 .addImm(ECQueueWaveAbort);1992 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_MOV_B32), AMDGPU::M0)1993 .addUse(SetWaveAbortBit);1994 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_SENDMSG))1995 .addImm(AMDGPU::SendMsg::ID_INTERRUPT);1996 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_MOV_B32), AMDGPU::M0)1997 .addUse(AMDGPU::TTMP2);1998 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_BRANCH)).addMBB(HaltLoopBB);1999 TrapBB->addSuccessor(HaltLoopBB);2000 2001 BuildMI(*HaltLoopBB, HaltLoopBB->end(), DL, get(AMDGPU::S_SETHALT)).addImm(5);2002 BuildMI(*HaltLoopBB, HaltLoopBB->end(), DL, get(AMDGPU::S_BRANCH))2003 .addMBB(HaltLoopBB);2004 MF->push_back(HaltLoopBB);2005 HaltLoopBB->addSuccessor(HaltLoopBB);2006 2007 return ContBB;2008}2009 2010unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) {2011 switch (MI.getOpcode()) {2012 default:2013 if (MI.isMetaInstruction())2014 return 0;2015 return 1; // FIXME: Do wait states equal cycles?2016 2017 case AMDGPU::S_NOP:2018 return MI.getOperand(0).getImm() + 1;2019 // SI_RETURN_TO_EPILOG is a fallthrough to code outside of the function. The2020 // hazard, even if one exist, won't really be visible. Should we handle it?2021 }2022}2023 2024bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {2025 MachineBasicBlock &MBB = *MI.getParent();2026 DebugLoc DL = MBB.findDebugLoc(MI);2027 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);2028 switch (MI.getOpcode()) {2029 default: return TargetInstrInfo::expandPostRAPseudo(MI);2030 case AMDGPU::S_MOV_B64_term:2031 // This is only a terminator to get the correct spill code placement during2032 // register allocation.2033 MI.setDesc(get(AMDGPU::S_MOV_B64));2034 break;2035 2036 case AMDGPU::S_MOV_B32_term:2037 // This is only a terminator to get the correct spill code placement during2038 // register allocation.2039 MI.setDesc(get(AMDGPU::S_MOV_B32));2040 break;2041 2042 case AMDGPU::S_XOR_B64_term:2043 // This is only a terminator to get the correct spill code placement during2044 // register allocation.2045 MI.setDesc(get(AMDGPU::S_XOR_B64));2046 break;2047 2048 case AMDGPU::S_XOR_B32_term:2049 // This is only a terminator to get the correct spill code placement during2050 // register allocation.2051 MI.setDesc(get(AMDGPU::S_XOR_B32));2052 break;2053 case AMDGPU::S_OR_B64_term:2054 // This is only a terminator to get the correct spill code placement during2055 // register allocation.2056 MI.setDesc(get(AMDGPU::S_OR_B64));2057 break;2058 case AMDGPU::S_OR_B32_term:2059 // This is only a terminator to get the correct spill code placement during2060 // register allocation.2061 MI.setDesc(get(AMDGPU::S_OR_B32));2062 break;2063 2064 case AMDGPU::S_ANDN2_B64_term:2065 // This is only a terminator to get the correct spill code placement during2066 // register allocation.2067 MI.setDesc(get(AMDGPU::S_ANDN2_B64));2068 break;2069 2070 case AMDGPU::S_ANDN2_B32_term:2071 // This is only a terminator to get the correct spill code placement during2072 // register allocation.2073 MI.setDesc(get(AMDGPU::S_ANDN2_B32));2074 break;2075 2076 case AMDGPU::S_AND_B64_term:2077 // This is only a terminator to get the correct spill code placement during2078 // register allocation.2079 MI.setDesc(get(AMDGPU::S_AND_B64));2080 break;2081 2082 case AMDGPU::S_AND_B32_term:2083 // This is only a terminator to get the correct spill code placement during2084 // register allocation.2085 MI.setDesc(get(AMDGPU::S_AND_B32));2086 break;2087 2088 case AMDGPU::S_AND_SAVEEXEC_B64_term:2089 // This is only a terminator to get the correct spill code placement during2090 // register allocation.2091 MI.setDesc(get(AMDGPU::S_AND_SAVEEXEC_B64));2092 break;2093 2094 case AMDGPU::S_AND_SAVEEXEC_B32_term:2095 // This is only a terminator to get the correct spill code placement during2096 // register allocation.2097 MI.setDesc(get(AMDGPU::S_AND_SAVEEXEC_B32));2098 break;2099 2100 case AMDGPU::SI_SPILL_S32_TO_VGPR:2101 MI.setDesc(get(AMDGPU::V_WRITELANE_B32));2102 break;2103 2104 case AMDGPU::SI_RESTORE_S32_FROM_VGPR:2105 MI.setDesc(get(AMDGPU::V_READLANE_B32));2106 break;2107 case AMDGPU::AV_MOV_B32_IMM_PSEUDO: {2108 Register Dst = MI.getOperand(0).getReg();2109 bool IsAGPR = SIRegisterInfo::isAGPRClass(RI.getPhysRegBaseClass(Dst));2110 MI.setDesc(2111 get(IsAGPR ? AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::V_MOV_B32_e32));2112 break;2113 }2114 case AMDGPU::AV_MOV_B64_IMM_PSEUDO: {2115 Register Dst = MI.getOperand(0).getReg();2116 if (SIRegisterInfo::isAGPRClass(RI.getPhysRegBaseClass(Dst))) {2117 int64_t Imm = MI.getOperand(1).getImm();2118 2119 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);2120 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);2121 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DstLo)2122 .addImm(SignExtend64<32>(Imm))2123 .addReg(Dst, RegState::Implicit | RegState::Define);2124 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DstHi)2125 .addImm(SignExtend64<32>(Imm >> 32))2126 .addReg(Dst, RegState::Implicit | RegState::Define);2127 MI.eraseFromParent();2128 break;2129 }2130 2131 [[fallthrough]];2132 }2133 case AMDGPU::V_MOV_B64_PSEUDO: {2134 Register Dst = MI.getOperand(0).getReg();2135 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);2136 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);2137 2138 const MachineOperand &SrcOp = MI.getOperand(1);2139 // FIXME: Will this work for 64-bit floating point immediates?2140 assert(!SrcOp.isFPImm());2141 if (ST.hasMovB64()) {2142 MI.setDesc(get(AMDGPU::V_MOV_B64_e32));2143 if (SrcOp.isReg() || isInlineConstant(MI, 1) ||2144 isUInt<32>(SrcOp.getImm()) || ST.has64BitLiterals())2145 break;2146 }2147 if (SrcOp.isImm()) {2148 APInt Imm(64, SrcOp.getImm());2149 APInt Lo(32, Imm.getLoBits(32).getZExtValue());2150 APInt Hi(32, Imm.getHiBits(32).getZExtValue());2151 if (ST.hasPkMovB32() && Lo == Hi && isInlineConstant(Lo)) {2152 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst)2153 .addImm(SISrcMods::OP_SEL_1)2154 .addImm(Lo.getSExtValue())2155 .addImm(SISrcMods::OP_SEL_1)2156 .addImm(Lo.getSExtValue())2157 .addImm(0) // op_sel_lo2158 .addImm(0) // op_sel_hi2159 .addImm(0) // neg_lo2160 .addImm(0) // neg_hi2161 .addImm(0); // clamp2162 } else {2163 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)2164 .addImm(Lo.getSExtValue())2165 .addReg(Dst, RegState::Implicit | RegState::Define);2166 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)2167 .addImm(Hi.getSExtValue())2168 .addReg(Dst, RegState::Implicit | RegState::Define);2169 }2170 } else {2171 assert(SrcOp.isReg());2172 if (ST.hasPkMovB32() &&2173 !RI.isAGPR(MBB.getParent()->getRegInfo(), SrcOp.getReg())) {2174 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst)2175 .addImm(SISrcMods::OP_SEL_1) // src0_mod2176 .addReg(SrcOp.getReg())2177 .addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) // src1_mod2178 .addReg(SrcOp.getReg())2179 .addImm(0) // op_sel_lo2180 .addImm(0) // op_sel_hi2181 .addImm(0) // neg_lo2182 .addImm(0) // neg_hi2183 .addImm(0); // clamp2184 } else {2185 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)2186 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0))2187 .addReg(Dst, RegState::Implicit | RegState::Define);2188 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)2189 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1))2190 .addReg(Dst, RegState::Implicit | RegState::Define);2191 }2192 }2193 MI.eraseFromParent();2194 break;2195 }2196 case AMDGPU::V_MOV_B64_DPP_PSEUDO: {2197 expandMovDPP64(MI);2198 break;2199 }2200 case AMDGPU::S_MOV_B64_IMM_PSEUDO: {2201 const MachineOperand &SrcOp = MI.getOperand(1);2202 assert(!SrcOp.isFPImm());2203 2204 if (ST.has64BitLiterals()) {2205 MI.setDesc(get(AMDGPU::S_MOV_B64));2206 break;2207 }2208 2209 APInt Imm(64, SrcOp.getImm());2210 if (Imm.isIntN(32) || isInlineConstant(Imm)) {2211 MI.setDesc(get(AMDGPU::S_MOV_B64));2212 break;2213 }2214 2215 Register Dst = MI.getOperand(0).getReg();2216 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);2217 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);2218 2219 APInt Lo(32, Imm.getLoBits(32).getZExtValue());2220 APInt Hi(32, Imm.getHiBits(32).getZExtValue());2221 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstLo)2222 .addImm(Lo.getSExtValue())2223 .addReg(Dst, RegState::Implicit | RegState::Define);2224 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstHi)2225 .addImm(Hi.getSExtValue())2226 .addReg(Dst, RegState::Implicit | RegState::Define);2227 MI.eraseFromParent();2228 break;2229 }2230 case AMDGPU::V_SET_INACTIVE_B32: {2231 // Lower V_SET_INACTIVE_B32 to V_CNDMASK_B32.2232 Register DstReg = MI.getOperand(0).getReg();2233 BuildMI(MBB, MI, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)2234 .add(MI.getOperand(3))2235 .add(MI.getOperand(4))2236 .add(MI.getOperand(1))2237 .add(MI.getOperand(2))2238 .add(MI.getOperand(5));2239 MI.eraseFromParent();2240 break;2241 }2242 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1:2243 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2:2244 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3:2245 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4:2246 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5:2247 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8:2248 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9:2249 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10:2250 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11:2251 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12:2252 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16:2253 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32:2254 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1:2255 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2:2256 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3:2257 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4:2258 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5:2259 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8:2260 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9:2261 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10:2262 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11:2263 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12:2264 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16:2265 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32:2266 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1:2267 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2:2268 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4:2269 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8:2270 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: {2271 const TargetRegisterClass *EltRC = getOpRegClass(MI, 2);2272 2273 unsigned Opc;2274 if (RI.hasVGPRs(EltRC)) {2275 Opc = AMDGPU::V_MOVRELD_B32_e32;2276 } else {2277 Opc = RI.getRegSizeInBits(*EltRC) == 64 ? AMDGPU::S_MOVRELD_B642278 : AMDGPU::S_MOVRELD_B32;2279 }2280 2281 const MCInstrDesc &OpDesc = get(Opc);2282 Register VecReg = MI.getOperand(0).getReg();2283 bool IsUndef = MI.getOperand(1).isUndef();2284 unsigned SubReg = MI.getOperand(3).getImm();2285 assert(VecReg == MI.getOperand(1).getReg());2286 2287 MachineInstrBuilder MIB =2288 BuildMI(MBB, MI, DL, OpDesc)2289 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)2290 .add(MI.getOperand(2))2291 .addReg(VecReg, RegState::ImplicitDefine)2292 .addReg(VecReg, RegState::Implicit | (IsUndef ? RegState::Undef : 0));2293 2294 const int ImpDefIdx =2295 OpDesc.getNumOperands() + OpDesc.implicit_uses().size();2296 const int ImpUseIdx = ImpDefIdx + 1;2297 MIB->tieOperands(ImpDefIdx, ImpUseIdx);2298 MI.eraseFromParent();2299 break;2300 }2301 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1:2302 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2:2303 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3:2304 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4:2305 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5:2306 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8:2307 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9:2308 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10:2309 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11:2310 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12:2311 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16:2312 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: {2313 assert(ST.useVGPRIndexMode());2314 Register VecReg = MI.getOperand(0).getReg();2315 bool IsUndef = MI.getOperand(1).isUndef();2316 MachineOperand &Idx = MI.getOperand(3);2317 Register SubReg = MI.getOperand(4).getImm();2318 2319 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON))2320 .add(Idx)2321 .addImm(AMDGPU::VGPRIndexMode::DST_ENABLE);2322 SetOn->getOperand(3).setIsUndef();2323 2324 const MCInstrDesc &OpDesc = get(AMDGPU::V_MOV_B32_indirect_write);2325 MachineInstrBuilder MIB =2326 BuildMI(MBB, MI, DL, OpDesc)2327 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)2328 .add(MI.getOperand(2))2329 .addReg(VecReg, RegState::ImplicitDefine)2330 .addReg(VecReg,2331 RegState::Implicit | (IsUndef ? RegState::Undef : 0));2332 2333 const int ImpDefIdx =2334 OpDesc.getNumOperands() + OpDesc.implicit_uses().size();2335 const int ImpUseIdx = ImpDefIdx + 1;2336 MIB->tieOperands(ImpDefIdx, ImpUseIdx);2337 2338 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF));2339 2340 finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator()));2341 2342 MI.eraseFromParent();2343 break;2344 }2345 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1:2346 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2:2347 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3:2348 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4:2349 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5:2350 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8:2351 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9:2352 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10:2353 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11:2354 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12:2355 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16:2356 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: {2357 assert(ST.useVGPRIndexMode());2358 Register Dst = MI.getOperand(0).getReg();2359 Register VecReg = MI.getOperand(1).getReg();2360 bool IsUndef = MI.getOperand(1).isUndef();2361 Register Idx = MI.getOperand(2).getReg();2362 Register SubReg = MI.getOperand(3).getImm();2363 2364 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON))2365 .addReg(Idx)2366 .addImm(AMDGPU::VGPRIndexMode::SRC0_ENABLE);2367 SetOn->getOperand(3).setIsUndef();2368 2369 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_indirect_read))2370 .addDef(Dst)2371 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)2372 .addReg(VecReg, RegState::Implicit | (IsUndef ? RegState::Undef : 0));2373 2374 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF));2375 2376 finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator()));2377 2378 MI.eraseFromParent();2379 break;2380 }2381 case AMDGPU::SI_PC_ADD_REL_OFFSET: {2382 MachineFunction &MF = *MBB.getParent();2383 Register Reg = MI.getOperand(0).getReg();2384 Register RegLo = RI.getSubReg(Reg, AMDGPU::sub0);2385 Register RegHi = RI.getSubReg(Reg, AMDGPU::sub1);2386 MachineOperand OpLo = MI.getOperand(1);2387 MachineOperand OpHi = MI.getOperand(2);2388 2389 // Create a bundle so these instructions won't be re-ordered by the2390 // post-RA scheduler.2391 MIBundleBuilder Bundler(MBB, MI);2392 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg));2393 2394 // What we want here is an offset from the value returned by s_getpc (which2395 // is the address of the s_add_u32 instruction) to the global variable, but2396 // since the encoding of $symbol starts 4 bytes after the start of the2397 // s_add_u32 instruction, we end up with an offset that is 4 bytes too2398 // small. This requires us to add 4 to the global variable offset in order2399 // to compute the correct address. Similarly for the s_addc_u32 instruction,2400 // the encoding of $symbol starts 12 bytes after the start of the s_add_u322401 // instruction.2402 2403 int64_t Adjust = 0;2404 if (ST.hasGetPCZeroExtension()) {2405 // Fix up hardware that does not sign-extend the 48-bit PC value by2406 // inserting: s_sext_i32_i16 reghi, reghi2407 Bundler.append(2408 BuildMI(MF, DL, get(AMDGPU::S_SEXT_I32_I16), RegHi).addReg(RegHi));2409 Adjust += 4;2410 }2411 2412 if (OpLo.isGlobal())2413 OpLo.setOffset(OpLo.getOffset() + Adjust + 4);2414 Bundler.append(2415 BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo).addReg(RegLo).add(OpLo));2416 2417 if (OpHi.isGlobal())2418 OpHi.setOffset(OpHi.getOffset() + Adjust + 12);2419 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi)2420 .addReg(RegHi)2421 .add(OpHi));2422 2423 finalizeBundle(MBB, Bundler.begin());2424 2425 MI.eraseFromParent();2426 break;2427 }2428 case AMDGPU::SI_PC_ADD_REL_OFFSET64: {2429 MachineFunction &MF = *MBB.getParent();2430 Register Reg = MI.getOperand(0).getReg();2431 MachineOperand Op = MI.getOperand(1);2432 2433 // Create a bundle so these instructions won't be re-ordered by the2434 // post-RA scheduler.2435 MIBundleBuilder Bundler(MBB, MI);2436 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg));2437 if (Op.isGlobal())2438 Op.setOffset(Op.getOffset() + 4);2439 Bundler.append(2440 BuildMI(MF, DL, get(AMDGPU::S_ADD_U64), Reg).addReg(Reg).add(Op));2441 2442 finalizeBundle(MBB, Bundler.begin());2443 2444 MI.eraseFromParent();2445 break;2446 }2447 case AMDGPU::ENTER_STRICT_WWM: {2448 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when2449 // Whole Wave Mode is entered.2450 MI.setDesc(get(LMC.OrSaveExecOpc));2451 break;2452 }2453 case AMDGPU::ENTER_STRICT_WQM: {2454 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when2455 // STRICT_WQM is entered.2456 BuildMI(MBB, MI, DL, get(LMC.MovOpc), MI.getOperand(0).getReg())2457 .addReg(LMC.ExecReg);2458 BuildMI(MBB, MI, DL, get(LMC.WQMOpc), LMC.ExecReg).addReg(LMC.ExecReg);2459 2460 MI.eraseFromParent();2461 break;2462 }2463 case AMDGPU::EXIT_STRICT_WWM:2464 case AMDGPU::EXIT_STRICT_WQM: {2465 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when2466 // WWM/STICT_WQM is exited.2467 MI.setDesc(get(LMC.MovOpc));2468 break;2469 }2470 case AMDGPU::SI_RETURN: {2471 const MachineFunction *MF = MBB.getParent();2472 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();2473 const SIRegisterInfo *TRI = ST.getRegisterInfo();2474 // Hiding the return address use with SI_RETURN may lead to extra kills in2475 // the function and missing live-ins. We are fine in practice because callee2476 // saved register handling ensures the register value is restored before2477 // RET, but we need the undef flag here to appease the MachineVerifier2478 // liveness checks.2479 MachineInstrBuilder MIB =2480 BuildMI(MBB, MI, DL, get(AMDGPU::S_SETPC_B64_return))2481 .addReg(TRI->getReturnAddressReg(*MF), RegState::Undef);2482 2483 MIB.copyImplicitOps(MI);2484 MI.eraseFromParent();2485 break;2486 }2487 2488 case AMDGPU::S_MUL_U64_U32_PSEUDO:2489 case AMDGPU::S_MUL_I64_I32_PSEUDO:2490 MI.setDesc(get(AMDGPU::S_MUL_U64));2491 break;2492 2493 case AMDGPU::S_GETPC_B64_pseudo:2494 MI.setDesc(get(AMDGPU::S_GETPC_B64));2495 if (ST.hasGetPCZeroExtension()) {2496 Register Dst = MI.getOperand(0).getReg();2497 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);2498 // Fix up hardware that does not sign-extend the 48-bit PC value by2499 // inserting: s_sext_i32_i16 dsthi, dsthi2500 BuildMI(MBB, std::next(MI.getIterator()), DL, get(AMDGPU::S_SEXT_I32_I16),2501 DstHi)2502 .addReg(DstHi);2503 }2504 break;2505 2506 case AMDGPU::V_MAX_BF16_PSEUDO_e64:2507 assert(ST.hasBF16PackedInsts());2508 MI.setDesc(get(AMDGPU::V_PK_MAX_NUM_BF16));2509 MI.addOperand(MachineOperand::CreateImm(0)); // op_sel2510 MI.addOperand(MachineOperand::CreateImm(0)); // neg_lo2511 MI.addOperand(MachineOperand::CreateImm(0)); // neg_hi2512 auto Op0 = getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);2513 Op0->setImm(Op0->getImm() | SISrcMods::OP_SEL_1);2514 auto Op1 = getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);2515 Op1->setImm(Op1->getImm() | SISrcMods::OP_SEL_1);2516 break;2517 }2518 2519 return true;2520}2521 2522void SIInstrInfo::reMaterialize(MachineBasicBlock &MBB,2523 MachineBasicBlock::iterator I, Register DestReg,2524 unsigned SubIdx,2525 const MachineInstr &Orig) const {2526 2527 // Try shrinking the instruction to remat only the part needed for current2528 // context.2529 // TODO: Handle more cases.2530 unsigned Opcode = Orig.getOpcode();2531 switch (Opcode) {2532 case AMDGPU::S_LOAD_DWORDX16_IMM:2533 case AMDGPU::S_LOAD_DWORDX8_IMM: {2534 if (SubIdx != 0)2535 break;2536 2537 if (I == MBB.end())2538 break;2539 2540 if (I->isBundled())2541 break;2542 2543 // Look for a single use of the register that is also a subreg.2544 Register RegToFind = Orig.getOperand(0).getReg();2545 MachineOperand *UseMO = nullptr;2546 for (auto &CandMO : I->operands()) {2547 if (!CandMO.isReg() || CandMO.getReg() != RegToFind || CandMO.isDef())2548 continue;2549 if (UseMO) {2550 UseMO = nullptr;2551 break;2552 }2553 UseMO = &CandMO;2554 }2555 if (!UseMO || UseMO->getSubReg() == AMDGPU::NoSubRegister)2556 break;2557 2558 unsigned Offset = RI.getSubRegIdxOffset(UseMO->getSubReg());2559 unsigned SubregSize = RI.getSubRegIdxSize(UseMO->getSubReg());2560 2561 MachineFunction *MF = MBB.getParent();2562 MachineRegisterInfo &MRI = MF->getRegInfo();2563 assert(MRI.use_nodbg_empty(DestReg) && "DestReg should have no users yet.");2564 2565 unsigned NewOpcode = -1;2566 if (SubregSize == 256)2567 NewOpcode = AMDGPU::S_LOAD_DWORDX8_IMM;2568 else if (SubregSize == 128)2569 NewOpcode = AMDGPU::S_LOAD_DWORDX4_IMM;2570 else2571 break;2572 2573 const MCInstrDesc &TID = get(NewOpcode);2574 const TargetRegisterClass *NewRC =2575 RI.getAllocatableClass(getRegClass(TID, 0));2576 MRI.setRegClass(DestReg, NewRC);2577 2578 UseMO->setReg(DestReg);2579 UseMO->setSubReg(AMDGPU::NoSubRegister);2580 2581 // Use a smaller load with the desired size, possibly with updated offset.2582 MachineInstr *MI = MF->CloneMachineInstr(&Orig);2583 MI->setDesc(TID);2584 MI->getOperand(0).setReg(DestReg);2585 MI->getOperand(0).setSubReg(AMDGPU::NoSubRegister);2586 if (Offset) {2587 MachineOperand *OffsetMO = getNamedOperand(*MI, AMDGPU::OpName::offset);2588 int64_t FinalOffset = OffsetMO->getImm() + Offset / 8;2589 OffsetMO->setImm(FinalOffset);2590 }2591 SmallVector<MachineMemOperand *> NewMMOs;2592 for (const MachineMemOperand *MemOp : Orig.memoperands())2593 NewMMOs.push_back(MF->getMachineMemOperand(MemOp, MemOp->getPointerInfo(),2594 SubregSize / 8));2595 MI->setMemRefs(*MF, NewMMOs);2596 2597 MBB.insert(I, MI);2598 return;2599 }2600 2601 default:2602 break;2603 }2604 2605 TargetInstrInfo::reMaterialize(MBB, I, DestReg, SubIdx, Orig);2606}2607 2608std::pair<MachineInstr*, MachineInstr*>2609SIInstrInfo::expandMovDPP64(MachineInstr &MI) const {2610 assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);2611 2612 if (ST.hasMovB64() && ST.hasFeature(AMDGPU::FeatureDPALU_DPP) &&2613 AMDGPU::isLegalDPALU_DPPControl(2614 ST, getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl)->getImm())) {2615 MI.setDesc(get(AMDGPU::V_MOV_B64_dpp));2616 return std::pair(&MI, nullptr);2617 }2618 2619 MachineBasicBlock &MBB = *MI.getParent();2620 DebugLoc DL = MBB.findDebugLoc(MI);2621 MachineFunction *MF = MBB.getParent();2622 MachineRegisterInfo &MRI = MF->getRegInfo();2623 Register Dst = MI.getOperand(0).getReg();2624 unsigned Part = 0;2625 MachineInstr *Split[2];2626 2627 for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) {2628 auto MovDPP = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_dpp));2629 if (Dst.isPhysical()) {2630 MovDPP.addDef(RI.getSubReg(Dst, Sub));2631 } else {2632 assert(MRI.isSSA());2633 auto Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);2634 MovDPP.addDef(Tmp);2635 }2636 2637 for (unsigned I = 1; I <= 2; ++I) { // old and src operands.2638 const MachineOperand &SrcOp = MI.getOperand(I);2639 assert(!SrcOp.isFPImm());2640 if (SrcOp.isImm()) {2641 APInt Imm(64, SrcOp.getImm());2642 Imm.ashrInPlace(Part * 32);2643 MovDPP.addImm(Imm.getLoBits(32).getZExtValue());2644 } else {2645 assert(SrcOp.isReg());2646 Register Src = SrcOp.getReg();2647 if (Src.isPhysical())2648 MovDPP.addReg(RI.getSubReg(Src, Sub));2649 else2650 MovDPP.addReg(Src, SrcOp.isUndef() ? RegState::Undef : 0, Sub);2651 }2652 }2653 2654 for (const MachineOperand &MO : llvm::drop_begin(MI.explicit_operands(), 3))2655 MovDPP.addImm(MO.getImm());2656 2657 Split[Part] = MovDPP;2658 ++Part;2659 }2660 2661 if (Dst.isVirtual())2662 BuildMI(MBB, MI, DL, get(AMDGPU::REG_SEQUENCE), Dst)2663 .addReg(Split[0]->getOperand(0).getReg())2664 .addImm(AMDGPU::sub0)2665 .addReg(Split[1]->getOperand(0).getReg())2666 .addImm(AMDGPU::sub1);2667 2668 MI.eraseFromParent();2669 return std::pair(Split[0], Split[1]);2670}2671 2672std::optional<DestSourcePair>2673SIInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {2674 if (MI.getOpcode() == AMDGPU::WWM_COPY)2675 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};2676 2677 return std::nullopt;2678}2679 2680bool SIInstrInfo::swapSourceModifiers(MachineInstr &MI, MachineOperand &Src0,2681 AMDGPU::OpName Src0OpName,2682 MachineOperand &Src1,2683 AMDGPU::OpName Src1OpName) const {2684 MachineOperand *Src0Mods = getNamedOperand(MI, Src0OpName);2685 if (!Src0Mods)2686 return false;2687 2688 MachineOperand *Src1Mods = getNamedOperand(MI, Src1OpName);2689 assert(Src1Mods &&2690 "All commutable instructions have both src0 and src1 modifiers");2691 2692 int Src0ModsVal = Src0Mods->getImm();2693 int Src1ModsVal = Src1Mods->getImm();2694 2695 Src1Mods->setImm(Src0ModsVal);2696 Src0Mods->setImm(Src1ModsVal);2697 return true;2698}2699 2700static MachineInstr *swapRegAndNonRegOperand(MachineInstr &MI,2701 MachineOperand &RegOp,2702 MachineOperand &NonRegOp) {2703 Register Reg = RegOp.getReg();2704 unsigned SubReg = RegOp.getSubReg();2705 bool IsKill = RegOp.isKill();2706 bool IsDead = RegOp.isDead();2707 bool IsUndef = RegOp.isUndef();2708 bool IsDebug = RegOp.isDebug();2709 2710 if (NonRegOp.isImm())2711 RegOp.ChangeToImmediate(NonRegOp.getImm());2712 else if (NonRegOp.isFI())2713 RegOp.ChangeToFrameIndex(NonRegOp.getIndex());2714 else if (NonRegOp.isGlobal()) {2715 RegOp.ChangeToGA(NonRegOp.getGlobal(), NonRegOp.getOffset(),2716 NonRegOp.getTargetFlags());2717 } else2718 return nullptr;2719 2720 // Make sure we don't reinterpret a subreg index in the target flags.2721 RegOp.setTargetFlags(NonRegOp.getTargetFlags());2722 2723 NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug);2724 NonRegOp.setSubReg(SubReg);2725 2726 return &MI;2727}2728 2729static MachineInstr *swapImmOperands(MachineInstr &MI,2730 MachineOperand &NonRegOp1,2731 MachineOperand &NonRegOp2) {2732 unsigned TargetFlags = NonRegOp1.getTargetFlags();2733 int64_t NonRegVal = NonRegOp1.getImm();2734 2735 NonRegOp1.setImm(NonRegOp2.getImm());2736 NonRegOp2.setImm(NonRegVal);2737 NonRegOp1.setTargetFlags(NonRegOp2.getTargetFlags());2738 NonRegOp2.setTargetFlags(TargetFlags);2739 return &MI;2740}2741 2742bool SIInstrInfo::isLegalToSwap(const MachineInstr &MI, unsigned OpIdx0,2743 unsigned OpIdx1) const {2744 const MCInstrDesc &InstDesc = MI.getDesc();2745 const MCOperandInfo &OpInfo0 = InstDesc.operands()[OpIdx0];2746 const MCOperandInfo &OpInfo1 = InstDesc.operands()[OpIdx1];2747 2748 unsigned Opc = MI.getOpcode();2749 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);2750 2751 const MachineOperand &MO0 = MI.getOperand(OpIdx0);2752 const MachineOperand &MO1 = MI.getOperand(OpIdx1);2753 2754 // Swap doesn't breach constant bus or literal limits2755 // It may move literal to position other than src0, this is not allowed2756 // pre-gfx10 However, most test cases need literals in Src0 for VOP2757 // FIXME: After gfx9, literal can be in place other than Src02758 if (isVALU(MI)) {2759 if ((int)OpIdx0 == Src0Idx && !MO0.isReg() &&2760 !isInlineConstant(MO0, OpInfo1))2761 return false;2762 if ((int)OpIdx1 == Src0Idx && !MO1.isReg() &&2763 !isInlineConstant(MO1, OpInfo0))2764 return false;2765 }2766 2767 if ((int)OpIdx1 != Src0Idx && MO0.isReg()) {2768 if (OpInfo1.RegClass == -1)2769 return OpInfo1.OperandType == MCOI::OPERAND_UNKNOWN;2770 return isLegalRegOperand(MI, OpIdx1, MO0) &&2771 (!MO1.isReg() || isLegalRegOperand(MI, OpIdx0, MO1));2772 }2773 if ((int)OpIdx0 != Src0Idx && MO1.isReg()) {2774 if (OpInfo0.RegClass == -1)2775 return OpInfo0.OperandType == MCOI::OPERAND_UNKNOWN;2776 return (!MO0.isReg() || isLegalRegOperand(MI, OpIdx1, MO0)) &&2777 isLegalRegOperand(MI, OpIdx0, MO1);2778 }2779 2780 // No need to check 64-bit literals since swapping does not bring new2781 // 64-bit literals into current instruction to fold to 32-bit2782 2783 return isImmOperandLegal(MI, OpIdx1, MO0);2784}2785 2786MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,2787 unsigned Src0Idx,2788 unsigned Src1Idx) const {2789 assert(!NewMI && "this should never be used");2790 2791 unsigned Opc = MI.getOpcode();2792 int CommutedOpcode = commuteOpcode(Opc);2793 if (CommutedOpcode == -1)2794 return nullptr;2795 2796 if (Src0Idx > Src1Idx)2797 std::swap(Src0Idx, Src1Idx);2798 2799 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==2800 static_cast<int>(Src0Idx) &&2801 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==2802 static_cast<int>(Src1Idx) &&2803 "inconsistency with findCommutedOpIndices");2804 2805 if (!isLegalToSwap(MI, Src0Idx, Src1Idx))2806 return nullptr;2807 2808 MachineInstr *CommutedMI = nullptr;2809 MachineOperand &Src0 = MI.getOperand(Src0Idx);2810 MachineOperand &Src1 = MI.getOperand(Src1Idx);2811 if (Src0.isReg() && Src1.isReg()) {2812 // Be sure to copy the source modifiers to the right place.2813 CommutedMI =2814 TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx);2815 } else if (Src0.isReg() && !Src1.isReg()) {2816 CommutedMI = swapRegAndNonRegOperand(MI, Src0, Src1);2817 } else if (!Src0.isReg() && Src1.isReg()) {2818 CommutedMI = swapRegAndNonRegOperand(MI, Src1, Src0);2819 } else if (Src0.isImm() && Src1.isImm()) {2820 CommutedMI = swapImmOperands(MI, Src0, Src1);2821 } else {2822 // FIXME: Found two non registers to commute. This does happen.2823 return nullptr;2824 }2825 2826 if (CommutedMI) {2827 swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers,2828 Src1, AMDGPU::OpName::src1_modifiers);2829 2830 swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_sel, Src1,2831 AMDGPU::OpName::src1_sel);2832 2833 CommutedMI->setDesc(get(CommutedOpcode));2834 }2835 2836 return CommutedMI;2837}2838 2839// This needs to be implemented because the source modifiers may be inserted2840// between the true commutable operands, and the base2841// TargetInstrInfo::commuteInstruction uses it.2842bool SIInstrInfo::findCommutedOpIndices(const MachineInstr &MI,2843 unsigned &SrcOpIdx0,2844 unsigned &SrcOpIdx1) const {2845 return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1);2846}2847 2848bool SIInstrInfo::findCommutedOpIndices(const MCInstrDesc &Desc,2849 unsigned &SrcOpIdx0,2850 unsigned &SrcOpIdx1) const {2851 if (!Desc.isCommutable())2852 return false;2853 2854 unsigned Opc = Desc.getOpcode();2855 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);2856 if (Src0Idx == -1)2857 return false;2858 2859 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);2860 if (Src1Idx == -1)2861 return false;2862 2863 return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx);2864}2865 2866bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp,2867 int64_t BrOffset) const {2868 // BranchRelaxation should never have to check s_setpc_b64 or s_add_pc_i642869 // because its dest block is unanalyzable.2870 assert(isSOPP(BranchOp) || isSOPK(BranchOp));2871 2872 // Convert to dwords.2873 BrOffset /= 4;2874 2875 // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is2876 // from the next instruction.2877 BrOffset -= 1;2878 2879 return isIntN(BranchOffsetBits, BrOffset);2880}2881 2882MachineBasicBlock *2883SIInstrInfo::getBranchDestBlock(const MachineInstr &MI) const {2884 return MI.getOperand(0).getMBB();2885}2886 2887bool SIInstrInfo::hasDivergentBranch(const MachineBasicBlock *MBB) const {2888 for (const MachineInstr &MI : MBB->terminators()) {2889 if (MI.getOpcode() == AMDGPU::SI_IF || MI.getOpcode() == AMDGPU::SI_ELSE ||2890 MI.getOpcode() == AMDGPU::SI_LOOP)2891 return true;2892 }2893 return false;2894}2895 2896void SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,2897 MachineBasicBlock &DestBB,2898 MachineBasicBlock &RestoreBB,2899 const DebugLoc &DL, int64_t BrOffset,2900 RegScavenger *RS) const {2901 assert(MBB.empty() &&2902 "new block should be inserted for expanding unconditional branch");2903 assert(MBB.pred_size() == 1);2904 assert(RestoreBB.empty() &&2905 "restore block should be inserted for restoring clobbered registers");2906 2907 MachineFunction *MF = MBB.getParent();2908 MachineRegisterInfo &MRI = MF->getRegInfo();2909 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();2910 auto I = MBB.end();2911 auto &MCCtx = MF->getContext();2912 2913 if (ST.hasAddPC64Inst()) {2914 MCSymbol *Offset =2915 MCCtx.createTempSymbol("offset", /*AlwaysAddSuffix=*/true);2916 auto AddPC = BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_PC_I64))2917 .addSym(Offset, MO_FAR_BRANCH_OFFSET);2918 MCSymbol *PostAddPCLabel =2919 MCCtx.createTempSymbol("post_addpc", /*AlwaysAddSuffix=*/true);2920 AddPC->setPostInstrSymbol(*MF, PostAddPCLabel);2921 auto *OffsetExpr = MCBinaryExpr::createSub(2922 MCSymbolRefExpr::create(DestBB.getSymbol(), MCCtx),2923 MCSymbolRefExpr::create(PostAddPCLabel, MCCtx), MCCtx);2924 Offset->setVariableValue(OffsetExpr);2925 return;2926 }2927 2928 assert(RS && "RegScavenger required for long branching");2929 2930 // FIXME: Virtual register workaround for RegScavenger not working with empty2931 // blocks.2932 Register PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);2933 2934 // Note: as this is used after hazard recognizer we need to apply some hazard2935 // workarounds directly.2936 const bool FlushSGPRWrites = (ST.isWave64() && ST.hasVALUMaskWriteHazard()) ||2937 ST.hasVALUReadSGPRHazard();2938 auto ApplyHazardWorkarounds = [this, &MBB, &I, &DL, FlushSGPRWrites]() {2939 if (FlushSGPRWrites)2940 BuildMI(MBB, I, DL, get(AMDGPU::S_WAITCNT_DEPCTR))2941 .addImm(AMDGPU::DepCtr::encodeFieldSaSdst(0, ST));2942 };2943 2944 // We need to compute the offset relative to the instruction immediately after2945 // s_getpc_b64. Insert pc arithmetic code before last terminator.2946 MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg);2947 ApplyHazardWorkarounds();2948 2949 MCSymbol *PostGetPCLabel =2950 MCCtx.createTempSymbol("post_getpc", /*AlwaysAddSuffix=*/true);2951 GetPC->setPostInstrSymbol(*MF, PostGetPCLabel);2952 2953 MCSymbol *OffsetLo =2954 MCCtx.createTempSymbol("offset_lo", /*AlwaysAddSuffix=*/true);2955 MCSymbol *OffsetHi =2956 MCCtx.createTempSymbol("offset_hi", /*AlwaysAddSuffix=*/true);2957 BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32))2958 .addReg(PCReg, RegState::Define, AMDGPU::sub0)2959 .addReg(PCReg, 0, AMDGPU::sub0)2960 .addSym(OffsetLo, MO_FAR_BRANCH_OFFSET);2961 BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32))2962 .addReg(PCReg, RegState::Define, AMDGPU::sub1)2963 .addReg(PCReg, 0, AMDGPU::sub1)2964 .addSym(OffsetHi, MO_FAR_BRANCH_OFFSET);2965 ApplyHazardWorkarounds();2966 2967 // Insert the indirect branch after the other terminator.2968 BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64))2969 .addReg(PCReg);2970 2971 // If a spill is needed for the pc register pair, we need to insert a spill2972 // restore block right before the destination block, and insert a short branch2973 // into the old destination block's fallthrough predecessor.2974 // e.g.:2975 //2976 // s_cbranch_scc0 skip_long_branch:2977 //2978 // long_branch_bb:2979 // spill s[8:9]2980 // s_getpc_b64 s[8:9]2981 // s_add_u32 s8, s8, restore_bb2982 // s_addc_u32 s9, s9, 02983 // s_setpc_b64 s[8:9]2984 //2985 // skip_long_branch:2986 // foo;2987 //2988 // .....2989 //2990 // dest_bb_fallthrough_predecessor:2991 // bar;2992 // s_branch dest_bb2993 //2994 // restore_bb:2995 // restore s[8:9]2996 // fallthrough dest_bb2997 ///2998 // dest_bb:2999 // buzz;3000 3001 Register LongBranchReservedReg = MFI->getLongBranchReservedReg();3002 Register Scav;3003 3004 // If we've previously reserved a register for long branches3005 // avoid running the scavenger and just use those registers3006 if (LongBranchReservedReg) {3007 RS->enterBasicBlock(MBB);3008 Scav = LongBranchReservedReg;3009 } else {3010 RS->enterBasicBlockEnd(MBB);3011 Scav = RS->scavengeRegisterBackwards(3012 AMDGPU::SReg_64RegClass, MachineBasicBlock::iterator(GetPC),3013 /* RestoreAfter */ false, 0, /* AllowSpill */ false);3014 }3015 if (Scav) {3016 RS->setRegUsed(Scav);3017 MRI.replaceRegWith(PCReg, Scav);3018 MRI.clearVirtRegs();3019 } else {3020 // As SGPR needs VGPR to be spilled, we reuse the slot of temporary VGPR for3021 // SGPR spill.3022 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();3023 const SIRegisterInfo *TRI = ST.getRegisterInfo();3024 TRI->spillEmergencySGPR(GetPC, RestoreBB, AMDGPU::SGPR0_SGPR1, RS);3025 MRI.replaceRegWith(PCReg, AMDGPU::SGPR0_SGPR1);3026 MRI.clearVirtRegs();3027 }3028 3029 MCSymbol *DestLabel = Scav ? DestBB.getSymbol() : RestoreBB.getSymbol();3030 // Now, the distance could be defined.3031 auto *Offset = MCBinaryExpr::createSub(3032 MCSymbolRefExpr::create(DestLabel, MCCtx),3033 MCSymbolRefExpr::create(PostGetPCLabel, MCCtx), MCCtx);3034 // Add offset assignments.3035 auto *Mask = MCConstantExpr::create(0xFFFFFFFFULL, MCCtx);3036 OffsetLo->setVariableValue(MCBinaryExpr::createAnd(Offset, Mask, MCCtx));3037 auto *ShAmt = MCConstantExpr::create(32, MCCtx);3038 OffsetHi->setVariableValue(MCBinaryExpr::createAShr(Offset, ShAmt, MCCtx));3039}3040 3041unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {3042 switch (Cond) {3043 case SIInstrInfo::SCC_TRUE:3044 return AMDGPU::S_CBRANCH_SCC1;3045 case SIInstrInfo::SCC_FALSE:3046 return AMDGPU::S_CBRANCH_SCC0;3047 case SIInstrInfo::VCCNZ:3048 return AMDGPU::S_CBRANCH_VCCNZ;3049 case SIInstrInfo::VCCZ:3050 return AMDGPU::S_CBRANCH_VCCZ;3051 case SIInstrInfo::EXECNZ:3052 return AMDGPU::S_CBRANCH_EXECNZ;3053 case SIInstrInfo::EXECZ:3054 return AMDGPU::S_CBRANCH_EXECZ;3055 default:3056 llvm_unreachable("invalid branch predicate");3057 }3058}3059 3060SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {3061 switch (Opcode) {3062 case AMDGPU::S_CBRANCH_SCC0:3063 return SCC_FALSE;3064 case AMDGPU::S_CBRANCH_SCC1:3065 return SCC_TRUE;3066 case AMDGPU::S_CBRANCH_VCCNZ:3067 return VCCNZ;3068 case AMDGPU::S_CBRANCH_VCCZ:3069 return VCCZ;3070 case AMDGPU::S_CBRANCH_EXECNZ:3071 return EXECNZ;3072 case AMDGPU::S_CBRANCH_EXECZ:3073 return EXECZ;3074 default:3075 return INVALID_BR;3076 }3077}3078 3079bool SIInstrInfo::analyzeBranchImpl(MachineBasicBlock &MBB,3080 MachineBasicBlock::iterator I,3081 MachineBasicBlock *&TBB,3082 MachineBasicBlock *&FBB,3083 SmallVectorImpl<MachineOperand> &Cond,3084 bool AllowModify) const {3085 if (I->getOpcode() == AMDGPU::S_BRANCH) {3086 // Unconditional Branch3087 TBB = I->getOperand(0).getMBB();3088 return false;3089 }3090 3091 BranchPredicate Pred = getBranchPredicate(I->getOpcode());3092 if (Pred == INVALID_BR)3093 return true;3094 3095 MachineBasicBlock *CondBB = I->getOperand(0).getMBB();3096 Cond.push_back(MachineOperand::CreateImm(Pred));3097 Cond.push_back(I->getOperand(1)); // Save the branch register.3098 3099 ++I;3100 3101 if (I == MBB.end()) {3102 // Conditional branch followed by fall-through.3103 TBB = CondBB;3104 return false;3105 }3106 3107 if (I->getOpcode() == AMDGPU::S_BRANCH) {3108 TBB = CondBB;3109 FBB = I->getOperand(0).getMBB();3110 return false;3111 }3112 3113 return true;3114}3115 3116bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,3117 MachineBasicBlock *&FBB,3118 SmallVectorImpl<MachineOperand> &Cond,3119 bool AllowModify) const {3120 MachineBasicBlock::iterator I = MBB.getFirstTerminator();3121 auto E = MBB.end();3122 if (I == E)3123 return false;3124 3125 // Skip over the instructions that are artificially terminators for special3126 // exec management.3127 while (I != E && !I->isBranch() && !I->isReturn()) {3128 switch (I->getOpcode()) {3129 case AMDGPU::S_MOV_B64_term:3130 case AMDGPU::S_XOR_B64_term:3131 case AMDGPU::S_OR_B64_term:3132 case AMDGPU::S_ANDN2_B64_term:3133 case AMDGPU::S_AND_B64_term:3134 case AMDGPU::S_AND_SAVEEXEC_B64_term:3135 case AMDGPU::S_MOV_B32_term:3136 case AMDGPU::S_XOR_B32_term:3137 case AMDGPU::S_OR_B32_term:3138 case AMDGPU::S_ANDN2_B32_term:3139 case AMDGPU::S_AND_B32_term:3140 case AMDGPU::S_AND_SAVEEXEC_B32_term:3141 break;3142 case AMDGPU::SI_IF:3143 case AMDGPU::SI_ELSE:3144 case AMDGPU::SI_KILL_I1_TERMINATOR:3145 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:3146 // FIXME: It's messy that these need to be considered here at all.3147 return true;3148 default:3149 llvm_unreachable("unexpected non-branch terminator inst");3150 }3151 3152 ++I;3153 }3154 3155 if (I == E)3156 return false;3157 3158 return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);3159}3160 3161unsigned SIInstrInfo::removeBranch(MachineBasicBlock &MBB,3162 int *BytesRemoved) const {3163 unsigned Count = 0;3164 unsigned RemovedSize = 0;3165 for (MachineInstr &MI : llvm::make_early_inc_range(MBB.terminators())) {3166 // Skip over artificial terminators when removing instructions.3167 if (MI.isBranch() || MI.isReturn()) {3168 RemovedSize += getInstSizeInBytes(MI);3169 MI.eraseFromParent();3170 ++Count;3171 }3172 }3173 3174 if (BytesRemoved)3175 *BytesRemoved = RemovedSize;3176 3177 return Count;3178}3179 3180// Copy the flags onto the implicit condition register operand.3181static void preserveCondRegFlags(MachineOperand &CondReg,3182 const MachineOperand &OrigCond) {3183 CondReg.setIsUndef(OrigCond.isUndef());3184 CondReg.setIsKill(OrigCond.isKill());3185}3186 3187unsigned SIInstrInfo::insertBranch(MachineBasicBlock &MBB,3188 MachineBasicBlock *TBB,3189 MachineBasicBlock *FBB,3190 ArrayRef<MachineOperand> Cond,3191 const DebugLoc &DL,3192 int *BytesAdded) const {3193 if (!FBB && Cond.empty()) {3194 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))3195 .addMBB(TBB);3196 if (BytesAdded)3197 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;3198 return 1;3199 }3200 3201 assert(TBB && Cond[0].isImm());3202 3203 unsigned Opcode3204 = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm()));3205 3206 if (!FBB) {3207 MachineInstr *CondBr =3208 BuildMI(&MBB, DL, get(Opcode))3209 .addMBB(TBB);3210 3211 // Copy the flags onto the implicit condition register operand.3212 preserveCondRegFlags(CondBr->getOperand(1), Cond[1]);3213 fixImplicitOperands(*CondBr);3214 3215 if (BytesAdded)3216 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;3217 return 1;3218 }3219 3220 assert(TBB && FBB);3221 3222 MachineInstr *CondBr =3223 BuildMI(&MBB, DL, get(Opcode))3224 .addMBB(TBB);3225 fixImplicitOperands(*CondBr);3226 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))3227 .addMBB(FBB);3228 3229 MachineOperand &CondReg = CondBr->getOperand(1);3230 CondReg.setIsUndef(Cond[1].isUndef());3231 CondReg.setIsKill(Cond[1].isKill());3232 3233 if (BytesAdded)3234 *BytesAdded = ST.hasOffset3fBug() ? 16 : 8;3235 3236 return 2;3237}3238 3239bool SIInstrInfo::reverseBranchCondition(3240 SmallVectorImpl<MachineOperand> &Cond) const {3241 if (Cond.size() != 2) {3242 return true;3243 }3244 3245 if (Cond[0].isImm()) {3246 Cond[0].setImm(-Cond[0].getImm());3247 return false;3248 }3249 3250 return true;3251}3252 3253bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,3254 ArrayRef<MachineOperand> Cond,3255 Register DstReg, Register TrueReg,3256 Register FalseReg, int &CondCycles,3257 int &TrueCycles, int &FalseCycles) const {3258 switch (Cond[0].getImm()) {3259 case VCCNZ:3260 case VCCZ: {3261 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();3262 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);3263 if (MRI.getRegClass(FalseReg) != RC)3264 return false;3265 3266 int NumInsts = AMDGPU::getRegBitWidth(*RC) / 32;3267 CondCycles = TrueCycles = FalseCycles = NumInsts; // ???3268 3269 // Limit to equal cost for branch vs. N v_cndmask_b32s.3270 return RI.hasVGPRs(RC) && NumInsts <= 6;3271 }3272 case SCC_TRUE:3273 case SCC_FALSE: {3274 // FIXME: We could insert for VGPRs if we could replace the original compare3275 // with a vector one.3276 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();3277 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);3278 if (MRI.getRegClass(FalseReg) != RC)3279 return false;3280 3281 int NumInsts = AMDGPU::getRegBitWidth(*RC) / 32;3282 3283 // Multiples of 8 can do s_cselect_b643284 if (NumInsts % 2 == 0)3285 NumInsts /= 2;3286 3287 CondCycles = TrueCycles = FalseCycles = NumInsts; // ???3288 return RI.isSGPRClass(RC);3289 }3290 default:3291 return false;3292 }3293}3294 3295void SIInstrInfo::insertSelect(MachineBasicBlock &MBB,3296 MachineBasicBlock::iterator I, const DebugLoc &DL,3297 Register DstReg, ArrayRef<MachineOperand> Cond,3298 Register TrueReg, Register FalseReg) const {3299 BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());3300 if (Pred == VCCZ || Pred == SCC_FALSE) {3301 Pred = static_cast<BranchPredicate>(-Pred);3302 std::swap(TrueReg, FalseReg);3303 }3304 3305 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();3306 const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg);3307 unsigned DstSize = RI.getRegSizeInBits(*DstRC);3308 3309 if (DstSize == 32) {3310 MachineInstr *Select;3311 if (Pred == SCC_TRUE) {3312 Select = BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B32), DstReg)3313 .addReg(TrueReg)3314 .addReg(FalseReg);3315 } else {3316 // Instruction's operands are backwards from what is expected.3317 Select = BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e32), DstReg)3318 .addReg(FalseReg)3319 .addReg(TrueReg);3320 }3321 3322 preserveCondRegFlags(Select->getOperand(3), Cond[1]);3323 return;3324 }3325 3326 if (DstSize == 64 && Pred == SCC_TRUE) {3327 MachineInstr *Select =3328 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg)3329 .addReg(TrueReg)3330 .addReg(FalseReg);3331 3332 preserveCondRegFlags(Select->getOperand(3), Cond[1]);3333 return;3334 }3335 3336 static const int16_t Sub0_15[] = {3337 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,3338 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,3339 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,3340 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,3341 };3342 3343 static const int16_t Sub0_15_64[] = {3344 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,3345 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,3346 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,3347 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,3348 };3349 3350 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;3351 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;3352 const int16_t *SubIndices = Sub0_15;3353 int NElts = DstSize / 32;3354 3355 // 64-bit select is only available for SALU.3356 // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit.3357 if (Pred == SCC_TRUE) {3358 if (NElts % 2) {3359 SelOp = AMDGPU::S_CSELECT_B32;3360 EltRC = &AMDGPU::SGPR_32RegClass;3361 } else {3362 SelOp = AMDGPU::S_CSELECT_B64;3363 EltRC = &AMDGPU::SGPR_64RegClass;3364 SubIndices = Sub0_15_64;3365 NElts /= 2;3366 }3367 }3368 3369 MachineInstrBuilder MIB = BuildMI(3370 MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg);3371 3372 I = MIB->getIterator();3373 3374 SmallVector<Register, 8> Regs;3375 for (int Idx = 0; Idx != NElts; ++Idx) {3376 Register DstElt = MRI.createVirtualRegister(EltRC);3377 Regs.push_back(DstElt);3378 3379 unsigned SubIdx = SubIndices[Idx];3380 3381 MachineInstr *Select;3382 if (SelOp == AMDGPU::V_CNDMASK_B32_e32) {3383 Select =3384 BuildMI(MBB, I, DL, get(SelOp), DstElt)3385 .addReg(FalseReg, 0, SubIdx)3386 .addReg(TrueReg, 0, SubIdx);3387 } else {3388 Select =3389 BuildMI(MBB, I, DL, get(SelOp), DstElt)3390 .addReg(TrueReg, 0, SubIdx)3391 .addReg(FalseReg, 0, SubIdx);3392 }3393 3394 preserveCondRegFlags(Select->getOperand(3), Cond[1]);3395 fixImplicitOperands(*Select);3396 3397 MIB.addReg(DstElt)3398 .addImm(SubIdx);3399 }3400}3401 3402bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) {3403 switch (MI.getOpcode()) {3404 case AMDGPU::V_MOV_B16_t16_e32:3405 case AMDGPU::V_MOV_B16_t16_e64:3406 case AMDGPU::V_MOV_B32_e32:3407 case AMDGPU::V_MOV_B32_e64:3408 case AMDGPU::V_MOV_B64_PSEUDO:3409 case AMDGPU::V_MOV_B64_e32:3410 case AMDGPU::V_MOV_B64_e64:3411 case AMDGPU::S_MOV_B32:3412 case AMDGPU::S_MOV_B64:3413 case AMDGPU::S_MOV_B64_IMM_PSEUDO:3414 case AMDGPU::COPY:3415 case AMDGPU::WWM_COPY:3416 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:3417 case AMDGPU::V_ACCVGPR_READ_B32_e64:3418 case AMDGPU::V_ACCVGPR_MOV_B32:3419 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:3420 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:3421 return true;3422 default:3423 return false;3424 }3425}3426 3427unsigned SIInstrInfo::getFoldableCopySrcIdx(const MachineInstr &MI) {3428 switch (MI.getOpcode()) {3429 case AMDGPU::V_MOV_B16_t16_e32:3430 case AMDGPU::V_MOV_B16_t16_e64:3431 return 2;3432 case AMDGPU::V_MOV_B32_e32:3433 case AMDGPU::V_MOV_B32_e64:3434 case AMDGPU::V_MOV_B64_PSEUDO:3435 case AMDGPU::V_MOV_B64_e32:3436 case AMDGPU::V_MOV_B64_e64:3437 case AMDGPU::S_MOV_B32:3438 case AMDGPU::S_MOV_B64:3439 case AMDGPU::S_MOV_B64_IMM_PSEUDO:3440 case AMDGPU::COPY:3441 case AMDGPU::WWM_COPY:3442 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:3443 case AMDGPU::V_ACCVGPR_READ_B32_e64:3444 case AMDGPU::V_ACCVGPR_MOV_B32:3445 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:3446 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:3447 return 1;3448 default:3449 llvm_unreachable("MI is not a foldable copy");3450 }3451}3452 3453static constexpr AMDGPU::OpName ModifierOpNames[] = {3454 AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src1_modifiers,3455 AMDGPU::OpName::src2_modifiers, AMDGPU::OpName::clamp,3456 AMDGPU::OpName::omod, AMDGPU::OpName::op_sel};3457 3458void SIInstrInfo::removeModOperands(MachineInstr &MI) const {3459 unsigned Opc = MI.getOpcode();3460 for (AMDGPU::OpName Name : reverse(ModifierOpNames)) {3461 int Idx = AMDGPU::getNamedOperandIdx(Opc, Name);3462 if (Idx >= 0)3463 MI.removeOperand(Idx);3464 }3465}3466 3467void SIInstrInfo::mutateAndCleanupImplicit(MachineInstr &MI,3468 const MCInstrDesc &NewDesc) const {3469 MI.setDesc(NewDesc);3470 3471 // Remove any leftover implicit operands from mutating the instruction. e.g.3472 // if we replace an s_and_b32 with a copy, we don't need the implicit scc def3473 // anymore.3474 const MCInstrDesc &Desc = MI.getDesc();3475 unsigned NumOps = Desc.getNumOperands() + Desc.implicit_uses().size() +3476 Desc.implicit_defs().size();3477 3478 for (unsigned I = MI.getNumOperands() - 1; I >= NumOps; --I)3479 MI.removeOperand(I);3480}3481 3482std::optional<int64_t> SIInstrInfo::extractSubregFromImm(int64_t Imm,3483 unsigned SubRegIndex) {3484 switch (SubRegIndex) {3485 case AMDGPU::NoSubRegister:3486 return Imm;3487 case AMDGPU::sub0:3488 return SignExtend64<32>(Imm);3489 case AMDGPU::sub1:3490 return SignExtend64<32>(Imm >> 32);3491 case AMDGPU::lo16:3492 return SignExtend64<16>(Imm);3493 case AMDGPU::hi16:3494 return SignExtend64<16>(Imm >> 16);3495 case AMDGPU::sub1_lo16:3496 return SignExtend64<16>(Imm >> 32);3497 case AMDGPU::sub1_hi16:3498 return SignExtend64<16>(Imm >> 48);3499 default:3500 return std::nullopt;3501 }3502 3503 llvm_unreachable("covered subregister switch");3504}3505 3506static unsigned getNewFMAAKInst(const GCNSubtarget &ST, unsigned Opc) {3507 switch (Opc) {3508 case AMDGPU::V_MAC_F16_e32:3509 case AMDGPU::V_MAC_F16_e64:3510 case AMDGPU::V_MAD_F16_e64:3511 return AMDGPU::V_MADAK_F16;3512 case AMDGPU::V_MAC_F32_e32:3513 case AMDGPU::V_MAC_F32_e64:3514 case AMDGPU::V_MAD_F32_e64:3515 return AMDGPU::V_MADAK_F32;3516 case AMDGPU::V_FMAC_F32_e32:3517 case AMDGPU::V_FMAC_F32_e64:3518 case AMDGPU::V_FMA_F32_e64:3519 return AMDGPU::V_FMAAK_F32;3520 case AMDGPU::V_FMAC_F16_e32:3521 case AMDGPU::V_FMAC_F16_e64:3522 case AMDGPU::V_FMAC_F16_t16_e64:3523 case AMDGPU::V_FMAC_F16_fake16_e64:3524 case AMDGPU::V_FMA_F16_e64:3525 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()3526 ? AMDGPU::V_FMAAK_F16_t163527 : AMDGPU::V_FMAAK_F16_fake163528 : AMDGPU::V_FMAAK_F16;3529 case AMDGPU::V_FMAC_F64_e32:3530 case AMDGPU::V_FMAC_F64_e64:3531 case AMDGPU::V_FMA_F64_e64:3532 return AMDGPU::V_FMAAK_F64;3533 default:3534 llvm_unreachable("invalid instruction");3535 }3536}3537 3538static unsigned getNewFMAMKInst(const GCNSubtarget &ST, unsigned Opc) {3539 switch (Opc) {3540 case AMDGPU::V_MAC_F16_e32:3541 case AMDGPU::V_MAC_F16_e64:3542 case AMDGPU::V_MAD_F16_e64:3543 return AMDGPU::V_MADMK_F16;3544 case AMDGPU::V_MAC_F32_e32:3545 case AMDGPU::V_MAC_F32_e64:3546 case AMDGPU::V_MAD_F32_e64:3547 return AMDGPU::V_MADMK_F32;3548 case AMDGPU::V_FMAC_F32_e32:3549 case AMDGPU::V_FMAC_F32_e64:3550 case AMDGPU::V_FMA_F32_e64:3551 return AMDGPU::V_FMAMK_F32;3552 case AMDGPU::V_FMAC_F16_e32:3553 case AMDGPU::V_FMAC_F16_e64:3554 case AMDGPU::V_FMAC_F16_t16_e64:3555 case AMDGPU::V_FMAC_F16_fake16_e64:3556 case AMDGPU::V_FMA_F16_e64:3557 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()3558 ? AMDGPU::V_FMAMK_F16_t163559 : AMDGPU::V_FMAMK_F16_fake163560 : AMDGPU::V_FMAMK_F16;3561 case AMDGPU::V_FMAC_F64_e32:3562 case AMDGPU::V_FMAC_F64_e64:3563 case AMDGPU::V_FMA_F64_e64:3564 return AMDGPU::V_FMAMK_F64;3565 default:3566 llvm_unreachable("invalid instruction");3567 }3568}3569 3570bool SIInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,3571 Register Reg, MachineRegisterInfo *MRI) const {3572 int64_t Imm;3573 if (!getConstValDefinedInReg(DefMI, Reg, Imm))3574 return false;3575 3576 const bool HasMultipleUses = !MRI->hasOneNonDBGUse(Reg);3577 3578 assert(!DefMI.getOperand(0).getSubReg() && "Expected SSA form");3579 3580 unsigned Opc = UseMI.getOpcode();3581 if (Opc == AMDGPU::COPY) {3582 assert(!UseMI.getOperand(0).getSubReg() && "Expected SSA form");3583 3584 Register DstReg = UseMI.getOperand(0).getReg();3585 Register UseSubReg = UseMI.getOperand(1).getSubReg();3586 3587 const TargetRegisterClass *DstRC = RI.getRegClassForReg(*MRI, DstReg);3588 3589 if (HasMultipleUses) {3590 // TODO: This should fold in more cases with multiple use, but we need to3591 // more carefully consider what those uses are.3592 unsigned ImmDefSize = RI.getRegSizeInBits(*MRI->getRegClass(Reg));3593 3594 // Avoid breaking up a 64-bit inline immediate into a subregister extract.3595 if (UseSubReg != AMDGPU::NoSubRegister && ImmDefSize == 64)3596 return false;3597 3598 // Most of the time folding a 32-bit inline constant is free (though this3599 // might not be true if we can't later fold it into a real user).3600 //3601 // FIXME: This isInlineConstant check is imprecise if3602 // getConstValDefinedInReg handled the tricky non-mov cases.3603 if (ImmDefSize == 32 &&3604 !isInlineConstant(Imm, AMDGPU::OPERAND_REG_IMM_INT32))3605 return false;3606 }3607 3608 bool Is16Bit = UseSubReg != AMDGPU::NoSubRegister &&3609 RI.getSubRegIdxSize(UseSubReg) == 16;3610 3611 if (Is16Bit) {3612 if (RI.hasVGPRs(DstRC))3613 return false; // Do not clobber vgpr_hi163614 3615 if (DstReg.isVirtual() && UseSubReg != AMDGPU::lo16)3616 return false;3617 }3618 3619 MachineFunction *MF = UseMI.getMF();3620 3621 unsigned NewOpc = AMDGPU::INSTRUCTION_LIST_END;3622 MCRegister MovDstPhysReg =3623 DstReg.isPhysical() ? DstReg.asMCReg() : MCRegister();3624 3625 std::optional<int64_t> SubRegImm = extractSubregFromImm(Imm, UseSubReg);3626 3627 // TODO: Try to fold with AMDGPU::V_MOV_B16_t16_e643628 for (unsigned MovOp :3629 {AMDGPU::S_MOV_B32, AMDGPU::V_MOV_B32_e32, AMDGPU::S_MOV_B64,3630 AMDGPU::V_MOV_B64_PSEUDO, AMDGPU::V_ACCVGPR_WRITE_B32_e64}) {3631 const MCInstrDesc &MovDesc = get(MovOp);3632 3633 const TargetRegisterClass *MovDstRC = getRegClass(MovDesc, 0);3634 if (Is16Bit) {3635 // We just need to find a correctly sized register class, so the3636 // subregister index compatibility doesn't matter since we're statically3637 // extracting the immediate value.3638 MovDstRC = RI.getMatchingSuperRegClass(MovDstRC, DstRC, AMDGPU::lo16);3639 if (!MovDstRC)3640 continue;3641 3642 if (MovDstPhysReg) {3643 // FIXME: We probably should not do this. If there is a live value in3644 // the high half of the register, it will be corrupted.3645 MovDstPhysReg =3646 RI.getMatchingSuperReg(MovDstPhysReg, AMDGPU::lo16, MovDstRC);3647 if (!MovDstPhysReg)3648 continue;3649 }3650 }3651 3652 // Result class isn't the right size, try the next instruction.3653 if (MovDstPhysReg) {3654 if (!MovDstRC->contains(MovDstPhysReg))3655 return false;3656 } else if (!MRI->constrainRegClass(DstReg, MovDstRC)) {3657 // TODO: This will be overly conservative in the case of 16-bit virtual3658 // SGPRs. We could hack up the virtual register uses to use a compatible3659 // 32-bit class.3660 continue;3661 }3662 3663 const MCOperandInfo &OpInfo = MovDesc.operands()[1];3664 3665 // Ensure the interpreted immediate value is a valid operand in the new3666 // mov.3667 //3668 // FIXME: isImmOperandLegal should have form that doesn't require existing3669 // MachineInstr or MachineOperand3670 if (!RI.opCanUseLiteralConstant(OpInfo.OperandType) &&3671 !isInlineConstant(*SubRegImm, OpInfo.OperandType))3672 break;3673 3674 NewOpc = MovOp;3675 break;3676 }3677 3678 if (NewOpc == AMDGPU::INSTRUCTION_LIST_END)3679 return false;3680 3681 if (Is16Bit) {3682 UseMI.getOperand(0).setSubReg(AMDGPU::NoSubRegister);3683 if (MovDstPhysReg)3684 UseMI.getOperand(0).setReg(MovDstPhysReg);3685 assert(UseMI.getOperand(1).getReg().isVirtual());3686 }3687 3688 const MCInstrDesc &NewMCID = get(NewOpc);3689 UseMI.setDesc(NewMCID);3690 UseMI.getOperand(1).ChangeToImmediate(*SubRegImm);3691 UseMI.addImplicitDefUseOperands(*MF);3692 return true;3693 }3694 3695 if (HasMultipleUses)3696 return false;3697 3698 if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 ||3699 Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||3700 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 ||3701 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 ||3702 Opc == AMDGPU::V_FMAC_F16_t16_e64 ||3703 Opc == AMDGPU::V_FMAC_F16_fake16_e64 || Opc == AMDGPU::V_FMA_F64_e64 ||3704 Opc == AMDGPU::V_FMAC_F64_e64) {3705 // Don't fold if we are using source or output modifiers. The new VOP23706 // instructions don't have them.3707 if (hasAnyModifiersSet(UseMI))3708 return false;3709 3710 // If this is a free constant, there's no reason to do this.3711 // TODO: We could fold this here instead of letting SIFoldOperands do it3712 // later.3713 int Src0Idx = getNamedOperandIdx(UseMI.getOpcode(), AMDGPU::OpName::src0);3714 3715 // Any src operand can be used for the legality check.3716 if (isInlineConstant(UseMI, Src0Idx, Imm))3717 return false;3718 3719 MachineOperand *Src0 = &UseMI.getOperand(Src0Idx);3720 3721 MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1);3722 MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2);3723 3724 // Multiplied part is the constant: Use v_madmk_{f16, f32}.3725 if ((Src0->isReg() && Src0->getReg() == Reg) ||3726 (Src1->isReg() && Src1->getReg() == Reg)) {3727 MachineOperand *RegSrc =3728 Src1->isReg() && Src1->getReg() == Reg ? Src0 : Src1;3729 if (!RegSrc->isReg())3730 return false;3731 if (RI.isSGPRClass(MRI->getRegClass(RegSrc->getReg())) &&3732 ST.getConstantBusLimit(Opc) < 2)3733 return false;3734 3735 if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg())))3736 return false;3737 3738 // If src2 is also a literal constant then we have to choose which one to3739 // fold. In general it is better to choose madak so that the other literal3740 // can be materialized in an sgpr instead of a vgpr:3741 // s_mov_b32 s0, literal3742 // v_madak_f32 v0, s0, v0, literal3743 // Instead of:3744 // v_mov_b32 v1, literal3745 // v_madmk_f32 v0, v0, literal, v13746 MachineInstr *Def = MRI->getUniqueVRegDef(Src2->getReg());3747 if (Def && Def->isMoveImmediate() &&3748 !isInlineConstant(Def->getOperand(1)))3749 return false;3750 3751 unsigned NewOpc = getNewFMAMKInst(ST, Opc);3752 if (pseudoToMCOpcode(NewOpc) == -1)3753 return false;3754 3755 // V_FMAMK_F16_t16 takes VGPR_16_Lo128 operands while V_FMAMK_F16_fake163756 // takes VGPR_32_Lo128 operands, so the rewrite would also require3757 // restricting their register classes. For now just bail out.3758 if (NewOpc == AMDGPU::V_FMAMK_F16_t16 ||3759 NewOpc == AMDGPU::V_FMAMK_F16_fake16)3760 return false;3761 3762 const std::optional<int64_t> SubRegImm = extractSubregFromImm(3763 Imm, RegSrc == Src1 ? Src0->getSubReg() : Src1->getSubReg());3764 3765 // FIXME: This would be a lot easier if we could return a new instruction3766 // instead of having to modify in place.3767 3768 Register SrcReg = RegSrc->getReg();3769 unsigned SrcSubReg = RegSrc->getSubReg();3770 Src0->setReg(SrcReg);3771 Src0->setSubReg(SrcSubReg);3772 Src0->setIsKill(RegSrc->isKill());3773 3774 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||3775 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 ||3776 Opc == AMDGPU::V_FMAC_F16_fake16_e64 ||3777 Opc == AMDGPU::V_FMAC_F16_e64 || Opc == AMDGPU::V_FMAC_F64_e64)3778 UseMI.untieRegOperand(3779 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));3780 3781 Src1->ChangeToImmediate(*SubRegImm);3782 3783 removeModOperands(UseMI);3784 UseMI.setDesc(get(NewOpc));3785 3786 bool DeleteDef = MRI->use_nodbg_empty(Reg);3787 if (DeleteDef)3788 DefMI.eraseFromParent();3789 3790 return true;3791 }3792 3793 // Added part is the constant: Use v_madak_{f16, f32}.3794 if (Src2->isReg() && Src2->getReg() == Reg) {3795 if (ST.getConstantBusLimit(Opc) < 2) {3796 // Not allowed to use constant bus for another operand.3797 // We can however allow an inline immediate as src0.3798 bool Src0Inlined = false;3799 if (Src0->isReg()) {3800 // Try to inline constant if possible.3801 // If the Def moves immediate and the use is single3802 // We are saving VGPR here.3803 MachineInstr *Def = MRI->getUniqueVRegDef(Src0->getReg());3804 if (Def && Def->isMoveImmediate() &&3805 isInlineConstant(Def->getOperand(1)) &&3806 MRI->hasOneNonDBGUse(Src0->getReg())) {3807 Src0->ChangeToImmediate(Def->getOperand(1).getImm());3808 Src0Inlined = true;3809 } else if (ST.getConstantBusLimit(Opc) <= 1 &&3810 RI.isSGPRReg(*MRI, Src0->getReg())) {3811 return false;3812 }3813 // VGPR is okay as Src0 - fallthrough3814 }3815 3816 if (Src1->isReg() && !Src0Inlined) {3817 // We have one slot for inlinable constant so far - try to fill it3818 MachineInstr *Def = MRI->getUniqueVRegDef(Src1->getReg());3819 if (Def && Def->isMoveImmediate() &&3820 isInlineConstant(Def->getOperand(1)) &&3821 MRI->hasOneNonDBGUse(Src1->getReg()) && commuteInstruction(UseMI))3822 Src0->ChangeToImmediate(Def->getOperand(1).getImm());3823 else if (RI.isSGPRReg(*MRI, Src1->getReg()))3824 return false;3825 // VGPR is okay as Src1 - fallthrough3826 }3827 }3828 3829 unsigned NewOpc = getNewFMAAKInst(ST, Opc);3830 if (pseudoToMCOpcode(NewOpc) == -1)3831 return false;3832 3833 // V_FMAAK_F16_t16 takes VGPR_16_Lo128 operands while V_FMAAK_F16_fake163834 // takes VGPR_32_Lo128 operands, so the rewrite would also require3835 // restricting their register classes. For now just bail out.3836 if (NewOpc == AMDGPU::V_FMAAK_F16_t16 ||3837 NewOpc == AMDGPU::V_FMAAK_F16_fake16)3838 return false;3839 3840 // FIXME: This would be a lot easier if we could return a new instruction3841 // instead of having to modify in place.3842 3843 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||3844 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 ||3845 Opc == AMDGPU::V_FMAC_F16_fake16_e64 ||3846 Opc == AMDGPU::V_FMAC_F16_e64 || Opc == AMDGPU::V_FMAC_F64_e64)3847 UseMI.untieRegOperand(3848 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));3849 3850 const std::optional<int64_t> SubRegImm =3851 extractSubregFromImm(Imm, Src2->getSubReg());3852 3853 // ChangingToImmediate adds Src2 back to the instruction.3854 Src2->ChangeToImmediate(*SubRegImm);3855 3856 // These come before src2.3857 removeModOperands(UseMI);3858 UseMI.setDesc(get(NewOpc));3859 // It might happen that UseMI was commuted3860 // and we now have SGPR as SRC1. If so 2 inlined3861 // constant and SGPR are illegal.3862 legalizeOperands(UseMI);3863 3864 bool DeleteDef = MRI->use_nodbg_empty(Reg);3865 if (DeleteDef)3866 DefMI.eraseFromParent();3867 3868 return true;3869 }3870 }3871 3872 return false;3873}3874 3875static bool3876memOpsHaveSameBaseOperands(ArrayRef<const MachineOperand *> BaseOps1,3877 ArrayRef<const MachineOperand *> BaseOps2) {3878 if (BaseOps1.size() != BaseOps2.size())3879 return false;3880 for (size_t I = 0, E = BaseOps1.size(); I < E; ++I) {3881 if (!BaseOps1[I]->isIdenticalTo(*BaseOps2[I]))3882 return false;3883 }3884 return true;3885}3886 3887static bool offsetsDoNotOverlap(LocationSize WidthA, int OffsetA,3888 LocationSize WidthB, int OffsetB) {3889 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;3890 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;3891 LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;3892 return LowWidth.hasValue() &&3893 LowOffset + (int)LowWidth.getValue() <= HighOffset;3894}3895 3896bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,3897 const MachineInstr &MIb) const {3898 SmallVector<const MachineOperand *, 4> BaseOps0, BaseOps1;3899 int64_t Offset0, Offset1;3900 LocationSize Dummy0 = LocationSize::precise(0);3901 LocationSize Dummy1 = LocationSize::precise(0);3902 bool Offset0IsScalable, Offset1IsScalable;3903 if (!getMemOperandsWithOffsetWidth(MIa, BaseOps0, Offset0, Offset0IsScalable,3904 Dummy0, &RI) ||3905 !getMemOperandsWithOffsetWidth(MIb, BaseOps1, Offset1, Offset1IsScalable,3906 Dummy1, &RI))3907 return false;3908 3909 if (!memOpsHaveSameBaseOperands(BaseOps0, BaseOps1))3910 return false;3911 3912 if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {3913 // FIXME: Handle ds_read2 / ds_write2.3914 return false;3915 }3916 LocationSize Width0 = MIa.memoperands().front()->getSize();3917 LocationSize Width1 = MIb.memoperands().front()->getSize();3918 return offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1);3919}3920 3921bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,3922 const MachineInstr &MIb) const {3923 assert(MIa.mayLoadOrStore() &&3924 "MIa must load from or modify a memory location");3925 assert(MIb.mayLoadOrStore() &&3926 "MIb must load from or modify a memory location");3927 3928 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects())3929 return false;3930 3931 // XXX - Can we relax this between address spaces?3932 if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())3933 return false;3934 3935 if (isLDSDMA(MIa) || isLDSDMA(MIb))3936 return false;3937 3938 if (MIa.isBundle() || MIb.isBundle())3939 return false;3940 3941 // TODO: Should we check the address space from the MachineMemOperand? That3942 // would allow us to distinguish objects we know don't alias based on the3943 // underlying address space, even if it was lowered to a different one,3944 // e.g. private accesses lowered to use MUBUF instructions on a scratch3945 // buffer.3946 if (isDS(MIa)) {3947 if (isDS(MIb))3948 return checkInstOffsetsDoNotOverlap(MIa, MIb);3949 3950 return !isFLAT(MIb) || isSegmentSpecificFLAT(MIb);3951 }3952 3953 if (isMUBUF(MIa) || isMTBUF(MIa)) {3954 if (isMUBUF(MIb) || isMTBUF(MIb))3955 return checkInstOffsetsDoNotOverlap(MIa, MIb);3956 3957 if (isFLAT(MIb))3958 return isFLATScratch(MIb);3959 3960 return !isSMRD(MIb);3961 }3962 3963 if (isSMRD(MIa)) {3964 if (isSMRD(MIb))3965 return checkInstOffsetsDoNotOverlap(MIa, MIb);3966 3967 if (isFLAT(MIb))3968 return isFLATScratch(MIb);3969 3970 return !isMUBUF(MIb) && !isMTBUF(MIb);3971 }3972 3973 if (isFLAT(MIa)) {3974 if (isFLAT(MIb)) {3975 if ((isFLATScratch(MIa) && isFLATGlobal(MIb)) ||3976 (isFLATGlobal(MIa) && isFLATScratch(MIb)))3977 return true;3978 3979 return checkInstOffsetsDoNotOverlap(MIa, MIb);3980 }3981 3982 return false;3983 }3984 3985 return false;3986}3987 3988static bool getFoldableImm(Register Reg, const MachineRegisterInfo &MRI,3989 int64_t &Imm, MachineInstr **DefMI = nullptr) {3990 if (Reg.isPhysical())3991 return false;3992 auto *Def = MRI.getUniqueVRegDef(Reg);3993 if (Def && SIInstrInfo::isFoldableCopy(*Def) && Def->getOperand(1).isImm()) {3994 Imm = Def->getOperand(1).getImm();3995 if (DefMI)3996 *DefMI = Def;3997 return true;3998 }3999 return false;4000}4001 4002static bool getFoldableImm(const MachineOperand *MO, int64_t &Imm,4003 MachineInstr **DefMI = nullptr) {4004 if (!MO->isReg())4005 return false;4006 const MachineFunction *MF = MO->getParent()->getMF();4007 const MachineRegisterInfo &MRI = MF->getRegInfo();4008 return getFoldableImm(MO->getReg(), MRI, Imm, DefMI);4009}4010 4011static void updateLiveVariables(LiveVariables *LV, MachineInstr &MI,4012 MachineInstr &NewMI) {4013 if (LV) {4014 unsigned NumOps = MI.getNumOperands();4015 for (unsigned I = 1; I < NumOps; ++I) {4016 MachineOperand &Op = MI.getOperand(I);4017 if (Op.isReg() && Op.isKill())4018 LV->replaceKillInstruction(Op.getReg(), MI, NewMI);4019 }4020 }4021}4022 4023static unsigned getNewFMAInst(const GCNSubtarget &ST, unsigned Opc) {4024 switch (Opc) {4025 case AMDGPU::V_MAC_F16_e32:4026 case AMDGPU::V_MAC_F16_e64:4027 return AMDGPU::V_MAD_F16_e64;4028 case AMDGPU::V_MAC_F32_e32:4029 case AMDGPU::V_MAC_F32_e64:4030 return AMDGPU::V_MAD_F32_e64;4031 case AMDGPU::V_MAC_LEGACY_F32_e32:4032 case AMDGPU::V_MAC_LEGACY_F32_e64:4033 return AMDGPU::V_MAD_LEGACY_F32_e64;4034 case AMDGPU::V_FMAC_LEGACY_F32_e32:4035 case AMDGPU::V_FMAC_LEGACY_F32_e64:4036 return AMDGPU::V_FMA_LEGACY_F32_e64;4037 case AMDGPU::V_FMAC_F16_e32:4038 case AMDGPU::V_FMAC_F16_e64:4039 case AMDGPU::V_FMAC_F16_t16_e64:4040 case AMDGPU::V_FMAC_F16_fake16_e64:4041 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()4042 ? AMDGPU::V_FMA_F16_gfx9_t16_e644043 : AMDGPU::V_FMA_F16_gfx9_fake16_e644044 : AMDGPU::V_FMA_F16_gfx9_e64;4045 case AMDGPU::V_FMAC_F32_e32:4046 case AMDGPU::V_FMAC_F32_e64:4047 return AMDGPU::V_FMA_F32_e64;4048 case AMDGPU::V_FMAC_F64_e32:4049 case AMDGPU::V_FMAC_F64_e64:4050 return AMDGPU::V_FMA_F64_e64;4051 default:4052 llvm_unreachable("invalid instruction");4053 }4054}4055 4056/// Helper struct for the implementation of 3-address conversion to communicate4057/// updates made to instruction operands.4058struct SIInstrInfo::ThreeAddressUpdates {4059 /// Other instruction whose def is no longer used by the converted4060 /// instruction.4061 MachineInstr *RemoveMIUse = nullptr;4062};4063 4064MachineInstr *SIInstrInfo::convertToThreeAddress(MachineInstr &MI,4065 LiveVariables *LV,4066 LiveIntervals *LIS) const {4067 MachineBasicBlock &MBB = *MI.getParent();4068 MachineInstr *CandidateMI = &MI;4069 4070 if (MI.isBundle()) {4071 // This is a temporary placeholder for bundle handling that enables us to4072 // exercise the relevant code paths in the two-address instruction pass.4073 if (MI.getBundleSize() != 1)4074 return nullptr;4075 CandidateMI = MI.getNextNode();4076 }4077 4078 ThreeAddressUpdates U;4079 MachineInstr *NewMI = convertToThreeAddressImpl(*CandidateMI, U);4080 if (!NewMI)4081 return nullptr;4082 4083 if (MI.isBundle()) {4084 CandidateMI->eraseFromBundle();4085 4086 for (MachineOperand &MO : MI.all_defs()) {4087 if (MO.isTied())4088 MI.untieRegOperand(MO.getOperandNo());4089 }4090 } else {4091 updateLiveVariables(LV, MI, *NewMI);4092 if (LIS) {4093 LIS->ReplaceMachineInstrInMaps(MI, *NewMI);4094 // SlotIndex of defs needs to be updated when converting to early-clobber4095 MachineOperand &Def = NewMI->getOperand(0);4096 if (Def.isEarlyClobber() && Def.isReg() &&4097 LIS->hasInterval(Def.getReg())) {4098 SlotIndex OldIndex = LIS->getInstructionIndex(*NewMI).getRegSlot(false);4099 SlotIndex NewIndex = LIS->getInstructionIndex(*NewMI).getRegSlot(true);4100 auto &LI = LIS->getInterval(Def.getReg());4101 auto UpdateDefIndex = [&](LiveRange &LR) {4102 auto *S = LR.find(OldIndex);4103 if (S != LR.end() && S->start == OldIndex) {4104 assert(S->valno && S->valno->def == OldIndex);4105 S->start = NewIndex;4106 S->valno->def = NewIndex;4107 }4108 };4109 UpdateDefIndex(LI);4110 for (auto &SR : LI.subranges())4111 UpdateDefIndex(SR);4112 }4113 }4114 }4115 4116 if (U.RemoveMIUse) {4117 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();4118 // The only user is the instruction which will be killed.4119 Register DefReg = U.RemoveMIUse->getOperand(0).getReg();4120 4121 if (MRI.hasOneNonDBGUse(DefReg)) {4122 // We cannot just remove the DefMI here, calling pass will crash.4123 U.RemoveMIUse->setDesc(get(AMDGPU::IMPLICIT_DEF));4124 U.RemoveMIUse->getOperand(0).setIsDead(true);4125 for (unsigned I = U.RemoveMIUse->getNumOperands() - 1; I != 0; --I)4126 U.RemoveMIUse->removeOperand(I);4127 if (LV)4128 LV->getVarInfo(DefReg).AliveBlocks.clear();4129 }4130 4131 if (MI.isBundle()) {4132 VirtRegInfo VRI = AnalyzeVirtRegInBundle(MI, DefReg);4133 if (!VRI.Reads && !VRI.Writes) {4134 for (MachineOperand &MO : MI.all_uses()) {4135 if (MO.isReg() && MO.getReg() == DefReg) {4136 assert(MO.getSubReg() == 0 &&4137 "tied sub-registers in bundles currently not supported");4138 MI.removeOperand(MO.getOperandNo());4139 break;4140 }4141 }4142 4143 if (LIS)4144 LIS->shrinkToUses(&LIS->getInterval(DefReg));4145 }4146 } else if (LIS) {4147 LiveInterval &DefLI = LIS->getInterval(DefReg);4148 4149 // We cannot delete the original instruction here, so hack out the use4150 // in the original instruction with a dummy register so we can use4151 // shrinkToUses to deal with any multi-use edge cases. Other targets do4152 // not have the complexity of deleting a use to consider here.4153 Register DummyReg = MRI.cloneVirtualRegister(DefReg);4154 for (MachineOperand &MIOp : MI.uses()) {4155 if (MIOp.isReg() && MIOp.getReg() == DefReg) {4156 MIOp.setIsUndef(true);4157 MIOp.setReg(DummyReg);4158 }4159 }4160 4161 if (MI.isBundle()) {4162 VirtRegInfo VRI = AnalyzeVirtRegInBundle(MI, DefReg);4163 if (!VRI.Reads && !VRI.Writes) {4164 for (MachineOperand &MIOp : MI.uses()) {4165 if (MIOp.isReg() && MIOp.getReg() == DefReg) {4166 MIOp.setIsUndef(true);4167 MIOp.setReg(DummyReg);4168 }4169 }4170 }4171 4172 MI.addOperand(MachineOperand::CreateReg(DummyReg, false, false, false,4173 false, /*isUndef=*/true));4174 }4175 4176 LIS->shrinkToUses(&DefLI);4177 }4178 }4179 4180 return MI.isBundle() ? &MI : NewMI;4181}4182 4183MachineInstr *4184SIInstrInfo::convertToThreeAddressImpl(MachineInstr &MI,4185 ThreeAddressUpdates &U) const {4186 MachineBasicBlock &MBB = *MI.getParent();4187 unsigned Opc = MI.getOpcode();4188 4189 // Handle MFMA.4190 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(Opc);4191 if (NewMFMAOpc != -1) {4192 MachineInstrBuilder MIB =4193 BuildMI(MBB, MI, MI.getDebugLoc(), get(NewMFMAOpc));4194 for (unsigned I = 0, E = MI.getNumExplicitOperands(); I != E; ++I)4195 MIB.add(MI.getOperand(I));4196 return MIB;4197 }4198 4199 if (SIInstrInfo::isWMMA(MI)) {4200 unsigned NewOpc = AMDGPU::mapWMMA2AddrTo3AddrOpcode(MI.getOpcode());4201 MachineInstrBuilder MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))4202 .setMIFlags(MI.getFlags());4203 for (unsigned I = 0, E = MI.getNumExplicitOperands(); I != E; ++I)4204 MIB->addOperand(MI.getOperand(I));4205 return MIB;4206 }4207 4208 assert(Opc != AMDGPU::V_FMAC_F16_t16_e32 &&4209 Opc != AMDGPU::V_FMAC_F16_fake16_e32 &&4210 "V_FMAC_F16_t16/fake16_e32 is not supported and not expected to be "4211 "present pre-RA");4212 4213 // Handle MAC/FMAC.4214 bool IsF64 = Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64;4215 bool IsLegacy = Opc == AMDGPU::V_MAC_LEGACY_F32_e32 ||4216 Opc == AMDGPU::V_MAC_LEGACY_F32_e64 ||4217 Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 ||4218 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64;4219 bool Src0Literal = false;4220 4221 switch (Opc) {4222 default:4223 return nullptr;4224 case AMDGPU::V_MAC_F16_e64:4225 case AMDGPU::V_FMAC_F16_e64:4226 case AMDGPU::V_FMAC_F16_t16_e64:4227 case AMDGPU::V_FMAC_F16_fake16_e64:4228 case AMDGPU::V_MAC_F32_e64:4229 case AMDGPU::V_MAC_LEGACY_F32_e64:4230 case AMDGPU::V_FMAC_F32_e64:4231 case AMDGPU::V_FMAC_LEGACY_F32_e64:4232 case AMDGPU::V_FMAC_F64_e64:4233 break;4234 case AMDGPU::V_MAC_F16_e32:4235 case AMDGPU::V_FMAC_F16_e32:4236 case AMDGPU::V_MAC_F32_e32:4237 case AMDGPU::V_MAC_LEGACY_F32_e32:4238 case AMDGPU::V_FMAC_F32_e32:4239 case AMDGPU::V_FMAC_LEGACY_F32_e32:4240 case AMDGPU::V_FMAC_F64_e32: {4241 int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),4242 AMDGPU::OpName::src0);4243 const MachineOperand *Src0 = &MI.getOperand(Src0Idx);4244 if (!Src0->isReg() && !Src0->isImm())4245 return nullptr;4246 4247 if (Src0->isImm() && !isInlineConstant(MI, Src0Idx, *Src0))4248 Src0Literal = true;4249 4250 break;4251 }4252 }4253 4254 MachineInstrBuilder MIB;4255 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);4256 const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0);4257 const MachineOperand *Src0Mods =4258 getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);4259 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);4260 const MachineOperand *Src1Mods =4261 getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);4262 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);4263 const MachineOperand *Src2Mods =4264 getNamedOperand(MI, AMDGPU::OpName::src2_modifiers);4265 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);4266 const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod);4267 const MachineOperand *OpSel = getNamedOperand(MI, AMDGPU::OpName::op_sel);4268 4269 if (!Src0Mods && !Src1Mods && !Src2Mods && !Clamp && !Omod && !IsLegacy &&4270 (!IsF64 || ST.hasFmaakFmamkF64Insts()) &&4271 // If we have an SGPR input, we will violate the constant bus restriction.4272 (ST.getConstantBusLimit(Opc) > 1 || !Src0->isReg() ||4273 !RI.isSGPRReg(MBB.getParent()->getRegInfo(), Src0->getReg()))) {4274 MachineInstr *DefMI;4275 4276 int64_t Imm;4277 if (!Src0Literal && getFoldableImm(Src2, Imm, &DefMI)) {4278 unsigned NewOpc = getNewFMAAKInst(ST, Opc);4279 if (pseudoToMCOpcode(NewOpc) != -1) {4280 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))4281 .add(*Dst)4282 .add(*Src0)4283 .add(*Src1)4284 .addImm(Imm)4285 .setMIFlags(MI.getFlags());4286 U.RemoveMIUse = DefMI;4287 return MIB;4288 }4289 }4290 unsigned NewOpc = getNewFMAMKInst(ST, Opc);4291 if (!Src0Literal && getFoldableImm(Src1, Imm, &DefMI)) {4292 if (pseudoToMCOpcode(NewOpc) != -1) {4293 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))4294 .add(*Dst)4295 .add(*Src0)4296 .addImm(Imm)4297 .add(*Src2)4298 .setMIFlags(MI.getFlags());4299 U.RemoveMIUse = DefMI;4300 return MIB;4301 }4302 }4303 if (Src0Literal || getFoldableImm(Src0, Imm, &DefMI)) {4304 if (Src0Literal) {4305 Imm = Src0->getImm();4306 DefMI = nullptr;4307 }4308 if (pseudoToMCOpcode(NewOpc) != -1 &&4309 isOperandLegal(4310 MI, AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::src0),4311 Src1)) {4312 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))4313 .add(*Dst)4314 .add(*Src1)4315 .addImm(Imm)4316 .add(*Src2)4317 .setMIFlags(MI.getFlags());4318 U.RemoveMIUse = DefMI;4319 return MIB;4320 }4321 }4322 }4323 4324 // VOP2 mac/fmac with a literal operand cannot be converted to VOP3 mad/fma4325 // if VOP3 does not allow a literal operand.4326 if (Src0Literal && !ST.hasVOP3Literal())4327 return nullptr;4328 4329 unsigned NewOpc = getNewFMAInst(ST, Opc);4330 4331 if (pseudoToMCOpcode(NewOpc) == -1)4332 return nullptr;4333 4334 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))4335 .add(*Dst)4336 .addImm(Src0Mods ? Src0Mods->getImm() : 0)4337 .add(*Src0)4338 .addImm(Src1Mods ? Src1Mods->getImm() : 0)4339 .add(*Src1)4340 .addImm(Src2Mods ? Src2Mods->getImm() : 0)4341 .add(*Src2)4342 .addImm(Clamp ? Clamp->getImm() : 0)4343 .addImm(Omod ? Omod->getImm() : 0)4344 .setMIFlags(MI.getFlags());4345 if (AMDGPU::hasNamedOperand(NewOpc, AMDGPU::OpName::op_sel))4346 MIB.addImm(OpSel ? OpSel->getImm() : 0);4347 return MIB;4348}4349 4350// It's not generally safe to move VALU instructions across these since it will4351// start using the register as a base index rather than directly.4352// XXX - Why isn't hasSideEffects sufficient for these?4353static bool changesVGPRIndexingMode(const MachineInstr &MI) {4354 switch (MI.getOpcode()) {4355 case AMDGPU::S_SET_GPR_IDX_ON:4356 case AMDGPU::S_SET_GPR_IDX_MODE:4357 case AMDGPU::S_SET_GPR_IDX_OFF:4358 return true;4359 default:4360 return false;4361 }4362}4363 4364bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI,4365 const MachineBasicBlock *MBB,4366 const MachineFunction &MF) const {4367 // Skipping the check for SP writes in the base implementation. The reason it4368 // was added was apparently due to compile time concerns.4369 //4370 // TODO: Do we really want this barrier? It triggers unnecessary hazard nops4371 // but is probably avoidable.4372 4373 // Copied from base implementation.4374 // Terminators and labels can't be scheduled around.4375 if (MI.isTerminator() || MI.isPosition())4376 return true;4377 4378 // INLINEASM_BR can jump to another block4379 if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)4380 return true;4381 4382 if (MI.getOpcode() == AMDGPU::SCHED_BARRIER && MI.getOperand(0).getImm() == 0)4383 return true;4384 4385 // Target-independent instructions do not have an implicit-use of EXEC, even4386 // when they operate on VGPRs. Treating EXEC modifications as scheduling4387 // boundaries prevents incorrect movements of such instructions.4388 return MI.modifiesRegister(AMDGPU::EXEC, &RI) ||4389 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||4390 MI.getOpcode() == AMDGPU::S_SETREG_B32 ||4391 MI.getOpcode() == AMDGPU::S_SETPRIO ||4392 MI.getOpcode() == AMDGPU::S_SETPRIO_INC_WG ||4393 changesVGPRIndexingMode(MI);4394}4395 4396bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const {4397 return Opcode == AMDGPU::DS_ORDERED_COUNT ||4398 Opcode == AMDGPU::DS_ADD_GS_REG_RTN ||4399 Opcode == AMDGPU::DS_SUB_GS_REG_RTN || isGWS(Opcode);4400}4401 4402bool SIInstrInfo::mayAccessScratchThroughFlat(const MachineInstr &MI) const {4403 if (!isFLAT(MI) || isFLATGlobal(MI))4404 return false;4405 4406 // If scratch is not initialized, we can never access it.4407 if (MI.getMF()->getFunction().hasFnAttribute("amdgpu-no-flat-scratch-init"))4408 return false;4409 4410 // SCRATCH instructions always access scratch.4411 if (isFLATScratch(MI))4412 return true;4413 4414 // If there are no memory operands then conservatively assume the flat4415 // operation may access scratch.4416 if (MI.memoperands_empty())4417 return true;4418 4419 // See if any memory operand specifies an address space that involves scratch.4420 return any_of(MI.memoperands(), [](const MachineMemOperand *Memop) {4421 unsigned AS = Memop->getAddrSpace();4422 if (AS == AMDGPUAS::FLAT_ADDRESS) {4423 const MDNode *MD = Memop->getAAInfo().NoAliasAddrSpace;4424 return !MD || !AMDGPU::hasValueInRangeLikeMetadata(4425 *MD, AMDGPUAS::PRIVATE_ADDRESS);4426 }4427 return AS == AMDGPUAS::PRIVATE_ADDRESS;4428 });4429}4430 4431bool SIInstrInfo::mayAccessVMEMThroughFlat(const MachineInstr &MI) const {4432 assert(isFLAT(MI));4433 4434 // All flat instructions use the VMEM counter except prefetch.4435 if (!usesVM_CNT(MI))4436 return false;4437 4438 // If there are no memory operands then conservatively assume the flat4439 // operation may access VMEM.4440 if (MI.memoperands_empty())4441 return true;4442 4443 // See if any memory operand specifies an address space that involves VMEM.4444 // Flat operations only supported FLAT, LOCAL (LDS), or address spaces4445 // involving VMEM such as GLOBAL, CONSTANT, PRIVATE (SCRATCH), etc. The REGION4446 // (GDS) address space is not supported by flat operations. Therefore, simply4447 // return true unless only the LDS address space is found.4448 for (const MachineMemOperand *Memop : MI.memoperands()) {4449 unsigned AS = Memop->getAddrSpace();4450 assert(AS != AMDGPUAS::REGION_ADDRESS);4451 if (AS != AMDGPUAS::LOCAL_ADDRESS)4452 return true;4453 }4454 4455 return false;4456}4457 4458bool SIInstrInfo::mayAccessLDSThroughFlat(const MachineInstr &MI) const {4459 assert(isFLAT(MI));4460 4461 // Flat instruction such as SCRATCH and GLOBAL do not use the lgkm counter.4462 if (!usesLGKM_CNT(MI))4463 return false;4464 4465 // If in tgsplit mode then there can be no use of LDS.4466 if (ST.isTgSplitEnabled())4467 return false;4468 4469 // If there are no memory operands then conservatively assume the flat4470 // operation may access LDS.4471 if (MI.memoperands_empty())4472 return true;4473 4474 // See if any memory operand specifies an address space that involves LDS.4475 for (const MachineMemOperand *Memop : MI.memoperands()) {4476 unsigned AS = Memop->getAddrSpace();4477 if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS)4478 return true;4479 }4480 4481 return false;4482}4483 4484bool SIInstrInfo::modifiesModeRegister(const MachineInstr &MI) {4485 // Skip the full operand and register alias search modifiesRegister4486 // does. There's only a handful of instructions that touch this, it's only an4487 // implicit def, and doesn't alias any other registers.4488 return is_contained(MI.getDesc().implicit_defs(), AMDGPU::MODE);4489}4490 4491bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const {4492 unsigned Opcode = MI.getOpcode();4493 4494 if (MI.mayStore() && isSMRD(MI))4495 return true; // scalar store or atomic4496 4497 // This will terminate the function when other lanes may need to continue.4498 if (MI.isReturn())4499 return true;4500 4501 // These instructions cause shader I/O that may cause hardware lockups4502 // when executed with an empty EXEC mask.4503 //4504 // Note: exp with VM = DONE = 0 is automatically skipped by hardware when4505 // EXEC = 0, but checking for that case here seems not worth it4506 // given the typical code patterns.4507 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||4508 isEXP(Opcode) || Opcode == AMDGPU::DS_ORDERED_COUNT ||4509 Opcode == AMDGPU::S_TRAP || Opcode == AMDGPU::S_WAIT_EVENT)4510 return true;4511 4512 if (MI.isCall() || MI.isInlineAsm())4513 return true; // conservative assumption4514 4515 // Assume that barrier interactions are only intended with active lanes.4516 if (isBarrier(Opcode))4517 return true;4518 4519 // A mode change is a scalar operation that influences vector instructions.4520 if (modifiesModeRegister(MI))4521 return true;4522 4523 // These are like SALU instructions in terms of effects, so it's questionable4524 // whether we should return true for those.4525 //4526 // However, executing them with EXEC = 0 causes them to operate on undefined4527 // data, which we avoid by returning true here.4528 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 ||4529 Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32 ||4530 Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||4531 Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR)4532 return true;4533 4534 return false;4535}4536 4537bool SIInstrInfo::mayReadEXEC(const MachineRegisterInfo &MRI,4538 const MachineInstr &MI) const {4539 if (MI.isMetaInstruction())4540 return false;4541 4542 // This won't read exec if this is an SGPR->SGPR copy.4543 if (MI.isCopyLike()) {4544 if (!RI.isSGPRReg(MRI, MI.getOperand(0).getReg()))4545 return true;4546 4547 // Make sure this isn't copying exec as a normal operand4548 return MI.readsRegister(AMDGPU::EXEC, &RI);4549 }4550 4551 // Make a conservative assumption about the callee.4552 if (MI.isCall())4553 return true;4554 4555 // Be conservative with any unhandled generic opcodes.4556 if (!isTargetSpecificOpcode(MI.getOpcode()))4557 return true;4558 4559 return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI);4560}4561 4562bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {4563 switch (Imm.getBitWidth()) {4564 case 1: // This likely will be a condition code mask.4565 return true;4566 4567 case 32:4568 return AMDGPU::isInlinableLiteral32(Imm.getSExtValue(),4569 ST.hasInv2PiInlineImm());4570 case 64:4571 return AMDGPU::isInlinableLiteral64(Imm.getSExtValue(),4572 ST.hasInv2PiInlineImm());4573 case 16:4574 return ST.has16BitInsts() &&4575 AMDGPU::isInlinableLiteralI16(Imm.getSExtValue(),4576 ST.hasInv2PiInlineImm());4577 default:4578 llvm_unreachable("invalid bitwidth");4579 }4580}4581 4582bool SIInstrInfo::isInlineConstant(const APFloat &Imm) const {4583 APInt IntImm = Imm.bitcastToAPInt();4584 int64_t IntImmVal = IntImm.getSExtValue();4585 bool HasInv2Pi = ST.hasInv2PiInlineImm();4586 switch (APFloat::SemanticsToEnum(Imm.getSemantics())) {4587 default:4588 llvm_unreachable("invalid fltSemantics");4589 case APFloatBase::S_IEEEsingle:4590 case APFloatBase::S_IEEEdouble:4591 return isInlineConstant(IntImm);4592 case APFloatBase::S_BFloat:4593 return ST.has16BitInsts() &&4594 AMDGPU::isInlinableLiteralBF16(IntImmVal, HasInv2Pi);4595 case APFloatBase::S_IEEEhalf:4596 return ST.has16BitInsts() &&4597 AMDGPU::isInlinableLiteralFP16(IntImmVal, HasInv2Pi);4598 }4599}4600 4601bool SIInstrInfo::isInlineConstant(int64_t Imm, uint8_t OperandType) const {4602 // MachineOperand provides no way to tell the true operand size, since it only4603 // records a 64-bit value. We need to know the size to determine if a 32-bit4604 // floating point immediate bit pattern is legal for an integer immediate. It4605 // would be for any 32-bit integer operand, but would not be for a 64-bit one.4606 switch (OperandType) {4607 case AMDGPU::OPERAND_REG_IMM_INT32:4608 case AMDGPU::OPERAND_REG_IMM_FP32:4609 case AMDGPU::OPERAND_REG_INLINE_C_INT32:4610 case AMDGPU::OPERAND_REG_INLINE_C_FP32:4611 case AMDGPU::OPERAND_REG_IMM_V2FP32:4612 case AMDGPU::OPERAND_REG_IMM_V2INT32:4613 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:4614 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:4615 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32: {4616 int32_t Trunc = static_cast<int32_t>(Imm);4617 return AMDGPU::isInlinableLiteral32(Trunc, ST.hasInv2PiInlineImm());4618 }4619 case AMDGPU::OPERAND_REG_IMM_INT64:4620 case AMDGPU::OPERAND_REG_IMM_FP64:4621 case AMDGPU::OPERAND_REG_INLINE_C_INT64:4622 case AMDGPU::OPERAND_REG_INLINE_C_FP64:4623 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:4624 return AMDGPU::isInlinableLiteral64(Imm, ST.hasInv2PiInlineImm());4625 case AMDGPU::OPERAND_REG_IMM_INT16:4626 case AMDGPU::OPERAND_REG_INLINE_C_INT16:4627 // We would expect inline immediates to not be concerned with an integer/fp4628 // distinction. However, in the case of 16-bit integer operations, the4629 // "floating point" values appear to not work. It seems read the low 16-bits4630 // of 32-bit immediates, which happens to always work for the integer4631 // values.4632 //4633 // See llvm bugzilla 46302.4634 //4635 // TODO: Theoretically we could use op-sel to use the high bits of the4636 // 32-bit FP values.4637 return AMDGPU::isInlinableIntLiteral(Imm);4638 case AMDGPU::OPERAND_REG_IMM_V2INT16:4639 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:4640 return AMDGPU::isInlinableLiteralV2I16(Imm);4641 case AMDGPU::OPERAND_REG_IMM_V2FP16:4642 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:4643 return AMDGPU::isInlinableLiteralV2F16(Imm);4644 case AMDGPU::OPERAND_REG_IMM_V2BF16:4645 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:4646 return AMDGPU::isInlinableLiteralV2BF16(Imm);4647 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:4648 return false;4649 case AMDGPU::OPERAND_REG_IMM_FP16:4650 case AMDGPU::OPERAND_REG_INLINE_C_FP16: {4651 if (isInt<16>(Imm) || isUInt<16>(Imm)) {4652 // A few special case instructions have 16-bit operands on subtargets4653 // where 16-bit instructions are not legal.4654 // TODO: Do the 32-bit immediates work? We shouldn't really need to handle4655 // constants in these cases4656 int16_t Trunc = static_cast<int16_t>(Imm);4657 return ST.has16BitInsts() &&4658 AMDGPU::isInlinableLiteralFP16(Trunc, ST.hasInv2PiInlineImm());4659 }4660 4661 return false;4662 }4663 case AMDGPU::OPERAND_REG_IMM_BF16:4664 case AMDGPU::OPERAND_REG_INLINE_C_BF16: {4665 if (isInt<16>(Imm) || isUInt<16>(Imm)) {4666 int16_t Trunc = static_cast<int16_t>(Imm);4667 return ST.has16BitInsts() &&4668 AMDGPU::isInlinableLiteralBF16(Trunc, ST.hasInv2PiInlineImm());4669 }4670 return false;4671 }4672 case AMDGPU::OPERAND_KIMM32:4673 case AMDGPU::OPERAND_KIMM16:4674 case AMDGPU::OPERAND_KIMM64:4675 return false;4676 case AMDGPU::OPERAND_INLINE_C_AV64_PSEUDO:4677 return isLegalAV64PseudoImm(Imm);4678 case AMDGPU::OPERAND_INPUT_MODS:4679 case MCOI::OPERAND_IMMEDIATE:4680 // Always embedded in the instruction for free.4681 return true;4682 case MCOI::OPERAND_UNKNOWN:4683 case MCOI::OPERAND_REGISTER:4684 case MCOI::OPERAND_PCREL:4685 case MCOI::OPERAND_GENERIC_0:4686 case MCOI::OPERAND_GENERIC_1:4687 case MCOI::OPERAND_GENERIC_2:4688 case MCOI::OPERAND_GENERIC_3:4689 case MCOI::OPERAND_GENERIC_4:4690 case MCOI::OPERAND_GENERIC_5:4691 // Just ignore anything else.4692 return true;4693 default:4694 llvm_unreachable("invalid operand type");4695 }4696}4697 4698static bool compareMachineOp(const MachineOperand &Op0,4699 const MachineOperand &Op1) {4700 if (Op0.getType() != Op1.getType())4701 return false;4702 4703 switch (Op0.getType()) {4704 case MachineOperand::MO_Register:4705 return Op0.getReg() == Op1.getReg();4706 case MachineOperand::MO_Immediate:4707 return Op0.getImm() == Op1.getImm();4708 default:4709 llvm_unreachable("Didn't expect to be comparing these operand types");4710 }4711}4712 4713bool SIInstrInfo::isLiteralOperandLegal(const MCInstrDesc &InstDesc,4714 const MCOperandInfo &OpInfo) const {4715 if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)4716 return true;4717 4718 if (!RI.opCanUseLiteralConstant(OpInfo.OperandType))4719 return false;4720 4721 if (!isVOP3(InstDesc) || !AMDGPU::isSISrcOperand(OpInfo))4722 return true;4723 4724 return ST.hasVOP3Literal();4725}4726 4727bool SIInstrInfo::isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,4728 int64_t ImmVal) const {4729 const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo];4730 if (isInlineConstant(ImmVal, OpInfo.OperandType)) {4731 if (isMAI(InstDesc) && ST.hasMFMAInlineLiteralBug() &&4732 OpNo == (unsigned)AMDGPU::getNamedOperandIdx(InstDesc.getOpcode(),4733 AMDGPU::OpName::src2))4734 return false;4735 return RI.opCanUseInlineConstant(OpInfo.OperandType);4736 }4737 4738 return isLiteralOperandLegal(InstDesc, OpInfo);4739}4740 4741bool SIInstrInfo::isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,4742 const MachineOperand &MO) const {4743 if (MO.isImm())4744 return isImmOperandLegal(InstDesc, OpNo, MO.getImm());4745 4746 assert((MO.isTargetIndex() || MO.isFI() || MO.isGlobal()) &&4747 "unexpected imm-like operand kind");4748 const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo];4749 return isLiteralOperandLegal(InstDesc, OpInfo);4750}4751 4752bool SIInstrInfo::isLegalAV64PseudoImm(uint64_t Imm) const {4753 // 2 32-bit inline constants packed into one.4754 return AMDGPU::isInlinableLiteral32(Lo_32(Imm), ST.hasInv2PiInlineImm()) &&4755 AMDGPU::isInlinableLiteral32(Hi_32(Imm), ST.hasInv2PiInlineImm());4756}4757 4758bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {4759 // GFX90A does not have V_MUL_LEGACY_F32_e32.4760 if (Opcode == AMDGPU::V_MUL_LEGACY_F32_e64 && ST.hasGFX90AInsts())4761 return false;4762 4763 int Op32 = AMDGPU::getVOPe32(Opcode);4764 if (Op32 == -1)4765 return false;4766 4767 return pseudoToMCOpcode(Op32) != -1;4768}4769 4770bool SIInstrInfo::hasModifiers(unsigned Opcode) const {4771 // The src0_modifier operand is present on all instructions4772 // that have modifiers.4773 4774 return AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src0_modifiers);4775}4776 4777bool SIInstrInfo::hasModifiersSet(const MachineInstr &MI,4778 AMDGPU::OpName OpName) const {4779 const MachineOperand *Mods = getNamedOperand(MI, OpName);4780 return Mods && Mods->getImm();4781}4782 4783bool SIInstrInfo::hasAnyModifiersSet(const MachineInstr &MI) const {4784 return any_of(ModifierOpNames,4785 [&](AMDGPU::OpName Name) { return hasModifiersSet(MI, Name); });4786}4787 4788bool SIInstrInfo::canShrink(const MachineInstr &MI,4789 const MachineRegisterInfo &MRI) const {4790 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);4791 // Can't shrink instruction with three operands.4792 if (Src2) {4793 switch (MI.getOpcode()) {4794 default: return false;4795 4796 case AMDGPU::V_ADDC_U32_e64:4797 case AMDGPU::V_SUBB_U32_e64:4798 case AMDGPU::V_SUBBREV_U32_e64: {4799 const MachineOperand *Src14800 = getNamedOperand(MI, AMDGPU::OpName::src1);4801 if (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()))4802 return false;4803 // Additional verification is needed for sdst/src2.4804 return true;4805 }4806 case AMDGPU::V_MAC_F16_e64:4807 case AMDGPU::V_MAC_F32_e64:4808 case AMDGPU::V_MAC_LEGACY_F32_e64:4809 case AMDGPU::V_FMAC_F16_e64:4810 case AMDGPU::V_FMAC_F16_t16_e64:4811 case AMDGPU::V_FMAC_F16_fake16_e64:4812 case AMDGPU::V_FMAC_F32_e64:4813 case AMDGPU::V_FMAC_F64_e64:4814 case AMDGPU::V_FMAC_LEGACY_F32_e64:4815 if (!Src2->isReg() || !RI.isVGPR(MRI, Src2->getReg()) ||4816 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers))4817 return false;4818 break;4819 4820 case AMDGPU::V_CNDMASK_B32_e64:4821 break;4822 }4823 }4824 4825 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);4826 if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()) ||4827 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers)))4828 return false;4829 4830 // We don't need to check src0, all input types are legal, so just make sure4831 // src0 isn't using any modifiers.4832 if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers))4833 return false;4834 4835 // Can it be shrunk to a valid 32 bit opcode?4836 if (!hasVALU32BitEncoding(MI.getOpcode()))4837 return false;4838 4839 // Check output modifiers4840 return !hasModifiersSet(MI, AMDGPU::OpName::omod) &&4841 !hasModifiersSet(MI, AMDGPU::OpName::clamp) &&4842 !hasModifiersSet(MI, AMDGPU::OpName::byte_sel) &&4843 // TODO: Can we avoid checking bound_ctrl/fi here?4844 // They are only used by permlane*_swap special case.4845 !hasModifiersSet(MI, AMDGPU::OpName::bound_ctrl) &&4846 !hasModifiersSet(MI, AMDGPU::OpName::fi);4847}4848 4849// Set VCC operand with all flags from \p Orig, except for setting it as4850// implicit.4851static void copyFlagsToImplicitVCC(MachineInstr &MI,4852 const MachineOperand &Orig) {4853 4854 for (MachineOperand &Use : MI.implicit_operands()) {4855 if (Use.isUse() &&4856 (Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) {4857 Use.setIsUndef(Orig.isUndef());4858 Use.setIsKill(Orig.isKill());4859 return;4860 }4861 }4862}4863 4864MachineInstr *SIInstrInfo::buildShrunkInst(MachineInstr &MI,4865 unsigned Op32) const {4866 MachineBasicBlock *MBB = MI.getParent();4867 4868 const MCInstrDesc &Op32Desc = get(Op32);4869 MachineInstrBuilder Inst32 =4870 BuildMI(*MBB, MI, MI.getDebugLoc(), Op32Desc)4871 .setMIFlags(MI.getFlags());4872 4873 // Add the dst operand if the 32-bit encoding also has an explicit $vdst.4874 // For VOPC instructions, this is replaced by an implicit def of vcc.4875 4876 // We assume the defs of the shrunk opcode are in the same order, and the4877 // shrunk opcode loses the last def (SGPR def, in the VOP3->VOPC case).4878 for (int I = 0, E = Op32Desc.getNumDefs(); I != E; ++I)4879 Inst32.add(MI.getOperand(I));4880 4881 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);4882 4883 int Idx = MI.getNumExplicitDefs();4884 for (const MachineOperand &Use : MI.explicit_uses()) {4885 int OpTy = MI.getDesc().operands()[Idx++].OperandType;4886 if (OpTy == AMDGPU::OPERAND_INPUT_MODS || OpTy == MCOI::OPERAND_IMMEDIATE)4887 continue;4888 4889 if (&Use == Src2) {4890 if (AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2) == -1) {4891 // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is4892 // replaced with an implicit read of vcc or vcc_lo. The implicit read4893 // of vcc was already added during the initial BuildMI, but we4894 // 1) may need to change vcc to vcc_lo to preserve the original register4895 // 2) have to preserve the original flags.4896 copyFlagsToImplicitVCC(*Inst32, *Src2);4897 continue;4898 }4899 }4900 4901 Inst32.add(Use);4902 }4903 4904 // FIXME: Losing implicit operands4905 fixImplicitOperands(*Inst32);4906 return Inst32;4907}4908 4909bool SIInstrInfo::physRegUsesConstantBus(const MachineOperand &RegOp) const {4910 // Null is free4911 Register Reg = RegOp.getReg();4912 if (Reg == AMDGPU::SGPR_NULL || Reg == AMDGPU::SGPR_NULL64)4913 return false;4914 4915 // SGPRs use the constant bus4916 4917 // FIXME: implicit registers that are not part of the MCInstrDesc's implicit4918 // physical register operands should also count, except for exec.4919 if (RegOp.isImplicit())4920 return Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::M0;4921 4922 // SGPRs use the constant bus4923 return AMDGPU::SReg_32RegClass.contains(Reg) ||4924 AMDGPU::SReg_64RegClass.contains(Reg);4925}4926 4927bool SIInstrInfo::regUsesConstantBus(const MachineOperand &RegOp,4928 const MachineRegisterInfo &MRI) const {4929 Register Reg = RegOp.getReg();4930 return Reg.isVirtual() ? RI.isSGPRClass(MRI.getRegClass(Reg))4931 : physRegUsesConstantBus(RegOp);4932}4933 4934bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI,4935 const MachineOperand &MO,4936 const MCOperandInfo &OpInfo) const {4937 // Literal constants use the constant bus.4938 if (!MO.isReg())4939 return !isInlineConstant(MO, OpInfo);4940 4941 Register Reg = MO.getReg();4942 return Reg.isVirtual() ? RI.isSGPRClass(MRI.getRegClass(Reg))4943 : physRegUsesConstantBus(MO);4944}4945 4946static Register findImplicitSGPRRead(const MachineInstr &MI) {4947 for (const MachineOperand &MO : MI.implicit_operands()) {4948 // We only care about reads.4949 if (MO.isDef())4950 continue;4951 4952 switch (MO.getReg()) {4953 case AMDGPU::VCC:4954 case AMDGPU::VCC_LO:4955 case AMDGPU::VCC_HI:4956 case AMDGPU::M0:4957 case AMDGPU::FLAT_SCR:4958 return MO.getReg();4959 4960 default:4961 break;4962 }4963 }4964 4965 return Register();4966}4967 4968static bool shouldReadExec(const MachineInstr &MI) {4969 if (SIInstrInfo::isVALU(MI)) {4970 switch (MI.getOpcode()) {4971 case AMDGPU::V_READLANE_B32:4972 case AMDGPU::SI_RESTORE_S32_FROM_VGPR:4973 case AMDGPU::V_WRITELANE_B32:4974 case AMDGPU::SI_SPILL_S32_TO_VGPR:4975 return false;4976 }4977 4978 return true;4979 }4980 4981 if (MI.isPreISelOpcode() ||4982 SIInstrInfo::isGenericOpcode(MI.getOpcode()) ||4983 SIInstrInfo::isSALU(MI) ||4984 SIInstrInfo::isSMRD(MI))4985 return false;4986 4987 return true;4988}4989 4990static bool isRegOrFI(const MachineOperand &MO) {4991 return MO.isReg() || MO.isFI();4992}4993 4994static bool isSubRegOf(const SIRegisterInfo &TRI,4995 const MachineOperand &SuperVec,4996 const MachineOperand &SubReg) {4997 if (SubReg.getReg().isPhysical())4998 return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg());4999 5000 return SubReg.getSubReg() != AMDGPU::NoSubRegister &&5001 SubReg.getReg() == SuperVec.getReg();5002}5003 5004// Verify the illegal copy from vector register to SGPR for generic opcode COPY5005bool SIInstrInfo::verifyCopy(const MachineInstr &MI,5006 const MachineRegisterInfo &MRI,5007 StringRef &ErrInfo) const {5008 Register DstReg = MI.getOperand(0).getReg();5009 Register SrcReg = MI.getOperand(1).getReg();5010 // This is a check for copy from vector register to SGPR5011 if (RI.isVectorRegister(MRI, SrcReg) && RI.isSGPRReg(MRI, DstReg)) {5012 ErrInfo = "illegal copy from vector register to SGPR";5013 return false;5014 }5015 return true;5016}5017 5018bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,5019 StringRef &ErrInfo) const {5020 uint16_t Opcode = MI.getOpcode();5021 const MachineFunction *MF = MI.getMF();5022 const MachineRegisterInfo &MRI = MF->getRegInfo();5023 5024 // FIXME: At this point the COPY verify is done only for non-ssa forms.5025 // Find a better property to recognize the point where instruction selection5026 // is just done.5027 // We can only enforce this check after SIFixSGPRCopies pass so that the5028 // illegal copies are legalized and thereafter we don't expect a pass5029 // inserting similar copies.5030 if (!MRI.isSSA() && MI.isCopy())5031 return verifyCopy(MI, MRI, ErrInfo);5032 5033 if (SIInstrInfo::isGenericOpcode(Opcode))5034 return true;5035 5036 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);5037 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);5038 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);5039 int Src3Idx = -1;5040 if (Src0Idx == -1) {5041 // VOPD V_DUAL_* instructions use different operand names.5042 Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0X);5043 Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1X);5044 Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0Y);5045 Src3Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1Y);5046 }5047 5048 // Make sure the number of operands is correct.5049 const MCInstrDesc &Desc = get(Opcode);5050 if (!Desc.isVariadic() &&5051 Desc.getNumOperands() != MI.getNumExplicitOperands()) {5052 ErrInfo = "Instruction has wrong number of operands.";5053 return false;5054 }5055 5056 if (MI.isInlineAsm()) {5057 // Verify register classes for inlineasm constraints.5058 for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();5059 I != E; ++I) {5060 const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI);5061 if (!RC)5062 continue;5063 5064 const MachineOperand &Op = MI.getOperand(I);5065 if (!Op.isReg())5066 continue;5067 5068 Register Reg = Op.getReg();5069 if (!Reg.isVirtual() && !RC->contains(Reg)) {5070 ErrInfo = "inlineasm operand has incorrect register class.";5071 return false;5072 }5073 }5074 5075 return true;5076 }5077 5078 if (isImage(MI) && MI.memoperands_empty() && MI.mayLoadOrStore()) {5079 ErrInfo = "missing memory operand from image instruction.";5080 return false;5081 }5082 5083 // Make sure the register classes are correct.5084 for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {5085 const MachineOperand &MO = MI.getOperand(i);5086 if (MO.isFPImm()) {5087 ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "5088 "all fp values to integers.";5089 return false;5090 }5091 5092 const MCOperandInfo &OpInfo = Desc.operands()[i];5093 int16_t RegClass = getOpRegClassID(OpInfo);5094 5095 switch (OpInfo.OperandType) {5096 case MCOI::OPERAND_REGISTER:5097 if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) {5098 ErrInfo = "Illegal immediate value for operand.";5099 return false;5100 }5101 break;5102 case AMDGPU::OPERAND_REG_IMM_INT32:5103 case AMDGPU::OPERAND_REG_IMM_INT64:5104 case AMDGPU::OPERAND_REG_IMM_INT16:5105 case AMDGPU::OPERAND_REG_IMM_FP32:5106 case AMDGPU::OPERAND_REG_IMM_V2FP32:5107 case AMDGPU::OPERAND_REG_IMM_BF16:5108 case AMDGPU::OPERAND_REG_IMM_FP16:5109 case AMDGPU::OPERAND_REG_IMM_FP64:5110 case AMDGPU::OPERAND_REG_IMM_V2FP16:5111 case AMDGPU::OPERAND_REG_IMM_V2INT16:5112 case AMDGPU::OPERAND_REG_IMM_V2INT32:5113 case AMDGPU::OPERAND_REG_IMM_V2BF16:5114 break;5115 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:5116 break;5117 break;5118 case AMDGPU::OPERAND_REG_INLINE_C_INT16:5119 case AMDGPU::OPERAND_REG_INLINE_C_INT32:5120 case AMDGPU::OPERAND_REG_INLINE_C_INT64:5121 case AMDGPU::OPERAND_REG_INLINE_C_FP32:5122 case AMDGPU::OPERAND_REG_INLINE_C_FP64:5123 case AMDGPU::OPERAND_REG_INLINE_C_BF16:5124 case AMDGPU::OPERAND_REG_INLINE_C_FP16:5125 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:5126 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:5127 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:5128 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:5129 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:5130 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: {5131 if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, i))) {5132 ErrInfo = "Illegal immediate value for operand.";5133 return false;5134 }5135 break;5136 }5137 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:5138 if (!MI.getOperand(i).isImm() || !isInlineConstant(MI, i)) {5139 ErrInfo = "Expected inline constant for operand.";5140 return false;5141 }5142 break;5143 case AMDGPU::OPERAND_INPUT_MODS:5144 case AMDGPU::OPERAND_SDWA_VOPC_DST:5145 case AMDGPU::OPERAND_KIMM16:5146 break;5147 case MCOI::OPERAND_IMMEDIATE:5148 case AMDGPU::OPERAND_KIMM32:5149 case AMDGPU::OPERAND_KIMM64:5150 case AMDGPU::OPERAND_INLINE_C_AV64_PSEUDO:5151 // Check if this operand is an immediate.5152 // FrameIndex operands will be replaced by immediates, so they are5153 // allowed.5154 if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {5155 ErrInfo = "Expected immediate, but got non-immediate";5156 return false;5157 }5158 break;5159 case MCOI::OPERAND_UNKNOWN:5160 case MCOI::OPERAND_MEMORY:5161 case MCOI::OPERAND_PCREL:5162 break;5163 default:5164 if (OpInfo.isGenericType())5165 continue;5166 break;5167 }5168 5169 if (!MO.isReg())5170 continue;5171 Register Reg = MO.getReg();5172 if (!Reg)5173 continue;5174 5175 // FIXME: Ideally we would have separate instruction definitions with the5176 // aligned register constraint.5177 // FIXME: We do not verify inline asm operands, but custom inline asm5178 // verification is broken anyway5179 if (ST.needsAlignedVGPRs() && Opcode != AMDGPU::AV_MOV_B64_IMM_PSEUDO) {5180 const TargetRegisterClass *RC = RI.getRegClassForReg(MRI, Reg);5181 if (RI.hasVectorRegisters(RC) && MO.getSubReg()) {5182 if (const TargetRegisterClass *SubRC =5183 RI.getSubRegisterClass(RC, MO.getSubReg())) {5184 RC = RI.getCompatibleSubRegClass(RC, SubRC, MO.getSubReg());5185 if (RC)5186 RC = SubRC;5187 }5188 }5189 5190 // Check that this is the aligned version of the class.5191 if (!RC || !RI.isProperlyAlignedRC(*RC)) {5192 ErrInfo = "Subtarget requires even aligned vector registers";5193 return false;5194 }5195 }5196 5197 if (RegClass != -1) {5198 if (Reg.isVirtual())5199 continue;5200 5201 const TargetRegisterClass *RC = RI.getRegClass(RegClass);5202 if (!RC->contains(Reg)) {5203 ErrInfo = "Operand has incorrect register class.";5204 return false;5205 }5206 }5207 }5208 5209 // Verify SDWA5210 if (isSDWA(MI)) {5211 if (!ST.hasSDWA()) {5212 ErrInfo = "SDWA is not supported on this target";5213 return false;5214 }5215 5216 for (auto Op : {AMDGPU::OpName::src0_sel, AMDGPU::OpName::src1_sel,5217 AMDGPU::OpName::dst_sel}) {5218 const MachineOperand *MO = getNamedOperand(MI, Op);5219 if (!MO)5220 continue;5221 int64_t Imm = MO->getImm();5222 if (Imm < 0 || Imm > AMDGPU::SDWA::SdwaSel::DWORD) {5223 ErrInfo = "Invalid SDWA selection";5224 return false;5225 }5226 }5227 5228 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);5229 5230 for (int OpIdx : {DstIdx, Src0Idx, Src1Idx, Src2Idx}) {5231 if (OpIdx == -1)5232 continue;5233 const MachineOperand &MO = MI.getOperand(OpIdx);5234 5235 if (!ST.hasSDWAScalar()) {5236 // Only VGPRS on VI5237 if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) {5238 ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";5239 return false;5240 }5241 } else {5242 // No immediates on GFX95243 if (!MO.isReg()) {5244 ErrInfo =5245 "Only reg allowed as operands in SDWA instructions on GFX9+";5246 return false;5247 }5248 }5249 }5250 5251 if (!ST.hasSDWAOmod()) {5252 // No omod allowed on VI5253 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);5254 if (OMod != nullptr &&5255 (!OMod->isImm() || OMod->getImm() != 0)) {5256 ErrInfo = "OMod not allowed in SDWA instructions on VI";5257 return false;5258 }5259 }5260 5261 if (Opcode == AMDGPU::V_CVT_F32_FP8_sdwa ||5262 Opcode == AMDGPU::V_CVT_F32_BF8_sdwa ||5263 Opcode == AMDGPU::V_CVT_PK_F32_FP8_sdwa ||5264 Opcode == AMDGPU::V_CVT_PK_F32_BF8_sdwa) {5265 const MachineOperand *Src0ModsMO =5266 getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);5267 unsigned Mods = Src0ModsMO->getImm();5268 if (Mods & SISrcMods::ABS || Mods & SISrcMods::NEG ||5269 Mods & SISrcMods::SEXT) {5270 ErrInfo = "sext, abs and neg are not allowed on this instruction";5271 return false;5272 }5273 }5274 5275 uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);5276 if (isVOPC(BasicOpcode)) {5277 if (!ST.hasSDWASdst() && DstIdx != -1) {5278 // Only vcc allowed as dst on VI for VOPC5279 const MachineOperand &Dst = MI.getOperand(DstIdx);5280 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {5281 ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";5282 return false;5283 }5284 } else if (!ST.hasSDWAOutModsVOPC()) {5285 // No clamp allowed on GFX9 for VOPC5286 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);5287 if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {5288 ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";5289 return false;5290 }5291 5292 // No omod allowed on GFX9 for VOPC5293 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);5294 if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) {5295 ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";5296 return false;5297 }5298 }5299 }5300 5301 const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused);5302 if (DstUnused && DstUnused->isImm() &&5303 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) {5304 const MachineOperand &Dst = MI.getOperand(DstIdx);5305 if (!Dst.isReg() || !Dst.isTied()) {5306 ErrInfo = "Dst register should have tied register";5307 return false;5308 }5309 5310 const MachineOperand &TiedMO =5311 MI.getOperand(MI.findTiedOperandIdx(DstIdx));5312 if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {5313 ErrInfo =5314 "Dst register should be tied to implicit use of preserved register";5315 return false;5316 }5317 if (TiedMO.getReg().isPhysical() && Dst.getReg() != TiedMO.getReg()) {5318 ErrInfo = "Dst register should use same physical register as preserved";5319 return false;5320 }5321 }5322 }5323 5324 // Verify MIMG / VIMAGE / VSAMPLE5325 if (isImage(Opcode) && !MI.mayStore()) {5326 // Ensure that the return type used is large enough for all the options5327 // being used TFE/LWE require an extra result register.5328 const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask);5329 if (DMask) {5330 uint64_t DMaskImm = DMask->getImm();5331 uint32_t RegCount = isGather4(Opcode) ? 4 : llvm::popcount(DMaskImm);5332 const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe);5333 const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe);5334 const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16);5335 5336 // Adjust for packed 16 bit values5337 if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem())5338 RegCount = divideCeil(RegCount, 2);5339 5340 // Adjust if using LWE or TFE5341 if ((LWE && LWE->getImm()) || (TFE && TFE->getImm()))5342 RegCount += 1;5343 5344 const uint32_t DstIdx =5345 AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);5346 const MachineOperand &Dst = MI.getOperand(DstIdx);5347 if (Dst.isReg()) {5348 const TargetRegisterClass *DstRC = getOpRegClass(MI, DstIdx);5349 uint32_t DstSize = RI.getRegSizeInBits(*DstRC) / 32;5350 if (RegCount > DstSize) {5351 ErrInfo = "Image instruction returns too many registers for dst "5352 "register class";5353 return false;5354 }5355 }5356 }5357 }5358 5359 // Verify VOP*. Ignore multiple sgpr operands on writelane.5360 if (isVALU(MI) && Desc.getOpcode() != AMDGPU::V_WRITELANE_B32) {5361 unsigned ConstantBusCount = 0;5362 bool UsesLiteral = false;5363 const MachineOperand *LiteralVal = nullptr;5364 5365 int ImmIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm);5366 if (ImmIdx != -1) {5367 ++ConstantBusCount;5368 UsesLiteral = true;5369 LiteralVal = &MI.getOperand(ImmIdx);5370 }5371 5372 SmallVector<Register, 2> SGPRsUsed;5373 Register SGPRUsed;5374 5375 // Only look at the true operands. Only a real operand can use the constant5376 // bus, and we don't want to check pseudo-operands like the source modifier5377 // flags.5378 for (int OpIdx : {Src0Idx, Src1Idx, Src2Idx, Src3Idx}) {5379 if (OpIdx == -1)5380 continue;5381 const MachineOperand &MO = MI.getOperand(OpIdx);5382 if (usesConstantBus(MRI, MO, MI.getDesc().operands()[OpIdx])) {5383 if (MO.isReg()) {5384 SGPRUsed = MO.getReg();5385 if (!llvm::is_contained(SGPRsUsed, SGPRUsed)) {5386 ++ConstantBusCount;5387 SGPRsUsed.push_back(SGPRUsed);5388 }5389 } else if (!MO.isFI()) { // Treat FI like a register.5390 if (!UsesLiteral) {5391 ++ConstantBusCount;5392 UsesLiteral = true;5393 LiteralVal = &MO;5394 } else if (!MO.isIdenticalTo(*LiteralVal)) {5395 assert(isVOP2(MI) || isVOP3(MI));5396 ErrInfo = "VOP2/VOP3 instruction uses more than one literal";5397 return false;5398 }5399 }5400 }5401 }5402 5403 SGPRUsed = findImplicitSGPRRead(MI);5404 if (SGPRUsed) {5405 // Implicit uses may safely overlap true operands5406 if (llvm::all_of(SGPRsUsed, [this, SGPRUsed](unsigned SGPR) {5407 return !RI.regsOverlap(SGPRUsed, SGPR);5408 })) {5409 ++ConstantBusCount;5410 SGPRsUsed.push_back(SGPRUsed);5411 }5412 }5413 5414 // v_writelane_b32 is an exception from constant bus restriction:5415 // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const5416 if (ConstantBusCount > ST.getConstantBusLimit(Opcode) &&5417 Opcode != AMDGPU::V_WRITELANE_B32) {5418 ErrInfo = "VOP* instruction violates constant bus restriction";5419 return false;5420 }5421 5422 if (isVOP3(MI) && UsesLiteral && !ST.hasVOP3Literal()) {5423 ErrInfo = "VOP3 instruction uses literal";5424 return false;5425 }5426 }5427 5428 // Special case for writelane - this can break the multiple constant bus rule,5429 // but still can't use more than one SGPR register5430 if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) {5431 unsigned SGPRCount = 0;5432 Register SGPRUsed;5433 5434 for (int OpIdx : {Src0Idx, Src1Idx}) {5435 if (OpIdx == -1)5436 break;5437 5438 const MachineOperand &MO = MI.getOperand(OpIdx);5439 5440 if (usesConstantBus(MRI, MO, MI.getDesc().operands()[OpIdx])) {5441 if (MO.isReg() && MO.getReg() != AMDGPU::M0) {5442 if (MO.getReg() != SGPRUsed)5443 ++SGPRCount;5444 SGPRUsed = MO.getReg();5445 }5446 }5447 if (SGPRCount > ST.getConstantBusLimit(Opcode)) {5448 ErrInfo = "WRITELANE instruction violates constant bus restriction";5449 return false;5450 }5451 }5452 }5453 5454 // Verify misc. restrictions on specific instructions.5455 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 ||5456 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) {5457 const MachineOperand &Src0 = MI.getOperand(Src0Idx);5458 const MachineOperand &Src1 = MI.getOperand(Src1Idx);5459 const MachineOperand &Src2 = MI.getOperand(Src2Idx);5460 if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {5461 if (!compareMachineOp(Src0, Src1) &&5462 !compareMachineOp(Src0, Src2)) {5463 ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";5464 return false;5465 }5466 }5467 if ((getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm() &5468 SISrcMods::ABS) ||5469 (getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm() &5470 SISrcMods::ABS) ||5471 (getNamedOperand(MI, AMDGPU::OpName::src2_modifiers)->getImm() &5472 SISrcMods::ABS)) {5473 ErrInfo = "ABS not allowed in VOP3B instructions";5474 return false;5475 }5476 }5477 5478 if (isSOP2(MI) || isSOPC(MI)) {5479 const MachineOperand &Src0 = MI.getOperand(Src0Idx);5480 const MachineOperand &Src1 = MI.getOperand(Src1Idx);5481 5482 if (!isRegOrFI(Src0) && !isRegOrFI(Src1) &&5483 !isInlineConstant(Src0, Desc.operands()[Src0Idx]) &&5484 !isInlineConstant(Src1, Desc.operands()[Src1Idx]) &&5485 !Src0.isIdenticalTo(Src1)) {5486 ErrInfo = "SOP2/SOPC instruction requires too many immediate constants";5487 return false;5488 }5489 }5490 5491 if (isSOPK(MI)) {5492 const auto *Op = getNamedOperand(MI, AMDGPU::OpName::simm16);5493 if (Desc.isBranch()) {5494 if (!Op->isMBB()) {5495 ErrInfo = "invalid branch target for SOPK instruction";5496 return false;5497 }5498 } else {5499 uint64_t Imm = Op->getImm();5500 if (sopkIsZext(Opcode)) {5501 if (!isUInt<16>(Imm)) {5502 ErrInfo = "invalid immediate for SOPK instruction";5503 return false;5504 }5505 } else {5506 if (!isInt<16>(Imm)) {5507 ErrInfo = "invalid immediate for SOPK instruction";5508 return false;5509 }5510 }5511 }5512 }5513 5514 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||5515 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||5516 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||5517 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {5518 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||5519 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;5520 5521 const unsigned StaticNumOps =5522 Desc.getNumOperands() + Desc.implicit_uses().size();5523 const unsigned NumImplicitOps = IsDst ? 2 : 1;5524 5525 // Require additional implicit operands. This allows a fixup done by the5526 // post RA scheduler where the main implicit operand is killed and5527 // implicit-defs are added for sub-registers that remain live after this5528 // instruction.5529 if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {5530 ErrInfo = "missing implicit register operands";5531 return false;5532 }5533 5534 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);5535 if (IsDst) {5536 if (!Dst->isUse()) {5537 ErrInfo = "v_movreld_b32 vdst should be a use operand";5538 return false;5539 }5540 5541 unsigned UseOpIdx;5542 if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||5543 UseOpIdx != StaticNumOps + 1) {5544 ErrInfo = "movrel implicit operands should be tied";5545 return false;5546 }5547 }5548 5549 const MachineOperand &Src0 = MI.getOperand(Src0Idx);5550 const MachineOperand &ImpUse5551 = MI.getOperand(StaticNumOps + NumImplicitOps - 1);5552 if (!ImpUse.isReg() || !ImpUse.isUse() ||5553 !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {5554 ErrInfo = "src0 should be subreg of implicit vector use";5555 return false;5556 }5557 }5558 5559 // Make sure we aren't losing exec uses in the td files. This mostly requires5560 // being careful when using let Uses to try to add other use registers.5561 if (shouldReadExec(MI)) {5562 if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) {5563 ErrInfo = "VALU instruction does not implicitly read exec mask";5564 return false;5565 }5566 }5567 5568 if (isSMRD(MI)) {5569 if (MI.mayStore() &&5570 ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS) {5571 // The register offset form of scalar stores may only use m0 as the5572 // soffset register.5573 const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soffset);5574 if (Soff && Soff->getReg() != AMDGPU::M0) {5575 ErrInfo = "scalar stores must use m0 as offset register";5576 return false;5577 }5578 }5579 }5580 5581 if (isFLAT(MI) && !ST.hasFlatInstOffsets()) {5582 const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);5583 if (Offset->getImm() != 0) {5584 ErrInfo = "subtarget does not support offsets in flat instructions";5585 return false;5586 }5587 }5588 5589 if (isDS(MI) && !ST.hasGDS()) {5590 const MachineOperand *GDSOp = getNamedOperand(MI, AMDGPU::OpName::gds);5591 if (GDSOp && GDSOp->getImm() != 0) {5592 ErrInfo = "GDS is not supported on this subtarget";5593 return false;5594 }5595 }5596 5597 if (isImage(MI)) {5598 const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim);5599 if (DimOp) {5600 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode,5601 AMDGPU::OpName::vaddr0);5602 AMDGPU::OpName RSrcOpName =5603 isMIMG(MI) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;5604 int RsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, RSrcOpName);5605 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode);5606 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =5607 AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode);5608 const AMDGPU::MIMGDimInfo *Dim =5609 AMDGPU::getMIMGDimInfoByEncoding(DimOp->getImm());5610 5611 if (!Dim) {5612 ErrInfo = "dim is out of range";5613 return false;5614 }5615 5616 bool IsA16 = false;5617 if (ST.hasR128A16()) {5618 const MachineOperand *R128A16 = getNamedOperand(MI, AMDGPU::OpName::r128);5619 IsA16 = R128A16->getImm() != 0;5620 } else if (ST.hasA16()) {5621 const MachineOperand *A16 = getNamedOperand(MI, AMDGPU::OpName::a16);5622 IsA16 = A16->getImm() != 0;5623 }5624 5625 bool IsNSA = RsrcIdx - VAddr0Idx > 1;5626 5627 unsigned AddrWords =5628 AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim, IsA16, ST.hasG16());5629 5630 unsigned VAddrWords;5631 if (IsNSA) {5632 VAddrWords = RsrcIdx - VAddr0Idx;5633 if (ST.hasPartialNSAEncoding() &&5634 AddrWords > ST.getNSAMaxSize(isVSAMPLE(MI))) {5635 unsigned LastVAddrIdx = RsrcIdx - 1;5636 VAddrWords += getOpSize(MI, LastVAddrIdx) / 4 - 1;5637 }5638 } else {5639 VAddrWords = getOpSize(MI, VAddr0Idx) / 4;5640 if (AddrWords > 12)5641 AddrWords = 16;5642 }5643 5644 if (VAddrWords != AddrWords) {5645 LLVM_DEBUG(dbgs() << "bad vaddr size, expected " << AddrWords5646 << " but got " << VAddrWords << "\n");5647 ErrInfo = "bad vaddr size";5648 return false;5649 }5650 }5651 }5652 5653 const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl);5654 if (DppCt) {5655 using namespace AMDGPU::DPP;5656 5657 unsigned DC = DppCt->getImm();5658 if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||5659 DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||5660 (DC >= DppCtrl::DPP_UNUSED4_FIRST && DC <= DppCtrl::DPP_UNUSED4_LAST) ||5661 (DC >= DppCtrl::DPP_UNUSED5_FIRST && DC <= DppCtrl::DPP_UNUSED5_LAST) ||5662 (DC >= DppCtrl::DPP_UNUSED6_FIRST && DC <= DppCtrl::DPP_UNUSED6_LAST) ||5663 (DC >= DppCtrl::DPP_UNUSED7_FIRST && DC <= DppCtrl::DPP_UNUSED7_LAST) ||5664 (DC >= DppCtrl::DPP_UNUSED8_FIRST && DC <= DppCtrl::DPP_UNUSED8_LAST)) {5665 ErrInfo = "Invalid dpp_ctrl value";5666 return false;5667 }5668 if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 &&5669 ST.getGeneration() >= AMDGPUSubtarget::GFX10) {5670 ErrInfo = "Invalid dpp_ctrl value: "5671 "wavefront shifts are not supported on GFX10+";5672 return false;5673 }5674 if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 &&5675 ST.getGeneration() >= AMDGPUSubtarget::GFX10) {5676 ErrInfo = "Invalid dpp_ctrl value: "5677 "broadcasts are not supported on GFX10+";5678 return false;5679 }5680 if (DC >= DppCtrl::ROW_SHARE_FIRST && DC <= DppCtrl::ROW_XMASK_LAST &&5681 ST.getGeneration() < AMDGPUSubtarget::GFX10) {5682 if (DC >= DppCtrl::ROW_NEWBCAST_FIRST &&5683 DC <= DppCtrl::ROW_NEWBCAST_LAST &&5684 !ST.hasGFX90AInsts()) {5685 ErrInfo = "Invalid dpp_ctrl value: "5686 "row_newbroadcast/row_share is not supported before "5687 "GFX90A/GFX10";5688 return false;5689 }5690 if (DC > DppCtrl::ROW_NEWBCAST_LAST || !ST.hasGFX90AInsts()) {5691 ErrInfo = "Invalid dpp_ctrl value: "5692 "row_share and row_xmask are not supported before GFX10";5693 return false;5694 }5695 }5696 5697 if (Opcode != AMDGPU::V_MOV_B64_DPP_PSEUDO &&5698 !AMDGPU::isLegalDPALU_DPPControl(ST, DC) &&5699 AMDGPU::isDPALU_DPP(Desc, *this, ST)) {5700 ErrInfo = "Invalid dpp_ctrl value: "5701 "DP ALU dpp only support row_newbcast";5702 return false;5703 }5704 }5705 5706 if ((MI.mayStore() || MI.mayLoad()) && !isVGPRSpill(MI)) {5707 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);5708 AMDGPU::OpName DataName =5709 isDS(Opcode) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata;5710 const MachineOperand *Data = getNamedOperand(MI, DataName);5711 const MachineOperand *Data2 = getNamedOperand(MI, AMDGPU::OpName::data1);5712 if (Data && !Data->isReg())5713 Data = nullptr;5714 5715 if (ST.hasGFX90AInsts()) {5716 if (Dst && Data && !Dst->isTied() && !Data->isTied() &&5717 (RI.isAGPR(MRI, Dst->getReg()) != RI.isAGPR(MRI, Data->getReg()))) {5718 ErrInfo = "Invalid register class: "5719 "vdata and vdst should be both VGPR or AGPR";5720 return false;5721 }5722 if (Data && Data2 &&5723 (RI.isAGPR(MRI, Data->getReg()) != RI.isAGPR(MRI, Data2->getReg()))) {5724 ErrInfo = "Invalid register class: "5725 "both data operands should be VGPR or AGPR";5726 return false;5727 }5728 } else {5729 if ((Dst && RI.isAGPR(MRI, Dst->getReg())) ||5730 (Data && RI.isAGPR(MRI, Data->getReg())) ||5731 (Data2 && RI.isAGPR(MRI, Data2->getReg()))) {5732 ErrInfo = "Invalid register class: "5733 "agpr loads and stores not supported on this GPU";5734 return false;5735 }5736 }5737 }5738 5739 if (ST.needsAlignedVGPRs()) {5740 const auto isAlignedReg = [&MI, &MRI, this](AMDGPU::OpName OpName) -> bool {5741 const MachineOperand *Op = getNamedOperand(MI, OpName);5742 if (!Op)5743 return true;5744 Register Reg = Op->getReg();5745 if (Reg.isPhysical())5746 return !(RI.getHWRegIndex(Reg) & 1);5747 const TargetRegisterClass &RC = *MRI.getRegClass(Reg);5748 return RI.getRegSizeInBits(RC) > 32 && RI.isProperlyAlignedRC(RC) &&5749 !(RI.getChannelFromSubReg(Op->getSubReg()) & 1);5750 };5751 5752 if (Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_SEMA_BR ||5753 Opcode == AMDGPU::DS_GWS_BARRIER) {5754 5755 if (!isAlignedReg(AMDGPU::OpName::data0)) {5756 ErrInfo = "Subtarget requires even aligned vector registers "5757 "for DS_GWS instructions";5758 return false;5759 }5760 }5761 5762 if (isMIMG(MI)) {5763 if (!isAlignedReg(AMDGPU::OpName::vaddr)) {5764 ErrInfo = "Subtarget requires even aligned vector registers "5765 "for vaddr operand of image instructions";5766 return false;5767 }5768 }5769 }5770 5771 if (Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts()) {5772 const MachineOperand *Src = getNamedOperand(MI, AMDGPU::OpName::src0);5773 if (Src->isReg() && RI.isSGPRReg(MRI, Src->getReg())) {5774 ErrInfo = "Invalid register class: "5775 "v_accvgpr_write with an SGPR is not supported on this GPU";5776 return false;5777 }5778 }5779 5780 if (Desc.getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS) {5781 const MachineOperand &SrcOp = MI.getOperand(1);5782 if (!SrcOp.isReg() || SrcOp.getReg().isVirtual()) {5783 ErrInfo = "pseudo expects only physical SGPRs";5784 return false;5785 }5786 }5787 5788 if (const MachineOperand *CPol = getNamedOperand(MI, AMDGPU::OpName::cpol)) {5789 if (CPol->getImm() & AMDGPU::CPol::SCAL) {5790 if (!ST.hasScaleOffset()) {5791 ErrInfo = "Subtarget does not support offset scaling";5792 return false;5793 }5794 if (!AMDGPU::supportsScaleOffset(*this, MI.getOpcode())) {5795 ErrInfo = "Instruction does not support offset scaling";5796 return false;5797 }5798 }5799 }5800 5801 // See SIInstrInfo::isLegalGFX12PlusPackedMathFP32Operand for more5802 // information.5803 if (AMDGPU::isPackedFP32Inst(Opcode) && AMDGPU::isGFX12Plus(ST)) {5804 for (unsigned I = 0; I < 3; ++I) {5805 if (!isLegalGFX12PlusPackedMathFP32Operand(MRI, MI, I))5806 return false;5807 }5808 }5809 5810 return true;5811}5812 5813// It is more readable to list mapped opcodes on the same line.5814// clang-format off5815 5816unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {5817 switch (MI.getOpcode()) {5818 default: return AMDGPU::INSTRUCTION_LIST_END;5819 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;5820 case AMDGPU::COPY: return AMDGPU::COPY;5821 case AMDGPU::PHI: return AMDGPU::PHI;5822 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;5823 case AMDGPU::WQM: return AMDGPU::WQM;5824 case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM;5825 case AMDGPU::STRICT_WWM: return AMDGPU::STRICT_WWM;5826 case AMDGPU::STRICT_WQM: return AMDGPU::STRICT_WQM;5827 case AMDGPU::S_MOV_B32: {5828 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();5829 return MI.getOperand(1).isReg() ||5830 RI.isAGPR(MRI, MI.getOperand(0).getReg()) ?5831 AMDGPU::COPY : AMDGPU::V_MOV_B32_e32;5832 }5833 case AMDGPU::S_ADD_I32:5834 return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;5835 case AMDGPU::S_ADDC_U32:5836 return AMDGPU::V_ADDC_U32_e32;5837 case AMDGPU::S_SUB_I32:5838 return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32;5839 // FIXME: These are not consistently handled, and selected when the carry is5840 // used.5841 case AMDGPU::S_ADD_U32:5842 return AMDGPU::V_ADD_CO_U32_e32;5843 case AMDGPU::S_SUB_U32:5844 return AMDGPU::V_SUB_CO_U32_e32;5845 case AMDGPU::S_ADD_U64_PSEUDO:5846 return AMDGPU::V_ADD_U64_PSEUDO;5847 case AMDGPU::S_SUB_U64_PSEUDO:5848 return AMDGPU::V_SUB_U64_PSEUDO;5849 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;5850 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32_e64;5851 case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32_e64;5852 case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32_e64;5853 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;5854 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;5855 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;5856 case AMDGPU::S_XNOR_B32:5857 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;5858 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;5859 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;5860 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;5861 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;5862 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;5863 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64_e64;5864 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;5865 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64_e64;5866 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;5867 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64_e64;5868 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32_e64;5869 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32_e64;5870 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32_e64;5871 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32_e64;5872 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;5873 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;5874 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;5875 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;5876 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e64;5877 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e64;5878 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e64;5879 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e64;5880 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e64;5881 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e64;5882 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e64;5883 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e64;5884 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e64;5885 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e64;5886 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e64;5887 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e64;5888 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e64;5889 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e64;5890 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;5891 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;5892 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;5893 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;5894 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;5895 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;5896 case AMDGPU::S_CVT_F32_I32: return AMDGPU::V_CVT_F32_I32_e64;5897 case AMDGPU::S_CVT_F32_U32: return AMDGPU::V_CVT_F32_U32_e64;5898 case AMDGPU::S_CVT_I32_F32: return AMDGPU::V_CVT_I32_F32_e64;5899 case AMDGPU::S_CVT_U32_F32: return AMDGPU::V_CVT_U32_F32_e64;5900 case AMDGPU::S_CVT_F32_F16:5901 case AMDGPU::S_CVT_HI_F32_F16:5902 return ST.useRealTrue16Insts() ? AMDGPU::V_CVT_F32_F16_t16_e645903 : AMDGPU::V_CVT_F32_F16_fake16_e64;5904 case AMDGPU::S_CVT_F16_F32:5905 return ST.useRealTrue16Insts() ? AMDGPU::V_CVT_F16_F32_t16_e645906 : AMDGPU::V_CVT_F16_F32_fake16_e64;5907 case AMDGPU::S_CEIL_F32: return AMDGPU::V_CEIL_F32_e64;5908 case AMDGPU::S_FLOOR_F32: return AMDGPU::V_FLOOR_F32_e64;5909 case AMDGPU::S_TRUNC_F32: return AMDGPU::V_TRUNC_F32_e64;5910 case AMDGPU::S_RNDNE_F32: return AMDGPU::V_RNDNE_F32_e64;5911 case AMDGPU::S_CEIL_F16:5912 return ST.useRealTrue16Insts() ? AMDGPU::V_CEIL_F16_t16_e645913 : AMDGPU::V_CEIL_F16_fake16_e64;5914 case AMDGPU::S_FLOOR_F16:5915 return ST.useRealTrue16Insts() ? AMDGPU::V_FLOOR_F16_t16_e645916 : AMDGPU::V_FLOOR_F16_fake16_e64;5917 case AMDGPU::S_TRUNC_F16:5918 return ST.useRealTrue16Insts() ? AMDGPU::V_TRUNC_F16_t16_e645919 : AMDGPU::V_TRUNC_F16_fake16_e64;5920 case AMDGPU::S_RNDNE_F16:5921 return ST.useRealTrue16Insts() ? AMDGPU::V_RNDNE_F16_t16_e645922 : AMDGPU::V_RNDNE_F16_fake16_e64;5923 case AMDGPU::S_ADD_F32: return AMDGPU::V_ADD_F32_e64;5924 case AMDGPU::S_SUB_F32: return AMDGPU::V_SUB_F32_e64;5925 case AMDGPU::S_MIN_F32: return AMDGPU::V_MIN_F32_e64;5926 case AMDGPU::S_MAX_F32: return AMDGPU::V_MAX_F32_e64;5927 case AMDGPU::S_MINIMUM_F32: return AMDGPU::V_MINIMUM_F32_e64;5928 case AMDGPU::S_MAXIMUM_F32: return AMDGPU::V_MAXIMUM_F32_e64;5929 case AMDGPU::S_MUL_F32: return AMDGPU::V_MUL_F32_e64;5930 case AMDGPU::S_ADD_F16:5931 return ST.useRealTrue16Insts() ? AMDGPU::V_ADD_F16_t16_e645932 : AMDGPU::V_ADD_F16_fake16_e64;5933 case AMDGPU::S_SUB_F16:5934 return ST.useRealTrue16Insts() ? AMDGPU::V_SUB_F16_t16_e645935 : AMDGPU::V_SUB_F16_fake16_e64;5936 case AMDGPU::S_MIN_F16:5937 return ST.useRealTrue16Insts() ? AMDGPU::V_MIN_F16_t16_e645938 : AMDGPU::V_MIN_F16_fake16_e64;5939 case AMDGPU::S_MAX_F16:5940 return ST.useRealTrue16Insts() ? AMDGPU::V_MAX_F16_t16_e645941 : AMDGPU::V_MAX_F16_fake16_e64;5942 case AMDGPU::S_MINIMUM_F16:5943 return ST.useRealTrue16Insts() ? AMDGPU::V_MINIMUM_F16_t16_e645944 : AMDGPU::V_MINIMUM_F16_fake16_e64;5945 case AMDGPU::S_MAXIMUM_F16:5946 return ST.useRealTrue16Insts() ? AMDGPU::V_MAXIMUM_F16_t16_e645947 : AMDGPU::V_MAXIMUM_F16_fake16_e64;5948 case AMDGPU::S_MUL_F16:5949 return ST.useRealTrue16Insts() ? AMDGPU::V_MUL_F16_t16_e645950 : AMDGPU::V_MUL_F16_fake16_e64;5951 case AMDGPU::S_CVT_PK_RTZ_F16_F32: return AMDGPU::V_CVT_PKRTZ_F16_F32_e64;5952 case AMDGPU::S_FMAC_F32: return AMDGPU::V_FMAC_F32_e64;5953 case AMDGPU::S_FMAC_F16:5954 return ST.useRealTrue16Insts() ? AMDGPU::V_FMAC_F16_t16_e645955 : AMDGPU::V_FMAC_F16_fake16_e64;5956 case AMDGPU::S_FMAMK_F32: return AMDGPU::V_FMAMK_F32;5957 case AMDGPU::S_FMAAK_F32: return AMDGPU::V_FMAAK_F32;5958 case AMDGPU::S_CMP_LT_F32: return AMDGPU::V_CMP_LT_F32_e64;5959 case AMDGPU::S_CMP_EQ_F32: return AMDGPU::V_CMP_EQ_F32_e64;5960 case AMDGPU::S_CMP_LE_F32: return AMDGPU::V_CMP_LE_F32_e64;5961 case AMDGPU::S_CMP_GT_F32: return AMDGPU::V_CMP_GT_F32_e64;5962 case AMDGPU::S_CMP_LG_F32: return AMDGPU::V_CMP_LG_F32_e64;5963 case AMDGPU::S_CMP_GE_F32: return AMDGPU::V_CMP_GE_F32_e64;5964 case AMDGPU::S_CMP_O_F32: return AMDGPU::V_CMP_O_F32_e64;5965 case AMDGPU::S_CMP_U_F32: return AMDGPU::V_CMP_U_F32_e64;5966 case AMDGPU::S_CMP_NGE_F32: return AMDGPU::V_CMP_NGE_F32_e64;5967 case AMDGPU::S_CMP_NLG_F32: return AMDGPU::V_CMP_NLG_F32_e64;5968 case AMDGPU::S_CMP_NGT_F32: return AMDGPU::V_CMP_NGT_F32_e64;5969 case AMDGPU::S_CMP_NLE_F32: return AMDGPU::V_CMP_NLE_F32_e64;5970 case AMDGPU::S_CMP_NEQ_F32: return AMDGPU::V_CMP_NEQ_F32_e64;5971 case AMDGPU::S_CMP_NLT_F32: return AMDGPU::V_CMP_NLT_F32_e64;5972 case AMDGPU::S_CMP_LT_F16:5973 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LT_F16_t16_e645974 : AMDGPU::V_CMP_LT_F16_fake16_e64;5975 case AMDGPU::S_CMP_EQ_F16:5976 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_EQ_F16_t16_e645977 : AMDGPU::V_CMP_EQ_F16_fake16_e64;5978 case AMDGPU::S_CMP_LE_F16:5979 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LE_F16_t16_e645980 : AMDGPU::V_CMP_LE_F16_fake16_e64;5981 case AMDGPU::S_CMP_GT_F16:5982 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_GT_F16_t16_e645983 : AMDGPU::V_CMP_GT_F16_fake16_e64;5984 case AMDGPU::S_CMP_LG_F16:5985 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LG_F16_t16_e645986 : AMDGPU::V_CMP_LG_F16_fake16_e64;5987 case AMDGPU::S_CMP_GE_F16:5988 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_GE_F16_t16_e645989 : AMDGPU::V_CMP_GE_F16_fake16_e64;5990 case AMDGPU::S_CMP_O_F16:5991 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_O_F16_t16_e645992 : AMDGPU::V_CMP_O_F16_fake16_e64;5993 case AMDGPU::S_CMP_U_F16:5994 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_U_F16_t16_e645995 : AMDGPU::V_CMP_U_F16_fake16_e64;5996 case AMDGPU::S_CMP_NGE_F16:5997 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NGE_F16_t16_e645998 : AMDGPU::V_CMP_NGE_F16_fake16_e64;5999 case AMDGPU::S_CMP_NLG_F16:6000 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLG_F16_t16_e646001 : AMDGPU::V_CMP_NLG_F16_fake16_e64;6002 case AMDGPU::S_CMP_NGT_F16:6003 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NGT_F16_t16_e646004 : AMDGPU::V_CMP_NGT_F16_fake16_e64;6005 case AMDGPU::S_CMP_NLE_F16:6006 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLE_F16_t16_e646007 : AMDGPU::V_CMP_NLE_F16_fake16_e64;6008 case AMDGPU::S_CMP_NEQ_F16:6009 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NEQ_F16_t16_e646010 : AMDGPU::V_CMP_NEQ_F16_fake16_e64;6011 case AMDGPU::S_CMP_NLT_F16:6012 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLT_F16_t16_e646013 : AMDGPU::V_CMP_NLT_F16_fake16_e64;6014 case AMDGPU::V_S_EXP_F32_e64: return AMDGPU::V_EXP_F32_e64;6015 case AMDGPU::V_S_EXP_F16_e64:6016 return ST.useRealTrue16Insts() ? AMDGPU::V_EXP_F16_t16_e646017 : AMDGPU::V_EXP_F16_fake16_e64;6018 case AMDGPU::V_S_LOG_F32_e64: return AMDGPU::V_LOG_F32_e64;6019 case AMDGPU::V_S_LOG_F16_e64:6020 return ST.useRealTrue16Insts() ? AMDGPU::V_LOG_F16_t16_e646021 : AMDGPU::V_LOG_F16_fake16_e64;6022 case AMDGPU::V_S_RCP_F32_e64: return AMDGPU::V_RCP_F32_e64;6023 case AMDGPU::V_S_RCP_F16_e64:6024 return ST.useRealTrue16Insts() ? AMDGPU::V_RCP_F16_t16_e646025 : AMDGPU::V_RCP_F16_fake16_e64;6026 case AMDGPU::V_S_RSQ_F32_e64: return AMDGPU::V_RSQ_F32_e64;6027 case AMDGPU::V_S_RSQ_F16_e64:6028 return ST.useRealTrue16Insts() ? AMDGPU::V_RSQ_F16_t16_e646029 : AMDGPU::V_RSQ_F16_fake16_e64;6030 case AMDGPU::V_S_SQRT_F32_e64: return AMDGPU::V_SQRT_F32_e64;6031 case AMDGPU::V_S_SQRT_F16_e64:6032 return ST.useRealTrue16Insts() ? AMDGPU::V_SQRT_F16_t16_e646033 : AMDGPU::V_SQRT_F16_fake16_e64;6034 }6035 llvm_unreachable(6036 "Unexpected scalar opcode without corresponding vector one!");6037}6038 6039// clang-format on6040 6041void SIInstrInfo::insertScratchExecCopy(MachineFunction &MF,6042 MachineBasicBlock &MBB,6043 MachineBasicBlock::iterator MBBI,6044 const DebugLoc &DL, Register Reg,6045 bool IsSCCLive,6046 SlotIndexes *Indexes) const {6047 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();6048 const SIInstrInfo *TII = ST.getInstrInfo();6049 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);6050 if (IsSCCLive) {6051 // Insert two move instructions, one to save the original value of EXEC and6052 // the other to turn on all bits in EXEC. This is required as we can't use6053 // the single instruction S_OR_SAVEEXEC that clobbers SCC.6054 auto StoreExecMI = BuildMI(MBB, MBBI, DL, TII->get(LMC.MovOpc), Reg)6055 .addReg(LMC.ExecReg, RegState::Kill);6056 auto FlipExecMI =6057 BuildMI(MBB, MBBI, DL, TII->get(LMC.MovOpc), LMC.ExecReg).addImm(-1);6058 if (Indexes) {6059 Indexes->insertMachineInstrInMaps(*StoreExecMI);6060 Indexes->insertMachineInstrInMaps(*FlipExecMI);6061 }6062 } else {6063 auto SaveExec =6064 BuildMI(MBB, MBBI, DL, TII->get(LMC.OrSaveExecOpc), Reg).addImm(-1);6065 SaveExec->getOperand(3).setIsDead(); // Mark SCC as dead.6066 if (Indexes)6067 Indexes->insertMachineInstrInMaps(*SaveExec);6068 }6069}6070 6071void SIInstrInfo::restoreExec(MachineFunction &MF, MachineBasicBlock &MBB,6072 MachineBasicBlock::iterator MBBI,6073 const DebugLoc &DL, Register Reg,6074 SlotIndexes *Indexes) const {6075 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);6076 auto ExecRestoreMI = BuildMI(MBB, MBBI, DL, get(LMC.MovOpc), LMC.ExecReg)6077 .addReg(Reg, RegState::Kill);6078 if (Indexes)6079 Indexes->insertMachineInstrInMaps(*ExecRestoreMI);6080}6081 6082MachineInstr *6083SIInstrInfo::getWholeWaveFunctionSetup(MachineFunction &MF) const {6084 assert(MF.getInfo<SIMachineFunctionInfo>()->isWholeWaveFunction() &&6085 "Not a whole wave func");6086 MachineBasicBlock &MBB = *MF.begin();6087 for (MachineInstr &MI : MBB)6088 if (MI.getOpcode() == AMDGPU::SI_WHOLE_WAVE_FUNC_SETUP ||6089 MI.getOpcode() == AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_SETUP)6090 return &MI;6091 6092 llvm_unreachable("Couldn't find SI_SETUP_WHOLE_WAVE_FUNC instruction");6093}6094 6095const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI,6096 unsigned OpNo) const {6097 const MCInstrDesc &Desc = get(MI.getOpcode());6098 if (MI.isVariadic() || OpNo >= Desc.getNumOperands() ||6099 Desc.operands()[OpNo].RegClass == -1) {6100 Register Reg = MI.getOperand(OpNo).getReg();6101 6102 if (Reg.isVirtual()) {6103 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();6104 return MRI.getRegClass(Reg);6105 }6106 return RI.getPhysRegBaseClass(Reg);6107 }6108 6109 int16_t RegClass = getOpRegClassID(Desc.operands()[OpNo]);6110 return RegClass < 0 ? nullptr : RI.getRegClass(RegClass);6111}6112 6113void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {6114 MachineBasicBlock::iterator I = MI;6115 MachineBasicBlock *MBB = MI.getParent();6116 MachineOperand &MO = MI.getOperand(OpIdx);6117 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();6118 unsigned RCID = getOpRegClassID(get(MI.getOpcode()).operands()[OpIdx]);6119 const TargetRegisterClass *RC = RI.getRegClass(RCID);6120 unsigned Size = RI.getRegSizeInBits(*RC);6121 unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO6122 : Size == 16 ? AMDGPU::V_MOV_B16_t16_e646123 : AMDGPU::V_MOV_B32_e32;6124 if (MO.isReg())6125 Opcode = AMDGPU::COPY;6126 else if (RI.isSGPRClass(RC))6127 Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;6128 6129 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC);6130 Register Reg = MRI.createVirtualRegister(VRC);6131 DebugLoc DL = MBB->findDebugLoc(I);6132 BuildMI(*MI.getParent(), I, DL, get(Opcode), Reg).add(MO);6133 MO.ChangeToRegister(Reg, false);6134}6135 6136unsigned SIInstrInfo::buildExtractSubReg(6137 MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI,6138 const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC,6139 unsigned SubIdx, const TargetRegisterClass *SubRC) const {6140 if (!SuperReg.getReg().isVirtual())6141 return RI.getSubReg(SuperReg.getReg(), SubIdx);6142 6143 MachineBasicBlock *MBB = MI->getParent();6144 const DebugLoc &DL = MI->getDebugLoc();6145 Register SubReg = MRI.createVirtualRegister(SubRC);6146 6147 unsigned NewSubIdx = RI.composeSubRegIndices(SuperReg.getSubReg(), SubIdx);6148 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)6149 .addReg(SuperReg.getReg(), 0, NewSubIdx);6150 return SubReg;6151}6152 6153MachineOperand SIInstrInfo::buildExtractSubRegOrImm(6154 MachineBasicBlock::iterator MII, MachineRegisterInfo &MRI,6155 const MachineOperand &Op, const TargetRegisterClass *SuperRC,6156 unsigned SubIdx, const TargetRegisterClass *SubRC) const {6157 if (Op.isImm()) {6158 if (SubIdx == AMDGPU::sub0)6159 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm()));6160 if (SubIdx == AMDGPU::sub1)6161 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm() >> 32));6162 6163 llvm_unreachable("Unhandled register index for immediate");6164 }6165 6166 unsigned SubReg = buildExtractSubReg(MII, MRI, Op, SuperRC,6167 SubIdx, SubRC);6168 return MachineOperand::CreateReg(SubReg, false);6169}6170 6171// Change the order of operands from (0, 1, 2) to (0, 2, 1)6172void SIInstrInfo::swapOperands(MachineInstr &Inst) const {6173 assert(Inst.getNumExplicitOperands() == 3);6174 MachineOperand Op1 = Inst.getOperand(1);6175 Inst.removeOperand(1);6176 Inst.addOperand(Op1);6177}6178 6179bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI,6180 const MCOperandInfo &OpInfo,6181 const MachineOperand &MO) const {6182 if (!MO.isReg())6183 return false;6184 6185 Register Reg = MO.getReg();6186 6187 const TargetRegisterClass *DRC = RI.getRegClass(getOpRegClassID(OpInfo));6188 if (Reg.isPhysical())6189 return DRC->contains(Reg);6190 6191 const TargetRegisterClass *RC = MRI.getRegClass(Reg);6192 6193 if (MO.getSubReg()) {6194 const MachineFunction *MF = MO.getParent()->getMF();6195 const TargetRegisterClass *SuperRC = RI.getLargestLegalSuperClass(RC, *MF);6196 if (!SuperRC)6197 return false;6198 return RI.getMatchingSuperRegClass(SuperRC, DRC, MO.getSubReg()) != nullptr;6199 }6200 6201 return RI.getCommonSubClass(DRC, RC) != nullptr;6202}6203 6204bool SIInstrInfo::isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx,6205 const MachineOperand &MO) const {6206 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();6207 const MCOperandInfo OpInfo = MI.getDesc().operands()[OpIdx];6208 unsigned Opc = MI.getOpcode();6209 6210 // See SIInstrInfo::isLegalGFX12PlusPackedMathFP32Operand for more6211 // information.6212 if (AMDGPU::isPackedFP32Inst(MI.getOpcode()) && AMDGPU::isGFX12Plus(ST) &&6213 MO.isReg() && RI.isSGPRReg(MRI, MO.getReg())) {6214 constexpr AMDGPU::OpName OpNames[] = {6215 AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2};6216 6217 for (auto [I, OpName] : enumerate(OpNames)) {6218 int SrcIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpNames[I]);6219 if (static_cast<unsigned>(SrcIdx) == OpIdx &&6220 !isLegalGFX12PlusPackedMathFP32Operand(MRI, MI, I, &MO))6221 return false;6222 }6223 }6224 6225 if (!isLegalRegOperand(MRI, OpInfo, MO))6226 return false;6227 6228 // check Accumulate GPR operand6229 bool IsAGPR = RI.isAGPR(MRI, MO.getReg());6230 if (IsAGPR && !ST.hasMAIInsts())6231 return false;6232 if (IsAGPR && (!ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) &&6233 (MI.mayLoad() || MI.mayStore() || isDS(Opc) || isMIMG(Opc)))6234 return false;6235 // Atomics should have both vdst and vdata either vgpr or agpr.6236 const int VDstIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);6237 const int DataIdx = AMDGPU::getNamedOperandIdx(6238 Opc, isDS(Opc) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata);6239 if ((int)OpIdx == VDstIdx && DataIdx != -1 &&6240 MI.getOperand(DataIdx).isReg() &&6241 RI.isAGPR(MRI, MI.getOperand(DataIdx).getReg()) != IsAGPR)6242 return false;6243 if ((int)OpIdx == DataIdx) {6244 if (VDstIdx != -1 &&6245 RI.isAGPR(MRI, MI.getOperand(VDstIdx).getReg()) != IsAGPR)6246 return false;6247 // DS instructions with 2 src operands also must have tied RC.6248 const int Data1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1);6249 if (Data1Idx != -1 && MI.getOperand(Data1Idx).isReg() &&6250 RI.isAGPR(MRI, MI.getOperand(Data1Idx).getReg()) != IsAGPR)6251 return false;6252 }6253 6254 // Check V_ACCVGPR_WRITE_B32_e646255 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts() &&6256 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) &&6257 RI.isSGPRReg(MRI, MO.getReg()))6258 return false;6259 return true;6260}6261 6262bool SIInstrInfo::isLegalVSrcOperand(const MachineRegisterInfo &MRI,6263 const MCOperandInfo &OpInfo,6264 const MachineOperand &MO) const {6265 if (MO.isReg())6266 return isLegalRegOperand(MRI, OpInfo, MO);6267 6268 // Handle non-register types that are treated like immediates.6269 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());6270 return true;6271}6272 6273bool SIInstrInfo::isLegalGFX12PlusPackedMathFP32Operand(6274 const MachineRegisterInfo &MRI, const MachineInstr &MI, unsigned SrcN,6275 const MachineOperand *MO) const {6276 constexpr unsigned NumOps = 3;6277 constexpr AMDGPU::OpName OpNames[NumOps * 2] = {6278 AMDGPU::OpName::src0, AMDGPU::OpName::src1,6279 AMDGPU::OpName::src2, AMDGPU::OpName::src0_modifiers,6280 AMDGPU::OpName::src1_modifiers, AMDGPU::OpName::src2_modifiers};6281 6282 assert(SrcN < NumOps);6283 6284 if (!MO) {6285 int SrcIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpNames[SrcN]);6286 if (SrcIdx == -1)6287 return true;6288 MO = &MI.getOperand(SrcIdx);6289 }6290 6291 if (!MO->isReg() || !RI.isSGPRReg(MRI, MO->getReg()))6292 return true;6293 6294 int ModsIdx =6295 AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpNames[NumOps + SrcN]);6296 if (ModsIdx == -1)6297 return true;6298 6299 unsigned Mods = MI.getOperand(ModsIdx).getImm();6300 bool OpSel = Mods & SISrcMods::OP_SEL_0;6301 bool OpSelHi = Mods & SISrcMods::OP_SEL_1;6302 6303 return !OpSel && !OpSelHi;6304}6305 6306bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,6307 const MachineOperand *MO) const {6308 const MachineFunction &MF = *MI.getMF();6309 const MachineRegisterInfo &MRI = MF.getRegInfo();6310 const MCInstrDesc &InstDesc = MI.getDesc();6311 const MCOperandInfo &OpInfo = InstDesc.operands()[OpIdx];6312 int64_t RegClass = getOpRegClassID(OpInfo);6313 const TargetRegisterClass *DefinedRC =6314 RegClass != -1 ? RI.getRegClass(RegClass) : nullptr;6315 if (!MO)6316 MO = &MI.getOperand(OpIdx);6317 6318 const bool IsInlineConst = !MO->isReg() && isInlineConstant(*MO, OpInfo);6319 6320 if (isVALU(MI) && !IsInlineConst && usesConstantBus(MRI, *MO, OpInfo)) {6321 const MachineOperand *UsedLiteral = nullptr;6322 6323 int ConstantBusLimit = ST.getConstantBusLimit(MI.getOpcode());6324 int LiteralLimit = !isVOP3(MI) || ST.hasVOP3Literal() ? 1 : 0;6325 6326 // TODO: Be more permissive with frame indexes.6327 if (!MO->isReg() && !isInlineConstant(*MO, OpInfo)) {6328 if (!LiteralLimit--)6329 return false;6330 6331 UsedLiteral = MO;6332 }6333 6334 SmallDenseSet<RegSubRegPair> SGPRsUsed;6335 if (MO->isReg())6336 SGPRsUsed.insert(RegSubRegPair(MO->getReg(), MO->getSubReg()));6337 6338 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {6339 if (i == OpIdx)6340 continue;6341 const MachineOperand &Op = MI.getOperand(i);6342 if (Op.isReg()) {6343 if (Op.isUse()) {6344 RegSubRegPair SGPR(Op.getReg(), Op.getSubReg());6345 if (regUsesConstantBus(Op, MRI) && SGPRsUsed.insert(SGPR).second) {6346 if (--ConstantBusLimit <= 0)6347 return false;6348 }6349 }6350 } else if (AMDGPU::isSISrcOperand(InstDesc.operands()[i]) &&6351 !isInlineConstant(Op, InstDesc.operands()[i])) {6352 // The same literal may be used multiple times.6353 if (!UsedLiteral)6354 UsedLiteral = &Op;6355 else if (UsedLiteral->isIdenticalTo(Op))6356 continue;6357 6358 if (!LiteralLimit--)6359 return false;6360 if (--ConstantBusLimit <= 0)6361 return false;6362 }6363 }6364 } else if (!IsInlineConst && !MO->isReg() && isSALU(MI)) {6365 // There can be at most one literal operand, but it can be repeated.6366 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {6367 if (i == OpIdx)6368 continue;6369 const MachineOperand &Op = MI.getOperand(i);6370 if (!Op.isReg() && !Op.isFI() && !Op.isRegMask() &&6371 !isInlineConstant(Op, InstDesc.operands()[i]) &&6372 !Op.isIdenticalTo(*MO))6373 return false;6374 6375 // Do not fold a non-inlineable and non-register operand into an6376 // instruction that already has a frame index. The frame index handling6377 // code could not handle well when a frame index co-exists with another6378 // non-register operand, unless that operand is an inlineable immediate.6379 if (Op.isFI())6380 return false;6381 }6382 } else if (IsInlineConst && ST.hasNoF16PseudoScalarTransInlineConstants() &&6383 isF16PseudoScalarTrans(MI.getOpcode())) {6384 return false;6385 }6386 6387 if (MO->isReg()) {6388 if (!DefinedRC)6389 return OpInfo.OperandType == MCOI::OPERAND_UNKNOWN;6390 return isLegalRegOperand(MI, OpIdx, *MO);6391 }6392 6393 if (MO->isImm()) {6394 uint64_t Imm = MO->getImm();6395 bool Is64BitFPOp = OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_FP64;6396 bool Is64BitOp = Is64BitFPOp ||6397 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_INT64 ||6398 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2INT32 ||6399 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP32;6400 if (Is64BitOp &&6401 !AMDGPU::isInlinableLiteral64(Imm, ST.hasInv2PiInlineImm())) {6402 if (!AMDGPU::isValid32BitLiteral(Imm, Is64BitFPOp) &&6403 (!ST.has64BitLiterals() || InstDesc.getSize() != 4))6404 return false;6405 6406 // FIXME: We can use sign extended 64-bit literals, but only for signed6407 // operands. At the moment we do not know if an operand is signed.6408 // Such operand will be encoded as its low 32 bits and then either6409 // correctly sign extended or incorrectly zero extended by HW.6410 // If 64-bit literals are supported and the literal will be encoded6411 // as full 64 bit we still can use it.6412 if (!Is64BitFPOp && (int32_t)Imm < 0 &&6413 (!ST.has64BitLiterals() || AMDGPU::isValid32BitLiteral(Imm, false)))6414 return false;6415 }6416 }6417 6418 // Handle non-register types that are treated like immediates.6419 assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal());6420 6421 if (!DefinedRC) {6422 // This operand expects an immediate.6423 return true;6424 }6425 6426 return isImmOperandLegal(MI, OpIdx, *MO);6427}6428 6429bool SIInstrInfo::isNeverCoissue(MachineInstr &MI) const {6430 bool IsGFX950Only = ST.hasGFX950Insts();6431 bool IsGFX940Only = ST.hasGFX940Insts();6432 6433 if (!IsGFX950Only && !IsGFX940Only)6434 return false;6435 6436 if (!isVALU(MI))6437 return false;6438 6439 // V_COS, V_EXP, V_RCP, etc.6440 if (isTRANS(MI))6441 return true;6442 6443 // DOT2, DOT2C, DOT4, etc.6444 if (isDOT(MI))6445 return true;6446 6447 // MFMA, SMFMA6448 if (isMFMA(MI))6449 return true;6450 6451 unsigned Opcode = MI.getOpcode();6452 switch (Opcode) {6453 case AMDGPU::V_CVT_PK_BF8_F32_e64:6454 case AMDGPU::V_CVT_PK_FP8_F32_e64:6455 case AMDGPU::V_MQSAD_PK_U16_U8_e64:6456 case AMDGPU::V_MQSAD_U32_U8_e64:6457 case AMDGPU::V_PK_ADD_F16:6458 case AMDGPU::V_PK_ADD_F32:6459 case AMDGPU::V_PK_ADD_I16:6460 case AMDGPU::V_PK_ADD_U16:6461 case AMDGPU::V_PK_ASHRREV_I16:6462 case AMDGPU::V_PK_FMA_F16:6463 case AMDGPU::V_PK_FMA_F32:6464 case AMDGPU::V_PK_FMAC_F16_e32:6465 case AMDGPU::V_PK_FMAC_F16_e64:6466 case AMDGPU::V_PK_LSHLREV_B16:6467 case AMDGPU::V_PK_LSHRREV_B16:6468 case AMDGPU::V_PK_MAD_I16:6469 case AMDGPU::V_PK_MAD_U16:6470 case AMDGPU::V_PK_MAX_F16:6471 case AMDGPU::V_PK_MAX_I16:6472 case AMDGPU::V_PK_MAX_U16:6473 case AMDGPU::V_PK_MIN_F16:6474 case AMDGPU::V_PK_MIN_I16:6475 case AMDGPU::V_PK_MIN_U16:6476 case AMDGPU::V_PK_MOV_B32:6477 case AMDGPU::V_PK_MUL_F16:6478 case AMDGPU::V_PK_MUL_F32:6479 case AMDGPU::V_PK_MUL_LO_U16:6480 case AMDGPU::V_PK_SUB_I16:6481 case AMDGPU::V_PK_SUB_U16:6482 case AMDGPU::V_QSAD_PK_U16_U8_e64:6483 return true;6484 default:6485 return false;6486 }6487}6488 6489void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo &MRI,6490 MachineInstr &MI) const {6491 unsigned Opc = MI.getOpcode();6492 const MCInstrDesc &InstrDesc = get(Opc);6493 6494 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);6495 MachineOperand &Src0 = MI.getOperand(Src0Idx);6496 6497 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);6498 MachineOperand &Src1 = MI.getOperand(Src1Idx);6499 6500 // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u326501 // we need to only have one constant bus use before GFX10.6502 bool HasImplicitSGPR = findImplicitSGPRRead(MI);6503 if (HasImplicitSGPR && ST.getConstantBusLimit(Opc) <= 1 && Src0.isReg() &&6504 RI.isSGPRReg(MRI, Src0.getReg()))6505 legalizeOpWithMove(MI, Src0Idx);6506 6507 // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for6508 // both the value to write (src0) and lane select (src1). Fix up non-SGPR6509 // src0/src1 with V_READFIRSTLANE.6510 if (Opc == AMDGPU::V_WRITELANE_B32) {6511 const DebugLoc &DL = MI.getDebugLoc();6512 if (Src0.isReg() && RI.isVGPR(MRI, Src0.getReg())) {6513 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);6514 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)6515 .add(Src0);6516 Src0.ChangeToRegister(Reg, false);6517 }6518 if (Src1.isReg() && RI.isVGPR(MRI, Src1.getReg())) {6519 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);6520 const DebugLoc &DL = MI.getDebugLoc();6521 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)6522 .add(Src1);6523 Src1.ChangeToRegister(Reg, false);6524 }6525 return;6526 }6527 6528 // Special case: V_FMAC_F32 and V_FMAC_F16 have src2.6529 if (Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F16_e32) {6530 int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);6531 if (!RI.isVGPR(MRI, MI.getOperand(Src2Idx).getReg()))6532 legalizeOpWithMove(MI, Src2Idx);6533 }6534 6535 // VOP2 src0 instructions support all operand types, so we don't need to check6536 // their legality. If src1 is already legal, we don't need to do anything.6537 if (isLegalRegOperand(MRI, InstrDesc.operands()[Src1Idx], Src1))6538 return;6539 6540 // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for6541 // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane6542 // select is uniform.6543 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() &&6544 RI.isVGPR(MRI, Src1.getReg())) {6545 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);6546 const DebugLoc &DL = MI.getDebugLoc();6547 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)6548 .add(Src1);6549 Src1.ChangeToRegister(Reg, false);6550 return;6551 }6552 6553 // We do not use commuteInstruction here because it is too aggressive and will6554 // commute if it is possible. We only want to commute here if it improves6555 // legality. This can be called a fairly large number of times so don't waste6556 // compile time pointlessly swapping and checking legality again.6557 if (HasImplicitSGPR || !MI.isCommutable()) {6558 legalizeOpWithMove(MI, Src1Idx);6559 return;6560 }6561 6562 // If src0 can be used as src1, commuting will make the operands legal.6563 // Otherwise we have to give up and insert a move.6564 //6565 // TODO: Other immediate-like operand kinds could be commuted if there was a6566 // MachineOperand::ChangeTo* for them.6567 if ((!Src1.isImm() && !Src1.isReg()) ||6568 !isLegalRegOperand(MRI, InstrDesc.operands()[Src1Idx], Src0)) {6569 legalizeOpWithMove(MI, Src1Idx);6570 return;6571 }6572 6573 int CommutedOpc = commuteOpcode(MI);6574 if (CommutedOpc == -1) {6575 legalizeOpWithMove(MI, Src1Idx);6576 return;6577 }6578 6579 MI.setDesc(get(CommutedOpc));6580 6581 Register Src0Reg = Src0.getReg();6582 unsigned Src0SubReg = Src0.getSubReg();6583 bool Src0Kill = Src0.isKill();6584 6585 if (Src1.isImm())6586 Src0.ChangeToImmediate(Src1.getImm());6587 else if (Src1.isReg()) {6588 Src0.ChangeToRegister(Src1.getReg(), false, false, Src1.isKill());6589 Src0.setSubReg(Src1.getSubReg());6590 } else6591 llvm_unreachable("Should only have register or immediate operands");6592 6593 Src1.ChangeToRegister(Src0Reg, false, false, Src0Kill);6594 Src1.setSubReg(Src0SubReg);6595 fixImplicitOperands(MI);6596}6597 6598// Legalize VOP3 operands. All operand types are supported for any operand6599// but only one literal constant and only starting from GFX10.6600void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI,6601 MachineInstr &MI) const {6602 unsigned Opc = MI.getOpcode();6603 6604 int VOP3Idx[3] = {6605 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0),6606 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1),6607 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)6608 };6609 6610 if (Opc == AMDGPU::V_PERMLANE16_B32_e64 ||6611 Opc == AMDGPU::V_PERMLANEX16_B32_e64 ||6612 Opc == AMDGPU::V_PERMLANE_BCAST_B32_e64 ||6613 Opc == AMDGPU::V_PERMLANE_UP_B32_e64 ||6614 Opc == AMDGPU::V_PERMLANE_DOWN_B32_e64 ||6615 Opc == AMDGPU::V_PERMLANE_XOR_B32_e64 ||6616 Opc == AMDGPU::V_PERMLANE_IDX_GEN_B32_e64) {6617 // src1 and src2 must be scalar6618 MachineOperand &Src1 = MI.getOperand(VOP3Idx[1]);6619 const DebugLoc &DL = MI.getDebugLoc();6620 if (Src1.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src1.getReg()))) {6621 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);6622 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)6623 .add(Src1);6624 Src1.ChangeToRegister(Reg, false);6625 }6626 if (VOP3Idx[2] != -1) {6627 MachineOperand &Src2 = MI.getOperand(VOP3Idx[2]);6628 if (Src2.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src2.getReg()))) {6629 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);6630 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)6631 .add(Src2);6632 Src2.ChangeToRegister(Reg, false);6633 }6634 }6635 }6636 6637 // Find the one SGPR operand we are allowed to use.6638 int ConstantBusLimit = ST.getConstantBusLimit(Opc);6639 int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;6640 SmallDenseSet<unsigned> SGPRsUsed;6641 Register SGPRReg = findUsedSGPR(MI, VOP3Idx);6642 if (SGPRReg) {6643 SGPRsUsed.insert(SGPRReg);6644 --ConstantBusLimit;6645 }6646 6647 for (int Idx : VOP3Idx) {6648 if (Idx == -1)6649 break;6650 MachineOperand &MO = MI.getOperand(Idx);6651 6652 if (!MO.isReg()) {6653 if (isInlineConstant(MO, get(Opc).operands()[Idx]))6654 continue;6655 6656 if (LiteralLimit > 0 && ConstantBusLimit > 0) {6657 --LiteralLimit;6658 --ConstantBusLimit;6659 continue;6660 }6661 6662 --LiteralLimit;6663 --ConstantBusLimit;6664 legalizeOpWithMove(MI, Idx);6665 continue;6666 }6667 6668 if (!RI.isSGPRClass(RI.getRegClassForReg(MRI, MO.getReg())))6669 continue; // VGPRs are legal6670 6671 // We can use one SGPR in each VOP3 instruction prior to GFX106672 // and two starting from GFX10.6673 if (SGPRsUsed.count(MO.getReg()))6674 continue;6675 if (ConstantBusLimit > 0) {6676 SGPRsUsed.insert(MO.getReg());6677 --ConstantBusLimit;6678 continue;6679 }6680 6681 // If we make it this far, then the operand is not legal and we must6682 // legalize it.6683 legalizeOpWithMove(MI, Idx);6684 }6685 6686 // Special case: V_FMAC_F32 and V_FMAC_F16 have src2 tied to vdst.6687 if ((Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_e64) &&6688 !RI.isVGPR(MRI, MI.getOperand(VOP3Idx[2]).getReg()))6689 legalizeOpWithMove(MI, VOP3Idx[2]);6690 6691 // Fix the register class of packed FP32 instructions on gfx12+. See6692 // SIInstrInfo::isLegalGFX12PlusPackedMathFP32Operand for more information.6693 if (AMDGPU::isPackedFP32Inst(Opc) && AMDGPU::isGFX12Plus(ST)) {6694 for (unsigned I = 0; I < 3; ++I) {6695 if (!isLegalGFX12PlusPackedMathFP32Operand(MRI, MI, /*SrcN=*/I))6696 legalizeOpWithMove(MI, VOP3Idx[I]);6697 }6698 }6699}6700 6701Register SIInstrInfo::readlaneVGPRToSGPR(6702 Register SrcReg, MachineInstr &UseMI, MachineRegisterInfo &MRI,6703 const TargetRegisterClass *DstRC /*=nullptr*/) const {6704 const TargetRegisterClass *VRC = MRI.getRegClass(SrcReg);6705 const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);6706 if (DstRC)6707 SRC = RI.getCommonSubClass(SRC, DstRC);6708 6709 Register DstReg = MRI.createVirtualRegister(SRC);6710 unsigned SubRegs = RI.getRegSizeInBits(*VRC) / 32;6711 6712 if (RI.hasAGPRs(VRC)) {6713 VRC = RI.getEquivalentVGPRClass(VRC);6714 Register NewSrcReg = MRI.createVirtualRegister(VRC);6715 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),6716 get(TargetOpcode::COPY), NewSrcReg)6717 .addReg(SrcReg);6718 SrcReg = NewSrcReg;6719 }6720 6721 if (SubRegs == 1) {6722 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),6723 get(AMDGPU::V_READFIRSTLANE_B32), DstReg)6724 .addReg(SrcReg);6725 return DstReg;6726 }6727 6728 SmallVector<Register, 8> SRegs;6729 for (unsigned i = 0; i < SubRegs; ++i) {6730 Register SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);6731 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),6732 get(AMDGPU::V_READFIRSTLANE_B32), SGPR)6733 .addReg(SrcReg, 0, RI.getSubRegFromChannel(i));6734 SRegs.push_back(SGPR);6735 }6736 6737 MachineInstrBuilder MIB =6738 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),6739 get(AMDGPU::REG_SEQUENCE), DstReg);6740 for (unsigned i = 0; i < SubRegs; ++i) {6741 MIB.addReg(SRegs[i]);6742 MIB.addImm(RI.getSubRegFromChannel(i));6743 }6744 return DstReg;6745}6746 6747void SIInstrInfo::legalizeOperandsSMRD(MachineRegisterInfo &MRI,6748 MachineInstr &MI) const {6749 6750 // If the pointer is store in VGPRs, then we need to move them to6751 // SGPRs using v_readfirstlane. This is safe because we only select6752 // loads with uniform pointers to SMRD instruction so we know the6753 // pointer value is uniform.6754 MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase);6755 if (SBase && !RI.isSGPRClass(MRI.getRegClass(SBase->getReg()))) {6756 Register SGPR = readlaneVGPRToSGPR(SBase->getReg(), MI, MRI);6757 SBase->setReg(SGPR);6758 }6759 MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soffset);6760 if (SOff && !RI.isSGPRReg(MRI, SOff->getReg())) {6761 Register SGPR = readlaneVGPRToSGPR(SOff->getReg(), MI, MRI);6762 SOff->setReg(SGPR);6763 }6764}6765 6766bool SIInstrInfo::moveFlatAddrToVGPR(MachineInstr &Inst) const {6767 unsigned Opc = Inst.getOpcode();6768 int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr);6769 if (OldSAddrIdx < 0)6770 return false;6771 6772 assert(isSegmentSpecificFLAT(Inst) || (isFLAT(Inst) && ST.hasFlatGVSMode()));6773 6774 int NewOpc = AMDGPU::getGlobalVaddrOp(Opc);6775 if (NewOpc < 0)6776 NewOpc = AMDGPU::getFlatScratchInstSVfromSS(Opc);6777 if (NewOpc < 0)6778 return false;6779 6780 MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo();6781 MachineOperand &SAddr = Inst.getOperand(OldSAddrIdx);6782 if (RI.isSGPRReg(MRI, SAddr.getReg()))6783 return false;6784 6785 int NewVAddrIdx = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr);6786 if (NewVAddrIdx < 0)6787 return false;6788 6789 int OldVAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr);6790 6791 // Check vaddr, it shall be zero or absent.6792 MachineInstr *VAddrDef = nullptr;6793 if (OldVAddrIdx >= 0) {6794 MachineOperand &VAddr = Inst.getOperand(OldVAddrIdx);6795 VAddrDef = MRI.getUniqueVRegDef(VAddr.getReg());6796 if (!VAddrDef || !VAddrDef->isMoveImmediate() ||6797 !VAddrDef->getOperand(1).isImm() ||6798 VAddrDef->getOperand(1).getImm() != 0)6799 return false;6800 }6801 6802 const MCInstrDesc &NewDesc = get(NewOpc);6803 Inst.setDesc(NewDesc);6804 6805 // Callers expect iterator to be valid after this call, so modify the6806 // instruction in place.6807 if (OldVAddrIdx == NewVAddrIdx) {6808 MachineOperand &NewVAddr = Inst.getOperand(NewVAddrIdx);6809 // Clear use list from the old vaddr holding a zero register.6810 MRI.removeRegOperandFromUseList(&NewVAddr);6811 MRI.moveOperands(&NewVAddr, &SAddr, 1);6812 Inst.removeOperand(OldSAddrIdx);6813 // Update the use list with the pointer we have just moved from vaddr to6814 // saddr position. Otherwise new vaddr will be missing from the use list.6815 MRI.removeRegOperandFromUseList(&NewVAddr);6816 MRI.addRegOperandToUseList(&NewVAddr);6817 } else {6818 assert(OldSAddrIdx == NewVAddrIdx);6819 6820 if (OldVAddrIdx >= 0) {6821 int NewVDstIn = AMDGPU::getNamedOperandIdx(NewOpc,6822 AMDGPU::OpName::vdst_in);6823 6824 // removeOperand doesn't try to fixup tied operand indexes at it goes, so6825 // it asserts. Untie the operands for now and retie them afterwards.6826 if (NewVDstIn != -1) {6827 int OldVDstIn = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst_in);6828 Inst.untieRegOperand(OldVDstIn);6829 }6830 6831 Inst.removeOperand(OldVAddrIdx);6832 6833 if (NewVDstIn != -1) {6834 int NewVDst = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vdst);6835 Inst.tieOperands(NewVDst, NewVDstIn);6836 }6837 }6838 }6839 6840 if (VAddrDef && MRI.use_nodbg_empty(VAddrDef->getOperand(0).getReg()))6841 VAddrDef->eraseFromParent();6842 6843 return true;6844}6845 6846// FIXME: Remove this when SelectionDAG is obsoleted.6847void SIInstrInfo::legalizeOperandsFLAT(MachineRegisterInfo &MRI,6848 MachineInstr &MI) const {6849 if (!isSegmentSpecificFLAT(MI) && !ST.hasFlatGVSMode())6850 return;6851 6852 // Fixup SGPR operands in VGPRs. We only select these when the DAG divergence6853 // thinks they are uniform, so a readfirstlane should be valid.6854 MachineOperand *SAddr = getNamedOperand(MI, AMDGPU::OpName::saddr);6855 if (!SAddr || RI.isSGPRClass(MRI.getRegClass(SAddr->getReg())))6856 return;6857 6858 if (moveFlatAddrToVGPR(MI))6859 return;6860 6861 const TargetRegisterClass *DeclaredRC =6862 getRegClass(MI.getDesc(), SAddr->getOperandNo());6863 6864 Register ToSGPR = readlaneVGPRToSGPR(SAddr->getReg(), MI, MRI, DeclaredRC);6865 SAddr->setReg(ToSGPR);6866}6867 6868void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB,6869 MachineBasicBlock::iterator I,6870 const TargetRegisterClass *DstRC,6871 MachineOperand &Op,6872 MachineRegisterInfo &MRI,6873 const DebugLoc &DL) const {6874 Register OpReg = Op.getReg();6875 unsigned OpSubReg = Op.getSubReg();6876 6877 const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg(6878 RI.getRegClassForReg(MRI, OpReg), OpSubReg);6879 6880 // Check if operand is already the correct register class.6881 if (DstRC == OpRC)6882 return;6883 6884 Register DstReg = MRI.createVirtualRegister(DstRC);6885 auto Copy =6886 BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).addReg(OpReg);6887 Op.setReg(DstReg);6888 6889 MachineInstr *Def = MRI.getVRegDef(OpReg);6890 if (!Def)6891 return;6892 6893 // Try to eliminate the copy if it is copying an immediate value.6894 if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass)6895 foldImmediate(*Copy, *Def, OpReg, &MRI);6896 6897 bool ImpDef = Def->isImplicitDef();6898 while (!ImpDef && Def && Def->isCopy()) {6899 if (Def->getOperand(1).getReg().isPhysical())6900 break;6901 Def = MRI.getUniqueVRegDef(Def->getOperand(1).getReg());6902 ImpDef = Def && Def->isImplicitDef();6903 }6904 if (!RI.isSGPRClass(DstRC) && !Copy->readsRegister(AMDGPU::EXEC, &RI) &&6905 !ImpDef)6906 Copy.addReg(AMDGPU::EXEC, RegState::Implicit);6907}6908 6909// Emit the actual waterfall loop, executing the wrapped instruction for each6910// unique value of \p ScalarOps across all lanes. In the best case we execute 16911// iteration, in the worst case we execute 64 (once per lane).6912static void6913emitLoadScalarOpsFromVGPRLoop(const SIInstrInfo &TII,6914 MachineRegisterInfo &MRI,6915 MachineBasicBlock &LoopBB,6916 MachineBasicBlock &BodyBB,6917 const DebugLoc &DL,6918 ArrayRef<MachineOperand *> ScalarOps) {6919 MachineFunction &MF = *LoopBB.getParent();6920 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();6921 const SIRegisterInfo *TRI = ST.getRegisterInfo();6922 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);6923 const auto *BoolXExecRC = TRI->getWaveMaskRegClass();6924 6925 MachineBasicBlock::iterator I = LoopBB.begin();6926 Register CondReg;6927 6928 for (MachineOperand *ScalarOp : ScalarOps) {6929 unsigned RegSize = TRI->getRegSizeInBits(ScalarOp->getReg(), MRI);6930 unsigned NumSubRegs = RegSize / 32;6931 Register VScalarOp = ScalarOp->getReg();6932 6933 if (NumSubRegs == 1) {6934 Register CurReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);6935 6936 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurReg)6937 .addReg(VScalarOp);6938 6939 Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);6940 6941 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U32_e64), NewCondReg)6942 .addReg(CurReg)6943 .addReg(VScalarOp);6944 6945 // Combine the comparison results with AND.6946 if (!CondReg) // First.6947 CondReg = NewCondReg;6948 else { // If not the first, we create an AND.6949 Register AndReg = MRI.createVirtualRegister(BoolXExecRC);6950 BuildMI(LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)6951 .addReg(CondReg)6952 .addReg(NewCondReg);6953 CondReg = AndReg;6954 }6955 6956 // Update ScalarOp operand to use the SGPR ScalarOp.6957 ScalarOp->setReg(CurReg);6958 ScalarOp->setIsKill();6959 } else {6960 SmallVector<Register, 8> ReadlanePieces;6961 unsigned VScalarOpUndef = getUndefRegState(ScalarOp->isUndef());6962 assert(NumSubRegs % 2 == 0 && NumSubRegs <= 32 &&6963 "Unhandled register size");6964 6965 for (unsigned Idx = 0; Idx < NumSubRegs; Idx += 2) {6966 Register CurRegLo =6967 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);6968 Register CurRegHi =6969 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);6970 6971 // Read the next variant <- also loop target.6972 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegLo)6973 .addReg(VScalarOp, VScalarOpUndef, TRI->getSubRegFromChannel(Idx));6974 6975 // Read the next variant <- also loop target.6976 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegHi)6977 .addReg(VScalarOp, VScalarOpUndef,6978 TRI->getSubRegFromChannel(Idx + 1));6979 6980 ReadlanePieces.push_back(CurRegLo);6981 ReadlanePieces.push_back(CurRegHi);6982 6983 // Comparison is to be done as 64-bit.6984 Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass);6985 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), CurReg)6986 .addReg(CurRegLo)6987 .addImm(AMDGPU::sub0)6988 .addReg(CurRegHi)6989 .addImm(AMDGPU::sub1);6990 6991 Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);6992 auto Cmp = BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64),6993 NewCondReg)6994 .addReg(CurReg);6995 if (NumSubRegs <= 2)6996 Cmp.addReg(VScalarOp);6997 else6998 Cmp.addReg(VScalarOp, VScalarOpUndef,6999 TRI->getSubRegFromChannel(Idx, 2));7000 7001 // Combine the comparison results with AND.7002 if (!CondReg) // First.7003 CondReg = NewCondReg;7004 else { // If not the first, we create an AND.7005 Register AndReg = MRI.createVirtualRegister(BoolXExecRC);7006 BuildMI(LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)7007 .addReg(CondReg)7008 .addReg(NewCondReg);7009 CondReg = AndReg;7010 }7011 } // End for loop.7012 7013 const auto *SScalarOpRC =7014 TRI->getEquivalentSGPRClass(MRI.getRegClass(VScalarOp));7015 Register SScalarOp = MRI.createVirtualRegister(SScalarOpRC);7016 7017 // Build scalar ScalarOp.7018 auto Merge =7019 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SScalarOp);7020 unsigned Channel = 0;7021 for (Register Piece : ReadlanePieces) {7022 Merge.addReg(Piece).addImm(TRI->getSubRegFromChannel(Channel++));7023 }7024 7025 // Update ScalarOp operand to use the SGPR ScalarOp.7026 ScalarOp->setReg(SScalarOp);7027 ScalarOp->setIsKill();7028 }7029 }7030 7031 Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);7032 MRI.setSimpleHint(SaveExec, CondReg);7033 7034 // Update EXEC to matching lanes, saving original to SaveExec.7035 BuildMI(LoopBB, I, DL, TII.get(LMC.AndSaveExecOpc), SaveExec)7036 .addReg(CondReg, RegState::Kill);7037 7038 // The original instruction is here; we insert the terminators after it.7039 I = BodyBB.end();7040 7041 // Update EXEC, switch all done bits to 0 and all todo bits to 1.7042 BuildMI(BodyBB, I, DL, TII.get(LMC.XorTermOpc), LMC.ExecReg)7043 .addReg(LMC.ExecReg)7044 .addReg(SaveExec);7045 7046 BuildMI(BodyBB, I, DL, TII.get(AMDGPU::SI_WATERFALL_LOOP)).addMBB(&LoopBB);7047}7048 7049// Build a waterfall loop around \p MI, replacing the VGPR \p ScalarOp register7050// with SGPRs by iterating over all unique values across all lanes.7051// Returns the loop basic block that now contains \p MI.7052static MachineBasicBlock *7053loadMBUFScalarOperandsFromVGPR(const SIInstrInfo &TII, MachineInstr &MI,7054 ArrayRef<MachineOperand *> ScalarOps,7055 MachineDominatorTree *MDT,7056 MachineBasicBlock::iterator Begin = nullptr,7057 MachineBasicBlock::iterator End = nullptr) {7058 MachineBasicBlock &MBB = *MI.getParent();7059 MachineFunction &MF = *MBB.getParent();7060 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();7061 const SIRegisterInfo *TRI = ST.getRegisterInfo();7062 MachineRegisterInfo &MRI = MF.getRegInfo();7063 if (!Begin.isValid())7064 Begin = &MI;7065 if (!End.isValid()) {7066 End = &MI;7067 ++End;7068 }7069 const DebugLoc &DL = MI.getDebugLoc();7070 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);7071 const auto *BoolXExecRC = TRI->getWaveMaskRegClass();7072 7073 // Save SCC. Waterfall Loop may overwrite SCC.7074 Register SaveSCCReg;7075 7076 // FIXME: We should maintain SCC liveness while doing the FixSGPRCopies walk7077 // rather than unlimited scan everywhere7078 bool SCCNotDead =7079 MBB.computeRegisterLiveness(TRI, AMDGPU::SCC, MI,7080 std::numeric_limits<unsigned>::max()) !=7081 MachineBasicBlock::LQR_Dead;7082 if (SCCNotDead) {7083 SaveSCCReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);7084 BuildMI(MBB, Begin, DL, TII.get(AMDGPU::S_CSELECT_B32), SaveSCCReg)7085 .addImm(1)7086 .addImm(0);7087 }7088 7089 Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);7090 7091 // Save the EXEC mask7092 BuildMI(MBB, Begin, DL, TII.get(LMC.MovOpc), SaveExec).addReg(LMC.ExecReg);7093 7094 // Killed uses in the instruction we are waterfalling around will be7095 // incorrect due to the added control-flow.7096 MachineBasicBlock::iterator AfterMI = MI;7097 ++AfterMI;7098 for (auto I = Begin; I != AfterMI; I++) {7099 for (auto &MO : I->all_uses())7100 MRI.clearKillFlags(MO.getReg());7101 }7102 7103 // To insert the loop we need to split the block. Move everything after this7104 // point to a new block, and insert a new empty block between the two.7105 MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock();7106 MachineBasicBlock *BodyBB = MF.CreateMachineBasicBlock();7107 MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();7108 MachineFunction::iterator MBBI(MBB);7109 ++MBBI;7110 7111 MF.insert(MBBI, LoopBB);7112 MF.insert(MBBI, BodyBB);7113 MF.insert(MBBI, RemainderBB);7114 7115 LoopBB->addSuccessor(BodyBB);7116 BodyBB->addSuccessor(LoopBB);7117 BodyBB->addSuccessor(RemainderBB);7118 7119 // Move Begin to MI to the BodyBB, and the remainder of the block to7120 // RemainderBB.7121 RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);7122 RemainderBB->splice(RemainderBB->begin(), &MBB, End, MBB.end());7123 BodyBB->splice(BodyBB->begin(), &MBB, Begin, MBB.end());7124 7125 MBB.addSuccessor(LoopBB);7126 7127 // Update dominators. We know that MBB immediately dominates LoopBB, that7128 // LoopBB immediately dominates BodyBB, and BodyBB immediately dominates7129 // RemainderBB. RemainderBB immediately dominates all of the successors7130 // transferred to it from MBB that MBB used to properly dominate.7131 if (MDT) {7132 MDT->addNewBlock(LoopBB, &MBB);7133 MDT->addNewBlock(BodyBB, LoopBB);7134 MDT->addNewBlock(RemainderBB, BodyBB);7135 for (auto &Succ : RemainderBB->successors()) {7136 if (MDT->properlyDominates(&MBB, Succ)) {7137 MDT->changeImmediateDominator(Succ, RemainderBB);7138 }7139 }7140 }7141 7142 emitLoadScalarOpsFromVGPRLoop(TII, MRI, *LoopBB, *BodyBB, DL, ScalarOps);7143 7144 MachineBasicBlock::iterator First = RemainderBB->begin();7145 // Restore SCC7146 if (SCCNotDead) {7147 BuildMI(*RemainderBB, First, DL, TII.get(AMDGPU::S_CMP_LG_U32))7148 .addReg(SaveSCCReg, RegState::Kill)7149 .addImm(0);7150 }7151 7152 // Restore the EXEC mask7153 BuildMI(*RemainderBB, First, DL, TII.get(LMC.MovOpc), LMC.ExecReg)7154 .addReg(SaveExec);7155 return BodyBB;7156}7157 7158// Extract pointer from Rsrc and return a zero-value Rsrc replacement.7159static std::tuple<unsigned, unsigned>7160extractRsrcPtr(const SIInstrInfo &TII, MachineInstr &MI, MachineOperand &Rsrc) {7161 MachineBasicBlock &MBB = *MI.getParent();7162 MachineFunction &MF = *MBB.getParent();7163 MachineRegisterInfo &MRI = MF.getRegInfo();7164 7165 // Extract the ptr from the resource descriptor.7166 unsigned RsrcPtr =7167 TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass,7168 AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass);7169 7170 // Create an empty resource descriptor7171 Register Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);7172 Register SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);7173 Register SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);7174 Register NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass);7175 uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat();7176 7177 // Zero64 = 07178 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64)7179 .addImm(0);7180 7181 // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}7182 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo)7183 .addImm(Lo_32(RsrcDataFormat));7184 7185 // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}7186 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi)7187 .addImm(Hi_32(RsrcDataFormat));7188 7189 // NewSRsrc = {Zero64, SRsrcFormat}7190 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc)7191 .addReg(Zero64)7192 .addImm(AMDGPU::sub0_sub1)7193 .addReg(SRsrcFormatLo)7194 .addImm(AMDGPU::sub2)7195 .addReg(SRsrcFormatHi)7196 .addImm(AMDGPU::sub3);7197 7198 return std::tuple(RsrcPtr, NewSRsrc);7199}7200 7201MachineBasicBlock *7202SIInstrInfo::legalizeOperands(MachineInstr &MI,7203 MachineDominatorTree *MDT) const {7204 MachineFunction &MF = *MI.getMF();7205 MachineRegisterInfo &MRI = MF.getRegInfo();7206 MachineBasicBlock *CreatedBB = nullptr;7207 7208 // Legalize VOP27209 if (isVOP2(MI) || isVOPC(MI)) {7210 legalizeOperandsVOP2(MRI, MI);7211 return CreatedBB;7212 }7213 7214 // Legalize VOP37215 if (isVOP3(MI)) {7216 legalizeOperandsVOP3(MRI, MI);7217 return CreatedBB;7218 }7219 7220 // Legalize SMRD7221 if (isSMRD(MI)) {7222 legalizeOperandsSMRD(MRI, MI);7223 return CreatedBB;7224 }7225 7226 // Legalize FLAT7227 if (isFLAT(MI)) {7228 legalizeOperandsFLAT(MRI, MI);7229 return CreatedBB;7230 }7231 7232 // Legalize REG_SEQUENCE and PHI7233 // The register class of the operands much be the same type as the register7234 // class of the output.7235 if (MI.getOpcode() == AMDGPU::PHI) {7236 const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr;7237 for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {7238 if (!MI.getOperand(i).isReg() || !MI.getOperand(i).getReg().isVirtual())7239 continue;7240 const TargetRegisterClass *OpRC =7241 MRI.getRegClass(MI.getOperand(i).getReg());7242 if (RI.hasVectorRegisters(OpRC)) {7243 VRC = OpRC;7244 } else {7245 SRC = OpRC;7246 }7247 }7248 7249 // If any of the operands are VGPR registers, then they all most be7250 // otherwise we will create illegal VGPR->SGPR copies when legalizing7251 // them.7252 if (VRC || !RI.isSGPRClass(getOpRegClass(MI, 0))) {7253 if (!VRC) {7254 assert(SRC);7255 if (getOpRegClass(MI, 0) == &AMDGPU::VReg_1RegClass) {7256 VRC = &AMDGPU::VReg_1RegClass;7257 } else7258 VRC = RI.isAGPRClass(getOpRegClass(MI, 0))7259 ? RI.getEquivalentAGPRClass(SRC)7260 : RI.getEquivalentVGPRClass(SRC);7261 } else {7262 VRC = RI.isAGPRClass(getOpRegClass(MI, 0))7263 ? RI.getEquivalentAGPRClass(VRC)7264 : RI.getEquivalentVGPRClass(VRC);7265 }7266 RC = VRC;7267 } else {7268 RC = SRC;7269 }7270 7271 // Update all the operands so they have the same type.7272 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {7273 MachineOperand &Op = MI.getOperand(I);7274 if (!Op.isReg() || !Op.getReg().isVirtual())7275 continue;7276 7277 // MI is a PHI instruction.7278 MachineBasicBlock *InsertBB = MI.getOperand(I + 1).getMBB();7279 MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator();7280 7281 // Avoid creating no-op copies with the same src and dst reg class. These7282 // confuse some of the machine passes.7283 legalizeGenericOperand(*InsertBB, Insert, RC, Op, MRI, MI.getDebugLoc());7284 }7285 }7286 7287 // REG_SEQUENCE doesn't really require operand legalization, but if one has a7288 // VGPR dest type and SGPR sources, insert copies so all operands are7289 // VGPRs. This seems to help operand folding / the register coalescer.7290 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {7291 MachineBasicBlock *MBB = MI.getParent();7292 const TargetRegisterClass *DstRC = getOpRegClass(MI, 0);7293 if (RI.hasVGPRs(DstRC)) {7294 // Update all the operands so they are VGPR register classes. These may7295 // not be the same register class because REG_SEQUENCE supports mixing7296 // subregister index types e.g. sub0_sub1 + sub2 + sub37297 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {7298 MachineOperand &Op = MI.getOperand(I);7299 if (!Op.isReg() || !Op.getReg().isVirtual())7300 continue;7301 7302 const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg());7303 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC);7304 if (VRC == OpRC)7305 continue;7306 7307 legalizeGenericOperand(*MBB, MI, VRC, Op, MRI, MI.getDebugLoc());7308 Op.setIsKill();7309 }7310 }7311 7312 return CreatedBB;7313 }7314 7315 // Legalize INSERT_SUBREG7316 // src0 must have the same register class as dst7317 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {7318 Register Dst = MI.getOperand(0).getReg();7319 Register Src0 = MI.getOperand(1).getReg();7320 const TargetRegisterClass *DstRC = MRI.getRegClass(Dst);7321 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0);7322 if (DstRC != Src0RC) {7323 MachineBasicBlock *MBB = MI.getParent();7324 MachineOperand &Op = MI.getOperand(1);7325 legalizeGenericOperand(*MBB, MI, DstRC, Op, MRI, MI.getDebugLoc());7326 }7327 return CreatedBB;7328 }7329 7330 // Legalize SI_INIT_M07331 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {7332 MachineOperand &Src = MI.getOperand(0);7333 if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg())))7334 Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));7335 return CreatedBB;7336 }7337 7338 // Legalize S_BITREPLICATE, S_QUADMASK and S_WQM7339 if (MI.getOpcode() == AMDGPU::S_BITREPLICATE_B64_B32 ||7340 MI.getOpcode() == AMDGPU::S_QUADMASK_B32 ||7341 MI.getOpcode() == AMDGPU::S_QUADMASK_B64 ||7342 MI.getOpcode() == AMDGPU::S_WQM_B32 ||7343 MI.getOpcode() == AMDGPU::S_WQM_B64 ||7344 MI.getOpcode() == AMDGPU::S_INVERSE_BALLOT_U32 ||7345 MI.getOpcode() == AMDGPU::S_INVERSE_BALLOT_U64) {7346 MachineOperand &Src = MI.getOperand(1);7347 if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg())))7348 Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));7349 return CreatedBB;7350 }7351 7352 // Legalize MIMG/VIMAGE/VSAMPLE and MUBUF/MTBUF for shaders.7353 //7354 // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via7355 // scratch memory access. In both cases, the legalization never involves7356 // conversion to the addr64 form.7357 if (isImage(MI) || (AMDGPU::isGraphics(MF.getFunction().getCallingConv()) &&7358 (isMUBUF(MI) || isMTBUF(MI)))) {7359 AMDGPU::OpName RSrcOpName = (isVIMAGE(MI) || isVSAMPLE(MI))7360 ? AMDGPU::OpName::rsrc7361 : AMDGPU::OpName::srsrc;7362 MachineOperand *SRsrc = getNamedOperand(MI, RSrcOpName);7363 if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(SRsrc->getReg())))7364 CreatedBB = loadMBUFScalarOperandsFromVGPR(*this, MI, {SRsrc}, MDT);7365 7366 AMDGPU::OpName SampOpName =7367 isMIMG(MI) ? AMDGPU::OpName::ssamp : AMDGPU::OpName::samp;7368 MachineOperand *SSamp = getNamedOperand(MI, SampOpName);7369 if (SSamp && !RI.isSGPRClass(MRI.getRegClass(SSamp->getReg())))7370 CreatedBB = loadMBUFScalarOperandsFromVGPR(*this, MI, {SSamp}, MDT);7371 7372 return CreatedBB;7373 }7374 7375 // Legalize SI_CALL7376 if (MI.getOpcode() == AMDGPU::SI_CALL_ISEL) {7377 MachineOperand *Dest = &MI.getOperand(0);7378 if (!RI.isSGPRClass(MRI.getRegClass(Dest->getReg()))) {7379 // Move everything between ADJCALLSTACKUP and ADJCALLSTACKDOWN and7380 // following copies, we also need to move copies from and to physical7381 // registers into the loop block.7382 unsigned FrameSetupOpcode = getCallFrameSetupOpcode();7383 unsigned FrameDestroyOpcode = getCallFrameDestroyOpcode();7384 7385 // Also move the copies to physical registers into the loop block7386 MachineBasicBlock &MBB = *MI.getParent();7387 MachineBasicBlock::iterator Start(&MI);7388 while (Start->getOpcode() != FrameSetupOpcode)7389 --Start;7390 MachineBasicBlock::iterator End(&MI);7391 while (End->getOpcode() != FrameDestroyOpcode)7392 ++End;7393 // Also include following copies of the return value7394 ++End;7395 while (End != MBB.end() && End->isCopy() && End->getOperand(1).isReg() &&7396 MI.definesRegister(End->getOperand(1).getReg(), /*TRI=*/nullptr))7397 ++End;7398 CreatedBB =7399 loadMBUFScalarOperandsFromVGPR(*this, MI, {Dest}, MDT, Start, End);7400 }7401 }7402 7403 // Legalize s_sleep_var.7404 if (MI.getOpcode() == AMDGPU::S_SLEEP_VAR) {7405 const DebugLoc &DL = MI.getDebugLoc();7406 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);7407 int Src0Idx =7408 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0);7409 MachineOperand &Src0 = MI.getOperand(Src0Idx);7410 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)7411 .add(Src0);7412 Src0.ChangeToRegister(Reg, false);7413 return nullptr;7414 }7415 7416 // Legalize TENSOR_LOAD_TO_LDS, TENSOR_LOAD_TO_LDS_D2, TENSOR_STORE_FROM_LDS,7417 // TENSOR_STORE_FROM_LDS_D2. All their operands are scalar.7418 if (MI.getOpcode() == AMDGPU::TENSOR_LOAD_TO_LDS ||7419 MI.getOpcode() == AMDGPU::TENSOR_LOAD_TO_LDS_D2 ||7420 MI.getOpcode() == AMDGPU::TENSOR_STORE_FROM_LDS ||7421 MI.getOpcode() == AMDGPU::TENSOR_STORE_FROM_LDS_D2) {7422 for (MachineOperand &Src : MI.explicit_operands()) {7423 if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg())))7424 Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));7425 }7426 return CreatedBB;7427 }7428 7429 // Legalize MUBUF instructions.7430 bool isSoffsetLegal = true;7431 int SoffsetIdx =7432 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::soffset);7433 if (SoffsetIdx != -1) {7434 MachineOperand *Soffset = &MI.getOperand(SoffsetIdx);7435 if (Soffset->isReg() && Soffset->getReg().isVirtual() &&7436 !RI.isSGPRClass(MRI.getRegClass(Soffset->getReg()))) {7437 isSoffsetLegal = false;7438 }7439 }7440 7441 bool isRsrcLegal = true;7442 int RsrcIdx =7443 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc);7444 if (RsrcIdx != -1) {7445 MachineOperand *Rsrc = &MI.getOperand(RsrcIdx);7446 if (Rsrc->isReg() && !RI.isSGPRReg(MRI, Rsrc->getReg()))7447 isRsrcLegal = false;7448 }7449 7450 // The operands are legal.7451 if (isRsrcLegal && isSoffsetLegal)7452 return CreatedBB;7453 7454 if (!isRsrcLegal) {7455 // Legalize a VGPR Rsrc7456 //7457 // If the instruction is _ADDR64, we can avoid a waterfall by extracting7458 // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using7459 // a zero-value SRsrc.7460 //7461 // If the instruction is _OFFSET (both idxen and offen disabled), and we7462 // support ADDR64 instructions, we can convert to ADDR64 and do the same as7463 // above.7464 //7465 // Otherwise we are on non-ADDR64 hardware, and/or we have7466 // idxen/offen/bothen and we fall back to a waterfall loop.7467 7468 MachineOperand *Rsrc = &MI.getOperand(RsrcIdx);7469 MachineBasicBlock &MBB = *MI.getParent();7470 7471 MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr);7472 if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) {7473 // This is already an ADDR64 instruction so we need to add the pointer7474 // extracted from the resource descriptor to the current value of VAddr.7475 Register NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);7476 Register NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);7477 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);7478 7479 const auto *BoolXExecRC = RI.getWaveMaskRegClass();7480 Register CondReg0 = MRI.createVirtualRegister(BoolXExecRC);7481 Register CondReg1 = MRI.createVirtualRegister(BoolXExecRC);7482 7483 unsigned RsrcPtr, NewSRsrc;7484 std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);7485 7486 // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub07487 const DebugLoc &DL = MI.getDebugLoc();7488 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_CO_U32_e64), NewVAddrLo)7489 .addDef(CondReg0)7490 .addReg(RsrcPtr, 0, AMDGPU::sub0)7491 .addReg(VAddr->getReg(), 0, AMDGPU::sub0)7492 .addImm(0);7493 7494 // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub17495 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e64), NewVAddrHi)7496 .addDef(CondReg1, RegState::Dead)7497 .addReg(RsrcPtr, 0, AMDGPU::sub1)7498 .addReg(VAddr->getReg(), 0, AMDGPU::sub1)7499 .addReg(CondReg0, RegState::Kill)7500 .addImm(0);7501 7502 // NewVaddr = {NewVaddrHi, NewVaddrLo}7503 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr)7504 .addReg(NewVAddrLo)7505 .addImm(AMDGPU::sub0)7506 .addReg(NewVAddrHi)7507 .addImm(AMDGPU::sub1);7508 7509 VAddr->setReg(NewVAddr);7510 Rsrc->setReg(NewSRsrc);7511 } else if (!VAddr && ST.hasAddr64()) {7512 // This instructions is the _OFFSET variant, so we need to convert it to7513 // ADDR64.7514 assert(ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS &&7515 "FIXME: Need to emit flat atomics here");7516 7517 unsigned RsrcPtr, NewSRsrc;7518 std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);7519 7520 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);7521 MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata);7522 MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);7523 MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset);7524 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode());7525 7526 // Atomics with return have an additional tied operand and are7527 // missing some of the special bits.7528 MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in);7529 MachineInstr *Addr64;7530 7531 if (!VDataIn) {7532 // Regular buffer load / store.7533 MachineInstrBuilder MIB =7534 BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))7535 .add(*VData)7536 .addReg(NewVAddr)7537 .addReg(NewSRsrc)7538 .add(*SOffset)7539 .add(*Offset);7540 7541 if (const MachineOperand *CPol =7542 getNamedOperand(MI, AMDGPU::OpName::cpol)) {7543 MIB.addImm(CPol->getImm());7544 }7545 7546 if (const MachineOperand *TFE =7547 getNamedOperand(MI, AMDGPU::OpName::tfe)) {7548 MIB.addImm(TFE->getImm());7549 }7550 7551 MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::swz));7552 7553 MIB.cloneMemRefs(MI);7554 Addr64 = MIB;7555 } else {7556 // Atomics with return.7557 Addr64 = BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))7558 .add(*VData)7559 .add(*VDataIn)7560 .addReg(NewVAddr)7561 .addReg(NewSRsrc)7562 .add(*SOffset)7563 .add(*Offset)7564 .addImm(getNamedImmOperand(MI, AMDGPU::OpName::cpol))7565 .cloneMemRefs(MI);7566 }7567 7568 MI.removeFromParent();7569 7570 // NewVaddr = {NewVaddrHi, NewVaddrLo}7571 BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE),7572 NewVAddr)7573 .addReg(RsrcPtr, 0, AMDGPU::sub0)7574 .addImm(AMDGPU::sub0)7575 .addReg(RsrcPtr, 0, AMDGPU::sub1)7576 .addImm(AMDGPU::sub1);7577 } else {7578 // Legalize a VGPR Rsrc and soffset together.7579 if (!isSoffsetLegal) {7580 MachineOperand *Soffset = getNamedOperand(MI, AMDGPU::OpName::soffset);7581 CreatedBB =7582 loadMBUFScalarOperandsFromVGPR(*this, MI, {Rsrc, Soffset}, MDT);7583 return CreatedBB;7584 }7585 CreatedBB = loadMBUFScalarOperandsFromVGPR(*this, MI, {Rsrc}, MDT);7586 return CreatedBB;7587 }7588 }7589 7590 // Legalize a VGPR soffset.7591 if (!isSoffsetLegal) {7592 MachineOperand *Soffset = getNamedOperand(MI, AMDGPU::OpName::soffset);7593 CreatedBB = loadMBUFScalarOperandsFromVGPR(*this, MI, {Soffset}, MDT);7594 return CreatedBB;7595 }7596 return CreatedBB;7597}7598 7599void SIInstrWorklist::insert(MachineInstr *MI) {7600 InstrList.insert(MI);7601 // Add MBUF instructiosn to deferred list.7602 int RsrcIdx =7603 AMDGPU::getNamedOperandIdx(MI->getOpcode(), AMDGPU::OpName::srsrc);7604 if (RsrcIdx != -1) {7605 DeferredList.insert(MI);7606 }7607}7608 7609bool SIInstrWorklist::isDeferred(MachineInstr *MI) {7610 return DeferredList.contains(MI);7611}7612 7613// Legalize size mismatches between 16bit and 32bit registers in v2s copy7614// lowering (change spgr to vgpr).7615// This is mainly caused by 16bit SALU and 16bit VALU using reg with different7616// size. Need to legalize the size of the operands during the vgpr lowering7617// chain. This can be removed after we have sgpr16 in place7618void SIInstrInfo::legalizeOperandsVALUt16(MachineInstr &MI, unsigned OpIdx,7619 MachineRegisterInfo &MRI) const {7620 if (!ST.useRealTrue16Insts())7621 return;7622 7623 unsigned Opcode = MI.getOpcode();7624 MachineBasicBlock *MBB = MI.getParent();7625 // Legalize operands and check for size mismatch7626 if (!OpIdx || OpIdx >= MI.getNumExplicitOperands() ||7627 OpIdx >= get(Opcode).getNumOperands() ||7628 get(Opcode).operands()[OpIdx].RegClass == -1)7629 return;7630 7631 MachineOperand &Op = MI.getOperand(OpIdx);7632 if (!Op.isReg() || !Op.getReg().isVirtual())7633 return;7634 7635 const TargetRegisterClass *CurrRC = MRI.getRegClass(Op.getReg());7636 if (!RI.isVGPRClass(CurrRC))7637 return;7638 7639 int16_t RCID = getOpRegClassID(get(Opcode).operands()[OpIdx]);7640 const TargetRegisterClass *ExpectedRC = RI.getRegClass(RCID);7641 if (RI.getMatchingSuperRegClass(CurrRC, ExpectedRC, AMDGPU::lo16)) {7642 Op.setSubReg(AMDGPU::lo16);7643 } else if (RI.getMatchingSuperRegClass(ExpectedRC, CurrRC, AMDGPU::lo16)) {7644 const DebugLoc &DL = MI.getDebugLoc();7645 Register NewDstReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);7646 Register Undef = MRI.createVirtualRegister(&AMDGPU::VGPR_16RegClass);7647 BuildMI(*MBB, MI, DL, get(AMDGPU::IMPLICIT_DEF), Undef);7648 BuildMI(*MBB, MI, DL, get(AMDGPU::REG_SEQUENCE), NewDstReg)7649 .addReg(Op.getReg())7650 .addImm(AMDGPU::lo16)7651 .addReg(Undef)7652 .addImm(AMDGPU::hi16);7653 Op.setReg(NewDstReg);7654 }7655}7656void SIInstrInfo::legalizeOperandsVALUt16(MachineInstr &MI,7657 MachineRegisterInfo &MRI) const {7658 for (unsigned OpIdx = 1; OpIdx < MI.getNumExplicitOperands(); OpIdx++)7659 legalizeOperandsVALUt16(MI, OpIdx, MRI);7660}7661 7662void SIInstrInfo::moveToVALU(SIInstrWorklist &Worklist,7663 MachineDominatorTree *MDT) const {7664 7665 while (!Worklist.empty()) {7666 MachineInstr &Inst = *Worklist.top();7667 Worklist.erase_top();7668 // Skip MachineInstr in the deferred list.7669 if (Worklist.isDeferred(&Inst))7670 continue;7671 moveToVALUImpl(Worklist, MDT, Inst);7672 }7673 7674 // Deferred list of instructions will be processed once7675 // all the MachineInstr in the worklist are done.7676 for (MachineInstr *Inst : Worklist.getDeferredList()) {7677 moveToVALUImpl(Worklist, MDT, *Inst);7678 assert(Worklist.empty() &&7679 "Deferred MachineInstr are not supposed to re-populate worklist");7680 }7681}7682 7683void SIInstrInfo::moveToVALUImpl(SIInstrWorklist &Worklist,7684 MachineDominatorTree *MDT,7685 MachineInstr &Inst) const {7686 7687 MachineBasicBlock *MBB = Inst.getParent();7688 if (!MBB)7689 return;7690 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();7691 unsigned Opcode = Inst.getOpcode();7692 unsigned NewOpcode = getVALUOp(Inst);7693 const DebugLoc &DL = Inst.getDebugLoc();7694 7695 // Handle some special cases7696 switch (Opcode) {7697 default:7698 break;7699 case AMDGPU::S_ADD_I32:7700 case AMDGPU::S_SUB_I32: {7701 // FIXME: The u32 versions currently selected use the carry.7702 bool Changed;7703 MachineBasicBlock *CreatedBBTmp = nullptr;7704 std::tie(Changed, CreatedBBTmp) = moveScalarAddSub(Worklist, Inst, MDT);7705 if (Changed)7706 return;7707 7708 // Default handling7709 break;7710 }7711 7712 case AMDGPU::S_MUL_U64:7713 if (ST.hasVectorMulU64()) {7714 NewOpcode = AMDGPU::V_MUL_U64_e64;7715 break;7716 }7717 // Split s_mul_u64 in 32-bit vector multiplications.7718 splitScalarSMulU64(Worklist, Inst, MDT);7719 Inst.eraseFromParent();7720 return;7721 7722 case AMDGPU::S_MUL_U64_U32_PSEUDO:7723 case AMDGPU::S_MUL_I64_I32_PSEUDO:7724 // This is a special case of s_mul_u64 where all the operands are either7725 // zero extended or sign extended.7726 splitScalarSMulPseudo(Worklist, Inst, MDT);7727 Inst.eraseFromParent();7728 return;7729 7730 case AMDGPU::S_AND_B64:7731 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT);7732 Inst.eraseFromParent();7733 return;7734 7735 case AMDGPU::S_OR_B64:7736 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT);7737 Inst.eraseFromParent();7738 return;7739 7740 case AMDGPU::S_XOR_B64:7741 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT);7742 Inst.eraseFromParent();7743 return;7744 7745 case AMDGPU::S_NAND_B64:7746 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT);7747 Inst.eraseFromParent();7748 return;7749 7750 case AMDGPU::S_NOR_B64:7751 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT);7752 Inst.eraseFromParent();7753 return;7754 7755 case AMDGPU::S_XNOR_B64:7756 if (ST.hasDLInsts())7757 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT);7758 else7759 splitScalar64BitXnor(Worklist, Inst, MDT);7760 Inst.eraseFromParent();7761 return;7762 7763 case AMDGPU::S_ANDN2_B64:7764 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT);7765 Inst.eraseFromParent();7766 return;7767 7768 case AMDGPU::S_ORN2_B64:7769 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT);7770 Inst.eraseFromParent();7771 return;7772 7773 case AMDGPU::S_BREV_B64:7774 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_BREV_B32, true);7775 Inst.eraseFromParent();7776 return;7777 7778 case AMDGPU::S_NOT_B64:7779 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32);7780 Inst.eraseFromParent();7781 return;7782 7783 case AMDGPU::S_BCNT1_I32_B64:7784 splitScalar64BitBCNT(Worklist, Inst);7785 Inst.eraseFromParent();7786 return;7787 7788 case AMDGPU::S_BFE_I64:7789 splitScalar64BitBFE(Worklist, Inst);7790 Inst.eraseFromParent();7791 return;7792 7793 case AMDGPU::S_FLBIT_I32_B64:7794 splitScalar64BitCountOp(Worklist, Inst, AMDGPU::V_FFBH_U32_e32);7795 Inst.eraseFromParent();7796 return;7797 case AMDGPU::S_FF1_I32_B64:7798 splitScalar64BitCountOp(Worklist, Inst, AMDGPU::V_FFBL_B32_e32);7799 Inst.eraseFromParent();7800 return;7801 7802 case AMDGPU::S_LSHL_B32:7803 if (ST.hasOnlyRevVALUShifts()) {7804 NewOpcode = AMDGPU::V_LSHLREV_B32_e64;7805 swapOperands(Inst);7806 }7807 break;7808 case AMDGPU::S_ASHR_I32:7809 if (ST.hasOnlyRevVALUShifts()) {7810 NewOpcode = AMDGPU::V_ASHRREV_I32_e64;7811 swapOperands(Inst);7812 }7813 break;7814 case AMDGPU::S_LSHR_B32:7815 if (ST.hasOnlyRevVALUShifts()) {7816 NewOpcode = AMDGPU::V_LSHRREV_B32_e64;7817 swapOperands(Inst);7818 }7819 break;7820 case AMDGPU::S_LSHL_B64:7821 if (ST.hasOnlyRevVALUShifts()) {7822 NewOpcode = ST.getGeneration() >= AMDGPUSubtarget::GFX127823 ? AMDGPU::V_LSHLREV_B64_pseudo_e647824 : AMDGPU::V_LSHLREV_B64_e64;7825 swapOperands(Inst);7826 }7827 break;7828 case AMDGPU::S_ASHR_I64:7829 if (ST.hasOnlyRevVALUShifts()) {7830 NewOpcode = AMDGPU::V_ASHRREV_I64_e64;7831 swapOperands(Inst);7832 }7833 break;7834 case AMDGPU::S_LSHR_B64:7835 if (ST.hasOnlyRevVALUShifts()) {7836 NewOpcode = AMDGPU::V_LSHRREV_B64_e64;7837 swapOperands(Inst);7838 }7839 break;7840 7841 case AMDGPU::S_ABS_I32:7842 lowerScalarAbs(Worklist, Inst);7843 Inst.eraseFromParent();7844 return;7845 7846 case AMDGPU::S_ABSDIFF_I32:7847 lowerScalarAbsDiff(Worklist, Inst);7848 Inst.eraseFromParent();7849 return;7850 7851 case AMDGPU::S_CBRANCH_SCC0:7852 case AMDGPU::S_CBRANCH_SCC1: {7853 // Clear unused bits of vcc7854 Register CondReg = Inst.getOperand(1).getReg();7855 bool IsSCC = CondReg == AMDGPU::SCC;7856 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);7857 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(LMC.AndOpc), LMC.VccReg)7858 .addReg(LMC.ExecReg)7859 .addReg(IsSCC ? LMC.VccReg : CondReg);7860 Inst.removeOperand(1);7861 } break;7862 7863 case AMDGPU::S_BFE_U64:7864 case AMDGPU::S_BFM_B64:7865 llvm_unreachable("Moving this op to VALU not implemented");7866 7867 case AMDGPU::S_PACK_LL_B32_B16:7868 case AMDGPU::S_PACK_LH_B32_B16:7869 case AMDGPU::S_PACK_HL_B32_B16:7870 case AMDGPU::S_PACK_HH_B32_B16:7871 movePackToVALU(Worklist, MRI, Inst);7872 Inst.eraseFromParent();7873 return;7874 7875 case AMDGPU::S_XNOR_B32:7876 lowerScalarXnor(Worklist, Inst);7877 Inst.eraseFromParent();7878 return;7879 7880 case AMDGPU::S_NAND_B32:7881 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32);7882 Inst.eraseFromParent();7883 return;7884 7885 case AMDGPU::S_NOR_B32:7886 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32);7887 Inst.eraseFromParent();7888 return;7889 7890 case AMDGPU::S_ANDN2_B32:7891 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32);7892 Inst.eraseFromParent();7893 return;7894 7895 case AMDGPU::S_ORN2_B32:7896 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32);7897 Inst.eraseFromParent();7898 return;7899 7900 // TODO: remove as soon as everything is ready7901 // to replace VGPR to SGPR copy with V_READFIRSTLANEs.7902 // S_ADD/SUB_CO_PSEUDO as well as S_UADDO/USUBO_PSEUDO7903 // can only be selected from the uniform SDNode.7904 case AMDGPU::S_ADD_CO_PSEUDO:7905 case AMDGPU::S_SUB_CO_PSEUDO: {7906 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO)7907 ? AMDGPU::V_ADDC_U32_e647908 : AMDGPU::V_SUBB_U32_e64;7909 const auto *CarryRC = RI.getWaveMaskRegClass();7910 7911 Register CarryInReg = Inst.getOperand(4).getReg();7912 if (!MRI.constrainRegClass(CarryInReg, CarryRC)) {7913 Register NewCarryReg = MRI.createVirtualRegister(CarryRC);7914 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::COPY), NewCarryReg)7915 .addReg(CarryInReg);7916 }7917 7918 Register CarryOutReg = Inst.getOperand(1).getReg();7919 7920 Register DestReg = MRI.createVirtualRegister(RI.getEquivalentVGPRClass(7921 MRI.getRegClass(Inst.getOperand(0).getReg())));7922 MachineInstr *CarryOp =7923 BuildMI(*MBB, &Inst, Inst.getDebugLoc(), get(Opc), DestReg)7924 .addReg(CarryOutReg, RegState::Define)7925 .add(Inst.getOperand(2))7926 .add(Inst.getOperand(3))7927 .addReg(CarryInReg)7928 .addImm(0);7929 legalizeOperands(*CarryOp);7930 MRI.replaceRegWith(Inst.getOperand(0).getReg(), DestReg);7931 addUsersToMoveToVALUWorklist(DestReg, MRI, Worklist);7932 Inst.eraseFromParent();7933 }7934 return;7935 case AMDGPU::S_UADDO_PSEUDO:7936 case AMDGPU::S_USUBO_PSEUDO: {7937 MachineOperand &Dest0 = Inst.getOperand(0);7938 MachineOperand &Dest1 = Inst.getOperand(1);7939 MachineOperand &Src0 = Inst.getOperand(2);7940 MachineOperand &Src1 = Inst.getOperand(3);7941 7942 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_UADDO_PSEUDO)7943 ? AMDGPU::V_ADD_CO_U32_e647944 : AMDGPU::V_SUB_CO_U32_e64;7945 const TargetRegisterClass *NewRC =7946 RI.getEquivalentVGPRClass(MRI.getRegClass(Dest0.getReg()));7947 Register DestReg = MRI.createVirtualRegister(NewRC);7948 MachineInstr *NewInstr = BuildMI(*MBB, &Inst, DL, get(Opc), DestReg)7949 .addReg(Dest1.getReg(), RegState::Define)7950 .add(Src0)7951 .add(Src1)7952 .addImm(0); // clamp bit7953 7954 legalizeOperands(*NewInstr, MDT);7955 MRI.replaceRegWith(Dest0.getReg(), DestReg);7956 addUsersToMoveToVALUWorklist(DestReg, MRI, Worklist);7957 Inst.eraseFromParent();7958 }7959 return;7960 case AMDGPU::S_LSHL1_ADD_U32:7961 case AMDGPU::S_LSHL2_ADD_U32:7962 case AMDGPU::S_LSHL3_ADD_U32:7963 case AMDGPU::S_LSHL4_ADD_U32: {7964 MachineOperand &Dest = Inst.getOperand(0);7965 MachineOperand &Src0 = Inst.getOperand(1);7966 MachineOperand &Src1 = Inst.getOperand(2);7967 unsigned ShiftAmt = (Opcode == AMDGPU::S_LSHL1_ADD_U32 ? 17968 : Opcode == AMDGPU::S_LSHL2_ADD_U32 ? 27969 : Opcode == AMDGPU::S_LSHL3_ADD_U32 ? 37970 : 4);7971 7972 const TargetRegisterClass *NewRC =7973 RI.getEquivalentVGPRClass(MRI.getRegClass(Dest.getReg()));7974 Register DestReg = MRI.createVirtualRegister(NewRC);7975 MachineInstr *NewInstr =7976 BuildMI(*MBB, &Inst, DL, get(AMDGPU::V_LSHL_ADD_U32_e64), DestReg)7977 .add(Src0)7978 .addImm(ShiftAmt)7979 .add(Src1);7980 7981 legalizeOperands(*NewInstr, MDT);7982 MRI.replaceRegWith(Dest.getReg(), DestReg);7983 addUsersToMoveToVALUWorklist(DestReg, MRI, Worklist);7984 Inst.eraseFromParent();7985 }7986 return;7987 case AMDGPU::S_CSELECT_B32:7988 case AMDGPU::S_CSELECT_B64:7989 lowerSelect(Worklist, Inst, MDT);7990 Inst.eraseFromParent();7991 return;7992 case AMDGPU::S_CMP_EQ_I32:7993 case AMDGPU::S_CMP_LG_I32:7994 case AMDGPU::S_CMP_GT_I32:7995 case AMDGPU::S_CMP_GE_I32:7996 case AMDGPU::S_CMP_LT_I32:7997 case AMDGPU::S_CMP_LE_I32:7998 case AMDGPU::S_CMP_EQ_U32:7999 case AMDGPU::S_CMP_LG_U32:8000 case AMDGPU::S_CMP_GT_U32:8001 case AMDGPU::S_CMP_GE_U32:8002 case AMDGPU::S_CMP_LT_U32:8003 case AMDGPU::S_CMP_LE_U32:8004 case AMDGPU::S_CMP_EQ_U64:8005 case AMDGPU::S_CMP_LG_U64:8006 case AMDGPU::S_CMP_LT_F32:8007 case AMDGPU::S_CMP_EQ_F32:8008 case AMDGPU::S_CMP_LE_F32:8009 case AMDGPU::S_CMP_GT_F32:8010 case AMDGPU::S_CMP_LG_F32:8011 case AMDGPU::S_CMP_GE_F32:8012 case AMDGPU::S_CMP_O_F32:8013 case AMDGPU::S_CMP_U_F32:8014 case AMDGPU::S_CMP_NGE_F32:8015 case AMDGPU::S_CMP_NLG_F32:8016 case AMDGPU::S_CMP_NGT_F32:8017 case AMDGPU::S_CMP_NLE_F32:8018 case AMDGPU::S_CMP_NEQ_F32:8019 case AMDGPU::S_CMP_NLT_F32: {8020 Register CondReg = MRI.createVirtualRegister(RI.getWaveMaskRegClass());8021 auto NewInstr =8022 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(NewOpcode), CondReg)8023 .setMIFlags(Inst.getFlags());8024 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::src0_modifiers) >=8025 0) {8026 NewInstr8027 .addImm(0) // src0_modifiers8028 .add(Inst.getOperand(0)) // src08029 .addImm(0) // src1_modifiers8030 .add(Inst.getOperand(1)) // src18031 .addImm(0); // clamp8032 } else {8033 NewInstr.add(Inst.getOperand(0)).add(Inst.getOperand(1));8034 }8035 legalizeOperands(*NewInstr, MDT);8036 int SCCIdx = Inst.findRegisterDefOperandIdx(AMDGPU::SCC, /*TRI=*/nullptr);8037 const MachineOperand &SCCOp = Inst.getOperand(SCCIdx);8038 addSCCDefUsersToVALUWorklist(SCCOp, Inst, Worklist, CondReg);8039 Inst.eraseFromParent();8040 return;8041 }8042 case AMDGPU::S_CMP_LT_F16:8043 case AMDGPU::S_CMP_EQ_F16:8044 case AMDGPU::S_CMP_LE_F16:8045 case AMDGPU::S_CMP_GT_F16:8046 case AMDGPU::S_CMP_LG_F16:8047 case AMDGPU::S_CMP_GE_F16:8048 case AMDGPU::S_CMP_O_F16:8049 case AMDGPU::S_CMP_U_F16:8050 case AMDGPU::S_CMP_NGE_F16:8051 case AMDGPU::S_CMP_NLG_F16:8052 case AMDGPU::S_CMP_NGT_F16:8053 case AMDGPU::S_CMP_NLE_F16:8054 case AMDGPU::S_CMP_NEQ_F16:8055 case AMDGPU::S_CMP_NLT_F16: {8056 Register CondReg = MRI.createVirtualRegister(RI.getWaveMaskRegClass());8057 auto NewInstr =8058 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(NewOpcode), CondReg)8059 .setMIFlags(Inst.getFlags());8060 if (AMDGPU::hasNamedOperand(NewOpcode, AMDGPU::OpName::src0_modifiers)) {8061 NewInstr8062 .addImm(0) // src0_modifiers8063 .add(Inst.getOperand(0)) // src08064 .addImm(0) // src1_modifiers8065 .add(Inst.getOperand(1)) // src18066 .addImm(0); // clamp8067 if (AMDGPU::hasNamedOperand(NewOpcode, AMDGPU::OpName::op_sel))8068 NewInstr.addImm(0); // op_sel08069 } else {8070 NewInstr8071 .add(Inst.getOperand(0))8072 .add(Inst.getOperand(1));8073 }8074 legalizeOperandsVALUt16(*NewInstr, MRI);8075 legalizeOperands(*NewInstr, MDT);8076 int SCCIdx = Inst.findRegisterDefOperandIdx(AMDGPU::SCC, /*TRI=*/nullptr);8077 const MachineOperand &SCCOp = Inst.getOperand(SCCIdx);8078 addSCCDefUsersToVALUWorklist(SCCOp, Inst, Worklist, CondReg);8079 Inst.eraseFromParent();8080 return;8081 }8082 case AMDGPU::S_CVT_HI_F32_F16: {8083 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);8084 Register NewDst = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);8085 if (ST.useRealTrue16Insts()) {8086 BuildMI(*MBB, Inst, DL, get(AMDGPU::COPY), TmpReg)8087 .add(Inst.getOperand(1));8088 BuildMI(*MBB, Inst, DL, get(NewOpcode), NewDst)8089 .addImm(0) // src0_modifiers8090 .addReg(TmpReg, 0, AMDGPU::hi16)8091 .addImm(0) // clamp8092 .addImm(0) // omod8093 .addImm(0); // op_sel08094 } else {8095 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg)8096 .addImm(16)8097 .add(Inst.getOperand(1));8098 BuildMI(*MBB, Inst, DL, get(NewOpcode), NewDst)8099 .addImm(0) // src0_modifiers8100 .addReg(TmpReg)8101 .addImm(0) // clamp8102 .addImm(0); // omod8103 }8104 8105 MRI.replaceRegWith(Inst.getOperand(0).getReg(), NewDst);8106 addUsersToMoveToVALUWorklist(NewDst, MRI, Worklist);8107 Inst.eraseFromParent();8108 return;8109 }8110 case AMDGPU::S_MINIMUM_F32:8111 case AMDGPU::S_MAXIMUM_F32: {8112 Register NewDst = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);8113 MachineInstr *NewInstr = BuildMI(*MBB, Inst, DL, get(NewOpcode), NewDst)8114 .addImm(0) // src0_modifiers8115 .add(Inst.getOperand(1))8116 .addImm(0) // src1_modifiers8117 .add(Inst.getOperand(2))8118 .addImm(0) // clamp8119 .addImm(0); // omod8120 MRI.replaceRegWith(Inst.getOperand(0).getReg(), NewDst);8121 8122 legalizeOperands(*NewInstr, MDT);8123 addUsersToMoveToVALUWorklist(NewDst, MRI, Worklist);8124 Inst.eraseFromParent();8125 return;8126 }8127 case AMDGPU::S_MINIMUM_F16:8128 case AMDGPU::S_MAXIMUM_F16: {8129 Register NewDst = MRI.createVirtualRegister(ST.useRealTrue16Insts()8130 ? &AMDGPU::VGPR_16RegClass8131 : &AMDGPU::VGPR_32RegClass);8132 MachineInstr *NewInstr = BuildMI(*MBB, Inst, DL, get(NewOpcode), NewDst)8133 .addImm(0) // src0_modifiers8134 .add(Inst.getOperand(1))8135 .addImm(0) // src1_modifiers8136 .add(Inst.getOperand(2))8137 .addImm(0) // clamp8138 .addImm(0) // omod8139 .addImm(0); // opsel08140 MRI.replaceRegWith(Inst.getOperand(0).getReg(), NewDst);8141 legalizeOperandsVALUt16(*NewInstr, MRI);8142 legalizeOperands(*NewInstr, MDT);8143 addUsersToMoveToVALUWorklist(NewDst, MRI, Worklist);8144 Inst.eraseFromParent();8145 return;8146 }8147 case AMDGPU::V_S_EXP_F16_e64:8148 case AMDGPU::V_S_LOG_F16_e64:8149 case AMDGPU::V_S_RCP_F16_e64:8150 case AMDGPU::V_S_RSQ_F16_e64:8151 case AMDGPU::V_S_SQRT_F16_e64: {8152 Register NewDst = MRI.createVirtualRegister(ST.useRealTrue16Insts()8153 ? &AMDGPU::VGPR_16RegClass8154 : &AMDGPU::VGPR_32RegClass);8155 auto NewInstr = BuildMI(*MBB, Inst, DL, get(NewOpcode), NewDst)8156 .add(Inst.getOperand(1)) // src0_modifiers8157 .add(Inst.getOperand(2))8158 .add(Inst.getOperand(3)) // clamp8159 .add(Inst.getOperand(4)) // omod8160 .setMIFlags(Inst.getFlags());8161 if (AMDGPU::hasNamedOperand(NewOpcode, AMDGPU::OpName::op_sel))8162 NewInstr.addImm(0); // opsel08163 MRI.replaceRegWith(Inst.getOperand(0).getReg(), NewDst);8164 legalizeOperandsVALUt16(*NewInstr, MRI);8165 legalizeOperands(*NewInstr, MDT);8166 addUsersToMoveToVALUWorklist(NewDst, MRI, Worklist);8167 Inst.eraseFromParent();8168 return;8169 }8170 }8171 8172 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {8173 // We cannot move this instruction to the VALU, so we should try to8174 // legalize its operands instead.8175 legalizeOperands(Inst, MDT);8176 return;8177 }8178 // Handle converting generic instructions like COPY-to-SGPR into8179 // COPY-to-VGPR.8180 if (NewOpcode == Opcode) {8181 Register DstReg = Inst.getOperand(0).getReg();8182 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst);8183 8184 // If it's a copy of a VGPR to a physical SGPR, insert a V_READFIRSTLANE and8185 // hope for the best.8186 if (Inst.isCopy() && DstReg.isPhysical() &&8187 RI.isVGPR(MRI, Inst.getOperand(1).getReg())) {8188 Register NewDst = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);8189 BuildMI(*Inst.getParent(), &Inst, Inst.getDebugLoc(),8190 get(AMDGPU::V_READFIRSTLANE_B32), NewDst)8191 .add(Inst.getOperand(1));8192 BuildMI(*Inst.getParent(), &Inst, Inst.getDebugLoc(), get(AMDGPU::COPY),8193 DstReg)8194 .addReg(NewDst);8195 8196 Inst.eraseFromParent();8197 return;8198 }8199 8200 if (Inst.isCopy() && Inst.getOperand(1).getReg().isVirtual()) {8201 Register NewDstReg = Inst.getOperand(1).getReg();8202 const TargetRegisterClass *SrcRC = RI.getRegClassForReg(MRI, NewDstReg);8203 if (const TargetRegisterClass *CommonRC =8204 RI.getCommonSubClass(NewDstRC, SrcRC)) {8205 // Instead of creating a copy where src and dst are the same register8206 // class, we just replace all uses of dst with src. These kinds of8207 // copies interfere with the heuristics MachineSink uses to decide8208 // whether or not to split a critical edge. Since the pass assumes8209 // that copies will end up as machine instructions and not be8210 // eliminated.8211 addUsersToMoveToVALUWorklist(DstReg, MRI, Worklist);8212 MRI.replaceRegWith(DstReg, NewDstReg);8213 MRI.clearKillFlags(NewDstReg);8214 Inst.getOperand(0).setReg(DstReg);8215 8216 if (!MRI.constrainRegClass(NewDstReg, CommonRC))8217 llvm_unreachable("failed to constrain register");8218 8219 Inst.eraseFromParent();8220 // Legalize t16 operand since replaceReg is called after addUsersToVALU8221 for (MachineOperand &MO :8222 make_early_inc_range(MRI.use_operands(NewDstReg))) {8223 legalizeOperandsVALUt16(*MO.getParent(), MRI);8224 }8225 8226 return;8227 }8228 }8229 8230 // If this is a v2s copy between 16bit and 32bit reg,8231 // replace vgpr copy to reg_sequence/extract_subreg8232 // This can be remove after we have sgpr16 in place8233 if (ST.useRealTrue16Insts() && Inst.isCopy() &&8234 Inst.getOperand(1).getReg().isVirtual() &&8235 RI.isVGPR(MRI, Inst.getOperand(1).getReg())) {8236 const TargetRegisterClass *SrcRegRC = getOpRegClass(Inst, 1);8237 if (RI.getMatchingSuperRegClass(NewDstRC, SrcRegRC, AMDGPU::lo16)) {8238 Register NewDstReg = MRI.createVirtualRegister(NewDstRC);8239 Register Undef = MRI.createVirtualRegister(&AMDGPU::VGPR_16RegClass);8240 BuildMI(*Inst.getParent(), &Inst, Inst.getDebugLoc(),8241 get(AMDGPU::IMPLICIT_DEF), Undef);8242 BuildMI(*Inst.getParent(), &Inst, Inst.getDebugLoc(),8243 get(AMDGPU::REG_SEQUENCE), NewDstReg)8244 .addReg(Inst.getOperand(1).getReg())8245 .addImm(AMDGPU::lo16)8246 .addReg(Undef)8247 .addImm(AMDGPU::hi16);8248 Inst.eraseFromParent();8249 MRI.replaceRegWith(DstReg, NewDstReg);8250 addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist);8251 return;8252 } else if (RI.getMatchingSuperRegClass(SrcRegRC, NewDstRC,8253 AMDGPU::lo16)) {8254 Inst.getOperand(1).setSubReg(AMDGPU::lo16);8255 Register NewDstReg = MRI.createVirtualRegister(NewDstRC);8256 MRI.replaceRegWith(DstReg, NewDstReg);8257 addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist);8258 return;8259 }8260 }8261 8262 Register NewDstReg = MRI.createVirtualRegister(NewDstRC);8263 MRI.replaceRegWith(DstReg, NewDstReg);8264 legalizeOperands(Inst, MDT);8265 addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist);8266 return;8267 }8268 8269 // Use the new VALU Opcode.8270 auto NewInstr = BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(NewOpcode))8271 .setMIFlags(Inst.getFlags());8272 if (isVOP3(NewOpcode) && !isVOP3(Opcode)) {8273 // Intersperse VOP3 modifiers among the SALU operands.8274 NewInstr->addOperand(Inst.getOperand(0));8275 if (AMDGPU::getNamedOperandIdx(NewOpcode,8276 AMDGPU::OpName::src0_modifiers) >= 0)8277 NewInstr.addImm(0);8278 if (AMDGPU::hasNamedOperand(NewOpcode, AMDGPU::OpName::src0)) {8279 const MachineOperand &Src = Inst.getOperand(1);8280 NewInstr->addOperand(Src);8281 }8282 8283 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {8284 // We are converting these to a BFE, so we need to add the missing8285 // operands for the size and offset.8286 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;8287 NewInstr.addImm(0);8288 NewInstr.addImm(Size);8289 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {8290 // The VALU version adds the second operand to the result, so insert an8291 // extra 0 operand.8292 NewInstr.addImm(0);8293 } else if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {8294 const MachineOperand &OffsetWidthOp = Inst.getOperand(2);8295 // If we need to move this to VGPRs, we need to unpack the second8296 // operand back into the 2 separate ones for bit offset and width.8297 assert(OffsetWidthOp.isImm() &&8298 "Scalar BFE is only implemented for constant width and offset");8299 uint32_t Imm = OffsetWidthOp.getImm();8300 8301 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].8302 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].8303 NewInstr.addImm(Offset);8304 NewInstr.addImm(BitWidth);8305 } else {8306 if (AMDGPU::getNamedOperandIdx(NewOpcode,8307 AMDGPU::OpName::src1_modifiers) >= 0)8308 NewInstr.addImm(0);8309 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::src1) >= 0)8310 NewInstr->addOperand(Inst.getOperand(2));8311 if (AMDGPU::getNamedOperandIdx(NewOpcode,8312 AMDGPU::OpName::src2_modifiers) >= 0)8313 NewInstr.addImm(0);8314 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::src2) >= 0)8315 NewInstr->addOperand(Inst.getOperand(3));8316 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::clamp) >= 0)8317 NewInstr.addImm(0);8318 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::omod) >= 0)8319 NewInstr.addImm(0);8320 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::op_sel) >= 0)8321 NewInstr.addImm(0);8322 }8323 } else {8324 // Just copy the SALU operands.8325 for (const MachineOperand &Op : Inst.explicit_operands())8326 NewInstr->addOperand(Op);8327 }8328 8329 // Remove any references to SCC. Vector instructions can't read from it, and8330 // We're just about to add the implicit use / defs of VCC, and we don't want8331 // both.8332 for (MachineOperand &Op : Inst.implicit_operands()) {8333 if (Op.getReg() == AMDGPU::SCC) {8334 // Only propagate through live-def of SCC.8335 if (Op.isDef() && !Op.isDead())8336 addSCCDefUsersToVALUWorklist(Op, Inst, Worklist);8337 if (Op.isUse())8338 addSCCDefsToVALUWorklist(NewInstr, Worklist);8339 }8340 }8341 Inst.eraseFromParent();8342 Register NewDstReg;8343 if (NewInstr->getOperand(0).isReg() && NewInstr->getOperand(0).isDef()) {8344 Register DstReg = NewInstr->getOperand(0).getReg();8345 assert(DstReg.isVirtual());8346 // Update the destination register class.8347 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(*NewInstr);8348 assert(NewDstRC);8349 NewDstReg = MRI.createVirtualRegister(NewDstRC);8350 MRI.replaceRegWith(DstReg, NewDstReg);8351 }8352 fixImplicitOperands(*NewInstr);8353 8354 legalizeOperandsVALUt16(*NewInstr, MRI);8355 8356 // Legalize the operands8357 legalizeOperands(*NewInstr, MDT);8358 if (NewDstReg)8359 addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist);8360}8361 8362// Add/sub require special handling to deal with carry outs.8363std::pair<bool, MachineBasicBlock *>8364SIInstrInfo::moveScalarAddSub(SIInstrWorklist &Worklist, MachineInstr &Inst,8365 MachineDominatorTree *MDT) const {8366 if (ST.hasAddNoCarry()) {8367 // Assume there is no user of scc since we don't select this in that case.8368 // Since scc isn't used, it doesn't really matter if the i32 or u32 variant8369 // is used.8370 8371 MachineBasicBlock &MBB = *Inst.getParent();8372 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();8373 8374 Register OldDstReg = Inst.getOperand(0).getReg();8375 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);8376 8377 unsigned Opc = Inst.getOpcode();8378 assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32);8379 8380 unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ?8381 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64;8382 8383 assert(Inst.getOperand(3).getReg() == AMDGPU::SCC);8384 Inst.removeOperand(3);8385 8386 Inst.setDesc(get(NewOpc));8387 Inst.addOperand(MachineOperand::CreateImm(0)); // clamp bit8388 Inst.addImplicitDefUseOperands(*MBB.getParent());8389 MRI.replaceRegWith(OldDstReg, ResultReg);8390 MachineBasicBlock *NewBB = legalizeOperands(Inst, MDT);8391 8392 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);8393 return std::pair(true, NewBB);8394 }8395 8396 return std::pair(false, nullptr);8397}8398 8399void SIInstrInfo::lowerSelect(SIInstrWorklist &Worklist, MachineInstr &Inst,8400 MachineDominatorTree *MDT) const {8401 8402 MachineBasicBlock &MBB = *Inst.getParent();8403 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();8404 MachineBasicBlock::iterator MII = Inst;8405 DebugLoc DL = Inst.getDebugLoc();8406 8407 MachineOperand &Dest = Inst.getOperand(0);8408 MachineOperand &Src0 = Inst.getOperand(1);8409 MachineOperand &Src1 = Inst.getOperand(2);8410 MachineOperand &Cond = Inst.getOperand(3);8411 8412 Register CondReg = Cond.getReg();8413 bool IsSCC = (CondReg == AMDGPU::SCC);8414 8415 // If this is a trivial select where the condition is effectively not SCC8416 // (CondReg is a source of copy to SCC), then the select is semantically8417 // equivalent to copying CondReg. Hence, there is no need to create8418 // V_CNDMASK, we can just use that and bail out.8419 if (!IsSCC && Src0.isImm() && (Src0.getImm() == -1) && Src1.isImm() &&8420 (Src1.getImm() == 0)) {8421 MRI.replaceRegWith(Dest.getReg(), CondReg);8422 return;8423 }8424 8425 Register NewCondReg = CondReg;8426 if (IsSCC) {8427 const TargetRegisterClass *TC = RI.getWaveMaskRegClass();8428 NewCondReg = MRI.createVirtualRegister(TC);8429 8430 // Now look for the closest SCC def if it is a copy8431 // replacing the CondReg with the COPY source register8432 bool CopyFound = false;8433 for (MachineInstr &CandI :8434 make_range(std::next(MachineBasicBlock::reverse_iterator(Inst)),8435 Inst.getParent()->rend())) {8436 if (CandI.findRegisterDefOperandIdx(AMDGPU::SCC, &RI, false, false) !=8437 -1) {8438 if (CandI.isCopy() && CandI.getOperand(0).getReg() == AMDGPU::SCC) {8439 BuildMI(MBB, MII, DL, get(AMDGPU::COPY), NewCondReg)8440 .addReg(CandI.getOperand(1).getReg());8441 CopyFound = true;8442 }8443 break;8444 }8445 }8446 if (!CopyFound) {8447 // SCC def is not a copy8448 // Insert a trivial select instead of creating a copy, because a copy from8449 // SCC would semantically mean just copying a single bit, but we may need8450 // the result to be a vector condition mask that needs preserving.8451 unsigned Opcode =8452 ST.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;8453 auto NewSelect =8454 BuildMI(MBB, MII, DL, get(Opcode), NewCondReg).addImm(-1).addImm(0);8455 NewSelect->getOperand(3).setIsUndef(Cond.isUndef());8456 }8457 }8458 8459 Register NewDestReg = MRI.createVirtualRegister(8460 RI.getEquivalentVGPRClass(MRI.getRegClass(Dest.getReg())));8461 MachineInstr *NewInst;8462 if (Inst.getOpcode() == AMDGPU::S_CSELECT_B32) {8463 NewInst = BuildMI(MBB, MII, DL, get(AMDGPU::V_CNDMASK_B32_e64), NewDestReg)8464 .addImm(0)8465 .add(Src1) // False8466 .addImm(0)8467 .add(Src0) // True8468 .addReg(NewCondReg);8469 } else {8470 NewInst =8471 BuildMI(MBB, MII, DL, get(AMDGPU::V_CNDMASK_B64_PSEUDO), NewDestReg)8472 .add(Src1) // False8473 .add(Src0) // True8474 .addReg(NewCondReg);8475 }8476 MRI.replaceRegWith(Dest.getReg(), NewDestReg);8477 legalizeOperands(*NewInst, MDT);8478 addUsersToMoveToVALUWorklist(NewDestReg, MRI, Worklist);8479}8480 8481void SIInstrInfo::lowerScalarAbs(SIInstrWorklist &Worklist,8482 MachineInstr &Inst) const {8483 MachineBasicBlock &MBB = *Inst.getParent();8484 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();8485 MachineBasicBlock::iterator MII = Inst;8486 DebugLoc DL = Inst.getDebugLoc();8487 8488 MachineOperand &Dest = Inst.getOperand(0);8489 MachineOperand &Src = Inst.getOperand(1);8490 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);8491 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);8492 8493 unsigned SubOp = ST.hasAddNoCarry() ?8494 AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_CO_U32_e32;8495 8496 BuildMI(MBB, MII, DL, get(SubOp), TmpReg)8497 .addImm(0)8498 .addReg(Src.getReg());8499 8500 BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg)8501 .addReg(Src.getReg())8502 .addReg(TmpReg);8503 8504 MRI.replaceRegWith(Dest.getReg(), ResultReg);8505 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);8506}8507 8508void SIInstrInfo::lowerScalarAbsDiff(SIInstrWorklist &Worklist,8509 MachineInstr &Inst) const {8510 MachineBasicBlock &MBB = *Inst.getParent();8511 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();8512 MachineBasicBlock::iterator MII = Inst;8513 const DebugLoc &DL = Inst.getDebugLoc();8514 8515 MachineOperand &Dest = Inst.getOperand(0);8516 MachineOperand &Src1 = Inst.getOperand(1);8517 MachineOperand &Src2 = Inst.getOperand(2);8518 Register SubResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);8519 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);8520 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);8521 8522 unsigned SubOp =8523 ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_CO_U32_e32;8524 8525 BuildMI(MBB, MII, DL, get(SubOp), SubResultReg)8526 .addReg(Src1.getReg())8527 .addReg(Src2.getReg());8528 8529 BuildMI(MBB, MII, DL, get(SubOp), TmpReg).addImm(0).addReg(SubResultReg);8530 8531 BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg)8532 .addReg(SubResultReg)8533 .addReg(TmpReg);8534 8535 MRI.replaceRegWith(Dest.getReg(), ResultReg);8536 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);8537}8538 8539void SIInstrInfo::lowerScalarXnor(SIInstrWorklist &Worklist,8540 MachineInstr &Inst) const {8541 MachineBasicBlock &MBB = *Inst.getParent();8542 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();8543 MachineBasicBlock::iterator MII = Inst;8544 const DebugLoc &DL = Inst.getDebugLoc();8545 8546 MachineOperand &Dest = Inst.getOperand(0);8547 MachineOperand &Src0 = Inst.getOperand(1);8548 MachineOperand &Src1 = Inst.getOperand(2);8549 8550 if (ST.hasDLInsts()) {8551 Register NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);8552 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL);8553 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL);8554 8555 BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest)8556 .add(Src0)8557 .add(Src1);8558 8559 MRI.replaceRegWith(Dest.getReg(), NewDest);8560 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);8561 } else {8562 // Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can8563 // invert either source and then perform the XOR. If either source is a8564 // scalar register, then we can leave the inversion on the scalar unit to8565 // achieve a better distribution of scalar and vector instructions.8566 bool Src0IsSGPR = Src0.isReg() &&8567 RI.isSGPRClass(MRI.getRegClass(Src0.getReg()));8568 bool Src1IsSGPR = Src1.isReg() &&8569 RI.isSGPRClass(MRI.getRegClass(Src1.getReg()));8570 MachineInstr *Xor;8571 Register Temp = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);8572 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);8573 8574 // Build a pair of scalar instructions and add them to the work list.8575 // The next iteration over the work list will lower these to the vector8576 // unit as necessary.8577 if (Src0IsSGPR) {8578 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src0);8579 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)8580 .addReg(Temp)8581 .add(Src1);8582 } else if (Src1IsSGPR) {8583 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src1);8584 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)8585 .add(Src0)8586 .addReg(Temp);8587 } else {8588 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), Temp)8589 .add(Src0)8590 .add(Src1);8591 MachineInstr *Not =8592 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest).addReg(Temp);8593 Worklist.insert(Not);8594 }8595 8596 MRI.replaceRegWith(Dest.getReg(), NewDest);8597 8598 Worklist.insert(Xor);8599 8600 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);8601 }8602}8603 8604void SIInstrInfo::splitScalarNotBinop(SIInstrWorklist &Worklist,8605 MachineInstr &Inst,8606 unsigned Opcode) const {8607 MachineBasicBlock &MBB = *Inst.getParent();8608 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();8609 MachineBasicBlock::iterator MII = Inst;8610 const DebugLoc &DL = Inst.getDebugLoc();8611 8612 MachineOperand &Dest = Inst.getOperand(0);8613 MachineOperand &Src0 = Inst.getOperand(1);8614 MachineOperand &Src1 = Inst.getOperand(2);8615 8616 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);8617 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);8618 8619 MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), Interm)8620 .add(Src0)8621 .add(Src1);8622 8623 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest)8624 .addReg(Interm);8625 8626 Worklist.insert(&Op);8627 Worklist.insert(&Not);8628 8629 MRI.replaceRegWith(Dest.getReg(), NewDest);8630 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);8631}8632 8633void SIInstrInfo::splitScalarBinOpN2(SIInstrWorklist &Worklist,8634 MachineInstr &Inst,8635 unsigned Opcode) const {8636 MachineBasicBlock &MBB = *Inst.getParent();8637 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();8638 MachineBasicBlock::iterator MII = Inst;8639 const DebugLoc &DL = Inst.getDebugLoc();8640 8641 MachineOperand &Dest = Inst.getOperand(0);8642 MachineOperand &Src0 = Inst.getOperand(1);8643 MachineOperand &Src1 = Inst.getOperand(2);8644 8645 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);8646 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);8647 8648 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Interm)8649 .add(Src1);8650 8651 MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), NewDest)8652 .add(Src0)8653 .addReg(Interm);8654 8655 Worklist.insert(&Not);8656 Worklist.insert(&Op);8657 8658 MRI.replaceRegWith(Dest.getReg(), NewDest);8659 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);8660}8661 8662void SIInstrInfo::splitScalar64BitUnaryOp(SIInstrWorklist &Worklist,8663 MachineInstr &Inst, unsigned Opcode,8664 bool Swap) const {8665 MachineBasicBlock &MBB = *Inst.getParent();8666 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();8667 8668 MachineOperand &Dest = Inst.getOperand(0);8669 MachineOperand &Src0 = Inst.getOperand(1);8670 DebugLoc DL = Inst.getDebugLoc();8671 8672 MachineBasicBlock::iterator MII = Inst;8673 8674 const MCInstrDesc &InstDesc = get(Opcode);8675 const TargetRegisterClass *Src0RC = Src0.isReg() ?8676 MRI.getRegClass(Src0.getReg()) :8677 &AMDGPU::SGPR_32RegClass;8678 8679 const TargetRegisterClass *Src0SubRC =8680 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);8681 8682 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,8683 AMDGPU::sub0, Src0SubRC);8684 8685 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());8686 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);8687 const TargetRegisterClass *NewDestSubRC =8688 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0);8689 8690 Register DestSub0 = MRI.createVirtualRegister(NewDestSubRC);8691 MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0).add(SrcReg0Sub0);8692 8693 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,8694 AMDGPU::sub1, Src0SubRC);8695 8696 Register DestSub1 = MRI.createVirtualRegister(NewDestSubRC);8697 MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1).add(SrcReg0Sub1);8698 8699 if (Swap)8700 std::swap(DestSub0, DestSub1);8701 8702 Register FullDestReg = MRI.createVirtualRegister(NewDestRC);8703 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)8704 .addReg(DestSub0)8705 .addImm(AMDGPU::sub0)8706 .addReg(DestSub1)8707 .addImm(AMDGPU::sub1);8708 8709 MRI.replaceRegWith(Dest.getReg(), FullDestReg);8710 8711 Worklist.insert(&LoHalf);8712 Worklist.insert(&HiHalf);8713 8714 // We don't need to legalizeOperands here because for a single operand, src08715 // will support any kind of input.8716 8717 // Move all users of this moved value.8718 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);8719}8720 8721// There is not a vector equivalent of s_mul_u64. For this reason, we need to8722// split the s_mul_u64 in 32-bit vector multiplications.8723void SIInstrInfo::splitScalarSMulU64(SIInstrWorklist &Worklist,8724 MachineInstr &Inst,8725 MachineDominatorTree *MDT) const {8726 MachineBasicBlock &MBB = *Inst.getParent();8727 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();8728 8729 Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);8730 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);8731 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);8732 8733 MachineOperand &Dest = Inst.getOperand(0);8734 MachineOperand &Src0 = Inst.getOperand(1);8735 MachineOperand &Src1 = Inst.getOperand(2);8736 const DebugLoc &DL = Inst.getDebugLoc();8737 MachineBasicBlock::iterator MII = Inst;8738 8739 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0.getReg());8740 const TargetRegisterClass *Src1RC = MRI.getRegClass(Src1.getReg());8741 const TargetRegisterClass *Src0SubRC =8742 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);8743 if (RI.isSGPRClass(Src0SubRC))8744 Src0SubRC = RI.getEquivalentVGPRClass(Src0SubRC);8745 const TargetRegisterClass *Src1SubRC =8746 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);8747 if (RI.isSGPRClass(Src1SubRC))8748 Src1SubRC = RI.getEquivalentVGPRClass(Src1SubRC);8749 8750 // First, we extract the low 32-bit and high 32-bit values from each of the8751 // operands.8752 MachineOperand Op0L =8753 buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC);8754 MachineOperand Op1L =8755 buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC);8756 MachineOperand Op0H =8757 buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, AMDGPU::sub1, Src0SubRC);8758 MachineOperand Op1H =8759 buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC);8760 8761 // The multilication is done as follows:8762 //8763 // Op1H Op1L8764 // * Op0H Op0L8765 // --------------------8766 // Op1H*Op0L Op1L*Op0L8767 // + Op1H*Op0H Op1L*Op0H8768 // -----------------------------------------8769 // (Op1H*Op0L + Op1L*Op0H + carry) Op1L*Op0L8770 //8771 // We drop Op1H*Op0H because the result of the multiplication is a 64-bit8772 // value and that would overflow.8773 // The low 32-bit value is Op1L*Op0L.8774 // The high 32-bit value is Op1H*Op0L + Op1L*Op0H + carry (from Op1L*Op0L).8775 8776 Register Op1L_Op0H_Reg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);8777 MachineInstr *Op1L_Op0H =8778 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), Op1L_Op0H_Reg)8779 .add(Op1L)8780 .add(Op0H);8781 8782 Register Op1H_Op0L_Reg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);8783 MachineInstr *Op1H_Op0L =8784 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), Op1H_Op0L_Reg)8785 .add(Op1H)8786 .add(Op0L);8787 8788 Register CarryReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);8789 MachineInstr *Carry =8790 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_HI_U32_e64), CarryReg)8791 .add(Op1L)8792 .add(Op0L);8793 8794 MachineInstr *LoHalf =8795 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), DestSub0)8796 .add(Op1L)8797 .add(Op0L);8798 8799 Register AddReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);8800 MachineInstr *Add = BuildMI(MBB, MII, DL, get(AMDGPU::V_ADD_U32_e32), AddReg)8801 .addReg(Op1L_Op0H_Reg)8802 .addReg(Op1H_Op0L_Reg);8803 8804 MachineInstr *HiHalf =8805 BuildMI(MBB, MII, DL, get(AMDGPU::V_ADD_U32_e32), DestSub1)8806 .addReg(AddReg)8807 .addReg(CarryReg);8808 8809 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)8810 .addReg(DestSub0)8811 .addImm(AMDGPU::sub0)8812 .addReg(DestSub1)8813 .addImm(AMDGPU::sub1);8814 8815 MRI.replaceRegWith(Dest.getReg(), FullDestReg);8816 8817 // Try to legalize the operands in case we need to swap the order to keep it8818 // valid.8819 legalizeOperands(*Op1L_Op0H, MDT);8820 legalizeOperands(*Op1H_Op0L, MDT);8821 legalizeOperands(*Carry, MDT);8822 legalizeOperands(*LoHalf, MDT);8823 legalizeOperands(*Add, MDT);8824 legalizeOperands(*HiHalf, MDT);8825 8826 // Move all users of this moved value.8827 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);8828}8829 8830// Lower S_MUL_U64_U32_PSEUDO/S_MUL_I64_I32_PSEUDO in two 32-bit vector8831// multiplications.8832void SIInstrInfo::splitScalarSMulPseudo(SIInstrWorklist &Worklist,8833 MachineInstr &Inst,8834 MachineDominatorTree *MDT) const {8835 MachineBasicBlock &MBB = *Inst.getParent();8836 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();8837 8838 Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);8839 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);8840 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);8841 8842 MachineOperand &Dest = Inst.getOperand(0);8843 MachineOperand &Src0 = Inst.getOperand(1);8844 MachineOperand &Src1 = Inst.getOperand(2);8845 const DebugLoc &DL = Inst.getDebugLoc();8846 MachineBasicBlock::iterator MII = Inst;8847 8848 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0.getReg());8849 const TargetRegisterClass *Src1RC = MRI.getRegClass(Src1.getReg());8850 const TargetRegisterClass *Src0SubRC =8851 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);8852 if (RI.isSGPRClass(Src0SubRC))8853 Src0SubRC = RI.getEquivalentVGPRClass(Src0SubRC);8854 const TargetRegisterClass *Src1SubRC =8855 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);8856 if (RI.isSGPRClass(Src1SubRC))8857 Src1SubRC = RI.getEquivalentVGPRClass(Src1SubRC);8858 8859 // First, we extract the low 32-bit and high 32-bit values from each of the8860 // operands.8861 MachineOperand Op0L =8862 buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC);8863 MachineOperand Op1L =8864 buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC);8865 8866 unsigned Opc = Inst.getOpcode();8867 unsigned NewOpc = Opc == AMDGPU::S_MUL_U64_U32_PSEUDO8868 ? AMDGPU::V_MUL_HI_U32_e648869 : AMDGPU::V_MUL_HI_I32_e64;8870 MachineInstr *HiHalf =8871 BuildMI(MBB, MII, DL, get(NewOpc), DestSub1).add(Op1L).add(Op0L);8872 8873 MachineInstr *LoHalf =8874 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), DestSub0)8875 .add(Op1L)8876 .add(Op0L);8877 8878 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)8879 .addReg(DestSub0)8880 .addImm(AMDGPU::sub0)8881 .addReg(DestSub1)8882 .addImm(AMDGPU::sub1);8883 8884 MRI.replaceRegWith(Dest.getReg(), FullDestReg);8885 8886 // Try to legalize the operands in case we need to swap the order to keep it8887 // valid.8888 legalizeOperands(*HiHalf, MDT);8889 legalizeOperands(*LoHalf, MDT);8890 8891 // Move all users of this moved value.8892 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);8893}8894 8895void SIInstrInfo::splitScalar64BitBinaryOp(SIInstrWorklist &Worklist,8896 MachineInstr &Inst, unsigned Opcode,8897 MachineDominatorTree *MDT) const {8898 MachineBasicBlock &MBB = *Inst.getParent();8899 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();8900 8901 MachineOperand &Dest = Inst.getOperand(0);8902 MachineOperand &Src0 = Inst.getOperand(1);8903 MachineOperand &Src1 = Inst.getOperand(2);8904 DebugLoc DL = Inst.getDebugLoc();8905 8906 MachineBasicBlock::iterator MII = Inst;8907 8908 const MCInstrDesc &InstDesc = get(Opcode);8909 const TargetRegisterClass *Src0RC = Src0.isReg() ?8910 MRI.getRegClass(Src0.getReg()) :8911 &AMDGPU::SGPR_32RegClass;8912 8913 const TargetRegisterClass *Src0SubRC =8914 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);8915 const TargetRegisterClass *Src1RC = Src1.isReg() ?8916 MRI.getRegClass(Src1.getReg()) :8917 &AMDGPU::SGPR_32RegClass;8918 8919 const TargetRegisterClass *Src1SubRC =8920 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);8921 8922 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,8923 AMDGPU::sub0, Src0SubRC);8924 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,8925 AMDGPU::sub0, Src1SubRC);8926 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,8927 AMDGPU::sub1, Src0SubRC);8928 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,8929 AMDGPU::sub1, Src1SubRC);8930 8931 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());8932 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);8933 const TargetRegisterClass *NewDestSubRC =8934 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0);8935 8936 Register DestSub0 = MRI.createVirtualRegister(NewDestSubRC);8937 MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0)8938 .add(SrcReg0Sub0)8939 .add(SrcReg1Sub0);8940 8941 Register DestSub1 = MRI.createVirtualRegister(NewDestSubRC);8942 MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1)8943 .add(SrcReg0Sub1)8944 .add(SrcReg1Sub1);8945 8946 Register FullDestReg = MRI.createVirtualRegister(NewDestRC);8947 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)8948 .addReg(DestSub0)8949 .addImm(AMDGPU::sub0)8950 .addReg(DestSub1)8951 .addImm(AMDGPU::sub1);8952 8953 MRI.replaceRegWith(Dest.getReg(), FullDestReg);8954 8955 Worklist.insert(&LoHalf);8956 Worklist.insert(&HiHalf);8957 8958 // Move all users of this moved value.8959 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);8960}8961 8962void SIInstrInfo::splitScalar64BitXnor(SIInstrWorklist &Worklist,8963 MachineInstr &Inst,8964 MachineDominatorTree *MDT) const {8965 MachineBasicBlock &MBB = *Inst.getParent();8966 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();8967 8968 MachineOperand &Dest = Inst.getOperand(0);8969 MachineOperand &Src0 = Inst.getOperand(1);8970 MachineOperand &Src1 = Inst.getOperand(2);8971 const DebugLoc &DL = Inst.getDebugLoc();8972 8973 MachineBasicBlock::iterator MII = Inst;8974 8975 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());8976 8977 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);8978 8979 MachineOperand* Op0;8980 MachineOperand* Op1;8981 8982 if (Src0.isReg() && RI.isSGPRReg(MRI, Src0.getReg())) {8983 Op0 = &Src0;8984 Op1 = &Src1;8985 } else {8986 Op0 = &Src1;8987 Op1 = &Src0;8988 }8989 8990 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B64), Interm)8991 .add(*Op0);8992 8993 Register NewDest = MRI.createVirtualRegister(DestRC);8994 8995 MachineInstr &Xor = *BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B64), NewDest)8996 .addReg(Interm)8997 .add(*Op1);8998 8999 MRI.replaceRegWith(Dest.getReg(), NewDest);9000 9001 Worklist.insert(&Xor);9002}9003 9004void SIInstrInfo::splitScalar64BitBCNT(SIInstrWorklist &Worklist,9005 MachineInstr &Inst) const {9006 MachineBasicBlock &MBB = *Inst.getParent();9007 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();9008 9009 MachineBasicBlock::iterator MII = Inst;9010 const DebugLoc &DL = Inst.getDebugLoc();9011 9012 MachineOperand &Dest = Inst.getOperand(0);9013 MachineOperand &Src = Inst.getOperand(1);9014 9015 const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64);9016 const TargetRegisterClass *SrcRC = Src.isReg() ?9017 MRI.getRegClass(Src.getReg()) :9018 &AMDGPU::SGPR_32RegClass;9019 9020 Register MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);9021 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);9022 9023 const TargetRegisterClass *SrcSubRC =9024 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);9025 9026 MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC,9027 AMDGPU::sub0, SrcSubRC);9028 MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC,9029 AMDGPU::sub1, SrcSubRC);9030 9031 BuildMI(MBB, MII, DL, InstDesc, MidReg).add(SrcRegSub0).addImm(0);9032 9033 BuildMI(MBB, MII, DL, InstDesc, ResultReg).add(SrcRegSub1).addReg(MidReg);9034 9035 MRI.replaceRegWith(Dest.getReg(), ResultReg);9036 9037 // We don't need to legalize operands here. src0 for either instruction can be9038 // an SGPR, and the second input is unused or determined here.9039 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);9040}9041 9042void SIInstrInfo::splitScalar64BitBFE(SIInstrWorklist &Worklist,9043 MachineInstr &Inst) const {9044 MachineBasicBlock &MBB = *Inst.getParent();9045 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();9046 MachineBasicBlock::iterator MII = Inst;9047 const DebugLoc &DL = Inst.getDebugLoc();9048 9049 MachineOperand &Dest = Inst.getOperand(0);9050 uint32_t Imm = Inst.getOperand(2).getImm();9051 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].9052 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].9053 9054 (void) Offset;9055 9056 // Only sext_inreg cases handled.9057 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 &&9058 Offset == 0 && "Not implemented");9059 9060 if (BitWidth < 32) {9061 Register MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);9062 Register MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);9063 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);9064 9065 BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32_e64), MidRegLo)9066 .addReg(Inst.getOperand(1).getReg(), 0, AMDGPU::sub0)9067 .addImm(0)9068 .addImm(BitWidth);9069 9070 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi)9071 .addImm(31)9072 .addReg(MidRegLo);9073 9074 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg)9075 .addReg(MidRegLo)9076 .addImm(AMDGPU::sub0)9077 .addReg(MidRegHi)9078 .addImm(AMDGPU::sub1);9079 9080 MRI.replaceRegWith(Dest.getReg(), ResultReg);9081 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);9082 return;9083 }9084 9085 MachineOperand &Src = Inst.getOperand(1);9086 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);9087 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);9088 9089 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e64), TmpReg)9090 .addImm(31)9091 .addReg(Src.getReg(), 0, AMDGPU::sub0);9092 9093 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg)9094 .addReg(Src.getReg(), 0, AMDGPU::sub0)9095 .addImm(AMDGPU::sub0)9096 .addReg(TmpReg)9097 .addImm(AMDGPU::sub1);9098 9099 MRI.replaceRegWith(Dest.getReg(), ResultReg);9100 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);9101}9102 9103void SIInstrInfo::splitScalar64BitCountOp(SIInstrWorklist &Worklist,9104 MachineInstr &Inst, unsigned Opcode,9105 MachineDominatorTree *MDT) const {9106 // (S_FLBIT_I32_B64 hi:lo) ->9107 // -> (umin (V_FFBH_U32_e32 hi), (uaddsat (V_FFBH_U32_e32 lo), 32))9108 // (S_FF1_I32_B64 hi:lo) ->9109 // ->(umin (uaddsat (V_FFBL_B32_e32 hi), 32) (V_FFBL_B32_e32 lo))9110 9111 MachineBasicBlock &MBB = *Inst.getParent();9112 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();9113 MachineBasicBlock::iterator MII = Inst;9114 const DebugLoc &DL = Inst.getDebugLoc();9115 9116 MachineOperand &Dest = Inst.getOperand(0);9117 MachineOperand &Src = Inst.getOperand(1);9118 9119 const MCInstrDesc &InstDesc = get(Opcode);9120 9121 bool IsCtlz = Opcode == AMDGPU::V_FFBH_U32_e32;9122 unsigned OpcodeAdd =9123 ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;9124 9125 const TargetRegisterClass *SrcRC =9126 Src.isReg() ? MRI.getRegClass(Src.getReg()) : &AMDGPU::SGPR_32RegClass;9127 const TargetRegisterClass *SrcSubRC =9128 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);9129 9130 MachineOperand SrcRegSub0 =9131 buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, AMDGPU::sub0, SrcSubRC);9132 MachineOperand SrcRegSub1 =9133 buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, AMDGPU::sub1, SrcSubRC);9134 9135 Register MidReg1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);9136 Register MidReg2 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);9137 Register MidReg3 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);9138 Register MidReg4 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);9139 9140 BuildMI(MBB, MII, DL, InstDesc, MidReg1).add(SrcRegSub0);9141 9142 BuildMI(MBB, MII, DL, InstDesc, MidReg2).add(SrcRegSub1);9143 9144 BuildMI(MBB, MII, DL, get(OpcodeAdd), MidReg3)9145 .addReg(IsCtlz ? MidReg1 : MidReg2)9146 .addImm(32)9147 .addImm(1); // enable clamp9148 9149 BuildMI(MBB, MII, DL, get(AMDGPU::V_MIN_U32_e64), MidReg4)9150 .addReg(MidReg3)9151 .addReg(IsCtlz ? MidReg2 : MidReg1);9152 9153 MRI.replaceRegWith(Dest.getReg(), MidReg4);9154 9155 addUsersToMoveToVALUWorklist(MidReg4, MRI, Worklist);9156}9157 9158void SIInstrInfo::addUsersToMoveToVALUWorklist(9159 Register DstReg, MachineRegisterInfo &MRI,9160 SIInstrWorklist &Worklist) const {9161 for (MachineOperand &MO : make_early_inc_range(MRI.use_operands(DstReg))) {9162 MachineInstr &UseMI = *MO.getParent();9163 9164 unsigned OpNo = 0;9165 9166 switch (UseMI.getOpcode()) {9167 case AMDGPU::COPY:9168 case AMDGPU::WQM:9169 case AMDGPU::SOFT_WQM:9170 case AMDGPU::STRICT_WWM:9171 case AMDGPU::STRICT_WQM:9172 case AMDGPU::REG_SEQUENCE:9173 case AMDGPU::PHI:9174 case AMDGPU::INSERT_SUBREG:9175 break;9176 default:9177 OpNo = MO.getOperandNo();9178 break;9179 }9180 9181 const TargetRegisterClass *OpRC = getOpRegClass(UseMI, OpNo);9182 MRI.constrainRegClass(DstReg, OpRC);9183 9184 if (!RI.hasVectorRegisters(OpRC))9185 Worklist.insert(&UseMI);9186 else9187 // Legalization could change user list.9188 legalizeOperandsVALUt16(UseMI, OpNo, MRI);9189 }9190}9191 9192void SIInstrInfo::movePackToVALU(SIInstrWorklist &Worklist,9193 MachineRegisterInfo &MRI,9194 MachineInstr &Inst) const {9195 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);9196 MachineBasicBlock *MBB = Inst.getParent();9197 MachineOperand &Src0 = Inst.getOperand(1);9198 MachineOperand &Src1 = Inst.getOperand(2);9199 const DebugLoc &DL = Inst.getDebugLoc();9200 9201 if (ST.useRealTrue16Insts()) {9202 Register SrcReg0, SrcReg1;9203 if (!Src0.isReg() || !RI.isVGPR(MRI, Src0.getReg())) {9204 SrcReg0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);9205 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), SrcReg0).add(Src0);9206 } else {9207 SrcReg0 = Src0.getReg();9208 }9209 9210 if (!Src1.isReg() || !RI.isVGPR(MRI, Src1.getReg())) {9211 SrcReg1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);9212 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), SrcReg1).add(Src1);9213 } else {9214 SrcReg1 = Src1.getReg();9215 }9216 9217 bool isSrc0Reg16 = MRI.constrainRegClass(SrcReg0, &AMDGPU::VGPR_16RegClass);9218 bool isSrc1Reg16 = MRI.constrainRegClass(SrcReg1, &AMDGPU::VGPR_16RegClass);9219 9220 auto NewMI = BuildMI(*MBB, Inst, DL, get(AMDGPU::REG_SEQUENCE), ResultReg);9221 switch (Inst.getOpcode()) {9222 case AMDGPU::S_PACK_LL_B32_B16:9223 NewMI9224 .addReg(SrcReg0, 0,9225 isSrc0Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)9226 .addImm(AMDGPU::lo16)9227 .addReg(SrcReg1, 0,9228 isSrc1Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)9229 .addImm(AMDGPU::hi16);9230 break;9231 case AMDGPU::S_PACK_LH_B32_B16:9232 NewMI9233 .addReg(SrcReg0, 0,9234 isSrc0Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)9235 .addImm(AMDGPU::lo16)9236 .addReg(SrcReg1, 0, AMDGPU::hi16)9237 .addImm(AMDGPU::hi16);9238 break;9239 case AMDGPU::S_PACK_HL_B32_B16:9240 NewMI.addReg(SrcReg0, 0, AMDGPU::hi16)9241 .addImm(AMDGPU::lo16)9242 .addReg(SrcReg1, 0,9243 isSrc1Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)9244 .addImm(AMDGPU::hi16);9245 break;9246 case AMDGPU::S_PACK_HH_B32_B16:9247 NewMI.addReg(SrcReg0, 0, AMDGPU::hi16)9248 .addImm(AMDGPU::lo16)9249 .addReg(SrcReg1, 0, AMDGPU::hi16)9250 .addImm(AMDGPU::hi16);9251 break;9252 default:9253 llvm_unreachable("unhandled s_pack_* instruction");9254 }9255 9256 MachineOperand &Dest = Inst.getOperand(0);9257 MRI.replaceRegWith(Dest.getReg(), ResultReg);9258 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);9259 return;9260 }9261 9262 switch (Inst.getOpcode()) {9263 case AMDGPU::S_PACK_LL_B32_B16: {9264 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);9265 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);9266 9267 // FIXME: Can do a lot better if we know the high bits of src0 or src1 are9268 // 0.9269 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)9270 .addImm(0xffff);9271 9272 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_B32_e64), TmpReg)9273 .addReg(ImmReg, RegState::Kill)9274 .add(Src0);9275 9276 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32_e64), ResultReg)9277 .add(Src1)9278 .addImm(16)9279 .addReg(TmpReg, RegState::Kill);9280 break;9281 }9282 case AMDGPU::S_PACK_LH_B32_B16: {9283 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);9284 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)9285 .addImm(0xffff);9286 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_BFI_B32_e64), ResultReg)9287 .addReg(ImmReg, RegState::Kill)9288 .add(Src0)9289 .add(Src1);9290 break;9291 }9292 case AMDGPU::S_PACK_HL_B32_B16: {9293 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);9294 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg)9295 .addImm(16)9296 .add(Src0);9297 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32_e64), ResultReg)9298 .add(Src1)9299 .addImm(16)9300 .addReg(TmpReg, RegState::Kill);9301 break;9302 }9303 case AMDGPU::S_PACK_HH_B32_B16: {9304 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);9305 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);9306 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg)9307 .addImm(16)9308 .add(Src0);9309 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)9310 .addImm(0xffff0000);9311 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_OR_B32_e64), ResultReg)9312 .add(Src1)9313 .addReg(ImmReg, RegState::Kill)9314 .addReg(TmpReg, RegState::Kill);9315 break;9316 }9317 default:9318 llvm_unreachable("unhandled s_pack_* instruction");9319 }9320 9321 MachineOperand &Dest = Inst.getOperand(0);9322 MRI.replaceRegWith(Dest.getReg(), ResultReg);9323 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);9324}9325 9326void SIInstrInfo::addSCCDefUsersToVALUWorklist(const MachineOperand &Op,9327 MachineInstr &SCCDefInst,9328 SIInstrWorklist &Worklist,9329 Register NewCond) const {9330 9331 // Ensure that def inst defines SCC, which is still live.9332 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() &&9333 !Op.isDead() && Op.getParent() == &SCCDefInst);9334 SmallVector<MachineInstr *, 4> CopyToDelete;9335 // This assumes that all the users of SCC are in the same block9336 // as the SCC def.9337 for (MachineInstr &MI : // Skip the def inst itself.9338 make_range(std::next(MachineBasicBlock::iterator(SCCDefInst)),9339 SCCDefInst.getParent()->end())) {9340 // Check if SCC is used first.9341 int SCCIdx = MI.findRegisterUseOperandIdx(AMDGPU::SCC, &RI, false);9342 if (SCCIdx != -1) {9343 if (MI.isCopy()) {9344 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();9345 Register DestReg = MI.getOperand(0).getReg();9346 9347 MRI.replaceRegWith(DestReg, NewCond);9348 CopyToDelete.push_back(&MI);9349 } else {9350 9351 if (NewCond.isValid())9352 MI.getOperand(SCCIdx).setReg(NewCond);9353 9354 Worklist.insert(&MI);9355 }9356 }9357 // Exit if we find another SCC def.9358 if (MI.findRegisterDefOperandIdx(AMDGPU::SCC, &RI, false, false) != -1)9359 break;9360 }9361 for (auto &Copy : CopyToDelete)9362 Copy->eraseFromParent();9363}9364 9365// Instructions that use SCC may be converted to VALU instructions. When that9366// happens, the SCC register is changed to VCC_LO. The instruction that defines9367// SCC must be changed to an instruction that defines VCC. This function makes9368// sure that the instruction that defines SCC is added to the moveToVALU9369// worklist.9370void SIInstrInfo::addSCCDefsToVALUWorklist(MachineInstr *SCCUseInst,9371 SIInstrWorklist &Worklist) const {9372 // Look for a preceding instruction that either defines VCC or SCC. If VCC9373 // then there is nothing to do because the defining instruction has been9374 // converted to a VALU already. If SCC then that instruction needs to be9375 // converted to a VALU.9376 for (MachineInstr &MI :9377 make_range(std::next(MachineBasicBlock::reverse_iterator(SCCUseInst)),9378 SCCUseInst->getParent()->rend())) {9379 if (MI.modifiesRegister(AMDGPU::VCC, &RI))9380 break;9381 if (MI.definesRegister(AMDGPU::SCC, &RI)) {9382 Worklist.insert(&MI);9383 break;9384 }9385 }9386}9387 9388const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass(9389 const MachineInstr &Inst) const {9390 const TargetRegisterClass *NewDstRC = getOpRegClass(Inst, 0);9391 9392 switch (Inst.getOpcode()) {9393 // For target instructions, getOpRegClass just returns the virtual register9394 // class associated with the operand, so we need to find an equivalent VGPR9395 // register class in order to move the instruction to the VALU.9396 case AMDGPU::COPY:9397 case AMDGPU::PHI:9398 case AMDGPU::REG_SEQUENCE:9399 case AMDGPU::INSERT_SUBREG:9400 case AMDGPU::WQM:9401 case AMDGPU::SOFT_WQM:9402 case AMDGPU::STRICT_WWM:9403 case AMDGPU::STRICT_WQM: {9404 const TargetRegisterClass *SrcRC = getOpRegClass(Inst, 1);9405 if (RI.isAGPRClass(SrcRC)) {9406 if (RI.isAGPRClass(NewDstRC))9407 return nullptr;9408 9409 switch (Inst.getOpcode()) {9410 case AMDGPU::PHI:9411 case AMDGPU::REG_SEQUENCE:9412 case AMDGPU::INSERT_SUBREG:9413 NewDstRC = RI.getEquivalentAGPRClass(NewDstRC);9414 break;9415 default:9416 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC);9417 }9418 9419 if (!NewDstRC)9420 return nullptr;9421 } else {9422 if (RI.isVGPRClass(NewDstRC) || NewDstRC == &AMDGPU::VReg_1RegClass)9423 return nullptr;9424 9425 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC);9426 if (!NewDstRC)9427 return nullptr;9428 }9429 9430 return NewDstRC;9431 }9432 default:9433 return NewDstRC;9434 }9435}9436 9437// Find the one SGPR operand we are allowed to use.9438Register SIInstrInfo::findUsedSGPR(const MachineInstr &MI,9439 int OpIndices[3]) const {9440 const MCInstrDesc &Desc = MI.getDesc();9441 9442 // Find the one SGPR operand we are allowed to use.9443 //9444 // First we need to consider the instruction's operand requirements before9445 // legalizing. Some operands are required to be SGPRs, such as implicit uses9446 // of VCC, but we are still bound by the constant bus requirement to only use9447 // one.9448 //9449 // If the operand's class is an SGPR, we can never move it.9450 9451 Register SGPRReg = findImplicitSGPRRead(MI);9452 if (SGPRReg)9453 return SGPRReg;9454 9455 Register UsedSGPRs[3] = {Register()};9456 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();9457 9458 for (unsigned i = 0; i < 3; ++i) {9459 int Idx = OpIndices[i];9460 if (Idx == -1)9461 break;9462 9463 const MachineOperand &MO = MI.getOperand(Idx);9464 if (!MO.isReg())9465 continue;9466 9467 // Is this operand statically required to be an SGPR based on the operand9468 // constraints?9469 const TargetRegisterClass *OpRC =9470 RI.getRegClass(getOpRegClassID(Desc.operands()[Idx]));9471 bool IsRequiredSGPR = RI.isSGPRClass(OpRC);9472 if (IsRequiredSGPR)9473 return MO.getReg();9474 9475 // If this could be a VGPR or an SGPR, Check the dynamic register class.9476 Register Reg = MO.getReg();9477 const TargetRegisterClass *RegRC = MRI.getRegClass(Reg);9478 if (RI.isSGPRClass(RegRC))9479 UsedSGPRs[i] = Reg;9480 }9481 9482 // We don't have a required SGPR operand, so we have a bit more freedom in9483 // selecting operands to move.9484 9485 // Try to select the most used SGPR. If an SGPR is equal to one of the9486 // others, we choose that.9487 //9488 // e.g.9489 // V_FMA_F32 v0, s0, s0, s0 -> No moves9490 // V_FMA_F32 v0, s0, s1, s0 -> Move s19491 9492 // TODO: If some of the operands are 64-bit SGPRs and some 32, we should9493 // prefer those.9494 9495 if (UsedSGPRs[0]) {9496 if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2])9497 SGPRReg = UsedSGPRs[0];9498 }9499 9500 if (!SGPRReg && UsedSGPRs[1]) {9501 if (UsedSGPRs[1] == UsedSGPRs[2])9502 SGPRReg = UsedSGPRs[1];9503 }9504 9505 return SGPRReg;9506}9507 9508MachineOperand *SIInstrInfo::getNamedOperand(MachineInstr &MI,9509 AMDGPU::OpName OperandName) const {9510 if (OperandName == AMDGPU::OpName::NUM_OPERAND_NAMES)9511 return nullptr;9512 9513 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName);9514 if (Idx == -1)9515 return nullptr;9516 9517 return &MI.getOperand(Idx);9518}9519 9520uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const {9521 if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {9522 int64_t Format = ST.getGeneration() >= AMDGPUSubtarget::GFX119523 ? (int64_t)AMDGPU::UfmtGFX11::UFMT_32_FLOAT9524 : (int64_t)AMDGPU::UfmtGFX10::UFMT_32_FLOAT;9525 return (Format << 44) |9526 (1ULL << 56) | // RESOURCE_LEVEL = 19527 (3ULL << 60); // OOB_SELECT = 39528 }9529 9530 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT;9531 if (ST.isAmdHsaOS()) {9532 // Set ATC = 1. GFX9 doesn't have this bit.9533 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS)9534 RsrcDataFormat |= (1ULL << 56);9535 9536 // Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this.9537 // BTW, it disables TC L2 and therefore decreases performance.9538 if (ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS)9539 RsrcDataFormat |= (2ULL << 59);9540 }9541 9542 return RsrcDataFormat;9543}9544 9545uint64_t SIInstrInfo::getScratchRsrcWords23() const {9546 uint64_t Rsrc23 = getDefaultRsrcDataFormat() |9547 AMDGPU::RSRC_TID_ENABLE |9548 0xffffffff; // Size;9549 9550 // GFX9 doesn't have ELEMENT_SIZE.9551 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) {9552 uint64_t EltSizeValue = Log2_32(ST.getMaxPrivateElementSize(true)) - 1;9553 Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT;9554 }9555 9556 // IndexStride = 64 / 32.9557 uint64_t IndexStride = ST.isWave64() ? 3 : 2;9558 Rsrc23 |= IndexStride << AMDGPU::RSRC_INDEX_STRIDE_SHIFT;9559 9560 // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17].9561 // Clear them unless we want a huge stride.9562 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&9563 ST.getGeneration() <= AMDGPUSubtarget::GFX9)9564 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT;9565 9566 return Rsrc23;9567}9568 9569bool SIInstrInfo::isLowLatencyInstruction(const MachineInstr &MI) const {9570 unsigned Opc = MI.getOpcode();9571 9572 return isSMRD(Opc);9573}9574 9575bool SIInstrInfo::isHighLatencyDef(int Opc) const {9576 return get(Opc).mayLoad() &&9577 (isMUBUF(Opc) || isMTBUF(Opc) || isMIMG(Opc) || isFLAT(Opc));9578}9579 9580Register SIInstrInfo::isStackAccess(const MachineInstr &MI,9581 int &FrameIndex) const {9582 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::vaddr);9583 if (!Addr || !Addr->isFI())9584 return Register();9585 9586 assert(!MI.memoperands_empty() &&9587 (*MI.memoperands_begin())->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS);9588 9589 FrameIndex = Addr->getIndex();9590 return getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg();9591}9592 9593Register SIInstrInfo::isSGPRStackAccess(const MachineInstr &MI,9594 int &FrameIndex) const {9595 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::addr);9596 assert(Addr && Addr->isFI());9597 FrameIndex = Addr->getIndex();9598 return getNamedOperand(MI, AMDGPU::OpName::data)->getReg();9599}9600 9601Register SIInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,9602 int &FrameIndex) const {9603 if (!MI.mayLoad())9604 return Register();9605 9606 if (isMUBUF(MI) || isVGPRSpill(MI))9607 return isStackAccess(MI, FrameIndex);9608 9609 if (isSGPRSpill(MI))9610 return isSGPRStackAccess(MI, FrameIndex);9611 9612 return Register();9613}9614 9615Register SIInstrInfo::isStoreToStackSlot(const MachineInstr &MI,9616 int &FrameIndex) const {9617 if (!MI.mayStore())9618 return Register();9619 9620 if (isMUBUF(MI) || isVGPRSpill(MI))9621 return isStackAccess(MI, FrameIndex);9622 9623 if (isSGPRSpill(MI))9624 return isSGPRStackAccess(MI, FrameIndex);9625 9626 return Register();9627}9628 9629unsigned SIInstrInfo::getInstBundleSize(const MachineInstr &MI) const {9630 unsigned Size = 0;9631 MachineBasicBlock::const_instr_iterator I = MI.getIterator();9632 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();9633 while (++I != E && I->isInsideBundle()) {9634 assert(!I->isBundle() && "No nested bundle!");9635 Size += getInstSizeInBytes(*I);9636 }9637 9638 return Size;9639}9640 9641unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {9642 unsigned Opc = MI.getOpcode();9643 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opc);9644 unsigned DescSize = Desc.getSize();9645 9646 // If we have a definitive size, we can use it. Otherwise we need to inspect9647 // the operands to know the size.9648 if (isFixedSize(MI)) {9649 unsigned Size = DescSize;9650 9651 // If we hit the buggy offset, an extra nop will be inserted in MC so9652 // estimate the worst case.9653 if (MI.isBranch() && ST.hasOffset3fBug())9654 Size += 4;9655 9656 return Size;9657 }9658 9659 // Instructions may have a 32-bit literal encoded after them. Check9660 // operands that could ever be literals.9661 if (isVALU(MI) || isSALU(MI)) {9662 if (isDPP(MI))9663 return DescSize;9664 bool HasLiteral = false;9665 unsigned LiteralSize = 4;9666 for (int I = 0, E = MI.getNumExplicitOperands(); I != E; ++I) {9667 const MachineOperand &Op = MI.getOperand(I);9668 const MCOperandInfo &OpInfo = Desc.operands()[I];9669 if (!Op.isReg() && !isInlineConstant(Op, OpInfo)) {9670 HasLiteral = true;9671 if (ST.has64BitLiterals()) {9672 switch (OpInfo.OperandType) {9673 default:9674 break;9675 case AMDGPU::OPERAND_REG_IMM_FP64:9676 if (!AMDGPU::isValid32BitLiteral(Op.getImm(), true))9677 LiteralSize = 8;9678 break;9679 case AMDGPU::OPERAND_REG_IMM_INT64:9680 if (!Op.isImm() || !AMDGPU::isValid32BitLiteral(Op.getImm(), false))9681 LiteralSize = 8;9682 break;9683 }9684 }9685 break;9686 }9687 }9688 return HasLiteral ? DescSize + LiteralSize : DescSize;9689 }9690 9691 // Check whether we have extra NSA words.9692 if (isMIMG(MI)) {9693 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0);9694 if (VAddr0Idx < 0)9695 return 8;9696 9697 int RSrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc);9698 return 8 + 4 * ((RSrcIdx - VAddr0Idx + 2) / 4);9699 }9700 9701 switch (Opc) {9702 case TargetOpcode::BUNDLE:9703 return getInstBundleSize(MI);9704 case TargetOpcode::INLINEASM:9705 case TargetOpcode::INLINEASM_BR: {9706 const MachineFunction *MF = MI.getMF();9707 const char *AsmStr = MI.getOperand(0).getSymbolName();9708 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo(), &ST);9709 }9710 default:9711 if (MI.isMetaInstruction())9712 return 0;9713 9714 // If D16 Pseudo inst, get correct MC code size9715 const auto *D16Info = AMDGPU::getT16D16Helper(Opc);9716 if (D16Info) {9717 // Assume d16_lo/hi inst are always in same size9718 unsigned LoInstOpcode = D16Info->LoOp;9719 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(LoInstOpcode);9720 DescSize = Desc.getSize();9721 }9722 9723 // If FMA Pseudo inst, get correct MC code size9724 if (Opc == AMDGPU::V_FMA_MIX_F16_t16 || Opc == AMDGPU::V_FMA_MIX_BF16_t16) {9725 // All potential lowerings are the same size; arbitrarily pick one.9726 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(AMDGPU::V_FMA_MIXLO_F16);9727 DescSize = Desc.getSize();9728 }9729 9730 return DescSize;9731 }9732}9733 9734bool SIInstrInfo::mayAccessFlatAddressSpace(const MachineInstr &MI) const {9735 if (!isFLAT(MI))9736 return false;9737 9738 if (MI.memoperands_empty())9739 return true;9740 9741 for (const MachineMemOperand *MMO : MI.memoperands()) {9742 if (MMO->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS)9743 return true;9744 }9745 return false;9746}9747 9748ArrayRef<std::pair<int, const char *>>9749SIInstrInfo::getSerializableTargetIndices() const {9750 static const std::pair<int, const char *> TargetIndices[] = {9751 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"},9752 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"},9753 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"},9754 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"},9755 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}};9756 return ArrayRef(TargetIndices);9757}9758 9759/// This is used by the post-RA scheduler (SchedulePostRAList.cpp). The9760/// post-RA version of misched uses CreateTargetMIHazardRecognizer.9761ScheduleHazardRecognizer *9762SIInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,9763 const ScheduleDAG *DAG) const {9764 return new GCNHazardRecognizer(DAG->MF);9765}9766 9767/// This is the hazard recognizer used at -O0 by the PostRAHazardRecognizer9768/// pass.9769ScheduleHazardRecognizer *9770SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const {9771 return new GCNHazardRecognizer(MF);9772}9773 9774// Called during:9775// - pre-RA scheduling and post-RA scheduling9776ScheduleHazardRecognizer *9777SIInstrInfo::CreateTargetMIHazardRecognizer(const InstrItineraryData *II,9778 const ScheduleDAGMI *DAG) const {9779 // Borrowed from Arm Target9780 // We would like to restrict this hazard recognizer to only9781 // post-RA scheduling; we can tell that we're post-RA because we don't9782 // track VRegLiveness.9783 if (!DAG->hasVRegLiveness())9784 return new GCNHazardRecognizer(DAG->MF);9785 return TargetInstrInfo::CreateTargetMIHazardRecognizer(II, DAG);9786}9787 9788std::pair<unsigned, unsigned>9789SIInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {9790 return std::pair(TF & MO_MASK, TF & ~MO_MASK);9791}9792 9793ArrayRef<std::pair<unsigned, const char *>>9794SIInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {9795 static const std::pair<unsigned, const char *> TargetFlags[] = {9796 {MO_GOTPCREL, "amdgpu-gotprel"},9797 {MO_GOTPCREL32_LO, "amdgpu-gotprel32-lo"},9798 {MO_GOTPCREL32_HI, "amdgpu-gotprel32-hi"},9799 {MO_GOTPCREL64, "amdgpu-gotprel64"},9800 {MO_REL32_LO, "amdgpu-rel32-lo"},9801 {MO_REL32_HI, "amdgpu-rel32-hi"},9802 {MO_REL64, "amdgpu-rel64"},9803 {MO_ABS32_LO, "amdgpu-abs32-lo"},9804 {MO_ABS32_HI, "amdgpu-abs32-hi"},9805 {MO_ABS64, "amdgpu-abs64"},9806 };9807 9808 return ArrayRef(TargetFlags);9809}9810 9811ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>9812SIInstrInfo::getSerializableMachineMemOperandTargetFlags() const {9813 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =9814 {9815 {MONoClobber, "amdgpu-noclobber"},9816 {MOLastUse, "amdgpu-last-use"},9817 {MOCooperative, "amdgpu-cooperative"},9818 };9819 9820 return ArrayRef(TargetFlags);9821}9822 9823unsigned SIInstrInfo::getLiveRangeSplitOpcode(Register SrcReg,9824 const MachineFunction &MF) const {9825 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();9826 assert(SrcReg.isVirtual());9827 if (MFI->checkFlag(SrcReg, AMDGPU::VirtRegFlag::WWM_REG))9828 return AMDGPU::WWM_COPY;9829 9830 return AMDGPU::COPY;9831}9832 9833bool SIInstrInfo::isBasicBlockPrologue(const MachineInstr &MI,9834 Register Reg) const {9835 // We need to handle instructions which may be inserted during register9836 // allocation to handle the prolog. The initial prolog instruction may have9837 // been separated from the start of the block by spills and copies inserted9838 // needed by the prolog. However, the insertions for scalar registers can9839 // always be placed at the BB top as they are independent of the exec mask9840 // value.9841 const MachineFunction *MF = MI.getMF();9842 bool IsNullOrVectorRegister = true;9843 if (Reg) {9844 const MachineRegisterInfo &MRI = MF->getRegInfo();9845 IsNullOrVectorRegister = !RI.isSGPRClass(RI.getRegClassForReg(MRI, Reg));9846 }9847 9848 uint16_t Opcode = MI.getOpcode();9849 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();9850 return IsNullOrVectorRegister &&9851 (isSGPRSpill(Opcode) || isWWMRegSpillOpcode(Opcode) ||9852 (Opcode == AMDGPU::IMPLICIT_DEF &&9853 MFI->isWWMReg(MI.getOperand(0).getReg())) ||9854 (!MI.isTerminator() && Opcode != AMDGPU::COPY &&9855 MI.modifiesRegister(AMDGPU::EXEC, &RI)));9856}9857 9858MachineInstrBuilder9859SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,9860 MachineBasicBlock::iterator I,9861 const DebugLoc &DL,9862 Register DestReg) const {9863 if (ST.hasAddNoCarry())9864 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e64), DestReg);9865 9866 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();9867 Register UnusedCarry = MRI.createVirtualRegister(RI.getBoolRC());9868 MRI.setRegAllocationHint(UnusedCarry, 0, RI.getVCC());9869 9870 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg)9871 .addReg(UnusedCarry, RegState::Define | RegState::Dead);9872}9873 9874MachineInstrBuilder SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,9875 MachineBasicBlock::iterator I,9876 const DebugLoc &DL,9877 Register DestReg,9878 RegScavenger &RS) const {9879 if (ST.hasAddNoCarry())9880 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e32), DestReg);9881 9882 // If available, prefer to use vcc.9883 Register UnusedCarry = !RS.isRegUsed(AMDGPU::VCC)9884 ? Register(RI.getVCC())9885 : RS.scavengeRegisterBackwards(9886 *RI.getBoolRC(), I, /* RestoreAfter */ false,9887 0, /* AllowSpill */ false);9888 9889 // TODO: Users need to deal with this.9890 if (!UnusedCarry.isValid())9891 return MachineInstrBuilder();9892 9893 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg)9894 .addReg(UnusedCarry, RegState::Define | RegState::Dead);9895}9896 9897bool SIInstrInfo::isKillTerminator(unsigned Opcode) {9898 switch (Opcode) {9899 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:9900 case AMDGPU::SI_KILL_I1_TERMINATOR:9901 return true;9902 default:9903 return false;9904 }9905}9906 9907const MCInstrDesc &SIInstrInfo::getKillTerminatorFromPseudo(unsigned Opcode) const {9908 switch (Opcode) {9909 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:9910 return get(AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR);9911 case AMDGPU::SI_KILL_I1_PSEUDO:9912 return get(AMDGPU::SI_KILL_I1_TERMINATOR);9913 default:9914 llvm_unreachable("invalid opcode, expected SI_KILL_*_PSEUDO");9915 }9916}9917 9918bool SIInstrInfo::isLegalMUBUFImmOffset(unsigned Imm) const {9919 return Imm <= getMaxMUBUFImmOffset(ST);9920}9921 9922unsigned SIInstrInfo::getMaxMUBUFImmOffset(const GCNSubtarget &ST) {9923 // GFX12 field is non-negative 24-bit signed byte offset.9924 const unsigned OffsetBits =9925 ST.getGeneration() >= AMDGPUSubtarget::GFX12 ? 23 : 12;9926 return (1 << OffsetBits) - 1;9927}9928 9929void SIInstrInfo::fixImplicitOperands(MachineInstr &MI) const {9930 if (!ST.isWave32())9931 return;9932 9933 if (MI.isInlineAsm())9934 return;9935 9936 for (auto &Op : MI.implicit_operands()) {9937 if (Op.isReg() && Op.getReg() == AMDGPU::VCC)9938 Op.setReg(AMDGPU::VCC_LO);9939 }9940}9941 9942bool SIInstrInfo::isBufferSMRD(const MachineInstr &MI) const {9943 if (!isSMRD(MI))9944 return false;9945 9946 // Check that it is using a buffer resource.9947 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::sbase);9948 if (Idx == -1) // e.g. s_memtime9949 return false;9950 9951 const int16_t RCID = getOpRegClassID(MI.getDesc().operands()[Idx]);9952 return RI.getRegClass(RCID)->hasSubClassEq(&AMDGPU::SGPR_128RegClass);9953}9954 9955// Given Imm, split it into the values to put into the SOffset and ImmOffset9956// fields in an MUBUF instruction. Return false if it is not possible (due to a9957// hardware bug needing a workaround).9958//9959// The required alignment ensures that individual address components remain9960// aligned if they are aligned to begin with. It also ensures that additional9961// offsets within the given alignment can be added to the resulting ImmOffset.9962bool SIInstrInfo::splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset,9963 uint32_t &ImmOffset, Align Alignment) const {9964 const uint32_t MaxOffset = SIInstrInfo::getMaxMUBUFImmOffset(ST);9965 const uint32_t MaxImm = alignDown(MaxOffset, Alignment.value());9966 uint32_t Overflow = 0;9967 9968 if (Imm > MaxImm) {9969 if (Imm <= MaxImm + 64) {9970 // Use an SOffset inline constant for 4..649971 Overflow = Imm - MaxImm;9972 Imm = MaxImm;9973 } else {9974 // Try to keep the same value in SOffset for adjacent loads, so that9975 // the corresponding register contents can be re-used.9976 //9977 // Load values with all low-bits (except for alignment bits) set into9978 // SOffset, so that a larger range of values can be covered using9979 // s_movk_i32.9980 //9981 // Atomic operations fail to work correctly when individual address9982 // components are unaligned, even if their sum is aligned.9983 uint32_t High = (Imm + Alignment.value()) & ~MaxOffset;9984 uint32_t Low = (Imm + Alignment.value()) & MaxOffset;9985 Imm = Low;9986 Overflow = High - Alignment.value();9987 }9988 }9989 9990 if (Overflow > 0) {9991 // There is a hardware bug in SI and CI which prevents address clamping in9992 // MUBUF instructions from working correctly with SOffsets. The immediate9993 // offset is unaffected.9994 if (ST.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS)9995 return false;9996 9997 // It is not possible to set immediate in SOffset field on some targets.9998 if (ST.hasRestrictedSOffset())9999 return false;10000 }10001 10002 ImmOffset = Imm;10003 SOffset = Overflow;10004 return true;10005}10006 10007// Depending on the used address space and instructions, some immediate offsets10008// are allowed and some are not.10009// Pre-GFX12, flat instruction offsets can only be non-negative, global and10010// scratch instruction offsets can also be negative. On GFX12, offsets can be10011// negative for all variants.10012//10013// There are several bugs related to these offsets:10014// On gfx10.1, flat instructions that go into the global address space cannot10015// use an offset.10016//10017// For scratch instructions, the address can be either an SGPR or a VGPR.10018// The following offsets can be used, depending on the architecture (x means10019// cannot be used):10020// +----------------------------+------+------+10021// | Address-Mode | SGPR | VGPR |10022// +----------------------------+------+------+10023// | gfx9 | | |10024// | negative, 4-aligned offset | x | ok |10025// | negative, unaligned offset | x | ok |10026// +----------------------------+------+------+10027// | gfx10 | | |10028// | negative, 4-aligned offset | ok | ok |10029// | negative, unaligned offset | ok | x |10030// +----------------------------+------+------+10031// | gfx10.3 | | |10032// | negative, 4-aligned offset | ok | ok |10033// | negative, unaligned offset | ok | ok |10034// +----------------------------+------+------+10035//10036// This function ignores the addressing mode, so if an offset cannot be used in10037// one addressing mode, it is considered illegal.10038bool SIInstrInfo::isLegalFLATOffset(int64_t Offset, unsigned AddrSpace,10039 uint64_t FlatVariant) const {10040 // TODO: Should 0 be special cased?10041 if (!ST.hasFlatInstOffsets())10042 return false;10043 10044 if (ST.hasFlatSegmentOffsetBug() && FlatVariant == SIInstrFlags::FLAT &&10045 (AddrSpace == AMDGPUAS::FLAT_ADDRESS ||10046 AddrSpace == AMDGPUAS::GLOBAL_ADDRESS))10047 return false;10048 10049 if (ST.hasNegativeUnalignedScratchOffsetBug() &&10050 FlatVariant == SIInstrFlags::FlatScratch && Offset < 0 &&10051 (Offset % 4) != 0) {10052 return false;10053 }10054 10055 bool AllowNegative = allowNegativeFlatOffset(FlatVariant);10056 unsigned N = AMDGPU::getNumFlatOffsetBits(ST);10057 return isIntN(N, Offset) && (AllowNegative || Offset >= 0);10058}10059 10060// See comment on SIInstrInfo::isLegalFLATOffset for what is legal and what not.10061std::pair<int64_t, int64_t>10062SIInstrInfo::splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace,10063 uint64_t FlatVariant) const {10064 int64_t RemainderOffset = COffsetVal;10065 int64_t ImmField = 0;10066 10067 bool AllowNegative = allowNegativeFlatOffset(FlatVariant);10068 const unsigned NumBits = AMDGPU::getNumFlatOffsetBits(ST) - 1;10069 10070 if (AllowNegative) {10071 // Use signed division by a power of two to truncate towards 0.10072 int64_t D = 1LL << NumBits;10073 RemainderOffset = (COffsetVal / D) * D;10074 ImmField = COffsetVal - RemainderOffset;10075 10076 if (ST.hasNegativeUnalignedScratchOffsetBug() &&10077 FlatVariant == SIInstrFlags::FlatScratch && ImmField < 0 &&10078 (ImmField % 4) != 0) {10079 // Make ImmField a multiple of 410080 RemainderOffset += ImmField % 4;10081 ImmField -= ImmField % 4;10082 }10083 } else if (COffsetVal >= 0) {10084 ImmField = COffsetVal & maskTrailingOnes<uint64_t>(NumBits);10085 RemainderOffset = COffsetVal - ImmField;10086 }10087 10088 assert(isLegalFLATOffset(ImmField, AddrSpace, FlatVariant));10089 assert(RemainderOffset + ImmField == COffsetVal);10090 return {ImmField, RemainderOffset};10091}10092 10093bool SIInstrInfo::allowNegativeFlatOffset(uint64_t FlatVariant) const {10094 if (ST.hasNegativeScratchOffsetBug() &&10095 FlatVariant == SIInstrFlags::FlatScratch)10096 return false;10097 10098 return FlatVariant != SIInstrFlags::FLAT || AMDGPU::isGFX12Plus(ST);10099}10100 10101static unsigned subtargetEncodingFamily(const GCNSubtarget &ST) {10102 switch (ST.getGeneration()) {10103 default:10104 break;10105 case AMDGPUSubtarget::SOUTHERN_ISLANDS:10106 case AMDGPUSubtarget::SEA_ISLANDS:10107 return SIEncodingFamily::SI;10108 case AMDGPUSubtarget::VOLCANIC_ISLANDS:10109 case AMDGPUSubtarget::GFX9:10110 return SIEncodingFamily::VI;10111 case AMDGPUSubtarget::GFX10:10112 return SIEncodingFamily::GFX10;10113 case AMDGPUSubtarget::GFX11:10114 return SIEncodingFamily::GFX11;10115 case AMDGPUSubtarget::GFX12:10116 return ST.hasGFX1250Insts() ? SIEncodingFamily::GFX125010117 : SIEncodingFamily::GFX12;10118 }10119 llvm_unreachable("Unknown subtarget generation!");10120}10121 10122bool SIInstrInfo::isAsmOnlyOpcode(int MCOp) const {10123 switch(MCOp) {10124 // These opcodes use indirect register addressing so10125 // they need special handling by codegen (currently missing).10126 // Therefore it is too risky to allow these opcodes10127 // to be selected by dpp combiner or sdwa peepholer.10128 case AMDGPU::V_MOVRELS_B32_dpp_gfx10:10129 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:10130 case AMDGPU::V_MOVRELD_B32_dpp_gfx10:10131 case AMDGPU::V_MOVRELD_B32_sdwa_gfx10:10132 case AMDGPU::V_MOVRELSD_B32_dpp_gfx10:10133 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:10134 case AMDGPU::V_MOVRELSD_2_B32_dpp_gfx10:10135 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:10136 return true;10137 default:10138 return false;10139 }10140}10141 10142#define GENERATE_RENAMED_GFX9_CASES(OPCODE) \10143 case OPCODE##_dpp: \10144 case OPCODE##_e32: \10145 case OPCODE##_e64: \10146 case OPCODE##_e64_dpp: \10147 case OPCODE##_sdwa:10148 10149static bool isRenamedInGFX9(int Opcode) {10150 switch (Opcode) {10151 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADDC_U32)10152 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADD_CO_U32)10153 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADD_U32)10154 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBBREV_U32)10155 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBB_U32)10156 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBREV_CO_U32)10157 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBREV_U32)10158 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUB_CO_U32)10159 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUB_U32)10160 //10161 case AMDGPU::V_DIV_FIXUP_F16_gfx9_e64:10162 case AMDGPU::V_DIV_FIXUP_F16_gfx9_fake16_e64:10163 case AMDGPU::V_FMA_F16_gfx9_e64:10164 case AMDGPU::V_FMA_F16_gfx9_fake16_e64:10165 case AMDGPU::V_INTERP_P2_F16:10166 case AMDGPU::V_MAD_F16_e64:10167 case AMDGPU::V_MAD_U16_e64:10168 case AMDGPU::V_MAD_I16_e64:10169 return true;10170 default:10171 return false;10172 }10173}10174 10175int SIInstrInfo::pseudoToMCOpcode(int Opcode) const {10176 Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode);10177 10178 unsigned Gen = subtargetEncodingFamily(ST);10179 10180 if (ST.getGeneration() == AMDGPUSubtarget::GFX9 && isRenamedInGFX9(Opcode))10181 Gen = SIEncodingFamily::GFX9;10182 10183 // Adjust the encoding family to GFX80 for D16 buffer instructions when the10184 // subtarget has UnpackedD16VMem feature.10185 // TODO: remove this when we discard GFX80 encoding.10186 if (ST.hasUnpackedD16VMem() && (get(Opcode).TSFlags & SIInstrFlags::D16Buf))10187 Gen = SIEncodingFamily::GFX80;10188 10189 if (get(Opcode).TSFlags & SIInstrFlags::SDWA) {10190 switch (ST.getGeneration()) {10191 default:10192 Gen = SIEncodingFamily::SDWA;10193 break;10194 case AMDGPUSubtarget::GFX9:10195 Gen = SIEncodingFamily::SDWA9;10196 break;10197 case AMDGPUSubtarget::GFX10:10198 Gen = SIEncodingFamily::SDWA10;10199 break;10200 }10201 }10202 10203 if (isMAI(Opcode)) {10204 int MFMAOp = AMDGPU::getMFMAEarlyClobberOp(Opcode);10205 if (MFMAOp != -1)10206 Opcode = MFMAOp;10207 }10208 10209 int MCOp = AMDGPU::getMCOpcode(Opcode, Gen);10210 10211 if (MCOp == (uint16_t)-1 && ST.hasGFX1250Insts())10212 MCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX12);10213 10214 // -1 means that Opcode is already a native instruction.10215 if (MCOp == -1)10216 return Opcode;10217 10218 if (ST.hasGFX90AInsts()) {10219 uint16_t NMCOp = (uint16_t)-1;10220 if (ST.hasGFX940Insts())10221 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX940);10222 if (NMCOp == (uint16_t)-1)10223 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX90A);10224 if (NMCOp == (uint16_t)-1)10225 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX9);10226 if (NMCOp != (uint16_t)-1)10227 MCOp = NMCOp;10228 }10229 10230 // (uint16_t)-1 means that Opcode is a pseudo instruction that has10231 // no encoding in the given subtarget generation.10232 if (MCOp == (uint16_t)-1)10233 return -1;10234 10235 if (isAsmOnlyOpcode(MCOp))10236 return -1;10237 10238 return MCOp;10239}10240 10241static10242TargetInstrInfo::RegSubRegPair getRegOrUndef(const MachineOperand &RegOpnd) {10243 assert(RegOpnd.isReg());10244 return RegOpnd.isUndef() ? TargetInstrInfo::RegSubRegPair() :10245 getRegSubRegPair(RegOpnd);10246}10247 10248TargetInstrInfo::RegSubRegPair10249llvm::getRegSequenceSubReg(MachineInstr &MI, unsigned SubReg) {10250 assert(MI.isRegSequence());10251 for (unsigned I = 0, E = (MI.getNumOperands() - 1)/ 2; I < E; ++I)10252 if (MI.getOperand(1 + 2 * I + 1).getImm() == SubReg) {10253 auto &RegOp = MI.getOperand(1 + 2 * I);10254 return getRegOrUndef(RegOp);10255 }10256 return TargetInstrInfo::RegSubRegPair();10257}10258 10259// Try to find the definition of reg:subreg in subreg-manipulation pseudos10260// Following a subreg of reg:subreg isn't supported10261static bool followSubRegDef(MachineInstr &MI,10262 TargetInstrInfo::RegSubRegPair &RSR) {10263 if (!RSR.SubReg)10264 return false;10265 switch (MI.getOpcode()) {10266 default: break;10267 case AMDGPU::REG_SEQUENCE:10268 RSR = getRegSequenceSubReg(MI, RSR.SubReg);10269 return true;10270 // EXTRACT_SUBREG ins't supported as this would follow a subreg of subreg10271 case AMDGPU::INSERT_SUBREG:10272 if (RSR.SubReg == (unsigned)MI.getOperand(3).getImm())10273 // inserted the subreg we're looking for10274 RSR = getRegOrUndef(MI.getOperand(2));10275 else { // the subreg in the rest of the reg10276 auto R1 = getRegOrUndef(MI.getOperand(1));10277 if (R1.SubReg) // subreg of subreg isn't supported10278 return false;10279 RSR.Reg = R1.Reg;10280 }10281 return true;10282 }10283 return false;10284}10285 10286MachineInstr *llvm::getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,10287 const MachineRegisterInfo &MRI) {10288 assert(MRI.isSSA());10289 if (!P.Reg.isVirtual())10290 return nullptr;10291 10292 auto RSR = P;10293 auto *DefInst = MRI.getVRegDef(RSR.Reg);10294 while (auto *MI = DefInst) {10295 DefInst = nullptr;10296 switch (MI->getOpcode()) {10297 case AMDGPU::COPY:10298 case AMDGPU::V_MOV_B32_e32: {10299 auto &Op1 = MI->getOperand(1);10300 if (Op1.isReg() && Op1.getReg().isVirtual()) {10301 if (Op1.isUndef())10302 return nullptr;10303 RSR = getRegSubRegPair(Op1);10304 DefInst = MRI.getVRegDef(RSR.Reg);10305 }10306 break;10307 }10308 default:10309 if (followSubRegDef(*MI, RSR)) {10310 if (!RSR.Reg)10311 return nullptr;10312 DefInst = MRI.getVRegDef(RSR.Reg);10313 }10314 }10315 if (!DefInst)10316 return MI;10317 }10318 return nullptr;10319}10320 10321bool llvm::execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI,10322 Register VReg,10323 const MachineInstr &DefMI,10324 const MachineInstr &UseMI) {10325 assert(MRI.isSSA() && "Must be run on SSA");10326 10327 auto *TRI = MRI.getTargetRegisterInfo();10328 auto *DefBB = DefMI.getParent();10329 10330 // Don't bother searching between blocks, although it is possible this block10331 // doesn't modify exec.10332 if (UseMI.getParent() != DefBB)10333 return true;10334 10335 const int MaxInstScan = 20;10336 int NumInst = 0;10337 10338 // Stop scan at the use.10339 auto E = UseMI.getIterator();10340 for (auto I = std::next(DefMI.getIterator()); I != E; ++I) {10341 if (I->isDebugInstr())10342 continue;10343 10344 if (++NumInst > MaxInstScan)10345 return true;10346 10347 if (I->modifiesRegister(AMDGPU::EXEC, TRI))10348 return true;10349 }10350 10351 return false;10352}10353 10354bool llvm::execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI,10355 Register VReg,10356 const MachineInstr &DefMI) {10357 assert(MRI.isSSA() && "Must be run on SSA");10358 10359 auto *TRI = MRI.getTargetRegisterInfo();10360 auto *DefBB = DefMI.getParent();10361 10362 const int MaxUseScan = 10;10363 int NumUse = 0;10364 10365 for (auto &Use : MRI.use_nodbg_operands(VReg)) {10366 auto &UseInst = *Use.getParent();10367 // Don't bother searching between blocks, although it is possible this block10368 // doesn't modify exec.10369 if (UseInst.getParent() != DefBB || UseInst.isPHI())10370 return true;10371 10372 if (++NumUse > MaxUseScan)10373 return true;10374 }10375 10376 if (NumUse == 0)10377 return false;10378 10379 const int MaxInstScan = 20;10380 int NumInst = 0;10381 10382 // Stop scan when we have seen all the uses.10383 for (auto I = std::next(DefMI.getIterator()); ; ++I) {10384 assert(I != DefBB->end());10385 10386 if (I->isDebugInstr())10387 continue;10388 10389 if (++NumInst > MaxInstScan)10390 return true;10391 10392 for (const MachineOperand &Op : I->operands()) {10393 // We don't check reg masks here as they're used only on calls:10394 // 1. EXEC is only considered const within one BB10395 // 2. Call should be a terminator instruction if present in a BB10396 10397 if (!Op.isReg())10398 continue;10399 10400 Register Reg = Op.getReg();10401 if (Op.isUse()) {10402 if (Reg == VReg && --NumUse == 0)10403 return false;10404 } else if (TRI->regsOverlap(Reg, AMDGPU::EXEC))10405 return true;10406 }10407 }10408}10409 10410MachineInstr *SIInstrInfo::createPHIDestinationCopy(10411 MachineBasicBlock &MBB, MachineBasicBlock::iterator LastPHIIt,10412 const DebugLoc &DL, Register Src, Register Dst) const {10413 auto Cur = MBB.begin();10414 if (Cur != MBB.end())10415 do {10416 if (!Cur->isPHI() && Cur->readsRegister(Dst, /*TRI=*/nullptr))10417 return BuildMI(MBB, Cur, DL, get(TargetOpcode::COPY), Dst).addReg(Src);10418 ++Cur;10419 } while (Cur != MBB.end() && Cur != LastPHIIt);10420 10421 return TargetInstrInfo::createPHIDestinationCopy(MBB, LastPHIIt, DL, Src,10422 Dst);10423}10424 10425MachineInstr *SIInstrInfo::createPHISourceCopy(10426 MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt,10427 const DebugLoc &DL, Register Src, unsigned SrcSubReg, Register Dst) const {10428 if (InsPt != MBB.end() &&10429 (InsPt->getOpcode() == AMDGPU::SI_IF ||10430 InsPt->getOpcode() == AMDGPU::SI_ELSE ||10431 InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) &&10432 InsPt->definesRegister(Src, /*TRI=*/nullptr)) {10433 InsPt++;10434 return BuildMI(MBB, InsPt, DL,10435 get(AMDGPU::LaneMaskConstants::get(ST).MovTermOpc), Dst)10436 .addReg(Src, 0, SrcSubReg)10437 .addReg(AMDGPU::EXEC, RegState::Implicit);10438 }10439 return TargetInstrInfo::createPHISourceCopy(MBB, InsPt, DL, Src, SrcSubReg,10440 Dst);10441}10442 10443bool llvm::SIInstrInfo::isWave32() const { return ST.isWave32(); }10444 10445MachineInstr *SIInstrInfo::foldMemoryOperandImpl(10446 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,10447 MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS,10448 VirtRegMap *VRM) const {10449 // This is a bit of a hack (copied from AArch64). Consider this instruction:10450 //10451 // %0:sreg_32 = COPY $m010452 //10453 // We explicitly chose SReg_32 for the virtual register so such a copy might10454 // be eliminated by RegisterCoalescer. However, that may not be possible, and10455 // %0 may even spill. We can't spill $m0 normally (it would require copying to10456 // a numbered SGPR anyway), and since it is in the SReg_32 register class,10457 // TargetInstrInfo::foldMemoryOperand() is going to try.10458 // A similar issue also exists with spilling and reloading $exec registers.10459 //10460 // To prevent that, constrain the %0 register class here.10461 if (isFullCopyInstr(MI)) {10462 Register DstReg = MI.getOperand(0).getReg();10463 Register SrcReg = MI.getOperand(1).getReg();10464 if ((DstReg.isVirtual() || SrcReg.isVirtual()) &&10465 (DstReg.isVirtual() != SrcReg.isVirtual())) {10466 MachineRegisterInfo &MRI = MF.getRegInfo();10467 Register VirtReg = DstReg.isVirtual() ? DstReg : SrcReg;10468 const TargetRegisterClass *RC = MRI.getRegClass(VirtReg);10469 if (RC->hasSuperClassEq(&AMDGPU::SReg_32RegClass)) {10470 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_32_XM0_XEXECRegClass);10471 return nullptr;10472 }10473 if (RC->hasSuperClassEq(&AMDGPU::SReg_64RegClass)) {10474 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_64_XEXECRegClass);10475 return nullptr;10476 }10477 }10478 }10479 10480 return nullptr;10481}10482 10483unsigned SIInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,10484 const MachineInstr &MI,10485 unsigned *PredCost) const {10486 if (MI.isBundle()) {10487 MachineBasicBlock::const_instr_iterator I(MI.getIterator());10488 MachineBasicBlock::const_instr_iterator E(MI.getParent()->instr_end());10489 unsigned Lat = 0, Count = 0;10490 for (++I; I != E && I->isBundledWithPred(); ++I) {10491 ++Count;10492 Lat = std::max(Lat, SchedModel.computeInstrLatency(&*I));10493 }10494 return Lat + Count - 1;10495 }10496 10497 return SchedModel.computeInstrLatency(&MI);10498}10499 10500InstructionUniformity10501SIInstrInfo::getGenericInstructionUniformity(const MachineInstr &MI) const {10502 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();10503 unsigned Opcode = MI.getOpcode();10504 10505 auto HandleAddrSpaceCast = [this, &MRI](const MachineInstr &MI) {10506 Register Dst = MI.getOperand(0).getReg();10507 Register Src = isa<GIntrinsic>(MI) ? MI.getOperand(2).getReg()10508 : MI.getOperand(1).getReg();10509 LLT DstTy = MRI.getType(Dst);10510 LLT SrcTy = MRI.getType(Src);10511 unsigned DstAS = DstTy.getAddressSpace();10512 unsigned SrcAS = SrcTy.getAddressSpace();10513 return SrcAS == AMDGPUAS::PRIVATE_ADDRESS &&10514 DstAS == AMDGPUAS::FLAT_ADDRESS &&10515 ST.hasGloballyAddressableScratch()10516 ? InstructionUniformity::NeverUniform10517 : InstructionUniformity::Default;10518 };10519 10520 // If the target supports globally addressable scratch, the mapping from10521 // scratch memory to the flat aperture changes therefore an address space cast10522 // is no longer uniform.10523 if (Opcode == TargetOpcode::G_ADDRSPACE_CAST)10524 return HandleAddrSpaceCast(MI);10525 10526 if (auto *GI = dyn_cast<GIntrinsic>(&MI)) {10527 auto IID = GI->getIntrinsicID();10528 if (AMDGPU::isIntrinsicSourceOfDivergence(IID))10529 return InstructionUniformity::NeverUniform;10530 if (AMDGPU::isIntrinsicAlwaysUniform(IID))10531 return InstructionUniformity::AlwaysUniform;10532 10533 switch (IID) {10534 case Intrinsic::amdgcn_addrspacecast_nonnull:10535 return HandleAddrSpaceCast(MI);10536 case Intrinsic::amdgcn_if:10537 case Intrinsic::amdgcn_else:10538 // FIXME: Uniform if second result10539 break;10540 }10541 10542 return InstructionUniformity::Default;10543 }10544 10545 // Loads from the private and flat address spaces are divergent, because10546 // threads can execute the load instruction with the same inputs and get10547 // different results.10548 //10549 // All other loads are not divergent, because if threads issue loads with the10550 // same arguments, they will always get the same result.10551 if (Opcode == AMDGPU::G_LOAD || Opcode == AMDGPU::G_ZEXTLOAD ||10552 Opcode == AMDGPU::G_SEXTLOAD) {10553 if (MI.memoperands_empty())10554 return InstructionUniformity::NeverUniform; // conservative assumption10555 10556 if (llvm::any_of(MI.memoperands(), [](const MachineMemOperand *mmo) {10557 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS ||10558 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS;10559 })) {10560 // At least one MMO in a non-global address space.10561 return InstructionUniformity::NeverUniform;10562 }10563 return InstructionUniformity::Default;10564 }10565 10566 if (SIInstrInfo::isGenericAtomicRMWOpcode(Opcode) ||10567 Opcode == AMDGPU::G_ATOMIC_CMPXCHG ||10568 Opcode == AMDGPU::G_ATOMIC_CMPXCHG_WITH_SUCCESS ||10569 AMDGPU::isGenericAtomic(Opcode)) {10570 return InstructionUniformity::NeverUniform;10571 }10572 return InstructionUniformity::Default;10573}10574 10575InstructionUniformity10576SIInstrInfo::getInstructionUniformity(const MachineInstr &MI) const {10577 10578 if (isNeverUniform(MI))10579 return InstructionUniformity::NeverUniform;10580 10581 unsigned opcode = MI.getOpcode();10582 if (opcode == AMDGPU::V_READLANE_B32 ||10583 opcode == AMDGPU::V_READFIRSTLANE_B32 ||10584 opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR)10585 return InstructionUniformity::AlwaysUniform;10586 10587 if (isCopyInstr(MI)) {10588 const MachineOperand &srcOp = MI.getOperand(1);10589 if (srcOp.isReg() && srcOp.getReg().isPhysical()) {10590 const TargetRegisterClass *regClass =10591 RI.getPhysRegBaseClass(srcOp.getReg());10592 return RI.isSGPRClass(regClass) ? InstructionUniformity::AlwaysUniform10593 : InstructionUniformity::NeverUniform;10594 }10595 return InstructionUniformity::Default;10596 }10597 10598 // GMIR handling10599 if (MI.isPreISelOpcode())10600 return SIInstrInfo::getGenericInstructionUniformity(MI);10601 10602 // Atomics are divergent because they are executed sequentially: when an10603 // atomic operation refers to the same address in each thread, then each10604 // thread after the first sees the value written by the previous thread as10605 // original value.10606 10607 if (isAtomic(MI))10608 return InstructionUniformity::NeverUniform;10609 10610 // Loads from the private and flat address spaces are divergent, because10611 // threads can execute the load instruction with the same inputs and get10612 // different results.10613 if (isFLAT(MI) && MI.mayLoad()) {10614 if (MI.memoperands_empty())10615 return InstructionUniformity::NeverUniform; // conservative assumption10616 10617 if (llvm::any_of(MI.memoperands(), [](const MachineMemOperand *mmo) {10618 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS ||10619 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS;10620 })) {10621 // At least one MMO in a non-global address space.10622 return InstructionUniformity::NeverUniform;10623 }10624 10625 return InstructionUniformity::Default;10626 }10627 10628 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();10629 const AMDGPURegisterBankInfo *RBI = ST.getRegBankInfo();10630 10631 // FIXME: It's conceptually broken to report this for an instruction, and not10632 // a specific def operand. For inline asm in particular, there could be mixed10633 // uniform and divergent results.10634 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {10635 const MachineOperand &SrcOp = MI.getOperand(I);10636 if (!SrcOp.isReg())10637 continue;10638 10639 Register Reg = SrcOp.getReg();10640 if (!Reg || !SrcOp.readsReg())10641 continue;10642 10643 // If RegBank is null, this is unassigned or an unallocatable special10644 // register, which are all scalars.10645 const RegisterBank *RegBank = RBI->getRegBank(Reg, MRI, RI);10646 if (RegBank && RegBank->getID() != AMDGPU::SGPRRegBankID)10647 return InstructionUniformity::NeverUniform;10648 }10649 10650 // TODO: Uniformity check condtions above can be rearranged for more10651 // redability10652 10653 // TODO: amdgcn.{ballot, [if]cmp} should be AlwaysUniform, but they are10654 // currently turned into no-op COPYs by SelectionDAG ISel and are10655 // therefore no longer recognizable.10656 10657 return InstructionUniformity::Default;10658}10659 10660unsigned SIInstrInfo::getDSShaderTypeValue(const MachineFunction &MF) {10661 switch (MF.getFunction().getCallingConv()) {10662 case CallingConv::AMDGPU_PS:10663 return 1;10664 case CallingConv::AMDGPU_VS:10665 return 2;10666 case CallingConv::AMDGPU_GS:10667 return 3;10668 case CallingConv::AMDGPU_HS:10669 case CallingConv::AMDGPU_LS:10670 case CallingConv::AMDGPU_ES: {10671 const Function &F = MF.getFunction();10672 F.getContext().diagnose(DiagnosticInfoUnsupported(10673 F, "ds_ordered_count unsupported for this calling conv"));10674 [[fallthrough]];10675 }10676 case CallingConv::AMDGPU_CS:10677 case CallingConv::AMDGPU_KERNEL:10678 case CallingConv::C:10679 case CallingConv::Fast:10680 default:10681 // Assume other calling conventions are various compute callable functions10682 return 0;10683 }10684}10685 10686bool SIInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,10687 Register &SrcReg2, int64_t &CmpMask,10688 int64_t &CmpValue) const {10689 if (!MI.getOperand(0).isReg() || MI.getOperand(0).getSubReg())10690 return false;10691 10692 switch (MI.getOpcode()) {10693 default:10694 break;10695 case AMDGPU::S_CMP_EQ_U32:10696 case AMDGPU::S_CMP_EQ_I32:10697 case AMDGPU::S_CMP_LG_U32:10698 case AMDGPU::S_CMP_LG_I32:10699 case AMDGPU::S_CMP_LT_U32:10700 case AMDGPU::S_CMP_LT_I32:10701 case AMDGPU::S_CMP_GT_U32:10702 case AMDGPU::S_CMP_GT_I32:10703 case AMDGPU::S_CMP_LE_U32:10704 case AMDGPU::S_CMP_LE_I32:10705 case AMDGPU::S_CMP_GE_U32:10706 case AMDGPU::S_CMP_GE_I32:10707 case AMDGPU::S_CMP_EQ_U64:10708 case AMDGPU::S_CMP_LG_U64:10709 SrcReg = MI.getOperand(0).getReg();10710 if (MI.getOperand(1).isReg()) {10711 if (MI.getOperand(1).getSubReg())10712 return false;10713 SrcReg2 = MI.getOperand(1).getReg();10714 CmpValue = 0;10715 } else if (MI.getOperand(1).isImm()) {10716 SrcReg2 = Register();10717 CmpValue = MI.getOperand(1).getImm();10718 } else {10719 return false;10720 }10721 CmpMask = ~0;10722 return true;10723 case AMDGPU::S_CMPK_EQ_U32:10724 case AMDGPU::S_CMPK_EQ_I32:10725 case AMDGPU::S_CMPK_LG_U32:10726 case AMDGPU::S_CMPK_LG_I32:10727 case AMDGPU::S_CMPK_LT_U32:10728 case AMDGPU::S_CMPK_LT_I32:10729 case AMDGPU::S_CMPK_GT_U32:10730 case AMDGPU::S_CMPK_GT_I32:10731 case AMDGPU::S_CMPK_LE_U32:10732 case AMDGPU::S_CMPK_LE_I32:10733 case AMDGPU::S_CMPK_GE_U32:10734 case AMDGPU::S_CMPK_GE_I32:10735 SrcReg = MI.getOperand(0).getReg();10736 SrcReg2 = Register();10737 CmpValue = MI.getOperand(1).getImm();10738 CmpMask = ~0;10739 return true;10740 }10741 10742 return false;10743}10744 10745// SCC is already valid after SCCValid.10746// SCCRedefine will redefine SCC to the same value already available after10747// SCCValid. If there are no intervening SCC conflicts delete SCCRedefine and10748// update kill/dead flags if necessary.10749static bool optimizeSCC(MachineInstr *SCCValid, MachineInstr *SCCRedefine,10750 const SIRegisterInfo &RI) {10751 MachineInstr *KillsSCC = nullptr;10752 if (SCCValid->getParent() != SCCRedefine->getParent())10753 return false;10754 for (MachineInstr &MI : make_range(std::next(SCCValid->getIterator()),10755 SCCRedefine->getIterator())) {10756 if (MI.modifiesRegister(AMDGPU::SCC, &RI))10757 return false;10758 if (MI.killsRegister(AMDGPU::SCC, &RI))10759 KillsSCC = &MI;10760 }10761 if (MachineOperand *SccDef =10762 SCCValid->findRegisterDefOperand(AMDGPU::SCC, /*TRI=*/nullptr))10763 SccDef->setIsDead(false);10764 if (KillsSCC)10765 KillsSCC->clearRegisterKills(AMDGPU::SCC, /*TRI=*/nullptr);10766 SCCRedefine->eraseFromParent();10767 return true;10768}10769 10770static bool foldableSelect(const MachineInstr &Def) {10771 if (Def.getOpcode() != AMDGPU::S_CSELECT_B32 &&10772 Def.getOpcode() != AMDGPU::S_CSELECT_B64)10773 return false;10774 bool Op1IsNonZeroImm =10775 Def.getOperand(1).isImm() && Def.getOperand(1).getImm() != 0;10776 bool Op2IsZeroImm =10777 Def.getOperand(2).isImm() && Def.getOperand(2).getImm() == 0;10778 if (!Op1IsNonZeroImm || !Op2IsZeroImm)10779 return false;10780 return true;10781}10782 10783bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,10784 Register SrcReg2, int64_t CmpMask,10785 int64_t CmpValue,10786 const MachineRegisterInfo *MRI) const {10787 if (!SrcReg || SrcReg.isPhysical())10788 return false;10789 10790 if (SrcReg2 && !getFoldableImm(SrcReg2, *MRI, CmpValue))10791 return false;10792 10793 const auto optimizeCmpSelect = [&CmpInstr, SrcReg, CmpValue, MRI,10794 this]() -> bool {10795 if (CmpValue != 0)10796 return false;10797 10798 MachineInstr *Def = MRI->getVRegDef(SrcReg);10799 if (!Def)10800 return false;10801 10802 // For S_OP that set SCC = DST!=0, do the transformation10803 //10804 // s_cmp_lg_* (S_OP ...), 0 => (S_OP ...)10805 10806 // If foldableSelect, s_cmp_lg_* is redundant because the SCC input value10807 // for S_CSELECT* already has the same value that will be calculated by10808 // s_cmp_lg_*10809 //10810 // s_cmp_lg_* (S_CSELECT* (non-zero imm), 0), 0 => (S_CSELECT* (non-zero10811 // imm), 0)10812 if (!setsSCCifResultIsNonZero(*Def) && !foldableSelect(*Def))10813 return false;10814 10815 if (!optimizeSCC(Def, &CmpInstr, RI))10816 return false;10817 10818 // If s_or_b32 result, sY, is unused (i.e. it is effectively a 64-bit10819 // s_cmp_lg of a register pair) and the inputs are the hi and lo-halves of a10820 // 64-bit foldableSelect then delete s_or_b32 in the sequence:10821 // sX = s_cselect_b64 (non-zero imm), 010822 // sLo = copy sX.sub010823 // sHi = copy sX.sub110824 // sY = s_or_b32 sLo, sHi10825 if (Def->getOpcode() == AMDGPU::S_OR_B32 &&10826 MRI->use_nodbg_empty(Def->getOperand(0).getReg())) {10827 const MachineOperand &OrOpnd1 = Def->getOperand(1);10828 const MachineOperand &OrOpnd2 = Def->getOperand(2);10829 if (OrOpnd1.isReg() && OrOpnd2.isReg()) {10830 MachineInstr *Def1 = MRI->getVRegDef(OrOpnd1.getReg());10831 MachineInstr *Def2 = MRI->getVRegDef(OrOpnd2.getReg());10832 if (Def1 && Def1->getOpcode() == AMDGPU::COPY && Def2 &&10833 Def2->getOpcode() == AMDGPU::COPY && Def1->getOperand(1).isReg() &&10834 Def2->getOperand(1).isReg() &&10835 Def1->getOperand(1).getSubReg() == AMDGPU::sub0 &&10836 Def2->getOperand(1).getSubReg() == AMDGPU::sub1 &&10837 Def1->getOperand(1).getReg() == Def2->getOperand(1).getReg()) {10838 MachineInstr *Select = MRI->getVRegDef(Def1->getOperand(1).getReg());10839 if (Select && foldableSelect(*Select))10840 optimizeSCC(Select, Def, RI);10841 }10842 }10843 }10844 return true;10845 };10846 10847 const auto optimizeCmpAnd = [&CmpInstr, SrcReg, CmpValue, MRI,10848 this](int64_t ExpectedValue, unsigned SrcSize,10849 bool IsReversible, bool IsSigned) -> bool {10850 // s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n10851 // s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n10852 // s_cmp_ge_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n10853 // s_cmp_ge_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n10854 // s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 1 << n => s_and_b64 $src, 1 << n10855 // s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n10856 // s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n10857 // s_cmp_gt_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n10858 // s_cmp_gt_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n10859 // s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 0 => s_and_b64 $src, 1 << n10860 //10861 // Signed ge/gt are not used for the sign bit.10862 //10863 // If result of the AND is unused except in the compare:10864 // s_and_b(32|64) $src, 1 << n => s_bitcmp1_b(32|64) $src, n10865 //10866 // s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n10867 // s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n10868 // s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 0 => s_bitcmp0_b64 $src, n10869 // s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n10870 // s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n10871 // s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 1 << n => s_bitcmp0_b64 $src, n10872 10873 MachineInstr *Def = MRI->getVRegDef(SrcReg);10874 if (!Def)10875 return false;10876 10877 if (Def->getOpcode() != AMDGPU::S_AND_B32 &&10878 Def->getOpcode() != AMDGPU::S_AND_B64)10879 return false;10880 10881 int64_t Mask;10882 const auto isMask = [&Mask, SrcSize](const MachineOperand *MO) -> bool {10883 if (MO->isImm())10884 Mask = MO->getImm();10885 else if (!getFoldableImm(MO, Mask))10886 return false;10887 Mask &= maxUIntN(SrcSize);10888 return isPowerOf2_64(Mask);10889 };10890 10891 MachineOperand *SrcOp = &Def->getOperand(1);10892 if (isMask(SrcOp))10893 SrcOp = &Def->getOperand(2);10894 else if (isMask(&Def->getOperand(2)))10895 SrcOp = &Def->getOperand(1);10896 else10897 return false;10898 10899 // A valid Mask is required to have a single bit set, hence a non-zero and10900 // power-of-two value. This verifies that we will not do 64-bit shift below.10901 assert(llvm::has_single_bit<uint64_t>(Mask) && "Invalid mask.");10902 unsigned BitNo = llvm::countr_zero((uint64_t)Mask);10903 if (IsSigned && BitNo == SrcSize - 1)10904 return false;10905 10906 ExpectedValue <<= BitNo;10907 10908 bool IsReversedCC = false;10909 if (CmpValue != ExpectedValue) {10910 if (!IsReversible)10911 return false;10912 IsReversedCC = CmpValue == (ExpectedValue ^ Mask);10913 if (!IsReversedCC)10914 return false;10915 }10916 10917 Register DefReg = Def->getOperand(0).getReg();10918 if (IsReversedCC && !MRI->hasOneNonDBGUse(DefReg))10919 return false;10920 10921 if (!optimizeSCC(Def, &CmpInstr, RI))10922 return false;10923 10924 if (!MRI->use_nodbg_empty(DefReg)) {10925 assert(!IsReversedCC);10926 return true;10927 }10928 10929 // Replace AND with unused result with a S_BITCMP.10930 MachineBasicBlock *MBB = Def->getParent();10931 10932 unsigned NewOpc = (SrcSize == 32) ? IsReversedCC ? AMDGPU::S_BITCMP0_B3210933 : AMDGPU::S_BITCMP1_B3210934 : IsReversedCC ? AMDGPU::S_BITCMP0_B6410935 : AMDGPU::S_BITCMP1_B64;10936 10937 BuildMI(*MBB, Def, Def->getDebugLoc(), get(NewOpc))10938 .add(*SrcOp)10939 .addImm(BitNo);10940 Def->eraseFromParent();10941 10942 return true;10943 };10944 10945 switch (CmpInstr.getOpcode()) {10946 default:10947 break;10948 case AMDGPU::S_CMP_EQ_U32:10949 case AMDGPU::S_CMP_EQ_I32:10950 case AMDGPU::S_CMPK_EQ_U32:10951 case AMDGPU::S_CMPK_EQ_I32:10952 return optimizeCmpAnd(1, 32, true, false);10953 case AMDGPU::S_CMP_GE_U32:10954 case AMDGPU::S_CMPK_GE_U32:10955 return optimizeCmpAnd(1, 32, false, false);10956 case AMDGPU::S_CMP_GE_I32:10957 case AMDGPU::S_CMPK_GE_I32:10958 return optimizeCmpAnd(1, 32, false, true);10959 case AMDGPU::S_CMP_EQ_U64:10960 return optimizeCmpAnd(1, 64, true, false);10961 case AMDGPU::S_CMP_LG_U32:10962 case AMDGPU::S_CMP_LG_I32:10963 case AMDGPU::S_CMPK_LG_U32:10964 case AMDGPU::S_CMPK_LG_I32:10965 return optimizeCmpAnd(0, 32, true, false) || optimizeCmpSelect();10966 case AMDGPU::S_CMP_GT_U32:10967 case AMDGPU::S_CMPK_GT_U32:10968 return optimizeCmpAnd(0, 32, false, false);10969 case AMDGPU::S_CMP_GT_I32:10970 case AMDGPU::S_CMPK_GT_I32:10971 return optimizeCmpAnd(0, 32, false, true);10972 case AMDGPU::S_CMP_LG_U64:10973 return optimizeCmpAnd(0, 64, true, false) || optimizeCmpSelect();10974 }10975 10976 return false;10977}10978 10979void SIInstrInfo::enforceOperandRCAlignment(MachineInstr &MI,10980 AMDGPU::OpName OpName) const {10981 if (!ST.needsAlignedVGPRs())10982 return;10983 10984 int OpNo = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpName);10985 if (OpNo < 0)10986 return;10987 MachineOperand &Op = MI.getOperand(OpNo);10988 if (getOpSize(MI, OpNo) > 4)10989 return;10990 10991 // Add implicit aligned super-reg to force alignment on the data operand.10992 const DebugLoc &DL = MI.getDebugLoc();10993 MachineBasicBlock *BB = MI.getParent();10994 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();10995 Register DataReg = Op.getReg();10996 bool IsAGPR = RI.isAGPR(MRI, DataReg);10997 Register Undef = MRI.createVirtualRegister(10998 IsAGPR ? &AMDGPU::AGPR_32RegClass : &AMDGPU::VGPR_32RegClass);10999 BuildMI(*BB, MI, DL, get(AMDGPU::IMPLICIT_DEF), Undef);11000 Register NewVR =11001 MRI.createVirtualRegister(IsAGPR ? &AMDGPU::AReg_64_Align2RegClass11002 : &AMDGPU::VReg_64_Align2RegClass);11003 BuildMI(*BB, MI, DL, get(AMDGPU::REG_SEQUENCE), NewVR)11004 .addReg(DataReg, 0, Op.getSubReg())11005 .addImm(AMDGPU::sub0)11006 .addReg(Undef)11007 .addImm(AMDGPU::sub1);11008 Op.setReg(NewVR);11009 Op.setSubReg(AMDGPU::sub0);11010 MI.addOperand(MachineOperand::CreateReg(NewVR, false, true));11011}11012 11013bool SIInstrInfo::isGlobalMemoryObject(const MachineInstr *MI) const {11014 if (isIGLP(*MI))11015 return false;11016 11017 return TargetInstrInfo::isGlobalMemoryObject(MI);11018}11019 11020bool SIInstrInfo::isXDLWMMA(const MachineInstr &MI) const {11021 if (!isWMMA(MI) && !isSWMMAC(MI))11022 return false;11023 11024 if (AMDGPU::isGFX1250(ST))11025 return AMDGPU::getWMMAIsXDL(MI.getOpcode());11026 11027 return true;11028}11029 11030bool SIInstrInfo::isXDL(const MachineInstr &MI) const {11031 unsigned Opcode = MI.getOpcode();11032 11033 if (AMDGPU::isGFX12Plus(ST))11034 return isDOT(MI) || isXDLWMMA(MI);11035 11036 if (!isMAI(MI) || isDGEMM(Opcode) ||11037 Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||11038 Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64)11039 return false;11040 11041 if (!ST.hasGFX940Insts())11042 return true;11043 11044 return AMDGPU::getMAIIsGFX940XDL(Opcode);11045}11046