834 lines · cpp
1//=======- GCNDPPCombine.cpp - optimization for DPP instructions ---==========//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8// The pass combines V_MOV_B32_dpp instruction with its VALU uses as a DPP src09// operand. If any of the use instruction cannot be combined with the mov the10// whole sequence is reverted.11//12// $old = ...13// $dpp_value = V_MOV_B32_dpp $old, $vgpr_to_be_read_from_other_lane,14// dpp_controls..., $row_mask, $bank_mask, $bound_ctrl15// $res = VALU $dpp_value [, src1]16//17// to18//19// $res = VALU_DPP $combined_old, $vgpr_to_be_read_from_other_lane, [src1,]20// dpp_controls..., $row_mask, $bank_mask, $combined_bound_ctrl21//22// Combining rules :23//24// if $row_mask and $bank_mask are fully enabled (0xF) and25// $bound_ctrl==DPP_BOUND_ZERO or $old==026// -> $combined_old = undef,27// $combined_bound_ctrl = DPP_BOUND_ZERO28//29// if the VALU op is binary and30// $bound_ctrl==DPP_BOUND_OFF and31// $old==identity value (immediate) for the VALU op32// -> $combined_old = src1,33// $combined_bound_ctrl = DPP_BOUND_OFF34//35// Otherwise cancel.36//37// The mov_dpp instruction should reside in the same BB as all its uses38//===----------------------------------------------------------------------===//39 40#include "GCNDPPCombine.h"41#include "AMDGPU.h"42#include "GCNSubtarget.h"43#include "MCTargetDesc/AMDGPUMCTargetDesc.h"44#include "llvm/ADT/Statistic.h"45#include "llvm/CodeGen/MachineFunctionPass.h"46 47using namespace llvm;48 49#define DEBUG_TYPE "gcn-dpp-combine"50 51STATISTIC(NumDPPMovsCombined, "Number of DPP moves combined.");52 53namespace {54 55class GCNDPPCombine {56 MachineRegisterInfo *MRI;57 const SIInstrInfo *TII;58 const GCNSubtarget *ST;59 60 using RegSubRegPair = TargetInstrInfo::RegSubRegPair;61 62 MachineOperand *getOldOpndValue(MachineOperand &OldOpnd) const;63 64 MachineInstr *createDPPInst(MachineInstr &OrigMI, MachineInstr &MovMI,65 RegSubRegPair CombOldVGPR,66 MachineOperand *OldOpnd, bool CombBCZ,67 bool IsShrinkable) const;68 69 MachineInstr *createDPPInst(MachineInstr &OrigMI, MachineInstr &MovMI,70 RegSubRegPair CombOldVGPR, bool CombBCZ,71 bool IsShrinkable) const;72 73 bool hasNoImmOrEqual(MachineInstr &MI, AMDGPU::OpName OpndName, int64_t Value,74 int64_t Mask = -1) const;75 76 bool combineDPPMov(MachineInstr &MI) const;77 78 int getDPPOp(unsigned Op, bool IsShrinkable) const;79 bool isShrinkable(MachineInstr &MI) const;80 81public:82 bool run(MachineFunction &MF);83};84 85class GCNDPPCombineLegacy : public MachineFunctionPass {86public:87 static char ID;88 89 GCNDPPCombineLegacy() : MachineFunctionPass(ID) {}90 91 bool runOnMachineFunction(MachineFunction &MF) override;92 93 StringRef getPassName() const override { return "GCN DPP Combine"; }94 95 void getAnalysisUsage(AnalysisUsage &AU) const override {96 AU.setPreservesCFG();97 MachineFunctionPass::getAnalysisUsage(AU);98 }99 100 MachineFunctionProperties getRequiredProperties() const override {101 return MachineFunctionProperties().setIsSSA();102 }103};104 105} // end anonymous namespace106 107INITIALIZE_PASS(GCNDPPCombineLegacy, DEBUG_TYPE, "GCN DPP Combine", false,108 false)109 110char GCNDPPCombineLegacy::ID = 0;111 112char &llvm::GCNDPPCombineLegacyID = GCNDPPCombineLegacy::ID;113 114FunctionPass *llvm::createGCNDPPCombinePass() {115 return new GCNDPPCombineLegacy();116}117 118bool GCNDPPCombine::isShrinkable(MachineInstr &MI) const {119 unsigned Op = MI.getOpcode();120 if (!TII->isVOP3(Op)) {121 return false;122 }123 if (!TII->hasVALU32BitEncoding(Op)) {124 LLVM_DEBUG(dbgs() << " Inst hasn't e32 equivalent\n");125 return false;126 }127 // Do not shrink True16 instructions pre-RA to avoid the restriction in128 // register allocation from only being able to use 128 VGPRs129 if (AMDGPU::isTrue16Inst(Op))130 return false;131 if (const auto *SDst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst)) {132 // Give up if there are any uses of the sdst in carry-out or VOPC.133 // The shrunken form of the instruction would write it to vcc instead of to134 // a virtual register. If we rewrote the uses the shrinking would be135 // possible.136 if (!MRI->use_nodbg_empty(SDst->getReg()))137 return false;138 }139 // check if other than abs|neg modifiers are set (opsel for example)140 const int64_t Mask = ~(SISrcMods::ABS | SISrcMods::NEG);141 if (!hasNoImmOrEqual(MI, AMDGPU::OpName::src0_modifiers, 0, Mask) ||142 !hasNoImmOrEqual(MI, AMDGPU::OpName::src1_modifiers, 0, Mask) ||143 !hasNoImmOrEqual(MI, AMDGPU::OpName::clamp, 0) ||144 !hasNoImmOrEqual(MI, AMDGPU::OpName::omod, 0) ||145 !hasNoImmOrEqual(MI, AMDGPU::OpName::byte_sel, 0)) {146 LLVM_DEBUG(dbgs() << " Inst has non-default modifiers\n");147 return false;148 }149 return true;150}151 152int GCNDPPCombine::getDPPOp(unsigned Op, bool IsShrinkable) const {153 int DPP32 = AMDGPU::getDPPOp32(Op);154 if (IsShrinkable) {155 assert(DPP32 == -1);156 int E32 = AMDGPU::getVOPe32(Op);157 DPP32 = (E32 == -1) ? -1 : AMDGPU::getDPPOp32(E32);158 }159 if (DPP32 != -1 && TII->pseudoToMCOpcode(DPP32) != -1)160 return DPP32;161 int DPP64 = -1;162 if (ST->hasVOP3DPP())163 DPP64 = AMDGPU::getDPPOp64(Op);164 if (DPP64 != -1 && TII->pseudoToMCOpcode(DPP64) != -1)165 return DPP64;166 return -1;167}168 169// tracks the register operand definition and returns:170// 1. immediate operand used to initialize the register if found171// 2. nullptr if the register operand is undef172// 3. the operand itself otherwise173MachineOperand *GCNDPPCombine::getOldOpndValue(MachineOperand &OldOpnd) const {174 auto *Def = getVRegSubRegDef(getRegSubRegPair(OldOpnd), *MRI);175 if (!Def)176 return nullptr;177 178 switch(Def->getOpcode()) {179 default: break;180 case AMDGPU::IMPLICIT_DEF:181 return nullptr;182 case AMDGPU::COPY:183 case AMDGPU::V_MOV_B32_e32:184 case AMDGPU::V_MOV_B64_PSEUDO:185 case AMDGPU::V_MOV_B64_e32:186 case AMDGPU::V_MOV_B64_e64: {187 auto &Op1 = Def->getOperand(1);188 if (Op1.isImm())189 return &Op1;190 break;191 }192 }193 return &OldOpnd;194}195 196MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI,197 MachineInstr &MovMI,198 RegSubRegPair CombOldVGPR,199 bool CombBCZ,200 bool IsShrinkable) const {201 assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp ||202 MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp ||203 MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);204 205 bool HasVOP3DPP = ST->hasVOP3DPP();206 auto OrigOp = OrigMI.getOpcode();207 if (ST->useRealTrue16Insts() && AMDGPU::isTrue16Inst(OrigOp)) {208 LLVM_DEBUG(209 dbgs() << " failed: Did not expect any 16-bit uses of dpp values\n");210 return nullptr;211 }212 auto DPPOp = getDPPOp(OrigOp, IsShrinkable);213 if (DPPOp == -1) {214 LLVM_DEBUG(dbgs() << " failed: no DPP opcode\n");215 return nullptr;216 }217 int OrigOpE32 = AMDGPU::getVOPe32(OrigOp);218 // Prior checks cover Mask with VOPC condition, but not on purpose219 auto *RowMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::row_mask);220 assert(RowMaskOpnd && RowMaskOpnd->isImm());221 auto *BankMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::bank_mask);222 assert(BankMaskOpnd && BankMaskOpnd->isImm());223 const bool MaskAllLanes =224 RowMaskOpnd->getImm() == 0xF && BankMaskOpnd->getImm() == 0xF;225 (void)MaskAllLanes;226 assert((MaskAllLanes ||227 !(TII->isVOPC(DPPOp) || (TII->isVOP3(DPPOp) && OrigOpE32 != -1 &&228 TII->isVOPC(OrigOpE32)))) &&229 "VOPC cannot form DPP unless mask is full");230 231 auto DPPInst = BuildMI(*OrigMI.getParent(), OrigMI,232 OrigMI.getDebugLoc(), TII->get(DPPOp))233 .setMIFlags(OrigMI.getFlags());234 235 bool Fail = false;236 do {237 int NumOperands = 0;238 if (auto *Dst = TII->getNamedOperand(OrigMI, AMDGPU::OpName::vdst)) {239 DPPInst.add(*Dst);240 ++NumOperands;241 }242 if (auto *SDst = TII->getNamedOperand(OrigMI, AMDGPU::OpName::sdst)) {243 if (AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::sdst)) {244 DPPInst.add(*SDst);245 ++NumOperands;246 }247 // If we shrunk a 64bit vop3b to 32bits, just ignore the sdst248 }249 250 const int OldIdx = AMDGPU::getNamedOperandIdx(DPPOp, AMDGPU::OpName::old);251 if (OldIdx != -1) {252 assert(OldIdx == NumOperands);253 assert(isOfRegClass(254 CombOldVGPR,255 *MRI->getRegClass(256 TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst)->getReg()),257 *MRI));258 auto *Def = getVRegSubRegDef(CombOldVGPR, *MRI);259 DPPInst.addReg(CombOldVGPR.Reg, Def ? 0 : RegState::Undef,260 CombOldVGPR.SubReg);261 ++NumOperands;262 } else if (TII->isVOPC(DPPOp) || (TII->isVOP3(DPPOp) && OrigOpE32 != -1 &&263 TII->isVOPC(OrigOpE32))) {264 // VOPC DPP and VOPC promoted to VOP3 DPP do not have an old operand265 // because they write to SGPRs not VGPRs266 } else {267 // TODO: this discards MAC/FMA instructions for now, let's add it later268 LLVM_DEBUG(dbgs() << " failed: no old operand in DPP instruction,"269 " TBD\n");270 Fail = true;271 break;272 }273 274 auto *Mod0 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src0_modifiers);275 if (Mod0) {276 assert(NumOperands == AMDGPU::getNamedOperandIdx(DPPOp,277 AMDGPU::OpName::src0_modifiers));278 assert(HasVOP3DPP ||279 (0LL == (Mod0->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG))));280 DPPInst.addImm(Mod0->getImm());281 ++NumOperands;282 } else if (AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::src0_modifiers)) {283 DPPInst.addImm(0);284 ++NumOperands;285 }286 auto *Src0 = TII->getNamedOperand(MovMI, AMDGPU::OpName::src0);287 assert(Src0);288 [[maybe_unused]] int Src0Idx = NumOperands;289 290 DPPInst.add(*Src0);291 DPPInst->getOperand(NumOperands).setIsKill(false);292 ++NumOperands;293 294 auto *Mod1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1_modifiers);295 if (Mod1) {296 assert(NumOperands == AMDGPU::getNamedOperandIdx(DPPOp,297 AMDGPU::OpName::src1_modifiers));298 assert(HasVOP3DPP ||299 (0LL == (Mod1->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG))));300 DPPInst.addImm(Mod1->getImm());301 ++NumOperands;302 } else if (AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::src1_modifiers)) {303 DPPInst.addImm(0);304 ++NumOperands;305 }306 auto *Src1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1);307 if (Src1) {308 assert(AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::src1) &&309 "dpp version of instruction missing src1");310 // If subtarget does not support SGPRs for src1 operand then the311 // requirements are the same as for src0. We check src0 instead because312 // pseudos are shared between subtargets and allow SGPR for src1 on all.313 if (!ST->hasDPPSrc1SGPR()) {314 assert(TII->getOpSize(*DPPInst, Src0Idx) ==315 TII->getOpSize(*DPPInst, NumOperands) &&316 "Src0 and Src1 operands should have the same size");317 }318 319 DPPInst.add(*Src1);320 ++NumOperands;321 }322 323 auto *Mod2 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src2_modifiers);324 if (Mod2) {325 assert(NumOperands ==326 AMDGPU::getNamedOperandIdx(DPPOp, AMDGPU::OpName::src2_modifiers));327 assert(HasVOP3DPP ||328 (0LL == (Mod2->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG))));329 DPPInst.addImm(Mod2->getImm());330 ++NumOperands;331 }332 auto *Src2 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src2);333 if (Src2) {334 if (!AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::src2)) {335 LLVM_DEBUG(dbgs() << " failed: dpp does not have src2\n");336 Fail = true;337 break;338 }339 DPPInst.add(*Src2);340 ++NumOperands;341 }342 343 if (HasVOP3DPP) {344 auto *ClampOpr = TII->getNamedOperand(OrigMI, AMDGPU::OpName::clamp);345 if (ClampOpr && AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::clamp)) {346 DPPInst.addImm(ClampOpr->getImm());347 }348 auto *VdstInOpr = TII->getNamedOperand(OrigMI, AMDGPU::OpName::vdst_in);349 if (VdstInOpr &&350 AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::vdst_in)) {351 DPPInst.add(*VdstInOpr);352 }353 auto *OmodOpr = TII->getNamedOperand(OrigMI, AMDGPU::OpName::omod);354 if (OmodOpr && AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::omod)) {355 DPPInst.addImm(OmodOpr->getImm());356 }357 // Validate OP_SEL has to be set to all 0 and OP_SEL_HI has to be set to358 // all 1.359 if (TII->getNamedOperand(OrigMI, AMDGPU::OpName::op_sel)) {360 int64_t OpSel = 0;361 OpSel |= (Mod0 ? (!!(Mod0->getImm() & SISrcMods::OP_SEL_0) << 0) : 0);362 OpSel |= (Mod1 ? (!!(Mod1->getImm() & SISrcMods::OP_SEL_0) << 1) : 0);363 OpSel |= (Mod2 ? (!!(Mod2->getImm() & SISrcMods::OP_SEL_0) << 2) : 0);364 if (Mod0 && TII->isVOP3(OrigMI) && !TII->isVOP3P(OrigMI))365 OpSel |= !!(Mod0->getImm() & SISrcMods::DST_OP_SEL) << 3;366 367 if (OpSel != 0) {368 LLVM_DEBUG(dbgs() << " failed: op_sel must be zero\n");369 Fail = true;370 break;371 }372 if (AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::op_sel))373 DPPInst.addImm(OpSel);374 }375 if (TII->getNamedOperand(OrigMI, AMDGPU::OpName::op_sel_hi)) {376 int64_t OpSelHi = 0;377 OpSelHi |= (Mod0 ? (!!(Mod0->getImm() & SISrcMods::OP_SEL_1) << 0) : 0);378 OpSelHi |= (Mod1 ? (!!(Mod1->getImm() & SISrcMods::OP_SEL_1) << 1) : 0);379 OpSelHi |= (Mod2 ? (!!(Mod2->getImm() & SISrcMods::OP_SEL_1) << 2) : 0);380 381 // Only vop3p has op_sel_hi, and all vop3p have 3 operands, so check382 // the bitmask for 3 op_sel_hi bits set383 assert(Src2 && "Expected vop3p with 3 operands");384 if (OpSelHi != 7) {385 LLVM_DEBUG(dbgs() << " failed: op_sel_hi must be all set to one\n");386 Fail = true;387 break;388 }389 if (AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::op_sel_hi))390 DPPInst.addImm(OpSelHi);391 }392 auto *NegOpr = TII->getNamedOperand(OrigMI, AMDGPU::OpName::neg_lo);393 if (NegOpr && AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::neg_lo)) {394 DPPInst.addImm(NegOpr->getImm());395 }396 auto *NegHiOpr = TII->getNamedOperand(OrigMI, AMDGPU::OpName::neg_hi);397 if (NegHiOpr && AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::neg_hi)) {398 DPPInst.addImm(NegHiOpr->getImm());399 }400 auto *ByteSelOpr = TII->getNamedOperand(OrigMI, AMDGPU::OpName::byte_sel);401 if (ByteSelOpr &&402 AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::byte_sel)) {403 DPPInst.addImm(ByteSelOpr->getImm());404 }405 if (MachineOperand *BitOp3 =406 TII->getNamedOperand(OrigMI, AMDGPU::OpName::bitop3)) {407 assert(AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::bitop3));408 DPPInst.add(*BitOp3);409 }410 }411 DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::dpp_ctrl));412 DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::row_mask));413 DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::bank_mask));414 DPPInst.addImm(CombBCZ ? 1 : 0);415 416 constexpr AMDGPU::OpName Srcs[] = {417 AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2};418 419 // FIXME: isOperandLegal expects to operate on an completely built420 // instruction. We should have better legality APIs to check if the421 // candidate operands will be legal without building the instruction first.422 for (auto [I, OpName] : enumerate(Srcs)) {423 int OpIdx = AMDGPU::getNamedOperandIdx(DPPOp, OpName);424 if (OpIdx == -1)425 break;426 427 if (!TII->isOperandLegal(*DPPInst, OpIdx)) {428 LLVM_DEBUG(dbgs() << " failed: src" << I << " operand is illegal\n");429 Fail = true;430 break;431 }432 }433 } while (false);434 435 if (Fail) {436 DPPInst.getInstr()->eraseFromParent();437 return nullptr;438 }439 LLVM_DEBUG(dbgs() << " combined: " << *DPPInst.getInstr());440 return DPPInst.getInstr();441}442 443static bool isIdentityValue(unsigned OrigMIOp, MachineOperand *OldOpnd) {444 assert(OldOpnd->isImm());445 switch (OrigMIOp) {446 default: break;447 case AMDGPU::V_ADD_U32_e32:448 case AMDGPU::V_ADD_U32_e64:449 case AMDGPU::V_ADD_CO_U32_e32:450 case AMDGPU::V_ADD_CO_U32_e64:451 case AMDGPU::V_OR_B32_e32:452 case AMDGPU::V_OR_B32_e64:453 case AMDGPU::V_SUBREV_U32_e32:454 case AMDGPU::V_SUBREV_U32_e64:455 case AMDGPU::V_SUBREV_CO_U32_e32:456 case AMDGPU::V_SUBREV_CO_U32_e64:457 case AMDGPU::V_MAX_U32_e32:458 case AMDGPU::V_MAX_U32_e64:459 case AMDGPU::V_XOR_B32_e32:460 case AMDGPU::V_XOR_B32_e64:461 if (OldOpnd->getImm() == 0)462 return true;463 break;464 case AMDGPU::V_AND_B32_e32:465 case AMDGPU::V_AND_B32_e64:466 case AMDGPU::V_MIN_U32_e32:467 case AMDGPU::V_MIN_U32_e64:468 if (static_cast<uint32_t>(OldOpnd->getImm()) ==469 std::numeric_limits<uint32_t>::max())470 return true;471 break;472 case AMDGPU::V_MIN_I32_e32:473 case AMDGPU::V_MIN_I32_e64:474 if (static_cast<int32_t>(OldOpnd->getImm()) ==475 std::numeric_limits<int32_t>::max())476 return true;477 break;478 case AMDGPU::V_MAX_I32_e32:479 case AMDGPU::V_MAX_I32_e64:480 if (static_cast<int32_t>(OldOpnd->getImm()) ==481 std::numeric_limits<int32_t>::min())482 return true;483 break;484 case AMDGPU::V_MUL_I32_I24_e32:485 case AMDGPU::V_MUL_I32_I24_e64:486 case AMDGPU::V_MUL_U32_U24_e32:487 case AMDGPU::V_MUL_U32_U24_e64:488 if (OldOpnd->getImm() == 1)489 return true;490 break;491 }492 return false;493}494 495MachineInstr *GCNDPPCombine::createDPPInst(496 MachineInstr &OrigMI, MachineInstr &MovMI, RegSubRegPair CombOldVGPR,497 MachineOperand *OldOpndValue, bool CombBCZ, bool IsShrinkable) const {498 assert(CombOldVGPR.Reg);499 if (!CombBCZ && OldOpndValue && OldOpndValue->isImm()) {500 auto *Src1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1);501 if (!Src1 || !Src1->isReg()) {502 LLVM_DEBUG(dbgs() << " failed: no src1 or it isn't a register\n");503 return nullptr;504 }505 if (!isIdentityValue(OrigMI.getOpcode(), OldOpndValue)) {506 LLVM_DEBUG(dbgs() << " failed: old immediate isn't an identity\n");507 return nullptr;508 }509 CombOldVGPR = getRegSubRegPair(*Src1);510 auto *MovDst = TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst);511 const TargetRegisterClass *RC = MRI->getRegClass(MovDst->getReg());512 if (!isOfRegClass(CombOldVGPR, *RC, *MRI)) {513 LLVM_DEBUG(dbgs() << " failed: src1 has wrong register class\n");514 return nullptr;515 }516 }517 return createDPPInst(OrigMI, MovMI, CombOldVGPR, CombBCZ, IsShrinkable);518}519 520// returns true if MI doesn't have OpndName immediate operand or the521// operand has Value522bool GCNDPPCombine::hasNoImmOrEqual(MachineInstr &MI, AMDGPU::OpName OpndName,523 int64_t Value, int64_t Mask) const {524 auto *Imm = TII->getNamedOperand(MI, OpndName);525 if (!Imm)526 return true;527 528 assert(Imm->isImm());529 return (Imm->getImm() & Mask) == Value;530}531 532bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const {533 assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp ||534 MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp ||535 MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);536 LLVM_DEBUG(dbgs() << "\nDPP combine: " << MovMI);537 538 auto *DstOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst);539 assert(DstOpnd && DstOpnd->isReg());540 auto DPPMovReg = DstOpnd->getReg();541 if (DPPMovReg.isPhysical()) {542 LLVM_DEBUG(dbgs() << " failed: dpp move writes physreg\n");543 return false;544 }545 if (execMayBeModifiedBeforeAnyUse(*MRI, DPPMovReg, MovMI)) {546 LLVM_DEBUG(dbgs() << " failed: EXEC mask should remain the same"547 " for all uses\n");548 return false;549 }550 551 auto *DppCtrl = TII->getNamedOperand(MovMI, AMDGPU::OpName::dpp_ctrl);552 assert(DppCtrl && DppCtrl->isImm());553 unsigned DppCtrlVal = DppCtrl->getImm();554 if ((MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO ||555 MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp)) {556 if (!ST->hasFeature(AMDGPU::FeatureDPALU_DPP)) {557 LLVM_DEBUG(dbgs() << " failed: 64 bit dpp move is unsupported\n");558 // Split it.559 return false;560 }561 if (!AMDGPU::isLegalDPALU_DPPControl(*ST, DppCtrlVal)) {562 LLVM_DEBUG(dbgs() << " failed: 64 bit dpp move uses unsupported"563 " control value\n");564 // Let it split, then control may become legal.565 return false;566 }567 }568 569 auto *RowMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::row_mask);570 assert(RowMaskOpnd && RowMaskOpnd->isImm());571 auto *BankMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::bank_mask);572 assert(BankMaskOpnd && BankMaskOpnd->isImm());573 const bool MaskAllLanes = RowMaskOpnd->getImm() == 0xF &&574 BankMaskOpnd->getImm() == 0xF;575 576 auto *BCZOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::bound_ctrl);577 assert(BCZOpnd && BCZOpnd->isImm());578 bool BoundCtrlZero = BCZOpnd->getImm();579 580 auto *OldOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::old);581 auto *SrcOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::src0);582 assert(OldOpnd && OldOpnd->isReg());583 assert(SrcOpnd && SrcOpnd->isReg());584 if (OldOpnd->getReg().isPhysical() || SrcOpnd->getReg().isPhysical()) {585 LLVM_DEBUG(dbgs() << " failed: dpp move reads physreg\n");586 return false;587 }588 589 auto * const OldOpndValue = getOldOpndValue(*OldOpnd);590 // OldOpndValue is either undef (IMPLICIT_DEF) or immediate or something else591 // We could use: assert(!OldOpndValue || OldOpndValue->isImm())592 // but the third option is used to distinguish undef from non-immediate593 // to reuse IMPLICIT_DEF instruction later594 assert(!OldOpndValue || OldOpndValue->isImm() || OldOpndValue == OldOpnd);595 596 bool CombBCZ = false;597 598 if (MaskAllLanes && BoundCtrlZero) { // [1]599 CombBCZ = true;600 } else {601 if (!OldOpndValue || !OldOpndValue->isImm()) {602 LLVM_DEBUG(dbgs() << " failed: the DPP mov isn't combinable\n");603 return false;604 }605 606 if (OldOpndValue->getImm() == 0) {607 if (MaskAllLanes) {608 assert(!BoundCtrlZero); // by check [1]609 CombBCZ = true;610 }611 } else if (BoundCtrlZero) {612 assert(!MaskAllLanes); // by check [1]613 LLVM_DEBUG(dbgs() <<614 " failed: old!=0 and bctrl:0 and not all lanes isn't combinable\n");615 return false;616 }617 }618 619 LLVM_DEBUG(dbgs() << " old=";620 if (!OldOpndValue)621 dbgs() << "undef";622 else623 dbgs() << *OldOpndValue;624 dbgs() << ", bound_ctrl=" << CombBCZ << '\n');625 626 SmallVector<MachineInstr*, 4> OrigMIs, DPPMIs;627 DenseMap<MachineInstr*, SmallVector<unsigned, 4>> RegSeqWithOpNos;628 auto CombOldVGPR = getRegSubRegPair(*OldOpnd);629 // try to reuse previous old reg if its undefined (IMPLICIT_DEF)630 if (CombBCZ && OldOpndValue) { // CombOldVGPR should be undef631 const TargetRegisterClass *RC = MRI->getRegClass(DPPMovReg);632 CombOldVGPR = RegSubRegPair(633 MRI->createVirtualRegister(RC));634 auto UndefInst = BuildMI(*MovMI.getParent(), MovMI, MovMI.getDebugLoc(),635 TII->get(AMDGPU::IMPLICIT_DEF), CombOldVGPR.Reg);636 DPPMIs.push_back(UndefInst.getInstr());637 }638 639 OrigMIs.push_back(&MovMI);640 bool Rollback = true;641 SmallVector<MachineOperand *, 16> Uses(642 llvm::make_pointer_range(MRI->use_nodbg_operands(DPPMovReg)));643 644 while (!Uses.empty()) {645 MachineOperand *Use = Uses.pop_back_val();646 Rollback = true;647 648 auto &OrigMI = *Use->getParent();649 LLVM_DEBUG(dbgs() << " try: " << OrigMI);650 651 auto OrigOp = OrigMI.getOpcode();652 assert((TII->get(OrigOp).getSize() != 4 || !AMDGPU::isTrue16Inst(OrigOp)) &&653 "There should not be e32 True16 instructions pre-RA");654 if (OrigOp == AMDGPU::REG_SEQUENCE) {655 Register FwdReg = OrigMI.getOperand(0).getReg();656 unsigned FwdSubReg = 0;657 658 if (execMayBeModifiedBeforeAnyUse(*MRI, FwdReg, OrigMI)) {659 LLVM_DEBUG(dbgs() << " failed: EXEC mask should remain the same"660 " for all uses\n");661 break;662 }663 664 unsigned OpNo, E = OrigMI.getNumOperands();665 for (OpNo = 1; OpNo < E; OpNo += 2) {666 if (OrigMI.getOperand(OpNo).getReg() == DPPMovReg) {667 FwdSubReg = OrigMI.getOperand(OpNo + 1).getImm();668 break;669 }670 }671 672 if (!FwdSubReg)673 break;674 675 for (auto &Op : MRI->use_nodbg_operands(FwdReg)) {676 if (Op.getSubReg() == FwdSubReg)677 Uses.push_back(&Op);678 }679 RegSeqWithOpNos[&OrigMI].push_back(OpNo);680 continue;681 }682 683 bool IsShrinkable = isShrinkable(OrigMI);684 if (!(IsShrinkable ||685 ((TII->isVOP3P(OrigOp) || TII->isVOPC(OrigOp) ||686 TII->isVOP3(OrigOp)) &&687 ST->hasVOP3DPP()) ||688 TII->isVOP1(OrigOp) || TII->isVOP2(OrigOp))) {689 LLVM_DEBUG(dbgs() << " failed: not VOP1/2/3/3P/C\n");690 break;691 }692 if (OrigMI.modifiesRegister(AMDGPU::EXEC, ST->getRegisterInfo())) {693 LLVM_DEBUG(dbgs() << " failed: can't combine v_cmpx\n");694 break;695 }696 697 auto *Src0 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src0);698 auto *Src1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1);699 if (Use != Src0 && !(Use == Src1 && OrigMI.isCommutable())) { // [1]700 LLVM_DEBUG(dbgs() << " failed: no suitable operands\n");701 break;702 }703 704 auto *Src2 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src2);705 assert(Src0 && "Src1 without Src0?");706 if ((Use == Src0 && ((Src1 && Src1->isIdenticalTo(*Src0)) ||707 (Src2 && Src2->isIdenticalTo(*Src0)))) ||708 (Use == Src1 && (Src1->isIdenticalTo(*Src0) ||709 (Src2 && Src2->isIdenticalTo(*Src1))))) {710 LLVM_DEBUG(711 dbgs()712 << " " << OrigMI713 << " failed: DPP register is used more than once per instruction\n");714 break;715 }716 717 if (!ST->hasFeature(AMDGPU::FeatureDPALU_DPP) &&718 AMDGPU::isDPALU_DPP32BitOpc(OrigOp)) {719 LLVM_DEBUG(dbgs() << " " << OrigMI720 << " failed: DPP ALU DPP is not supported\n");721 break;722 }723 724 if (!AMDGPU::isLegalDPALU_DPPControl(*ST, DppCtrlVal) &&725 AMDGPU::isDPALU_DPP(TII->get(OrigOp), *TII, *ST)) {726 LLVM_DEBUG(dbgs() << " " << OrigMI727 << " failed: not valid 64-bit DPP control value\n");728 break;729 }730 731 LLVM_DEBUG(dbgs() << " combining: " << OrigMI);732 if (Use == Src0) {733 if (auto *DPPInst = createDPPInst(OrigMI, MovMI, CombOldVGPR,734 OldOpndValue, CombBCZ, IsShrinkable)) {735 DPPMIs.push_back(DPPInst);736 Rollback = false;737 }738 } else {739 assert(Use == Src1 && OrigMI.isCommutable()); // by check [1]740 auto *BB = OrigMI.getParent();741 auto *NewMI = BB->getParent()->CloneMachineInstr(&OrigMI);742 BB->insert(OrigMI, NewMI);743 if (TII->commuteInstruction(*NewMI)) {744 LLVM_DEBUG(dbgs() << " commuted: " << *NewMI);745 if (auto *DPPInst =746 createDPPInst(*NewMI, MovMI, CombOldVGPR, OldOpndValue, CombBCZ,747 IsShrinkable)) {748 DPPMIs.push_back(DPPInst);749 Rollback = false;750 }751 } else752 LLVM_DEBUG(dbgs() << " failed: cannot be commuted\n");753 NewMI->eraseFromParent();754 }755 if (Rollback)756 break;757 OrigMIs.push_back(&OrigMI);758 }759 760 Rollback |= !Uses.empty();761 762 for (auto *MI : *(Rollback? &DPPMIs : &OrigMIs))763 MI->eraseFromParent();764 765 if (!Rollback) {766 for (auto &S : RegSeqWithOpNos) {767 if (MRI->use_nodbg_empty(S.first->getOperand(0).getReg())) {768 S.first->eraseFromParent();769 continue;770 }771 while (!S.second.empty())772 S.first->getOperand(S.second.pop_back_val()).setIsUndef();773 }774 }775 776 return !Rollback;777}778 779bool GCNDPPCombineLegacy::runOnMachineFunction(MachineFunction &MF) {780 if (skipFunction(MF.getFunction()))781 return false;782 783 return GCNDPPCombine().run(MF);784}785 786bool GCNDPPCombine::run(MachineFunction &MF) {787 ST = &MF.getSubtarget<GCNSubtarget>();788 if (!ST->hasDPP())789 return false;790 791 MRI = &MF.getRegInfo();792 TII = ST->getInstrInfo();793 794 bool Changed = false;795 for (auto &MBB : MF) {796 for (MachineInstr &MI : llvm::make_early_inc_range(llvm::reverse(MBB))) {797 if (MI.getOpcode() == AMDGPU::V_MOV_B32_dpp && combineDPPMov(MI)) {798 Changed = true;799 ++NumDPPMovsCombined;800 } else if (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO ||801 MI.getOpcode() == AMDGPU::V_MOV_B64_dpp) {802 if (ST->hasDPALU_DPP() && combineDPPMov(MI)) {803 Changed = true;804 ++NumDPPMovsCombined;805 } else {806 auto Split = TII->expandMovDPP64(MI);807 for (auto *M : {Split.first, Split.second}) {808 if (M && combineDPPMov(*M))809 ++NumDPPMovsCombined;810 }811 Changed = true;812 }813 }814 }815 }816 return Changed;817}818 819PreservedAnalyses GCNDPPCombinePass::run(MachineFunction &MF,820 MachineFunctionAnalysisManager &) {821 MFPropsModifier _(*this, MF);822 823 if (MF.getFunction().hasOptNone())824 return PreservedAnalyses::all();825 826 bool Changed = GCNDPPCombine().run(MF);827 if (!Changed)828 return PreservedAnalyses::all();829 830 auto PA = getMachineFunctionPassPreservedAnalyses();831 PA.preserveSet<CFGAnalyses>();832 return PA;833}834