1128 lines · cpp
1//===-- SIShrinkInstructions.cpp - Shrink Instructions --------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7/// The pass tries to use the 32-bit encoding for instructions when possible.8//===----------------------------------------------------------------------===//9//10 11#include "SIShrinkInstructions.h"12#include "AMDGPU.h"13#include "GCNSubtarget.h"14#include "MCTargetDesc/AMDGPUMCTargetDesc.h"15#include "Utils/AMDGPUBaseInfo.h"16#include "llvm/ADT/Statistic.h"17#include "llvm/CodeGen/MachineFunctionPass.h"18 19#define DEBUG_TYPE "si-shrink-instructions"20 21STATISTIC(NumInstructionsShrunk,22 "Number of 64-bit instruction reduced to 32-bit.");23STATISTIC(NumLiteralConstantsFolded,24 "Number of literal constants folded into 32-bit instructions.");25 26using namespace llvm;27 28namespace {29 30class SIShrinkInstructions {31 MachineFunction *MF;32 MachineRegisterInfo *MRI;33 const GCNSubtarget *ST;34 const SIInstrInfo *TII;35 const SIRegisterInfo *TRI;36 bool IsPostRA;37 38 bool foldImmediates(MachineInstr &MI, bool TryToCommute = true) const;39 bool shouldShrinkTrue16(MachineInstr &MI) const;40 bool isKImmOperand(const MachineOperand &Src) const;41 bool isKUImmOperand(const MachineOperand &Src) const;42 bool isKImmOrKUImmOperand(const MachineOperand &Src, bool &IsUnsigned) const;43 void copyExtraImplicitOps(MachineInstr &NewMI, MachineInstr &MI) const;44 void shrinkScalarCompare(MachineInstr &MI) const;45 void shrinkMIMG(MachineInstr &MI) const;46 void shrinkMadFma(MachineInstr &MI) const;47 bool shrinkScalarLogicOp(MachineInstr &MI) const;48 bool tryReplaceDeadSDST(MachineInstr &MI) const;49 bool instAccessReg(iterator_range<MachineInstr::const_mop_iterator> &&R,50 Register Reg, unsigned SubReg) const;51 bool instReadsReg(const MachineInstr *MI, unsigned Reg,52 unsigned SubReg) const;53 bool instModifiesReg(const MachineInstr *MI, unsigned Reg,54 unsigned SubReg) const;55 TargetInstrInfo::RegSubRegPair getSubRegForIndex(Register Reg, unsigned Sub,56 unsigned I) const;57 void dropInstructionKeepingImpDefs(MachineInstr &MI) const;58 MachineInstr *matchSwap(MachineInstr &MovT) const;59 60public:61 SIShrinkInstructions() = default;62 bool run(MachineFunction &MF);63};64 65class SIShrinkInstructionsLegacy : public MachineFunctionPass {66 67public:68 static char ID;69 70 SIShrinkInstructionsLegacy() : MachineFunctionPass(ID) {}71 72 bool runOnMachineFunction(MachineFunction &MF) override;73 74 StringRef getPassName() const override { return "SI Shrink Instructions"; }75 76 void getAnalysisUsage(AnalysisUsage &AU) const override {77 AU.setPreservesCFG();78 MachineFunctionPass::getAnalysisUsage(AU);79 }80};81 82} // End anonymous namespace.83 84INITIALIZE_PASS(SIShrinkInstructionsLegacy, DEBUG_TYPE,85 "SI Shrink Instructions", false, false)86 87char SIShrinkInstructionsLegacy::ID = 0;88 89FunctionPass *llvm::createSIShrinkInstructionsLegacyPass() {90 return new SIShrinkInstructionsLegacy();91}92 93/// This function checks \p MI for operands defined by a move immediate94/// instruction and then folds the literal constant into the instruction if it95/// can. This function assumes that \p MI is a VOP1, VOP2, or VOPC instructions.96bool SIShrinkInstructions::foldImmediates(MachineInstr &MI,97 bool TryToCommute) const {98 assert(TII->isVOP1(MI) || TII->isVOP2(MI) || TII->isVOPC(MI));99 100 int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0);101 102 // Try to fold Src0103 MachineOperand &Src0 = MI.getOperand(Src0Idx);104 if (Src0.isReg()) {105 Register Reg = Src0.getReg();106 if (Reg.isVirtual()) {107 MachineInstr *Def = MRI->getUniqueVRegDef(Reg);108 if (Def && Def->isMoveImmediate()) {109 MachineOperand &MovSrc = Def->getOperand(1);110 bool ConstantFolded = false;111 112 if (TII->isOperandLegal(MI, Src0Idx, &MovSrc)) {113 if (MovSrc.isImm()) {114 Src0.ChangeToImmediate(MovSrc.getImm());115 ConstantFolded = true;116 } else if (MovSrc.isFI()) {117 Src0.ChangeToFrameIndex(MovSrc.getIndex());118 ConstantFolded = true;119 } else if (MovSrc.isGlobal()) {120 Src0.ChangeToGA(MovSrc.getGlobal(), MovSrc.getOffset(),121 MovSrc.getTargetFlags());122 ConstantFolded = true;123 }124 }125 126 if (ConstantFolded) {127 if (MRI->use_nodbg_empty(Reg))128 Def->eraseFromParent();129 ++NumLiteralConstantsFolded;130 return true;131 }132 }133 }134 }135 136 // We have failed to fold src0, so commute the instruction and try again.137 if (TryToCommute && MI.isCommutable()) {138 if (TII->commuteInstruction(MI)) {139 if (foldImmediates(MI, false))140 return true;141 142 // Commute back.143 TII->commuteInstruction(MI);144 }145 }146 147 return false;148}149 150/// Do not shrink the instruction if its registers are not expressible in the151/// shrunk encoding.152bool SIShrinkInstructions::shouldShrinkTrue16(MachineInstr &MI) const {153 for (unsigned I = 0, E = MI.getNumExplicitOperands(); I != E; ++I) {154 const MachineOperand &MO = MI.getOperand(I);155 if (MO.isReg()) {156 Register Reg = MO.getReg();157 assert(!Reg.isVirtual() && "Prior checks should ensure we only shrink "158 "True16 Instructions post-RA");159 if (AMDGPU::VGPR_32RegClass.contains(Reg) &&160 !AMDGPU::VGPR_32_Lo128RegClass.contains(Reg))161 return false;162 163 if (AMDGPU::VGPR_16RegClass.contains(Reg) &&164 !AMDGPU::VGPR_16_Lo128RegClass.contains(Reg))165 return false;166 }167 }168 return true;169}170 171bool SIShrinkInstructions::isKImmOperand(const MachineOperand &Src) const {172 return isInt<16>(SignExtend64(Src.getImm(), 32)) &&173 !TII->isInlineConstant(*Src.getParent(), Src.getOperandNo());174}175 176bool SIShrinkInstructions::isKUImmOperand(const MachineOperand &Src) const {177 return isUInt<16>(Src.getImm()) &&178 !TII->isInlineConstant(*Src.getParent(), Src.getOperandNo());179}180 181bool SIShrinkInstructions::isKImmOrKUImmOperand(const MachineOperand &Src,182 bool &IsUnsigned) const {183 if (isInt<16>(SignExtend64(Src.getImm(), 32))) {184 IsUnsigned = false;185 return !TII->isInlineConstant(Src);186 }187 188 if (isUInt<16>(Src.getImm())) {189 IsUnsigned = true;190 return !TII->isInlineConstant(Src);191 }192 193 return false;194}195 196/// \returns the opcode of an instruction a move immediate of the constant \p197/// Src can be replaced with if the constant is replaced with \p ModifiedImm.198/// i.e.199///200/// If the bitreverse of a constant is an inline immediate, reverse the201/// immediate and return the bitreverse opcode.202///203/// If the bitwise negation of a constant is an inline immediate, reverse the204/// immediate and return the bitwise not opcode.205static unsigned canModifyToInlineImmOp32(const SIInstrInfo *TII,206 const MachineOperand &Src,207 int32_t &ModifiedImm, bool Scalar) {208 if (TII->isInlineConstant(Src))209 return 0;210 int32_t SrcImm = static_cast<int32_t>(Src.getImm());211 212 if (!Scalar) {213 // We could handle the scalar case with here, but we would need to check214 // that SCC is not live as S_NOT_B32 clobbers it. It's probably not worth215 // it, as the reasonable values are already covered by s_movk_i32.216 ModifiedImm = ~SrcImm;217 if (TII->isInlineConstant(APInt(32, ModifiedImm, true)))218 return AMDGPU::V_NOT_B32_e32;219 }220 221 ModifiedImm = reverseBits<int32_t>(SrcImm);222 if (TII->isInlineConstant(APInt(32, ModifiedImm, true)))223 return Scalar ? AMDGPU::S_BREV_B32 : AMDGPU::V_BFREV_B32_e32;224 225 return 0;226}227 228/// Copy implicit register operands from specified instruction to this229/// instruction that are not part of the instruction definition.230void SIShrinkInstructions::copyExtraImplicitOps(MachineInstr &NewMI,231 MachineInstr &MI) const {232 MachineFunction &MF = *MI.getMF();233 for (unsigned i = MI.getDesc().getNumOperands() +234 MI.getDesc().implicit_uses().size() +235 MI.getDesc().implicit_defs().size(),236 e = MI.getNumOperands();237 i != e; ++i) {238 const MachineOperand &MO = MI.getOperand(i);239 if ((MO.isReg() && MO.isImplicit()) || MO.isRegMask())240 NewMI.addOperand(MF, MO);241 }242}243 244void SIShrinkInstructions::shrinkScalarCompare(MachineInstr &MI) const {245 if (!ST->hasSCmpK())246 return;247 248 // cmpk instructions do scc = dst <cc op> imm16, so commute the instruction to249 // get constants on the RHS.250 if (!MI.getOperand(0).isReg())251 TII->commuteInstruction(MI, false, 0, 1);252 253 // cmpk requires src0 to be a register254 const MachineOperand &Src0 = MI.getOperand(0);255 if (!Src0.isReg())256 return;257 258 MachineOperand &Src1 = MI.getOperand(1);259 if (!Src1.isImm())260 return;261 262 int SOPKOpc = AMDGPU::getSOPKOp(MI.getOpcode());263 if (SOPKOpc == -1)264 return;265 266 // eq/ne is special because the imm16 can be treated as signed or unsigned,267 // and initially selected to the unsigned versions.268 if (SOPKOpc == AMDGPU::S_CMPK_EQ_U32 || SOPKOpc == AMDGPU::S_CMPK_LG_U32) {269 bool HasUImm;270 if (isKImmOrKUImmOperand(Src1, HasUImm)) {271 if (!HasUImm) {272 SOPKOpc = (SOPKOpc == AMDGPU::S_CMPK_EQ_U32) ?273 AMDGPU::S_CMPK_EQ_I32 : AMDGPU::S_CMPK_LG_I32;274 Src1.setImm(SignExtend32(Src1.getImm(), 32));275 }276 277 MI.setDesc(TII->get(SOPKOpc));278 }279 280 return;281 }282 283 const MCInstrDesc &NewDesc = TII->get(SOPKOpc);284 285 if ((SIInstrInfo::sopkIsZext(SOPKOpc) && isKUImmOperand(Src1)) ||286 (!SIInstrInfo::sopkIsZext(SOPKOpc) && isKImmOperand(Src1))) {287 if (!SIInstrInfo::sopkIsZext(SOPKOpc))288 Src1.setImm(SignExtend64(Src1.getImm(), 32));289 MI.setDesc(NewDesc);290 }291}292 293// Shrink NSA encoded instructions with contiguous VGPRs to non-NSA encoding.294void SIShrinkInstructions::shrinkMIMG(MachineInstr &MI) const {295 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(MI.getOpcode());296 if (!Info)297 return;298 299 uint8_t NewEncoding;300 switch (Info->MIMGEncoding) {301 case AMDGPU::MIMGEncGfx10NSA:302 NewEncoding = AMDGPU::MIMGEncGfx10Default;303 break;304 case AMDGPU::MIMGEncGfx11NSA:305 NewEncoding = AMDGPU::MIMGEncGfx11Default;306 break;307 default:308 return;309 }310 311 int VAddr0Idx =312 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vaddr0);313 unsigned NewAddrDwords = Info->VAddrDwords;314 const TargetRegisterClass *RC;315 316 if (Info->VAddrDwords == 2) {317 RC = &AMDGPU::VReg_64RegClass;318 } else if (Info->VAddrDwords == 3) {319 RC = &AMDGPU::VReg_96RegClass;320 } else if (Info->VAddrDwords == 4) {321 RC = &AMDGPU::VReg_128RegClass;322 } else if (Info->VAddrDwords == 5) {323 RC = &AMDGPU::VReg_160RegClass;324 } else if (Info->VAddrDwords == 6) {325 RC = &AMDGPU::VReg_192RegClass;326 } else if (Info->VAddrDwords == 7) {327 RC = &AMDGPU::VReg_224RegClass;328 } else if (Info->VAddrDwords == 8) {329 RC = &AMDGPU::VReg_256RegClass;330 } else if (Info->VAddrDwords == 9) {331 RC = &AMDGPU::VReg_288RegClass;332 } else if (Info->VAddrDwords == 10) {333 RC = &AMDGPU::VReg_320RegClass;334 } else if (Info->VAddrDwords == 11) {335 RC = &AMDGPU::VReg_352RegClass;336 } else if (Info->VAddrDwords == 12) {337 RC = &AMDGPU::VReg_384RegClass;338 } else {339 RC = &AMDGPU::VReg_512RegClass;340 NewAddrDwords = 16;341 }342 343 unsigned VgprBase = 0;344 unsigned NextVgpr = 0;345 bool IsUndef = true;346 bool IsKill = NewAddrDwords == Info->VAddrDwords;347 const unsigned NSAMaxSize = ST->getNSAMaxSize();348 const bool IsPartialNSA = NewAddrDwords > NSAMaxSize;349 const unsigned EndVAddr = IsPartialNSA ? NSAMaxSize : Info->VAddrOperands;350 for (unsigned Idx = 0; Idx < EndVAddr; ++Idx) {351 const MachineOperand &Op = MI.getOperand(VAddr0Idx + Idx);352 unsigned Vgpr = TRI->getHWRegIndex(Op.getReg());353 unsigned Dwords = TRI->getRegSizeInBits(Op.getReg(), *MRI) / 32;354 assert(Dwords > 0 && "Un-implemented for less than 32 bit regs");355 356 if (Idx == 0) {357 VgprBase = Vgpr;358 NextVgpr = Vgpr + Dwords;359 } else if (Vgpr == NextVgpr) {360 NextVgpr = Vgpr + Dwords;361 } else {362 return;363 }364 365 if (!Op.isUndef())366 IsUndef = false;367 if (!Op.isKill())368 IsKill = false;369 }370 371 if (VgprBase + NewAddrDwords > 256)372 return;373 374 // Further check for implicit tied operands - this may be present if TFE is375 // enabled376 int TFEIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::tfe);377 int LWEIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::lwe);378 unsigned TFEVal = (TFEIdx == -1) ? 0 : MI.getOperand(TFEIdx).getImm();379 unsigned LWEVal = (LWEIdx == -1) ? 0 : MI.getOperand(LWEIdx).getImm();380 int ToUntie = -1;381 if (TFEVal || LWEVal) {382 // TFE/LWE is enabled so we need to deal with an implicit tied operand383 for (unsigned i = LWEIdx + 1, e = MI.getNumOperands(); i != e; ++i) {384 if (MI.getOperand(i).isReg() && MI.getOperand(i).isTied() &&385 MI.getOperand(i).isImplicit()) {386 // This is the tied operand387 assert(388 ToUntie == -1 &&389 "found more than one tied implicit operand when expecting only 1");390 ToUntie = i;391 MI.untieRegOperand(ToUntie);392 }393 }394 }395 396 unsigned NewOpcode = AMDGPU::getMIMGOpcode(Info->BaseOpcode, NewEncoding,397 Info->VDataDwords, NewAddrDwords);398 MI.setDesc(TII->get(NewOpcode));399 MI.getOperand(VAddr0Idx).setReg(RC->getRegister(VgprBase));400 MI.getOperand(VAddr0Idx).setIsUndef(IsUndef);401 MI.getOperand(VAddr0Idx).setIsKill(IsKill);402 403 for (unsigned i = 1; i < EndVAddr; ++i)404 MI.removeOperand(VAddr0Idx + 1);405 406 if (ToUntie >= 0) {407 MI.tieOperands(408 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata),409 ToUntie - (EndVAddr - 1));410 }411}412 413// Shrink MAD to MADAK/MADMK and FMA to FMAAK/FMAMK.414void SIShrinkInstructions::shrinkMadFma(MachineInstr &MI) const {415 // Pre-GFX10 VOP3 instructions like MAD/FMA cannot take a literal operand so416 // there is no reason to try to shrink them.417 if (!ST->hasVOP3Literal())418 return;419 420 // There is no advantage to doing this pre-RA.421 if (!IsPostRA)422 return;423 424 if (TII->hasAnyModifiersSet(MI))425 return;426 427 const unsigned Opcode = MI.getOpcode();428 MachineOperand &Src0 = *TII->getNamedOperand(MI, AMDGPU::OpName::src0);429 MachineOperand &Src1 = *TII->getNamedOperand(MI, AMDGPU::OpName::src1);430 MachineOperand &Src2 = *TII->getNamedOperand(MI, AMDGPU::OpName::src2);431 unsigned NewOpcode = AMDGPU::INSTRUCTION_LIST_END;432 433 bool Swap;434 435 // Detect "Dst = VSrc * VGPR + Imm" and convert to AK form.436 if (Src2.isImm() && !TII->isInlineConstant(Src2)) {437 if (Src1.isReg() && TRI->isVGPR(*MRI, Src1.getReg()))438 Swap = false;439 else if (Src0.isReg() && TRI->isVGPR(*MRI, Src0.getReg()))440 Swap = true;441 else442 return;443 444 switch (Opcode) {445 default:446 llvm_unreachable("Unexpected mad/fma opcode!");447 case AMDGPU::V_MAD_F32_e64:448 NewOpcode = AMDGPU::V_MADAK_F32;449 break;450 case AMDGPU::V_FMA_F32_e64:451 NewOpcode = AMDGPU::V_FMAAK_F32;452 break;453 case AMDGPU::V_MAD_F16_e64:454 NewOpcode = AMDGPU::V_MADAK_F16;455 break;456 case AMDGPU::V_FMA_F16_e64:457 case AMDGPU::V_FMA_F16_gfx9_e64:458 NewOpcode = AMDGPU::V_FMAAK_F16;459 break;460 case AMDGPU::V_FMA_F16_gfx9_t16_e64:461 NewOpcode = AMDGPU::V_FMAAK_F16_t16;462 break;463 case AMDGPU::V_FMA_F16_gfx9_fake16_e64:464 NewOpcode = AMDGPU::V_FMAAK_F16_fake16;465 break;466 case AMDGPU::V_FMA_F64_e64:467 if (ST->hasFmaakFmamkF64Insts())468 NewOpcode = AMDGPU::V_FMAAK_F64;469 break;470 }471 }472 473 // Detect "Dst = VSrc * Imm + VGPR" and convert to MK form.474 if (Src2.isReg() && TRI->isVGPR(*MRI, Src2.getReg())) {475 if (Src1.isImm() && !TII->isInlineConstant(Src1))476 Swap = false;477 else if (Src0.isImm() && !TII->isInlineConstant(Src0))478 Swap = true;479 else480 return;481 482 switch (Opcode) {483 default:484 llvm_unreachable("Unexpected mad/fma opcode!");485 case AMDGPU::V_MAD_F32_e64:486 NewOpcode = AMDGPU::V_MADMK_F32;487 break;488 case AMDGPU::V_FMA_F32_e64:489 NewOpcode = AMDGPU::V_FMAMK_F32;490 break;491 case AMDGPU::V_MAD_F16_e64:492 NewOpcode = AMDGPU::V_MADMK_F16;493 break;494 case AMDGPU::V_FMA_F16_e64:495 case AMDGPU::V_FMA_F16_gfx9_e64:496 NewOpcode = AMDGPU::V_FMAMK_F16;497 break;498 case AMDGPU::V_FMA_F16_gfx9_t16_e64:499 NewOpcode = AMDGPU::V_FMAMK_F16_t16;500 break;501 case AMDGPU::V_FMA_F16_gfx9_fake16_e64:502 NewOpcode = AMDGPU::V_FMAMK_F16_fake16;503 break;504 case AMDGPU::V_FMA_F64_e64:505 if (ST->hasFmaakFmamkF64Insts())506 NewOpcode = AMDGPU::V_FMAMK_F64;507 break;508 }509 }510 511 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END)512 return;513 514 if (AMDGPU::isTrue16Inst(NewOpcode) && !shouldShrinkTrue16(MI))515 return;516 517 if (Swap) {518 // Swap Src0 and Src1 by building a new instruction.519 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII->get(NewOpcode),520 MI.getOperand(0).getReg())521 .add(Src1)522 .add(Src0)523 .add(Src2)524 .setMIFlags(MI.getFlags());525 MI.eraseFromParent();526 } else {527 TII->removeModOperands(MI);528 MI.setDesc(TII->get(NewOpcode));529 }530}531 532/// Attempt to shrink AND/OR/XOR operations requiring non-inlineable literals.533/// For AND or OR, try using S_BITSET{0,1} to clear or set bits.534/// If the inverse of the immediate is legal, use ANDN2, ORN2 or535/// XNOR (as a ^ b == ~(a ^ ~b)).536/// \returns true if the caller should continue the machine function iterator537bool SIShrinkInstructions::shrinkScalarLogicOp(MachineInstr &MI) const {538 unsigned Opc = MI.getOpcode();539 const MachineOperand *Dest = &MI.getOperand(0);540 MachineOperand *Src0 = &MI.getOperand(1);541 MachineOperand *Src1 = &MI.getOperand(2);542 MachineOperand *SrcReg = Src0;543 MachineOperand *SrcImm = Src1;544 545 if (!SrcImm->isImm() ||546 AMDGPU::isInlinableLiteral32(SrcImm->getImm(), ST->hasInv2PiInlineImm()))547 return false;548 549 uint32_t Imm = static_cast<uint32_t>(SrcImm->getImm());550 uint32_t NewImm = 0;551 552 if (Opc == AMDGPU::S_AND_B32) {553 if (isPowerOf2_32(~Imm) &&554 MI.findRegisterDefOperand(AMDGPU::SCC, /*TRI=*/nullptr)->isDead()) {555 NewImm = llvm::countr_one(Imm);556 Opc = AMDGPU::S_BITSET0_B32;557 } else if (AMDGPU::isInlinableLiteral32(~Imm, ST->hasInv2PiInlineImm())) {558 NewImm = ~Imm;559 Opc = AMDGPU::S_ANDN2_B32;560 }561 } else if (Opc == AMDGPU::S_OR_B32) {562 if (isPowerOf2_32(Imm) &&563 MI.findRegisterDefOperand(AMDGPU::SCC, /*TRI=*/nullptr)->isDead()) {564 NewImm = llvm::countr_zero(Imm);565 Opc = AMDGPU::S_BITSET1_B32;566 } else if (AMDGPU::isInlinableLiteral32(~Imm, ST->hasInv2PiInlineImm())) {567 NewImm = ~Imm;568 Opc = AMDGPU::S_ORN2_B32;569 }570 } else if (Opc == AMDGPU::S_XOR_B32) {571 if (AMDGPU::isInlinableLiteral32(~Imm, ST->hasInv2PiInlineImm())) {572 NewImm = ~Imm;573 Opc = AMDGPU::S_XNOR_B32;574 }575 } else {576 llvm_unreachable("unexpected opcode");577 }578 579 if (NewImm != 0) {580 if (Dest->getReg().isVirtual() && SrcReg->isReg()) {581 MRI->setRegAllocationHint(Dest->getReg(), 0, SrcReg->getReg());582 MRI->setRegAllocationHint(SrcReg->getReg(), 0, Dest->getReg());583 return true;584 }585 586 if (SrcReg->isReg() && SrcReg->getReg() == Dest->getReg()) {587 const bool IsUndef = SrcReg->isUndef();588 const bool IsKill = SrcReg->isKill();589 TII->mutateAndCleanupImplicit(MI, TII->get(Opc));590 if (Opc == AMDGPU::S_BITSET0_B32 ||591 Opc == AMDGPU::S_BITSET1_B32) {592 Src0->ChangeToImmediate(NewImm);593 // Remove the immediate and add the tied input.594 MI.getOperand(2).ChangeToRegister(Dest->getReg(), /*IsDef*/ false,595 /*isImp*/ false, IsKill,596 /*isDead*/ false, IsUndef);597 MI.tieOperands(0, 2);598 } else {599 SrcImm->setImm(NewImm);600 }601 }602 }603 604 return false;605}606 607// This is the same as MachineInstr::readsRegister/modifiesRegister except608// it takes subregs into account.609bool SIShrinkInstructions::instAccessReg(610 iterator_range<MachineInstr::const_mop_iterator> &&R, Register Reg,611 unsigned SubReg) const {612 for (const MachineOperand &MO : R) {613 if (!MO.isReg())614 continue;615 616 if (Reg.isPhysical() && MO.getReg().isPhysical()) {617 if (TRI->regsOverlap(Reg, MO.getReg()))618 return true;619 } else if (MO.getReg() == Reg && Reg.isVirtual()) {620 LaneBitmask Overlap = TRI->getSubRegIndexLaneMask(SubReg) &621 TRI->getSubRegIndexLaneMask(MO.getSubReg());622 if (Overlap.any())623 return true;624 }625 }626 return false;627}628 629bool SIShrinkInstructions::instReadsReg(const MachineInstr *MI, unsigned Reg,630 unsigned SubReg) const {631 return instAccessReg(MI->uses(), Reg, SubReg);632}633 634bool SIShrinkInstructions::instModifiesReg(const MachineInstr *MI, unsigned Reg,635 unsigned SubReg) const {636 return instAccessReg(MI->defs(), Reg, SubReg);637}638 639TargetInstrInfo::RegSubRegPair640SIShrinkInstructions::getSubRegForIndex(Register Reg, unsigned Sub,641 unsigned I) const {642 if (TRI->getRegSizeInBits(Reg, *MRI) != 32) {643 if (Reg.isPhysical()) {644 Reg = TRI->getSubReg(Reg, TRI->getSubRegFromChannel(I));645 } else {646 Sub = TRI->getSubRegFromChannel(I + TRI->getChannelFromSubReg(Sub));647 }648 }649 return TargetInstrInfo::RegSubRegPair(Reg, Sub);650}651 652void SIShrinkInstructions::dropInstructionKeepingImpDefs(653 MachineInstr &MI) const {654 for (unsigned i = MI.getDesc().getNumOperands() +655 MI.getDesc().implicit_uses().size() +656 MI.getDesc().implicit_defs().size(),657 e = MI.getNumOperands();658 i != e; ++i) {659 const MachineOperand &Op = MI.getOperand(i);660 if (!Op.isDef())661 continue;662 BuildMI(*MI.getParent(), MI.getIterator(), MI.getDebugLoc(),663 TII->get(AMDGPU::IMPLICIT_DEF), Op.getReg());664 }665 666 MI.eraseFromParent();667}668 669// Match:670// mov t, x671// mov x, y672// mov y, t673//674// =>675//676// mov t, x (t is potentially dead and move eliminated)677// v_swap_b32 x, y678//679// Returns next valid instruction pointer if was able to create v_swap_b32.680//681// This shall not be done too early not to prevent possible folding which may682// remove matched moves, and this should preferably be done before RA to683// release saved registers and also possibly after RA which can insert copies684// too.685//686// This is really just a generic peephole that is not a canonical shrinking,687// although requirements match the pass placement and it reduces code size too.688MachineInstr *SIShrinkInstructions::matchSwap(MachineInstr &MovT) const {689 assert(MovT.getOpcode() == AMDGPU::V_MOV_B32_e32 ||690 MovT.getOpcode() == AMDGPU::V_MOV_B16_t16_e32 ||691 MovT.getOpcode() == AMDGPU::COPY);692 693 Register T = MovT.getOperand(0).getReg();694 unsigned Tsub = MovT.getOperand(0).getSubReg();695 MachineOperand &Xop = MovT.getOperand(1);696 697 if (!Xop.isReg())698 return nullptr;699 Register X = Xop.getReg();700 unsigned Xsub = Xop.getSubReg();701 702 unsigned Size = TII->getOpSize(MovT, 0);703 704 // We can't match v_swap_b16 pre-RA, because VGPR_16_Lo128 registers705 // are not allocatble.706 if (Size == 2 && X.isVirtual())707 return nullptr;708 709 if (!TRI->isVGPR(*MRI, X))710 return nullptr;711 712 const unsigned SearchLimit = 16;713 unsigned Count = 0;714 bool KilledT = false;715 for (auto Iter = std::next(MovT.getIterator()),716 E = MovT.getParent()->instr_end();717 Iter != E && Count < SearchLimit && !KilledT; ++Iter) {718 719 MachineInstr *MovY = &*Iter;720 KilledT = MovY->killsRegister(T, TRI);721 if (MovY->isDebugInstr())722 continue;723 ++Count;724 725 if ((MovY->getOpcode() != AMDGPU::V_MOV_B32_e32 &&726 MovY->getOpcode() != AMDGPU::V_MOV_B16_t16_e32 &&727 MovY->getOpcode() != AMDGPU::COPY) ||728 !MovY->getOperand(1).isReg() || MovY->getOperand(1).getReg() != T ||729 MovY->getOperand(1).getSubReg() != Tsub)730 continue;731 732 Register Y = MovY->getOperand(0).getReg();733 unsigned Ysub = MovY->getOperand(0).getSubReg();734 735 if (!TRI->isVGPR(*MRI, Y))736 continue;737 738 MachineInstr *MovX = nullptr;739 for (auto IY = MovY->getIterator(), I = std::next(MovT.getIterator());740 I != IY; ++I) {741 if (I->isDebugInstr())742 continue;743 if (instReadsReg(&*I, X, Xsub) || instModifiesReg(&*I, Y, Ysub) ||744 instModifiesReg(&*I, T, Tsub) ||745 (MovX && instModifiesReg(&*I, X, Xsub))) {746 MovX = nullptr;747 break;748 }749 if (!instReadsReg(&*I, Y, Ysub)) {750 if (!MovX && instModifiesReg(&*I, X, Xsub)) {751 MovX = nullptr;752 break;753 }754 continue;755 }756 if (MovX ||757 (I->getOpcode() != AMDGPU::V_MOV_B32_e32 &&758 I->getOpcode() != AMDGPU::V_MOV_B16_t16_e32 &&759 I->getOpcode() != AMDGPU::COPY) ||760 I->getOperand(0).getReg() != X ||761 I->getOperand(0).getSubReg() != Xsub) {762 MovX = nullptr;763 break;764 }765 766 if (Size > 4 && (I->getNumImplicitOperands() > (I->isCopy() ? 0U : 1U)))767 continue;768 769 MovX = &*I;770 }771 772 if (!MovX)773 continue;774 775 LLVM_DEBUG(dbgs() << "Matched v_swap:\n" << MovT << *MovX << *MovY);776 777 MachineBasicBlock &MBB = *MovT.getParent();778 SmallVector<MachineInstr *, 4> Swaps;779 if (Size == 2) {780 auto *MIB = BuildMI(MBB, MovX->getIterator(), MovT.getDebugLoc(),781 TII->get(AMDGPU::V_SWAP_B16))782 .addDef(X)783 .addDef(Y)784 .addReg(Y)785 .addReg(X)786 .getInstr();787 Swaps.push_back(MIB);788 } else {789 assert(Size > 0 && Size % 4 == 0);790 for (unsigned I = 0; I < Size / 4; ++I) {791 TargetInstrInfo::RegSubRegPair X1, Y1;792 X1 = getSubRegForIndex(X, Xsub, I);793 Y1 = getSubRegForIndex(Y, Ysub, I);794 auto *MIB = BuildMI(MBB, MovX->getIterator(), MovT.getDebugLoc(),795 TII->get(AMDGPU::V_SWAP_B32))796 .addDef(X1.Reg, 0, X1.SubReg)797 .addDef(Y1.Reg, 0, Y1.SubReg)798 .addReg(Y1.Reg, 0, Y1.SubReg)799 .addReg(X1.Reg, 0, X1.SubReg)800 .getInstr();801 Swaps.push_back(MIB);802 }803 }804 // Drop implicit EXEC.805 if (MovX->hasRegisterImplicitUseOperand(AMDGPU::EXEC)) {806 for (MachineInstr *Swap : Swaps) {807 Swap->removeOperand(Swap->getNumExplicitOperands());808 Swap->copyImplicitOps(*MBB.getParent(), *MovX);809 }810 }811 MovX->eraseFromParent();812 dropInstructionKeepingImpDefs(*MovY);813 MachineInstr *Next = &*std::next(MovT.getIterator());814 815 if (T.isVirtual() && MRI->use_nodbg_empty(T)) {816 dropInstructionKeepingImpDefs(MovT);817 } else {818 Xop.setIsKill(false);819 for (int I = MovT.getNumImplicitOperands() - 1; I >= 0; --I ) {820 unsigned OpNo = MovT.getNumExplicitOperands() + I;821 const MachineOperand &Op = MovT.getOperand(OpNo);822 if (Op.isKill() && TRI->regsOverlap(X, Op.getReg()))823 MovT.removeOperand(OpNo);824 }825 }826 827 return Next;828 }829 830 return nullptr;831}832 833// If an instruction has dead sdst replace it with NULL register on gfx1030+834bool SIShrinkInstructions::tryReplaceDeadSDST(MachineInstr &MI) const {835 if (!ST->hasGFX10_3Insts())836 return false;837 838 MachineOperand *Op = TII->getNamedOperand(MI, AMDGPU::OpName::sdst);839 if (!Op)840 return false;841 Register SDstReg = Op->getReg();842 if (SDstReg.isPhysical() || !MRI->use_nodbg_empty(SDstReg))843 return false;844 845 Op->setReg(ST->isWave32() ? AMDGPU::SGPR_NULL : AMDGPU::SGPR_NULL64);846 return true;847}848 849bool SIShrinkInstructions::run(MachineFunction &MF) {850 851 this->MF = &MF;852 MRI = &MF.getRegInfo();853 ST = &MF.getSubtarget<GCNSubtarget>();854 TII = ST->getInstrInfo();855 TRI = &TII->getRegisterInfo();856 IsPostRA = MF.getProperties().hasNoVRegs();857 858 unsigned VCCReg = ST->isWave32() ? AMDGPU::VCC_LO : AMDGPU::VCC;859 860 for (MachineBasicBlock &MBB : MF) {861 MachineBasicBlock::iterator I, Next;862 for (I = MBB.begin(); I != MBB.end(); I = Next) {863 Next = std::next(I);864 MachineInstr &MI = *I;865 866 if (MI.getOpcode() == AMDGPU::V_MOV_B32_e32) {867 // If this has a literal constant source that is the same as the868 // reversed bits of an inline immediate, replace with a bitreverse of869 // that constant. This saves 4 bytes in the common case of materializing870 // sign bits.871 872 // Test if we are after regalloc. We only want to do this after any873 // optimizations happen because this will confuse them.874 MachineOperand &Src = MI.getOperand(1);875 if (Src.isImm() && IsPostRA) {876 int32_t ModImm;877 unsigned ModOpcode =878 canModifyToInlineImmOp32(TII, Src, ModImm, /*Scalar=*/false);879 if (ModOpcode != 0) {880 MI.setDesc(TII->get(ModOpcode));881 Src.setImm(static_cast<int64_t>(ModImm));882 continue;883 }884 }885 }886 887 if (ST->hasSwap() && (MI.getOpcode() == AMDGPU::V_MOV_B32_e32 ||888 MI.getOpcode() == AMDGPU::V_MOV_B16_t16_e32 ||889 MI.getOpcode() == AMDGPU::COPY)) {890 if (auto *NextMI = matchSwap(MI)) {891 Next = NextMI->getIterator();892 continue;893 }894 }895 896 // Try to use S_ADDK_I32 and S_MULK_I32.897 if (MI.getOpcode() == AMDGPU::S_ADD_I32 ||898 MI.getOpcode() == AMDGPU::S_MUL_I32) {899 const MachineOperand *Dest = &MI.getOperand(0);900 MachineOperand *Src0 = &MI.getOperand(1);901 MachineOperand *Src1 = &MI.getOperand(2);902 903 if (!Src0->isReg() && Src1->isReg()) {904 if (TII->commuteInstruction(MI, false, 1, 2))905 std::swap(Src0, Src1);906 }907 908 // FIXME: This could work better if hints worked with subregisters. If909 // we have a vector add of a constant, we usually don't get the correct910 // allocation due to the subregister usage.911 if (Dest->getReg().isVirtual() && Src0->isReg()) {912 MRI->setRegAllocationHint(Dest->getReg(), 0, Src0->getReg());913 MRI->setRegAllocationHint(Src0->getReg(), 0, Dest->getReg());914 continue;915 }916 917 if (Src0->isReg() && Src0->getReg() == Dest->getReg()) {918 if (Src1->isImm() && isKImmOperand(*Src1)) {919 unsigned Opc = (MI.getOpcode() == AMDGPU::S_ADD_I32) ?920 AMDGPU::S_ADDK_I32 : AMDGPU::S_MULK_I32;921 922 Src1->setImm(SignExtend64(Src1->getImm(), 32));923 MI.setDesc(TII->get(Opc));924 MI.tieOperands(0, 1);925 }926 }927 }928 929 // Try to use s_cmpk_*930 if (MI.isCompare() && TII->isSOPC(MI)) {931 shrinkScalarCompare(MI);932 continue;933 }934 935 // Try to use S_MOVK_I32, which will save 4 bytes for small immediates.936 if (MI.getOpcode() == AMDGPU::S_MOV_B32) {937 const MachineOperand &Dst = MI.getOperand(0);938 MachineOperand &Src = MI.getOperand(1);939 940 if (Src.isImm() && Dst.getReg().isPhysical()) {941 unsigned ModOpc;942 int32_t ModImm;943 if (isKImmOperand(Src)) {944 MI.setDesc(TII->get(AMDGPU::S_MOVK_I32));945 Src.setImm(SignExtend64(Src.getImm(), 32));946 } else if ((ModOpc = canModifyToInlineImmOp32(TII, Src, ModImm,947 /*Scalar=*/true))) {948 MI.setDesc(TII->get(ModOpc));949 Src.setImm(static_cast<int64_t>(ModImm));950 }951 }952 953 continue;954 }955 956 // Shrink scalar logic operations.957 if (MI.getOpcode() == AMDGPU::S_AND_B32 ||958 MI.getOpcode() == AMDGPU::S_OR_B32 ||959 MI.getOpcode() == AMDGPU::S_XOR_B32) {960 if (shrinkScalarLogicOp(MI))961 continue;962 }963 964 if (IsPostRA && TII->isMIMG(MI.getOpcode()) &&965 ST->getGeneration() >= AMDGPUSubtarget::GFX10) {966 shrinkMIMG(MI);967 continue;968 }969 970 if (!TII->isVOP3(MI))971 continue;972 973 if (MI.getOpcode() == AMDGPU::V_MAD_F32_e64 ||974 MI.getOpcode() == AMDGPU::V_FMA_F32_e64 ||975 MI.getOpcode() == AMDGPU::V_MAD_F16_e64 ||976 MI.getOpcode() == AMDGPU::V_FMA_F16_e64 ||977 MI.getOpcode() == AMDGPU::V_FMA_F16_gfx9_e64 ||978 MI.getOpcode() == AMDGPU::V_FMA_F16_gfx9_t16_e64 ||979 MI.getOpcode() == AMDGPU::V_FMA_F16_gfx9_fake16_e64 ||980 (MI.getOpcode() == AMDGPU::V_FMA_F64_e64 &&981 ST->hasFmaakFmamkF64Insts())) {982 shrinkMadFma(MI);983 continue;984 }985 986 // If there is no chance we will shrink it and use VCC as sdst to get987 // a 32 bit form try to replace dead sdst with NULL.988 if (TII->isVOP3(MI.getOpcode())) {989 tryReplaceDeadSDST(MI);990 if (!TII->hasVALU32BitEncoding(MI.getOpcode())) {991 continue;992 }993 }994 995 if (!TII->canShrink(MI, *MRI)) {996 // Try commuting the instruction and see if that enables us to shrink997 // it.998 if (!MI.isCommutable() || !TII->commuteInstruction(MI) ||999 !TII->canShrink(MI, *MRI)) {1000 tryReplaceDeadSDST(MI);1001 continue;1002 }1003 }1004 1005 int Op32 = AMDGPU::getVOPe32(MI.getOpcode());1006 1007 if (TII->isVOPC(Op32)) {1008 MachineOperand &Op0 = MI.getOperand(0);1009 if (Op0.isReg()) {1010 // Exclude VOPCX instructions as these don't explicitly write a1011 // dst.1012 Register DstReg = Op0.getReg();1013 if (DstReg.isVirtual()) {1014 // VOPC instructions can only write to the VCC register. We can't1015 // force them to use VCC here, because this is only one register and1016 // cannot deal with sequences which would require multiple copies of1017 // VCC, e.g. S_AND_B64 (vcc = V_CMP_...), (vcc = V_CMP_...)1018 //1019 // So, instead of forcing the instruction to write to VCC, we1020 // provide a hint to the register allocator to use VCC and then we1021 // will run this pass again after RA and shrink it if it outputs to1022 // VCC.1023 MRI->setRegAllocationHint(DstReg, 0, VCCReg);1024 continue;1025 }1026 if (DstReg != VCCReg)1027 continue;1028 }1029 }1030 1031 if (Op32 == AMDGPU::V_CNDMASK_B32_e32) {1032 // We shrink V_CNDMASK_B32_e64 using regalloc hints like we do for VOPC1033 // instructions.1034 const MachineOperand *Src2 =1035 TII->getNamedOperand(MI, AMDGPU::OpName::src2);1036 if (!Src2->isReg())1037 continue;1038 Register SReg = Src2->getReg();1039 if (SReg.isVirtual()) {1040 MRI->setRegAllocationHint(SReg, 0, VCCReg);1041 continue;1042 }1043 if (SReg != VCCReg)1044 continue;1045 }1046 1047 // Check for the bool flag output for instructions like V_ADD_I32_e64.1048 const MachineOperand *SDst = TII->getNamedOperand(MI,1049 AMDGPU::OpName::sdst);1050 1051 if (SDst) {1052 bool Next = false;1053 1054 if (SDst->getReg() != VCCReg) {1055 if (SDst->getReg().isVirtual())1056 MRI->setRegAllocationHint(SDst->getReg(), 0, VCCReg);1057 Next = true;1058 }1059 1060 // All of the instructions with carry outs also have an SGPR input in1061 // src2.1062 const MachineOperand *Src2 = TII->getNamedOperand(MI,1063 AMDGPU::OpName::src2);1064 if (Src2 && Src2->getReg() != VCCReg) {1065 if (Src2->getReg().isVirtual())1066 MRI->setRegAllocationHint(Src2->getReg(), 0, VCCReg);1067 Next = true;1068 }1069 1070 if (Next)1071 continue;1072 }1073 1074 // Pre-GFX10, shrinking VOP3 instructions pre-RA gave us the chance to1075 // fold an immediate into the shrunk instruction as a literal operand. In1076 // GFX10 VOP3 instructions can take a literal operand anyway, so there is1077 // no advantage to doing this.1078 // However, if 64-bit literals are allowed we still need to shrink it1079 // for such literal to be able to fold.1080 if (ST->hasVOP3Literal() &&1081 (!ST->has64BitLiterals() || AMDGPU::isTrue16Inst(MI.getOpcode())) &&1082 !IsPostRA)1083 continue;1084 1085 if (ST->hasTrue16BitInsts() && AMDGPU::isTrue16Inst(MI.getOpcode()) &&1086 !shouldShrinkTrue16(MI))1087 continue;1088 1089 // We can shrink this instruction1090 LLVM_DEBUG(dbgs() << "Shrinking " << MI);1091 1092 MachineInstr *Inst32 = TII->buildShrunkInst(MI, Op32);1093 ++NumInstructionsShrunk;1094 1095 // Copy extra operands not present in the instruction definition.1096 copyExtraImplicitOps(*Inst32, MI);1097 1098 // Copy deadness from the old explicit vcc def to the new implicit def.1099 if (SDst && SDst->isDead())1100 Inst32->findRegisterDefOperand(VCCReg, /*TRI=*/nullptr)->setIsDead();1101 1102 MI.eraseFromParent();1103 foldImmediates(*Inst32);1104 1105 LLVM_DEBUG(dbgs() << "e32 MI = " << *Inst32 << '\n');1106 }1107 }1108 return false;1109}1110 1111bool SIShrinkInstructionsLegacy::runOnMachineFunction(MachineFunction &MF) {1112 if (skipFunction(MF.getFunction()))1113 return false;1114 1115 return SIShrinkInstructions().run(MF);1116}1117 1118PreservedAnalyses1119SIShrinkInstructionsPass::run(MachineFunction &MF,1120 MachineFunctionAnalysisManager &) {1121 if (MF.getFunction().hasOptNone() || !SIShrinkInstructions().run(MF))1122 return PreservedAnalyses::all();1123 1124 auto PA = getMachineFunctionPassPreservedAnalyses();1125 PA.preserveSet<CFGAnalyses>();1126 return PA;1127}1128