885 lines · cpp
1//===- RISCVVectorPeephole.cpp - MI Vector Pseudo Peepholes ---------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This pass performs various vector pseudo peephole optimisations after10// instruction selection.11//12// Currently it converts vmerge.vvm to vmv.v.v13// PseudoVMERGE_VVM %false, %false, %true, %allonesmask, %vl, %sew14// ->15// PseudoVMV_V_V %false, %true, %vl, %sew16//17// And masked pseudos to unmasked pseudos18// PseudoVADD_V_V_MASK %passthru, %a, %b, %allonesmask, %vl, sew, policy19// ->20// PseudoVADD_V_V %passthru %a, %b, %vl, sew, policy21//22// It also converts AVLs to VLMAX where possible23// %vl = VLENB * something24// PseudoVADD_V_V %passthru, %a, %b, %vl, sew, policy25// ->26// PseudoVADD_V_V %passthru, %a, %b, -1, sew, policy27//28//===----------------------------------------------------------------------===//29 30#include "RISCV.h"31#include "RISCVSubtarget.h"32#include "llvm/CodeGen/MachineFunctionPass.h"33#include "llvm/CodeGen/MachineRegisterInfo.h"34#include "llvm/CodeGen/TargetInstrInfo.h"35#include "llvm/CodeGen/TargetRegisterInfo.h"36 37using namespace llvm;38 39#define DEBUG_TYPE "riscv-vector-peephole"40 41namespace {42 43class RISCVVectorPeephole : public MachineFunctionPass {44public:45 static char ID;46 const TargetInstrInfo *TII;47 MachineRegisterInfo *MRI;48 const TargetRegisterInfo *TRI;49 const RISCVSubtarget *ST;50 RISCVVectorPeephole() : MachineFunctionPass(ID) {}51 52 bool runOnMachineFunction(MachineFunction &MF) override;53 MachineFunctionProperties getRequiredProperties() const override {54 return MachineFunctionProperties().setIsSSA();55 }56 57 StringRef getPassName() const override {58 return "RISC-V Vector Peephole Optimization";59 }60 61private:62 bool tryToReduceVL(MachineInstr &MI) const;63 bool convertToVLMAX(MachineInstr &MI) const;64 bool convertToWholeRegister(MachineInstr &MI) const;65 bool convertToUnmasked(MachineInstr &MI) const;66 bool convertAllOnesVMergeToVMv(MachineInstr &MI) const;67 bool convertSameMaskVMergeToVMv(MachineInstr &MI);68 bool foldUndefPassthruVMV_V_V(MachineInstr &MI);69 bool foldVMV_V_V(MachineInstr &MI);70 bool foldVMergeToMask(MachineInstr &MI) const;71 72 bool hasSameEEW(const MachineInstr &User, const MachineInstr &Src) const;73 bool isAllOnesMask(const MachineInstr *MaskDef) const;74 std::optional<unsigned> getConstant(const MachineOperand &VL) const;75 bool ensureDominates(const MachineOperand &Use, MachineInstr &Src) const;76 bool isKnownSameDefs(Register A, Register B) const;77};78 79} // namespace80 81char RISCVVectorPeephole::ID = 0;82 83INITIALIZE_PASS(RISCVVectorPeephole, DEBUG_TYPE, "RISC-V Fold Masks", false,84 false)85 86/// Given \p User that has an input operand with EEW=SEW, which uses the dest87/// operand of \p Src with an unknown EEW, return true if their EEWs match.88bool RISCVVectorPeephole::hasSameEEW(const MachineInstr &User,89 const MachineInstr &Src) const {90 unsigned UserLog2SEW =91 User.getOperand(RISCVII::getSEWOpNum(User.getDesc())).getImm();92 unsigned SrcLog2SEW =93 Src.getOperand(RISCVII::getSEWOpNum(Src.getDesc())).getImm();94 unsigned SrcLog2EEW = RISCV::getDestLog2EEW(95 TII->get(RISCV::getRVVMCOpcode(Src.getOpcode())), SrcLog2SEW);96 return SrcLog2EEW == UserLog2SEW;97}98 99// Attempt to reduce the VL of an instruction whose sole use is feeding a100// instruction with a narrower VL. This currently works backwards from the101// user instruction (which might have a smaller VL).102bool RISCVVectorPeephole::tryToReduceVL(MachineInstr &MI) const {103 // Note that the goal here is a bit multifaceted.104 // 1) For store's reducing the VL of the value being stored may help to105 // reduce VL toggles. This is somewhat of an artifact of the fact we106 // promote arithmetic instructions but VL predicate stores.107 // 2) For vmv.v.v reducing VL eagerly on the source instruction allows us108 // to share code with the foldVMV_V_V transform below.109 //110 // Note that to the best of our knowledge, reducing VL is generally not111 // a significant win on real hardware unless we can also reduce LMUL which112 // this code doesn't try to do.113 //114 // TODO: We can handle a bunch more instructions here, and probably115 // recurse backwards through operands too.116 SmallVector<unsigned, 2> SrcIndices = {0};117 switch (RISCV::getRVVMCOpcode(MI.getOpcode())) {118 default:119 return false;120 case RISCV::VSE8_V:121 case RISCV::VSE16_V:122 case RISCV::VSE32_V:123 case RISCV::VSE64_V:124 break;125 case RISCV::VMV_V_V:126 SrcIndices[0] = 2;127 break;128 case RISCV::VMERGE_VVM:129 SrcIndices.assign({2, 3});130 break;131 case RISCV::VREDSUM_VS:132 case RISCV::VREDMAXU_VS:133 case RISCV::VREDMAX_VS:134 case RISCV::VREDMINU_VS:135 case RISCV::VREDMIN_VS:136 case RISCV::VREDAND_VS:137 case RISCV::VREDOR_VS:138 case RISCV::VREDXOR_VS:139 case RISCV::VWREDSUM_VS:140 case RISCV::VWREDSUMU_VS:141 case RISCV::VFREDUSUM_VS:142 case RISCV::VFREDOSUM_VS:143 case RISCV::VFREDMAX_VS:144 case RISCV::VFREDMIN_VS:145 case RISCV::VFWREDUSUM_VS:146 case RISCV::VFWREDOSUM_VS:147 SrcIndices[0] = 2;148 break;149 }150 151 MachineOperand &VL = MI.getOperand(RISCVII::getVLOpNum(MI.getDesc()));152 if (VL.isImm() && VL.getImm() == RISCV::VLMaxSentinel)153 return false;154 155 bool Changed = false;156 for (unsigned SrcIdx : SrcIndices) {157 Register SrcReg = MI.getOperand(SrcIdx).getReg();158 // Note: one *use*, not one *user*.159 if (!MRI->hasOneUse(SrcReg))160 continue;161 162 MachineInstr *Src = MRI->getVRegDef(SrcReg);163 if (!Src || Src->hasUnmodeledSideEffects() ||164 Src->getParent() != MI.getParent() || Src->getNumDefs() != 1 ||165 !RISCVII::hasVLOp(Src->getDesc().TSFlags) ||166 !RISCVII::hasSEWOp(Src->getDesc().TSFlags))167 continue;168 169 // Src's dest needs to have the same EEW as MI's input.170 if (!hasSameEEW(MI, *Src))171 continue;172 173 bool ElementsDependOnVL = RISCVII::elementsDependOnVL(174 TII->get(RISCV::getRVVMCOpcode(Src->getOpcode())).TSFlags);175 if (ElementsDependOnVL || Src->mayRaiseFPException())176 continue;177 178 MachineOperand &SrcVL =179 Src->getOperand(RISCVII::getVLOpNum(Src->getDesc()));180 if (VL.isIdenticalTo(SrcVL) || !RISCV::isVLKnownLE(VL, SrcVL))181 continue;182 183 if (!ensureDominates(VL, *Src))184 continue;185 186 if (VL.isImm())187 SrcVL.ChangeToImmediate(VL.getImm());188 else if (VL.isReg())189 SrcVL.ChangeToRegister(VL.getReg(), false);190 191 Changed = true;192 }193 194 // TODO: For instructions with a passthru, we could clear the passthru195 // and tail policy since we've just proven the tail is not demanded.196 return Changed;197}198 199/// Check if an operand is an immediate or a materialized ADDI $x0, imm.200std::optional<unsigned>201RISCVVectorPeephole::getConstant(const MachineOperand &VL) const {202 if (VL.isImm())203 return VL.getImm();204 205 MachineInstr *Def = MRI->getVRegDef(VL.getReg());206 if (!Def || Def->getOpcode() != RISCV::ADDI ||207 Def->getOperand(1).getReg() != RISCV::X0)208 return std::nullopt;209 return Def->getOperand(2).getImm();210}211 212/// Convert AVLs that are known to be VLMAX to the VLMAX sentinel.213bool RISCVVectorPeephole::convertToVLMAX(MachineInstr &MI) const {214 if (!RISCVII::hasVLOp(MI.getDesc().TSFlags) ||215 !RISCVII::hasSEWOp(MI.getDesc().TSFlags))216 return false;217 218 auto LMUL = RISCVVType::decodeVLMUL(RISCVII::getLMul(MI.getDesc().TSFlags));219 // Fixed-point value, denominator=8220 unsigned LMULFixed = LMUL.second ? (8 / LMUL.first) : 8 * LMUL.first;221 unsigned Log2SEW = MI.getOperand(RISCVII::getSEWOpNum(MI.getDesc())).getImm();222 // A Log2SEW of 0 is an operation on mask registers only223 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;224 assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW");225 assert(8 * LMULFixed / SEW > 0);226 227 // If the exact VLEN is known then we know VLMAX, check if the AVL == VLMAX.228 MachineOperand &VL = MI.getOperand(RISCVII::getVLOpNum(MI.getDesc()));229 if (auto VLen = ST->getRealVLen(), AVL = getConstant(VL);230 VLen && AVL && (*VLen * LMULFixed) / SEW == *AVL * 8) {231 VL.ChangeToImmediate(RISCV::VLMaxSentinel);232 return true;233 }234 235 // If an AVL is a VLENB that's possibly scaled to be equal to VLMAX, convert236 // it to the VLMAX sentinel value.237 if (!VL.isReg())238 return false;239 MachineInstr *Def = MRI->getVRegDef(VL.getReg());240 if (!Def)241 return false;242 243 // Fixed-point value, denominator=8244 uint64_t ScaleFixed = 8;245 // Check if the VLENB was potentially scaled with slli/srli246 if (Def->getOpcode() == RISCV::SLLI) {247 assert(Def->getOperand(2).getImm() < 64);248 ScaleFixed <<= Def->getOperand(2).getImm();249 Def = MRI->getVRegDef(Def->getOperand(1).getReg());250 } else if (Def->getOpcode() == RISCV::SRLI) {251 assert(Def->getOperand(2).getImm() < 64);252 ScaleFixed >>= Def->getOperand(2).getImm();253 Def = MRI->getVRegDef(Def->getOperand(1).getReg());254 }255 256 if (!Def || Def->getOpcode() != RISCV::PseudoReadVLENB)257 return false;258 259 // AVL = (VLENB * Scale)260 //261 // VLMAX = (VLENB * 8 * LMUL) / SEW262 //263 // AVL == VLMAX264 // -> VLENB * Scale == (VLENB * 8 * LMUL) / SEW265 // -> Scale == (8 * LMUL) / SEW266 if (ScaleFixed != 8 * LMULFixed / SEW)267 return false;268 269 VL.ChangeToImmediate(RISCV::VLMaxSentinel);270 271 return true;272}273 274bool RISCVVectorPeephole::isAllOnesMask(const MachineInstr *MaskDef) const {275 while (MaskDef->isCopy() && MaskDef->getOperand(1).getReg().isVirtual())276 MaskDef = MRI->getVRegDef(MaskDef->getOperand(1).getReg());277 278 // TODO: Check that the VMSET is the expected bitwidth? The pseudo has279 // undefined behaviour if it's the wrong bitwidth, so we could choose to280 // assume that it's all-ones? Same applies to its VL.281 switch (MaskDef->getOpcode()) {282 case RISCV::PseudoVMSET_M_B1:283 case RISCV::PseudoVMSET_M_B2:284 case RISCV::PseudoVMSET_M_B4:285 case RISCV::PseudoVMSET_M_B8:286 case RISCV::PseudoVMSET_M_B16:287 case RISCV::PseudoVMSET_M_B32:288 case RISCV::PseudoVMSET_M_B64:289 return true;290 default:291 return false;292 }293}294 295/// Convert unit strided unmasked loads and stores to whole-register equivalents296/// to avoid the dependency on $vl and $vtype.297///298/// %x = PseudoVLE8_V_M1 %passthru, %ptr, %vlmax, policy299/// PseudoVSE8_V_M1 %v, %ptr, %vlmax300///301/// ->302///303/// %x = VL1RE8_V %ptr304/// VS1R_V %v, %ptr305bool RISCVVectorPeephole::convertToWholeRegister(MachineInstr &MI) const {306#define CASE_WHOLE_REGISTER_LMUL_SEW(lmul, sew) \307 case RISCV::PseudoVLE##sew##_V_M##lmul: \308 NewOpc = RISCV::VL##lmul##RE##sew##_V; \309 break; \310 case RISCV::PseudoVSE##sew##_V_M##lmul: \311 NewOpc = RISCV::VS##lmul##R_V; \312 break;313#define CASE_WHOLE_REGISTER_LMUL(lmul) \314 CASE_WHOLE_REGISTER_LMUL_SEW(lmul, 8) \315 CASE_WHOLE_REGISTER_LMUL_SEW(lmul, 16) \316 CASE_WHOLE_REGISTER_LMUL_SEW(lmul, 32) \317 CASE_WHOLE_REGISTER_LMUL_SEW(lmul, 64)318 319 unsigned NewOpc;320 switch (MI.getOpcode()) {321 CASE_WHOLE_REGISTER_LMUL(1)322 CASE_WHOLE_REGISTER_LMUL(2)323 CASE_WHOLE_REGISTER_LMUL(4)324 CASE_WHOLE_REGISTER_LMUL(8)325 default:326 return false;327 }328 329 MachineOperand &VLOp = MI.getOperand(RISCVII::getVLOpNum(MI.getDesc()));330 if (!VLOp.isImm() || VLOp.getImm() != RISCV::VLMaxSentinel)331 return false;332 333 // Whole register instructions aren't pseudos so they don't have334 // policy/SEW/AVL ops, and they don't have passthrus.335 if (RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags))336 MI.removeOperand(RISCVII::getVecPolicyOpNum(MI.getDesc()));337 MI.removeOperand(RISCVII::getSEWOpNum(MI.getDesc()));338 MI.removeOperand(RISCVII::getVLOpNum(MI.getDesc()));339 if (RISCVII::isFirstDefTiedToFirstUse(MI.getDesc()))340 MI.removeOperand(1);341 342 MI.setDesc(TII->get(NewOpc));343 344 return true;345}346 347static unsigned getVMV_V_VOpcodeForVMERGE_VVM(const MachineInstr &MI) {348#define CASE_VMERGE_TO_VMV(lmul) \349 case RISCV::PseudoVMERGE_VVM_##lmul: \350 return RISCV::PseudoVMV_V_V_##lmul;351 switch (MI.getOpcode()) {352 default:353 return 0;354 CASE_VMERGE_TO_VMV(MF8)355 CASE_VMERGE_TO_VMV(MF4)356 CASE_VMERGE_TO_VMV(MF2)357 CASE_VMERGE_TO_VMV(M1)358 CASE_VMERGE_TO_VMV(M2)359 CASE_VMERGE_TO_VMV(M4)360 CASE_VMERGE_TO_VMV(M8)361 }362}363 364/// Convert a PseudoVMERGE_VVM with an all ones mask to a PseudoVMV_V_V.365///366/// %x = PseudoVMERGE_VVM %passthru, %false, %true, %allones, sew, vl367/// ->368/// %x = PseudoVMV_V_V %passthru, %true, vl, sew, tu_mu369bool RISCVVectorPeephole::convertAllOnesVMergeToVMv(MachineInstr &MI) const {370 unsigned NewOpc = getVMV_V_VOpcodeForVMERGE_VVM(MI);371 if (!NewOpc)372 return false;373 if (!isAllOnesMask(MRI->getVRegDef(MI.getOperand(4).getReg())))374 return false;375 376 MI.setDesc(TII->get(NewOpc));377 MI.removeOperand(2); // False operand378 MI.removeOperand(3); // Mask operand379 MI.addOperand(380 MachineOperand::CreateImm(RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED));381 382 // vmv.v.v doesn't have a mask operand, so we may be able to inflate the383 // register class for the destination and passthru operands e.g. VRNoV0 -> VR384 MRI->recomputeRegClass(MI.getOperand(0).getReg());385 if (MI.getOperand(1).getReg().isValid())386 MRI->recomputeRegClass(MI.getOperand(1).getReg());387 return true;388}389 390bool RISCVVectorPeephole::isKnownSameDefs(Register A, Register B) const {391 if (A.isPhysical() || B.isPhysical())392 return false;393 394 auto LookThruVirtRegCopies = [this](Register Reg) {395 while (MachineInstr *Def = MRI->getUniqueVRegDef(Reg)) {396 if (!Def->isFullCopy())397 break;398 Register Src = Def->getOperand(1).getReg();399 if (!Src.isVirtual())400 break;401 Reg = Src;402 }403 return Reg;404 };405 406 return LookThruVirtRegCopies(A) == LookThruVirtRegCopies(B);407}408 409/// If a PseudoVMERGE_VVM's true operand is a masked pseudo and both have the410/// same mask, and the masked pseudo's passthru is the same as the false411/// operand, we can convert the PseudoVMERGE_VVM to a PseudoVMV_V_V.412///413/// %true = PseudoVADD_VV_M1_MASK %false, %x, %y, %mask, vl1, sew, policy414/// %x = PseudoVMERGE_VVM %passthru, %false, %true, %mask, vl2, sew415/// ->416/// %true = PseudoVADD_VV_M1_MASK %false, %x, %y, %mask, vl1, sew, policy417/// %x = PseudoVMV_V_V %passthru, %true, vl2, sew, tu_mu418bool RISCVVectorPeephole::convertSameMaskVMergeToVMv(MachineInstr &MI) {419 unsigned NewOpc = getVMV_V_VOpcodeForVMERGE_VVM(MI);420 if (!NewOpc)421 return false;422 MachineInstr *True = MRI->getVRegDef(MI.getOperand(3).getReg());423 424 if (!True || True->getParent() != MI.getParent())425 return false;426 427 auto *TrueMaskedInfo = RISCV::getMaskedPseudoInfo(True->getOpcode());428 if (!TrueMaskedInfo || !hasSameEEW(MI, *True))429 return false;430 431 const MachineOperand &TrueMask =432 True->getOperand(TrueMaskedInfo->MaskOpIdx + True->getNumExplicitDefs());433 const MachineOperand &MIMask = MI.getOperand(4);434 if (!isKnownSameDefs(TrueMask.getReg(), MIMask.getReg()))435 return false;436 437 // Masked off lanes past TrueVL will come from False, and converting to vmv438 // will lose these lanes unless MIVL <= TrueVL.439 // TODO: We could relax this for False == Passthru and True policy == TU440 const MachineOperand &MIVL = MI.getOperand(RISCVII::getVLOpNum(MI.getDesc()));441 const MachineOperand &TrueVL =442 True->getOperand(RISCVII::getVLOpNum(True->getDesc()));443 if (!RISCV::isVLKnownLE(MIVL, TrueVL))444 return false;445 446 // True's passthru needs to be equivalent to False447 Register TruePassthruReg = True->getOperand(1).getReg();448 Register FalseReg = MI.getOperand(2).getReg();449 if (TruePassthruReg != FalseReg) {450 // If True's passthru is undef see if we can change it to False451 if (TruePassthruReg.isValid() ||452 !MRI->hasOneUse(MI.getOperand(3).getReg()) ||453 !ensureDominates(MI.getOperand(2), *True))454 return false;455 True->getOperand(1).setReg(MI.getOperand(2).getReg());456 // If True is masked then its passthru needs to be in VRNoV0.457 MRI->constrainRegClass(True->getOperand(1).getReg(),458 TII->getRegClass(True->getDesc(), 1));459 }460 461 MI.setDesc(TII->get(NewOpc));462 MI.removeOperand(2); // False operand463 MI.removeOperand(3); // Mask operand464 MI.addOperand(465 MachineOperand::CreateImm(RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED));466 467 // vmv.v.v doesn't have a mask operand, so we may be able to inflate the468 // register class for the destination and passthru operands e.g. VRNoV0 -> VR469 MRI->recomputeRegClass(MI.getOperand(0).getReg());470 if (MI.getOperand(1).getReg().isValid())471 MRI->recomputeRegClass(MI.getOperand(1).getReg());472 return true;473}474 475bool RISCVVectorPeephole::convertToUnmasked(MachineInstr &MI) const {476 const RISCV::RISCVMaskedPseudoInfo *I =477 RISCV::getMaskedPseudoInfo(MI.getOpcode());478 if (!I)479 return false;480 481 if (!isAllOnesMask(MRI->getVRegDef(482 MI.getOperand(I->MaskOpIdx + MI.getNumExplicitDefs()).getReg())))483 return false;484 485 // There are two classes of pseudos in the table - compares and486 // everything else. See the comment on RISCVMaskedPseudo for details.487 const unsigned Opc = I->UnmaskedPseudo;488 const MCInstrDesc &MCID = TII->get(Opc);489 [[maybe_unused]] const bool HasPolicyOp =490 RISCVII::hasVecPolicyOp(MCID.TSFlags);491 const bool HasPassthru = RISCVII::isFirstDefTiedToFirstUse(MCID);492 const MCInstrDesc &MaskedMCID = TII->get(MI.getOpcode());493 assert((RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags) ||494 !RISCVII::hasVecPolicyOp(MCID.TSFlags)) &&495 "Unmasked pseudo has policy but masked pseudo doesn't?");496 assert(HasPolicyOp == HasPassthru && "Unexpected pseudo structure");497 assert(!(HasPassthru && !RISCVII::isFirstDefTiedToFirstUse(MaskedMCID)) &&498 "Unmasked with passthru but masked with no passthru?");499 (void)HasPolicyOp;500 501 MI.setDesc(MCID);502 503 // Drop the policy operand if unmasked doesn't need it.504 if (RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags) &&505 !RISCVII::hasVecPolicyOp(MCID.TSFlags))506 MI.removeOperand(RISCVII::getVecPolicyOpNum(MaskedMCID));507 508 // TODO: Increment all MaskOpIdxs in tablegen by num of explicit defs?509 unsigned MaskOpIdx = I->MaskOpIdx + MI.getNumExplicitDefs();510 MI.removeOperand(MaskOpIdx);511 512 // The unmasked pseudo will no longer be constrained to the vrnov0 reg class,513 // so try and relax it to vr.514 MRI->recomputeRegClass(MI.getOperand(0).getReg());515 516 // If the original masked pseudo had a passthru, relax it or remove it.517 if (RISCVII::isFirstDefTiedToFirstUse(MaskedMCID)) {518 unsigned PassthruOpIdx = MI.getNumExplicitDefs();519 if (HasPassthru) {520 if (MI.getOperand(PassthruOpIdx).getReg())521 MRI->recomputeRegClass(MI.getOperand(PassthruOpIdx).getReg());522 } else523 MI.removeOperand(PassthruOpIdx);524 }525 526 return true;527}528 529/// Check if it's safe to move From down to To, checking that no physical530/// registers are clobbered.531static bool isSafeToMove(const MachineInstr &From, const MachineInstr &To) {532 assert(From.getParent() == To.getParent());533 SmallVector<Register> PhysUses, PhysDefs;534 for (const MachineOperand &MO : From.all_uses())535 if (MO.getReg().isPhysical())536 PhysUses.push_back(MO.getReg());537 for (const MachineOperand &MO : From.all_defs())538 if (MO.getReg().isPhysical())539 PhysDefs.push_back(MO.getReg());540 bool SawStore = false;541 for (auto II = std::next(From.getIterator()); II != To.getIterator(); II++) {542 for (Register PhysReg : PhysUses)543 if (II->definesRegister(PhysReg, nullptr))544 return false;545 for (Register PhysReg : PhysDefs)546 if (II->definesRegister(PhysReg, nullptr) ||547 II->readsRegister(PhysReg, nullptr))548 return false;549 if (II->mayStore()) {550 SawStore = true;551 break;552 }553 }554 return From.isSafeToMove(SawStore);555}556 557/// Given A and B are in the same MBB, returns true if A comes before B.558static bool dominates(MachineBasicBlock::const_iterator A,559 MachineBasicBlock::const_iterator B) {560 assert(A->getParent() == B->getParent());561 const MachineBasicBlock *MBB = A->getParent();562 auto MBBEnd = MBB->end();563 if (B == MBBEnd)564 return true;565 566 MachineBasicBlock::const_iterator I = MBB->begin();567 for (; &*I != A && &*I != B; ++I)568 ;569 570 return &*I == A;571}572 573/// If the register in \p MO doesn't dominate \p Src, try to move \p Src so it574/// does. Returns false if doesn't dominate and we can't move. \p MO must be in575/// the same basic block as \Src.576bool RISCVVectorPeephole::ensureDominates(const MachineOperand &MO,577 MachineInstr &Src) const {578 assert(MO.getParent()->getParent() == Src.getParent());579 if (!MO.isReg() || !MO.getReg().isValid())580 return true;581 582 MachineInstr *Def = MRI->getVRegDef(MO.getReg());583 if (Def->getParent() == Src.getParent() && !dominates(Def, Src)) {584 if (!isSafeToMove(Src, *Def->getNextNode()))585 return false;586 Src.moveBefore(Def->getNextNode());587 }588 589 return true;590}591 592/// If a PseudoVMV_V_V's passthru is undef then we can replace it with its input593bool RISCVVectorPeephole::foldUndefPassthruVMV_V_V(MachineInstr &MI) {594 if (RISCV::getRVVMCOpcode(MI.getOpcode()) != RISCV::VMV_V_V)595 return false;596 if (MI.getOperand(1).getReg().isValid())597 return false;598 599 // If the input was a pseudo with a policy operand, we can give it a tail600 // agnostic policy if MI's undef tail subsumes the input's.601 MachineInstr *Src = MRI->getVRegDef(MI.getOperand(2).getReg());602 if (Src && !Src->hasUnmodeledSideEffects() &&603 MRI->hasOneUse(MI.getOperand(2).getReg()) &&604 RISCVII::hasVLOp(Src->getDesc().TSFlags) &&605 RISCVII::hasVecPolicyOp(Src->getDesc().TSFlags) && hasSameEEW(MI, *Src)) {606 const MachineOperand &MIVL = MI.getOperand(3);607 const MachineOperand &SrcVL =608 Src->getOperand(RISCVII::getVLOpNum(Src->getDesc()));609 610 MachineOperand &SrcPolicy =611 Src->getOperand(RISCVII::getVecPolicyOpNum(Src->getDesc()));612 613 if (RISCV::isVLKnownLE(MIVL, SrcVL))614 SrcPolicy.setImm(SrcPolicy.getImm() | RISCVVType::TAIL_AGNOSTIC);615 }616 617 MRI->constrainRegClass(MI.getOperand(2).getReg(),618 MRI->getRegClass(MI.getOperand(0).getReg()));619 MRI->replaceRegWith(MI.getOperand(0).getReg(), MI.getOperand(2).getReg());620 MRI->clearKillFlags(MI.getOperand(2).getReg());621 MI.eraseFromParent();622 return true;623}624 625/// If a PseudoVMV_V_V is the only user of its input, fold its passthru and VL626/// into it.627///628/// %x = PseudoVADD_V_V_M1 %passthru, %a, %b, %vl1, sew, policy629/// %y = PseudoVMV_V_V_M1 %passthru, %x, %vl2, sew, policy630/// (where %vl1 <= %vl2, see related tryToReduceVL)631///632/// ->633///634/// %y = PseudoVADD_V_V_M1 %passthru, %a, %b, vl1, sew, policy635bool RISCVVectorPeephole::foldVMV_V_V(MachineInstr &MI) {636 if (RISCV::getRVVMCOpcode(MI.getOpcode()) != RISCV::VMV_V_V)637 return false;638 639 MachineOperand &Passthru = MI.getOperand(1);640 641 if (!MRI->hasOneUse(MI.getOperand(2).getReg()))642 return false;643 644 MachineInstr *Src = MRI->getVRegDef(MI.getOperand(2).getReg());645 if (!Src || Src->hasUnmodeledSideEffects() ||646 Src->getParent() != MI.getParent() ||647 !RISCVII::isFirstDefTiedToFirstUse(Src->getDesc()) ||648 !RISCVII::hasVLOp(Src->getDesc().TSFlags))649 return false;650 651 // Src's dest needs to have the same EEW as MI's input.652 if (!hasSameEEW(MI, *Src))653 return false;654 655 // Src needs to have the same passthru as VMV_V_V656 MachineOperand &SrcPassthru = Src->getOperand(Src->getNumExplicitDefs());657 if (SrcPassthru.getReg().isValid() &&658 SrcPassthru.getReg() != Passthru.getReg())659 return false;660 661 // Src VL will have already been reduced if legal (see tryToReduceVL),662 // so we don't need to handle a smaller source VL here. However, the663 // user's VL may be larger664 MachineOperand &SrcVL = Src->getOperand(RISCVII::getVLOpNum(Src->getDesc()));665 if (!RISCV::isVLKnownLE(SrcVL, MI.getOperand(3)))666 return false;667 668 // If the new passthru doesn't dominate Src, try to move Src so it does.669 if (!ensureDominates(Passthru, *Src))670 return false;671 672 if (SrcPassthru.getReg() != Passthru.getReg()) {673 SrcPassthru.setReg(Passthru.getReg());674 // If Src is masked then its passthru needs to be in VRNoV0.675 if (Passthru.getReg().isValid())676 MRI->constrainRegClass(677 Passthru.getReg(),678 TII->getRegClass(Src->getDesc(), SrcPassthru.getOperandNo()));679 }680 681 if (RISCVII::hasVecPolicyOp(Src->getDesc().TSFlags)) {682 // If MI was tail agnostic and the VL didn't increase, preserve it.683 int64_t Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;684 if ((MI.getOperand(5).getImm() & RISCVVType::TAIL_AGNOSTIC) &&685 RISCV::isVLKnownLE(MI.getOperand(3), SrcVL))686 Policy |= RISCVVType::TAIL_AGNOSTIC;687 Src->getOperand(RISCVII::getVecPolicyOpNum(Src->getDesc())).setImm(Policy);688 }689 690 MRI->constrainRegClass(Src->getOperand(0).getReg(),691 MRI->getRegClass(MI.getOperand(0).getReg()));692 MRI->replaceRegWith(MI.getOperand(0).getReg(), Src->getOperand(0).getReg());693 MI.eraseFromParent();694 695 return true;696}697 698/// Try to fold away VMERGE_VVM instructions into their operands:699///700/// %true = PseudoVADD_VV ...701/// %x = PseudoVMERGE_VVM_M1 %false, %false, %true, %mask702/// ->703/// %x = PseudoVADD_VV_M1_MASK %false, ..., %mask704///705/// We can only fold if vmerge's passthru operand, vmerge's false operand and706/// %true's passthru operand (if it has one) are the same. This is because we707/// have to consolidate them into one passthru operand in the result.708///709/// If %true is masked, then we can use its mask instead of vmerge's if vmerge's710/// mask is all ones.711///712/// The resulting VL is the minimum of the two VLs.713///714/// The resulting policy is the effective policy the vmerge would have had,715/// i.e. whether or not it's passthru operand was implicit-def.716bool RISCVVectorPeephole::foldVMergeToMask(MachineInstr &MI) const {717 if (RISCV::getRVVMCOpcode(MI.getOpcode()) != RISCV::VMERGE_VVM)718 return false;719 720 Register PassthruReg = MI.getOperand(1).getReg();721 Register FalseReg = MI.getOperand(2).getReg();722 Register TrueReg = MI.getOperand(3).getReg();723 if (!TrueReg.isVirtual() || !MRI->hasOneUse(TrueReg))724 return false;725 MachineInstr &True = *MRI->getUniqueVRegDef(TrueReg);726 if (True.getParent() != MI.getParent())727 return false;728 const MachineOperand &MaskOp = MI.getOperand(4);729 MachineInstr *Mask = MRI->getUniqueVRegDef(MaskOp.getReg());730 assert(Mask);731 732 const RISCV::RISCVMaskedPseudoInfo *Info =733 RISCV::lookupMaskedIntrinsicByUnmasked(True.getOpcode());734 if (!Info)735 return false;736 737 // If the EEW of True is different from vmerge's SEW, then we can't fold.738 if (!hasSameEEW(MI, True))739 return false;740 741 // We require that either passthru and false are the same, or that passthru742 // is undefined.743 if (PassthruReg && !isKnownSameDefs(PassthruReg, FalseReg))744 return false;745 746 std::optional<std::pair<unsigned, unsigned>> NeedsCommute;747 748 // If True has a passthru operand then it needs to be the same as vmerge's749 // False, since False will be used for the result's passthru operand.750 Register TruePassthru = True.getOperand(True.getNumExplicitDefs()).getReg();751 if (RISCVII::isFirstDefTiedToFirstUse(True.getDesc()) && TruePassthru &&752 !isKnownSameDefs(TruePassthru, FalseReg)) {753 // If True's passthru != False, check if it uses False in another operand754 // and try to commute it.755 int OtherIdx = True.findRegisterUseOperandIdx(FalseReg, TRI);756 if (OtherIdx == -1)757 return false;758 unsigned OpIdx1 = OtherIdx;759 unsigned OpIdx2 = True.getNumExplicitDefs();760 if (!TII->findCommutedOpIndices(True, OpIdx1, OpIdx2))761 return false;762 NeedsCommute = {OpIdx1, OpIdx2};763 }764 765 // Make sure it doesn't raise any observable fp exceptions, since changing the766 // active elements will affect how fflags is set.767 if (True.hasUnmodeledSideEffects() || True.mayRaiseFPException())768 return false;769 770 const MachineOperand &VMergeVL =771 MI.getOperand(RISCVII::getVLOpNum(MI.getDesc()));772 const MachineOperand &TrueVL =773 True.getOperand(RISCVII::getVLOpNum(True.getDesc()));774 775 MachineOperand MinVL = MachineOperand::CreateImm(0);776 if (RISCV::isVLKnownLE(TrueVL, VMergeVL))777 MinVL = TrueVL;778 else if (RISCV::isVLKnownLE(VMergeVL, TrueVL))779 MinVL = VMergeVL;780 else781 return false;782 783 unsigned RVVTSFlags =784 TII->get(RISCV::getRVVMCOpcode(True.getOpcode())).TSFlags;785 if (RISCVII::elementsDependOnVL(RVVTSFlags) && !TrueVL.isIdenticalTo(MinVL))786 return false;787 if (RISCVII::elementsDependOnMask(RVVTSFlags) && !isAllOnesMask(Mask))788 return false;789 790 // Use a tumu policy, relaxing it to tail agnostic provided that the passthru791 // operand is undefined.792 //793 // However, if the VL became smaller than what the vmerge had originally, then794 // elements past VL that were previously in the vmerge's body will have moved795 // to the tail. In that case we always need to use tail undisturbed to796 // preserve them.797 uint64_t Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;798 if (!PassthruReg && RISCV::isVLKnownLE(VMergeVL, MinVL))799 Policy |= RISCVVType::TAIL_AGNOSTIC;800 801 assert(RISCVII::hasVecPolicyOp(True.getDesc().TSFlags) &&802 "Foldable unmasked pseudo should have a policy op already");803 804 // Make sure the mask dominates True, otherwise move down True so it does.805 // VL will always dominate since if it's a register they need to be the same.806 if (!ensureDominates(MaskOp, True))807 return false;808 809 if (NeedsCommute) {810 auto [OpIdx1, OpIdx2] = *NeedsCommute;811 [[maybe_unused]] bool Commuted =812 TII->commuteInstruction(True, /*NewMI=*/false, OpIdx1, OpIdx2);813 assert(Commuted && "Failed to commute True?");814 Info = RISCV::lookupMaskedIntrinsicByUnmasked(True.getOpcode());815 }816 817 True.setDesc(TII->get(Info->MaskedPseudo));818 819 // Insert the mask operand.820 // TODO: Increment MaskOpIdx by number of explicit defs?821 True.insert(True.operands_begin() + Info->MaskOpIdx +822 True.getNumExplicitDefs(),823 MachineOperand::CreateReg(MaskOp.getReg(), false));824 825 // Update the passthru, AVL and policy.826 True.getOperand(True.getNumExplicitDefs()).setReg(FalseReg);827 True.removeOperand(RISCVII::getVLOpNum(True.getDesc()));828 True.insert(True.operands_begin() + RISCVII::getVLOpNum(True.getDesc()),829 MinVL);830 True.getOperand(RISCVII::getVecPolicyOpNum(True.getDesc())).setImm(Policy);831 832 MRI->replaceRegWith(True.getOperand(0).getReg(), MI.getOperand(0).getReg());833 // Now that True is masked, constrain its operands from vr -> vrnov0.834 for (MachineOperand &MO : True.explicit_operands()) {835 if (!MO.isReg() || !MO.getReg().isVirtual())836 continue;837 MRI->constrainRegClass(838 MO.getReg(), True.getRegClassConstraint(MO.getOperandNo(), TII, TRI));839 }840 MI.eraseFromParent();841 842 return true;843}844 845bool RISCVVectorPeephole::runOnMachineFunction(MachineFunction &MF) {846 if (skipFunction(MF.getFunction()))847 return false;848 849 // Skip if the vector extension is not enabled.850 ST = &MF.getSubtarget<RISCVSubtarget>();851 if (!ST->hasVInstructions())852 return false;853 854 TII = ST->getInstrInfo();855 MRI = &MF.getRegInfo();856 TRI = MRI->getTargetRegisterInfo();857 858 bool Changed = false;859 860 for (MachineBasicBlock &MBB : MF) {861 for (MachineInstr &MI : make_early_inc_range(MBB))862 Changed |= foldVMergeToMask(MI);863 864 for (MachineInstr &MI : make_early_inc_range(MBB)) {865 Changed |= convertToVLMAX(MI);866 Changed |= tryToReduceVL(MI);867 Changed |= convertToUnmasked(MI);868 Changed |= convertToWholeRegister(MI);869 Changed |= convertAllOnesVMergeToVMv(MI);870 Changed |= convertSameMaskVMergeToVMv(MI);871 if (foldUndefPassthruVMV_V_V(MI)) {872 Changed |= true;873 continue; // MI is erased874 }875 Changed |= foldVMV_V_V(MI);876 }877 }878 879 return Changed;880}881 882FunctionPass *llvm::createRISCVVectorPeepholePass() {883 return new RISCVVectorPeephole();884}885