2104 lines · cpp
1//===- RISCVInsertVSETVLI.cpp - Insert VSETVLI instructions ---------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements a function pass that inserts VSETVLI instructions where10// needed and expands the vl outputs of VLEFF/VLSEGFF to PseudoReadVL11// instructions.12//13// This pass consists of 3 phases:14//15// Phase 1 collects how each basic block affects VL/VTYPE.16//17// Phase 2 uses the information from phase 1 to do a data flow analysis to18// propagate the VL/VTYPE changes through the function. This gives us the19// VL/VTYPE at the start of each basic block.20//21// Phase 3 inserts VSETVLI instructions in each basic block. Information from22// phase 2 is used to prevent inserting a VSETVLI before the first vector23// instruction in the block if possible.24//25//===----------------------------------------------------------------------===//26 27#include "RISCV.h"28#include "RISCVSubtarget.h"29#include "llvm/ADT/PostOrderIterator.h"30#include "llvm/ADT/Statistic.h"31#include "llvm/CodeGen/LiveDebugVariables.h"32#include "llvm/CodeGen/LiveIntervals.h"33#include "llvm/CodeGen/LiveStacks.h"34#include "llvm/CodeGen/MachineFunctionPass.h"35#include <queue>36using namespace llvm;37 38#define DEBUG_TYPE "riscv-insert-vsetvli"39#define RISCV_INSERT_VSETVLI_NAME "RISC-V Insert VSETVLI pass"40 41STATISTIC(NumInsertedVSETVL, "Number of VSETVL inst inserted");42STATISTIC(NumCoalescedVSETVL, "Number of VSETVL inst coalesced");43 44static cl::opt<bool> EnsureWholeVectorRegisterMoveValidVTYPE(45 DEBUG_TYPE "-whole-vector-register-move-valid-vtype", cl::Hidden,46 cl::desc("Insert vsetvlis before vmvNr.vs to ensure vtype is valid and "47 "vill is cleared"),48 cl::init(true));49 50namespace {51 52/// Given a virtual register \p Reg, return the corresponding VNInfo for it.53/// This will return nullptr if the virtual register is an implicit_def or54/// if LiveIntervals is not available.55static VNInfo *getVNInfoFromReg(Register Reg, const MachineInstr &MI,56 const LiveIntervals *LIS) {57 assert(Reg.isVirtual());58 if (!LIS)59 return nullptr;60 auto &LI = LIS->getInterval(Reg);61 SlotIndex SI = LIS->getSlotIndexes()->getInstructionIndex(MI);62 return LI.getVNInfoBefore(SI);63}64 65static unsigned getVLOpNum(const MachineInstr &MI) {66 return RISCVII::getVLOpNum(MI.getDesc());67}68 69static unsigned getSEWOpNum(const MachineInstr &MI) {70 return RISCVII::getSEWOpNum(MI.getDesc());71}72 73static unsigned getVecPolicyOpNum(const MachineInstr &MI) {74 return RISCVII::getVecPolicyOpNum(MI.getDesc());75}76 77/// Get the EEW for a load or store instruction. Return std::nullopt if MI is78/// not a load or store which ignores SEW.79static std::optional<unsigned> getEEWForLoadStore(const MachineInstr &MI) {80 switch (RISCV::getRVVMCOpcode(MI.getOpcode())) {81 default:82 return std::nullopt;83 case RISCV::VLE8_V:84 case RISCV::VLSE8_V:85 case RISCV::VSE8_V:86 case RISCV::VSSE8_V:87 return 8;88 case RISCV::VLE16_V:89 case RISCV::VLSE16_V:90 case RISCV::VSE16_V:91 case RISCV::VSSE16_V:92 return 16;93 case RISCV::VLE32_V:94 case RISCV::VLSE32_V:95 case RISCV::VSE32_V:96 case RISCV::VSSE32_V:97 return 32;98 case RISCV::VLE64_V:99 case RISCV::VLSE64_V:100 case RISCV::VSE64_V:101 case RISCV::VSSE64_V:102 return 64;103 }104}105 106/// Return true if this is an operation on mask registers. Note that107/// this includes both arithmetic/logical ops and load/store (vlm/vsm).108static bool isMaskRegOp(const MachineInstr &MI) {109 if (!RISCVII::hasSEWOp(MI.getDesc().TSFlags))110 return false;111 const unsigned Log2SEW = MI.getOperand(getSEWOpNum(MI)).getImm();112 // A Log2SEW of 0 is an operation on mask registers only.113 return Log2SEW == 0;114}115 116/// Return true if the inactive elements in the result are entirely undefined.117/// Note that this is different from "agnostic" as defined by the vector118/// specification. Agnostic requires each lane to either be undisturbed, or119/// take the value -1; no other value is allowed.120static bool hasUndefinedPassthru(const MachineInstr &MI) {121 122 unsigned UseOpIdx;123 if (!MI.isRegTiedToUseOperand(0, &UseOpIdx))124 // If there is no passthrough operand, then the pass through125 // lanes are undefined.126 return true;127 128 // All undefined passthrus should be $noreg: see129 // RISCVDAGToDAGISel::doPeepholeNoRegPassThru130 const MachineOperand &UseMO = MI.getOperand(UseOpIdx);131 return !UseMO.getReg().isValid() || UseMO.isUndef();132}133 134/// Return true if \p MI is a copy that will be lowered to one or more vmvNr.vs.135static bool isVectorCopy(const TargetRegisterInfo *TRI,136 const MachineInstr &MI) {137 return MI.isCopy() && MI.getOperand(0).getReg().isPhysical() &&138 RISCVRegisterInfo::isRVVRegClass(139 TRI->getMinimalPhysRegClass(MI.getOperand(0).getReg()));140}141 142/// Which subfields of VL or VTYPE have values we need to preserve?143struct DemandedFields {144 // Some unknown property of VL is used. If demanded, must preserve entire145 // value.146 bool VLAny = false;147 // Only zero vs non-zero is used. If demanded, can change non-zero values.148 bool VLZeroness = false;149 // What properties of SEW we need to preserve.150 enum : uint8_t {151 SEWEqual = 3, // The exact value of SEW needs to be preserved.152 SEWGreaterThanOrEqualAndLessThan64 =153 2, // SEW can be changed as long as it's greater154 // than or equal to the original value, but must be less155 // than 64.156 SEWGreaterThanOrEqual = 1, // SEW can be changed as long as it's greater157 // than or equal to the original value.158 SEWNone = 0 // We don't need to preserve SEW at all.159 } SEW = SEWNone;160 enum : uint8_t {161 LMULEqual = 2, // The exact value of LMUL needs to be preserved.162 LMULLessThanOrEqualToM1 = 1, // We can use any LMUL <= M1.163 LMULNone = 0 // We don't need to preserve LMUL at all.164 } LMUL = LMULNone;165 bool SEWLMULRatio = false;166 bool TailPolicy = false;167 bool MaskPolicy = false;168 // If this is true, we demand that VTYPE is set to some legal state, i.e. that169 // vill is unset.170 bool VILL = false;171 bool TWiden = false;172 bool AltFmt = false;173 174 // Return true if any part of VTYPE was used175 bool usedVTYPE() const {176 return SEW || LMUL || SEWLMULRatio || TailPolicy || MaskPolicy || VILL ||177 TWiden || AltFmt;178 }179 180 // Return true if any property of VL was used181 bool usedVL() {182 return VLAny || VLZeroness;183 }184 185 // Mark all VTYPE subfields and properties as demanded186 void demandVTYPE() {187 SEW = SEWEqual;188 LMUL = LMULEqual;189 SEWLMULRatio = true;190 TailPolicy = true;191 MaskPolicy = true;192 VILL = true;193 TWiden = true;194 AltFmt = true;195 }196 197 // Mark all VL properties as demanded198 void demandVL() {199 VLAny = true;200 VLZeroness = true;201 }202 203 static DemandedFields all() {204 DemandedFields DF;205 DF.demandVTYPE();206 DF.demandVL();207 return DF;208 }209 210 // Make this the result of demanding both the fields in this and B.211 void doUnion(const DemandedFields &B) {212 VLAny |= B.VLAny;213 VLZeroness |= B.VLZeroness;214 SEW = std::max(SEW, B.SEW);215 LMUL = std::max(LMUL, B.LMUL);216 SEWLMULRatio |= B.SEWLMULRatio;217 TailPolicy |= B.TailPolicy;218 MaskPolicy |= B.MaskPolicy;219 VILL |= B.VILL;220 AltFmt |= B.AltFmt;221 TWiden |= B.TWiden;222 }223 224#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)225 /// Support for debugging, callable in GDB: V->dump()226 LLVM_DUMP_METHOD void dump() const {227 print(dbgs());228 dbgs() << "\n";229 }230 231 /// Implement operator<<.232 void print(raw_ostream &OS) const {233 OS << "{";234 OS << "VLAny=" << VLAny << ", ";235 OS << "VLZeroness=" << VLZeroness << ", ";236 OS << "SEW=";237 switch (SEW) {238 case SEWEqual:239 OS << "SEWEqual";240 break;241 case SEWGreaterThanOrEqual:242 OS << "SEWGreaterThanOrEqual";243 break;244 case SEWGreaterThanOrEqualAndLessThan64:245 OS << "SEWGreaterThanOrEqualAndLessThan64";246 break;247 case SEWNone:248 OS << "SEWNone";249 break;250 };251 OS << ", ";252 OS << "LMUL=";253 switch (LMUL) {254 case LMULEqual:255 OS << "LMULEqual";256 break;257 case LMULLessThanOrEqualToM1:258 OS << "LMULLessThanOrEqualToM1";259 break;260 case LMULNone:261 OS << "LMULNone";262 break;263 };264 OS << ", ";265 OS << "SEWLMULRatio=" << SEWLMULRatio << ", ";266 OS << "TailPolicy=" << TailPolicy << ", ";267 OS << "MaskPolicy=" << MaskPolicy << ", ";268 OS << "VILL=" << VILL << ", ";269 OS << "AltFmt=" << AltFmt << ", ";270 OS << "TWiden=" << TWiden;271 OS << "}";272 }273#endif274};275 276#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)277LLVM_ATTRIBUTE_USED278inline raw_ostream &operator<<(raw_ostream &OS, const DemandedFields &DF) {279 DF.print(OS);280 return OS;281}282#endif283 284static bool isLMUL1OrSmaller(RISCVVType::VLMUL LMUL) {285 auto [LMul, Fractional] = RISCVVType::decodeVLMUL(LMUL);286 return Fractional || LMul == 1;287}288 289/// Return true if moving from CurVType to NewVType is290/// indistinguishable from the perspective of an instruction (or set291/// of instructions) which use only the Used subfields and properties.292static bool areCompatibleVTYPEs(uint64_t CurVType, uint64_t NewVType,293 const DemandedFields &Used) {294 switch (Used.SEW) {295 case DemandedFields::SEWNone:296 break;297 case DemandedFields::SEWEqual:298 if (RISCVVType::getSEW(CurVType) != RISCVVType::getSEW(NewVType))299 return false;300 break;301 case DemandedFields::SEWGreaterThanOrEqual:302 if (RISCVVType::getSEW(NewVType) < RISCVVType::getSEW(CurVType))303 return false;304 break;305 case DemandedFields::SEWGreaterThanOrEqualAndLessThan64:306 if (RISCVVType::getSEW(NewVType) < RISCVVType::getSEW(CurVType) ||307 RISCVVType::getSEW(NewVType) >= 64)308 return false;309 break;310 }311 312 switch (Used.LMUL) {313 case DemandedFields::LMULNone:314 break;315 case DemandedFields::LMULEqual:316 if (RISCVVType::getVLMUL(CurVType) != RISCVVType::getVLMUL(NewVType))317 return false;318 break;319 case DemandedFields::LMULLessThanOrEqualToM1:320 if (!isLMUL1OrSmaller(RISCVVType::getVLMUL(NewVType)))321 return false;322 break;323 }324 325 if (Used.SEWLMULRatio) {326 auto Ratio1 = RISCVVType::getSEWLMULRatio(RISCVVType::getSEW(CurVType),327 RISCVVType::getVLMUL(CurVType));328 auto Ratio2 = RISCVVType::getSEWLMULRatio(RISCVVType::getSEW(NewVType),329 RISCVVType::getVLMUL(NewVType));330 if (Ratio1 != Ratio2)331 return false;332 }333 334 if (Used.TailPolicy && RISCVVType::isTailAgnostic(CurVType) !=335 RISCVVType::isTailAgnostic(NewVType))336 return false;337 if (Used.MaskPolicy && RISCVVType::isMaskAgnostic(CurVType) !=338 RISCVVType::isMaskAgnostic(NewVType))339 return false;340 if (Used.TWiden && (RISCVVType::hasXSfmmWiden(CurVType) !=341 RISCVVType::hasXSfmmWiden(NewVType) ||342 (RISCVVType::hasXSfmmWiden(CurVType) &&343 RISCVVType::getXSfmmWiden(CurVType) !=344 RISCVVType::getXSfmmWiden(NewVType))))345 return false;346 if (Used.AltFmt &&347 RISCVVType::isAltFmt(CurVType) != RISCVVType::isAltFmt(NewVType))348 return false;349 return true;350}351 352/// Return the fields and properties demanded by the provided instruction.353DemandedFields getDemanded(const MachineInstr &MI, const RISCVSubtarget *ST) {354 // This function works in coalesceVSETVLI too. We can still use the value of a355 // SEW, VL, or Policy operand even though it might not be the exact value in356 // the VL or VTYPE, since we only care about what the instruction originally357 // demanded.358 359 // Most instructions don't use any of these subfeilds.360 DemandedFields Res;361 // Start conservative if registers are used362 if (MI.isCall() || MI.isInlineAsm() ||363 MI.readsRegister(RISCV::VL, /*TRI=*/nullptr))364 Res.demandVL();365 if (MI.isCall() || MI.isInlineAsm() ||366 MI.readsRegister(RISCV::VTYPE, /*TRI=*/nullptr))367 Res.demandVTYPE();368 // Start conservative on the unlowered form too369 uint64_t TSFlags = MI.getDesc().TSFlags;370 if (RISCVII::hasSEWOp(TSFlags)) {371 Res.demandVTYPE();372 if (RISCVII::hasVLOp(TSFlags))373 if (const MachineOperand &VLOp = MI.getOperand(getVLOpNum(MI));374 !VLOp.isReg() || !VLOp.isUndef())375 Res.demandVL();376 377 // Behavior is independent of mask policy.378 if (!RISCVII::usesMaskPolicy(TSFlags))379 Res.MaskPolicy = false;380 }381 382 // Loads and stores with implicit EEW do not demand SEW or LMUL directly.383 // They instead demand the ratio of the two which is used in computing384 // EMUL, but which allows us the flexibility to change SEW and LMUL385 // provided we don't change the ratio.386 // Note: We assume that the instructions initial SEW is the EEW encoded387 // in the opcode. This is asserted when constructing the VSETVLIInfo.388 if (getEEWForLoadStore(MI)) {389 Res.SEW = DemandedFields::SEWNone;390 Res.LMUL = DemandedFields::LMULNone;391 }392 393 // Store instructions don't use the policy fields.394 if (RISCVII::hasSEWOp(TSFlags) && MI.getNumExplicitDefs() == 0) {395 Res.TailPolicy = false;396 Res.MaskPolicy = false;397 }398 399 // If this is a mask reg operation, it only cares about VLMAX.400 // TODO: Possible extensions to this logic401 // * Probably ok if available VLMax is larger than demanded402 // * The policy bits can probably be ignored..403 if (isMaskRegOp(MI)) {404 Res.SEW = DemandedFields::SEWNone;405 Res.LMUL = DemandedFields::LMULNone;406 }407 408 // For vmv.s.x and vfmv.s.f, there are only two behaviors, VL = 0 and VL > 0.409 if (RISCVInstrInfo::isScalarInsertInstr(MI)) {410 Res.LMUL = DemandedFields::LMULNone;411 Res.SEWLMULRatio = false;412 Res.VLAny = false;413 // For vmv.s.x and vfmv.s.f, if the passthru is *undefined*, we don't414 // need to preserve any other bits and are thus compatible with any larger,415 // etype and can disregard policy bits. Warning: It's tempting to try doing416 // this for any tail agnostic operation, but we can't as TA requires417 // tail lanes to either be the original value or -1. We are writing418 // unknown bits to the lanes here.419 if (hasUndefinedPassthru(MI)) {420 if (RISCVInstrInfo::isFloatScalarMoveOrScalarSplatInstr(MI) &&421 !ST->hasVInstructionsF64())422 Res.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;423 else424 Res.SEW = DemandedFields::SEWGreaterThanOrEqual;425 Res.TailPolicy = false;426 }427 }428 429 // vmv.x.s, and vfmv.f.s are unconditional and ignore everything except SEW.430 if (RISCVInstrInfo::isScalarExtractInstr(MI)) {431 assert(!RISCVII::hasVLOp(TSFlags));432 Res.LMUL = DemandedFields::LMULNone;433 Res.SEWLMULRatio = false;434 Res.TailPolicy = false;435 Res.MaskPolicy = false;436 }437 438 if (RISCVII::hasVLOp(MI.getDesc().TSFlags)) {439 const MachineOperand &VLOp = MI.getOperand(getVLOpNum(MI));440 // A slidedown/slideup with an *undefined* passthru can freely clobber441 // elements not copied from the source vector (e.g. masked off, tail, or442 // slideup's prefix). Notes:443 // * We can't modify SEW here since the slide amount is in units of SEW.444 // * VL=1 is special only because we have existing support for zero vs445 // non-zero VL. We could generalize this if we had a VL > C predicate.446 // * The LMUL1 restriction is for machines whose latency may depend on LMUL.447 // * As above, this is only legal for tail "undefined" not "agnostic".448 // * We avoid increasing vl if the subtarget has +vl-dependent-latency449 if (RISCVInstrInfo::isVSlideInstr(MI) && VLOp.isImm() &&450 VLOp.getImm() == 1 && hasUndefinedPassthru(MI) &&451 !ST->hasVLDependentLatency()) {452 Res.VLAny = false;453 Res.VLZeroness = true;454 Res.LMUL = DemandedFields::LMULLessThanOrEqualToM1;455 Res.TailPolicy = false;456 }457 458 // A tail undefined vmv.v.i/x or vfmv.v.f with VL=1 can be treated in the459 // same semantically as vmv.s.x. This is particularly useful since we don't460 // have an immediate form of vmv.s.x, and thus frequently use vmv.v.i in461 // it's place. Since a splat is non-constant time in LMUL, we do need to be462 // careful to not increase the number of active vector registers (unlike for463 // vmv.s.x.)464 if (RISCVInstrInfo::isScalarSplatInstr(MI) && VLOp.isImm() &&465 VLOp.getImm() == 1 && hasUndefinedPassthru(MI) &&466 !ST->hasVLDependentLatency()) {467 Res.LMUL = DemandedFields::LMULLessThanOrEqualToM1;468 Res.SEWLMULRatio = false;469 Res.VLAny = false;470 if (RISCVInstrInfo::isFloatScalarMoveOrScalarSplatInstr(MI) &&471 !ST->hasVInstructionsF64())472 Res.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;473 else474 Res.SEW = DemandedFields::SEWGreaterThanOrEqual;475 Res.TailPolicy = false;476 }477 }478 479 // In §32.16.6, whole vector register moves have a dependency on SEW. At the480 // MIR level though we don't encode the element type, and it gives the same481 // result whatever the SEW may be.482 //483 // However it does need valid SEW, i.e. vill must be cleared. The entry to a484 // function, calls and inline assembly may all set it, so make sure we clear485 // it for whole register copies. Do this by leaving VILL demanded.486 if (isVectorCopy(ST->getRegisterInfo(), MI)) {487 Res.LMUL = DemandedFields::LMULNone;488 Res.SEW = DemandedFields::SEWNone;489 Res.SEWLMULRatio = false;490 Res.TailPolicy = false;491 Res.MaskPolicy = false;492 }493 494 if (RISCVInstrInfo::isVExtractInstr(MI)) {495 assert(!RISCVII::hasVLOp(TSFlags));496 // TODO: LMUL can be any larger value (without cost)497 Res.TailPolicy = false;498 }499 500 Res.AltFmt = RISCVII::getAltFmtType(MI.getDesc().TSFlags) !=501 RISCVII::AltFmtType::DontCare;502 Res.TWiden = RISCVII::hasTWidenOp(MI.getDesc().TSFlags) ||503 RISCVInstrInfo::isXSfmmVectorConfigInstr(MI);504 505 return Res;506}507 508/// Defines the abstract state with which the forward dataflow models the509/// values of the VL and VTYPE registers after insertion.510class VSETVLIInfo {511 struct AVLDef {512 // Every AVLDef should have a VNInfo, unless we're running without513 // LiveIntervals in which case this will be nullptr.514 const VNInfo *ValNo;515 Register DefReg;516 };517 union {518 AVLDef AVLRegDef;519 unsigned AVLImm;520 };521 522 enum class AVLState : uint8_t {523 Uninitialized,524 AVLIsReg,525 AVLIsImm,526 AVLIsVLMAX,527 Unknown, // AVL and VTYPE are fully unknown528 } State = AVLState::Uninitialized;529 530 // Fields from VTYPE.531 RISCVVType::VLMUL VLMul = RISCVVType::LMUL_1;532 uint8_t SEW = 0;533 uint8_t TailAgnostic : 1;534 uint8_t MaskAgnostic : 1;535 uint8_t SEWLMULRatioOnly : 1;536 uint8_t AltFmt : 1;537 uint8_t TWiden : 3;538 539public:540 VSETVLIInfo()541 : AVLImm(0), TailAgnostic(false), MaskAgnostic(false),542 SEWLMULRatioOnly(false), AltFmt(false), TWiden(0) {}543 544 static VSETVLIInfo getUnknown() {545 VSETVLIInfo Info;546 Info.setUnknown();547 return Info;548 }549 550 bool isValid() const { return State != AVLState::Uninitialized; }551 void setUnknown() { State = AVLState::Unknown; }552 bool isUnknown() const { return State == AVLState::Unknown; }553 554 void setAVLRegDef(const VNInfo *VNInfo, Register AVLReg) {555 assert(AVLReg.isVirtual());556 AVLRegDef.ValNo = VNInfo;557 AVLRegDef.DefReg = AVLReg;558 State = AVLState::AVLIsReg;559 }560 561 void setAVLImm(unsigned Imm) {562 AVLImm = Imm;563 State = AVLState::AVLIsImm;564 }565 566 void setAVLVLMAX() { State = AVLState::AVLIsVLMAX; }567 568 bool hasAVLImm() const { return State == AVLState::AVLIsImm; }569 bool hasAVLReg() const { return State == AVLState::AVLIsReg; }570 bool hasAVLVLMAX() const { return State == AVLState::AVLIsVLMAX; }571 Register getAVLReg() const {572 assert(hasAVLReg() && AVLRegDef.DefReg.isVirtual());573 return AVLRegDef.DefReg;574 }575 unsigned getAVLImm() const {576 assert(hasAVLImm());577 return AVLImm;578 }579 const VNInfo *getAVLVNInfo() const {580 assert(hasAVLReg());581 return AVLRegDef.ValNo;582 }583 // Most AVLIsReg infos will have a single defining MachineInstr, unless it was584 // a PHI node. In that case getAVLVNInfo()->def will point to the block585 // boundary slot and this will return nullptr. If LiveIntervals isn't586 // available, nullptr is also returned.587 const MachineInstr *getAVLDefMI(const LiveIntervals *LIS) const {588 assert(hasAVLReg());589 if (!LIS || getAVLVNInfo()->isPHIDef())590 return nullptr;591 auto *MI = LIS->getInstructionFromIndex(getAVLVNInfo()->def);592 assert(MI);593 return MI;594 }595 596 void setAVL(const VSETVLIInfo &Info) {597 assert(Info.isValid());598 if (Info.isUnknown())599 setUnknown();600 else if (Info.hasAVLReg())601 setAVLRegDef(Info.getAVLVNInfo(), Info.getAVLReg());602 else if (Info.hasAVLVLMAX())603 setAVLVLMAX();604 else {605 assert(Info.hasAVLImm());606 setAVLImm(Info.getAVLImm());607 }608 }609 610 unsigned getSEW() const {611 assert(isValid() && !isUnknown() &&612 "Can't use VTYPE for uninitialized or unknown");613 return SEW;614 }615 RISCVVType::VLMUL getVLMUL() const {616 assert(isValid() && !isUnknown() &&617 "Can't use VTYPE for uninitialized or unknown");618 return VLMul;619 }620 bool getTailAgnostic() const {621 assert(isValid() && !isUnknown() &&622 "Can't use VTYPE for uninitialized or unknown");623 return TailAgnostic;624 }625 bool getMaskAgnostic() const {626 assert(isValid() && !isUnknown() &&627 "Can't use VTYPE for uninitialized or unknown");628 return MaskAgnostic;629 }630 bool getAltFmt() const {631 assert(isValid() && !isUnknown() &&632 "Can't use VTYPE for uninitialized or unknown");633 return AltFmt;634 }635 unsigned getTWiden() const {636 assert(isValid() && !isUnknown() &&637 "Can't use VTYPE for uninitialized or unknown");638 return TWiden;639 }640 641 bool hasNonZeroAVL(const LiveIntervals *LIS) const {642 if (hasAVLImm())643 return getAVLImm() > 0;644 if (hasAVLReg()) {645 if (auto *DefMI = getAVLDefMI(LIS))646 return RISCVInstrInfo::isNonZeroLoadImmediate(*DefMI);647 }648 if (hasAVLVLMAX())649 return true;650 return false;651 }652 653 bool hasEquallyZeroAVL(const VSETVLIInfo &Other,654 const LiveIntervals *LIS) const {655 if (hasSameAVL(Other))656 return true;657 return (hasNonZeroAVL(LIS) && Other.hasNonZeroAVL(LIS));658 }659 660 bool hasSameAVLLatticeValue(const VSETVLIInfo &Other) const {661 if (hasAVLReg() && Other.hasAVLReg()) {662 assert(!getAVLVNInfo() == !Other.getAVLVNInfo() &&663 "we either have intervals or we don't");664 if (!getAVLVNInfo())665 return getAVLReg() == Other.getAVLReg();666 return getAVLVNInfo()->id == Other.getAVLVNInfo()->id &&667 getAVLReg() == Other.getAVLReg();668 }669 670 if (hasAVLImm() && Other.hasAVLImm())671 return getAVLImm() == Other.getAVLImm();672 673 if (hasAVLVLMAX())674 return Other.hasAVLVLMAX() && hasSameVLMAX(Other);675 676 return false;677 }678 679 // Return true if the two lattice values are guaranteed to have680 // the same AVL value at runtime.681 bool hasSameAVL(const VSETVLIInfo &Other) const {682 // Without LiveIntervals, we don't know which instruction defines a683 // register. Since a register may be redefined, this means all AVLIsReg684 // states must be treated as possibly distinct.685 if (hasAVLReg() && Other.hasAVLReg()) {686 assert(!getAVLVNInfo() == !Other.getAVLVNInfo() &&687 "we either have intervals or we don't");688 if (!getAVLVNInfo())689 return false;690 }691 return hasSameAVLLatticeValue(Other);692 }693 694 void setVTYPE(unsigned VType) {695 assert(isValid() && !isUnknown() &&696 "Can't set VTYPE for uninitialized or unknown");697 VLMul = RISCVVType::getVLMUL(VType);698 SEW = RISCVVType::getSEW(VType);699 TailAgnostic = RISCVVType::isTailAgnostic(VType);700 MaskAgnostic = RISCVVType::isMaskAgnostic(VType);701 AltFmt = RISCVVType::isAltFmt(VType);702 TWiden =703 RISCVVType::hasXSfmmWiden(VType) ? RISCVVType::getXSfmmWiden(VType) : 0;704 }705 void setVTYPE(RISCVVType::VLMUL L, unsigned S, bool TA, bool MA, bool Altfmt,706 unsigned W) {707 assert(isValid() && !isUnknown() &&708 "Can't set VTYPE for uninitialized or unknown");709 VLMul = L;710 SEW = S;711 TailAgnostic = TA;712 MaskAgnostic = MA;713 AltFmt = Altfmt;714 TWiden = W;715 }716 717 void setAltFmt(bool AF) { AltFmt = AF; }718 719 void setVLMul(RISCVVType::VLMUL VLMul) { this->VLMul = VLMul; }720 721 unsigned encodeVTYPE() const {722 assert(isValid() && !isUnknown() && !SEWLMULRatioOnly &&723 "Can't encode VTYPE for uninitialized or unknown");724 if (TWiden != 0)725 return RISCVVType::encodeXSfmmVType(SEW, TWiden, AltFmt);726 return RISCVVType::encodeVTYPE(VLMul, SEW, TailAgnostic, MaskAgnostic,727 AltFmt);728 }729 730 bool hasSEWLMULRatioOnly() const { return SEWLMULRatioOnly; }731 732 bool hasSameVTYPE(const VSETVLIInfo &Other) const {733 assert(isValid() && Other.isValid() &&734 "Can't compare invalid VSETVLIInfos");735 assert(!isUnknown() && !Other.isUnknown() &&736 "Can't compare VTYPE in unknown state");737 assert(!SEWLMULRatioOnly && !Other.SEWLMULRatioOnly &&738 "Can't compare when only LMUL/SEW ratio is valid.");739 return std::tie(VLMul, SEW, TailAgnostic, MaskAgnostic, AltFmt, TWiden) ==740 std::tie(Other.VLMul, Other.SEW, Other.TailAgnostic,741 Other.MaskAgnostic, Other.AltFmt, Other.TWiden);742 }743 744 unsigned getSEWLMULRatio() const {745 assert(isValid() && !isUnknown() &&746 "Can't use VTYPE for uninitialized or unknown");747 return RISCVVType::getSEWLMULRatio(SEW, VLMul);748 }749 750 // Check if the VTYPE for these two VSETVLIInfos produce the same VLMAX.751 // Note that having the same VLMAX ensures that both share the same752 // function from AVL to VL; that is, they must produce the same VL value753 // for any given AVL value.754 bool hasSameVLMAX(const VSETVLIInfo &Other) const {755 assert(isValid() && Other.isValid() &&756 "Can't compare invalid VSETVLIInfos");757 assert(!isUnknown() && !Other.isUnknown() &&758 "Can't compare VTYPE in unknown state");759 return getSEWLMULRatio() == Other.getSEWLMULRatio();760 }761 762 bool hasCompatibleVTYPE(const DemandedFields &Used,763 const VSETVLIInfo &Require) const {764 return areCompatibleVTYPEs(Require.encodeVTYPE(), encodeVTYPE(), Used);765 }766 767 // Determine whether the vector instructions requirements represented by768 // Require are compatible with the previous vsetvli instruction represented769 // by this. MI is the instruction whose requirements we're considering.770 bool isCompatible(const DemandedFields &Used, const VSETVLIInfo &Require,771 const LiveIntervals *LIS) const {772 assert(isValid() && Require.isValid() &&773 "Can't compare invalid VSETVLIInfos");774 // Nothing is compatible with Unknown.775 if (isUnknown() || Require.isUnknown())776 return false;777 778 // If only our VLMAX ratio is valid, then this isn't compatible.779 if (SEWLMULRatioOnly || Require.SEWLMULRatioOnly)780 return false;781 782 if (Used.VLAny && !(hasSameAVL(Require) && hasSameVLMAX(Require)))783 return false;784 785 if (Used.VLZeroness && !hasEquallyZeroAVL(Require, LIS))786 return false;787 788 return hasCompatibleVTYPE(Used, Require);789 }790 791 bool operator==(const VSETVLIInfo &Other) const {792 // Uninitialized is only equal to another Uninitialized.793 if (!isValid())794 return !Other.isValid();795 if (!Other.isValid())796 return !isValid();797 798 // Unknown is only equal to another Unknown.799 if (isUnknown())800 return Other.isUnknown();801 if (Other.isUnknown())802 return isUnknown();803 804 if (!hasSameAVLLatticeValue(Other))805 return false;806 807 // If the SEWLMULRatioOnly bits are different, then they aren't equal.808 if (SEWLMULRatioOnly != Other.SEWLMULRatioOnly)809 return false;810 811 // If only the VLMAX is valid, check that it is the same.812 if (SEWLMULRatioOnly)813 return hasSameVLMAX(Other);814 815 // If the full VTYPE is valid, check that it is the same.816 return hasSameVTYPE(Other);817 }818 819 bool operator!=(const VSETVLIInfo &Other) const {820 return !(*this == Other);821 }822 823 // Calculate the VSETVLIInfo visible to a block assuming this and Other are824 // both predecessors.825 VSETVLIInfo intersect(const VSETVLIInfo &Other) const {826 // If the new value isn't valid, ignore it.827 if (!Other.isValid())828 return *this;829 830 // If this value isn't valid, this must be the first predecessor, use it.831 if (!isValid())832 return Other;833 834 // If either is unknown, the result is unknown.835 if (isUnknown() || Other.isUnknown())836 return VSETVLIInfo::getUnknown();837 838 // If we have an exact, match return this.839 if (*this == Other)840 return *this;841 842 // Not an exact match, but maybe the AVL and VLMAX are the same. If so,843 // return an SEW/LMUL ratio only value.844 if (hasSameAVL(Other) && hasSameVLMAX(Other)) {845 VSETVLIInfo MergeInfo = *this;846 MergeInfo.SEWLMULRatioOnly = true;847 return MergeInfo;848 }849 850 // Otherwise the result is unknown.851 return VSETVLIInfo::getUnknown();852 }853 854#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)855 /// Support for debugging, callable in GDB: V->dump()856 LLVM_DUMP_METHOD void dump() const {857 print(dbgs());858 dbgs() << "\n";859 }860 861 /// Implement operator<<.862 /// @{863 void print(raw_ostream &OS) const {864 OS << '{';865 switch (State) {866 case AVLState::Uninitialized:867 OS << "Uninitialized";868 break;869 case AVLState::Unknown:870 OS << "unknown";871 break;872 case AVLState::AVLIsReg:873 OS << "AVLReg=" << llvm::printReg(getAVLReg());874 break;875 case AVLState::AVLIsImm:876 OS << "AVLImm=" << (unsigned)AVLImm;877 break;878 case AVLState::AVLIsVLMAX:879 OS << "AVLVLMAX";880 break;881 }882 if (isValid() && !isUnknown()) {883 OS << ", ";884 885 unsigned LMul;886 bool Fractional;887 std::tie(LMul, Fractional) = decodeVLMUL(VLMul);888 889 OS << "VLMul=m";890 if (Fractional)891 OS << 'f';892 OS << LMul << ", "893 << "SEW=e" << (unsigned)SEW << ", "894 << "TailAgnostic=" << (bool)TailAgnostic << ", "895 << "MaskAgnostic=" << (bool)MaskAgnostic << ", "896 << "SEWLMULRatioOnly=" << (bool)SEWLMULRatioOnly << ", "897 << "TWiden=" << (unsigned)TWiden << ", "898 << "AltFmt=" << (bool)AltFmt;899 }900 901 OS << '}';902 }903#endif904};905 906#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)907LLVM_ATTRIBUTE_USED908inline raw_ostream &operator<<(raw_ostream &OS, const VSETVLIInfo &V) {909 V.print(OS);910 return OS;911}912#endif913 914struct BlockData {915 // The VSETVLIInfo that represents the VL/VTYPE settings on exit from this916 // block. Calculated in Phase 2.917 VSETVLIInfo Exit;918 919 // The VSETVLIInfo that represents the VL/VTYPE settings from all predecessor920 // blocks. Calculated in Phase 2, and used by Phase 3.921 VSETVLIInfo Pred;922 923 // Keeps track of whether the block is already in the queue.924 bool InQueue = false;925 926 BlockData() = default;927};928 929enum TKTMMode {930 VSETTK = 0,931 VSETTM = 1,932};933 934class RISCVInsertVSETVLI : public MachineFunctionPass {935 const RISCVSubtarget *ST;936 const TargetInstrInfo *TII;937 MachineRegisterInfo *MRI;938 // Possibly null!939 LiveIntervals *LIS;940 941 std::vector<BlockData> BlockInfo;942 std::queue<const MachineBasicBlock *> WorkList;943 944public:945 static char ID;946 947 RISCVInsertVSETVLI() : MachineFunctionPass(ID) {}948 bool runOnMachineFunction(MachineFunction &MF) override;949 950 void getAnalysisUsage(AnalysisUsage &AU) const override {951 AU.setPreservesCFG();952 953 AU.addUsedIfAvailable<LiveIntervalsWrapperPass>();954 AU.addPreserved<LiveIntervalsWrapperPass>();955 AU.addPreserved<SlotIndexesWrapperPass>();956 AU.addPreserved<LiveDebugVariablesWrapperLegacy>();957 AU.addPreserved<LiveStacksWrapperLegacy>();958 959 MachineFunctionPass::getAnalysisUsage(AU);960 }961 962 StringRef getPassName() const override { return RISCV_INSERT_VSETVLI_NAME; }963 964private:965 bool needVSETVLI(const DemandedFields &Used, const VSETVLIInfo &Require,966 const VSETVLIInfo &CurInfo) const;967 bool needVSETVLIPHI(const VSETVLIInfo &Require,968 const MachineBasicBlock &MBB) const;969 void insertVSETVLI(MachineBasicBlock &MBB,970 MachineBasicBlock::iterator InsertPt, DebugLoc DL,971 const VSETVLIInfo &Info, const VSETVLIInfo &PrevInfo);972 973 void transferBefore(VSETVLIInfo &Info, const MachineInstr &MI) const;974 void transferAfter(VSETVLIInfo &Info, const MachineInstr &MI) const;975 bool computeVLVTYPEChanges(const MachineBasicBlock &MBB,976 VSETVLIInfo &Info) const;977 void computeIncomingVLVTYPE(const MachineBasicBlock &MBB);978 void emitVSETVLIs(MachineBasicBlock &MBB);979 void doPRE(MachineBasicBlock &MBB);980 void insertReadVL(MachineBasicBlock &MBB);981 982 bool canMutatePriorConfig(const MachineInstr &PrevMI, const MachineInstr &MI,983 const DemandedFields &Used) const;984 void coalesceVSETVLIs(MachineBasicBlock &MBB) const;985 986 VSETVLIInfo getInfoForVSETVLI(const MachineInstr &MI) const;987 VSETVLIInfo computeInfoForInstr(const MachineInstr &MI) const;988 void forwardVSETVLIAVL(VSETVLIInfo &Info) const;989 bool insertVSETMTK(MachineBasicBlock &MBB, TKTMMode Mode) const;990};991 992} // end anonymous namespace993 994char RISCVInsertVSETVLI::ID = 0;995char &llvm::RISCVInsertVSETVLIID = RISCVInsertVSETVLI::ID;996 997INITIALIZE_PASS(RISCVInsertVSETVLI, DEBUG_TYPE, RISCV_INSERT_VSETVLI_NAME,998 false, false)999 1000// If the AVL is defined by a vsetvli's output vl with the same VLMAX, we can1001// replace the AVL operand with the AVL of the defining vsetvli. E.g.1002//1003// %vl = PseudoVSETVLI %avl:gpr, SEW=32, LMUL=M11004// $x0 = PseudoVSETVLI %vl:gpr, SEW=32, LMUL=M11005// ->1006// %vl = PseudoVSETVLI %avl:gpr, SEW=32, LMUL=M11007// $x0 = PseudoVSETVLI %avl:gpr, SEW=32, LMUL=M11008void RISCVInsertVSETVLI::forwardVSETVLIAVL(VSETVLIInfo &Info) const {1009 if (!Info.hasAVLReg())1010 return;1011 const MachineInstr *DefMI = Info.getAVLDefMI(LIS);1012 if (!DefMI || !RISCVInstrInfo::isVectorConfigInstr(*DefMI))1013 return;1014 VSETVLIInfo DefInstrInfo = getInfoForVSETVLI(*DefMI);1015 if (!DefInstrInfo.hasSameVLMAX(Info))1016 return;1017 Info.setAVL(DefInstrInfo);1018}1019 1020// Return a VSETVLIInfo representing the changes made by this VSETVLI or1021// VSETIVLI instruction.1022VSETVLIInfo1023RISCVInsertVSETVLI::getInfoForVSETVLI(const MachineInstr &MI) const {1024 VSETVLIInfo NewInfo;1025 if (MI.getOpcode() == RISCV::PseudoVSETIVLI) {1026 NewInfo.setAVLImm(MI.getOperand(1).getImm());1027 } else if (RISCVInstrInfo::isXSfmmVectorConfigTNInstr(MI)) {1028 assert(MI.getOpcode() == RISCV::PseudoSF_VSETTNT ||1029 MI.getOpcode() == RISCV::PseudoSF_VSETTNTX0);1030 switch (MI.getOpcode()) {1031 case RISCV::PseudoSF_VSETTNTX0:1032 NewInfo.setAVLVLMAX();1033 break;1034 case RISCV::PseudoSF_VSETTNT:1035 Register ATNReg = MI.getOperand(1).getReg();1036 NewInfo.setAVLRegDef(getVNInfoFromReg(ATNReg, MI, LIS), ATNReg);1037 break;1038 }1039 } else {1040 assert(MI.getOpcode() == RISCV::PseudoVSETVLI ||1041 MI.getOpcode() == RISCV::PseudoVSETVLIX0);1042 if (MI.getOpcode() == RISCV::PseudoVSETVLIX0)1043 NewInfo.setAVLVLMAX();1044 else if (MI.getOperand(1).isUndef())1045 // Otherwise use an AVL of 1 to avoid depending on previous vl.1046 NewInfo.setAVLImm(1);1047 else {1048 Register AVLReg = MI.getOperand(1).getReg();1049 VNInfo *VNI = getVNInfoFromReg(AVLReg, MI, LIS);1050 NewInfo.setAVLRegDef(VNI, AVLReg);1051 }1052 }1053 NewInfo.setVTYPE(MI.getOperand(2).getImm());1054 1055 forwardVSETVLIAVL(NewInfo);1056 1057 return NewInfo;1058}1059 1060static unsigned computeVLMAX(unsigned VLEN, unsigned SEW,1061 RISCVVType::VLMUL VLMul) {1062 auto [LMul, Fractional] = RISCVVType::decodeVLMUL(VLMul);1063 if (Fractional)1064 VLEN = VLEN / LMul;1065 else1066 VLEN = VLEN * LMul;1067 return VLEN/SEW;1068}1069 1070VSETVLIInfo1071RISCVInsertVSETVLI::computeInfoForInstr(const MachineInstr &MI) const {1072 VSETVLIInfo InstrInfo;1073 const uint64_t TSFlags = MI.getDesc().TSFlags;1074 1075 bool TailAgnostic = true;1076 bool MaskAgnostic = true;1077 if (!hasUndefinedPassthru(MI)) {1078 // Start with undisturbed.1079 TailAgnostic = false;1080 MaskAgnostic = false;1081 1082 // If there is a policy operand, use it.1083 if (RISCVII::hasVecPolicyOp(TSFlags)) {1084 const MachineOperand &Op = MI.getOperand(getVecPolicyOpNum(MI));1085 uint64_t Policy = Op.getImm();1086 assert(Policy <=1087 (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC) &&1088 "Invalid Policy Value");1089 TailAgnostic = Policy & RISCVVType::TAIL_AGNOSTIC;1090 MaskAgnostic = Policy & RISCVVType::MASK_AGNOSTIC;1091 }1092 1093 if (!RISCVII::usesMaskPolicy(TSFlags))1094 MaskAgnostic = true;1095 }1096 1097 RISCVVType::VLMUL VLMul = RISCVII::getLMul(TSFlags);1098 1099 bool AltFmt = RISCVII::getAltFmtType(TSFlags) == RISCVII::AltFmtType::AltFmt;1100 InstrInfo.setAltFmt(AltFmt);1101 1102 unsigned Log2SEW = MI.getOperand(getSEWOpNum(MI)).getImm();1103 // A Log2SEW of 0 is an operation on mask registers only.1104 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;1105 assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW");1106 1107 if (RISCVII::hasTWidenOp(TSFlags)) {1108 const MachineOperand &TWidenOp =1109 MI.getOperand(MI.getNumExplicitOperands() - 1);1110 unsigned TWiden = TWidenOp.getImm();1111 1112 InstrInfo.setAVLVLMAX();1113 if (RISCVII::hasVLOp(TSFlags)) {1114 const MachineOperand &TNOp =1115 MI.getOperand(RISCVII::getTNOpNum(MI.getDesc()));1116 1117 if (TNOp.getReg().isVirtual())1118 InstrInfo.setAVLRegDef(getVNInfoFromReg(TNOp.getReg(), MI, LIS),1119 TNOp.getReg());1120 }1121 1122 InstrInfo.setVTYPE(VLMul, SEW, TailAgnostic, MaskAgnostic, AltFmt, TWiden);1123 1124 return InstrInfo;1125 }1126 1127 if (RISCVII::hasVLOp(TSFlags)) {1128 const MachineOperand &VLOp = MI.getOperand(getVLOpNum(MI));1129 if (VLOp.isImm()) {1130 int64_t Imm = VLOp.getImm();1131 // Convert the VLMax sentintel to X0 register.1132 if (Imm == RISCV::VLMaxSentinel) {1133 // If we know the exact VLEN, see if we can use the constant encoding1134 // for the VLMAX instead. This reduces register pressure slightly.1135 const unsigned VLMAX = computeVLMAX(ST->getRealMaxVLen(), SEW, VLMul);1136 if (ST->getRealMinVLen() == ST->getRealMaxVLen() && VLMAX <= 31)1137 InstrInfo.setAVLImm(VLMAX);1138 else1139 InstrInfo.setAVLVLMAX();1140 }1141 else1142 InstrInfo.setAVLImm(Imm);1143 } else if (VLOp.isUndef()) {1144 // Otherwise use an AVL of 1 to avoid depending on previous vl.1145 InstrInfo.setAVLImm(1);1146 } else {1147 VNInfo *VNI = getVNInfoFromReg(VLOp.getReg(), MI, LIS);1148 InstrInfo.setAVLRegDef(VNI, VLOp.getReg());1149 }1150 } else {1151 assert(RISCVInstrInfo::isScalarExtractInstr(MI) ||1152 RISCVInstrInfo::isVExtractInstr(MI));1153 // Pick a random value for state tracking purposes, will be ignored via1154 // the demanded fields mechanism1155 InstrInfo.setAVLImm(1);1156 }1157#ifndef NDEBUG1158 if (std::optional<unsigned> EEW = getEEWForLoadStore(MI)) {1159 assert(SEW == EEW && "Initial SEW doesn't match expected EEW");1160 }1161#endif1162 // TODO: Propagate the twiden from previous vtype for potential reuse.1163 InstrInfo.setVTYPE(VLMul, SEW, TailAgnostic, MaskAgnostic, AltFmt,1164 /*TWiden*/ 0);1165 1166 forwardVSETVLIAVL(InstrInfo);1167 1168 return InstrInfo;1169}1170 1171void RISCVInsertVSETVLI::insertVSETVLI(MachineBasicBlock &MBB,1172 MachineBasicBlock::iterator InsertPt,1173 DebugLoc DL, const VSETVLIInfo &Info,1174 const VSETVLIInfo &PrevInfo) {1175 ++NumInsertedVSETVL;1176 1177 if (Info.getTWiden()) {1178 if (Info.hasAVLVLMAX()) {1179 Register DestReg = MRI->createVirtualRegister(&RISCV::GPRNoX0RegClass);1180 auto MI = BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoSF_VSETTNTX0))1181 .addReg(DestReg, RegState::Define | RegState::Dead)1182 .addReg(RISCV::X0, RegState::Kill)1183 .addImm(Info.encodeVTYPE());1184 if (LIS) {1185 LIS->InsertMachineInstrInMaps(*MI);1186 LIS->createAndComputeVirtRegInterval(DestReg);1187 }1188 } else {1189 auto MI = BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoSF_VSETTNT))1190 .addReg(RISCV::X0, RegState::Define | RegState::Dead)1191 .addReg(Info.getAVLReg())1192 .addImm(Info.encodeVTYPE());1193 if (LIS)1194 LIS->InsertMachineInstrInMaps(*MI);1195 }1196 return;1197 }1198 1199 if (PrevInfo.isValid() && !PrevInfo.isUnknown()) {1200 // Use X0, X0 form if the AVL is the same and the SEW+LMUL gives the same1201 // VLMAX.1202 if (Info.hasSameAVL(PrevInfo) && Info.hasSameVLMAX(PrevInfo)) {1203 auto MI = BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0X0))1204 .addReg(RISCV::X0, RegState::Define | RegState::Dead)1205 .addReg(RISCV::X0, RegState::Kill)1206 .addImm(Info.encodeVTYPE())1207 .addReg(RISCV::VL, RegState::Implicit);1208 if (LIS)1209 LIS->InsertMachineInstrInMaps(*MI);1210 return;1211 }1212 1213 // If our AVL is a virtual register, it might be defined by a VSET(I)VLI. If1214 // it has the same VLMAX we want and the last VL/VTYPE we observed is the1215 // same, we can use the X0, X0 form.1216 if (Info.hasSameVLMAX(PrevInfo) && Info.hasAVLReg()) {1217 if (const MachineInstr *DefMI = Info.getAVLDefMI(LIS);1218 DefMI && RISCVInstrInfo::isVectorConfigInstr(*DefMI)) {1219 VSETVLIInfo DefInfo = getInfoForVSETVLI(*DefMI);1220 if (DefInfo.hasSameAVL(PrevInfo) && DefInfo.hasSameVLMAX(PrevInfo)) {1221 auto MI =1222 BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0X0))1223 .addReg(RISCV::X0, RegState::Define | RegState::Dead)1224 .addReg(RISCV::X0, RegState::Kill)1225 .addImm(Info.encodeVTYPE())1226 .addReg(RISCV::VL, RegState::Implicit);1227 if (LIS)1228 LIS->InsertMachineInstrInMaps(*MI);1229 return;1230 }1231 }1232 }1233 }1234 1235 if (Info.hasAVLImm()) {1236 auto MI = BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETIVLI))1237 .addReg(RISCV::X0, RegState::Define | RegState::Dead)1238 .addImm(Info.getAVLImm())1239 .addImm(Info.encodeVTYPE());1240 if (LIS)1241 LIS->InsertMachineInstrInMaps(*MI);1242 return;1243 }1244 1245 if (Info.hasAVLVLMAX()) {1246 Register DestReg = MRI->createVirtualRegister(&RISCV::GPRNoX0RegClass);1247 auto MI = BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0))1248 .addReg(DestReg, RegState::Define | RegState::Dead)1249 .addReg(RISCV::X0, RegState::Kill)1250 .addImm(Info.encodeVTYPE());1251 if (LIS) {1252 LIS->InsertMachineInstrInMaps(*MI);1253 LIS->createAndComputeVirtRegInterval(DestReg);1254 }1255 return;1256 }1257 1258 Register AVLReg = Info.getAVLReg();1259 MRI->constrainRegClass(AVLReg, &RISCV::GPRNoX0RegClass);1260 auto MI = BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLI))1261 .addReg(RISCV::X0, RegState::Define | RegState::Dead)1262 .addReg(AVLReg)1263 .addImm(Info.encodeVTYPE());1264 if (LIS) {1265 LIS->InsertMachineInstrInMaps(*MI);1266 LiveInterval &LI = LIS->getInterval(AVLReg);1267 SlotIndex SI = LIS->getInstructionIndex(*MI).getRegSlot();1268 const VNInfo *CurVNI = Info.getAVLVNInfo();1269 // If the AVL value isn't live at MI, do a quick check to see if it's easily1270 // extendable. Otherwise, we need to copy it.1271 if (LI.getVNInfoBefore(SI) != CurVNI) {1272 if (!LI.liveAt(SI) && LI.containsOneValue())1273 LIS->extendToIndices(LI, SI);1274 else {1275 Register AVLCopyReg =1276 MRI->createVirtualRegister(&RISCV::GPRNoX0RegClass);1277 MachineBasicBlock *MBB = LIS->getMBBFromIndex(CurVNI->def);1278 MachineBasicBlock::iterator II;1279 if (CurVNI->isPHIDef())1280 II = MBB->getFirstNonPHI();1281 else {1282 II = LIS->getInstructionFromIndex(CurVNI->def);1283 II = std::next(II);1284 }1285 assert(II.isValid());1286 auto AVLCopy = BuildMI(*MBB, II, DL, TII->get(RISCV::COPY), AVLCopyReg)1287 .addReg(AVLReg);1288 LIS->InsertMachineInstrInMaps(*AVLCopy);1289 MI->getOperand(1).setReg(AVLCopyReg);1290 LIS->createAndComputeVirtRegInterval(AVLCopyReg);1291 }1292 }1293 }1294}1295 1296/// Return true if a VSETVLI is required to transition from CurInfo to Require1297/// given a set of DemandedFields \p Used.1298bool RISCVInsertVSETVLI::needVSETVLI(const DemandedFields &Used,1299 const VSETVLIInfo &Require,1300 const VSETVLIInfo &CurInfo) const {1301 if (!CurInfo.isValid() || CurInfo.isUnknown() || CurInfo.hasSEWLMULRatioOnly())1302 return true;1303 1304 if (CurInfo.isCompatible(Used, Require, LIS))1305 return false;1306 1307 return true;1308}1309 1310// If we don't use LMUL or the SEW/LMUL ratio, then adjust LMUL so that we1311// maintain the SEW/LMUL ratio. This allows us to eliminate VL toggles in more1312// places.1313static VSETVLIInfo adjustIncoming(const VSETVLIInfo &PrevInfo,1314 const VSETVLIInfo &NewInfo,1315 DemandedFields &Demanded) {1316 VSETVLIInfo Info = NewInfo;1317 1318 if (!Demanded.LMUL && !Demanded.SEWLMULRatio && PrevInfo.isValid() &&1319 !PrevInfo.isUnknown()) {1320 if (auto NewVLMul = RISCVVType::getSameRatioLMUL(1321 PrevInfo.getSEW(), PrevInfo.getVLMUL(), Info.getSEW()))1322 Info.setVLMul(*NewVLMul);1323 Demanded.LMUL = DemandedFields::LMULEqual;1324 }1325 1326 return Info;1327}1328 1329// Given an incoming state reaching MI, minimally modifies that state so that it1330// is compatible with MI. The resulting state is guaranteed to be semantically1331// legal for MI, but may not be the state requested by MI.1332void RISCVInsertVSETVLI::transferBefore(VSETVLIInfo &Info,1333 const MachineInstr &MI) const {1334 if (isVectorCopy(ST->getRegisterInfo(), MI) &&1335 (Info.isUnknown() || !Info.isValid() || Info.hasSEWLMULRatioOnly())) {1336 // Use an arbitrary but valid AVL and VTYPE so vill will be cleared. It may1337 // be coalesced into another vsetvli since we won't demand any fields.1338 VSETVLIInfo NewInfo; // Need a new VSETVLIInfo to clear SEWLMULRatioOnly1339 NewInfo.setAVLImm(1);1340 NewInfo.setVTYPE(RISCVVType::LMUL_1, /*sew*/ 8, /*ta*/ true, /*ma*/ true,1341 /*AltFmt*/ false, /*W*/ 0);1342 Info = NewInfo;1343 return;1344 }1345 1346 if (!RISCVII::hasSEWOp(MI.getDesc().TSFlags))1347 return;1348 1349 DemandedFields Demanded = getDemanded(MI, ST);1350 1351 const VSETVLIInfo NewInfo = computeInfoForInstr(MI);1352 assert(NewInfo.isValid() && !NewInfo.isUnknown());1353 if (Info.isValid() && !needVSETVLI(Demanded, NewInfo, Info))1354 return;1355 1356 const VSETVLIInfo PrevInfo = Info;1357 if (!Info.isValid() || Info.isUnknown())1358 Info = NewInfo;1359 1360 const VSETVLIInfo IncomingInfo = adjustIncoming(PrevInfo, NewInfo, Demanded);1361 1362 // If MI only demands that VL has the same zeroness, we only need to set the1363 // AVL if the zeroness differs. This removes a vsetvli entirely if the types1364 // match or allows use of cheaper avl preserving variant if VLMAX doesn't1365 // change. If VLMAX might change, we couldn't use the 'vsetvli x0, x0, vtype"1366 // variant, so we avoid the transform to prevent extending live range of an1367 // avl register operand.1368 // TODO: We can probably relax this for immediates.1369 bool EquallyZero = IncomingInfo.hasEquallyZeroAVL(PrevInfo, LIS) &&1370 IncomingInfo.hasSameVLMAX(PrevInfo);1371 if (Demanded.VLAny || (Demanded.VLZeroness && !EquallyZero))1372 Info.setAVL(IncomingInfo);1373 1374 Info.setVTYPE(1375 ((Demanded.LMUL || Demanded.SEWLMULRatio) ? IncomingInfo : Info)1376 .getVLMUL(),1377 ((Demanded.SEW || Demanded.SEWLMULRatio) ? IncomingInfo : Info).getSEW(),1378 // Prefer tail/mask agnostic since it can be relaxed to undisturbed later1379 // if needed.1380 (Demanded.TailPolicy ? IncomingInfo : Info).getTailAgnostic() ||1381 IncomingInfo.getTailAgnostic(),1382 (Demanded.MaskPolicy ? IncomingInfo : Info).getMaskAgnostic() ||1383 IncomingInfo.getMaskAgnostic(),1384 (Demanded.AltFmt ? IncomingInfo : Info).getAltFmt(),1385 Demanded.TWiden ? IncomingInfo.getTWiden() : 0);1386 1387 // If we only knew the sew/lmul ratio previously, replace the VTYPE but keep1388 // the AVL.1389 if (Info.hasSEWLMULRatioOnly()) {1390 VSETVLIInfo RatiolessInfo = IncomingInfo;1391 RatiolessInfo.setAVL(Info);1392 Info = RatiolessInfo;1393 }1394}1395 1396// Given a state with which we evaluated MI (see transferBefore above for why1397// this might be different that the state MI requested), modify the state to1398// reflect the changes MI might make.1399void RISCVInsertVSETVLI::transferAfter(VSETVLIInfo &Info,1400 const MachineInstr &MI) const {1401 if (RISCVInstrInfo::isVectorConfigInstr(MI)) {1402 Info = getInfoForVSETVLI(MI);1403 return;1404 }1405 1406 if (RISCVInstrInfo::isFaultOnlyFirstLoad(MI)) {1407 // Update AVL to vl-output of the fault first load.1408 assert(MI.getOperand(1).getReg().isVirtual());1409 if (LIS) {1410 auto &LI = LIS->getInterval(MI.getOperand(1).getReg());1411 SlotIndex SI =1412 LIS->getSlotIndexes()->getInstructionIndex(MI).getRegSlot();1413 VNInfo *VNI = LI.getVNInfoAt(SI);1414 Info.setAVLRegDef(VNI, MI.getOperand(1).getReg());1415 } else1416 Info.setAVLRegDef(nullptr, MI.getOperand(1).getReg());1417 return;1418 }1419 1420 // If this is something that updates VL/VTYPE that we don't know about, set1421 // the state to unknown.1422 if (MI.isCall() || MI.isInlineAsm() ||1423 MI.modifiesRegister(RISCV::VL, /*TRI=*/nullptr) ||1424 MI.modifiesRegister(RISCV::VTYPE, /*TRI=*/nullptr))1425 Info = VSETVLIInfo::getUnknown();1426}1427 1428bool RISCVInsertVSETVLI::computeVLVTYPEChanges(const MachineBasicBlock &MBB,1429 VSETVLIInfo &Info) const {1430 bool HadVectorOp = false;1431 1432 Info = BlockInfo[MBB.getNumber()].Pred;1433 for (const MachineInstr &MI : MBB) {1434 transferBefore(Info, MI);1435 1436 if (RISCVInstrInfo::isVectorConfigInstr(MI) ||1437 RISCVII::hasSEWOp(MI.getDesc().TSFlags) ||1438 isVectorCopy(ST->getRegisterInfo(), MI) ||1439 RISCVInstrInfo::isXSfmmVectorConfigInstr(MI))1440 HadVectorOp = true;1441 1442 transferAfter(Info, MI);1443 }1444 1445 return HadVectorOp;1446}1447 1448void RISCVInsertVSETVLI::computeIncomingVLVTYPE(const MachineBasicBlock &MBB) {1449 1450 BlockData &BBInfo = BlockInfo[MBB.getNumber()];1451 1452 BBInfo.InQueue = false;1453 1454 // Start with the previous entry so that we keep the most conservative state1455 // we have ever found.1456 VSETVLIInfo InInfo = BBInfo.Pred;1457 if (MBB.pred_empty()) {1458 // There are no predecessors, so use the default starting status.1459 InInfo.setUnknown();1460 } else {1461 for (MachineBasicBlock *P : MBB.predecessors())1462 InInfo = InInfo.intersect(BlockInfo[P->getNumber()].Exit);1463 }1464 1465 // If we don't have any valid predecessor value, wait until we do.1466 if (!InInfo.isValid())1467 return;1468 1469 // If no change, no need to rerun block1470 if (InInfo == BBInfo.Pred)1471 return;1472 1473 BBInfo.Pred = InInfo;1474 LLVM_DEBUG(dbgs() << "Entry state of " << printMBBReference(MBB)1475 << " changed to " << BBInfo.Pred << "\n");1476 1477 // Note: It's tempting to cache the state changes here, but due to the1478 // compatibility checks performed a blocks output state can change based on1479 // the input state. To cache, we'd have to add logic for finding1480 // never-compatible state changes.1481 VSETVLIInfo TmpStatus;1482 computeVLVTYPEChanges(MBB, TmpStatus);1483 1484 // If the new exit value matches the old exit value, we don't need to revisit1485 // any blocks.1486 if (BBInfo.Exit == TmpStatus)1487 return;1488 1489 BBInfo.Exit = TmpStatus;1490 LLVM_DEBUG(dbgs() << "Exit state of " << printMBBReference(MBB)1491 << " changed to " << BBInfo.Exit << "\n");1492 1493 // Add the successors to the work list so we can propagate the changed exit1494 // status.1495 for (MachineBasicBlock *S : MBB.successors())1496 if (!BlockInfo[S->getNumber()].InQueue) {1497 BlockInfo[S->getNumber()].InQueue = true;1498 WorkList.push(S);1499 }1500}1501 1502// If we weren't able to prove a vsetvli was directly unneeded, it might still1503// be unneeded if the AVL was a phi node where all incoming values are VL1504// outputs from the last VSETVLI in their respective basic blocks.1505bool RISCVInsertVSETVLI::needVSETVLIPHI(const VSETVLIInfo &Require,1506 const MachineBasicBlock &MBB) const {1507 if (!Require.hasAVLReg())1508 return true;1509 1510 if (!LIS)1511 return true;1512 1513 // We need the AVL to have been produced by a PHI node in this basic block.1514 const VNInfo *Valno = Require.getAVLVNInfo();1515 if (!Valno->isPHIDef() || LIS->getMBBFromIndex(Valno->def) != &MBB)1516 return true;1517 1518 const LiveRange &LR = LIS->getInterval(Require.getAVLReg());1519 1520 for (auto *PBB : MBB.predecessors()) {1521 const VSETVLIInfo &PBBExit = BlockInfo[PBB->getNumber()].Exit;1522 1523 // We need the PHI input to the be the output of a VSET(I)VLI.1524 const VNInfo *Value = LR.getVNInfoBefore(LIS->getMBBEndIdx(PBB));1525 if (!Value)1526 return true;1527 MachineInstr *DefMI = LIS->getInstructionFromIndex(Value->def);1528 if (!DefMI || !RISCVInstrInfo::isVectorConfigInstr(*DefMI))1529 return true;1530 1531 // We found a VSET(I)VLI make sure it matches the output of the1532 // predecessor block.1533 VSETVLIInfo DefInfo = getInfoForVSETVLI(*DefMI);1534 if (DefInfo != PBBExit)1535 return true;1536 1537 // Require has the same VL as PBBExit, so if the exit from the1538 // predecessor has the VTYPE we are looking for we might be able1539 // to avoid a VSETVLI.1540 if (PBBExit.isUnknown() || !PBBExit.hasSameVTYPE(Require))1541 return true;1542 }1543 1544 // If all the incoming values to the PHI checked out, we don't need1545 // to insert a VSETVLI.1546 return false;1547}1548 1549void RISCVInsertVSETVLI::emitVSETVLIs(MachineBasicBlock &MBB) {1550 VSETVLIInfo CurInfo = BlockInfo[MBB.getNumber()].Pred;1551 // Track whether the prefix of the block we've scanned is transparent1552 // (meaning has not yet changed the abstract state).1553 bool PrefixTransparent = true;1554 for (MachineInstr &MI : MBB) {1555 const VSETVLIInfo PrevInfo = CurInfo;1556 transferBefore(CurInfo, MI);1557 1558 // If this is an explicit VSETVLI or VSETIVLI, update our state.1559 if (RISCVInstrInfo::isVectorConfigInstr(MI)) {1560 // Conservatively, mark the VL and VTYPE as live.1561 assert(MI.getOperand(3).getReg() == RISCV::VL &&1562 MI.getOperand(4).getReg() == RISCV::VTYPE &&1563 "Unexpected operands where VL and VTYPE should be");1564 MI.getOperand(3).setIsDead(false);1565 MI.getOperand(4).setIsDead(false);1566 PrefixTransparent = false;1567 }1568 1569 if (EnsureWholeVectorRegisterMoveValidVTYPE &&1570 isVectorCopy(ST->getRegisterInfo(), MI)) {1571 if (!PrevInfo.isCompatible(DemandedFields::all(), CurInfo, LIS)) {1572 insertVSETVLI(MBB, MI, MI.getDebugLoc(), CurInfo, PrevInfo);1573 PrefixTransparent = false;1574 }1575 MI.addOperand(MachineOperand::CreateReg(RISCV::VTYPE, /*isDef*/ false,1576 /*isImp*/ true));1577 }1578 1579 uint64_t TSFlags = MI.getDesc().TSFlags;1580 if (RISCVII::hasSEWOp(TSFlags)) {1581 if (!PrevInfo.isCompatible(DemandedFields::all(), CurInfo, LIS)) {1582 // If this is the first implicit state change, and the state change1583 // requested can be proven to produce the same register contents, we1584 // can skip emitting the actual state change and continue as if we1585 // had since we know the GPR result of the implicit state change1586 // wouldn't be used and VL/VTYPE registers are correct. Note that1587 // we *do* need to model the state as if it changed as while the1588 // register contents are unchanged, the abstract model can change.1589 if (!PrefixTransparent || needVSETVLIPHI(CurInfo, MBB))1590 insertVSETVLI(MBB, MI, MI.getDebugLoc(), CurInfo, PrevInfo);1591 PrefixTransparent = false;1592 }1593 1594 if (RISCVII::hasVLOp(TSFlags)) {1595 MachineOperand &VLOp = MI.getOperand(getVLOpNum(MI));1596 if (VLOp.isReg()) {1597 Register Reg = VLOp.getReg();1598 1599 // Erase the AVL operand from the instruction.1600 VLOp.setReg(Register());1601 VLOp.setIsKill(false);1602 if (LIS) {1603 LiveInterval &LI = LIS->getInterval(Reg);1604 SmallVector<MachineInstr *> DeadMIs;1605 LIS->shrinkToUses(&LI, &DeadMIs);1606 // We might have separate components that need split due to1607 // needVSETVLIPHI causing us to skip inserting a new VL def.1608 SmallVector<LiveInterval *> SplitLIs;1609 LIS->splitSeparateComponents(LI, SplitLIs);1610 1611 // If the AVL was an immediate > 31, then it would have been emitted1612 // as an ADDI. However, the ADDI might not have been used in the1613 // vsetvli, or a vsetvli might not have been emitted, so it may be1614 // dead now.1615 for (MachineInstr *DeadMI : DeadMIs) {1616 if (!TII->isAddImmediate(*DeadMI, Reg))1617 continue;1618 LIS->RemoveMachineInstrFromMaps(*DeadMI);1619 Register AddReg = DeadMI->getOperand(1).getReg();1620 DeadMI->eraseFromParent();1621 if (AddReg.isVirtual())1622 LIS->shrinkToUses(&LIS->getInterval(AddReg));1623 }1624 }1625 }1626 MI.addOperand(MachineOperand::CreateReg(RISCV::VL, /*isDef*/ false,1627 /*isImp*/ true));1628 }1629 MI.addOperand(MachineOperand::CreateReg(RISCV::VTYPE, /*isDef*/ false,1630 /*isImp*/ true));1631 }1632 1633 if (MI.isInlineAsm()) {1634 MI.addOperand(MachineOperand::CreateReg(RISCV::VL, /*isDef*/ true,1635 /*isImp*/ true));1636 MI.addOperand(MachineOperand::CreateReg(RISCV::VTYPE, /*isDef*/ true,1637 /*isImp*/ true));1638 }1639 1640 if (MI.isCall() || MI.isInlineAsm() ||1641 MI.modifiesRegister(RISCV::VL, /*TRI=*/nullptr) ||1642 MI.modifiesRegister(RISCV::VTYPE, /*TRI=*/nullptr))1643 PrefixTransparent = false;1644 1645 transferAfter(CurInfo, MI);1646 }1647 1648 const auto &Info = BlockInfo[MBB.getNumber()];1649 if (CurInfo != Info.Exit) {1650 LLVM_DEBUG(dbgs() << "in block " << printMBBReference(MBB) << "\n");1651 LLVM_DEBUG(dbgs() << " begin state: " << Info.Pred << "\n");1652 LLVM_DEBUG(dbgs() << " expected end state: " << Info.Exit << "\n");1653 LLVM_DEBUG(dbgs() << " actual end state: " << CurInfo << "\n");1654 }1655 assert(CurInfo == Info.Exit && "InsertVSETVLI dataflow invariant violated");1656}1657 1658/// Perform simple partial redundancy elimination of the VSETVLI instructions1659/// we're about to insert by looking for cases where we can PRE from the1660/// beginning of one block to the end of one of its predecessors. Specifically,1661/// this is geared to catch the common case of a fixed length vsetvl in a single1662/// block loop when it could execute once in the preheader instead.1663void RISCVInsertVSETVLI::doPRE(MachineBasicBlock &MBB) {1664 if (!BlockInfo[MBB.getNumber()].Pred.isUnknown())1665 return;1666 1667 MachineBasicBlock *UnavailablePred = nullptr;1668 VSETVLIInfo AvailableInfo;1669 for (MachineBasicBlock *P : MBB.predecessors()) {1670 const VSETVLIInfo &PredInfo = BlockInfo[P->getNumber()].Exit;1671 if (PredInfo.isUnknown()) {1672 if (UnavailablePred)1673 return;1674 UnavailablePred = P;1675 } else if (!AvailableInfo.isValid()) {1676 AvailableInfo = PredInfo;1677 } else if (AvailableInfo != PredInfo) {1678 return;1679 }1680 }1681 1682 // Unreachable, single pred, or full redundancy. Note that FRE is handled by1683 // phase 3.1684 if (!UnavailablePred || !AvailableInfo.isValid())1685 return;1686 1687 if (!LIS)1688 return;1689 1690 // If we don't know the exact VTYPE, we can't copy the vsetvli to the exit of1691 // the unavailable pred.1692 if (AvailableInfo.hasSEWLMULRatioOnly())1693 return;1694 1695 // Critical edge - TODO: consider splitting?1696 if (UnavailablePred->succ_size() != 1)1697 return;1698 1699 // If the AVL value is a register (other than our VLMAX sentinel),1700 // we need to prove the value is available at the point we're going1701 // to insert the vsetvli at.1702 if (AvailableInfo.hasAVLReg()) {1703 SlotIndex SI = AvailableInfo.getAVLVNInfo()->def;1704 // This is an inline dominance check which covers the case of1705 // UnavailablePred being the preheader of a loop.1706 if (LIS->getMBBFromIndex(SI) != UnavailablePred)1707 return;1708 if (!UnavailablePred->terminators().empty() &&1709 SI >= LIS->getInstructionIndex(*UnavailablePred->getFirstTerminator()))1710 return;1711 }1712 1713 // Model the effect of changing the input state of the block MBB to1714 // AvailableInfo. We're looking for two issues here; one legality,1715 // one profitability.1716 // 1) If the block doesn't use some of the fields from VL or VTYPE, we1717 // may hit the end of the block with a different end state. We can1718 // not make this change without reflowing later blocks as well.1719 // 2) If we don't actually remove a transition, inserting a vsetvli1720 // into the predecessor block would be correct, but unprofitable.1721 VSETVLIInfo OldInfo = BlockInfo[MBB.getNumber()].Pred;1722 VSETVLIInfo CurInfo = AvailableInfo;1723 int TransitionsRemoved = 0;1724 for (const MachineInstr &MI : MBB) {1725 const VSETVLIInfo LastInfo = CurInfo;1726 const VSETVLIInfo LastOldInfo = OldInfo;1727 transferBefore(CurInfo, MI);1728 transferBefore(OldInfo, MI);1729 if (CurInfo == LastInfo)1730 TransitionsRemoved++;1731 if (LastOldInfo == OldInfo)1732 TransitionsRemoved--;1733 transferAfter(CurInfo, MI);1734 transferAfter(OldInfo, MI);1735 if (CurInfo == OldInfo)1736 // Convergence. All transitions after this must match by construction.1737 break;1738 }1739 if (CurInfo != OldInfo || TransitionsRemoved <= 0)1740 // Issues 1 and 2 above1741 return;1742 1743 // Finally, update both data flow state and insert the actual vsetvli.1744 // Doing both keeps the code in sync with the dataflow results, which1745 // is critical for correctness of phase 3.1746 auto OldExit = BlockInfo[UnavailablePred->getNumber()].Exit;1747 LLVM_DEBUG(dbgs() << "PRE VSETVLI from " << MBB.getName() << " to "1748 << UnavailablePred->getName() << " with state "1749 << AvailableInfo << "\n");1750 BlockInfo[UnavailablePred->getNumber()].Exit = AvailableInfo;1751 BlockInfo[MBB.getNumber()].Pred = AvailableInfo;1752 1753 // Note there's an implicit assumption here that terminators never use1754 // or modify VL or VTYPE. Also, fallthrough will return end().1755 auto InsertPt = UnavailablePred->getFirstInstrTerminator();1756 insertVSETVLI(*UnavailablePred, InsertPt,1757 UnavailablePred->findDebugLoc(InsertPt),1758 AvailableInfo, OldExit);1759}1760 1761// Return true if we can mutate PrevMI to match MI without changing any the1762// fields which would be observed.1763bool RISCVInsertVSETVLI::canMutatePriorConfig(1764 const MachineInstr &PrevMI, const MachineInstr &MI,1765 const DemandedFields &Used) const {1766 // If the VL values aren't equal, return false if either a) the former is1767 // demanded, or b) we can't rewrite the former to be the later for1768 // implementation reasons.1769 if (!RISCVInstrInfo::isVLPreservingConfig(MI)) {1770 if (Used.VLAny)1771 return false;1772 1773 if (Used.VLZeroness) {1774 if (RISCVInstrInfo::isVLPreservingConfig(PrevMI))1775 return false;1776 if (!getInfoForVSETVLI(PrevMI).hasEquallyZeroAVL(getInfoForVSETVLI(MI),1777 LIS))1778 return false;1779 }1780 1781 auto &AVL = MI.getOperand(1);1782 1783 // If the AVL is a register, we need to make sure its definition is the same1784 // at PrevMI as it was at MI.1785 if (AVL.isReg() && AVL.getReg() != RISCV::X0) {1786 VNInfo *VNI = getVNInfoFromReg(AVL.getReg(), MI, LIS);1787 VNInfo *PrevVNI = getVNInfoFromReg(AVL.getReg(), PrevMI, LIS);1788 if (!VNI || !PrevVNI || VNI != PrevVNI)1789 return false;1790 }1791 1792 // If we define VL and need to move the definition up, check we can extend1793 // the live interval upwards from MI to PrevMI.1794 Register VL = MI.getOperand(0).getReg();1795 if (VL.isVirtual() && LIS &&1796 LIS->getInterval(VL).overlaps(LIS->getInstructionIndex(PrevMI),1797 LIS->getInstructionIndex(MI)))1798 return false;1799 }1800 1801 assert(PrevMI.getOperand(2).isImm() && MI.getOperand(2).isImm());1802 auto PriorVType = PrevMI.getOperand(2).getImm();1803 auto VType = MI.getOperand(2).getImm();1804 return areCompatibleVTYPEs(PriorVType, VType, Used);1805}1806 1807void RISCVInsertVSETVLI::coalesceVSETVLIs(MachineBasicBlock &MBB) const {1808 MachineInstr *NextMI = nullptr;1809 // We can have arbitrary code in successors, so VL and VTYPE1810 // must be considered demanded.1811 DemandedFields Used;1812 Used.demandVL();1813 Used.demandVTYPE();1814 SmallVector<MachineInstr*> ToDelete;1815 1816 auto dropAVLUse = [&](MachineOperand &MO) {1817 if (!MO.isReg() || !MO.getReg().isVirtual())1818 return;1819 Register OldVLReg = MO.getReg();1820 MO.setReg(Register());1821 1822 if (LIS)1823 LIS->shrinkToUses(&LIS->getInterval(OldVLReg));1824 1825 MachineInstr *VLOpDef = MRI->getUniqueVRegDef(OldVLReg);1826 if (VLOpDef && TII->isAddImmediate(*VLOpDef, OldVLReg) &&1827 MRI->use_nodbg_empty(OldVLReg))1828 ToDelete.push_back(VLOpDef);1829 };1830 1831 for (MachineInstr &MI : make_early_inc_range(reverse(MBB))) {1832 // TODO: Support XSfmm.1833 if (RISCVII::hasTWidenOp(MI.getDesc().TSFlags) ||1834 RISCVInstrInfo::isXSfmmVectorConfigInstr(MI)) {1835 NextMI = nullptr;1836 continue;1837 }1838 1839 if (!RISCVInstrInfo::isVectorConfigInstr(MI)) {1840 Used.doUnion(getDemanded(MI, ST));1841 if (MI.isCall() || MI.isInlineAsm() ||1842 MI.modifiesRegister(RISCV::VL, /*TRI=*/nullptr) ||1843 MI.modifiesRegister(RISCV::VTYPE, /*TRI=*/nullptr))1844 NextMI = nullptr;1845 continue;1846 }1847 1848 if (!MI.getOperand(0).isDead())1849 Used.demandVL();1850 1851 if (NextMI) {1852 if (!Used.usedVL() && !Used.usedVTYPE()) {1853 dropAVLUse(MI.getOperand(1));1854 if (LIS)1855 LIS->RemoveMachineInstrFromMaps(MI);1856 MI.eraseFromParent();1857 NumCoalescedVSETVL++;1858 // Leave NextMI unchanged1859 continue;1860 }1861 1862 if (canMutatePriorConfig(MI, *NextMI, Used)) {1863 if (!RISCVInstrInfo::isVLPreservingConfig(*NextMI)) {1864 Register DefReg = NextMI->getOperand(0).getReg();1865 1866 MI.getOperand(0).setReg(DefReg);1867 MI.getOperand(0).setIsDead(false);1868 1869 // Move the AVL from NextMI to MI1870 dropAVLUse(MI.getOperand(1));1871 if (NextMI->getOperand(1).isImm())1872 MI.getOperand(1).ChangeToImmediate(NextMI->getOperand(1).getImm());1873 else1874 MI.getOperand(1).ChangeToRegister(NextMI->getOperand(1).getReg(),1875 false);1876 dropAVLUse(NextMI->getOperand(1));1877 1878 // The def of DefReg moved to MI, so extend the LiveInterval up to1879 // it.1880 if (DefReg.isVirtual() && LIS) {1881 LiveInterval &DefLI = LIS->getInterval(DefReg);1882 SlotIndex MISlot = LIS->getInstructionIndex(MI).getRegSlot();1883 SlotIndex NextMISlot =1884 LIS->getInstructionIndex(*NextMI).getRegSlot();1885 VNInfo *DefVNI = DefLI.getVNInfoAt(NextMISlot);1886 LiveInterval::Segment S(MISlot, NextMISlot, DefVNI);1887 DefLI.addSegment(S);1888 DefVNI->def = MISlot;1889 // Mark DefLI as spillable if it was previously unspillable1890 DefLI.setWeight(0);1891 1892 // DefReg may have had no uses, in which case we need to shrink1893 // the LiveInterval up to MI.1894 LIS->shrinkToUses(&DefLI);1895 }1896 1897 MI.setDesc(NextMI->getDesc());1898 }1899 MI.getOperand(2).setImm(NextMI->getOperand(2).getImm());1900 1901 dropAVLUse(NextMI->getOperand(1));1902 if (LIS)1903 LIS->RemoveMachineInstrFromMaps(*NextMI);1904 NextMI->eraseFromParent();1905 NumCoalescedVSETVL++;1906 // fallthrough1907 }1908 }1909 NextMI = &MI;1910 Used = getDemanded(MI, ST);1911 }1912 1913 // Loop over the dead AVL values, and delete them now. This has1914 // to be outside the above loop to avoid invalidating iterators.1915 for (auto *MI : ToDelete) {1916 assert(MI->getOpcode() == RISCV::ADDI);1917 Register AddReg = MI->getOperand(1).getReg();1918 if (LIS) {1919 LIS->removeInterval(MI->getOperand(0).getReg());1920 LIS->RemoveMachineInstrFromMaps(*MI);1921 }1922 MI->eraseFromParent();1923 if (LIS && AddReg.isVirtual())1924 LIS->shrinkToUses(&LIS->getInterval(AddReg));1925 }1926}1927 1928void RISCVInsertVSETVLI::insertReadVL(MachineBasicBlock &MBB) {1929 for (auto I = MBB.begin(), E = MBB.end(); I != E;) {1930 MachineInstr &MI = *I++;1931 if (RISCVInstrInfo::isFaultOnlyFirstLoad(MI)) {1932 Register VLOutput = MI.getOperand(1).getReg();1933 assert(VLOutput.isVirtual());1934 if (!MI.getOperand(1).isDead()) {1935 auto ReadVLMI = BuildMI(MBB, I, MI.getDebugLoc(),1936 TII->get(RISCV::PseudoReadVL), VLOutput);1937 // Move the LiveInterval's definition down to PseudoReadVL.1938 if (LIS) {1939 SlotIndex NewDefSI =1940 LIS->InsertMachineInstrInMaps(*ReadVLMI).getRegSlot();1941 LiveInterval &DefLI = LIS->getInterval(VLOutput);1942 LiveRange::Segment *DefSeg = DefLI.getSegmentContaining(NewDefSI);1943 VNInfo *DefVNI = DefLI.getVNInfoAt(DefSeg->start);1944 DefLI.removeSegment(DefSeg->start, NewDefSI);1945 DefVNI->def = NewDefSI;1946 }1947 }1948 // We don't use the vl output of the VLEFF/VLSEGFF anymore.1949 MI.getOperand(1).setReg(RISCV::X0);1950 MI.addRegisterDefined(RISCV::VL, MRI->getTargetRegisterInfo());1951 }1952 }1953}1954 1955bool RISCVInsertVSETVLI::insertVSETMTK(MachineBasicBlock &MBB,1956 TKTMMode Mode) const {1957 1958 bool Changed = false;1959 for (auto &MI : MBB) {1960 uint64_t TSFlags = MI.getDesc().TSFlags;1961 if (RISCVInstrInfo::isXSfmmVectorConfigTMTKInstr(MI) ||1962 !RISCVII::hasSEWOp(TSFlags) || !RISCVII::hasTWidenOp(TSFlags))1963 continue;1964 1965 VSETVLIInfo CurrInfo = computeInfoForInstr(MI);1966 1967 if (Mode == VSETTK && !RISCVII::hasTKOp(TSFlags))1968 continue;1969 1970 if (Mode == VSETTM && !RISCVII::hasTMOp(TSFlags))1971 continue;1972 1973 unsigned OpNum = 0;1974 unsigned Opcode = 0;1975 switch (Mode) {1976 case VSETTK:1977 OpNum = RISCVII::getTKOpNum(MI.getDesc());1978 Opcode = RISCV::PseudoSF_VSETTK;1979 break;1980 case VSETTM:1981 OpNum = RISCVII::getTMOpNum(MI.getDesc());1982 Opcode = RISCV::PseudoSF_VSETTM;1983 break;1984 }1985 1986 assert(OpNum && Opcode && "Invalid OpNum or Opcode");1987 1988 MachineOperand &Op = MI.getOperand(OpNum);1989 1990 auto TmpMI = BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(Opcode))1991 .addReg(RISCV::X0, RegState::Define | RegState::Dead)1992 .addReg(Op.getReg())1993 .addImm(Log2_32(CurrInfo.getSEW()))1994 .addImm(Log2_32(CurrInfo.getTWiden()) + 1);1995 1996 Changed = true;1997 Register Reg = Op.getReg();1998 Op.setReg(Register());1999 Op.setIsKill(false);2000 if (LIS) {2001 LIS->InsertMachineInstrInMaps(*TmpMI);2002 LiveInterval &LI = LIS->getInterval(Reg);2003 2004 // Erase the AVL operand from the instruction.2005 LIS->shrinkToUses(&LI);2006 // TODO: Enable this once needVSETVLIPHI is supported.2007 // SmallVector<LiveInterval *> SplitLIs;2008 // LIS->splitSeparateComponents(LI, SplitLIs);2009 }2010 }2011 return Changed;2012}2013 2014bool RISCVInsertVSETVLI::runOnMachineFunction(MachineFunction &MF) {2015 // Skip if the vector extension is not enabled.2016 ST = &MF.getSubtarget<RISCVSubtarget>();2017 if (!ST->hasVInstructions())2018 return false;2019 2020 LLVM_DEBUG(dbgs() << "Entering InsertVSETVLI for " << MF.getName() << "\n");2021 2022 TII = ST->getInstrInfo();2023 MRI = &MF.getRegInfo();2024 auto *LISWrapper = getAnalysisIfAvailable<LiveIntervalsWrapperPass>();2025 LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;2026 2027 assert(BlockInfo.empty() && "Expect empty block infos");2028 BlockInfo.resize(MF.getNumBlockIDs());2029 2030 bool HaveVectorOp = false;2031 2032 // Phase 1 - determine how VL/VTYPE are affected by the each block.2033 for (const MachineBasicBlock &MBB : MF) {2034 VSETVLIInfo TmpStatus;2035 HaveVectorOp |= computeVLVTYPEChanges(MBB, TmpStatus);2036 // Initial exit state is whatever change we found in the block.2037 BlockData &BBInfo = BlockInfo[MBB.getNumber()];2038 BBInfo.Exit = TmpStatus;2039 LLVM_DEBUG(dbgs() << "Initial exit state of " << printMBBReference(MBB)2040 << " is " << BBInfo.Exit << "\n");2041 2042 }2043 2044 // If we didn't find any instructions that need VSETVLI, we're done.2045 if (!HaveVectorOp) {2046 BlockInfo.clear();2047 return false;2048 }2049 2050 // Phase 2 - determine the exit VL/VTYPE from each block. We add all2051 // blocks to the list here, but will also add any that need to be revisited2052 // during Phase 2 processing.2053 for (const MachineBasicBlock &MBB : MF) {2054 WorkList.push(&MBB);2055 BlockInfo[MBB.getNumber()].InQueue = true;2056 }2057 while (!WorkList.empty()) {2058 const MachineBasicBlock &MBB = *WorkList.front();2059 WorkList.pop();2060 computeIncomingVLVTYPE(MBB);2061 }2062 2063 // Perform partial redundancy elimination of vsetvli transitions.2064 for (MachineBasicBlock &MBB : MF)2065 doPRE(MBB);2066 2067 // Phase 3 - add any vsetvli instructions needed in the block. Use the2068 // Phase 2 information to avoid adding vsetvlis before the first vector2069 // instruction in the block if the VL/VTYPE is satisfied by its2070 // predecessors.2071 for (MachineBasicBlock &MBB : MF)2072 emitVSETVLIs(MBB);2073 2074 // Now that all vsetvlis are explicit, go through and do block local2075 // DSE and peephole based demanded fields based transforms. Note that2076 // this *must* be done outside the main dataflow so long as we allow2077 // any cross block analysis within the dataflow. We can't have both2078 // demanded fields based mutation and non-local analysis in the2079 // dataflow at the same time without introducing inconsistencies.2080 // We're visiting blocks from the bottom up because a VSETVLI in the2081 // earlier block might become dead when its uses in later blocks are2082 // optimized away.2083 for (MachineBasicBlock *MBB : post_order(&MF))2084 coalesceVSETVLIs(*MBB);2085 2086 // Insert PseudoReadVL after VLEFF/VLSEGFF and replace it with the vl output2087 // of VLEFF/VLSEGFF.2088 for (MachineBasicBlock &MBB : MF)2089 insertReadVL(MBB);2090 2091 for (MachineBasicBlock &MBB : MF) {2092 insertVSETMTK(MBB, VSETTM);2093 insertVSETMTK(MBB, VSETTK);2094 }2095 2096 BlockInfo.clear();2097 return HaveVectorOp;2098}2099 2100/// Returns an instance of the Insert VSETVLI pass.2101FunctionPass *llvm::createRISCVInsertVSETVLIPass() {2102 return new RISCVInsertVSETVLI();2103}2104