666 lines · c
1//===- LoopVectorizationPlanner.h - Planner for LoopVectorization ---------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8///9/// \file10/// This file provides a LoopVectorizationPlanner class.11/// InnerLoopVectorizer vectorizes loops which contain only one basic12/// LoopVectorizationPlanner - drives the vectorization process after having13/// passed Legality checks.14/// The planner builds and optimizes the Vectorization Plans which record the15/// decisions how to vectorize the given loop. In particular, represent the16/// control-flow of the vectorized version, the replication of instructions that17/// are to be scalarized, and interleave access groups.18///19/// Also provides a VPlan-based builder utility analogous to IRBuilder.20/// It provides an instruction-level API for generating VPInstructions while21/// abstracting away the Recipe manipulation details.22//===----------------------------------------------------------------------===//23 24#ifndef LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONPLANNER_H25#define LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONPLANNER_H26 27#include "VPlan.h"28#include "llvm/ADT/SmallSet.h"29#include "llvm/Support/InstructionCost.h"30 31namespace {32class GeneratedRTChecks;33}34 35namespace llvm {36 37class LoopInfo;38class DominatorTree;39class LoopVectorizationLegality;40class LoopVectorizationCostModel;41class PredicatedScalarEvolution;42class LoopVectorizeHints;43class LoopVersioning;44class OptimizationRemarkEmitter;45class TargetTransformInfo;46class TargetLibraryInfo;47class VPRecipeBuilder;48struct VFRange;49 50extern cl::opt<bool> EnableVPlanNativePath;51extern cl::opt<unsigned> ForceTargetInstructionCost;52 53/// VPlan-based builder utility analogous to IRBuilder.54class VPBuilder {55 VPBasicBlock *BB = nullptr;56 VPBasicBlock::iterator InsertPt = VPBasicBlock::iterator();57 58 /// Insert \p VPI in BB at InsertPt if BB is set.59 template <typename T> T *tryInsertInstruction(T *R) {60 if (BB)61 BB->insert(R, InsertPt);62 return R;63 }64 65 VPInstruction *createInstruction(unsigned Opcode,66 ArrayRef<VPValue *> Operands,67 const VPIRMetadata &MD, DebugLoc DL,68 const Twine &Name = "") {69 return tryInsertInstruction(70 new VPInstruction(Opcode, Operands, {}, MD, DL, Name));71 }72 73public:74 VPBuilder() = default;75 VPBuilder(VPBasicBlock *InsertBB) { setInsertPoint(InsertBB); }76 VPBuilder(VPRecipeBase *InsertPt) { setInsertPoint(InsertPt); }77 VPBuilder(VPBasicBlock *TheBB, VPBasicBlock::iterator IP) {78 setInsertPoint(TheBB, IP);79 }80 81 /// Clear the insertion point: created instructions will not be inserted into82 /// a block.83 void clearInsertionPoint() {84 BB = nullptr;85 InsertPt = VPBasicBlock::iterator();86 }87 88 VPBasicBlock *getInsertBlock() const { return BB; }89 VPBasicBlock::iterator getInsertPoint() const { return InsertPt; }90 91 /// Create a VPBuilder to insert after \p R.92 static VPBuilder getToInsertAfter(VPRecipeBase *R) {93 VPBuilder B;94 B.setInsertPoint(R->getParent(), std::next(R->getIterator()));95 return B;96 }97 98 /// InsertPoint - A saved insertion point.99 class VPInsertPoint {100 VPBasicBlock *Block = nullptr;101 VPBasicBlock::iterator Point;102 103 public:104 /// Creates a new insertion point which doesn't point to anything.105 VPInsertPoint() = default;106 107 /// Creates a new insertion point at the given location.108 VPInsertPoint(VPBasicBlock *InsertBlock, VPBasicBlock::iterator InsertPoint)109 : Block(InsertBlock), Point(InsertPoint) {}110 111 /// Returns true if this insert point is set.112 bool isSet() const { return Block != nullptr; }113 114 VPBasicBlock *getBlock() const { return Block; }115 VPBasicBlock::iterator getPoint() const { return Point; }116 };117 118 /// Sets the current insert point to a previously-saved location.119 void restoreIP(VPInsertPoint IP) {120 if (IP.isSet())121 setInsertPoint(IP.getBlock(), IP.getPoint());122 else123 clearInsertionPoint();124 }125 126 /// This specifies that created VPInstructions should be appended to the end127 /// of the specified block.128 void setInsertPoint(VPBasicBlock *TheBB) {129 assert(TheBB && "Attempting to set a null insert point");130 BB = TheBB;131 InsertPt = BB->end();132 }133 134 /// This specifies that created instructions should be inserted at the135 /// specified point.136 void setInsertPoint(VPBasicBlock *TheBB, VPBasicBlock::iterator IP) {137 BB = TheBB;138 InsertPt = IP;139 }140 141 /// This specifies that created instructions should be inserted at the142 /// specified point.143 void setInsertPoint(VPRecipeBase *IP) {144 BB = IP->getParent();145 InsertPt = IP->getIterator();146 }147 148 /// Insert \p R at the current insertion point.149 void insert(VPRecipeBase *R) { BB->insert(R, InsertPt); }150 151 /// Create an N-ary operation with \p Opcode, \p Operands and set \p Inst as152 /// its underlying Instruction.153 VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,154 Instruction *Inst = nullptr,155 const VPIRFlags &Flags = {},156 const VPIRMetadata &MD = {},157 DebugLoc DL = DebugLoc::getUnknown(),158 const Twine &Name = "") {159 VPInstruction *NewVPInst = tryInsertInstruction(160 new VPInstruction(Opcode, Operands, Flags, MD, DL, Name));161 NewVPInst->setUnderlyingValue(Inst);162 return NewVPInst;163 }164 VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,165 DebugLoc DL, const Twine &Name = "") {166 return createInstruction(Opcode, Operands, {}, DL, Name);167 }168 VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,169 const VPIRFlags &Flags,170 DebugLoc DL = DebugLoc::getUnknown(),171 const Twine &Name = "") {172 return tryInsertInstruction(173 new VPInstruction(Opcode, Operands, Flags, {}, DL, Name));174 }175 176 VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,177 Type *ResultTy, const VPIRFlags &Flags = {},178 DebugLoc DL = DebugLoc::getUnknown(),179 const Twine &Name = "") {180 return tryInsertInstruction(new VPInstructionWithType(181 Opcode, Operands, ResultTy, Flags, {}, DL, Name));182 }183 184 VPInstruction *createOverflowingOp(185 unsigned Opcode, ArrayRef<VPValue *> Operands,186 VPRecipeWithIRFlags::WrapFlagsTy WrapFlags = {false, false},187 DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "") {188 return tryInsertInstruction(189 new VPInstruction(Opcode, Operands, WrapFlags, {}, DL, Name));190 }191 192 VPInstruction *createNot(VPValue *Operand,193 DebugLoc DL = DebugLoc::getUnknown(),194 const Twine &Name = "") {195 return createInstruction(VPInstruction::Not, {Operand}, {}, DL, Name);196 }197 198 VPInstruction *createAnd(VPValue *LHS, VPValue *RHS,199 DebugLoc DL = DebugLoc::getUnknown(),200 const Twine &Name = "") {201 return createInstruction(Instruction::BinaryOps::And, {LHS, RHS}, {}, DL,202 Name);203 }204 205 VPInstruction *createOr(VPValue *LHS, VPValue *RHS,206 DebugLoc DL = DebugLoc::getUnknown(),207 const Twine &Name = "") {208 209 return tryInsertInstruction(new VPInstruction(210 Instruction::BinaryOps::Or, {LHS, RHS},211 VPRecipeWithIRFlags::DisjointFlagsTy(false), {}, DL, Name));212 }213 214 VPInstruction *createLogicalAnd(VPValue *LHS, VPValue *RHS,215 DebugLoc DL = DebugLoc::getUnknown(),216 const Twine &Name = "") {217 return createNaryOp(VPInstruction::LogicalAnd, {LHS, RHS}, DL, Name);218 }219 220 VPInstruction *221 createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal,222 DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "",223 std::optional<FastMathFlags> FMFs = std::nullopt) {224 if (!FMFs)225 return createNaryOp(Instruction::Select, {Cond, TrueVal, FalseVal}, DL,226 Name);227 return tryInsertInstruction(new VPInstruction(228 Instruction::Select, {Cond, TrueVal, FalseVal}, *FMFs, {}, DL, Name));229 }230 231 /// Create a new ICmp VPInstruction with predicate \p Pred and operands \p A232 /// and \p B.233 VPInstruction *createICmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B,234 DebugLoc DL = DebugLoc::getUnknown(),235 const Twine &Name = "") {236 assert(Pred >= CmpInst::FIRST_ICMP_PREDICATE &&237 Pred <= CmpInst::LAST_ICMP_PREDICATE && "invalid predicate");238 return tryInsertInstruction(239 new VPInstruction(Instruction::ICmp, {A, B}, Pred, {}, DL, Name));240 }241 242 /// Create a new FCmp VPInstruction with predicate \p Pred and operands \p A243 /// and \p B.244 VPInstruction *createFCmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B,245 DebugLoc DL = DebugLoc::getUnknown(),246 const Twine &Name = "") {247 assert(Pred >= CmpInst::FIRST_FCMP_PREDICATE &&248 Pred <= CmpInst::LAST_FCMP_PREDICATE && "invalid predicate");249 return tryInsertInstruction(250 new VPInstruction(Instruction::FCmp, {A, B}, Pred, {}, DL, Name));251 }252 253 VPInstruction *createPtrAdd(VPValue *Ptr, VPValue *Offset,254 DebugLoc DL = DebugLoc::getUnknown(),255 const Twine &Name = "") {256 return tryInsertInstruction(257 new VPInstruction(VPInstruction::PtrAdd, {Ptr, Offset},258 GEPNoWrapFlags::none(), {}, DL, Name));259 }260 261 VPInstruction *createNoWrapPtrAdd(VPValue *Ptr, VPValue *Offset,262 GEPNoWrapFlags GEPFlags,263 DebugLoc DL = DebugLoc::getUnknown(),264 const Twine &Name = "") {265 return tryInsertInstruction(new VPInstruction(266 VPInstruction::PtrAdd, {Ptr, Offset}, GEPFlags, {}, DL, Name));267 }268 269 VPInstruction *createWidePtrAdd(VPValue *Ptr, VPValue *Offset,270 DebugLoc DL = DebugLoc::getUnknown(),271 const Twine &Name = "") {272 return tryInsertInstruction(273 new VPInstruction(VPInstruction::WidePtrAdd, {Ptr, Offset},274 GEPNoWrapFlags::none(), {}, DL, Name));275 }276 277 VPPhi *createScalarPhi(ArrayRef<VPValue *> IncomingValues, DebugLoc DL,278 const Twine &Name = "") {279 return tryInsertInstruction(new VPPhi(IncomingValues, DL, Name));280 }281 282 VPValue *createElementCount(Type *Ty, ElementCount EC) {283 VPlan &Plan = *getInsertBlock()->getPlan();284 VPValue *RuntimeEC = Plan.getConstantInt(Ty, EC.getKnownMinValue());285 if (EC.isScalable()) {286 VPValue *VScale = createNaryOp(VPInstruction::VScale, {}, Ty);287 RuntimeEC = EC.getKnownMinValue() == 1288 ? VScale289 : createOverflowingOp(Instruction::Mul,290 {VScale, RuntimeEC}, {true, false});291 }292 return RuntimeEC;293 }294 295 /// Convert the input value \p Current to the corresponding value of an296 /// induction with \p Start and \p Step values, using \p Start + \p Current *297 /// \p Step.298 VPDerivedIVRecipe *createDerivedIV(InductionDescriptor::InductionKind Kind,299 FPMathOperator *FPBinOp, VPValue *Start,300 VPValue *Current, VPValue *Step,301 const Twine &Name = "") {302 return tryInsertInstruction(303 new VPDerivedIVRecipe(Kind, FPBinOp, Start, Current, Step, Name));304 }305 306 VPInstruction *createScalarCast(Instruction::CastOps Opcode, VPValue *Op,307 Type *ResultTy, DebugLoc DL,308 const VPIRFlags &Flags = {},309 const VPIRMetadata &Metadata = {}) {310 return tryInsertInstruction(311 new VPInstructionWithType(Opcode, Op, ResultTy, Flags, Metadata, DL));312 }313 314 VPValue *createScalarZExtOrTrunc(VPValue *Op, Type *ResultTy, Type *SrcTy,315 DebugLoc DL) {316 if (ResultTy == SrcTy)317 return Op;318 Instruction::CastOps CastOp =319 ResultTy->getScalarSizeInBits() < SrcTy->getScalarSizeInBits()320 ? Instruction::Trunc321 : Instruction::ZExt;322 return createScalarCast(CastOp, Op, ResultTy, DL);323 }324 325 VPWidenCastRecipe *createWidenCast(Instruction::CastOps Opcode, VPValue *Op,326 Type *ResultTy) {327 VPIRFlags Flags;328 if (Opcode == Instruction::Trunc)329 Flags = VPIRFlags::TruncFlagsTy(false, false);330 else if (Opcode == Instruction::ZExt)331 Flags = VPIRFlags::NonNegFlagsTy(false);332 return tryInsertInstruction(333 new VPWidenCastRecipe(Opcode, Op, ResultTy, nullptr, Flags));334 }335 336 VPScalarIVStepsRecipe *337 createScalarIVSteps(Instruction::BinaryOps InductionOpcode,338 FPMathOperator *FPBinOp, VPValue *IV, VPValue *Step,339 VPValue *VF, DebugLoc DL) {340 return tryInsertInstruction(new VPScalarIVStepsRecipe(341 IV, Step, VF, InductionOpcode,342 FPBinOp ? FPBinOp->getFastMathFlags() : FastMathFlags(), DL));343 }344 345 VPExpandSCEVRecipe *createExpandSCEV(const SCEV *Expr) {346 return tryInsertInstruction(new VPExpandSCEVRecipe(Expr));347 }348 349 //===--------------------------------------------------------------------===//350 // RAII helpers.351 //===--------------------------------------------------------------------===//352 353 /// RAII object that stores the current insertion point and restores it when354 /// the object is destroyed.355 class InsertPointGuard {356 VPBuilder &Builder;357 VPBasicBlock *Block;358 VPBasicBlock::iterator Point;359 360 public:361 InsertPointGuard(VPBuilder &B)362 : Builder(B), Block(B.getInsertBlock()), Point(B.getInsertPoint()) {}363 364 InsertPointGuard(const InsertPointGuard &) = delete;365 InsertPointGuard &operator=(const InsertPointGuard &) = delete;366 367 ~InsertPointGuard() { Builder.restoreIP(VPInsertPoint(Block, Point)); }368 };369};370 371/// TODO: The following VectorizationFactor was pulled out of372/// LoopVectorizationCostModel class. LV also deals with373/// VectorizerParams::VectorizationFactor.374/// We need to streamline them.375 376/// Information about vectorization costs.377struct VectorizationFactor {378 /// Vector width with best cost.379 ElementCount Width;380 381 /// Cost of the loop with that width.382 InstructionCost Cost;383 384 /// Cost of the scalar loop.385 InstructionCost ScalarCost;386 387 /// The minimum trip count required to make vectorization profitable, e.g. due388 /// to runtime checks.389 ElementCount MinProfitableTripCount;390 391 VectorizationFactor(ElementCount Width, InstructionCost Cost,392 InstructionCost ScalarCost)393 : Width(Width), Cost(Cost), ScalarCost(ScalarCost) {}394 395 /// Width 1 means no vectorization, cost 0 means uncomputed cost.396 static VectorizationFactor Disabled() {397 return {ElementCount::getFixed(1), 0, 0};398 }399 400 bool operator==(const VectorizationFactor &rhs) const {401 return Width == rhs.Width && Cost == rhs.Cost;402 }403 404 bool operator!=(const VectorizationFactor &rhs) const {405 return !(*this == rhs);406 }407};408 409/// A class that represents two vectorization factors (initialized with 0 by410/// default). One for fixed-width vectorization and one for scalable411/// vectorization. This can be used by the vectorizer to choose from a range of412/// fixed and/or scalable VFs in order to find the most cost-effective VF to413/// vectorize with.414struct FixedScalableVFPair {415 ElementCount FixedVF;416 ElementCount ScalableVF;417 418 FixedScalableVFPair()419 : FixedVF(ElementCount::getFixed(0)),420 ScalableVF(ElementCount::getScalable(0)) {}421 FixedScalableVFPair(const ElementCount &Max) : FixedScalableVFPair() {422 *(Max.isScalable() ? &ScalableVF : &FixedVF) = Max;423 }424 FixedScalableVFPair(const ElementCount &FixedVF,425 const ElementCount &ScalableVF)426 : FixedVF(FixedVF), ScalableVF(ScalableVF) {427 assert(!FixedVF.isScalable() && ScalableVF.isScalable() &&428 "Invalid scalable properties");429 }430 431 static FixedScalableVFPair getNone() { return FixedScalableVFPair(); }432 433 /// \return true if either fixed- or scalable VF is non-zero.434 explicit operator bool() const { return FixedVF || ScalableVF; }435 436 /// \return true if either fixed- or scalable VF is a valid vector VF.437 bool hasVector() const { return FixedVF.isVector() || ScalableVF.isVector(); }438};439 440/// Planner drives the vectorization process after having passed441/// Legality checks.442class LoopVectorizationPlanner {443 /// The loop that we evaluate.444 Loop *OrigLoop;445 446 /// Loop Info analysis.447 LoopInfo *LI;448 449 /// The dominator tree.450 DominatorTree *DT;451 452 /// Target Library Info.453 const TargetLibraryInfo *TLI;454 455 /// Target Transform Info.456 const TargetTransformInfo &TTI;457 458 /// The legality analysis.459 LoopVectorizationLegality *Legal;460 461 /// The profitability analysis.462 LoopVectorizationCostModel &CM;463 464 /// The interleaved access analysis.465 InterleavedAccessInfo &IAI;466 467 PredicatedScalarEvolution &PSE;468 469 const LoopVectorizeHints &Hints;470 471 OptimizationRemarkEmitter *ORE;472 473 SmallVector<VPlanPtr, 4> VPlans;474 475 /// Profitable vector factors.476 SmallVector<VectorizationFactor, 8> ProfitableVFs;477 478 /// A builder used to construct the current plan.479 VPBuilder Builder;480 481 /// Computes the cost of \p Plan for vectorization factor \p VF.482 ///483 /// The current implementation requires access to the484 /// LoopVectorizationLegality to handle inductions and reductions, which is485 /// why it is kept separate from the VPlan-only cost infrastructure.486 ///487 /// TODO: Move to VPlan::cost once the use of LoopVectorizationLegality has488 /// been retired.489 InstructionCost cost(VPlan &Plan, ElementCount VF) const;490 491 /// Precompute costs for certain instructions using the legacy cost model. The492 /// function is used to bring up the VPlan-based cost model to initially avoid493 /// taking different decisions due to inaccuracies in the legacy cost model.494 InstructionCost precomputeCosts(VPlan &Plan, ElementCount VF,495 VPCostContext &CostCtx) const;496 497public:498 LoopVectorizationPlanner(499 Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,500 const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,501 LoopVectorizationCostModel &CM, InterleavedAccessInfo &IAI,502 PredicatedScalarEvolution &PSE, const LoopVectorizeHints &Hints,503 OptimizationRemarkEmitter *ORE)504 : OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI), Legal(Legal), CM(CM),505 IAI(IAI), PSE(PSE), Hints(Hints), ORE(ORE) {}506 507 /// Build VPlans for the specified \p UserVF and \p UserIC if they are508 /// non-zero or all applicable candidate VFs otherwise. If vectorization and509 /// interleaving should be avoided up-front, no plans are generated.510 void plan(ElementCount UserVF, unsigned UserIC);511 512 /// Use the VPlan-native path to plan how to best vectorize, return the best513 /// VF and its cost.514 VectorizationFactor planInVPlanNativePath(ElementCount UserVF);515 516 /// Return the VPlan for \p VF. At the moment, there is always a single VPlan517 /// for each VF.518 VPlan &getPlanFor(ElementCount VF) const;519 520 /// Compute and return the most profitable vectorization factor. Also collect521 /// all profitable VFs in ProfitableVFs.522 VectorizationFactor computeBestVF();523 524 /// \return The desired interleave count.525 /// If interleave count has been specified by metadata it will be returned.526 /// Otherwise, the interleave count is computed and returned. VF and LoopCost527 /// are the selected vectorization factor and the cost of the selected VF.528 unsigned selectInterleaveCount(VPlan &Plan, ElementCount VF,529 InstructionCost LoopCost);530 531 /// Generate the IR code for the vectorized loop captured in VPlan \p BestPlan532 /// according to the best selected \p VF and \p UF.533 ///534 /// TODO: \p VectorizingEpilogue indicates if the executed VPlan is for the535 /// epilogue vector loop. It should be removed once the re-use issue has been536 /// fixed.537 ///538 /// Returns a mapping of SCEVs to their expanded IR values.539 /// Note that this is a temporary workaround needed due to the current540 /// epilogue handling.541 DenseMap<const SCEV *, Value *> executePlan(ElementCount VF, unsigned UF,542 VPlan &BestPlan,543 InnerLoopVectorizer &LB,544 DominatorTree *DT,545 bool VectorizingEpilogue);546 547#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)548 void printPlans(raw_ostream &O);549#endif550 551 /// Look through the existing plans and return true if we have one with552 /// vectorization factor \p VF.553 bool hasPlanWithVF(ElementCount VF) const {554 return any_of(VPlans,555 [&](const VPlanPtr &Plan) { return Plan->hasVF(VF); });556 }557 558 /// Test a \p Predicate on a \p Range of VF's. Return the value of applying559 /// \p Predicate on Range.Start, possibly decreasing Range.End such that the560 /// returned value holds for the entire \p Range.561 static bool562 getDecisionAndClampRange(const std::function<bool(ElementCount)> &Predicate,563 VFRange &Range);564 565 /// \return The most profitable vectorization factor and the cost of that VF566 /// for vectorizing the epilogue. Returns VectorizationFactor::Disabled if567 /// epilogue vectorization is not supported for the loop.568 VectorizationFactor569 selectEpilogueVectorizationFactor(const ElementCount MaxVF, unsigned IC);570 571 /// Emit remarks for recipes with invalid costs in the available VPlans.572 void emitInvalidCostRemarks(OptimizationRemarkEmitter *ORE);573 574 /// Create a check to \p Plan to see if the vector loop should be executed575 /// based on its trip count.576 void addMinimumIterationCheck(VPlan &Plan, ElementCount VF, unsigned UF,577 ElementCount MinProfitableTripCount) const;578 579 /// Update loop metadata and profile info for both the scalar remainder loop580 /// and \p VectorLoop, if it exists. Keeps all loop hints from the original581 /// loop on the vector loop and replaces vectorizer-specific metadata. The582 /// loop ID of the original loop \p OrigLoopID must be passed, together with583 /// the average trip count and invocation weight of the original loop (\p584 /// OrigAverageTripCount and \p OrigLoopInvocationWeight respectively). They585 /// cannot be retrieved after the plan has been executed, as the original loop586 /// may have been removed.587 void updateLoopMetadataAndProfileInfo(588 Loop *VectorLoop, VPBasicBlock *HeaderVPBB, const VPlan &Plan,589 bool VectorizingEpilogue, MDNode *OrigLoopID,590 std::optional<unsigned> OrigAverageTripCount,591 unsigned OrigLoopInvocationWeight, unsigned EstimatedVFxUF,592 bool DisableRuntimeUnroll);593 594protected:595 /// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,596 /// according to the information gathered by Legal when it checked if it is597 /// legal to vectorize the loop.598 void buildVPlans(ElementCount MinVF, ElementCount MaxVF);599 600private:601 /// Build a VPlan according to the information gathered by Legal. \return a602 /// VPlan for vectorization factors \p Range.Start and up to \p Range.End603 /// exclusive, possibly decreasing \p Range.End. If no VPlan can be built for604 /// the input range, set the largest included VF to the maximum VF for which605 /// no plan could be built.606 VPlanPtr tryToBuildVPlan(VFRange &Range);607 608 /// Build a VPlan using VPRecipes according to the information gather by609 /// Legal. This method is only used for the legacy inner loop vectorizer.610 /// \p Range's largest included VF is restricted to the maximum VF the611 /// returned VPlan is valid for. If no VPlan can be built for the input range,612 /// set the largest included VF to the maximum VF for which no plan could be613 /// built. Each VPlan is built starting from a copy of \p InitialPlan, which614 /// is a plain CFG VPlan wrapping the original scalar loop.615 VPlanPtr tryToBuildVPlanWithVPRecipes(VPlanPtr InitialPlan, VFRange &Range,616 LoopVersioning *LVer);617 618 /// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,619 /// according to the information gathered by Legal when it checked if it is620 /// legal to vectorize the loop. This method creates VPlans using VPRecipes.621 void buildVPlansWithVPRecipes(ElementCount MinVF, ElementCount MaxVF);622 623 // Adjust the recipes for reductions. For in-loop reductions the chain of624 // instructions leading from the loop exit instr to the phi need to be625 // converted to reductions, with one operand being vector and the other being626 // the scalar reduction chain. For other reductions, a select is introduced627 // between the phi and users outside the vector region when folding the tail.628 void adjustRecipesForReductions(VPlanPtr &Plan,629 VPRecipeBuilder &RecipeBuilder,630 ElementCount MinVF);631 632 /// Attach the runtime checks of \p RTChecks to \p Plan.633 void attachRuntimeChecks(VPlan &Plan, GeneratedRTChecks &RTChecks,634 bool HasBranchWeights) const;635 636#ifndef NDEBUG637 /// \return The most profitable vectorization factor for the available VPlans638 /// and the cost of that VF.639 /// This is now only used to verify the decisions by the new VPlan-based640 /// cost-model and will be retired once the VPlan-based cost-model is641 /// stabilized.642 VectorizationFactor selectVectorizationFactor();643#endif644 645 /// Returns true if the per-lane cost of VectorizationFactor A is lower than646 /// that of B.647 bool isMoreProfitable(const VectorizationFactor &A,648 const VectorizationFactor &B, bool HasTail,649 bool IsEpilogue = false) const;650 651 /// Returns true if the per-lane cost of VectorizationFactor A is lower than652 /// that of B in the context of vectorizing a loop with known \p MaxTripCount.653 bool isMoreProfitable(const VectorizationFactor &A,654 const VectorizationFactor &B,655 const unsigned MaxTripCount, bool HasTail,656 bool IsEpilogue = false) const;657 658 /// Determines if we have the infrastructure to vectorize the loop and its659 /// epilogue, assuming the main loop is vectorized by \p VF.660 bool isCandidateForEpilogueVectorization(const ElementCount VF) const;661};662 663} // namespace llvm664 665#endif // LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONPLANNER_H666