611 lines · cpp
1//===-- VPlanUnroll.cpp - VPlan unroller ----------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8///9/// \file10/// This file implements explicit unrolling for VPlans.11///12//===----------------------------------------------------------------------===//13 14#include "VPRecipeBuilder.h"15#include "VPlan.h"16#include "VPlanAnalysis.h"17#include "VPlanCFG.h"18#include "VPlanHelpers.h"19#include "VPlanPatternMatch.h"20#include "VPlanTransforms.h"21#include "VPlanUtils.h"22#include "llvm/ADT/PostOrderIterator.h"23#include "llvm/ADT/STLExtras.h"24#include "llvm/ADT/ScopeExit.h"25#include "llvm/Analysis/IVDescriptors.h"26#include "llvm/IR/Intrinsics.h"27 28using namespace llvm;29using namespace llvm::VPlanPatternMatch;30 31namespace {32 33/// Helper to hold state needed for unrolling. It holds the Plan to unroll by34/// UF. It also holds copies of VPValues across UF-1 unroll parts to facilitate35/// the unrolling transformation, where the original VPValues are retained for36/// part zero.37class UnrollState {38 /// Plan to unroll.39 VPlan &Plan;40 /// Unroll factor to unroll by.41 const unsigned UF;42 /// Analysis for types.43 VPTypeAnalysis TypeInfo;44 45 /// Unrolling may create recipes that should not be unrolled themselves.46 /// Those are tracked in ToSkip.47 SmallPtrSet<VPRecipeBase *, 8> ToSkip;48 49 // Associate with each VPValue of part 0 its unrolled instances of parts 1,50 // ..., UF-1.51 DenseMap<VPValue *, SmallVector<VPValue *>> VPV2Parts;52 53 /// Unroll replicate region \p VPR by cloning the region UF - 1 times.54 void unrollReplicateRegionByUF(VPRegionBlock *VPR);55 56 /// Unroll recipe \p R by cloning it UF - 1 times, unless it is uniform across57 /// all parts.58 void unrollRecipeByUF(VPRecipeBase &R);59 60 /// Unroll header phi recipe \p R. How exactly the recipe gets unrolled61 /// depends on the concrete header phi. Inserts newly created recipes at \p62 /// InsertPtForPhi.63 void unrollHeaderPHIByUF(VPHeaderPHIRecipe *R,64 VPBasicBlock::iterator InsertPtForPhi);65 66 /// Unroll a widen induction recipe \p IV. This introduces recipes to compute67 /// the induction steps for each part.68 void unrollWidenInductionByUF(VPWidenInductionRecipe *IV,69 VPBasicBlock::iterator InsertPtForPhi);70 71 VPValue *getConstantInt(unsigned Part) {72 Type *CanIVIntTy = Plan.getVectorLoopRegion()->getCanonicalIVType();73 return Plan.getConstantInt(CanIVIntTy, Part);74 }75 76public:77 UnrollState(VPlan &Plan, unsigned UF) : Plan(Plan), UF(UF), TypeInfo(Plan) {}78 79 void unrollBlock(VPBlockBase *VPB);80 81 VPValue *getValueForPart(VPValue *V, unsigned Part) {82 if (Part == 0 || V->isLiveIn())83 return V;84 assert((VPV2Parts.contains(V) && VPV2Parts[V].size() >= Part) &&85 "accessed value does not exist");86 return VPV2Parts[V][Part - 1];87 }88 89 /// Given a single original recipe \p OrigR (of part zero), and its copy \p90 /// CopyR for part \p Part, map every VPValue defined by \p OrigR to its91 /// corresponding VPValue defined by \p CopyR.92 void addRecipeForPart(VPRecipeBase *OrigR, VPRecipeBase *CopyR,93 unsigned Part) {94 for (const auto &[Idx, VPV] : enumerate(OrigR->definedValues())) {95 const auto &[V, _] = VPV2Parts.try_emplace(VPV);96 assert(V->second.size() == Part - 1 && "earlier parts not set");97 V->second.push_back(CopyR->getVPValue(Idx));98 }99 }100 101 /// Given a uniform recipe \p R, add it for all parts.102 void addUniformForAllParts(VPSingleDefRecipe *R) {103 const auto &[V, Inserted] = VPV2Parts.try_emplace(R);104 assert(Inserted && "uniform value already added");105 for (unsigned Part = 0; Part != UF; ++Part)106 V->second.push_back(R);107 }108 109 bool contains(VPValue *VPV) const { return VPV2Parts.contains(VPV); }110 111 /// Update \p R's operand at \p OpIdx with its corresponding VPValue for part112 /// \p P.113 void remapOperand(VPRecipeBase *R, unsigned OpIdx, unsigned Part) {114 auto *Op = R->getOperand(OpIdx);115 R->setOperand(OpIdx, getValueForPart(Op, Part));116 }117 118 /// Update \p R's operands with their corresponding VPValues for part \p P.119 void remapOperands(VPRecipeBase *R, unsigned Part) {120 for (const auto &[OpIdx, Op] : enumerate(R->operands()))121 R->setOperand(OpIdx, getValueForPart(Op, Part));122 }123};124} // namespace125 126void UnrollState::unrollReplicateRegionByUF(VPRegionBlock *VPR) {127 VPBlockBase *InsertPt = VPR->getSingleSuccessor();128 for (unsigned Part = 1; Part != UF; ++Part) {129 auto *Copy = VPR->clone();130 VPBlockUtils::insertBlockBefore(Copy, InsertPt);131 132 auto PartI = vp_depth_first_shallow(Copy->getEntry());133 auto Part0 = vp_depth_first_shallow(VPR->getEntry());134 for (const auto &[PartIVPBB, Part0VPBB] :135 zip(VPBlockUtils::blocksOnly<VPBasicBlock>(PartI),136 VPBlockUtils::blocksOnly<VPBasicBlock>(Part0))) {137 for (const auto &[PartIR, Part0R] : zip(*PartIVPBB, *Part0VPBB)) {138 remapOperands(&PartIR, Part);139 if (auto *ScalarIVSteps = dyn_cast<VPScalarIVStepsRecipe>(&PartIR)) {140 ScalarIVSteps->addOperand(getConstantInt(Part));141 }142 143 addRecipeForPart(&Part0R, &PartIR, Part);144 }145 }146 }147}148 149void UnrollState::unrollWidenInductionByUF(150 VPWidenInductionRecipe *IV, VPBasicBlock::iterator InsertPtForPhi) {151 VPBasicBlock *PH = cast<VPBasicBlock>(152 IV->getParent()->getEnclosingLoopRegion()->getSinglePredecessor());153 Type *IVTy = TypeInfo.inferScalarType(IV);154 auto &ID = IV->getInductionDescriptor();155 VPIRFlags Flags;156 if (isa_and_present<FPMathOperator>(ID.getInductionBinOp()))157 Flags = ID.getInductionBinOp()->getFastMathFlags();158 159 VPValue *ScalarStep = IV->getStepValue();160 VPBuilder Builder(PH);161 Type *VectorStepTy =162 IVTy->isPointerTy() ? TypeInfo.inferScalarType(ScalarStep) : IVTy;163 VPInstruction *VectorStep = Builder.createNaryOp(164 VPInstruction::WideIVStep, {&Plan.getVF(), ScalarStep}, VectorStepTy,165 Flags, IV->getDebugLoc());166 167 ToSkip.insert(VectorStep);168 169 // Now create recipes to compute the induction steps for part 1 .. UF. Part 0170 // remains the header phi. Parts > 0 are computed by adding Step to the171 // previous part. The header phi recipe will get 2 new operands: the step172 // value for a single part and the last part, used to compute the backedge173 // value during VPWidenInductionRecipe::execute.174 // %Part.0 = VPWidenInductionRecipe %Start, %ScalarStep, %VectorStep, %Part.3175 // %Part.1 = %Part.0 + %VectorStep176 // %Part.2 = %Part.1 + %VectorStep177 // %Part.3 = %Part.2 + %VectorStep178 //179 // The newly added recipes are added to ToSkip to avoid interleaving them180 // again.181 VPValue *Prev = IV;182 Builder.setInsertPoint(IV->getParent(), InsertPtForPhi);183 unsigned AddOpc;184 if (IVTy->isPointerTy())185 AddOpc = VPInstruction::WidePtrAdd;186 else if (IVTy->isFloatingPointTy())187 AddOpc = ID.getInductionOpcode();188 else189 AddOpc = Instruction::Add;190 for (unsigned Part = 1; Part != UF; ++Part) {191 std::string Name =192 Part > 1 ? "step.add." + std::to_string(Part) : "step.add";193 194 VPInstruction *Add = Builder.createNaryOp(AddOpc,195 {196 Prev,197 VectorStep,198 },199 Flags, IV->getDebugLoc(), Name);200 ToSkip.insert(Add);201 addRecipeForPart(IV, Add, Part);202 Prev = Add;203 }204 IV->addOperand(VectorStep);205 IV->addOperand(Prev);206}207 208void UnrollState::unrollHeaderPHIByUF(VPHeaderPHIRecipe *R,209 VPBasicBlock::iterator InsertPtForPhi) {210 // First-order recurrences pass a single vector or scalar through their header211 // phis, irrespective of interleaving.212 if (isa<VPFirstOrderRecurrencePHIRecipe>(R))213 return;214 215 // Generate step vectors for each unrolled part.216 if (auto *IV = dyn_cast<VPWidenInductionRecipe>(R)) {217 unrollWidenInductionByUF(IV, InsertPtForPhi);218 return;219 }220 221 auto *RdxPhi = dyn_cast<VPReductionPHIRecipe>(R);222 if (RdxPhi && RdxPhi->isOrdered())223 return;224 225 auto InsertPt = std::next(R->getIterator());226 for (unsigned Part = 1; Part != UF; ++Part) {227 VPRecipeBase *Copy = R->clone();228 Copy->insertBefore(*R->getParent(), InsertPt);229 addRecipeForPart(R, Copy, Part);230 if (RdxPhi) {231 // If the start value is a ReductionStartVector, use the identity value232 // (second operand) for unrolled parts. If the scaling factor is > 1,233 // create a new ReductionStartVector with the scale factor and both234 // operands set to the identity value.235 if (auto *VPI = dyn_cast<VPInstruction>(RdxPhi->getStartValue())) {236 assert(VPI->getOpcode() == VPInstruction::ReductionStartVector &&237 "unexpected start VPInstruction");238 if (Part != 1)239 continue;240 VPValue *StartV;241 if (match(VPI->getOperand(2), m_One())) {242 StartV = VPI->getOperand(1);243 } else {244 auto *C = VPI->clone();245 C->setOperand(0, C->getOperand(1));246 C->insertAfter(VPI);247 StartV = C;248 }249 for (unsigned Part = 1; Part != UF; ++Part)250 VPV2Parts[VPI][Part - 1] = StartV;251 }252 Copy->addOperand(getConstantInt(Part));253 } else {254 assert(isa<VPActiveLaneMaskPHIRecipe>(R) &&255 "unexpected header phi recipe not needing unrolled part");256 }257 }258}259 260/// Handle non-header-phi recipes.261void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {262 if (match(&R, m_CombineOr(m_BranchOnCond(), m_BranchOnCount())))263 return;264 265 if (auto *VPI = dyn_cast<VPInstruction>(&R)) {266 if (vputils::onlyFirstPartUsed(VPI)) {267 addUniformForAllParts(VPI);268 return;269 }270 }271 if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {272 if (isa<StoreInst>(RepR->getUnderlyingValue()) &&273 RepR->getOperand(1)->isDefinedOutsideLoopRegions()) {274 // Stores to an invariant address only need to store the last part.275 remapOperands(&R, UF - 1);276 return;277 }278 if (match(RepR,279 m_Intrinsic<Intrinsic::experimental_noalias_scope_decl>())) {280 addUniformForAllParts(RepR);281 return;282 }283 }284 285 // Unroll non-uniform recipes.286 auto InsertPt = std::next(R.getIterator());287 VPBasicBlock &VPBB = *R.getParent();288 for (unsigned Part = 1; Part != UF; ++Part) {289 VPRecipeBase *Copy = R.clone();290 Copy->insertBefore(VPBB, InsertPt);291 addRecipeForPart(&R, Copy, Part);292 293 VPValue *Op;294 if (match(&R, m_VPInstruction<VPInstruction::FirstOrderRecurrenceSplice>(295 m_VPValue(), m_VPValue(Op)))) {296 Copy->setOperand(0, getValueForPart(Op, Part - 1));297 Copy->setOperand(1, getValueForPart(Op, Part));298 continue;299 }300 if (auto *Red = dyn_cast<VPReductionRecipe>(&R)) {301 auto *Phi = dyn_cast<VPReductionPHIRecipe>(R.getOperand(0));302 if (Phi && Phi->isOrdered()) {303 auto &Parts = VPV2Parts[Phi];304 if (Part == 1) {305 Parts.clear();306 Parts.push_back(Red);307 }308 Parts.push_back(Copy->getVPSingleValue());309 Phi->setOperand(1, Copy->getVPSingleValue());310 }311 }312 remapOperands(Copy, Part);313 314 // Add operand indicating the part to generate code for, to recipes still315 // requiring it.316 if (isa<VPScalarIVStepsRecipe, VPWidenCanonicalIVRecipe,317 VPVectorPointerRecipe, VPVectorEndPointerRecipe>(Copy) ||318 match(Copy,319 m_VPInstruction<VPInstruction::CanonicalIVIncrementForPart>()))320 Copy->addOperand(getConstantInt(Part));321 322 if (isa<VPVectorPointerRecipe, VPVectorEndPointerRecipe>(R))323 Copy->setOperand(0, R.getOperand(0));324 }325}326 327void UnrollState::unrollBlock(VPBlockBase *VPB) {328 auto *VPR = dyn_cast<VPRegionBlock>(VPB);329 if (VPR) {330 if (VPR->isReplicator())331 return unrollReplicateRegionByUF(VPR);332 333 // Traverse blocks in region in RPO to ensure defs are visited before uses334 // across blocks.335 ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>>336 RPOT(VPR->getEntry());337 for (VPBlockBase *VPB : RPOT)338 unrollBlock(VPB);339 return;340 }341 342 // VPB is a VPBasicBlock; unroll it, i.e., unroll its recipes.343 auto *VPBB = cast<VPBasicBlock>(VPB);344 auto InsertPtForPhi = VPBB->getFirstNonPhi();345 for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {346 if (ToSkip.contains(&R) || isa<VPIRInstruction>(&R))347 continue;348 349 // Add all VPValues for all parts to AnyOf, FirstActiveLaneMask and350 // Compute*Result which combine all parts to compute the final value.351 VPValue *Op1;352 if (match(&R, m_VPInstruction<VPInstruction::AnyOf>(m_VPValue(Op1))) ||353 match(&R, m_FirstActiveLane(m_VPValue(Op1))) ||354 match(&R, m_LastActiveLane(m_VPValue(Op1))) ||355 match(&R, m_VPInstruction<VPInstruction::ComputeAnyOfResult>(356 m_VPValue(), m_VPValue(), m_VPValue(Op1))) ||357 match(&R, m_VPInstruction<VPInstruction::ComputeReductionResult>(358 m_VPValue(), m_VPValue(Op1))) ||359 match(&R, m_VPInstruction<VPInstruction::ComputeFindIVResult>(360 m_VPValue(), m_VPValue(), m_VPValue(), m_VPValue(Op1)))) {361 addUniformForAllParts(cast<VPInstruction>(&R));362 for (unsigned Part = 1; Part != UF; ++Part)363 R.addOperand(getValueForPart(Op1, Part));364 continue;365 }366 VPValue *Op0;367 if (match(&R, m_ExtractLane(m_VPValue(Op0), m_VPValue(Op1)))) {368 addUniformForAllParts(cast<VPInstruction>(&R));369 for (unsigned Part = 1; Part != UF; ++Part)370 R.addOperand(getValueForPart(Op1, Part));371 continue;372 }373 if (match(&R, m_ExtractLastElement(m_VPValue(Op0))) ||374 match(&R, m_ExtractPenultimateElement(m_VPValue(Op0)))) {375 addUniformForAllParts(cast<VPSingleDefRecipe>(&R));376 if (isa<VPFirstOrderRecurrencePHIRecipe>(Op0)) {377 assert(match(&R, m_ExtractLastElement(m_VPValue())) &&378 "can only extract last element of FOR");379 continue;380 }381 382 if (Plan.hasScalarVFOnly()) {383 auto *I = cast<VPInstruction>(&R);384 // Extracting from end with VF = 1 implies retrieving the last or385 // penultimate scalar part (UF-1 or UF-2).386 unsigned Offset =387 I->getOpcode() == VPInstruction::ExtractLastElement ? 1 : 2;388 I->replaceAllUsesWith(getValueForPart(Op0, UF - Offset));389 R.eraseFromParent();390 } else {391 // Otherwise we extract from the last part.392 remapOperands(&R, UF - 1);393 }394 continue;395 }396 397 auto *SingleDef = dyn_cast<VPSingleDefRecipe>(&R);398 if (SingleDef && vputils::isUniformAcrossVFsAndUFs(SingleDef)) {399 addUniformForAllParts(SingleDef);400 continue;401 }402 403 if (auto *H = dyn_cast<VPHeaderPHIRecipe>(&R)) {404 unrollHeaderPHIByUF(H, InsertPtForPhi);405 continue;406 }407 408 unrollRecipeByUF(R);409 }410}411 412void VPlanTransforms::unrollByUF(VPlan &Plan, unsigned UF) {413 assert(UF > 0 && "Unroll factor must be positive");414 Plan.setUF(UF);415 auto Cleanup = make_scope_exit([&Plan]() {416 auto Iter = vp_depth_first_deep(Plan.getEntry());417 // Remove recipes that are redundant after unrolling.418 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {419 for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {420 auto *VPI = dyn_cast<VPInstruction>(&R);421 if (VPI &&422 VPI->getOpcode() == VPInstruction::CanonicalIVIncrementForPart &&423 VPI->getNumOperands() == 1) {424 VPI->replaceAllUsesWith(VPI->getOperand(0));425 VPI->eraseFromParent();426 }427 }428 }429 });430 if (UF == 1) {431 return;432 }433 434 UnrollState Unroller(Plan, UF);435 436 // Iterate over all blocks in the plan starting from Entry, and unroll437 // recipes inside them. This includes the vector preheader and middle blocks,438 // which may set up or post-process per-part values.439 ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(440 Plan.getEntry());441 for (VPBlockBase *VPB : RPOT)442 Unroller.unrollBlock(VPB);443 444 unsigned Part = 1;445 // Remap operands of cloned header phis to update backedge values. The header446 // phis cloned during unrolling are just after the header phi for part 0.447 // Reset Part to 1 when reaching the first (part 0) recipe of a block.448 for (VPRecipeBase &H :449 Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) {450 // The second operand of Fixed Order Recurrence phi's, feeding the spliced451 // value across the backedge, needs to remap to the last part of the spliced452 // value.453 if (isa<VPFirstOrderRecurrencePHIRecipe>(&H)) {454 Unroller.remapOperand(&H, 1, UF - 1);455 continue;456 }457 if (Unroller.contains(H.getVPSingleValue())) {458 Part = 1;459 continue;460 }461 Unroller.remapOperands(&H, Part);462 Part++;463 }464 465 VPlanTransforms::removeDeadRecipes(Plan);466}467 468/// Create a single-scalar clone of \p DefR (must be a VPReplicateRecipe or469/// VPInstruction) for lane \p Lane. Use \p Def2LaneDefs to look up scalar470/// definitions for operands of \DefR.471static VPValue *472cloneForLane(VPlan &Plan, VPBuilder &Builder, Type *IdxTy,473 VPSingleDefRecipe *DefR, VPLane Lane,474 const DenseMap<VPValue *, SmallVector<VPValue *>> &Def2LaneDefs) {475 VPValue *Op;476 if (match(DefR, m_VPInstruction<VPInstruction::Unpack>(m_VPValue(Op)))) {477 auto LaneDefs = Def2LaneDefs.find(Op);478 if (LaneDefs != Def2LaneDefs.end())479 return LaneDefs->second[Lane.getKnownLane()];480 481 VPValue *Idx = Plan.getConstantInt(IdxTy, Lane.getKnownLane());482 return Builder.createNaryOp(Instruction::ExtractElement, {Op, Idx});483 }484 485 // Collect the operands at Lane, creating extracts as needed.486 SmallVector<VPValue *> NewOps;487 for (VPValue *Op : DefR->operands()) {488 // If Op is a definition that has been unrolled, directly use the clone for489 // the corresponding lane.490 auto LaneDefs = Def2LaneDefs.find(Op);491 if (LaneDefs != Def2LaneDefs.end()) {492 NewOps.push_back(LaneDefs->second[Lane.getKnownLane()]);493 continue;494 }495 if (Lane.getKind() == VPLane::Kind::ScalableLast) {496 // Look through mandatory Unpack.497 [[maybe_unused]] bool Matched =498 match(Op, m_VPInstruction<VPInstruction::Unpack>(m_VPValue(Op)));499 assert(Matched && "original op must have been Unpack");500 NewOps.push_back(501 Builder.createNaryOp(VPInstruction::ExtractLastElement, {Op}));502 continue;503 }504 if (vputils::isSingleScalar(Op)) {505 NewOps.push_back(Op);506 continue;507 }508 509 // Look through buildvector to avoid unnecessary extracts.510 if (match(Op, m_BuildVector())) {511 NewOps.push_back(512 cast<VPInstruction>(Op)->getOperand(Lane.getKnownLane()));513 continue;514 }515 VPValue *Idx = Plan.getConstantInt(IdxTy, Lane.getKnownLane());516 VPValue *Ext = Builder.createNaryOp(Instruction::ExtractElement, {Op, Idx});517 NewOps.push_back(Ext);518 }519 520 VPSingleDefRecipe *New;521 if (auto *RepR = dyn_cast<VPReplicateRecipe>(DefR)) {522 // TODO: have cloning of replicate recipes also provide the desired result523 // coupled with setting its operands to NewOps (deriving IsSingleScalar and524 // Mask from the operands?)525 New = new VPReplicateRecipe(RepR->getUnderlyingInstr(), NewOps,526 /*IsSingleScalar=*/true, /*Mask=*/nullptr,527 *RepR, *RepR, RepR->getDebugLoc());528 } else {529 assert(isa<VPInstruction>(DefR) &&530 "DefR must be a VPReplicateRecipe or VPInstruction");531 New = DefR->clone();532 for (const auto &[Idx, Op] : enumerate(NewOps)) {533 New->setOperand(Idx, Op);534 }535 }536 New->insertBefore(DefR);537 return New;538}539 540void VPlanTransforms::replicateByVF(VPlan &Plan, ElementCount VF) {541 Type *IdxTy = IntegerType::get(542 Plan.getScalarHeader()->getIRBasicBlock()->getContext(), 32);543 544 // Visit all VPBBs outside the loop region and directly inside the top-level545 // loop region.546 auto VPBBsOutsideLoopRegion = VPBlockUtils::blocksOnly<VPBasicBlock>(547 vp_depth_first_shallow(Plan.getEntry()));548 auto VPBBsInsideLoopRegion = VPBlockUtils::blocksOnly<VPBasicBlock>(549 vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()));550 auto VPBBsToUnroll =551 concat<VPBasicBlock *>(VPBBsOutsideLoopRegion, VPBBsInsideLoopRegion);552 // A mapping of current VPValue definitions to collections of new VPValues553 // defined per lane. Serves to hook-up potential users of current VPValue554 // definition that are replicated-per-VF later.555 DenseMap<VPValue *, SmallVector<VPValue *>> Def2LaneDefs;556 // The removal of current recipes being replaced by new ones needs to be557 // delayed after Def2LaneDefs is no longer in use.558 SmallVector<VPRecipeBase *> ToRemove;559 for (VPBasicBlock *VPBB : VPBBsToUnroll) {560 for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {561 if (!isa<VPInstruction, VPReplicateRecipe>(&R) ||562 (isa<VPReplicateRecipe>(&R) &&563 cast<VPReplicateRecipe>(&R)->isSingleScalar()) ||564 (isa<VPInstruction>(&R) &&565 !cast<VPInstruction>(&R)->doesGeneratePerAllLanes() &&566 cast<VPInstruction>(&R)->getOpcode() != VPInstruction::Unpack))567 continue;568 569 auto *DefR = cast<VPSingleDefRecipe>(&R);570 VPBuilder Builder(DefR);571 if (DefR->getNumUsers() == 0) {572 // Create single-scalar version of DefR for all lanes.573 for (unsigned I = 0; I != VF.getKnownMinValue(); ++I)574 cloneForLane(Plan, Builder, IdxTy, DefR, VPLane(I), Def2LaneDefs);575 DefR->eraseFromParent();576 continue;577 }578 /// Create single-scalar version of DefR for all lanes.579 SmallVector<VPValue *> LaneDefs;580 for (unsigned I = 0; I != VF.getKnownMinValue(); ++I)581 LaneDefs.push_back(582 cloneForLane(Plan, Builder, IdxTy, DefR, VPLane(I), Def2LaneDefs));583 584 Def2LaneDefs[DefR] = LaneDefs;585 /// Users that only demand the first lane can use the definition for lane586 /// 0.587 DefR->replaceUsesWithIf(LaneDefs[0], [DefR](VPUser &U, unsigned) {588 return U.usesFirstLaneOnly(DefR);589 });590 591 // Update each build vector user that currently has DefR as its only592 // operand, to have all LaneDefs as its operands.593 for (VPUser *U : to_vector(DefR->users())) {594 auto *VPI = dyn_cast<VPInstruction>(U);595 if (!VPI || (VPI->getOpcode() != VPInstruction::BuildVector &&596 VPI->getOpcode() != VPInstruction::BuildStructVector))597 continue;598 assert(VPI->getNumOperands() == 1 &&599 "Build(Struct)Vector must have a single operand before "600 "replicating by VF");601 VPI->setOperand(0, LaneDefs[0]);602 for (VPValue *LaneDef : drop_begin(LaneDefs))603 VPI->addOperand(LaneDef);604 }605 ToRemove.push_back(DefR);606 }607 }608 for (auto *R : reverse(ToRemove))609 R->eraseFromParent();610}611