920 lines · c
1//===- VPlanPatternMatch.h - Match on VPValues and recipes ------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file provides a simple and efficient mechanism for performing general10// tree-based pattern matches on the VPlan values and recipes, based on11// LLVM's IR pattern matchers.12//13//===----------------------------------------------------------------------===//14 15#ifndef LLVM_TRANSFORM_VECTORIZE_VPLANPATTERNMATCH_H16#define LLVM_TRANSFORM_VECTORIZE_VPLANPATTERNMATCH_H17 18#include "VPlan.h"19 20namespace llvm::VPlanPatternMatch {21 22template <typename Val, typename Pattern> bool match(Val *V, const Pattern &P) {23 return P.match(V);24}25 26template <typename Pattern> bool match(VPUser *U, const Pattern &P) {27 auto *R = dyn_cast<VPRecipeBase>(U);28 return R && match(R, P);29}30 31template <typename Pattern> bool match(VPSingleDefRecipe *R, const Pattern &P) {32 return P.match(static_cast<const VPRecipeBase *>(R));33}34 35template <typename Val, typename Pattern> struct VPMatchFunctor {36 const Pattern &P;37 VPMatchFunctor(const Pattern &P) : P(P) {}38 bool operator()(Val *V) const { return match(V, P); }39};40 41/// A match functor that can be used as a UnaryPredicate in functional42/// algorithms like all_of.43template <typename Val = VPUser, typename Pattern>44VPMatchFunctor<Val, Pattern> match_fn(const Pattern &P) {45 return P;46}47 48template <typename Class> struct class_match {49 template <typename ITy> bool match(ITy *V) const { return isa<Class>(V); }50};51 52/// Match an arbitrary VPValue and ignore it.53inline class_match<VPValue> m_VPValue() { return class_match<VPValue>(); }54 55template <typename Class> struct bind_ty {56 Class *&VR;57 58 bind_ty(Class *&V) : VR(V) {}59 60 template <typename ITy> bool match(ITy *V) const {61 if (auto *CV = dyn_cast<Class>(V)) {62 VR = CV;63 return true;64 }65 return false;66 }67};68 69/// Match a specified VPValue.70struct specificval_ty {71 const VPValue *Val;72 73 specificval_ty(const VPValue *V) : Val(V) {}74 75 bool match(VPValue *VPV) const { return VPV == Val; }76};77 78inline specificval_ty m_Specific(const VPValue *VPV) { return VPV; }79 80/// Stores a reference to the VPValue *, not the VPValue * itself,81/// thus can be used in commutative matchers.82struct deferredval_ty {83 VPValue *const &Val;84 85 deferredval_ty(VPValue *const &V) : Val(V) {}86 87 bool match(VPValue *const V) const { return V == Val; }88};89 90/// Like m_Specific(), but works if the specific value to match is determined91/// as part of the same match() expression. For example:92/// m_Mul(m_VPValue(X), m_Specific(X)) is incorrect, because m_Specific() will93/// bind X before the pattern match starts.94/// m_Mul(m_VPValue(X), m_Deferred(X)) is correct, and will check against95/// whichever value m_VPValue(X) populated.96inline deferredval_ty m_Deferred(VPValue *const &V) { return V; }97 98/// Match an integer constant or vector of constants if Pred::isValue returns99/// true for the APInt. \p BitWidth optionally specifies the bitwidth the100/// matched constant must have. If it is 0, the matched constant can have any101/// bitwidth.102template <typename Pred, unsigned BitWidth = 0> struct int_pred_ty {103 Pred P;104 105 int_pred_ty(Pred P) : P(std::move(P)) {}106 int_pred_ty() : P() {}107 108 bool match(VPValue *VPV) const {109 if (!VPV->isLiveIn())110 return false;111 Value *V = VPV->getLiveInIRValue();112 if (!V)113 return false;114 assert(!V->getType()->isVectorTy() && "Unexpected vector live-in");115 const auto *CI = dyn_cast<ConstantInt>(V);116 if (!CI)117 return false;118 119 if (BitWidth != 0 && CI->getBitWidth() != BitWidth)120 return false;121 return P.isValue(CI->getValue());122 }123};124 125/// Match a specified integer value or vector of all elements of that126/// value. \p BitWidth optionally specifies the bitwidth the matched constant127/// must have. If it is 0, the matched constant can have any bitwidth.128struct is_specific_int {129 APInt Val;130 131 is_specific_int(APInt Val) : Val(std::move(Val)) {}132 133 bool isValue(const APInt &C) const { return APInt::isSameValue(Val, C); }134};135 136template <unsigned Bitwidth = 0>137using specific_intval = int_pred_ty<is_specific_int, Bitwidth>;138 139inline specific_intval<0> m_SpecificInt(uint64_t V) {140 return specific_intval<0>(is_specific_int(APInt(64, V)));141}142 143inline specific_intval<1> m_False() {144 return specific_intval<1>(is_specific_int(APInt(64, 0)));145}146 147inline specific_intval<1> m_True() {148 return specific_intval<1>(is_specific_int(APInt(64, 1)));149}150 151struct is_all_ones {152 bool isValue(const APInt &C) const { return C.isAllOnes(); }153};154 155/// Match an integer or vector with all bits set.156/// For vectors, this includes constants with undefined elements.157inline int_pred_ty<is_all_ones> m_AllOnes() {158 return int_pred_ty<is_all_ones>();159}160 161struct is_zero_int {162 bool isValue(const APInt &C) const { return C.isZero(); }163};164 165struct is_one {166 bool isValue(const APInt &C) const { return C.isOne(); }167};168 169/// Match an integer 0 or a vector with all elements equal to 0.170/// For vectors, this includes constants with undefined elements.171inline int_pred_ty<is_zero_int> m_ZeroInt() {172 return int_pred_ty<is_zero_int>();173}174 175/// Match an integer 1 or a vector with all elements equal to 1.176/// For vectors, this includes constants with undefined elements.177inline int_pred_ty<is_one> m_One() { return int_pred_ty<is_one>(); }178 179struct bind_apint {180 const APInt *&Res;181 182 bind_apint(const APInt *&Res) : Res(Res) {}183 184 bool match(VPValue *VPV) const {185 if (!VPV->isLiveIn())186 return false;187 Value *V = VPV->getLiveInIRValue();188 if (!V)189 return false;190 assert(!V->getType()->isVectorTy() && "Unexpected vector live-in");191 const auto *CI = dyn_cast<ConstantInt>(V);192 if (!CI)193 return false;194 Res = &CI->getValue();195 return true;196 }197};198 199inline bind_apint m_APInt(const APInt *&C) { return C; }200 201struct bind_const_int {202 uint64_t &Res;203 204 bind_const_int(uint64_t &Res) : Res(Res) {}205 206 bool match(VPValue *VPV) const {207 const APInt *APConst;208 if (!bind_apint(APConst).match(VPV))209 return false;210 if (auto C = APConst->tryZExtValue()) {211 Res = *C;212 return true;213 }214 return false;215 }216};217 218/// Match a plain integer constant no wider than 64-bits, capturing it if we219/// match.220inline bind_const_int m_ConstantInt(uint64_t &C) { return C; }221 222/// Matching combinators223template <typename LTy, typename RTy> struct match_combine_or {224 LTy L;225 RTy R;226 227 match_combine_or(const LTy &Left, const RTy &Right) : L(Left), R(Right) {}228 229 template <typename ITy> bool match(ITy *V) const {230 return L.match(V) || R.match(V);231 }232};233 234template <typename LTy, typename RTy> struct match_combine_and {235 LTy L;236 RTy R;237 238 match_combine_and(const LTy &Left, const RTy &Right) : L(Left), R(Right) {}239 240 template <typename ITy> bool match(ITy *V) const {241 return L.match(V) && R.match(V);242 }243};244 245/// Combine two pattern matchers matching L || R246template <typename LTy, typename RTy>247inline match_combine_or<LTy, RTy> m_CombineOr(const LTy &L, const RTy &R) {248 return match_combine_or<LTy, RTy>(L, R);249}250 251/// Combine two pattern matchers matching L && R252template <typename LTy, typename RTy>253inline match_combine_and<LTy, RTy> m_CombineAnd(const LTy &L, const RTy &R) {254 return match_combine_and<LTy, RTy>(L, R);255}256 257/// Match a VPValue, capturing it if we match.258inline bind_ty<VPValue> m_VPValue(VPValue *&V) { return V; }259 260/// Match a VPInstruction, capturing if we match.261inline bind_ty<VPInstruction> m_VPInstruction(VPInstruction *&V) { return V; }262 263template <typename Ops_t, unsigned Opcode, bool Commutative,264 typename... RecipeTys>265struct Recipe_match {266 Ops_t Ops;267 268 template <typename... OpTy> Recipe_match(OpTy... Ops) : Ops(Ops...) {269 static_assert(std::tuple_size<Ops_t>::value == sizeof...(Ops) &&270 "number of operands in constructor doesn't match Ops_t");271 static_assert((!Commutative || std::tuple_size<Ops_t>::value == 2) &&272 "only binary ops can be commutative");273 }274 275 bool match(const VPValue *V) const {276 auto *DefR = V->getDefiningRecipe();277 return DefR && match(DefR);278 }279 280 bool match(const VPSingleDefRecipe *R) const {281 return match(static_cast<const VPRecipeBase *>(R));282 }283 284 bool match(const VPRecipeBase *R) const {285 if (std::tuple_size_v<Ops_t> == 0) {286 auto *VPI = dyn_cast<VPInstruction>(R);287 return VPI && VPI->getOpcode() == Opcode;288 }289 290 if ((!matchRecipeAndOpcode<RecipeTys>(R) && ...))291 return false;292 293 if (R->getNumOperands() != std::tuple_size<Ops_t>::value) {294 assert(Opcode == Instruction::PHI &&295 "non-variadic recipe with matched opcode does not have the "296 "expected number of operands");297 return false;298 }299 300 auto IdxSeq = std::make_index_sequence<std::tuple_size<Ops_t>::value>();301 if (all_of_tuple_elements(IdxSeq, [R](auto Op, unsigned Idx) {302 return Op.match(R->getOperand(Idx));303 }))304 return true;305 306 return Commutative &&307 all_of_tuple_elements(IdxSeq, [R](auto Op, unsigned Idx) {308 return Op.match(R->getOperand(R->getNumOperands() - Idx - 1));309 });310 }311 312private:313 template <typename RecipeTy>314 static bool matchRecipeAndOpcode(const VPRecipeBase *R) {315 auto *DefR = dyn_cast<RecipeTy>(R);316 // Check for recipes that do not have opcodes.317 if constexpr (std::is_same_v<RecipeTy, VPScalarIVStepsRecipe> ||318 std::is_same_v<RecipeTy, VPCanonicalIVPHIRecipe> ||319 std::is_same_v<RecipeTy, VPDerivedIVRecipe> ||320 std::is_same_v<RecipeTy, VPVectorEndPointerRecipe>)321 return DefR;322 else323 return DefR && DefR->getOpcode() == Opcode;324 }325 326 /// Helper to check if predicate \p P holds on all tuple elements in Ops using327 /// the provided index sequence.328 template <typename Fn, std::size_t... Is>329 bool all_of_tuple_elements(std::index_sequence<Is...>, Fn P) const {330 return (P(std::get<Is>(Ops), Is) && ...);331 }332};333 334template <unsigned Opcode, typename... OpTys>335using AllRecipe_match =336 Recipe_match<std::tuple<OpTys...>, Opcode, /*Commutative*/ false,337 VPWidenRecipe, VPReplicateRecipe, VPWidenCastRecipe,338 VPInstruction, VPWidenSelectRecipe>;339 340template <unsigned Opcode, typename... OpTys>341using AllRecipe_commutative_match =342 Recipe_match<std::tuple<OpTys...>, Opcode, /*Commutative*/ true,343 VPWidenRecipe, VPReplicateRecipe, VPInstruction>;344 345template <unsigned Opcode, typename... OpTys>346using VPInstruction_match = Recipe_match<std::tuple<OpTys...>, Opcode,347 /*Commutative*/ false, VPInstruction>;348 349template <unsigned Opcode, typename... OpTys>350inline VPInstruction_match<Opcode, OpTys...>351m_VPInstruction(const OpTys &...Ops) {352 return VPInstruction_match<Opcode, OpTys...>(Ops...);353}354 355/// BuildVector is matches only its opcode, w/o matching its operands as the356/// number of operands is not fixed.357inline VPInstruction_match<VPInstruction::BuildVector> m_BuildVector() {358 return m_VPInstruction<VPInstruction::BuildVector>();359}360 361template <typename Op0_t>362inline VPInstruction_match<Instruction::Freeze, Op0_t>363m_Freeze(const Op0_t &Op0) {364 return m_VPInstruction<Instruction::Freeze>(Op0);365}366 367inline VPInstruction_match<VPInstruction::BranchOnCond> m_BranchOnCond() {368 return m_VPInstruction<VPInstruction::BranchOnCond>();369}370 371template <typename Op0_t>372inline VPInstruction_match<VPInstruction::BranchOnCond, Op0_t>373m_BranchOnCond(const Op0_t &Op0) {374 return m_VPInstruction<VPInstruction::BranchOnCond>(Op0);375}376 377template <typename Op0_t>378inline VPInstruction_match<VPInstruction::Broadcast, Op0_t>379m_Broadcast(const Op0_t &Op0) {380 return m_VPInstruction<VPInstruction::Broadcast>(Op0);381}382 383template <typename Op0_t>384inline VPInstruction_match<VPInstruction::ExplicitVectorLength, Op0_t>385m_EVL(const Op0_t &Op0) {386 return m_VPInstruction<VPInstruction::ExplicitVectorLength>(Op0);387}388 389template <typename Op0_t>390inline VPInstruction_match<VPInstruction::ExtractLastElement, Op0_t>391m_ExtractLastElement(const Op0_t &Op0) {392 return m_VPInstruction<VPInstruction::ExtractLastElement>(Op0);393}394 395template <typename Op0_t, typename Op1_t>396inline VPInstruction_match<Instruction::ExtractElement, Op0_t, Op1_t>397m_ExtractElement(const Op0_t &Op0, const Op1_t &Op1) {398 return m_VPInstruction<Instruction::ExtractElement>(Op0, Op1);399}400 401template <typename Op0_t, typename Op1_t>402inline VPInstruction_match<VPInstruction::ExtractLane, Op0_t, Op1_t>403m_ExtractLane(const Op0_t &Op0, const Op1_t &Op1) {404 return m_VPInstruction<VPInstruction::ExtractLane>(Op0, Op1);405}406 407template <typename Op0_t>408inline VPInstruction_match<VPInstruction::ExtractLastLanePerPart, Op0_t>409m_ExtractLastLanePerPart(const Op0_t &Op0) {410 return m_VPInstruction<VPInstruction::ExtractLastLanePerPart>(Op0);411}412 413template <typename Op0_t>414inline VPInstruction_match<VPInstruction::ExtractPenultimateElement, Op0_t>415m_ExtractPenultimateElement(const Op0_t &Op0) {416 return m_VPInstruction<VPInstruction::ExtractPenultimateElement>(Op0);417}418 419template <typename Op0_t, typename Op1_t, typename Op2_t>420inline VPInstruction_match<VPInstruction::ActiveLaneMask, Op0_t, Op1_t, Op2_t>421m_ActiveLaneMask(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {422 return m_VPInstruction<VPInstruction::ActiveLaneMask>(Op0, Op1, Op2);423}424 425inline VPInstruction_match<VPInstruction::BranchOnCount> m_BranchOnCount() {426 return m_VPInstruction<VPInstruction::BranchOnCount>();427}428 429template <typename Op0_t, typename Op1_t>430inline VPInstruction_match<VPInstruction::BranchOnCount, Op0_t, Op1_t>431m_BranchOnCount(const Op0_t &Op0, const Op1_t &Op1) {432 return m_VPInstruction<VPInstruction::BranchOnCount>(Op0, Op1);433}434 435inline VPInstruction_match<VPInstruction::AnyOf> m_AnyOf() {436 return m_VPInstruction<VPInstruction::AnyOf>();437}438 439template <typename Op0_t>440inline VPInstruction_match<VPInstruction::AnyOf, Op0_t>441m_AnyOf(const Op0_t &Op0) {442 return m_VPInstruction<VPInstruction::AnyOf>(Op0);443}444 445template <typename Op0_t>446inline VPInstruction_match<VPInstruction::FirstActiveLane, Op0_t>447m_FirstActiveLane(const Op0_t &Op0) {448 return m_VPInstruction<VPInstruction::FirstActiveLane>(Op0);449}450 451template <typename Op0_t>452inline VPInstruction_match<VPInstruction::LastActiveLane, Op0_t>453m_LastActiveLane(const Op0_t &Op0) {454 return m_VPInstruction<VPInstruction::LastActiveLane>(Op0);455}456 457inline VPInstruction_match<VPInstruction::StepVector> m_StepVector() {458 return m_VPInstruction<VPInstruction::StepVector>();459}460 461template <unsigned Opcode, typename Op0_t>462inline AllRecipe_match<Opcode, Op0_t> m_Unary(const Op0_t &Op0) {463 return AllRecipe_match<Opcode, Op0_t>(Op0);464}465 466template <typename Op0_t>467inline AllRecipe_match<Instruction::Trunc, Op0_t> m_Trunc(const Op0_t &Op0) {468 return m_Unary<Instruction::Trunc, Op0_t>(Op0);469}470 471template <typename Op0_t>472inline match_combine_or<AllRecipe_match<Instruction::Trunc, Op0_t>, Op0_t>473m_TruncOrSelf(const Op0_t &Op0) {474 return m_CombineOr(m_Trunc(Op0), Op0);475}476 477template <typename Op0_t>478inline AllRecipe_match<Instruction::ZExt, Op0_t> m_ZExt(const Op0_t &Op0) {479 return m_Unary<Instruction::ZExt, Op0_t>(Op0);480}481 482template <typename Op0_t>483inline AllRecipe_match<Instruction::SExt, Op0_t> m_SExt(const Op0_t &Op0) {484 return m_Unary<Instruction::SExt, Op0_t>(Op0);485}486 487template <typename Op0_t>488inline match_combine_or<AllRecipe_match<Instruction::ZExt, Op0_t>,489 AllRecipe_match<Instruction::SExt, Op0_t>>490m_ZExtOrSExt(const Op0_t &Op0) {491 return m_CombineOr(m_ZExt(Op0), m_SExt(Op0));492}493 494template <typename Op0_t>495inline match_combine_or<AllRecipe_match<Instruction::ZExt, Op0_t>, Op0_t>496m_ZExtOrSelf(const Op0_t &Op0) {497 return m_CombineOr(m_ZExt(Op0), Op0);498}499 500template <unsigned Opcode, typename Op0_t, typename Op1_t>501inline AllRecipe_match<Opcode, Op0_t, Op1_t> m_Binary(const Op0_t &Op0,502 const Op1_t &Op1) {503 return AllRecipe_match<Opcode, Op0_t, Op1_t>(Op0, Op1);504}505 506template <unsigned Opcode, typename Op0_t, typename Op1_t>507inline AllRecipe_commutative_match<Opcode, Op0_t, Op1_t>508m_c_Binary(const Op0_t &Op0, const Op1_t &Op1) {509 return AllRecipe_commutative_match<Opcode, Op0_t, Op1_t>(Op0, Op1);510}511 512template <typename Op0_t, typename Op1_t>513inline AllRecipe_match<Instruction::Add, Op0_t, Op1_t> m_Add(const Op0_t &Op0,514 const Op1_t &Op1) {515 return m_Binary<Instruction::Add, Op0_t, Op1_t>(Op0, Op1);516}517 518template <typename Op0_t, typename Op1_t>519inline AllRecipe_commutative_match<Instruction::Add, Op0_t, Op1_t>520m_c_Add(const Op0_t &Op0, const Op1_t &Op1) {521 return m_c_Binary<Instruction::Add, Op0_t, Op1_t>(Op0, Op1);522}523 524template <typename Op0_t, typename Op1_t>525inline AllRecipe_match<Instruction::Sub, Op0_t, Op1_t> m_Sub(const Op0_t &Op0,526 const Op1_t &Op1) {527 return m_Binary<Instruction::Sub, Op0_t, Op1_t>(Op0, Op1);528}529 530template <typename Op0_t, typename Op1_t>531inline AllRecipe_match<Instruction::Mul, Op0_t, Op1_t> m_Mul(const Op0_t &Op0,532 const Op1_t &Op1) {533 return m_Binary<Instruction::Mul, Op0_t, Op1_t>(Op0, Op1);534}535 536template <typename Op0_t, typename Op1_t>537inline AllRecipe_commutative_match<Instruction::Mul, Op0_t, Op1_t>538m_c_Mul(const Op0_t &Op0, const Op1_t &Op1) {539 return m_c_Binary<Instruction::Mul, Op0_t, Op1_t>(Op0, Op1);540}541 542/// Match a binary AND operation.543template <typename Op0_t, typename Op1_t>544inline AllRecipe_commutative_match<Instruction::And, Op0_t, Op1_t>545m_c_BinaryAnd(const Op0_t &Op0, const Op1_t &Op1) {546 return m_c_Binary<Instruction::And, Op0_t, Op1_t>(Op0, Op1);547}548 549/// Match a binary OR operation. Note that while conceptually the operands can550/// be matched commutatively, \p Commutative defaults to false in line with the551/// IR-based pattern matching infrastructure. Use m_c_BinaryOr for a commutative552/// version of the matcher.553template <typename Op0_t, typename Op1_t>554inline AllRecipe_match<Instruction::Or, Op0_t, Op1_t>555m_BinaryOr(const Op0_t &Op0, const Op1_t &Op1) {556 return m_Binary<Instruction::Or, Op0_t, Op1_t>(Op0, Op1);557}558 559template <typename Op0_t, typename Op1_t>560inline AllRecipe_commutative_match<Instruction::Or, Op0_t, Op1_t>561m_c_BinaryOr(const Op0_t &Op0, const Op1_t &Op1) {562 return m_c_Binary<Instruction::Or, Op0_t, Op1_t>(Op0, Op1);563}564 565/// Cmp_match is a variant of BinaryRecipe_match that also binds the comparison566/// predicate. Opcodes must either be Instruction::ICmp or Instruction::FCmp, or567/// both.568template <typename Op0_t, typename Op1_t, unsigned... Opcodes>569struct Cmp_match {570 static_assert((sizeof...(Opcodes) == 1 || sizeof...(Opcodes) == 2) &&571 "Expected one or two opcodes");572 static_assert(573 ((Opcodes == Instruction::ICmp || Opcodes == Instruction::FCmp) && ...) &&574 "Expected a compare instruction opcode");575 576 CmpPredicate *Predicate = nullptr;577 Op0_t Op0;578 Op1_t Op1;579 580 Cmp_match(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1)581 : Predicate(&Pred), Op0(Op0), Op1(Op1) {}582 Cmp_match(const Op0_t &Op0, const Op1_t &Op1) : Op0(Op0), Op1(Op1) {}583 584 bool match(const VPValue *V) const {585 auto *DefR = V->getDefiningRecipe();586 return DefR && match(DefR);587 }588 589 bool match(const VPRecipeBase *V) const {590 if ((m_Binary<Opcodes>(Op0, Op1).match(V) || ...)) {591 if (Predicate)592 *Predicate = cast<VPRecipeWithIRFlags>(V)->getPredicate();593 return true;594 }595 return false;596 }597};598 599/// SpecificCmp_match is a variant of Cmp_match that matches the comparison600/// predicate, instead of binding it.601template <typename Op0_t, typename Op1_t, unsigned... Opcodes>602struct SpecificCmp_match {603 const CmpPredicate Predicate;604 Op0_t Op0;605 Op1_t Op1;606 607 SpecificCmp_match(CmpPredicate Pred, const Op0_t &LHS, const Op1_t &RHS)608 : Predicate(Pred), Op0(LHS), Op1(RHS) {}609 610 bool match(const VPValue *V) const {611 CmpPredicate CurrentPred;612 return Cmp_match<Op0_t, Op1_t, Opcodes...>(CurrentPred, Op0, Op1)613 .match(V) &&614 CmpPredicate::getMatching(CurrentPred, Predicate);615 }616};617 618template <typename Op0_t, typename Op1_t>619inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp> m_ICmp(const Op0_t &Op0,620 const Op1_t &Op1) {621 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp>(Op0, Op1);622}623 624template <typename Op0_t, typename Op1_t>625inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp>626m_ICmp(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1) {627 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp>(Pred, Op0, Op1);628}629 630template <typename Op0_t, typename Op1_t>631inline SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp>632m_SpecificICmp(CmpPredicate MatchPred, const Op0_t &Op0, const Op1_t &Op1) {633 return SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp>(MatchPred, Op0,634 Op1);635}636 637template <typename Op0_t, typename Op1_t>638inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>639m_Cmp(const Op0_t &Op0, const Op1_t &Op1) {640 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(Op0,641 Op1);642}643 644template <typename Op0_t, typename Op1_t>645inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>646m_Cmp(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1) {647 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(648 Pred, Op0, Op1);649}650 651template <typename Op0_t, typename Op1_t>652inline SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>653m_SpecificCmp(CmpPredicate MatchPred, const Op0_t &Op0, const Op1_t &Op1) {654 return SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(655 MatchPred, Op0, Op1);656}657 658template <typename Op0_t, typename Op1_t>659using GEPLikeRecipe_match = match_combine_or<660 Recipe_match<std::tuple<Op0_t, Op1_t>, Instruction::GetElementPtr,661 /*Commutative*/ false, VPReplicateRecipe, VPWidenGEPRecipe>,662 match_combine_or<663 VPInstruction_match<VPInstruction::PtrAdd, Op0_t, Op1_t>,664 VPInstruction_match<VPInstruction::WidePtrAdd, Op0_t, Op1_t>>>;665 666template <typename Op0_t, typename Op1_t>667inline GEPLikeRecipe_match<Op0_t, Op1_t> m_GetElementPtr(const Op0_t &Op0,668 const Op1_t &Op1) {669 return m_CombineOr(670 Recipe_match<std::tuple<Op0_t, Op1_t>, Instruction::GetElementPtr,671 /*Commutative*/ false, VPReplicateRecipe, VPWidenGEPRecipe>(672 Op0, Op1),673 m_CombineOr(674 VPInstruction_match<VPInstruction::PtrAdd, Op0_t, Op1_t>(Op0, Op1),675 VPInstruction_match<VPInstruction::WidePtrAdd, Op0_t, Op1_t>(Op0,676 Op1)));677}678 679template <typename Op0_t, typename Op1_t, typename Op2_t>680inline AllRecipe_match<Instruction::Select, Op0_t, Op1_t, Op2_t>681m_Select(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {682 return AllRecipe_match<Instruction::Select, Op0_t, Op1_t, Op2_t>(683 {Op0, Op1, Op2});684}685 686template <typename Op0_t>687inline match_combine_or<VPInstruction_match<VPInstruction::Not, Op0_t>,688 AllRecipe_commutative_match<689 Instruction::Xor, int_pred_ty<is_all_ones>, Op0_t>>690m_Not(const Op0_t &Op0) {691 return m_CombineOr(m_VPInstruction<VPInstruction::Not>(Op0),692 m_c_Binary<Instruction::Xor>(m_AllOnes(), Op0));693}694 695template <typename Op0_t, typename Op1_t>696inline match_combine_or<697 VPInstruction_match<VPInstruction::LogicalAnd, Op0_t, Op1_t>,698 AllRecipe_match<Instruction::Select, Op0_t, Op1_t, specific_intval<1>>>699m_LogicalAnd(const Op0_t &Op0, const Op1_t &Op1) {700 return m_CombineOr(701 m_VPInstruction<VPInstruction::LogicalAnd, Op0_t, Op1_t>(Op0, Op1),702 m_Select(Op0, Op1, m_False()));703}704 705template <typename Op0_t, typename Op1_t>706inline AllRecipe_match<Instruction::Select, Op0_t, specific_intval<1>, Op1_t>707m_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) {708 return m_Select(Op0, m_True(), Op1);709}710 711template <typename Op0_t, typename Op1_t, typename Op2_t>712using VPScalarIVSteps_match = Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t>, 0,713 false, VPScalarIVStepsRecipe>;714 715template <typename Op0_t, typename Op1_t, typename Op2_t>716inline VPScalarIVSteps_match<Op0_t, Op1_t, Op2_t>717m_ScalarIVSteps(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {718 return VPScalarIVSteps_match<Op0_t, Op1_t, Op2_t>({Op0, Op1, Op2});719}720 721template <typename Op0_t, typename Op1_t, typename Op2_t>722using VPDerivedIV_match =723 Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t>, 0, false, VPDerivedIVRecipe>;724 725template <typename Op0_t, typename Op1_t, typename Op2_t>726inline VPDerivedIV_match<Op0_t, Op1_t, Op2_t>727m_DerivedIV(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {728 return VPDerivedIV_match<Op0_t, Op1_t, Op2_t>({Op0, Op1, Op2});729}730 731template <typename Addr_t, typename Mask_t> struct Load_match {732 Addr_t Addr;733 Mask_t Mask;734 735 Load_match(Addr_t Addr, Mask_t Mask) : Addr(Addr), Mask(Mask) {}736 737 template <typename OpTy> bool match(const OpTy *V) const {738 auto *Load = dyn_cast<VPWidenLoadRecipe>(V);739 if (!Load || !Addr.match(Load->getAddr()) || !Load->isMasked() ||740 !Mask.match(Load->getMask()))741 return false;742 return true;743 }744};745 746/// Match a (possibly reversed) masked load.747template <typename Addr_t, typename Mask_t>748inline Load_match<Addr_t, Mask_t> m_MaskedLoad(const Addr_t &Addr,749 const Mask_t &Mask) {750 return Load_match<Addr_t, Mask_t>(Addr, Mask);751}752 753template <typename Addr_t, typename Val_t, typename Mask_t> struct Store_match {754 Addr_t Addr;755 Val_t Val;756 Mask_t Mask;757 758 Store_match(Addr_t Addr, Val_t Val, Mask_t Mask)759 : Addr(Addr), Val(Val), Mask(Mask) {}760 761 template <typename OpTy> bool match(const OpTy *V) const {762 auto *Store = dyn_cast<VPWidenStoreRecipe>(V);763 if (!Store || !Addr.match(Store->getAddr()) ||764 !Val.match(Store->getStoredValue()) || !Store->isMasked() ||765 !Mask.match(Store->getMask()))766 return false;767 return true;768 }769};770 771/// Match a (possibly reversed) masked store.772template <typename Addr_t, typename Val_t, typename Mask_t>773inline Store_match<Addr_t, Val_t, Mask_t>774m_MaskedStore(const Addr_t &Addr, const Val_t &Val, const Mask_t &Mask) {775 return Store_match<Addr_t, Val_t, Mask_t>(Addr, Val, Mask);776}777 778template <typename Op0_t, typename Op1_t>779using VectorEndPointerRecipe_match =780 Recipe_match<std::tuple<Op0_t, Op1_t>, 0,781 /*Commutative*/ false, VPVectorEndPointerRecipe>;782 783template <typename Op0_t, typename Op1_t>784VectorEndPointerRecipe_match<Op0_t, Op1_t> m_VecEndPtr(const Op0_t &Op0,785 const Op1_t &Op1) {786 return VectorEndPointerRecipe_match<Op0_t, Op1_t>(Op0, Op1);787}788 789/// Match a call argument at a given argument index.790template <typename Opnd_t> struct Argument_match {791 /// Call argument index to match.792 unsigned OpI;793 Opnd_t Val;794 795 Argument_match(unsigned OpIdx, const Opnd_t &V) : OpI(OpIdx), Val(V) {}796 797 template <typename OpTy> bool match(OpTy *V) const {798 if (const auto *R = dyn_cast<VPWidenIntrinsicRecipe>(V))799 return Val.match(R->getOperand(OpI));800 if (const auto *R = dyn_cast<VPWidenCallRecipe>(V))801 return Val.match(R->getOperand(OpI));802 if (const auto *R = dyn_cast<VPReplicateRecipe>(V))803 if (isa<CallInst>(R->getUnderlyingInstr()))804 return Val.match(R->getOperand(OpI + 1));805 return false;806 }807};808 809/// Match a call argument.810template <unsigned OpI, typename Opnd_t>811inline Argument_match<Opnd_t> m_Argument(const Opnd_t &Op) {812 return Argument_match<Opnd_t>(OpI, Op);813}814 815/// Intrinsic matchers.816struct IntrinsicID_match {817 unsigned ID;818 819 IntrinsicID_match(Intrinsic::ID IntrID) : ID(IntrID) {}820 821 template <typename OpTy> bool match(OpTy *V) const {822 if (const auto *R = dyn_cast<VPWidenIntrinsicRecipe>(V))823 return R->getVectorIntrinsicID() == ID;824 if (const auto *R = dyn_cast<VPWidenCallRecipe>(V))825 return R->getCalledScalarFunction()->getIntrinsicID() == ID;826 if (const auto *R = dyn_cast<VPReplicateRecipe>(V))827 if (const auto *CI = dyn_cast<CallInst>(R->getUnderlyingInstr()))828 if (const auto *F = CI->getCalledFunction())829 return F->getIntrinsicID() == ID;830 return false;831 }832};833 834/// Intrinsic matches are combinations of ID matchers, and argument835/// matchers. Higher arity matcher are defined recursively in terms of and-ing836/// them with lower arity matchers. Here's some convenient typedefs for up to837/// several arguments, and more can be added as needed838template <typename T0 = void, typename T1 = void, typename T2 = void,839 typename T3 = void>840struct m_Intrinsic_Ty;841template <typename T0> struct m_Intrinsic_Ty<T0> {842 using Ty = match_combine_and<IntrinsicID_match, Argument_match<T0>>;843};844template <typename T0, typename T1> struct m_Intrinsic_Ty<T0, T1> {845 using Ty =846 match_combine_and<typename m_Intrinsic_Ty<T0>::Ty, Argument_match<T1>>;847};848template <typename T0, typename T1, typename T2>849struct m_Intrinsic_Ty<T0, T1, T2> {850 using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1>::Ty,851 Argument_match<T2>>;852};853template <typename T0, typename T1, typename T2, typename T3>854struct m_Intrinsic_Ty {855 using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2>::Ty,856 Argument_match<T3>>;857};858 859/// Match intrinsic calls like this:860/// m_Intrinsic<Intrinsic::fabs>(m_VPValue(X), ...)861template <Intrinsic::ID IntrID> inline IntrinsicID_match m_Intrinsic() {862 return IntrinsicID_match(IntrID);863}864 865/// Match intrinsic calls with a runtime intrinsic ID.866inline IntrinsicID_match m_Intrinsic(Intrinsic::ID IntrID) {867 return IntrinsicID_match(IntrID);868}869 870template <Intrinsic::ID IntrID, typename T0>871inline typename m_Intrinsic_Ty<T0>::Ty m_Intrinsic(const T0 &Op0) {872 return m_CombineAnd(m_Intrinsic<IntrID>(), m_Argument<0>(Op0));873}874 875template <Intrinsic::ID IntrID, typename T0, typename T1>876inline typename m_Intrinsic_Ty<T0, T1>::Ty m_Intrinsic(const T0 &Op0,877 const T1 &Op1) {878 return m_CombineAnd(m_Intrinsic<IntrID>(Op0), m_Argument<1>(Op1));879}880 881template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2>882inline typename m_Intrinsic_Ty<T0, T1, T2>::Ty883m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2) {884 return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1), m_Argument<2>(Op2));885}886 887template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2,888 typename T3>889inline typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty890m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3) {891 return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2), m_Argument<3>(Op3));892}893 894struct live_in_vpvalue {895 template <typename ITy> bool match(ITy *V) const {896 VPValue *Val = dyn_cast<VPValue>(V);897 return Val && Val->isLiveIn();898 }899};900 901inline live_in_vpvalue m_LiveIn() { return live_in_vpvalue(); }902 903template <typename SubPattern_t> struct OneUse_match {904 SubPattern_t SubPattern;905 906 OneUse_match(const SubPattern_t &SP) : SubPattern(SP) {}907 908 template <typename OpTy> bool match(OpTy *V) {909 return V->hasOneUse() && SubPattern.match(V);910 }911};912 913template <typename T> inline OneUse_match<T> m_OneUse(const T &SubPattern) {914 return SubPattern;915}916 917} // namespace llvm::VPlanPatternMatch918 919#endif920