brintos / llvm-project-archived public Read only

1//===- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ----===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the SelectionDAG::LegalizeVectors method.10//11// The vector legalizer looks for vector operations which might need to be12// scalarized and legalizes them. This is a separate step from Legalize because13// scalarizing can introduce illegal types.  For example, suppose we have an14// ISD::SDIV of type v2i64 on x86-32.  The type is legal (for example, addition15// on a v2i64 is legal), but ISD::SDIV isn't legal, so we have to unroll the16// operation, which introduces nodes with the illegal type i64 which must be17// expanded.  Similarly, suppose we have an ISD::SRA of type v16i8 on PowerPC;18// the operation must be unrolled, which introduces nodes with the illegal19// type i8 which must be promoted.20//21// This does not legalize vector manipulations like ISD::BUILD_VECTOR,22// or operations that happen to take a vector which are custom-lowered;23// the legalization for such operations never produces nodes24// with illegal types, so it's okay to put off legalizing them until25// SelectionDAG::Legalize runs.26//27//===----------------------------------------------------------------------===//28 29#include "llvm/ADT/DenseMap.h"30#include "llvm/ADT/SmallVector.h"31#include "llvm/Analysis/TargetLibraryInfo.h"32#include "llvm/Analysis/VectorUtils.h"33#include "llvm/CodeGen/ISDOpcodes.h"34#include "llvm/CodeGen/SelectionDAG.h"35#include "llvm/CodeGen/SelectionDAGNodes.h"36#include "llvm/CodeGen/TargetLowering.h"37#include "llvm/CodeGen/ValueTypes.h"38#include "llvm/CodeGenTypes/MachineValueType.h"39#include "llvm/IR/DataLayout.h"40#include "llvm/Support/Casting.h"41#include "llvm/Support/Compiler.h"42#include "llvm/Support/Debug.h"43#include "llvm/Support/ErrorHandling.h"44#include <cassert>45#include <cstdint>46#include <iterator>47#include <utility>48 49using namespace llvm;50 51#define DEBUG_TYPE "legalizevectorops"52 53namespace {54 55class VectorLegalizer {56  SelectionDAG& DAG;57  const TargetLowering &TLI;58  bool Changed = false; // Keep track of whether anything changed59 60  /// For nodes that are of legal width, and that have more than one use, this61  /// map indicates what regularized operand to use.  This allows us to avoid62  /// legalizing the same thing more than once.63  SmallDenseMap<SDValue, SDValue, 64> LegalizedNodes;64 65  /// Adds a node to the translation cache.66  void AddLegalizedOperand(SDValue From, SDValue To) {67    LegalizedNodes.insert(std::make_pair(From, To));68    // If someone requests legalization of the new node, return itself.69    if (From != To)70      LegalizedNodes.insert(std::make_pair(To, To));71  }72 73  /// Legalizes the given node.74  SDValue LegalizeOp(SDValue Op);75 76  /// Assuming the node is legal, "legalize" the results.77  SDValue TranslateLegalizeResults(SDValue Op, SDNode *Result);78 79  /// Make sure Results are legal and update the translation cache.80  SDValue RecursivelyLegalizeResults(SDValue Op,81                                     MutableArrayRef<SDValue> Results);82 83  /// Wrapper to interface LowerOperation with a vector of Results.84  /// Returns false if the target wants to use default expansion. Otherwise85  /// returns true. If return is true and the Results are empty, then the86  /// target wants to keep the input node as is.87  bool LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results);88 89  /// Implements unrolling a VSETCC.90  SDValue UnrollVSETCC(SDNode *Node);91 92  /// Implement expand-based legalization of vector operations.93  ///94  /// This is just a high-level routine to dispatch to specific code paths for95  /// operations to legalize them.96  void Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results);97 98  /// Implements expansion for FP_TO_UINT; falls back to UnrollVectorOp if99  /// FP_TO_SINT isn't legal.100  void ExpandFP_TO_UINT(SDNode *Node, SmallVectorImpl<SDValue> &Results);101 102  /// Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if103  /// SINT_TO_FLOAT and SHR on vectors isn't legal.104  void ExpandUINT_TO_FLOAT(SDNode *Node, SmallVectorImpl<SDValue> &Results);105 106  /// Implement expansion for SIGN_EXTEND_INREG using SRL and SRA.107  SDValue ExpandSEXTINREG(SDNode *Node);108 109  /// Implement expansion for ANY_EXTEND_VECTOR_INREG.110  ///111  /// Shuffles the low lanes of the operand into place and bitcasts to the proper112  /// type. The contents of the bits in the extended part of each element are113  /// undef.114  SDValue ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node);115 116  /// Implement expansion for SIGN_EXTEND_VECTOR_INREG.117  ///118  /// Shuffles the low lanes of the operand into place, bitcasts to the proper119  /// type, then shifts left and arithmetic shifts right to introduce a sign120  /// extension.121  SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node);122 123  /// Implement expansion for ZERO_EXTEND_VECTOR_INREG.124  ///125  /// Shuffles the low lanes of the operand into place and blends zeros into126  /// the remaining lanes, finally bitcasting to the proper type.127  SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node);128 129  /// Expand bswap of vectors into a shuffle if legal.130  SDValue ExpandBSWAP(SDNode *Node);131 132  /// Implement vselect in terms of XOR, AND, OR when blend is not133  /// supported by the target.134  SDValue ExpandVSELECT(SDNode *Node);135  SDValue ExpandVP_SELECT(SDNode *Node);136  SDValue ExpandVP_MERGE(SDNode *Node);137  SDValue ExpandVP_REM(SDNode *Node);138  SDValue ExpandVP_FNEG(SDNode *Node);139  SDValue ExpandVP_FABS(SDNode *Node);140  SDValue ExpandVP_FCOPYSIGN(SDNode *Node);141  SDValue ExpandLOOP_DEPENDENCE_MASK(SDNode *N);142  SDValue ExpandSELECT(SDNode *Node);143  std::pair<SDValue, SDValue> ExpandLoad(SDNode *N);144  SDValue ExpandStore(SDNode *N);145  SDValue ExpandFNEG(SDNode *Node);146  SDValue ExpandFABS(SDNode *Node);147  SDValue ExpandFCOPYSIGN(SDNode *Node);148  void ExpandFSUB(SDNode *Node, SmallVectorImpl<SDValue> &Results);149  void ExpandSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);150  SDValue ExpandBITREVERSE(SDNode *Node);151  void ExpandUADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);152  void ExpandSADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);153  void ExpandMULO(SDNode *Node, SmallVectorImpl<SDValue> &Results);154  void ExpandFixedPointDiv(SDNode *Node, SmallVectorImpl<SDValue> &Results);155  void ExpandStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);156  void ExpandREM(SDNode *Node, SmallVectorImpl<SDValue> &Results);157 158  bool tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall LC,159                            SmallVectorImpl<SDValue> &Results);160  bool tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall Call_F32,161                            RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,162                            RTLIB::Libcall Call_F128,163                            RTLIB::Libcall Call_PPCF128,164                            SmallVectorImpl<SDValue> &Results);165 166  void UnrollStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);167 168  /// Implements vector promotion.169  ///170  /// This is essentially just bitcasting the operands to a different type and171  /// bitcasting the result back to the original type.172  void Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results);173 174  /// Implements [SU]INT_TO_FP vector promotion.175  ///176  /// This is a [zs]ext of the input operand to a larger integer type.177  void PromoteINT_TO_FP(SDNode *Node, SmallVectorImpl<SDValue> &Results);178 179  /// Implements FP_TO_[SU]INT vector promotion of the result type.180  ///181  /// It is promoted to a larger integer type.  The result is then182  /// truncated back to the original type.183  void PromoteFP_TO_INT(SDNode *Node, SmallVectorImpl<SDValue> &Results);184 185  /// Implements vector setcc operation promotion.186  ///187  /// All vector operands are promoted to a vector type with larger element188  /// type.189  void PromoteSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);190 191  void PromoteSTRICT(SDNode *Node, SmallVectorImpl<SDValue> &Results);192 193  /// Calculate the reduction using a type of higher precision and round the194  /// result to match the original type. Setting NonArithmetic signifies the195  /// rounding of the result does not affect its value.196  void PromoteFloatVECREDUCE(SDNode *Node, SmallVectorImpl<SDValue> &Results,197                             bool NonArithmetic);198 199public:200  VectorLegalizer(SelectionDAG& dag) :201      DAG(dag), TLI(dag.getTargetLoweringInfo()) {}202 203  /// Begin legalizer the vector operations in the DAG.204  bool Run();205};206 207} // end anonymous namespace208 209bool VectorLegalizer::Run() {210  // Before we start legalizing vector nodes, check if there are any vectors.211  bool HasVectors = false;212  for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),213       E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I) {214    // Check if the values of the nodes contain vectors. We don't need to check215    // the operands because we are going to check their values at some point.216    HasVectors = llvm::any_of(I->values(), [](EVT T) { return T.isVector(); });217 218    // If we found a vector node we can start the legalization.219    if (HasVectors)220      break;221  }222 223  // If this basic block has no vectors then no need to legalize vectors.224  if (!HasVectors)225    return false;226 227  // The legalize process is inherently a bottom-up recursive process (users228  // legalize their uses before themselves).  Given infinite stack space, we229  // could just start legalizing on the root and traverse the whole graph.  In230  // practice however, this causes us to run out of stack space on large basic231  // blocks.  To avoid this problem, compute an ordering of the nodes where each232  // node is only legalized after all of its operands are legalized.233  DAG.AssignTopologicalOrder();234  for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),235       E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I)236    LegalizeOp(SDValue(&*I, 0));237 238  // Finally, it's possible the root changed.  Get the new root.239  SDValue OldRoot = DAG.getRoot();240  assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?");241  DAG.setRoot(LegalizedNodes[OldRoot]);242 243  LegalizedNodes.clear();244 245  // Remove dead nodes now.246  DAG.RemoveDeadNodes();247 248  return Changed;249}250 251SDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDNode *Result) {252  assert(Op->getNumValues() == Result->getNumValues() &&253         "Unexpected number of results");254  // Generic legalization: just pass the operand through.255  for (unsigned i = 0, e = Op->getNumValues(); i != e; ++i)256    AddLegalizedOperand(Op.getValue(i), SDValue(Result, i));257  return SDValue(Result, Op.getResNo());258}259 260SDValue261VectorLegalizer::RecursivelyLegalizeResults(SDValue Op,262                                            MutableArrayRef<SDValue> Results) {263  assert(Results.size() == Op->getNumValues() &&264         "Unexpected number of results");265  // Make sure that the generated code is itself legal.266  for (unsigned i = 0, e = Results.size(); i != e; ++i) {267    Results[i] = LegalizeOp(Results[i]);268    AddLegalizedOperand(Op.getValue(i), Results[i]);269  }270 271  return Results[Op.getResNo()];272}273 274SDValue VectorLegalizer::LegalizeOp(SDValue Op) {275  // Note that LegalizeOp may be reentered even from single-use nodes, which276  // means that we always must cache transformed nodes.277  DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op);278  if (I != LegalizedNodes.end()) return I->second;279 280  // Legalize the operands281  SmallVector<SDValue, 8> Ops;282  for (const SDValue &Oper : Op->op_values())283    Ops.push_back(LegalizeOp(Oper));284 285  SDNode *Node = DAG.UpdateNodeOperands(Op.getNode(), Ops);286 287  bool HasVectorValueOrOp =288      llvm::any_of(Node->values(), [](EVT T) { return T.isVector(); }) ||289      llvm::any_of(Node->op_values(),290                   [](SDValue O) { return O.getValueType().isVector(); });291  if (!HasVectorValueOrOp)292    return TranslateLegalizeResults(Op, Node);293 294  TargetLowering::LegalizeAction Action = TargetLowering::Legal;295  EVT ValVT;296  switch (Op.getOpcode()) {297  default:298    return TranslateLegalizeResults(Op, Node);299  case ISD::LOAD: {300    LoadSDNode *LD = cast<LoadSDNode>(Node);301    ISD::LoadExtType ExtType = LD->getExtensionType();302    EVT LoadedVT = LD->getMemoryVT();303    if (LoadedVT.isVector() && ExtType != ISD::NON_EXTLOAD)304      Action = TLI.getLoadExtAction(ExtType, LD->getValueType(0), LoadedVT);305    break;306  }307  case ISD::STORE: {308    StoreSDNode *ST = cast<StoreSDNode>(Node);309    EVT StVT = ST->getMemoryVT();310    MVT ValVT = ST->getValue().getSimpleValueType();311    if (StVT.isVector() && ST->isTruncatingStore())312      Action = TLI.getTruncStoreAction(ValVT, StVT);313    break;314  }315  case ISD::MERGE_VALUES:316    Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));317    // This operation lies about being legal: when it claims to be legal,318    // it should actually be expanded.319    if (Action == TargetLowering::Legal)320      Action = TargetLowering::Expand;321    break;322#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN)               \323  case ISD::STRICT_##DAGN:324#include "llvm/IR/ConstrainedOps.def"325    ValVT = Node->getValueType(0);326    if (Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||327        Op.getOpcode() == ISD::STRICT_UINT_TO_FP)328      ValVT = Node->getOperand(1).getValueType();329    if (Op.getOpcode() == ISD::STRICT_FSETCC ||330        Op.getOpcode() == ISD::STRICT_FSETCCS) {331      MVT OpVT = Node->getOperand(1).getSimpleValueType();332      ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(3))->get();333      Action = TLI.getCondCodeAction(CCCode, OpVT);334      if (Action == TargetLowering::Legal)335        Action = TLI.getOperationAction(Node->getOpcode(), OpVT);336    } else {337      Action = TLI.getOperationAction(Node->getOpcode(), ValVT);338    }339    // If we're asked to expand a strict vector floating-point operation,340    // by default we're going to simply unroll it.  That is usually the341    // best approach, except in the case where the resulting strict (scalar)342    // operations would themselves use the fallback mutation to non-strict.343    // In that specific case, just do the fallback on the vector op.344    if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&345        TLI.getStrictFPOperationAction(Node->getOpcode(), ValVT) ==346            TargetLowering::Legal) {347      EVT EltVT = ValVT.getVectorElementType();348      if (TLI.getOperationAction(Node->getOpcode(), EltVT)349          == TargetLowering::Expand &&350          TLI.getStrictFPOperationAction(Node->getOpcode(), EltVT)351          == TargetLowering::Legal)352        Action = TargetLowering::Legal;353    }354    break;355  case ISD::ADD:356  case ISD::SUB:357  case ISD::MUL:358  case ISD::MULHS:359  case ISD::MULHU:360  case ISD::SDIV:361  case ISD::UDIV:362  case ISD::SREM:363  case ISD::UREM:364  case ISD::SDIVREM:365  case ISD::UDIVREM:366  case ISD::FADD:367  case ISD::FSUB:368  case ISD::FMUL:369  case ISD::FDIV:370  case ISD::FREM:371  case ISD::AND:372  case ISD::OR:373  case ISD::XOR:374  case ISD::SHL:375  case ISD::SRA:376  case ISD::SRL:377  case ISD::FSHL:378  case ISD::FSHR:379  case ISD::ROTL:380  case ISD::ROTR:381  case ISD::ABS:382  case ISD::ABDS:383  case ISD::ABDU:384  case ISD::AVGCEILS:385  case ISD::AVGCEILU:386  case ISD::AVGFLOORS:387  case ISD::AVGFLOORU:388  case ISD::BSWAP:389  case ISD::BITREVERSE:390  case ISD::CTLZ:391  case ISD::CTTZ:392  case ISD::CTLZ_ZERO_UNDEF:393  case ISD::CTTZ_ZERO_UNDEF:394  case ISD::CTPOP:395  case ISD::SELECT:396  case ISD::VSELECT:397  case ISD::SELECT_CC:398  case ISD::ZERO_EXTEND:399  case ISD::ANY_EXTEND:400  case ISD::TRUNCATE:401  case ISD::SIGN_EXTEND:402  case ISD::FP_TO_SINT:403  case ISD::FP_TO_UINT:404  case ISD::FNEG:405  case ISD::FABS:406  case ISD::FMINNUM:407  case ISD::FMAXNUM:408  case ISD::FMINNUM_IEEE:409  case ISD::FMAXNUM_IEEE:410  case ISD::FMINIMUM:411  case ISD::FMAXIMUM:412  case ISD::FMINIMUMNUM:413  case ISD::FMAXIMUMNUM:414  case ISD::FCOPYSIGN:415  case ISD::FSQRT:416  case ISD::FSIN:417  case ISD::FCOS:418  case ISD::FTAN:419  case ISD::FASIN:420  case ISD::FACOS:421  case ISD::FATAN:422  case ISD::FATAN2:423  case ISD::FSINH:424  case ISD::FCOSH:425  case ISD::FTANH:426  case ISD::FLDEXP:427  case ISD::FPOWI:428  case ISD::FPOW:429  case ISD::FLOG:430  case ISD::FLOG2:431  case ISD::FLOG10:432  case ISD::FEXP:433  case ISD::FEXP2:434  case ISD::FEXP10:435  case ISD::FCEIL:436  case ISD::FTRUNC:437  case ISD::FRINT:438  case ISD::FNEARBYINT:439  case ISD::FROUND:440  case ISD::FROUNDEVEN:441  case ISD::FFLOOR:442  case ISD::FP_ROUND:443  case ISD::FP_EXTEND:444  case ISD::FPTRUNC_ROUND:445  case ISD::FMA:446  case ISD::SIGN_EXTEND_INREG:447  case ISD::ANY_EXTEND_VECTOR_INREG:448  case ISD::SIGN_EXTEND_VECTOR_INREG:449  case ISD::ZERO_EXTEND_VECTOR_INREG:450  case ISD::SMIN:451  case ISD::SMAX:452  case ISD::UMIN:453  case ISD::UMAX:454  case ISD::SMUL_LOHI:455  case ISD::UMUL_LOHI:456  case ISD::SADDO:457  case ISD::UADDO:458  case ISD::SSUBO:459  case ISD::USUBO:460  case ISD::SMULO:461  case ISD::UMULO:462  case ISD::FCANONICALIZE:463  case ISD::FFREXP:464  case ISD::FMODF:465  case ISD::FSINCOS:466  case ISD::FSINCOSPI:467  case ISD::SADDSAT:468  case ISD::UADDSAT:469  case ISD::SSUBSAT:470  case ISD::USUBSAT:471  case ISD::SSHLSAT:472  case ISD::USHLSAT:473  case ISD::FP_TO_SINT_SAT:474  case ISD::FP_TO_UINT_SAT:475  case ISD::MGATHER:476  case ISD::VECTOR_COMPRESS:477  case ISD::SCMP:478  case ISD::UCMP:479  case ISD::LOOP_DEPENDENCE_WAR_MASK:480  case ISD::LOOP_DEPENDENCE_RAW_MASK:481    Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));482    break;483  case ISD::SMULFIX:484  case ISD::SMULFIXSAT:485  case ISD::UMULFIX:486  case ISD::UMULFIXSAT:487  case ISD::SDIVFIX:488  case ISD::SDIVFIXSAT:489  case ISD::UDIVFIX:490  case ISD::UDIVFIXSAT: {491    unsigned Scale = Node->getConstantOperandVal(2);492    Action = TLI.getFixedPointOperationAction(Node->getOpcode(),493                                              Node->getValueType(0), Scale);494    break;495  }496  case ISD::LROUND:497  case ISD::LLROUND:498  case ISD::LRINT:499  case ISD::LLRINT:500  case ISD::SINT_TO_FP:501  case ISD::UINT_TO_FP:502  case ISD::VECREDUCE_ADD:503  case ISD::VECREDUCE_MUL:504  case ISD::VECREDUCE_AND:505  case ISD::VECREDUCE_OR:506  case ISD::VECREDUCE_XOR:507  case ISD::VECREDUCE_SMAX:508  case ISD::VECREDUCE_SMIN:509  case ISD::VECREDUCE_UMAX:510  case ISD::VECREDUCE_UMIN:511  case ISD::VECREDUCE_FADD:512  case ISD::VECREDUCE_FMAX:513  case ISD::VECREDUCE_FMAXIMUM:514  case ISD::VECREDUCE_FMIN:515  case ISD::VECREDUCE_FMINIMUM:516  case ISD::VECREDUCE_FMUL:517  case ISD::VECTOR_FIND_LAST_ACTIVE:518    Action = TLI.getOperationAction(Node->getOpcode(),519                                    Node->getOperand(0).getValueType());520    break;521  case ISD::VECREDUCE_SEQ_FADD:522  case ISD::VECREDUCE_SEQ_FMUL:523    Action = TLI.getOperationAction(Node->getOpcode(),524                                    Node->getOperand(1).getValueType());525    break;526  case ISD::SETCC: {527    MVT OpVT = Node->getOperand(0).getSimpleValueType();528    ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();529    Action = TLI.getCondCodeAction(CCCode, OpVT);530    if (Action == TargetLowering::Legal)531      Action = TLI.getOperationAction(Node->getOpcode(), OpVT);532    break;533  }534  case ISD::PARTIAL_REDUCE_UMLA:535  case ISD::PARTIAL_REDUCE_SMLA:536  case ISD::PARTIAL_REDUCE_SUMLA:537  case ISD::PARTIAL_REDUCE_FMLA:538    Action =539        TLI.getPartialReduceMLAAction(Op.getOpcode(), Node->getValueType(0),540                                      Node->getOperand(1).getValueType());541    break;542 543#define BEGIN_REGISTER_VP_SDNODE(VPID, LEGALPOS, ...)                          \544  case ISD::VPID: {                                                            \545    EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS))        \546                                  : Node->getOperand(LEGALPOS).getValueType(); \547    if (ISD::VPID == ISD::VP_SETCC) {                                          \548      ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); \549      Action = TLI.getCondCodeAction(CCCode, LegalizeVT.getSimpleVT());        \550      if (Action != TargetLowering::Legal)                                     \551        break;                                                                 \552    }                                                                          \553    /* Defer non-vector results to LegalizeDAG. */                             \554    if (!Node->getValueType(0).isVector() &&                                   \555        Node->getValueType(0) != MVT::Other) {                                 \556      Action = TargetLowering::Legal;                                          \557      break;                                                                   \558    }                                                                          \559    Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT);            \560  } break;561#include "llvm/IR/VPIntrinsics.def"562  }563 564  LLVM_DEBUG(dbgs() << "\nLegalizing vector op: "; Node->dump(&DAG));565 566  SmallVector<SDValue, 8> ResultVals;567  switch (Action) {568  default: llvm_unreachable("This action is not supported yet!");569  case TargetLowering::Promote:570    assert((Op.getOpcode() != ISD::LOAD && Op.getOpcode() != ISD::STORE) &&571           "This action is not supported yet!");572    LLVM_DEBUG(dbgs() << "Promoting\n");573    Promote(Node, ResultVals);574    assert(!ResultVals.empty() && "No results for promotion?");575    break;576  case TargetLowering::Legal:577    LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");578    break;579  case TargetLowering::Custom:580    LLVM_DEBUG(dbgs() << "Trying custom legalization\n");581    if (LowerOperationWrapper(Node, ResultVals))582      break;583    LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");584    [[fallthrough]];585  case TargetLowering::Expand:586    LLVM_DEBUG(dbgs() << "Expanding\n");587    Expand(Node, ResultVals);588    break;589  }590 591  if (ResultVals.empty())592    return TranslateLegalizeResults(Op, Node);593 594  Changed = true;595  return RecursivelyLegalizeResults(Op, ResultVals);596}597 598// FIXME: This is very similar to TargetLowering::LowerOperationWrapper. Can we599// merge them somehow?600bool VectorLegalizer::LowerOperationWrapper(SDNode *Node,601                                            SmallVectorImpl<SDValue> &Results) {602  SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);603 604  if (!Res.getNode())605    return false;606 607  if (Res == SDValue(Node, 0))608    return true;609 610  // If the original node has one result, take the return value from611  // LowerOperation as is. It might not be result number 0.612  if (Node->getNumValues() == 1) {613    Results.push_back(Res);614    return true;615  }616 617  // If the original node has multiple results, then the return node should618  // have the same number of results.619  assert((Node->getNumValues() == Res->getNumValues()) &&620         "Lowering returned the wrong number of results!");621 622  // Places new result values base on N result number.623  for (unsigned I = 0, E = Node->getNumValues(); I != E; ++I)624    Results.push_back(Res.getValue(I));625 626  return true;627}628 629void VectorLegalizer::PromoteSETCC(SDNode *Node,630                                   SmallVectorImpl<SDValue> &Results) {631  MVT VecVT = Node->getOperand(0).getSimpleValueType();632  MVT NewVecVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VecVT);633 634  unsigned ExtOp = VecVT.isFloatingPoint() ? ISD::FP_EXTEND : ISD::ANY_EXTEND;635 636  SDLoc DL(Node);637  SmallVector<SDValue, 5> Operands(Node->getNumOperands());638 639  Operands[0] = DAG.getNode(ExtOp, DL, NewVecVT, Node->getOperand(0));640  Operands[1] = DAG.getNode(ExtOp, DL, NewVecVT, Node->getOperand(1));641  Operands[2] = Node->getOperand(2);642 643  if (Node->getOpcode() == ISD::VP_SETCC) {644    Operands[3] = Node->getOperand(3); // mask645    Operands[4] = Node->getOperand(4); // evl646  }647 648  SDValue Res = DAG.getNode(Node->getOpcode(), DL, Node->getSimpleValueType(0),649                            Operands, Node->getFlags());650 651  Results.push_back(Res);652}653 654void VectorLegalizer::PromoteSTRICT(SDNode *Node,655                                    SmallVectorImpl<SDValue> &Results) {656  MVT VecVT = Node->getOperand(1).getSimpleValueType();657  MVT NewVecVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VecVT);658 659  assert(VecVT.isFloatingPoint());660 661  SDLoc DL(Node);662  SmallVector<SDValue, 5> Operands(Node->getNumOperands());663  SmallVector<SDValue, 2> Chains;664 665  for (unsigned j = 1; j != Node->getNumOperands(); ++j)666    if (Node->getOperand(j).getValueType().isVector() &&667        !(ISD::isVPOpcode(Node->getOpcode()) &&668          ISD::getVPMaskIdx(Node->getOpcode()) == j)) // Skip mask operand.669    {670      // promote the vector operand.671      SDValue Ext =672          DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {NewVecVT, MVT::Other},673                      {Node->getOperand(0), Node->getOperand(j)});674      Operands[j] = Ext.getValue(0);675      Chains.push_back(Ext.getValue(1));676    } else677      Operands[j] = Node->getOperand(j); // Skip no vector operand.678 679  SDVTList VTs = DAG.getVTList(NewVecVT, Node->getValueType(1));680 681  Operands[0] = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);682 683  SDValue Res =684      DAG.getNode(Node->getOpcode(), DL, VTs, Operands, Node->getFlags());685 686  SDValue Round =687      DAG.getNode(ISD::STRICT_FP_ROUND, DL, {VecVT, MVT::Other},688                  {Res.getValue(1), Res.getValue(0),689                   DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)});690 691  Results.push_back(Round.getValue(0));692  Results.push_back(Round.getValue(1));693}694 695void VectorLegalizer::PromoteFloatVECREDUCE(SDNode *Node,696                                            SmallVectorImpl<SDValue> &Results,697                                            bool NonArithmetic) {698  MVT OpVT = Node->getOperand(0).getSimpleValueType();699  assert(OpVT.isFloatingPoint() && "Expected floating point reduction!");700  MVT NewOpVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OpVT);701 702  SDLoc DL(Node);703  SDValue NewOp = DAG.getNode(ISD::FP_EXTEND, DL, NewOpVT, Node->getOperand(0));704  SDValue Rdx =705      DAG.getNode(Node->getOpcode(), DL, NewOpVT.getVectorElementType(), NewOp,706                  Node->getFlags());707  SDValue Res =708      DAG.getNode(ISD::FP_ROUND, DL, Node->getValueType(0), Rdx,709                  DAG.getIntPtrConstant(NonArithmetic, DL, /*isTarget=*/true));710  Results.push_back(Res);711}712 713void VectorLegalizer::Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results) {714  // For a few operations there is a specific concept for promotion based on715  // the operand's type.716  switch (Node->getOpcode()) {717  case ISD::SINT_TO_FP:718  case ISD::UINT_TO_FP:719  case ISD::STRICT_SINT_TO_FP:720  case ISD::STRICT_UINT_TO_FP:721    // "Promote" the operation by extending the operand.722    PromoteINT_TO_FP(Node, Results);723    return;724  case ISD::FP_TO_UINT:725  case ISD::FP_TO_SINT:726  case ISD::STRICT_FP_TO_UINT:727  case ISD::STRICT_FP_TO_SINT:728    // Promote the operation by extending the operand.729    PromoteFP_TO_INT(Node, Results);730    return;731  case ISD::VP_SETCC:732  case ISD::SETCC:733    // Promote the operation by extending the operand.734    PromoteSETCC(Node, Results);735    return;736  case ISD::STRICT_FADD:737  case ISD::STRICT_FSUB:738  case ISD::STRICT_FMUL:739  case ISD::STRICT_FDIV:740  case ISD::STRICT_FSQRT:741  case ISD::STRICT_FMA:742    PromoteSTRICT(Node, Results);743    return;744  case ISD::VECREDUCE_FADD:745    PromoteFloatVECREDUCE(Node, Results, /*NonArithmetic=*/false);746    return;747  case ISD::VECREDUCE_FMAX:748  case ISD::VECREDUCE_FMAXIMUM:749  case ISD::VECREDUCE_FMIN:750  case ISD::VECREDUCE_FMINIMUM:751    PromoteFloatVECREDUCE(Node, Results, /*NonArithmetic=*/true);752    return;753  case ISD::FP_ROUND:754  case ISD::FP_EXTEND:755    // These operations are used to do promotion so they can't be promoted756    // themselves.757    llvm_unreachable("Don't know how to promote this operation!");758  case ISD::VP_FABS:759  case ISD::VP_FCOPYSIGN:760  case ISD::VP_FNEG:761    // Promoting fabs, fneg, and fcopysign changes their semantics.762    llvm_unreachable("These operations should not be promoted");763  }764 765  // There are currently two cases of vector promotion:766  // 1) Bitcasting a vector of integers to a different type to a vector of the767  //    same overall length. For example, x86 promotes ISD::AND v2i32 to v1i64.768  // 2) Extending a vector of floats to a vector of the same number of larger769  //    floats. For example, AArch64 promotes ISD::FADD on v4f16 to v4f32.770  assert(Node->getNumValues() == 1 &&771         "Can't promote a vector with multiple results!");772  MVT VT = Node->getSimpleValueType(0);773  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);774  SDLoc dl(Node);775  SmallVector<SDValue, 4> Operands(Node->getNumOperands());776 777  for (unsigned j = 0; j != Node->getNumOperands(); ++j) {778    // Do not promote the mask operand of a VP OP.779    bool SkipPromote = ISD::isVPOpcode(Node->getOpcode()) &&780                       ISD::getVPMaskIdx(Node->getOpcode()) == j;781    if (Node->getOperand(j).getValueType().isVector() && !SkipPromote)782      if (Node->getOperand(j)783              .getValueType()784              .getVectorElementType()785              .isFloatingPoint() &&786          NVT.isVector() && NVT.getVectorElementType().isFloatingPoint())787        if (ISD::isVPOpcode(Node->getOpcode())) {788          unsigned EVLIdx =789              *ISD::getVPExplicitVectorLengthIdx(Node->getOpcode());790          unsigned MaskIdx = *ISD::getVPMaskIdx(Node->getOpcode());791          Operands[j] =792              DAG.getNode(ISD::VP_FP_EXTEND, dl, NVT, Node->getOperand(j),793                          Node->getOperand(MaskIdx), Node->getOperand(EVLIdx));794        } else {795          Operands[j] =796              DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(j));797        }798      else799        Operands[j] = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(j));800    else801      Operands[j] = Node->getOperand(j);802  }803 804  SDValue Res =805      DAG.getNode(Node->getOpcode(), dl, NVT, Operands, Node->getFlags());806 807  if ((VT.isFloatingPoint() && NVT.isFloatingPoint()) ||808      (VT.isVector() && VT.getVectorElementType().isFloatingPoint() &&809       NVT.isVector() && NVT.getVectorElementType().isFloatingPoint()))810    if (ISD::isVPOpcode(Node->getOpcode())) {811      unsigned EVLIdx = *ISD::getVPExplicitVectorLengthIdx(Node->getOpcode());812      unsigned MaskIdx = *ISD::getVPMaskIdx(Node->getOpcode());813      Res = DAG.getNode(ISD::VP_FP_ROUND, dl, VT, Res,814                        Node->getOperand(MaskIdx), Node->getOperand(EVLIdx));815    } else {816      Res = DAG.getNode(ISD::FP_ROUND, dl, VT, Res,817                        DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));818    }819  else820    Res = DAG.getNode(ISD::BITCAST, dl, VT, Res);821 822  Results.push_back(Res);823}824 825void VectorLegalizer::PromoteINT_TO_FP(SDNode *Node,826                                       SmallVectorImpl<SDValue> &Results) {827  // INT_TO_FP operations may require the input operand be promoted even828  // when the type is otherwise legal.829  bool IsStrict = Node->isStrictFPOpcode();830  MVT VT = Node->getOperand(IsStrict ? 1 : 0).getSimpleValueType();831  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);832  assert(NVT.getVectorNumElements() == VT.getVectorNumElements() &&833         "Vectors have different number of elements!");834 835  SDLoc dl(Node);836  SmallVector<SDValue, 4> Operands(Node->getNumOperands());837 838  unsigned Opc = (Node->getOpcode() == ISD::UINT_TO_FP ||839                  Node->getOpcode() == ISD::STRICT_UINT_TO_FP)840                     ? ISD::ZERO_EXTEND841                     : ISD::SIGN_EXTEND;842  for (unsigned j = 0; j != Node->getNumOperands(); ++j) {843    if (Node->getOperand(j).getValueType().isVector())844      Operands[j] = DAG.getNode(Opc, dl, NVT, Node->getOperand(j));845    else846      Operands[j] = Node->getOperand(j);847  }848 849  if (IsStrict) {850    SDValue Res = DAG.getNode(Node->getOpcode(), dl,851                              {Node->getValueType(0), MVT::Other}, Operands);852    Results.push_back(Res);853    Results.push_back(Res.getValue(1));854    return;855  }856 857  SDValue Res =858      DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Operands);859  Results.push_back(Res);860}861 862// For FP_TO_INT we promote the result type to a vector type with wider863// elements and then truncate the result.  This is different from the default864// PromoteVector which uses bitcast to promote thus assumning that the865// promoted vector type has the same overall size.866void VectorLegalizer::PromoteFP_TO_INT(SDNode *Node,867                                       SmallVectorImpl<SDValue> &Results) {868  MVT VT = Node->getSimpleValueType(0);869  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);870  bool IsStrict = Node->isStrictFPOpcode();871  assert(NVT.getVectorNumElements() == VT.getVectorNumElements() &&872         "Vectors have different number of elements!");873 874  unsigned NewOpc = Node->getOpcode();875  // Change FP_TO_UINT to FP_TO_SINT if possible.876  // TODO: Should we only do this if FP_TO_UINT itself isn't legal?877  if (NewOpc == ISD::FP_TO_UINT &&878      TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))879    NewOpc = ISD::FP_TO_SINT;880 881  if (NewOpc == ISD::STRICT_FP_TO_UINT &&882      TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT))883    NewOpc = ISD::STRICT_FP_TO_SINT;884 885  SDLoc dl(Node);886  SDValue Promoted, Chain;887  if (IsStrict) {888    Promoted = DAG.getNode(NewOpc, dl, {NVT, MVT::Other},889                           {Node->getOperand(0), Node->getOperand(1)});890    Chain = Promoted.getValue(1);891  } else892    Promoted = DAG.getNode(NewOpc, dl, NVT, Node->getOperand(0));893 894  // Assert that the converted value fits in the original type.  If it doesn't895  // (eg: because the value being converted is too big), then the result of the896  // original operation was undefined anyway, so the assert is still correct.897  if (Node->getOpcode() == ISD::FP_TO_UINT ||898      Node->getOpcode() == ISD::STRICT_FP_TO_UINT)899    NewOpc = ISD::AssertZext;900  else901    NewOpc = ISD::AssertSext;902 903  Promoted = DAG.getNode(NewOpc, dl, NVT, Promoted,904                         DAG.getValueType(VT.getScalarType()));905  Promoted = DAG.getNode(ISD::TRUNCATE, dl, VT, Promoted);906  Results.push_back(Promoted);907  if (IsStrict)908    Results.push_back(Chain);909}910 911std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(SDNode *N) {912  LoadSDNode *LD = cast<LoadSDNode>(N);913  return TLI.scalarizeVectorLoad(LD, DAG);914}915 916SDValue VectorLegalizer::ExpandStore(SDNode *N) {917  StoreSDNode *ST = cast<StoreSDNode>(N);918  SDValue TF = TLI.scalarizeVectorStore(ST, DAG);919  return TF;920}921 922void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {923  switch (Node->getOpcode()) {924  case ISD::LOAD: {925    std::pair<SDValue, SDValue> Tmp = ExpandLoad(Node);926    Results.push_back(Tmp.first);927    Results.push_back(Tmp.second);928    return;929  }930  case ISD::STORE:931    Results.push_back(ExpandStore(Node));932    return;933  case ISD::MERGE_VALUES:934    for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)935      Results.push_back(Node->getOperand(i));936    return;937  case ISD::SIGN_EXTEND_INREG:938    if (SDValue Expanded = ExpandSEXTINREG(Node)) {939      Results.push_back(Expanded);940      return;941    }942    break;943  case ISD::ANY_EXTEND_VECTOR_INREG:944    Results.push_back(ExpandANY_EXTEND_VECTOR_INREG(Node));945    return;946  case ISD::SIGN_EXTEND_VECTOR_INREG:947    Results.push_back(ExpandSIGN_EXTEND_VECTOR_INREG(Node));948    return;949  case ISD::ZERO_EXTEND_VECTOR_INREG:950    Results.push_back(ExpandZERO_EXTEND_VECTOR_INREG(Node));951    return;952  case ISD::BSWAP:953    if (SDValue Expanded = ExpandBSWAP(Node)) {954      Results.push_back(Expanded);955      return;956    }957    break;958  case ISD::VP_BSWAP:959    Results.push_back(TLI.expandVPBSWAP(Node, DAG));960    return;961  case ISD::VSELECT:962    if (SDValue Expanded = ExpandVSELECT(Node)) {963      Results.push_back(Expanded);964      return;965    }966    break;967  case ISD::VP_SELECT:968    if (SDValue Expanded = ExpandVP_SELECT(Node)) {969      Results.push_back(Expanded);970      return;971    }972    break;973  case ISD::VP_SREM:974  case ISD::VP_UREM:975    if (SDValue Expanded = ExpandVP_REM(Node)) {976      Results.push_back(Expanded);977      return;978    }979    break;980  case ISD::VP_FNEG:981    if (SDValue Expanded = ExpandVP_FNEG(Node)) {982      Results.push_back(Expanded);983      return;984    }985    break;986  case ISD::VP_FABS:987    if (SDValue Expanded = ExpandVP_FABS(Node)) {988      Results.push_back(Expanded);989      return;990    }991    break;992  case ISD::VP_FCOPYSIGN:993    if (SDValue Expanded = ExpandVP_FCOPYSIGN(Node)) {994      Results.push_back(Expanded);995      return;996    }997    break;998  case ISD::SELECT:999    if (SDValue Expanded = ExpandSELECT(Node)) {1000      Results.push_back(Expanded);1001      return;1002    }1003    break;1004  case ISD::SELECT_CC: {1005    if (Node->getValueType(0).isScalableVector()) {1006      EVT CondVT = TLI.getSetCCResultType(1007          DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));1008      SDValue SetCC =1009          DAG.getNode(ISD::SETCC, SDLoc(Node), CondVT, Node->getOperand(0),1010                      Node->getOperand(1), Node->getOperand(4));1011      Results.push_back(DAG.getSelect(SDLoc(Node), Node->getValueType(0), SetCC,1012                                      Node->getOperand(2),1013                                      Node->getOperand(3)));1014      return;1015    }1016    break;1017  }1018  case ISD::FP_TO_UINT:1019    ExpandFP_TO_UINT(Node, Results);1020    return;1021  case ISD::UINT_TO_FP:1022    ExpandUINT_TO_FLOAT(Node, Results);1023    return;1024  case ISD::FNEG:1025    if (SDValue Expanded = ExpandFNEG(Node)) {1026      Results.push_back(Expanded);1027      return;1028    }1029    break;1030  case ISD::FABS:1031    if (SDValue Expanded = ExpandFABS(Node)) {1032      Results.push_back(Expanded);1033      return;1034    }1035    break;1036  case ISD::FCOPYSIGN:1037    if (SDValue Expanded = ExpandFCOPYSIGN(Node)) {1038      Results.push_back(Expanded);1039      return;1040    }1041    break;1042  case ISD::FSUB:1043    ExpandFSUB(Node, Results);1044    return;1045  case ISD::SETCC:1046  case ISD::VP_SETCC:1047    ExpandSETCC(Node, Results);1048    return;1049  case ISD::ABS:1050    if (SDValue Expanded = TLI.expandABS(Node, DAG)) {1051      Results.push_back(Expanded);1052      return;1053    }1054    break;1055  case ISD::ABDS:1056  case ISD::ABDU:1057    if (SDValue Expanded = TLI.expandABD(Node, DAG)) {1058      Results.push_back(Expanded);1059      return;1060    }1061    break;1062  case ISD::AVGCEILS:1063  case ISD::AVGCEILU:1064  case ISD::AVGFLOORS:1065  case ISD::AVGFLOORU:1066    if (SDValue Expanded = TLI.expandAVG(Node, DAG)) {1067      Results.push_back(Expanded);1068      return;1069    }1070    break;1071  case ISD::BITREVERSE:1072    if (SDValue Expanded = ExpandBITREVERSE(Node)) {1073      Results.push_back(Expanded);1074      return;1075    }1076    break;1077  case ISD::VP_BITREVERSE:1078    if (SDValue Expanded = TLI.expandVPBITREVERSE(Node, DAG)) {1079      Results.push_back(Expanded);1080      return;1081    }1082    break;1083  case ISD::CTPOP:1084    if (SDValue Expanded = TLI.expandCTPOP(Node, DAG)) {1085      Results.push_back(Expanded);1086      return;1087    }1088    break;1089  case ISD::VP_CTPOP:1090    if (SDValue Expanded = TLI.expandVPCTPOP(Node, DAG)) {1091      Results.push_back(Expanded);1092      return;1093    }1094    break;1095  case ISD::CTLZ:1096  case ISD::CTLZ_ZERO_UNDEF:1097    if (SDValue Expanded = TLI.expandCTLZ(Node, DAG)) {1098      Results.push_back(Expanded);1099      return;1100    }1101    break;1102  case ISD::VP_CTLZ:1103  case ISD::VP_CTLZ_ZERO_UNDEF:1104    if (SDValue Expanded = TLI.expandVPCTLZ(Node, DAG)) {1105      Results.push_back(Expanded);1106      return;1107    }1108    break;1109  case ISD::CTTZ:1110  case ISD::CTTZ_ZERO_UNDEF:1111    if (SDValue Expanded = TLI.expandCTTZ(Node, DAG)) {1112      Results.push_back(Expanded);1113      return;1114    }1115    break;1116  case ISD::VP_CTTZ:1117  case ISD::VP_CTTZ_ZERO_UNDEF:1118    if (SDValue Expanded = TLI.expandVPCTTZ(Node, DAG)) {1119      Results.push_back(Expanded);1120      return;1121    }1122    break;1123  case ISD::FSHL:1124  case ISD::VP_FSHL:1125  case ISD::FSHR:1126  case ISD::VP_FSHR:1127    if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG)) {1128      Results.push_back(Expanded);1129      return;1130    }1131    break;1132  case ISD::ROTL:1133  case ISD::ROTR:1134    if (SDValue Expanded = TLI.expandROT(Node, false /*AllowVectorOps*/, DAG)) {1135      Results.push_back(Expanded);1136      return;1137    }1138    break;1139  case ISD::FMINNUM:1140  case ISD::FMAXNUM:1141    if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG)) {1142      Results.push_back(Expanded);1143      return;1144    }1145    break;1146  case ISD::FMINIMUM:1147  case ISD::FMAXIMUM:1148    Results.push_back(TLI.expandFMINIMUM_FMAXIMUM(Node, DAG));1149    return;1150  case ISD::FMINIMUMNUM:1151  case ISD::FMAXIMUMNUM:1152    Results.push_back(TLI.expandFMINIMUMNUM_FMAXIMUMNUM(Node, DAG));1153    return;1154  case ISD::SMIN:1155  case ISD::SMAX:1156  case ISD::UMIN:1157  case ISD::UMAX:1158    if (SDValue Expanded = TLI.expandIntMINMAX(Node, DAG)) {1159      Results.push_back(Expanded);1160      return;1161    }1162    break;1163  case ISD::UADDO:1164  case ISD::USUBO:1165    ExpandUADDSUBO(Node, Results);1166    return;1167  case ISD::SADDO:1168  case ISD::SSUBO:1169    ExpandSADDSUBO(Node, Results);1170    return;1171  case ISD::UMULO:1172  case ISD::SMULO:1173    ExpandMULO(Node, Results);1174    return;1175  case ISD::USUBSAT:1176  case ISD::SSUBSAT:1177  case ISD::UADDSAT:1178  case ISD::SADDSAT:1179    if (SDValue Expanded = TLI.expandAddSubSat(Node, DAG)) {1180      Results.push_back(Expanded);1181      return;1182    }1183    break;1184  case ISD::USHLSAT:1185  case ISD::SSHLSAT:1186    if (SDValue Expanded = TLI.expandShlSat(Node, DAG)) {1187      Results.push_back(Expanded);1188      return;1189    }1190    break;1191  case ISD::FP_TO_SINT_SAT:1192  case ISD::FP_TO_UINT_SAT:1193    // Expand the fpsosisat if it is scalable to prevent it from unrolling below.1194    if (Node->getValueType(0).isScalableVector()) {1195      if (SDValue Expanded = TLI.expandFP_TO_INT_SAT(Node, DAG)) {1196        Results.push_back(Expanded);1197        return;1198      }1199    }1200    break;1201  case ISD::SMULFIX:1202  case ISD::UMULFIX:1203    if (SDValue Expanded = TLI.expandFixedPointMul(Node, DAG)) {1204      Results.push_back(Expanded);1205      return;1206    }1207    break;1208  case ISD::SMULFIXSAT:1209  case ISD::UMULFIXSAT:1210    // FIXME: We do not expand SMULFIXSAT/UMULFIXSAT here yet, not sure exactly1211    // why. Maybe it results in worse codegen compared to the unroll for some1212    // targets? This should probably be investigated. And if we still prefer to1213    // unroll an explanation could be helpful.1214    break;1215  case ISD::SDIVFIX:1216  case ISD::UDIVFIX:1217    ExpandFixedPointDiv(Node, Results);1218    return;1219  case ISD::SDIVFIXSAT:1220  case ISD::UDIVFIXSAT:1221    break;1222#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN)               \1223  case ISD::STRICT_##DAGN:1224#include "llvm/IR/ConstrainedOps.def"1225    ExpandStrictFPOp(Node, Results);1226    return;1227  case ISD::VECREDUCE_ADD:1228  case ISD::VECREDUCE_MUL:1229  case ISD::VECREDUCE_AND:1230  case ISD::VECREDUCE_OR:1231  case ISD::VECREDUCE_XOR:1232  case ISD::VECREDUCE_SMAX:1233  case ISD::VECREDUCE_SMIN:1234  case ISD::VECREDUCE_UMAX:1235  case ISD::VECREDUCE_UMIN:1236  case ISD::VECREDUCE_FADD:1237  case ISD::VECREDUCE_FMUL:1238  case ISD::VECREDUCE_FMAX:1239  case ISD::VECREDUCE_FMIN:1240  case ISD::VECREDUCE_FMAXIMUM:1241  case ISD::VECREDUCE_FMINIMUM:1242    Results.push_back(TLI.expandVecReduce(Node, DAG));1243    return;1244  case ISD::PARTIAL_REDUCE_UMLA:1245  case ISD::PARTIAL_REDUCE_SMLA:1246  case ISD::PARTIAL_REDUCE_SUMLA:1247  case ISD::PARTIAL_REDUCE_FMLA:1248    Results.push_back(TLI.expandPartialReduceMLA(Node, DAG));1249    return;1250  case ISD::VECREDUCE_SEQ_FADD:1251  case ISD::VECREDUCE_SEQ_FMUL:1252    Results.push_back(TLI.expandVecReduceSeq(Node, DAG));1253    return;1254  case ISD::SREM:1255  case ISD::UREM:1256    ExpandREM(Node, Results);1257    return;1258  case ISD::VP_MERGE:1259    if (SDValue Expanded = ExpandVP_MERGE(Node)) {1260      Results.push_back(Expanded);1261      return;1262    }1263    break;1264  case ISD::FREM:1265    if (tryExpandVecMathCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,1266                             RTLIB::REM_F80, RTLIB::REM_F128,1267                             RTLIB::REM_PPCF128, Results))1268      return;1269 1270    break;1271  case ISD::FSINCOS:1272  case ISD::FSINCOSPI: {1273    EVT VT = Node->getValueType(0);1274    RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS1275                            ? RTLIB::getSINCOS(VT)1276                            : RTLIB::getSINCOSPI(VT);1277    if (LC != RTLIB::UNKNOWN_LIBCALL &&1278        TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results))1279      return;1280 1281    // TODO: Try to see if there's a narrower call available to use before1282    // scalarizing.1283    break;1284  }1285  case ISD::FMODF: {1286    EVT VT = Node->getValueType(0);1287    RTLIB::Libcall LC = RTLIB::getMODF(VT);1288    if (LC != RTLIB::UNKNOWN_LIBCALL &&1289        TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results,1290                                          /*CallRetResNo=*/0))1291      return;1292    break;1293  }1294  case ISD::VECTOR_COMPRESS:1295    Results.push_back(TLI.expandVECTOR_COMPRESS(Node, DAG));1296    return;1297  case ISD::VECTOR_FIND_LAST_ACTIVE:1298    Results.push_back(TLI.expandVectorFindLastActive(Node, DAG));1299    return;1300  case ISD::SCMP:1301  case ISD::UCMP:1302    Results.push_back(TLI.expandCMP(Node, DAG));1303    return;1304  case ISD::LOOP_DEPENDENCE_WAR_MASK:1305  case ISD::LOOP_DEPENDENCE_RAW_MASK:1306    Results.push_back(ExpandLOOP_DEPENDENCE_MASK(Node));1307    return;1308 1309  case ISD::FADD:1310  case ISD::FMUL:1311  case ISD::FMA:1312  case ISD::FDIV:1313  case ISD::FCEIL:1314  case ISD::FFLOOR:1315  case ISD::FNEARBYINT:1316  case ISD::FRINT:1317  case ISD::FROUND:1318  case ISD::FROUNDEVEN:1319  case ISD::FTRUNC:1320  case ISD::FSQRT:1321    if (SDValue Expanded = TLI.expandVectorNaryOpBySplitting(Node, DAG)) {1322      Results.push_back(Expanded);1323      return;1324    }1325    break;1326  }1327 1328  SDValue Unrolled = DAG.UnrollVectorOp(Node);1329  if (Node->getNumValues() == 1) {1330    Results.push_back(Unrolled);1331  } else {1332    assert(Node->getNumValues() == Unrolled->getNumValues() &&1333      "VectorLegalizer Expand returned wrong number of results!");1334    for (unsigned I = 0, E = Unrolled->getNumValues(); I != E; ++I)1335      Results.push_back(Unrolled.getValue(I));1336  }1337}1338 1339SDValue VectorLegalizer::ExpandSELECT(SDNode *Node) {1340  // Lower a select instruction where the condition is a scalar and the1341  // operands are vectors. Lower this select to VSELECT and implement it1342  // using XOR AND OR. The selector bit is broadcasted.1343  EVT VT = Node->getValueType(0);1344  SDLoc DL(Node);1345 1346  SDValue Mask = Node->getOperand(0);1347  SDValue Op1 = Node->getOperand(1);1348  SDValue Op2 = Node->getOperand(2);1349 1350  assert(VT.isVector() && !Mask.getValueType().isVector()1351         && Op1.getValueType() == Op2.getValueType() && "Invalid type");1352 1353  // If we can't even use the basic vector operations of1354  // AND,OR,XOR, we will have to scalarize the op.1355  // Notice that the operation may be 'promoted' which means that it is1356  // 'bitcasted' to another type which is handled.1357  // Also, we need to be able to construct a splat vector using either1358  // BUILD_VECTOR or SPLAT_VECTOR.1359  // FIXME: Should we also permit fixed-length SPLAT_VECTOR as a fallback to1360  // BUILD_VECTOR?1361  if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||1362      TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||1363      TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand ||1364      TLI.getOperationAction(VT.isFixedLengthVector() ? ISD::BUILD_VECTOR1365                                                      : ISD::SPLAT_VECTOR,1366                             VT) == TargetLowering::Expand)1367    return SDValue();1368 1369  // Generate a mask operand.1370  EVT MaskTy = VT.changeVectorElementTypeToInteger();1371 1372  // What is the size of each element in the vector mask.1373  EVT BitTy = MaskTy.getScalarType();1374 1375  Mask = DAG.getSelect(DL, BitTy, Mask, DAG.getAllOnesConstant(DL, BitTy),1376                       DAG.getConstant(0, DL, BitTy));1377 1378  // Broadcast the mask so that the entire vector is all one or all zero.1379  Mask = DAG.getSplat(MaskTy, DL, Mask);1380 1381  // Bitcast the operands to be the same type as the mask.1382  // This is needed when we select between FP types because1383  // the mask is a vector of integers.1384  Op1 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op1);1385  Op2 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op2);1386 1387  SDValue NotMask = DAG.getNOT(DL, Mask, MaskTy);1388 1389  Op1 = DAG.getNode(ISD::AND, DL, MaskTy, Op1, Mask);1390  Op2 = DAG.getNode(ISD::AND, DL, MaskTy, Op2, NotMask);1391  SDValue Val = DAG.getNode(ISD::OR, DL, MaskTy, Op1, Op2);1392  return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val);1393}1394 1395SDValue VectorLegalizer::ExpandSEXTINREG(SDNode *Node) {1396  EVT VT = Node->getValueType(0);1397 1398  // Make sure that the SRA and SHL instructions are available.1399  if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Expand ||1400      TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Expand)1401    return SDValue();1402 1403  SDLoc DL(Node);1404  EVT OrigTy = cast<VTSDNode>(Node->getOperand(1))->getVT();1405 1406  unsigned BW = VT.getScalarSizeInBits();1407  unsigned OrigBW = OrigTy.getScalarSizeInBits();1408  SDValue ShiftSz = DAG.getConstant(BW - OrigBW, DL, VT);1409 1410  SDValue Op = DAG.getNode(ISD::SHL, DL, VT, Node->getOperand(0), ShiftSz);1411  return DAG.getNode(ISD::SRA, DL, VT, Op, ShiftSz);1412}1413 1414// Generically expand a vector anyext in register to a shuffle of the relevant1415// lanes into the appropriate locations, with other lanes left undef.1416SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node) {1417  SDLoc DL(Node);1418  EVT VT = Node->getValueType(0);1419  int NumElements = VT.getVectorNumElements();1420  SDValue Src = Node->getOperand(0);1421  EVT SrcVT = Src.getValueType();1422  int NumSrcElements = SrcVT.getVectorNumElements();1423 1424  // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector1425  // into a larger vector type.1426  if (SrcVT.bitsLE(VT)) {1427    assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&1428           "ANY_EXTEND_VECTOR_INREG vector size mismatch");1429    NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();1430    SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),1431                             NumSrcElements);1432    Src = DAG.getInsertSubvector(DL, DAG.getUNDEF(SrcVT), Src, 0);1433  }1434 1435  // Build a base mask of undef shuffles.1436  SmallVector<int, 16> ShuffleMask;1437  ShuffleMask.resize(NumSrcElements, -1);1438 1439  // Place the extended lanes into the correct locations.1440  int ExtLaneScale = NumSrcElements / NumElements;1441  int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;1442  for (int i = 0; i < NumElements; ++i)1443    ShuffleMask[i * ExtLaneScale + EndianOffset] = i;1444 1445  return DAG.getNode(1446      ISD::BITCAST, DL, VT,1447      DAG.getVectorShuffle(SrcVT, DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask));1448}1449 1450SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node) {1451  SDLoc DL(Node);1452  EVT VT = Node->getValueType(0);1453  SDValue Src = Node->getOperand(0);1454  EVT SrcVT = Src.getValueType();1455 1456  // First build an any-extend node which can be legalized above when we1457  // recurse through it.1458  SDValue Op = DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Src);1459 1460  // Now we need sign extend. Do this by shifting the elements. Even if these1461  // aren't legal operations, they have a better chance of being legalized1462  // without full scalarization than the sign extension does.1463  unsigned EltWidth = VT.getScalarSizeInBits();1464  unsigned SrcEltWidth = SrcVT.getScalarSizeInBits();1465  SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth, DL, VT);1466  return DAG.getNode(ISD::SRA, DL, VT,1467                     DAG.getNode(ISD::SHL, DL, VT, Op, ShiftAmount),1468                     ShiftAmount);1469}1470 1471// Generically expand a vector zext in register to a shuffle of the relevant1472// lanes into the appropriate locations, a blend of zero into the high bits,1473// and a bitcast to the wider element type.1474SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node) {1475  SDLoc DL(Node);1476  EVT VT = Node->getValueType(0);1477  int NumElements = VT.getVectorNumElements();1478  SDValue Src = Node->getOperand(0);1479  EVT SrcVT = Src.getValueType();1480  int NumSrcElements = SrcVT.getVectorNumElements();1481 1482  // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector1483  // into a larger vector type.1484  if (SrcVT.bitsLE(VT)) {1485    assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&1486           "ZERO_EXTEND_VECTOR_INREG vector size mismatch");1487    NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();1488    SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),1489                             NumSrcElements);1490    Src = DAG.getInsertSubvector(DL, DAG.getUNDEF(SrcVT), Src, 0);1491  }1492 1493  // Build up a zero vector to blend into this one.1494  SDValue Zero = DAG.getConstant(0, DL, SrcVT);1495 1496  // Shuffle the incoming lanes into the correct position, and pull all other1497  // lanes from the zero vector.1498  auto ShuffleMask = llvm::to_vector<16>(llvm::seq<int>(0, NumSrcElements));1499 1500  int ExtLaneScale = NumSrcElements / NumElements;1501  int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;1502  for (int i = 0; i < NumElements; ++i)1503    ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;1504 1505  return DAG.getNode(ISD::BITCAST, DL, VT,1506                     DAG.getVectorShuffle(SrcVT, DL, Zero, Src, ShuffleMask));1507}1508 1509static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl<int> &ShuffleMask) {1510  int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8;1511  for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I)1512    for (int J = ScalarSizeInBytes - 1; J >= 0; --J)1513      ShuffleMask.push_back((I * ScalarSizeInBytes) + J);1514}1515 1516SDValue VectorLegalizer::ExpandBSWAP(SDNode *Node) {1517  EVT VT = Node->getValueType(0);1518 1519  // Scalable vectors can't use shuffle expansion.1520  if (VT.isScalableVector())1521    return TLI.expandBSWAP(Node, DAG);1522 1523  // Generate a byte wise shuffle mask for the BSWAP.1524  SmallVector<int, 16> ShuffleMask;1525  createBSWAPShuffleMask(VT, ShuffleMask);1526  EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, ShuffleMask.size());1527 1528  // Only emit a shuffle if the mask is legal.1529  if (TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) {1530    SDLoc DL(Node);1531    SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0));1532    Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT), ShuffleMask);1533    return DAG.getNode(ISD::BITCAST, DL, VT, Op);1534  }1535 1536  // If we have the appropriate vector bit operations, it is better to use them1537  // than unrolling and expanding each component.1538  if (TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&1539      TLI.isOperationLegalOrCustom(ISD::SRL, VT) &&1540      TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) &&1541      TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT))1542    return TLI.expandBSWAP(Node, DAG);1543 1544  // Otherwise let the caller unroll.1545  return SDValue();1546}1547 1548SDValue VectorLegalizer::ExpandBITREVERSE(SDNode *Node) {1549  EVT VT = Node->getValueType(0);1550 1551  // We can't unroll or use shuffles for scalable vectors.1552  if (VT.isScalableVector())1553    return TLI.expandBITREVERSE(Node, DAG);1554 1555  // If we have the scalar operation, it's probably cheaper to unroll it.1556  if (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, VT.getScalarType()))1557    return SDValue();1558 1559  // If the vector element width is a whole number of bytes, test if its legal1560  // to BSWAP shuffle the bytes and then perform the BITREVERSE on the byte1561  // vector. This greatly reduces the number of bit shifts necessary.1562  unsigned ScalarSizeInBits = VT.getScalarSizeInBits();1563  if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {1564    SmallVector<int, 16> BSWAPMask;1565    createBSWAPShuffleMask(VT, BSWAPMask);1566 1567    EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, BSWAPMask.size());1568    if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&1569        (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, ByteVT) ||1570         (TLI.isOperationLegalOrCustom(ISD::SHL, ByteVT) &&1571          TLI.isOperationLegalOrCustom(ISD::SRL, ByteVT) &&1572          TLI.isOperationLegalOrCustomOrPromote(ISD::AND, ByteVT) &&1573          TLI.isOperationLegalOrCustomOrPromote(ISD::OR, ByteVT)))) {1574      SDLoc DL(Node);1575      SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0));1576      Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT),1577                                BSWAPMask);1578      Op = DAG.getNode(ISD::BITREVERSE, DL, ByteVT, Op);1579      Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);1580      return Op;1581    }1582  }1583 1584  // If we have the appropriate vector bit operations, it is better to use them1585  // than unrolling and expanding each component.1586  if (TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&1587      TLI.isOperationLegalOrCustom(ISD::SRL, VT) &&1588      TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) &&1589      TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT))1590    return TLI.expandBITREVERSE(Node, DAG);1591 1592  // Otherwise unroll.1593  return SDValue();1594}1595 1596SDValue VectorLegalizer::ExpandVSELECT(SDNode *Node) {1597  // Implement VSELECT in terms of XOR, AND, OR1598  // on platforms which do not support blend natively.1599  SDLoc DL(Node);1600 1601  SDValue Mask = Node->getOperand(0);1602  SDValue Op1 = Node->getOperand(1);1603  SDValue Op2 = Node->getOperand(2);1604 1605  EVT VT = Mask.getValueType();1606 1607  // If we can't even use the basic vector operations of1608  // AND,OR,XOR, we will have to scalarize the op.1609  // Notice that the operation may be 'promoted' which means that it is1610  // 'bitcasted' to another type which is handled.1611  if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||1612      TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||1613      TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand)1614    return SDValue();1615 1616  // This operation also isn't safe with AND, OR, XOR when the boolean type is1617  // 0/1 and the select operands aren't also booleans, as we need an all-ones1618  // vector constant to mask with.1619  // FIXME: Sign extend 1 to all ones if that's legal on the target.1620  auto BoolContents = TLI.getBooleanContents(Op1.getValueType());1621  if (BoolContents != TargetLowering::ZeroOrNegativeOneBooleanContent &&1622      !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&1623        Op1.getValueType().getVectorElementType() == MVT::i1))1624    return SDValue();1625 1626  // If the mask and the type are different sizes, unroll the vector op. This1627  // can occur when getSetCCResultType returns something that is different in1628  // size from the operand types. For example, v4i8 = select v4i32, v4i8, v4i8.1629  if (VT.getSizeInBits() != Op1.getValueSizeInBits())1630    return SDValue();1631 1632  // Bitcast the operands to be the same type as the mask.1633  // This is needed when we select between FP types because1634  // the mask is a vector of integers.1635  Op1 = DAG.getNode(ISD::BITCAST, DL, VT, Op1);1636  Op2 = DAG.getNode(ISD::BITCAST, DL, VT, Op2);1637 1638  SDValue NotMask = DAG.getNOT(DL, Mask, VT);1639 1640  Op1 = DAG.getNode(ISD::AND, DL, VT, Op1, Mask);1641  Op2 = DAG.getNode(ISD::AND, DL, VT, Op2, NotMask);1642  SDValue Val = DAG.getNode(ISD::OR, DL, VT, Op1, Op2);1643  return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val);1644}1645 1646SDValue VectorLegalizer::ExpandVP_SELECT(SDNode *Node) {1647  // Implement VP_SELECT in terms of VP_XOR, VP_AND and VP_OR on platforms which1648  // do not support it natively.1649  SDLoc DL(Node);1650 1651  SDValue Mask = Node->getOperand(0);1652  SDValue Op1 = Node->getOperand(1);1653  SDValue Op2 = Node->getOperand(2);1654  SDValue EVL = Node->getOperand(3);1655 1656  EVT VT = Mask.getValueType();1657 1658  // If we can't even use the basic vector operations of1659  // VP_AND,VP_OR,VP_XOR, we will have to scalarize the op.1660  if (TLI.getOperationAction(ISD::VP_AND, VT) == TargetLowering::Expand ||1661      TLI.getOperationAction(ISD::VP_XOR, VT) == TargetLowering::Expand ||1662      TLI.getOperationAction(ISD::VP_OR, VT) == TargetLowering::Expand)1663    return SDValue();1664 1665  // This operation also isn't safe when the operands aren't also booleans.1666  if (Op1.getValueType().getVectorElementType() != MVT::i1)1667    return SDValue();1668 1669  SDValue Ones = DAG.getAllOnesConstant(DL, VT);1670  SDValue NotMask = DAG.getNode(ISD::VP_XOR, DL, VT, Mask, Ones, Ones, EVL);1671 1672  Op1 = DAG.getNode(ISD::VP_AND, DL, VT, Op1, Mask, Ones, EVL);1673  Op2 = DAG.getNode(ISD::VP_AND, DL, VT, Op2, NotMask, Ones, EVL);1674  return DAG.getNode(ISD::VP_OR, DL, VT, Op1, Op2, Ones, EVL);1675}1676 1677SDValue VectorLegalizer::ExpandVP_MERGE(SDNode *Node) {1678  // Implement VP_MERGE in terms of VSELECT. Construct a mask where vector1679  // indices less than the EVL/pivot are true. Combine that with the original1680  // mask for a full-length mask. Use a full-length VSELECT to select between1681  // the true and false values.1682  SDLoc DL(Node);1683 1684  SDValue Mask = Node->getOperand(0);1685  SDValue Op1 = Node->getOperand(1);1686  SDValue Op2 = Node->getOperand(2);1687  SDValue EVL = Node->getOperand(3);1688 1689  EVT MaskVT = Mask.getValueType();1690  bool IsFixedLen = MaskVT.isFixedLengthVector();1691 1692  EVT EVLVecVT = EVT::getVectorVT(*DAG.getContext(), EVL.getValueType(),1693                                  MaskVT.getVectorElementCount());1694 1695  // If we can't construct the EVL mask efficiently, it's better to unroll.1696  if ((IsFixedLen &&1697       !TLI.isOperationLegalOrCustom(ISD::BUILD_VECTOR, EVLVecVT)) ||1698      (!IsFixedLen &&1699       (!TLI.isOperationLegalOrCustom(ISD::STEP_VECTOR, EVLVecVT) ||1700        !TLI.isOperationLegalOrCustom(ISD::SPLAT_VECTOR, EVLVecVT))))1701    return SDValue();1702 1703  // If using a SETCC would result in a different type than the mask type,1704  // unroll.1705  if (TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),1706                             EVLVecVT) != MaskVT)1707    return SDValue();1708 1709  SDValue StepVec = DAG.getStepVector(DL, EVLVecVT);1710  SDValue SplatEVL = DAG.getSplat(EVLVecVT, DL, EVL);1711  SDValue EVLMask =1712      DAG.getSetCC(DL, MaskVT, StepVec, SplatEVL, ISD::CondCode::SETULT);1713 1714  SDValue FullMask = DAG.getNode(ISD::AND, DL, MaskVT, Mask, EVLMask);1715  return DAG.getSelect(DL, Node->getValueType(0), FullMask, Op1, Op2);1716}1717 1718SDValue VectorLegalizer::ExpandVP_REM(SDNode *Node) {1719  // Implement VP_SREM/UREM in terms of VP_SDIV/VP_UDIV, VP_MUL, VP_SUB.1720  EVT VT = Node->getValueType(0);1721 1722  unsigned DivOpc = Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV;1723 1724  if (!TLI.isOperationLegalOrCustom(DivOpc, VT) ||1725      !TLI.isOperationLegalOrCustom(ISD::VP_MUL, VT) ||1726      !TLI.isOperationLegalOrCustom(ISD::VP_SUB, VT))1727    return SDValue();1728 1729  SDLoc DL(Node);1730 1731  SDValue Dividend = Node->getOperand(0);1732  SDValue Divisor = Node->getOperand(1);1733  SDValue Mask = Node->getOperand(2);1734  SDValue EVL = Node->getOperand(3);1735 1736  // X % Y -> X-X/Y*Y1737  SDValue Div = DAG.getNode(DivOpc, DL, VT, Dividend, Divisor, Mask, EVL);1738  SDValue Mul = DAG.getNode(ISD::VP_MUL, DL, VT, Divisor, Div, Mask, EVL);1739  return DAG.getNode(ISD::VP_SUB, DL, VT, Dividend, Mul, Mask, EVL);1740}1741 1742SDValue VectorLegalizer::ExpandVP_FNEG(SDNode *Node) {1743  EVT VT = Node->getValueType(0);1744  EVT IntVT = VT.changeVectorElementTypeToInteger();1745 1746  if (!TLI.isOperationLegalOrCustom(ISD::VP_XOR, IntVT))1747    return SDValue();1748 1749  SDValue Mask = Node->getOperand(1);1750  SDValue EVL = Node->getOperand(2);1751 1752  SDLoc DL(Node);1753  SDValue Cast = DAG.getNode(ISD::BITCAST, DL, IntVT, Node->getOperand(0));1754  SDValue SignMask = DAG.getConstant(1755      APInt::getSignMask(IntVT.getScalarSizeInBits()), DL, IntVT);1756  SDValue Xor = DAG.getNode(ISD::VP_XOR, DL, IntVT, Cast, SignMask, Mask, EVL);1757  return DAG.getNode(ISD::BITCAST, DL, VT, Xor);1758}1759 1760SDValue VectorLegalizer::ExpandVP_FABS(SDNode *Node) {1761  EVT VT = Node->getValueType(0);1762  EVT IntVT = VT.changeVectorElementTypeToInteger();1763 1764  if (!TLI.isOperationLegalOrCustom(ISD::VP_AND, IntVT))1765    return SDValue();1766 1767  SDValue Mask = Node->getOperand(1);1768  SDValue EVL = Node->getOperand(2);1769 1770  SDLoc DL(Node);1771  SDValue Cast = DAG.getNode(ISD::BITCAST, DL, IntVT, Node->getOperand(0));1772  SDValue ClearSignMask = DAG.getConstant(1773      APInt::getSignedMaxValue(IntVT.getScalarSizeInBits()), DL, IntVT);1774  SDValue ClearSign =1775      DAG.getNode(ISD::VP_AND, DL, IntVT, Cast, ClearSignMask, Mask, EVL);1776  return DAG.getNode(ISD::BITCAST, DL, VT, ClearSign);1777}1778 1779SDValue VectorLegalizer::ExpandVP_FCOPYSIGN(SDNode *Node) {1780  EVT VT = Node->getValueType(0);1781 1782  if (VT != Node->getOperand(1).getValueType())1783    return SDValue();1784 1785  EVT IntVT = VT.changeVectorElementTypeToInteger();1786  if (!TLI.isOperationLegalOrCustom(ISD::VP_AND, IntVT) ||1787      !TLI.isOperationLegalOrCustom(ISD::VP_XOR, IntVT))1788    return SDValue();1789 1790  SDValue Mask = Node->getOperand(2);1791  SDValue EVL = Node->getOperand(3);1792 1793  SDLoc DL(Node);1794  SDValue Mag = DAG.getNode(ISD::BITCAST, DL, IntVT, Node->getOperand(0));1795  SDValue Sign = DAG.getNode(ISD::BITCAST, DL, IntVT, Node->getOperand(1));1796 1797  SDValue SignMask = DAG.getConstant(1798      APInt::getSignMask(IntVT.getScalarSizeInBits()), DL, IntVT);1799  SDValue SignBit =1800      DAG.getNode(ISD::VP_AND, DL, IntVT, Sign, SignMask, Mask, EVL);1801 1802  SDValue ClearSignMask = DAG.getConstant(1803      APInt::getSignedMaxValue(IntVT.getScalarSizeInBits()), DL, IntVT);1804  SDValue ClearedSign =1805      DAG.getNode(ISD::VP_AND, DL, IntVT, Mag, ClearSignMask, Mask, EVL);1806 1807  SDValue CopiedSign = DAG.getNode(ISD::VP_OR, DL, IntVT, ClearedSign, SignBit,1808                                   Mask, EVL, SDNodeFlags::Disjoint);1809 1810  return DAG.getNode(ISD::BITCAST, DL, VT, CopiedSign);1811}1812 1813SDValue VectorLegalizer::ExpandLOOP_DEPENDENCE_MASK(SDNode *N) {1814  SDLoc DL(N);1815  SDValue SourceValue = N->getOperand(0);1816  SDValue SinkValue = N->getOperand(1);1817  SDValue EltSize = N->getOperand(2);1818 1819  bool IsReadAfterWrite = N->getOpcode() == ISD::LOOP_DEPENDENCE_RAW_MASK;1820  EVT VT = N->getValueType(0);1821  EVT PtrVT = SourceValue->getValueType(0);1822 1823  SDValue Diff = DAG.getNode(ISD::SUB, DL, PtrVT, SinkValue, SourceValue);1824  if (IsReadAfterWrite)1825    Diff = DAG.getNode(ISD::ABS, DL, PtrVT, Diff);1826 1827  Diff = DAG.getNode(ISD::SDIV, DL, PtrVT, Diff, EltSize);1828 1829  // If the difference is positive then some elements may alias1830  EVT CmpVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),1831                                     Diff.getValueType());1832  SDValue Zero = DAG.getConstant(0, DL, PtrVT);1833  SDValue Cmp = DAG.getSetCC(DL, CmpVT, Diff, Zero,1834                             IsReadAfterWrite ? ISD::SETEQ : ISD::SETLE);1835 1836  // Create the lane mask1837  EVT SplatVT = VT.changeElementType(PtrVT);1838  SDValue DiffSplat = DAG.getSplat(SplatVT, DL, Diff);1839  SDValue VectorStep = DAG.getStepVector(DL, SplatVT);1840  EVT MaskVT = VT.changeElementType(MVT::i1);1841  SDValue DiffMask =1842      DAG.getSetCC(DL, MaskVT, VectorStep, DiffSplat, ISD::CondCode::SETULT);1843 1844  EVT EltVT = VT.getVectorElementType();1845  // Extend the diff setcc in case the intrinsic has been promoted to a vector1846  // type with elements larger than i11847  if (EltVT.getScalarSizeInBits() > MaskVT.getScalarSizeInBits())1848    DiffMask = DAG.getNode(ISD::ANY_EXTEND, DL, VT, DiffMask);1849 1850  // Splat the compare result then OR it with the lane mask1851  if (CmpVT.getScalarSizeInBits() < EltVT.getScalarSizeInBits())1852    Cmp = DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Cmp);1853  SDValue Splat = DAG.getSplat(VT, DL, Cmp);1854  return DAG.getNode(ISD::OR, DL, VT, DiffMask, Splat);1855}1856 1857void VectorLegalizer::ExpandFP_TO_UINT(SDNode *Node,1858                                       SmallVectorImpl<SDValue> &Results) {1859  // Attempt to expand using TargetLowering.1860  SDValue Result, Chain;1861  if (TLI.expandFP_TO_UINT(Node, Result, Chain, DAG)) {1862    Results.push_back(Result);1863    if (Node->isStrictFPOpcode())1864      Results.push_back(Chain);1865    return;1866  }1867 1868  // Otherwise go ahead and unroll.1869  if (Node->isStrictFPOpcode()) {1870    UnrollStrictFPOp(Node, Results);1871    return;1872  }1873 1874  Results.push_back(DAG.UnrollVectorOp(Node));1875}1876 1877void VectorLegalizer::ExpandUINT_TO_FLOAT(SDNode *Node,1878                                          SmallVectorImpl<SDValue> &Results) {1879  bool IsStrict = Node->isStrictFPOpcode();1880  unsigned OpNo = IsStrict ? 1 : 0;1881  SDValue Src = Node->getOperand(OpNo);1882  EVT SrcVT = Src.getValueType();1883  EVT DstVT = Node->getValueType(0);1884  SDLoc DL(Node);1885 1886  // Attempt to expand using TargetLowering.1887  SDValue Result;1888  SDValue Chain;1889  if (TLI.expandUINT_TO_FP(Node, Result, Chain, DAG)) {1890    Results.push_back(Result);1891    if (IsStrict)1892      Results.push_back(Chain);1893    return;1894  }1895 1896  // Make sure that the SINT_TO_FP and SRL instructions are available.1897  if (((!IsStrict && TLI.getOperationAction(ISD::SINT_TO_FP, SrcVT) ==1898                         TargetLowering::Expand) ||1899       (IsStrict && TLI.getOperationAction(ISD::STRICT_SINT_TO_FP, SrcVT) ==1900                        TargetLowering::Expand)) ||1901      TLI.getOperationAction(ISD::SRL, SrcVT) == TargetLowering::Expand) {1902    if (IsStrict) {1903      UnrollStrictFPOp(Node, Results);1904      return;1905    }1906 1907    Results.push_back(DAG.UnrollVectorOp(Node));1908    return;1909  }1910 1911  unsigned BW = SrcVT.getScalarSizeInBits();1912  assert((BW == 64 || BW == 32) &&1913         "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");1914 1915  // If STRICT_/FMUL is not supported by the target (in case of f16) replace the1916  // UINT_TO_FP with a larger float and round to the smaller type1917  if ((!IsStrict && !TLI.isOperationLegalOrCustom(ISD::FMUL, DstVT)) ||1918      (IsStrict && !TLI.isOperationLegalOrCustom(ISD::STRICT_FMUL, DstVT))) {1919    EVT FPVT = BW == 32 ? MVT::f32 : MVT::f64;1920    SDValue UIToFP;1921    SDValue Result;1922    SDValue TargetZero = DAG.getIntPtrConstant(0, DL, /*isTarget=*/true);1923    EVT FloatVecVT = SrcVT.changeVectorElementType(FPVT);1924    if (IsStrict) {1925      UIToFP = DAG.getNode(ISD::STRICT_UINT_TO_FP, DL, {FloatVecVT, MVT::Other},1926                           {Node->getOperand(0), Src});1927      Result = DAG.getNode(ISD::STRICT_FP_ROUND, DL, {DstVT, MVT::Other},1928                           {Node->getOperand(0), UIToFP, TargetZero});1929      Results.push_back(Result);1930      Results.push_back(Result.getValue(1));1931    } else {1932      UIToFP = DAG.getNode(ISD::UINT_TO_FP, DL, FloatVecVT, Src);1933      Result = DAG.getNode(ISD::FP_ROUND, DL, DstVT, UIToFP, TargetZero);1934      Results.push_back(Result);1935    }1936 1937    return;1938  }1939 1940  SDValue HalfWord = DAG.getConstant(BW / 2, DL, SrcVT);1941 1942  // Constants to clear the upper part of the word.1943  // Notice that we can also use SHL+SHR, but using a constant is slightly1944  // faster on x86.1945  uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;1946  SDValue HalfWordMask = DAG.getConstant(HWMask, DL, SrcVT);1947 1948  // Two to the power of half-word-size.1949  SDValue TWOHW = DAG.getConstantFP(1ULL << (BW / 2), DL, DstVT);1950 1951  // Clear upper part of LO, lower HI1952  SDValue HI = DAG.getNode(ISD::SRL, DL, SrcVT, Src, HalfWord);1953  SDValue LO = DAG.getNode(ISD::AND, DL, SrcVT, Src, HalfWordMask);1954 1955  if (IsStrict) {1956    // Convert hi and lo to floats1957    // Convert the hi part back to the upper values1958    // TODO: Can any fast-math-flags be set on these nodes?1959    SDValue fHI = DAG.getNode(ISD::STRICT_SINT_TO_FP, DL, {DstVT, MVT::Other},1960                              {Node->getOperand(0), HI});1961    fHI = DAG.getNode(ISD::STRICT_FMUL, DL, {DstVT, MVT::Other},1962                      {fHI.getValue(1), fHI, TWOHW});1963    SDValue fLO = DAG.getNode(ISD::STRICT_SINT_TO_FP, DL, {DstVT, MVT::Other},1964                              {Node->getOperand(0), LO});1965 1966    SDValue TF = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, fHI.getValue(1),1967                             fLO.getValue(1));1968 1969    // Add the two halves1970    SDValue Result =1971        DAG.getNode(ISD::STRICT_FADD, DL, {DstVT, MVT::Other}, {TF, fHI, fLO});1972 1973    Results.push_back(Result);1974    Results.push_back(Result.getValue(1));1975    return;1976  }1977 1978  // Convert hi and lo to floats1979  // Convert the hi part back to the upper values1980  // TODO: Can any fast-math-flags be set on these nodes?1981  SDValue fHI = DAG.getNode(ISD::SINT_TO_FP, DL, DstVT, HI);1982  fHI = DAG.getNode(ISD::FMUL, DL, DstVT, fHI, TWOHW);1983  SDValue fLO = DAG.getNode(ISD::SINT_TO_FP, DL, DstVT, LO);1984 1985  // Add the two halves1986  Results.push_back(DAG.getNode(ISD::FADD, DL, DstVT, fHI, fLO));1987}1988 1989SDValue VectorLegalizer::ExpandFNEG(SDNode *Node) {1990  EVT VT = Node->getValueType(0);1991  EVT IntVT = VT.changeVectorElementTypeToInteger();1992 1993  if (!TLI.isOperationLegalOrCustom(ISD::XOR, IntVT))1994    return SDValue();1995 1996  // FIXME: The FSUB check is here to force unrolling v1f64 vectors on AArch64.1997  if (!TLI.isOperationLegalOrCustomOrPromote(ISD::FSUB, VT) &&1998      !VT.isScalableVector())1999    return SDValue();2000 2001  SDLoc DL(Node);2002  SDValue Cast = DAG.getNode(ISD::BITCAST, DL, IntVT, Node->getOperand(0));2003  SDValue SignMask = DAG.getConstant(2004      APInt::getSignMask(IntVT.getScalarSizeInBits()), DL, IntVT);2005  SDValue Xor = DAG.getNode(ISD::XOR, DL, IntVT, Cast, SignMask);2006  return DAG.getNode(ISD::BITCAST, DL, VT, Xor);2007}2008 2009SDValue VectorLegalizer::ExpandFABS(SDNode *Node) {2010  EVT VT = Node->getValueType(0);2011  EVT IntVT = VT.changeVectorElementTypeToInteger();2012 2013  if (!TLI.isOperationLegalOrCustom(ISD::AND, IntVT))2014    return SDValue();2015 2016  // FIXME: The FSUB check is here to force unrolling v1f64 vectors on AArch64.2017  if (!TLI.isOperationLegalOrCustomOrPromote(ISD::FSUB, VT) &&2018      !VT.isScalableVector())2019    return SDValue();2020 2021  SDLoc DL(Node);2022  SDValue Cast = DAG.getNode(ISD::BITCAST, DL, IntVT, Node->getOperand(0));2023  SDValue ClearSignMask = DAG.getConstant(2024      APInt::getSignedMaxValue(IntVT.getScalarSizeInBits()), DL, IntVT);2025  SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, Cast, ClearSignMask);2026  return DAG.getNode(ISD::BITCAST, DL, VT, ClearedSign);2027}2028 2029SDValue VectorLegalizer::ExpandFCOPYSIGN(SDNode *Node) {2030  EVT VT = Node->getValueType(0);2031  EVT IntVT = VT.changeVectorElementTypeToInteger();2032 2033  if (VT != Node->getOperand(1).getValueType() ||2034      !TLI.isOperationLegalOrCustom(ISD::AND, IntVT) ||2035      !TLI.isOperationLegalOrCustom(ISD::OR, IntVT))2036    return SDValue();2037 2038  // FIXME: The FSUB check is here to force unrolling v1f64 vectors on AArch64.2039  if (!TLI.isOperationLegalOrCustomOrPromote(ISD::FSUB, VT) &&2040      !VT.isScalableVector())2041    return SDValue();2042 2043  SDLoc DL(Node);2044  SDValue Mag = DAG.getNode(ISD::BITCAST, DL, IntVT, Node->getOperand(0));2045  SDValue Sign = DAG.getNode(ISD::BITCAST, DL, IntVT, Node->getOperand(1));2046 2047  SDValue SignMask = DAG.getConstant(2048      APInt::getSignMask(IntVT.getScalarSizeInBits()), DL, IntVT);2049  SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, Sign, SignMask);2050 2051  SDValue ClearSignMask = DAG.getConstant(2052      APInt::getSignedMaxValue(IntVT.getScalarSizeInBits()), DL, IntVT);2053  SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, Mag, ClearSignMask);2054 2055  SDValue CopiedSign = DAG.getNode(ISD::OR, DL, IntVT, ClearedSign, SignBit,2056                                   SDNodeFlags::Disjoint);2057 2058  return DAG.getNode(ISD::BITCAST, DL, VT, CopiedSign);2059}2060 2061void VectorLegalizer::ExpandFSUB(SDNode *Node,2062                                 SmallVectorImpl<SDValue> &Results) {2063  // For floating-point values, (a-b) is the same as a+(-b). If FNEG is legal,2064  // we can defer this to operation legalization where it will be lowered as2065  // a+(-b).2066  EVT VT = Node->getValueType(0);2067  if (TLI.isOperationLegalOrCustom(ISD::FNEG, VT) &&2068      TLI.isOperationLegalOrCustom(ISD::FADD, VT))2069    return; // Defer to LegalizeDAG2070 2071  if (SDValue Expanded = TLI.expandVectorNaryOpBySplitting(Node, DAG)) {2072    Results.push_back(Expanded);2073    return;2074  }2075 2076  SDValue Tmp = DAG.UnrollVectorOp(Node);2077  Results.push_back(Tmp);2078}2079 2080void VectorLegalizer::ExpandSETCC(SDNode *Node,2081                                  SmallVectorImpl<SDValue> &Results) {2082  bool NeedInvert = false;2083  bool IsVP = Node->getOpcode() == ISD::VP_SETCC;2084  bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||2085                  Node->getOpcode() == ISD::STRICT_FSETCCS;2086  bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;2087  unsigned Offset = IsStrict ? 1 : 0;2088 2089  SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();2090  SDValue LHS = Node->getOperand(0 + Offset);2091  SDValue RHS = Node->getOperand(1 + Offset);2092  SDValue CC = Node->getOperand(2 + Offset);2093 2094  MVT OpVT = LHS.getSimpleValueType();2095  ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();2096 2097  if (TLI.getCondCodeAction(CCCode, OpVT) != TargetLowering::Expand) {2098    if (IsStrict) {2099      UnrollStrictFPOp(Node, Results);2100      return;2101    }2102    Results.push_back(UnrollVSETCC(Node));2103    return;2104  }2105 2106  SDValue Mask, EVL;2107  if (IsVP) {2108    Mask = Node->getOperand(3 + Offset);2109    EVL = Node->getOperand(4 + Offset);2110  }2111 2112  SDLoc dl(Node);2113  bool Legalized =2114      TLI.LegalizeSetCCCondCode(DAG, Node->getValueType(0), LHS, RHS, CC, Mask,2115                                EVL, NeedInvert, dl, Chain, IsSignaling);2116 2117  if (Legalized) {2118    // If we expanded the SETCC by swapping LHS and RHS, or by inverting the2119    // condition code, create a new SETCC node.2120    if (CC.getNode()) {2121      if (IsStrict) {2122        LHS = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(),2123                          {Chain, LHS, RHS, CC}, Node->getFlags());2124        Chain = LHS.getValue(1);2125      } else if (IsVP) {2126        LHS = DAG.getNode(ISD::VP_SETCC, dl, Node->getValueType(0),2127                          {LHS, RHS, CC, Mask, EVL}, Node->getFlags());2128      } else {2129        LHS = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), LHS, RHS, CC,2130                          Node->getFlags());2131      }2132    }2133 2134    // If we expanded the SETCC by inverting the condition code, then wrap2135    // the existing SETCC in a NOT to restore the intended condition.2136    if (NeedInvert) {2137      if (!IsVP)2138        LHS = DAG.getLogicalNOT(dl, LHS, LHS->getValueType(0));2139      else2140        LHS = DAG.getVPLogicalNOT(dl, LHS, Mask, EVL, LHS->getValueType(0));2141    }2142  } else {2143    assert(!IsStrict && "Don't know how to expand for strict nodes.");2144 2145    // Otherwise, SETCC for the given comparison type must be completely2146    // illegal; expand it into a SELECT_CC.2147    EVT VT = Node->getValueType(0);2148    LHS = DAG.getNode(ISD::SELECT_CC, dl, VT, LHS, RHS,2149                      DAG.getBoolConstant(true, dl, VT, LHS.getValueType()),2150                      DAG.getBoolConstant(false, dl, VT, LHS.getValueType()),2151                      CC, Node->getFlags());2152  }2153 2154  Results.push_back(LHS);2155  if (IsStrict)2156    Results.push_back(Chain);2157}2158 2159void VectorLegalizer::ExpandUADDSUBO(SDNode *Node,2160                                     SmallVectorImpl<SDValue> &Results) {2161  SDValue Result, Overflow;2162  TLI.expandUADDSUBO(Node, Result, Overflow, DAG);2163  Results.push_back(Result);2164  Results.push_back(Overflow);2165}2166 2167void VectorLegalizer::ExpandSADDSUBO(SDNode *Node,2168                                     SmallVectorImpl<SDValue> &Results) {2169  SDValue Result, Overflow;2170  TLI.expandSADDSUBO(Node, Result, Overflow, DAG);2171  Results.push_back(Result);2172  Results.push_back(Overflow);2173}2174 2175void VectorLegalizer::ExpandMULO(SDNode *Node,2176                                 SmallVectorImpl<SDValue> &Results) {2177  SDValue Result, Overflow;2178  if (!TLI.expandMULO(Node, Result, Overflow, DAG))2179    std::tie(Result, Overflow) = DAG.UnrollVectorOverflowOp(Node);2180 2181  Results.push_back(Result);2182  Results.push_back(Overflow);2183}2184 2185void VectorLegalizer::ExpandFixedPointDiv(SDNode *Node,2186                                          SmallVectorImpl<SDValue> &Results) {2187  SDNode *N = Node;2188  if (SDValue Expanded = TLI.expandFixedPointDiv(N->getOpcode(), SDLoc(N),2189          N->getOperand(0), N->getOperand(1), N->getConstantOperandVal(2), DAG))2190    Results.push_back(Expanded);2191}2192 2193void VectorLegalizer::ExpandStrictFPOp(SDNode *Node,2194                                       SmallVectorImpl<SDValue> &Results) {2195  if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP) {2196    ExpandUINT_TO_FLOAT(Node, Results);2197    return;2198  }2199  if (Node->getOpcode() == ISD::STRICT_FP_TO_UINT) {2200    ExpandFP_TO_UINT(Node, Results);2201    return;2202  }2203 2204  if (Node->getOpcode() == ISD::STRICT_FSETCC ||2205      Node->getOpcode() == ISD::STRICT_FSETCCS) {2206    ExpandSETCC(Node, Results);2207    return;2208  }2209 2210  UnrollStrictFPOp(Node, Results);2211}2212 2213void VectorLegalizer::ExpandREM(SDNode *Node,2214                                SmallVectorImpl<SDValue> &Results) {2215  assert((Node->getOpcode() == ISD::SREM || Node->getOpcode() == ISD::UREM) &&2216         "Expected REM node");2217 2218  SDValue Result;2219  if (!TLI.expandREM(Node, Result, DAG))2220    Result = DAG.UnrollVectorOp(Node);2221  Results.push_back(Result);2222}2223 2224// Try to expand libm nodes into vector math routine calls. Callers provide the2225// LibFunc equivalent of the passed in Node, which is used to lookup mappings2226// within TargetLibraryInfo. The only mappings considered are those where the2227// result and all operands are the same vector type. While predicated nodes are2228// not supported, we will emit calls to masked routines by passing in an all2229// true mask.2230bool VectorLegalizer::tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall LC,2231                                           SmallVectorImpl<SDValue> &Results) {2232  // Chain must be propagated but currently strict fp operations are down2233  // converted to their none strict counterpart.2234  assert(!Node->isStrictFPOpcode() && "Unexpected strict fp operation!");2235 2236  const char *LCName = TLI.getLibcallName(LC);2237  if (!LCName)2238    return false;2239  LLVM_DEBUG(dbgs() << "Looking for vector variant of " << LCName << "\n");2240 2241  EVT VT = Node->getValueType(0);2242  ElementCount VL = VT.getVectorElementCount();2243 2244  // Lookup a vector function equivalent to the specified libcall. Prefer2245  // unmasked variants but we will generate a mask if need be.2246  const TargetLibraryInfo &TLibInfo = DAG.getLibInfo();2247  const VecDesc *VD = TLibInfo.getVectorMappingInfo(LCName, VL, false);2248  if (!VD)2249    VD = TLibInfo.getVectorMappingInfo(LCName, VL, /*Masked=*/true);2250  if (!VD)2251    return false;2252 2253  LLVMContext *Ctx = DAG.getContext();2254  Type *Ty = VT.getTypeForEVT(*Ctx);2255  Type *ScalarTy = Ty->getScalarType();2256 2257  // Construct a scalar function type based on Node's operands.2258  SmallVector<Type *, 8> ArgTys;2259  for (unsigned i = 0; i < Node->getNumOperands(); ++i) {2260    assert(Node->getOperand(i).getValueType() == VT &&2261           "Expected matching vector types!");2262    ArgTys.push_back(ScalarTy);2263  }2264  FunctionType *ScalarFTy = FunctionType::get(ScalarTy, ArgTys, false);2265 2266  // Generate call information for the vector function.2267  const std::string MangledName = VD->getVectorFunctionABIVariantString();2268  auto OptVFInfo = VFABI::tryDemangleForVFABI(MangledName, ScalarFTy);2269  if (!OptVFInfo)2270    return false;2271 2272  LLVM_DEBUG(dbgs() << "Found vector variant " << VD->getVectorFnName()2273                    << "\n");2274 2275  // Sanity check just in case OptVFInfo has unexpected parameters.2276  if (OptVFInfo->Shape.Parameters.size() !=2277      Node->getNumOperands() + VD->isMasked())2278    return false;2279 2280  // Collect vector call operands.2281 2282  SDLoc DL(Node);2283  TargetLowering::ArgListTy Args;2284 2285  unsigned OpNum = 0;2286  for (auto &VFParam : OptVFInfo->Shape.Parameters) {2287    if (VFParam.ParamKind == VFParamKind::GlobalPredicate) {2288      EVT MaskVT = TLI.getSetCCResultType(DAG.getDataLayout(), *Ctx, VT);2289      Args.emplace_back(DAG.getBoolConstant(true, DL, MaskVT, VT),2290                        MaskVT.getTypeForEVT(*Ctx));2291      continue;2292    }2293 2294    // Only vector operands are supported.2295    if (VFParam.ParamKind != VFParamKind::Vector)2296      return false;2297 2298    Args.emplace_back(Node->getOperand(OpNum++), Ty);2299  }2300 2301  // Emit a call to the vector function.2302  SDValue Callee = DAG.getExternalSymbol(VD->getVectorFnName().data(),2303                                         TLI.getPointerTy(DAG.getDataLayout()));2304  TargetLowering::CallLoweringInfo CLI(DAG);2305  CLI.setDebugLoc(DL)2306      .setChain(DAG.getEntryNode())2307      .setLibCallee(CallingConv::C, Ty, Callee, std::move(Args));2308 2309  std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);2310  Results.push_back(CallResult.first);2311  return true;2312}2313 2314/// Try to expand the node to a vector libcall based on the result type.2315bool VectorLegalizer::tryExpandVecMathCall(2316    SDNode *Node, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,2317    RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,2318    RTLIB::Libcall Call_PPCF128, SmallVectorImpl<SDValue> &Results) {2319  RTLIB::Libcall LC = RTLIB::getFPLibCall(2320      Node->getValueType(0).getVectorElementType(), Call_F32, Call_F64,2321      Call_F80, Call_F128, Call_PPCF128);2322 2323  if (LC == RTLIB::UNKNOWN_LIBCALL)2324    return false;2325 2326  return tryExpandVecMathCall(Node, LC, Results);2327}2328 2329void VectorLegalizer::UnrollStrictFPOp(SDNode *Node,2330                                       SmallVectorImpl<SDValue> &Results) {2331  EVT VT = Node->getValueType(0);2332  EVT EltVT = VT.getVectorElementType();2333  unsigned NumElems = VT.getVectorNumElements();2334  unsigned NumOpers = Node->getNumOperands();2335  const TargetLowering &TLI = DAG.getTargetLoweringInfo();2336 2337  EVT TmpEltVT = EltVT;2338  if (Node->getOpcode() == ISD::STRICT_FSETCC ||2339      Node->getOpcode() == ISD::STRICT_FSETCCS)2340    TmpEltVT = TLI.getSetCCResultType(DAG.getDataLayout(),2341                                      *DAG.getContext(), TmpEltVT);2342 2343  EVT ValueVTs[] = {TmpEltVT, MVT::Other};2344  SDValue Chain = Node->getOperand(0);2345  SDLoc dl(Node);2346 2347  SmallVector<SDValue, 32> OpValues;2348  SmallVector<SDValue, 32> OpChains;2349  for (unsigned i = 0; i < NumElems; ++i) {2350    SmallVector<SDValue, 4> Opers;2351    SDValue Idx = DAG.getVectorIdxConstant(i, dl);2352 2353    // The Chain is the first operand.2354    Opers.push_back(Chain);2355 2356    // Now process the remaining operands.2357    for (unsigned j = 1; j < NumOpers; ++j) {2358      SDValue Oper = Node->getOperand(j);2359      EVT OperVT = Oper.getValueType();2360 2361      if (OperVT.isVector())2362        Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,2363                           OperVT.getVectorElementType(), Oper, Idx);2364 2365      Opers.push_back(Oper);2366    }2367 2368    SDValue ScalarOp = DAG.getNode(Node->getOpcode(), dl, ValueVTs, Opers);2369    SDValue ScalarResult = ScalarOp.getValue(0);2370    SDValue ScalarChain = ScalarOp.getValue(1);2371 2372    if (Node->getOpcode() == ISD::STRICT_FSETCC ||2373        Node->getOpcode() == ISD::STRICT_FSETCCS)2374      ScalarResult = DAG.getSelect(dl, EltVT, ScalarResult,2375                                   DAG.getAllOnesConstant(dl, EltVT),2376                                   DAG.getConstant(0, dl, EltVT));2377 2378    OpValues.push_back(ScalarResult);2379    OpChains.push_back(ScalarChain);2380  }2381 2382  SDValue Result = DAG.getBuildVector(VT, dl, OpValues);2383  SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);2384 2385  Results.push_back(Result);2386  Results.push_back(NewChain);2387}2388 2389SDValue VectorLegalizer::UnrollVSETCC(SDNode *Node) {2390  EVT VT = Node->getValueType(0);2391  unsigned NumElems = VT.getVectorNumElements();2392  EVT EltVT = VT.getVectorElementType();2393  SDValue LHS = Node->getOperand(0);2394  SDValue RHS = Node->getOperand(1);2395  SDValue CC = Node->getOperand(2);2396  EVT TmpEltVT = LHS.getValueType().getVectorElementType();2397  SDLoc dl(Node);2398  SmallVector<SDValue, 8> Ops(NumElems);2399  for (unsigned i = 0; i < NumElems; ++i) {2400    SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,2401                                  DAG.getVectorIdxConstant(i, dl));2402    SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,2403                                  DAG.getVectorIdxConstant(i, dl));2404    // FIXME: We should use i1 setcc + boolext here, but it causes regressions.2405    Ops[i] = DAG.getNode(ISD::SETCC, dl,2406                         TLI.getSetCCResultType(DAG.getDataLayout(),2407                                                *DAG.getContext(), TmpEltVT),2408                         LHSElem, RHSElem, CC);2409    Ops[i] = DAG.getSelect(dl, EltVT, Ops[i],2410                           DAG.getBoolConstant(true, dl, EltVT, VT),2411                           DAG.getConstant(0, dl, EltVT));2412  }2413  return DAG.getBuildVector(VT, dl, Ops);2414}2415 2416bool SelectionDAG::LegalizeVectors() {2417  return VectorLegalizer(*this).Run();2418}2419