14333 lines · cpp
1//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This implements the SelectionDAG class.10//11//===----------------------------------------------------------------------===//12 13#include "llvm/CodeGen/SelectionDAG.h"14#include "SDNodeDbgValue.h"15#include "llvm/ADT/APFloat.h"16#include "llvm/ADT/APInt.h"17#include "llvm/ADT/APSInt.h"18#include "llvm/ADT/ArrayRef.h"19#include "llvm/ADT/BitVector.h"20#include "llvm/ADT/DenseSet.h"21#include "llvm/ADT/FoldingSet.h"22#include "llvm/ADT/STLExtras.h"23#include "llvm/ADT/SmallPtrSet.h"24#include "llvm/ADT/SmallVector.h"25#include "llvm/ADT/Twine.h"26#include "llvm/Analysis/AliasAnalysis.h"27#include "llvm/Analysis/MemoryLocation.h"28#include "llvm/Analysis/TargetLibraryInfo.h"29#include "llvm/Analysis/ValueTracking.h"30#include "llvm/Analysis/VectorUtils.h"31#include "llvm/BinaryFormat/Dwarf.h"32#include "llvm/CodeGen/Analysis.h"33#include "llvm/CodeGen/FunctionLoweringInfo.h"34#include "llvm/CodeGen/ISDOpcodes.h"35#include "llvm/CodeGen/MachineBasicBlock.h"36#include "llvm/CodeGen/MachineConstantPool.h"37#include "llvm/CodeGen/MachineFrameInfo.h"38#include "llvm/CodeGen/MachineFunction.h"39#include "llvm/CodeGen/MachineMemOperand.h"40#include "llvm/CodeGen/RuntimeLibcallUtil.h"41#include "llvm/CodeGen/SDPatternMatch.h"42#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"43#include "llvm/CodeGen/SelectionDAGNodes.h"44#include "llvm/CodeGen/SelectionDAGTargetInfo.h"45#include "llvm/CodeGen/TargetFrameLowering.h"46#include "llvm/CodeGen/TargetLowering.h"47#include "llvm/CodeGen/TargetRegisterInfo.h"48#include "llvm/CodeGen/TargetSubtargetInfo.h"49#include "llvm/CodeGen/ValueTypes.h"50#include "llvm/CodeGenTypes/MachineValueType.h"51#include "llvm/IR/Constant.h"52#include "llvm/IR/Constants.h"53#include "llvm/IR/DataLayout.h"54#include "llvm/IR/DebugInfoMetadata.h"55#include "llvm/IR/DebugLoc.h"56#include "llvm/IR/DerivedTypes.h"57#include "llvm/IR/Function.h"58#include "llvm/IR/GlobalValue.h"59#include "llvm/IR/Metadata.h"60#include "llvm/IR/Type.h"61#include "llvm/Support/Casting.h"62#include "llvm/Support/CodeGen.h"63#include "llvm/Support/Compiler.h"64#include "llvm/Support/Debug.h"65#include "llvm/Support/ErrorHandling.h"66#include "llvm/Support/KnownBits.h"67#include "llvm/Support/MathExtras.h"68#include "llvm/Support/raw_ostream.h"69#include "llvm/Target/TargetMachine.h"70#include "llvm/Target/TargetOptions.h"71#include "llvm/TargetParser/Triple.h"72#include "llvm/Transforms/Utils/SizeOpts.h"73#include <algorithm>74#include <cassert>75#include <cstdint>76#include <cstdlib>77#include <limits>78#include <optional>79#include <string>80#include <utility>81#include <vector>82 83using namespace llvm;84using namespace llvm::SDPatternMatch;85 86/// makeVTList - Return an instance of the SDVTList struct initialized with the87/// specified members.88static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {89 SDVTList Res = {VTs, NumVTs};90 return Res;91}92 93// Default null implementations of the callbacks.94void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}95void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}96void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {}97 98void SelectionDAG::DAGNodeDeletedListener::anchor() {}99void SelectionDAG::DAGNodeInsertedListener::anchor() {}100 101#define DEBUG_TYPE "selectiondag"102 103static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",104 cl::Hidden, cl::init(true),105 cl::desc("Gang up loads and stores generated by inlining of memcpy"));106 107static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",108 cl::desc("Number limit for gluing ld/st of memcpy."),109 cl::Hidden, cl::init(0));110 111static cl::opt<unsigned>112 MaxSteps("has-predecessor-max-steps", cl::Hidden, cl::init(8192),113 cl::desc("DAG combiner limit number of steps when searching DAG "114 "for predecessor nodes"));115 116static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {117 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););118}119 120unsigned SelectionDAG::getHasPredecessorMaxSteps() { return MaxSteps; }121 122//===----------------------------------------------------------------------===//123// ConstantFPSDNode Class124//===----------------------------------------------------------------------===//125 126/// isExactlyValue - We don't rely on operator== working on double values, as127/// it returns true for things that are clearly not equal, like -0.0 and 0.0.128/// As such, this method can be used to do an exact bit-for-bit comparison of129/// two floating point values.130bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {131 return getValueAPF().bitwiseIsEqual(V);132}133 134bool ConstantFPSDNode::isValueValidForType(EVT VT,135 const APFloat& Val) {136 assert(VT.isFloatingPoint() && "Can only convert between FP types");137 138 // convert modifies in place, so make a copy.139 APFloat Val2 = APFloat(Val);140 bool losesInfo;141 (void)Val2.convert(VT.getFltSemantics(), APFloat::rmNearestTiesToEven,142 &losesInfo);143 return !losesInfo;144}145 146//===----------------------------------------------------------------------===//147// ISD Namespace148//===----------------------------------------------------------------------===//149 150bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {151 if (N->getOpcode() == ISD::SPLAT_VECTOR) {152 if (auto OptAPInt = N->getOperand(0)->bitcastToAPInt()) {153 unsigned EltSize =154 N->getValueType(0).getVectorElementType().getSizeInBits();155 SplatVal = OptAPInt->trunc(EltSize);156 return true;157 }158 }159 160 auto *BV = dyn_cast<BuildVectorSDNode>(N);161 if (!BV)162 return false;163 164 APInt SplatUndef;165 unsigned SplatBitSize;166 bool HasUndefs;167 unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();168 // Endianness does not matter here. We are checking for a splat given the169 // element size of the vector, and if we find such a splat for little endian170 // layout, then that should be valid also for big endian (as the full vector171 // size is known to be a multiple of the element size).172 const bool IsBigEndian = false;173 return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,174 EltSize, IsBigEndian) &&175 EltSize == SplatBitSize;176}177 178// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be179// specializations of the more general isConstantSplatVector()?180 181bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) {182 // Look through a bit convert.183 while (N->getOpcode() == ISD::BITCAST)184 N = N->getOperand(0).getNode();185 186 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {187 APInt SplatVal;188 return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnes();189 }190 191 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;192 193 unsigned i = 0, e = N->getNumOperands();194 195 // Skip over all of the undef values.196 while (i != e && N->getOperand(i).isUndef())197 ++i;198 199 // Do not accept an all-undef vector.200 if (i == e) return false;201 202 // Do not accept build_vectors that aren't all constants or which have non-~0203 // elements. We have to be a bit careful here, as the type of the constant204 // may not be the same as the type of the vector elements due to type205 // legalization (the elements are promoted to a legal type for the target and206 // a vector of a type may be legal when the base element type is not).207 // We only want to check enough bits to cover the vector elements, because208 // we care if the resultant vector is all ones, not whether the individual209 // constants are.210 SDValue NotZero = N->getOperand(i);211 if (auto OptAPInt = NotZero->bitcastToAPInt()) {212 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();213 if (OptAPInt->countr_one() < EltSize)214 return false;215 } else216 return false;217 218 // Okay, we have at least one ~0 value, check to see if the rest match or are219 // undefs. Even with the above element type twiddling, this should be OK, as220 // the same type legalization should have applied to all the elements.221 for (++i; i != e; ++i)222 if (N->getOperand(i) != NotZero && !N->getOperand(i).isUndef())223 return false;224 return true;225}226 227bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) {228 // Look through a bit convert.229 while (N->getOpcode() == ISD::BITCAST)230 N = N->getOperand(0).getNode();231 232 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {233 APInt SplatVal;234 return isConstantSplatVector(N, SplatVal) && SplatVal.isZero();235 }236 237 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;238 239 bool IsAllUndef = true;240 for (const SDValue &Op : N->op_values()) {241 if (Op.isUndef())242 continue;243 IsAllUndef = false;244 // Do not accept build_vectors that aren't all constants or which have non-0245 // elements. We have to be a bit careful here, as the type of the constant246 // may not be the same as the type of the vector elements due to type247 // legalization (the elements are promoted to a legal type for the target248 // and a vector of a type may be legal when the base element type is not).249 // We only want to check enough bits to cover the vector elements, because250 // we care if the resultant vector is all zeros, not whether the individual251 // constants are.252 if (auto OptAPInt = Op->bitcastToAPInt()) {253 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();254 if (OptAPInt->countr_zero() < EltSize)255 return false;256 } else257 return false;258 }259 260 // Do not accept an all-undef vector.261 if (IsAllUndef)262 return false;263 return true;264}265 266bool ISD::isBuildVectorAllOnes(const SDNode *N) {267 return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true);268}269 270bool ISD::isBuildVectorAllZeros(const SDNode *N) {271 return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true);272}273 274bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {275 if (N->getOpcode() != ISD::BUILD_VECTOR)276 return false;277 278 for (const SDValue &Op : N->op_values()) {279 if (Op.isUndef())280 continue;281 if (!isa<ConstantSDNode>(Op))282 return false;283 }284 return true;285}286 287bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {288 if (N->getOpcode() != ISD::BUILD_VECTOR)289 return false;290 291 for (const SDValue &Op : N->op_values()) {292 if (Op.isUndef())293 continue;294 if (!isa<ConstantFPSDNode>(Op))295 return false;296 }297 return true;298}299 300bool ISD::isVectorShrinkable(const SDNode *N, unsigned NewEltSize,301 bool Signed) {302 assert(N->getValueType(0).isVector() && "Expected a vector!");303 304 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();305 if (EltSize <= NewEltSize)306 return false;307 308 if (N->getOpcode() == ISD::ZERO_EXTEND) {309 return (N->getOperand(0).getValueType().getScalarSizeInBits() <=310 NewEltSize) &&311 !Signed;312 }313 if (N->getOpcode() == ISD::SIGN_EXTEND) {314 return (N->getOperand(0).getValueType().getScalarSizeInBits() <=315 NewEltSize) &&316 Signed;317 }318 if (N->getOpcode() != ISD::BUILD_VECTOR)319 return false;320 321 for (const SDValue &Op : N->op_values()) {322 if (Op.isUndef())323 continue;324 if (!isa<ConstantSDNode>(Op))325 return false;326 327 APInt C = Op->getAsAPIntVal().trunc(EltSize);328 if (Signed && C.trunc(NewEltSize).sext(EltSize) != C)329 return false;330 if (!Signed && C.trunc(NewEltSize).zext(EltSize) != C)331 return false;332 }333 334 return true;335}336 337bool ISD::allOperandsUndef(const SDNode *N) {338 // Return false if the node has no operands.339 // This is "logically inconsistent" with the definition of "all" but340 // is probably the desired behavior.341 if (N->getNumOperands() == 0)342 return false;343 return all_of(N->op_values(), [](SDValue Op) { return Op.isUndef(); });344}345 346bool ISD::isFreezeUndef(const SDNode *N) {347 return N->getOpcode() == ISD::FREEZE && N->getOperand(0).isUndef();348}349 350template <typename ConstNodeType>351bool ISD::matchUnaryPredicateImpl(SDValue Op,352 std::function<bool(ConstNodeType *)> Match,353 bool AllowUndefs, bool AllowTruncation) {354 // FIXME: Add support for scalar UNDEF cases?355 if (auto *C = dyn_cast<ConstNodeType>(Op))356 return Match(C);357 358 // FIXME: Add support for vector UNDEF cases?359 if (ISD::BUILD_VECTOR != Op.getOpcode() &&360 ISD::SPLAT_VECTOR != Op.getOpcode())361 return false;362 363 EVT SVT = Op.getValueType().getScalarType();364 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {365 if (AllowUndefs && Op.getOperand(i).isUndef()) {366 if (!Match(nullptr))367 return false;368 continue;369 }370 371 auto *Cst = dyn_cast<ConstNodeType>(Op.getOperand(i));372 if (!Cst || (!AllowTruncation && Cst->getValueType(0) != SVT) ||373 !Match(Cst))374 return false;375 }376 return true;377}378// Build used template types.379template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(380 SDValue, std::function<bool(ConstantSDNode *)>, bool, bool);381template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(382 SDValue, std::function<bool(ConstantFPSDNode *)>, bool, bool);383 384bool ISD::matchBinaryPredicate(385 SDValue LHS, SDValue RHS,386 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,387 bool AllowUndefs, bool AllowTypeMismatch) {388 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())389 return false;390 391 // TODO: Add support for scalar UNDEF cases?392 if (auto *LHSCst = dyn_cast<ConstantSDNode>(LHS))393 if (auto *RHSCst = dyn_cast<ConstantSDNode>(RHS))394 return Match(LHSCst, RHSCst);395 396 // TODO: Add support for vector UNDEF cases?397 if (LHS.getOpcode() != RHS.getOpcode() ||398 (LHS.getOpcode() != ISD::BUILD_VECTOR &&399 LHS.getOpcode() != ISD::SPLAT_VECTOR))400 return false;401 402 EVT SVT = LHS.getValueType().getScalarType();403 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {404 SDValue LHSOp = LHS.getOperand(i);405 SDValue RHSOp = RHS.getOperand(i);406 bool LHSUndef = AllowUndefs && LHSOp.isUndef();407 bool RHSUndef = AllowUndefs && RHSOp.isUndef();408 auto *LHSCst = dyn_cast<ConstantSDNode>(LHSOp);409 auto *RHSCst = dyn_cast<ConstantSDNode>(RHSOp);410 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))411 return false;412 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||413 LHSOp.getValueType() != RHSOp.getValueType()))414 return false;415 if (!Match(LHSCst, RHSCst))416 return false;417 }418 return true;419}420 421ISD::NodeType ISD::getInverseMinMaxOpcode(unsigned MinMaxOpc) {422 switch (MinMaxOpc) {423 default:424 llvm_unreachable("unrecognized opcode");425 case ISD::UMIN:426 return ISD::UMAX;427 case ISD::UMAX:428 return ISD::UMIN;429 case ISD::SMIN:430 return ISD::SMAX;431 case ISD::SMAX:432 return ISD::SMIN;433 }434}435 436ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {437 switch (VecReduceOpcode) {438 default:439 llvm_unreachable("Expected VECREDUCE opcode");440 case ISD::VECREDUCE_FADD:441 case ISD::VECREDUCE_SEQ_FADD:442 case ISD::VP_REDUCE_FADD:443 case ISD::VP_REDUCE_SEQ_FADD:444 return ISD::FADD;445 case ISD::VECREDUCE_FMUL:446 case ISD::VECREDUCE_SEQ_FMUL:447 case ISD::VP_REDUCE_FMUL:448 case ISD::VP_REDUCE_SEQ_FMUL:449 return ISD::FMUL;450 case ISD::VECREDUCE_ADD:451 case ISD::VP_REDUCE_ADD:452 return ISD::ADD;453 case ISD::VECREDUCE_MUL:454 case ISD::VP_REDUCE_MUL:455 return ISD::MUL;456 case ISD::VECREDUCE_AND:457 case ISD::VP_REDUCE_AND:458 return ISD::AND;459 case ISD::VECREDUCE_OR:460 case ISD::VP_REDUCE_OR:461 return ISD::OR;462 case ISD::VECREDUCE_XOR:463 case ISD::VP_REDUCE_XOR:464 return ISD::XOR;465 case ISD::VECREDUCE_SMAX:466 case ISD::VP_REDUCE_SMAX:467 return ISD::SMAX;468 case ISD::VECREDUCE_SMIN:469 case ISD::VP_REDUCE_SMIN:470 return ISD::SMIN;471 case ISD::VECREDUCE_UMAX:472 case ISD::VP_REDUCE_UMAX:473 return ISD::UMAX;474 case ISD::VECREDUCE_UMIN:475 case ISD::VP_REDUCE_UMIN:476 return ISD::UMIN;477 case ISD::VECREDUCE_FMAX:478 case ISD::VP_REDUCE_FMAX:479 return ISD::FMAXNUM;480 case ISD::VECREDUCE_FMIN:481 case ISD::VP_REDUCE_FMIN:482 return ISD::FMINNUM;483 case ISD::VECREDUCE_FMAXIMUM:484 case ISD::VP_REDUCE_FMAXIMUM:485 return ISD::FMAXIMUM;486 case ISD::VECREDUCE_FMINIMUM:487 case ISD::VP_REDUCE_FMINIMUM:488 return ISD::FMINIMUM;489 }490}491 492bool ISD::isVPOpcode(unsigned Opcode) {493 switch (Opcode) {494 default:495 return false;496#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \497 case ISD::VPSD: \498 return true;499#include "llvm/IR/VPIntrinsics.def"500 }501}502 503bool ISD::isVPBinaryOp(unsigned Opcode) {504 switch (Opcode) {505 default:506 break;507#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:508#define VP_PROPERTY_BINARYOP return true;509#define END_REGISTER_VP_SDNODE(VPSD) break;510#include "llvm/IR/VPIntrinsics.def"511 }512 return false;513}514 515bool ISD::isVPReduction(unsigned Opcode) {516 switch (Opcode) {517 default:518 return false;519 case ISD::VP_REDUCE_ADD:520 case ISD::VP_REDUCE_MUL:521 case ISD::VP_REDUCE_AND:522 case ISD::VP_REDUCE_OR:523 case ISD::VP_REDUCE_XOR:524 case ISD::VP_REDUCE_SMAX:525 case ISD::VP_REDUCE_SMIN:526 case ISD::VP_REDUCE_UMAX:527 case ISD::VP_REDUCE_UMIN:528 case ISD::VP_REDUCE_FMAX:529 case ISD::VP_REDUCE_FMIN:530 case ISD::VP_REDUCE_FMAXIMUM:531 case ISD::VP_REDUCE_FMINIMUM:532 case ISD::VP_REDUCE_FADD:533 case ISD::VP_REDUCE_FMUL:534 case ISD::VP_REDUCE_SEQ_FADD:535 case ISD::VP_REDUCE_SEQ_FMUL:536 return true;537 }538}539 540/// The operand position of the vector mask.541std::optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) {542 switch (Opcode) {543 default:544 return std::nullopt;545#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \546 case ISD::VPSD: \547 return MASKPOS;548#include "llvm/IR/VPIntrinsics.def"549 }550}551 552/// The operand position of the explicit vector length parameter.553std::optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) {554 switch (Opcode) {555 default:556 return std::nullopt;557#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \558 case ISD::VPSD: \559 return EVLPOS;560#include "llvm/IR/VPIntrinsics.def"561 }562}563 564std::optional<unsigned> ISD::getBaseOpcodeForVP(unsigned VPOpcode,565 bool hasFPExcept) {566 // FIXME: Return strict opcodes in case of fp exceptions.567 switch (VPOpcode) {568 default:569 return std::nullopt;570#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC:571#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC;572#define END_REGISTER_VP_SDNODE(VPOPC) break;573#include "llvm/IR/VPIntrinsics.def"574 }575 return std::nullopt;576}577 578std::optional<unsigned> ISD::getVPForBaseOpcode(unsigned Opcode) {579 switch (Opcode) {580 default:581 return std::nullopt;582#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break;583#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC:584#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC;585#include "llvm/IR/VPIntrinsics.def"586 }587}588 589ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {590 switch (ExtType) {591 case ISD::EXTLOAD:592 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;593 case ISD::SEXTLOAD:594 return ISD::SIGN_EXTEND;595 case ISD::ZEXTLOAD:596 return ISD::ZERO_EXTEND;597 default:598 break;599 }600 601 llvm_unreachable("Invalid LoadExtType");602}603 604ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {605 // To perform this operation, we just need to swap the L and G bits of the606 // operation.607 unsigned OldL = (Operation >> 2) & 1;608 unsigned OldG = (Operation >> 1) & 1;609 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits610 (OldL << 1) | // New G bit611 (OldG << 2)); // New L bit.612}613 614static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) {615 unsigned Operation = Op;616 if (isIntegerLike)617 Operation ^= 7; // Flip L, G, E bits, but not U.618 else619 Operation ^= 15; // Flip all of the condition bits.620 621 if (Operation > ISD::SETTRUE2)622 Operation &= ~8; // Don't let N and U bits get set.623 624 return ISD::CondCode(Operation);625}626 627ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) {628 return getSetCCInverseImpl(Op, Type.isInteger());629}630 631ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op,632 bool isIntegerLike) {633 return getSetCCInverseImpl(Op, isIntegerLike);634}635 636/// For an integer comparison, return 1 if the comparison is a signed operation637/// and 2 if the result is an unsigned comparison. Return zero if the operation638/// does not depend on the sign of the input (setne and seteq).639static int isSignedOp(ISD::CondCode Opcode) {640 switch (Opcode) {641 default: llvm_unreachable("Illegal integer setcc operation!");642 case ISD::SETEQ:643 case ISD::SETNE: return 0;644 case ISD::SETLT:645 case ISD::SETLE:646 case ISD::SETGT:647 case ISD::SETGE: return 1;648 case ISD::SETULT:649 case ISD::SETULE:650 case ISD::SETUGT:651 case ISD::SETUGE: return 2;652 }653}654 655ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,656 EVT Type) {657 bool IsInteger = Type.isInteger();658 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)659 // Cannot fold a signed integer setcc with an unsigned integer setcc.660 return ISD::SETCC_INVALID;661 662 unsigned Op = Op1 | Op2; // Combine all of the condition bits.663 664 // If the N and U bits get set, then the resultant comparison DOES suddenly665 // care about orderedness, and it is true when ordered.666 if (Op > ISD::SETTRUE2)667 Op &= ~16; // Clear the U bit if the N bit is set.668 669 // Canonicalize illegal integer setcc's.670 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT671 Op = ISD::SETNE;672 673 return ISD::CondCode(Op);674}675 676ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,677 EVT Type) {678 bool IsInteger = Type.isInteger();679 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)680 // Cannot fold a signed setcc with an unsigned setcc.681 return ISD::SETCC_INVALID;682 683 // Combine all of the condition bits.684 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);685 686 // Canonicalize illegal integer setcc's.687 if (IsInteger) {688 switch (Result) {689 default: break;690 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT691 case ISD::SETOEQ: // SETEQ & SETU[LG]E692 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE693 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE694 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE695 }696 }697 698 return Result;699}700 701//===----------------------------------------------------------------------===//702// SDNode Profile Support703//===----------------------------------------------------------------------===//704 705/// AddNodeIDOpcode - Add the node opcode to the NodeID data.706static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {707 ID.AddInteger(OpC);708}709 710/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them711/// solely with their pointer.712static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {713 ID.AddPointer(VTList.VTs);714}715 716/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.717static void AddNodeIDOperands(FoldingSetNodeID &ID,718 ArrayRef<SDValue> Ops) {719 for (const auto &Op : Ops) {720 ID.AddPointer(Op.getNode());721 ID.AddInteger(Op.getResNo());722 }723}724 725/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.726static void AddNodeIDOperands(FoldingSetNodeID &ID,727 ArrayRef<SDUse> Ops) {728 for (const auto &Op : Ops) {729 ID.AddPointer(Op.getNode());730 ID.AddInteger(Op.getResNo());731 }732}733 734static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC,735 SDVTList VTList, ArrayRef<SDValue> OpList) {736 AddNodeIDOpcode(ID, OpC);737 AddNodeIDValueTypes(ID, VTList);738 AddNodeIDOperands(ID, OpList);739}740 741/// If this is an SDNode with special info, add this info to the NodeID data.742static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {743 switch (N->getOpcode()) {744 case ISD::TargetExternalSymbol:745 case ISD::ExternalSymbol:746 case ISD::MCSymbol:747 llvm_unreachable("Should only be used on nodes with operands");748 default: break; // Normal nodes don't need extra info.749 case ISD::TargetConstant:750 case ISD::Constant: {751 const ConstantSDNode *C = cast<ConstantSDNode>(N);752 ID.AddPointer(C->getConstantIntValue());753 ID.AddBoolean(C->isOpaque());754 break;755 }756 case ISD::TargetConstantFP:757 case ISD::ConstantFP:758 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());759 break;760 case ISD::TargetGlobalAddress:761 case ISD::GlobalAddress:762 case ISD::TargetGlobalTLSAddress:763 case ISD::GlobalTLSAddress: {764 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);765 ID.AddPointer(GA->getGlobal());766 ID.AddInteger(GA->getOffset());767 ID.AddInteger(GA->getTargetFlags());768 break;769 }770 case ISD::BasicBlock:771 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());772 break;773 case ISD::Register:774 ID.AddInteger(cast<RegisterSDNode>(N)->getReg().id());775 break;776 case ISD::RegisterMask:777 ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());778 break;779 case ISD::SRCVALUE:780 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());781 break;782 case ISD::FrameIndex:783 case ISD::TargetFrameIndex:784 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());785 break;786 case ISD::PSEUDO_PROBE:787 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getGuid());788 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getIndex());789 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getAttributes());790 break;791 case ISD::JumpTable:792 case ISD::TargetJumpTable:793 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());794 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());795 break;796 case ISD::ConstantPool:797 case ISD::TargetConstantPool: {798 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);799 ID.AddInteger(CP->getAlign().value());800 ID.AddInteger(CP->getOffset());801 if (CP->isMachineConstantPoolEntry())802 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);803 else804 ID.AddPointer(CP->getConstVal());805 ID.AddInteger(CP->getTargetFlags());806 break;807 }808 case ISD::TargetIndex: {809 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);810 ID.AddInteger(TI->getIndex());811 ID.AddInteger(TI->getOffset());812 ID.AddInteger(TI->getTargetFlags());813 break;814 }815 case ISD::LOAD: {816 const LoadSDNode *LD = cast<LoadSDNode>(N);817 ID.AddInteger(LD->getMemoryVT().getRawBits());818 ID.AddInteger(LD->getRawSubclassData());819 ID.AddInteger(LD->getPointerInfo().getAddrSpace());820 ID.AddInteger(LD->getMemOperand()->getFlags());821 break;822 }823 case ISD::STORE: {824 const StoreSDNode *ST = cast<StoreSDNode>(N);825 ID.AddInteger(ST->getMemoryVT().getRawBits());826 ID.AddInteger(ST->getRawSubclassData());827 ID.AddInteger(ST->getPointerInfo().getAddrSpace());828 ID.AddInteger(ST->getMemOperand()->getFlags());829 break;830 }831 case ISD::VP_LOAD: {832 const VPLoadSDNode *ELD = cast<VPLoadSDNode>(N);833 ID.AddInteger(ELD->getMemoryVT().getRawBits());834 ID.AddInteger(ELD->getRawSubclassData());835 ID.AddInteger(ELD->getPointerInfo().getAddrSpace());836 ID.AddInteger(ELD->getMemOperand()->getFlags());837 break;838 }839 case ISD::VP_LOAD_FF: {840 const auto *LD = cast<VPLoadFFSDNode>(N);841 ID.AddInteger(LD->getMemoryVT().getRawBits());842 ID.AddInteger(LD->getRawSubclassData());843 ID.AddInteger(LD->getPointerInfo().getAddrSpace());844 ID.AddInteger(LD->getMemOperand()->getFlags());845 break;846 }847 case ISD::VP_STORE: {848 const VPStoreSDNode *EST = cast<VPStoreSDNode>(N);849 ID.AddInteger(EST->getMemoryVT().getRawBits());850 ID.AddInteger(EST->getRawSubclassData());851 ID.AddInteger(EST->getPointerInfo().getAddrSpace());852 ID.AddInteger(EST->getMemOperand()->getFlags());853 break;854 }855 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {856 const VPStridedLoadSDNode *SLD = cast<VPStridedLoadSDNode>(N);857 ID.AddInteger(SLD->getMemoryVT().getRawBits());858 ID.AddInteger(SLD->getRawSubclassData());859 ID.AddInteger(SLD->getPointerInfo().getAddrSpace());860 break;861 }862 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {863 const VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(N);864 ID.AddInteger(SST->getMemoryVT().getRawBits());865 ID.AddInteger(SST->getRawSubclassData());866 ID.AddInteger(SST->getPointerInfo().getAddrSpace());867 break;868 }869 case ISD::VP_GATHER: {870 const VPGatherSDNode *EG = cast<VPGatherSDNode>(N);871 ID.AddInteger(EG->getMemoryVT().getRawBits());872 ID.AddInteger(EG->getRawSubclassData());873 ID.AddInteger(EG->getPointerInfo().getAddrSpace());874 ID.AddInteger(EG->getMemOperand()->getFlags());875 break;876 }877 case ISD::VP_SCATTER: {878 const VPScatterSDNode *ES = cast<VPScatterSDNode>(N);879 ID.AddInteger(ES->getMemoryVT().getRawBits());880 ID.AddInteger(ES->getRawSubclassData());881 ID.AddInteger(ES->getPointerInfo().getAddrSpace());882 ID.AddInteger(ES->getMemOperand()->getFlags());883 break;884 }885 case ISD::MLOAD: {886 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(N);887 ID.AddInteger(MLD->getMemoryVT().getRawBits());888 ID.AddInteger(MLD->getRawSubclassData());889 ID.AddInteger(MLD->getPointerInfo().getAddrSpace());890 ID.AddInteger(MLD->getMemOperand()->getFlags());891 break;892 }893 case ISD::MSTORE: {894 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);895 ID.AddInteger(MST->getMemoryVT().getRawBits());896 ID.AddInteger(MST->getRawSubclassData());897 ID.AddInteger(MST->getPointerInfo().getAddrSpace());898 ID.AddInteger(MST->getMemOperand()->getFlags());899 break;900 }901 case ISD::MGATHER: {902 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(N);903 ID.AddInteger(MG->getMemoryVT().getRawBits());904 ID.AddInteger(MG->getRawSubclassData());905 ID.AddInteger(MG->getPointerInfo().getAddrSpace());906 ID.AddInteger(MG->getMemOperand()->getFlags());907 break;908 }909 case ISD::MSCATTER: {910 const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(N);911 ID.AddInteger(MS->getMemoryVT().getRawBits());912 ID.AddInteger(MS->getRawSubclassData());913 ID.AddInteger(MS->getPointerInfo().getAddrSpace());914 ID.AddInteger(MS->getMemOperand()->getFlags());915 break;916 }917 case ISD::ATOMIC_CMP_SWAP:918 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:919 case ISD::ATOMIC_SWAP:920 case ISD::ATOMIC_LOAD_ADD:921 case ISD::ATOMIC_LOAD_SUB:922 case ISD::ATOMIC_LOAD_AND:923 case ISD::ATOMIC_LOAD_CLR:924 case ISD::ATOMIC_LOAD_OR:925 case ISD::ATOMIC_LOAD_XOR:926 case ISD::ATOMIC_LOAD_NAND:927 case ISD::ATOMIC_LOAD_MIN:928 case ISD::ATOMIC_LOAD_MAX:929 case ISD::ATOMIC_LOAD_UMIN:930 case ISD::ATOMIC_LOAD_UMAX:931 case ISD::ATOMIC_LOAD:932 case ISD::ATOMIC_STORE: {933 const AtomicSDNode *AT = cast<AtomicSDNode>(N);934 ID.AddInteger(AT->getMemoryVT().getRawBits());935 ID.AddInteger(AT->getRawSubclassData());936 ID.AddInteger(AT->getPointerInfo().getAddrSpace());937 ID.AddInteger(AT->getMemOperand()->getFlags());938 break;939 }940 case ISD::VECTOR_SHUFFLE: {941 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(N)->getMask();942 for (int M : Mask)943 ID.AddInteger(M);944 break;945 }946 case ISD::ADDRSPACECAST: {947 const AddrSpaceCastSDNode *ASC = cast<AddrSpaceCastSDNode>(N);948 ID.AddInteger(ASC->getSrcAddressSpace());949 ID.AddInteger(ASC->getDestAddressSpace());950 break;951 }952 case ISD::TargetBlockAddress:953 case ISD::BlockAddress: {954 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);955 ID.AddPointer(BA->getBlockAddress());956 ID.AddInteger(BA->getOffset());957 ID.AddInteger(BA->getTargetFlags());958 break;959 }960 case ISD::AssertAlign:961 ID.AddInteger(cast<AssertAlignSDNode>(N)->getAlign().value());962 break;963 case ISD::PREFETCH:964 case ISD::INTRINSIC_VOID:965 case ISD::INTRINSIC_W_CHAIN:966 // Handled by MemIntrinsicSDNode check after the switch.967 break;968 case ISD::MDNODE_SDNODE:969 ID.AddPointer(cast<MDNodeSDNode>(N)->getMD());970 break;971 } // end switch (N->getOpcode())972 973 // MemIntrinsic nodes could also have subclass data, address spaces, and flags974 // to check.975 if (auto *MN = dyn_cast<MemIntrinsicSDNode>(N)) {976 ID.AddInteger(MN->getRawSubclassData());977 ID.AddInteger(MN->getPointerInfo().getAddrSpace());978 ID.AddInteger(MN->getMemOperand()->getFlags());979 ID.AddInteger(MN->getMemoryVT().getRawBits());980 }981}982 983/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID984/// data.985static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {986 AddNodeIDOpcode(ID, N->getOpcode());987 // Add the return value info.988 AddNodeIDValueTypes(ID, N->getVTList());989 // Add the operand info.990 AddNodeIDOperands(ID, N->ops());991 992 // Handle SDNode leafs with special info.993 AddNodeIDCustom(ID, N);994}995 996//===----------------------------------------------------------------------===//997// SelectionDAG Class998//===----------------------------------------------------------------------===//999 1000/// doNotCSE - Return true if CSE should not be performed for this node.1001static bool doNotCSE(SDNode *N) {1002 if (N->getValueType(0) == MVT::Glue)1003 return true; // Never CSE anything that produces a glue result.1004 1005 switch (N->getOpcode()) {1006 default: break;1007 case ISD::HANDLENODE:1008 case ISD::EH_LABEL:1009 return true; // Never CSE these nodes.1010 }1011 1012 // Check that remaining values produced are not flags.1013 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)1014 if (N->getValueType(i) == MVT::Glue)1015 return true; // Never CSE anything that produces a glue result.1016 1017 return false;1018}1019 1020/// RemoveDeadNodes - This method deletes all unreachable nodes in the1021/// SelectionDAG.1022void SelectionDAG::RemoveDeadNodes() {1023 // Create a dummy node (which is not added to allnodes), that adds a reference1024 // to the root node, preventing it from being deleted.1025 HandleSDNode Dummy(getRoot());1026 1027 SmallVector<SDNode*, 128> DeadNodes;1028 1029 // Add all obviously-dead nodes to the DeadNodes worklist.1030 for (SDNode &Node : allnodes())1031 if (Node.use_empty())1032 DeadNodes.push_back(&Node);1033 1034 RemoveDeadNodes(DeadNodes);1035 1036 // If the root changed (e.g. it was a dead load, update the root).1037 setRoot(Dummy.getValue());1038}1039 1040/// RemoveDeadNodes - This method deletes the unreachable nodes in the1041/// given list, and any nodes that become unreachable as a result.1042void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {1043 1044 // Process the worklist, deleting the nodes and adding their uses to the1045 // worklist.1046 while (!DeadNodes.empty()) {1047 SDNode *N = DeadNodes.pop_back_val();1048 // Skip to next node if we've already managed to delete the node. This could1049 // happen if replacing a node causes a node previously added to the node to1050 // be deleted.1051 if (N->getOpcode() == ISD::DELETED_NODE)1052 continue;1053 1054 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)1055 DUL->NodeDeleted(N, nullptr);1056 1057 // Take the node out of the appropriate CSE map.1058 RemoveNodeFromCSEMaps(N);1059 1060 // Next, brutally remove the operand list. This is safe to do, as there are1061 // no cycles in the graph.1062 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {1063 SDUse &Use = *I++;1064 SDNode *Operand = Use.getNode();1065 Use.set(SDValue());1066 1067 // Now that we removed this operand, see if there are no uses of it left.1068 if (Operand->use_empty())1069 DeadNodes.push_back(Operand);1070 }1071 1072 DeallocateNode(N);1073 }1074}1075 1076void SelectionDAG::RemoveDeadNode(SDNode *N){1077 SmallVector<SDNode*, 16> DeadNodes(1, N);1078 1079 // Create a dummy node that adds a reference to the root node, preventing1080 // it from being deleted. (This matters if the root is an operand of the1081 // dead node.)1082 HandleSDNode Dummy(getRoot());1083 1084 RemoveDeadNodes(DeadNodes);1085}1086 1087void SelectionDAG::DeleteNode(SDNode *N) {1088 // First take this out of the appropriate CSE map.1089 RemoveNodeFromCSEMaps(N);1090 1091 // Finally, remove uses due to operands of this node, remove from the1092 // AllNodes list, and delete the node.1093 DeleteNodeNotInCSEMaps(N);1094}1095 1096void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {1097 assert(N->getIterator() != AllNodes.begin() &&1098 "Cannot delete the entry node!");1099 assert(N->use_empty() && "Cannot delete a node that is not dead!");1100 1101 // Drop all of the operands and decrement used node's use counts.1102 N->DropOperands();1103 1104 DeallocateNode(N);1105}1106 1107void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {1108 assert(!(V->isVariadic() && isParameter));1109 if (isParameter)1110 ByvalParmDbgValues.push_back(V);1111 else1112 DbgValues.push_back(V);1113 for (const SDNode *Node : V->getSDNodes())1114 if (Node)1115 DbgValMap[Node].push_back(V);1116}1117 1118void SDDbgInfo::erase(const SDNode *Node) {1119 DbgValMapType::iterator I = DbgValMap.find(Node);1120 if (I == DbgValMap.end())1121 return;1122 for (auto &Val: I->second)1123 Val->setIsInvalidated();1124 DbgValMap.erase(I);1125}1126 1127void SelectionDAG::DeallocateNode(SDNode *N) {1128 // If we have operands, deallocate them.1129 removeOperands(N);1130 1131 NodeAllocator.Deallocate(AllNodes.remove(N));1132 1133 // Set the opcode to DELETED_NODE to help catch bugs when node1134 // memory is reallocated.1135 // FIXME: There are places in SDag that have grown a dependency on the opcode1136 // value in the released node.1137 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));1138 N->NodeType = ISD::DELETED_NODE;1139 1140 // If any of the SDDbgValue nodes refer to this SDNode, invalidate1141 // them and forget about that node.1142 DbgInfo->erase(N);1143 1144 // Invalidate extra info.1145 SDEI.erase(N);1146}1147 1148#ifndef NDEBUG1149/// VerifySDNode - Check the given SDNode. Aborts if it is invalid.1150void SelectionDAG::verifyNode(SDNode *N) const {1151 switch (N->getOpcode()) {1152 default:1153 if (N->isTargetOpcode())1154 getSelectionDAGInfo().verifyTargetNode(*this, N);1155 break;1156 case ISD::BUILD_PAIR: {1157 EVT VT = N->getValueType(0);1158 assert(N->getNumValues() == 1 && "Too many results!");1159 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&1160 "Wrong return type!");1161 assert(N->getNumOperands() == 2 && "Wrong number of operands!");1162 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&1163 "Mismatched operand types!");1164 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&1165 "Wrong operand type!");1166 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&1167 "Wrong return type size");1168 break;1169 }1170 case ISD::BUILD_VECTOR: {1171 assert(N->getNumValues() == 1 && "Too many results!");1172 assert(N->getValueType(0).isVector() && "Wrong return type!");1173 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&1174 "Wrong number of operands!");1175 EVT EltVT = N->getValueType(0).getVectorElementType();1176 for (const SDUse &Op : N->ops()) {1177 assert((Op.getValueType() == EltVT ||1178 (EltVT.isInteger() && Op.getValueType().isInteger() &&1179 EltVT.bitsLE(Op.getValueType()))) &&1180 "Wrong operand type!");1181 assert(Op.getValueType() == N->getOperand(0).getValueType() &&1182 "Operands must all have the same type");1183 }1184 break;1185 }1186 }1187}1188#endif // NDEBUG1189 1190/// Insert a newly allocated node into the DAG.1191///1192/// Handles insertion into the all nodes list and CSE map, as well as1193/// verification and other common operations when a new node is allocated.1194void SelectionDAG::InsertNode(SDNode *N) {1195 AllNodes.push_back(N);1196#ifndef NDEBUG1197 N->PersistentId = NextPersistentId++;1198 verifyNode(N);1199#endif1200 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)1201 DUL->NodeInserted(N);1202}1203 1204/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that1205/// correspond to it. This is useful when we're about to delete or repurpose1206/// the node. We don't want future request for structurally identical nodes1207/// to return N anymore.1208bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {1209 bool Erased = false;1210 switch (N->getOpcode()) {1211 case ISD::HANDLENODE: return false; // noop.1212 case ISD::CONDCODE:1213 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&1214 "Cond code doesn't exist!");1215 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;1216 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;1217 break;1218 case ISD::ExternalSymbol:1219 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());1220 break;1221 case ISD::TargetExternalSymbol: {1222 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);1223 Erased = TargetExternalSymbols.erase(std::pair<std::string, unsigned>(1224 ESN->getSymbol(), ESN->getTargetFlags()));1225 break;1226 }1227 case ISD::MCSymbol: {1228 auto *MCSN = cast<MCSymbolSDNode>(N);1229 Erased = MCSymbols.erase(MCSN->getMCSymbol());1230 break;1231 }1232 case ISD::VALUETYPE: {1233 EVT VT = cast<VTSDNode>(N)->getVT();1234 if (VT.isExtended()) {1235 Erased = ExtendedValueTypeNodes.erase(VT);1236 } else {1237 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;1238 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;1239 }1240 break;1241 }1242 default:1243 // Remove it from the CSE Map.1244 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");1245 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");1246 Erased = CSEMap.RemoveNode(N);1247 break;1248 }1249#ifndef NDEBUG1250 // Verify that the node was actually in one of the CSE maps, unless it has a1251 // glue result (which cannot be CSE'd) or is one of the special cases that are1252 // not subject to CSE.1253 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&1254 !N->isMachineOpcode() && !doNotCSE(N)) {1255 N->dump(this);1256 dbgs() << "\n";1257 llvm_unreachable("Node is not in map!");1258 }1259#endif1260 return Erased;1261}1262 1263/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE1264/// maps and modified in place. Add it back to the CSE maps, unless an identical1265/// node already exists, in which case transfer all its users to the existing1266/// node. This transfer can potentially trigger recursive merging.1267void1268SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {1269 // For node types that aren't CSE'd, just act as if no identical node1270 // already exists.1271 if (!doNotCSE(N)) {1272 SDNode *Existing = CSEMap.GetOrInsertNode(N);1273 if (Existing != N) {1274 // If there was already an existing matching node, use ReplaceAllUsesWith1275 // to replace the dead one with the existing one. This can cause1276 // recursive merging of other unrelated nodes down the line.1277 Existing->intersectFlagsWith(N->getFlags());1278 if (auto *MemNode = dyn_cast<MemSDNode>(Existing))1279 MemNode->refineRanges(cast<MemSDNode>(N)->getMemOperand());1280 ReplaceAllUsesWith(N, Existing);1281 1282 // N is now dead. Inform the listeners and delete it.1283 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)1284 DUL->NodeDeleted(N, Existing);1285 DeleteNodeNotInCSEMaps(N);1286 return;1287 }1288 }1289 1290 // If the node doesn't already exist, we updated it. Inform listeners.1291 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)1292 DUL->NodeUpdated(N);1293}1294 1295/// FindModifiedNodeSlot - Find a slot for the specified node if its operands1296/// were replaced with those specified. If this node is never memoized,1297/// return null, otherwise return a pointer to the slot it would take. If a1298/// node already exists with these operands, the slot will be non-null.1299SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,1300 void *&InsertPos) {1301 if (doNotCSE(N))1302 return nullptr;1303 1304 SDValue Ops[] = { Op };1305 FoldingSetNodeID ID;1306 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);1307 AddNodeIDCustom(ID, N);1308 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);1309 if (Node)1310 Node->intersectFlagsWith(N->getFlags());1311 return Node;1312}1313 1314/// FindModifiedNodeSlot - Find a slot for the specified node if its operands1315/// were replaced with those specified. If this node is never memoized,1316/// return null, otherwise return a pointer to the slot it would take. If a1317/// node already exists with these operands, the slot will be non-null.1318SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,1319 SDValue Op1, SDValue Op2,1320 void *&InsertPos) {1321 if (doNotCSE(N))1322 return nullptr;1323 1324 SDValue Ops[] = { Op1, Op2 };1325 FoldingSetNodeID ID;1326 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);1327 AddNodeIDCustom(ID, N);1328 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);1329 if (Node)1330 Node->intersectFlagsWith(N->getFlags());1331 return Node;1332}1333 1334/// FindModifiedNodeSlot - Find a slot for the specified node if its operands1335/// were replaced with those specified. If this node is never memoized,1336/// return null, otherwise return a pointer to the slot it would take. If a1337/// node already exists with these operands, the slot will be non-null.1338SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,1339 void *&InsertPos) {1340 if (doNotCSE(N))1341 return nullptr;1342 1343 FoldingSetNodeID ID;1344 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);1345 AddNodeIDCustom(ID, N);1346 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);1347 if (Node)1348 Node->intersectFlagsWith(N->getFlags());1349 return Node;1350}1351 1352Align SelectionDAG::getEVTAlign(EVT VT) const {1353 Type *Ty = VT == MVT::iPTR ? PointerType::get(*getContext(), 0)1354 : VT.getTypeForEVT(*getContext());1355 1356 return getDataLayout().getABITypeAlign(Ty);1357}1358 1359// EntryNode could meaningfully have debug info if we can find it...1360SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOptLevel OL)1361 : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(),1362 getVTList(MVT::Other, MVT::Glue)),1363 Root(getEntryNode()) {1364 InsertNode(&EntryNode);1365 DbgInfo = new SDDbgInfo();1366}1367 1368void SelectionDAG::init(MachineFunction &NewMF,1369 OptimizationRemarkEmitter &NewORE, Pass *PassPtr,1370 const TargetLibraryInfo *LibraryInfo,1371 UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,1372 BlockFrequencyInfo *BFIin, MachineModuleInfo &MMIin,1373 FunctionVarLocs const *VarLocs) {1374 MF = &NewMF;1375 SDAGISelPass = PassPtr;1376 ORE = &NewORE;1377 TLI = getSubtarget().getTargetLowering();1378 TSI = getSubtarget().getSelectionDAGInfo();1379 LibInfo = LibraryInfo;1380 Context = &MF->getFunction().getContext();1381 UA = NewUA;1382 PSI = PSIin;1383 BFI = BFIin;1384 MMI = &MMIin;1385 FnVarLocs = VarLocs;1386}1387 1388SelectionDAG::~SelectionDAG() {1389 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");1390 allnodes_clear();1391 OperandRecycler.clear(OperandAllocator);1392 delete DbgInfo;1393}1394 1395bool SelectionDAG::shouldOptForSize() const {1396 return llvm::shouldOptimizeForSize(FLI->MBB->getBasicBlock(), PSI, BFI);1397}1398 1399void SelectionDAG::allnodes_clear() {1400 assert(&*AllNodes.begin() == &EntryNode);1401 AllNodes.remove(AllNodes.begin());1402 while (!AllNodes.empty())1403 DeallocateNode(&AllNodes.front());1404#ifndef NDEBUG1405 NextPersistentId = 0;1406#endif1407}1408 1409SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,1410 void *&InsertPos) {1411 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);1412 if (N) {1413 switch (N->getOpcode()) {1414 default: break;1415 case ISD::Constant:1416 case ISD::ConstantFP:1417 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "1418 "debug location. Use another overload.");1419 }1420 }1421 return N;1422}1423 1424SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,1425 const SDLoc &DL, void *&InsertPos) {1426 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);1427 if (N) {1428 switch (N->getOpcode()) {1429 case ISD::Constant:1430 case ISD::ConstantFP:1431 // Erase debug location from the node if the node is used at several1432 // different places. Do not propagate one location to all uses as it1433 // will cause a worse single stepping debugging experience.1434 if (N->getDebugLoc() != DL.getDebugLoc())1435 N->setDebugLoc(DebugLoc());1436 break;1437 default:1438 // When the node's point of use is located earlier in the instruction1439 // sequence than its prior point of use, update its debug info to the1440 // earlier location.1441 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())1442 N->setDebugLoc(DL.getDebugLoc());1443 break;1444 }1445 }1446 return N;1447}1448 1449void SelectionDAG::clear() {1450 allnodes_clear();1451 OperandRecycler.clear(OperandAllocator);1452 OperandAllocator.Reset();1453 CSEMap.clear();1454 1455 ExtendedValueTypeNodes.clear();1456 ExternalSymbols.clear();1457 TargetExternalSymbols.clear();1458 MCSymbols.clear();1459 SDEI.clear();1460 llvm::fill(CondCodeNodes, nullptr);1461 llvm::fill(ValueTypeNodes, nullptr);1462 1463 EntryNode.UseList = nullptr;1464 InsertNode(&EntryNode);1465 Root = getEntryNode();1466 DbgInfo->clear();1467}1468 1469SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {1470 return VT.bitsGT(Op.getValueType())1471 ? getNode(ISD::FP_EXTEND, DL, VT, Op)1472 : getNode(ISD::FP_ROUND, DL, VT, Op,1473 getIntPtrConstant(0, DL, /*isTarget=*/true));1474}1475 1476std::pair<SDValue, SDValue>1477SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain,1478 const SDLoc &DL, EVT VT) {1479 assert(!VT.bitsEq(Op.getValueType()) &&1480 "Strict no-op FP extend/round not allowed.");1481 SDValue Res =1482 VT.bitsGT(Op.getValueType())1483 ? getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op})1484 : getNode(ISD::STRICT_FP_ROUND, DL, {VT, MVT::Other},1485 {Chain, Op, getIntPtrConstant(0, DL, /*isTarget=*/true)});1486 1487 return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));1488}1489 1490SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {1491 return VT.bitsGT(Op.getValueType()) ?1492 getNode(ISD::ANY_EXTEND, DL, VT, Op) :1493 getNode(ISD::TRUNCATE, DL, VT, Op);1494}1495 1496SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {1497 return VT.bitsGT(Op.getValueType()) ?1498 getNode(ISD::SIGN_EXTEND, DL, VT, Op) :1499 getNode(ISD::TRUNCATE, DL, VT, Op);1500}1501 1502SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {1503 return VT.bitsGT(Op.getValueType()) ?1504 getNode(ISD::ZERO_EXTEND, DL, VT, Op) :1505 getNode(ISD::TRUNCATE, DL, VT, Op);1506}1507 1508SDValue SelectionDAG::getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL,1509 EVT VT) {1510 assert(!VT.isVector());1511 auto Type = Op.getValueType();1512 SDValue DestOp;1513 if (Type == VT)1514 return Op;1515 auto Size = Op.getValueSizeInBits();1516 DestOp = getBitcast(EVT::getIntegerVT(*Context, Size), Op);1517 if (DestOp.getValueType() == VT)1518 return DestOp;1519 1520 return getAnyExtOrTrunc(DestOp, DL, VT);1521}1522 1523SDValue SelectionDAG::getBitcastedSExtOrTrunc(SDValue Op, const SDLoc &DL,1524 EVT VT) {1525 assert(!VT.isVector());1526 auto Type = Op.getValueType();1527 SDValue DestOp;1528 if (Type == VT)1529 return Op;1530 auto Size = Op.getValueSizeInBits();1531 DestOp = getBitcast(MVT::getIntegerVT(Size), Op);1532 if (DestOp.getValueType() == VT)1533 return DestOp;1534 1535 return getSExtOrTrunc(DestOp, DL, VT);1536}1537 1538SDValue SelectionDAG::getBitcastedZExtOrTrunc(SDValue Op, const SDLoc &DL,1539 EVT VT) {1540 assert(!VT.isVector());1541 auto Type = Op.getValueType();1542 SDValue DestOp;1543 if (Type == VT)1544 return Op;1545 auto Size = Op.getValueSizeInBits();1546 DestOp = getBitcast(MVT::getIntegerVT(Size), Op);1547 if (DestOp.getValueType() == VT)1548 return DestOp;1549 1550 return getZExtOrTrunc(DestOp, DL, VT);1551}1552 1553SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,1554 EVT OpVT) {1555 if (VT.bitsLE(Op.getValueType()))1556 return getNode(ISD::TRUNCATE, SL, VT, Op);1557 1558 TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);1559 return getNode(TLI->getExtendForContent(BType), SL, VT, Op);1560}1561 1562SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {1563 EVT OpVT = Op.getValueType();1564 assert(VT.isInteger() && OpVT.isInteger() &&1565 "Cannot getZeroExtendInReg FP types");1566 assert(VT.isVector() == OpVT.isVector() &&1567 "getZeroExtendInReg type should be vector iff the operand "1568 "type is vector!");1569 assert((!VT.isVector() ||1570 VT.getVectorElementCount() == OpVT.getVectorElementCount()) &&1571 "Vector element counts must match in getZeroExtendInReg");1572 assert(VT.bitsLE(OpVT) && "Not extending!");1573 if (OpVT == VT)1574 return Op;1575 APInt Imm = APInt::getLowBitsSet(OpVT.getScalarSizeInBits(),1576 VT.getScalarSizeInBits());1577 return getNode(ISD::AND, DL, OpVT, Op, getConstant(Imm, DL, OpVT));1578}1579 1580SDValue SelectionDAG::getVPZeroExtendInReg(SDValue Op, SDValue Mask,1581 SDValue EVL, const SDLoc &DL,1582 EVT VT) {1583 EVT OpVT = Op.getValueType();1584 assert(VT.isInteger() && OpVT.isInteger() &&1585 "Cannot getVPZeroExtendInReg FP types");1586 assert(VT.isVector() && OpVT.isVector() &&1587 "getVPZeroExtendInReg type and operand type should be vector!");1588 assert(VT.getVectorElementCount() == OpVT.getVectorElementCount() &&1589 "Vector element counts must match in getZeroExtendInReg");1590 assert(VT.bitsLE(OpVT) && "Not extending!");1591 if (OpVT == VT)1592 return Op;1593 APInt Imm = APInt::getLowBitsSet(OpVT.getScalarSizeInBits(),1594 VT.getScalarSizeInBits());1595 return getNode(ISD::VP_AND, DL, OpVT, Op, getConstant(Imm, DL, OpVT), Mask,1596 EVL);1597}1598 1599SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {1600 // Only unsigned pointer semantics are supported right now. In the future this1601 // might delegate to TLI to check pointer signedness.1602 return getZExtOrTrunc(Op, DL, VT);1603}1604 1605SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {1606 // Only unsigned pointer semantics are supported right now. In the future this1607 // might delegate to TLI to check pointer signedness.1608 return getZeroExtendInReg(Op, DL, VT);1609}1610 1611SDValue SelectionDAG::getNegative(SDValue Val, const SDLoc &DL, EVT VT) {1612 return getNode(ISD::SUB, DL, VT, getConstant(0, DL, VT), Val);1613}1614 1615/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).1616SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {1617 return getNode(ISD::XOR, DL, VT, Val, getAllOnesConstant(DL, VT));1618}1619 1620SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {1621 SDValue TrueValue = getBoolConstant(true, DL, VT, VT);1622 return getNode(ISD::XOR, DL, VT, Val, TrueValue);1623}1624 1625SDValue SelectionDAG::getVPLogicalNOT(const SDLoc &DL, SDValue Val,1626 SDValue Mask, SDValue EVL, EVT VT) {1627 SDValue TrueValue = getBoolConstant(true, DL, VT, VT);1628 return getNode(ISD::VP_XOR, DL, VT, Val, TrueValue, Mask, EVL);1629}1630 1631SDValue SelectionDAG::getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,1632 SDValue Mask, SDValue EVL) {1633 return getVPZExtOrTrunc(DL, VT, Op, Mask, EVL);1634}1635 1636SDValue SelectionDAG::getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,1637 SDValue Mask, SDValue EVL) {1638 if (VT.bitsGT(Op.getValueType()))1639 return getNode(ISD::VP_ZERO_EXTEND, DL, VT, Op, Mask, EVL);1640 if (VT.bitsLT(Op.getValueType()))1641 return getNode(ISD::VP_TRUNCATE, DL, VT, Op, Mask, EVL);1642 return Op;1643}1644 1645SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,1646 EVT OpVT) {1647 if (!V)1648 return getConstant(0, DL, VT);1649 1650 switch (TLI->getBooleanContents(OpVT)) {1651 case TargetLowering::ZeroOrOneBooleanContent:1652 case TargetLowering::UndefinedBooleanContent:1653 return getConstant(1, DL, VT);1654 case TargetLowering::ZeroOrNegativeOneBooleanContent:1655 return getAllOnesConstant(DL, VT);1656 }1657 llvm_unreachable("Unexpected boolean content enum!");1658}1659 1660SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,1661 bool isT, bool isO) {1662 return getConstant(APInt(VT.getScalarSizeInBits(), Val, /*isSigned=*/false),1663 DL, VT, isT, isO);1664}1665 1666SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,1667 bool isT, bool isO) {1668 return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO);1669}1670 1671SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,1672 EVT VT, bool isT, bool isO) {1673 assert(VT.isInteger() && "Cannot create FP integer constant!");1674 1675 EVT EltVT = VT.getScalarType();1676 const ConstantInt *Elt = &Val;1677 1678 // Vector splats are explicit within the DAG, with ConstantSDNode holding the1679 // to-be-splatted scalar ConstantInt.1680 if (isa<VectorType>(Elt->getType()))1681 Elt = ConstantInt::get(*getContext(), Elt->getValue());1682 1683 // In some cases the vector type is legal but the element type is illegal and1684 // needs to be promoted, for example v8i8 on ARM. In this case, promote the1685 // inserted value (the type does not need to match the vector element type).1686 // Any extra bits introduced will be truncated away.1687 if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==1688 TargetLowering::TypePromoteInteger) {1689 EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);1690 APInt NewVal;1691 if (TLI->isSExtCheaperThanZExt(VT.getScalarType(), EltVT))1692 NewVal = Elt->getValue().sextOrTrunc(EltVT.getSizeInBits());1693 else1694 NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits());1695 Elt = ConstantInt::get(*getContext(), NewVal);1696 }1697 // In other cases the element type is illegal and needs to be expanded, for1698 // example v2i64 on MIPS32. In this case, find the nearest legal type, split1699 // the value into n parts and use a vector type with n-times the elements.1700 // Then bitcast to the type requested.1701 // Legalizing constants too early makes the DAGCombiner's job harder so we1702 // only legalize if the DAG tells us we must produce legal types.1703 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&1704 TLI->getTypeAction(*getContext(), EltVT) ==1705 TargetLowering::TypeExpandInteger) {1706 const APInt &NewVal = Elt->getValue();1707 EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);1708 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();1709 1710 // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.1711 if (VT.isScalableVector() ||1712 TLI->isOperationLegal(ISD::SPLAT_VECTOR, VT)) {1713 assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&1714 "Can only handle an even split!");1715 unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;1716 1717 SmallVector<SDValue, 2> ScalarParts;1718 for (unsigned i = 0; i != Parts; ++i)1719 ScalarParts.push_back(getConstant(1720 NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL,1721 ViaEltVT, isT, isO));1722 1723 return getNode(ISD::SPLAT_VECTOR_PARTS, DL, VT, ScalarParts);1724 }1725 1726 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;1727 EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);1728 1729 // Check the temporary vector is the correct size. If this fails then1730 // getTypeToTransformTo() probably returned a type whose size (in bits)1731 // isn't a power-of-2 factor of the requested type size.1732 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());1733 1734 SmallVector<SDValue, 2> EltParts;1735 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)1736 EltParts.push_back(getConstant(1737 NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL,1738 ViaEltVT, isT, isO));1739 1740 // EltParts is currently in little endian order. If we actually want1741 // big-endian order then reverse it now.1742 if (getDataLayout().isBigEndian())1743 std::reverse(EltParts.begin(), EltParts.end());1744 1745 // The elements must be reversed when the element order is different1746 // to the endianness of the elements (because the BITCAST is itself a1747 // vector shuffle in this situation). However, we do not need any code to1748 // perform this reversal because getConstant() is producing a vector1749 // splat.1750 // This situation occurs in MIPS MSA.1751 1752 SmallVector<SDValue, 8> Ops;1753 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)1754 llvm::append_range(Ops, EltParts);1755 1756 SDValue V =1757 getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops));1758 return V;1759 }1760 1761 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&1762 "APInt size does not match type size!");1763 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;1764 SDVTList VTs = getVTList(EltVT);1765 FoldingSetNodeID ID;1766 AddNodeIDNode(ID, Opc, VTs, {});1767 ID.AddPointer(Elt);1768 ID.AddBoolean(isO);1769 void *IP = nullptr;1770 SDNode *N = nullptr;1771 if ((N = FindNodeOrInsertPos(ID, DL, IP)))1772 if (!VT.isVector())1773 return SDValue(N, 0);1774 1775 if (!N) {1776 N = newSDNode<ConstantSDNode>(isT, isO, Elt, VTs);1777 CSEMap.InsertNode(N, IP);1778 InsertNode(N);1779 NewSDValueDbgMsg(SDValue(N, 0), "Creating constant: ", this);1780 }1781 1782 SDValue Result(N, 0);1783 if (VT.isVector())1784 Result = getSplat(VT, DL, Result);1785 return Result;1786}1787 1788SDValue SelectionDAG::getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT,1789 bool isT, bool isO) {1790 unsigned Size = VT.getScalarSizeInBits();1791 return getConstant(APInt(Size, Val, /*isSigned=*/true), DL, VT, isT, isO);1792}1793 1794SDValue SelectionDAG::getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget,1795 bool IsOpaque) {1796 return getConstant(APInt::getAllOnes(VT.getScalarSizeInBits()), DL, VT,1797 IsTarget, IsOpaque);1798}1799 1800SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,1801 bool isTarget) {1802 return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget);1803}1804 1805SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,1806 const SDLoc &DL) {1807 assert(VT.isInteger() && "Shift amount is not an integer type!");1808 EVT ShiftVT = TLI->getShiftAmountTy(VT, getDataLayout());1809 return getConstant(Val, DL, ShiftVT);1810}1811 1812SDValue SelectionDAG::getShiftAmountConstant(const APInt &Val, EVT VT,1813 const SDLoc &DL) {1814 assert(Val.ult(VT.getScalarSizeInBits()) && "Out of range shift");1815 return getShiftAmountConstant(Val.getZExtValue(), VT, DL);1816}1817 1818SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL,1819 bool isTarget) {1820 return getConstant(Val, DL, TLI->getVectorIdxTy(getDataLayout()), isTarget);1821}1822 1823SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,1824 bool isTarget) {1825 return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget);1826}1827 1828SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,1829 EVT VT, bool isTarget) {1830 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");1831 1832 EVT EltVT = VT.getScalarType();1833 const ConstantFP *Elt = &V;1834 1835 // Vector splats are explicit within the DAG, with ConstantFPSDNode holding1836 // the to-be-splatted scalar ConstantFP.1837 if (isa<VectorType>(Elt->getType()))1838 Elt = ConstantFP::get(*getContext(), Elt->getValue());1839 1840 // Do the map lookup using the actual bit pattern for the floating point1841 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and1842 // we don't have issues with SNANs.1843 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;1844 SDVTList VTs = getVTList(EltVT);1845 FoldingSetNodeID ID;1846 AddNodeIDNode(ID, Opc, VTs, {});1847 ID.AddPointer(Elt);1848 void *IP = nullptr;1849 SDNode *N = nullptr;1850 if ((N = FindNodeOrInsertPos(ID, DL, IP)))1851 if (!VT.isVector())1852 return SDValue(N, 0);1853 1854 if (!N) {1855 N = newSDNode<ConstantFPSDNode>(isTarget, Elt, VTs);1856 CSEMap.InsertNode(N, IP);1857 InsertNode(N);1858 }1859 1860 SDValue Result(N, 0);1861 if (VT.isVector())1862 Result = getSplat(VT, DL, Result);1863 NewSDValueDbgMsg(Result, "Creating fp constant: ", this);1864 return Result;1865}1866 1867SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,1868 bool isTarget) {1869 EVT EltVT = VT.getScalarType();1870 if (EltVT == MVT::f32)1871 return getConstantFP(APFloat((float)Val), DL, VT, isTarget);1872 if (EltVT == MVT::f64)1873 return getConstantFP(APFloat(Val), DL, VT, isTarget);1874 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||1875 EltVT == MVT::f16 || EltVT == MVT::bf16) {1876 bool Ignored;1877 APFloat APF = APFloat(Val);1878 APF.convert(EltVT.getFltSemantics(), APFloat::rmNearestTiesToEven,1879 &Ignored);1880 return getConstantFP(APF, DL, VT, isTarget);1881 }1882 llvm_unreachable("Unsupported type in getConstantFP");1883}1884 1885SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,1886 EVT VT, int64_t Offset, bool isTargetGA,1887 unsigned TargetFlags) {1888 assert((TargetFlags == 0 || isTargetGA) &&1889 "Cannot set target flags on target-independent globals");1890 1891 // Truncate (with sign-extension) the offset value to the pointer size.1892 unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());1893 if (BitWidth < 64)1894 Offset = SignExtend64(Offset, BitWidth);1895 1896 unsigned Opc;1897 if (GV->isThreadLocal())1898 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;1899 else1900 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;1901 1902 SDVTList VTs = getVTList(VT);1903 FoldingSetNodeID ID;1904 AddNodeIDNode(ID, Opc, VTs, {});1905 ID.AddPointer(GV);1906 ID.AddInteger(Offset);1907 ID.AddInteger(TargetFlags);1908 void *IP = nullptr;1909 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))1910 return SDValue(E, 0);1911 1912 auto *N = newSDNode<GlobalAddressSDNode>(1913 Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VTs, Offset, TargetFlags);1914 CSEMap.InsertNode(N, IP);1915 InsertNode(N);1916 return SDValue(N, 0);1917}1918 1919SDValue SelectionDAG::getDeactivationSymbol(const GlobalValue *GV) {1920 SDVTList VTs = getVTList(MVT::Untyped);1921 FoldingSetNodeID ID;1922 AddNodeIDNode(ID, ISD::DEACTIVATION_SYMBOL, VTs, {});1923 ID.AddPointer(GV);1924 void *IP = nullptr;1925 if (SDNode *E = FindNodeOrInsertPos(ID, SDLoc(), IP))1926 return SDValue(E, 0);1927 1928 auto *N = newSDNode<DeactivationSymbolSDNode>(GV, VTs);1929 CSEMap.InsertNode(N, IP);1930 InsertNode(N);1931 return SDValue(N, 0);1932}1933 1934SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {1935 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;1936 SDVTList VTs = getVTList(VT);1937 FoldingSetNodeID ID;1938 AddNodeIDNode(ID, Opc, VTs, {});1939 ID.AddInteger(FI);1940 void *IP = nullptr;1941 if (SDNode *E = FindNodeOrInsertPos(ID, IP))1942 return SDValue(E, 0);1943 1944 auto *N = newSDNode<FrameIndexSDNode>(FI, VTs, isTarget);1945 CSEMap.InsertNode(N, IP);1946 InsertNode(N);1947 return SDValue(N, 0);1948}1949 1950SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,1951 unsigned TargetFlags) {1952 assert((TargetFlags == 0 || isTarget) &&1953 "Cannot set target flags on target-independent jump tables");1954 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;1955 SDVTList VTs = getVTList(VT);1956 FoldingSetNodeID ID;1957 AddNodeIDNode(ID, Opc, VTs, {});1958 ID.AddInteger(JTI);1959 ID.AddInteger(TargetFlags);1960 void *IP = nullptr;1961 if (SDNode *E = FindNodeOrInsertPos(ID, IP))1962 return SDValue(E, 0);1963 1964 auto *N = newSDNode<JumpTableSDNode>(JTI, VTs, isTarget, TargetFlags);1965 CSEMap.InsertNode(N, IP);1966 InsertNode(N);1967 return SDValue(N, 0);1968}1969 1970SDValue SelectionDAG::getJumpTableDebugInfo(int JTI, SDValue Chain,1971 const SDLoc &DL) {1972 EVT PTy = getTargetLoweringInfo().getPointerTy(getDataLayout());1973 return getNode(ISD::JUMP_TABLE_DEBUG_INFO, DL, MVT::Glue, Chain,1974 getTargetConstant(static_cast<uint64_t>(JTI), DL, PTy, true));1975}1976 1977SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,1978 MaybeAlign Alignment, int Offset,1979 bool isTarget, unsigned TargetFlags) {1980 assert((TargetFlags == 0 || isTarget) &&1981 "Cannot set target flags on target-independent globals");1982 if (!Alignment)1983 Alignment = shouldOptForSize()1984 ? getDataLayout().getABITypeAlign(C->getType())1985 : getDataLayout().getPrefTypeAlign(C->getType());1986 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;1987 SDVTList VTs = getVTList(VT);1988 FoldingSetNodeID ID;1989 AddNodeIDNode(ID, Opc, VTs, {});1990 ID.AddInteger(Alignment->value());1991 ID.AddInteger(Offset);1992 ID.AddPointer(C);1993 ID.AddInteger(TargetFlags);1994 void *IP = nullptr;1995 if (SDNode *E = FindNodeOrInsertPos(ID, IP))1996 return SDValue(E, 0);1997 1998 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VTs, Offset, *Alignment,1999 TargetFlags);2000 CSEMap.InsertNode(N, IP);2001 InsertNode(N);2002 SDValue V = SDValue(N, 0);2003 NewSDValueDbgMsg(V, "Creating new constant pool: ", this);2004 return V;2005}2006 2007SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,2008 MaybeAlign Alignment, int Offset,2009 bool isTarget, unsigned TargetFlags) {2010 assert((TargetFlags == 0 || isTarget) &&2011 "Cannot set target flags on target-independent globals");2012 if (!Alignment)2013 Alignment = getDataLayout().getPrefTypeAlign(C->getType());2014 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;2015 SDVTList VTs = getVTList(VT);2016 FoldingSetNodeID ID;2017 AddNodeIDNode(ID, Opc, VTs, {});2018 ID.AddInteger(Alignment->value());2019 ID.AddInteger(Offset);2020 C->addSelectionDAGCSEId(ID);2021 ID.AddInteger(TargetFlags);2022 void *IP = nullptr;2023 if (SDNode *E = FindNodeOrInsertPos(ID, IP))2024 return SDValue(E, 0);2025 2026 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VTs, Offset, *Alignment,2027 TargetFlags);2028 CSEMap.InsertNode(N, IP);2029 InsertNode(N);2030 return SDValue(N, 0);2031}2032 2033SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {2034 FoldingSetNodeID ID;2035 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), {});2036 ID.AddPointer(MBB);2037 void *IP = nullptr;2038 if (SDNode *E = FindNodeOrInsertPos(ID, IP))2039 return SDValue(E, 0);2040 2041 auto *N = newSDNode<BasicBlockSDNode>(MBB);2042 CSEMap.InsertNode(N, IP);2043 InsertNode(N);2044 return SDValue(N, 0);2045}2046 2047SDValue SelectionDAG::getValueType(EVT VT) {2048 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=2049 ValueTypeNodes.size())2050 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);2051 2052 SDNode *&N = VT.isExtended() ?2053 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];2054 2055 if (N) return SDValue(N, 0);2056 N = newSDNode<VTSDNode>(VT);2057 InsertNode(N);2058 return SDValue(N, 0);2059}2060 2061SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {2062 SDNode *&N = ExternalSymbols[Sym];2063 if (N) return SDValue(N, 0);2064 N = newSDNode<ExternalSymbolSDNode>(false, Sym, 0, getVTList(VT));2065 InsertNode(N);2066 return SDValue(N, 0);2067}2068 2069SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {2070 SDNode *&N = MCSymbols[Sym];2071 if (N)2072 return SDValue(N, 0);2073 N = newSDNode<MCSymbolSDNode>(Sym, getVTList(VT));2074 InsertNode(N);2075 return SDValue(N, 0);2076}2077 2078SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,2079 unsigned TargetFlags) {2080 SDNode *&N =2081 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];2082 if (N) return SDValue(N, 0);2083 N = newSDNode<ExternalSymbolSDNode>(true, Sym, TargetFlags, getVTList(VT));2084 InsertNode(N);2085 return SDValue(N, 0);2086}2087 2088SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {2089 if ((unsigned)Cond >= CondCodeNodes.size())2090 CondCodeNodes.resize(Cond+1);2091 2092 if (!CondCodeNodes[Cond]) {2093 auto *N = newSDNode<CondCodeSDNode>(Cond);2094 CondCodeNodes[Cond] = N;2095 InsertNode(N);2096 }2097 2098 return SDValue(CondCodeNodes[Cond], 0);2099}2100 2101SDValue SelectionDAG::getVScale(const SDLoc &DL, EVT VT, APInt MulImm) {2102 assert(MulImm.getBitWidth() == VT.getSizeInBits() &&2103 "APInt size does not match type size!");2104 2105 if (MulImm == 0)2106 return getConstant(0, DL, VT);2107 2108 const MachineFunction &MF = getMachineFunction();2109 const Function &F = MF.getFunction();2110 ConstantRange CR = getVScaleRange(&F, 64);2111 if (const APInt *C = CR.getSingleElement())2112 return getConstant(MulImm * C->getZExtValue(), DL, VT);2113 2114 return getNode(ISD::VSCALE, DL, VT, getConstant(MulImm, DL, VT));2115}2116 2117/// \returns a value of type \p VT that represents the runtime value of \p2118/// Quantity, i.e. scaled by vscale if it's scalable, or a fixed constant2119/// otherwise. Quantity should be a FixedOrScalableQuantity, i.e. ElementCount2120/// or TypeSize.2121template <typename Ty>2122static SDValue getFixedOrScalableQuantity(SelectionDAG &DAG, const SDLoc &DL,2123 EVT VT, Ty Quantity) {2124 if (Quantity.isScalable())2125 return DAG.getVScale(2126 DL, VT, APInt(VT.getSizeInBits(), Quantity.getKnownMinValue()));2127 2128 return DAG.getConstant(Quantity.getKnownMinValue(), DL, VT);2129}2130 2131SDValue SelectionDAG::getElementCount(const SDLoc &DL, EVT VT,2132 ElementCount EC) {2133 return getFixedOrScalableQuantity(*this, DL, VT, EC);2134}2135 2136SDValue SelectionDAG::getTypeSize(const SDLoc &DL, EVT VT, TypeSize TS) {2137 return getFixedOrScalableQuantity(*this, DL, VT, TS);2138}2139 2140SDValue SelectionDAG::getMaskFromElementCount(const SDLoc &DL, EVT DataVT,2141 ElementCount EC) {2142 EVT IdxVT = TLI->getVectorIdxTy(getDataLayout());2143 EVT MaskVT = TLI->getSetCCResultType(getDataLayout(), *getContext(), DataVT);2144 return getNode(ISD::GET_ACTIVE_LANE_MASK, DL, MaskVT,2145 getConstant(0, DL, IdxVT), getElementCount(DL, IdxVT, EC));2146}2147 2148SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) {2149 APInt One(ResVT.getScalarSizeInBits(), 1);2150 return getStepVector(DL, ResVT, One);2151}2152 2153SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT,2154 const APInt &StepVal) {2155 assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth());2156 if (ResVT.isScalableVector())2157 return getNode(2158 ISD::STEP_VECTOR, DL, ResVT,2159 getTargetConstant(StepVal, DL, ResVT.getVectorElementType()));2160 2161 SmallVector<SDValue, 16> OpsStepConstants;2162 for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)2163 OpsStepConstants.push_back(2164 getConstant(StepVal * i, DL, ResVT.getVectorElementType()));2165 return getBuildVector(ResVT, DL, OpsStepConstants);2166}2167 2168/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that2169/// point at N1 to point at N2 and indices that point at N2 to point at N1.2170static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {2171 std::swap(N1, N2);2172 ShuffleVectorSDNode::commuteMask(M);2173}2174 2175SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,2176 SDValue N2, ArrayRef<int> Mask) {2177 assert(VT.getVectorNumElements() == Mask.size() &&2178 "Must have the same number of vector elements as mask elements!");2179 assert(VT == N1.getValueType() && VT == N2.getValueType() &&2180 "Invalid VECTOR_SHUFFLE");2181 2182 // Canonicalize shuffle undef, undef -> undef2183 if (N1.isUndef() && N2.isUndef())2184 return getUNDEF(VT);2185 2186 // Validate that all indices in Mask are within the range of the elements2187 // input to the shuffle.2188 int NElts = Mask.size();2189 assert(llvm::all_of(Mask,2190 [&](int M) { return M < (NElts * 2) && M >= -1; }) &&2191 "Index out of range");2192 2193 // Copy the mask so we can do any needed cleanup.2194 SmallVector<int, 8> MaskVec(Mask);2195 2196 // Canonicalize shuffle v, v -> v, undef2197 if (N1 == N2) {2198 N2 = getUNDEF(VT);2199 for (int i = 0; i != NElts; ++i)2200 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;2201 }2202 2203 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.2204 if (N1.isUndef())2205 commuteShuffle(N1, N2, MaskVec);2206 2207 if (TLI->hasVectorBlend()) {2208 // If shuffling a splat, try to blend the splat instead. We do this here so2209 // that even when this arises during lowering we don't have to re-handle it.2210 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {2211 BitVector UndefElements;2212 SDValue Splat = BV->getSplatValue(&UndefElements);2213 if (!Splat)2214 return;2215 2216 for (int i = 0; i < NElts; ++i) {2217 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))2218 continue;2219 2220 // If this input comes from undef, mark it as such.2221 if (UndefElements[MaskVec[i] - Offset]) {2222 MaskVec[i] = -1;2223 continue;2224 }2225 2226 // If we can blend a non-undef lane, use that instead.2227 if (!UndefElements[i])2228 MaskVec[i] = i + Offset;2229 }2230 };2231 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))2232 BlendSplat(N1BV, 0);2233 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))2234 BlendSplat(N2BV, NElts);2235 }2236 2237 // Canonicalize all index into lhs, -> shuffle lhs, undef2238 // Canonicalize all index into rhs, -> shuffle rhs, undef2239 bool AllLHS = true, AllRHS = true;2240 bool N2Undef = N2.isUndef();2241 for (int i = 0; i != NElts; ++i) {2242 if (MaskVec[i] >= NElts) {2243 if (N2Undef)2244 MaskVec[i] = -1;2245 else2246 AllLHS = false;2247 } else if (MaskVec[i] >= 0) {2248 AllRHS = false;2249 }2250 }2251 if (AllLHS && AllRHS)2252 return getUNDEF(VT);2253 if (AllLHS && !N2Undef)2254 N2 = getUNDEF(VT);2255 if (AllRHS) {2256 N1 = getUNDEF(VT);2257 commuteShuffle(N1, N2, MaskVec);2258 }2259 // Reset our undef status after accounting for the mask.2260 N2Undef = N2.isUndef();2261 // Re-check whether both sides ended up undef.2262 if (N1.isUndef() && N2Undef)2263 return getUNDEF(VT);2264 2265 // If Identity shuffle return that node.2266 bool Identity = true, AllSame = true;2267 for (int i = 0; i != NElts; ++i) {2268 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;2269 if (MaskVec[i] != MaskVec[0]) AllSame = false;2270 }2271 if (Identity && NElts)2272 return N1;2273 2274 // Shuffling a constant splat doesn't change the result.2275 if (N2Undef) {2276 SDValue V = N1;2277 2278 // Look through any bitcasts. We check that these don't change the number2279 // (and size) of elements and just changes their types.2280 while (V.getOpcode() == ISD::BITCAST)2281 V = V->getOperand(0);2282 2283 // A splat should always show up as a build vector node.2284 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {2285 BitVector UndefElements;2286 SDValue Splat = BV->getSplatValue(&UndefElements);2287 // If this is a splat of an undef, shuffling it is also undef.2288 if (Splat && Splat.isUndef())2289 return getUNDEF(VT);2290 2291 bool SameNumElts =2292 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();2293 2294 // We only have a splat which can skip shuffles if there is a splatted2295 // value and no undef lanes rearranged by the shuffle.2296 if (Splat && UndefElements.none()) {2297 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the2298 // number of elements match or the value splatted is a zero constant.2299 if (SameNumElts || isNullConstant(Splat))2300 return N1;2301 }2302 2303 // If the shuffle itself creates a splat, build the vector directly.2304 if (AllSame && SameNumElts) {2305 EVT BuildVT = BV->getValueType(0);2306 const SDValue &Splatted = BV->getOperand(MaskVec[0]);2307 SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted);2308 2309 // We may have jumped through bitcasts, so the type of the2310 // BUILD_VECTOR may not match the type of the shuffle.2311 if (BuildVT != VT)2312 NewBV = getNode(ISD::BITCAST, dl, VT, NewBV);2313 return NewBV;2314 }2315 }2316 }2317 2318 SDVTList VTs = getVTList(VT);2319 FoldingSetNodeID ID;2320 SDValue Ops[2] = { N1, N2 };2321 AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, VTs, Ops);2322 for (int i = 0; i != NElts; ++i)2323 ID.AddInteger(MaskVec[i]);2324 2325 void* IP = nullptr;2326 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))2327 return SDValue(E, 0);2328 2329 // Allocate the mask array for the node out of the BumpPtrAllocator, since2330 // SDNode doesn't have access to it. This memory will be "leaked" when2331 // the node is deallocated, but recovered when the NodeAllocator is released.2332 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);2333 llvm::copy(MaskVec, MaskAlloc);2334 2335 auto *N = newSDNode<ShuffleVectorSDNode>(VTs, dl.getIROrder(),2336 dl.getDebugLoc(), MaskAlloc);2337 createOperands(N, Ops);2338 2339 CSEMap.InsertNode(N, IP);2340 InsertNode(N);2341 SDValue V = SDValue(N, 0);2342 NewSDValueDbgMsg(V, "Creating new node: ", this);2343 return V;2344}2345 2346SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {2347 EVT VT = SV.getValueType(0);2348 SmallVector<int, 8> MaskVec(SV.getMask());2349 ShuffleVectorSDNode::commuteMask(MaskVec);2350 2351 SDValue Op0 = SV.getOperand(0);2352 SDValue Op1 = SV.getOperand(1);2353 return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec);2354}2355 2356SDValue SelectionDAG::getRegister(Register Reg, EVT VT) {2357 SDVTList VTs = getVTList(VT);2358 FoldingSetNodeID ID;2359 AddNodeIDNode(ID, ISD::Register, VTs, {});2360 ID.AddInteger(Reg.id());2361 void *IP = nullptr;2362 if (SDNode *E = FindNodeOrInsertPos(ID, IP))2363 return SDValue(E, 0);2364 2365 auto *N = newSDNode<RegisterSDNode>(Reg, VTs);2366 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA);2367 CSEMap.InsertNode(N, IP);2368 InsertNode(N);2369 return SDValue(N, 0);2370}2371 2372SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {2373 FoldingSetNodeID ID;2374 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), {});2375 ID.AddPointer(RegMask);2376 void *IP = nullptr;2377 if (SDNode *E = FindNodeOrInsertPos(ID, IP))2378 return SDValue(E, 0);2379 2380 auto *N = newSDNode<RegisterMaskSDNode>(RegMask);2381 CSEMap.InsertNode(N, IP);2382 InsertNode(N);2383 return SDValue(N, 0);2384}2385 2386SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,2387 MCSymbol *Label) {2388 return getLabelNode(ISD::EH_LABEL, dl, Root, Label);2389}2390 2391SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,2392 SDValue Root, MCSymbol *Label) {2393 FoldingSetNodeID ID;2394 SDValue Ops[] = { Root };2395 AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops);2396 ID.AddPointer(Label);2397 void *IP = nullptr;2398 if (SDNode *E = FindNodeOrInsertPos(ID, IP))2399 return SDValue(E, 0);2400 2401 auto *N =2402 newSDNode<LabelSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), Label);2403 createOperands(N, Ops);2404 2405 CSEMap.InsertNode(N, IP);2406 InsertNode(N);2407 return SDValue(N, 0);2408}2409 2410SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,2411 int64_t Offset, bool isTarget,2412 unsigned TargetFlags) {2413 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;2414 SDVTList VTs = getVTList(VT);2415 2416 FoldingSetNodeID ID;2417 AddNodeIDNode(ID, Opc, VTs, {});2418 ID.AddPointer(BA);2419 ID.AddInteger(Offset);2420 ID.AddInteger(TargetFlags);2421 void *IP = nullptr;2422 if (SDNode *E = FindNodeOrInsertPos(ID, IP))2423 return SDValue(E, 0);2424 2425 auto *N = newSDNode<BlockAddressSDNode>(Opc, VTs, BA, Offset, TargetFlags);2426 CSEMap.InsertNode(N, IP);2427 InsertNode(N);2428 return SDValue(N, 0);2429}2430 2431SDValue SelectionDAG::getSrcValue(const Value *V) {2432 FoldingSetNodeID ID;2433 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), {});2434 ID.AddPointer(V);2435 2436 void *IP = nullptr;2437 if (SDNode *E = FindNodeOrInsertPos(ID, IP))2438 return SDValue(E, 0);2439 2440 auto *N = newSDNode<SrcValueSDNode>(V);2441 CSEMap.InsertNode(N, IP);2442 InsertNode(N);2443 return SDValue(N, 0);2444}2445 2446SDValue SelectionDAG::getMDNode(const MDNode *MD) {2447 FoldingSetNodeID ID;2448 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), {});2449 ID.AddPointer(MD);2450 2451 void *IP = nullptr;2452 if (SDNode *E = FindNodeOrInsertPos(ID, IP))2453 return SDValue(E, 0);2454 2455 auto *N = newSDNode<MDNodeSDNode>(MD);2456 CSEMap.InsertNode(N, IP);2457 InsertNode(N);2458 return SDValue(N, 0);2459}2460 2461SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {2462 if (VT == V.getValueType())2463 return V;2464 2465 return getNode(ISD::BITCAST, SDLoc(V), VT, V);2466}2467 2468SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,2469 unsigned SrcAS, unsigned DestAS) {2470 SDVTList VTs = getVTList(VT);2471 SDValue Ops[] = {Ptr};2472 FoldingSetNodeID ID;2473 AddNodeIDNode(ID, ISD::ADDRSPACECAST, VTs, Ops);2474 ID.AddInteger(SrcAS);2475 ID.AddInteger(DestAS);2476 2477 void *IP = nullptr;2478 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))2479 return SDValue(E, 0);2480 2481 auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(),2482 VTs, SrcAS, DestAS);2483 createOperands(N, Ops);2484 2485 CSEMap.InsertNode(N, IP);2486 InsertNode(N);2487 return SDValue(N, 0);2488}2489 2490SDValue SelectionDAG::getFreeze(SDValue V) {2491 return getNode(ISD::FREEZE, SDLoc(V), V.getValueType(), V);2492}2493 2494/// getShiftAmountOperand - Return the specified value casted to2495/// the target's desired shift amount type.2496SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {2497 EVT OpTy = Op.getValueType();2498 EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout());2499 if (OpTy == ShTy || OpTy.isVector()) return Op;2500 2501 return getZExtOrTrunc(Op, SDLoc(Op), ShTy);2502}2503 2504SDValue SelectionDAG::expandVAArg(SDNode *Node) {2505 SDLoc dl(Node);2506 const TargetLowering &TLI = getTargetLoweringInfo();2507 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();2508 EVT VT = Node->getValueType(0);2509 SDValue Tmp1 = Node->getOperand(0);2510 SDValue Tmp2 = Node->getOperand(1);2511 const MaybeAlign MA(Node->getConstantOperandVal(3));2512 2513 SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1,2514 Tmp2, MachinePointerInfo(V));2515 SDValue VAList = VAListLoad;2516 2517 if (MA && *MA > TLI.getMinStackArgumentAlignment()) {2518 VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,2519 getConstant(MA->value() - 1, dl, VAList.getValueType()));2520 2521 VAList = getNode(2522 ISD::AND, dl, VAList.getValueType(), VAList,2523 getSignedConstant(-(int64_t)MA->value(), dl, VAList.getValueType()));2524 }2525 2526 // Increment the pointer, VAList, to the next vaarg2527 Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,2528 getConstant(getDataLayout().getTypeAllocSize(2529 VT.getTypeForEVT(*getContext())),2530 dl, VAList.getValueType()));2531 // Store the incremented VAList to the legalized pointer2532 Tmp1 =2533 getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V));2534 // Load the actual argument out of the pointer VAList2535 return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo());2536}2537 2538SDValue SelectionDAG::expandVACopy(SDNode *Node) {2539 SDLoc dl(Node);2540 const TargetLowering &TLI = getTargetLoweringInfo();2541 // This defaults to loading a pointer from the input and storing it to the2542 // output, returning the chain.2543 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();2544 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();2545 SDValue Tmp1 =2546 getLoad(TLI.getPointerTy(getDataLayout()), dl, Node->getOperand(0),2547 Node->getOperand(2), MachinePointerInfo(VS));2548 return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1),2549 MachinePointerInfo(VD));2550}2551 2552Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) {2553 const DataLayout &DL = getDataLayout();2554 Type *Ty = VT.getTypeForEVT(*getContext());2555 Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);2556 2557 if (TLI->isTypeLegal(VT) || !VT.isVector())2558 return RedAlign;2559 2560 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();2561 const Align StackAlign = TFI->getStackAlign();2562 2563 // See if we can choose a smaller ABI alignment in cases where it's an2564 // illegal vector type that will get broken down.2565 if (RedAlign > StackAlign) {2566 EVT IntermediateVT;2567 MVT RegisterVT;2568 unsigned NumIntermediates;2569 TLI->getVectorTypeBreakdown(*getContext(), VT, IntermediateVT,2570 NumIntermediates, RegisterVT);2571 Ty = IntermediateVT.getTypeForEVT(*getContext());2572 Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);2573 if (RedAlign2 < RedAlign)2574 RedAlign = RedAlign2;2575 2576 if (!getMachineFunction().getFrameInfo().isStackRealignable())2577 // If the stack is not realignable, the alignment should be limited to the2578 // StackAlignment2579 RedAlign = std::min(RedAlign, StackAlign);2580 }2581 2582 return RedAlign;2583}2584 2585SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) {2586 MachineFrameInfo &MFI = MF->getFrameInfo();2587 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();2588 int StackID = 0;2589 if (Bytes.isScalable())2590 StackID = TFI->getStackIDForScalableVectors();2591 // The stack id gives an indication of whether the object is scalable or2592 // not, so it's safe to pass in the minimum size here.2593 int FrameIdx = MFI.CreateStackObject(Bytes.getKnownMinValue(), Alignment,2594 false, nullptr, StackID);2595 return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));2596}2597 2598SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {2599 Type *Ty = VT.getTypeForEVT(*getContext());2600 Align StackAlign =2601 std::max(getDataLayout().getPrefTypeAlign(Ty), Align(minAlign));2602 return CreateStackTemporary(VT.getStoreSize(), StackAlign);2603}2604 2605SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {2606 TypeSize VT1Size = VT1.getStoreSize();2607 TypeSize VT2Size = VT2.getStoreSize();2608 assert(VT1Size.isScalable() == VT2Size.isScalable() &&2609 "Don't know how to choose the maximum size when creating a stack "2610 "temporary");2611 TypeSize Bytes = VT1Size.getKnownMinValue() > VT2Size.getKnownMinValue()2612 ? VT1Size2613 : VT2Size;2614 2615 Type *Ty1 = VT1.getTypeForEVT(*getContext());2616 Type *Ty2 = VT2.getTypeForEVT(*getContext());2617 const DataLayout &DL = getDataLayout();2618 Align Align = std::max(DL.getPrefTypeAlign(Ty1), DL.getPrefTypeAlign(Ty2));2619 return CreateStackTemporary(Bytes, Align);2620}2621 2622SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,2623 ISD::CondCode Cond, const SDLoc &dl) {2624 EVT OpVT = N1.getValueType();2625 2626 auto GetUndefBooleanConstant = [&]() {2627 if (VT.getScalarType() == MVT::i1 ||2628 TLI->getBooleanContents(OpVT) ==2629 TargetLowering::UndefinedBooleanContent)2630 return getUNDEF(VT);2631 // ZeroOrOne / ZeroOrNegative require specific values for the high bits,2632 // so we cannot use getUNDEF(). Return zero instead.2633 return getConstant(0, dl, VT);2634 };2635 2636 // These setcc operations always fold.2637 switch (Cond) {2638 default: break;2639 case ISD::SETFALSE:2640 case ISD::SETFALSE2: return getBoolConstant(false, dl, VT, OpVT);2641 case ISD::SETTRUE:2642 case ISD::SETTRUE2: return getBoolConstant(true, dl, VT, OpVT);2643 2644 case ISD::SETOEQ:2645 case ISD::SETOGT:2646 case ISD::SETOGE:2647 case ISD::SETOLT:2648 case ISD::SETOLE:2649 case ISD::SETONE:2650 case ISD::SETO:2651 case ISD::SETUO:2652 case ISD::SETUEQ:2653 case ISD::SETUNE:2654 assert(!OpVT.isInteger() && "Illegal setcc for integer!");2655 break;2656 }2657 2658 if (OpVT.isInteger()) {2659 // For EQ and NE, we can always pick a value for the undef to make the2660 // predicate pass or fail, so we can return undef.2661 // Matches behavior in llvm::ConstantFoldCompareInstruction.2662 // icmp eq/ne X, undef -> undef.2663 if ((N1.isUndef() || N2.isUndef()) &&2664 (Cond == ISD::SETEQ || Cond == ISD::SETNE))2665 return GetUndefBooleanConstant();2666 2667 // If both operands are undef, we can return undef for int comparison.2668 // icmp undef, undef -> undef.2669 if (N1.isUndef() && N2.isUndef())2670 return GetUndefBooleanConstant();2671 2672 // icmp X, X -> true/false2673 // icmp X, undef -> true/false because undef could be X.2674 if (N1.isUndef() || N2.isUndef() || N1 == N2)2675 return getBoolConstant(ISD::isTrueWhenEqual(Cond), dl, VT, OpVT);2676 }2677 2678 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2)) {2679 const APInt &C2 = N2C->getAPIntValue();2680 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {2681 const APInt &C1 = N1C->getAPIntValue();2682 2683 return getBoolConstant(ICmpInst::compare(C1, C2, getICmpCondCode(Cond)),2684 dl, VT, OpVT);2685 }2686 }2687 2688 auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1);2689 auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2);2690 2691 if (N1CFP && N2CFP) {2692 APFloat::cmpResult R = N1CFP->getValueAPF().compare(N2CFP->getValueAPF());2693 switch (Cond) {2694 default: break;2695 case ISD::SETEQ: if (R==APFloat::cmpUnordered)2696 return GetUndefBooleanConstant();2697 [[fallthrough]];2698 case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT,2699 OpVT);2700 case ISD::SETNE: if (R==APFloat::cmpUnordered)2701 return GetUndefBooleanConstant();2702 [[fallthrough]];2703 case ISD::SETONE: return getBoolConstant(R==APFloat::cmpGreaterThan ||2704 R==APFloat::cmpLessThan, dl, VT,2705 OpVT);2706 case ISD::SETLT: if (R==APFloat::cmpUnordered)2707 return GetUndefBooleanConstant();2708 [[fallthrough]];2709 case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT,2710 OpVT);2711 case ISD::SETGT: if (R==APFloat::cmpUnordered)2712 return GetUndefBooleanConstant();2713 [[fallthrough]];2714 case ISD::SETOGT: return getBoolConstant(R==APFloat::cmpGreaterThan, dl,2715 VT, OpVT);2716 case ISD::SETLE: if (R==APFloat::cmpUnordered)2717 return GetUndefBooleanConstant();2718 [[fallthrough]];2719 case ISD::SETOLE: return getBoolConstant(R==APFloat::cmpLessThan ||2720 R==APFloat::cmpEqual, dl, VT,2721 OpVT);2722 case ISD::SETGE: if (R==APFloat::cmpUnordered)2723 return GetUndefBooleanConstant();2724 [[fallthrough]];2725 case ISD::SETOGE: return getBoolConstant(R==APFloat::cmpGreaterThan ||2726 R==APFloat::cmpEqual, dl, VT, OpVT);2727 case ISD::SETO: return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT,2728 OpVT);2729 case ISD::SETUO: return getBoolConstant(R==APFloat::cmpUnordered, dl, VT,2730 OpVT);2731 case ISD::SETUEQ: return getBoolConstant(R==APFloat::cmpUnordered ||2732 R==APFloat::cmpEqual, dl, VT,2733 OpVT);2734 case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT,2735 OpVT);2736 case ISD::SETULT: return getBoolConstant(R==APFloat::cmpUnordered ||2737 R==APFloat::cmpLessThan, dl, VT,2738 OpVT);2739 case ISD::SETUGT: return getBoolConstant(R==APFloat::cmpGreaterThan ||2740 R==APFloat::cmpUnordered, dl, VT,2741 OpVT);2742 case ISD::SETULE: return getBoolConstant(R!=APFloat::cmpGreaterThan, dl,2743 VT, OpVT);2744 case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT,2745 OpVT);2746 }2747 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {2748 // Ensure that the constant occurs on the RHS.2749 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);2750 if (!TLI->isCondCodeLegal(SwappedCond, OpVT.getSimpleVT()))2751 return SDValue();2752 return getSetCC(dl, VT, N2, N1, SwappedCond);2753 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||2754 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {2755 // If an operand is known to be a nan (or undef that could be a nan), we can2756 // fold it.2757 // Choosing NaN for the undef will always make unordered comparison succeed2758 // and ordered comparison fails.2759 // Matches behavior in llvm::ConstantFoldCompareInstruction.2760 switch (ISD::getUnorderedFlavor(Cond)) {2761 default:2762 llvm_unreachable("Unknown flavor!");2763 case 0: // Known false.2764 return getBoolConstant(false, dl, VT, OpVT);2765 case 1: // Known true.2766 return getBoolConstant(true, dl, VT, OpVT);2767 case 2: // Undefined.2768 return GetUndefBooleanConstant();2769 }2770 }2771 2772 // Could not fold it.2773 return SDValue();2774}2775 2776/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We2777/// use this predicate to simplify operations downstream.2778bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {2779 unsigned BitWidth = Op.getScalarValueSizeInBits();2780 return MaskedValueIsZero(Op, APInt::getSignMask(BitWidth), Depth);2781}2782 2783bool SelectionDAG::SignBitIsZeroFP(SDValue Op, unsigned Depth) const {2784 if (Depth >= MaxRecursionDepth)2785 return false; // Limit search depth.2786 2787 unsigned Opc = Op.getOpcode();2788 switch (Opc) {2789 case ISD::FABS:2790 return true;2791 case ISD::AssertNoFPClass: {2792 FPClassTest NoFPClass =2793 static_cast<FPClassTest>(Op.getConstantOperandVal(1));2794 2795 const FPClassTest TestMask = fcNan | fcNegative;2796 return (NoFPClass & TestMask) == TestMask;2797 }2798 case ISD::ARITH_FENCE:2799 return SignBitIsZeroFP(Op, Depth + 1);2800 case ISD::FEXP:2801 case ISD::FEXP2:2802 case ISD::FEXP10:2803 return Op->getFlags().hasNoNaNs();2804 default:2805 return false;2806 }2807 2808 llvm_unreachable("covered opcode switch");2809}2810 2811/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use2812/// this predicate to simplify operations downstream. Mask is known to be zero2813/// for bits that V cannot have.2814bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,2815 unsigned Depth) const {2816 return Mask.isSubsetOf(computeKnownBits(V, Depth).Zero);2817}2818 2819/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in2820/// DemandedElts. We use this predicate to simplify operations downstream.2821/// Mask is known to be zero for bits that V cannot have.2822bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,2823 const APInt &DemandedElts,2824 unsigned Depth) const {2825 return Mask.isSubsetOf(computeKnownBits(V, DemandedElts, Depth).Zero);2826}2827 2828/// MaskedVectorIsZero - Return true if 'Op' is known to be zero in2829/// DemandedElts. We use this predicate to simplify operations downstream.2830bool SelectionDAG::MaskedVectorIsZero(SDValue V, const APInt &DemandedElts,2831 unsigned Depth /* = 0 */) const {2832 return computeKnownBits(V, DemandedElts, Depth).isZero();2833}2834 2835/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.2836bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,2837 unsigned Depth) const {2838 return Mask.isSubsetOf(computeKnownBits(V, Depth).One);2839}2840 2841APInt SelectionDAG::computeVectorKnownZeroElements(SDValue Op,2842 const APInt &DemandedElts,2843 unsigned Depth) const {2844 EVT VT = Op.getValueType();2845 assert(VT.isVector() && !VT.isScalableVector() && "Only for fixed vectors!");2846 2847 unsigned NumElts = VT.getVectorNumElements();2848 assert(DemandedElts.getBitWidth() == NumElts && "Unexpected demanded mask.");2849 2850 APInt KnownZeroElements = APInt::getZero(NumElts);2851 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {2852 if (!DemandedElts[EltIdx])2853 continue; // Don't query elements that are not demanded.2854 APInt Mask = APInt::getOneBitSet(NumElts, EltIdx);2855 if (MaskedVectorIsZero(Op, Mask, Depth))2856 KnownZeroElements.setBit(EltIdx);2857 }2858 return KnownZeroElements;2859}2860 2861/// isSplatValue - Return true if the vector V has the same value2862/// across all DemandedElts. For scalable vectors, we don't know the2863/// number of lanes at compile time. Instead, we use a 1 bit APInt2864/// to represent a conservative value for all lanes; that is, that2865/// one bit value is implicitly splatted across all lanes.2866bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,2867 APInt &UndefElts, unsigned Depth) const {2868 unsigned Opcode = V.getOpcode();2869 EVT VT = V.getValueType();2870 assert(VT.isVector() && "Vector type expected");2871 assert((!VT.isScalableVector() || DemandedElts.getBitWidth() == 1) &&2872 "scalable demanded bits are ignored");2873 2874 if (!DemandedElts)2875 return false; // No demanded elts, better to assume we don't know anything.2876 2877 if (Depth >= MaxRecursionDepth)2878 return false; // Limit search depth.2879 2880 // Deal with some common cases here that work for both fixed and scalable2881 // vector types.2882 switch (Opcode) {2883 case ISD::SPLAT_VECTOR:2884 UndefElts = V.getOperand(0).isUndef()2885 ? APInt::getAllOnes(DemandedElts.getBitWidth())2886 : APInt(DemandedElts.getBitWidth(), 0);2887 return true;2888 case ISD::ADD:2889 case ISD::SUB:2890 case ISD::AND:2891 case ISD::XOR:2892 case ISD::OR: {2893 APInt UndefLHS, UndefRHS;2894 SDValue LHS = V.getOperand(0);2895 SDValue RHS = V.getOperand(1);2896 // Only recognize splats with the same demanded undef elements for both2897 // operands, otherwise we might fail to handle binop-specific undef2898 // handling.2899 // e.g. (and undef, 0) -> 0 etc.2900 if (isSplatValue(LHS, DemandedElts, UndefLHS, Depth + 1) &&2901 isSplatValue(RHS, DemandedElts, UndefRHS, Depth + 1) &&2902 (DemandedElts & UndefLHS) == (DemandedElts & UndefRHS)) {2903 UndefElts = UndefLHS | UndefRHS;2904 return true;2905 }2906 return false;2907 }2908 case ISD::ABS:2909 case ISD::TRUNCATE:2910 case ISD::SIGN_EXTEND:2911 case ISD::ZERO_EXTEND:2912 return isSplatValue(V.getOperand(0), DemandedElts, UndefElts, Depth + 1);2913 default:2914 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||2915 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)2916 return TLI->isSplatValueForTargetNode(V, DemandedElts, UndefElts, *this,2917 Depth);2918 break;2919 }2920 2921 // We don't support other cases than those above for scalable vectors at2922 // the moment.2923 if (VT.isScalableVector())2924 return false;2925 2926 unsigned NumElts = VT.getVectorNumElements();2927 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");2928 UndefElts = APInt::getZero(NumElts);2929 2930 switch (Opcode) {2931 case ISD::BUILD_VECTOR: {2932 SDValue Scl;2933 for (unsigned i = 0; i != NumElts; ++i) {2934 SDValue Op = V.getOperand(i);2935 if (Op.isUndef()) {2936 UndefElts.setBit(i);2937 continue;2938 }2939 if (!DemandedElts[i])2940 continue;2941 if (Scl && Scl != Op)2942 return false;2943 Scl = Op;2944 }2945 return true;2946 }2947 case ISD::VECTOR_SHUFFLE: {2948 // Check if this is a shuffle node doing a splat or a shuffle of a splat.2949 APInt DemandedLHS = APInt::getZero(NumElts);2950 APInt DemandedRHS = APInt::getZero(NumElts);2951 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(V)->getMask();2952 for (int i = 0; i != (int)NumElts; ++i) {2953 int M = Mask[i];2954 if (M < 0) {2955 UndefElts.setBit(i);2956 continue;2957 }2958 if (!DemandedElts[i])2959 continue;2960 if (M < (int)NumElts)2961 DemandedLHS.setBit(M);2962 else2963 DemandedRHS.setBit(M - NumElts);2964 }2965 2966 // If we aren't demanding either op, assume there's no splat.2967 // If we are demanding both ops, assume there's no splat.2968 if ((DemandedLHS.isZero() && DemandedRHS.isZero()) ||2969 (!DemandedLHS.isZero() && !DemandedRHS.isZero()))2970 return false;2971 2972 // See if the demanded elts of the source op is a splat or we only demand2973 // one element, which should always be a splat.2974 // TODO: Handle source ops splats with undefs.2975 auto CheckSplatSrc = [&](SDValue Src, const APInt &SrcElts) {2976 APInt SrcUndefs;2977 return (SrcElts.popcount() == 1) ||2978 (isSplatValue(Src, SrcElts, SrcUndefs, Depth + 1) &&2979 (SrcElts & SrcUndefs).isZero());2980 };2981 if (!DemandedLHS.isZero())2982 return CheckSplatSrc(V.getOperand(0), DemandedLHS);2983 return CheckSplatSrc(V.getOperand(1), DemandedRHS);2984 }2985 case ISD::EXTRACT_SUBVECTOR: {2986 // Offset the demanded elts by the subvector index.2987 SDValue Src = V.getOperand(0);2988 // We don't support scalable vectors at the moment.2989 if (Src.getValueType().isScalableVector())2990 return false;2991 uint64_t Idx = V.getConstantOperandVal(1);2992 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();2993 APInt UndefSrcElts;2994 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);2995 if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) {2996 UndefElts = UndefSrcElts.extractBits(NumElts, Idx);2997 return true;2998 }2999 break;3000 }3001 case ISD::ANY_EXTEND_VECTOR_INREG:3002 case ISD::SIGN_EXTEND_VECTOR_INREG:3003 case ISD::ZERO_EXTEND_VECTOR_INREG: {3004 // Widen the demanded elts by the src element count.3005 SDValue Src = V.getOperand(0);3006 // We don't support scalable vectors at the moment.3007 if (Src.getValueType().isScalableVector())3008 return false;3009 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();3010 APInt UndefSrcElts;3011 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts);3012 if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) {3013 UndefElts = UndefSrcElts.trunc(NumElts);3014 return true;3015 }3016 break;3017 }3018 case ISD::BITCAST: {3019 SDValue Src = V.getOperand(0);3020 EVT SrcVT = Src.getValueType();3021 unsigned SrcBitWidth = SrcVT.getScalarSizeInBits();3022 unsigned BitWidth = VT.getScalarSizeInBits();3023 3024 // Ignore bitcasts from unsupported types.3025 // TODO: Add fp support?3026 if (!SrcVT.isVector() || !SrcVT.isInteger() || !VT.isInteger())3027 break;3028 3029 // Bitcast 'small element' vector to 'large element' vector.3030 if ((BitWidth % SrcBitWidth) == 0) {3031 // See if each sub element is a splat.3032 unsigned Scale = BitWidth / SrcBitWidth;3033 unsigned NumSrcElts = SrcVT.getVectorNumElements();3034 APInt ScaledDemandedElts =3035 APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);3036 for (unsigned I = 0; I != Scale; ++I) {3037 APInt SubUndefElts;3038 APInt SubDemandedElt = APInt::getOneBitSet(Scale, I);3039 APInt SubDemandedElts = APInt::getSplat(NumSrcElts, SubDemandedElt);3040 SubDemandedElts &= ScaledDemandedElts;3041 if (!isSplatValue(Src, SubDemandedElts, SubUndefElts, Depth + 1))3042 return false;3043 // TODO: Add support for merging sub undef elements.3044 if (!SubUndefElts.isZero())3045 return false;3046 }3047 return true;3048 }3049 break;3050 }3051 }3052 3053 return false;3054}3055 3056/// Helper wrapper to main isSplatValue function.3057bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const {3058 EVT VT = V.getValueType();3059 assert(VT.isVector() && "Vector type expected");3060 3061 APInt UndefElts;3062 // Since the number of lanes in a scalable vector is unknown at compile time,3063 // we track one bit which is implicitly broadcast to all lanes. This means3064 // that all lanes in a scalable vector are considered demanded.3065 APInt DemandedElts3066 = APInt::getAllOnes(VT.isScalableVector() ? 1 : VT.getVectorNumElements());3067 return isSplatValue(V, DemandedElts, UndefElts) &&3068 (AllowUndefs || !UndefElts);3069}3070 3071SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {3072 V = peekThroughExtractSubvectors(V);3073 3074 EVT VT = V.getValueType();3075 unsigned Opcode = V.getOpcode();3076 switch (Opcode) {3077 default: {3078 APInt UndefElts;3079 // Since the number of lanes in a scalable vector is unknown at compile time,3080 // we track one bit which is implicitly broadcast to all lanes. This means3081 // that all lanes in a scalable vector are considered demanded.3082 APInt DemandedElts3083 = APInt::getAllOnes(VT.isScalableVector() ? 1 : VT.getVectorNumElements());3084 3085 if (isSplatValue(V, DemandedElts, UndefElts)) {3086 if (VT.isScalableVector()) {3087 // DemandedElts and UndefElts are ignored for scalable vectors, since3088 // the only supported cases are SPLAT_VECTOR nodes.3089 SplatIdx = 0;3090 } else {3091 // Handle case where all demanded elements are UNDEF.3092 if (DemandedElts.isSubsetOf(UndefElts)) {3093 SplatIdx = 0;3094 return getUNDEF(VT);3095 }3096 SplatIdx = (UndefElts & DemandedElts).countr_one();3097 }3098 return V;3099 }3100 break;3101 }3102 case ISD::SPLAT_VECTOR:3103 SplatIdx = 0;3104 return V;3105 case ISD::VECTOR_SHUFFLE: {3106 assert(!VT.isScalableVector());3107 // Check if this is a shuffle node doing a splat.3108 // TODO - remove this and rely purely on SelectionDAG::isSplatValue,3109 // getTargetVShiftNode currently struggles without the splat source.3110 auto *SVN = cast<ShuffleVectorSDNode>(V);3111 if (!SVN->isSplat())3112 break;3113 int Idx = SVN->getSplatIndex();3114 int NumElts = V.getValueType().getVectorNumElements();3115 SplatIdx = Idx % NumElts;3116 return V.getOperand(Idx / NumElts);3117 }3118 }3119 3120 return SDValue();3121}3122 3123SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) {3124 int SplatIdx;3125 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {3126 EVT SVT = SrcVector.getValueType().getScalarType();3127 EVT LegalSVT = SVT;3128 if (LegalTypes && !TLI->isTypeLegal(SVT)) {3129 if (!SVT.isInteger())3130 return SDValue();3131 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);3132 if (LegalSVT.bitsLT(SVT))3133 return SDValue();3134 }3135 return getExtractVectorElt(SDLoc(V), LegalSVT, SrcVector, SplatIdx);3136 }3137 return SDValue();3138}3139 3140std::optional<ConstantRange>3141SelectionDAG::getValidShiftAmountRange(SDValue V, const APInt &DemandedElts,3142 unsigned Depth) const {3143 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||3144 V.getOpcode() == ISD::SRA) &&3145 "Unknown shift node");3146 // Shifting more than the bitwidth is not valid.3147 unsigned BitWidth = V.getScalarValueSizeInBits();3148 3149 if (auto *Cst = dyn_cast<ConstantSDNode>(V.getOperand(1))) {3150 const APInt &ShAmt = Cst->getAPIntValue();3151 if (ShAmt.uge(BitWidth))3152 return std::nullopt;3153 return ConstantRange(ShAmt);3154 }3155 3156 if (auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1))) {3157 const APInt *MinAmt = nullptr, *MaxAmt = nullptr;3158 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {3159 if (!DemandedElts[i])3160 continue;3161 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));3162 if (!SA) {3163 MinAmt = MaxAmt = nullptr;3164 break;3165 }3166 const APInt &ShAmt = SA->getAPIntValue();3167 if (ShAmt.uge(BitWidth))3168 return std::nullopt;3169 if (!MinAmt || MinAmt->ugt(ShAmt))3170 MinAmt = &ShAmt;3171 if (!MaxAmt || MaxAmt->ult(ShAmt))3172 MaxAmt = &ShAmt;3173 }3174 assert(((!MinAmt && !MaxAmt) || (MinAmt && MaxAmt)) &&3175 "Failed to find matching min/max shift amounts");3176 if (MinAmt && MaxAmt)3177 return ConstantRange(*MinAmt, *MaxAmt + 1);3178 }3179 3180 // Use computeKnownBits to find a hidden constant/knownbits (usually type3181 // legalized). e.g. Hidden behind multiple bitcasts/build_vector/casts etc.3182 KnownBits KnownAmt = computeKnownBits(V.getOperand(1), DemandedElts, Depth);3183 if (KnownAmt.getMaxValue().ult(BitWidth))3184 return ConstantRange::fromKnownBits(KnownAmt, /*IsSigned=*/false);3185 3186 return std::nullopt;3187}3188 3189std::optional<unsigned>3190SelectionDAG::getValidShiftAmount(SDValue V, const APInt &DemandedElts,3191 unsigned Depth) const {3192 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||3193 V.getOpcode() == ISD::SRA) &&3194 "Unknown shift node");3195 if (std::optional<ConstantRange> AmtRange =3196 getValidShiftAmountRange(V, DemandedElts, Depth))3197 if (const APInt *ShAmt = AmtRange->getSingleElement())3198 return ShAmt->getZExtValue();3199 return std::nullopt;3200}3201 3202std::optional<unsigned>3203SelectionDAG::getValidShiftAmount(SDValue V, unsigned Depth) const {3204 EVT VT = V.getValueType();3205 APInt DemandedElts = VT.isFixedLengthVector()3206 ? APInt::getAllOnes(VT.getVectorNumElements())3207 : APInt(1, 1);3208 return getValidShiftAmount(V, DemandedElts, Depth);3209}3210 3211std::optional<unsigned>3212SelectionDAG::getValidMinimumShiftAmount(SDValue V, const APInt &DemandedElts,3213 unsigned Depth) const {3214 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||3215 V.getOpcode() == ISD::SRA) &&3216 "Unknown shift node");3217 if (std::optional<ConstantRange> AmtRange =3218 getValidShiftAmountRange(V, DemandedElts, Depth))3219 return AmtRange->getUnsignedMin().getZExtValue();3220 return std::nullopt;3221}3222 3223std::optional<unsigned>3224SelectionDAG::getValidMinimumShiftAmount(SDValue V, unsigned Depth) const {3225 EVT VT = V.getValueType();3226 APInt DemandedElts = VT.isFixedLengthVector()3227 ? APInt::getAllOnes(VT.getVectorNumElements())3228 : APInt(1, 1);3229 return getValidMinimumShiftAmount(V, DemandedElts, Depth);3230}3231 3232std::optional<unsigned>3233SelectionDAG::getValidMaximumShiftAmount(SDValue V, const APInt &DemandedElts,3234 unsigned Depth) const {3235 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||3236 V.getOpcode() == ISD::SRA) &&3237 "Unknown shift node");3238 if (std::optional<ConstantRange> AmtRange =3239 getValidShiftAmountRange(V, DemandedElts, Depth))3240 return AmtRange->getUnsignedMax().getZExtValue();3241 return std::nullopt;3242}3243 3244std::optional<unsigned>3245SelectionDAG::getValidMaximumShiftAmount(SDValue V, unsigned Depth) const {3246 EVT VT = V.getValueType();3247 APInt DemandedElts = VT.isFixedLengthVector()3248 ? APInt::getAllOnes(VT.getVectorNumElements())3249 : APInt(1, 1);3250 return getValidMaximumShiftAmount(V, DemandedElts, Depth);3251}3252 3253/// Determine which bits of Op are known to be either zero or one and return3254/// them in Known. For vectors, the known bits are those that are shared by3255/// every vector element.3256KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {3257 EVT VT = Op.getValueType();3258 3259 // Since the number of lanes in a scalable vector is unknown at compile time,3260 // we track one bit which is implicitly broadcast to all lanes. This means3261 // that all lanes in a scalable vector are considered demanded.3262 APInt DemandedElts = VT.isFixedLengthVector()3263 ? APInt::getAllOnes(VT.getVectorNumElements())3264 : APInt(1, 1);3265 return computeKnownBits(Op, DemandedElts, Depth);3266}3267 3268/// Determine which bits of Op are known to be either zero or one and return3269/// them in Known. The DemandedElts argument allows us to only collect the known3270/// bits that are shared by the requested vector elements.3271KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,3272 unsigned Depth) const {3273 unsigned BitWidth = Op.getScalarValueSizeInBits();3274 3275 KnownBits Known(BitWidth); // Don't know anything.3276 3277 if (auto OptAPInt = Op->bitcastToAPInt()) {3278 // We know all of the bits for a constant!3279 return KnownBits::makeConstant(*std::move(OptAPInt));3280 }3281 3282 if (Depth >= MaxRecursionDepth)3283 return Known; // Limit search depth.3284 3285 KnownBits Known2;3286 unsigned NumElts = DemandedElts.getBitWidth();3287 assert((!Op.getValueType().isFixedLengthVector() ||3288 NumElts == Op.getValueType().getVectorNumElements()) &&3289 "Unexpected vector size");3290 3291 if (!DemandedElts)3292 return Known; // No demanded elts, better to assume we don't know anything.3293 3294 unsigned Opcode = Op.getOpcode();3295 switch (Opcode) {3296 case ISD::MERGE_VALUES:3297 return computeKnownBits(Op.getOperand(Op.getResNo()), DemandedElts,3298 Depth + 1);3299 case ISD::SPLAT_VECTOR: {3300 SDValue SrcOp = Op.getOperand(0);3301 assert(SrcOp.getValueSizeInBits() >= BitWidth &&3302 "Expected SPLAT_VECTOR implicit truncation");3303 // Implicitly truncate the bits to match the official semantics of3304 // SPLAT_VECTOR.3305 Known = computeKnownBits(SrcOp, Depth + 1).trunc(BitWidth);3306 break;3307 }3308 case ISD::SPLAT_VECTOR_PARTS: {3309 unsigned ScalarSize = Op.getOperand(0).getScalarValueSizeInBits();3310 assert(ScalarSize * Op.getNumOperands() == BitWidth &&3311 "Expected SPLAT_VECTOR_PARTS scalars to cover element width");3312 for (auto [I, SrcOp] : enumerate(Op->ops())) {3313 Known.insertBits(computeKnownBits(SrcOp, Depth + 1), ScalarSize * I);3314 }3315 break;3316 }3317 case ISD::STEP_VECTOR: {3318 const APInt &Step = Op.getConstantOperandAPInt(0);3319 3320 if (Step.isPowerOf2())3321 Known.Zero.setLowBits(Step.logBase2());3322 3323 const Function &F = getMachineFunction().getFunction();3324 3325 if (!isUIntN(BitWidth, Op.getValueType().getVectorMinNumElements()))3326 break;3327 const APInt MinNumElts =3328 APInt(BitWidth, Op.getValueType().getVectorMinNumElements());3329 3330 bool Overflow;3331 const APInt MaxNumElts = getVScaleRange(&F, BitWidth)3332 .getUnsignedMax()3333 .umul_ov(MinNumElts, Overflow);3334 if (Overflow)3335 break;3336 3337 const APInt MaxValue = (MaxNumElts - 1).umul_ov(Step, Overflow);3338 if (Overflow)3339 break;3340 3341 Known.Zero.setHighBits(MaxValue.countl_zero());3342 break;3343 }3344 case ISD::BUILD_VECTOR:3345 assert(!Op.getValueType().isScalableVector());3346 // Collect the known bits that are shared by every demanded vector element.3347 Known.setAllConflict();3348 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {3349 if (!DemandedElts[i])3350 continue;3351 3352 SDValue SrcOp = Op.getOperand(i);3353 Known2 = computeKnownBits(SrcOp, Depth + 1);3354 3355 // BUILD_VECTOR can implicitly truncate sources, we must handle this.3356 if (SrcOp.getValueSizeInBits() != BitWidth) {3357 assert(SrcOp.getValueSizeInBits() > BitWidth &&3358 "Expected BUILD_VECTOR implicit truncation");3359 Known2 = Known2.trunc(BitWidth);3360 }3361 3362 // Known bits are the values that are shared by every demanded element.3363 Known = Known.intersectWith(Known2);3364 3365 // If we don't know any bits, early out.3366 if (Known.isUnknown())3367 break;3368 }3369 break;3370 case ISD::VECTOR_COMPRESS: {3371 SDValue Vec = Op.getOperand(0);3372 SDValue PassThru = Op.getOperand(2);3373 Known = computeKnownBits(PassThru, DemandedElts, Depth + 1);3374 // If we don't know any bits, early out.3375 if (Known.isUnknown())3376 break;3377 Known2 = computeKnownBits(Vec, Depth + 1);3378 Known = Known.intersectWith(Known2);3379 break;3380 }3381 case ISD::VECTOR_SHUFFLE: {3382 assert(!Op.getValueType().isScalableVector());3383 // Collect the known bits that are shared by every vector element referenced3384 // by the shuffle.3385 APInt DemandedLHS, DemandedRHS;3386 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);3387 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");3388 if (!getShuffleDemandedElts(NumElts, SVN->getMask(), DemandedElts,3389 DemandedLHS, DemandedRHS))3390 break;3391 3392 // Known bits are the values that are shared by every demanded element.3393 Known.setAllConflict();3394 if (!!DemandedLHS) {3395 SDValue LHS = Op.getOperand(0);3396 Known2 = computeKnownBits(LHS, DemandedLHS, Depth + 1);3397 Known = Known.intersectWith(Known2);3398 }3399 // If we don't know any bits, early out.3400 if (Known.isUnknown())3401 break;3402 if (!!DemandedRHS) {3403 SDValue RHS = Op.getOperand(1);3404 Known2 = computeKnownBits(RHS, DemandedRHS, Depth + 1);3405 Known = Known.intersectWith(Known2);3406 }3407 break;3408 }3409 case ISD::VSCALE: {3410 const Function &F = getMachineFunction().getFunction();3411 const APInt &Multiplier = Op.getConstantOperandAPInt(0);3412 Known = getVScaleRange(&F, BitWidth).multiply(Multiplier).toKnownBits();3413 break;3414 }3415 case ISD::CONCAT_VECTORS: {3416 if (Op.getValueType().isScalableVector())3417 break;3418 // Split DemandedElts and test each of the demanded subvectors.3419 Known.setAllConflict();3420 EVT SubVectorVT = Op.getOperand(0).getValueType();3421 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();3422 unsigned NumSubVectors = Op.getNumOperands();3423 for (unsigned i = 0; i != NumSubVectors; ++i) {3424 APInt DemandedSub =3425 DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts);3426 if (!!DemandedSub) {3427 SDValue Sub = Op.getOperand(i);3428 Known2 = computeKnownBits(Sub, DemandedSub, Depth + 1);3429 Known = Known.intersectWith(Known2);3430 }3431 // If we don't know any bits, early out.3432 if (Known.isUnknown())3433 break;3434 }3435 break;3436 }3437 case ISD::INSERT_SUBVECTOR: {3438 if (Op.getValueType().isScalableVector())3439 break;3440 // Demand any elements from the subvector and the remainder from the src its3441 // inserted into.3442 SDValue Src = Op.getOperand(0);3443 SDValue Sub = Op.getOperand(1);3444 uint64_t Idx = Op.getConstantOperandVal(2);3445 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();3446 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);3447 APInt DemandedSrcElts = DemandedElts;3448 DemandedSrcElts.clearBits(Idx, Idx + NumSubElts);3449 3450 Known.setAllConflict();3451 if (!!DemandedSubElts) {3452 Known = computeKnownBits(Sub, DemandedSubElts, Depth + 1);3453 if (Known.isUnknown())3454 break; // early-out.3455 }3456 if (!!DemandedSrcElts) {3457 Known2 = computeKnownBits(Src, DemandedSrcElts, Depth + 1);3458 Known = Known.intersectWith(Known2);3459 }3460 break;3461 }3462 case ISD::EXTRACT_SUBVECTOR: {3463 // Offset the demanded elts by the subvector index.3464 SDValue Src = Op.getOperand(0);3465 // Bail until we can represent demanded elements for scalable vectors.3466 if (Op.getValueType().isScalableVector() || Src.getValueType().isScalableVector())3467 break;3468 uint64_t Idx = Op.getConstantOperandVal(1);3469 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();3470 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);3471 Known = computeKnownBits(Src, DemandedSrcElts, Depth + 1);3472 break;3473 }3474 case ISD::SCALAR_TO_VECTOR: {3475 if (Op.getValueType().isScalableVector())3476 break;3477 // We know about scalar_to_vector as much as we know about it source,3478 // which becomes the first element of otherwise unknown vector.3479 if (DemandedElts != 1)3480 break;3481 3482 SDValue N0 = Op.getOperand(0);3483 Known = computeKnownBits(N0, Depth + 1);3484 if (N0.getValueSizeInBits() != BitWidth)3485 Known = Known.trunc(BitWidth);3486 3487 break;3488 }3489 case ISD::BITCAST: {3490 if (Op.getValueType().isScalableVector())3491 break;3492 3493 SDValue N0 = Op.getOperand(0);3494 EVT SubVT = N0.getValueType();3495 unsigned SubBitWidth = SubVT.getScalarSizeInBits();3496 3497 // Ignore bitcasts from unsupported types.3498 if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))3499 break;3500 3501 // Fast handling of 'identity' bitcasts.3502 if (BitWidth == SubBitWidth) {3503 Known = computeKnownBits(N0, DemandedElts, Depth + 1);3504 break;3505 }3506 3507 bool IsLE = getDataLayout().isLittleEndian();3508 3509 // Bitcast 'small element' vector to 'large element' scalar/vector.3510 if ((BitWidth % SubBitWidth) == 0) {3511 assert(N0.getValueType().isVector() && "Expected bitcast from vector");3512 3513 // Collect known bits for the (larger) output by collecting the known3514 // bits from each set of sub elements and shift these into place.3515 // We need to separately call computeKnownBits for each set of3516 // sub elements as the knownbits for each is likely to be different.3517 unsigned SubScale = BitWidth / SubBitWidth;3518 APInt SubDemandedElts(NumElts * SubScale, 0);3519 for (unsigned i = 0; i != NumElts; ++i)3520 if (DemandedElts[i])3521 SubDemandedElts.setBit(i * SubScale);3522 3523 for (unsigned i = 0; i != SubScale; ++i) {3524 Known2 = computeKnownBits(N0, SubDemandedElts.shl(i),3525 Depth + 1);3526 unsigned Shifts = IsLE ? i : SubScale - 1 - i;3527 Known.insertBits(Known2, SubBitWidth * Shifts);3528 }3529 }3530 3531 // Bitcast 'large element' scalar/vector to 'small element' vector.3532 if ((SubBitWidth % BitWidth) == 0) {3533 assert(Op.getValueType().isVector() && "Expected bitcast to vector");3534 3535 // Collect known bits for the (smaller) output by collecting the known3536 // bits from the overlapping larger input elements and extracting the3537 // sub sections we actually care about.3538 unsigned SubScale = SubBitWidth / BitWidth;3539 APInt SubDemandedElts =3540 APIntOps::ScaleBitMask(DemandedElts, NumElts / SubScale);3541 Known2 = computeKnownBits(N0, SubDemandedElts, Depth + 1);3542 3543 Known.setAllConflict();3544 for (unsigned i = 0; i != NumElts; ++i)3545 if (DemandedElts[i]) {3546 unsigned Shifts = IsLE ? i : NumElts - 1 - i;3547 unsigned Offset = (Shifts % SubScale) * BitWidth;3548 Known = Known.intersectWith(Known2.extractBits(BitWidth, Offset));3549 // If we don't know any bits, early out.3550 if (Known.isUnknown())3551 break;3552 }3553 }3554 break;3555 }3556 case ISD::AND:3557 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);3558 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3559 3560 Known &= Known2;3561 break;3562 case ISD::OR:3563 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);3564 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3565 3566 Known |= Known2;3567 break;3568 case ISD::XOR:3569 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);3570 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3571 3572 Known ^= Known2;3573 break;3574 case ISD::MUL: {3575 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);3576 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3577 bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1);3578 // TODO: SelfMultiply can be poison, but not undef.3579 if (SelfMultiply)3580 SelfMultiply &= isGuaranteedNotToBeUndefOrPoison(3581 Op.getOperand(0), DemandedElts, false, Depth + 1);3582 Known = KnownBits::mul(Known, Known2, SelfMultiply);3583 3584 // If the multiplication is known not to overflow, the product of a number3585 // with itself is non-negative. Only do this if we didn't already computed3586 // the opposite value for the sign bit.3587 if (Op->getFlags().hasNoSignedWrap() &&3588 Op.getOperand(0) == Op.getOperand(1) &&3589 !Known.isNegative())3590 Known.makeNonNegative();3591 break;3592 }3593 case ISD::MULHU: {3594 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);3595 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3596 Known = KnownBits::mulhu(Known, Known2);3597 break;3598 }3599 case ISD::MULHS: {3600 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);3601 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3602 Known = KnownBits::mulhs(Known, Known2);3603 break;3604 }3605 case ISD::ABDU: {3606 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);3607 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3608 Known = KnownBits::abdu(Known, Known2);3609 break;3610 }3611 case ISD::ABDS: {3612 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);3613 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3614 Known = KnownBits::abds(Known, Known2);3615 unsigned SignBits1 =3616 ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);3617 if (SignBits1 == 1)3618 break;3619 unsigned SignBits0 =3620 ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);3621 Known.Zero.setHighBits(std::min(SignBits0, SignBits1) - 1);3622 break;3623 }3624 case ISD::UMUL_LOHI: {3625 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");3626 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);3627 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3628 bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1);3629 if (Op.getResNo() == 0)3630 Known = KnownBits::mul(Known, Known2, SelfMultiply);3631 else3632 Known = KnownBits::mulhu(Known, Known2);3633 break;3634 }3635 case ISD::SMUL_LOHI: {3636 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");3637 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);3638 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3639 bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1);3640 if (Op.getResNo() == 0)3641 Known = KnownBits::mul(Known, Known2, SelfMultiply);3642 else3643 Known = KnownBits::mulhs(Known, Known2);3644 break;3645 }3646 case ISD::AVGFLOORU: {3647 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3648 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);3649 Known = KnownBits::avgFloorU(Known, Known2);3650 break;3651 }3652 case ISD::AVGCEILU: {3653 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3654 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);3655 Known = KnownBits::avgCeilU(Known, Known2);3656 break;3657 }3658 case ISD::AVGFLOORS: {3659 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3660 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);3661 Known = KnownBits::avgFloorS(Known, Known2);3662 break;3663 }3664 case ISD::AVGCEILS: {3665 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3666 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);3667 Known = KnownBits::avgCeilS(Known, Known2);3668 break;3669 }3670 case ISD::SELECT:3671 case ISD::VSELECT:3672 Known = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);3673 // If we don't know any bits, early out.3674 if (Known.isUnknown())3675 break;3676 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth+1);3677 3678 // Only known if known in both the LHS and RHS.3679 Known = Known.intersectWith(Known2);3680 break;3681 case ISD::SELECT_CC:3682 Known = computeKnownBits(Op.getOperand(3), DemandedElts, Depth+1);3683 // If we don't know any bits, early out.3684 if (Known.isUnknown())3685 break;3686 Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);3687 3688 // Only known if known in both the LHS and RHS.3689 Known = Known.intersectWith(Known2);3690 break;3691 case ISD::SMULO:3692 case ISD::UMULO:3693 if (Op.getResNo() != 1)3694 break;3695 // The boolean result conforms to getBooleanContents.3696 // If we know the result of a setcc has the top bits zero, use this info.3697 // We know that we have an integer-based boolean since these operations3698 // are only available for integer.3699 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==3700 TargetLowering::ZeroOrOneBooleanContent &&3701 BitWidth > 1)3702 Known.Zero.setBitsFrom(1);3703 break;3704 case ISD::SETCC:3705 case ISD::SETCCCARRY:3706 case ISD::STRICT_FSETCC:3707 case ISD::STRICT_FSETCCS: {3708 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;3709 // If we know the result of a setcc has the top bits zero, use this info.3710 if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==3711 TargetLowering::ZeroOrOneBooleanContent &&3712 BitWidth > 1)3713 Known.Zero.setBitsFrom(1);3714 break;3715 }3716 case ISD::SHL: {3717 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3718 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);3719 3720 bool NUW = Op->getFlags().hasNoUnsignedWrap();3721 bool NSW = Op->getFlags().hasNoSignedWrap();3722 3723 bool ShAmtNonZero = Known2.isNonZero();3724 3725 Known = KnownBits::shl(Known, Known2, NUW, NSW, ShAmtNonZero);3726 3727 // Minimum shift low bits are known zero.3728 if (std::optional<unsigned> ShMinAmt =3729 getValidMinimumShiftAmount(Op, DemandedElts, Depth + 1))3730 Known.Zero.setLowBits(*ShMinAmt);3731 break;3732 }3733 case ISD::SRL:3734 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3735 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);3736 Known = KnownBits::lshr(Known, Known2, /*ShAmtNonZero=*/false,3737 Op->getFlags().hasExact());3738 3739 // Minimum shift high bits are known zero.3740 if (std::optional<unsigned> ShMinAmt =3741 getValidMinimumShiftAmount(Op, DemandedElts, Depth + 1))3742 Known.Zero.setHighBits(*ShMinAmt);3743 break;3744 case ISD::SRA:3745 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3746 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);3747 Known = KnownBits::ashr(Known, Known2, /*ShAmtNonZero=*/false,3748 Op->getFlags().hasExact());3749 break;3750 case ISD::ROTL:3751 case ISD::ROTR:3752 if (ConstantSDNode *C =3753 isConstOrConstSplat(Op.getOperand(1), DemandedElts)) {3754 unsigned Amt = C->getAPIntValue().urem(BitWidth);3755 3756 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3757 3758 // Canonicalize to ROTR.3759 if (Opcode == ISD::ROTL && Amt != 0)3760 Amt = BitWidth - Amt;3761 3762 Known.Zero = Known.Zero.rotr(Amt);3763 Known.One = Known.One.rotr(Amt);3764 }3765 break;3766 case ISD::FSHL:3767 case ISD::FSHR:3768 if (ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(2), DemandedElts)) {3769 unsigned Amt = C->getAPIntValue().urem(BitWidth);3770 3771 // For fshl, 0-shift returns the 1st arg.3772 // For fshr, 0-shift returns the 2nd arg.3773 if (Amt == 0) {3774 Known = computeKnownBits(Op.getOperand(Opcode == ISD::FSHL ? 0 : 1),3775 DemandedElts, Depth + 1);3776 break;3777 }3778 3779 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))3780 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))3781 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3782 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);3783 if (Opcode == ISD::FSHL) {3784 Known <<= Amt;3785 Known2 >>= BitWidth - Amt;3786 } else {3787 Known <<= BitWidth - Amt;3788 Known2 >>= Amt;3789 }3790 Known = Known.unionWith(Known2);3791 }3792 break;3793 case ISD::SHL_PARTS:3794 case ISD::SRA_PARTS:3795 case ISD::SRL_PARTS: {3796 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");3797 3798 // Collect lo/hi source values and concatenate.3799 unsigned LoBits = Op.getOperand(0).getScalarValueSizeInBits();3800 unsigned HiBits = Op.getOperand(1).getScalarValueSizeInBits();3801 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3802 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);3803 Known = Known2.concat(Known);3804 3805 // Collect shift amount.3806 Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);3807 3808 if (Opcode == ISD::SHL_PARTS)3809 Known = KnownBits::shl(Known, Known2);3810 else if (Opcode == ISD::SRA_PARTS)3811 Known = KnownBits::ashr(Known, Known2);3812 else // if (Opcode == ISD::SRL_PARTS)3813 Known = KnownBits::lshr(Known, Known2);3814 3815 // TODO: Minimum shift low/high bits are known zero.3816 3817 if (Op.getResNo() == 0)3818 Known = Known.extractBits(LoBits, 0);3819 else3820 Known = Known.extractBits(HiBits, LoBits);3821 break;3822 }3823 case ISD::SIGN_EXTEND_INREG: {3824 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3825 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();3826 Known = Known.sextInReg(EVT.getScalarSizeInBits());3827 break;3828 }3829 case ISD::CTTZ:3830 case ISD::CTTZ_ZERO_UNDEF: {3831 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3832 // If we have a known 1, its position is our upper bound.3833 unsigned PossibleTZ = Known2.countMaxTrailingZeros();3834 unsigned LowBits = llvm::bit_width(PossibleTZ);3835 Known.Zero.setBitsFrom(LowBits);3836 break;3837 }3838 case ISD::CTLZ:3839 case ISD::CTLZ_ZERO_UNDEF: {3840 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3841 // If we have a known 1, its position is our upper bound.3842 unsigned PossibleLZ = Known2.countMaxLeadingZeros();3843 unsigned LowBits = llvm::bit_width(PossibleLZ);3844 Known.Zero.setBitsFrom(LowBits);3845 break;3846 }3847 case ISD::CTPOP: {3848 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3849 // If we know some of the bits are zero, they can't be one.3850 unsigned PossibleOnes = Known2.countMaxPopulation();3851 Known.Zero.setBitsFrom(llvm::bit_width(PossibleOnes));3852 break;3853 }3854 case ISD::PARITY: {3855 // Parity returns 0 everywhere but the LSB.3856 Known.Zero.setBitsFrom(1);3857 break;3858 }3859 case ISD::MGATHER:3860 case ISD::MLOAD: {3861 ISD::LoadExtType ETy =3862 (Opcode == ISD::MGATHER)3863 ? cast<MaskedGatherSDNode>(Op)->getExtensionType()3864 : cast<MaskedLoadSDNode>(Op)->getExtensionType();3865 if (ETy == ISD::ZEXTLOAD) {3866 EVT MemVT = cast<MemSDNode>(Op)->getMemoryVT();3867 KnownBits Known0(MemVT.getScalarSizeInBits());3868 return Known0.zext(BitWidth);3869 }3870 break;3871 }3872 case ISD::LOAD: {3873 LoadSDNode *LD = cast<LoadSDNode>(Op);3874 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);3875 if (ISD::isNON_EXTLoad(LD) && Cst) {3876 // Determine any common known bits from the loaded constant pool value.3877 Type *CstTy = Cst->getType();3878 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits() &&3879 !Op.getValueType().isScalableVector()) {3880 // If its a vector splat, then we can (quickly) reuse the scalar path.3881 // NOTE: We assume all elements match and none are UNDEF.3882 if (CstTy->isVectorTy()) {3883 if (const Constant *Splat = Cst->getSplatValue()) {3884 Cst = Splat;3885 CstTy = Cst->getType();3886 }3887 }3888 // TODO - do we need to handle different bitwidths?3889 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {3890 // Iterate across all vector elements finding common known bits.3891 Known.setAllConflict();3892 for (unsigned i = 0; i != NumElts; ++i) {3893 if (!DemandedElts[i])3894 continue;3895 if (Constant *Elt = Cst->getAggregateElement(i)) {3896 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {3897 const APInt &Value = CInt->getValue();3898 Known.One &= Value;3899 Known.Zero &= ~Value;3900 continue;3901 }3902 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {3903 APInt Value = CFP->getValueAPF().bitcastToAPInt();3904 Known.One &= Value;3905 Known.Zero &= ~Value;3906 continue;3907 }3908 }3909 Known.One.clearAllBits();3910 Known.Zero.clearAllBits();3911 break;3912 }3913 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {3914 if (auto *CInt = dyn_cast<ConstantInt>(Cst)) {3915 Known = KnownBits::makeConstant(CInt->getValue());3916 } else if (auto *CFP = dyn_cast<ConstantFP>(Cst)) {3917 Known =3918 KnownBits::makeConstant(CFP->getValueAPF().bitcastToAPInt());3919 }3920 }3921 }3922 } else if (Op.getResNo() == 0) {3923 unsigned ScalarMemorySize = LD->getMemoryVT().getScalarSizeInBits();3924 KnownBits KnownScalarMemory(ScalarMemorySize);3925 if (const MDNode *MD = LD->getRanges())3926 computeKnownBitsFromRangeMetadata(*MD, KnownScalarMemory);3927 3928 // Extend the Known bits from memory to the size of the scalar result.3929 if (ISD::isZEXTLoad(Op.getNode()))3930 Known = KnownScalarMemory.zext(BitWidth);3931 else if (ISD::isSEXTLoad(Op.getNode()))3932 Known = KnownScalarMemory.sext(BitWidth);3933 else if (ISD::isEXTLoad(Op.getNode()))3934 Known = KnownScalarMemory.anyext(BitWidth);3935 else3936 Known = KnownScalarMemory;3937 assert(Known.getBitWidth() == BitWidth);3938 return Known;3939 }3940 break;3941 }3942 case ISD::ZERO_EXTEND_VECTOR_INREG: {3943 if (Op.getValueType().isScalableVector())3944 break;3945 EVT InVT = Op.getOperand(0).getValueType();3946 APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements());3947 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);3948 Known = Known.zext(BitWidth);3949 break;3950 }3951 case ISD::ZERO_EXTEND: {3952 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3953 Known = Known.zext(BitWidth);3954 break;3955 }3956 case ISD::SIGN_EXTEND_VECTOR_INREG: {3957 if (Op.getValueType().isScalableVector())3958 break;3959 EVT InVT = Op.getOperand(0).getValueType();3960 APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements());3961 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);3962 // If the sign bit is known to be zero or one, then sext will extend3963 // it to the top bits, else it will just zext.3964 Known = Known.sext(BitWidth);3965 break;3966 }3967 case ISD::SIGN_EXTEND: {3968 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3969 // If the sign bit is known to be zero or one, then sext will extend3970 // it to the top bits, else it will just zext.3971 Known = Known.sext(BitWidth);3972 break;3973 }3974 case ISD::ANY_EXTEND_VECTOR_INREG: {3975 if (Op.getValueType().isScalableVector())3976 break;3977 EVT InVT = Op.getOperand(0).getValueType();3978 APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements());3979 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);3980 Known = Known.anyext(BitWidth);3981 break;3982 }3983 case ISD::ANY_EXTEND: {3984 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3985 Known = Known.anyext(BitWidth);3986 break;3987 }3988 case ISD::TRUNCATE: {3989 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3990 Known = Known.trunc(BitWidth);3991 break;3992 }3993 case ISD::AssertZext: {3994 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();3995 APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());3996 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);3997 Known.Zero |= (~InMask);3998 Known.One &= (~Known.Zero);3999 break;4000 }4001 case ISD::AssertAlign: {4002 unsigned LogOfAlign = Log2(cast<AssertAlignSDNode>(Op)->getAlign());4003 assert(LogOfAlign != 0);4004 4005 // TODO: Should use maximum with source4006 // If a node is guaranteed to be aligned, set low zero bits accordingly as4007 // well as clearing one bits.4008 Known.Zero.setLowBits(LogOfAlign);4009 Known.One.clearLowBits(LogOfAlign);4010 break;4011 }4012 case ISD::AssertNoFPClass: {4013 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);4014 4015 FPClassTest NoFPClass =4016 static_cast<FPClassTest>(Op.getConstantOperandVal(1));4017 const FPClassTest NegativeTestMask = fcNan | fcNegative;4018 if ((NoFPClass & NegativeTestMask) == NegativeTestMask) {4019 // Cannot be negative.4020 Known.makeNonNegative();4021 }4022 4023 const FPClassTest PositiveTestMask = fcNan | fcPositive;4024 if ((NoFPClass & PositiveTestMask) == PositiveTestMask) {4025 // Cannot be positive.4026 Known.makeNegative();4027 }4028 4029 break;4030 }4031 case ISD::FGETSIGN:4032 // All bits are zero except the low bit.4033 Known.Zero.setBitsFrom(1);4034 break;4035 case ISD::ADD:4036 case ISD::SUB: {4037 SDNodeFlags Flags = Op.getNode()->getFlags();4038 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);4039 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);4040 Known = KnownBits::computeForAddSub(4041 Op.getOpcode() == ISD::ADD, Flags.hasNoSignedWrap(),4042 Flags.hasNoUnsignedWrap(), Known, Known2);4043 break;4044 }4045 case ISD::USUBO:4046 case ISD::SSUBO:4047 case ISD::USUBO_CARRY:4048 case ISD::SSUBO_CARRY:4049 if (Op.getResNo() == 1) {4050 // If we know the result of a setcc has the top bits zero, use this info.4051 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==4052 TargetLowering::ZeroOrOneBooleanContent &&4053 BitWidth > 1)4054 Known.Zero.setBitsFrom(1);4055 break;4056 }4057 [[fallthrough]];4058 case ISD::SUBC: {4059 assert(Op.getResNo() == 0 &&4060 "We only compute knownbits for the difference here.");4061 4062 // With USUBO_CARRY and SSUBO_CARRY a borrow bit may be added in.4063 KnownBits Borrow(1);4064 if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) {4065 Borrow = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);4066 // Borrow has bit width 14067 Borrow = Borrow.trunc(1);4068 } else {4069 Borrow.setAllZero();4070 }4071 4072 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);4073 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);4074 Known = KnownBits::computeForSubBorrow(Known, Known2, Borrow);4075 break;4076 }4077 case ISD::UADDO:4078 case ISD::SADDO:4079 case ISD::UADDO_CARRY:4080 case ISD::SADDO_CARRY:4081 if (Op.getResNo() == 1) {4082 // If we know the result of a setcc has the top bits zero, use this info.4083 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==4084 TargetLowering::ZeroOrOneBooleanContent &&4085 BitWidth > 1)4086 Known.Zero.setBitsFrom(1);4087 break;4088 }4089 [[fallthrough]];4090 case ISD::ADDC:4091 case ISD::ADDE: {4092 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");4093 4094 // With ADDE and UADDO_CARRY, a carry bit may be added in.4095 KnownBits Carry(1);4096 if (Opcode == ISD::ADDE)4097 // Can't track carry from glue, set carry to unknown.4098 Carry.resetAll();4099 else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) {4100 Carry = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);4101 // Carry has bit width 14102 Carry = Carry.trunc(1);4103 } else {4104 Carry.setAllZero();4105 }4106 4107 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);4108 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);4109 Known = KnownBits::computeForAddCarry(Known, Known2, Carry);4110 break;4111 }4112 case ISD::UDIV: {4113 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);4114 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);4115 Known = KnownBits::udiv(Known, Known2, Op->getFlags().hasExact());4116 break;4117 }4118 case ISD::SDIV: {4119 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);4120 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);4121 Known = KnownBits::sdiv(Known, Known2, Op->getFlags().hasExact());4122 break;4123 }4124 case ISD::SREM: {4125 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);4126 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);4127 Known = KnownBits::srem(Known, Known2);4128 break;4129 }4130 case ISD::UREM: {4131 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);4132 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);4133 Known = KnownBits::urem(Known, Known2);4134 break;4135 }4136 case ISD::EXTRACT_ELEMENT: {4137 Known = computeKnownBits(Op.getOperand(0), Depth+1);4138 const unsigned Index = Op.getConstantOperandVal(1);4139 const unsigned EltBitWidth = Op.getValueSizeInBits();4140 4141 // Remove low part of known bits mask4142 Known.Zero = Known.Zero.getHiBits(Known.getBitWidth() - Index * EltBitWidth);4143 Known.One = Known.One.getHiBits(Known.getBitWidth() - Index * EltBitWidth);4144 4145 // Remove high part of known bit mask4146 Known = Known.trunc(EltBitWidth);4147 break;4148 }4149 case ISD::EXTRACT_VECTOR_ELT: {4150 SDValue InVec = Op.getOperand(0);4151 SDValue EltNo = Op.getOperand(1);4152 EVT VecVT = InVec.getValueType();4153 // computeKnownBits not yet implemented for scalable vectors.4154 if (VecVT.isScalableVector())4155 break;4156 const unsigned EltBitWidth = VecVT.getScalarSizeInBits();4157 const unsigned NumSrcElts = VecVT.getVectorNumElements();4158 4159 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know4160 // anything about the extended bits.4161 if (BitWidth > EltBitWidth)4162 Known = Known.trunc(EltBitWidth);4163 4164 // If we know the element index, just demand that vector element, else for4165 // an unknown element index, ignore DemandedElts and demand them all.4166 APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);4167 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);4168 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))4169 DemandedSrcElts =4170 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());4171 4172 Known = computeKnownBits(InVec, DemandedSrcElts, Depth + 1);4173 if (BitWidth > EltBitWidth)4174 Known = Known.anyext(BitWidth);4175 break;4176 }4177 case ISD::INSERT_VECTOR_ELT: {4178 if (Op.getValueType().isScalableVector())4179 break;4180 4181 // If we know the element index, split the demand between the4182 // source vector and the inserted element, otherwise assume we need4183 // the original demanded vector elements and the value.4184 SDValue InVec = Op.getOperand(0);4185 SDValue InVal = Op.getOperand(1);4186 SDValue EltNo = Op.getOperand(2);4187 bool DemandedVal = true;4188 APInt DemandedVecElts = DemandedElts;4189 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);4190 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {4191 unsigned EltIdx = CEltNo->getZExtValue();4192 DemandedVal = !!DemandedElts[EltIdx];4193 DemandedVecElts.clearBit(EltIdx);4194 }4195 Known.setAllConflict();4196 if (DemandedVal) {4197 Known2 = computeKnownBits(InVal, Depth + 1);4198 Known = Known.intersectWith(Known2.zextOrTrunc(BitWidth));4199 }4200 if (!!DemandedVecElts) {4201 Known2 = computeKnownBits(InVec, DemandedVecElts, Depth + 1);4202 Known = Known.intersectWith(Known2);4203 }4204 break;4205 }4206 case ISD::BITREVERSE: {4207 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);4208 Known = Known2.reverseBits();4209 break;4210 }4211 case ISD::BSWAP: {4212 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);4213 Known = Known2.byteSwap();4214 break;4215 }4216 case ISD::ABS: {4217 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);4218 Known = Known2.abs();4219 Known.Zero.setHighBits(4220 ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1) - 1);4221 break;4222 }4223 case ISD::USUBSAT: {4224 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);4225 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);4226 Known = KnownBits::usub_sat(Known, Known2);4227 break;4228 }4229 case ISD::UMIN: {4230 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);4231 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);4232 Known = KnownBits::umin(Known, Known2);4233 break;4234 }4235 case ISD::UMAX: {4236 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);4237 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);4238 Known = KnownBits::umax(Known, Known2);4239 break;4240 }4241 case ISD::SMIN:4242 case ISD::SMAX: {4243 // If we have a clamp pattern, we know that the number of sign bits will be4244 // the minimum of the clamp min/max range.4245 bool IsMax = (Opcode == ISD::SMAX);4246 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;4247 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))4248 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))4249 CstHigh =4250 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);4251 if (CstLow && CstHigh) {4252 if (!IsMax)4253 std::swap(CstLow, CstHigh);4254 4255 const APInt &ValueLow = CstLow->getAPIntValue();4256 const APInt &ValueHigh = CstHigh->getAPIntValue();4257 if (ValueLow.sle(ValueHigh)) {4258 unsigned LowSignBits = ValueLow.getNumSignBits();4259 unsigned HighSignBits = ValueHigh.getNumSignBits();4260 unsigned MinSignBits = std::min(LowSignBits, HighSignBits);4261 if (ValueLow.isNegative() && ValueHigh.isNegative()) {4262 Known.One.setHighBits(MinSignBits);4263 break;4264 }4265 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {4266 Known.Zero.setHighBits(MinSignBits);4267 break;4268 }4269 }4270 }4271 4272 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);4273 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);4274 if (IsMax)4275 Known = KnownBits::smax(Known, Known2);4276 else4277 Known = KnownBits::smin(Known, Known2);4278 4279 // For SMAX, if CstLow is non-negative we know the result will be4280 // non-negative and thus all sign bits are 0.4281 // TODO: There's an equivalent of this for smin with negative constant for4282 // known ones.4283 if (IsMax && CstLow) {4284 const APInt &ValueLow = CstLow->getAPIntValue();4285 if (ValueLow.isNonNegative()) {4286 unsigned SignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);4287 Known.Zero.setHighBits(std::min(SignBits, ValueLow.getNumSignBits()));4288 }4289 }4290 4291 break;4292 }4293 case ISD::UINT_TO_FP: {4294 Known.makeNonNegative();4295 break;4296 }4297 case ISD::SINT_TO_FP: {4298 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);4299 if (Known2.isNonNegative())4300 Known.makeNonNegative();4301 else if (Known2.isNegative())4302 Known.makeNegative();4303 break;4304 }4305 case ISD::FP_TO_UINT_SAT: {4306 // FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT.4307 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();4308 Known.Zero |= APInt::getBitsSetFrom(BitWidth, VT.getScalarSizeInBits());4309 break;4310 }4311 case ISD::ATOMIC_LOAD: {4312 // If we are looking at the loaded value.4313 if (Op.getResNo() == 0) {4314 auto *AT = cast<AtomicSDNode>(Op);4315 unsigned ScalarMemorySize = AT->getMemoryVT().getScalarSizeInBits();4316 KnownBits KnownScalarMemory(ScalarMemorySize);4317 if (const MDNode *MD = AT->getRanges())4318 computeKnownBitsFromRangeMetadata(*MD, KnownScalarMemory);4319 4320 switch (AT->getExtensionType()) {4321 case ISD::ZEXTLOAD:4322 Known = KnownScalarMemory.zext(BitWidth);4323 break;4324 case ISD::SEXTLOAD:4325 Known = KnownScalarMemory.sext(BitWidth);4326 break;4327 case ISD::EXTLOAD:4328 switch (TLI->getExtendForAtomicOps()) {4329 case ISD::ZERO_EXTEND:4330 Known = KnownScalarMemory.zext(BitWidth);4331 break;4332 case ISD::SIGN_EXTEND:4333 Known = KnownScalarMemory.sext(BitWidth);4334 break;4335 default:4336 Known = KnownScalarMemory.anyext(BitWidth);4337 break;4338 }4339 break;4340 case ISD::NON_EXTLOAD:4341 Known = KnownScalarMemory;4342 break;4343 }4344 assert(Known.getBitWidth() == BitWidth);4345 }4346 break;4347 }4348 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:4349 if (Op.getResNo() == 1) {4350 // The boolean result conforms to getBooleanContents.4351 // If we know the result of a setcc has the top bits zero, use this info.4352 // We know that we have an integer-based boolean since these operations4353 // are only available for integer.4354 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==4355 TargetLowering::ZeroOrOneBooleanContent &&4356 BitWidth > 1)4357 Known.Zero.setBitsFrom(1);4358 break;4359 }4360 [[fallthrough]];4361 case ISD::ATOMIC_CMP_SWAP:4362 case ISD::ATOMIC_SWAP:4363 case ISD::ATOMIC_LOAD_ADD:4364 case ISD::ATOMIC_LOAD_SUB:4365 case ISD::ATOMIC_LOAD_AND:4366 case ISD::ATOMIC_LOAD_CLR:4367 case ISD::ATOMIC_LOAD_OR:4368 case ISD::ATOMIC_LOAD_XOR:4369 case ISD::ATOMIC_LOAD_NAND:4370 case ISD::ATOMIC_LOAD_MIN:4371 case ISD::ATOMIC_LOAD_MAX:4372 case ISD::ATOMIC_LOAD_UMIN:4373 case ISD::ATOMIC_LOAD_UMAX: {4374 // If we are looking at the loaded value.4375 if (Op.getResNo() == 0) {4376 auto *AT = cast<AtomicSDNode>(Op);4377 unsigned MemBits = AT->getMemoryVT().getScalarSizeInBits();4378 4379 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)4380 Known.Zero.setBitsFrom(MemBits);4381 }4382 break;4383 }4384 case ISD::FrameIndex:4385 case ISD::TargetFrameIndex:4386 TLI->computeKnownBitsForFrameIndex(cast<FrameIndexSDNode>(Op)->getIndex(),4387 Known, getMachineFunction());4388 break;4389 4390 default:4391 if (Opcode < ISD::BUILTIN_OP_END)4392 break;4393 [[fallthrough]];4394 case ISD::INTRINSIC_WO_CHAIN:4395 case ISD::INTRINSIC_W_CHAIN:4396 case ISD::INTRINSIC_VOID:4397 // TODO: Probably okay to remove after audit; here to reduce change size4398 // in initial enablement patch for scalable vectors4399 if (Op.getValueType().isScalableVector())4400 break;4401 4402 // Allow the target to implement this method for its nodes.4403 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth);4404 break;4405 }4406 4407 return Known;4408}4409 4410/// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind.4411static SelectionDAG::OverflowKind mapOverflowResult(ConstantRange::OverflowResult OR) {4412 switch (OR) {4413 case ConstantRange::OverflowResult::MayOverflow:4414 return SelectionDAG::OFK_Sometime;4415 case ConstantRange::OverflowResult::AlwaysOverflowsLow:4416 case ConstantRange::OverflowResult::AlwaysOverflowsHigh:4417 return SelectionDAG::OFK_Always;4418 case ConstantRange::OverflowResult::NeverOverflows:4419 return SelectionDAG::OFK_Never;4420 }4421 llvm_unreachable("Unknown OverflowResult");4422}4423 4424SelectionDAG::OverflowKind4425SelectionDAG::computeOverflowForSignedAdd(SDValue N0, SDValue N1) const {4426 // X + 0 never overflow4427 if (isNullConstant(N1))4428 return OFK_Never;4429 4430 // If both operands each have at least two sign bits, the addition4431 // cannot overflow.4432 if (ComputeNumSignBits(N0) > 1 && ComputeNumSignBits(N1) > 1)4433 return OFK_Never;4434 4435 // TODO: Add ConstantRange::signedAddMayOverflow handling.4436 return OFK_Sometime;4437}4438 4439SelectionDAG::OverflowKind4440SelectionDAG::computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const {4441 // X + 0 never overflow4442 if (isNullConstant(N1))4443 return OFK_Never;4444 4445 // mulhi + 1 never overflow4446 KnownBits N1Known = computeKnownBits(N1);4447 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&4448 N1Known.getMaxValue().ult(2))4449 return OFK_Never;4450 4451 KnownBits N0Known = computeKnownBits(N0);4452 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 &&4453 N0Known.getMaxValue().ult(2))4454 return OFK_Never;4455 4456 // Fallback to ConstantRange::unsignedAddMayOverflow handling.4457 ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, false);4458 ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, false);4459 return mapOverflowResult(N0Range.unsignedAddMayOverflow(N1Range));4460}4461 4462SelectionDAG::OverflowKind4463SelectionDAG::computeOverflowForSignedSub(SDValue N0, SDValue N1) const {4464 // X - 0 never overflow4465 if (isNullConstant(N1))4466 return OFK_Never;4467 4468 // If both operands each have at least two sign bits, the subtraction4469 // cannot overflow.4470 if (ComputeNumSignBits(N0) > 1 && ComputeNumSignBits(N1) > 1)4471 return OFK_Never;4472 4473 KnownBits N0Known = computeKnownBits(N0);4474 KnownBits N1Known = computeKnownBits(N1);4475 ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, true);4476 ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, true);4477 return mapOverflowResult(N0Range.signedSubMayOverflow(N1Range));4478}4479 4480SelectionDAG::OverflowKind4481SelectionDAG::computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const {4482 // X - 0 never overflow4483 if (isNullConstant(N1))4484 return OFK_Never;4485 4486 KnownBits N0Known = computeKnownBits(N0);4487 KnownBits N1Known = computeKnownBits(N1);4488 ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, false);4489 ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, false);4490 return mapOverflowResult(N0Range.unsignedSubMayOverflow(N1Range));4491}4492 4493SelectionDAG::OverflowKind4494SelectionDAG::computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const {4495 // X * 0 and X * 1 never overflow.4496 if (isNullConstant(N1) || isOneConstant(N1))4497 return OFK_Never;4498 4499 KnownBits N0Known = computeKnownBits(N0);4500 KnownBits N1Known = computeKnownBits(N1);4501 ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, false);4502 ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, false);4503 return mapOverflowResult(N0Range.unsignedMulMayOverflow(N1Range));4504}4505 4506SelectionDAG::OverflowKind4507SelectionDAG::computeOverflowForSignedMul(SDValue N0, SDValue N1) const {4508 // X * 0 and X * 1 never overflow.4509 if (isNullConstant(N1) || isOneConstant(N1))4510 return OFK_Never;4511 4512 // Get the size of the result.4513 unsigned BitWidth = N0.getScalarValueSizeInBits();4514 4515 // Sum of the sign bits.4516 unsigned SignBits = ComputeNumSignBits(N0) + ComputeNumSignBits(N1);4517 4518 // If we have enough sign bits, then there's no overflow.4519 if (SignBits > BitWidth + 1)4520 return OFK_Never;4521 4522 if (SignBits == BitWidth + 1) {4523 // The overflow occurs when the true multiplication of the4524 // the operands is the minimum negative number.4525 KnownBits N0Known = computeKnownBits(N0);4526 KnownBits N1Known = computeKnownBits(N1);4527 // If one of the operands is non-negative, then there's no4528 // overflow.4529 if (N0Known.isNonNegative() || N1Known.isNonNegative())4530 return OFK_Never;4531 }4532 4533 return OFK_Sometime;4534}4535 4536bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth) const {4537 if (Depth >= MaxRecursionDepth)4538 return false; // Limit search depth.4539 4540 EVT OpVT = Val.getValueType();4541 unsigned BitWidth = OpVT.getScalarSizeInBits();4542 4543 // Is the constant a known power of 2?4544 if (ISD::matchUnaryPredicate(Val, [BitWidth](ConstantSDNode *C) {4545 return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();4546 }))4547 return true;4548 4549 // A left-shift of a constant one will have exactly one bit set because4550 // shifting the bit off the end is undefined.4551 if (Val.getOpcode() == ISD::SHL) {4552 auto *C = isConstOrConstSplat(Val.getOperand(0));4553 if (C && C->getAPIntValue() == 1)4554 return true;4555 return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1) &&4556 isKnownNeverZero(Val, Depth);4557 }4558 4559 // Similarly, a logical right-shift of a constant sign-bit will have exactly4560 // one bit set.4561 if (Val.getOpcode() == ISD::SRL) {4562 auto *C = isConstOrConstSplat(Val.getOperand(0));4563 if (C && C->getAPIntValue().isSignMask())4564 return true;4565 return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1) &&4566 isKnownNeverZero(Val, Depth);4567 }4568 4569 if (Val.getOpcode() == ISD::ROTL || Val.getOpcode() == ISD::ROTR)4570 return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1);4571 4572 // Are all operands of a build vector constant powers of two?4573 if (Val.getOpcode() == ISD::BUILD_VECTOR)4574 if (llvm::all_of(Val->ops(), [BitWidth](SDValue E) {4575 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E))4576 return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();4577 return false;4578 }))4579 return true;4580 4581 // Is the operand of a splat vector a constant power of two?4582 if (Val.getOpcode() == ISD::SPLAT_VECTOR)4583 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val->getOperand(0)))4584 if (C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2())4585 return true;4586 4587 // vscale(power-of-two) is a power-of-two for some targets4588 if (Val.getOpcode() == ISD::VSCALE &&4589 getTargetLoweringInfo().isVScaleKnownToBeAPowerOfTwo() &&4590 isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1))4591 return true;4592 4593 if (Val.getOpcode() == ISD::SMIN || Val.getOpcode() == ISD::SMAX ||4594 Val.getOpcode() == ISD::UMIN || Val.getOpcode() == ISD::UMAX)4595 return isKnownToBeAPowerOfTwo(Val.getOperand(1), Depth + 1) &&4596 isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1);4597 4598 if (Val.getOpcode() == ISD::SELECT || Val.getOpcode() == ISD::VSELECT)4599 return isKnownToBeAPowerOfTwo(Val.getOperand(2), Depth + 1) &&4600 isKnownToBeAPowerOfTwo(Val.getOperand(1), Depth + 1);4601 4602 // Looking for `x & -x` pattern:4603 // If x == 0:4604 // x & -x -> 04605 // If x != 0:4606 // x & -x -> non-zero pow24607 // so if we find the pattern return whether we know `x` is non-zero.4608 SDValue X;4609 if (sd_match(Val, m_And(m_Value(X), m_Neg(m_Deferred(X)))))4610 return isKnownNeverZero(X, Depth);4611 4612 if (Val.getOpcode() == ISD::ZERO_EXTEND)4613 return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1);4614 4615 // More could be done here, though the above checks are enough4616 // to handle some common cases.4617 return false;4618}4619 4620bool SelectionDAG::isKnownToBeAPowerOfTwoFP(SDValue Val, unsigned Depth) const {4621 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(Val, true))4622 return C1->getValueAPF().getExactLog2Abs() >= 0;4623 4624 if (Val.getOpcode() == ISD::UINT_TO_FP || Val.getOpcode() == ISD::SINT_TO_FP)4625 return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1);4626 4627 return false;4628}4629 4630unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {4631 EVT VT = Op.getValueType();4632 4633 // Since the number of lanes in a scalable vector is unknown at compile time,4634 // we track one bit which is implicitly broadcast to all lanes. This means4635 // that all lanes in a scalable vector are considered demanded.4636 APInt DemandedElts = VT.isFixedLengthVector()4637 ? APInt::getAllOnes(VT.getVectorNumElements())4638 : APInt(1, 1);4639 return ComputeNumSignBits(Op, DemandedElts, Depth);4640}4641 4642unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,4643 unsigned Depth) const {4644 EVT VT = Op.getValueType();4645 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");4646 unsigned VTBits = VT.getScalarSizeInBits();4647 unsigned NumElts = DemandedElts.getBitWidth();4648 unsigned Tmp, Tmp2;4649 unsigned FirstAnswer = 1;4650 4651 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {4652 const APInt &Val = C->getAPIntValue();4653 return Val.getNumSignBits();4654 }4655 4656 if (Depth >= MaxRecursionDepth)4657 return 1; // Limit search depth.4658 4659 if (!DemandedElts)4660 return 1; // No demanded elts, better to assume we don't know anything.4661 4662 unsigned Opcode = Op.getOpcode();4663 switch (Opcode) {4664 default: break;4665 case ISD::AssertSext:4666 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();4667 return VTBits-Tmp+1;4668 case ISD::AssertZext:4669 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();4670 return VTBits-Tmp;4671 case ISD::FREEZE:4672 if (isGuaranteedNotToBeUndefOrPoison(Op.getOperand(0), DemandedElts,4673 /*PoisonOnly=*/false))4674 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);4675 break;4676 case ISD::MERGE_VALUES:4677 return ComputeNumSignBits(Op.getOperand(Op.getResNo()), DemandedElts,4678 Depth + 1);4679 case ISD::SPLAT_VECTOR: {4680 // Check if the sign bits of source go down as far as the truncated value.4681 unsigned NumSrcBits = Op.getOperand(0).getValueSizeInBits();4682 unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);4683 if (NumSrcSignBits > (NumSrcBits - VTBits))4684 return NumSrcSignBits - (NumSrcBits - VTBits);4685 break;4686 }4687 case ISD::BUILD_VECTOR:4688 assert(!VT.isScalableVector());4689 Tmp = VTBits;4690 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {4691 if (!DemandedElts[i])4692 continue;4693 4694 SDValue SrcOp = Op.getOperand(i);4695 // BUILD_VECTOR can implicitly truncate sources, we handle this specially4696 // for constant nodes to ensure we only look at the sign bits.4697 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(SrcOp)) {4698 APInt T = C->getAPIntValue().trunc(VTBits);4699 Tmp2 = T.getNumSignBits();4700 } else {4701 Tmp2 = ComputeNumSignBits(SrcOp, Depth + 1);4702 4703 if (SrcOp.getValueSizeInBits() != VTBits) {4704 assert(SrcOp.getValueSizeInBits() > VTBits &&4705 "Expected BUILD_VECTOR implicit truncation");4706 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;4707 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);4708 }4709 }4710 Tmp = std::min(Tmp, Tmp2);4711 }4712 return Tmp;4713 4714 case ISD::VECTOR_COMPRESS: {4715 SDValue Vec = Op.getOperand(0);4716 SDValue PassThru = Op.getOperand(2);4717 Tmp = ComputeNumSignBits(PassThru, DemandedElts, Depth + 1);4718 if (Tmp == 1)4719 return 1;4720 Tmp2 = ComputeNumSignBits(Vec, Depth + 1);4721 Tmp = std::min(Tmp, Tmp2);4722 return Tmp;4723 }4724 4725 case ISD::VECTOR_SHUFFLE: {4726 // Collect the minimum number of sign bits that are shared by every vector4727 // element referenced by the shuffle.4728 APInt DemandedLHS, DemandedRHS;4729 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);4730 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");4731 if (!getShuffleDemandedElts(NumElts, SVN->getMask(), DemandedElts,4732 DemandedLHS, DemandedRHS))4733 return 1;4734 4735 Tmp = std::numeric_limits<unsigned>::max();4736 if (!!DemandedLHS)4737 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1);4738 if (!!DemandedRHS) {4739 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1);4740 Tmp = std::min(Tmp, Tmp2);4741 }4742 // If we don't know anything, early out and try computeKnownBits fall-back.4743 if (Tmp == 1)4744 break;4745 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");4746 return Tmp;4747 }4748 4749 case ISD::BITCAST: {4750 if (VT.isScalableVector())4751 break;4752 SDValue N0 = Op.getOperand(0);4753 EVT SrcVT = N0.getValueType();4754 unsigned SrcBits = SrcVT.getScalarSizeInBits();4755 4756 // Ignore bitcasts from unsupported types..4757 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))4758 break;4759 4760 // Fast handling of 'identity' bitcasts.4761 if (VTBits == SrcBits)4762 return ComputeNumSignBits(N0, DemandedElts, Depth + 1);4763 4764 bool IsLE = getDataLayout().isLittleEndian();4765 4766 // Bitcast 'large element' scalar/vector to 'small element' vector.4767 if ((SrcBits % VTBits) == 0) {4768 assert(VT.isVector() && "Expected bitcast to vector");4769 4770 unsigned Scale = SrcBits / VTBits;4771 APInt SrcDemandedElts =4772 APIntOps::ScaleBitMask(DemandedElts, NumElts / Scale);4773 4774 // Fast case - sign splat can be simply split across the small elements.4775 Tmp = ComputeNumSignBits(N0, SrcDemandedElts, Depth + 1);4776 if (Tmp == SrcBits)4777 return VTBits;4778 4779 // Slow case - determine how far the sign extends into each sub-element.4780 Tmp2 = VTBits;4781 for (unsigned i = 0; i != NumElts; ++i)4782 if (DemandedElts[i]) {4783 unsigned SubOffset = i % Scale;4784 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);4785 SubOffset = SubOffset * VTBits;4786 if (Tmp <= SubOffset)4787 return 1;4788 Tmp2 = std::min(Tmp2, Tmp - SubOffset);4789 }4790 return Tmp2;4791 }4792 break;4793 }4794 4795 case ISD::FP_TO_SINT_SAT:4796 // FP_TO_SINT_SAT produces a signed value that fits in the saturating VT.4797 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();4798 return VTBits - Tmp + 1;4799 case ISD::SIGN_EXTEND:4800 Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits();4801 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1) + Tmp;4802 case ISD::SIGN_EXTEND_INREG:4803 // Max of the input and what this extends.4804 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();4805 Tmp = VTBits-Tmp+1;4806 Tmp2 = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);4807 return std::max(Tmp, Tmp2);4808 case ISD::SIGN_EXTEND_VECTOR_INREG: {4809 if (VT.isScalableVector())4810 break;4811 SDValue Src = Op.getOperand(0);4812 EVT SrcVT = Src.getValueType();4813 APInt DemandedSrcElts = DemandedElts.zext(SrcVT.getVectorNumElements());4814 Tmp = VTBits - SrcVT.getScalarSizeInBits();4815 return ComputeNumSignBits(Src, DemandedSrcElts, Depth+1) + Tmp;4816 }4817 case ISD::SRA:4818 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);4819 // SRA X, C -> adds C sign bits.4820 if (std::optional<unsigned> ShAmt =4821 getValidMinimumShiftAmount(Op, DemandedElts, Depth + 1))4822 Tmp = std::min(Tmp + *ShAmt, VTBits);4823 return Tmp;4824 case ISD::SHL:4825 if (std::optional<ConstantRange> ShAmtRange =4826 getValidShiftAmountRange(Op, DemandedElts, Depth + 1)) {4827 unsigned MaxShAmt = ShAmtRange->getUnsignedMax().getZExtValue();4828 unsigned MinShAmt = ShAmtRange->getUnsignedMin().getZExtValue();4829 // Try to look through ZERO/SIGN/ANY_EXTEND. If all extended bits are4830 // shifted out, then we can compute the number of sign bits for the4831 // operand being extended. A future improvement could be to pass along the4832 // "shifted left by" information in the recursive calls to4833 // ComputeKnownSignBits. Allowing us to handle this more generically.4834 if (ISD::isExtOpcode(Op.getOperand(0).getOpcode())) {4835 SDValue Ext = Op.getOperand(0);4836 EVT ExtVT = Ext.getValueType();4837 SDValue Extendee = Ext.getOperand(0);4838 EVT ExtendeeVT = Extendee.getValueType();4839 unsigned SizeDifference =4840 ExtVT.getScalarSizeInBits() - ExtendeeVT.getScalarSizeInBits();4841 if (SizeDifference <= MinShAmt) {4842 Tmp = SizeDifference +4843 ComputeNumSignBits(Extendee, DemandedElts, Depth + 1);4844 if (MaxShAmt < Tmp)4845 return Tmp - MaxShAmt;4846 }4847 }4848 // shl destroys sign bits, ensure it doesn't shift out all sign bits.4849 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);4850 if (MaxShAmt < Tmp)4851 return Tmp - MaxShAmt;4852 }4853 break;4854 case ISD::AND:4855 case ISD::OR:4856 case ISD::XOR: // NOT is handled here.4857 // Logical binary ops preserve the number of sign bits at the worst.4858 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);4859 if (Tmp != 1) {4860 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);4861 FirstAnswer = std::min(Tmp, Tmp2);4862 // We computed what we know about the sign bits as our first4863 // answer. Now proceed to the generic code that uses4864 // computeKnownBits, and pick whichever answer is better.4865 }4866 break;4867 4868 case ISD::SELECT:4869 case ISD::VSELECT:4870 Tmp = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);4871 if (Tmp == 1) return 1; // Early out.4872 Tmp2 = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);4873 return std::min(Tmp, Tmp2);4874 case ISD::SELECT_CC:4875 Tmp = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);4876 if (Tmp == 1) return 1; // Early out.4877 Tmp2 = ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth+1);4878 return std::min(Tmp, Tmp2);4879 4880 case ISD::SMIN:4881 case ISD::SMAX: {4882 // If we have a clamp pattern, we know that the number of sign bits will be4883 // the minimum of the clamp min/max range.4884 bool IsMax = (Opcode == ISD::SMAX);4885 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;4886 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))4887 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))4888 CstHigh =4889 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);4890 if (CstLow && CstHigh) {4891 if (!IsMax)4892 std::swap(CstLow, CstHigh);4893 if (CstLow->getAPIntValue().sle(CstHigh->getAPIntValue())) {4894 Tmp = CstLow->getAPIntValue().getNumSignBits();4895 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();4896 return std::min(Tmp, Tmp2);4897 }4898 }4899 4900 // Fallback - just get the minimum number of sign bits of the operands.4901 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);4902 if (Tmp == 1)4903 return 1; // Early out.4904 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);4905 return std::min(Tmp, Tmp2);4906 }4907 case ISD::UMIN:4908 case ISD::UMAX:4909 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);4910 if (Tmp == 1)4911 return 1; // Early out.4912 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);4913 return std::min(Tmp, Tmp2);4914 case ISD::SSUBO_CARRY:4915 case ISD::USUBO_CARRY:4916 // sub_carry(x,x,c) -> 0/-1 (sext carry)4917 if (Op.getResNo() == 0 && Op.getOperand(0) == Op.getOperand(1))4918 return VTBits;4919 [[fallthrough]];4920 case ISD::SADDO:4921 case ISD::UADDO:4922 case ISD::SADDO_CARRY:4923 case ISD::UADDO_CARRY:4924 case ISD::SSUBO:4925 case ISD::USUBO:4926 case ISD::SMULO:4927 case ISD::UMULO:4928 if (Op.getResNo() != 1)4929 break;4930 // The boolean result conforms to getBooleanContents. Fall through.4931 // If setcc returns 0/-1, all bits are sign bits.4932 // We know that we have an integer-based boolean since these operations4933 // are only available for integer.4934 if (TLI->getBooleanContents(VT.isVector(), false) ==4935 TargetLowering::ZeroOrNegativeOneBooleanContent)4936 return VTBits;4937 break;4938 case ISD::SETCC:4939 case ISD::SETCCCARRY:4940 case ISD::STRICT_FSETCC:4941 case ISD::STRICT_FSETCCS: {4942 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;4943 // If setcc returns 0/-1, all bits are sign bits.4944 if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==4945 TargetLowering::ZeroOrNegativeOneBooleanContent)4946 return VTBits;4947 break;4948 }4949 case ISD::ROTL:4950 case ISD::ROTR:4951 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);4952 4953 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.4954 if (Tmp == VTBits)4955 return VTBits;4956 4957 if (ConstantSDNode *C =4958 isConstOrConstSplat(Op.getOperand(1), DemandedElts)) {4959 unsigned RotAmt = C->getAPIntValue().urem(VTBits);4960 4961 // Handle rotate right by N like a rotate left by 32-N.4962 if (Opcode == ISD::ROTR)4963 RotAmt = (VTBits - RotAmt) % VTBits;4964 4965 // If we aren't rotating out all of the known-in sign bits, return the4966 // number that are left. This handles rotl(sext(x), 1) for example.4967 if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt);4968 }4969 break;4970 case ISD::ADD:4971 case ISD::ADDC:4972 // TODO: Move Operand 1 check before Operand 0 check4973 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);4974 if (Tmp == 1) return 1; // Early out.4975 4976 // Special case decrementing a value (ADD X, -1):4977 if (ConstantSDNode *CRHS =4978 isConstOrConstSplat(Op.getOperand(1), DemandedElts))4979 if (CRHS->isAllOnes()) {4980 KnownBits Known =4981 computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);4982 4983 // If the input is known to be 0 or 1, the output is 0/-1, which is all4984 // sign bits set.4985 if ((Known.Zero | 1).isAllOnes())4986 return VTBits;4987 4988 // If we are subtracting one from a positive number, there is no carry4989 // out of the result.4990 if (Known.isNonNegative())4991 return Tmp;4992 }4993 4994 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);4995 if (Tmp2 == 1) return 1; // Early out.4996 4997 // Add can have at most one carry bit. Thus we know that the output4998 // is, at worst, one more bit than the inputs.4999 return std::min(Tmp, Tmp2) - 1;5000 case ISD::SUB:5001 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);5002 if (Tmp2 == 1) return 1; // Early out.5003 5004 // Handle NEG.5005 if (ConstantSDNode *CLHS =5006 isConstOrConstSplat(Op.getOperand(0), DemandedElts))5007 if (CLHS->isZero()) {5008 KnownBits Known =5009 computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);5010 // If the input is known to be 0 or 1, the output is 0/-1, which is all5011 // sign bits set.5012 if ((Known.Zero | 1).isAllOnes())5013 return VTBits;5014 5015 // If the input is known to be positive (the sign bit is known clear),5016 // the output of the NEG has the same number of sign bits as the input.5017 if (Known.isNonNegative())5018 return Tmp2;5019 5020 // Otherwise, we treat this like a SUB.5021 }5022 5023 // Sub can have at most one carry bit. Thus we know that the output5024 // is, at worst, one more bit than the inputs.5025 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);5026 if (Tmp == 1) return 1; // Early out.5027 return std::min(Tmp, Tmp2) - 1;5028 case ISD::MUL: {5029 // The output of the Mul can be at most twice the valid bits in the inputs.5030 unsigned SignBitsOp0 = ComputeNumSignBits(Op.getOperand(0), Depth + 1);5031 if (SignBitsOp0 == 1)5032 break;5033 unsigned SignBitsOp1 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);5034 if (SignBitsOp1 == 1)5035 break;5036 unsigned OutValidBits =5037 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);5038 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;5039 }5040 case ISD::AVGCEILS:5041 case ISD::AVGFLOORS:5042 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);5043 if (Tmp == 1)5044 return 1; // Early out.5045 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);5046 return std::min(Tmp, Tmp2);5047 case ISD::SREM:5048 // The sign bit is the LHS's sign bit, except when the result of the5049 // remainder is zero. The magnitude of the result should be less than or5050 // equal to the magnitude of the LHS. Therefore, the result should have5051 // at least as many sign bits as the left hand side.5052 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);5053 case ISD::TRUNCATE: {5054 // Check if the sign bits of source go down as far as the truncated value.5055 unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits();5056 unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);5057 if (NumSrcSignBits > (NumSrcBits - VTBits))5058 return NumSrcSignBits - (NumSrcBits - VTBits);5059 break;5060 }5061 case ISD::EXTRACT_ELEMENT: {5062 if (VT.isScalableVector())5063 break;5064 const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);5065 const int BitWidth = Op.getValueSizeInBits();5066 const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth;5067 5068 // Get reverse index (starting from 1), Op1 value indexes elements from5069 // little end. Sign starts at big end.5070 const int rIndex = Items - 1 - Op.getConstantOperandVal(1);5071 5072 // If the sign portion ends in our element the subtraction gives correct5073 // result. Otherwise it gives either negative or > bitwidth result5074 return std::clamp(KnownSign - rIndex * BitWidth, 1, BitWidth);5075 }5076 case ISD::INSERT_VECTOR_ELT: {5077 if (VT.isScalableVector())5078 break;5079 // If we know the element index, split the demand between the5080 // source vector and the inserted element, otherwise assume we need5081 // the original demanded vector elements and the value.5082 SDValue InVec = Op.getOperand(0);5083 SDValue InVal = Op.getOperand(1);5084 SDValue EltNo = Op.getOperand(2);5085 bool DemandedVal = true;5086 APInt DemandedVecElts = DemandedElts;5087 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);5088 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {5089 unsigned EltIdx = CEltNo->getZExtValue();5090 DemandedVal = !!DemandedElts[EltIdx];5091 DemandedVecElts.clearBit(EltIdx);5092 }5093 Tmp = std::numeric_limits<unsigned>::max();5094 if (DemandedVal) {5095 // TODO - handle implicit truncation of inserted elements.5096 if (InVal.getScalarValueSizeInBits() != VTBits)5097 break;5098 Tmp2 = ComputeNumSignBits(InVal, Depth + 1);5099 Tmp = std::min(Tmp, Tmp2);5100 }5101 if (!!DemandedVecElts) {5102 Tmp2 = ComputeNumSignBits(InVec, DemandedVecElts, Depth + 1);5103 Tmp = std::min(Tmp, Tmp2);5104 }5105 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");5106 return Tmp;5107 }5108 case ISD::EXTRACT_VECTOR_ELT: {5109 assert(!VT.isScalableVector());5110 SDValue InVec = Op.getOperand(0);5111 SDValue EltNo = Op.getOperand(1);5112 EVT VecVT = InVec.getValueType();5113 // ComputeNumSignBits not yet implemented for scalable vectors.5114 if (VecVT.isScalableVector())5115 break;5116 const unsigned BitWidth = Op.getValueSizeInBits();5117 const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits();5118 const unsigned NumSrcElts = VecVT.getVectorNumElements();5119 5120 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know5121 // anything about sign bits. But if the sizes match we can derive knowledge5122 // about sign bits from the vector operand.5123 if (BitWidth != EltBitWidth)5124 break;5125 5126 // If we know the element index, just demand that vector element, else for5127 // an unknown element index, ignore DemandedElts and demand them all.5128 APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);5129 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);5130 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))5131 DemandedSrcElts =5132 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());5133 5134 return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1);5135 }5136 case ISD::EXTRACT_SUBVECTOR: {5137 // Offset the demanded elts by the subvector index.5138 SDValue Src = Op.getOperand(0);5139 // Bail until we can represent demanded elements for scalable vectors.5140 if (Src.getValueType().isScalableVector())5141 break;5142 uint64_t Idx = Op.getConstantOperandVal(1);5143 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();5144 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);5145 return ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);5146 }5147 case ISD::CONCAT_VECTORS: {5148 if (VT.isScalableVector())5149 break;5150 // Determine the minimum number of sign bits across all demanded5151 // elts of the input vectors. Early out if the result is already 1.5152 Tmp = std::numeric_limits<unsigned>::max();5153 EVT SubVectorVT = Op.getOperand(0).getValueType();5154 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();5155 unsigned NumSubVectors = Op.getNumOperands();5156 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {5157 APInt DemandedSub =5158 DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts);5159 if (!DemandedSub)5160 continue;5161 Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1);5162 Tmp = std::min(Tmp, Tmp2);5163 }5164 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");5165 return Tmp;5166 }5167 case ISD::INSERT_SUBVECTOR: {5168 if (VT.isScalableVector())5169 break;5170 // Demand any elements from the subvector and the remainder from the src its5171 // inserted into.5172 SDValue Src = Op.getOperand(0);5173 SDValue Sub = Op.getOperand(1);5174 uint64_t Idx = Op.getConstantOperandVal(2);5175 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();5176 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);5177 APInt DemandedSrcElts = DemandedElts;5178 DemandedSrcElts.clearBits(Idx, Idx + NumSubElts);5179 5180 Tmp = std::numeric_limits<unsigned>::max();5181 if (!!DemandedSubElts) {5182 Tmp = ComputeNumSignBits(Sub, DemandedSubElts, Depth + 1);5183 if (Tmp == 1)5184 return 1; // early-out5185 }5186 if (!!DemandedSrcElts) {5187 Tmp2 = ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);5188 Tmp = std::min(Tmp, Tmp2);5189 }5190 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");5191 return Tmp;5192 }5193 case ISD::LOAD: {5194 LoadSDNode *LD = cast<LoadSDNode>(Op);5195 if (const MDNode *Ranges = LD->getRanges()) {5196 if (DemandedElts != 1)5197 break;5198 5199 ConstantRange CR = getConstantRangeFromMetadata(*Ranges);5200 if (VTBits > CR.getBitWidth()) {5201 switch (LD->getExtensionType()) {5202 case ISD::SEXTLOAD:5203 CR = CR.signExtend(VTBits);5204 break;5205 case ISD::ZEXTLOAD:5206 CR = CR.zeroExtend(VTBits);5207 break;5208 default:5209 break;5210 }5211 }5212 5213 if (VTBits != CR.getBitWidth())5214 break;5215 return std::min(CR.getSignedMin().getNumSignBits(),5216 CR.getSignedMax().getNumSignBits());5217 }5218 5219 break;5220 }5221 case ISD::ATOMIC_CMP_SWAP:5222 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:5223 case ISD::ATOMIC_SWAP:5224 case ISD::ATOMIC_LOAD_ADD:5225 case ISD::ATOMIC_LOAD_SUB:5226 case ISD::ATOMIC_LOAD_AND:5227 case ISD::ATOMIC_LOAD_CLR:5228 case ISD::ATOMIC_LOAD_OR:5229 case ISD::ATOMIC_LOAD_XOR:5230 case ISD::ATOMIC_LOAD_NAND:5231 case ISD::ATOMIC_LOAD_MIN:5232 case ISD::ATOMIC_LOAD_MAX:5233 case ISD::ATOMIC_LOAD_UMIN:5234 case ISD::ATOMIC_LOAD_UMAX:5235 case ISD::ATOMIC_LOAD: {5236 auto *AT = cast<AtomicSDNode>(Op);5237 // If we are looking at the loaded value.5238 if (Op.getResNo() == 0) {5239 Tmp = AT->getMemoryVT().getScalarSizeInBits();5240 if (Tmp == VTBits)5241 return 1; // early-out5242 5243 // For atomic_load, prefer to use the extension type.5244 if (Op->getOpcode() == ISD::ATOMIC_LOAD) {5245 switch (AT->getExtensionType()) {5246 default:5247 break;5248 case ISD::SEXTLOAD:5249 return VTBits - Tmp + 1;5250 case ISD::ZEXTLOAD:5251 return VTBits - Tmp;5252 }5253 }5254 5255 if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND)5256 return VTBits - Tmp + 1;5257 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)5258 return VTBits - Tmp;5259 }5260 break;5261 }5262 }5263 5264 // If we are looking at the loaded value of the SDNode.5265 if (Op.getResNo() == 0) {5266 // Handle LOADX separately here. EXTLOAD case will fallthrough.5267 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {5268 unsigned ExtType = LD->getExtensionType();5269 switch (ExtType) {5270 default: break;5271 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.5272 Tmp = LD->getMemoryVT().getScalarSizeInBits();5273 return VTBits - Tmp + 1;5274 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.5275 Tmp = LD->getMemoryVT().getScalarSizeInBits();5276 return VTBits - Tmp;5277 case ISD::NON_EXTLOAD:5278 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {5279 // We only need to handle vectors - computeKnownBits should handle5280 // scalar cases.5281 Type *CstTy = Cst->getType();5282 if (CstTy->isVectorTy() && !VT.isScalableVector() &&5283 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits() &&5284 VTBits == CstTy->getScalarSizeInBits()) {5285 Tmp = VTBits;5286 for (unsigned i = 0; i != NumElts; ++i) {5287 if (!DemandedElts[i])5288 continue;5289 if (Constant *Elt = Cst->getAggregateElement(i)) {5290 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {5291 const APInt &Value = CInt->getValue();5292 Tmp = std::min(Tmp, Value.getNumSignBits());5293 continue;5294 }5295 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {5296 APInt Value = CFP->getValueAPF().bitcastToAPInt();5297 Tmp = std::min(Tmp, Value.getNumSignBits());5298 continue;5299 }5300 }5301 // Unknown type. Conservatively assume no bits match sign bit.5302 return 1;5303 }5304 return Tmp;5305 }5306 }5307 break;5308 }5309 }5310 }5311 5312 // Allow the target to implement this method for its nodes.5313 if (Opcode >= ISD::BUILTIN_OP_END ||5314 Opcode == ISD::INTRINSIC_WO_CHAIN ||5315 Opcode == ISD::INTRINSIC_W_CHAIN ||5316 Opcode == ISD::INTRINSIC_VOID) {5317 // TODO: This can probably be removed once target code is audited. This5318 // is here purely to reduce patch size and review complexity.5319 if (!VT.isScalableVector()) {5320 unsigned NumBits =5321 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth);5322 if (NumBits > 1)5323 FirstAnswer = std::max(FirstAnswer, NumBits);5324 }5325 }5326 5327 // Finally, if we can prove that the top bits of the result are 0's or 1's,5328 // use this information.5329 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);5330 return std::max(FirstAnswer, Known.countMinSignBits());5331}5332 5333unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,5334 unsigned Depth) const {5335 unsigned SignBits = ComputeNumSignBits(Op, Depth);5336 return Op.getScalarValueSizeInBits() - SignBits + 1;5337}5338 5339unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,5340 const APInt &DemandedElts,5341 unsigned Depth) const {5342 unsigned SignBits = ComputeNumSignBits(Op, DemandedElts, Depth);5343 return Op.getScalarValueSizeInBits() - SignBits + 1;5344}5345 5346bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly,5347 unsigned Depth) const {5348 // Early out for FREEZE.5349 if (Op.getOpcode() == ISD::FREEZE)5350 return true;5351 5352 EVT VT = Op.getValueType();5353 APInt DemandedElts = VT.isFixedLengthVector()5354 ? APInt::getAllOnes(VT.getVectorNumElements())5355 : APInt(1, 1);5356 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, PoisonOnly, Depth);5357}5358 5359bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,5360 const APInt &DemandedElts,5361 bool PoisonOnly,5362 unsigned Depth) const {5363 unsigned Opcode = Op.getOpcode();5364 5365 // Early out for FREEZE.5366 if (Opcode == ISD::FREEZE)5367 return true;5368 5369 if (Depth >= MaxRecursionDepth)5370 return false; // Limit search depth.5371 5372 if (isIntOrFPConstant(Op))5373 return true;5374 5375 switch (Opcode) {5376 case ISD::CONDCODE:5377 case ISD::VALUETYPE:5378 case ISD::FrameIndex:5379 case ISD::TargetFrameIndex:5380 case ISD::CopyFromReg:5381 return true;5382 5383 case ISD::POISON:5384 return false;5385 5386 case ISD::UNDEF:5387 return PoisonOnly;5388 5389 case ISD::BUILD_VECTOR:5390 // NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements -5391 // this shouldn't affect the result.5392 for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) {5393 if (!DemandedElts[i])5394 continue;5395 if (!isGuaranteedNotToBeUndefOrPoison(Op.getOperand(i), PoisonOnly,5396 Depth + 1))5397 return false;5398 }5399 return true;5400 5401 case ISD::EXTRACT_SUBVECTOR: {5402 SDValue Src = Op.getOperand(0);5403 if (Src.getValueType().isScalableVector())5404 break;5405 uint64_t Idx = Op.getConstantOperandVal(1);5406 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();5407 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);5408 return isGuaranteedNotToBeUndefOrPoison(Src, DemandedSrcElts, PoisonOnly,5409 Depth + 1);5410 }5411 5412 case ISD::INSERT_SUBVECTOR: {5413 if (Op.getValueType().isScalableVector())5414 break;5415 SDValue Src = Op.getOperand(0);5416 SDValue Sub = Op.getOperand(1);5417 uint64_t Idx = Op.getConstantOperandVal(2);5418 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();5419 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);5420 APInt DemandedSrcElts = DemandedElts;5421 DemandedSrcElts.clearBits(Idx, Idx + NumSubElts);5422 5423 if (!!DemandedSubElts && !isGuaranteedNotToBeUndefOrPoison(5424 Sub, DemandedSubElts, PoisonOnly, Depth + 1))5425 return false;5426 if (!!DemandedSrcElts && !isGuaranteedNotToBeUndefOrPoison(5427 Src, DemandedSrcElts, PoisonOnly, Depth + 1))5428 return false;5429 return true;5430 }5431 5432 case ISD::EXTRACT_VECTOR_ELT: {5433 SDValue Src = Op.getOperand(0);5434 auto *IndexC = dyn_cast<ConstantSDNode>(Op.getOperand(1));5435 EVT SrcVT = Src.getValueType();5436 if (SrcVT.isFixedLengthVector() && IndexC &&5437 IndexC->getAPIntValue().ult(SrcVT.getVectorNumElements())) {5438 APInt DemandedSrcElts = APInt::getOneBitSet(SrcVT.getVectorNumElements(),5439 IndexC->getZExtValue());5440 return isGuaranteedNotToBeUndefOrPoison(Src, DemandedSrcElts, PoisonOnly,5441 Depth + 1);5442 }5443 break;5444 }5445 5446 case ISD::INSERT_VECTOR_ELT: {5447 SDValue InVec = Op.getOperand(0);5448 SDValue InVal = Op.getOperand(1);5449 SDValue EltNo = Op.getOperand(2);5450 EVT VT = InVec.getValueType();5451 auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);5452 if (IndexC && VT.isFixedLengthVector() &&5453 IndexC->getAPIntValue().ult(VT.getVectorNumElements())) {5454 if (DemandedElts[IndexC->getZExtValue()] &&5455 !isGuaranteedNotToBeUndefOrPoison(InVal, PoisonOnly, Depth + 1))5456 return false;5457 APInt InVecDemandedElts = DemandedElts;5458 InVecDemandedElts.clearBit(IndexC->getZExtValue());5459 if (!!InVecDemandedElts &&5460 !isGuaranteedNotToBeUndefOrPoison(5461 peekThroughInsertVectorElt(InVec, InVecDemandedElts),5462 InVecDemandedElts, PoisonOnly, Depth + 1))5463 return false;5464 return true;5465 }5466 break;5467 }5468 5469 case ISD::SCALAR_TO_VECTOR:5470 // Check upper (known undef) elements.5471 if (DemandedElts.ugt(1) && !PoisonOnly)5472 return false;5473 // Check element zero.5474 if (DemandedElts[0] && !isGuaranteedNotToBeUndefOrPoison(5475 Op.getOperand(0), PoisonOnly, Depth + 1))5476 return false;5477 return true;5478 5479 case ISD::SPLAT_VECTOR:5480 return isGuaranteedNotToBeUndefOrPoison(Op.getOperand(0), PoisonOnly,5481 Depth + 1);5482 5483 case ISD::VECTOR_SHUFFLE: {5484 APInt DemandedLHS, DemandedRHS;5485 auto *SVN = cast<ShuffleVectorSDNode>(Op);5486 if (!getShuffleDemandedElts(DemandedElts.getBitWidth(), SVN->getMask(),5487 DemandedElts, DemandedLHS, DemandedRHS,5488 /*AllowUndefElts=*/false))5489 return false;5490 if (!DemandedLHS.isZero() &&5491 !isGuaranteedNotToBeUndefOrPoison(Op.getOperand(0), DemandedLHS,5492 PoisonOnly, Depth + 1))5493 return false;5494 if (!DemandedRHS.isZero() &&5495 !isGuaranteedNotToBeUndefOrPoison(Op.getOperand(1), DemandedRHS,5496 PoisonOnly, Depth + 1))5497 return false;5498 return true;5499 }5500 5501 case ISD::SHL:5502 case ISD::SRL:5503 case ISD::SRA:5504 // Shift amount operand is checked by canCreateUndefOrPoison. So it is5505 // enough to check operand 0 if Op can't create undef/poison.5506 return !canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly,5507 /*ConsiderFlags*/ true, Depth) &&5508 isGuaranteedNotToBeUndefOrPoison(Op.getOperand(0), DemandedElts,5509 PoisonOnly, Depth + 1);5510 5511 case ISD::BSWAP:5512 case ISD::CTPOP:5513 case ISD::BITREVERSE:5514 case ISD::AND:5515 case ISD::OR:5516 case ISD::XOR:5517 case ISD::ADD:5518 case ISD::SUB:5519 case ISD::MUL:5520 case ISD::SADDSAT:5521 case ISD::UADDSAT:5522 case ISD::SSUBSAT:5523 case ISD::USUBSAT:5524 case ISD::SSHLSAT:5525 case ISD::USHLSAT:5526 case ISD::SMIN:5527 case ISD::SMAX:5528 case ISD::UMIN:5529 case ISD::UMAX:5530 case ISD::ZERO_EXTEND:5531 case ISD::SIGN_EXTEND:5532 case ISD::ANY_EXTEND:5533 case ISD::TRUNCATE:5534 case ISD::VSELECT: {5535 // If Op can't create undef/poison and none of its operands are undef/poison5536 // then Op is never undef/poison. A difference from the more common check5537 // below, outside the switch, is that we handle elementwise operations for5538 // which the DemandedElts mask is valid for all operands here.5539 return !canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly,5540 /*ConsiderFlags*/ true, Depth) &&5541 all_of(Op->ops(), [&](SDValue V) {5542 return isGuaranteedNotToBeUndefOrPoison(V, DemandedElts,5543 PoisonOnly, Depth + 1);5544 });5545 }5546 5547 // TODO: Search for noundef attributes from library functions.5548 5549 // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.5550 5551 default:5552 // Allow the target to implement this method for its nodes.5553 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||5554 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)5555 return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode(5556 Op, DemandedElts, *this, PoisonOnly, Depth);5557 break;5558 }5559 5560 // If Op can't create undef/poison and none of its operands are undef/poison5561 // then Op is never undef/poison.5562 // NOTE: TargetNodes can handle this in themselves in5563 // isGuaranteedNotToBeUndefOrPoisonForTargetNode or let5564 // TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode handle it.5565 return !canCreateUndefOrPoison(Op, PoisonOnly, /*ConsiderFlags*/ true,5566 Depth) &&5567 all_of(Op->ops(), [&](SDValue V) {5568 return isGuaranteedNotToBeUndefOrPoison(V, PoisonOnly, Depth + 1);5569 });5570}5571 5572bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, bool PoisonOnly,5573 bool ConsiderFlags,5574 unsigned Depth) const {5575 EVT VT = Op.getValueType();5576 APInt DemandedElts = VT.isFixedLengthVector()5577 ? APInt::getAllOnes(VT.getVectorNumElements())5578 : APInt(1, 1);5579 return canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly, ConsiderFlags,5580 Depth);5581}5582 5583bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,5584 bool PoisonOnly, bool ConsiderFlags,5585 unsigned Depth) const {5586 if (ConsiderFlags && Op->hasPoisonGeneratingFlags())5587 return true;5588 5589 unsigned Opcode = Op.getOpcode();5590 switch (Opcode) {5591 case ISD::AssertSext:5592 case ISD::AssertZext:5593 case ISD::AssertAlign:5594 case ISD::AssertNoFPClass:5595 // Assertion nodes can create poison if the assertion fails.5596 return true;5597 5598 case ISD::FREEZE:5599 case ISD::CONCAT_VECTORS:5600 case ISD::INSERT_SUBVECTOR:5601 case ISD::EXTRACT_SUBVECTOR:5602 case ISD::SADDSAT:5603 case ISD::UADDSAT:5604 case ISD::SSUBSAT:5605 case ISD::USUBSAT:5606 case ISD::MULHU:5607 case ISD::MULHS:5608 case ISD::AVGFLOORS:5609 case ISD::AVGFLOORU:5610 case ISD::AVGCEILS:5611 case ISD::AVGCEILU:5612 case ISD::ABDU:5613 case ISD::ABDS:5614 case ISD::SMIN:5615 case ISD::SMAX:5616 case ISD::SCMP:5617 case ISD::UMIN:5618 case ISD::UMAX:5619 case ISD::UCMP:5620 case ISD::AND:5621 case ISD::XOR:5622 case ISD::ROTL:5623 case ISD::ROTR:5624 case ISD::FSHL:5625 case ISD::FSHR:5626 case ISD::BSWAP:5627 case ISD::CTTZ:5628 case ISD::CTLZ:5629 case ISD::CTPOP:5630 case ISD::BITREVERSE:5631 case ISD::PARITY:5632 case ISD::SIGN_EXTEND:5633 case ISD::TRUNCATE:5634 case ISD::SIGN_EXTEND_INREG:5635 case ISD::SIGN_EXTEND_VECTOR_INREG:5636 case ISD::ZERO_EXTEND_VECTOR_INREG:5637 case ISD::BITCAST:5638 case ISD::BUILD_VECTOR:5639 case ISD::BUILD_PAIR:5640 case ISD::SPLAT_VECTOR:5641 case ISD::FABS:5642 return false;5643 5644 case ISD::ABS:5645 // ISD::ABS defines abs(INT_MIN) -> INT_MIN and never generates poison.5646 // Different to Intrinsic::abs.5647 return false;5648 5649 case ISD::ADDC:5650 case ISD::SUBC:5651 case ISD::ADDE:5652 case ISD::SUBE:5653 case ISD::SADDO:5654 case ISD::SSUBO:5655 case ISD::SMULO:5656 case ISD::SADDO_CARRY:5657 case ISD::SSUBO_CARRY:5658 case ISD::UADDO:5659 case ISD::USUBO:5660 case ISD::UMULO:5661 case ISD::UADDO_CARRY:5662 case ISD::USUBO_CARRY:5663 // No poison on result or overflow flags.5664 return false;5665 5666 case ISD::SELECT_CC:5667 case ISD::SETCC: {5668 // Integer setcc cannot create undef or poison.5669 if (Op.getOperand(0).getValueType().isInteger())5670 return false;5671 5672 // FP compares are more complicated. They can create poison for nan/infinity5673 // based on options and flags. The options and flags also cause special5674 // nonan condition codes to be used. Those condition codes may be preserved5675 // even if the nonan flag is dropped somewhere.5676 unsigned CCOp = Opcode == ISD::SETCC ? 2 : 4;5677 ISD::CondCode CCCode = cast<CondCodeSDNode>(Op.getOperand(CCOp))->get();5678 return (unsigned)CCCode & 0x10U;5679 }5680 5681 case ISD::OR:5682 case ISD::ZERO_EXTEND:5683 case ISD::SELECT:5684 case ISD::VSELECT:5685 case ISD::ADD:5686 case ISD::SUB:5687 case ISD::MUL:5688 case ISD::FNEG:5689 case ISD::FADD:5690 case ISD::FSUB:5691 case ISD::FMUL:5692 case ISD::FDIV:5693 case ISD::FREM:5694 case ISD::FCOPYSIGN:5695 case ISD::FMA:5696 case ISD::FMAD:5697 case ISD::FMULADD:5698 case ISD::FP_EXTEND:5699 case ISD::FP_TO_SINT_SAT:5700 case ISD::FP_TO_UINT_SAT:5701 // No poison except from flags (which is handled above)5702 return false;5703 5704 case ISD::SHL:5705 case ISD::SRL:5706 case ISD::SRA:5707 // If the max shift amount isn't in range, then the shift can5708 // create poison.5709 return !getValidMaximumShiftAmount(Op, DemandedElts, Depth + 1);5710 5711 case ISD::CTTZ_ZERO_UNDEF:5712 case ISD::CTLZ_ZERO_UNDEF:5713 // If the amount is zero then the result will be poison.5714 // TODO: Add isKnownNeverZero DemandedElts handling.5715 return !isKnownNeverZero(Op.getOperand(0), Depth + 1);5716 5717 case ISD::SCALAR_TO_VECTOR:5718 // Check if we demand any upper (undef) elements.5719 return !PoisonOnly && DemandedElts.ugt(1);5720 5721 case ISD::INSERT_VECTOR_ELT:5722 case ISD::EXTRACT_VECTOR_ELT: {5723 // Ensure that the element index is in bounds.5724 EVT VecVT = Op.getOperand(0).getValueType();5725 SDValue Idx = Op.getOperand(Opcode == ISD::INSERT_VECTOR_ELT ? 2 : 1);5726 KnownBits KnownIdx = computeKnownBits(Idx, Depth + 1);5727 return KnownIdx.getMaxValue().uge(VecVT.getVectorMinNumElements());5728 }5729 5730 case ISD::VECTOR_SHUFFLE: {5731 // Check for any demanded shuffle element that is undef.5732 auto *SVN = cast<ShuffleVectorSDNode>(Op);5733 for (auto [Idx, Elt] : enumerate(SVN->getMask()))5734 if (Elt < 0 && DemandedElts[Idx])5735 return true;5736 return false;5737 }5738 5739 case ISD::VECTOR_COMPRESS:5740 return false;5741 5742 default:5743 // Allow the target to implement this method for its nodes.5744 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||5745 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)5746 return TLI->canCreateUndefOrPoisonForTargetNode(5747 Op, DemandedElts, *this, PoisonOnly, ConsiderFlags, Depth);5748 break;5749 }5750 5751 // Be conservative and return true.5752 return true;5753}5754 5755bool SelectionDAG::isADDLike(SDValue Op, bool NoWrap) const {5756 unsigned Opcode = Op.getOpcode();5757 if (Opcode == ISD::OR)5758 return Op->getFlags().hasDisjoint() ||5759 haveNoCommonBitsSet(Op.getOperand(0), Op.getOperand(1));5760 if (Opcode == ISD::XOR)5761 return !NoWrap && isMinSignedConstant(Op.getOperand(1));5762 return false;5763}5764 5765bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {5766 return Op.getNumOperands() == 2 && isa<ConstantSDNode>(Op.getOperand(1)) &&5767 (Op.isAnyAdd() || isADDLike(Op));5768}5769 5770bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN,5771 unsigned Depth) const {5772 EVT VT = Op.getValueType();5773 5774 // Since the number of lanes in a scalable vector is unknown at compile time,5775 // we track one bit which is implicitly broadcast to all lanes. This means5776 // that all lanes in a scalable vector are considered demanded.5777 APInt DemandedElts = VT.isFixedLengthVector()5778 ? APInt::getAllOnes(VT.getVectorNumElements())5779 : APInt(1, 1);5780 5781 return isKnownNeverNaN(Op, DemandedElts, SNaN, Depth);5782}5783 5784bool SelectionDAG::isKnownNeverNaN(SDValue Op, const APInt &DemandedElts,5785 bool SNaN, unsigned Depth) const {5786 assert(!DemandedElts.isZero() && "No demanded elements");5787 5788 // If we're told that NaNs won't happen, assume they won't.5789 if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs())5790 return true;5791 5792 if (Depth >= MaxRecursionDepth)5793 return false; // Limit search depth.5794 5795 // If the value is a constant, we can obviously see if it is a NaN or not.5796 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) {5797 return !C->getValueAPF().isNaN() ||5798 (SNaN && !C->getValueAPF().isSignaling());5799 }5800 5801 unsigned Opcode = Op.getOpcode();5802 switch (Opcode) {5803 case ISD::FADD:5804 case ISD::FSUB:5805 case ISD::FMUL:5806 case ISD::FDIV:5807 case ISD::FREM:5808 case ISD::FSIN:5809 case ISD::FCOS:5810 case ISD::FTAN:5811 case ISD::FASIN:5812 case ISD::FACOS:5813 case ISD::FATAN:5814 case ISD::FATAN2:5815 case ISD::FSINH:5816 case ISD::FCOSH:5817 case ISD::FTANH:5818 case ISD::FMA:5819 case ISD::FMULADD:5820 case ISD::FMAD: {5821 if (SNaN)5822 return true;5823 // TODO: Need isKnownNeverInfinity5824 return false;5825 }5826 case ISD::FCANONICALIZE:5827 case ISD::FEXP:5828 case ISD::FEXP2:5829 case ISD::FEXP10:5830 case ISD::FTRUNC:5831 case ISD::FFLOOR:5832 case ISD::FCEIL:5833 case ISD::FROUND:5834 case ISD::FROUNDEVEN:5835 case ISD::LROUND:5836 case ISD::LLROUND:5837 case ISD::FRINT:5838 case ISD::LRINT:5839 case ISD::LLRINT:5840 case ISD::FNEARBYINT:5841 case ISD::FLDEXP: {5842 if (SNaN)5843 return true;5844 return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1);5845 }5846 case ISD::FABS:5847 case ISD::FNEG:5848 case ISD::FCOPYSIGN: {5849 return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1);5850 }5851 case ISD::SELECT:5852 return isKnownNeverNaN(Op.getOperand(1), DemandedElts, SNaN, Depth + 1) &&5853 isKnownNeverNaN(Op.getOperand(2), DemandedElts, SNaN, Depth + 1);5854 case ISD::FP_EXTEND:5855 case ISD::FP_ROUND: {5856 if (SNaN)5857 return true;5858 return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1);5859 }5860 case ISD::SINT_TO_FP:5861 case ISD::UINT_TO_FP:5862 return true;5863 case ISD::FSQRT: // Need is known positive5864 case ISD::FLOG:5865 case ISD::FLOG2:5866 case ISD::FLOG10:5867 case ISD::FPOWI:5868 case ISD::FPOW: {5869 if (SNaN)5870 return true;5871 // TODO: Refine on operand5872 return false;5873 }5874 case ISD::FMINNUM:5875 case ISD::FMAXNUM:5876 case ISD::FMINIMUMNUM:5877 case ISD::FMAXIMUMNUM: {5878 // Only one needs to be known not-nan, since it will be returned if the5879 // other ends up being one.5880 return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1) ||5881 isKnownNeverNaN(Op.getOperand(1), DemandedElts, SNaN, Depth + 1);5882 }5883 case ISD::FMINNUM_IEEE:5884 case ISD::FMAXNUM_IEEE: {5885 if (SNaN)5886 return true;5887 // This can return a NaN if either operand is an sNaN, or if both operands5888 // are NaN.5889 return (isKnownNeverNaN(Op.getOperand(0), DemandedElts, false, Depth + 1) &&5890 isKnownNeverSNaN(Op.getOperand(1), DemandedElts, Depth + 1)) ||5891 (isKnownNeverNaN(Op.getOperand(1), DemandedElts, false, Depth + 1) &&5892 isKnownNeverSNaN(Op.getOperand(0), DemandedElts, Depth + 1));5893 }5894 case ISD::FMINIMUM:5895 case ISD::FMAXIMUM: {5896 // TODO: Does this quiet or return the origina NaN as-is?5897 return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1) &&5898 isKnownNeverNaN(Op.getOperand(1), DemandedElts, SNaN, Depth + 1);5899 }5900 case ISD::EXTRACT_VECTOR_ELT: {5901 SDValue Src = Op.getOperand(0);5902 auto *Idx = dyn_cast<ConstantSDNode>(Op.getOperand(1));5903 EVT SrcVT = Src.getValueType();5904 if (SrcVT.isFixedLengthVector() && Idx &&5905 Idx->getAPIntValue().ult(SrcVT.getVectorNumElements())) {5906 APInt DemandedSrcElts = APInt::getOneBitSet(SrcVT.getVectorNumElements(),5907 Idx->getZExtValue());5908 return isKnownNeverNaN(Src, DemandedSrcElts, SNaN, Depth + 1);5909 }5910 return isKnownNeverNaN(Src, SNaN, Depth + 1);5911 }5912 case ISD::EXTRACT_SUBVECTOR: {5913 SDValue Src = Op.getOperand(0);5914 if (Src.getValueType().isFixedLengthVector()) {5915 unsigned Idx = Op.getConstantOperandVal(1);5916 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();5917 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);5918 return isKnownNeverNaN(Src, DemandedSrcElts, SNaN, Depth + 1);5919 }5920 return isKnownNeverNaN(Src, SNaN, Depth + 1);5921 }5922 case ISD::INSERT_SUBVECTOR: {5923 SDValue BaseVector = Op.getOperand(0);5924 SDValue SubVector = Op.getOperand(1);5925 EVT BaseVectorVT = BaseVector.getValueType();5926 if (BaseVectorVT.isFixedLengthVector()) {5927 unsigned Idx = Op.getConstantOperandVal(2);5928 unsigned NumBaseElts = BaseVectorVT.getVectorNumElements();5929 unsigned NumSubElts = SubVector.getValueType().getVectorNumElements();5930 5931 // Clear/Extract the bits at the position where the subvector will be5932 // inserted.5933 APInt DemandedMask =5934 APInt::getBitsSet(NumBaseElts, Idx, Idx + NumSubElts);5935 APInt DemandedSrcElts = DemandedElts & ~DemandedMask;5936 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);5937 5938 bool NeverNaN = true;5939 if (!DemandedSrcElts.isZero())5940 NeverNaN &=5941 isKnownNeverNaN(BaseVector, DemandedSrcElts, SNaN, Depth + 1);5942 if (NeverNaN && !DemandedSubElts.isZero())5943 NeverNaN &=5944 isKnownNeverNaN(SubVector, DemandedSubElts, SNaN, Depth + 1);5945 return NeverNaN;5946 }5947 return isKnownNeverNaN(BaseVector, SNaN, Depth + 1) &&5948 isKnownNeverNaN(SubVector, SNaN, Depth + 1);5949 }5950 case ISD::BUILD_VECTOR: {5951 unsigned NumElts = Op.getNumOperands();5952 for (unsigned I = 0; I != NumElts; ++I)5953 if (DemandedElts[I] &&5954 !isKnownNeverNaN(Op.getOperand(I), SNaN, Depth + 1))5955 return false;5956 return true;5957 }5958 case ISD::AssertNoFPClass: {5959 FPClassTest NoFPClass =5960 static_cast<FPClassTest>(Op.getConstantOperandVal(1));5961 if ((NoFPClass & fcNan) == fcNan)5962 return true;5963 if (SNaN && (NoFPClass & fcSNan) == fcSNan)5964 return true;5965 return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1);5966 }5967 default:5968 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||5969 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {5970 return TLI->isKnownNeverNaNForTargetNode(Op, DemandedElts, *this, SNaN,5971 Depth);5972 }5973 5974 return false;5975 }5976}5977 5978bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const {5979 assert(Op.getValueType().isFloatingPoint() &&5980 "Floating point type expected");5981 5982 // If the value is a constant, we can obviously see if it is a zero or not.5983 return ISD::matchUnaryFpPredicate(5984 Op, [](ConstantFPSDNode *C) { return !C->isZero(); });5985}5986 5987bool SelectionDAG::isKnownNeverZero(SDValue Op, unsigned Depth) const {5988 if (Depth >= MaxRecursionDepth)5989 return false; // Limit search depth.5990 5991 assert(!Op.getValueType().isFloatingPoint() &&5992 "Floating point types unsupported - use isKnownNeverZeroFloat");5993 5994 // If the value is a constant, we can obviously see if it is a zero or not.5995 if (ISD::matchUnaryPredicate(Op,5996 [](ConstantSDNode *C) { return !C->isZero(); }))5997 return true;5998 5999 // TODO: Recognize more cases here. Most of the cases are also incomplete to6000 // some degree.6001 switch (Op.getOpcode()) {6002 default:6003 break;6004 6005 case ISD::OR:6006 return isKnownNeverZero(Op.getOperand(1), Depth + 1) ||6007 isKnownNeverZero(Op.getOperand(0), Depth + 1);6008 6009 case ISD::VSELECT:6010 case ISD::SELECT:6011 return isKnownNeverZero(Op.getOperand(1), Depth + 1) &&6012 isKnownNeverZero(Op.getOperand(2), Depth + 1);6013 6014 case ISD::SHL: {6015 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())6016 return isKnownNeverZero(Op.getOperand(0), Depth + 1);6017 KnownBits ValKnown = computeKnownBits(Op.getOperand(0), Depth + 1);6018 // 1 << X is never zero.6019 if (ValKnown.One[0])6020 return true;6021 // If max shift cnt of known ones is non-zero, result is non-zero.6022 APInt MaxCnt = computeKnownBits(Op.getOperand(1), Depth + 1).getMaxValue();6023 if (MaxCnt.ult(ValKnown.getBitWidth()) &&6024 !ValKnown.One.shl(MaxCnt).isZero())6025 return true;6026 break;6027 }6028 case ISD::UADDSAT:6029 case ISD::UMAX:6030 return isKnownNeverZero(Op.getOperand(1), Depth + 1) ||6031 isKnownNeverZero(Op.getOperand(0), Depth + 1);6032 6033 // For smin/smax: If either operand is known negative/positive6034 // respectively we don't need the other to be known at all.6035 case ISD::SMAX: {6036 KnownBits Op1 = computeKnownBits(Op.getOperand(1), Depth + 1);6037 if (Op1.isStrictlyPositive())6038 return true;6039 6040 KnownBits Op0 = computeKnownBits(Op.getOperand(0), Depth + 1);6041 if (Op0.isStrictlyPositive())6042 return true;6043 6044 if (Op1.isNonZero() && Op0.isNonZero())6045 return true;6046 6047 return isKnownNeverZero(Op.getOperand(1), Depth + 1) &&6048 isKnownNeverZero(Op.getOperand(0), Depth + 1);6049 }6050 case ISD::SMIN: {6051 KnownBits Op1 = computeKnownBits(Op.getOperand(1), Depth + 1);6052 if (Op1.isNegative())6053 return true;6054 6055 KnownBits Op0 = computeKnownBits(Op.getOperand(0), Depth + 1);6056 if (Op0.isNegative())6057 return true;6058 6059 if (Op1.isNonZero() && Op0.isNonZero())6060 return true;6061 6062 return isKnownNeverZero(Op.getOperand(1), Depth + 1) &&6063 isKnownNeverZero(Op.getOperand(0), Depth + 1);6064 }6065 case ISD::UMIN:6066 return isKnownNeverZero(Op.getOperand(1), Depth + 1) &&6067 isKnownNeverZero(Op.getOperand(0), Depth + 1);6068 6069 case ISD::ROTL:6070 case ISD::ROTR:6071 case ISD::BITREVERSE:6072 case ISD::BSWAP:6073 case ISD::CTPOP:6074 case ISD::ABS:6075 return isKnownNeverZero(Op.getOperand(0), Depth + 1);6076 6077 case ISD::SRA:6078 case ISD::SRL: {6079 if (Op->getFlags().hasExact())6080 return isKnownNeverZero(Op.getOperand(0), Depth + 1);6081 KnownBits ValKnown = computeKnownBits(Op.getOperand(0), Depth + 1);6082 if (ValKnown.isNegative())6083 return true;6084 // If max shift cnt of known ones is non-zero, result is non-zero.6085 APInt MaxCnt = computeKnownBits(Op.getOperand(1), Depth + 1).getMaxValue();6086 if (MaxCnt.ult(ValKnown.getBitWidth()) &&6087 !ValKnown.One.lshr(MaxCnt).isZero())6088 return true;6089 break;6090 }6091 case ISD::UDIV:6092 case ISD::SDIV:6093 // div exact can only produce a zero if the dividend is zero.6094 // TODO: For udiv this is also true if Op1 u<= Op06095 if (Op->getFlags().hasExact())6096 return isKnownNeverZero(Op.getOperand(0), Depth + 1);6097 break;6098 6099 case ISD::ADD:6100 if (Op->getFlags().hasNoUnsignedWrap())6101 if (isKnownNeverZero(Op.getOperand(1), Depth + 1) ||6102 isKnownNeverZero(Op.getOperand(0), Depth + 1))6103 return true;6104 // TODO: There are a lot more cases we can prove for add.6105 break;6106 6107 case ISD::SUB: {6108 if (isNullConstant(Op.getOperand(0)))6109 return isKnownNeverZero(Op.getOperand(1), Depth + 1);6110 6111 std::optional<bool> ne =6112 KnownBits::ne(computeKnownBits(Op.getOperand(0), Depth + 1),6113 computeKnownBits(Op.getOperand(1), Depth + 1));6114 return ne && *ne;6115 }6116 6117 case ISD::MUL:6118 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())6119 if (isKnownNeverZero(Op.getOperand(1), Depth + 1) &&6120 isKnownNeverZero(Op.getOperand(0), Depth + 1))6121 return true;6122 break;6123 6124 case ISD::ZERO_EXTEND:6125 case ISD::SIGN_EXTEND:6126 return isKnownNeverZero(Op.getOperand(0), Depth + 1);6127 case ISD::VSCALE: {6128 const Function &F = getMachineFunction().getFunction();6129 const APInt &Multiplier = Op.getConstantOperandAPInt(0);6130 ConstantRange CR =6131 getVScaleRange(&F, Op.getScalarValueSizeInBits()).multiply(Multiplier);6132 if (!CR.contains(APInt(CR.getBitWidth(), 0)))6133 return true;6134 break;6135 }6136 }6137 6138 return computeKnownBits(Op, Depth).isNonZero();6139}6140 6141bool SelectionDAG::cannotBeOrderedNegativeFP(SDValue Op) const {6142 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(Op, true))6143 return !C1->isNegative();6144 6145 switch (Op.getOpcode()) {6146 case ISD::FABS:6147 case ISD::FEXP:6148 case ISD::FEXP2:6149 case ISD::FEXP10:6150 return true;6151 default:6152 return false;6153 }6154 6155 llvm_unreachable("covered opcode switch");6156}6157 6158bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {6159 // Check the obvious case.6160 if (A == B) return true;6161 6162 // For negative and positive zero.6163 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))6164 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))6165 if (CA->isZero() && CB->isZero()) return true;6166 6167 // Otherwise they may not be equal.6168 return false;6169}6170 6171// Only bits set in Mask must be negated, other bits may be arbitrary.6172SDValue llvm::getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs) {6173 if (isBitwiseNot(V, AllowUndefs))6174 return V.getOperand(0);6175 6176 // Handle any_extend (not (truncate X)) pattern, where Mask only sets6177 // bits in the non-extended part.6178 ConstantSDNode *MaskC = isConstOrConstSplat(Mask);6179 if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND)6180 return SDValue();6181 SDValue ExtArg = V.getOperand(0);6182 if (ExtArg.getScalarValueSizeInBits() >=6183 MaskC->getAPIntValue().getActiveBits() &&6184 isBitwiseNot(ExtArg, AllowUndefs) &&6185 ExtArg.getOperand(0).getOpcode() == ISD::TRUNCATE &&6186 ExtArg.getOperand(0).getOperand(0).getValueType() == V.getValueType())6187 return ExtArg.getOperand(0).getOperand(0);6188 return SDValue();6189}6190 6191static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B) {6192 // Match masked merge pattern (X & ~M) op (Y & M)6193 // Including degenerate case (X & ~M) op M6194 auto MatchNoCommonBitsPattern = [&](SDValue Not, SDValue Mask,6195 SDValue Other) {6196 if (SDValue NotOperand =6197 getBitwiseNotOperand(Not, Mask, /* AllowUndefs */ true)) {6198 if (NotOperand->getOpcode() == ISD::ZERO_EXTEND ||6199 NotOperand->getOpcode() == ISD::TRUNCATE)6200 NotOperand = NotOperand->getOperand(0);6201 6202 if (Other == NotOperand)6203 return true;6204 if (Other->getOpcode() == ISD::AND)6205 return NotOperand == Other->getOperand(0) ||6206 NotOperand == Other->getOperand(1);6207 }6208 return false;6209 };6210 6211 if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE)6212 A = A->getOperand(0);6213 6214 if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE)6215 B = B->getOperand(0);6216 6217 if (A->getOpcode() == ISD::AND)6218 return MatchNoCommonBitsPattern(A->getOperand(0), A->getOperand(1), B) ||6219 MatchNoCommonBitsPattern(A->getOperand(1), A->getOperand(0), B);6220 return false;6221}6222 6223// FIXME: unify with llvm::haveNoCommonBitsSet.6224bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {6225 assert(A.getValueType() == B.getValueType() &&6226 "Values must have the same type");6227 if (haveNoCommonBitsSetCommutative(A, B) ||6228 haveNoCommonBitsSetCommutative(B, A))6229 return true;6230 return KnownBits::haveNoCommonBitsSet(computeKnownBits(A),6231 computeKnownBits(B));6232}6233 6234static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,6235 SelectionDAG &DAG) {6236 if (cast<ConstantSDNode>(Step)->isZero())6237 return DAG.getConstant(0, DL, VT);6238 6239 return SDValue();6240}6241 6242static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,6243 ArrayRef<SDValue> Ops,6244 SelectionDAG &DAG) {6245 int NumOps = Ops.size();6246 assert(NumOps != 0 && "Can't build an empty vector!");6247 assert(!VT.isScalableVector() &&6248 "BUILD_VECTOR cannot be used with scalable types");6249 assert(VT.getVectorNumElements() == (unsigned)NumOps &&6250 "Incorrect element count in BUILD_VECTOR!");6251 6252 // BUILD_VECTOR of UNDEFs is UNDEF.6253 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))6254 return DAG.getUNDEF(VT);6255 6256 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.6257 SDValue IdentitySrc;6258 bool IsIdentity = true;6259 for (int i = 0; i != NumOps; ++i) {6260 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||6261 Ops[i].getOperand(0).getValueType() != VT ||6262 (IdentitySrc && Ops[i].getOperand(0) != IdentitySrc) ||6263 !isa<ConstantSDNode>(Ops[i].getOperand(1)) ||6264 Ops[i].getConstantOperandAPInt(1) != i) {6265 IsIdentity = false;6266 break;6267 }6268 IdentitySrc = Ops[i].getOperand(0);6269 }6270 if (IsIdentity)6271 return IdentitySrc;6272 6273 return SDValue();6274}6275 6276/// Try to simplify vector concatenation to an input value, undef, or build6277/// vector.6278static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,6279 ArrayRef<SDValue> Ops,6280 SelectionDAG &DAG) {6281 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");6282 assert(llvm::all_of(Ops,6283 [Ops](SDValue Op) {6284 return Ops[0].getValueType() == Op.getValueType();6285 }) &&6286 "Concatenation of vectors with inconsistent value types!");6287 assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==6288 VT.getVectorElementCount() &&6289 "Incorrect element count in vector concatenation!");6290 6291 if (Ops.size() == 1)6292 return Ops[0];6293 6294 // Concat of UNDEFs is UNDEF.6295 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))6296 return DAG.getUNDEF(VT);6297 6298 // Scan the operands and look for extract operations from a single source6299 // that correspond to insertion at the same location via this concatenation:6300 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...6301 SDValue IdentitySrc;6302 bool IsIdentity = true;6303 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {6304 SDValue Op = Ops[i];6305 unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();6306 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||6307 Op.getOperand(0).getValueType() != VT ||6308 (IdentitySrc && Op.getOperand(0) != IdentitySrc) ||6309 Op.getConstantOperandVal(1) != IdentityIndex) {6310 IsIdentity = false;6311 break;6312 }6313 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&6314 "Unexpected identity source vector for concat of extracts");6315 IdentitySrc = Op.getOperand(0);6316 }6317 if (IsIdentity) {6318 assert(IdentitySrc && "Failed to set source vector of extracts");6319 return IdentitySrc;6320 }6321 6322 // The code below this point is only designed to work for fixed width6323 // vectors, so we bail out for now.6324 if (VT.isScalableVector())6325 return SDValue();6326 6327 // A CONCAT_VECTOR of scalar sources, such as UNDEF, BUILD_VECTOR and6328 // single-element INSERT_VECTOR_ELT operands can be simplified to one big6329 // BUILD_VECTOR.6330 // FIXME: Add support for SCALAR_TO_VECTOR as well.6331 EVT SVT = VT.getScalarType();6332 SmallVector<SDValue, 16> Elts;6333 for (SDValue Op : Ops) {6334 EVT OpVT = Op.getValueType();6335 if (Op.isUndef())6336 Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT));6337 else if (Op.getOpcode() == ISD::BUILD_VECTOR)6338 Elts.append(Op->op_begin(), Op->op_end());6339 else if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT &&6340 OpVT.getVectorNumElements() == 1 &&6341 isNullConstant(Op.getOperand(2)))6342 Elts.push_back(Op.getOperand(1));6343 else6344 return SDValue();6345 }6346 6347 // BUILD_VECTOR requires all inputs to be of the same type, find the6348 // maximum type and extend them all.6349 for (SDValue Op : Elts)6350 SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);6351 6352 if (SVT.bitsGT(VT.getScalarType())) {6353 for (SDValue &Op : Elts) {6354 if (Op.isUndef())6355 Op = DAG.getUNDEF(SVT);6356 else6357 Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT)6358 ? DAG.getZExtOrTrunc(Op, DL, SVT)6359 : DAG.getSExtOrTrunc(Op, DL, SVT);6360 }6361 }6362 6363 SDValue V = DAG.getBuildVector(VT, DL, Elts);6364 NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG);6365 return V;6366}6367 6368/// Gets or creates the specified node.6369SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {6370 SDVTList VTs = getVTList(VT);6371 FoldingSetNodeID ID;6372 AddNodeIDNode(ID, Opcode, VTs, {});6373 void *IP = nullptr;6374 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))6375 return SDValue(E, 0);6376 6377 auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);6378 CSEMap.InsertNode(N, IP);6379 6380 InsertNode(N);6381 SDValue V = SDValue(N, 0);6382 NewSDValueDbgMsg(V, "Creating new node: ", this);6383 return V;6384}6385 6386SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,6387 SDValue N1) {6388 SDNodeFlags Flags;6389 if (Inserter)6390 Flags = Inserter->getFlags();6391 return getNode(Opcode, DL, VT, N1, Flags);6392}6393 6394SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,6395 SDValue N1, const SDNodeFlags Flags) {6396 assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!");6397 6398 // Constant fold unary operations with a vector integer or float operand.6399 switch (Opcode) {6400 default:6401 // FIXME: Entirely reasonable to perform folding of other unary6402 // operations here as the need arises.6403 break;6404 case ISD::FNEG:6405 case ISD::FABS:6406 case ISD::FCEIL:6407 case ISD::FTRUNC:6408 case ISD::FFLOOR:6409 case ISD::FP_EXTEND:6410 case ISD::FP_TO_SINT:6411 case ISD::FP_TO_UINT:6412 case ISD::FP_TO_FP16:6413 case ISD::FP_TO_BF16:6414 case ISD::TRUNCATE:6415 case ISD::ANY_EXTEND:6416 case ISD::ZERO_EXTEND:6417 case ISD::SIGN_EXTEND:6418 case ISD::UINT_TO_FP:6419 case ISD::SINT_TO_FP:6420 case ISD::FP16_TO_FP:6421 case ISD::BF16_TO_FP:6422 case ISD::BITCAST:6423 case ISD::ABS:6424 case ISD::BITREVERSE:6425 case ISD::BSWAP:6426 case ISD::CTLZ:6427 case ISD::CTLZ_ZERO_UNDEF:6428 case ISD::CTTZ:6429 case ISD::CTTZ_ZERO_UNDEF:6430 case ISD::CTPOP:6431 case ISD::STEP_VECTOR: {6432 SDValue Ops = {N1};6433 if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops))6434 return Fold;6435 }6436 }6437 6438 unsigned OpOpcode = N1.getNode()->getOpcode();6439 switch (Opcode) {6440 case ISD::STEP_VECTOR:6441 assert(VT.isScalableVector() &&6442 "STEP_VECTOR can only be used with scalable types");6443 assert(OpOpcode == ISD::TargetConstant &&6444 VT.getVectorElementType() == N1.getValueType() &&6445 "Unexpected step operand");6446 break;6447 case ISD::FREEZE:6448 assert(VT == N1.getValueType() && "Unexpected VT!");6449 if (isGuaranteedNotToBeUndefOrPoison(N1, /*PoisonOnly=*/false))6450 return N1;6451 break;6452 case ISD::TokenFactor:6453 case ISD::MERGE_VALUES:6454 case ISD::CONCAT_VECTORS:6455 return N1; // Factor, merge or concat of one node? No need.6456 case ISD::BUILD_VECTOR: {6457 // Attempt to simplify BUILD_VECTOR.6458 SDValue Ops[] = {N1};6459 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))6460 return V;6461 break;6462 }6463 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");6464 case ISD::FP_EXTEND:6465 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&6466 "Invalid FP cast!");6467 if (N1.getValueType() == VT) return N1; // noop conversion.6468 assert((!VT.isVector() || VT.getVectorElementCount() ==6469 N1.getValueType().getVectorElementCount()) &&6470 "Vector element count mismatch!");6471 assert(N1.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!");6472 if (N1.isUndef())6473 return getUNDEF(VT);6474 break;6475 case ISD::FP_TO_SINT:6476 case ISD::FP_TO_UINT:6477 if (N1.isUndef())6478 return getUNDEF(VT);6479 break;6480 case ISD::SINT_TO_FP:6481 case ISD::UINT_TO_FP:6482 // [us]itofp(undef) = 0, because the result value is bounded.6483 if (N1.isUndef())6484 return getConstantFP(0.0, DL, VT);6485 break;6486 case ISD::SIGN_EXTEND:6487 assert(VT.isInteger() && N1.getValueType().isInteger() &&6488 "Invalid SIGN_EXTEND!");6489 assert(VT.isVector() == N1.getValueType().isVector() &&6490 "SIGN_EXTEND result type type should be vector iff the operand "6491 "type is vector!");6492 if (N1.getValueType() == VT) return N1; // noop extension6493 assert((!VT.isVector() || VT.getVectorElementCount() ==6494 N1.getValueType().getVectorElementCount()) &&6495 "Vector element count mismatch!");6496 assert(N1.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!");6497 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) {6498 SDNodeFlags Flags;6499 if (OpOpcode == ISD::ZERO_EXTEND)6500 Flags.setNonNeg(N1->getFlags().hasNonNeg());6501 SDValue NewVal = getNode(OpOpcode, DL, VT, N1.getOperand(0), Flags);6502 transferDbgValues(N1, NewVal);6503 return NewVal;6504 }6505 6506 if (OpOpcode == ISD::POISON)6507 return getPOISON(VT);6508 6509 if (N1.isUndef())6510 // sext(undef) = 0, because the top bits will all be the same.6511 return getConstant(0, DL, VT);6512 6513 // Skip unnecessary sext_inreg pattern:6514 // (sext (trunc x)) -> x iff the upper bits are all signbits.6515 if (OpOpcode == ISD::TRUNCATE) {6516 SDValue OpOp = N1.getOperand(0);6517 if (OpOp.getValueType() == VT) {6518 unsigned NumSignExtBits =6519 VT.getScalarSizeInBits() - N1.getScalarValueSizeInBits();6520 if (ComputeNumSignBits(OpOp) > NumSignExtBits) {6521 transferDbgValues(N1, OpOp);6522 return OpOp;6523 }6524 }6525 }6526 break;6527 case ISD::ZERO_EXTEND:6528 assert(VT.isInteger() && N1.getValueType().isInteger() &&6529 "Invalid ZERO_EXTEND!");6530 assert(VT.isVector() == N1.getValueType().isVector() &&6531 "ZERO_EXTEND result type type should be vector iff the operand "6532 "type is vector!");6533 if (N1.getValueType() == VT) return N1; // noop extension6534 assert((!VT.isVector() || VT.getVectorElementCount() ==6535 N1.getValueType().getVectorElementCount()) &&6536 "Vector element count mismatch!");6537 assert(N1.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!");6538 if (OpOpcode == ISD::ZERO_EXTEND) { // (zext (zext x)) -> (zext x)6539 SDNodeFlags Flags;6540 Flags.setNonNeg(N1->getFlags().hasNonNeg());6541 SDValue NewVal =6542 getNode(ISD::ZERO_EXTEND, DL, VT, N1.getOperand(0), Flags);6543 transferDbgValues(N1, NewVal);6544 return NewVal;6545 }6546 6547 if (OpOpcode == ISD::POISON)6548 return getPOISON(VT);6549 6550 if (N1.isUndef())6551 // zext(undef) = 0, because the top bits will be zero.6552 return getConstant(0, DL, VT);6553 6554 // Skip unnecessary zext_inreg pattern:6555 // (zext (trunc x)) -> x iff the upper bits are known zero.6556 // TODO: Remove (zext (trunc (and x, c))) exception which some targets6557 // use to recognise zext_inreg patterns.6558 if (OpOpcode == ISD::TRUNCATE) {6559 SDValue OpOp = N1.getOperand(0);6560 if (OpOp.getValueType() == VT) {6561 if (OpOp.getOpcode() != ISD::AND) {6562 APInt HiBits = APInt::getBitsSetFrom(VT.getScalarSizeInBits(),6563 N1.getScalarValueSizeInBits());6564 if (MaskedValueIsZero(OpOp, HiBits)) {6565 transferDbgValues(N1, OpOp);6566 return OpOp;6567 }6568 }6569 }6570 }6571 break;6572 case ISD::ANY_EXTEND:6573 assert(VT.isInteger() && N1.getValueType().isInteger() &&6574 "Invalid ANY_EXTEND!");6575 assert(VT.isVector() == N1.getValueType().isVector() &&6576 "ANY_EXTEND result type type should be vector iff the operand "6577 "type is vector!");6578 if (N1.getValueType() == VT) return N1; // noop extension6579 assert((!VT.isVector() || VT.getVectorElementCount() ==6580 N1.getValueType().getVectorElementCount()) &&6581 "Vector element count mismatch!");6582 assert(N1.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!");6583 6584 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||6585 OpOpcode == ISD::ANY_EXTEND) {6586 SDNodeFlags Flags;6587 if (OpOpcode == ISD::ZERO_EXTEND)6588 Flags.setNonNeg(N1->getFlags().hasNonNeg());6589 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)6590 return getNode(OpOpcode, DL, VT, N1.getOperand(0), Flags);6591 }6592 if (N1.isUndef())6593 return getUNDEF(VT);6594 6595 // (ext (trunc x)) -> x6596 if (OpOpcode == ISD::TRUNCATE) {6597 SDValue OpOp = N1.getOperand(0);6598 if (OpOp.getValueType() == VT) {6599 transferDbgValues(N1, OpOp);6600 return OpOp;6601 }6602 }6603 break;6604 case ISD::TRUNCATE:6605 assert(VT.isInteger() && N1.getValueType().isInteger() &&6606 "Invalid TRUNCATE!");6607 assert(VT.isVector() == N1.getValueType().isVector() &&6608 "TRUNCATE result type type should be vector iff the operand "6609 "type is vector!");6610 if (N1.getValueType() == VT) return N1; // noop truncate6611 assert((!VT.isVector() || VT.getVectorElementCount() ==6612 N1.getValueType().getVectorElementCount()) &&6613 "Vector element count mismatch!");6614 assert(N1.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!");6615 if (OpOpcode == ISD::TRUNCATE)6616 return getNode(ISD::TRUNCATE, DL, VT, N1.getOperand(0));6617 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||6618 OpOpcode == ISD::ANY_EXTEND) {6619 // If the source is smaller than the dest, we still need an extend.6620 if (N1.getOperand(0).getValueType().getScalarType().bitsLT(6621 VT.getScalarType())) {6622 SDNodeFlags Flags;6623 if (OpOpcode == ISD::ZERO_EXTEND)6624 Flags.setNonNeg(N1->getFlags().hasNonNeg());6625 return getNode(OpOpcode, DL, VT, N1.getOperand(0), Flags);6626 }6627 if (N1.getOperand(0).getValueType().bitsGT(VT))6628 return getNode(ISD::TRUNCATE, DL, VT, N1.getOperand(0));6629 return N1.getOperand(0);6630 }6631 if (N1.isUndef())6632 return getUNDEF(VT);6633 if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes)6634 return getVScale(DL, VT,6635 N1.getConstantOperandAPInt(0).trunc(VT.getSizeInBits()));6636 break;6637 case ISD::ANY_EXTEND_VECTOR_INREG:6638 case ISD::ZERO_EXTEND_VECTOR_INREG:6639 case ISD::SIGN_EXTEND_VECTOR_INREG:6640 assert(VT.isVector() && "This DAG node is restricted to vector types.");6641 assert(N1.getValueType().bitsLE(VT) &&6642 "The input must be the same size or smaller than the result.");6643 assert(VT.getVectorMinNumElements() <6644 N1.getValueType().getVectorMinNumElements() &&6645 "The destination vector type must have fewer lanes than the input.");6646 break;6647 case ISD::ABS:6648 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid ABS!");6649 if (N1.isUndef())6650 return getConstant(0, DL, VT);6651 break;6652 case ISD::BSWAP:6653 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BSWAP!");6654 assert((VT.getScalarSizeInBits() % 16 == 0) &&6655 "BSWAP types must be a multiple of 16 bits!");6656 if (N1.isUndef())6657 return getUNDEF(VT);6658 // bswap(bswap(X)) -> X.6659 if (OpOpcode == ISD::BSWAP)6660 return N1.getOperand(0);6661 break;6662 case ISD::BITREVERSE:6663 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BITREVERSE!");6664 if (N1.isUndef())6665 return getUNDEF(VT);6666 break;6667 case ISD::BITCAST:6668 assert(VT.getSizeInBits() == N1.getValueSizeInBits() &&6669 "Cannot BITCAST between types of different sizes!");6670 if (VT == N1.getValueType()) return N1; // noop conversion.6671 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)6672 return getNode(ISD::BITCAST, DL, VT, N1.getOperand(0));6673 if (N1.isUndef())6674 return getUNDEF(VT);6675 break;6676 case ISD::SCALAR_TO_VECTOR:6677 assert(VT.isVector() && !N1.getValueType().isVector() &&6678 (VT.getVectorElementType() == N1.getValueType() ||6679 (VT.getVectorElementType().isInteger() &&6680 N1.getValueType().isInteger() &&6681 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&6682 "Illegal SCALAR_TO_VECTOR node!");6683 if (N1.isUndef())6684 return getUNDEF(VT);6685 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.6686 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&6687 isa<ConstantSDNode>(N1.getOperand(1)) &&6688 N1.getConstantOperandVal(1) == 0 &&6689 N1.getOperand(0).getValueType() == VT)6690 return N1.getOperand(0);6691 break;6692 case ISD::FNEG:6693 // Negation of an unknown bag of bits is still completely undefined.6694 if (N1.isUndef())6695 return getUNDEF(VT);6696 6697 if (OpOpcode == ISD::FNEG) // --X -> X6698 return N1.getOperand(0);6699 break;6700 case ISD::FABS:6701 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)6702 return getNode(ISD::FABS, DL, VT, N1.getOperand(0));6703 break;6704 case ISD::VSCALE:6705 assert(VT == N1.getValueType() && "Unexpected VT!");6706 break;6707 case ISD::CTPOP:6708 if (N1.getValueType().getScalarType() == MVT::i1)6709 return N1;6710 break;6711 case ISD::CTLZ:6712 case ISD::CTTZ:6713 if (N1.getValueType().getScalarType() == MVT::i1)6714 return getNOT(DL, N1, N1.getValueType());6715 break;6716 case ISD::VECREDUCE_ADD:6717 if (N1.getValueType().getScalarType() == MVT::i1)6718 return getNode(ISD::VECREDUCE_XOR, DL, VT, N1);6719 break;6720 case ISD::VECREDUCE_SMIN:6721 case ISD::VECREDUCE_UMAX:6722 if (N1.getValueType().getScalarType() == MVT::i1)6723 return getNode(ISD::VECREDUCE_OR, DL, VT, N1);6724 break;6725 case ISD::VECREDUCE_SMAX:6726 case ISD::VECREDUCE_UMIN:6727 if (N1.getValueType().getScalarType() == MVT::i1)6728 return getNode(ISD::VECREDUCE_AND, DL, VT, N1);6729 break;6730 case ISD::SPLAT_VECTOR:6731 assert(VT.isVector() && "Wrong return type!");6732 // FIXME: Hexagon uses i32 scalar for a floating point zero vector so allow6733 // that for now.6734 assert((VT.getVectorElementType() == N1.getValueType() ||6735 (VT.isFloatingPoint() && N1.getValueType() == MVT::i32) ||6736 (VT.getVectorElementType().isInteger() &&6737 N1.getValueType().isInteger() &&6738 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&6739 "Wrong operand type!");6740 break;6741 }6742 6743 SDNode *N;6744 SDVTList VTs = getVTList(VT);6745 SDValue Ops[] = {N1};6746 if (VT != MVT::Glue) { // Don't CSE glue producing nodes6747 FoldingSetNodeID ID;6748 AddNodeIDNode(ID, Opcode, VTs, Ops);6749 void *IP = nullptr;6750 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {6751 E->intersectFlagsWith(Flags);6752 return SDValue(E, 0);6753 }6754 6755 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);6756 N->setFlags(Flags);6757 createOperands(N, Ops);6758 CSEMap.InsertNode(N, IP);6759 } else {6760 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);6761 createOperands(N, Ops);6762 }6763 6764 InsertNode(N);6765 SDValue V = SDValue(N, 0);6766 NewSDValueDbgMsg(V, "Creating new node: ", this);6767 return V;6768}6769 6770static std::optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,6771 const APInt &C2) {6772 switch (Opcode) {6773 case ISD::ADD: return C1 + C2;6774 case ISD::SUB: return C1 - C2;6775 case ISD::MUL: return C1 * C2;6776 case ISD::AND: return C1 & C2;6777 case ISD::OR: return C1 | C2;6778 case ISD::XOR: return C1 ^ C2;6779 case ISD::SHL: return C1 << C2;6780 case ISD::SRL: return C1.lshr(C2);6781 case ISD::SRA: return C1.ashr(C2);6782 case ISD::ROTL: return C1.rotl(C2);6783 case ISD::ROTR: return C1.rotr(C2);6784 case ISD::SMIN: return C1.sle(C2) ? C1 : C2;6785 case ISD::SMAX: return C1.sge(C2) ? C1 : C2;6786 case ISD::UMIN: return C1.ule(C2) ? C1 : C2;6787 case ISD::UMAX: return C1.uge(C2) ? C1 : C2;6788 case ISD::SADDSAT: return C1.sadd_sat(C2);6789 case ISD::UADDSAT: return C1.uadd_sat(C2);6790 case ISD::SSUBSAT: return C1.ssub_sat(C2);6791 case ISD::USUBSAT: return C1.usub_sat(C2);6792 case ISD::SSHLSAT: return C1.sshl_sat(C2);6793 case ISD::USHLSAT: return C1.ushl_sat(C2);6794 case ISD::UDIV:6795 if (!C2.getBoolValue())6796 break;6797 return C1.udiv(C2);6798 case ISD::UREM:6799 if (!C2.getBoolValue())6800 break;6801 return C1.urem(C2);6802 case ISD::SDIV:6803 if (!C2.getBoolValue())6804 break;6805 return C1.sdiv(C2);6806 case ISD::SREM:6807 if (!C2.getBoolValue())6808 break;6809 return C1.srem(C2);6810 case ISD::AVGFLOORS:6811 return APIntOps::avgFloorS(C1, C2);6812 case ISD::AVGFLOORU:6813 return APIntOps::avgFloorU(C1, C2);6814 case ISD::AVGCEILS:6815 return APIntOps::avgCeilS(C1, C2);6816 case ISD::AVGCEILU:6817 return APIntOps::avgCeilU(C1, C2);6818 case ISD::ABDS:6819 return APIntOps::abds(C1, C2);6820 case ISD::ABDU:6821 return APIntOps::abdu(C1, C2);6822 case ISD::MULHS:6823 return APIntOps::mulhs(C1, C2);6824 case ISD::MULHU:6825 return APIntOps::mulhu(C1, C2);6826 }6827 return std::nullopt;6828}6829// Handle constant folding with UNDEF.6830// TODO: Handle more cases.6831static std::optional<APInt> FoldValueWithUndef(unsigned Opcode, const APInt &C1,6832 bool IsUndef1, const APInt &C2,6833 bool IsUndef2) {6834 if (!(IsUndef1 || IsUndef2))6835 return FoldValue(Opcode, C1, C2);6836 6837 // Fold and(x, undef) -> 06838 // Fold mul(x, undef) -> 06839 if (Opcode == ISD::AND || Opcode == ISD::MUL)6840 return APInt::getZero(C1.getBitWidth());6841 6842 return std::nullopt;6843}6844 6845SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,6846 const GlobalAddressSDNode *GA,6847 const SDNode *N2) {6848 if (GA->getOpcode() != ISD::GlobalAddress)6849 return SDValue();6850 if (!TLI->isOffsetFoldingLegal(GA))6851 return SDValue();6852 auto *C2 = dyn_cast<ConstantSDNode>(N2);6853 if (!C2)6854 return SDValue();6855 int64_t Offset = C2->getSExtValue();6856 switch (Opcode) {6857 case ISD::ADD:6858 case ISD::PTRADD:6859 break;6860 case ISD::SUB: Offset = -uint64_t(Offset); break;6861 default: return SDValue();6862 }6863 return getGlobalAddress(GA->getGlobal(), SDLoc(C2), VT,6864 GA->getOffset() + uint64_t(Offset));6865}6866 6867bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {6868 switch (Opcode) {6869 case ISD::SDIV:6870 case ISD::UDIV:6871 case ISD::SREM:6872 case ISD::UREM: {6873 // If a divisor is zero/undef or any element of a divisor vector is6874 // zero/undef, the whole op is undef.6875 assert(Ops.size() == 2 && "Div/rem should have 2 operands");6876 SDValue Divisor = Ops[1];6877 if (Divisor.isUndef() || isNullConstant(Divisor))6878 return true;6879 6880 return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) &&6881 llvm::any_of(Divisor->op_values(),6882 [](SDValue V) { return V.isUndef() ||6883 isNullConstant(V); });6884 // TODO: Handle signed overflow.6885 }6886 // TODO: Handle oversized shifts.6887 default:6888 return false;6889 }6890}6891 6892SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,6893 EVT VT, ArrayRef<SDValue> Ops,6894 SDNodeFlags Flags) {6895 // If the opcode is a target-specific ISD node, there's nothing we can6896 // do here and the operand rules may not line up with the below, so6897 // bail early.6898 // We can't create a scalar CONCAT_VECTORS so skip it. It will break6899 // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by6900 // foldCONCAT_VECTORS in getNode before this is called.6901 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)6902 return SDValue();6903 6904 unsigned NumOps = Ops.size();6905 if (NumOps == 0)6906 return SDValue();6907 6908 if (isUndef(Opcode, Ops))6909 return getUNDEF(VT);6910 6911 // Handle unary special cases.6912 if (NumOps == 1) {6913 SDValue N1 = Ops[0];6914 6915 // Constant fold unary operations with an integer constant operand. Even6916 // opaque constant will be folded, because the folding of unary operations6917 // doesn't create new constants with different values. Nevertheless, the6918 // opaque flag is preserved during folding to prevent future folding with6919 // other constants.6920 if (auto *C = dyn_cast<ConstantSDNode>(N1)) {6921 const APInt &Val = C->getAPIntValue();6922 switch (Opcode) {6923 case ISD::SIGN_EXTEND:6924 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,6925 C->isTargetOpcode(), C->isOpaque());6926 case ISD::TRUNCATE:6927 if (C->isOpaque())6928 break;6929 [[fallthrough]];6930 case ISD::ZERO_EXTEND:6931 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,6932 C->isTargetOpcode(), C->isOpaque());6933 case ISD::ANY_EXTEND:6934 // Some targets like RISCV prefer to sign extend some types.6935 if (TLI->isSExtCheaperThanZExt(N1.getValueType(), VT))6936 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,6937 C->isTargetOpcode(), C->isOpaque());6938 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,6939 C->isTargetOpcode(), C->isOpaque());6940 case ISD::ABS:6941 return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(),6942 C->isOpaque());6943 case ISD::BITREVERSE:6944 return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(),6945 C->isOpaque());6946 case ISD::BSWAP:6947 return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(),6948 C->isOpaque());6949 case ISD::CTPOP:6950 return getConstant(Val.popcount(), DL, VT, C->isTargetOpcode(),6951 C->isOpaque());6952 case ISD::CTLZ:6953 case ISD::CTLZ_ZERO_UNDEF:6954 return getConstant(Val.countl_zero(), DL, VT, C->isTargetOpcode(),6955 C->isOpaque());6956 case ISD::CTTZ:6957 case ISD::CTTZ_ZERO_UNDEF:6958 return getConstant(Val.countr_zero(), DL, VT, C->isTargetOpcode(),6959 C->isOpaque());6960 case ISD::UINT_TO_FP:6961 case ISD::SINT_TO_FP: {6962 APFloat FPV(VT.getFltSemantics(), APInt::getZero(VT.getSizeInBits()));6963 (void)FPV.convertFromAPInt(Val, Opcode == ISD::SINT_TO_FP,6964 APFloat::rmNearestTiesToEven);6965 return getConstantFP(FPV, DL, VT);6966 }6967 case ISD::FP16_TO_FP:6968 case ISD::BF16_TO_FP: {6969 bool Ignored;6970 APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf()6971 : APFloat::BFloat(),6972 (Val.getBitWidth() == 16) ? Val : Val.trunc(16));6973 6974 // This can return overflow, underflow, or inexact; we don't care.6975 // FIXME need to be more flexible about rounding mode.6976 (void)FPV.convert(VT.getFltSemantics(), APFloat::rmNearestTiesToEven,6977 &Ignored);6978 return getConstantFP(FPV, DL, VT);6979 }6980 case ISD::STEP_VECTOR:6981 if (SDValue V = FoldSTEP_VECTOR(DL, VT, N1, *this))6982 return V;6983 break;6984 case ISD::BITCAST:6985 if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)6986 return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT);6987 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)6988 return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT);6989 if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)6990 return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT);6991 if (VT == MVT::f128 && C->getValueType(0) == MVT::i128)6992 return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT);6993 break;6994 }6995 }6996 6997 // Constant fold unary operations with a floating point constant operand.6998 if (auto *C = dyn_cast<ConstantFPSDNode>(N1)) {6999 APFloat V = C->getValueAPF(); // make copy7000 switch (Opcode) {7001 case ISD::FNEG:7002 V.changeSign();7003 return getConstantFP(V, DL, VT);7004 case ISD::FABS:7005 V.clearSign();7006 return getConstantFP(V, DL, VT);7007 case ISD::FCEIL: {7008 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);7009 if (fs == APFloat::opOK || fs == APFloat::opInexact)7010 return getConstantFP(V, DL, VT);7011 return SDValue();7012 }7013 case ISD::FTRUNC: {7014 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);7015 if (fs == APFloat::opOK || fs == APFloat::opInexact)7016 return getConstantFP(V, DL, VT);7017 return SDValue();7018 }7019 case ISD::FFLOOR: {7020 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);7021 if (fs == APFloat::opOK || fs == APFloat::opInexact)7022 return getConstantFP(V, DL, VT);7023 return SDValue();7024 }7025 case ISD::FP_EXTEND: {7026 bool ignored;7027 // This can return overflow, underflow, or inexact; we don't care.7028 // FIXME need to be more flexible about rounding mode.7029 (void)V.convert(VT.getFltSemantics(), APFloat::rmNearestTiesToEven,7030 &ignored);7031 return getConstantFP(V, DL, VT);7032 }7033 case ISD::FP_TO_SINT:7034 case ISD::FP_TO_UINT: {7035 bool ignored;7036 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);7037 // FIXME need to be more flexible about rounding mode.7038 APFloat::opStatus s =7039 V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored);7040 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual7041 break;7042 return getConstant(IntVal, DL, VT);7043 }7044 case ISD::FP_TO_FP16:7045 case ISD::FP_TO_BF16: {7046 bool Ignored;7047 // This can return overflow, underflow, or inexact; we don't care.7048 // FIXME need to be more flexible about rounding mode.7049 (void)V.convert(Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf()7050 : APFloat::BFloat(),7051 APFloat::rmNearestTiesToEven, &Ignored);7052 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);7053 }7054 case ISD::BITCAST:7055 if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)7056 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL,7057 VT);7058 if (VT == MVT::i16 && C->getValueType(0) == MVT::bf16)7059 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL,7060 VT);7061 if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)7062 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL,7063 VT);7064 if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)7065 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);7066 break;7067 }7068 }7069 7070 // Early-out if we failed to constant fold a bitcast.7071 if (Opcode == ISD::BITCAST)7072 return SDValue();7073 }7074 7075 // Handle binops special cases.7076 if (NumOps == 2) {7077 if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops))7078 return CFP;7079 7080 if (auto *C1 = dyn_cast<ConstantSDNode>(Ops[0])) {7081 if (auto *C2 = dyn_cast<ConstantSDNode>(Ops[1])) {7082 if (C1->isOpaque() || C2->isOpaque())7083 return SDValue();7084 7085 std::optional<APInt> FoldAttempt =7086 FoldValue(Opcode, C1->getAPIntValue(), C2->getAPIntValue());7087 if (!FoldAttempt)7088 return SDValue();7089 7090 SDValue Folded = getConstant(*FoldAttempt, DL, VT);7091 assert((!Folded || !VT.isVector()) &&7092 "Can't fold vectors ops with scalar operands");7093 return Folded;7094 }7095 }7096 7097 // fold (add Sym, c) -> Sym+c7098 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Ops[0]))7099 return FoldSymbolOffset(Opcode, VT, GA, Ops[1].getNode());7100 if (TLI->isCommutativeBinOp(Opcode))7101 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Ops[1]))7102 return FoldSymbolOffset(Opcode, VT, GA, Ops[0].getNode());7103 7104 // fold (sext_in_reg c1) -> c27105 if (Opcode == ISD::SIGN_EXTEND_INREG) {7106 EVT EVT = cast<VTSDNode>(Ops[1])->getVT();7107 7108 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {7109 unsigned FromBits = EVT.getScalarSizeInBits();7110 Val <<= Val.getBitWidth() - FromBits;7111 Val.ashrInPlace(Val.getBitWidth() - FromBits);7112 return getConstant(Val, DL, ConstantVT);7113 };7114 7115 if (auto *C1 = dyn_cast<ConstantSDNode>(Ops[0])) {7116 const APInt &Val = C1->getAPIntValue();7117 return SignExtendInReg(Val, VT);7118 }7119 7120 if (ISD::isBuildVectorOfConstantSDNodes(Ops[0].getNode())) {7121 SmallVector<SDValue, 8> ScalarOps;7122 llvm::EVT OpVT = Ops[0].getOperand(0).getValueType();7123 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I) {7124 SDValue Op = Ops[0].getOperand(I);7125 if (Op.isUndef()) {7126 ScalarOps.push_back(getUNDEF(OpVT));7127 continue;7128 }7129 const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue();7130 ScalarOps.push_back(SignExtendInReg(Val, OpVT));7131 }7132 return getBuildVector(VT, DL, ScalarOps);7133 }7134 7135 if (Ops[0].getOpcode() == ISD::SPLAT_VECTOR &&7136 isa<ConstantSDNode>(Ops[0].getOperand(0)))7137 return getNode(ISD::SPLAT_VECTOR, DL, VT,7138 SignExtendInReg(Ops[0].getConstantOperandAPInt(0),7139 Ops[0].getOperand(0).getValueType()));7140 }7141 }7142 7143 // Handle fshl/fshr special cases.7144 if (Opcode == ISD::FSHL || Opcode == ISD::FSHR) {7145 auto *C1 = dyn_cast<ConstantSDNode>(Ops[0]);7146 auto *C2 = dyn_cast<ConstantSDNode>(Ops[1]);7147 auto *C3 = dyn_cast<ConstantSDNode>(Ops[2]);7148 7149 if (C1 && C2 && C3) {7150 if (C1->isOpaque() || C2->isOpaque() || C3->isOpaque())7151 return SDValue();7152 const APInt &V1 = C1->getAPIntValue(), &V2 = C2->getAPIntValue(),7153 &V3 = C3->getAPIntValue();7154 7155 APInt FoldedVal = Opcode == ISD::FSHL ? APIntOps::fshl(V1, V2, V3)7156 : APIntOps::fshr(V1, V2, V3);7157 return getConstant(FoldedVal, DL, VT);7158 }7159 }7160 7161 // Handle fma/fmad special cases.7162 if (Opcode == ISD::FMA || Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {7163 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");7164 assert(Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&7165 Ops[2].getValueType() == VT && "FMA types must match!");7166 ConstantFPSDNode *C1 = dyn_cast<ConstantFPSDNode>(Ops[0]);7167 ConstantFPSDNode *C2 = dyn_cast<ConstantFPSDNode>(Ops[1]);7168 ConstantFPSDNode *C3 = dyn_cast<ConstantFPSDNode>(Ops[2]);7169 if (C1 && C2 && C3) {7170 APFloat V1 = C1->getValueAPF();7171 const APFloat &V2 = C2->getValueAPF();7172 const APFloat &V3 = C3->getValueAPF();7173 if (Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {7174 V1.multiply(V2, APFloat::rmNearestTiesToEven);7175 V1.add(V3, APFloat::rmNearestTiesToEven);7176 } else7177 V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);7178 return getConstantFP(V1, DL, VT);7179 }7180 }7181 7182 // This is for vector folding only from here on.7183 if (!VT.isVector())7184 return SDValue();7185 7186 ElementCount NumElts = VT.getVectorElementCount();7187 7188 // See if we can fold through any bitcasted integer ops.7189 if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() &&7190 Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&7191 (Ops[0].getOpcode() == ISD::BITCAST ||7192 Ops[1].getOpcode() == ISD::BITCAST)) {7193 SDValue N1 = peekThroughBitcasts(Ops[0]);7194 SDValue N2 = peekThroughBitcasts(Ops[1]);7195 auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);7196 auto *BV2 = dyn_cast<BuildVectorSDNode>(N2);7197 if (BV1 && BV2 && N1.getValueType().isInteger() &&7198 N2.getValueType().isInteger()) {7199 bool IsLE = getDataLayout().isLittleEndian();7200 unsigned EltBits = VT.getScalarSizeInBits();7201 SmallVector<APInt> RawBits1, RawBits2;7202 BitVector UndefElts1, UndefElts2;7203 if (BV1->getConstantRawBits(IsLE, EltBits, RawBits1, UndefElts1) &&7204 BV2->getConstantRawBits(IsLE, EltBits, RawBits2, UndefElts2)) {7205 SmallVector<APInt> RawBits;7206 for (unsigned I = 0, E = NumElts.getFixedValue(); I != E; ++I) {7207 std::optional<APInt> Fold = FoldValueWithUndef(7208 Opcode, RawBits1[I], UndefElts1[I], RawBits2[I], UndefElts2[I]);7209 if (!Fold)7210 break;7211 RawBits.push_back(*Fold);7212 }7213 if (RawBits.size() == NumElts.getFixedValue()) {7214 // We have constant folded, but we might need to cast this again back7215 // to the original (possibly legalized) type.7216 EVT BVVT, BVEltVT;7217 if (N1.getValueType() == VT) {7218 BVVT = N1.getValueType();7219 BVEltVT = BV1->getOperand(0).getValueType();7220 } else {7221 BVVT = N2.getValueType();7222 BVEltVT = BV2->getOperand(0).getValueType();7223 }7224 unsigned BVEltBits = BVEltVT.getSizeInBits();7225 SmallVector<APInt> DstBits;7226 BitVector DstUndefs;7227 BuildVectorSDNode::recastRawBits(IsLE, BVVT.getScalarSizeInBits(),7228 DstBits, RawBits, DstUndefs,7229 BitVector(RawBits.size(), false));7230 SmallVector<SDValue> Ops(DstBits.size(), getUNDEF(BVEltVT));7231 for (unsigned I = 0, E = DstBits.size(); I != E; ++I) {7232 if (DstUndefs[I])7233 continue;7234 Ops[I] = getConstant(DstBits[I].sext(BVEltBits), DL, BVEltVT);7235 }7236 return getBitcast(VT, getBuildVector(BVVT, DL, Ops));7237 }7238 }7239 }7240 }7241 7242 // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).7243 // (shl step_vector(C0), C1) -> (step_vector(C0 << C1))7244 if ((Opcode == ISD::MUL || Opcode == ISD::SHL) &&7245 Ops[0].getOpcode() == ISD::STEP_VECTOR) {7246 APInt RHSVal;7247 if (ISD::isConstantSplatVector(Ops[1].getNode(), RHSVal)) {7248 APInt NewStep = Opcode == ISD::MUL7249 ? Ops[0].getConstantOperandAPInt(0) * RHSVal7250 : Ops[0].getConstantOperandAPInt(0) << RHSVal;7251 return getStepVector(DL, VT, NewStep);7252 }7253 }7254 7255 auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {7256 return !Op.getValueType().isVector() ||7257 Op.getValueType().getVectorElementCount() == NumElts;7258 };7259 7260 auto IsBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {7261 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||7262 Op.getOpcode() == ISD::BUILD_VECTOR ||7263 Op.getOpcode() == ISD::SPLAT_VECTOR;7264 };7265 7266 // All operands must be vector types with the same number of elements as7267 // the result type and must be either UNDEF or a build/splat vector7268 // or UNDEF scalars.7269 if (!llvm::all_of(Ops, IsBuildVectorSplatVectorOrUndef) ||7270 !llvm::all_of(Ops, IsScalarOrSameVectorSize))7271 return SDValue();7272 7273 // If we are comparing vectors, then the result needs to be a i1 boolean that7274 // is then extended back to the legal result type depending on how booleans7275 // are represented.7276 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());7277 ISD::NodeType ExtendCode =7278 (Opcode == ISD::SETCC && SVT != VT.getScalarType())7279 ? TargetLowering::getExtendForContent(TLI->getBooleanContents(VT))7280 : ISD::SIGN_EXTEND;7281 7282 // Find legal integer scalar type for constant promotion and7283 // ensure that its scalar size is at least as large as source.7284 EVT LegalSVT = VT.getScalarType();7285 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {7286 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);7287 if (LegalSVT.bitsLT(VT.getScalarType()))7288 return SDValue();7289 }7290 7291 // For scalable vector types we know we're dealing with SPLAT_VECTORs. We7292 // only have one operand to check. For fixed-length vector types we may have7293 // a combination of BUILD_VECTOR and SPLAT_VECTOR.7294 unsigned NumVectorElts = NumElts.isScalable() ? 1 : NumElts.getFixedValue();7295 7296 // Constant fold each scalar lane separately.7297 SmallVector<SDValue, 4> ScalarResults;7298 for (unsigned I = 0; I != NumVectorElts; I++) {7299 SmallVector<SDValue, 4> ScalarOps;7300 for (SDValue Op : Ops) {7301 EVT InSVT = Op.getValueType().getScalarType();7302 if (Op.getOpcode() != ISD::BUILD_VECTOR &&7303 Op.getOpcode() != ISD::SPLAT_VECTOR) {7304 if (Op.isUndef())7305 ScalarOps.push_back(getUNDEF(InSVT));7306 else7307 ScalarOps.push_back(Op);7308 continue;7309 }7310 7311 SDValue ScalarOp =7312 Op.getOperand(Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);7313 EVT ScalarVT = ScalarOp.getValueType();7314 7315 // Build vector (integer) scalar operands may need implicit7316 // truncation - do this before constant folding.7317 if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT)) {7318 // Don't create illegally-typed nodes unless they're constants or undef7319 // - if we fail to constant fold we can't guarantee the (dead) nodes7320 // we're creating will be cleaned up before being visited for7321 // legalization.7322 if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() &&7323 !isa<ConstantSDNode>(ScalarOp) &&7324 TLI->getTypeAction(*getContext(), InSVT) !=7325 TargetLowering::TypeLegal)7326 return SDValue();7327 ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp);7328 }7329 7330 ScalarOps.push_back(ScalarOp);7331 }7332 7333 // Constant fold the scalar operands.7334 SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps, Flags);7335 7336 // Scalar folding only succeeded if the result is a constant or UNDEF.7337 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&7338 ScalarResult.getOpcode() != ISD::ConstantFP)7339 return SDValue();7340 7341 // Legalize the (integer) scalar constant if necessary. We only do7342 // this once we know the folding succeeded, since otherwise we would7343 // get a node with illegal type which has a user.7344 if (LegalSVT != SVT)7345 ScalarResult = getNode(ExtendCode, DL, LegalSVT, ScalarResult);7346 7347 ScalarResults.push_back(ScalarResult);7348 }7349 7350 SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, ScalarResults[0])7351 : getBuildVector(VT, DL, ScalarResults);7352 NewSDValueDbgMsg(V, "New node fold constant vector: ", this);7353 return V;7354}7355 7356SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,7357 EVT VT, ArrayRef<SDValue> Ops) {7358 // TODO: Add support for unary/ternary fp opcodes.7359 if (Ops.size() != 2)7360 return SDValue();7361 7362 // TODO: We don't do any constant folding for strict FP opcodes here, but we7363 // should. That will require dealing with a potentially non-default7364 // rounding mode, checking the "opStatus" return value from the APFloat7365 // math calculations, and possibly other variations.7366 SDValue N1 = Ops[0];7367 SDValue N2 = Ops[1];7368 ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1, /*AllowUndefs*/ false);7369 ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N2, /*AllowUndefs*/ false);7370 if (N1CFP && N2CFP) {7371 APFloat C1 = N1CFP->getValueAPF(); // make copy7372 const APFloat &C2 = N2CFP->getValueAPF();7373 switch (Opcode) {7374 case ISD::FADD:7375 C1.add(C2, APFloat::rmNearestTiesToEven);7376 return getConstantFP(C1, DL, VT);7377 case ISD::FSUB:7378 C1.subtract(C2, APFloat::rmNearestTiesToEven);7379 return getConstantFP(C1, DL, VT);7380 case ISD::FMUL:7381 C1.multiply(C2, APFloat::rmNearestTiesToEven);7382 return getConstantFP(C1, DL, VT);7383 case ISD::FDIV:7384 C1.divide(C2, APFloat::rmNearestTiesToEven);7385 return getConstantFP(C1, DL, VT);7386 case ISD::FREM:7387 C1.mod(C2);7388 return getConstantFP(C1, DL, VT);7389 case ISD::FCOPYSIGN:7390 C1.copySign(C2);7391 return getConstantFP(C1, DL, VT);7392 case ISD::FMINNUM:7393 return getConstantFP(minnum(C1, C2), DL, VT);7394 case ISD::FMAXNUM:7395 return getConstantFP(maxnum(C1, C2), DL, VT);7396 case ISD::FMINIMUM:7397 return getConstantFP(minimum(C1, C2), DL, VT);7398 case ISD::FMAXIMUM:7399 return getConstantFP(maximum(C1, C2), DL, VT);7400 case ISD::FMINIMUMNUM:7401 return getConstantFP(minimumnum(C1, C2), DL, VT);7402 case ISD::FMAXIMUMNUM:7403 return getConstantFP(maximumnum(C1, C2), DL, VT);7404 default: break;7405 }7406 }7407 if (N1CFP && Opcode == ISD::FP_ROUND) {7408 APFloat C1 = N1CFP->getValueAPF(); // make copy7409 bool Unused;7410 // This can return overflow, underflow, or inexact; we don't care.7411 // FIXME need to be more flexible about rounding mode.7412 (void)C1.convert(VT.getFltSemantics(), APFloat::rmNearestTiesToEven,7413 &Unused);7414 return getConstantFP(C1, DL, VT);7415 }7416 7417 switch (Opcode) {7418 case ISD::FSUB:7419 // -0.0 - undef --> undef (consistent with "fneg undef")7420 if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N1, /*AllowUndefs*/ true))7421 if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef())7422 return getUNDEF(VT);7423 [[fallthrough]];7424 7425 case ISD::FADD:7426 case ISD::FMUL:7427 case ISD::FDIV:7428 case ISD::FREM:7429 // If both operands are undef, the result is undef. If 1 operand is undef,7430 // the result is NaN. This should match the behavior of the IR optimizer.7431 if (N1.isUndef() && N2.isUndef())7432 return getUNDEF(VT);7433 if (N1.isUndef() || N2.isUndef())7434 return getConstantFP(APFloat::getNaN(VT.getFltSemantics()), DL, VT);7435 }7436 return SDValue();7437}7438 7439SDValue SelectionDAG::FoldConstantBuildVector(BuildVectorSDNode *BV,7440 const SDLoc &DL, EVT DstEltVT) {7441 EVT SrcEltVT = BV->getValueType(0).getVectorElementType();7442 7443 // If this is already the right type, we're done.7444 if (SrcEltVT == DstEltVT)7445 return SDValue(BV, 0);7446 7447 unsigned SrcBitSize = SrcEltVT.getSizeInBits();7448 unsigned DstBitSize = DstEltVT.getSizeInBits();7449 7450 // If this is a conversion of N elements of one type to N elements of another7451 // type, convert each element. This handles FP<->INT cases.7452 if (SrcBitSize == DstBitSize) {7453 SmallVector<SDValue, 8> Ops;7454 for (SDValue Op : BV->op_values()) {7455 // If the vector element type is not legal, the BUILD_VECTOR operands7456 // are promoted and implicitly truncated. Make that explicit here.7457 if (Op.getValueType() != SrcEltVT)7458 Op = getNode(ISD::TRUNCATE, DL, SrcEltVT, Op);7459 Ops.push_back(getBitcast(DstEltVT, Op));7460 }7461 EVT VT = EVT::getVectorVT(*getContext(), DstEltVT,7462 BV->getValueType(0).getVectorNumElements());7463 return getBuildVector(VT, DL, Ops);7464 }7465 7466 // Otherwise, we're growing or shrinking the elements. To avoid having to7467 // handle annoying details of growing/shrinking FP values, we convert them to7468 // int first.7469 if (SrcEltVT.isFloatingPoint()) {7470 // Convert the input float vector to a int vector where the elements are the7471 // same sizes.7472 EVT IntEltVT = EVT::getIntegerVT(*getContext(), SrcEltVT.getSizeInBits());7473 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, IntEltVT))7474 return FoldConstantBuildVector(cast<BuildVectorSDNode>(Tmp), DL,7475 DstEltVT);7476 return SDValue();7477 }7478 7479 // Now we know the input is an integer vector. If the output is a FP type,7480 // convert to integer first, then to FP of the right size.7481 if (DstEltVT.isFloatingPoint()) {7482 EVT IntEltVT = EVT::getIntegerVT(*getContext(), DstEltVT.getSizeInBits());7483 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, IntEltVT))7484 return FoldConstantBuildVector(cast<BuildVectorSDNode>(Tmp), DL,7485 DstEltVT);7486 return SDValue();7487 }7488 7489 // Okay, we know the src/dst types are both integers of differing types.7490 assert(SrcEltVT.isInteger() && DstEltVT.isInteger());7491 7492 // Extract the constant raw bit data.7493 BitVector UndefElements;7494 SmallVector<APInt> RawBits;7495 bool IsLE = getDataLayout().isLittleEndian();7496 if (!BV->getConstantRawBits(IsLE, DstBitSize, RawBits, UndefElements))7497 return SDValue();7498 7499 SmallVector<SDValue, 8> Ops;7500 for (unsigned I = 0, E = RawBits.size(); I != E; ++I) {7501 if (UndefElements[I])7502 Ops.push_back(getUNDEF(DstEltVT));7503 else7504 Ops.push_back(getConstant(RawBits[I], DL, DstEltVT));7505 }7506 7507 EVT VT = EVT::getVectorVT(*getContext(), DstEltVT, Ops.size());7508 return getBuildVector(VT, DL, Ops);7509}7510 7511SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) {7512 assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");7513 7514 // There's no need to assert on a byte-aligned pointer. All pointers are at7515 // least byte aligned.7516 if (A == Align(1))7517 return Val;7518 7519 SDVTList VTs = getVTList(Val.getValueType());7520 FoldingSetNodeID ID;7521 AddNodeIDNode(ID, ISD::AssertAlign, VTs, {Val});7522 ID.AddInteger(A.value());7523 7524 void *IP = nullptr;7525 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))7526 return SDValue(E, 0);7527 7528 auto *N =7529 newSDNode<AssertAlignSDNode>(DL.getIROrder(), DL.getDebugLoc(), VTs, A);7530 createOperands(N, {Val});7531 7532 CSEMap.InsertNode(N, IP);7533 InsertNode(N);7534 7535 SDValue V(N, 0);7536 NewSDValueDbgMsg(V, "Creating new node: ", this);7537 return V;7538}7539 7540SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,7541 SDValue N1, SDValue N2) {7542 SDNodeFlags Flags;7543 if (Inserter)7544 Flags = Inserter->getFlags();7545 return getNode(Opcode, DL, VT, N1, N2, Flags);7546}7547 7548void SelectionDAG::canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1,7549 SDValue &N2) const {7550 if (!TLI->isCommutativeBinOp(Opcode))7551 return;7552 7553 // Canonicalize:7554 // binop(const, nonconst) -> binop(nonconst, const)7555 bool N1C = isConstantIntBuildVectorOrConstantInt(N1);7556 bool N2C = isConstantIntBuildVectorOrConstantInt(N2);7557 bool N1CFP = isConstantFPBuildVectorOrConstantFP(N1);7558 bool N2CFP = isConstantFPBuildVectorOrConstantFP(N2);7559 if ((N1C && !N2C) || (N1CFP && !N2CFP))7560 std::swap(N1, N2);7561 7562 // Canonicalize:7563 // binop(splat(x), step_vector) -> binop(step_vector, splat(x))7564 else if (N1.getOpcode() == ISD::SPLAT_VECTOR &&7565 N2.getOpcode() == ISD::STEP_VECTOR)7566 std::swap(N1, N2);7567}7568 7569SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,7570 SDValue N1, SDValue N2, const SDNodeFlags Flags) {7571 assert(N1.getOpcode() != ISD::DELETED_NODE &&7572 N2.getOpcode() != ISD::DELETED_NODE &&7573 "Operand is DELETED_NODE!");7574 7575 canonicalizeCommutativeBinop(Opcode, N1, N2);7576 7577 auto *N1C = dyn_cast<ConstantSDNode>(N1);7578 auto *N2C = dyn_cast<ConstantSDNode>(N2);7579 7580 // Don't allow undefs in vector splats - we might be returning N2 when folding7581 // to zero etc.7582 ConstantSDNode *N2CV =7583 isConstOrConstSplat(N2, /*AllowUndefs*/ false, /*AllowTruncation*/ true);7584 7585 switch (Opcode) {7586 default: break;7587 case ISD::TokenFactor:7588 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&7589 N2.getValueType() == MVT::Other && "Invalid token factor!");7590 // Fold trivial token factors.7591 if (N1.getOpcode() == ISD::EntryToken) return N2;7592 if (N2.getOpcode() == ISD::EntryToken) return N1;7593 if (N1 == N2) return N1;7594 break;7595 case ISD::BUILD_VECTOR: {7596 // Attempt to simplify BUILD_VECTOR.7597 SDValue Ops[] = {N1, N2};7598 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))7599 return V;7600 break;7601 }7602 case ISD::CONCAT_VECTORS: {7603 SDValue Ops[] = {N1, N2};7604 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))7605 return V;7606 break;7607 }7608 case ISD::AND:7609 assert(VT.isInteger() && "This operator does not apply to FP types!");7610 assert(N1.getValueType() == N2.getValueType() &&7611 N1.getValueType() == VT && "Binary operator types must match!");7612 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's7613 // worth handling here.7614 if (N2CV && N2CV->isZero())7615 return N2;7616 if (N2CV && N2CV->isAllOnes()) // X & -1 -> X7617 return N1;7618 break;7619 case ISD::OR:7620 case ISD::XOR:7621 case ISD::ADD:7622 case ISD::PTRADD:7623 case ISD::SUB:7624 assert(VT.isInteger() && "This operator does not apply to FP types!");7625 assert(N1.getValueType() == N2.getValueType() &&7626 N1.getValueType() == VT && "Binary operator types must match!");7627 // The equal operand types requirement is unnecessarily strong for PTRADD.7628 // However, the SelectionDAGBuilder does not generate PTRADDs with different7629 // operand types, and we'd need to re-implement GEP's non-standard wrapping7630 // logic everywhere where PTRADDs may be folded or combined to properly7631 // support them. If/when we introduce pointer types to the SDAG, we will7632 // need to relax this constraint.7633 7634 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so7635 // it's worth handling here.7636 if (N2CV && N2CV->isZero())7637 return N1;7638 if ((Opcode == ISD::ADD || Opcode == ISD::SUB) &&7639 VT.getScalarType() == MVT::i1)7640 return getNode(ISD::XOR, DL, VT, N1, N2);7641 // Fold (add (vscale * C0), (vscale * C1)) to (vscale * (C0 + C1)).7642 if (Opcode == ISD::ADD && N1.getOpcode() == ISD::VSCALE &&7643 N2.getOpcode() == ISD::VSCALE) {7644 const APInt &C1 = N1->getConstantOperandAPInt(0);7645 const APInt &C2 = N2->getConstantOperandAPInt(0);7646 return getVScale(DL, VT, C1 + C2);7647 }7648 break;7649 case ISD::MUL:7650 assert(VT.isInteger() && "This operator does not apply to FP types!");7651 assert(N1.getValueType() == N2.getValueType() &&7652 N1.getValueType() == VT && "Binary operator types must match!");7653 if (VT.getScalarType() == MVT::i1)7654 return getNode(ISD::AND, DL, VT, N1, N2);7655 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {7656 const APInt &MulImm = N1->getConstantOperandAPInt(0);7657 const APInt &N2CImm = N2C->getAPIntValue();7658 return getVScale(DL, VT, MulImm * N2CImm);7659 }7660 break;7661 case ISD::UDIV:7662 case ISD::UREM:7663 case ISD::MULHU:7664 case ISD::MULHS:7665 case ISD::SDIV:7666 case ISD::SREM:7667 case ISD::SADDSAT:7668 case ISD::SSUBSAT:7669 case ISD::UADDSAT:7670 case ISD::USUBSAT:7671 assert(VT.isInteger() && "This operator does not apply to FP types!");7672 assert(N1.getValueType() == N2.getValueType() &&7673 N1.getValueType() == VT && "Binary operator types must match!");7674 if (VT.getScalarType() == MVT::i1) {7675 // fold (add_sat x, y) -> (or x, y) for bool types.7676 if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)7677 return getNode(ISD::OR, DL, VT, N1, N2);7678 // fold (sub_sat x, y) -> (and x, ~y) for bool types.7679 if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT)7680 return getNode(ISD::AND, DL, VT, N1, getNOT(DL, N2, VT));7681 }7682 break;7683 case ISD::SCMP:7684 case ISD::UCMP:7685 assert(N1.getValueType() == N2.getValueType() &&7686 "Types of operands of UCMP/SCMP must match");7687 assert(N1.getValueType().isVector() == VT.isVector() &&7688 "Operands and return type of must both be scalars or vectors");7689 if (VT.isVector())7690 assert(VT.getVectorElementCount() ==7691 N1.getValueType().getVectorElementCount() &&7692 "Result and operands must have the same number of elements");7693 break;7694 case ISD::AVGFLOORS:7695 case ISD::AVGFLOORU:7696 case ISD::AVGCEILS:7697 case ISD::AVGCEILU:7698 assert(VT.isInteger() && "This operator does not apply to FP types!");7699 assert(N1.getValueType() == N2.getValueType() &&7700 N1.getValueType() == VT && "Binary operator types must match!");7701 break;7702 case ISD::ABDS:7703 case ISD::ABDU:7704 assert(VT.isInteger() && "This operator does not apply to FP types!");7705 assert(N1.getValueType() == N2.getValueType() &&7706 N1.getValueType() == VT && "Binary operator types must match!");7707 if (VT.getScalarType() == MVT::i1)7708 return getNode(ISD::XOR, DL, VT, N1, N2);7709 break;7710 case ISD::SMIN:7711 case ISD::UMAX:7712 assert(VT.isInteger() && "This operator does not apply to FP types!");7713 assert(N1.getValueType() == N2.getValueType() &&7714 N1.getValueType() == VT && "Binary operator types must match!");7715 if (VT.getScalarType() == MVT::i1)7716 return getNode(ISD::OR, DL, VT, N1, N2);7717 break;7718 case ISD::SMAX:7719 case ISD::UMIN:7720 assert(VT.isInteger() && "This operator does not apply to FP types!");7721 assert(N1.getValueType() == N2.getValueType() &&7722 N1.getValueType() == VT && "Binary operator types must match!");7723 if (VT.getScalarType() == MVT::i1)7724 return getNode(ISD::AND, DL, VT, N1, N2);7725 break;7726 case ISD::FADD:7727 case ISD::FSUB:7728 case ISD::FMUL:7729 case ISD::FDIV:7730 case ISD::FREM:7731 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");7732 assert(N1.getValueType() == N2.getValueType() &&7733 N1.getValueType() == VT && "Binary operator types must match!");7734 if (SDValue V = simplifyFPBinop(Opcode, N1, N2, Flags))7735 return V;7736 break;7737 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.7738 assert(N1.getValueType() == VT &&7739 N1.getValueType().isFloatingPoint() &&7740 N2.getValueType().isFloatingPoint() &&7741 "Invalid FCOPYSIGN!");7742 break;7743 case ISD::SHL:7744 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {7745 const APInt &MulImm = N1->getConstantOperandAPInt(0);7746 const APInt &ShiftImm = N2C->getAPIntValue();7747 return getVScale(DL, VT, MulImm << ShiftImm);7748 }7749 [[fallthrough]];7750 case ISD::SRA:7751 case ISD::SRL:7752 if (SDValue V = simplifyShift(N1, N2))7753 return V;7754 [[fallthrough]];7755 case ISD::ROTL:7756 case ISD::ROTR:7757 assert(VT == N1.getValueType() &&7758 "Shift operators return type must be the same as their first arg");7759 assert(VT.isInteger() && N2.getValueType().isInteger() &&7760 "Shifts only work on integers");7761 assert((!VT.isVector() || VT == N2.getValueType()) &&7762 "Vector shift amounts must be in the same as their first arg");7763 // Verify that the shift amount VT is big enough to hold valid shift7764 // amounts. This catches things like trying to shift an i1024 value by an7765 // i8, which is easy to fall into in generic code that uses7766 // TLI.getShiftAmount().7767 assert(N2.getValueType().getScalarSizeInBits() >=7768 Log2_32_Ceil(VT.getScalarSizeInBits()) &&7769 "Invalid use of small shift amount with oversized value!");7770 7771 // Always fold shifts of i1 values so the code generator doesn't need to7772 // handle them. Since we know the size of the shift has to be less than the7773 // size of the value, the shift/rotate count is guaranteed to be zero.7774 if (VT == MVT::i1)7775 return N1;7776 if (N2CV && N2CV->isZero())7777 return N1;7778 break;7779 case ISD::FP_ROUND:7780 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&7781 VT.bitsLE(N1.getValueType()) && N2C &&7782 (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&7783 N2.getOpcode() == ISD::TargetConstant && "Invalid FP_ROUND!");7784 if (N1.getValueType() == VT) return N1; // noop conversion.7785 break;7786 case ISD::AssertNoFPClass: {7787 assert(N1.getValueType().isFloatingPoint() &&7788 "AssertNoFPClass is used for a non-floating type");7789 assert(isa<ConstantSDNode>(N2) && "NoFPClass is not Constant");7790 FPClassTest NoFPClass = static_cast<FPClassTest>(N2->getAsZExtVal());7791 assert(llvm::to_underlying(NoFPClass) <=7792 BitmaskEnumDetail::Mask<FPClassTest>() &&7793 "FPClassTest value too large");7794 (void)NoFPClass;7795 break;7796 }7797 case ISD::AssertSext:7798 case ISD::AssertZext: {7799 EVT EVT = cast<VTSDNode>(N2)->getVT();7800 assert(VT == N1.getValueType() && "Not an inreg extend!");7801 assert(VT.isInteger() && EVT.isInteger() &&7802 "Cannot *_EXTEND_INREG FP types");7803 assert(!EVT.isVector() &&7804 "AssertSExt/AssertZExt type should be the vector element type "7805 "rather than the vector type!");7806 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!");7807 if (VT.getScalarType() == EVT) return N1; // noop assertion.7808 break;7809 }7810 case ISD::SIGN_EXTEND_INREG: {7811 EVT EVT = cast<VTSDNode>(N2)->getVT();7812 assert(VT == N1.getValueType() && "Not an inreg extend!");7813 assert(VT.isInteger() && EVT.isInteger() &&7814 "Cannot *_EXTEND_INREG FP types");7815 assert(EVT.isVector() == VT.isVector() &&7816 "SIGN_EXTEND_INREG type should be vector iff the operand "7817 "type is vector!");7818 assert((!EVT.isVector() ||7819 EVT.getVectorElementCount() == VT.getVectorElementCount()) &&7820 "Vector element counts must match in SIGN_EXTEND_INREG");7821 assert(EVT.bitsLE(VT) && "Not extending!");7822 if (EVT == VT) return N1; // Not actually extending7823 break;7824 }7825 case ISD::FP_TO_SINT_SAT:7826 case ISD::FP_TO_UINT_SAT: {7827 assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&7828 N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT");7829 assert(N1.getValueType().isVector() == VT.isVector() &&7830 "FP_TO_*INT_SAT type should be vector iff the operand type is "7831 "vector!");7832 assert((!VT.isVector() || VT.getVectorElementCount() ==7833 N1.getValueType().getVectorElementCount()) &&7834 "Vector element counts must match in FP_TO_*INT_SAT");7835 assert(!cast<VTSDNode>(N2)->getVT().isVector() &&7836 "Type to saturate to must be a scalar.");7837 assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&7838 "Not extending!");7839 break;7840 }7841 case ISD::EXTRACT_VECTOR_ELT:7842 assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&7843 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \7844 element type of the vector.");7845 7846 // Extract from an undefined value or using an undefined index is undefined.7847 if (N1.isUndef() || N2.isUndef())7848 return getUNDEF(VT);7849 7850 // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF for fixed length7851 // vectors. For scalable vectors we will provide appropriate support for7852 // dealing with arbitrary indices.7853 if (N2C && N1.getValueType().isFixedLengthVector() &&7854 N2C->getAPIntValue().uge(N1.getValueType().getVectorNumElements()))7855 return getUNDEF(VT);7856 7857 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is7858 // expanding copies of large vectors from registers. This only works for7859 // fixed length vectors, since we need to know the exact number of7860 // elements.7861 if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&7862 N1.getOperand(0).getValueType().isFixedLengthVector()) {7863 unsigned Factor = N1.getOperand(0).getValueType().getVectorNumElements();7864 return getExtractVectorElt(DL, VT,7865 N1.getOperand(N2C->getZExtValue() / Factor),7866 N2C->getZExtValue() % Factor);7867 }7868 7869 // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while7870 // lowering is expanding large vector constants.7871 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR ||7872 N1.getOpcode() == ISD::SPLAT_VECTOR)) {7873 assert((N1.getOpcode() != ISD::BUILD_VECTOR ||7874 N1.getValueType().isFixedLengthVector()) &&7875 "BUILD_VECTOR used for scalable vectors");7876 unsigned Index =7877 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0;7878 SDValue Elt = N1.getOperand(Index);7879 7880 if (VT != Elt.getValueType())7881 // If the vector element type is not legal, the BUILD_VECTOR operands7882 // are promoted and implicitly truncated, and the result implicitly7883 // extended. Make that explicit here.7884 Elt = getAnyExtOrTrunc(Elt, DL, VT);7885 7886 return Elt;7887 }7888 7889 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector7890 // operations are lowered to scalars.7891 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {7892 // If the indices are the same, return the inserted element else7893 // if the indices are known different, extract the element from7894 // the original vector.7895 SDValue N1Op2 = N1.getOperand(2);7896 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2);7897 7898 if (N1Op2C && N2C) {7899 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {7900 if (VT == N1.getOperand(1).getValueType())7901 return N1.getOperand(1);7902 if (VT.isFloatingPoint()) {7903 assert(VT.getSizeInBits() > N1.getOperand(1).getValueType().getSizeInBits());7904 return getFPExtendOrRound(N1.getOperand(1), DL, VT);7905 }7906 return getSExtOrTrunc(N1.getOperand(1), DL, VT);7907 }7908 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);7909 }7910 }7911 7912 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed7913 // when vector types are scalarized and v1iX is legal.7914 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx).7915 // Here we are completely ignoring the extract element index (N2),7916 // which is fine for fixed width vectors, since any index other than 07917 // is undefined anyway. However, this cannot be ignored for scalable7918 // vectors - in theory we could support this, but we don't want to do this7919 // without a profitability check.7920 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&7921 N1.getValueType().isFixedLengthVector() &&7922 N1.getValueType().getVectorNumElements() == 1) {7923 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0),7924 N1.getOperand(1));7925 }7926 break;7927 case ISD::EXTRACT_ELEMENT:7928 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");7929 assert(!N1.getValueType().isVector() && !VT.isVector() &&7930 (N1.getValueType().isInteger() == VT.isInteger()) &&7931 N1.getValueType() != VT &&7932 "Wrong types for EXTRACT_ELEMENT!");7933 7934 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding7935 // 64-bit integers into 32-bit parts. Instead of building the extract of7936 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.7937 if (N1.getOpcode() == ISD::BUILD_PAIR)7938 return N1.getOperand(N2C->getZExtValue());7939 7940 // EXTRACT_ELEMENT of a constant int is also very common.7941 if (N1C) {7942 unsigned ElementSize = VT.getSizeInBits();7943 unsigned Shift = ElementSize * N2C->getZExtValue();7944 const APInt &Val = N1C->getAPIntValue();7945 return getConstant(Val.extractBits(ElementSize, Shift), DL, VT);7946 }7947 break;7948 case ISD::EXTRACT_SUBVECTOR: {7949 EVT N1VT = N1.getValueType();7950 assert(VT.isVector() && N1VT.isVector() &&7951 "Extract subvector VTs must be vectors!");7952 assert(VT.getVectorElementType() == N1VT.getVectorElementType() &&7953 "Extract subvector VTs must have the same element type!");7954 assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) &&7955 "Cannot extract a scalable vector from a fixed length vector!");7956 assert((VT.isScalableVector() != N1VT.isScalableVector() ||7957 VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) &&7958 "Extract subvector must be from larger vector to smaller vector!");7959 assert(N2C && "Extract subvector index must be a constant");7960 assert((VT.isScalableVector() != N1VT.isScalableVector() ||7961 (VT.getVectorMinNumElements() + N2C->getZExtValue()) <=7962 N1VT.getVectorMinNumElements()) &&7963 "Extract subvector overflow!");7964 assert(N2C->getAPIntValue().getBitWidth() ==7965 TLI->getVectorIdxWidth(getDataLayout()) &&7966 "Constant index for EXTRACT_SUBVECTOR has an invalid size");7967 assert(N2C->getZExtValue() % VT.getVectorMinNumElements() == 0 &&7968 "Extract index is not a multiple of the output vector length");7969 7970 // Trivial extraction.7971 if (VT == N1VT)7972 return N1;7973 7974 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.7975 if (N1.isUndef())7976 return getUNDEF(VT);7977 7978 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of7979 // the concat have the same type as the extract.7980 if (N1.getOpcode() == ISD::CONCAT_VECTORS &&7981 VT == N1.getOperand(0).getValueType()) {7982 unsigned Factor = VT.getVectorMinNumElements();7983 return N1.getOperand(N2C->getZExtValue() / Factor);7984 }7985 7986 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created7987 // during shuffle legalization.7988 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(2) &&7989 VT == N1.getOperand(1).getValueType())7990 return N1.getOperand(1);7991 break;7992 }7993 }7994 7995 if (N1.getOpcode() == ISD::POISON || N2.getOpcode() == ISD::POISON) {7996 switch (Opcode) {7997 case ISD::XOR:7998 case ISD::ADD:7999 case ISD::PTRADD:8000 case ISD::SUB:8001 case ISD::SIGN_EXTEND_INREG:8002 case ISD::UDIV:8003 case ISD::SDIV:8004 case ISD::UREM:8005 case ISD::SREM:8006 case ISD::MUL:8007 case ISD::AND:8008 case ISD::SSUBSAT:8009 case ISD::USUBSAT:8010 case ISD::UMIN:8011 case ISD::OR:8012 case ISD::SADDSAT:8013 case ISD::UADDSAT:8014 case ISD::UMAX:8015 case ISD::SMAX:8016 case ISD::SMIN:8017 // fold op(arg1, poison) -> poison, fold op(poison, arg2) -> poison.8018 return N2.getOpcode() == ISD::POISON ? N2 : N1;8019 }8020 }8021 8022 // Canonicalize an UNDEF to the RHS, even over a constant.8023 if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() != ISD::UNDEF) {8024 if (TLI->isCommutativeBinOp(Opcode)) {8025 std::swap(N1, N2);8026 } else {8027 switch (Opcode) {8028 case ISD::PTRADD:8029 case ISD::SUB:8030 // fold op(undef, non_undef_arg2) -> undef.8031 return N1;8032 case ISD::SIGN_EXTEND_INREG:8033 case ISD::UDIV:8034 case ISD::SDIV:8035 case ISD::UREM:8036 case ISD::SREM:8037 case ISD::SSUBSAT:8038 case ISD::USUBSAT:8039 // fold op(undef, non_undef_arg2) -> 0.8040 return getConstant(0, DL, VT);8041 }8042 }8043 }8044 8045 // Fold a bunch of operators when the RHS is undef.8046 if (N2.getOpcode() == ISD::UNDEF) {8047 switch (Opcode) {8048 case ISD::XOR:8049 if (N1.getOpcode() == ISD::UNDEF)8050 // Handle undef ^ undef -> 0 special case. This is a common8051 // idiom (misuse).8052 return getConstant(0, DL, VT);8053 [[fallthrough]];8054 case ISD::ADD:8055 case ISD::PTRADD:8056 case ISD::SUB:8057 // fold op(arg1, undef) -> undef.8058 return N2;8059 case ISD::UDIV:8060 case ISD::SDIV:8061 case ISD::UREM:8062 case ISD::SREM:8063 // fold op(arg1, undef) -> poison.8064 return getPOISON(VT);8065 case ISD::MUL:8066 case ISD::AND:8067 case ISD::SSUBSAT:8068 case ISD::USUBSAT:8069 case ISD::UMIN:8070 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> 0.8071 return N1.getOpcode() == ISD::UNDEF ? N2 : getConstant(0, DL, VT);8072 case ISD::OR:8073 case ISD::SADDSAT:8074 case ISD::UADDSAT:8075 case ISD::UMAX:8076 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> -1.8077 return N1.getOpcode() == ISD::UNDEF ? N2 : getAllOnesConstant(DL, VT);8078 case ISD::SMAX:8079 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MAX_INT.8080 return N1.getOpcode() == ISD::UNDEF8081 ? N28082 : getConstant(8083 APInt::getSignedMaxValue(VT.getScalarSizeInBits()), DL,8084 VT);8085 case ISD::SMIN:8086 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MIN_INT.8087 return N1.getOpcode() == ISD::UNDEF8088 ? N28089 : getConstant(8090 APInt::getSignedMinValue(VT.getScalarSizeInBits()), DL,8091 VT);8092 }8093 }8094 8095 // Perform trivial constant folding.8096 if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, {N1, N2}, Flags))8097 return SV;8098 8099 // Memoize this node if possible.8100 SDNode *N;8101 SDVTList VTs = getVTList(VT);8102 SDValue Ops[] = {N1, N2};8103 if (VT != MVT::Glue) {8104 FoldingSetNodeID ID;8105 AddNodeIDNode(ID, Opcode, VTs, Ops);8106 void *IP = nullptr;8107 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {8108 E->intersectFlagsWith(Flags);8109 return SDValue(E, 0);8110 }8111 8112 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);8113 N->setFlags(Flags);8114 createOperands(N, Ops);8115 CSEMap.InsertNode(N, IP);8116 } else {8117 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);8118 createOperands(N, Ops);8119 }8120 8121 InsertNode(N);8122 SDValue V = SDValue(N, 0);8123 NewSDValueDbgMsg(V, "Creating new node: ", this);8124 return V;8125}8126 8127SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,8128 SDValue N1, SDValue N2, SDValue N3) {8129 SDNodeFlags Flags;8130 if (Inserter)8131 Flags = Inserter->getFlags();8132 return getNode(Opcode, DL, VT, N1, N2, N3, Flags);8133}8134 8135SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,8136 SDValue N1, SDValue N2, SDValue N3,8137 const SDNodeFlags Flags) {8138 assert(N1.getOpcode() != ISD::DELETED_NODE &&8139 N2.getOpcode() != ISD::DELETED_NODE &&8140 N3.getOpcode() != ISD::DELETED_NODE &&8141 "Operand is DELETED_NODE!");8142 // Perform various simplifications.8143 switch (Opcode) {8144 case ISD::BUILD_VECTOR: {8145 // Attempt to simplify BUILD_VECTOR.8146 SDValue Ops[] = {N1, N2, N3};8147 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))8148 return V;8149 break;8150 }8151 case ISD::CONCAT_VECTORS: {8152 SDValue Ops[] = {N1, N2, N3};8153 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))8154 return V;8155 break;8156 }8157 case ISD::SETCC: {8158 assert(VT.isInteger() && "SETCC result type must be an integer!");8159 assert(N1.getValueType() == N2.getValueType() &&8160 "SETCC operands must have the same type!");8161 assert(VT.isVector() == N1.getValueType().isVector() &&8162 "SETCC type should be vector iff the operand type is vector!");8163 assert((!VT.isVector() || VT.getVectorElementCount() ==8164 N1.getValueType().getVectorElementCount()) &&8165 "SETCC vector element counts must match!");8166 // Use FoldSetCC to simplify SETCC's.8167 if (SDValue V = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL))8168 return V;8169 break;8170 }8171 case ISD::SELECT:8172 case ISD::VSELECT:8173 if (SDValue V = simplifySelect(N1, N2, N3))8174 return V;8175 break;8176 case ISD::VECTOR_SHUFFLE:8177 llvm_unreachable("should use getVectorShuffle constructor!");8178 case ISD::VECTOR_SPLICE: {8179 if (cast<ConstantSDNode>(N3)->isZero())8180 return N1;8181 break;8182 }8183 case ISD::INSERT_VECTOR_ELT: {8184 assert(VT.isVector() && VT == N1.getValueType() &&8185 "INSERT_VECTOR_ELT vector type mismatch");8186 assert(VT.isFloatingPoint() == N2.getValueType().isFloatingPoint() &&8187 "INSERT_VECTOR_ELT scalar fp/int mismatch");8188 assert((!VT.isFloatingPoint() ||8189 VT.getVectorElementType() == N2.getValueType()) &&8190 "INSERT_VECTOR_ELT fp scalar type mismatch");8191 assert((!VT.isInteger() ||8192 VT.getScalarSizeInBits() <= N2.getScalarValueSizeInBits()) &&8193 "INSERT_VECTOR_ELT int scalar size mismatch");8194 8195 auto *N3C = dyn_cast<ConstantSDNode>(N3);8196 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except8197 // for scalable vectors where we will generate appropriate code to8198 // deal with out-of-bounds cases correctly.8199 if (N3C && VT.isFixedLengthVector() &&8200 N3C->getZExtValue() >= VT.getVectorNumElements())8201 return getUNDEF(VT);8202 8203 // Undefined index can be assumed out-of-bounds, so that's UNDEF too.8204 if (N3.isUndef())8205 return getUNDEF(VT);8206 8207 // If inserting poison, just use the input vector.8208 if (N2.getOpcode() == ISD::POISON)8209 return N1;8210 8211 // Inserting undef into undef/poison is still undef.8212 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())8213 return getUNDEF(VT);8214 8215 // If the inserted element is an UNDEF, just use the input vector.8216 // But not if skipping the insert could make the result more poisonous.8217 if (N2.isUndef()) {8218 if (N3C && VT.isFixedLengthVector()) {8219 APInt EltMask =8220 APInt::getOneBitSet(VT.getVectorNumElements(), N3C->getZExtValue());8221 if (isGuaranteedNotToBePoison(N1, EltMask))8222 return N1;8223 } else if (isGuaranteedNotToBePoison(N1))8224 return N1;8225 }8226 break;8227 }8228 case ISD::INSERT_SUBVECTOR: {8229 // If inserting poison, just use the input vector,8230 if (N2.getOpcode() == ISD::POISON)8231 return N1;8232 8233 // Inserting undef into undef/poison is still undef.8234 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())8235 return getUNDEF(VT);8236 8237 EVT N2VT = N2.getValueType();8238 assert(VT == N1.getValueType() &&8239 "Dest and insert subvector source types must match!");8240 assert(VT.isVector() && N2VT.isVector() &&8241 "Insert subvector VTs must be vectors!");8242 assert(VT.getVectorElementType() == N2VT.getVectorElementType() &&8243 "Insert subvector VTs must have the same element type!");8244 assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) &&8245 "Cannot insert a scalable vector into a fixed length vector!");8246 assert((VT.isScalableVector() != N2VT.isScalableVector() ||8247 VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) &&8248 "Insert subvector must be from smaller vector to larger vector!");8249 assert(isa<ConstantSDNode>(N3) &&8250 "Insert subvector index must be constant");8251 assert((VT.isScalableVector() != N2VT.isScalableVector() ||8252 (N2VT.getVectorMinNumElements() + N3->getAsZExtVal()) <=8253 VT.getVectorMinNumElements()) &&8254 "Insert subvector overflow!");8255 assert(N3->getAsAPIntVal().getBitWidth() ==8256 TLI->getVectorIdxWidth(getDataLayout()) &&8257 "Constant index for INSERT_SUBVECTOR has an invalid size");8258 8259 // Trivial insertion.8260 if (VT == N2VT)8261 return N2;8262 8263 // If this is an insert of an extracted vector into an undef/poison vector,8264 // we can just use the input to the extract. But not if skipping the8265 // extract+insert could make the result more poisonous.8266 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&8267 N2.getOperand(1) == N3 && N2.getOperand(0).getValueType() == VT) {8268 if (N1.getOpcode() == ISD::POISON)8269 return N2.getOperand(0);8270 if (VT.isFixedLengthVector() && N2VT.isFixedLengthVector()) {8271 unsigned LoBit = N3->getAsZExtVal();8272 unsigned HiBit = LoBit + N2VT.getVectorNumElements();8273 APInt EltMask =8274 APInt::getBitsSet(VT.getVectorNumElements(), LoBit, HiBit);8275 if (isGuaranteedNotToBePoison(N2.getOperand(0), ~EltMask))8276 return N2.getOperand(0);8277 } else if (isGuaranteedNotToBePoison(N2.getOperand(0)))8278 return N2.getOperand(0);8279 }8280 8281 // If the inserted subvector is UNDEF, just use the input vector.8282 // But not if skipping the insert could make the result more poisonous.8283 if (N2.isUndef()) {8284 if (VT.isFixedLengthVector()) {8285 unsigned LoBit = N3->getAsZExtVal();8286 unsigned HiBit = LoBit + N2VT.getVectorNumElements();8287 APInt EltMask =8288 APInt::getBitsSet(VT.getVectorNumElements(), LoBit, HiBit);8289 if (isGuaranteedNotToBePoison(N1, EltMask))8290 return N1;8291 } else if (isGuaranteedNotToBePoison(N1))8292 return N1;8293 }8294 break;8295 }8296 case ISD::BITCAST:8297 // Fold bit_convert nodes from a type to themselves.8298 if (N1.getValueType() == VT)8299 return N1;8300 break;8301 case ISD::VP_TRUNCATE:8302 case ISD::VP_SIGN_EXTEND:8303 case ISD::VP_ZERO_EXTEND:8304 // Don't create noop casts.8305 if (N1.getValueType() == VT)8306 return N1;8307 break;8308 case ISD::VECTOR_COMPRESS: {8309 [[maybe_unused]] EVT VecVT = N1.getValueType();8310 [[maybe_unused]] EVT MaskVT = N2.getValueType();8311 [[maybe_unused]] EVT PassthruVT = N3.getValueType();8312 assert(VT == VecVT && "Vector and result type don't match.");8313 assert(VecVT.isVector() && MaskVT.isVector() && PassthruVT.isVector() &&8314 "All inputs must be vectors.");8315 assert(VecVT == PassthruVT && "Vector and passthru types don't match.");8316 assert(VecVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&8317 "Vector and mask must have same number of elements.");8318 8319 if (N1.isUndef() || N2.isUndef())8320 return N3;8321 8322 break;8323 }8324 case ISD::PARTIAL_REDUCE_UMLA:8325 case ISD::PARTIAL_REDUCE_SMLA:8326 case ISD::PARTIAL_REDUCE_SUMLA:8327 case ISD::PARTIAL_REDUCE_FMLA: {8328 [[maybe_unused]] EVT AccVT = N1.getValueType();8329 [[maybe_unused]] EVT Input1VT = N2.getValueType();8330 [[maybe_unused]] EVT Input2VT = N3.getValueType();8331 assert(Input1VT.isVector() && Input1VT == Input2VT &&8332 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "8333 "node to have the same type!");8334 assert(VT.isVector() && VT == AccVT &&8335 "Expected the first operand of the PARTIAL_REDUCE_MLA node to have "8336 "the same type as its result!");8337 assert(Input1VT.getVectorElementCount().hasKnownScalarFactor(8338 AccVT.getVectorElementCount()) &&8339 "Expected the element count of the second and third operands of the "8340 "PARTIAL_REDUCE_MLA node to be a positive integer multiple of the "8341 "element count of the first operand and the result!");8342 assert(N2.getScalarValueSizeInBits() <= N1.getScalarValueSizeInBits() &&8343 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "8344 "node to have an element type which is the same as or smaller than "8345 "the element type of the first operand and result!");8346 break;8347 }8348 }8349 8350 // Perform trivial constant folding for arithmetic operators.8351 switch (Opcode) {8352 case ISD::FMA:8353 case ISD::FMAD:8354 case ISD::SETCC:8355 case ISD::FSHL:8356 case ISD::FSHR:8357 if (SDValue SV =8358 FoldConstantArithmetic(Opcode, DL, VT, {N1, N2, N3}, Flags))8359 return SV;8360 break;8361 }8362 8363 // Memoize node if it doesn't produce a glue result.8364 SDNode *N;8365 SDVTList VTs = getVTList(VT);8366 SDValue Ops[] = {N1, N2, N3};8367 if (VT != MVT::Glue) {8368 FoldingSetNodeID ID;8369 AddNodeIDNode(ID, Opcode, VTs, Ops);8370 void *IP = nullptr;8371 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {8372 E->intersectFlagsWith(Flags);8373 return SDValue(E, 0);8374 }8375 8376 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);8377 N->setFlags(Flags);8378 createOperands(N, Ops);8379 CSEMap.InsertNode(N, IP);8380 } else {8381 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);8382 createOperands(N, Ops);8383 }8384 8385 InsertNode(N);8386 SDValue V = SDValue(N, 0);8387 NewSDValueDbgMsg(V, "Creating new node: ", this);8388 return V;8389}8390 8391SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,8392 SDValue N1, SDValue N2, SDValue N3, SDValue N4,8393 const SDNodeFlags Flags) {8394 SDValue Ops[] = { N1, N2, N3, N4 };8395 return getNode(Opcode, DL, VT, Ops, Flags);8396}8397 8398SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,8399 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {8400 SDNodeFlags Flags;8401 if (Inserter)8402 Flags = Inserter->getFlags();8403 return getNode(Opcode, DL, VT, N1, N2, N3, N4, Flags);8404}8405 8406SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,8407 SDValue N1, SDValue N2, SDValue N3, SDValue N4,8408 SDValue N5, const SDNodeFlags Flags) {8409 SDValue Ops[] = { N1, N2, N3, N4, N5 };8410 return getNode(Opcode, DL, VT, Ops, Flags);8411}8412 8413SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,8414 SDValue N1, SDValue N2, SDValue N3, SDValue N4,8415 SDValue N5) {8416 SDNodeFlags Flags;8417 if (Inserter)8418 Flags = Inserter->getFlags();8419 return getNode(Opcode, DL, VT, N1, N2, N3, N4, N5, Flags);8420}8421 8422/// getStackArgumentTokenFactor - Compute a TokenFactor to force all8423/// the incoming stack arguments to be loaded from the stack.8424SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {8425 SmallVector<SDValue, 8> ArgChains;8426 8427 // Include the original chain at the beginning of the list. When this is8428 // used by target LowerCall hooks, this helps legalize find the8429 // CALLSEQ_BEGIN node.8430 ArgChains.push_back(Chain);8431 8432 // Add a chain value for each stack argument.8433 for (SDNode *U : getEntryNode().getNode()->users())8434 if (LoadSDNode *L = dyn_cast<LoadSDNode>(U))8435 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))8436 if (FI->getIndex() < 0)8437 ArgChains.push_back(SDValue(L, 1));8438 8439 // Build a tokenfactor for all the chains.8440 return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);8441}8442 8443/// getMemsetValue - Vectorized representation of the memset value8444/// operand.8445static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,8446 const SDLoc &dl) {8447 assert(!Value.isUndef());8448 8449 unsigned NumBits = VT.getScalarSizeInBits();8450 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {8451 assert(C->getAPIntValue().getBitWidth() == 8);8452 APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());8453 if (VT.isInteger()) {8454 bool IsOpaque = VT.getSizeInBits() > 64 ||8455 !DAG.getTargetLoweringInfo().isLegalStoreImmediate(C->getSExtValue());8456 return DAG.getConstant(Val, dl, VT, false, IsOpaque);8457 }8458 return DAG.getConstantFP(APFloat(VT.getFltSemantics(), Val), dl, VT);8459 }8460 8461 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");8462 EVT IntVT = VT.getScalarType();8463 if (!IntVT.isInteger())8464 IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits());8465 8466 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value);8467 if (NumBits > 8) {8468 // Use a multiplication with 0x010101... to extend the input to the8469 // required length.8470 APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));8471 Value = DAG.getNode(ISD::MUL, dl, IntVT, Value,8472 DAG.getConstant(Magic, dl, IntVT));8473 }8474 8475 if (VT != Value.getValueType() && !VT.isInteger())8476 Value = DAG.getBitcast(VT.getScalarType(), Value);8477 if (VT != Value.getValueType())8478 Value = DAG.getSplatBuildVector(VT, dl, Value);8479 8480 return Value;8481}8482 8483/// getMemsetStringVal - Similar to getMemsetValue. Except this is only8484/// used when a memcpy is turned into a memset when the source is a constant8485/// string ptr.8486static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,8487 const TargetLowering &TLI,8488 const ConstantDataArraySlice &Slice) {8489 // Handle vector with all elements zero.8490 if (Slice.Array == nullptr) {8491 if (VT.isInteger())8492 return DAG.getConstant(0, dl, VT);8493 return DAG.getNode(ISD::BITCAST, dl, VT,8494 DAG.getConstant(0, dl, VT.changeTypeToInteger()));8495 }8496 8497 assert(!VT.isVector() && "Can't handle vector type here!");8498 unsigned NumVTBits = VT.getSizeInBits();8499 unsigned NumVTBytes = NumVTBits / 8;8500 unsigned NumBytes = std::min(NumVTBytes, unsigned(Slice.Length));8501 8502 APInt Val(NumVTBits, 0);8503 if (DAG.getDataLayout().isLittleEndian()) {8504 for (unsigned i = 0; i != NumBytes; ++i)8505 Val |= (uint64_t)(unsigned char)Slice[i] << i*8;8506 } else {8507 for (unsigned i = 0; i != NumBytes; ++i)8508 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;8509 }8510 8511 // If the "cost" of materializing the integer immediate is less than the cost8512 // of a load, then it is cost effective to turn the load into the immediate.8513 Type *Ty = VT.getTypeForEVT(*DAG.getContext());8514 if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))8515 return DAG.getConstant(Val, dl, VT);8516 return SDValue();8517}8518 8519SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset,8520 const SDLoc &DL,8521 const SDNodeFlags Flags) {8522 SDValue Index = getTypeSize(DL, Base.getValueType(), Offset);8523 return getMemBasePlusOffset(Base, Index, DL, Flags);8524}8525 8526SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset,8527 const SDLoc &DL,8528 const SDNodeFlags Flags) {8529 assert(Offset.getValueType().isInteger());8530 EVT BasePtrVT = Ptr.getValueType();8531 if (TLI->shouldPreservePtrArith(this->getMachineFunction().getFunction(),8532 BasePtrVT))8533 return getNode(ISD::PTRADD, DL, BasePtrVT, Ptr, Offset, Flags);8534 // InBounds only applies to PTRADD, don't set it if we generate ADD.8535 SDNodeFlags AddFlags = Flags;8536 AddFlags.setInBounds(false);8537 return getNode(ISD::ADD, DL, BasePtrVT, Ptr, Offset, AddFlags);8538}8539 8540/// Returns true if memcpy source is constant data.8541static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {8542 uint64_t SrcDelta = 0;8543 GlobalAddressSDNode *G = nullptr;8544 if (Src.getOpcode() == ISD::GlobalAddress)8545 G = cast<GlobalAddressSDNode>(Src);8546 else if (Src->isAnyAdd() &&8547 Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&8548 Src.getOperand(1).getOpcode() == ISD::Constant) {8549 G = cast<GlobalAddressSDNode>(Src.getOperand(0));8550 SrcDelta = Src.getConstantOperandVal(1);8551 }8552 if (!G)8553 return false;8554 8555 return getConstantDataArrayInfo(G->getGlobal(), Slice, 8,8556 SrcDelta + G->getOffset());8557}8558 8559static bool shouldLowerMemFuncForSize(const MachineFunction &MF,8560 SelectionDAG &DAG) {8561 // On Darwin, -Os means optimize for size without hurting performance, so8562 // only really optimize for size when -Oz (MinSize) is used.8563 if (MF.getTarget().getTargetTriple().isOSDarwin())8564 return MF.getFunction().hasMinSize();8565 return DAG.shouldOptForSize();8566}8567 8568static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,8569 SmallVector<SDValue, 32> &OutChains, unsigned From,8570 unsigned To, SmallVector<SDValue, 16> &OutLoadChains,8571 SmallVector<SDValue, 16> &OutStoreChains) {8572 assert(OutLoadChains.size() && "Missing loads in memcpy inlining");8573 assert(OutStoreChains.size() && "Missing stores in memcpy inlining");8574 SmallVector<SDValue, 16> GluedLoadChains;8575 for (unsigned i = From; i < To; ++i) {8576 OutChains.push_back(OutLoadChains[i]);8577 GluedLoadChains.push_back(OutLoadChains[i]);8578 }8579 8580 // Chain for all loads.8581 SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,8582 GluedLoadChains);8583 8584 for (unsigned i = From; i < To; ++i) {8585 StoreSDNode *ST = dyn_cast<StoreSDNode>(OutStoreChains[i]);8586 SDValue NewStore = DAG.getTruncStore(LoadToken, dl, ST->getValue(),8587 ST->getBasePtr(), ST->getMemoryVT(),8588 ST->getMemOperand());8589 OutChains.push_back(NewStore);8590 }8591}8592 8593static SDValue getMemcpyLoadsAndStores(8594 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,8595 uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline,8596 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo,8597 const AAMDNodes &AAInfo, BatchAAResults *BatchAA) {8598 // Turn a memcpy of undef to nop.8599 // FIXME: We need to honor volatile even is Src is undef.8600 if (Src.isUndef())8601 return Chain;8602 8603 // Expand memcpy to a series of load and store ops if the size operand falls8604 // below a certain threshold.8605 // TODO: In the AlwaysInline case, if the size is big then generate a loop8606 // rather than maybe a humongous number of loads and stores.8607 const TargetLowering &TLI = DAG.getTargetLoweringInfo();8608 const DataLayout &DL = DAG.getDataLayout();8609 LLVMContext &C = *DAG.getContext();8610 std::vector<EVT> MemOps;8611 bool DstAlignCanChange = false;8612 MachineFunction &MF = DAG.getMachineFunction();8613 MachineFrameInfo &MFI = MF.getFrameInfo();8614 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);8615 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);8616 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))8617 DstAlignCanChange = true;8618 MaybeAlign SrcAlign = DAG.InferPtrAlign(Src);8619 if (!SrcAlign || Alignment > *SrcAlign)8620 SrcAlign = Alignment;8621 assert(SrcAlign && "SrcAlign must be set");8622 ConstantDataArraySlice Slice;8623 // If marked as volatile, perform a copy even when marked as constant.8624 bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);8625 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;8626 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);8627 const MemOp Op = isZeroConstant8628 ? MemOp::Set(Size, DstAlignCanChange, Alignment,8629 /*IsZeroMemset*/ true, isVol)8630 : MemOp::Copy(Size, DstAlignCanChange, Alignment,8631 *SrcAlign, isVol, CopyFromConstant);8632 if (!TLI.findOptimalMemOpLowering(8633 C, MemOps, Limit, Op, DstPtrInfo.getAddrSpace(),8634 SrcPtrInfo.getAddrSpace(), MF.getFunction().getAttributes()))8635 return SDValue();8636 8637 if (DstAlignCanChange) {8638 Type *Ty = MemOps[0].getTypeForEVT(C);8639 Align NewAlign = DL.getABITypeAlign(Ty);8640 8641 // Don't promote to an alignment that would require dynamic stack8642 // realignment which may conflict with optimizations such as tail call8643 // optimization.8644 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();8645 if (!TRI->hasStackRealignment(MF))8646 if (MaybeAlign StackAlign = DL.getStackAlignment())8647 NewAlign = std::min(NewAlign, *StackAlign);8648 8649 if (NewAlign > Alignment) {8650 // Give the stack frame object a larger alignment if needed.8651 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)8652 MFI.setObjectAlignment(FI->getIndex(), NewAlign);8653 Alignment = NewAlign;8654 }8655 }8656 8657 // Prepare AAInfo for loads/stores after lowering this memcpy.8658 AAMDNodes NewAAInfo = AAInfo;8659 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;8660 8661 const Value *SrcVal = dyn_cast_if_present<const Value *>(SrcPtrInfo.V);8662 bool isConstant =8663 BatchAA && SrcVal &&8664 BatchAA->pointsToConstantMemory(MemoryLocation(SrcVal, Size, AAInfo));8665 8666 MachineMemOperand::Flags MMOFlags =8667 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;8668 SmallVector<SDValue, 16> OutLoadChains;8669 SmallVector<SDValue, 16> OutStoreChains;8670 SmallVector<SDValue, 32> OutChains;8671 unsigned NumMemOps = MemOps.size();8672 uint64_t SrcOff = 0, DstOff = 0;8673 for (unsigned i = 0; i != NumMemOps; ++i) {8674 EVT VT = MemOps[i];8675 unsigned VTSize = VT.getSizeInBits() / 8;8676 SDValue Value, Store;8677 8678 if (VTSize > Size) {8679 // Issuing an unaligned load / store pair that overlaps with the previous8680 // pair. Adjust the offset accordingly.8681 assert(i == NumMemOps-1 && i != 0);8682 SrcOff -= VTSize - Size;8683 DstOff -= VTSize - Size;8684 }8685 8686 if (CopyFromConstant &&8687 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {8688 // It's unlikely a store of a vector immediate can be done in a single8689 // instruction. It would require a load from a constantpool first.8690 // We only handle zero vectors here.8691 // FIXME: Handle other cases where store of vector immediate is done in8692 // a single instruction.8693 ConstantDataArraySlice SubSlice;8694 if (SrcOff < Slice.Length) {8695 SubSlice = Slice;8696 SubSlice.move(SrcOff);8697 } else {8698 // This is an out-of-bounds access and hence UB. Pretend we read zero.8699 SubSlice.Array = nullptr;8700 SubSlice.Offset = 0;8701 SubSlice.Length = VTSize;8702 }8703 Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice);8704 if (Value.getNode()) {8705 Store = DAG.getStore(8706 Chain, dl, Value,8707 DAG.getObjectPtrOffset(dl, Dst, TypeSize::getFixed(DstOff)),8708 DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo);8709 OutChains.push_back(Store);8710 }8711 }8712 8713 if (!Store.getNode()) {8714 // The type might not be legal for the target. This should only happen8715 // if the type is smaller than a legal type, as on PPC, so the right8716 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify8717 // to Load/Store if NVT==VT.8718 // FIXME does the case above also need this?8719 EVT NVT = TLI.getTypeToTransformTo(C, VT);8720 assert(NVT.bitsGE(VT));8721 8722 bool isDereferenceable =8723 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);8724 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;8725 if (isDereferenceable)8726 SrcMMOFlags |= MachineMemOperand::MODereferenceable;8727 if (isConstant)8728 SrcMMOFlags |= MachineMemOperand::MOInvariant;8729 8730 Value = DAG.getExtLoad(8731 ISD::EXTLOAD, dl, NVT, Chain,8732 DAG.getObjectPtrOffset(dl, Src, TypeSize::getFixed(SrcOff)),8733 SrcPtrInfo.getWithOffset(SrcOff), VT,8734 commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags, NewAAInfo);8735 OutLoadChains.push_back(Value.getValue(1));8736 8737 Store = DAG.getTruncStore(8738 Chain, dl, Value,8739 DAG.getObjectPtrOffset(dl, Dst, TypeSize::getFixed(DstOff)),8740 DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags, NewAAInfo);8741 OutStoreChains.push_back(Store);8742 }8743 SrcOff += VTSize;8744 DstOff += VTSize;8745 Size -= VTSize;8746 }8747 8748 unsigned GluedLdStLimit = MaxLdStGlue == 0 ?8749 TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue;8750 unsigned NumLdStInMemcpy = OutStoreChains.size();8751 8752 if (NumLdStInMemcpy) {8753 // It may be that memcpy might be converted to memset if it's memcpy8754 // of constants. In such a case, we won't have loads and stores, but8755 // just stores. In the absence of loads, there is nothing to gang up.8756 if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) {8757 // If target does not care, just leave as it.8758 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {8759 OutChains.push_back(OutLoadChains[i]);8760 OutChains.push_back(OutStoreChains[i]);8761 }8762 } else {8763 // Ld/St less than/equal limit set by target.8764 if (NumLdStInMemcpy <= GluedLdStLimit) {8765 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,8766 NumLdStInMemcpy, OutLoadChains,8767 OutStoreChains);8768 } else {8769 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;8770 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;8771 unsigned GlueIter = 0;8772 8773 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {8774 unsigned IndexFrom = NumLdStInMemcpy - GlueIter - GluedLdStLimit;8775 unsigned IndexTo = NumLdStInMemcpy - GlueIter;8776 8777 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, IndexFrom, IndexTo,8778 OutLoadChains, OutStoreChains);8779 GlueIter += GluedLdStLimit;8780 }8781 8782 // Residual ld/st.8783 if (RemainingLdStInMemcpy) {8784 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,8785 RemainingLdStInMemcpy, OutLoadChains,8786 OutStoreChains);8787 }8788 }8789 }8790 }8791 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);8792}8793 8794static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,8795 SDValue Chain, SDValue Dst, SDValue Src,8796 uint64_t Size, Align Alignment,8797 bool isVol, bool AlwaysInline,8798 MachinePointerInfo DstPtrInfo,8799 MachinePointerInfo SrcPtrInfo,8800 const AAMDNodes &AAInfo) {8801 // Turn a memmove of undef to nop.8802 // FIXME: We need to honor volatile even is Src is undef.8803 if (Src.isUndef())8804 return Chain;8805 8806 // Expand memmove to a series of load and store ops if the size operand falls8807 // below a certain threshold.8808 const TargetLowering &TLI = DAG.getTargetLoweringInfo();8809 const DataLayout &DL = DAG.getDataLayout();8810 LLVMContext &C = *DAG.getContext();8811 std::vector<EVT> MemOps;8812 bool DstAlignCanChange = false;8813 MachineFunction &MF = DAG.getMachineFunction();8814 MachineFrameInfo &MFI = MF.getFrameInfo();8815 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);8816 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);8817 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))8818 DstAlignCanChange = true;8819 MaybeAlign SrcAlign = DAG.InferPtrAlign(Src);8820 if (!SrcAlign || Alignment > *SrcAlign)8821 SrcAlign = Alignment;8822 assert(SrcAlign && "SrcAlign must be set");8823 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);8824 if (!TLI.findOptimalMemOpLowering(8825 C, MemOps, Limit,8826 MemOp::Copy(Size, DstAlignCanChange, Alignment, *SrcAlign,8827 /*IsVolatile*/ true),8828 DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),8829 MF.getFunction().getAttributes()))8830 return SDValue();8831 8832 if (DstAlignCanChange) {8833 Type *Ty = MemOps[0].getTypeForEVT(C);8834 Align NewAlign = DL.getABITypeAlign(Ty);8835 8836 // Don't promote to an alignment that would require dynamic stack8837 // realignment which may conflict with optimizations such as tail call8838 // optimization.8839 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();8840 if (!TRI->hasStackRealignment(MF))8841 if (MaybeAlign StackAlign = DL.getStackAlignment())8842 NewAlign = std::min(NewAlign, *StackAlign);8843 8844 if (NewAlign > Alignment) {8845 // Give the stack frame object a larger alignment if needed.8846 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)8847 MFI.setObjectAlignment(FI->getIndex(), NewAlign);8848 Alignment = NewAlign;8849 }8850 }8851 8852 // Prepare AAInfo for loads/stores after lowering this memmove.8853 AAMDNodes NewAAInfo = AAInfo;8854 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;8855 8856 MachineMemOperand::Flags MMOFlags =8857 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;8858 uint64_t SrcOff = 0, DstOff = 0;8859 SmallVector<SDValue, 8> LoadValues;8860 SmallVector<SDValue, 8> LoadChains;8861 SmallVector<SDValue, 8> OutChains;8862 unsigned NumMemOps = MemOps.size();8863 for (unsigned i = 0; i < NumMemOps; i++) {8864 EVT VT = MemOps[i];8865 unsigned VTSize = VT.getSizeInBits() / 8;8866 SDValue Value;8867 8868 bool isDereferenceable =8869 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);8870 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;8871 if (isDereferenceable)8872 SrcMMOFlags |= MachineMemOperand::MODereferenceable;8873 8874 Value = DAG.getLoad(8875 VT, dl, Chain,8876 DAG.getObjectPtrOffset(dl, Src, TypeSize::getFixed(SrcOff)),8877 SrcPtrInfo.getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags, NewAAInfo);8878 LoadValues.push_back(Value);8879 LoadChains.push_back(Value.getValue(1));8880 SrcOff += VTSize;8881 }8882 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);8883 OutChains.clear();8884 for (unsigned i = 0; i < NumMemOps; i++) {8885 EVT VT = MemOps[i];8886 unsigned VTSize = VT.getSizeInBits() / 8;8887 SDValue Store;8888 8889 Store = DAG.getStore(8890 Chain, dl, LoadValues[i],8891 DAG.getObjectPtrOffset(dl, Dst, TypeSize::getFixed(DstOff)),8892 DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo);8893 OutChains.push_back(Store);8894 DstOff += VTSize;8895 }8896 8897 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);8898}8899 8900/// Lower the call to 'memset' intrinsic function into a series of store8901/// operations.8902///8903/// \param DAG Selection DAG where lowered code is placed.8904/// \param dl Link to corresponding IR location.8905/// \param Chain Control flow dependency.8906/// \param Dst Pointer to destination memory location.8907/// \param Src Value of byte to write into the memory.8908/// \param Size Number of bytes to write.8909/// \param Alignment Alignment of the destination in bytes.8910/// \param isVol True if destination is volatile.8911/// \param AlwaysInline Makes sure no function call is generated.8912/// \param DstPtrInfo IR information on the memory pointer.8913/// \returns New head in the control flow, if lowering was successful, empty8914/// SDValue otherwise.8915///8916/// The function tries to replace 'llvm.memset' intrinsic with several store8917/// operations and value calculation code. This is usually profitable for small8918/// memory size or when the semantic requires inlining.8919static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,8920 SDValue Chain, SDValue Dst, SDValue Src,8921 uint64_t Size, Align Alignment, bool isVol,8922 bool AlwaysInline, MachinePointerInfo DstPtrInfo,8923 const AAMDNodes &AAInfo) {8924 // Turn a memset of undef to nop.8925 // FIXME: We need to honor volatile even is Src is undef.8926 if (Src.isUndef())8927 return Chain;8928 8929 // Expand memset to a series of load/store ops if the size operand8930 // falls below a certain threshold.8931 const TargetLowering &TLI = DAG.getTargetLoweringInfo();8932 std::vector<EVT> MemOps;8933 bool DstAlignCanChange = false;8934 LLVMContext &C = *DAG.getContext();8935 MachineFunction &MF = DAG.getMachineFunction();8936 MachineFrameInfo &MFI = MF.getFrameInfo();8937 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);8938 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);8939 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))8940 DstAlignCanChange = true;8941 bool IsZeroVal = isNullConstant(Src);8942 unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize);8943 8944 if (!TLI.findOptimalMemOpLowering(8945 C, MemOps, Limit,8946 MemOp::Set(Size, DstAlignCanChange, Alignment, IsZeroVal, isVol),8947 DstPtrInfo.getAddrSpace(), ~0u, MF.getFunction().getAttributes()))8948 return SDValue();8949 8950 if (DstAlignCanChange) {8951 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());8952 const DataLayout &DL = DAG.getDataLayout();8953 Align NewAlign = DL.getABITypeAlign(Ty);8954 8955 // Don't promote to an alignment that would require dynamic stack8956 // realignment which may conflict with optimizations such as tail call8957 // optimization.8958 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();8959 if (!TRI->hasStackRealignment(MF))8960 if (MaybeAlign StackAlign = DL.getStackAlignment())8961 NewAlign = std::min(NewAlign, *StackAlign);8962 8963 if (NewAlign > Alignment) {8964 // Give the stack frame object a larger alignment if needed.8965 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)8966 MFI.setObjectAlignment(FI->getIndex(), NewAlign);8967 Alignment = NewAlign;8968 }8969 }8970 8971 SmallVector<SDValue, 8> OutChains;8972 uint64_t DstOff = 0;8973 unsigned NumMemOps = MemOps.size();8974 8975 // Find the largest store and generate the bit pattern for it.8976 EVT LargestVT = MemOps[0];8977 for (unsigned i = 1; i < NumMemOps; i++)8978 if (MemOps[i].bitsGT(LargestVT))8979 LargestVT = MemOps[i];8980 SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);8981 8982 // Prepare AAInfo for loads/stores after lowering this memset.8983 AAMDNodes NewAAInfo = AAInfo;8984 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;8985 8986 for (unsigned i = 0; i < NumMemOps; i++) {8987 EVT VT = MemOps[i];8988 unsigned VTSize = VT.getSizeInBits() / 8;8989 if (VTSize > Size) {8990 // Issuing an unaligned load / store pair that overlaps with the previous8991 // pair. Adjust the offset accordingly.8992 assert(i == NumMemOps-1 && i != 0);8993 DstOff -= VTSize - Size;8994 }8995 8996 // If this store is smaller than the largest store see whether we can get8997 // the smaller value for free with a truncate or extract vector element and8998 // then store.8999 SDValue Value = MemSetValue;9000 if (VT.bitsLT(LargestVT)) {9001 unsigned Index;9002 unsigned NElts = LargestVT.getSizeInBits() / VT.getSizeInBits();9003 EVT SVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), NElts);9004 if (!LargestVT.isVector() && !VT.isVector() &&9005 TLI.isTruncateFree(LargestVT, VT))9006 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);9007 else if (LargestVT.isVector() && !VT.isVector() &&9008 TLI.shallExtractConstSplatVectorElementToStore(9009 LargestVT.getTypeForEVT(*DAG.getContext()),9010 VT.getSizeInBits(), Index) &&9011 TLI.isTypeLegal(SVT) &&9012 LargestVT.getSizeInBits() == SVT.getSizeInBits()) {9013 // Target which can combine store(extractelement VectorTy, Idx) can get9014 // the smaller value for free.9015 SDValue TailValue = DAG.getNode(ISD::BITCAST, dl, SVT, MemSetValue);9016 Value = DAG.getExtractVectorElt(dl, VT, TailValue, Index);9017 } else9018 Value = getMemsetValue(Src, VT, DAG, dl);9019 }9020 assert(Value.getValueType() == VT && "Value with wrong type.");9021 SDValue Store = DAG.getStore(9022 Chain, dl, Value,9023 DAG.getObjectPtrOffset(dl, Dst, TypeSize::getFixed(DstOff)),9024 DstPtrInfo.getWithOffset(DstOff), Alignment,9025 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone,9026 NewAAInfo);9027 OutChains.push_back(Store);9028 DstOff += VT.getSizeInBits() / 8;9029 Size -= VTSize;9030 }9031 9032 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);9033}9034 9035static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,9036 unsigned AS) {9037 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all9038 // pointer operands can be losslessly bitcasted to pointers of address space 09039 if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(AS, 0)) {9040 report_fatal_error("cannot lower memory intrinsic in address space " +9041 Twine(AS));9042 }9043}9044 9045static bool isInTailCallPositionWrapper(const CallInst *CI,9046 const SelectionDAG *SelDAG,9047 bool AllowReturnsFirstArg) {9048 if (!CI || !CI->isTailCall())9049 return false;9050 // TODO: Fix "returns-first-arg" determination so it doesn't depend on which9051 // helper symbol we lower to.9052 return isInTailCallPosition(*CI, SelDAG->getTarget(),9053 AllowReturnsFirstArg &&9054 funcReturnsFirstArgOfCall(*CI));9055}9056 9057std::pair<SDValue, SDValue>9058SelectionDAG::getMemcmp(SDValue Chain, const SDLoc &dl, SDValue Mem0,9059 SDValue Mem1, SDValue Size, const CallInst *CI) {9060 const char *LibCallName = TLI->getLibcallName(RTLIB::MEMCMP);9061 if (!LibCallName)9062 return {};9063 9064 PointerType *PT = PointerType::getUnqual(*getContext());9065 TargetLowering::ArgListTy Args = {9066 {Mem0, PT},9067 {Mem1, PT},9068 {Size, getDataLayout().getIntPtrType(*getContext())}};9069 9070 TargetLowering::CallLoweringInfo CLI(*this);9071 bool IsTailCall =9072 isInTailCallPositionWrapper(CI, this, /*AllowReturnsFirstArg*/ true);9073 9074 CLI.setDebugLoc(dl)9075 .setChain(Chain)9076 .setLibCallee(9077 TLI->getLibcallCallingConv(RTLIB::MEMCMP),9078 Type::getInt32Ty(*getContext()),9079 getExternalSymbol(LibCallName, TLI->getPointerTy(getDataLayout())),9080 std::move(Args))9081 .setTailCall(IsTailCall);9082 9083 return TLI->LowerCallTo(CLI);9084}9085 9086std::pair<SDValue, SDValue> SelectionDAG::getStrlen(SDValue Chain,9087 const SDLoc &dl,9088 SDValue Src,9089 const CallInst *CI) {9090 const char *LibCallName = TLI->getLibcallName(RTLIB::STRLEN);9091 if (!LibCallName)9092 return {};9093 9094 // Emit a library call.9095 TargetLowering::ArgListTy Args = {9096 {Src, PointerType::getUnqual(*getContext())}};9097 9098 TargetLowering::CallLoweringInfo CLI(*this);9099 bool IsTailCall =9100 isInTailCallPositionWrapper(CI, this, /*AllowReturnsFirstArg*/ true);9101 9102 CLI.setDebugLoc(dl)9103 .setChain(Chain)9104 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::STRLEN), CI->getType(),9105 getExternalSymbol(9106 LibCallName, TLI->getProgramPointerTy(getDataLayout())),9107 std::move(Args))9108 .setTailCall(IsTailCall);9109 9110 return TLI->LowerCallTo(CLI);9111}9112 9113SDValue SelectionDAG::getMemcpy(9114 SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size,9115 Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI,9116 std::optional<bool> OverrideTailCall, MachinePointerInfo DstPtrInfo,9117 MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo,9118 BatchAAResults *BatchAA) {9119 // Check to see if we should lower the memcpy to loads and stores first.9120 // For cases within the target-specified limits, this is the best choice.9121 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);9122 if (ConstantSize) {9123 // Memcpy with size zero? Just return the original chain.9124 if (ConstantSize->isZero())9125 return Chain;9126 9127 SDValue Result = getMemcpyLoadsAndStores(9128 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,9129 isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA);9130 if (Result.getNode())9131 return Result;9132 }9133 9134 // Then check to see if we should lower the memcpy with target-specific9135 // code. If the target chooses to do this, this is the next best.9136 if (TSI) {9137 SDValue Result = TSI->EmitTargetCodeForMemcpy(9138 *this, dl, Chain, Dst, Src, Size, Alignment, isVol, AlwaysInline,9139 DstPtrInfo, SrcPtrInfo);9140 if (Result.getNode())9141 return Result;9142 }9143 9144 // If we really need inline code and the target declined to provide it,9145 // use a (potentially long) sequence of loads and stores.9146 if (AlwaysInline) {9147 assert(ConstantSize && "AlwaysInline requires a constant size!");9148 return getMemcpyLoadsAndStores(9149 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,9150 isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA);9151 }9152 9153 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());9154 checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());9155 9156 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc9157 // memcpy is not guaranteed to be safe. libc memcpys aren't required to9158 // respect volatile, so they may do things like read or write memory9159 // beyond the given memory regions. But fixing this isn't easy, and most9160 // people don't care.9161 9162 // Emit a library call.9163 TargetLowering::ArgListTy Args;9164 Type *PtrTy = PointerType::getUnqual(*getContext());9165 Args.emplace_back(Dst, PtrTy);9166 Args.emplace_back(Src, PtrTy);9167 Args.emplace_back(Size, getDataLayout().getIntPtrType(*getContext()));9168 // FIXME: pass in SDLoc9169 TargetLowering::CallLoweringInfo CLI(*this);9170 bool IsTailCall = false;9171 RTLIB::LibcallImpl MemCpyImpl = TLI->getMemcpyImpl();9172 9173 if (OverrideTailCall.has_value()) {9174 IsTailCall = *OverrideTailCall;9175 } else {9176 bool LowersToMemcpy = MemCpyImpl == RTLIB::impl_memcpy;9177 IsTailCall = isInTailCallPositionWrapper(CI, this, LowersToMemcpy);9178 }9179 9180 CLI.setDebugLoc(dl)9181 .setChain(Chain)9182 .setLibCallee(9183 TLI->getLibcallImplCallingConv(MemCpyImpl),9184 Dst.getValueType().getTypeForEVT(*getContext()),9185 getExternalSymbol(TLI->getLibcallImplName(MemCpyImpl).data(),9186 TLI->getPointerTy(getDataLayout())),9187 std::move(Args))9188 .setDiscardResult()9189 .setTailCall(IsTailCall);9190 9191 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);9192 return CallResult.second;9193}9194 9195SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,9196 SDValue Dst, SDValue Src, SDValue Size,9197 Type *SizeTy, unsigned ElemSz,9198 bool isTailCall,9199 MachinePointerInfo DstPtrInfo,9200 MachinePointerInfo SrcPtrInfo) {9201 // Emit a library call.9202 TargetLowering::ArgListTy Args;9203 Type *ArgTy = getDataLayout().getIntPtrType(*getContext());9204 Args.emplace_back(Dst, ArgTy);9205 Args.emplace_back(Src, ArgTy);9206 Args.emplace_back(Size, SizeTy);9207 9208 RTLIB::Libcall LibraryCall =9209 RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElemSz);9210 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)9211 report_fatal_error("Unsupported element size");9212 9213 TargetLowering::CallLoweringInfo CLI(*this);9214 CLI.setDebugLoc(dl)9215 .setChain(Chain)9216 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),9217 Type::getVoidTy(*getContext()),9218 getExternalSymbol(TLI->getLibcallName(LibraryCall),9219 TLI->getPointerTy(getDataLayout())),9220 std::move(Args))9221 .setDiscardResult()9222 .setTailCall(isTailCall);9223 9224 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);9225 return CallResult.second;9226}9227 9228SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,9229 SDValue Src, SDValue Size, Align Alignment,9230 bool isVol, const CallInst *CI,9231 std::optional<bool> OverrideTailCall,9232 MachinePointerInfo DstPtrInfo,9233 MachinePointerInfo SrcPtrInfo,9234 const AAMDNodes &AAInfo,9235 BatchAAResults *BatchAA) {9236 // Check to see if we should lower the memmove to loads and stores first.9237 // For cases within the target-specified limits, this is the best choice.9238 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);9239 if (ConstantSize) {9240 // Memmove with size zero? Just return the original chain.9241 if (ConstantSize->isZero())9242 return Chain;9243 9244 SDValue Result = getMemmoveLoadsAndStores(9245 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,9246 isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo);9247 if (Result.getNode())9248 return Result;9249 }9250 9251 // Then check to see if we should lower the memmove with target-specific9252 // code. If the target chooses to do this, this is the next best.9253 if (TSI) {9254 SDValue Result =9255 TSI->EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size,9256 Alignment, isVol, DstPtrInfo, SrcPtrInfo);9257 if (Result.getNode())9258 return Result;9259 }9260 9261 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());9262 checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());9263 9264 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may9265 // not be safe. See memcpy above for more details.9266 9267 // Emit a library call.9268 TargetLowering::ArgListTy Args;9269 Type *PtrTy = PointerType::getUnqual(*getContext());9270 Args.emplace_back(Dst, PtrTy);9271 Args.emplace_back(Src, PtrTy);9272 Args.emplace_back(Size, getDataLayout().getIntPtrType(*getContext()));9273 // FIXME: pass in SDLoc9274 TargetLowering::CallLoweringInfo CLI(*this);9275 9276 RTLIB::LibcallImpl MemmoveImpl = TLI->getLibcallImpl(RTLIB::MEMMOVE);9277 9278 bool IsTailCall = false;9279 if (OverrideTailCall.has_value()) {9280 IsTailCall = *OverrideTailCall;9281 } else {9282 bool LowersToMemmove = MemmoveImpl == RTLIB::impl_memmove;9283 IsTailCall = isInTailCallPositionWrapper(CI, this, LowersToMemmove);9284 }9285 9286 CLI.setDebugLoc(dl)9287 .setChain(Chain)9288 .setLibCallee(9289 TLI->getLibcallImplCallingConv(MemmoveImpl),9290 Dst.getValueType().getTypeForEVT(*getContext()),9291 getExternalSymbol(TLI->getLibcallImplName(MemmoveImpl).data(),9292 TLI->getPointerTy(getDataLayout())),9293 std::move(Args))9294 .setDiscardResult()9295 .setTailCall(IsTailCall);9296 9297 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);9298 return CallResult.second;9299}9300 9301SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,9302 SDValue Dst, SDValue Src, SDValue Size,9303 Type *SizeTy, unsigned ElemSz,9304 bool isTailCall,9305 MachinePointerInfo DstPtrInfo,9306 MachinePointerInfo SrcPtrInfo) {9307 // Emit a library call.9308 TargetLowering::ArgListTy Args;9309 Type *IntPtrTy = getDataLayout().getIntPtrType(*getContext());9310 Args.emplace_back(Dst, IntPtrTy);9311 Args.emplace_back(Src, IntPtrTy);9312 Args.emplace_back(Size, SizeTy);9313 9314 RTLIB::Libcall LibraryCall =9315 RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElemSz);9316 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)9317 report_fatal_error("Unsupported element size");9318 9319 TargetLowering::CallLoweringInfo CLI(*this);9320 CLI.setDebugLoc(dl)9321 .setChain(Chain)9322 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),9323 Type::getVoidTy(*getContext()),9324 getExternalSymbol(TLI->getLibcallName(LibraryCall),9325 TLI->getPointerTy(getDataLayout())),9326 std::move(Args))9327 .setDiscardResult()9328 .setTailCall(isTailCall);9329 9330 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);9331 return CallResult.second;9332}9333 9334SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,9335 SDValue Src, SDValue Size, Align Alignment,9336 bool isVol, bool AlwaysInline,9337 const CallInst *CI,9338 MachinePointerInfo DstPtrInfo,9339 const AAMDNodes &AAInfo) {9340 // Check to see if we should lower the memset to stores first.9341 // For cases within the target-specified limits, this is the best choice.9342 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);9343 if (ConstantSize) {9344 // Memset with size zero? Just return the original chain.9345 if (ConstantSize->isZero())9346 return Chain;9347 9348 SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src,9349 ConstantSize->getZExtValue(), Alignment,9350 isVol, false, DstPtrInfo, AAInfo);9351 9352 if (Result.getNode())9353 return Result;9354 }9355 9356 // Then check to see if we should lower the memset with target-specific9357 // code. If the target chooses to do this, this is the next best.9358 if (TSI) {9359 SDValue Result = TSI->EmitTargetCodeForMemset(9360 *this, dl, Chain, Dst, Src, Size, Alignment, isVol, AlwaysInline, DstPtrInfo);9361 if (Result.getNode())9362 return Result;9363 }9364 9365 // If we really need inline code and the target declined to provide it,9366 // use a (potentially long) sequence of loads and stores.9367 if (AlwaysInline) {9368 assert(ConstantSize && "AlwaysInline requires a constant size!");9369 SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src,9370 ConstantSize->getZExtValue(), Alignment,9371 isVol, true, DstPtrInfo, AAInfo);9372 assert(Result &&9373 "getMemsetStores must return a valid sequence when AlwaysInline");9374 return Result;9375 }9376 9377 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());9378 9379 // Emit a library call.9380 auto &Ctx = *getContext();9381 const auto& DL = getDataLayout();9382 9383 TargetLowering::CallLoweringInfo CLI(*this);9384 // FIXME: pass in SDLoc9385 CLI.setDebugLoc(dl).setChain(Chain);9386 9387 const char *BzeroName = getTargetLoweringInfo().getLibcallName(RTLIB::BZERO);9388 9389 bool UseBZero = isNullConstant(Src) && BzeroName;9390 // If zeroing out and bzero is present, use it.9391 if (UseBZero) {9392 TargetLowering::ArgListTy Args;9393 Args.emplace_back(Dst, PointerType::getUnqual(Ctx));9394 Args.emplace_back(Size, DL.getIntPtrType(Ctx));9395 CLI.setLibCallee(9396 TLI->getLibcallCallingConv(RTLIB::BZERO), Type::getVoidTy(Ctx),9397 getExternalSymbol(BzeroName, TLI->getPointerTy(DL)), std::move(Args));9398 } else {9399 TargetLowering::ArgListTy Args;9400 Args.emplace_back(Dst, PointerType::getUnqual(Ctx));9401 Args.emplace_back(Src, Src.getValueType().getTypeForEVT(Ctx));9402 Args.emplace_back(Size, DL.getIntPtrType(Ctx));9403 CLI.setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),9404 Dst.getValueType().getTypeForEVT(Ctx),9405 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),9406 TLI->getPointerTy(DL)),9407 std::move(Args));9408 }9409 9410 RTLIB::LibcallImpl MemsetImpl = TLI->getLibcallImpl(RTLIB::MEMSET);9411 bool LowersToMemset = MemsetImpl == RTLIB::impl_memset;9412 9413 // If we're going to use bzero, make sure not to tail call unless the9414 // subsequent return doesn't need a value, as bzero doesn't return the first9415 // arg unlike memset.9416 bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(*CI) && !UseBZero;9417 bool IsTailCall =9418 CI && CI->isTailCall() &&9419 isInTailCallPosition(*CI, getTarget(), ReturnsFirstArg && LowersToMemset);9420 CLI.setDiscardResult().setTailCall(IsTailCall);9421 9422 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);9423 return CallResult.second;9424}9425 9426SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,9427 SDValue Dst, SDValue Value, SDValue Size,9428 Type *SizeTy, unsigned ElemSz,9429 bool isTailCall,9430 MachinePointerInfo DstPtrInfo) {9431 // Emit a library call.9432 TargetLowering::ArgListTy Args;9433 Args.emplace_back(Dst, getDataLayout().getIntPtrType(*getContext()));9434 Args.emplace_back(Value, Type::getInt8Ty(*getContext()));9435 Args.emplace_back(Size, SizeTy);9436 9437 RTLIB::Libcall LibraryCall =9438 RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElemSz);9439 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)9440 report_fatal_error("Unsupported element size");9441 9442 TargetLowering::CallLoweringInfo CLI(*this);9443 CLI.setDebugLoc(dl)9444 .setChain(Chain)9445 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),9446 Type::getVoidTy(*getContext()),9447 getExternalSymbol(TLI->getLibcallName(LibraryCall),9448 TLI->getPointerTy(getDataLayout())),9449 std::move(Args))9450 .setDiscardResult()9451 .setTailCall(isTailCall);9452 9453 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);9454 return CallResult.second;9455}9456 9457SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,9458 SDVTList VTList, ArrayRef<SDValue> Ops,9459 MachineMemOperand *MMO,9460 ISD::LoadExtType ExtType) {9461 FoldingSetNodeID ID;9462 AddNodeIDNode(ID, Opcode, VTList, Ops);9463 ID.AddInteger(MemVT.getRawBits());9464 ID.AddInteger(getSyntheticNodeSubclassData<AtomicSDNode>(9465 dl.getIROrder(), Opcode, VTList, MemVT, MMO, ExtType));9466 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());9467 ID.AddInteger(MMO->getFlags());9468 void* IP = nullptr;9469 if (auto *E = cast_or_null<AtomicSDNode>(FindNodeOrInsertPos(ID, dl, IP))) {9470 E->refineAlignment(MMO);9471 E->refineRanges(MMO);9472 return SDValue(E, 0);9473 }9474 9475 auto *N = newSDNode<AtomicSDNode>(dl.getIROrder(), dl.getDebugLoc(), Opcode,9476 VTList, MemVT, MMO, ExtType);9477 createOperands(N, Ops);9478 9479 CSEMap.InsertNode(N, IP);9480 InsertNode(N);9481 SDValue V(N, 0);9482 NewSDValueDbgMsg(V, "Creating new node: ", this);9483 return V;9484}9485 9486SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,9487 EVT MemVT, SDVTList VTs, SDValue Chain,9488 SDValue Ptr, SDValue Cmp, SDValue Swp,9489 MachineMemOperand *MMO) {9490 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||9491 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);9492 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");9493 9494 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};9495 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);9496}9497 9498SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,9499 SDValue Chain, SDValue Ptr, SDValue Val,9500 MachineMemOperand *MMO) {9501 assert((Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB ||9502 Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR ||9503 Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR ||9504 Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN ||9505 Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN ||9506 Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD ||9507 Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_LOAD_FMAX ||9508 Opcode == ISD::ATOMIC_LOAD_FMIN ||9509 Opcode == ISD::ATOMIC_LOAD_FMINIMUM ||9510 Opcode == ISD::ATOMIC_LOAD_FMAXIMUM ||9511 Opcode == ISD::ATOMIC_LOAD_UINC_WRAP ||9512 Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP ||9513 Opcode == ISD::ATOMIC_LOAD_USUB_COND ||9514 Opcode == ISD::ATOMIC_LOAD_USUB_SAT || Opcode == ISD::ATOMIC_SWAP ||9515 Opcode == ISD::ATOMIC_STORE) &&9516 "Invalid Atomic Op");9517 9518 EVT VT = Val.getValueType();9519 9520 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :9521 getVTList(VT, MVT::Other);9522 SDValue Ops[] = {Chain, Ptr, Val};9523 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);9524}9525 9526SDValue SelectionDAG::getAtomicLoad(ISD::LoadExtType ExtType, const SDLoc &dl,9527 EVT MemVT, EVT VT, SDValue Chain,9528 SDValue Ptr, MachineMemOperand *MMO) {9529 SDVTList VTs = getVTList(VT, MVT::Other);9530 SDValue Ops[] = {Chain, Ptr};9531 return getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, VTs, Ops, MMO, ExtType);9532}9533 9534/// getMergeValues - Create a MERGE_VALUES node from the given operands.9535SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {9536 if (Ops.size() == 1)9537 return Ops[0];9538 9539 SmallVector<EVT, 4> VTs;9540 VTs.reserve(Ops.size());9541 for (const SDValue &Op : Ops)9542 VTs.push_back(Op.getValueType());9543 return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);9544}9545 9546SDValue SelectionDAG::getMemIntrinsicNode(9547 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,9548 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,9549 MachineMemOperand::Flags Flags, LocationSize Size,9550 const AAMDNodes &AAInfo) {9551 if (Size.hasValue() && !Size.getValue())9552 Size = LocationSize::precise(MemVT.getStoreSize());9553 9554 MachineFunction &MF = getMachineFunction();9555 MachineMemOperand *MMO =9556 MF.getMachineMemOperand(PtrInfo, Flags, Size, Alignment, AAInfo);9557 9558 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);9559}9560 9561SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,9562 SDVTList VTList,9563 ArrayRef<SDValue> Ops, EVT MemVT,9564 MachineMemOperand *MMO) {9565 assert(9566 (Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN ||9567 Opcode == ISD::PREFETCH ||9568 (Opcode <= (unsigned)std::numeric_limits<int>::max() &&9569 Opcode >= ISD::BUILTIN_OP_END && TSI->isTargetMemoryOpcode(Opcode))) &&9570 "Opcode is not a memory-accessing opcode!");9571 9572 // Memoize the node unless it returns a glue result.9573 MemIntrinsicSDNode *N;9574 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {9575 FoldingSetNodeID ID;9576 AddNodeIDNode(ID, Opcode, VTList, Ops);9577 ID.AddInteger(getSyntheticNodeSubclassData<MemIntrinsicSDNode>(9578 Opcode, dl.getIROrder(), VTList, MemVT, MMO));9579 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());9580 ID.AddInteger(MMO->getFlags());9581 ID.AddInteger(MemVT.getRawBits());9582 void *IP = nullptr;9583 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {9584 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);9585 return SDValue(E, 0);9586 }9587 9588 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),9589 VTList, MemVT, MMO);9590 createOperands(N, Ops);9591 9592 CSEMap.InsertNode(N, IP);9593 } else {9594 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),9595 VTList, MemVT, MMO);9596 createOperands(N, Ops);9597 }9598 InsertNode(N);9599 SDValue V(N, 0);9600 NewSDValueDbgMsg(V, "Creating new node: ", this);9601 return V;9602}9603 9604SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl,9605 SDValue Chain, int FrameIndex) {9606 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;9607 const auto VTs = getVTList(MVT::Other);9608 SDValue Ops[2] = {9609 Chain,9610 getFrameIndex(FrameIndex,9611 getTargetLoweringInfo().getFrameIndexTy(getDataLayout()),9612 true)};9613 9614 FoldingSetNodeID ID;9615 AddNodeIDNode(ID, Opcode, VTs, Ops);9616 ID.AddInteger(FrameIndex);9617 void *IP = nullptr;9618 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))9619 return SDValue(E, 0);9620 9621 LifetimeSDNode *N =9622 newSDNode<LifetimeSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs);9623 createOperands(N, Ops);9624 CSEMap.InsertNode(N, IP);9625 InsertNode(N);9626 SDValue V(N, 0);9627 NewSDValueDbgMsg(V, "Creating new node: ", this);9628 return V;9629}9630 9631SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain,9632 uint64_t Guid, uint64_t Index,9633 uint32_t Attr) {9634 const unsigned Opcode = ISD::PSEUDO_PROBE;9635 const auto VTs = getVTList(MVT::Other);9636 SDValue Ops[] = {Chain};9637 FoldingSetNodeID ID;9638 AddNodeIDNode(ID, Opcode, VTs, Ops);9639 ID.AddInteger(Guid);9640 ID.AddInteger(Index);9641 void *IP = nullptr;9642 if (SDNode *E = FindNodeOrInsertPos(ID, Dl, IP))9643 return SDValue(E, 0);9644 9645 auto *N = newSDNode<PseudoProbeSDNode>(9646 Opcode, Dl.getIROrder(), Dl.getDebugLoc(), VTs, Guid, Index, Attr);9647 createOperands(N, Ops);9648 CSEMap.InsertNode(N, IP);9649 InsertNode(N);9650 SDValue V(N, 0);9651 NewSDValueDbgMsg(V, "Creating new node: ", this);9652 return V;9653}9654 9655/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a9656/// MachinePointerInfo record from it. This is particularly useful because the9657/// code generator has many cases where it doesn't bother passing in a9658/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".9659static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,9660 SelectionDAG &DAG, SDValue Ptr,9661 int64_t Offset = 0) {9662 // If this is FI+Offset, we can model it.9663 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))9664 return MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),9665 FI->getIndex(), Offset);9666 9667 // If this is (FI+Offset1)+Offset2, we can model it.9668 if (Ptr.getOpcode() != ISD::ADD ||9669 !isa<ConstantSDNode>(Ptr.getOperand(1)) ||9670 !isa<FrameIndexSDNode>(Ptr.getOperand(0)))9671 return Info;9672 9673 int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();9674 return MachinePointerInfo::getFixedStack(9675 DAG.getMachineFunction(), FI,9676 Offset + cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());9677}9678 9679/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a9680/// MachinePointerInfo record from it. This is particularly useful because the9681/// code generator has many cases where it doesn't bother passing in a9682/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".9683static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,9684 SelectionDAG &DAG, SDValue Ptr,9685 SDValue OffsetOp) {9686 // If the 'Offset' value isn't a constant, we can't handle this.9687 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))9688 return InferPointerInfo(Info, DAG, Ptr, OffsetNode->getSExtValue());9689 if (OffsetOp.isUndef())9690 return InferPointerInfo(Info, DAG, Ptr);9691 return Info;9692}9693 9694SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,9695 EVT VT, const SDLoc &dl, SDValue Chain,9696 SDValue Ptr, SDValue Offset,9697 MachinePointerInfo PtrInfo, EVT MemVT,9698 Align Alignment,9699 MachineMemOperand::Flags MMOFlags,9700 const AAMDNodes &AAInfo, const MDNode *Ranges) {9701 assert(Chain.getValueType() == MVT::Other &&9702 "Invalid chain type");9703 9704 MMOFlags |= MachineMemOperand::MOLoad;9705 assert((MMOFlags & MachineMemOperand::MOStore) == 0);9706 // If we don't have a PtrInfo, infer the trivial frame index case to simplify9707 // clients.9708 if (PtrInfo.V.isNull())9709 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);9710 9711 TypeSize Size = MemVT.getStoreSize();9712 MachineFunction &MF = getMachineFunction();9713 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,9714 Alignment, AAInfo, Ranges);9715 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);9716}9717 9718SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,9719 EVT VT, const SDLoc &dl, SDValue Chain,9720 SDValue Ptr, SDValue Offset, EVT MemVT,9721 MachineMemOperand *MMO) {9722 if (VT == MemVT) {9723 ExtType = ISD::NON_EXTLOAD;9724 } else if (ExtType == ISD::NON_EXTLOAD) {9725 assert(VT == MemVT && "Non-extending load from different memory type!");9726 } else {9727 // Extending load.9728 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&9729 "Should only be an extending load, not truncating!");9730 assert(VT.isInteger() == MemVT.isInteger() &&9731 "Cannot convert from FP to Int or Int -> FP!");9732 assert(VT.isVector() == MemVT.isVector() &&9733 "Cannot use an ext load to convert to or from a vector!");9734 assert((!VT.isVector() ||9735 VT.getVectorElementCount() == MemVT.getVectorElementCount()) &&9736 "Cannot use an ext load to change the number of vector elements!");9737 }9738 9739 assert((!MMO->getRanges() ||9740 (mdconst::extract<ConstantInt>(MMO->getRanges()->getOperand(0))9741 ->getBitWidth() == MemVT.getScalarSizeInBits() &&9742 MemVT.isInteger())) &&9743 "Range metadata and load type must match!");9744 9745 bool Indexed = AM != ISD::UNINDEXED;9746 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");9747 9748 SDVTList VTs = Indexed ?9749 getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);9750 SDValue Ops[] = { Chain, Ptr, Offset };9751 FoldingSetNodeID ID;9752 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);9753 ID.AddInteger(MemVT.getRawBits());9754 ID.AddInteger(getSyntheticNodeSubclassData<LoadSDNode>(9755 dl.getIROrder(), VTs, AM, ExtType, MemVT, MMO));9756 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());9757 ID.AddInteger(MMO->getFlags());9758 void *IP = nullptr;9759 if (auto *E = cast_or_null<LoadSDNode>(FindNodeOrInsertPos(ID, dl, IP))) {9760 E->refineAlignment(MMO);9761 E->refineRanges(MMO);9762 return SDValue(E, 0);9763 }9764 auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,9765 ExtType, MemVT, MMO);9766 createOperands(N, Ops);9767 9768 CSEMap.InsertNode(N, IP);9769 InsertNode(N);9770 SDValue V(N, 0);9771 NewSDValueDbgMsg(V, "Creating new node: ", this);9772 return V;9773}9774 9775SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,9776 SDValue Ptr, MachinePointerInfo PtrInfo,9777 MaybeAlign Alignment,9778 MachineMemOperand::Flags MMOFlags,9779 const AAMDNodes &AAInfo, const MDNode *Ranges) {9780 SDValue Undef = getUNDEF(Ptr.getValueType());9781 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,9782 PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges);9783}9784 9785SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,9786 SDValue Ptr, MachineMemOperand *MMO) {9787 SDValue Undef = getUNDEF(Ptr.getValueType());9788 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,9789 VT, MMO);9790}9791 9792SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,9793 EVT VT, SDValue Chain, SDValue Ptr,9794 MachinePointerInfo PtrInfo, EVT MemVT,9795 MaybeAlign Alignment,9796 MachineMemOperand::Flags MMOFlags,9797 const AAMDNodes &AAInfo) {9798 SDValue Undef = getUNDEF(Ptr.getValueType());9799 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo,9800 MemVT, Alignment, MMOFlags, AAInfo);9801}9802 9803SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,9804 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,9805 MachineMemOperand *MMO) {9806 SDValue Undef = getUNDEF(Ptr.getValueType());9807 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,9808 MemVT, MMO);9809}9810 9811SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,9812 SDValue Base, SDValue Offset,9813 ISD::MemIndexedMode AM) {9814 LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);9815 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");9816 // Don't propagate the invariant or dereferenceable flags.9817 auto MMOFlags =9818 LD->getMemOperand()->getFlags() &9819 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);9820 return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,9821 LD->getChain(), Base, Offset, LD->getPointerInfo(),9822 LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo());9823}9824 9825SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,9826 SDValue Ptr, MachinePointerInfo PtrInfo,9827 Align Alignment,9828 MachineMemOperand::Flags MMOFlags,9829 const AAMDNodes &AAInfo) {9830 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");9831 9832 MMOFlags |= MachineMemOperand::MOStore;9833 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);9834 9835 if (PtrInfo.V.isNull())9836 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);9837 9838 MachineFunction &MF = getMachineFunction();9839 TypeSize Size = Val.getValueType().getStoreSize();9840 MachineMemOperand *MMO =9841 MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo);9842 return getStore(Chain, dl, Val, Ptr, MMO);9843}9844 9845SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,9846 SDValue Ptr, MachineMemOperand *MMO) {9847 SDValue Undef = getUNDEF(Ptr.getValueType());9848 return getStore(Chain, dl, Val, Ptr, Undef, Val.getValueType(), MMO,9849 ISD::UNINDEXED);9850}9851 9852SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,9853 SDValue Ptr, SDValue Offset, EVT SVT,9854 MachineMemOperand *MMO, ISD::MemIndexedMode AM,9855 bool IsTruncating) {9856 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");9857 EVT VT = Val.getValueType();9858 if (VT == SVT) {9859 IsTruncating = false;9860 } else if (!IsTruncating) {9861 assert(VT == SVT && "No-truncating store from different memory type!");9862 } else {9863 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&9864 "Should only be a truncating store, not extending!");9865 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");9866 assert(VT.isVector() == SVT.isVector() &&9867 "Cannot use trunc store to convert to or from a vector!");9868 assert((!VT.isVector() ||9869 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&9870 "Cannot use trunc store to change the number of vector elements!");9871 }9872 9873 bool Indexed = AM != ISD::UNINDEXED;9874 assert((Indexed || Offset.isUndef()) && "Unindexed store with an offset!");9875 SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other)9876 : getVTList(MVT::Other);9877 SDValue Ops[] = {Chain, Val, Ptr, Offset};9878 FoldingSetNodeID ID;9879 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);9880 ID.AddInteger(SVT.getRawBits());9881 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(9882 dl.getIROrder(), VTs, AM, IsTruncating, SVT, MMO));9883 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());9884 ID.AddInteger(MMO->getFlags());9885 void *IP = nullptr;9886 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {9887 cast<StoreSDNode>(E)->refineAlignment(MMO);9888 return SDValue(E, 0);9889 }9890 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,9891 IsTruncating, SVT, MMO);9892 createOperands(N, Ops);9893 9894 CSEMap.InsertNode(N, IP);9895 InsertNode(N);9896 SDValue V(N, 0);9897 NewSDValueDbgMsg(V, "Creating new node: ", this);9898 return V;9899}9900 9901SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,9902 SDValue Ptr, MachinePointerInfo PtrInfo,9903 EVT SVT, Align Alignment,9904 MachineMemOperand::Flags MMOFlags,9905 const AAMDNodes &AAInfo) {9906 assert(Chain.getValueType() == MVT::Other &&9907 "Invalid chain type");9908 9909 MMOFlags |= MachineMemOperand::MOStore;9910 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);9911 9912 if (PtrInfo.V.isNull())9913 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);9914 9915 MachineFunction &MF = getMachineFunction();9916 MachineMemOperand *MMO = MF.getMachineMemOperand(9917 PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo);9918 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);9919}9920 9921SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,9922 SDValue Ptr, EVT SVT,9923 MachineMemOperand *MMO) {9924 SDValue Undef = getUNDEF(Ptr.getValueType());9925 return getStore(Chain, dl, Val, Ptr, Undef, SVT, MMO, ISD::UNINDEXED, true);9926}9927 9928SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,9929 SDValue Base, SDValue Offset,9930 ISD::MemIndexedMode AM) {9931 StoreSDNode *ST = cast<StoreSDNode>(OrigStore);9932 assert(ST->getOffset().isUndef() && "Store is already a indexed store!");9933 return getStore(ST->getChain(), dl, ST->getValue(), Base, Offset,9934 ST->getMemoryVT(), ST->getMemOperand(), AM,9935 ST->isTruncatingStore());9936}9937 9938SDValue SelectionDAG::getLoadVP(9939 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,9940 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL,9941 MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,9942 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,9943 const MDNode *Ranges, bool IsExpanding) {9944 MMOFlags |= MachineMemOperand::MOLoad;9945 assert((MMOFlags & MachineMemOperand::MOStore) == 0);9946 // If we don't have a PtrInfo, infer the trivial frame index case to simplify9947 // clients.9948 if (PtrInfo.V.isNull())9949 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);9950 9951 TypeSize Size = MemVT.getStoreSize();9952 MachineFunction &MF = getMachineFunction();9953 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,9954 Alignment, AAInfo, Ranges);9955 return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,9956 MMO, IsExpanding);9957}9958 9959SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM,9960 ISD::LoadExtType ExtType, EVT VT,9961 const SDLoc &dl, SDValue Chain, SDValue Ptr,9962 SDValue Offset, SDValue Mask, SDValue EVL,9963 EVT MemVT, MachineMemOperand *MMO,9964 bool IsExpanding) {9965 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");9966 assert(Mask.getValueType().getVectorElementCount() ==9967 VT.getVectorElementCount() &&9968 "Vector width mismatch between mask and data");9969 9970 bool Indexed = AM != ISD::UNINDEXED;9971 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");9972 9973 SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other)9974 : getVTList(VT, MVT::Other);9975 SDValue Ops[] = {Chain, Ptr, Offset, Mask, EVL};9976 FoldingSetNodeID ID;9977 AddNodeIDNode(ID, ISD::VP_LOAD, VTs, Ops);9978 ID.AddInteger(MemVT.getRawBits());9979 ID.AddInteger(getSyntheticNodeSubclassData<VPLoadSDNode>(9980 dl.getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO));9981 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());9982 ID.AddInteger(MMO->getFlags());9983 void *IP = nullptr;9984 if (auto *E = cast_or_null<VPLoadSDNode>(FindNodeOrInsertPos(ID, dl, IP))) {9985 E->refineAlignment(MMO);9986 E->refineRanges(MMO);9987 return SDValue(E, 0);9988 }9989 auto *N = newSDNode<VPLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,9990 ExtType, IsExpanding, MemVT, MMO);9991 createOperands(N, Ops);9992 9993 CSEMap.InsertNode(N, IP);9994 InsertNode(N);9995 SDValue V(N, 0);9996 NewSDValueDbgMsg(V, "Creating new node: ", this);9997 return V;9998}9999 10000SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,10001 SDValue Ptr, SDValue Mask, SDValue EVL,10002 MachinePointerInfo PtrInfo,10003 MaybeAlign Alignment,10004 MachineMemOperand::Flags MMOFlags,10005 const AAMDNodes &AAInfo, const MDNode *Ranges,10006 bool IsExpanding) {10007 SDValue Undef = getUNDEF(Ptr.getValueType());10008 return getLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,10009 Mask, EVL, PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges,10010 IsExpanding);10011}10012 10013SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,10014 SDValue Ptr, SDValue Mask, SDValue EVL,10015 MachineMemOperand *MMO, bool IsExpanding) {10016 SDValue Undef = getUNDEF(Ptr.getValueType());10017 return getLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,10018 Mask, EVL, VT, MMO, IsExpanding);10019}10020 10021SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,10022 EVT VT, SDValue Chain, SDValue Ptr,10023 SDValue Mask, SDValue EVL,10024 MachinePointerInfo PtrInfo, EVT MemVT,10025 MaybeAlign Alignment,10026 MachineMemOperand::Flags MMOFlags,10027 const AAMDNodes &AAInfo, bool IsExpanding) {10028 SDValue Undef = getUNDEF(Ptr.getValueType());10029 return getLoadVP(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, Mask,10030 EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo, nullptr,10031 IsExpanding);10032}10033 10034SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,10035 EVT VT, SDValue Chain, SDValue Ptr,10036 SDValue Mask, SDValue EVL, EVT MemVT,10037 MachineMemOperand *MMO, bool IsExpanding) {10038 SDValue Undef = getUNDEF(Ptr.getValueType());10039 return getLoadVP(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, Mask,10040 EVL, MemVT, MMO, IsExpanding);10041}10042 10043SDValue SelectionDAG::getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl,10044 SDValue Base, SDValue Offset,10045 ISD::MemIndexedMode AM) {10046 auto *LD = cast<VPLoadSDNode>(OrigLoad);10047 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");10048 // Don't propagate the invariant or dereferenceable flags.10049 auto MMOFlags =10050 LD->getMemOperand()->getFlags() &10051 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);10052 return getLoadVP(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,10053 LD->getChain(), Base, Offset, LD->getMask(),10054 LD->getVectorLength(), LD->getPointerInfo(),10055 LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo(),10056 nullptr, LD->isExpandingLoad());10057}10058 10059SDValue SelectionDAG::getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val,10060 SDValue Ptr, SDValue Offset, SDValue Mask,10061 SDValue EVL, EVT MemVT, MachineMemOperand *MMO,10062 ISD::MemIndexedMode AM, bool IsTruncating,10063 bool IsCompressing) {10064 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");10065 assert(Mask.getValueType().getVectorElementCount() ==10066 Val.getValueType().getVectorElementCount() &&10067 "Vector width mismatch between mask and data");10068 10069 bool Indexed = AM != ISD::UNINDEXED;10070 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");10071 SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other)10072 : getVTList(MVT::Other);10073 SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL};10074 FoldingSetNodeID ID;10075 AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops);10076 ID.AddInteger(MemVT.getRawBits());10077 ID.AddInteger(getSyntheticNodeSubclassData<VPStoreSDNode>(10078 dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));10079 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());10080 ID.AddInteger(MMO->getFlags());10081 void *IP = nullptr;10082 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {10083 cast<VPStoreSDNode>(E)->refineAlignment(MMO);10084 return SDValue(E, 0);10085 }10086 auto *N = newSDNode<VPStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,10087 IsTruncating, IsCompressing, MemVT, MMO);10088 createOperands(N, Ops);10089 10090 CSEMap.InsertNode(N, IP);10091 InsertNode(N);10092 SDValue V(N, 0);10093 NewSDValueDbgMsg(V, "Creating new node: ", this);10094 return V;10095}10096 10097SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,10098 SDValue Val, SDValue Ptr, SDValue Mask,10099 SDValue EVL, MachinePointerInfo PtrInfo,10100 EVT SVT, Align Alignment,10101 MachineMemOperand::Flags MMOFlags,10102 const AAMDNodes &AAInfo,10103 bool IsCompressing) {10104 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");10105 10106 MMOFlags |= MachineMemOperand::MOStore;10107 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);10108 10109 if (PtrInfo.V.isNull())10110 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);10111 10112 MachineFunction &MF = getMachineFunction();10113 MachineMemOperand *MMO = MF.getMachineMemOperand(10114 PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo);10115 return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,10116 IsCompressing);10117}10118 10119SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,10120 SDValue Val, SDValue Ptr, SDValue Mask,10121 SDValue EVL, EVT SVT,10122 MachineMemOperand *MMO,10123 bool IsCompressing) {10124 EVT VT = Val.getValueType();10125 10126 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");10127 if (VT == SVT)10128 return getStoreVP(Chain, dl, Val, Ptr, getUNDEF(Ptr.getValueType()), Mask,10129 EVL, VT, MMO, ISD::UNINDEXED,10130 /*IsTruncating*/ false, IsCompressing);10131 10132 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&10133 "Should only be a truncating store, not extending!");10134 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");10135 assert(VT.isVector() == SVT.isVector() &&10136 "Cannot use trunc store to convert to or from a vector!");10137 assert((!VT.isVector() ||10138 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&10139 "Cannot use trunc store to change the number of vector elements!");10140 10141 SDVTList VTs = getVTList(MVT::Other);10142 SDValue Undef = getUNDEF(Ptr.getValueType());10143 SDValue Ops[] = {Chain, Val, Ptr, Undef, Mask, EVL};10144 FoldingSetNodeID ID;10145 AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops);10146 ID.AddInteger(SVT.getRawBits());10147 ID.AddInteger(getSyntheticNodeSubclassData<VPStoreSDNode>(10148 dl.getIROrder(), VTs, ISD::UNINDEXED, true, IsCompressing, SVT, MMO));10149 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());10150 ID.AddInteger(MMO->getFlags());10151 void *IP = nullptr;10152 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {10153 cast<VPStoreSDNode>(E)->refineAlignment(MMO);10154 return SDValue(E, 0);10155 }10156 auto *N =10157 newSDNode<VPStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,10158 ISD::UNINDEXED, true, IsCompressing, SVT, MMO);10159 createOperands(N, Ops);10160 10161 CSEMap.InsertNode(N, IP);10162 InsertNode(N);10163 SDValue V(N, 0);10164 NewSDValueDbgMsg(V, "Creating new node: ", this);10165 return V;10166}10167 10168SDValue SelectionDAG::getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl,10169 SDValue Base, SDValue Offset,10170 ISD::MemIndexedMode AM) {10171 auto *ST = cast<VPStoreSDNode>(OrigStore);10172 assert(ST->getOffset().isUndef() && "Store is already an indexed store!");10173 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);10174 SDValue Ops[] = {ST->getChain(), ST->getValue(), Base,10175 Offset, ST->getMask(), ST->getVectorLength()};10176 FoldingSetNodeID ID;10177 AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops);10178 ID.AddInteger(ST->getMemoryVT().getRawBits());10179 ID.AddInteger(ST->getRawSubclassData());10180 ID.AddInteger(ST->getPointerInfo().getAddrSpace());10181 ID.AddInteger(ST->getMemOperand()->getFlags());10182 void *IP = nullptr;10183 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))10184 return SDValue(E, 0);10185 10186 auto *N = newSDNode<VPStoreSDNode>(10187 dl.getIROrder(), dl.getDebugLoc(), VTs, AM, ST->isTruncatingStore(),10188 ST->isCompressingStore(), ST->getMemoryVT(), ST->getMemOperand());10189 createOperands(N, Ops);10190 10191 CSEMap.InsertNode(N, IP);10192 InsertNode(N);10193 SDValue V(N, 0);10194 NewSDValueDbgMsg(V, "Creating new node: ", this);10195 return V;10196}10197 10198SDValue SelectionDAG::getStridedLoadVP(10199 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,10200 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,10201 SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) {10202 bool Indexed = AM != ISD::UNINDEXED;10203 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");10204 10205 SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL};10206 SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other)10207 : getVTList(VT, MVT::Other);10208 FoldingSetNodeID ID;10209 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTs, Ops);10210 ID.AddInteger(VT.getRawBits());10211 ID.AddInteger(getSyntheticNodeSubclassData<VPStridedLoadSDNode>(10212 DL.getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO));10213 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());10214 10215 void *IP = nullptr;10216 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {10217 cast<VPStridedLoadSDNode>(E)->refineAlignment(MMO);10218 return SDValue(E, 0);10219 }10220 10221 auto *N =10222 newSDNode<VPStridedLoadSDNode>(DL.getIROrder(), DL.getDebugLoc(), VTs, AM,10223 ExtType, IsExpanding, MemVT, MMO);10224 createOperands(N, Ops);10225 CSEMap.InsertNode(N, IP);10226 InsertNode(N);10227 SDValue V(N, 0);10228 NewSDValueDbgMsg(V, "Creating new node: ", this);10229 return V;10230}10231 10232SDValue SelectionDAG::getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain,10233 SDValue Ptr, SDValue Stride,10234 SDValue Mask, SDValue EVL,10235 MachineMemOperand *MMO,10236 bool IsExpanding) {10237 SDValue Undef = getUNDEF(Ptr.getValueType());10238 return getStridedLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, DL, Chain, Ptr,10239 Undef, Stride, Mask, EVL, VT, MMO, IsExpanding);10240}10241 10242SDValue SelectionDAG::getExtStridedLoadVP(10243 ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,10244 SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT,10245 MachineMemOperand *MMO, bool IsExpanding) {10246 SDValue Undef = getUNDEF(Ptr.getValueType());10247 return getStridedLoadVP(ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Undef,10248 Stride, Mask, EVL, MemVT, MMO, IsExpanding);10249}10250 10251SDValue SelectionDAG::getStridedStoreVP(SDValue Chain, const SDLoc &DL,10252 SDValue Val, SDValue Ptr,10253 SDValue Offset, SDValue Stride,10254 SDValue Mask, SDValue EVL, EVT MemVT,10255 MachineMemOperand *MMO,10256 ISD::MemIndexedMode AM,10257 bool IsTruncating, bool IsCompressing) {10258 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");10259 bool Indexed = AM != ISD::UNINDEXED;10260 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");10261 SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other)10262 : getVTList(MVT::Other);10263 SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL};10264 FoldingSetNodeID ID;10265 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops);10266 ID.AddInteger(MemVT.getRawBits());10267 ID.AddInteger(getSyntheticNodeSubclassData<VPStridedStoreSDNode>(10268 DL.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));10269 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());10270 void *IP = nullptr;10271 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {10272 cast<VPStridedStoreSDNode>(E)->refineAlignment(MMO);10273 return SDValue(E, 0);10274 }10275 auto *N = newSDNode<VPStridedStoreSDNode>(DL.getIROrder(), DL.getDebugLoc(),10276 VTs, AM, IsTruncating,10277 IsCompressing, MemVT, MMO);10278 createOperands(N, Ops);10279 10280 CSEMap.InsertNode(N, IP);10281 InsertNode(N);10282 SDValue V(N, 0);10283 NewSDValueDbgMsg(V, "Creating new node: ", this);10284 return V;10285}10286 10287SDValue SelectionDAG::getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL,10288 SDValue Val, SDValue Ptr,10289 SDValue Stride, SDValue Mask,10290 SDValue EVL, EVT SVT,10291 MachineMemOperand *MMO,10292 bool IsCompressing) {10293 EVT VT = Val.getValueType();10294 10295 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");10296 if (VT == SVT)10297 return getStridedStoreVP(Chain, DL, Val, Ptr, getUNDEF(Ptr.getValueType()),10298 Stride, Mask, EVL, VT, MMO, ISD::UNINDEXED,10299 /*IsTruncating*/ false, IsCompressing);10300 10301 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&10302 "Should only be a truncating store, not extending!");10303 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");10304 assert(VT.isVector() == SVT.isVector() &&10305 "Cannot use trunc store to convert to or from a vector!");10306 assert((!VT.isVector() ||10307 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&10308 "Cannot use trunc store to change the number of vector elements!");10309 10310 SDVTList VTs = getVTList(MVT::Other);10311 SDValue Undef = getUNDEF(Ptr.getValueType());10312 SDValue Ops[] = {Chain, Val, Ptr, Undef, Stride, Mask, EVL};10313 FoldingSetNodeID ID;10314 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops);10315 ID.AddInteger(SVT.getRawBits());10316 ID.AddInteger(getSyntheticNodeSubclassData<VPStridedStoreSDNode>(10317 DL.getIROrder(), VTs, ISD::UNINDEXED, true, IsCompressing, SVT, MMO));10318 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());10319 void *IP = nullptr;10320 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {10321 cast<VPStridedStoreSDNode>(E)->refineAlignment(MMO);10322 return SDValue(E, 0);10323 }10324 auto *N = newSDNode<VPStridedStoreSDNode>(DL.getIROrder(), DL.getDebugLoc(),10325 VTs, ISD::UNINDEXED, true,10326 IsCompressing, SVT, MMO);10327 createOperands(N, Ops);10328 10329 CSEMap.InsertNode(N, IP);10330 InsertNode(N);10331 SDValue V(N, 0);10332 NewSDValueDbgMsg(V, "Creating new node: ", this);10333 return V;10334}10335 10336SDValue SelectionDAG::getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl,10337 ArrayRef<SDValue> Ops, MachineMemOperand *MMO,10338 ISD::MemIndexType IndexType) {10339 assert(Ops.size() == 6 && "Incompatible number of operands");10340 10341 FoldingSetNodeID ID;10342 AddNodeIDNode(ID, ISD::VP_GATHER, VTs, Ops);10343 ID.AddInteger(VT.getRawBits());10344 ID.AddInteger(getSyntheticNodeSubclassData<VPGatherSDNode>(10345 dl.getIROrder(), VTs, VT, MMO, IndexType));10346 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());10347 ID.AddInteger(MMO->getFlags());10348 void *IP = nullptr;10349 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {10350 cast<VPGatherSDNode>(E)->refineAlignment(MMO);10351 return SDValue(E, 0);10352 }10353 10354 auto *N = newSDNode<VPGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,10355 VT, MMO, IndexType);10356 createOperands(N, Ops);10357 10358 assert(N->getMask().getValueType().getVectorElementCount() ==10359 N->getValueType(0).getVectorElementCount() &&10360 "Vector width mismatch between mask and data");10361 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==10362 N->getValueType(0).getVectorElementCount().isScalable() &&10363 "Scalable flags of index and data do not match");10364 assert(ElementCount::isKnownGE(10365 N->getIndex().getValueType().getVectorElementCount(),10366 N->getValueType(0).getVectorElementCount()) &&10367 "Vector width mismatch between index and data");10368 assert(isa<ConstantSDNode>(N->getScale()) &&10369 N->getScale()->getAsAPIntVal().isPowerOf2() &&10370 "Scale should be a constant power of 2");10371 10372 CSEMap.InsertNode(N, IP);10373 InsertNode(N);10374 SDValue V(N, 0);10375 NewSDValueDbgMsg(V, "Creating new node: ", this);10376 return V;10377}10378 10379SDValue SelectionDAG::getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl,10380 ArrayRef<SDValue> Ops,10381 MachineMemOperand *MMO,10382 ISD::MemIndexType IndexType) {10383 assert(Ops.size() == 7 && "Incompatible number of operands");10384 10385 FoldingSetNodeID ID;10386 AddNodeIDNode(ID, ISD::VP_SCATTER, VTs, Ops);10387 ID.AddInteger(VT.getRawBits());10388 ID.AddInteger(getSyntheticNodeSubclassData<VPScatterSDNode>(10389 dl.getIROrder(), VTs, VT, MMO, IndexType));10390 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());10391 ID.AddInteger(MMO->getFlags());10392 void *IP = nullptr;10393 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {10394 cast<VPScatterSDNode>(E)->refineAlignment(MMO);10395 return SDValue(E, 0);10396 }10397 auto *N = newSDNode<VPScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,10398 VT, MMO, IndexType);10399 createOperands(N, Ops);10400 10401 assert(N->getMask().getValueType().getVectorElementCount() ==10402 N->getValue().getValueType().getVectorElementCount() &&10403 "Vector width mismatch between mask and data");10404 assert(10405 N->getIndex().getValueType().getVectorElementCount().isScalable() ==10406 N->getValue().getValueType().getVectorElementCount().isScalable() &&10407 "Scalable flags of index and data do not match");10408 assert(ElementCount::isKnownGE(10409 N->getIndex().getValueType().getVectorElementCount(),10410 N->getValue().getValueType().getVectorElementCount()) &&10411 "Vector width mismatch between index and data");10412 assert(isa<ConstantSDNode>(N->getScale()) &&10413 N->getScale()->getAsAPIntVal().isPowerOf2() &&10414 "Scale should be a constant power of 2");10415 10416 CSEMap.InsertNode(N, IP);10417 InsertNode(N);10418 SDValue V(N, 0);10419 NewSDValueDbgMsg(V, "Creating new node: ", this);10420 return V;10421}10422 10423SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,10424 SDValue Base, SDValue Offset, SDValue Mask,10425 SDValue PassThru, EVT MemVT,10426 MachineMemOperand *MMO,10427 ISD::MemIndexedMode AM,10428 ISD::LoadExtType ExtTy, bool isExpanding) {10429 bool Indexed = AM != ISD::UNINDEXED;10430 assert((Indexed || Offset.isUndef()) &&10431 "Unindexed masked load with an offset!");10432 SDVTList VTs = Indexed ? getVTList(VT, Base.getValueType(), MVT::Other)10433 : getVTList(VT, MVT::Other);10434 SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru};10435 FoldingSetNodeID ID;10436 AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops);10437 ID.AddInteger(MemVT.getRawBits());10438 ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>(10439 dl.getIROrder(), VTs, AM, ExtTy, isExpanding, MemVT, MMO));10440 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());10441 ID.AddInteger(MMO->getFlags());10442 void *IP = nullptr;10443 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {10444 cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);10445 return SDValue(E, 0);10446 }10447 auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,10448 AM, ExtTy, isExpanding, MemVT, MMO);10449 createOperands(N, Ops);10450 10451 CSEMap.InsertNode(N, IP);10452 InsertNode(N);10453 SDValue V(N, 0);10454 NewSDValueDbgMsg(V, "Creating new node: ", this);10455 return V;10456}10457 10458SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl,10459 SDValue Base, SDValue Offset,10460 ISD::MemIndexedMode AM) {10461 MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(OrigLoad);10462 assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!");10463 return getMaskedLoad(OrigLoad.getValueType(), dl, LD->getChain(), Base,10464 Offset, LD->getMask(), LD->getPassThru(),10465 LD->getMemoryVT(), LD->getMemOperand(), AM,10466 LD->getExtensionType(), LD->isExpandingLoad());10467}10468 10469SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,10470 SDValue Val, SDValue Base, SDValue Offset,10471 SDValue Mask, EVT MemVT,10472 MachineMemOperand *MMO,10473 ISD::MemIndexedMode AM, bool IsTruncating,10474 bool IsCompressing) {10475 assert(Chain.getValueType() == MVT::Other &&10476 "Invalid chain type");10477 bool Indexed = AM != ISD::UNINDEXED;10478 assert((Indexed || Offset.isUndef()) &&10479 "Unindexed masked store with an offset!");10480 SDVTList VTs = Indexed ? getVTList(Base.getValueType(), MVT::Other)10481 : getVTList(MVT::Other);10482 SDValue Ops[] = {Chain, Val, Base, Offset, Mask};10483 FoldingSetNodeID ID;10484 AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops);10485 ID.AddInteger(MemVT.getRawBits());10486 ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>(10487 dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));10488 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());10489 ID.AddInteger(MMO->getFlags());10490 void *IP = nullptr;10491 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {10492 cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);10493 return SDValue(E, 0);10494 }10495 auto *N =10496 newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,10497 IsTruncating, IsCompressing, MemVT, MMO);10498 createOperands(N, Ops);10499 10500 CSEMap.InsertNode(N, IP);10501 InsertNode(N);10502 SDValue V(N, 0);10503 NewSDValueDbgMsg(V, "Creating new node: ", this);10504 return V;10505}10506 10507SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,10508 SDValue Base, SDValue Offset,10509 ISD::MemIndexedMode AM) {10510 MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(OrigStore);10511 assert(ST->getOffset().isUndef() &&10512 "Masked store is already a indexed store!");10513 return getMaskedStore(ST->getChain(), dl, ST->getValue(), Base, Offset,10514 ST->getMask(), ST->getMemoryVT(), ST->getMemOperand(),10515 AM, ST->isTruncatingStore(), ST->isCompressingStore());10516}10517 10518SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl,10519 ArrayRef<SDValue> Ops,10520 MachineMemOperand *MMO,10521 ISD::MemIndexType IndexType,10522 ISD::LoadExtType ExtTy) {10523 assert(Ops.size() == 6 && "Incompatible number of operands");10524 10525 FoldingSetNodeID ID;10526 AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);10527 ID.AddInteger(MemVT.getRawBits());10528 ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>(10529 dl.getIROrder(), VTs, MemVT, MMO, IndexType, ExtTy));10530 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());10531 ID.AddInteger(MMO->getFlags());10532 void *IP = nullptr;10533 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {10534 cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);10535 return SDValue(E, 0);10536 }10537 10538 auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(),10539 VTs, MemVT, MMO, IndexType, ExtTy);10540 createOperands(N, Ops);10541 10542 assert(N->getPassThru().getValueType() == N->getValueType(0) &&10543 "Incompatible type of the PassThru value in MaskedGatherSDNode");10544 assert(N->getMask().getValueType().getVectorElementCount() ==10545 N->getValueType(0).getVectorElementCount() &&10546 "Vector width mismatch between mask and data");10547 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==10548 N->getValueType(0).getVectorElementCount().isScalable() &&10549 "Scalable flags of index and data do not match");10550 assert(ElementCount::isKnownGE(10551 N->getIndex().getValueType().getVectorElementCount(),10552 N->getValueType(0).getVectorElementCount()) &&10553 "Vector width mismatch between index and data");10554 assert(isa<ConstantSDNode>(N->getScale()) &&10555 N->getScale()->getAsAPIntVal().isPowerOf2() &&10556 "Scale should be a constant power of 2");10557 10558 CSEMap.InsertNode(N, IP);10559 InsertNode(N);10560 SDValue V(N, 0);10561 NewSDValueDbgMsg(V, "Creating new node: ", this);10562 return V;10563}10564 10565SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,10566 ArrayRef<SDValue> Ops,10567 MachineMemOperand *MMO,10568 ISD::MemIndexType IndexType,10569 bool IsTrunc) {10570 assert(Ops.size() == 6 && "Incompatible number of operands");10571 10572 FoldingSetNodeID ID;10573 AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops);10574 ID.AddInteger(MemVT.getRawBits());10575 ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>(10576 dl.getIROrder(), VTs, MemVT, MMO, IndexType, IsTrunc));10577 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());10578 ID.AddInteger(MMO->getFlags());10579 void *IP = nullptr;10580 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {10581 cast<MaskedScatterSDNode>(E)->refineAlignment(MMO);10582 return SDValue(E, 0);10583 }10584 10585 auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(),10586 VTs, MemVT, MMO, IndexType, IsTrunc);10587 createOperands(N, Ops);10588 10589 assert(N->getMask().getValueType().getVectorElementCount() ==10590 N->getValue().getValueType().getVectorElementCount() &&10591 "Vector width mismatch between mask and data");10592 assert(10593 N->getIndex().getValueType().getVectorElementCount().isScalable() ==10594 N->getValue().getValueType().getVectorElementCount().isScalable() &&10595 "Scalable flags of index and data do not match");10596 assert(ElementCount::isKnownGE(10597 N->getIndex().getValueType().getVectorElementCount(),10598 N->getValue().getValueType().getVectorElementCount()) &&10599 "Vector width mismatch between index and data");10600 assert(isa<ConstantSDNode>(N->getScale()) &&10601 N->getScale()->getAsAPIntVal().isPowerOf2() &&10602 "Scale should be a constant power of 2");10603 10604 CSEMap.InsertNode(N, IP);10605 InsertNode(N);10606 SDValue V(N, 0);10607 NewSDValueDbgMsg(V, "Creating new node: ", this);10608 return V;10609}10610 10611SDValue SelectionDAG::getMaskedHistogram(SDVTList VTs, EVT MemVT,10612 const SDLoc &dl, ArrayRef<SDValue> Ops,10613 MachineMemOperand *MMO,10614 ISD::MemIndexType IndexType) {10615 assert(Ops.size() == 7 && "Incompatible number of operands");10616 10617 FoldingSetNodeID ID;10618 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VECTOR_HISTOGRAM, VTs, Ops);10619 ID.AddInteger(MemVT.getRawBits());10620 ID.AddInteger(getSyntheticNodeSubclassData<MaskedHistogramSDNode>(10621 dl.getIROrder(), VTs, MemVT, MMO, IndexType));10622 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());10623 ID.AddInteger(MMO->getFlags());10624 void *IP = nullptr;10625 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {10626 cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);10627 return SDValue(E, 0);10628 }10629 10630 auto *N = newSDNode<MaskedHistogramSDNode>(dl.getIROrder(), dl.getDebugLoc(),10631 VTs, MemVT, MMO, IndexType);10632 createOperands(N, Ops);10633 10634 assert(N->getMask().getValueType().getVectorElementCount() ==10635 N->getIndex().getValueType().getVectorElementCount() &&10636 "Vector width mismatch between mask and data");10637 assert(isa<ConstantSDNode>(N->getScale()) &&10638 N->getScale()->getAsAPIntVal().isPowerOf2() &&10639 "Scale should be a constant power of 2");10640 assert(N->getInc().getValueType().isInteger() && "Non integer update value");10641 10642 CSEMap.InsertNode(N, IP);10643 InsertNode(N);10644 SDValue V(N, 0);10645 NewSDValueDbgMsg(V, "Creating new node: ", this);10646 return V;10647}10648 10649SDValue SelectionDAG::getLoadFFVP(EVT VT, const SDLoc &DL, SDValue Chain,10650 SDValue Ptr, SDValue Mask, SDValue EVL,10651 MachineMemOperand *MMO) {10652 SDVTList VTs = getVTList(VT, EVL.getValueType(), MVT::Other);10653 SDValue Ops[] = {Chain, Ptr, Mask, EVL};10654 FoldingSetNodeID ID;10655 AddNodeIDNode(ID, ISD::VP_LOAD_FF, VTs, Ops);10656 ID.AddInteger(VT.getRawBits());10657 ID.AddInteger(getSyntheticNodeSubclassData<VPLoadFFSDNode>(DL.getIROrder(),10658 VTs, VT, MMO));10659 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());10660 ID.AddInteger(MMO->getFlags());10661 void *IP = nullptr;10662 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {10663 cast<VPLoadFFSDNode>(E)->refineAlignment(MMO);10664 return SDValue(E, 0);10665 }10666 auto *N = newSDNode<VPLoadFFSDNode>(DL.getIROrder(), DL.getDebugLoc(), VTs,10667 VT, MMO);10668 createOperands(N, Ops);10669 10670 CSEMap.InsertNode(N, IP);10671 InsertNode(N);10672 SDValue V(N, 0);10673 NewSDValueDbgMsg(V, "Creating new node: ", this);10674 return V;10675}10676 10677SDValue SelectionDAG::getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,10678 EVT MemVT, MachineMemOperand *MMO) {10679 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");10680 SDVTList VTs = getVTList(MVT::Other);10681 SDValue Ops[] = {Chain, Ptr};10682 FoldingSetNodeID ID;10683 AddNodeIDNode(ID, ISD::GET_FPENV_MEM, VTs, Ops);10684 ID.AddInteger(MemVT.getRawBits());10685 ID.AddInteger(getSyntheticNodeSubclassData<FPStateAccessSDNode>(10686 ISD::GET_FPENV_MEM, dl.getIROrder(), VTs, MemVT, MMO));10687 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());10688 ID.AddInteger(MMO->getFlags());10689 void *IP = nullptr;10690 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))10691 return SDValue(E, 0);10692 10693 auto *N = newSDNode<FPStateAccessSDNode>(ISD::GET_FPENV_MEM, dl.getIROrder(),10694 dl.getDebugLoc(), VTs, MemVT, MMO);10695 createOperands(N, Ops);10696 10697 CSEMap.InsertNode(N, IP);10698 InsertNode(N);10699 SDValue V(N, 0);10700 NewSDValueDbgMsg(V, "Creating new node: ", this);10701 return V;10702}10703 10704SDValue SelectionDAG::getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,10705 EVT MemVT, MachineMemOperand *MMO) {10706 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");10707 SDVTList VTs = getVTList(MVT::Other);10708 SDValue Ops[] = {Chain, Ptr};10709 FoldingSetNodeID ID;10710 AddNodeIDNode(ID, ISD::SET_FPENV_MEM, VTs, Ops);10711 ID.AddInteger(MemVT.getRawBits());10712 ID.AddInteger(getSyntheticNodeSubclassData<FPStateAccessSDNode>(10713 ISD::SET_FPENV_MEM, dl.getIROrder(), VTs, MemVT, MMO));10714 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());10715 ID.AddInteger(MMO->getFlags());10716 void *IP = nullptr;10717 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))10718 return SDValue(E, 0);10719 10720 auto *N = newSDNode<FPStateAccessSDNode>(ISD::SET_FPENV_MEM, dl.getIROrder(),10721 dl.getDebugLoc(), VTs, MemVT, MMO);10722 createOperands(N, Ops);10723 10724 CSEMap.InsertNode(N, IP);10725 InsertNode(N);10726 SDValue V(N, 0);10727 NewSDValueDbgMsg(V, "Creating new node: ", this);10728 return V;10729}10730 10731SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {10732 // select undef, T, F --> T (if T is a constant), otherwise F10733 // select, ?, undef, F --> F10734 // select, ?, T, undef --> T10735 if (Cond.isUndef())10736 return isConstantValueOfAnyType(T) ? T : F;10737 if (T.isUndef())10738 return F;10739 if (F.isUndef())10740 return T;10741 10742 // select true, T, F --> T10743 // select false, T, F --> F10744 if (auto C = isBoolConstant(Cond))10745 return *C ? T : F;10746 10747 // select ?, T, T --> T10748 if (T == F)10749 return T;10750 10751 return SDValue();10752}10753 10754SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {10755 // shift undef, Y --> 0 (can always assume that the undef value is 0)10756 if (X.isUndef())10757 return getConstant(0, SDLoc(X.getNode()), X.getValueType());10758 // shift X, undef --> undef (because it may shift by the bitwidth)10759 if (Y.isUndef())10760 return getUNDEF(X.getValueType());10761 10762 // shift 0, Y --> 010763 // shift X, 0 --> X10764 if (isNullOrNullSplat(X) || isNullOrNullSplat(Y))10765 return X;10766 10767 // shift X, C >= bitwidth(X) --> undef10768 // All vector elements must be too big (or undef) to avoid partial undefs.10769 auto isShiftTooBig = [X](ConstantSDNode *Val) {10770 return !Val || Val->getAPIntValue().uge(X.getScalarValueSizeInBits());10771 };10772 if (ISD::matchUnaryPredicate(Y, isShiftTooBig, true))10773 return getUNDEF(X.getValueType());10774 10775 // shift i1/vXi1 X, Y --> X (any non-zero shift amount is undefined).10776 if (X.getValueType().getScalarType() == MVT::i1)10777 return X;10778 10779 return SDValue();10780}10781 10782SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y,10783 SDNodeFlags Flags) {10784 // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand10785 // (an undef operand can be chosen to be Nan/Inf), then the result of this10786 // operation is poison. That result can be relaxed to undef.10787 ConstantFPSDNode *XC = isConstOrConstSplatFP(X, /* AllowUndefs */ true);10788 ConstantFPSDNode *YC = isConstOrConstSplatFP(Y, /* AllowUndefs */ true);10789 bool HasNan = (XC && XC->getValueAPF().isNaN()) ||10790 (YC && YC->getValueAPF().isNaN());10791 bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||10792 (YC && YC->getValueAPF().isInfinity());10793 10794 if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef()))10795 return getUNDEF(X.getValueType());10796 10797 if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef()))10798 return getUNDEF(X.getValueType());10799 10800 if (!YC)10801 return SDValue();10802 10803 // X + -0.0 --> X10804 if (Opcode == ISD::FADD)10805 if (YC->getValueAPF().isNegZero())10806 return X;10807 10808 // X - +0.0 --> X10809 if (Opcode == ISD::FSUB)10810 if (YC->getValueAPF().isPosZero())10811 return X;10812 10813 // X * 1.0 --> X10814 // X / 1.0 --> X10815 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)10816 if (YC->getValueAPF().isExactlyValue(1.0))10817 return X;10818 10819 // X * 0.0 --> 0.010820 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())10821 if (YC->getValueAPF().isZero())10822 return getConstantFP(0.0, SDLoc(Y), Y.getValueType());10823 10824 return SDValue();10825}10826 10827SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,10828 SDValue Ptr, SDValue SV, unsigned Align) {10829 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) };10830 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);10831}10832 10833SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,10834 ArrayRef<SDUse> Ops) {10835 switch (Ops.size()) {10836 case 0: return getNode(Opcode, DL, VT);10837 case 1: return getNode(Opcode, DL, VT, Ops[0].get());10838 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);10839 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);10840 default: break;10841 }10842 10843 // Copy from an SDUse array into an SDValue array for use with10844 // the regular getNode logic.10845 SmallVector<SDValue, 8> NewOps(Ops);10846 return getNode(Opcode, DL, VT, NewOps);10847}10848 10849SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,10850 ArrayRef<SDValue> Ops) {10851 SDNodeFlags Flags;10852 if (Inserter)10853 Flags = Inserter->getFlags();10854 return getNode(Opcode, DL, VT, Ops, Flags);10855}10856 10857SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,10858 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {10859 unsigned NumOps = Ops.size();10860 switch (NumOps) {10861 case 0: return getNode(Opcode, DL, VT);10862 case 1: return getNode(Opcode, DL, VT, Ops[0], Flags);10863 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Flags);10864 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2], Flags);10865 default: break;10866 }10867 10868#ifndef NDEBUG10869 for (const auto &Op : Ops)10870 assert(Op.getOpcode() != ISD::DELETED_NODE &&10871 "Operand is DELETED_NODE!");10872#endif10873 10874 switch (Opcode) {10875 default: break;10876 case ISD::BUILD_VECTOR:10877 // Attempt to simplify BUILD_VECTOR.10878 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))10879 return V;10880 break;10881 case ISD::CONCAT_VECTORS:10882 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))10883 return V;10884 break;10885 case ISD::SELECT_CC:10886 assert(NumOps == 5 && "SELECT_CC takes 5 operands!");10887 assert(Ops[0].getValueType() == Ops[1].getValueType() &&10888 "LHS and RHS of condition must have same type!");10889 assert(Ops[2].getValueType() == Ops[3].getValueType() &&10890 "True and False arms of SelectCC must have same type!");10891 assert(Ops[2].getValueType() == VT &&10892 "select_cc node must be of same type as true and false value!");10893 assert((!Ops[0].getValueType().isVector() ||10894 Ops[0].getValueType().getVectorElementCount() ==10895 VT.getVectorElementCount()) &&10896 "Expected select_cc with vector result to have the same sized "10897 "comparison type!");10898 break;10899 case ISD::BR_CC:10900 assert(NumOps == 5 && "BR_CC takes 5 operands!");10901 assert(Ops[2].getValueType() == Ops[3].getValueType() &&10902 "LHS/RHS of comparison should match types!");10903 break;10904 case ISD::VP_ADD:10905 case ISD::VP_SUB:10906 // If it is VP_ADD/VP_SUB mask operation then turn it to VP_XOR10907 if (VT.getScalarType() == MVT::i1)10908 Opcode = ISD::VP_XOR;10909 break;10910 case ISD::VP_MUL:10911 // If it is VP_MUL mask operation then turn it to VP_AND10912 if (VT.getScalarType() == MVT::i1)10913 Opcode = ISD::VP_AND;10914 break;10915 case ISD::VP_REDUCE_MUL:10916 // If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND10917 if (VT == MVT::i1)10918 Opcode = ISD::VP_REDUCE_AND;10919 break;10920 case ISD::VP_REDUCE_ADD:10921 // If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR10922 if (VT == MVT::i1)10923 Opcode = ISD::VP_REDUCE_XOR;10924 break;10925 case ISD::VP_REDUCE_SMAX:10926 case ISD::VP_REDUCE_UMIN:10927 // If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to10928 // VP_REDUCE_AND.10929 if (VT == MVT::i1)10930 Opcode = ISD::VP_REDUCE_AND;10931 break;10932 case ISD::VP_REDUCE_SMIN:10933 case ISD::VP_REDUCE_UMAX:10934 // If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to10935 // VP_REDUCE_OR.10936 if (VT == MVT::i1)10937 Opcode = ISD::VP_REDUCE_OR;10938 break;10939 }10940 10941 // Memoize nodes.10942 SDNode *N;10943 SDVTList VTs = getVTList(VT);10944 10945 if (VT != MVT::Glue) {10946 FoldingSetNodeID ID;10947 AddNodeIDNode(ID, Opcode, VTs, Ops);10948 void *IP = nullptr;10949 10950 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {10951 E->intersectFlagsWith(Flags);10952 return SDValue(E, 0);10953 }10954 10955 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);10956 createOperands(N, Ops);10957 10958 CSEMap.InsertNode(N, IP);10959 } else {10960 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);10961 createOperands(N, Ops);10962 }10963 10964 N->setFlags(Flags);10965 InsertNode(N);10966 SDValue V(N, 0);10967 NewSDValueDbgMsg(V, "Creating new node: ", this);10968 return V;10969}10970 10971SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,10972 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {10973 SDNodeFlags Flags;10974 if (Inserter)10975 Flags = Inserter->getFlags();10976 return getNode(Opcode, DL, getVTList(ResultTys), Ops, Flags);10977}10978 10979SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,10980 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops,10981 const SDNodeFlags Flags) {10982 return getNode(Opcode, DL, getVTList(ResultTys), Ops, Flags);10983}10984 10985SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,10986 ArrayRef<SDValue> Ops) {10987 SDNodeFlags Flags;10988 if (Inserter)10989 Flags = Inserter->getFlags();10990 return getNode(Opcode, DL, VTList, Ops, Flags);10991}10992 10993SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,10994 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {10995 if (VTList.NumVTs == 1)10996 return getNode(Opcode, DL, VTList.VTs[0], Ops, Flags);10997 10998#ifndef NDEBUG10999 for (const auto &Op : Ops)11000 assert(Op.getOpcode() != ISD::DELETED_NODE &&11001 "Operand is DELETED_NODE!");11002#endif11003 11004 switch (Opcode) {11005 case ISD::SADDO:11006 case ISD::UADDO:11007 case ISD::SSUBO:11008 case ISD::USUBO: {11009 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&11010 "Invalid add/sub overflow op!");11011 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&11012 Ops[0].getValueType() == Ops[1].getValueType() &&11013 Ops[0].getValueType() == VTList.VTs[0] &&11014 "Binary operator types must match!");11015 SDValue N1 = Ops[0], N2 = Ops[1];11016 canonicalizeCommutativeBinop(Opcode, N1, N2);11017 11018 // (X +- 0) -> X with zero-overflow.11019 ConstantSDNode *N2CV = isConstOrConstSplat(N2, /*AllowUndefs*/ false,11020 /*AllowTruncation*/ true);11021 if (N2CV && N2CV->isZero()) {11022 SDValue ZeroOverFlow = getConstant(0, DL, VTList.VTs[1]);11023 return getNode(ISD::MERGE_VALUES, DL, VTList, {N1, ZeroOverFlow}, Flags);11024 }11025 11026 if (VTList.VTs[0].getScalarType() == MVT::i1 &&11027 VTList.VTs[1].getScalarType() == MVT::i1) {11028 SDValue F1 = getFreeze(N1);11029 SDValue F2 = getFreeze(N2);11030 // {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)}11031 if (Opcode == ISD::UADDO || Opcode == ISD::SADDO)11032 return getNode(ISD::MERGE_VALUES, DL, VTList,11033 {getNode(ISD::XOR, DL, VTList.VTs[0], F1, F2),11034 getNode(ISD::AND, DL, VTList.VTs[1], F1, F2)},11035 Flags);11036 // {vXi1,vXi1} (u/s)subo(vXi1 x, vXi1y) -> {xor(x,y),and(~x,y)}11037 if (Opcode == ISD::USUBO || Opcode == ISD::SSUBO) {11038 SDValue NotF1 = getNOT(DL, F1, VTList.VTs[0]);11039 return getNode(ISD::MERGE_VALUES, DL, VTList,11040 {getNode(ISD::XOR, DL, VTList.VTs[0], F1, F2),11041 getNode(ISD::AND, DL, VTList.VTs[1], NotF1, F2)},11042 Flags);11043 }11044 }11045 break;11046 }11047 case ISD::SADDO_CARRY:11048 case ISD::UADDO_CARRY:11049 case ISD::SSUBO_CARRY:11050 case ISD::USUBO_CARRY:11051 assert(VTList.NumVTs == 2 && Ops.size() == 3 &&11052 "Invalid add/sub overflow op!");11053 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&11054 Ops[0].getValueType() == Ops[1].getValueType() &&11055 Ops[0].getValueType() == VTList.VTs[0] &&11056 Ops[2].getValueType() == VTList.VTs[1] &&11057 "Binary operator types must match!");11058 break;11059 case ISD::SMUL_LOHI:11060 case ISD::UMUL_LOHI: {11061 assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!");11062 assert(VTList.VTs[0].isInteger() && VTList.VTs[0] == VTList.VTs[1] &&11063 VTList.VTs[0] == Ops[0].getValueType() &&11064 VTList.VTs[0] == Ops[1].getValueType() &&11065 "Binary operator types must match!");11066 // Constant fold.11067 ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(Ops[0]);11068 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ops[1]);11069 if (LHS && RHS) {11070 unsigned Width = VTList.VTs[0].getScalarSizeInBits();11071 unsigned OutWidth = Width * 2;11072 APInt Val = LHS->getAPIntValue();11073 APInt Mul = RHS->getAPIntValue();11074 if (Opcode == ISD::SMUL_LOHI) {11075 Val = Val.sext(OutWidth);11076 Mul = Mul.sext(OutWidth);11077 } else {11078 Val = Val.zext(OutWidth);11079 Mul = Mul.zext(OutWidth);11080 }11081 Val *= Mul;11082 11083 SDValue Hi =11084 getConstant(Val.extractBits(Width, Width), DL, VTList.VTs[0]);11085 SDValue Lo = getConstant(Val.trunc(Width), DL, VTList.VTs[0]);11086 return getNode(ISD::MERGE_VALUES, DL, VTList, {Lo, Hi}, Flags);11087 }11088 break;11089 }11090 case ISD::FFREXP: {11091 assert(VTList.NumVTs == 2 && Ops.size() == 1 && "Invalid ffrexp op!");11092 assert(VTList.VTs[0].isFloatingPoint() && VTList.VTs[1].isInteger() &&11093 VTList.VTs[0] == Ops[0].getValueType() && "frexp type mismatch");11094 11095 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Ops[0])) {11096 int FrexpExp;11097 APFloat FrexpMant =11098 frexp(C->getValueAPF(), FrexpExp, APFloat::rmNearestTiesToEven);11099 SDValue Result0 = getConstantFP(FrexpMant, DL, VTList.VTs[0]);11100 SDValue Result1 = getSignedConstant(FrexpMant.isFinite() ? FrexpExp : 0,11101 DL, VTList.VTs[1]);11102 return getNode(ISD::MERGE_VALUES, DL, VTList, {Result0, Result1}, Flags);11103 }11104 11105 break;11106 }11107 case ISD::STRICT_FP_EXTEND:11108 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&11109 "Invalid STRICT_FP_EXTEND!");11110 assert(VTList.VTs[0].isFloatingPoint() &&11111 Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!");11112 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&11113 "STRICT_FP_EXTEND result type should be vector iff the operand "11114 "type is vector!");11115 assert((!VTList.VTs[0].isVector() ||11116 VTList.VTs[0].getVectorElementCount() ==11117 Ops[1].getValueType().getVectorElementCount()) &&11118 "Vector element count mismatch!");11119 assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) &&11120 "Invalid fpext node, dst <= src!");11121 break;11122 case ISD::STRICT_FP_ROUND:11123 assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!");11124 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&11125 "STRICT_FP_ROUND result type should be vector iff the operand "11126 "type is vector!");11127 assert((!VTList.VTs[0].isVector() ||11128 VTList.VTs[0].getVectorElementCount() ==11129 Ops[1].getValueType().getVectorElementCount()) &&11130 "Vector element count mismatch!");11131 assert(VTList.VTs[0].isFloatingPoint() &&11132 Ops[1].getValueType().isFloatingPoint() &&11133 VTList.VTs[0].bitsLT(Ops[1].getValueType()) &&11134 Ops[2].getOpcode() == ISD::TargetConstant &&11135 (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) &&11136 "Invalid STRICT_FP_ROUND!");11137 break;11138 }11139 11140 // Memoize the node unless it returns a glue result.11141 SDNode *N;11142 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {11143 FoldingSetNodeID ID;11144 AddNodeIDNode(ID, Opcode, VTList, Ops);11145 void *IP = nullptr;11146 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {11147 E->intersectFlagsWith(Flags);11148 return SDValue(E, 0);11149 }11150 11151 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);11152 createOperands(N, Ops);11153 CSEMap.InsertNode(N, IP);11154 } else {11155 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);11156 createOperands(N, Ops);11157 }11158 11159 N->setFlags(Flags);11160 InsertNode(N);11161 SDValue V(N, 0);11162 NewSDValueDbgMsg(V, "Creating new node: ", this);11163 return V;11164}11165 11166SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,11167 SDVTList VTList) {11168 return getNode(Opcode, DL, VTList, ArrayRef<SDValue>());11169}11170 11171SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,11172 SDValue N1) {11173 SDValue Ops[] = { N1 };11174 return getNode(Opcode, DL, VTList, Ops);11175}11176 11177SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,11178 SDValue N1, SDValue N2) {11179 SDValue Ops[] = { N1, N2 };11180 return getNode(Opcode, DL, VTList, Ops);11181}11182 11183SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,11184 SDValue N1, SDValue N2, SDValue N3) {11185 SDValue Ops[] = { N1, N2, N3 };11186 return getNode(Opcode, DL, VTList, Ops);11187}11188 11189SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,11190 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {11191 SDValue Ops[] = { N1, N2, N3, N4 };11192 return getNode(Opcode, DL, VTList, Ops);11193}11194 11195SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,11196 SDValue N1, SDValue N2, SDValue N3, SDValue N4,11197 SDValue N5) {11198 SDValue Ops[] = { N1, N2, N3, N4, N5 };11199 return getNode(Opcode, DL, VTList, Ops);11200}11201 11202SDVTList SelectionDAG::getVTList(EVT VT) {11203 if (!VT.isExtended())11204 return makeVTList(SDNode::getValueTypeList(VT.getSimpleVT()), 1);11205 11206 return makeVTList(&(*EVTs.insert(VT).first), 1);11207}11208 11209SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {11210 FoldingSetNodeID ID;11211 ID.AddInteger(2U);11212 ID.AddInteger(VT1.getRawBits());11213 ID.AddInteger(VT2.getRawBits());11214 11215 void *IP = nullptr;11216 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);11217 if (!Result) {11218 EVT *Array = Allocator.Allocate<EVT>(2);11219 Array[0] = VT1;11220 Array[1] = VT2;11221 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);11222 VTListMap.InsertNode(Result, IP);11223 }11224 return Result->getSDVTList();11225}11226 11227SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {11228 FoldingSetNodeID ID;11229 ID.AddInteger(3U);11230 ID.AddInteger(VT1.getRawBits());11231 ID.AddInteger(VT2.getRawBits());11232 ID.AddInteger(VT3.getRawBits());11233 11234 void *IP = nullptr;11235 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);11236 if (!Result) {11237 EVT *Array = Allocator.Allocate<EVT>(3);11238 Array[0] = VT1;11239 Array[1] = VT2;11240 Array[2] = VT3;11241 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);11242 VTListMap.InsertNode(Result, IP);11243 }11244 return Result->getSDVTList();11245}11246 11247SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {11248 FoldingSetNodeID ID;11249 ID.AddInteger(4U);11250 ID.AddInteger(VT1.getRawBits());11251 ID.AddInteger(VT2.getRawBits());11252 ID.AddInteger(VT3.getRawBits());11253 ID.AddInteger(VT4.getRawBits());11254 11255 void *IP = nullptr;11256 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);11257 if (!Result) {11258 EVT *Array = Allocator.Allocate<EVT>(4);11259 Array[0] = VT1;11260 Array[1] = VT2;11261 Array[2] = VT3;11262 Array[3] = VT4;11263 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);11264 VTListMap.InsertNode(Result, IP);11265 }11266 return Result->getSDVTList();11267}11268 11269SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {11270 unsigned NumVTs = VTs.size();11271 FoldingSetNodeID ID;11272 ID.AddInteger(NumVTs);11273 for (unsigned index = 0; index < NumVTs; index++) {11274 ID.AddInteger(VTs[index].getRawBits());11275 }11276 11277 void *IP = nullptr;11278 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);11279 if (!Result) {11280 EVT *Array = Allocator.Allocate<EVT>(NumVTs);11281 llvm::copy(VTs, Array);11282 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);11283 VTListMap.InsertNode(Result, IP);11284 }11285 return Result->getSDVTList();11286}11287 11288 11289/// UpdateNodeOperands - *Mutate* the specified node in-place to have the11290/// specified operands. If the resultant node already exists in the DAG,11291/// this does not modify the specified node, instead it returns the node that11292/// already exists. If the resultant node does not exist in the DAG, the11293/// input node is returned. As a degenerate case, if you specify the same11294/// input operands as the node already has, the input node is returned.11295SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {11296 assert(N->getNumOperands() == 1 && "Update with wrong number of operands");11297 11298 // Check to see if there is no change.11299 if (Op == N->getOperand(0)) return N;11300 11301 // See if the modified node already exists.11302 void *InsertPos = nullptr;11303 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))11304 return Existing;11305 11306 // Nope it doesn't. Remove the node from its current place in the maps.11307 if (InsertPos)11308 if (!RemoveNodeFromCSEMaps(N))11309 InsertPos = nullptr;11310 11311 // Now we update the operands.11312 N->OperandList[0].set(Op);11313 11314 updateDivergence(N);11315 // If this gets put into a CSE map, add it.11316 if (InsertPos) CSEMap.InsertNode(N, InsertPos);11317 return N;11318}11319 11320SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {11321 assert(N->getNumOperands() == 2 && "Update with wrong number of operands");11322 11323 // Check to see if there is no change.11324 if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))11325 return N; // No operands changed, just return the input node.11326 11327 // See if the modified node already exists.11328 void *InsertPos = nullptr;11329 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))11330 return Existing;11331 11332 // Nope it doesn't. Remove the node from its current place in the maps.11333 if (InsertPos)11334 if (!RemoveNodeFromCSEMaps(N))11335 InsertPos = nullptr;11336 11337 // Now we update the operands.11338 if (N->OperandList[0] != Op1)11339 N->OperandList[0].set(Op1);11340 if (N->OperandList[1] != Op2)11341 N->OperandList[1].set(Op2);11342 11343 updateDivergence(N);11344 // If this gets put into a CSE map, add it.11345 if (InsertPos) CSEMap.InsertNode(N, InsertPos);11346 return N;11347}11348 11349SDNode *SelectionDAG::11350UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {11351 SDValue Ops[] = { Op1, Op2, Op3 };11352 return UpdateNodeOperands(N, Ops);11353}11354 11355SDNode *SelectionDAG::11356UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,11357 SDValue Op3, SDValue Op4) {11358 SDValue Ops[] = { Op1, Op2, Op3, Op4 };11359 return UpdateNodeOperands(N, Ops);11360}11361 11362SDNode *SelectionDAG::11363UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,11364 SDValue Op3, SDValue Op4, SDValue Op5) {11365 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };11366 return UpdateNodeOperands(N, Ops);11367}11368 11369SDNode *SelectionDAG::11370UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {11371 unsigned NumOps = Ops.size();11372 assert(N->getNumOperands() == NumOps &&11373 "Update with wrong number of operands");11374 11375 // If no operands changed just return the input node.11376 if (std::equal(Ops.begin(), Ops.end(), N->op_begin()))11377 return N;11378 11379 // See if the modified node already exists.11380 void *InsertPos = nullptr;11381 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))11382 return Existing;11383 11384 // Nope it doesn't. Remove the node from its current place in the maps.11385 if (InsertPos)11386 if (!RemoveNodeFromCSEMaps(N))11387 InsertPos = nullptr;11388 11389 // Now we update the operands.11390 for (unsigned i = 0; i != NumOps; ++i)11391 if (N->OperandList[i] != Ops[i])11392 N->OperandList[i].set(Ops[i]);11393 11394 updateDivergence(N);11395 // If this gets put into a CSE map, add it.11396 if (InsertPos) CSEMap.InsertNode(N, InsertPos);11397 return N;11398}11399 11400/// DropOperands - Release the operands and set this node to have11401/// zero operands.11402void SDNode::DropOperands() {11403 // Unlike the code in MorphNodeTo that does this, we don't need to11404 // watch for dead nodes here.11405 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {11406 SDUse &Use = *I++;11407 Use.set(SDValue());11408 }11409}11410 11411void SelectionDAG::setNodeMemRefs(MachineSDNode *N,11412 ArrayRef<MachineMemOperand *> NewMemRefs) {11413 if (NewMemRefs.empty()) {11414 N->clearMemRefs();11415 return;11416 }11417 11418 // Check if we can avoid allocating by storing a single reference directly.11419 if (NewMemRefs.size() == 1) {11420 N->MemRefs = NewMemRefs[0];11421 N->NumMemRefs = 1;11422 return;11423 }11424 11425 MachineMemOperand **MemRefsBuffer =11426 Allocator.template Allocate<MachineMemOperand *>(NewMemRefs.size());11427 llvm::copy(NewMemRefs, MemRefsBuffer);11428 N->MemRefs = MemRefsBuffer;11429 N->NumMemRefs = static_cast<int>(NewMemRefs.size());11430}11431 11432/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a11433/// machine opcode.11434///11435SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,11436 EVT VT) {11437 SDVTList VTs = getVTList(VT);11438 return SelectNodeTo(N, MachineOpc, VTs, {});11439}11440 11441SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,11442 EVT VT, SDValue Op1) {11443 SDVTList VTs = getVTList(VT);11444 SDValue Ops[] = { Op1 };11445 return SelectNodeTo(N, MachineOpc, VTs, Ops);11446}11447 11448SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,11449 EVT VT, SDValue Op1,11450 SDValue Op2) {11451 SDVTList VTs = getVTList(VT);11452 SDValue Ops[] = { Op1, Op2 };11453 return SelectNodeTo(N, MachineOpc, VTs, Ops);11454}11455 11456SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,11457 EVT VT, SDValue Op1,11458 SDValue Op2, SDValue Op3) {11459 SDVTList VTs = getVTList(VT);11460 SDValue Ops[] = { Op1, Op2, Op3 };11461 return SelectNodeTo(N, MachineOpc, VTs, Ops);11462}11463 11464SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,11465 EVT VT, ArrayRef<SDValue> Ops) {11466 SDVTList VTs = getVTList(VT);11467 return SelectNodeTo(N, MachineOpc, VTs, Ops);11468}11469 11470SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,11471 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {11472 SDVTList VTs = getVTList(VT1, VT2);11473 return SelectNodeTo(N, MachineOpc, VTs, Ops);11474}11475 11476SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,11477 EVT VT1, EVT VT2) {11478 SDVTList VTs = getVTList(VT1, VT2);11479 return SelectNodeTo(N, MachineOpc, VTs, {});11480}11481 11482SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,11483 EVT VT1, EVT VT2, EVT VT3,11484 ArrayRef<SDValue> Ops) {11485 SDVTList VTs = getVTList(VT1, VT2, VT3);11486 return SelectNodeTo(N, MachineOpc, VTs, Ops);11487}11488 11489SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,11490 EVT VT1, EVT VT2,11491 SDValue Op1, SDValue Op2) {11492 SDVTList VTs = getVTList(VT1, VT2);11493 SDValue Ops[] = { Op1, Op2 };11494 return SelectNodeTo(N, MachineOpc, VTs, Ops);11495}11496 11497SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,11498 SDVTList VTs,ArrayRef<SDValue> Ops) {11499 SDNode *New = MorphNodeTo(N, ~MachineOpc, VTs, Ops);11500 // Reset the NodeID to -1.11501 New->setNodeId(-1);11502 if (New != N) {11503 ReplaceAllUsesWith(N, New);11504 RemoveDeadNode(N);11505 }11506 return New;11507}11508 11509/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away11510/// the line number information on the merged node since it is not possible to11511/// preserve the information that operation is associated with multiple lines.11512/// This will make the debugger working better at -O0, were there is a higher11513/// probability having other instructions associated with that line.11514///11515/// For IROrder, we keep the smaller of the two11516SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {11517 DebugLoc NLoc = N->getDebugLoc();11518 if (NLoc && OptLevel == CodeGenOptLevel::None && OLoc.getDebugLoc() != NLoc) {11519 N->setDebugLoc(DebugLoc());11520 }11521 unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());11522 N->setIROrder(Order);11523 return N;11524}11525 11526/// MorphNodeTo - This *mutates* the specified node to have the specified11527/// return type, opcode, and operands.11528///11529/// Note that MorphNodeTo returns the resultant node. If there is already a11530/// node of the specified opcode and operands, it returns that node instead of11531/// the current one. Note that the SDLoc need not be the same.11532///11533/// Using MorphNodeTo is faster than creating a new node and swapping it in11534/// with ReplaceAllUsesWith both because it often avoids allocating a new11535/// node, and because it doesn't require CSE recalculation for any of11536/// the node's users.11537///11538/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.11539/// As a consequence it isn't appropriate to use from within the DAG combiner or11540/// the legalizer which maintain worklists that would need to be updated when11541/// deleting things.11542SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,11543 SDVTList VTs, ArrayRef<SDValue> Ops) {11544 // If an identical node already exists, use it.11545 void *IP = nullptr;11546 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {11547 FoldingSetNodeID ID;11548 AddNodeIDNode(ID, Opc, VTs, Ops);11549 if (SDNode *ON = FindNodeOrInsertPos(ID, SDLoc(N), IP))11550 return UpdateSDLocOnMergeSDNode(ON, SDLoc(N));11551 }11552 11553 if (!RemoveNodeFromCSEMaps(N))11554 IP = nullptr;11555 11556 // Start the morphing.11557 N->NodeType = Opc;11558 N->ValueList = VTs.VTs;11559 N->NumValues = VTs.NumVTs;11560 11561 // Clear the operands list, updating used nodes to remove this from their11562 // use list. Keep track of any operands that become dead as a result.11563 SmallPtrSet<SDNode*, 16> DeadNodeSet;11564 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {11565 SDUse &Use = *I++;11566 SDNode *Used = Use.getNode();11567 Use.set(SDValue());11568 if (Used->use_empty())11569 DeadNodeSet.insert(Used);11570 }11571 11572 // For MachineNode, initialize the memory references information.11573 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N))11574 MN->clearMemRefs();11575 11576 // Swap for an appropriately sized array from the recycler.11577 removeOperands(N);11578 createOperands(N, Ops);11579 11580 // Delete any nodes that are still dead after adding the uses for the11581 // new operands.11582 if (!DeadNodeSet.empty()) {11583 SmallVector<SDNode *, 16> DeadNodes;11584 for (SDNode *N : DeadNodeSet)11585 if (N->use_empty())11586 DeadNodes.push_back(N);11587 RemoveDeadNodes(DeadNodes);11588 }11589 11590 if (IP)11591 CSEMap.InsertNode(N, IP); // Memoize the new node.11592 return N;11593}11594 11595SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {11596 unsigned OrigOpc = Node->getOpcode();11597 unsigned NewOpc;11598 switch (OrigOpc) {11599 default:11600 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");11601#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \11602 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;11603#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \11604 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;11605#include "llvm/IR/ConstrainedOps.def"11606 }11607 11608 assert(Node->getNumValues() == 2 && "Unexpected number of results!");11609 11610 // We're taking this node out of the chain, so we need to re-link things.11611 SDValue InputChain = Node->getOperand(0);11612 SDValue OutputChain = SDValue(Node, 1);11613 ReplaceAllUsesOfValueWith(OutputChain, InputChain);11614 11615 SmallVector<SDValue, 3> Ops;11616 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)11617 Ops.push_back(Node->getOperand(i));11618 11619 SDVTList VTs = getVTList(Node->getValueType(0));11620 SDNode *Res = MorphNodeTo(Node, NewOpc, VTs, Ops);11621 11622 // MorphNodeTo can operate in two ways: if an existing node with the11623 // specified operands exists, it can just return it. Otherwise, it11624 // updates the node in place to have the requested operands.11625 if (Res == Node) {11626 // If we updated the node in place, reset the node ID. To the isel,11627 // this should be just like a newly allocated machine node.11628 Res->setNodeId(-1);11629 } else {11630 ReplaceAllUsesWith(Node, Res);11631 RemoveDeadNode(Node);11632 }11633 11634 return Res;11635}11636 11637/// getMachineNode - These are used for target selectors to create a new node11638/// with specified return type(s), MachineInstr opcode, and operands.11639///11640/// Note that getMachineNode returns the resultant node. If there is already a11641/// node of the specified opcode and operands, it returns that node instead of11642/// the current one.11643MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,11644 EVT VT) {11645 SDVTList VTs = getVTList(VT);11646 return getMachineNode(Opcode, dl, VTs, {});11647}11648 11649MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,11650 EVT VT, SDValue Op1) {11651 SDVTList VTs = getVTList(VT);11652 SDValue Ops[] = { Op1 };11653 return getMachineNode(Opcode, dl, VTs, Ops);11654}11655 11656MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,11657 EVT VT, SDValue Op1, SDValue Op2) {11658 SDVTList VTs = getVTList(VT);11659 SDValue Ops[] = { Op1, Op2 };11660 return getMachineNode(Opcode, dl, VTs, Ops);11661}11662 11663MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,11664 EVT VT, SDValue Op1, SDValue Op2,11665 SDValue Op3) {11666 SDVTList VTs = getVTList(VT);11667 SDValue Ops[] = { Op1, Op2, Op3 };11668 return getMachineNode(Opcode, dl, VTs, Ops);11669}11670 11671MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,11672 EVT VT, ArrayRef<SDValue> Ops) {11673 SDVTList VTs = getVTList(VT);11674 return getMachineNode(Opcode, dl, VTs, Ops);11675}11676 11677MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,11678 EVT VT1, EVT VT2, SDValue Op1,11679 SDValue Op2) {11680 SDVTList VTs = getVTList(VT1, VT2);11681 SDValue Ops[] = { Op1, Op2 };11682 return getMachineNode(Opcode, dl, VTs, Ops);11683}11684 11685MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,11686 EVT VT1, EVT VT2, SDValue Op1,11687 SDValue Op2, SDValue Op3) {11688 SDVTList VTs = getVTList(VT1, VT2);11689 SDValue Ops[] = { Op1, Op2, Op3 };11690 return getMachineNode(Opcode, dl, VTs, Ops);11691}11692 11693MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,11694 EVT VT1, EVT VT2,11695 ArrayRef<SDValue> Ops) {11696 SDVTList VTs = getVTList(VT1, VT2);11697 return getMachineNode(Opcode, dl, VTs, Ops);11698}11699 11700MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,11701 EVT VT1, EVT VT2, EVT VT3,11702 SDValue Op1, SDValue Op2) {11703 SDVTList VTs = getVTList(VT1, VT2, VT3);11704 SDValue Ops[] = { Op1, Op2 };11705 return getMachineNode(Opcode, dl, VTs, Ops);11706}11707 11708MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,11709 EVT VT1, EVT VT2, EVT VT3,11710 SDValue Op1, SDValue Op2,11711 SDValue Op3) {11712 SDVTList VTs = getVTList(VT1, VT2, VT3);11713 SDValue Ops[] = { Op1, Op2, Op3 };11714 return getMachineNode(Opcode, dl, VTs, Ops);11715}11716 11717MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,11718 EVT VT1, EVT VT2, EVT VT3,11719 ArrayRef<SDValue> Ops) {11720 SDVTList VTs = getVTList(VT1, VT2, VT3);11721 return getMachineNode(Opcode, dl, VTs, Ops);11722}11723 11724MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,11725 ArrayRef<EVT> ResultTys,11726 ArrayRef<SDValue> Ops) {11727 SDVTList VTs = getVTList(ResultTys);11728 return getMachineNode(Opcode, dl, VTs, Ops);11729}11730 11731MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,11732 SDVTList VTs,11733 ArrayRef<SDValue> Ops) {11734 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;11735 MachineSDNode *N;11736 void *IP = nullptr;11737 11738 if (DoCSE) {11739 FoldingSetNodeID ID;11740 AddNodeIDNode(ID, ~Opcode, VTs, Ops);11741 IP = nullptr;11742 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {11743 return cast<MachineSDNode>(UpdateSDLocOnMergeSDNode(E, DL));11744 }11745 }11746 11747 // Allocate a new MachineSDNode.11748 N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);11749 createOperands(N, Ops);11750 11751 if (DoCSE)11752 CSEMap.InsertNode(N, IP);11753 11754 InsertNode(N);11755 NewSDValueDbgMsg(SDValue(N, 0), "Creating new machine node: ", this);11756 return N;11757}11758 11759/// getTargetExtractSubreg - A convenience function for creating11760/// TargetOpcode::EXTRACT_SUBREG nodes.11761SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,11762 SDValue Operand) {11763 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);11764 SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,11765 VT, Operand, SRIdxVal);11766 return SDValue(Subreg, 0);11767}11768 11769/// getTargetInsertSubreg - A convenience function for creating11770/// TargetOpcode::INSERT_SUBREG nodes.11771SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,11772 SDValue Operand, SDValue Subreg) {11773 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);11774 SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,11775 VT, Operand, Subreg, SRIdxVal);11776 return SDValue(Result, 0);11777}11778 11779/// getNodeIfExists - Get the specified node if it's already available, or11780/// else return NULL.11781SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,11782 ArrayRef<SDValue> Ops,11783 bool AllowCommute) {11784 SDNodeFlags Flags;11785 if (Inserter)11786 Flags = Inserter->getFlags();11787 return getNodeIfExists(Opcode, VTList, Ops, Flags, AllowCommute);11788}11789 11790SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,11791 ArrayRef<SDValue> Ops,11792 const SDNodeFlags Flags,11793 bool AllowCommute) {11794 if (VTList.VTs[VTList.NumVTs - 1] == MVT::Glue)11795 return nullptr;11796 11797 auto Lookup = [&](ArrayRef<SDValue> LookupOps) -> SDNode * {11798 FoldingSetNodeID ID;11799 AddNodeIDNode(ID, Opcode, VTList, LookupOps);11800 void *IP = nullptr;11801 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) {11802 E->intersectFlagsWith(Flags);11803 return E;11804 }11805 return nullptr;11806 };11807 11808 if (SDNode *Existing = Lookup(Ops))11809 return Existing;11810 11811 if (AllowCommute && TLI->isCommutativeBinOp(Opcode))11812 return Lookup({Ops[1], Ops[0]});11813 11814 return nullptr;11815}11816 11817/// doesNodeExist - Check if a node exists without modifying its flags.11818bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList,11819 ArrayRef<SDValue> Ops) {11820 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {11821 FoldingSetNodeID ID;11822 AddNodeIDNode(ID, Opcode, VTList, Ops);11823 void *IP = nullptr;11824 if (FindNodeOrInsertPos(ID, SDLoc(), IP))11825 return true;11826 }11827 return false;11828}11829 11830/// getDbgValue - Creates a SDDbgValue node.11831///11832/// SDNode11833SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,11834 SDNode *N, unsigned R, bool IsIndirect,11835 const DebugLoc &DL, unsigned O) {11836 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&11837 "Expected inlined-at fields to agree");11838 return new (DbgInfo->getAlloc())11839 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(N, R),11840 {}, IsIndirect, DL, O,11841 /*IsVariadic=*/false);11842}11843 11844/// Constant11845SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,11846 DIExpression *Expr,11847 const Value *C,11848 const DebugLoc &DL, unsigned O) {11849 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&11850 "Expected inlined-at fields to agree");11851 return new (DbgInfo->getAlloc())11852 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(C), {},11853 /*IsIndirect=*/false, DL, O,11854 /*IsVariadic=*/false);11855}11856 11857/// FrameIndex11858SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,11859 DIExpression *Expr, unsigned FI,11860 bool IsIndirect,11861 const DebugLoc &DL,11862 unsigned O) {11863 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&11864 "Expected inlined-at fields to agree");11865 return getFrameIndexDbgValue(Var, Expr, FI, {}, IsIndirect, DL, O);11866}11867 11868/// FrameIndex with dependencies11869SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,11870 DIExpression *Expr, unsigned FI,11871 ArrayRef<SDNode *> Dependencies,11872 bool IsIndirect,11873 const DebugLoc &DL,11874 unsigned O) {11875 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&11876 "Expected inlined-at fields to agree");11877 return new (DbgInfo->getAlloc())11878 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FI),11879 Dependencies, IsIndirect, DL, O,11880 /*IsVariadic=*/false);11881}11882 11883/// VReg11884SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr,11885 Register VReg, bool IsIndirect,11886 const DebugLoc &DL, unsigned O) {11887 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&11888 "Expected inlined-at fields to agree");11889 return new (DbgInfo->getAlloc())11890 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg),11891 {}, IsIndirect, DL, O,11892 /*IsVariadic=*/false);11893}11894 11895SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr,11896 ArrayRef<SDDbgOperand> Locs,11897 ArrayRef<SDNode *> Dependencies,11898 bool IsIndirect, const DebugLoc &DL,11899 unsigned O, bool IsVariadic) {11900 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&11901 "Expected inlined-at fields to agree");11902 return new (DbgInfo->getAlloc())11903 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,11904 DL, O, IsVariadic);11905}11906 11907void SelectionDAG::transferDbgValues(SDValue From, SDValue To,11908 unsigned OffsetInBits, unsigned SizeInBits,11909 bool InvalidateDbg) {11910 SDNode *FromNode = From.getNode();11911 SDNode *ToNode = To.getNode();11912 assert(FromNode && ToNode && "Can't modify dbg values");11913 11914 // PR3533811915 // TODO: assert(From != To && "Redundant dbg value transfer");11916 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");11917 if (From == To || FromNode == ToNode)11918 return;11919 11920 if (!FromNode->getHasDebugValue())11921 return;11922 11923 SDDbgOperand FromLocOp =11924 SDDbgOperand::fromNode(From.getNode(), From.getResNo());11925 SDDbgOperand ToLocOp = SDDbgOperand::fromNode(To.getNode(), To.getResNo());11926 11927 SmallVector<SDDbgValue *, 2> ClonedDVs;11928 for (SDDbgValue *Dbg : GetDbgValues(FromNode)) {11929 if (Dbg->isInvalidated())11930 continue;11931 11932 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");11933 11934 // Create a new location ops vector that is equal to the old vector, but11935 // with each instance of FromLocOp replaced with ToLocOp.11936 bool Changed = false;11937 auto NewLocOps = Dbg->copyLocationOps();11938 std::replace_if(11939 NewLocOps.begin(), NewLocOps.end(),11940 [&Changed, FromLocOp](const SDDbgOperand &Op) {11941 bool Match = Op == FromLocOp;11942 Changed |= Match;11943 return Match;11944 },11945 ToLocOp);11946 // Ignore this SDDbgValue if we didn't find a matching location.11947 if (!Changed)11948 continue;11949 11950 DIVariable *Var = Dbg->getVariable();11951 auto *Expr = Dbg->getExpression();11952 // If a fragment is requested, update the expression.11953 if (SizeInBits) {11954 // When splitting a larger (e.g., sign-extended) value whose11955 // lower bits are described with an SDDbgValue, do not attempt11956 // to transfer the SDDbgValue to the upper bits.11957 if (auto FI = Expr->getFragmentInfo())11958 if (OffsetInBits + SizeInBits > FI->SizeInBits)11959 continue;11960 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,11961 SizeInBits);11962 if (!Fragment)11963 continue;11964 Expr = *Fragment;11965 }11966 11967 auto AdditionalDependencies = Dbg->getAdditionalDependencies();11968 // Clone the SDDbgValue and move it to To.11969 SDDbgValue *Clone = getDbgValueList(11970 Var, Expr, NewLocOps, AdditionalDependencies, Dbg->isIndirect(),11971 Dbg->getDebugLoc(), std::max(ToNode->getIROrder(), Dbg->getOrder()),11972 Dbg->isVariadic());11973 ClonedDVs.push_back(Clone);11974 11975 if (InvalidateDbg) {11976 // Invalidate value and indicate the SDDbgValue should not be emitted.11977 Dbg->setIsInvalidated();11978 Dbg->setIsEmitted();11979 }11980 }11981 11982 for (SDDbgValue *Dbg : ClonedDVs) {11983 assert(is_contained(Dbg->getSDNodes(), ToNode) &&11984 "Transferred DbgValues should depend on the new SDNode");11985 AddDbgValue(Dbg, false);11986 }11987}11988 11989void SelectionDAG::salvageDebugInfo(SDNode &N) {11990 if (!N.getHasDebugValue())11991 return;11992 11993 auto GetLocationOperand = [](SDNode *Node, unsigned ResNo) {11994 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Node))11995 return SDDbgOperand::fromFrameIdx(FISDN->getIndex());11996 return SDDbgOperand::fromNode(Node, ResNo);11997 };11998 11999 SmallVector<SDDbgValue *, 2> ClonedDVs;12000 for (auto *DV : GetDbgValues(&N)) {12001 if (DV->isInvalidated())12002 continue;12003 switch (N.getOpcode()) {12004 default:12005 break;12006 case ISD::ADD: {12007 SDValue N0 = N.getOperand(0);12008 SDValue N1 = N.getOperand(1);12009 if (!isa<ConstantSDNode>(N0)) {12010 bool RHSConstant = isa<ConstantSDNode>(N1);12011 uint64_t Offset;12012 if (RHSConstant)12013 Offset = N.getConstantOperandVal(1);12014 // We are not allowed to turn indirect debug values variadic, so12015 // don't salvage those.12016 if (!RHSConstant && DV->isIndirect())12017 continue;12018 12019 // Rewrite an ADD constant node into a DIExpression. Since we are12020 // performing arithmetic to compute the variable's *value* in the12021 // DIExpression, we need to mark the expression with a12022 // DW_OP_stack_value.12023 auto *DIExpr = DV->getExpression();12024 auto NewLocOps = DV->copyLocationOps();12025 bool Changed = false;12026 size_t OrigLocOpsSize = NewLocOps.size();12027 for (size_t i = 0; i < OrigLocOpsSize; ++i) {12028 // We're not given a ResNo to compare against because the whole12029 // node is going away. We know that any ISD::ADD only has one12030 // result, so we can assume any node match is using the result.12031 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||12032 NewLocOps[i].getSDNode() != &N)12033 continue;12034 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());12035 if (RHSConstant) {12036 SmallVector<uint64_t, 3> ExprOps;12037 DIExpression::appendOffset(ExprOps, Offset);12038 DIExpr = DIExpression::appendOpsToArg(DIExpr, ExprOps, i, true);12039 } else {12040 // Convert to a variadic expression (if not already).12041 // convertToVariadicExpression() returns a const pointer, so we use12042 // a temporary const variable here.12043 const auto *TmpDIExpr =12044 DIExpression::convertToVariadicExpression(DIExpr);12045 SmallVector<uint64_t, 3> ExprOps;12046 ExprOps.push_back(dwarf::DW_OP_LLVM_arg);12047 ExprOps.push_back(NewLocOps.size());12048 ExprOps.push_back(dwarf::DW_OP_plus);12049 SDDbgOperand RHS =12050 SDDbgOperand::fromNode(N1.getNode(), N1.getResNo());12051 NewLocOps.push_back(RHS);12052 DIExpr = DIExpression::appendOpsToArg(TmpDIExpr, ExprOps, i, true);12053 }12054 Changed = true;12055 }12056 (void)Changed;12057 assert(Changed && "Salvage target doesn't use N");12058 12059 bool IsVariadic =12060 DV->isVariadic() || OrigLocOpsSize != NewLocOps.size();12061 12062 auto AdditionalDependencies = DV->getAdditionalDependencies();12063 SDDbgValue *Clone = getDbgValueList(12064 DV->getVariable(), DIExpr, NewLocOps, AdditionalDependencies,12065 DV->isIndirect(), DV->getDebugLoc(), DV->getOrder(), IsVariadic);12066 ClonedDVs.push_back(Clone);12067 DV->setIsInvalidated();12068 DV->setIsEmitted();12069 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";12070 N0.getNode()->dumprFull(this);12071 dbgs() << " into " << *DIExpr << '\n');12072 }12073 break;12074 }12075 case ISD::TRUNCATE: {12076 SDValue N0 = N.getOperand(0);12077 TypeSize FromSize = N0.getValueSizeInBits();12078 TypeSize ToSize = N.getValueSizeInBits(0);12079 12080 DIExpression *DbgExpression = DV->getExpression();12081 auto ExtOps = DIExpression::getExtOps(FromSize, ToSize, false);12082 auto NewLocOps = DV->copyLocationOps();12083 bool Changed = false;12084 for (size_t i = 0; i < NewLocOps.size(); ++i) {12085 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||12086 NewLocOps[i].getSDNode() != &N)12087 continue;12088 12089 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());12090 DbgExpression = DIExpression::appendOpsToArg(DbgExpression, ExtOps, i);12091 Changed = true;12092 }12093 assert(Changed && "Salvage target doesn't use N");12094 (void)Changed;12095 12096 SDDbgValue *Clone =12097 getDbgValueList(DV->getVariable(), DbgExpression, NewLocOps,12098 DV->getAdditionalDependencies(), DV->isIndirect(),12099 DV->getDebugLoc(), DV->getOrder(), DV->isVariadic());12100 12101 ClonedDVs.push_back(Clone);12102 DV->setIsInvalidated();12103 DV->setIsEmitted();12104 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; N0.getNode()->dumprFull(this);12105 dbgs() << " into " << *DbgExpression << '\n');12106 break;12107 }12108 }12109 }12110 12111 for (SDDbgValue *Dbg : ClonedDVs) {12112 assert((!Dbg->getSDNodes().empty() ||12113 llvm::any_of(Dbg->getLocationOps(),12114 [&](const SDDbgOperand &Op) {12115 return Op.getKind() == SDDbgOperand::FRAMEIX;12116 })) &&12117 "Salvaged DbgValue should depend on a new SDNode");12118 AddDbgValue(Dbg, false);12119 }12120}12121 12122/// Creates a SDDbgLabel node.12123SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label,12124 const DebugLoc &DL, unsigned O) {12125 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&12126 "Expected inlined-at fields to agree");12127 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);12128}12129 12130namespace {12131 12132/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node12133/// pointed to by a use iterator is deleted, increment the use iterator12134/// so that it doesn't dangle.12135///12136class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {12137 SDNode::use_iterator &UI;12138 SDNode::use_iterator &UE;12139 12140 void NodeDeleted(SDNode *N, SDNode *E) override {12141 // Increment the iterator as needed.12142 while (UI != UE && N == UI->getUser())12143 ++UI;12144 }12145 12146public:12147 RAUWUpdateListener(SelectionDAG &d,12148 SDNode::use_iterator &ui,12149 SDNode::use_iterator &ue)12150 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}12151};12152 12153} // end anonymous namespace12154 12155/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.12156/// This can cause recursive merging of nodes in the DAG.12157///12158/// This version assumes From has a single result value.12159///12160void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {12161 SDNode *From = FromN.getNode();12162 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&12163 "Cannot replace with this method!");12164 assert(From != To.getNode() && "Cannot replace uses of with self");12165 12166 // Preserve Debug Values12167 transferDbgValues(FromN, To);12168 // Preserve extra info.12169 copyExtraInfo(From, To.getNode());12170 12171 // Iterate over all the existing uses of From. New uses will be added12172 // to the beginning of the use list, which we avoid visiting.12173 // This specifically avoids visiting uses of From that arise while the12174 // replacement is happening, because any such uses would be the result12175 // of CSE: If an existing node looks like From after one of its operands12176 // is replaced by To, we don't want to replace of all its users with To12177 // too. See PR3018 for more info.12178 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();12179 RAUWUpdateListener Listener(*this, UI, UE);12180 while (UI != UE) {12181 SDNode *User = UI->getUser();12182 12183 // This node is about to morph, remove its old self from the CSE maps.12184 RemoveNodeFromCSEMaps(User);12185 12186 // A user can appear in a use list multiple times, and when this12187 // happens the uses are usually next to each other in the list.12188 // To help reduce the number of CSE recomputations, process all12189 // the uses of this user that we can find this way.12190 do {12191 SDUse &Use = *UI;12192 ++UI;12193 Use.set(To);12194 if (To->isDivergent() != From->isDivergent())12195 updateDivergence(User);12196 } while (UI != UE && UI->getUser() == User);12197 // Now that we have modified User, add it back to the CSE maps. If it12198 // already exists there, recursively merge the results together.12199 AddModifiedNodeToCSEMaps(User);12200 }12201 12202 // If we just RAUW'd the root, take note.12203 if (FromN == getRoot())12204 setRoot(To);12205}12206 12207/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.12208/// This can cause recursive merging of nodes in the DAG.12209///12210/// This version assumes that for each value of From, there is a12211/// corresponding value in To in the same position with the same type.12212///12213void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {12214#ifndef NDEBUG12215 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)12216 assert((!From->hasAnyUseOfValue(i) ||12217 From->getValueType(i) == To->getValueType(i)) &&12218 "Cannot use this version of ReplaceAllUsesWith!");12219#endif12220 12221 // Handle the trivial case.12222 if (From == To)12223 return;12224 12225 // Preserve Debug Info. Only do this if there's a use.12226 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)12227 if (From->hasAnyUseOfValue(i)) {12228 assert((i < To->getNumValues()) && "Invalid To location");12229 transferDbgValues(SDValue(From, i), SDValue(To, i));12230 }12231 // Preserve extra info.12232 copyExtraInfo(From, To);12233 12234 // Iterate over just the existing users of From. See the comments in12235 // the ReplaceAllUsesWith above.12236 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();12237 RAUWUpdateListener Listener(*this, UI, UE);12238 while (UI != UE) {12239 SDNode *User = UI->getUser();12240 12241 // This node is about to morph, remove its old self from the CSE maps.12242 RemoveNodeFromCSEMaps(User);12243 12244 // A user can appear in a use list multiple times, and when this12245 // happens the uses are usually next to each other in the list.12246 // To help reduce the number of CSE recomputations, process all12247 // the uses of this user that we can find this way.12248 do {12249 SDUse &Use = *UI;12250 ++UI;12251 Use.setNode(To);12252 if (To->isDivergent() != From->isDivergent())12253 updateDivergence(User);12254 } while (UI != UE && UI->getUser() == User);12255 12256 // Now that we have modified User, add it back to the CSE maps. If it12257 // already exists there, recursively merge the results together.12258 AddModifiedNodeToCSEMaps(User);12259 }12260 12261 // If we just RAUW'd the root, take note.12262 if (From == getRoot().getNode())12263 setRoot(SDValue(To, getRoot().getResNo()));12264}12265 12266/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.12267/// This can cause recursive merging of nodes in the DAG.12268///12269/// This version can replace From with any result values. To must match the12270/// number and types of values returned by From.12271void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {12272 if (From->getNumValues() == 1) // Handle the simple case efficiently.12273 return ReplaceAllUsesWith(SDValue(From, 0), To[0]);12274 12275 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) {12276 // Preserve Debug Info.12277 transferDbgValues(SDValue(From, i), To[i]);12278 // Preserve extra info.12279 copyExtraInfo(From, To[i].getNode());12280 }12281 12282 // Iterate over just the existing users of From. See the comments in12283 // the ReplaceAllUsesWith above.12284 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();12285 RAUWUpdateListener Listener(*this, UI, UE);12286 while (UI != UE) {12287 SDNode *User = UI->getUser();12288 12289 // This node is about to morph, remove its old self from the CSE maps.12290 RemoveNodeFromCSEMaps(User);12291 12292 // A user can appear in a use list multiple times, and when this happens the12293 // uses are usually next to each other in the list. To help reduce the12294 // number of CSE and divergence recomputations, process all the uses of this12295 // user that we can find this way.12296 bool To_IsDivergent = false;12297 do {12298 SDUse &Use = *UI;12299 const SDValue &ToOp = To[Use.getResNo()];12300 ++UI;12301 Use.set(ToOp);12302 To_IsDivergent |= ToOp->isDivergent();12303 } while (UI != UE && UI->getUser() == User);12304 12305 if (To_IsDivergent != From->isDivergent())12306 updateDivergence(User);12307 12308 // Now that we have modified User, add it back to the CSE maps. If it12309 // already exists there, recursively merge the results together.12310 AddModifiedNodeToCSEMaps(User);12311 }12312 12313 // If we just RAUW'd the root, take note.12314 if (From == getRoot().getNode())12315 setRoot(SDValue(To[getRoot().getResNo()]));12316}12317 12318/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving12319/// uses of other values produced by From.getNode() alone. The Deleted12320/// vector is handled the same way as for ReplaceAllUsesWith.12321void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){12322 // Handle the really simple, really trivial case efficiently.12323 if (From == To) return;12324 12325 // Handle the simple, trivial, case efficiently.12326 if (From.getNode()->getNumValues() == 1) {12327 ReplaceAllUsesWith(From, To);12328 return;12329 }12330 12331 // Preserve Debug Info.12332 transferDbgValues(From, To);12333 copyExtraInfo(From.getNode(), To.getNode());12334 12335 // Iterate over just the existing users of From. See the comments in12336 // the ReplaceAllUsesWith above.12337 SDNode::use_iterator UI = From.getNode()->use_begin(),12338 UE = From.getNode()->use_end();12339 RAUWUpdateListener Listener(*this, UI, UE);12340 while (UI != UE) {12341 SDNode *User = UI->getUser();12342 bool UserRemovedFromCSEMaps = false;12343 12344 // A user can appear in a use list multiple times, and when this12345 // happens the uses are usually next to each other in the list.12346 // To help reduce the number of CSE recomputations, process all12347 // the uses of this user that we can find this way.12348 do {12349 SDUse &Use = *UI;12350 12351 // Skip uses of different values from the same node.12352 if (Use.getResNo() != From.getResNo()) {12353 ++UI;12354 continue;12355 }12356 12357 // If this node hasn't been modified yet, it's still in the CSE maps,12358 // so remove its old self from the CSE maps.12359 if (!UserRemovedFromCSEMaps) {12360 RemoveNodeFromCSEMaps(User);12361 UserRemovedFromCSEMaps = true;12362 }12363 12364 ++UI;12365 Use.set(To);12366 if (To->isDivergent() != From->isDivergent())12367 updateDivergence(User);12368 } while (UI != UE && UI->getUser() == User);12369 // We are iterating over all uses of the From node, so if a use12370 // doesn't use the specific value, no changes are made.12371 if (!UserRemovedFromCSEMaps)12372 continue;12373 12374 // Now that we have modified User, add it back to the CSE maps. If it12375 // already exists there, recursively merge the results together.12376 AddModifiedNodeToCSEMaps(User);12377 }12378 12379 // If we just RAUW'd the root, take note.12380 if (From == getRoot())12381 setRoot(To);12382}12383 12384namespace {12385 12386/// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith12387/// to record information about a use.12388struct UseMemo {12389 SDNode *User;12390 unsigned Index;12391 SDUse *Use;12392};12393 12394/// operator< - Sort Memos by User.12395bool operator<(const UseMemo &L, const UseMemo &R) {12396 return (intptr_t)L.User < (intptr_t)R.User;12397}12398 12399/// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node12400/// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that12401/// the node already has been taken care of recursively.12402class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener {12403 SmallVectorImpl<UseMemo> &Uses;12404 12405 void NodeDeleted(SDNode *N, SDNode *E) override {12406 for (UseMemo &Memo : Uses)12407 if (Memo.User == N)12408 Memo.User = nullptr;12409 }12410 12411public:12412 RAUOVWUpdateListener(SelectionDAG &d, SmallVectorImpl<UseMemo> &uses)12413 : SelectionDAG::DAGUpdateListener(d), Uses(uses) {}12414};12415 12416} // end anonymous namespace12417 12418/// Return true if a glue output should propagate divergence information.12419static bool gluePropagatesDivergence(const SDNode *Node) {12420 switch (Node->getOpcode()) {12421 case ISD::CopyFromReg:12422 case ISD::CopyToReg:12423 return false;12424 default:12425 return true;12426 }12427 12428 llvm_unreachable("covered opcode switch");12429}12430 12431bool SelectionDAG::calculateDivergence(SDNode *N) {12432 if (TLI->isSDNodeAlwaysUniform(N)) {12433 assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&12434 "Conflicting divergence information!");12435 return false;12436 }12437 if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA))12438 return true;12439 for (const auto &Op : N->ops()) {12440 EVT VT = Op.getValueType();12441 12442 // Skip Chain. It does not carry divergence.12443 if (VT != MVT::Other && Op.getNode()->isDivergent() &&12444 (VT != MVT::Glue || gluePropagatesDivergence(Op.getNode())))12445 return true;12446 }12447 return false;12448}12449 12450void SelectionDAG::updateDivergence(SDNode *N) {12451 SmallVector<SDNode *, 16> Worklist(1, N);12452 do {12453 N = Worklist.pop_back_val();12454 bool IsDivergent = calculateDivergence(N);12455 if (N->SDNodeBits.IsDivergent != IsDivergent) {12456 N->SDNodeBits.IsDivergent = IsDivergent;12457 llvm::append_range(Worklist, N->users());12458 }12459 } while (!Worklist.empty());12460}12461 12462void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {12463 DenseMap<SDNode *, unsigned> Degree;12464 Order.reserve(AllNodes.size());12465 for (auto &N : allnodes()) {12466 unsigned NOps = N.getNumOperands();12467 Degree[&N] = NOps;12468 if (0 == NOps)12469 Order.push_back(&N);12470 }12471 for (size_t I = 0; I != Order.size(); ++I) {12472 SDNode *N = Order[I];12473 for (auto *U : N->users()) {12474 unsigned &UnsortedOps = Degree[U];12475 if (0 == --UnsortedOps)12476 Order.push_back(U);12477 }12478 }12479}12480 12481#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS12482void SelectionDAG::VerifyDAGDivergence() {12483 std::vector<SDNode *> TopoOrder;12484 CreateTopologicalOrder(TopoOrder);12485 for (auto *N : TopoOrder) {12486 assert(calculateDivergence(N) == N->isDivergent() &&12487 "Divergence bit inconsistency detected");12488 }12489}12490#endif12491 12492/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving12493/// uses of other values produced by From.getNode() alone. The same value12494/// may appear in both the From and To list. The Deleted vector is12495/// handled the same way as for ReplaceAllUsesWith.12496void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,12497 const SDValue *To,12498 unsigned Num){12499 // Handle the simple, trivial case efficiently.12500 if (Num == 1)12501 return ReplaceAllUsesOfValueWith(*From, *To);12502 12503 transferDbgValues(*From, *To);12504 copyExtraInfo(From->getNode(), To->getNode());12505 12506 // Read up all the uses and make records of them. This helps12507 // processing new uses that are introduced during the12508 // replacement process.12509 SmallVector<UseMemo, 4> Uses;12510 for (unsigned i = 0; i != Num; ++i) {12511 unsigned FromResNo = From[i].getResNo();12512 SDNode *FromNode = From[i].getNode();12513 for (SDUse &Use : FromNode->uses()) {12514 if (Use.getResNo() == FromResNo) {12515 UseMemo Memo = {Use.getUser(), i, &Use};12516 Uses.push_back(Memo);12517 }12518 }12519 }12520 12521 // Sort the uses, so that all the uses from a given User are together.12522 llvm::sort(Uses);12523 RAUOVWUpdateListener Listener(*this, Uses);12524 12525 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();12526 UseIndex != UseIndexEnd; ) {12527 // We know that this user uses some value of From. If it is the right12528 // value, update it.12529 SDNode *User = Uses[UseIndex].User;12530 // If the node has been deleted by recursive CSE updates when updating12531 // another node, then just skip this entry.12532 if (User == nullptr) {12533 ++UseIndex;12534 continue;12535 }12536 12537 // This node is about to morph, remove its old self from the CSE maps.12538 RemoveNodeFromCSEMaps(User);12539 12540 // The Uses array is sorted, so all the uses for a given User12541 // are next to each other in the list.12542 // To help reduce the number of CSE recomputations, process all12543 // the uses of this user that we can find this way.12544 do {12545 unsigned i = Uses[UseIndex].Index;12546 SDUse &Use = *Uses[UseIndex].Use;12547 ++UseIndex;12548 12549 Use.set(To[i]);12550 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);12551 12552 // Now that we have modified User, add it back to the CSE maps. If it12553 // already exists there, recursively merge the results together.12554 AddModifiedNodeToCSEMaps(User);12555 }12556}12557 12558/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG12559/// based on their topological order. It returns the maximum id and a vector12560/// of the SDNodes* in assigned order by reference.12561unsigned SelectionDAG::AssignTopologicalOrder() {12562 unsigned DAGSize = 0;12563 12564 // SortedPos tracks the progress of the algorithm. Nodes before it are12565 // sorted, nodes after it are unsorted. When the algorithm completes12566 // it is at the end of the list.12567 allnodes_iterator SortedPos = allnodes_begin();12568 12569 // Visit all the nodes. Move nodes with no operands to the front of12570 // the list immediately. Annotate nodes that do have operands with their12571 // operand count. Before we do this, the Node Id fields of the nodes12572 // may contain arbitrary values. After, the Node Id fields for nodes12573 // before SortedPos will contain the topological sort index, and the12574 // Node Id fields for nodes At SortedPos and after will contain the12575 // count of outstanding operands.12576 for (SDNode &N : llvm::make_early_inc_range(allnodes())) {12577 checkForCycles(&N, this);12578 unsigned Degree = N.getNumOperands();12579 if (Degree == 0) {12580 // A node with no uses, add it to the result array immediately.12581 N.setNodeId(DAGSize++);12582 allnodes_iterator Q(&N);12583 if (Q != SortedPos)12584 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));12585 assert(SortedPos != AllNodes.end() && "Overran node list");12586 ++SortedPos;12587 } else {12588 // Temporarily use the Node Id as scratch space for the degree count.12589 N.setNodeId(Degree);12590 }12591 }12592 12593 // Visit all the nodes. As we iterate, move nodes into sorted order,12594 // such that by the time the end is reached all nodes will be sorted.12595 for (SDNode &Node : allnodes()) {12596 SDNode *N = &Node;12597 checkForCycles(N, this);12598 // N is in sorted position, so all its uses have one less operand12599 // that needs to be sorted.12600 for (SDNode *P : N->users()) {12601 unsigned Degree = P->getNodeId();12602 assert(Degree != 0 && "Invalid node degree");12603 --Degree;12604 if (Degree == 0) {12605 // All of P's operands are sorted, so P may sorted now.12606 P->setNodeId(DAGSize++);12607 if (P->getIterator() != SortedPos)12608 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));12609 assert(SortedPos != AllNodes.end() && "Overran node list");12610 ++SortedPos;12611 } else {12612 // Update P's outstanding operand count.12613 P->setNodeId(Degree);12614 }12615 }12616 if (Node.getIterator() == SortedPos) {12617#ifndef NDEBUG12618 allnodes_iterator I(N);12619 SDNode *S = &*++I;12620 dbgs() << "Overran sorted position:\n";12621 S->dumprFull(this); dbgs() << "\n";12622 dbgs() << "Checking if this is due to cycles\n";12623 checkForCycles(this, true);12624#endif12625 llvm_unreachable(nullptr);12626 }12627 }12628 12629 assert(SortedPos == AllNodes.end() &&12630 "Topological sort incomplete!");12631 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&12632 "First node in topological sort is not the entry token!");12633 assert(AllNodes.front().getNodeId() == 0 &&12634 "First node in topological sort has non-zero id!");12635 assert(AllNodes.front().getNumOperands() == 0 &&12636 "First node in topological sort has operands!");12637 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&12638 "Last node in topologic sort has unexpected id!");12639 assert(AllNodes.back().use_empty() &&12640 "Last node in topologic sort has users!");12641 assert(DAGSize == allnodes_size() && "Node count mismatch!");12642 return DAGSize;12643}12644 12645void SelectionDAG::getTopologicallyOrderedNodes(12646 SmallVectorImpl<const SDNode *> &SortedNodes) const {12647 SortedNodes.clear();12648 // Node -> remaining number of outstanding operands.12649 DenseMap<const SDNode *, unsigned> RemainingOperands;12650 12651 // Put nodes without any operands into SortedNodes first.12652 for (const SDNode &N : allnodes()) {12653 checkForCycles(&N, this);12654 unsigned NumOperands = N.getNumOperands();12655 if (NumOperands == 0)12656 SortedNodes.push_back(&N);12657 else12658 // Record their total number of outstanding operands.12659 RemainingOperands[&N] = NumOperands;12660 }12661 12662 // A node is pushed into SortedNodes when all of its operands (predecessors in12663 // the graph) are also in SortedNodes.12664 for (unsigned i = 0U; i < SortedNodes.size(); ++i) {12665 const SDNode *N = SortedNodes[i];12666 for (const SDNode *U : N->users()) {12667 // HandleSDNode is never part of a DAG and therefore has no entry in12668 // RemainingOperands.12669 if (U->getOpcode() == ISD::HANDLENODE)12670 continue;12671 unsigned &NumRemOperands = RemainingOperands[U];12672 assert(NumRemOperands && "Invalid number of remaining operands");12673 --NumRemOperands;12674 if (!NumRemOperands)12675 SortedNodes.push_back(U);12676 }12677 }12678 12679 assert(SortedNodes.size() == AllNodes.size() && "Node count mismatch");12680 assert(SortedNodes.front()->getOpcode() == ISD::EntryToken &&12681 "First node in topological sort is not the entry token");12682 assert(SortedNodes.front()->getNumOperands() == 0 &&12683 "First node in topological sort has operands");12684}12685 12686/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the12687/// value is produced by SD.12688void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) {12689 for (SDNode *SD : DB->getSDNodes()) {12690 if (!SD)12691 continue;12692 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());12693 SD->setHasDebugValue(true);12694 }12695 DbgInfo->add(DB, isParameter);12696}12697 12698void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(DB); }12699 12700SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain,12701 SDValue NewMemOpChain) {12702 assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node");12703 assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT");12704 // The new memory operation must have the same position as the old load in12705 // terms of memory dependency. Create a TokenFactor for the old load and new12706 // memory operation and update uses of the old load's output chain to use that12707 // TokenFactor.12708 if (OldChain == NewMemOpChain || OldChain.use_empty())12709 return NewMemOpChain;12710 12711 SDValue TokenFactor = getNode(ISD::TokenFactor, SDLoc(OldChain), MVT::Other,12712 OldChain, NewMemOpChain);12713 ReplaceAllUsesOfValueWith(OldChain, TokenFactor);12714 UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewMemOpChain);12715 return TokenFactor;12716}12717 12718SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,12719 SDValue NewMemOp) {12720 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");12721 SDValue OldChain = SDValue(OldLoad, 1);12722 SDValue NewMemOpChain = NewMemOp.getValue(1);12723 return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain);12724}12725 12726SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,12727 Function **OutFunction) {12728 assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol");12729 12730 auto *Symbol = cast<ExternalSymbolSDNode>(Op)->getSymbol();12731 auto *Module = MF->getFunction().getParent();12732 auto *Function = Module->getFunction(Symbol);12733 12734 if (OutFunction != nullptr)12735 *OutFunction = Function;12736 12737 if (Function != nullptr) {12738 auto PtrTy = TLI->getPointerTy(getDataLayout(), Function->getAddressSpace());12739 return getGlobalAddress(Function, SDLoc(Op), PtrTy);12740 }12741 12742 std::string ErrorStr;12743 raw_string_ostream ErrorFormatter(ErrorStr);12744 ErrorFormatter << "Undefined external symbol ";12745 ErrorFormatter << '"' << Symbol << '"';12746 report_fatal_error(Twine(ErrorStr));12747}12748 12749//===----------------------------------------------------------------------===//12750// SDNode Class12751//===----------------------------------------------------------------------===//12752 12753bool llvm::isNullConstant(SDValue V) {12754 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);12755 return Const != nullptr && Const->isZero();12756}12757 12758bool llvm::isNullConstantOrUndef(SDValue V) {12759 return V.isUndef() || isNullConstant(V);12760}12761 12762bool llvm::isNullFPConstant(SDValue V) {12763 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(V);12764 return Const != nullptr && Const->isZero() && !Const->isNegative();12765}12766 12767bool llvm::isAllOnesConstant(SDValue V) {12768 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);12769 return Const != nullptr && Const->isAllOnes();12770}12771 12772bool llvm::isOneConstant(SDValue V) {12773 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);12774 return Const != nullptr && Const->isOne();12775}12776 12777bool llvm::isMinSignedConstant(SDValue V) {12778 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);12779 return Const != nullptr && Const->isMinSignedValue();12780}12781 12782bool llvm::isNeutralConstant(unsigned Opcode, SDNodeFlags Flags, SDValue V,12783 unsigned OperandNo) {12784 // NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity().12785 // TODO: Target-specific opcodes could be added.12786 if (auto *ConstV = isConstOrConstSplat(V, /*AllowUndefs*/ false,12787 /*AllowTruncation*/ true)) {12788 APInt Const = ConstV->getAPIntValue().trunc(V.getScalarValueSizeInBits());12789 switch (Opcode) {12790 case ISD::ADD:12791 case ISD::OR:12792 case ISD::XOR:12793 case ISD::UMAX:12794 return Const.isZero();12795 case ISD::MUL:12796 return Const.isOne();12797 case ISD::AND:12798 case ISD::UMIN:12799 return Const.isAllOnes();12800 case ISD::SMAX:12801 return Const.isMinSignedValue();12802 case ISD::SMIN:12803 return Const.isMaxSignedValue();12804 case ISD::SUB:12805 case ISD::SHL:12806 case ISD::SRA:12807 case ISD::SRL:12808 return OperandNo == 1 && Const.isZero();12809 case ISD::UDIV:12810 case ISD::SDIV:12811 return OperandNo == 1 && Const.isOne();12812 }12813 } else if (auto *ConstFP = isConstOrConstSplatFP(V)) {12814 switch (Opcode) {12815 case ISD::FADD:12816 return ConstFP->isZero() &&12817 (Flags.hasNoSignedZeros() || ConstFP->isNegative());12818 case ISD::FSUB:12819 return OperandNo == 1 && ConstFP->isZero() &&12820 (Flags.hasNoSignedZeros() || !ConstFP->isNegative());12821 case ISD::FMUL:12822 return ConstFP->isExactlyValue(1.0);12823 case ISD::FDIV:12824 return OperandNo == 1 && ConstFP->isExactlyValue(1.0);12825 case ISD::FMINNUM:12826 case ISD::FMAXNUM: {12827 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.12828 EVT VT = V.getValueType();12829 const fltSemantics &Semantics = VT.getFltSemantics();12830 APFloat NeutralAF = !Flags.hasNoNaNs()12831 ? APFloat::getQNaN(Semantics)12832 : !Flags.hasNoInfs()12833 ? APFloat::getInf(Semantics)12834 : APFloat::getLargest(Semantics);12835 if (Opcode == ISD::FMAXNUM)12836 NeutralAF.changeSign();12837 12838 return ConstFP->isExactlyValue(NeutralAF);12839 }12840 }12841 }12842 return false;12843}12844 12845SDValue llvm::peekThroughBitcasts(SDValue V) {12846 while (V.getOpcode() == ISD::BITCAST)12847 V = V.getOperand(0);12848 return V;12849}12850 12851SDValue llvm::peekThroughOneUseBitcasts(SDValue V) {12852 while (V.getOpcode() == ISD::BITCAST && V.getOperand(0).hasOneUse())12853 V = V.getOperand(0);12854 return V;12855}12856 12857SDValue llvm::peekThroughExtractSubvectors(SDValue V) {12858 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)12859 V = V.getOperand(0);12860 return V;12861}12862 12863SDValue llvm::peekThroughInsertVectorElt(SDValue V, const APInt &DemandedElts) {12864 while (V.getOpcode() == ISD::INSERT_VECTOR_ELT) {12865 SDValue InVec = V.getOperand(0);12866 SDValue EltNo = V.getOperand(2);12867 EVT VT = InVec.getValueType();12868 auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);12869 if (IndexC && VT.isFixedLengthVector() &&12870 IndexC->getAPIntValue().ult(VT.getVectorNumElements()) &&12871 !DemandedElts[IndexC->getZExtValue()]) {12872 V = InVec;12873 continue;12874 }12875 break;12876 }12877 return V;12878}12879 12880SDValue llvm::peekThroughTruncates(SDValue V) {12881 while (V.getOpcode() == ISD::TRUNCATE)12882 V = V.getOperand(0);12883 return V;12884}12885 12886bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {12887 if (V.getOpcode() != ISD::XOR)12888 return false;12889 V = peekThroughBitcasts(V.getOperand(1));12890 unsigned NumBits = V.getScalarValueSizeInBits();12891 ConstantSDNode *C =12892 isConstOrConstSplat(V, AllowUndefs, /*AllowTruncation*/ true);12893 return C && (C->getAPIntValue().countr_one() >= NumBits);12894}12895 12896ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs,12897 bool AllowTruncation) {12898 EVT VT = N.getValueType();12899 APInt DemandedElts = VT.isFixedLengthVector()12900 ? APInt::getAllOnes(VT.getVectorMinNumElements())12901 : APInt(1, 1);12902 return isConstOrConstSplat(N, DemandedElts, AllowUndefs, AllowTruncation);12903}12904 12905ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,12906 bool AllowUndefs,12907 bool AllowTruncation) {12908 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))12909 return CN;12910 12911 // SplatVectors can truncate their operands. Ignore that case here unless12912 // AllowTruncation is set.12913 if (N->getOpcode() == ISD::SPLAT_VECTOR) {12914 EVT VecEltVT = N->getValueType(0).getVectorElementType();12915 if (auto *CN = dyn_cast<ConstantSDNode>(N->getOperand(0))) {12916 EVT CVT = CN->getValueType(0);12917 assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension");12918 if (AllowTruncation || CVT == VecEltVT)12919 return CN;12920 }12921 }12922 12923 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {12924 BitVector UndefElements;12925 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, &UndefElements);12926 12927 // BuildVectors can truncate their operands. Ignore that case here unless12928 // AllowTruncation is set.12929 // TODO: Look into whether we should allow UndefElements in non-DemandedElts12930 if (CN && (UndefElements.none() || AllowUndefs)) {12931 EVT CVT = CN->getValueType(0);12932 EVT NSVT = N.getValueType().getScalarType();12933 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");12934 if (AllowTruncation || (CVT == NSVT))12935 return CN;12936 }12937 }12938 12939 return nullptr;12940}12941 12942ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) {12943 EVT VT = N.getValueType();12944 APInt DemandedElts = VT.isFixedLengthVector()12945 ? APInt::getAllOnes(VT.getVectorMinNumElements())12946 : APInt(1, 1);12947 return isConstOrConstSplatFP(N, DemandedElts, AllowUndefs);12948}12949 12950ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N,12951 const APInt &DemandedElts,12952 bool AllowUndefs) {12953 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N))12954 return CN;12955 12956 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {12957 BitVector UndefElements;12958 ConstantFPSDNode *CN =12959 BV->getConstantFPSplatNode(DemandedElts, &UndefElements);12960 // TODO: Look into whether we should allow UndefElements in non-DemandedElts12961 if (CN && (UndefElements.none() || AllowUndefs))12962 return CN;12963 }12964 12965 if (N.getOpcode() == ISD::SPLAT_VECTOR)12966 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N.getOperand(0)))12967 return CN;12968 12969 return nullptr;12970}12971 12972bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {12973 // TODO: may want to use peekThroughBitcast() here.12974 ConstantSDNode *C =12975 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);12976 return C && C->isZero();12977}12978 12979bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) {12980 ConstantSDNode *C =12981 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation*/ true);12982 return C && C->isOne();12983}12984 12985bool llvm::isOneOrOneSplatFP(SDValue N, bool AllowUndefs) {12986 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);12987 return C && C->isExactlyValue(1.0);12988}12989 12990bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {12991 N = peekThroughBitcasts(N);12992 unsigned BitWidth = N.getScalarValueSizeInBits();12993 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);12994 return C && C->isAllOnes() && C->getValueSizeInBits(0) == BitWidth;12995}12996 12997bool llvm::isOnesOrOnesSplat(SDValue N, bool AllowUndefs) {12998 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);12999 return C && APInt::isSameValue(C->getAPIntValue(),13000 APInt(C->getAPIntValue().getBitWidth(), 1));13001}13002 13003bool llvm::isZeroOrZeroSplat(SDValue N, bool AllowUndefs) {13004 N = peekThroughBitcasts(N);13005 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs, true);13006 return C && C->isZero();13007}13008 13009bool llvm::isZeroOrZeroSplatFP(SDValue N, bool AllowUndefs) {13010 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);13011 return C && C->isZero();13012}13013 13014HandleSDNode::~HandleSDNode() {13015 DropOperands();13016}13017 13018MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,13019 SDVTList VTs, EVT memvt, MachineMemOperand *mmo)13020 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {13021 MemSDNodeBits.IsVolatile = MMO->isVolatile();13022 MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal();13023 MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable();13024 MemSDNodeBits.IsInvariant = MMO->isInvariant();13025 13026 // We check here that the size of the memory operand fits within the size of13027 // the MMO. This is because the MMO might indicate only a possible address13028 // range instead of specifying the affected memory addresses precisely.13029 assert(13030 (!MMO->getType().isValid() ||13031 TypeSize::isKnownLE(memvt.getStoreSize(), MMO->getSize().getValue())) &&13032 "Size mismatch!");13033}13034 13035/// Profile - Gather unique data for the node.13036///13037void SDNode::Profile(FoldingSetNodeID &ID) const {13038 AddNodeIDNode(ID, this);13039}13040 13041namespace {13042 13043 struct EVTArray {13044 std::vector<EVT> VTs;13045 13046 EVTArray() {13047 VTs.reserve(MVT::VALUETYPE_SIZE);13048 for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i)13049 VTs.push_back(MVT((MVT::SimpleValueType)i));13050 }13051 };13052 13053} // end anonymous namespace13054 13055/// getValueTypeList - Return a pointer to the specified value type.13056///13057const EVT *SDNode::getValueTypeList(MVT VT) {13058 static EVTArray SimpleVTArray;13059 13060 assert(VT < MVT::VALUETYPE_SIZE && "Value type out of range!");13061 return &SimpleVTArray.VTs[VT.SimpleTy];13062}13063 13064/// hasAnyUseOfValue - Return true if there are any use of the indicated13065/// value. This method ignores uses of other values defined by this operation.13066bool SDNode::hasAnyUseOfValue(unsigned Value) const {13067 assert(Value < getNumValues() && "Bad value!");13068 13069 for (SDUse &U : uses())13070 if (U.getResNo() == Value)13071 return true;13072 13073 return false;13074}13075 13076/// isOnlyUserOf - Return true if this node is the only use of N.13077bool SDNode::isOnlyUserOf(const SDNode *N) const {13078 bool Seen = false;13079 for (const SDNode *User : N->users()) {13080 if (User == this)13081 Seen = true;13082 else13083 return false;13084 }13085 13086 return Seen;13087}13088 13089/// Return true if the only users of N are contained in Nodes.13090bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {13091 bool Seen = false;13092 for (const SDNode *User : N->users()) {13093 if (llvm::is_contained(Nodes, User))13094 Seen = true;13095 else13096 return false;13097 }13098 13099 return Seen;13100}13101 13102/// Return true if the referenced return value is an operand of N.13103bool SDValue::isOperandOf(const SDNode *N) const {13104 return is_contained(N->op_values(), *this);13105}13106 13107bool SDNode::isOperandOf(const SDNode *N) const {13108 return any_of(N->op_values(),13109 [this](SDValue Op) { return this == Op.getNode(); });13110}13111 13112/// reachesChainWithoutSideEffects - Return true if this operand (which must13113/// be a chain) reaches the specified operand without crossing any13114/// side-effecting instructions on any chain path. In practice, this looks13115/// through token factors and non-volatile loads. In order to remain efficient,13116/// this only looks a couple of nodes in, it does not do an exhaustive search.13117///13118/// Note that we only need to examine chains when we're searching for13119/// side-effects; SelectionDAG requires that all side-effects are represented13120/// by chains, even if another operand would force a specific ordering. This13121/// constraint is necessary to allow transformations like splitting loads.13122bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,13123 unsigned Depth) const {13124 if (*this == Dest) return true;13125 13126 // Don't search too deeply, we just want to be able to see through13127 // TokenFactor's etc.13128 if (Depth == 0) return false;13129 13130 // If this is a token factor, all inputs to the TF happen in parallel.13131 if (getOpcode() == ISD::TokenFactor) {13132 // First, try a shallow search.13133 if (is_contained((*this)->ops(), Dest)) {13134 // We found the chain we want as an operand of this TokenFactor.13135 // Essentially, we reach the chain without side-effects if we could13136 // serialize the TokenFactor into a simple chain of operations with13137 // Dest as the last operation. This is automatically true if the13138 // chain has one use: there are no other ordering constraints.13139 // If the chain has more than one use, we give up: some other13140 // use of Dest might force a side-effect between Dest and the current13141 // node.13142 if (Dest.hasOneUse())13143 return true;13144 }13145 // Next, try a deep search: check whether every operand of the TokenFactor13146 // reaches Dest.13147 return llvm::all_of((*this)->ops(), [=](SDValue Op) {13148 return Op.reachesChainWithoutSideEffects(Dest, Depth - 1);13149 });13150 }13151 13152 // Loads don't have side effects, look through them.13153 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) {13154 if (Ld->isUnordered())13155 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1);13156 }13157 return false;13158}13159 13160bool SDNode::hasPredecessor(const SDNode *N) const {13161 SmallPtrSet<const SDNode *, 32> Visited;13162 SmallVector<const SDNode *, 16> Worklist;13163 Worklist.push_back(this);13164 return hasPredecessorHelper(N, Visited, Worklist);13165}13166 13167void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {13168 this->Flags &= Flags;13169}13170 13171SDValue13172SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,13173 ArrayRef<ISD::NodeType> CandidateBinOps,13174 bool AllowPartials) {13175 // The pattern must end in an extract from index 0.13176 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||13177 !isNullConstant(Extract->getOperand(1)))13178 return SDValue();13179 13180 // Match against one of the candidate binary ops.13181 SDValue Op = Extract->getOperand(0);13182 if (llvm::none_of(CandidateBinOps, [Op](ISD::NodeType BinOp) {13183 return Op.getOpcode() == unsigned(BinOp);13184 }))13185 return SDValue();13186 13187 // Floating-point reductions may require relaxed constraints on the final step13188 // of the reduction because they may reorder intermediate operations.13189 unsigned CandidateBinOp = Op.getOpcode();13190 if (Op.getValueType().isFloatingPoint()) {13191 SDNodeFlags Flags = Op->getFlags();13192 switch (CandidateBinOp) {13193 case ISD::FADD:13194 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())13195 return SDValue();13196 break;13197 default:13198 llvm_unreachable("Unhandled FP opcode for binop reduction");13199 }13200 }13201 13202 // Matching failed - attempt to see if we did enough stages that a partial13203 // reduction from a subvector is possible.13204 auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) {13205 if (!AllowPartials || !Op)13206 return SDValue();13207 EVT OpVT = Op.getValueType();13208 EVT OpSVT = OpVT.getScalarType();13209 EVT SubVT = EVT::getVectorVT(*getContext(), OpSVT, NumSubElts);13210 if (!TLI->isExtractSubvectorCheap(SubVT, OpVT, 0))13211 return SDValue();13212 BinOp = (ISD::NodeType)CandidateBinOp;13213 return getExtractSubvector(SDLoc(Op), SubVT, Op, 0);13214 };13215 13216 // At each stage, we're looking for something that looks like:13217 // %s = shufflevector <8 x i32> %op, <8 x i32> undef,13218 // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,13219 // i32 undef, i32 undef, i32 undef, i32 undef>13220 // %a = binop <8 x i32> %op, %s13221 // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,13222 // we expect something like:13223 // <4,5,6,7,u,u,u,u>13224 // <2,3,u,u,u,u,u,u>13225 // <1,u,u,u,u,u,u,u>13226 // While a partial reduction match would be:13227 // <2,3,u,u,u,u,u,u>13228 // <1,u,u,u,u,u,u,u>13229 unsigned Stages = Log2_32(Op.getValueType().getVectorNumElements());13230 SDValue PrevOp;13231 for (unsigned i = 0; i < Stages; ++i) {13232 unsigned MaskEnd = (1 << i);13233 13234 if (Op.getOpcode() != CandidateBinOp)13235 return PartialReduction(PrevOp, MaskEnd);13236 13237 SDValue Op0 = Op.getOperand(0);13238 SDValue Op1 = Op.getOperand(1);13239 13240 ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Op0);13241 if (Shuffle) {13242 Op = Op1;13243 } else {13244 Shuffle = dyn_cast<ShuffleVectorSDNode>(Op1);13245 Op = Op0;13246 }13247 13248 // The first operand of the shuffle should be the same as the other operand13249 // of the binop.13250 if (!Shuffle || Shuffle->getOperand(0) != Op)13251 return PartialReduction(PrevOp, MaskEnd);13252 13253 // Verify the shuffle has the expected (at this stage of the pyramid) mask.13254 for (int Index = 0; Index < (int)MaskEnd; ++Index)13255 if (Shuffle->getMaskElt(Index) != (int)(MaskEnd + Index))13256 return PartialReduction(PrevOp, MaskEnd);13257 13258 PrevOp = Op;13259 }13260 13261 // Handle subvector reductions, which tend to appear after the shuffle13262 // reduction stages.13263 while (Op.getOpcode() == CandidateBinOp) {13264 unsigned NumElts = Op.getValueType().getVectorNumElements();13265 SDValue Op0 = Op.getOperand(0);13266 SDValue Op1 = Op.getOperand(1);13267 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||13268 Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||13269 Op0.getOperand(0) != Op1.getOperand(0))13270 break;13271 SDValue Src = Op0.getOperand(0);13272 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();13273 if (NumSrcElts != (2 * NumElts))13274 break;13275 if (!(Op0.getConstantOperandAPInt(1) == 0 &&13276 Op1.getConstantOperandAPInt(1) == NumElts) &&13277 !(Op1.getConstantOperandAPInt(1) == 0 &&13278 Op0.getConstantOperandAPInt(1) == NumElts))13279 break;13280 Op = Src;13281 }13282 13283 BinOp = (ISD::NodeType)CandidateBinOp;13284 return Op;13285}13286 13287SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {13288 EVT VT = N->getValueType(0);13289 EVT EltVT = VT.getVectorElementType();13290 unsigned NE = VT.getVectorNumElements();13291 13292 SDLoc dl(N);13293 13294 // If ResNE is 0, fully unroll the vector op.13295 if (ResNE == 0)13296 ResNE = NE;13297 else if (NE > ResNE)13298 NE = ResNE;13299 13300 if (N->getNumValues() == 2) {13301 SmallVector<SDValue, 8> Scalars0, Scalars1;13302 SmallVector<SDValue, 4> Operands(N->getNumOperands());13303 EVT VT1 = N->getValueType(1);13304 EVT EltVT1 = VT1.getVectorElementType();13305 13306 unsigned i;13307 for (i = 0; i != NE; ++i) {13308 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {13309 SDValue Operand = N->getOperand(j);13310 EVT OperandVT = Operand.getValueType();13311 13312 // A vector operand; extract a single element.13313 EVT OperandEltVT = OperandVT.getVectorElementType();13314 Operands[j] = getExtractVectorElt(dl, OperandEltVT, Operand, i);13315 }13316 13317 SDValue EltOp = getNode(N->getOpcode(), dl, {EltVT, EltVT1}, Operands);13318 Scalars0.push_back(EltOp);13319 Scalars1.push_back(EltOp.getValue(1));13320 }13321 13322 for (; i < ResNE; ++i) {13323 Scalars0.push_back(getUNDEF(EltVT));13324 Scalars1.push_back(getUNDEF(EltVT1));13325 }13326 13327 EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE);13328 EVT VecVT1 = EVT::getVectorVT(*getContext(), EltVT1, ResNE);13329 SDValue Vec0 = getBuildVector(VecVT, dl, Scalars0);13330 SDValue Vec1 = getBuildVector(VecVT1, dl, Scalars1);13331 return getMergeValues({Vec0, Vec1}, dl);13332 }13333 13334 assert(N->getNumValues() == 1 &&13335 "Can't unroll a vector with multiple results!");13336 13337 SmallVector<SDValue, 8> Scalars;13338 SmallVector<SDValue, 4> Operands(N->getNumOperands());13339 13340 unsigned i;13341 for (i= 0; i != NE; ++i) {13342 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {13343 SDValue Operand = N->getOperand(j);13344 EVT OperandVT = Operand.getValueType();13345 if (OperandVT.isVector()) {13346 // A vector operand; extract a single element.13347 EVT OperandEltVT = OperandVT.getVectorElementType();13348 Operands[j] = getExtractVectorElt(dl, OperandEltVT, Operand, i);13349 } else {13350 // A scalar operand; just use it as is.13351 Operands[j] = Operand;13352 }13353 }13354 13355 switch (N->getOpcode()) {13356 default: {13357 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands,13358 N->getFlags()));13359 break;13360 }13361 case ISD::VSELECT:13362 Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands));13363 break;13364 case ISD::SHL:13365 case ISD::SRA:13366 case ISD::SRL:13367 case ISD::ROTL:13368 case ISD::ROTR:13369 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0],13370 getShiftAmountOperand(Operands[0].getValueType(),13371 Operands[1])));13372 break;13373 case ISD::SIGN_EXTEND_INREG: {13374 EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType();13375 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,13376 Operands[0],13377 getValueType(ExtVT)));13378 break;13379 }13380 case ISD::ADDRSPACECAST: {13381 const auto *ASC = cast<AddrSpaceCastSDNode>(N);13382 Scalars.push_back(getAddrSpaceCast(dl, EltVT, Operands[0],13383 ASC->getSrcAddressSpace(),13384 ASC->getDestAddressSpace()));13385 break;13386 }13387 }13388 }13389 13390 for (; i < ResNE; ++i)13391 Scalars.push_back(getUNDEF(EltVT));13392 13393 EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE);13394 return getBuildVector(VecVT, dl, Scalars);13395}13396 13397std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(13398 SDNode *N, unsigned ResNE) {13399 unsigned Opcode = N->getOpcode();13400 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||13401 Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||13402 Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&13403 "Expected an overflow opcode");13404 13405 EVT ResVT = N->getValueType(0);13406 EVT OvVT = N->getValueType(1);13407 EVT ResEltVT = ResVT.getVectorElementType();13408 EVT OvEltVT = OvVT.getVectorElementType();13409 SDLoc dl(N);13410 13411 // If ResNE is 0, fully unroll the vector op.13412 unsigned NE = ResVT.getVectorNumElements();13413 if (ResNE == 0)13414 ResNE = NE;13415 else if (NE > ResNE)13416 NE = ResNE;13417 13418 SmallVector<SDValue, 8> LHSScalars;13419 SmallVector<SDValue, 8> RHSScalars;13420 ExtractVectorElements(N->getOperand(0), LHSScalars, 0, NE);13421 ExtractVectorElements(N->getOperand(1), RHSScalars, 0, NE);13422 13423 EVT SVT = TLI->getSetCCResultType(getDataLayout(), *getContext(), ResEltVT);13424 SDVTList VTs = getVTList(ResEltVT, SVT);13425 SmallVector<SDValue, 8> ResScalars;13426 SmallVector<SDValue, 8> OvScalars;13427 for (unsigned i = 0; i < NE; ++i) {13428 SDValue Res = getNode(Opcode, dl, VTs, LHSScalars[i], RHSScalars[i]);13429 SDValue Ov =13430 getSelect(dl, OvEltVT, Res.getValue(1),13431 getBoolConstant(true, dl, OvEltVT, ResVT),13432 getConstant(0, dl, OvEltVT));13433 13434 ResScalars.push_back(Res);13435 OvScalars.push_back(Ov);13436 }13437 13438 ResScalars.append(ResNE - NE, getUNDEF(ResEltVT));13439 OvScalars.append(ResNE - NE, getUNDEF(OvEltVT));13440 13441 EVT NewResVT = EVT::getVectorVT(*getContext(), ResEltVT, ResNE);13442 EVT NewOvVT = EVT::getVectorVT(*getContext(), OvEltVT, ResNE);13443 return std::make_pair(getBuildVector(NewResVT, dl, ResScalars),13444 getBuildVector(NewOvVT, dl, OvScalars));13445}13446 13447bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,13448 LoadSDNode *Base,13449 unsigned Bytes,13450 int Dist) const {13451 if (LD->isVolatile() || Base->isVolatile())13452 return false;13453 // TODO: probably too restrictive for atomics, revisit13454 if (!LD->isSimple())13455 return false;13456 if (LD->isIndexed() || Base->isIndexed())13457 return false;13458 if (LD->getChain() != Base->getChain())13459 return false;13460 EVT VT = LD->getMemoryVT();13461 if (VT.getSizeInBits() / 8 != Bytes)13462 return false;13463 13464 auto BaseLocDecomp = BaseIndexOffset::match(Base, *this);13465 auto LocDecomp = BaseIndexOffset::match(LD, *this);13466 13467 int64_t Offset = 0;13468 if (BaseLocDecomp.equalBaseIndex(LocDecomp, *this, Offset))13469 return (Dist * (int64_t)Bytes == Offset);13470 return false;13471}13472 13473/// InferPtrAlignment - Infer alignment of a load / store address. Return13474/// std::nullopt if it cannot be inferred.13475MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const {13476 // If this is a GlobalAddress + cst, return the alignment.13477 const GlobalValue *GV = nullptr;13478 int64_t GVOffset = 0;13479 if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) {13480 unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());13481 KnownBits Known(PtrWidth);13482 llvm::computeKnownBits(GV, Known, getDataLayout());13483 unsigned AlignBits = Known.countMinTrailingZeros();13484 if (AlignBits)13485 return commonAlignment(Align(1ull << std::min(31U, AlignBits)), GVOffset);13486 }13487 13488 // If this is a direct reference to a stack slot, use information about the13489 // stack slot's alignment.13490 int FrameIdx = INT_MIN;13491 int64_t FrameOffset = 0;13492 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) {13493 FrameIdx = FI->getIndex();13494 } else if (isBaseWithConstantOffset(Ptr) &&13495 isa<FrameIndexSDNode>(Ptr.getOperand(0))) {13496 // Handle FI+Cst13497 FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();13498 FrameOffset = Ptr.getConstantOperandVal(1);13499 }13500 13501 if (FrameIdx != INT_MIN) {13502 const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();13503 return commonAlignment(MFI.getObjectAlign(FrameIdx), FrameOffset);13504 }13505 13506 return std::nullopt;13507}13508 13509/// Split the scalar node with EXTRACT_ELEMENT using the provided13510/// VTs and return the low/high part.13511std::pair<SDValue, SDValue> SelectionDAG::SplitScalar(const SDValue &N,13512 const SDLoc &DL,13513 const EVT &LoVT,13514 const EVT &HiVT) {13515 assert(!LoVT.isVector() && !HiVT.isVector() && !N.getValueType().isVector() &&13516 "Split node must be a scalar type");13517 SDValue Lo =13518 getNode(ISD::EXTRACT_ELEMENT, DL, LoVT, N, getIntPtrConstant(0, DL));13519 SDValue Hi =13520 getNode(ISD::EXTRACT_ELEMENT, DL, HiVT, N, getIntPtrConstant(1, DL));13521 return std::make_pair(Lo, Hi);13522}13523 13524/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type13525/// which is split (or expanded) into two not necessarily identical pieces.13526std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {13527 // Currently all types are split in half.13528 EVT LoVT, HiVT;13529 if (!VT.isVector())13530 LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT);13531 else13532 LoVT = HiVT = VT.getHalfNumVectorElementsVT(*getContext());13533 13534 return std::make_pair(LoVT, HiVT);13535}13536 13537/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a13538/// type, dependent on an enveloping VT that has been split into two identical13539/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.13540std::pair<EVT, EVT>13541SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,13542 bool *HiIsEmpty) const {13543 EVT EltTp = VT.getVectorElementType();13544 // Examples:13545 // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty)13546 // custom VL=9 with enveloping VL=8/8 yields 8/113547 // custom VL=10 with enveloping VL=8/8 yields 8/213548 // etc.13549 ElementCount VTNumElts = VT.getVectorElementCount();13550 ElementCount EnvNumElts = EnvVT.getVectorElementCount();13551 assert(VTNumElts.isScalable() == EnvNumElts.isScalable() &&13552 "Mixing fixed width and scalable vectors when enveloping a type");13553 EVT LoVT, HiVT;13554 if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) {13555 LoVT = EVT::getVectorVT(*getContext(), EltTp, EnvNumElts);13556 HiVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts - EnvNumElts);13557 *HiIsEmpty = false;13558 } else {13559 // Flag that hi type has zero storage size, but return split envelop type13560 // (this would be easier if vector types with zero elements were allowed).13561 LoVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts);13562 HiVT = EVT::getVectorVT(*getContext(), EltTp, EnvNumElts);13563 *HiIsEmpty = true;13564 }13565 return std::make_pair(LoVT, HiVT);13566}13567 13568/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the13569/// low/high part.13570std::pair<SDValue, SDValue>13571SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,13572 const EVT &HiVT) {13573 assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&13574 LoVT.isScalableVector() == N.getValueType().isScalableVector() &&13575 "Splitting vector with an invalid mixture of fixed and scalable "13576 "vector types");13577 assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <=13578 N.getValueType().getVectorMinNumElements() &&13579 "More vector elements requested than available!");13580 SDValue Lo, Hi;13581 Lo = getExtractSubvector(DL, LoVT, N, 0);13582 // For scalable vectors it is safe to use LoVT.getVectorMinNumElements()13583 // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales13584 // IDX with the runtime scaling factor of the result vector type. For13585 // fixed-width result vectors, that runtime scaling factor is 1.13586 Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N,13587 getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL));13588 return std::make_pair(Lo, Hi);13589}13590 13591std::pair<SDValue, SDValue> SelectionDAG::SplitEVL(SDValue N, EVT VecVT,13592 const SDLoc &DL) {13593 // Split the vector length parameter.13594 // %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts).13595 EVT VT = N.getValueType();13596 assert(VecVT.getVectorElementCount().isKnownEven() &&13597 "Expecting the mask to be an evenly-sized vector");13598 SDValue HalfNumElts = getElementCount(13599 DL, VT, VecVT.getVectorElementCount().divideCoefficientBy(2));13600 SDValue Lo = getNode(ISD::UMIN, DL, VT, N, HalfNumElts);13601 SDValue Hi = getNode(ISD::USUBSAT, DL, VT, N, HalfNumElts);13602 return std::make_pair(Lo, Hi);13603}13604 13605/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.13606SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) {13607 EVT VT = N.getValueType();13608 EVT WideVT = EVT::getVectorVT(*getContext(), VT.getVectorElementType(),13609 NextPowerOf2(VT.getVectorNumElements()));13610 return getInsertSubvector(DL, getUNDEF(WideVT), N, 0);13611}13612 13613void SelectionDAG::ExtractVectorElements(SDValue Op,13614 SmallVectorImpl<SDValue> &Args,13615 unsigned Start, unsigned Count,13616 EVT EltVT) {13617 EVT VT = Op.getValueType();13618 if (Count == 0)13619 Count = VT.getVectorNumElements();13620 if (EltVT == EVT())13621 EltVT = VT.getVectorElementType();13622 SDLoc SL(Op);13623 for (unsigned i = Start, e = Start + Count; i != e; ++i) {13624 Args.push_back(getExtractVectorElt(SL, EltVT, Op, i));13625 }13626}13627 13628// getAddressSpace - Return the address space this GlobalAddress belongs to.13629unsigned GlobalAddressSDNode::getAddressSpace() const {13630 return getGlobal()->getType()->getAddressSpace();13631}13632 13633Type *ConstantPoolSDNode::getType() const {13634 if (isMachineConstantPoolEntry())13635 return Val.MachineCPVal->getType();13636 return Val.ConstVal->getType();13637}13638 13639bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,13640 unsigned &SplatBitSize,13641 bool &HasAnyUndefs,13642 unsigned MinSplatBits,13643 bool IsBigEndian) const {13644 EVT VT = getValueType(0);13645 assert(VT.isVector() && "Expected a vector type");13646 unsigned VecWidth = VT.getSizeInBits();13647 if (MinSplatBits > VecWidth)13648 return false;13649 13650 // FIXME: The widths are based on this node's type, but build vectors can13651 // truncate their operands.13652 SplatValue = APInt(VecWidth, 0);13653 SplatUndef = APInt(VecWidth, 0);13654 13655 // Get the bits. Bits with undefined values (when the corresponding element13656 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared13657 // in SplatValue. If any of the values are not constant, give up and return13658 // false.13659 unsigned int NumOps = getNumOperands();13660 assert(NumOps > 0 && "isConstantSplat has 0-size build vector");13661 unsigned EltWidth = VT.getScalarSizeInBits();13662 13663 for (unsigned j = 0; j < NumOps; ++j) {13664 unsigned i = IsBigEndian ? NumOps - 1 - j : j;13665 SDValue OpVal = getOperand(i);13666 unsigned BitPos = j * EltWidth;13667 13668 if (OpVal.isUndef())13669 SplatUndef.setBits(BitPos, BitPos + EltWidth);13670 else if (auto *CN = dyn_cast<ConstantSDNode>(OpVal))13671 SplatValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos);13672 else if (auto *CN = dyn_cast<ConstantFPSDNode>(OpVal))13673 SplatValue.insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos);13674 else13675 return false;13676 }13677 13678 // The build_vector is all constants or undefs. Find the smallest element13679 // size that splats the vector.13680 HasAnyUndefs = (SplatUndef != 0);13681 13682 // FIXME: This does not work for vectors with elements less than 8 bits.13683 while (VecWidth > 8) {13684 // If we can't split in half, stop here.13685 if (VecWidth & 1)13686 break;13687 13688 unsigned HalfSize = VecWidth / 2;13689 APInt HighValue = SplatValue.extractBits(HalfSize, HalfSize);13690 APInt LowValue = SplatValue.extractBits(HalfSize, 0);13691 APInt HighUndef = SplatUndef.extractBits(HalfSize, HalfSize);13692 APInt LowUndef = SplatUndef.extractBits(HalfSize, 0);13693 13694 // If the two halves do not match (ignoring undef bits), stop here.13695 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||13696 MinSplatBits > HalfSize)13697 break;13698 13699 SplatValue = HighValue | LowValue;13700 SplatUndef = HighUndef & LowUndef;13701 13702 VecWidth = HalfSize;13703 }13704 13705 // FIXME: The loop above only tries to split in halves. But if the input13706 // vector for example is <3 x i16> it wouldn't be able to detect a13707 // SplatBitSize of 16. No idea if that is a design flaw currently limiting13708 // optimizations. I guess that back in the days when this helper was created13709 // vectors normally was power-of-2 sized.13710 13711 SplatBitSize = VecWidth;13712 return true;13713}13714 13715SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts,13716 BitVector *UndefElements) const {13717 unsigned NumOps = getNumOperands();13718 if (UndefElements) {13719 UndefElements->clear();13720 UndefElements->resize(NumOps);13721 }13722 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");13723 if (!DemandedElts)13724 return SDValue();13725 SDValue Splatted;13726 for (unsigned i = 0; i != NumOps; ++i) {13727 if (!DemandedElts[i])13728 continue;13729 SDValue Op = getOperand(i);13730 if (Op.isUndef()) {13731 if (UndefElements)13732 (*UndefElements)[i] = true;13733 } else if (!Splatted) {13734 Splatted = Op;13735 } else if (Splatted != Op) {13736 return SDValue();13737 }13738 }13739 13740 if (!Splatted) {13741 unsigned FirstDemandedIdx = DemandedElts.countr_zero();13742 assert(getOperand(FirstDemandedIdx).isUndef() &&13743 "Can only have a splat without a constant for all undefs.");13744 return getOperand(FirstDemandedIdx);13745 }13746 13747 return Splatted;13748}13749 13750SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {13751 APInt DemandedElts = APInt::getAllOnes(getNumOperands());13752 return getSplatValue(DemandedElts, UndefElements);13753}13754 13755bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts,13756 SmallVectorImpl<SDValue> &Sequence,13757 BitVector *UndefElements) const {13758 unsigned NumOps = getNumOperands();13759 Sequence.clear();13760 if (UndefElements) {13761 UndefElements->clear();13762 UndefElements->resize(NumOps);13763 }13764 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");13765 if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(NumOps))13766 return false;13767 13768 // Set the undefs even if we don't find a sequence (like getSplatValue).13769 if (UndefElements)13770 for (unsigned I = 0; I != NumOps; ++I)13771 if (DemandedElts[I] && getOperand(I).isUndef())13772 (*UndefElements)[I] = true;13773 13774 // Iteratively widen the sequence length looking for repetitions.13775 for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {13776 Sequence.append(SeqLen, SDValue());13777 for (unsigned I = 0; I != NumOps; ++I) {13778 if (!DemandedElts[I])13779 continue;13780 SDValue &SeqOp = Sequence[I % SeqLen];13781 SDValue Op = getOperand(I);13782 if (Op.isUndef()) {13783 if (!SeqOp)13784 SeqOp = Op;13785 continue;13786 }13787 if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) {13788 Sequence.clear();13789 break;13790 }13791 SeqOp = Op;13792 }13793 if (!Sequence.empty())13794 return true;13795 }13796 13797 assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern");13798 return false;13799}13800 13801bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,13802 BitVector *UndefElements) const {13803 APInt DemandedElts = APInt::getAllOnes(getNumOperands());13804 return getRepeatedSequence(DemandedElts, Sequence, UndefElements);13805}13806 13807ConstantSDNode *13808BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts,13809 BitVector *UndefElements) const {13810 return dyn_cast_or_null<ConstantSDNode>(13811 getSplatValue(DemandedElts, UndefElements));13812}13813 13814ConstantSDNode *13815BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {13816 return dyn_cast_or_null<ConstantSDNode>(getSplatValue(UndefElements));13817}13818 13819ConstantFPSDNode *13820BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts,13821 BitVector *UndefElements) const {13822 return dyn_cast_or_null<ConstantFPSDNode>(13823 getSplatValue(DemandedElts, UndefElements));13824}13825 13826ConstantFPSDNode *13827BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {13828 return dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements));13829}13830 13831int32_t13832BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,13833 uint32_t BitWidth) const {13834 if (ConstantFPSDNode *CN =13835 dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements))) {13836 bool IsExact;13837 APSInt IntVal(BitWidth);13838 const APFloat &APF = CN->getValueAPF();13839 if (APF.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) !=13840 APFloat::opOK ||13841 !IsExact)13842 return -1;13843 13844 return IntVal.exactLogBase2();13845 }13846 return -1;13847}13848 13849bool BuildVectorSDNode::getConstantRawBits(13850 bool IsLittleEndian, unsigned DstEltSizeInBits,13851 SmallVectorImpl<APInt> &RawBitElements, BitVector &UndefElements) const {13852 // Early-out if this contains anything but Undef/Constant/ConstantFP.13853 if (!isConstant())13854 return false;13855 13856 unsigned NumSrcOps = getNumOperands();13857 unsigned SrcEltSizeInBits = getValueType(0).getScalarSizeInBits();13858 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&13859 "Invalid bitcast scale");13860 13861 // Extract raw src bits.13862 SmallVector<APInt> SrcBitElements(NumSrcOps,13863 APInt::getZero(SrcEltSizeInBits));13864 BitVector SrcUndeElements(NumSrcOps, false);13865 13866 for (unsigned I = 0; I != NumSrcOps; ++I) {13867 SDValue Op = getOperand(I);13868 if (Op.isUndef()) {13869 SrcUndeElements.set(I);13870 continue;13871 }13872 auto *CInt = dyn_cast<ConstantSDNode>(Op);13873 auto *CFP = dyn_cast<ConstantFPSDNode>(Op);13874 assert((CInt || CFP) && "Unknown constant");13875 SrcBitElements[I] = CInt ? CInt->getAPIntValue().trunc(SrcEltSizeInBits)13876 : CFP->getValueAPF().bitcastToAPInt();13877 }13878 13879 // Recast to dst width.13880 recastRawBits(IsLittleEndian, DstEltSizeInBits, RawBitElements,13881 SrcBitElements, UndefElements, SrcUndeElements);13882 return true;13883}13884 13885void BuildVectorSDNode::recastRawBits(bool IsLittleEndian,13886 unsigned DstEltSizeInBits,13887 SmallVectorImpl<APInt> &DstBitElements,13888 ArrayRef<APInt> SrcBitElements,13889 BitVector &DstUndefElements,13890 const BitVector &SrcUndefElements) {13891 unsigned NumSrcOps = SrcBitElements.size();13892 unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth();13893 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&13894 "Invalid bitcast scale");13895 assert(NumSrcOps == SrcUndefElements.size() &&13896 "Vector size mismatch");13897 13898 unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits;13899 DstUndefElements.clear();13900 DstUndefElements.resize(NumDstOps, false);13901 DstBitElements.assign(NumDstOps, APInt::getZero(DstEltSizeInBits));13902 13903 // Concatenate src elements constant bits together into dst element.13904 if (SrcEltSizeInBits <= DstEltSizeInBits) {13905 unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits;13906 for (unsigned I = 0; I != NumDstOps; ++I) {13907 DstUndefElements.set(I);13908 APInt &DstBits = DstBitElements[I];13909 for (unsigned J = 0; J != Scale; ++J) {13910 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));13911 if (SrcUndefElements[Idx])13912 continue;13913 DstUndefElements.reset(I);13914 const APInt &SrcBits = SrcBitElements[Idx];13915 assert(SrcBits.getBitWidth() == SrcEltSizeInBits &&13916 "Illegal constant bitwidths");13917 DstBits.insertBits(SrcBits, J * SrcEltSizeInBits);13918 }13919 }13920 return;13921 }13922 13923 // Split src element constant bits into dst elements.13924 unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits;13925 for (unsigned I = 0; I != NumSrcOps; ++I) {13926 if (SrcUndefElements[I]) {13927 DstUndefElements.set(I * Scale, (I + 1) * Scale);13928 continue;13929 }13930 const APInt &SrcBits = SrcBitElements[I];13931 for (unsigned J = 0; J != Scale; ++J) {13932 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));13933 APInt &DstBits = DstBitElements[Idx];13934 DstBits = SrcBits.extractBits(DstEltSizeInBits, J * DstEltSizeInBits);13935 }13936 }13937}13938 13939bool BuildVectorSDNode::isConstant() const {13940 for (const SDValue &Op : op_values()) {13941 unsigned Opc = Op.getOpcode();13942 if (!Op.isUndef() && Opc != ISD::Constant && Opc != ISD::ConstantFP)13943 return false;13944 }13945 return true;13946}13947 13948std::optional<std::pair<APInt, APInt>>13949BuildVectorSDNode::isConstantSequence() const {13950 unsigned NumOps = getNumOperands();13951 if (NumOps < 2)13952 return std::nullopt;13953 13954 if (!isa<ConstantSDNode>(getOperand(0)) ||13955 !isa<ConstantSDNode>(getOperand(1)))13956 return std::nullopt;13957 13958 unsigned EltSize = getValueType(0).getScalarSizeInBits();13959 APInt Start = getConstantOperandAPInt(0).trunc(EltSize);13960 APInt Stride = getConstantOperandAPInt(1).trunc(EltSize) - Start;13961 13962 if (Stride.isZero())13963 return std::nullopt;13964 13965 for (unsigned i = 2; i < NumOps; ++i) {13966 if (!isa<ConstantSDNode>(getOperand(i)))13967 return std::nullopt;13968 13969 APInt Val = getConstantOperandAPInt(i).trunc(EltSize);13970 if (Val != (Start + (Stride * i)))13971 return std::nullopt;13972 }13973 13974 return std::make_pair(Start, Stride);13975}13976 13977bool ShuffleVectorSDNode::isSplatMask(ArrayRef<int> Mask) {13978 // Find the first non-undef value in the shuffle mask.13979 unsigned i, e;13980 for (i = 0, e = Mask.size(); i != e && Mask[i] < 0; ++i)13981 /* search */;13982 13983 // If all elements are undefined, this shuffle can be considered a splat13984 // (although it should eventually get simplified away completely).13985 if (i == e)13986 return true;13987 13988 // Make sure all remaining elements are either undef or the same as the first13989 // non-undef value.13990 for (int Idx = Mask[i]; i != e; ++i)13991 if (Mask[i] >= 0 && Mask[i] != Idx)13992 return false;13993 return true;13994}13995 13996// Returns true if it is a constant integer BuildVector or constant integer,13997// possibly hidden by a bitcast.13998bool SelectionDAG::isConstantIntBuildVectorOrConstantInt(13999 SDValue N, bool AllowOpaques) const {14000 N = peekThroughBitcasts(N);14001 14002 if (auto *C = dyn_cast<ConstantSDNode>(N))14003 return AllowOpaques || !C->isOpaque();14004 14005 if (ISD::isBuildVectorOfConstantSDNodes(N.getNode()))14006 return true;14007 14008 // Treat a GlobalAddress supporting constant offset folding as a14009 // constant integer.14010 if (auto *GA = dyn_cast<GlobalAddressSDNode>(N))14011 if (GA->getOpcode() == ISD::GlobalAddress &&14012 TLI->isOffsetFoldingLegal(GA))14013 return true;14014 14015 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&14016 isa<ConstantSDNode>(N.getOperand(0)))14017 return true;14018 return false;14019}14020 14021// Returns true if it is a constant float BuildVector or constant float.14022bool SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const {14023 if (isa<ConstantFPSDNode>(N))14024 return true;14025 14026 if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode()))14027 return true;14028 14029 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&14030 isa<ConstantFPSDNode>(N.getOperand(0)))14031 return true;14032 14033 return false;14034}14035 14036std::optional<bool> SelectionDAG::isBoolConstant(SDValue N) const {14037 ConstantSDNode *Const =14038 isConstOrConstSplat(N, false, /*AllowTruncation=*/true);14039 if (!Const)14040 return std::nullopt;14041 14042 EVT VT = N->getValueType(0);14043 const APInt CVal = Const->getAPIntValue().trunc(VT.getScalarSizeInBits());14044 switch (TLI->getBooleanContents(N.getValueType())) {14045 case TargetLowering::ZeroOrOneBooleanContent:14046 if (CVal.isOne())14047 return true;14048 if (CVal.isZero())14049 return false;14050 return std::nullopt;14051 case TargetLowering::ZeroOrNegativeOneBooleanContent:14052 if (CVal.isAllOnes())14053 return true;14054 if (CVal.isZero())14055 return false;14056 return std::nullopt;14057 case TargetLowering::UndefinedBooleanContent:14058 return CVal[0];14059 }14060 llvm_unreachable("Unknown BooleanContent enum");14061}14062 14063void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {14064 assert(!Node->OperandList && "Node already has operands");14065 assert(SDNode::getMaxNumOperands() >= Vals.size() &&14066 "too many operands to fit into SDNode");14067 SDUse *Ops = OperandRecycler.allocate(14068 ArrayRecycler<SDUse>::Capacity::get(Vals.size()), OperandAllocator);14069 14070 bool IsDivergent = false;14071 for (unsigned I = 0; I != Vals.size(); ++I) {14072 Ops[I].setUser(Node);14073 Ops[I].setInitial(Vals[I]);14074 EVT VT = Ops[I].getValueType();14075 14076 // Skip Chain. It does not carry divergence.14077 if (VT != MVT::Other &&14078 (VT != MVT::Glue || gluePropagatesDivergence(Ops[I].getNode())) &&14079 Ops[I].getNode()->isDivergent()) {14080 IsDivergent = true;14081 }14082 }14083 Node->NumOperands = Vals.size();14084 Node->OperandList = Ops;14085 if (!TLI->isSDNodeAlwaysUniform(Node)) {14086 IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, UA);14087 Node->SDNodeBits.IsDivergent = IsDivergent;14088 }14089 checkForCycles(Node);14090}14091 14092SDValue SelectionDAG::getTokenFactor(const SDLoc &DL,14093 SmallVectorImpl<SDValue> &Vals) {14094 size_t Limit = SDNode::getMaxNumOperands();14095 while (Vals.size() > Limit) {14096 unsigned SliceIdx = Vals.size() - Limit;14097 auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(SliceIdx, Limit);14098 SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs);14099 Vals.erase(Vals.begin() + SliceIdx, Vals.end());14100 Vals.emplace_back(NewTF);14101 }14102 return getNode(ISD::TokenFactor, DL, MVT::Other, Vals);14103}14104 14105SDValue SelectionDAG::getNeutralElement(unsigned Opcode, const SDLoc &DL,14106 EVT VT, SDNodeFlags Flags) {14107 switch (Opcode) {14108 default:14109 return SDValue();14110 case ISD::ADD:14111 case ISD::OR:14112 case ISD::XOR:14113 case ISD::UMAX:14114 return getConstant(0, DL, VT);14115 case ISD::MUL:14116 return getConstant(1, DL, VT);14117 case ISD::AND:14118 case ISD::UMIN:14119 return getAllOnesConstant(DL, VT);14120 case ISD::SMAX:14121 return getConstant(APInt::getSignedMinValue(VT.getSizeInBits()), DL, VT);14122 case ISD::SMIN:14123 return getConstant(APInt::getSignedMaxValue(VT.getSizeInBits()), DL, VT);14124 case ISD::FADD:14125 // If flags allow, prefer positive zero since it's generally cheaper14126 // to materialize on most targets.14127 return getConstantFP(Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, VT);14128 case ISD::FMUL:14129 return getConstantFP(1.0, DL, VT);14130 case ISD::FMINNUM:14131 case ISD::FMAXNUM: {14132 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.14133 const fltSemantics &Semantics = VT.getFltSemantics();14134 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Semantics) :14135 !Flags.hasNoInfs() ? APFloat::getInf(Semantics) :14136 APFloat::getLargest(Semantics);14137 if (Opcode == ISD::FMAXNUM)14138 NeutralAF.changeSign();14139 14140 return getConstantFP(NeutralAF, DL, VT);14141 }14142 case ISD::FMINIMUM:14143 case ISD::FMAXIMUM: {14144 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.14145 const fltSemantics &Semantics = VT.getFltSemantics();14146 APFloat NeutralAF = !Flags.hasNoInfs() ? APFloat::getInf(Semantics)14147 : APFloat::getLargest(Semantics);14148 if (Opcode == ISD::FMAXIMUM)14149 NeutralAF.changeSign();14150 14151 return getConstantFP(NeutralAF, DL, VT);14152 }14153 14154 }14155}14156 14157/// Helper used to make a call to a library function that has one argument of14158/// pointer type.14159///14160/// Such functions include 'fegetmode', 'fesetenv' and some others, which are14161/// used to get or set floating-point state. They have one argument of pointer14162/// type, which points to the memory region containing bits of the14163/// floating-point state. The value returned by such function is ignored in the14164/// created call.14165///14166/// \param LibFunc Reference to library function (value of RTLIB::Libcall).14167/// \param Ptr Pointer used to save/load state.14168/// \param InChain Ingoing token chain.14169/// \returns Outgoing chain token.14170SDValue SelectionDAG::makeStateFunctionCall(unsigned LibFunc, SDValue Ptr,14171 SDValue InChain,14172 const SDLoc &DLoc) {14173 assert(InChain.getValueType() == MVT::Other && "Expected token chain");14174 TargetLowering::ArgListTy Args;14175 Args.emplace_back(Ptr, Ptr.getValueType().getTypeForEVT(*getContext()));14176 RTLIB::Libcall LC = static_cast<RTLIB::Libcall>(LibFunc);14177 SDValue Callee = getExternalSymbol(TLI->getLibcallName(LC),14178 TLI->getPointerTy(getDataLayout()));14179 TargetLowering::CallLoweringInfo CLI(*this);14180 CLI.setDebugLoc(DLoc).setChain(InChain).setLibCallee(14181 TLI->getLibcallCallingConv(LC), Type::getVoidTy(*getContext()), Callee,14182 std::move(Args));14183 return TLI->LowerCallTo(CLI).second;14184}14185 14186void SelectionDAG::copyExtraInfo(SDNode *From, SDNode *To) {14187 assert(From && To && "Invalid SDNode; empty source SDValue?");14188 auto I = SDEI.find(From);14189 if (I == SDEI.end())14190 return;14191 14192 // Use of operator[] on the DenseMap may cause an insertion, which invalidates14193 // the iterator, hence the need to make a copy to prevent a use-after-free.14194 NodeExtraInfo NEI = I->second;14195 if (LLVM_LIKELY(!NEI.PCSections)) {14196 // No deep copy required for the types of extra info set.14197 //14198 // FIXME: Investigate if other types of extra info also need deep copy. This14199 // depends on the types of nodes they can be attached to: if some extra info14200 // is only ever attached to nodes where a replacement To node is always the14201 // node where later use and propagation of the extra info has the intended14202 // semantics, no deep copy is required.14203 SDEI[To] = std::move(NEI);14204 return;14205 }14206 14207 const SDNode *EntrySDN = getEntryNode().getNode();14208 14209 // We need to copy NodeExtraInfo to all _new_ nodes that are being introduced14210 // through the replacement of From with To. Otherwise, replacements of a node14211 // (From) with more complex nodes (To and its operands) may result in lost14212 // extra info where the root node (To) is insignificant in further propagating14213 // and using extra info when further lowering to MIR.14214 //14215 // In the first step pre-populate the visited set with the nodes reachable14216 // from the old From node. This avoids copying NodeExtraInfo to parts of the14217 // DAG that is not new and should be left untouched.14218 SmallVector<const SDNode *> Leafs{From}; // Leafs reachable with VisitFrom.14219 DenseSet<const SDNode *> FromReach; // The set of nodes reachable from From.14220 auto VisitFrom = [&](auto &&Self, const SDNode *N, int MaxDepth) {14221 if (MaxDepth == 0) {14222 // Remember this node in case we need to increase MaxDepth and continue14223 // populating FromReach from this node.14224 Leafs.emplace_back(N);14225 return;14226 }14227 if (!FromReach.insert(N).second)14228 return;14229 for (const SDValue &Op : N->op_values())14230 Self(Self, Op.getNode(), MaxDepth - 1);14231 };14232 14233 // Copy extra info to To and all its transitive operands (that are new).14234 SmallPtrSet<const SDNode *, 8> Visited;14235 auto DeepCopyTo = [&](auto &&Self, const SDNode *N) {14236 if (FromReach.contains(N))14237 return true;14238 if (!Visited.insert(N).second)14239 return true;14240 if (EntrySDN == N)14241 return false;14242 for (const SDValue &Op : N->op_values()) {14243 if (N == To && Op.getNode() == EntrySDN) {14244 // Special case: New node's operand is the entry node; just need to14245 // copy extra info to new node.14246 break;14247 }14248 if (!Self(Self, Op.getNode()))14249 return false;14250 }14251 // Copy only if entry node was not reached.14252 SDEI[N] = NEI;14253 return true;14254 };14255 14256 // We first try with a lower MaxDepth, assuming that the path to common14257 // operands between From and To is relatively short. This significantly14258 // improves performance in the common case. The initial MaxDepth is big14259 // enough to avoid retry in the common case; the last MaxDepth is large14260 // enough to avoid having to use the fallback below (and protects from14261 // potential stack exhaustion from recursion).14262 for (int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024;14263 PrevDepth = MaxDepth, MaxDepth *= 2, Visited.clear()) {14264 // StartFrom is the previous (or initial) set of leafs reachable at the14265 // previous maximum depth.14266 SmallVector<const SDNode *> StartFrom;14267 std::swap(StartFrom, Leafs);14268 for (const SDNode *N : StartFrom)14269 VisitFrom(VisitFrom, N, MaxDepth - PrevDepth);14270 if (LLVM_LIKELY(DeepCopyTo(DeepCopyTo, To)))14271 return;14272 // This should happen very rarely (reached the entry node).14273 LLVM_DEBUG(dbgs() << __func__ << ": MaxDepth=" << MaxDepth << " too low\n");14274 assert(!Leafs.empty());14275 }14276 14277 // This should not happen - but if it did, that means the subgraph reachable14278 // from From has depth greater or equal to maximum MaxDepth, and VisitFrom()14279 // could not visit all reachable common operands. Consequently, we were able14280 // to reach the entry node.14281 errs() << "warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n";14282 assert(false && "From subgraph too complex - increase max. MaxDepth?");14283 // Best-effort fallback if assertions disabled.14284 SDEI[To] = std::move(NEI);14285}14286 14287#ifndef NDEBUG14288static void checkForCyclesHelper(const SDNode *N,14289 SmallPtrSetImpl<const SDNode*> &Visited,14290 SmallPtrSetImpl<const SDNode*> &Checked,14291 const llvm::SelectionDAG *DAG) {14292 // If this node has already been checked, don't check it again.14293 if (Checked.count(N))14294 return;14295 14296 // If a node has already been visited on this depth-first walk, reject it as14297 // a cycle.14298 if (!Visited.insert(N).second) {14299 errs() << "Detected cycle in SelectionDAG\n";14300 dbgs() << "Offending node:\n";14301 N->dumprFull(DAG); dbgs() << "\n";14302 abort();14303 }14304 14305 for (const SDValue &Op : N->op_values())14306 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);14307 14308 Checked.insert(N);14309 Visited.erase(N);14310}14311#endif14312 14313void llvm::checkForCycles(const llvm::SDNode *N,14314 const llvm::SelectionDAG *DAG,14315 bool force) {14316#ifndef NDEBUG14317 bool check = force;14318#ifdef EXPENSIVE_CHECKS14319 check = true;14320#endif // EXPENSIVE_CHECKS14321 if (check) {14322 assert(N && "Checking nonexistent SDNode");14323 SmallPtrSet<const SDNode*, 32> visited;14324 SmallPtrSet<const SDNode*, 32> checked;14325 checkForCyclesHelper(N, visited, checked, DAG);14326 }14327#endif // !NDEBUG14328}14329 14330void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {14331 checkForCycles(DAG->getRoot().getNode(), DAG, force);14332}14333