6232 lines · cpp
1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the SelectionDAG::Legalize method.10//11//===----------------------------------------------------------------------===//12 13#include "llvm/ADT/APFloat.h"14#include "llvm/ADT/APInt.h"15#include "llvm/ADT/ArrayRef.h"16#include "llvm/ADT/FloatingPointMode.h"17#include "llvm/ADT/SetVector.h"18#include "llvm/ADT/SmallPtrSet.h"19#include "llvm/ADT/SmallSet.h"20#include "llvm/ADT/SmallVector.h"21#include "llvm/Analysis/ConstantFolding.h"22#include "llvm/Analysis/TargetLibraryInfo.h"23#include "llvm/CodeGen/ISDOpcodes.h"24#include "llvm/CodeGen/MachineFrameInfo.h"25#include "llvm/CodeGen/MachineFunction.h"26#include "llvm/CodeGen/MachineJumpTableInfo.h"27#include "llvm/CodeGen/MachineMemOperand.h"28#include "llvm/CodeGen/RuntimeLibcallUtil.h"29#include "llvm/CodeGen/SelectionDAG.h"30#include "llvm/CodeGen/SelectionDAGNodes.h"31#include "llvm/CodeGen/TargetFrameLowering.h"32#include "llvm/CodeGen/TargetLowering.h"33#include "llvm/CodeGen/TargetSubtargetInfo.h"34#include "llvm/CodeGen/ValueTypes.h"35#include "llvm/CodeGenTypes/MachineValueType.h"36#include "llvm/IR/CallingConv.h"37#include "llvm/IR/Constants.h"38#include "llvm/IR/DataLayout.h"39#include "llvm/IR/DerivedTypes.h"40#include "llvm/IR/Function.h"41#include "llvm/IR/Metadata.h"42#include "llvm/IR/Type.h"43#include "llvm/Support/Casting.h"44#include "llvm/Support/Compiler.h"45#include "llvm/Support/Debug.h"46#include "llvm/Support/ErrorHandling.h"47#include "llvm/Support/MathExtras.h"48#include "llvm/Support/raw_ostream.h"49#include "llvm/Target/TargetMachine.h"50#include "llvm/Target/TargetOptions.h"51#include <cassert>52#include <cstdint>53#include <tuple>54#include <utility>55 56using namespace llvm;57 58#define DEBUG_TYPE "legalizedag"59 60namespace {61 62/// Keeps track of state when getting the sign of a floating-point value as an63/// integer.64struct FloatSignAsInt {65 EVT FloatVT;66 SDValue Chain;67 SDValue FloatPtr;68 SDValue IntPtr;69 MachinePointerInfo IntPointerInfo;70 MachinePointerInfo FloatPointerInfo;71 SDValue IntValue;72 APInt SignMask;73 uint8_t SignBit;74};75 76//===----------------------------------------------------------------------===//77/// This takes an arbitrary SelectionDAG as input and78/// hacks on it until the target machine can handle it. This involves79/// eliminating value sizes the machine cannot handle (promoting small sizes to80/// large sizes or splitting up large values into small values) as well as81/// eliminating operations the machine cannot handle.82///83/// This code also does a small amount of optimization and recognition of idioms84/// as part of its processing. For example, if a target does not support a85/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this86/// will attempt merge setcc and brc instructions into brcc's.87class SelectionDAGLegalize {88 const TargetMachine &TM;89 const TargetLowering &TLI;90 SelectionDAG &DAG;91 92 /// The set of nodes which have already been legalized. We hold a93 /// reference to it in order to update as necessary on node deletion.94 SmallPtrSetImpl<SDNode *> &LegalizedNodes;95 96 /// A set of all the nodes updated during legalization.97 SmallSetVector<SDNode *, 16> *UpdatedNodes;98 99 EVT getSetCCResultType(EVT VT) const {100 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);101 }102 103 // Libcall insertion helpers.104 105public:106 SelectionDAGLegalize(SelectionDAG &DAG,107 SmallPtrSetImpl<SDNode *> &LegalizedNodes,108 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)109 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),110 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}111 112 /// Legalizes the given operation.113 void LegalizeOp(SDNode *Node);114 115private:116 SDValue OptimizeFloatStore(StoreSDNode *ST);117 118 void LegalizeLoadOps(SDNode *Node);119 void LegalizeStoreOps(SDNode *Node);120 121 SDValue ExpandINSERT_VECTOR_ELT(SDValue Op);122 123 /// Return a vector shuffle operation which124 /// performs the same shuffe in terms of order or result bytes, but on a type125 /// whose vector element type is narrower than the original shuffle type.126 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>127 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,128 SDValue N1, SDValue N2,129 ArrayRef<int> Mask) const;130 131 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,132 TargetLowering::ArgListTy &&Args,133 bool IsSigned, EVT RetVT);134 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);135 136 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,137 SmallVectorImpl<SDValue> &Results);138 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,139 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,140 RTLIB::Libcall Call_F128,141 RTLIB::Libcall Call_PPCF128,142 SmallVectorImpl<SDValue> &Results);143 144 void145 ExpandFastFPLibCall(SDNode *Node, bool IsFast,146 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,147 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,148 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,149 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,150 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,151 SmallVectorImpl<SDValue> &Results);152 153 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, RTLIB::Libcall Call_I8,154 RTLIB::Libcall Call_I16, RTLIB::Libcall Call_I32,155 RTLIB::Libcall Call_I64, RTLIB::Libcall Call_I128);156 void ExpandArgFPLibCall(SDNode *Node,157 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,158 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,159 RTLIB::Libcall Call_PPCF128,160 SmallVectorImpl<SDValue> &Results);161 SDValue ExpandBitCountingLibCall(SDNode *Node, RTLIB::Libcall CallI32,162 RTLIB::Libcall CallI64,163 RTLIB::Libcall CallI128);164 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);165 166 SDValue ExpandSincosStretLibCall(SDNode *Node) const;167 168 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,169 const SDLoc &dl);170 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,171 const SDLoc &dl, SDValue ChainIn);172 SDValue ExpandBUILD_VECTOR(SDNode *Node);173 SDValue ExpandSPLAT_VECTOR(SDNode *Node);174 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);175 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,176 SmallVectorImpl<SDValue> &Results);177 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,178 SDValue Value) const;179 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,180 SDValue NewIntValue) const;181 SDValue ExpandFCOPYSIGN(SDNode *Node) const;182 SDValue ExpandFABS(SDNode *Node) const;183 SDValue ExpandFNEG(SDNode *Node) const;184 SDValue expandLdexp(SDNode *Node) const;185 SDValue expandFrexp(SDNode *Node) const;186 187 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);188 void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,189 SmallVectorImpl<SDValue> &Results);190 void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,191 SmallVectorImpl<SDValue> &Results);192 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);193 194 /// Implements vector reduce operation promotion.195 ///196 /// All vector operands are promoted to a vector type with larger element197 /// type, and the start value is promoted to a larger scalar type. Then the198 /// result is truncated back to the original scalar type.199 SDValue PromoteReduction(SDNode *Node);200 201 SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);202 203 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);204 SDValue ExpandInsertToVectorThroughStack(SDValue Op);205 SDValue ExpandVectorBuildThroughStack(SDNode* Node);206 SDValue ExpandConcatVectors(SDNode *Node);207 208 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);209 SDValue ExpandConstant(ConstantSDNode *CP);210 211 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall212 bool ExpandNode(SDNode *Node);213 void ConvertNodeToLibcall(SDNode *Node);214 void PromoteNode(SDNode *Node);215 216public:217 // Node replacement helpers218 219 void ReplacedNode(SDNode *N) {220 LegalizedNodes.erase(N);221 if (UpdatedNodes)222 UpdatedNodes->insert(N);223 }224 225 void ReplaceNode(SDNode *Old, SDNode *New) {226 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);227 dbgs() << " with: "; New->dump(&DAG));228 229 assert(Old->getNumValues() == New->getNumValues() &&230 "Replacing one node with another that produces a different number "231 "of values!");232 DAG.ReplaceAllUsesWith(Old, New);233 if (UpdatedNodes)234 UpdatedNodes->insert(New);235 ReplacedNode(Old);236 }237 238 void ReplaceNode(SDValue Old, SDValue New) {239 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);240 dbgs() << " with: "; New->dump(&DAG));241 242 DAG.ReplaceAllUsesWith(Old, New);243 if (UpdatedNodes)244 UpdatedNodes->insert(New.getNode());245 ReplacedNode(Old.getNode());246 }247 248 void ReplaceNode(SDNode *Old, const SDValue *New) {249 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));250 251 DAG.ReplaceAllUsesWith(Old, New);252 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {253 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");254 New[i]->dump(&DAG));255 if (UpdatedNodes)256 UpdatedNodes->insert(New[i].getNode());257 }258 ReplacedNode(Old);259 }260 261 void ReplaceNodeWithValue(SDValue Old, SDValue New) {262 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);263 dbgs() << " with: "; New->dump(&DAG));264 265 DAG.ReplaceAllUsesOfValueWith(Old, New);266 if (UpdatedNodes)267 UpdatedNodes->insert(New.getNode());268 ReplacedNode(Old.getNode());269 }270};271 272} // end anonymous namespace273 274// Helper function that generates an MMO that considers the alignment of the275// stack, and the size of the stack object276static MachineMemOperand *getStackAlignedMMO(SDValue StackPtr,277 MachineFunction &MF,278 bool isObjectScalable) {279 auto &MFI = MF.getFrameInfo();280 int FI = cast<FrameIndexSDNode>(StackPtr)->getIndex();281 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);282 LocationSize ObjectSize = isObjectScalable283 ? LocationSize::beforeOrAfterPointer()284 : LocationSize::precise(MFI.getObjectSize(FI));285 return MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,286 ObjectSize, MFI.getObjectAlign(FI));287}288 289/// Return a vector shuffle operation which290/// performs the same shuffle in terms of order or result bytes, but on a type291/// whose vector element type is narrower than the original shuffle type.292/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>293SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(294 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,295 ArrayRef<int> Mask) const {296 unsigned NumMaskElts = VT.getVectorNumElements();297 unsigned NumDestElts = NVT.getVectorNumElements();298 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;299 300 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");301 302 if (NumEltsGrowth == 1)303 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);304 305 SmallVector<int, 8> NewMask;306 for (unsigned i = 0; i != NumMaskElts; ++i) {307 int Idx = Mask[i];308 for (unsigned j = 0; j != NumEltsGrowth; ++j) {309 if (Idx < 0)310 NewMask.push_back(-1);311 else312 NewMask.push_back(Idx * NumEltsGrowth + j);313 }314 }315 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");316 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");317 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);318}319 320/// Expands the ConstantFP node to an integer constant or321/// a load from the constant pool.322SDValue323SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {324 bool Extend = false;325 SDLoc dl(CFP);326 327 // If a FP immediate is precise when represented as a float and if the328 // target can do an extending load from float to double, we put it into329 // the constant pool as a float, even if it's is statically typed as a330 // double. This shrinks FP constants and canonicalizes them for targets where331 // an FP extending load is the same cost as a normal load (such as on the x87332 // fp stack or PPC FP unit).333 EVT VT = CFP->getValueType(0);334 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());335 if (!UseCP) {336 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");337 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,338 (VT == MVT::f64) ? MVT::i64 : MVT::i32);339 }340 341 APFloat APF = CFP->getValueAPF();342 EVT OrigVT = VT;343 EVT SVT = VT;344 345 // We don't want to shrink SNaNs. Converting the SNaN back to its real type346 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).347 if (!APF.isSignaling()) {348 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {349 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);350 if (ConstantFPSDNode::isValueValidForType(SVT, APF) &&351 // Only do this if the target has a native EXTLOAD instruction from352 // smaller type.353 TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&354 TLI.ShouldShrinkFPConstant(OrigVT)) {355 Type *SType = SVT.getTypeForEVT(*DAG.getContext());356 LLVMC = cast<ConstantFP>(ConstantFoldCastOperand(357 Instruction::FPTrunc, LLVMC, SType, DAG.getDataLayout()));358 VT = SVT;359 Extend = true;360 }361 }362 }363 364 SDValue CPIdx =365 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));366 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();367 if (Extend) {368 SDValue Result = DAG.getExtLoad(369 ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,370 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,371 Alignment);372 return Result;373 }374 SDValue Result = DAG.getLoad(375 OrigVT, dl, DAG.getEntryNode(), CPIdx,376 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);377 return Result;378}379 380/// Expands the Constant node to a load from the constant pool.381SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {382 SDLoc dl(CP);383 EVT VT = CP->getValueType(0);384 SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),385 TLI.getPointerTy(DAG.getDataLayout()));386 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();387 SDValue Result = DAG.getLoad(388 VT, dl, DAG.getEntryNode(), CPIdx,389 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);390 return Result;391}392 393SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Op) {394 SDValue Vec = Op.getOperand(0);395 SDValue Val = Op.getOperand(1);396 SDValue Idx = Op.getOperand(2);397 SDLoc dl(Op);398 399 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {400 // SCALAR_TO_VECTOR requires that the type of the value being inserted401 // match the element type of the vector being created, except for402 // integers in which case the inserted value can be over width.403 EVT EltVT = Vec.getValueType().getVectorElementType();404 if (Val.getValueType() == EltVT ||405 (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {406 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,407 Vec.getValueType(), Val);408 409 unsigned NumElts = Vec.getValueType().getVectorNumElements();410 // We generate a shuffle of InVec and ScVec, so the shuffle mask411 // should be 0,1,2,3,4,5... with the appropriate element replaced with412 // elt 0 of the RHS.413 SmallVector<int, 8> ShufOps;414 for (unsigned i = 0; i != NumElts; ++i)415 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);416 417 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);418 }419 }420 return ExpandInsertToVectorThroughStack(Op);421}422 423SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {424 if (!ISD::isNormalStore(ST))425 return SDValue();426 427 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");428 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'429 // FIXME: move this to the DAG Combiner! Note that we can't regress due430 // to phase ordering between legalized code and the dag combiner. This431 // probably means that we need to integrate dag combiner and legalizer432 // together.433 // We generally can't do this one for long doubles.434 SDValue Chain = ST->getChain();435 SDValue Ptr = ST->getBasePtr();436 SDValue Value = ST->getValue();437 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();438 AAMDNodes AAInfo = ST->getAAInfo();439 SDLoc dl(ST);440 441 // Don't optimise TargetConstantFP442 if (Value.getOpcode() == ISD::TargetConstantFP)443 return SDValue();444 445 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Value)) {446 if (CFP->getValueType(0) == MVT::f32 &&447 TLI.isTypeLegal(MVT::i32)) {448 SDValue Con = DAG.getConstant(CFP->getValueAPF().449 bitcastToAPInt().zextOrTrunc(32),450 SDLoc(CFP), MVT::i32);451 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),452 ST->getBaseAlign(), MMOFlags, AAInfo);453 }454 455 if (CFP->getValueType(0) == MVT::f64 &&456 !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) {457 // If this target supports 64-bit registers, do a single 64-bit store.458 if (TLI.isTypeLegal(MVT::i64)) {459 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().460 zextOrTrunc(64), SDLoc(CFP), MVT::i64);461 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),462 ST->getBaseAlign(), MMOFlags, AAInfo);463 }464 465 if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {466 // Otherwise, if the target supports 32-bit registers, use 2 32-bit467 // stores. If the target supports neither 32- nor 64-bits, this468 // xform is certainly not worth it.469 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();470 SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);471 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);472 if (DAG.getDataLayout().isBigEndian())473 std::swap(Lo, Hi);474 475 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(),476 ST->getBaseAlign(), MMOFlags, AAInfo);477 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(4), dl);478 Hi = DAG.getStore(Chain, dl, Hi, Ptr,479 ST->getPointerInfo().getWithOffset(4),480 ST->getBaseAlign(), MMOFlags, AAInfo);481 482 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);483 }484 }485 }486 return SDValue();487}488 489void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {490 StoreSDNode *ST = cast<StoreSDNode>(Node);491 SDValue Chain = ST->getChain();492 SDValue Ptr = ST->getBasePtr();493 SDLoc dl(Node);494 495 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();496 AAMDNodes AAInfo = ST->getAAInfo();497 498 if (!ST->isTruncatingStore()) {499 LLVM_DEBUG(dbgs() << "Legalizing store operation\n");500 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {501 ReplaceNode(ST, OptStore);502 return;503 }504 505 SDValue Value = ST->getValue();506 MVT VT = Value.getSimpleValueType();507 switch (TLI.getOperationAction(ISD::STORE, VT)) {508 default: llvm_unreachable("This action is not supported yet!");509 case TargetLowering::Legal: {510 // If this is an unaligned store and the target doesn't support it,511 // expand it.512 EVT MemVT = ST->getMemoryVT();513 const DataLayout &DL = DAG.getDataLayout();514 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,515 *ST->getMemOperand())) {516 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");517 SDValue Result = TLI.expandUnalignedStore(ST, DAG);518 ReplaceNode(SDValue(ST, 0), Result);519 } else520 LLVM_DEBUG(dbgs() << "Legal store\n");521 break;522 }523 case TargetLowering::Custom: {524 LLVM_DEBUG(dbgs() << "Trying custom lowering\n");525 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);526 if (Res && Res != SDValue(Node, 0))527 ReplaceNode(SDValue(Node, 0), Res);528 return;529 }530 case TargetLowering::Promote: {531 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);532 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&533 "Can only promote stores to same size type");534 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);535 SDValue Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),536 ST->getBaseAlign(), MMOFlags, AAInfo);537 ReplaceNode(SDValue(Node, 0), Result);538 break;539 }540 }541 return;542 }543 544 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");545 SDValue Value = ST->getValue();546 EVT StVT = ST->getMemoryVT();547 TypeSize StWidth = StVT.getSizeInBits();548 TypeSize StSize = StVT.getStoreSizeInBits();549 auto &DL = DAG.getDataLayout();550 551 if (StWidth != StSize) {552 // Promote to a byte-sized store with upper bits zero if not553 // storing an integral number of bytes. For example, promote554 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)555 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), StSize.getFixedValue());556 Value = DAG.getZeroExtendInReg(Value, dl, StVT);557 SDValue Result =558 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,559 ST->getBaseAlign(), MMOFlags, AAInfo);560 ReplaceNode(SDValue(Node, 0), Result);561 } else if (!StVT.isVector() && !isPowerOf2_64(StWidth.getFixedValue())) {562 // If not storing a power-of-2 number of bits, expand as two stores.563 assert(!StVT.isVector() && "Unsupported truncstore!");564 unsigned StWidthBits = StWidth.getFixedValue();565 unsigned LogStWidth = Log2_32(StWidthBits);566 assert(LogStWidth < 32);567 unsigned RoundWidth = 1 << LogStWidth;568 assert(RoundWidth < StWidthBits);569 unsigned ExtraWidth = StWidthBits - RoundWidth;570 assert(ExtraWidth < RoundWidth);571 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&572 "Store size not an integral number of bytes!");573 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);574 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);575 SDValue Lo, Hi;576 unsigned IncrementSize;577 578 if (DL.isLittleEndian()) {579 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)580 // Store the bottom RoundWidth bits.581 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),582 RoundVT, ST->getBaseAlign(), MMOFlags, AAInfo);583 584 // Store the remaining ExtraWidth bits.585 IncrementSize = RoundWidth / 8;586 Ptr =587 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);588 Hi = DAG.getNode(589 ISD::SRL, dl, Value.getValueType(), Value,590 DAG.getShiftAmountConstant(RoundWidth, Value.getValueType(), dl));591 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr,592 ST->getPointerInfo().getWithOffset(IncrementSize),593 ExtraVT, ST->getBaseAlign(), MMOFlags, AAInfo);594 } else {595 // Big endian - avoid unaligned stores.596 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X597 // Store the top RoundWidth bits.598 Hi = DAG.getNode(599 ISD::SRL, dl, Value.getValueType(), Value,600 DAG.getShiftAmountConstant(ExtraWidth, Value.getValueType(), dl));601 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), RoundVT,602 ST->getBaseAlign(), MMOFlags, AAInfo);603 604 // Store the remaining ExtraWidth bits.605 IncrementSize = RoundWidth / 8;606 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,607 DAG.getConstant(IncrementSize, dl,608 Ptr.getValueType()));609 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr,610 ST->getPointerInfo().getWithOffset(IncrementSize),611 ExtraVT, ST->getBaseAlign(), MMOFlags, AAInfo);612 }613 614 // The order of the stores doesn't matter.615 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);616 ReplaceNode(SDValue(Node, 0), Result);617 } else {618 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {619 default: llvm_unreachable("This action is not supported yet!");620 case TargetLowering::Legal: {621 EVT MemVT = ST->getMemoryVT();622 // If this is an unaligned store and the target doesn't support it,623 // expand it.624 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,625 *ST->getMemOperand())) {626 SDValue Result = TLI.expandUnalignedStore(ST, DAG);627 ReplaceNode(SDValue(ST, 0), Result);628 }629 break;630 }631 case TargetLowering::Custom: {632 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);633 if (Res && Res != SDValue(Node, 0))634 ReplaceNode(SDValue(Node, 0), Res);635 return;636 }637 case TargetLowering::Expand:638 assert(!StVT.isVector() &&639 "Vector Stores are handled in LegalizeVectorOps");640 641 SDValue Result;642 643 // TRUNCSTORE:i16 i32 -> STORE i16644 if (TLI.isTypeLegal(StVT)) {645 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);646 Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),647 ST->getBaseAlign(), MMOFlags, AAInfo);648 } else {649 // The in-memory type isn't legal. Truncate to the type it would promote650 // to, and then do a truncstore.651 Value = DAG.getNode(ISD::TRUNCATE, dl,652 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),653 Value);654 Result = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),655 StVT, ST->getBaseAlign(), MMOFlags, AAInfo);656 }657 658 ReplaceNode(SDValue(Node, 0), Result);659 break;660 }661 }662}663 664void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {665 LoadSDNode *LD = cast<LoadSDNode>(Node);666 SDValue Chain = LD->getChain(); // The chain.667 SDValue Ptr = LD->getBasePtr(); // The base pointer.668 SDValue Value; // The value returned by the load op.669 SDLoc dl(Node);670 671 ISD::LoadExtType ExtType = LD->getExtensionType();672 if (ExtType == ISD::NON_EXTLOAD) {673 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");674 MVT VT = Node->getSimpleValueType(0);675 SDValue RVal = SDValue(Node, 0);676 SDValue RChain = SDValue(Node, 1);677 678 switch (TLI.getOperationAction(Node->getOpcode(), VT)) {679 default: llvm_unreachable("This action is not supported yet!");680 case TargetLowering::Legal: {681 EVT MemVT = LD->getMemoryVT();682 const DataLayout &DL = DAG.getDataLayout();683 // If this is an unaligned load and the target doesn't support it,684 // expand it.685 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,686 *LD->getMemOperand())) {687 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);688 }689 break;690 }691 case TargetLowering::Custom:692 if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {693 RVal = Res;694 RChain = Res.getValue(1);695 }696 break;697 698 case TargetLowering::Promote: {699 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);700 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&701 "Can only promote loads to same size type");702 703 // If the range metadata type does not match the legalized memory704 // operation type, remove the range metadata.705 if (const MDNode *MD = LD->getRanges()) {706 ConstantInt *Lower = mdconst::extract<ConstantInt>(MD->getOperand(0));707 if (Lower->getBitWidth() != NVT.getScalarSizeInBits() ||708 !NVT.isInteger())709 LD->getMemOperand()->clearRanges();710 }711 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());712 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);713 RChain = Res.getValue(1);714 break;715 }716 }717 if (RChain.getNode() != Node) {718 assert(RVal.getNode() != Node && "Load must be completely replaced");719 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);720 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);721 if (UpdatedNodes) {722 UpdatedNodes->insert(RVal.getNode());723 UpdatedNodes->insert(RChain.getNode());724 }725 ReplacedNode(Node);726 }727 return;728 }729 730 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");731 EVT SrcVT = LD->getMemoryVT();732 TypeSize SrcWidth = SrcVT.getSizeInBits();733 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();734 AAMDNodes AAInfo = LD->getAAInfo();735 736 if (SrcWidth != SrcVT.getStoreSizeInBits() &&737 // Some targets pretend to have an i1 loading operation, and actually738 // load an i8. This trick is correct for ZEXTLOAD because the top 7739 // bits are guaranteed to be zero; it helps the optimizers understand740 // that these bits are zero. It is also useful for EXTLOAD, since it741 // tells the optimizers that those bits are undefined. It would be742 // nice to have an effective generic way of getting these benefits...743 // Until such a way is found, don't insist on promoting i1 here.744 (SrcVT != MVT::i1 ||745 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==746 TargetLowering::Promote)) {747 // Promote to a byte-sized load if not loading an integral number of748 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.749 unsigned NewWidth = SrcVT.getStoreSizeInBits();750 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);751 SDValue Ch;752 753 // The extra bits are guaranteed to be zero, since we stored them that754 // way. A zext load from NVT thus automatically gives zext from SrcVT.755 756 ISD::LoadExtType NewExtType =757 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;758 759 SDValue Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0),760 Chain, Ptr, LD->getPointerInfo(), NVT,761 LD->getBaseAlign(), MMOFlags, AAInfo);762 763 Ch = Result.getValue(1); // The chain.764 765 if (ExtType == ISD::SEXTLOAD)766 // Having the top bits zero doesn't help when sign extending.767 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,768 Result.getValueType(),769 Result, DAG.getValueType(SrcVT));770 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())771 // All the top bits are guaranteed to be zero - inform the optimizers.772 Result = DAG.getNode(ISD::AssertZext, dl,773 Result.getValueType(), Result,774 DAG.getValueType(SrcVT));775 776 Value = Result;777 Chain = Ch;778 } else if (!isPowerOf2_64(SrcWidth.getKnownMinValue())) {779 // If not loading a power-of-2 number of bits, expand as two loads.780 assert(!SrcVT.isVector() && "Unsupported extload!");781 unsigned SrcWidthBits = SrcWidth.getFixedValue();782 unsigned LogSrcWidth = Log2_32(SrcWidthBits);783 assert(LogSrcWidth < 32);784 unsigned RoundWidth = 1 << LogSrcWidth;785 assert(RoundWidth < SrcWidthBits);786 unsigned ExtraWidth = SrcWidthBits - RoundWidth;787 assert(ExtraWidth < RoundWidth);788 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&789 "Load size not an integral number of bytes!");790 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);791 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);792 SDValue Lo, Hi, Ch;793 unsigned IncrementSize;794 auto &DL = DAG.getDataLayout();795 796 if (DL.isLittleEndian()) {797 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)798 // Load the bottom RoundWidth bits.799 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,800 LD->getPointerInfo(), RoundVT, LD->getBaseAlign(),801 MMOFlags, AAInfo);802 803 // Load the remaining ExtraWidth bits.804 IncrementSize = RoundWidth / 8;805 Ptr =806 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);807 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,808 LD->getPointerInfo().getWithOffset(IncrementSize),809 ExtraVT, LD->getBaseAlign(), MMOFlags, AAInfo);810 811 // Build a factor node to remember that this load is independent of812 // the other one.813 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),814 Hi.getValue(1));815 816 // Move the top bits to the right place.817 Hi = DAG.getNode(818 ISD::SHL, dl, Hi.getValueType(), Hi,819 DAG.getShiftAmountConstant(RoundWidth, Hi.getValueType(), dl));820 821 // Join the hi and lo parts.822 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);823 } else {824 // Big endian - avoid unaligned loads.825 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8826 // Load the top RoundWidth bits.827 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,828 LD->getPointerInfo(), RoundVT, LD->getBaseAlign(),829 MMOFlags, AAInfo);830 831 // Load the remaining ExtraWidth bits.832 IncrementSize = RoundWidth / 8;833 Ptr =834 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);835 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,836 LD->getPointerInfo().getWithOffset(IncrementSize),837 ExtraVT, LD->getBaseAlign(), MMOFlags, AAInfo);838 839 // Build a factor node to remember that this load is independent of840 // the other one.841 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),842 Hi.getValue(1));843 844 // Move the top bits to the right place.845 Hi = DAG.getNode(846 ISD::SHL, dl, Hi.getValueType(), Hi,847 DAG.getShiftAmountConstant(ExtraWidth, Hi.getValueType(), dl));848 849 // Join the hi and lo parts.850 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);851 }852 853 Chain = Ch;854 } else {855 bool isCustom = false;856 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),857 SrcVT.getSimpleVT())) {858 default: llvm_unreachable("This action is not supported yet!");859 case TargetLowering::Custom:860 isCustom = true;861 [[fallthrough]];862 case TargetLowering::Legal:863 Value = SDValue(Node, 0);864 Chain = SDValue(Node, 1);865 866 if (isCustom) {867 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {868 Value = Res;869 Chain = Res.getValue(1);870 }871 } else {872 // If this is an unaligned load and the target doesn't support it,873 // expand it.874 EVT MemVT = LD->getMemoryVT();875 const DataLayout &DL = DAG.getDataLayout();876 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,877 *LD->getMemOperand())) {878 std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);879 }880 }881 break;882 883 case TargetLowering::Expand: {884 EVT DestVT = Node->getValueType(0);885 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {886 // If the source type is not legal, see if there is a legal extload to887 // an intermediate type that we can then extend further.888 EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());889 if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) &&890 (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?891 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT))) {892 // If we are loading a legal type, this is a non-extload followed by a893 // full extend.894 ISD::LoadExtType MidExtType =895 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;896 897 SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,898 SrcVT, LD->getMemOperand());899 unsigned ExtendOp =900 ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);901 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);902 Chain = Load.getValue(1);903 break;904 }905 906 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the907 // normal undefined upper bits behavior to allow using an in-reg extend908 // with the illegal FP type, so load as an integer and do the909 // from-integer conversion.910 EVT SVT = SrcVT.getScalarType();911 if (SVT == MVT::f16 || SVT == MVT::bf16) {912 EVT ISrcVT = SrcVT.changeTypeToInteger();913 EVT IDestVT = DestVT.changeTypeToInteger();914 EVT ILoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());915 916 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, ILoadVT, Chain,917 Ptr, ISrcVT, LD->getMemOperand());918 Value =919 DAG.getNode(SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP,920 dl, DestVT, Result);921 Chain = Result.getValue(1);922 break;923 }924 }925 926 assert(!SrcVT.isVector() &&927 "Vector Loads are handled in LegalizeVectorOps");928 929 // FIXME: This does not work for vectors on most targets. Sign-930 // and zero-extend operations are currently folded into extending931 // loads, whether they are legal or not, and then we end up here932 // without any support for legalizing them.933 assert(ExtType != ISD::EXTLOAD &&934 "EXTLOAD should always be supported!");935 // Turn the unsupported load into an EXTLOAD followed by an936 // explicit zero/sign extend inreg.937 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,938 Node->getValueType(0),939 Chain, Ptr, SrcVT,940 LD->getMemOperand());941 SDValue ValRes;942 if (ExtType == ISD::SEXTLOAD)943 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,944 Result.getValueType(),945 Result, DAG.getValueType(SrcVT));946 else947 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT);948 Value = ValRes;949 Chain = Result.getValue(1);950 break;951 }952 }953 }954 955 // Since loads produce two values, make sure to remember that we legalized956 // both of them.957 if (Chain.getNode() != Node) {958 assert(Value.getNode() != Node && "Load must be completely replaced");959 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);960 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);961 if (UpdatedNodes) {962 UpdatedNodes->insert(Value.getNode());963 UpdatedNodes->insert(Chain.getNode());964 }965 ReplacedNode(Node);966 }967}968 969/// Return a legal replacement for the given operation, with all legal operands.970void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {971 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));972 973 // Allow illegal target nodes and illegal registers.974 if (Node->getOpcode() == ISD::TargetConstant ||975 Node->getOpcode() == ISD::Register)976 return;977 978#ifndef NDEBUG979 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)980 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==981 TargetLowering::TypeLegal &&982 "Unexpected illegal type!");983 984 for (const SDValue &Op : Node->op_values())985 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==986 TargetLowering::TypeLegal ||987 Op.getOpcode() == ISD::TargetConstant ||988 Op.getOpcode() == ISD::Register) &&989 "Unexpected illegal type!");990#endif991 992 // Figure out the correct action; the way to query this varies by opcode993 TargetLowering::LegalizeAction Action = TargetLowering::Legal;994 bool SimpleFinishLegalizing = true;995 switch (Node->getOpcode()) {996 // TODO: Currently, POISON is being lowered to UNDEF here. However, there is997 // an open concern that this transformation may not be ideal, as targets998 // should ideally handle POISON directly. Changing this behavior would require999 // adding support for POISON in TableGen, which is a large change.1000 // Additionally, many existing test cases rely on the current behavior (e.g.,1001 // llvm/test/CodeGen/PowerPC/vec_shuffle.ll). A broader discussion and1002 // incremental changes might be needed to properly1003 // support POISON without breaking existing targets and tests.1004 case ISD::POISON: {1005 SDValue UndefNode = DAG.getUNDEF(Node->getValueType(0));1006 ReplaceNode(Node, UndefNode.getNode());1007 break;1008 }1009 case ISD::INTRINSIC_W_CHAIN:1010 case ISD::INTRINSIC_WO_CHAIN:1011 case ISD::INTRINSIC_VOID:1012 case ISD::STACKSAVE:1013 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);1014 break;1015 case ISD::GET_DYNAMIC_AREA_OFFSET:1016 Action = TLI.getOperationAction(Node->getOpcode(),1017 Node->getValueType(0));1018 break;1019 case ISD::VAARG:1020 Action = TLI.getOperationAction(Node->getOpcode(),1021 Node->getValueType(0));1022 if (Action != TargetLowering::Promote)1023 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);1024 break;1025 case ISD::SET_FPENV:1026 case ISD::SET_FPMODE:1027 Action = TLI.getOperationAction(Node->getOpcode(),1028 Node->getOperand(1).getValueType());1029 break;1030 case ISD::FP_TO_FP16:1031 case ISD::FP_TO_BF16:1032 case ISD::SINT_TO_FP:1033 case ISD::UINT_TO_FP:1034 case ISD::EXTRACT_VECTOR_ELT:1035 case ISD::LROUND:1036 case ISD::LLROUND:1037 case ISD::LRINT:1038 case ISD::LLRINT:1039 Action = TLI.getOperationAction(Node->getOpcode(),1040 Node->getOperand(0).getValueType());1041 break;1042 case ISD::STRICT_FP_TO_FP16:1043 case ISD::STRICT_FP_TO_BF16:1044 case ISD::STRICT_SINT_TO_FP:1045 case ISD::STRICT_UINT_TO_FP:1046 case ISD::STRICT_LRINT:1047 case ISD::STRICT_LLRINT:1048 case ISD::STRICT_LROUND:1049 case ISD::STRICT_LLROUND:1050 // These pseudo-ops are the same as the other STRICT_ ops except1051 // they are registered with setOperationAction() using the input type1052 // instead of the output type.1053 Action = TLI.getOperationAction(Node->getOpcode(),1054 Node->getOperand(1).getValueType());1055 break;1056 case ISD::SIGN_EXTEND_INREG: {1057 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();1058 Action = TLI.getOperationAction(Node->getOpcode(), InnerType);1059 break;1060 }1061 case ISD::ATOMIC_STORE:1062 Action = TLI.getOperationAction(Node->getOpcode(),1063 Node->getOperand(1).getValueType());1064 break;1065 case ISD::SELECT_CC:1066 case ISD::STRICT_FSETCC:1067 case ISD::STRICT_FSETCCS:1068 case ISD::SETCC:1069 case ISD::SETCCCARRY:1070 case ISD::VP_SETCC:1071 case ISD::BR_CC: {1072 unsigned Opc = Node->getOpcode();1073 unsigned CCOperand = Opc == ISD::SELECT_CC ? 41074 : Opc == ISD::STRICT_FSETCC ? 31075 : Opc == ISD::STRICT_FSETCCS ? 31076 : Opc == ISD::SETCCCARRY ? 31077 : (Opc == ISD::SETCC || Opc == ISD::VP_SETCC) ? 21078 : 1;1079 unsigned CompareOperand = Opc == ISD::BR_CC ? 21080 : Opc == ISD::STRICT_FSETCC ? 11081 : Opc == ISD::STRICT_FSETCCS ? 11082 : 0;1083 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();1084 ISD::CondCode CCCode =1085 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();1086 Action = TLI.getCondCodeAction(CCCode, OpVT);1087 if (Action == TargetLowering::Legal) {1088 if (Node->getOpcode() == ISD::SELECT_CC)1089 Action = TLI.getOperationAction(Node->getOpcode(),1090 Node->getValueType(0));1091 else1092 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);1093 }1094 break;1095 }1096 case ISD::LOAD:1097 case ISD::STORE:1098 // FIXME: Model these properly. LOAD and STORE are complicated, and1099 // STORE expects the unlegalized operand in some cases.1100 SimpleFinishLegalizing = false;1101 break;1102 case ISD::CALLSEQ_START:1103 case ISD::CALLSEQ_END:1104 // FIXME: This shouldn't be necessary. These nodes have special properties1105 // dealing with the recursive nature of legalization. Removing this1106 // special case should be done as part of making LegalizeDAG non-recursive.1107 SimpleFinishLegalizing = false;1108 break;1109 case ISD::EXTRACT_ELEMENT:1110 case ISD::GET_ROUNDING:1111 case ISD::MERGE_VALUES:1112 case ISD::EH_RETURN:1113 case ISD::FRAME_TO_ARGS_OFFSET:1114 case ISD::EH_DWARF_CFA:1115 case ISD::EH_SJLJ_SETJMP:1116 case ISD::EH_SJLJ_LONGJMP:1117 case ISD::EH_SJLJ_SETUP_DISPATCH:1118 // These operations lie about being legal: when they claim to be legal,1119 // they should actually be expanded.1120 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));1121 if (Action == TargetLowering::Legal)1122 Action = TargetLowering::Expand;1123 break;1124 case ISD::INIT_TRAMPOLINE:1125 case ISD::ADJUST_TRAMPOLINE:1126 case ISD::FRAMEADDR:1127 case ISD::RETURNADDR:1128 case ISD::ADDROFRETURNADDR:1129 case ISD::SPONENTRY:1130 // These operations lie about being legal: when they claim to be legal,1131 // they should actually be custom-lowered.1132 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));1133 if (Action == TargetLowering::Legal)1134 Action = TargetLowering::Custom;1135 break;1136 case ISD::CLEAR_CACHE:1137 // This operation is typically going to be LibCall unless the target wants1138 // something differrent.1139 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));1140 break;1141 case ISD::READCYCLECOUNTER:1142 case ISD::READSTEADYCOUNTER:1143 // READCYCLECOUNTER and READSTEADYCOUNTER return a i64, even if type1144 // legalization might have expanded that to several smaller types.1145 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);1146 break;1147 case ISD::READ_REGISTER:1148 case ISD::WRITE_REGISTER:1149 // Named register is legal in the DAG, but blocked by register name1150 // selection if not implemented by target (to chose the correct register)1151 // They'll be converted to Copy(To/From)Reg.1152 Action = TargetLowering::Legal;1153 break;1154 case ISD::UBSANTRAP:1155 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));1156 if (Action == TargetLowering::Expand) {1157 // replace ISD::UBSANTRAP with ISD::TRAP1158 SDValue NewVal;1159 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),1160 Node->getOperand(0));1161 ReplaceNode(Node, NewVal.getNode());1162 LegalizeOp(NewVal.getNode());1163 return;1164 }1165 break;1166 case ISD::DEBUGTRAP:1167 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));1168 if (Action == TargetLowering::Expand) {1169 // replace ISD::DEBUGTRAP with ISD::TRAP1170 SDValue NewVal;1171 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),1172 Node->getOperand(0));1173 ReplaceNode(Node, NewVal.getNode());1174 LegalizeOp(NewVal.getNode());1175 return;1176 }1177 break;1178 case ISD::SADDSAT:1179 case ISD::UADDSAT:1180 case ISD::SSUBSAT:1181 case ISD::USUBSAT:1182 case ISD::SSHLSAT:1183 case ISD::USHLSAT:1184 case ISD::SCMP:1185 case ISD::UCMP:1186 case ISD::FP_TO_SINT_SAT:1187 case ISD::FP_TO_UINT_SAT:1188 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));1189 break;1190 case ISD::SMULFIX:1191 case ISD::SMULFIXSAT:1192 case ISD::UMULFIX:1193 case ISD::UMULFIXSAT:1194 case ISD::SDIVFIX:1195 case ISD::SDIVFIXSAT:1196 case ISD::UDIVFIX:1197 case ISD::UDIVFIXSAT: {1198 unsigned Scale = Node->getConstantOperandVal(2);1199 Action = TLI.getFixedPointOperationAction(Node->getOpcode(),1200 Node->getValueType(0), Scale);1201 break;1202 }1203 case ISD::MSCATTER:1204 Action = TLI.getOperationAction(Node->getOpcode(),1205 cast<MaskedScatterSDNode>(Node)->getValue().getValueType());1206 break;1207 case ISD::MSTORE:1208 Action = TLI.getOperationAction(Node->getOpcode(),1209 cast<MaskedStoreSDNode>(Node)->getValue().getValueType());1210 break;1211 case ISD::VP_SCATTER:1212 Action = TLI.getOperationAction(1213 Node->getOpcode(),1214 cast<VPScatterSDNode>(Node)->getValue().getValueType());1215 break;1216 case ISD::VP_STORE:1217 Action = TLI.getOperationAction(1218 Node->getOpcode(),1219 cast<VPStoreSDNode>(Node)->getValue().getValueType());1220 break;1221 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:1222 Action = TLI.getOperationAction(1223 Node->getOpcode(),1224 cast<VPStridedStoreSDNode>(Node)->getValue().getValueType());1225 break;1226 case ISD::VECREDUCE_FADD:1227 case ISD::VECREDUCE_FMUL:1228 case ISD::VECREDUCE_ADD:1229 case ISD::VECREDUCE_MUL:1230 case ISD::VECREDUCE_AND:1231 case ISD::VECREDUCE_OR:1232 case ISD::VECREDUCE_XOR:1233 case ISD::VECREDUCE_SMAX:1234 case ISD::VECREDUCE_SMIN:1235 case ISD::VECREDUCE_UMAX:1236 case ISD::VECREDUCE_UMIN:1237 case ISD::VECREDUCE_FMAX:1238 case ISD::VECREDUCE_FMIN:1239 case ISD::VECREDUCE_FMAXIMUM:1240 case ISD::VECREDUCE_FMINIMUM:1241 case ISD::IS_FPCLASS:1242 Action = TLI.getOperationAction(1243 Node->getOpcode(), Node->getOperand(0).getValueType());1244 break;1245 case ISD::VECREDUCE_SEQ_FADD:1246 case ISD::VECREDUCE_SEQ_FMUL:1247 case ISD::VP_REDUCE_FADD:1248 case ISD::VP_REDUCE_FMUL:1249 case ISD::VP_REDUCE_ADD:1250 case ISD::VP_REDUCE_MUL:1251 case ISD::VP_REDUCE_AND:1252 case ISD::VP_REDUCE_OR:1253 case ISD::VP_REDUCE_XOR:1254 case ISD::VP_REDUCE_SMAX:1255 case ISD::VP_REDUCE_SMIN:1256 case ISD::VP_REDUCE_UMAX:1257 case ISD::VP_REDUCE_UMIN:1258 case ISD::VP_REDUCE_FMAX:1259 case ISD::VP_REDUCE_FMIN:1260 case ISD::VP_REDUCE_FMAXIMUM:1261 case ISD::VP_REDUCE_FMINIMUM:1262 case ISD::VP_REDUCE_SEQ_FADD:1263 case ISD::VP_REDUCE_SEQ_FMUL:1264 Action = TLI.getOperationAction(1265 Node->getOpcode(), Node->getOperand(1).getValueType());1266 break;1267 case ISD::VP_CTTZ_ELTS:1268 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:1269 Action = TLI.getOperationAction(Node->getOpcode(),1270 Node->getOperand(0).getValueType());1271 break;1272 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:1273 Action = TLI.getOperationAction(1274 Node->getOpcode(),1275 cast<MaskedHistogramSDNode>(Node)->getIndex().getValueType());1276 break;1277 default:1278 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {1279 Action = TLI.getCustomOperationAction(*Node);1280 } else {1281 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));1282 }1283 break;1284 }1285 1286 if (SimpleFinishLegalizing) {1287 SDNode *NewNode = Node;1288 switch (Node->getOpcode()) {1289 default: break;1290 case ISD::SHL:1291 case ISD::SRL:1292 case ISD::SRA:1293 case ISD::ROTL:1294 case ISD::ROTR: {1295 // Legalizing shifts/rotates requires adjusting the shift amount1296 // to the appropriate width.1297 SDValue Op0 = Node->getOperand(0);1298 SDValue Op1 = Node->getOperand(1);1299 if (!Op1.getValueType().isVector()) {1300 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);1301 // The getShiftAmountOperand() may create a new operand node or1302 // return the existing one. If new operand is created we need1303 // to update the parent node.1304 // Do not try to legalize SAO here! It will be automatically legalized1305 // in the next round.1306 if (SAO != Op1)1307 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);1308 }1309 }1310 break;1311 case ISD::FSHL:1312 case ISD::FSHR:1313 case ISD::SRL_PARTS:1314 case ISD::SRA_PARTS:1315 case ISD::SHL_PARTS: {1316 // Legalizing shifts/rotates requires adjusting the shift amount1317 // to the appropriate width.1318 SDValue Op0 = Node->getOperand(0);1319 SDValue Op1 = Node->getOperand(1);1320 SDValue Op2 = Node->getOperand(2);1321 if (!Op2.getValueType().isVector()) {1322 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);1323 // The getShiftAmountOperand() may create a new operand node or1324 // return the existing one. If new operand is created we need1325 // to update the parent node.1326 if (SAO != Op2)1327 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);1328 }1329 break;1330 }1331 }1332 1333 if (NewNode != Node) {1334 ReplaceNode(Node, NewNode);1335 Node = NewNode;1336 }1337 switch (Action) {1338 case TargetLowering::Legal:1339 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");1340 return;1341 case TargetLowering::Custom:1342 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");1343 // FIXME: The handling for custom lowering with multiple results is1344 // a complete mess.1345 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {1346 if (!(Res.getNode() != Node || Res.getResNo() != 0))1347 return;1348 1349 if (Node->getNumValues() == 1) {1350 // Verify the new types match the original. Glue is waived because1351 // ISD::ADDC can be legalized by replacing Glue with an integer type.1352 assert((Res.getValueType() == Node->getValueType(0) ||1353 Node->getValueType(0) == MVT::Glue) &&1354 "Type mismatch for custom legalized operation");1355 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");1356 // We can just directly replace this node with the lowered value.1357 ReplaceNode(SDValue(Node, 0), Res);1358 return;1359 }1360 1361 SmallVector<SDValue, 8> ResultVals;1362 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {1363 // Verify the new types match the original. Glue is waived because1364 // ISD::ADDC can be legalized by replacing Glue with an integer type.1365 assert((Res->getValueType(i) == Node->getValueType(i) ||1366 Node->getValueType(i) == MVT::Glue) &&1367 "Type mismatch for custom legalized operation");1368 ResultVals.push_back(Res.getValue(i));1369 }1370 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");1371 ReplaceNode(Node, ResultVals.data());1372 return;1373 }1374 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");1375 [[fallthrough]];1376 case TargetLowering::Expand:1377 if (ExpandNode(Node))1378 return;1379 [[fallthrough]];1380 case TargetLowering::LibCall:1381 ConvertNodeToLibcall(Node);1382 return;1383 case TargetLowering::Promote:1384 PromoteNode(Node);1385 return;1386 }1387 }1388 1389 switch (Node->getOpcode()) {1390 default:1391#ifndef NDEBUG1392 dbgs() << "NODE: ";1393 Node->dump( &DAG);1394 dbgs() << "\n";1395#endif1396 llvm_unreachable("Do not know how to legalize this operator!");1397 1398 case ISD::CALLSEQ_START:1399 case ISD::CALLSEQ_END:1400 break;1401 case ISD::LOAD:1402 return LegalizeLoadOps(Node);1403 case ISD::STORE:1404 return LegalizeStoreOps(Node);1405 }1406}1407 1408SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {1409 SDValue Vec = Op.getOperand(0);1410 SDValue Idx = Op.getOperand(1);1411 SDLoc dl(Op);1412 1413 // Before we generate a new store to a temporary stack slot, see if there is1414 // already one that we can use. There often is because when we scalarize1415 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole1416 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in1417 // the vector. If all are expanded here, we don't want one store per vector1418 // element.1419 1420 // Caches for hasPredecessorHelper1421 SmallPtrSet<const SDNode *, 32> Visited;1422 SmallVector<const SDNode *, 16> Worklist;1423 Visited.insert(Op.getNode());1424 Worklist.push_back(Idx.getNode());1425 SDValue StackPtr, Ch;1426 for (SDNode *User : Vec.getNode()->users()) {1427 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {1428 if (ST->isIndexed() || ST->isTruncatingStore() ||1429 ST->getValue() != Vec)1430 continue;1431 1432 // Make sure that nothing else could have stored into the destination of1433 // this store.1434 if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))1435 continue;1436 1437 // If the index is dependent on the store we will introduce a cycle when1438 // creating the load (the load uses the index, and by replacing the chain1439 // we will make the index dependent on the load). Also, the store might be1440 // dependent on the extractelement and introduce a cycle when creating1441 // the load.1442 if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||1443 ST->hasPredecessor(Op.getNode()))1444 continue;1445 1446 StackPtr = ST->getBasePtr();1447 Ch = SDValue(ST, 0);1448 break;1449 }1450 }1451 1452 EVT VecVT = Vec.getValueType();1453 1454 if (!Ch.getNode()) {1455 // Store the value to a temporary stack slot, then LOAD the returned part.1456 StackPtr = DAG.CreateStackTemporary(VecVT);1457 MachineMemOperand *StoreMMO = getStackAlignedMMO(1458 StackPtr, DAG.getMachineFunction(), VecVT.isScalableVector());1459 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, StoreMMO);1460 }1461 1462 SDValue NewLoad;1463 Align ElementAlignment =1464 std::min(cast<StoreSDNode>(Ch)->getAlign(),1465 DAG.getDataLayout().getPrefTypeAlign(1466 Op.getValueType().getTypeForEVT(*DAG.getContext())));1467 1468 if (Op.getValueType().isVector()) {1469 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT,1470 Op.getValueType(), Idx);1471 NewLoad = DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr,1472 MachinePointerInfo(), ElementAlignment);1473 } else {1474 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);1475 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,1476 MachinePointerInfo(), VecVT.getVectorElementType(),1477 ElementAlignment);1478 }1479 1480 // Replace the chain going out of the store, by the one out of the load.1481 DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));1482 1483 // We introduced a cycle though, so update the loads operands, making sure1484 // to use the original store's chain as an incoming chain.1485 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->ops());1486 NewLoadOperands[0] = Ch;1487 NewLoad =1488 SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);1489 return NewLoad;1490}1491 1492SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {1493 assert(Op.getValueType().isVector() && "Non-vector insert subvector!");1494 1495 SDValue Vec = Op.getOperand(0);1496 SDValue Part = Op.getOperand(1);1497 SDValue Idx = Op.getOperand(2);1498 SDLoc dl(Op);1499 1500 // Store the value to a temporary stack slot, then LOAD the returned part.1501 EVT VecVT = Vec.getValueType();1502 EVT PartVT = Part.getValueType();1503 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);1504 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();1505 MachinePointerInfo PtrInfo =1506 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);1507 1508 // First store the whole vector.1509 Align BaseVecAlignment =1510 DAG.getMachineFunction().getFrameInfo().getObjectAlign(FI);1511 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,1512 BaseVecAlignment);1513 1514 // Freeze the index so we don't poison the clamping code we're about to emit.1515 Idx = DAG.getFreeze(Idx);1516 1517 Type *PartTy = PartVT.getTypeForEVT(*DAG.getContext());1518 Align PartAlignment = DAG.getDataLayout().getPrefTypeAlign(PartTy);1519 1520 // Then store the inserted part.1521 if (PartVT.isVector()) {1522 SDValue SubStackPtr =1523 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, PartVT, Idx);1524 1525 // Store the subvector.1526 Ch = DAG.getStore(1527 Ch, dl, Part, SubStackPtr,1528 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()),1529 PartAlignment);1530 } else {1531 SDValue SubStackPtr =1532 TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);1533 1534 // Store the scalar value.1535 Ch = DAG.getTruncStore(1536 Ch, dl, Part, SubStackPtr,1537 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()),1538 VecVT.getVectorElementType(), PartAlignment);1539 }1540 1541 assert(cast<StoreSDNode>(Ch)->getAlign() == PartAlignment &&1542 "ElementAlignment does not match!");1543 1544 // Finally, load the updated vector.1545 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo,1546 BaseVecAlignment);1547}1548 1549SDValue SelectionDAGLegalize::ExpandConcatVectors(SDNode *Node) {1550 assert(Node->getOpcode() == ISD::CONCAT_VECTORS && "Unexpected opcode!");1551 SDLoc DL(Node);1552 SmallVector<SDValue, 16> Ops;1553 unsigned NumOperands = Node->getNumOperands();1554 MVT VectorIdxType = TLI.getVectorIdxTy(DAG.getDataLayout());1555 EVT VectorValueType = Node->getOperand(0).getValueType();1556 unsigned NumSubElem = VectorValueType.getVectorNumElements();1557 EVT ElementValueType = TLI.getTypeToTransformTo(1558 *DAG.getContext(), VectorValueType.getVectorElementType());1559 for (unsigned I = 0; I < NumOperands; ++I) {1560 SDValue SubOp = Node->getOperand(I);1561 for (unsigned Idx = 0; Idx < NumSubElem; ++Idx) {1562 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ElementValueType,1563 SubOp,1564 DAG.getConstant(Idx, DL, VectorIdxType)));1565 }1566 }1567 return DAG.getBuildVector(Node->getValueType(0), DL, Ops);1568}1569 1570SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {1571 assert((Node->getOpcode() == ISD::BUILD_VECTOR ||1572 Node->getOpcode() == ISD::CONCAT_VECTORS) &&1573 "Unexpected opcode!");1574 1575 // We can't handle this case efficiently. Allocate a sufficiently1576 // aligned object on the stack, store each operand into it, then load1577 // the result as a vector.1578 // Create the stack frame object.1579 EVT VT = Node->getValueType(0);1580 EVT MemVT = isa<BuildVectorSDNode>(Node) ? VT.getVectorElementType()1581 : Node->getOperand(0).getValueType();1582 SDLoc dl(Node);1583 SDValue FIPtr = DAG.CreateStackTemporary(VT);1584 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();1585 MachinePointerInfo PtrInfo =1586 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);1587 1588 // Emit a store of each element to the stack slot.1589 SmallVector<SDValue, 8> Stores;1590 unsigned TypeByteSize = MemVT.getSizeInBits() / 8;1591 assert(TypeByteSize > 0 && "Vector element type too small for stack store!");1592 1593 // If the destination vector element type of a BUILD_VECTOR is narrower than1594 // the source element type, only store the bits necessary.1595 bool Truncate = isa<BuildVectorSDNode>(Node) &&1596 MemVT.bitsLT(Node->getOperand(0).getValueType());1597 1598 // Store (in the right endianness) the elements to memory.1599 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {1600 // Ignore undef elements.1601 if (Node->getOperand(i).isUndef()) continue;1602 1603 unsigned Offset = TypeByteSize*i;1604 1605 SDValue Idx =1606 DAG.getMemBasePlusOffset(FIPtr, TypeSize::getFixed(Offset), dl);1607 1608 if (Truncate)1609 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,1610 Node->getOperand(i), Idx,1611 PtrInfo.getWithOffset(Offset), MemVT));1612 else1613 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),1614 Idx, PtrInfo.getWithOffset(Offset)));1615 }1616 1617 SDValue StoreChain;1618 if (!Stores.empty()) // Not all undef elements?1619 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);1620 else1621 StoreChain = DAG.getEntryNode();1622 1623 // Result is a load from the stack slot.1624 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);1625}1626 1627/// Bitcast a floating-point value to an integer value. Only bitcast the part1628/// containing the sign bit if the target has no integer value capable of1629/// holding all bits of the floating-point value.1630void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,1631 const SDLoc &DL,1632 SDValue Value) const {1633 EVT FloatVT = Value.getValueType();1634 unsigned NumBits = FloatVT.getScalarSizeInBits();1635 State.FloatVT = FloatVT;1636 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);1637 // Convert to an integer of the same size.1638 if (TLI.isTypeLegal(IVT)) {1639 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);1640 State.SignMask = APInt::getSignMask(NumBits);1641 State.SignBit = NumBits - 1;1642 return;1643 }1644 1645 auto &DataLayout = DAG.getDataLayout();1646 // Store the float to memory, then load the sign part out as an integer.1647 MVT LoadTy = TLI.getRegisterType(MVT::i8);1648 // First create a temporary that is aligned for both the load and store.1649 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);1650 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();1651 // Then store the float to it.1652 State.FloatPtr = StackPtr;1653 MachineFunction &MF = DAG.getMachineFunction();1654 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);1655 State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,1656 State.FloatPointerInfo);1657 1658 SDValue IntPtr;1659 if (DataLayout.isBigEndian()) {1660 assert(FloatVT.isByteSized() && "Unsupported floating point type!");1661 // Load out a legal integer with the same sign bit as the float.1662 IntPtr = StackPtr;1663 State.IntPointerInfo = State.FloatPointerInfo;1664 } else {1665 // Advance the pointer so that the loaded byte will contain the sign bit.1666 unsigned ByteOffset = (NumBits / 8) - 1;1667 IntPtr =1668 DAG.getMemBasePlusOffset(StackPtr, TypeSize::getFixed(ByteOffset), DL);1669 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,1670 ByteOffset);1671 }1672 1673 State.IntPtr = IntPtr;1674 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,1675 State.IntPointerInfo, MVT::i8);1676 State.SignMask = APInt::getOneBitSet(LoadTy.getScalarSizeInBits(), 7);1677 State.SignBit = 7;1678}1679 1680/// Replace the integer value produced by getSignAsIntValue() with a new value1681/// and cast the result back to a floating-point type.1682SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,1683 const SDLoc &DL,1684 SDValue NewIntValue) const {1685 if (!State.Chain)1686 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);1687 1688 // Override the part containing the sign bit in the value stored on the stack.1689 SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,1690 State.IntPointerInfo, MVT::i8);1691 return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,1692 State.FloatPointerInfo);1693}1694 1695SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {1696 SDLoc DL(Node);1697 SDValue Mag = Node->getOperand(0);1698 SDValue Sign = Node->getOperand(1);1699 1700 // Get sign bit into an integer value.1701 FloatSignAsInt SignAsInt;1702 getSignAsIntValue(SignAsInt, DL, Sign);1703 1704 EVT IntVT = SignAsInt.IntValue.getValueType();1705 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);1706 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,1707 SignMask);1708 1709 // If FABS is legal transform1710 // FCOPYSIGN(x, y) => SignBit(y) ? -FABS(x) : FABS(x)1711 EVT FloatVT = Mag.getValueType();1712 if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&1713 TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {1714 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);1715 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);1716 SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,1717 DAG.getConstant(0, DL, IntVT), ISD::SETNE);1718 return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);1719 }1720 1721 // Transform Mag value to integer, and clear the sign bit.1722 FloatSignAsInt MagAsInt;1723 getSignAsIntValue(MagAsInt, DL, Mag);1724 EVT MagVT = MagAsInt.IntValue.getValueType();1725 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);1726 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,1727 ClearSignMask);1728 1729 // Get the signbit at the right position for MagAsInt.1730 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;1731 EVT ShiftVT = IntVT;1732 if (SignBit.getScalarValueSizeInBits() <1733 ClearedSign.getScalarValueSizeInBits()) {1734 SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);1735 ShiftVT = MagVT;1736 }1737 if (ShiftAmount > 0) {1738 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT);1739 SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst);1740 } else if (ShiftAmount < 0) {1741 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT);1742 SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst);1743 }1744 if (SignBit.getScalarValueSizeInBits() >1745 ClearedSign.getScalarValueSizeInBits()) {1746 SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);1747 }1748 1749 // Store the part with the modified sign and convert back to float.1750 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit,1751 SDNodeFlags::Disjoint);1752 1753 return modifySignAsInt(MagAsInt, DL, CopiedSign);1754}1755 1756SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {1757 // Get the sign bit as an integer.1758 SDLoc DL(Node);1759 FloatSignAsInt SignAsInt;1760 getSignAsIntValue(SignAsInt, DL, Node->getOperand(0));1761 EVT IntVT = SignAsInt.IntValue.getValueType();1762 1763 // Flip the sign.1764 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);1765 SDValue SignFlip =1766 DAG.getNode(ISD::XOR, DL, IntVT, SignAsInt.IntValue, SignMask);1767 1768 // Convert back to float.1769 return modifySignAsInt(SignAsInt, DL, SignFlip);1770}1771 1772SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {1773 SDLoc DL(Node);1774 SDValue Value = Node->getOperand(0);1775 1776 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.1777 EVT FloatVT = Value.getValueType();1778 if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {1779 SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);1780 return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);1781 }1782 1783 // Transform value to integer, clear the sign bit and transform back.1784 FloatSignAsInt ValueAsInt;1785 getSignAsIntValue(ValueAsInt, DL, Value);1786 EVT IntVT = ValueAsInt.IntValue.getValueType();1787 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);1788 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,1789 ClearSignMask);1790 return modifySignAsInt(ValueAsInt, DL, ClearedSign);1791}1792 1793void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,1794 SmallVectorImpl<SDValue> &Results) {1795 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();1796 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"1797 " not tell us which reg is the stack pointer!");1798 SDLoc dl(Node);1799 EVT VT = Node->getValueType(0);1800 SDValue Tmp1 = SDValue(Node, 0);1801 SDValue Tmp2 = SDValue(Node, 1);1802 SDValue Tmp3 = Node->getOperand(2);1803 SDValue Chain = Tmp1.getOperand(0);1804 1805 // Chain the dynamic stack allocation so that it doesn't modify the stack1806 // pointer when other instructions are using the stack.1807 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);1808 1809 SDValue Size = Tmp2.getOperand(1);1810 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);1811 Chain = SP.getValue(1);1812 Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue();1813 const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();1814 unsigned Opc =1815 TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ?1816 ISD::ADD : ISD::SUB;1817 1818 Align StackAlign = TFL->getStackAlign();1819 Tmp1 = DAG.getNode(Opc, dl, VT, SP, Size); // Value1820 if (Alignment > StackAlign)1821 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,1822 DAG.getSignedConstant(-Alignment.value(), dl, VT));1823 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain1824 1825 Tmp2 = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl);1826 1827 Results.push_back(Tmp1);1828 Results.push_back(Tmp2);1829}1830 1831/// Emit a store/load combination to the stack. This stores1832/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does1833/// a load from the stack slot to DestVT, extending it if needed.1834/// The resultant code need not be legal.1835SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,1836 EVT DestVT, const SDLoc &dl) {1837 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());1838}1839 1840SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,1841 EVT DestVT, const SDLoc &dl,1842 SDValue Chain) {1843 EVT SrcVT = SrcOp.getValueType();1844 Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());1845 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);1846 1847 // Don't convert with stack if the load/store is expensive.1848 if ((SrcVT.bitsGT(SlotVT) &&1849 !TLI.isTruncStoreLegalOrCustom(SrcOp.getValueType(), SlotVT)) ||1850 (SlotVT.bitsLT(DestVT) &&1851 !TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, DestVT, SlotVT)))1852 return SDValue();1853 1854 // Create the stack frame object.1855 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(1856 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));1857 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT.getStoreSize(), SrcAlign);1858 1859 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);1860 int SPFI = StackPtrFI->getIndex();1861 MachinePointerInfo PtrInfo =1862 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);1863 1864 // Emit a store to the stack slot. Use a truncstore if the input value is1865 // later than DestVT.1866 SDValue Store;1867 1868 if (SrcVT.bitsGT(SlotVT))1869 Store = DAG.getTruncStore(Chain, dl, SrcOp, FIPtr, PtrInfo,1870 SlotVT, SrcAlign);1871 else {1872 assert(SrcVT.bitsEq(SlotVT) && "Invalid store");1873 Store = DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign);1874 }1875 1876 // Result is a load from the stack slot.1877 if (SlotVT.bitsEq(DestVT))1878 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);1879 1880 assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");1881 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,1882 DestAlign);1883}1884 1885SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {1886 SDLoc dl(Node);1887 // Create a vector sized/aligned stack slot, store the value to element #0,1888 // then load the whole vector back out.1889 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));1890 1891 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);1892 int SPFI = StackPtrFI->getIndex();1893 1894 SDValue Ch = DAG.getTruncStore(1895 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,1896 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),1897 Node->getValueType(0).getVectorElementType());1898 return DAG.getLoad(1899 Node->getValueType(0), dl, Ch, StackPtr,1900 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));1901}1902 1903static bool1904ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,1905 const TargetLowering &TLI, SDValue &Res) {1906 unsigned NumElems = Node->getNumOperands();1907 SDLoc dl(Node);1908 EVT VT = Node->getValueType(0);1909 1910 // Try to group the scalars into pairs, shuffle the pairs together, then1911 // shuffle the pairs of pairs together, etc. until the vector has1912 // been built. This will work only if all of the necessary shuffle masks1913 // are legal.1914 1915 // We do this in two phases; first to check the legality of the shuffles,1916 // and next, assuming that all shuffles are legal, to create the new nodes.1917 for (int Phase = 0; Phase < 2; ++Phase) {1918 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,1919 NewIntermedVals;1920 for (unsigned i = 0; i < NumElems; ++i) {1921 SDValue V = Node->getOperand(i);1922 if (V.isUndef())1923 continue;1924 1925 SDValue Vec;1926 if (Phase)1927 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);1928 IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));1929 }1930 1931 while (IntermedVals.size() > 2) {1932 NewIntermedVals.clear();1933 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {1934 // This vector and the next vector are shuffled together (simply to1935 // append the one to the other).1936 SmallVector<int, 16> ShuffleVec(NumElems, -1);1937 1938 SmallVector<int, 16> FinalIndices;1939 FinalIndices.reserve(IntermedVals[i].second.size() +1940 IntermedVals[i+1].second.size());1941 1942 int k = 0;1943 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;1944 ++j, ++k) {1945 ShuffleVec[k] = j;1946 FinalIndices.push_back(IntermedVals[i].second[j]);1947 }1948 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;1949 ++j, ++k) {1950 ShuffleVec[k] = NumElems + j;1951 FinalIndices.push_back(IntermedVals[i+1].second[j]);1952 }1953 1954 SDValue Shuffle;1955 if (Phase)1956 Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,1957 IntermedVals[i+1].first,1958 ShuffleVec);1959 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))1960 return false;1961 NewIntermedVals.push_back(1962 std::make_pair(Shuffle, std::move(FinalIndices)));1963 }1964 1965 // If we had an odd number of defined values, then append the last1966 // element to the array of new vectors.1967 if ((IntermedVals.size() & 1) != 0)1968 NewIntermedVals.push_back(IntermedVals.back());1969 1970 IntermedVals.swap(NewIntermedVals);1971 }1972 1973 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&1974 "Invalid number of intermediate vectors");1975 SDValue Vec1 = IntermedVals[0].first;1976 SDValue Vec2;1977 if (IntermedVals.size() > 1)1978 Vec2 = IntermedVals[1].first;1979 else if (Phase)1980 Vec2 = DAG.getUNDEF(VT);1981 1982 SmallVector<int, 16> ShuffleVec(NumElems, -1);1983 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)1984 ShuffleVec[IntermedVals[0].second[i]] = i;1985 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)1986 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;1987 1988 if (Phase)1989 Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);1990 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))1991 return false;1992 }1993 1994 return true;1995}1996 1997/// Expand a BUILD_VECTOR node on targets that don't1998/// support the operation, but do support the resultant vector type.1999SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {2000 unsigned NumElems = Node->getNumOperands();2001 SDValue Value1, Value2;2002 SDLoc dl(Node);2003 EVT VT = Node->getValueType(0);2004 EVT OpVT = Node->getOperand(0).getValueType();2005 EVT EltVT = VT.getVectorElementType();2006 2007 // If the only non-undef value is the low element, turn this into a2008 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.2009 bool isOnlyLowElement = true;2010 bool MoreThanTwoValues = false;2011 bool isConstant = true;2012 for (unsigned i = 0; i < NumElems; ++i) {2013 SDValue V = Node->getOperand(i);2014 if (V.isUndef())2015 continue;2016 if (i > 0)2017 isOnlyLowElement = false;2018 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))2019 isConstant = false;2020 2021 if (!Value1.getNode()) {2022 Value1 = V;2023 } else if (!Value2.getNode()) {2024 if (V != Value1)2025 Value2 = V;2026 } else if (V != Value1 && V != Value2) {2027 MoreThanTwoValues = true;2028 }2029 }2030 2031 if (!Value1.getNode())2032 return DAG.getUNDEF(VT);2033 2034 if (isOnlyLowElement)2035 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));2036 2037 // If all elements are constants, create a load from the constant pool.2038 if (isConstant) {2039 SmallVector<Constant*, 16> CV;2040 for (unsigned i = 0, e = NumElems; i != e; ++i) {2041 if (ConstantFPSDNode *V =2042 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {2043 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));2044 } else if (ConstantSDNode *V =2045 dyn_cast<ConstantSDNode>(Node->getOperand(i))) {2046 if (OpVT==EltVT)2047 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));2048 else {2049 // If OpVT and EltVT don't match, EltVT is not legal and the2050 // element values have been promoted/truncated earlier. Undo this;2051 // we don't want a v16i8 to become a v16i32 for example.2052 const ConstantInt *CI = V->getConstantIntValue();2053 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),2054 CI->getZExtValue()));2055 }2056 } else {2057 assert(Node->getOperand(i).isUndef());2058 Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());2059 CV.push_back(UndefValue::get(OpNTy));2060 }2061 }2062 Constant *CP = ConstantVector::get(CV);2063 SDValue CPIdx =2064 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));2065 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();2066 return DAG.getLoad(2067 VT, dl, DAG.getEntryNode(), CPIdx,2068 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),2069 Alignment);2070 }2071 2072 SmallSet<SDValue, 16> DefinedValues;2073 for (unsigned i = 0; i < NumElems; ++i) {2074 if (Node->getOperand(i).isUndef())2075 continue;2076 DefinedValues.insert(Node->getOperand(i));2077 }2078 2079 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {2080 if (!MoreThanTwoValues) {2081 SmallVector<int, 8> ShuffleVec(NumElems, -1);2082 for (unsigned i = 0; i < NumElems; ++i) {2083 SDValue V = Node->getOperand(i);2084 if (V.isUndef())2085 continue;2086 ShuffleVec[i] = V == Value1 ? 0 : NumElems;2087 }2088 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {2089 // Get the splatted value into the low element of a vector register.2090 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);2091 SDValue Vec2;2092 if (Value2.getNode())2093 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);2094 else2095 Vec2 = DAG.getUNDEF(VT);2096 2097 // Return shuffle(LowValVec, undef, <0,0,0,0>)2098 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);2099 }2100 } else {2101 SDValue Res;2102 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))2103 return Res;2104 }2105 }2106 2107 // Otherwise, we can't handle this case efficiently.2108 return ExpandVectorBuildThroughStack(Node);2109}2110 2111SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {2112 SDLoc DL(Node);2113 EVT VT = Node->getValueType(0);2114 SDValue SplatVal = Node->getOperand(0);2115 2116 return DAG.getSplatBuildVector(VT, DL, SplatVal);2117}2118 2119// Expand a node into a call to a libcall, returning the value as the first2120// result and the chain as the second. If the result value does not fit into a2121// register, return the lo part and set the hi part to the by-reg argument in2122// the first. If it does fit into a single register, return the result and2123// leave the Hi part unset.2124std::pair<SDValue, SDValue>2125SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,2126 TargetLowering::ArgListTy &&Args,2127 bool IsSigned, EVT RetVT) {2128 EVT CodePtrTy = TLI.getPointerTy(DAG.getDataLayout());2129 SDValue Callee;2130 if (const char *LibcallName = TLI.getLibcallName(LC))2131 Callee = DAG.getExternalSymbol(LibcallName, CodePtrTy);2132 else {2133 Callee = DAG.getUNDEF(CodePtrTy);2134 DAG.getContext()->emitError(Twine("no libcall available for ") +2135 Node->getOperationName(&DAG));2136 }2137 2138 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());2139 2140 // By default, the input chain to this libcall is the entry node of the2141 // function. If the libcall is going to be emitted as a tail call then2142 // TLI.isUsedByReturnOnly will change it to the right chain if the return2143 // node which is being folded has a non-entry input chain.2144 SDValue InChain = DAG.getEntryNode();2145 2146 // isTailCall may be true since the callee does not reference caller stack2147 // frame. Check if it's in the right position and that the return types match.2148 SDValue TCChain = InChain;2149 const Function &F = DAG.getMachineFunction().getFunction();2150 bool isTailCall =2151 TLI.isInTailCallPosition(DAG, Node, TCChain) &&2152 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());2153 if (isTailCall)2154 InChain = TCChain;2155 2156 TargetLowering::CallLoweringInfo CLI(DAG);2157 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetTy, IsSigned);2158 CLI.setDebugLoc(SDLoc(Node))2159 .setChain(InChain)2160 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,2161 std::move(Args))2162 .setTailCall(isTailCall)2163 .setSExtResult(signExtend)2164 .setZExtResult(!signExtend)2165 .setIsPostTypeLegalization(true);2166 2167 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);2168 2169 if (!CallInfo.second.getNode()) {2170 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));2171 // It's a tailcall, return the chain (which is the DAG root).2172 return {DAG.getRoot(), DAG.getRoot()};2173 }2174 2175 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));2176 return CallInfo;2177}2178 2179std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,2180 bool isSigned) {2181 TargetLowering::ArgListTy Args;2182 for (const SDValue &Op : Node->op_values()) {2183 EVT ArgVT = Op.getValueType();2184 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());2185 TargetLowering::ArgListEntry Entry(Op, ArgTy);2186 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgTy, isSigned);2187 Entry.IsZExt = !Entry.IsSExt;2188 Args.push_back(Entry);2189 }2190 2191 return ExpandLibCall(LC, Node, std::move(Args), isSigned,2192 Node->getValueType(0));2193}2194 2195void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,2196 RTLIB::Libcall LC,2197 SmallVectorImpl<SDValue> &Results) {2198 if (LC == RTLIB::UNKNOWN_LIBCALL)2199 llvm_unreachable("Can't create an unknown libcall!");2200 2201 if (Node->isStrictFPOpcode()) {2202 EVT RetVT = Node->getValueType(0);2203 SmallVector<SDValue, 4> Ops(drop_begin(Node->ops()));2204 TargetLowering::MakeLibCallOptions CallOptions;2205 CallOptions.IsPostTypeLegalization = true;2206 // FIXME: This doesn't support tail calls.2207 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,2208 Ops, CallOptions,2209 SDLoc(Node),2210 Node->getOperand(0));2211 Results.push_back(Tmp.first);2212 Results.push_back(Tmp.second);2213 } else {2214 bool IsSignedArgument = Node->getOpcode() == ISD::FLDEXP;2215 SDValue Tmp = ExpandLibCall(LC, Node, IsSignedArgument).first;2216 Results.push_back(Tmp);2217 }2218}2219 2220/// Expand the node to a libcall based on the result type.2221void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,2222 RTLIB::Libcall Call_F32,2223 RTLIB::Libcall Call_F64,2224 RTLIB::Libcall Call_F80,2225 RTLIB::Libcall Call_F128,2226 RTLIB::Libcall Call_PPCF128,2227 SmallVectorImpl<SDValue> &Results) {2228 RTLIB::Libcall LC = RTLIB::getFPLibCall(Node->getSimpleValueType(0),2229 Call_F32, Call_F64, Call_F80,2230 Call_F128, Call_PPCF128);2231 ExpandFPLibCall(Node, LC, Results);2232}2233 2234void SelectionDAGLegalize::ExpandFastFPLibCall(2235 SDNode *Node, bool IsFast,2236 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,2237 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,2238 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,2239 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,2240 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,2241 SmallVectorImpl<SDValue> &Results) {2242 2243 EVT VT = Node->getSimpleValueType(0);2244 2245 RTLIB::Libcall LC;2246 2247 // FIXME: Probably should define fast to respect nan/inf and only be2248 // approximate functions.2249 2250 if (IsFast) {2251 LC = RTLIB::getFPLibCall(VT, Call_F32.first, Call_F64.first, Call_F80.first,2252 Call_F128.first, Call_PPCF128.first);2253 }2254 2255 if (!IsFast || TLI.getLibcallImpl(LC) == RTLIB::Unsupported) {2256 // Fall back if we don't have a fast implementation.2257 LC = RTLIB::getFPLibCall(VT, Call_F32.second, Call_F64.second,2258 Call_F80.second, Call_F128.second,2259 Call_PPCF128.second);2260 }2261 2262 ExpandFPLibCall(Node, LC, Results);2263}2264 2265SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,2266 RTLIB::Libcall Call_I8,2267 RTLIB::Libcall Call_I16,2268 RTLIB::Libcall Call_I32,2269 RTLIB::Libcall Call_I64,2270 RTLIB::Libcall Call_I128) {2271 RTLIB::Libcall LC;2272 switch (Node->getSimpleValueType(0).SimpleTy) {2273 default: llvm_unreachable("Unexpected request for libcall!");2274 case MVT::i8: LC = Call_I8; break;2275 case MVT::i16: LC = Call_I16; break;2276 case MVT::i32: LC = Call_I32; break;2277 case MVT::i64: LC = Call_I64; break;2278 case MVT::i128: LC = Call_I128; break;2279 }2280 return ExpandLibCall(LC, Node, isSigned).first;2281}2282 2283/// Expand the node to a libcall based on first argument type (for instance2284/// lround and its variant).2285void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,2286 RTLIB::Libcall Call_F32,2287 RTLIB::Libcall Call_F64,2288 RTLIB::Libcall Call_F80,2289 RTLIB::Libcall Call_F128,2290 RTLIB::Libcall Call_PPCF128,2291 SmallVectorImpl<SDValue> &Results) {2292 EVT InVT = Node->getOperand(Node->isStrictFPOpcode() ? 1 : 0).getValueType();2293 RTLIB::Libcall LC = RTLIB::getFPLibCall(InVT.getSimpleVT(),2294 Call_F32, Call_F64, Call_F80,2295 Call_F128, Call_PPCF128);2296 ExpandFPLibCall(Node, LC, Results);2297}2298 2299SDValue SelectionDAGLegalize::ExpandBitCountingLibCall(2300 SDNode *Node, RTLIB::Libcall CallI32, RTLIB::Libcall CallI64,2301 RTLIB::Libcall CallI128) {2302 RTLIB::Libcall LC;2303 switch (Node->getSimpleValueType(0).SimpleTy) {2304 default:2305 llvm_unreachable("Unexpected request for libcall!");2306 case MVT::i32:2307 LC = CallI32;2308 break;2309 case MVT::i64:2310 LC = CallI64;2311 break;2312 case MVT::i128:2313 LC = CallI128;2314 break;2315 }2316 2317 // Bit-counting libcalls have one unsigned argument and return `int`.2318 // Note that `int` may be illegal on this target; ExpandLibCall will2319 // take care of promoting it to a legal type.2320 SDValue Op = Node->getOperand(0);2321 EVT IntVT =2322 EVT::getIntegerVT(*DAG.getContext(), DAG.getLibInfo().getIntSize());2323 2324 EVT ArgVT = Op.getValueType();2325 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());2326 TargetLowering::ArgListEntry Arg(Op, ArgTy);2327 Arg.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgTy, /*IsSigned=*/false);2328 Arg.IsZExt = !Arg.IsSExt;2329 2330 SDValue Res = ExpandLibCall(LC, Node, TargetLowering::ArgListTy{Arg},2331 /*IsSigned=*/true, IntVT)2332 .first;2333 2334 // If ExpandLibCall created a tail call, the result was already2335 // of the correct type. Otherwise, we need to sign extend it.2336 if (Res.getValueType() != MVT::Other)2337 Res = DAG.getSExtOrTrunc(Res, SDLoc(Node), Node->getValueType(0));2338 return Res;2339}2340 2341/// Issue libcalls to __{u}divmod to compute div / rem pairs.2342void2343SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,2344 SmallVectorImpl<SDValue> &Results) {2345 unsigned Opcode = Node->getOpcode();2346 bool isSigned = Opcode == ISD::SDIVREM;2347 2348 RTLIB::Libcall LC;2349 switch (Node->getSimpleValueType(0).SimpleTy) {2350 default: llvm_unreachable("Unexpected request for libcall!");2351 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;2352 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;2353 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;2354 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;2355 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;2356 }2357 2358 // The input chain to this libcall is the entry node of the function.2359 // Legalizing the call will automatically add the previous call to the2360 // dependence.2361 SDValue InChain = DAG.getEntryNode();2362 2363 EVT RetVT = Node->getValueType(0);2364 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());2365 2366 TargetLowering::ArgListTy Args;2367 for (const SDValue &Op : Node->op_values()) {2368 EVT ArgVT = Op.getValueType();2369 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());2370 TargetLowering::ArgListEntry Entry(Op, ArgTy);2371 Entry.IsSExt = isSigned;2372 Entry.IsZExt = !isSigned;2373 Args.push_back(Entry);2374 }2375 2376 // Also pass the return address of the remainder.2377 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);2378 TargetLowering::ArgListEntry Entry(2379 FIPtr, PointerType::getUnqual(RetTy->getContext()));2380 Entry.IsSExt = isSigned;2381 Entry.IsZExt = !isSigned;2382 Args.push_back(Entry);2383 2384 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),2385 TLI.getPointerTy(DAG.getDataLayout()));2386 2387 SDLoc dl(Node);2388 TargetLowering::CallLoweringInfo CLI(DAG);2389 CLI.setDebugLoc(dl)2390 .setChain(InChain)2391 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,2392 std::move(Args))2393 .setSExtResult(isSigned)2394 .setZExtResult(!isSigned);2395 2396 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);2397 2398 // Remainder is loaded back from the stack frame.2399 SDValue Rem =2400 DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());2401 Results.push_back(CallInfo.first);2402 Results.push_back(Rem);2403}2404 2405/// Return true if sincos or __sincos_stret libcall is available.2406static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {2407 MVT::SimpleValueType VT = Node->getSimpleValueType(0).SimpleTy;2408 return TLI.getLibcallImpl(RTLIB::getSINCOS(VT)) != RTLIB::Unsupported ||2409 TLI.getLibcallImpl(RTLIB::getSINCOS_STRET(VT)) != RTLIB::Unsupported;2410}2411 2412/// Only issue sincos libcall if both sin and cos are needed.2413static bool useSinCos(SDNode *Node) {2414 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN2415 ? ISD::FCOS : ISD::FSIN;2416 2417 SDValue Op0 = Node->getOperand(0);2418 for (const SDNode *User : Op0.getNode()->users()) {2419 if (User == Node)2420 continue;2421 // The other user might have been turned into sincos already.2422 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)2423 return true;2424 }2425 return false;2426}2427 2428SDValue SelectionDAGLegalize::ExpandSincosStretLibCall(SDNode *Node) const {2429 // For iOS, we want to call an alternative entry point: __sincos_stret,2430 // which returns the values in two S / D registers.2431 SDLoc dl(Node);2432 SDValue Arg = Node->getOperand(0);2433 EVT ArgVT = Arg.getValueType();2434 RTLIB::Libcall LC = RTLIB::getSINCOS_STRET(ArgVT);2435 RTLIB::LibcallImpl SincosStret = TLI.getLibcallImpl(LC);2436 if (SincosStret == RTLIB::Unsupported)2437 return SDValue();2438 2439 /// There are 3 different ABI cases to handle:2440 /// - Direct return of separate fields in registers2441 /// - Single return as vector elements2442 /// - sret struct2443 2444 const RTLIB::RuntimeLibcallsInfo &CallsInfo = TLI.getRuntimeLibcallsInfo();2445 2446 const DataLayout &DL = DAG.getDataLayout();2447 2448 auto [FuncTy, FuncAttrs] = CallsInfo.getFunctionTy(2449 *DAG.getContext(), TM.getTargetTriple(), DL, SincosStret);2450 2451 Type *SincosStretRetTy = FuncTy->getReturnType();2452 CallingConv::ID CallConv = CallsInfo.getLibcallImplCallingConv(SincosStret);2453 StringRef LibcallImplName = CallsInfo.getLibcallImplName(SincosStret);2454 2455 SDValue Callee = DAG.getExternalSymbol(LibcallImplName.data(),2456 TLI.getProgramPointerTy(DL));2457 2458 TargetLowering::ArgListTy Args;2459 SDValue SRet;2460 2461 int FrameIdx;2462 if (FuncTy->getParamType(0)->isPointerTy()) {2463 // Uses sret2464 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();2465 2466 AttributeSet PtrAttrs = FuncAttrs.getParamAttrs(0);2467 Type *StructTy = PtrAttrs.getStructRetType();2468 const uint64_t ByteSize = DL.getTypeAllocSize(StructTy);2469 const Align StackAlign = DL.getPrefTypeAlign(StructTy);2470 2471 FrameIdx = MFI.CreateStackObject(ByteSize, StackAlign, false);2472 SRet = DAG.getFrameIndex(FrameIdx, TLI.getFrameIndexTy(DL));2473 2474 TargetLowering::ArgListEntry Entry(SRet, FuncTy->getParamType(0));2475 Entry.IsSRet = true;2476 Entry.IndirectType = StructTy;2477 Entry.Alignment = StackAlign;2478 2479 Args.push_back(Entry);2480 Args.emplace_back(Arg, FuncTy->getParamType(1));2481 } else {2482 Args.emplace_back(Arg, FuncTy->getParamType(0));2483 }2484 2485 TargetLowering::CallLoweringInfo CLI(DAG);2486 CLI.setDebugLoc(dl)2487 .setChain(DAG.getEntryNode())2488 .setLibCallee(CallConv, SincosStretRetTy, Callee, std::move(Args))2489 .setIsPostTypeLegalization();2490 2491 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);2492 2493 if (SRet) {2494 MachinePointerInfo PtrInfo =2495 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);2496 SDValue LoadSin = DAG.getLoad(ArgVT, dl, CallResult.second, SRet, PtrInfo);2497 2498 TypeSize StoreSize = ArgVT.getStoreSize();2499 2500 // Address of cos field.2501 SDValue Add = DAG.getObjectPtrOffset(dl, SRet, StoreSize);2502 SDValue LoadCos = DAG.getLoad(ArgVT, dl, LoadSin.getValue(1), Add,2503 PtrInfo.getWithOffset(StoreSize));2504 2505 SDVTList Tys = DAG.getVTList(ArgVT, ArgVT);2506 return DAG.getNode(ISD::MERGE_VALUES, dl, Tys, LoadSin.getValue(0),2507 LoadCos.getValue(0));2508 }2509 2510 if (!CallResult.first.getValueType().isVector())2511 return CallResult.first;2512 2513 SDValue SinVal =2514 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT, CallResult.first,2515 DAG.getVectorIdxConstant(0, dl));2516 SDValue CosVal =2517 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT, CallResult.first,2518 DAG.getVectorIdxConstant(1, dl));2519 SDVTList Tys = DAG.getVTList(ArgVT, ArgVT);2520 return DAG.getNode(ISD::MERGE_VALUES, dl, Tys, SinVal, CosVal);2521}2522 2523SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const {2524 SDLoc dl(Node);2525 EVT VT = Node->getValueType(0);2526 SDValue X = Node->getOperand(0);2527 SDValue N = Node->getOperand(1);2528 EVT ExpVT = N.getValueType();2529 EVT AsIntVT = VT.changeTypeToInteger();2530 if (AsIntVT == EVT()) // TODO: How to handle f80?2531 return SDValue();2532 2533 if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO2534 return SDValue();2535 2536 SDNodeFlags NSW;2537 NSW.setNoSignedWrap(true);2538 SDNodeFlags NUW_NSW;2539 NUW_NSW.setNoUnsignedWrap(true);2540 NUW_NSW.setNoSignedWrap(true);2541 2542 EVT SetCCVT =2543 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ExpVT);2544 const fltSemantics &FltSem = VT.getFltSemantics();2545 2546 const APFloat::ExponentType MaxExpVal = APFloat::semanticsMaxExponent(FltSem);2547 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);2548 const int Precision = APFloat::semanticsPrecision(FltSem);2549 2550 const SDValue MaxExp = DAG.getSignedConstant(MaxExpVal, dl, ExpVT);2551 const SDValue MinExp = DAG.getSignedConstant(MinExpVal, dl, ExpVT);2552 2553 const SDValue DoubleMaxExp = DAG.getSignedConstant(2 * MaxExpVal, dl, ExpVT);2554 2555 const APFloat One(FltSem, "1.0");2556 APFloat ScaleUpK = scalbn(One, MaxExpVal, APFloat::rmNearestTiesToEven);2557 2558 // Offset by precision to avoid denormal range.2559 APFloat ScaleDownK =2560 scalbn(One, MinExpVal + Precision, APFloat::rmNearestTiesToEven);2561 2562 // TODO: Should really introduce control flow and use a block for the >2563 // MaxExp, < MinExp cases2564 2565 // First, handle exponents Exp > MaxExp and scale down.2566 SDValue NGtMaxExp = DAG.getSetCC(dl, SetCCVT, N, MaxExp, ISD::SETGT);2567 2568 SDValue DecN0 = DAG.getNode(ISD::SUB, dl, ExpVT, N, MaxExp, NSW);2569 SDValue ClampMaxVal = DAG.getConstant(3 * MaxExpVal, dl, ExpVT);2570 SDValue ClampN_Big = DAG.getNode(ISD::SMIN, dl, ExpVT, N, ClampMaxVal);2571 SDValue DecN1 =2572 DAG.getNode(ISD::SUB, dl, ExpVT, ClampN_Big, DoubleMaxExp, NSW);2573 2574 SDValue ScaleUpTwice =2575 DAG.getSetCC(dl, SetCCVT, N, DoubleMaxExp, ISD::SETUGT);2576 2577 const SDValue ScaleUpVal = DAG.getConstantFP(ScaleUpK, dl, VT);2578 SDValue ScaleUp0 = DAG.getNode(ISD::FMUL, dl, VT, X, ScaleUpVal);2579 SDValue ScaleUp1 = DAG.getNode(ISD::FMUL, dl, VT, ScaleUp0, ScaleUpVal);2580 2581 SDValue SelectN_Big =2582 DAG.getNode(ISD::SELECT, dl, ExpVT, ScaleUpTwice, DecN1, DecN0);2583 SDValue SelectX_Big =2584 DAG.getNode(ISD::SELECT, dl, VT, ScaleUpTwice, ScaleUp1, ScaleUp0);2585 2586 // Now handle exponents Exp < MinExp2587 SDValue NLtMinExp = DAG.getSetCC(dl, SetCCVT, N, MinExp, ISD::SETLT);2588 2589 SDValue Increment0 = DAG.getConstant(-(MinExpVal + Precision), dl, ExpVT);2590 SDValue Increment1 = DAG.getConstant(-2 * (MinExpVal + Precision), dl, ExpVT);2591 2592 SDValue IncN0 = DAG.getNode(ISD::ADD, dl, ExpVT, N, Increment0, NUW_NSW);2593 2594 SDValue ClampMinVal =2595 DAG.getSignedConstant(3 * MinExpVal + 2 * Precision, dl, ExpVT);2596 SDValue ClampN_Small = DAG.getNode(ISD::SMAX, dl, ExpVT, N, ClampMinVal);2597 SDValue IncN1 =2598 DAG.getNode(ISD::ADD, dl, ExpVT, ClampN_Small, Increment1, NSW);2599 2600 const SDValue ScaleDownVal = DAG.getConstantFP(ScaleDownK, dl, VT);2601 SDValue ScaleDown0 = DAG.getNode(ISD::FMUL, dl, VT, X, ScaleDownVal);2602 SDValue ScaleDown1 = DAG.getNode(ISD::FMUL, dl, VT, ScaleDown0, ScaleDownVal);2603 2604 SDValue ScaleDownTwice = DAG.getSetCC(2605 dl, SetCCVT, N,2606 DAG.getSignedConstant(2 * MinExpVal + Precision, dl, ExpVT), ISD::SETULT);2607 2608 SDValue SelectN_Small =2609 DAG.getNode(ISD::SELECT, dl, ExpVT, ScaleDownTwice, IncN1, IncN0);2610 SDValue SelectX_Small =2611 DAG.getNode(ISD::SELECT, dl, VT, ScaleDownTwice, ScaleDown1, ScaleDown0);2612 2613 // Now combine the two out of range exponent handling cases with the base2614 // case.2615 SDValue NewX = DAG.getNode(2616 ISD::SELECT, dl, VT, NGtMaxExp, SelectX_Big,2617 DAG.getNode(ISD::SELECT, dl, VT, NLtMinExp, SelectX_Small, X));2618 2619 SDValue NewN = DAG.getNode(2620 ISD::SELECT, dl, ExpVT, NGtMaxExp, SelectN_Big,2621 DAG.getNode(ISD::SELECT, dl, ExpVT, NLtMinExp, SelectN_Small, N));2622 2623 SDValue BiasedN = DAG.getNode(ISD::ADD, dl, ExpVT, NewN, MaxExp, NSW);2624 2625 SDValue ExponentShiftAmt =2626 DAG.getShiftAmountConstant(Precision - 1, ExpVT, dl);2627 SDValue CastExpToValTy = DAG.getZExtOrTrunc(BiasedN, dl, AsIntVT);2628 2629 SDValue AsInt = DAG.getNode(ISD::SHL, dl, AsIntVT, CastExpToValTy,2630 ExponentShiftAmt, NUW_NSW);2631 SDValue AsFP = DAG.getNode(ISD::BITCAST, dl, VT, AsInt);2632 return DAG.getNode(ISD::FMUL, dl, VT, NewX, AsFP);2633}2634 2635SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const {2636 SDLoc dl(Node);2637 SDValue Val = Node->getOperand(0);2638 EVT VT = Val.getValueType();2639 EVT ExpVT = Node->getValueType(1);2640 EVT AsIntVT = VT.changeTypeToInteger();2641 if (AsIntVT == EVT()) // TODO: How to handle f80?2642 return SDValue();2643 2644 const fltSemantics &FltSem = VT.getFltSemantics();2645 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);2646 const unsigned Precision = APFloat::semanticsPrecision(FltSem);2647 const unsigned BitSize = VT.getScalarSizeInBits();2648 2649 // TODO: Could introduce control flow and skip over the denormal handling.2650 2651 // scale_up = fmul value, scalbn(1.0, precision + 1)2652 // extracted_exp = (bitcast value to uint) >> precision - 12653 // biased_exp = extracted_exp + min_exp2654 // extracted_fract = (bitcast value to uint) & (fract_mask | sign_mask)2655 //2656 // is_denormal = val < smallest_normalized2657 // computed_fract = is_denormal ? scale_up : extracted_fract2658 // computed_exp = is_denormal ? biased_exp + (-precision - 1) : biased_exp2659 //2660 // result_0 = (!isfinite(val) || iszero(val)) ? val : computed_fract2661 // result_1 = (!isfinite(val) || iszero(val)) ? 0 : computed_exp2662 2663 SDValue NegSmallestNormalizedInt = DAG.getConstant(2664 APFloat::getSmallestNormalized(FltSem, true).bitcastToAPInt(), dl,2665 AsIntVT);2666 2667 SDValue SmallestNormalizedInt = DAG.getConstant(2668 APFloat::getSmallestNormalized(FltSem, false).bitcastToAPInt(), dl,2669 AsIntVT);2670 2671 // Masks out the exponent bits.2672 SDValue ExpMask =2673 DAG.getConstant(APFloat::getInf(FltSem).bitcastToAPInt(), dl, AsIntVT);2674 2675 // Mask out the exponent part of the value.2676 //2677 // e.g, for f32 FractSignMaskVal = 0x807fffff2678 APInt FractSignMaskVal = APInt::getBitsSet(BitSize, 0, Precision - 1);2679 FractSignMaskVal.setBit(BitSize - 1); // Set the sign bit2680 2681 APInt SignMaskVal = APInt::getSignedMaxValue(BitSize);2682 SDValue SignMask = DAG.getConstant(SignMaskVal, dl, AsIntVT);2683 2684 SDValue FractSignMask = DAG.getConstant(FractSignMaskVal, dl, AsIntVT);2685 2686 const APFloat One(FltSem, "1.0");2687 // Scale a possible denormal input.2688 // e.g., for f64, 0x1p+542689 APFloat ScaleUpKVal =2690 scalbn(One, Precision + 1, APFloat::rmNearestTiesToEven);2691 2692 SDValue ScaleUpK = DAG.getConstantFP(ScaleUpKVal, dl, VT);2693 SDValue ScaleUp = DAG.getNode(ISD::FMUL, dl, VT, Val, ScaleUpK);2694 2695 EVT SetCCVT =2696 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);2697 2698 SDValue AsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, Val);2699 2700 SDValue Abs = DAG.getNode(ISD::AND, dl, AsIntVT, AsInt, SignMask);2701 2702 SDValue AddNegSmallestNormal =2703 DAG.getNode(ISD::ADD, dl, AsIntVT, Abs, NegSmallestNormalizedInt);2704 SDValue DenormOrZero = DAG.getSetCC(dl, SetCCVT, AddNegSmallestNormal,2705 NegSmallestNormalizedInt, ISD::SETULE);2706 2707 SDValue IsDenormal =2708 DAG.getSetCC(dl, SetCCVT, Abs, SmallestNormalizedInt, ISD::SETULT);2709 2710 SDValue MinExp = DAG.getSignedConstant(MinExpVal, dl, ExpVT);2711 SDValue Zero = DAG.getConstant(0, dl, ExpVT);2712 2713 SDValue ScaledAsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, ScaleUp);2714 SDValue ScaledSelect =2715 DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ScaledAsInt, AsInt);2716 2717 SDValue ExpMaskScaled =2718 DAG.getNode(ISD::AND, dl, AsIntVT, ScaledAsInt, ExpMask);2719 2720 SDValue ScaledValue =2721 DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ExpMaskScaled, Abs);2722 2723 // Extract the exponent bits.2724 SDValue ExponentShiftAmt =2725 DAG.getShiftAmountConstant(Precision - 1, AsIntVT, dl);2726 SDValue ShiftedExp =2727 DAG.getNode(ISD::SRL, dl, AsIntVT, ScaledValue, ExponentShiftAmt);2728 SDValue Exp = DAG.getSExtOrTrunc(ShiftedExp, dl, ExpVT);2729 2730 SDValue NormalBiasedExp = DAG.getNode(ISD::ADD, dl, ExpVT, Exp, MinExp);2731 SDValue DenormalOffset = DAG.getConstant(-Precision - 1, dl, ExpVT);2732 SDValue DenormalExpBias =2733 DAG.getNode(ISD::SELECT, dl, ExpVT, IsDenormal, DenormalOffset, Zero);2734 2735 SDValue MaskedFractAsInt =2736 DAG.getNode(ISD::AND, dl, AsIntVT, ScaledSelect, FractSignMask);2737 const APFloat Half(FltSem, "0.5");2738 SDValue FPHalf = DAG.getConstant(Half.bitcastToAPInt(), dl, AsIntVT);2739 SDValue Or = DAG.getNode(ISD::OR, dl, AsIntVT, MaskedFractAsInt, FPHalf);2740 SDValue MaskedFract = DAG.getNode(ISD::BITCAST, dl, VT, Or);2741 2742 SDValue ComputedExp =2743 DAG.getNode(ISD::ADD, dl, ExpVT, NormalBiasedExp, DenormalExpBias);2744 2745 SDValue Result0 =2746 DAG.getNode(ISD::SELECT, dl, VT, DenormOrZero, Val, MaskedFract);2747 2748 SDValue Result1 =2749 DAG.getNode(ISD::SELECT, dl, ExpVT, DenormOrZero, Zero, ComputedExp);2750 2751 return DAG.getMergeValues({Result0, Result1}, dl);2752}2753 2754/// This function is responsible for legalizing a2755/// INT_TO_FP operation of the specified operand when the target requests that2756/// we expand it. At this point, we know that the result and operand types are2757/// legal for the target.2758SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,2759 SDValue &Chain) {2760 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||2761 Node->getOpcode() == ISD::SINT_TO_FP);2762 EVT DestVT = Node->getValueType(0);2763 SDLoc dl(Node);2764 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;2765 SDValue Op0 = Node->getOperand(OpNo);2766 EVT SrcVT = Op0.getValueType();2767 2768 // TODO: Should any fast-math-flags be set for the created nodes?2769 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");2770 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&2771 (DestVT.bitsLE(MVT::f64) ||2772 TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND2773 : ISD::FP_EXTEND,2774 DestVT))) {2775 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "2776 "expansion\n");2777 2778 // Get the stack frame index of a 8 byte buffer.2779 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);2780 2781 SDValue Lo = Op0;2782 // if signed map to unsigned space2783 if (isSigned) {2784 // Invert sign bit (signed to unsigned mapping).2785 Lo = DAG.getNode(ISD::XOR, dl, MVT::i32, Lo,2786 DAG.getConstant(0x80000000u, dl, MVT::i32));2787 }2788 // Initial hi portion of constructed double.2789 SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32);2790 2791 // If this a big endian target, swap the lo and high data.2792 if (DAG.getDataLayout().isBigEndian())2793 std::swap(Lo, Hi);2794 2795 SDValue MemChain = DAG.getEntryNode();2796 2797 // Store the lo of the constructed double.2798 SDValue Store1 = DAG.getStore(MemChain, dl, Lo, StackSlot,2799 MachinePointerInfo());2800 // Store the hi of the constructed double.2801 SDValue HiPtr =2802 DAG.getMemBasePlusOffset(StackSlot, TypeSize::getFixed(4), dl);2803 SDValue Store2 =2804 DAG.getStore(MemChain, dl, Hi, HiPtr, MachinePointerInfo());2805 MemChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);2806 2807 // load the constructed double2808 SDValue Load =2809 DAG.getLoad(MVT::f64, dl, MemChain, StackSlot, MachinePointerInfo());2810 // FP constant to bias correct the final result2811 SDValue Bias = DAG.getConstantFP(2812 isSigned ? llvm::bit_cast<double>(0x4330000080000000ULL)2813 : llvm::bit_cast<double>(0x4330000000000000ULL),2814 dl, MVT::f64);2815 // Subtract the bias and get the final result.2816 SDValue Sub;2817 SDValue Result;2818 if (Node->isStrictFPOpcode()) {2819 Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other},2820 {Node->getOperand(0), Load, Bias});2821 Chain = Sub.getValue(1);2822 if (DestVT != Sub.getValueType()) {2823 std::pair<SDValue, SDValue> ResultPair;2824 ResultPair =2825 DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT);2826 Result = ResultPair.first;2827 Chain = ResultPair.second;2828 }2829 else2830 Result = Sub;2831 } else {2832 Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);2833 Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);2834 }2835 return Result;2836 }2837 2838 if (isSigned)2839 return SDValue();2840 2841 // TODO: Generalize this for use with other types.2842 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||2843 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {2844 LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");2845 // For unsigned conversions, convert them to signed conversions using the2846 // algorithm from the x86_64 __floatundisf in compiler_rt. That method2847 // should be valid for i32->f32 as well.2848 2849 // More generally this transform should be valid if there are 3 more bits2850 // in the integer type than the significand. Rounding uses the first bit2851 // after the width of the significand and the OR of all bits after that. So2852 // we need to be able to OR the shifted out bit into one of the bits that2853 // participate in the OR.2854 2855 // TODO: This really should be implemented using a branch rather than a2856 // select. We happen to get lucky and machinesink does the right2857 // thing most of the time. This would be a good candidate for a2858 // pseudo-op, or, even better, for whole-function isel.2859 EVT SetCCVT = getSetCCResultType(SrcVT);2860 2861 SDValue SignBitTest = DAG.getSetCC(2862 dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT), ISD::SETLT);2863 2864 SDValue ShiftConst = DAG.getShiftAmountConstant(1, SrcVT, dl);2865 SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Op0, ShiftConst);2866 SDValue AndConst = DAG.getConstant(1, dl, SrcVT);2867 SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Op0, AndConst);2868 SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr);2869 2870 SDValue Slow, Fast;2871 if (Node->isStrictFPOpcode()) {2872 // In strict mode, we must avoid spurious exceptions, and therefore2873 // must make sure to only emit a single STRICT_SINT_TO_FP.2874 SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest, Or, Op0);2875 // The STRICT_SINT_TO_FP inherits the exception mode from the2876 // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can2877 // never raise any exception.2878 SDNodeFlags Flags;2879 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());2880 Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {DestVT, MVT::Other},2881 {Node->getOperand(0), InCvt}, Flags);2882 Flags.setNoFPExcept(true);2883 Slow = DAG.getNode(ISD::STRICT_FADD, dl, {DestVT, MVT::Other},2884 {Fast.getValue(1), Fast, Fast}, Flags);2885 Chain = Slow.getValue(1);2886 } else {2887 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Or);2888 Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt);2889 Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);2890 }2891 2892 return DAG.getSelect(dl, DestVT, SignBitTest, Slow, Fast);2893 }2894 2895 // Don't expand it if there isn't cheap fadd.2896 if (!TLI.isOperationLegalOrCustom(2897 Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT))2898 return SDValue();2899 2900 // The following optimization is valid only if every value in SrcVT (when2901 // treated as signed) is representable in DestVT. Check that the mantissa2902 // size of DestVT is >= than the number of bits in SrcVT -1.2903 assert(APFloat::semanticsPrecision(DestVT.getFltSemantics()) >=2904 SrcVT.getSizeInBits() - 1 &&2905 "Cannot perform lossless SINT_TO_FP!");2906 2907 SDValue Tmp1;2908 if (Node->isStrictFPOpcode()) {2909 Tmp1 = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },2910 { Node->getOperand(0), Op0 });2911 } else2912 Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);2913 2914 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,2915 DAG.getConstant(0, dl, SrcVT), ISD::SETLT);2916 SDValue Zero = DAG.getIntPtrConstant(0, dl),2917 Four = DAG.getIntPtrConstant(4, dl);2918 SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),2919 SignSet, Four, Zero);2920 2921 // If the sign bit of the integer is set, the large number will be treated2922 // as a negative number. To counteract this, the dynamic code adds an2923 // offset depending on the data type.2924 uint64_t FF;2925 switch (SrcVT.getSimpleVT().SimpleTy) {2926 default:2927 return SDValue();2928 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)2929 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)2930 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)2931 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)2932 }2933 if (DAG.getDataLayout().isLittleEndian())2934 FF <<= 32;2935 Constant *FudgeFactor = ConstantInt::get(2936 Type::getInt64Ty(*DAG.getContext()), FF);2937 2938 SDValue CPIdx =2939 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));2940 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();2941 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);2942 Alignment = commonAlignment(Alignment, 4);2943 SDValue FudgeInReg;2944 if (DestVT == MVT::f32)2945 FudgeInReg = DAG.getLoad(2946 MVT::f32, dl, DAG.getEntryNode(), CPIdx,2947 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),2948 Alignment);2949 else {2950 SDValue Load = DAG.getExtLoad(2951 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,2952 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,2953 Alignment);2954 HandleSDNode Handle(Load);2955 LegalizeOp(Load.getNode());2956 FudgeInReg = Handle.getValue();2957 }2958 2959 if (Node->isStrictFPOpcode()) {2960 SDValue Result = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },2961 { Tmp1.getValue(1), Tmp1, FudgeInReg });2962 Chain = Result.getValue(1);2963 return Result;2964 }2965 2966 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);2967}2968 2969/// This function is responsible for legalizing a2970/// *INT_TO_FP operation of the specified operand when the target requests that2971/// we promote it. At this point, we know that the result and operand types are2972/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP2973/// operation that takes a larger input.2974void SelectionDAGLegalize::PromoteLegalINT_TO_FP(2975 SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) {2976 bool IsStrict = N->isStrictFPOpcode();2977 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||2978 N->getOpcode() == ISD::STRICT_SINT_TO_FP;2979 EVT DestVT = N->getValueType(0);2980 SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);2981 unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;2982 unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;2983 2984 // First step, figure out the appropriate *INT_TO_FP operation to use.2985 EVT NewInTy = LegalOp.getValueType();2986 2987 unsigned OpToUse = 0;2988 2989 // Scan for the appropriate larger type to use.2990 while (true) {2991 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);2992 assert(NewInTy.isInteger() && "Ran out of possibilities!");2993 2994 // If the target supports SINT_TO_FP of this type, use it.2995 if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) {2996 OpToUse = SIntOp;2997 break;2998 }2999 if (IsSigned)3000 continue;3001 3002 // If the target supports UINT_TO_FP of this type, use it.3003 if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) {3004 OpToUse = UIntOp;3005 break;3006 }3007 3008 // Otherwise, try a larger type.3009 }3010 3011 // Okay, we found the operation and type to use. Zero extend our input to the3012 // desired type then run the operation on it.3013 if (IsStrict) {3014 SDValue Res =3015 DAG.getNode(OpToUse, dl, {DestVT, MVT::Other},3016 {N->getOperand(0),3017 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,3018 dl, NewInTy, LegalOp)});3019 Results.push_back(Res);3020 Results.push_back(Res.getValue(1));3021 return;3022 }3023 3024 Results.push_back(3025 DAG.getNode(OpToUse, dl, DestVT,3026 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,3027 dl, NewInTy, LegalOp)));3028}3029 3030/// This function is responsible for legalizing a3031/// FP_TO_*INT operation of the specified operand when the target requests that3032/// we promote it. At this point, we know that the result and operand types are3033/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT3034/// operation that returns a larger result.3035void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,3036 SmallVectorImpl<SDValue> &Results) {3037 bool IsStrict = N->isStrictFPOpcode();3038 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||3039 N->getOpcode() == ISD::STRICT_FP_TO_SINT;3040 EVT DestVT = N->getValueType(0);3041 SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);3042 // First step, figure out the appropriate FP_TO*INT operation to use.3043 EVT NewOutTy = DestVT;3044 3045 unsigned OpToUse = 0;3046 3047 // Scan for the appropriate larger type to use.3048 while (true) {3049 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);3050 assert(NewOutTy.isInteger() && "Ran out of possibilities!");3051 3052 // A larger signed type can hold all unsigned values of the requested type,3053 // so using FP_TO_SINT is valid3054 OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;3055 if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))3056 break;3057 3058 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.3059 OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;3060 if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))3061 break;3062 3063 // Otherwise, try a larger type.3064 }3065 3066 // Okay, we found the operation and type to use.3067 SDValue Operation;3068 if (IsStrict) {3069 SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);3070 Operation = DAG.getNode(OpToUse, dl, VTs, N->getOperand(0), LegalOp);3071 } else3072 Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);3073 3074 // Truncate the result of the extended FP_TO_*INT operation to the desired3075 // size.3076 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);3077 Results.push_back(Trunc);3078 if (IsStrict)3079 Results.push_back(Operation.getValue(1));3080}3081 3082/// Promote FP_TO_*INT_SAT operation to a larger result type. At this point3083/// the result and operand types are legal and there must be a legal3084/// FP_TO_*INT_SAT operation for a larger result type.3085SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,3086 const SDLoc &dl) {3087 unsigned Opcode = Node->getOpcode();3088 3089 // Scan for the appropriate larger type to use.3090 EVT NewOutTy = Node->getValueType(0);3091 while (true) {3092 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);3093 assert(NewOutTy.isInteger() && "Ran out of possibilities!");3094 3095 if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy))3096 break;3097 }3098 3099 // Saturation width is determined by second operand, so we don't have to3100 // perform any fixup and can directly truncate the result.3101 SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0),3102 Node->getOperand(1));3103 return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result);3104}3105 3106/// Open code the operations for PARITY of the specified operation.3107SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {3108 EVT VT = Op.getValueType();3109 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());3110 unsigned Sz = VT.getScalarSizeInBits();3111 3112 // If CTPOP is legal, use it. Otherwise use shifts and xor.3113 SDValue Result;3114 if (TLI.isOperationLegalOrPromote(ISD::CTPOP, VT)) {3115 Result = DAG.getNode(ISD::CTPOP, dl, VT, Op);3116 } else {3117 Result = Op;3118 for (unsigned i = Log2_32_Ceil(Sz); i != 0;) {3119 SDValue Shift = DAG.getNode(ISD::SRL, dl, VT, Result,3120 DAG.getConstant(1ULL << (--i), dl, ShVT));3121 Result = DAG.getNode(ISD::XOR, dl, VT, Result, Shift);3122 }3123 }3124 3125 return DAG.getNode(ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT));3126}3127 3128SDValue SelectionDAGLegalize::PromoteReduction(SDNode *Node) {3129 bool IsVPOpcode = ISD::isVPOpcode(Node->getOpcode());3130 MVT VecVT = IsVPOpcode ? Node->getOperand(1).getSimpleValueType()3131 : Node->getOperand(0).getSimpleValueType();3132 MVT NewVecVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VecVT);3133 MVT ScalarVT = Node->getSimpleValueType(0);3134 MVT NewScalarVT = NewVecVT.getVectorElementType();3135 3136 SDLoc DL(Node);3137 SmallVector<SDValue, 4> Operands(Node->getNumOperands());3138 3139 // FIXME: Support integer.3140 assert(Node->getOperand(0).getValueType().isFloatingPoint() &&3141 "Only FP promotion is supported");3142 3143 for (unsigned j = 0; j != Node->getNumOperands(); ++j)3144 if (Node->getOperand(j).getValueType().isVector() &&3145 !(IsVPOpcode &&3146 ISD::getVPMaskIdx(Node->getOpcode()) == j)) { // Skip mask operand.3147 // promote the vector operand.3148 // FIXME: Support integer.3149 assert(Node->getOperand(j).getValueType().isFloatingPoint() &&3150 "Only FP promotion is supported");3151 Operands[j] =3152 DAG.getNode(ISD::FP_EXTEND, DL, NewVecVT, Node->getOperand(j));3153 } else if (Node->getOperand(j).getValueType().isFloatingPoint()) {3154 // promote the initial value.3155 Operands[j] =3156 DAG.getNode(ISD::FP_EXTEND, DL, NewScalarVT, Node->getOperand(j));3157 } else {3158 Operands[j] = Node->getOperand(j); // Skip VL operand.3159 }3160 3161 SDValue Res = DAG.getNode(Node->getOpcode(), DL, NewScalarVT, Operands,3162 Node->getFlags());3163 3164 assert(ScalarVT.isFloatingPoint() && "Only FP promotion is supported");3165 return DAG.getNode(ISD::FP_ROUND, DL, ScalarVT, Res,3166 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));3167}3168 3169bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {3170 LLVM_DEBUG(dbgs() << "Trying to expand node\n");3171 SmallVector<SDValue, 8> Results;3172 SDLoc dl(Node);3173 SDValue Tmp1, Tmp2, Tmp3, Tmp4;3174 bool NeedInvert;3175 switch (Node->getOpcode()) {3176 case ISD::ABS:3177 if ((Tmp1 = TLI.expandABS(Node, DAG)))3178 Results.push_back(Tmp1);3179 break;3180 case ISD::ABDS:3181 case ISD::ABDU:3182 if ((Tmp1 = TLI.expandABD(Node, DAG)))3183 Results.push_back(Tmp1);3184 break;3185 case ISD::AVGCEILS:3186 case ISD::AVGCEILU:3187 case ISD::AVGFLOORS:3188 case ISD::AVGFLOORU:3189 if ((Tmp1 = TLI.expandAVG(Node, DAG)))3190 Results.push_back(Tmp1);3191 break;3192 case ISD::CTPOP:3193 if ((Tmp1 = TLI.expandCTPOP(Node, DAG)))3194 Results.push_back(Tmp1);3195 break;3196 case ISD::CTLZ:3197 case ISD::CTLZ_ZERO_UNDEF:3198 if ((Tmp1 = TLI.expandCTLZ(Node, DAG)))3199 Results.push_back(Tmp1);3200 break;3201 case ISD::CTTZ:3202 case ISD::CTTZ_ZERO_UNDEF:3203 if ((Tmp1 = TLI.expandCTTZ(Node, DAG)))3204 Results.push_back(Tmp1);3205 break;3206 case ISD::BITREVERSE:3207 if ((Tmp1 = TLI.expandBITREVERSE(Node, DAG)))3208 Results.push_back(Tmp1);3209 break;3210 case ISD::BSWAP:3211 if ((Tmp1 = TLI.expandBSWAP(Node, DAG)))3212 Results.push_back(Tmp1);3213 break;3214 case ISD::PARITY:3215 Results.push_back(ExpandPARITY(Node->getOperand(0), dl));3216 break;3217 case ISD::FRAMEADDR:3218 case ISD::RETURNADDR:3219 case ISD::FRAME_TO_ARGS_OFFSET:3220 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));3221 break;3222 case ISD::EH_DWARF_CFA: {3223 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,3224 TLI.getPointerTy(DAG.getDataLayout()));3225 SDValue Offset = DAG.getNode(ISD::ADD, dl,3226 CfaArg.getValueType(),3227 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,3228 CfaArg.getValueType()),3229 CfaArg);3230 SDValue FA = DAG.getNode(3231 ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),3232 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));3233 Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),3234 FA, Offset));3235 break;3236 }3237 case ISD::GET_ROUNDING:3238 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));3239 Results.push_back(Node->getOperand(0));3240 break;3241 case ISD::EH_RETURN:3242 case ISD::EH_LABEL:3243 case ISD::PREFETCH:3244 case ISD::VAEND:3245 case ISD::EH_SJLJ_LONGJMP:3246 // If the target didn't expand these, there's nothing to do, so just3247 // preserve the chain and be done.3248 Results.push_back(Node->getOperand(0));3249 break;3250 case ISD::READCYCLECOUNTER:3251 case ISD::READSTEADYCOUNTER:3252 // If the target didn't expand this, just return 'zero' and preserve the3253 // chain.3254 Results.append(Node->getNumValues() - 1,3255 DAG.getConstant(0, dl, Node->getValueType(0)));3256 Results.push_back(Node->getOperand(0));3257 break;3258 case ISD::EH_SJLJ_SETJMP:3259 // If the target didn't expand this, just return 'zero' and preserve the3260 // chain.3261 Results.push_back(DAG.getConstant(0, dl, MVT::i32));3262 Results.push_back(Node->getOperand(0));3263 break;3264 case ISD::ATOMIC_LOAD: {3265 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.3266 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));3267 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);3268 SDValue Swap = DAG.getAtomicCmpSwap(3269 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,3270 Node->getOperand(0), Node->getOperand(1), Zero, Zero,3271 cast<AtomicSDNode>(Node)->getMemOperand());3272 Results.push_back(Swap.getValue(0));3273 Results.push_back(Swap.getValue(1));3274 break;3275 }3276 case ISD::ATOMIC_STORE: {3277 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.3278 SDValue Swap = DAG.getAtomic(3279 ISD::ATOMIC_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(),3280 Node->getOperand(0), Node->getOperand(2), Node->getOperand(1),3281 cast<AtomicSDNode>(Node)->getMemOperand());3282 Results.push_back(Swap.getValue(1));3283 break;3284 }3285 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {3286 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and3287 // splits out the success value as a comparison. Expanding the resulting3288 // ATOMIC_CMP_SWAP will produce a libcall.3289 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);3290 SDValue Res = DAG.getAtomicCmpSwap(3291 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,3292 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),3293 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());3294 3295 SDValue ExtRes = Res;3296 SDValue LHS = Res;3297 SDValue RHS = Node->getOperand(1);3298 3299 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();3300 EVT OuterType = Node->getValueType(0);3301 switch (TLI.getExtendForAtomicOps()) {3302 case ISD::SIGN_EXTEND:3303 LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,3304 DAG.getValueType(AtomicType));3305 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,3306 Node->getOperand(2), DAG.getValueType(AtomicType));3307 ExtRes = LHS;3308 break;3309 case ISD::ZERO_EXTEND:3310 LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,3311 DAG.getValueType(AtomicType));3312 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);3313 ExtRes = LHS;3314 break;3315 case ISD::ANY_EXTEND:3316 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);3317 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);3318 break;3319 default:3320 llvm_unreachable("Invalid atomic op extension");3321 }3322 3323 SDValue Success =3324 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);3325 3326 Results.push_back(ExtRes.getValue(0));3327 Results.push_back(Success);3328 Results.push_back(Res.getValue(1));3329 break;3330 }3331 case ISD::ATOMIC_LOAD_SUB: {3332 SDLoc DL(Node);3333 EVT VT = Node->getValueType(0);3334 SDValue RHS = Node->getOperand(2);3335 AtomicSDNode *AN = cast<AtomicSDNode>(Node);3336 if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG &&3337 cast<VTSDNode>(RHS->getOperand(1))->getVT() == AN->getMemoryVT())3338 RHS = RHS->getOperand(0);3339 SDValue NewRHS =3340 DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), RHS);3341 SDValue Res = DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, AN->getMemoryVT(),3342 Node->getOperand(0), Node->getOperand(1),3343 NewRHS, AN->getMemOperand());3344 Results.push_back(Res);3345 Results.push_back(Res.getValue(1));3346 break;3347 }3348 case ISD::DYNAMIC_STACKALLOC:3349 ExpandDYNAMIC_STACKALLOC(Node, Results);3350 break;3351 case ISD::MERGE_VALUES:3352 for (unsigned i = 0; i < Node->getNumValues(); i++)3353 Results.push_back(Node->getOperand(i));3354 break;3355 case ISD::POISON:3356 case ISD::UNDEF: {3357 EVT VT = Node->getValueType(0);3358 if (VT.isInteger())3359 Results.push_back(DAG.getConstant(0, dl, VT));3360 else {3361 assert(VT.isFloatingPoint() && "Unknown value type!");3362 Results.push_back(DAG.getConstantFP(0, dl, VT));3363 }3364 break;3365 }3366 case ISD::STRICT_FP_ROUND:3367 // When strict mode is enforced we can't do expansion because it3368 // does not honor the "strict" properties. Only libcall is allowed.3369 if (TLI.isStrictFPEnabled())3370 break;3371 // We might as well mutate to FP_ROUND when FP_ROUND operation is legal3372 // since this operation is more efficient than stack operation.3373 if (TLI.getStrictFPOperationAction(Node->getOpcode(),3374 Node->getValueType(0))3375 == TargetLowering::Legal)3376 break;3377 // We fall back to use stack operation when the FP_ROUND operation3378 // isn't available.3379 if ((Tmp1 = EmitStackConvert(Node->getOperand(1), Node->getValueType(0),3380 Node->getValueType(0), dl,3381 Node->getOperand(0)))) {3382 ReplaceNode(Node, Tmp1.getNode());3383 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");3384 return true;3385 }3386 break;3387 case ISD::FP_ROUND: {3388 if ((Tmp1 = TLI.expandFP_ROUND(Node, DAG))) {3389 Results.push_back(Tmp1);3390 break;3391 }3392 3393 [[fallthrough]];3394 }3395 case ISD::BITCAST:3396 if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),3397 Node->getValueType(0), dl)))3398 Results.push_back(Tmp1);3399 break;3400 case ISD::STRICT_FP_EXTEND:3401 // When strict mode is enforced we can't do expansion because it3402 // does not honor the "strict" properties. Only libcall is allowed.3403 if (TLI.isStrictFPEnabled())3404 break;3405 // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal3406 // since this operation is more efficient than stack operation.3407 if (TLI.getStrictFPOperationAction(Node->getOpcode(),3408 Node->getValueType(0))3409 == TargetLowering::Legal)3410 break;3411 // We fall back to use stack operation when the FP_EXTEND operation3412 // isn't available.3413 if ((Tmp1 = EmitStackConvert(3414 Node->getOperand(1), Node->getOperand(1).getValueType(),3415 Node->getValueType(0), dl, Node->getOperand(0)))) {3416 ReplaceNode(Node, Tmp1.getNode());3417 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");3418 return true;3419 }3420 break;3421 case ISD::FP_EXTEND: {3422 SDValue Op = Node->getOperand(0);3423 EVT SrcVT = Op.getValueType();3424 EVT DstVT = Node->getValueType(0);3425 if (SrcVT.getScalarType() == MVT::bf16) {3426 Results.push_back(DAG.getNode(ISD::BF16_TO_FP, SDLoc(Node), DstVT, Op));3427 break;3428 }3429 3430 if ((Tmp1 = EmitStackConvert(Op, SrcVT, DstVT, dl)))3431 Results.push_back(Tmp1);3432 break;3433 }3434 case ISD::BF16_TO_FP: {3435 // Always expand bf16 to f32 casts, they lower to ext + shift.3436 //3437 // Note that the operand of this code can be bf16 or an integer type in case3438 // bf16 is not supported on the target and was softened.3439 SDValue Op = Node->getOperand(0);3440 if (Op.getValueType() == MVT::bf16) {3441 Op = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32,3442 DAG.getNode(ISD::BITCAST, dl, MVT::i16, Op));3443 } else {3444 Op = DAG.getAnyExtOrTrunc(Op, dl, MVT::i32);3445 }3446 Op = DAG.getNode(ISD::SHL, dl, MVT::i32, Op,3447 DAG.getShiftAmountConstant(16, MVT::i32, dl));3448 Op = DAG.getNode(ISD::BITCAST, dl, MVT::f32, Op);3449 // Add fp_extend in case the output is bigger than f32.3450 if (Node->getValueType(0) != MVT::f32)3451 Op = DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Op);3452 Results.push_back(Op);3453 break;3454 }3455 case ISD::FP_TO_BF16: {3456 SDValue Op = Node->getOperand(0);3457 if (Op.getValueType() != MVT::f32)3458 Op = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,3459 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));3460 // Certain SNaNs will turn into infinities if we do a simple shift right.3461 if (!DAG.isKnownNeverSNaN(Op)) {3462 Op = DAG.getNode(ISD::FCANONICALIZE, dl, MVT::f32, Op, Node->getFlags());3463 }3464 Op = DAG.getNode(ISD::SRL, dl, MVT::i32,3465 DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op),3466 DAG.getShiftAmountConstant(16, MVT::i32, dl));3467 // The result of this node can be bf16 or an integer type in case bf16 is3468 // not supported on the target and was softened to i16 for storage.3469 if (Node->getValueType(0) == MVT::bf16) {3470 Op = DAG.getNode(ISD::BITCAST, dl, MVT::bf16,3471 DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Op));3472 } else {3473 Op = DAG.getAnyExtOrTrunc(Op, dl, Node->getValueType(0));3474 }3475 Results.push_back(Op);3476 break;3477 }3478 case ISD::FCANONICALIZE: {3479 // This implements llvm.canonicalize.f* by multiplication with 1.0, as3480 // suggested in3481 // https://llvm.org/docs/LangRef.html#llvm-canonicalize-intrinsic.3482 // It uses strict_fp operations even outside a strict_fp context in order3483 // to guarantee that the canonicalization is not optimized away by later3484 // passes. The result chain introduced by that is intentionally ignored3485 // since no ordering requirement is intended here.3486 3487 // Create strict multiplication by 1.0.3488 SDValue Operand = Node->getOperand(0);3489 EVT VT = Operand.getValueType();3490 SDValue One = DAG.getConstantFP(1.0, dl, VT);3491 SDValue Chain = DAG.getEntryNode();3492 // Propagate existing flags on canonicalize, and additionally set3493 // NoFPExcept.3494 SDNodeFlags CanonicalizeFlags = Node->getFlags();3495 CanonicalizeFlags.setNoFPExcept(true);3496 SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, dl, {VT, MVT::Other},3497 {Chain, Operand, One}, CanonicalizeFlags);3498 3499 Results.push_back(Mul);3500 break;3501 }3502 case ISD::SIGN_EXTEND_INREG: {3503 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();3504 EVT VT = Node->getValueType(0);3505 3506 // An in-register sign-extend of a boolean is a negation:3507 // 'true' (1) sign-extended is -1.3508 // 'false' (0) sign-extended is 0.3509 // However, we must mask the high bits of the source operand because the3510 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.3511 3512 // TODO: Do this for vectors too?3513 if (ExtraVT.isScalarInteger() && ExtraVT.getSizeInBits() == 1) {3514 SDValue One = DAG.getConstant(1, dl, VT);3515 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);3516 SDValue Zero = DAG.getConstant(0, dl, VT);3517 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);3518 Results.push_back(Neg);3519 break;3520 }3521 3522 // NOTE: we could fall back on load/store here too for targets without3523 // SRA. However, it is doubtful that any exist.3524 unsigned BitsDiff = VT.getScalarSizeInBits() -3525 ExtraVT.getScalarSizeInBits();3526 SDValue ShiftCst = DAG.getShiftAmountConstant(BitsDiff, VT, dl);3527 Tmp1 = DAG.getNode(ISD::SHL, dl, VT, Node->getOperand(0), ShiftCst);3528 Tmp1 = DAG.getNode(ISD::SRA, dl, VT, Tmp1, ShiftCst);3529 Results.push_back(Tmp1);3530 break;3531 }3532 case ISD::UINT_TO_FP:3533 case ISD::STRICT_UINT_TO_FP:3534 if (TLI.expandUINT_TO_FP(Node, Tmp1, Tmp2, DAG)) {3535 Results.push_back(Tmp1);3536 if (Node->isStrictFPOpcode())3537 Results.push_back(Tmp2);3538 break;3539 }3540 [[fallthrough]];3541 case ISD::SINT_TO_FP:3542 case ISD::STRICT_SINT_TO_FP:3543 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Tmp2))) {3544 Results.push_back(Tmp1);3545 if (Node->isStrictFPOpcode())3546 Results.push_back(Tmp2);3547 }3548 break;3549 case ISD::FP_TO_SINT:3550 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))3551 Results.push_back(Tmp1);3552 break;3553 case ISD::STRICT_FP_TO_SINT:3554 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) {3555 ReplaceNode(Node, Tmp1.getNode());3556 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");3557 return true;3558 }3559 break;3560 case ISD::FP_TO_UINT:3561 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG))3562 Results.push_back(Tmp1);3563 break;3564 case ISD::STRICT_FP_TO_UINT:3565 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) {3566 // Relink the chain.3567 DAG.ReplaceAllUsesOfValueWith(SDValue(Node,1), Tmp2);3568 // Replace the new UINT result.3569 ReplaceNodeWithValue(SDValue(Node, 0), Tmp1);3570 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");3571 return true;3572 }3573 break;3574 case ISD::FP_TO_SINT_SAT:3575 case ISD::FP_TO_UINT_SAT:3576 Results.push_back(TLI.expandFP_TO_INT_SAT(Node, DAG));3577 break;3578 case ISD::LROUND:3579 case ISD::LLROUND: {3580 SDValue Arg = Node->getOperand(0);3581 EVT ArgVT = Arg.getValueType();3582 EVT ResVT = Node->getValueType(0);3583 SDLoc dl(Node);3584 SDValue RoundNode = DAG.getNode(ISD::FROUND, dl, ArgVT, Arg);3585 Results.push_back(DAG.getNode(ISD::FP_TO_SINT, dl, ResVT, RoundNode));3586 break;3587 }3588 case ISD::VAARG:3589 Results.push_back(DAG.expandVAArg(Node));3590 Results.push_back(Results[0].getValue(1));3591 break;3592 case ISD::VACOPY:3593 Results.push_back(DAG.expandVACopy(Node));3594 break;3595 case ISD::EXTRACT_VECTOR_ELT:3596 if (Node->getOperand(0).getValueType().getVectorElementCount().isScalar())3597 // This must be an access of the only element. Return it.3598 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),3599 Node->getOperand(0));3600 else3601 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));3602 Results.push_back(Tmp1);3603 break;3604 case ISD::EXTRACT_SUBVECTOR:3605 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));3606 break;3607 case ISD::INSERT_SUBVECTOR:3608 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));3609 break;3610 case ISD::CONCAT_VECTORS:3611 if (EVT VectorValueType = Node->getOperand(0).getValueType();3612 VectorValueType.isScalableVector() ||3613 TLI.isOperationExpand(ISD::EXTRACT_VECTOR_ELT, VectorValueType))3614 Results.push_back(ExpandVectorBuildThroughStack(Node));3615 else3616 Results.push_back(ExpandConcatVectors(Node));3617 break;3618 case ISD::SCALAR_TO_VECTOR:3619 Results.push_back(ExpandSCALAR_TO_VECTOR(Node));3620 break;3621 case ISD::INSERT_VECTOR_ELT:3622 Results.push_back(ExpandINSERT_VECTOR_ELT(SDValue(Node, 0)));3623 break;3624 case ISD::VECTOR_SHUFFLE: {3625 SmallVector<int, 32> NewMask;3626 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();3627 3628 EVT VT = Node->getValueType(0);3629 EVT EltVT = VT.getVectorElementType();3630 SDValue Op0 = Node->getOperand(0);3631 SDValue Op1 = Node->getOperand(1);3632 if (!TLI.isTypeLegal(EltVT)) {3633 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);3634 3635 // BUILD_VECTOR operands are allowed to be wider than the element type.3636 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept3637 // it.3638 if (NewEltVT.bitsLT(EltVT)) {3639 // Convert shuffle node.3640 // If original node was v4i64 and the new EltVT is i32,3641 // cast operands to v8i32 and re-build the mask.3642 3643 // Calculate new VT, the size of the new VT should be equal to original.3644 EVT NewVT =3645 EVT::getVectorVT(*DAG.getContext(), NewEltVT,3646 VT.getSizeInBits() / NewEltVT.getSizeInBits());3647 assert(NewVT.bitsEq(VT));3648 3649 // cast operands to new VT3650 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);3651 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);3652 3653 // Convert the shuffle mask3654 unsigned int factor =3655 NewVT.getVectorNumElements()/VT.getVectorNumElements();3656 3657 // EltVT gets smaller3658 assert(factor > 0);3659 3660 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {3661 if (Mask[i] < 0) {3662 for (unsigned fi = 0; fi < factor; ++fi)3663 NewMask.push_back(Mask[i]);3664 }3665 else {3666 for (unsigned fi = 0; fi < factor; ++fi)3667 NewMask.push_back(Mask[i]*factor+fi);3668 }3669 }3670 Mask = NewMask;3671 VT = NewVT;3672 }3673 EltVT = NewEltVT;3674 }3675 unsigned NumElems = VT.getVectorNumElements();3676 SmallVector<SDValue, 16> Ops;3677 for (unsigned i = 0; i != NumElems; ++i) {3678 if (Mask[i] < 0) {3679 Ops.push_back(DAG.getUNDEF(EltVT));3680 continue;3681 }3682 unsigned Idx = Mask[i];3683 if (Idx < NumElems)3684 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,3685 DAG.getVectorIdxConstant(Idx, dl)));3686 else3687 Ops.push_back(3688 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,3689 DAG.getVectorIdxConstant(Idx - NumElems, dl)));3690 }3691 3692 Tmp1 = DAG.getBuildVector(VT, dl, Ops);3693 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.3694 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);3695 Results.push_back(Tmp1);3696 break;3697 }3698 case ISD::VECTOR_SPLICE: {3699 Results.push_back(TLI.expandVectorSplice(Node, DAG));3700 break;3701 }3702 case ISD::VECTOR_DEINTERLEAVE: {3703 unsigned Factor = Node->getNumOperands();3704 if (Factor <= 2 || !isPowerOf2_32(Factor))3705 break;3706 SmallVector<SDValue, 8> Ops(Node->ops());3707 EVT VecVT = Node->getValueType(0);3708 SmallVector<EVT> HalfVTs(Factor / 2, VecVT);3709 // Deinterleave at Factor/2 so each result contains two factors interleaved:3710 // a0b0 c0d0 a1b1 c1d1 -> [a0c0 b0d0] [a1c1 b1d1]3711 SDValue L = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, dl, HalfVTs,3712 ArrayRef(Ops).take_front(Factor / 2));3713 SDValue R = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, dl, HalfVTs,3714 ArrayRef(Ops).take_back(Factor / 2));3715 Results.resize(Factor);3716 // Deinterleave the 2 factors out:3717 // [a0c0 a1c1] [b0d0 b1d1] -> a0a1 b0b1 c0c1 d0d13718 for (unsigned I = 0; I < Factor / 2; I++) {3719 SDValue Deinterleave =3720 DAG.getNode(ISD::VECTOR_DEINTERLEAVE, dl, {VecVT, VecVT},3721 {L.getValue(I), R.getValue(I)});3722 Results[I] = Deinterleave.getValue(0);3723 Results[I + Factor / 2] = Deinterleave.getValue(1);3724 }3725 break;3726 }3727 case ISD::VECTOR_INTERLEAVE: {3728 unsigned Factor = Node->getNumOperands();3729 if (Factor <= 2 || !isPowerOf2_32(Factor))3730 break;3731 EVT VecVT = Node->getValueType(0);3732 SmallVector<EVT> HalfVTs(Factor / 2, VecVT);3733 SmallVector<SDValue, 8> LOps, ROps;3734 // Interleave so we have 2 factors per result:3735 // a0a1 b0b1 c0c1 d0d1 -> [a0c0 b0d0] [a1c1 b1d1]3736 for (unsigned I = 0; I < Factor / 2; I++) {3737 SDValue Interleave =3738 DAG.getNode(ISD::VECTOR_INTERLEAVE, dl, {VecVT, VecVT},3739 {Node->getOperand(I), Node->getOperand(I + Factor / 2)});3740 LOps.push_back(Interleave.getValue(0));3741 ROps.push_back(Interleave.getValue(1));3742 }3743 // Interleave at Factor/2:3744 // [a0c0 b0d0] [a1c1 b1d1] -> a0b0 c0d0 a1b1 c1d13745 SDValue L = DAG.getNode(ISD::VECTOR_INTERLEAVE, dl, HalfVTs, LOps);3746 SDValue R = DAG.getNode(ISD::VECTOR_INTERLEAVE, dl, HalfVTs, ROps);3747 for (unsigned I = 0; I < Factor / 2; I++)3748 Results.push_back(L.getValue(I));3749 for (unsigned I = 0; I < Factor / 2; I++)3750 Results.push_back(R.getValue(I));3751 break;3752 }3753 case ISD::EXTRACT_ELEMENT: {3754 EVT OpTy = Node->getOperand(0).getValueType();3755 if (Node->getConstantOperandVal(1)) {3756 // 1 -> Hi3757 Tmp1 = DAG.getNode(3758 ISD::SRL, dl, OpTy, Node->getOperand(0),3759 DAG.getShiftAmountConstant(OpTy.getSizeInBits() / 2, OpTy, dl));3760 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);3761 } else {3762 // 0 -> Lo3763 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),3764 Node->getOperand(0));3765 }3766 Results.push_back(Tmp1);3767 break;3768 }3769 case ISD::STACKSAVE:3770 // Expand to CopyFromReg if the target set3771 // StackPointerRegisterToSaveRestore.3772 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {3773 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,3774 Node->getValueType(0)));3775 Results.push_back(Results[0].getValue(1));3776 } else {3777 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));3778 Results.push_back(Node->getOperand(0));3779 }3780 break;3781 case ISD::STACKRESTORE:3782 // Expand to CopyToReg if the target set3783 // StackPointerRegisterToSaveRestore.3784 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {3785 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,3786 Node->getOperand(1)));3787 } else {3788 Results.push_back(Node->getOperand(0));3789 }3790 break;3791 case ISD::GET_DYNAMIC_AREA_OFFSET:3792 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));3793 Results.push_back(Results[0].getValue(0));3794 break;3795 case ISD::FCOPYSIGN:3796 Results.push_back(ExpandFCOPYSIGN(Node));3797 break;3798 case ISD::FNEG:3799 Results.push_back(ExpandFNEG(Node));3800 break;3801 case ISD::FABS:3802 Results.push_back(ExpandFABS(Node));3803 break;3804 case ISD::IS_FPCLASS: {3805 auto Test = static_cast<FPClassTest>(Node->getConstantOperandVal(1));3806 if (SDValue Expanded =3807 TLI.expandIS_FPCLASS(Node->getValueType(0), Node->getOperand(0),3808 Test, Node->getFlags(), SDLoc(Node), DAG))3809 Results.push_back(Expanded);3810 break;3811 }3812 case ISD::SMIN:3813 case ISD::SMAX:3814 case ISD::UMIN:3815 case ISD::UMAX: {3816 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B3817 ISD::CondCode Pred;3818 switch (Node->getOpcode()) {3819 default: llvm_unreachable("How did we get here?");3820 case ISD::SMAX: Pred = ISD::SETGT; break;3821 case ISD::SMIN: Pred = ISD::SETLT; break;3822 case ISD::UMAX: Pred = ISD::SETUGT; break;3823 case ISD::UMIN: Pred = ISD::SETULT; break;3824 }3825 Tmp1 = Node->getOperand(0);3826 Tmp2 = Node->getOperand(1);3827 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);3828 Results.push_back(Tmp1);3829 break;3830 }3831 case ISD::FMINNUM:3832 case ISD::FMAXNUM: {3833 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG))3834 Results.push_back(Expanded);3835 break;3836 }3837 case ISD::FMINIMUM:3838 case ISD::FMAXIMUM: {3839 if (SDValue Expanded = TLI.expandFMINIMUM_FMAXIMUM(Node, DAG))3840 Results.push_back(Expanded);3841 break;3842 }3843 case ISD::FMINIMUMNUM:3844 case ISD::FMAXIMUMNUM: {3845 Results.push_back(TLI.expandFMINIMUMNUM_FMAXIMUMNUM(Node, DAG));3846 break;3847 }3848 case ISD::FSIN:3849 case ISD::FCOS: {3850 EVT VT = Node->getValueType(0);3851 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /3852 // fcos which share the same operand and both are used.3853 if ((TLI.isOperationLegal(ISD::FSINCOS, VT) ||3854 isSinCosLibcallAvailable(Node, TLI)) &&3855 useSinCos(Node)) {3856 SDVTList VTs = DAG.getVTList(VT, VT);3857 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));3858 if (Node->getOpcode() == ISD::FCOS)3859 Tmp1 = Tmp1.getValue(1);3860 Results.push_back(Tmp1);3861 }3862 break;3863 }3864 case ISD::FLDEXP:3865 case ISD::STRICT_FLDEXP: {3866 EVT VT = Node->getValueType(0);3867 RTLIB::Libcall LC = RTLIB::getLDEXP(VT);3868 // Use the LibCall instead, it is very likely faster3869 // FIXME: Use separate LibCall action.3870 if (TLI.getLibcallName(LC))3871 break;3872 3873 if (SDValue Expanded = expandLdexp(Node)) {3874 Results.push_back(Expanded);3875 if (Node->getOpcode() == ISD::STRICT_FLDEXP)3876 Results.push_back(Expanded.getValue(1));3877 }3878 3879 break;3880 }3881 case ISD::FFREXP: {3882 RTLIB::Libcall LC = RTLIB::getFREXP(Node->getValueType(0));3883 // Use the LibCall instead, it is very likely faster3884 // FIXME: Use separate LibCall action.3885 if (TLI.getLibcallName(LC))3886 break;3887 3888 if (SDValue Expanded = expandFrexp(Node)) {3889 Results.push_back(Expanded);3890 Results.push_back(Expanded.getValue(1));3891 }3892 break;3893 }3894 case ISD::FSINCOS: {3895 if (isSinCosLibcallAvailable(Node, TLI))3896 break;3897 EVT VT = Node->getValueType(0);3898 SDValue Op = Node->getOperand(0);3899 SDNodeFlags Flags = Node->getFlags();3900 Tmp1 = DAG.getNode(ISD::FSIN, dl, VT, Op, Flags);3901 Tmp2 = DAG.getNode(ISD::FCOS, dl, VT, Op, Flags);3902 Results.append({Tmp1, Tmp2});3903 break;3904 }3905 case ISD::FMAD:3906 llvm_unreachable("Illegal fmad should never be formed");3907 3908 case ISD::FP16_TO_FP:3909 if (Node->getValueType(0) != MVT::f32) {3910 // We can extend to types bigger than f32 in two steps without changing3911 // the result. Since "f16 -> f32" is much more commonly available, give3912 // CodeGen the option of emitting that before resorting to a libcall.3913 SDValue Res =3914 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));3915 Results.push_back(3916 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));3917 }3918 break;3919 case ISD::STRICT_BF16_TO_FP:3920 case ISD::STRICT_FP16_TO_FP:3921 if (Node->getValueType(0) != MVT::f32) {3922 // We can extend to types bigger than f32 in two steps without changing3923 // the result. Since "f16 -> f32" is much more commonly available, give3924 // CodeGen the option of emitting that before resorting to a libcall.3925 SDValue Res = DAG.getNode(Node->getOpcode(), dl, {MVT::f32, MVT::Other},3926 {Node->getOperand(0), Node->getOperand(1)});3927 Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl,3928 {Node->getValueType(0), MVT::Other},3929 {Res.getValue(1), Res});3930 Results.push_back(Res);3931 Results.push_back(Res.getValue(1));3932 }3933 break;3934 case ISD::FP_TO_FP16:3935 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");3936 if (Node->getFlags().hasApproximateFuncs() && !TLI.useSoftFloat()) {3937 SDValue Op = Node->getOperand(0);3938 MVT SVT = Op.getSimpleValueType();3939 if ((SVT == MVT::f64 || SVT == MVT::f80) &&3940 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {3941 // Under fastmath, we can expand this node into a fround followed by3942 // a float-half conversion.3943 SDValue FloatVal =3944 DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,3945 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));3946 Results.push_back(3947 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));3948 }3949 }3950 break;3951 case ISD::ConstantFP: {3952 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);3953 // Check to see if this FP immediate is already legal.3954 // If this is a legal constant, turn it into a TargetConstantFP node.3955 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0),3956 DAG.shouldOptForSize()))3957 Results.push_back(ExpandConstantFP(CFP, true));3958 break;3959 }3960 case ISD::Constant: {3961 ConstantSDNode *CP = cast<ConstantSDNode>(Node);3962 Results.push_back(ExpandConstant(CP));3963 break;3964 }3965 case ISD::FSUB: {3966 EVT VT = Node->getValueType(0);3967 if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&3968 TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {3969 const SDNodeFlags Flags = Node->getFlags();3970 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));3971 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);3972 Results.push_back(Tmp1);3973 }3974 break;3975 }3976 case ISD::SUB: {3977 EVT VT = Node->getValueType(0);3978 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&3979 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&3980 "Don't know how to expand this subtraction!");3981 Tmp1 = DAG.getNOT(dl, Node->getOperand(1), VT);3982 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));3983 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));3984 break;3985 }3986 case ISD::UREM:3987 case ISD::SREM:3988 if (TLI.expandREM(Node, Tmp1, DAG))3989 Results.push_back(Tmp1);3990 break;3991 case ISD::UDIV:3992 case ISD::SDIV: {3993 bool isSigned = Node->getOpcode() == ISD::SDIV;3994 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;3995 EVT VT = Node->getValueType(0);3996 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {3997 SDVTList VTs = DAG.getVTList(VT, VT);3998 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),3999 Node->getOperand(1));4000 Results.push_back(Tmp1);4001 }4002 break;4003 }4004 case ISD::MULHU:4005 case ISD::MULHS: {4006 unsigned ExpandOpcode =4007 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;4008 EVT VT = Node->getValueType(0);4009 SDVTList VTs = DAG.getVTList(VT, VT);4010 4011 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),4012 Node->getOperand(1));4013 Results.push_back(Tmp1.getValue(1));4014 break;4015 }4016 case ISD::UMUL_LOHI:4017 case ISD::SMUL_LOHI: {4018 SDValue LHS = Node->getOperand(0);4019 SDValue RHS = Node->getOperand(1);4020 MVT VT = LHS.getSimpleValueType();4021 unsigned MULHOpcode =4022 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;4023 4024 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {4025 Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));4026 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));4027 break;4028 }4029 4030 SmallVector<SDValue, 4> Halves;4031 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());4032 assert(TLI.isTypeLegal(HalfType));4033 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves,4034 HalfType, DAG,4035 TargetLowering::MulExpansionKind::Always)) {4036 for (unsigned i = 0; i < 2; ++i) {4037 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);4038 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);4039 SDValue Shift =4040 DAG.getShiftAmountConstant(HalfType.getScalarSizeInBits(), VT, dl);4041 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);4042 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));4043 }4044 break;4045 }4046 break;4047 }4048 case ISD::MUL: {4049 EVT VT = Node->getValueType(0);4050 SDVTList VTs = DAG.getVTList(VT, VT);4051 // See if multiply or divide can be lowered using two-result operations.4052 // We just need the low half of the multiply; try both the signed4053 // and unsigned forms. If the target supports both SMUL_LOHI and4054 // UMUL_LOHI, form a preference by checking which forms of plain4055 // MULH it supports.4056 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);4057 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);4058 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);4059 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);4060 unsigned OpToUse = 0;4061 if (HasSMUL_LOHI && !HasMULHS) {4062 OpToUse = ISD::SMUL_LOHI;4063 } else if (HasUMUL_LOHI && !HasMULHU) {4064 OpToUse = ISD::UMUL_LOHI;4065 } else if (HasSMUL_LOHI) {4066 OpToUse = ISD::SMUL_LOHI;4067 } else if (HasUMUL_LOHI) {4068 OpToUse = ISD::UMUL_LOHI;4069 }4070 if (OpToUse) {4071 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),4072 Node->getOperand(1)));4073 break;4074 }4075 4076 SDValue Lo, Hi;4077 EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());4078 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&4079 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&4080 TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&4081 TLI.isOperationLegalOrCustom(ISD::OR, VT) &&4082 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,4083 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {4084 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);4085 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);4086 SDValue Shift =4087 DAG.getShiftAmountConstant(HalfType.getSizeInBits(), VT, dl);4088 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);4089 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));4090 }4091 break;4092 }4093 case ISD::FSHL:4094 case ISD::FSHR:4095 if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG))4096 Results.push_back(Expanded);4097 break;4098 case ISD::ROTL:4099 case ISD::ROTR:4100 if (SDValue Expanded = TLI.expandROT(Node, true /*AllowVectorOps*/, DAG))4101 Results.push_back(Expanded);4102 break;4103 case ISD::SADDSAT:4104 case ISD::UADDSAT:4105 case ISD::SSUBSAT:4106 case ISD::USUBSAT:4107 Results.push_back(TLI.expandAddSubSat(Node, DAG));4108 break;4109 case ISD::SCMP:4110 case ISD::UCMP:4111 Results.push_back(TLI.expandCMP(Node, DAG));4112 break;4113 case ISD::SSHLSAT:4114 case ISD::USHLSAT:4115 Results.push_back(TLI.expandShlSat(Node, DAG));4116 break;4117 case ISD::SMULFIX:4118 case ISD::SMULFIXSAT:4119 case ISD::UMULFIX:4120 case ISD::UMULFIXSAT:4121 Results.push_back(TLI.expandFixedPointMul(Node, DAG));4122 break;4123 case ISD::SDIVFIX:4124 case ISD::SDIVFIXSAT:4125 case ISD::UDIVFIX:4126 case ISD::UDIVFIXSAT:4127 if (SDValue V = TLI.expandFixedPointDiv(Node->getOpcode(), SDLoc(Node),4128 Node->getOperand(0),4129 Node->getOperand(1),4130 Node->getConstantOperandVal(2),4131 DAG)) {4132 Results.push_back(V);4133 break;4134 }4135 // FIXME: We might want to retry here with a wider type if we fail, if that4136 // type is legal.4137 // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is4138 // <= 128 (which is the case for all of the default Embedded-C types),4139 // we will only get here with types and scales that we could always expand4140 // if we were allowed to generate libcalls to division functions of illegal4141 // type. But we cannot do that.4142 llvm_unreachable("Cannot expand DIVFIX!");4143 case ISD::UADDO_CARRY:4144 case ISD::USUBO_CARRY: {4145 SDValue LHS = Node->getOperand(0);4146 SDValue RHS = Node->getOperand(1);4147 SDValue Carry = Node->getOperand(2);4148 4149 bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY;4150 4151 // Initial add of the 2 operands.4152 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;4153 EVT VT = LHS.getValueType();4154 SDValue Sum = DAG.getNode(Op, dl, VT, LHS, RHS);4155 4156 // Initial check for overflow.4157 EVT CarryType = Node->getValueType(1);4158 EVT SetCCType = getSetCCResultType(Node->getValueType(0));4159 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;4160 SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);4161 4162 // Add of the sum and the carry.4163 SDValue One = DAG.getConstant(1, dl, VT);4164 SDValue CarryExt =4165 DAG.getNode(ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One);4166 SDValue Sum2 = DAG.getNode(Op, dl, VT, Sum, CarryExt);4167 4168 // Second check for overflow. If we are adding, we can only overflow if the4169 // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.4170 // If we are subtracting, we can only overflow if the initial sum is 0 and4171 // the carry is set, resulting in a new sum of all 1s.4172 SDValue Zero = DAG.getConstant(0, dl, VT);4173 SDValue Overflow2 =4174 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ)4175 : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ);4176 Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2,4177 DAG.getZExtOrTrunc(Carry, dl, SetCCType));4178 4179 SDValue ResultCarry =4180 DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2);4181 4182 Results.push_back(Sum2);4183 Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));4184 break;4185 }4186 case ISD::SADDO:4187 case ISD::SSUBO: {4188 SDValue Result, Overflow;4189 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);4190 Results.push_back(Result);4191 Results.push_back(Overflow);4192 break;4193 }4194 case ISD::UADDO:4195 case ISD::USUBO: {4196 SDValue Result, Overflow;4197 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);4198 Results.push_back(Result);4199 Results.push_back(Overflow);4200 break;4201 }4202 case ISD::UMULO:4203 case ISD::SMULO: {4204 SDValue Result, Overflow;4205 if (TLI.expandMULO(Node, Result, Overflow, DAG)) {4206 Results.push_back(Result);4207 Results.push_back(Overflow);4208 }4209 break;4210 }4211 case ISD::BUILD_PAIR: {4212 EVT PairTy = Node->getValueType(0);4213 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));4214 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));4215 Tmp2 = DAG.getNode(4216 ISD::SHL, dl, PairTy, Tmp2,4217 DAG.getShiftAmountConstant(PairTy.getSizeInBits() / 2, PairTy, dl));4218 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));4219 break;4220 }4221 case ISD::SELECT:4222 Tmp1 = Node->getOperand(0);4223 Tmp2 = Node->getOperand(1);4224 Tmp3 = Node->getOperand(2);4225 if (Tmp1.getOpcode() == ISD::SETCC) {4226 Tmp1 = DAG.getSelectCC(4227 dl, Tmp1.getOperand(0), Tmp1.getOperand(1), Tmp2, Tmp3,4228 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get(), Node->getFlags());4229 } else {4230 Tmp1 =4231 DAG.getSelectCC(dl, Tmp1, DAG.getConstant(0, dl, Tmp1.getValueType()),4232 Tmp2, Tmp3, ISD::SETNE, Node->getFlags());4233 }4234 Results.push_back(Tmp1);4235 break;4236 case ISD::BR_JT: {4237 SDValue Chain = Node->getOperand(0);4238 SDValue Table = Node->getOperand(1);4239 SDValue Index = Node->getOperand(2);4240 int JTI = cast<JumpTableSDNode>(Table.getNode())->getIndex();4241 4242 const DataLayout &TD = DAG.getDataLayout();4243 EVT PTy = TLI.getPointerTy(TD);4244 4245 unsigned EntrySize =4246 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);4247 4248 // For power-of-two jumptable entry sizes convert multiplication to a shift.4249 // This transformation needs to be done here since otherwise the MIPS4250 // backend will end up emitting a three instruction multiply sequence4251 // instead of a single shift and MSP430 will call a runtime function.4252 if (llvm::isPowerOf2_32(EntrySize))4253 Index = DAG.getNode(4254 ISD::SHL, dl, Index.getValueType(), Index,4255 DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType()));4256 else4257 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,4258 DAG.getConstant(EntrySize, dl, Index.getValueType()));4259 SDValue Addr = DAG.getMemBasePlusOffset(Table, Index, dl);4260 4261 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);4262 SDValue LD = DAG.getExtLoad(4263 ISD::SEXTLOAD, dl, PTy, Chain, Addr,4264 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);4265 Addr = LD;4266 if (TLI.isJumpTableRelative()) {4267 // For PIC, the sequence is:4268 // BRIND(RelocBase + load(Jumptable + index))4269 // RelocBase can be JumpTable, GOT or some sort of global base.4270 Addr = DAG.getMemBasePlusOffset(TLI.getPICJumpTableRelocBase(Table, DAG),4271 Addr, dl);4272 }4273 4274 Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, JTI, DAG);4275 Results.push_back(Tmp1);4276 break;4277 }4278 case ISD::BRCOND:4279 // Expand brcond's setcc into its constituent parts and create a BR_CC4280 // Node.4281 Tmp1 = Node->getOperand(0);4282 Tmp2 = Node->getOperand(1);4283 if (Tmp2.getOpcode() == ISD::SETCC &&4284 TLI.isOperationLegalOrCustom(ISD::BR_CC,4285 Tmp2.getOperand(0).getValueType())) {4286 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, Tmp2.getOperand(2),4287 Tmp2.getOperand(0), Tmp2.getOperand(1),4288 Node->getOperand(2));4289 } else {4290 // We test only the i1 bit. Skip the AND if UNDEF or another AND.4291 if (Tmp2.isUndef() ||4292 (Tmp2.getOpcode() == ISD::AND && isOneConstant(Tmp2.getOperand(1))))4293 Tmp3 = Tmp2;4294 else4295 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,4296 DAG.getConstant(1, dl, Tmp2.getValueType()));4297 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,4298 DAG.getCondCode(ISD::SETNE), Tmp3,4299 DAG.getConstant(0, dl, Tmp3.getValueType()),4300 Node->getOperand(2));4301 }4302 Results.push_back(Tmp1);4303 break;4304 case ISD::SETCC:4305 case ISD::VP_SETCC:4306 case ISD::STRICT_FSETCC:4307 case ISD::STRICT_FSETCCS: {4308 bool IsVP = Node->getOpcode() == ISD::VP_SETCC;4309 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||4310 Node->getOpcode() == ISD::STRICT_FSETCCS;4311 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;4312 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();4313 unsigned Offset = IsStrict ? 1 : 0;4314 Tmp1 = Node->getOperand(0 + Offset);4315 Tmp2 = Node->getOperand(1 + Offset);4316 Tmp3 = Node->getOperand(2 + Offset);4317 SDValue Mask, EVL;4318 if (IsVP) {4319 Mask = Node->getOperand(3 + Offset);4320 EVL = Node->getOperand(4 + Offset);4321 }4322 bool Legalized = TLI.LegalizeSetCCCondCode(4323 DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl,4324 Chain, IsSignaling);4325 4326 if (Legalized) {4327 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the4328 // condition code, create a new SETCC node.4329 if (Tmp3.getNode()) {4330 if (IsStrict) {4331 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(),4332 {Chain, Tmp1, Tmp2, Tmp3}, Node->getFlags());4333 Chain = Tmp1.getValue(1);4334 } else if (IsVP) {4335 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0),4336 {Tmp1, Tmp2, Tmp3, Mask, EVL}, Node->getFlags());4337 } else {4338 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Tmp1,4339 Tmp2, Tmp3, Node->getFlags());4340 }4341 }4342 4343 // If we expanded the SETCC by inverting the condition code, then wrap4344 // the existing SETCC in a NOT to restore the intended condition.4345 if (NeedInvert) {4346 if (!IsVP)4347 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));4348 else4349 Tmp1 =4350 DAG.getVPLogicalNOT(dl, Tmp1, Mask, EVL, Tmp1->getValueType(0));4351 }4352 4353 Results.push_back(Tmp1);4354 if (IsStrict)4355 Results.push_back(Chain);4356 4357 break;4358 }4359 4360 // FIXME: It seems Legalized is false iff CCCode is Legal. I don't4361 // understand if this code is useful for strict nodes.4362 assert(!IsStrict && "Don't know how to expand for strict nodes.");4363 4364 // Otherwise, SETCC for the given comparison type must be completely4365 // illegal; expand it into a SELECT_CC.4366 // FIXME: This drops the mask/evl for VP_SETCC.4367 EVT VT = Node->getValueType(0);4368 EVT Tmp1VT = Tmp1.getValueType();4369 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,4370 DAG.getBoolConstant(true, dl, VT, Tmp1VT),4371 DAG.getBoolConstant(false, dl, VT, Tmp1VT), Tmp3,4372 Node->getFlags());4373 Results.push_back(Tmp1);4374 break;4375 }4376 case ISD::SELECT_CC: {4377 // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS4378 Tmp1 = Node->getOperand(0); // LHS4379 Tmp2 = Node->getOperand(1); // RHS4380 Tmp3 = Node->getOperand(2); // True4381 Tmp4 = Node->getOperand(3); // False4382 EVT VT = Node->getValueType(0);4383 SDValue Chain;4384 SDValue CC = Node->getOperand(4);4385 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();4386 4387 if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) {4388 // If the condition code is legal, then we need to expand this4389 // node using SETCC and SELECT.4390 EVT CmpVT = Tmp1.getValueType();4391 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&4392 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "4393 "expanded.");4394 EVT CCVT = getSetCCResultType(CmpVT);4395 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags());4396 Results.push_back(4397 DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4, Node->getFlags()));4398 break;4399 }4400 4401 // SELECT_CC is legal, so the condition code must not be.4402 bool Legalized = false;4403 // Try to legalize by inverting the condition. This is for targets that4404 // might support an ordered version of a condition, but not the unordered4405 // version (or vice versa).4406 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, Tmp1.getValueType());4407 if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) {4408 // Use the new condition code and swap true and false4409 Legalized = true;4410 Tmp1 =4411 DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC, Node->getFlags());4412 } else {4413 // If The inverse is not legal, then try to swap the arguments using4414 // the inverse condition code.4415 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);4416 if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) {4417 // The swapped inverse condition is legal, so swap true and false,4418 // lhs and rhs.4419 Legalized = true;4420 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC,4421 Node->getFlags());4422 }4423 }4424 4425 if (!Legalized) {4426 Legalized = TLI.LegalizeSetCCCondCode(4427 DAG, getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC,4428 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);4429 4430 assert(Legalized && "Can't legalize SELECT_CC with legal condition!");4431 4432 // If we expanded the SETCC by inverting the condition code, then swap4433 // the True/False operands to match.4434 if (NeedInvert)4435 std::swap(Tmp3, Tmp4);4436 4437 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the4438 // condition code, create a new SELECT_CC node.4439 if (CC.getNode()) {4440 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,4441 Tmp2, Tmp3, Tmp4, CC, Node->getFlags());4442 } else {4443 Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());4444 CC = DAG.getCondCode(ISD::SETNE);4445 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,4446 Tmp2, Tmp3, Tmp4, CC, Node->getFlags());4447 }4448 }4449 Results.push_back(Tmp1);4450 break;4451 }4452 case ISD::BR_CC: {4453 // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS4454 SDValue Chain;4455 Tmp1 = Node->getOperand(0); // Chain4456 Tmp2 = Node->getOperand(2); // LHS4457 Tmp3 = Node->getOperand(3); // RHS4458 Tmp4 = Node->getOperand(1); // CC4459 4460 bool Legalized = TLI.LegalizeSetCCCondCode(4461 DAG, getSetCCResultType(Tmp2.getValueType()), Tmp2, Tmp3, Tmp4,4462 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);4463 (void)Legalized;4464 assert(Legalized && "Can't legalize BR_CC with legal condition!");4465 4466 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC4467 // node.4468 if (Tmp4.getNode()) {4469 assert(!NeedInvert && "Don't know how to invert BR_CC!");4470 4471 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,4472 Tmp4, Tmp2, Tmp3, Node->getOperand(4));4473 } else {4474 Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());4475 Tmp4 = DAG.getCondCode(NeedInvert ? ISD::SETEQ : ISD::SETNE);4476 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,4477 Tmp2, Tmp3, Node->getOperand(4));4478 }4479 Results.push_back(Tmp1);4480 break;4481 }4482 case ISD::BUILD_VECTOR:4483 Results.push_back(ExpandBUILD_VECTOR(Node));4484 break;4485 case ISD::SPLAT_VECTOR:4486 Results.push_back(ExpandSPLAT_VECTOR(Node));4487 break;4488 case ISD::SRA:4489 case ISD::SRL:4490 case ISD::SHL: {4491 // Scalarize vector SRA/SRL/SHL.4492 EVT VT = Node->getValueType(0);4493 assert(VT.isVector() && "Unable to legalize non-vector shift");4494 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");4495 unsigned NumElem = VT.getVectorNumElements();4496 4497 SmallVector<SDValue, 8> Scalars;4498 for (unsigned Idx = 0; Idx < NumElem; Idx++) {4499 SDValue Ex =4500 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(),4501 Node->getOperand(0), DAG.getVectorIdxConstant(Idx, dl));4502 SDValue Sh =4503 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(),4504 Node->getOperand(1), DAG.getVectorIdxConstant(Idx, dl));4505 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,4506 VT.getScalarType(), Ex, Sh));4507 }4508 4509 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);4510 Results.push_back(Result);4511 break;4512 }4513 case ISD::VECREDUCE_FADD:4514 case ISD::VECREDUCE_FMUL:4515 case ISD::VECREDUCE_ADD:4516 case ISD::VECREDUCE_MUL:4517 case ISD::VECREDUCE_AND:4518 case ISD::VECREDUCE_OR:4519 case ISD::VECREDUCE_XOR:4520 case ISD::VECREDUCE_SMAX:4521 case ISD::VECREDUCE_SMIN:4522 case ISD::VECREDUCE_UMAX:4523 case ISD::VECREDUCE_UMIN:4524 case ISD::VECREDUCE_FMAX:4525 case ISD::VECREDUCE_FMIN:4526 case ISD::VECREDUCE_FMAXIMUM:4527 case ISD::VECREDUCE_FMINIMUM:4528 Results.push_back(TLI.expandVecReduce(Node, DAG));4529 break;4530 case ISD::VP_CTTZ_ELTS:4531 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:4532 Results.push_back(TLI.expandVPCTTZElements(Node, DAG));4533 break;4534 case ISD::CLEAR_CACHE:4535 // The default expansion of llvm.clear_cache is simply a no-op for those4536 // targets where it is not needed.4537 Results.push_back(Node->getOperand(0));4538 break;4539 case ISD::LRINT:4540 case ISD::LLRINT: {4541 SDValue Arg = Node->getOperand(0);4542 EVT ArgVT = Arg.getValueType();4543 EVT ResVT = Node->getValueType(0);4544 SDLoc dl(Node);4545 SDValue RoundNode = DAG.getNode(ISD::FRINT, dl, ArgVT, Arg);4546 Results.push_back(DAG.getNode(ISD::FP_TO_SINT, dl, ResVT, RoundNode));4547 break;4548 }4549 case ISD::ADDRSPACECAST:4550 Results.push_back(DAG.UnrollVectorOp(Node));4551 break;4552 case ISD::GLOBAL_OFFSET_TABLE:4553 case ISD::GlobalAddress:4554 case ISD::GlobalTLSAddress:4555 case ISD::ExternalSymbol:4556 case ISD::ConstantPool:4557 case ISD::JumpTable:4558 case ISD::INTRINSIC_W_CHAIN:4559 case ISD::INTRINSIC_WO_CHAIN:4560 case ISD::INTRINSIC_VOID:4561 // FIXME: Custom lowering for these operations shouldn't return null!4562 // Return true so that we don't call ConvertNodeToLibcall which also won't4563 // do anything.4564 return true;4565 }4566 4567 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {4568 // FIXME: We were asked to expand a strict floating-point operation,4569 // but there is currently no expansion implemented that would preserve4570 // the "strict" properties. For now, we just fall back to the non-strict4571 // version if that is legal on the target. The actual mutation of the4572 // operation will happen in SelectionDAGISel::DoInstructionSelection.4573 switch (Node->getOpcode()) {4574 default:4575 if (TLI.getStrictFPOperationAction(Node->getOpcode(),4576 Node->getValueType(0))4577 == TargetLowering::Legal)4578 return true;4579 break;4580 case ISD::STRICT_FSUB: {4581 if (TLI.getStrictFPOperationAction(4582 ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal)4583 return true;4584 if (TLI.getStrictFPOperationAction(4585 ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal)4586 break;4587 4588 EVT VT = Node->getValueType(0);4589 const SDNodeFlags Flags = Node->getFlags();4590 SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags);4591 SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(),4592 {Node->getOperand(0), Node->getOperand(1), Neg},4593 Flags);4594 4595 Results.push_back(Fadd);4596 Results.push_back(Fadd.getValue(1));4597 break;4598 }4599 case ISD::STRICT_SINT_TO_FP:4600 case ISD::STRICT_UINT_TO_FP:4601 case ISD::STRICT_LRINT:4602 case ISD::STRICT_LLRINT:4603 case ISD::STRICT_LROUND:4604 case ISD::STRICT_LLROUND:4605 // These are registered by the operand type instead of the value4606 // type. Reflect that here.4607 if (TLI.getStrictFPOperationAction(Node->getOpcode(),4608 Node->getOperand(1).getValueType())4609 == TargetLowering::Legal)4610 return true;4611 break;4612 }4613 }4614 4615 // Replace the original node with the legalized result.4616 if (Results.empty()) {4617 LLVM_DEBUG(dbgs() << "Cannot expand node\n");4618 return false;4619 }4620 4621 LLVM_DEBUG(dbgs() << "Successfully expanded node\n");4622 ReplaceNode(Node, Results.data());4623 return true;4624}4625 4626/// Return if we can use the FAST_* variant of a math libcall for the node.4627/// FIXME: This is just guessing, we probably should have unique specific sets4628/// flags required per libcall.4629static bool canUseFastMathLibcall(const SDNode *Node) {4630 // FIXME: Probably should define fast to respect nan/inf and only be4631 // approximate functions.4632 4633 SDNodeFlags Flags = Node->getFlags();4634 return Flags.hasApproximateFuncs() && Flags.hasNoNaNs() &&4635 Flags.hasNoInfs() && Flags.hasNoSignedZeros();4636}4637 4638void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {4639 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");4640 SmallVector<SDValue, 8> Results;4641 SDLoc dl(Node);4642 TargetLowering::MakeLibCallOptions CallOptions;4643 CallOptions.IsPostTypeLegalization = true;4644 // FIXME: Check flags on the node to see if we can use a finite call.4645 unsigned Opc = Node->getOpcode();4646 switch (Opc) {4647 case ISD::ATOMIC_FENCE: {4648 // If the target didn't lower this, lower it to '__sync_synchronize()' call4649 // FIXME: handle "fence singlethread" more efficiently.4650 TargetLowering::ArgListTy Args;4651 4652 TargetLowering::CallLoweringInfo CLI(DAG);4653 CLI.setDebugLoc(dl)4654 .setChain(Node->getOperand(0))4655 .setLibCallee(4656 CallingConv::C, Type::getVoidTy(*DAG.getContext()),4657 DAG.getExternalSymbol("__sync_synchronize",4658 TLI.getPointerTy(DAG.getDataLayout())),4659 std::move(Args));4660 4661 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);4662 4663 Results.push_back(CallResult.second);4664 break;4665 }4666 // By default, atomic intrinsics are marked Legal and lowered. Targets4667 // which don't support them directly, however, may want libcalls, in which4668 // case they mark them Expand, and we get here.4669 case ISD::ATOMIC_SWAP:4670 case ISD::ATOMIC_LOAD_ADD:4671 case ISD::ATOMIC_LOAD_SUB:4672 case ISD::ATOMIC_LOAD_AND:4673 case ISD::ATOMIC_LOAD_CLR:4674 case ISD::ATOMIC_LOAD_OR:4675 case ISD::ATOMIC_LOAD_XOR:4676 case ISD::ATOMIC_LOAD_NAND:4677 case ISD::ATOMIC_LOAD_MIN:4678 case ISD::ATOMIC_LOAD_MAX:4679 case ISD::ATOMIC_LOAD_UMIN:4680 case ISD::ATOMIC_LOAD_UMAX:4681 case ISD::ATOMIC_CMP_SWAP: {4682 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();4683 AtomicOrdering Order = cast<AtomicSDNode>(Node)->getMergedOrdering();4684 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);4685 EVT RetVT = Node->getValueType(0);4686 SmallVector<SDValue, 4> Ops;4687 if (TLI.getLibcallName(LC)) {4688 // If outline atomic available, prepare its arguments and expand.4689 Ops.append(Node->op_begin() + 2, Node->op_end());4690 Ops.push_back(Node->getOperand(1));4691 4692 } else {4693 LC = RTLIB::getSYNC(Opc, VT);4694 assert(LC != RTLIB::UNKNOWN_LIBCALL &&4695 "Unexpected atomic op or value type!");4696 // Arguments for expansion to sync libcall4697 Ops.append(Node->op_begin() + 1, Node->op_end());4698 }4699 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,4700 Ops, CallOptions,4701 SDLoc(Node),4702 Node->getOperand(0));4703 Results.push_back(Tmp.first);4704 Results.push_back(Tmp.second);4705 break;4706 }4707 case ISD::TRAP: {4708 // If this operation is not supported, lower it to 'abort()' call4709 TargetLowering::ArgListTy Args;4710 TargetLowering::CallLoweringInfo CLI(DAG);4711 CLI.setDebugLoc(dl)4712 .setChain(Node->getOperand(0))4713 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),4714 DAG.getExternalSymbol(4715 "abort", TLI.getPointerTy(DAG.getDataLayout())),4716 std::move(Args));4717 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);4718 4719 Results.push_back(CallResult.second);4720 break;4721 }4722 case ISD::CLEAR_CACHE: {4723 SDValue InputChain = Node->getOperand(0);4724 SDValue StartVal = Node->getOperand(1);4725 SDValue EndVal = Node->getOperand(2);4726 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(4727 DAG, RTLIB::CLEAR_CACHE, MVT::isVoid, {StartVal, EndVal}, CallOptions,4728 SDLoc(Node), InputChain);4729 Results.push_back(Tmp.second);4730 break;4731 }4732 case ISD::FMINNUM:4733 case ISD::STRICT_FMINNUM:4734 ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,4735 RTLIB::FMIN_F80, RTLIB::FMIN_F128,4736 RTLIB::FMIN_PPCF128, Results);4737 break;4738 // FIXME: We do not have libcalls for FMAXIMUM and FMINIMUM. So, we cannot use4739 // libcall legalization for these nodes, but there is no default expasion for4740 // these nodes either (see PR63267 for example).4741 case ISD::FMAXNUM:4742 case ISD::STRICT_FMAXNUM:4743 ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,4744 RTLIB::FMAX_F80, RTLIB::FMAX_F128,4745 RTLIB::FMAX_PPCF128, Results);4746 break;4747 case ISD::FMINIMUMNUM:4748 ExpandFPLibCall(Node, RTLIB::FMINIMUM_NUM_F32, RTLIB::FMINIMUM_NUM_F64,4749 RTLIB::FMINIMUM_NUM_F80, RTLIB::FMINIMUM_NUM_F128,4750 RTLIB::FMINIMUM_NUM_PPCF128, Results);4751 break;4752 case ISD::FMAXIMUMNUM:4753 ExpandFPLibCall(Node, RTLIB::FMAXIMUM_NUM_F32, RTLIB::FMAXIMUM_NUM_F64,4754 RTLIB::FMAXIMUM_NUM_F80, RTLIB::FMAXIMUM_NUM_F128,4755 RTLIB::FMAXIMUM_NUM_PPCF128, Results);4756 break;4757 case ISD::FSQRT:4758 case ISD::STRICT_FSQRT: {4759 // FIXME: Probably should define fast to respect nan/inf and only be4760 // approximate functions.4761 ExpandFastFPLibCall(Node, canUseFastMathLibcall(Node),4762 {RTLIB::FAST_SQRT_F32, RTLIB::SQRT_F32},4763 {RTLIB::FAST_SQRT_F64, RTLIB::SQRT_F64},4764 {RTLIB::FAST_SQRT_F80, RTLIB::SQRT_F80},4765 {RTLIB::FAST_SQRT_F128, RTLIB::SQRT_F128},4766 {RTLIB::FAST_SQRT_PPCF128, RTLIB::SQRT_PPCF128},4767 Results);4768 break;4769 }4770 case ISD::FCBRT:4771 ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,4772 RTLIB::CBRT_F80, RTLIB::CBRT_F128,4773 RTLIB::CBRT_PPCF128, Results);4774 break;4775 case ISD::FSIN:4776 case ISD::STRICT_FSIN:4777 ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,4778 RTLIB::SIN_F80, RTLIB::SIN_F128,4779 RTLIB::SIN_PPCF128, Results);4780 break;4781 case ISD::FCOS:4782 case ISD::STRICT_FCOS:4783 ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,4784 RTLIB::COS_F80, RTLIB::COS_F128,4785 RTLIB::COS_PPCF128, Results);4786 break;4787 case ISD::FTAN:4788 case ISD::STRICT_FTAN:4789 ExpandFPLibCall(Node, RTLIB::TAN_F32, RTLIB::TAN_F64, RTLIB::TAN_F80,4790 RTLIB::TAN_F128, RTLIB::TAN_PPCF128, Results);4791 break;4792 case ISD::FASIN:4793 case ISD::STRICT_FASIN:4794 ExpandFPLibCall(Node, RTLIB::ASIN_F32, RTLIB::ASIN_F64, RTLIB::ASIN_F80,4795 RTLIB::ASIN_F128, RTLIB::ASIN_PPCF128, Results);4796 break;4797 case ISD::FACOS:4798 case ISD::STRICT_FACOS:4799 ExpandFPLibCall(Node, RTLIB::ACOS_F32, RTLIB::ACOS_F64, RTLIB::ACOS_F80,4800 RTLIB::ACOS_F128, RTLIB::ACOS_PPCF128, Results);4801 break;4802 case ISD::FATAN:4803 case ISD::STRICT_FATAN:4804 ExpandFPLibCall(Node, RTLIB::ATAN_F32, RTLIB::ATAN_F64, RTLIB::ATAN_F80,4805 RTLIB::ATAN_F128, RTLIB::ATAN_PPCF128, Results);4806 break;4807 case ISD::FATAN2:4808 case ISD::STRICT_FATAN2:4809 ExpandFPLibCall(Node, RTLIB::ATAN2_F32, RTLIB::ATAN2_F64, RTLIB::ATAN2_F80,4810 RTLIB::ATAN2_F128, RTLIB::ATAN2_PPCF128, Results);4811 break;4812 case ISD::FSINH:4813 case ISD::STRICT_FSINH:4814 ExpandFPLibCall(Node, RTLIB::SINH_F32, RTLIB::SINH_F64, RTLIB::SINH_F80,4815 RTLIB::SINH_F128, RTLIB::SINH_PPCF128, Results);4816 break;4817 case ISD::FCOSH:4818 case ISD::STRICT_FCOSH:4819 ExpandFPLibCall(Node, RTLIB::COSH_F32, RTLIB::COSH_F64, RTLIB::COSH_F80,4820 RTLIB::COSH_F128, RTLIB::COSH_PPCF128, Results);4821 break;4822 case ISD::FTANH:4823 case ISD::STRICT_FTANH:4824 ExpandFPLibCall(Node, RTLIB::TANH_F32, RTLIB::TANH_F64, RTLIB::TANH_F80,4825 RTLIB::TANH_F128, RTLIB::TANH_PPCF128, Results);4826 break;4827 case ISD::FSINCOS:4828 case ISD::FSINCOSPI: {4829 EVT VT = Node->getValueType(0);4830 4831 if (Node->getOpcode() == ISD::FSINCOS) {4832 RTLIB::Libcall SincosStret = RTLIB::getSINCOS_STRET(VT);4833 if (SincosStret != RTLIB::UNKNOWN_LIBCALL) {4834 if (SDValue Expanded = ExpandSincosStretLibCall(Node)) {4835 Results.push_back(Expanded);4836 Results.push_back(Expanded.getValue(1));4837 break;4838 }4839 }4840 }4841 4842 RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS4843 ? RTLIB::getSINCOS(VT)4844 : RTLIB::getSINCOSPI(VT);4845 bool Expanded = TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results);4846 if (!Expanded) {4847 DAG.getContext()->emitError(Twine("no libcall available for ") +4848 Node->getOperationName(&DAG));4849 SDValue Poison = DAG.getPOISON(VT);4850 Results.push_back(Poison);4851 Results.push_back(Poison);4852 }4853 4854 break;4855 }4856 case ISD::FLOG:4857 case ISD::STRICT_FLOG:4858 ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,4859 RTLIB::LOG_F128, RTLIB::LOG_PPCF128, Results);4860 break;4861 case ISD::FLOG2:4862 case ISD::STRICT_FLOG2:4863 ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,4864 RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128, Results);4865 break;4866 case ISD::FLOG10:4867 case ISD::STRICT_FLOG10:4868 ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,4869 RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128, Results);4870 break;4871 case ISD::FEXP:4872 case ISD::STRICT_FEXP:4873 ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,4874 RTLIB::EXP_F128, RTLIB::EXP_PPCF128, Results);4875 break;4876 case ISD::FEXP2:4877 case ISD::STRICT_FEXP2:4878 ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,4879 RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128, Results);4880 break;4881 case ISD::FEXP10:4882 ExpandFPLibCall(Node, RTLIB::EXP10_F32, RTLIB::EXP10_F64, RTLIB::EXP10_F80,4883 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128, Results);4884 break;4885 case ISD::FTRUNC:4886 case ISD::STRICT_FTRUNC:4887 ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,4888 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,4889 RTLIB::TRUNC_PPCF128, Results);4890 break;4891 case ISD::FFLOOR:4892 case ISD::STRICT_FFLOOR:4893 ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,4894 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,4895 RTLIB::FLOOR_PPCF128, Results);4896 break;4897 case ISD::FCEIL:4898 case ISD::STRICT_FCEIL:4899 ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,4900 RTLIB::CEIL_F80, RTLIB::CEIL_F128,4901 RTLIB::CEIL_PPCF128, Results);4902 break;4903 case ISD::FRINT:4904 case ISD::STRICT_FRINT:4905 ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,4906 RTLIB::RINT_F80, RTLIB::RINT_F128,4907 RTLIB::RINT_PPCF128, Results);4908 break;4909 case ISD::FNEARBYINT:4910 case ISD::STRICT_FNEARBYINT:4911 ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,4912 RTLIB::NEARBYINT_F64,4913 RTLIB::NEARBYINT_F80,4914 RTLIB::NEARBYINT_F128,4915 RTLIB::NEARBYINT_PPCF128, Results);4916 break;4917 case ISD::FROUND:4918 case ISD::STRICT_FROUND:4919 ExpandFPLibCall(Node, RTLIB::ROUND_F32,4920 RTLIB::ROUND_F64,4921 RTLIB::ROUND_F80,4922 RTLIB::ROUND_F128,4923 RTLIB::ROUND_PPCF128, Results);4924 break;4925 case ISD::FROUNDEVEN:4926 case ISD::STRICT_FROUNDEVEN:4927 ExpandFPLibCall(Node, RTLIB::ROUNDEVEN_F32,4928 RTLIB::ROUNDEVEN_F64,4929 RTLIB::ROUNDEVEN_F80,4930 RTLIB::ROUNDEVEN_F128,4931 RTLIB::ROUNDEVEN_PPCF128, Results);4932 break;4933 case ISD::FLDEXP:4934 case ISD::STRICT_FLDEXP:4935 ExpandFPLibCall(Node, RTLIB::LDEXP_F32, RTLIB::LDEXP_F64, RTLIB::LDEXP_F80,4936 RTLIB::LDEXP_F128, RTLIB::LDEXP_PPCF128, Results);4937 break;4938 case ISD::FMODF:4939 case ISD::FFREXP: {4940 EVT VT = Node->getValueType(0);4941 RTLIB::Libcall LC = Node->getOpcode() == ISD::FMODF ? RTLIB::getMODF(VT)4942 : RTLIB::getFREXP(VT);4943 bool Expanded = TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results,4944 /*CallRetResNo=*/0);4945 if (!Expanded)4946 llvm_unreachable("Expected scalar FFREXP/FMODF to expand to libcall!");4947 break;4948 }4949 case ISD::FPOWI:4950 case ISD::STRICT_FPOWI: {4951 RTLIB::Libcall LC = RTLIB::getPOWI(Node->getSimpleValueType(0));4952 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");4953 if (!TLI.getLibcallName(LC)) {4954 // Some targets don't have a powi libcall; use pow instead.4955 if (Node->isStrictFPOpcode()) {4956 SDValue Exponent =4957 DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(Node),4958 {Node->getValueType(0), Node->getValueType(1)},4959 {Node->getOperand(0), Node->getOperand(2)});4960 SDValue FPOW =4961 DAG.getNode(ISD::STRICT_FPOW, SDLoc(Node),4962 {Node->getValueType(0), Node->getValueType(1)},4963 {Exponent.getValue(1), Node->getOperand(1), Exponent});4964 Results.push_back(FPOW);4965 Results.push_back(FPOW.getValue(1));4966 } else {4967 SDValue Exponent =4968 DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), Node->getValueType(0),4969 Node->getOperand(1));4970 Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node),4971 Node->getValueType(0),4972 Node->getOperand(0), Exponent));4973 }4974 break;4975 }4976 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;4977 bool ExponentHasSizeOfInt =4978 DAG.getLibInfo().getIntSize() ==4979 Node->getOperand(1 + Offset).getValueType().getSizeInBits();4980 if (!ExponentHasSizeOfInt) {4981 // If the exponent does not match with sizeof(int) a libcall to4982 // RTLIB::POWI would use the wrong type for the argument.4983 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");4984 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));4985 break;4986 }4987 ExpandFPLibCall(Node, LC, Results);4988 break;4989 }4990 case ISD::FPOW:4991 case ISD::STRICT_FPOW:4992 ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,4993 RTLIB::POW_F128, RTLIB::POW_PPCF128, Results);4994 break;4995 case ISD::LROUND:4996 case ISD::STRICT_LROUND:4997 ExpandArgFPLibCall(Node, RTLIB::LROUND_F32,4998 RTLIB::LROUND_F64, RTLIB::LROUND_F80,4999 RTLIB::LROUND_F128,5000 RTLIB::LROUND_PPCF128, Results);5001 break;5002 case ISD::LLROUND:5003 case ISD::STRICT_LLROUND:5004 ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32,5005 RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,5006 RTLIB::LLROUND_F128,5007 RTLIB::LLROUND_PPCF128, Results);5008 break;5009 case ISD::LRINT:5010 case ISD::STRICT_LRINT:5011 ExpandArgFPLibCall(Node, RTLIB::LRINT_F32,5012 RTLIB::LRINT_F64, RTLIB::LRINT_F80,5013 RTLIB::LRINT_F128,5014 RTLIB::LRINT_PPCF128, Results);5015 break;5016 case ISD::LLRINT:5017 case ISD::STRICT_LLRINT:5018 ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32,5019 RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,5020 RTLIB::LLRINT_F128,5021 RTLIB::LLRINT_PPCF128, Results);5022 break;5023 case ISD::FDIV:5024 case ISD::STRICT_FDIV: {5025 ExpandFastFPLibCall(Node, canUseFastMathLibcall(Node),5026 {RTLIB::FAST_DIV_F32, RTLIB::DIV_F32},5027 {RTLIB::FAST_DIV_F64, RTLIB::DIV_F64},5028 {RTLIB::FAST_DIV_F80, RTLIB::DIV_F80},5029 {RTLIB::FAST_DIV_F128, RTLIB::DIV_F128},5030 {RTLIB::FAST_DIV_PPCF128, RTLIB::DIV_PPCF128}, Results);5031 break;5032 }5033 case ISD::FREM:5034 case ISD::STRICT_FREM:5035 ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,5036 RTLIB::REM_F80, RTLIB::REM_F128,5037 RTLIB::REM_PPCF128, Results);5038 break;5039 case ISD::FMA:5040 case ISD::STRICT_FMA:5041 ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,5042 RTLIB::FMA_F80, RTLIB::FMA_F128,5043 RTLIB::FMA_PPCF128, Results);5044 break;5045 case ISD::FADD:5046 case ISD::STRICT_FADD: {5047 ExpandFastFPLibCall(Node, canUseFastMathLibcall(Node),5048 {RTLIB::FAST_ADD_F32, RTLIB::ADD_F32},5049 {RTLIB::FAST_ADD_F64, RTLIB::ADD_F64},5050 {RTLIB::FAST_ADD_F80, RTLIB::ADD_F80},5051 {RTLIB::FAST_ADD_F128, RTLIB::ADD_F128},5052 {RTLIB::FAST_ADD_PPCF128, RTLIB::ADD_PPCF128}, Results);5053 break;5054 }5055 case ISD::FMUL:5056 case ISD::STRICT_FMUL: {5057 ExpandFastFPLibCall(Node, canUseFastMathLibcall(Node),5058 {RTLIB::FAST_MUL_F32, RTLIB::MUL_F32},5059 {RTLIB::FAST_MUL_F64, RTLIB::MUL_F64},5060 {RTLIB::FAST_MUL_F80, RTLIB::MUL_F80},5061 {RTLIB::FAST_MUL_F128, RTLIB::MUL_F128},5062 {RTLIB::FAST_MUL_PPCF128, RTLIB::MUL_PPCF128}, Results);5063 break;5064 }5065 case ISD::FP16_TO_FP:5066 if (Node->getValueType(0) == MVT::f32) {5067 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false).first);5068 }5069 break;5070 case ISD::STRICT_BF16_TO_FP:5071 if (Node->getValueType(0) == MVT::f32) {5072 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(5073 DAG, RTLIB::FPEXT_BF16_F32, MVT::f32, Node->getOperand(1),5074 CallOptions, SDLoc(Node), Node->getOperand(0));5075 Results.push_back(Tmp.first);5076 Results.push_back(Tmp.second);5077 }5078 break;5079 case ISD::STRICT_FP16_TO_FP: {5080 if (Node->getValueType(0) == MVT::f32) {5081 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(5082 DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions,5083 SDLoc(Node), Node->getOperand(0));5084 Results.push_back(Tmp.first);5085 Results.push_back(Tmp.second);5086 }5087 break;5088 }5089 case ISD::FP_TO_FP16: {5090 RTLIB::Libcall LC =5091 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);5092 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");5093 Results.push_back(ExpandLibCall(LC, Node, false).first);5094 break;5095 }5096 case ISD::FP_TO_BF16: {5097 RTLIB::Libcall LC =5098 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::bf16);5099 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_bf16");5100 Results.push_back(ExpandLibCall(LC, Node, false).first);5101 break;5102 }5103 case ISD::STRICT_SINT_TO_FP:5104 case ISD::STRICT_UINT_TO_FP:5105 case ISD::SINT_TO_FP:5106 case ISD::UINT_TO_FP: {5107 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP5108 bool IsStrict = Node->isStrictFPOpcode();5109 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP ||5110 Node->getOpcode() == ISD::STRICT_SINT_TO_FP;5111 EVT SVT = Node->getOperand(IsStrict ? 1 : 0).getValueType();5112 EVT RVT = Node->getValueType(0);5113 EVT NVT = EVT();5114 SDLoc dl(Node);5115 5116 // Even if the input is legal, no libcall may exactly match, eg. we don't5117 // have i1 -> fp conversions. So, it needs to be promoted to a larger type,5118 // eg: i13 -> fp. Then, look for an appropriate libcall.5119 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;5120 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;5121 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;5122 ++t) {5123 NVT = (MVT::SimpleValueType)t;5124 // The source needs to big enough to hold the operand.5125 if (NVT.bitsGE(SVT))5126 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT)5127 : RTLIB::getUINTTOFP(NVT, RVT);5128 }5129 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");5130 5131 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();5132 // Sign/zero extend the argument if the libcall takes a larger type.5133 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,5134 NVT, Node->getOperand(IsStrict ? 1 : 0));5135 CallOptions.setIsSigned(Signed);5136 std::pair<SDValue, SDValue> Tmp =5137 TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, dl, Chain);5138 Results.push_back(Tmp.first);5139 if (IsStrict)5140 Results.push_back(Tmp.second);5141 break;5142 }5143 case ISD::FP_TO_SINT:5144 case ISD::FP_TO_UINT:5145 case ISD::STRICT_FP_TO_SINT:5146 case ISD::STRICT_FP_TO_UINT: {5147 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT.5148 bool IsStrict = Node->isStrictFPOpcode();5149 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT ||5150 Node->getOpcode() == ISD::STRICT_FP_TO_SINT;5151 5152 SDValue Op = Node->getOperand(IsStrict ? 1 : 0);5153 EVT SVT = Op.getValueType();5154 EVT RVT = Node->getValueType(0);5155 EVT NVT = EVT();5156 SDLoc dl(Node);5157 5158 // Even if the result is legal, no libcall may exactly match, eg. we don't5159 // have fp -> i1 conversions. So, it needs to be promoted to a larger type,5160 // eg: fp -> i32. Then, look for an appropriate libcall.5161 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;5162 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;5163 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;5164 ++IntVT) {5165 NVT = (MVT::SimpleValueType)IntVT;5166 // The type needs to big enough to hold the result.5167 if (NVT.bitsGE(RVT))5168 LC = Signed ? RTLIB::getFPTOSINT(SVT, NVT)5169 : RTLIB::getFPTOUINT(SVT, NVT);5170 }5171 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");5172 5173 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();5174 std::pair<SDValue, SDValue> Tmp =5175 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);5176 5177 // Truncate the result if the libcall returns a larger type.5178 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first));5179 if (IsStrict)5180 Results.push_back(Tmp.second);5181 break;5182 }5183 5184 case ISD::FP_ROUND:5185 case ISD::STRICT_FP_ROUND: {5186 // X = FP_ROUND(Y, TRUNC)5187 // TRUNC is a flag, which is always an integer that is zero or one.5188 // If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND5189 // is known to not change the value of Y.5190 // We can only expand it into libcall if the TRUNC is 0.5191 bool IsStrict = Node->isStrictFPOpcode();5192 SDValue Op = Node->getOperand(IsStrict ? 1 : 0);5193 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();5194 EVT VT = Node->getValueType(0);5195 assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() &&5196 "Unable to expand as libcall if it is not normal rounding");5197 5198 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), VT);5199 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");5200 5201 std::pair<SDValue, SDValue> Tmp =5202 TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, SDLoc(Node), Chain);5203 Results.push_back(Tmp.first);5204 if (IsStrict)5205 Results.push_back(Tmp.second);5206 break;5207 }5208 case ISD::FP_EXTEND: {5209 Results.push_back(5210 ExpandLibCall(RTLIB::getFPEXT(Node->getOperand(0).getValueType(),5211 Node->getValueType(0)),5212 Node, false).first);5213 break;5214 }5215 case ISD::STRICT_FP_EXTEND:5216 case ISD::STRICT_FP_TO_FP16:5217 case ISD::STRICT_FP_TO_BF16: {5218 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;5219 if (Node->getOpcode() == ISD::STRICT_FP_TO_FP16)5220 LC = RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::f16);5221 else if (Node->getOpcode() == ISD::STRICT_FP_TO_BF16)5222 LC = RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::bf16);5223 else5224 LC = RTLIB::getFPEXT(Node->getOperand(1).getValueType(),5225 Node->getValueType(0));5226 5227 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");5228 5229 std::pair<SDValue, SDValue> Tmp =5230 TLI.makeLibCall(DAG, LC, Node->getValueType(0), Node->getOperand(1),5231 CallOptions, SDLoc(Node), Node->getOperand(0));5232 Results.push_back(Tmp.first);5233 Results.push_back(Tmp.second);5234 break;5235 }5236 case ISD::FSUB:5237 case ISD::STRICT_FSUB: {5238 ExpandFastFPLibCall(Node, canUseFastMathLibcall(Node),5239 {RTLIB::FAST_SUB_F32, RTLIB::SUB_F32},5240 {RTLIB::FAST_SUB_F64, RTLIB::SUB_F64},5241 {RTLIB::FAST_SUB_F80, RTLIB::SUB_F80},5242 {RTLIB::FAST_SUB_F128, RTLIB::SUB_F128},5243 {RTLIB::FAST_SUB_PPCF128, RTLIB::SUB_PPCF128}, Results);5244 break;5245 }5246 case ISD::SREM:5247 Results.push_back(ExpandIntLibCall(Node, true,5248 RTLIB::SREM_I8,5249 RTLIB::SREM_I16, RTLIB::SREM_I32,5250 RTLIB::SREM_I64, RTLIB::SREM_I128));5251 break;5252 case ISD::UREM:5253 Results.push_back(ExpandIntLibCall(Node, false,5254 RTLIB::UREM_I8,5255 RTLIB::UREM_I16, RTLIB::UREM_I32,5256 RTLIB::UREM_I64, RTLIB::UREM_I128));5257 break;5258 case ISD::SDIV:5259 Results.push_back(ExpandIntLibCall(Node, true,5260 RTLIB::SDIV_I8,5261 RTLIB::SDIV_I16, RTLIB::SDIV_I32,5262 RTLIB::SDIV_I64, RTLIB::SDIV_I128));5263 break;5264 case ISD::UDIV:5265 Results.push_back(ExpandIntLibCall(Node, false,5266 RTLIB::UDIV_I8,5267 RTLIB::UDIV_I16, RTLIB::UDIV_I32,5268 RTLIB::UDIV_I64, RTLIB::UDIV_I128));5269 break;5270 case ISD::SDIVREM:5271 case ISD::UDIVREM:5272 // Expand into divrem libcall5273 ExpandDivRemLibCall(Node, Results);5274 break;5275 case ISD::MUL:5276 Results.push_back(ExpandIntLibCall(Node, false,5277 RTLIB::MUL_I8,5278 RTLIB::MUL_I16, RTLIB::MUL_I32,5279 RTLIB::MUL_I64, RTLIB::MUL_I128));5280 break;5281 case ISD::CTLZ_ZERO_UNDEF:5282 Results.push_back(ExpandBitCountingLibCall(5283 Node, RTLIB::CTLZ_I32, RTLIB::CTLZ_I64, RTLIB::CTLZ_I128));5284 break;5285 case ISD::CTPOP:5286 Results.push_back(ExpandBitCountingLibCall(5287 Node, RTLIB::CTPOP_I32, RTLIB::CTPOP_I64, RTLIB::CTPOP_I128));5288 break;5289 case ISD::RESET_FPENV: {5290 // It is legalized to call 'fesetenv(FE_DFL_ENV)'. On most targets5291 // FE_DFL_ENV is defined as '((const fenv_t *) -1)' in glibc.5292 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());5293 SDValue Ptr = DAG.getAllOnesConstant(dl, PtrTy);5294 SDValue Chain = Node->getOperand(0);5295 Results.push_back(5296 DAG.makeStateFunctionCall(RTLIB::FESETENV, Ptr, Chain, dl));5297 break;5298 }5299 case ISD::GET_FPENV_MEM: {5300 SDValue Chain = Node->getOperand(0);5301 SDValue EnvPtr = Node->getOperand(1);5302 Results.push_back(5303 DAG.makeStateFunctionCall(RTLIB::FEGETENV, EnvPtr, Chain, dl));5304 break;5305 }5306 case ISD::SET_FPENV_MEM: {5307 SDValue Chain = Node->getOperand(0);5308 SDValue EnvPtr = Node->getOperand(1);5309 Results.push_back(5310 DAG.makeStateFunctionCall(RTLIB::FESETENV, EnvPtr, Chain, dl));5311 break;5312 }5313 case ISD::GET_FPMODE: {5314 // Call fegetmode, which saves control modes into a stack slot. Then load5315 // the value to return from the stack.5316 EVT ModeVT = Node->getValueType(0);5317 SDValue StackPtr = DAG.CreateStackTemporary(ModeVT);5318 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();5319 SDValue Chain = DAG.makeStateFunctionCall(RTLIB::FEGETMODE, StackPtr,5320 Node->getOperand(0), dl);5321 SDValue LdInst = DAG.getLoad(5322 ModeVT, dl, Chain, StackPtr,5323 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));5324 Results.push_back(LdInst);5325 Results.push_back(LdInst.getValue(1));5326 break;5327 }5328 case ISD::SET_FPMODE: {5329 // Move control modes to stack slot and then call fesetmode with the pointer5330 // to the slot as argument.5331 SDValue Mode = Node->getOperand(1);5332 EVT ModeVT = Mode.getValueType();5333 SDValue StackPtr = DAG.CreateStackTemporary(ModeVT);5334 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();5335 SDValue StInst = DAG.getStore(5336 Node->getOperand(0), dl, Mode, StackPtr,5337 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));5338 Results.push_back(5339 DAG.makeStateFunctionCall(RTLIB::FESETMODE, StackPtr, StInst, dl));5340 break;5341 }5342 case ISD::RESET_FPMODE: {5343 // It is legalized to a call 'fesetmode(FE_DFL_MODE)'. On most targets5344 // FE_DFL_MODE is defined as '((const femode_t *) -1)' in glibc. If not, the5345 // target must provide custom lowering.5346 const DataLayout &DL = DAG.getDataLayout();5347 EVT PtrTy = TLI.getPointerTy(DL);5348 SDValue Mode = DAG.getAllOnesConstant(dl, PtrTy);5349 Results.push_back(DAG.makeStateFunctionCall(RTLIB::FESETMODE, Mode,5350 Node->getOperand(0), dl));5351 break;5352 }5353 }5354 5355 // Replace the original node with the legalized result.5356 if (!Results.empty()) {5357 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");5358 ReplaceNode(Node, Results.data());5359 } else5360 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");5361}5362 5363// Determine the vector type to use in place of an original scalar element when5364// promoting equally sized vectors.5365static MVT getPromotedVectorElementType(const TargetLowering &TLI,5366 MVT EltVT, MVT NewEltVT) {5367 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();5368 MVT MidVT = OldEltsPerNewElt == 15369 ? NewEltVT5370 : MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);5371 assert(TLI.isTypeLegal(MidVT) && "unexpected");5372 return MidVT;5373}5374 5375void SelectionDAGLegalize::PromoteNode(SDNode *Node) {5376 LLVM_DEBUG(dbgs() << "Trying to promote node\n");5377 SmallVector<SDValue, 8> Results;5378 MVT OVT = Node->getSimpleValueType(0);5379 if (Node->getOpcode() == ISD::UINT_TO_FP ||5380 Node->getOpcode() == ISD::SINT_TO_FP ||5381 Node->getOpcode() == ISD::SETCC ||5382 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||5383 Node->getOpcode() == ISD::INSERT_VECTOR_ELT ||5384 Node->getOpcode() == ISD::VECREDUCE_FMAX ||5385 Node->getOpcode() == ISD::VECREDUCE_FMIN ||5386 Node->getOpcode() == ISD::VECREDUCE_FMAXIMUM ||5387 Node->getOpcode() == ISD::VECREDUCE_FMINIMUM) {5388 OVT = Node->getOperand(0).getSimpleValueType();5389 }5390 if (Node->getOpcode() == ISD::ATOMIC_STORE ||5391 Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||5392 Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||5393 Node->getOpcode() == ISD::STRICT_FSETCC ||5394 Node->getOpcode() == ISD::STRICT_FSETCCS ||5395 Node->getOpcode() == ISD::VP_REDUCE_FADD ||5396 Node->getOpcode() == ISD::VP_REDUCE_FMUL ||5397 Node->getOpcode() == ISD::VP_REDUCE_FMAX ||5398 Node->getOpcode() == ISD::VP_REDUCE_FMIN ||5399 Node->getOpcode() == ISD::VP_REDUCE_FMAXIMUM ||5400 Node->getOpcode() == ISD::VP_REDUCE_FMINIMUM ||5401 Node->getOpcode() == ISD::VP_REDUCE_SEQ_FADD)5402 OVT = Node->getOperand(1).getSimpleValueType();5403 if (Node->getOpcode() == ISD::BR_CC ||5404 Node->getOpcode() == ISD::SELECT_CC)5405 OVT = Node->getOperand(2).getSimpleValueType();5406 // Preserve fast math flags5407 SDNodeFlags FastMathFlags = Node->getFlags() & SDNodeFlags::FastMathFlags;5408 SelectionDAG::FlagInserter FlagsInserter(DAG, FastMathFlags);5409 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);5410 SDLoc dl(Node);5411 SDValue Tmp1, Tmp2, Tmp3, Tmp4;5412 switch (Node->getOpcode()) {5413 case ISD::CTTZ:5414 case ISD::CTTZ_ZERO_UNDEF:5415 case ISD::CTLZ:5416 case ISD::CTPOP: {5417 // Zero extend the argument unless its cttz, then use any_extend.5418 if (Node->getOpcode() == ISD::CTTZ ||5419 Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF)5420 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0));5421 else5422 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));5423 5424 unsigned NewOpc = Node->getOpcode();5425 if (NewOpc == ISD::CTTZ) {5426 // The count is the same in the promoted type except if the original5427 // value was zero. This can be handled by setting the bit just off5428 // the top of the original type.5429 auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),5430 OVT.getSizeInBits());5431 Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,5432 DAG.getConstant(TopBit, dl, NVT));5433 NewOpc = ISD::CTTZ_ZERO_UNDEF;5434 }5435 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is5436 // already the correct result.5437 Tmp1 = DAG.getNode(NewOpc, dl, NVT, Tmp1);5438 if (NewOpc == ISD::CTLZ) {5439 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))5440 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,5441 DAG.getConstant(NVT.getSizeInBits() -5442 OVT.getSizeInBits(), dl, NVT));5443 }5444 Results.push_back(5445 DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1, SDNodeFlags::NoWrap));5446 break;5447 }5448 case ISD::CTLZ_ZERO_UNDEF: {5449 // We know that the argument is unlikely to be zero, hence we can take a5450 // different approach as compared to ISD::CTLZ5451 5452 // Any Extend the argument5453 auto AnyExtendedNode =5454 DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0));5455 5456 // Tmp1 = Tmp1 << (sizeinbits(NVT) - sizeinbits(Old VT))5457 auto ShiftConstant = DAG.getShiftAmountConstant(5458 NVT.getSizeInBits() - OVT.getSizeInBits(), NVT, dl);5459 auto LeftShiftResult =5460 DAG.getNode(ISD::SHL, dl, NVT, AnyExtendedNode, ShiftConstant);5461 5462 // Perform the larger operation5463 auto CTLZResult = DAG.getNode(Node->getOpcode(), dl, NVT, LeftShiftResult);5464 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, CTLZResult));5465 break;5466 }5467 case ISD::BITREVERSE:5468 case ISD::BSWAP: {5469 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();5470 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));5471 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);5472 Tmp1 = DAG.getNode(ISD::SRL, dl, NVT, Tmp1,5473 DAG.getShiftAmountConstant(DiffBits, NVT, dl));5474 5475 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));5476 break;5477 }5478 case ISD::FP_TO_UINT:5479 case ISD::STRICT_FP_TO_UINT:5480 case ISD::FP_TO_SINT:5481 case ISD::STRICT_FP_TO_SINT:5482 PromoteLegalFP_TO_INT(Node, dl, Results);5483 break;5484 case ISD::FP_TO_UINT_SAT:5485 case ISD::FP_TO_SINT_SAT:5486 Results.push_back(PromoteLegalFP_TO_INT_SAT(Node, dl));5487 break;5488 case ISD::UINT_TO_FP:5489 case ISD::STRICT_UINT_TO_FP:5490 case ISD::SINT_TO_FP:5491 case ISD::STRICT_SINT_TO_FP:5492 PromoteLegalINT_TO_FP(Node, dl, Results);5493 break;5494 case ISD::VAARG: {5495 SDValue Chain = Node->getOperand(0); // Get the chain.5496 SDValue Ptr = Node->getOperand(1); // Get the pointer.5497 5498 unsigned TruncOp;5499 if (OVT.isVector()) {5500 TruncOp = ISD::BITCAST;5501 } else {5502 assert(OVT.isInteger()5503 && "VAARG promotion is supported only for vectors or integer types");5504 TruncOp = ISD::TRUNCATE;5505 }5506 5507 // Perform the larger operation, then convert back5508 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),5509 Node->getConstantOperandVal(3));5510 Chain = Tmp1.getValue(1);5511 5512 Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);5513 5514 // Modified the chain result - switch anything that used the old chain to5515 // use the new one.5516 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);5517 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);5518 if (UpdatedNodes) {5519 UpdatedNodes->insert(Tmp2.getNode());5520 UpdatedNodes->insert(Chain.getNode());5521 }5522 ReplacedNode(Node);5523 break;5524 }5525 case ISD::MUL:5526 case ISD::SDIV:5527 case ISD::SREM:5528 case ISD::UDIV:5529 case ISD::UREM:5530 case ISD::SMIN:5531 case ISD::SMAX:5532 case ISD::UMIN:5533 case ISD::UMAX:5534 case ISD::AND:5535 case ISD::OR:5536 case ISD::XOR: {5537 unsigned ExtOp, TruncOp;5538 if (OVT.isVector()) {5539 ExtOp = ISD::BITCAST;5540 TruncOp = ISD::BITCAST;5541 } else {5542 assert(OVT.isInteger() && "Cannot promote logic operation");5543 5544 switch (Node->getOpcode()) {5545 default:5546 ExtOp = ISD::ANY_EXTEND;5547 break;5548 case ISD::SDIV:5549 case ISD::SREM:5550 case ISD::SMIN:5551 case ISD::SMAX:5552 ExtOp = ISD::SIGN_EXTEND;5553 break;5554 case ISD::UDIV:5555 case ISD::UREM:5556 ExtOp = ISD::ZERO_EXTEND;5557 break;5558 case ISD::UMIN:5559 case ISD::UMAX:5560 if (TLI.isSExtCheaperThanZExt(OVT, NVT))5561 ExtOp = ISD::SIGN_EXTEND;5562 else5563 ExtOp = ISD::ZERO_EXTEND;5564 break;5565 }5566 TruncOp = ISD::TRUNCATE;5567 }5568 // Promote each of the values to the new type.5569 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));5570 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));5571 // Perform the larger operation, then convert back5572 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);5573 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));5574 break;5575 }5576 case ISD::UMUL_LOHI:5577 case ISD::SMUL_LOHI: {5578 // Promote to a multiply in a wider integer type.5579 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND5580 : ISD::SIGN_EXTEND;5581 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));5582 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));5583 Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);5584 5585 unsigned OriginalSize = OVT.getScalarSizeInBits();5586 Tmp2 = DAG.getNode(ISD::SRL, dl, NVT, Tmp1,5587 DAG.getShiftAmountConstant(OriginalSize, NVT, dl));5588 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));5589 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));5590 break;5591 }5592 case ISD::SELECT: {5593 unsigned ExtOp, TruncOp;5594 if (Node->getValueType(0).isVector() ||5595 Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {5596 ExtOp = ISD::BITCAST;5597 TruncOp = ISD::BITCAST;5598 } else if (Node->getValueType(0).isInteger()) {5599 ExtOp = ISD::ANY_EXTEND;5600 TruncOp = ISD::TRUNCATE;5601 } else {5602 ExtOp = ISD::FP_EXTEND;5603 TruncOp = ISD::FP_ROUND;5604 }5605 Tmp1 = Node->getOperand(0);5606 // Promote each of the values to the new type.5607 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));5608 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));5609 // Perform the larger operation, then round down.5610 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);5611 if (TruncOp != ISD::FP_ROUND)5612 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);5613 else5614 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,5615 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));5616 Results.push_back(Tmp1);5617 break;5618 }5619 case ISD::VECTOR_SHUFFLE: {5620 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();5621 5622 // Cast the two input vectors.5623 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));5624 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));5625 5626 // Convert the shuffle mask to the right # elements.5627 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);5628 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);5629 Results.push_back(Tmp1);5630 break;5631 }5632 case ISD::VECTOR_SPLICE: {5633 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0));5634 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(1));5635 Tmp3 = DAG.getNode(ISD::VECTOR_SPLICE, dl, NVT, Tmp1, Tmp2,5636 Node->getOperand(2));5637 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp3));5638 break;5639 }5640 case ISD::SELECT_CC: {5641 SDValue Cond = Node->getOperand(4);5642 ISD::CondCode CCCode = cast<CondCodeSDNode>(Cond)->get();5643 // Type of the comparison operands.5644 MVT CVT = Node->getSimpleValueType(0);5645 assert(CVT == OVT && "not handled");5646 5647 unsigned ExtOp = ISD::FP_EXTEND;5648 if (NVT.isInteger()) {5649 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;5650 }5651 5652 // Promote the comparison operands, if needed.5653 if (TLI.isCondCodeLegal(CCCode, CVT)) {5654 Tmp1 = Node->getOperand(0);5655 Tmp2 = Node->getOperand(1);5656 } else {5657 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));5658 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));5659 }5660 // Cast the true/false operands.5661 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));5662 Tmp4 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));5663 5664 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, NVT, {Tmp1, Tmp2, Tmp3, Tmp4, Cond},5665 Node->getFlags());5666 5667 // Cast the result back to the original type.5668 if (ExtOp != ISD::FP_EXTEND)5669 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1);5670 else5671 Tmp1 = DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp1,5672 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));5673 5674 Results.push_back(Tmp1);5675 break;5676 }5677 case ISD::SETCC:5678 case ISD::STRICT_FSETCC:5679 case ISD::STRICT_FSETCCS: {5680 unsigned ExtOp = ISD::FP_EXTEND;5681 if (NVT.isInteger()) {5682 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();5683 if (isSignedIntSetCC(CCCode) ||5684 TLI.isSExtCheaperThanZExt(Node->getOperand(0).getValueType(), NVT))5685 ExtOp = ISD::SIGN_EXTEND;5686 else5687 ExtOp = ISD::ZERO_EXTEND;5688 }5689 if (Node->isStrictFPOpcode()) {5690 SDValue InChain = Node->getOperand(0);5691 std::tie(Tmp1, std::ignore) =5692 DAG.getStrictFPExtendOrRound(Node->getOperand(1), InChain, dl, NVT);5693 std::tie(Tmp2, std::ignore) =5694 DAG.getStrictFPExtendOrRound(Node->getOperand(2), InChain, dl, NVT);5695 SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(1), Tmp2.getValue(1)};5696 SDValue OutChain = DAG.getTokenFactor(dl, TmpChains);5697 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);5698 Results.push_back(DAG.getNode(Node->getOpcode(), dl, VTs,5699 {OutChain, Tmp1, Tmp2, Node->getOperand(3)},5700 Node->getFlags()));5701 Results.push_back(Results.back().getValue(1));5702 break;5703 }5704 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));5705 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));5706 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1,5707 Tmp2, Node->getOperand(2), Node->getFlags()));5708 break;5709 }5710 case ISD::BR_CC: {5711 unsigned ExtOp = ISD::FP_EXTEND;5712 if (NVT.isInteger()) {5713 ISD::CondCode CCCode =5714 cast<CondCodeSDNode>(Node->getOperand(1))->get();5715 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;5716 }5717 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));5718 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));5719 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),5720 Node->getOperand(0), Node->getOperand(1),5721 Tmp1, Tmp2, Node->getOperand(4)));5722 break;5723 }5724 case ISD::FADD:5725 case ISD::FSUB:5726 case ISD::FMUL:5727 case ISD::FDIV:5728 case ISD::FREM:5729 case ISD::FMINNUM:5730 case ISD::FMAXNUM:5731 case ISD::FMINIMUM:5732 case ISD::FMAXIMUM:5733 case ISD::FMINIMUMNUM:5734 case ISD::FMAXIMUMNUM:5735 case ISD::FPOW:5736 case ISD::FATAN2:5737 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));5738 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));5739 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);5740 Results.push_back(5741 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp3,5742 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));5743 break;5744 5745 case ISD::STRICT_FMINIMUM:5746 case ISD::STRICT_FMAXIMUM: {5747 SDValue InChain = Node->getOperand(0);5748 SDVTList VTs = DAG.getVTList(NVT, MVT::Other);5749 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, VTs, InChain,5750 Node->getOperand(1));5751 Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, VTs, InChain,5752 Node->getOperand(2));5753 SmallVector<SDValue, 4> Ops = {InChain, Tmp1, Tmp2};5754 Tmp3 = DAG.getNode(Node->getOpcode(), dl, VTs, Ops, Node->getFlags());5755 Tmp4 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, DAG.getVTList(OVT, MVT::Other),5756 InChain, Tmp3,5757 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));5758 Results.push_back(Tmp4);5759 Results.push_back(Tmp4.getValue(1));5760 break;5761 }5762 5763 case ISD::STRICT_FADD:5764 case ISD::STRICT_FSUB:5765 case ISD::STRICT_FMUL:5766 case ISD::STRICT_FDIV:5767 case ISD::STRICT_FMINNUM:5768 case ISD::STRICT_FMAXNUM:5769 case ISD::STRICT_FREM:5770 case ISD::STRICT_FPOW:5771 case ISD::STRICT_FATAN2:5772 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},5773 {Node->getOperand(0), Node->getOperand(1)});5774 Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},5775 {Node->getOperand(0), Node->getOperand(2)});5776 Tmp3 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1),5777 Tmp2.getValue(1));5778 Tmp1 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},5779 {Tmp3, Tmp1, Tmp2});5780 Tmp1 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},5781 {Tmp1.getValue(1), Tmp1,5782 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)});5783 Results.push_back(Tmp1);5784 Results.push_back(Tmp1.getValue(1));5785 break;5786 case ISD::FMA:5787 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));5788 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));5789 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));5790 Results.push_back(5791 DAG.getNode(ISD::FP_ROUND, dl, OVT,5792 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),5793 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));5794 break;5795 case ISD::STRICT_FMA:5796 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},5797 {Node->getOperand(0), Node->getOperand(1)});5798 Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},5799 {Node->getOperand(0), Node->getOperand(2)});5800 Tmp3 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},5801 {Node->getOperand(0), Node->getOperand(3)});5802 Tmp4 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1),5803 Tmp2.getValue(1), Tmp3.getValue(1));5804 Tmp4 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},5805 {Tmp4, Tmp1, Tmp2, Tmp3});5806 Tmp4 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},5807 {Tmp4.getValue(1), Tmp4,5808 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)});5809 Results.push_back(Tmp4);5810 Results.push_back(Tmp4.getValue(1));5811 break;5812 case ISD::FCOPYSIGN:5813 case ISD::FLDEXP:5814 case ISD::FPOWI: {5815 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));5816 Tmp2 = Node->getOperand(1);5817 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);5818 5819 // fcopysign doesn't change anything but the sign bit, so5820 // (fp_round (fcopysign (fpext a), b))5821 // is as precise as5822 // (fp_round (fpext a))5823 // which is a no-op. Mark it as a TRUNCating FP_ROUND.5824 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);5825 Results.push_back(5826 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp3,5827 DAG.getIntPtrConstant(isTrunc, dl, /*isTarget=*/true)));5828 break;5829 }5830 case ISD::STRICT_FLDEXP: {5831 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},5832 {Node->getOperand(0), Node->getOperand(1)});5833 Tmp2 = Node->getOperand(2);5834 Tmp3 = DAG.getNode(ISD::STRICT_FLDEXP, dl, {NVT, MVT::Other},5835 {Tmp1.getValue(1), Tmp1, Tmp2});5836 Tmp4 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},5837 {Tmp3.getValue(1), Tmp3,5838 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)});5839 Results.push_back(Tmp4);5840 Results.push_back(Tmp4.getValue(1));5841 break;5842 }5843 case ISD::STRICT_FPOWI:5844 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},5845 {Node->getOperand(0), Node->getOperand(1)});5846 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},5847 {Tmp1.getValue(1), Tmp1, Node->getOperand(2)});5848 Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},5849 {Tmp2.getValue(1), Tmp2,5850 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)});5851 Results.push_back(Tmp3);5852 Results.push_back(Tmp3.getValue(1));5853 break;5854 case ISD::FFREXP: {5855 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));5856 Tmp2 = DAG.getNode(ISD::FFREXP, dl, {NVT, Node->getValueType(1)}, Tmp1);5857 5858 Results.push_back(5859 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2,5860 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));5861 5862 Results.push_back(Tmp2.getValue(1));5863 break;5864 }5865 case ISD::FMODF:5866 case ISD::FSINCOS:5867 case ISD::FSINCOSPI: {5868 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));5869 Tmp2 = DAG.getNode(Node->getOpcode(), dl, DAG.getVTList(NVT, NVT), Tmp1);5870 Tmp3 = DAG.getIntPtrConstant(0, dl, /*isTarget=*/true);5871 for (unsigned ResNum = 0; ResNum < Node->getNumValues(); ResNum++)5872 Results.push_back(5873 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2.getValue(ResNum), Tmp3));5874 break;5875 }5876 case ISD::FFLOOR:5877 case ISD::FCEIL:5878 case ISD::FRINT:5879 case ISD::FNEARBYINT:5880 case ISD::FROUND:5881 case ISD::FROUNDEVEN:5882 case ISD::FTRUNC:5883 case ISD::FNEG:5884 case ISD::FSQRT:5885 case ISD::FSIN:5886 case ISD::FCOS:5887 case ISD::FTAN:5888 case ISD::FASIN:5889 case ISD::FACOS:5890 case ISD::FATAN:5891 case ISD::FSINH:5892 case ISD::FCOSH:5893 case ISD::FTANH:5894 case ISD::FLOG:5895 case ISD::FLOG2:5896 case ISD::FLOG10:5897 case ISD::FABS:5898 case ISD::FEXP:5899 case ISD::FEXP2:5900 case ISD::FEXP10:5901 case ISD::FCANONICALIZE:5902 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));5903 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);5904 Results.push_back(5905 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2,5906 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));5907 break;5908 case ISD::STRICT_FFLOOR:5909 case ISD::STRICT_FCEIL:5910 case ISD::STRICT_FRINT:5911 case ISD::STRICT_FNEARBYINT:5912 case ISD::STRICT_FROUND:5913 case ISD::STRICT_FROUNDEVEN:5914 case ISD::STRICT_FTRUNC:5915 case ISD::STRICT_FSQRT:5916 case ISD::STRICT_FSIN:5917 case ISD::STRICT_FCOS:5918 case ISD::STRICT_FTAN:5919 case ISD::STRICT_FASIN:5920 case ISD::STRICT_FACOS:5921 case ISD::STRICT_FATAN:5922 case ISD::STRICT_FSINH:5923 case ISD::STRICT_FCOSH:5924 case ISD::STRICT_FTANH:5925 case ISD::STRICT_FLOG:5926 case ISD::STRICT_FLOG2:5927 case ISD::STRICT_FLOG10:5928 case ISD::STRICT_FEXP:5929 case ISD::STRICT_FEXP2:5930 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},5931 {Node->getOperand(0), Node->getOperand(1)});5932 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},5933 {Tmp1.getValue(1), Tmp1});5934 Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},5935 {Tmp2.getValue(1), Tmp2,5936 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)});5937 Results.push_back(Tmp3);5938 Results.push_back(Tmp3.getValue(1));5939 break;5940 case ISD::BUILD_VECTOR: {5941 MVT EltVT = OVT.getVectorElementType();5942 MVT NewEltVT = NVT.getVectorElementType();5943 5944 // Handle bitcasts to a different vector type with the same total bit size5945 //5946 // e.g. v2i64 = build_vector i64:x, i64:y => v4i325947 // =>5948 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))5949 5950 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&5951 "Invalid promote type for build_vector");5952 assert(NewEltVT.bitsLE(EltVT) && "not handled");5953 5954 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);5955 5956 SmallVector<SDValue, 8> NewOps;5957 for (const SDValue &Op : Node->op_values())5958 NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));5959 5960 SDLoc SL(Node);5961 SDValue Concat =5962 DAG.getNode(MidVT == NewEltVT ? ISD::BUILD_VECTOR : ISD::CONCAT_VECTORS,5963 SL, NVT, NewOps);5964 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);5965 Results.push_back(CvtVec);5966 break;5967 }5968 case ISD::EXTRACT_VECTOR_ELT: {5969 MVT EltVT = OVT.getVectorElementType();5970 MVT NewEltVT = NVT.getVectorElementType();5971 5972 // Handle bitcasts to a different vector type with the same total bit size.5973 //5974 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i325975 // =>5976 // v4i32:castx = bitcast x:v2i645977 //5978 // i64 = bitcast5979 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),5980 // (i32 (extract_vector_elt castx, (2 * y + 1)))5981 //5982 5983 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&5984 "Invalid promote type for extract_vector_elt");5985 assert(NewEltVT.bitsLT(EltVT) && "not handled");5986 5987 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);5988 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();5989 5990 SDValue Idx = Node->getOperand(1);5991 EVT IdxVT = Idx.getValueType();5992 SDLoc SL(Node);5993 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);5994 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);5995 5996 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));5997 5998 SmallVector<SDValue, 8> NewOps;5999 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {6000 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);6001 SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);6002 6003 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,6004 CastVec, TmpIdx);6005 NewOps.push_back(Elt);6006 }6007 6008 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);6009 Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));6010 break;6011 }6012 case ISD::INSERT_VECTOR_ELT: {6013 MVT EltVT = OVT.getVectorElementType();6014 MVT NewEltVT = NVT.getVectorElementType();6015 6016 // Handle bitcasts to a different vector type with the same total bit size6017 //6018 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i326019 // =>6020 // v4i32:castx = bitcast x:v2i646021 // v2i32:casty = bitcast y:i646022 //6023 // v2i64 = bitcast6024 // (v4i32 insert_vector_elt6025 // (v4i32 insert_vector_elt v4i32:castx,6026 // (extract_vector_elt casty, 0), 2 * z),6027 // (extract_vector_elt casty, 1), (2 * z + 1))6028 6029 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&6030 "Invalid promote type for insert_vector_elt");6031 assert(NewEltVT.bitsLT(EltVT) && "not handled");6032 6033 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);6034 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();6035 6036 SDValue Val = Node->getOperand(1);6037 SDValue Idx = Node->getOperand(2);6038 EVT IdxVT = Idx.getValueType();6039 SDLoc SL(Node);6040 6041 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);6042 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);6043 6044 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));6045 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);6046 6047 SDValue NewVec = CastVec;6048 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {6049 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);6050 SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);6051 6052 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,6053 CastVal, IdxOffset);6054 6055 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,6056 NewVec, Elt, InEltIdx);6057 }6058 6059 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));6060 break;6061 }6062 case ISD::SCALAR_TO_VECTOR: {6063 MVT EltVT = OVT.getVectorElementType();6064 MVT NewEltVT = NVT.getVectorElementType();6065 6066 // Handle bitcasts to different vector type with the same total bit size.6067 //6068 // e.g. v2i64 = scalar_to_vector x:i646069 // =>6070 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)6071 //6072 6073 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);6074 SDValue Val = Node->getOperand(0);6075 SDLoc SL(Node);6076 6077 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);6078 SDValue Undef = DAG.getUNDEF(MidVT);6079 6080 SmallVector<SDValue, 8> NewElts;6081 NewElts.push_back(CastVal);6082 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)6083 NewElts.push_back(Undef);6084 6085 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);6086 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);6087 Results.push_back(CvtVec);6088 break;6089 }6090 case ISD::ATOMIC_SWAP:6091 case ISD::ATOMIC_STORE: {6092 AtomicSDNode *AM = cast<AtomicSDNode>(Node);6093 SDLoc SL(Node);6094 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal());6095 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&6096 "unexpected promotion type");6097 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&6098 "unexpected atomic_swap with illegal type");6099 6100 SDValue Op0 = AM->getBasePtr();6101 SDValue Op1 = CastVal;6102 6103 // ATOMIC_STORE uses a swapped operand order from every other AtomicSDNode,6104 // but really it should merge with ISD::STORE.6105 if (AM->getOpcode() == ISD::ATOMIC_STORE)6106 std::swap(Op0, Op1);6107 6108 SDValue NewAtomic = DAG.getAtomic(AM->getOpcode(), SL, NVT, AM->getChain(),6109 Op0, Op1, AM->getMemOperand());6110 6111 if (AM->getOpcode() != ISD::ATOMIC_STORE) {6112 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic));6113 Results.push_back(NewAtomic.getValue(1));6114 } else6115 Results.push_back(NewAtomic);6116 break;6117 }6118 case ISD::ATOMIC_LOAD: {6119 AtomicSDNode *AM = cast<AtomicSDNode>(Node);6120 SDLoc SL(Node);6121 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&6122 "unexpected promotion type");6123 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&6124 "unexpected atomic_load with illegal type");6125 6126 SDValue NewAtomic =6127 DAG.getAtomic(ISD::ATOMIC_LOAD, SL, NVT, DAG.getVTList(NVT, MVT::Other),6128 {AM->getChain(), AM->getBasePtr()}, AM->getMemOperand());6129 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic));6130 Results.push_back(NewAtomic.getValue(1));6131 break;6132 }6133 case ISD::SPLAT_VECTOR: {6134 SDValue Scalar = Node->getOperand(0);6135 MVT ScalarType = Scalar.getSimpleValueType();6136 MVT NewScalarType = NVT.getVectorElementType();6137 if (ScalarType.isInteger()) {6138 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NewScalarType, Scalar);6139 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);6140 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));6141 break;6142 }6143 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NewScalarType, Scalar);6144 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);6145 Results.push_back(6146 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2,6147 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));6148 break;6149 }6150 case ISD::VECREDUCE_FMAX:6151 case ISD::VECREDUCE_FMIN:6152 case ISD::VECREDUCE_FMAXIMUM:6153 case ISD::VECREDUCE_FMINIMUM:6154 case ISD::VP_REDUCE_FMAX:6155 case ISD::VP_REDUCE_FMIN:6156 case ISD::VP_REDUCE_FMAXIMUM:6157 case ISD::VP_REDUCE_FMINIMUM:6158 Results.push_back(PromoteReduction(Node));6159 break;6160 }6161 6162 // Replace the original node with the legalized result.6163 if (!Results.empty()) {6164 LLVM_DEBUG(dbgs() << "Successfully promoted node\n");6165 ReplaceNode(Node, Results.data());6166 } else6167 LLVM_DEBUG(dbgs() << "Could not promote node\n");6168}6169 6170/// This is the entry point for the file.6171void SelectionDAG::Legalize() {6172 AssignTopologicalOrder();6173 6174 SmallPtrSet<SDNode *, 16> LegalizedNodes;6175 // Use a delete listener to remove nodes which were deleted during6176 // legalization from LegalizeNodes. This is needed to handle the situation6177 // where a new node is allocated by the object pool to the same address of a6178 // previously deleted node.6179 DAGNodeDeletedListener DeleteListener(6180 *this,6181 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); });6182 6183 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);6184 6185 // Visit all the nodes. We start in topological order, so that we see6186 // nodes with their original operands intact. Legalization can produce6187 // new nodes which may themselves need to be legalized. Iterate until all6188 // nodes have been legalized.6189 while (true) {6190 bool AnyLegalized = false;6191 for (auto NI = allnodes_end(); NI != allnodes_begin();) {6192 --NI;6193 6194 SDNode *N = &*NI;6195 if (N->use_empty() && N != getRoot().getNode()) {6196 ++NI;6197 DeleteNode(N);6198 continue;6199 }6200 6201 if (LegalizedNodes.insert(N).second) {6202 AnyLegalized = true;6203 Legalizer.LegalizeOp(N);6204 6205 if (N->use_empty() && N != getRoot().getNode()) {6206 ++NI;6207 DeleteNode(N);6208 }6209 }6210 }6211 if (!AnyLegalized)6212 break;6213 6214 }6215 6216 // Remove dead nodes now.6217 RemoveDeadNodes();6218}6219 6220bool SelectionDAG::LegalizeOp(SDNode *N,6221 SmallSetVector<SDNode *, 16> &UpdatedNodes) {6222 SmallPtrSet<SDNode *, 16> LegalizedNodes;6223 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);6224 6225 // Directly insert the node in question, and legalize it. This will recurse6226 // as needed through operands.6227 LegalizedNodes.insert(N);6228 Legalizer.LegalizeOp(N);6229 6230 return LegalizedNodes.count(N);6231}6232