7216 lines · cpp
1//===-- NVPTXISelLowering.cpp - NVPTX DAG Lowering Implementation ---------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file defines the interfaces that NVPTX uses to lower LLVM code into a10// selection DAG.11//12//===----------------------------------------------------------------------===//13 14#include "NVPTXISelLowering.h"15#include "MCTargetDesc/NVPTXBaseInfo.h"16#include "NVPTX.h"17#include "NVPTXISelDAGToDAG.h"18#include "NVPTXSelectionDAGInfo.h"19#include "NVPTXSubtarget.h"20#include "NVPTXTargetMachine.h"21#include "NVPTXTargetObjectFile.h"22#include "NVPTXUtilities.h"23#include "llvm/ADT/APFloat.h"24#include "llvm/ADT/APInt.h"25#include "llvm/ADT/STLExtras.h"26#include "llvm/ADT/SmallVector.h"27#include "llvm/ADT/StringRef.h"28#include "llvm/CodeGen/Analysis.h"29#include "llvm/CodeGen/ISDOpcodes.h"30#include "llvm/CodeGen/MachineFunction.h"31#include "llvm/CodeGen/MachineJumpTableInfo.h"32#include "llvm/CodeGen/MachineMemOperand.h"33#include "llvm/CodeGen/SelectionDAG.h"34#include "llvm/CodeGen/SelectionDAGNodes.h"35#include "llvm/CodeGen/TargetCallingConv.h"36#include "llvm/CodeGen/TargetLowering.h"37#include "llvm/CodeGen/ValueTypes.h"38#include "llvm/CodeGenTypes/MachineValueType.h"39#include "llvm/IR/Argument.h"40#include "llvm/IR/Attributes.h"41#include "llvm/IR/Constants.h"42#include "llvm/IR/DataLayout.h"43#include "llvm/IR/DerivedTypes.h"44#include "llvm/IR/DiagnosticInfo.h"45#include "llvm/IR/FPEnv.h"46#include "llvm/IR/Function.h"47#include "llvm/IR/GlobalValue.h"48#include "llvm/IR/IRBuilder.h"49#include "llvm/IR/Instruction.h"50#include "llvm/IR/Instructions.h"51#include "llvm/IR/IntrinsicsNVPTX.h"52#include "llvm/IR/Module.h"53#include "llvm/IR/Type.h"54#include "llvm/IR/Value.h"55#include "llvm/Support/Alignment.h"56#include "llvm/Support/AtomicOrdering.h"57#include "llvm/Support/Casting.h"58#include "llvm/Support/CodeGen.h"59#include "llvm/Support/CommandLine.h"60#include "llvm/Support/ErrorHandling.h"61#include "llvm/Support/KnownBits.h"62#include "llvm/Support/NVPTXAddrSpace.h"63#include "llvm/Support/raw_ostream.h"64#include "llvm/Target/TargetMachine.h"65#include "llvm/Target/TargetOptions.h"66#include <algorithm>67#include <cassert>68#include <cmath>69#include <cstdint>70#include <iterator>71#include <optional>72#include <string>73#include <tuple>74#include <utility>75#include <vector>76 77#define DEBUG_TYPE "nvptx-lower"78 79using namespace llvm;80 81static cl::opt<bool> sched4reg(82 "nvptx-sched4reg",83 cl::desc("NVPTX Specific: schedule for register pressue"), cl::init(false));84 85static cl::opt<unsigned> FMAContractLevelOpt(86 "nvptx-fma-level", cl::Hidden,87 cl::desc("NVPTX Specific: FMA contraction (0: don't do it"88 " 1: do it 2: do it aggressively"),89 cl::init(2));90 91static cl::opt<NVPTX::DivPrecisionLevel> UsePrecDivF32(92 "nvptx-prec-divf32", cl::Hidden,93 cl::desc(94 "NVPTX Specific: Override the precision of the lowering for f32 fdiv"),95 cl::values(96 clEnumValN(NVPTX::DivPrecisionLevel::Approx, "0", "Use div.approx"),97 clEnumValN(NVPTX::DivPrecisionLevel::Full, "1", "Use div.full"),98 clEnumValN(NVPTX::DivPrecisionLevel::IEEE754, "2",99 "Use IEEE Compliant F32 div.rnd if available (default)"),100 clEnumValN(NVPTX::DivPrecisionLevel::IEEE754_NoFTZ, "3",101 "Use IEEE Compliant F32 div.rnd if available, no FTZ")),102 cl::init(NVPTX::DivPrecisionLevel::IEEE754));103 104static cl::opt<bool> UsePrecSqrtF32(105 "nvptx-prec-sqrtf32", cl::Hidden,106 cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),107 cl::init(true));108 109/// Whereas CUDA's implementation (see libdevice) uses ex2.approx for exp2(), it110/// does NOT use lg2.approx for log2, so this is disabled by default.111static cl::opt<bool> UseApproxLog2F32(112 "nvptx-approx-log2f32",113 cl::desc("NVPTX Specific: whether to use lg2.approx for log2"),114 cl::init(false));115 116static cl::opt<bool> ForceMinByValParamAlign(117 "nvptx-force-min-byval-param-align", cl::Hidden,118 cl::desc("NVPTX Specific: force 4-byte minimal alignment for byval"119 " params of device functions."),120 cl::init(false));121 122NVPTX::DivPrecisionLevel123NVPTXTargetLowering::getDivF32Level(const MachineFunction &MF,124 const SDNode &N) const {125 // If nvptx-prec-div32=N is used on the command-line, always honor it126 if (UsePrecDivF32.getNumOccurrences() > 0)127 return UsePrecDivF32;128 129 const SDNodeFlags Flags = N.getFlags();130 if (Flags.hasApproximateFuncs())131 return NVPTX::DivPrecisionLevel::Approx;132 133 return NVPTX::DivPrecisionLevel::IEEE754;134}135 136bool NVPTXTargetLowering::usePrecSqrtF32(const SDNode *N) const {137 // If nvptx-prec-sqrtf32 is used on the command-line, always honor it138 if (UsePrecSqrtF32.getNumOccurrences() > 0)139 return UsePrecSqrtF32;140 141 if (N) {142 const SDNodeFlags Flags = N->getFlags();143 if (Flags.hasApproximateFuncs())144 return false;145 }146 147 return true;148}149 150bool NVPTXTargetLowering::useF32FTZ(const MachineFunction &MF) const {151 return MF.getDenormalMode(APFloat::IEEEsingle()).Output ==152 DenormalMode::PreserveSign;153}154 155static bool IsPTXVectorType(MVT VT) {156 switch (VT.SimpleTy) {157 default:158 return false;159 case MVT::v2i1:160 case MVT::v4i1:161 case MVT::v2i8:162 case MVT::v4i8:163 case MVT::v8i8: // <2 x i8x4>164 case MVT::v16i8: // <4 x i8x4>165 case MVT::v2i16:166 case MVT::v4i16:167 case MVT::v8i16: // <4 x i16x2>168 case MVT::v2i32:169 case MVT::v4i32:170 case MVT::v2i64:171 case MVT::v2f16:172 case MVT::v4f16:173 case MVT::v8f16: // <4 x f16x2>174 case MVT::v2bf16:175 case MVT::v4bf16:176 case MVT::v8bf16: // <4 x bf16x2>177 case MVT::v2f32:178 case MVT::v4f32:179 case MVT::v2f64:180 case MVT::v4i64:181 case MVT::v4f64:182 case MVT::v8i32:183 case MVT::v8f32:184 case MVT::v16f16: // <8 x f16x2>185 case MVT::v16bf16: // <8 x bf16x2>186 case MVT::v16i16: // <8 x i16x2>187 case MVT::v32i8: // <8 x i8x4>188 return true;189 }190}191 192// When legalizing vector loads/stores, this function is called, which does two193// things:194// 1. Determines Whether the vector is something we want to custom lower,195// std::nullopt is returned if we do not want to custom lower it.196// 2. If we do want to handle it, returns two parameters:197// - unsigned int NumElts - The number of elements in the final vector198// - EVT EltVT - The type of the elements in the final vector199static std::optional<std::pair<unsigned int, MVT>>200getVectorLoweringShape(EVT VectorEVT, const NVPTXSubtarget &STI,201 unsigned AddressSpace) {202 const bool CanLowerTo256Bit = STI.has256BitVectorLoadStore(AddressSpace);203 204 if (CanLowerTo256Bit && VectorEVT.isScalarInteger() &&205 VectorEVT.getSizeInBits() == 256)206 return {{4, MVT::i64}};207 208 if (!VectorEVT.isSimple())209 return std::nullopt;210 const MVT VectorVT = VectorEVT.getSimpleVT();211 212 if (!VectorVT.isVector()) {213 if (VectorVT == MVT::i128 || VectorVT == MVT::f128)214 return {{2, MVT::i64}};215 return std::nullopt;216 }217 218 const MVT EltVT = VectorVT.getVectorElementType();219 const unsigned NumElts = VectorVT.getVectorNumElements();220 221 // The size of the PTX virtual register that holds a packed type.222 unsigned PackRegSize;223 224 // We only handle "native" vector sizes for now, e.g. <4 x double> is not225 // legal. We can (and should) split that into 2 stores of <2 x double> here226 // but I'm leaving that as a TODO for now.227 switch (VectorVT.SimpleTy) {228 default:229 return std::nullopt;230 231 case MVT::v4i64:232 case MVT::v4f64:233 // This is a "native" vector type iff the address space is global and the234 // target supports 256-bit loads/stores235 if (!CanLowerTo256Bit)236 return std::nullopt;237 [[fallthrough]];238 case MVT::v2i8:239 case MVT::v2i64:240 case MVT::v2f64:241 // This is a "native" vector type242 return std::pair(NumElts, EltVT);243 244 case MVT::v16f16: // <8 x f16x2>245 case MVT::v16bf16: // <8 x bf16x2>246 case MVT::v16i16: // <8 x i16x2>247 case MVT::v32i8: // <8 x i8x4>248 // This can be upsized into a "native" vector type iff the address space is249 // global and the target supports 256-bit loads/stores.250 if (!CanLowerTo256Bit)251 return std::nullopt;252 [[fallthrough]];253 case MVT::v2i16: // <1 x i16x2>254 case MVT::v2f16: // <1 x f16x2>255 case MVT::v2bf16: // <1 x bf16x2>256 case MVT::v4i8: // <1 x i8x4>257 case MVT::v4i16: // <2 x i16x2>258 case MVT::v4f16: // <2 x f16x2>259 case MVT::v4bf16: // <2 x bf16x2>260 case MVT::v8i8: // <2 x i8x4>261 case MVT::v8f16: // <4 x f16x2>262 case MVT::v8bf16: // <4 x bf16x2>263 case MVT::v8i16: // <4 x i16x2>264 case MVT::v16i8: // <4 x i8x4>265 PackRegSize = 32;266 break;267 268 case MVT::v8f32: // <4 x f32x2>269 case MVT::v8i32: // <4 x i32x2>270 // This is a "native" vector type iff the address space is global and the271 // target supports 256-bit loads/stores272 if (!CanLowerTo256Bit)273 return std::nullopt;274 [[fallthrough]];275 case MVT::v2f32: // <1 x f32x2>276 case MVT::v4f32: // <2 x f32x2>277 case MVT::v2i32: // <1 x i32x2>278 case MVT::v4i32: // <2 x i32x2>279 if (!STI.hasF32x2Instructions())280 return std::pair(NumElts, EltVT);281 PackRegSize = 64;282 break;283 }284 285 // If we reach here, then we can pack 2 or more elements into a single 32-bit286 // or 64-bit PTX register and treat the vector as a new vector containing287 // packed elements.288 289 // Number of elements to pack in one word.290 const unsigned NPerReg = PackRegSize / EltVT.getSizeInBits();291 292 return std::pair(NumElts / NPerReg, MVT::getVectorVT(EltVT, NPerReg));293}294 295/// ComputePTXValueVTs - For the given Type \p Ty, returns the set of primitive296/// legal-ish MVTs that compose it. Unlike ComputeValueVTs, this will legalize297/// the types as required by the calling convention (with special handling for298/// i8s).299/// NOTE: This is a band-aid for code that expects ComputeValueVTs to return the300/// same number of types as the Ins/Outs arrays in LowerFormalArguments,301/// LowerCall, and LowerReturn.302static void ComputePTXValueVTs(const TargetLowering &TLI, const DataLayout &DL,303 LLVMContext &Ctx, CallingConv::ID CallConv,304 Type *Ty, SmallVectorImpl<EVT> &ValueVTs,305 SmallVectorImpl<uint64_t> &Offsets,306 uint64_t StartingOffset = 0) {307 SmallVector<EVT, 16> TempVTs;308 SmallVector<uint64_t, 16> TempOffsets;309 ComputeValueVTs(TLI, DL, Ty, TempVTs, /*MemVTs=*/nullptr, &TempOffsets,310 StartingOffset);311 312 for (const auto [VT, Off] : zip(TempVTs, TempOffsets)) {313 MVT RegisterVT = TLI.getRegisterTypeForCallingConv(Ctx, CallConv, VT);314 unsigned NumRegs = TLI.getNumRegistersForCallingConv(Ctx, CallConv, VT);315 316 // Since we actually can load/store b8, we need to ensure that we'll use317 // the original sized type for any i8s or i8 vectors.318 if (VT.getScalarType() == MVT::i8) {319 if (RegisterVT == MVT::i16)320 RegisterVT = MVT::i8;321 else if (RegisterVT == MVT::v2i16)322 RegisterVT = MVT::v2i8;323 else324 assert(RegisterVT == MVT::v4i8 &&325 "Expected v4i8, v2i16, or i16 for i8 RegisterVT");326 }327 328 // TODO: This is horribly incorrect for cases where the vector elements are329 // not a multiple of bytes (ex i1) and legal or i8. However, this problem330 // has existed for as long as NVPTX has and no one has complained, so we'll331 // leave it for now.332 for (unsigned I : seq(NumRegs)) {333 ValueVTs.push_back(RegisterVT);334 Offsets.push_back(Off + I * RegisterVT.getStoreSize());335 }336 }337}338 339// We return an EVT that can hold N VTs340// If the VT is a vector, the resulting EVT is a flat vector with the same341// element type as VT's element type.342static EVT getVectorizedVT(EVT VT, unsigned N, LLVMContext &C) {343 if (N == 1)344 return VT;345 346 return VT.isVector() ? EVT::getVectorVT(C, VT.getScalarType(),347 VT.getVectorNumElements() * N)348 : EVT::getVectorVT(C, VT, N);349}350 351static SDValue getExtractVectorizedValue(SDValue V, unsigned I, EVT VT,352 const SDLoc &dl, SelectionDAG &DAG) {353 if (V.getValueType() == VT) {354 assert(I == 0 && "Index must be 0 for scalar value");355 return V;356 }357 358 if (!VT.isVector())359 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, V,360 DAG.getVectorIdxConstant(I, dl));361 362 return DAG.getNode(363 ISD::EXTRACT_SUBVECTOR, dl, VT, V,364 DAG.getVectorIdxConstant(I * VT.getVectorNumElements(), dl));365}366 367template <typename T>368static inline SDValue getBuildVectorizedValue(unsigned N, const SDLoc &dl,369 SelectionDAG &DAG, T GetElement) {370 if (N == 1)371 return GetElement(0);372 373 SmallVector<SDValue, 8> Values;374 for (const unsigned I : llvm::seq(N)) {375 SDValue Val = GetElement(I);376 if (Val.getValueType().isVector())377 DAG.ExtractVectorElements(Val, Values);378 else379 Values.push_back(Val);380 }381 382 EVT VT = EVT::getVectorVT(*DAG.getContext(), Values[0].getValueType(),383 Values.size());384 return DAG.getBuildVector(VT, dl, Values);385}386 387/// PromoteScalarIntegerPTX388/// Used to make sure the arguments/returns are suitable for passing389/// and promote them to a larger size if they're not.390///391/// The promoted type is placed in \p PromoteVT if the function returns true.392static EVT promoteScalarIntegerPTX(const EVT VT) {393 if (VT.isScalarInteger()) {394 switch (PowerOf2Ceil(VT.getFixedSizeInBits())) {395 default:396 llvm_unreachable(397 "Promotion is not suitable for scalars of size larger than 64-bits");398 case 1:399 return MVT::i1;400 case 2:401 case 4:402 case 8:403 return MVT::i8;404 case 16:405 return MVT::i16;406 case 32:407 return MVT::i32;408 case 64:409 return MVT::i64;410 }411 }412 return VT;413}414 415// Check whether we can merge loads/stores of some of the pieces of a416// flattened function parameter or return value into a single vector417// load/store.418//419// The flattened parameter is represented as a list of EVTs and420// offsets, and the whole structure is aligned to ParamAlignment. This421// function determines whether we can load/store pieces of the422// parameter starting at index Idx using a single vectorized op of423// size AccessSize. If so, it returns the number of param pieces424// covered by the vector op. Otherwise, it returns 1.425template <typename T>426static unsigned canMergeParamLoadStoresStartingAt(427 unsigned Idx, uint32_t AccessSize, const SmallVectorImpl<EVT> &ValueVTs,428 const SmallVectorImpl<T> &Offsets, Align ParamAlignment) {429 430 // Can't vectorize if param alignment is not sufficient.431 if (ParamAlignment < AccessSize)432 return 1;433 // Can't vectorize if offset is not aligned.434 if (Offsets[Idx] & (AccessSize - 1))435 return 1;436 437 EVT EltVT = ValueVTs[Idx];438 unsigned EltSize = EltVT.getStoreSize();439 440 // Element is too large to vectorize.441 if (EltSize >= AccessSize)442 return 1;443 444 unsigned NumElts = AccessSize / EltSize;445 // Can't vectorize if AccessBytes if not a multiple of EltSize.446 if (AccessSize != EltSize * NumElts)447 return 1;448 449 // We don't have enough elements to vectorize.450 if (Idx + NumElts > ValueVTs.size())451 return 1;452 453 // PTX ISA can only deal with 2- and 4-element vector ops.454 if (NumElts != 4 && NumElts != 2)455 return 1;456 457 for (unsigned j = Idx + 1; j < Idx + NumElts; ++j) {458 // Types do not match.459 if (ValueVTs[j] != EltVT)460 return 1;461 462 // Elements are not contiguous.463 if (Offsets[j] - Offsets[j - 1] != EltSize)464 return 1;465 }466 // OK. We can vectorize ValueVTs[i..i+NumElts)467 return NumElts;468}469 470// Computes whether and how we can vectorize the loads/stores of a471// flattened function parameter or return value.472//473// The flattened parameter is represented as the list of ValueVTs and474// Offsets, and is aligned to ParamAlignment bytes. We return a vector475// of the same size as ValueVTs indicating how each piece should be476// loaded/stored (i.e. as a scalar, or as part of a vector477// load/store).478template <typename T>479static SmallVector<unsigned, 16>480VectorizePTXValueVTs(const SmallVectorImpl<EVT> &ValueVTs,481 const SmallVectorImpl<T> &Offsets, Align ParamAlignment,482 bool IsVAArg = false) {483 // Set vector size to match ValueVTs and mark all elements as484 // scalars by default.485 486 if (IsVAArg)487 return SmallVector<unsigned>(ValueVTs.size(), 1);488 489 SmallVector<unsigned, 16> VectorInfo;490 491 const auto GetNumElts = [&](unsigned I) -> unsigned {492 for (const unsigned AccessSize : {16, 8, 4, 2}) {493 const unsigned NumElts = canMergeParamLoadStoresStartingAt(494 I, AccessSize, ValueVTs, Offsets, ParamAlignment);495 assert((NumElts == 1 || NumElts == 2 || NumElts == 4) &&496 "Unexpected vectorization size");497 if (NumElts != 1)498 return NumElts;499 }500 return 1;501 };502 503 // Check what we can vectorize using 128/64/32-bit accesses.504 for (unsigned I = 0, E = ValueVTs.size(); I != E;) {505 const unsigned NumElts = GetNumElts(I);506 VectorInfo.push_back(NumElts);507 I += NumElts;508 }509 assert(std::accumulate(VectorInfo.begin(), VectorInfo.end(), 0u) ==510 ValueVTs.size());511 return VectorInfo;512}513 514// NVPTXTargetLowering Constructor.515NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,516 const NVPTXSubtarget &STI)517 : TargetLowering(TM, STI), nvTM(&TM), STI(STI), GlobalUniqueCallSite(0) {518 // always lower memset, memcpy, and memmove intrinsics to load/store519 // instructions, rather520 // then generating calls to memset, mempcy or memmove.521 MaxStoresPerMemset = MaxStoresPerMemsetOptSize = (unsigned)0xFFFFFFFF;522 MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = (unsigned) 0xFFFFFFFF;523 MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = (unsigned) 0xFFFFFFFF;524 525 setBooleanContents(ZeroOrNegativeOneBooleanContent);526 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);527 528 // Jump is Expensive. Don't create extra control flow for 'and', 'or'529 // condition branches.530 setJumpIsExpensive(true);531 532 // Wide divides are _very_ slow. Try to reduce the width of the divide if533 // possible.534 addBypassSlowDiv(64, 32);535 536 // By default, use the Source scheduling537 if (sched4reg)538 setSchedulingPreference(Sched::RegPressure);539 else540 setSchedulingPreference(Sched::Source);541 542 auto setFP16OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,543 LegalizeAction NoF16Action) {544 bool IsOpSupported = STI.allowFP16Math();545 switch (Op) {546 // Several FP16 instructions are available on sm_80 only.547 case ISD::FMINNUM:548 case ISD::FMAXNUM:549 case ISD::FMAXNUM_IEEE:550 case ISD::FMINNUM_IEEE:551 case ISD::FMAXIMUM:552 case ISD::FMINIMUM:553 case ISD::FMAXIMUMNUM:554 case ISD::FMINIMUMNUM:555 IsOpSupported &= STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70;556 break;557 case ISD::FEXP2:558 IsOpSupported &= STI.getSmVersion() >= 75 && STI.getPTXVersion() >= 70;559 break;560 }561 setOperationAction(Op, VT, IsOpSupported ? Action : NoF16Action);562 };563 564 auto setBF16OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,565 LegalizeAction NoBF16Action) {566 bool IsOpSupported = STI.hasNativeBF16Support(Op);567 setOperationAction(568 Op, VT, IsOpSupported ? Action : NoBF16Action);569 };570 571 auto setI16x2OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,572 LegalizeAction NoI16x2Action) {573 bool IsOpSupported = false;574 // instructions are available on sm_90 only575 switch (Op) {576 case ISD::ADD:577 case ISD::SMAX:578 case ISD::SMIN:579 case ISD::UMIN:580 case ISD::UMAX:581 IsOpSupported = STI.getSmVersion() >= 90 && STI.getPTXVersion() >= 80;582 break;583 }584 setOperationAction(Op, VT, IsOpSupported ? Action : NoI16x2Action);585 };586 587 addRegisterClass(MVT::i1, &NVPTX::B1RegClass);588 addRegisterClass(MVT::i16, &NVPTX::B16RegClass);589 addRegisterClass(MVT::v2i16, &NVPTX::B32RegClass);590 addRegisterClass(MVT::v4i8, &NVPTX::B32RegClass);591 addRegisterClass(MVT::i32, &NVPTX::B32RegClass);592 addRegisterClass(MVT::i64, &NVPTX::B64RegClass);593 addRegisterClass(MVT::f32, &NVPTX::B32RegClass);594 addRegisterClass(MVT::f64, &NVPTX::B64RegClass);595 addRegisterClass(MVT::f16, &NVPTX::B16RegClass);596 addRegisterClass(MVT::v2f16, &NVPTX::B32RegClass);597 addRegisterClass(MVT::bf16, &NVPTX::B16RegClass);598 addRegisterClass(MVT::v2bf16, &NVPTX::B32RegClass);599 600 if (STI.hasF32x2Instructions()) {601 addRegisterClass(MVT::v2f32, &NVPTX::B64RegClass);602 addRegisterClass(MVT::v2i32, &NVPTX::B64RegClass);603 }604 605 // Conversion to/from FP16/FP16x2 is always legal.606 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f16, Custom);607 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f16, Custom);608 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f16, Expand);609 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f16, Expand);610 611 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal);612 if (STI.getSmVersion() >= 30 && STI.getPTXVersion() > 31)613 setOperationAction(ISD::READSTEADYCOUNTER, MVT::i64, Legal);614 615 setFP16OperationAction(ISD::SETCC, MVT::f16, Legal, Promote);616 setFP16OperationAction(ISD::SETCC, MVT::v2f16, Legal, Expand);617 618 // Conversion to/from BFP16/BFP16x2 is always legal.619 setOperationAction(ISD::BUILD_VECTOR, MVT::v2bf16, Custom);620 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2bf16, Custom);621 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2bf16, Expand);622 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2bf16, Expand);623 624 setBF16OperationAction(ISD::SETCC, MVT::v2bf16, Legal, Expand);625 setBF16OperationAction(ISD::SETCC, MVT::bf16, Legal, Promote);626 if (getOperationAction(ISD::SETCC, MVT::bf16) == Promote)627 AddPromotedToType(ISD::SETCC, MVT::bf16, MVT::f32);628 629 // Conversion to/from i16/i16x2 is always legal.630 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i16, Custom);631 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i16, Custom);632 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i16, Expand);633 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i16, Expand);634 635 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i8, Custom);636 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i8, Custom);637 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i8, Custom);638 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i8, Custom);639 640 // No support for these operations with v2f32/v2i32641 setOperationAction(ISD::INSERT_VECTOR_ELT, {MVT::v2f32, MVT::v2i32}, Expand);642 setOperationAction(ISD::VECTOR_SHUFFLE, {MVT::v2f32, MVT::v2i32}, Expand);643 644 setOperationAction(ISD::TRUNCATE, MVT::v2i16, Expand);645 setOperationAction({ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},646 MVT::v2i32, Expand);647 648 // Need custom lowering in case the index is dynamic.649 if (STI.hasF32x2Instructions())650 setOperationAction(ISD::EXTRACT_VECTOR_ELT, {MVT::v2f32, MVT::v2i32},651 Custom);652 653 // Custom conversions to/from v2i8.654 setOperationAction(ISD::BITCAST, MVT::v2i8, Custom);655 656 // Only logical ops can be done on v4i8/v2i32 directly, others must be done657 // elementwise.658 setOperationAction(659 {ISD::ABS, ISD::ADD, ISD::ADDC, ISD::ADDE,660 ISD::BITREVERSE, ISD::CTLZ, ISD::CTPOP, ISD::CTTZ,661 ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::FSHL, ISD::FSHR,662 ISD::MUL, ISD::MULHS, ISD::MULHU, ISD::PARITY,663 ISD::ROTL, ISD::ROTR, ISD::SADDO, ISD::SADDO_CARRY,664 ISD::SADDSAT, ISD::SDIV, ISD::SDIVREM, ISD::SELECT_CC,665 ISD::SETCC, ISD::SHL, ISD::SINT_TO_FP, ISD::SMAX,666 ISD::SMIN, ISD::SMULO, ISD::SMUL_LOHI, ISD::SRA,667 ISD::SREM, ISD::SRL, ISD::SSHLSAT, ISD::SSUBO,668 ISD::SSUBO_CARRY, ISD::SSUBSAT, ISD::SUB, ISD::SUBC,669 ISD::SUBE, ISD::UADDO, ISD::UADDO_CARRY, ISD::UADDSAT,670 ISD::UDIV, ISD::UDIVREM, ISD::UINT_TO_FP, ISD::UMAX,671 ISD::UMIN, ISD::UMULO, ISD::UMUL_LOHI, ISD::UREM,672 ISD::USHLSAT, ISD::USUBO, ISD::USUBO_CARRY, ISD::VSELECT,673 ISD::USUBSAT},674 {MVT::v4i8, MVT::v2i32}, Expand);675 676 // Operations not directly supported by NVPTX.677 for (MVT VT : {MVT::bf16, MVT::f16, MVT::v2bf16, MVT::v2f16, MVT::f32,678 MVT::v2f32, MVT::f64, MVT::i1, MVT::i8, MVT::i16, MVT::v2i16,679 MVT::v4i8, MVT::i32, MVT::v2i32, MVT::i64}) {680 setOperationAction(ISD::SELECT_CC, VT, Expand);681 setOperationAction(ISD::BR_CC, VT, Expand);682 }683 684 // We don't want ops like FMINIMUM or UMAX to be lowered to SETCC+VSELECT.685 setOperationAction(ISD::VSELECT, {MVT::v2f32, MVT::v2i32}, Expand);686 687 // Some SIGN_EXTEND_INREG can be done using cvt instruction.688 // For others we will expand to a SHL/SRA pair.689 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i64, Legal);690 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal);691 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Legal);692 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal);693 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);694 setOperationAction(ISD::SIGN_EXTEND_INREG, {MVT::v2i16, MVT::v2i32}, Expand);695 696 setOperationAction(ISD::SHL_PARTS, MVT::i32 , Custom);697 setOperationAction(ISD::SRA_PARTS, MVT::i32 , Custom);698 setOperationAction(ISD::SRL_PARTS, MVT::i32 , Custom);699 setOperationAction(ISD::SHL_PARTS, MVT::i64 , Custom);700 setOperationAction(ISD::SRA_PARTS, MVT::i64 , Custom);701 setOperationAction(ISD::SRL_PARTS, MVT::i64 , Custom);702 703 setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);704 setOperationAction(ISD::BITREVERSE, MVT::i64, Legal);705 706 setOperationAction({ISD::ROTL, ISD::ROTR},707 {MVT::i8, MVT::i16, MVT::v2i16, MVT::i32, MVT::i64},708 Expand);709 710 if (STI.hasHWROT32()) {711 setOperationAction({ISD::FSHL, ISD::FSHR}, MVT::i32, Legal);712 setOperationAction({ISD::ROTL, ISD::ROTR, ISD::FSHL, ISD::FSHR}, MVT::i64,713 Custom);714 }715 716 setOperationAction(ISD::BR_JT, MVT::Other, Custom);717 setOperationAction(ISD::BRIND, MVT::Other, Expand);718 719 // We want to legalize constant related memmove and memcopy720 // intrinsics.721 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);722 723 // FP extload/truncstore is not legal in PTX. We need to expand all these.724 for (auto FloatVTs :725 {MVT::fp_valuetypes(), MVT::fp_fixedlen_vector_valuetypes()}) {726 for (MVT ValVT : FloatVTs) {727 for (MVT MemVT : FloatVTs) {728 setLoadExtAction(ISD::EXTLOAD, ValVT, MemVT, Expand);729 setTruncStoreAction(ValVT, MemVT, Expand);730 }731 }732 }733 734 // To improve CodeGen we'll legalize any-extend loads to zext loads. This is735 // how they'll be lowered in ISel anyway, and by doing this a little earlier736 // we allow for more DAG combine opportunities.737 for (auto IntVTs :738 {MVT::integer_valuetypes(), MVT::integer_fixedlen_vector_valuetypes()})739 for (MVT ValVT : IntVTs)740 for (MVT MemVT : IntVTs)741 if (isTypeLegal(ValVT))742 setLoadExtAction(ISD::EXTLOAD, ValVT, MemVT, Custom);743 744 // PTX does not support load / store predicate registers745 setOperationAction({ISD::LOAD, ISD::STORE}, MVT::i1, Custom);746 for (MVT VT : MVT::integer_valuetypes()) {747 setLoadExtAction({ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}, VT, MVT::i1,748 Promote);749 setTruncStoreAction(VT, MVT::i1, Expand);750 }751 752 // Disable generations of extload/truncstore for v2i32/v2i16/v2i8. The generic753 // expansion for these nodes when they are unaligned is incorrect if the754 // type is a vector.755 //756 // TODO: Fix the generic expansion for these nodes found in757 // TargetLowering::expandUnalignedLoad/Store.758 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, MVT::v2i16,759 MVT::v2i8, Expand);760 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, MVT::v2i32,761 {MVT::v2i8, MVT::v2i16}, Expand);762 setTruncStoreAction(MVT::v2i16, MVT::v2i8, Expand);763 setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand);764 setTruncStoreAction(MVT::v2i32, MVT::v2i8, Expand);765 766 // Register custom handling for illegal type loads/stores. We'll try to custom767 // lower almost all illegal types and logic in the lowering will discard cases768 // we can't handle.769 setOperationAction({ISD::LOAD, ISD::STORE}, {MVT::i128, MVT::i256, MVT::f128},770 Custom);771 for (MVT VT : MVT::fixedlen_vector_valuetypes())772 if (!isTypeLegal(VT) && VT.getStoreSizeInBits() <= 256)773 setOperationAction({ISD::STORE, ISD::LOAD, ISD::MSTORE, ISD::MLOAD}, VT,774 Custom);775 776 // Custom legalization for LDU intrinsics.777 // TODO: The logic to lower these is not very robust and we should rewrite it.778 // Perhaps LDU should not be represented as an intrinsic at all.779 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom);780 for (MVT VT : MVT::fixedlen_vector_valuetypes())781 if (IsPTXVectorType(VT))782 setOperationAction(ISD::INTRINSIC_W_CHAIN, VT, Custom);783 784 setCondCodeAction({ISD::SETNE, ISD::SETEQ, ISD::SETUGE, ISD::SETULE,785 ISD::SETUGT, ISD::SETULT, ISD::SETGT, ISD::SETLT,786 ISD::SETGE, ISD::SETLE},787 MVT::i1, Expand);788 789 // This is legal in NVPTX790 setOperationAction(ISD::ConstantFP, MVT::f64, Legal);791 setOperationAction(ISD::ConstantFP, MVT::f32, Legal);792 setOperationAction(ISD::ConstantFP, MVT::f16, Legal);793 setOperationAction(ISD::ConstantFP, MVT::bf16, Legal);794 795 setOperationAction(ISD::DYNAMIC_STACKALLOC, {MVT::i32, MVT::i64}, Custom);796 setOperationAction({ISD::STACKRESTORE, ISD::STACKSAVE}, MVT::Other, Custom);797 798 // TRAP can be lowered to PTX trap799 setOperationAction(ISD::TRAP, MVT::Other, Legal);800 // DEBUGTRAP can be lowered to PTX brkpt801 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);802 803 // Support varargs.804 setOperationAction(ISD::VASTART, MVT::Other, Custom);805 setOperationAction(ISD::VAARG, MVT::Other, Custom);806 setOperationAction(ISD::VACOPY, MVT::Other, Expand);807 setOperationAction(ISD::VAEND, MVT::Other, Expand);808 809 setOperationAction({ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX},810 {MVT::i16, MVT::i32, MVT::i64}, Legal);811 812 setOperationAction({ISD::CTPOP, ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF}, MVT::i16,813 Promote);814 setOperationAction({ISD::CTPOP, ISD::CTLZ}, MVT::i32, Legal);815 setOperationAction({ISD::CTPOP, ISD::CTLZ}, MVT::i64, Custom);816 817 setI16x2OperationAction(ISD::ABS, MVT::v2i16, Legal, Custom);818 setI16x2OperationAction(ISD::SMIN, MVT::v2i16, Legal, Custom);819 setI16x2OperationAction(ISD::SMAX, MVT::v2i16, Legal, Custom);820 setI16x2OperationAction(ISD::UMIN, MVT::v2i16, Legal, Custom);821 setI16x2OperationAction(ISD::UMAX, MVT::v2i16, Legal, Custom);822 setI16x2OperationAction(ISD::CTPOP, MVT::v2i16, Legal, Expand);823 setI16x2OperationAction(ISD::CTLZ, MVT::v2i16, Legal, Expand);824 825 setI16x2OperationAction(ISD::ADD, MVT::v2i16, Legal, Custom);826 setI16x2OperationAction(ISD::SUB, MVT::v2i16, Legal, Custom);827 setI16x2OperationAction(ISD::MUL, MVT::v2i16, Legal, Custom);828 setI16x2OperationAction(ISD::SHL, MVT::v2i16, Legal, Custom);829 setI16x2OperationAction(ISD::SREM, MVT::v2i16, Legal, Custom);830 setI16x2OperationAction(ISD::UREM, MVT::v2i16, Legal, Custom);831 832 // Other arithmetic and logic ops are unsupported.833 setOperationAction({ISD::SDIV, ISD::UDIV, ISD::SRA, ISD::SRL, ISD::MULHS,834 ISD::MULHU, ISD::FP_TO_SINT, ISD::FP_TO_UINT,835 ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::SETCC},836 {MVT::v2i16, MVT::v2i32}, Expand);837 838 // v2i32 is not supported for any arithmetic operations839 setOperationAction({ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX,840 ISD::CTPOP, ISD::CTLZ, ISD::ADD, ISD::SUB, ISD::MUL,841 ISD::SHL, ISD::SRA, ISD::SRL, ISD::OR, ISD::AND, ISD::XOR,842 ISD::SREM, ISD::UREM},843 MVT::v2i32, Expand);844 845 setOperationAction(ISD::ADDC, MVT::i32, Legal);846 setOperationAction(ISD::ADDE, MVT::i32, Legal);847 setOperationAction(ISD::SUBC, MVT::i32, Legal);848 setOperationAction(ISD::SUBE, MVT::i32, Legal);849 if (STI.getPTXVersion() >= 43) {850 setOperationAction(ISD::ADDC, MVT::i64, Legal);851 setOperationAction(ISD::ADDE, MVT::i64, Legal);852 setOperationAction(ISD::SUBC, MVT::i64, Legal);853 setOperationAction(ISD::SUBE, MVT::i64, Legal);854 }855 856 setOperationAction(ISD::CTTZ, MVT::i16, Expand);857 setOperationAction(ISD::CTTZ, {MVT::v2i16, MVT::v2i32}, Expand);858 setOperationAction(ISD::CTTZ, MVT::i32, Expand);859 setOperationAction(ISD::CTTZ, MVT::i64, Expand);860 861 // PTX does not directly support SELP of i1, so promote to i32 first862 setOperationAction(ISD::SELECT, MVT::i1, Custom);863 864 // PTX cannot multiply two i64s in a single instruction.865 setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);866 setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);867 868 // We have some custom DAG combine patterns for these nodes869 setTargetDAGCombine(870 {ISD::ADD, ISD::AND, ISD::EXTRACT_VECTOR_ELT,871 ISD::FADD, ISD::FMAXNUM, ISD::FMINNUM,872 ISD::FMAXIMUM, ISD::FMINIMUM, ISD::FMAXIMUMNUM,873 ISD::FMINIMUMNUM, ISD::MUL, ISD::SHL,874 ISD::SREM, ISD::UREM, ISD::VSELECT,875 ISD::BUILD_VECTOR, ISD::ADDRSPACECAST, ISD::LOAD,876 ISD::STORE, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND});877 878 // setcc for f16x2 and bf16x2 needs special handling to prevent879 // legalizer's attempt to scalarize it due to v2i1 not being legal.880 if (STI.allowFP16Math() || STI.hasBF16Math())881 setTargetDAGCombine(ISD::SETCC);882 883 // Vector reduction operations. These may be turned into shuffle or tree884 // reductions depending on what instructions are available for each type.885 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {886 MVT EltVT = VT.getVectorElementType();887 if (EltVT == MVT::f32 || EltVT == MVT::f64) {888 setOperationAction({ISD::VECREDUCE_FMAX, ISD::VECREDUCE_FMIN,889 ISD::VECREDUCE_FMAXIMUM, ISD::VECREDUCE_FMINIMUM},890 VT, Custom);891 }892 }893 894 // Promote fp16 arithmetic if fp16 hardware isn't available or the895 // user passed --nvptx-no-fp16-math. The flag is useful because,896 // although sm_53+ GPUs have some sort of FP16 support in897 // hardware, only sm_53 and sm_60 have full implementation. Others898 // only have token amount of hardware and are likely to run faster899 // by using fp32 units instead.900 for (const auto &Op : {ISD::FADD, ISD::FMUL, ISD::FSUB, ISD::FMA}) {901 setFP16OperationAction(Op, MVT::f16, Legal, Promote);902 setFP16OperationAction(Op, MVT::v2f16, Legal, Expand);903 setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);904 // bf16 must be promoted to f32.905 setBF16OperationAction(Op, MVT::bf16, Legal, Promote);906 if (getOperationAction(Op, MVT::bf16) == Promote)907 AddPromotedToType(Op, MVT::bf16, MVT::f32);908 setOperationAction(Op, MVT::v2f32,909 STI.hasF32x2Instructions() ? Legal : Expand);910 }911 912 // On SM80, we select add/mul/sub as fma to avoid promotion to float913 for (const auto &Op : {ISD::FADD, ISD::FMUL, ISD::FSUB}) {914 for (const auto &VT : {MVT::bf16, MVT::v2bf16}) {915 if (!STI.hasNativeBF16Support(Op) && STI.hasNativeBF16Support(ISD::FMA)) {916 setOperationAction(Op, VT, Custom);917 }918 }919 }920 921 // f16/f16x2 neg was introduced in PTX 60, SM_53.922 const bool IsFP16FP16x2NegAvailable = STI.getSmVersion() >= 53 &&923 STI.getPTXVersion() >= 60 &&924 STI.allowFP16Math();925 for (const auto &VT : {MVT::f16, MVT::v2f16})926 setOperationAction(ISD::FNEG, VT,927 IsFP16FP16x2NegAvailable ? Legal : Expand);928 929 setBF16OperationAction(ISD::FNEG, MVT::bf16, Legal, Expand);930 setBF16OperationAction(ISD::FNEG, MVT::v2bf16, Legal, Expand);931 setOperationAction(ISD::FNEG, MVT::v2f32, Expand);932 // (would be) Library functions.933 934 // These map to conversion instructions for scalar FP types.935 for (const auto &Op : {ISD::FCEIL, ISD::FFLOOR, ISD::FNEARBYINT, ISD::FRINT,936 ISD::FROUNDEVEN, ISD::FTRUNC}) {937 setOperationAction(Op, MVT::f16, Legal);938 setOperationAction(Op, MVT::f32, Legal);939 setOperationAction(Op, MVT::f64, Legal);940 setOperationAction(Op, MVT::v2f16, Expand);941 setOperationAction(Op, MVT::v2bf16, Expand);942 setOperationAction(Op, MVT::v2f32, Expand);943 setBF16OperationAction(Op, MVT::bf16, Legal, Promote);944 if (getOperationAction(Op, MVT::bf16) == Promote)945 AddPromotedToType(Op, MVT::bf16, MVT::f32);946 }947 948 if (STI.getSmVersion() < 80 || STI.getPTXVersion() < 71) {949 setOperationAction(ISD::BF16_TO_FP, MVT::f32, Expand);950 }951 if (STI.getSmVersion() < 90 || STI.getPTXVersion() < 78) {952 for (MVT VT : {MVT::bf16, MVT::f32, MVT::f64}) {953 setOperationAction(ISD::FP_EXTEND, VT, Custom);954 setOperationAction(ISD::FP_ROUND, VT, Custom);955 }956 }957 958 // Expand v2f32 = fp_extend959 setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Expand);960 // Expand v2[b]f16 = fp_round v2f32961 setOperationAction(ISD::FP_ROUND, {MVT::v2bf16, MVT::v2f16}, Expand);962 963 // sm_80 only has conversions between f32 and bf16. Custom lower all other964 // bf16 conversions.965 if (STI.getSmVersion() < 90 || STI.getPTXVersion() < 78) {966 for (MVT VT : {MVT::i1, MVT::i16, MVT::i32, MVT::i64}) {967 setOperationAction(968 {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT},969 VT, Custom);970 }971 setOperationAction(972 {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT},973 MVT::bf16, Custom);974 }975 976 setOperationAction(ISD::FROUND, MVT::f16, Promote);977 setOperationAction(ISD::FROUND, MVT::v2f16, Expand);978 setOperationAction(ISD::FROUND, MVT::v2bf16, Expand);979 setOperationAction(ISD::FROUND, MVT::f32, Custom);980 setOperationAction(ISD::FROUND, MVT::f64, Custom);981 setOperationAction(ISD::FROUND, MVT::bf16, Promote);982 AddPromotedToType(ISD::FROUND, MVT::bf16, MVT::f32);983 984 // 'Expand' implements FCOPYSIGN without calling an external library.985 setOperationAction(ISD::FCOPYSIGN, MVT::f16, Expand);986 setOperationAction(ISD::FCOPYSIGN, MVT::v2f16, Expand);987 setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Expand);988 setOperationAction(ISD::FCOPYSIGN, MVT::v2bf16, Expand);989 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);990 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);991 992 // These map to corresponding instructions for f32/f64. f16 must be993 // promoted to f32. v2f16 is expanded to f16, which is then promoted994 // to f32.995 for (const auto &Op :996 {ISD::FDIV, ISD::FREM, ISD::FSQRT, ISD::FSIN, ISD::FCOS, ISD::FTANH}) {997 setOperationAction(Op, MVT::f16, Promote);998 setOperationAction(Op, MVT::f32, Legal);999 // only div/rem/sqrt are legal for f641000 if (Op == ISD::FDIV || Op == ISD::FREM || Op == ISD::FSQRT) {1001 setOperationAction(Op, MVT::f64, Legal);1002 }1003 setOperationAction(Op, {MVT::v2f16, MVT::v2bf16, MVT::v2f32}, Expand);1004 setOperationAction(Op, MVT::bf16, Promote);1005 AddPromotedToType(Op, MVT::bf16, MVT::f32);1006 }1007 setOperationAction(ISD::FREM, {MVT::f32, MVT::f64}, Custom);1008 1009 setOperationAction(ISD::FABS, {MVT::f32, MVT::f64}, Legal);1010 setOperationAction(ISD::FABS, MVT::v2f32, Expand);1011 if (STI.getPTXVersion() >= 65) {1012 setFP16OperationAction(ISD::FABS, MVT::f16, Legal, Promote);1013 setFP16OperationAction(ISD::FABS, MVT::v2f16, Legal, Expand);1014 } else {1015 setOperationAction(ISD::FABS, MVT::f16, Promote);1016 setOperationAction(ISD::FABS, MVT::v2f16, Expand);1017 }1018 setBF16OperationAction(ISD::FABS, MVT::v2bf16, Legal, Expand);1019 setBF16OperationAction(ISD::FABS, MVT::bf16, Legal, Promote);1020 if (getOperationAction(ISD::FABS, MVT::bf16) == Promote)1021 AddPromotedToType(ISD::FABS, MVT::bf16, MVT::f32);1022 1023 for (const auto &Op :1024 {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM}) {1025 setOperationAction(Op, MVT::f32, Legal);1026 setOperationAction(Op, MVT::f64, Legal);1027 setFP16OperationAction(Op, MVT::f16, Legal, Promote);1028 setFP16OperationAction(Op, MVT::v2f16, Legal, Expand);1029 setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);1030 setBF16OperationAction(Op, MVT::bf16, Legal, Promote);1031 if (getOperationAction(Op, MVT::bf16) == Promote)1032 AddPromotedToType(Op, MVT::bf16, MVT::f32);1033 setOperationAction(Op, MVT::v2f32, Expand);1034 }1035 bool SupportsF32MinMaxNaN =1036 STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70;1037 for (const auto &Op : {ISD::FMINIMUM, ISD::FMAXIMUM}) {1038 setOperationAction(Op, MVT::f32, SupportsF32MinMaxNaN ? Legal : Expand);1039 setFP16OperationAction(Op, MVT::f16, Legal, Expand);1040 setFP16OperationAction(Op, MVT::v2f16, Legal, Expand);1041 setBF16OperationAction(Op, MVT::bf16, Legal, Expand);1042 setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);1043 setOperationAction(Op, MVT::v2f32, Expand);1044 }1045 1046 // Custom lowering for inline asm with 128-bit operands1047 setOperationAction(ISD::CopyToReg, MVT::i128, Custom);1048 setOperationAction(ISD::CopyFromReg, MVT::i128, Custom);1049 1050 // FEXP2 support:1051 // - f321052 // - f16/f16x2 (sm_70+, PTX 7.0+)1053 // - bf16/bf16x2 (sm_90+, PTX 7.8+)1054 // When f16/bf16 types aren't supported, they are promoted/expanded to f32.1055 setOperationAction(ISD::FEXP2, MVT::f32, Legal);1056 setOperationAction(ISD::FEXP2, MVT::v2f32, Expand);1057 setFP16OperationAction(ISD::FEXP2, MVT::f16, Legal, Promote);1058 setFP16OperationAction(ISD::FEXP2, MVT::v2f16, Legal, Expand);1059 setBF16OperationAction(ISD::FEXP2, MVT::bf16, Legal, Promote);1060 setBF16OperationAction(ISD::FEXP2, MVT::v2bf16, Legal, Expand);1061 1062 // FLOG2 supports f32 only1063 // f16/bf16 types aren't supported, but they are promoted/expanded to f32.1064 if (UseApproxLog2F32) {1065 setOperationAction(ISD::FLOG2, MVT::f32, Legal);1066 setOperationPromotedToType(ISD::FLOG2, MVT::f16, MVT::f32);1067 setOperationPromotedToType(ISD::FLOG2, MVT::bf16, MVT::f32);1068 setOperationAction(ISD::FLOG2, {MVT::v2f16, MVT::v2bf16, MVT::v2f32},1069 Expand);1070 }1071 1072 setOperationAction(ISD::ADDRSPACECAST, {MVT::i32, MVT::i64}, Custom);1073 1074 setOperationAction(ISD::ATOMIC_LOAD_SUB, {MVT::i32, MVT::i64}, Expand);1075 1076 // atom.b128 is legal in PTX but since we don't represent i128 as a legal1077 // type, we need to custom lower it.1078 setOperationAction({ISD::ATOMIC_CMP_SWAP, ISD::ATOMIC_SWAP}, MVT::i128,1079 Custom);1080 1081 // Now deduce the information based on the above mentioned1082 // actions1083 computeRegisterProperties(STI.getRegisterInfo());1084 1085 // PTX support for 16-bit CAS is emulated. Only use 32+1086 setMinCmpXchgSizeInBits(STI.getMinCmpXchgSizeInBits());1087 setMaxAtomicSizeInBitsSupported(STI.hasAtomSwap128() ? 128 : 64);1088 setMaxDivRemBitWidthSupported(64);1089 1090 // Custom lowering for tcgen05.ld vector operands1091 setOperationAction(ISD::INTRINSIC_W_CHAIN,1092 {MVT::v2i32, MVT::v4i32, MVT::v8i32, MVT::v16i32,1093 MVT::v32i32, MVT::v64i32, MVT::v128i32},1094 Custom);1095 1096 // Custom lowering for tcgen05.st vector operands1097 setOperationAction(ISD::INTRINSIC_VOID,1098 {MVT::v2i32, MVT::v4i32, MVT::v8i32, MVT::v16i32,1099 MVT::v32i32, MVT::v64i32, MVT::v128i32, MVT::Other},1100 Custom);1101 1102 // Enable custom lowering for the following:1103 // * MVT::i128 - clusterlaunchcontrol1104 // * MVT::i32 - prmt1105 // * MVT::v4f32 - cvt_rs fp{4/6/8}x4 intrinsics1106 // * MVT::Other - internal.addrspace.wrap1107 setOperationAction(ISD::INTRINSIC_WO_CHAIN,1108 {MVT::i32, MVT::i128, MVT::v4f32, MVT::Other}, Custom);1109 1110 // Custom lowering for bswap1111 setOperationAction(ISD::BSWAP, {MVT::i16, MVT::i32, MVT::i64, MVT::v2i16},1112 Custom);1113}1114 1115TargetLoweringBase::LegalizeTypeAction1116NVPTXTargetLowering::getPreferredVectorAction(MVT VT) const {1117 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&1118 VT.getScalarType() == MVT::i1)1119 return TypeSplitVector;1120 return TargetLoweringBase::getPreferredVectorAction(VT);1121}1122 1123SDValue NVPTXTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,1124 int Enabled, int &ExtraSteps,1125 bool &UseOneConst,1126 bool Reciprocal) const {1127 if (!(Enabled == ReciprocalEstimate::Enabled ||1128 (Enabled == ReciprocalEstimate::Unspecified && !usePrecSqrtF32())))1129 return SDValue();1130 1131 if (ExtraSteps == ReciprocalEstimate::Unspecified)1132 ExtraSteps = 0;1133 1134 SDLoc DL(Operand);1135 EVT VT = Operand.getValueType();1136 bool Ftz = useF32FTZ(DAG.getMachineFunction());1137 1138 auto MakeIntrinsicCall = [&](Intrinsic::ID IID) {1139 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,1140 DAG.getConstant(IID, DL, MVT::i32), Operand);1141 };1142 1143 // The sqrt and rsqrt refinement processes assume we always start out with an1144 // approximation of the rsqrt. Therefore, if we're going to do any refinement1145 // (i.e. ExtraSteps > 0), we must return an rsqrt. But if we're *not* doing1146 // any refinement, we must return a regular sqrt.1147 if (Reciprocal || ExtraSteps > 0) {1148 if (VT == MVT::f32)1149 return MakeIntrinsicCall(Ftz ? Intrinsic::nvvm_rsqrt_approx_ftz_f1150 : Intrinsic::nvvm_rsqrt_approx_f);1151 else if (VT == MVT::f64)1152 return MakeIntrinsicCall(Intrinsic::nvvm_rsqrt_approx_d);1153 else1154 return SDValue();1155 } else {1156 if (VT == MVT::f32)1157 return MakeIntrinsicCall(Ftz ? Intrinsic::nvvm_sqrt_approx_ftz_f1158 : Intrinsic::nvvm_sqrt_approx_f);1159 else {1160 // There's no sqrt.approx.f64 instruction, so we emit1161 // reciprocal(rsqrt(x)). This is faster than1162 // select(x == 0, 0, x * rsqrt(x)). (In fact, it's faster than plain1163 // x * rsqrt(x).)1164 return DAG.getNode(1165 ISD::INTRINSIC_WO_CHAIN, DL, VT,1166 DAG.getConstant(Intrinsic::nvvm_rcp_approx_ftz_d, DL, MVT::i32),1167 MakeIntrinsicCall(Intrinsic::nvvm_rsqrt_approx_d));1168 }1169 }1170}1171 1172std::string NVPTXTargetLowering::getPrototype(1173 const DataLayout &DL, Type *RetTy, const ArgListTy &Args,1174 const SmallVectorImpl<ISD::OutputArg> &Outs,1175 std::optional<unsigned> FirstVAArg, const CallBase &CB,1176 unsigned UniqueCallSite) const {1177 auto PtrVT = getPointerTy(DL);1178 1179 std::string Prototype;1180 raw_string_ostream O(Prototype);1181 O << "prototype_" << UniqueCallSite << " : .callprototype ";1182 1183 if (RetTy->isVoidTy()) {1184 O << "()";1185 } else {1186 O << "(";1187 if (shouldPassAsArray(RetTy)) {1188 const Align RetAlign = getArgumentAlignment(&CB, RetTy, 0, DL);1189 O << ".param .align " << RetAlign.value() << " .b8 _["1190 << DL.getTypeAllocSize(RetTy) << "]";1191 } else if (RetTy->isFloatingPointTy() || RetTy->isIntegerTy()) {1192 unsigned size = 0;1193 if (auto *ITy = dyn_cast<IntegerType>(RetTy)) {1194 size = ITy->getBitWidth();1195 } else {1196 assert(RetTy->isFloatingPointTy() &&1197 "Floating point type expected here");1198 size = RetTy->getPrimitiveSizeInBits();1199 }1200 // PTX ABI requires all scalar return values to be at least 321201 // bits in size. fp16 normally uses .b16 as its storage type in1202 // PTX, so its size must be adjusted here, too.1203 size = promoteScalarArgumentSize(size);1204 1205 O << ".param .b" << size << " _";1206 } else if (isa<PointerType>(RetTy)) {1207 O << ".param .b" << PtrVT.getSizeInBits() << " _";1208 } else {1209 llvm_unreachable("Unknown return type");1210 }1211 O << ") ";1212 }1213 O << "_ (";1214 1215 bool first = true;1216 1217 const unsigned NumArgs = FirstVAArg.value_or(Args.size());1218 auto AllOuts = ArrayRef(Outs);1219 for (const unsigned I : llvm::seq(NumArgs)) {1220 const auto ArgOuts =1221 AllOuts.take_while([I](auto O) { return O.OrigArgIndex == I; });1222 AllOuts = AllOuts.drop_front(ArgOuts.size());1223 1224 Type *Ty = Args[I].Ty;1225 if (!first) {1226 O << ", ";1227 }1228 first = false;1229 1230 if (ArgOuts[0].Flags.isByVal()) {1231 // Indirect calls need strict ABI alignment so we disable optimizations by1232 // not providing a function to optimize.1233 Type *ETy = Args[I].IndirectType;1234 Align InitialAlign = ArgOuts[0].Flags.getNonZeroByValAlign();1235 Align ParamByValAlign =1236 getFunctionByValParamAlign(/*F=*/nullptr, ETy, InitialAlign, DL);1237 1238 O << ".param .align " << ParamByValAlign.value() << " .b8 _["1239 << ArgOuts[0].Flags.getByValSize() << "]";1240 } else {1241 if (shouldPassAsArray(Ty)) {1242 Align ParamAlign =1243 getArgumentAlignment(&CB, Ty, I + AttributeList::FirstArgIndex, DL);1244 O << ".param .align " << ParamAlign.value() << " .b8 _["1245 << DL.getTypeAllocSize(Ty) << "]";1246 continue;1247 }1248 // i8 types in IR will be i16 types in SDAG1249 assert((getValueType(DL, Ty) == ArgOuts[0].VT ||1250 (getValueType(DL, Ty) == MVT::i8 && ArgOuts[0].VT == MVT::i16)) &&1251 "type mismatch between callee prototype and arguments");1252 // scalar type1253 unsigned sz = 0;1254 if (auto *ITy = dyn_cast<IntegerType>(Ty)) {1255 sz = promoteScalarArgumentSize(ITy->getBitWidth());1256 } else if (isa<PointerType>(Ty)) {1257 sz = PtrVT.getSizeInBits();1258 } else {1259 sz = Ty->getPrimitiveSizeInBits();1260 }1261 O << ".param .b" << sz << " _";1262 }1263 }1264 1265 if (FirstVAArg)1266 O << (first ? "" : ",") << " .param .align "1267 << STI.getMaxRequiredAlignment() << " .b8 _[]";1268 O << ")";1269 if (shouldEmitPTXNoReturn(&CB, *nvTM))1270 O << " .noreturn";1271 O << ";";1272 1273 return Prototype;1274}1275 1276Align NVPTXTargetLowering::getFunctionArgumentAlignment(1277 const Function *F, Type *Ty, unsigned Idx, const DataLayout &DL) const {1278 return getAlign(*F, Idx).value_or(getFunctionParamOptimizedAlign(F, Ty, DL));1279}1280 1281Align NVPTXTargetLowering::getArgumentAlignment(const CallBase *CB, Type *Ty,1282 unsigned Idx,1283 const DataLayout &DL) const {1284 if (!CB) {1285 // CallSite is zero, fallback to ABI type alignment1286 return DL.getABITypeAlign(Ty);1287 }1288 1289 const Function *DirectCallee = CB->getCalledFunction();1290 1291 if (!DirectCallee) {1292 // We don't have a direct function symbol, but that may be because of1293 // constant cast instructions in the call.1294 1295 // With bitcast'd call targets, the instruction will be the call1296 if (const auto *CI = dyn_cast<CallInst>(CB)) {1297 // Check if we have call alignment metadata1298 if (MaybeAlign StackAlign = getAlign(*CI, Idx))1299 return StackAlign.value();1300 }1301 DirectCallee = getMaybeBitcastedCallee(CB);1302 }1303 1304 // Check for function alignment information if we found that the1305 // ultimate target is a Function1306 if (DirectCallee)1307 return getFunctionArgumentAlignment(DirectCallee, Ty, Idx, DL);1308 1309 // Call is indirect, fall back to the ABI type alignment1310 return DL.getABITypeAlign(Ty);1311}1312 1313static bool shouldConvertToIndirectCall(const CallBase *CB,1314 const GlobalAddressSDNode *Func) {1315 if (!Func)1316 return false;1317 if (auto *CalleeFunc = dyn_cast<Function>(Func->getGlobal()))1318 return CB->getFunctionType() != CalleeFunc->getFunctionType();1319 return false;1320}1321 1322static MachinePointerInfo refinePtrAS(SDValue &Ptr, SelectionDAG &DAG,1323 const DataLayout &DL,1324 const TargetLowering &TL) {1325 if (Ptr->getOpcode() == ISD::FrameIndex) {1326 auto Ty = TL.getPointerTy(DL, ADDRESS_SPACE_LOCAL);1327 Ptr = DAG.getAddrSpaceCast(SDLoc(), Ty, Ptr, ADDRESS_SPACE_GENERIC,1328 ADDRESS_SPACE_LOCAL);1329 1330 return MachinePointerInfo(ADDRESS_SPACE_LOCAL);1331 }1332 1333 // Peel of an addrspacecast to generic and load directly from the specific1334 // address space.1335 if (Ptr->getOpcode() == ISD::ADDRSPACECAST) {1336 const auto *ASC = cast<AddrSpaceCastSDNode>(Ptr);1337 if (ASC->getDestAddressSpace() == ADDRESS_SPACE_GENERIC) {1338 Ptr = ASC->getOperand(0);1339 return MachinePointerInfo(ASC->getSrcAddressSpace());1340 }1341 }1342 1343 return MachinePointerInfo();1344}1345 1346static ISD::NodeType getExtOpcode(const ISD::ArgFlagsTy &Flags) {1347 if (Flags.isSExt())1348 return ISD::SIGN_EXTEND;1349 if (Flags.isZExt())1350 return ISD::ZERO_EXTEND;1351 return ISD::ANY_EXTEND;1352}1353 1354static SDValue correctParamType(SDValue V, EVT ExpectedVT,1355 ISD::ArgFlagsTy Flags, SelectionDAG &DAG,1356 SDLoc dl) {1357 const EVT ActualVT = V.getValueType();1358 assert((ActualVT == ExpectedVT ||1359 (ExpectedVT.isInteger() && ActualVT.isInteger())) &&1360 "Non-integer argument type size mismatch");1361 if (ExpectedVT.bitsGT(ActualVT))1362 return DAG.getNode(getExtOpcode(Flags), dl, ExpectedVT, V);1363 if (ExpectedVT.bitsLT(ActualVT))1364 return DAG.getNode(ISD::TRUNCATE, dl, ExpectedVT, V);1365 1366 return V;1367}1368 1369SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,1370 SmallVectorImpl<SDValue> &InVals) const {1371 1372 if (CLI.IsVarArg && (STI.getPTXVersion() < 60 || STI.getSmVersion() < 30))1373 report_fatal_error(1374 "Support for variadic functions (unsized array parameter) introduced "1375 "in PTX ISA version 6.0 and requires target sm_30.");1376 1377 SelectionDAG &DAG = CLI.DAG;1378 SDLoc dl = CLI.DL;1379 const SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;1380 SDValue Callee = CLI.Callee;1381 ArgListTy &Args = CLI.getArgs();1382 Type *RetTy = CLI.RetTy;1383 const CallBase *CB = CLI.CB;1384 const DataLayout &DL = DAG.getDataLayout();1385 LLVMContext &Ctx = *DAG.getContext();1386 1387 const auto GetI32 = [&](const unsigned I) {1388 return DAG.getConstant(I, dl, MVT::i32);1389 };1390 1391 const unsigned UniqueCallSite = GlobalUniqueCallSite++;1392 const SDValue CallChain = CLI.Chain;1393 const SDValue StartChain =1394 DAG.getCALLSEQ_START(CallChain, UniqueCallSite, 0, dl);1395 SDValue DeclareGlue = StartChain.getValue(1);1396 1397 SmallVector<SDValue, 16> CallPrereqs{StartChain};1398 1399 const auto MakeDeclareScalarParam = [&](SDValue Symbol, unsigned Size) {1400 // PTX ABI requires integral types to be at least 32 bits in size. FP16 is1401 // loaded/stored using i16, so it's handled here as well.1402 const unsigned SizeBits = promoteScalarArgumentSize(Size * 8);1403 SDValue Declare =1404 DAG.getNode(NVPTXISD::DeclareScalarParam, dl, {MVT::Other, MVT::Glue},1405 {StartChain, Symbol, GetI32(SizeBits), DeclareGlue});1406 CallPrereqs.push_back(Declare);1407 DeclareGlue = Declare.getValue(1);1408 return Declare;1409 };1410 1411 const auto MakeDeclareArrayParam = [&](SDValue Symbol, Align Align,1412 unsigned Size) {1413 SDValue Declare = DAG.getNode(1414 NVPTXISD::DeclareArrayParam, dl, {MVT::Other, MVT::Glue},1415 {StartChain, Symbol, GetI32(Align.value()), GetI32(Size), DeclareGlue});1416 CallPrereqs.push_back(Declare);1417 DeclareGlue = Declare.getValue(1);1418 return Declare;1419 };1420 1421 // Variadic arguments.1422 //1423 // Normally, for each argument, we declare a param scalar or a param1424 // byte array in the .param space, and store the argument value to that1425 // param scalar or array starting at offset 0.1426 //1427 // In the case of the first variadic argument, we declare a vararg byte array1428 // with size 0. The exact size of this array isn't known at this point, so1429 // it'll be patched later. All the variadic arguments will be stored to this1430 // array at a certain offset (which gets tracked by 'VAOffset'). The offset is1431 // initially set to 0, so it can be used for non-variadic arguments (which use1432 // 0 offset) to simplify the code.1433 //1434 // After all vararg is processed, 'VAOffset' holds the size of the1435 // vararg byte array.1436 assert((CLI.IsVarArg || CLI.Args.size() == CLI.NumFixedArgs) &&1437 "Non-VarArg function with extra arguments");1438 1439 const unsigned FirstVAArg = CLI.NumFixedArgs; // position of first variadic1440 unsigned VAOffset = 0; // current offset in the param array1441 1442 const SDValue VADeclareParam =1443 CLI.Args.size() > FirstVAArg1444 ? MakeDeclareArrayParam(getCallParamSymbol(DAG, FirstVAArg, MVT::i32),1445 Align(STI.getMaxRequiredAlignment()), 0)1446 : SDValue();1447 1448 // Args.size() and Outs.size() need not match.1449 // Outs.size() will be larger1450 // * if there is an aggregate argument with multiple fields (each field1451 // showing up separately in Outs)1452 // * if there is a vector argument with more than typical vector-length1453 // elements (generally if more than 4) where each vector element is1454 // individually present in Outs.1455 // So a different index should be used for indexing into Outs/OutVals.1456 // See similar issue in LowerFormalArguments.1457 auto AllOuts = ArrayRef(CLI.Outs);1458 auto AllOutVals = ArrayRef(CLI.OutVals);1459 assert(AllOuts.size() == AllOutVals.size() &&1460 "Outs and OutVals must be the same size");1461 // Declare the .params or .reg need to pass values1462 // to the function1463 for (const auto E : llvm::enumerate(Args)) {1464 const auto ArgI = E.index();1465 const auto Arg = E.value();1466 const auto ArgOuts =1467 AllOuts.take_while([&](auto O) { return O.OrigArgIndex == ArgI; });1468 const auto ArgOutVals = AllOutVals.take_front(ArgOuts.size());1469 AllOuts = AllOuts.drop_front(ArgOuts.size());1470 AllOutVals = AllOutVals.drop_front(ArgOuts.size());1471 1472 const bool IsVAArg = (ArgI >= FirstVAArg);1473 const bool IsByVal = Arg.IsByVal;1474 1475 const SDValue ParamSymbol =1476 getCallParamSymbol(DAG, IsVAArg ? FirstVAArg : ArgI, MVT::i32);1477 1478 assert((!IsByVal || Arg.IndirectType) &&1479 "byval arg must have indirect type");1480 Type *ETy = (IsByVal ? Arg.IndirectType : Arg.Ty);1481 1482 const Align ArgAlign = [&]() {1483 if (IsByVal) {1484 // The ByValAlign in the Outs[OIdx].Flags is always set at this point,1485 // so we don't need to worry whether it's naturally aligned or not.1486 // See TargetLowering::LowerCallTo().1487 const Align InitialAlign = ArgOuts[0].Flags.getNonZeroByValAlign();1488 return getFunctionByValParamAlign(CB->getCalledFunction(), ETy,1489 InitialAlign, DL);1490 }1491 return getArgumentAlignment(CB, Arg.Ty, ArgI + 1, DL);1492 }();1493 1494 const unsigned TySize = DL.getTypeAllocSize(ETy);1495 assert((!IsByVal || TySize == ArgOuts[0].Flags.getByValSize()) &&1496 "type size mismatch");1497 1498 const SDValue ArgDeclare = [&]() {1499 if (IsVAArg)1500 return VADeclareParam;1501 1502 if (IsByVal || shouldPassAsArray(Arg.Ty))1503 return MakeDeclareArrayParam(ParamSymbol, ArgAlign, TySize);1504 1505 assert(ArgOuts.size() == 1 && "We must pass only one value as non-array");1506 assert((ArgOuts[0].VT.isInteger() || ArgOuts[0].VT.isFloatingPoint()) &&1507 "Only int and float types are supported as non-array arguments");1508 1509 return MakeDeclareScalarParam(ParamSymbol, TySize);1510 }();1511 1512 if (IsByVal) {1513 assert(ArgOutVals.size() == 1 && "We must pass only one value as byval");1514 SDValue SrcPtr = ArgOutVals[0];1515 const auto PointerInfo = refinePtrAS(SrcPtr, DAG, DL, *this);1516 const Align BaseSrcAlign = ArgOuts[0].Flags.getNonZeroByValAlign();1517 1518 if (IsVAArg)1519 VAOffset = alignTo(VAOffset, ArgAlign);1520 1521 SmallVector<EVT, 4> ValueVTs, MemVTs;1522 SmallVector<TypeSize, 4> Offsets;1523 ComputeValueVTs(*this, DL, ETy, ValueVTs, &MemVTs, &Offsets);1524 1525 unsigned J = 0;1526 const auto VI = VectorizePTXValueVTs(MemVTs, Offsets, ArgAlign, IsVAArg);1527 for (const unsigned NumElts : VI) {1528 EVT LoadVT = getVectorizedVT(MemVTs[J], NumElts, Ctx);1529 Align SrcAlign = commonAlignment(BaseSrcAlign, Offsets[J]);1530 SDValue SrcAddr = DAG.getObjectPtrOffset(dl, SrcPtr, Offsets[J]);1531 SDValue SrcLoad =1532 DAG.getLoad(LoadVT, dl, CallChain, SrcAddr, PointerInfo, SrcAlign);1533 1534 TypeSize ParamOffset = Offsets[J].getWithIncrement(VAOffset);1535 Align ParamAlign = commonAlignment(ArgAlign, ParamOffset);1536 SDValue ParamAddr =1537 DAG.getObjectPtrOffset(dl, ParamSymbol, ParamOffset);1538 SDValue StoreParam =1539 DAG.getStore(ArgDeclare, dl, SrcLoad, ParamAddr,1540 MachinePointerInfo(ADDRESS_SPACE_PARAM), ParamAlign);1541 CallPrereqs.push_back(StoreParam);1542 1543 J += NumElts;1544 }1545 if (IsVAArg)1546 VAOffset += TySize;1547 } else {1548 SmallVector<EVT, 16> VTs;1549 SmallVector<uint64_t, 16> Offsets;1550 ComputePTXValueVTs(*this, DL, Ctx, CLI.CallConv, Arg.Ty, VTs, Offsets,1551 VAOffset);1552 assert(VTs.size() == Offsets.size() && "Size mismatch");1553 assert(VTs.size() == ArgOuts.size() && "Size mismatch");1554 1555 // PTX Interoperability Guide 3.3(A): [Integer] Values shorter1556 // than 32-bits are sign extended or zero extended, depending on1557 // whether they are signed or unsigned types. This case applies1558 // only to scalar parameters and not to aggregate values.1559 const bool ExtendIntegerParam =1560 Arg.Ty->isIntegerTy() && DL.getTypeAllocSizeInBits(Arg.Ty) < 32;1561 1562 const auto GetStoredValue = [&](const unsigned I) {1563 SDValue StVal = ArgOutVals[I];1564 assert(promoteScalarIntegerPTX(StVal.getValueType()) ==1565 StVal.getValueType() &&1566 "OutVal type should always be legal");1567 1568 const EVT VTI = promoteScalarIntegerPTX(VTs[I]);1569 const EVT StoreVT =1570 ExtendIntegerParam ? MVT::i32 : (VTI == MVT::i1 ? MVT::i8 : VTI);1571 1572 return correctParamType(StVal, StoreVT, ArgOuts[I].Flags, DAG, dl);1573 };1574 1575 unsigned J = 0;1576 const auto VI = VectorizePTXValueVTs(VTs, Offsets, ArgAlign, IsVAArg);1577 for (const unsigned NumElts : VI) {1578 const EVT EltVT = promoteScalarIntegerPTX(VTs[J]);1579 1580 unsigned Offset;1581 if (IsVAArg) {1582 // TODO: We may need to support vector types that can be passed1583 // as scalars in variadic arguments.1584 assert(NumElts == 1 &&1585 "Vectorization should be disabled for vaargs.");1586 1587 // Align each part of the variadic argument to their type.1588 VAOffset = alignTo(VAOffset, DAG.getEVTAlign(EltVT));1589 Offset = VAOffset;1590 1591 const EVT TheStoreType = ExtendIntegerParam ? MVT::i32 : EltVT;1592 VAOffset += DL.getTypeAllocSize(TheStoreType.getTypeForEVT(Ctx));1593 } else {1594 assert(VAOffset == 0 && "VAOffset must be 0 for non-VA args");1595 Offset = Offsets[J];1596 }1597 1598 SDValue Ptr =1599 DAG.getObjectPtrOffset(dl, ParamSymbol, TypeSize::getFixed(Offset));1600 1601 const MaybeAlign CurrentAlign = ExtendIntegerParam1602 ? MaybeAlign(std::nullopt)1603 : commonAlignment(ArgAlign, Offset);1604 1605 SDValue Val =1606 getBuildVectorizedValue(NumElts, dl, DAG, [&](unsigned K) {1607 return GetStoredValue(J + K);1608 });1609 1610 SDValue StoreParam =1611 DAG.getStore(ArgDeclare, dl, Val, Ptr,1612 MachinePointerInfo(ADDRESS_SPACE_PARAM), CurrentAlign);1613 CallPrereqs.push_back(StoreParam);1614 1615 J += NumElts;1616 }1617 }1618 }1619 1620 // Handle Result1621 if (!Ins.empty()) {1622 const SDValue RetSymbol = DAG.getExternalSymbol("retval0", MVT::i32);1623 const unsigned ResultSize = DL.getTypeAllocSize(RetTy);1624 if (shouldPassAsArray(RetTy)) {1625 const Align RetAlign = getArgumentAlignment(CB, RetTy, 0, DL);1626 MakeDeclareArrayParam(RetSymbol, RetAlign, ResultSize);1627 } else {1628 MakeDeclareScalarParam(RetSymbol, ResultSize);1629 }1630 }1631 1632 // Set the size of the vararg param byte array if the callee is a variadic1633 // function and the variadic part is not empty.1634 if (VADeclareParam) {1635 SDValue DeclareParamOps[] = {VADeclareParam.getOperand(0),1636 VADeclareParam.getOperand(1),1637 VADeclareParam.getOperand(2), GetI32(VAOffset),1638 VADeclareParam.getOperand(4)};1639 DAG.MorphNodeTo(VADeclareParam.getNode(), VADeclareParam.getOpcode(),1640 VADeclareParam->getVTList(), DeclareParamOps);1641 }1642 1643 const auto *Func = dyn_cast<GlobalAddressSDNode>(Callee.getNode());1644 // If the type of the callsite does not match that of the function, convert1645 // the callsite to an indirect call.1646 const bool ConvertToIndirectCall = shouldConvertToIndirectCall(CB, Func);1647 1648 // Both indirect calls and libcalls have nullptr Func. In order to distinguish1649 // between them we must rely on the call site value which is valid for1650 // indirect calls but is always null for libcalls.1651 const bool IsIndirectCall = (!Func && CB) || ConvertToIndirectCall;1652 1653 if (isa<ExternalSymbolSDNode>(Callee)) {1654 Function* CalleeFunc = nullptr;1655 1656 // Try to find the callee in the current module.1657 Callee = DAG.getSymbolFunctionGlobalAddress(Callee, &CalleeFunc);1658 assert(CalleeFunc != nullptr && "Libcall callee must be set.");1659 1660 // Set the "libcall callee" attribute to indicate that the function1661 // must always have a declaration.1662 CalleeFunc->addFnAttr("nvptx-libcall-callee", "true");1663 }1664 1665 if (IsIndirectCall) {1666 // This is indirect function call case : PTX requires a prototype of the1667 // form1668 // proto_0 : .callprototype(.param .b32 _) _ (.param .b32 _);1669 // to be emitted, and the label has to used as the last arg of call1670 // instruction.1671 // The prototype is embedded in a string and put as the operand for a1672 // CallPrototype SDNode which will print out to the value of the string.1673 const bool HasVAArgs = CLI.IsVarArg && (CLI.Args.size() > CLI.NumFixedArgs);1674 std::string Proto =1675 getPrototype(DL, RetTy, Args, CLI.Outs,1676 HasVAArgs ? std::optional(FirstVAArg) : std::nullopt, *CB,1677 UniqueCallSite);1678 const char *ProtoStr = nvTM->getStrPool().save(Proto).data();1679 const SDValue PrototypeDeclare = DAG.getNode(1680 NVPTXISD::CallPrototype, dl, MVT::Other,1681 {StartChain, DAG.getTargetExternalSymbol(ProtoStr, MVT::i32)});1682 CallPrereqs.push_back(PrototypeDeclare);1683 }1684 1685 const unsigned Proto = IsIndirectCall ? UniqueCallSite : 0;1686 const unsigned NumArgs =1687 std::min<unsigned>(CLI.NumFixedArgs + 1, Args.size());1688 /// CALL(Chain, IsConvergent, IsIndirectCall/IsUniform, NumReturns,1689 /// NumParams, Callee, Proto)1690 const SDValue CallToken = DAG.getTokenFactor(dl, CallPrereqs);1691 const SDValue Call = DAG.getNode(1692 NVPTXISD::CALL, dl, MVT::Other,1693 {CallToken, GetI32(CLI.IsConvergent), GetI32(IsIndirectCall),1694 GetI32(Ins.empty() ? 0 : 1), GetI32(NumArgs), Callee, GetI32(Proto)});1695 1696 SmallVector<SDValue, 16> LoadChains{Call};1697 SmallVector<SDValue, 16> ProxyRegOps;1698 if (!Ins.empty()) {1699 SmallVector<EVT, 16> VTs;1700 SmallVector<uint64_t, 16> Offsets;1701 ComputePTXValueVTs(*this, DL, Ctx, CLI.CallConv, RetTy, VTs, Offsets);1702 assert(VTs.size() == Ins.size() && "Bad value decomposition");1703 1704 const Align RetAlign = getArgumentAlignment(CB, RetTy, 0, DL);1705 const SDValue RetSymbol = DAG.getExternalSymbol("retval0", MVT::i32);1706 1707 // PTX Interoperability Guide 3.3(A): [Integer] Values shorter than1708 // 32-bits are sign extended or zero extended, depending on whether1709 // they are signed or unsigned types.1710 const bool ExtendIntegerRetVal =1711 RetTy->isIntegerTy() && DL.getTypeAllocSizeInBits(RetTy) < 32;1712 1713 unsigned I = 0;1714 const auto VI = VectorizePTXValueVTs(VTs, Offsets, RetAlign);1715 for (const unsigned NumElts : VI) {1716 const MaybeAlign CurrentAlign =1717 ExtendIntegerRetVal ? MaybeAlign(std::nullopt)1718 : commonAlignment(RetAlign, Offsets[I]);1719 1720 const EVT VTI = promoteScalarIntegerPTX(VTs[I]);1721 const EVT LoadVT =1722 ExtendIntegerRetVal ? MVT::i32 : (VTI == MVT::i1 ? MVT::i8 : VTI);1723 const EVT VecVT = getVectorizedVT(LoadVT, NumElts, Ctx);1724 SDValue Ptr =1725 DAG.getObjectPtrOffset(dl, RetSymbol, TypeSize::getFixed(Offsets[I]));1726 1727 SDValue R =1728 DAG.getLoad(VecVT, dl, Call, Ptr,1729 MachinePointerInfo(ADDRESS_SPACE_PARAM), CurrentAlign);1730 1731 LoadChains.push_back(R.getValue(1));1732 for (const unsigned J : llvm::seq(NumElts))1733 ProxyRegOps.push_back(getExtractVectorizedValue(R, J, LoadVT, dl, DAG));1734 I += NumElts;1735 }1736 }1737 1738 const SDValue EndToken = DAG.getTokenFactor(dl, LoadChains);1739 const SDValue CallEnd = DAG.getCALLSEQ_END(EndToken, UniqueCallSite,1740 UniqueCallSite + 1, SDValue(), dl);1741 1742 // Append ProxyReg instructions to the chain to make sure that `callseq_end`1743 // will not get lost. Otherwise, during libcalls expansion, the nodes can become1744 // dangling.1745 for (const auto [I, Reg] : llvm::enumerate(ProxyRegOps)) {1746 SDValue Proxy =1747 DAG.getNode(NVPTXISD::ProxyReg, dl, Reg.getValueType(), {CallEnd, Reg});1748 SDValue Ret = correctParamType(Proxy, Ins[I].VT, Ins[I].Flags, DAG, dl);1749 InVals.push_back(Ret);1750 }1751 1752 // set IsTailCall to false for now, until we figure out how to express1753 // tail call optimization in PTX1754 CLI.IsTailCall = false;1755 return CallEnd;1756}1757 1758SDValue NVPTXTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,1759 SelectionDAG &DAG) const {1760 1761 if (STI.getPTXVersion() < 73 || STI.getSmVersion() < 52) {1762 const Function &Fn = DAG.getMachineFunction().getFunction();1763 1764 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(1765 Fn,1766 "Support for dynamic alloca introduced in PTX ISA version 7.3 and "1767 "requires target sm_52.",1768 SDLoc(Op).getDebugLoc()));1769 auto Ops = {DAG.getConstant(0, SDLoc(), Op.getValueType()),1770 Op.getOperand(0)};1771 return DAG.getMergeValues(Ops, SDLoc());1772 }1773 1774 SDLoc DL(Op.getNode());1775 SDValue Chain = Op.getOperand(0);1776 SDValue Size = Op.getOperand(1);1777 uint64_t Align = Op.getConstantOperandVal(2);1778 1779 // The alignment on a ISD::DYNAMIC_STACKALLOC node may be 0 to indicate that1780 // the default stack alignment should be used.1781 if (Align == 0)1782 Align = DAG.getSubtarget().getFrameLowering()->getStackAlign().value();1783 1784 // The size for ptx alloca instruction is 64-bit for m64 and 32-bit for m32.1785 const MVT LocalVT = getPointerTy(DAG.getDataLayout(), ADDRESS_SPACE_LOCAL);1786 1787 SDValue Alloc =1788 DAG.getNode(NVPTXISD::DYNAMIC_STACKALLOC, DL, {LocalVT, MVT::Other},1789 {Chain, DAG.getZExtOrTrunc(Size, DL, LocalVT),1790 DAG.getTargetConstant(Align, DL, MVT::i32)});1791 1792 SDValue ASC = DAG.getAddrSpaceCast(1793 DL, Op.getValueType(), Alloc, ADDRESS_SPACE_LOCAL, ADDRESS_SPACE_GENERIC);1794 1795 return DAG.getMergeValues({ASC, SDValue(Alloc.getNode(), 1)}, DL);1796}1797 1798SDValue NVPTXTargetLowering::LowerSTACKRESTORE(SDValue Op,1799 SelectionDAG &DAG) const {1800 SDLoc DL(Op.getNode());1801 if (STI.getPTXVersion() < 73 || STI.getSmVersion() < 52) {1802 const Function &Fn = DAG.getMachineFunction().getFunction();1803 1804 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(1805 Fn,1806 "Support for stackrestore requires PTX ISA version >= 7.3 and target "1807 ">= sm_52.",1808 DL.getDebugLoc()));1809 return Op.getOperand(0);1810 }1811 1812 const MVT LocalVT = getPointerTy(DAG.getDataLayout(), ADDRESS_SPACE_LOCAL);1813 SDValue Chain = Op.getOperand(0);1814 SDValue Ptr = Op.getOperand(1);1815 SDValue ASC = DAG.getAddrSpaceCast(DL, LocalVT, Ptr, ADDRESS_SPACE_GENERIC,1816 ADDRESS_SPACE_LOCAL);1817 return DAG.getNode(NVPTXISD::STACKRESTORE, DL, MVT::Other, {Chain, ASC});1818}1819 1820SDValue NVPTXTargetLowering::LowerSTACKSAVE(SDValue Op,1821 SelectionDAG &DAG) const {1822 SDLoc DL(Op.getNode());1823 if (STI.getPTXVersion() < 73 || STI.getSmVersion() < 52) {1824 const Function &Fn = DAG.getMachineFunction().getFunction();1825 1826 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(1827 Fn,1828 "Support for stacksave requires PTX ISA version >= 7.3 and target >= "1829 "sm_52.",1830 DL.getDebugLoc()));1831 auto Ops = {DAG.getConstant(0, DL, Op.getValueType()), Op.getOperand(0)};1832 return DAG.getMergeValues(Ops, DL);1833 }1834 1835 const MVT LocalVT = getPointerTy(DAG.getDataLayout(), ADDRESS_SPACE_LOCAL);1836 SDValue Chain = Op.getOperand(0);1837 SDValue SS =1838 DAG.getNode(NVPTXISD::STACKSAVE, DL, {LocalVT, MVT::Other}, Chain);1839 SDValue ASC = DAG.getAddrSpaceCast(1840 DL, Op.getValueType(), SS, ADDRESS_SPACE_LOCAL, ADDRESS_SPACE_GENERIC);1841 return DAG.getMergeValues({ASC, SDValue(SS.getNode(), 1)}, DL);1842}1843 1844// By default CONCAT_VECTORS is lowered by ExpandVectorBuildThroughStack()1845// (see LegalizeDAG.cpp). This is slow and uses local memory.1846// We use extract/insert/build vector just as what LegalizeOp() does in llvm 2.51847SDValue1848NVPTXTargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const {1849 SDNode *Node = Op.getNode();1850 SDLoc dl(Node);1851 SmallVector<SDValue, 8> Ops;1852 unsigned NumOperands = Node->getNumOperands();1853 for (unsigned i = 0; i < NumOperands; ++i) {1854 SDValue SubOp = Node->getOperand(i);1855 EVT VVT = SubOp.getNode()->getValueType(0);1856 EVT EltVT = VVT.getVectorElementType();1857 unsigned NumSubElem = VVT.getVectorNumElements();1858 for (unsigned j = 0; j < NumSubElem; ++j) {1859 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, SubOp,1860 DAG.getIntPtrConstant(j, dl)));1861 }1862 }1863 return DAG.getBuildVector(Node->getValueType(0), dl, Ops);1864}1865 1866static SDValue getPRMT(SDValue A, SDValue B, SDValue Selector, SDLoc DL,1867 SelectionDAG &DAG,1868 unsigned Mode = NVPTX::PTXPrmtMode::NONE) {1869 assert(A.getValueType() == MVT::i32 && B.getValueType() == MVT::i32 &&1870 Selector.getValueType() == MVT::i32 && "PRMT must have i32 operands");1871 return DAG.getNode(NVPTXISD::PRMT, DL, MVT::i32,1872 {A, B, Selector, DAG.getConstant(Mode, DL, MVT::i32)});1873}1874 1875static SDValue getPRMT(SDValue A, SDValue B, uint64_t Selector, SDLoc DL,1876 SelectionDAG &DAG,1877 unsigned Mode = NVPTX::PTXPrmtMode::NONE) {1878 return getPRMT(A, B, DAG.getConstant(Selector, DL, MVT::i32), DL, DAG, Mode);1879}1880 1881/// Reduces the elements using the scalar operations provided. The operations1882/// are sorted descending in number of inputs they take. The flags on the1883/// original reduction operation will be propagated to each scalar operation.1884/// Nearby elements are grouped in tree reduction, unlike the shuffle reduction1885/// used in ExpandReductions and SelectionDAG.1886static SDValue buildTreeReduction(1887 const SmallVector<SDValue> &Elements, EVT EltTy,1888 ArrayRef<std::pair<unsigned /*NodeType*/, unsigned /*NumInputs*/>> Ops,1889 const SDLoc &DL, const SDNodeFlags Flags, SelectionDAG &DAG) {1890 // Build the reduction tree at each level, starting with all the elements.1891 SmallVector<SDValue> Level = Elements;1892 1893 unsigned OpIdx = 0;1894 while (Level.size() > 1) {1895 // Try to reduce this level using the current operator.1896 const auto [Op, NumInputs] = Ops[OpIdx];1897 1898 // Build the next level by partially reducing all elements.1899 SmallVector<SDValue> ReducedLevel;1900 unsigned I = 0, E = Level.size();1901 for (; I + NumInputs <= E; I += NumInputs) {1902 // Reduce elements in groups of [NumInputs], as much as possible.1903 ReducedLevel.push_back(DAG.getNode(1904 Op, DL, EltTy, ArrayRef<SDValue>(Level).slice(I, NumInputs), Flags));1905 }1906 1907 if (I < E) {1908 // Handle leftover elements.1909 1910 if (ReducedLevel.empty()) {1911 // We didn't reduce anything at this level. We need to pick a smaller1912 // operator.1913 ++OpIdx;1914 assert(OpIdx < Ops.size() && "no smaller operators for reduction");1915 continue;1916 }1917 1918 // We reduced some things but there's still more left, meaning the1919 // operator's number of inputs doesn't evenly divide this level size. Move1920 // these elements to the next level.1921 for (; I < E; ++I)1922 ReducedLevel.push_back(Level[I]);1923 }1924 1925 // Process the next level.1926 Level = ReducedLevel;1927 }1928 1929 return *Level.begin();1930}1931 1932// Get scalar reduction opcode1933static ISD::NodeType getScalarOpcodeForReduction(unsigned ReductionOpcode) {1934 switch (ReductionOpcode) {1935 case ISD::VECREDUCE_FMAX:1936 return ISD::FMAXNUM;1937 case ISD::VECREDUCE_FMIN:1938 return ISD::FMINNUM;1939 case ISD::VECREDUCE_FMAXIMUM:1940 return ISD::FMAXIMUM;1941 case ISD::VECREDUCE_FMINIMUM:1942 return ISD::FMINIMUM;1943 default:1944 llvm_unreachable("unhandled reduction opcode");1945 }1946}1947 1948/// Get 3-input scalar reduction opcode1949static std::optional<unsigned>1950getScalar3OpcodeForReduction(unsigned ReductionOpcode) {1951 switch (ReductionOpcode) {1952 case ISD::VECREDUCE_FMAX:1953 return NVPTXISD::FMAXNUM3;1954 case ISD::VECREDUCE_FMIN:1955 return NVPTXISD::FMINNUM3;1956 case ISD::VECREDUCE_FMAXIMUM:1957 return NVPTXISD::FMAXIMUM3;1958 case ISD::VECREDUCE_FMINIMUM:1959 return NVPTXISD::FMINIMUM3;1960 default:1961 return std::nullopt;1962 }1963}1964 1965/// Lower reductions to either a sequence of operations or a tree if1966/// reassociations are allowed. This method will use larger operations like1967/// max3/min3 when the target supports them.1968SDValue NVPTXTargetLowering::LowerVECREDUCE(SDValue Op,1969 SelectionDAG &DAG) const {1970 SDLoc DL(Op);1971 const SDNodeFlags Flags = Op->getFlags();1972 SDValue Vector = Op.getOperand(0);1973 1974 const unsigned Opcode = Op->getOpcode();1975 const EVT EltTy = Vector.getValueType().getVectorElementType();1976 1977 // Whether we can use 3-input min/max when expanding the reduction.1978 const bool CanUseMinMax3 =1979 EltTy == MVT::f32 && STI.getSmVersion() >= 100 &&1980 STI.getPTXVersion() >= 88 &&1981 (Opcode == ISD::VECREDUCE_FMAX || Opcode == ISD::VECREDUCE_FMIN ||1982 Opcode == ISD::VECREDUCE_FMAXIMUM || Opcode == ISD::VECREDUCE_FMINIMUM);1983 1984 // A list of SDNode opcodes with equivalent semantics, sorted descending by1985 // number of inputs they take.1986 SmallVector<std::pair<unsigned /*Op*/, unsigned /*NumIn*/>, 2> ScalarOps;1987 1988 if (auto Opcode3Elem = getScalar3OpcodeForReduction(Opcode);1989 CanUseMinMax3 && Opcode3Elem)1990 ScalarOps.push_back({*Opcode3Elem, 3});1991 ScalarOps.push_back({getScalarOpcodeForReduction(Opcode), 2});1992 1993 SmallVector<SDValue> Elements;1994 DAG.ExtractVectorElements(Vector, Elements);1995 1996 return buildTreeReduction(Elements, EltTy, ScalarOps, DL, Flags, DAG);1997}1998 1999SDValue NVPTXTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {2000 // Handle bitcasting from v2i8 without hitting the default promotion2001 // strategy which goes through stack memory.2002 EVT FromVT = Op->getOperand(0)->getValueType(0);2003 if (FromVT != MVT::v2i8) {2004 return Op;2005 }2006 2007 // Pack vector elements into i16 and bitcast to final type2008 SDLoc DL(Op);2009 SDValue Vec0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i8,2010 Op->getOperand(0), DAG.getIntPtrConstant(0, DL));2011 SDValue Vec1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i8,2012 Op->getOperand(0), DAG.getIntPtrConstant(1, DL));2013 SDValue Extend0 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i16, Vec0);2014 SDValue Extend1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i16, Vec1);2015 SDValue Const8 = DAG.getConstant(8, DL, MVT::i16);2016 SDValue AsInt = DAG.getNode(2017 ISD::OR, DL, MVT::i16,2018 {Extend0, DAG.getNode(ISD::SHL, DL, MVT::i16, {Extend1, Const8})});2019 EVT ToVT = Op->getValueType(0);2020 return DAG.getBitcast(ToVT, AsInt);2021}2022 2023// We can init constant f16x2/v2i16/v4i8 with a single .b32 move. Normally it2024// would get lowered as two constant loads and vector-packing move.2025// Instead we want just a constant move:2026// mov.b32 %r2, 0x40003C002027SDValue NVPTXTargetLowering::LowerBUILD_VECTOR(SDValue Op,2028 SelectionDAG &DAG) const {2029 EVT VT = Op->getValueType(0);2030 if (!(NVPTX::isPackedVectorTy(VT) && VT.is32BitVector()))2031 return Op;2032 SDLoc DL(Op);2033 2034 if (!llvm::all_of(Op->ops(), [](SDValue Operand) {2035 return Operand->isUndef() || isa<ConstantSDNode>(Operand) ||2036 isa<ConstantFPSDNode>(Operand);2037 })) {2038 if (VT != MVT::v4i8)2039 return Op;2040 // Lower non-const v4i8 vector as byte-wise constructed i32, which allows us2041 // to optimize calculation of constant parts.2042 auto GetPRMT = [&](const SDValue Left, const SDValue Right, bool Cast,2043 uint64_t SelectionValue) -> SDValue {2044 SDValue L = Left;2045 SDValue R = Right;2046 if (Cast) {2047 L = DAG.getAnyExtOrTrunc(L, DL, MVT::i32);2048 R = DAG.getAnyExtOrTrunc(R, DL, MVT::i32);2049 }2050 return getPRMT(L, R, SelectionValue, DL, DAG);2051 };2052 auto PRMT__10 = GetPRMT(Op->getOperand(0), Op->getOperand(1), true, 0x3340);2053 auto PRMT__32 = GetPRMT(Op->getOperand(2), Op->getOperand(3), true, 0x3340);2054 auto PRMT3210 = GetPRMT(PRMT__10, PRMT__32, false, 0x5410);2055 return DAG.getBitcast(VT, PRMT3210);2056 }2057 2058 // Get value or the Nth operand as an APInt(32). Undef values treated as 0.2059 auto GetOperand = [](SDValue Op, int N) -> APInt {2060 const SDValue &Operand = Op->getOperand(N);2061 EVT VT = Op->getValueType(0);2062 if (Operand->isUndef())2063 return APInt(32, 0);2064 APInt Value;2065 if (VT == MVT::v2f16 || VT == MVT::v2bf16)2066 Value = cast<ConstantFPSDNode>(Operand)->getValueAPF().bitcastToAPInt();2067 else if (VT == MVT::v2i16 || VT == MVT::v4i8)2068 Value = Operand->getAsAPIntVal();2069 else2070 llvm_unreachable("Unsupported type");2071 // i8 values are carried around as i16, so we need to zero out upper bits,2072 // so they do not get in the way of combining individual byte values2073 if (VT == MVT::v4i8)2074 Value = Value.trunc(8);2075 return Value.zext(32);2076 };2077 2078 // Construct a 32-bit constant by shifting into place smaller values2079 // (elements of the vector type VT).2080 // For example, if VT has 2 elements, then N == 2:2081 // ShiftAmount = 32 / N = 162082 // Value |= Op0 (b16) << 02083 // Value |= Op1 (b16) << 162084 // If N == 4:2085 // ShiftAmount = 32 / N = 82086 // Value |= Op0 (b8) << 02087 // Value |= Op1 (b8) << 82088 // Value |= Op2 (b8) << 162089 // Value |= Op3 (b8) << 242090 // ...etc2091 APInt Value(32, 0);2092 const unsigned NumElements = VT.getVectorNumElements();2093 assert(32 % NumElements == 0 && "must evenly divide bit length");2094 const unsigned ShiftAmount = 32 / NumElements;2095 for (unsigned ElementNo : seq(NumElements))2096 Value |= GetOperand(Op, ElementNo).shl(ElementNo * ShiftAmount);2097 SDValue Const = DAG.getConstant(Value, DL, MVT::i32);2098 return DAG.getNode(ISD::BITCAST, DL, Op->getValueType(0), Const);2099}2100 2101SDValue NVPTXTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,2102 SelectionDAG &DAG) const {2103 SDValue Index = Op->getOperand(1);2104 SDValue Vector = Op->getOperand(0);2105 SDLoc DL(Op);2106 EVT VectorVT = Vector.getValueType();2107 2108 if (VectorVT == MVT::v4i8) {2109 SDValue Selector = DAG.getNode(ISD::OR, DL, MVT::i32,2110 DAG.getZExtOrTrunc(Index, DL, MVT::i32),2111 DAG.getConstant(0x7770, DL, MVT::i32));2112 SDValue PRMT = getPRMT(DAG.getBitcast(MVT::i32, Vector),2113 DAG.getConstant(0, DL, MVT::i32), Selector, DL, DAG);2114 SDValue Ext = DAG.getAnyExtOrTrunc(PRMT, DL, Op->getValueType(0));2115 SDNodeFlags Flags;2116 Flags.setNoSignedWrap(Ext.getScalarValueSizeInBits() > 8);2117 Flags.setNoUnsignedWrap(Ext.getScalarValueSizeInBits() >= 8);2118 Ext->setFlags(Flags);2119 return Ext;2120 }2121 2122 // Constant index will be matched by tablegen.2123 if (isa<ConstantSDNode>(Index.getNode()))2124 return Op;2125 2126 // Extract individual elements and select one of them.2127 assert(NVPTX::isPackedVectorTy(VectorVT) &&2128 VectorVT.getVectorNumElements() == 2 && "Unexpected vector type.");2129 EVT EltVT = VectorVT.getVectorElementType();2130 2131 SDLoc dl(Op.getNode());2132 SDValue E0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Vector,2133 DAG.getIntPtrConstant(0, dl));2134 SDValue E1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Vector,2135 DAG.getIntPtrConstant(1, dl));2136 return DAG.getSelectCC(dl, Index, DAG.getIntPtrConstant(0, dl), E0, E1,2137 ISD::CondCode::SETEQ);2138}2139 2140SDValue NVPTXTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,2141 SelectionDAG &DAG) const {2142 SDValue Vector = Op->getOperand(0);2143 EVT VectorVT = Vector.getValueType();2144 2145 if (VectorVT != MVT::v4i8)2146 return Op;2147 SDLoc DL(Op);2148 SDValue Value = Op->getOperand(1);2149 if (Value->isUndef())2150 return Vector;2151 2152 SDValue Index = Op->getOperand(2);2153 2154 SDValue BFI =2155 DAG.getNode(NVPTXISD::BFI, DL, MVT::i32,2156 {DAG.getZExtOrTrunc(Value, DL, MVT::i32), Vector,2157 DAG.getNode(ISD::MUL, DL, MVT::i32,2158 DAG.getZExtOrTrunc(Index, DL, MVT::i32),2159 DAG.getConstant(8, DL, MVT::i32)),2160 DAG.getConstant(8, DL, MVT::i32)});2161 return DAG.getNode(ISD::BITCAST, DL, Op->getValueType(0), BFI);2162}2163 2164SDValue NVPTXTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,2165 SelectionDAG &DAG) const {2166 SDValue V1 = Op.getOperand(0);2167 EVT VectorVT = V1.getValueType();2168 if (VectorVT != MVT::v4i8 || Op.getValueType() != MVT::v4i8)2169 return Op;2170 2171 // Lower shuffle to PRMT instruction.2172 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());2173 SDValue V2 = Op.getOperand(1);2174 uint32_t Selector = 0;2175 for (auto I : llvm::enumerate(SVN->getMask())) {2176 if (I.value() != -1) // -1 is a placeholder for undef.2177 Selector |= (I.value() << (I.index() * 4));2178 }2179 2180 SDLoc DL(Op);2181 SDValue PRMT = getPRMT(DAG.getBitcast(MVT::i32, V1),2182 DAG.getBitcast(MVT::i32, V2), Selector, DL, DAG);2183 return DAG.getBitcast(Op.getValueType(), PRMT);2184}2185/// LowerShiftRightParts - Lower SRL_PARTS, SRA_PARTS, which2186/// 1) returns two i32 values and take a 2 x i32 value to shift plus a shift2187/// amount, or2188/// 2) returns two i64 values and take a 2 x i64 value to shift plus a shift2189/// amount.2190SDValue NVPTXTargetLowering::LowerShiftRightParts(SDValue Op,2191 SelectionDAG &DAG) const {2192 assert(Op.getNumOperands() == 3 && "Not a double-shift!");2193 assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS);2194 2195 EVT VT = Op.getValueType();2196 unsigned VTBits = VT.getSizeInBits();2197 SDLoc dl(Op);2198 SDValue ShOpLo = Op.getOperand(0);2199 SDValue ShOpHi = Op.getOperand(1);2200 SDValue ShAmt = Op.getOperand(2);2201 unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;2202 2203 if (VTBits == 32 && STI.getSmVersion() >= 35) {2204 // For 32bit and sm35, we can use the funnel shift 'shf' instruction.2205 // {dHi, dLo} = {aHi, aLo} >> Amt2206 // dHi = aHi >> Amt2207 // dLo = shf.r.clamp aLo, aHi, Amt2208 2209 SDValue Hi = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);2210 SDValue Lo =2211 DAG.getNode(NVPTXISD::FSHR_CLAMP, dl, VT, ShOpHi, ShOpLo, ShAmt);2212 2213 SDValue Ops[2] = { Lo, Hi };2214 return DAG.getMergeValues(Ops, dl);2215 }2216 else {2217 // {dHi, dLo} = {aHi, aLo} >> Amt2218 // - if (Amt>=size) then2219 // dLo = aHi >> (Amt-size)2220 // dHi = aHi >> Amt (this is either all 0 or all 1)2221 // else2222 // dLo = (aLo >>logic Amt) | (aHi << (size-Amt))2223 // dHi = aHi >> Amt2224 2225 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,2226 DAG.getConstant(VTBits, dl, MVT::i32),2227 ShAmt);2228 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);2229 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,2230 DAG.getConstant(VTBits, dl, MVT::i32));2231 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);2232 SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);2233 SDValue TrueVal = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);2234 2235 SDValue Cmp = DAG.getSetCC(dl, MVT::i1, ShAmt,2236 DAG.getConstant(VTBits, dl, MVT::i32),2237 ISD::SETGE);2238 SDValue Hi = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);2239 SDValue Lo = DAG.getNode(ISD::SELECT, dl, VT, Cmp, TrueVal, FalseVal);2240 2241 SDValue Ops[2] = { Lo, Hi };2242 return DAG.getMergeValues(Ops, dl);2243 }2244}2245 2246/// LowerShiftLeftParts - Lower SHL_PARTS, which2247/// 1) returns two i32 values and take a 2 x i32 value to shift plus a shift2248/// amount, or2249/// 2) returns two i64 values and take a 2 x i64 value to shift plus a shift2250/// amount.2251SDValue NVPTXTargetLowering::LowerShiftLeftParts(SDValue Op,2252 SelectionDAG &DAG) const {2253 assert(Op.getNumOperands() == 3 && "Not a double-shift!");2254 assert(Op.getOpcode() == ISD::SHL_PARTS);2255 2256 EVT VT = Op.getValueType();2257 unsigned VTBits = VT.getSizeInBits();2258 SDLoc dl(Op);2259 SDValue ShOpLo = Op.getOperand(0);2260 SDValue ShOpHi = Op.getOperand(1);2261 SDValue ShAmt = Op.getOperand(2);2262 2263 if (VTBits == 32 && STI.getSmVersion() >= 35) {2264 // For 32bit and sm35, we can use the funnel shift 'shf' instruction.2265 // {dHi, dLo} = {aHi, aLo} << Amt2266 // dHi = shf.l.clamp aLo, aHi, Amt2267 // dLo = aLo << Amt2268 2269 SDValue Hi =2270 DAG.getNode(NVPTXISD::FSHL_CLAMP, dl, VT, ShOpHi, ShOpLo, ShAmt);2271 SDValue Lo = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);2272 2273 SDValue Ops[2] = { Lo, Hi };2274 return DAG.getMergeValues(Ops, dl);2275 }2276 else {2277 // {dHi, dLo} = {aHi, aLo} << Amt2278 // - if (Amt>=size) then2279 // dLo = aLo << Amt (all 0)2280 // dLo = aLo << (Amt-size)2281 // else2282 // dLo = aLo << Amt2283 // dHi = (aHi << Amt) | (aLo >> (size-Amt))2284 2285 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,2286 DAG.getConstant(VTBits, dl, MVT::i32),2287 ShAmt);2288 SDValue Tmp1 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);2289 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,2290 DAG.getConstant(VTBits, dl, MVT::i32));2291 SDValue Tmp2 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);2292 SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);2293 SDValue TrueVal = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);2294 2295 SDValue Cmp = DAG.getSetCC(dl, MVT::i1, ShAmt,2296 DAG.getConstant(VTBits, dl, MVT::i32),2297 ISD::SETGE);2298 SDValue Lo = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);2299 SDValue Hi = DAG.getNode(ISD::SELECT, dl, VT, Cmp, TrueVal, FalseVal);2300 2301 SDValue Ops[2] = { Lo, Hi };2302 return DAG.getMergeValues(Ops, dl);2303 }2304}2305 2306/// If the types match, convert the generic copysign to the NVPTXISD version,2307/// otherwise bail ensuring that mismatched cases are properly expaned.2308SDValue NVPTXTargetLowering::LowerFCOPYSIGN(SDValue Op,2309 SelectionDAG &DAG) const {2310 EVT VT = Op.getValueType();2311 SDLoc DL(Op);2312 2313 SDValue In1 = Op.getOperand(0);2314 SDValue In2 = Op.getOperand(1);2315 EVT SrcVT = In2.getValueType();2316 2317 if (!SrcVT.bitsEq(VT))2318 return SDValue();2319 2320 return DAG.getNode(NVPTXISD::FCOPYSIGN, DL, VT, In1, In2);2321}2322 2323SDValue NVPTXTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const {2324 EVT VT = Op.getValueType();2325 2326 if (VT == MVT::f32)2327 return LowerFROUND32(Op, DAG);2328 2329 if (VT == MVT::f64)2330 return LowerFROUND64(Op, DAG);2331 2332 llvm_unreachable("unhandled type");2333}2334 2335// This is the the rounding method used in CUDA libdevice in C like code:2336// float roundf(float A)2337// {2338// float RoundedA = (float) (int) ( A > 0 ? (A + 0.5f) : (A - 0.5f));2339// RoundedA = abs(A) > 0x1.0p23 ? A : RoundedA;2340// return abs(A) < 0.5 ? (float)(int)A : RoundedA;2341// }2342SDValue NVPTXTargetLowering::LowerFROUND32(SDValue Op,2343 SelectionDAG &DAG) const {2344 SDLoc SL(Op);2345 SDValue A = Op.getOperand(0);2346 EVT VT = Op.getValueType();2347 2348 SDValue AbsA = DAG.getNode(ISD::FABS, SL, VT, A);2349 2350 // RoundedA = (float) (int) ( A > 0 ? (A + 0.5f) : (A - 0.5f))2351 SDValue Bitcast = DAG.getNode(ISD::BITCAST, SL, MVT::i32, A);2352 const unsigned SignBitMask = 0x80000000;2353 SDValue Sign = DAG.getNode(ISD::AND, SL, MVT::i32, Bitcast,2354 DAG.getConstant(SignBitMask, SL, MVT::i32));2355 const unsigned PointFiveInBits = 0x3F000000;2356 SDValue PointFiveWithSignRaw =2357 DAG.getNode(ISD::OR, SL, MVT::i32, Sign,2358 DAG.getConstant(PointFiveInBits, SL, MVT::i32));2359 SDValue PointFiveWithSign =2360 DAG.getNode(ISD::BITCAST, SL, VT, PointFiveWithSignRaw);2361 SDValue AdjustedA = DAG.getNode(ISD::FADD, SL, VT, A, PointFiveWithSign);2362 SDValue RoundedA = DAG.getNode(ISD::FTRUNC, SL, VT, AdjustedA);2363 2364 // RoundedA = abs(A) > 0x1.0p23 ? A : RoundedA;2365 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);2366 SDValue IsLarge =2367 DAG.getSetCC(SL, SetCCVT, AbsA, DAG.getConstantFP(pow(2.0, 23.0), SL, VT),2368 ISD::SETOGT);2369 RoundedA = DAG.getNode(ISD::SELECT, SL, VT, IsLarge, A, RoundedA);2370 2371 // return abs(A) < 0.5 ? (float)(int)A : RoundedA;2372 SDValue IsSmall =DAG.getSetCC(SL, SetCCVT, AbsA,2373 DAG.getConstantFP(0.5, SL, VT), ISD::SETOLT);2374 SDValue RoundedAForSmallA = DAG.getNode(ISD::FTRUNC, SL, VT, A);2375 return DAG.getNode(ISD::SELECT, SL, VT, IsSmall, RoundedAForSmallA, RoundedA);2376}2377 2378// The implementation of round(double) is similar to that of round(float) in2379// that they both separate the value range into three regions and use a method2380// specific to the region to round the values. However, round(double) first2381// calculates the round of the absolute value and then adds the sign back while2382// round(float) directly rounds the value with sign.2383SDValue NVPTXTargetLowering::LowerFROUND64(SDValue Op,2384 SelectionDAG &DAG) const {2385 SDLoc SL(Op);2386 SDValue A = Op.getOperand(0);2387 EVT VT = Op.getValueType();2388 2389 SDValue AbsA = DAG.getNode(ISD::FABS, SL, VT, A);2390 2391 // double RoundedA = (double) (int) (abs(A) + 0.5f);2392 SDValue AdjustedA = DAG.getNode(ISD::FADD, SL, VT, AbsA,2393 DAG.getConstantFP(0.5, SL, VT));2394 SDValue RoundedA = DAG.getNode(ISD::FTRUNC, SL, VT, AdjustedA);2395 2396 // RoundedA = abs(A) < 0.5 ? (double)0 : RoundedA;2397 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);2398 SDValue IsSmall =DAG.getSetCC(SL, SetCCVT, AbsA,2399 DAG.getConstantFP(0.5, SL, VT), ISD::SETOLT);2400 RoundedA = DAG.getNode(ISD::SELECT, SL, VT, IsSmall,2401 DAG.getConstantFP(0, SL, VT),2402 RoundedA);2403 2404 // Add sign to rounded_A2405 RoundedA = DAG.getNode(ISD::FCOPYSIGN, SL, VT, RoundedA, A);2406 DAG.getNode(ISD::FTRUNC, SL, VT, A);2407 2408 // RoundedA = abs(A) > 0x1.0p52 ? A : RoundedA;2409 SDValue IsLarge =2410 DAG.getSetCC(SL, SetCCVT, AbsA, DAG.getConstantFP(pow(2.0, 52.0), SL, VT),2411 ISD::SETOGT);2412 return DAG.getNode(ISD::SELECT, SL, VT, IsLarge, A, RoundedA);2413}2414 2415static SDValue PromoteBinOpToF32(SDNode *N, SelectionDAG &DAG) {2416 EVT VT = N->getValueType(0);2417 EVT NVT = MVT::f32;2418 if (VT.isVector()) {2419 NVT = EVT::getVectorVT(*DAG.getContext(), NVT, VT.getVectorElementCount());2420 }2421 SDLoc DL(N);2422 SDValue Tmp0 = DAG.getFPExtendOrRound(N->getOperand(0), DL, NVT);2423 SDValue Tmp1 = DAG.getFPExtendOrRound(N->getOperand(1), DL, NVT);2424 SDValue Res = DAG.getNode(N->getOpcode(), DL, NVT, Tmp0, Tmp1, N->getFlags());2425 return DAG.getFPExtendOrRound(Res, DL, VT);2426}2427 2428SDValue NVPTXTargetLowering::PromoteBinOpIfF32FTZ(SDValue Op,2429 SelectionDAG &DAG) const {2430 if (useF32FTZ(DAG.getMachineFunction())) {2431 return PromoteBinOpToF32(Op.getNode(), DAG);2432 }2433 return Op;2434}2435 2436SDValue NVPTXTargetLowering::LowerINT_TO_FP(SDValue Op,2437 SelectionDAG &DAG) const {2438 assert(STI.getSmVersion() < 90 || STI.getPTXVersion() < 78);2439 2440 if (Op.getValueType() == MVT::bf16) {2441 SDLoc Loc(Op);2442 return DAG.getNode(2443 ISD::FP_ROUND, Loc, MVT::bf16,2444 DAG.getNode(Op.getOpcode(), Loc, MVT::f32, Op.getOperand(0)),2445 DAG.getIntPtrConstant(0, Loc, /*isTarget=*/true));2446 }2447 2448 // Everything else is considered legal.2449 return Op;2450}2451 2452SDValue NVPTXTargetLowering::LowerFP_TO_INT(SDValue Op,2453 SelectionDAG &DAG) const {2454 assert(STI.getSmVersion() < 90 || STI.getPTXVersion() < 78);2455 2456 if (Op.getOperand(0).getValueType() == MVT::bf16) {2457 SDLoc Loc(Op);2458 return DAG.getNode(2459 Op.getOpcode(), Loc, Op.getValueType(),2460 DAG.getNode(ISD::FP_EXTEND, Loc, MVT::f32, Op.getOperand(0)));2461 }2462 2463 // Everything else is considered legal.2464 return Op;2465}2466 2467SDValue NVPTXTargetLowering::LowerFP_ROUND(SDValue Op,2468 SelectionDAG &DAG) const {2469 EVT NarrowVT = Op.getValueType();2470 SDValue Wide = Op.getOperand(0);2471 EVT WideVT = Wide.getValueType();2472 if (NarrowVT.getScalarType() == MVT::bf16) {2473 const TargetLowering *TLI = STI.getTargetLowering();2474 if (STI.getSmVersion() < 80 || STI.getPTXVersion() < 70) {2475 return TLI->expandFP_ROUND(Op.getNode(), DAG);2476 }2477 if (STI.getSmVersion() < 90 || STI.getPTXVersion() < 78) {2478 // This combination was the first to support f32 -> bf16.2479 if (STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70) {2480 if (WideVT.getScalarType() == MVT::f32) {2481 return Op;2482 }2483 if (WideVT.getScalarType() == MVT::f64) {2484 SDLoc Loc(Op);2485 // Round-inexact-to-odd f64 to f32, then do the final rounding using2486 // the hardware f32 -> bf16 instruction.2487 SDValue rod = TLI->expandRoundInexactToOdd(2488 WideVT.isVector() ? WideVT.changeVectorElementType(MVT::f32)2489 : MVT::f32,2490 Wide, Loc, DAG);2491 return DAG.getFPExtendOrRound(rod, Loc, NarrowVT);2492 }2493 }2494 return TLI->expandFP_ROUND(Op.getNode(), DAG);2495 }2496 }2497 2498 // Everything else is considered legal.2499 return Op;2500}2501 2502SDValue NVPTXTargetLowering::LowerFP_EXTEND(SDValue Op,2503 SelectionDAG &DAG) const {2504 SDValue Narrow = Op.getOperand(0);2505 EVT NarrowVT = Narrow.getValueType();2506 EVT WideVT = Op.getValueType();2507 if (NarrowVT.getScalarType() == MVT::bf16) {2508 if (WideVT.getScalarType() == MVT::f32 &&2509 (STI.getSmVersion() < 80 || STI.getPTXVersion() < 71)) {2510 SDLoc Loc(Op);2511 return DAG.getNode(ISD::BF16_TO_FP, Loc, WideVT, Narrow);2512 }2513 if (WideVT.getScalarType() == MVT::f64 &&2514 (STI.getSmVersion() < 90 || STI.getPTXVersion() < 78)) {2515 EVT F32 = NarrowVT.isVector() ? NarrowVT.changeVectorElementType(MVT::f32)2516 : MVT::f32;2517 SDLoc Loc(Op);2518 if (STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 71) {2519 Op = DAG.getNode(ISD::FP_EXTEND, Loc, F32, Narrow);2520 } else {2521 Op = DAG.getNode(ISD::BF16_TO_FP, Loc, F32, Narrow);2522 }2523 return DAG.getNode(ISD::FP_EXTEND, Loc, WideVT, Op);2524 }2525 }2526 2527 // Everything else is considered legal.2528 return Op;2529}2530 2531static SDValue LowerVectorArith(SDValue Op, SelectionDAG &DAG) {2532 SDLoc DL(Op);2533 if (Op.getValueType() != MVT::v2i16)2534 return Op;2535 EVT EltVT = Op.getValueType().getVectorElementType();2536 SmallVector<SDValue> VecElements;2537 for (int I = 0, E = Op.getValueType().getVectorNumElements(); I < E; I++) {2538 SmallVector<SDValue> ScalarArgs;2539 llvm::transform(Op->ops(), std::back_inserter(ScalarArgs),2540 [&](const SDUse &O) {2541 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT,2542 O.get(), DAG.getIntPtrConstant(I, DL));2543 });2544 VecElements.push_back(DAG.getNode(Op.getOpcode(), DL, EltVT, ScalarArgs));2545 }2546 SDValue V =2547 DAG.getNode(ISD::BUILD_VECTOR, DL, Op.getValueType(), VecElements);2548 return V;2549}2550 2551static SDValue lowerTcgen05St(SDValue Op, SelectionDAG &DAG) {2552 SDNode *N = Op.getNode();2553 SDLoc DL(N);2554 SmallVector<SDValue, 32> Ops;2555 2556 // split the vector argument2557 for (size_t I = 0; I < N->getNumOperands(); I++) {2558 SDValue Val = N->getOperand(I);2559 EVT ValVT = Val.getValueType();2560 if (ValVT.isVector()) {2561 EVT EltVT = ValVT.getVectorElementType();2562 for (unsigned J = 0, NElts = ValVT.getVectorNumElements(); J < NElts; J++)2563 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Val,2564 DAG.getIntPtrConstant(J, DL)));2565 } else2566 Ops.push_back(Val);2567 }2568 2569 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(N);2570 SDValue Tcgen05StNode =2571 DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, N->getVTList(), Ops,2572 MemSD->getMemoryVT(), MemSD->getMemOperand());2573 2574 return Tcgen05StNode;2575}2576 2577static SDValue lowerBSWAP(SDValue Op, SelectionDAG &DAG) {2578 SDLoc DL(Op);2579 SDValue Src = Op.getOperand(0);2580 EVT VT = Op.getValueType();2581 2582 switch (VT.getSimpleVT().SimpleTy) {2583 case MVT::i16: {2584 SDValue Extended = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Src);2585 SDValue Swapped =2586 getPRMT(Extended, DAG.getConstant(0, DL, MVT::i32), 0x7701, DL, DAG);2587 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Swapped);2588 }2589 case MVT::i32: {2590 return getPRMT(Src, DAG.getConstant(0, DL, MVT::i32), 0x0123, DL, DAG);2591 }2592 case MVT::v2i16: {2593 SDValue Converted = DAG.getBitcast(MVT::i32, Src);2594 SDValue Swapped =2595 getPRMT(Converted, DAG.getConstant(0, DL, MVT::i32), 0x2301, DL, DAG);2596 return DAG.getNode(ISD::BITCAST, DL, MVT::v2i16, Swapped);2597 }2598 case MVT::i64: {2599 SDValue UnpackSrc =2600 DAG.getNode(NVPTXISD::UNPACK_VECTOR, DL, {MVT::i32, MVT::i32}, Src);2601 SDValue SwappedLow =2602 getPRMT(UnpackSrc.getValue(0), DAG.getConstant(0, DL, MVT::i32), 0x0123,2603 DL, DAG);2604 SDValue SwappedHigh =2605 getPRMT(UnpackSrc.getValue(1), DAG.getConstant(0, DL, MVT::i32), 0x0123,2606 DL, DAG);2607 return DAG.getNode(NVPTXISD::BUILD_VECTOR, DL, MVT::i64,2608 {SwappedHigh, SwappedLow});2609 }2610 default:2611 llvm_unreachable("unsupported type for bswap");2612 }2613}2614 2615static unsigned getTcgen05MMADisableOutputLane(unsigned IID) {2616 switch (IID) {2617 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg1:2618 return NVPTXISD::TCGEN05_MMA_SHARED_DISABLE_OUTPUT_LANE_CG1;2619 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg2:2620 return NVPTXISD::TCGEN05_MMA_SHARED_DISABLE_OUTPUT_LANE_CG2;2621 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg1:2622 return NVPTXISD::TCGEN05_MMA_SHARED_SCALE_D_DISABLE_OUTPUT_LANE_CG1;2623 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg2:2624 return NVPTXISD::TCGEN05_MMA_SHARED_SCALE_D_DISABLE_OUTPUT_LANE_CG2;2625 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1:2626 return NVPTXISD::TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG1;2627 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2:2628 return NVPTXISD::TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG2;2629 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1:2630 return NVPTXISD::TCGEN05_MMA_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG1;2631 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2:2632 return NVPTXISD::TCGEN05_MMA_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG2;2633 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1_ashift:2634 return NVPTXISD::TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG1_ASHIFT;2635 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2_ashift:2636 return NVPTXISD::TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG2_ASHIFT;2637 case Intrinsic::2638 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1_ashift:2639 return NVPTXISD::TCGEN05_MMA_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG1_ASHIFT;2640 case Intrinsic::2641 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2_ashift:2642 return NVPTXISD::TCGEN05_MMA_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG2_ASHIFT;2643 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg1:2644 return NVPTXISD::TCGEN05_MMA_SP_SHARED_DISABLE_OUTPUT_LANE_CG1;2645 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg2:2646 return NVPTXISD::TCGEN05_MMA_SP_SHARED_DISABLE_OUTPUT_LANE_CG2;2647 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg1:2648 return NVPTXISD::TCGEN05_MMA_SP_SHARED_SCALE_D_DISABLE_OUTPUT_LANE_CG1;2649 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg2:2650 return NVPTXISD::TCGEN05_MMA_SP_SHARED_SCALE_D_DISABLE_OUTPUT_LANE_CG2;2651 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1:2652 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_DISABLE_OUTPUT_LANE_CG1;2653 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2:2654 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_DISABLE_OUTPUT_LANE_CG2;2655 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1_ashift:2656 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_DISABLE_OUTPUT_LANE_CG1_ASHIFT;2657 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2_ashift:2658 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_DISABLE_OUTPUT_LANE_CG2_ASHIFT;2659 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1:2660 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG1;2661 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2:2662 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG2;2663 case Intrinsic::2664 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1_ashift:2665 return NVPTXISD::2666 TCGEN05_MMA_SP_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG1_ASHIFT;2667 case Intrinsic::2668 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2_ashift:2669 return NVPTXISD::2670 TCGEN05_MMA_SP_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG2_ASHIFT;2671 };2672 llvm_unreachable("unhandled tcgen05.mma.disable_output_lane intrinsic");2673}2674 2675static SDValue LowerTcgen05MMADisableOutputLane(SDValue Op, SelectionDAG &DAG) {2676 SDNode *N = Op.getNode();2677 SDLoc DL(N);2678 unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();2679 2680 SmallVector<SDValue, 16> Ops;2681 // split the vector argument2682 for (size_t I = 0; I < N->getNumOperands(); I++) {2683 if (I == 1)2684 continue; // skip IID2685 SDValue Val = N->getOperand(I);2686 EVT ValVT = Val.getValueType();2687 if (ValVT.isVector()) {2688 EVT EltVT = ValVT.getVectorElementType();2689 for (unsigned J = 0, NElts = ValVT.getVectorNumElements(); J < NElts; J++)2690 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Val,2691 DAG.getIntPtrConstant(J, DL)));2692 } else2693 Ops.push_back(Val);2694 }2695 2696 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(N);2697 SDValue Tcgen05MMANode = DAG.getMemIntrinsicNode(2698 getTcgen05MMADisableOutputLane(IID), DL, N->getVTList(), Ops,2699 MemSD->getMemoryVT(), MemSD->getMemOperand());2700 2701 return Tcgen05MMANode;2702}2703 2704// Lower vector return type of tcgen05.ld intrinsics2705static std::optional<std::pair<SDValue, SDValue>>2706lowerTcgen05Ld(SDNode *N, SelectionDAG &DAG, bool HasOffset = false) {2707 SDLoc DL(N);2708 EVT ResVT = N->getValueType(0);2709 if (!ResVT.isVector())2710 return {}; // already legalized.2711 2712 const unsigned NumElts = ResVT.getVectorNumElements();2713 2714 // Create the return type of the instructions2715 SmallVector<EVT, 5> ListVTs;2716 for (unsigned i = 0; i < NumElts; ++i)2717 ListVTs.push_back(MVT::i32);2718 2719 ListVTs.push_back(N->getValueType(1)); // Chain2720 2721 SDVTList ResVTs = DAG.getVTList(ListVTs);2722 2723 SmallVector<SDValue, 8> Ops{N->getOperand(0), N->getOperand(1),2724 N->getOperand(2)};2725 2726 if (HasOffset) {2727 Ops.push_back(N->getOperand(3)); // offset2728 Ops.push_back(N->getOperand(4)); // Pack flag2729 } else2730 Ops.push_back(N->getOperand(3)); // Pack flag2731 2732 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(N);2733 SDValue NewNode =2734 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, ResVTs, Ops,2735 MemSD->getMemoryVT(), MemSD->getMemOperand());2736 2737 // split the vector result2738 SmallVector<SDValue, 4> ScalarRes;2739 for (unsigned i = 0; i < NumElts; ++i) {2740 SDValue Res = NewNode.getValue(i);2741 ScalarRes.push_back(Res);2742 }2743 2744 SDValue Chain = NewNode.getValue(NumElts);2745 SDValue BuildVector = DAG.getNode(ISD::BUILD_VECTOR, DL, ResVT, ScalarRes);2746 return {{BuildVector, Chain}};2747}2748 2749static SDValue lowerIntrinsicVoid(SDValue Op, SelectionDAG &DAG) {2750 SDNode *N = Op.getNode();2751 SDValue Intrin = N->getOperand(1);2752 2753 // Get the intrinsic ID2754 unsigned IntrinNo = cast<ConstantSDNode>(Intrin.getNode())->getZExtValue();2755 switch (IntrinNo) {2756 default:2757 break;2758 case Intrinsic::nvvm_tcgen05_st_16x64b_x1:2759 case Intrinsic::nvvm_tcgen05_st_16x64b_x2:2760 case Intrinsic::nvvm_tcgen05_st_16x64b_x4:2761 case Intrinsic::nvvm_tcgen05_st_16x64b_x8:2762 case Intrinsic::nvvm_tcgen05_st_16x64b_x16:2763 case Intrinsic::nvvm_tcgen05_st_16x64b_x32:2764 case Intrinsic::nvvm_tcgen05_st_16x64b_x128:2765 case Intrinsic::nvvm_tcgen05_st_16x128b_x1:2766 case Intrinsic::nvvm_tcgen05_st_16x128b_x2:2767 case Intrinsic::nvvm_tcgen05_st_16x128b_x4:2768 case Intrinsic::nvvm_tcgen05_st_16x128b_x8:2769 case Intrinsic::nvvm_tcgen05_st_16x128b_x16:2770 case Intrinsic::nvvm_tcgen05_st_16x128b_x32:2771 case Intrinsic::nvvm_tcgen05_st_16x128b_x64:2772 case Intrinsic::nvvm_tcgen05_st_16x256b_x1:2773 case Intrinsic::nvvm_tcgen05_st_16x256b_x2:2774 case Intrinsic::nvvm_tcgen05_st_16x256b_x4:2775 case Intrinsic::nvvm_tcgen05_st_16x256b_x8:2776 case Intrinsic::nvvm_tcgen05_st_16x256b_x16:2777 case Intrinsic::nvvm_tcgen05_st_16x256b_x32:2778 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x1:2779 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x2:2780 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x4:2781 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x8:2782 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x16:2783 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x32:2784 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x64:2785 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x128:2786 case Intrinsic::nvvm_tcgen05_st_32x32b_x1:2787 case Intrinsic::nvvm_tcgen05_st_32x32b_x2:2788 case Intrinsic::nvvm_tcgen05_st_32x32b_x4:2789 case Intrinsic::nvvm_tcgen05_st_32x32b_x8:2790 case Intrinsic::nvvm_tcgen05_st_32x32b_x16:2791 case Intrinsic::nvvm_tcgen05_st_32x32b_x32:2792 case Intrinsic::nvvm_tcgen05_st_16x64b_x64:2793 case Intrinsic::nvvm_tcgen05_st_32x32b_x64:2794 case Intrinsic::nvvm_tcgen05_st_32x32b_x128:2795 return lowerTcgen05St(Op, DAG);2796 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg1:2797 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg2:2798 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg1:2799 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg2:2800 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg1:2801 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg2:2802 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg1:2803 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg2:2804 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1:2805 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2:2806 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1:2807 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2:2808 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1:2809 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2:2810 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1:2811 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2:2812 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1_ashift:2813 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2_ashift:2814 case Intrinsic::2815 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1_ashift:2816 case Intrinsic::2817 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2_ashift:2818 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1_ashift:2819 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2_ashift:2820 case Intrinsic::2821 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1_ashift:2822 case Intrinsic::2823 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2_ashift:2824 return LowerTcgen05MMADisableOutputLane(Op, DAG);2825 }2826 return Op;2827}2828 2829static SDValue LowerClusterLaunchControlQueryCancel(SDValue Op,2830 SelectionDAG &DAG) {2831 2832 SDNode *N = Op.getNode();2833 if (N->getOperand(1).getValueType() != MVT::i128) {2834 // return, if the operand is already lowered2835 return SDValue();2836 }2837 2838 unsigned IID =2839 cast<ConstantSDNode>(N->getOperand(0).getNode())->getZExtValue();2840 auto Opcode = [&]() {2841 switch (IID) {2842 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_is_canceled:2843 return NVPTXISD::CLUSTERLAUNCHCONTROL_QUERY_CANCEL_IS_CANCELED;2844 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_x:2845 return NVPTXISD::CLUSTERLAUNCHCONTROL_QUERY_CANCEL_GET_FIRST_CTAID_X;2846 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_y:2847 return NVPTXISD::CLUSTERLAUNCHCONTROL_QUERY_CANCEL_GET_FIRST_CTAID_Y;2848 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_z:2849 return NVPTXISD::CLUSTERLAUNCHCONTROL_QUERY_CANCEL_GET_FIRST_CTAID_Z;2850 default:2851 llvm_unreachable("unsupported/unhandled intrinsic");2852 }2853 }();2854 2855 SDLoc DL(N);2856 SDValue TryCancelResponse = N->getOperand(1);2857 SDValue Cast = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, TryCancelResponse);2858 SDValue TryCancelResponse0 =2859 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, Cast,2860 DAG.getIntPtrConstant(0, DL));2861 SDValue TryCancelResponse1 =2862 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, Cast,2863 DAG.getIntPtrConstant(1, DL));2864 2865 return DAG.getNode(Opcode, DL, N->getVTList(),2866 {TryCancelResponse0, TryCancelResponse1});2867}2868 2869static SDValue lowerCvtRSIntrinsics(SDValue Op, SelectionDAG &DAG) {2870 SDNode *N = Op.getNode();2871 SDLoc DL(N);2872 SDValue F32Vec = N->getOperand(1);2873 SDValue RBits = N->getOperand(2);2874 2875 unsigned IntrinsicID = N->getConstantOperandVal(0);2876 2877 // Extract the 4 float elements from the vector2878 SmallVector<SDValue, 6> Ops;2879 for (unsigned i = 0; i < 4; ++i)2880 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, F32Vec,2881 DAG.getIntPtrConstant(i, DL)));2882 2883 using NVPTX::PTXCvtMode::CvtMode;2884 2885 auto [OpCode, RetTy, CvtModeFlag] =2886 [&]() -> std::tuple<unsigned, MVT::SimpleValueType, uint32_t> {2887 switch (IntrinsicID) {2888 case Intrinsic::nvvm_f32x4_to_e4m3x4_rs_relu_satfinite:2889 return {NVPTXISD::CVT_E4M3X4_F32X4_RS_SF, MVT::v4i8,2890 CvtMode::RS | CvtMode::RELU_FLAG};2891 case Intrinsic::nvvm_f32x4_to_e4m3x4_rs_satfinite:2892 return {NVPTXISD::CVT_E4M3X4_F32X4_RS_SF, MVT::v4i8, CvtMode::RS};2893 case Intrinsic::nvvm_f32x4_to_e5m2x4_rs_relu_satfinite:2894 return {NVPTXISD::CVT_E5M2X4_F32X4_RS_SF, MVT::v4i8,2895 CvtMode::RS | CvtMode::RELU_FLAG};2896 case Intrinsic::nvvm_f32x4_to_e5m2x4_rs_satfinite:2897 return {NVPTXISD::CVT_E5M2X4_F32X4_RS_SF, MVT::v4i8, CvtMode::RS};2898 case Intrinsic::nvvm_f32x4_to_e2m3x4_rs_relu_satfinite:2899 return {NVPTXISD::CVT_E2M3X4_F32X4_RS_SF, MVT::v4i8,2900 CvtMode::RS | CvtMode::RELU_FLAG};2901 case Intrinsic::nvvm_f32x4_to_e2m3x4_rs_satfinite:2902 return {NVPTXISD::CVT_E2M3X4_F32X4_RS_SF, MVT::v4i8, CvtMode::RS};2903 case Intrinsic::nvvm_f32x4_to_e3m2x4_rs_relu_satfinite:2904 return {NVPTXISD::CVT_E3M2X4_F32X4_RS_SF, MVT::v4i8,2905 CvtMode::RS | CvtMode::RELU_FLAG};2906 case Intrinsic::nvvm_f32x4_to_e3m2x4_rs_satfinite:2907 return {NVPTXISD::CVT_E3M2X4_F32X4_RS_SF, MVT::v4i8, CvtMode::RS};2908 case Intrinsic::nvvm_f32x4_to_e2m1x4_rs_relu_satfinite:2909 return {NVPTXISD::CVT_E2M1X4_F32X4_RS_SF, MVT::i16,2910 CvtMode::RS | CvtMode::RELU_FLAG};2911 case Intrinsic::nvvm_f32x4_to_e2m1x4_rs_satfinite:2912 return {NVPTXISD::CVT_E2M1X4_F32X4_RS_SF, MVT::i16, CvtMode::RS};2913 default:2914 llvm_unreachable("unsupported/unhandled intrinsic");2915 }2916 }();2917 2918 Ops.push_back(RBits);2919 Ops.push_back(DAG.getConstant(CvtModeFlag, DL, MVT::i32));2920 2921 return DAG.getNode(OpCode, DL, RetTy, Ops);2922}2923 2924static SDValue lowerPrmtIntrinsic(SDValue Op, SelectionDAG &DAG) {2925 const unsigned Mode = [&]() {2926 switch (Op->getConstantOperandVal(0)) {2927 case Intrinsic::nvvm_prmt:2928 return NVPTX::PTXPrmtMode::NONE;2929 case Intrinsic::nvvm_prmt_b4e:2930 return NVPTX::PTXPrmtMode::B4E;2931 case Intrinsic::nvvm_prmt_ecl:2932 return NVPTX::PTXPrmtMode::ECL;2933 case Intrinsic::nvvm_prmt_ecr:2934 return NVPTX::PTXPrmtMode::ECR;2935 case Intrinsic::nvvm_prmt_f4e:2936 return NVPTX::PTXPrmtMode::F4E;2937 case Intrinsic::nvvm_prmt_rc16:2938 return NVPTX::PTXPrmtMode::RC16;2939 case Intrinsic::nvvm_prmt_rc8:2940 return NVPTX::PTXPrmtMode::RC8;2941 default:2942 llvm_unreachable("unsupported/unhandled intrinsic");2943 }2944 }();2945 SDLoc DL(Op);2946 SDValue A = Op->getOperand(1);2947 SDValue B = Op.getNumOperands() == 4 ? Op.getOperand(2)2948 : DAG.getConstant(0, DL, MVT::i32);2949 SDValue Selector = (Op->op_end() - 1)->get();2950 return getPRMT(A, B, Selector, DL, DAG, Mode);2951}2952 2953static SDValue lowerIntrinsicWChain(SDValue Op, SelectionDAG &DAG) {2954 switch (Op->getConstantOperandVal(1)) {2955 default:2956 return Op;2957 2958 // These tcgen05 intrinsics return a v2i32, which is legal, so we have to2959 // lower them through LowerOperation() instead of ReplaceNodeResults().2960 case Intrinsic::nvvm_tcgen05_ld_16x64b_x2:2961 case Intrinsic::nvvm_tcgen05_ld_16x128b_x1:2962 case Intrinsic::nvvm_tcgen05_ld_32x32b_x2:2963 if (auto Res = lowerTcgen05Ld(Op.getNode(), DAG))2964 return DAG.getMergeValues({Res->first, Res->second}, SDLoc(Op));2965 return SDValue();2966 2967 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x2:2968 if (auto Res = lowerTcgen05Ld(Op.getNode(), DAG, /*HasOffset=*/true))2969 return DAG.getMergeValues({Res->first, Res->second}, SDLoc(Op));2970 return SDValue();2971 }2972}2973 2974static SDValue lowerIntrinsicWOChain(SDValue Op, SelectionDAG &DAG) {2975 switch (Op->getConstantOperandVal(0)) {2976 default:2977 return Op;2978 case Intrinsic::nvvm_prmt:2979 case Intrinsic::nvvm_prmt_b4e:2980 case Intrinsic::nvvm_prmt_ecl:2981 case Intrinsic::nvvm_prmt_ecr:2982 case Intrinsic::nvvm_prmt_f4e:2983 case Intrinsic::nvvm_prmt_rc16:2984 case Intrinsic::nvvm_prmt_rc8:2985 return lowerPrmtIntrinsic(Op, DAG);2986 case Intrinsic::nvvm_internal_addrspace_wrap:2987 return Op.getOperand(1);2988 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_is_canceled:2989 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_x:2990 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_y:2991 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_z:2992 return LowerClusterLaunchControlQueryCancel(Op, DAG);2993 case Intrinsic::nvvm_f32x4_to_e4m3x4_rs_satfinite:2994 case Intrinsic::nvvm_f32x4_to_e4m3x4_rs_relu_satfinite:2995 case Intrinsic::nvvm_f32x4_to_e5m2x4_rs_satfinite:2996 case Intrinsic::nvvm_f32x4_to_e5m2x4_rs_relu_satfinite:2997 case Intrinsic::nvvm_f32x4_to_e2m3x4_rs_satfinite:2998 case Intrinsic::nvvm_f32x4_to_e2m3x4_rs_relu_satfinite:2999 case Intrinsic::nvvm_f32x4_to_e3m2x4_rs_satfinite:3000 case Intrinsic::nvvm_f32x4_to_e3m2x4_rs_relu_satfinite:3001 case Intrinsic::nvvm_f32x4_to_e2m1x4_rs_satfinite:3002 case Intrinsic::nvvm_f32x4_to_e2m1x4_rs_relu_satfinite:3003 return lowerCvtRSIntrinsics(Op, DAG);3004 }3005}3006 3007// In PTX 64-bit CTLZ and CTPOP are supported, but they return a 32-bit value.3008// Lower these into a node returning the correct type which is zero-extended3009// back to the correct size.3010static SDValue lowerCTLZCTPOP(SDValue Op, SelectionDAG &DAG) {3011 SDValue V = Op->getOperand(0);3012 assert(V.getValueType() == MVT::i64 &&3013 "Unexpected CTLZ/CTPOP type to legalize");3014 3015 SDLoc DL(Op);3016 SDValue CT = DAG.getNode(Op->getOpcode(), DL, MVT::i32, V);3017 return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, CT, SDNodeFlags::NonNeg);3018}3019 3020static SDValue expandFSH64(SDValue A, SDValue B, SDValue ShiftAmount, SDLoc DL,3021 unsigned Opcode, SelectionDAG &DAG) {3022 assert(A.getValueType() == MVT::i64 && B.getValueType() == MVT::i64);3023 3024 const auto *AmtConst = dyn_cast<ConstantSDNode>(ShiftAmount);3025 if (!AmtConst)3026 return SDValue();3027 const auto Amt = AmtConst->getZExtValue() & 63;3028 3029 SDValue UnpackA =3030 DAG.getNode(NVPTXISD::UNPACK_VECTOR, DL, {MVT::i32, MVT::i32}, A);3031 SDValue UnpackB =3032 DAG.getNode(NVPTXISD::UNPACK_VECTOR, DL, {MVT::i32, MVT::i32}, B);3033 3034 // Arch is Little endiain: 0 = low bits, 1 = high bits3035 SDValue ALo = UnpackA.getValue(0);3036 SDValue AHi = UnpackA.getValue(1);3037 SDValue BLo = UnpackB.getValue(0);3038 SDValue BHi = UnpackB.getValue(1);3039 3040 // The bitfeild consists of { AHi : ALo : BHi : BLo }3041 //3042 // * FSHL, Amt < 32 - The window will contain { AHi : ALo : BHi }3043 // * FSHL, Amt >= 32 - The window will contain { ALo : BHi : BLo }3044 // * FSHR, Amt < 32 - The window will contain { ALo : BHi : BLo }3045 // * FSHR, Amt >= 32 - The window will contain { AHi : ALo : BHi }3046 //3047 // Note that Amt = 0 and Amt = 32 are special cases where 32-bit funnel shifts3048 // are not needed at all. Amt = 0 is a no-op producing either A or B depending3049 // on the direction. Amt = 32 can be implemented by a packing and unpacking3050 // move to select and arrange the 32bit values. For simplicity, these cases3051 // are not handled here explicitly and instead we rely on DAGCombiner to3052 // remove the no-op funnel shifts we insert.3053 auto [High, Mid, Low] = ((Opcode == ISD::FSHL) == (Amt < 32))3054 ? std::make_tuple(AHi, ALo, BHi)3055 : std::make_tuple(ALo, BHi, BLo);3056 3057 SDValue NewAmt = DAG.getConstant(Amt & 31, DL, MVT::i32);3058 SDValue RHi = DAG.getNode(Opcode, DL, MVT::i32, {High, Mid, NewAmt});3059 SDValue RLo = DAG.getNode(Opcode, DL, MVT::i32, {Mid, Low, NewAmt});3060 3061 return DAG.getNode(NVPTXISD::BUILD_VECTOR, DL, MVT::i64, {RLo, RHi});3062}3063 3064static SDValue lowerFSH(SDValue Op, SelectionDAG &DAG) {3065 return expandFSH64(Op->getOperand(0), Op->getOperand(1), Op->getOperand(2),3066 SDLoc(Op), Op->getOpcode(), DAG);3067}3068 3069static SDValue lowerROT(SDValue Op, SelectionDAG &DAG) {3070 unsigned Opcode = Op->getOpcode() == ISD::ROTL ? ISD::FSHL : ISD::FSHR;3071 return expandFSH64(Op->getOperand(0), Op->getOperand(0), Op->getOperand(1),3072 SDLoc(Op), Opcode, DAG);3073}3074 3075static SDValue lowerFREM(SDValue Op, SelectionDAG &DAG) {3076 // Lower (frem x, y) into (sub x, (mul (ftrunc (div x, y)) y)),3077 // i.e. "poor man's fmod()". When y is infinite, x is returned. This matches3078 // the semantics of LLVM's frem.3079 SDLoc DL(Op);3080 SDValue X = Op->getOperand(0);3081 SDValue Y = Op->getOperand(1);3082 EVT Ty = Op.getValueType();3083 SDNodeFlags Flags = Op->getFlags();3084 3085 SDValue Div = DAG.getNode(ISD::FDIV, DL, Ty, X, Y, Flags);3086 SDValue Trunc = DAG.getNode(ISD::FTRUNC, DL, Ty, Div, Flags);3087 SDValue Mul = DAG.getNode(ISD::FMUL, DL, Ty, Trunc, Y,3088 Flags | SDNodeFlags::AllowContract);3089 SDValue Sub = DAG.getNode(ISD::FSUB, DL, Ty, X, Mul,3090 Flags | SDNodeFlags::AllowContract);3091 3092 if (Flags.hasNoInfs())3093 return Sub;3094 3095 // If Y is infinite, return X3096 SDValue AbsY = DAG.getNode(ISD::FABS, DL, Ty, Y);3097 SDValue Inf =3098 DAG.getConstantFP(APFloat::getInf(Ty.getFltSemantics()), DL, Ty);3099 SDValue IsInf = DAG.getSetCC(DL, MVT::i1, AbsY, Inf, ISD::SETEQ);3100 return DAG.getSelect(DL, Ty, IsInf, X, Sub);3101}3102 3103static SDValue lowerSELECT(SDValue Op, SelectionDAG &DAG) {3104 assert(Op.getValueType() == MVT::i1 && "Custom lowering enabled only for i1");3105 3106 SDValue Cond = Op->getOperand(0);3107 SDValue TrueVal = Op->getOperand(1);3108 SDValue FalseVal = Op->getOperand(2);3109 SDLoc DL(Op);3110 3111 // If both operands are truncated, we push the select through the truncates.3112 if (TrueVal.getOpcode() == ISD::TRUNCATE &&3113 FalseVal.getOpcode() == ISD::TRUNCATE) {3114 TrueVal = TrueVal.getOperand(0);3115 FalseVal = FalseVal.getOperand(0);3116 3117 EVT VT = TrueVal.getSimpleValueType().bitsLE(FalseVal.getSimpleValueType())3118 ? TrueVal.getValueType()3119 : FalseVal.getValueType();3120 TrueVal = DAG.getAnyExtOrTrunc(TrueVal, DL, VT);3121 FalseVal = DAG.getAnyExtOrTrunc(FalseVal, DL, VT);3122 SDValue Select = DAG.getSelect(DL, VT, Cond, TrueVal, FalseVal);3123 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Select);3124 }3125 3126 // Otherwise, expand the select into a series of logical operations. These3127 // often can be folded into other operations either by us or ptxas.3128 TrueVal = DAG.getFreeze(TrueVal);3129 FalseVal = DAG.getFreeze(FalseVal);3130 SDValue And1 = DAG.getNode(ISD::AND, DL, MVT::i1, Cond, TrueVal);3131 SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);3132 SDValue And2 = DAG.getNode(ISD::AND, DL, MVT::i1, NotCond, FalseVal);3133 SDValue Or = DAG.getNode(ISD::OR, DL, MVT::i1, And1, And2);3134 return Or;3135}3136 3137static SDValue lowerMSTORE(SDValue Op, SelectionDAG &DAG) {3138 SDNode *N = Op.getNode();3139 3140 SDValue Chain = N->getOperand(0);3141 SDValue Val = N->getOperand(1);3142 SDValue BasePtr = N->getOperand(2);3143 SDValue Offset = N->getOperand(3);3144 SDValue Mask = N->getOperand(4);3145 3146 SDLoc DL(N);3147 EVT ValVT = Val.getValueType();3148 MemSDNode *MemSD = cast<MemSDNode>(N);3149 assert(ValVT.isVector() && "Masked vector store must have vector type");3150 assert(MemSD->getAlign() >= DAG.getEVTAlign(ValVT) &&3151 "Unexpected alignment for masked store");3152 3153 unsigned Opcode = 0;3154 switch (ValVT.getSimpleVT().SimpleTy) {3155 default:3156 llvm_unreachable("Unexpected masked vector store type");3157 case MVT::v4i64:3158 case MVT::v4f64: {3159 Opcode = NVPTXISD::StoreV4;3160 break;3161 }3162 case MVT::v8i32:3163 case MVT::v8f32: {3164 Opcode = NVPTXISD::StoreV8;3165 break;3166 }3167 }3168 3169 SmallVector<SDValue, 8> Ops;3170 3171 // Construct the new SDNode. First operand is the chain.3172 Ops.push_back(Chain);3173 3174 // The next N operands are the values to store. Encode the mask into the3175 // values using the sentinel register 0 to represent a masked-off element.3176 assert(Mask.getValueType().isVector() &&3177 Mask.getValueType().getVectorElementType() == MVT::i1 &&3178 "Mask must be a vector of i1");3179 assert(Mask.getOpcode() == ISD::BUILD_VECTOR &&3180 "Mask expected to be a BUILD_VECTOR");3181 assert(Mask.getValueType().getVectorNumElements() ==3182 ValVT.getVectorNumElements() &&3183 "Mask size must be the same as the vector size");3184 for (auto [I, Op] : enumerate(Mask->ops())) {3185 // Mask elements must be constants.3186 if (Op.getNode()->getAsZExtVal() == 0) {3187 // Append a sentinel register 0 to the Ops vector to represent a masked3188 // off element, this will be handled in tablegen3189 Ops.push_back(DAG.getRegister(MCRegister::NoRegister,3190 ValVT.getVectorElementType()));3191 } else {3192 // Extract the element from the vector to store3193 SDValue ExtVal =3194 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ValVT.getVectorElementType(),3195 Val, DAG.getIntPtrConstant(I, DL));3196 Ops.push_back(ExtVal);3197 }3198 }3199 3200 // Next, the pointer operand.3201 Ops.push_back(BasePtr);3202 3203 // Finally, the offset operand. We expect this to always be undef, and it will3204 // be ignored in lowering, but to mirror the handling of the other vector3205 // store instructions we include it in the new SDNode.3206 assert(Offset.getOpcode() == ISD::UNDEF &&3207 "Offset operand expected to be undef");3208 Ops.push_back(Offset);3209 3210 SDValue NewSt =3211 DAG.getMemIntrinsicNode(Opcode, DL, DAG.getVTList(MVT::Other), Ops,3212 MemSD->getMemoryVT(), MemSD->getMemOperand());3213 3214 return NewSt;3215}3216 3217SDValue3218NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {3219 switch (Op.getOpcode()) {3220 case ISD::RETURNADDR:3221 return SDValue();3222 case ISD::FRAMEADDR:3223 return SDValue();3224 case ISD::ADDRSPACECAST:3225 return LowerADDRSPACECAST(Op, DAG);3226 case ISD::INTRINSIC_W_CHAIN:3227 return lowerIntrinsicWChain(Op, DAG);3228 case ISD::INTRINSIC_WO_CHAIN:3229 return lowerIntrinsicWOChain(Op, DAG);3230 case ISD::INTRINSIC_VOID:3231 return lowerIntrinsicVoid(Op, DAG);3232 case ISD::BUILD_VECTOR:3233 return LowerBUILD_VECTOR(Op, DAG);3234 case ISD::BITCAST:3235 return LowerBITCAST(Op, DAG);3236 case ISD::EXTRACT_SUBVECTOR:3237 return Op;3238 case ISD::EXTRACT_VECTOR_ELT:3239 return LowerEXTRACT_VECTOR_ELT(Op, DAG);3240 case ISD::INSERT_VECTOR_ELT:3241 return LowerINSERT_VECTOR_ELT(Op, DAG);3242 case ISD::VECTOR_SHUFFLE:3243 return LowerVECTOR_SHUFFLE(Op, DAG);3244 case ISD::CONCAT_VECTORS:3245 return LowerCONCAT_VECTORS(Op, DAG);3246 case ISD::VECREDUCE_FMAX:3247 case ISD::VECREDUCE_FMIN:3248 case ISD::VECREDUCE_FMAXIMUM:3249 case ISD::VECREDUCE_FMINIMUM:3250 return LowerVECREDUCE(Op, DAG);3251 case ISD::STORE:3252 return LowerSTORE(Op, DAG);3253 case ISD::MSTORE: {3254 assert(STI.has256BitVectorLoadStore(3255 cast<MemSDNode>(Op.getNode())->getAddressSpace()) &&3256 "Masked store vector not supported on subtarget.");3257 return lowerMSTORE(Op, DAG);3258 }3259 case ISD::LOAD:3260 return LowerLOAD(Op, DAG);3261 case ISD::MLOAD:3262 return LowerMLOAD(Op, DAG);3263 case ISD::SHL_PARTS:3264 return LowerShiftLeftParts(Op, DAG);3265 case ISD::SRA_PARTS:3266 case ISD::SRL_PARTS:3267 return LowerShiftRightParts(Op, DAG);3268 case ISD::SELECT:3269 return lowerSELECT(Op, DAG);3270 case ISD::FROUND:3271 return LowerFROUND(Op, DAG);3272 case ISD::FCOPYSIGN:3273 return LowerFCOPYSIGN(Op, DAG);3274 case ISD::SINT_TO_FP:3275 case ISD::UINT_TO_FP:3276 return LowerINT_TO_FP(Op, DAG);3277 case ISD::FP_TO_SINT:3278 case ISD::FP_TO_UINT:3279 return LowerFP_TO_INT(Op, DAG);3280 case ISD::FP_ROUND:3281 return LowerFP_ROUND(Op, DAG);3282 case ISD::FP_EXTEND:3283 return LowerFP_EXTEND(Op, DAG);3284 case ISD::BR_JT:3285 return LowerBR_JT(Op, DAG);3286 case ISD::VAARG:3287 return LowerVAARG(Op, DAG);3288 case ISD::VASTART:3289 return LowerVASTART(Op, DAG);3290 case ISD::FSHL:3291 case ISD::FSHR:3292 return lowerFSH(Op, DAG);3293 case ISD::ROTL:3294 case ISD::ROTR:3295 return lowerROT(Op, DAG);3296 case ISD::ABS:3297 case ISD::SMIN:3298 case ISD::SMAX:3299 case ISD::UMIN:3300 case ISD::UMAX:3301 case ISD::ADD:3302 case ISD::SUB:3303 case ISD::MUL:3304 case ISD::SHL:3305 case ISD::SREM:3306 case ISD::UREM:3307 return LowerVectorArith(Op, DAG);3308 case ISD::DYNAMIC_STACKALLOC:3309 return LowerDYNAMIC_STACKALLOC(Op, DAG);3310 case ISD::STACKRESTORE:3311 return LowerSTACKRESTORE(Op, DAG);3312 case ISD::STACKSAVE:3313 return LowerSTACKSAVE(Op, DAG);3314 case ISD::CopyToReg:3315 return LowerCopyToReg_128(Op, DAG);3316 case ISD::FADD:3317 case ISD::FSUB:3318 case ISD::FMUL:3319 // Used only for bf16 on SM80, where we select fma for non-ftz operation3320 return PromoteBinOpIfF32FTZ(Op, DAG);3321 case ISD::CTPOP:3322 case ISD::CTLZ:3323 return lowerCTLZCTPOP(Op, DAG);3324 case ISD::FREM:3325 return lowerFREM(Op, DAG);3326 case ISD::BSWAP:3327 return lowerBSWAP(Op, DAG);3328 default:3329 llvm_unreachable("Custom lowering not defined for operation");3330 }3331}3332 3333SDValue NVPTXTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) const {3334 SDLoc DL(Op);3335 SDValue Chain = Op.getOperand(0);3336 const auto *JT = cast<JumpTableSDNode>(Op.getOperand(1));3337 SDValue Index = Op.getOperand(2);3338 3339 unsigned JId = JT->getIndex();3340 MachineJumpTableInfo *MJTI = DAG.getMachineFunction().getJumpTableInfo();3341 ArrayRef<MachineBasicBlock *> MBBs = MJTI->getJumpTables()[JId].MBBs;3342 3343 SDValue IdV = DAG.getConstant(JId, DL, MVT::i32);3344 3345 // Generate BrxStart node3346 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);3347 Chain = DAG.getNode(NVPTXISD::BrxStart, DL, VTs, Chain, IdV);3348 3349 // Generate BrxItem nodes3350 assert(!MBBs.empty());3351 for (MachineBasicBlock *MBB : MBBs.drop_back())3352 Chain = DAG.getNode(NVPTXISD::BrxItem, DL, VTs, Chain.getValue(0),3353 DAG.getBasicBlock(MBB), Chain.getValue(1));3354 3355 // Generate BrxEnd nodes3356 SDValue EndOps[] = {Chain.getValue(0), DAG.getBasicBlock(MBBs.back()), Index,3357 IdV, Chain.getValue(1)};3358 SDValue BrxEnd = DAG.getNode(NVPTXISD::BrxEnd, DL, MVT::Other, EndOps);3359 3360 return BrxEnd;3361}3362 3363// This will prevent AsmPrinter from trying to print the jump tables itself.3364unsigned NVPTXTargetLowering::getJumpTableEncoding() const {3365 return MachineJumpTableInfo::EK_Inline;3366}3367 3368SDValue NVPTXTargetLowering::LowerADDRSPACECAST(SDValue Op,3369 SelectionDAG &DAG) const {3370 AddrSpaceCastSDNode *N = cast<AddrSpaceCastSDNode>(Op.getNode());3371 unsigned SrcAS = N->getSrcAddressSpace();3372 unsigned DestAS = N->getDestAddressSpace();3373 if (SrcAS != llvm::ADDRESS_SPACE_GENERIC &&3374 DestAS != llvm::ADDRESS_SPACE_GENERIC) {3375 // Shared and SharedCluster can be converted to each other through generic3376 // space3377 if ((SrcAS == llvm::ADDRESS_SPACE_SHARED &&3378 DestAS == llvm::ADDRESS_SPACE_SHARED_CLUSTER) ||3379 (SrcAS == llvm::ADDRESS_SPACE_SHARED_CLUSTER &&3380 DestAS == llvm::ADDRESS_SPACE_SHARED)) {3381 SDLoc DL(Op.getNode());3382 const MVT GenerictVT =3383 getPointerTy(DAG.getDataLayout(), ADDRESS_SPACE_GENERIC);3384 SDValue GenericConversion = DAG.getAddrSpaceCast(3385 DL, GenerictVT, Op.getOperand(0), SrcAS, ADDRESS_SPACE_GENERIC);3386 SDValue SharedClusterConversion =3387 DAG.getAddrSpaceCast(DL, Op.getValueType(), GenericConversion,3388 ADDRESS_SPACE_GENERIC, DestAS);3389 return SharedClusterConversion;3390 }3391 3392 return DAG.getUNDEF(Op.getValueType());3393 }3394 3395 return Op;3396}3397 3398// This function is almost a copy of SelectionDAG::expandVAArg().3399// The only diff is that this one produces loads from local address space.3400SDValue NVPTXTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {3401 const TargetLowering *TLI = STI.getTargetLowering();3402 SDLoc DL(Op);3403 3404 SDNode *Node = Op.getNode();3405 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();3406 EVT VT = Node->getValueType(0);3407 auto *Ty = VT.getTypeForEVT(*DAG.getContext());3408 SDValue Tmp1 = Node->getOperand(0);3409 SDValue Tmp2 = Node->getOperand(1);3410 const MaybeAlign MA(Node->getConstantOperandVal(3));3411 3412 SDValue VAListLoad = DAG.getLoad(TLI->getPointerTy(DAG.getDataLayout()), DL,3413 Tmp1, Tmp2, MachinePointerInfo(V));3414 SDValue VAList = VAListLoad;3415 3416 if (MA && *MA > TLI->getMinStackArgumentAlignment()) {3417 VAList = DAG.getNode(3418 ISD::ADD, DL, VAList.getValueType(), VAList,3419 DAG.getConstant(MA->value() - 1, DL, VAList.getValueType()));3420 3421 VAList = DAG.getNode(ISD::AND, DL, VAList.getValueType(), VAList,3422 DAG.getSignedConstant(-(int64_t)MA->value(), DL,3423 VAList.getValueType()));3424 }3425 3426 // Increment the pointer, VAList, to the next vaarg3427 Tmp1 = DAG.getNode(ISD::ADD, DL, VAList.getValueType(), VAList,3428 DAG.getConstant(DAG.getDataLayout().getTypeAllocSize(Ty),3429 DL, VAList.getValueType()));3430 3431 // Store the incremented VAList to the legalized pointer3432 Tmp1 = DAG.getStore(VAListLoad.getValue(1), DL, Tmp1, Tmp2,3433 MachinePointerInfo(V));3434 3435 const Value *SrcV = Constant::getNullValue(3436 PointerType::get(*DAG.getContext(), ADDRESS_SPACE_LOCAL));3437 3438 // Load the actual argument out of the pointer VAList3439 return DAG.getLoad(VT, DL, Tmp1, VAList, MachinePointerInfo(SrcV));3440}3441 3442SDValue NVPTXTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {3443 const TargetLowering *TLI = STI.getTargetLowering();3444 SDLoc DL(Op);3445 EVT PtrVT = TLI->getPointerTy(DAG.getDataLayout());3446 3447 // Store the address of unsized array <function>_vararg[] in the ap object.3448 SDValue VAReg = getParamSymbol(DAG, /* vararg */ -1, PtrVT);3449 3450 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();3451 return DAG.getStore(Op.getOperand(0), DL, VAReg, Op.getOperand(1),3452 MachinePointerInfo(SV));3453}3454 3455static std::pair<MemSDNode *, uint32_t>3456convertMLOADToLoadWithUsedBytesMask(MemSDNode *N, SelectionDAG &DAG) {3457 SDValue Chain = N->getOperand(0);3458 SDValue BasePtr = N->getOperand(1);3459 SDValue Mask = N->getOperand(3);3460 [[maybe_unused]] SDValue Passthru = N->getOperand(4);3461 3462 SDLoc DL(N);3463 EVT ResVT = N->getValueType(0);3464 assert(ResVT.isVector() && "Masked vector load must have vector type");3465 // While we only expect poison passthru vectors as an input to the backend,3466 // when the legalization framework splits a poison vector in half, it creates3467 // two undef vectors, so we can technically expect those too.3468 assert((Passthru.getOpcode() == ISD::POISON ||3469 Passthru.getOpcode() == ISD::UNDEF) &&3470 "Passthru operand expected to be poison or undef");3471 3472 // Extract the mask and convert it to a uint32_t representing the used bytes3473 // of the entire vector load3474 uint32_t UsedBytesMask = 0;3475 uint32_t ElementSizeInBits = ResVT.getVectorElementType().getSizeInBits();3476 assert(ElementSizeInBits % 8 == 0 && "Unexpected element size");3477 uint32_t ElementSizeInBytes = ElementSizeInBits / 8;3478 uint32_t ElementMask = (1u << ElementSizeInBytes) - 1u;3479 3480 for (SDValue Op : reverse(Mask->ops())) {3481 // We technically only want to do this shift for every3482 // iteration *but* the first, but in the first iteration UsedBytesMask is 0,3483 // so this shift is a no-op.3484 UsedBytesMask <<= ElementSizeInBytes;3485 3486 // Mask elements must be constants.3487 if (Op->getAsZExtVal() != 0)3488 UsedBytesMask |= ElementMask;3489 }3490 3491 assert(UsedBytesMask != 0 && UsedBytesMask != UINT32_MAX &&3492 "Unexpected masked load with elements masked all on or all off");3493 3494 // Create a new load sd node to be handled normally by ReplaceLoadVector.3495 MemSDNode *NewLD = cast<MemSDNode>(3496 DAG.getLoad(ResVT, DL, Chain, BasePtr, N->getMemOperand()).getNode());3497 3498 return {NewLD, UsedBytesMask};3499}3500 3501/// replaceLoadVector - Convert vector loads into multi-output scalar loads.3502static std::optional<std::pair<SDValue, SDValue>>3503replaceLoadVector(SDNode *N, SelectionDAG &DAG, const NVPTXSubtarget &STI) {3504 MemSDNode *LD = cast<MemSDNode>(N);3505 const EVT ResVT = LD->getValueType(0);3506 const EVT MemVT = LD->getMemoryVT();3507 3508 // If we're doing sign/zero extension as part of the load, avoid lowering to3509 // a LoadV node. TODO: consider relaxing this restriction.3510 if (ResVT != MemVT)3511 return std::nullopt;3512 3513 const auto NumEltsAndEltVT =3514 getVectorLoweringShape(ResVT, STI, LD->getAddressSpace());3515 if (!NumEltsAndEltVT)3516 return std::nullopt;3517 const auto [NumElts, EltVT] = NumEltsAndEltVT.value();3518 3519 Align Alignment = LD->getAlign();3520 const auto &TD = DAG.getDataLayout();3521 Align PrefAlign = TD.getPrefTypeAlign(MemVT.getTypeForEVT(*DAG.getContext()));3522 if (Alignment < PrefAlign) {3523 // This load is not sufficiently aligned, so bail out and let this vector3524 // load be scalarized. Note that we may still be able to emit smaller3525 // vector loads. For example, if we are loading a <4 x float> with an3526 // alignment of 8, this check will fail but the legalizer will try again3527 // with 2 x <2 x float>, which will succeed with an alignment of 8.3528 return std::nullopt;3529 }3530 3531 // If we have a masked load, convert it to a normal load now3532 std::optional<uint32_t> UsedBytesMask = std::nullopt;3533 if (LD->getOpcode() == ISD::MLOAD)3534 std::tie(LD, UsedBytesMask) = convertMLOADToLoadWithUsedBytesMask(LD, DAG);3535 3536 // Since LoadV2 is a target node, we cannot rely on DAG type legalization.3537 // Therefore, we must ensure the type is legal. For i1 and i8, we set the3538 // loaded type to i16 and propagate the "real" type as the memory type.3539 const MVT LoadEltVT = (EltVT.getSizeInBits() < 16) ? MVT::i16 : EltVT;3540 3541 unsigned Opcode;3542 switch (NumElts) {3543 default:3544 return std::nullopt;3545 case 2:3546 Opcode = NVPTXISD::LoadV2;3547 break;3548 case 4:3549 Opcode = NVPTXISD::LoadV4;3550 break;3551 case 8:3552 Opcode = NVPTXISD::LoadV8;3553 break;3554 }3555 auto ListVTs = SmallVector<EVT, 9>(NumElts, LoadEltVT);3556 ListVTs.push_back(MVT::Other);3557 SDVTList LdResVTs = DAG.getVTList(ListVTs);3558 3559 SDLoc DL(LD);3560 3561 // Copy regular operands3562 SmallVector<SDValue, 8> OtherOps(LD->ops());3563 3564 OtherOps.push_back(3565 DAG.getConstant(UsedBytesMask.value_or(UINT32_MAX), DL, MVT::i32));3566 3567 // The select routine does not have access to the LoadSDNode instance, so3568 // pass along the extension information3569 OtherOps.push_back(3570 DAG.getIntPtrConstant(cast<LoadSDNode>(LD)->getExtensionType(), DL));3571 3572 SDValue NewLD = DAG.getMemIntrinsicNode(Opcode, DL, LdResVTs, OtherOps, MemVT,3573 LD->getMemOperand());3574 3575 SmallVector<SDValue> ScalarRes;3576 if (EltVT.isVector()) {3577 assert(EVT(EltVT.getVectorElementType()) == ResVT.getVectorElementType());3578 assert(NumElts * EltVT.getVectorNumElements() ==3579 ResVT.getVectorNumElements());3580 // Generate EXTRACT_VECTOR_ELTs to split v2[i,f,bf]16/v4i8 subvectors back3581 // into individual elements.3582 for (const unsigned I : llvm::seq(NumElts)) {3583 SDValue SubVector = NewLD.getValue(I);3584 DAG.ExtractVectorElements(SubVector, ScalarRes);3585 }3586 } else {3587 for (const unsigned I : llvm::seq(NumElts)) {3588 SDValue Res = NewLD.getValue(I);3589 if (LoadEltVT != EltVT)3590 Res = DAG.getNode(ISD::TRUNCATE, DL, EltVT, Res);3591 ScalarRes.push_back(Res);3592 }3593 }3594 3595 SDValue LoadChain = NewLD.getValue(NumElts);3596 3597 const MVT BuildVecVT =3598 MVT::getVectorVT(EltVT.getScalarType(), ScalarRes.size());3599 SDValue BuildVec = DAG.getBuildVector(BuildVecVT, DL, ScalarRes);3600 SDValue LoadValue = DAG.getBitcast(ResVT, BuildVec);3601 3602 return {{LoadValue, LoadChain}};3603}3604 3605static void replaceLoadVector(SDNode *N, SelectionDAG &DAG,3606 SmallVectorImpl<SDValue> &Results,3607 const NVPTXSubtarget &STI) {3608 if (auto Res = replaceLoadVector(N, DAG, STI))3609 Results.append({Res->first, Res->second});3610}3611 3612static SDValue lowerLoadVector(SDNode *N, SelectionDAG &DAG,3613 const NVPTXSubtarget &STI) {3614 if (auto Res = replaceLoadVector(N, DAG, STI))3615 return DAG.getMergeValues({Res->first, Res->second}, SDLoc(N));3616 return SDValue();3617}3618 3619// v = ld i1* addr3620// =>3621// v1 = ld i8* addr (-> i16)3622// v = trunc i16 to i13623static SDValue lowerLOADi1(LoadSDNode *LD, SelectionDAG &DAG) {3624 SDLoc dl(LD);3625 assert(LD->getExtensionType() == ISD::NON_EXTLOAD);3626 assert(LD->getValueType(0) == MVT::i1 && "Custom lowering for i1 load only");3627 SDValue newLD = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i16, LD->getChain(),3628 LD->getBasePtr(), LD->getPointerInfo(),3629 MVT::i8, LD->getAlign(),3630 LD->getMemOperand()->getFlags());3631 SDValue result = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, newLD);3632 // The legalizer (the caller) is expecting two values from the legalized3633 // load, so we build a MergeValues node for it. See ExpandUnalignedLoad()3634 // in LegalizeDAG.cpp which also uses MergeValues.3635 return DAG.getMergeValues({result, LD->getChain()}, dl);3636}3637 3638SDValue NVPTXTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {3639 LoadSDNode *LD = cast<LoadSDNode>(Op);3640 3641 if (Op.getValueType() == MVT::i1)3642 return lowerLOADi1(LD, DAG);3643 3644 // To improve CodeGen we'll legalize any-extend loads to zext loads. This is3645 // how they'll be lowered in ISel anyway, and by doing this a little earlier3646 // we allow for more DAG combine opportunities.3647 if (LD->getExtensionType() == ISD::EXTLOAD) {3648 assert(LD->getValueType(0).isInteger() && LD->getMemoryVT().isInteger() &&3649 "Unexpected fpext-load");3650 return DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(Op), Op.getValueType(),3651 LD->getChain(), LD->getBasePtr(), LD->getMemoryVT(),3652 LD->getMemOperand());3653 }3654 3655 llvm_unreachable("Unexpected custom lowering for load");3656}3657 3658SDValue NVPTXTargetLowering::LowerMLOAD(SDValue Op, SelectionDAG &DAG) const {3659 // v2f16/v2bf16/v2i16/v4i8 are legal, so we can't rely on legalizer to handle3660 // masked loads of these types and have to handle them here.3661 // v2f32 also needs to be handled here if the subtarget has f32x23662 // instructions, making it legal.3663 //3664 // Note: misaligned masked loads should never reach this point3665 // because the override of isLegalMaskedLoad in NVPTXTargetTransformInfo.cpp3666 // will validate alignment. Therefore, we do not need to special case handle3667 // them here.3668 EVT VT = Op.getValueType();3669 if (NVPTX::isPackedVectorTy(VT)) {3670 auto Result =3671 convertMLOADToLoadWithUsedBytesMask(cast<MemSDNode>(Op.getNode()), DAG);3672 MemSDNode *LD = std::get<0>(Result);3673 uint32_t UsedBytesMask = std::get<1>(Result);3674 3675 SDLoc DL(LD);3676 3677 // Copy regular operands3678 SmallVector<SDValue, 8> OtherOps(LD->ops());3679 3680 OtherOps.push_back(DAG.getConstant(UsedBytesMask, DL, MVT::i32));3681 3682 // We currently are not lowering extending loads, but pass the extension3683 // type anyway as later handling expects it.3684 OtherOps.push_back(3685 DAG.getIntPtrConstant(cast<LoadSDNode>(LD)->getExtensionType(), DL));3686 SDValue NewLD =3687 DAG.getMemIntrinsicNode(NVPTXISD::MLoad, DL, LD->getVTList(), OtherOps,3688 LD->getMemoryVT(), LD->getMemOperand());3689 return NewLD;3690 }3691 return SDValue();3692}3693 3694static SDValue lowerSTOREVector(SDValue Op, SelectionDAG &DAG,3695 const NVPTXSubtarget &STI) {3696 MemSDNode *N = cast<MemSDNode>(Op.getNode());3697 SDValue Val = N->getOperand(1);3698 SDLoc DL(N);3699 const EVT ValVT = Val.getValueType();3700 const EVT MemVT = N->getMemoryVT();3701 3702 // If we're truncating as part of the store, avoid lowering to a StoreV node.3703 // TODO: consider relaxing this restriction.3704 if (ValVT != MemVT)3705 return SDValue();3706 3707 const auto NumEltsAndEltVT =3708 getVectorLoweringShape(ValVT, STI, N->getAddressSpace());3709 if (!NumEltsAndEltVT)3710 return SDValue();3711 const auto [NumElts, EltVT] = NumEltsAndEltVT.value();3712 3713 const DataLayout &TD = DAG.getDataLayout();3714 3715 Align Alignment = N->getAlign();3716 Align PrefAlign = TD.getPrefTypeAlign(ValVT.getTypeForEVT(*DAG.getContext()));3717 if (Alignment < PrefAlign) {3718 // This store is not sufficiently aligned, so bail out and let this vector3719 // store be scalarized. Note that we may still be able to emit smaller3720 // vector stores. For example, if we are storing a <4 x float> with an3721 // alignment of 8, this check will fail but the legalizer will try again3722 // with 2 x <2 x float>, which will succeed with an alignment of 8.3723 return SDValue();3724 }3725 3726 unsigned Opcode;3727 switch (NumElts) {3728 default:3729 return SDValue();3730 case 2:3731 Opcode = NVPTXISD::StoreV2;3732 break;3733 case 4:3734 Opcode = NVPTXISD::StoreV4;3735 break;3736 case 8:3737 Opcode = NVPTXISD::StoreV8;3738 break;3739 }3740 3741 SmallVector<SDValue, 8> Ops;3742 3743 // First is the chain3744 Ops.push_back(N->getOperand(0));3745 3746 // Then the split values3747 if (EltVT.isVector()) {3748 assert(EVT(EltVT.getVectorElementType()) == ValVT.getVectorElementType());3749 assert(NumElts * EltVT.getVectorNumElements() ==3750 ValVT.getVectorNumElements());3751 // Combine individual elements into v2[i,f,bf]16/v4i8 subvectors to be3752 // stored as b32s3753 const unsigned NumEltsPerSubVector = EltVT.getVectorNumElements();3754 for (const unsigned I : llvm::seq(NumElts)) {3755 SmallVector<SDValue, 4> SubVectorElts;3756 DAG.ExtractVectorElements(Val, SubVectorElts, I * NumEltsPerSubVector,3757 NumEltsPerSubVector);3758 Ops.push_back(DAG.getBuildVector(EltVT, DL, SubVectorElts));3759 }3760 } else {3761 SDValue V = DAG.getBitcast(MVT::getVectorVT(EltVT, NumElts), Val);3762 for (const unsigned I : llvm::seq(NumElts)) {3763 SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, V,3764 DAG.getIntPtrConstant(I, DL));3765 3766 // Since StoreV2 is a target node, we cannot rely on DAG type3767 // legalization. Therefore, we must ensure the type is legal. For i1 and3768 // i8, we set the stored type to i16 and propagate the "real" type as the3769 // memory type.3770 if (EltVT.getSizeInBits() < 16)3771 ExtVal = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i16, ExtVal);3772 Ops.push_back(ExtVal);3773 }3774 }3775 3776 // Then any remaining arguments3777 Ops.append(N->op_begin() + 2, N->op_end());3778 3779 SDValue NewSt =3780 DAG.getMemIntrinsicNode(Opcode, DL, DAG.getVTList(MVT::Other), Ops,3781 N->getMemoryVT(), N->getMemOperand());3782 3783 // return DCI.CombineTo(N, NewSt, true);3784 return NewSt;3785}3786 3787SDValue NVPTXTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {3788 StoreSDNode *Store = cast<StoreSDNode>(Op);3789 EVT VT = Store->getMemoryVT();3790 3791 if (VT == MVT::i1)3792 return LowerSTOREi1(Op, DAG);3793 3794 // Lower store of any other vector type, including v2f32 as we want to break3795 // it apart since this is not a widely-supported type.3796 return lowerSTOREVector(Op, DAG, STI);3797}3798 3799// st i1 v, addr3800// =>3801// v1 = zxt v to i163802// st.u8 i16, addr3803SDValue NVPTXTargetLowering::LowerSTOREi1(SDValue Op, SelectionDAG &DAG) const {3804 SDNode *Node = Op.getNode();3805 SDLoc dl(Node);3806 StoreSDNode *ST = cast<StoreSDNode>(Node);3807 SDValue Tmp1 = ST->getChain();3808 SDValue Tmp2 = ST->getBasePtr();3809 SDValue Tmp3 = ST->getValue();3810 assert(Tmp3.getValueType() == MVT::i1 && "Custom lowering for i1 store only");3811 Tmp3 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Tmp3);3812 SDValue Result =3813 DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2, ST->getPointerInfo(), MVT::i8,3814 ST->getAlign(), ST->getMemOperand()->getFlags());3815 return Result;3816}3817 3818SDValue NVPTXTargetLowering::LowerCopyToReg_128(SDValue Op,3819 SelectionDAG &DAG) const {3820 // Change the CopyToReg to take in two 64-bit operands instead of a 128-bit3821 // operand so that it can pass the legalization.3822 3823 assert(Op.getOperand(1).getValueType() == MVT::i128 &&3824 "Custom lowering for 128-bit CopyToReg only");3825 3826 SDNode *Node = Op.getNode();3827 SDLoc DL(Node);3828 3829 SDValue Cast = DAG.getBitcast(MVT::v2i64, Op->getOperand(2));3830 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, Cast,3831 DAG.getIntPtrConstant(0, DL));3832 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, Cast,3833 DAG.getIntPtrConstant(1, DL));3834 3835 SmallVector<SDValue, 5> NewOps(Op->getNumOperands() + 1);3836 SmallVector<EVT, 3> ResultsType(Node->values());3837 3838 NewOps[0] = Op->getOperand(0); // Chain3839 NewOps[1] = Op->getOperand(1); // Dst Reg3840 NewOps[2] = Lo; // Lower 64-bit3841 NewOps[3] = Hi; // Higher 64-bit3842 if (Op.getNumOperands() == 4)3843 NewOps[4] = Op->getOperand(3); // Glue if exists3844 3845 return DAG.getNode(ISD::CopyToReg, DL, ResultsType, NewOps);3846}3847 3848unsigned NVPTXTargetLowering::getNumRegisters(3849 LLVMContext &Context, EVT VT,3850 std::optional<MVT> RegisterVT = std::nullopt) const {3851 if (VT == MVT::i128 && RegisterVT == MVT::i128)3852 return 1;3853 return TargetLoweringBase::getNumRegisters(Context, VT, RegisterVT);3854}3855 3856bool NVPTXTargetLowering::splitValueIntoRegisterParts(3857 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,3858 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {3859 if (Val.getValueType() == MVT::i128 && NumParts == 1) {3860 Parts[0] = Val;3861 return true;3862 }3863 return false;3864}3865 3866// This creates target external symbol for a function parameter.3867// Name of the symbol is composed from its index and the function name.3868// Negative index corresponds to special parameter (unsized array) used for3869// passing variable arguments.3870SDValue NVPTXTargetLowering::getParamSymbol(SelectionDAG &DAG, int I,3871 EVT T) const {3872 StringRef SavedStr = nvTM->getStrPool().save(3873 getParamName(&DAG.getMachineFunction().getFunction(), I));3874 return DAG.getExternalSymbol(SavedStr.data(), T);3875}3876 3877SDValue NVPTXTargetLowering::getCallParamSymbol(SelectionDAG &DAG, int I,3878 EVT T) const {3879 const StringRef SavedStr = nvTM->getStrPool().save("param" + Twine(I));3880 return DAG.getExternalSymbol(SavedStr.data(), T);3881}3882 3883SDValue NVPTXTargetLowering::LowerFormalArguments(3884 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,3885 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,3886 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {3887 const DataLayout &DL = DAG.getDataLayout();3888 LLVMContext &Ctx = *DAG.getContext();3889 auto PtrVT = getPointerTy(DAG.getDataLayout());3890 3891 const Function &F = DAG.getMachineFunction().getFunction();3892 3893 SDValue Root = DAG.getRoot();3894 SmallVector<SDValue, 16> OutChains;3895 3896 // argTypes.size() (or theArgs.size()) and Ins.size() need not match.3897 // Ins.size() will be larger3898 // * if there is an aggregate argument with multiple fields (each field3899 // showing up separately in Ins)3900 // * if there is a vector argument with more than typical vector-length3901 // elements (generally if more than 4) where each vector element is3902 // individually present in Ins.3903 // So a different index should be used for indexing into Ins.3904 // See similar issue in LowerCall.3905 3906 auto AllIns = ArrayRef(Ins);3907 for (const auto &Arg : F.args()) {3908 const auto ArgIns = AllIns.take_while(3909 [&](auto I) { return I.OrigArgIndex == Arg.getArgNo(); });3910 AllIns = AllIns.drop_front(ArgIns.size());3911 3912 Type *Ty = Arg.getType();3913 3914 if (ArgIns.empty())3915 report_fatal_error("Empty parameter types are not supported");3916 3917 if (Arg.use_empty()) {3918 // argument is dead3919 for (const auto &In : ArgIns) {3920 assert(!In.Used && "Arg.use_empty() is true but Arg is used?");3921 InVals.push_back(DAG.getUNDEF(In.VT));3922 }3923 continue;3924 }3925 3926 SDValue ArgSymbol = getParamSymbol(DAG, Arg.getArgNo(), PtrVT);3927 3928 // In the following cases, assign a node order of "i+1"3929 // to newly created nodes. The SDNodes for params have to3930 // appear in the same order as their order of appearance3931 // in the original function. "i+1" holds that order.3932 if (Arg.hasByValAttr()) {3933 // Param has ByVal attribute3934 // Return MoveParam(param symbol).3935 // Ideally, the param symbol can be returned directly,3936 // but when SDNode builder decides to use it in a CopyToReg(),3937 // machine instruction fails because TargetExternalSymbol3938 // (not lowered) is target dependent, and CopyToReg assumes3939 // the source is lowered.3940 assert(ArgIns.size() == 1 && "ByVal argument must be a pointer");3941 const auto &ByvalIn = ArgIns[0];3942 assert(getValueType(DL, Ty) == ByvalIn.VT &&3943 "Ins type did not match function type");3944 assert(ByvalIn.VT == PtrVT && "ByVal argument must be a pointer");3945 3946 SDValue P;3947 if (isKernelFunction(F)) {3948 P = ArgSymbol;3949 P.getNode()->setIROrder(Arg.getArgNo() + 1);3950 } else {3951 P = DAG.getNode(NVPTXISD::MoveParam, dl, ByvalIn.VT, ArgSymbol);3952 P.getNode()->setIROrder(Arg.getArgNo() + 1);3953 P = DAG.getAddrSpaceCast(dl, ByvalIn.VT, P, ADDRESS_SPACE_LOCAL,3954 ADDRESS_SPACE_GENERIC);3955 }3956 InVals.push_back(P);3957 } else {3958 SmallVector<EVT, 16> VTs;3959 SmallVector<uint64_t, 16> Offsets;3960 ComputePTXValueVTs(*this, DL, Ctx, CallConv, Ty, VTs, Offsets);3961 assert(VTs.size() == ArgIns.size() && "Size mismatch");3962 assert(VTs.size() == Offsets.size() && "Size mismatch");3963 3964 const Align ArgAlign = getFunctionArgumentAlignment(3965 &F, Ty, Arg.getArgNo() + AttributeList::FirstArgIndex, DL);3966 3967 unsigned I = 0;3968 const auto VI = VectorizePTXValueVTs(VTs, Offsets, ArgAlign);3969 for (const unsigned NumElts : VI) {3970 // i1 is loaded/stored as i83971 const EVT LoadVT = VTs[I] == MVT::i1 ? MVT::i8 : VTs[I];3972 const EVT VecVT = getVectorizedVT(LoadVT, NumElts, Ctx);3973 3974 SDValue VecAddr = DAG.getObjectPtrOffset(3975 dl, ArgSymbol, TypeSize::getFixed(Offsets[I]));3976 3977 const Align PartAlign = commonAlignment(ArgAlign, Offsets[I]);3978 SDValue P =3979 DAG.getLoad(VecVT, dl, Root, VecAddr,3980 MachinePointerInfo(ADDRESS_SPACE_PARAM), PartAlign,3981 MachineMemOperand::MODereferenceable |3982 MachineMemOperand::MOInvariant);3983 P.getNode()->setIROrder(Arg.getArgNo() + 1);3984 for (const unsigned J : llvm::seq(NumElts)) {3985 SDValue Elt = getExtractVectorizedValue(P, J, LoadVT, dl, DAG);3986 3987 Elt = correctParamType(Elt, ArgIns[I + J].VT, ArgIns[I + J].Flags,3988 DAG, dl);3989 InVals.push_back(Elt);3990 }3991 I += NumElts;3992 }3993 }3994 }3995 3996 if (!OutChains.empty())3997 DAG.setRoot(DAG.getTokenFactor(dl, OutChains));3998 3999 return Chain;4000}4001 4002SDValue4003NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,4004 bool isVarArg,4005 const SmallVectorImpl<ISD::OutputArg> &Outs,4006 const SmallVectorImpl<SDValue> &OutVals,4007 const SDLoc &dl, SelectionDAG &DAG) const {4008 const Function &F = DAG.getMachineFunction().getFunction();4009 Type *RetTy = F.getReturnType();4010 4011 if (RetTy->isVoidTy()) {4012 assert(OutVals.empty() && Outs.empty() && "Return value expected for void");4013 return DAG.getNode(NVPTXISD::RET_GLUE, dl, MVT::Other, Chain);4014 }4015 4016 const DataLayout &DL = DAG.getDataLayout();4017 LLVMContext &Ctx = *DAG.getContext();4018 4019 const SDValue RetSymbol = DAG.getExternalSymbol("func_retval0", MVT::i32);4020 const auto RetAlign = getFunctionParamOptimizedAlign(&F, RetTy, DL);4021 4022 // PTX Interoperability Guide 3.3(A): [Integer] Values shorter than4023 // 32-bits are sign extended or zero extended, depending on whether4024 // they are signed or unsigned types.4025 const bool ExtendIntegerRetVal =4026 RetTy->isIntegerTy() && DL.getTypeAllocSizeInBits(RetTy) < 32;4027 4028 SmallVector<EVT, 16> VTs;4029 SmallVector<uint64_t, 16> Offsets;4030 ComputePTXValueVTs(*this, DL, Ctx, CallConv, RetTy, VTs, Offsets);4031 assert(VTs.size() == OutVals.size() && "Bad return value decomposition");4032 4033 const auto GetRetVal = [&](unsigned I) -> SDValue {4034 SDValue RetVal = OutVals[I];4035 assert(promoteScalarIntegerPTX(RetVal.getValueType()) ==4036 RetVal.getValueType() &&4037 "OutVal type should always be legal");4038 4039 const EVT VTI = promoteScalarIntegerPTX(VTs[I]);4040 const EVT StoreVT =4041 ExtendIntegerRetVal ? MVT::i32 : (VTI == MVT::i1 ? MVT::i8 : VTI);4042 return correctParamType(RetVal, StoreVT, Outs[I].Flags, DAG, dl);4043 };4044 4045 unsigned I = 0;4046 const auto VI = VectorizePTXValueVTs(VTs, Offsets, RetAlign);4047 for (const unsigned NumElts : VI) {4048 const MaybeAlign CurrentAlign = ExtendIntegerRetVal4049 ? MaybeAlign(std::nullopt)4050 : commonAlignment(RetAlign, Offsets[I]);4051 4052 SDValue Val = getBuildVectorizedValue(4053 NumElts, dl, DAG, [&](unsigned K) { return GetRetVal(I + K); });4054 4055 SDValue Ptr =4056 DAG.getObjectPtrOffset(dl, RetSymbol, TypeSize::getFixed(Offsets[I]));4057 4058 Chain = DAG.getStore(Chain, dl, Val, Ptr,4059 MachinePointerInfo(ADDRESS_SPACE_PARAM), CurrentAlign);4060 4061 I += NumElts;4062 }4063 4064 return DAG.getNode(NVPTXISD::RET_GLUE, dl, MVT::Other, Chain);4065}4066 4067void NVPTXTargetLowering::LowerAsmOperandForConstraint(4068 SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops,4069 SelectionDAG &DAG) const {4070 if (Constraint.size() > 1)4071 return;4072 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);4073}4074 4075// llvm.ptx.memcpy.const and llvm.ptx.memmove.const need to be modeled as4076// TgtMemIntrinsic4077// because we need the information that is only available in the "Value" type4078// of destination4079// pointer. In particular, the address space information.4080bool NVPTXTargetLowering::getTgtMemIntrinsic(4081 IntrinsicInfo &Info, const CallInst &I,4082 MachineFunction &MF, unsigned Intrinsic) const {4083 switch (Intrinsic) {4084 default:4085 return false;4086 case Intrinsic::nvvm_match_all_sync_i32p:4087 case Intrinsic::nvvm_match_all_sync_i64p:4088 Info.opc = ISD::INTRINSIC_W_CHAIN;4089 // memVT is bogus. These intrinsics have IntrInaccessibleMemOnly attribute4090 // in order to model data exchange with other threads, but perform no real4091 // memory accesses.4092 Info.memVT = MVT::i1;4093 4094 // Our result depends on both our and other thread's arguments.4095 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;4096 return true;4097 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_col:4098 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_row:4099 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_col_stride:4100 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_row_stride:4101 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_col:4102 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_row:4103 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_col_stride:4104 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_row_stride:4105 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_col:4106 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_row:4107 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_col_stride:4108 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_row_stride:4109 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_col:4110 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_row:4111 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_col_stride:4112 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_row_stride:4113 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_col:4114 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_row:4115 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_col_stride:4116 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_row_stride:4117 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_col:4118 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_row:4119 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_col_stride:4120 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_row_stride: {4121 Info.opc = ISD::INTRINSIC_W_CHAIN;4122 Info.memVT = MVT::v8f16;4123 Info.ptrVal = I.getArgOperand(0);4124 Info.offset = 0;4125 Info.flags = MachineMemOperand::MOLoad;4126 Info.align = Align(16);4127 return true;4128 }4129 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_col:4130 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_col_stride:4131 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_col_stride:4132 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_col:4133 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_row:4134 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_row_stride:4135 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_row_stride:4136 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_row:4137 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_col:4138 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_col_stride:4139 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_row:4140 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_row_stride:4141 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_col:4142 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_col_stride:4143 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_col_stride:4144 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_col:4145 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_row:4146 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_row_stride:4147 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_row_stride:4148 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_row:4149 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_col:4150 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_col_stride:4151 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_row:4152 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_row_stride: {4153 Info.opc = ISD::INTRINSIC_W_CHAIN;4154 Info.memVT = MVT::v2i32;4155 Info.ptrVal = I.getArgOperand(0);4156 Info.offset = 0;4157 Info.flags = MachineMemOperand::MOLoad;4158 Info.align = Align(8);4159 return true;4160 }4161 4162 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_col:4163 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_col_stride:4164 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_col_stride:4165 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_col:4166 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_row:4167 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_row_stride:4168 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_row_stride:4169 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_row:4170 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_col:4171 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_col_stride:4172 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_row:4173 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_row_stride:4174 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_col:4175 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_col_stride:4176 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_row:4177 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_row_stride:4178 4179 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_col:4180 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_col_stride:4181 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_col_stride:4182 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_col:4183 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_row:4184 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_row_stride:4185 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_row_stride:4186 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_row:4187 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_col:4188 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_col_stride:4189 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_row:4190 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_row_stride:4191 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_col:4192 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_col_stride:4193 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_row:4194 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_row_stride:4195 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x4_b16:4196 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x4_trans_b16:4197 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8:4198 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8x16_b4x16_p64:4199 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8x16_b6x16_p32:4200 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x4_b8x16_b4x16_p64:4201 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x4_b8x16_b6x16_p32: {4202 Info.opc = ISD::INTRINSIC_W_CHAIN;4203 Info.memVT = MVT::v4i32;4204 Info.ptrVal = I.getArgOperand(0);4205 Info.offset = 0;4206 Info.flags = MachineMemOperand::MOLoad;4207 Info.align = Align(16);4208 return true;4209 }4210 4211 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_col:4212 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_col_stride:4213 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_col_stride:4214 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_col:4215 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_row:4216 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_row_stride:4217 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_row_stride:4218 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_row:4219 4220 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_col:4221 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_col_stride:4222 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_col_stride:4223 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_col:4224 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_row:4225 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_row_stride:4226 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_row_stride:4227 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_row:4228 case Intrinsic::nvvm_wmma_m8n8k128_load_a_b1_row:4229 case Intrinsic::nvvm_wmma_m8n8k128_load_a_b1_row_stride:4230 case Intrinsic::nvvm_wmma_m8n8k128_load_b_b1_col:4231 case Intrinsic::nvvm_wmma_m8n8k128_load_b_b1_col_stride:4232 case Intrinsic::nvvm_wmma_m8n8k32_load_a_s4_row:4233 case Intrinsic::nvvm_wmma_m8n8k32_load_a_s4_row_stride:4234 case Intrinsic::nvvm_wmma_m8n8k32_load_a_u4_row_stride:4235 case Intrinsic::nvvm_wmma_m8n8k32_load_a_u4_row:4236 case Intrinsic::nvvm_wmma_m8n8k32_load_b_s4_col:4237 case Intrinsic::nvvm_wmma_m8n8k32_load_b_s4_col_stride:4238 case Intrinsic::nvvm_wmma_m8n8k32_load_b_u4_col_stride:4239 case Intrinsic::nvvm_wmma_m8n8k32_load_b_u4_col:4240 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x1_b16:4241 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x1_trans_b16:4242 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x1_b8x16_b4x16_p64:4243 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x1_b8x16_b6x16_p32: {4244 Info.opc = ISD::INTRINSIC_W_CHAIN;4245 Info.memVT = MVT::i32;4246 Info.ptrVal = I.getArgOperand(0);4247 Info.offset = 0;4248 Info.flags = MachineMemOperand::MOLoad;4249 Info.align = Align(4);4250 return true;4251 }4252 4253 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_col:4254 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_row:4255 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_col_stride:4256 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_row_stride:4257 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_col:4258 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_row:4259 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_col_stride:4260 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_row_stride:4261 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_col:4262 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_row:4263 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_col_stride:4264 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_row_stride: {4265 Info.opc = ISD::INTRINSIC_W_CHAIN;4266 Info.memVT = MVT::v4f16;4267 Info.ptrVal = I.getArgOperand(0);4268 Info.offset = 0;4269 Info.flags = MachineMemOperand::MOLoad;4270 Info.align = Align(16);4271 return true;4272 }4273 4274 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_col:4275 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_row:4276 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_col_stride:4277 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_row_stride:4278 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_col:4279 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_row:4280 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_col_stride:4281 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_row_stride:4282 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_col:4283 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_row:4284 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_col_stride:4285 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_row_stride:4286 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_col:4287 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_row:4288 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_col_stride:4289 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_row_stride: {4290 Info.opc = ISD::INTRINSIC_W_CHAIN;4291 Info.memVT = MVT::v8f32;4292 Info.ptrVal = I.getArgOperand(0);4293 Info.offset = 0;4294 Info.flags = MachineMemOperand::MOLoad;4295 Info.align = Align(16);4296 return true;4297 }4298 4299 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_col:4300 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_col_stride:4301 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_row:4302 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_row_stride:4303 4304 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_col:4305 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_col_stride:4306 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_row:4307 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_row_stride:4308 4309 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_col:4310 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_col_stride:4311 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_row:4312 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_row_stride:4313 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_col:4314 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_col_stride:4315 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_row:4316 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_row_stride:4317 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_col:4318 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_col_stride:4319 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_row:4320 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_row_stride: {4321 Info.opc = ISD::INTRINSIC_W_CHAIN;4322 Info.memVT = MVT::v8i32;4323 Info.ptrVal = I.getArgOperand(0);4324 Info.offset = 0;4325 Info.flags = MachineMemOperand::MOLoad;4326 Info.align = Align(16);4327 return true;4328 }4329 4330 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_col:4331 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_col_stride:4332 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_row:4333 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_row_stride:4334 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_col:4335 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_col_stride:4336 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_row:4337 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_row_stride:4338 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x2_b16:4339 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x2_trans_b16:4340 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8:4341 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8x16_b4x16_p64:4342 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8x16_b6x16_p32:4343 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x2_b8x16_b4x16_p64:4344 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x2_b8x16_b6x16_p32: {4345 Info.opc = ISD::INTRINSIC_W_CHAIN;4346 Info.memVT = MVT::v2i32;4347 Info.ptrVal = I.getArgOperand(0);4348 Info.offset = 0;4349 Info.flags = MachineMemOperand::MOLoad;4350 Info.align = Align(8);4351 return true;4352 }4353 4354 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_col:4355 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_col_stride:4356 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_row:4357 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_row_stride:4358 4359 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_col:4360 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_col_stride:4361 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_row:4362 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_row_stride: {4363 Info.opc = ISD::INTRINSIC_W_CHAIN;4364 Info.memVT = MVT::f64;4365 Info.ptrVal = I.getArgOperand(0);4366 Info.offset = 0;4367 Info.flags = MachineMemOperand::MOLoad;4368 Info.align = Align(8);4369 return true;4370 }4371 4372 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_col:4373 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_col_stride:4374 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_row:4375 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_row_stride: {4376 Info.opc = ISD::INTRINSIC_W_CHAIN;4377 Info.memVT = MVT::v2f64;4378 Info.ptrVal = I.getArgOperand(0);4379 Info.offset = 0;4380 Info.flags = MachineMemOperand::MOLoad;4381 Info.align = Align(16);4382 return true;4383 }4384 4385 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_col:4386 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_row:4387 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_col_stride:4388 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_row_stride:4389 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_col:4390 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_row:4391 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_col_stride:4392 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_row_stride:4393 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_col:4394 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_row:4395 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_col_stride:4396 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_row_stride: {4397 Info.opc = ISD::INTRINSIC_VOID;4398 Info.memVT = MVT::v4f16;4399 Info.ptrVal = I.getArgOperand(0);4400 Info.offset = 0;4401 Info.flags = MachineMemOperand::MOStore;4402 Info.align = Align(16);4403 return true;4404 }4405 4406 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_col:4407 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_row:4408 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_col_stride:4409 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_row_stride:4410 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_col:4411 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_row:4412 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_col_stride:4413 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_row_stride:4414 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_col:4415 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_row:4416 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_col_stride:4417 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_row_stride:4418 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_col:4419 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_row:4420 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_col_stride:4421 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_row_stride: {4422 Info.opc = ISD::INTRINSIC_VOID;4423 Info.memVT = MVT::v8f32;4424 Info.ptrVal = I.getArgOperand(0);4425 Info.offset = 0;4426 Info.flags = MachineMemOperand::MOStore;4427 Info.align = Align(16);4428 return true;4429 }4430 4431 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_col:4432 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_col_stride:4433 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_row:4434 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_row_stride:4435 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_col:4436 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_col_stride:4437 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_row:4438 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_row_stride:4439 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_col:4440 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_col_stride:4441 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_row:4442 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_row_stride: {4443 Info.opc = ISD::INTRINSIC_VOID;4444 Info.memVT = MVT::v8i32;4445 Info.ptrVal = I.getArgOperand(0);4446 Info.offset = 0;4447 Info.flags = MachineMemOperand::MOStore;4448 Info.align = Align(16);4449 return true;4450 }4451 4452 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_col:4453 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_col_stride:4454 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_row:4455 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_row_stride:4456 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_col:4457 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_col_stride:4458 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_row:4459 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_row_stride:4460 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x2_b16:4461 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x2_trans_b16:4462 case Intrinsic::nvvm_stmatrix_sync_aligned_m16n8_x2_trans_b8: {4463 Info.opc = ISD::INTRINSIC_VOID;4464 Info.memVT = MVT::v2i32;4465 Info.ptrVal = I.getArgOperand(0);4466 Info.offset = 0;4467 Info.flags = MachineMemOperand::MOStore;4468 Info.align = Align(8);4469 return true;4470 }4471 4472 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_col:4473 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_col_stride:4474 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_row:4475 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_row_stride: {4476 Info.opc = ISD::INTRINSIC_VOID;4477 Info.memVT = MVT::v2f64;4478 Info.ptrVal = I.getArgOperand(0);4479 Info.offset = 0;4480 Info.flags = MachineMemOperand::MOStore;4481 Info.align = Align(16);4482 return true;4483 }4484 4485 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x1_b16:4486 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x1_trans_b16:4487 case Intrinsic::nvvm_stmatrix_sync_aligned_m16n8_x1_trans_b8: {4488 Info.opc = ISD::INTRINSIC_VOID;4489 Info.memVT = MVT::i32;4490 Info.ptrVal = I.getArgOperand(0);4491 Info.offset = 0;4492 Info.flags = MachineMemOperand::MOStore;4493 Info.align = Align(4);4494 return true;4495 }4496 4497 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x4_b16:4498 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x4_trans_b16:4499 case Intrinsic::nvvm_stmatrix_sync_aligned_m16n8_x4_trans_b8: {4500 Info.opc = ISD::INTRINSIC_VOID;4501 Info.memVT = MVT::v4i32;4502 Info.ptrVal = I.getArgOperand(0);4503 Info.offset = 0;4504 Info.flags = MachineMemOperand::MOStore;4505 Info.align = Align(16);4506 return true;4507 }4508 4509 case Intrinsic::nvvm_atomic_add_gen_f_cta:4510 case Intrinsic::nvvm_atomic_add_gen_f_sys:4511 case Intrinsic::nvvm_atomic_add_gen_i_cta:4512 case Intrinsic::nvvm_atomic_add_gen_i_sys:4513 case Intrinsic::nvvm_atomic_and_gen_i_cta:4514 case Intrinsic::nvvm_atomic_and_gen_i_sys:4515 case Intrinsic::nvvm_atomic_cas_gen_i_cta:4516 case Intrinsic::nvvm_atomic_cas_gen_i_sys:4517 case Intrinsic::nvvm_atomic_dec_gen_i_cta:4518 case Intrinsic::nvvm_atomic_dec_gen_i_sys:4519 case Intrinsic::nvvm_atomic_inc_gen_i_cta:4520 case Intrinsic::nvvm_atomic_inc_gen_i_sys:4521 case Intrinsic::nvvm_atomic_max_gen_i_cta:4522 case Intrinsic::nvvm_atomic_max_gen_i_sys:4523 case Intrinsic::nvvm_atomic_min_gen_i_cta:4524 case Intrinsic::nvvm_atomic_min_gen_i_sys:4525 case Intrinsic::nvvm_atomic_or_gen_i_cta:4526 case Intrinsic::nvvm_atomic_or_gen_i_sys:4527 case Intrinsic::nvvm_atomic_exch_gen_i_cta:4528 case Intrinsic::nvvm_atomic_exch_gen_i_sys:4529 case Intrinsic::nvvm_atomic_xor_gen_i_cta:4530 case Intrinsic::nvvm_atomic_xor_gen_i_sys: {4531 auto &DL = I.getDataLayout();4532 Info.opc = ISD::INTRINSIC_W_CHAIN;4533 Info.memVT = getValueType(DL, I.getType());4534 Info.ptrVal = I.getArgOperand(0);4535 Info.offset = 0;4536 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;4537 Info.align.reset();4538 return true;4539 }4540 4541 case Intrinsic::nvvm_prefetch_tensormap: {4542 auto &DL = I.getDataLayout();4543 Info.opc = ISD::INTRINSIC_VOID;4544 Info.memVT = getPointerTy(DL);4545 Info.ptrVal = I.getArgOperand(0);4546 Info.offset = 0;4547 Info.flags =4548 MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable;4549 Info.align.reset();4550 return true;4551 }4552 4553 case Intrinsic::nvvm_ldu_global_i:4554 case Intrinsic::nvvm_ldu_global_f:4555 case Intrinsic::nvvm_ldu_global_p: {4556 Info.opc = ISD::INTRINSIC_W_CHAIN;4557 Info.memVT = getValueType(I.getDataLayout(), I.getType());4558 Info.ptrVal = I.getArgOperand(0);4559 Info.offset = 0;4560 Info.flags = MachineMemOperand::MOLoad;4561 Info.align = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue();4562 4563 return true;4564 }4565 case Intrinsic::nvvm_tex_1d_v4f32_s32:4566 case Intrinsic::nvvm_tex_1d_v4f32_f32:4567 case Intrinsic::nvvm_tex_1d_level_v4f32_f32:4568 case Intrinsic::nvvm_tex_1d_grad_v4f32_f32:4569 case Intrinsic::nvvm_tex_1d_array_v4f32_s32:4570 case Intrinsic::nvvm_tex_1d_array_v4f32_f32:4571 case Intrinsic::nvvm_tex_1d_array_level_v4f32_f32:4572 case Intrinsic::nvvm_tex_1d_array_grad_v4f32_f32:4573 case Intrinsic::nvvm_tex_2d_v4f32_s32:4574 case Intrinsic::nvvm_tex_2d_v4f32_f32:4575 case Intrinsic::nvvm_tex_2d_level_v4f32_f32:4576 case Intrinsic::nvvm_tex_2d_grad_v4f32_f32:4577 case Intrinsic::nvvm_tex_2d_array_v4f32_s32:4578 case Intrinsic::nvvm_tex_2d_array_v4f32_f32:4579 case Intrinsic::nvvm_tex_2d_array_level_v4f32_f32:4580 case Intrinsic::nvvm_tex_2d_array_grad_v4f32_f32:4581 case Intrinsic::nvvm_tex_3d_v4f32_s32:4582 case Intrinsic::nvvm_tex_3d_v4f32_f32:4583 case Intrinsic::nvvm_tex_3d_level_v4f32_f32:4584 case Intrinsic::nvvm_tex_3d_grad_v4f32_f32:4585 case Intrinsic::nvvm_tex_cube_v4f32_f32:4586 case Intrinsic::nvvm_tex_cube_level_v4f32_f32:4587 case Intrinsic::nvvm_tex_cube_array_v4f32_f32:4588 case Intrinsic::nvvm_tex_cube_array_level_v4f32_f32:4589 case Intrinsic::nvvm_tld4_r_2d_v4f32_f32:4590 case Intrinsic::nvvm_tld4_g_2d_v4f32_f32:4591 case Intrinsic::nvvm_tld4_b_2d_v4f32_f32:4592 case Intrinsic::nvvm_tld4_a_2d_v4f32_f32:4593 case Intrinsic::nvvm_tex_unified_1d_v4f32_s32:4594 case Intrinsic::nvvm_tex_unified_1d_v4f32_f32:4595 case Intrinsic::nvvm_tex_unified_1d_level_v4f32_f32:4596 case Intrinsic::nvvm_tex_unified_1d_grad_v4f32_f32:4597 case Intrinsic::nvvm_tex_unified_1d_array_v4f32_s32:4598 case Intrinsic::nvvm_tex_unified_1d_array_v4f32_f32:4599 case Intrinsic::nvvm_tex_unified_1d_array_level_v4f32_f32:4600 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4f32_f32:4601 case Intrinsic::nvvm_tex_unified_2d_v4f32_s32:4602 case Intrinsic::nvvm_tex_unified_2d_v4f32_f32:4603 case Intrinsic::nvvm_tex_unified_2d_level_v4f32_f32:4604 case Intrinsic::nvvm_tex_unified_2d_grad_v4f32_f32:4605 case Intrinsic::nvvm_tex_unified_2d_array_v4f32_s32:4606 case Intrinsic::nvvm_tex_unified_2d_array_v4f32_f32:4607 case Intrinsic::nvvm_tex_unified_2d_array_level_v4f32_f32:4608 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4f32_f32:4609 case Intrinsic::nvvm_tex_unified_3d_v4f32_s32:4610 case Intrinsic::nvvm_tex_unified_3d_v4f32_f32:4611 case Intrinsic::nvvm_tex_unified_3d_level_v4f32_f32:4612 case Intrinsic::nvvm_tex_unified_3d_grad_v4f32_f32:4613 case Intrinsic::nvvm_tex_unified_cube_v4f32_f32:4614 case Intrinsic::nvvm_tex_unified_cube_level_v4f32_f32:4615 case Intrinsic::nvvm_tex_unified_cube_array_v4f32_f32:4616 case Intrinsic::nvvm_tex_unified_cube_array_level_v4f32_f32:4617 case Intrinsic::nvvm_tex_unified_cube_grad_v4f32_f32:4618 case Intrinsic::nvvm_tex_unified_cube_array_grad_v4f32_f32:4619 case Intrinsic::nvvm_tld4_unified_r_2d_v4f32_f32:4620 case Intrinsic::nvvm_tld4_unified_g_2d_v4f32_f32:4621 case Intrinsic::nvvm_tld4_unified_b_2d_v4f32_f32:4622 case Intrinsic::nvvm_tld4_unified_a_2d_v4f32_f32:4623 Info.opc = ISD::INTRINSIC_W_CHAIN;4624 Info.memVT = MVT::v4f32;4625 Info.ptrVal = nullptr;4626 Info.offset = 0;4627 Info.flags = MachineMemOperand::MOLoad;4628 Info.align = Align(16);4629 return true;4630 4631 case Intrinsic::nvvm_tex_1d_v4s32_s32:4632 case Intrinsic::nvvm_tex_1d_v4s32_f32:4633 case Intrinsic::nvvm_tex_1d_level_v4s32_f32:4634 case Intrinsic::nvvm_tex_1d_grad_v4s32_f32:4635 case Intrinsic::nvvm_tex_1d_array_v4s32_s32:4636 case Intrinsic::nvvm_tex_1d_array_v4s32_f32:4637 case Intrinsic::nvvm_tex_1d_array_level_v4s32_f32:4638 case Intrinsic::nvvm_tex_1d_array_grad_v4s32_f32:4639 case Intrinsic::nvvm_tex_2d_v4s32_s32:4640 case Intrinsic::nvvm_tex_2d_v4s32_f32:4641 case Intrinsic::nvvm_tex_2d_level_v4s32_f32:4642 case Intrinsic::nvvm_tex_2d_grad_v4s32_f32:4643 case Intrinsic::nvvm_tex_2d_array_v4s32_s32:4644 case Intrinsic::nvvm_tex_2d_array_v4s32_f32:4645 case Intrinsic::nvvm_tex_2d_array_level_v4s32_f32:4646 case Intrinsic::nvvm_tex_2d_array_grad_v4s32_f32:4647 case Intrinsic::nvvm_tex_3d_v4s32_s32:4648 case Intrinsic::nvvm_tex_3d_v4s32_f32:4649 case Intrinsic::nvvm_tex_3d_level_v4s32_f32:4650 case Intrinsic::nvvm_tex_3d_grad_v4s32_f32:4651 case Intrinsic::nvvm_tex_cube_v4s32_f32:4652 case Intrinsic::nvvm_tex_cube_level_v4s32_f32:4653 case Intrinsic::nvvm_tex_cube_array_v4s32_f32:4654 case Intrinsic::nvvm_tex_cube_array_level_v4s32_f32:4655 case Intrinsic::nvvm_tex_cube_v4u32_f32:4656 case Intrinsic::nvvm_tex_cube_level_v4u32_f32:4657 case Intrinsic::nvvm_tex_cube_array_v4u32_f32:4658 case Intrinsic::nvvm_tex_cube_array_level_v4u32_f32:4659 case Intrinsic::nvvm_tex_1d_v4u32_s32:4660 case Intrinsic::nvvm_tex_1d_v4u32_f32:4661 case Intrinsic::nvvm_tex_1d_level_v4u32_f32:4662 case Intrinsic::nvvm_tex_1d_grad_v4u32_f32:4663 case Intrinsic::nvvm_tex_1d_array_v4u32_s32:4664 case Intrinsic::nvvm_tex_1d_array_v4u32_f32:4665 case Intrinsic::nvvm_tex_1d_array_level_v4u32_f32:4666 case Intrinsic::nvvm_tex_1d_array_grad_v4u32_f32:4667 case Intrinsic::nvvm_tex_2d_v4u32_s32:4668 case Intrinsic::nvvm_tex_2d_v4u32_f32:4669 case Intrinsic::nvvm_tex_2d_level_v4u32_f32:4670 case Intrinsic::nvvm_tex_2d_grad_v4u32_f32:4671 case Intrinsic::nvvm_tex_2d_array_v4u32_s32:4672 case Intrinsic::nvvm_tex_2d_array_v4u32_f32:4673 case Intrinsic::nvvm_tex_2d_array_level_v4u32_f32:4674 case Intrinsic::nvvm_tex_2d_array_grad_v4u32_f32:4675 case Intrinsic::nvvm_tex_3d_v4u32_s32:4676 case Intrinsic::nvvm_tex_3d_v4u32_f32:4677 case Intrinsic::nvvm_tex_3d_level_v4u32_f32:4678 case Intrinsic::nvvm_tex_3d_grad_v4u32_f32:4679 case Intrinsic::nvvm_tld4_r_2d_v4s32_f32:4680 case Intrinsic::nvvm_tld4_g_2d_v4s32_f32:4681 case Intrinsic::nvvm_tld4_b_2d_v4s32_f32:4682 case Intrinsic::nvvm_tld4_a_2d_v4s32_f32:4683 case Intrinsic::nvvm_tld4_r_2d_v4u32_f32:4684 case Intrinsic::nvvm_tld4_g_2d_v4u32_f32:4685 case Intrinsic::nvvm_tld4_b_2d_v4u32_f32:4686 case Intrinsic::nvvm_tld4_a_2d_v4u32_f32:4687 case Intrinsic::nvvm_tex_unified_1d_v4s32_s32:4688 case Intrinsic::nvvm_tex_unified_1d_v4s32_f32:4689 case Intrinsic::nvvm_tex_unified_1d_level_v4s32_f32:4690 case Intrinsic::nvvm_tex_unified_1d_grad_v4s32_f32:4691 case Intrinsic::nvvm_tex_unified_1d_array_v4s32_s32:4692 case Intrinsic::nvvm_tex_unified_1d_array_v4s32_f32:4693 case Intrinsic::nvvm_tex_unified_1d_array_level_v4s32_f32:4694 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4s32_f32:4695 case Intrinsic::nvvm_tex_unified_2d_v4s32_s32:4696 case Intrinsic::nvvm_tex_unified_2d_v4s32_f32:4697 case Intrinsic::nvvm_tex_unified_2d_level_v4s32_f32:4698 case Intrinsic::nvvm_tex_unified_2d_grad_v4s32_f32:4699 case Intrinsic::nvvm_tex_unified_2d_array_v4s32_s32:4700 case Intrinsic::nvvm_tex_unified_2d_array_v4s32_f32:4701 case Intrinsic::nvvm_tex_unified_2d_array_level_v4s32_f32:4702 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4s32_f32:4703 case Intrinsic::nvvm_tex_unified_3d_v4s32_s32:4704 case Intrinsic::nvvm_tex_unified_3d_v4s32_f32:4705 case Intrinsic::nvvm_tex_unified_3d_level_v4s32_f32:4706 case Intrinsic::nvvm_tex_unified_3d_grad_v4s32_f32:4707 case Intrinsic::nvvm_tex_unified_1d_v4u32_s32:4708 case Intrinsic::nvvm_tex_unified_1d_v4u32_f32:4709 case Intrinsic::nvvm_tex_unified_1d_level_v4u32_f32:4710 case Intrinsic::nvvm_tex_unified_1d_grad_v4u32_f32:4711 case Intrinsic::nvvm_tex_unified_1d_array_v4u32_s32:4712 case Intrinsic::nvvm_tex_unified_1d_array_v4u32_f32:4713 case Intrinsic::nvvm_tex_unified_1d_array_level_v4u32_f32:4714 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4u32_f32:4715 case Intrinsic::nvvm_tex_unified_2d_v4u32_s32:4716 case Intrinsic::nvvm_tex_unified_2d_v4u32_f32:4717 case Intrinsic::nvvm_tex_unified_2d_level_v4u32_f32:4718 case Intrinsic::nvvm_tex_unified_2d_grad_v4u32_f32:4719 case Intrinsic::nvvm_tex_unified_2d_array_v4u32_s32:4720 case Intrinsic::nvvm_tex_unified_2d_array_v4u32_f32:4721 case Intrinsic::nvvm_tex_unified_2d_array_level_v4u32_f32:4722 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4u32_f32:4723 case Intrinsic::nvvm_tex_unified_3d_v4u32_s32:4724 case Intrinsic::nvvm_tex_unified_3d_v4u32_f32:4725 case Intrinsic::nvvm_tex_unified_3d_level_v4u32_f32:4726 case Intrinsic::nvvm_tex_unified_3d_grad_v4u32_f32:4727 case Intrinsic::nvvm_tex_unified_cube_v4s32_f32:4728 case Intrinsic::nvvm_tex_unified_cube_level_v4s32_f32:4729 case Intrinsic::nvvm_tex_unified_cube_array_v4s32_f32:4730 case Intrinsic::nvvm_tex_unified_cube_array_level_v4s32_f32:4731 case Intrinsic::nvvm_tex_unified_cube_v4u32_f32:4732 case Intrinsic::nvvm_tex_unified_cube_level_v4u32_f32:4733 case Intrinsic::nvvm_tex_unified_cube_array_v4u32_f32:4734 case Intrinsic::nvvm_tex_unified_cube_array_level_v4u32_f32:4735 case Intrinsic::nvvm_tex_unified_cube_grad_v4s32_f32:4736 case Intrinsic::nvvm_tex_unified_cube_grad_v4u32_f32:4737 case Intrinsic::nvvm_tex_unified_cube_array_grad_v4s32_f32:4738 case Intrinsic::nvvm_tex_unified_cube_array_grad_v4u32_f32:4739 case Intrinsic::nvvm_tld4_unified_r_2d_v4s32_f32:4740 case Intrinsic::nvvm_tld4_unified_g_2d_v4s32_f32:4741 case Intrinsic::nvvm_tld4_unified_b_2d_v4s32_f32:4742 case Intrinsic::nvvm_tld4_unified_a_2d_v4s32_f32:4743 case Intrinsic::nvvm_tld4_unified_r_2d_v4u32_f32:4744 case Intrinsic::nvvm_tld4_unified_g_2d_v4u32_f32:4745 case Intrinsic::nvvm_tld4_unified_b_2d_v4u32_f32:4746 case Intrinsic::nvvm_tld4_unified_a_2d_v4u32_f32:4747 Info.opc = ISD::INTRINSIC_W_CHAIN;4748 Info.memVT = MVT::v4i32;4749 Info.ptrVal = nullptr;4750 Info.offset = 0;4751 Info.flags = MachineMemOperand::MOLoad;4752 Info.align = Align(16);4753 return true;4754 4755 case Intrinsic::nvvm_suld_1d_i8_clamp:4756 case Intrinsic::nvvm_suld_1d_v2i8_clamp:4757 case Intrinsic::nvvm_suld_1d_v4i8_clamp:4758 case Intrinsic::nvvm_suld_1d_array_i8_clamp:4759 case Intrinsic::nvvm_suld_1d_array_v2i8_clamp:4760 case Intrinsic::nvvm_suld_1d_array_v4i8_clamp:4761 case Intrinsic::nvvm_suld_2d_i8_clamp:4762 case Intrinsic::nvvm_suld_2d_v2i8_clamp:4763 case Intrinsic::nvvm_suld_2d_v4i8_clamp:4764 case Intrinsic::nvvm_suld_2d_array_i8_clamp:4765 case Intrinsic::nvvm_suld_2d_array_v2i8_clamp:4766 case Intrinsic::nvvm_suld_2d_array_v4i8_clamp:4767 case Intrinsic::nvvm_suld_3d_i8_clamp:4768 case Intrinsic::nvvm_suld_3d_v2i8_clamp:4769 case Intrinsic::nvvm_suld_3d_v4i8_clamp:4770 case Intrinsic::nvvm_suld_1d_i8_trap:4771 case Intrinsic::nvvm_suld_1d_v2i8_trap:4772 case Intrinsic::nvvm_suld_1d_v4i8_trap:4773 case Intrinsic::nvvm_suld_1d_array_i8_trap:4774 case Intrinsic::nvvm_suld_1d_array_v2i8_trap:4775 case Intrinsic::nvvm_suld_1d_array_v4i8_trap:4776 case Intrinsic::nvvm_suld_2d_i8_trap:4777 case Intrinsic::nvvm_suld_2d_v2i8_trap:4778 case Intrinsic::nvvm_suld_2d_v4i8_trap:4779 case Intrinsic::nvvm_suld_2d_array_i8_trap:4780 case Intrinsic::nvvm_suld_2d_array_v2i8_trap:4781 case Intrinsic::nvvm_suld_2d_array_v4i8_trap:4782 case Intrinsic::nvvm_suld_3d_i8_trap:4783 case Intrinsic::nvvm_suld_3d_v2i8_trap:4784 case Intrinsic::nvvm_suld_3d_v4i8_trap:4785 case Intrinsic::nvvm_suld_1d_i8_zero:4786 case Intrinsic::nvvm_suld_1d_v2i8_zero:4787 case Intrinsic::nvvm_suld_1d_v4i8_zero:4788 case Intrinsic::nvvm_suld_1d_array_i8_zero:4789 case Intrinsic::nvvm_suld_1d_array_v2i8_zero:4790 case Intrinsic::nvvm_suld_1d_array_v4i8_zero:4791 case Intrinsic::nvvm_suld_2d_i8_zero:4792 case Intrinsic::nvvm_suld_2d_v2i8_zero:4793 case Intrinsic::nvvm_suld_2d_v4i8_zero:4794 case Intrinsic::nvvm_suld_2d_array_i8_zero:4795 case Intrinsic::nvvm_suld_2d_array_v2i8_zero:4796 case Intrinsic::nvvm_suld_2d_array_v4i8_zero:4797 case Intrinsic::nvvm_suld_3d_i8_zero:4798 case Intrinsic::nvvm_suld_3d_v2i8_zero:4799 case Intrinsic::nvvm_suld_3d_v4i8_zero:4800 Info.opc = ISD::INTRINSIC_W_CHAIN;4801 Info.memVT = MVT::i8;4802 Info.ptrVal = nullptr;4803 Info.offset = 0;4804 Info.flags = MachineMemOperand::MOLoad;4805 Info.align = Align(16);4806 return true;4807 4808 case Intrinsic::nvvm_suld_1d_i16_clamp:4809 case Intrinsic::nvvm_suld_1d_v2i16_clamp:4810 case Intrinsic::nvvm_suld_1d_v4i16_clamp:4811 case Intrinsic::nvvm_suld_1d_array_i16_clamp:4812 case Intrinsic::nvvm_suld_1d_array_v2i16_clamp:4813 case Intrinsic::nvvm_suld_1d_array_v4i16_clamp:4814 case Intrinsic::nvvm_suld_2d_i16_clamp:4815 case Intrinsic::nvvm_suld_2d_v2i16_clamp:4816 case Intrinsic::nvvm_suld_2d_v4i16_clamp:4817 case Intrinsic::nvvm_suld_2d_array_i16_clamp:4818 case Intrinsic::nvvm_suld_2d_array_v2i16_clamp:4819 case Intrinsic::nvvm_suld_2d_array_v4i16_clamp:4820 case Intrinsic::nvvm_suld_3d_i16_clamp:4821 case Intrinsic::nvvm_suld_3d_v2i16_clamp:4822 case Intrinsic::nvvm_suld_3d_v4i16_clamp:4823 case Intrinsic::nvvm_suld_1d_i16_trap:4824 case Intrinsic::nvvm_suld_1d_v2i16_trap:4825 case Intrinsic::nvvm_suld_1d_v4i16_trap:4826 case Intrinsic::nvvm_suld_1d_array_i16_trap:4827 case Intrinsic::nvvm_suld_1d_array_v2i16_trap:4828 case Intrinsic::nvvm_suld_1d_array_v4i16_trap:4829 case Intrinsic::nvvm_suld_2d_i16_trap:4830 case Intrinsic::nvvm_suld_2d_v2i16_trap:4831 case Intrinsic::nvvm_suld_2d_v4i16_trap:4832 case Intrinsic::nvvm_suld_2d_array_i16_trap:4833 case Intrinsic::nvvm_suld_2d_array_v2i16_trap:4834 case Intrinsic::nvvm_suld_2d_array_v4i16_trap:4835 case Intrinsic::nvvm_suld_3d_i16_trap:4836 case Intrinsic::nvvm_suld_3d_v2i16_trap:4837 case Intrinsic::nvvm_suld_3d_v4i16_trap:4838 case Intrinsic::nvvm_suld_1d_i16_zero:4839 case Intrinsic::nvvm_suld_1d_v2i16_zero:4840 case Intrinsic::nvvm_suld_1d_v4i16_zero:4841 case Intrinsic::nvvm_suld_1d_array_i16_zero:4842 case Intrinsic::nvvm_suld_1d_array_v2i16_zero:4843 case Intrinsic::nvvm_suld_1d_array_v4i16_zero:4844 case Intrinsic::nvvm_suld_2d_i16_zero:4845 case Intrinsic::nvvm_suld_2d_v2i16_zero:4846 case Intrinsic::nvvm_suld_2d_v4i16_zero:4847 case Intrinsic::nvvm_suld_2d_array_i16_zero:4848 case Intrinsic::nvvm_suld_2d_array_v2i16_zero:4849 case Intrinsic::nvvm_suld_2d_array_v4i16_zero:4850 case Intrinsic::nvvm_suld_3d_i16_zero:4851 case Intrinsic::nvvm_suld_3d_v2i16_zero:4852 case Intrinsic::nvvm_suld_3d_v4i16_zero:4853 Info.opc = ISD::INTRINSIC_W_CHAIN;4854 Info.memVT = MVT::i16;4855 Info.ptrVal = nullptr;4856 Info.offset = 0;4857 Info.flags = MachineMemOperand::MOLoad;4858 Info.align = Align(16);4859 return true;4860 4861 case Intrinsic::nvvm_suld_1d_i32_clamp:4862 case Intrinsic::nvvm_suld_1d_v2i32_clamp:4863 case Intrinsic::nvvm_suld_1d_v4i32_clamp:4864 case Intrinsic::nvvm_suld_1d_array_i32_clamp:4865 case Intrinsic::nvvm_suld_1d_array_v2i32_clamp:4866 case Intrinsic::nvvm_suld_1d_array_v4i32_clamp:4867 case Intrinsic::nvvm_suld_2d_i32_clamp:4868 case Intrinsic::nvvm_suld_2d_v2i32_clamp:4869 case Intrinsic::nvvm_suld_2d_v4i32_clamp:4870 case Intrinsic::nvvm_suld_2d_array_i32_clamp:4871 case Intrinsic::nvvm_suld_2d_array_v2i32_clamp:4872 case Intrinsic::nvvm_suld_2d_array_v4i32_clamp:4873 case Intrinsic::nvvm_suld_3d_i32_clamp:4874 case Intrinsic::nvvm_suld_3d_v2i32_clamp:4875 case Intrinsic::nvvm_suld_3d_v4i32_clamp:4876 case Intrinsic::nvvm_suld_1d_i32_trap:4877 case Intrinsic::nvvm_suld_1d_v2i32_trap:4878 case Intrinsic::nvvm_suld_1d_v4i32_trap:4879 case Intrinsic::nvvm_suld_1d_array_i32_trap:4880 case Intrinsic::nvvm_suld_1d_array_v2i32_trap:4881 case Intrinsic::nvvm_suld_1d_array_v4i32_trap:4882 case Intrinsic::nvvm_suld_2d_i32_trap:4883 case Intrinsic::nvvm_suld_2d_v2i32_trap:4884 case Intrinsic::nvvm_suld_2d_v4i32_trap:4885 case Intrinsic::nvvm_suld_2d_array_i32_trap:4886 case Intrinsic::nvvm_suld_2d_array_v2i32_trap:4887 case Intrinsic::nvvm_suld_2d_array_v4i32_trap:4888 case Intrinsic::nvvm_suld_3d_i32_trap:4889 case Intrinsic::nvvm_suld_3d_v2i32_trap:4890 case Intrinsic::nvvm_suld_3d_v4i32_trap:4891 case Intrinsic::nvvm_suld_1d_i32_zero:4892 case Intrinsic::nvvm_suld_1d_v2i32_zero:4893 case Intrinsic::nvvm_suld_1d_v4i32_zero:4894 case Intrinsic::nvvm_suld_1d_array_i32_zero:4895 case Intrinsic::nvvm_suld_1d_array_v2i32_zero:4896 case Intrinsic::nvvm_suld_1d_array_v4i32_zero:4897 case Intrinsic::nvvm_suld_2d_i32_zero:4898 case Intrinsic::nvvm_suld_2d_v2i32_zero:4899 case Intrinsic::nvvm_suld_2d_v4i32_zero:4900 case Intrinsic::nvvm_suld_2d_array_i32_zero:4901 case Intrinsic::nvvm_suld_2d_array_v2i32_zero:4902 case Intrinsic::nvvm_suld_2d_array_v4i32_zero:4903 case Intrinsic::nvvm_suld_3d_i32_zero:4904 case Intrinsic::nvvm_suld_3d_v2i32_zero:4905 case Intrinsic::nvvm_suld_3d_v4i32_zero:4906 Info.opc = ISD::INTRINSIC_W_CHAIN;4907 Info.memVT = MVT::i32;4908 Info.ptrVal = nullptr;4909 Info.offset = 0;4910 Info.flags = MachineMemOperand::MOLoad;4911 Info.align = Align(16);4912 return true;4913 4914 case Intrinsic::nvvm_suld_1d_i64_clamp:4915 case Intrinsic::nvvm_suld_1d_v2i64_clamp:4916 case Intrinsic::nvvm_suld_1d_array_i64_clamp:4917 case Intrinsic::nvvm_suld_1d_array_v2i64_clamp:4918 case Intrinsic::nvvm_suld_2d_i64_clamp:4919 case Intrinsic::nvvm_suld_2d_v2i64_clamp:4920 case Intrinsic::nvvm_suld_2d_array_i64_clamp:4921 case Intrinsic::nvvm_suld_2d_array_v2i64_clamp:4922 case Intrinsic::nvvm_suld_3d_i64_clamp:4923 case Intrinsic::nvvm_suld_3d_v2i64_clamp:4924 case Intrinsic::nvvm_suld_1d_i64_trap:4925 case Intrinsic::nvvm_suld_1d_v2i64_trap:4926 case Intrinsic::nvvm_suld_1d_array_i64_trap:4927 case Intrinsic::nvvm_suld_1d_array_v2i64_trap:4928 case Intrinsic::nvvm_suld_2d_i64_trap:4929 case Intrinsic::nvvm_suld_2d_v2i64_trap:4930 case Intrinsic::nvvm_suld_2d_array_i64_trap:4931 case Intrinsic::nvvm_suld_2d_array_v2i64_trap:4932 case Intrinsic::nvvm_suld_3d_i64_trap:4933 case Intrinsic::nvvm_suld_3d_v2i64_trap:4934 case Intrinsic::nvvm_suld_1d_i64_zero:4935 case Intrinsic::nvvm_suld_1d_v2i64_zero:4936 case Intrinsic::nvvm_suld_1d_array_i64_zero:4937 case Intrinsic::nvvm_suld_1d_array_v2i64_zero:4938 case Intrinsic::nvvm_suld_2d_i64_zero:4939 case Intrinsic::nvvm_suld_2d_v2i64_zero:4940 case Intrinsic::nvvm_suld_2d_array_i64_zero:4941 case Intrinsic::nvvm_suld_2d_array_v2i64_zero:4942 case Intrinsic::nvvm_suld_3d_i64_zero:4943 case Intrinsic::nvvm_suld_3d_v2i64_zero:4944 Info.opc = ISD::INTRINSIC_W_CHAIN;4945 Info.memVT = MVT::i64;4946 Info.ptrVal = nullptr;4947 Info.offset = 0;4948 Info.flags = MachineMemOperand::MOLoad;4949 Info.align = Align(16);4950 return true;4951 4952 case Intrinsic::nvvm_tcgen05_ld_16x64b_x1:4953 case Intrinsic::nvvm_tcgen05_ld_32x32b_x1:4954 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x1: {4955 Info.opc = ISD::INTRINSIC_W_CHAIN;4956 Info.memVT = MVT::v1i32;4957 Info.ptrVal = I.getArgOperand(0);4958 Info.offset = 0;4959 Info.flags = MachineMemOperand::MOLoad;4960 Info.align.reset();4961 return true;4962 }4963 4964 case Intrinsic::nvvm_tcgen05_ld_16x64b_x2:4965 case Intrinsic::nvvm_tcgen05_ld_16x128b_x1:4966 case Intrinsic::nvvm_tcgen05_ld_32x32b_x2:4967 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x2: {4968 Info.opc = ISD::INTRINSIC_W_CHAIN;4969 Info.memVT = MVT::v2i32;4970 Info.ptrVal = I.getArgOperand(0);4971 Info.offset = 0;4972 Info.flags = MachineMemOperand::MOLoad;4973 Info.align.reset();4974 return true;4975 }4976 4977 case Intrinsic::nvvm_tcgen05_ld_16x64b_x4:4978 case Intrinsic::nvvm_tcgen05_ld_16x128b_x2:4979 case Intrinsic::nvvm_tcgen05_ld_32x32b_x4:4980 case Intrinsic::nvvm_tcgen05_ld_16x256b_x1:4981 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x4: {4982 Info.opc = ISD::INTRINSIC_W_CHAIN;4983 Info.memVT = MVT::v4i32;4984 Info.ptrVal = I.getArgOperand(0);4985 Info.offset = 0;4986 Info.flags = MachineMemOperand::MOLoad;4987 Info.align.reset();4988 return true;4989 }4990 4991 case Intrinsic::nvvm_tcgen05_ld_16x64b_x8:4992 case Intrinsic::nvvm_tcgen05_ld_16x128b_x4:4993 case Intrinsic::nvvm_tcgen05_ld_16x256b_x2:4994 case Intrinsic::nvvm_tcgen05_ld_32x32b_x8:4995 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x8: {4996 Info.opc = ISD::INTRINSIC_W_CHAIN;4997 Info.memVT = MVT::v8i32;4998 Info.ptrVal = I.getArgOperand(0);4999 Info.offset = 0;5000 Info.flags = MachineMemOperand::MOLoad;5001 Info.align.reset();5002 return true;5003 }5004 5005 case Intrinsic::nvvm_tcgen05_ld_16x64b_x16:5006 case Intrinsic::nvvm_tcgen05_ld_16x128b_x8:5007 case Intrinsic::nvvm_tcgen05_ld_16x256b_x4:5008 case Intrinsic::nvvm_tcgen05_ld_32x32b_x16:5009 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x16: {5010 Info.opc = ISD::INTRINSIC_W_CHAIN;5011 Info.memVT = MVT::v16i32;5012 Info.ptrVal = I.getArgOperand(0);5013 Info.offset = 0;5014 Info.flags = MachineMemOperand::MOLoad;5015 Info.align.reset();5016 return true;5017 }5018 5019 case Intrinsic::nvvm_tcgen05_ld_16x64b_x32:5020 case Intrinsic::nvvm_tcgen05_ld_16x128b_x16:5021 case Intrinsic::nvvm_tcgen05_ld_16x256b_x8:5022 case Intrinsic::nvvm_tcgen05_ld_32x32b_x32:5023 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x32: {5024 Info.opc = ISD::INTRINSIC_W_CHAIN;5025 Info.memVT = MVT::v32i32;5026 Info.ptrVal = I.getArgOperand(0);5027 Info.offset = 0;5028 Info.flags = MachineMemOperand::MOLoad;5029 Info.align.reset();5030 return true;5031 }5032 5033 case Intrinsic::nvvm_tcgen05_ld_16x64b_x64:5034 case Intrinsic::nvvm_tcgen05_ld_16x128b_x32:5035 case Intrinsic::nvvm_tcgen05_ld_16x256b_x16:5036 case Intrinsic::nvvm_tcgen05_ld_32x32b_x64:5037 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x64: {5038 Info.opc = ISD::INTRINSIC_W_CHAIN;5039 Info.memVT = MVT::v64i32;5040 Info.ptrVal = I.getArgOperand(0);5041 Info.offset = 0;5042 Info.flags = MachineMemOperand::MOLoad;5043 Info.align.reset();5044 return true;5045 }5046 5047 case Intrinsic::nvvm_tcgen05_ld_16x64b_x128:5048 case Intrinsic::nvvm_tcgen05_ld_16x128b_x64:5049 case Intrinsic::nvvm_tcgen05_ld_16x256b_x32:5050 case Intrinsic::nvvm_tcgen05_ld_32x32b_x128:5051 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x128: {5052 Info.opc = ISD::INTRINSIC_W_CHAIN;5053 Info.memVT = MVT::v128i32;5054 Info.ptrVal = I.getArgOperand(0);5055 Info.offset = 0;5056 Info.flags = MachineMemOperand::MOLoad;5057 Info.align.reset();5058 return true;5059 }5060 5061 case Intrinsic::nvvm_tcgen05_st_16x64b_x1:5062 case Intrinsic::nvvm_tcgen05_st_32x32b_x1:5063 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x1: {5064 Info.opc = ISD::INTRINSIC_VOID;5065 Info.memVT = MVT::i32;5066 Info.ptrVal = I.getArgOperand(0);5067 Info.offset = 0;5068 Info.flags = MachineMemOperand::MOStore;5069 Info.align.reset();5070 return true;5071 }5072 5073 case Intrinsic::nvvm_tcgen05_st_16x64b_x2:5074 case Intrinsic::nvvm_tcgen05_st_16x128b_x1:5075 case Intrinsic::nvvm_tcgen05_st_32x32b_x2:5076 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x2: {5077 Info.opc = ISD::INTRINSIC_VOID;5078 Info.memVT = MVT::v2i32;5079 Info.ptrVal = I.getArgOperand(0);5080 Info.offset = 0;5081 Info.flags = MachineMemOperand::MOStore;5082 Info.align.reset();5083 return true;5084 }5085 5086 case Intrinsic::nvvm_tcgen05_st_16x64b_x4:5087 case Intrinsic::nvvm_tcgen05_st_16x128b_x2:5088 case Intrinsic::nvvm_tcgen05_st_16x256b_x1:5089 case Intrinsic::nvvm_tcgen05_st_32x32b_x4:5090 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x4: {5091 Info.opc = ISD::INTRINSIC_VOID;5092 Info.memVT = MVT::v4i32;5093 Info.ptrVal = I.getArgOperand(0);5094 Info.offset = 0;5095 Info.flags = MachineMemOperand::MOStore;5096 Info.align.reset();5097 return true;5098 }5099 5100 case Intrinsic::nvvm_tcgen05_st_16x64b_x8:5101 case Intrinsic::nvvm_tcgen05_st_16x128b_x4:5102 case Intrinsic::nvvm_tcgen05_st_16x256b_x2:5103 case Intrinsic::nvvm_tcgen05_st_32x32b_x8:5104 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x8: {5105 Info.opc = ISD::INTRINSIC_VOID;5106 Info.memVT = MVT::v8i32;5107 Info.ptrVal = I.getArgOperand(0);5108 Info.offset = 0;5109 Info.flags = MachineMemOperand::MOStore;5110 Info.align.reset();5111 return true;5112 }5113 5114 case Intrinsic::nvvm_tcgen05_st_16x64b_x16:5115 case Intrinsic::nvvm_tcgen05_st_16x128b_x8:5116 case Intrinsic::nvvm_tcgen05_st_16x256b_x4:5117 case Intrinsic::nvvm_tcgen05_st_32x32b_x16:5118 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x16: {5119 Info.opc = ISD::INTRINSIC_VOID;5120 Info.memVT = MVT::v16i32;5121 Info.ptrVal = I.getArgOperand(0);5122 Info.offset = 0;5123 Info.flags = MachineMemOperand::MOStore;5124 Info.align.reset();5125 return true;5126 }5127 5128 case Intrinsic::nvvm_tcgen05_st_16x64b_x32:5129 case Intrinsic::nvvm_tcgen05_st_16x128b_x16:5130 case Intrinsic::nvvm_tcgen05_st_16x256b_x8:5131 case Intrinsic::nvvm_tcgen05_st_32x32b_x32:5132 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x32: {5133 Info.opc = ISD::INTRINSIC_VOID;5134 Info.memVT = MVT::v32i32;5135 Info.ptrVal = I.getArgOperand(0);5136 Info.offset = 0;5137 Info.flags = MachineMemOperand::MOStore;5138 Info.align.reset();5139 return true;5140 }5141 5142 case Intrinsic::nvvm_tcgen05_st_16x64b_x64:5143 case Intrinsic::nvvm_tcgen05_st_16x128b_x32:5144 case Intrinsic::nvvm_tcgen05_st_16x256b_x16:5145 case Intrinsic::nvvm_tcgen05_st_32x32b_x64:5146 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x64: {5147 Info.opc = ISD::INTRINSIC_VOID;5148 Info.memVT = MVT::v64i32;5149 Info.ptrVal = I.getArgOperand(0);5150 Info.offset = 0;5151 Info.flags = MachineMemOperand::MOStore;5152 Info.align.reset();5153 return true;5154 }5155 5156 case Intrinsic::nvvm_tcgen05_st_16x64b_x128:5157 case Intrinsic::nvvm_tcgen05_st_16x128b_x64:5158 case Intrinsic::nvvm_tcgen05_st_16x256b_x32:5159 case Intrinsic::nvvm_tcgen05_st_32x32b_x128:5160 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x128: {5161 Info.opc = ISD::INTRINSIC_VOID;5162 Info.memVT = MVT::v128i32;5163 Info.ptrVal = I.getArgOperand(0);5164 Info.offset = 0;5165 Info.flags = MachineMemOperand::MOStore;5166 Info.align.reset();5167 return true;5168 }5169 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg1:5170 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg1:5171 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg1:5172 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg1:5173 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1:5174 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1:5175 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1_ashift:5176 case Intrinsic::5177 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1_ashift:5178 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1:5179 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1:5180 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1_ashift:5181 case Intrinsic::5182 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1_ashift: {5183 // We are reading and writing back to TMem5184 Info.opc = ISD::INTRINSIC_VOID;5185 Info.memVT = MVT::v4i32;5186 Info.ptrVal = I.getArgOperand(0);5187 Info.offset = 0;5188 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;5189 Info.align = Align(16);5190 return true;5191 }5192 5193 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg2:5194 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg2:5195 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg2:5196 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg2:5197 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2:5198 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2:5199 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2:5200 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2:5201 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2_ashift:5202 case Intrinsic::5203 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2_ashift:5204 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2_ashift:5205 case Intrinsic::5206 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2_ashift: {5207 // We are reading and writing back to TMem5208 Info.opc = ISD::INTRINSIC_VOID;5209 Info.memVT = MVT::v8i32;5210 Info.ptrVal = I.getArgOperand(0);5211 Info.offset = 0;5212 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;5213 Info.align = Align(16);5214 return true;5215 }5216 }5217 return false;5218}5219 5220/// getFunctionParamOptimizedAlign - since function arguments are passed via5221/// .param space, we may want to increase their alignment in a way that5222/// ensures that we can effectively vectorize their loads & stores. We can5223/// increase alignment only if the function has internal or has private5224/// linkage as for other linkage types callers may already rely on default5225/// alignment. To allow using 128-bit vectorized loads/stores, this function5226/// ensures that alignment is 16 or greater.5227Align NVPTXTargetLowering::getFunctionParamOptimizedAlign(5228 const Function *F, Type *ArgTy, const DataLayout &DL) const {5229 // Capping the alignment to 128 bytes as that is the maximum alignment5230 // supported by PTX.5231 const Align ABITypeAlign = std::min(Align(128), DL.getABITypeAlign(ArgTy));5232 5233 // If a function has linkage different from internal or private, we5234 // must use default ABI alignment as external users rely on it. Same5235 // for a function that may be called from a function pointer.5236 if (!F || !F->hasLocalLinkage() ||5237 F->hasAddressTaken(/*Users=*/nullptr,5238 /*IgnoreCallbackUses=*/false,5239 /*IgnoreAssumeLikeCalls=*/true,5240 /*IgnoreLLVMUsed=*/true))5241 return ABITypeAlign;5242 5243 assert(!isKernelFunction(*F) && "Expect kernels to have non-local linkage");5244 return std::max(Align(16), ABITypeAlign);5245}5246 5247/// Helper for computing alignment of a device function byval parameter.5248Align NVPTXTargetLowering::getFunctionByValParamAlign(5249 const Function *F, Type *ArgTy, Align InitialAlign,5250 const DataLayout &DL) const {5251 Align ArgAlign = InitialAlign;5252 // Try to increase alignment to enhance vectorization options.5253 if (F)5254 ArgAlign = std::max(ArgAlign, getFunctionParamOptimizedAlign(F, ArgTy, DL));5255 5256 // Old ptx versions have a bug. When PTX code takes address of5257 // byval parameter with alignment < 4, ptxas generates code to5258 // spill argument into memory. Alas on sm_50+ ptxas generates5259 // SASS code that fails with misaligned access. To work around5260 // the problem, make sure that we align byval parameters by at5261 // least 4. This bug seems to be fixed at least starting from5262 // ptxas > 9.0.5263 // TODO: remove this after verifying the bug is not reproduced5264 // on non-deprecated ptxas versions.5265 if (ForceMinByValParamAlign)5266 ArgAlign = std::max(ArgAlign, Align(4));5267 5268 return ArgAlign;5269}5270 5271// Helper for getting a function parameter name. Name is composed from5272// its index and the function name. Negative index corresponds to special5273// parameter (unsized array) used for passing variable arguments.5274std::string NVPTXTargetLowering::getParamName(const Function *F,5275 int Idx) const {5276 std::string ParamName;5277 raw_string_ostream ParamStr(ParamName);5278 5279 ParamStr << getTargetMachine().getSymbol(F)->getName();5280 if (Idx < 0)5281 ParamStr << "_vararg";5282 else5283 ParamStr << "_param_" << Idx;5284 5285 return ParamName;5286}5287 5288/// isLegalAddressingMode - Return true if the addressing mode represented5289/// by AM is legal for this target, for a load/store of the specified type.5290/// Used to guide target specific optimizations, like loop strength reduction5291/// (LoopStrengthReduce.cpp) and memory optimization for address mode5292/// (CodeGenPrepare.cpp)5293bool NVPTXTargetLowering::isLegalAddressingMode(const DataLayout &DL,5294 const AddrMode &AM, Type *Ty,5295 unsigned AS, Instruction *I) const {5296 // AddrMode - This represents an addressing mode of:5297 // BaseGV + BaseOffs + BaseReg + Scale*ScaleReg5298 //5299 // The legal address modes are5300 // - [avar]5301 // - [areg]5302 // - [areg+immoff]5303 // - [immAddr]5304 5305 // immoff must fit in a signed 32-bit int5306 if (!APInt(64, AM.BaseOffs).isSignedIntN(32))5307 return false;5308 5309 if (AM.BaseGV)5310 return !AM.BaseOffs && !AM.HasBaseReg && !AM.Scale;5311 5312 switch (AM.Scale) {5313 case 0: // "r", "r+i" or "i" is allowed5314 break;5315 case 1:5316 if (AM.HasBaseReg) // "r+r+i" or "r+r" is not allowed.5317 return false;5318 // Otherwise we have r+i.5319 break;5320 default:5321 // No scale > 1 is allowed5322 return false;5323 }5324 return true;5325}5326 5327//===----------------------------------------------------------------------===//5328// NVPTX Inline Assembly Support5329//===----------------------------------------------------------------------===//5330 5331/// getConstraintType - Given a constraint letter, return the type of5332/// constraint it is for this target.5333NVPTXTargetLowering::ConstraintType5334NVPTXTargetLowering::getConstraintType(StringRef Constraint) const {5335 if (Constraint.size() == 1) {5336 switch (Constraint[0]) {5337 default:5338 break;5339 case 'b':5340 case 'r':5341 case 'h':5342 case 'c':5343 case 'l':5344 case 'f':5345 case 'd':5346 case 'q':5347 case '0':5348 case 'N':5349 return C_RegisterClass;5350 }5351 }5352 return TargetLowering::getConstraintType(Constraint);5353}5354 5355std::pair<unsigned, const TargetRegisterClass *>5356NVPTXTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,5357 StringRef Constraint,5358 MVT VT) const {5359 if (Constraint.size() == 1) {5360 switch (Constraint[0]) {5361 case 'b':5362 return std::make_pair(0U, &NVPTX::B1RegClass);5363 case 'c':5364 case 'h':5365 return std::make_pair(0U, &NVPTX::B16RegClass);5366 case 'r':5367 case 'f':5368 return std::make_pair(0U, &NVPTX::B32RegClass);5369 case 'l':5370 case 'N':5371 case 'd':5372 return std::make_pair(0U, &NVPTX::B64RegClass);5373 case 'q': {5374 if (STI.getSmVersion() < 70)5375 report_fatal_error("Inline asm with 128 bit operands is only "5376 "supported for sm_70 and higher!");5377 return std::make_pair(0U, &NVPTX::B128RegClass);5378 }5379 }5380 }5381 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);5382}5383 5384//===----------------------------------------------------------------------===//5385// NVPTX DAG Combining5386//===----------------------------------------------------------------------===//5387 5388bool NVPTXTargetLowering::allowFMA(MachineFunction &MF,5389 CodeGenOptLevel OptLevel) const {5390 // Always honor command-line argument5391 if (FMAContractLevelOpt.getNumOccurrences() > 0)5392 return FMAContractLevelOpt > 0;5393 5394 // Do not contract if we're not optimizing the code.5395 if (OptLevel == CodeGenOptLevel::None)5396 return false;5397 5398 // Honor TargetOptions flags that explicitly say fusion is okay.5399 if (MF.getTarget().Options.AllowFPOpFusion == FPOpFusion::Fast)5400 return true;5401 5402 return false;5403}5404 5405static bool isConstZero(const SDValue &Operand) {5406 const auto *Const = dyn_cast<ConstantSDNode>(Operand);5407 return Const && Const->getZExtValue() == 0;5408}5409 5410/// PerformADDCombineWithOperands - Try DAG combinations for an ADD with5411/// operands N0 and N1. This is a helper for PerformADDCombine that is5412/// called with the default operands, and if that fails, with commuted5413/// operands.5414static SDValue5415PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,5416 TargetLowering::DAGCombinerInfo &DCI) {5417 EVT VT = N0.getValueType();5418 5419 // Since integer multiply-add costs the same as integer multiply5420 // but is more costly than integer add, do the fusion only when5421 // the mul is only used in the add.5422 // TODO: this may not be true for later architectures, consider relaxing this5423 if (!N0.getNode()->hasOneUse())5424 return SDValue();5425 5426 // fold (add (select cond, 0, (mul a, b)), c)5427 // -> (select cond, c, (add (mul a, b), c))5428 //5429 if (N0.getOpcode() == ISD::SELECT) {5430 unsigned ZeroOpNum;5431 if (isConstZero(N0->getOperand(1)))5432 ZeroOpNum = 1;5433 else if (isConstZero(N0->getOperand(2)))5434 ZeroOpNum = 2;5435 else5436 return SDValue();5437 5438 SDValue M = N0->getOperand((ZeroOpNum == 1) ? 2 : 1);5439 if (M->getOpcode() != ISD::MUL || !M.getNode()->hasOneUse())5440 return SDValue();5441 5442 SDLoc DL(N);5443 SDValue Mul =5444 DCI.DAG.getNode(ISD::MUL, DL, VT, M->getOperand(0), M->getOperand(1));5445 SDValue MAD = DCI.DAG.getNode(ISD::ADD, DL, VT, Mul, N1);5446 return DCI.DAG.getSelect(SDLoc(N), VT, N0->getOperand(0),5447 ((ZeroOpNum == 1) ? N1 : MAD),5448 ((ZeroOpNum == 1) ? MAD : N1));5449 }5450 5451 return SDValue();5452}5453 5454static SDValue5455PerformFADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,5456 TargetLowering::DAGCombinerInfo &DCI,5457 CodeGenOptLevel OptLevel) {5458 EVT VT = N0.getValueType();5459 if (N0.getOpcode() == ISD::FMUL) {5460 const auto *TLI = static_cast<const NVPTXTargetLowering *>(5461 &DCI.DAG.getTargetLoweringInfo());5462 if (!(TLI->allowFMA(DCI.DAG.getMachineFunction(), OptLevel) ||5463 (N->getFlags().hasAllowContract() &&5464 N0->getFlags().hasAllowContract())))5465 return SDValue();5466 5467 // For floating point:5468 // Do the fusion only when the mul has less than 5 uses and all5469 // are add.5470 // The heuristic is that if a use is not an add, then that use5471 // cannot be fused into fma, therefore mul is still needed anyway.5472 // If there are more than 4 uses, even if they are all add, fusing5473 // them will increase register pressue.5474 //5475 int numUses = 0;5476 int nonAddCount = 0;5477 for (const SDNode *User : N0.getNode()->users()) {5478 numUses++;5479 if (User->getOpcode() != ISD::FADD)5480 ++nonAddCount;5481 if (numUses >= 5)5482 return SDValue();5483 }5484 if (nonAddCount) {5485 int orderNo = N->getIROrder();5486 int orderNo2 = N0.getNode()->getIROrder();5487 // simple heuristics here for considering potential register5488 // pressure, the logics here is that the differnce are used5489 // to measure the distance between def and use, the longer distance5490 // more likely cause register pressure.5491 if (orderNo - orderNo2 < 500)5492 return SDValue();5493 5494 // Now, check if at least one of the FMUL's operands is live beyond the5495 // node N, which guarantees that the FMA will not increase register5496 // pressure at node N.5497 bool opIsLive = false;5498 const SDNode *left = N0.getOperand(0).getNode();5499 const SDNode *right = N0.getOperand(1).getNode();5500 5501 if (isa<ConstantSDNode>(left) || isa<ConstantSDNode>(right))5502 opIsLive = true;5503 5504 if (!opIsLive)5505 for (const SDNode *User : left->users()) {5506 int orderNo3 = User->getIROrder();5507 if (orderNo3 > orderNo) {5508 opIsLive = true;5509 break;5510 }5511 }5512 5513 if (!opIsLive)5514 for (const SDNode *User : right->users()) {5515 int orderNo3 = User->getIROrder();5516 if (orderNo3 > orderNo) {5517 opIsLive = true;5518 break;5519 }5520 }5521 5522 if (!opIsLive)5523 return SDValue();5524 }5525 5526 return DCI.DAG.getNode(ISD::FMA, SDLoc(N), VT, N0.getOperand(0),5527 N0.getOperand(1), N1);5528 }5529 5530 return SDValue();5531}5532 5533/// Fold unpacking movs into a load by increasing the number of return values.5534///5535/// ex:5536/// L: v2f16,ch = load <p>5537/// a: f16 = extractelt L:0, 05538/// b: f16 = extractelt L:0, 15539/// use(a, b)5540///5541/// ...is turned into...5542///5543/// L: f16,f16,ch = LoadV2 <p>5544/// use(L:0, L:1)5545static SDValue5546combineUnpackingMovIntoLoad(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {5547 // Don't run this optimization before the legalizer5548 if (!DCI.isAfterLegalizeDAG())5549 return SDValue();5550 5551 EVT ElementVT = N->getValueType(0);5552 // Avoid non-packed types and v4i85553 if (!NVPTX::isPackedVectorTy(ElementVT) || ElementVT == MVT::v4i8)5554 return SDValue();5555 5556 // Check whether all outputs are either used by an extractelt or are5557 // glue/chain nodes5558 if (!all_of(N->uses(), [&](SDUse &U) {5559 // Skip glue, chain nodes5560 if (U.getValueType() == MVT::Glue || U.getValueType() == MVT::Other)5561 return true;5562 if (U.getUser()->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {5563 if (N->getOpcode() != ISD::LOAD)5564 return true;5565 // Since this is an ISD::LOAD, check all extractelts are used. If5566 // any are not used, we don't want to defeat another optimization that5567 // will narrow the load.5568 //5569 // For example:5570 //5571 // L: v2f16,ch = load <p>5572 // e0: f16 = extractelt L:0, 05573 // e1: f16 = extractelt L:0, 1 <-- unused5574 // store e05575 //5576 // Can be optimized by DAGCombiner to:5577 //5578 // L: f16,ch = load <p>5579 // store L:05580 return !U.getUser()->use_empty();5581 }5582 5583 // Otherwise, this use prevents us from splitting a value.5584 return false;5585 }))5586 return SDValue();5587 5588 auto *LD = cast<MemSDNode>(N);5589 SDLoc DL(LD);5590 5591 // the new opcode after we double the number of operands5592 unsigned Opcode;5593 SmallVector<SDValue> Operands(LD->ops());5594 unsigned OldNumOutputs; // non-glue, non-chain outputs5595 switch (LD->getOpcode()) {5596 case ISD::LOAD:5597 OldNumOutputs = 1;5598 // Any packed type is legal, so the legalizer will not have lowered5599 // ISD::LOAD -> NVPTXISD::Load (unless it's under-aligned). We have to do it5600 // here.5601 Opcode = NVPTXISD::LoadV2;5602 // append a "full" used bytes mask operand right before the extension type5603 // operand, signifying that all bytes are used.5604 Operands.push_back(DCI.DAG.getConstant(UINT32_MAX, DL, MVT::i32));5605 Operands.push_back(DCI.DAG.getIntPtrConstant(5606 cast<LoadSDNode>(LD)->getExtensionType(), DL));5607 break;5608 case NVPTXISD::LoadV2:5609 OldNumOutputs = 2;5610 Opcode = NVPTXISD::LoadV4;5611 break;5612 case NVPTXISD::LoadV4:5613 // V8 is only supported for f32/i32. Don't forget, we're not changing the5614 // load size here. This is already a 256-bit load.5615 if (ElementVT != MVT::v2f32 && ElementVT != MVT::v2i32)5616 return SDValue();5617 OldNumOutputs = 4;5618 Opcode = NVPTXISD::LoadV8;5619 break;5620 case NVPTXISD::LoadV8:5621 // PTX doesn't support the next doubling of outputs5622 return SDValue();5623 }5624 5625 // the non-glue, non-chain outputs in the new load5626 const unsigned NewNumOutputs = OldNumOutputs * 2;5627 SmallVector<EVT> NewVTs(NewNumOutputs, ElementVT.getVectorElementType());5628 // add remaining chain and glue values5629 NewVTs.append(LD->value_begin() + OldNumOutputs, LD->value_end());5630 5631 // Create the new load5632 SDValue NewLoad = DCI.DAG.getMemIntrinsicNode(5633 Opcode, DL, DCI.DAG.getVTList(NewVTs), Operands, LD->getMemoryVT(),5634 LD->getMemOperand());5635 5636 // Now we use a combination of BUILD_VECTORs and a MERGE_VALUES node to keep5637 // the outputs the same. These nodes will be optimized away in later5638 // DAGCombiner iterations.5639 SmallVector<SDValue> Results;5640 for (unsigned I : seq(OldNumOutputs))5641 Results.push_back(DCI.DAG.getBuildVector(5642 ElementVT, DL, {NewLoad.getValue(I * 2), NewLoad.getValue(I * 2 + 1)}));5643 // Add remaining chain and glue nodes5644 for (unsigned I : seq(NewLoad->getNumValues() - NewNumOutputs))5645 Results.push_back(NewLoad.getValue(NewNumOutputs + I));5646 5647 return DCI.DAG.getMergeValues(Results, DL);5648}5649 5650/// Fold packing movs into a store.5651///5652/// ex:5653/// v1: v2f16 = BUILD_VECTOR a:f16, b:f165654/// v2: v2f16 = BUILD_VECTOR c:f16, d:f165655/// StoreV2 v1, v25656///5657/// ...is turned into...5658///5659/// StoreV4 a, b, c, d5660static SDValue combinePackingMovIntoStore(SDNode *N,5661 TargetLowering::DAGCombinerInfo &DCI,5662 unsigned Front, unsigned Back) {5663 // We want to run this as late as possible since other optimizations may5664 // eliminate the BUILD_VECTORs.5665 if (!DCI.isAfterLegalizeDAG())5666 return SDValue();5667 5668 // Get the type of the operands being stored.5669 EVT ElementVT = N->getOperand(Front).getValueType();5670 5671 // Avoid non-packed types and v4i85672 if (!NVPTX::isPackedVectorTy(ElementVT) || ElementVT == MVT::v4i8)5673 return SDValue();5674 5675 auto *ST = cast<MemSDNode>(N);5676 5677 // The new opcode after we double the number of operands.5678 unsigned Opcode;5679 switch (N->getOpcode()) {5680 case ISD::STORE:5681 // Any packed type is legal, so the legalizer will not have lowered5682 // ISD::STORE -> NVPTXISD::Store (unless it's under-aligned). We have to do5683 // it here.5684 Opcode = NVPTXISD::StoreV2;5685 break;5686 case NVPTXISD::StoreV2:5687 Opcode = NVPTXISD::StoreV4;5688 break;5689 case NVPTXISD::StoreV4:5690 // V8 is only supported for f32/i32. Don't forget, we're not changing the5691 // store size here. This is already a 256-bit store.5692 if (ElementVT != MVT::v2f32 && ElementVT != MVT::v2i32)5693 return SDValue();5694 Opcode = NVPTXISD::StoreV8;5695 break;5696 case NVPTXISD::StoreV8:5697 // PTX doesn't support the next doubling of operands5698 return SDValue();5699 default:5700 llvm_unreachable("Unhandled store opcode");5701 }5702 5703 // Scan the operands and if they're all BUILD_VECTORs, we'll have gathered5704 // their elements.5705 SmallVector<SDValue, 4> Operands(N->ops().take_front(Front));5706 for (SDValue BV : N->ops().drop_front(Front).drop_back(Back)) {5707 if (BV.getOpcode() != ISD::BUILD_VECTOR)5708 return SDValue();5709 5710 // If the operand has multiple uses, this optimization can increase register5711 // pressure.5712 if (!BV.hasOneUse())5713 return SDValue();5714 5715 // DAGCombiner visits nodes bottom-up. Check the BUILD_VECTOR operands for5716 // any signs they may be folded by some other pattern or rule.5717 for (SDValue Op : BV->ops()) {5718 // Peek through bitcasts5719 if (Op.getOpcode() == ISD::BITCAST)5720 Op = Op.getOperand(0);5721 5722 // This may be folded into a PRMT.5723 if (Op.getValueType() == MVT::i16 && Op.getOpcode() == ISD::TRUNCATE &&5724 Op->getOperand(0).getValueType() == MVT::i32)5725 return SDValue();5726 5727 // This may be folded into cvt.bf16x25728 if (Op.getOpcode() == ISD::FP_ROUND)5729 return SDValue();5730 }5731 Operands.append({BV.getOperand(0), BV.getOperand(1)});5732 }5733 Operands.append(N->op_end() - Back, N->op_end());5734 5735 // Now we replace the store5736 return DCI.DAG.getMemIntrinsicNode(Opcode, SDLoc(N), N->getVTList(), Operands,5737 ST->getMemoryVT(), ST->getMemOperand());5738}5739 5740static SDValue combineSTORE(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,5741 const NVPTXSubtarget &STI) {5742 5743 if (DCI.isBeforeLegalize() && N->getOpcode() == ISD::STORE) {5744 // Here is our chance to custom lower a store with a non-simple type.5745 // Unfortunately, we can't do this in the legalizer because there is no5746 // way to setOperationAction for an non-simple type.5747 StoreSDNode *ST = cast<StoreSDNode>(N);5748 if (!ST->getValue().getValueType().isSimple())5749 return lowerSTOREVector(SDValue(ST, 0), DCI.DAG, STI);5750 }5751 5752 return combinePackingMovIntoStore(N, DCI, 1, 2);5753}5754 5755static SDValue combineLOAD(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,5756 const NVPTXSubtarget &STI) {5757 if (DCI.isBeforeLegalize() && N->getOpcode() == ISD::LOAD) {5758 // Here is our chance to custom lower a load with a non-simple type.5759 // Unfortunately, we can't do this in the legalizer because there is no5760 // way to setOperationAction for an non-simple type.5761 if (!N->getValueType(0).isSimple())5762 return lowerLoadVector(N, DCI.DAG, STI);5763 }5764 5765 return combineUnpackingMovIntoLoad(N, DCI);5766}5767 5768/// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.5769///5770static SDValue PerformADDCombine(SDNode *N,5771 TargetLowering::DAGCombinerInfo &DCI,5772 CodeGenOptLevel OptLevel) {5773 if (OptLevel == CodeGenOptLevel::None)5774 return SDValue();5775 5776 SDValue N0 = N->getOperand(0);5777 SDValue N1 = N->getOperand(1);5778 5779 // Skip non-integer, non-scalar case5780 EVT VT = N0.getValueType();5781 if (VT.isVector() || VT != MVT::i32)5782 return SDValue();5783 5784 // First try with the default operand order.5785 if (SDValue Result = PerformADDCombineWithOperands(N, N0, N1, DCI))5786 return Result;5787 5788 // If that didn't work, try again with the operands commuted.5789 return PerformADDCombineWithOperands(N, N1, N0, DCI);5790}5791 5792/// PerformFADDCombine - Target-specific dag combine xforms for ISD::FADD.5793///5794static SDValue PerformFADDCombine(SDNode *N,5795 TargetLowering::DAGCombinerInfo &DCI,5796 CodeGenOptLevel OptLevel) {5797 SDValue N0 = N->getOperand(0);5798 SDValue N1 = N->getOperand(1);5799 5800 EVT VT = N0.getValueType();5801 if (VT.isVector() || !(VT == MVT::f32 || VT == MVT::f64))5802 return SDValue();5803 5804 // First try with the default operand order.5805 if (SDValue Result = PerformFADDCombineWithOperands(N, N0, N1, DCI, OptLevel))5806 return Result;5807 5808 // If that didn't work, try again with the operands commuted.5809 return PerformFADDCombineWithOperands(N, N1, N0, DCI, OptLevel);5810}5811 5812/// Get 3-input version of a 2-input min/max opcode5813static unsigned getMinMax3Opcode(unsigned MinMax2Opcode) {5814 switch (MinMax2Opcode) {5815 case ISD::FMAXNUM:5816 case ISD::FMAXIMUMNUM:5817 return NVPTXISD::FMAXNUM3;5818 case ISD::FMINNUM:5819 case ISD::FMINIMUMNUM:5820 return NVPTXISD::FMINNUM3;5821 case ISD::FMAXIMUM:5822 return NVPTXISD::FMAXIMUM3;5823 case ISD::FMINIMUM:5824 return NVPTXISD::FMINIMUM3;5825 default:5826 llvm_unreachable("Invalid 2-input min/max opcode");5827 }5828}5829 5830/// PerformFMinMaxCombine - Combine (fmaxnum (fmaxnum a, b), c) into5831/// (fmaxnum3 a, b, c). Also covers other llvm min/max intrinsics.5832static SDValue PerformFMinMaxCombine(SDNode *N,5833 TargetLowering::DAGCombinerInfo &DCI,5834 unsigned PTXVersion, unsigned SmVersion) {5835 5836 // 3-input min/max requires PTX 8.8+ and SM_100+, and only supports f32s5837 EVT VT = N->getValueType(0);5838 if (VT != MVT::f32 || PTXVersion < 88 || SmVersion < 100)5839 return SDValue();5840 5841 SDValue Op0 = N->getOperand(0);5842 SDValue Op1 = N->getOperand(1);5843 unsigned MinMaxOp2 = N->getOpcode();5844 unsigned MinMaxOp3 = getMinMax3Opcode(MinMaxOp2);5845 5846 if (Op0.getOpcode() == MinMaxOp2 && Op0.hasOneUse()) {5847 // (maxnum (maxnum a, b), c) -> (maxnum3 a, b, c)5848 SDValue A = Op0.getOperand(0);5849 SDValue B = Op0.getOperand(1);5850 SDValue C = Op1;5851 return DCI.DAG.getNode(MinMaxOp3, SDLoc(N), VT, A, B, C, N->getFlags());5852 } else if (Op1.getOpcode() == MinMaxOp2 && Op1.hasOneUse()) {5853 // (maxnum a, (maxnum b, c)) -> (maxnum3 a, b, c)5854 SDValue A = Op0;5855 SDValue B = Op1.getOperand(0);5856 SDValue C = Op1.getOperand(1);5857 return DCI.DAG.getNode(MinMaxOp3, SDLoc(N), VT, A, B, C, N->getFlags());5858 }5859 return SDValue();5860}5861 5862static SDValue PerformREMCombine(SDNode *N,5863 TargetLowering::DAGCombinerInfo &DCI,5864 CodeGenOptLevel OptLevel) {5865 assert(N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM);5866 5867 // Don't do anything at less than -O2.5868 if (OptLevel < CodeGenOptLevel::Default)5869 return SDValue();5870 5871 SelectionDAG &DAG = DCI.DAG;5872 SDLoc DL(N);5873 EVT VT = N->getValueType(0);5874 bool IsSigned = N->getOpcode() == ISD::SREM;5875 unsigned DivOpc = IsSigned ? ISD::SDIV : ISD::UDIV;5876 5877 const SDValue &Num = N->getOperand(0);5878 const SDValue &Den = N->getOperand(1);5879 5880 for (const SDNode *U : Num->users()) {5881 if (U->getOpcode() == DivOpc && U->getOperand(0) == Num &&5882 U->getOperand(1) == Den) {5883 // Num % Den -> Num - (Num / Den) * Den5884 return DAG.getNode(ISD::SUB, DL, VT, Num,5885 DAG.getNode(ISD::MUL, DL, VT,5886 DAG.getNode(DivOpc, DL, VT, Num, Den),5887 Den));5888 }5889 }5890 return SDValue();5891}5892 5893// (sign_extend|zero_extend (mul|shl) x, y) -> (mul.wide x, y)5894static SDValue combineMulWide(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,5895 CodeGenOptLevel OptLevel) {5896 if (OptLevel == CodeGenOptLevel::None)5897 return SDValue();5898 5899 SDValue Op = N->getOperand(0);5900 if (!Op.hasOneUse())5901 return SDValue();5902 EVT ToVT = N->getValueType(0);5903 EVT FromVT = Op.getValueType();5904 if (!((ToVT == MVT::i32 && FromVT == MVT::i16) ||5905 (ToVT == MVT::i64 && FromVT == MVT::i32)))5906 return SDValue();5907 if (!(Op.getOpcode() == ISD::MUL ||5908 (Op.getOpcode() == ISD::SHL && isa<ConstantSDNode>(Op.getOperand(1)))))5909 return SDValue();5910 5911 SDLoc DL(N);5912 unsigned ExtOpcode = N->getOpcode();5913 unsigned Opcode = 0;5914 if (ExtOpcode == ISD::SIGN_EXTEND && Op->getFlags().hasNoSignedWrap())5915 Opcode = NVPTXISD::MUL_WIDE_SIGNED;5916 else if (ExtOpcode == ISD::ZERO_EXTEND && Op->getFlags().hasNoUnsignedWrap())5917 Opcode = NVPTXISD::MUL_WIDE_UNSIGNED;5918 else5919 return SDValue();5920 SDValue RHS = Op.getOperand(1);5921 if (Op.getOpcode() == ISD::SHL) {5922 const auto ShiftAmt = Op.getConstantOperandVal(1);5923 const auto MulVal = APInt(ToVT.getSizeInBits(), 1) << ShiftAmt;5924 RHS = DCI.DAG.getConstant(MulVal, DL, ToVT);5925 }5926 return DCI.DAG.getNode(Opcode, DL, ToVT, Op.getOperand(0), RHS);5927}5928 5929enum OperandSignedness {5930 Signed = 0,5931 Unsigned,5932 Unknown5933};5934 5935/// IsMulWideOperandDemotable - Checks if the provided DAG node is an operand5936/// that can be demoted to \p OptSize bits without loss of information. The5937/// signedness of the operand, if determinable, is placed in \p S.5938static bool IsMulWideOperandDemotable(SDValue Op,5939 unsigned OptSize,5940 OperandSignedness &S) {5941 S = Unknown;5942 5943 if (Op.getOpcode() == ISD::SIGN_EXTEND ||5944 Op.getOpcode() == ISD::SIGN_EXTEND_INREG) {5945 EVT OrigVT = Op.getOperand(0).getValueType();5946 if (OrigVT.getFixedSizeInBits() <= OptSize) {5947 S = Signed;5948 return true;5949 }5950 } else if (Op.getOpcode() == ISD::ZERO_EXTEND) {5951 EVT OrigVT = Op.getOperand(0).getValueType();5952 if (OrigVT.getFixedSizeInBits() <= OptSize) {5953 S = Unsigned;5954 return true;5955 }5956 }5957 5958 return false;5959}5960 5961/// AreMulWideOperandsDemotable - Checks if the given LHS and RHS operands can5962/// be demoted to \p OptSize bits without loss of information. If the operands5963/// contain a constant, it should appear as the RHS operand. The signedness of5964/// the operands is placed in \p IsSigned.5965static bool AreMulWideOperandsDemotable(SDValue LHS, SDValue RHS,5966 unsigned OptSize,5967 bool &IsSigned) {5968 OperandSignedness LHSSign;5969 5970 // The LHS operand must be a demotable op5971 if (!IsMulWideOperandDemotable(LHS, OptSize, LHSSign))5972 return false;5973 5974 // We should have been able to determine the signedness from the LHS5975 if (LHSSign == Unknown)5976 return false;5977 5978 IsSigned = (LHSSign == Signed);5979 5980 // The RHS can be a demotable op or a constant5981 if (ConstantSDNode *CI = dyn_cast<ConstantSDNode>(RHS)) {5982 const APInt &Val = CI->getAPIntValue();5983 if (LHSSign == Unsigned) {5984 return Val.isIntN(OptSize);5985 } else {5986 return Val.isSignedIntN(OptSize);5987 }5988 } else {5989 OperandSignedness RHSSign;5990 if (!IsMulWideOperandDemotable(RHS, OptSize, RHSSign))5991 return false;5992 5993 return LHSSign == RHSSign;5994 }5995}5996 5997/// TryMULWIDECombine - Attempt to replace a multiply of M bits with a multiply5998/// of M/2 bits that produces an M-bit result (i.e. mul.wide). This transform5999/// works on both multiply DAG nodes and SHL DAG nodes with a constant shift6000/// amount.6001static SDValue TryMULWIDECombine(SDNode *N,6002 TargetLowering::DAGCombinerInfo &DCI) {6003 EVT MulType = N->getValueType(0);6004 if (MulType != MVT::i32 && MulType != MVT::i64) {6005 return SDValue();6006 }6007 6008 SDLoc DL(N);6009 unsigned OptSize = MulType.getSizeInBits() >> 1;6010 SDValue LHS = N->getOperand(0);6011 SDValue RHS = N->getOperand(1);6012 6013 // Canonicalize the multiply so the constant (if any) is on the right6014 if (N->getOpcode() == ISD::MUL) {6015 if (isa<ConstantSDNode>(LHS)) {6016 std::swap(LHS, RHS);6017 }6018 }6019 6020 // If we have a SHL, determine the actual multiply amount6021 if (N->getOpcode() == ISD::SHL) {6022 ConstantSDNode *ShlRHS = dyn_cast<ConstantSDNode>(RHS);6023 if (!ShlRHS) {6024 return SDValue();6025 }6026 6027 APInt ShiftAmt = ShlRHS->getAPIntValue();6028 unsigned BitWidth = MulType.getSizeInBits();6029 if (ShiftAmt.sge(0) && ShiftAmt.slt(BitWidth)) {6030 APInt MulVal = APInt(BitWidth, 1) << ShiftAmt;6031 RHS = DCI.DAG.getConstant(MulVal, DL, MulType);6032 } else {6033 return SDValue();6034 }6035 }6036 6037 bool Signed;6038 // Verify that our operands are demotable6039 if (!AreMulWideOperandsDemotable(LHS, RHS, OptSize, Signed)) {6040 return SDValue();6041 }6042 6043 EVT DemotedVT;6044 if (MulType == MVT::i32) {6045 DemotedVT = MVT::i16;6046 } else {6047 DemotedVT = MVT::i32;6048 }6049 6050 // Truncate the operands to the correct size. Note that these are just for6051 // type consistency and will (likely) be eliminated in later phases.6052 SDValue TruncLHS =6053 DCI.DAG.getNode(ISD::TRUNCATE, DL, DemotedVT, LHS);6054 SDValue TruncRHS =6055 DCI.DAG.getNode(ISD::TRUNCATE, DL, DemotedVT, RHS);6056 6057 unsigned Opc;6058 if (Signed) {6059 Opc = NVPTXISD::MUL_WIDE_SIGNED;6060 } else {6061 Opc = NVPTXISD::MUL_WIDE_UNSIGNED;6062 }6063 6064 return DCI.DAG.getNode(Opc, DL, MulType, TruncLHS, TruncRHS);6065}6066 6067static bool isConstOne(const SDValue &Operand) {6068 const auto *Const = dyn_cast<ConstantSDNode>(Operand);6069 return Const && Const->getZExtValue() == 1;6070}6071 6072static SDValue matchMADConstOnePattern(SDValue Add) {6073 if (Add->getOpcode() != ISD::ADD)6074 return SDValue();6075 6076 if (isConstOne(Add->getOperand(0)))6077 return Add->getOperand(1);6078 6079 if (isConstOne(Add->getOperand(1)))6080 return Add->getOperand(0);6081 6082 return SDValue();6083}6084 6085static SDValue combineMADConstOne(SDValue X, SDValue Add, EVT VT, SDLoc DL,6086 TargetLowering::DAGCombinerInfo &DCI) {6087 6088 if (SDValue Y = matchMADConstOnePattern(Add)) {6089 SDValue Mul = DCI.DAG.getNode(ISD::MUL, DL, VT, X, Y);6090 return DCI.DAG.getNode(ISD::ADD, DL, VT, Mul, X);6091 }6092 6093 return SDValue();6094}6095 6096static SDValue combineMulSelectConstOne(SDValue X, SDValue Select, EVT VT,6097 SDLoc DL,6098 TargetLowering::DAGCombinerInfo &DCI) {6099 if (Select->getOpcode() != ISD::SELECT)6100 return SDValue();6101 6102 SDValue Cond = Select->getOperand(0);6103 6104 unsigned ConstOpNo;6105 if (isConstOne(Select->getOperand(1)))6106 ConstOpNo = 1;6107 else if (isConstOne(Select->getOperand(2)))6108 ConstOpNo = 2;6109 else6110 return SDValue();6111 6112 SDValue Y = Select->getOperand((ConstOpNo == 1) ? 2 : 1);6113 6114 // Do not combine if the resulting sequence is not obviously profitable.6115 if (!matchMADConstOnePattern(Y))6116 return SDValue();6117 6118 SDValue NewMul = DCI.DAG.getNode(ISD::MUL, DL, VT, X, Y);6119 6120 return DCI.DAG.getNode(ISD::SELECT, DL, VT, Cond,6121 (ConstOpNo == 1) ? X : NewMul,6122 (ConstOpNo == 1) ? NewMul : X);6123}6124 6125static SDValue6126PerformMULCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,6127 TargetLowering::DAGCombinerInfo &DCI) {6128 6129 EVT VT = N0.getValueType();6130 if (VT.isVector())6131 return SDValue();6132 6133 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)6134 return SDValue();6135 6136 SDLoc DL(N);6137 6138 // (mul x, (add y, 1)) -> (add (mul x, y), x)6139 if (SDValue Res = combineMADConstOne(N0, N1, VT, DL, DCI))6140 return Res;6141 if (SDValue Res = combineMADConstOne(N1, N0, VT, DL, DCI))6142 return Res;6143 6144 // (mul x, (select y, 1)) -> (select (mul x, y), x)6145 if (SDValue Res = combineMulSelectConstOne(N0, N1, VT, DL, DCI))6146 return Res;6147 if (SDValue Res = combineMulSelectConstOne(N1, N0, VT, DL, DCI))6148 return Res;6149 6150 return SDValue();6151}6152 6153/// PerformMULCombine - Runs PTX-specific DAG combine patterns on MUL nodes.6154static SDValue PerformMULCombine(SDNode *N,6155 TargetLowering::DAGCombinerInfo &DCI,6156 CodeGenOptLevel OptLevel) {6157 if (OptLevel == CodeGenOptLevel::None)6158 return SDValue();6159 6160 if (SDValue Ret = TryMULWIDECombine(N, DCI))6161 return Ret;6162 6163 SDValue N0 = N->getOperand(0);6164 SDValue N1 = N->getOperand(1);6165 return PerformMULCombineWithOperands(N, N0, N1, DCI);6166}6167 6168/// PerformSHLCombine - Runs PTX-specific DAG combine patterns on SHL nodes.6169static SDValue PerformSHLCombine(SDNode *N,6170 TargetLowering::DAGCombinerInfo &DCI,6171 CodeGenOptLevel OptLevel) {6172 if (OptLevel > CodeGenOptLevel::None) {6173 // Try mul.wide combining at OptLevel > 06174 if (SDValue Ret = TryMULWIDECombine(N, DCI))6175 return Ret;6176 }6177 6178 return SDValue();6179}6180 6181static SDValue PerformSETCCCombine(SDNode *N,6182 TargetLowering::DAGCombinerInfo &DCI,6183 unsigned int SmVersion) {6184 EVT CCType = N->getValueType(0);6185 SDValue A = N->getOperand(0);6186 SDValue B = N->getOperand(1);6187 6188 EVT AType = A.getValueType();6189 if (!(CCType == MVT::v2i1 && (AType == MVT::v2f16 || AType == MVT::v2bf16)))6190 return SDValue();6191 6192 if (A.getValueType() == MVT::v2bf16 && SmVersion < 90)6193 return SDValue();6194 6195 SDLoc DL(N);6196 // setp.f16x2 returns two scalar predicates, which we need to6197 // convert back to v2i1. The returned result will be scalarized by6198 // the legalizer, but the comparison will remain a single vector6199 // instruction.6200 SDValue CCNode = DCI.DAG.getNode(6201 A.getValueType() == MVT::v2f16 ? NVPTXISD::SETP_F16X26202 : NVPTXISD::SETP_BF16X2,6203 DL, DCI.DAG.getVTList(MVT::i1, MVT::i1), {A, B, N->getOperand(2)});6204 return DCI.DAG.getNode(ISD::BUILD_VECTOR, DL, CCType, CCNode.getValue(0),6205 CCNode.getValue(1));6206}6207 6208static SDValue PerformEXTRACTCombine(SDNode *N,6209 TargetLowering::DAGCombinerInfo &DCI) {6210 SDValue Vector = N->getOperand(0);6211 if (Vector->getOpcode() == ISD::FREEZE)6212 Vector = Vector->getOperand(0);6213 SDLoc DL(N);6214 EVT VectorVT = Vector.getValueType();6215 if (Vector->getOpcode() == ISD::LOAD && VectorVT.isSimple() &&6216 IsPTXVectorType(VectorVT.getSimpleVT()))6217 return SDValue(); // Native vector loads already combine nicely w/6218 // extract_vector_elt.6219 // Don't mess with singletons or packed types (v2*32, v2*16, v4i8 and v8i8),6220 // we already handle them OK.6221 if (VectorVT.getVectorNumElements() == 1 ||6222 NVPTX::isPackedVectorTy(VectorVT) || VectorVT == MVT::v8i8)6223 return SDValue();6224 6225 // Don't mess with undef values as sra may be simplified to 0, not undef.6226 if (Vector->isUndef() || ISD::allOperandsUndef(Vector.getNode()))6227 return SDValue();6228 6229 uint64_t VectorBits = VectorVT.getSizeInBits();6230 // We only handle the types we can extract in-register.6231 if (!(VectorBits == 16 || VectorBits == 32 || VectorBits == 64))6232 return SDValue();6233 6234 ConstantSDNode *Index = dyn_cast<ConstantSDNode>(N->getOperand(1));6235 // Index == 0 is handled by generic DAG combiner.6236 if (!Index || Index->getZExtValue() == 0)6237 return SDValue();6238 6239 MVT IVT = MVT::getIntegerVT(VectorBits);6240 EVT EltVT = VectorVT.getVectorElementType();6241 EVT EltIVT = EltVT.changeTypeToInteger();6242 uint64_t EltBits = EltVT.getScalarSizeInBits();6243 6244 SDValue Result = DCI.DAG.getNode(6245 ISD::TRUNCATE, DL, EltIVT,6246 DCI.DAG.getNode(6247 ISD::SRA, DL, IVT, DCI.DAG.getNode(ISD::BITCAST, DL, IVT, Vector),6248 DCI.DAG.getConstant(Index->getZExtValue() * EltBits, DL, IVT)));6249 6250 // If element has non-integer type, bitcast it back to the expected type.6251 if (EltVT != EltIVT)6252 Result = DCI.DAG.getNode(ISD::BITCAST, DL, EltVT, Result);6253 // Past legalizer, we may need to extent i8 -> i16 to match the register type.6254 if (EltVT != N->getValueType(0))6255 Result = DCI.DAG.getNode(ISD::ANY_EXTEND, DL, N->getValueType(0), Result);6256 6257 return Result;6258}6259 6260static SDValue PerformVSELECTCombine(SDNode *N,6261 TargetLowering::DAGCombinerInfo &DCI) {6262 SDValue VA = N->getOperand(1);6263 EVT VectorVT = VA.getValueType();6264 if (VectorVT != MVT::v4i8)6265 return SDValue();6266 6267 // We need to split vselect into individual per-element operations Because we6268 // use BFE/BFI instruction for byte extraction/insertion, we do end up with6269 // 32-bit values, so we may as well do comparison as i32 to avoid conversions6270 // to/from i16 normally used for i8 values.6271 SmallVector<SDValue, 4> E;6272 SDLoc DL(N);6273 SDValue VCond = N->getOperand(0);6274 SDValue VB = N->getOperand(2);6275 for (int I = 0; I < 4; ++I) {6276 SDValue C = DCI.DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i1, VCond,6277 DCI.DAG.getConstant(I, DL, MVT::i32));6278 SDValue EA = DCI.DAG.getAnyExtOrTrunc(6279 DCI.DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i8, VA,6280 DCI.DAG.getConstant(I, DL, MVT::i32)),6281 DL, MVT::i32);6282 SDValue EB = DCI.DAG.getAnyExtOrTrunc(6283 DCI.DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i8, VB,6284 DCI.DAG.getConstant(I, DL, MVT::i32)),6285 DL, MVT::i32);6286 E.push_back(DCI.DAG.getAnyExtOrTrunc(6287 DCI.DAG.getNode(ISD::SELECT, DL, MVT::i32, C, EA, EB), DL, MVT::i8));6288 }6289 return DCI.DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v4i8, E);6290}6291 6292static SDValue6293PerformBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {6294 auto VT = N->getValueType(0);6295 if (!DCI.isAfterLegalizeDAG() ||6296 // only process v2*16 types6297 !(NVPTX::isPackedVectorTy(VT) && VT.is32BitVector() &&6298 VT.getVectorNumElements() == 2))6299 return SDValue();6300 6301 auto Op0 = N->getOperand(0);6302 auto Op1 = N->getOperand(1);6303 6304 // Start out by assuming we want to take the lower 2 bytes of each i326305 // operand.6306 uint64_t Op0Bytes = 0x10;6307 uint64_t Op1Bytes = 0x54;6308 6309 std::pair<SDValue *, uint64_t *> OpData[2] = {{&Op0, &Op0Bytes},6310 {&Op1, &Op1Bytes}};6311 6312 // Check that each operand is an i16, truncated from an i32 operand. We'll6313 // select individual bytes from those original operands. Optionally, fold in a6314 // shift right of that original operand.6315 for (auto &[Op, OpBytes] : OpData) {6316 // Eat up any bitcast6317 if (Op->getOpcode() == ISD::BITCAST)6318 *Op = Op->getOperand(0);6319 6320 if (!(Op->getValueType() == MVT::i16 && Op->getOpcode() == ISD::TRUNCATE &&6321 Op->getOperand(0).getValueType() == MVT::i32))6322 return SDValue();6323 6324 // If the truncate has multiple uses, this optimization can increase6325 // register pressure6326 if (!Op->hasOneUse())6327 return SDValue();6328 6329 *Op = Op->getOperand(0);6330 6331 // Optionally, fold in a shift-right of the original operand and let permute6332 // pick the two higher bytes of the original value directly.6333 if (Op->getOpcode() == ISD::SRL && isa<ConstantSDNode>(Op->getOperand(1))) {6334 if (cast<ConstantSDNode>(Op->getOperand(1))->getZExtValue() == 16) {6335 // Shift the PRMT byte selector to pick upper bytes from each respective6336 // value, instead of the lower ones: 0x10 -> 0x32, 0x54 -> 0x766337 assert((*OpBytes == 0x10 || *OpBytes == 0x54) &&6338 "PRMT selector values out of range");6339 *OpBytes += 0x22;6340 *Op = Op->getOperand(0);6341 }6342 }6343 }6344 6345 SDLoc DL(N);6346 auto &DAG = DCI.DAG;6347 6348 auto PRMT =6349 getPRMT(DAG.getBitcast(MVT::i32, Op0), DAG.getBitcast(MVT::i32, Op1),6350 (Op1Bytes << 8) | Op0Bytes, DL, DAG);6351 return DAG.getBitcast(VT, PRMT);6352}6353 6354static SDValue combineADDRSPACECAST(SDNode *N,6355 TargetLowering::DAGCombinerInfo &DCI) {6356 auto *ASCN1 = cast<AddrSpaceCastSDNode>(N);6357 6358 if (auto *ASCN2 = dyn_cast<AddrSpaceCastSDNode>(ASCN1->getOperand(0))) {6359 assert(ASCN2->getDestAddressSpace() == ASCN1->getSrcAddressSpace());6360 6361 // Fold asc[B -> A](asc[A -> B](x)) -> x6362 if (ASCN1->getDestAddressSpace() == ASCN2->getSrcAddressSpace())6363 return ASCN2->getOperand(0);6364 }6365 6366 return SDValue();6367}6368 6369// Given a constant selector value and a prmt mode, return the selector value6370// normalized to the generic prmt mode. See the PTX ISA documentation for more6371// details:6372// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-prmt6373static APInt getPRMTSelector(const APInt &Selector, unsigned Mode) {6374 assert(Selector.getBitWidth() == 32 && "PRMT must have i32 operands");6375 6376 if (Mode == NVPTX::PTXPrmtMode::NONE)6377 return Selector;6378 6379 const unsigned V = Selector.trunc(2).getZExtValue();6380 6381 const auto GetSelector = [](unsigned S0, unsigned S1, unsigned S2,6382 unsigned S3) {6383 return APInt(32, S0 | (S1 << 4) | (S2 << 8) | (S3 << 12));6384 };6385 6386 switch (Mode) {6387 case NVPTX::PTXPrmtMode::F4E:6388 return GetSelector(V, V + 1, V + 2, V + 3);6389 case NVPTX::PTXPrmtMode::B4E:6390 return GetSelector(V, (V - 1) & 7, (V - 2) & 7, (V - 3) & 7);6391 case NVPTX::PTXPrmtMode::RC8:6392 return GetSelector(V, V, V, V);6393 case NVPTX::PTXPrmtMode::ECL:6394 return GetSelector(V, std::max(V, 1U), std::max(V, 2U), 3U);6395 case NVPTX::PTXPrmtMode::ECR:6396 return GetSelector(0, std::min(V, 1U), std::min(V, 2U), V);6397 case NVPTX::PTXPrmtMode::RC16: {6398 unsigned V1 = (V & 1) << 1;6399 return GetSelector(V1, V1 + 1, V1, V1 + 1);6400 }6401 default:6402 llvm_unreachable("Invalid PRMT mode");6403 }6404}6405 6406static APInt computePRMT(APInt A, APInt B, APInt Selector, unsigned Mode) {6407 assert(A.getBitWidth() == 32 && B.getBitWidth() == 32 &&6408 Selector.getBitWidth() == 32 && "PRMT must have i32 operands");6409 // {b, a} = {{b7, b6, b5, b4}, {b3, b2, b1, b0}}6410 APInt BitField = B.concat(A);6411 APInt SelectorVal = getPRMTSelector(Selector, Mode);6412 APInt Result(32, 0);6413 for (unsigned I : llvm::seq(4U)) {6414 APInt Sel = SelectorVal.extractBits(4, I * 4);6415 unsigned Idx = Sel.getLoBits(3).getZExtValue();6416 unsigned Sign = Sel.getHiBits(1).getZExtValue();6417 APInt Byte = BitField.extractBits(8, Idx * 8);6418 if (Sign)6419 Byte = Byte.ashr(8);6420 Result.insertBits(Byte, I * 8);6421 }6422 return Result;6423}6424 6425static SDValue combinePRMT(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,6426 CodeGenOptLevel OptLevel) {6427 if (OptLevel == CodeGenOptLevel::None)6428 return SDValue();6429 6430 // Constant fold PRMT6431 if (isa<ConstantSDNode>(N->getOperand(0)) &&6432 isa<ConstantSDNode>(N->getOperand(1)) &&6433 isa<ConstantSDNode>(N->getOperand(2)))6434 return DCI.DAG.getConstant(computePRMT(N->getConstantOperandAPInt(0),6435 N->getConstantOperandAPInt(1),6436 N->getConstantOperandAPInt(2),6437 N->getConstantOperandVal(3)),6438 SDLoc(N), N->getValueType(0));6439 return SDValue();6440}6441 6442// During call lowering we wrap the return values in a ProxyReg node which6443// depend on the chain value produced by the completed call. This ensures that6444// the full call is emitted in cases where libcalls are used to legalize6445// operations. To improve the functioning of other DAG combines we pull all6446// operations we can through one of these nodes, ensuring that the ProxyReg6447// directly wraps a load. That is:6448//6449// (ProxyReg (zext (load retval0))) => (zext (ProxyReg (load retval0)))6450//6451static SDValue sinkProxyReg(SDValue R, SDValue Chain,6452 TargetLowering::DAGCombinerInfo &DCI) {6453 switch (R.getOpcode()) {6454 case ISD::TRUNCATE:6455 case ISD::ANY_EXTEND:6456 case ISD::SIGN_EXTEND:6457 case ISD::ZERO_EXTEND:6458 case ISD::BITCAST: {6459 if (SDValue V = sinkProxyReg(R.getOperand(0), Chain, DCI))6460 return DCI.DAG.getNode(R.getOpcode(), SDLoc(R), R.getValueType(), V);6461 return SDValue();6462 }6463 case ISD::SHL:6464 case ISD::SRL:6465 case ISD::SRA:6466 case ISD::OR: {6467 if (SDValue A = sinkProxyReg(R.getOperand(0), Chain, DCI))6468 if (SDValue B = sinkProxyReg(R.getOperand(1), Chain, DCI))6469 return DCI.DAG.getNode(R.getOpcode(), SDLoc(R), R.getValueType(), A, B);6470 return SDValue();6471 }6472 case ISD::Constant:6473 return R;6474 case ISD::LOAD:6475 case NVPTXISD::LoadV2:6476 case NVPTXISD::LoadV4: {6477 return DCI.DAG.getNode(NVPTXISD::ProxyReg, SDLoc(R), R.getValueType(),6478 {Chain, R});6479 }6480 case ISD::BUILD_VECTOR: {6481 if (DCI.isBeforeLegalize())6482 return SDValue();6483 6484 SmallVector<SDValue, 16> Ops;6485 for (auto &Op : R->ops()) {6486 SDValue V = sinkProxyReg(Op, Chain, DCI);6487 if (!V)6488 return SDValue();6489 Ops.push_back(V);6490 }6491 return DCI.DAG.getNode(ISD::BUILD_VECTOR, SDLoc(R), R.getValueType(), Ops);6492 }6493 case ISD::EXTRACT_VECTOR_ELT: {6494 if (DCI.isBeforeLegalize())6495 return SDValue();6496 6497 if (SDValue V = sinkProxyReg(R.getOperand(0), Chain, DCI))6498 return DCI.DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(R),6499 R.getValueType(), V, R.getOperand(1));6500 return SDValue();6501 }6502 default:6503 return SDValue();6504 }6505}6506 6507static SDValue combineProxyReg(SDNode *N,6508 TargetLowering::DAGCombinerInfo &DCI) {6509 6510 SDValue Chain = N->getOperand(0);6511 SDValue Reg = N->getOperand(1);6512 6513 // If the ProxyReg is not wrapping a load, try to pull the operations through6514 // the ProxyReg.6515 if (Reg.getOpcode() != ISD::LOAD) {6516 if (SDValue V = sinkProxyReg(Reg, Chain, DCI))6517 return V;6518 }6519 6520 return SDValue();6521}6522 6523SDValue NVPTXTargetLowering::PerformDAGCombine(SDNode *N,6524 DAGCombinerInfo &DCI) const {6525 CodeGenOptLevel OptLevel = getTargetMachine().getOptLevel();6526 switch (N->getOpcode()) {6527 default:6528 break;6529 case ISD::ADD:6530 return PerformADDCombine(N, DCI, OptLevel);6531 case ISD::ADDRSPACECAST:6532 return combineADDRSPACECAST(N, DCI);6533 case ISD::SIGN_EXTEND:6534 case ISD::ZERO_EXTEND:6535 return combineMulWide(N, DCI, OptLevel);6536 case ISD::BUILD_VECTOR:6537 return PerformBUILD_VECTORCombine(N, DCI);6538 case ISD::EXTRACT_VECTOR_ELT:6539 return PerformEXTRACTCombine(N, DCI);6540 case ISD::FADD:6541 return PerformFADDCombine(N, DCI, OptLevel);6542 case ISD::FMAXNUM:6543 case ISD::FMINNUM:6544 case ISD::FMAXIMUM:6545 case ISD::FMINIMUM:6546 case ISD::FMAXIMUMNUM:6547 case ISD::FMINIMUMNUM:6548 return PerformFMinMaxCombine(N, DCI, STI.getPTXVersion(),6549 STI.getSmVersion());6550 case ISD::LOAD:6551 case NVPTXISD::LoadV2:6552 case NVPTXISD::LoadV4:6553 return combineLOAD(N, DCI, STI);6554 case ISD::MUL:6555 return PerformMULCombine(N, DCI, OptLevel);6556 case NVPTXISD::PRMT:6557 return combinePRMT(N, DCI, OptLevel);6558 case NVPTXISD::ProxyReg:6559 return combineProxyReg(N, DCI);6560 case ISD::SETCC:6561 return PerformSETCCCombine(N, DCI, STI.getSmVersion());6562 case ISD::SHL:6563 return PerformSHLCombine(N, DCI, OptLevel);6564 case ISD::SREM:6565 case ISD::UREM:6566 return PerformREMCombine(N, DCI, OptLevel);6567 case ISD::STORE:6568 case NVPTXISD::StoreV2:6569 case NVPTXISD::StoreV4:6570 return combineSTORE(N, DCI, STI);6571 case ISD::VSELECT:6572 return PerformVSELECTCombine(N, DCI);6573 }6574 return SDValue();6575}6576 6577static void ReplaceBITCAST(SDNode *Node, SelectionDAG &DAG,6578 SmallVectorImpl<SDValue> &Results) {6579 // Handle bitcasting to v2i8 without hitting the default promotion6580 // strategy which goes through stack memory.6581 SDValue Op(Node, 0);6582 EVT ToVT = Op->getValueType(0);6583 if (ToVT != MVT::v2i8) {6584 return;6585 }6586 6587 // Bitcast to i16 and unpack elements into a vector6588 SDLoc DL(Node);6589 SDValue AsInt = DAG.getBitcast(MVT::i16, Op->getOperand(0));6590 SDValue Vec0 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, AsInt);6591 SDValue Const8 = DAG.getConstant(8, DL, MVT::i16);6592 SDValue Vec1 =6593 DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,6594 DAG.getNode(ISD::SRL, DL, MVT::i16, {AsInt, Const8}));6595 Results.push_back(6596 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2i8, {Vec0, Vec1}));6597}6598 6599static void ReplaceINTRINSIC_W_CHAIN(SDNode *N, SelectionDAG &DAG,6600 SmallVectorImpl<SDValue> &Results) {6601 SDValue Chain = N->getOperand(0);6602 SDValue Intrin = N->getOperand(1);6603 SDLoc DL(N);6604 6605 // Get the intrinsic ID6606 unsigned IntrinNo = Intrin.getNode()->getAsZExtVal();6607 switch (IntrinNo) {6608 default:6609 return;6610 case Intrinsic::nvvm_ldu_global_i:6611 case Intrinsic::nvvm_ldu_global_f:6612 case Intrinsic::nvvm_ldu_global_p: {6613 EVT ResVT = N->getValueType(0);6614 6615 if (ResVT.isVector()) {6616 // Vector LDG/LDU6617 6618 unsigned NumElts = ResVT.getVectorNumElements();6619 EVT EltVT = ResVT.getVectorElementType();6620 6621 // Since LDU/LDG are target nodes, we cannot rely on DAG type6622 // legalization.6623 // Therefore, we must ensure the type is legal. For i1 and i8, we set the6624 // loaded type to i16 and propagate the "real" type as the memory type.6625 bool NeedTrunc = false;6626 if (EltVT.getSizeInBits() < 16) {6627 EltVT = MVT::i16;6628 NeedTrunc = true;6629 }6630 6631 unsigned Opcode = 0;6632 SDVTList LdResVTs;6633 6634 switch (NumElts) {6635 default:6636 return;6637 case 2:6638 Opcode = NVPTXISD::LDUV2;6639 LdResVTs = DAG.getVTList(EltVT, EltVT, MVT::Other);6640 break;6641 case 4: {6642 Opcode = NVPTXISD::LDUV4;6643 EVT ListVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other };6644 LdResVTs = DAG.getVTList(ListVTs);6645 break;6646 }6647 }6648 6649 SmallVector<SDValue, 8> OtherOps;6650 6651 // Copy regular operands6652 6653 OtherOps.push_back(Chain); // Chain6654 // Skip operand 1 (intrinsic ID)6655 // Others6656 OtherOps.append(N->op_begin() + 2, N->op_end());6657 6658 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(N);6659 6660 SDValue NewLD = DAG.getMemIntrinsicNode(Opcode, DL, LdResVTs, OtherOps,6661 MemSD->getMemoryVT(),6662 MemSD->getMemOperand());6663 6664 SmallVector<SDValue, 4> ScalarRes;6665 6666 for (unsigned i = 0; i < NumElts; ++i) {6667 SDValue Res = NewLD.getValue(i);6668 if (NeedTrunc)6669 Res =6670 DAG.getNode(ISD::TRUNCATE, DL, ResVT.getVectorElementType(), Res);6671 ScalarRes.push_back(Res);6672 }6673 6674 SDValue LoadChain = NewLD.getValue(NumElts);6675 6676 SDValue BuildVec =6677 DAG.getBuildVector(ResVT, DL, ScalarRes);6678 6679 Results.push_back(BuildVec);6680 Results.push_back(LoadChain);6681 } else {6682 // i8 LDG/LDU6683 assert(ResVT.isSimple() && ResVT.getSimpleVT().SimpleTy == MVT::i8 &&6684 "Custom handling of non-i8 ldu/ldg?");6685 6686 // Just copy all operands as-is6687 SmallVector<SDValue, 4> Ops(N->ops());6688 6689 // Force output to i166690 SDVTList LdResVTs = DAG.getVTList(MVT::i16, MVT::Other);6691 6692 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(N);6693 6694 // We make sure the memory type is i8, which will be used during isel6695 // to select the proper instruction.6696 SDValue NewLD =6697 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, LdResVTs, Ops,6698 MVT::i8, MemSD->getMemOperand());6699 6700 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,6701 NewLD.getValue(0)));6702 Results.push_back(NewLD.getValue(1));6703 }6704 return;6705 }6706 6707 case Intrinsic::nvvm_tcgen05_ld_16x64b_x4:6708 case Intrinsic::nvvm_tcgen05_ld_16x64b_x8:6709 case Intrinsic::nvvm_tcgen05_ld_16x64b_x16:6710 case Intrinsic::nvvm_tcgen05_ld_16x64b_x32:6711 case Intrinsic::nvvm_tcgen05_ld_16x64b_x64:6712 case Intrinsic::nvvm_tcgen05_ld_16x64b_x128:6713 case Intrinsic::nvvm_tcgen05_ld_32x32b_x4:6714 case Intrinsic::nvvm_tcgen05_ld_32x32b_x8:6715 case Intrinsic::nvvm_tcgen05_ld_32x32b_x16:6716 case Intrinsic::nvvm_tcgen05_ld_32x32b_x32:6717 case Intrinsic::nvvm_tcgen05_ld_32x32b_x64:6718 case Intrinsic::nvvm_tcgen05_ld_32x32b_x128:6719 case Intrinsic::nvvm_tcgen05_ld_16x128b_x2:6720 case Intrinsic::nvvm_tcgen05_ld_16x128b_x4:6721 case Intrinsic::nvvm_tcgen05_ld_16x128b_x8:6722 case Intrinsic::nvvm_tcgen05_ld_16x128b_x16:6723 case Intrinsic::nvvm_tcgen05_ld_16x128b_x32:6724 case Intrinsic::nvvm_tcgen05_ld_16x128b_x64:6725 case Intrinsic::nvvm_tcgen05_ld_16x256b_x1:6726 case Intrinsic::nvvm_tcgen05_ld_16x256b_x2:6727 case Intrinsic::nvvm_tcgen05_ld_16x256b_x4:6728 case Intrinsic::nvvm_tcgen05_ld_16x256b_x8:6729 case Intrinsic::nvvm_tcgen05_ld_16x256b_x16:6730 case Intrinsic::nvvm_tcgen05_ld_16x256b_x32:6731 if (auto Res = lowerTcgen05Ld(N, DAG)) {6732 Results.push_back(Res->first);6733 Results.push_back(Res->second);6734 }6735 return;6736 6737 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x4:6738 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x8:6739 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x16:6740 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x32:6741 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x64:6742 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x128:6743 if (auto Res = lowerTcgen05Ld(N, DAG, /*HasOffset=*/true)) {6744 Results.push_back(Res->first);6745 Results.push_back(Res->second);6746 }6747 return;6748 }6749}6750 6751static void ReplaceCopyFromReg_128(SDNode *N, SelectionDAG &DAG,6752 SmallVectorImpl<SDValue> &Results) {6753 // Change the CopyFromReg to output 2 64-bit results instead of a 128-bit6754 // result so that it can pass the legalization6755 SDLoc DL(N);6756 SDValue Chain = N->getOperand(0);6757 SDValue Reg = N->getOperand(1);6758 SDValue Glue = N->getOperand(2);6759 6760 assert(Reg.getValueType() == MVT::i128 &&6761 "Custom lowering for CopyFromReg with 128-bit reg only");6762 SmallVector<EVT, 4> ResultsType = {MVT::i64, MVT::i64, N->getValueType(1),6763 N->getValueType(2)};6764 SmallVector<SDValue, 3> NewOps = {Chain, Reg, Glue};6765 6766 SDValue NewValue = DAG.getNode(ISD::CopyFromReg, DL, ResultsType, NewOps);6767 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i128,6768 {NewValue.getValue(0), NewValue.getValue(1)});6769 6770 Results.push_back(Pair);6771 Results.push_back(NewValue.getValue(2));6772 Results.push_back(NewValue.getValue(3));6773}6774 6775static void replaceProxyReg(SDNode *N, SelectionDAG &DAG,6776 const TargetLowering &TLI,6777 SmallVectorImpl<SDValue> &Results) {6778 SDValue Chain = N->getOperand(0);6779 SDValue Reg = N->getOperand(1);6780 6781 MVT VT = TLI.getRegisterType(*DAG.getContext(), Reg.getValueType());6782 6783 SDValue NewReg = DAG.getAnyExtOrTrunc(Reg, SDLoc(N), VT);6784 SDValue NewProxy =6785 DAG.getNode(NVPTXISD::ProxyReg, SDLoc(N), VT, {Chain, NewReg});6786 SDValue Res = DAG.getAnyExtOrTrunc(NewProxy, SDLoc(N), N->getValueType(0));6787 6788 Results.push_back(Res);6789}6790 6791static void replaceAtomicSwap128(SDNode *N, SelectionDAG &DAG,6792 const NVPTXSubtarget &STI,6793 SmallVectorImpl<SDValue> &Results) {6794 assert(N->getValueType(0) == MVT::i128 &&6795 "Custom lowering for atomic128 only supports i128");6796 6797 AtomicSDNode *AN = cast<AtomicSDNode>(N);6798 SDLoc dl(N);6799 6800 if (!STI.hasAtomSwap128()) {6801 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(6802 DAG.getMachineFunction().getFunction(),6803 "Support for b128 atomics introduced in PTX ISA version 8.3 and "6804 "requires target sm_90.",6805 dl.getDebugLoc()));6806 6807 Results.push_back(DAG.getUNDEF(MVT::i128));6808 Results.push_back(AN->getOperand(0)); // Chain6809 return;6810 }6811 6812 SmallVector<SDValue, 6> Ops;6813 Ops.push_back(AN->getOperand(0)); // Chain6814 Ops.push_back(AN->getOperand(1)); // Ptr6815 for (const auto &Op : AN->ops().drop_front(2)) {6816 // Low part6817 Ops.push_back(DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i64, Op,6818 DAG.getIntPtrConstant(0, dl)));6819 // High part6820 Ops.push_back(DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i64, Op,6821 DAG.getIntPtrConstant(1, dl)));6822 }6823 unsigned Opcode = N->getOpcode() == ISD::ATOMIC_SWAP6824 ? NVPTXISD::ATOMIC_SWAP_B1286825 : NVPTXISD::ATOMIC_CMP_SWAP_B128;6826 SDVTList Tys = DAG.getVTList(MVT::i64, MVT::i64, MVT::Other);6827 SDValue Result = DAG.getMemIntrinsicNode(Opcode, dl, Tys, Ops, MVT::i128,6828 AN->getMemOperand());6829 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i128,6830 {Result.getValue(0), Result.getValue(1)}));6831 Results.push_back(Result.getValue(2));6832}6833 6834void NVPTXTargetLowering::ReplaceNodeResults(6835 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {6836 switch (N->getOpcode()) {6837 default:6838 report_fatal_error("Unhandled custom legalization");6839 case ISD::BITCAST:6840 ReplaceBITCAST(N, DAG, Results);6841 return;6842 case ISD::LOAD:6843 case ISD::MLOAD:6844 replaceLoadVector(N, DAG, Results, STI);6845 return;6846 case ISD::INTRINSIC_W_CHAIN:6847 ReplaceINTRINSIC_W_CHAIN(N, DAG, Results);6848 return;6849 case ISD::CopyFromReg:6850 ReplaceCopyFromReg_128(N, DAG, Results);6851 return;6852 case NVPTXISD::ProxyReg:6853 replaceProxyReg(N, DAG, *this, Results);6854 return;6855 case ISD::ATOMIC_CMP_SWAP:6856 case ISD::ATOMIC_SWAP:6857 replaceAtomicSwap128(N, DAG, STI, Results);6858 return;6859 }6860}6861 6862NVPTXTargetLowering::AtomicExpansionKind6863NVPTXTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {6864 Type *Ty = AI->getValOperand()->getType();6865 6866 if (AI->isFloatingPointOperation()) {6867 if (AI->getOperation() == AtomicRMWInst::BinOp::FAdd) {6868 if (Ty->isHalfTy() && STI.getSmVersion() >= 70 &&6869 STI.getPTXVersion() >= 63)6870 return AtomicExpansionKind::None;6871 if (Ty->isBFloatTy() && STI.getSmVersion() >= 90 &&6872 STI.getPTXVersion() >= 78)6873 return AtomicExpansionKind::None;6874 if (Ty->isFloatTy())6875 return AtomicExpansionKind::None;6876 if (Ty->isDoubleTy() && STI.hasAtomAddF64())6877 return AtomicExpansionKind::None;6878 }6879 return AtomicExpansionKind::CmpXChg;6880 }6881 6882 assert(Ty->isIntegerTy() && "Ty should be integer at this point");6883 const unsigned BitWidth = cast<IntegerType>(Ty)->getBitWidth();6884 6885 switch (AI->getOperation()) {6886 default:6887 return AtomicExpansionKind::CmpXChg;6888 case AtomicRMWInst::BinOp::Xchg:6889 if (BitWidth == 128)6890 return AtomicExpansionKind::None;6891 [[fallthrough]];6892 case AtomicRMWInst::BinOp::And:6893 case AtomicRMWInst::BinOp::Or:6894 case AtomicRMWInst::BinOp::Xor:6895 switch (BitWidth) {6896 case 8:6897 case 16:6898 return AtomicExpansionKind::CmpXChg;6899 case 32:6900 return AtomicExpansionKind::None;6901 case 64:6902 if (STI.hasAtomBitwise64())6903 return AtomicExpansionKind::None;6904 return AtomicExpansionKind::CmpXChg;6905 case 128:6906 return AtomicExpansionKind::CmpXChg;6907 default:6908 llvm_unreachable("unsupported width encountered");6909 }6910 case AtomicRMWInst::BinOp::Add:6911 case AtomicRMWInst::BinOp::Sub:6912 case AtomicRMWInst::BinOp::Max:6913 case AtomicRMWInst::BinOp::Min:6914 case AtomicRMWInst::BinOp::UMax:6915 case AtomicRMWInst::BinOp::UMin:6916 switch (BitWidth) {6917 case 8:6918 case 16:6919 return AtomicExpansionKind::CmpXChg;6920 case 32:6921 return AtomicExpansionKind::None;6922 case 64:6923 if (STI.hasAtomMinMax64())6924 return AtomicExpansionKind::None;6925 return AtomicExpansionKind::CmpXChg;6926 case 128:6927 return AtomicExpansionKind::CmpXChg;6928 default:6929 llvm_unreachable("unsupported width encountered");6930 }6931 case AtomicRMWInst::BinOp::UIncWrap:6932 case AtomicRMWInst::BinOp::UDecWrap:6933 switch (BitWidth) {6934 case 32:6935 return AtomicExpansionKind::None;6936 case 8:6937 case 16:6938 case 64:6939 case 128:6940 return AtomicExpansionKind::CmpXChg;6941 default:6942 llvm_unreachable("unsupported width encountered");6943 }6944 }6945 6946 return AtomicExpansionKind::CmpXChg;6947}6948 6949bool NVPTXTargetLowering::shouldInsertFencesForAtomic(6950 const Instruction *I) const {6951 auto *CI = dyn_cast<AtomicCmpXchgInst>(I);6952 // When CAS bitwidth is not supported on the hardware, the CAS is emulated6953 // using a retry loop that uses a higher-bitwidth monotonic CAS. We enforce6954 // the memory order using explicit fences around the retry loop.6955 // The memory order of natively supported CAS operations can be enforced6956 // by lowering to an atom.cas with the right memory synchronizing effect.6957 // However, atom.cas only supports relaxed, acquire, release and acq_rel.6958 // So we also use explicit fences for enforcing memory order for6959 // seq_cast CAS with natively-supported bitwidths.6960 return CI &&6961 (cast<IntegerType>(CI->getCompareOperand()->getType())->getBitWidth() <6962 STI.getMinCmpXchgSizeInBits() ||6963 CI->getMergedOrdering() == AtomicOrdering::SequentiallyConsistent);6964}6965 6966AtomicOrdering NVPTXTargetLowering::atomicOperationOrderAfterFenceSplit(6967 const Instruction *I) const {6968 auto *CI = dyn_cast<AtomicCmpXchgInst>(I);6969 bool BitwidthSupportedAndIsSeqCst =6970 CI && CI->getMergedOrdering() == AtomicOrdering::SequentiallyConsistent &&6971 cast<IntegerType>(CI->getCompareOperand()->getType())->getBitWidth() >=6972 STI.getMinCmpXchgSizeInBits();6973 return BitwidthSupportedAndIsSeqCst ? AtomicOrdering::Acquire6974 : AtomicOrdering::Monotonic;6975}6976 6977Instruction *NVPTXTargetLowering::emitLeadingFence(IRBuilderBase &Builder,6978 Instruction *Inst,6979 AtomicOrdering Ord) const {6980 if (!isa<AtomicCmpXchgInst>(Inst))6981 return TargetLoweringBase::emitLeadingFence(Builder, Inst, Ord);6982 6983 // Specialize for cmpxchg6984 // Emit a fence.sc leading fence for cmpxchg seq_cst which are not emulated6985 SyncScope::ID SSID = cast<AtomicCmpXchgInst>(Inst)->getSyncScopeID();6986 if (isReleaseOrStronger(Ord))6987 return Builder.CreateFence(Ord == AtomicOrdering::SequentiallyConsistent6988 ? Ord6989 : AtomicOrdering::Release,6990 SSID);6991 6992 return nullptr;6993}6994 6995Instruction *NVPTXTargetLowering::emitTrailingFence(IRBuilderBase &Builder,6996 Instruction *Inst,6997 AtomicOrdering Ord) const {6998 // Specialize for cmpxchg6999 if (!isa<AtomicCmpXchgInst>(Inst))7000 return TargetLoweringBase::emitTrailingFence(Builder, Inst, Ord);7001 7002 auto *CI = cast<AtomicCmpXchgInst>(Inst);7003 auto CASWidth =7004 cast<IntegerType>(CI->getCompareOperand()->getType())->getBitWidth();7005 SyncScope::ID SSID = CI->getSyncScopeID();7006 // Do not emit a trailing fence for cmpxchg seq_cst which are not emulated7007 if (isAcquireOrStronger(Ord) &&7008 (Ord != AtomicOrdering::SequentiallyConsistent ||7009 CASWidth < STI.getMinCmpXchgSizeInBits()))7010 return Builder.CreateFence(AtomicOrdering::Acquire, SSID);7011 7012 return nullptr;7013}7014 7015// Rather than default to SINT when both UINT and SINT are custom, we only7016// change the opcode when UINT is not legal and SINT is. UINT is preferred when7017// both are custom since unsigned CVT instructions can lead to slightly better7018// SASS code with fewer instructions.7019unsigned NVPTXTargetLowering::getPreferredFPToIntOpcode(unsigned Op, EVT FromVT,7020 EVT ToVT) const {7021 if (isOperationLegal(Op, ToVT))7022 return Op;7023 switch (Op) {7024 case ISD::FP_TO_UINT:7025 if (isOperationLegal(ISD::FP_TO_SINT, ToVT))7026 return ISD::FP_TO_SINT;7027 break;7028 case ISD::STRICT_FP_TO_UINT:7029 if (isOperationLegal(ISD::STRICT_FP_TO_SINT, ToVT))7030 return ISD::STRICT_FP_TO_SINT;7031 break;7032 case ISD::VP_FP_TO_UINT:7033 if (isOperationLegal(ISD::VP_FP_TO_SINT, ToVT))7034 return ISD::VP_FP_TO_SINT;7035 break;7036 default:7037 break;7038 }7039 return Op;7040}7041 7042// Pin NVPTXTargetObjectFile's vtables to this file.7043NVPTXTargetObjectFile::~NVPTXTargetObjectFile() = default;7044 7045MCSection *NVPTXTargetObjectFile::SelectSectionForGlobal(7046 const GlobalObject *GO, SectionKind Kind, const TargetMachine &TM) const {7047 return getDataSection();7048}7049 7050static void computeKnownBitsForPRMT(const SDValue Op, KnownBits &Known,7051 const SelectionDAG &DAG, unsigned Depth) {7052 SDValue A = Op.getOperand(0);7053 SDValue B = Op.getOperand(1);7054 ConstantSDNode *Selector = dyn_cast<ConstantSDNode>(Op.getOperand(2));7055 unsigned Mode = Op.getConstantOperandVal(3);7056 7057 if (!Selector)7058 return;7059 7060 KnownBits AKnown = DAG.computeKnownBits(A, Depth);7061 KnownBits BKnown = DAG.computeKnownBits(B, Depth);7062 7063 // {b, a} = {{b7, b6, b5, b4}, {b3, b2, b1, b0}}7064 assert(AKnown.getBitWidth() == 32 && BKnown.getBitWidth() == 32 &&7065 "PRMT must have i32 operands");7066 assert(Known.getBitWidth() == 32 && "PRMT must have i32 result");7067 KnownBits BitField = BKnown.concat(AKnown);7068 7069 APInt SelectorVal = getPRMTSelector(Selector->getAPIntValue(), Mode);7070 for (unsigned I : llvm::seq(4)) {7071 APInt Sel = SelectorVal.extractBits(4, I * 4);7072 unsigned Idx = Sel.getLoBits(3).getZExtValue();7073 unsigned Sign = Sel.getHiBits(1).getZExtValue();7074 KnownBits Byte = BitField.extractBits(8, Idx * 8);7075 if (Sign)7076 Byte = KnownBits::ashr(Byte, 8);7077 Known.insertBits(Byte, I * 8);7078 }7079}7080 7081static void computeKnownBitsForLoadV(const SDValue Op, KnownBits &Known) {7082 MemSDNode *LD = cast<MemSDNode>(Op);7083 7084 // We can't do anything without knowing the sign bit.7085 auto ExtType = LD->getConstantOperandVal(LD->getNumOperands() - 1);7086 if (ExtType == ISD::SEXTLOAD)7087 return;7088 7089 // ExtLoading to vector types is weird and may not work well with known bits.7090 auto DestVT = LD->getValueType(0);7091 if (DestVT.isVector())7092 return;7093 7094 assert(Known.getBitWidth() == DestVT.getSizeInBits());7095 auto ElementBitWidth = NVPTXDAGToDAGISel::getFromTypeWidthForLoad(LD);7096 Known.Zero.setHighBits(Known.getBitWidth() - ElementBitWidth);7097}7098 7099void NVPTXTargetLowering::computeKnownBitsForTargetNode(7100 const SDValue Op, KnownBits &Known, const APInt &DemandedElts,7101 const SelectionDAG &DAG, unsigned Depth) const {7102 Known.resetAll();7103 7104 switch (Op.getOpcode()) {7105 case NVPTXISD::PRMT:7106 computeKnownBitsForPRMT(Op, Known, DAG, Depth);7107 break;7108 case NVPTXISD::LoadV2:7109 case NVPTXISD::LoadV4:7110 case NVPTXISD::LoadV8:7111 computeKnownBitsForLoadV(Op, Known);7112 break;7113 default:7114 break;7115 }7116}7117 7118static std::pair<APInt, APInt> getPRMTDemandedBits(const APInt &SelectorVal,7119 const APInt &DemandedBits) {7120 APInt DemandedLHS = APInt(32, 0);7121 APInt DemandedRHS = APInt(32, 0);7122 7123 for (unsigned I : llvm::seq(4)) {7124 if (DemandedBits.extractBits(8, I * 8).isZero())7125 continue;7126 7127 APInt Sel = SelectorVal.extractBits(4, I * 4);7128 unsigned Idx = Sel.getLoBits(3).getZExtValue();7129 unsigned Sign = Sel.getHiBits(1).getZExtValue();7130 7131 APInt &Src = Idx < 4 ? DemandedLHS : DemandedRHS;7132 unsigned ByteStart = (Idx % 4) * 8;7133 if (Sign)7134 Src.setBit(ByteStart + 7);7135 else7136 Src.setBits(ByteStart, ByteStart + 8);7137 }7138 7139 return {DemandedLHS, DemandedRHS};7140}7141 7142// Replace undef with 0 as this is easier for other optimizations such as7143// known bits.7144static SDValue canonicalizePRMTInput(SDValue Op, SelectionDAG &DAG) {7145 if (!Op)7146 return SDValue();7147 if (Op.isUndef())7148 return DAG.getConstant(0, SDLoc(), MVT::i32);7149 return Op;7150}7151 7152static SDValue simplifyDemandedBitsForPRMT(SDValue PRMT,7153 const APInt &DemandedBits,7154 SelectionDAG &DAG,7155 const TargetLowering &TLI,7156 unsigned Depth) {7157 assert(PRMT.getOpcode() == NVPTXISD::PRMT);7158 SDValue Op0 = PRMT.getOperand(0);7159 SDValue Op1 = PRMT.getOperand(1);7160 auto *SelectorConst = dyn_cast<ConstantSDNode>(PRMT.getOperand(2));7161 if (!SelectorConst)7162 return SDValue();7163 7164 unsigned Mode = PRMT.getConstantOperandVal(3);7165 const APInt Selector = getPRMTSelector(SelectorConst->getAPIntValue(), Mode);7166 7167 // Try to simplify the PRMT to one of the inputs if the used bytes are all7168 // from the same input in the correct order.7169 const unsigned LeadingBytes = DemandedBits.countLeadingZeros() / 8;7170 const unsigned SelBits = (4 - LeadingBytes) * 4;7171 if (Selector.getLoBits(SelBits) == APInt(32, 0x3210).getLoBits(SelBits))7172 return Op0;7173 if (Selector.getLoBits(SelBits) == APInt(32, 0x7654).getLoBits(SelBits))7174 return Op1;7175 7176 auto [DemandedLHS, DemandedRHS] = getPRMTDemandedBits(Selector, DemandedBits);7177 7178 // Attempt to avoid multi-use ops if we don't need anything from them.7179 SDValue DemandedOp0 =7180 TLI.SimplifyMultipleUseDemandedBits(Op0, DemandedLHS, DAG, Depth + 1);7181 SDValue DemandedOp1 =7182 TLI.SimplifyMultipleUseDemandedBits(Op1, DemandedRHS, DAG, Depth + 1);7183 7184 DemandedOp0 = canonicalizePRMTInput(DemandedOp0, DAG);7185 DemandedOp1 = canonicalizePRMTInput(DemandedOp1, DAG);7186 if ((DemandedOp0 && DemandedOp0 != Op0) ||7187 (DemandedOp1 && DemandedOp1 != Op1)) {7188 Op0 = DemandedOp0 ? DemandedOp0 : Op0;7189 Op1 = DemandedOp1 ? DemandedOp1 : Op1;7190 return getPRMT(Op0, Op1, Selector.getZExtValue(), SDLoc(PRMT), DAG);7191 }7192 7193 return SDValue();7194}7195 7196bool NVPTXTargetLowering::SimplifyDemandedBitsForTargetNode(7197 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,7198 KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {7199 Known.resetAll();7200 7201 switch (Op.getOpcode()) {7202 case NVPTXISD::PRMT:7203 if (SDValue Result = simplifyDemandedBitsForPRMT(Op, DemandedBits, TLO.DAG,7204 *this, Depth)) {7205 TLO.CombineTo(Op, Result);7206 return true;7207 }7208 break;7209 default:7210 break;7211 }7212 7213 computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);7214 return false;7215}7216