6125 lines · plain
1//===- NVPTXIntrinsics.td - PTX Intrinsics Instructions -------*- tblgen -*-==//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9def AS_match {10 code generic = [{11 return cast<MemSDNode>(N)->getAddressSpace() == llvm::ADDRESS_SPACE_GENERIC;12 }];13 code shared = [{14 return cast<MemSDNode>(N)->getAddressSpace() == llvm::ADDRESS_SPACE_SHARED;15 }];16 code shared_cluster = [{17 return cast<MemSDNode>(N)->getAddressSpace() == llvm::ADDRESS_SPACE_SHARED_CLUSTER;18 }];19 code global = [{20 return cast<MemSDNode>(N)->getAddressSpace() == llvm::ADDRESS_SPACE_GLOBAL;21 }];22 code const = [{23 return cast<MemSDNode>(N)->getAddressSpace() == llvm::ADDRESS_SPACE_CONST;24 }];25 code param = [{26 return cast<MemSDNode>(N)->getAddressSpace() == llvm::ADDRESS_SPACE_PARAM;27 }];28}29 30 31//===----------------------------------------------------------------------===//32// NVPTX Scope Constants33// These map to the Scope enum in NVPTX.h34//===----------------------------------------------------------------------===//35 36def Scope_thread : PatLeaf<(i32 0)>; // Thread = 037def Scope_cta : PatLeaf<(i32 1)>; // Block = 138def Scope_cluster : PatLeaf<(i32 2)>; // Cluster = 239def Scope_device : PatLeaf<(i32 3)>; // Device = 340def Scope_sys : PatLeaf<(i32 4)>; // System = 441 42//===----------------------------------------------------------------------===//43// NVPTX Address Space Constants44// These map to the AddressSpace enum in NVPTX.h45//===----------------------------------------------------------------------===//46 47def AddrSpace_gen : PatLeaf<(i32 0)>; // Generic = 048def AddrSpace_global : PatLeaf<(i32 1)>; // Global = 149def AddrSpace_shared : PatLeaf<(i32 3)>; // Shared = 350def AddrSpace_const : PatLeaf<(i32 4)>; // Const = 451def AddrSpace_local : PatLeaf<(i32 5)>; // Local = 552def AddrSpace_shared_cluster : PatLeaf<(i32 7)>; // SharedCluster = 753def AddrSpace_param : PatLeaf<(i32 101)>; // Param = 10154 55//===----------------------------------------------------------------------===//56// NVPTX Ordering Constants57// These map to the Ordering enum in NVPTX.h58//===----------------------------------------------------------------------===//59 60def Ordering_not_atomic : PatLeaf<(i32 0)>; // NotAtomic = 061def Ordering_relaxed : PatLeaf<(i32 2)>; // Relaxed = 162def Ordering_acquire : PatLeaf<(i32 4)>; // Acquire = 463def Ordering_release : PatLeaf<(i32 5)>; // Release = 564def Ordering_acquire_release : PatLeaf<(i32 6)>; // AcquireRelease = 665def Ordering_sequentially_consistent : PatLeaf<(i32 7)>; // SequentiallyConsistent = 766def Ordering_volatile : PatLeaf<(i32 8)>; // Volatile = 867def Ordering_relaxed_mmio : PatLeaf<(i32 9)>; // RelaxedMMIO = 968 69 70// A node that will be replaced with the current PTX version.71class PTX {72 SDNodeXForm PTXVerXform = SDNodeXForm<imm, [{73 return getI32Imm(Subtarget->getPTXVersion(), SDLoc(N));74 }]>;75 // (i32 0) will be XForm'ed to the currently used PTX version.76 dag version = (PTXVerXform (i32 0));77}78def ptx : PTX;79 80// Generates list of n sequential register names.81// E.g. RegNames<3, "r">.ret -> ["r0", "r1", "r2" ]82class RegSeq<int n, string prefix> {83 list<string> ret = !if(n, !listconcat(RegSeq<!sub(n, 1), prefix>.ret,84 [prefix # !sub(n, 1)]),85 []);86}87 88//-----------------------------------89// Synchronization and shuffle functions90//-----------------------------------91let isConvergent = true in {92def INT_BARRIER0_POPC : NVPTXInst<(outs B32:$dst), (ins B32:$pred),93 !strconcat("{{ \n\t",94 ".reg .pred \t%p1; \n\t",95 "setp.ne.u32 \t%p1, $pred, 0; \n\t",96 "bar.red.popc.u32 \t$dst, 0, %p1; \n\t",97 "}}"),98 [(set i32:$dst, (int_nvvm_barrier0_popc i32:$pred))]>;99def INT_BARRIER0_AND : NVPTXInst<(outs B32:$dst), (ins B32:$pred),100 !strconcat("{{ \n\t",101 ".reg .pred \t%p1; \n\t",102 ".reg .pred \t%p2; \n\t",103 "setp.ne.u32 \t%p1, $pred, 0; \n\t",104 "bar.red.and.pred \t%p2, 0, %p1; \n\t",105 "selp.u32 \t$dst, 1, 0, %p2; \n\t",106 "}}"),107 [(set i32:$dst, (int_nvvm_barrier0_and i32:$pred))]>;108def INT_BARRIER0_OR : NVPTXInst<(outs B32:$dst), (ins B32:$pred),109 !strconcat("{{ \n\t",110 ".reg .pred \t%p1; \n\t",111 ".reg .pred \t%p2; \n\t",112 "setp.ne.u32 \t%p1, $pred, 0; \n\t",113 "bar.red.or.pred \t%p2, 0, %p1; \n\t",114 "selp.u32 \t$dst, 1, 0, %p2; \n\t",115 "}}"),116 [(set i32:$dst, (int_nvvm_barrier0_or i32:$pred))]>;117 118def INT_BAR_WARP_SYNC_I : BasicNVPTXInst<(outs), (ins i32imm:$i), "bar.warp.sync",119 [(int_nvvm_bar_warp_sync imm:$i)]>,120 Requires<[hasPTX<60>, hasSM<30>]>;121def INT_BAR_WARP_SYNC_R : BasicNVPTXInst<(outs), (ins B32:$i), "bar.warp.sync",122 [(int_nvvm_bar_warp_sync i32:$i)]>,123 Requires<[hasPTX<60>, hasSM<30>]>;124 125multiclass BARRIER1<string asmstr, Intrinsic intrinsic, list<Predicate> requires = []> {126 def _i : BasicNVPTXInst<(outs), (ins i32imm:$i), asmstr,127 [(intrinsic imm:$i)]>,128 Requires<requires>;129 130 def _r : BasicNVPTXInst<(outs), (ins B32:$i), asmstr,131 [(intrinsic i32:$i)]>,132 Requires<requires>;133}134 135multiclass BARRIER2<string asmstr, Intrinsic intrinsic, list<Predicate> requires = []> {136 def _rr : BasicNVPTXInst<(outs), (ins B32:$i, B32:$j), asmstr,137 [(intrinsic i32:$i, i32:$j)]>,138 Requires<requires>;139 140 def _ri : BasicNVPTXInst<(outs), (ins B32:$i, i32imm:$j), asmstr,141 [(intrinsic i32:$i, imm:$j)]>,142 Requires<requires>;143 144 def _ir : BasicNVPTXInst<(outs), (ins i32imm:$i, B32:$j), asmstr,145 [(intrinsic imm:$i, i32:$j)]>,146 Requires<requires>;147 148 def _ii : BasicNVPTXInst<(outs), (ins i32imm:$i, i32imm:$j), asmstr,149 [(intrinsic imm:$i, imm:$j)]>,150 Requires<requires>;151}152 153// Note the "bar.sync" variants could be renamed to the equivalent corresponding154// "barrier.*.aligned" variants. We use the older syntax for compatibility with155// older versions of the PTX ISA.156defm BARRIER_CTA_SYNC_ALIGNED_ALL : BARRIER1<"bar.sync", int_nvvm_barrier_cta_sync_aligned_all>;157defm BARRIER_CTA_SYNC_ALIGNED : BARRIER2<"bar.sync", int_nvvm_barrier_cta_sync_aligned_count>;158defm BARRIER_CTA_ARRIVE_ALIGNED : BARRIER2<"bar.arrive", int_nvvm_barrier_cta_arrive_aligned_count>;159 160defm BARRIER_CTA_SYNC_ALL : BARRIER1<"barrier.sync", int_nvvm_barrier_cta_sync_all, [hasPTX<60>]>;161defm BARRIER_CTA_SYNC : BARRIER2<"barrier.sync", int_nvvm_barrier_cta_sync_count, [hasPTX<60>]>;162defm BARRIER_CTA_ARRIVE : BARRIER2<"barrier.arrive", int_nvvm_barrier_cta_arrive_count, [hasPTX<60>]>;163 164class INT_BARRIER_CLUSTER<string variant, Intrinsic Intr,165 list<Predicate> Preds = [hasPTX<78>, hasSM<90>]>:166 BasicNVPTXInst<(outs), (ins), "barrier.cluster."# variant, [(Intr)]>,167 Requires<Preds>;168 169def barrier_cluster_arrive:170 INT_BARRIER_CLUSTER<"arrive", int_nvvm_barrier_cluster_arrive>;171def barrier_cluster_arrive_relaxed:172 INT_BARRIER_CLUSTER<"arrive.relaxed",173 int_nvvm_barrier_cluster_arrive_relaxed, [hasPTX<80>, hasSM<90>]>;174def barrier_cluster_wait:175 INT_BARRIER_CLUSTER<"wait", int_nvvm_barrier_cluster_wait>;176 177// 'aligned' versions of the cluster barrier intrinsics178def barrier_cluster_arrive_aligned:179 INT_BARRIER_CLUSTER<"arrive.aligned", int_nvvm_barrier_cluster_arrive_aligned>;180def barrier_cluster_arrive_relaxed_aligned:181 INT_BARRIER_CLUSTER<"arrive.relaxed.aligned",182 int_nvvm_barrier_cluster_arrive_relaxed_aligned, [hasPTX<80>, hasSM<90>]>;183def barrier_cluster_wait_aligned:184 INT_BARRIER_CLUSTER<"wait.aligned", int_nvvm_barrier_cluster_wait_aligned>;185 186foreach sync = [false, true] in {187 foreach mode = ["up", "down", "bfly", "idx"] in {188 foreach regclass = ["i32", "f32"] in {189 foreach return_pred = [false, true] in {190 foreach offset_imm = [false, true] in {191 foreach mask_imm = [false, true] in {192 foreach threadmask_imm = !if(sync, [0, 1], [0]) in {193 defvar Intr = !cast<Intrinsic>("int_nvvm_shfl_"194 # !if(sync, "sync_", "")195 # mode196 # "_" # regclass197 # !if(return_pred, "p", ""));198 defvar InOperandList = !con(199 (ins B32:$src),200 !dag(ins, !if(offset_imm, [i32imm], [B32]), ["offset"]),201 !dag(ins, !if(mask_imm, [i32imm], [B32]), ["mask"]),202 !if(sync,203 !dag(ins, !if(threadmask_imm, [i32imm], [B32]), ["threadmask"]),204 (ins)));205 defvar Pattern = !con(206 (set B32:$dst),207 !if(return_pred, (set B1:$pred), (set)),208 (set !con(209 !if(sync,210 !dag(Intr, !if(threadmask_imm, [imm], [B32]), ["threadmask"]),211 (Intr)),212 (Intr B32:$src),213 !dag(Intr, !if(offset_imm, [imm], [B32]), ["offset"]),214 !dag(Intr, !if(mask_imm, [imm], [B32]), ["mask"]))));215 216 def : BasicNVPTXInst<217 !if(return_pred, (outs B32:$dst, B1:$pred),218 (outs B32:$dst)),219 InOperandList,220 "shfl." # !if(sync, "sync.", "") # mode # ".b32",221 [Pattern]>,222 Requires<!if(sync, [hasSM<30>, hasPTX<60>], [hasSM<30>, hasSHFL])>;223 }224 }225 }226 }227 }228 }229}230 231// vote.{all,any,uni,ballot}232let Predicates = [hasPTX<60>, hasSM<30>] in {233 multiclass VOTE<string mode, RegTyInfo t, Intrinsic op> {234 def : BasicNVPTXInst<(outs t.RC:$dest), (ins B1:$pred),235 "vote." # mode # "." # t.PtxType,236 [(set t.Ty:$dest, (op i1:$pred))]>;237 }238 239 defm VOTE_ALL : VOTE<"all", I1RT, int_nvvm_vote_all>;240 defm VOTE_ANY : VOTE<"any", I1RT, int_nvvm_vote_any>;241 defm VOTE_UNI : VOTE<"uni", I1RT, int_nvvm_vote_uni>;242 defm VOTE_BALLOT : VOTE<"ballot", I32RT, int_nvvm_vote_ballot>;243 244 // vote.sync.{all,any,uni,ballot}245 multiclass VOTE_SYNC<string mode, RegTyInfo t, Intrinsic op> {246 def i : BasicNVPTXInst<(outs t.RC:$dest), (ins B1:$pred, i32imm:$mask),247 "vote.sync." # mode # "." # t.PtxType,248 [(set t.Ty:$dest, (op imm:$mask, i1:$pred))]>;249 def r : BasicNVPTXInst<(outs t.RC:$dest), (ins B1:$pred, B32:$mask),250 "vote.sync." # mode # "." # t.PtxType,251 [(set t.Ty:$dest, (op i32:$mask, i1:$pred))]>;252 }253 254 defm VOTE_SYNC_ALL : VOTE_SYNC<"all", I1RT, int_nvvm_vote_all_sync>;255 defm VOTE_SYNC_ANY : VOTE_SYNC<"any", I1RT, int_nvvm_vote_any_sync>;256 defm VOTE_SYNC_UNI : VOTE_SYNC<"uni", I1RT, int_nvvm_vote_uni_sync>;257 defm VOTE_SYNC_BALLOT : VOTE_SYNC<"ballot", I32RT, int_nvvm_vote_ballot_sync>;258}259// elect.sync260let Predicates = [hasPTX<80>, hasSM<90>] in {261def INT_ELECT_SYNC_I : BasicNVPTXInst<(outs B32:$dest, B1:$pred), (ins i32imm:$mask),262 "elect.sync",263 [(set i32:$dest, i1:$pred, (int_nvvm_elect_sync imm:$mask))]>;264def INT_ELECT_SYNC_R : BasicNVPTXInst<(outs B32:$dest, B1:$pred), (ins B32:$mask),265 "elect.sync",266 [(set i32:$dest, i1:$pred, (int_nvvm_elect_sync i32:$mask))]>;267}268 269let Predicates = [hasPTX<60>, hasSM<70>] in {270 multiclass MATCH_ANY_SYNC<Intrinsic op, RegTyInfo t> {271 def ii : BasicNVPTXInst<(outs B32:$dest), (ins t.Imm:$value, i32imm:$mask),272 "match.any.sync." # t.PtxType,273 [(set i32:$dest, (op imm:$mask, imm:$value))]>;274 def ir : BasicNVPTXInst<(outs B32:$dest), (ins t.Imm:$value, B32:$mask),275 "match.any.sync." # t.PtxType,276 [(set i32:$dest, (op i32:$mask, imm:$value))]>;277 def ri : BasicNVPTXInst<(outs B32:$dest), (ins t.RC:$value, i32imm:$mask),278 "match.any.sync." # t.PtxType,279 [(set i32:$dest, (op imm:$mask, t.Ty:$value))]>;280 def rr : BasicNVPTXInst<(outs B32:$dest), (ins t.RC:$value, B32:$mask),281 "match.any.sync." # t.PtxType,282 [(set i32:$dest, (op i32:$mask, t.Ty:$value))]>;283 }284 285 defm MATCH_ANY_SYNC_32 : MATCH_ANY_SYNC<int_nvvm_match_any_sync_i32, I32RT>;286 defm MATCH_ANY_SYNC_64 : MATCH_ANY_SYNC<int_nvvm_match_any_sync_i64, I64RT>;287 288 multiclass MATCH_ALLP_SYNC<RegTyInfo t, Intrinsic op> {289 def ii : BasicNVPTXInst<(outs B32:$dest, B1:$pred),290 (ins t.Imm:$value, i32imm:$mask),291 "match.all.sync." # t.PtxType,292 [(set i32:$dest, i1:$pred, (op imm:$mask, imm:$value))]>;293 def ir : BasicNVPTXInst<(outs B32:$dest, B1:$pred),294 (ins t.Imm:$value, B32:$mask),295 "match.all.sync." # t.PtxType,296 [(set i32:$dest, i1:$pred, (op i32:$mask, imm:$value))]>;297 def ri : BasicNVPTXInst<(outs B32:$dest, B1:$pred),298 (ins t.RC:$value, i32imm:$mask),299 "match.all.sync." # t.PtxType,300 [(set i32:$dest, i1:$pred, (op imm:$mask, t.Ty:$value))]>;301 def rr : BasicNVPTXInst<(outs B32:$dest, B1:$pred),302 (ins t.RC:$value, B32:$mask),303 "match.all.sync." # t.PtxType,304 [(set i32:$dest, i1:$pred, (op i32:$mask, t.Ty:$value))]>;305 }306 defm MATCH_ALLP_SYNC_32 : MATCH_ALLP_SYNC<I32RT, int_nvvm_match_all_sync_i32p>;307 defm MATCH_ALLP_SYNC_64 : MATCH_ALLP_SYNC<I64RT, int_nvvm_match_all_sync_i64p>;308}309 310// activemask.b32311def ACTIVEMASK : BasicNVPTXInst<(outs B32:$dest), (ins),312 "activemask.b32",313 [(set i32:$dest, (int_nvvm_activemask))]>,314 Requires<[hasPTX<62>, hasSM<30>]>;315 316multiclass REDUX_SYNC<string BinOp, string PTXType, Intrinsic Intrin> {317 def : BasicNVPTXInst<(outs B32:$dst), (ins B32:$src, B32:$mask),318 "redux.sync." # BinOp # "." # PTXType,319 [(set i32:$dst, (Intrin i32:$src, B32:$mask))]>,320 Requires<[hasPTX<70>, hasSM<80>]>;321}322 323defm REDUX_SYNC_UMIN : REDUX_SYNC<"min", "u32", int_nvvm_redux_sync_umin>;324defm REDUX_SYNC_UMAX : REDUX_SYNC<"max", "u32", int_nvvm_redux_sync_umax>;325defm REDUX_SYNC_ADD : REDUX_SYNC<"add", "s32", int_nvvm_redux_sync_add>;326defm REDUX_SYNC_MIN : REDUX_SYNC<"min", "s32", int_nvvm_redux_sync_min>;327defm REDUX_SYNC_MAX : REDUX_SYNC<"max", "s32", int_nvvm_redux_sync_max>;328defm REDUX_SYNC_AND : REDUX_SYNC<"and", "b32", int_nvvm_redux_sync_and>;329defm REDUX_SYNC_XOR : REDUX_SYNC<"xor", "b32", int_nvvm_redux_sync_xor>;330defm REDUX_SYNC_OR : REDUX_SYNC<"or", "b32", int_nvvm_redux_sync_or>;331 332multiclass REDUX_SYNC_F<string BinOp, string abs, string NaN> {333 defvar intr_name = "int_nvvm_redux_sync_f" # BinOp # !subst(".", "_", abs) # !subst(".", "_", NaN);334 335 def : BasicNVPTXInst<(outs B32:$dst),336 (ins B32:$src, B32:$mask),337 "redux.sync." # BinOp # abs # NaN # ".f32",338 [(set f32:$dst, (!cast<Intrinsic>(intr_name) f32:$src, B32:$mask))]>,339 Requires<[hasPTX<86>, hasSM100a]>; 340}341 342defm REDUX_SYNC_FMIN : REDUX_SYNC_F<"min", "", "">;343defm REDUX_SYNC_FMIN_ABS : REDUX_SYNC_F<"min", ".abs", "">;344defm REDUX_SYNC_FMIN_NAN: REDUX_SYNC_F<"min", "", ".NaN">;345defm REDUX_SYNC_FMIN_ABS_NAN: REDUX_SYNC_F<"min", ".abs", ".NaN">;346defm REDUX_SYNC_FMAX : REDUX_SYNC_F<"max", "", "">;347defm REDUX_SYNC_FMAX_ABS : REDUX_SYNC_F<"max", ".abs", "">;348defm REDUX_SYNC_FMAX_NAN: REDUX_SYNC_F<"max", "", ".NaN">;349defm REDUX_SYNC_FMAX_ABS_NAN: REDUX_SYNC_F<"max", ".abs", ".NaN">;350 351} // isConvergent = true352 353//-----------------------------------354// Explicit Memory Fence Functions355//-----------------------------------356class NullaryInst<string StrOp, Intrinsic IntOP> :357 BasicNVPTXInst<(outs), (ins), StrOp, [(IntOP)]>;358 359def INT_MEMBAR_CTA : NullaryInst<"membar.cta", int_nvvm_membar_cta>;360def INT_MEMBAR_GL : NullaryInst<"membar.gl", int_nvvm_membar_gl>;361def INT_MEMBAR_SYS : NullaryInst<"membar.sys", int_nvvm_membar_sys>;362 363def INT_FENCE_SC_CLUSTER:364 NullaryInst<"fence.sc.cluster", int_nvvm_fence_sc_cluster>,365 Requires<[hasPTX<78>, hasSM<90>]>;366 367def INT_FENCE_MBARRIER_INIT_RELEASE_CLUSTER:368 NullaryInst<"fence.mbarrier_init.release.cluster",369 int_nvvm_fence_mbarrier_init_release_cluster>,370 Requires<[hasPTX<80>, hasSM<90>]>;371 372let Predicates = [hasPTX<86>, hasSM<90>] in {373def INT_FENCE_ACQUIRE_SYNC_RESTRICT_CLUSTER_CLUSTER:374 NullaryInst<"fence.acquire.sync_restrict::shared::cluster.cluster",375 int_nvvm_fence_acquire_sync_restrict_space_cluster_scope_cluster>;376 377def INT_FENCE_RELEASE_SYNC_RESTRICT_CTA_CLUSTER:378 NullaryInst<"fence.release.sync_restrict::shared::cta.cluster",379 int_nvvm_fence_release_sync_restrict_space_cta_scope_cluster>;380}381 382// Proxy fence (uni-directional)383let Predicates = [hasPTX<86>, hasSM<90>] in {384def INT_NVVM_FENCE_PROXY_ASYNC_GENERIC_ACQUIRE_SYNC_RESTRICT_SPACE_CLUSTER_SCOPE_CLUSTER:385 NullaryInst<"fence.proxy.async::generic.acquire.sync_restrict::shared::cluster.cluster",386 int_nvvm_fence_proxy_async_generic_acquire_sync_restrict_space_cluster_scope_cluster>;387 388def INT_NVVM_FENCE_PROXY_ASYNC_GENERIC_RELEASE_SYNC_RESTRICT_SPACE_CTA_SCOPE_CLUSTER:389 NullaryInst<"fence.proxy.async::generic.release.sync_restrict::shared::cta.cluster",390 int_nvvm_fence_proxy_async_generic_release_sync_restrict_space_cta_scope_cluster>;391}392 393// Proxy fence (bi-directional)394foreach proxykind = ["alias", "async", "async.global", "async.shared_cta",395 "async.shared_cluster"] in {396 defvar Preds = !if(!eq(proxykind, "alias"), [hasPTX<75>, hasSM<70>],397 [hasPTX<80>, hasSM<90>]);398 defvar Intr = IntrinsicName<"llvm.nvvm.fence.proxy." # proxykind>;399 def : NullaryInst<"fence.proxy." # !subst("_", "::", proxykind),400 !cast<Intrinsic>(Intr.record_name)>, Requires<Preds>;401}402 403class FENCE_PROXY_TENSORMAP_GENERIC_RELEASE<string Scope, Intrinsic Intr> :404 NullaryInst<"fence.proxy.tensormap::generic.release." # Scope, Intr>,405 Requires<[hasPTX<83>, hasSM<90>]>;406 407def INT_FENCE_PROXY_TENSORMAP_GENERIC_RELEASE_CTA:408 FENCE_PROXY_TENSORMAP_GENERIC_RELEASE<"cta",409 int_nvvm_fence_proxy_tensormap_generic_release_cta>;410def INT_FENCE_PROXY_TENSORMAP_GENERIC_RELEASE_CLUSTER:411 FENCE_PROXY_TENSORMAP_GENERIC_RELEASE<"cluster",412 int_nvvm_fence_proxy_tensormap_generic_release_cluster>;413def INT_FENCE_PROXY_TENSORMAP_GENERIC_RELEASE_GPU:414 FENCE_PROXY_TENSORMAP_GENERIC_RELEASE<"gpu",415 int_nvvm_fence_proxy_tensormap_generic_release_gpu>;416def INT_FENCE_PROXY_TENSORMAP_GENERIC_RELEASE_SYS:417 FENCE_PROXY_TENSORMAP_GENERIC_RELEASE<"sys",418 int_nvvm_fence_proxy_tensormap_generic_release_sys>;419 420// fence.proxy.tensormap.acquire variants421 422class FENCE_PROXY_TENSORMAP_GENERIC_ACQUIRE<string Scope, Intrinsic Intr> :423 NVPTXInst<(outs), (ins B64:$addr),424 "fence.proxy.tensormap::generic.acquire." # Scope # " [$addr], 128;",425 [(Intr i64:$addr, (i32 128))]>,426 Requires<[hasPTX<83>, hasSM<90>]>;427 428def INT_FENCE_PROXY_TENSORMAP_GENERIC_ACQUIRE_CTA :429 FENCE_PROXY_TENSORMAP_GENERIC_ACQUIRE<"cta",430 int_nvvm_fence_proxy_tensormap_generic_acquire_cta>;431def INT_FENCE_PROXY_TENSORMAP_GENERIC_ACQUIRE_CLUSTER :432 FENCE_PROXY_TENSORMAP_GENERIC_ACQUIRE<"cluster",433 int_nvvm_fence_proxy_tensormap_generic_acquire_cluster>;434def INT_FENCE_PROXY_TENSORMAP_GENERIC_ACQUIRE_GPU :435 FENCE_PROXY_TENSORMAP_GENERIC_ACQUIRE<"gpu",436 int_nvvm_fence_proxy_tensormap_generic_acquire_gpu>;437def INT_FENCE_PROXY_TENSORMAP_GENERIC_ACQUIRE_SYS :438 FENCE_PROXY_TENSORMAP_GENERIC_ACQUIRE<"sys",439 int_nvvm_fence_proxy_tensormap_generic_acquire_sys>;440 441//-----------------------------------442// Async Copy Functions443//-----------------------------------444 445multiclass CP_ASYNC_MBARRIER_ARRIVE<string NoInc, string AddrSpace, Intrinsic Intrin> {446 def "" : BasicNVPTXInst<(outs), (ins ADDR:$addr),447 "cp.async.mbarrier.arrive" # NoInc # AddrSpace # ".b64",448 [(Intrin addr:$addr)]>,449 Requires<[hasPTX<70>, hasSM<80>]>;450}451 452defm CP_ASYNC_MBARRIER_ARRIVE :453 CP_ASYNC_MBARRIER_ARRIVE<"", "", int_nvvm_cp_async_mbarrier_arrive>;454defm CP_ASYNC_MBARRIER_ARRIVE_SHARED :455 CP_ASYNC_MBARRIER_ARRIVE<"", ".shared", int_nvvm_cp_async_mbarrier_arrive_shared>;456defm CP_ASYNC_MBARRIER_ARRIVE_NOINC :457 CP_ASYNC_MBARRIER_ARRIVE<".noinc", "", int_nvvm_cp_async_mbarrier_arrive_noinc>;458defm CP_ASYNC_MBARRIER_ARRIVE_NOINC_SHARED :459 CP_ASYNC_MBARRIER_ARRIVE<".noinc", ".shared", int_nvvm_cp_async_mbarrier_arrive_noinc_shared>;460 461multiclass CP_ASYNC_SHARED_GLOBAL_I<string cc, string cpsize, Intrinsic Intrin, Intrinsic IntrinS> {462 def "" : NVPTXInst<(outs), (ins ADDR:$dst, ADDR:$src),463 "cp.async." # cc # ".shared.global" # " [$dst], [$src], " # cpsize # ";",464 [(Intrin addr:$dst, addr:$src)]>,465 Requires<[hasPTX<70>, hasSM<80>]>;466 467 // Variant with src_size parameter468 def _s : NVPTXInst<(outs), (ins ADDR:$dst, ADDR:$src, B32:$src_size),469 "cp.async." # cc # ".shared.global" # " [$dst], [$src], " # cpsize # ", $src_size;",470 [(IntrinS addr:$dst, addr:$src, i32:$src_size)]>,471 Requires<[hasPTX<70>, hasSM<80>]>;472 def _si: NVPTXInst<(outs), (ins ADDR:$dst, ADDR:$src, i32imm:$src_size),473 "cp.async." # cc # ".shared.global" # " [$dst], [$src], " # cpsize # ", $src_size;",474 [(IntrinS addr:$dst, addr:$src, imm:$src_size)]>,475 Requires<[hasPTX<70>, hasSM<80>]>;476}477 478defm CP_ASYNC_CA_SHARED_GLOBAL_4 :479 CP_ASYNC_SHARED_GLOBAL_I<"ca", "4", int_nvvm_cp_async_ca_shared_global_4,480 int_nvvm_cp_async_ca_shared_global_4_s>;481 482defm CP_ASYNC_CA_SHARED_GLOBAL_8 :483 CP_ASYNC_SHARED_GLOBAL_I<"ca", "8", int_nvvm_cp_async_ca_shared_global_8,484 int_nvvm_cp_async_ca_shared_global_8_s>;485 486defm CP_ASYNC_CA_SHARED_GLOBAL_16 :487 CP_ASYNC_SHARED_GLOBAL_I<"ca", "16", int_nvvm_cp_async_ca_shared_global_16,488 int_nvvm_cp_async_ca_shared_global_16_s>;489 490defm CP_ASYNC_CG_SHARED_GLOBAL_16 :491 CP_ASYNC_SHARED_GLOBAL_I<"cg", "16", int_nvvm_cp_async_cg_shared_global_16,492 int_nvvm_cp_async_cg_shared_global_16_s>;493 494let Predicates = [hasPTX<70>, hasSM<80>] in {495 def CP_ASYNC_COMMIT_GROUP :496 NullaryInst<"cp.async.commit_group", int_nvvm_cp_async_commit_group>;497 498 def CP_ASYNC_WAIT_GROUP :499 BasicNVPTXInst<(outs), (ins i32imm:$n), "cp.async.wait_group",500 [(int_nvvm_cp_async_wait_group timm:$n)]>;501 502 def CP_ASYNC_WAIT_ALL :503 NullaryInst<"cp.async.wait_all", int_nvvm_cp_async_wait_all>;504}505 506let Predicates = [hasPTX<80>, hasSM<90>] in {507 // cp.async.bulk variants of the commit/wait group508 def CP_ASYNC_BULK_COMMIT_GROUP :509 NullaryInst<"cp.async.bulk.commit_group", int_nvvm_cp_async_bulk_commit_group>;510 511 def CP_ASYNC_BULK_WAIT_GROUP :512 BasicNVPTXInst<(outs), (ins i32imm:$n), "cp.async.bulk.wait_group",513 [(int_nvvm_cp_async_bulk_wait_group timm:$n)]>;514 515 def CP_ASYNC_BULK_WAIT_GROUP_READ :516 BasicNVPTXInst<(outs), (ins i32imm:$n), "cp.async.bulk.wait_group.read",517 [(int_nvvm_cp_async_bulk_wait_group_read timm:$n)]>;518}519 520//------------------------------521// TMA Async Bulk Copy Functions522//------------------------------523 524class CpAsyncBulkStr<bit mc, bit ch, bit mask = 0> {525 // Shared to Global memory526 string S2G = "cp.async.bulk.global.shared::cta.bulk_group"527 # !if(ch, ".L2::cache_hint", "")528 # !if(mask, ".cp_mask", "");529 530 // Global to Shared cluster memory531 string G2S = "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes"532 # !if(mc, ".multicast::cluster", "")533 # !if(ch, ".L2::cache_hint", "");534 535 // Global to Shared CTA memory536 string G2S_CTA = "cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes"537 # !if(ch, ".L2::cache_hint", "");538 539 // Shared CTA to Cluster memory540 string C2C = "cp.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes";541}542 543multiclass CP_ASYNC_BULK_S2G_INTR<bit has_ch> {544 def "" : NVPTXInst<(outs), (ins ADDR:$dst, ADDR:$src, B32:$size, B64:$ch),545 !if(has_ch,546 CpAsyncBulkStr<0, 1>.S2G # " [$dst], [$src], $size, $ch;",547 CpAsyncBulkStr<0, 0>.S2G # " [$dst], [$src], $size;"),548 [(int_nvvm_cp_async_bulk_shared_cta_to_global addr:$dst, addr:$src, i32:$size, i64:$ch, !if(has_ch, -1, 0))]>,549 Requires<[hasPTX<80>, hasSM<90>]>;550 551 def _BM : NVPTXInst<(outs), (ins ADDR:$dst, ADDR:$src, B32:$size, B64:$ch, B16:$mask),552 !if(has_ch,553 CpAsyncBulkStr<0, 1, 1>.S2G # " [$dst], [$src], $size, $ch, $mask;",554 CpAsyncBulkStr<0, 0, 1>.S2G # " [$dst], [$src], $size, $mask;"),555 [(int_nvvm_cp_async_bulk_shared_cta_to_global_bytemask addr:$dst, addr:$src, i32:$size, i64:$ch, !if(has_ch, -1, 0), i16:$mask)]>,556 Requires<[hasPTX<86>, hasSM<100>]>;557}558defm CP_ASYNC_BULK_S2G : CP_ASYNC_BULK_S2G_INTR<has_ch = 0>;559defm CP_ASYNC_BULK_S2G_CH : CP_ASYNC_BULK_S2G_INTR<has_ch = 1>;560 561multiclass CP_ASYNC_BULK_G2S_INTR<bit has_ch> {562 defvar Intr = int_nvvm_cp_async_bulk_global_to_shared_cluster;563 564 def "" : NVPTXInst<(outs),565 (ins ADDR:$dst, ADDR:$mbar, ADDR:$src,566 B32:$size, B16:$mask, B64:$ch),567 !if(has_ch,568 CpAsyncBulkStr<0, 1>.G2S # " [$dst], [$src], $size, [$mbar], $ch;",569 CpAsyncBulkStr<0, 0>.G2S # " [$dst], [$src], $size, [$mbar];"),570 [(Intr addr:$dst, addr:$mbar, addr:$src, i32:$size, i16:$mask, i64:$ch, 0, !if(has_ch, -1, 0))]>,571 Requires<[hasPTX<80>, hasSM<90>]>;572 573 def _MC : NVPTXInst<(outs),574 (ins ADDR:$dst, ADDR:$mbar, ADDR:$src,575 B32:$size, B16:$mask, B64:$ch),576 !if(has_ch,577 CpAsyncBulkStr<1, 1>.G2S # " [$dst], [$src], $size, [$mbar], $mask, $ch;",578 CpAsyncBulkStr<1, 0>.G2S # " [$dst], [$src], $size, [$mbar], $mask;"),579 [(Intr addr:$dst, addr:$mbar, addr:$src, i32:$size, i16:$mask, i64:$ch, -1, !if(has_ch, -1, 0))]>,580 Requires<[hasPTX<80>, hasSM<90>]>;581}582defm CP_ASYNC_BULK_G2S : CP_ASYNC_BULK_G2S_INTR<has_ch = 0>;583defm CP_ASYNC_BULK_G2S_CH : CP_ASYNC_BULK_G2S_INTR<has_ch = 1>;584 585multiclass CP_ASYNC_BULK_G2S_CTA_INTR<bit has_ch> {586 defvar Intr = int_nvvm_cp_async_bulk_global_to_shared_cta;587 588 def "" : NVPTXInst<(outs),589 (ins ADDR:$dst, ADDR:$mbar, ADDR:$src,590 B32:$size, B64:$ch),591 !if(has_ch,592 CpAsyncBulkStr<0, 1>.G2S_CTA # " [$dst], [$src], $size, [$mbar], $ch;",593 CpAsyncBulkStr<0, 0>.G2S_CTA # " [$dst], [$src], $size, [$mbar];"),594 [(Intr addr:$dst, addr:$mbar, addr:$src, i32:$size, i64:$ch, !if(has_ch, -1, 0))]>,595 Requires<[hasPTX<86>, hasSM<90>]>;596}597defm CP_ASYNC_BULK_G2S_CTA : CP_ASYNC_BULK_G2S_CTA_INTR<has_ch = 0>;598defm CP_ASYNC_BULK_G2S_CTA_CH : CP_ASYNC_BULK_G2S_CTA_INTR<has_ch = 1>;599 600def CP_ASYNC_BULK_CTA_TO_CLUSTER : NVPTXInst<(outs),601 (ins ADDR:$dst, ADDR:$mbar, ADDR:$src, B32:$size),602 CpAsyncBulkStr<0, 0>.C2C # " [$dst], [$src], $size, [$mbar];",603 [(int_nvvm_cp_async_bulk_shared_cta_to_cluster addr:$dst, addr:$mbar, addr:$src, i32:$size)]>,604 Requires<[hasPTX<80>, hasSM<90>]>;605 606multiclass CP_ASYNC_BULK_PREFETCH_INTR<bit has_ch> {607 def "" : NVPTXInst<(outs), (ins ADDR:$src, B32:$size, B64:$ch),608 !if(has_ch,609 "cp.async.bulk.prefetch.L2.global.L2::cache_hint" # " [$src], $size, $ch;",610 "cp.async.bulk.prefetch.L2.global" # " [$src], $size;"),611 [(int_nvvm_cp_async_bulk_prefetch_L2 addr:$src, i32:$size, i64:$ch, !if(has_ch, -1, 0))]>,612 Requires<[hasPTX<80>, hasSM<90>]>;613}614defm CP_ASYNC_BULK_PREFETCH : CP_ASYNC_BULK_PREFETCH_INTR<has_ch = 0>;615defm CP_ASYNC_BULK_PREFETCH_CH : CP_ASYNC_BULK_PREFETCH_INTR<has_ch = 1>;616 617//-------------------------------------618// TMA Async Bulk Tensor Copy Functions619//-------------------------------------620 621class TMA_DIMS_UTIL<int dim, string mode = ""> {622 // For example, when 'dim' is 3, this generates:623 // an ins_dag: B32:$d0, B32:$d1, B32:$d2624 // with base_str: $d0, $d1, $d2625 dag ins_dag = !dag(ins, !listsplat(B32, dim), !foreach(i, !range(dim), "d" # i));626 string base_str = !interleave(!foreach(i, !range(dim), "$d" # i), ", ");627 628 // Tile::Gather4/scatter4 actually operate on a 2D tensor,629 // though they take 5 co-ordinates.630 //631 // The scatter-gather happens over 4 rows with a fixed632 // column-index. The first co-ordinate represents the633 // col-index followed by four row-indices.634 int num_dims = !cond(635 !eq(mode, "tile_scatter4") : 2,636 !eq(mode, "tile_gather4") : 2,637 true : dim); // for all other modes638}639 640class TMA_IM2COL_UTIL<int dim, string mode> {641 // For im2col_w/w_128 modes, number of offsets is always 2.642 // For im2col mode, offsets is (dim - 2).643 // For non-im2col modes (i.e. tile) there are no offsets.644 int offsets = !cond(645 !eq(mode, "im2col") : !sub(dim, 2),646 !eq(mode, "im2col_w") : 2,647 !eq(mode, "im2col_w_128") : 2,648 true : 0); // for all other modes649 650 dag ins_dag = !if(!gt(offsets, 0),651 !dag(ins, !listsplat(B16, offsets), !foreach(i, !range(offsets), "im2col" # i)),652 (ins));653 string base_str = !interleave(!foreach(i, !range(offsets), "$im2col" # i), ", ");654}655 656def CTAGroupFlags : Operand<i32> {657 let PrintMethod = "printCTAGroup";658}659 660def tma_cta_group_imm0 : TImmLeaf<i32, [{return Imm == 0;}]>;661def tma_cta_group_imm_any : TImmLeaf<i32, [{return Imm >= 0;}]>;662 663multiclass TMA_TENSOR_G2S_INTR<int dim, string mode, list<Predicate> pred,664 TImmLeaf cta_group_type = tma_cta_group_imm_any> {665 defvar dims_dag = TMA_DIMS_UTIL<dim>.ins_dag;666 defvar dims_str = TMA_DIMS_UTIL<dim>.base_str;667 defvar asm_str_base = "$cg [$dst], [$tmap, {{" # dims_str # "}}], [$mbar]";668 669 defvar im2col_dag = TMA_IM2COL_UTIL<dim, mode>.ins_dag;670 defvar im2col_str = TMA_IM2COL_UTIL<dim, mode>.base_str;671 defvar asm_str = !if(!empty(im2col_str),672 asm_str_base,673 asm_str_base # ", {{" # im2col_str # "}}");674 675 defvar dim_val = TMA_DIMS_UTIL<dim, mode>.num_dims;676 defvar inst_name = "cp.async.bulk.tensor"677 # "." # dim_val # "d"678 # "." # "shared::cluster.global"679 # "." # !subst("_", "::", mode)680 # "." # "mbarrier::complete_tx::bytes";681 defvar intr = !cast<Intrinsic>(682 "int_nvvm_cp_async_bulk_tensor_g2s_" # mode # "_" # dim_val # "d");683 684 defvar ins_dag = !con(685 (ins ADDR:$dst, ADDR:$mbar, B64:$tmap),686 dims_dag, im2col_dag,687 (ins B16:$mc, B64:$ch, CTAGroupFlags:$cg));688 689 defvar intr_dag_base = !con(690 (intr addr:$dst, addr:$mbar, B64:$tmap),691 !setdagop(dims_dag, intr),692 !setdagop(im2col_dag, intr),693 (intr B16:$mc, B64:$ch));694 defvar intr_dag_no_hints = !con(intr_dag_base, (intr 0, 0, cta_group_type:$cg));695 defvar intr_dag_with_mc = !con(intr_dag_base, (intr -1, 0, cta_group_type:$cg));696 defvar intr_dag_with_ch = !con(intr_dag_base, (intr 0, -1, cta_group_type:$cg));697 defvar intr_dag_with_mc_ch = !con(intr_dag_base, (intr -1, -1, cta_group_type:$cg));698 699 def "" : NVPTXInst<(outs), ins_dag,700 inst_name # asm_str # ";",701 [intr_dag_no_hints]>,702 Requires<pred>;703 def _MC : NVPTXInst<(outs), ins_dag,704 inst_name # ".multicast::cluster" # asm_str # ", $mc;",705 [intr_dag_with_mc]>,706 Requires<pred>;707 def _CH : NVPTXInst<(outs), ins_dag,708 inst_name # ".L2::cache_hint" # asm_str # ", $ch;",709 [intr_dag_with_ch]>,710 Requires<pred>;711 def _MC_CH : NVPTXInst<(outs), ins_dag,712 inst_name # ".multicast::cluster.L2::cache_hint" # asm_str # ", $mc, $ch;",713 [intr_dag_with_mc_ch]>,714 Requires<pred>;715}716 717foreach dim = 1...5 in {718 defm TMA_G2S_TILE_CG0_ # dim # "D"719 : TMA_TENSOR_G2S_INTR<dim, "tile", [hasPTX<80>, hasSM<90>],720 tma_cta_group_imm0>;721 defm TMA_G2S_TILE_ # dim # "D"722 : TMA_TENSOR_G2S_INTR<dim, "tile",723 [callSubtarget<"hasTMABlackwellSupport">]>;724}725foreach dim = 3...5 in {726 defm TMA_G2S_IM2COL_CG0_ # dim # "D"727 : TMA_TENSOR_G2S_INTR<dim, "im2col", [hasPTX<80>, hasSM<90>],728 tma_cta_group_imm0>;729 defm TMA_G2S_IM2COL_ # dim # "D"730 : TMA_TENSOR_G2S_INTR<dim, "im2col",731 [callSubtarget<"hasTMABlackwellSupport">]>;732 foreach mode = ["im2col_w", "im2col_w_128"] in {733 defm TMA_G2S_ # !toupper(mode) # "_" # dim # "D"734 : TMA_TENSOR_G2S_INTR<dim, mode,735 [callSubtarget<"hasTMABlackwellSupport">]>;736 }737}738defm TMA_G2S_TILE_GATHER4_2D : TMA_TENSOR_G2S_INTR<5, "tile_gather4",739 [callSubtarget<"hasTMABlackwellSupport">]>;740 741multiclass TMA_TENSOR_G2S_CTA_INTR<int dim, string mode, list<Predicate> pred = []> {742 defvar dims_dag = TMA_DIMS_UTIL<dim>.ins_dag;743 defvar dims_str = TMA_DIMS_UTIL<dim>.base_str;744 defvar asm_str_base = " [$dst], [$tmap, {{" # dims_str # "}}], [$mbar]";745 746 defvar im2col_dag = TMA_IM2COL_UTIL<dim, mode>.ins_dag;747 defvar im2col_str = TMA_IM2COL_UTIL<dim, mode>.base_str;748 defvar asm_str = !if(!empty(im2col_str),749 asm_str_base,750 asm_str_base # ", {{" # im2col_str # "}}");751 752 defvar ins_dag = !con(753 (ins ADDR:$dst, ADDR:$mbar, B64:$tmap),754 dims_dag, im2col_dag,755 (ins B64:$ch));756 757 defvar dim_val = TMA_DIMS_UTIL<dim, mode>.num_dims;758 defvar intr = !cast<Intrinsic>(759 "int_nvvm_cp_async_bulk_tensor_g2s_cta_" # mode # "_" # dim_val # "d");760 defvar intr_dag = !con(761 (intr addr:$dst, addr:$mbar, B64:$tmap),762 !setdagop(dims_dag, intr),763 !setdagop(im2col_dag, intr),764 (intr B64:$ch, 0));765 defvar intr_dag_with_ch = !con(766 (intr addr:$dst, addr:$mbar, B64:$tmap),767 !setdagop(dims_dag, intr),768 !setdagop(im2col_dag, intr),769 (intr B64:$ch, -1));770 defvar inst_name = "cp.async.bulk.tensor"771 # "." # dim_val # "d"772 # "." # "shared::cta.global"773 # "." # !subst("_", "::", mode)774 # "." # "mbarrier::complete_tx::bytes";775 776 def "" : NVPTXInst<(outs), ins_dag,777 inst_name # asm_str # ";",778 [intr_dag]>,779 Requires<pred>;780 def _CH : NVPTXInst<(outs), ins_dag,781 inst_name # ".L2::cache_hint" # asm_str # ", $ch;",782 [intr_dag_with_ch]>,783 Requires<pred>;784}785foreach dim = 1...5 in {786 defm TMA_G2S_CTA_TILE_ # dim # "D"787 : TMA_TENSOR_G2S_CTA_INTR<dim, "tile", [hasPTX<86>, hasSM<90>]>;788}789foreach dim = 3...5 in {790 defm TMA_G2S_CTA_IM2COL_ # dim # "D"791 : TMA_TENSOR_G2S_CTA_INTR<dim, "im2col", [hasPTX<86>, hasSM<90>]>;792 793 defm TMA_G2S_CTA_IM2COL_W_ # dim # "D"794 : TMA_TENSOR_G2S_CTA_INTR<dim, "im2col_w", [hasPTX<86>, hasSM<100>]>;795 796 defm TMA_G2S_CTA_IM2COL_W_128_ # dim # "D"797 : TMA_TENSOR_G2S_CTA_INTR<dim, "im2col_w_128",798 [callSubtarget<"hasTMABlackwellSupport">]>;799}800defm TMA_G2S_CTA_TILE_GATHER4_2D : TMA_TENSOR_G2S_CTA_INTR<5, "tile_gather4",801 [hasPTX<86>, hasSM<100>]>;802 803multiclass TMA_TENSOR_S2G_INTR<int dim, string mode,804 list<Predicate> pred = [hasPTX<80>, hasSM<90>]> {805 defvar dims_dag = TMA_DIMS_UTIL<dim>.ins_dag;806 defvar dims_str = TMA_DIMS_UTIL<dim>.base_str;807 defvar asm_str = " [$tmap, {{" # dims_str # "}}], [$src]";808 809 defvar dim_val = TMA_DIMS_UTIL<dim, mode>.num_dims;810 defvar intr = !cast<Intrinsic>(811 "int_nvvm_cp_async_bulk_tensor_s2g_" # mode # "_" # dim_val # "d");812 813 defvar intr_dag = !con((intr addr:$src, B64:$tmap),814 !setdagop(dims_dag, intr),815 (intr B64:$ch, 0));816 defvar intr_dag_with_ch = !con((intr addr:$src, B64:$tmap),817 !setdagop(dims_dag, intr),818 (intr B64:$ch, -1));819 820 // Fix-up the asm_str when it is im2col/scatter4.821 defvar mode_asm_str = !cond(822 !eq(mode, "im2col") : "im2col_no_offs",823 !eq(mode, "tile_scatter4") : "tile::scatter4",824 true : mode);825 defvar prefix = "cp.async.bulk.tensor"826 # "." # dim_val # "d"827 # ".global.shared::cta"828 # "." # mode_asm_str829 # ".bulk_group";830 831 def "" : NVPTXInst<(outs),832 !con((ins ADDR:$src, B64:$tmap), dims_dag, (ins B64:$ch)),833 prefix # asm_str # ";",834 [intr_dag]>,835 Requires<pred>;836 def _CH : NVPTXInst<(outs),837 !con((ins ADDR:$src, B64:$tmap), dims_dag, (ins B64:$ch)),838 prefix # ".L2::cache_hint" # asm_str # ", $ch;",839 [intr_dag_with_ch]>,840 Requires<pred>;841}842foreach dim = 1...5 in {843 foreach mode = !if(!ge(dim, 3), ["tile", "im2col"], ["tile"]) in {844 defvar suffix = !toupper(mode) # "_" # dim # "D";845 defm TMA_TENSOR_S2G_ # suffix : TMA_TENSOR_S2G_INTR<dim, mode>;846 }847}848defm TMA_S2G_TILE_SCATTER4_2D : TMA_TENSOR_S2G_INTR<5, "tile_scatter4",849 [callSubtarget<"hasTMABlackwellSupport">]>;850 851def TMAReductionFlags : Operand<i32> {852 let PrintMethod = "printTmaReductionMode";853}854 855// TMA Copy from Shared to Global memory with Reduction856multiclass CP_ASYNC_BULK_TENSOR_REDUCE_INTR<int dim, bit shared32, string mode> {857 defvar dims_dag = TMA_DIMS_UTIL<dim>.ins_dag;858 defvar dims_str = TMA_DIMS_UTIL<dim>.base_str;859 defvar asm_str = " [$tmap, {{" # dims_str # "}}], [$src]";860 defvar rc = !if(shared32, B32, B64);861 862 // For im2col mode, the actual asm_str is "im2col_no_offs"863 defvar mode_asm_str = !if(!eq(mode, "im2col"),864 "im2col_no_offs", mode);865 defvar prefix = "cp.reduce.async.bulk.tensor" # "." # dim # "d" # ".global.shared::cta";866 defvar suffix = "." # mode_asm_str # ".bulk_group";867 868 def "" : NVPTXInst<(outs),869 !con((ins rc:$src, B64:$tmap), dims_dag, (ins TMAReductionFlags:$red_op)),870 !strconcat(prefix, "${red_op}", suffix, asm_str, ";")>,871 Requires<[hasPTX<80>, hasSM<90>]>;872 def _CH : NVPTXInst<(outs),873 !con((ins rc:$src, B64:$tmap), dims_dag, (ins B64:$ch, TMAReductionFlags:$red_op)),874 !strconcat(prefix, "${red_op}", suffix, ".L2::cache_hint", asm_str, ", $ch;")>,875 Requires<[hasPTX<80>, hasSM<90>]>;876}877 878foreach dim = [1, 2, 3, 4, 5] in {879 foreach shared32 = [true, false] in {880 foreach mode = !if(!ge(dim, 3), ["tile", "im2col"], ["tile"]) in {881 defvar suffix = dim # "D"882 # !if(shared32, "_SHARED32", "")883 # "_" # !toupper(mode);884 defm CP_ASYNC_BULK_TENSOR_RED_ # suffix :885 CP_ASYNC_BULK_TENSOR_REDUCE_INTR<dim, shared32, mode>;886 }887 }888}889 890// TMA Prefetch from Global memory to L2 cache891multiclass TMA_TENSOR_PREFETCH_INTR<int dim, string mode,892 list<Predicate> pred = [hasPTX<80>, hasSM<90>]> {893 defvar dims_dag = TMA_DIMS_UTIL<dim>.ins_dag;894 defvar dims_str = TMA_DIMS_UTIL<dim>.base_str;895 defvar asm_str_base = " [$tmap, {{" # dims_str # "}}]";896 897 defvar im2col_dag = TMA_IM2COL_UTIL<dim, mode>.ins_dag;898 defvar im2col_str = TMA_IM2COL_UTIL<dim, mode>.base_str;899 defvar asm_str = !if(!empty(im2col_str),900 asm_str_base,901 asm_str_base # ", {{" # im2col_str # "}}");902 903 defvar dim_val = TMA_DIMS_UTIL<dim, mode>.num_dims;904 defvar inst_name = "cp.async.bulk.prefetch.tensor"905 # "." # dim_val # "d"906 # "." # "L2.global"907 # "." # !subst("_", "::", mode);908 909 defvar intr = !cast<Intrinsic>(910 "int_nvvm_cp_async_bulk_tensor_prefetch_" # mode # "_" # dim_val # "d");911 912 defvar ins_dag = !con((ins B64:$tmap),913 dims_dag,914 im2col_dag,915 (ins B64:$ch));916 defvar intr_dag = !con((intr B64:$tmap),917 !setdagop(dims_dag, intr),918 !setdagop(im2col_dag, intr),919 (intr B64:$ch, 0));920 defvar intr_dag_with_ch = !con((intr B64:$tmap),921 !setdagop(dims_dag, intr),922 !setdagop(im2col_dag, intr),923 (intr B64:$ch, -1));924 925 def "" : NVPTXInst<(outs), ins_dag,926 inst_name # asm_str # ";",927 [intr_dag]>,928 Requires<pred>;929 def _CH : NVPTXInst<(outs), ins_dag,930 inst_name # ".L2::cache_hint" # asm_str # ", $ch;",931 [intr_dag_with_ch]>,932 Requires<pred>;933}934foreach dim = 1...5 in {935 foreach mode = !if(!ge(dim, 3), ["tile", "im2col"], ["tile"]) in {936 defvar suffix = !toupper(mode) # "_" # dim # "D";937 defm TMA_TENSOR_PF_ # suffix : TMA_TENSOR_PREFETCH_INTR<dim, mode>;938 }939}940foreach dim = 3...5 in {941 foreach mode = ["im2col_w", "im2col_w_128"] in {942 defvar suffix = !toupper(mode) # "_" # dim # "D";943 defm TMA_TENSOR_PF_ # suffix : TMA_TENSOR_PREFETCH_INTR<dim, mode,944 [callSubtarget<"hasTMABlackwellSupport">]>;945 }946}947defm TMA_TENSOR_PF_TILE_GATHER4_2D : TMA_TENSOR_PREFETCH_INTR<5, "tile_gather4",948 [callSubtarget<"hasTMABlackwellSupport">]>;949 950//Prefetchu and Prefetch951 952defvar frag_pat = (int_nvvm_prefetch_tensormap node:$addr);953 954multiclass PREFETCH_TENSORMAP_PATFRAG<string suffix, code predicate> {955 def !tolower(suffix) : PatFrag<!setdagop(frag_pat, ops), frag_pat, predicate>;956}957 958defm prefetch_tensormap_ : PREFETCH_TENSORMAP_PATFRAG<"CONST", AS_match.const>;959defm prefetch_tensormap_ : PREFETCH_TENSORMAP_PATFRAG<"GENERIC", AS_match.generic>;960defm prefetch_tensormap_ : PREFETCH_TENSORMAP_PATFRAG<"PARAM", AS_match.param>;961 962multiclass PREFETCH_TENSORMAP_INST<string addrspace_name, PatFrag pattern_frag> {963 def "" : BasicNVPTXInst<(outs), (ins ADDR:$addr),964 "prefetch" # addrspace_name # ".tensormap",965 [(pattern_frag addr:$addr)]>,966 Requires<[hasPTX<80>, hasSM<90>]>;967}968 969defm PREFETCH_CONST_TENSORMAP : PREFETCH_TENSORMAP_INST<".const", prefetch_tensormap_const>;970defm PREFETCH_GENERIC_TENSORMAP : PREFETCH_TENSORMAP_INST<"", prefetch_tensormap_generic>;971defm PREFETCH_PARAM_TENSORMAP : PREFETCH_TENSORMAP_INST<".param", prefetch_tensormap_param>;972 973class PREFETCH_INTRS<string InstName, Intrinsic Intr> :974 BasicNVPTXInst<(outs), (ins ADDR:$addr),975 InstName,976 [(Intr addr:$addr)]>,977 Requires<[hasPTX<80>, hasSM<90>]>;978 979def PREFETCHU_L1 : PREFETCH_INTRS<"prefetchu.L1", int_nvvm_prefetchu_L1>; 980def PREFETCH_L1 : PREFETCH_INTRS<"prefetch.L1", int_nvvm_prefetch_L1>;981def PREFETCH_L2 : PREFETCH_INTRS<"prefetch.L2", int_nvvm_prefetch_L2>;982def PREFETCH_GLOBAL_L1 : PREFETCH_INTRS<"prefetch.global.L1", int_nvvm_prefetch_global_L1>;983def PREFETCH_LOCAL_L1 : PREFETCH_INTRS<"prefetch.local.L1", int_nvvm_prefetch_local_L1>;984def PREFETCH_GLOBAL_L2 : PREFETCH_INTRS<"prefetch.global.L2", int_nvvm_prefetch_global_L2>;985def PREFETCH_LOCAL_L2 : PREFETCH_INTRS<"prefetch.local.L2", int_nvvm_prefetch_local_L2>;986def PREFETCH_GLOBAL_L2_EVICT_NORMAL : PREFETCH_INTRS<"prefetch.global.L2::evict_normal", 987 int_nvvm_prefetch_global_L2_evict_normal>;988def PREFETCH_GLOBAL_L2_EVICT_LAST : PREFETCH_INTRS<"prefetch.global.L2::evict_last", 989 int_nvvm_prefetch_global_L2_evict_last>;990 991//Applypriority intrinsics992class APPLYPRIORITY_L2_INTRS<string addrspace> :993 BasicNVPTXInst<(outs), (ins ADDR:$addr, B64:$size),994 StrJoin<".", ["applypriority", addrspace , "L2::evict_normal"]>.ret,995 [(!cast<Intrinsic>(StrJoin<"_", ["int_nvvm_applypriority", addrspace , "L2_evict_normal"]>.ret)996 addr:$addr, i64:$size)]>,997 Requires<[hasPTX<74>, hasSM<80>]>;998 999def APPLYPRIORITY_L2_EVICT_NORMAL : APPLYPRIORITY_L2_INTRS<"">;1000def APPLYPRIORITY_GLOBAL_L2_EVICT_NORMAL : APPLYPRIORITY_L2_INTRS<"global">;1001 1002//Discard Intrinsics1003 1004def discard_size_imm : TImmLeaf<i64, [{ return Imm == 128; }]>;1005 1006class DISCARD_L2_INTRS<string addrspace> :1007 BasicNVPTXInst<(outs), (ins ADDR:$addr, i64imm:$size),1008 StrJoin<".", ["discard", addrspace , "L2"]>.ret,1009 [(!cast<Intrinsic>(StrJoin<"_", ["int_nvvm_discard", addrspace , "L2"]>.ret)1010 addr:$addr, discard_size_imm:$size)]>,1011 Requires<[hasPTX<74>, hasSM<80>]>;1012 1013def DISCARD_L2 : DISCARD_L2_INTRS<"">;1014def DISCARD_GLOBAL_L2 : DISCARD_L2_INTRS<"global">;1015 1016//-----------------------------------1017// MBarrier Functions1018//-----------------------------------1019 1020let Predicates = [hasPTX<70>, hasSM<80>] in {1021 class MBARRIER_INIT<string AddrSpace, Intrinsic Intrin> :1022 BasicNVPTXInst<(outs), (ins ADDR:$addr, B32:$count),1023 "mbarrier.init" # AddrSpace # ".b64",1024 [(Intrin addr:$addr, i32:$count)]>;1025 1026 def MBARRIER_INIT : MBARRIER_INIT<"", int_nvvm_mbarrier_init>;1027 def MBARRIER_INIT_SHARED : MBARRIER_INIT<".shared",1028 int_nvvm_mbarrier_init_shared>;1029 1030 class MBARRIER_INVAL<string AddrSpace, Intrinsic Intrin> :1031 BasicNVPTXInst<(outs), (ins ADDR:$addr),1032 "mbarrier.inval" # AddrSpace # ".b64",1033 [(Intrin addr:$addr)]>;1034 1035 def MBARRIER_INVAL : MBARRIER_INVAL<"", int_nvvm_mbarrier_inval>;1036 def MBARRIER_INVAL_SHARED : MBARRIER_INVAL<".shared",1037 int_nvvm_mbarrier_inval_shared>;1038 1039 class MBARRIER_ARRIVE<string AddrSpace, Intrinsic Intrin> :1040 BasicNVPTXInst<(outs B64:$state), (ins ADDR:$addr),1041 "mbarrier.arrive" # AddrSpace # ".b64",1042 [(set i64:$state, (Intrin addr:$addr))]>;1043 1044 def MBARRIER_ARRIVE : MBARRIER_ARRIVE<"", int_nvvm_mbarrier_arrive>;1045 def MBARRIER_ARRIVE_SHARED :1046 MBARRIER_ARRIVE<".shared", int_nvvm_mbarrier_arrive_shared>;1047 1048 class MBARRIER_ARRIVE_NOCOMPLETE<string AddrSpace, Intrinsic Intrin> :1049 BasicNVPTXInst<(outs B64:$state),1050 (ins ADDR:$addr, B32:$count),1051 "mbarrier.arrive.noComplete" # AddrSpace # ".b64",1052 [(set i64:$state, (Intrin addr:$addr, i32:$count))]>;1053 1054 def MBARRIER_ARRIVE_NOCOMPLETE :1055 MBARRIER_ARRIVE_NOCOMPLETE<"", int_nvvm_mbarrier_arrive_noComplete>;1056 def MBARRIER_ARRIVE_NOCOMPLETE_SHARED :1057 MBARRIER_ARRIVE_NOCOMPLETE<".shared", int_nvvm_mbarrier_arrive_noComplete_shared>;1058 1059 class MBARRIER_ARRIVE_DROP<string AddrSpace, Intrinsic Intrin> :1060 BasicNVPTXInst<(outs B64:$state), (ins ADDR:$addr),1061 "mbarrier.arrive_drop" # AddrSpace # ".b64",1062 [(set i64:$state, (Intrin addr:$addr))]>;1063 1064 def MBARRIER_ARRIVE_DROP :1065 MBARRIER_ARRIVE_DROP<"", int_nvvm_mbarrier_arrive_drop>;1066 def MBARRIER_ARRIVE_DROP_SHARED :1067 MBARRIER_ARRIVE_DROP<".shared", int_nvvm_mbarrier_arrive_drop_shared>;1068 1069 class MBARRIER_ARRIVE_DROP_NOCOMPLETE<string AddrSpace, Intrinsic Intrin> :1070 BasicNVPTXInst<(outs B64:$state),1071 (ins ADDR:$addr, B32:$count),1072 "mbarrier.arrive_drop.noComplete" # AddrSpace # ".b64",1073 [(set i64:$state, (Intrin addr:$addr, i32:$count))]>;1074 1075 def MBARRIER_ARRIVE_DROP_NOCOMPLETE :1076 MBARRIER_ARRIVE_DROP_NOCOMPLETE<"", int_nvvm_mbarrier_arrive_drop_noComplete>;1077 def MBARRIER_ARRIVE_DROP_NOCOMPLETE_SHARED :1078 MBARRIER_ARRIVE_DROP_NOCOMPLETE<".shared",1079 int_nvvm_mbarrier_arrive_drop_noComplete_shared>;1080 1081 class MBARRIER_TEST_WAIT<string AddrSpace, Intrinsic Intrin> :1082 BasicNVPTXInst<(outs B1:$res), (ins ADDR:$addr, B64:$state),1083 "mbarrier.test_wait" # AddrSpace # ".b64",1084 [(set i1:$res, (Intrin addr:$addr, i64:$state))]>;1085 1086 def MBARRIER_TEST_WAIT :1087 MBARRIER_TEST_WAIT<"", int_nvvm_mbarrier_test_wait>;1088 def MBARRIER_TEST_WAIT_SHARED :1089 MBARRIER_TEST_WAIT<".shared", int_nvvm_mbarrier_test_wait_shared>;1090 1091 def MBARRIER_PENDING_COUNT :1092 BasicNVPTXInst<(outs B32:$res), (ins B64:$state),1093 "mbarrier.pending_count.b64",1094 [(set i32:$res, (int_nvvm_mbarrier_pending_count i64:$state))]>;1095}1096 1097class MBAR_UTIL<string op, string scope,1098 string space = "", string sem = "",1099 bit tl = 0, bit parity = 0> {1100 // The mbarrier instructions in PTX ISA are of the general form:1101 // mbarrier.op.semantics.scope.space.b64 arg1, arg2 ...1102 // where:1103 // op -> arrive, expect_tx, complete_tx, arrive.expect_tx etc.1104 // semantics -> acquire, release, relaxed (default depends on the op)1105 // scope -> cta or cluster (default is cta-scope)1106 // space -> shared::cta or shared::cluster (default is shared::cta)1107 //1108 // The 'semantics' and 'scope' go together. If one is specified,1109 // then the other _must_ be specified. For example:1110 // (A) mbarrier.arrive <args> (valid, release and cta are default)1111 // (B) mbarrier.arrive.release.cta <args> (valid, sem/scope mentioned explicitly)1112 // (C) mbarrier.arrive.release <args> (invalid, needs scope)1113 // (D) mbarrier.arrive.cta <args> (invalid, needs order)1114 //1115 // Wherever possible, we prefer form (A) to (B) since it is available1116 // from early PTX versions. In most cases, explicitly specifying the1117 // scope requires a later version of PTX.1118 string _scope_asm = !cond(1119 !eq(scope, "scope_cluster") : "cluster",1120 !eq(scope, "scope_cta") : !if(!empty(sem), "", "cta"),1121 true : scope);1122 string _space_asm = !cond(1123 !eq(space, "space_cta") : "shared",1124 !eq(space, "space_cluster") : "shared::cluster",1125 true : space);1126 1127 string _parity = !if(parity, "parity", "");1128 string asm_str = StrJoin<".", ["mbarrier", op, _parity,1129 sem, _scope_asm, _space_asm, "b64"]>.ret;1130 1131 string _intr_suffix = StrJoin<"_", [!subst(".", "_", op), _parity,1132 !if(tl, "tl", ""),1133 sem, scope, space]>.ret;1134 string intr_name = "int_nvvm_mbarrier_" # _intr_suffix;1135 1136 // Predicate checks:1137 // These are used only for the "test_wait/try_wait" variants as they1138 // have evolved since sm80 and are complex. The predicates for the1139 // remaining instructions are straightforward and have already been1140 // applied directly.1141 Predicate _sm_pred = !cond(!or(1142 !eq(op, "try_wait"),1143 !eq(scope, "scope_cluster"),1144 !eq(sem, "relaxed")) : hasSM<90>,1145 true : hasSM<80>);1146 Predicate _ptx_pred = !cond(1147 !eq(sem, "relaxed") : hasPTX<86>,1148 !ne(_scope_asm, "") : hasPTX<80>,1149 !eq(op, "try_wait") : hasPTX<78>,1150 parity : hasPTX<71>,1151 true : hasPTX<70>);1152 list<Predicate> preds = [_ptx_pred, _sm_pred];1153}1154 1155foreach op = ["expect_tx", "complete_tx"] in {1156 foreach scope = ["scope_cta", "scope_cluster"] in {1157 foreach space = ["space_cta", "space_cluster"] in {1158 defvar intr = !cast<Intrinsic>(MBAR_UTIL<op, scope, space>.intr_name);1159 defvar suffix = StrJoin<"_", [op, scope, space]>.ret;1160 def mbar_ # suffix : BasicNVPTXInst<(outs), (ins ADDR:$addr, B32:$tx_count),1161 MBAR_UTIL<op, scope, space, "relaxed">.asm_str,1162 [(intr addr:$addr, i32:$tx_count)]>,1163 Requires<[hasPTX<80>, hasSM<90>]>;1164 } // space1165 } // scope1166} // op1167 1168multiclass MBAR_ARR_INTR<string op, string scope, string sem,1169 list<Predicate> pred = []> {1170 // When either of sem or scope is non-default, both have to1171 // be explicitly specified. So, explicitly state that1172 // sem is `release` when scope is `cluster`.1173 defvar asm_sem = !if(!and(!empty(sem), !eq(scope, "scope_cluster")),1174 "release", sem);1175 1176 defvar asm_cta = MBAR_UTIL<op, scope, "space_cta", asm_sem>.asm_str;1177 defvar intr_cta = !cast<Intrinsic>(MBAR_UTIL<op, scope,1178 "space_cta", sem>.intr_name);1179 1180 defvar asm_cluster = MBAR_UTIL<op, scope, "space_cluster", asm_sem>.asm_str;1181 defvar intr_cluster = !cast<Intrinsic>(MBAR_UTIL<op, scope,1182 "space_cluster", sem>.intr_name);1183 1184 def _CTA : NVPTXInst<(outs B64:$state),1185 (ins ADDR:$addr, B32:$tx_count),1186 asm_cta # " $state, [$addr], $tx_count;",1187 [(set i64:$state, (intr_cta addr:$addr, i32:$tx_count))]>,1188 Requires<pred>;1189 def _CLUSTER : NVPTXInst<(outs),1190 (ins ADDR:$addr, B32:$tx_count),1191 asm_cluster # " _, [$addr], $tx_count;",1192 [(intr_cluster addr:$addr, i32:$tx_count)]>,1193 Requires<pred>;1194}1195foreach op = ["arrive", "arrive.expect_tx",1196 "arrive_drop", "arrive_drop.expect_tx"] in {1197 foreach scope = ["scope_cta", "scope_cluster"] in {1198 defvar suffix = !subst(".", "_", op) # scope;1199 defm mbar_ # suffix # _release : MBAR_ARR_INTR<op, scope, "", [hasPTX<80>, hasSM<90>]>;1200 defm mbar_ # suffix # _relaxed : MBAR_ARR_INTR<op, scope, "relaxed", [hasPTX<86>, hasSM<90>]>;1201 } // scope1202} // op1203 1204multiclass MBAR_WAIT_INTR<string op, string scope, string sem, bit time_limit> {1205 // When either of sem or scope is non-default, both have to1206 // be explicitly specified. So, explicitly state that the1207 // semantics is `acquire` when the scope is `cluster`.1208 defvar asm_sem = !if(!and(!empty(sem), !eq(scope, "scope_cluster")),1209 "acquire", sem);1210 1211 defvar asm_parity = MBAR_UTIL<op, scope, "space_cta", asm_sem,1212 time_limit, 1>.asm_str;1213 defvar pred_parity = MBAR_UTIL<op, scope, "space_cta", asm_sem,1214 time_limit, 1>.preds;1215 defvar intr_parity = !cast<Intrinsic>(MBAR_UTIL<op, scope, "space_cta",1216 sem, time_limit, 1>.intr_name);1217 1218 defvar asm_state = MBAR_UTIL<op, scope, "space_cta", asm_sem,1219 time_limit>.asm_str;1220 defvar pred_state = MBAR_UTIL<op, scope, "space_cta", asm_sem,1221 time_limit>.preds;1222 defvar intr_state = !cast<Intrinsic>(MBAR_UTIL<op, scope, "space_cta",1223 sem, time_limit>.intr_name);1224 1225 defvar ins_tl_dag = !if(time_limit, (ins B32:$tl), (ins));1226 defvar tl_suffix = !if(time_limit, ", $tl;", ";");1227 defvar intr_state_dag = !con((intr_state addr:$addr, i64:$state),1228 !if(time_limit, (intr_state i32:$tl), (intr_state)));1229 defvar intr_parity_dag = !con((intr_parity addr:$addr, i32:$phase),1230 !if(time_limit, (intr_parity i32:$tl), (intr_parity)));1231 1232 def _STATE : NVPTXInst<(outs B1:$res), !con((ins ADDR:$addr, B64:$state), ins_tl_dag),1233 asm_state # " $res, [$addr], $state" # tl_suffix,1234 [(set i1:$res, intr_state_dag)]>,1235 Requires<pred_state>;1236 def _PARITY : NVPTXInst<(outs B1:$res), !con((ins ADDR:$addr, B32:$phase), ins_tl_dag),1237 asm_parity # " $res, [$addr], $phase" # tl_suffix,1238 [(set i1:$res, intr_parity_dag)]>,1239 Requires<pred_parity>;1240}1241foreach op = ["test_wait", "try_wait"] in {1242 foreach scope = ["scope_cta", "scope_cluster"] in {1243 foreach time_limit = !if(!eq(op, "try_wait"), [true, false], [false]) in {1244 defvar suffix = StrJoin<"_", [op, scope, !if(time_limit, "tl", "")]>.ret;1245 defm mbar_ # suffix # "_acquire" : MBAR_WAIT_INTR<op, scope, "", time_limit>;1246 defm mbar_ # suffix # "_relaxed" : MBAR_WAIT_INTR<op, scope, "relaxed", time_limit>;1247 } // time_limit1248 } // scope1249} // op1250 1251//-----------------------------------1252// Math Functions1253//-----------------------------------1254 1255// Map min(1.0, max(0.0, x)) to sat(x)1256// Note that max(0.0, min(x, 1.0)) cannot be mapped to sat(x) because when x is1257// NaN1258// max(0.0, min(x, 1.0)) is 1.0 while sat(x) is 0.1259// Same story for fmax, fmin.1260 1261def : Pat<(int_nvvm_fmin_f fpimm_1,1262 (int_nvvm_fmax_f fpimm_0, f32:$a)),1263 (CVT_f32_f32 $a, CvtSAT)>;1264def : Pat<(int_nvvm_fmin_f fpimm_1,1265 (int_nvvm_fmax_f f32:$a, fpimm_0)),1266 (CVT_f32_f32 $a, CvtSAT)>;1267def : Pat<(int_nvvm_fmin_f1268 (int_nvvm_fmax_f fpimm_0, f32:$a), fpimm_1),1269 (CVT_f32_f32 $a, CvtSAT)>;1270def : Pat<(int_nvvm_fmin_f1271 (int_nvvm_fmax_f f32:$a, fpimm_0), fpimm_1),1272 (CVT_f32_f32 $a, CvtSAT)>;1273 1274def : Pat<(int_nvvm_fmin_d fpimm_1,1275 (int_nvvm_fmax_d fpimm_0, f64:$a)),1276 (CVT_f64_f64 $a, CvtSAT)>;1277def : Pat<(int_nvvm_fmin_d fpimm_1,1278 (int_nvvm_fmax_d f64:$a, fpimm_0)),1279 (CVT_f64_f64 $a, CvtSAT)>;1280def : Pat<(int_nvvm_fmin_d1281 (int_nvvm_fmax_d fpimm_0, f64:$a), fpimm_1),1282 (CVT_f64_f64 $a, CvtSAT)>;1283def : Pat<(int_nvvm_fmin_d1284 (int_nvvm_fmax_d f64:$a, fpimm_0), fpimm_1),1285 (CVT_f64_f64 $a, CvtSAT)>;1286 1287 1288// We need a full string for OpcStr here because we need to deal with case like1289// INT_PTX_RECIP.1290class F_MATH_1<string OpcStr, RegTyInfo dst, RegTyInfo src, Intrinsic IntOP,1291 list<Predicate> Preds = []>1292 : BasicNVPTXInst<(outs dst.RC:$dst),1293 (ins src.RC:$src0),1294 OpcStr,1295 [(set dst.Ty:$dst, (IntOP src.Ty:$src0))]>,1296 Requires<Preds>;1297 1298// We need a full string for OpcStr here because we need to deal with the case1299// like INT_PTX_NATIVE_POWR_F.1300class F_MATH_2<string OpcStr, NVPTXRegClass t_regclass,1301 NVPTXRegClass s0_regclass, NVPTXRegClass s1_regclass, Intrinsic IntOP,1302 list<Predicate> Preds = []>1303 : BasicNVPTXInst<(outs t_regclass:$dst),1304 (ins s0_regclass:$src0, s1_regclass:$src1),1305 OpcStr,1306 [(set t_regclass:$dst, (IntOP s0_regclass:$src0, s1_regclass:$src1))]>,1307 Requires<Preds>;1308 1309class F_MATH_3<string OpcStr, NVPTXRegClass t_regclass,1310 NVPTXRegClass s0_regclass, NVPTXRegClass s1_regclass,1311 NVPTXRegClass s2_regclass, Intrinsic IntOP, list<Predicate> Preds = []>1312 : BasicNVPTXInst<(outs t_regclass:$dst),1313 (ins s0_regclass:$src0, s1_regclass:$src1, s2_regclass:$src2),1314 OpcStr,1315 [(set t_regclass:$dst,1316 (IntOP s0_regclass:$src0, s1_regclass:$src1, s2_regclass:$src2))]>,1317 Requires<Preds>;1318 1319//1320// MISC1321//1322 1323def INT_NVVM_NANOSLEEP_I : BasicNVPTXInst<(outs), (ins i32imm:$i), "nanosleep.u32",1324 [(int_nvvm_nanosleep imm:$i)]>,1325 Requires<[hasPTX<63>, hasSM<70>]>;1326def INT_NVVM_NANOSLEEP_R : BasicNVPTXInst<(outs), (ins B32:$i), "nanosleep.u32",1327 [(int_nvvm_nanosleep i32:$i)]>,1328 Requires<[hasPTX<63>, hasSM<70>]>;1329 1330let hasSideEffects = 1 in {1331// Performance Monitor events1332def INT_PM_EVENT_MASK : BasicNVPTXInst<(outs),1333 (ins i16imm:$mask),1334 "pmevent.mask",1335 [(int_nvvm_pm_event_mask timm:$mask)]>,1336 Requires<[hasSM<20>, hasPTX<30>]>;1337} // hasSideEffects1338 1339//1340// Min Max1341//1342 1343def : Pat<(int_nvvm_fmin_f f32:$a, f32:$b), (MIN_f32_rr $a, $b, NoFTZ)>;1344def : Pat<(int_nvvm_fmin_ftz_f f32:$a, f32:$b), (MIN_f32_rr $a, $b, FTZ)>;1345 1346let Predicates = [hasPTX<70>, hasSM<80>] in {1347 def : Pat<(int_nvvm_fmin_nan_f f32:$a, f32:$b), (MIN_NAN_f32_rr $a, $b, NoFTZ)>;1348 def : Pat<(int_nvvm_fmin_ftz_nan_f f32:$a, f32:$b), (MIN_NAN_f32_rr $a, $b, FTZ)>;1349}1350 1351def INT_NVVM_FMIN_XORSIGN_ABS_F :1352 F_MATH_2<"min.xorsign.abs.f32", B32, B32, B32, int_nvvm_fmin_xorsign_abs_f,1353 [hasPTX<72>, hasSM<86>]>;1354def INT_NVVM_FMIN_FTZ_XORSIGN_ABS_F :1355 F_MATH_2<"min.ftz.xorsign.abs.f32", B32, B32, B32, int_nvvm_fmin_ftz_xorsign_abs_f,1356 [hasPTX<72>, hasSM<86>]>;1357def INT_NVVM_FMIN_NAN_XORSIGN_ABS_F :1358 F_MATH_2<"min.NaN.xorsign.abs.f32", B32, B32, B32, int_nvvm_fmin_nan_xorsign_abs_f,1359 [hasPTX<72>, hasSM<86>]>;1360def INT_NVVM_FMIN_FTZ_NAN_XORSIGN_ABS_F :1361 F_MATH_2<"min.ftz.NaN.xorsign.abs.f32", B32, B32, B32, int_nvvm_fmin_ftz_nan_xorsign_abs_f,1362 [hasPTX<72>, hasSM<86>]>;1363 1364 1365def : Pat<(int_nvvm_fmax_f f32:$a, f32:$b), (MAX_f32_rr $a, $b, NoFTZ)>;1366def : Pat<(int_nvvm_fmax_ftz_f f32:$a, f32:$b), (MAX_f32_rr $a, $b, FTZ)>;1367 1368let Predicates = [hasPTX<70>, hasSM<80>] in {1369 def : Pat<(int_nvvm_fmax_nan_f f32:$a, f32:$b), (MAX_NAN_f32_rr $a, $b, NoFTZ)>;1370 def : Pat<(int_nvvm_fmax_ftz_nan_f f32:$a, f32:$b), (MAX_NAN_f32_rr $a, $b, FTZ)>;1371}1372 1373def INT_NVVM_FMAX_XORSIGN_ABS_F :1374 F_MATH_2<"max.xorsign.abs.f32", B32, B32, B32, int_nvvm_fmax_xorsign_abs_f,1375 [hasPTX<72>, hasSM<86>]>;1376def INT_NVVM_FMAX_FTZ_XORSIGN_ABS_F :1377 F_MATH_2<"max.ftz.xorsign.abs.f32", B32, B32, B32, int_nvvm_fmax_ftz_xorsign_abs_f,1378 [hasPTX<72>, hasSM<86>]>;1379def INT_NVVM_FMAX_NAN_XORSIGN_ABS_F :1380 F_MATH_2<"max.NaN.xorsign.abs.f32", B32, B32, B32, int_nvvm_fmax_nan_xorsign_abs_f,1381 [hasPTX<72>, hasSM<86>]>;1382def INT_NVVM_FMAX_FTZ_NAN_XORSIGN_ABS_F :1383 F_MATH_2<"max.ftz.NaN.xorsign.abs.f32", B32, B32, B32, int_nvvm_fmax_ftz_nan_xorsign_abs_f,1384 [hasPTX<72>, hasSM<86>]>;1385 1386def : Pat<(int_nvvm_fmin_d f64:$a, f64:$b), (MIN_f64_rr $a, $b)>;1387def : Pat<(int_nvvm_fmax_d f64:$a, f64:$b), (MAX_f64_rr $a, $b)>;1388 1389//1390// Min Max f16, f16x2, bf16, bf16x21391//1392 1393class MIN_MAX_TUPLE<string V, Intrinsic I, NVPTXRegClass RC,1394 list<Predicate> Preds = [hasPTX<70>, hasSM<80>]> {1395 string Variant = V;1396 Intrinsic Intr = I;1397 NVPTXRegClass RegClass = RC;1398 list<Predicate> Predicates = Preds;1399}1400 1401multiclass MIN_MAX<string IntName> {1402 foreach P = [1403 MIN_MAX_TUPLE<"_f16", !if(!eq(IntName, "min"), int_nvvm_fmin_f16,1404 int_nvvm_fmax_f16), B16>,1405 MIN_MAX_TUPLE<"_ftz_f16", !if(!eq(IntName, "min"), int_nvvm_fmin_ftz_f16,1406 int_nvvm_fmax_ftz_f16), B16>,1407 MIN_MAX_TUPLE<"_NaN_f16", !if(!eq(IntName, "min"), int_nvvm_fmin_nan_f16,1408 int_nvvm_fmax_nan_f16), B16>,1409 MIN_MAX_TUPLE<"_ftz_NaN_f16", !if(!eq(IntName, "min"),1410 int_nvvm_fmin_ftz_nan_f16, int_nvvm_fmax_ftz_nan_f16), B16>,1411 MIN_MAX_TUPLE<"_xorsign_abs_f16", !if(!eq(IntName, "min"),1412 int_nvvm_fmin_xorsign_abs_f16, int_nvvm_fmax_xorsign_abs_f16),1413 B16, [hasPTX<72>, hasSM<86>]>,1414 MIN_MAX_TUPLE<"_ftz_xorsign_abs_f16", !if(!eq(IntName, "min"),1415 int_nvvm_fmin_ftz_xorsign_abs_f16, int_nvvm_fmax_ftz_xorsign_abs_f16),1416 B16, [hasPTX<72>, hasSM<86>]>,1417 MIN_MAX_TUPLE<"_NaN_xorsign_abs_f16", !if(!eq(IntName, "min"),1418 int_nvvm_fmin_nan_xorsign_abs_f16, int_nvvm_fmax_nan_xorsign_abs_f16),1419 B16, [hasPTX<72>, hasSM<86>]>,1420 MIN_MAX_TUPLE<"_ftz_NaN_xorsign_abs_f16", !if(!eq(IntName, "min"),1421 int_nvvm_fmin_ftz_nan_xorsign_abs_f16,1422 int_nvvm_fmax_ftz_nan_xorsign_abs_f16), B16, [hasPTX<72>, hasSM<86>]>,1423 MIN_MAX_TUPLE<"_f16x2", !if(!eq(IntName, "min"), int_nvvm_fmin_f16x2,1424 int_nvvm_fmax_f16x2), B32>,1425 MIN_MAX_TUPLE<"_ftz_f16x2", !if(!eq(IntName, "min"),1426 int_nvvm_fmin_ftz_f16x2, int_nvvm_fmax_ftz_f16x2), B32>,1427 MIN_MAX_TUPLE<"_NaN_f16x2", !if(!eq(IntName, "min"),1428 int_nvvm_fmin_nan_f16x2, int_nvvm_fmax_nan_f16x2), B32>,1429 MIN_MAX_TUPLE<"_ftz_NaN_f16x2", !if(!eq(IntName, "min"),1430 int_nvvm_fmin_ftz_nan_f16x2, int_nvvm_fmax_ftz_nan_f16x2), B32>,1431 MIN_MAX_TUPLE<"_xorsign_abs_f16x2", !if(!eq(IntName, "min"),1432 int_nvvm_fmin_xorsign_abs_f16x2, int_nvvm_fmax_xorsign_abs_f16x2),1433 B32, [hasPTX<72>, hasSM<86>]>,1434 MIN_MAX_TUPLE<"_ftz_xorsign_abs_f16x2", !if(!eq(IntName, "min"),1435 int_nvvm_fmin_ftz_xorsign_abs_f16x2, int_nvvm_fmax_ftz_xorsign_abs_f16x2),1436 B32, [hasPTX<72>, hasSM<86>]>,1437 MIN_MAX_TUPLE<"_NaN_xorsign_abs_f16x2", !if(!eq(IntName, "min"),1438 int_nvvm_fmin_nan_xorsign_abs_f16x2, int_nvvm_fmax_nan_xorsign_abs_f16x2),1439 B32, [hasPTX<72>, hasSM<86>]>,1440 MIN_MAX_TUPLE<"_ftz_NaN_xorsign_abs_f16x2", !if(!eq(IntName, "min"),1441 int_nvvm_fmin_ftz_nan_xorsign_abs_f16x2,1442 int_nvvm_fmax_ftz_nan_xorsign_abs_f16x2),1443 B32, [hasPTX<72>, hasSM<86>]>,1444 MIN_MAX_TUPLE<"_bf16", !if(!eq(IntName, "min"),1445 int_nvvm_fmin_bf16, int_nvvm_fmax_bf16), B16>,1446 MIN_MAX_TUPLE<"_NaN_bf16", !if(!eq(IntName, "min"), int_nvvm_fmin_nan_bf16,1447 int_nvvm_fmax_nan_bf16), B16>,1448 MIN_MAX_TUPLE<"_xorsign_abs_bf16", !if(!eq(IntName, "min"),1449 int_nvvm_fmin_xorsign_abs_bf16, int_nvvm_fmax_xorsign_abs_bf16),1450 B16, [hasPTX<72>, hasSM<86>]>,1451 MIN_MAX_TUPLE<"_NaN_xorsign_abs_bf16", !if(!eq(IntName, "min"),1452 int_nvvm_fmin_nan_xorsign_abs_bf16, int_nvvm_fmax_nan_xorsign_abs_bf16),1453 B16, [hasPTX<72>, hasSM<86>]>,1454 MIN_MAX_TUPLE<"_bf16x2", !if(!eq(IntName, "min"), int_nvvm_fmin_bf16x2,1455 int_nvvm_fmax_bf16x2), B32>,1456 MIN_MAX_TUPLE<"_NaN_bf16x2", !if(!eq(IntName, "min"),1457 int_nvvm_fmin_nan_bf16x2, int_nvvm_fmax_nan_bf16x2), B32>,1458 MIN_MAX_TUPLE<"_xorsign_abs_bf16x2", !if(!eq(IntName, "min"),1459 int_nvvm_fmin_xorsign_abs_bf16x2, int_nvvm_fmax_xorsign_abs_bf16x2),1460 B32, [hasPTX<72>, hasSM<86>]>,1461 MIN_MAX_TUPLE<"_NaN_xorsign_abs_bf16x2", !if(!eq(IntName, "min"),1462 int_nvvm_fmin_nan_xorsign_abs_bf16x2,1463 int_nvvm_fmax_nan_xorsign_abs_bf16x2),1464 B32, [hasPTX<72>, hasSM<86>]>] in {1465 def P.Variant : F_MATH_2<!strconcat(1466 IntName, !subst("_", ".", P.Variant)),1467 P.RegClass, P.RegClass, P.RegClass, P.Intr, P.Predicates>;1468 }1469}1470 1471defm INT_NVVM_FMIN : MIN_MAX<"min">;1472defm INT_NVVM_FMAN : MIN_MAX<"max">;1473 1474//1475// Multiplication1476//1477 1478def : Pat<(int_nvvm_mulhi_s i16:$a, i16:$b), (MUL_HI_S16rr $a, $b)>;1479def : Pat<(int_nvvm_mulhi_us i16:$a, i16:$b), (MUL_HI_U16rr $a, $b)>;1480def : Pat<(int_nvvm_mulhi_i i32:$a, i32:$b), (MUL_HI_S32rr $a, $b)>;1481def : Pat<(int_nvvm_mulhi_ui i32:$a, i32:$b), (MUL_HI_U32rr $a, $b)>;1482def : Pat<(int_nvvm_mulhi_ll i64:$a, i64:$b), (MUL_HI_S64rr $a, $b)>;1483def : Pat<(int_nvvm_mulhi_ull i64:$a, i64:$b), (MUL_HI_U64rr $a, $b)>;1484 1485def INT_NVVM_MUL_RN_FTZ_F : F_MATH_2<"mul.rn.ftz.f32", B32, B32, B32, int_nvvm_mul_rn_ftz_f>;1486def INT_NVVM_MUL_RN_F : F_MATH_2<"mul.rn.f32", B32, B32, B32, int_nvvm_mul_rn_f>;1487def INT_NVVM_MUL_RZ_FTZ_F : F_MATH_2<"mul.rz.ftz.f32", B32, B32, B32, int_nvvm_mul_rz_ftz_f>;1488def INT_NVVM_MUL_RZ_F : F_MATH_2<"mul.rz.f32", B32, B32, B32, int_nvvm_mul_rz_f>;1489def INT_NVVM_MUL_RM_FTZ_F : F_MATH_2<"mul.rm.ftz.f32", B32, B32, B32, int_nvvm_mul_rm_ftz_f>;1490def INT_NVVM_MUL_RM_F : F_MATH_2<"mul.rm.f32", B32, B32, B32, int_nvvm_mul_rm_f>;1491def INT_NVVM_MUL_RP_FTZ_F : F_MATH_2<"mul.rp.ftz.f32", B32, B32, B32, int_nvvm_mul_rp_ftz_f>;1492def INT_NVVM_MUL_RP_F : F_MATH_2<"mul.rp.f32", B32, B32, B32, int_nvvm_mul_rp_f>;1493 1494def INT_NVVM_MUL_RN_D : F_MATH_2<"mul.rn.f64", B64, B64, B64, int_nvvm_mul_rn_d>;1495def INT_NVVM_MUL_RZ_D : F_MATH_2<"mul.rz.f64", B64, B64, B64, int_nvvm_mul_rz_d>;1496def INT_NVVM_MUL_RM_D : F_MATH_2<"mul.rm.f64", B64, B64, B64, int_nvvm_mul_rm_d>;1497def INT_NVVM_MUL_RP_D : F_MATH_2<"mul.rp.f64", B64, B64, B64, int_nvvm_mul_rp_d>;1498 1499def INT_NVVM_MUL24_I : F_MATH_2<"mul24.lo.s32", B32, B32, B32, int_nvvm_mul24_i>;1500def INT_NVVM_MUL24_UI : F_MATH_2<"mul24.lo.u32", B32, B32, B32, int_nvvm_mul24_ui>;1501 1502//1503// Div1504//1505 1506def : Pat<(int_nvvm_div_approx_ftz_f f32:$a, f32:$b), (DIV_APPROX_F32_rr $a, $b, FTZ)>;1507def : Pat<(int_nvvm_div_approx_f f32:$a, f32:$b), (DIV_APPROX_F32_rr $a, $b, NoFTZ)>;1508 1509def INT_NVVM_DIV_RN_FTZ_F : F_MATH_2<"div.rn.ftz.f32", B32, B32, B32, int_nvvm_div_rn_ftz_f>;1510def INT_NVVM_DIV_RN_F : F_MATH_2<"div.rn.f32", B32, B32, B32, int_nvvm_div_rn_f>;1511def INT_NVVM_DIV_RZ_FTZ_F : F_MATH_2<"div.rz.ftz.f32", B32, B32, B32, int_nvvm_div_rz_ftz_f>;1512def INT_NVVM_DIV_RZ_F : F_MATH_2<"div.rz.f32", B32, B32, B32, int_nvvm_div_rz_f>;1513def INT_NVVM_DIV_RM_FTZ_F : F_MATH_2<"div.rm.ftz.f32", B32, B32, B32, int_nvvm_div_rm_ftz_f>;1514def INT_NVVM_DIV_RM_F : F_MATH_2<"div.rm.f32", B32, B32, B32, int_nvvm_div_rm_f>;1515def INT_NVVM_DIV_RP_FTZ_F : F_MATH_2<"div.rp.ftz.f32", B32, B32, B32, int_nvvm_div_rp_ftz_f>;1516def INT_NVVM_DIV_RP_F : F_MATH_2<"div.rp.f32", B32, B32, B32, int_nvvm_div_rp_f>;1517 1518def INT_NVVM_DIV_RN_D : F_MATH_2<"div.rn.f64", B64, B64, B64, int_nvvm_div_rn_d>;1519def INT_NVVM_DIV_RZ_D : F_MATH_2<"div.rz.f64", B64, B64, B64, int_nvvm_div_rz_d>;1520def INT_NVVM_DIV_RM_D : F_MATH_2<"div.rm.f64", B64, B64, B64, int_nvvm_div_rm_d>;1521def INT_NVVM_DIV_RP_D : F_MATH_2<"div.rp.f64", B64, B64, B64, int_nvvm_div_rp_d>;1522 1523def : Pat<(int_nvvm_div_full f32:$a, f32:$b), (FDIV32rr $a, $b, NoFTZ)>;1524def : Pat<(int_nvvm_div_full f32:$a, fpimm:$b), (FDIV32ri $a, f32imm:$b, NoFTZ)>;1525def : Pat<(int_nvvm_div_full_ftz f32:$a, f32:$b), (FDIV32rr $a, $b, FTZ)>;1526def : Pat<(int_nvvm_div_full_ftz f32:$a, fpimm:$b), (FDIV32ri $a, f32imm:$b, FTZ)>;1527 1528//1529// Sad1530//1531 1532def INT_NVVM_SAD_S : F_MATH_3<"sad.s16", B16, B16, B16, B16, int_nvvm_sad_s>;1533def INT_NVVM_SAD_US : F_MATH_3<"sad.u16", B16, B16, B16, B16, int_nvvm_sad_us>;1534def INT_NVVM_SAD_I : F_MATH_3<"sad.s32", B32, B32, B32, B32, int_nvvm_sad_i>;1535def INT_NVVM_SAD_UI : F_MATH_3<"sad.u32", B32, B32, B32, B32, int_nvvm_sad_ui>;1536def INT_NVVM_SAD_LL : F_MATH_3<"sad.s64", B64, B64, B64, B64, int_nvvm_sad_ll>;1537def INT_NVVM_SAD_ULL : F_MATH_3<"sad.u64", B64, B64, B64, B64, int_nvvm_sad_ull>;1538 1539//1540// Floor Ceil1541//1542 1543def : Pat<(int_nvvm_floor_ftz_f f32:$a), (CVT_f32_f32 $a, CvtRMI_FTZ)>;1544def : Pat<(int_nvvm_floor_f f32:$a), (CVT_f32_f32 $a, CvtRMI)>;1545def : Pat<(int_nvvm_floor_d f64:$a), (CVT_f64_f64 $a, CvtRMI)>;1546 1547def : Pat<(int_nvvm_ceil_ftz_f f32:$a), (CVT_f32_f32 $a, CvtRPI_FTZ)>;1548def : Pat<(int_nvvm_ceil_f f32:$a), (CVT_f32_f32 $a, CvtRPI)>;1549def : Pat<(int_nvvm_ceil_d f64:$a), (CVT_f64_f64 $a, CvtRPI)>;1550 1551//1552// Abs1553//1554 1555multiclass F_ABS<string suffix, RegTyInfo RT, bit support_ftz, list<Predicate> preds = []> {1556 def "" : F_MATH_1<"abs." # suffix, RT, RT, int_nvvm_fabs, preds>;1557 if support_ftz then1558 def _FTZ : F_MATH_1<"abs.ftz." # suffix, RT, RT, int_nvvm_fabs_ftz, preds>;1559}1560 1561defm ABS_F16 : F_ABS<"f16", F16RT, support_ftz = true, preds = [hasPTX<65>, hasSM<53>]>;1562defm ABS_F16X2 : F_ABS<"f16x2", F16X2RT, support_ftz = true, preds = [hasPTX<65>, hasSM<53>]>;1563 1564defm ABS_BF16 : F_ABS<"bf16", BF16RT, support_ftz = false, preds = [hasPTX<70>, hasSM<80>]>;1565defm ABS_BF16X2 : F_ABS<"bf16x2", BF16X2RT, support_ftz = false, preds = [hasPTX<70>, hasSM<80>]>;1566 1567defm ABS_F32 : F_ABS<"f32", F32RT, support_ftz = true>;1568defm ABS_F64 : F_ABS<"f64", F64RT, support_ftz = false>;1569 1570//1571// copysign1572//1573 1574def fcopysign_nvptx : SDNode<"NVPTXISD::FCOPYSIGN", SDTFPBinOp>;1575 1576foreach t = [F32RT, F64RT] in1577 def COPYSIGN_ # t :1578 BasicNVPTXInst<(outs t.RC:$dst), (ins t.RC:$src0, t.RC:$src1),1579 "copysign." # t.PtxType,1580 [(set t.Ty:$dst, (fcopysign_nvptx t.Ty:$src1, t.Ty:$src0))]>;1581 1582//1583// Neg bf16, bf16x21584//1585 1586def INT_NVVM_NEG_BF16 : F_MATH_1<"neg.bf16", BF16RT,1587 BF16RT, int_nvvm_neg_bf16, [hasPTX<70>, hasSM<80>]>;1588def INT_NVVM_NEG_BF16X2 : F_MATH_1<"neg.bf16x2", BF16X2RT,1589 BF16X2RT, int_nvvm_neg_bf16x2, [hasPTX<70>, hasSM<80>]>;1590 1591//1592// Round1593//1594 1595def : Pat<(int_nvvm_round_ftz_f f32:$a), (CVT_f32_f32 $a, CvtRNI_FTZ)>;1596def : Pat<(int_nvvm_round_f f32:$a), (CVT_f32_f32 $a, CvtRNI)>;1597def : Pat<(int_nvvm_round_d f64:$a), (CVT_f64_f64 $a, CvtRNI)>;1598 1599//1600// Trunc1601//1602 1603def : Pat<(int_nvvm_trunc_ftz_f f32:$a), (CVT_f32_f32 $a, CvtRZI_FTZ)>;1604def : Pat<(int_nvvm_trunc_f f32:$a), (CVT_f32_f32 $a, CvtRZI)>;1605def : Pat<(int_nvvm_trunc_d f64:$a), (CVT_f64_f64 $a, CvtRZI)>;1606 1607//1608// Saturate1609//1610 1611def : Pat<(int_nvvm_saturate_ftz_f f32:$a), (CVT_f32_f32 $a, CvtSAT_FTZ)>;1612def : Pat<(int_nvvm_saturate_f f32:$a), (CVT_f32_f32 $a, CvtSAT)>;1613def : Pat<(int_nvvm_saturate_d f64:$a), (CVT_f64_f64 $a, CvtSAT)>;1614 1615//1616// Exp2 Log21617//1618 1619def : Pat<(f32 (int_nvvm_ex2_approx_ftz f32:$a)), (EX2_APPROX_f32 $a, FTZ)>;1620def : Pat<(f32 (int_nvvm_ex2_approx f32:$a)), (EX2_APPROX_f32 $a, NoFTZ)>;1621 1622let Predicates = [hasPTX<70>, hasSM<75>] in {1623 def : Pat<(f16 (int_nvvm_ex2_approx f16:$a)), (EX2_APPROX_f16 $a)>;1624 def : Pat<(v2f16 (int_nvvm_ex2_approx v2f16:$a)), (EX2_APPROX_f16x2 $a)>;1625}1626 1627let Predicates = [hasPTX<78>, hasSM<90>] in {1628 def : Pat<(bf16 (int_nvvm_ex2_approx_ftz bf16:$a)), (EX2_APPROX_bf16 $a)>;1629 def : Pat<(v2bf16 (int_nvvm_ex2_approx_ftz v2bf16:$a)), (EX2_APPROX_bf16x2 $a)>;1630}1631 1632def LG2_APPROX_f32 :1633 BasicFlagsNVPTXInst<(outs B32:$dst), (ins B32:$src), (ins FTZFlag:$ftz),1634 "lg2.approx$ftz.f32",1635 [(set f32:$dst, (flog2 f32:$src))]>;1636 1637def LG2_APPROX_f64 :1638 BasicNVPTXInst<(outs B64:$dst), (ins B64:$src),1639 "lg2.approx.f64",1640 [(set f64:$dst, (flog2 f64:$src))]>;1641 1642def : Pat<(int_nvvm_lg2_approx_ftz_f f32:$a), (LG2_APPROX_f32 $a, FTZ)>;1643def : Pat<(int_nvvm_lg2_approx_f f32:$a), (LG2_APPROX_f32 $a, NoFTZ)>;1644def : Pat<(int_nvvm_lg2_approx_d f64:$a), (LG2_APPROX_f64 $a)>;1645 1646//1647// Sin Cos1648//1649 1650def : Pat<(int_nvvm_sin_approx_ftz_f f32:$a), (SIN_APPROX_f32 $a, FTZ)>;1651def : Pat<(int_nvvm_sin_approx_f f32:$a), (SIN_APPROX_f32 $a, NoFTZ)>;1652def : Pat<(int_nvvm_cos_approx_ftz_f f32:$a), (COS_APPROX_f32 $a, FTZ)>;1653def : Pat<(int_nvvm_cos_approx_f f32:$a), (COS_APPROX_f32 $a, NoFTZ)>;1654 1655//1656// Fma1657//1658 1659class FMA_TUPLE<string V, Intrinsic I, NVPTXRegClass RC,1660 list<Predicate> Preds = []> {1661 string Variant = V;1662 Intrinsic Intr = I;1663 NVPTXRegClass RegClass = RC;1664 list<Predicate> Predicates = Preds;1665}1666 1667multiclass FMA_INST {1668 foreach P = [1669 FMA_TUPLE<"_rn_f64", int_nvvm_fma_rn_d, B64>,1670 FMA_TUPLE<"_rz_f64", int_nvvm_fma_rz_d, B64>,1671 FMA_TUPLE<"_rm_f64", int_nvvm_fma_rm_d, B64>,1672 FMA_TUPLE<"_rp_f64", int_nvvm_fma_rp_d, B64>,1673 1674 FMA_TUPLE<"_rn_ftz_f32", int_nvvm_fma_rn_ftz_f, B32>,1675 FMA_TUPLE<"_rn_f32", int_nvvm_fma_rn_f, B32>,1676 FMA_TUPLE<"_rz_ftz_f32", int_nvvm_fma_rz_ftz_f, B32>,1677 FMA_TUPLE<"_rz_f32", int_nvvm_fma_rz_f, B32>,1678 FMA_TUPLE<"_rm_f32", int_nvvm_fma_rm_f, B32>,1679 FMA_TUPLE<"_rm_ftz_f32", int_nvvm_fma_rm_ftz_f, B32>,1680 FMA_TUPLE<"_rp_f32", int_nvvm_fma_rp_f, B32>,1681 FMA_TUPLE<"_rp_ftz_f32", int_nvvm_fma_rp_ftz_f, B32>,1682 1683 FMA_TUPLE<"_rn_f16", int_nvvm_fma_rn_f16, B16, [hasPTX<42>, hasSM<53>]>,1684 FMA_TUPLE<"_rn_ftz_f16", int_nvvm_fma_rn_ftz_f16, B16,1685 [hasPTX<42>, hasSM<53>]>,1686 FMA_TUPLE<"_rn_sat_f16", int_nvvm_fma_rn_sat_f16, B16,1687 [hasPTX<42>, hasSM<53>]>,1688 FMA_TUPLE<"_rn_ftz_sat_f16", int_nvvm_fma_rn_ftz_sat_f16, B16,1689 [hasPTX<42>, hasSM<53>]>,1690 FMA_TUPLE<"_rn_relu_f16", int_nvvm_fma_rn_relu_f16, B16,1691 [hasPTX<70>, hasSM<80>]>,1692 FMA_TUPLE<"_rn_ftz_relu_f16", int_nvvm_fma_rn_ftz_relu_f16, B16,1693 [hasPTX<70>, hasSM<80>]>,1694 1695 FMA_TUPLE<"_rn_bf16", int_nvvm_fma_rn_bf16, B16, [hasPTX<70>, hasSM<80>]>,1696 FMA_TUPLE<"_rn_ftz_bf16", int_nvvm_fma_rn_ftz_bf16, B16,1697 [hasPTX<70>, hasSM<80>]>,1698 FMA_TUPLE<"_rn_sat_bf16", int_nvvm_fma_rn_sat_bf16, B16,1699 [hasPTX<70>, hasSM<80>]>,1700 FMA_TUPLE<"_rn_ftz_sat_bf16", int_nvvm_fma_rn_ftz_sat_bf16, B16,1701 [hasPTX<70>, hasSM<80>]>,1702 FMA_TUPLE<"_rn_relu_bf16", int_nvvm_fma_rn_relu_bf16, B16,1703 [hasPTX<70>, hasSM<80>]>,1704 FMA_TUPLE<"_rn_ftz_relu_bf16", int_nvvm_fma_rn_ftz_relu_bf16, B16,1705 [hasPTX<70>, hasSM<80>]>,1706 1707 FMA_TUPLE<"_rn_f16x2", int_nvvm_fma_rn_f16x2, B32,1708 [hasPTX<42>, hasSM<53>]>,1709 FMA_TUPLE<"_rn_ftz_f16x2", int_nvvm_fma_rn_ftz_f16x2, B32,1710 [hasPTX<42>, hasSM<53>]>,1711 FMA_TUPLE<"_rn_sat_f16x2", int_nvvm_fma_rn_sat_f16x2, B32,1712 [hasPTX<42>, hasSM<53>]>,1713 FMA_TUPLE<"_rn_ftz_sat_f16x2", int_nvvm_fma_rn_ftz_sat_f16x2,1714 B32, [hasPTX<42>, hasSM<53>]>,1715 FMA_TUPLE<"_rn_relu_f16x2", int_nvvm_fma_rn_relu_f16x2, B32,1716 [hasPTX<70>, hasSM<80>]>,1717 FMA_TUPLE<"_rn_ftz_relu_f16x2", int_nvvm_fma_rn_ftz_relu_f16x2,1718 B32, [hasPTX<70>, hasSM<80>]>,1719 FMA_TUPLE<"_rn_bf16x2", int_nvvm_fma_rn_bf16x2, B32,1720 [hasPTX<70>, hasSM<80>]>,1721 FMA_TUPLE<"_rn_relu_bf16x2", int_nvvm_fma_rn_relu_bf16x2, B32,1722 [hasPTX<70>, hasSM<80>]>1723 ] in {1724 def P.Variant :1725 F_MATH_3<!strconcat("fma", !subst("_", ".", P.Variant)),1726 P.RegClass, P.RegClass, P.RegClass, P.RegClass, P.Intr, P.Predicates>;1727 }1728}1729 1730defm INT_NVVM_FMA : FMA_INST;1731 1732//1733// Rcp1734//1735 1736def INT_NVVM_RCP_RN_FTZ_F : F_MATH_1<"rcp.rn.ftz.f32", F32RT, F32RT, int_nvvm_rcp_rn_ftz_f>;1737def INT_NVVM_RCP_RN_F : F_MATH_1<"rcp.rn.f32", F32RT, F32RT, int_nvvm_rcp_rn_f>;1738def INT_NVVM_RCP_RZ_FTZ_F : F_MATH_1<"rcp.rz.ftz.f32", F32RT, F32RT, int_nvvm_rcp_rz_ftz_f>;1739def INT_NVVM_RCP_RZ_F : F_MATH_1<"rcp.rz.f32", F32RT, F32RT, int_nvvm_rcp_rz_f>;1740def INT_NVVM_RCP_RM_FTZ_F : F_MATH_1<"rcp.rm.ftz.f32", F32RT, F32RT, int_nvvm_rcp_rm_ftz_f>;1741def INT_NVVM_RCP_RM_F : F_MATH_1<"rcp.rm.f32", F32RT, F32RT, int_nvvm_rcp_rm_f>;1742def INT_NVVM_RCP_RP_FTZ_F : F_MATH_1<"rcp.rp.ftz.f32", F32RT, F32RT, int_nvvm_rcp_rp_ftz_f>;1743def INT_NVVM_RCP_RP_F : F_MATH_1<"rcp.rp.f32", F32RT, F32RT, int_nvvm_rcp_rp_f>;1744 1745def INT_NVVM_RCP_RN_D : F_MATH_1<"rcp.rn.f64", F64RT, F64RT, int_nvvm_rcp_rn_d>;1746def INT_NVVM_RCP_RZ_D : F_MATH_1<"rcp.rz.f64", F64RT, F64RT, int_nvvm_rcp_rz_d>;1747def INT_NVVM_RCP_RM_D : F_MATH_1<"rcp.rm.f64", F64RT, F64RT, int_nvvm_rcp_rm_d>;1748def INT_NVVM_RCP_RP_D : F_MATH_1<"rcp.rp.f64", F64RT, F64RT, int_nvvm_rcp_rp_d>;1749 1750def INT_NVVM_RCP_APPROX_FTZ_F : F_MATH_1<"rcp.approx.ftz.f32",1751 F32RT, F32RT, int_nvvm_rcp_approx_ftz_f>;1752def INT_NVVM_RCP_APPROX_FTZ_D : F_MATH_1<"rcp.approx.ftz.f64",1753 F64RT, F64RT, int_nvvm_rcp_approx_ftz_d>;1754 1755//1756// Sqrt1757//1758 1759def INT_NVVM_SQRT_RN_FTZ_F : F_MATH_1<"sqrt.rn.ftz.f32",1760 F32RT, F32RT, int_nvvm_sqrt_rn_ftz_f>;1761def INT_NVVM_SQRT_RN_F : F_MATH_1<"sqrt.rn.f32", F32RT,1762 F32RT, int_nvvm_sqrt_rn_f>;1763def INT_NVVM_SQRT_RZ_FTZ_F : F_MATH_1<"sqrt.rz.ftz.f32",1764 F32RT, F32RT, int_nvvm_sqrt_rz_ftz_f>;1765def INT_NVVM_SQRT_RZ_F : F_MATH_1<"sqrt.rz.f32", F32RT,1766 F32RT, int_nvvm_sqrt_rz_f>;1767def INT_NVVM_SQRT_RM_FTZ_F : F_MATH_1<"sqrt.rm.ftz.f32",1768 F32RT, F32RT, int_nvvm_sqrt_rm_ftz_f>;1769def INT_NVVM_SQRT_RM_F : F_MATH_1<"sqrt.rm.f32", F32RT,1770 F32RT, int_nvvm_sqrt_rm_f>;1771def INT_NVVM_SQRT_RP_FTZ_F : F_MATH_1<"sqrt.rp.ftz.f32",1772 F32RT, F32RT, int_nvvm_sqrt_rp_ftz_f>;1773def INT_NVVM_SQRT_RP_F : F_MATH_1<"sqrt.rp.f32", F32RT,1774 F32RT, int_nvvm_sqrt_rp_f>;1775def INT_NVVM_SQRT_APPROX_FTZ_F : F_MATH_1<"sqrt.approx.ftz.f32",1776 F32RT, F32RT, int_nvvm_sqrt_approx_ftz_f>;1777def INT_NVVM_SQRT_APPROX_F : F_MATH_1<"sqrt.approx.f32",1778 F32RT, F32RT, int_nvvm_sqrt_approx_f>;1779 1780def INT_NVVM_SQRT_RN_D : F_MATH_1<"sqrt.rn.f64", F64RT, F64RT, int_nvvm_sqrt_rn_d>;1781def INT_NVVM_SQRT_RZ_D : F_MATH_1<"sqrt.rz.f64", F64RT, F64RT, int_nvvm_sqrt_rz_d>;1782def INT_NVVM_SQRT_RM_D : F_MATH_1<"sqrt.rm.f64", F64RT, F64RT, int_nvvm_sqrt_rm_d>;1783def INT_NVVM_SQRT_RP_D : F_MATH_1<"sqrt.rp.f64", F64RT, F64RT, int_nvvm_sqrt_rp_d>;1784 1785def fsqrt_approx : PatFrags<(ops node:$a),1786 [(fsqrt node:$a),1787 (int_nvvm_sqrt_f node:$a)], [{1788 return !usePrecSqrtF32(N);1789}]>;1790 1791// nvvm_sqrt intrinsic1792def : Pat<(int_nvvm_sqrt_f f32:$a), (INT_NVVM_SQRT_RN_FTZ_F $a)>, Requires<[doF32FTZ]>;1793def : Pat<(int_nvvm_sqrt_f f32:$a), (INT_NVVM_SQRT_RN_F $a)>;1794 1795def : Pat<(fsqrt_approx f32:$a), (INT_NVVM_SQRT_APPROX_FTZ_F $a)>, Requires<[doF32FTZ]>;1796def : Pat<(fsqrt_approx f32:$a), (INT_NVVM_SQRT_APPROX_F $a)>;1797 1798//1799// Rsqrt1800//1801 1802foreach t = [F32RT, F64RT] in {1803 def RSQRT_APPROX_ # t.Ty :1804 BasicFlagsNVPTXInst<(outs t.RC:$dst),1805 (ins t.RC:$a), (ins FTZFlag:$ftz),1806 "rsqrt.approx$ftz.f" # t.Size>;1807}1808 1809def : Pat<(int_nvvm_rsqrt_approx_ftz_f f32:$a), (RSQRT_APPROX_f32 $a, FTZ)>;1810def : Pat<(int_nvvm_rsqrt_approx_ftz_d f64:$a), (RSQRT_APPROX_f64 $a, FTZ)>;1811def : Pat<(int_nvvm_rsqrt_approx_f f32:$a), (RSQRT_APPROX_f32 $a, NoFTZ)>;1812def : Pat<(int_nvvm_rsqrt_approx_d f64:$a), (RSQRT_APPROX_f64 $a, NoFTZ)>;1813 1814 1815// 1.0f / sqrt_approx -> rsqrt_approx1816let Predicates = [doRsqrtOpt] in {1817 def : Pat<(fdiv fpimm_1, (int_nvvm_sqrt_approx_f f32:$a)),1818 (RSQRT_APPROX_f32 $a, NoFTZ)>;1819 def : Pat<(fdiv fpimm_1, (int_nvvm_sqrt_approx_ftz_f f32:$a)),1820 (RSQRT_APPROX_f32 $a, FTZ)>;1821 1822 // same for int_nvvm_sqrt_f when non-precision sqrt is requested1823 def : Pat<(fdiv fpimm_1, (fsqrt_approx f32:$a)),1824 (RSQRT_APPROX_f32 $a)>;1825}1826//1827// Add1828//1829 1830def INT_NVVM_ADD_RN_FTZ_F : F_MATH_2<"add.rn.ftz.f32", B32, B32, B32, int_nvvm_add_rn_ftz_f>;1831def INT_NVVM_ADD_RN_F : F_MATH_2<"add.rn.f32", B32, B32, B32, int_nvvm_add_rn_f>;1832def INT_NVVM_ADD_RZ_FTZ_F : F_MATH_2<"add.rz.ftz.f32", B32, B32, B32, int_nvvm_add_rz_ftz_f>;1833def INT_NVVM_ADD_RZ_F : F_MATH_2<"add.rz.f32", B32, B32, B32, int_nvvm_add_rz_f>;1834def INT_NVVM_ADD_RM_FTZ_F : F_MATH_2<"add.rm.ftz.f32", B32, B32, B32, int_nvvm_add_rm_ftz_f>;1835def INT_NVVM_ADD_RM_F : F_MATH_2<"add.rm.f32", B32, B32, B32, int_nvvm_add_rm_f>;1836def INT_NVVM_ADD_RP_FTZ_F : F_MATH_2<"add.rp.ftz.f32", B32, B32, B32, int_nvvm_add_rp_ftz_f>;1837def INT_NVVM_ADD_RP_F : F_MATH_2<"add.rp.f32", B32, B32, B32, int_nvvm_add_rp_f>;1838 1839def INT_NVVM_ADD_RN_D : F_MATH_2<"add.rn.f64", B64, B64, B64, int_nvvm_add_rn_d>;1840def INT_NVVM_ADD_RZ_D : F_MATH_2<"add.rz.f64", B64, B64, B64, int_nvvm_add_rz_d>;1841def INT_NVVM_ADD_RM_D : F_MATH_2<"add.rm.f64", B64, B64, B64, int_nvvm_add_rm_d>;1842def INT_NVVM_ADD_RP_D : F_MATH_2<"add.rp.f64", B64, B64, B64, int_nvvm_add_rp_d>;1843 1844//1845// BFIND1846//1847 1848foreach t = [I32RT, I64RT] in {1849 foreach sign = ["s", "u"] in {1850 defvar flo_intrin = !cast<Intrinsic>("int_nvvm_flo_" # sign);1851 def BFIND_ # sign # t.Size1852 : BasicNVPTXInst<(outs B32:$dst), (ins t.RC:$src),1853 "bfind." # sign # t.Size,1854 [(set i32:$dst, (flo_intrin t.Ty:$src, 0))]>;1855 1856 def BFIND_SHIFTAMT_ # sign # t.Size1857 : BasicNVPTXInst<(outs B32:$dst), (ins t.RC:$src),1858 "bfind.shiftamt." # sign # t.Size,1859 [(set i32:$dst, (flo_intrin t.Ty:$src, -1))]>;1860 }1861}1862 1863//1864// szext1865//1866 1867foreach sign = ["s", "u"] in {1868 foreach mode = ["wrap", "clamp"] in {1869 defvar ext = !if(!eq(sign, "s"), "sext", "zext");1870 defvar intrin = !cast<Intrinsic>("int_nvvm_" # ext # "_" # mode);1871 defm SZEXT_ # sign # _ # mode1872 : I3Inst<"szext." # mode # "." # sign # "32",1873 intrin, I32RT, commutative = false,1874 requires = [hasSM<70>, hasPTX<76>]>;1875 }1876}1877 1878//1879// BMSK1880//1881 1882foreach mode = ["wrap", "clamp"] in {1883 defvar intrin = !cast<Intrinsic>("int_nvvm_bmsk_" # mode);1884 defm BMSK_ # mode1885 : I3Inst<"bmsk." # mode # ".b32",1886 intrin, I32RT, commutative = false,1887 requires = [hasSM<70>, hasPTX<76>]>;1888}1889 1890//1891// Convert1892//1893 1894def : Pat<(int_nvvm_d2f_rn_ftz f64:$a), (CVT_f32_f64 $a, CvtRN_FTZ)>;1895def : Pat<(int_nvvm_d2f_rn f64:$a), (CVT_f32_f64 $a, CvtRN)>;1896def : Pat<(int_nvvm_d2f_rz_ftz f64:$a), (CVT_f32_f64 $a, CvtRZ_FTZ)>;1897def : Pat<(int_nvvm_d2f_rz f64:$a), (CVT_f32_f64 $a, CvtRZ)>;1898def : Pat<(int_nvvm_d2f_rm_ftz f64:$a), (CVT_f32_f64 $a, CvtRM_FTZ)>;1899def : Pat<(int_nvvm_d2f_rm f64:$a), (CVT_f32_f64 $a, CvtRM)>;1900def : Pat<(int_nvvm_d2f_rp_ftz f64:$a), (CVT_f32_f64 $a, CvtRP_FTZ)>;1901def : Pat<(int_nvvm_d2f_rp f64:$a), (CVT_f32_f64 $a, CvtRP)>;1902 1903def : Pat<(int_nvvm_d2i_rn f64:$a), (CVT_s32_f64 $a, CvtRNI)>;1904def : Pat<(int_nvvm_d2i_rz f64:$a), (CVT_s32_f64 $a, CvtRZI)>;1905def : Pat<(int_nvvm_d2i_rm f64:$a), (CVT_s32_f64 $a, CvtRMI)>;1906def : Pat<(int_nvvm_d2i_rp f64:$a), (CVT_s32_f64 $a, CvtRPI)>;1907 1908def : Pat<(int_nvvm_d2ui_rn f64:$a), (CVT_u32_f64 $a, CvtRNI)>;1909def : Pat<(int_nvvm_d2ui_rz f64:$a), (CVT_u32_f64 $a, CvtRZI)>;1910def : Pat<(int_nvvm_d2ui_rm f64:$a), (CVT_u32_f64 $a, CvtRMI)>;1911def : Pat<(int_nvvm_d2ui_rp f64:$a), (CVT_u32_f64 $a, CvtRPI)>;1912 1913def : Pat<(int_nvvm_i2d_rn i32:$a), (CVT_f64_s32 $a, CvtRN)>;1914def : Pat<(int_nvvm_i2d_rz i32:$a), (CVT_f64_s32 $a, CvtRZ)>;1915def : Pat<(int_nvvm_i2d_rm i32:$a), (CVT_f64_s32 $a, CvtRM)>;1916def : Pat<(int_nvvm_i2d_rp i32:$a), (CVT_f64_s32 $a, CvtRP)>;1917 1918def : Pat<(int_nvvm_ui2d_rn i32:$a), (CVT_f64_u32 $a, CvtRN)>;1919def : Pat<(int_nvvm_ui2d_rz i32:$a), (CVT_f64_u32 $a, CvtRZ)>;1920def : Pat<(int_nvvm_ui2d_rm i32:$a), (CVT_f64_u32 $a, CvtRM)>;1921def : Pat<(int_nvvm_ui2d_rp i32:$a), (CVT_f64_u32 $a, CvtRP)>;1922 1923def : Pat<(int_nvvm_f2i_rn_ftz f32:$a), (CVT_s32_f32 $a, CvtRNI_FTZ)>;1924def : Pat<(int_nvvm_f2i_rn f32:$a), (CVT_s32_f32 $a, CvtRNI)>;1925def : Pat<(int_nvvm_f2i_rz_ftz f32:$a), (CVT_s32_f32 $a, CvtRZI_FTZ)>;1926def : Pat<(int_nvvm_f2i_rz f32:$a), (CVT_s32_f32 $a, CvtRZI)>;1927def : Pat<(int_nvvm_f2i_rm_ftz f32:$a), (CVT_s32_f32 $a, CvtRMI_FTZ)>;1928def : Pat<(int_nvvm_f2i_rm f32:$a), (CVT_s32_f32 $a, CvtRMI)>;1929def : Pat<(int_nvvm_f2i_rp_ftz f32:$a), (CVT_s32_f32 $a, CvtRPI_FTZ)>;1930def : Pat<(int_nvvm_f2i_rp f32:$a), (CVT_s32_f32 $a, CvtRPI)>;1931 1932def : Pat<(int_nvvm_f2ui_rn_ftz f32:$a), (CVT_u32_f32 $a, CvtRNI_FTZ)>;1933def : Pat<(int_nvvm_f2ui_rn f32:$a), (CVT_u32_f32 $a, CvtRNI)>;1934def : Pat<(int_nvvm_f2ui_rz_ftz f32:$a), (CVT_u32_f32 $a, CvtRZI_FTZ)>;1935def : Pat<(int_nvvm_f2ui_rz f32:$a), (CVT_u32_f32 $a, CvtRZI)>;1936def : Pat<(int_nvvm_f2ui_rm_ftz f32:$a), (CVT_u32_f32 $a, CvtRMI_FTZ)>;1937def : Pat<(int_nvvm_f2ui_rm f32:$a), (CVT_u32_f32 $a, CvtRMI)>;1938def : Pat<(int_nvvm_f2ui_rp_ftz f32:$a), (CVT_u32_f32 $a, CvtRPI_FTZ)>;1939def : Pat<(int_nvvm_f2ui_rp f32:$a), (CVT_u32_f32 $a, CvtRPI)>;1940 1941def : Pat<(int_nvvm_i2f_rn i32:$a), (CVT_f32_s32 $a, CvtRN)>;1942def : Pat<(int_nvvm_i2f_rz i32:$a), (CVT_f32_s32 $a, CvtRZ)>;1943def : Pat<(int_nvvm_i2f_rm i32:$a), (CVT_f32_s32 $a, CvtRM)>;1944def : Pat<(int_nvvm_i2f_rp i32:$a), (CVT_f32_s32 $a, CvtRP)>;1945 1946def : Pat<(int_nvvm_ui2f_rn i32:$a), (CVT_f32_u32 $a, CvtRN)>;1947def : Pat<(int_nvvm_ui2f_rz i32:$a), (CVT_f32_u32 $a, CvtRZ)>;1948def : Pat<(int_nvvm_ui2f_rm i32:$a), (CVT_f32_u32 $a, CvtRM)>;1949def : Pat<(int_nvvm_ui2f_rp i32:$a), (CVT_f32_u32 $a, CvtRP)>;1950 1951def : Pat<(int_nvvm_ff2bf16x2_rn f32:$a, f32:$b), (CVT_bf16x2_f32 $a, $b, CvtRN)>;1952def : Pat<(int_nvvm_ff2bf16x2_rn_relu f32:$a, f32:$b), (CVT_bf16x2_f32 $a, $b, CvtRN_RELU)>;1953def : Pat<(int_nvvm_ff2bf16x2_rz f32:$a, f32:$b), (CVT_bf16x2_f32 $a, $b, CvtRZ)>;1954def : Pat<(int_nvvm_ff2bf16x2_rz_relu f32:$a, f32:$b), (CVT_bf16x2_f32 $a, $b, CvtRZ_RELU)>;1955let Predicates = [hasPTX<81>, hasSM<80>] in {1956 def : Pat<(int_nvvm_ff2bf16x2_rn_satfinite f32:$a, f32:$b), (CVT_bf16x2_f32_sf $a, $b, CvtRN)>;1957 def : Pat<(int_nvvm_ff2bf16x2_rn_relu_satfinite f32:$a, f32:$b), (CVT_bf16x2_f32_sf $a, $b, CvtRN_RELU)>;1958 def : Pat<(int_nvvm_ff2bf16x2_rz_satfinite f32:$a, f32:$b), (CVT_bf16x2_f32_sf $a, $b, CvtRZ)>;1959 def : Pat<(int_nvvm_ff2bf16x2_rz_relu_satfinite f32:$a, f32:$b), (CVT_bf16x2_f32_sf $a, $b, CvtRZ_RELU)>;1960}1961let Predicates = [hasPTX<87>, hasSM100aOrSM103a] in {1962def : Pat<(int_nvvm_ff2bf16x2_rs f32:$a, f32:$b, i32:$c),1963 (CVT_bf16x2_f32_rs $a, $b, $c, CvtRS)>;1964def : Pat<(int_nvvm_ff2bf16x2_rs_relu f32:$a, f32:$b, i32:$c),1965 (CVT_bf16x2_f32_rs $a, $b, $c, CvtRS_RELU)>;1966def : Pat<(int_nvvm_ff2bf16x2_rs_satfinite f32:$a, f32:$b, i32:$c), 1967 (CVT_bf16x2_f32_rs_sf $a, $b, $c, CvtRS)>;1968def : Pat<(int_nvvm_ff2bf16x2_rs_relu_satfinite f32:$a, f32:$b, i32:$c), 1969 (CVT_bf16x2_f32_rs_sf $a, $b, $c, CvtRS_RELU)>;1970}1971 1972def : Pat<(int_nvvm_ff2f16x2_rn f32:$a, f32:$b), (CVT_f16x2_f32 $a, $b, CvtRN)>;1973def : Pat<(int_nvvm_ff2f16x2_rn_relu f32:$a, f32:$b), (CVT_f16x2_f32 $a, $b, CvtRN_RELU)>;1974def : Pat<(int_nvvm_ff2f16x2_rz f32:$a, f32:$b), (CVT_f16x2_f32 $a, $b, CvtRZ)>;1975def : Pat<(int_nvvm_ff2f16x2_rz_relu f32:$a, f32:$b), (CVT_f16x2_f32 $a, $b, CvtRZ_RELU)>;1976let Predicates = [hasPTX<81>, hasSM<80>] in {1977 def : Pat<(int_nvvm_ff2f16x2_rn_satfinite f32:$a, f32:$b), (CVT_f16x2_f32_sf $a, $b, CvtRN)>;1978 def : Pat<(int_nvvm_ff2f16x2_rn_relu_satfinite f32:$a, f32:$b), (CVT_f16x2_f32_sf $a, $b, CvtRN_RELU)>;1979 def : Pat<(int_nvvm_ff2f16x2_rz_satfinite f32:$a, f32:$b), (CVT_f16x2_f32_sf $a, $b, CvtRZ)>;1980 def : Pat<(int_nvvm_ff2f16x2_rz_relu_satfinite f32:$a, f32:$b), (CVT_f16x2_f32_sf $a, $b, CvtRZ_RELU)>;1981}1982 1983let Predicates = [hasPTX<87>, hasSM100aOrSM103a] in {1984def : Pat<(int_nvvm_ff2f16x2_rs f32:$a, f32:$b, i32:$c),1985 (CVT_f16x2_f32_rs $a, $b, $c, CvtRS)>;1986def : Pat<(int_nvvm_ff2f16x2_rs_relu f32:$a, f32:$b, i32:$c),1987 (CVT_f16x2_f32_rs $a, $b, $c, CvtRS_RELU)>;1988def : Pat<(int_nvvm_ff2f16x2_rs_satfinite f32:$a, f32:$b, i32:$c), 1989 (CVT_f16x2_f32_rs_sf $a, $b, $c, CvtRS)>;1990def : Pat<(int_nvvm_ff2f16x2_rs_relu_satfinite f32:$a, f32:$b, i32:$c), 1991 (CVT_f16x2_f32_rs_sf $a, $b, $c, CvtRS_RELU)>;1992}1993def : Pat<(int_nvvm_f2bf16_rn f32:$a), (CVT_bf16_f32 $a, CvtRN)>;1994def : Pat<(int_nvvm_f2bf16_rn_relu f32:$a), (CVT_bf16_f32 $a, CvtRN_RELU)>;1995def : Pat<(int_nvvm_f2bf16_rz f32:$a), (CVT_bf16_f32 $a, CvtRZ)>;1996def : Pat<(int_nvvm_f2bf16_rz_relu f32:$a), (CVT_bf16_f32 $a, CvtRZ_RELU)>;1997let Predicates = [hasPTX<81>, hasSM<80>] in {1998 def : Pat<(int_nvvm_f2bf16_rz_satfinite f32:$a), (CVT_bf16_f32_sf $a, CvtRZ)>;1999 def : Pat<(int_nvvm_f2bf16_rz_relu_satfinite f32:$a), (CVT_bf16_f32_sf $a, CvtRZ_RELU)>;2000 def : Pat<(int_nvvm_f2bf16_rn_satfinite f32:$a), (CVT_bf16_f32_sf $a, CvtRN)>;2001 def : Pat<(int_nvvm_f2bf16_rn_relu_satfinite f32:$a), (CVT_bf16_f32_sf $a, CvtRN_RELU)>;2002}2003 2004def : Pat<(int_nvvm_f2f16_rn f32:$a), (CVT_f16_f32 $a, CvtRN)>;2005def : Pat<(int_nvvm_f2f16_rn_relu f32:$a), (CVT_f16_f32 $a, CvtRN_RELU)>;2006def : Pat<(int_nvvm_f2f16_rz f32:$a), (CVT_f16_f32 $a, CvtRZ)>;2007def : Pat<(int_nvvm_f2f16_rz_relu f32:$a), (CVT_f16_f32 $a, CvtRZ_RELU)>;2008let Predicates = [hasPTX<81>, hasSM<80>] in {2009 def : Pat<(int_nvvm_f2f16_rz_satfinite f32:$a), (CVT_f16_f32_sf $a, CvtRZ)>;2010 def : Pat<(int_nvvm_f2f16_rz_relu_satfinite f32:$a), (CVT_f16_f32_sf $a, CvtRZ_RELU)>;2011 def : Pat<(int_nvvm_f2f16_rn_satfinite f32:$a), (CVT_f16_f32_sf $a, CvtRN)>;2012 def : Pat<(int_nvvm_f2f16_rn_relu_satfinite f32:$a), (CVT_f16_f32_sf $a, CvtRN_RELU)>;2013}2014 2015def : Pat<(int_nvvm_lohi_i2d i32:$a, i32:$b), (V2I32toI64 $a, $b)>;2016def : Pat<(int_nvvm_d2i_lo f64:$a), (I64toI32L $a)>;2017def : Pat<(int_nvvm_d2i_hi f64:$a), (I64toI32H $a)>;2018 2019def : Pat<(int_nvvm_d2i_lo f64:$a), (I64toI32L_Sink $a)>, Requires<[hasPTX<71>]>;2020def : Pat<(int_nvvm_d2i_hi f64:$a), (I64toI32H_Sink $a)>, Requires<[hasPTX<71>]>;2021 2022def : Pat<(int_nvvm_f2ll_rn_ftz f32:$a), (CVT_s64_f32 $a, CvtRNI_FTZ)>;2023def : Pat<(int_nvvm_f2ll_rn f32:$a), (CVT_s64_f32 $a, CvtRNI)>;2024def : Pat<(int_nvvm_f2ll_rz_ftz f32:$a), (CVT_s64_f32 $a, CvtRZI_FTZ)>;2025def : Pat<(int_nvvm_f2ll_rz f32:$a), (CVT_s64_f32 $a, CvtRZI)>;2026def : Pat<(int_nvvm_f2ll_rm_ftz f32:$a), (CVT_s64_f32 $a, CvtRMI_FTZ)>;2027def : Pat<(int_nvvm_f2ll_rm f32:$a), (CVT_s64_f32 $a, CvtRMI)>;2028def : Pat<(int_nvvm_f2ll_rp_ftz f32:$a), (CVT_s64_f32 $a, CvtRPI_FTZ)>;2029def : Pat<(int_nvvm_f2ll_rp f32:$a), (CVT_s64_f32 $a, CvtRPI)>;2030 2031def : Pat<(int_nvvm_f2ull_rn_ftz f32:$a), (CVT_u64_f32 $a, CvtRNI_FTZ)>;2032def : Pat<(int_nvvm_f2ull_rn f32:$a), (CVT_u64_f32 $a, CvtRNI)>;2033def : Pat<(int_nvvm_f2ull_rz_ftz f32:$a), (CVT_u64_f32 $a, CvtRZI_FTZ)>;2034def : Pat<(int_nvvm_f2ull_rz f32:$a), (CVT_u64_f32 $a, CvtRZI)>;2035def : Pat<(int_nvvm_f2ull_rm_ftz f32:$a), (CVT_u64_f32 $a, CvtRMI_FTZ)>;2036def : Pat<(int_nvvm_f2ull_rm f32:$a), (CVT_u64_f32 $a, CvtRMI)>;2037def : Pat<(int_nvvm_f2ull_rp_ftz f32:$a), (CVT_u64_f32 $a, CvtRPI_FTZ)>;2038def : Pat<(int_nvvm_f2ull_rp f32:$a), (CVT_u64_f32 $a, CvtRPI)>;2039 2040def : Pat<(int_nvvm_d2ll_rn f64:$a), (CVT_s64_f64 $a, CvtRNI)>;2041def : Pat<(int_nvvm_d2ll_rz f64:$a), (CVT_s64_f64 $a, CvtRZI)>;2042def : Pat<(int_nvvm_d2ll_rm f64:$a), (CVT_s64_f64 $a, CvtRMI)>;2043def : Pat<(int_nvvm_d2ll_rp f64:$a), (CVT_s64_f64 $a, CvtRPI)>;2044 2045def : Pat<(int_nvvm_d2ull_rn f64:$a), (CVT_u64_f64 $a, CvtRNI)>;2046def : Pat<(int_nvvm_d2ull_rz f64:$a), (CVT_u64_f64 $a, CvtRZI)>;2047def : Pat<(int_nvvm_d2ull_rm f64:$a), (CVT_u64_f64 $a, CvtRMI)>;2048def : Pat<(int_nvvm_d2ull_rp f64:$a), (CVT_u64_f64 $a, CvtRPI)>;2049 2050def : Pat<(int_nvvm_ll2f_rn i64:$a), (CVT_f32_s64 $a, CvtRN)>;2051def : Pat<(int_nvvm_ll2f_rz i64:$a), (CVT_f32_s64 $a, CvtRZ)>;2052def : Pat<(int_nvvm_ll2f_rm i64:$a), (CVT_f32_s64 $a, CvtRM)>;2053def : Pat<(int_nvvm_ll2f_rp i64:$a), (CVT_f32_s64 $a, CvtRP)>;2054 2055def : Pat<(int_nvvm_ull2f_rn i64:$a), (CVT_f32_u64 $a, CvtRN)>;2056def : Pat<(int_nvvm_ull2f_rz i64:$a), (CVT_f32_u64 $a, CvtRZ)>;2057def : Pat<(int_nvvm_ull2f_rm i64:$a), (CVT_f32_u64 $a, CvtRM)>;2058def : Pat<(int_nvvm_ull2f_rp i64:$a), (CVT_f32_u64 $a, CvtRP)>;2059 2060def : Pat<(int_nvvm_ll2d_rn i64:$a), (CVT_f64_s64 $a, CvtRN)>;2061def : Pat<(int_nvvm_ll2d_rz i64:$a), (CVT_f64_s64 $a, CvtRZ)>;2062def : Pat<(int_nvvm_ll2d_rm i64:$a), (CVT_f64_s64 $a, CvtRM)>;2063def : Pat<(int_nvvm_ll2d_rp i64:$a), (CVT_f64_s64 $a, CvtRP)>;2064 2065def : Pat<(int_nvvm_ull2d_rn i64:$a), (CVT_f64_u64 $a, CvtRN)>;2066def : Pat<(int_nvvm_ull2d_rz i64:$a), (CVT_f64_u64 $a, CvtRZ)>;2067def : Pat<(int_nvvm_ull2d_rm i64:$a), (CVT_f64_u64 $a, CvtRM)>;2068def : Pat<(int_nvvm_ull2d_rp i64:$a), (CVT_f64_u64 $a, CvtRP)>;2069 2070 2071def : Pat<(int_nvvm_f2h_rn_ftz f32:$a), (CVT_f16_f32 $a, CvtRN_FTZ)>;2072def : Pat<(int_nvvm_f2h_rn f32:$a), (CVT_f16_f32 $a, CvtRN)>;2073 2074let Predicates = [callSubtarget<"hasFP8ConversionSupport">] in {2075 def : Pat<(int_nvvm_ff_to_e4m3x2_rn f32:$a, f32:$b),2076 (CVT_e4m3x2_f32 $a, $b, CvtRN)>;2077 def : Pat<(int_nvvm_ff_to_e4m3x2_rn_relu f32:$a, f32:$b),2078 (CVT_e4m3x2_f32 $a, $b, CvtRN_RELU)>;2079 def : Pat<(int_nvvm_ff_to_e5m2x2_rn f32:$a, f32:$b),2080 (CVT_e5m2x2_f32 $a, $b, CvtRN)>;2081 def : Pat<(int_nvvm_ff_to_e5m2x2_rn_relu f32:$a, f32:$b),2082 (CVT_e5m2x2_f32 $a, $b, CvtRN_RELU)>;2083 2084 def : Pat<(int_nvvm_f16x2_to_e4m3x2_rn v2f16:$a),2085 (CVT_e4m3x2_f16x2 $a, CvtRN)>;2086 def : Pat<(int_nvvm_f16x2_to_e4m3x2_rn_relu v2f16:$a),2087 (CVT_e4m3x2_f16x2 $a, CvtRN_RELU)>;2088 def : Pat<(int_nvvm_f16x2_to_e5m2x2_rn v2f16:$a),2089 (CVT_e5m2x2_f16x2 $a, CvtRN)>;2090 def : Pat<(int_nvvm_f16x2_to_e5m2x2_rn_relu v2f16:$a),2091 (CVT_e5m2x2_f16x2 $a, CvtRN_RELU)>;2092 2093 def : Pat<(int_nvvm_e4m3x2_to_f16x2_rn i16:$a),2094 (CVT_f16x2_e4m3x2 $a, CvtRN)>;2095 def : Pat<(int_nvvm_e4m3x2_to_f16x2_rn_relu i16:$a),2096 (CVT_f16x2_e4m3x2 $a, CvtRN_RELU)>;2097 def : Pat<(int_nvvm_e5m2x2_to_f16x2_rn i16:$a),2098 (CVT_f16x2_e5m2x2 $a, CvtRN)>;2099 def : Pat<(int_nvvm_e5m2x2_to_f16x2_rn_relu i16:$a),2100 (CVT_f16x2_e5m2x2 $a, CvtRN_RELU)>;2101}2102 2103let Predicates = [callSubtarget<"hasNarrowFPConversionSupport">] in {2104 def : Pat<(int_nvvm_ff_to_e2m3x2_rn_satfinite f32:$a, f32:$b),2105 (CVT_e2m3x2_f32_sf $a, $b, CvtRN)>;2106 def : Pat<(int_nvvm_ff_to_e2m3x2_rn_relu_satfinite f32:$a, f32:$b),2107 (CVT_e2m3x2_f32_sf $a, $b, CvtRN_RELU)>;2108 def : Pat<(int_nvvm_ff_to_e3m2x2_rn_satfinite f32:$a, f32:$b),2109 (CVT_e3m2x2_f32_sf $a, $b, CvtRN)>;2110 def : Pat<(int_nvvm_ff_to_e3m2x2_rn_relu_satfinite f32:$a, f32:$b),2111 (CVT_e3m2x2_f32_sf $a, $b, CvtRN_RELU)>;2112 2113 def : Pat<(int_nvvm_e2m3x2_to_f16x2_rn i16:$a),2114 (CVT_f16x2_e2m3x2 $a, CvtRN)>;2115 def : Pat<(int_nvvm_e2m3x2_to_f16x2_rn_relu i16:$a),2116 (CVT_f16x2_e2m3x2 $a, CvtRN_RELU)>;2117 def : Pat<(int_nvvm_e3m2x2_to_f16x2_rn i16:$a),2118 (CVT_f16x2_e3m2x2 $a, CvtRN)>;2119 def : Pat<(int_nvvm_e3m2x2_to_f16x2_rn_relu i16:$a),2120 (CVT_f16x2_e3m2x2 $a, CvtRN_RELU)>;2121 2122 def : Pat<(int_nvvm_ff_to_e2m1x2_rn_satfinite f32:$a, f32:$b),2123 (CVT_e2m1x2_f32_sf $a, $b, CvtRN)>;2124 def : Pat<(int_nvvm_ff_to_e2m1x2_rn_relu_satfinite f32:$a, f32:$b),2125 (CVT_e2m1x2_f32_sf $a, $b, CvtRN_RELU)>;2126 2127 def : Pat<(int_nvvm_e2m1x2_to_f16x2_rn i16:$a),2128 (CVT_f16x2_e2m1x2 $a, CvtRN)>;2129 def : Pat<(int_nvvm_e2m1x2_to_f16x2_rn_relu i16:$a),2130 (CVT_f16x2_e2m1x2 $a, CvtRN_RELU)>;2131 2132 def : Pat<(int_nvvm_ff_to_ue8m0x2_rz f32:$a, f32:$b),2133 (CVT_ue8m0x2_f32 $a, $b, CvtRZ)>;2134 def : Pat<(int_nvvm_ff_to_ue8m0x2_rz_satfinite f32:$a, f32:$b),2135 (CVT_ue8m0x2_f32_sf $a, $b, CvtRZ)>;2136 def : Pat<(int_nvvm_ff_to_ue8m0x2_rp f32:$a, f32:$b),2137 (CVT_ue8m0x2_f32 $a, $b, CvtRP)>;2138 def : Pat<(int_nvvm_ff_to_ue8m0x2_rp_satfinite f32:$a, f32:$b),2139 (CVT_ue8m0x2_f32_sf $a, $b, CvtRP)>;2140 2141 def : Pat<(int_nvvm_bf16x2_to_ue8m0x2_rz v2bf16:$a),2142 (CVT_ue8m0x2_bf16x2 $a, CvtRZ)>;2143 def : Pat<(int_nvvm_bf16x2_to_ue8m0x2_rz_satfinite v2bf16:$a),2144 (CVT_ue8m0x2_bf16x2_sf $a, CvtRZ)>;2145 def : Pat<(int_nvvm_bf16x2_to_ue8m0x2_rp v2bf16:$a),2146 (CVT_ue8m0x2_bf16x2 $a, CvtRP)>;2147 def : Pat<(int_nvvm_bf16x2_to_ue8m0x2_rp_satfinite v2bf16:$a),2148 (CVT_ue8m0x2_bf16x2_sf $a, CvtRP)>;2149 2150 def : Pat<(int_nvvm_ue8m0x2_to_bf16x2 i16:$a),2151 (CVT_bf16x2_ue8m0x2 $a)>;2152}2153 2154def SDT_CVT_F32X4_TO_FPX4_RS_VEC :2155 SDTypeProfile<1, 6, [SDTCisVec<0>, SDTCisFP<1>, SDTCisFP<2>, SDTCisFP<3>, 2156 SDTCisFP<4>, SDTCisInt<5>, SDTCisInt<6>]>;2157 2158def SDT_CVT_F32X4_TO_FPX4_RS_INT :2159 SDTypeProfile<1, 6, [SDTCisInt<0>, SDTCisFP<1>, SDTCisFP<2>, SDTCisFP<3>, 2160 SDTCisFP<4>, SDTCisInt<5>, SDTCisInt<6>]>;2161 2162class CVT_F32X4_TO_FPX4_RS_SF_NODE<string FPName, SDTypeProfile SDT> :2163 SDNode<"NVPTXISD::CVT_" # FPName # "X4_F32X4_RS_SF", SDT, []>;2164 2165multiclass CVT_F32X4_TO_FPX4_RS_SF_VEC<string FPName, VTVec RetTy> {2166 def : Pat<(RetTy (CVT_F32X4_TO_FPX4_RS_SF_NODE<!toupper(FPName),2167 SDT_CVT_F32X4_TO_FPX4_RS_VEC>2168 f32:$f1, f32:$f2, f32:$f3, f32:$f4, i32:$rbits, CvtRS)),2169 (!cast<NVPTXInst>("CVT_" # FPName # "x4_f32x4_rs_sf") 2170 $f1, $f2, $f3, $f4, $rbits, CvtRS)>;2171 2172 def : Pat<(RetTy (CVT_F32X4_TO_FPX4_RS_SF_NODE<!toupper(FPName), 2173 SDT_CVT_F32X4_TO_FPX4_RS_VEC>2174 f32:$f1, f32:$f2, f32:$f3, f32:$f4, i32:$rbits, CvtRS_RELU)),2175 (!cast<NVPTXInst>("CVT_" # FPName # "x4_f32x4_rs_sf") 2176 $f1, $f2, $f3, $f4, $rbits, CvtRS_RELU)>;2177}2178 2179// RS rounding mode conversions2180let Predicates = [hasPTX<87>, hasSM100aOrSM103a] in {2181// FP8x4 conversions2182defm : CVT_F32X4_TO_FPX4_RS_SF_VEC<"e4m3", v4i8>;2183defm : CVT_F32X4_TO_FPX4_RS_SF_VEC<"e5m2", v4i8>;2184 2185// FP6x4 conversions2186defm : CVT_F32X4_TO_FPX4_RS_SF_VEC<"e2m3", v4i8>;2187defm : CVT_F32X4_TO_FPX4_RS_SF_VEC<"e3m2", v4i8>;2188 2189// FP4x4 conversions2190def : Pat<(i16 (CVT_F32X4_TO_FPX4_RS_SF_NODE<"E2M1", 2191 SDT_CVT_F32X4_TO_FPX4_RS_INT>2192 f32:$f1, f32:$f2, f32:$f3, f32:$f4, i32:$rbits, CvtRS)),2193 (CVT_e2m1x4_f32x4_rs_sf $f1, $f2, $f3, $f4, $rbits, CvtRS)>;2194def : Pat<(i16 (CVT_F32X4_TO_FPX4_RS_SF_NODE<"E2M1", 2195 SDT_CVT_F32X4_TO_FPX4_RS_INT>2196 f32:$f1, f32:$f2, f32:$f3, f32:$f4, i32:$rbits, CvtRS_RELU)),2197 (CVT_e2m1x4_f32x4_rs_sf $f1, $f2, $f3, $f4, $rbits, CvtRS_RELU)>;2198}2199 2200//2201// FNS2202//2203 2204class INT_FNS_MBO<dag ins, dag Operands>2205 : BasicNVPTXInst<(outs B32:$dst), ins,2206 "fns.b32",2207 [(set i32:$dst, Operands)]>,2208 Requires<[hasPTX<60>, hasSM<30>]>;2209 2210def INT_FNS_rrr : INT_FNS_MBO<(ins B32:$mask, B32:$base, B32:$offset),2211 (int_nvvm_fns i32:$mask, i32:$base, i32:$offset)>;2212def INT_FNS_rri : INT_FNS_MBO<(ins B32:$mask, B32:$base, i32imm:$offset),2213 (int_nvvm_fns i32:$mask, i32:$base, imm:$offset)>;2214def INT_FNS_rir : INT_FNS_MBO<(ins B32:$mask, i32imm:$base, B32:$offset),2215 (int_nvvm_fns i32:$mask, imm:$base, i32:$offset)>;2216def INT_FNS_rii : INT_FNS_MBO<(ins B32:$mask, i32imm:$base, i32imm:$offset),2217 (int_nvvm_fns i32:$mask, imm:$base, imm:$offset)>;2218def INT_FNS_irr : INT_FNS_MBO<(ins i32imm:$mask, B32:$base, B32:$offset),2219 (int_nvvm_fns imm:$mask, i32:$base, i32:$offset)>;2220def INT_FNS_iri : INT_FNS_MBO<(ins i32imm:$mask, B32:$base, i32imm:$offset),2221 (int_nvvm_fns imm:$mask, i32:$base, imm:$offset)>;2222def INT_FNS_iir : INT_FNS_MBO<(ins i32imm:$mask, i32imm:$base, B32:$offset),2223 (int_nvvm_fns imm:$mask, imm:$base, i32:$offset)>;2224def INT_FNS_iii : INT_FNS_MBO<(ins i32imm:$mask, i32imm:$base, i32imm:$offset),2225 (int_nvvm_fns imm:$mask, imm:$base, imm:$offset)>;2226 2227//-----------------------------------2228// Atomic Functions2229//-----------------------------------2230 2231class ATOMIC_GLOBAL_CHK <dag frag>2232 : PatFrag<!setdagop(frag, ops), frag, AS_match.global>;2233class ATOMIC_SHARED_CHK <dag frag>2234 : PatFrag<!setdagop(frag, ops), frag, AS_match.shared>;2235class ATOMIC_SHARED_CLUSTER_CHK <dag frag>2236 : PatFrag<!setdagop(frag, ops), frag, AS_match.shared_cluster>;2237class ATOMIC_GENERIC_CHK <dag frag>2238 : PatFrag<!setdagop(frag, ops), frag, AS_match.generic>;2239 2240multiclass F_ATOMIC_2<RegTyInfo t, string sem_str, string as_str, string op_str,2241 SDPatternOperator op, list<Predicate> preds> {2242 defvar asm_str = "atom" # sem_str # as_str # "." # op_str;2243 let mayLoad = 1, mayStore = 1, hasSideEffects = 1 in {2244 def r : BasicNVPTXInst<(outs t.RC:$dst), (ins ADDR:$addr, t.RC:$b),2245 asm_str,2246 [(set t.Ty:$dst, (op addr:$addr, t.Ty:$b))]>,2247 Requires<preds>;2248 if t.SupportsImm then2249 def i : BasicNVPTXInst<(outs t.RC:$dst), (ins ADDR:$addr, t.Imm:$b),2250 asm_str,2251 [(set t.Ty:$dst, (op addr:$addr, (t.Ty t.ImmNode:$b)))]>,2252 Requires<preds>;2253 }2254}2255 2256multiclass F_ATOMIC_3<RegTyInfo t, string op_str, SDPatternOperator op, SDNode atomic> {2257 defvar asm_str = "atom${sem:sem}${scope:scope}${addsp:addsp}" # op_str;2258 2259 let mayLoad = 1, mayStore = 1, hasSideEffects = 1 in {2260 def _rr : BasicFlagsNVPTXInst<(outs t.RC:$dst),2261 (ins ADDR:$addr, t.RC:$b, t.RC:$c),2262 (ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp),2263 asm_str>;2264 2265 def _ir : BasicFlagsNVPTXInst<(outs t.RC:$dst),2266 (ins ADDR:$addr, t.Imm:$b, t.RC:$c),2267 (ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp),2268 asm_str>;2269 2270 def _ri : BasicFlagsNVPTXInst<(outs t.RC:$dst),2271 (ins ADDR:$addr, t.RC:$b, t.Imm:$c),2272 (ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp),2273 asm_str>;2274 2275 def _ii : BasicFlagsNVPTXInst<(outs t.RC:$dst),2276 (ins ADDR:$addr, t.Imm:$b, t.Imm:$c),2277 (ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp),2278 asm_str>;2279 }2280 2281 defvar GetSem = SDNodeXForm<atomic, [{2282 return getI32Imm(getMemOrder(cast<MemSDNode>(N)), SDLoc(N));2283 }]>;2284 2285 defvar GetScope = SDNodeXForm<atomic, [{2286 return getI32Imm(getAtomicScope(cast<MemSDNode>(N)), SDLoc(N));2287 }]>;2288 2289 defvar GetAddSp = SDNodeXForm<atomic, [{2290 return getI32Imm(getAddrSpace(cast<MemSDNode>(N)), SDLoc(N));2291 }]>;2292 2293 def : Pat<(op:$this addr:$addr, t.Ty:$b, t.Ty:$c),2294 (!cast<Instruction>(NAME # _rr) ADDR:$addr, t.Ty:$b, t.Ty:$c, (GetSem $this), (GetScope $this), (GetAddSp $this))>;2295 2296 def : Pat<(op:$this addr:$addr, (t.Ty t.ImmNode:$b), t.Ty:$c),2297 (!cast<Instruction>(NAME # _ir) ADDR:$addr, (t.Ty t.ImmNode:$b), t.Ty:$c, (GetSem $this), (GetScope $this), (GetAddSp $this))>;2298 2299 def : Pat<(op:$this addr:$addr, t.Ty:$b, (t.Ty t.ImmNode:$c)),2300 (!cast<Instruction>(NAME # _ri) ADDR:$addr, t.Ty:$b, (t.Ty t.ImmNode:$c), (GetSem $this), (GetScope $this), (GetAddSp $this))>;2301 2302 def : Pat<(op:$this addr:$addr, (t.Ty t.ImmNode:$b), (t.Ty t.ImmNode:$c)),2303 (!cast<Instruction>(NAME # _ii) ADDR:$addr, (t.Ty t.ImmNode:$b), (t.Ty t.ImmNode:$c), (GetSem $this), (GetScope $this), (GetAddSp $this))>;2304}2305 2306multiclass F_ATOMIC_2_AS<RegTyInfo t, SDPatternOperator frag, string op_str, list<Predicate> preds = []> {2307 defvar frag_pat = (frag node:$a, node:$b);2308 defm _G : F_ATOMIC_2<t, "", ".global", op_str, ATOMIC_GLOBAL_CHK<frag_pat>, preds>;2309 defm _S : F_ATOMIC_2<t, "", ".shared", op_str, ATOMIC_SHARED_CHK<frag_pat>, preds>;2310 defm _S_C : F_ATOMIC_2<t, "", ".shared::cluster", op_str, ATOMIC_SHARED_CLUSTER_CHK<frag_pat>, !listconcat([hasClusters], preds)>;2311 defm _GEN : F_ATOMIC_2<t, "", "", op_str, ATOMIC_GENERIC_CHK<frag_pat>, preds>;2312}2313 2314// atom_add2315defm INT_PTX_ATOM_ADD_32 : F_ATOMIC_2_AS<I32RT, atomic_load_add_i32, "add.u32">;2316defm INT_PTX_ATOM_ADD_64 : F_ATOMIC_2_AS<I64RT, atomic_load_add_i64, "add.u64">;2317 2318defm INT_PTX_ATOM_ADD_F16 : F_ATOMIC_2_AS<F16RT, atomic_load_fadd, "add.noftz.f16", [hasSM<70>, hasPTX<63>]>;2319defm INT_PTX_ATOM_ADD_BF16 : F_ATOMIC_2_AS<BF16RT, atomic_load_fadd, "add.noftz.bf16", [hasSM<90>, hasPTX<78>]>;2320defm INT_PTX_ATOM_ADD_F32 : F_ATOMIC_2_AS<F32RT, atomic_load_fadd, "add.f32">;2321defm INT_PTX_ATOM_ADD_F64 : F_ATOMIC_2_AS<F64RT, atomic_load_fadd, "add.f64", [hasAtomAddF64]>;2322 2323// atom_swap2324defm INT_PTX_ATOM_SWAP_32 : F_ATOMIC_2_AS<I32RT, atomic_swap_i32, "exch.b32">;2325defm INT_PTX_ATOM_SWAP_64 : F_ATOMIC_2_AS<I64RT, atomic_swap_i64, "exch.b64">;2326 2327// atom_max2328defm INT_PTX_ATOMIC_MAX_32 : F_ATOMIC_2_AS<I32RT, atomic_load_max_i32, "max.s32">;2329defm INT_PTX_ATOMIC_MAX_64 : F_ATOMIC_2_AS<I64RT, atomic_load_max_i64, "max.s64", [hasSM<32>]>;2330defm INT_PTX_ATOMIC_UMAX_32 : F_ATOMIC_2_AS<I32RT, atomic_load_umax_i32, "max.u32">;2331defm INT_PTX_ATOMIC_UMAX_64 : F_ATOMIC_2_AS<I64RT, atomic_load_umax_i64, "max.u64", [hasSM<32>]>;2332 2333// atom_min2334defm INT_PTX_ATOMIC_MIN_32 : F_ATOMIC_2_AS<I32RT, atomic_load_min_i32, "min.s32">;2335defm INT_PTX_ATOMIC_MIN_64 : F_ATOMIC_2_AS<I64RT, atomic_load_min_i64, "min.s64", [hasSM<32>]>;2336defm INT_PTX_ATOMIC_UMIN_32 : F_ATOMIC_2_AS<I32RT, atomic_load_umin_i32, "min.u32">;2337defm INT_PTX_ATOMIC_UMIN_64 : F_ATOMIC_2_AS<I64RT, atomic_load_umin_i64, "min.u64", [hasSM<32>]>;2338 2339// atom_inc atom_dec2340defm INT_PTX_ATOM_INC_32 : F_ATOMIC_2_AS<I32RT, atomic_load_uinc_wrap_i32, "inc.u32">;2341defm INT_PTX_ATOM_DEC_32 : F_ATOMIC_2_AS<I32RT, atomic_load_udec_wrap_i32, "dec.u32">;2342 2343// atom_and2344defm INT_PTX_ATOM_AND_32 : F_ATOMIC_2_AS<I32RT, atomic_load_and_i32, "and.b32">;2345defm INT_PTX_ATOM_AND_64 : F_ATOMIC_2_AS<I64RT, atomic_load_and_i64, "and.b64", [hasSM<32>]>;2346 2347// atom_or2348defm INT_PTX_ATOM_OR_32 : F_ATOMIC_2_AS<I32RT, atomic_load_or_i32, "or.b32">;2349defm INT_PTX_ATOM_OR_64 : F_ATOMIC_2_AS<I64RT, atomic_load_or_i64, "or.b64", [hasSM<32>]>;2350 2351// atom_xor2352defm INT_PTX_ATOM_XOR_32 : F_ATOMIC_2_AS<I32RT, atomic_load_xor_i32, "xor.b32">;2353defm INT_PTX_ATOM_XOR_64 : F_ATOMIC_2_AS<I64RT, atomic_load_xor_i64, "xor.b64", [hasSM<32>]>;2354 2355 2356// Define atom.cas for all combinations of size x addrspace x memory order2357// supported in PTX *and* on the hardware.2358foreach t = [I16RT, I32RT, I64RT] in {2359 defvar atomic_cmp_swap_pat = !cast<PatFrag>("atomic_cmp_swap_i"#t.Size);2360 defm INT_PTX_ATOM_CAS_#t.Size2361 : F_ATOMIC_3<t, ".cas.b"#t.Size, atomic_cmp_swap_pat, atomic_cmp_swap>;2362}2363 2364// Support for scoped atomic operations. Matches2365// int_nvvm_atomic_{op}_{space}_{type}_{scope}2366// and converts it into the appropriate instruction.2367// NOTE: not all possible combinations are implemented2368// 'space' is limited to generic as it's the only one needed to support CUDA.2369// 'scope' = 'gpu' is default and is handled by regular atomic instructions.2370 2371// Define instruction variants for all addressing modes.2372 2373// Constructs intrinsic name and instruction asm strings.2374multiclass ATOM2N_impl<string OpStr, string IntTypeStr, string TypeStr,2375 string ScopeStr, string SpaceStr,2376 RegTyInfo t, list<Predicate> Preds> {2377 defm "" : F_ATOMIC_2<t,2378 as_str = !if(!eq(ScopeStr, "gpu"), "", "." # ScopeStr),2379 sem_str = !if(!eq(SpaceStr, "gen"), "", "." # SpaceStr),2380 op_str = OpStr # "." # TypeStr,2381 op = !cast<Intrinsic>(2382 "int_nvvm_atomic_" # OpStr2383 # "_" # SpaceStr # "_" # IntTypeStr2384 # !if(!empty(ScopeStr), "", "_" # ScopeStr)),2385 preds = Preds>;2386}2387 2388// Constructs variants for different scopes of atomic op.2389multiclass ATOM2S_impl<string OpStr, string IntTypeStr, string TypeStr,2390 RegTyInfo t, list<Predicate> Preds> {2391 // .gpu scope is default and is currently covered by existing2392 // atomics w/o explicitly specified scope.2393 foreach scope = ["cta", "sys"] in {2394 // For now we only need variants for generic space pointers.2395 foreach space = ["gen"] in {2396 defm _#scope#space : ATOM2N_impl<OpStr, IntTypeStr, TypeStr, scope, space,2397 t, !listconcat(Preds, [hasAtomScope])>;2398 }2399 }2400}2401 2402multiclass F_ATOMIC_3_INTRINSIC_PATTERN<RegTyInfo t, string OpStr, string InstructionName> {2403 foreach scope = ["cta", "sys"] in {2404 foreach space = ["gen"] in {2405 defvar intrinsic = !cast<SDPatternOperator>("int_nvvm_atomic_" # OpStr # "_" # space # "_i_" # scope);2406 def : Pat<(t.Ty (intrinsic addr:$addr, t.Ty:$b, t.Ty:$c)),2407 (!cast<Instruction>(InstructionName # "_rr") ADDR:$addr, t.Ty:$b, t.Ty:$c, Ordering_not_atomic, !cast<PatLeaf>("Scope_" # scope), !cast<PatLeaf>("AddrSpace_" # space))>;2408 2409 def : Pat<(t.Ty (intrinsic addr:$addr, (t.Ty t.ImmNode:$b), t.Ty:$c)),2410 (!cast<Instruction>(InstructionName # "_ir") ADDR:$addr, (t.Ty t.ImmNode:$b), t.Ty:$c, Ordering_not_atomic, !cast<PatLeaf>("Scope_" # scope), !cast<PatLeaf>("AddrSpace_" # space))>;2411 2412 def : Pat<(t.Ty (intrinsic addr:$addr, t.Ty:$b, (t.Ty t.ImmNode:$c))),2413 (!cast<Instruction>(InstructionName # "_ri") ADDR:$addr, t.Ty:$b, (t.Ty t.ImmNode:$c), Ordering_not_atomic, !cast<PatLeaf>("Scope_" # scope), !cast<PatLeaf>("AddrSpace_" # space))>;2414 2415 def : Pat<(t.Ty (intrinsic addr:$addr, (t.Ty t.ImmNode:$b), (t.Ty t.ImmNode:$c))),2416 (!cast<Instruction>(InstructionName # "_ii") ADDR:$addr, (t.Ty t.ImmNode:$b), (t.Ty t.ImmNode:$c), Ordering_not_atomic, !cast<PatLeaf>("Scope_" # scope), !cast<PatLeaf>("AddrSpace_" # space))>;2417 }2418 }2419}2420 2421// atom.add2422multiclass ATOM2_add_impl<string OpStr> {2423 defm _s32 : ATOM2S_impl<OpStr, "i", "s32", I32RT, []>;2424 defm _u32 : ATOM2S_impl<OpStr, "i", "u32", I32RT, []>;2425 defm _u64 : ATOM2S_impl<OpStr, "i", "u64", I64RT, []>;2426 defm _bf16 : ATOM2S_impl<OpStr, "f", "bf16", BF16RT, [hasSM<90>, hasPTX<78>]>;2427 defm _f16 : ATOM2S_impl<OpStr, "f", "f16", F16RT, []>;2428 defm _f32 : ATOM2S_impl<OpStr, "f", "f32", F32RT, []>;2429 defm _f64 : ATOM2S_impl<OpStr, "f", "f64", F64RT, []>;2430}2431 2432// atom.{and,or,xor}2433multiclass ATOM2_bitwise_impl<string OpStr> {2434 defm _b32 : ATOM2S_impl<OpStr, "i", "b32", I32RT, []>;2435 defm _b64 : ATOM2S_impl<OpStr, "i", "b64", I64RT, [hasAtomBitwise64]>;2436}2437 2438// atom.exch2439multiclass ATOM2_exch_impl<string OpStr> {2440 defm _b32 : ATOM2S_impl<OpStr, "i", "b32", I32RT, []>;2441 defm _b64 : ATOM2S_impl<OpStr, "i", "b64", I64RT, []>;2442}2443 2444// atom.{min,max}2445multiclass ATOM2_minmax_impl<string OpStr> {2446 defm _s32 : ATOM2S_impl<OpStr, "i", "s32", I32RT, []>;2447 defm _u32 : ATOM2S_impl<OpStr, "i", "u32", I32RT, []>;2448 defm _s64 : ATOM2S_impl<OpStr, "i", "s64", I64RT, [hasAtomMinMax64]>;2449 defm _u64 : ATOM2S_impl<OpStr, "i", "u64", I64RT, [hasAtomMinMax64]>;2450}2451 2452// atom.{inc,dec}2453multiclass ATOM2_incdec_impl<string OpStr> {2454 defm _u32 : ATOM2S_impl<OpStr, "i", "u32", I32RT, []>;2455}2456 2457// atom.cas2458multiclass ATOM3_cas_impl<string OpStr> {2459 defm _b16 : F_ATOMIC_3_INTRINSIC_PATTERN<I16RT, OpStr, "INT_PTX_ATOM_CAS_16">;2460 defm _b32 : F_ATOMIC_3_INTRINSIC_PATTERN<I32RT, OpStr, "INT_PTX_ATOM_CAS_32">;2461 defm _b64 : F_ATOMIC_3_INTRINSIC_PATTERN<I64RT, OpStr, "INT_PTX_ATOM_CAS_64">;2462}2463 2464defm INT_PTX_SATOM_ADD : ATOM2_add_impl<"add">;2465defm INT_PTX_SATOM_AND : ATOM2_bitwise_impl<"and">;2466defm INT_PTX_SATOM_CAS : ATOM3_cas_impl<"cas">;2467defm INT_PTX_SATOM_DEC : ATOM2_incdec_impl<"dec">;2468defm INT_PTX_SATOM_EXCH : ATOM2_exch_impl<"exch">;2469defm INT_PTX_SATOM_INC : ATOM2_incdec_impl<"inc">;2470defm INT_PTX_SATOM_MAX : ATOM2_minmax_impl<"max">;2471defm INT_PTX_SATOM_MIN : ATOM2_minmax_impl<"min">;2472defm INT_PTX_SATOM_OR : ATOM2_bitwise_impl<"or">;2473defm INT_PTX_SATOM_XOR : ATOM2_bitwise_impl<"xor">;2474 2475// atom.*.b1282476 2477let mayLoad = true, mayStore = true, hasSideEffects = true,2478 Predicates = [hasAtomSwap128] in {2479 def ATOM_CAS_B128 :2480 NVPTXInst<2481 (outs B64:$dst0, B64:$dst1),2482 (ins ADDR:$addr, B64:$cmp0, B64:$cmp1, B64:$swap0, B64:$swap1,2483 AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp),2484 "{{\n\t"2485 ".reg .b128 cmp, swap, dst;\n\t"2486 "mov.b128 cmp, {$cmp0, $cmp1};\n\t"2487 "mov.b128 swap, {$swap0, $swap1};\n\t"2488 "atom${sem:sem}${scope:scope}${addsp:addsp}.cas.b128 dst, [$addr], cmp, swap;\n\t"2489 "mov.b128 {$dst0, $dst1}, dst;\n\t"2490 "}}">;2491 2492 def ATOM_EXCH_B128 :2493 NVPTXInst<2494 (outs B64:$dst0, B64:$dst1),2495 (ins ADDR:$addr, B64:$amt0, B64:$amt1,2496 AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp),2497 "{{\n\t"2498 ".reg .b128 amt, dst;\n\t"2499 "mov.b128 amt, {$amt0, $amt1};\n\t"2500 "atom${sem:sem}${scope:scope}${addsp:addsp}.exch.b128 dst, [$addr], amt;\n\t"2501 "mov.b128 {$dst0, $dst1}, dst;\n\t"2502 "}}">;2503}2504 2505 2506//-----------------------------------2507// Support for ldu on sm_20 or later2508//-----------------------------------2509 2510// Don't annotate ldu instructions as mayLoad, as they load from memory that is2511// read-only in a kernel.2512 2513// Scalar2514 2515class LDU_G<NVPTXRegClass regclass>2516 : NVPTXInst<(outs regclass:$result), (ins i32imm:$fromWidth, ADDR:$src),2517 "ldu.global.b$fromWidth \t$result, [$src];">;2518 2519def LDU_GLOBAL_i16 : LDU_G<B16>;2520def LDU_GLOBAL_i32 : LDU_G<B32>;2521def LDU_GLOBAL_i64 : LDU_G<B64>;2522 2523// vector2524 2525// Elementized vector ldu2526class VLDU_G_ELE_V2<NVPTXRegClass regclass>2527 : NVPTXInst<(outs regclass:$dst1, regclass:$dst2),2528 (ins i32imm:$fromWidth, ADDR:$src),2529 "ldu.global.v2.b$fromWidth \t{{$dst1, $dst2}}, [$src];">;2530 2531 2532class VLDU_G_ELE_V4<NVPTXRegClass regclass>2533 : NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3, regclass:$dst4),2534 (ins i32imm:$fromWidth, ADDR:$src),2535 "ldu.global.v4.b$fromWidth \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];">;2536 2537 2538def LDU_GLOBAL_v2i16 : VLDU_G_ELE_V2<B16>;2539def LDU_GLOBAL_v2i32 : VLDU_G_ELE_V2<B32>;2540def LDU_GLOBAL_v2i64 : VLDU_G_ELE_V2<B64>;2541 2542def LDU_GLOBAL_v4i16 : VLDU_G_ELE_V4<B16>;2543def LDU_GLOBAL_v4i32 : VLDU_G_ELE_V4<B32>;2544 2545 2546//-----------------------------------2547// Support for ldg on sm_35 or later2548//-----------------------------------2549 2550// Don't annotate ld.global.nc as mayLoad, because these loads go through the2551// non-coherent texture cache, and therefore the values read must be read-only2552// during the lifetime of the kernel.2553 2554class LDG_G<NVPTXRegClass regclass>2555 : NVPTXInst<(outs regclass:$result),2556 (ins AtomicCode:$Sign, i32imm:$fromWidth,2557 UsedBytesMask:$usedBytes, ADDR:$src),2558 "${usedBytes}"2559 "ld.global.nc.${Sign:sign}$fromWidth \t$result, [$src];">;2560 2561def LD_GLOBAL_NC_i16 : LDG_G<B16>;2562def LD_GLOBAL_NC_i32 : LDG_G<B32>;2563def LD_GLOBAL_NC_i64 : LDG_G<B64>;2564 2565// vector2566 2567// Elementized vector ldg2568class VLDG_G_ELE_V2<NVPTXRegClass regclass> :2569 NVPTXInst<(outs regclass:$dst1, regclass:$dst2),2570 (ins AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,2571 ADDR:$src),2572 "${usedBytes}"2573 "ld.global.nc.v2.${Sign:sign}$fromWidth \t{{$dst1, $dst2}}, [$src];">;2574 2575 2576class VLDG_G_ELE_V4<NVPTXRegClass regclass> :2577 NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3, regclass:$dst4), 2578 (ins AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,2579 ADDR:$src),2580 "${usedBytes}"2581 "ld.global.nc.v4.${Sign:sign}$fromWidth \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];">;2582 2583class VLDG_G_ELE_V8<NVPTXRegClass regclass> :2584 NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3, regclass:$dst4,2585 regclass:$dst5, regclass:$dst6, regclass:$dst7, regclass:$dst8),2586 (ins AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,2587 ADDR:$src),2588 "${usedBytes}"2589 "ld.global.nc.v8.${Sign:sign}$fromWidth \t{{$dst1, $dst2, $dst3, $dst4, $dst5, $dst6, $dst7, $dst8}}, [$src];">;2590 2591// FIXME: 8-bit LDG should be fixed once LDG/LDU nodes are made into proper loads.2592def LD_GLOBAL_NC_v2i16 : VLDG_G_ELE_V2<B16>;2593def LD_GLOBAL_NC_v2i32 : VLDG_G_ELE_V2<B32>;2594def LD_GLOBAL_NC_v2i64 : VLDG_G_ELE_V2<B64>;2595 2596def LD_GLOBAL_NC_v4i16 : VLDG_G_ELE_V4<B16>;2597def LD_GLOBAL_NC_v4i32 : VLDG_G_ELE_V4<B32>;2598 2599def LD_GLOBAL_NC_v4i64 : VLDG_G_ELE_V4<B64>;2600def LD_GLOBAL_NC_v8i32 : VLDG_G_ELE_V8<B32>;2601 2602multiclass NG_TO_G<string Str, bit Supports32 = 1, list<Predicate> Preds = []> {2603 if Supports32 then2604 def "" : BasicNVPTXInst<(outs B32:$result), (ins B32:$src),2605 "cvta." # Str # ".u32">, Requires<Preds>;2606 2607 def _64 : BasicNVPTXInst<(outs B64:$result), (ins B64:$src),2608 "cvta." # Str # ".u64">, Requires<Preds>;2609}2610 2611multiclass G_TO_NG<string Str, bit Supports32 = 1, list<Predicate> Preds = []> {2612 if Supports32 then2613 def "" : BasicNVPTXInst<(outs B32:$result), (ins B32:$src),2614 "cvta.to." # Str # ".u32">, Requires<Preds>;2615 2616 def _64 : BasicNVPTXInst<(outs B64:$result), (ins B64:$src),2617 "cvta.to." # Str # ".u64">, Requires<Preds>;2618}2619 2620foreach space = ["local", "shared", "global", "const", "param"] in {2621 defm cvta_#space : NG_TO_G<space>;2622 defm cvta_to_#space : G_TO_NG<space>;2623}2624 2625defm cvta_shared_cluster : NG_TO_G<"shared::cluster", false, [hasClusters]>;2626defm cvta_to_shared_cluster : G_TO_NG<"shared::cluster", false, [hasClusters]>;2627 2628 2629// nvvm.move intrinsicc2630def nvvm_move_i16 : BasicNVPTXInst<(outs B16:$r), (ins B16:$s),2631 "mov.b16",2632 [(set i16:$r,2633 (int_nvvm_move_i16 i16:$s))]>;2634def nvvm_move_i32 : BasicNVPTXInst<(outs B32:$r), (ins B32:$s),2635 "mov.b32",2636 [(set i32:$r,2637 (int_nvvm_move_i32 i32:$s))]>;2638def nvvm_move_i64 : BasicNVPTXInst<(outs B64:$r), (ins B64:$s),2639 "mov.b64",2640 [(set i64:$r,2641 (int_nvvm_move_i64 i64:$s))]>;2642def nvvm_move_float : BasicNVPTXInst<(outs B32:$r), (ins B32:$s),2643 "mov.f32",2644 [(set f32:$r,2645 (int_nvvm_move_float f32:$s))]>;2646def nvvm_move_double : BasicNVPTXInst<(outs B64:$r), (ins B64:$s),2647 "mov.f64",2648 [(set f64:$r,2649 (int_nvvm_move_double f64:$s))]>;2650def nvvm_move_ptr32 : BasicNVPTXInst<(outs B32:$r), (ins B32:$s),2651 "mov.u32",2652 [(set i32:$r,2653 (int_nvvm_move_ptr i32:$s))]>;2654def nvvm_move_ptr64 : BasicNVPTXInst<(outs B64:$r), (ins B64:$s),2655 "mov.u64",2656 [(set i64:$r,2657 (int_nvvm_move_ptr i64:$s))]>;2658 2659// @TODO: Are these actually needed, or will we always just see symbols2660// copied to registers first?2661/*def nvvm_move_sym32 : NVPTXInst<(outs B32:$r), (ins ADDR_base:$s),2662 "mov.u32 \t$r, $s;",2663 [(set B32:$r,2664 (int_nvvm_move_ptr texternalsym:$s))]>;2665def nvvm_move_sym64 : NVPTXInst<(outs B64:$r), (ins ADDR_base:$s),2666 "mov.u64 \t$r, $s;",2667 [(set B64:$r,2668 (int_nvvm_move_ptr texternalsym:$s))]>;*/2669 2670def texsurf_handles2671 : BasicNVPTXInst<(outs B64:$result), (ins ADDR_base:$src), "mov.u64">;2672def : Pat<(int_nvvm_texsurf_handle_internal globaladdr:$src),2673 (texsurf_handles (to_tglobaladdr $src))>;2674 2675//-----------------------------------2676// Compiler Error Warn2677// - Just ignore them in codegen2678//-----------------------------------2679 2680def INT_NVVM_COMPILER_WARN_32 : NVPTXInst<(outs), (ins B32:$a),2681 "// llvm.nvvm.compiler.warn()",2682 [(int_nvvm_compiler_warn i32:$a)]>;2683def INT_NVVM_COMPILER_WARN_64 : NVPTXInst<(outs), (ins B64:$a),2684 "// llvm.nvvm.compiler.warn()",2685 [(int_nvvm_compiler_warn i64:$a)]>;2686def INT_NVVM_COMPILER_ERROR_32 : NVPTXInst<(outs), (ins B32:$a),2687 "// llvm.nvvm.compiler.error()",2688 [(int_nvvm_compiler_error i32:$a)]>;2689def INT_NVVM_COMPILER_ERROR_64 : NVPTXInst<(outs), (ins B64:$a),2690 "// llvm.nvvm.compiler.error()",2691 [(int_nvvm_compiler_error i64:$a)]>;2692 2693 2694// isspacep2695 2696multiclass ISSPACEP<string suffix, Intrinsic Intr, list<Predicate> Preds = []> {2697 def _32: BasicNVPTXInst<(outs B1:$d), (ins B32:$a),2698 "isspacep." # suffix,2699 [(set i1:$d, (Intr i32:$a))]>,2700 Requires<Preds>;2701 def _64: BasicNVPTXInst<(outs B1:$d), (ins B64:$a),2702 "isspacep." # suffix,2703 [(set i1:$d, (Intr i64:$a))]>,2704 Requires<Preds>;2705}2706 2707defm isspace_const : ISSPACEP<"const", int_nvvm_isspacep_const, [hasPTX<31>]>;2708defm isspace_global : ISSPACEP<"global", int_nvvm_isspacep_global>;2709defm isspace_local : ISSPACEP<"local", int_nvvm_isspacep_local>;2710defm isspace_shared : ISSPACEP<"shared", int_nvvm_isspacep_shared>;2711defm isspace_shared_cluster : ISSPACEP<"shared::cluster",2712 int_nvvm_isspacep_shared_cluster,2713 [hasPTX<78>, hasSM<90>]>;2714 2715// Special register reads2716def MOV_SPECIAL : BasicNVPTXInst<(outs B32:$d),2717 (ins SpecialRegs:$r),2718 "mov.b32", []>;2719 2720def : Pat<(int_nvvm_read_ptx_sreg_envreg0), (MOV_SPECIAL ENVREG0)>;2721def : Pat<(int_nvvm_read_ptx_sreg_envreg1), (MOV_SPECIAL ENVREG1)>;2722def : Pat<(int_nvvm_read_ptx_sreg_envreg2), (MOV_SPECIAL ENVREG2)>;2723def : Pat<(int_nvvm_read_ptx_sreg_envreg3), (MOV_SPECIAL ENVREG3)>;2724def : Pat<(int_nvvm_read_ptx_sreg_envreg4), (MOV_SPECIAL ENVREG4)>;2725def : Pat<(int_nvvm_read_ptx_sreg_envreg5), (MOV_SPECIAL ENVREG5)>;2726def : Pat<(int_nvvm_read_ptx_sreg_envreg6), (MOV_SPECIAL ENVREG6)>;2727def : Pat<(int_nvvm_read_ptx_sreg_envreg7), (MOV_SPECIAL ENVREG7)>;2728def : Pat<(int_nvvm_read_ptx_sreg_envreg8), (MOV_SPECIAL ENVREG8)>;2729def : Pat<(int_nvvm_read_ptx_sreg_envreg9), (MOV_SPECIAL ENVREG9)>;2730def : Pat<(int_nvvm_read_ptx_sreg_envreg10), (MOV_SPECIAL ENVREG10)>;2731def : Pat<(int_nvvm_read_ptx_sreg_envreg11), (MOV_SPECIAL ENVREG11)>;2732def : Pat<(int_nvvm_read_ptx_sreg_envreg12), (MOV_SPECIAL ENVREG12)>;2733def : Pat<(int_nvvm_read_ptx_sreg_envreg13), (MOV_SPECIAL ENVREG13)>;2734def : Pat<(int_nvvm_read_ptx_sreg_envreg14), (MOV_SPECIAL ENVREG14)>;2735def : Pat<(int_nvvm_read_ptx_sreg_envreg15), (MOV_SPECIAL ENVREG15)>;2736def : Pat<(int_nvvm_read_ptx_sreg_envreg16), (MOV_SPECIAL ENVREG16)>;2737def : Pat<(int_nvvm_read_ptx_sreg_envreg17), (MOV_SPECIAL ENVREG17)>;2738def : Pat<(int_nvvm_read_ptx_sreg_envreg18), (MOV_SPECIAL ENVREG18)>;2739def : Pat<(int_nvvm_read_ptx_sreg_envreg19), (MOV_SPECIAL ENVREG19)>;2740def : Pat<(int_nvvm_read_ptx_sreg_envreg20), (MOV_SPECIAL ENVREG20)>;2741def : Pat<(int_nvvm_read_ptx_sreg_envreg21), (MOV_SPECIAL ENVREG21)>;2742def : Pat<(int_nvvm_read_ptx_sreg_envreg22), (MOV_SPECIAL ENVREG22)>;2743def : Pat<(int_nvvm_read_ptx_sreg_envreg23), (MOV_SPECIAL ENVREG23)>;2744def : Pat<(int_nvvm_read_ptx_sreg_envreg24), (MOV_SPECIAL ENVREG24)>;2745def : Pat<(int_nvvm_read_ptx_sreg_envreg25), (MOV_SPECIAL ENVREG25)>;2746def : Pat<(int_nvvm_read_ptx_sreg_envreg26), (MOV_SPECIAL ENVREG26)>;2747def : Pat<(int_nvvm_read_ptx_sreg_envreg27), (MOV_SPECIAL ENVREG27)>;2748def : Pat<(int_nvvm_read_ptx_sreg_envreg28), (MOV_SPECIAL ENVREG28)>;2749def : Pat<(int_nvvm_read_ptx_sreg_envreg29), (MOV_SPECIAL ENVREG29)>;2750def : Pat<(int_nvvm_read_ptx_sreg_envreg30), (MOV_SPECIAL ENVREG30)>;2751def : Pat<(int_nvvm_read_ptx_sreg_envreg31), (MOV_SPECIAL ENVREG31)>;2752 2753//-----------------------------------2754// Texture Intrinsics2755//-----------------------------------2756 2757// NOTE: For Fermi support, any new texture/surface/sampler intrinsics must be2758// also defined in NVPTXReplaceImageHandles.cpp2759 2760// texmode_independent2761let IsTex = true, IsTexModeUnified = false in {2762// Texture fetch instructions using handles2763 2764class TEX_1D_base<string inst, dag texsamp, list<dag> pattern = []>2765 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),2766 !con(texsamp, (ins B32:$x)),2767 inst # " \t\\{$r, $g, $b, $a\\}, [$t, $s, \\{$x\\}];",2768 pattern>;2769 2770multiclass TEX_1D<string inst, Intrinsic intr> {2771 def _RR : TEX_1D_base<inst, (ins B64:$t, B64:$s),2772 [(set B32:$r, B32:$g, B32:$b, B32:$a,2773 (intr i64:$t, i64:$s, B32:$x))]>;2774 def _RI : TEX_1D_base<inst, (ins B64:$t, i64imm:$s)>;2775 def _IR : TEX_1D_base<inst, (ins i64imm:$t, B64:$s)>;2776 def _II : TEX_1D_base<inst, (ins i64imm:$t, i64imm:$s)>;2777}2778 2779defm TEX_1D_F32_S32 : TEX_1D<"tex.1d.v4.f32.s32", int_nvvm_tex_1d_v4f32_s32>;2780defm TEX_1D_F32_F32 : TEX_1D<"tex.1d.v4.f32.f32", int_nvvm_tex_1d_v4f32_f32>;2781defm TEX_1D_S32_S32 : TEX_1D<"tex.1d.v4.s32.s32", int_nvvm_tex_1d_v4s32_s32>;2782defm TEX_1D_S32_F32 : TEX_1D<"tex.1d.v4.s32.f32", int_nvvm_tex_1d_v4s32_f32>;2783defm TEX_1D_U32_S32 : TEX_1D<"tex.1d.v4.u32.s32", int_nvvm_tex_1d_v4u32_s32>;2784defm TEX_1D_U32_F32 : TEX_1D<"tex.1d.v4.u32.f32", int_nvvm_tex_1d_v4u32_f32>;2785 2786class TEX_1D_LEVEL_base<string inst, dag texsamp, list<dag> pattern = []>2787 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),2788 !con(texsamp, (ins B32:$x, B32:$lod)),2789 inst # " \t\\{$r, $g, $b, $a\\}, [$t, $s, \\{$x\\}], $lod;",2790 pattern>;2791 2792multiclass TEX_1D_LEVEL<string inst, Intrinsic intr> {2793 def _RR : TEX_1D_LEVEL_base<inst, (ins B64:$t, B64:$s),2794 [(set B32:$r, B32:$g, B32:$b, B32:$a,2795 (intr i64:$t, i64:$s, B32:$x, B32:$lod))]>;2796 def _RI : TEX_1D_LEVEL_base<inst, (ins B64:$t, i64imm:$s)>;2797 def _IR : TEX_1D_LEVEL_base<inst, (ins i64imm:$t, B64:$s)>;2798 def _II : TEX_1D_LEVEL_base<inst, (ins i64imm:$t, i64imm:$s)>;2799}2800 2801defm TEX_1D_F32_F32_LEVEL :2802 TEX_1D_LEVEL<"tex.level.1d.v4.f32.f32", int_nvvm_tex_1d_level_v4f32_f32>;2803defm TEX_1D_S32_F32_LEVEL :2804 TEX_1D_LEVEL<"tex.level.1d.v4.s32.f32", int_nvvm_tex_1d_level_v4s32_f32>;2805defm TEX_1D_U32_F32_LEVEL :2806 TEX_1D_LEVEL<"tex.level.1d.v4.u32.f32", int_nvvm_tex_1d_level_v4u32_f32>;2807 2808class TEX_1D_GRAD_base<string inst, dag texsamp, list<dag> pattern = []>2809 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),2810 !con(texsamp, (ins B32:$x, B32:$gradx, B32:$grady)),2811 inst # " \t\\{$r, $g, $b, $a\\}, [$t, $s, \\{$x\\}],"2812 " \\{$gradx\\}, \\{$grady\\};",2813 pattern>;2814 2815multiclass TEX_1D_GRAD<string inst, Intrinsic intr> {2816 def _RR : TEX_1D_GRAD_base<inst, (ins B64:$t, B64:$s),2817 [(set B32:$r, B32:$g, B32:$b, B32:$a,2818 (intr i64:$t, i64:$s, B32:$x, B32:$gradx, B32:$grady))]>;2819 def _RI : TEX_1D_GRAD_base<inst, (ins B64:$t, i64imm:$s)>;2820 def _IR : TEX_1D_GRAD_base<inst, (ins i64imm:$t, B64:$s)>;2821 def _II : TEX_1D_GRAD_base<inst, (ins i64imm:$t, i64imm:$s)>;2822}2823 2824defm TEX_1D_F32_F32_GRAD2825 : TEX_1D_GRAD<"tex.grad.1d.v4.f32.f32", int_nvvm_tex_1d_grad_v4f32_f32>;2826defm TEX_1D_S32_F32_GRAD2827 : TEX_1D_GRAD<"tex.grad.1d.v4.s32.f32", int_nvvm_tex_1d_grad_v4s32_f32>;2828defm TEX_1D_U32_F32_GRAD2829 : TEX_1D_GRAD<"tex.grad.1d.v4.u32.f32", int_nvvm_tex_1d_grad_v4u32_f32>;2830 2831class TEX_1D_ARRAY_base<string inst, dag texsamp, list<dag> pattern = []>2832 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),2833 !con(texsamp, (ins B32:$l, B32:$x)),2834 inst # " \t\\{$r, $g, $b, $a\\}, [$t, $s, \\{$l, $x\\}];",2835 pattern>;2836 2837multiclass TEX_1D_ARRAY<string inst, Intrinsic intr> {2838 def _RR : TEX_1D_ARRAY_base<inst, (ins B64:$t, B64:$s),2839 [(set B32:$r, B32:$g, B32:$b, B32:$a,2840 (intr i64:$t, i64:$s, B32:$l, B32:$x))]>;2841 def _RI : TEX_1D_ARRAY_base<inst, (ins B64:$t, i64imm:$s)>;2842 def _IR : TEX_1D_ARRAY_base<inst, (ins i64imm:$t, B64:$s)>;2843 def _II : TEX_1D_ARRAY_base<inst, (ins i64imm:$t, i64imm:$s)>;2844}2845 2846defm TEX_1D_ARRAY_F32_F322847 : TEX_1D_ARRAY<"tex.a1d.v4.f32.f32", int_nvvm_tex_1d_array_v4f32_f32>;2848defm TEX_1D_ARRAY_F32_S322849 : TEX_1D_ARRAY<"tex.a1d.v4.f32.s32", int_nvvm_tex_1d_array_v4f32_s32>;2850defm TEX_1D_ARRAY_S32_S322851 : TEX_1D_ARRAY<"tex.a1d.v4.s32.s32", int_nvvm_tex_1d_array_v4s32_s32>;2852defm TEX_1D_ARRAY_S32_F322853 : TEX_1D_ARRAY<"tex.a1d.v4.s32.f32", int_nvvm_tex_1d_array_v4s32_f32>;2854defm TEX_1D_ARRAY_U32_S322855 : TEX_1D_ARRAY<"tex.a1d.v4.u32.s32", int_nvvm_tex_1d_array_v4u32_s32>;2856defm TEX_1D_ARRAY_U32_F322857 : TEX_1D_ARRAY<"tex.a1d.v4.u32.f32", int_nvvm_tex_1d_array_v4u32_f32>;2858 2859class TEX_1D_ARRAY_LEVEL_base<string inst, dag texsamp, list<dag> pattern = []>2860 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),2861 !con(texsamp, (ins B32:$l, B32:$x, B32:$lod)),2862 inst # " \t\\{$r, $g, $b, $a\\},"2863 " [$t, $s, \\{$l, $x\\}], $lod;",2864 pattern>;2865 2866multiclass TEX_1D_ARRAY_LEVEL<string inst, Intrinsic intr> {2867 def _RR : TEX_1D_ARRAY_LEVEL_base<inst, (ins B64:$t, B64:$s),2868 [(set B32:$r, B32:$g, B32:$b, B32:$a,2869 (intr i64:$t, i64:$s, B32:$l, B32:$x, B32:$lod))]>;2870 def _RI : TEX_1D_ARRAY_LEVEL_base<inst, (ins B64:$t, i64imm:$s)>;2871 def _IR : TEX_1D_ARRAY_LEVEL_base<inst, (ins i64imm:$t, B64:$s)>;2872 def _II : TEX_1D_ARRAY_LEVEL_base<inst, (ins i64imm:$t, i64imm:$s)>;2873}2874 2875defm TEX_1D_ARRAY_F32_F32_LEVEL2876 : TEX_1D_ARRAY_LEVEL<"tex.level.a1d.v4.f32.f32", int_nvvm_tex_1d_array_level_v4f32_f32>;2877defm TEX_1D_ARRAY_S32_F32_LEVEL2878 : TEX_1D_ARRAY_LEVEL<"tex.level.a1d.v4.s32.f32", int_nvvm_tex_1d_array_level_v4s32_f32>;2879defm TEX_1D_ARRAY_U32_F32_LEVEL2880 : TEX_1D_ARRAY_LEVEL<"tex.level.a1d.v4.u32.f32", int_nvvm_tex_1d_array_level_v4u32_f32>;2881 2882class TEX_1D_ARRAY_GRAD_base<string inst, dag texsamp, list<dag> pattern = []>2883 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),2884 !con(texsamp, (ins B32:$l, B32:$x, B32:$gradx, B32:$grady)),2885 inst # " \t\\{$r, $g, $b, $a\\}, [$t, $s, \\{$l, $x\\}],"2886 " \\{$gradx\\}, \\{$grady\\};",2887 pattern>;2888 2889multiclass TEX_1D_ARRAY_GRAD<string inst, Intrinsic intr> {2890 def _RR : TEX_1D_ARRAY_GRAD_base<inst, (ins B64:$t, B64:$s),2891 [(set B32:$r, B32:$g, B32:$b, B32:$a,2892 (intr i64:$t, i64:$s, B32:$l, B32:$x,2893 B32:$gradx, B32:$grady))]>;2894 def _RI : TEX_1D_ARRAY_GRAD_base<inst, (ins B64:$t, i64imm:$s)>;2895 def _IR : TEX_1D_ARRAY_GRAD_base<inst, (ins i64imm:$t, B64:$s)>;2896 def _II : TEX_1D_ARRAY_GRAD_base<inst, (ins i64imm:$t, i64imm:$s)>;2897}2898 2899defm TEX_1D_ARRAY_F32_F32_GRAD2900 : TEX_1D_ARRAY_GRAD<"tex.grad.a1d.v4.f32.f32", int_nvvm_tex_1d_array_grad_v4f32_f32>;2901defm TEX_1D_ARRAY_S32_F32_GRAD2902 : TEX_1D_ARRAY_GRAD<"tex.grad.a1d.v4.s32.f32", int_nvvm_tex_1d_array_grad_v4s32_f32>;2903defm TEX_1D_ARRAY_U32_F32_GRAD2904 : TEX_1D_ARRAY_GRAD<"tex.grad.a1d.v4.u32.f32", int_nvvm_tex_1d_array_grad_v4u32_f32>;2905 2906class TEX_2D_base<string inst, dag texsamp, list<dag> pattern = []>2907 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),2908 !con(texsamp, (ins B32:$x, B32:$y)),2909 inst # " \t\\{$r, $g, $b, $a\\}, [$t, $s, \\{$x, $y\\}];",2910 pattern>;2911 2912multiclass TEX_2D<string inst, Intrinsic intr> {2913 def _RR : TEX_2D_base<inst, (ins B64:$t, B64:$s),2914 [(set B32:$r, B32:$g, B32:$b, B32:$a,2915 (intr i64:$t, i64:$s, B32:$x, B32:$y))]>;2916 def _RI : TEX_2D_base<inst, (ins B64:$t, i64imm:$s)>;2917 def _IR : TEX_2D_base<inst, (ins i64imm:$t, B64:$s)>;2918 def _II : TEX_2D_base<inst, (ins i64imm:$t, i64imm:$s)>;2919}2920 2921defm TEX_2D_F32_F32 : TEX_2D<"tex.2d.v4.f32.f32", int_nvvm_tex_2d_v4f32_f32>;2922defm TEX_2D_F32_S32 : TEX_2D<"tex.2d.v4.f32.s32", int_nvvm_tex_2d_v4f32_s32>;2923defm TEX_2D_S32_S32 : TEX_2D<"tex.2d.v4.s32.s32", int_nvvm_tex_2d_v4s32_s32>;2924defm TEX_2D_S32_F32 : TEX_2D<"tex.2d.v4.s32.f32", int_nvvm_tex_2d_v4s32_f32>;2925defm TEX_2D_U32_S32 : TEX_2D<"tex.2d.v4.u32.s32", int_nvvm_tex_2d_v4u32_s32>;2926defm TEX_2D_U32_F32 : TEX_2D<"tex.2d.v4.u32.f32", int_nvvm_tex_2d_v4u32_f32>;2927 2928class TEX_2D_LEVEL_base<string inst, dag texsamp, list<dag> pattern = []>2929 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),2930 !con(texsamp, (ins B32:$x, B32:$y, B32:$lod)),2931 inst # " \t\\{$r, $g, $b, $a\\},"2932 " [$t, $s, \\{$x, $y\\}], $lod;",2933 pattern>;2934 2935multiclass TEX_2D_LEVEL<string inst, Intrinsic intr> {2936 def _RR : TEX_2D_LEVEL_base<inst, (ins B64:$t, B64:$s),2937 [(set B32:$r, B32:$g, B32:$b, B32:$a,2938 (intr i64:$t, i64:$s, B32:$x, B32:$y, B32:$lod))]>;2939 def _RI : TEX_2D_LEVEL_base<inst, (ins B64:$t, i64imm:$s)>;2940 def _IR : TEX_2D_LEVEL_base<inst, (ins i64imm:$t, B64:$s)>;2941 def _II : TEX_2D_LEVEL_base<inst, (ins i64imm:$t, i64imm:$s)>;2942}2943 2944defm TEX_2D_F32_F32_LEVEL :2945 TEX_2D_LEVEL<"tex.level.2d.v4.f32.f32", int_nvvm_tex_2d_level_v4f32_f32>;2946defm TEX_2D_S32_F32_LEVEL :2947 TEX_2D_LEVEL<"tex.level.2d.v4.s32.f32", int_nvvm_tex_2d_level_v4s32_f32>;2948defm TEX_2D_U32_F32_LEVEL :2949 TEX_2D_LEVEL<"tex.level.2d.v4.u32.f32", int_nvvm_tex_2d_level_v4u32_f32>;2950 2951class TEX_2D_GRAD_base<string inst, dag texsamp, list<dag> pattern = []>2952 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),2953 !con(texsamp, (ins B32:$x, B32:$y,2954 B32:$gradx0, B32:$gradx1,2955 B32:$grady0, B32:$grady1)),2956 inst # " \t\\{$r, $g, $b, $a\\}, [$t, $s, \\{$x, $y\\}],"2957 " \\{$gradx0, $gradx1\\}, \\{$grady0, $grady1\\};",2958 pattern>;2959 2960multiclass TEX_2D_GRAD<string inst, Intrinsic intr> {2961 def _RR : TEX_2D_GRAD_base<inst, (ins B64:$t, B64:$s),2962 [(set B32:$r, B32:$g, B32:$b, B32:$a,2963 (intr i64:$t, i64:$s, B32:$x, B32:$y,2964 B32:$gradx0, B32:$gradx1,2965 B32:$grady0, B32:$grady1))]>;2966 def _RI : TEX_2D_GRAD_base<inst, (ins B64:$t, i64imm:$s)>;2967 def _IR : TEX_2D_GRAD_base<inst, (ins i64imm:$t, B64:$s)>;2968 def _II : TEX_2D_GRAD_base<inst, (ins i64imm:$t, i64imm:$s)>;2969}2970 2971defm TEX_2D_F32_F32_GRAD :2972 TEX_2D_GRAD<"tex.grad.2d.v4.f32.f32", int_nvvm_tex_2d_grad_v4f32_f32>;2973defm TEX_2D_S32_F32_GRAD :2974 TEX_2D_GRAD<"tex.grad.2d.v4.s32.f32", int_nvvm_tex_2d_grad_v4s32_f32>;2975defm TEX_2D_U32_F32_GRAD :2976 TEX_2D_GRAD<"tex.grad.2d.v4.u32.f32", int_nvvm_tex_2d_grad_v4u32_f32>;2977 2978class TEX_2D_ARRAY_base<string inst, dag texsamp, list<dag> pattern = []>2979 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),2980 !con(texsamp, (ins B32:$l, B32:$x, B32:$y)),2981 inst # " \t\\{$r, $g, $b, $a\\},"2982 " [$t, $s, \\{$l, $x, $y, $y\\}];",2983 pattern>;2984 2985multiclass TEX_2D_ARRAY<string inst, Intrinsic intr> {2986 def _RR : TEX_2D_ARRAY_base<inst, (ins B64:$t, B64:$s),2987 [(set B32:$r, B32:$g, B32:$b, B32:$a,2988 (intr i64:$t, i64:$s, B32:$l, B32:$x, B32:$y))]>;2989 def _RI : TEX_2D_ARRAY_base<inst, (ins B64:$t, i64imm:$s)>;2990 def _IR : TEX_2D_ARRAY_base<inst, (ins i64imm:$t, B64:$s)>;2991 def _II : TEX_2D_ARRAY_base<inst, (ins i64imm:$t, i64imm:$s)>;2992}2993 2994defm TEX_2D_ARRAY_F32_F322995 : TEX_2D_ARRAY<"tex.a2d.v4.f32.f32", int_nvvm_tex_2d_array_v4f32_f32>;2996defm TEX_2D_ARRAY_F32_S322997 : TEX_2D_ARRAY<"tex.a2d.v4.f32.s32", int_nvvm_tex_2d_array_v4f32_s32>;2998defm TEX_2D_ARRAY_S32_S322999 : TEX_2D_ARRAY<"tex.a2d.v4.s32.s32", int_nvvm_tex_2d_array_v4s32_s32>;3000defm TEX_2D_ARRAY_S32_F323001 : TEX_2D_ARRAY<"tex.a2d.v4.s32.f32", int_nvvm_tex_2d_array_v4s32_f32>;3002defm TEX_2D_ARRAY_U32_S323003 : TEX_2D_ARRAY<"tex.a2d.v4.u32.s32", int_nvvm_tex_2d_array_v4u32_s32>;3004defm TEX_2D_ARRAY_U32_F323005 : TEX_2D_ARRAY<"tex.a2d.v4.u32.f32", int_nvvm_tex_2d_array_v4u32_f32>;3006 3007class TEX_2D_ARRAY_LEVEL_base<string inst, dag texsamp, list<dag> pattern = []>3008 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3009 !con(texsamp, (ins B32:$l, B32:$x, B32:$y, B32:$lod)),3010 inst # " \t\\{$r, $g, $b, $a\\},"3011 " [$t, $s, \\{$l, $x, $y, $y\\}], $lod;",3012 pattern>;3013 3014multiclass TEX_2D_ARRAY_LEVEL<string inst, Intrinsic intr> {3015 def _RR : TEX_2D_ARRAY_LEVEL_base<inst, (ins B64:$t, B64:$s),3016 [(set B32:$r, B32:$g, B32:$b, B32:$a,3017 (intr i64:$t, i64:$s, B32:$l, B32:$x, B32:$y, B32:$lod))]>;3018 def _RI : TEX_2D_ARRAY_LEVEL_base<inst, (ins B64:$t, i64imm:$s)>;3019 def _IR : TEX_2D_ARRAY_LEVEL_base<inst, (ins i64imm:$t, B64:$s)>;3020 def _II : TEX_2D_ARRAY_LEVEL_base<inst, (ins i64imm:$t, i64imm:$s)>;3021}3022 3023defm TEX_2D_ARRAY_F32_F32_LEVEL3024 : TEX_2D_ARRAY_LEVEL<"tex.level.a2d.v4.f32.f32", int_nvvm_tex_2d_array_level_v4f32_f32>;3025defm TEX_2D_ARRAY_S32_F32_LEVEL3026 : TEX_2D_ARRAY_LEVEL<"tex.level.a2d.v4.s32.f32", int_nvvm_tex_2d_array_level_v4s32_f32>;3027defm TEX_2D_ARRAY_U32_F32_LEVEL3028 : TEX_2D_ARRAY_LEVEL<"tex.level.a2d.v4.u32.f32", int_nvvm_tex_2d_array_level_v4u32_f32>;3029 3030class TEX_2D_ARRAY_GRAD_base<string inst, dag texsamp, list<dag> pattern = []>3031 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3032 !con(texsamp, (ins B32:$l, B32:$x, B32:$y,3033 B32:$gradx0, B32:$gradx1,3034 B32:$grady0, B32:$grady1)),3035 inst # " \t\\{$r, $g, $b, $a\\},"3036 " [$t, $s, \\{$l, $x, $y, $y\\}],"3037 " \\{$gradx0, $gradx1\\}, \\{$grady0, $grady1\\};",3038 pattern>;3039 3040multiclass TEX_2D_ARRAY_GRAD<string inst, Intrinsic intr> {3041 def _RR : TEX_2D_ARRAY_GRAD_base<inst, (ins B64:$t, B64:$s),3042 [(set B32:$r, B32:$g, B32:$b, B32:$a,3043 (intr i64:$t, i64:$s, B32:$l, B32:$x, B32:$y,3044 B32:$gradx0, B32:$gradx1,3045 B32:$grady0, B32:$grady1))]>;3046 def _RI : TEX_2D_ARRAY_GRAD_base<inst, (ins B64:$t, i64imm:$s)>;3047 def _IR : TEX_2D_ARRAY_GRAD_base<inst, (ins i64imm:$t, B64:$s)>;3048 def _II : TEX_2D_ARRAY_GRAD_base<inst, (ins i64imm:$t, i64imm:$s)>;3049}3050 3051defm TEX_2D_ARRAY_F32_F32_GRAD3052 : TEX_2D_ARRAY_GRAD<"tex.grad.a2d.v4.f32.f32", int_nvvm_tex_2d_array_grad_v4f32_f32>;3053defm TEX_2D_ARRAY_S32_F32_GRAD3054 : TEX_2D_ARRAY_GRAD<"tex.grad.a2d.v4.s32.f32", int_nvvm_tex_2d_array_grad_v4s32_f32>;3055defm TEX_2D_ARRAY_U32_F32_GRAD3056 : TEX_2D_ARRAY_GRAD<"tex.grad.a2d.v4.u32.f32", int_nvvm_tex_2d_array_grad_v4u32_f32>;3057 3058class TEX_3D_base<string inst, dag texsamp, list<dag> pattern = []>3059 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3060 !con(texsamp, (ins B32:$x, B32:$y, B32:$z)),3061 inst # " \t\\{$r, $g, $b, $a\\},"3062 " [$t, $s, \\{$x, $y, $z, $z\\}];",3063 pattern>;3064 3065multiclass TEX_3D<string inst, Intrinsic intr> {3066 def _RR : TEX_3D_base<inst, (ins B64:$t, B64:$s),3067 [(set B32:$r, B32:$g, B32:$b, B32:$a,3068 (intr i64:$t, i64:$s, B32:$x, B32:$y, B32:$z))]>;3069 def _RI : TEX_3D_base<inst, (ins B64:$t, i64imm:$s)>;3070 def _IR : TEX_3D_base<inst, (ins i64imm:$t, B64:$s)>;3071 def _II : TEX_3D_base<inst, (ins i64imm:$t, i64imm:$s)>;3072}3073 3074defm TEX_3D_F32_F32 : TEX_3D<"tex.3d.v4.f32.f32", int_nvvm_tex_3d_v4f32_f32>;3075defm TEX_3D_F32_S32 : TEX_3D<"tex.3d.v4.f32.s32", int_nvvm_tex_3d_v4f32_s32>;3076defm TEX_3D_S32_S32 : TEX_3D<"tex.3d.v4.s32.s32", int_nvvm_tex_3d_v4s32_s32>;3077defm TEX_3D_S32_F32 : TEX_3D<"tex.3d.v4.s32.f32", int_nvvm_tex_3d_v4s32_f32>;3078defm TEX_3D_U32_S32 : TEX_3D<"tex.3d.v4.u32.s32", int_nvvm_tex_3d_v4u32_s32>;3079defm TEX_3D_U32_F32 : TEX_3D<"tex.3d.v4.u32.f32", int_nvvm_tex_3d_v4u32_f32>;3080 3081class TEX_3D_LEVEL_base<string inst, dag texsamp, list<dag> pattern = []>3082 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3083 !con(texsamp, (ins B32:$x, B32:$y, B32:$z, B32:$lod)),3084 inst # " \t\\{$r, $g, $b, $a\\},"3085 " [$t, $s, \\{$x, $y, $z, $z\\}], $lod;",3086 pattern>;3087 3088multiclass TEX_3D_LEVEL<string inst, Intrinsic intr> {3089 def _RR : TEX_3D_LEVEL_base<inst, (ins B64:$t, B64:$s),3090 [(set B32:$r, B32:$g, B32:$b, B32:$a,3091 (intr i64:$t, i64:$s, B32:$x, B32:$y, B32:$z, B32:$lod))]>;3092 def _RI : TEX_3D_LEVEL_base<inst, (ins B64:$t, i64imm:$s)>;3093 def _IR : TEX_3D_LEVEL_base<inst, (ins i64imm:$t, B64:$s)>;3094 def _II : TEX_3D_LEVEL_base<inst, (ins i64imm:$t, i64imm:$s)>;3095}3096 3097defm TEX_3D_F32_F32_LEVEL3098 : TEX_3D_LEVEL<"tex.level.3d.v4.f32.f32", int_nvvm_tex_3d_level_v4f32_f32>;3099defm TEX_3D_S32_F32_LEVEL3100 : TEX_3D_LEVEL<"tex.level.3d.v4.s32.f32", int_nvvm_tex_3d_level_v4s32_f32>;3101defm TEX_3D_U32_F32_LEVEL3102 : TEX_3D_LEVEL<"tex.level.3d.v4.u32.f32", int_nvvm_tex_3d_level_v4u32_f32>;3103 3104class TEX_3D_GRAD_base<string inst, dag texsamp, list<dag> pattern = []>3105 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3106 !con(texsamp, (ins B32:$x, B32:$y, B32:$z,3107 B32:$gradx0, B32:$gradx1,3108 B32:$gradx2, B32:$grady0,3109 B32:$grady1, B32:$grady2)),3110 inst # " \t\\{$r, $g, $b, $a\\},"3111 " [$t, $s, \\{$x, $y, $z, $z\\}],"3112 " \\{$gradx0, $gradx1, $gradx2, $gradx2\\},"3113 " \\{$grady0, $grady1, $grady2, $grady2\\};",3114 pattern>;3115 3116multiclass TEX_3D_GRAD<string inst, Intrinsic intr> {3117 def _RR : TEX_3D_GRAD_base<inst, (ins B64:$t, B64:$s),3118 [(set B32:$r, B32:$g, B32:$b, B32:$a,3119 (intr i64:$t, i64:$s, B32:$x, B32:$y, B32:$z,3120 B32:$gradx0, B32:$gradx1, B32:$gradx2,3121 B32:$grady0, B32:$grady1, B32:$grady2))]>;3122 def _RI : TEX_3D_GRAD_base<inst, (ins B64:$t, i64imm:$s)>;3123 def _IR : TEX_3D_GRAD_base<inst, (ins i64imm:$t, B64:$s)>;3124 def _II : TEX_3D_GRAD_base<inst, (ins i64imm:$t, i64imm:$s)>;3125}3126 3127defm TEX_3D_F32_F32_GRAD3128 : TEX_3D_GRAD<"tex.grad.3d.v4.f32.f32", int_nvvm_tex_3d_grad_v4f32_f32>;3129defm TEX_3D_S32_F32_GRAD3130 : TEX_3D_GRAD<"tex.grad.3d.v4.s32.f32", int_nvvm_tex_3d_grad_v4s32_f32>;3131defm TEX_3D_U32_F32_GRAD3132 : TEX_3D_GRAD<"tex.grad.3d.v4.u32.f32", int_nvvm_tex_3d_grad_v4u32_f32>;3133 3134class TEX_CUBE_base<string inst, dag texsamp, list<dag> pattern = []>3135 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3136 !con(texsamp, (ins B32:$x, B32:$y, B32:$z)),3137 inst # " \t\\{$r, $g, $b, $a\\},"3138 " [$t, $s, \\{$x, $y, $z, $z\\}];",3139 pattern>;3140 3141multiclass TEX_CUBE<string inst, Intrinsic intr> {3142 def _RR : TEX_CUBE_base<inst, (ins B64:$t, B64:$s),3143 [(set B32:$r, B32:$g, B32:$b, B32:$a,3144 (intr i64:$t, i64:$s, B32:$x, B32:$y, B32:$z))]>;3145 def _RI : TEX_CUBE_base<inst, (ins B64:$t, i64imm:$s)>;3146 def _IR : TEX_CUBE_base<inst, (ins i64imm:$t, B64:$s)>;3147 def _II : TEX_CUBE_base<inst, (ins i64imm:$t, i64imm:$s)>;3148}3149 3150defm TEX_CUBE_F32_F323151 : TEX_CUBE<"tex.cube.v4.f32.f32", int_nvvm_tex_cube_v4f32_f32>;3152defm TEX_CUBE_S32_F323153 : TEX_CUBE<"tex.cube.v4.s32.f32", int_nvvm_tex_cube_v4s32_f32>;3154defm TEX_CUBE_U32_F323155 : TEX_CUBE<"tex.cube.v4.u32.f32", int_nvvm_tex_cube_v4u32_f32>;3156 3157class TEX_CUBE_LEVEL_base<string inst, dag texsamp, list<dag> pattern = []>3158 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3159 !con(texsamp, (ins B32:$x, B32:$y, B32:$z, B32:$lod)),3160 inst # " \t\\{$r, $g, $b, $a\\},"3161 " [$t, $s, \\{$x, $y, $z, $z\\}], $lod;",3162 pattern>;3163 3164multiclass TEX_CUBE_LEVEL<string inst, Intrinsic intr> {3165 def _RR : TEX_CUBE_LEVEL_base<inst, (ins B64:$t, B64:$s),3166 [(set B32:$r, B32:$g, B32:$b, B32:$a,3167 (intr i64:$t, i64:$s, B32:$x, B32:$y, B32:$z,3168 B32:$lod))]>;3169 def _RI : TEX_CUBE_LEVEL_base<inst, (ins B64:$t, i64imm:$s)>;3170 def _IR : TEX_CUBE_LEVEL_base<inst, (ins i64imm:$t, B64:$s)>;3171 def _II : TEX_CUBE_LEVEL_base<inst, (ins i64imm:$t, i64imm:$s)>;3172}3173 3174defm TEX_CUBE_F32_F32_LEVEL3175 : TEX_CUBE_LEVEL<"tex.level.cube.v4.f32.f32", int_nvvm_tex_cube_level_v4f32_f32>;3176defm TEX_CUBE_S32_F32_LEVEL3177 : TEX_CUBE_LEVEL<"tex.level.cube.v4.s32.f32", int_nvvm_tex_cube_level_v4s32_f32>;3178defm TEX_CUBE_U32_F32_LEVEL3179 : TEX_CUBE_LEVEL<"tex.level.cube.v4.u32.f32", int_nvvm_tex_cube_level_v4u32_f32>;3180 3181class TEX_CUBE_ARRAY_base<string inst, dag texsamp, list<dag> pattern = []>3182 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3183 !con(texsamp, (ins B32:$l, B32:$x, B32:$y, B32:$z)),3184 inst # " \t\\{$r, $g, $b, $a\\},"3185 " [$t, $s, \\{$l, $x, $y, $z\\}];",3186 pattern>;3187 3188multiclass TEX_CUBE_ARRAY<string inst, Intrinsic intr> {3189 def _RR : TEX_CUBE_ARRAY_base<inst, (ins B64:$t, B64:$s),3190 [(set B32:$r, B32:$g, B32:$b, B32:$a,3191 (intr i64:$t, i64:$s, B32:$l, B32:$x, B32:$y, B32:$z))]>;3192 def _RI : TEX_CUBE_ARRAY_base<inst, (ins B64:$t, i64imm:$s)>;3193 def _IR : TEX_CUBE_ARRAY_base<inst, (ins i64imm:$t, B64:$s)>;3194 def _II : TEX_CUBE_ARRAY_base<inst, (ins i64imm:$t, i64imm:$s)>;3195}3196 3197defm TEX_CUBE_ARRAY_F32_F323198 : TEX_CUBE_ARRAY<"tex.acube.v4.f32.f32", int_nvvm_tex_cube_array_v4f32_f32>;3199defm TEX_CUBE_ARRAY_S32_F323200 : TEX_CUBE_ARRAY<"tex.acube.v4.s32.f32", int_nvvm_tex_cube_array_v4s32_f32>;3201defm TEX_CUBE_ARRAY_U32_F323202 : TEX_CUBE_ARRAY<"tex.acube.v4.u32.f32", int_nvvm_tex_cube_array_v4u32_f32>;3203 3204class TEX_CUBE_ARRAY_LEVEL_base<string inst, dag texsamp, list<dag> pattern = []>3205 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3206 !con(texsamp, (ins B32:$l, B32:$x, B32:$y, B32:$z, B32:$lod)),3207 inst # " \t\\{$r, $g, $b, $a\\},"3208 " [$t, $s, \\{$l, $x, $y, $z\\}], $lod;",3209 pattern>;3210 3211multiclass TEX_CUBE_ARRAY_LEVEL<string inst, Intrinsic intr> {3212 def _RR : TEX_CUBE_ARRAY_LEVEL_base<inst, (ins B64:$t, B64:$s),3213 [(set B32:$r, B32:$g, B32:$b, B32:$a,3214 (intr i64:$t, i64:$s, B32:$l, B32:$x, B32:$y, B32:$z,3215 B32:$lod))]>;3216 def _RI : TEX_CUBE_ARRAY_LEVEL_base<inst, (ins B64:$t, i64imm:$s)>;3217 def _IR : TEX_CUBE_ARRAY_LEVEL_base<inst, (ins i64imm:$t, B64:$s)>;3218 def _II : TEX_CUBE_ARRAY_LEVEL_base<inst, (ins i64imm:$t, i64imm:$s)>;3219}3220 3221defm TEX_CUBE_ARRAY_F32_F32_LEVEL3222 : TEX_CUBE_ARRAY_LEVEL<"tex.level.acube.v4.f32.f32",3223 int_nvvm_tex_cube_array_level_v4f32_f32>;3224defm TEX_CUBE_ARRAY_S32_F32_LEVEL3225 : TEX_CUBE_ARRAY_LEVEL<"tex.level.acube.v4.s32.f32",3226 int_nvvm_tex_cube_array_level_v4s32_f32>;3227defm TEX_CUBE_ARRAY_U32_F32_LEVEL3228 : TEX_CUBE_ARRAY_LEVEL<"tex.level.acube.v4.u32.f32",3229 int_nvvm_tex_cube_array_level_v4u32_f32>;3230 3231class TLD4_2D_base<string inst, dag texsamp, list<dag> pattern = []>3232 : NVPTXInst<(outs B32:$v0, B32:$v1, B32:$v2, B32:$v3),3233 !con(texsamp, (ins B32:$x, B32:$y)),3234 inst # " \t\\{$v0, $v1, $v2, $v3\\}, [$t, $s, \\{$x, $y\\}];",3235 pattern>;3236 3237multiclass TLD4_2D<string inst, Intrinsic intr> {3238 def _RR : TLD4_2D_base<inst, (ins B64:$t, B64:$s),3239 [(set B32:$v0, B32:$v1, B32:$v2, B32:$v3,3240 (intr i64:$t, i64:$s, B32:$x, B32:$y))]>;3241 def _RI : TLD4_2D_base<inst, (ins B64:$t, i64imm:$s)>;3242 def _IR : TLD4_2D_base<inst, (ins i64imm:$t, B64:$s)>;3243 def _II : TLD4_2D_base<inst, (ins i64imm:$t, i64imm:$s)>;3244}3245 3246defm TLD4_R_2D_F32_F323247 : TLD4_2D<"tld4.r.2d.v4.f32.f32", int_nvvm_tld4_r_2d_v4f32_f32>;3248defm TLD4_G_2D_F32_F323249 : TLD4_2D<"tld4.g.2d.v4.f32.f32", int_nvvm_tld4_g_2d_v4f32_f32>;3250defm TLD4_B_2D_F32_F323251 : TLD4_2D<"tld4.b.2d.v4.f32.f32", int_nvvm_tld4_b_2d_v4f32_f32>;3252defm TLD4_A_2D_F32_F323253 : TLD4_2D<"tld4.a.2d.v4.f32.f32", int_nvvm_tld4_a_2d_v4f32_f32>;3254 3255defm TLD4_R_2D_S32_F323256 : TLD4_2D<"tld4.r.2d.v4.s32.f32", int_nvvm_tld4_r_2d_v4s32_f32>;3257defm TLD4_G_2D_S32_F323258 : TLD4_2D<"tld4.g.2d.v4.s32.f32", int_nvvm_tld4_g_2d_v4s32_f32>;3259defm TLD4_B_2D_S32_F323260 : TLD4_2D<"tld4.b.2d.v4.s32.f32", int_nvvm_tld4_b_2d_v4s32_f32>;3261defm TLD4_A_2D_S32_F323262 : TLD4_2D<"tld4.a.2d.v4.s32.f32", int_nvvm_tld4_a_2d_v4s32_f32>;3263 3264defm TLD4_R_2D_U32_F323265 : TLD4_2D<"tld4.r.2d.v4.u32.f32", int_nvvm_tld4_r_2d_v4u32_f32>;3266defm TLD4_G_2D_U32_F323267 : TLD4_2D<"tld4.g.2d.v4.u32.f32", int_nvvm_tld4_g_2d_v4u32_f32>;3268defm TLD4_B_2D_U32_F323269 : TLD4_2D<"tld4.b.2d.v4.u32.f32", int_nvvm_tld4_b_2d_v4u32_f32>;3270defm TLD4_A_2D_U32_F323271 : TLD4_2D<"tld4.a.2d.v4.u32.f32", int_nvvm_tld4_a_2d_v4u32_f32>;3272 3273}3274 3275 3276// texmode_unified3277let IsTex = true, IsTexModeUnified = true in {3278// Texture fetch instructions using handles3279 3280class TEX_UNIFIED_1D_base<string inst, dag tex, list<dag> pattern = []>3281 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3282 !con(tex, (ins B32:$x)),3283 inst # " \t\\{$r, $g, $b, $a\\}, [$t, \\{$x\\}];",3284 pattern>;3285 3286multiclass TEX_UNIFIED_1D<string inst, Intrinsic intr> {3287 def _R : TEX_UNIFIED_1D_base<inst, (ins B64:$t),3288 [(set B32:$r, B32:$g, B32:$b, B32:$a, (intr i64:$t, B32:$x))]>;3289 def _I : TEX_UNIFIED_1D_base<inst, (ins i64imm:$t)>;3290}3291 3292defm TEX_UNIFIED_1D_F32_S323293 : TEX_UNIFIED_1D<"tex.1d.v4.f32.s32", int_nvvm_tex_unified_1d_v4f32_s32>;3294defm TEX_UNIFIED_1D_F32_F323295 : TEX_UNIFIED_1D<"tex.1d.v4.f32.f32", int_nvvm_tex_unified_1d_v4f32_f32>;3296defm TEX_UNIFIED_1D_S32_S323297 : TEX_UNIFIED_1D<"tex.1d.v4.s32.s32", int_nvvm_tex_unified_1d_v4s32_s32>;3298defm TEX_UNIFIED_1D_S32_F323299 : TEX_UNIFIED_1D<"tex.1d.v4.s32.f32", int_nvvm_tex_unified_1d_v4s32_f32>;3300defm TEX_UNIFIED_1D_U32_S323301 : TEX_UNIFIED_1D<"tex.1d.v4.u32.s32", int_nvvm_tex_unified_1d_v4u32_s32>;3302defm TEX_UNIFIED_1D_U32_F323303 : TEX_UNIFIED_1D<"tex.1d.v4.u32.f32", int_nvvm_tex_unified_1d_v4u32_f32>;3304 3305class TEX_UNIFIED_1D_LEVEL_base<string inst, dag tex, list<dag> pattern = []>3306 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3307 !con(tex, (ins B32:$x, B32:$lod)),3308 inst # " \t\\{$r, $g, $b, $a\\}, [$t, \\{$x\\}], $lod;",3309 pattern>;3310 3311multiclass TEX_UNIFIED_1D_LEVEL<string inst, Intrinsic intr> {3312 def _R : TEX_UNIFIED_1D_LEVEL_base<inst, (ins B64:$t),3313 [(set B32:$r, B32:$g, B32:$b, B32:$a,3314 (intr i64:$t, B32:$x, B32:$lod))]>;3315 def _I : TEX_UNIFIED_1D_LEVEL_base<inst, (ins i64imm:$t)>;3316}3317 3318defm TEX_UNIFIED_1D_F32_F32_LEVEL3319 : TEX_UNIFIED_1D_LEVEL<"tex.level.1d.v4.f32.f32", int_nvvm_tex_unified_1d_level_v4f32_f32>;3320defm TEX_UNIFIED_1D_S32_F32_LEVEL3321 : TEX_UNIFIED_1D_LEVEL<"tex.level.1d.v4.s32.f32", int_nvvm_tex_unified_1d_level_v4s32_f32>;3322defm TEX_UNIFIED_1D_U32_F32_LEVEL3323 : TEX_UNIFIED_1D_LEVEL<"tex.level.1d.v4.u32.f32", int_nvvm_tex_unified_1d_level_v4u32_f32>;3324 3325class TEX_UNIFIED_1D_GRAD_base<string inst, dag tex, list<dag> pattern = []>3326 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3327 !con(tex, (ins B32:$x, B32:$gradx, B32:$grady)),3328 inst # " \t\\{$r, $g, $b, $a\\},"3329 " [$t, \\{$x\\}], \\{$gradx\\}, \\{$grady\\};",3330 pattern>;3331 3332multiclass TEX_UNIFIED_1D_GRAD<string inst, Intrinsic intr> {3333 def _R : TEX_UNIFIED_1D_GRAD_base<inst, (ins B64:$t),3334 [(set B32:$r, B32:$g, B32:$b, B32:$a,3335 (intr i64:$t, B32:$x, B32:$gradx, B32:$grady))]>;3336 def _I : TEX_UNIFIED_1D_GRAD_base<inst, (ins i64imm:$t)>;3337}3338 3339defm TEX_UNIFIED_1D_F32_F32_GRAD3340 : TEX_UNIFIED_1D_GRAD<"tex.grad.1d.v4.f32.f32", int_nvvm_tex_unified_1d_grad_v4f32_f32>;3341defm TEX_UNIFIED_1D_S32_F32_GRAD3342 : TEX_UNIFIED_1D_GRAD<"tex.grad.1d.v4.s32.f32", int_nvvm_tex_unified_1d_grad_v4s32_f32>;3343defm TEX_UNIFIED_1D_U32_F32_GRAD3344 : TEX_UNIFIED_1D_GRAD<"tex.grad.1d.v4.u32.f32", int_nvvm_tex_unified_1d_grad_v4u32_f32>;3345 3346class TEX_UNIFIED_1D_ARRAY_base<string inst, dag tex, list<dag> pattern = []>3347 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3348 !con(tex, (ins B32:$l, B32:$x)),3349 inst # " \t\\{$r, $g, $b, $a\\}, [$t, \\{$l, $x\\}];",3350 pattern>;3351 3352multiclass TEX_UNIFIED_1D_ARRAY<string inst, Intrinsic intr> {3353 def _R : TEX_UNIFIED_1D_ARRAY_base<inst, (ins B64:$t),3354 [(set B32:$r, B32:$g, B32:$b, B32:$a, (intr i64:$t, B32:$l, B32:$x))]>;3355 def _I : TEX_UNIFIED_1D_ARRAY_base<inst, (ins i64imm:$t)>;3356}3357 3358defm TEX_UNIFIED_1D_ARRAY_F32_S323359 : TEX_UNIFIED_1D_ARRAY<"tex.a1d.v4.f32.s32", int_nvvm_tex_unified_1d_array_v4f32_s32>;3360defm TEX_UNIFIED_1D_ARRAY_F32_F323361 : TEX_UNIFIED_1D_ARRAY<"tex.a1d.v4.f32.f32", int_nvvm_tex_unified_1d_array_v4f32_f32>;3362defm TEX_UNIFIED_1D_ARRAY_S32_S323363 : TEX_UNIFIED_1D_ARRAY<"tex.a1d.v4.s32.s32", int_nvvm_tex_unified_1d_array_v4s32_s32>;3364defm TEX_UNIFIED_1D_ARRAY_S32_F323365 : TEX_UNIFIED_1D_ARRAY<"tex.a1d.v4.s32.f32", int_nvvm_tex_unified_1d_array_v4s32_f32>;3366defm TEX_UNIFIED_1D_ARRAY_U32_S323367 : TEX_UNIFIED_1D_ARRAY<"tex.a1d.v4.u32.s32", int_nvvm_tex_unified_1d_array_v4u32_s32>;3368defm TEX_UNIFIED_1D_ARRAY_U32_F323369 : TEX_UNIFIED_1D_ARRAY<"tex.a1d.v4.u32.f32", int_nvvm_tex_unified_1d_array_v4u32_f32>;3370 3371class TEX_UNIFIED_1D_ARRAY_LEVEL_base<string inst, dag tex, list<dag> pattern = []>3372 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3373 !con(tex, (ins B32:$l, B32:$x, B32:$lod)),3374 inst # " \t\\{$r, $g, $b, $a\\}, [$t, \\{$l, $x\\}], $lod;",3375 pattern>;3376 3377multiclass TEX_UNIFIED_1D_ARRAY_LEVEL<string inst, Intrinsic intr> {3378 def _R : TEX_UNIFIED_1D_ARRAY_LEVEL_base<inst, (ins B64:$t),3379 [(set B32:$r, B32:$g, B32:$b, B32:$a,3380 (intr i64:$t, B32:$l, B32:$x, B32:$lod))]>;3381 def _I : TEX_UNIFIED_1D_ARRAY_LEVEL_base<inst, (ins i64imm:$t)>;3382}3383 3384defm TEX_UNIFIED_1D_ARRAY_F32_F32_LEVEL3385 : TEX_UNIFIED_1D_ARRAY_LEVEL<"tex.level.a1d.v4.f32.f32",3386 int_nvvm_tex_unified_1d_array_level_v4f32_f32>;3387defm TEX_UNIFIED_1D_ARRAY_S32_F32_LEVEL3388 : TEX_UNIFIED_1D_ARRAY_LEVEL<"tex.level.a1d.v4.s32.f32",3389 int_nvvm_tex_unified_1d_array_level_v4s32_f32>;3390defm TEX_UNIFIED_1D_ARRAY_U32_F32_LEVEL3391 : TEX_UNIFIED_1D_ARRAY_LEVEL<"tex.level.a1d.v4.u32.f32",3392 int_nvvm_tex_unified_1d_array_level_v4u32_f32>;3393 3394class TEX_UNIFIED_1D_ARRAY_GRAD_base<string inst, dag tex, list<dag> pattern = []>3395 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3396 !con(tex, (ins B32:$l, B32:$x, B32:$gradx, B32:$grady)),3397 inst # " \t\\{$r, $g, $b, $a\\},"3398 " [$t, \\{$l, $x\\}], \\{$gradx\\}, \\{$grady\\};",3399 pattern>;3400 3401multiclass TEX_UNIFIED_1D_ARRAY_GRAD<string inst, Intrinsic intr> {3402 def _R : TEX_UNIFIED_1D_ARRAY_GRAD_base<inst, (ins B64:$t),3403 [(set B32:$r, B32:$g, B32:$b, B32:$a,3404 (intr i64:$t, B32:$l, B32:$x, B32:$gradx, B32:$grady))]>;3405 def _I : TEX_UNIFIED_1D_ARRAY_GRAD_base<inst, (ins i64imm:$t)>;3406}3407 3408defm TEX_UNIFIED_1D_ARRAY_F32_F32_GRAD3409 : TEX_UNIFIED_1D_ARRAY_GRAD<"tex.grad.a1d.v4.f32.f32",3410 int_nvvm_tex_unified_1d_array_grad_v4f32_f32>;3411defm TEX_UNIFIED_1D_ARRAY_S32_F32_GRAD3412 : TEX_UNIFIED_1D_ARRAY_GRAD<"tex.grad.a1d.v4.s32.f32",3413 int_nvvm_tex_unified_1d_array_grad_v4s32_f32>;3414defm TEX_UNIFIED_1D_ARRAY_U32_F32_GRAD3415 : TEX_UNIFIED_1D_ARRAY_GRAD<"tex.grad.a1d.v4.u32.f32",3416 int_nvvm_tex_unified_1d_array_grad_v4u32_f32>;3417 3418class TEX_UNIFIED_2D_base<string inst, dag tex, list<dag> pattern = []>3419 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3420 !con(tex, (ins B32:$x, B32:$y)),3421 inst # " \t\\{$r, $g, $b, $a\\}, [$t, \\{$x, $y\\}];",3422 pattern>;3423 3424multiclass TEX_UNIFIED_2D<string inst, Intrinsic intr> {3425 def _R : TEX_UNIFIED_2D_base<inst, (ins B64:$t),3426 [(set B32:$r, B32:$g, B32:$b, B32:$a,3427 (intr i64:$t, B32:$x, B32:$y))]>;3428 def _I : TEX_UNIFIED_2D_base<inst, (ins i64imm:$t)>;3429}3430 3431defm TEX_UNIFIED_2D_F32_S323432 : TEX_UNIFIED_2D<"tex.2d.v4.f32.s32", int_nvvm_tex_unified_2d_v4f32_s32>;3433defm TEX_UNIFIED_2D_F32_F323434 : TEX_UNIFIED_2D<"tex.2d.v4.f32.f32", int_nvvm_tex_unified_2d_v4f32_f32>;3435defm TEX_UNIFIED_2D_S32_S323436 : TEX_UNIFIED_2D<"tex.2d.v4.s32.s32", int_nvvm_tex_unified_2d_v4s32_s32>;3437defm TEX_UNIFIED_2D_S32_F323438 : TEX_UNIFIED_2D<"tex.2d.v4.s32.f32", int_nvvm_tex_unified_2d_v4s32_f32>;3439defm TEX_UNIFIED_2D_U32_S323440 : TEX_UNIFIED_2D<"tex.2d.v4.u32.s32", int_nvvm_tex_unified_2d_v4u32_s32>;3441defm TEX_UNIFIED_2D_U32_F323442 : TEX_UNIFIED_2D<"tex.2d.v4.u32.f32", int_nvvm_tex_unified_2d_v4u32_f32>;3443 3444class TEX_UNIFIED_2D_LEVEL_base<string inst, dag tex, list<dag> pattern = []>3445 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3446 !con(tex, (ins B32:$x, B32:$y, B32:$lod)),3447 inst # " \t\\{$r, $g, $b, $a\\}, [$t, \\{$x, $y\\}], $lod;",3448 pattern>;3449 3450multiclass TEX_UNIFIED_2D_LEVEL<string inst, Intrinsic intr> {3451 def _R : TEX_UNIFIED_2D_LEVEL_base<inst, (ins B64:$t),3452 [(set B32:$r, B32:$g, B32:$b, B32:$a,3453 (intr i64:$t, B32:$x, B32:$y, B32:$lod))]>;3454 def _I : TEX_UNIFIED_2D_LEVEL_base<inst, (ins i64imm:$t)>;3455}3456 3457defm TEX_UNIFIED_2D_F32_F32_LEVEL3458 : TEX_UNIFIED_2D_LEVEL<"tex.level.2d.v4.f32.f32", int_nvvm_tex_unified_2d_level_v4f32_f32>;3459defm TEX_UNIFIED_2D_S32_F32_LEVEL3460 : TEX_UNIFIED_2D_LEVEL<"tex.level.2d.v4.s32.f32", int_nvvm_tex_unified_2d_level_v4s32_f32>;3461defm TEX_UNIFIED_2D_U32_F32_LEVEL3462 : TEX_UNIFIED_2D_LEVEL<"tex.level.2d.v4.u32.f32", int_nvvm_tex_unified_2d_level_v4u32_f32>;3463 3464class TEX_UNIFIED_2D_GRAD_base<string inst, dag tex, list<dag> pattern = []>3465 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3466 !con(tex, (ins B32:$x, B32:$y,3467 B32:$gradx0, B32:$gradx1,3468 B32:$grady0, B32:$grady1)),3469 inst # " \t\\{$r, $g, $b, $a\\}, [$t, \\{$x, $y\\}],"3470 " \\{$gradx0, $gradx1\\}, \\{$grady0, $grady1\\};",3471 pattern>;3472multiclass TEX_UNIFIED_2D_GRAD<string inst, Intrinsic intr> {3473 def _R : TEX_UNIFIED_2D_GRAD_base<inst, (ins B64:$t),3474 [(set B32:$r, B32:$g, B32:$b, B32:$a,3475 (intr i64:$t, B32:$x, B32:$y,3476 B32:$gradx0, B32:$gradx1,3477 B32:$grady0, B32:$grady1))]>;3478 def _I : TEX_UNIFIED_2D_GRAD_base<inst, (ins i64imm:$t)>;3479}3480 3481defm TEX_UNIFIED_2D_F32_F32_GRAD3482 : TEX_UNIFIED_2D_GRAD<"tex.grad.2d.v4.f32.f32", int_nvvm_tex_unified_2d_grad_v4f32_f32>;3483defm TEX_UNIFIED_2D_S32_F32_GRAD3484 : TEX_UNIFIED_2D_GRAD<"tex.grad.2d.v4.s32.f32", int_nvvm_tex_unified_2d_grad_v4s32_f32>;3485defm TEX_UNIFIED_2D_U32_F32_GRAD3486 : TEX_UNIFIED_2D_GRAD<"tex.grad.2d.v4.u32.f32", int_nvvm_tex_unified_2d_grad_v4u32_f32>;3487 3488class TEX_UNIFIED_2D_ARRAY_base<string inst, dag tex, list<dag> pattern = []>3489 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3490 !con(tex, (ins B32:$l, B32:$x, B32:$y)),3491 inst # " \t\\{$r, $g, $b, $a\\}, [$t, \\{$l, $x, $y, $y\\}];",3492 pattern>;3493multiclass TEX_UNIFIED_2D_ARRAY<string inst, Intrinsic intr> {3494 def _R : TEX_UNIFIED_2D_ARRAY_base<inst, (ins B64:$t),3495 [(set B32:$r, B32:$g, B32:$b, B32:$a,3496 (intr i64:$t, B32:$l, B32:$x, B32:$y))]>;3497 def _I : TEX_UNIFIED_2D_ARRAY_base<inst, (ins i64imm:$t)>;3498}3499 3500defm TEX_UNIFIED_2D_ARRAY_F32_S323501 : TEX_UNIFIED_2D_ARRAY<"tex.a2d.v4.f32.s32", int_nvvm_tex_unified_2d_array_v4f32_s32>;3502defm TEX_UNIFIED_2D_ARRAY_F32_F323503 : TEX_UNIFIED_2D_ARRAY<"tex.a2d.v4.f32.f32", int_nvvm_tex_unified_2d_array_v4f32_f32>;3504defm TEX_UNIFIED_2D_ARRAY_S32_S323505 : TEX_UNIFIED_2D_ARRAY<"tex.a2d.v4.s32.s32", int_nvvm_tex_unified_2d_array_v4s32_s32>;3506defm TEX_UNIFIED_2D_ARRAY_S32_F323507 : TEX_UNIFIED_2D_ARRAY<"tex.a2d.v4.s32.f32", int_nvvm_tex_unified_2d_array_v4s32_f32>;3508defm TEX_UNIFIED_2D_ARRAY_U32_S323509 : TEX_UNIFIED_2D_ARRAY<"tex.a2d.v4.u32.s32", int_nvvm_tex_unified_2d_array_v4u32_s32>;3510defm TEX_UNIFIED_2D_ARRAY_U32_F323511 : TEX_UNIFIED_2D_ARRAY<"tex.a2d.v4.u32.f32", int_nvvm_tex_unified_2d_array_v4u32_f32>;3512 3513class TEX_UNIFIED_2D_ARRAY_LEVEL_base<string inst, dag tex, list<dag> pattern = []>3514 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3515 !con(tex, (ins B32:$l, B32:$x, B32:$y, B32:$lod)),3516 inst # " \t\\{$r, $g, $b, $a\\},"3517 " [$t, \\{$l, $x, $y, $y\\}], $lod;",3518 pattern>;3519multiclass TEX_UNIFIED_2D_ARRAY_LEVEL<string inst, Intrinsic intr> {3520 def _R : TEX_UNIFIED_2D_ARRAY_LEVEL_base<inst, (ins B64:$t),3521 [(set B32:$r, B32:$g, B32:$b, B32:$a,3522 (intr i64:$t, B32:$l, B32:$x, B32:$y, B32:$lod))]>;3523 def _I : TEX_UNIFIED_2D_ARRAY_LEVEL_base<inst, (ins i64imm:$t)>;3524}3525 3526defm TEX_UNIFIED_2D_ARRAY_F32_F32_LEVEL3527 : TEX_UNIFIED_2D_ARRAY_LEVEL<"tex.level.a2d.v4.f32.f32",3528 int_nvvm_tex_unified_2d_array_level_v4f32_f32>;3529defm TEX_UNIFIED_2D_ARRAY_S32_F32_LEVEL3530 : TEX_UNIFIED_2D_ARRAY_LEVEL<"tex.level.a2d.v4.s32.f32",3531 int_nvvm_tex_unified_2d_array_level_v4s32_f32>;3532defm TEX_UNIFIED_2D_ARRAY_U32_F32_LEVEL3533 : TEX_UNIFIED_2D_ARRAY_LEVEL<"tex.level.a2d.v4.u32.f32",3534 int_nvvm_tex_unified_2d_array_level_v4u32_f32>;3535 3536class TEX_UNIFIED_2D_ARRAY_GRAD_base<string inst, dag tex, list<dag> pattern = []>3537 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3538 !con(tex, (ins B32:$l, B32:$x, B32:$y,3539 B32:$gradx0, B32:$gradx1,3540 B32:$grady0, B32:$grady1)),3541 inst # " \t\\{$r, $g, $b, $a\\}, [$t, \\{$l, $x, $y, $y\\}],"3542 " \\{$gradx0, $gradx1\\}, \\{$grady0, $grady1\\};",3543 pattern>;3544multiclass TEX_UNIFIED_2D_ARRAY_GRAD<string inst, Intrinsic intr> {3545 def _R : TEX_UNIFIED_2D_ARRAY_GRAD_base<inst, (ins B64:$t),3546 [(set B32:$r, B32:$g, B32:$b, B32:$a,3547 (intr i64:$t, B32:$l, B32:$x, B32:$y,3548 B32:$gradx0, B32:$gradx1,3549 B32:$grady0, B32:$grady1))]>;3550 def _I : TEX_UNIFIED_2D_ARRAY_GRAD_base<inst, (ins i64imm:$t)>;3551}3552 3553defm TEX_UNIFIED_2D_ARRAY_F32_F32_GRAD3554 : TEX_UNIFIED_2D_ARRAY_GRAD<"tex.grad.a2d.v4.f32.f32",3555 int_nvvm_tex_unified_2d_array_grad_v4f32_f32>;3556defm TEX_UNIFIED_2D_ARRAY_S32_F32_GRAD3557 : TEX_UNIFIED_2D_ARRAY_GRAD<"tex.grad.a2d.v4.s32.f32",3558 int_nvvm_tex_unified_2d_array_grad_v4s32_f32>;3559defm TEX_UNIFIED_2D_ARRAY_U32_F32_GRAD3560 : TEX_UNIFIED_2D_ARRAY_GRAD<"tex.grad.a2d.v4.u32.f32",3561 int_nvvm_tex_unified_2d_array_grad_v4u32_f32>;3562 3563class TEX_UNIFIED_3D_base<string inst, dag tex, list<dag> pattern = []>3564 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3565 !con(tex, (ins B32:$x, B32:$y, B32:$z)),3566 inst # " \t\\{$r, $g, $b, $a\\}, [$t, \\{$x, $y, $z, $z\\}];",3567 pattern>;3568multiclass TEX_UNIFIED_3D<string inst, Intrinsic intr> {3569 def _R : TEX_UNIFIED_3D_base<inst, (ins B64:$t),3570 [(set B32:$r, B32:$g, B32:$b, B32:$a,3571 (intr i64:$t, B32:$x, B32:$y, B32:$z))]>;3572 def _I : TEX_UNIFIED_3D_base<inst, (ins i64imm:$t)>;3573}3574 3575defm TEX_UNIFIED_3D_F32_S323576 : TEX_UNIFIED_3D<"tex.3d.v4.f32.s32", int_nvvm_tex_unified_3d_v4f32_s32>;3577defm TEX_UNIFIED_3D_F32_F323578 : TEX_UNIFIED_3D<"tex.3d.v4.f32.f32", int_nvvm_tex_unified_3d_v4f32_f32>;3579defm TEX_UNIFIED_3D_S32_S323580 : TEX_UNIFIED_3D<"tex.3d.v4.s32.s32", int_nvvm_tex_unified_3d_v4s32_s32>;3581defm TEX_UNIFIED_3D_S32_F323582 : TEX_UNIFIED_3D<"tex.3d.v4.s32.f32", int_nvvm_tex_unified_3d_v4s32_f32>;3583defm TEX_UNIFIED_3D_U32_S323584 : TEX_UNIFIED_3D<"tex.3d.v4.u32.s32", int_nvvm_tex_unified_3d_v4u32_s32>;3585defm TEX_UNIFIED_3D_U32_F323586 : TEX_UNIFIED_3D<"tex.3d.v4.u32.f32", int_nvvm_tex_unified_3d_v4u32_f32>;3587 3588class TEX_UNIFIED_3D_LEVEL_base<string inst, dag tex, list<dag> pattern = []>3589 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3590 !con(tex, (ins B32:$x, B32:$y, B32:$z, B32:$lod)),3591 inst # " \t\\{$r, $g, $b, $a\\},"3592 " [$t, \\{$x, $y, $z, $z\\}], $lod;",3593 pattern>;3594multiclass TEX_UNIFIED_3D_LEVEL<string inst, Intrinsic intr> {3595 def _R : TEX_UNIFIED_3D_LEVEL_base<inst, (ins B64:$t),3596 [(set B32:$r, B32:$g, B32:$b, B32:$a,3597 (intr i64:$t, B32:$x, B32:$y, B32:$z, B32:$lod))]>;3598 def _I : TEX_UNIFIED_3D_LEVEL_base<inst, (ins i64imm:$t)>;3599}3600 3601defm TEX_UNIFIED_3D_F32_F32_LEVEL3602 : TEX_UNIFIED_3D_LEVEL<"tex.level.3d.v4.f32.f32", int_nvvm_tex_unified_3d_level_v4f32_f32>;3603defm TEX_UNIFIED_3D_S32_F32_LEVEL3604 : TEX_UNIFIED_3D_LEVEL<"tex.level.3d.v4.s32.f32", int_nvvm_tex_unified_3d_level_v4s32_f32>;3605defm TEX_UNIFIED_3D_U32_F32_LEVEL3606 : TEX_UNIFIED_3D_LEVEL<"tex.level.3d.v4.u32.f32", int_nvvm_tex_unified_3d_level_v4u32_f32>;3607 3608class TEX_UNIFIED_3D_GRAD_base<string inst, dag tex, list<dag> pattern = []>3609 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3610 !con(tex, (ins B32:$x, B32:$y, B32:$z,3611 B32:$gradx0, B32:$gradx1,3612 B32:$gradx2, B32:$grady0,3613 B32:$grady1, B32:$grady2)),3614 inst # " \t\\{$r, $g, $b, $a\\}, [$t, \\{$x, $y, $z, $z\\}],"3615 " \\{$gradx0, $gradx1, $gradx2, $gradx2\\},"3616 " \\{$grady0, $grady1, $grady2, $grady2\\};",3617 pattern>;3618multiclass TEX_UNIFIED_3D_GRAD<string inst, Intrinsic intr> {3619 def _R : TEX_UNIFIED_3D_GRAD_base<inst, (ins B64:$t),3620 [(set B32:$r, B32:$g, B32:$b, B32:$a,3621 (intr i64:$t, B32:$x, B32:$y, B32:$z,3622 B32:$gradx0, B32:$gradx1, B32:$gradx2,3623 B32:$grady0, B32:$grady1, B32:$grady2))]>;3624 def _I : TEX_UNIFIED_3D_GRAD_base<inst, (ins i64imm:$t)>;3625}3626 3627defm TEX_UNIFIED_3D_F32_F32_GRAD3628 : TEX_UNIFIED_3D_GRAD<"tex.grad.3d.v4.f32.f32", int_nvvm_tex_unified_3d_grad_v4f32_f32>;3629defm TEX_UNIFIED_3D_S32_F32_GRAD3630 : TEX_UNIFIED_3D_GRAD<"tex.grad.3d.v4.s32.f32", int_nvvm_tex_unified_3d_grad_v4s32_f32>;3631defm TEX_UNIFIED_3D_U32_F32_GRAD3632 : TEX_UNIFIED_3D_GRAD<"tex.grad.3d.v4.u32.f32", int_nvvm_tex_unified_3d_grad_v4u32_f32>;3633 3634class TEX_UNIFIED_CUBE_base<string inst, dag tex, list<dag> pattern = []>3635 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3636 !con(tex, (ins B32:$x, B32:$y, B32:$z)),3637 inst # " \t\\{$r, $g, $b, $a\\}, [$t, \\{$x, $y, $z, $z\\}];",3638 pattern>;3639multiclass TEX_UNIFIED_CUBE<string inst, Intrinsic intr> {3640 def _R : TEX_UNIFIED_CUBE_base<inst, (ins B64:$t),3641 [(set B32:$r, B32:$g, B32:$b, B32:$a,3642 (intr i64:$t, B32:$x, B32:$y, B32:$z))]>;3643 def _I : TEX_UNIFIED_CUBE_base<inst, (ins i64imm:$t)>;3644}3645 3646defm TEX_UNIFIED_CUBE_F32_F323647 : TEX_UNIFIED_CUBE<"tex.cube.v4.f32.f32", int_nvvm_tex_unified_cube_v4f32_f32>;3648defm TEX_UNIFIED_CUBE_S32_F323649 : TEX_UNIFIED_CUBE<"tex.cube.v4.s32.f32", int_nvvm_tex_unified_cube_v4s32_f32>;3650defm TEX_UNIFIED_CUBE_U32_F323651 : TEX_UNIFIED_CUBE<"tex.cube.v4.u32.f32", int_nvvm_tex_unified_cube_v4u32_f32>;3652 3653class TEX_UNIFIED_CUBE_LEVEL_base<string inst, dag tex, list<dag> pattern = []>3654 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3655 !con(tex, (ins B32:$x, B32:$y, B32:$z, B32:$lod)),3656 inst # " \t\\{$r, $g, $b, $a\\},"3657 " [$t, \\{$x, $y, $z, $z\\}], $lod;",3658 pattern>;3659multiclass TEX_UNIFIED_CUBE_LEVEL<string inst, Intrinsic intr> {3660 def _R : TEX_UNIFIED_CUBE_LEVEL_base<inst, (ins B64:$t),3661 [(set B32:$r, B32:$g, B32:$b, B32:$a,3662 (intr i64:$t, B32:$x, B32:$y, B32:$z, B32:$lod))]>;3663 def _I : TEX_UNIFIED_CUBE_LEVEL_base<inst, (ins i64imm:$t)>;3664}3665 3666defm TEX_UNIFIED_CUBE_F32_F32_LEVEL3667 : TEX_UNIFIED_CUBE_LEVEL<"tex.level.cube.v4.f32.f32",3668 int_nvvm_tex_unified_cube_level_v4f32_f32>;3669defm TEX_UNIFIED_CUBE_S32_F32_LEVEL3670 : TEX_UNIFIED_CUBE_LEVEL<"tex.level.cube.v4.s32.f32",3671 int_nvvm_tex_unified_cube_level_v4s32_f32>;3672defm TEX_UNIFIED_CUBE_U32_F32_LEVEL3673 : TEX_UNIFIED_CUBE_LEVEL<"tex.level.cube.v4.u32.f32",3674 int_nvvm_tex_unified_cube_level_v4u32_f32>;3675 3676class TEX_UNIFIED_CUBE_ARRAY_base<string inst, dag tex, list<dag> pattern = []>3677 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3678 !con(tex, (ins B32:$l, B32:$x, B32:$y, B32:$z)),3679 inst # " \t\\{$r, $g, $b, $a\\}, [$t, \\{$l, $x, $y, $z\\}];",3680 pattern>;3681multiclass TEX_UNIFIED_CUBE_ARRAY<string inst, Intrinsic intr> {3682 def _R : TEX_UNIFIED_CUBE_ARRAY_base<inst, (ins B64:$t),3683 [(set B32:$r, B32:$g, B32:$b, B32:$a,3684 (intr i64:$t, i32:$l, B32:$x, B32:$y, B32:$z))]>;3685 def _I : TEX_UNIFIED_CUBE_ARRAY_base<inst, (ins i64imm:$t)>;3686}3687 3688defm TEX_UNIFIED_CUBE_ARRAY_F32_F323689 : TEX_UNIFIED_CUBE_ARRAY<"tex.acube.v4.f32.f32", int_nvvm_tex_unified_cube_array_v4f32_f32>;3690defm TEX_UNIFIED_CUBE_ARRAY_S32_F323691 : TEX_UNIFIED_CUBE_ARRAY<"tex.acube.v4.s32.f32", int_nvvm_tex_unified_cube_array_v4s32_f32>;3692defm TEX_UNIFIED_CUBE_ARRAY_U32_F323693 : TEX_UNIFIED_CUBE_ARRAY<"tex.acube.v4.u32.f32", int_nvvm_tex_unified_cube_array_v4u32_f32>;3694 3695class TEX_UNIFIED_CUBE_ARRAY_LEVEL_base<string inst, dag tex, list<dag> pattern = []>3696 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3697 !con(tex, (ins B32:$l, B32:$x, B32:$y, B32:$z, B32:$lod)),3698 inst # " \t\\{$r, $g, $b, $a\\},"3699 " [$t, \\{$l, $x, $y, $z\\}], $lod;",3700 pattern>;3701multiclass TEX_UNIFIED_CUBE_ARRAY_LEVEL<string inst, Intrinsic intr> {3702 def _R : TEX_UNIFIED_CUBE_ARRAY_LEVEL_base<inst, (ins B64:$t),3703 [(set B32:$r, B32:$g, B32:$b, B32:$a,3704 (intr i64:$t, i32:$l, B32:$x, B32:$y, B32:$z, B32:$lod))]>;3705 def _I : TEX_UNIFIED_CUBE_ARRAY_LEVEL_base<inst, (ins i64imm:$t)>;3706}3707 3708defm TEX_UNIFIED_CUBE_ARRAY_F32_F32_LEVEL3709 : TEX_UNIFIED_CUBE_ARRAY_LEVEL<"tex.level.acube.v4.f32.f32",3710 int_nvvm_tex_unified_cube_array_level_v4f32_f32>;3711defm TEX_UNIFIED_CUBE_ARRAY_S32_F32_LEVEL3712 : TEX_UNIFIED_CUBE_ARRAY_LEVEL<"tex.level.acube.v4.s32.f32",3713 int_nvvm_tex_unified_cube_array_level_v4s32_f32>;3714defm TEX_UNIFIED_CUBE_ARRAY_U32_F32_LEVEL3715 : TEX_UNIFIED_CUBE_ARRAY_LEVEL<"tex.level.acube.v4.u32.f32",3716 int_nvvm_tex_unified_cube_array_level_v4u32_f32>;3717 3718class TEX_UNIFIED_CUBE_GRAD_base<string inst, dag tex, list<dag> pattern = []>3719 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3720 !con(tex, (ins B32:$x, B32:$y, B32:$z,3721 B32:$gradx0, B32:$gradx1,3722 B32:$gradx2, B32:$grady0,3723 B32:$grady1, B32:$grady2)),3724 inst # " \t\\{$r, $g, $b, $a\\}, [$t, \\{$x, $y, $z, $z\\}],"3725 " \\{$gradx0, $gradx1, $gradx2, $gradx2\\},"3726 " \\{$grady0, $grady1, $grady2, $grady2\\};",3727 pattern>;3728 3729multiclass TEX_UNIFIED_CUBE_GRAD<string inst, Intrinsic intr> {3730 def _R : TEX_UNIFIED_CUBE_GRAD_base<inst, (ins B64:$t),3731 [(set B32:$r, B32:$g, B32:$b, B32:$a,3732 (intr i64:$t, B32:$x, B32:$y, B32:$z,3733 B32:$gradx0, B32:$gradx1, B32:$gradx2,3734 B32:$grady0, B32:$grady1, B32:$grady2))]>;3735 def _I : TEX_UNIFIED_CUBE_GRAD_base<inst, (ins i64imm:$t)>;3736}3737 3738defm TEX_UNIFIED_CUBE_F32_F32_GRAD3739 : TEX_UNIFIED_CUBE_GRAD<"tex.grad.cube.v4.f32.f32", int_nvvm_tex_unified_cube_grad_v4f32_f32>;3740defm TEX_UNIFIED_CUBE_S32_F32_GRAD3741 : TEX_UNIFIED_CUBE_GRAD<"tex.grad.cube.v4.s32.f32", int_nvvm_tex_unified_cube_grad_v4s32_f32>;3742defm TEX_UNIFIED_CUBE_U32_F32_GRAD3743 : TEX_UNIFIED_CUBE_GRAD<"tex.grad.cube.v4.u32.f32", int_nvvm_tex_unified_cube_grad_v4u32_f32>;3744 3745class TEX_UNIFIED_CUBE_ARRAY_GRAD_base<string inst, dag tex, list<dag> pattern = []>3746 : NVPTXInst<(outs B32:$r, B32:$g, B32:$b, B32:$a),3747 !con(tex, (ins B32:$l, B32:$x, B32:$y, B32:$z,3748 B32:$gradx0, B32:$gradx1,3749 B32:$gradx2, B32:$grady0,3750 B32:$grady1, B32:$grady2)),3751 inst # " \t\\{$r, $g, $b, $a\\}, [$t, \\{$l, $x, $y, $z\\}],"3752 " \\{$gradx0, $gradx1, $gradx2, $gradx2\\},"3753 " \\{$grady0, $grady1, $grady2, $grady2\\};",3754 pattern>;3755multiclass TEX_UNIFIED_CUBE_ARRAY_GRAD<string inst, Intrinsic intr> {3756 def _R : TEX_UNIFIED_CUBE_ARRAY_GRAD_base<inst, (ins B64:$t),3757 [(set B32:$r, B32:$g, B32:$b, B32:$a,3758 (intr i64:$t, i32:$l, B32:$x, B32:$y, B32:$z,3759 B32:$gradx0, B32:$gradx1, B32:$gradx2,3760 B32:$grady0, B32:$grady1, B32:$grady2))]>;3761 def _I : TEX_UNIFIED_CUBE_ARRAY_GRAD_base<inst, (ins i64imm:$t)>;3762}3763 3764defm TEX_UNIFIED_CUBE_ARRAY_F32_F32_GRAD3765 : TEX_UNIFIED_CUBE_ARRAY_GRAD<"tex.grad.acube.v4.f32.f32",3766 int_nvvm_tex_unified_cube_array_grad_v4f32_f32>;3767defm TEX_UNIFIED_CUBE_ARRAY_S32_F32_GRAD3768 : TEX_UNIFIED_CUBE_ARRAY_GRAD<"tex.grad.acube.v4.s32.f32",3769 int_nvvm_tex_unified_cube_array_grad_v4s32_f32>;3770defm TEX_UNIFIED_CUBE_ARRAY_U32_F32_GRAD3771 : TEX_UNIFIED_CUBE_ARRAY_GRAD<"tex.grad.acube.v4.u32.f32",3772 int_nvvm_tex_unified_cube_array_grad_v4u32_f32>;3773 3774class TLD4_UNIFIED_2D_base<string inst, dag tex, list<dag> pattern = []>3775 : NVPTXInst<(outs B32:$v0, B32:$v1, B32:$v2, B32:$v3),3776 !con(tex, (ins B32:$x, B32:$y)),3777 inst # " \t\\{$v0, $v1, $v2, $v3\\}, [$t, \\{$x, $y\\}];",3778 pattern>;3779multiclass TLD4_UNIFIED_2D<string inst, Intrinsic intr> {3780 def _R : TLD4_UNIFIED_2D_base<inst, (ins B64:$t),3781 [(set B32:$v0, B32:$v1, B32:$v2, B32:$v3,3782 (intr i64:$t, B32:$x, B32:$y))]>;3783 def _I : TLD4_UNIFIED_2D_base<inst, (ins i64imm:$t)>;3784}3785 3786defm TLD4_UNIFIED_R_2D_F32_F323787 : TLD4_UNIFIED_2D<"tld4.r.2d.v4.f32.f32", int_nvvm_tld4_unified_r_2d_v4f32_f32>;3788defm TLD4_UNIFIED_G_2D_F32_F323789 : TLD4_UNIFIED_2D<"tld4.g.2d.v4.f32.f32", int_nvvm_tld4_unified_g_2d_v4f32_f32>;3790defm TLD4_UNIFIED_B_2D_F32_F323791 : TLD4_UNIFIED_2D<"tld4.b.2d.v4.f32.f32", int_nvvm_tld4_unified_b_2d_v4f32_f32>;3792defm TLD4_UNIFIED_A_2D_F32_F323793 : TLD4_UNIFIED_2D<"tld4.a.2d.v4.f32.f32", int_nvvm_tld4_unified_a_2d_v4f32_f32>;3794 3795defm TLD4_UNIFIED_R_2D_S32_F323796 : TLD4_UNIFIED_2D<"tld4.r.2d.v4.s32.f32", int_nvvm_tld4_unified_r_2d_v4s32_f32>;3797defm TLD4_UNIFIED_G_2D_S32_F323798 : TLD4_UNIFIED_2D<"tld4.g.2d.v4.s32.f32", int_nvvm_tld4_unified_g_2d_v4s32_f32>;3799defm TLD4_UNIFIED_B_2D_S32_F323800 : TLD4_UNIFIED_2D<"tld4.b.2d.v4.s32.f32", int_nvvm_tld4_unified_b_2d_v4s32_f32>;3801defm TLD4_UNIFIED_A_2D_S32_F323802 : TLD4_UNIFIED_2D<"tld4.a.2d.v4.s32.f32", int_nvvm_tld4_unified_a_2d_v4s32_f32>;3803 3804defm TLD4_UNIFIED_R_2D_U32_F323805 : TLD4_UNIFIED_2D<"tld4.r.2d.v4.u32.f32", int_nvvm_tld4_unified_r_2d_v4u32_f32>;3806defm TLD4_UNIFIED_G_2D_U32_F323807 : TLD4_UNIFIED_2D<"tld4.g.2d.v4.u32.f32", int_nvvm_tld4_unified_g_2d_v4u32_f32>;3808defm TLD4_UNIFIED_B_2D_U32_F323809 : TLD4_UNIFIED_2D<"tld4.b.2d.v4.u32.f32", int_nvvm_tld4_unified_b_2d_v4u32_f32>;3810defm TLD4_UNIFIED_A_2D_U32_F323811 : TLD4_UNIFIED_2D<"tld4.a.2d.v4.u32.f32", int_nvvm_tld4_unified_a_2d_v4u32_f32>;3812 3813}3814 3815 3816//=== Surface load instructions3817 3818let IsSuld = true in {3819 3820class SULD_1D_base<string inst, NVPTXRegClass outtype, dag surf,3821 list<dag> pattern = []>3822 : NVPTXInst<(outs outtype:$r),3823 !con(surf, (ins B32:$x)),3824 inst # " \\{$r\\}, [$s, \\{$x\\}];",3825 pattern>;3826multiclass SULD_1D<string inst, NVPTXRegClass outtype> {3827 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));3828 3829 def _R : SULD_1D_base<inst, outtype, (ins B64:$s),3830 [(set outtype:$r, (intr i64:$s, i32:$x))]>;3831 def _I : SULD_1D_base<inst, outtype, (ins i64imm:$s)>;3832}3833 3834foreach op = ["clamp", "trap", "zero"] in {3835 defvar op_upper = !toupper(op);3836 defm SULD_1D_I8_ # op_upper : SULD_1D<"suld.b.1d.b8." # op, B16>;3837 defm SULD_1D_I16_ # op_upper : SULD_1D<"suld.b.1d.b16." # op, B16>;3838 defm SULD_1D_I32_ # op_upper : SULD_1D<"suld.b.1d.b32." # op, B32>;3839 defm SULD_1D_I64_ # op_upper : SULD_1D<"suld.b.1d.b64." # op, B64>;3840}3841 3842class SULD_1D_ARRAY_base<string inst, NVPTXRegClass outtype, dag surf,3843 list<dag> pattern = []>3844 : NVPTXInst<(outs outtype:$r),3845 !con(surf, (ins B32:$l, B32:$x)),3846 inst # " \\{$r\\}, [$s, \\{$l, $x\\}];",3847 pattern>;3848multiclass SULD_1D_ARRAY<string inst, NVPTXRegClass outtype> {3849 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));3850 3851 def _R : SULD_1D_ARRAY_base<inst, outtype, (ins B64:$s),3852 [(set outtype:$r,3853 (intr i64:$s, i32:$l, i32:$x))]>;3854 def _I : SULD_1D_ARRAY_base<inst, outtype, (ins i64imm:$s)>;3855}3856 3857foreach op = ["clamp", "trap", "zero"] in {3858 defvar op_upper = !toupper(op);3859 defm SULD_1D_ARRAY_I8_ # op_upper : SULD_1D_ARRAY<"suld.b.a1d.b8." # op, B16>;3860 defm SULD_1D_ARRAY_I16_ # op_upper : SULD_1D_ARRAY<"suld.b.a1d.b16." # op, B16>;3861 defm SULD_1D_ARRAY_I32_ # op_upper : SULD_1D_ARRAY<"suld.b.a1d.b32." # op, B32>;3862 defm SULD_1D_ARRAY_I64_ # op_upper : SULD_1D_ARRAY<"suld.b.a1d.b64." # op, B64>;3863}3864 3865class SULD_2D_base<string inst, NVPTXRegClass outtype, dag surf,3866 list<dag> pattern = []>3867 : NVPTXInst<(outs outtype:$r),3868 !con(surf, (ins B32:$x, B32:$y)),3869 inst # " \\{$r\\}, [$s, \\{$x, $y\\}];",3870 pattern>;3871multiclass SULD_2D<string inst, NVPTXRegClass outtype> {3872 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));3873 3874 def _R : SULD_2D_base<inst, outtype, (ins B64:$s),3875 [(set outtype:$r, (intr i64:$s, i32:$x, i32:$y))]>;3876 def _I : SULD_2D_base<inst, outtype, (ins i64imm:$s)>;3877}3878 3879foreach op = ["clamp", "trap", "zero"] in {3880 defvar op_upper = !toupper(op);3881 defm SULD_2D_I8_ # op_upper : SULD_2D<"suld.b.2d.b8." # op, B16>;3882 defm SULD_2D_I16_ # op_upper : SULD_2D<"suld.b.2d.b16." # op, B16>;3883 defm SULD_2D_I32_ # op_upper : SULD_2D<"suld.b.2d.b32." # op, B32>;3884 defm SULD_2D_I64_ # op_upper : SULD_2D<"suld.b.2d.b64." # op, B64>;3885}3886 3887class SULD_2D_ARRAY_base<string inst, NVPTXRegClass outtype, dag surf,3888 list<dag> pattern = []>3889 : NVPTXInst<(outs outtype:$r),3890 !con(surf, (ins B32:$l, B32:$x, B32:$y)),3891 inst # " \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",3892 pattern>;3893multiclass SULD_2D_ARRAY<string inst, NVPTXRegClass outtype> {3894 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));3895 3896 def _R : SULD_2D_ARRAY_base<inst, outtype, (ins B64:$s),3897 [(set outtype:$r,3898 (intr i64:$s, i32:$l, i32:$x, i32:$y))]>;3899 def _I : SULD_2D_ARRAY_base<inst, outtype, (ins i64imm:$s)>;3900}3901 3902foreach op = ["clamp", "trap", "zero"] in {3903 defvar op_upper = !toupper(op);3904 defm SULD_2D_ARRAY_I8_ # op_upper : SULD_2D_ARRAY<"suld.b.a2d.b8." # op, B16>;3905 defm SULD_2D_ARRAY_I16_ # op_upper : SULD_2D_ARRAY<"suld.b.a2d.b16." # op, B16>;3906 defm SULD_2D_ARRAY_I32_ # op_upper : SULD_2D_ARRAY<"suld.b.a2d.b32." # op, B32>;3907 defm SULD_2D_ARRAY_I64_ # op_upper : SULD_2D_ARRAY<"suld.b.a2d.b64." # op, B64>;3908}3909 3910class SULD_3D_base<string inst, NVPTXRegClass outtype, dag surf,3911 list<dag> pattern = []>3912 : NVPTXInst<(outs outtype:$r),3913 !con(surf, (ins B32:$x, B32:$y, B32:$z)),3914 inst # " \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",3915 pattern>;3916multiclass SULD_3D<string inst, NVPTXRegClass outtype> {3917 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));3918 3919 def _R : SULD_3D_base<inst, outtype, (ins B64:$s),3920 [(set outtype:$r,3921 (intr i64:$s, i32:$x, i32:$y, i32:$z))]>;3922 def _I : SULD_3D_base<inst, outtype, (ins i64imm:$s)>;3923}3924 3925foreach op = ["clamp", "trap", "zero"] in {3926 defvar op_upper = !toupper(op);3927 defm SULD_3D_I8_ # op_upper : SULD_3D<"suld.b.3d.b8." # op, B16>;3928 defm SULD_3D_I16_ # op_upper : SULD_3D<"suld.b.3d.b16." # op, B16>;3929 defm SULD_3D_I32_ # op_upper : SULD_3D<"suld.b.3d.b32." # op, B32>;3930 defm SULD_3D_I64_ # op_upper : SULD_3D<"suld.b.3d.b64." # op, B64>;3931}3932}3933 3934let IsSuld = 2 in {3935 3936class SULD_1D_V2_base<string inst, NVPTXRegClass outtype, dag surf,3937 list<dag> pattern = []>3938 : NVPTXInst<(outs outtype:$r, outtype:$g),3939 !con(surf, (ins B32:$x)),3940 inst # " \\{$r, $g\\}, [$s, \\{$x\\}];",3941 pattern>;3942multiclass SULD_1D_V2<string inst, NVPTXRegClass outtype> {3943 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));3944 3945 def _R : SULD_1D_V2_base<inst, outtype, (ins B64:$s),3946 [(set outtype:$r, outtype:$g,3947 (intr i64:$s, i32:$x))]>;3948 def _I : SULD_1D_V2_base<inst, outtype, (ins i64imm:$s)>;3949}3950 3951foreach op = ["clamp", "trap", "zero"] in {3952 defvar op_upper = !toupper(op);3953 defm SULD_1D_V2I8_ # op_upper : SULD_1D_V2<"suld.b.1d.v2.b8." # op, B16>;3954 defm SULD_1D_V2I16_ # op_upper : SULD_1D_V2<"suld.b.1d.v2.b16." # op, B16>;3955 defm SULD_1D_V2I32_ # op_upper : SULD_1D_V2<"suld.b.1d.v2.b32." # op, B32>;3956 defm SULD_1D_V2I64_ # op_upper : SULD_1D_V2<"suld.b.1d.v2.b64." # op, B64>;3957}3958 3959class SULD_1D_ARRAY_V2_base<string inst, NVPTXRegClass outtype, dag surf,3960 list<dag> pattern = []>3961 : NVPTXInst<(outs outtype:$r, outtype:$g),3962 !con(surf, (ins B32:$l, B32:$x)),3963 inst # " \\{$r, $g\\}, [$s, \\{$l, $x\\}];",3964 pattern>;3965multiclass SULD_1D_ARRAY_V2<string inst, NVPTXRegClass outtype> {3966 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));3967 3968 def _R : SULD_1D_ARRAY_V2_base<inst, outtype, (ins B64:$s),3969 [(set outtype:$r, outtype:$g,3970 (intr i64:$s, i32:$l, i32:$x))]>;3971 def _I : SULD_1D_ARRAY_V2_base<inst, outtype, (ins i64imm:$s)>;3972}3973 3974foreach op = ["clamp", "trap", "zero"] in {3975 defvar op_upper = !toupper(op);3976 defm SULD_1D_ARRAY_V2I8_ # op_upper : SULD_1D_ARRAY_V2<"suld.b.a1d.v2.b8." # op, B16>;3977 defm SULD_1D_ARRAY_V2I16_ # op_upper : SULD_1D_ARRAY_V2<"suld.b.a1d.v2.b16." # op, B16>;3978 defm SULD_1D_ARRAY_V2I32_ # op_upper : SULD_1D_ARRAY_V2<"suld.b.a1d.v2.b32." # op, B32>;3979 defm SULD_1D_ARRAY_V2I64_ # op_upper : SULD_1D_ARRAY_V2<"suld.b.a1d.v2.b64." # op, B64>;3980}3981 3982class SULD_2D_V2_base<string inst, NVPTXRegClass outtype, dag surf,3983 list<dag> pattern = []>3984 : NVPTXInst<(outs outtype:$r, outtype:$g),3985 !con(surf, (ins B32:$x, B32:$y)),3986 inst # " \\{$r, $g\\}, [$s, \\{$x, $y\\}];",3987 pattern>;3988multiclass SULD_2D_V2<string inst, NVPTXRegClass outtype> {3989 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));3990 3991 def _R : SULD_2D_V2_base<inst, outtype, (ins B64:$s),3992 [(set outtype:$r, outtype:$g,3993 (intr i64:$s, i32:$x, i32:$y))]>;3994 def _I : SULD_2D_V2_base<inst, outtype, (ins i64imm:$s)>;3995}3996 3997foreach op = ["clamp", "trap", "zero"] in {3998 defvar op_upper = !toupper(op);3999 defm SULD_2D_V2I8_ # op_upper : SULD_2D_V2<"suld.b.2d.v2.b8." # op, B16>;4000 defm SULD_2D_V2I16_ # op_upper : SULD_2D_V2<"suld.b.2d.v2.b16." # op, B16>;4001 defm SULD_2D_V2I32_ # op_upper : SULD_2D_V2<"suld.b.2d.v2.b32." # op, B32>;4002 defm SULD_2D_V2I64_ # op_upper : SULD_2D_V2<"suld.b.2d.v2.b64." # op, B64>;4003}4004 4005class SULD_2D_ARRAY_V2_base<string inst, NVPTXRegClass outtype, dag surf,4006 list<dag> pattern = []>4007 : NVPTXInst<(outs outtype:$r, outtype:$g),4008 !con(surf, (ins B32:$l, B32:$x, B32:$y)),4009 inst # " \\{$r, $g\\}, [$s, \\{$l, $x, $y, $y\\}];",4010 pattern>;4011multiclass SULD_2D_ARRAY_V2<string inst, NVPTXRegClass outtype> {4012 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));4013 4014 def _R : SULD_2D_ARRAY_V2_base<inst, outtype, (ins B64:$s),4015 [(set outtype:$r, outtype:$g,4016 (intr i64:$s, i32:$l, i32:$x, i32:$y))]>;4017 def _I : SULD_2D_ARRAY_V2_base<inst, outtype, (ins i64imm:$s)>;4018}4019 4020foreach op = ["clamp", "trap", "zero"] in {4021 defvar op_upper = !toupper(op);4022 defm SULD_2D_ARRAY_V2I8_ # op_upper : SULD_2D_ARRAY_V2<"suld.b.a2d.v2.b8." # op, B16>;4023 defm SULD_2D_ARRAY_V2I16_ # op_upper : SULD_2D_ARRAY_V2<"suld.b.a2d.v2.b16." # op, B16>;4024 defm SULD_2D_ARRAY_V2I32_ # op_upper : SULD_2D_ARRAY_V2<"suld.b.a2d.v2.b32." # op, B32>;4025 defm SULD_2D_ARRAY_V2I64_ # op_upper : SULD_2D_ARRAY_V2<"suld.b.a2d.v2.b64." # op, B64>;4026}4027 4028class SULD_3D_V2_base<string inst, NVPTXRegClass outtype, dag surf,4029 list<dag> pattern = []>4030 : NVPTXInst<(outs outtype:$r, outtype:$g),4031 !con(surf, (ins B32:$x, B32:$y, B32:$z)),4032 inst # " \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",4033 pattern>;4034multiclass SULD_3D_V2<string inst, NVPTXRegClass outtype> {4035 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));4036 4037 def _R : SULD_3D_V2_base<inst, outtype, (ins B64:$s),4038 [(set outtype:$r, outtype:$g,4039 (intr i64:$s, i32:$x, i32:$y, i32:$z))]>;4040 def _I : SULD_3D_V2_base<inst, outtype, (ins i64imm:$s)>;4041}4042 4043foreach op = ["clamp", "trap", "zero"] in {4044 defvar op_upper = !toupper(op);4045 defm SULD_3D_V2I8_ # op_upper : SULD_3D_V2<"suld.b.3d.v2.b8." # op, B16>;4046 defm SULD_3D_V2I16_ # op_upper : SULD_3D_V2<"suld.b.3d.v2.b16." # op, B16>;4047 defm SULD_3D_V2I32_ # op_upper : SULD_3D_V2<"suld.b.3d.v2.b32." # op, B32>;4048 defm SULD_3D_V2I64_ # op_upper : SULD_3D_V2<"suld.b.3d.v2.b64." # op, B64>;4049}4050 4051}4052 4053let IsSuld = 3 in {4054 4055class SULD_1D_V4_base<string inst, NVPTXRegClass outtype, dag surf,4056 list<dag> pattern = []>4057 : NVPTXInst<(outs outtype:$r, outtype:$g, outtype:$b, outtype:$a),4058 !con(surf, (ins B32:$x)),4059 inst # " \\{$r, $g, $b, $a\\}, [$s, \\{$x\\}];",4060 pattern>;4061multiclass SULD_1D_V4<string inst, NVPTXRegClass outtype> {4062 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));4063 4064 def _R : SULD_1D_V4_base<inst, outtype, (ins B64:$s),4065 [(set outtype:$r, outtype:$g, outtype:$b, outtype:$a,4066 (intr i64:$s, i32:$x))]>;4067 def _I : SULD_1D_V4_base<inst, outtype, (ins i64imm:$s)>;4068}4069 4070foreach op = ["clamp", "trap", "zero"] in {4071 defvar op_upper = !toupper(op);4072 defm SULD_1D_V4I8_ # op_upper : SULD_1D_V4<"suld.b.1d.v4.b8." # op, B16>;4073 defm SULD_1D_V4I16_ # op_upper : SULD_1D_V4<"suld.b.1d.v4.b16." # op, B16>;4074 defm SULD_1D_V4I32_ # op_upper : SULD_1D_V4<"suld.b.1d.v4.b32." # op, B32>;4075}4076 4077class SULD_1D_ARRAY_V4_base<string inst, NVPTXRegClass outtype, dag surf,4078 list<dag> pattern = []>4079 : NVPTXInst<(outs outtype:$r, outtype:$g, outtype:$b, outtype:$a),4080 !con(surf, (ins B32:$l, B32:$x)),4081 inst # " \\{$r, $g, $b, $a\\}, [$s, \\{$l, $x\\}];",4082 pattern>;4083multiclass SULD_1D_ARRAY_V4<string inst, NVPTXRegClass outtype> {4084 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));4085 4086 def _R : SULD_1D_ARRAY_V4_base<inst, outtype, (ins B64:$s),4087 [(set outtype:$r, outtype:$g, outtype:$b,4088 outtype:$a,4089 (intr i64:$s, i32:$l, i32:$x))]>;4090 def _I : SULD_1D_ARRAY_V4_base<inst, outtype, (ins i64imm:$s)>;4091}4092 4093foreach op = ["clamp", "trap", "zero"] in {4094 defvar op_upper = !toupper(op);4095 defm SULD_1D_ARRAY_V4I8_ # op_upper : SULD_1D_ARRAY_V4<"suld.b.a1d.v4.b8." # op, B16>;4096 defm SULD_1D_ARRAY_V4I16_ # op_upper : SULD_1D_ARRAY_V4<"suld.b.a1d.v4.b16." # op, B16>;4097 defm SULD_1D_ARRAY_V4I32_ # op_upper : SULD_1D_ARRAY_V4<"suld.b.a1d.v4.b32." # op, B32>;4098}4099 4100class SULD_2D_V4_base<string inst, NVPTXRegClass outtype, dag surf,4101 list<dag> pattern = []>4102 : NVPTXInst<(outs outtype:$r, outtype:$g, outtype:$b, outtype:$a),4103 !con(surf, (ins B32:$x, B32:$y)),4104 inst # " \\{$r, $g, $b, $a\\}, [$s, \\{$x, $y\\}];",4105 pattern>;4106multiclass SULD_2D_V4<string inst, NVPTXRegClass outtype> {4107 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));4108 4109 def _R : SULD_2D_V4_base<inst, outtype, (ins B64:$s),4110 [(set outtype:$r, outtype:$g, outtype:$b, outtype:$a,4111 (intr i64:$s, i32:$x, i32:$y))]>;4112 def _I : SULD_2D_V4_base<inst, outtype, (ins i64imm:$s)>;4113}4114 4115foreach op = ["clamp", "trap", "zero"] in {4116 defvar op_upper = !toupper(op);4117 defm SULD_2D_V4I8_ # op_upper : SULD_2D_V4<"suld.b.2d.v4.b8." # op, B16>;4118 defm SULD_2D_V4I16_ # op_upper : SULD_2D_V4<"suld.b.2d.v4.b16." # op, B16>;4119 defm SULD_2D_V4I32_ # op_upper : SULD_2D_V4<"suld.b.2d.v4.b32." # op, B32>;4120}4121 4122class SULD_2D_ARRAY_V4_base<string inst, NVPTXRegClass outtype, dag surf,4123 list<dag> pattern = []>4124 : NVPTXInst<(outs outtype:$r, outtype:$g, outtype:$b, outtype:$a),4125 !con(surf, (ins B32:$l, B32:$x, B32:$y)),4126 inst # " \\{$r, $g, $b, $a\\}, [$s, \\{$l, $x, $y, $y\\}];",4127 pattern>;4128multiclass SULD_2D_ARRAY_V4<string inst, NVPTXRegClass outtype> {4129 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));4130 4131 def _R : SULD_2D_ARRAY_V4_base<inst, outtype, (ins B64:$s),4132 [(set outtype:$r, outtype:$g, outtype:$b,4133 outtype:$a,4134 (intr i64:$s, i32:$l, i32:$x, i32:$y))]>;4135 def _I : SULD_2D_ARRAY_V4_base<inst, outtype, (ins i64imm:$s)>;4136}4137 4138foreach op = ["clamp", "trap", "zero"] in {4139 defvar op_upper = !toupper(op);4140 defm SULD_2D_ARRAY_V4I8_ # op_upper : SULD_2D_ARRAY_V4<"suld.b.a2d.v4.b8." # op, B16>;4141 defm SULD_2D_ARRAY_V4I16_ # op_upper : SULD_2D_ARRAY_V4<"suld.b.a2d.v4.b16." # op, B16>;4142 defm SULD_2D_ARRAY_V4I32_ # op_upper : SULD_2D_ARRAY_V4<"suld.b.a2d.v4.b32." # op, B32>;4143}4144 4145class SULD_3D_V4_base<string inst, NVPTXRegClass outtype, dag surf,4146 list<dag> pattern = []>4147 : NVPTXInst<(outs outtype:$r, outtype:$g, outtype:$b, outtype:$a),4148 !con(surf, (ins B32:$x, B32:$y, B32:$z)),4149 inst # " \\{$r, $g, $b, $a\\}, [$s, \\{$x, $y, $z, $z\\}];",4150 pattern>;4151multiclass SULD_3D_V4<string inst, NVPTXRegClass outtype> {4152 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));4153 4154 def _R : SULD_3D_V4_base<inst, outtype, (ins B64:$s),4155 [(set outtype:$r, outtype:$g, outtype:$b, outtype:$a,4156 (intr i64:$s, i32:$x, i32:$y, i32:$z))]>;4157 def _I : SULD_3D_V4_base<inst, outtype, (ins i64imm:$s)>;4158}4159 4160foreach op = ["clamp", "trap", "zero"] in {4161 defvar op_upper = !toupper(op);4162 defm SULD_3D_V4I8_ # op_upper : SULD_3D_V4<"suld.b.3d.v4.b8." # op, B16>;4163 defm SULD_3D_V4I16_ # op_upper : SULD_3D_V4<"suld.b.3d.v4.b16." # op, B16>;4164 defm SULD_3D_V4I32_ # op_upper : SULD_3D_V4<"suld.b.3d.v4.b32." # op, B32>;4165}4166 4167}4168 4169//-----------------------------------4170// Texture Query Intrinsics4171//-----------------------------------4172 4173let IsSurfTexQuery = true in {4174 foreach query = ["channel_order", "channel_data_type", "width", "height", 4175 "depth", "array_size", "num_samples", "num_mipmap_levels"] in {4176 def TXQ_ # !toupper(query) # _R4177 : NVPTXInst<(outs B32:$d), (ins B64:$a),4178 "txq." # query # ".b32 \t$d, [$a];",4179 [(set i32:$d, (!cast<Intrinsic>("int_nvvm_txq_" # query) i64:$a))]>;4180 def TXQ_ # !toupper(query) # _I4181 : NVPTXInst<(outs B32:$d), (ins i64imm:$a),4182 "txq." # query # ".b32 \t$d, [$a];",4183 []>;4184 }4185}4186 4187//-----------------------------------4188// Surface Query Intrinsics4189//-----------------------------------4190 4191let IsSurfTexQuery = true in {4192 foreach query = ["channel_order", "channel_data_type", "width", "height", "depth", "array_size"] in {4193 def SUQ_ # !toupper(query) # _R4194 : NVPTXInst<(outs B32:$d), (ins B64:$a),4195 "suq." # query # ".b32 \t$d, [$a];",4196 [(set i32:$d, (!cast<Intrinsic>("int_nvvm_suq_" # query) i64:$a))]>;4197 def SUQ_ # !toupper(query) # _I4198 : NVPTXInst<(outs B32:$d), (ins i64imm:$a),4199 "suq." # query # ".b32 \t$d, [$a];",4200 []>;4201 }4202}4203 4204//===- Handle Query -------------------------------------------------------===//4205 4206// TODO: These intrinsics are not yet finalized, pending PTX ISA design work4207def ISTYPEP_SAMPLER4208 : BasicNVPTXInst<(outs B1:$d), (ins B64:$a),4209 "istypep.samplerref",4210 [(set i1:$d, (int_nvvm_istypep_sampler i64:$a))]>;4211def ISTYPEP_SURFACE4212 : BasicNVPTXInst<(outs B1:$d), (ins B64:$a),4213 "istypep.surfref",4214 [(set i1:$d, (int_nvvm_istypep_surface i64:$a))]>;4215def ISTYPEP_TEXTURE4216 : BasicNVPTXInst<(outs B1:$d), (ins B64:$a),4217 "istypep.texref",4218 [(set i1:$d, (int_nvvm_istypep_texture i64:$a))]>;4219 4220//===- Surface Stores -----------------------------------------------------===//4221 4222let IsSust = true in {4223 4224class SUST_1D_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>4225 : NVPTXInst<(outs),4226 !con(surf, (ins B32:$x, intype:$r)),4227 inst # " \t[$s, \\{$x\\}], \\{$r\\};", pat>;4228multiclass SUST_1D<string inst, NVPTXRegClass intype> {4229 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));4230 4231 def _R : SUST_1D_base<inst, intype, (ins B64:$s),4232 [(intr B64:$s, B32:$x, intype:$r)]>;4233 def _I : SUST_1D_base<inst, intype, (ins i64imm:$s), []>;4234}4235 4236foreach op = ["clamp", "trap", "zero"] in {4237 defvar op_upper = !toupper(op);4238 defm SUST_B_1D_I8_ # op_upper : SUST_1D<"sust.b.1d.b8." # op, B16>;4239 defm SUST_B_1D_I16_ # op_upper : SUST_1D<"sust.b.1d.b16." # op, B16>;4240 defm SUST_B_1D_I32_ # op_upper : SUST_1D<"sust.b.1d.b32." # op, B32>;4241 defm SUST_B_1D_I64_ # op_upper : SUST_1D<"sust.b.1d.b64." # op, B64>;4242}4243 4244defm SUST_P_1D_I8_TRAP : SUST_1D<"sust.p.1d.b8.trap", B16>;4245defm SUST_P_1D_I16_TRAP : SUST_1D<"sust.p.1d.b16.trap", B16>;4246defm SUST_P_1D_I32_TRAP : SUST_1D<"sust.p.1d.b32.trap", B32>;4247 4248class SUST_1D_V2_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>4249 : NVPTXInst<(outs),4250 !con(surf, (ins B32:$x, intype:$r, intype:$g)),4251 inst # " \t[$s, \\{$x\\}], \\{$r, $g\\};",4252 pat>;4253multiclass SUST_1D_V2<string inst, NVPTXRegClass intype> {4254 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));4255 def _R : SUST_1D_V2_base<inst, intype, (ins B64:$s),4256 [(intr B64:$s, B32:$x, intype:$r, intype:$g)]>;4257 def _I : SUST_1D_V2_base<inst, intype, (ins i64imm:$s), []>;4258}4259 4260foreach op = ["clamp", "trap", "zero"] in {4261 defvar op_upper = !toupper(op);4262 defm SUST_B_1D_V2I8_ # op_upper : SUST_1D_V2<"sust.b.1d.v2.b8." # op, B16>;4263 defm SUST_B_1D_V2I16_ # op_upper : SUST_1D_V2<"sust.b.1d.v2.b16." # op, B16>;4264 defm SUST_B_1D_V2I32_ # op_upper : SUST_1D_V2<"sust.b.1d.v2.b32." # op, B32>;4265 defm SUST_B_1D_V2I64_ # op_upper : SUST_1D_V2<"sust.b.1d.v2.b64." # op, B64>;4266}4267defm SUST_P_1D_V2I8_TRAP : SUST_1D_V2<"sust.p.1d.v2.b8.trap", B16>;4268defm SUST_P_1D_V2I16_TRAP : SUST_1D_V2<"sust.p.1d.v2.b16.trap", B16>;4269defm SUST_P_1D_V2I32_TRAP : SUST_1D_V2<"sust.p.1d.v2.b32.trap", B32>;4270 4271class SUST_1D_V4_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>4272 : NVPTXInst<(outs),4273 !con(surf, (ins B32:$x, intype:$r, intype:$g,4274 intype:$b, intype:$a)),4275 inst # " \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",4276 pat>;4277multiclass SUST_1D_V4<string inst, NVPTXRegClass intype> {4278 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));4279 def _R : SUST_1D_V4_base<inst, intype, (ins B64:$s),4280 [(intr B64:$s, B32:$x, intype:$r, intype:$g,4281 intype:$b, intype:$a)]>;4282 def _I : SUST_1D_V4_base<inst, intype, (ins i64imm:$s), []>;4283}4284 4285foreach op = ["clamp", "trap", "zero"] in {4286 defvar op_upper = !toupper(op);4287 defm SUST_B_1D_V4I8_ # op_upper : SUST_1D_V4<"sust.b.1d.v4.b8." # op, B16>;4288 defm SUST_B_1D_V4I16_ # op_upper : SUST_1D_V4<"sust.b.1d.v4.b16." # op, B16>;4289 defm SUST_B_1D_V4I32_ # op_upper : SUST_1D_V4<"sust.b.1d.v4.b32." # op, B32>;4290}4291 4292defm SUST_P_1D_V4I8_TRAP : SUST_1D_V4<"sust.p.1d.v4.b8.trap", B16>;4293defm SUST_P_1D_V4I16_TRAP : SUST_1D_V4<"sust.p.1d.v4.b16.trap", B16>;4294defm SUST_P_1D_V4I32_TRAP : SUST_1D_V4<"sust.p.1d.v4.b32.trap", B32>;4295 4296class SUST_1D_ARRAY_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>4297 : NVPTXInst<(outs),4298 !con(surf, (ins B32:$idx, B32:$x, intype:$r)),4299 inst # " \t[$s, \\{$idx, $x\\}], \\{$r\\};",4300 pat>;4301multiclass SUST_1D_ARRAY<string inst, NVPTXRegClass intype> {4302 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));4303 def _R : SUST_1D_ARRAY_base<inst, intype, (ins B64:$s),4304 [(intr B64:$s, B32:$idx, B32:$x, intype:$r)]>;4305 def _I : SUST_1D_ARRAY_base<inst, intype, (ins i64imm:$s), []>;4306}4307 4308foreach op = ["clamp", "trap", "zero"] in {4309 defvar op_upper = !toupper(op);4310 defm SUST_B_1D_ARRAY_I8_ # op_upper : SUST_1D_ARRAY<"sust.b.a1d.b8." # op, B16>;4311 defm SUST_B_1D_ARRAY_I16_ # op_upper : SUST_1D_ARRAY<"sust.b.a1d.b16." # op, B16>;4312 defm SUST_B_1D_ARRAY_I32_ # op_upper : SUST_1D_ARRAY<"sust.b.a1d.b32." # op, B32>;4313 defm SUST_B_1D_ARRAY_I64_ # op_upper : SUST_1D_ARRAY<"sust.b.a1d.b64." # op, B64>;4314}4315 4316defm SUST_P_1D_ARRAY_I8_TRAP : SUST_1D_ARRAY<"sust.p.a1d.b8.trap", B16>;4317defm SUST_P_1D_ARRAY_I16_TRAP : SUST_1D_ARRAY<"sust.p.a1d.b16.trap", B16>;4318defm SUST_P_1D_ARRAY_I32_TRAP : SUST_1D_ARRAY<"sust.p.a1d.b32.trap", B32>;4319 4320class SUST_1D_ARRAY_V2_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>4321 : NVPTXInst<(outs),4322 !con(surf, (ins B32:$idx, B32:$x,4323 intype:$r, intype:$g)),4324 inst # " \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",4325 pat>;4326multiclass SUST_1D_ARRAY_V2<string inst, NVPTXRegClass intype> {4327 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));4328 def _R : SUST_1D_ARRAY_V2_base<inst, intype, (ins B64:$s),4329 [(intr B64:$s, B32:$idx, B32:$x,4330 intype:$r, intype:$g)]>;4331 def _I : SUST_1D_ARRAY_V2_base<inst, intype, (ins i64imm:$s), []>;4332}4333 4334foreach op = ["clamp", "trap", "zero"] in {4335 defvar op_upper = !toupper(op);4336 defm SUST_B_1D_ARRAY_V2I8_ # op_upper : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b8." # op, B16>;4337 defm SUST_B_1D_ARRAY_V2I16_ # op_upper : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b16." # op, B16>;4338 defm SUST_B_1D_ARRAY_V2I32_ # op_upper : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b32." # op, B32>;4339 defm SUST_B_1D_ARRAY_V2I64_ # op_upper : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b64." # op, B64>;4340}4341 4342defm SUST_P_1D_ARRAY_V2I8_TRAP : SUST_1D_ARRAY_V2<"sust.p.a1d.v2.b8.trap", B16>;4343defm SUST_P_1D_ARRAY_V2I16_TRAP : SUST_1D_ARRAY_V2<"sust.p.a1d.v2.b16.trap", B16>;4344defm SUST_P_1D_ARRAY_V2I32_TRAP : SUST_1D_ARRAY_V2<"sust.p.a1d.v2.b32.trap", B32>;4345 4346class SUST_1D_ARRAY_V4_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>4347 : NVPTXInst<(outs),4348 !con(surf, (ins B32:$idx, B32:$x,4349 intype:$r, intype:$g, intype:$b, intype:$a)),4350 inst # " \t[$s, \\{$idx, $x\\}], \\{$r, $g, $b, $a\\};",4351 pat>;4352multiclass SUST_1D_ARRAY_V4<string inst, NVPTXRegClass intype> {4353 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));4354 def _R : SUST_1D_ARRAY_V4_base<inst, intype, (ins B64:$s),4355 [(intr B64:$s, B32:$idx, B32:$x,4356 intype:$r, intype:$g, intype:$b, intype:$a)]>;4357 def _I : SUST_1D_ARRAY_V4_base<inst, intype, (ins i64imm:$s), []>;4358}4359 4360foreach op = ["clamp", "trap", "zero"] in {4361 defvar op_upper = !toupper(op);4362 defm SUST_B_1D_ARRAY_V4I8_ # op_upper : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b8." # op, B16>;4363 defm SUST_B_1D_ARRAY_V4I16_ # op_upper : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b16." # op, B16>;4364 defm SUST_B_1D_ARRAY_V4I32_ # op_upper : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b32." # op, B32>;4365}4366 4367defm SUST_P_1D_ARRAY_V4I8_TRAP : SUST_1D_ARRAY_V4<"sust.p.a1d.v4.b8.trap", B16>;4368defm SUST_P_1D_ARRAY_V4I16_TRAP : SUST_1D_ARRAY_V4<"sust.p.a1d.v4.b16.trap", B16>;4369defm SUST_P_1D_ARRAY_V4I32_TRAP : SUST_1D_ARRAY_V4<"sust.p.a1d.v4.b32.trap", B32>;4370 4371class SUST_2D_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>4372 : NVPTXInst<(outs),4373 !con(surf, (ins B32:$x, B32:$y, intype:$r)),4374 inst # " \t[$s, \\{$x, $y\\}], \\{$r\\};",4375 pat>;4376multiclass SUST_2D<string inst, NVPTXRegClass intype> {4377 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));4378 def _R : SUST_2D_base<inst, intype, (ins B64:$s),4379 [(intr B64:$s, B32:$x, B32:$y, intype:$r)]>;4380 def _I : SUST_2D_base<inst, intype, (ins i64imm:$s), []>;4381}4382 4383foreach op = ["clamp", "trap", "zero"] in {4384 defvar op_upper = !toupper(op);4385 defm SUST_B_2D_I8_ # op_upper : SUST_2D<"sust.b.2d.b8." # op, B16>;4386 defm SUST_B_2D_I16_ # op_upper : SUST_2D<"sust.b.2d.b16." # op, B16>;4387 defm SUST_B_2D_I32_ # op_upper : SUST_2D<"sust.b.2d.b32." # op, B32>;4388 defm SUST_B_2D_I64_ # op_upper : SUST_2D<"sust.b.2d.b64." # op, B64>;4389}4390 4391defm SUST_P_2D_I8_TRAP : SUST_2D<"sust.p.2d.b8.trap", B16>;4392defm SUST_P_2D_I16_TRAP : SUST_2D<"sust.p.2d.b16.trap", B16>;4393defm SUST_P_2D_I32_TRAP : SUST_2D<"sust.p.2d.b32.trap", B32>;4394 4395class SUST_2D_V2_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>4396 : NVPTXInst<(outs),4397 !con(surf, (ins B32:$x, B32:$y,4398 intype:$r, intype:$g)),4399 inst # " \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",4400 pat>;4401multiclass SUST_2D_V2<string inst, NVPTXRegClass intype> {4402 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));4403 def _R : SUST_2D_V2_base<inst, intype, (ins B64:$s),4404 [(intr B64:$s, B32:$x, B32:$y,4405 intype:$r, intype:$g)]>;4406 def _I : SUST_2D_V2_base<inst, intype, (ins i64imm:$s), []>;4407}4408 4409foreach op = ["clamp", "trap", "zero"] in {4410 defvar op_upper = !toupper(op);4411 defm SUST_B_2D_V2I8_ # op_upper : SUST_2D_V2<"sust.b.2d.v2.b8." # op, B16>;4412 defm SUST_B_2D_V2I16_ # op_upper : SUST_2D_V2<"sust.b.2d.v2.b16." # op, B16>;4413 defm SUST_B_2D_V2I32_ # op_upper : SUST_2D_V2<"sust.b.2d.v2.b32." # op, B32>;4414 defm SUST_B_2D_V2I64_ # op_upper : SUST_2D_V2<"sust.b.2d.v2.b64." # op, B64>;4415}4416 4417defm SUST_P_2D_V2I8_TRAP : SUST_2D_V2<"sust.p.2d.v2.b8.trap", B16>;4418defm SUST_P_2D_V2I16_TRAP : SUST_2D_V2<"sust.p.2d.v2.b16.trap", B16>;4419defm SUST_P_2D_V2I32_TRAP : SUST_2D_V2<"sust.p.2d.v2.b32.trap", B32>;4420 4421class SUST_2D_V4_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>4422 : NVPTXInst<(outs),4423 !con(surf, (ins B32:$x, B32:$y,4424 intype:$r, intype:$g, intype:$b, intype:$a)),4425 inst # " \t[$s, \\{$x, $y\\}], \\{$r, $g, $b, $a\\};",4426 pat>;4427multiclass SUST_2D_V4<string inst, NVPTXRegClass intype> {4428 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));4429 def _R : SUST_2D_V4_base<inst, intype, (ins B64:$s),4430 [(intr B64:$s, B32:$x, B32:$y,4431 intype:$r, intype:$g, intype:$b, intype:$a)]>;4432 def _I : SUST_2D_V4_base<inst, intype, (ins i64imm:$s), []>;4433}4434 4435foreach op = ["clamp", "trap", "zero"] in {4436 defvar op_upper = !toupper(op);4437 defm SUST_B_2D_V4I8_ # op_upper : SUST_2D_V4<"sust.b.2d.v4.b8." # op, B16>;4438 defm SUST_B_2D_V4I16_ # op_upper : SUST_2D_V4<"sust.b.2d.v4.b16." # op, B16>;4439 defm SUST_B_2D_V4I32_ # op_upper : SUST_2D_V4<"sust.b.2d.v4.b32." # op, B32>;4440}4441 4442defm SUST_P_2D_V4I8_TRAP : SUST_2D_V4<"sust.p.2d.v4.b8.trap", B16>;4443defm SUST_P_2D_V4I16_TRAP : SUST_2D_V4<"sust.p.2d.v4.b16.trap", B16>;4444defm SUST_P_2D_V4I32_TRAP : SUST_2D_V4<"sust.p.2d.v4.b32.trap", B32>;4445 4446class SUST_2D_ARRAY_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>4447 : NVPTXInst<(outs),4448 !con(surf, (ins B32:$idx, B32:$x, B32:$y,4449 intype:$r)),4450 inst # " \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",4451 pat>;4452multiclass SUST_2D_ARRAY<string inst, NVPTXRegClass intype> {4453 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));4454 def _R : SUST_2D_ARRAY_base<inst, intype, (ins B64:$s),4455 [(intr B64:$s, B32:$idx, B32:$x, B32:$y,4456 intype:$r)]>;4457 def _I : SUST_2D_ARRAY_base<inst, intype, (ins i64imm:$s), []>;4458}4459 4460foreach op = ["clamp", "trap", "zero"] in {4461 defvar op_upper = !toupper(op);4462 defm SUST_B_2D_ARRAY_I8_ # op_upper : SUST_2D_ARRAY<"sust.b.a2d.b8." # op, B16>;4463 defm SUST_B_2D_ARRAY_I16_ # op_upper : SUST_2D_ARRAY<"sust.b.a2d.b16." # op, B16>;4464 defm SUST_B_2D_ARRAY_I32_ # op_upper : SUST_2D_ARRAY<"sust.b.a2d.b32." # op, B32>;4465 defm SUST_B_2D_ARRAY_I64_ # op_upper : SUST_2D_ARRAY<"sust.b.a2d.b64." # op, B64>;4466}4467 4468defm SUST_P_2D_ARRAY_I8_TRAP : SUST_2D_ARRAY<"sust.p.a2d.b8.trap", B16>;4469defm SUST_P_2D_ARRAY_I16_TRAP : SUST_2D_ARRAY<"sust.p.a2d.b16.trap", B16>;4470defm SUST_P_2D_ARRAY_I32_TRAP : SUST_2D_ARRAY<"sust.p.a2d.b32.trap", B32>;4471 4472class SUST_2D_ARRAY_V2_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>4473 : NVPTXInst<(outs),4474 !con(surf, (ins B32:$idx, B32:$x, B32:$y,4475 intype:$r, intype:$g)),4476 inst # " \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r, $g\\};",4477 pat>;4478multiclass SUST_2D_ARRAY_V2<string inst, NVPTXRegClass intype> {4479 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));4480 def _R : SUST_2D_ARRAY_V2_base<inst, intype, (ins B64:$s),4481 [(intr B64:$s, B32:$idx, B32:$x, B32:$y,4482 intype:$r, intype:$g)]>;4483 def _I : SUST_2D_ARRAY_V2_base<inst, intype, (ins i64imm:$s), []>;4484}4485 4486foreach op = ["clamp", "trap", "zero"] in {4487 defvar op_upper = !toupper(op);4488 defm SUST_B_2D_ARRAY_V2I8_ # op_upper : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b8." # op, B16>;4489 defm SUST_B_2D_ARRAY_V2I16_ # op_upper : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b16." # op, B16>;4490 defm SUST_B_2D_ARRAY_V2I32_ # op_upper : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b32." # op, B32>;4491 defm SUST_B_2D_ARRAY_V2I64_ # op_upper : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b64." # op, B64>;4492}4493 4494defm SUST_P_2D_ARRAY_V2I8_TRAP : SUST_2D_ARRAY_V2<"sust.p.a2d.v2.b8.trap", B16>;4495defm SUST_P_2D_ARRAY_V2I16_TRAP : SUST_2D_ARRAY_V2<"sust.p.a2d.v2.b16.trap", B16>;4496defm SUST_P_2D_ARRAY_V2I32_TRAP : SUST_2D_ARRAY_V2<"sust.p.a2d.v2.b32.trap", B32>;4497 4498class SUST_2D_ARRAY_V4_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>4499 : NVPTXInst<(outs),4500 !con(surf, (ins B32:$idx, B32:$x, B32:$y,4501 intype:$r, intype:$g, intype:$b, intype:$a)),4502 inst # " \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r, $g, $b, $a\\};",4503 pat>;4504multiclass SUST_2D_ARRAY_V4<string inst, NVPTXRegClass intype> {4505 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));4506 def _R : SUST_2D_ARRAY_V4_base<inst, intype, (ins B64:$s),4507 [(intr B64:$s, B32:$idx, B32:$x, B32:$y,4508 intype:$r, intype:$g, intype:$b, intype:$a)]>;4509 def _I : SUST_2D_ARRAY_V4_base<inst, intype, (ins i64imm:$s), []>;4510}4511 4512foreach op = ["clamp", "trap", "zero"] in {4513 defvar op_upper = !toupper(op);4514 defm SUST_B_2D_ARRAY_V4I8_ # op_upper : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b8." # op, B16>;4515 defm SUST_B_2D_ARRAY_V4I16_ # op_upper : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b16." # op, B16>;4516 defm SUST_B_2D_ARRAY_V4I32_ # op_upper : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b32." # op, B32>;4517}4518 4519defm SUST_P_2D_ARRAY_V4I8_TRAP : SUST_2D_ARRAY_V4<"sust.p.a2d.v4.b8.trap", B16>;4520defm SUST_P_2D_ARRAY_V4I16_TRAP : SUST_2D_ARRAY_V4<"sust.p.a2d.v4.b16.trap", B16>;4521defm SUST_P_2D_ARRAY_V4I32_TRAP : SUST_2D_ARRAY_V4<"sust.p.a2d.v4.b32.trap", B32>;4522 4523class SUST_3D_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>4524 : NVPTXInst<(outs),4525 !con(surf, (ins B32:$x, B32:$y, B32:$z,4526 intype:$r)),4527 inst # " \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",4528 pat>;4529multiclass SUST_3D<string inst, NVPTXRegClass intype> {4530 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));4531 def _R : SUST_3D_base<inst, intype, (ins B64:$s),4532 [(intr B64:$s, B32:$x, B32:$y, B32:$z,4533 intype:$r)]>;4534 def _I : SUST_3D_base<inst, intype, (ins i64imm:$s), []>;4535}4536 4537foreach op = ["clamp", "trap", "zero"] in {4538 defvar op_upper = !toupper(op);4539 defm SUST_B_3D_I8_ # op_upper : SUST_3D<"sust.b.3d.b8." # op, B16>;4540 defm SUST_B_3D_I16_ # op_upper : SUST_3D<"sust.b.3d.b16." # op, B16>;4541 defm SUST_B_3D_I32_ # op_upper : SUST_3D<"sust.b.3d.b32." # op, B32>;4542 defm SUST_B_3D_I64_ # op_upper : SUST_3D<"sust.b.3d.b64." # op, B64>;4543}4544defm SUST_P_3D_I8_TRAP : SUST_3D<"sust.p.3d.b8.trap", B16>;4545defm SUST_P_3D_I16_TRAP : SUST_3D<"sust.p.3d.b16.trap", B16>;4546defm SUST_P_3D_I32_TRAP : SUST_3D<"sust.p.3d.b32.trap", B32>;4547 4548class SUST_3D_V2_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>4549 : NVPTXInst<(outs),4550 !con(surf, (ins B32:$x, B32:$y, B32:$z,4551 intype:$r, intype:$g)),4552 inst # " \t[$s, \\{$x, $y, $z, $z\\}], \\{$r, $g\\};",4553 pat>;4554multiclass SUST_3D_V2<string inst, NVPTXRegClass intype> {4555 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));4556 def _R : SUST_3D_V2_base<inst, intype, (ins B64:$s),4557 [(intr B64:$s, B32:$x, B32:$y, B32:$z,4558 intype:$r, intype:$g)]>;4559 def _I : SUST_3D_V2_base<inst, intype, (ins i64imm:$s), []>;4560}4561 4562foreach op = ["clamp", "trap", "zero"] in {4563 defvar op_upper = !toupper(op);4564 defm SUST_B_3D_V2I8_ # op_upper : SUST_3D_V2<"sust.b.3d.v2.b8." # op, B16>;4565 defm SUST_B_3D_V2I16_ # op_upper : SUST_3D_V2<"sust.b.3d.v2.b16." # op, B16>;4566 defm SUST_B_3D_V2I32_ # op_upper : SUST_3D_V2<"sust.b.3d.v2.b32." # op, B32>;4567 defm SUST_B_3D_V2I64_ # op_upper : SUST_3D_V2<"sust.b.3d.v2.b64." # op, B64>;4568}4569defm SUST_P_3D_V2I8_TRAP : SUST_3D_V2<"sust.p.3d.v2.b8.trap", B16>;4570defm SUST_P_3D_V2I16_TRAP : SUST_3D_V2<"sust.p.3d.v2.b16.trap", B16>;4571defm SUST_P_3D_V2I32_TRAP : SUST_3D_V2<"sust.p.3d.v2.b32.trap", B32>;4572 4573class SUST_3D_V4_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>4574 : NVPTXInst<(outs),4575 !con(surf, (ins B32:$x, B32:$y, B32:$z,4576 intype:$r, intype:$g, intype:$b, intype:$a)),4577 inst # " \t[$s, \\{$x, $y, $z, $z\\}], \\{$r, $g, $b, $a\\};",4578 pat>;4579multiclass SUST_3D_V4<string inst, NVPTXRegClass intype> {4580 defvar intr = !cast<Intrinsic>("int_nvvm_" # !tolower(NAME));4581 def _R : SUST_3D_V4_base<inst, intype, (ins B64:$s),4582 [(intr B64:$s, B32:$x, B32:$y, B32:$z,4583 intype:$r, intype:$g, intype:$b, intype:$a)]>;4584 def _I : SUST_3D_V4_base<inst, intype, (ins i64imm:$s), []>;4585}4586 4587foreach op = ["clamp", "trap", "zero"] in {4588 defvar op_upper = !toupper(op);4589 defm SUST_B_3D_V4I8_ # op_upper : SUST_3D_V4<"sust.b.3d.v4.b8." # op, B16>;4590 defm SUST_B_3D_V4I16_ # op_upper : SUST_3D_V4<"sust.b.3d.v4.b16." # op, B16>;4591 defm SUST_B_3D_V4I32_ # op_upper : SUST_3D_V4<"sust.b.3d.v4.b32." # op, B32>;4592}4593 4594defm SUST_P_3D_V4I8_TRAP : SUST_3D_V4<"sust.p.3d.v4.b8.trap", B16>;4595defm SUST_P_3D_V4I16_TRAP : SUST_3D_V4<"sust.p.3d.v4.b16.trap", B16>;4596defm SUST_P_3D_V4I32_TRAP : SUST_3D_V4<"sust.p.3d.v4.b32.trap", B32>;4597 4598}4599 4600 4601//-----------------------------------4602// Read Special Registers4603//-----------------------------------4604 4605class PTX_READ_SREG_R64<string regname, Intrinsic intop, list<Predicate> Preds=[]>4606 : NVPTXInst<(outs B64:$d), (ins),4607 "mov.u64 \t$d, %" # regname # ";",4608 [(set i64:$d, (intop))]>,4609 Requires<Preds>;4610 4611class PTX_READ_SREG_R32<string regname, Intrinsic intop, list<Predicate> Preds=[]>4612 : NVPTXInst<(outs B32:$d), (ins),4613 "mov.u32 \t$d, %" # regname # ";",4614 [(set i32:$d, (intop))]>,4615 Requires<Preds>;4616 4617multiclass PTX_READ_SREG_R32V4<string regname, list<Predicate> Preds=[]> {4618 foreach suffix = ["x", "y", "z", "w"] in {4619 defvar reg = regname # "." # suffix;4620 defvar intr = !cast<Intrinsic>("int_nvvm_read_ptx_sreg_" # regname # "_" # suffix);4621 def "_"#suffix : PTX_READ_SREG_R32<reg, intr, Preds>;4622 }4623}4624 4625// TODO Add read vector-version of special registers4626 4627defm INT_PTX_SREG_TID : PTX_READ_SREG_R32V4<"tid">;4628defm INT_PTX_SREG_NTID : PTX_READ_SREG_R32V4<"ntid">;4629defm INT_PTX_SREG_CTAID : PTX_READ_SREG_R32V4<"ctaid">;4630defm INT_PTX_SREG_NCTAID: PTX_READ_SREG_R32V4<"nctaid">;4631 4632defm INT_PTX_SREG_CLUSTERID :4633 PTX_READ_SREG_R32V4<"clusterid", [hasSM<90>, hasPTX<78>]>;4634defm INT_PTX_SREG_NCLUSTERID :4635 PTX_READ_SREG_R32V4<"nclusterid", [hasSM<90>, hasPTX<78>]>;4636defm INT_PTX_SREG_CLUSTER_CTAID :4637 PTX_READ_SREG_R32V4<"cluster_ctaid", [hasSM<90>, hasPTX<78>]>;4638defm INT_PTX_SREG_CLUSTER_NCTAID:4639 PTX_READ_SREG_R32V4<"cluster_nctaid", [hasSM<90>, hasPTX<78>]>;4640 4641def INT_PTX_SREG_CLUSTER_CTARANK :4642 PTX_READ_SREG_R32<"cluster_ctarank",4643 int_nvvm_read_ptx_sreg_cluster_ctarank,4644 [hasSM<90>, hasPTX<78>]>;4645def INT_PTX_SREG_CLUSTER_NCTARANK:4646 PTX_READ_SREG_R32<"cluster_nctarank",4647 int_nvvm_read_ptx_sreg_cluster_nctarank,4648 [hasSM<90>, hasPTX<78>]>;4649 4650 4651def SREG_LANEID : PTX_READ_SREG_R32<"laneid", int_nvvm_read_ptx_sreg_laneid>;4652def SREG_WARPID : PTX_READ_SREG_R32<"warpid", int_nvvm_read_ptx_sreg_warpid>;4653def SREG_NWARPID : PTX_READ_SREG_R32<"nwarpid", int_nvvm_read_ptx_sreg_nwarpid>;4654def SREG_SMID : PTX_READ_SREG_R32<"smid", int_nvvm_read_ptx_sreg_smid>;4655def SREG_NSMID : PTX_READ_SREG_R32<"nsmid", int_nvvm_read_ptx_sreg_nsmid>;4656def SREG_GRIDID : PTX_READ_SREG_R32<"gridid", int_nvvm_read_ptx_sreg_gridid>;4657 4658def INT_PTX_SREG_LANEMASK_EQ :4659 PTX_READ_SREG_R32<"lanemask_eq", int_nvvm_read_ptx_sreg_lanemask_eq>;4660def INT_PTX_SREG_LANEMASK_LE :4661 PTX_READ_SREG_R32<"lanemask_le", int_nvvm_read_ptx_sreg_lanemask_le>;4662def INT_PTX_SREG_LANEMASK_LT :4663 PTX_READ_SREG_R32<"lanemask_lt", int_nvvm_read_ptx_sreg_lanemask_lt>;4664def INT_PTX_SREG_LANEMASK_GE :4665 PTX_READ_SREG_R32<"lanemask_ge", int_nvvm_read_ptx_sreg_lanemask_ge>;4666def INT_PTX_SREG_LANEMASK_GT :4667 PTX_READ_SREG_R32<"lanemask_gt", int_nvvm_read_ptx_sreg_lanemask_gt>;4668 4669let hasSideEffects = 1 in {4670 def SREG_CLOCK : PTX_READ_SREG_R32<"clock", int_nvvm_read_ptx_sreg_clock>;4671 def SREG_CLOCK64 : PTX_READ_SREG_R64<"clock64", int_nvvm_read_ptx_sreg_clock64>;4672 def SREG_GLOBALTIMER : PTX_READ_SREG_R64<"globaltimer", int_nvvm_read_ptx_sreg_globaltimer>;4673 def SREG_GLOBALTIMER_LO : PTX_READ_SREG_R32<"globaltimer_lo", int_nvvm_read_ptx_sreg_globaltimer_lo>;4674}4675 4676def: Pat <(i64 (readcyclecounter)), (SREG_CLOCK64)>;4677def: Pat <(i64 (readsteadycounter)), (SREG_GLOBALTIMER)>;4678def: Pat <(i32 (readsteadycounter)), (SREG_GLOBALTIMER_LO)>;4679 4680def INT_PTX_SREG_PM0 : PTX_READ_SREG_R32<"pm0", int_nvvm_read_ptx_sreg_pm0>;4681def INT_PTX_SREG_PM1 : PTX_READ_SREG_R32<"pm1", int_nvvm_read_ptx_sreg_pm1>;4682def INT_PTX_SREG_PM2 : PTX_READ_SREG_R32<"pm2", int_nvvm_read_ptx_sreg_pm2>;4683def INT_PTX_SREG_PM3 : PTX_READ_SREG_R32<"pm3", int_nvvm_read_ptx_sreg_pm3>;4684 4685// TODO: It would be nice to use PTX_READ_SREG here, but it doesn't4686// handle the constant.4687def INT_PTX_SREG_WARPSIZE :4688 NVPTXInst<(outs B32:$dst), (ins), "mov.u32 \t$dst, WARP_SZ;",4689 [(set i32:$dst, (int_nvvm_read_ptx_sreg_warpsize))]>;4690 4691// Helper class that represents a 'fragment' of an NVPTX *MMA instruction.4692// In addition to target-independent fields provided by WMMA_REGS, it adds4693// the fields commonly used to implement specific PTX instruction -- register4694// types and names, constraints, parts of assembly, etc.4695class WMMA_REGINFO<WMMA_REGS r, string op, string metadata = "", string kind = "">4696 : WMMA_REGS<r.geom, r.frag, r.ptx_elt_type,4697 !or(!eq(op, "mma.sp"), !eq(op, "mma.sp.block_scale"))> {4698 // NVPTX register types used to carry fragment data.4699 NVPTXRegClass regclass = !cond(4700 !eq(ptx_elt_type, "e4m3") : B32,4701 !eq(ptx_elt_type, "e5m2") : B32,4702 !eq(ptx_elt_type, "e3m2") : B32,4703 !eq(ptx_elt_type, "e2m3") : B32,4704 !eq(ptx_elt_type, "e2m1") : B32,4705 !eq(ptx_elt_type, "f16") : B32,4706 !eq(ptx_elt_type, "f32") : B32,4707 !eq(ptx_elt_type, "f64") : B64,4708 !eq(ptx_elt_type, "bf16") : B32,4709 !eq(ptx_elt_type, "tf32") : B32,4710 !eq(ptx_elt_type, "s32") : B32,4711 !eq(ptx_elt_type, "b16") : B32,4712 !eq(ptx_elt_type, "b8") : B32,4713 !eq(ptx_elt_type, "b8x16.b6x16_p32") : B32,4714 !eq(ptx_elt_type, "b8x16.b4x16_p64") : B32,4715 !eq(ptx_elt_type, "s8") : B32,4716 !eq(ptx_elt_type, "u8") : B32,4717 !eq(ptx_elt_type, "s4") : B32,4718 !eq(ptx_elt_type, "u4") : B32,4719 !eq(ptx_elt_type, "b1") : B32);4720 4721 // Instruction input/output arguments for the fragment.4722 list<NVPTXRegClass> ptx_regs = !listsplat(regclass, !size(regs));4723 4724 // List of register names for the fragment -- ["ra0", "ra1",...]4725 list<string> reg_names = RegSeq<!size(ptx_regs), "r"#frag>.ret;4726 4727 // Generates "{{$r0, $r1,.... $rN-1}}" for use in asm string construction.4728 string regstring = "{{$" # !interleave(reg_names, ", $") # "}}";4729 4730 // Predicates for particular fragment variant. Technically those are4731 // per-instruction predicates, but currently all fragments that can be used in4732 // a given instruction are subject to the same constraints, so an instruction4733 // can use predicates from any of its fragments. If/when this is no4734 // longer the case, we can concat all per-fragment predicates to enforce that4735 // all fragments of the instruction are viable.4736 list<Predicate> Predicates = !cond(4737 !or(!eq(op, "mma.block_scale"),4738 !eq(op, "mma.sp.block_scale")) : [hasSM120a, hasPTX<88>],4739 4740 !or(!eq(ptx_elt_type, "e3m2"),4741 !eq(ptx_elt_type, "e2m3"),4742 !eq(ptx_elt_type, "e2m1"),4743 !ne(kind, "")) : [hasSM120a, hasPTX<87>],4744 4745 !and(!or(!eq(ptx_elt_type,"e4m3"),4746 !eq(ptx_elt_type,"e5m2")),4747 !eq(geom, "m16n8k16")) : [hasSM<89>, hasPTX<87>],4748 4749 !or(!eq(ptx_elt_type, "e4m3"),4750 !eq(ptx_elt_type, "e5m2")) : [hasSM<89>, hasPTX<84>],4751 4752 !and(isSparse,4753 !ne(metadata, "sp")) : [hasSM<80>, hasPTX<85>],4754 isSparse : [hasSM<80>, hasPTX<71>],4755 4756 // fp16 -> fp16/fp32 @ m16n16k164757 !and(!eq(geom, "m16n16k16"),4758 !or(!eq(ptx_elt_type, "f16"),4759 !eq(ptx_elt_type, "f32"))) : [hasSM<70>, hasPTX<60>],4760 4761 !and(!eq(geom, "m8n8k4"),4762 !eq(ptx_elt_type, "f64")) : [hasSM<80>, hasPTX<70>],4763 4764 !and(!or(!eq(geom, "m16n8k4"),4765 !eq(geom, "m16n8k8"),4766 !eq(geom, "m16n8k16")),4767 !eq(ptx_elt_type, "f64")) : [hasSM<90>, hasPTX<78>],4768 4769 // fp16 -> fp16/fp32 @ m8n32k16/m32n8k164770 !and(!or(!eq(geom, "m8n32k16"),4771 !eq(geom, "m32n8k16")),4772 !or(!eq(ptx_elt_type, "f16"),4773 !eq(ptx_elt_type, "f32"))) : [hasSM<70>, hasPTX<61>],4774 4775 // u8/s8 -> s32 @ m16n16k16/m8n32k16/m32n8k164776 !and(!or(!eq(geom, "m16n16k16"),4777 !eq(geom, "m8n32k16"),4778 !eq(geom, "m32n8k16")),4779 !or(!eq(ptx_elt_type, "u8"),4780 !eq(ptx_elt_type, "s8"),4781 !eq(ptx_elt_type, "s32"))) : [hasSM<72>, hasPTX<63>],4782 4783 !and(!or(!eq(geom, "m16n16k16"),4784 !eq(geom, "m8n32k16"),4785 !eq(geom, "m32n8k16")),4786 !eq(ptx_elt_type, "bf16")) : [hasSM<80>, hasPTX<70>],4787 4788 !and(!eq(geom, "m16n16k8"),4789 !eq(ptx_elt_type, "tf32")) : [hasSM<80>, hasPTX<70>],4790 4791 !and(!eq(geom, "m16n16k8"),4792 !eq(ptx_elt_type, "f32")) : [hasSM<80>, hasPTX<70>],4793 4794 // b1 -> s32 @ m8n8k128(b1)4795 !and(!ne(op, "mma"),4796 !eq(geom, "m8n8k128")) : [hasSM<75>, hasPTX<63>],4797 4798 // u4/s4 -> s32 @ m8n8k32 (u4/s4)4799 !and(!ne(op, "mma"),4800 !eq(geom, "m8n8k32")) : [hasSM<75>, hasPTX<63>],4801 4802 !or(!eq(geom, "m16n8k8"),4803 !eq(geom, "m8n8k16")) : [hasSM<75>, hasPTX<65>],4804 4805 !and(!ne(ptx_elt_type, "f64"),4806 !eq(geom, "m8n8k4")) : [hasSM<70>, hasPTX<64>],4807 4808 // mma m8n8k32 requires higher PTX version4809 !and(!eq(op, "mma"),4810 !eq(geom, "m8n8k32")) : [hasSM<75>, hasPTX<65>],4811 4812 !and(!eq(ptx_elt_type, "f64"),4813 !eq(geom, "m8n8k4")) : [hasSM<80>, hasPTX<70>],4814 4815 !and(!eq(op, "mma"),4816 !or(!eq(geom, "m16n8k16"),4817 !eq(geom, "m16n8k4"),4818 !eq(geom, "m16n8k32"),4819 !eq(geom, "m16n8k64"),4820 !eq(geom, "m8n8k128"),4821 !eq(geom, "m16n8k128"),4822 !eq(geom, "m16n8k256"))) : [hasSM<80>, hasPTX<70>],4823 4824 !and(!eq(op, "ldmatrix"),4825 !eq(ptx_elt_type, "b16"),4826 !eq(geom, "m8n8")) : [hasSM<75>, hasPTX<65>],4827 4828 !and(!eq(op, "ldmatrix"),4829 !eq(ptx_elt_type, "b8"),4830 !eq(geom, "m16n16")) : [hasSM<100>, hasArchAccelFeatures, hasPTX<86>],4831 4832 !and(!eq(op, "ldmatrix"),4833 !eq(ptx_elt_type, "b8x16.b6x16_p32"),4834 !eq(geom, "m16n16")) : [hasSM<100>, hasArchAccelFeatures, hasPTX<86>],4835 4836 !and(!eq(op, "ldmatrix"),4837 !eq(ptx_elt_type, "b8x16.b4x16_p64"),4838 !eq(geom, "m16n16")) : [hasSM<100>, hasArchAccelFeatures, hasPTX<86>],4839 4840 !and(!eq(op, "ldmatrix"),4841 !eq(ptx_elt_type, "b8x16.b6x16_p32"),4842 !eq(geom, "m8n16")) : [hasSM<100>, hasArchAccelFeatures, hasPTX<86>],4843 4844 !and(!eq(op, "ldmatrix"),4845 !eq(ptx_elt_type, "b8x16.b4x16_p64"),4846 !eq(geom, "m8n16")) : [hasSM<100>, hasArchAccelFeatures, hasPTX<86>],4847 4848 !and(!eq(op, "stmatrix"),!eq(ptx_elt_type, "b16"),4849 !eq(geom, "m8n8")) : [hasSM<90>, hasPTX<78>],4850 4851 !and(!eq(op, "stmatrix"),4852 !eq(ptx_elt_type, "b8"),4853 !eq(geom, "m16n8")) : [hasSM<100>, hasArchAccelFeatures, hasPTX<86>]);4854 4855 // template DAGs for instruction inputs/output.4856 dag Outs = !dag(outs, ptx_regs, reg_names);4857 dag Ins = !dag(ins, ptx_regs, reg_names);4858}4859 4860// Convert dag of arguments into a dag to match given intrinsic.4861class BuildPatternI<Intrinsic Intr, dag Ins> {4862 // Build a dag pattern that matches the intrinsic call.4863 dag ret = !foreach(tmp, Ins,4864 !subst(ADDR, addr,4865 !subst(ins, Intr,4866 !subst(i32imm, timm, tmp))));4867}4868 4869// Same as above, but uses PatFrag instead of an Intrinsic.4870class BuildPatternPF<PatFrag Intr, dag Ins> {4871 // Build a dag pattern that matches the intrinsic call.4872 dag ret = !foreach(tmp, Ins,4873 !subst(ADDR, addr,4874 !subst(ins, Intr, tmp)));4875}4876 4877// Common WMMA-related fields used for building patterns for all MMA instructions.4878class WMMA_INSTR<string _Intr, list<dag> _Args>4879 : NVPTXInst<(outs), (ins), "?", []> {4880 Intrinsic Intr = !cast<Intrinsic>(_Intr);4881 // Concatenate all arguments into a single dag.4882 dag Args = !foldl((ins), _Args, a, b, !con(a, b));4883 // Pre-build the pattern to match (intrinsic arg0, arg1, ...).4884 dag IntrinsicPattern = BuildPatternI<!cast<Intrinsic>(Intr), Args>.ret;4885}4886 4887//4888// wmma.load.[a|b|c].sync.[row|col].m16n16k16[|.global|.shared].[f16|f32]4889//4890 4891class WMMA_LOAD<WMMA_REGINFO Frag, string Layout, string Space, bit WithStride>4892 : WMMA_INSTR<WMMA_NAME_LDST<"load", Frag, Layout, WithStride>.record_name,4893 [!con((ins ADDR:$src),4894 !if(WithStride, (ins B32:$ldm), (ins)))]>,4895 Requires<Frag.Predicates> {4896 // Load/store intrinsics are overloaded on pointer's address space.4897 // To match the right intrinsic, we need to build AS-constrained PatFrag.4898 // Operands is a dag equivalent in shape to Args, but using (ops node:$name, .....).4899 dag PFOperands = !if(WithStride, (ops node:$src, node:$ldm), (ops node:$src));4900 dag PFOperandsIntr = !if(WithStride, (Intr node:$src, node:$ldm), (Intr node:$src));4901 // Build PatFrag that only matches particular address space.4902 PatFrag IntrFrag = PatFrag<PFOperands,4903 PFOperandsIntr,4904 !cond(!eq(Space, ".shared"): AS_match.shared,4905 !eq(Space, ".global"): AS_match.global,4906 true: AS_match.generic)>;4907 // Build AS-constrained pattern.4908 let IntrinsicPattern = BuildPatternPF<IntrFrag, Args>.ret;4909 4910 let OutOperandList = Frag.Outs;4911 let InOperandList = !con(Args, (ins MmaCode:$ptx));4912 let AsmString = "wmma.load."4913 # Frag.frag4914 # ".sync"4915 # "${ptx:aligned}"4916 # "." # Layout4917 # "." # Frag.geom4918 # Space4919 # "." # Frag.ptx_elt_type # " \t"4920 # Frag.regstring4921 # ", [$src]"4922 # !if(WithStride, ", $ldm", "")4923 # ";";4924}4925 4926//4927// wmma.store.d.sync.[row|col].m16n16k16[|.global|.shared].[f16|f32]4928//4929class WMMA_STORE_D<WMMA_REGINFO Frag, string Layout, string Space,4930 bit WithStride>4931 : WMMA_INSTR<WMMA_NAME_LDST<"store", Frag, Layout, WithStride>.record_name,4932 [!con((ins ADDR:$dst),4933 Frag.Ins,4934 !if(WithStride, (ins B32:$ldm), (ins)))]>,4935 Requires<Frag.Predicates> {4936 4937 // Load/store intrinsics are overloaded on pointer's address space.4938 // To match the right intrinsic, we need to build AS-constrained PatFrag.4939 // Operands is a dag equivalent in shape to Args, but using (ops node:$name, .....).4940 dag PFOperands = !con((ops node:$dst),4941 !dag(ops, !listsplat(node, !size(Frag.regs)), Frag.reg_names),4942 !if(WithStride, (ops node:$ldm), (ops)));4943 // Build PatFrag that only matches particular address space.4944 PatFrag IntrFrag = PatFrag<PFOperands,4945 !foreach(tmp, PFOperands, !subst(ops, Intr, tmp)),4946 !cond(!eq(Space, ".shared"): AS_match.shared,4947 !eq(Space, ".global"): AS_match.global,4948 true: AS_match.generic)>;4949 // Build AS-constrained pattern.4950 let IntrinsicPattern = BuildPatternPF<IntrFrag, Args>.ret;4951 4952 let InOperandList = !con(Args, (ins MmaCode:$ptx));4953 let OutOperandList = (outs);4954 let AsmString = "wmma.store.d.sync"4955 # "${ptx:aligned}"4956 # "." # Layout4957 # "." # Frag.geom4958 # Space4959 # "." # Frag.ptx_elt_type4960 # " \t[$dst],"4961 # Frag.regstring4962 # !if(WithStride, ", $ldm", "")4963 # ";";4964}4965 4966// Create all load/store variants4967defset list<WMMA_INSTR> MMA_LDSTs = {4968 foreach layout = ["row", "col"] in {4969 foreach stride = [false, true] in {4970 foreach space = [".global", ".shared", ""] in {4971 foreach frag = NVVM_MMA_OPS.all_ld_ops in4972 if NVVM_WMMA_LDST_SUPPORTED<frag, layout>.ret then4973 def : WMMA_LOAD<WMMA_REGINFO<frag, "load">, layout, space, stride>;4974 foreach frag = NVVM_MMA_OPS.all_st_ops in4975 if NVVM_WMMA_LDST_SUPPORTED<frag, layout>.ret then4976 def : WMMA_STORE_D<WMMA_REGINFO<frag, "store">, layout, space, stride>;4977 } // space4978 } // stride4979 } // layout4980} // defset4981 4982// B1 instruction variants need extra constraints.4983class MMA_OP_PREDICATES<WMMA_REGINFO FragA, string b1op> {4984 string Op = b1op;4985 WMMA_REGINFO Frag = FragA;4986 list<Predicate> ret = !listconcat(4987 FragA.Predicates,4988 !if(!eq(b1op, ".and.popc"), [hasSM<80>, hasPTX<71>], [])4989 );4990}4991// WMMA.MMA4992class WMMA_MMA<WMMA_REGINFO FragA, WMMA_REGINFO FragB,4993 WMMA_REGINFO FragC, WMMA_REGINFO FragD,4994 string ALayout, string BLayout, int Satfinite, string rnd, string b1op>4995 : WMMA_INSTR<WMMA_NAME<ALayout, BLayout, Satfinite, rnd, b1op, FragA, FragB, FragC, FragD>.record_name,4996 [FragA.Ins, FragB.Ins, FragC.Ins]>,4997 // Requires does not seem to have effect on Instruction w/o Patterns.4998 // We set it here anyways and propagate to the Pat<> we construct below.4999 Requires<MMA_OP_PREDICATES<FragA, b1op>.ret> {5000 let OutOperandList = FragD.Outs;5001 let InOperandList = !con(Args, (ins MmaCode:$ptx));5002 string TypeList = !cond(5003 !eq(FragA.ptx_elt_type, "f16") : "." # FragD.ptx_elt_type5004 # "." # FragC.ptx_elt_type,5005 1: "." # FragD.ptx_elt_type5006 # "." # FragA.ptx_elt_type5007 # "." # FragB.ptx_elt_type5008 # "." # FragC.ptx_elt_type,5009 );5010 let AsmString = "wmma.mma"5011 # b1op5012 # ".sync"5013 # "${ptx:aligned}"5014 # "." # ALayout5015 # "." # BLayout5016 # "." # FragA.geom5017 # !if(!ne(rnd, ""), !strconcat(".", rnd), "")5018 # TypeList5019 # !if(Satfinite, ".satfinite", "") # "\n\t\t"5020 # FragD.regstring # ",\n\t\t"5021 # FragA.regstring # ",\n\t\t"5022 # FragB.regstring # ",\n\t\t"5023 # FragC.regstring # ";";5024}5025 5026let isConvergent = true in {5027defset list<WMMA_INSTR> WMMAs = {5028 foreach layout_a = ["row", "col"] in {5029 foreach layout_b = ["row", "col"] in {5030 foreach satf = [0, 1] in {5031 foreach rnd = ["", "rn", "rz", "rm", "rp"] in {5032 foreach op = NVVM_MMA_OPS.all_wmma_ops in {5033 foreach b1op = NVVM_MMA_B1OPS<op>.ret in {5034 if NVVM_WMMA_SUPPORTED<op, layout_a, layout_b, satf, rnd>.ret then {5035 def : WMMA_MMA<WMMA_REGINFO<op[0], "wmma.mma">,5036 WMMA_REGINFO<op[1], "wmma.mma">,5037 WMMA_REGINFO<op[2], "wmma.mma">,5038 WMMA_REGINFO<op[3], "wmma.mma">,5039 layout_a, layout_b, satf, rnd, b1op>;5040 }5041 } // b1op5042 } // op5043 } // rnd5044 } // satf5045 } // layout_b5046 } // layout_a5047} // defset5048}5049 5050// MMA5051class MMA<WMMA_REGINFO FragA, WMMA_REGINFO FragB,5052 WMMA_REGINFO FragC, WMMA_REGINFO FragD,5053 string ALayout, string BLayout, int Satfinite, string b1op, string Kind>5054 : WMMA_INSTR<MMA_NAME<ALayout, BLayout, Satfinite, b1op, Kind, FragA, FragB, FragC, FragD>.record_name,5055 [FragA.Ins, FragB.Ins, FragC.Ins]>,5056 // Requires does not seem to have effect on Instruction w/o Patterns.5057 // We set it here anyways and propagate to the Pat<> we construct below.5058 Requires<MMA_OP_PREDICATES<FragA, b1op>.ret> {5059 let OutOperandList = FragD.Outs;5060 let InOperandList = !con(Args, (ins MmaCode:$ptx));5061 string TypeList = "." # FragD.ptx_elt_type5062 # "." # FragA.ptx_elt_type5063 # "." # FragB.ptx_elt_type5064 # "." # FragC.ptx_elt_type;5065 let AsmString = "mma.sync.aligned."5066 # FragA.geom5067 # "." # ALayout5068 # "." # BLayout5069 # !if(!ne(Kind, ""), "." # Kind, "")5070 # !if(Satfinite, ".satfinite", "")5071 # TypeList5072 # b1op # "\n\t\t"5073 # FragD.regstring # ",\n\t\t"5074 # FragA.regstring # ",\n\t\t"5075 # FragB.regstring # ",\n\t\t"5076 # FragC.regstring # ";";5077}5078 5079let isConvergent = true in {5080defset list<WMMA_INSTR> MMAs = {5081 foreach layout_a = ["row", "col"] in {5082 foreach layout_b = ["row", "col"] in {5083 foreach satf = [0, 1] in {5084 foreach op = NVVM_MMA_OPS.all_mma_ops in {5085 foreach b1op = NVVM_MMA_B1OPS<op>.ret in {5086 foreach kind = ["", "kind::f8f6f4"] in {5087 if NVVM_MMA_SUPPORTED<op, layout_a, layout_b, kind, satf>.ret then {5088 def : MMA<WMMA_REGINFO<op[0], "mma", "", kind>,5089 WMMA_REGINFO<op[1], "mma", "", kind>,5090 WMMA_REGINFO<op[2], "mma", "", kind>,5091 WMMA_REGINFO<op[3], "mma", "", kind>,5092 layout_a, layout_b, satf, b1op, kind>;5093 }5094 } // kind5095 } // b1op5096 } // op5097 } // satf5098 } // layout_b5099 } // layout_a5100} // defset5101}5102 5103// MMA.block_scale5104class MMA_BLOCK_SCALE<WMMA_REGINFO FragA, WMMA_REGINFO FragB,5105 WMMA_REGINFO FragC, WMMA_REGINFO FragD,5106 string Kind, string SType, string ScaleVecSize>5107 : WMMA_INSTR<MMA_BLOCK_SCALE_NAME<Kind, SType, ScaleVecSize,5108 FragA, FragB, FragC, FragD>.record_name,5109 [FragA.Ins, FragB.Ins, FragC.Ins,5110 (ins B32:$scale_a, B16:$byte_id_a,5111 B16:$thread_id_a, B32:$scale_b,5112 B16:$byte_id_b, B16:$thread_id_b)]>,5113 // Requires does not seem to have effect on Instruction w/o Patterns.5114 // We set it here anyways and propagate to the Pat<> we construct below.5115 Requires<FragA.Predicates> {5116 let OutOperandList = FragD.Outs;5117 let InOperandList = !con(Args, (ins MmaCode:$ptx));5118 string TypeList = !interleave([FragD.ptx_elt_type,5119 FragA.ptx_elt_type,5120 FragB.ptx_elt_type,5121 FragC.ptx_elt_type], ".");5122 string ScaleVecSizeStr = !cond(5123 !eq(ScaleVecSize, "") : "",5124 !eq(ScaleVecSize, ".scale_1x") : ".scale_vec::1X",5125 !eq(ScaleVecSize, ".scale_2x") : ".scale_vec::2X",5126 !eq(ScaleVecSize, ".scale_4x") : ".scale_vec::4X"5127 );5128 let AsmString = "mma.sync.aligned."5129 # FragA.geom5130 # ".row.col"5131 # ".kind::" # Kind5132 # ".block_scale"5133 # ScaleVecSizeStr5134 # "." # TypeList5135 # "." # SType # " \n\t\t"5136 # FragD.regstring # ",\n\t\t"5137 # FragA.regstring # ",\n\t\t"5138 # FragB.regstring # ",\n\t\t"5139 # FragC.regstring # ",\n\t\t"5140 # "$scale_a, {{$byte_id_a, $thread_id_a}}" # ",\n\t\t"5141 # "$scale_b, {{$byte_id_b, $thread_id_b}};";5142}5143 5144let isConvergent = true in {5145defset list<WMMA_INSTR> MMA_BLOCK_SCALEs = {5146 foreach kind = ["mxf4", "mxf4nvf4", "mxf8f6f4"] in {5147 foreach scale_vec_size = ["", ".scale_1x", ".scale_2x", ".scale_4x"] in {5148 foreach stype = ["ue8m0", "ue4m3"] in {5149 foreach op = NVVM_MMA_OPS.all_mma_block_scale_ops in {5150 if NVVM_MMA_BLOCK_SCALE_SUPPORTED<op, kind, stype, scale_vec_size>.ret then {5151 def : MMA_BLOCK_SCALE<WMMA_REGINFO<op[0], "mma.block_scale", "", kind>,5152 WMMA_REGINFO<op[1], "mma.block_scale", "", kind>,5153 WMMA_REGINFO<op[2], "mma.block_scale", "", kind>,5154 WMMA_REGINFO<op[3], "mma.block_scale", "", kind>,5155 kind, stype, scale_vec_size>;5156 }5157 } // op5158 } // stype5159 } // scale_vec_size5160 } // kind5161} // defset5162}5163 5164// MMA SP5165class MMA_SP<WMMA_REGINFO FragA, WMMA_REGINFO FragB,5166 WMMA_REGINFO FragC, WMMA_REGINFO FragD,5167 string Metadata, string Kind, int Satfinite>5168 : WMMA_INSTR<MMA_SP_NAME<Metadata, Kind, Satfinite,5169 FragA, FragB, FragC, FragD>.record_name,5170 [FragA.Ins, FragB.Ins, FragC.Ins,5171 (ins B32:$metadata, i32imm:$selector)]>,5172 // Requires does not seem to have effect on Instruction w/o Patterns.5173 // We set it here anyways and propagate to the Pat<> we construct below.5174 Requires<!listconcat(FragA.Predicates,5175 FragB.Predicates,5176 FragC.Predicates,5177 FragD.Predicates)> {5178 let OutOperandList = FragD.Outs;5179 let InOperandList = !con(Args, (ins MmaCode:$ptx));5180 string TypeList = "." # FragD.ptx_elt_type5181 # "." # FragA.ptx_elt_type5182 # "." # FragB.ptx_elt_type5183 # "." # FragC.ptx_elt_type;5184 let AsmString = "mma"5185 # "." # Metadata5186 # ".sync.aligned."5187 # FragA.geom5188 # ".row.col"5189 # !if(!ne(Kind, ""), "." # Kind, "")5190 # !if(Satfinite, ".satfinite", "")5191 # TypeList # "\n\t\t"5192 # FragD.regstring # ",\n\t\t"5193 # FragA.regstring # ",\n\t\t"5194 # FragB.regstring # ",\n\t\t"5195 # FragC.regstring # ",\n\t\t"5196 # "$metadata" # ",\n\t\t"5197 # "$selector" # ";";5198}5199 5200let isConvergent = true in {5201defset list<WMMA_INSTR> MMA_SPs = {5202 foreach metadata = ["sp", "sp::ordered_metadata"] in {5203 foreach kind = ["", "kind::f8f6f4"] in {5204 foreach satf = [0, 1] in {5205 foreach op = NVVM_MMA_OPS.all_mma_sp_ops in {5206 if NVVM_MMA_SP_SUPPORTED<op, metadata, kind, satf>.ret then {5207 def : MMA_SP<WMMA_REGINFO<op[0], "mma.sp", metadata, kind>,5208 WMMA_REGINFO<op[1], "mma.sp", metadata, kind>,5209 WMMA_REGINFO<op[2], "mma.sp", metadata, kind>,5210 WMMA_REGINFO<op[3], "mma.sp", metadata, kind>,5211 metadata, kind, satf>;5212 }5213 } // op5214 } // satf5215 } // kind5216 } // metadata5217} // defset5218}5219 5220// MMA SP BLOCK SCALE5221class MMA_SP_BLOCK_SCALE<WMMA_REGINFO FragA, WMMA_REGINFO FragB,5222 WMMA_REGINFO FragC, WMMA_REGINFO FragD,5223 string Kind, string SType, string ScaleVecSize>5224 : WMMA_INSTR<MMA_SP_BLOCK_SCALE_NAME<Kind, SType, ScaleVecSize,5225 FragA, FragB, FragC, FragD>.record_name,5226 [FragA.Ins, FragB.Ins, FragC.Ins,5227 (ins B32:$metadata, i32imm:$selector,5228 B32:$scale_a, B16:$byte_id_a, B16:$thread_id_a,5229 B32:$scale_b, B16:$byte_id_b, B16:$thread_id_b)]>,5230 // Requires does not seem to have effect on Instruction w/o Patterns.5231 // We set it here anyways and propagate to the Pat<> we construct below.5232 Requires<!listconcat(FragA.Predicates,5233 FragB.Predicates,5234 FragC.Predicates,5235 FragD.Predicates)> {5236 let OutOperandList = FragD.Outs;5237 let InOperandList = !con(Args, (ins MmaCode:$ptx));5238 string TypeList = "." # FragD.ptx_elt_type5239 # "." # FragA.ptx_elt_type5240 # "." # FragB.ptx_elt_type5241 # "." # FragC.ptx_elt_type;5242 string ScaleVecSizeStr = !cond(5243 !eq(ScaleVecSize, "") : "",5244 !eq(ScaleVecSize, ".scale_1x") : ".scale_vec::1X",5245 !eq(ScaleVecSize, ".scale_2x") : ".scale_vec::2X",5246 !eq(ScaleVecSize, ".scale_4x") : ".scale_vec::4X"5247 );5248 let AsmString = "mma.sp::ordered_metadata.sync.aligned."5249 # FragA.geom5250 # ".row.col"5251 # ".kind::" # Kind5252 # ".block_scale"5253 # ScaleVecSizeStr5254 # TypeList5255 # "." # SType # "\n\t\t"5256 # FragD.regstring # ",\n\t\t"5257 # FragA.regstring # ",\n\t\t"5258 # FragB.regstring # ",\n\t\t"5259 # FragC.regstring # ",\n\t\t"5260 # "$metadata" # ",\n\t\t"5261 # "$selector" # ",\n\t\t"5262 # "$scale_a, {{$byte_id_a, $thread_id_a}}" # ",\n\t\t"5263 # "$scale_b, {{$byte_id_b, $thread_id_b}};";5264}5265 5266let isConvergent = true in {5267defset list<WMMA_INSTR> MMA_SP_BLOCK_SCALEs = {5268 foreach kind = ["mxf4", "mxf4nvf4", "mxf8f6f4"] in {5269 foreach scale_vec_size = ["", ".scale_1x", ".scale_2x", ".scale_4x"] in {5270 foreach stype = ["ue8m0", "ue4m3"] in {5271 foreach op = NVVM_MMA_OPS.all_mma_sp_block_scale_ops in {5272 if NVVM_MMA_SP_BLOCK_SCALE_SUPPORTED<op, kind, stype, scale_vec_size>.ret then {5273 def : MMA_SP_BLOCK_SCALE<WMMA_REGINFO<op[0], "mma.sp.block_scale", "sp::ordered_metadata", kind>,5274 WMMA_REGINFO<op[1], "mma.sp.block_scale", "sp::ordered_metadata", kind>,5275 WMMA_REGINFO<op[2], "mma.sp.block_scale", "sp::ordered_metadata", kind>,5276 WMMA_REGINFO<op[3], "mma.sp.block_scale", "sp::ordered_metadata", kind>,5277 kind, stype, scale_vec_size>;5278 }5279 } // op5280 } // stype5281 } // scale_vec_size5282 } // kind5283} // defset5284}5285 5286//5287// ldmatrix.sync.aligned.m8n8[|.trans][|.shared].b165288//5289class LDMATRIX<WMMA_REGINFO Frag, bit Transposed, string Space>5290 : WMMA_INSTR<LDMATRIX_NAME<Frag, Transposed>.record_name, [(ins ADDR:$src)]>,5291 Requires<Frag.Predicates> {5292 // Build PatFrag that only matches particular address space.5293 PatFrag IntrFrag = PatFrag<(ops node:$src), (Intr node:$src),5294 !cond(!eq(Space, ".shared"): AS_match.shared,5295 true: AS_match.generic)>;5296 // Build AS-constrained pattern.5297 let IntrinsicPattern = BuildPatternPF<IntrFrag, Args>.ret;5298 5299 let OutOperandList = Frag.Outs;5300 let InOperandList = !con(Args, (ins MmaCode:$ptx));5301 let AsmString = "ldmatrix.sync.aligned."5302 # Frag.geom5303 # "." # Frag.frag5304 # !if(Transposed, ".trans", "")5305 # Space5306 # "." # Frag.ptx_elt_type5307 # " " # Frag.regstring # ", [$src];";5308}5309 5310// Create all ldmatrix variants5311defset list<WMMA_INSTR> LDMATRIXs = {5312 foreach transposed = [false, true] in {5313 foreach space = [".shared", ""] in {5314 foreach frag = NVVM_MMA_OPS.all_ldmatrix_ops in5315 if NVVM_LDMATRIX_SUPPORTED<frag, transposed>.ret then5316 def : LDMATRIX<WMMA_REGINFO<frag, "ldmatrix">, transposed, space>;5317 } // space5318 } // transposed5319} // defset5320 5321//5322// stmatrix.sync.aligned.m8n8[|.trans][|.shared].b165323//5324class STMATRIX<WMMA_REGINFO Frag, bit Transposed, string Space>5325 : WMMA_INSTR<STMATRIX_NAME<Frag, Transposed>.record_name, [!con((ins ADDR:$dst), Frag.Ins)]>,5326 Requires<Frag.Predicates> {5327 // Build PatFrag that only matches particular address space.5328 dag PFOperands = !con((ops node:$dst),5329 !dag(ops, !listsplat(node, !size(Frag.regs)), Frag.reg_names));5330 PatFrag IntrFrag = PatFrag<PFOperands,5331 !foreach(tmp, PFOperands, !subst(ops, Intr, tmp)),5332 !cond(!eq(Space, ".shared"): AS_match.shared,5333 true: AS_match.generic)>;5334 // Build AS-constrained pattern.5335 let IntrinsicPattern = BuildPatternPF<IntrFrag, Args>.ret;5336 let OutOperandList = (outs);5337 let InOperandList = !con(Args, (ins MmaCode:$ptx));5338 let AsmString = "stmatrix.sync.aligned."5339 # Frag.geom5340 # "." # Frag.frag5341 # !if(Transposed, ".trans", "")5342 # Space5343 # "." # Frag.ptx_elt_type5344 # " [$dst], " # Frag.regstring # ";";5345}5346 5347// Create all stmatrix variants5348defset list<WMMA_INSTR> STMATRIXs = {5349 foreach transposed = [false, true] in {foreach space = [".shared", ""] in {5350 foreach frag = NVVM_MMA_OPS.all_stmatrix_ops in5351 if NVVM_STMATRIX_SUPPORTED<frag, transposed>.ret then5352 def : STMATRIX<WMMA_REGINFO<frag, "stmatrix">, transposed, space>;5353 } // space5354 } // transposed5355} // defset5356 5357// Constructing non-flat DAGs is still a pain. I can't !subst a dag node with a5358// dag, so the ptx.version must be appended *after* foreach replaces 'ins' with5359// the instruction record.5360class MMA_PAT<WMMA_INSTR wi>5361 : Pat<wi.IntrinsicPattern,5362 !con(!foreach(tmp, wi.Args, !subst(ins, wi, tmp)),5363 (wi ptx.version))>,5364 Requires<wi.Predicates>;5365 5366// Build intrinsic->instruction patterns for all MMA instructions.5367foreach mma = !listconcat(MMAs, MMA_BLOCK_SCALEs, WMMAs, MMA_LDSTs, LDMATRIXs,5368 STMATRIXs, MMA_SPs, MMA_SP_BLOCK_SCALEs) in5369 def : MMA_PAT<mma>;5370 5371multiclass MAPA<string suffix, Intrinsic Intr> {5372 let Predicates = [hasSM<90>, hasPTX<78>] in {5373 def _32: BasicNVPTXInst<(outs B32:$d), (ins B32:$a, B32:$b),5374 "mapa" # suffix # ".u32",5375 [(set i32:$d, (Intr i32:$a, i32:$b))]>;5376 def _32i: BasicNVPTXInst<(outs B32:$d), (ins B32:$a, i32imm:$b),5377 "mapa" # suffix # ".u32",5378 [(set i32:$d, (Intr i32:$a, imm:$b))]>;5379 def _64: BasicNVPTXInst<(outs B64:$d), (ins B64:$a, B32:$b),5380 "mapa" # suffix # ".u64",5381 [(set i64:$d, (Intr i64:$a, i32:$b))]>;5382 def _64i: BasicNVPTXInst<(outs B64:$d), (ins B64:$a, i32imm:$b),5383 "mapa" # suffix # ".u64",5384 [(set i64:$d, (Intr i64:$a, imm:$b))]>;5385 }5386}5387 5388 5389defm mapa : MAPA<"", int_nvvm_mapa>;5390defm mapa_shared_cluster : MAPA<".shared::cluster", int_nvvm_mapa_shared_cluster>;5391 5392 5393multiclass GETCTARANK<string suffix, Intrinsic Intr> {5394 let Predicates = [hasSM<90>, hasPTX<78>] in {5395 def _32: BasicNVPTXInst<(outs B32:$d), (ins B32:$a),5396 "getctarank" # suffix # ".u32",5397 [(set i32:$d, (Intr i32:$a))]>;5398 def _64: BasicNVPTXInst<(outs B32:$d), (ins B64:$a),5399 "getctarank" # suffix # ".u64",5400 [(set i32:$d, (Intr i64:$a))]>;5401 }5402}5403 5404defm getctarank : GETCTARANK<"", int_nvvm_getctarank>;5405defm getctarank_shared_cluster : GETCTARANK<".shared::cluster", int_nvvm_getctarank_shared_cluster>;5406 5407def is_explicit_cluster: NVPTXInst<(outs B1:$d), (ins),5408 "mov.pred\t$d, %is_explicit_cluster;",5409 [(set i1:$d, (int_nvvm_is_explicit_cluster))]>,5410 Requires<[hasSM<90>, hasPTX<78>]>;5411 5412// setmaxnreg inc/dec intrinsics5413let isConvergent = true in {5414multiclass SET_MAXNREG<string Action, Intrinsic Intr> {5415 def : BasicNVPTXInst<(outs), (ins i32imm:$reg_count),5416 "setmaxnreg." # Action # ".sync.aligned.u32",5417 [(Intr timm:$reg_count)]>,5418 Requires<[hasArchAccelFeatures, hasSM<90>, hasPTX<80>]>;5419}5420 5421defm INT_SET_MAXNREG_INC : SET_MAXNREG<"inc", int_nvvm_setmaxnreg_inc_sync_aligned_u32>;5422defm INT_SET_MAXNREG_DEC : SET_MAXNREG<"dec", int_nvvm_setmaxnreg_dec_sync_aligned_u32>;5423 5424} // isConvergent5425 5426//5427// WGMMA fence instructions5428//5429let isConvergent = true, Predicates = [hasSM90a, hasPTX<80>] in {5430 def WGMMA_FENCE_SYNC_ALIGNED : NullaryInst<"wgmma.fence.sync.aligned", int_nvvm_wgmma_fence_sync_aligned>;5431 5432 def WGMMA_COMMIT_GROUP_SYNC_ALIGNED : NullaryInst<"wgmma.commit_group.sync.aligned", int_nvvm_wgmma_commit_group_sync_aligned>;5433 5434 def WGMMA_WAIT_GROUP_SYNC_ALIGNED : BasicNVPTXInst<(outs), (ins i64imm:$n), "wgmma.wait_group.sync.aligned",5435 [(int_nvvm_wgmma_wait_group_sync_aligned timm:$n)]>;5436}5437 5438let Predicates = [hasSM<90>, hasPTX<78>] in {5439 def GRIDDEPCONTROL_LAUNCH_DEPENDENTS :5440 NullaryInst<"griddepcontrol.launch_dependents", int_nvvm_griddepcontrol_launch_dependents>;5441 def GRIDDEPCONTROL_WAIT :5442 NullaryInst<"griddepcontrol.wait", int_nvvm_griddepcontrol_wait>;5443}5444 5445def EXIT : NullaryInst<"exit", int_nvvm_exit>;5446 5447// Tcgen05 intrinsics5448let isConvergent = true in {5449let Predicates = [callSubtarget<"hasTcgen05InstSupport">] in {5450multiclass TCGEN05_ALLOC_INTR<string AS, string num, Intrinsic Intr> {5451 def "" : BasicNVPTXInst<(outs),5452 (ins ADDR:$dst, B32:$ncols),5453 "tcgen05.alloc.cta_group::" # num # ".sync.aligned" # AS # ".b32",5454 [(Intr addr:$dst, B32:$ncols)]>;5455}5456 5457defm TCGEN05_ALLOC_CG1 : TCGEN05_ALLOC_INTR<"", "1", int_nvvm_tcgen05_alloc_cg1>;5458defm TCGEN05_ALLOC_CG2 : TCGEN05_ALLOC_INTR<"", "2", int_nvvm_tcgen05_alloc_cg2>;5459 5460defm TCGEN05_ALLOC_S64_CG1 : TCGEN05_ALLOC_INTR<".shared::cta", "1", int_nvvm_tcgen05_alloc_shared_cg1>;5461defm TCGEN05_ALLOC_S64_CG2 : TCGEN05_ALLOC_INTR<".shared::cta", "2", int_nvvm_tcgen05_alloc_shared_cg2>;5462 5463multiclass TCGEN05_DEALLOC_INTR<string num, Intrinsic Intr> {5464 def "" : BasicNVPTXInst<(outs),5465 (ins B32:$tmem_addr, B32:$ncols),5466 "tcgen05.dealloc.cta_group::" # num # ".sync.aligned.b32",5467 [(Intr B32:$tmem_addr, B32:$ncols)]>;5468}5469defm TCGEN05_DEALLOC_CG1: TCGEN05_DEALLOC_INTR<"1", int_nvvm_tcgen05_dealloc_cg1>;5470defm TCGEN05_DEALLOC_CG2: TCGEN05_DEALLOC_INTR<"2", int_nvvm_tcgen05_dealloc_cg2>;5471 5472multiclass TCGEN05_RELINQ_PERMIT_INTR<string num, Intrinsic Intr> {5473 def "" : NullaryInst<"tcgen05.relinquish_alloc_permit.cta_group::" # num # ".sync.aligned", Intr>;5474}5475defm TCGEN05_RELINQ_CG1: TCGEN05_RELINQ_PERMIT_INTR<"1", int_nvvm_tcgen05_relinq_alloc_permit_cg1>;5476defm TCGEN05_RELINQ_CG2: TCGEN05_RELINQ_PERMIT_INTR<"2", int_nvvm_tcgen05_relinq_alloc_permit_cg2>;5477 5478def tcgen05_wait_ld: NullaryInst<"tcgen05.wait::ld.sync.aligned", int_nvvm_tcgen05_wait_ld>;5479def tcgen05_wait_st: NullaryInst<"tcgen05.wait::st.sync.aligned", int_nvvm_tcgen05_wait_st>;5480 5481multiclass TCGEN05_COMMIT_INTR<string AS, string num> {5482 defvar prefix = "tcgen05.commit.cta_group::" # num #".mbarrier::arrive::one.shared::cluster";5483 5484 defvar intr_suffix = !if(!eq(AS, "shared"), "_shared", "") # "_cg" # num;5485 defvar Intr = !cast<Intrinsic>("int_nvvm_tcgen05_commit" # intr_suffix);5486 defvar IntrMC = !cast<Intrinsic>("int_nvvm_tcgen05_commit_mc" # intr_suffix);5487 5488 def "" : BasicNVPTXInst<(outs), (ins ADDR:$mbar),5489 prefix # ".b64",5490 [(Intr addr:$mbar)]>;5491 def _MC : BasicNVPTXInst<(outs), (ins ADDR:$mbar, B16:$mc),5492 prefix # ".multicast::cluster.b64",5493 [(IntrMC addr:$mbar, B16:$mc)]>;5494}5495 5496defm TCGEN05_COMMIT_CG1 : TCGEN05_COMMIT_INTR<"", "1">;5497defm TCGEN05_COMMIT_CG2 : TCGEN05_COMMIT_INTR<"", "2">;5498defm TCGEN05_COMMIT_S64_CG1 : TCGEN05_COMMIT_INTR<"shared", "1">;5499defm TCGEN05_COMMIT_S64_CG2 : TCGEN05_COMMIT_INTR<"shared", "2">;5500 5501multiclass TCGEN05_CP_INTR<string shape, string src_fmt, string mc = ""> {5502 defvar dst_fmt = !if(!eq(src_fmt, ""), "", ".b8x16");5503 defvar fmt_asm = StrJoin<".", [dst_fmt, src_fmt]>.ret;5504 defvar fmt_intr = StrJoin<"_", [src_fmt]>.ret;5505 5506 defvar shape_mc_asm = StrJoin<".", [shape, mc]>.ret;5507 defvar shape_mc_intr = !subst("::", "_", !subst(".", "_", shape_mc_asm));5508 5509 defvar intr_prefix = StrJoin<"_", ["int_nvvm_tcgen05_cp", shape_mc_intr, fmt_intr]>.ret;5510 defvar IntrCG1 = !cast<Intrinsic>(intr_prefix # "_cg1");5511 defvar IntrCG2 = !cast<Intrinsic>(intr_prefix # "_cg2");5512 5513 def _cg1 : BasicNVPTXInst<(outs),5514 (ins ADDR:$tmem_addr, B64:$sdesc),5515 "tcgen05.cp.cta_group::1." # shape_mc_asm # fmt_asm,5516 [(IntrCG1 addr:$tmem_addr, B64:$sdesc)]>;5517 def _cg2 : BasicNVPTXInst<(outs),5518 (ins ADDR:$tmem_addr, B64:$sdesc),5519 "tcgen05.cp.cta_group::2." # shape_mc_asm # fmt_asm,5520 [(IntrCG2 addr:$tmem_addr, B64:$sdesc)]>;5521}5522 5523foreach src_fmt = ["", "b6x16_p32", "b4x16_p64"] in {5524 defm TCGEN05_CP_128x256b # src_fmt : TCGEN05_CP_INTR<"128x256b", src_fmt>;5525 defm TCGEN05_CP_4x256b # src_fmt : TCGEN05_CP_INTR<"4x256b", src_fmt>;5526 defm TCGEN05_CP_128x128b # src_fmt : TCGEN05_CP_INTR<"128x128b", src_fmt>;5527 defm TCGEN05_CP_64x128_1 # src_fmt : TCGEN05_CP_INTR<"64x128b", src_fmt, "warpx2::02_13">;5528 defm TCGEN05_CP_64x128_2 # src_fmt : TCGEN05_CP_INTR<"64x128b", src_fmt, "warpx2::01_23">;5529 defm TCGEN05_CP_32x128 # src_fmt : TCGEN05_CP_INTR<"32x128b", src_fmt, "warpx4">;5530}5531} // Predicates5532 5533let Predicates = [callSubtarget<"hasTcgen05ShiftSupport">] in {5534multiclass TCGEN05_SHIFT_INTR<string num, Intrinsic Intr> {5535 def "" : BasicNVPTXInst<(outs),5536 (ins ADDR:$tmem_addr),5537 "tcgen05.shift.cta_group::" # num # ".down",5538 [(Intr addr:$tmem_addr)]>;5539}5540defm TCGEN05_SHIFT_CG1: TCGEN05_SHIFT_INTR<"1", int_nvvm_tcgen05_shift_down_cg1>;5541defm TCGEN05_SHIFT_CG2: TCGEN05_SHIFT_INTR<"2", int_nvvm_tcgen05_shift_down_cg2>;5542} // Predicates5543 5544} // isConvergent5545 5546let hasSideEffects = 1, Predicates = [callSubtarget<"hasTcgen05InstSupport">] in {5547 5548 def tcgen05_fence_before_thread_sync: NullaryInst<5549 "tcgen05.fence::before_thread_sync", int_nvvm_tcgen05_fence_before_thread_sync>;5550 5551 def tcgen05_fence_after_thread_sync: NullaryInst<5552 "tcgen05.fence::after_thread_sync", int_nvvm_tcgen05_fence_after_thread_sync>;5553 5554} // hasSideEffects5555 5556// name class for tcgen05.{ld, st}5557class TCGEN05_LDST_INST_NAME<string Op, string shape, int lg2Count, bit packOrUnpack> {5558 string name = "TCGEN05_" # Op5559 # "_" # shape5560 # "_x" # !shl(1, lg2Count)5561 # !if(!eq(packOrUnpack, 1), !if(!eq(Op, "LD"), "_PACK", "_UNPACK"), "");5562}5563 5564// reginfo class tcgen05.{ld, st}5565class TCGEN05_LDST_REGINFO<int Veclen> {5566 // create a list of types for load/store operands5567 list<NVPTXRegClass> regs = !listsplat(B32, Veclen);5568 // generate list of regnames for load/store operands5569 list<string> reg_names = !foreach(x, !range(0, Veclen), "r" # x);5570 string regstring = "{{" # !interleave(!foreach(n, !range(0, Veclen), "$r" # n), ", ") # "}}";5571 dag Ins = !dag(ins, regs, reg_names);5572 dag Outs = !dag(outs, regs, reg_names);5573}5574 5575//5576// tcgen05.ld.sync.aligned.shape.x[1, 2, 4, 8, 16, 32, 64, 128][|.pack::16b].[b32]5577//5578 5579class TCGEN05_LD_INST<string Shape, int Num, bit Pack> :5580 NVPTXInst<(outs), (ins), "?", []> {5581 5582 TCGEN05_LDST_REGINFO Info = TCGEN05_LDST_REGINFO<5583 NVVM_TCGEN05_LDST_ACCESS_SIZE<Shape, Num>.veclen>;5584 5585 let InOperandList = !con((ins B32:$taddr),5586 !if(!eq(Shape, "16x32bx2"), (ins i64imm:$offset), (ins)));5587 let OutOperandList = Info.Outs;5588 let AsmString = "tcgen05.ld.sync.aligned"5589 # "." # Shape5590 # ".x" # !shl(1, Num)5591 # !if(!eq(Pack, 1), ".pack::16b", "")5592 # ".b32 "5593 # Info.regstring # ", "5594 # "[$taddr]"5595 # !if(!eq(Shape, "16x32bx2"), ", $offset", "")5596 # ";";5597}5598 5599//5600// tcgen05.st.sync.aligned.shape.x[1, 2, 4, 8, 16, 32, 64, 128][|.unpack::16b].[b32]5601//5602 5603class TCGEN05_ST_INST<string Shape, int Num, bit Unpack> :5604 NVPTXInst<(outs), (ins), "?", []> {5605 5606 TCGEN05_LDST_REGINFO Info = TCGEN05_LDST_REGINFO<5607 NVVM_TCGEN05_LDST_ACCESS_SIZE<Shape, Num>.veclen>;5608 5609 let InOperandList = !con((ins B32:$taddr),5610 !if(!eq(Shape, "16x32bx2"), (ins i64imm:$offset), (ins)),5611 Info.Ins);5612 let OutOperandList = (outs);5613 let AsmString = "tcgen05.st.sync.aligned"5614 # "." # Shape5615 # ".x" # !shl(1, Num)5616 # !if(!eq(Unpack, 1), ".unpack::16b", "")5617 # ".b32 [$taddr]"5618 # !if(!eq(Shape, "16x32bx2"), ", $offset", "")5619 # ", " # Info.regstring5620 # ";";5621}5622 5623let isConvergent = true in {5624 5625foreach shape = ["16x64b", "16x128b", "16x256b", "32x32b", "16x32bx2"] in {5626 foreach num = !range(0, 8) in {5627 foreach packOrUnpack = [false, true] in {5628 if NVVM_TCGEN05_LDST_ACCESS_SIZE<shape, num>.valid then {5629 def TCGEN05_LDST_INST_NAME<"LD", shape, num, packOrUnpack>.name :5630 TCGEN05_LD_INST<shape, num, packOrUnpack>;5631 def TCGEN05_LDST_INST_NAME<"ST", shape, num, packOrUnpack>.name :5632 TCGEN05_ST_INST<shape, num, packOrUnpack>;5633 }5634 }5635 }5636}5637 5638} // isConvergent5639 5640// Bulk store instructions5641def st_bulk_imm : TImmLeaf<i64, [{ return Imm == 0; }]>;5642 5643let Predicates = [hasSM<100>, hasPTX<86>] in {5644 def INT_NVVM_ST_BULK_GENERIC :5645 BasicNVPTXInst<(outs), (ins ADDR:$dest_addr, B64:$size, i64imm:$value),5646 "st.bulk",5647 [(int_nvvm_st_bulk addr:$dest_addr, i64:$size, st_bulk_imm:$value)]>;5648 5649 def INT_NVVM_ST_BULK_SHARED_CTA:5650 BasicNVPTXInst<(outs), (ins ADDR:$dest_addr, B64:$size, i64imm:$value),5651 "st.bulk.shared::cta",5652 [(int_nvvm_st_bulk_shared_cta addr:$dest_addr, i64:$size, st_bulk_imm:$value)]>;5653}5654 5655//5656// clusterlaunchcontorl Instructions5657//5658 5659def CLUSTERLAUNCHCONTRL_TRY_CANCEL:5660 BasicNVPTXInst<(outs), (ins ADDR:$addr, ADDR:$mbar),5661 "clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128",5662 [(int_nvvm_clusterlaunchcontrol_try_cancel_async_shared addr:$addr, addr:$mbar)]>,5663 Requires<[hasSM<100>, hasPTX<86>]>;5664 5665def CLUSTERLAUNCHCONTRL_TRY_CANCEL_MULTICAST:5666 BasicNVPTXInst<(outs), (ins ADDR:$addr, ADDR:$mbar),5667 "clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes" #5668 ".multicast::cluster::all.b128",5669 [(int_nvvm_clusterlaunchcontrol_try_cancel_async_multicast_shared addr:$addr, addr:$mbar)]>,5670 Requires<[hasSM<100>, hasArchAccelFeatures, hasPTX<86>]>;5671 5672def SDTClusterLaunchControlQueryCancelIsCanceled: SDTypeProfile<1, 2, []>;5673def clusterlaunchcontrol_query_cancel_is_canceled:5674 SDNode<"NVPTXISD::CLUSTERLAUNCHCONTROL_QUERY_CANCEL_IS_CANCELED",5675 SDTClusterLaunchControlQueryCancelIsCanceled, []>;5676 5677def CLUSTERLAUNCHCONTROL_QUERY_CANCEL_IS_CANCELED:5678 NVPTXInst<(outs B1:$pred), (ins B64:$try_cancel_response0, B64:$try_cancel_response1),5679 "{{\n\t" #5680 ".reg .b128 %clc_handle;\n\t" #5681 "mov.b128 %clc_handle, {$try_cancel_response0, $try_cancel_response1};\n\t" #5682 "clusterlaunchcontrol.query_cancel.is_canceled.pred.b128 $pred, %clc_handle;\n\t" #5683 "}}", [(set i1:$pred,5684 (clusterlaunchcontrol_query_cancel_is_canceled i64:$try_cancel_response0, i64:$try_cancel_response1))]>,5685 Requires<[hasSM<100>, hasPTX<86>]>;5686 5687class CLUSTERLAUNCHCONTROL_QUERY_CANCEL_GET_FIRST_CTAID<string Dim>:5688 NVPTXInst<(outs B32:$reg), (ins B64:$try_cancel_response0, B64:$try_cancel_response1),5689 "{{\n\t" #5690 ".reg .b128 %clc_handle;\n\t" #5691 "mov.b128 %clc_handle, {$try_cancel_response0, $try_cancel_response1};\n\t" #5692 "clusterlaunchcontrol.query_cancel.get_first_ctaid::" # Dim # ".b32.b128 $reg, %clc_handle;\n\t" #5693 "}}", [(set i32:$reg,5694 (!cast<SDNode>("clusterlaunchcontrol_query_cancel_first_cta_id_" # Dim)5695 i64:$try_cancel_response0, i64:$try_cancel_response1))]>,5696 Requires<[hasSM<100>, hasPTX<86>]>;5697 5698foreach dim = ["x", "y", "z"] in {5699 def SDTClusterLaunchControlQueryCancelGetFirstCtaId # dim: SDTypeProfile<1, 2, []>;5700 5701 def clusterlaunchcontrol_query_cancel_first_cta_id_ # dim :5702 SDNode<"NVPTXISD::CLUSTERLAUNCHCONTROL_QUERY_CANCEL_GET_FIRST_CTAID_" # !toupper(dim),5703 !cast<SDTypeProfile>("SDTClusterLaunchControlQueryCancelGetFirstCtaId" # dim), []>;5704 5705 def CLUSTERLAUNCHCONTROL_QUERY_CANCEL_GET_FIRST_CTAID_ # dim:5706 CLUSTERLAUNCHCONTROL_QUERY_CANCEL_GET_FIRST_CTAID<dim>;5707}5708 5709//5710// tcgen05.mma Instructions5711//5712 5713class Tcgen05MMAInst<bit Sp, string KindStr, string ASpace,5714 int CtaGroup, string CollectorUsage,5715 bit ScaleInputD, bit AShift,5716 list<Predicate> PTXPredicates> :5717 NVPTXInst<(outs), (ins), "?", []>,5718 Requires<PTXPredicates> {5719 5720 Intrinsic Intrin = !cast<Intrinsic>(5721 NVVM_TCGEN05_MMA<Sp, ASpace, AShift, ScaleInputD>.record_name5722 );5723 5724 dag ScaleInpIns = !if(!eq(ScaleInputD, 1), (ins i64imm:$scale_input_d), (ins));5725 string ScaleInpStr = !if(!eq(ScaleInputD, 1), ", $scale_input_d", "");5726 dag ScaleInpInput = !if(!eq(ScaleInputD, 1), (Intrin i64:$scale_input_d), (Intrin));5727 5728 dag SparseMetadataIns = !if(!eq(Sp, 1), (ins B32:$spmetadata), (ins));5729 dag SparseMetadataIntr = !if(!eq(Sp, 1), (Intrin B32:$spmetadata), (Intrin));5730 string SparseMetadataStr = !if(!eq(Sp, 1), ", [$spmetadata]", "");5731 5732 int KindVal = !cond(5733 !eq(KindStr, "f16"): 0,5734 !eq(KindStr, "tf32"): 1,5735 !eq(KindStr, "f8f6f4"): 2,5736 !eq(KindStr, "i8"): 3,5737 );5738 5739 int CollectorUsageVal = !cond(5740 !eq(CollectorUsage, "discard"): 0,5741 !eq(CollectorUsage, "lastuse"): 1,5742 !eq(CollectorUsage, "fill"): 2,5743 !eq(CollectorUsage, "use"): 35744 );5745 5746 string AOperandStr = !if(!eq(ASpace, "tensor"), "[$a]", "$a");5747 5748 NVPTXRegClass ARegClass = !if(!eq(ASpace, "tensor"), B32, B64);5749 5750 dag input = !con((ins B32:$dtmem,5751 ARegClass:$a, ADDR:$b,5752 B32:$idesc,5753 B1:$enable_inp_d),5754 SparseMetadataIns,5755 ScaleInpIns);5756 5757 let InOperandList = input;5758 let OutOperandList = (outs);5759 let AsmString = "tcgen05.mma"5760 # !if(!eq(Sp, 1), ".sp", "")5761 # ".cta_group::" # CtaGroup5762 # ".kind::" # KindStr5763 # ".collector::a::" # CollectorUsage5764 # !if(!eq(AShift, 1), ".ashift", "")5765 # " [$dtmem], " # AOperandStr # ", $b"5766 # SparseMetadataStr5767 # ", $idesc, $enable_inp_d"5768 # ScaleInpStr5769 # ";";5770 5771 dag IntrinsicPattern = !con((Intrin i32:$dtmem,5772 ARegClass:$a, addr:$b,5773 i32:$idesc,5774 i1:$enable_inp_d),5775 SparseMetadataIntr,5776 ScaleInpInput);5777 5778 dag FlagOperands = (Intrin (i32 KindVal), (i32 CtaGroup),5779 (i32 CollectorUsageVal));5780 5781 let Pattern = [!con(IntrinsicPattern, FlagOperands)];5782}5783 5784// tcgen05.mma5785foreach sp = [0, 1] in {5786 foreach space = ["tensor", "shared"] in {5787 foreach kind = ["f16", "tf32", "f8f6f4", "i8"] in {5788 foreach cta_group = [1, 2] in {5789 foreach collector_usage = ["discard", "lastuse", "fill", "use"] in {5790 foreach scale_input_d = !if(!or(!eq(kind, "f16"),5791 !eq(kind, "tf32")), [0, 1], [0]) in {5792 foreach ashift = !if(!eq(space, "tensor"), [0, 1], [0]) in {5793 5794 def : Tcgen05MMAInst<sp, kind, space, cta_group, collector_usage,5795 scale_input_d, ashift,5796 !if(!eq(scale_input_d, 1),5797 [hasTcgen05MMAScaleInputDImm],5798 [hasTcgen05Instructions])>;5799 }5800 }5801 }5802 }5803 }5804 }5805}5806 5807class Tcgen05MMADisableOutputLaneTypeProfile<bit Sp, string ASpace,5808 int CtaGroup, bit ScaleInputD>:5809 SDTypeProfile<0, 0, []> {5810 int DisableOutputLaneVecSize = !mul(4, CtaGroup);5811 5812 list<ValueType> VTs = !listconcat(5813 [i32], // d5814 !if(!eq(ASpace, "tensor"), [i32], [i64]), // a5815 [i64, i32, i1], // b, idesc, enable_inp_d5816 !if(!eq(Sp, 1), [i32], []), // spmetadata5817 !if(!eq(ScaleInputD, 1), [i64], []), // scale_input_d5818 !listsplat(i32, DisableOutputLaneVecSize), // disable_output_lane5819 [i32, i32] // kind, collector_usage5820 );5821 let Constraints = !foreach(x, !range(!size(VTs)), SDTCisVT<x, VTs[x]>);5822 let NumOperands = !size(Constraints);5823}5824 5825class Tcgen05MMADisableOutputLaneSDNode<bit Sp, string ASpace,5826 int CtaGroup, bit ScaleInput, bit AShift>:5827 SDNode<"NVPTXISD::TCGEN05_MMA"5828 # !if(!eq(Sp, 1), "_SP", "")5829 # "_" # !toupper(ASpace)5830 # !if(!eq(ScaleInput, 1), "_SCALE_D", "")5831 # "_DISABLE_OUTPUT_LANE_CG" # CtaGroup5832 # !if(!eq(AShift, 1), "_ASHIFT", ""),5833 Tcgen05MMADisableOutputLaneTypeProfile<Sp, ASpace, CtaGroup, ScaleInput>,5834 [SDNPHasChain, SDNPSideEffect, SDNPMemOperand]>;5835 5836class Tcgen05MMADisableOutputLaneInst<bit Sp, string ASpace,5837 string Kind, int CtaGroup, string CollectorUsageStr,5838 bit ScaleInputD, bit AShift,5839 list<Predicate> PTXPredicates> :5840 NVPTXInst<(outs), (ins), "?", []>,5841 Requires<PTXPredicates> {5842 5843 SDNode Opcode = Tcgen05MMADisableOutputLaneSDNode<Sp, ASpace, CtaGroup,5844 ScaleInputD, AShift>;5845 5846 5847 dag ScaleInpIns = !if(!eq(ScaleInputD, 1), (ins i64imm:$scale_input_d), (ins));5848 string ScaleInpStr = !if(!eq(ScaleInputD, 1), ", $scale_input_d", "");5849 dag ScaleInpInput = !if(!eq(ScaleInputD, 1), (Opcode i64:$scale_input_d), (Opcode));5850 5851 // disable output lane5852 int DisableOutputLaneVecSize = !mul(4, CtaGroup);5853 5854 dag DisableOutputLaneIns = !dag(ins,5855 !listsplat(B32, DisableOutputLaneVecSize),5856 !foreach(x,5857 !range(DisableOutputLaneVecSize),5858 "disable_output_lane" # x));5859 5860 dag DisableOutputLaneInput = !dag(Opcode,5861 !listsplat(i32, DisableOutputLaneVecSize),5862 !foreach(x,5863 !range(DisableOutputLaneVecSize),5864 "disable_output_lane" # x));5865 5866 string DisableOutputLaneStr = "{{" #5867 !interleave(5868 !foreach(x,5869 !range(DisableOutputLaneVecSize),5870 "$disable_output_lane" # x),5871 ", ")5872 # "}}";5873 5874 dag SparseMetadataIns = !if(!eq(Sp, 1), (ins B32:$spmetadata), (ins));5875 dag SparseMetadataIntr = !if(!eq(Sp, 1), (Opcode i32:$spmetadata), (Opcode));5876 string SparseMetadataStr = !if(!eq(Sp, 1), ", [$spmetadata]", "");5877 5878 int KindVal = !cond(5879 !eq(Kind, "f16"): 0,5880 !eq(Kind, "tf32"): 1,5881 !eq(Kind, "f8f6f4"): 2,5882 !eq(Kind, "i8"): 3,5883 );5884 5885 int CollectorUsage = !cond(5886 !eq(CollectorUsageStr, "discard"): 0,5887 !eq(CollectorUsageStr, "lastuse"): 1,5888 !eq(CollectorUsageStr, "fill"): 2,5889 !eq(CollectorUsageStr, "use"): 3,5890 );5891 5892 string AOperandStr = !if(!eq(ASpace, "tensor"), "[$a]", "$a");5893 5894 NVPTXRegClass ARegClass = !if(!eq(ASpace, "tensor"), B32, B64);5895 5896 dag InOperandList = !con((ins B32:$dtmem,5897 ARegClass:$a, B64:$b,5898 B32:$idesc,5899 B1:$enable_inp_d),5900 SparseMetadataIns,5901 ScaleInpIns,5902 DisableOutputLaneIns);5903 5904 let OutOperandList = (outs);5905 let AsmString = "tcgen05.mma"5906 # !if(!eq(Sp, 1), ".sp", "")5907 # ".cta_group::" # CtaGroup5908 # ".kind::" # Kind5909 # !if(!eq(AShift, 1), ".ashift", "")5910 # ".collector::a::" # CollectorUsageStr5911 # " " # "[$dtmem], " # AOperandStr # ", $b"5912 # SparseMetadataStr5913 # ", " # "$idesc"5914 # ", " # DisableOutputLaneStr5915 # ", $enable_inp_d"5916 # ScaleInpStr5917 # ";";5918 5919 dag IntrinsicPattern = !con((Opcode i32:$dtmem,5920 ARegClass:$a, i64:$b,5921 i32:$idesc,5922 i1:$enable_inp_d),5923 SparseMetadataIntr,5924 ScaleInpInput,5925 DisableOutputLaneInput);5926 5927 dag FlagOperands = (Opcode (i32 KindVal), (i32 CollectorUsage));5928 5929 let Pattern = [!con(IntrinsicPattern, FlagOperands)];5930}5931 5932// tcgen05.mma.disable_output_lane5933foreach sp = [0, 1] in {5934 foreach space = ["tensor", "shared"] in {5935 foreach kind = ["f16", "tf32", "f8f6f4", "i8"] in {5936 foreach cta_group = [1, 2] in {5937 foreach collector_usage = ["fill", "use", "lastuse", "discard"] in {5938 foreach scale_input_d = !if(!or(!eq(kind, "f16"),5939 !eq(kind, "tf32")), [0, 1], [0]) in {5940 foreach ashift = !if(!eq(space, "tensor"), [0, 1], [0]) in {5941 def :5942 Tcgen05MMADisableOutputLaneInst<sp, space, kind, cta_group,5943 collector_usage, scale_input_d,5944 ashift,5945 !if(!eq(scale_input_d, 1),5946 [hasTcgen05MMAScaleInputDImm],5947 [hasTcgen05Instructions])>;5948 }5949 }5950 }5951 }5952 }5953 }5954}5955 5956class Tcgen05MMABlockScaleInst<bit Sp, string ASpace, string KindStr,5957 int CtaGroup, string ScaleVecSize, string CollectorUsageStr,5958 Predicate PTXPredicate>:5959 NVPTXInst<(outs), (ins), "?", []>,5960 Requires<[hasTcgen05Instructions, PTXPredicate]> {5961 5962 Intrinsic Intrin = !cast<Intrinsic>(5963 NVVM_TCGEN05_MMA_BLOCKSCALE<Sp, ASpace, KindStr, ScaleVecSize>.record_name);5964 5965 dag SparseMetadataIns = !if(!eq(Sp, 1), (ins B32:$spmetadata), (ins));5966 dag SparseMetadataIntr = !if(!eq(Sp, 1), (Intrin i32:$spmetadata), (Intrin));5967 string SparseMetadataStr = !if(!eq(Sp, 1), ", [$spmetadata]", "");5968 5969 int KindVal = !cond(5970 !eq(KindStr, "mxf8f6f4") : 0,5971 !eq(KindStr, "mxf4") : 1,5972 !eq(KindStr, "mxf4nvf4") : 2,5973 );5974 5975 int CollectorUsage = !cond(5976 !eq(CollectorUsageStr, "discard") : 0,5977 !eq(CollectorUsageStr, "lastuse") : 1,5978 !eq(CollectorUsageStr, "fill") : 2,5979 !eq(CollectorUsageStr, "use") : 3,5980 );5981 5982 string AOperandStr = !if(!eq(ASpace, "tensor"), "[$a]", "$a");5983 NVPTXRegClass ARegClass = !if(!eq(ASpace, "tensor"), B32, B64);5984 5985 dag input = !con((ins B32:$dtmem, ARegClass:$a, B64:$b,5986 B32:$idesc, B1:$enable_inp_d),5987 SparseMetadataIns,5988 (ins B32:$scale_a,5989 B32:$scale_b));5990 5991 let InOperandList = input;5992 let OutOperandList = (outs);5993 let AsmString = "tcgen05.mma"5994 # !if(!eq(Sp, 1), ".sp", "")5995 # ".cta_group::" # CtaGroup5996 # ".kind::" # KindStr5997 # ".block_scale" # ScaleVecSize5998 # ".collector::a::" # CollectorUsageStr5999 # " [$dtmem], " # AOperandStr # ", $b"6000 # SparseMetadataStr6001 # ", $idesc, [$scale_a], [$scale_b], $enable_inp_d;";6002 6003 dag IntrinsicPattern = !con((Intrin i32:$dtmem,6004 ARegClass:$a, i64:$b,6005 i32:$idesc,6006 i1:$enable_inp_d),6007 SparseMetadataIntr,6008 (Intrin i32:$scale_a,6009 i32:$scale_b));6010 6011 dag FlagOperands = (Intrin (i32 CtaGroup), (i32 CollectorUsage));6012 6013 let Pattern = [!con(IntrinsicPattern, FlagOperands)];6014}6015 6016// tcgen05.mma.block_scale6017foreach sp = [0, 1] in {6018 foreach space = ["tensor", "shared"] in {6019 foreach kind = ["mxf8f6f4", "mxf4", "mxf4nvf4"] in {6020 foreach scale_vec_size = ["", ".block16", ".block32"] in {6021 foreach cta_group = [1, 2] in {6022 foreach collector_usage = ["fill", "use", "lastuse", "discard"] in {6023 if NVVM_TCGEN05_MMA_BLOCKSCALE_SUPPORTED<kind, scale_vec_size>.ret then {6024 def : Tcgen05MMABlockScaleInst<sp, space, kind, cta_group, scale_vec_size,6025 collector_usage,6026 !if(!ne(scale_vec_size, ""),6027 hasPTX<88>, hasPTX<86>)>;6028 }6029 }6030 }6031 }6032 }6033 }6034}6035 6036//6037// tcgen05.mma.ws Instructions6038//6039 6040class Tcgen05MMAWSInst<bit Sp, string ASpace, string KindStr,6041 int CollectorBufferB, string CollectorUsageOpStr,6042 bit HasZeroColMask> :6043 NVPTXInst<(outs), (ins), "?", []>,6044 Requires<[hasTcgen05Instructions]> {6045 6046 Intrinsic Intrin = !cast<Intrinsic>(6047 NVVM_TCGEN05_MMA_WS<Sp, ASpace, HasZeroColMask>.record_name);6048 6049 dag ZeroColMaskIns = !if(!eq(HasZeroColMask, 1),6050 (ins B64:$zero_col_mask), (ins));6051 string ZeroColMaskStr = !if(!eq(HasZeroColMask, 1), ", $zero_col_mask", "");6052 dag ZeroColMaskIntr = !if(!eq(HasZeroColMask, 1),6053 (Intrin i64:$zero_col_mask), (Intrin));6054 6055 dag SparseMetadataIns = !if(!eq(Sp, 1), (ins B32:$spmetadata), (ins));6056 dag SparseMetadataIntr = !if(!eq(Sp, 1), (Intrin B32:$spmetadata), (Intrin));6057 string SparseMetadataStr = !if(!eq(Sp, 1), ", [$spmetadata]", "");6058 6059 int KindVal = !cond(6060 !eq(KindStr, "f16") : 0,6061 !eq(KindStr, "tf32") : 1,6062 !eq(KindStr, "f8f6f4"): 2,6063 !eq(KindStr, "i8") : 3,6064 );6065 6066 int CollectorUsageOp = !cond(6067 !eq(CollectorUsageOpStr, "discard"): 0,6068 !eq(CollectorUsageOpStr, "lastuse"): 1,6069 !eq(CollectorUsageOpStr, "fill") : 2,6070 !eq(CollectorUsageOpStr, "use") : 3,6071 );6072 6073 string AOperandStr = !if(!eq(ASpace, "tensor"), "[$a]", "$a");6074 NVPTXRegClass ARegClass = !if(!eq(ASpace, "tensor"), B32, B64);6075 6076 dag input = !con((ins B32:$dtmem,6077 ARegClass:$a, B64:$b,6078 B32:$idesc,6079 B1:$enable_inp_d),6080 SparseMetadataIns,6081 ZeroColMaskIns);6082 6083 let InOperandList = input;6084 let OutOperandList = (outs);6085 let AsmString = "tcgen05.mma.ws"6086 # !if(!eq(Sp, 1), ".sp", "")6087 # ".cta_group::1"6088 # ".kind::" # KindStr6089 # ".collector::b" # CollectorBufferB6090 # "::" # CollectorUsageOpStr6091 # " [$dtmem], " # AOperandStr # ", $b"6092 # SparseMetadataStr6093 # ", $idesc, $enable_inp_d"6094 # ZeroColMaskStr6095 # ";";6096 6097 dag IntrinsicPattern = !con((Intrin i32:$dtmem,6098 ARegClass:$a, i64:$b,6099 i32:$idesc,6100 i1:$enable_inp_d),6101 SparseMetadataIntr,6102 ZeroColMaskIntr);6103 6104 dag FlagOperands = (Intrin (i32 KindVal), (i32 CollectorBufferB),6105 (i32 CollectorUsageOp));6106 6107 let Pattern = [!con(IntrinsicPattern, FlagOperands)];6108}6109 6110// tcgen05.mma.ws6111foreach sp = [0, 1] in {6112 foreach space = ["shared", "tensor"] in {6113 foreach kind = ["f16", "tf32", "f8f6f4", "i8"] in {6114 foreach collector_buffer_b = [0, 1, 2, 3] in {6115 foreach collector_usage_op = ["discard", "fill", "use", "lastuse"] in {6116 foreach zero_col_mask = [0, 1] in {6117 def : Tcgen05MMAWSInst<sp, space, kind, collector_buffer_b,6118 collector_usage_op, zero_col_mask>;6119 }6120 }6121 }6122 }6123 }6124}6125