2144 lines · plain
1//===-- MIMGInstructions.td - MIMG Instruction Definitions ----------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9// MIMG-specific encoding families to distinguish between semantically10// equivalent machine instructions with different encoding.11//12// - MIMGEncGfx6: encoding introduced with gfx6 (obsoleted for atomics in gfx8)13// - MIMGEncGfx8: encoding introduced with gfx8 for atomics14// - MIMGEncGfx90a: encoding for gfx90a for atomics15// - MIMGEncGfx10Default: gfx10 default (non-NSA) encoding16// - MIMGEncGfx10NSA: gfx10 NSA encoding17// - MIMGEncGfx11Default: gfx11 default (non-NSA) encoding18// - MIMGEncGfx11NSA: gfx11 partial NSA encoding19// - MIMGEncGfx12: gfx12 encoding (partial NSA)20class MIMGEncoding;21 22def MIMGEncGfx6 : MIMGEncoding;23def MIMGEncGfx8 : MIMGEncoding;24def MIMGEncGfx90a : MIMGEncoding;25def MIMGEncGfx10Default : MIMGEncoding;26def MIMGEncGfx10NSA : MIMGEncoding;27def MIMGEncGfx11Default : MIMGEncoding;28def MIMGEncGfx11NSA : MIMGEncoding;29def MIMGEncGfx12 : MIMGEncoding;30 31def MIMGEncoding : GenericEnum {32 let FilterClass = "MIMGEncoding";33}34 35// Represent an ISA-level opcode, independent of the encoding and the36// vdata/vaddr size.37class MIMGBaseOpcode : PredicateControl {38 MIMGBaseOpcode BaseOpcode = !cast<MIMGBaseOpcode>(NAME);39 bit Store = 0;40 bit Atomic = 0;41 bit AtomicX2 = 0; // (f)cmpswap42 bit Sampler = 0;43 bit Gather4 = 0;44 bits<8> NumExtraArgs = 0;45 bit Gradients = 0;46 bit G16 = 0;47 bit Coordinates = 1;48 bit LodOrClampOrMip = 0;49 bit HasD16 = 0;50 bit IsAtomicRet = 0;51 bit MSAA = 0;52 bit BVH = 0;53 bit A16 = 0;54 bit NoReturn = 0;55 bit PointSampleAccel = 0; // Opcode eligible for gfx11.5 point sample acceleration56}57 58def MIMGBaseOpcode : GenericEnum {59 let FilterClass = "MIMGBaseOpcode";60}61 62def MIMGBaseOpcodesTable : GenericTable {63 let FilterClass = "MIMGBaseOpcode";64 let CppTypeName = "MIMGBaseOpcodeInfo";65 let Fields = ["BaseOpcode", "Store", "Atomic", "AtomicX2", "Sampler",66 "Gather4", "NumExtraArgs", "Gradients", "G16", "Coordinates",67 "LodOrClampOrMip", "HasD16", "MSAA", "BVH", "A16", "NoReturn",68 "PointSampleAccel"];69 string TypeOf_BaseOpcode = "MIMGBaseOpcode";70 71 let PrimaryKey = ["BaseOpcode"];72 let PrimaryKeyName = "getMIMGBaseOpcodeInfo";73}74 75def MIMGDim : GenericEnum {76 let FilterClass = "AMDGPUDimProps";77}78 79def MIMGDimInfoTable : GenericTable {80 let FilterClass = "AMDGPUDimProps";81 let CppTypeName = "MIMGDimInfo";82 let Fields = ["Dim", "NumCoords", "NumGradients", "MSAA", "DA", "Encoding", "AsmSuffix"];83 string TypeOf_Dim = "MIMGDim";84 85 let PrimaryKey = ["Dim"];86 let PrimaryKeyName = "getMIMGDimInfo";87}88 89def getMIMGDimInfoByEncoding : SearchIndex {90 let Table = MIMGDimInfoTable;91 let Key = ["Encoding"];92}93 94def getMIMGDimInfoByAsmSuffix : SearchIndex {95 let Table = MIMGDimInfoTable;96 let Key = ["AsmSuffix"];97}98 99def MIMG {100 int NOP = -1;101}102 103class mimgopc <int gfx12, int gfx11, int gfx10m, int vi = gfx10m, int si = gfx10m> {104 field bits<8> GFX12 = gfx12;105 field bits<8> GFX11 = gfx11;106 field bits<8> GFX10M = gfx10m; // GFX10minus for all but atomics107 field bits<8> VI = vi; // VI is only used for atomic/sampler/gather instructions108 field bits<8> SI = si; // SI is only used for atomic instructions109 bit HAS_GFX12 = !ne(gfx12, MIMG.NOP);110 bit HAS_GFX11 = !ne(gfx11, MIMG.NOP);111 bit HAS_GFX10M = !ne(gfx10m, MIMG.NOP);112 bit HAS_VI = !ne(vi, MIMG.NOP);113 bit HAS_SI = !ne(si, MIMG.NOP);114}115 116class MIMGLZMapping<MIMGBaseOpcode l, MIMGBaseOpcode lz> {117 MIMGBaseOpcode L = l;118 MIMGBaseOpcode LZ = lz;119}120 121def MIMGLZMappingTable : GenericTable {122 let FilterClass = "MIMGLZMapping";123 let CppTypeName = "MIMGLZMappingInfo";124 let Fields = ["L", "LZ"];125 string TypeOf_L = "MIMGBaseOpcode";126 string TypeOf_LZ = "MIMGBaseOpcode";127 128 let PrimaryKey = ["L"];129 let PrimaryKeyName = "getMIMGLZMappingInfo";130}131 132class MIMGMIPMapping<MIMGBaseOpcode mip, MIMGBaseOpcode nonmip> {133 MIMGBaseOpcode MIP = mip;134 MIMGBaseOpcode NONMIP = nonmip;135}136 137def MIMGMIPMappingTable : GenericTable {138 let FilterClass = "MIMGMIPMapping";139 let CppTypeName = "MIMGMIPMappingInfo";140 let Fields = ["MIP", "NONMIP"];141 string TypeOf_MIP = "MIMGBaseOpcode";142 string TypeOf_NONMIP = "MIMGBaseOpcode";143 144 let PrimaryKey = ["MIP"];145 let PrimaryKeyName = "getMIMGMIPMappingInfo";146}147 148class MIMGBiasMapping<MIMGBaseOpcode bias, MIMGBaseOpcode nobias> {149 MIMGBaseOpcode Bias = bias;150 MIMGBaseOpcode NoBias = nobias;151}152 153def MIMGBiasMappingTable : GenericTable {154 let FilterClass = "MIMGBiasMapping";155 let CppTypeName = "MIMGBiasMappingInfo";156 let Fields = ["Bias", "NoBias"];157 string TypeOf_Bias = "MIMGBaseOpcode";158 string TypeOf_NoBias = "MIMGBaseOpcode";159 160 let PrimaryKey = ["Bias"];161 let PrimaryKeyName = "getMIMGBiasMappingInfo";162}163 164class MIMGOffsetMapping<MIMGBaseOpcode offset, MIMGBaseOpcode nooffset> {165 MIMGBaseOpcode Offset = offset;166 MIMGBaseOpcode NoOffset = nooffset;167}168 169def MIMGOffsetMappingTable : GenericTable {170 let FilterClass = "MIMGOffsetMapping";171 let CppTypeName = "MIMGOffsetMappingInfo";172 let Fields = ["Offset", "NoOffset"];173 string TypeOf_Offset = "MIMGBaseOpcode";174 string TypeOf_NoOffset = "MIMGBaseOpcode";175 176 let PrimaryKey = ["Offset"];177 let PrimaryKeyName = "getMIMGOffsetMappingInfo";178}179 180class MIMGG16Mapping<MIMGBaseOpcode g, MIMGBaseOpcode g16> {181 MIMGBaseOpcode G = g;182 MIMGBaseOpcode G16 = g16;183}184 185def MIMGG16MappingTable : GenericTable {186 let FilterClass = "MIMGG16Mapping";187 let CppTypeName = "MIMGG16MappingInfo";188 let Fields = ["G", "G16"];189 string TypeOf_G = "MIMGBaseOpcode";190 string TypeOf_G16 = "MIMGBaseOpcode";191 192 let PrimaryKey = ["G"];193 let PrimaryKeyName = "getMIMGG16MappingInfo";194}195 196class MIMG_Base <dag outs, string dns = "">197 : InstSI <outs, (ins), "", []> {198 199 let VM_CNT = 1;200 let EXP_CNT = 1;201 let MIMG = 1;202 let Uses = [EXEC];203 let mayLoad = 1;204 let mayStore = 0;205 let SchedRW = [WriteVMEM];206 let UseNamedOperandTable = 1;207 let hasSideEffects = 0; // XXX ????208 209 let DecoderNamespace = dns;210 let isAsmParserOnly = !eq(dns, "");211}212 213class MIMG <dag outs, string dns = "">214 : MIMG_Base <outs, dns> {215 216 let hasPostISelHook = 1;217 let usesCustomInserter = 1;218 219 Instruction Opcode = !cast<Instruction>(NAME);220 MIMGBaseOpcode BaseOpcode;221 MIMGEncoding MIMGEncoding;222 bits<8> VDataDwords;223 bits<8> VAddrDwords;224 225 // If NSA is used this counts number of operands VAddrDwords is split into.226 bits<8> VAddrOperands;227}228 229class VIMAGE <dag outs, string dns = ""> : MIMG<outs, dns> {230 let MIMG = 0;231 let VIMAGE = 1;232}233 234class VSAMPLE <dag outs, string dns = ""> : MIMG<outs, dns> {235 let MIMG = 0;236 let VSAMPLE = 1;237}238 239def MIMGInfoTable : GenericTable {240 let FilterClass = "MIMG";241 let CppTypeName = "MIMGInfo";242 let Fields = ["Opcode", "BaseOpcode", "MIMGEncoding", "VDataDwords",243 "VAddrDwords", "VAddrOperands"];244 string TypeOf_BaseOpcode = "MIMGBaseOpcode";245 string TypeOf_MIMGEncoding = "MIMGEncoding";246 247 let PrimaryKey = ["BaseOpcode", "MIMGEncoding", "VDataDwords", "VAddrDwords"];248 let PrimaryKeyName = "getMIMGOpcodeHelper";249}250 251def getMIMGInfo : SearchIndex {252 let Table = MIMGInfoTable;253 let Key = ["Opcode"];254}255 256class NSAHelper {257 dag AddrIns;258 string AddrAsm;259 int NSA;260}261 262class MIMGNSAHelper<int num_addrs,263 list<RegisterOperand> addr_types_in=[]>264 : NSAHelper<> {265 list<RegisterOperand> addr_types =266 !if(!empty(addr_types_in), !listsplat(VGPROp_32, num_addrs),267 addr_types_in);268 269 list<string> AddrAsmNames = !foreach(i, !range(num_addrs), "vaddr" # i);270 let AddrIns = !dag(ins, addr_types, AddrAsmNames);271 let AddrAsm = "[$" # !interleave(AddrAsmNames, ", $") # "]";272 273 let NSA = !if(!le(num_addrs, 1), ?,274 !if(!le(num_addrs, 5), 1,275 !if(!le(num_addrs, 9), 2,276 !if(!le(num_addrs, 13), 3, ?))));277}278 279class PartialNSAHelper<int num_addrs, int max_addr, RegisterOperand LastAddrRC>280 : NSAHelper<> {281 282 list<RegisterOperand> addr_types =283 !if(!ge(num_addrs, max_addr),284 !listconcat(!listsplat(VGPROp_32, !sub(max_addr, 1)), [LastAddrRC]),285 !listsplat(VGPROp_32, num_addrs));286 287 int VAddrCount = !if(!gt(num_addrs, max_addr), max_addr, num_addrs);288 list<string> AddrAsmNames = !foreach(i, !range(VAddrCount), "vaddr" # i);289 290 let AddrIns = !dag(ins, addr_types, AddrAsmNames);291 let AddrAsm = "[$" # !interleave(AddrAsmNames, ", $") # "]";292 let NSA = 1;293}294 295// Base class of all pre-gfx10 MIMG instructions.296class MIMG_gfx6789<bits<8> op, dag outs, string dns = "">297 : MIMG<outs, dns>, MIMGe_gfx6789<op> {298 let SubtargetPredicate = isGFX6GFX7GFX8GFX9NotGFX90A;299 let AssemblerPredicate = isGFX6GFX7GFX8GFX9NotGFX90A;300 301 let MIMGEncoding = MIMGEncGfx6;302 let VAddrOperands = 1;303 304 let d16 = !if(BaseOpcode.HasD16, ?, 0);305}306 307class MIMG_gfx90a<bits<8> op, dag outs, string dns = "">308 : MIMG<outs, dns>, MIMGe_gfx90a<op> {309 let SubtargetPredicate = isGFX90APlus;310 let AssemblerPredicate = isGFX90APlus;311 312 let MIMGEncoding = MIMGEncGfx90a;313 let VAddrOperands = 1;314 315 let d16 = !if(BaseOpcode.HasD16, ?, 0);316}317 318// Base class of all non-NSA gfx10 MIMG instructions.319class MIMG_gfx10<int op, dag outs, string dns = "">320 : MIMG<outs, dns>, MIMGe_gfx10<op> {321 let SubtargetPredicate = isGFX10Only;322 let AssemblerPredicate = isGFX10Only;323 324 let MIMGEncoding = MIMGEncGfx10Default;325 let VAddrOperands = 1;326 327 let d16 = !if(BaseOpcode.HasD16, ?, 0);328 let nsa = 0;329}330 331// Base class for all NSA MIMG instructions.332// Note that 1-dword addresses always use non-NSA variants.333class MIMG_nsa_gfx10<int op, dag outs, int num_addrs, string dns="">334 : MIMG<outs, dns>, MIMGe_gfx10<op> {335 let SubtargetPredicate = isGFX10Only;336 let AssemblerPredicate = isGFX10Only;337 338 let MIMGEncoding = MIMGEncGfx10NSA;339 let VAddrOperands = num_addrs;340 341 MIMGNSAHelper nsah = MIMGNSAHelper<num_addrs>;342 dag AddrIns = nsah.AddrIns;343 string AddrAsm = nsah.AddrAsm;344 345 let d16 = !if(BaseOpcode.HasD16, ?, 0);346 let nsa = nsah.NSA;347}348 349// Base class of all non-NSA gfx11 MIMG instructions.350class MIMG_gfx11<int op, dag outs, string dns = "">351 : MIMG<outs, dns>, MIMGe_gfx11<op> {352 let SubtargetPredicate = isGFX11Only;353 let AssemblerPredicate = isGFX11Only;354 355 let MIMGEncoding = MIMGEncGfx11Default;356 let VAddrOperands = 1;357 358 let d16 = !if(BaseOpcode.HasD16, ?, 0);359 let nsa = 0;360}361 362// Base class for all NSA MIMG instructions.363// Note that 1-dword addresses always use non-NSA variants.364class MIMG_nsa_gfx11<int op, dag outs, int num_addrs, string dns="",365 list<RegisterOperand> addr_types=[],366 RegisterOperand LastAddrRC = VGPROp_32>367 : MIMG<outs, dns>, MIMGe_gfx11<op> {368 let SubtargetPredicate = isGFX11Only;369 let AssemblerPredicate = isGFX11Only;370 371 let MIMGEncoding = MIMGEncGfx11NSA;372 let VAddrOperands = num_addrs;373 374 NSAHelper nsah = !if(!empty(addr_types),375 PartialNSAHelper<num_addrs, 5, LastAddrRC>,376 MIMGNSAHelper<num_addrs, addr_types>);377 dag AddrIns = nsah.AddrIns;378 string AddrAsm = nsah.AddrAsm;379 380 let d16 = !if(BaseOpcode.HasD16, ?, 0);381 let nsa = nsah.NSA;382}383 384class VIMAGE_gfx12<int op, dag outs, int num_addrs, string dns="",385 list<RegisterOperand> addr_types=[]>386 : VIMAGE<outs, dns>, VIMAGEe<op> {387 let SubtargetPredicate = isGFX12Plus;388 let AssemblerPredicate = isGFX12Plus;389 390 let MIMGEncoding = MIMGEncGfx12;391 let VAddrOperands = num_addrs;392 393 MIMGNSAHelper nsah = !if(!empty(addr_types),394 MIMGNSAHelper<num_addrs>,395 MIMGNSAHelper<num_addrs, addr_types>);396 dag AddrIns = nsah.AddrIns;397 string AddrAsm = !if(!eq(num_addrs, 1), "$vaddr0", nsah.AddrAsm);398 399 let d16 = !if(BaseOpcode.HasD16, ?, 0);400 let vaddr1 = !if(!lt(num_addrs, 2), 0, ?);401 let vaddr2 = !if(!lt(num_addrs, 3), 0, ?);402 let vaddr3 = !if(!lt(num_addrs, 4), 0, ?);403 let vaddr4 = !if(!lt(num_addrs, 5), 0, ?);404}405 406class VSAMPLE_gfx12<int op, dag outs, int num_addrs, string dns="",407 RegisterOperand Addr3RC>408 : VSAMPLE<outs, dns>, VSAMPLEe<op> {409 let SubtargetPredicate = isGFX12Plus;410 let AssemblerPredicate = isGFX12Plus;411 412 let MIMGEncoding = MIMGEncGfx12;413 let VAddrOperands = num_addrs;414 415 PartialNSAHelper nsah = PartialNSAHelper<num_addrs, 4, Addr3RC>;416 417 dag AddrIns = nsah.AddrIns;418 string AddrAsm = !if(!eq(num_addrs, 1), "$vaddr0", nsah.AddrAsm);419 420 let d16 = !if(BaseOpcode.HasD16, ?, 0);421 let vaddr1 = !if(!lt(num_addrs, 2), 0, ?);422 let vaddr2 = !if(!lt(num_addrs, 3), 0, ?);423 let vaddr3 = !if(!lt(num_addrs, 4), 0, ?);424}425 426class MIMG_NoSampler_Helper <mimgopc op, string asm,427 RegisterOperand dst_rc,428 RegisterOperand addr_rc,429 string dns="">430 : MIMG_gfx6789 <op.GFX10M, (outs dst_rc:$vdata), dns> {431 let InOperandList = !con((ins addr_rc:$vaddr, SReg_256_XNULL:$srsrc,432 DMask:$dmask, UNorm:$unorm, CPol:$cpol,433 R128A16:$r128, TFE:$tfe, LWE:$lwe, DA:$da),434 !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));435 let AsmString = asm#" $vdata, $vaddr, $srsrc$dmask$unorm$cpol$r128$tfe$lwe$da"436 #!if(BaseOpcode.HasD16, "$d16", "");437}438 439class MIMG_NoSampler_Helper_gfx90a <mimgopc op, string asm,440 RegisterOperand dst_rc,441 RegisterOperand addr_rc,442 string dns="">443 : MIMG_gfx90a <op.GFX10M, (outs getAlign2RegOp<dst_rc>.ret:$vdata), dns> {444 let InOperandList = !con((ins getAlign2RegOp<addr_rc>.ret:$vaddr, SReg_256_XNULL:$srsrc,445 DMask:$dmask, UNorm:$unorm, CPol:$cpol,446 R128A16:$r128, LWE:$lwe, DA:$da),447 !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));448 let AsmString = asm#" $vdata, $vaddr, $srsrc$dmask$unorm$cpol$r128$lwe$da"449 #!if(BaseOpcode.HasD16, "$d16", "");450}451 452class MIMG_NoSampler_gfx10<mimgopc op, string opcode,453 RegisterOperand DataRC, RegisterOperand AddrRC,454 string dns="">455 : MIMG_gfx10<op.GFX10M, (outs DataRC:$vdata), dns> {456 let InOperandList = !con((ins AddrRC:$vaddr0, SReg_256_XNULL:$srsrc, DMask:$dmask,457 Dim:$dim, UNorm:$unorm, CPol:$cpol,458 R128A16:$r128, A16:$a16, TFE:$tfe, LWE:$lwe),459 !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));460 let AsmString = opcode#" $vdata, $vaddr0, $srsrc$dmask$dim$unorm$cpol$r128$a16$tfe$lwe"461 #!if(BaseOpcode.HasD16, "$d16", "");462}463 464class MIMG_NoSampler_nsa_gfx10<mimgopc op, string opcode,465 RegisterOperand DataRC, int num_addrs,466 string dns="">467 : MIMG_nsa_gfx10<op.GFX10M, (outs DataRC:$vdata), num_addrs, dns> {468 let InOperandList = !con(AddrIns,469 (ins SReg_256_XNULL:$srsrc, DMask:$dmask,470 Dim:$dim, UNorm:$unorm, CPol:$cpol,471 R128A16:$r128, A16:$a16, TFE:$tfe, LWE:$lwe),472 !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));473 let AsmString = opcode#" $vdata, "#AddrAsm#", $srsrc$dmask$dim$unorm$cpol$r128$a16$tfe$lwe"474 #!if(BaseOpcode.HasD16, "$d16", "");475}476 477class MIMG_NoSampler_gfx11<mimgopc op, string opcode,478 RegisterOperand DataRC, RegisterOperand AddrRC,479 string dns="">480 : MIMG_gfx11<op.GFX11, (outs DataRC:$vdata), dns> {481 let InOperandList = !con((ins AddrRC:$vaddr0, SReg_256_XNULL:$srsrc, DMask:$dmask,482 Dim:$dim, UNorm:$unorm, CPol:$cpol,483 R128A16:$r128, A16:$a16, TFE:$tfe, LWE:$lwe),484 !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));485 let AsmString = opcode#" $vdata, $vaddr0, $srsrc$dmask$dim$unorm$cpol$r128$a16$tfe$lwe"486 #!if(BaseOpcode.HasD16, "$d16", "");487}488 489class MIMG_NoSampler_nsa_gfx11<mimgopc op, string opcode,490 RegisterOperand DataRC, int num_addrs,491 string dns="">492 : MIMG_nsa_gfx11<op.GFX11, (outs DataRC:$vdata), num_addrs, dns> {493 let InOperandList = !con(AddrIns,494 (ins SReg_256_XNULL:$srsrc, DMask:$dmask,495 Dim:$dim, UNorm:$unorm, CPol:$cpol,496 R128A16:$r128, A16:$a16, TFE:$tfe, LWE:$lwe),497 !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));498 let AsmString = opcode#" $vdata, "#AddrAsm#", $srsrc$dmask$dim$unorm$cpol$r128$a16$tfe$lwe"499 #!if(BaseOpcode.HasD16, "$d16", "");500}501 502class VIMAGE_NoSampler_gfx12<mimgopc op, string opcode,503 RegisterOperand DataRC, int num_addrs,504 string dns="">505 : VIMAGE_gfx12<op.GFX12, (outs DataRC:$vdata), num_addrs, dns> {506 let InOperandList = !con(AddrIns,507 (ins SReg_256_XNULL:$rsrc, DMask:$dmask, Dim:$dim,508 CPol:$cpol, R128A16:$r128, A16:$a16, TFE:$tfe),509 !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));510 let AsmString = opcode#" $vdata, "#AddrAsm#", $rsrc$dmask$dim$cpol$r128$a16$tfe"511 #!if(BaseOpcode.HasD16, "$d16", "");512}513 514class VSAMPLE_Sampler_gfx12<mimgopc op, string opcode, RegisterOperand DataRC,515 int num_addrs, RegisterOperand Addr3RC = VGPROp_32,516 string dns="">517 : VSAMPLE_gfx12<op.GFX12, (outs DataRC:$vdata), num_addrs, dns, Addr3RC> {518 let InOperandList = !con(AddrIns,519 (ins SReg_256_XNULL:$rsrc),520 !if(BaseOpcode.Sampler, (ins SReg_128_XNULL:$samp), (ins)),521 (ins DMask:$dmask, Dim:$dim, UNorm:$unorm,522 CPol:$cpol, R128A16:$r128, A16:$a16, TFE:$tfe,523 LWE:$lwe),524 !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));525 let AsmString = opcode#" $vdata, "#AddrAsm#", $rsrc"526 #!if(BaseOpcode.Sampler, ", $samp", "")527 #"$dmask$dim$unorm$cpol$r128$a16$tfe$lwe"528 #!if(BaseOpcode.HasD16, "$d16", "");529}530 531class VSAMPLE_Sampler_nortn_gfx12<mimgopc op, string opcode,532 int num_addrs, RegisterOperand Addr3RC = VGPROp_32,533 string dns="">534 : VSAMPLE_gfx12<op.GFX12, (outs), num_addrs, dns, Addr3RC> {535 let InOperandList = !con(AddrIns,536 (ins SReg_256_XNULL:$rsrc),537 !if(BaseOpcode.Sampler, (ins SReg_128_XNULL:$samp), (ins)),538 (ins DMask:$dmask, Dim:$dim, UNorm:$unorm,539 CPol:$cpol, R128A16:$r128, A16:$a16, TFE:$tfe,540 LWE:$lwe),541 !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));542 let AsmString = opcode#" off, "#AddrAsm#", $rsrc"543 #!if(BaseOpcode.Sampler, ", $samp", "")544 #"$dmask$dim$unorm$cpol$r128$a16$tfe$lwe"545 #!if(BaseOpcode.HasD16, "$d16", "");546 // Force vdata to VGPR0 as no result will be returned.547 let vdata = 0;548}549 550multiclass MIMG_NoSampler_Src_Helper <mimgopc op, string asm,551 RegisterOperand dst_rc, bit enableDisasm,552 bit ExtendedImageInst = 1,553 bit isVSample = 0> {554 let VAddrDwords = 1 in {555 let ssamp = 0 in {556 if op.HAS_GFX10M then {557 def _V1 : MIMG_NoSampler_Helper <op, asm, dst_rc, VGPROp_32,558 !if(enableDisasm, "GFX8", "")>;559 if !not(ExtendedImageInst) then560 def _V1_gfx90a : MIMG_NoSampler_Helper_gfx90a <op, asm, dst_rc, VGPROp_32,561 !if(enableDisasm, "GFX90A", "")>;562 def _V1_gfx10 : MIMG_NoSampler_gfx10<op, asm, dst_rc, VGPROp_32,563 !if(enableDisasm, "GFX10", "")>;564 }565 if op.HAS_GFX11 then {566 def _V1_gfx11 : MIMG_NoSampler_gfx11<op, asm, dst_rc, VGPROp_32,567 !if(enableDisasm, "GFX11", "")>;568 }569 }570 if op.HAS_GFX12 then {571 if isVSample then {572 let samp = 0 in573 def _V1_gfx12 : VSAMPLE_Sampler_gfx12<op, asm, dst_rc, 1>;574 }575 else {576 def _V1_gfx12 : VIMAGE_NoSampler_gfx12<op, asm, dst_rc, 1>;577 }578 }579 }580 let VAddrDwords = 2 in {581 let ssamp = 0 in {582 if op.HAS_GFX10M then {583 def _V2 : MIMG_NoSampler_Helper <op, asm, dst_rc, VGPROp_64>;584 if !not(ExtendedImageInst) then585 def _V2_gfx90a : MIMG_NoSampler_Helper_gfx90a <op, asm, dst_rc, VGPROp_64_Align2>;586 def _V2_gfx10 : MIMG_NoSampler_gfx10<op, asm, dst_rc, VGPROp_64>;587 def _V2_nsa_gfx10 : MIMG_NoSampler_nsa_gfx10<op, asm, dst_rc, 2>;588 }589 if op.HAS_GFX11 then {590 def _V2_gfx11 : MIMG_NoSampler_gfx11<op, asm, dst_rc, VGPROp_64>;591 def _V2_nsa_gfx11 : MIMG_NoSampler_nsa_gfx11<op, asm, dst_rc, 2>;592 }593 }594 if op.HAS_GFX12 then {595 if isVSample then {596 let samp = 0 in597 def _V2_gfx12 : VSAMPLE_Sampler_gfx12<op, asm, dst_rc, 2>;598 }599 else {600 def _V2_gfx12 : VIMAGE_NoSampler_gfx12<op, asm, dst_rc, 2>;601 }602 }603 }604 let VAddrDwords = 3 in {605 let ssamp = 0 in {606 if op.HAS_GFX10M then {607 def _V3 : MIMG_NoSampler_Helper <op, asm, dst_rc, VGPROp_96>;608 if !not(ExtendedImageInst) then609 def _V3_gfx90a : MIMG_NoSampler_Helper_gfx90a <op, asm, dst_rc, VGPROp_96_Align2>;610 def _V3_gfx10 : MIMG_NoSampler_gfx10<op, asm, dst_rc, VGPROp_96>;611 def _V3_nsa_gfx10 : MIMG_NoSampler_nsa_gfx10<op, asm, dst_rc, 3>;612 }613 if op.HAS_GFX11 then {614 def _V3_gfx11 : MIMG_NoSampler_gfx11<op, asm, dst_rc, VGPROp_96>;615 def _V3_nsa_gfx11 : MIMG_NoSampler_nsa_gfx11<op, asm, dst_rc, 3>;616 }617 }618 if op.HAS_GFX12 then {619 if isVSample then {620 let samp = 0 in621 def _V3_gfx12 : VSAMPLE_Sampler_gfx12<op, asm, dst_rc, 3>;622 }623 else {624 def _V3_gfx12 : VIMAGE_NoSampler_gfx12<op, asm, dst_rc, 3>;625 }626 }627 }628 let VAddrDwords = 4 in {629 let ssamp = 0 in {630 if op.HAS_GFX10M then {631 def _V4 : MIMG_NoSampler_Helper <op, asm, dst_rc, VGPROp_128>;632 if !not(ExtendedImageInst) then633 def _V4_gfx90a : MIMG_NoSampler_Helper_gfx90a <op, asm, dst_rc, VGPROp_128_Align2>;634 def _V4_gfx10 : MIMG_NoSampler_gfx10<op, asm, dst_rc, VGPROp_128>;635 def _V4_nsa_gfx10 : MIMG_NoSampler_nsa_gfx10<op, asm, dst_rc, 4,636 !if(enableDisasm, "GFX10", "")>;637 }638 if op.HAS_GFX11 then {639 def _V4_gfx11 : MIMG_NoSampler_gfx11<op, asm, dst_rc, VGPROp_128>;640 def _V4_nsa_gfx11 : MIMG_NoSampler_nsa_gfx11<op, asm, dst_rc, 4,641 !if(enableDisasm, "GFX11", "")>;642 }643 }644 if op.HAS_GFX12 then {645 if isVSample then {646 let samp = 0 in647 def _V4_gfx12 : VSAMPLE_Sampler_gfx12<op, asm, dst_rc, 4, VGPROp_32,648 !if(enableDisasm, "GFX12", "")>;649 }650 else {651 def _V4_gfx12 : VIMAGE_NoSampler_gfx12<op, asm, dst_rc, 4,652 !if(enableDisasm, "GFX12", "")>;653 }654 }655 }656}657 658multiclass MIMG_NoSampler <mimgopc op, string asm, bit has_d16, bit mip = 0,659 bit isResInfo = 0,660 bit msaa = 0> {661 def "" : MIMGBaseOpcode {662 let Coordinates = !not(isResInfo);663 let LodOrClampOrMip = mip;664 let HasD16 = has_d16;665 let MSAA = msaa;666 }667 668 let BaseOpcode = !cast<MIMGBaseOpcode>(NAME),669 mayLoad = !not(isResInfo) in {670 let VDataDwords = 1 in671 defm _V1 : MIMG_NoSampler_Src_Helper <op, asm, AVLdSt_32, 1, msaa>;672 let VDataDwords = 2 in673 defm _V2 : MIMG_NoSampler_Src_Helper <op, asm, AVLdSt_64, 0, msaa>;674 let VDataDwords = 3 in675 defm _V3 : MIMG_NoSampler_Src_Helper <op, asm, AVLdSt_96, 0, msaa>;676 let VDataDwords = 4 in677 defm _V4 : MIMG_NoSampler_Src_Helper <op, asm, AVLdSt_128, 0, msaa>;678 let VDataDwords = 5 in679 defm _V5 : MIMG_NoSampler_Src_Helper <op, asm, AVLdSt_160, 0, msaa>;680 }681}682 683class MIMG_Store_Helper <mimgopc op, string asm,684 RegisterOperand data_rc,685 RegisterClass addr_rc,686 string dns = "">687 : MIMG_gfx6789<op.GFX10M, (outs), dns> {688 let InOperandList = !con((ins data_rc:$vdata, addr_rc:$vaddr, SReg_256_XNULL:$srsrc,689 DMask:$dmask, UNorm:$unorm, CPol:$cpol,690 R128A16:$r128, TFE:$tfe, LWE:$lwe, DA:$da),691 !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));692 let AsmString = asm#" $vdata, $vaddr, $srsrc$dmask$unorm$cpol$r128$tfe$lwe$da"693 #!if(BaseOpcode.HasD16, "$d16", "");694}695 696class MIMG_Store_Helper_gfx90a <mimgopc op, string asm,697 RegisterOperand data_rc,698 RegisterClass addr_rc,699 string dns = "">700 : MIMG_gfx90a<op.GFX10M, (outs), dns> {701 let InOperandList = !con((ins getAlign2RegOp<data_rc>.ret:$vdata,702 addr_rc:$vaddr, SReg_256_XNULL:$srsrc,703 DMask:$dmask, UNorm:$unorm, CPol:$cpol,704 R128A16:$r128, LWE:$lwe, DA:$da),705 !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));706 let AsmString = asm#" $vdata, $vaddr, $srsrc$dmask$unorm$cpol$r128$lwe$da"707 #!if(BaseOpcode.HasD16, "$d16", "");708}709 710class MIMG_Store_gfx10<mimgopc op, string opcode,711 RegisterOperand DataRC, RegisterClass AddrRC,712 string dns="">713 : MIMG_gfx10<op.GFX10M, (outs), dns> {714 let InOperandList = !con((ins DataRC:$vdata, AddrRC:$vaddr0, SReg_256_XNULL:$srsrc,715 DMask:$dmask, Dim:$dim, UNorm:$unorm, CPol:$cpol,716 R128A16:$r128, A16:$a16, TFE:$tfe, LWE:$lwe),717 !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));718 let AsmString = opcode#" $vdata, $vaddr0, $srsrc$dmask$dim$unorm$cpol$r128$a16$tfe$lwe"719 #!if(BaseOpcode.HasD16, "$d16", "");720}721 722class MIMG_Store_nsa_gfx10<mimgopc op, string opcode,723 RegisterOperand DataRC, int num_addrs,724 string dns="">725 : MIMG_nsa_gfx10<op.GFX10M, (outs), num_addrs, dns> {726 let InOperandList = !con((ins DataRC:$vdata),727 AddrIns,728 (ins SReg_256_XNULL:$srsrc, DMask:$dmask,729 Dim:$dim, UNorm:$unorm, CPol:$cpol,730 R128A16:$r128, A16:$a16, TFE:$tfe, LWE:$lwe),731 !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));732 let AsmString = opcode#" $vdata, "#AddrAsm#", $srsrc$dmask$dim$unorm$cpol$r128$a16$tfe$lwe"733 #!if(BaseOpcode.HasD16, "$d16", "");734}735 736class MIMG_Store_gfx11<mimgopc op, string opcode,737 RegisterOperand DataRC, RegisterClass AddrRC,738 string dns="">739 : MIMG_gfx11<op.GFX11, (outs), dns> {740 let InOperandList = !con((ins DataRC:$vdata, AddrRC:$vaddr0, SReg_256_XNULL:$srsrc,741 DMask:$dmask, Dim:$dim, UNorm:$unorm, CPol:$cpol,742 R128A16:$r128, A16:$a16, TFE:$tfe, LWE:$lwe),743 !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));744 let AsmString = opcode#" $vdata, $vaddr0, $srsrc$dmask$dim$unorm$cpol$r128$a16$tfe$lwe"745 #!if(BaseOpcode.HasD16, "$d16", "");746}747 748class MIMG_Store_nsa_gfx11<mimgopc op, string opcode,749 RegisterOperand DataRC, int num_addrs,750 string dns="">751 : MIMG_nsa_gfx11<op.GFX11, (outs), num_addrs, dns> {752 let InOperandList = !con((ins DataRC:$vdata),753 AddrIns,754 (ins SReg_256_XNULL:$srsrc, DMask:$dmask,755 Dim:$dim, UNorm:$unorm, CPol:$cpol,756 R128A16:$r128, A16:$a16, TFE:$tfe, LWE:$lwe),757 !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));758 let AsmString = opcode#" $vdata, "#AddrAsm#", $srsrc$dmask$dim$unorm$cpol$r128$a16$tfe$lwe"759 #!if(BaseOpcode.HasD16, "$d16", "");760}761 762class VIMAGE_Store_gfx12<mimgopc op, string opcode,763 RegisterOperand DataRC, int num_addrs,764 string dns="">765 : VIMAGE_gfx12<op.GFX12, (outs), num_addrs, dns> {766 let InOperandList = !con((ins DataRC:$vdata),767 AddrIns,768 (ins SReg_256_XNULL:$rsrc, DMask:$dmask, Dim:$dim,769 CPol:$cpol, R128A16:$r128, A16:$a16, TFE:$tfe),770 !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));771 let AsmString = opcode#" $vdata, "#AddrAsm#", $rsrc$dmask$dim$cpol$r128$a16$tfe"772 #!if(BaseOpcode.HasD16, "$d16", "");773}774 775multiclass MIMG_Store_Addr_Helper <mimgopc op, string asm,776 RegisterOperand data_rc,777 bit enableDisasm> {778 let mayLoad = 0, mayStore = 1, hasSideEffects = 0, hasPostISelHook = 0,779 DisableWQM = 1 in {780 let VAddrDwords = 1 in {781 let ssamp = 0 in {782 if op.HAS_GFX10M then {783 def _V1 : MIMG_Store_Helper <op, asm, data_rc, VGPR_32,784 !if(enableDisasm, "GFX8", "")>;785 let hasPostISelHook = 1 in786 def _V1_gfx90a : MIMG_Store_Helper_gfx90a <op, asm, data_rc, VGPR_32,787 !if(enableDisasm, "GFX90A", "")>;788 def _V1_gfx10 : MIMG_Store_gfx10 <op, asm, data_rc, VGPR_32,789 !if(enableDisasm, "GFX10", "")>;790 }791 if op.HAS_GFX11 then {792 def _V1_gfx11 : MIMG_Store_gfx11 <op, asm, data_rc, VGPR_32,793 !if(enableDisasm, "GFX11", "")>;794 }795 }796 if op.HAS_GFX12 then {797 def _V1_gfx12 : VIMAGE_Store_gfx12 <op, asm, data_rc, 1>;798 }799 }800 let VAddrDwords = 2 in {801 let ssamp = 0 in {802 if op.HAS_GFX10M then {803 def _V2 : MIMG_Store_Helper <op, asm, data_rc, VReg_64>;804 def _V2_gfx90a : MIMG_Store_Helper_gfx90a <op, asm, data_rc, VReg_64_Align2>;805 def _V2_gfx10 : MIMG_Store_gfx10 <op, asm, data_rc, VReg_64>;806 def _V2_nsa_gfx10 : MIMG_Store_nsa_gfx10 <op, asm, data_rc, 2>;807 }808 if op.HAS_GFX11 then {809 def _V2_gfx11 : MIMG_Store_gfx11 <op, asm, data_rc, VReg_64>;810 def _V2_nsa_gfx11 : MIMG_Store_nsa_gfx11 <op, asm, data_rc, 2>;811 }812 }813 if op.HAS_GFX12 then {814 def _V2_gfx12 : VIMAGE_Store_gfx12 <op, asm, data_rc, 2>;815 }816 }817 let VAddrDwords = 3 in {818 let ssamp = 0 in {819 if op.HAS_GFX10M then {820 def _V3 : MIMG_Store_Helper <op, asm, data_rc, VReg_96>;821 def _V3_gfx90a : MIMG_Store_Helper_gfx90a <op, asm, data_rc, VReg_96_Align2>;822 def _V3_gfx10 : MIMG_Store_gfx10 <op, asm, data_rc, VReg_96>;823 def _V3_nsa_gfx10 : MIMG_Store_nsa_gfx10 <op, asm, data_rc, 3>;824 }825 if op.HAS_GFX11 then {826 def _V3_gfx11 : MIMG_Store_gfx11 <op, asm, data_rc, VReg_96>;827 def _V3_nsa_gfx11 : MIMG_Store_nsa_gfx11 <op, asm, data_rc, 3>;828 }829 }830 if op.HAS_GFX12 then {831 def _V3_gfx12 : VIMAGE_Store_gfx12 <op, asm, data_rc, 3>;832 }833 }834 let VAddrDwords = 4 in {835 let ssamp = 0 in {836 if op.HAS_GFX10M then {837 def _V4 : MIMG_Store_Helper <op, asm, data_rc, VReg_128>;838 def _V4_gfx90a : MIMG_Store_Helper_gfx90a <op, asm, data_rc, VReg_128_Align2>;839 def _V4_gfx10 : MIMG_Store_gfx10 <op, asm, data_rc, VReg_128>;840 def _V4_nsa_gfx10 : MIMG_Store_nsa_gfx10 <op, asm, data_rc, 4,841 !if(enableDisasm, "GFX10", "")>;842 }843 if op.HAS_GFX11 then {844 def _V4_gfx11 : MIMG_Store_gfx11 <op, asm, data_rc, VReg_128>;845 def _V4_nsa_gfx11 : MIMG_Store_nsa_gfx11 <op, asm, data_rc, 4,846 !if(enableDisasm, "GFX11", "")>;847 }848 }849 if op.HAS_GFX12 then {850 def _V4_gfx12 : VIMAGE_Store_gfx12 <op, asm, data_rc, 4,851 !if(enableDisasm, "GFX12", "")>;852 }853 }854 }855}856 857multiclass MIMG_Store <mimgopc op, string asm, bit has_d16, bit mip = 0> {858 def "" : MIMGBaseOpcode {859 let Store = 1;860 let LodOrClampOrMip = mip;861 let HasD16 = has_d16;862 let NoReturn = 1;863 }864 865 let BaseOpcode = !cast<MIMGBaseOpcode>(NAME) in {866 let VDataDwords = 1 in867 defm _V1 : MIMG_Store_Addr_Helper <op, asm, AVLdSt_32, 1>;868 let VDataDwords = 2 in869 defm _V2 : MIMG_Store_Addr_Helper <op, asm, AVLdSt_64, 0>;870 let VDataDwords = 3 in871 defm _V3 : MIMG_Store_Addr_Helper <op, asm, AVLdSt_96, 0>;872 let VDataDwords = 4 in873 defm _V4 : MIMG_Store_Addr_Helper <op, asm, AVLdSt_128, 0>;874 let VDataDwords = 5 in875 defm _V5 : MIMG_Store_Addr_Helper <op, asm, AVLdSt_160, 0>;876 }877}878 879class MIMG_Atomic_gfx6789_base <bits<8> op, string asm, RegisterOperand data_rc,880 RegisterClass addr_rc, bit noRtn, string dns="">881 : MIMG_gfx6789 <op, !if(noRtn, (outs), (outs data_rc:$vdst)), dns> {882 let Constraints = !if(noRtn, "", "$vdst = $vdata");883 let isCodeGenOnly = noRtn;884 let InOperandList = (ins data_rc:$vdata, addr_rc:$vaddr, SReg_256_XNULL:$srsrc,885 DMask:$dmask, UNorm:$unorm, CPol:$cpol,886 R128A16:$r128, TFE:$tfe, LWE:$lwe, DA:$da);887 let AsmString = asm#" $vdata, $vaddr, $srsrc$dmask$unorm$cpol$r128$tfe$lwe$da";888}889 890class MIMG_Atomic_gfx90a_base <bits<8> op, string asm, RegisterOperand data_rc,891 RegisterClass addr_rc, bit noRtn, string dns="">892 : MIMG_gfx90a <op, !if(noRtn, (outs), (outs getAlign2RegOp<data_rc>.ret:$vdst)), dns> {893 let Constraints = !if(noRtn, "", "$vdst = $vdata");894 let isCodeGenOnly = noRtn;895 let InOperandList = (ins getAlign2RegOp<data_rc>.ret:$vdata,896 addr_rc:$vaddr, SReg_256_XNULL:$srsrc,897 DMask:$dmask, UNorm:$unorm, CPol:$cpol,898 R128A16:$r128, LWE:$lwe, DA:$da);899 let AsmString = asm#" $vdata, $vaddr, $srsrc$dmask$unorm$cpol$r128$lwe$da";900}901 902class MIMG_Atomic_si<mimgopc op, string asm, RegisterOperand data_rc,903 RegisterClass addr_rc, bit noRtn = 0, bit enableDasm = 0>904 : MIMG_Atomic_gfx6789_base<op.SI, asm, data_rc, addr_rc, noRtn,905 !if(enableDasm, "GFX6GFX7", "")> {906 let AssemblerPredicate = isGFX6GFX7;907}908 909class MIMG_Atomic_vi<mimgopc op, string asm, RegisterOperand data_rc,910 RegisterClass addr_rc, bit noRtn = 0, bit enableDasm = 0>911 : MIMG_Atomic_gfx6789_base<op.VI, asm, data_rc, addr_rc, noRtn, !if(enableDasm, "GFX8", "")> {912 let AssemblerPredicate = isGFX8GFX9NotGFX90A;913 let MIMGEncoding = MIMGEncGfx8;914}915 916class MIMG_Atomic_gfx90a<mimgopc op, string asm, RegisterOperand data_rc,917 RegisterClass addr_rc, bit noRtn = 0, bit enableDasm = 0>918 : MIMG_Atomic_gfx90a_base<op.VI, asm, data_rc, addr_rc, noRtn, !if(enableDasm, "GFX90A", "")> {919 let AssemblerPredicate = isGFX90APlus;920 let MIMGEncoding = MIMGEncGfx90a;921}922 923class MIMG_Atomic_gfx10<mimgopc op, string opcode,924 RegisterOperand DataRC, RegisterClass AddrRC,925 bit noRtn = 0, bit enableDisasm = 0>926 : MIMG_gfx10<op.GFX10M, !if(noRtn, (outs), (outs DataRC:$vdst)),927 !if(enableDisasm, "GFX10", "")> {928 let Constraints = !if(noRtn, "", "$vdst = $vdata");929 let isCodeGenOnly = noRtn;930 let InOperandList = (ins DataRC:$vdata, AddrRC:$vaddr0, SReg_256_XNULL:$srsrc,931 DMask:$dmask, Dim:$dim, UNorm:$unorm, CPol:$cpol,932 R128A16:$r128, A16:$a16, TFE:$tfe, LWE:$lwe);933 let AsmString = opcode#" $vdata, $vaddr0, $srsrc$dmask$dim$unorm$cpol$r128$a16$tfe$lwe";934}935 936class MIMG_Atomic_nsa_gfx10<mimgopc op, string opcode,937 RegisterOperand DataRC, int num_addrs,938 bit noRtn = 0, bit enableDisasm = 0>939 : MIMG_nsa_gfx10<op.GFX10M, !if(noRtn, (outs), (outs DataRC:$vdst)), num_addrs,940 !if(enableDisasm, "GFX10", "")> {941 let Constraints = !if(noRtn, "", "$vdst = $vdata");942 let isCodeGenOnly = noRtn;943 let InOperandList = !con((ins DataRC:$vdata),944 AddrIns,945 (ins SReg_256_XNULL:$srsrc, DMask:$dmask,946 Dim:$dim, UNorm:$unorm, CPol:$cpol,947 R128A16:$r128, A16:$a16, TFE:$tfe, LWE:$lwe));948 let AsmString = opcode#" $vdata, "#AddrAsm#", $srsrc$dmask$dim$unorm$cpol$r128$a16$tfe$lwe";949}950 951class MIMG_Atomic_gfx11<mimgopc op, string opcode,952 RegisterOperand DataRC, RegisterClass AddrRC,953 bit noRtn = 0, bit enableDisasm = 0>954 : MIMG_gfx11<op.GFX11, !if(noRtn, (outs), (outs DataRC:$vdst)),955 !if(enableDisasm, "GFX11", "")> {956 let Constraints = !if(noRtn, "", "$vdst = $vdata");957 let isCodeGenOnly = noRtn;958 let InOperandList = (ins DataRC:$vdata, AddrRC:$vaddr0, SReg_256_XNULL:$srsrc,959 DMask:$dmask, Dim:$dim, UNorm:$unorm, CPol:$cpol,960 R128A16:$r128, A16:$a16, TFE:$tfe, LWE:$lwe);961 let AsmString = opcode#" $vdata, $vaddr0, $srsrc$dmask$dim$unorm$cpol$r128$a16$tfe$lwe";962}963 964class MIMG_Atomic_nsa_gfx11<mimgopc op, string opcode,965 RegisterOperand DataRC, int num_addrs,966 bit noRtn = 0, bit enableDisasm = 0>967 : MIMG_nsa_gfx11<op.GFX11, !if(noRtn, (outs), (outs DataRC:$vdst)), num_addrs,968 !if(enableDisasm, "GFX11", "")> {969 let Constraints = !if(noRtn, "", "$vdst = $vdata");970 let isCodeGenOnly = noRtn;971 let InOperandList = !con((ins DataRC:$vdata),972 AddrIns,973 (ins SReg_256_XNULL:$srsrc, DMask:$dmask,974 Dim:$dim, UNorm:$unorm, CPol:$cpol,975 R128A16:$r128, A16:$a16, TFE:$tfe, LWE:$lwe));976 let AsmString = opcode#" $vdata, "#AddrAsm#", $srsrc$dmask$dim$unorm$cpol$r128$a16$tfe$lwe";977}978 979class VIMAGE_Atomic_gfx12<mimgopc op, string opcode, RegisterOperand DataRC,980 int num_addrs, string renamed, bit noRtn = 0, bit enableDisasm = 0>981 : VIMAGE_gfx12<op.GFX12, !if(noRtn, (outs), (outs DataRC:$vdst)), num_addrs,982 !if(enableDisasm, "GFX12", "")> {983 let Constraints = !if(noRtn, "", "$vdst = $vdata");984 let isCodeGenOnly = noRtn;985 let InOperandList = !con((ins DataRC:$vdata),986 AddrIns,987 (ins SReg_256_XNULL:$rsrc, DMask:$dmask, Dim:$dim,988 CPol:$cpol, R128A16:$r128, A16:$a16, TFE:$tfe));989 let AsmString = !if(!empty(renamed), opcode, renamed)#" $vdata, "#AddrAsm#990 ", $rsrc$dmask$dim$cpol$r128$a16$tfe";991}992 993multiclass MIMG_Atomic_Addr_Helper_m <mimgopc op, string asm,994 RegisterOperand data_rc,995 bit enableDasm = 0,996 bit isFP = 0,997 bit noRtn = 0,998 string renamed = ""> {999 let hasSideEffects = 1, // FIXME: remove this1000 mayLoad = 1, mayStore = 1, hasPostISelHook = 0, DisableWQM = 1,1001 FPAtomic = isFP, IsAtomicNoRet = noRtn in {1002 let VAddrDwords = 1 in {1003 let ssamp = 0 in {1004 if op.HAS_SI then {1005 def _V1_si : MIMG_Atomic_si <op, asm, data_rc, VGPR_32, noRtn, enableDasm>;1006 }1007 if op.HAS_VI then {1008 def _V1_vi : MIMG_Atomic_vi <op, asm, data_rc, VGPR_32, noRtn, enableDasm>;1009 let hasPostISelHook = 1 in1010 def _V1_gfx90a : MIMG_Atomic_gfx90a <op, asm, data_rc, VGPR_32, noRtn, enableDasm>;1011 }1012 if op.HAS_GFX10M then {1013 def _V1_gfx10 : MIMG_Atomic_gfx10 <op, asm, data_rc, VGPR_32, noRtn, enableDasm>;1014 }1015 if op.HAS_GFX11 then {1016 def _V1_gfx11 : MIMG_Atomic_gfx11 <op, asm, data_rc, VGPR_32, noRtn, enableDasm>;1017 }1018 }1019 if op.HAS_GFX12 then {1020 def _V1_gfx12 : VIMAGE_Atomic_gfx12 <op, asm, data_rc, 1, renamed, noRtn>;1021 }1022 }1023 let VAddrDwords = 2 in {1024 let ssamp = 0 in {1025 if op.HAS_SI then {1026 def _V2_si : MIMG_Atomic_si <op, asm, data_rc, VReg_64, noRtn, 0>;1027 }1028 if op.HAS_VI then {1029 def _V2_vi : MIMG_Atomic_vi <op, asm, data_rc, VReg_64, noRtn, 0>;1030 def _V2_gfx90a : MIMG_Atomic_gfx90a <op, asm, data_rc, VReg_64_Align2, noRtn, 0>;1031 }1032 if op.HAS_GFX10M then {1033 def _V2_gfx10 : MIMG_Atomic_gfx10 <op, asm, data_rc, VReg_64, noRtn, 0>;1034 def _V2_nsa_gfx10 : MIMG_Atomic_nsa_gfx10 <op, asm, data_rc, 2, noRtn, 0>;1035 }1036 if op.HAS_GFX11 then {1037 def _V2_gfx11 : MIMG_Atomic_gfx11 <op, asm, data_rc, VReg_64, noRtn, 0>;1038 def _V2_nsa_gfx11 : MIMG_Atomic_nsa_gfx11 <op, asm, data_rc, 2, noRtn, 0>;1039 }1040 }1041 if op.HAS_GFX12 then {1042 def _V2_gfx12 : VIMAGE_Atomic_gfx12 <op, asm, data_rc, 2, renamed, noRtn>;1043 }1044 }1045 let VAddrDwords = 3 in {1046 let ssamp = 0 in {1047 if op.HAS_SI then {1048 def _V3_si : MIMG_Atomic_si <op, asm, data_rc, VReg_96, noRtn, 0>;1049 }1050 if op.HAS_VI then {1051 def _V3_vi : MIMG_Atomic_vi <op, asm, data_rc, VReg_96, noRtn, 0>;1052 def _V3_gfx90a : MIMG_Atomic_gfx90a <op, asm, data_rc, VReg_96_Align2, noRtn, 0>;1053 }1054 if op.HAS_GFX10M then {1055 def _V3_gfx10 : MIMG_Atomic_gfx10 <op, asm, data_rc, VReg_96, noRtn, 0>;1056 def _V3_nsa_gfx10 : MIMG_Atomic_nsa_gfx10 <op, asm, data_rc, 3, noRtn, 0>;1057 }1058 if op.HAS_GFX11 then {1059 def _V3_gfx11 : MIMG_Atomic_gfx11 <op, asm, data_rc, VReg_96, noRtn, 0>;1060 def _V3_nsa_gfx11 : MIMG_Atomic_nsa_gfx11 <op, asm, data_rc, 3, noRtn, 0>;1061 }1062 }1063 if op.HAS_GFX12 then {1064 def _V3_gfx12 : VIMAGE_Atomic_gfx12 <op, asm, data_rc, 3, renamed, noRtn>;1065 }1066 }1067 let VAddrDwords = 4 in {1068 let ssamp = 0 in {1069 if op.HAS_SI then {1070 def _V4_si : MIMG_Atomic_si <op, asm, data_rc, VReg_128, noRtn, 0>;1071 }1072 if op.HAS_VI then {1073 def _V4_vi : MIMG_Atomic_vi <op, asm, data_rc, VReg_128, noRtn, 0>;1074 def _V4_gfx90a : MIMG_Atomic_gfx90a <op, asm, data_rc, VReg_128_Align2, noRtn, 0>;1075 }1076 if op.HAS_GFX10M then {1077 def _V4_gfx10 : MIMG_Atomic_gfx10 <op, asm, data_rc, VReg_128, noRtn, 0>;1078 def _V4_nsa_gfx10 : MIMG_Atomic_nsa_gfx10 <op, asm, data_rc, 4, noRtn, enableDasm>;1079 }1080 if op.HAS_GFX11 then {1081 def _V4_gfx11 : MIMG_Atomic_gfx11 <op, asm, data_rc, VReg_128, noRtn, 0>;1082 def _V4_nsa_gfx11 : MIMG_Atomic_nsa_gfx11 <op, asm, data_rc, 4, noRtn, enableDasm>;1083 }1084 }1085 if op.HAS_GFX12 then {1086 def _V4_gfx12 : VIMAGE_Atomic_gfx12 <op, asm, data_rc, 4, renamed, noRtn, enableDasm>;1087 }1088 }1089 }1090 if !and(op.HAS_GFX12, !not(!empty(renamed))) then1091 def : AMDGPUMnemonicAlias<asm, renamed> {1092 let AssemblerPredicate = isGFX12Plus;1093 bit IsAtomicRet; // Unused1094 MIMGBaseOpcode BaseOpcode; // Unused1095 int VDataDwords; // Unused1096 }1097}1098 1099multiclass MIMG_Atomic_Base <mimgopc op, string asm, bit isCmpSwap = 0, bit isFP = 0,1100 bit noRtn = 0, string renamed = ""> { // 64-bit atomics1101 let IsAtomicRet = !not(noRtn) in {1102 def "" : MIMGBaseOpcode {1103 let Atomic = 1;1104 let AtomicX2 = isCmpSwap;1105 let NoReturn = noRtn;1106 }1107 1108 let BaseOpcode = !cast<MIMGBaseOpcode>(NAME) in {1109 // _V* variants have different dst size, but the size is encoded implicitly,1110 // using dmask and tfe. Only 32-bit variant is registered with disassembler.1111 // Other variants are reconstructed by disassembler using dmask and tfe.1112 if !not(isCmpSwap) then {1113 let VDataDwords = 1 in1114 defm _V1 : MIMG_Atomic_Addr_Helper_m <op, asm, AVLdSt_32, 1, isFP, noRtn, renamed>;1115 }1116 1117 let VDataDwords = 2 in1118 defm _V2 : MIMG_Atomic_Addr_Helper_m <op, asm, AVLdSt_64, isCmpSwap, isFP, noRtn, renamed>;1119 let VDataDwords = 3 in1120 defm _V3 : MIMG_Atomic_Addr_Helper_m <op, asm, AVLdSt_96, 0, isFP, noRtn, renamed>;1121 1122 if isCmpSwap then {1123 let VDataDwords = 4 in1124 defm _V4 : MIMG_Atomic_Addr_Helper_m <op, asm, AVLdSt_128, 0, isFP, noRtn, renamed>;1125 let VDataDwords = 5 in1126 defm _V5 : MIMG_Atomic_Addr_Helper_m <op, asm, AVLdSt_160, 0, isFP, noRtn, renamed>;1127 }1128 }1129 }1130}1131 1132multiclass MIMG_Atomic <mimgopc op, string asm, bit isCmpSwap = 0, bit isFP = 0,1133 string renamed = ""> {1134 defm "" : MIMG_Atomic_Base <op, asm, isCmpSwap, isFP, /*noRtn=*/0, renamed>;1135 defm "_NORTN" : MIMG_Atomic_Base <op, asm, isCmpSwap, isFP, /*noRtn=*/1, renamed>;1136}1137 1138multiclass MIMG_Atomic_Renamed <mimgopc op, string asm, string renamed,1139 bit isCmpSwap = 0, bit isFP = 0>1140 : MIMG_Atomic <op, asm, isCmpSwap, isFP, renamed>;1141 1142class MIMG_Sampler_Helper <mimgopc op, string asm, RegisterOperand dst_rc,1143 RegisterOperand src_rc, string dns="">1144 : MIMG_gfx6789 <op.VI, (outs dst_rc:$vdata), dns> {1145 let InOperandList = !con((ins src_rc:$vaddr, SReg_256_XNULL:$srsrc, SReg_128_XNULL:$ssamp,1146 DMask:$dmask, UNorm:$unorm, CPol:$cpol,1147 R128A16:$r128, TFE:$tfe, LWE:$lwe, DA:$da),1148 !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));1149 let AsmString = asm#" $vdata, $vaddr, $srsrc, $ssamp$dmask$unorm$cpol$r128$tfe$lwe$da"1150 #!if(BaseOpcode.HasD16, "$d16", "");1151}1152 1153class MIMG_Sampler_gfx90a<mimgopc op, string asm, RegisterOperand dst_rc,1154 RegisterOperand src_rc, string dns="">1155 : MIMG_gfx90a<op.GFX10M, (outs dst_rc:$vdata), dns> {1156 let InOperandList = !con((ins src_rc:$vaddr, SReg_256_XNULL:$srsrc, SReg_128_XNULL:$ssamp,1157 DMask:$dmask, UNorm:$unorm, CPol:$cpol,1158 R128A16:$r128, LWE:$lwe, DA:$da),1159 !if(BaseOpcode.HasD16, (ins D16:$d16), (ins)));1160 let AsmString = asm#" $vdata, $vaddr, $srsrc, $ssamp$dmask$unorm$cpol$r128$lwe$da"1161 #!if(BaseOpcode.HasD16, "$d16", "");1162}1163 1164class MIMG_Sampler_OpList_gfx10p<dag OpPrefix, bit HasD16> {1165 dag ret = !con(OpPrefix,1166 (ins SReg_256_XNULL:$srsrc, SReg_128_XNULL:$ssamp,1167 DMask:$dmask, Dim:$dim, UNorm:$unorm, CPol:$cpol,1168 R128A16:$r128, A16:$a16, TFE:$tfe, LWE:$lwe),1169 !if(HasD16, (ins D16:$d16), (ins)));1170}1171 1172class MIMG_Sampler_Asm_gfx10p<string opcode, string AsmPrefix, bit HasD16> {1173 string ret = opcode#" "#AsmPrefix#", $srsrc, $ssamp$dmask$dim$unorm"1174 #"$cpol$r128$a16$tfe$lwe"1175 #!if(HasD16, "$d16", "");1176}1177 1178class MIMG_Sampler_gfx10<mimgopc op, string opcode,1179 RegisterOperand DataRC, RegisterOperand AddrRC,1180 string dns="">1181 : MIMG_gfx10<op.GFX10M, (outs DataRC:$vdata), dns> {1182 let InOperandList = MIMG_Sampler_OpList_gfx10p<(ins AddrRC:$vaddr0), BaseOpcode.HasD16>.ret;1183 let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, "$vdata, $vaddr0", BaseOpcode.HasD16>.ret;1184}1185 1186class MIMG_Sampler_nsa_gfx10<mimgopc op, string opcode,1187 RegisterOperand DataRC, int num_addrs,1188 string dns="">1189 : MIMG_nsa_gfx10<op.GFX10M, (outs DataRC:$vdata), num_addrs, dns> {1190 let InOperandList = MIMG_Sampler_OpList_gfx10p<AddrIns, BaseOpcode.HasD16>.ret;1191 let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, " $vdata, "#AddrAsm, BaseOpcode.HasD16>.ret;1192}1193 1194class MIMG_Sampler_nortn_gfx10<mimgopc op, string opcode,1195 RegisterOperand AddrRC,1196 string dns="">1197 : MIMG_gfx10<op.GFX10M, (outs), dns> {1198 let InOperandList = MIMG_Sampler_OpList_gfx10p<(ins AddrRC:$vaddr0), BaseOpcode.HasD16>.ret;1199 let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, "off, $vaddr0", BaseOpcode.HasD16>.ret;1200 // Force vdata to VGPR0 as no result will be returned.1201 let vdata = 0;1202}1203 1204class MIMG_Sampler_nortn_nsa_gfx10<mimgopc op, string opcode,1205 int num_addrs,1206 string dns="">1207 : MIMG_nsa_gfx10<op.GFX10M, (outs), num_addrs, dns> {1208 let InOperandList = MIMG_Sampler_OpList_gfx10p<AddrIns, BaseOpcode.HasD16>.ret;1209 let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, " off, "#AddrAsm, BaseOpcode.HasD16>.ret;1210 // Force vdata to VGPR0 as no result will be returned.1211 let vdata = 0;1212}1213 1214class MIMG_Sampler_gfx11<mimgopc op, string opcode,1215 RegisterOperand DataRC, RegisterOperand AddrRC,1216 string dns="">1217 : MIMG_gfx11<op.GFX11, (outs DataRC:$vdata), dns> {1218 let InOperandList = MIMG_Sampler_OpList_gfx10p<(ins AddrRC:$vaddr0), BaseOpcode.HasD16>.ret;1219 let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, "$vdata, $vaddr0", BaseOpcode.HasD16>.ret;1220}1221 1222class MIMG_Sampler_nsa_gfx11<mimgopc op, string opcode,1223 RegisterOperand DataRC, int num_addrs,1224 RegisterOperand LastVAddrSize, string dns="">1225 : MIMG_nsa_gfx11<op.GFX11, (outs DataRC:$vdata), num_addrs, dns, [],1226 LastVAddrSize> {1227 let InOperandList = MIMG_Sampler_OpList_gfx10p<AddrIns, BaseOpcode.HasD16>.ret;1228 let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, " $vdata, "#AddrAsm, BaseOpcode.HasD16>.ret;1229}1230 1231class MIMG_Sampler_nortn_gfx11<mimgopc op, string opcode,1232 RegisterOperand AddrRC,1233 string dns="">1234 : MIMG_gfx11<op.GFX11, (outs), dns> {1235 let InOperandList = MIMG_Sampler_OpList_gfx10p<(ins AddrRC:$vaddr0), BaseOpcode.HasD16>.ret;1236 let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, "off, $vaddr0", BaseOpcode.HasD16>.ret;1237 let vdata = 0;1238}1239 1240class MIMG_Sampler_nortn_nsa_gfx11<mimgopc op, string opcode,1241 int num_addrs,1242 RegisterOperand LastVAddrSize, string dns="">1243 : MIMG_nsa_gfx11<op.GFX11, (outs), num_addrs, dns, [], LastVAddrSize> {1244 let InOperandList = MIMG_Sampler_OpList_gfx10p<AddrIns, BaseOpcode.HasD16>.ret;1245 let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, "off, "#AddrAsm, BaseOpcode.HasD16>.ret;1246 let vdata = 0;1247}1248 1249class MIMGAddrSize<int dw, bit enable_disasm, int AddrDW = dw> {1250 int NumWords = dw;1251 1252 RegisterOperand RegClass = !if(!le(AddrDW, 0), ?,1253 !if(!eq(AddrDW, 1), VGPROp_32,1254 !if(!eq(AddrDW, 2), VGPROp_64,1255 !if(!eq(AddrDW, 3), VGPROp_96,1256 !if(!eq(AddrDW, 4), VGPROp_128,1257 !if(!eq(AddrDW, 5), VGPROp_160,1258 !if(!eq(AddrDW, 6), VGPROp_192,1259 !if(!eq(AddrDW, 7), VGPROp_224,1260 !if(!eq(AddrDW, 8), VGPROp_256,1261 !if(!eq(AddrDW, 9), VGPROp_288,1262 !if(!eq(AddrDW, 10), VGPROp_320,1263 !if(!eq(AddrDW, 11), VGPROp_352,1264 !if(!eq(AddrDW, 12), VGPROp_384,1265 !if(!le(AddrDW, 16), VGPROp_512, ?))))))))))))));1266 1267 // Whether the instruction variant with this vaddr size should be enabled for1268 // the auto-generated disassembler.1269 bit Disassemble = enable_disasm;1270}1271 1272// Returns the MIMGAddrSize with the size of last VAddr for partial NSA1273class LastVAddrSize <int dw, int max_idx, bit enable_disasm>1274 : MIMGAddrSize<dw, enable_disasm,1275 !if(!gt(dw, max_idx), !sub(dw, max_idx), 0)>;1276 1277// Return whether x is in lst.1278class isIntInList<int x, list<int> lst> {1279 bit ret = !foldl(0, lst, lhs, y, !or(lhs, !eq(x, y)));1280}1281 1282// Return whether a value inside the range [min, max] (endpoints inclusive)1283// is in the given list.1284class isRangeInList<int min, int max, list<int> lst> {1285 bit ret = !foldl(0, lst, lhs, y, !or(lhs, !and(!le(min, y), !le(y, max))));1286}1287 1288class MIMGAddrSizes_dw_range<list<int> range> {1289 int Min = !head(range);1290 int Max = !if(!empty(!tail(range)), Min, !head(!tail(range)));1291}1292 1293class MIMG_Sampler_AddrSizes<AMDGPUSampleVariant sample, bit isG16,1294 int nsa_max_addr = 5, bit includeNSA1 = 0> {1295 // List of all possible numbers of address words, taking all combinations of1296 // A16 and image dimension into account (note: no MSAA, since this is for1297 // sample/gather ops).1298 list<int> AllNumAddrWords =1299 !foreach(dw, !if(sample.Gradients,1300 !if(isG16,1301 !if(!eq(sample.LodOrClamp, ""),1302 [2, 3, 4, 5, 6, 7],1303 [2, 3, 4, 5, 6, 7, 8]),1304 !if(!eq(sample.LodOrClamp, ""),1305 [2, 3, 4, 5, 6, 7, 8, 9],1306 [2, 3, 4, 5, 6, 7, 8, 9, 10])),1307 !if(!eq(sample.LodOrClamp, ""),1308 [1, 2, 3],1309 [1, 2, 3, 4])),1310 !add(dw, !size(sample.ExtraAddrArgs)));1311 1312 // Generate machine instructions based on possible register classes for the1313 // required numbers of address words. The disassembler defaults to the1314 // smallest register class.1315 list<MIMGAddrSize> MachineInstrs =1316 !foldl([]<MIMGAddrSize>,1317 !foreach(range,1318 // V4 is generated for V3 and V41319 // V8 is generated for V5 through V81320 // V16 is generated for V13 through V161321 [[1],[2],[3],[3,4],[5],[6],[7],[5,8],[9],[10],[11],[12],[13,16]],1322 MIMGAddrSizes_dw_range<range>),1323 lhs, dw,1324 !if(isRangeInList<dw.Min, dw.Max, AllNumAddrWords>.ret,1325 !listconcat(lhs, [MIMGAddrSize<dw.Max, !empty(lhs)>]),1326 lhs));1327 1328 // For NSA, generate machine instructions for all possible numbers of words1329 // except 1 (which is already covered by the non-NSA case).1330 // The disassembler defaults to the largest number of arguments among the1331 // variants with the same number of NSA words, and custom code then derives1332 // the exact variant based on the sample variant and the image dimension.1333 list<MIMGAddrSize> NSAInstrs =1334 !foldl([]<MIMGAddrSize>, [[12, 11, 10], [9, 8, 7, 6], [5, 4, 3, 2]], prev, nsa_group,1335 !listconcat(prev,1336 !foldl([]<MIMGAddrSize>, nsa_group, lhs, dw,1337 !if(isIntInList<dw, AllNumAddrWords>.ret,1338 !listconcat(lhs, [MIMGAddrSize<dw, !empty(lhs)>]),1339 lhs))));1340 1341 // In NSA format if there is a requirement for more VGPRs than the format1342 // supports, then the rest are sequential after the last one. Generate1343 // machine instructions for all possible number of words. The disassembler1344 // defaults to the largest number of arguments but no larger than max nsa1345 // size. List is generated with the register class needed for last vaddr since1346 // it is the only one that could have a register other than VGPR32.1347 int EnableDisasmNum = !foldl(!head(AllNumAddrWords), !tail(AllNumAddrWords),1348 acc, var, !if(!le(var, nsa_max_addr), var, acc));1349 list<int> PossibleVariants =1350 !listconcat([12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2], !if(includeNSA1, [1], []));1351 list<LastVAddrSize> PartialNSAInstrs =1352 !foldl([]<LastVAddrSize>, PossibleVariants, lhs, dw,1353 !if(isIntInList<dw, AllNumAddrWords>.ret,1354 !listconcat(lhs, [LastVAddrSize<dw, !sub(nsa_max_addr, 1),1355 !eq(dw, EnableDisasmNum)>]),1356 lhs));1357}1358 1359multiclass MIMG_Sampler_Src_Helper <mimgopc op, string asm,1360 AMDGPUSampleVariant sample, RegisterOperand dst_rc,1361 bit enableDisasm = 0,1362 bit ExtendedImageInst = 1, bit isG16 = 0> {1363 foreach addr = MIMG_Sampler_AddrSizes<sample, isG16>.MachineInstrs in {1364 let VAddrDwords = addr.NumWords in {1365 if op.HAS_GFX10M then {1366 def _V # addr.NumWords1367 : MIMG_Sampler_Helper <op, asm, dst_rc, addr.RegClass,1368 !if(!and(enableDisasm, addr.Disassemble), "GFX8", "")>;1369 if !not(ExtendedImageInst) then1370 def _V # addr.NumWords # _gfx90a1371 : MIMG_Sampler_gfx90a <op, asm, dst_rc, addr.RegClass,1372 !if(!and(enableDisasm, addr.Disassemble), "GFX90A", "")>;1373 def _V # addr.NumWords # _gfx101374 : MIMG_Sampler_gfx10 <op, asm, dst_rc, addr.RegClass,1375 !if(!and(enableDisasm, addr.Disassemble), "GFX10", "")>;1376 }1377 if op.HAS_GFX11 then {1378 def _V # addr.NumWords # _gfx111379 : MIMG_Sampler_gfx11 <op, asm, dst_rc, addr.RegClass,1380 !if(!and(enableDisasm, addr.Disassemble), "GFX11", "")>;1381 }1382 }1383 }1384 1385 foreach addr = MIMG_Sampler_AddrSizes<sample, isG16>.NSAInstrs in {1386 let VAddrDwords = addr.NumWords in {1387 if op.HAS_GFX10M then {1388 def _V # addr.NumWords # _nsa_gfx101389 : MIMG_Sampler_nsa_gfx10<op, asm, dst_rc, addr.NumWords,1390 !if(!and(enableDisasm, addr.Disassemble), "GFX10", "")>;1391 }1392 }1393 }1394 1395 foreach addr = MIMG_Sampler_AddrSizes<sample, isG16, 5/*MaxNSASize*/>.PartialNSAInstrs in {1396 let VAddrDwords = addr.NumWords in {1397 if op.HAS_GFX11 then {1398 def _V # addr.NumWords # _nsa_gfx111399 : MIMG_Sampler_nsa_gfx11<op, asm, dst_rc, addr.NumWords, addr.RegClass,1400 !if(!and(enableDisasm, addr.Disassemble), "GFX11", "")>;1401 }1402 }1403 }1404 1405 foreach addr = MIMG_Sampler_AddrSizes<sample, isG16, 4/*MaxNSASize*/, 1>.PartialNSAInstrs in {1406 let VAddrDwords = addr.NumWords in {1407 if op.HAS_GFX12 then {1408 def _V # addr.NumWords # _gfx121409 : VSAMPLE_Sampler_gfx12<op, asm, dst_rc, addr.NumWords, addr.RegClass,1410 !if(!and(enableDisasm, addr.Disassemble), "GFX12", "")>;1411 }1412 }1413 }1414}1415 1416class MIMG_Sampler_BaseOpcode<AMDGPUSampleVariant sample>1417 : MIMGBaseOpcode {1418 let Sampler = 1;1419 let NumExtraArgs = !size(sample.ExtraAddrArgs);1420 let Gradients = sample.Gradients;1421 let LodOrClampOrMip = !ne(sample.LodOrClamp, "");1422}1423 1424multiclass MIMG_Sampler_NoReturn <mimgopc op, AMDGPUSampleVariant sample, bit wqm = 0, bit isG16, string asm> {1425 def "" : MIMG_Sampler_BaseOpcode<sample> {1426 let HasD16 = 1;1427 let G16 = isG16;1428 let NoReturn = 1;1429 }1430 1431 let BaseOpcode = !cast<MIMGBaseOpcode>(NAME), WQM = wqm,1432 mayLoad = 1, mayStore = 1, VDataDwords = 0 in {1433 foreach addr = MIMG_Sampler_AddrSizes<sample, isG16>.MachineInstrs in {1434 let VAddrDwords = addr.NumWords in {1435 if op.HAS_GFX10M then {1436 def _V # addr.NumWords # _gfx101437 : MIMG_Sampler_nortn_gfx10 <op, asm, addr.RegClass>;1438 }1439 if op.HAS_GFX11 then {1440 def _V # addr.NumWords # _gfx111441 : MIMG_Sampler_nortn_gfx11 <op, asm, addr.RegClass>;1442 }1443 }1444 }1445 1446 foreach addr = MIMG_Sampler_AddrSizes<sample, isG16>.NSAInstrs in {1447 let VAddrDwords = addr.NumWords in {1448 if op.HAS_GFX10M then {1449 def _V # addr.NumWords # _nsa_gfx101450 : MIMG_Sampler_nortn_nsa_gfx10<op, asm, addr.NumWords>;1451 }1452 }1453 }1454 1455 foreach addr = MIMG_Sampler_AddrSizes<sample, isG16, 5/*MaxNSASize*/>.PartialNSAInstrs in {1456 let VAddrDwords = addr.NumWords in {1457 if op.HAS_GFX11 then {1458 def _V # addr.NumWords # _nsa_gfx111459 : MIMG_Sampler_nortn_nsa_gfx11<op, asm, addr.NumWords, addr.RegClass>;1460 }1461 }1462 }1463 1464 foreach addr = MIMG_Sampler_AddrSizes<sample, isG16, 4/*MaxNSASize*/, 1>.PartialNSAInstrs in {1465 let VAddrDwords = addr.NumWords in {1466 if op.HAS_GFX12 then {1467 def _V # addr.NumWords # _gfx121468 : VSAMPLE_Sampler_nortn_gfx12<op, asm, addr.NumWords, addr.RegClass>;1469 }1470 }1471 }1472 }1473}1474 1475multiclass MIMG_Sampler <mimgopc op, AMDGPUSampleVariant sample, bit isPointSampleAccel = 0,1476 bit wqm = 0, bit isG16 = 0, bit isGetLod = 0,1477 string asm = "image_sample"#sample.LowerCaseMod#!if(isG16, "_g16", ""),1478 bit ExtendedImageInst = !ne(sample.LowerCaseMod, "")> {1479 def "" : MIMG_Sampler_BaseOpcode<sample> {1480 let HasD16 = !not(isGetLod);1481 let G16 = isG16;1482 let PointSampleAccel = isPointSampleAccel;1483 }1484 1485 let BaseOpcode = !cast<MIMGBaseOpcode>(NAME), WQM = wqm,1486 mayLoad = !not(isGetLod) in {1487 let VDataDwords = 1 in1488 defm _V1 : MIMG_Sampler_Src_Helper<op, asm, sample, AVLdSt_32, 1, ExtendedImageInst, isG16>;1489 let VDataDwords = 2 in1490 defm _V2 : MIMG_Sampler_Src_Helper<op, asm, sample, AVLdSt_64, 0, ExtendedImageInst, isG16>;1491 let VDataDwords = 3 in1492 defm _V3 : MIMG_Sampler_Src_Helper<op, asm, sample, AVLdSt_96, 0, ExtendedImageInst, isG16>;1493 let VDataDwords = 4 in1494 defm _V4 : MIMG_Sampler_Src_Helper<op, asm, sample, AVLdSt_128, 0, ExtendedImageInst, isG16>;1495 let VDataDwords = 5 in1496 defm _V5 : MIMG_Sampler_Src_Helper<op, asm, sample, AVLdSt_160, 0, ExtendedImageInst, isG16>;1497 }1498 1499 if !not(isGetLod) then1500 defm "_nortn" : MIMG_Sampler_NoReturn <op, sample, wqm, isG16, asm>;1501}1502 1503multiclass MIMG_Sampler_WQM <mimgopc op, AMDGPUSampleVariant sample, bit isPointSampleAccel = 0>1504 : MIMG_Sampler<op, sample, isPointSampleAccel, 1>;1505 1506multiclass MIMG_Gather <mimgopc op, AMDGPUSampleVariant sample, bit wqm = 0,1507 string asm = "image_gather4"#sample.LowerCaseMod> {1508 def "" : MIMG_Sampler_BaseOpcode<sample> {1509 let HasD16 = 1;1510 let Gather4 = 1;1511 }1512 1513 let BaseOpcode = !cast<MIMGBaseOpcode>(NAME), WQM = wqm,1514 Gather4 = 1 in {1515 let VDataDwords = 2 in1516 defm _V2 : MIMG_Sampler_Src_Helper<op, asm, sample, AVLdSt_64, /*enableDisasm*/ true>; /* for packed D16 only */1517 let VDataDwords = 4 in1518 defm _V4 : MIMG_Sampler_Src_Helper<op, asm, sample, AVLdSt_128>;1519 let VDataDwords = 5 in1520 defm _V5 : MIMG_Sampler_Src_Helper<op, asm, sample, AVLdSt_160>;1521 }1522}1523 1524multiclass MIMG_Gather_WQM <mimgopc op, AMDGPUSampleVariant sample>1525 : MIMG_Gather<op, sample, 1>;1526 1527class MIMG_IntersectRay_Helper<bit Is64, bit IsA16, bit isDual, bit isBVH8> {1528 int num_addrs = !if(isBVH8, 11, !if(Is64, !if(IsA16, 9, 12), !if(IsA16, 8, 11)));1529 RegisterOperand RegClass = MIMGAddrSize<num_addrs, 0>.RegClass;1530 1531 defvar Size = !cast<SIRegisterClassLike>(RegClass.RegClass).Size;1532 1533 int VAddrDwords = !srl(Size, 5);1534 1535 int GFX11PlusNSAAddrs = !if(IsA16, 4, 5);1536 RegisterOperand node_ptr_type = !if(Is64, VGPROp_64, VGPROp_32);1537 list<RegisterOperand> GFX11PlusAddrTypes =1538 !cond(isBVH8 : [node_ptr_type, VGPROp_64, VGPROp_96, VGPROp_96, VGPROp_32],1539 isDual : [node_ptr_type, VGPROp_64, VGPROp_96, VGPROp_96, VGPROp_64],1540 IsA16 : [node_ptr_type, VGPROp_32, VGPROp_96, VGPROp_96],1541 true : [node_ptr_type, VGPROp_32, VGPROp_96, VGPROp_96, VGPROp_96]);1542}1543 1544class MIMG_IntersectRay_gfx10<mimgopc op, string opcode, RegisterOperand AddrRC>1545 : MIMG_gfx10<op.GFX10M, (outs VReg_128:$vdata), "GFX10"> {1546 let InOperandList = (ins AddrRC:$vaddr0, SReg_128_XNULL:$srsrc, A16:$a16);1547 let AsmString = opcode#" $vdata, $vaddr0, $srsrc$a16";1548 1549 let nsa = 0;1550}1551 1552class MIMG_IntersectRay_nsa_gfx10<mimgopc op, string opcode, int num_addrs>1553 : MIMG_nsa_gfx10<op.GFX10M, (outs VReg_128:$vdata), num_addrs, "GFX10"> {1554 let InOperandList = !con(nsah.AddrIns, (ins SReg_128_XNULL:$srsrc, A16:$a16));1555 let AsmString = opcode#" $vdata, "#nsah.AddrAsm#", $srsrc$a16";1556}1557 1558class MIMG_IntersectRay_gfx11<mimgopc op, string opcode, RegisterOperand AddrRC>1559 : MIMG_gfx11<op.GFX11, (outs VReg_128:$vdata), "GFX11"> {1560 let InOperandList = (ins AddrRC:$vaddr0, SReg_128_XNULL:$srsrc, A16:$a16);1561 let AsmString = opcode#" $vdata, $vaddr0, $srsrc$a16";1562 1563 let nsa = 0;1564}1565 1566class MIMG_IntersectRay_nsa_gfx11<mimgopc op, string opcode, int num_addrs,1567 list<RegisterOperand> addr_types>1568 : MIMG_nsa_gfx11<op.GFX11, (outs VReg_128:$vdata), num_addrs, "GFX11",1569 addr_types> {1570 let InOperandList = !con(nsah.AddrIns, (ins SReg_128_XNULL:$srsrc, A16:$a16));1571 let AsmString = opcode#" $vdata, "#nsah.AddrAsm#", $srsrc$a16";1572}1573 1574class VIMAGE_IntersectRay_gfx12<mimgopc op, string opcode, int num_addrs,1575 bit isDual, bit isBVH8,1576 list<RegisterOperand> addr_types>1577 : VIMAGE_gfx12<op.GFX12, !if(!or(isDual, isBVH8),1578 (outs VReg_320:$vdata, VReg_96:$ray_origin_out,1579 VReg_96:$ray_dir_out),1580 (outs VReg_128:$vdata)),1581 num_addrs, "GFX12", addr_types> {1582 let Constraints = !if(!or(isDual, isBVH8),1583 "$ray_origin_out = $vaddr2, $ray_dir_out = $vaddr3", "");1584 let InOperandList = !con(nsah.AddrIns, (ins SReg_128_XNULL:$rsrc),1585 !if(!or(isDual, isBVH8), (ins), (ins A16:$a16)));1586 let AsmString = opcode#" $vdata, "#nsah.AddrAsm#", $rsrc"#1587 !if(!or(isDual, isBVH8), "", "$a16");1588 let SchedRW = !if(!or(isDual, isBVH8),1589 [WriteVMEM, WriteVMEM, WriteVMEM], [WriteVMEM]);1590}1591 1592multiclass MIMG_IntersectRay<mimgopc op, string opcode, bit Is64, bit IsA16,1593 bit isDual, bit isBVH8 = 0> {1594 defvar info = MIMG_IntersectRay_Helper<Is64, IsA16, isDual, isBVH8>;1595 def "" : MIMGBaseOpcode {1596 let BVH = 1;1597 let A16 = IsA16;1598 }1599 let dmask = 0xf,1600 d16 = 0,1601 cpol = 0,1602 tfe = 0,1603 r128 = 1,1604 dim = {0, 0, 0},1605 a16 = IsA16,1606 d16 = 0,1607 BaseOpcode = !cast<MIMGBaseOpcode>(NAME),1608 VDataDwords = 4 in {1609 let unorm = 1,1610 lwe = 0,1611 ssamp = 0 in {1612 if op.HAS_GFX10M then1613 def _sa_gfx10 : MIMG_IntersectRay_gfx10<op, opcode, info.RegClass> {1614 let VAddrDwords = info.VAddrDwords;1615 }1616 if op.HAS_GFX11 then1617 def _sa_gfx11 : MIMG_IntersectRay_gfx11<op, opcode, info.RegClass> {1618 let VAddrDwords = info.VAddrDwords;1619 }1620 if op.HAS_GFX10M then1621 def _nsa_gfx10 : MIMG_IntersectRay_nsa_gfx10<op, opcode, info.num_addrs> {1622 let VAddrDwords = info.num_addrs;1623 }1624 if op.HAS_GFX11 then1625 def _nsa_gfx11 : MIMG_IntersectRay_nsa_gfx11<op, opcode,1626 info.GFX11PlusNSAAddrs,1627 info.GFX11PlusAddrTypes> {1628 let VAddrDwords = info.num_addrs;1629 }1630 }1631 def _gfx12 : VIMAGE_IntersectRay_gfx12<op, opcode, info.GFX11PlusNSAAddrs,1632 isDual, isBVH8,1633 info.GFX11PlusAddrTypes> {1634 let VDataDwords = !if(!or(isDual, isBVH8), 10, 4);1635 let VAddrDwords = info.num_addrs;1636 }1637 }1638}1639 1640multiclass MIMG_MSAA_Load <mimgopc op, string asm> {1641 def "" : MIMGBaseOpcode {1642 let HasD16 = 1;1643 let Gather4 = 1; /* for appropriate dmask handling */1644 let MSAA = 1;1645 }1646 1647 let BaseOpcode = !cast<MIMGBaseOpcode>(NAME),1648 Gather4 = 1, hasPostISelHook = 0, mayLoad = 1 in {1649 let VDataDwords = 2 in1650 defm _V2 : MIMG_NoSampler_Src_Helper<op, asm, AVLdSt_64, 0, 0, 1>; /* packed D16 */1651 let VDataDwords = 3 in1652 defm _V3 : MIMG_NoSampler_Src_Helper<op, asm, AVLdSt_96, 0, 0, 1>; /* packed D16 + tfe */1653 let VDataDwords = 4 in1654 defm _V4 : MIMG_NoSampler_Src_Helper<op, asm, AVLdSt_128, 1, 0, 1>;1655 let VDataDwords = 5 in1656 defm _V5 : MIMG_NoSampler_Src_Helper<op, asm, AVLdSt_160, 0, 0, 1>;1657 }1658}1659 1660//===----------------------------------------------------------------------===//1661// MIMG Instructions1662//===----------------------------------------------------------------------===//1663let OtherPredicates = [HasImageInsts] in {1664 1665defm IMAGE_LOAD : MIMG_NoSampler <mimgopc<0x00, 0x00, 0x00>, "image_load", 1>;1666defm IMAGE_LOAD_MIP : MIMG_NoSampler <mimgopc<0x01, 0x01, 0x01>, "image_load_mip", 1, 1>;1667defm IMAGE_LOAD_PCK : MIMG_NoSampler <mimgopc<0x02, 0x02, 0x02>, "image_load_pck", 0>;1668defm IMAGE_LOAD_PCK_SGN : MIMG_NoSampler <mimgopc<0x03, 0x03, 0x03>, "image_load_pck_sgn", 0>;1669defm IMAGE_LOAD_MIP_PCK : MIMG_NoSampler <mimgopc<0x04, 0x04, 0x04>, "image_load_mip_pck", 0, 1>;1670defm IMAGE_LOAD_MIP_PCK_SGN : MIMG_NoSampler <mimgopc<0x05, 0x05, 0x05>, "image_load_mip_pck_sgn", 0, 1>;1671defm IMAGE_STORE : MIMG_Store <mimgopc<0x06, 0x06, 0x08>, "image_store", 1>;1672defm IMAGE_STORE_MIP : MIMG_Store <mimgopc<0x07, 0x07, 0x09>, "image_store_mip", 1, 1>;1673defm IMAGE_STORE_PCK : MIMG_Store <mimgopc<0x08, 0x08, 0x0a>, "image_store_pck", 0>;1674defm IMAGE_STORE_MIP_PCK : MIMG_Store <mimgopc<0x09, 0x09, 0x0b>, "image_store_mip_pck", 0, 1>;1675 1676defm IMAGE_GET_RESINFO : MIMG_NoSampler <mimgopc<0x17, 0x17, 0x0e, 0x0e, 0x0e>, "image_get_resinfo", 0, 1, 1>;1677 1678defm IMAGE_ATOMIC_SWAP : MIMG_Atomic <mimgopc<0x0a, 0x0a, 0x0f, 0x10, 0x0f>, "image_atomic_swap">;1679defm IMAGE_ATOMIC_CMPSWAP : MIMG_Atomic <mimgopc<0x0b, 0x0b, 0x10, 0x11, 0x10>, "image_atomic_cmpswap", 1>;1680defm IMAGE_ATOMIC_ADD : MIMG_Atomic_Renamed <mimgopc<0x0c, 0x0c, 0x11, 0x12, 0x11>, "image_atomic_add", "image_atomic_add_uint">;1681defm IMAGE_ATOMIC_SUB : MIMG_Atomic_Renamed <mimgopc<0x0d, 0x0d, 0x12, 0x13, 0x12>, "image_atomic_sub", "image_atomic_sub_uint">;1682defm IMAGE_ATOMIC_RSUB : MIMG_Atomic <mimgopc<MIMG.NOP, MIMG.NOP, MIMG.NOP, MIMG.NOP, 0x13>, "image_atomic_rsub">;1683defm IMAGE_ATOMIC_SMIN : MIMG_Atomic_Renamed <mimgopc<0x0e, 0x0e, 0x14>, "image_atomic_smin", "image_atomic_min_int">;1684defm IMAGE_ATOMIC_UMIN : MIMG_Atomic_Renamed <mimgopc<0x0f, 0x0f, 0x15>, "image_atomic_umin", "image_atomic_min_uint">;1685defm IMAGE_ATOMIC_SMAX : MIMG_Atomic_Renamed <mimgopc<0x10, 0x10, 0x16>, "image_atomic_smax", "image_atomic_max_int">;1686defm IMAGE_ATOMIC_UMAX : MIMG_Atomic_Renamed <mimgopc<0x11, 0x11, 0x17>, "image_atomic_umax", "image_atomic_max_uint">;1687defm IMAGE_ATOMIC_AND : MIMG_Atomic <mimgopc<0x12, 0x12, 0x18>, "image_atomic_and">;1688defm IMAGE_ATOMIC_OR : MIMG_Atomic <mimgopc<0x13, 0x13, 0x19>, "image_atomic_or">;1689defm IMAGE_ATOMIC_XOR : MIMG_Atomic <mimgopc<0x14, 0x14, 0x1a>, "image_atomic_xor">;1690defm IMAGE_ATOMIC_INC : MIMG_Atomic_Renamed <mimgopc<0x15, 0x15, 0x1b>, "image_atomic_inc", "image_atomic_inc_uint">;1691defm IMAGE_ATOMIC_DEC : MIMG_Atomic_Renamed <mimgopc<0x16, 0x16, 0x1c>, "image_atomic_dec", "image_atomic_dec_uint">;1692defm IMAGE_ATOMIC_FCMPSWAP : MIMG_Atomic <mimgopc<MIMG.NOP, MIMG.NOP, 0x1d, MIMG.NOP>, "image_atomic_fcmpswap", 1, 1>;1693defm IMAGE_ATOMIC_FMIN : MIMG_Atomic <mimgopc<MIMG.NOP, MIMG.NOP, 0x1e, MIMG.NOP>, "image_atomic_fmin", 0, 1>;1694defm IMAGE_ATOMIC_FMAX : MIMG_Atomic <mimgopc<MIMG.NOP, MIMG.NOP, 0x1f, MIMG.NOP>, "image_atomic_fmax", 0, 1>;1695defm IMAGE_ATOMIC_PK_ADD_F16 : MIMG_Atomic <mimgopc<0x86, MIMG.NOP, MIMG.NOP, MIMG.NOP, MIMG.NOP>, "image_atomic_pk_add_f16", 0, 1>;1696defm IMAGE_ATOMIC_PK_ADD_BF16 : MIMG_Atomic <mimgopc<0x87, MIMG.NOP, MIMG.NOP, MIMG.NOP, MIMG.NOP>, "image_atomic_pk_add_bf16", 0, 1>;1697defm IMAGE_ATOMIC_ADD_FLT : MIMG_Atomic <mimgopc<0x83, MIMG.NOP, MIMG.NOP, MIMG.NOP>, "image_atomic_add_flt", 0, 1>;1698defm IMAGE_ATOMIC_MIN_FLT : MIMG_Atomic <mimgopc<0x84, MIMG.NOP, MIMG.NOP, MIMG.NOP>, "image_atomic_min_num_flt", 0, 1, "image_atomic_min_flt">;1699defm IMAGE_ATOMIC_MAX_FLT : MIMG_Atomic <mimgopc<0x85, MIMG.NOP, MIMG.NOP, MIMG.NOP>, "image_atomic_max_num_flt", 0, 1, "image_atomic_max_flt">;1700let AssemblerPredicate = isGFX12Plus in {1701 def : AMDGPUMnemonicAlias<"image_atomic_fmin", "image_atomic_min_flt">;1702 def : AMDGPUMnemonicAlias<"image_atomic_fmax", "image_atomic_max_flt">;1703}1704 1705defm IMAGE_SAMPLE : MIMG_Sampler_WQM <mimgopc<0x1b, 0x1b, 0x20>, AMDGPUSample, 1>;1706let OtherPredicates = [HasImageInsts, HasExtendedImageInsts] in {1707defm IMAGE_SAMPLE_CL : MIMG_Sampler_WQM <mimgopc<0x40, 0x40, 0x21>, AMDGPUSample_cl>;1708defm IMAGE_SAMPLE_D : MIMG_Sampler <mimgopc<0x1c, 0x1c, 0x22>, AMDGPUSample_d>;1709defm IMAGE_SAMPLE_D_CL : MIMG_Sampler <mimgopc<0x41, 0x41, 0x23>, AMDGPUSample_d_cl>;1710defm IMAGE_SAMPLE_L : MIMG_Sampler <mimgopc<0x1d, 0x1d, 0x24>, AMDGPUSample_l, 1>;1711defm IMAGE_SAMPLE_B : MIMG_Sampler_WQM <mimgopc<0x1e, 0x1e, 0x25>, AMDGPUSample_b>;1712defm IMAGE_SAMPLE_B_CL : MIMG_Sampler_WQM <mimgopc<0x42, 0x42, 0x26>, AMDGPUSample_b_cl>;1713defm IMAGE_SAMPLE_LZ : MIMG_Sampler <mimgopc<0x1f, 0x1f, 0x27>, AMDGPUSample_lz, 1>;1714defm IMAGE_SAMPLE_C : MIMG_Sampler_WQM <mimgopc<0x20, 0x20, 0x28>, AMDGPUSample_c>;1715defm IMAGE_SAMPLE_C_CL : MIMG_Sampler_WQM <mimgopc<0x43, 0x43, 0x29>, AMDGPUSample_c_cl>;1716defm IMAGE_SAMPLE_C_D : MIMG_Sampler <mimgopc<0x21, 0x21, 0x2a>, AMDGPUSample_c_d>;1717defm IMAGE_SAMPLE_C_D_CL : MIMG_Sampler <mimgopc<0x44, 0x44, 0x2b>, AMDGPUSample_c_d_cl>;1718defm IMAGE_SAMPLE_C_L : MIMG_Sampler <mimgopc<0x22, 0x22, 0x2c>, AMDGPUSample_c_l>;1719defm IMAGE_SAMPLE_C_B : MIMG_Sampler_WQM <mimgopc<0x23, 0x23, 0x2d>, AMDGPUSample_c_b>;1720defm IMAGE_SAMPLE_C_B_CL : MIMG_Sampler_WQM <mimgopc<0x45, 0x45, 0x2e>, AMDGPUSample_c_b_cl>;1721defm IMAGE_SAMPLE_C_LZ : MIMG_Sampler <mimgopc<0x24, 0x24, 0x2f>, AMDGPUSample_c_lz>;1722defm IMAGE_SAMPLE_O : MIMG_Sampler_WQM <mimgopc<0x25, 0x25, 0x30>, AMDGPUSample_o>;1723defm IMAGE_SAMPLE_CL_O : MIMG_Sampler_WQM <mimgopc<0x46, 0x46, 0x31>, AMDGPUSample_cl_o>;1724defm IMAGE_SAMPLE_D_O : MIMG_Sampler <mimgopc<0x26, 0x26, 0x32>, AMDGPUSample_d_o>;1725defm IMAGE_SAMPLE_D_CL_O : MIMG_Sampler <mimgopc<0x47, 0x47, 0x33>, AMDGPUSample_d_cl_o>;1726defm IMAGE_SAMPLE_L_O : MIMG_Sampler <mimgopc<0x27, 0x27, 0x34>, AMDGPUSample_l_o>;1727defm IMAGE_SAMPLE_B_O : MIMG_Sampler_WQM <mimgopc<0x28, 0x28, 0x35>, AMDGPUSample_b_o>;1728defm IMAGE_SAMPLE_B_CL_O : MIMG_Sampler_WQM <mimgopc<0x48, 0x48, 0x36>, AMDGPUSample_b_cl_o>;1729defm IMAGE_SAMPLE_LZ_O : MIMG_Sampler <mimgopc<0x29, 0x29, 0x37>, AMDGPUSample_lz_o>;1730defm IMAGE_SAMPLE_C_O : MIMG_Sampler_WQM <mimgopc<0x2a, 0x2a, 0x38>, AMDGPUSample_c_o>;1731defm IMAGE_SAMPLE_C_CL_O : MIMG_Sampler_WQM <mimgopc<0x49, 0x49, 0x39>, AMDGPUSample_c_cl_o>;1732defm IMAGE_SAMPLE_C_D_O : MIMG_Sampler <mimgopc<0x2b, 0x2b, 0x3a>, AMDGPUSample_c_d_o>;1733defm IMAGE_SAMPLE_C_D_CL_O : MIMG_Sampler <mimgopc<0x4a, 0x4a, 0x3b>, AMDGPUSample_c_d_cl_o>;1734defm IMAGE_SAMPLE_C_L_O : MIMG_Sampler <mimgopc<0x2c, 0x2c, 0x3c>, AMDGPUSample_c_l_o>;1735defm IMAGE_SAMPLE_C_B_CL_O : MIMG_Sampler_WQM <mimgopc<0x4b, 0x4b, 0x3e>, AMDGPUSample_c_b_cl_o>;1736defm IMAGE_SAMPLE_C_B_O : MIMG_Sampler_WQM <mimgopc<0x2d, 0x2d, 0x3d>, AMDGPUSample_c_b_o>;1737defm IMAGE_SAMPLE_C_LZ_O : MIMG_Sampler <mimgopc<0x2e, 0x2e, 0x3f>, AMDGPUSample_c_lz_o>;1738defm IMAGE_GATHER4 : MIMG_Gather_WQM <mimgopc<0x2f, 0x2f, 0x40>, AMDGPUSample>;1739defm IMAGE_GATHER4_CL : MIMG_Gather_WQM <mimgopc<0x60, 0x60, 0x41>, AMDGPUSample_cl>;1740defm IMAGE_GATHER4_L : MIMG_Gather <mimgopc<0x30, 0x30, 0x44>, AMDGPUSample_l>;1741defm IMAGE_GATHER4_B : MIMG_Gather_WQM <mimgopc<0x31, 0x31, 0x45>, AMDGPUSample_b>;1742defm IMAGE_GATHER4_B_CL : MIMG_Gather_WQM <mimgopc<0x61, 0x61, 0x46>, AMDGPUSample_b_cl>;1743defm IMAGE_GATHER4_LZ : MIMG_Gather <mimgopc<0x32, 0x32, 0x47>, AMDGPUSample_lz>;1744defm IMAGE_GATHER4_C : MIMG_Gather_WQM <mimgopc<0x33, 0x33, 0x48>, AMDGPUSample_c>;1745defm IMAGE_GATHER4_C_CL : MIMG_Gather_WQM <mimgopc<0x62, 0x62, 0x49>, AMDGPUSample_c_cl>;1746defm IMAGE_GATHER4_C_L : MIMG_Gather <mimgopc<0x63, 0x63, 0x4c>, AMDGPUSample_c_l>;1747defm IMAGE_GATHER4_C_B : MIMG_Gather_WQM <mimgopc<0x64, 0x64, 0x4d>, AMDGPUSample_c_b>;1748defm IMAGE_GATHER4_C_B_CL : MIMG_Gather_WQM <mimgopc<0x65, 0x65, 0x4e>, AMDGPUSample_c_b_cl>;1749defm IMAGE_GATHER4_C_LZ : MIMG_Gather <mimgopc<0x34, 0x34, 0x4f>, AMDGPUSample_c_lz>;1750defm IMAGE_GATHER4_O : MIMG_Gather_WQM <mimgopc<0x35, 0x35, 0x50>, AMDGPUSample_o>;1751defm IMAGE_GATHER4_CL_O : MIMG_Gather_WQM <mimgopc<MIMG.NOP, MIMG.NOP, 0x51>, AMDGPUSample_cl_o>;1752defm IMAGE_GATHER4_L_O : MIMG_Gather <mimgopc<MIMG.NOP, MIMG.NOP, 0x54>, AMDGPUSample_l_o>;1753defm IMAGE_GATHER4_B_O : MIMG_Gather_WQM <mimgopc<MIMG.NOP, MIMG.NOP, 0x55>, AMDGPUSample_b_o>;1754defm IMAGE_GATHER4_B_CL_O : MIMG_Gather <mimgopc<MIMG.NOP, MIMG.NOP, 0x56>, AMDGPUSample_b_cl_o>;1755defm IMAGE_GATHER4_LZ_O : MIMG_Gather <mimgopc<0x36, 0x36, 0x57>, AMDGPUSample_lz_o>;1756defm IMAGE_GATHER4_C_O : MIMG_Gather_WQM <mimgopc<MIMG.NOP, MIMG.NOP, 0x58>, AMDGPUSample_c_o>;1757defm IMAGE_GATHER4_C_CL_O : MIMG_Gather_WQM <mimgopc<MIMG.NOP, MIMG.NOP, 0x59>, AMDGPUSample_c_cl_o>;1758defm IMAGE_GATHER4_C_L_O : MIMG_Gather <mimgopc<MIMG.NOP, MIMG.NOP, 0x5c>, AMDGPUSample_c_l_o>;1759defm IMAGE_GATHER4_C_B_O : MIMG_Gather_WQM <mimgopc<MIMG.NOP, MIMG.NOP, 0x5d>, AMDGPUSample_c_b_o>;1760defm IMAGE_GATHER4_C_B_CL_O : MIMG_Gather_WQM <mimgopc<MIMG.NOP, MIMG.NOP, 0x5e>, AMDGPUSample_c_b_cl_o>;1761defm IMAGE_GATHER4_C_LZ_O : MIMG_Gather <mimgopc<0x37, 0x37, 0x5f>, AMDGPUSample_c_lz_o>;1762 1763let OtherPredicates = [HasImageInsts, HasExtendedImageInsts, isGFX9Plus] in1764defm IMAGE_GATHER4H : MIMG_Gather <mimgopc<0x90, 0x90, 0x61, 0x42>, AMDGPUSample, 1, "image_gather4h">;1765 1766defm IMAGE_GET_LOD : MIMG_Sampler <mimgopc<0x38, 0x38, 0x60>, AMDGPUSample, 0, 1, 0, 1, "image_get_lod">;1767 1768defm IMAGE_SAMPLE_CD : MIMG_Sampler <mimgopc<MIMG.NOP, MIMG.NOP, 0x68>, AMDGPUSample_cd>;1769defm IMAGE_SAMPLE_CD_CL : MIMG_Sampler <mimgopc<MIMG.NOP, MIMG.NOP, 0x69>, AMDGPUSample_cd_cl>;1770defm IMAGE_SAMPLE_C_CD : MIMG_Sampler <mimgopc<MIMG.NOP, MIMG.NOP, 0x6a>, AMDGPUSample_c_cd>;1771defm IMAGE_SAMPLE_C_CD_CL : MIMG_Sampler <mimgopc<MIMG.NOP, MIMG.NOP, 0x6b>, AMDGPUSample_c_cd_cl>;1772defm IMAGE_SAMPLE_CD_O : MIMG_Sampler <mimgopc<MIMG.NOP, MIMG.NOP, 0x6c>, AMDGPUSample_cd_o>;1773defm IMAGE_SAMPLE_CD_CL_O : MIMG_Sampler <mimgopc<MIMG.NOP, MIMG.NOP, 0x6d>, AMDGPUSample_cd_cl_o>;1774defm IMAGE_SAMPLE_C_CD_O : MIMG_Sampler <mimgopc<MIMG.NOP, MIMG.NOP, 0x6e>, AMDGPUSample_c_cd_o>;1775defm IMAGE_SAMPLE_C_CD_CL_O : MIMG_Sampler <mimgopc<MIMG.NOP, MIMG.NOP, 0x6f>, AMDGPUSample_c_cd_cl_o>;1776} // End OtherPredicates = [HasImageInsts, HasExtendedImageInsts]1777 1778let OtherPredicates = [HasImageInsts, HasExtendedImageInsts, HasG16] in {1779defm IMAGE_SAMPLE_D_G16 : MIMG_Sampler <mimgopc<0x39, 0x39, 0xa2>, AMDGPUSample_d, 0, 0, 1>;1780defm IMAGE_SAMPLE_D_CL_G16 : MIMG_Sampler <mimgopc<0x5f, 0x5f, 0xa3>, AMDGPUSample_d_cl, 0, 0, 1>;1781defm IMAGE_SAMPLE_C_D_G16 : MIMG_Sampler <mimgopc<0x3a, 0x3a, 0xaa>, AMDGPUSample_c_d, 0, 0, 1>;1782defm IMAGE_SAMPLE_C_D_CL_G16 : MIMG_Sampler <mimgopc<0x54, 0x54, 0xab>, AMDGPUSample_c_d_cl, 0, 0, 1>;1783defm IMAGE_SAMPLE_D_O_G16 : MIMG_Sampler <mimgopc<0x3b, 0x3b, 0xb2>, AMDGPUSample_d_o, 0, 0, 1>;1784defm IMAGE_SAMPLE_D_CL_O_G16 : MIMG_Sampler <mimgopc<0x55, 0x55, 0xb3>, AMDGPUSample_d_cl_o, 0, 0, 1>;1785defm IMAGE_SAMPLE_C_D_O_G16 : MIMG_Sampler <mimgopc<0x3c, 0x3c, 0xba>, AMDGPUSample_c_d_o, 0, 0, 1>;1786defm IMAGE_SAMPLE_C_D_CL_O_G16 : MIMG_Sampler <mimgopc<0x56, 0x56, 0xbb>, AMDGPUSample_c_d_cl_o, 0, 0, 1>;1787defm IMAGE_SAMPLE_CD_G16 : MIMG_Sampler <mimgopc<MIMG.NOP, MIMG.NOP, 0xe8>, AMDGPUSample_cd, 0, 0, 1>;1788defm IMAGE_SAMPLE_CD_CL_G16 : MIMG_Sampler <mimgopc<MIMG.NOP, MIMG.NOP, 0xe9>, AMDGPUSample_cd_cl, 0, 0, 1>;1789defm IMAGE_SAMPLE_C_CD_G16 : MIMG_Sampler <mimgopc<MIMG.NOP, MIMG.NOP, 0xea>, AMDGPUSample_c_cd, 0, 0, 1>;1790defm IMAGE_SAMPLE_C_CD_CL_G16 : MIMG_Sampler <mimgopc<MIMG.NOP, MIMG.NOP, 0xeb>, AMDGPUSample_c_cd_cl, 0, 0, 1>;1791defm IMAGE_SAMPLE_CD_O_G16 : MIMG_Sampler <mimgopc<MIMG.NOP, MIMG.NOP, 0xec>, AMDGPUSample_cd_o, 0, 0, 1>;1792defm IMAGE_SAMPLE_CD_CL_O_G16 : MIMG_Sampler <mimgopc<MIMG.NOP, MIMG.NOP, 0xed>, AMDGPUSample_cd_cl_o, 0, 0, 1>;1793defm IMAGE_SAMPLE_C_CD_O_G16 : MIMG_Sampler <mimgopc<MIMG.NOP, MIMG.NOP, 0xee>, AMDGPUSample_c_cd_o, 0, 0, 1>;1794defm IMAGE_SAMPLE_C_CD_CL_O_G16 : MIMG_Sampler <mimgopc<MIMG.NOP, MIMG.NOP, 0xef>, AMDGPUSample_c_cd_cl_o, 0, 0, 1>;1795} // End OtherPredicates = [HasImageInsts, HasExtendedImageInsts, HasG16]1796 1797//def IMAGE_RSRC256 : MIMG_NoPattern_RSRC256 <"image_rsrc256", mimgopc<0x7e>>;1798//def IMAGE_SAMPLER : MIMG_NoPattern_ <"image_sampler", mimgopc<0x7f>>;1799 1800let OtherPredicates = [HasImageInsts, HasGFX10_AEncoding, isGFX10Only] in1801defm IMAGE_MSAA_LOAD_X : MIMG_NoSampler <mimgopc<MIMG.NOP, MIMG.NOP, 0x80>, "image_msaa_load", 1, 0, 0, 1>;1802 1803let OtherPredicates = [HasImageInsts, HasGFX10_AEncoding] in {1804defm IMAGE_MSAA_LOAD : MIMG_MSAA_Load <mimgopc<0x18, 0x18, MIMG.NOP>, "image_msaa_load">;1805 1806defm IMAGE_BVH_INTERSECT_RAY : MIMG_IntersectRay<mimgopc<0x19, 0x19, 0xe6>, "image_bvh_intersect_ray", 0, 0, 0>;1807defm IMAGE_BVH_INTERSECT_RAY_a16 : MIMG_IntersectRay<mimgopc<0x19, 0x19, 0xe6>, "image_bvh_intersect_ray", 0, 1, 0>;1808defm IMAGE_BVH64_INTERSECT_RAY : MIMG_IntersectRay<mimgopc<0x1a, 0x1a, 0xe7>, "image_bvh64_intersect_ray", 1, 0, 0>;1809defm IMAGE_BVH64_INTERSECT_RAY_a16 : MIMG_IntersectRay<mimgopc<0x1a, 0x1a, 0xe7>, "image_bvh64_intersect_ray", 1, 1, 0>;1810} // End OtherPredicates = [HasImageInsts, HasGFX10_AEncoding]1811 1812defm IMAGE_BVH_DUAL_INTERSECT_RAY : MIMG_IntersectRay<mimgopc<0x80, MIMG.NOP, MIMG.NOP>, "image_bvh_dual_intersect_ray", 1, 0, 1>;1813defm IMAGE_BVH8_INTERSECT_RAY : MIMG_IntersectRay<mimgopc<0x81, MIMG.NOP, MIMG.NOP>, "image_bvh8_intersect_ray", 1, 0, 0, 1>;1814 1815let SubtargetPredicate = isGFX12Plus in {1816 def : AMDGPUMnemonicAlias<"bvh_intersect_ray", "image_bvh_intersect_ray">;1817 def : AMDGPUMnemonicAlias<"bvh64_intersect_ray", "image_bvh64_intersect_ray">;1818 def : AMDGPUMnemonicAlias<"bvh_dual_intersect_ray", "image_bvh_dual_intersect_ray">;1819 def : AMDGPUMnemonicAlias<"bvh8_intersect_ray", "image_bvh8_intersect_ray">;1820}1821 1822} // End let OtherPredicates = [HasImageInsts]1823 1824/********** ========================================= **********/1825/********** Table of dimension-aware image intrinsics **********/1826/********** ========================================= **********/1827 1828class ImageDimIntrinsicInfo<AMDGPUImageDimIntrinsic I> {1829 Intrinsic Intr = I;1830 MIMGBaseOpcode BaseOpcode = !cast<MIMGBaseOpcode>(!strconcat("IMAGE_", I.P.OpMod));1831 MIMGBaseOpcode AtomicNoRetBaseOpcode = BaseOpcode;1832 AMDGPUDimProps Dim = I.P.Dim;1833 AMDGPUImageDimIntrinsicEval DimEval = AMDGPUImageDimIntrinsicEval<I.P>;1834 1835 bits<8> NumOffsetArgs = DimEval.NumOffsetArgs;1836 bits<8> NumBiasArgs = DimEval.NumBiasArgs;1837 bits<8> NumZCompareArgs = DimEval.NumZCompareArgs;1838 bits<8> NumGradients = DimEval.NumGradientArgs;1839 bits<8> NumDmask = DimEval.NumDmaskArgs;1840 bits<8> NumData = DimEval.NumDataArgs;1841 bits<8> NumVAddrs = DimEval.NumVAddrArgs;1842 bits<8> NumArgs = !add(DimEval.CachePolicyArgIndex, 1);1843 1844 bits<8> DMaskIndex = DimEval.DmaskArgIndex;1845 bits<8> VAddrStart = DimEval.VAddrArgIndex;1846 bits<8> OffsetIndex = DimEval.OffsetArgIndex;1847 bits<8> BiasIndex = DimEval.BiasArgIndex;1848 bits<8> ZCompareIndex = DimEval.ZCompareArgIndex;1849 bits<8> GradientStart = DimEval.GradientArgIndex;1850 bits<8> CoordStart = DimEval.CoordArgIndex;1851 bits<8> LodIndex = DimEval.LodArgIndex;1852 bits<8> MipIndex = DimEval.MipArgIndex;1853 bits<8> VAddrEnd = !add(DimEval.VAddrArgIndex, DimEval.NumVAddrArgs);1854 bits<8> RsrcIndex = DimEval.RsrcArgIndex;1855 bits<8> SampIndex = DimEval.SampArgIndex;1856 bits<8> UnormIndex = DimEval.UnormArgIndex;1857 bits<8> TexFailCtrlIndex = DimEval.TexFailCtrlArgIndex;1858 bits<8> CachePolicyIndex = DimEval.CachePolicyArgIndex;1859 1860 bits<8> BiasTyArg = !add(I.P.NumRetAndDataAnyTypes,1861 !if(!eq(NumOffsetArgs, 0), 0, I.P.ExtraAddrArgs[0].Type.isAny));1862 bits<8> GradientTyArg = !add(I.P.NumRetAndDataAnyTypes,1863 !foldl(0, I.P.ExtraAddrArgs, cnt, arg, !add(cnt, arg.Type.isAny)));1864 bits<8> CoordTyArg = !add(GradientTyArg, !if(I.P.Gradients, 1, 0));1865}1866 1867class ImageDimAtomicIntrinsicInfo<AMDGPUImageDimIntrinsic I>1868 : ImageDimIntrinsicInfo<I> {1869 MIMGBaseOpcode AtomicNoRetBaseOpcode =1870 !cast<MIMGBaseOpcode>(!strconcat("IMAGE_", I.P.OpMod, "_NORTN"));1871}1872 1873def ImageDimIntrinsicTable : GenericTable {1874 let FilterClass = "ImageDimIntrinsicInfo";1875 let Fields = ["Intr", "BaseOpcode", "AtomicNoRetBaseOpcode", "Dim", "NumOffsetArgs", "NumBiasArgs", "NumZCompareArgs", "NumGradients", "NumDmask", "NumData",1876 "NumVAddrs", "NumArgs", "DMaskIndex", "VAddrStart", "OffsetIndex", "BiasIndex", "ZCompareIndex", "GradientStart", "CoordStart", "LodIndex", "MipIndex",1877 "VAddrEnd", "RsrcIndex", "SampIndex", "UnormIndex", "TexFailCtrlIndex", "CachePolicyIndex",1878 "BiasTyArg", "GradientTyArg", "CoordTyArg"];1879 string TypeOf_BaseOpcode = "MIMGBaseOpcode";1880 string TypeOf_AtomicNoRetBaseOpcode = "MIMGBaseOpcode";1881 string TypeOf_Dim = "MIMGDim";1882 1883 let PrimaryKey = ["Intr"];1884 let PrimaryKeyName = "getImageDimIntrinsicInfo";1885 let PrimaryKeyEarlyOut = 1;1886}1887 1888def getImageDimIntrinsicByBaseOpcode : SearchIndex {1889 let Table = ImageDimIntrinsicTable;1890 let Key = ["BaseOpcode", "Dim"];1891}1892 1893foreach intr = AMDGPUImageDimIntrinsics in {1894 def : ImageDimIntrinsicInfo<intr>;1895}1896 1897foreach intr = AMDGPUImageDimAtomicIntrinsics in {1898 def : ImageDimAtomicIntrinsicInfo<intr>;1899}1900 1901// L to LZ Optimization Mapping1902def : MIMGLZMapping<IMAGE_SAMPLE_L, IMAGE_SAMPLE_LZ>;1903def : MIMGLZMapping<IMAGE_SAMPLE_C_L, IMAGE_SAMPLE_C_LZ>;1904def : MIMGLZMapping<IMAGE_SAMPLE_L_O, IMAGE_SAMPLE_LZ_O>;1905def : MIMGLZMapping<IMAGE_SAMPLE_C_L_O, IMAGE_SAMPLE_C_LZ_O>;1906def : MIMGLZMapping<IMAGE_GATHER4_L, IMAGE_GATHER4_LZ>;1907def : MIMGLZMapping<IMAGE_GATHER4_C_L, IMAGE_GATHER4_C_LZ>;1908def : MIMGLZMapping<IMAGE_GATHER4_L_O, IMAGE_GATHER4_LZ_O>;1909def : MIMGLZMapping<IMAGE_GATHER4_C_L_O, IMAGE_GATHER4_C_LZ_O>;1910def : MIMGLZMapping<IMAGE_SAMPLE_L_nortn, IMAGE_SAMPLE_LZ_nortn>;1911def : MIMGLZMapping<IMAGE_SAMPLE_C_L_nortn, IMAGE_SAMPLE_C_LZ_nortn>;1912def : MIMGLZMapping<IMAGE_SAMPLE_L_O_nortn, IMAGE_SAMPLE_LZ_O_nortn>;1913def : MIMGLZMapping<IMAGE_SAMPLE_C_L_O_nortn, IMAGE_SAMPLE_C_LZ_O_nortn>;1914 1915// MIP to NONMIP Optimization Mapping1916def : MIMGMIPMapping<IMAGE_LOAD_MIP, IMAGE_LOAD>;1917def : MIMGMIPMapping<IMAGE_STORE_MIP, IMAGE_STORE>;1918 1919// Bias to NoBias Optimization Mapping1920def : MIMGBiasMapping<IMAGE_SAMPLE_B, IMAGE_SAMPLE>;1921def : MIMGBiasMapping<IMAGE_SAMPLE_B_CL, IMAGE_SAMPLE_CL>;1922def : MIMGBiasMapping<IMAGE_SAMPLE_C_B, IMAGE_SAMPLE_C>;1923def : MIMGBiasMapping<IMAGE_SAMPLE_C_B_CL, IMAGE_SAMPLE_C_CL>;1924def : MIMGBiasMapping<IMAGE_SAMPLE_B_O, IMAGE_SAMPLE_O>;1925def : MIMGBiasMapping<IMAGE_SAMPLE_B_CL_O, IMAGE_SAMPLE_CL_O>;1926def : MIMGBiasMapping<IMAGE_SAMPLE_C_B_O, IMAGE_SAMPLE_C_O>;1927def : MIMGBiasMapping<IMAGE_SAMPLE_C_B_CL_O, IMAGE_SAMPLE_C_CL_O>;1928def : MIMGBiasMapping<IMAGE_GATHER4_B, IMAGE_GATHER4>;1929def : MIMGBiasMapping<IMAGE_GATHER4_B_CL, IMAGE_GATHER4_CL>;1930def : MIMGBiasMapping<IMAGE_GATHER4_C_B, IMAGE_GATHER4_C>;1931def : MIMGBiasMapping<IMAGE_GATHER4_C_B_CL, IMAGE_GATHER4_C_CL>;1932def : MIMGBiasMapping<IMAGE_GATHER4_B_O, IMAGE_GATHER4_O>;1933def : MIMGBiasMapping<IMAGE_GATHER4_B_CL_O, IMAGE_GATHER4_CL_O>;1934def : MIMGBiasMapping<IMAGE_GATHER4_C_B_O, IMAGE_GATHER4_C_O>;1935def : MIMGBiasMapping<IMAGE_GATHER4_C_B_CL_O, IMAGE_GATHER4_C_CL_O>;1936def : MIMGBiasMapping<IMAGE_SAMPLE_B_nortn, IMAGE_SAMPLE_nortn>;1937def : MIMGBiasMapping<IMAGE_SAMPLE_B_CL_nortn, IMAGE_SAMPLE_CL_nortn>;1938def : MIMGBiasMapping<IMAGE_SAMPLE_C_B_nortn, IMAGE_SAMPLE_C_nortn>;1939def : MIMGBiasMapping<IMAGE_SAMPLE_C_B_CL_nortn, IMAGE_SAMPLE_C_CL_nortn>;1940def : MIMGBiasMapping<IMAGE_SAMPLE_B_O_nortn, IMAGE_SAMPLE_O_nortn>;1941def : MIMGBiasMapping<IMAGE_SAMPLE_B_CL_O_nortn, IMAGE_SAMPLE_CL_O_nortn>;1942def : MIMGBiasMapping<IMAGE_SAMPLE_C_B_O_nortn, IMAGE_SAMPLE_C_O_nortn>;1943def : MIMGBiasMapping<IMAGE_SAMPLE_C_B_CL_O_nortn, IMAGE_SAMPLE_C_CL_O_nortn>;1944 1945// Offset to NoOffset Optimization Mapping1946def : MIMGOffsetMapping<IMAGE_SAMPLE_O, IMAGE_SAMPLE>;1947def : MIMGOffsetMapping<IMAGE_SAMPLE_CL_O, IMAGE_SAMPLE_CL>;1948def : MIMGOffsetMapping<IMAGE_SAMPLE_D_O, IMAGE_SAMPLE_D>;1949def : MIMGOffsetMapping<IMAGE_SAMPLE_D_CL_O, IMAGE_SAMPLE_D_CL>;1950def : MIMGOffsetMapping<IMAGE_SAMPLE_D_O_G16, IMAGE_SAMPLE_D_G16>;1951def : MIMGOffsetMapping<IMAGE_SAMPLE_D_CL_O_G16, IMAGE_SAMPLE_D_CL_G16>;1952def : MIMGOffsetMapping<IMAGE_SAMPLE_L_O, IMAGE_SAMPLE_L>;1953def : MIMGOffsetMapping<IMAGE_SAMPLE_B_O, IMAGE_SAMPLE_B>;1954def : MIMGOffsetMapping<IMAGE_SAMPLE_B_CL_O, IMAGE_SAMPLE_B_CL>;1955def : MIMGOffsetMapping<IMAGE_SAMPLE_LZ_O, IMAGE_SAMPLE_LZ>;1956def : MIMGOffsetMapping<IMAGE_SAMPLE_C_O, IMAGE_SAMPLE_C>;1957def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CL_O, IMAGE_SAMPLE_C_CL>;1958def : MIMGOffsetMapping<IMAGE_SAMPLE_C_D_O, IMAGE_SAMPLE_C_D>;1959def : MIMGOffsetMapping<IMAGE_SAMPLE_C_D_CL_O, IMAGE_SAMPLE_C_D_CL>;1960def : MIMGOffsetMapping<IMAGE_SAMPLE_C_D_O_G16, IMAGE_SAMPLE_C_D_G16>;1961def : MIMGOffsetMapping<IMAGE_SAMPLE_C_D_CL_O_G16, IMAGE_SAMPLE_C_D_CL_G16>;1962def : MIMGOffsetMapping<IMAGE_SAMPLE_C_L_O, IMAGE_SAMPLE_C_L>;1963def : MIMGOffsetMapping<IMAGE_SAMPLE_C_B_CL_O, IMAGE_SAMPLE_C_B_CL>;1964def : MIMGOffsetMapping<IMAGE_SAMPLE_C_B_O, IMAGE_SAMPLE_C_B>;1965def : MIMGOffsetMapping<IMAGE_SAMPLE_C_LZ_O, IMAGE_SAMPLE_C_LZ>;1966def : MIMGOffsetMapping<IMAGE_GATHER4_O, IMAGE_GATHER4>;1967def : MIMGOffsetMapping<IMAGE_GATHER4_CL_O, IMAGE_GATHER4_CL>;1968def : MIMGOffsetMapping<IMAGE_GATHER4_L_O, IMAGE_GATHER4_L>;1969def : MIMGOffsetMapping<IMAGE_GATHER4_B_O, IMAGE_GATHER4_B>;1970def : MIMGOffsetMapping<IMAGE_GATHER4_B_CL_O, IMAGE_GATHER4_B_CL>;1971def : MIMGOffsetMapping<IMAGE_GATHER4_LZ_O, IMAGE_GATHER4_LZ>;1972def : MIMGOffsetMapping<IMAGE_GATHER4_C_O, IMAGE_GATHER4_C>;1973def : MIMGOffsetMapping<IMAGE_GATHER4_C_CL_O, IMAGE_GATHER4_C_CL>;1974def : MIMGOffsetMapping<IMAGE_GATHER4_C_L_O, IMAGE_GATHER4_C_L>;1975def : MIMGOffsetMapping<IMAGE_GATHER4_C_B_O, IMAGE_GATHER4_C_B>;1976def : MIMGOffsetMapping<IMAGE_GATHER4_C_B_CL_O, IMAGE_GATHER4_C_B_CL>;1977def : MIMGOffsetMapping<IMAGE_GATHER4_C_LZ_O, IMAGE_GATHER4_C_LZ>;1978def : MIMGOffsetMapping<IMAGE_SAMPLE_CD_O, IMAGE_SAMPLE_CD>;1979def : MIMGOffsetMapping<IMAGE_SAMPLE_CD_CL_O, IMAGE_SAMPLE_CD_CL>;1980def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CD_O, IMAGE_SAMPLE_C_CD>;1981def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CD_CL_O, IMAGE_SAMPLE_C_CD_CL>;1982def : MIMGOffsetMapping<IMAGE_SAMPLE_CD_O_G16, IMAGE_SAMPLE_CD_G16>;1983def : MIMGOffsetMapping<IMAGE_SAMPLE_CD_CL_O_G16, IMAGE_SAMPLE_CD_CL_G16>;1984def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CD_O_G16, IMAGE_SAMPLE_C_CD_G16>;1985def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CD_CL_O_G16, IMAGE_SAMPLE_C_CD_CL_G16>;1986def : MIMGOffsetMapping<IMAGE_SAMPLE_O_nortn, IMAGE_SAMPLE_nortn>;1987def : MIMGOffsetMapping<IMAGE_SAMPLE_CL_O_nortn, IMAGE_SAMPLE_CL_nortn>;1988def : MIMGOffsetMapping<IMAGE_SAMPLE_D_O_nortn, IMAGE_SAMPLE_D_nortn>;1989def : MIMGOffsetMapping<IMAGE_SAMPLE_D_CL_O_nortn, IMAGE_SAMPLE_D_CL_nortn>;1990def : MIMGOffsetMapping<IMAGE_SAMPLE_D_O_G16_nortn, IMAGE_SAMPLE_D_G16_nortn>;1991def : MIMGOffsetMapping<IMAGE_SAMPLE_D_CL_O_G16_nortn, IMAGE_SAMPLE_D_CL_G16_nortn>;1992def : MIMGOffsetMapping<IMAGE_SAMPLE_L_O_nortn, IMAGE_SAMPLE_L_nortn>;1993def : MIMGOffsetMapping<IMAGE_SAMPLE_B_O_nortn, IMAGE_SAMPLE_B_nortn>;1994def : MIMGOffsetMapping<IMAGE_SAMPLE_B_CL_O_nortn, IMAGE_SAMPLE_B_CL_nortn>;1995def : MIMGOffsetMapping<IMAGE_SAMPLE_LZ_O_nortn, IMAGE_SAMPLE_LZ_nortn>;1996def : MIMGOffsetMapping<IMAGE_SAMPLE_C_O_nortn, IMAGE_SAMPLE_C_nortn>;1997def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CL_O_nortn, IMAGE_SAMPLE_C_CL_nortn>;1998def : MIMGOffsetMapping<IMAGE_SAMPLE_C_D_O_nortn, IMAGE_SAMPLE_C_D_nortn>;1999def : MIMGOffsetMapping<IMAGE_SAMPLE_C_D_CL_O_nortn, IMAGE_SAMPLE_C_D_CL_nortn>;2000def : MIMGOffsetMapping<IMAGE_SAMPLE_C_D_O_G16_nortn, IMAGE_SAMPLE_C_D_G16_nortn>;2001def : MIMGOffsetMapping<IMAGE_SAMPLE_C_D_CL_O_G16_nortn, IMAGE_SAMPLE_C_D_CL_G16_nortn>;2002def : MIMGOffsetMapping<IMAGE_SAMPLE_C_L_O_nortn, IMAGE_SAMPLE_C_L_nortn>;2003def : MIMGOffsetMapping<IMAGE_SAMPLE_C_B_CL_O_nortn, IMAGE_SAMPLE_C_B_CL_nortn>;2004def : MIMGOffsetMapping<IMAGE_SAMPLE_C_B_O_nortn, IMAGE_SAMPLE_C_B_nortn>;2005def : MIMGOffsetMapping<IMAGE_SAMPLE_C_LZ_O_nortn, IMAGE_SAMPLE_C_LZ_nortn>;2006def : MIMGOffsetMapping<IMAGE_SAMPLE_CD_O_nortn, IMAGE_SAMPLE_CD>;2007def : MIMGOffsetMapping<IMAGE_SAMPLE_CD_CL_O_nortn, IMAGE_SAMPLE_CD_CL_nortn>;2008def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CD_O_nortn, IMAGE_SAMPLE_C_CD_nortn>;2009def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CD_CL_O_nortn, IMAGE_SAMPLE_C_CD_CL_nortn>;2010def : MIMGOffsetMapping<IMAGE_SAMPLE_CD_O_G16_nortn, IMAGE_SAMPLE_CD_G16_nortn>;2011def : MIMGOffsetMapping<IMAGE_SAMPLE_CD_CL_O_G16_nortn, IMAGE_SAMPLE_CD_CL_G16_nortn>;2012def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CD_O_G16_nortn, IMAGE_SAMPLE_C_CD_G16_nortn>;2013def : MIMGOffsetMapping<IMAGE_SAMPLE_C_CD_CL_O_G16_nortn, IMAGE_SAMPLE_C_CD_CL_G16_nortn>;2014 2015// G to G16 Optimization Mapping2016def : MIMGG16Mapping<IMAGE_SAMPLE_D, IMAGE_SAMPLE_D_G16>;2017def : MIMGG16Mapping<IMAGE_SAMPLE_D_CL, IMAGE_SAMPLE_D_CL_G16>;2018def : MIMGG16Mapping<IMAGE_SAMPLE_C_D, IMAGE_SAMPLE_C_D_G16>;2019def : MIMGG16Mapping<IMAGE_SAMPLE_C_D_CL, IMAGE_SAMPLE_C_D_CL_G16>;2020def : MIMGG16Mapping<IMAGE_SAMPLE_D_O, IMAGE_SAMPLE_D_O_G16>;2021def : MIMGG16Mapping<IMAGE_SAMPLE_D_CL_O, IMAGE_SAMPLE_D_CL_O_G16>;2022def : MIMGG16Mapping<IMAGE_SAMPLE_C_D_O, IMAGE_SAMPLE_C_D_O_G16>;2023def : MIMGG16Mapping<IMAGE_SAMPLE_C_D_CL_O, IMAGE_SAMPLE_C_D_CL_O_G16>;2024def : MIMGG16Mapping<IMAGE_SAMPLE_CD, IMAGE_SAMPLE_CD_G16>;2025def : MIMGG16Mapping<IMAGE_SAMPLE_CD_CL, IMAGE_SAMPLE_CD_CL_G16>;2026def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD, IMAGE_SAMPLE_C_CD_G16>;2027def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD_CL, IMAGE_SAMPLE_C_CD_CL_G16>;2028def : MIMGG16Mapping<IMAGE_SAMPLE_CD_O, IMAGE_SAMPLE_CD_O_G16>;2029def : MIMGG16Mapping<IMAGE_SAMPLE_CD_CL_O, IMAGE_SAMPLE_CD_CL_O_G16>;2030def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD_O, IMAGE_SAMPLE_C_CD_O_G16>;2031def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD_CL_O, IMAGE_SAMPLE_C_CD_CL_O_G16>;2032def : MIMGG16Mapping<IMAGE_SAMPLE_D_nortn, IMAGE_SAMPLE_D_G16_nortn>;2033def : MIMGG16Mapping<IMAGE_SAMPLE_D_CL_nortn, IMAGE_SAMPLE_D_CL_G16_nortn>;2034def : MIMGG16Mapping<IMAGE_SAMPLE_C_D_nortn, IMAGE_SAMPLE_C_D_G16_nortn>;2035def : MIMGG16Mapping<IMAGE_SAMPLE_C_D_CL_nortn, IMAGE_SAMPLE_C_D_CL_G16_nortn>;2036def : MIMGG16Mapping<IMAGE_SAMPLE_D_O_nortn, IMAGE_SAMPLE_D_O_G16_nortn>;2037def : MIMGG16Mapping<IMAGE_SAMPLE_D_CL_O_nortn, IMAGE_SAMPLE_D_CL_O_G16_nortn>;2038def : MIMGG16Mapping<IMAGE_SAMPLE_C_D_O_nortn, IMAGE_SAMPLE_C_D_O_G16_nortn>;2039def : MIMGG16Mapping<IMAGE_SAMPLE_C_D_CL_O_nortn, IMAGE_SAMPLE_C_D_CL_O_G16_nortn>;2040def : MIMGG16Mapping<IMAGE_SAMPLE_CD_nortn, IMAGE_SAMPLE_CD_G16_nortn>;2041def : MIMGG16Mapping<IMAGE_SAMPLE_CD_CL_nortn, IMAGE_SAMPLE_CD_CL_G16_nortn>;2042def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD_nortn, IMAGE_SAMPLE_C_CD_G16_nortn>;2043def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD_CL_nortn, IMAGE_SAMPLE_C_CD_CL_G16_nortn>;2044def : MIMGG16Mapping<IMAGE_SAMPLE_CD_O_nortn, IMAGE_SAMPLE_CD_O_G16_nortn>;2045def : MIMGG16Mapping<IMAGE_SAMPLE_CD_CL_O_nortn, IMAGE_SAMPLE_CD_CL_O_G16_nortn>;2046def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD_O_nortn, IMAGE_SAMPLE_C_CD_O_G16_nortn>;2047def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD_CL_O_nortn, IMAGE_SAMPLE_C_CD_CL_O_G16_nortn>;2048 2049//===----------------------------------------------------------------------===//2050// VIMAGE Tensor Instructions2051//===----------------------------------------------------------------------===//2052 2053class VIMAGE_TENSOR_Pseudo<string opName, bit _UpTo2D = 0> :2054 InstSI<(outs ), (ins ), "", []>,2055 SIMCInstr<opName#!if(_UpTo2D, "_D2", ""), SIEncodingFamily.NONE> {2056 2057 let isPseudo = 1;2058 let isCodeGenOnly = 1;2059 string Mnemonic = opName;2060 2061 let VALU = 1;2062 let maybeAtomic = 0;2063 let TENSOR_CNT = 1;2064 let mayLoad = 1;2065 let mayStore = 1;2066 let Uses = [EXEC, TENSORcnt];2067 let Defs = [TENSORcnt];2068 let SchedRW = [WriteVMEM, WriteLDS];2069 let UseNamedOperandTable = 1;2070 let hasSideEffects = 0;2071 2072 bit UpTo2D = _UpTo2D;2073 let InOperandList = !if(UpTo2D, (ins SReg_128_XNULL:$vaddr0, SReg_256_XNULL:$vaddr1, R128A16:$r128, CPol:$cpol),2074 (ins SReg_128_XNULL:$vaddr0, SReg_256_XNULL:$vaddr1, SReg_128_XNULL:$vaddr2,2075 SReg_128_XNULL:$vaddr3, R128A16:$r128, CPol:$cpol));2076 string AsmOperands = " $vaddr0, $vaddr1"#!if(UpTo2D, "", ", $vaddr2, $vaddr3")#"$r128$cpol";2077}2078 2079let SubtargetPredicate = isGFX1250Plus in {2080def TENSOR_LOAD_TO_LDS : VIMAGE_TENSOR_Pseudo<"tensor_load_to_lds">;2081def TENSOR_STORE_FROM_LDS : VIMAGE_TENSOR_Pseudo<"tensor_store_from_lds">;2082def TENSOR_LOAD_TO_LDS_D2 : VIMAGE_TENSOR_Pseudo<"tensor_load_to_lds", 1>;2083def TENSOR_STORE_FROM_LDS_D2 : VIMAGE_TENSOR_Pseudo<"tensor_store_from_lds", 1>;2084} // End SubtargetPredicate = isGFX1250Plus.2085 2086class TensorPat <VIMAGE_TENSOR_Pseudo inst, SDPatternOperator node> : GCNPat <2087 (node v4i32:$vaddr0, v8i32:$vaddr1, v4i32:$vaddr2, v4i32:$vaddr3, (i32 timm:$cpol)),2088 (inst $vaddr0, $vaddr1, $vaddr2, $vaddr3, 0, $cpol)2089>;2090 2091class TensorD2Pat <VIMAGE_TENSOR_Pseudo inst, SDPatternOperator node> : GCNPat <2092 (node v4i32:$vaddr0, v8i32:$vaddr1, (i32 timm:$cpol)),2093 (inst $vaddr0, $vaddr1, 0, $cpol)2094>;2095 2096let SubtargetPredicate = isGFX1250Plus in {2097def : TensorPat <TENSOR_LOAD_TO_LDS, int_amdgcn_tensor_load_to_lds>;2098def : TensorPat <TENSOR_STORE_FROM_LDS, int_amdgcn_tensor_store_from_lds>;2099def : TensorD2Pat <TENSOR_LOAD_TO_LDS_D2, int_amdgcn_tensor_load_to_lds_d2>;2100def : TensorD2Pat <TENSOR_STORE_FROM_LDS_D2, int_amdgcn_tensor_store_from_lds_d2>;2101}2102 2103class VIMAGE_TENSOR_Real <bits<8> op, VIMAGE_TENSOR_Pseudo ps, string opName = ps.Mnemonic> :2104 InstSI <ps.OutOperandList, ps.InOperandList, opName # ps.AsmOperands, []>,2105 VIMAGEe<op> {2106 2107 // copy relevant pseudo op flags2108 let SubtargetPredicate = ps.SubtargetPredicate;2109 let TSFlags = ps.TSFlags;2110 let mayLoad = ps.mayLoad;2111 let mayStore = ps.mayStore;2112 let UseNamedOperandTable = ps.UseNamedOperandTable;2113 let SchedRW = ps.SchedRW;2114 2115 // D# group 2 and 3 set to NULL for 2D or less.2116 let vaddr2 = !if(ps.UpTo2D, !cast<int>(SGPR_NULL_gfx11plus.HWEncoding), ?);2117 let vaddr3 = !if(ps.UpTo2D, !cast<int>(SGPR_NULL_gfx11plus.HWEncoding), ?);2118 2119 // Set VADDR4 to NULL2120 let vaddr4 = !cast<int>(SGPR_NULL_gfx11plus.HWEncoding);2121 2122 // set to 0 based on SPG.2123 let rsrc = 0;2124 let vdata = 0;2125 let d16 = 0;2126 let a16 = 0;2127 let tfe = 0;2128 let dmask = 1; // sp32129 let dim = 1; // sp32130}2131 2132multiclass VIMAGE_TENSOR_Real_gfx1250<bits<8> op> {2133 let AssemblerPredicate = isGFX1250Plus, DecoderNamespace = "GFX1250" in {2134 foreach DSuffix = ["_D2", ""] in {2135 defvar ps = !cast<VIMAGE_TENSOR_Pseudo>(NAME # DSuffix);2136 def DSuffix # _gfx1250 : VIMAGE_TENSOR_Real<op, ps, ps.Mnemonic>,2137 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX1250>;2138 }2139 }2140}2141 2142defm TENSOR_LOAD_TO_LDS : VIMAGE_TENSOR_Real_gfx1250<0xc4>;2143defm TENSOR_STORE_FROM_LDS : VIMAGE_TENSOR_Real_gfx1250<0xc5>;2144