3444 lines · plain
1//===-- BUFInstructions.td - Buffer Instruction Definitions ---------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9def MUBUFAddr64 : ComplexPattern<iPTR, 4, "SelectMUBUFAddr64">;10def MUBUFOffset : ComplexPattern<iPTR, 3, "SelectMUBUFOffset">;11 12let WantsParent = true in {13 def MUBUFScratchOffen : ComplexPattern<iPTR, 4, "SelectMUBUFScratchOffen">;14 def MUBUFScratchOffset : ComplexPattern<iPTR, 3, "SelectMUBUFScratchOffset", [], [], 20>;15}16 17def BUFSOffset : ComplexPattern<iPTR, 1, "SelectBUFSOffset">;18 19def BUFAddrKind {20 int Offset = 0;21 int OffEn = 1;22 int IdxEn = 2;23 int BothEn = 3;24 int Addr64 = 4;25}26 27class getAddrName<int addrKind> {28 string ret =29 !if(!eq(addrKind, BUFAddrKind.Offset), "offset",30 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen",31 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen",32 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen",33 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64",34 "")))));35}36 37class MUBUFAddr64Table <bit is_addr64, string Name> {38 bit IsAddr64 = is_addr64;39 string OpName = Name;40}41 42class MTBUFAddr64Table <bit is_addr64, string Name> {43 bit IsAddr64 = is_addr64;44 string OpName = Name;45}46 47 48//===----------------------------------------------------------------------===//49// BUF class (base class for MTBUF and MUBUF pseudos)50//===----------------------------------------------------------------------===//51 52class BUF_Pseudo <string opName, dag outs, dag ins,53 string asmOps, list<dag> pattern=[]> :54 InstSI<outs, ins, "", pattern>,55 SIMCInstr<opName, SIEncodingFamily.NONE> {56 57 let isPseudo = 1;58 let isCodeGenOnly = 1;59 let Size = 8;60 let UseNamedOperandTable = 1;61 62 string Mnemonic = opName;63 string AsmOperands = asmOps;64 65 Instruction Opcode = !cast<Instruction>(NAME);66 67 68 let VM_CNT = 1;69 let EXP_CNT = 1;70 71 let Uses = [EXEC];72 let hasSideEffects = 0;73 let SchedRW = [WriteVMEM];74 75 76 77 bits<1> offen = 0;78 bits<1> idxen = 0;79 bits<1> addr64 = 0;80 bits<1> lds = 0;81 bits<1> has_vdata = !not(lds);82 bits<1> has_vaddr = 1;83 bits<1> has_glc = 1;84 bits<1> has_dlc = 1;85 bits<1> glc_value = 0; // the value for glc if no such operand86 bits<1> dlc_value = 0; // the value for dlc if no such operand87 bits<1> has_srsrc = 1;88 bits<1> has_soffset = 1;89 bits<1> has_offset = 1;90 bits<1> has_slc = 1;91 bits<1> tfe = 0;92 bits<4> elements = 0;93 bits<1> has_sccb = 1;94 bits<1> sccb_value = 0;95 bits<1> IsBufferInv = 0;96}97 98 99 100//===----------------------------------------------------------------------===//101// MTBUF classes102//===----------------------------------------------------------------------===//103 104class MTBUFGetBaseOpcode<string Op> {105 string ret = !subst("FORMAT_XY", "FORMAT_X",106 !subst("FORMAT_XYZ", "FORMAT_X",107 !subst("FORMAT_XYZW", "FORMAT_X", Op)));108}109 110 111class MTBUF_Pseudo <string opName, dag outs, dag ins,112 string asmOps, list<dag> pattern=[]> :113 BUF_Pseudo <opName, outs, ins, asmOps, pattern> {114 115 Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret);116 let MTBUF = 1;117}118 119class MTBUF_Real <MTBUF_Pseudo ps, string real_name = ps.Mnemonic> :120 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []> {121 122 let isPseudo = 0;123 let isCodeGenOnly = 0;124 125 let VM_CNT = 1;126 let EXP_CNT = 1;127 let MTBUF = 1;128 129 // copy relevant pseudo op flags130 let UseNamedOperandTable = ps.UseNamedOperandTable;131 let SubtargetPredicate = ps.SubtargetPredicate;132 let OtherPredicates = ps.OtherPredicates;133 let AsmMatchConverter = ps.AsmMatchConverter;134 let Constraints = ps.Constraints;135 let TSFlags = ps.TSFlags;136 let SchedRW = ps.SchedRW;137 let mayLoad = ps.mayLoad;138 let mayStore = ps.mayStore;139 let IsAtomicRet = ps.IsAtomicRet;140 let IsAtomicNoRet = ps.IsAtomicNoRet;141 let Uses = ps.Uses;142 let Defs = ps.Defs;143 let isConvergent = ps.isConvergent;144 145 bits<12> offset;146 bits<5> cpol;147 bits<7> format;148 bits<8> vaddr;149 bits<10> vdata;150 bits<7> srsrc;151 bits<8> soffset;152 153 bits<4> dfmt = format{3-0};154 bits<3> nfmt = format{6-4};155 156 // GFX90A+ only: instruction uses AccVGPR for data157 // Bit supersedes tfe.158 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0);159}160 161class getMTBUFInsDA<list<RegisterOperand> vdataList,162 list<RegisterClass> vaddrList=[], bit hasRestrictedSOffset> {163 RegisterOperand vdata_op = !if(!empty(vdataList), ?, !head(vdataList));164 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));165 166 dag SOffset = !if(hasRestrictedSOffset, (ins SReg_32:$soffset),167 (ins SCSrc_b32:$soffset));168 169 dag NonVaddrInputs = !con((ins SReg_128_XNULL:$srsrc), SOffset,170 (ins Offset:$offset, FORMAT:$format, CPol_0:$cpol, i1imm_0:$swz));171 172 dag Inputs = !if(!empty(vaddrList),173 NonVaddrInputs,174 !con((ins vaddrClass:$vaddr), NonVaddrInputs));175 dag ret = !if(!empty(vdataList),176 Inputs,177 !con((ins vdata_op:$vdata), Inputs));178}179 180class getMTBUFIns<int addrKind, list<RegisterOperand> vdataList=[], bit hasRestrictedSOffset> {181 dag ret =182 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList, [], hasRestrictedSOffset>.ret,183 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32], hasRestrictedSOffset>.ret,184 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32], hasRestrictedSOffset>.ret,185 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64], hasRestrictedSOffset>.ret,186 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64], hasRestrictedSOffset>.ret,187 (ins))))));188}189 190class getMTBUFAsmOps<int addrKind> {191 string Pfx =192 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc,$format $soffset",193 !if(!eq(addrKind, BUFAddrKind.OffEn),194 "$vaddr, $srsrc,$format $soffset offen",195 !if(!eq(addrKind, BUFAddrKind.IdxEn),196 "$vaddr, $srsrc,$format $soffset idxen",197 !if(!eq(addrKind, BUFAddrKind.BothEn),198 "$vaddr, $srsrc,$format $soffset idxen offen",199 !if(!eq(addrKind, BUFAddrKind.Addr64),200 "$vaddr, $srsrc,$format $soffset addr64",201 "")))));202 string ret = " $vdata, " # Pfx # "$offset$cpol";203}204 205class MTBUF_SetupAddr<int addrKind> {206 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn),207 !eq(addrKind, BUFAddrKind.BothEn));208 209 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn),210 !eq(addrKind, BUFAddrKind.BothEn));211 212 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64);213 214 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset);215}216 217class MTBUF_Load_Pseudo <string opName,218 int addrKind,219 RegisterOperand vdataClass,220 int elems,221 bit hasRestrictedSOffset = 0,222 list<dag> pattern=[]>223 : MTBUF_Pseudo<opName,224 (outs vdataClass:$vdata),225 getMTBUFIns<addrKind, [], hasRestrictedSOffset>.ret,226 getMTBUFAsmOps<addrKind>.ret,227 pattern>,228 MTBUF_SetupAddr<addrKind> {229 let PseudoInstr = opName # "_" # getAddrName<addrKind>.ret;230 let mayLoad = 1;231 let mayStore = 0;232 let elements = elems;233}234 235multiclass MTBUF_Pseudo_Loads_Helper<string opName, RegisterOperand vdataClass,236 int elems, bit hasRestrictedSOffset> {237 238 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset>,239 MTBUFAddr64Table<0, NAME>;240 241 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, hasRestrictedSOffset>,242 MTBUFAddr64Table<1, NAME>;243 244 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset>;245 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset>;246 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset>;247 248 let DisableWQM = 1 in {249 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset>;250 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset>;251 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset>;252 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset>;253 }254}255 256multiclass MTBUF_Pseudo_Loads<string opName, RegisterOperand vdataClass,257 int elems> {258 defm NAME : MTBUF_Pseudo_Loads_Helper<opName, vdataClass, elems, 0>;259 defm _VBUFFER : MTBUF_Pseudo_Loads_Helper<opName, vdataClass, elems, 1>;260}261 262class MTBUF_Store_Pseudo <string opName,263 int addrKind,264 RegisterOperand vdataClass,265 int elems,266 bit hasRestrictedSOffset = 0,267 list<dag> pattern=[]>268 : MTBUF_Pseudo<opName,269 (outs),270 getMTBUFIns<addrKind, [vdataClass], hasRestrictedSOffset>.ret,271 getMTBUFAsmOps<addrKind>.ret,272 pattern>,273 MTBUF_SetupAddr<addrKind> {274 let PseudoInstr = opName # "_" # getAddrName<addrKind>.ret;275 let mayLoad = 0;276 let mayStore = 1;277 let elements = elems;278}279 280multiclass MTBUF_Pseudo_Stores_Helper<string opName, RegisterOperand vdataClass,281 int elems, bit hasRestrictedSOffset> {282 283 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset>,284 MTBUFAddr64Table<0, NAME>;285 286 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, hasRestrictedSOffset>,287 MTBUFAddr64Table<1, NAME>;288 289 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset>;290 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset>;291 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset>;292 293 let DisableWQM = 1 in {294 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset>;295 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset>;296 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset>;297 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset>;298 }299}300 301multiclass MTBUF_Pseudo_Stores<string opName, RegisterOperand vdataClass,302 int elems> {303 defm NAME : MTBUF_Pseudo_Stores_Helper<opName, vdataClass, elems, 0>;304 defm _VBUFFER : MTBUF_Pseudo_Stores_Helper<opName, vdataClass, elems, 1>;305}306 307//===----------------------------------------------------------------------===//308// MUBUF classes309//===----------------------------------------------------------------------===//310 311class MUBUFGetBaseOpcode<string Op> {312 string ret = !subst("DWORDX2", "DWORD",313 !subst("DWORDX3", "DWORD",314 !subst("DWORDX4", "DWORD", Op)));315}316 317class MUBUF_Pseudo <string opName, dag outs, dag ins,318 string asmOps, list<dag> pattern=[]> :319 BUF_Pseudo <opName, outs, ins, asmOps, pattern> {320 321 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret);322 let MUBUF = 1;323 let AsmMatchConverter = "cvtMubuf";324 let usesCustomInserter = 1;325}326 327class MUBUF_Real <MUBUF_Pseudo ps, string real_name = ps.Mnemonic> :328 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []> {329 330 let isPseudo = 0;331 let isCodeGenOnly = 0;332 333 let VM_CNT = 1;334 let EXP_CNT = 1;335 let MUBUF = 1;336 337 // copy relevant pseudo op flags338 let SubtargetPredicate = ps.SubtargetPredicate;339 let AsmMatchConverter = ps.AsmMatchConverter;340 let OtherPredicates = ps.OtherPredicates;341 let Constraints = ps.Constraints;342 let TSFlags = ps.TSFlags;343 let UseNamedOperandTable = ps.UseNamedOperandTable;344 let SchedRW = ps.SchedRW;345 let mayLoad = ps.mayLoad;346 let mayStore = ps.mayStore;347 let IsAtomicRet = ps.IsAtomicRet;348 let IsAtomicNoRet = ps.IsAtomicNoRet;349 let VALU = ps.VALU;350 let LGKM_CNT = ps.LGKM_CNT;351 let Uses = ps.Uses;352 let Defs = ps.Defs;353 let isConvergent = ps.isConvergent;354 355 bits<12> offset;356 bits<5> cpol;357 bits<8> vaddr;358 bits<10> vdata;359 bits<7> srsrc;360 bits<8> soffset;361 362 // GFX90A+ only: instruction uses AccVGPR for data363 // Bit supersedes tfe.364 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0);365}366 367// For cache invalidation instructions.368class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> :369 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> {370 371 let AsmMatchConverter = "";372 373 let hasSideEffects = 1;374 let mayLoad = 0;375 let mayStore = 0;376 377 let IsBufferInv = 1;378 // Set everything else to 0.379 let offen = 0;380 let idxen = 0;381 let addr64 = 0;382 let has_vdata = 0;383 let has_vaddr = 0;384 let has_glc = 0;385 let has_dlc = 0;386 let glc_value = 0;387 let dlc_value = 0;388 let has_srsrc = 0;389 let has_soffset = 0;390 let has_offset = 0;391 let has_slc = 0;392 let has_sccb = 0;393 let sccb_value = 0;394}395 396class getBUFVDataRegisterOperand<int Size, bit isTFE> {397 defvar tfeVDataOp =398 !cond(!eq(Size, 16) : AVLdSt_64,399 !eq(Size, 32) : AVLdSt_64,400 !eq(Size, 64) : AVLdSt_96,401 !eq(Size, 96) : AVLdSt_128,402 !eq(Size, 128) : AVLdSt_160);403 404 defvar VDataOp =405 !cond(!eq(Size, 16) : AVLdSt_32,406 !eq(Size, 32) : AVLdSt_32,407 !eq(Size, 64) : AVLdSt_64,408 !eq(Size, 96) : AVLdSt_96,409 !eq(Size, 128) : AVLdSt_128);410 411 RegisterOperand ret = !if(isTFE, tfeVDataOp, VDataOp);412}413 414class getBUFVDataRegisterOperandForOp<RegisterOperand Op, bit isTFE> {415 defvar Size = !cast<RegisterClass>(Op.RegClass).Size;416 RegisterOperand ret = getBUFVDataRegisterOperand<Size, isTFE>.ret;417}418 419class getMUBUFInsDA<list<RegisterOperand> vdataList,420 list<RegisterClassLike> vaddrList, bit isTFE, bit hasRestrictedSOffset> {421 RegisterOperand vdataClass = !if(!empty(vdataList), ?, !head(vdataList));422 RegisterClassLike vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));423 RegisterOperand vdata_op = getBUFVDataRegisterOperand<!cast<SIRegisterClassLike>(vdataClass.RegClass).Size, isTFE>.ret;424 425 dag SOffset = !if(hasRestrictedSOffset, (ins SReg_32:$soffset), (ins SCSrc_b32:$soffset));426 dag NonVaddrInputs = !con((ins SReg_128_XNULL:$srsrc), SOffset, (ins Offset:$offset, CPol_0:$cpol, i1imm_0:$swz));427 428 dag Inputs = !if(!empty(vaddrList), NonVaddrInputs, !con((ins vaddrClass:$vaddr), NonVaddrInputs));429 dag ret = !if(!empty(vdataList), Inputs, !con((ins vdata_op:$vdata), Inputs));430}431 432class getMUBUFElements<ValueType vt> {433 int ret =434 !if(!eq(vt, f16), 1,435 !if(!eq(vt, v2f16), 2,436 !if(!eq(vt, v3f16), 3,437 !if(!eq(vt, v4f16), 4,438 !if(!eq(vt.Size, 32), 1,439 !if(!eq(vt.Size, 64), 2,440 !if(!eq(vt.Size, 96), 3,441 !if(!eq(vt.Size, 128), 4, 0)442 )443 )444 )445 )446 )447 )448 );449}450 451class getMUBUFIns<int addrKind, list<RegisterOperand> vdataList, bit isTFE, bit hasRestrictedSOffset> {452 dag ret =453 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isTFE, hasRestrictedSOffset>.ret,454 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isTFE, hasRestrictedSOffset>.ret,455 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isTFE, hasRestrictedSOffset>.ret,456 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64_AlignTarget], isTFE, hasRestrictedSOffset>.ret,457 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64_AlignTarget], isTFE, hasRestrictedSOffset>.ret,458 (ins))))));459}460 461class getMUBUFAsmOps<int addrKind, bit noVdata = 0, bit isLds = 0, bit isTFE = 0> {462 string Vdata = !if(noVdata, " ", " $vdata, ");463 string Lds = !if(isLds, " lds", "");464 string TFE = !if(isTFE, " tfe", "");465 string MainArgs =466 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset",467 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen",468 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen",469 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen",470 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64",471 "")))));472 string Offset = "$offset";473 string OtherArgs = "$cpol";474 475 string ret = Vdata # MainArgs # Offset # OtherArgs # Lds # TFE;476}477 478class MUBUF_SetupAddr<int addrKind> {479 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn),480 !eq(addrKind, BUFAddrKind.BothEn));481 482 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn),483 !eq(addrKind, BUFAddrKind.BothEn));484 485 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64);486 487 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset);488}489 490class MUBUF_Load_Pseudo <string opName,491 int addrKind,492 ValueType vdata_vt,493 bit HasTiedDest = 0,494 bit isLds = 0,495 bit isLdsOpc = 0,496 bit isTFE = 0,497 bit hasRestrictedSOffset = 0,498 list<dag> pattern=[],499 RegisterOperand vdata_op = getBUFVDataRegisterOperand<vdata_vt.Size, isTFE>.ret>500 : MUBUF_Pseudo<opName,501 !if(!or(isLds, isLdsOpc), (outs), (outs vdata_op:$vdata)),502 !con(getMUBUFIns<addrKind, [], isTFE, hasRestrictedSOffset>.ret,503 !if(HasTiedDest, (ins vdata_op:$vdata_in), (ins))),504 getMUBUFAsmOps<addrKind, !or(isLds, isLdsOpc), isLds, isTFE>.ret,505 pattern>,506 MUBUF_SetupAddr<addrKind> {507 let PseudoInstr = opName # !if(isLds, "_lds", "") # !if(isTFE, "_tfe", "") #508 "_" # getAddrName<addrKind>.ret;509 let AsmMatchConverter = "cvtMubuf";510 511 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", "");512 let LGKM_CNT = isLds;513 let has_vdata = !not(!or(isLds, isLdsOpc));514 let mayLoad = 1;515 let mayStore = isLds;516 let Uses = !if(!or(isLds, isLdsOpc) , [EXEC, M0], [EXEC]);517 let tfe = isTFE;518 let lds = isLds;519 let elements = getMUBUFElements<vdata_vt>.ret;520 let VALU = isLds;521}522 523class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : GCNPat <524 (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset))),525 (load_vt (inst v4i32:$srsrc, i32:$soffset, i32:$offset))526>;527 528class MUBUF_Addr64_Load_Pat <Instruction inst,529 ValueType load_vt = i32,530 SDPatternOperator ld = null_frag> : GCNPat <531 (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset))),532 (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i32:$offset))533>;534 535multiclass MUBUF_Pseudo_Load_Pats_Common<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> {536 def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>;537 def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>;538}539 540multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag>{541 let OtherPredicates = [HasUnrestrictedSOffset] in {542 defm : MUBUF_Pseudo_Load_Pats_Common<BaseInst, load_vt, ld>;543 }544 defm : MUBUF_Pseudo_Load_Pats_Common<BaseInst # "_VBUFFER", load_vt, ld>;545}546 547multiclass MUBUF_Pseudo_Loads_Helper<string opName, ValueType load_vt,548 bit TiedDest, bit isLds, bit isTFE, bit hasRestrictedSOffset> {549 defvar legal_load_vt = !if(!eq(load_vt, v3f16), v4f16, load_vt);550 551 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>,552 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>;553 554 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>,555 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>;556 557 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>;558 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>;559 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>;560 561 let DisableWQM = 1 in {562 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>;563 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>;564 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>;565 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>;566 }567}568 569multiclass MUBUF_Pseudo_Loads<string opName, ValueType load_vt = i32,570 bit TiedDest = 0, bit isLds = 0> {571 defm NAME : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 0, 0>;572 defm _VBUFFER : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 0, 1>;573 574 if !not(isLds) then {575 defm _TFE : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 1, 0>;576 defm _TFE_VBUFFER : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 1, 1>;577 }578}579 580multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32, Predicate LDSPred = TruePredicate> {581 defm NAME : MUBUF_Pseudo_Loads<opName, load_vt>;582 583 if !ne(LDSPred, TruePredicate) then {584 let SubtargetPredicate = LDSPred in {585 defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, 0, 1>;586 }587 } else {588 defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, 0, 1>;589 }590 591}592 593class MUBUF_Store_Pseudo <string opName,594 int addrKind,595 ValueType store_vt,596 bit isTFE = 0,597 bit hasRestrictedSOffset = 0,598 list<dag> pattern=[]>599 : MUBUF_Pseudo<opName,600 (outs),601 getMUBUFIns<addrKind, [getVregSrcForVT<store_vt>.ret], isTFE, hasRestrictedSOffset>.ret,602 getMUBUFAsmOps<addrKind, 0, 0, isTFE>.ret,603 pattern>,604 MUBUF_SetupAddr<addrKind> {605 let PseudoInstr = opName # "_" # !if(isTFE, "_tfe", "") #606 getAddrName<addrKind>.ret;607 let mayLoad = 0;608 let mayStore = 1;609 let elements = getMUBUFElements<store_vt>.ret;610 let tfe = isTFE;611}612 613multiclass MUBUF_Pseudo_Store_Pats_Common<string BaseInst, ValueType store_vt = i32, SDPatternOperator st = null_frag> {614 615 def : GCNPat <616 (st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset)),617 (!cast<MUBUF_Pseudo>(BaseInst # _OFFSET) store_vt:$vdata, v4i32:$srsrc, i32:$soffset, i32:$offset)>;618 619 def : GCNPat <620 (st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset)),621 (!cast<MUBUF_Pseudo>(BaseInst # _ADDR64) store_vt:$vdata, i64:$vaddr, v4i32:$srsrc, i32:$soffset, i32:$offset)>;622}623 624multiclass MUBUF_Pseudo_Store_Pats<string BaseInst, ValueType store_vt = i32, SDPatternOperator st = null_frag> {625 let OtherPredicates = [HasUnrestrictedSOffset] in {626 defm : MUBUF_Pseudo_Store_Pats_Common<BaseInst, store_vt, st>;627 }628 defm : MUBUF_Pseudo_Store_Pats_Common<BaseInst # "_VBUFFER", store_vt, st>;629}630 631multiclass MUBUF_Pseudo_Stores_Helper<string opName, ValueType store_vt,632 bit isTFE, bit hasRestrictedSOffset> {633 defvar legal_store_vt = !if(!eq(store_vt, v3f16), v4f16, store_vt);634 635 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE, hasRestrictedSOffset>,636 MUBUFAddr64Table<0, NAME>;637 638 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt, isTFE, hasRestrictedSOffset>,639 MUBUFAddr64Table<1, NAME>;640 641 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE, hasRestrictedSOffset>;642 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE, hasRestrictedSOffset>;643 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE, hasRestrictedSOffset>;644 645 let DisableWQM = 1 in {646 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE, hasRestrictedSOffset>;647 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE, hasRestrictedSOffset>;648 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE, hasRestrictedSOffset>;649 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE, hasRestrictedSOffset>;650 }651}652 653multiclass MUBUF_Pseudo_Stores<string opName, ValueType store_vt = i32> {654 defm NAME : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 0, 0>;655 defm _TFE : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 1, 0>;656 657 defm _VBUFFER : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 0, 1>;658 defm _TFE_VBUFFER : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 1, 1>;659}660 661class MUBUF_Pseudo_Store_Lds<string opName>662 : MUBUF_Pseudo<opName,663 (outs),664 (ins SReg_128_XNULL:$srsrc, SCSrc_b32:$soffset, Offset:$offset, CPol:$cpol, i1imm:$swz),665 " $srsrc, $soffset$offset lds$cpol"> {666 let LGKM_CNT = 1;667 let mayLoad = 1;668 let mayStore = 1;669 670 let has_vdata = 0;671 let has_vaddr = 0;672 let lds = 1;673 let VALU = 1;674 675 let Uses = [EXEC, M0];676 let AsmMatchConverter = "cvtMubuf";677}678 679class getMUBUFAtomicInsDA<RegisterOperand vdata_op, bit vdata_in, bit hasRestrictedSOffset,680 list<RegisterClassLike> vaddrList=[]> {681 RegisterClassLike vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));682 683 dag VData = !if(vdata_in, (ins vdata_op:$vdata_in), (ins vdata_op:$vdata));684 dag Data = !if(!empty(vaddrList), VData, !con(VData, (ins vaddrClass:$vaddr)));685 dag SOffset = !if(hasRestrictedSOffset, (ins SReg_32:$soffset), (ins SCSrc_b32:$soffset));686 dag MainInputs = !con((ins SReg_128_XNULL:$srsrc), SOffset, (ins Offset:$offset));687 dag CPol = !if(vdata_in, (ins CPol_GLC_WithDefault:$cpol),688 (ins CPol_NonGLC_WithDefault:$cpol));689 690 dag ret = !con(Data, MainInputs, CPol);691}692 693class getMUBUFAtomicIns<int addrKind,694 RegisterOperand vdataClass,695 bit vdata_in,696 bit hasRestrictedSOffset> {697 dag ret =698 !if(!eq(addrKind, BUFAddrKind.Offset),699 getMUBUFAtomicInsDA<vdataClass, vdata_in, hasRestrictedSOffset>.ret,700 !if(!eq(addrKind, BUFAddrKind.OffEn),701 getMUBUFAtomicInsDA<vdataClass, vdata_in, hasRestrictedSOffset, [VGPR_32]>.ret,702 !if(!eq(addrKind, BUFAddrKind.IdxEn),703 getMUBUFAtomicInsDA<vdataClass, vdata_in, hasRestrictedSOffset, [VGPR_32]>.ret,704 !if(!eq(addrKind, BUFAddrKind.BothEn),705 getMUBUFAtomicInsDA<vdataClass, vdata_in, hasRestrictedSOffset, [VReg_64_AlignTarget]>.ret,706 !if(!eq(addrKind, BUFAddrKind.Addr64),707 getMUBUFAtomicInsDA<vdataClass, vdata_in, hasRestrictedSOffset, [VReg_64_AlignTarget]>.ret,708 (ins))))));709}710 711class MUBUF_Atomic_Pseudo<string opName,712 int addrKind,713 dag outs,714 dag ins,715 string asmOps,716 list<dag> pattern=[]>717 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>,718 MUBUF_SetupAddr<addrKind> {719 let mayStore = 1;720 let mayLoad = 1;721 let hasSideEffects = 1;722 let DisableWQM = 1;723 let has_glc = 0;724 let has_dlc = 0;725 let has_sccb = 1;726 let AsmMatchConverter = "cvtMubufAtomic";727}728 729class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind,730 RegisterOperand vdataClass,731 bit hasRestrictedSOffset = 0,732 list<dag> pattern=[]>733 : MUBUF_Atomic_Pseudo<opName, addrKind,734 (outs),735 getMUBUFAtomicIns<addrKind, vdataClass, 0, hasRestrictedSOffset>.ret,736 getMUBUFAsmOps<addrKind>.ret,737 pattern> {738 let PseudoInstr = opName # "_" # getAddrName<addrKind>.ret;739 let glc_value = 0;740 let dlc_value = 0;741 let sccb_value = 0;742 let IsAtomicNoRet = 1;743}744 745class MUBUF_AtomicRet_Pseudo<string opName, int addrKind,746 RegisterOperand vdata_op,747 bit hasRestrictedSOffset = 0,748 list<dag> pattern=[]>749 : MUBUF_Atomic_Pseudo<opName, addrKind,750 (outs vdata_op:$vdata),751 getMUBUFAtomicIns<addrKind, vdata_op, 1, hasRestrictedSOffset>.ret,752 getMUBUFAsmOps<addrKind>.ret,753 pattern> {754 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKind>.ret;755 let glc_value = 1;756 let dlc_value = 0;757 let sccb_value = 0;758 let IsAtomicRet = 1;759 let Constraints = "$vdata = $vdata_in";760}761 762multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName,763 RegisterOperand vdataClass,764 ValueType vdataType> {765 let FPAtomic = vdataType.isFP in {766 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 0>,767 MUBUFAddr64Table <0, NAME>;768 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 0>,769 MUBUFAddr64Table <1, NAME>;770 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, 0>;771 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, 0>;772 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, 0>;773 774 def _VBUFFER_OFFSET : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.Offset, vdataClass, 1>,775 MUBUFAddr64Table <0, NAME # "_VBUFFER">;776 def _VBUFFER_ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.Addr64, vdataClass, 1>,777 MUBUFAddr64Table <1, NAME # "_VBUFFER">;778 def _VBUFFER_OFFEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.OffEn, vdataClass, 1>;779 def _VBUFFER_IDXEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.IdxEn, vdataClass, 1>;780 def _VBUFFER_BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.BothEn, vdataClass, 1>;781 }782}783 784multiclass MUBUF_Pseudo_Atomics_RTN <string opName,785 RegisterOperand vdataClass,786 ValueType vdataType,787 SDPatternOperator atomic> {788 let FPAtomic = vdataType.isFP in {789 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 0,790 [(set vdataType:$vdata,791 (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset),792 vdataType:$vdata_in))]>,793 MUBUFAddr64Table <0, NAME # "_RTN">;794 795 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 0,796 [(set vdataType:$vdata,797 (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset),798 vdataType:$vdata_in))]>,799 MUBUFAddr64Table <1, NAME # "_RTN">;800 801 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, 0>;802 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, 0>;803 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, 0>;804 805 def _VBUFFER_OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.Offset, vdataClass, 1,806 [(set vdataType:$vdata,807 (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset),808 vdataType:$vdata_in))]>,809 MUBUFAddr64Table <0, NAME # "_VBUFFER_RTN">;810 811 def _VBUFFER_ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.Addr64, vdataClass, 1,812 [(set vdataType:$vdata,813 (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset),814 vdataType:$vdata_in))]>,815 MUBUFAddr64Table <1, NAME # "_VBUFFER_RTN">;816 817 def _VBUFFER_OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.OffEn, vdataClass, 1>;818 def _VBUFFER_IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.IdxEn, vdataClass, 1>;819 def _VBUFFER_BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.BothEn, vdataClass, 1>;820 }821}822 823multiclass MUBUF_Pseudo_Atomics <string opName,824 RegisterOperand vdataClass,825 ValueType vdataType,826 SDPatternOperator atomic = null_frag> :827 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType>,828 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>;829 830 831//===----------------------------------------------------------------------===//832// MUBUF Instructions833//===----------------------------------------------------------------------===//834 835let OtherPredicates = [HasFormattedMUBUFInsts] in836defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds <837 "buffer_load_format_x", f32838>;839 840let SubtargetPredicate = HasFormattedMUBUFInsts in {841defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads <842 "buffer_load_format_xy", v2f32843>;844defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads <845 "buffer_load_format_xyz", v3f32846>;847defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads <848 "buffer_load_format_xyzw", v4f32849>;850defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores <851 "buffer_store_format_x", f32852>;853defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores <854 "buffer_store_format_xy", v2f32855>;856defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores <857 "buffer_store_format_xyz", v3f32858>;859defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores <860 "buffer_store_format_xyzw", v4f32861>;862 863let OtherPredicates = [HasUnpackedD16VMem], D16Buf = 1 in {864let TiedSourceNotRead = 1 in {865 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads <866 "buffer_load_format_d16_x", i32867 >;868 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads <869 "buffer_load_format_d16_xy", v2i32870 >;871 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads <872 "buffer_load_format_d16_xyz", v3i32873 >;874 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads <875 "buffer_load_format_d16_xyzw", v4i32876 >;877}878 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores <879 "buffer_store_format_d16_x", i32880 >;881 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores <882 "buffer_store_format_d16_xy", v2i32883 >;884 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores <885 "buffer_store_format_d16_xyz", v3i32886 >;887 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores <888 "buffer_store_format_d16_xyzw", v4i32889 >;890} // End OtherPredicates = [HasUnpackedD16VMem], D16Buf = 1.891 892let OtherPredicates = [HasPackedD16VMem], D16Buf = 1 in {893let TiedSourceNotRead = 1 in {894 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads <895 "buffer_load_format_d16_x", f16896 >;897 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads <898 "buffer_load_format_d16_xy", v2f16899 >;900 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads <901 "buffer_load_format_d16_xyz", v3f16902 >;903 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads <904 "buffer_load_format_d16_xyzw", v4f16905 >;906}907 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores <908 "buffer_store_format_d16_x", f16909 >;910 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores <911 "buffer_store_format_d16_xy", v2f16912 >;913 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores <914 "buffer_store_format_d16_xyz", v3f16915 >;916 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores <917 "buffer_store_format_d16_xyzw", v4f16918 >;919} // End OtherPredicates = [HasPackedD16VMem], D16Buf = 1.920} // End SubtargetPredicate = HasFormattedMUBUFInsts.921 922defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds <923 "buffer_load_ubyte", i32924>;925defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds <926 "buffer_load_sbyte", i32927>;928defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds <929 "buffer_load_ushort", i32930>;931defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds <932 "buffer_load_sshort", i32933>;934defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds <935 "buffer_load_dword", i32936>;937defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads <938 "buffer_load_dwordx2", v2i32939>;940defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads_Lds <941 "buffer_load_dwordx3", v3i32, /*LDSPred=*/HasGFX950Insts942>;943defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads_Lds <944 "buffer_load_dwordx4", v4i32, /*LDSPred=*/HasGFX950Insts945>;946 947defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, atomic_load_aext_8_global>;948defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, atomic_load_zext_8_global>;949defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, atomic_load_aext_16_global>;950defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, atomic_load_zext_16_global>;951defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i16, atomic_load_aext_8_global>;952defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i16, atomic_load_zext_8_global>;953defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i16, atomic_load_nonext_16_global>;954defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>;955defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>;956defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>;957defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, atomic_load_sext_8_global>;958defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, atomic_load_sext_16_global>;959defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>;960defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>;961defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>;962defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, atomic_load_sext_16_global>;963 964foreach vt = Reg32Types.types in {965defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", vt, load_global>;966}967 968foreach vt = VReg_64.RegTypes in {969defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", vt, load_global>;970}971 972foreach vt = VReg_96.RegTypes in {973defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", vt, load_global>;974}975 976foreach vt = VReg_128.RegTypes in {977defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", vt, load_global>;978}979 980defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores <981 "buffer_store_byte", i32982>;983defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores <984 "buffer_store_short", i32985>;986defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores <987 "buffer_store_dword", i32988>;989defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores <990 "buffer_store_dwordx2", v2i32991>;992defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores <993 "buffer_store_dwordx3", v3i32994>;995defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores <996 "buffer_store_dwordx4", v4i32997>;998 999defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_BYTE", i32, truncstorei8_global>;1000defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_SHORT", i32, truncstorei16_global>;1001 1002foreach vt = Reg32Types.types in {1003defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORD", vt, store_global>;1004}1005 1006foreach vt = VReg_64.RegTypes in {1007defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX2", vt, store_global>;1008}1009 1010foreach vt = VReg_96.RegTypes in {1011defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX3", vt, store_global>;1012}1013 1014foreach vt = VReg_128.RegTypes in {1015defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX4", vt, store_global>;1016}1017 1018defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics <1019 "buffer_atomic_swap", AVLdSt_32, i321020>;1021defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics <1022 "buffer_atomic_cmpswap", AVLdSt_64, v2i321023>;1024defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics <1025 "buffer_atomic_add", AVLdSt_32, i321026>;1027defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics <1028 "buffer_atomic_sub", AVLdSt_32, i321029>;1030defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics <1031 "buffer_atomic_smin", AVLdSt_32, i321032>;1033defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics <1034 "buffer_atomic_umin", AVLdSt_32, i321035>;1036defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics <1037 "buffer_atomic_smax", AVLdSt_32, i321038>;1039defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics <1040 "buffer_atomic_umax", AVLdSt_32, i321041>;1042defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics <1043 "buffer_atomic_and", AVLdSt_32, i321044>;1045defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics <1046 "buffer_atomic_or", AVLdSt_32, i321047>;1048defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics <1049 "buffer_atomic_xor", AVLdSt_32, i321050>;1051defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics <1052 "buffer_atomic_inc", AVLdSt_32, i321053>;1054defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics <1055 "buffer_atomic_dec", AVLdSt_32, i321056>;1057defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics <1058 "buffer_atomic_swap_x2", AVLdSt_64, i641059>;1060defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics <1061 "buffer_atomic_cmpswap_x2", AVLdSt_128, v2i641062>;1063defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics <1064 "buffer_atomic_add_x2", AVLdSt_64, i641065>;1066defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics <1067 "buffer_atomic_sub_x2", AVLdSt_64, i641068>;1069defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics <1070 "buffer_atomic_smin_x2", AVLdSt_64, i641071>;1072defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics <1073 "buffer_atomic_umin_x2", AVLdSt_64, i641074>;1075defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics <1076 "buffer_atomic_smax_x2", AVLdSt_64, i641077>;1078defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics <1079 "buffer_atomic_umax_x2", AVLdSt_64, i641080>;1081defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics <1082 "buffer_atomic_and_x2", AVLdSt_64, i641083>;1084defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics <1085 "buffer_atomic_or_x2", AVLdSt_64, i641086>;1087defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics <1088 "buffer_atomic_xor_x2", AVLdSt_64, i641089>;1090defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics <1091 "buffer_atomic_inc_x2", AVLdSt_64, i641092>;1093defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics <1094 "buffer_atomic_dec_x2", AVLdSt_64, i641095>;1096 1097let OtherPredicates = [HasGFX10_BEncoding] in {1098 defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics <1099 "buffer_atomic_csub", VGPROp_32, i32, int_amdgcn_global_atomic_csub1100 >;1101}1102 1103let SubtargetPredicate = isGFX8GFX9NotGFX940 in {1104def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">;1105}1106 1107let SubtargetPredicate = isGFX6 in { // isn't on CI & VI1108/*1109defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">;1110defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">;1111*/1112 1113def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc",1114 int_amdgcn_buffer_wbinvl1_sc>;1115}1116 1117let SubtargetPredicate = isGFX6GFX7GFX10Plus in {1118 1119defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics <1120 "buffer_atomic_fcmpswap", AVLdSt_64, v2f32, null_frag1121>;1122}1123 1124let SubtargetPredicate = HasAtomicFMinFMaxF32GlobalInsts in {1125defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics <1126 "buffer_atomic_fmin", AVLdSt_32, f32, null_frag1127>;1128defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics <1129 "buffer_atomic_fmax", AVLdSt_32, f32, null_frag1130>;1131}1132 1133let SubtargetPredicate = isGFX6GFX7GFX10 in {1134defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <1135 "buffer_atomic_fcmpswap_x2", VGPROp_128, v2f64, null_frag1136>;1137}1138 1139let SubtargetPredicate = HasD16LoadStore in {1140let TiedSourceNotRead = 1 in {1141 1142defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads <1143 "buffer_load_ubyte_d16", i32, 11144>;1145 1146defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads <1147 "buffer_load_ubyte_d16_hi", i32, 11148>;1149 1150defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads <1151 "buffer_load_sbyte_d16", i32, 11152>;1153 1154defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads <1155 "buffer_load_sbyte_d16_hi", i32, 11156>;1157 1158defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads <1159 "buffer_load_short_d16", i32, 11160>;1161 1162defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads <1163 "buffer_load_short_d16_hi", i32, 11164>;1165 1166let OtherPredicates = [HasFormattedMUBUFInsts] in1167defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads <1168 "buffer_load_format_d16_hi_x", i321169>;1170} // End TiedSourceNotRead1171 1172defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores <1173 "buffer_store_byte_d16_hi", i321174>;1175 1176defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores <1177 "buffer_store_short_d16_hi", i321178>;1179 1180let OtherPredicates = [HasFormattedMUBUFInsts] in1181defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores <1182 "buffer_store_format_d16_hi_x", i321183>;1184 1185} // End HasD16LoadStore1186 1187let SubtargetPredicate = isNotGFX940Plus in1188def BUFFER_WBINVL1 : MUBUF_Invalidate <1189 "buffer_wbinvl1", int_amdgcn_buffer_wbinvl11190>;1191 1192let SubtargetPredicate = HasAtomicFaddNoRtnInsts in1193defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN<1194 "buffer_atomic_add_f32", AVLdSt_32, f321195>;1196 1197let SubtargetPredicate = HasAtomicBufferGlobalPkAddF16NoRtnInsts in1198defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN <1199 "buffer_atomic_pk_add_f16", AVLdSt_32, v2f161200>;1201 1202let SubtargetPredicate = HasAtomicFaddRtnInsts in1203defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_RTN<1204 "buffer_atomic_add_f32", AVLdSt_32, f32, null_frag1205>;1206 1207let SubtargetPredicate = HasAtomicBufferGlobalPkAddF16Insts in1208defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_RTN <1209 "buffer_atomic_pk_add_f16", AVLdSt_32, v2f16, null_frag1210>;1211 1212let SubtargetPredicate = isGFX12Plus in {1213defm BUFFER_ATOMIC_COND_SUB_U32 : MUBUF_Pseudo_Atomics <1214 "buffer_atomic_cond_sub_u32", VGPROp_32, i321215>;1216}1217 1218let SubtargetPredicate = HasAtomicBufferPkAddBF16Inst in {1219let FPAtomic = 1 in1220defm BUFFER_ATOMIC_PK_ADD_BF16 : MUBUF_Pseudo_Atomics <1221 "buffer_atomic_pk_add_bf16", AVLdSt_32, v2bf161222>;1223}1224 1225//===----------------------------------------------------------------------===//1226// MTBUF Instructions1227//===----------------------------------------------------------------------===//1228let OtherPredicates = [HasMTBUFInsts] in {1229defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", AVLdSt_32, 1>;1230defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", AVLdSt_64, 2>;1231defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", AVLdSt_96, 3>;1232defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", AVLdSt_128, 4>;1233defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", AVLdSt_32, 1>;1234defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", AVLdSt_64, 2>;1235defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", AVLdSt_96, 3>;1236defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", AVLdSt_128, 4>;1237 1238let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in {1239let TiedSourceNotRead = 1 in {1240 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", AVLdSt_32, 1>;1241 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", AVLdSt_64, 2>;1242 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", AVLdSt_96, 3>;1243 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", AVLdSt_128, 4>;1244}1245 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", AVLdSt_32, 1>;1246 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", AVLdSt_64, 2>;1247 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", AVLdSt_96, 3>;1248 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", AVLdSt_128, 4>;1249} // End HasUnpackedD16VMem.1250 1251let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in {1252let TiedSourceNotRead = 1 in {1253 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", AVLdSt_32, 1>;1254 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", AVLdSt_32, 2>;1255 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", AVLdSt_64, 3>;1256 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", AVLdSt_64, 4>;1257}1258 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", AVLdSt_32, 1>;1259 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", AVLdSt_32, 2>;1260 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", AVLdSt_64, 3>;1261 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", AVLdSt_64, 4>;1262} // End HasPackedD16VMem.1263} // End HasMTBUFInsts.1264 1265let SubtargetPredicate = isGFX7Plus in {1266 1267//===----------------------------------------------------------------------===//1268// Instruction definitions for CI and newer.1269//===----------------------------------------------------------------------===//1270 1271let SubtargetPredicate = isNotGFX940Plus in1272def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol",1273 int_amdgcn_buffer_wbinvl1_vol>;1274 1275} // End let SubtargetPredicate = isGFX7Plus1276 1277let SubtargetPredicate = isGFX90APlus in {1278 def BUFFER_WBL2 : MUBUF_Invalidate<"buffer_wbl2"> {1279 let has_glc = 1;1280 let has_sccb = 1;1281 let InOperandList = (ins CPol_0:$cpol);1282 let AsmOperands = "$cpol";1283 }1284 def BUFFER_INVL2 : MUBUF_Invalidate<"buffer_invl2"> {1285 let SubtargetPredicate = isGFX90AOnly;1286 }1287} // End SubtargetPredicate = isGFX90APlus1288 1289let SubtargetPredicate = HasFlatBufferGlobalAtomicFaddF64Inst in {1290 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_add_f64", AVLdSt_64, f64>;1291} // End SubtargetPredicate = HasFlatBufferGlobalAtomicFaddF64Inst1292 1293let SubtargetPredicate = HasAtomicFMinFMaxF64GlobalInsts in {1294 // Note the names can be buffer_atomic_fmin_x2/buffer_atomic_fmax_x21295 // depending on some subtargets.1296 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_min_f64", AVLdSt_64, f64>;1297 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_max_f64", AVLdSt_64, f64>;1298}1299 1300def BUFFER_INV : MUBUF_Invalidate<"buffer_inv"> {1301 let SubtargetPredicate = isGFX940Plus;1302 let has_glc = 1;1303 let has_sccb = 1;1304 let InOperandList = (ins CPol_0:$cpol);1305 let AsmOperands = "$cpol";1306}1307 1308def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">;1309def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">;1310 1311//===----------------------------------------------------------------------===//1312// MUBUF Patterns1313//===----------------------------------------------------------------------===//1314 1315//===----------------------------------------------------------------------===//1316// buffer_load/store_format patterns1317//===----------------------------------------------------------------------===//1318 1319multiclass MUBUF_LoadIntrinsicPat_Common<SDPatternOperator name, ValueType vt,1320 string opcode, ValueType memoryVt = vt> {1321 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_load<name, memoryVt>);1322 1323 def : GCNPat<1324 (vt (st v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset,1325 timm:$auxiliary, 0)),1326 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,1327 (extract_cpol $auxiliary), (extract_swz $auxiliary))1328 >;1329 1330 def : GCNPat<1331 (vt (st v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset,1332 timm:$auxiliary, 0)),1333 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,1334 (extract_cpol $auxiliary), (extract_swz $auxiliary))1335 >;1336 1337 def : GCNPat<1338 (vt (st v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset,1339 timm:$auxiliary, timm)),1340 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,1341 (extract_cpol $auxiliary), (extract_swz $auxiliary))1342 >;1343 1344 def : GCNPat<1345 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset,1346 timm:$auxiliary, timm)),1347 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)1348 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),1349 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,1350 (extract_cpol $auxiliary), (extract_swz $auxiliary))1351 >;1352}1353 1354multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,1355 string opcode, ValueType memoryVt = vt>{1356 let SubtargetPredicate = HasUnrestrictedSOffset in {1357 defm : MUBUF_LoadIntrinsicPat_Common<name, vt, opcode, memoryVt>;1358 }1359 defm : MUBUF_LoadIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>;1360}1361 1362let OtherPredicates = [HasFormattedMUBUFInsts] in {1363defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">;1364defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">;1365defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">;1366defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">;1367defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">;1368defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">;1369defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">;1370defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">;1371 1372defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v2i32, "BUFFER_LOAD_FORMAT_X_TFE">;1373defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v3i32, "BUFFER_LOAD_FORMAT_XY_TFE">;1374defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v4i32, "BUFFER_LOAD_FORMAT_XYZ_TFE">;1375defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v5i32, "BUFFER_LOAD_FORMAT_XYZW_TFE">;1376} // End OtherPredicates = [HasFormattedMUBUFInsts].1377 1378let OtherPredicates = [HasUnpackedD16VMem, HasFormattedMUBUFInsts] in {1379 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;1380 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;1381 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;1382 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">;1383 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80">;1384 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;1385} // End OtherPredicates = [HasUnpackedD16VMem, HasFormattedMUBUFInsts].1386 1387let OtherPredicates = [HasPackedD16VMem, HasFormattedMUBUFInsts] in {1388 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">;1389 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">;1390 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">;1391 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">;1392 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">;1393 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3f16>;1394 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3i16>;1395 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">;1396 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">;1397} // End OtherPredicates = [HasPackedD16VMem, HasFormattedMUBUFInsts].1398 1399foreach vt = Reg32Types.types in {1400defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, vt, "BUFFER_LOAD_DWORD">;1401}1402 1403foreach vt = Reg64Types.types in {1404defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, vt, "BUFFER_LOAD_DWORDX2">;1405}1406 1407foreach vt = Reg96Types.types in {1408defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, vt, "BUFFER_LOAD_DWORDX3">;1409}1410 1411foreach vt = Reg128Types.types in {1412defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, vt, "BUFFER_LOAD_DWORDX4">;1413}1414 1415defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">;1416defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">;1417defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">;1418defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">;1419 1420defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_tfe, v2i32, "BUFFER_LOAD_DWORD_TFE">;1421defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_tfe, v3i32, "BUFFER_LOAD_DWORDX2_TFE">;1422defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_tfe, v4i32, "BUFFER_LOAD_DWORDX3_TFE">;1423defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_tfe, v5i32, "BUFFER_LOAD_DWORDX4_TFE">;1424defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte_tfe, v2i32, "BUFFER_LOAD_SBYTE_TFE">;1425defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short_tfe, v2i32, "BUFFER_LOAD_SSHORT_TFE">;1426defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte_tfe, v2i32, "BUFFER_LOAD_UBYTE_TFE">;1427defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort_tfe, v2i32, "BUFFER_LOAD_USHORT_TFE">;1428 1429multiclass MUBUF_StoreIntrinsicPat_Common<SDPatternOperator name, ValueType vt,1430 string opcode, ValueType memoryVt = vt> {1431 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_store<name, memoryVt>);1432 1433 def : GCNPat<1434 (st vt:$vdata, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset,1435 timm:$auxiliary, 0),1436 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,1437 (extract_cpol $auxiliary), (extract_swz $auxiliary))1438 >;1439 1440 def : GCNPat<1441 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset,1442 timm:$auxiliary, 0),1443 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset,1444 timm:$offset, (extract_cpol $auxiliary), (extract_swz $auxiliary))1445 >;1446 1447 def : GCNPat<1448 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset,1449 timm:$auxiliary, timm),1450 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset,1451 timm:$offset, (extract_cpol $auxiliary), (extract_swz $auxiliary))1452 >;1453 1454 def : GCNPat<1455 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset,1456 timm:$auxiliary, timm),1457 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact)1458 getVregSrcForVT<vt>.ret:$vdata,1459 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),1460 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, (extract_cpol $auxiliary),1461 (extract_swz $auxiliary))1462 >;1463}1464 1465multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,1466 string opcode, ValueType memoryVt = vt> {1467 let SubtargetPredicate = HasUnrestrictedSOffset in {1468 defm : MUBUF_StoreIntrinsicPat_Common<name, vt, opcode, memoryVt>;1469 }1470 defm : MUBUF_StoreIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>;1471}1472 1473let OtherPredicates = [HasFormattedMUBUFInsts] in {1474defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">;1475defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">;1476defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">;1477defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">;1478defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">;1479defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">;1480defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">;1481defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">;1482} // End OtherPredicates = [HasFormattedMUBUFInsts].1483 1484let OtherPredicates = [HasUnpackedD16VMem, HasFormattedMUBUFInsts] in {1485 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">;1486 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">;1487 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">;1488 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">;1489 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, v3i32, "BUFFER_STORE_FORMAT_D16_XYZ_gfx80">;1490 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">;1491} // End OtherPredicates = [HasUnpackedD16VMem, HasFormattedMUBUFInsts].1492 1493let OtherPredicates = [HasPackedD16VMem, HasFormattedMUBUFInsts] in {1494 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">;1495 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">;1496 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">;1497 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">;1498 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">;1499 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZ", v3f16>;1500 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZ", v3i16>;1501 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">;1502 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">;1503} // End OtherPredicates = [HasPackedD16VMem, HasFormattedMUBUFInsts].1504 1505foreach vt = Reg32Types.types in {1506defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, vt, "BUFFER_STORE_DWORD">;1507}1508 1509foreach vt = Reg64Types.types in {1510defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, vt, "BUFFER_STORE_DWORDX2">;1511}1512 1513foreach vt = Reg96Types.types in {1514defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, vt, "BUFFER_STORE_DWORDX3">;1515}1516 1517foreach vt = Reg128Types.types in {1518defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, vt, "BUFFER_STORE_DWORDX4">;1519}1520 1521defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">;1522defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">;1523 1524//===----------------------------------------------------------------------===//1525// buffer_atomic patterns1526//===----------------------------------------------------------------------===//1527 1528multiclass BufferAtomicPat_Common<string OpPrefix, ValueType vt, string Inst, bit isIntr = 0> {1529 foreach RtnMode = ["ret", "noret"] in {1530 1531 defvar Op = !cast<SDPatternOperator>(OpPrefix1532 # !if(!eq(RtnMode, "ret"), "", "_noret")1533 # !if(isIntr, "", "_" # vt));1534 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", "");1535 1536 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in {1537 def : GCNPat<1538 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), vt:$vdata_in)),1539 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in,1540 SReg_128:$srsrc, SCSrc_b32:$soffset, Offset:$offset)1541 >;1542 1543 def : GCNPat<1544 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset),1545 vt:$vdata_in)),1546 (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in,1547 VReg_64:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, Offset:$offset)1548 >;1549 } // end let AddedComplexity1550 1551 } // end foreach RtnMode1552}1553 1554multiclass BufferAtomicPat<string OpPrefix, ValueType vt, string Inst, bit isIntr = 0> {1555 let SubtargetPredicate = HasUnrestrictedSOffset in {1556 defm : BufferAtomicPat_Common<OpPrefix, vt, Inst, isIntr>;1557 }1558 defm : BufferAtomicPat_Common<OpPrefix, vt, Inst # "_VBUFFER", isIntr>;1559}1560 1561multiclass BufferAtomicIntrPat<string OpPrefix, ValueType vt, string Inst> :1562 BufferAtomicPat<OpPrefix, vt, Inst, /* isIntr */ 1>;1563 1564multiclass BufferAtomicCmpSwapPat_Common<ValueType vt, ValueType data_vt, string Inst> {1565 foreach RtnMode = ["ret", "noret"] in {1566 1567 defvar Op = !cast<SDPatternOperator>("AMDGPUatomic_cmp_swap_global"1568 # !if(!eq(RtnMode, "ret"), "", "_noret")1569 # "_" # vt);1570 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", "");1571 defvar data_op = getVregSrcForVT<data_vt>.ret;1572 defvar data_vt_RC = getVregClassForVT<data_vt>.ret;1573 1574 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in {1575 defvar OffsetResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix)1576 data_op:$vdata_in, SReg_128:$srsrc, SCSrc_b32:$soffset,1577 Offset:$offset);1578 def : GCNPat<1579 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), data_vt:$vdata_in)),1580 !if(!eq(RtnMode, "ret"),1581 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS OffsetResDag, data_vt_RC)),1582 !if(!eq(vt, i32), sub0, sub0_sub1)),1583 OffsetResDag)1584 >;1585 1586 defvar Addr64ResDag = (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix)1587 data_op:$vdata_in, VReg_64:$vaddr, SReg_128:$srsrc,1588 SCSrc_b32:$soffset, Offset:$offset);1589 def : GCNPat<1590 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset),1591 data_vt:$vdata_in)),1592 !if(!eq(RtnMode, "ret"),1593 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS Addr64ResDag, data_vt_RC)),1594 !if(!eq(vt, i32), sub0, sub0_sub1)),1595 Addr64ResDag)1596 >;1597 } // end let AddedComplexity1598 1599 } // end foreach RtnMode1600}1601 1602multiclass BufferAtomicCmpSwapPat<ValueType vt, ValueType data_vt, string Inst> {1603 let SubtargetPredicate = HasUnrestrictedSOffset in {1604 defm : BufferAtomicCmpSwapPat_Common<vt, data_vt, Inst>;1605 }1606 defm : BufferAtomicCmpSwapPat_Common<vt, data_vt, Inst # "_VBUFFER">;1607}1608 1609 1610foreach Ty = [i32, i64] in {1611 1612defvar Suffix = !if(!eq(Ty, i64), "_X2", "");1613 1614defm : BufferAtomicPat<"atomic_swap_global", Ty, "BUFFER_ATOMIC_SWAP" # Suffix>;1615defm : BufferAtomicPat<"atomic_load_add_global", Ty, "BUFFER_ATOMIC_ADD" # Suffix>;1616defm : BufferAtomicPat<"atomic_load_sub_global", Ty, "BUFFER_ATOMIC_SUB" # Suffix>;1617defm : BufferAtomicPat<"atomic_load_min_global", Ty, "BUFFER_ATOMIC_SMIN" # Suffix>;1618defm : BufferAtomicPat<"atomic_load_umin_global", Ty, "BUFFER_ATOMIC_UMIN" # Suffix>;1619defm : BufferAtomicPat<"atomic_load_max_global", Ty, "BUFFER_ATOMIC_SMAX" # Suffix>;1620defm : BufferAtomicPat<"atomic_load_umax_global", Ty, "BUFFER_ATOMIC_UMAX" # Suffix>;1621defm : BufferAtomicPat<"atomic_load_and_global", Ty, "BUFFER_ATOMIC_AND" # Suffix>;1622defm : BufferAtomicPat<"atomic_load_or_global", Ty, "BUFFER_ATOMIC_OR" # Suffix>;1623defm : BufferAtomicPat<"atomic_load_xor_global", Ty, "BUFFER_ATOMIC_XOR" # Suffix>;1624defm : BufferAtomicPat<"atomic_load_uinc_wrap_global", Ty, "BUFFER_ATOMIC_INC" # Suffix>;1625defm : BufferAtomicPat<"atomic_load_udec_wrap_global", Ty, "BUFFER_ATOMIC_DEC" # Suffix>;1626 1627} // end foreach Ty1628 1629let SubtargetPredicate = HasAtomicFMinFMaxF32GlobalInsts in {1630defm : BufferAtomicPat<"atomic_load_fmin_global", f32, "BUFFER_ATOMIC_FMIN">;1631defm : BufferAtomicPat<"atomic_load_fmax_global", f32, "BUFFER_ATOMIC_FMAX">;1632}1633 1634let SubtargetPredicate = HasAtomicFMinFMaxF64GlobalInsts in {1635defm : BufferAtomicPat<"atomic_load_fmin_global", f64, "BUFFER_ATOMIC_MIN_F64">;1636defm : BufferAtomicPat<"atomic_load_fmax_global", f64, "BUFFER_ATOMIC_MAX_F64">;1637}1638 1639defm : BufferAtomicCmpSwapPat<i32, v2i32, "BUFFER_ATOMIC_CMPSWAP">;1640defm : BufferAtomicCmpSwapPat<i64, v2i64, "BUFFER_ATOMIC_CMPSWAP_X2">;1641 1642multiclass SIBufferAtomicPat_Common<string OpPrefix, ValueType vt, string Inst,1643 list<string> RtnModes = ["ret", "noret"]> {1644 foreach RtnMode = RtnModes in {1645 1646 defvar Op = !cast<SDPatternOperator>(OpPrefix1647 # !if(!eq(RtnMode, "ret"), "", "_noret"));1648 1649 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", "");1650 defvar CachePolicy = !if(!eq(RtnMode, "ret"),1651 (extract_cpol_set_glc $auxiliary), (extract_cpol $auxiliary));1652 1653 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in {1654 def : GCNPat<1655 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset),1656 timm:$offset, timm:$auxiliary, 0)),1657 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix)1658 getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset,1659 timm:$offset, CachePolicy)1660 >;1661 1662 def : GCNPat<1663 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset),1664 timm:$offset, timm:$auxiliary, timm)),1665 (!cast<MUBUF_Pseudo>(Inst # "_IDXEN" # InstSuffix)1666 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc,1667 SCSrc_b32:$soffset, timm:$offset, CachePolicy)1668 >;1669 1670 def : GCNPat<1671 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, i32:$voffset,1672 (BUFSOffset i32:$soffset), timm:$offset, timm:$auxiliary, 0)),1673 (!cast<MUBUF_Pseudo>(Inst # "_OFFEN" # InstSuffix)1674 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc,1675 SCSrc_b32:$soffset, timm:$offset, CachePolicy)1676 >;1677 1678 def : GCNPat<1679 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, i32:$voffset,1680 (BUFSOffset i32:$soffset), timm:$offset, timm:$auxiliary, timm)),1681 (!cast<MUBUF_Pseudo>(Inst # "_BOTHEN" # InstSuffix)1682 getVregSrcForVT<vt>.ret:$vdata_in,1683 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),1684 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, CachePolicy)1685 >;1686 } // end let AddedComplexity1687 1688 } // end foreach RtnMode1689}1690 1691multiclass SIBufferAtomicPat<string OpPrefix, ValueType vt, string Inst,1692 list<string> RtnModes = ["ret", "noret"]> {1693 let OtherPredicates = [HasUnrestrictedSOffset] in {1694 defm : SIBufferAtomicPat_Common<OpPrefix, vt, Inst, RtnModes>;1695 }1696 1697 // FIXME: This needs a !HasUnrestrictedSOffset predicate1698 defm : SIBufferAtomicPat_Common<OpPrefix, vt, Inst # "_VBUFFER", RtnModes>;1699}1700 1701defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i32, "BUFFER_ATOMIC_SWAP">;1702defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", f32, "BUFFER_ATOMIC_SWAP">;1703defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i32, "BUFFER_ATOMIC_ADD">;1704defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i32, "BUFFER_ATOMIC_SUB">;1705defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i32, "BUFFER_ATOMIC_SMIN">;1706defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i32, "BUFFER_ATOMIC_UMIN">;1707defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i32, "BUFFER_ATOMIC_SMAX">;1708defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i32, "BUFFER_ATOMIC_UMAX">;1709defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i32, "BUFFER_ATOMIC_AND">;1710defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i32, "BUFFER_ATOMIC_OR">;1711defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i32, "BUFFER_ATOMIC_XOR">;1712defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i32, "BUFFER_ATOMIC_INC">;1713defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i32, "BUFFER_ATOMIC_DEC">;1714defm : SIBufferAtomicPat<"SIbuffer_atomic_csub", i32, "BUFFER_ATOMIC_CSUB", ["ret"]>;1715defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i64, "BUFFER_ATOMIC_SWAP_X2">;1716defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i64, "BUFFER_ATOMIC_ADD_X2">;1717defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i64, "BUFFER_ATOMIC_SUB_X2">;1718defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i64, "BUFFER_ATOMIC_SMIN_X2">;1719defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i64, "BUFFER_ATOMIC_UMIN_X2">;1720defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i64, "BUFFER_ATOMIC_SMAX_X2">;1721defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i64, "BUFFER_ATOMIC_UMAX_X2">;1722defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i64, "BUFFER_ATOMIC_AND_X2">;1723defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i64, "BUFFER_ATOMIC_OR_X2">;1724defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i64, "BUFFER_ATOMIC_XOR_X2">;1725defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i64, "BUFFER_ATOMIC_INC_X2">;1726defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i64, "BUFFER_ATOMIC_DEC_X2">;1727 1728let SubtargetPredicate = HasAtomicCSubNoRtnInsts in1729defm : SIBufferAtomicPat<"SIbuffer_atomic_csub", i32, "BUFFER_ATOMIC_CSUB", ["noret"]>;1730 1731let SubtargetPredicate = HasAtomicBufferPkAddBF16Inst in {1732 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", v2bf16, "BUFFER_ATOMIC_PK_ADD_BF16">;1733}1734 1735let SubtargetPredicate = isGFX12Plus in {1736 defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_cond_sub_u32", i32, "BUFFER_ATOMIC_COND_SUB_U32_VBUFFER", ["ret"]>;1737}1738 1739let SubtargetPredicate = HasAtomicCSubNoRtnInsts in {1740defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_cond_sub_u32", i32, "BUFFER_ATOMIC_COND_SUB_U32_VBUFFER", ["noret"]>;1741}1742 1743let SubtargetPredicate = HasAtomicFMinFMaxF32GlobalInsts in {1744 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f32, "BUFFER_ATOMIC_FMIN">;1745 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f32, "BUFFER_ATOMIC_FMAX">;1746}1747 1748let SubtargetPredicate = HasAtomicFMinFMaxF64GlobalInsts in {1749 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_MIN_F64">;1750 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_MAX_F64">;1751}1752 1753class NoUseBufferAtomic<SDPatternOperator Op, ValueType vt> : PatFrag <1754 (ops node:$src0, node:$src1, node:$src2, node:$src3, node:$src4, node:$src5, node:$src6, node:$src7),1755 (vt (Op $src0, $src1, $src2, $src3, $src4, $src5, $src6, $src7))> {1756 let HasNoUse = true;1757}1758 1759multiclass BufferAtomicPatterns_NO_RTN_Common<SDPatternOperator name, ValueType vt,1760 string opcode> {1761 def : GCNPat<1762 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0,1763 0, (BUFSOffset i32:$soffset), timm:$offset,1764 timm:$auxiliary, 0),1765 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset,1766 timm:$offset, (extract_cpol $auxiliary))1767 >;1768 1769 def : GCNPat<1770 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex,1771 0, (BUFSOffset i32:$soffset), timm:$offset,1772 timm:$auxiliary, timm),1773 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset,1774 timm:$offset, (extract_cpol $auxiliary))1775 >;1776 1777 def : GCNPat<1778 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0,1779 i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset,1780 timm:$auxiliary, 0),1781 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset,1782 timm:$offset, (extract_cpol $auxiliary))1783 >;1784 1785 def : GCNPat<1786 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex,1787 i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset,1788 timm:$auxiliary, timm),1789 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)1790 getVregSrcForVT<vt>.ret:$vdata_in,1791 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),1792 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, (extract_cpol $auxiliary))1793 >;1794}1795 1796multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt,1797 string opcode> {1798 let SubtargetPredicate = HasUnrestrictedSOffset in {1799 defm : BufferAtomicPatterns_NO_RTN_Common<name, vt, opcode>;1800 }1801 defm : BufferAtomicPatterns_NO_RTN_Common<name, vt, opcode # "_VBUFFER">;1802}1803 1804let SubtargetPredicate = HasAtomicFaddNoRtnInsts in1805 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32", ["noret"]>;1806 1807let SubtargetPredicate = HasAtomicBufferGlobalPkAddF16NoRtnInsts in {1808 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16", ["noret"]>;1809} // End SubtargetPredicate = HasAtomicBufferGlobalPkAddF16NoRtnInsts1810 1811let SubtargetPredicate = HasAtomicFaddRtnInsts in1812 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32", ["ret"]>;1813 1814let SubtargetPredicate = HasAtomicBufferGlobalPkAddF16Insts in {1815 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16", ["ret"]>;1816} // End SubtargetPredicate = HasAtomicBufferGlobalPkAddF16Insts1817 1818let SubtargetPredicate = HasFlatBufferGlobalAtomicFaddF64Inst in {1819 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f64, "BUFFER_ATOMIC_ADD_F64">;1820} // End SubtargetPredicate = HasFlatBufferGlobalAtomicFaddF64Inst1821 1822let SubtargetPredicate = HasAtomicFMinFMaxF64GlobalInsts in {1823 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_MIN_F64">;1824 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_MAX_F64">;1825} //End let SubtargetPredicate = HasAtomicFMinFMaxF64GlobalInsts1826 1827multiclass SIBufferAtomicCmpSwapPat_Common<ValueType vt, ValueType data_vt, string Inst> {1828 foreach RtnMode = ["ret", "noret"] in {1829 defvar Op = !cast<SDPatternOperator>(SIbuffer_atomic_cmpswap1830 # !if(!eq(RtnMode, "ret"), "", "_noret"));1831 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", "");1832 defvar CachePolicy = !if(!eq(RtnMode, "ret"),1833 (extract_cpol_set_glc $auxiliary),1834 (extract_cpol $auxiliary));1835 defvar SrcRC = getVregSrcForVT<vt>.ret;1836 defvar DataRC = getVregClassForVT<data_vt>.ret;1837 defvar SubLo = !if(!eq(vt, i32), sub0, sub0_sub1);1838 defvar SubHi = !if(!eq(vt, i32), sub1, sub2_sub3);1839 1840 defvar OffsetResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix)1841 (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi),1842 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, CachePolicy);1843 def : GCNPat<1844 (vt (Op1845 vt:$data, vt:$cmp, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset),1846 timm:$offset, timm:$auxiliary, 0)),1847 !if(!eq(RtnMode, "ret"),1848 (EXTRACT_SUBREG OffsetResDag, SubLo),1849 OffsetResDag)1850 >;1851 1852 defvar IdxenResDag = (!cast<MUBUF_Pseudo>(Inst # "_IDXEN" # InstSuffix)1853 (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi),1854 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,1855 CachePolicy);1856 def : GCNPat<1857 (vt (Op1858 vt:$data, vt:$cmp, v4i32:$rsrc, i32:$vindex,1859 0, (BUFSOffset i32:$soffset), timm:$offset,1860 timm:$auxiliary, timm)),1861 !if(!eq(RtnMode, "ret"),1862 (EXTRACT_SUBREG IdxenResDag, SubLo),1863 IdxenResDag)1864 >;1865 1866 defvar OffenResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFEN" # InstSuffix)1867 (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi),1868 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,1869 CachePolicy);1870 def : GCNPat<1871 (vt (Op1872 vt:$data, vt:$cmp, v4i32:$rsrc, 0,1873 i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset,1874 timm:$auxiliary, 0)),1875 !if(!eq(RtnMode, "ret"),1876 (EXTRACT_SUBREG OffenResDag, SubLo),1877 OffenResDag)1878 >;1879 1880 defvar BothenResDag = (!cast<MUBUF_Pseudo>(Inst # "_BOTHEN" # InstSuffix)1881 (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi),1882 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),1883 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, CachePolicy);1884 def : GCNPat<1885 (vt (Op1886 vt:$data, vt:$cmp, v4i32:$rsrc, i32:$vindex,1887 i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset,1888 timm:$auxiliary, timm)),1889 !if(!eq(RtnMode, "ret"),1890 (EXTRACT_SUBREG BothenResDag, SubLo),1891 BothenResDag)1892 >;1893 } // end foreach RtnMode1894}1895 1896multiclass SIBufferAtomicCmpSwapPat<ValueType vt, ValueType data_vt, string Inst> {1897 let OtherPredicates = [HasUnrestrictedSOffset] in {1898 defm : SIBufferAtomicCmpSwapPat_Common<vt, data_vt, Inst>;1899 }1900 defm : SIBufferAtomicCmpSwapPat_Common<vt, data_vt, Inst # "_VBUFFER">;1901}1902 1903defm : SIBufferAtomicCmpSwapPat<i32, v2i32, "BUFFER_ATOMIC_CMPSWAP">;1904defm : SIBufferAtomicCmpSwapPat<i64, v2i64, "BUFFER_ATOMIC_CMPSWAP_X2">;1905 1906class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt,1907 PatFrag constant_ld> : GCNPat <1908 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,1909 i32:$offset))),1910 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset)1911 >;1912 1913multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,1914 ValueType vt, PatFrag atomic_ld> {1915 def : GCNPat <1916 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset))),1917 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset)1918 >;1919 1920 def : GCNPat <1921 (vt (atomic_ld (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset))),1922 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset))1923 >;1924}1925 1926let SubtargetPredicate = isGFX6GFX7 in {1927def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>;1928def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>;1929def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>;1930def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>;1931def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>;1932def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>;1933 1934defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_nonext_32_global>;1935defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_nonext_64_global>;1936} // End SubtargetPredicate = isGFX6GFX71937 1938multiclass MUBUFLoad_PatternOffset_Common <string Instr, ValueType vt,1939 PatFrag ld> {1940 def : GCNPat <1941 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset))),1942 (!cast<MUBUF_Pseudo>(Instr # "_OFFSET") $srsrc, $soffset, $offset)1943 >;1944}1945 1946multiclass MUBUFLoad_PatternOffset <string Instr, ValueType vt,1947 PatFrag ld> {1948 let OtherPredicates = [HasUnrestrictedSOffset] in {1949 defm : MUBUFLoad_PatternOffset_Common<Instr, vt, ld>;1950 }1951 defm : MUBUFLoad_PatternOffset_Common<Instr # "_VBUFFER", vt, ld>;1952}1953 1954let OtherPredicates = [Has16BitInsts] in {1955 1956defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_SBYTE", i16, sextloadi8_constant>;1957defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, extloadi8_constant>;1958defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, zextloadi8_constant>;1959defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_SBYTE", i16, sextloadi8_global>;1960defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, extloadi8_global>;1961defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, zextloadi8_global>;1962 1963defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_USHORT", i16, load_global>;1964 1965} // End OtherPredicates = [Has16BitInsts]1966 1967multiclass MUBUFScratchLoadPat_Common <string Instr,1968 ValueType vt, PatFrag ld> {1969 def : GCNPat <1970 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,1971 i32:$soffset, i32:$offset))),1972 (!cast<MUBUF_Pseudo>(Instr # _OFFEN) $vaddr, $srsrc, $soffset, $offset, 0, 0)1973 >;1974 1975 def : GCNPat <1976 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, i32:$offset))),1977 (!cast<MUBUF_Pseudo>(Instr # _OFFSET) $srsrc, $soffset, $offset, 0, 0)1978 >;1979}1980 1981multiclass MUBUFScratchLoadPat <string Instr,1982 ValueType vt, PatFrag ld> {1983 let SubtargetPredicate = HasUnrestrictedSOffset in {1984 defm : MUBUFScratchLoadPat_Common<Instr, vt, ld>;1985 }1986 defm : MUBUFScratchLoadPat_Common<Instr # "_VBUFFER", vt, ld>;1987}1988 1989// XXX - Is it possible to have a complex pattern in a PatFrag?1990multiclass MUBUFScratchLoadPat_D16_Common <string Instr,1991 ValueType vt, PatFrag ld_frag> {1992 def : GCNPat <1993 (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, i32:$offset), vt:$in),1994 (!cast<MUBUF_Pseudo>(Instr # _OFFEN) $vaddr, $srsrc, $soffset, $offset, $in)1995 >;1996 1997 def : GCNPat <1998 (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, i32:$offset), vt:$in),1999 (!cast<MUBUF_Pseudo>(Instr # _OFFSET) $srsrc, $soffset, $offset, $in)2000 >;2001}2002 2003multiclass MUBUFScratchLoadPat_D16 <string Instr,2004 ValueType vt, PatFrag ld_frag> {2005 let SubtargetPredicate = HasUnrestrictedSOffset in {2006 defm : MUBUFScratchLoadPat_D16_Common<Instr, vt, ld_frag>;2007 }2008 defm : MUBUFScratchLoadPat_D16_Common<Instr # "_VBUFFER", vt, ld_frag>;2009}2010 2011let OtherPredicates = [DisableFlatScratch] in {2012defm : MUBUFScratchLoadPat <"BUFFER_LOAD_SBYTE", i32, sextloadi8_private>;2013defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i32, extloadi8_private>;2014defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i32, zextloadi8_private>;2015defm : MUBUFScratchLoadPat <"BUFFER_LOAD_SBYTE", i16, sextloadi8_private>;2016defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i16, extloadi8_private>;2017defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i16, zextloadi8_private>;2018defm : MUBUFScratchLoadPat <"BUFFER_LOAD_SSHORT", i32, sextloadi16_private>;2019defm : MUBUFScratchLoadPat <"BUFFER_LOAD_USHORT", i32, extloadi16_private>;2020defm : MUBUFScratchLoadPat <"BUFFER_LOAD_USHORT", i32, zextloadi16_private>;2021defm : MUBUFScratchLoadPat <"BUFFER_LOAD_USHORT", i16, load_private>;2022 2023foreach vt = Reg32Types.types in {2024defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORD", vt, load_private>;2025}2026defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORDX2", v2i32, load_private>;2027defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORDX3", v3i32, load_private>;2028defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORDX4", v4i32, load_private>;2029 2030let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in {2031defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16_HI", v2i16, load_d16_hi_private>;2032defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16_HI", v2i16, az_extloadi8_d16_hi_private>;2033defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16_HI", v2i16, sextloadi8_d16_hi_private>;2034defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16_HI", v2f16, load_d16_hi_private>;2035defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16_HI", v2f16, az_extloadi8_d16_hi_private>;2036defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16_HI", v2f16, sextloadi8_d16_hi_private>;2037 2038defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16", v2i16, load_d16_lo_private>;2039defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16", v2i16, az_extloadi8_d16_lo_private>;2040defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16", v2i16, sextloadi8_d16_lo_private>;2041defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16", v2f16, load_d16_lo_private>;2042defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16", v2f16, az_extloadi8_d16_lo_private>;2043defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16", v2f16, sextloadi8_d16_lo_private>;2044}2045 2046} // End OtherPredicates = [DisableFlatScratch]2047 2048multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,2049 ValueType vt, PatFrag atomic_st> {2050 def : GCNPat <2051 (atomic_st vt:$val, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset)),2052 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset)2053 >;2054 2055 def : GCNPat <2056 (atomic_st vt:$val, (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset)),2057 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset))2058 >;2059}2060let SubtargetPredicate = isGFX6GFX7 in {2061defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i32, atomic_store_8_global>;2062defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i16, atomic_store_8_global>;2063defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i32, atomic_store_16_global>;2064defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i16, atomic_store_16_global>;2065defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_32_global>;2066defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_64_global>;2067} // End Predicates = isGFX6GFX72068 2069 2070multiclass MUBUFStore_PatternOffset_Common <string Instr, ValueType vt,2071 PatFrag st> {2072 2073 def : GCNPat <2074 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset)),2075 (!cast<MUBUF_Pseudo>(Instr # "_OFFSET") $vdata, $srsrc, $soffset, $offset)2076 >;2077}2078 2079multiclass MUBUFStore_PatternOffset <string Instr, ValueType vt,2080 PatFrag st> {2081 let SubtargetPredicate = HasUnrestrictedSOffset in {2082 defm : MUBUFStore_PatternOffset_Common<Instr, vt, st>;2083 }2084 defm : MUBUFStore_PatternOffset_Common<Instr # "_VBUFFER", vt, st>;2085}2086 2087defm : MUBUFStore_PatternOffset <"BUFFER_STORE_BYTE", i16, truncstorei8_global>;2088defm : MUBUFStore_PatternOffset <"BUFFER_STORE_SHORT", i16, store_global>;2089 2090multiclass MUBUFScratchStorePat_Common <string Instr,2091 ValueType vt, PatFrag st,2092 RegisterClassLike rc = VGPR_32> {2093 def : GCNPat <2094 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,2095 i32:$soffset, i32:$offset)),2096 (!cast<MUBUF_Pseudo>(Instr # _OFFEN) rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0)2097 >;2098 2099 def : GCNPat <2100 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset,2101 i32:$offset)),2102 (!cast<MUBUF_Pseudo>(Instr # _OFFSET) rc:$value, $srsrc, $soffset, $offset, 0, 0)2103 >;2104}2105 2106multiclass MUBUFScratchStorePat <string Instr,2107 ValueType vt, PatFrag st,2108 RegisterClassLike rc = VGPR_32> {2109 let SubtargetPredicate = HasUnrestrictedSOffset in {2110 defm : MUBUFScratchStorePat_Common<Instr, vt, st, rc>;2111 }2112 defm : MUBUFScratchStorePat_Common<Instr # "_VBUFFER", vt, st, rc>;2113}2114 2115let OtherPredicates = [DisableFlatScratch] in {2116defm : MUBUFScratchStorePat <"BUFFER_STORE_BYTE", i32, truncstorei8_private>;2117defm : MUBUFScratchStorePat <"BUFFER_STORE_SHORT", i32, truncstorei16_private>;2118defm : MUBUFScratchStorePat <"BUFFER_STORE_BYTE", i16, truncstorei8_private>;2119defm : MUBUFScratchStorePat <"BUFFER_STORE_SHORT", i16, store_private>;2120 2121foreach vt = Reg32Types.types in {2122defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORD", vt, store_private>;2123}2124 2125defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORDX2", v2i32, store_private, VReg_64>;2126defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORDX3", v3i32, store_private, VReg_96>;2127defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORDX4", v4i32, store_private, VReg_128>;2128 2129 2130let OtherPredicates = [HasD16LoadStore, DisableFlatScratch] in {2131 // Hiding the extract high pattern in the PatFrag seems to not2132 // automatically increase the complexity.2133let AddedComplexity = 1 in {2134defm : MUBUFScratchStorePat <"BUFFER_STORE_SHORT_D16_HI", i32, store_hi16_private>;2135defm : MUBUFScratchStorePat <"BUFFER_STORE_BYTE_D16_HI", i32, truncstorei8_hi16_private>;2136}2137}2138} // End OtherPredicates = [DisableFlatScratch]2139 2140//===----------------------------------------------------------------------===//2141// MTBUF Patterns2142//===----------------------------------------------------------------------===//2143 2144//===----------------------------------------------------------------------===//2145// tbuffer_load/store_format patterns2146//===----------------------------------------------------------------------===//2147 2148multiclass MTBUF_LoadIntrinsicPat_Common<SDPatternOperator name, ValueType vt,2149 string opcode, ValueType memoryVt = vt> {2150 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_load<name, memoryVt>);2151 2152 def : GCNPat<2153 (vt (st v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset,2154 timm:$format, timm:$auxiliary, 0)),2155 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,2156 (as_i8timm $format),2157 (extract_cpol $auxiliary), (extract_swz $auxiliary))2158 >;2159 2160 def : GCNPat<2161 (vt (st v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset,2162 timm:$format, timm:$auxiliary, timm)),2163 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,2164 (as_i8timm $format),2165 (extract_cpol $auxiliary), (extract_swz $auxiliary))2166 >;2167 2168 def : GCNPat<2169 (vt (st v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset,2170 timm:$format, timm:$auxiliary, 0)),2171 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,2172 (as_i8timm $format),2173 (extract_cpol $auxiliary), (extract_swz $auxiliary))2174 >;2175 2176 def : GCNPat<2177 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset,2178 timm:$format, timm:$auxiliary, timm)),2179 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN)2180 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),2181 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset,2182 (as_i8timm $format),2183 (extract_cpol $auxiliary), (extract_swz $auxiliary))2184 >;2185}2186 2187multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,2188 string opcode, ValueType memoryVt = vt> {2189 let SubtargetPredicate = HasUnrestrictedSOffset in {2190 defm : MTBUF_LoadIntrinsicPat_Common<name, vt, opcode, memoryVt>;2191 }2192 defm : MTBUF_LoadIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>;2193}2194 2195let OtherPredicates = [HasMTBUFInsts] in {2196defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">;2197defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">;2198defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">;2199defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">;2200defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">;2201defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">;2202defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">;2203defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">;2204} // End HasMTBUFInsts.2205 2206let OtherPredicates = [HasUnpackedD16VMem,HasMTBUFInsts] in {2207 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">;2208 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">;2209 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">;2210 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v3i32, "TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80">;2211 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;2212} // End HasUnpackedD16VMem,HasMTBUFInsts.2213 2214let OtherPredicates = [HasPackedD16VMem,HasMTBUFInsts] in {2215 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">;2216 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X">;2217 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">;2218 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3f16>;2219 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">;2220} // End HasPackedD16VMem,HasMTBUFInsts.2221 2222multiclass MTBUF_StoreIntrinsicPat_Common<SDPatternOperator name, ValueType vt,2223 string opcode, ValueType memoryVt = vt> {2224 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_store<name, memoryVt>);2225 2226 def : GCNPat<2227 (st vt:$vdata, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset,2228 timm:$format, timm:$auxiliary, 0),2229 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset,2230 timm:$offset, (as_i8timm $format),2231 (extract_cpol $auxiliary), (extract_swz $auxiliary))2232 >;2233 2234 def : GCNPat<2235 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset,2236 timm:$format, timm:$auxiliary, timm),2237 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset,2238 timm:$offset, (as_i8timm $format),2239 (extract_cpol $auxiliary), (extract_swz $auxiliary))2240 >;2241 2242 def : GCNPat<2243 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset,2244 timm:$format, timm:$auxiliary, 0),2245 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset,2246 timm:$offset, (as_i8timm $format),2247 (extract_cpol $auxiliary), (extract_swz $auxiliary))2248 >;2249 2250 def : GCNPat<2251 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset),2252 timm:$offset, timm:$format, timm:$auxiliary, timm),2253 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact)2254 getVregSrcForVT<vt>.ret:$vdata,2255 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),2256 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, (as_i8timm $format),2257 (extract_cpol $auxiliary), (extract_swz $auxiliary))2258 >;2259}2260 2261multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,2262 string opcode, ValueType memoryVt = vt> {2263 let SubtargetPredicate = HasUnrestrictedSOffset in {2264 defm : MTBUF_StoreIntrinsicPat_Common<name, vt, opcode, memoryVt>;2265 }2266 defm : MTBUF_StoreIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>;2267}2268 2269let OtherPredicates = [HasMTBUFInsts] in {2270defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">;2271defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">;2272defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">;2273defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">;2274defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">;2275defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">;2276defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">;2277defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">;2278} // End HasMTBUFInsts.2279 2280let OtherPredicates = [HasUnpackedD16VMem,HasMTBUFInsts] in {2281 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">;2282 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X_gfx80">;2283 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">;2284 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v3i32, "TBUFFER_STORE_FORMAT_D16_XYZ_gfx80">;2285 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">;2286} // End HasUnpackedD16VMem,HasMTBUFInsts.2287 2288let OtherPredicates = [HasPackedD16VMem,HasMTBUFInsts] in {2289 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">;2290 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X">;2291 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">;2292 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3f16>;2293 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">;2294} // End HasPackedD16VMem,HasMTBUFInsts.2295 2296//===----------------------------------------------------------------------===//2297// Target-specific instruction encodings.2298//===----------------------------------------------------------------------===//2299 2300// Shortcut to default Mnemonic from BUF_Pseudo. Hides the cast to the2301// specific pseudo (bothen in this case) since any of them will work.2302class get_BUF_ps<string name> {2303 string Mnemonic = !cast<BUF_Pseudo>(name # "_OFFSET").Mnemonic;2304}2305 2306//===----------------------------------------------------------------------===//2307// Base ENC_MUBUF for GFX6, GFX7, GFX10, GFX11.2308//===----------------------------------------------------------------------===//2309 2310class Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11 <MUBUF_Pseudo ps, int ef,2311 string real_name = ps.Mnemonic> :2312 MUBUF_Real<ps, real_name>, Enc64, SIMCInstr<ps.PseudoInstr, ef> {2313 let Inst{11-0} = !if(ps.has_offset, offset, ?);2314 let Inst{31-26} = 0x38;2315 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);2316 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?);2317 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);2318 let Inst{63-56} = !if(ps.has_soffset, soffset, ?);2319}2320 2321multiclass MUBUF_Real_gfx11<bits<8> op, string real_name = !cast<MUBUF_Pseudo>(NAME).Mnemonic> {2322 defvar ps = !cast<MUBUF_Pseudo>(NAME);2323 def _gfx11 : Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, SIEncodingFamily.GFX11, real_name> {2324 let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?);2325 // In GFX11 dlc is applicable to all loads/stores/atomics.2326 let Inst{13} = !if(!or(ps.mayLoad, ps.mayStore), cpol{CPolBit.DLC}, ps.dlc_value);2327 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value);2328 let Inst{25-18} = op;2329 let Inst{53} = ps.tfe;2330 let Inst{54} = ps.offen;2331 let Inst{55} = ps.idxen;2332 let AssemblerPredicate = isGFX11Only;2333 let DecoderNamespace = "GFX11";2334 }2335}2336 2337class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef, string asmName> :2338 Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, ef, asmName> {2339 let Inst{12} = ps.offen;2340 let Inst{13} = ps.idxen;2341 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value);2342 let Inst{16} = ps.lds;2343 let Inst{24-18} = op;2344 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?);2345 let Inst{55} = ps.tfe;2346}2347 2348multiclass MUBUF_Real_gfx10<bits<8> op, string psName = NAME,2349 string asmName = !cast<MUBUF_Pseudo>(psName).Mnemonic> {2350 defvar ps = !cast<MUBUF_Pseudo>(psName);2351 def _gfx10 : Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10, asmName> {2352 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value);2353 let Inst{25} = op{7};2354 let AssemblerPredicate = isGFX10Only;2355 let DecoderNamespace = "GFX10";2356 }2357}2358 2359multiclass MUBUF_Real_gfx6_gfx7<bits<8> op, string psName = NAME,2360 string asmName = !cast<MUBUF_Pseudo>(psName).Mnemonic> {2361 defvar ps = !cast<MUBUF_Pseudo>(psName);2362 def _gfx6_gfx7 : Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI, asmName> {2363 let Inst{15} = ps.addr64;2364 let AssemblerPredicate = isGFX6GFX7;2365 let DecoderNamespace = "GFX6GFX7";2366 }2367}2368 2369multiclass MUBUF_Real_gfx6<bits<8> op> {2370 defvar ps = !cast<MUBUF_Pseudo>(NAME);2371 def _gfx6 : Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI, ps.Mnemonic> {2372 let Inst{15} = ps.addr64;2373 let AssemblerPredicate = isGFX6;2374 let DecoderNamespace = "GFX6";2375 }2376}2377 2378multiclass MUBUF_Real_gfx7<bits<8> op> {2379 defvar ps = !cast<MUBUF_Pseudo>(NAME);2380 def _gfx7 : Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI, ps.Mnemonic> {2381 let Inst{15} = ps.addr64;2382 let AssemblerPredicate = isGFX7Only;2383 let DecoderNamespace = "GFX7";2384 }2385}2386 2387//===----------------------------------------------------------------------===//2388// Base ENC_VBUFFER for GFX12.2389//===----------------------------------------------------------------------===//2390 2391class VBUFFER_Real <bits<8> op, BUF_Pseudo ps, string real_name> :2392 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []>, Enc96 {2393 2394 let isPseudo = 0;2395 let isCodeGenOnly = 0;2396 2397 let VM_CNT = 1;2398 let EXP_CNT = 1;2399 2400 // copy relevant pseudo op flags2401 let SubtargetPredicate = ps.SubtargetPredicate;2402 let AsmMatchConverter = ps.AsmMatchConverter;2403 let OtherPredicates = ps.OtherPredicates;2404 let Constraints = ps.Constraints;2405 let TSFlags = ps.TSFlags;2406 let UseNamedOperandTable = ps.UseNamedOperandTable;2407 let SchedRW = ps.SchedRW;2408 let mayLoad = ps.mayLoad;2409 let mayStore = ps.mayStore;2410 let IsAtomicRet = ps.IsAtomicRet;2411 let IsAtomicNoRet = ps.IsAtomicNoRet;2412 let VALU = ps.VALU;2413 let LGKM_CNT = ps.LGKM_CNT;2414 let MUBUF = ps.MUBUF;2415 let MTBUF = ps.MTBUF;2416 let Uses = ps.Uses;2417 let Defs = ps.Defs;2418 let isConvergent = ps.isConvergent;2419 2420 bits<24> offset;2421 bits<8> vaddr;2422 bits<10> vdata;2423 2424 bits<7> srsrc;2425 bits<7> soffset;2426 bits<6> cpol;2427 2428 let Inst{95-72} = !if(ps.has_offset, offset, ?);2429 let Inst{71-64} = !if(ps.has_vaddr, vaddr, ?);2430 let Inst{39-32} = !if(ps.has_vdata, vdata{7-0}, ?);2431 2432 let Inst{47-41} = !if(ps.has_srsrc, srsrc, ?);2433 let Inst{49-48} = 0b00;2434 let Inst{6-0} = !if(ps.has_soffset, soffset, ?);2435 let Inst{21-14} = op;2436 let Inst{22} = ps.tfe;2437 let Inst{62} = ps.offen;2438 let Inst{63} = ps.idxen;2439 2440 let Inst{7} = cpol{5}; // nv2441 let Inst{54-53} = cpol{2-1}; // th{2-1}2442 let Inst{52} = !if(ps.IsAtomicRet, 1, cpol{0}); // th{0}2443 let Inst{51-50} = cpol{4-3}; // scope2444 2445 let Inst{31-26} = 0b110001;2446}2447 2448class VBUFFER_Real_gfx12<bits<8> op, BUF_Pseudo ps, string real_name> :2449 VBUFFER_Real<op, ps, real_name>,2450 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX12> {2451 let AssemblerPredicate = isGFX12Only;2452 let DecoderNamespace = "GFX12";2453}2454 2455multiclass VBUFFER_MUBUF_Real_gfx12<bits<8> op, string real_name> {2456 defvar ps = !cast<MUBUF_Pseudo>(NAME);2457 def _gfx12 : VBUFFER_Real_gfx12<op, ps, real_name> {2458 // Set the format field to be 1 to avoid round-trip issues, as some tools2459 // print BUF_FMT_INVALID for format 0.2460 let Inst{61-55} = 0b0000001;2461 }2462 // Have a version of the instruction to disassemble to for any other2463 // format field values.2464 def _gfx12_format : VBUFFER_Real<op, ps, real_name> {2465 let AsmVariantName = "NonParsable";2466 let DecoderNamespace = "GFX12";2467 }2468}2469 2470multiclass VBUFFER_MTBUF_Real_gfx12<bits<4> op, string real_name> {2471 defvar ps = !cast<MTBUF_Pseudo>(NAME);2472 def _gfx12 : VBUFFER_Real_gfx12<{0b1000, op}, ps, real_name> {2473 bits<7> format;2474 let Inst{61-55} = format;2475 }2476}2477 2478//===----------------------------------------------------------------------===//2479// MUBUF - GFX11, GFX12, GFX1250.2480//===----------------------------------------------------------------------===//2481 2482// gfx11 instruction that accept both old and new assembler name.2483class Mnem_gfx11_gfx12 <string mnemonic, string real_name> :2484 AMDGPUMnemonicAlias<mnemonic, real_name> {2485 let AssemblerPredicate = isGFX11Plus;2486}2487 2488class Mnem_gfx11 <string mnemonic, string real_name> :2489 AMDGPUMnemonicAlias<mnemonic, real_name> {2490 let AssemblerPredicate = isGFX11Only;2491}2492 2493class Mnem_gfx12 <string mnemonic, string real_name> :2494 AMDGPUMnemonicAlias<mnemonic, real_name> {2495 let AssemblerPredicate = isGFX12Plus;2496}2497 2498multiclass MUBUF_Real_AllAddr_gfx11_Impl2<bits<8> op, string real_name> {2499 defm _BOTHEN : MUBUF_Real_gfx11<op, real_name>;2500 defm _IDXEN : MUBUF_Real_gfx11<op, real_name>;2501 defm _OFFEN : MUBUF_Real_gfx11<op, real_name>;2502 defm _OFFSET : MUBUF_Real_gfx11<op, real_name>;2503}2504 2505multiclass MUBUF_Real_AllAddr_gfx12_Impl2<bits<8> op, string real_name> {2506 defm _VBUFFER_BOTHEN : VBUFFER_MUBUF_Real_gfx12<op, real_name>;2507 defm _VBUFFER_IDXEN : VBUFFER_MUBUF_Real_gfx12<op, real_name>;2508 defm _VBUFFER_OFFEN : VBUFFER_MUBUF_Real_gfx12<op, real_name>;2509 defm _VBUFFER_OFFSET : VBUFFER_MUBUF_Real_gfx12<op, real_name>;2510}2511 2512multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Impl2<bits<8> op, string real_name> :2513 MUBUF_Real_AllAddr_gfx11_Impl2<op, real_name>,2514 MUBUF_Real_AllAddr_gfx12_Impl2<op, real_name>;2515 2516multiclass MUBUF_Real_AllAddr_gfx11_Impl<bits<8> op, bit hasTFE,2517 string real_name = get_BUF_ps<NAME>.Mnemonic> {2518 defm NAME : MUBUF_Real_AllAddr_gfx11_Impl2<op, real_name>;2519 if hasTFE then2520 defm _TFE : MUBUF_Real_AllAddr_gfx11_Impl2<op, real_name>;2521}2522 2523multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Impl<bits<8> op, string real_name> {2524 defm NAME : MUBUF_Real_AllAddr_gfx11_gfx12_Impl2<op, real_name>;2525 defm _TFE : MUBUF_Real_AllAddr_gfx11_gfx12_Impl2<op, real_name>;2526}2527 2528// Non-renamed, non-atomic gfx11/gfx12 mubuf instructions.2529multiclass MUBUF_Real_AllAddr_gfx11<bits<8> op, bit hasTFE = 1> :2530 MUBUF_Real_AllAddr_gfx11_Impl<op, hasTFE>;2531 2532multiclass MUBUF_Real_AllAddr_gfx11_gfx12<bits<8> op,2533 string real_name = get_BUF_ps<NAME>.Mnemonic> :2534 MUBUF_Real_AllAddr_gfx11_gfx12_Impl<op, real_name> {2535 defvar ps = get_BUF_ps<NAME>;2536 if !ne(ps.Mnemonic, real_name) then2537 def : Mnem_gfx11_gfx12<ps.Mnemonic, real_name>;2538}2539 2540multiclass MUBUF_Real_Atomic_gfx11_impl<bits<8> op, bit is_return,2541 string real_name> {2542 defvar Rtn = !if(is_return, "_RTN", "");2543 defm _BOTHEN#Rtn : MUBUF_Real_gfx11<op, real_name>;2544 defm _IDXEN#Rtn : MUBUF_Real_gfx11<op, real_name>;2545 defm _OFFEN#Rtn : MUBUF_Real_gfx11<op, real_name>;2546 defm _OFFSET#Rtn : MUBUF_Real_gfx11<op, real_name>;2547}2548 2549multiclass MUBUF_Real_Atomic_gfx12_impl<bits<8> op, bit is_return,2550 string real_name = get_BUF_ps<NAME>.Mnemonic> {2551 defvar Rtn = !if(is_return, "_RTN", "");2552 defm _VBUFFER_BOTHEN#Rtn : VBUFFER_MUBUF_Real_gfx12<op, real_name>;2553 defm _VBUFFER_IDXEN#Rtn : VBUFFER_MUBUF_Real_gfx12<op, real_name>;2554 defm _VBUFFER_OFFEN#Rtn : VBUFFER_MUBUF_Real_gfx12<op, real_name>;2555 defm _VBUFFER_OFFSET#Rtn : VBUFFER_MUBUF_Real_gfx12<op, real_name>;2556}2557 2558multiclass MUBUF_Real_Atomic_gfx11_gfx12_impl<bits<8> op, bit is_return,2559 string real_name> :2560 MUBUF_Real_Atomic_gfx11_impl<op, is_return, real_name>,2561 MUBUF_Real_Atomic_gfx12_impl<op, is_return, real_name>;2562 2563multiclass MUBUF_Real_Atomic_gfx12<bits<8> op> :2564 MUBUF_Real_Atomic_gfx12_impl<op, 0>,2565 MUBUF_Real_Atomic_gfx12_impl<op, 1>;2566 2567multiclass MUBUF_Real_Atomic_gfx11<bits<8> op, string real_name> :2568 MUBUF_Real_Atomic_gfx11_impl<op, 0, real_name>,2569 MUBUF_Real_Atomic_gfx11_impl<op, 1, real_name> {2570 defvar ps = get_BUF_ps<NAME>;2571 def : Mnem_gfx11_gfx12<ps.Mnemonic, real_name>;2572}2573 2574multiclass MUBUF_Real_Atomic_gfx11_gfx12<bits<8> op,2575 string gfx12_name = get_BUF_ps<NAME>.Mnemonic,2576 string gfx11_name = gfx12_name> :2577 MUBUF_Real_Atomic_gfx11_impl<op, 0, gfx11_name>,2578 MUBUF_Real_Atomic_gfx11_impl<op, 1, gfx11_name>,2579 MUBUF_Real_Atomic_gfx12_impl<op, 0, gfx12_name>,2580 MUBUF_Real_Atomic_gfx12_impl<op, 1, gfx12_name> {2581 defvar ps = get_BUF_ps<NAME>;2582 if !ne(ps.Mnemonic, gfx11_name) then2583 def : Mnem_gfx11<ps.Mnemonic, gfx11_name>;2584 if !ne(ps.Mnemonic, gfx12_name) then2585 def : Mnem_gfx12<ps.Mnemonic, gfx12_name>;2586 if !ne(gfx11_name, gfx12_name) then2587 def : Mnem_gfx12<gfx11_name, gfx12_name>;2588}2589 2590multiclass MUBUF_Real_Atomic_gfx12_Renamed<bits<8> op, string real_name> :2591 MUBUF_Real_Atomic_gfx12_impl<op, 0, real_name>,2592 MUBUF_Real_Atomic_gfx12_impl<op, 1, real_name> {2593 def : Mnem_gfx12<get_BUF_ps<NAME>.Mnemonic, real_name>;2594}2595 2596defm BUFFER_GL0_INV : MUBUF_Real_gfx11<0x02B>;2597defm BUFFER_GL1_INV : MUBUF_Real_gfx11<0x02C>;2598 2599defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_gfx11_gfx12<0x014, "buffer_load_b32">;2600defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx11_gfx12<0x015, "buffer_load_b64">;2601defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx11_gfx12<0x016, "buffer_load_b96">;2602defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx11_gfx12<0x017, "buffer_load_b128">;2603defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx11_gfx12<0x020, "buffer_load_d16_b16">;2604defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x008, "buffer_load_d16_format_x">;2605defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx11_gfx12<0x009, "buffer_load_d16_format_xy">;2606defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12<0x00a, "buffer_load_d16_format_xyz">;2607defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12<0x00b, "buffer_load_d16_format_xyzw">;2608defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12<0x023, "buffer_load_d16_hi_b16">;2609defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x026, "buffer_load_d16_hi_format_x">;2610defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12<0x022, "buffer_load_d16_hi_i8">;2611defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12<0x021, "buffer_load_d16_hi_u8">;2612defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx11_gfx12<0x01f, "buffer_load_d16_i8">;2613defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx11_gfx12<0x01e, "buffer_load_d16_u8">;2614defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x000>;2615defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx11_gfx12<0x001>;2616defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12<0x002>;2617defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12<0x003>;2618defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_gfx11_gfx12<0x011, "buffer_load_i8">;2619defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_gfx11_gfx12<0x013, "buffer_load_i16">;2620defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_gfx11_gfx12<0x010, "buffer_load_u8">;2621defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_gfx11_gfx12<0x012, "buffer_load_u16">;2622defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx11_gfx12<0x018, "buffer_store_b8">;2623defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx11_gfx12<0x019, "buffer_store_b16">;2624defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx11_gfx12<0x01A, "buffer_store_b32">;2625defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx11_gfx12<0x01B, "buffer_store_b64">;2626defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx11_gfx12<0x01C, "buffer_store_b96">;2627defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx11_gfx12<0x01D, "buffer_store_b128">;2628defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x00C, "buffer_store_d16_format_x">;2629defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx11_gfx12<0x00D, "buffer_store_d16_format_xy">;2630defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12<0x00E, "buffer_store_d16_format_xyz">;2631defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12<0x00F, "buffer_store_d16_format_xyzw">;2632defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12<0x024, "buffer_store_d16_hi_b8">;2633defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12<0x025, "buffer_store_d16_hi_b16">;2634defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x027, "buffer_store_d16_hi_format_x">;2635defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x004>;2636defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx11_gfx12<0x005>;2637defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12<0x006>;2638defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12<0x007>;2639defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_gfx11_gfx12<0x056>;2640defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_gfx11_gfx12<0x035, "buffer_atomic_add_u32">;2641defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x043, "buffer_atomic_add_u64">;2642defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_gfx11_gfx12<0x03C, "buffer_atomic_and_b32">;2643defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x049, "buffer_atomic_and_b64">;2644defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_gfx11_gfx12<0x034, "buffer_atomic_cmpswap_b32">;2645defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x042, "buffer_atomic_cmpswap_b64">;2646defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomic_gfx11<0x050, "buffer_atomic_cmpswap_f32">;2647defm BUFFER_ATOMIC_COND_SUB_U32 : MUBUF_Real_Atomic_gfx12<0x050>;2648defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomic_gfx11_gfx12<0x037, "buffer_atomic_sub_clamp_u32", "buffer_atomic_csub_u32">;2649defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_gfx11_gfx12<0x040, "buffer_atomic_dec_u32">;2650defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x04D, "buffer_atomic_dec_u64">;2651defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_gfx11_gfx12<0x03F, "buffer_atomic_inc_u32">;2652defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x04C, "buffer_atomic_inc_u64">;2653defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomic_gfx11_gfx12<0x052, "buffer_atomic_max_num_f32", "buffer_atomic_max_f32">;2654defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_gfx11_gfx12<0x03A, "buffer_atomic_max_i32">;2655defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x047, "buffer_atomic_max_i64">;2656defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_gfx11_gfx12<0x03B, "buffer_atomic_max_u32">;2657defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x048, "buffer_atomic_max_u64">;2658defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomic_gfx11_gfx12<0x051, "buffer_atomic_min_num_f32", "buffer_atomic_min_f32">;2659defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_gfx11_gfx12<0x038, "buffer_atomic_min_i32">;2660defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x045, "buffer_atomic_min_i64">;2661defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_gfx11_gfx12<0x039, "buffer_atomic_min_u32">;2662defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x046, "buffer_atomic_min_u64">;2663defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_gfx11_gfx12<0x03D, "buffer_atomic_or_b32">;2664defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x04A, "buffer_atomic_or_b64">;2665defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_gfx11_gfx12<0x036, "buffer_atomic_sub_u32">;2666defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x044, "buffer_atomic_sub_u64">;2667defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_gfx11_gfx12<0x033, "buffer_atomic_swap_b32">;2668defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x041, "buffer_atomic_swap_b64">;2669defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_gfx11_gfx12<0x03E, "buffer_atomic_xor_b32">;2670defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x04B, "buffer_atomic_xor_b64">;2671defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_gfx12<0x059>;2672defm BUFFER_ATOMIC_PK_ADD_BF16 : MUBUF_Real_Atomic_gfx12<0x05a>;2673 2674defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Real_Atomic_gfx12<0x055>;2675defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Real_Atomic_gfx12_Renamed<0x05b, "buffer_atomic_min_num_f64">;2676defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Real_Atomic_gfx12_Renamed<0x05c, "buffer_atomic_max_num_f64">;2677 2678//===----------------------------------------------------------------------===//2679// MUBUF - GFX10.2680//===----------------------------------------------------------------------===//2681 2682multiclass MUBUF_Real_AllAddr_Helper_gfx10<bits<8> op> {2683 defm _BOTHEN : MUBUF_Real_gfx10<op>;2684 defm _IDXEN : MUBUF_Real_gfx10<op>;2685 defm _OFFEN : MUBUF_Real_gfx10<op>;2686 defm _OFFSET : MUBUF_Real_gfx10<op>;2687}2688multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> {2689 defm NAME : MUBUF_Real_AllAddr_Helper_gfx10<op>;2690 defm _TFE : MUBUF_Real_AllAddr_Helper_gfx10<op>;2691}2692multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op, bit isTFE = 0> {2693 defm _OFFSET : MUBUF_Real_gfx10<op>;2694 defm _OFFEN : MUBUF_Real_gfx10<op>;2695 defm _IDXEN : MUBUF_Real_gfx10<op>;2696 defm _BOTHEN : MUBUF_Real_gfx10<op>;2697 2698 if !not(isTFE) then {2699 defm _LDS_OFFSET : MUBUF_Real_gfx10<op>;2700 defm _LDS_OFFEN : MUBUF_Real_gfx10<op>;2701 defm _LDS_IDXEN : MUBUF_Real_gfx10<op>;2702 defm _LDS_BOTHEN : MUBUF_Real_gfx10<op>;2703 }2704}2705multiclass MUBUF_Real_Atomics_RTN_gfx10<bits<8> op, string psName = NAME,2706 string asmName = !cast<MUBUF_Pseudo>(psName).Mnemonic> {2707 defm _BOTHEN_RTN : MUBUF_Real_gfx10<op, psName#"_BOTHEN_RTN", asmName>;2708 defm _IDXEN_RTN : MUBUF_Real_gfx10<op, psName#"_IDXEN_RTN", asmName>;2709 defm _OFFEN_RTN : MUBUF_Real_gfx10<op, psName#"_OFFEN_RTN", asmName>;2710 defm _OFFSET_RTN : MUBUF_Real_gfx10<op, psName#"_OFFSET_RTN", asmName>;2711}2712multiclass MUBUF_Real_Atomics_gfx10<bits<8> op, string psName = NAME,2713 string asmName = get_BUF_ps<psName>.Mnemonic> :2714 MUBUF_Real_Atomics_RTN_gfx10<op, psName, asmName> {2715 defm _BOTHEN : MUBUF_Real_gfx10<op, psName#"_BOTHEN", asmName>;2716 defm _IDXEN : MUBUF_Real_gfx10<op, psName#"_IDXEN", asmName>;2717 defm _OFFEN : MUBUF_Real_gfx10<op, psName#"_OFFEN", asmName>;2718 defm _OFFSET : MUBUF_Real_gfx10<op, psName#"_OFFSET", asmName>;2719}2720 2721defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>;2722defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x01b>;2723defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x020>;2724defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x021>;2725defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x022>;2726defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x023>;2727defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx10<0x024>;2728defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x025>;2729defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x026>;2730defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>;2731defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x080>;2732defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x081>;2733defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x082>;2734defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x083>;2735defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x084>;2736defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>;2737defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>;2738defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>;2739 2740defm BUFFER_GL0_INV : MUBUF_Real_gfx10<0x071>;2741defm BUFFER_GL1_INV : MUBUF_Real_gfx10<0x072>;2742 2743//===----------------------------------------------------------------------===//2744// MUBUF - GFX6, GFX7, GFX10.2745//===----------------------------------------------------------------------===//2746 2747multiclass MUBUF_Real_AllAddr_Helper_gfx6_gfx7<bits<8> op> {2748 defm _ADDR64 : MUBUF_Real_gfx6_gfx7<op>;2749 defm _BOTHEN : MUBUF_Real_gfx6_gfx7<op>;2750 defm _IDXEN : MUBUF_Real_gfx6_gfx7<op>;2751 defm _OFFEN : MUBUF_Real_gfx6_gfx7<op>;2752 defm _OFFSET : MUBUF_Real_gfx6_gfx7<op>;2753}2754multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> {2755 defm NAME : MUBUF_Real_AllAddr_Helper_gfx6_gfx7<op>;2756 defm _TFE : MUBUF_Real_AllAddr_Helper_gfx6_gfx7<op>;2757}2758multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op, bit isTFE = 0> {2759 defm _OFFSET : MUBUF_Real_gfx6_gfx7<op>;2760 defm _ADDR64 : MUBUF_Real_gfx6_gfx7<op>;2761 defm _OFFEN : MUBUF_Real_gfx6_gfx7<op>;2762 defm _IDXEN : MUBUF_Real_gfx6_gfx7<op>;2763 defm _BOTHEN : MUBUF_Real_gfx6_gfx7<op>;2764 2765 if !not(isTFE) then {2766 defm _LDS_OFFSET : MUBUF_Real_gfx6_gfx7<op>;2767 defm _LDS_ADDR64 : MUBUF_Real_gfx6_gfx7<op>;2768 defm _LDS_OFFEN : MUBUF_Real_gfx6_gfx7<op>;2769 defm _LDS_IDXEN : MUBUF_Real_gfx6_gfx7<op>;2770 defm _LDS_BOTHEN : MUBUF_Real_gfx6_gfx7<op>;2771 }2772}2773multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op, string psName, string asmName> {2774 defm _ADDR64 : MUBUF_Real_gfx6_gfx7<op, psName#"_ADDR64", asmName>;2775 defm _BOTHEN : MUBUF_Real_gfx6_gfx7<op, psName#"_BOTHEN", asmName>;2776 defm _IDXEN : MUBUF_Real_gfx6_gfx7<op, psName#"_IDXEN", asmName>;2777 defm _OFFEN : MUBUF_Real_gfx6_gfx7<op, psName#"_OFFEN", asmName>;2778 defm _OFFSET : MUBUF_Real_gfx6_gfx7<op, psName#"_OFFSET", asmName>;2779 2780 defm _ADDR64_RTN : MUBUF_Real_gfx6_gfx7<op, psName#"_ADDR64_RTN", asmName>;2781 defm _BOTHEN_RTN : MUBUF_Real_gfx6_gfx7<op, psName#"_BOTHEN_RTN", asmName>;2782 defm _IDXEN_RTN : MUBUF_Real_gfx6_gfx7<op, psName#"_IDXEN_RTN", asmName>;2783 defm _OFFEN_RTN : MUBUF_Real_gfx6_gfx7<op, psName#"_OFFEN_RTN", asmName>;2784 defm _OFFSET_RTN : MUBUF_Real_gfx6_gfx7<op, psName#"_OFFSET_RTN", asmName>;2785}2786 2787multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> :2788 MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>;2789 2790multiclass MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<bits<8> op, bit isTFE = 0> :2791 MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op, isTFE>,2792 MUBUF_Real_AllAddr_Lds_gfx10<op, isTFE>;2793 2794multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> {2795 defm NAME : MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<op>;2796 defm _TFE : MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<op, 1>;2797}2798 2799multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op, string psName = NAME,2800 string asmName = get_BUF_ps<psName>.Mnemonic> :2801 MUBUF_Real_Atomics_gfx6_gfx7<op, psName, asmName>,2802 MUBUF_Real_Atomics_gfx10<op, psName, asmName>;2803 2804// FIXME-GFX6: Following instructions are available only on GFX6.2805//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomics_gfx6 <0x034>;2806//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomics_gfx6 <0x054>;2807 2808defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>;2809defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>;2810defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>;2811defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>;2812defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>;2813defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>;2814defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>;2815defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>;2816defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>;2817defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>;2818defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>;2819defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>;2820defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>;2821defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>;2822defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>;2823defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>;2824defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>;2825defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>;2826defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>;2827defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>;2828defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>;2829defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>;2830 2831defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>;2832defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>;2833defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>;2834defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>;2835defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>;2836defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>;2837defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>;2838defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>;2839defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>;2840defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>;2841defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>;2842defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>;2843defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>;2844defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>;2845defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>;2846defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>;2847defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>;2848defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>;2849defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>;2850defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>;2851defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>;2852defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>;2853defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>;2854defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>;2855defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>;2856defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>;2857defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>;2858defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>;2859defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>;2860// FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7.2861defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>;2862defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f, "BUFFER_ATOMIC_MIN_F64", "buffer_atomic_fmin_x2">;2863defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060, "BUFFER_ATOMIC_MAX_F64", "buffer_atomic_fmax_x2">;2864 2865defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_gfx10<0x034>;2866 2867defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>;2868defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>;2869defm BUFFER_WBINVL1 : MUBUF_Real_gfx6_gfx7<0x071>;2870 2871//===----------------------------------------------------------------------===//2872// Base ENC_MTBUF for GFX6, GFX7, GFX10, GFX11.2873//===----------------------------------------------------------------------===//2874 2875class Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<MTBUF_Pseudo ps, int ef,2876 string real_name = ps.Mnemonic> :2877 MTBUF_Real<ps, real_name>, Enc64, SIMCInstr<ps.PseudoInstr, ef> {2878 let Inst{11-0} = !if(ps.has_offset, offset, ?);2879 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value);2880 let Inst{31-26} = 0x3a; //encoding2881 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);2882 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?);2883 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);2884 let Inst{63-56} = !if(ps.has_soffset, soffset, ?);2885}2886 2887multiclass MTBUF_Real_gfx11<bits<4> op, string real_name> {2888 defvar ps = !cast<MTBUF_Pseudo>(NAME);2889 def _gfx11 : Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, SIEncodingFamily.GFX11, real_name> {2890 let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?);2891 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value);2892 let Inst{18-15} = op;2893 let Inst{25-19} = format;2894 let Inst{53} = ps.tfe;2895 let Inst{54} = ps.offen;2896 let Inst{55} = ps.idxen;2897 let AssemblerPredicate = isGFX11Only;2898 let DecoderNamespace = "GFX11";2899 }2900}2901 2902class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> :2903 Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, ef> {2904 let Inst{12} = ps.offen;2905 let Inst{13} = ps.idxen;2906 let Inst{18-16} = op;2907 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?);2908 let Inst{55} = ps.tfe;2909}2910 2911//===----------------------------------------------------------------------===//2912// MTBUF - GFX11.2913//===----------------------------------------------------------------------===//2914 2915multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Impl<bits<4> op, string real_name> {2916 defm _BOTHEN : MTBUF_Real_gfx11<op, real_name>;2917 defm _IDXEN : MTBUF_Real_gfx11<op, real_name>;2918 defm _OFFEN : MTBUF_Real_gfx11<op, real_name>;2919 defm _OFFSET : MTBUF_Real_gfx11<op, real_name>;2920 2921 defm _VBUFFER_BOTHEN : VBUFFER_MTBUF_Real_gfx12<op, real_name>;2922 defm _VBUFFER_IDXEN : VBUFFER_MTBUF_Real_gfx12<op, real_name>;2923 defm _VBUFFER_OFFEN : VBUFFER_MTBUF_Real_gfx12<op, real_name>;2924 defm _VBUFFER_OFFSET : VBUFFER_MTBUF_Real_gfx12<op, real_name>;2925}2926 2927multiclass MTBUF_Real_AllAddr_gfx11_gfx12<bits<4> op,2928 string real_name = get_BUF_ps<NAME>.Mnemonic>2929 : MTBUF_Real_AllAddr_gfx11_gfx12_Impl<op, real_name> {2930 defvar ps = get_BUF_ps<NAME>;2931 if !ne(ps.Mnemonic, real_name) then2932 def : Mnem_gfx11_gfx12<ps.Mnemonic, real_name>;2933}2934 2935defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_gfx12<0x008, "tbuffer_load_d16_format_x">;2936defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx11_gfx12<0x009, "tbuffer_load_d16_format_xy">;2937defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12<0x00a, "tbuffer_load_d16_format_xyz">;2938defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12<0x00b, "tbuffer_load_d16_format_xyzw">;2939defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx11_gfx12<0x000>;2940defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx11_gfx12<0x001>;2941defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12<0x002>;2942defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12<0x003>;2943defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_gfx12<0x00c, "tbuffer_store_d16_format_x">;2944defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx11_gfx12<0x00d, "tbuffer_store_d16_format_xy">;2945defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12<0x00e, "tbuffer_store_d16_format_xyz">;2946defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12<0x00f, "tbuffer_store_d16_format_xyzw">;2947defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx11_gfx12<0x004>;2948defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx11_gfx12<0x005>;2949defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12<0x006>;2950defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12<0x007>;2951 2952//===----------------------------------------------------------------------===//2953// MTBUF - GFX10.2954//===----------------------------------------------------------------------===//2955 2956multiclass MTBUF_Real_gfx10<bits<4> op> {2957 defvar ps = !cast<MTBUF_Pseudo>(NAME);2958 def _gfx10 : Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> {2959 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value);2960 let Inst{25-19} = format;2961 let Inst{53} = op{3};2962 let AssemblerPredicate = isGFX10Only;2963 let DecoderNamespace = "GFX10";2964 }2965}2966 2967multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> {2968 defm _BOTHEN : MTBUF_Real_gfx10<op>;2969 defm _IDXEN : MTBUF_Real_gfx10<op>;2970 defm _OFFEN : MTBUF_Real_gfx10<op>;2971 defm _OFFSET : MTBUF_Real_gfx10<op>;2972}2973 2974defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>;2975defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x009>;2976defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00a>;2977defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00b>;2978defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x00c>;2979defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x00d>;2980defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00e>;2981defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>;2982 2983//===----------------------------------------------------------------------===//2984// MTBUF - GFX6, GFX7, GFX10.2985//===----------------------------------------------------------------------===//2986 2987multiclass MTBUF_Real_gfx6_gfx7<bits<4> op> {2988 defvar ps = !cast<MTBUF_Pseudo>(NAME);2989 def _gfx6_gfx7 : Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> {2990 let Inst{15} = ps.addr64;2991 let Inst{22-19} = dfmt;2992 let Inst{25-23} = nfmt;2993 let AssemblerPredicate = isGFX6GFX7;2994 let DecoderNamespace = "GFX6GFX7";2995 }2996}2997 2998multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> {2999 defm _ADDR64 : MTBUF_Real_gfx6_gfx7<op>;3000 defm _BOTHEN : MTBUF_Real_gfx6_gfx7<op>;3001 defm _IDXEN : MTBUF_Real_gfx6_gfx7<op>;3002 defm _OFFEN : MTBUF_Real_gfx6_gfx7<op>;3003 defm _OFFSET : MTBUF_Real_gfx6_gfx7<op>;3004}3005 3006multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> :3007 MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>;3008 3009defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>;3010defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>;3011defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>;3012defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>;3013defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>;3014defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>;3015defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>;3016defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>;3017 3018//===----------------------------------------------------------------------===//3019// GFX8, GFX9 (VI).3020//===----------------------------------------------------------------------===//3021 3022class MUBUF_Real_Base_vi <bits<7> op, MUBUF_Pseudo ps, int Enc,3023 bit has_sccb = ps.has_sccb> :3024 MUBUF_Real<ps>,3025 Enc64,3026 SIMCInstr<ps.PseudoInstr, Enc> {3027 3028 let Inst{11-0} = !if(ps.has_offset, offset, ?);3029 let Inst{12} = ps.offen;3030 let Inst{13} = ps.idxen;3031 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value);3032 let Inst{15} = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccb_value);3033 let Inst{16} = ps.lds;3034 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?);3035 let Inst{24-18} = op;3036 let Inst{31-26} = 0x38; //encoding3037 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);3038 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?);3039 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);3040 let Inst{63-56} = !if(ps.has_soffset, soffset, ?);3041}3042 3043multiclass MUBUF_Real_vi <bits<7> op,3044 bit has_sccb = !cast<MUBUF_Pseudo>(NAME).has_sccb> {3045 defvar ps = !cast<MUBUF_Pseudo>(NAME);3046 def _vi : MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.VI, has_sccb> {3047 let AssemblerPredicate = isGFX8GFX9NotGFX90A;3048 let DecoderNamespace = "GFX8";3049 3050 let Inst{55} = ps.tfe;3051 }3052}3053 3054multiclass MUBUF_Real_gfx90a <bits<7> op,3055 bit has_sccb = !cast<MUBUF_Pseudo>(NAME).has_sccb> {3056 defvar ps = !cast<MUBUF_Pseudo>(NAME);3057 def _gfx90a : MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX90A, has_sccb> {3058 let AssemblerPredicate = isGFX90APlus;3059 let DecoderNamespace = "GFX90A";3060 let AsmString = ps.Mnemonic # !subst("$sccb", !if(has_sccb, "$sccb",""),3061 ps.AsmOperands);3062 3063 let Inst{55} = acc;3064 }3065}3066 3067class MUBUF_Real_gfx940 <bits<7> op, MUBUF_Pseudo ps> :3068 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX940> {3069 let AssemblerPredicate = isGFX940Plus;3070 let DecoderNamespace = "GFX9";3071 let AsmString = ps.Mnemonic # ps.AsmOperands;3072 3073 let Inst{55} = acc;3074}3075 3076multiclass MUBUF_Real_vi_gfx90a<bits<7> op, bit isTFE = 0> : MUBUF_Real_vi<op> {3077 defvar ps = !cast<MUBUF_Pseudo>(NAME);3078 3079 if !not(isTFE) then {3080 if !not(ps.FPAtomic) then3081 defm NAME : MUBUF_Real_gfx90a<op>;3082 }3083 3084 if ps.FPAtomic then {3085 let AssemblerPredicate = isGFX90AOnly in3086 defm NAME : MUBUF_Real_gfx90a<op, 0>;3087 3088 def _gfx940 : MUBUF_Real_gfx940<op, ps>;3089 }3090}3091 3092multiclass MUBUF_Real_AllAddr_Helper_vi<bits<7> op, bit isTFE = 0> {3093 defm _OFFSET : MUBUF_Real_vi_gfx90a <op, isTFE>;3094 defm _OFFEN : MUBUF_Real_vi_gfx90a <op, isTFE>;3095 defm _IDXEN : MUBUF_Real_vi_gfx90a <op, isTFE>;3096 defm _BOTHEN : MUBUF_Real_vi_gfx90a <op, isTFE>;3097}3098 3099multiclass MUBUF_Real_AllAddr_vi<bits<7> op, bit hasTFE = 1> {3100 defm NAME : MUBUF_Real_AllAddr_Helper_vi<op>;3101 if hasTFE then3102 defm _TFE : MUBUF_Real_AllAddr_Helper_vi<op, 1>;3103}3104 3105multiclass MUBUF_Real_AllAddr_Lds_Helper_vi<bits<7> op, bit isTFE = 0> {3106 defm _OFFSET : MUBUF_Real_vi <op>;3107 defm _OFFEN : MUBUF_Real_vi <op>;3108 defm _IDXEN : MUBUF_Real_vi <op>;3109 defm _BOTHEN : MUBUF_Real_vi <op>;3110 3111 if !not(isTFE) then {3112 defm _LDS_OFFSET : MUBUF_Real_vi <op>;3113 defm _LDS_OFFEN : MUBUF_Real_vi <op>;3114 defm _LDS_IDXEN : MUBUF_Real_vi <op>;3115 defm _LDS_BOTHEN : MUBUF_Real_vi <op>;3116 3117 defm _OFFSET : MUBUF_Real_gfx90a <op>;3118 defm _OFFEN : MUBUF_Real_gfx90a <op>;3119 defm _IDXEN : MUBUF_Real_gfx90a <op>;3120 defm _BOTHEN : MUBUF_Real_gfx90a <op>;3121 3122 defm _LDS_OFFSET : MUBUF_Real_gfx90a <op>;3123 defm _LDS_OFFEN : MUBUF_Real_gfx90a <op>;3124 defm _LDS_IDXEN : MUBUF_Real_gfx90a <op>;3125 defm _LDS_BOTHEN : MUBUF_Real_gfx90a <op>;3126 }3127}3128 3129multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> {3130 defm NAME : MUBUF_Real_AllAddr_Lds_Helper_vi<op>;3131 defm _TFE : MUBUF_Real_AllAddr_Lds_Helper_vi<op, 1>;3132}3133 3134multiclass MUBUF_Real_gfx80 <bits<7> op> {3135 defvar ps = !cast<MUBUF_Pseudo>(NAME);3136 def _gfx80 : MUBUF_Real<ps>,3137 Enc64,3138 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> {3139 let AssemblerPredicate=HasUnpackedD16VMem;3140 let DecoderNamespace="GFX80_UNPACKED";3141 3142 let Inst{11-0} = !if(ps.has_offset, offset, ?);3143 let Inst{12} = ps.offen;3144 let Inst{13} = ps.idxen;3145 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value);3146 let Inst{16} = ps.lds;3147 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?);3148 let Inst{24-18} = op;3149 let Inst{31-26} = 0x38; //encoding3150 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);3151 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?);3152 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);3153 let Inst{55} = ps.tfe;3154 let Inst{63-56} = !if(ps.has_soffset, soffset, ?);3155 }3156}3157 3158multiclass MUBUF_Real_AllAddr_Helper_gfx80<bits<7> op> {3159 defm _OFFSET : MUBUF_Real_gfx80 <op>;3160 defm _OFFEN : MUBUF_Real_gfx80 <op>;3161 defm _IDXEN : MUBUF_Real_gfx80 <op>;3162 defm _BOTHEN : MUBUF_Real_gfx80 <op>;3163}3164 3165multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> {3166 defm NAME : MUBUF_Real_AllAddr_Helper_gfx80<op>;3167 defm _TFE : MUBUF_Real_AllAddr_Helper_gfx80<op>;3168}3169 3170multiclass MUBUF_Real_Atomic_vi<bits<7> op> :3171 MUBUF_Real_AllAddr_vi<op, 0> {3172 defm _OFFSET_RTN : MUBUF_Real_vi_gfx90a <op>;3173 defm _OFFEN_RTN : MUBUF_Real_vi_gfx90a <op>;3174 defm _IDXEN_RTN : MUBUF_Real_vi_gfx90a <op>;3175 defm _BOTHEN_RTN : MUBUF_Real_vi_gfx90a <op>;3176}3177 3178defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>;3179defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>;3180defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>;3181defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>;3182defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>;3183defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>;3184defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>;3185defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>;3186let SubtargetPredicate = HasUnpackedD16VMem in {3187 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>;3188 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>;3189 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>;3190 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>;3191 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>;3192 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>;3193 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>;3194 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>;3195} // End HasUnpackedD16VMem.3196let SubtargetPredicate = HasPackedD16VMem in {3197 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>;3198 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>;3199 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>;3200 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>;3201 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>;3202 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>;3203 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>;3204 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>;3205} // End HasPackedD16VMem.3206defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>;3207defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>;3208defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>;3209defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>;3210defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>;3211defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_vi <0x15>;3212defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_Lds_vi <0x16>;3213defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_Lds_vi <0x17>;3214defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>;3215defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>;3216defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>;3217defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>;3218defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>;3219defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>;3220defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>;3221defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>;3222 3223defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>;3224defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>;3225defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>;3226defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>;3227defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>;3228defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>;3229 3230defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>;3231defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>;3232 3233defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>;3234defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>;3235defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>;3236defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>;3237defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>;3238defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>;3239defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>;3240defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>;3241defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>;3242defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>;3243defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>;3244defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>;3245defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>;3246 3247defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>;3248defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>;3249defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>;3250defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>;3251defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>;3252defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>;3253defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>;3254defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>;3255defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>;3256defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>;3257defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>;3258defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>;3259defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>;3260 3261defm BUFFER_STORE_LDS_DWORD : MUBUF_Real_vi_gfx90a <0x3d>;3262 3263let AssemblerPredicate = isGFX8GFX9 in {3264defm BUFFER_WBINVL1 : MUBUF_Real_vi <0x3e>;3265defm BUFFER_WBINVL1_VOL : MUBUF_Real_vi <0x3f>;3266} // End AssemblerPredicate = isGFX8GFX93267 3268 3269defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_vi <0x4e>;3270defm BUFFER_ATOMIC_PK_ADD_BF16 : MUBUF_Real_Atomic_vi <0x52>;3271 3272defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_vi <0x4d>;3273 3274let SubtargetPredicate = isGFX90APlus in {3275 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Real_Atomic_vi<0x4f>;3276 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Real_Atomic_vi<0x50>;3277 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Real_Atomic_vi<0x51>;3278} // End SubtargetPredicate = isGFX90APlus3279 3280let AsmString = BUFFER_WBL2.Mnemonic, // drop flags3281 AssemblerPredicate = isGFX90AOnly,3282 SubtargetPredicate = isGFX90AOnly in3283defm BUFFER_WBL2 : MUBUF_Real_gfx90a<0x28>;3284defm BUFFER_INVL2 : MUBUF_Real_gfx90a<0x29>;3285 3286let SubtargetPredicate = isGFX940Plus in {3287def BUFFER_WBL2_gfx940 : MUBUF_Real_gfx940<0x28, BUFFER_WBL2>;3288def BUFFER_INV_gfx940 : MUBUF_Real_gfx940<0x29, BUFFER_INV>;3289}3290 3291class MTBUF_Real_Base_vi <bits<4> op, MTBUF_Pseudo ps, int Enc> :3292 MTBUF_Real<ps>,3293 Enc64,3294 SIMCInstr<ps.PseudoInstr, Enc> {3295 3296 let Inst{11-0} = !if(ps.has_offset, offset, ?);3297 let Inst{12} = ps.offen;3298 let Inst{13} = ps.idxen;3299 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value);3300 let Inst{18-15} = op;3301 let Inst{22-19} = dfmt;3302 let Inst{25-23} = nfmt;3303 let Inst{31-26} = 0x3a; //encoding3304 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);3305 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?);3306 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);3307 let Inst{53} = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccb_value);3308 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?);3309 let Inst{55} = ps.tfe;3310 let Inst{63-56} = !if(ps.has_soffset, soffset, ?);3311}3312 3313class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> :3314 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.VI> {3315 let AssemblerPredicate = isGFX8GFX9NotGFX90A;3316 let DecoderNamespace = "GFX8";3317 3318 let Inst{55} = ps.tfe;3319}3320 3321class MTBUF_Real_gfx90a <bits<4> op, MTBUF_Pseudo ps> :3322 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.GFX90A> {3323 let AssemblerPredicate = isGFX90APlus;3324 let DecoderNamespace = "GFX90A";3325 let AsmString = ps.Mnemonic # ps.AsmOperands;3326 3327 let Inst{55} = acc;3328}3329 3330multiclass MTBUF_Real_vi_gfx90a<bits<4> op> {3331 defvar ps = !cast<MTBUF_Pseudo>(NAME);3332 def _vi : MTBUF_Real_vi<op, ps>;3333 def _gfx90a : MTBUF_Real_gfx90a<op, ps>;3334}3335 3336multiclass MTBUF_Real_AllAddr_vi<bits<4> op> {3337 defm _OFFSET : MTBUF_Real_vi_gfx90a <op>;3338 defm _OFFEN : MTBUF_Real_vi_gfx90a <op>;3339 defm _IDXEN : MTBUF_Real_vi_gfx90a <op>;3340 defm _BOTHEN : MTBUF_Real_vi_gfx90a <op>;3341}3342 3343multiclass MTBUF_Real_gfx80 <bits<4> op> {3344 defvar ps = !cast<MTBUF_Pseudo>(NAME);3345 def _gfx80 : MTBUF_Real<ps>,3346 Enc64,3347 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> {3348 let AssemblerPredicate=HasUnpackedD16VMem;3349 let DecoderNamespace="GFX80_UNPACKED";3350 3351 let Inst{11-0} = !if(ps.has_offset, offset, ?);3352 let Inst{12} = ps.offen;3353 let Inst{13} = ps.idxen;3354 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value);3355 let Inst{18-15} = op;3356 let Inst{22-19} = dfmt;3357 let Inst{25-23} = nfmt;3358 let Inst{31-26} = 0x3a; //encoding3359 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);3360 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?);3361 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);3362 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?);3363 let Inst{55} = ps.tfe;3364 let Inst{63-56} = !if(ps.has_soffset, soffset, ?);3365 }3366}3367 3368multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> {3369 defm _OFFSET : MTBUF_Real_gfx80 <op>;3370 defm _OFFEN : MTBUF_Real_gfx80 <op>;3371 defm _IDXEN : MTBUF_Real_gfx80 <op>;3372 defm _BOTHEN : MTBUF_Real_gfx80 <op>;3373}3374 3375defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>;3376defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>;3377defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>;3378defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>;3379defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>;3380defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>;3381defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>;3382defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>;3383let SubtargetPredicate = HasUnpackedD16VMem in {3384 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>;3385 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>;3386 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>;3387 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>;3388 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>;3389 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>;3390 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>;3391 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>;3392} // End HasUnpackedD16VMem.3393let SubtargetPredicate = HasPackedD16VMem in {3394 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>;3395 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>;3396 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>;3397 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>;3398 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>;3399 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>;3400 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>;3401 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>;3402} // End HasUnpackedD16VMem.3403 3404def MUBUFInfoTable : GenericTable {3405 let FilterClass = "MUBUF_Pseudo";3406 let CppTypeName = "MUBUFInfo";3407 let Fields = [3408 "Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset",3409 "IsBufferInv", "tfe"3410 ];3411 3412 let PrimaryKey = ["Opcode"];3413 let PrimaryKeyName = "getMUBUFOpcodeHelper";3414}3415 3416def getMUBUFInfoFromOpcode : SearchIndex {3417 let Table = MUBUFInfoTable;3418 let Key = ["Opcode"];3419}3420 3421def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex {3422 let Table = MUBUFInfoTable;3423 let Key = ["BaseOpcode", "elements"];3424}3425 3426def MTBUFInfoTable : GenericTable {3427 let FilterClass = "MTBUF_Pseudo";3428 let CppTypeName = "MTBUFInfo";3429 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"];3430 3431 let PrimaryKey = ["Opcode"];3432 let PrimaryKeyName = "getMTBUFOpcodeHelper";3433}3434 3435def getMTBUFInfoFromOpcode : SearchIndex {3436 let Table = MTBUFInfoTable;3437 let Key = ["Opcode"];3438}3439 3440def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex {3441 let Table = MTBUFInfoTable;3442 let Key = ["BaseOpcode", "elements"];3443}3444