3745 lines · plain
1//===-- FLATInstructions.td - FLAT Instruction Definitions ----------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9let WantsRoot = true in {10 def FlatOffset : ComplexPattern<iPTR, 2, "SelectFlatOffset", [], [], -10>;11 def GlobalOffset : ComplexPattern<iPTR, 2, "SelectGlobalOffset", [], [], -10>;12 def ScratchOffset : ComplexPattern<iPTR, 2, "SelectScratchOffset", [], [], -10>;13 14 def GlobalSAddrNoIOffset : ComplexPattern<iPTR, 3, "SelectGlobalSAddrNoIOffset", [], [], -3>;15 def GlobalSAddrNoIOffsetM0 : ComplexPattern<iPTR, 3, "SelectGlobalSAddrNoIOffsetM0", [], [], -3>;16 def GlobalSAddr : ComplexPattern<iPTR, 4, "SelectGlobalSAddr", [], [], -10>;17 def GlobalSAddrGLC : ComplexPattern<iPTR, 4, "SelectGlobalSAddrGLC", [], [], -10>;18 def GlobalSAddrCPol : ComplexPattern<iPTR, 4, "SelectGlobalSAddrCPol", [], [], -10>;19 def GlobalSAddrCPolM0 : ComplexPattern<iPTR, 4, "SelectGlobalSAddrCPolM0", [], [], -10>;20 def ScratchSAddr : ComplexPattern<iPTR, 2, "SelectScratchSAddr", [], [], -10>;21 def ScratchSVAddr : ComplexPattern<iPTR, 4, "SelectScratchSVAddr", [], [], -10>;22}23 24class True16D16Table <string hiOp, string loOp> {25 Instruction T16Op = !cast<Instruction>(NAME);26 Instruction HiOp = !cast<Instruction>(hiOp);27 Instruction LoOp = !cast<Instruction>(loOp);28}29 30//===----------------------------------------------------------------------===//31// FLAT classes32//===----------------------------------------------------------------------===//33 34class FLAT_Pseudo<string opName, dag outs, dag ins,35 string asmOps, list<dag> pattern=[]> :36 InstSI<outs, ins, "", pattern>,37 SIMCInstr<NAME, SIEncodingFamily.NONE> {38 39 let isPseudo = 1;40 let isCodeGenOnly = 1;41 42 let FLAT = 1;43 44 let UseNamedOperandTable = 1;45 let hasSideEffects = 0;46 let SchedRW = [WriteVMEM];47 48 string Mnemonic = opName;49 string AsmOperands = asmOps;50 51 bits<1> is_flat_global = 0;52 bits<1> is_flat_scratch = 0;53 54 bits<1> has_vdst = 1;55 56 // We need to distinguish having saddr and enabling saddr because57 // saddr is only valid for scratch and global instructions. Pre-gfx958 // these bits were reserved, so we also don't necessarily want to59 // set these bits to the disabled value for the original flat60 // segment instructions.61 bits<1> has_saddr = 0;62 bits<1> enabled_saddr = 0;63 bits<7> saddr_value = 0;64 bits<1> has_vaddr = 1;65 66 bits<1> has_data = 1;67 bits<1> has_glc = 1;68 bits<1> glcValue = 0;69 bits<1> has_dlc = 1;70 bits<1> dlcValue = 0;71 bits<1> has_sccb = 1;72 bits<1> sccbValue = 0;73 bits<1> has_sve = 0; // Scratch VGPR Enable74 bits<1> lds = 0;75 bits<1> sve = 0;76 bits<1> has_offset = 1;77 78 let SubtargetPredicate = !if(is_flat_global, HasFlatGlobalInsts,79 !if(is_flat_scratch, HasFlatScratchInsts, HasFlatAddressSpace));80 81 // TODO: M0 if it could possibly access LDS (before gfx9? only)?82 let Uses = !if(is_flat_global, [EXEC], [EXEC, FLAT_SCR]);83 84 // Internally, FLAT instruction are executed as both an LDS and a85 // Buffer instruction; so, they increment both VM_CNT and LGKM_CNT86 // and are not considered done until both have been decremented.87 let VM_CNT = 1;88 let LGKM_CNT = !not(!or(is_flat_global, is_flat_scratch));89 90 let FlatGlobal = is_flat_global;91 92 let FlatScratch = is_flat_scratch;93}94 95class FLAT_Real <bits<7> op, FLAT_Pseudo ps, string opName = ps.Mnemonic> :96 InstSI <ps.OutOperandList, ps.InOperandList, opName # ps.AsmOperands, []>,97 Enc64 {98 99 let isPseudo = 0;100 let isCodeGenOnly = 0;101 102 let FLAT = 1;103 104 // copy relevant pseudo op flags105 let SubtargetPredicate = ps.SubtargetPredicate;106 let AsmMatchConverter = ps.AsmMatchConverter;107 let OtherPredicates = ps.OtherPredicates;108 let TSFlags = ps.TSFlags;109 let UseNamedOperandTable = ps.UseNamedOperandTable;110 let SchedRW = ps.SchedRW;111 let mayLoad = ps.mayLoad;112 let mayStore = ps.mayStore;113 let IsAtomicRet = ps.IsAtomicRet;114 let IsAtomicNoRet = ps.IsAtomicNoRet;115 let VM_CNT = ps.VM_CNT;116 let LGKM_CNT = ps.LGKM_CNT;117 let VALU = ps.VALU;118 let Uses = ps.Uses;119 let Defs = ps.Defs;120 let isConvergent = ps.isConvergent;121 122 // encoding fields123 bits<8> vaddr;124 bits<10> vdata;125 bits<7> saddr;126 bits<10> vdst;127 128 bits<5> cpol;129 130 // Only valid on gfx9131 bits<1> lds = ps.lds; // LDS DMA for global and scratch132 133 // Segment, 00=flat, 01=scratch, 10=global, 11=reserved134 bits<2> seg = {ps.is_flat_global, ps.is_flat_scratch};135 136 // Signed offset. Highest bit ignored for flat and treated as 12-bit137 // unsigned for flat accesses.138 bits<13> offset;139 // GFX90A+ only: instruction uses AccVGPR for data140 defvar DstOpIsAV = !if(ps.has_vdst,141 VDstOperandIsAV<ps.OutOperandList>.ret, 0);142 defvar DstOpIsAGPR = !if(ps.has_vdst,143 VDstOperandIsAGPR<ps.OutOperandList>.ret, 0);144 defvar DataOpIsAV = !if(ps.has_data,145 VDataOperandIsAV<ps.InOperandList>.ret, 0);146 defvar DataOpIsAGPR = !if(ps.has_data,147 VDataOperandIsAGPR<ps.InOperandList>.ret, 0);148 149 bits<1> acc = !if(ps.has_vdst,150 !if(DstOpIsAV, vdst{9}, DstOpIsAGPR),151 !if(DataOpIsAV, vdata{9}, DataOpIsAGPR));152 153 // We don't use tfe right now, and it was removed in gfx9.154 bits<1> tfe = 0;155 156 // Only valid on GFX9+157 let Inst{12-0} = offset;158 let Inst{13} = !if(ps.has_sve, ps.sve, lds);159 let Inst{15-14} = seg;160 161 let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glcValue);162 let Inst{17} = cpol{CPolBit.SLC};163 let Inst{24-18} = op;164 let Inst{31-26} = 0x37; // Encoding.165 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);166 let Inst{47-40} = !if(ps.has_data, vdata{7-0}, ?);167 let Inst{54-48} = !if(ps.has_saddr, !if(ps.enabled_saddr, saddr, 0x7f), 0);168 169 // 54-48 is reserved.170 let Inst{55} = acc; // nv on GFX9+, TFE before. AccVGPR for data on GFX90A.171 let Inst{63-56} = !if(ps.has_vdst, vdst{7-0}, ?);172}173 174class VFLAT_Real <bits<8> op, FLAT_Pseudo ps, string opName = ps.Mnemonic> :175 InstSI <ps.OutOperandList, ps.InOperandList, opName # ps.AsmOperands, []>,176 Enc96 {177 178 let FLAT = 1;179 180 // copy relevant pseudo op flags181 let SubtargetPredicate = ps.SubtargetPredicate;182 let WaveSizePredicate = ps.WaveSizePredicate;183 let AsmMatchConverter = ps.AsmMatchConverter;184 let OtherPredicates = ps.OtherPredicates;185 let TSFlags = ps.TSFlags;186 let UseNamedOperandTable = ps.UseNamedOperandTable;187 let SchedRW = ps.SchedRW;188 let mayLoad = ps.mayLoad;189 let mayStore = ps.mayStore;190 let IsAtomicRet = ps.IsAtomicRet;191 let IsAtomicNoRet = ps.IsAtomicNoRet;192 let VM_CNT = ps.VM_CNT;193 let LGKM_CNT = ps.LGKM_CNT;194 let VALU = ps.VALU;195 let Uses = ps.Uses;196 let Defs = ps.Defs;197 let isConvergent = ps.isConvergent;198 199 bits<7> saddr;200 bits<8> vdst;201 bits<12> cpol;202 bits<8> vdata; // vsrc203 bits<8> vaddr;204 bits<24> offset;205 206 let Inst{6-0} = !if(ps.enabled_saddr, saddr, SGPR_NULL_gfx11plus.Index);207 let Inst{21-14} = op;208 let Inst{31-26} = 0x3b;209 let Inst{39-32} = !if(ps.has_vdst, vdst, ?);210 let Inst{49} = ps.sve;211 let Inst{7} = cpol{5}; // nv212 let Inst{54-53} = cpol{2-1}; // th{2-1}213 let Inst{52} = !if(ps.IsAtomicRet, 1, cpol{0}); // th{0}214 let Inst{51-50} = cpol{4-3}; // scope215 let Inst{62-55} = !if(ps.has_data, vdata{7-0}, ?);216 let Inst{71-64} = !if(ps.has_vaddr, vaddr, ?);217 let Inst{95-72} = !if(ps.has_offset, offset, ?);218}219 220// TODO: Rename to FlatSaddrTable, it now handles both global and flat GVS addressing mode.221class GlobalSaddrTable <bit is_saddr, string Name = ""> {222 bit IsSaddr = is_saddr;223 string SaddrOp = Name;224}225 226// TODO: Is exec allowed for saddr? The disabled value 0x7f is the227// same encoding value as exec_hi, so it isn't possible to use that if228// saddr is 32-bit (which isn't handled here yet).229class FLAT_Load_Pseudo<230 string opName, RegisterOperand vdata_op, bit HasTiedOutput = 0,231 bit HasSaddr = 0, bit EnableSaddr = 0,232 RegisterClassLike VaddrRC = !if(EnableSaddr, VGPR_32, VReg_64_AlignTarget)>233 : FLAT_Pseudo<opName, (outs), (ins), ""> {234 235 let OutOperandList = (outs vdata_op:$vdst);236 let InOperandList = !con(237 !if(EnableSaddr, (ins SReg_64_XEXEC_XNULL:$saddr), (ins)),238 (ins VaddrRC:$vaddr, flat_offset:$offset),239 // FIXME: Operands with default values do not work with following240 // non-optional operands.241 !if(HasTiedOutput, (ins CPol:$cpol, vdata_op:$vdst_in),242 (ins CPol_0:$cpol)));243 let AsmOperands = " $vdst, $vaddr"244 # !if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")245 # "$offset$cpol";246 247 let has_data = 0;248 let mayLoad = 1;249 let has_saddr = HasSaddr;250 let enabled_saddr = EnableSaddr;251 252 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", "");253}254 255multiclass FLAT_Flat_Load_Pseudo<string opName, RegisterOperand regClass = AVLdSt_32, bit HasTiedInput = 0> {256 def "" : FLAT_Load_Pseudo<opName, regClass, HasTiedInput>,257 GlobalSaddrTable<0, opName>;258 let OtherPredicates = [HasFlatGVSMode] in259 def _SADDR : FLAT_Load_Pseudo<opName, regClass, HasTiedInput, 1, 1>,260 GlobalSaddrTable<1, opName>;261}262 263multiclass FLAT_Flat_Load_Pseudo_t16<string opName> {264 defm "" : FLAT_Flat_Load_Pseudo<opName, AVLdSt_32, 1>;265 266 defvar Name16 = opName#"_t16";267 let True16Predicate = UseRealTrue16Insts in {268 def _t16 : FLAT_Load_Pseudo<Name16, VGPROp_16>,269 GlobalSaddrTable<0, Name16>,270 True16D16Table<NAME#"_HI", NAME>;271 272 let OtherPredicates = [HasFlatGVSMode] in273 def _t16_SADDR : FLAT_Load_Pseudo<Name16, VGPROp_16, 0, 1, 1>,274 GlobalSaddrTable<1, Name16>,275 True16D16Table<NAME#"_HI_SADDR", NAME#"_SADDR">;276 }277}278 279class FLAT_Store_Pseudo <string opName, RegisterOperand vdataClass,280 bit HasSaddr = 0, bit EnableSaddr = 0,281 RegisterClassLike VaddrRC = !if(EnableSaddr, VGPR_32, VReg_64_AlignTarget)> : FLAT_Pseudo<opName, (outs), (ins), ""> {282 let InOperandList = !con(283 (ins VaddrRC:$vaddr, vdataClass:$vdata),284 !if(EnableSaddr, (ins SReg_64_XEXEC_XNULL:$saddr), (ins)),285 (ins flat_offset:$offset, CPol_0:$cpol));286 let AsmOperands = " $vaddr, $vdata"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$cpol";287 let mayLoad = 0;288 let mayStore = 1;289 let has_vdst = 0;290 let has_saddr = HasSaddr;291 let enabled_saddr = EnableSaddr;292}293 294multiclass FLAT_Flat_Store_Pseudo<string opName, RegisterOperand regClass = AVLdSt_32> {295 def "" : FLAT_Store_Pseudo<opName, regClass>,296 GlobalSaddrTable<0, opName>;297 let OtherPredicates = [HasFlatGVSMode] in298 def _SADDR : FLAT_Store_Pseudo<opName, regClass, 1, 1>,299 GlobalSaddrTable<1, opName>;300}301 302multiclass FLAT_Flat_Store_Pseudo_t16<string opName> {303 defm "" : FLAT_Flat_Store_Pseudo<opName, AVLdSt_32>;304 305 defvar Name16 = opName#"_t16";306 let OtherPredicates = [HasFlatGVSMode, HasTrue16BitInsts] in {307 def _t16 : FLAT_Store_Pseudo<Name16, VGPROp_16, 1>,308 GlobalSaddrTable<0, Name16>,309 True16D16Table<NAME#"_D16_HI", NAME>;310 def _SADDR_t16 : FLAT_Store_Pseudo<Name16, VGPROp_16, 1, 1>,311 GlobalSaddrTable<1, Name16>,312 True16D16Table<NAME#"_D16_HI_SADDR", NAME#"_SADDR">;313 }314}315 316multiclass FLAT_Global_Load_Pseudo<string opName, RegisterOperand regClass = AVLdSt_32,317 bit HasTiedInput = 0> {318 let is_flat_global = 1, SubtargetPredicate = HasFlatGlobalInsts in {319 def "" : FLAT_Load_Pseudo<opName, regClass, HasTiedInput, 1>,320 GlobalSaddrTable<0, opName>;321 def _SADDR : FLAT_Load_Pseudo<opName, regClass, HasTiedInput, 1, 1>,322 GlobalSaddrTable<1, opName>;323 }324}325 326multiclass FLAT_Global_Load_Pseudo_t16<string opName> {327 defm "" : FLAT_Global_Load_Pseudo<opName, AVLdSt_32, 1>;328 329 defvar Name16 = opName#"_t16";330 let OtherPredicates = [HasTrue16BitInsts],331 SubtargetPredicate = HasFlatGlobalInsts, is_flat_global = 1 in {332 def _t16 : FLAT_Load_Pseudo<Name16, VGPROp_16, 0, 1>,333 GlobalSaddrTable<0, Name16>,334 True16D16Table<NAME#"_HI", NAME>;335 def _SADDR_t16 : FLAT_Load_Pseudo<Name16, VGPROp_16, 0, 1, 1>,336 GlobalSaddrTable<1, Name16>,337 True16D16Table<NAME#"_HI_SADDR", NAME#"_SADDR">;338 }339}340 341class FLAT_Global_Load_AddTid_Pseudo <string opName, RegisterOperand regClass,342 bit HasTiedOutput = 0, bit EnableSaddr = 0> : FLAT_Pseudo<343 opName,344 (outs regClass:$vdst),345 !con(!if(EnableSaddr, (ins SReg_64:$saddr), (ins)),346 (ins flat_offset:$offset, CPol_0:$cpol),347 !if(HasTiedOutput, (ins regClass:$vdst_in), (ins))),348 " $vdst, "#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> {349 let is_flat_global = 1;350 let has_data = 0;351 let mayLoad = 1;352 let has_vaddr = 0;353 let has_saddr = 1;354 let enabled_saddr = EnableSaddr;355 356 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", "");357}358 359multiclass FLAT_Global_Load_AddTid_Pseudo<string opName, RegisterOperand regClass,360 bit HasTiedOutput = 0> {361 def "" : FLAT_Global_Load_AddTid_Pseudo<opName, regClass, HasTiedOutput>,362 GlobalSaddrTable<0, opName>;363 def _SADDR : FLAT_Global_Load_AddTid_Pseudo<opName, regClass, HasTiedOutput, 1>,364 GlobalSaddrTable<1, opName>;365}366 367multiclass FLAT_Global_Store_Pseudo<string opName, RegisterOperand regClass = AVLdSt_32> {368 let is_flat_global = 1, SubtargetPredicate = HasFlatGlobalInsts in {369 def "" : FLAT_Store_Pseudo<opName, regClass, 1>,370 GlobalSaddrTable<0, opName>;371 def _SADDR : FLAT_Store_Pseudo<opName, regClass, 1, 1>,372 GlobalSaddrTable<1, opName>;373 }374}375 376multiclass FLAT_Global_Store_Pseudo_t16<string opName> {377 defm "" : FLAT_Global_Store_Pseudo<opName, AVLdSt_32>;378 379 defvar Name16 = opName#"_t16";380 let OtherPredicates = [HasTrue16BitInsts],381 SubtargetPredicate = HasFlatGlobalInsts, is_flat_global = 1 in {382 def _t16 : FLAT_Store_Pseudo<Name16, VGPROp_16, 1>,383 GlobalSaddrTable<0, Name16>,384 True16D16Table<NAME#"_D16_HI", NAME>;385 def _SADDR_t16 : FLAT_Store_Pseudo<Name16, VGPROp_16, 1, 1>,386 GlobalSaddrTable<1, Name16>,387 True16D16Table<NAME#"_D16_HI_SADDR", NAME#"_SADDR">;388 }389}390 391// Async loads, introduced in gfx1250, will store directly392// to a DS address in vdst (they will not use M0 for DS addess).393class FLAT_Global_Load_LDS_Pseudo <string opName, bit EnableSaddr = 0, bit IsAsync = 0> : FLAT_Pseudo<394 opName,395 (outs ),396 !con(397 !if(IsAsync, (ins VGPR_32:$vdst), (ins)),398 !if(EnableSaddr, (ins SReg_64:$saddr, VGPR_32:$vaddr), (ins VReg_64_AlignTarget:$vaddr)),399 (ins flat_offset:$offset, CPol_0:$cpol)),400 !if(IsAsync, " $vdst,", "")#" $vaddr"#!if(EnableSaddr, ", $saddr", ", off")#"$offset$cpol"> {401 let LGKM_CNT = !not(IsAsync);402 let VM_CNT = !not(IsAsync);403 let ASYNC_CNT = IsAsync;404 let is_flat_global = 1;405 let lds = 1;406 let has_data = 0;407 let has_vdst = IsAsync; // vdst for ds address with IsAsync408 let mayLoad = 1;409 let mayStore = 1;410 let has_saddr = 1;411 let enabled_saddr = EnableSaddr;412 let VALU = 1;413 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", "");414 let Uses = !if(IsAsync, [EXEC, ASYNCcnt], [M0, EXEC]);415 let Defs = !if(IsAsync, [ASYNCcnt], []);416 let SchedRW = [WriteVMEM, WriteLDS];417}418 419multiclass FLAT_Global_Load_LDS_Pseudo<string opName, bit IsAsync = 0> {420 def "" : FLAT_Global_Load_LDS_Pseudo<opName, 0, IsAsync>,421 GlobalSaddrTable<0, opName>;422 def _SADDR : FLAT_Global_Load_LDS_Pseudo<opName, 1, IsAsync>,423 GlobalSaddrTable<1, opName>;424}425 426class FLAT_Global_STORE_LDS_Pseudo <string opName, bit EnableSaddr = 0> : FLAT_Pseudo<427 opName,428 (outs ),429 !con(430 !if(EnableSaddr, (ins SReg_64:$saddr, VGPR_32:$vaddr), (ins VReg_64_AlignTarget:$vaddr)), (ins VGPR_32:$vdata),431 (ins flat_offset:$offset, CPol_0:$cpol)),432 " $vaddr, $vdata"#!if(EnableSaddr, ", $saddr", ", off")#"$offset$cpol"> {433 let VM_CNT = 0;434 let ASYNC_CNT = 1;435 let is_flat_global = 1;436 let lds = 1;437 let has_data = 1; // vdata for ds address438 let has_vdst = 0;439 let mayLoad = 1;440 let mayStore = 1;441 let has_saddr = 1;442 let enabled_saddr = EnableSaddr;443 let VALU = 1;444 let Uses = [EXEC, ASYNCcnt];445 let Defs = [ASYNCcnt];446 let SchedRW = [WriteVMEM, WriteLDS];447}448 449multiclass FLAT_Global_STORE_LDS_Pseudo<string opName> {450 def "" : FLAT_Global_STORE_LDS_Pseudo<opName>,451 GlobalSaddrTable<0, opName>;452 def _SADDR : FLAT_Global_STORE_LDS_Pseudo<opName, 1>,453 GlobalSaddrTable<1, opName>;454}455 456class FLAT_Global_Store_AddTid_Pseudo <string opName, RegisterOperand vdataClass,457 bit EnableSaddr = 0> : FLAT_Pseudo<458 opName,459 (outs),460 !con(!if(EnableSaddr, (ins vdataClass:$vdata, SReg_64:$saddr), (ins vdataClass:$vdata)),461 (ins flat_offset:$offset, CPol:$cpol)),462 " $vdata, "#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> {463 let is_flat_global = 1;464 let mayLoad = 0;465 let mayStore = 1;466 let has_vdst = 0;467 let has_vaddr = 0;468 let has_saddr = 1;469 let enabled_saddr = EnableSaddr;470}471 472multiclass FLAT_Global_Store_AddTid_Pseudo<string opName, RegisterOperand regClass> {473 def "" : FLAT_Global_Store_AddTid_Pseudo<opName, regClass>,474 GlobalSaddrTable<0, opName>;475 def _SADDR : FLAT_Global_Store_AddTid_Pseudo<opName, regClass, 1>,476 GlobalSaddrTable<1, opName>;477}478 479class FLAT_Global_Tensor_Pseudo<string opName, bit EnableSaddr = 0> : FLAT_Pseudo<480 opName,481 (outs ),482 !con(!if(EnableSaddr, (ins SReg_64:$saddr, flat_offset:$offset), (ins )), (ins CPol_0:$cpol)),483 !if(EnableSaddr, " $saddr$offset", " ")#"$cpol"> {484 485 let is_flat_global = 1;486 let has_vdst = 0;487 let has_data = 0;488 let has_vaddr = 0;489 let mayLoad = 0;490 let mayStore = 1;491 let has_saddr = 1;492 let enabled_saddr = EnableSaddr;493 let has_offset = EnableSaddr;494}495 496class FLAT_Global_Invalidate_Writeback<string opName, SDPatternOperator node = null_frag> :497 FLAT_Pseudo<opName, (outs), (ins CPol:$cpol), "$cpol", [(node)]> {498 499 let AsmMatchConverter = "";500 501 let hasSideEffects = 1;502 let mayLoad = 0;503 let mayStore = 0;504 let is_flat_global = 1;505 506 let has_offset = 0;507 let has_saddr = 0;508 let enabled_saddr = 0;509 let saddr_value = 0;510 let has_vdst = 0;511 let has_data = 0;512 let has_vaddr = 0;513 let has_glc = 0;514 let has_dlc = 0;515 let glcValue = 0;516 let dlcValue = 0;517 let has_sccb = 0;518 let sccbValue = 0;519 let has_sve = 0;520 let lds = 0;521 let sve = 0;522}523 524class FLAT_Prefetch_Pseudo<string opName, dag addr = (ins VReg_64_AlignTarget:$vaddr), string asm = " $vaddr"> :525 FLAT_Pseudo<opName, (outs), !con(addr, (ins flat_offset:$offset, CPol_0:$cpol)), asm#"$offset$cpol"> {526 let has_vdst = 0;527 let has_data = 0;528 let mayLoad = 1;529 let mayStore = 1;530 let VM_CNT = 0;531 let LGKM_CNT = 0;532}533 534multiclass FLAT_Flat_Prefetch_Pseudo<string opName> {535 def "" : FLAT_Prefetch_Pseudo<opName>,536 GlobalSaddrTable<0, opName>;537 def _SADDR : FLAT_Prefetch_Pseudo<opName, (ins SReg_64:$saddr, VGPR_32:$vaddr), " $vaddr, $saddr">,538 GlobalSaddrTable<1, opName> {539 let OtherPredicates = [HasFlatGVSMode];540 let enabled_saddr = 1;541 }542}543 544multiclass FLAT_Global_Prefetch_Pseudo<string opName> {545 let is_flat_global = 1, has_saddr = 1 in {546 def "" : FLAT_Prefetch_Pseudo<opName, (ins VReg_64_AlignTarget:$vaddr), " $vaddr, off">,547 GlobalSaddrTable<0, opName>;548 def _SADDR : FLAT_Prefetch_Pseudo<opName, (ins SReg_64:$saddr, VGPR_32:$vaddr), " $vaddr, $saddr">,549 GlobalSaddrTable<1, opName> {550 let enabled_saddr = 1;551 }552 }553}554 555class FlatScratchInst <string sv_op, string mode> {556 string SVOp = sv_op;557 string Mode = mode;558}559 560class FLAT_Scratch_Load_Pseudo <string opName, RegisterOperand regClass = AVLdSt_32,561 bit HasTiedOutput = 0,562 bit EnableSaddr = 0,563 bit EnableSVE = 0,564 bit EnableVaddr = !or(EnableSVE, !not(EnableSaddr))>565 : FLAT_Pseudo<566 opName,567 (outs regClass:$vdst),568 !con(569 !if(EnableSVE,570 (ins VGPR_32:$vaddr, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset),571 !if(EnableSaddr,572 (ins SReg_32_XEXEC_HI:$saddr, flat_offset:$offset),573 !if(EnableVaddr,574 (ins VGPR_32:$vaddr, flat_offset:$offset),575 (ins flat_offset:$offset)))),576 !if(HasTiedOutput, (ins CPol:$cpol, regClass:$vdst_in),577 (ins CPol_0:$cpol))),578 " $vdst, "#!if(EnableVaddr, "$vaddr, ", "off, ")#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> {579 let is_flat_scratch = 1;580 let has_data = 0;581 let mayLoad = 1;582 let has_saddr = 1;583 let enabled_saddr = EnableSaddr;584 let has_vaddr = EnableVaddr;585 let has_sve = EnableSVE;586 let sve = EnableVaddr;587 588 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", "");589}590 591class FLAT_Scratch_Store_Pseudo <string opName, RegisterOperand vdata_op, bit EnableSaddr = 0,592 bit EnableSVE = 0,593 bit EnableVaddr = !or(EnableSVE, !not(EnableSaddr))> : FLAT_Pseudo<594 opName,595 (outs),596 !if(EnableSVE,597 (ins vdata_op:$vdata, VGPR_32:$vaddr, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, CPol_0:$cpol),598 !if(EnableSaddr,599 (ins vdata_op:$vdata, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, CPol_0:$cpol),600 !if(EnableVaddr,601 (ins vdata_op:$vdata, VGPR_32:$vaddr, flat_offset:$offset, CPol_0:$cpol),602 (ins vdata_op:$vdata, flat_offset:$offset, CPol_0:$cpol)))),603 " "#!if(EnableVaddr, "$vaddr", "off")#", $vdata, "#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> {604 let is_flat_scratch = 1;605 let mayLoad = 0;606 let mayStore = 1;607 let has_vdst = 0;608 let has_saddr = 1;609 let enabled_saddr = EnableSaddr;610 let has_vaddr = EnableVaddr;611 let has_sve = EnableSVE;612 let sve = EnableVaddr;613}614 615multiclass FLAT_Scratch_Load_Pseudo<string opName, RegisterOperand regClass = AVLdSt_32,616 bit HasTiedOutput = 0> {617 def "" : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput>,618 FlatScratchInst<opName, "SV">;619 def _SADDR : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 1>,620 FlatScratchInst<opName, "SS">;621 622 let SubtargetPredicate = HasFlatScratchSVSMode in623 def _SVS : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 1, 1>,624 FlatScratchInst<opName, "SVS">;625 626 let SubtargetPredicate = HasFlatScratchSTMode in627 def _ST : FLAT_Scratch_Load_Pseudo<opName, regClass, HasTiedOutput, 0, 0, 0>,628 FlatScratchInst<opName, "ST">;629}630 631multiclass FLAT_Scratch_Load_Pseudo_t16<string opName> {632 defm "" : FLAT_Scratch_Load_Pseudo<opName, AVLdSt_32, 1>;633 634 defvar Name16 = opName#"_t16";635 let OtherPredicates = [HasTrue16BitInsts], is_flat_scratch = 1 in {636 def _t16 : FLAT_Scratch_Load_Pseudo<Name16, VGPROp_16, 0>,637 FlatScratchInst<Name16, "SV">,638 True16D16Table<NAME#"_HI", NAME>;639 def _SADDR_t16 : FLAT_Scratch_Load_Pseudo<Name16, VGPROp_16, 0, 1>,640 FlatScratchInst<Name16, "SS">,641 True16D16Table<NAME#"_HI_SADDR", NAME#"_SADDR">;642 let SubtargetPredicate = HasFlatScratchSVSMode in643 def _SVS_t16 : FLAT_Scratch_Load_Pseudo<Name16, VGPROp_16, 0, 1, 1>,644 FlatScratchInst<Name16, "SVS">,645 True16D16Table<NAME#"_HI_SVS", NAME#"_SVS">;646 647 let SubtargetPredicate = HasFlatScratchSTMode in648 def _ST_t16 : FLAT_Scratch_Load_Pseudo<Name16, VGPROp_16, 0, 0, 0, 0>,649 FlatScratchInst<Name16, "ST">,650 True16D16Table<NAME#"_HI_ST", NAME#"_ST">;651 }652}653 654multiclass FLAT_Scratch_Store_Pseudo<string opName, RegisterOperand regClass = AVLdSt_32> {655 def "" : FLAT_Scratch_Store_Pseudo<opName, regClass>,656 FlatScratchInst<opName, "SV">;657 def _SADDR : FLAT_Scratch_Store_Pseudo<opName, regClass, 1>,658 FlatScratchInst<opName, "SS">;659 660 let SubtargetPredicate = HasFlatScratchSVSMode in661 def _SVS : FLAT_Scratch_Store_Pseudo<opName, regClass, 1, 1>,662 FlatScratchInst<opName, "SVS">;663 664 let SubtargetPredicate = HasFlatScratchSTMode in665 def _ST : FLAT_Scratch_Store_Pseudo<opName, regClass, 0, 0, 0>,666 FlatScratchInst<opName, "ST">;667}668 669multiclass FLAT_Scratch_Store_Pseudo_t16<string opName> {670 defm "" : FLAT_Scratch_Store_Pseudo<opName, AVLdSt_32>;671 672 defvar Name16 = opName#"_t16";673 let OtherPredicates = [HasTrue16BitInsts], is_flat_scratch = 1 in {674 def _t16 : FLAT_Scratch_Store_Pseudo<Name16, VGPROp_16>,675 FlatScratchInst<Name16, "SV">,676 True16D16Table<NAME#"_D16_HI", NAME>;677 def _SADDR_t16 : FLAT_Scratch_Store_Pseudo<Name16, VGPROp_16, 1>,678 FlatScratchInst<Name16, "SS">,679 True16D16Table<NAME#"_D16_HI_SADDR", NAME#"_SADDR">;680 681 let SubtargetPredicate = HasFlatScratchSVSMode in682 def _SVS_t16 : FLAT_Scratch_Store_Pseudo<Name16, VGPROp_16, 1, 1>,683 FlatScratchInst<Name16, "SVS">,684 True16D16Table<NAME#"_D16_HI_SVS", NAME#"_SVS">;685 686 let SubtargetPredicate = HasFlatScratchSTMode in687 def _ST_t16 : FLAT_Scratch_Store_Pseudo<Name16, VGPROp_16, 0, 0, 0>,688 FlatScratchInst<Name16, "ST">,689 True16D16Table<NAME#"_D16_HI_ST", NAME#"_ST">;690 }691}692 693 694class FLAT_Scratch_Load_LDS_Pseudo <string opName, bit EnableSaddr = 0,695 bit EnableSVE = 0,696 bit EnableVaddr = !or(EnableSVE, !not(EnableSaddr))> : FLAT_Pseudo<697 opName,698 (outs ),699 !if(EnableSVE,700 (ins VGPR_32:$vaddr, SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, CPol:$cpol),701 !if(EnableSaddr,702 (ins SReg_32_XEXEC_HI:$saddr, flat_offset:$offset, CPol:$cpol),703 !if(EnableVaddr,704 (ins VGPR_32:$vaddr, flat_offset:$offset, CPol:$cpol),705 (ins flat_offset:$offset, CPol:$cpol)))),706 " "#!if(EnableVaddr, "$vaddr, ", "off, ")#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol"> {707 708 let LGKM_CNT = 1;709 let is_flat_scratch = 1;710 let lds = 1;711 let has_data = 0;712 let has_vdst = 0;713 let mayLoad = 1;714 let mayStore = 1;715 let has_saddr = 1;716 let enabled_saddr = EnableSaddr;717 let has_vaddr = EnableVaddr;718 let has_sve = EnableSVE;719 let sve = EnableVaddr;720 let VALU = 1;721 let Uses = [M0, EXEC];722 let SchedRW = [WriteVMEM, WriteLDS];723}724 725multiclass FLAT_Scratch_Load_LDS_Pseudo<string opName> {726 def "" : FLAT_Scratch_Load_LDS_Pseudo<opName>,727 FlatScratchInst<opName, "SV">;728 def _SADDR : FLAT_Scratch_Load_LDS_Pseudo<opName, 1>,729 FlatScratchInst<opName, "SS">;730 def _SVS : FLAT_Scratch_Load_LDS_Pseudo<opName, 1, 1>,731 FlatScratchInst<opName, "SVS">;732 def _ST : FLAT_Scratch_Load_LDS_Pseudo<opName, 0, 0, 0>,733 FlatScratchInst<opName, "ST">;734}735 736class FLAT_AtomicNoRet_Pseudo<string opName, dag outs, dag ins,737 string asm, list<dag> pattern = []> :738 FLAT_Pseudo<opName, outs, ins, asm, pattern> {739 let mayLoad = 1;740 let mayStore = 1;741 let has_glc = 0;742 let glcValue = 0;743 let has_vdst = 0;744 let has_sccb = 1;745 let sccbValue = 0;746 let IsAtomicNoRet = 1;747}748 749class FLAT_AtomicRet_Pseudo<string opName, dag outs, dag ins,750 string asm, list<dag> pattern = []>751 : FLAT_AtomicNoRet_Pseudo<opName, outs, ins, asm, pattern> {752 let has_vdst = 1;753 let glcValue = 1;754 let sccbValue = 0;755 let IsAtomicNoRet = 0;756 let IsAtomicRet = 1;757}758 759multiclass FLAT_Atomic_Pseudo_NO_RTN<760 string opName,761 RegisterOperand vdst_op,762 ValueType vt,763 ValueType data_vt = vt,764 RegisterOperand data_op = vdst_op> {765 def "" : FLAT_AtomicNoRet_Pseudo <opName,766 (outs),767 (ins VReg_64_AlignTarget:$vaddr, data_op:$vdata, flat_offset:$offset, CPol_0:$cpol),768 " $vaddr, $vdata$offset$cpol">,769 GlobalSaddrTable<0, opName> {770 let FPAtomic = data_vt.isFP;771 let AddedComplexity = -1; // Prefer global atomics if available772 }773 774 def _SADDR : FLAT_AtomicNoRet_Pseudo <opName,775 (outs),776 (ins VGPR_32:$vaddr, data_op:$vdata, SReg_64:$saddr, flat_offset:$offset, CPol_0:$cpol),777 " $vaddr, $vdata, $saddr$offset$cpol">,778 GlobalSaddrTable<1, opName> {779 let OtherPredicates = [HasFlatGVSMode];780 let has_saddr = 1;781 let enabled_saddr = 1;782 let FPAtomic = data_vt.isFP;783 let AddedComplexity = -1; // Prefer global atomics if available784 }785}786 787multiclass FLAT_Atomic_Pseudo_RTN<788 string opName,789 RegisterOperand vdst_op,790 ValueType vt,791 ValueType data_vt = vt,792 RegisterOperand data_op = vdst_op> {793 794 defvar vdst_op_vgpr = getEquivalentVGPROperand<vdst_op>.ret;795 defvar data_op_vgpr = getEquivalentVGPROperand<data_op>.ret;796 797 def _RTN : FLAT_AtomicRet_Pseudo <opName,798 (outs vdst_op_vgpr:$vdst),799 (ins VReg_64_AlignTarget:$vaddr, data_op_vgpr:$vdata, flat_offset:$offset, CPol_GLC1:$cpol),800 " $vdst, $vaddr, $vdata$offset$cpol">,801 GlobalSaddrTable<0, opName#"_rtn"> {802 let FPAtomic = data_vt.isFP;803 let AddedComplexity = -1; // Prefer global atomics if available804 }805 806 def _SADDR_RTN : FLAT_AtomicRet_Pseudo <opName,807 (outs vdst_op_vgpr:$vdst),808 (ins VGPR_32:$vaddr, data_op_vgpr:$vdata, SReg_64:$saddr, flat_offset:$offset, CPol_GLC1:$cpol),809 " $vdst, $vaddr, $vdata, $saddr$offset$cpol">,810 GlobalSaddrTable<1, opName#"_rtn"> {811 let OtherPredicates = [HasFlatGVSMode];812 let has_saddr = 1;813 let enabled_saddr = 1;814 let PseudoInstr = NAME#"_SADDR_RTN";815 let FPAtomic = data_vt.isFP;816 let AddedComplexity = -1; // Prefer global atomics if available817 }818 819 defvar vdst_op_agpr = getEquivalentAGPROperand<vdst_op>.ret;820 defvar data_op_agpr = getEquivalentAGPROperand<data_op>.ret;821 822 def _RTN_agpr : FLAT_AtomicRet_Pseudo <opName,823 (outs vdst_op_agpr:$vdst),824 (ins VReg_64_AlignTarget:$vaddr, data_op_agpr:$vdata, flat_offset:$offset, CPol_GLC1:$cpol),825 " $vdst, $vaddr, $vdata$offset$cpol">,826 GlobalSaddrTable<0, opName#"_rtn_agpr"> {827 let FPAtomic = data_vt.isFP;828 let AddedComplexity = -1; // Prefer global atomics if available829 }830 // No saddr agpr form. HasFlatGVSMode targets do not have AGPRs.831}832 833multiclass FLAT_Atomic_Pseudo<834 string opName,835 RegisterOperand vdst_op,836 ValueType vt,837 ValueType data_vt = vt,838 RegisterOperand data_op = vdst_op> {839 defm "" : FLAT_Atomic_Pseudo_NO_RTN<opName, vdst_op, vt, data_vt, data_op>;840 defm "" : FLAT_Atomic_Pseudo_RTN<opName, vdst_op, vt, data_vt, data_op>;841}842 843class FLAT_Global_Atomic_Pseudo_NO_RTN<844 string opName,845 RegisterOperand vdst_op,846 ValueType vt,847 ValueType data_vt = vt,848 RegisterOperand data_op = vdst_op,849 bit EnableSaddr = false,850 RegisterClassLike VaddrRC = !if(EnableSaddr, VGPR_32, VReg_64_AlignTarget)>851 : FLAT_AtomicNoRet_Pseudo<opName, (outs), (ins), "">, GlobalSaddrTable<EnableSaddr, opName> {852 let InOperandList = !con(853 (ins VaddrRC:$vaddr, data_op:$vdata),854 !if(EnableSaddr, (ins SReg_64_XEXEC_XNULL:$saddr), (ins)),855 (ins flat_offset:$offset, CPol_0:$cpol));856 let AsmOperands = " $vaddr, $vdata, "#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol";857 let has_saddr = 1;858 let enabled_saddr = EnableSaddr;859 let FPAtomic = data_vt.isFP;860 let is_flat_global = 1;861}862 863multiclass FLAT_Global_Atomic_Pseudo_Helper_NO_RTN<string opName,864 RegisterOperand vdst_op,865 ValueType vt,866 ValueType data_vt = vt,867 RegisterOperand data_op = vdst_op> {868 def "" : FLAT_Global_Atomic_Pseudo_NO_RTN<opName, vdst_op, vt, data_vt, data_op, 0>;869 def _SADDR : FLAT_Global_Atomic_Pseudo_NO_RTN<opName, vdst_op, vt, data_vt, data_op, 1>;870}871 872class FLAT_Global_Atomic_Pseudo_RTN<873 string opName,874 RegisterOperand vdst_op,875 ValueType vt,876 ValueType data_vt = vt,877 RegisterOperand data_op = vdst_op,878 bit EnableSaddr = false,879 bit IsVGPR = false,880 RegisterClassLike VaddrRC = !if(EnableSaddr, VGPR_32, VReg_64_AlignTarget)>881 : FLAT_AtomicRet_Pseudo<opName, (outs), (ins), "">, GlobalSaddrTable<EnableSaddr, opName#"_rtn"#!if(IsVGPR, "", "_agpr")> {882 883 defvar vdst_rc= !if(IsVGPR, getEquivalentVGPROperand<vdst_op>.ret, getEquivalentAGPROperand<vdst_op>.ret);884 defvar data_rc = !if(IsVGPR, getEquivalentVGPROperand<data_op>.ret, getEquivalentAGPROperand<data_op>.ret);885 886 let OutOperandList = (outs vdst_rc:$vdst);887 let InOperandList = !con(888 (ins VaddrRC:$vaddr, data_rc:$vdata),889 !if(EnableSaddr, (ins SReg_64_XEXEC_XNULL:$saddr), (ins)),890 (ins flat_offset:$offset, CPol_GLC1:$cpol));891 let AsmOperands = " $vdst, $vaddr, $vdata, "#!if(EnableSaddr, "$saddr", "off")#"$offset$cpol";892 let has_saddr = 1;893 let enabled_saddr = EnableSaddr;894 let FPAtomic = data_vt.isFP;895 let is_flat_global = 1;896}897 898multiclass FLAT_Global_Atomic_Pseudo_Helper_RTN<string opName,899 RegisterOperand vdst_op,900 ValueType vt,901 ValueType data_vt = vt,902 RegisterOperand data_op = vdst_op> {903 def _RTN : FLAT_Global_Atomic_Pseudo_RTN<opName, vdst_op, vt, data_vt, data_op, 0, 1>;904 def _SADDR_RTN : FLAT_Global_Atomic_Pseudo_RTN<opName, vdst_op, vt, data_vt, data_op, 1, 1>;905 let SubtargetPredicate = isGFX90APlus in {906 def _RTN_agpr : FLAT_Global_Atomic_Pseudo_RTN<opName, vdst_op, vt, data_vt, data_op, 0, 0>;907 def _SADDR_RTN_agpr : FLAT_Global_Atomic_Pseudo_RTN<opName, vdst_op, vt, data_vt, data_op, 1, 0>;908 }909}910 911 912multiclass FLAT_Global_Atomic_Pseudo<913 string opName,914 RegisterOperand vdst_rc,915 ValueType vt,916 ValueType data_vt = vt,917 RegisterOperand data_rc = vdst_rc> {918 defm "" : FLAT_Global_Atomic_Pseudo_Helper_NO_RTN<opName, vdst_rc, vt, data_vt, data_rc>;919 defm "" : FLAT_Global_Atomic_Pseudo_Helper_RTN<opName, vdst_rc, vt, data_vt, data_rc>;920}921 922//===----------------------------------------------------------------------===//923// Flat Instructions924//===----------------------------------------------------------------------===//925 926defm FLAT_LOAD_UBYTE : FLAT_Flat_Load_Pseudo <"flat_load_ubyte">;927defm FLAT_LOAD_SBYTE : FLAT_Flat_Load_Pseudo <"flat_load_sbyte">;928defm FLAT_LOAD_USHORT : FLAT_Flat_Load_Pseudo <"flat_load_ushort">;929defm FLAT_LOAD_SSHORT : FLAT_Flat_Load_Pseudo <"flat_load_sshort">;930defm FLAT_LOAD_DWORD : FLAT_Flat_Load_Pseudo <"flat_load_dword">;931defm FLAT_LOAD_DWORDX2 : FLAT_Flat_Load_Pseudo <"flat_load_dwordx2", AVLdSt_64>;932defm FLAT_LOAD_DWORDX4 : FLAT_Flat_Load_Pseudo <"flat_load_dwordx4", AVLdSt_128>;933defm FLAT_LOAD_DWORDX3 : FLAT_Flat_Load_Pseudo <"flat_load_dwordx3", AVLdSt_96>;934 935defm FLAT_STORE_DWORD : FLAT_Flat_Store_Pseudo <"flat_store_dword">;936defm FLAT_STORE_DWORDX2 : FLAT_Flat_Store_Pseudo <"flat_store_dwordx2", AVLdSt_64>;937defm FLAT_STORE_DWORDX4 : FLAT_Flat_Store_Pseudo <"flat_store_dwordx4", AVLdSt_128>;938defm FLAT_STORE_DWORDX3 : FLAT_Flat_Store_Pseudo <"flat_store_dwordx3", AVLdSt_96>;939 940let SubtargetPredicate = HasD16LoadStore in {941let TiedSourceNotRead = 1 in {942defm FLAT_LOAD_UBYTE_D16_HI : FLAT_Flat_Load_Pseudo <"flat_load_ubyte_d16_hi", AVLdSt_32, 1>;943defm FLAT_LOAD_UBYTE_D16 : FLAT_Flat_Load_Pseudo_t16 <"flat_load_ubyte_d16">;944defm FLAT_LOAD_SBYTE_D16_HI : FLAT_Flat_Load_Pseudo <"flat_load_sbyte_d16_hi", AVLdSt_32, 1>;945defm FLAT_LOAD_SBYTE_D16 : FLAT_Flat_Load_Pseudo_t16 <"flat_load_sbyte_d16">;946defm FLAT_LOAD_SHORT_D16_HI : FLAT_Flat_Load_Pseudo <"flat_load_short_d16_hi", AVLdSt_32, 1>;947defm FLAT_LOAD_SHORT_D16 : FLAT_Flat_Load_Pseudo_t16 <"flat_load_short_d16">;948}949 950defm FLAT_STORE_BYTE_D16_HI : FLAT_Flat_Store_Pseudo <"flat_store_byte_d16_hi">;951defm FLAT_STORE_SHORT_D16_HI : FLAT_Flat_Store_Pseudo <"flat_store_short_d16_hi">;952}953 954defm FLAT_STORE_BYTE : FLAT_Flat_Store_Pseudo_t16 <"flat_store_byte">;955defm FLAT_STORE_SHORT : FLAT_Flat_Store_Pseudo_t16 <"flat_store_short">;956 957defm FLAT_ATOMIC_CMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap",958 AVLdSt_32, i32, v2i32, AVLdSt_64>;959 960defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap_x2",961 AVLdSt_64, i64, v2i64, AVLdSt_128>;962 963defm FLAT_ATOMIC_SWAP : FLAT_Atomic_Pseudo <"flat_atomic_swap",964 AVLdSt_32, i32>;965 966defm FLAT_ATOMIC_SWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_swap_x2",967 AVLdSt_64, i64>;968 969defm FLAT_ATOMIC_ADD : FLAT_Atomic_Pseudo <"flat_atomic_add",970 AVLdSt_32, i32>;971 972defm FLAT_ATOMIC_SUB : FLAT_Atomic_Pseudo <"flat_atomic_sub",973 AVLdSt_32, i32>;974 975defm FLAT_ATOMIC_SMIN : FLAT_Atomic_Pseudo <"flat_atomic_smin",976 AVLdSt_32, i32>;977 978defm FLAT_ATOMIC_UMIN : FLAT_Atomic_Pseudo <"flat_atomic_umin",979 AVLdSt_32, i32>;980 981defm FLAT_ATOMIC_SMAX : FLAT_Atomic_Pseudo <"flat_atomic_smax",982 AVLdSt_32, i32>;983 984defm FLAT_ATOMIC_UMAX : FLAT_Atomic_Pseudo <"flat_atomic_umax",985 AVLdSt_32, i32>;986 987defm FLAT_ATOMIC_AND : FLAT_Atomic_Pseudo <"flat_atomic_and",988 AVLdSt_32, i32>;989 990defm FLAT_ATOMIC_OR : FLAT_Atomic_Pseudo <"flat_atomic_or",991 AVLdSt_32, i32>;992 993defm FLAT_ATOMIC_XOR : FLAT_Atomic_Pseudo <"flat_atomic_xor",994 AVLdSt_32, i32>;995 996defm FLAT_ATOMIC_INC : FLAT_Atomic_Pseudo <"flat_atomic_inc",997 AVLdSt_32, i32>;998 999defm FLAT_ATOMIC_DEC : FLAT_Atomic_Pseudo <"flat_atomic_dec",1000 AVLdSt_32, i32>;1001 1002defm FLAT_ATOMIC_ADD_X2 : FLAT_Atomic_Pseudo <"flat_atomic_add_x2",1003 AVLdSt_64, i64>;1004 1005defm FLAT_ATOMIC_SUB_X2 : FLAT_Atomic_Pseudo <"flat_atomic_sub_x2",1006 AVLdSt_64, i64>;1007 1008defm FLAT_ATOMIC_SMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smin_x2",1009 AVLdSt_64, i64>;1010 1011defm FLAT_ATOMIC_UMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umin_x2",1012 AVLdSt_64, i64>;1013 1014defm FLAT_ATOMIC_SMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smax_x2",1015 AVLdSt_64, i64>;1016 1017defm FLAT_ATOMIC_UMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umax_x2",1018 AVLdSt_64, i64>;1019 1020defm FLAT_ATOMIC_AND_X2 : FLAT_Atomic_Pseudo <"flat_atomic_and_x2",1021 AVLdSt_64, i64>;1022 1023defm FLAT_ATOMIC_OR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_or_x2",1024 AVLdSt_64, i64>;1025 1026defm FLAT_ATOMIC_XOR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_xor_x2",1027 AVLdSt_64, i64>;1028 1029defm FLAT_ATOMIC_INC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_inc_x2",1030 AVLdSt_64, i64>;1031 1032defm FLAT_ATOMIC_DEC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_dec_x2",1033 AVLdSt_64, i64>;1034 1035// GFX7-, GFX10-only flat instructions.1036let SubtargetPredicate = isGFX7GFX10 in {1037defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fcmpswap_x2",1038 AVLdSt_64, f64, v2f64, AVLdSt_128>;1039} // End SubtargetPredicate = isGFX7GFX101040 1041 1042// The names may be flat_atomic_fmin_x2 on some subtargets, but we1043// choose this as the canonical name.1044let SubtargetPredicate = HasAtomicFMinFMaxF64FlatInsts in {1045defm FLAT_ATOMIC_MIN_F64 : FLAT_Atomic_Pseudo <"flat_atomic_min_f64",1046 AVLdSt_64, f64>;1047 1048defm FLAT_ATOMIC_MAX_F64 : FLAT_Atomic_Pseudo <"flat_atomic_max_f64",1049 AVLdSt_64, f64>;1050}1051 1052let SubtargetPredicate = HasAtomicFMinFMaxF64GlobalInsts in {1053defm GLOBAL_ATOMIC_MIN_F64 : FLAT_Global_Atomic_Pseudo<"global_atomic_min_f64", AVLdSt_64, f64>;1054defm GLOBAL_ATOMIC_MAX_F64 : FLAT_Global_Atomic_Pseudo<"global_atomic_max_f64", AVLdSt_64, f64>;1055}1056 1057let SubtargetPredicate = HasFlatBufferGlobalAtomicFaddF64Inst in {1058 defm FLAT_ATOMIC_ADD_F64 : FLAT_Atomic_Pseudo<"flat_atomic_add_f64", AVLdSt_64, f64>;1059 defm GLOBAL_ATOMIC_ADD_F64 : FLAT_Global_Atomic_Pseudo<"global_atomic_add_f64", AVLdSt_64, f64>;1060} // End SubtargetPredicate = HasFlatBufferGlobalAtomicFaddF64Inst1061 1062let SubtargetPredicate = HasAtomicFlatPkAdd16Insts in {1063 defm FLAT_ATOMIC_PK_ADD_F16 : FLAT_Atomic_Pseudo<"flat_atomic_pk_add_f16", AVLdSt_32, v2f16>;1064 let FPAtomic = 1 in1065 defm FLAT_ATOMIC_PK_ADD_BF16 : FLAT_Atomic_Pseudo<"flat_atomic_pk_add_bf16", AVLdSt_32, v2i16>;1066} // End SubtargetPredicate = HasAtomicFlatPkAdd16Insts1067 1068let SubtargetPredicate = HasAtomicGlobalPkAddBF16Inst, FPAtomic = 1 in1069 defm GLOBAL_ATOMIC_PK_ADD_BF16 : FLAT_Global_Atomic_Pseudo<"global_atomic_pk_add_bf16", AVLdSt_32, v2i16>;1070 1071// GFX7-, GFX10-, GFX11-only flat instructions.1072let SubtargetPredicate = isGFX7GFX10GFX11 in {1073 1074defm FLAT_ATOMIC_FCMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_fcmpswap",1075 AVLdSt_32, f32, v2f32, AVLdSt_64>;1076 1077defm FLAT_ATOMIC_FMIN : FLAT_Atomic_Pseudo <"flat_atomic_fmin",1078 AVLdSt_32, f32>;1079 1080defm FLAT_ATOMIC_FMAX : FLAT_Atomic_Pseudo <"flat_atomic_fmax",1081 AVLdSt_32, f32>;1082 1083} // End SubtargetPredicate = isGFX7GFX10GFX111084 1085// GFX942-, GFX11-only flat instructions.1086let SubtargetPredicate = HasFlatAtomicFaddF32Inst in {1087 defm FLAT_ATOMIC_ADD_F32 : FLAT_Atomic_Pseudo<"flat_atomic_add_f32", AVLdSt_32, f32>;1088} // End SubtargetPredicate = HasFlatAtomicFaddF32Inst1089 1090let SubtargetPredicate = isGFX12Plus in {1091 defm FLAT_ATOMIC_CSUB_U32 : FLAT_Atomic_Pseudo <"flat_atomic_csub_u32", VGPROp_32, i32>;1092 defm FLAT_ATOMIC_COND_SUB_U32 : FLAT_Atomic_Pseudo_RTN<"flat_atomic_cond_sub_u32", VGPROp_32, i32>;1093}1094 1095let SubtargetPredicate = HasAtomicCSubNoRtnInsts in {1096 defm FLAT_ATOMIC_COND_SUB_U32 : FLAT_Atomic_Pseudo_NO_RTN<"flat_atomic_cond_sub_u32", VGPROp_32, i32>;1097}1098 1099defm GLOBAL_LOAD_UBYTE : FLAT_Global_Load_Pseudo <"global_load_ubyte">;1100defm GLOBAL_LOAD_SBYTE : FLAT_Global_Load_Pseudo <"global_load_sbyte">;1101defm GLOBAL_LOAD_USHORT : FLAT_Global_Load_Pseudo <"global_load_ushort">;1102defm GLOBAL_LOAD_SSHORT : FLAT_Global_Load_Pseudo <"global_load_sshort">;1103defm GLOBAL_LOAD_DWORD : FLAT_Global_Load_Pseudo <"global_load_dword">;1104defm GLOBAL_LOAD_DWORDX2 : FLAT_Global_Load_Pseudo <"global_load_dwordx2", AVLdSt_64>;1105defm GLOBAL_LOAD_DWORDX3 : FLAT_Global_Load_Pseudo <"global_load_dwordx3", AVLdSt_96>;1106defm GLOBAL_LOAD_DWORDX4 : FLAT_Global_Load_Pseudo <"global_load_dwordx4", AVLdSt_128>;1107 1108let TiedSourceNotRead = 1 in {1109defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16_hi", AVLdSt_32, 1>;1110defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Global_Load_Pseudo <"global_load_short_d16_hi", AVLdSt_32, 1>;1111defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16_hi", AVLdSt_32, 1>;1112defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Global_Load_Pseudo_t16 <"global_load_sbyte_d16">;1113defm GLOBAL_LOAD_SHORT_D16 : FLAT_Global_Load_Pseudo_t16 <"global_load_short_d16">;1114defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Global_Load_Pseudo_t16 <"global_load_ubyte_d16">;1115}1116 1117defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Global_Store_Pseudo <"global_store_byte_d16_hi">;1118defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Global_Store_Pseudo <"global_store_short_d16_hi">;1119 1120let OtherPredicates = [HasGFX10_BEncoding] in1121defm GLOBAL_LOAD_DWORD_ADDTID : FLAT_Global_Load_AddTid_Pseudo <"global_load_dword_addtid", VGPROp_32>;1122 1123defm GLOBAL_STORE_BYTE : FLAT_Global_Store_Pseudo_t16 <"global_store_byte">;1124defm GLOBAL_STORE_SHORT : FLAT_Global_Store_Pseudo_t16 <"global_store_short">;1125defm GLOBAL_STORE_DWORD : FLAT_Global_Store_Pseudo <"global_store_dword">;1126defm GLOBAL_STORE_DWORDX2 : FLAT_Global_Store_Pseudo <"global_store_dwordx2", AVLdSt_64>;1127defm GLOBAL_STORE_DWORDX3 : FLAT_Global_Store_Pseudo <"global_store_dwordx3", AVLdSt_96>;1128defm GLOBAL_STORE_DWORDX4 : FLAT_Global_Store_Pseudo <"global_store_dwordx4", AVLdSt_128>;1129let OtherPredicates = [HasGFX10_BEncoding] in1130defm GLOBAL_STORE_DWORD_ADDTID : FLAT_Global_Store_AddTid_Pseudo <"global_store_dword_addtid", VGPROp_32>;1131 1132defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap",1133 AVLdSt_32, i32, v2i32, AVLdSt_64>;1134 1135defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap_x2",1136 AVLdSt_64, i64, v2i64, AVLdSt_128>;1137 1138defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_swap",1139 AVLdSt_32, i32>;1140 1141defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_swap_x2",1142 AVLdSt_64, i64>;1143 1144defm GLOBAL_ATOMIC_ADD : FLAT_Global_Atomic_Pseudo <"global_atomic_add",1145 AVLdSt_32, i32>;1146 1147defm GLOBAL_ATOMIC_SUB : FLAT_Global_Atomic_Pseudo <"global_atomic_sub",1148 AVLdSt_32, i32>;1149 1150defm GLOBAL_ATOMIC_SMIN : FLAT_Global_Atomic_Pseudo <"global_atomic_smin",1151 AVLdSt_32, i32>;1152 1153defm GLOBAL_ATOMIC_UMIN : FLAT_Global_Atomic_Pseudo <"global_atomic_umin",1154 AVLdSt_32, i32>;1155 1156defm GLOBAL_ATOMIC_SMAX : FLAT_Global_Atomic_Pseudo <"global_atomic_smax",1157 AVLdSt_32, i32>;1158 1159defm GLOBAL_ATOMIC_UMAX : FLAT_Global_Atomic_Pseudo <"global_atomic_umax",1160 AVLdSt_32, i32>;1161 1162defm GLOBAL_ATOMIC_AND : FLAT_Global_Atomic_Pseudo <"global_atomic_and",1163 AVLdSt_32, i32>;1164 1165defm GLOBAL_ATOMIC_OR : FLAT_Global_Atomic_Pseudo <"global_atomic_or",1166 AVLdSt_32, i32>;1167 1168defm GLOBAL_ATOMIC_XOR : FLAT_Global_Atomic_Pseudo <"global_atomic_xor",1169 AVLdSt_32, i32>;1170 1171defm GLOBAL_ATOMIC_INC : FLAT_Global_Atomic_Pseudo <"global_atomic_inc",1172 AVLdSt_32, i32>;1173 1174defm GLOBAL_ATOMIC_DEC : FLAT_Global_Atomic_Pseudo <"global_atomic_dec",1175 AVLdSt_32, i32>;1176 1177defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_add_x2",1178 AVLdSt_64, i64>;1179 1180defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_sub_x2",1181 AVLdSt_64, i64>;1182 1183defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_smin_x2",1184 AVLdSt_64, i64>;1185 1186defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_umin_x2",1187 AVLdSt_64, i64>;1188 1189defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_smax_x2",1190 AVLdSt_64, i64>;1191 1192defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_umax_x2",1193 AVLdSt_64, i64>;1194 1195defm GLOBAL_ATOMIC_AND_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_and_x2",1196 AVLdSt_64, i64>;1197 1198defm GLOBAL_ATOMIC_OR_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_or_x2",1199 AVLdSt_64, i64>;1200 1201defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_xor_x2",1202 AVLdSt_64, i64>;1203 1204defm GLOBAL_ATOMIC_INC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_inc_x2",1205 AVLdSt_64, i64>;1206 1207defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_dec_x2",1208 AVLdSt_64, i64>;1209 1210let SubtargetPredicate = HasGFX10_BEncoding in {1211 defm GLOBAL_ATOMIC_CSUB : FLAT_Global_Atomic_Pseudo <"global_atomic_csub",1212 VGPROp_32, i32>;1213}1214 1215defm GLOBAL_LOAD_LDS_UBYTE : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_ubyte">;1216defm GLOBAL_LOAD_LDS_SBYTE : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_sbyte">;1217defm GLOBAL_LOAD_LDS_USHORT : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_ushort">;1218defm GLOBAL_LOAD_LDS_SSHORT : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_sshort">;1219defm GLOBAL_LOAD_LDS_DWORD : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_dword">;1220 1221let SubtargetPredicate = HasGFX950Insts in {1222defm GLOBAL_LOAD_LDS_DWORDX3 : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_dwordx3">;1223defm GLOBAL_LOAD_LDS_DWORDX4 : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_dwordx4">;1224}1225 1226let SubtargetPredicate = isGFX12PlusNot12_50 in1227 defm GLOBAL_ATOMIC_ORDERED_ADD_B64 : FLAT_Global_Atomic_Pseudo <"global_atomic_ordered_add_b64", VGPROp_64, i64>;1228 1229let SubtargetPredicate = isGFX12Plus in {1230 defm GLOBAL_ATOMIC_COND_SUB_U32 : FLAT_Global_Atomic_Pseudo <"global_atomic_cond_sub_u32", VGPROp_32, i32>;1231 1232 def GLOBAL_INV : FLAT_Global_Invalidate_Writeback<"global_inv">;1233 def GLOBAL_WB : FLAT_Global_Invalidate_Writeback<"global_wb">;1234 def GLOBAL_WBINV : FLAT_Global_Invalidate_Writeback<"global_wbinv">;1235} // End SubtargetPredicate = isGFX12Plus1236 1237let SubtargetPredicate = isGFX1250Plus in {1238 1239let Uses = [M0, EXEC, ASYNCcnt], WaveSizePredicate = isWave32 in {1240defm CLUSTER_LOAD_ASYNC_TO_LDS_B8 : FLAT_Global_Load_LDS_Pseudo<"cluster_load_async_to_lds_b8", 1>;1241defm CLUSTER_LOAD_ASYNC_TO_LDS_B32 : FLAT_Global_Load_LDS_Pseudo<"cluster_load_async_to_lds_b32", 1>;1242defm CLUSTER_LOAD_ASYNC_TO_LDS_B64 : FLAT_Global_Load_LDS_Pseudo<"cluster_load_async_to_lds_b64", 1>;1243defm CLUSTER_LOAD_ASYNC_TO_LDS_B128 : FLAT_Global_Load_LDS_Pseudo<"cluster_load_async_to_lds_b128", 1>;1244} // End Uses = [M0, EXEC, ASYNCcnt], WaveSizePredicate = isWave321245defm GLOBAL_LOAD_ASYNC_TO_LDS_B8 : FLAT_Global_Load_LDS_Pseudo<"global_load_async_to_lds_b8", 1>;1246defm GLOBAL_LOAD_ASYNC_TO_LDS_B32 : FLAT_Global_Load_LDS_Pseudo<"global_load_async_to_lds_b32", 1>;1247defm GLOBAL_LOAD_ASYNC_TO_LDS_B64 : FLAT_Global_Load_LDS_Pseudo<"global_load_async_to_lds_b64", 1>;1248defm GLOBAL_LOAD_ASYNC_TO_LDS_B128 : FLAT_Global_Load_LDS_Pseudo<"global_load_async_to_lds_b128", 1>;1249defm GLOBAL_STORE_ASYNC_FROM_LDS_B8 : FLAT_Global_STORE_LDS_Pseudo<"global_store_async_from_lds_b8">;1250defm GLOBAL_STORE_ASYNC_FROM_LDS_B32 : FLAT_Global_STORE_LDS_Pseudo<"global_store_async_from_lds_b32">;1251defm GLOBAL_STORE_ASYNC_FROM_LDS_B64 : FLAT_Global_STORE_LDS_Pseudo<"global_store_async_from_lds_b64">;1252defm GLOBAL_STORE_ASYNC_FROM_LDS_B128 : FLAT_Global_STORE_LDS_Pseudo<"global_store_async_from_lds_b128">;1253 1254def TENSOR_SAVE : FLAT_Global_Tensor_Pseudo<"tensor_save", 1>;1255def TENSOR_STOP : FLAT_Global_Tensor_Pseudo<"tensor_stop">;1256} // End SubtargetPredicate = isGFX1250Plus1257 1258defm SCRATCH_LOAD_UBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte">;1259defm SCRATCH_LOAD_SBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte">;1260defm SCRATCH_LOAD_USHORT : FLAT_Scratch_Load_Pseudo <"scratch_load_ushort">;1261defm SCRATCH_LOAD_SSHORT : FLAT_Scratch_Load_Pseudo <"scratch_load_sshort">;1262defm SCRATCH_LOAD_DWORD : FLAT_Scratch_Load_Pseudo <"scratch_load_dword">;1263defm SCRATCH_LOAD_DWORDX2 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx2", AVLdSt_64>;1264defm SCRATCH_LOAD_DWORDX3 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx3", AVLdSt_96>;1265defm SCRATCH_LOAD_DWORDX4 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx4", AVLdSt_128>;1266 1267let TiedSourceNotRead = 1 in {1268defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16_hi", AVLdSt_32, 1>;1269defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16_hi", AVLdSt_32, 1>;1270defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16_hi", AVLdSt_32, 1>;1271defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Scratch_Load_Pseudo_t16 <"scratch_load_ubyte_d16">;1272defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Scratch_Load_Pseudo_t16 <"scratch_load_sbyte_d16">;1273defm SCRATCH_LOAD_SHORT_D16 : FLAT_Scratch_Load_Pseudo_t16 <"scratch_load_short_d16">;1274}1275 1276defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_byte_d16_hi">;1277defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_short_d16_hi">;1278 1279defm SCRATCH_STORE_BYTE : FLAT_Scratch_Store_Pseudo_t16 <"scratch_store_byte">;1280defm SCRATCH_STORE_SHORT : FLAT_Scratch_Store_Pseudo_t16 <"scratch_store_short">;1281defm SCRATCH_STORE_DWORD : FLAT_Scratch_Store_Pseudo <"scratch_store_dword">;1282defm SCRATCH_STORE_DWORDX2 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx2", AVLdSt_64>;1283defm SCRATCH_STORE_DWORDX3 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx3", AVLdSt_96>;1284defm SCRATCH_STORE_DWORDX4 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx4", AVLdSt_128>;1285 1286defm SCRATCH_LOAD_LDS_UBYTE : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_ubyte">;1287defm SCRATCH_LOAD_LDS_SBYTE : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_sbyte">;1288defm SCRATCH_LOAD_LDS_USHORT : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_ushort">;1289defm SCRATCH_LOAD_LDS_SSHORT : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_sshort">;1290defm SCRATCH_LOAD_LDS_DWORD : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_dword">;1291 1292let SubtargetPredicate = isGFX125xOnly in {1293defm FLAT_LOAD_MONITOR_B32 : FLAT_Flat_Load_Pseudo <"flat_load_monitor_b32", VGPROp_32>;1294defm FLAT_LOAD_MONITOR_B64 : FLAT_Flat_Load_Pseudo <"flat_load_monitor_b64", VGPROp_64>;1295defm FLAT_LOAD_MONITOR_B128 : FLAT_Flat_Load_Pseudo <"flat_load_monitor_b128", VGPROp_128>;1296 1297defm GLOBAL_LOAD_MONITOR_B32 : FLAT_Global_Load_Pseudo <"global_load_monitor_b32", VGPROp_32>;1298defm GLOBAL_LOAD_MONITOR_B64 : FLAT_Global_Load_Pseudo <"global_load_monitor_b64", VGPROp_64>;1299defm GLOBAL_LOAD_MONITOR_B128 : FLAT_Global_Load_Pseudo <"global_load_monitor_b128", VGPROp_128>;1300} // End SubtargetPredicate = isGFX125xOnly1301 1302let SubtargetPredicate = isGFX1250Plus, WaveSizePredicate = isWave32 in {1303let Uses = [M0, EXEC] in { // Use M0 for broadcast workgroup mask.1304defm CLUSTER_LOAD_B32 : FLAT_Global_Load_Pseudo <"cluster_load_b32", VGPROp_32>;1305defm CLUSTER_LOAD_B64 : FLAT_Global_Load_Pseudo <"cluster_load_b64", VGPROp_64>;1306defm CLUSTER_LOAD_B128 : FLAT_Global_Load_Pseudo <"cluster_load_b128", VGPROp_128>;1307} // End Uses = [M0, EXEC]1308} // End SubtargetPredicate = isGFX1250Plus, WaveSizePredicate = isWave321309 1310let SubtargetPredicate = isGFX12Plus in {1311 let Uses = [EXEC, M0] in {1312 defm GLOBAL_LOAD_BLOCK : FLAT_Global_Load_Pseudo <"global_load_block", VGPROp_1024>;1313 defm GLOBAL_STORE_BLOCK : FLAT_Global_Store_Pseudo <"global_store_block", VGPROp_1024>;1314 }1315 let Uses = [EXEC, FLAT_SCR, M0] in {1316 defm SCRATCH_LOAD_BLOCK : FLAT_Scratch_Load_Pseudo <"scratch_load_block", VGPROp_1024>;1317 defm SCRATCH_STORE_BLOCK : FLAT_Scratch_Store_Pseudo <"scratch_store_block", VGPROp_1024>;1318 }1319 1320 let WaveSizePredicate = isWave32 in {1321 defm GLOBAL_LOAD_TR_B128_w32 : FLAT_Global_Load_Pseudo <"global_load_tr_b128", VGPROp_128>;1322 defm GLOBAL_LOAD_TR_B64_w32 : FLAT_Global_Load_Pseudo <"global_load_tr_b64", VGPROp_64>;1323 }1324} // End SubtargetPredicate = isGFX12Plus1325 1326let WaveSizePredicate = isWave64, SubtargetPredicate = isGFX12PlusNot12_50 in {1327 let Mnemonic = "global_load_tr_b128" in1328 defm GLOBAL_LOAD_TR_B128_w64 : FLAT_Global_Load_Pseudo <"global_load_tr_b128_w64", VGPROp_64>;1329 let Mnemonic = "global_load_tr_b64" in1330 defm GLOBAL_LOAD_TR_B64_w64 : FLAT_Global_Load_Pseudo <"global_load_tr_b64_w64", VGPROp_32>;1331}1332 1333let WaveSizePredicate = isWave32, SubtargetPredicate = HasTransposeLoadF4F6Insts in {1334 defm GLOBAL_LOAD_TR6_B96 : FLAT_Global_Load_Pseudo <"global_load_tr6_b96", VGPROp_96_Align1>;1335 defm GLOBAL_LOAD_TR4_B64 : FLAT_Global_Load_Pseudo <"global_load_tr4_b64", VGPROp_64>;1336}1337 1338let SubtargetPredicate = isGFX10Plus in {1339 defm GLOBAL_ATOMIC_FCMPSWAP :1340 FLAT_Global_Atomic_Pseudo<"global_atomic_fcmpswap", AVLdSt_32, f32, v2f32, AVLdSt_64>;1341 defm GLOBAL_ATOMIC_FMIN :1342 FLAT_Global_Atomic_Pseudo<"global_atomic_fmin", AVLdSt_32, f32>;1343 defm GLOBAL_ATOMIC_FMAX :1344 FLAT_Global_Atomic_Pseudo<"global_atomic_fmax", AVLdSt_32, f32>;1345 defm GLOBAL_ATOMIC_FCMPSWAP_X2 :1346 FLAT_Global_Atomic_Pseudo<"global_atomic_fcmpswap_x2", AVLdSt_64, f64, v2f64, AVLdSt_128>;1347} // End SubtargetPredicate = isGFX10Plus1348 1349let SubtargetPredicate = HasAtomicFaddNoRtnInsts in1350 defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Atomic_Pseudo_Helper_NO_RTN <1351 "global_atomic_add_f32", AVLdSt_32, f321352 >;1353let SubtargetPredicate = HasAtomicBufferGlobalPkAddF16NoRtnInsts in1354 defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Atomic_Pseudo_Helper_NO_RTN <1355 "global_atomic_pk_add_f16", AVLdSt_32, v2f161356 >;1357let SubtargetPredicate = HasAtomicFaddRtnInsts in1358 defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Atomic_Pseudo_Helper_RTN <1359 "global_atomic_add_f32", AVLdSt_32, f321360 >;1361let SubtargetPredicate = HasAtomicBufferGlobalPkAddF16Insts in1362 defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Atomic_Pseudo_Helper_RTN <1363 "global_atomic_pk_add_f16", AVLdSt_32, v2f161364 >;1365 1366let SubtargetPredicate = HasVmemPrefInsts in {1367 defm FLAT_PREFETCH_B8 : FLAT_Flat_Prefetch_Pseudo<"flat_prefetch_b8">;1368 defm GLOBAL_PREFETCH_B8 : FLAT_Global_Prefetch_Pseudo<"global_prefetch_b8">;1369}1370 1371//===----------------------------------------------------------------------===//1372// Flat Patterns1373//===----------------------------------------------------------------------===//1374 1375// Patterns for global loads with no offset.1376class FlatLoadPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1377 (vt (node (FlatOffset i64:$vaddr, i32:$offset))),1378 (inst $vaddr, $offset)1379>;1380 1381class FlatLoadPat_CPOL <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1382 (vt (node (FlatOffset i64:$vaddr, i32:$offset), (i32 timm:$cpol))),1383 (inst $vaddr, $offset, $cpol)1384>;1385 1386class FlatLoadPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1387 (node (FlatOffset (i64 VReg_64:$vaddr), i32:$offset), vt:$in),1388 (inst $vaddr, $offset, 0, $in)1389>;1390 1391class FlatLoadPat_D16_t16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1392 (vt (node (FlatOffset (i64 VReg_64:$vaddr), i32:$offset))),1393 (inst $vaddr, $offset, (i32 0))1394>;1395 1396class FlatLoadPat_t16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1397 (vt (node (FlatOffset i64:$vaddr, i32:$offset))),1398 (EXTRACT_SUBREG (inst $vaddr, $offset), lo16)1399>;1400 1401class FlatSignedLoadPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1402 (node (GlobalOffset (i64 VReg_64:$vaddr), i32:$offset), vt:$in),1403 (inst $vaddr, $offset, 0, $in)1404>;1405 1406class FlatSignedLoadPat_D16_t16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1407 (vt (node (GlobalOffset (i64 VReg_64:$vaddr), i32:$offset))),1408 (inst $vaddr, $offset, (i32 0))1409>;1410 1411class FlatSignedLoadPat_t16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1412 (vt (node (GlobalOffset (i64 VReg_64:$vaddr), i32:$offset))),1413 (EXTRACT_SUBREG (inst $vaddr, $offset, (i32 0)), lo16)1414>;1415 1416class GlobalLoadSaddrPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1417 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset, CPol:$cpol), vt:$in)),1418 (inst $saddr, $voffset, $offset, $cpol, $in)1419>;1420 1421class FlatLoadSaddrPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1422 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset, CPol:$cpol), vt:$in)),1423 (inst $saddr, $voffset, $offset, $cpol, $in)1424>;1425 1426class FlatLoadSaddrPat_D16_t16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1427 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset, CPol:$cpol))),1428 (inst $saddr, $voffset, $offset, $cpol)1429>;1430 1431class FlatLoadSaddrPat_t16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1432 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset, CPol:$cpol))),1433 (EXTRACT_SUBREG (inst $saddr, $voffset, $offset, $cpol), lo16)1434>;1435 1436class FlatLoadLDSSignedPat_M0 <FLAT_Pseudo inst, SDPatternOperator node> : GCNPat <1437 (node (i64 VReg_64:$vaddr), (i32 VGPR_32:$dsaddr), (i32 timm:$offset), (i32 timm:$cpol), M0),1438 (inst $dsaddr, $vaddr, $offset, $cpol)1439>;1440 1441class GlobalLoadLDSSaddrPat_M0 <FLAT_Pseudo inst, SDPatternOperator node> : GCNPat <1442 (node (GlobalSAddrNoIOffsetM0 (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), CPol:$cpol), (i32 VGPR_32:$dsaddr), (i32 timm:$offset), (i32 timm), M0),1443 (inst $dsaddr, $saddr, $voffset, $offset, $cpol)1444>;1445 1446class FlatLoadLDSSignedPat <FLAT_Pseudo inst, SDPatternOperator node> : GCNPat <1447 (node (i64 VReg_64:$vaddr), (i32 VGPR_32:$dsaddr), (i32 timm:$offset), (i32 timm:$cpol)),1448 (inst $dsaddr, $vaddr, $offset, $cpol)1449>;1450 1451class GlobalLoadLDSSaddrPat <FLAT_Pseudo inst, SDPatternOperator node> : GCNPat <1452 (node (GlobalSAddrNoIOffset (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), CPol:$cpol), (i32 VGPR_32:$dsaddr), (i32 timm:$offset), (i32 timm)),1453 (inst $dsaddr, $saddr, $voffset, $offset, $cpol)1454>;1455 1456class FlatStoreLDSSignedPat <FLAT_Pseudo inst, SDPatternOperator node> : GCNPat <1457 (node (i64 VReg_64:$vaddr), (i32 VGPR_32:$dsaddr), (i32 timm:$offset), (i32 timm:$cpol)),1458 (inst $vaddr, $dsaddr, $offset, $cpol)1459>;1460 1461class GlobalStoreLDSSaddrPat <FLAT_Pseudo inst, SDPatternOperator node> : GCNPat <1462 (node (GlobalSAddrNoIOffset (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), CPol:$cpol), (i32 VGPR_32:$dsaddr), (i32 timm:$offset), (i32 timm)),1463 (inst $saddr, $voffset, $dsaddr, $offset, $cpol)1464>;1465 1466class GlobalLoadSaddrPat_D16_t16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1467 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset, CPol:$cpol))),1468 (inst $saddr, $voffset, $offset, $cpol)1469>;1470 1471class GlobalLoadSaddrPat_t16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1472 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset, CPol:$cpol))),1473 (EXTRACT_SUBREG (inst $saddr, $voffset, $offset, $cpol), lo16)1474>;1475 1476class FlatLoadSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1477 (vt (node (GlobalOffset (i64 VReg_64:$vaddr), i32:$offset))),1478 (inst $vaddr, $offset)1479>;1480 1481class FlatLoadSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1482 (vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset, CPol:$cpol))),1483 (inst $saddr, $voffset, $offset, $cpol)1484>;1485 1486class FlatLoadSignedPat_M0 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1487 (vt (node (GlobalOffset (i64 VReg_64:$vaddr), i32:$offset), (i32 timm:$cpol), M0)),1488 (inst $vaddr, $offset, $cpol)1489>;1490 1491class GlobalLoadSaddrPat_M0 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1492 (vt (node (GlobalSAddrCPolM0 (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset, CPol:$cpol), (i32 timm), M0)),1493 (inst $saddr, $voffset, $offset, $cpol)1494>;1495 1496class FlatLoadSignedPat_CPOL <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1497 (vt (node (GlobalOffset (i64 VReg_64:$vaddr), i32:$offset), (i32 timm:$cpol))),1498 (inst $vaddr, $offset, $cpol)1499>;1500 1501class GlobalLoadSaddrPat_CPOL <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1502 (vt (node (GlobalSAddrCPol (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset, CPol:$cpol), (i32 timm))),1503 (inst $saddr, $voffset, $offset, $cpol)1504>;1505 1506class FlatStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node,1507 ValueType vt> : GCNPat <1508 (node vt:$data, (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset, CPol:$cpol)),1509 (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset, $cpol)1510>;1511 1512class FlatAtomicSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, ComplexPattern pat,1513 ValueType vt, ValueType data_vt = vt> : GCNPat <1514 (vt (node (pat (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset, CPol:$cpol), data_vt:$data)),1515 (inst $voffset, getVregSrcForVT<data_vt>.ret:$data, $saddr, $offset, $cpol)> {1516 let SubtargetPredicate = inst.SubtargetPredicate;1517 let OtherPredicates = inst.OtherPredicates;1518}1519 1520class GlobalAtomicNoRtnSaddrPat <FLAT_Pseudo inst, SDPatternOperator node,1521 ValueType vt> : GCNPat <1522 (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset, CPol:$cpol), vt:$data),1523 (inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset, $cpol)1524>;1525 1526class FlatStorePat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1527 (node vt:$data, (FlatOffset i64:$vaddr, i32:$offset)),1528 (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset)1529>;1530 1531class FlatStoreSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1532 (node vt:$data, (GlobalOffset i64:$vaddr, i32:$offset)),1533 (inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset)1534>;1535 1536class FlatStoreSignedAtomicPat <FLAT_Pseudo inst, SDPatternOperator node,1537 ValueType vt, ValueType data_vt = vt> : GCNPat <1538 // atomic store follows atomic binop convention so the address comes1539 // first.1540 (node (GlobalOffset i64:$vaddr, i32:$offset), data_vt:$data),1541 (inst $vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset)1542>;1543 1544multiclass FlatAtomicNoRtnPatBase <string base_inst_name, string node, ValueType vt,1545 ValueType data_vt = vt> {1546 defvar inst = !cast<FLAT_Pseudo>(base_inst_name);1547 defvar inst_saddr = !cast<FLAT_Pseudo>(inst#"_SADDR");1548 defvar noRtnNode = !cast<PatFrags>(node);1549 1550 let AddedComplexity = 1 in1551 def : GCNPat <(vt (noRtnNode (FlatOffset i64:$vaddr, i32:$offset), data_vt:$data)),1552 (inst VReg_64_AlignTarget:$vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset)> {1553 let SubtargetPredicate = inst.SubtargetPredicate;1554 let OtherPredicates = inst.OtherPredicates;1555 }1556 1557 def : FlatAtomicSaddrPat<inst_saddr, !cast<SDPatternOperator>(node),1558 GlobalSAddr, vt, data_vt> {1559 let AddedComplexity = 9;1560 let SubtargetPredicate = inst_saddr.SubtargetPredicate;1561 let OtherPredicates = inst_saddr.OtherPredicates;1562 }1563}1564 1565multiclass FlatAtomicNoRtnPatWithAddrSpace<string inst, string node, string addrSpaceSuffix,1566 ValueType vt> :1567 FlatAtomicNoRtnPatBase<inst, node # "_noret_" # addrSpaceSuffix, vt, vt>;1568 1569multiclass FlatAtomicNoRtnPat <string inst, string node, ValueType vt,1570 ValueType data_vt = vt, bit isIntr = 0> :1571 FlatAtomicNoRtnPatBase<inst, node # "_noret" # !if(isIntr, "", "_"#vt), vt, data_vt>;1572 1573 1574multiclass FlatAtomicRtnPatBase <string inst_name, string node, ValueType vt,1575 ValueType data_vt = vt> {1576 defvar inst = !cast<FLAT_Pseudo>(inst_name#"_RTN");1577 defvar inst_saddr = !cast<FLAT_Pseudo>(inst_name#"_SADDR_RTN");1578 defvar rtnNode = !cast<SDPatternOperator>(node);1579 1580 def : GCNPat <(vt (rtnNode (FlatOffset i64:$vaddr, i32:$offset), data_vt:$data)),1581 (inst VReg_64_AlignTarget:$vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset)> {1582 let SubtargetPredicate = inst.SubtargetPredicate;1583 let OtherPredicates = inst.OtherPredicates;1584 }1585 1586 def : FlatAtomicSaddrPat<inst_saddr, rtnNode, GlobalSAddrGLC, vt, data_vt> {1587 let AddedComplexity = 8;1588 let SubtargetPredicate = inst_saddr.SubtargetPredicate;1589 let OtherPredicates = inst_saddr.OtherPredicates;1590 }1591}1592 1593multiclass FlatAtomicRtnPatWithAddrSpace<string inst, string intr, string addrSpaceSuffix,1594 ValueType vt> :1595 FlatAtomicRtnPatBase<inst, intr # "_" # addrSpaceSuffix, vt, vt>;1596 1597multiclass FlatAtomicRtnPat <string inst, string node, ValueType vt,1598 ValueType data_vt = vt, bit isIntr = 0> :1599 FlatAtomicRtnPatBase<inst, node # !if(isIntr, "", "_"#vt), vt, data_vt>;1600 1601 1602multiclass FlatAtomicPat <string inst, string node, ValueType vt,1603 ValueType data_vt = vt, bit isIntr = 0> :1604 FlatAtomicRtnPat<inst, node, vt, data_vt, isIntr>,1605 FlatAtomicNoRtnPat<inst, node, vt, data_vt, isIntr>;1606 1607multiclass FlatAtomicIntrNoRtnPat <string inst, string node, ValueType vt,1608 ValueType data_vt = vt> {1609 defm : FlatAtomicNoRtnPat<inst, node, vt, data_vt, /* isIntr */ 1>;1610}1611 1612multiclass FlatAtomicIntrRtnPat <string inst, string node, ValueType vt,1613 ValueType data_vt = vt> {1614 defm : FlatAtomicRtnPat<inst, node, vt, data_vt, /* isIntr */ 1>;1615}1616 1617multiclass FlatAtomicIntrPat <string inst, string node, ValueType vt,1618 ValueType data_vt = vt> :1619 FlatAtomicRtnPat<inst, node, vt, data_vt, /* isIntr */ 1>,1620 FlatAtomicNoRtnPat<inst, node, vt, data_vt, /* isIntr */ 1>;1621 1622class FlatSignedAtomicPatBase <FLAT_Pseudo inst, SDPatternOperator node,1623 ValueType vt, ValueType data_vt = vt> : GCNPat <1624 (vt (node (GlobalOffset i64:$vaddr, i32:$offset), data_vt:$data)),1625 (inst VReg_64_AlignTarget:$vaddr, getVregSrcForVT<data_vt>.ret:$data, $offset)> {1626 let SubtargetPredicate = inst.SubtargetPredicate;1627 let OtherPredicates = inst.OtherPredicates;1628}1629 1630multiclass FlatSignedAtomicPat <string inst, string node, ValueType vt,1631 ValueType data_vt = vt, int complexity = 0,1632 bit isIntr = 0> {1633 defvar rtnNode = !cast<SDPatternOperator>(node # !if(isIntr, "", "_" # vt));1634 defvar noRtnNode = !cast<PatFrags>(node # "_noret" # !if(isIntr, "", "_" # vt));1635 1636 let AddedComplexity = complexity in1637 def : FlatSignedAtomicPatBase<!cast<FLAT_Pseudo>(inst#"_RTN"), rtnNode, vt, data_vt>;1638 1639 let AddedComplexity = !add(complexity, 1) in1640 def : FlatSignedAtomicPatBase<!cast<FLAT_Pseudo>(inst), noRtnNode, vt, data_vt>;1641}1642 1643class ScratchLoadSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1644 (vt (node (ScratchOffset (i32 VGPR_32:$vaddr), i32:$offset))),1645 (inst $vaddr, $offset)1646>;1647 1648class ScratchLoadSignedPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1649 (node (ScratchOffset (i32 VGPR_32:$vaddr), i32:$offset), vt:$in),1650 (inst $vaddr, $offset, 0, $in)1651>;1652 1653class ScratchLoadSignedPat_D16_t16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1654 (vt (node (ScratchOffset (i32 VGPR_32:$vaddr), i32:$offset))),1655 (inst $vaddr, $offset, 0)1656>;1657 1658class ScratchLoadSignedPat_t16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1659 (vt (node (ScratchOffset (i32 VGPR_32:$vaddr), i32:$offset))),1660 (EXTRACT_SUBREG (inst $vaddr, $offset), lo16)1661>;1662 1663class ScratchStoreSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1664 (node vt:$data, (ScratchOffset (i32 VGPR_32:$vaddr), i32:$offset)),1665 (inst getVregSrcForVT<vt>.ret:$data, $vaddr, $offset)1666>;1667 1668class ScratchLoadSaddrPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1669 (vt (node (ScratchSAddr (i32 SGPR_32:$saddr), i32:$offset))),1670 (inst $saddr, $offset)1671>;1672 1673class ScratchLoadSaddrPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1674 (vt (node (ScratchSAddr (i32 SGPR_32:$saddr), i32:$offset), vt:$in)),1675 (inst $saddr, $offset, 0, $in)1676>;1677 1678class ScratchLoadSaddrPat_D16_t16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1679 (vt (node (ScratchSAddr (i32 SGPR_32:$saddr), i32:$offset))),1680 (inst $saddr, $offset, 0)1681>;1682 1683class ScratchLoadSaddrPat_t16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1684 (vt (node (ScratchSAddr (i32 SGPR_32:$saddr), i32:$offset))),1685 (EXTRACT_SUBREG (inst $saddr, $offset), lo16)1686>;1687 1688class ScratchStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node,1689 ValueType vt> : GCNPat <1690 (node vt:$data, (ScratchSAddr (i32 SGPR_32:$saddr), i32:$offset)),1691 (inst getVregSrcForVT<vt>.ret:$data, $saddr, $offset)1692>;1693 1694class ScratchLoadSVaddrPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1695 (vt (node (ScratchSVAddr (i32 VGPR_32:$vaddr), (i32 SGPR_32:$saddr), i32:$offset, CPol:$cpol))),1696 (inst $vaddr, $saddr, $offset, $cpol)1697>;1698 1699class ScratchStoreSVaddrPat <FLAT_Pseudo inst, SDPatternOperator node,1700 ValueType vt> : GCNPat <1701 (node vt:$data, (ScratchSVAddr (i32 VGPR_32:$vaddr), (i32 SGPR_32:$saddr), i32:$offset, CPol:$cpol)),1702 (inst getVregSrcForVT<vt>.ret:$data, $vaddr, $saddr, $offset, $cpol)1703>;1704 1705class ScratchLoadSVaddrPat_D16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1706 (vt (node (ScratchSVAddr (i32 VGPR_32:$vaddr), (i32 SGPR_32:$saddr), i32:$offset, CPol:$cpol), vt:$in)),1707 (inst $vaddr, $saddr, $offset, $cpol, $in)1708>;1709 1710class ScratchLoadSVaddrPat_D16_t16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1711 (vt (node (ScratchSVAddr (i32 VGPR_32:$vaddr), (i32 SGPR_32:$saddr), i32:$offset, CPol:$cpol))),1712 (inst $vaddr, $saddr, $offset, $cpol)1713>;1714 1715class ScratchLoadSVaddrPat_t16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <1716 (vt (node (ScratchSVAddr (i32 VGPR_32:$vaddr), (i32 SGPR_32:$saddr), i32:$offset, CPol:$cpol))),1717 (EXTRACT_SUBREG (inst $vaddr, $saddr, $offset, $cpol), lo16)1718>;1719 1720multiclass GlobalLoadLDSPats_M0<FLAT_Pseudo inst, SDPatternOperator node> {1721 def : FlatLoadLDSSignedPat_M0 <inst, node> {1722 let AddedComplexity = 10;1723 }1724 1725 def : GlobalLoadLDSSaddrPat_M0<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node> {1726 let AddedComplexity = 11;1727 }1728}1729 1730multiclass GlobalLoadLDSPats<FLAT_Pseudo inst, SDPatternOperator node> {1731 def : FlatLoadLDSSignedPat <inst, node> {1732 let AddedComplexity = 10;1733 }1734 1735 def : GlobalLoadLDSSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node> {1736 let AddedComplexity = 11;1737 }1738}1739 1740multiclass GlobalStoreLDSPats<FLAT_Pseudo inst, SDPatternOperator node> {1741 def : FlatStoreLDSSignedPat <inst, node> {1742 let AddedComplexity = 10;1743 }1744 1745 def : GlobalStoreLDSSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node> {1746 let AddedComplexity = 11;1747 }1748}1749 1750multiclass GlobalFLATLoadPats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> {1751 def : FlatLoadSignedPat <inst, node, vt> {1752 let AddedComplexity = 10;1753 let SubtargetPredicate = inst.SubtargetPredicate;1754 let OtherPredicates = inst.OtherPredicates;1755 }1756 1757 def : FlatLoadSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> {1758 let AddedComplexity = 11;1759 let SubtargetPredicate = inst.SubtargetPredicate;1760 let OtherPredicates = inst.OtherPredicates;1761 }1762}1763 1764multiclass GlobalFLATLoadPats_M0<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> {1765 def : FlatLoadSignedPat_M0 <inst, node, vt> {1766 let AddedComplexity = 10;1767 let SubtargetPredicate = inst.SubtargetPredicate;1768 let OtherPredicates = inst.OtherPredicates;1769 }1770 1771 def : GlobalLoadSaddrPat_M0<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> {1772 let AddedComplexity = 11;1773 let SubtargetPredicate = inst.SubtargetPredicate;1774 let OtherPredicates = inst.OtherPredicates;1775 }1776}1777 1778multiclass GlobalFLATLoadPats_CPOL<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> {1779 def : FlatLoadSignedPat_CPOL<inst, node, vt> {1780 let AddedComplexity = 10;1781 let SubtargetPredicate = inst.SubtargetPredicate;1782 let OtherPredicates = inst.OtherPredicates;1783 }1784 1785 def : GlobalLoadSaddrPat_CPOL<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> {1786 let AddedComplexity = 11;1787 let SubtargetPredicate = inst.SubtargetPredicate;1788 let OtherPredicates = inst.OtherPredicates;1789 }1790}1791 1792multiclass GlobalFLATLoadPats_D16<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> {1793 def : FlatSignedLoadPat_D16 <inst, node, vt> {1794 let AddedComplexity = 10;1795 }1796 1797 def : FlatLoadSaddrPat_D16<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> {1798 let AddedComplexity = 11;1799 }1800}1801 1802multiclass GlobalFLATLoadPats_D16_t16<string inst, SDPatternOperator node, ValueType vt> {1803 def : FlatSignedLoadPat_D16_t16<!cast<FLAT_Pseudo>(inst#"_t16"), node, vt> {1804 let AddedComplexity = 10;1805 }1806 1807 def : GlobalLoadSaddrPat_D16_t16<!cast<FLAT_Pseudo>(inst#"_SADDR_t16"), node, vt> {1808 let AddedComplexity = 11;1809 }1810}1811 1812multiclass GlobalFLATLoadPats_t16<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> {1813 def : FlatSignedLoadPat_t16<inst, node, vt> {1814 let AddedComplexity = 10;1815 }1816 1817 def : GlobalLoadSaddrPat_t16<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> {1818 let AddedComplexity = 11;1819 }1820}1821 1822multiclass GlobalFLATStorePats<FLAT_Pseudo inst, SDPatternOperator node,1823 ValueType vt> {1824 def : FlatStoreSignedPat <inst, node, vt> {1825 let SubtargetPredicate = inst.SubtargetPredicate;1826 let OtherPredicates = inst.OtherPredicates;1827 let AddedComplexity = 10;1828 }1829 1830 def : FlatStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> {1831 let SubtargetPredicate = inst.SubtargetPredicate;1832 let OtherPredicates = inst.OtherPredicates;1833 let AddedComplexity = 11;1834 }1835}1836 1837multiclass GlobalFLATStorePats_D16_t16<string inst, SDPatternOperator node, ValueType vt> {1838 def : FlatStoreSignedPat<!cast<FLAT_Pseudo>(inst#"_t16"), node, vt> {1839 let AddedComplexity = 10;1840 }1841 1842 def : FlatStoreSaddrPat<!cast<FLAT_Pseudo>(inst#"_SADDR_t16"), node, vt> {1843 let AddedComplexity = 11;1844 }1845}1846 1847multiclass GlobalFLATAtomicPatsNoRtnBase<string inst, string node, ValueType vt,1848 ValueType data_vt = vt> {1849 let AddedComplexity = 11 in1850 def : FlatSignedAtomicPatBase<!cast<FLAT_Pseudo>(inst), !cast<SDPatternOperator>(node), vt, data_vt>;1851 1852 let AddedComplexity = 13 in1853 def : FlatAtomicSaddrPat<!cast<FLAT_Pseudo>(inst#"_SADDR"), !cast<SDPatternOperator>(node),1854 GlobalSAddr, vt, data_vt>;1855}1856 1857multiclass GlobalFLATAtomicPatsRtnBase<string inst, string node, ValueType vt,1858 ValueType data_vt = vt, bit isPatFrags = 0> {1859 defvar rtnNode = !if(isPatFrags, !cast<PatFrags>(node), !cast<SDPatternOperator>(node));1860 1861 let AddedComplexity = 10 in1862 def : FlatSignedAtomicPatBase<!cast<FLAT_Pseudo>(inst#"_RTN"), rtnNode, vt, data_vt>;1863 1864 let AddedComplexity = 12 in1865 def : FlatAtomicSaddrPat<!cast<FLAT_Pseudo>(inst#"_SADDR_RTN"), rtnNode, GlobalSAddrGLC, vt, data_vt>;1866}1867 1868multiclass GlobalFLATAtomicPatsNoRtn<string inst, string node, ValueType vt,1869 ValueType data_vt = vt, bit isIntr = 0> :1870 GlobalFLATAtomicPatsNoRtnBase<inst, node # "_noret" # !if(isIntr, "", "_" # vt), vt, data_vt>;1871 1872multiclass GlobalFLATAtomicPatsRtn<string inst, string node, ValueType vt,1873 ValueType data_vt = vt, bit isIntr = 0> :1874 GlobalFLATAtomicPatsRtnBase<inst, node # !if(isIntr, "", "_" # vt), vt, data_vt>;1875 1876multiclass GlobalFLATAtomicPats<string inst, string node, ValueType vt,1877 ValueType data_vt = vt, bit isIntr = 0> :1878 GlobalFLATAtomicPatsNoRtn<inst, node, vt, data_vt, isIntr>,1879 GlobalFLATAtomicPatsRtn<inst, node, vt, data_vt, isIntr>;1880 1881multiclass GlobalFLATAtomicPatsNoRtnWithAddrSpace<string inst, string intr, string addrSpaceSuffix,1882 ValueType vt, ValueType data_vt = vt> :1883 GlobalFLATAtomicPatsNoRtnBase<inst, intr # "_noret_" # addrSpaceSuffix, vt, data_vt>;1884 1885multiclass GlobalFLATAtomicPatsRtnWithAddrSpace<string inst, string intr, string addrSpaceSuffix,1886 ValueType vt, ValueType data_vt = vt> :1887 GlobalFLATAtomicPatsRtnBase<inst, intr # "_" # addrSpaceSuffix, vt, data_vt, /*isPatFrags*/ 1>;1888 1889multiclass GlobalFLATAtomicPatsWithAddrSpace<string inst, string intr, string addrSpaceSuffix,1890 ValueType vt, ValueType data_vt = vt> :1891 GlobalFLATAtomicPatsNoRtnWithAddrSpace<inst, intr, addrSpaceSuffix, vt, data_vt>,1892 GlobalFLATAtomicPatsRtnWithAddrSpace<inst, intr, addrSpaceSuffix, vt, data_vt>;1893 1894multiclass GlobalFLATAtomicIntrPats<string inst, string node, ValueType vt,1895 ValueType data_vt = vt> {1896 defm : GlobalFLATAtomicPats<inst, node, vt, data_vt, /* isIntr */ 1>;1897}1898 1899multiclass ScratchFLATLoadPats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> {1900 def : ScratchLoadSignedPat <inst, node, vt> {1901 let AddedComplexity = 25;1902 }1903 1904 def : ScratchLoadSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> {1905 let AddedComplexity = 26;1906 }1907 1908 def : ScratchLoadSVaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SVS"), node, vt> {1909 let SubtargetPredicate = HasFlatScratchSVSMode;1910 let AddedComplexity = 27;1911 }1912}1913 1914multiclass ScratchFLATStorePats<FLAT_Pseudo inst, SDPatternOperator node,1915 ValueType vt> {1916 def : ScratchStoreSignedPat <inst, node, vt> {1917 let AddedComplexity = 25;1918 }1919 1920 def : ScratchStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> {1921 let AddedComplexity = 26;1922 }1923 1924 def : ScratchStoreSVaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SVS"), node, vt> {1925 let SubtargetPredicate = HasFlatScratchSVSMode;1926 let AddedComplexity = 27;1927 }1928}1929 1930multiclass ScratchFLATStorePats_D16_t16<string inst, SDPatternOperator node,1931 ValueType vt> {1932 def : ScratchStoreSignedPat <!cast<FLAT_Pseudo>(inst#"_t16"), node, vt> {1933 let AddedComplexity = 25;1934 }1935 1936 def : ScratchStoreSaddrPat<!cast<FLAT_Pseudo>(inst#"_SADDR_t16"), node, vt> {1937 let AddedComplexity = 26;1938 }1939 1940 def : ScratchStoreSVaddrPat<!cast<FLAT_Pseudo>(inst#"_SVS_t16"), node, vt> {1941 let SubtargetPredicate = HasFlatScratchSVSMode;1942 let AddedComplexity = 27;1943 }1944}1945 1946multiclass ScratchFLATLoadPats_D16<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> {1947 def : ScratchLoadSignedPat_D16 <inst, node, vt> {1948 let AddedComplexity = 25;1949 }1950 1951 def : ScratchLoadSaddrPat_D16<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> {1952 let AddedComplexity = 26;1953 }1954 1955 def : ScratchLoadSVaddrPat_D16 <!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SVS"), node, vt> {1956 let SubtargetPredicate = HasFlatScratchSVSMode;1957 let AddedComplexity = 27;1958 }1959}1960 1961multiclass ScratchFLATLoadPats_D16_t16<string inst, SDPatternOperator node, ValueType vt> {1962 def : ScratchLoadSignedPat_D16_t16 <!cast<FLAT_Pseudo>(inst#"_t16"), node, vt> {1963 let AddedComplexity = 25;1964 }1965 1966 def : ScratchLoadSaddrPat_D16_t16<!cast<FLAT_Pseudo>(inst#"_SADDR_t16"), node, vt> {1967 let AddedComplexity = 26;1968 }1969 1970 def : ScratchLoadSVaddrPat_D16_t16 <!cast<FLAT_Pseudo>(inst#"_SVS_t16"), node, vt> {1971 let SubtargetPredicate = HasFlatScratchSVSMode;1972 let AddedComplexity = 27;1973 }1974}1975 1976multiclass ScratchFLATLoadPats_t16<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> {1977 def : ScratchLoadSignedPat_t16 <inst, node, vt> {1978 let AddedComplexity = 25;1979 }1980 1981 def : ScratchLoadSaddrPat_t16<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> {1982 let AddedComplexity = 26;1983 }1984 1985 def : ScratchLoadSVaddrPat_t16<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SVS"), node, vt> {1986 let SubtargetPredicate = HasFlatScratchSVSMode;1987 let AddedComplexity = 27;1988 }1989}1990 1991multiclass FlatLoadPats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> {1992 def : FlatLoadPat <inst, node, vt> {1993 let OtherPredicates = [HasFlatAddressSpace];1994 }1995 1996 def : FlatLoadSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> {1997 let AddedComplexity = 9;1998 let SubtargetPredicate = HasFlatGVSMode;1999 }2000}2001 2002multiclass FlatLoadPats_D16<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> {2003 def : FlatLoadPat_D16 <inst, node, vt>;2004 2005 def : FlatLoadSaddrPat_D16<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> {2006 let AddedComplexity = 9;2007 let SubtargetPredicate = HasFlatGVSMode;2008 }2009}2010 2011multiclass FlatLoadPats_D16_t16<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> {2012 def : FlatLoadPat_D16_t16 <inst, node, vt>;2013 2014 def : FlatLoadSaddrPat_D16_t16<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> {2015 let AddedComplexity = 9;2016 let SubtargetPredicate = HasFlatGVSMode;2017 }2018}2019 2020multiclass FlatLoadPats_t16<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> {2021 def : FlatLoadPat_t16 <inst, node, vt> {2022 let OtherPredicates = [HasFlatAddressSpace];2023 }2024 2025 def : FlatLoadSaddrPat_t16<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> {2026 let AddedComplexity = 9;2027 let SubtargetPredicate = HasFlatGVSMode;2028 }2029}2030 2031multiclass FlatStorePats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> {2032 def : FlatStorePat <inst, node, vt> {2033 let OtherPredicates = [HasFlatAddressSpace];2034 }2035 2036 def : FlatStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> {2037 let AddedComplexity = 9;2038 let SubtargetPredicate = HasFlatGVSMode;2039 }2040}2041 2042multiclass FlatStorePats_t16<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> {2043 def : FlatStorePat <!cast<FLAT_Pseudo>(!cast<string>(inst)#"_t16"), node, vt>;2044 2045 def : FlatStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR_t16"), node, vt> {2046 let AddedComplexity = 9;2047 let SubtargetPredicate = HasFlatGVSMode;2048 }2049}2050 2051defm : FlatLoadPats <FLAT_LOAD_UBYTE, atomic_load_aext_8_flat, i32>;2052defm : FlatLoadPats <FLAT_LOAD_UBYTE, atomic_load_zext_8_flat, i32>;2053defm : FlatLoadPats <FLAT_LOAD_USHORT, atomic_load_aext_16_flat, i32>;2054defm : FlatLoadPats <FLAT_LOAD_USHORT, atomic_load_zext_16_flat, i32>;2055defm : FlatLoadPats <FLAT_LOAD_USHORT, atomic_load_zext_16_flat, i16>;2056defm : FlatLoadPats <FLAT_LOAD_UBYTE, extloadi8_flat, i32>;2057defm : FlatLoadPats <FLAT_LOAD_UBYTE, zextloadi8_flat, i32>;2058defm : FlatLoadPats <FLAT_LOAD_SBYTE, sextloadi8_flat, i32>;2059defm : FlatLoadPats <FLAT_LOAD_SBYTE, atomic_load_sext_8_flat, i32>;2060defm : FlatLoadPats <FLAT_LOAD_USHORT, extloadi16_flat, i32>;2061defm : FlatLoadPats <FLAT_LOAD_USHORT, zextloadi16_flat, i32>;2062defm : FlatLoadPats <FLAT_LOAD_SSHORT, sextloadi16_flat, i32>;2063defm : FlatLoadPats <FLAT_LOAD_SSHORT, atomic_load_sext_16_flat, i32>;2064defm : FlatLoadPats <FLAT_LOAD_DWORDX3, load_flat, v3i32>;2065 2066let True16Predicate = NotUseRealTrue16Insts in {2067 defm : FlatLoadPats <FLAT_LOAD_UBYTE, extloadi8_flat, i16>;2068 defm : FlatLoadPats <FLAT_LOAD_UBYTE, zextloadi8_flat, i16>;2069 defm : FlatLoadPats <FLAT_LOAD_SBYTE, sextloadi8_flat, i16>;2070 defm : FlatLoadPats <FLAT_LOAD_USHORT, load_flat, i16>;2071 defm : FlatLoadPats <FLAT_LOAD_UBYTE, atomic_load_aext_8_flat, i16>;2072 defm : FlatLoadPats <FLAT_LOAD_UBYTE, atomic_load_zext_8_flat, i16>;2073 defm : FlatLoadPats <FLAT_LOAD_USHORT, atomic_load_nonext_16_flat, i16>;2074 defm : FlatLoadPats <FLAT_LOAD_SBYTE, atomic_load_sext_8_flat, i16>;2075 defm : FlatStorePats <FLAT_STORE_BYTE, truncstorei8_flat, i16>;2076 defm : FlatStorePats <FLAT_STORE_SHORT, store_flat, i16>;2077 defm : FlatStorePats <FLAT_STORE_BYTE, atomic_store_8_flat, i16>;2078 defm : FlatStorePats <FLAT_STORE_SHORT, atomic_store_16_flat, i16>;2079}2080 2081let True16Predicate = UseTrue16WithSramECC in {2082 defm : FlatLoadPats_t16 <FLAT_LOAD_UBYTE, extloadi8_flat, i16>;2083 defm : FlatLoadPats_t16 <FLAT_LOAD_UBYTE, zextloadi8_flat, i16>;2084 defm : FlatLoadPats_t16 <FLAT_LOAD_SBYTE, sextloadi8_flat, i16>;2085 defm : FlatLoadPats_t16 <FLAT_LOAD_USHORT, load_flat, i16>;2086 defm : FlatLoadPats_t16 <FLAT_LOAD_UBYTE, atomic_load_aext_8_flat, i16>;2087 defm : FlatLoadPats_t16 <FLAT_LOAD_UBYTE, atomic_load_zext_8_flat, i16>;2088 defm : FlatLoadPats_t16 <FLAT_LOAD_USHORT, atomic_load_nonext_16_flat, i16>;2089 defm : FlatLoadPats_t16 <FLAT_LOAD_SBYTE, atomic_load_sext_8_flat, i16>;2090}2091 2092let OtherPredicates = [D16PreservesUnusedBits, HasFlatAddressSpace], True16Predicate = UseRealTrue16Insts in {2093 defm : FlatLoadPats_D16_t16<FLAT_LOAD_UBYTE_D16_t16, extloadi8_flat, i16>;2094 defm : FlatLoadPats_D16_t16<FLAT_LOAD_UBYTE_D16_t16, zextloadi8_flat, i16>;2095 defm : FlatLoadPats_D16_t16<FLAT_LOAD_SBYTE_D16_t16, sextloadi8_flat, i16>;2096 defm : FlatLoadPats_D16_t16<FLAT_LOAD_SHORT_D16_t16, load_flat, i16>;2097 defm : FlatLoadPats_D16_t16<FLAT_LOAD_UBYTE_D16_t16, atomic_load_aext_8_flat, i16>;2098 defm : FlatLoadPats_D16_t16<FLAT_LOAD_UBYTE_D16_t16, atomic_load_zext_8_flat, i16>;2099 defm : FlatLoadPats_D16_t16<FLAT_LOAD_SHORT_D16_t16, atomic_load_nonext_16_flat, i16>;2100 defm : FlatLoadPats_D16_t16<FLAT_LOAD_SBYTE_D16_t16, atomic_load_sext_8_flat, i16>;2101} // End let OtherPredicates = [D16PreservesUnusedBits, HasFlatAddressSpace], True16Predicate = UseRealTrue16Insts2102 2103let OtherPredicates = [D16PreservesUnusedBits], True16Predicate = UseRealTrue16Insts in {2104 defm : FlatStorePats_t16 <FLAT_STORE_BYTE, truncstorei8_flat, i16>;2105 defm : FlatStorePats_t16 <FLAT_STORE_SHORT, store_flat, i16>;2106 defm : FlatStorePats_t16 <FLAT_STORE_BYTE, atomic_store_8_flat, i16>;2107 defm : FlatStorePats_t16 <FLAT_STORE_SHORT, atomic_store_16_flat, i16>;2108}2109 2110defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_nonext_32_flat, i32>;2111defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, i64>;2112defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, v2i32>;2113defm : FlatLoadPats <FLAT_LOAD_DWORDX4, atomic_load_nonext_128_flat, v4i32>;2114 2115defm : FlatStorePats <FLAT_STORE_BYTE, truncstorei8_flat, i32>;2116defm : FlatStorePats <FLAT_STORE_SHORT, truncstorei16_flat, i32>;2117 2118foreach vt = Reg32Types.types in {2119defm : FlatLoadPats <FLAT_LOAD_DWORD, load_flat, vt>;2120defm : FlatStorePats <FLAT_STORE_DWORD, store_flat, vt>;2121}2122 2123foreach vt = VReg_64.RegTypes in {2124defm : FlatStorePats <FLAT_STORE_DWORDX2, store_flat, vt>;2125defm : FlatLoadPats <FLAT_LOAD_DWORDX2, load_flat, vt>;2126}2127 2128defm : FlatStorePats <FLAT_STORE_DWORDX3, store_flat, v3i32>;2129 2130foreach vt = VReg_128.RegTypes in {2131defm : FlatLoadPats <FLAT_LOAD_DWORDX4, load_flat, vt>;2132defm : FlatStorePats <FLAT_STORE_DWORDX4, store_flat, vt>;2133}2134 2135defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_32_flat, i32>;2136defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, i64>;2137defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, v2i32>;2138defm : FlatStorePats <FLAT_STORE_DWORDX4, atomic_store_128_flat, v4i32>;2139defm : FlatStorePats <FLAT_STORE_BYTE, atomic_store_8_flat, i32>;2140defm : FlatStorePats <FLAT_STORE_SHORT, atomic_store_16_flat, i32>;2141 2142 2143foreach as = [ "flat", "global" ] in {2144defm : FlatAtomicPat <"FLAT_ATOMIC_ADD", "atomic_load_add_"#as, i32>;2145defm : FlatAtomicPat <"FLAT_ATOMIC_SUB", "atomic_load_sub_"#as, i32>;2146defm : FlatAtomicPat <"FLAT_ATOMIC_INC", "atomic_load_uinc_wrap_"#as, i32>;2147defm : FlatAtomicPat <"FLAT_ATOMIC_DEC", "atomic_load_udec_wrap_"#as, i32>;2148defm : FlatAtomicPat <"FLAT_ATOMIC_AND", "atomic_load_and_"#as, i32>;2149defm : FlatAtomicPat <"FLAT_ATOMIC_SMAX", "atomic_load_max_"#as, i32>;2150defm : FlatAtomicPat <"FLAT_ATOMIC_UMAX", "atomic_load_umax_"#as, i32>;2151defm : FlatAtomicPat <"FLAT_ATOMIC_SMIN", "atomic_load_min_"#as, i32>;2152defm : FlatAtomicPat <"FLAT_ATOMIC_UMIN", "atomic_load_umin_"#as, i32>;2153defm : FlatAtomicPat <"FLAT_ATOMIC_OR", "atomic_load_or_"#as, i32>;2154defm : FlatAtomicPat <"FLAT_ATOMIC_SWAP", "atomic_swap_"#as, i32>;2155defm : FlatAtomicPat <"FLAT_ATOMIC_CMPSWAP", "AMDGPUatomic_cmp_swap_"#as, i32, v2i32>;2156defm : FlatAtomicPat <"FLAT_ATOMIC_XOR", "atomic_load_xor_"#as, i32>;2157 2158defm : FlatAtomicPat <"FLAT_ATOMIC_ADD_X2", "atomic_load_add_"#as, i64>;2159defm : FlatAtomicPat <"FLAT_ATOMIC_SUB_X2", "atomic_load_sub_"#as, i64>;2160defm : FlatAtomicPat <"FLAT_ATOMIC_INC_X2", "atomic_load_uinc_wrap_"#as, i64>;2161defm : FlatAtomicPat <"FLAT_ATOMIC_DEC_X2", "atomic_load_udec_wrap_"#as, i64>;2162defm : FlatAtomicPat <"FLAT_ATOMIC_AND_X2", "atomic_load_and_"#as, i64>;2163defm : FlatAtomicPat <"FLAT_ATOMIC_SMAX_X2", "atomic_load_max_"#as, i64>;2164defm : FlatAtomicPat <"FLAT_ATOMIC_UMAX_X2", "atomic_load_umax_"#as, i64>;2165defm : FlatAtomicPat <"FLAT_ATOMIC_SMIN_X2", "atomic_load_min_"#as, i64>;2166defm : FlatAtomicPat <"FLAT_ATOMIC_UMIN_X2", "atomic_load_umin_"#as, i64>;2167defm : FlatAtomicPat <"FLAT_ATOMIC_OR_X2", "atomic_load_or_"#as, i64>;2168defm : FlatAtomicPat <"FLAT_ATOMIC_SWAP_X2", "atomic_swap_"#as, i64>;2169defm : FlatAtomicPat <"FLAT_ATOMIC_CMPSWAP_X2", "AMDGPUatomic_cmp_swap_"#as, i64, v2i64>;2170defm : FlatAtomicPat <"FLAT_ATOMIC_XOR_X2", "atomic_load_xor_"#as, i64>;2171 2172let SubtargetPredicate = HasAtomicFMinFMaxF32FlatInsts in {2173defm : FlatAtomicPat <"FLAT_ATOMIC_FMIN", "atomic_load_fmin_"#as, f32>;2174defm : FlatAtomicPat <"FLAT_ATOMIC_FMAX", "atomic_load_fmax_"#as, f32>;2175}2176 2177let SubtargetPredicate = HasAtomicFMinFMaxF64FlatInsts in {2178defm : FlatAtomicPat <"FLAT_ATOMIC_MIN_F64", "atomic_load_fmin_"#as, f64>;2179defm : FlatAtomicPat <"FLAT_ATOMIC_MAX_F64", "atomic_load_fmax_"#as, f64>;2180}2181 2182} // end foreach as2183 2184defm : FlatStorePats <FLAT_STORE_BYTE, truncstorei8_flat, i16>;2185defm : FlatStorePats <FLAT_STORE_SHORT, store_flat, i16>;2186 2187defm : FlatAtomicRtnPatWithAddrSpace<"FLAT_ATOMIC_COND_SUB_U32", "int_amdgcn_atomic_cond_sub_u32", "flat_addrspace", i32>;2188defm : FlatAtomicNoRtnPatWithAddrSpace<"FLAT_ATOMIC_COND_SUB_U32", "int_amdgcn_atomic_cond_sub_u32", "flat_addrspace", i32>;2189 2190let OtherPredicates = [HasD16LoadStore] in {2191defm : FlatStorePats <FLAT_STORE_SHORT_D16_HI, truncstorei16_hi16_flat, i32>;2192defm : FlatStorePats <FLAT_STORE_BYTE_D16_HI, truncstorei8_hi16_flat, i32>;2193}2194 2195let OtherPredicates = [D16PreservesUnusedBits] in {2196// TODO: Handle atomic loads2197defm : FlatLoadPats_D16 <FLAT_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_flat, v2i16>;2198defm : FlatLoadPats_D16 <FLAT_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_flat, v2f16>;2199defm : FlatLoadPats_D16 <FLAT_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_flat, v2i16>;2200defm : FlatLoadPats_D16 <FLAT_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_flat, v2f16>;2201defm : FlatLoadPats_D16 <FLAT_LOAD_SHORT_D16_HI, load_d16_hi_flat, v2i16>;2202defm : FlatLoadPats_D16 <FLAT_LOAD_SHORT_D16_HI, load_d16_hi_flat, v2f16>;2203 2204defm : FlatLoadPats_D16 <FLAT_LOAD_UBYTE_D16, az_extloadi8_d16_lo_flat, v2i16>;2205defm : FlatLoadPats_D16 <FLAT_LOAD_UBYTE_D16, az_extloadi8_d16_lo_flat, v2f16>;2206defm : FlatLoadPats_D16 <FLAT_LOAD_SBYTE_D16, sextloadi8_d16_lo_flat, v2i16>;2207defm : FlatLoadPats_D16 <FLAT_LOAD_SBYTE_D16, sextloadi8_d16_lo_flat, v2f16>;2208defm : FlatLoadPats_D16 <FLAT_LOAD_SHORT_D16, load_d16_lo_flat, v2i16>;2209defm : FlatLoadPats_D16 <FLAT_LOAD_SHORT_D16, load_d16_lo_flat, v2f16>;2210}2211 2212defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, atomic_load_aext_8_global, i32>;2213defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, atomic_load_zext_8_global, i32>;2214defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_aext_16_global, i32>;2215defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_zext_16_global, i32>;2216defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, atomic_load_sext_8_global, i32>;2217defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, extloadi8_global, i32>;2218defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, zextloadi8_global, i32>;2219defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, sextloadi8_global, i32>;2220defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, extloadi16_global, i32>;2221defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, zextloadi16_global, i32>;2222defm : GlobalFLATLoadPats <GLOBAL_LOAD_SSHORT, sextloadi16_global, i32>;2223 2224let True16Predicate = NotUseRealTrue16Insts in {2225defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, extloadi8_global, i16>;2226defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, zextloadi8_global, i16>;2227defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, sextloadi8_global, i16>;2228defm : GlobalFLATLoadPats <GLOBAL_LOAD_SSHORT, atomic_load_sext_16_global, i32>;2229defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_zext_16_global, i32>;2230defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, load_global, i16>;2231defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, atomic_load_aext_8_global, i16>;2232defm : GlobalFLATLoadPats <GLOBAL_LOAD_UBYTE, atomic_load_zext_8_global, i16>;2233defm : GlobalFLATLoadPats <GLOBAL_LOAD_SBYTE, atomic_load_sext_8_global, i16>;2234defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_nonext_16_global, i16>;2235defm : GlobalFLATLoadPats <GLOBAL_LOAD_USHORT, atomic_load_zext_16_global, i16>;2236}2237 2238let True16Predicate = UseTrue16WithSramECC in {2239defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_UBYTE, extloadi8_global, i16>;2240defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_UBYTE, zextloadi8_global, i16>;2241defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_SBYTE, sextloadi8_global, i16>;2242defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_SSHORT, atomic_load_sext_16_global, i32>;2243defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_USHORT, atomic_load_zext_16_global, i32>;2244defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_USHORT, load_global, i16>;2245defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_UBYTE, atomic_load_aext_8_global, i16>;2246defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_UBYTE, atomic_load_zext_8_global, i16>;2247defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_SBYTE, atomic_load_sext_8_global, i16>;2248defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_USHORT, atomic_load_nonext_16_global, i16>;2249defm : GlobalFLATLoadPats_t16 <GLOBAL_LOAD_USHORT, atomic_load_zext_16_global, i16>;2250}2251 2252let OtherPredicates = [D16PreservesUnusedBits], True16Predicate = UseRealTrue16Insts in {2253defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_UBYTE_D16", extloadi8_global, i16>;2254defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_UBYTE_D16", zextloadi8_global, i16>;2255defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_SBYTE_D16", sextloadi8_global, i16>;2256defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_SHORT_D16", load_global, i16>;2257defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_UBYTE_D16", atomic_load_aext_8_global, i16>;2258defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_UBYTE_D16", atomic_load_zext_8_global, i16>;2259defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_SBYTE_D16", atomic_load_sext_8_global, i16>;2260defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_SHORT_D16", atomic_load_nonext_16_global, i16>;2261defm : GlobalFLATLoadPats_D16_t16<"GLOBAL_LOAD_SHORT_D16", atomic_load_zext_16_global, i16>;2262defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_BYTE", truncstorei8_global, i16>;2263defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_SHORT", store_global, i16>;2264defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_BYTE", atomic_store_8_global, i16>;2265defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_SHORT", atomic_store_16_global, i16>;2266} // end OtherPredicates = [HasFlatGlobalInsts, D16PreservesUnusedBits], True16Predicate = UseRealTrue16Insts2267 2268foreach vt = Reg32Types.types in {2269defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, load_global, vt>;2270defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, store_global, vt>;2271}2272 2273foreach vt = VReg_64.RegTypes in {2274defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, load_global, vt>;2275defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, store_global, vt>;2276}2277 2278defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX3, load_global, v3i32>;2279 2280foreach vt = VReg_128.RegTypes in {2281defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, load_global, vt>;2282defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, store_global, vt>;2283}2284 2285// There is no distinction for atomic load lowering during selection;2286// the memory legalizer will set the cache bits and insert the2287// appropriate waits.2288defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_nonext_32_global, i32>;2289defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, i64>;2290defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, v2i32>;2291defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, atomic_load_nonext_128_global, v4i32>;2292 2293defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i32>;2294defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, truncstorei16_global, i32>;2295defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX3, store_global, v3i32>;2296 2297let OtherPredicates = [HasFlatGlobalInsts], True16Predicate = NotUseRealTrue16Insts in {2298defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i16>;2299defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, store_global, i16>;2300defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i16>;2301defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i16>;2302}2303 2304let OtherPredicates = [HasFlatGlobalInsts], True16Predicate = UseRealTrue16Insts in {2305defm : GlobalFLATStorePats_D16_t16 <"GLOBAL_STORE_BYTE", truncstorei8_global, i16>;2306defm : GlobalFLATStorePats_D16_t16 <"GLOBAL_STORE_SHORT", store_global, i16>;2307defm : GlobalFLATStorePats_D16_t16 <"GLOBAL_STORE_BYTE", atomic_store_8_global, i16>;2308defm : GlobalFLATStorePats_D16_t16 <"GLOBAL_STORE_SHORT", atomic_store_16_global, i16>;2309}2310 2311let OtherPredicates = [HasD16LoadStore] in {2312defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT_D16_HI, truncstorei16_hi16_global, i32>;2313defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE_D16_HI, truncstorei8_hi16_global, i32>;2314}2315 2316let OtherPredicates = [D16PreservesUnusedBits] in {2317// TODO: Handle atomic loads2318defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_global, v2i16>;2319defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_global, v2f16>;2320defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_global, v2i16>;2321defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_global, v2f16>;2322defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16_HI, load_d16_hi_global, v2i16>;2323defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16_HI, load_d16_hi_global, v2f16>;2324 2325defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16, az_extloadi8_d16_lo_global, v2i16>;2326defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_UBYTE_D16, az_extloadi8_d16_lo_global, v2f16>;2327defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16, sextloadi8_d16_lo_global, v2i16>;2328defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SBYTE_D16, sextloadi8_d16_lo_global, v2f16>;2329defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2i16>;2330defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2f16>;2331}2332 2333defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i32>;2334defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i32>;2335defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, i32>;2336defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, i64>;2337defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, v2i32>;2338defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, atomic_store_128_global, v4i32>;2339 2340defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD", "atomic_load_add_global", i32>;2341defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB", "atomic_load_sub_global", i32>;2342defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_INC", "atomic_load_uinc_wrap_global", i32>;2343defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_DEC", "atomic_load_udec_wrap_global", i32>;2344defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_AND", "atomic_load_and_global", i32>;2345defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMAX", "atomic_load_max_global", i32>;2346defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMAX", "atomic_load_umax_global", i32>;2347defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMIN", "atomic_load_min_global", i32>;2348defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMIN", "atomic_load_umin_global", i32>;2349defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_OR", "atomic_load_or_global", i32>;2350defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SWAP", "atomic_swap_global", i32>;2351defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CMPSWAP", "AMDGPUatomic_cmp_swap_global", i32, v2i32>;2352defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_XOR", "atomic_load_xor_global", i32>;2353defm : GlobalFLATAtomicPatsRtn <"GLOBAL_ATOMIC_CSUB", "int_amdgcn_global_atomic_csub", i32, i32, /* isIntr */ 1>;2354 2355let SubtargetPredicate = HasAtomicCSubNoRtnInsts in2356defm : GlobalFLATAtomicPatsNoRtn <"GLOBAL_ATOMIC_CSUB", "int_amdgcn_global_atomic_csub", i32, i32, /* isIntr */ 1>;2357 2358defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD_X2", "atomic_load_add_global", i64>;2359defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB_X2", "atomic_load_sub_global", i64>;2360defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_INC_X2", "atomic_load_uinc_wrap_global", i64>;2361defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_DEC_X2", "atomic_load_udec_wrap_global", i64>;2362defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_AND_X2", "atomic_load_and_global", i64>;2363defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMAX_X2", "atomic_load_max_global", i64>;2364defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMAX_X2", "atomic_load_umax_global", i64>;2365defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMIN_X2", "atomic_load_min_global", i64>;2366defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMIN_X2", "atomic_load_umin_global", i64>;2367defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_OR_X2", "atomic_load_or_global", i64>;2368defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SWAP_X2", "atomic_swap_global", i64>;2369defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CMPSWAP_X2", "AMDGPUatomic_cmp_swap_global", i64, v2i64>;2370defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_XOR_X2", "atomic_load_xor_global", i64>;2371 2372let SubtargetPredicate = isGFX12Plus in {2373 defm : GlobalFLATAtomicPatsRtnWithAddrSpace <"GLOBAL_ATOMIC_COND_SUB_U32", "int_amdgcn_atomic_cond_sub_u32", "global_addrspace", i32>;2374 2375 let SubtargetPredicate = HasAtomicCSubNoRtnInsts in2376 defm : GlobalFLATAtomicPatsNoRtnWithAddrSpace <"GLOBAL_ATOMIC_COND_SUB_U32", "int_amdgcn_atomic_cond_sub_u32", "global_addrspace", i32>;2377}2378 2379let OtherPredicates = [isGFX12PlusNot12_50] in2380 defm : GlobalFLATAtomicPatsRtn <"GLOBAL_ATOMIC_ORDERED_ADD_B64", "int_amdgcn_global_atomic_ordered_add_b64", i64, i64, /* isIntr */ 1>;2381 2382let WaveSizePredicate = isWave32, OtherPredicates = [isGFX12Plus] in {2383 defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B64_w32, int_amdgcn_global_load_tr_b64, v2i32>;2384 foreach vt = [v8i16, v8f16, v8bf16] in2385 defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w32, int_amdgcn_global_load_tr_b128, vt>;2386}2387 2388let WaveSizePredicate = isWave64, OtherPredicates = [isGFX12PlusNot12_50] in {2389 defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B64_w64, int_amdgcn_global_load_tr_b64, i32>;2390 foreach vt = [v4i16, v4f16, v4bf16] in2391 defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w64, int_amdgcn_global_load_tr_b128, vt>;2392}2393 2394let WaveSizePredicate = isWave32, OtherPredicates = [HasTransposeLoadF4F6Insts] in {2395 defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR4_B64, int_amdgcn_global_load_tr4_b64, v2i32>;2396 defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR6_B96, int_amdgcn_global_load_tr6_b96, v3i32>;2397}2398 2399let OtherPredicates = [isGFX125xOnly] in {2400 def : FlatLoadPat_CPOL <FLAT_LOAD_MONITOR_B32, int_amdgcn_flat_load_monitor_b32, i32>;2401 def : FlatLoadPat_CPOL <FLAT_LOAD_MONITOR_B64, int_amdgcn_flat_load_monitor_b64, v2i32>;2402 def : FlatLoadPat_CPOL <FLAT_LOAD_MONITOR_B128, int_amdgcn_flat_load_monitor_b128, v4i32>;2403 2404 defm : GlobalFLATLoadPats_CPOL <GLOBAL_LOAD_MONITOR_B32, int_amdgcn_global_load_monitor_b32, i32>;2405 defm : GlobalFLATLoadPats_CPOL <GLOBAL_LOAD_MONITOR_B64, int_amdgcn_global_load_monitor_b64, v2i32>;2406 defm : GlobalFLATLoadPats_CPOL <GLOBAL_LOAD_MONITOR_B128, int_amdgcn_global_load_monitor_b128, v4i32>;2407} // End SubtargetPredicate = isGFX125xOnly2408 2409let OtherPredicates = [isGFX1250Plus] in {2410 defm : GlobalFLATLoadPats_M0 <CLUSTER_LOAD_B32, int_amdgcn_cluster_load_b32, i32>;2411 defm : GlobalFLATLoadPats_M0 <CLUSTER_LOAD_B64, int_amdgcn_cluster_load_b64, v2i32>;2412 defm : GlobalFLATLoadPats_M0 <CLUSTER_LOAD_B128, int_amdgcn_cluster_load_b128, v4i32>;2413 2414 defm : GlobalLoadLDSPats_M0 <CLUSTER_LOAD_ASYNC_TO_LDS_B8, int_amdgcn_cluster_load_async_to_lds_b8>;2415 defm : GlobalLoadLDSPats_M0 <CLUSTER_LOAD_ASYNC_TO_LDS_B32, int_amdgcn_cluster_load_async_to_lds_b32>;2416 defm : GlobalLoadLDSPats_M0 <CLUSTER_LOAD_ASYNC_TO_LDS_B64, int_amdgcn_cluster_load_async_to_lds_b64>;2417 defm : GlobalLoadLDSPats_M0 <CLUSTER_LOAD_ASYNC_TO_LDS_B128, int_amdgcn_cluster_load_async_to_lds_b128>;2418 2419 defm : GlobalLoadLDSPats <GLOBAL_LOAD_ASYNC_TO_LDS_B8, int_amdgcn_global_load_async_to_lds_b8>;2420 defm : GlobalLoadLDSPats <GLOBAL_LOAD_ASYNC_TO_LDS_B32, int_amdgcn_global_load_async_to_lds_b32>;2421 defm : GlobalLoadLDSPats <GLOBAL_LOAD_ASYNC_TO_LDS_B64, int_amdgcn_global_load_async_to_lds_b64>;2422 defm : GlobalLoadLDSPats <GLOBAL_LOAD_ASYNC_TO_LDS_B128, int_amdgcn_global_load_async_to_lds_b128>;2423 2424 defm : GlobalStoreLDSPats <GLOBAL_STORE_ASYNC_FROM_LDS_B8, int_amdgcn_global_store_async_from_lds_b8>;2425 defm : GlobalStoreLDSPats <GLOBAL_STORE_ASYNC_FROM_LDS_B32, int_amdgcn_global_store_async_from_lds_b32>;2426 defm : GlobalStoreLDSPats <GLOBAL_STORE_ASYNC_FROM_LDS_B64, int_amdgcn_global_store_async_from_lds_b64>;2427 defm : GlobalStoreLDSPats <GLOBAL_STORE_ASYNC_FROM_LDS_B128, int_amdgcn_global_store_async_from_lds_b128>;2428}2429 2430defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMIN", "atomic_load_fmin_global", f32>;2431defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMAX", "atomic_load_fmax_global", f32>;2432defm : FlatAtomicPat <"FLAT_ATOMIC_FMIN", "atomic_load_fmin_flat", f32>;2433defm : FlatAtomicPat <"FLAT_ATOMIC_FMAX", "atomic_load_fmax_flat", f32>;2434 2435// FIXME: Remove these intrinsics2436let SubtargetPredicate = isGFX12Only in {2437defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_FMIN", "int_amdgcn_global_atomic_fmin_num", f32>;2438defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_FMAX", "int_amdgcn_global_atomic_fmax_num", f32>;2439defm : FlatAtomicIntrPat <"FLAT_ATOMIC_FMIN", "int_amdgcn_flat_atomic_fmin_num", f32>;2440defm : FlatAtomicIntrPat <"FLAT_ATOMIC_FMAX", "int_amdgcn_flat_atomic_fmax_num", f32>;2441}2442 2443defm : GlobalFLATAtomicPatsNoRtn <"GLOBAL_ATOMIC_ADD_F32", "atomic_load_fadd_global", f32>;2444 2445defm : GlobalFLATAtomicPatsNoRtn <"GLOBAL_ATOMIC_PK_ADD_F16", "atomic_load_fadd_global", v2f16>;2446 2447defm : GlobalFLATAtomicPatsRtn <"GLOBAL_ATOMIC_ADD_F32", "atomic_load_fadd_global", f32>;2448 2449defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_PK_ADD_F16", "atomic_load_fadd_global", v2f16>;2450 2451defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_MIN_F64", "atomic_load_fmin_global", f64>;2452defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_MAX_F64", "atomic_load_fmax_global", f64>;2453 2454defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD_F64", "atomic_load_fadd_global", f64>;2455defm : FlatAtomicPat <"FLAT_ATOMIC_ADD_F64", "atomic_load_fadd_flat", f64>;2456 2457defm : FlatAtomicPat <"FLAT_ATOMIC_ADD_F32", "atomic_load_fadd_flat", f32>;2458defm : FlatAtomicPat <"FLAT_ATOMIC_PK_ADD_F16", "atomic_load_fadd_flat", v2f16>;2459defm : FlatAtomicPat <"FLAT_ATOMIC_PK_ADD_BF16", "atomic_load_fadd_flat", v2bf16>;2460 2461defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_PK_ADD_BF16", "atomic_load_fadd_global", v2bf16>;2462 2463let OtherPredicates = [HasFlatScratchInsts, EnableFlatScratch] in {2464 2465defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, extloadi8_private, i32>;2466defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, zextloadi8_private, i32>;2467defm : ScratchFLATLoadPats <SCRATCH_LOAD_SBYTE, sextloadi8_private, i32>;2468defm : ScratchFLATLoadPats <SCRATCH_LOAD_USHORT, extloadi16_private, i32>;2469defm : ScratchFLATLoadPats <SCRATCH_LOAD_USHORT, zextloadi16_private, i32>;2470defm : ScratchFLATLoadPats <SCRATCH_LOAD_SSHORT, sextloadi16_private, i32>;2471 2472let True16Predicate = NotUseRealTrue16Insts in {2473defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, extloadi8_private, i16>;2474defm : ScratchFLATLoadPats <SCRATCH_LOAD_UBYTE, zextloadi8_private, i16>;2475defm : ScratchFLATLoadPats <SCRATCH_LOAD_SBYTE, sextloadi8_private, i16>;2476defm : ScratchFLATLoadPats <SCRATCH_LOAD_USHORT, load_private, i16>;2477defm : ScratchFLATStorePats <SCRATCH_STORE_SHORT, store_private, i16>;2478defm : ScratchFLATStorePats <SCRATCH_STORE_BYTE, truncstorei8_private, i16>;2479}2480 2481let True16Predicate = UseTrue16WithSramECC in {2482defm : ScratchFLATLoadPats_t16 <SCRATCH_LOAD_UBYTE, extloadi8_private, i16>;2483defm : ScratchFLATLoadPats_t16 <SCRATCH_LOAD_UBYTE, zextloadi8_private, i16>;2484defm : ScratchFLATLoadPats_t16 <SCRATCH_LOAD_SBYTE, sextloadi8_private, i16>;2485defm : ScratchFLATLoadPats_t16 <SCRATCH_LOAD_USHORT, load_private, i16>;2486}2487 2488let OtherPredicates = [D16PreservesUnusedBits], True16Predicate = UseRealTrue16Insts in {2489defm : ScratchFLATLoadPats_D16_t16<"SCRATCH_LOAD_UBYTE_D16", extloadi8_private, i16>;2490defm : ScratchFLATLoadPats_D16_t16<"SCRATCH_LOAD_UBYTE_D16", zextloadi8_private, i16>;2491defm : ScratchFLATLoadPats_D16_t16<"SCRATCH_LOAD_SBYTE_D16", sextloadi8_private, i16>;2492defm : ScratchFLATLoadPats_D16_t16<"SCRATCH_LOAD_SHORT_D16", load_private, i16>;2493} // End OtherPredicates = [D16PreservesUnusedBits], True16Predicate = UseRealTrue16Insts2494 2495let True16Predicate = UseRealTrue16Insts in {2496defm : ScratchFLATStorePats_D16_t16 <"SCRATCH_STORE_SHORT", store_private, i16>;2497defm : ScratchFLATStorePats_D16_t16 <"SCRATCH_STORE_BYTE", truncstorei8_private, i16>;2498}2499 2500foreach vt = Reg32Types.types in {2501defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORD, load_private, vt>;2502defm : ScratchFLATStorePats <SCRATCH_STORE_DWORD, store_private, vt>;2503}2504 2505foreach vt = VReg_64.RegTypes in {2506defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORDX2, load_private, vt>;2507defm : ScratchFLATStorePats <SCRATCH_STORE_DWORDX2, store_private, vt>;2508}2509 2510defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORDX3, load_private, v3i32>;2511 2512foreach vt = VReg_128.RegTypes in {2513defm : ScratchFLATLoadPats <SCRATCH_LOAD_DWORDX4, load_private, vt>;2514defm : ScratchFLATStorePats <SCRATCH_STORE_DWORDX4, store_private, vt>;2515}2516 2517defm : ScratchFLATStorePats <SCRATCH_STORE_BYTE, truncstorei8_private, i32>;2518defm : ScratchFLATStorePats <SCRATCH_STORE_SHORT, truncstorei16_private, i32>;2519defm : ScratchFLATStorePats <SCRATCH_STORE_DWORDX3, store_private, v3i32>;2520 2521let OtherPredicates = [HasD16LoadStore, HasFlatScratchInsts, EnableFlatScratch] in {2522defm : ScratchFLATStorePats <SCRATCH_STORE_SHORT_D16_HI, truncstorei16_hi16_private, i32>;2523defm : ScratchFLATStorePats <SCRATCH_STORE_BYTE_D16_HI, truncstorei8_hi16_private, i32>;2524}2525 2526let OtherPredicates = [D16PreservesUnusedBits, HasFlatScratchInsts, EnableFlatScratch] in {2527defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_private, v2i16>;2528defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16_HI, az_extloadi8_d16_hi_private, v2f16>;2529defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_private, v2i16>;2530defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16_HI, sextloadi8_d16_hi_private, v2f16>;2531defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16_HI, load_d16_hi_private, v2i16>;2532defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16_HI, load_d16_hi_private, v2f16>;2533 2534defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16, az_extloadi8_d16_lo_private, v2i16>;2535defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_UBYTE_D16, az_extloadi8_d16_lo_private, v2f16>;2536defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16, sextloadi8_d16_lo_private, v2i16>;2537defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SBYTE_D16, sextloadi8_d16_lo_private, v2f16>;2538defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16, load_d16_lo_private, v2i16>;2539defm : ScratchFLATLoadPats_D16 <SCRATCH_LOAD_SHORT_D16, load_d16_lo_private, v2f16>;2540}2541 2542} // End OtherPredicates = [HasFlatScratchInsts,EnableFlatScratch]2543 2544def PrefetchLoc: SDNodeXForm<timm, [{2545 uint32_t V = N->getZExtValue();2546 V = (AMDGPU::CPol::SCOPE_MASK - (V & AMDGPU::CPol::SCOPE_MASK)) << AMDGPU::CPol::SCOPE_SHIFT;2547 if (!Subtarget->hasSafeCUPrefetch())2548 V = std::max(V, (uint32_t)AMDGPU::CPol::SCOPE_SE); // CU scope is unsafe2549 return CurDAG->getTargetConstant(V, SDLoc(N), MVT::i32);2550}]>;2551 2552def prefetch_flat : PatFrag <(ops node:$ptr, node:$rw, node:$loc, node:$type),2553 (prefetch node:$ptr, node:$rw, node:$loc, node:$type),2554 [{ return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::FLAT_ADDRESS; }]> {2555 let GISelPredicateCode = [{2556 return (*MI.memoperands_begin())->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS;2557 }];2558}2559 2560def prefetch_global : PatFrag <(ops node:$ptr, node:$rw, node:$loc, node:$type),2561 (prefetch node:$ptr, node:$rw, node:$loc, node:$type),2562 [{ return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS ||2563 (cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS &&2564 !Subtarget->hasSafeSmemPrefetch()); }]> {2565 let GISelPredicateCode = [{2566 return (*MI.memoperands_begin())->getAddrSpace() == AMDGPUAS::GLOBAL_ADDRESS ||2567 ((*MI.memoperands_begin())->getAddrSpace() == AMDGPUAS::CONSTANT_ADDRESS &&2568 !Subtarget->hasSafeSmemPrefetch());2569 }];2570}2571 2572multiclass FlatPrefetchPats<string inst, SDPatternOperator prefetch_kind, SDPatternOperator rw> {2573 def : GCNPat <2574 (prefetch_kind (GlobalOffset (i64 VReg_64:$vaddr), i32:$offset), rw, (i32 timm:$loc), i32imm_one),2575 (!cast<FLAT_Pseudo>(inst) $vaddr, $offset, (i32 (PrefetchLoc $loc)))2576 > {2577 let AddedComplexity = !if(!eq(rw, i32imm_zero), 0, 25);2578 }2579 2580 def : GCNPat <2581 (prefetch_kind (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset), rw, (i32 timm:$loc), i32imm_one),2582 (!cast<FLAT_Pseudo>(inst#"_SADDR") $saddr, $voffset, $offset, (i32 (PrefetchLoc $loc)))2583 > {2584 let AddedComplexity = !if(!eq(rw, i32imm_zero), 11, 30);2585 }2586}2587 2588multiclass FlatIntrPrefetchPats<string inst, SDPatternOperator intr> {2589 def : GCNPat <2590 (intr (FlatOffset i64:$vaddr, i32:$offset), timm:$cpol),2591 (!cast<FLAT_Pseudo>(inst) $vaddr, $offset, $cpol)2592 >;2593 2594 def : GCNPat <2595 (intr (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset), timm:$cpol),2596 (!cast<FLAT_Pseudo>(inst#"_SADDR") $saddr, $voffset, $offset, $cpol)> {2597 let AddedComplexity = 11;2598 }2599}2600 2601let SubtargetPredicate = HasVmemPrefInsts in {2602 defm : FlatPrefetchPats<"FLAT_PREFETCH_B8", prefetch_flat, i32imm_zero>;2603 defm : FlatPrefetchPats<"GLOBAL_PREFETCH_B8", prefetch_global, i32imm_zero>;2604 2605 // Patterns for forced vector prefetch with rw = 1.2606 defm : FlatPrefetchPats<"FLAT_PREFETCH_B8", prefetch_flat, i32imm_one>;2607 defm : FlatPrefetchPats<"GLOBAL_PREFETCH_B8", prefetch_global, i32imm_one>;2608 2609 2610 // Patterns for target intrinsics2611 defm : FlatIntrPrefetchPats<"FLAT_PREFETCH_B8", int_amdgcn_flat_prefetch>;2612 defm : FlatIntrPrefetchPats<"GLOBAL_PREFETCH_B8", int_amdgcn_global_prefetch>;2613} // End SubtargetPredicate = HasVmemPrefInsts2614 2615//===----------------------------------------------------------------------===//2616// Target2617//===----------------------------------------------------------------------===//2618 2619//===----------------------------------------------------------------------===//2620// CI2621//===----------------------------------------------------------------------===//2622 2623class FLAT_Real_ci <bits<7> op, FLAT_Pseudo ps, string asmName = ps.Mnemonic> :2624 FLAT_Real <op, ps, asmName>,2625 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.SI> {2626 let AssemblerPredicate = isGFX7Only;2627 let DecoderNamespace="GFX7";2628}2629 2630def FLAT_LOAD_UBYTE_ci : FLAT_Real_ci <0x8, FLAT_LOAD_UBYTE>;2631def FLAT_LOAD_SBYTE_ci : FLAT_Real_ci <0x9, FLAT_LOAD_SBYTE>;2632def FLAT_LOAD_USHORT_ci : FLAT_Real_ci <0xa, FLAT_LOAD_USHORT>;2633def FLAT_LOAD_SSHORT_ci : FLAT_Real_ci <0xb, FLAT_LOAD_SSHORT>;2634def FLAT_LOAD_DWORD_ci : FLAT_Real_ci <0xc, FLAT_LOAD_DWORD>;2635def FLAT_LOAD_DWORDX2_ci : FLAT_Real_ci <0xd, FLAT_LOAD_DWORDX2>;2636def FLAT_LOAD_DWORDX4_ci : FLAT_Real_ci <0xe, FLAT_LOAD_DWORDX4>;2637def FLAT_LOAD_DWORDX3_ci : FLAT_Real_ci <0xf, FLAT_LOAD_DWORDX3>;2638 2639def FLAT_STORE_BYTE_ci : FLAT_Real_ci <0x18, FLAT_STORE_BYTE>;2640def FLAT_STORE_SHORT_ci : FLAT_Real_ci <0x1a, FLAT_STORE_SHORT>;2641def FLAT_STORE_DWORD_ci : FLAT_Real_ci <0x1c, FLAT_STORE_DWORD>;2642def FLAT_STORE_DWORDX2_ci : FLAT_Real_ci <0x1d, FLAT_STORE_DWORDX2>;2643def FLAT_STORE_DWORDX4_ci : FLAT_Real_ci <0x1e, FLAT_STORE_DWORDX4>;2644def FLAT_STORE_DWORDX3_ci : FLAT_Real_ci <0x1f, FLAT_STORE_DWORDX3>;2645 2646multiclass FLAT_Real_Atomics_ci <bits<7> op, string opName = NAME,2647 string asmName = !cast<FLAT_Pseudo>(opName).Mnemonic> {2648 defvar ps = !cast<FLAT_Pseudo>(opName);2649 defvar ps_rtn = !cast<FLAT_Pseudo>(opName#"_RTN");2650 2651 def _ci : FLAT_Real_ci<op, ps, asmName>;2652 def _RTN_ci : FLAT_Real_ci<op, ps_rtn, asmName>;2653}2654 2655defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_ci <0x30>;2656defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_ci <0x31>;2657defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_ci <0x32>;2658defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_ci <0x33>;2659defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_ci <0x35>;2660defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_ci <0x36>;2661defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_ci <0x37>;2662defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_ci <0x38>;2663defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_ci <0x39>;2664defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_ci <0x3a>;2665defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_ci <0x3b>;2666defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_ci <0x3c>;2667defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_ci <0x3d>;2668defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_ci <0x50>;2669defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_ci <0x51>;2670defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_ci <0x52>;2671defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_ci <0x53>;2672defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_ci <0x55>;2673defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_ci <0x56>;2674defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_ci <0x57>;2675defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_ci <0x58>;2676defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_ci <0x59>;2677defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_ci <0x5a>;2678defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_ci <0x5b>;2679defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_ci <0x5c>;2680defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_ci <0x5d>;2681 2682// CI Only flat instructions2683defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_ci <0x3e>;2684defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_ci <0x3f>;2685defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_ci <0x40>;2686defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Real_Atomics_ci <0x5e>;2687defm FLAT_ATOMIC_FMIN_X2 : FLAT_Real_Atomics_ci <0x5f, "FLAT_ATOMIC_MIN_F64", "flat_atomic_fmin_x2">;2688defm FLAT_ATOMIC_FMAX_X2 : FLAT_Real_Atomics_ci <0x60, "FLAT_ATOMIC_MAX_F64", "flat_atomic_fmax_x2">;2689 2690 2691//===----------------------------------------------------------------------===//2692// VI2693//===----------------------------------------------------------------------===//2694 2695class FLAT_Real_vi <bits<7> op, FLAT_Pseudo ps, bit has_sccb = ps.has_sccb> :2696 FLAT_Real <op, ps>,2697 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.VI> {2698 let AssemblerPredicate = isGFX8GFX9;2699 let DecoderNamespace = "GFX8";2700 2701 let Inst{25} = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccbValue);2702 let AsmString = ps.Mnemonic #2703 !subst("$sccb", !if(has_sccb, "$sccb",""), ps.AsmOperands);2704}2705 2706multiclass FLAT_Real_AllAddr_vi<bits<7> op,2707 bit has_sccb = !cast<FLAT_Pseudo>(NAME).has_sccb> {2708 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME), has_sccb>;2709 def _SADDR_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME#"_SADDR"), has_sccb>;2710}2711 2712class FLAT_Real_gfx940 <bits<7> op, FLAT_Pseudo ps> :2713 FLAT_Real <op, ps>,2714 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.GFX940> {2715 let AssemblerPredicate = isGFX940Plus;2716 let DecoderNamespace = "GFX9";2717 let Inst{13} = ps.sve;2718 let Inst{25} = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccbValue);2719}2720 2721multiclass FLAT_Real_AllAddr_SVE_vi<bits<7> op> {2722 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME)> {2723 let AssemblerPredicate = isGFX8GFX9NotGFX940;2724 let OtherPredicates = [isGFX8GFX9NotGFX940];2725 }2726 def _SADDR_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")> {2727 let DecoderNamespace = "GFX9";2728 }2729 let AssemblerPredicate = isGFX940Plus in {2730 def _VE_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME)>;2731 def _SVS_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME#"_SVS")>;2732 def _ST_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME#"_ST")>;2733 }2734}2735 2736multiclass FLAT_Real_AllAddr_LDS<bits<7> op, bits<7> pre_gfx940_op,2737 string pre_gfx940_name = !subst("_lds", "", !cast<FLAT_Pseudo>(NAME).Mnemonic),2738 bit has_sccb = !cast<FLAT_Pseudo>(NAME).has_sccb> {2739 2740 let OtherPredicates = [isGFX8GFX9NotGFX940] in {2741 def _vi : FLAT_Real_vi<pre_gfx940_op, !cast<FLAT_Pseudo>(NAME), has_sccb> {2742 let AsmString = pre_gfx940_name # !cast<FLAT_Pseudo>(NAME).AsmOperands # " lds";2743 }2744 def _SADDR_vi : FLAT_Real_vi<pre_gfx940_op, !cast<FLAT_Pseudo>(NAME#"_SADDR"), has_sccb> {2745 let AsmString = pre_gfx940_name # !cast<FLAT_Pseudo>(NAME#"_SADDR").AsmOperands # " lds";2746 }2747 }2748 2749 let AssemblerPredicate = isGFX940Plus in {2750 def _gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME)>;2751 def _SADDR_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")>;2752 }2753}2754 2755multiclass FLAT_Real_AllAddr_SVE_LDS<bits<7> op, bits<7> pre_gfx940_op> {2756 defm "" : FLAT_Real_AllAddr_LDS<op, pre_gfx940_op>;2757 def _SVS_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME#"_SVS")>;2758 def _ST_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME#"_ST")>;2759}2760 2761def FLAT_LOAD_UBYTE_vi : FLAT_Real_vi <0x10, FLAT_LOAD_UBYTE>;2762def FLAT_LOAD_SBYTE_vi : FLAT_Real_vi <0x11, FLAT_LOAD_SBYTE>;2763def FLAT_LOAD_USHORT_vi : FLAT_Real_vi <0x12, FLAT_LOAD_USHORT>;2764def FLAT_LOAD_SSHORT_vi : FLAT_Real_vi <0x13, FLAT_LOAD_SSHORT>;2765def FLAT_LOAD_DWORD_vi : FLAT_Real_vi <0x14, FLAT_LOAD_DWORD>;2766def FLAT_LOAD_DWORDX2_vi : FLAT_Real_vi <0x15, FLAT_LOAD_DWORDX2>;2767def FLAT_LOAD_DWORDX4_vi : FLAT_Real_vi <0x17, FLAT_LOAD_DWORDX4>;2768def FLAT_LOAD_DWORDX3_vi : FLAT_Real_vi <0x16, FLAT_LOAD_DWORDX3>;2769 2770def FLAT_STORE_BYTE_vi : FLAT_Real_vi <0x18, FLAT_STORE_BYTE>;2771def FLAT_STORE_BYTE_D16_HI_vi : FLAT_Real_vi <0x19, FLAT_STORE_BYTE_D16_HI>;2772def FLAT_STORE_SHORT_vi : FLAT_Real_vi <0x1a, FLAT_STORE_SHORT>;2773def FLAT_STORE_SHORT_D16_HI_vi : FLAT_Real_vi <0x1b, FLAT_STORE_SHORT_D16_HI>;2774def FLAT_STORE_DWORD_vi : FLAT_Real_vi <0x1c, FLAT_STORE_DWORD>;2775def FLAT_STORE_DWORDX2_vi : FLAT_Real_vi <0x1d, FLAT_STORE_DWORDX2>;2776def FLAT_STORE_DWORDX4_vi : FLAT_Real_vi <0x1f, FLAT_STORE_DWORDX4>;2777def FLAT_STORE_DWORDX3_vi : FLAT_Real_vi <0x1e, FLAT_STORE_DWORDX3>;2778 2779def FLAT_LOAD_UBYTE_D16_vi : FLAT_Real_vi <0x20, FLAT_LOAD_UBYTE_D16>;2780def FLAT_LOAD_UBYTE_D16_HI_vi : FLAT_Real_vi <0x21, FLAT_LOAD_UBYTE_D16_HI>;2781def FLAT_LOAD_SBYTE_D16_vi : FLAT_Real_vi <0x22, FLAT_LOAD_SBYTE_D16>;2782def FLAT_LOAD_SBYTE_D16_HI_vi : FLAT_Real_vi <0x23, FLAT_LOAD_SBYTE_D16_HI>;2783def FLAT_LOAD_SHORT_D16_vi : FLAT_Real_vi <0x24, FLAT_LOAD_SHORT_D16>;2784def FLAT_LOAD_SHORT_D16_HI_vi : FLAT_Real_vi <0x25, FLAT_LOAD_SHORT_D16_HI>;2785 2786multiclass FLAT_Real_Atomics_vi <bits<7> op,2787 bit has_sccb = !cast<FLAT_Pseudo>(NAME).has_sccb> {2788 defvar ps = !cast<FLAT_Pseudo>(NAME);2789 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr), has_sccb>;2790 def _RTN_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN"), has_sccb>;2791 def _RTN_agpr_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN_agpr"), has_sccb>;2792}2793 2794multiclass FLAT_Global_Real_Atomics_vi<bits<7> op,2795 bit has_sccb = !cast<FLAT_Pseudo>(NAME).has_sccb> :2796 FLAT_Real_AllAddr_vi<op, has_sccb> {2797 def _RTN_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_RTN"), has_sccb>;2798 def _SADDR_RTN_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN"), has_sccb>;2799 2800 def _RTN_agpr_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_RTN_agpr"), has_sccb>;2801 def _SADDR_RTN_agpr_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN_agpr"), has_sccb>;2802}2803 2804defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_vi <0x40>;2805defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_vi <0x41>;2806defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_vi <0x42>;2807defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_vi <0x43>;2808defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_vi <0x44>;2809defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_vi <0x45>;2810defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_vi <0x46>;2811defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_vi <0x47>;2812defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_vi <0x48>;2813defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_vi <0x49>;2814defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_vi <0x4a>;2815defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_vi <0x4b>;2816defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_vi <0x4c>;2817defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_vi <0x60>;2818defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_vi <0x61>;2819defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_vi <0x62>;2820defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_vi <0x63>;2821defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_vi <0x64>;2822defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_vi <0x65>;2823defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_vi <0x66>;2824defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_vi <0x67>;2825defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_vi <0x68>;2826defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_vi <0x69>;2827defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_vi <0x6a>;2828defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_vi <0x6b>;2829defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_vi <0x6c>;2830 2831defm GLOBAL_LOAD_UBYTE : FLAT_Real_AllAddr_vi <0x10>;2832defm GLOBAL_LOAD_SBYTE : FLAT_Real_AllAddr_vi <0x11>;2833defm GLOBAL_LOAD_USHORT : FLAT_Real_AllAddr_vi <0x12>;2834defm GLOBAL_LOAD_SSHORT : FLAT_Real_AllAddr_vi <0x13>;2835defm GLOBAL_LOAD_DWORD : FLAT_Real_AllAddr_vi <0x14>;2836defm GLOBAL_LOAD_DWORDX2 : FLAT_Real_AllAddr_vi <0x15>;2837defm GLOBAL_LOAD_DWORDX3 : FLAT_Real_AllAddr_vi <0x16>;2838defm GLOBAL_LOAD_DWORDX4 : FLAT_Real_AllAddr_vi <0x17>;2839 2840defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_vi <0x20>;2841defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x21>;2842defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_vi <0x22>;2843defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x23>;2844defm GLOBAL_LOAD_SHORT_D16 : FLAT_Real_AllAddr_vi <0x24>;2845defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x25>;2846 2847defm GLOBAL_STORE_BYTE : FLAT_Real_AllAddr_vi <0x18>;2848defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_vi <0x19>;2849defm GLOBAL_STORE_SHORT : FLAT_Real_AllAddr_vi <0x1a>;2850defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x1b>;2851defm GLOBAL_STORE_DWORD : FLAT_Real_AllAddr_vi <0x1c>;2852defm GLOBAL_STORE_DWORDX2 : FLAT_Real_AllAddr_vi <0x1d>;2853defm GLOBAL_STORE_DWORDX3 : FLAT_Real_AllAddr_vi <0x1e>;2854defm GLOBAL_STORE_DWORDX4 : FLAT_Real_AllAddr_vi <0x1f>;2855 2856defm GLOBAL_LOAD_LDS_UBYTE : FLAT_Real_AllAddr_LDS <0x026, 0x10>;2857defm GLOBAL_LOAD_LDS_SBYTE : FLAT_Real_AllAddr_LDS <0x027, 0x11>;2858defm GLOBAL_LOAD_LDS_USHORT : FLAT_Real_AllAddr_LDS <0x028, 0x12>;2859defm GLOBAL_LOAD_LDS_SSHORT : FLAT_Real_AllAddr_LDS <0x029, 0x13>;2860defm GLOBAL_LOAD_LDS_DWORD : FLAT_Real_AllAddr_LDS <0x02a, 0x14>;2861 2862defm GLOBAL_LOAD_LDS_DWORDX3 : FLAT_Real_AllAddr_LDS <0x07e, 0x07e>;2863defm GLOBAL_LOAD_LDS_DWORDX4 : FLAT_Real_AllAddr_LDS <0x07d, 0x07d>;2864 2865 2866defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Real_Atomics_vi <0x40>;2867defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Real_Atomics_vi <0x41>;2868defm GLOBAL_ATOMIC_ADD : FLAT_Global_Real_Atomics_vi <0x42>;2869defm GLOBAL_ATOMIC_SUB : FLAT_Global_Real_Atomics_vi <0x43>;2870defm GLOBAL_ATOMIC_SMIN : FLAT_Global_Real_Atomics_vi <0x44>;2871defm GLOBAL_ATOMIC_UMIN : FLAT_Global_Real_Atomics_vi <0x45>;2872defm GLOBAL_ATOMIC_SMAX : FLAT_Global_Real_Atomics_vi <0x46>;2873defm GLOBAL_ATOMIC_UMAX : FLAT_Global_Real_Atomics_vi <0x47>;2874defm GLOBAL_ATOMIC_AND : FLAT_Global_Real_Atomics_vi <0x48>;2875defm GLOBAL_ATOMIC_OR : FLAT_Global_Real_Atomics_vi <0x49>;2876defm GLOBAL_ATOMIC_XOR : FLAT_Global_Real_Atomics_vi <0x4a>;2877defm GLOBAL_ATOMIC_INC : FLAT_Global_Real_Atomics_vi <0x4b>;2878defm GLOBAL_ATOMIC_DEC : FLAT_Global_Real_Atomics_vi <0x4c>;2879defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Global_Real_Atomics_vi <0x60>;2880defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Real_Atomics_vi <0x61>;2881defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Global_Real_Atomics_vi <0x62>;2882defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Global_Real_Atomics_vi <0x63>;2883defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Global_Real_Atomics_vi <0x64>;2884defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Global_Real_Atomics_vi <0x65>;2885defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Global_Real_Atomics_vi <0x66>;2886defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Global_Real_Atomics_vi <0x67>;2887defm GLOBAL_ATOMIC_AND_X2 : FLAT_Global_Real_Atomics_vi <0x68>;2888defm GLOBAL_ATOMIC_OR_X2 : FLAT_Global_Real_Atomics_vi <0x69>;2889defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Global_Real_Atomics_vi <0x6a>;2890defm GLOBAL_ATOMIC_INC_X2 : FLAT_Global_Real_Atomics_vi <0x6b>;2891defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Real_Atomics_vi <0x6c>;2892 2893defm SCRATCH_LOAD_LDS_UBYTE : FLAT_Real_AllAddr_SVE_LDS <0x026, 0x10>;2894defm SCRATCH_LOAD_LDS_SBYTE : FLAT_Real_AllAddr_SVE_LDS <0x027, 0x11>;2895defm SCRATCH_LOAD_LDS_USHORT : FLAT_Real_AllAddr_SVE_LDS <0x028, 0x12>;2896defm SCRATCH_LOAD_LDS_SSHORT : FLAT_Real_AllAddr_SVE_LDS <0x029, 0x13>;2897defm SCRATCH_LOAD_LDS_DWORD : FLAT_Real_AllAddr_SVE_LDS <0x02a, 0x14>;2898 2899defm SCRATCH_LOAD_UBYTE : FLAT_Real_AllAddr_SVE_vi <0x10>;2900defm SCRATCH_LOAD_SBYTE : FLAT_Real_AllAddr_SVE_vi <0x11>;2901defm SCRATCH_LOAD_USHORT : FLAT_Real_AllAddr_SVE_vi <0x12>;2902defm SCRATCH_LOAD_SSHORT : FLAT_Real_AllAddr_SVE_vi <0x13>;2903defm SCRATCH_LOAD_DWORD : FLAT_Real_AllAddr_SVE_vi <0x14>;2904defm SCRATCH_LOAD_DWORDX2 : FLAT_Real_AllAddr_SVE_vi <0x15>;2905defm SCRATCH_LOAD_DWORDX3 : FLAT_Real_AllAddr_SVE_vi <0x16>;2906defm SCRATCH_LOAD_DWORDX4 : FLAT_Real_AllAddr_SVE_vi <0x17>;2907defm SCRATCH_STORE_BYTE : FLAT_Real_AllAddr_SVE_vi <0x18>;2908defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x19>;2909defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_SVE_vi <0x20>;2910defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x21>;2911defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_SVE_vi <0x22>;2912defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x23>;2913defm SCRATCH_LOAD_SHORT_D16 : FLAT_Real_AllAddr_SVE_vi <0x24>;2914defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x25>;2915defm SCRATCH_STORE_SHORT : FLAT_Real_AllAddr_SVE_vi <0x1a>;2916defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_SVE_vi <0x1b>;2917defm SCRATCH_STORE_DWORD : FLAT_Real_AllAddr_SVE_vi <0x1c>;2918defm SCRATCH_STORE_DWORDX2 : FLAT_Real_AllAddr_SVE_vi <0x1d>;2919defm SCRATCH_STORE_DWORDX3 : FLAT_Real_AllAddr_SVE_vi <0x1e>;2920defm SCRATCH_STORE_DWORDX4 : FLAT_Real_AllAddr_SVE_vi <0x1f>;2921 2922let AssemblerPredicate = isGFX8GFX9NotGFX940 in {2923 // These instructions are encoded differently on gfx90* and gfx94*.2924 defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Real_Atomics_vi <0x04d, 0>;2925 defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Real_Atomics_vi <0x04e, 0>;2926}2927 2928defm FLAT_ATOMIC_ADD_F64 : FLAT_Real_Atomics_vi<0x4f, 0>;2929defm FLAT_ATOMIC_MIN_F64 : FLAT_Real_Atomics_vi<0x50, 0>;2930defm FLAT_ATOMIC_MAX_F64 : FLAT_Real_Atomics_vi<0x51, 0>;2931defm GLOBAL_ATOMIC_ADD_F64 : FLAT_Global_Real_Atomics_vi<0x4f, 0>;2932defm GLOBAL_ATOMIC_MIN_F64 : FLAT_Global_Real_Atomics_vi<0x50, 0>;2933defm GLOBAL_ATOMIC_MAX_F64 : FLAT_Global_Real_Atomics_vi<0x51, 0>;2934 2935multiclass FLAT_Real_AllAddr_gfx940<bits<7> op> {2936 def _gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME)>;2937 def _SADDR_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")>;2938}2939 2940multiclass FLAT_Real_Atomics_gfx940 <bits<7> op> {2941 defvar ps = !cast<FLAT_Pseudo>(NAME);2942 def _gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(ps.PseudoInstr)>;2943 def _RTN_gfx940 : FLAT_Real_gfx940<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN")>;2944}2945 2946multiclass FLAT_Global_Real_Atomics_gfx940<bits<7> op> :2947 FLAT_Real_AllAddr_gfx940<op> {2948 def _RTN_gfx940 : FLAT_Real_gfx940 <op, !cast<FLAT_Pseudo>(NAME#"_RTN")>;2949 def _SADDR_RTN_gfx940 : FLAT_Real_gfx940 <op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN")>;2950}2951 2952let AssemblerPredicate = isGFX940Plus in {2953 // These instructions are encoded differently on gfx90* and gfx940.2954 defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Real_Atomics_gfx940 <0x04d>;2955 defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Real_Atomics_gfx940 <0x04e>;2956 2957 defm FLAT_ATOMIC_ADD_F64 : FLAT_Real_Atomics_gfx940<0x4f>;2958 defm FLAT_ATOMIC_MIN_F64 : FLAT_Real_Atomics_gfx940<0x50>;2959 defm FLAT_ATOMIC_MAX_F64 : FLAT_Real_Atomics_gfx940<0x51>;2960 defm GLOBAL_ATOMIC_ADD_F64 : FLAT_Global_Real_Atomics_gfx940<0x4f>;2961 defm GLOBAL_ATOMIC_MIN_F64 : FLAT_Global_Real_Atomics_gfx940<0x50>;2962 defm GLOBAL_ATOMIC_MAX_F64 : FLAT_Global_Real_Atomics_gfx940<0x51>;2963 defm FLAT_ATOMIC_ADD_F32 : FLAT_Real_Atomics_vi<0x4d>;2964 defm FLAT_ATOMIC_PK_ADD_F16 : FLAT_Real_Atomics_vi<0x4e>;2965 defm FLAT_ATOMIC_PK_ADD_BF16 : FLAT_Real_Atomics_vi<0x52>;2966 defm GLOBAL_ATOMIC_PK_ADD_BF16 : FLAT_Global_Real_Atomics_vi<0x52>;2967} // End AssemblerPredicate = isGFX940Plus2968 2969//===----------------------------------------------------------------------===//2970// GFX10.2971//===----------------------------------------------------------------------===//2972 2973class FLAT_Real_gfx10<bits<7> op, FLAT_Pseudo ps, string opName = ps.Mnemonic> :2974 FLAT_Real<op, ps, opName>, SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX10> {2975 let AssemblerPredicate = isGFX10Only;2976 let DecoderNamespace = "GFX10";2977 2978 let Inst{11-0} = offset{11-0};2979 let Inst{12} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlcValue);2980 let Inst{54-48} = !cond(ps.enabled_saddr : saddr,2981 !and(ps.is_flat_scratch, !not(ps.has_vaddr)) : EXEC_HI.Index{6-0}, // ST mode2982 true : SGPR_NULL_gfxpre11.Index{6-0});2983 let Inst{55} = 0;2984}2985 2986multiclass FLAT_Real_Base_gfx10<bits<7> op, string psName = NAME,2987 string asmName = !cast<FLAT_Pseudo>(psName).Mnemonic> {2988 def _gfx10 :2989 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(psName), asmName>;2990}2991 2992multiclass FLAT_Real_RTN_gfx10<bits<7> op, string psName = NAME,2993 string asmName = !cast<FLAT_Pseudo>(psName).Mnemonic> {2994 def _RTN_gfx10 :2995 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(psName#"_RTN"), asmName>;2996}2997 2998multiclass FLAT_Real_SADDR_gfx10<bits<7> op, string psName = NAME,2999 string asmName = !cast<FLAT_Pseudo>(psName#"_SADDR").Mnemonic> {3000 def _SADDR_gfx10 :3001 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(psName#"_SADDR"), asmName>;3002}3003 3004multiclass FLAT_Real_SADDR_RTN_gfx10<bits<7> op, string psName = NAME,3005 string asmName = !cast<FLAT_Pseudo>(psName#"_SADDR_RTN").Mnemonic> {3006 def _SADDR_RTN_gfx10 :3007 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(psName#"_SADDR_RTN"), asmName>;3008}3009 3010multiclass FLAT_Real_ST_gfx10<bits<7> op> {3011 def _ST_gfx10 :3012 FLAT_Real_gfx10<op, !cast<FLAT_Pseudo>(NAME#"_ST")>;3013}3014 3015multiclass FLAT_Real_AllAddr_gfx10<bits<7> op, string OpName = NAME,3016 string asmName = !cast<FLAT_Pseudo>(OpName).Mnemonic> :3017 FLAT_Real_Base_gfx10<op, OpName, asmName>,3018 FLAT_Real_SADDR_gfx10<op, OpName, asmName>;3019 3020multiclass FLAT_Real_Atomics_gfx10<bits<7> op, string OpName = NAME,3021 string asmName = !cast<FLAT_Pseudo>(OpName).Mnemonic> :3022 FLAT_Real_Base_gfx10<op, OpName, asmName>,3023 FLAT_Real_RTN_gfx10<op, OpName, asmName>;3024 3025multiclass FLAT_Real_GlblAtomics_gfx10<bits<7> op, string OpName = NAME,3026 string asmName = !cast<FLAT_Pseudo>(OpName).Mnemonic> :3027 FLAT_Real_AllAddr_gfx10<op, OpName, asmName>,3028 FLAT_Real_RTN_gfx10<op, OpName, asmName>,3029 FLAT_Real_SADDR_RTN_gfx10<op, OpName, asmName>;3030 3031multiclass FLAT_Real_GlblAtomics_RTN_gfx10<bits<7> op, string OpName = NAME> :3032 FLAT_Real_RTN_gfx10<op, OpName>,3033 FLAT_Real_SADDR_RTN_gfx10<op, OpName>;3034 3035multiclass FLAT_Real_ScratchAllAddr_gfx10<bits<7> op> :3036 FLAT_Real_Base_gfx10<op>,3037 FLAT_Real_SADDR_gfx10<op>,3038 FLAT_Real_ST_gfx10<op>;3039 3040multiclass FLAT_Real_AllAddr_LDS_gfx10<bits<7> op,3041 string opname = !subst("_lds", "", !cast<FLAT_Pseudo>(NAME).Mnemonic)> {3042 let AsmString = opname # !cast<FLAT_Pseudo>(NAME).AsmOperands # " lds" in3043 defm "" : FLAT_Real_Base_gfx10<op>;3044 3045 let AsmString = opname # !cast<FLAT_Pseudo>(NAME#"_SADDR").AsmOperands # " lds" in3046 defm "" : FLAT_Real_SADDR_gfx10<op>;3047}3048 3049multiclass FLAT_Real_ScratchAllAddr_LDS_gfx10<bits<7> op,3050 string opname = !subst("_lds", "", !cast<FLAT_Pseudo>(NAME).Mnemonic)> {3051 defm "" : FLAT_Real_AllAddr_LDS_gfx10<op>;3052 3053 let AsmString = opname # !cast<FLAT_Pseudo>(NAME#"_ST").AsmOperands # " lds" in3054 defm "" : FLAT_Real_ST_gfx10<op>;3055}3056 3057// ENC_FLAT.3058defm FLAT_LOAD_UBYTE : FLAT_Real_Base_gfx10<0x008>;3059defm FLAT_LOAD_SBYTE : FLAT_Real_Base_gfx10<0x009>;3060defm FLAT_LOAD_USHORT : FLAT_Real_Base_gfx10<0x00a>;3061defm FLAT_LOAD_SSHORT : FLAT_Real_Base_gfx10<0x00b>;3062defm FLAT_LOAD_DWORD : FLAT_Real_Base_gfx10<0x00c>;3063defm FLAT_LOAD_DWORDX2 : FLAT_Real_Base_gfx10<0x00d>;3064defm FLAT_LOAD_DWORDX4 : FLAT_Real_Base_gfx10<0x00e>;3065defm FLAT_LOAD_DWORDX3 : FLAT_Real_Base_gfx10<0x00f>;3066defm FLAT_STORE_BYTE : FLAT_Real_Base_gfx10<0x018>;3067defm FLAT_STORE_BYTE_D16_HI : FLAT_Real_Base_gfx10<0x019>;3068defm FLAT_STORE_SHORT : FLAT_Real_Base_gfx10<0x01a>;3069defm FLAT_STORE_SHORT_D16_HI : FLAT_Real_Base_gfx10<0x01b>;3070defm FLAT_STORE_DWORD : FLAT_Real_Base_gfx10<0x01c>;3071defm FLAT_STORE_DWORDX2 : FLAT_Real_Base_gfx10<0x01d>;3072defm FLAT_STORE_DWORDX4 : FLAT_Real_Base_gfx10<0x01e>;3073defm FLAT_STORE_DWORDX3 : FLAT_Real_Base_gfx10<0x01f>;3074defm FLAT_LOAD_UBYTE_D16 : FLAT_Real_Base_gfx10<0x020>;3075defm FLAT_LOAD_UBYTE_D16_HI : FLAT_Real_Base_gfx10<0x021>;3076defm FLAT_LOAD_SBYTE_D16 : FLAT_Real_Base_gfx10<0x022>;3077defm FLAT_LOAD_SBYTE_D16_HI : FLAT_Real_Base_gfx10<0x023>;3078defm FLAT_LOAD_SHORT_D16 : FLAT_Real_Base_gfx10<0x024>;3079defm FLAT_LOAD_SHORT_D16_HI : FLAT_Real_Base_gfx10<0x025>;3080defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_gfx10<0x030>;3081defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_gfx10<0x031>;3082defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_gfx10<0x032>;3083defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_gfx10<0x033>;3084defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_gfx10<0x035>;3085defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_gfx10<0x036>;3086defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_gfx10<0x037>;3087defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_gfx10<0x038>;3088defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_gfx10<0x039>;3089defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_gfx10<0x03a>;3090defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_gfx10<0x03b>;3091defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_gfx10<0x03c>;3092defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_gfx10<0x03d>;3093defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_gfx10<0x03e>;3094defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_gfx10<0x03f>;3095defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_gfx10<0x040>;3096defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_gfx10<0x050>;3097defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_gfx10<0x051>;3098defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_gfx10<0x052>;3099defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_gfx10<0x053>;3100defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_gfx10<0x055>;3101defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_gfx10<0x056>;3102defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_gfx10<0x057>;3103defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_gfx10<0x058>;3104defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_gfx10<0x059>;3105defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_gfx10<0x05a>;3106defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_gfx10<0x05b>;3107defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_gfx10<0x05c>;3108defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_gfx10<0x05d>;3109defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Real_Atomics_gfx10<0x05e>;3110defm FLAT_ATOMIC_FMIN_X2 : FLAT_Real_Atomics_gfx10<0x05f, "FLAT_ATOMIC_MIN_F64", "flat_atomic_fmin_x2">;3111defm FLAT_ATOMIC_FMAX_X2 : FLAT_Real_Atomics_gfx10<0x060, "FLAT_ATOMIC_MAX_F64", "flat_atomic_fmax_x2">;3112 3113 3114// ENC_FLAT_GLBL.3115defm GLOBAL_LOAD_UBYTE : FLAT_Real_AllAddr_gfx10<0x008>;3116defm GLOBAL_LOAD_SBYTE : FLAT_Real_AllAddr_gfx10<0x009>;3117defm GLOBAL_LOAD_USHORT : FLAT_Real_AllAddr_gfx10<0x00a>;3118defm GLOBAL_LOAD_SSHORT : FLAT_Real_AllAddr_gfx10<0x00b>;3119defm GLOBAL_LOAD_DWORD : FLAT_Real_AllAddr_gfx10<0x00c>;3120defm GLOBAL_LOAD_DWORDX2 : FLAT_Real_AllAddr_gfx10<0x00d>;3121defm GLOBAL_LOAD_DWORDX4 : FLAT_Real_AllAddr_gfx10<0x00e>;3122defm GLOBAL_LOAD_DWORDX3 : FLAT_Real_AllAddr_gfx10<0x00f>;3123defm GLOBAL_STORE_BYTE : FLAT_Real_AllAddr_gfx10<0x018>;3124defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x019>;3125defm GLOBAL_STORE_SHORT : FLAT_Real_AllAddr_gfx10<0x01a>;3126defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_gfx10<0x01b>;3127defm GLOBAL_STORE_DWORD : FLAT_Real_AllAddr_gfx10<0x01c>;3128defm GLOBAL_STORE_DWORDX2 : FLAT_Real_AllAddr_gfx10<0x01d>;3129defm GLOBAL_STORE_DWORDX4 : FLAT_Real_AllAddr_gfx10<0x01e>;3130defm GLOBAL_STORE_DWORDX3 : FLAT_Real_AllAddr_gfx10<0x01f>;3131defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_gfx10<0x020>;3132defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x021>;3133defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_gfx10<0x022>;3134defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_gfx10<0x023>;3135defm GLOBAL_LOAD_SHORT_D16 : FLAT_Real_AllAddr_gfx10<0x024>;3136defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_gfx10<0x025>;3137defm GLOBAL_ATOMIC_SWAP : FLAT_Real_GlblAtomics_gfx10<0x030>;3138defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Real_GlblAtomics_gfx10<0x031>;3139defm GLOBAL_ATOMIC_ADD : FLAT_Real_GlblAtomics_gfx10<0x032>;3140defm GLOBAL_ATOMIC_SUB : FLAT_Real_GlblAtomics_gfx10<0x033>;3141defm GLOBAL_ATOMIC_CSUB : FLAT_Real_GlblAtomics_gfx10<0x034>;3142defm GLOBAL_ATOMIC_SMIN : FLAT_Real_GlblAtomics_gfx10<0x035>;3143defm GLOBAL_ATOMIC_UMIN : FLAT_Real_GlblAtomics_gfx10<0x036>;3144defm GLOBAL_ATOMIC_SMAX : FLAT_Real_GlblAtomics_gfx10<0x037>;3145defm GLOBAL_ATOMIC_UMAX : FLAT_Real_GlblAtomics_gfx10<0x038>;3146defm GLOBAL_ATOMIC_AND : FLAT_Real_GlblAtomics_gfx10<0x039>;3147defm GLOBAL_ATOMIC_OR : FLAT_Real_GlblAtomics_gfx10<0x03a>;3148defm GLOBAL_ATOMIC_XOR : FLAT_Real_GlblAtomics_gfx10<0x03b>;3149defm GLOBAL_ATOMIC_INC : FLAT_Real_GlblAtomics_gfx10<0x03c>;3150defm GLOBAL_ATOMIC_DEC : FLAT_Real_GlblAtomics_gfx10<0x03d>;3151defm GLOBAL_ATOMIC_FCMPSWAP : FLAT_Real_GlblAtomics_gfx10<0x03e>;3152defm GLOBAL_ATOMIC_FMIN : FLAT_Real_GlblAtomics_gfx10<0x03f>;3153defm GLOBAL_ATOMIC_FMAX : FLAT_Real_GlblAtomics_gfx10<0x040>;3154defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x050>;3155defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x051>;3156defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Real_GlblAtomics_gfx10<0x052>;3157defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Real_GlblAtomics_gfx10<0x053>;3158defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x055>;3159defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x056>;3160defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x057>;3161defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x058>;3162defm GLOBAL_ATOMIC_AND_X2 : FLAT_Real_GlblAtomics_gfx10<0x059>;3163defm GLOBAL_ATOMIC_OR_X2 : FLAT_Real_GlblAtomics_gfx10<0x05a>;3164defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Real_GlblAtomics_gfx10<0x05b>;3165defm GLOBAL_ATOMIC_INC_X2 : FLAT_Real_GlblAtomics_gfx10<0x05c>;3166defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Real_GlblAtomics_gfx10<0x05d>;3167defm GLOBAL_ATOMIC_FCMPSWAP_X2 : FLAT_Real_GlblAtomics_gfx10<0x05e>;3168defm GLOBAL_ATOMIC_FMIN_X2 : FLAT_Real_GlblAtomics_gfx10<0x05f, "GLOBAL_ATOMIC_MIN_F64", "global_atomic_fmin_x2">;3169defm GLOBAL_ATOMIC_FMAX_X2 : FLAT_Real_GlblAtomics_gfx10<0x060, "GLOBAL_ATOMIC_MAX_F64", "global_atomic_fmax_x2">;3170defm GLOBAL_LOAD_DWORD_ADDTID : FLAT_Real_AllAddr_gfx10<0x016>;3171defm GLOBAL_STORE_DWORD_ADDTID : FLAT_Real_AllAddr_gfx10<0x017>;3172 3173defm GLOBAL_LOAD_LDS_UBYTE : FLAT_Real_AllAddr_LDS_gfx10 <0x008>;3174defm GLOBAL_LOAD_LDS_SBYTE : FLAT_Real_AllAddr_LDS_gfx10 <0x009>;3175defm GLOBAL_LOAD_LDS_USHORT : FLAT_Real_AllAddr_LDS_gfx10 <0x00a>;3176defm GLOBAL_LOAD_LDS_SSHORT : FLAT_Real_AllAddr_LDS_gfx10 <0x00b>;3177defm GLOBAL_LOAD_LDS_DWORD : FLAT_Real_AllAddr_LDS_gfx10 <0x00c>;3178 3179// ENC_FLAT_SCRATCH.3180defm SCRATCH_LOAD_UBYTE : FLAT_Real_ScratchAllAddr_gfx10<0x008>;3181defm SCRATCH_LOAD_SBYTE : FLAT_Real_ScratchAllAddr_gfx10<0x009>;3182defm SCRATCH_LOAD_USHORT : FLAT_Real_ScratchAllAddr_gfx10<0x00a>;3183defm SCRATCH_LOAD_SSHORT : FLAT_Real_ScratchAllAddr_gfx10<0x00b>;3184defm SCRATCH_LOAD_DWORD : FLAT_Real_ScratchAllAddr_gfx10<0x00c>;3185defm SCRATCH_LOAD_DWORDX2 : FLAT_Real_ScratchAllAddr_gfx10<0x00d>;3186defm SCRATCH_LOAD_DWORDX4 : FLAT_Real_ScratchAllAddr_gfx10<0x00e>;3187defm SCRATCH_LOAD_DWORDX3 : FLAT_Real_ScratchAllAddr_gfx10<0x00f>;3188defm SCRATCH_STORE_BYTE : FLAT_Real_ScratchAllAddr_gfx10<0x018>;3189defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x019>;3190defm SCRATCH_STORE_SHORT : FLAT_Real_ScratchAllAddr_gfx10<0x01a>;3191defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x01b>;3192defm SCRATCH_STORE_DWORD : FLAT_Real_ScratchAllAddr_gfx10<0x01c>;3193defm SCRATCH_STORE_DWORDX2 : FLAT_Real_ScratchAllAddr_gfx10<0x01d>;3194defm SCRATCH_STORE_DWORDX4 : FLAT_Real_ScratchAllAddr_gfx10<0x01e>;3195defm SCRATCH_STORE_DWORDX3 : FLAT_Real_ScratchAllAddr_gfx10<0x01f>;3196defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x020>;3197defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x021>;3198defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x022>;3199defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x023>;3200defm SCRATCH_LOAD_SHORT_D16 : FLAT_Real_ScratchAllAddr_gfx10<0x024>;3201defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Real_ScratchAllAddr_gfx10<0x025>;3202 3203defm SCRATCH_LOAD_LDS_UBYTE : FLAT_Real_ScratchAllAddr_LDS_gfx10 <0x008>;3204defm SCRATCH_LOAD_LDS_SBYTE : FLAT_Real_ScratchAllAddr_LDS_gfx10 <0x009>;3205defm SCRATCH_LOAD_LDS_USHORT : FLAT_Real_ScratchAllAddr_LDS_gfx10 <0x00a>;3206defm SCRATCH_LOAD_LDS_SSHORT : FLAT_Real_ScratchAllAddr_LDS_gfx10 <0x00b>;3207defm SCRATCH_LOAD_LDS_DWORD : FLAT_Real_ScratchAllAddr_LDS_gfx10 <0x00c>;3208 3209//===----------------------------------------------------------------------===//3210// GFX113211//===----------------------------------------------------------------------===//3212 3213class get_FLAT_ps<string name> {3214 string Mnemonic = !cast<FLAT_Pseudo>(name).Mnemonic;3215}3216 3217multiclass FLAT_Real_gfx11 <bits<7> op,3218 string name = get_FLAT_ps<NAME>.Mnemonic> {3219 defvar ps = !cast<FLAT_Pseudo>(NAME);3220 def _gfx11 : FLAT_Real <op, ps, name>,3221 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.GFX11> {3222 let AssemblerPredicate = isGFX11Only;3223 let DecoderNamespace = "GFX11";3224 3225 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlcValue);3226 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glcValue);3227 let Inst{15} = cpol{CPolBit.SLC};3228 let Inst{17-16} = seg;3229 let Inst{54-48} = !if(ps.enabled_saddr, saddr, SGPR_NULL_gfx11plus.Index);3230 let Inst{55} = ps.sve;3231 }3232}3233 3234multiclass FLAT_Aliases_gfx11<string name> {3235 defvar ps = get_FLAT_ps<NAME>;3236 if !ne(ps.Mnemonic, name) then3237 def : AMDGPUMnemonicAlias<ps.Mnemonic, name> {3238 let AssemblerPredicate = isGFX11Only;3239 }3240}3241 3242multiclass FLAT_Real_Base_gfx11<bits<7> op,3243 string name = get_FLAT_ps<NAME>.Mnemonic> :3244 FLAT_Aliases_gfx11<name>,3245 FLAT_Real_gfx11<op, name>;3246 3247multiclass FLAT_Real_Atomics_gfx11<bits<7> op,3248 string name = get_FLAT_ps<NAME>.Mnemonic> :3249 FLAT_Real_Base_gfx11<op, name> {3250 defm _RTN : FLAT_Real_gfx11<op, name>;3251}3252 3253multiclass GLOBAL_Real_AllAddr_gfx11<bits<7> op,3254 string name = get_FLAT_ps<NAME>.Mnemonic> :3255 FLAT_Real_Base_gfx11<op, name> {3256 defm _SADDR : FLAT_Real_gfx11<op, name>;3257}3258 3259multiclass GLOBAL_Real_Atomics_gfx11<bits<7> op,3260 string name = get_FLAT_ps<NAME>.Mnemonic> :3261 GLOBAL_Real_AllAddr_gfx11<op, name> {3262 defm _RTN : FLAT_Real_gfx11<op, name>;3263 defm _SADDR_RTN : FLAT_Real_gfx11<op, name>;3264}3265 3266multiclass SCRATCH_Real_AllAddr_gfx11<bits<7> op,3267 string name = get_FLAT_ps<NAME>.Mnemonic> :3268 FLAT_Real_Base_gfx11<op, name> {3269 defm _SADDR : FLAT_Real_gfx11<op, name>;3270 defm _ST : FLAT_Real_gfx11<op, name>;3271 defm _SVS : FLAT_Real_gfx11<op, name>;3272}3273 3274// ENC_FLAT.3275defm FLAT_LOAD_UBYTE : FLAT_Real_Base_gfx11<0x010, "flat_load_u8">;3276defm FLAT_LOAD_SBYTE : FLAT_Real_Base_gfx11<0x011, "flat_load_i8">;3277defm FLAT_LOAD_USHORT : FLAT_Real_Base_gfx11<0x012, "flat_load_u16">;3278defm FLAT_LOAD_SSHORT : FLAT_Real_Base_gfx11<0x013, "flat_load_i16">;3279defm FLAT_LOAD_DWORD : FLAT_Real_Base_gfx11<0x014, "flat_load_b32">;3280defm FLAT_LOAD_DWORDX2 : FLAT_Real_Base_gfx11<0x015, "flat_load_b64">;3281defm FLAT_LOAD_DWORDX3 : FLAT_Real_Base_gfx11<0x016, "flat_load_b96">;3282defm FLAT_LOAD_DWORDX4 : FLAT_Real_Base_gfx11<0x017, "flat_load_b128">;3283defm FLAT_STORE_BYTE : FLAT_Real_Base_gfx11<0x018, "flat_store_b8">;3284defm FLAT_STORE_SHORT : FLAT_Real_Base_gfx11<0x019, "flat_store_b16">;3285defm FLAT_STORE_DWORD : FLAT_Real_Base_gfx11<0x01a, "flat_store_b32">;3286defm FLAT_STORE_DWORDX2 : FLAT_Real_Base_gfx11<0x01b, "flat_store_b64">;3287defm FLAT_STORE_DWORDX3 : FLAT_Real_Base_gfx11<0x01c, "flat_store_b96">;3288defm FLAT_STORE_DWORDX4 : FLAT_Real_Base_gfx11<0x01d, "flat_store_b128">;3289defm FLAT_LOAD_UBYTE_D16 : FLAT_Real_Base_gfx11<0x01e, "flat_load_d16_u8">;3290defm FLAT_LOAD_SBYTE_D16 : FLAT_Real_Base_gfx11<0x01f, "flat_load_d16_i8">;3291defm FLAT_LOAD_SHORT_D16 : FLAT_Real_Base_gfx11<0x020, "flat_load_d16_b16">;3292defm FLAT_LOAD_UBYTE_D16_HI : FLAT_Real_Base_gfx11<0x021, "flat_load_d16_hi_u8">;3293defm FLAT_LOAD_SBYTE_D16_HI : FLAT_Real_Base_gfx11<0x022, "flat_load_d16_hi_i8">;3294defm FLAT_LOAD_SHORT_D16_HI : FLAT_Real_Base_gfx11<0x023, "flat_load_d16_hi_b16">;3295defm FLAT_STORE_BYTE_D16_HI : FLAT_Real_Base_gfx11<0x024, "flat_store_d16_hi_b8">;3296defm FLAT_STORE_SHORT_D16_HI : FLAT_Real_Base_gfx11<0x025, "flat_store_d16_hi_b16">;3297defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_gfx11<0x033, "flat_atomic_swap_b32">;3298defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_gfx11<0x034, "flat_atomic_cmpswap_b32">;3299defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_gfx11<0x035, "flat_atomic_add_u32">;3300defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_gfx11<0x036, "flat_atomic_sub_u32">;3301defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_gfx11<0x038, "flat_atomic_min_i32">;3302defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_gfx11<0x039, "flat_atomic_min_u32">;3303defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_gfx11<0x03a, "flat_atomic_max_i32">;3304defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_gfx11<0x03b, "flat_atomic_max_u32">;3305defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_gfx11<0x03c, "flat_atomic_and_b32">;3306defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_gfx11<0x03d, "flat_atomic_or_b32">;3307defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_gfx11<0x03e, "flat_atomic_xor_b32">;3308defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_gfx11<0x03f, "flat_atomic_inc_u32">;3309defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_gfx11<0x040, "flat_atomic_dec_u32">;3310defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_gfx11<0x041, "flat_atomic_swap_b64">;3311defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_gfx11<0x042, "flat_atomic_cmpswap_b64">;3312defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_gfx11<0x043, "flat_atomic_add_u64">;3313defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_gfx11<0x044, "flat_atomic_sub_u64">;3314defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_gfx11<0x045, "flat_atomic_min_i64">;3315defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_gfx11<0x046, "flat_atomic_min_u64">;3316defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_gfx11<0x047, "flat_atomic_max_i64">;3317defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_gfx11<0x048, "flat_atomic_max_u64">;3318defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_gfx11<0x049, "flat_atomic_and_b64">;3319defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_gfx11<0x04a, "flat_atomic_or_b64">;3320defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_gfx11<0x04b, "flat_atomic_xor_b64">;3321defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_gfx11<0x04c, "flat_atomic_inc_u64">;3322defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_gfx11<0x04d, "flat_atomic_dec_u64">;3323defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_gfx11<0x050, "flat_atomic_cmpswap_f32">;3324defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_gfx11<0x051, "flat_atomic_min_f32">;3325defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_gfx11<0x052, "flat_atomic_max_f32">;3326defm FLAT_ATOMIC_ADD_F32 : FLAT_Real_Atomics_gfx11<0x056>;3327 3328// ENC_FLAT_GLBL.3329defm GLOBAL_LOAD_UBYTE : GLOBAL_Real_AllAddr_gfx11<0x010, "global_load_u8">;3330defm GLOBAL_LOAD_SBYTE : GLOBAL_Real_AllAddr_gfx11<0x011, "global_load_i8">;3331defm GLOBAL_LOAD_USHORT : GLOBAL_Real_AllAddr_gfx11<0x012, "global_load_u16">;3332defm GLOBAL_LOAD_SSHORT : GLOBAL_Real_AllAddr_gfx11<0x013, "global_load_i16">;3333defm GLOBAL_LOAD_DWORD : GLOBAL_Real_AllAddr_gfx11<0x014, "global_load_b32">;3334defm GLOBAL_LOAD_DWORDX2 : GLOBAL_Real_AllAddr_gfx11<0x015, "global_load_b64">;3335defm GLOBAL_LOAD_DWORDX3 : GLOBAL_Real_AllAddr_gfx11<0x016, "global_load_b96">;3336defm GLOBAL_LOAD_DWORDX4 : GLOBAL_Real_AllAddr_gfx11<0x017, "global_load_b128">;3337defm GLOBAL_STORE_BYTE : GLOBAL_Real_AllAddr_gfx11<0x018, "global_store_b8">;3338defm GLOBAL_STORE_SHORT : GLOBAL_Real_AllAddr_gfx11<0x019, "global_store_b16">;3339defm GLOBAL_STORE_DWORD : GLOBAL_Real_AllAddr_gfx11<0x01a, "global_store_b32">;3340defm GLOBAL_STORE_DWORDX2 : GLOBAL_Real_AllAddr_gfx11<0x01b, "global_store_b64">;3341defm GLOBAL_STORE_DWORDX3 : GLOBAL_Real_AllAddr_gfx11<0x01c, "global_store_b96">;3342defm GLOBAL_STORE_DWORDX4 : GLOBAL_Real_AllAddr_gfx11<0x01d, "global_store_b128">;3343defm GLOBAL_LOAD_UBYTE_D16 : GLOBAL_Real_AllAddr_gfx11<0x01e, "global_load_d16_u8">;3344defm GLOBAL_LOAD_SBYTE_D16 : GLOBAL_Real_AllAddr_gfx11<0x01f, "global_load_d16_i8">;3345defm GLOBAL_LOAD_SHORT_D16 : GLOBAL_Real_AllAddr_gfx11<0x020, "global_load_d16_b16">;3346defm GLOBAL_LOAD_UBYTE_D16_HI : GLOBAL_Real_AllAddr_gfx11<0x021, "global_load_d16_hi_u8">;3347defm GLOBAL_LOAD_SBYTE_D16_HI : GLOBAL_Real_AllAddr_gfx11<0x022, "global_load_d16_hi_i8">;3348defm GLOBAL_LOAD_SHORT_D16_HI : GLOBAL_Real_AllAddr_gfx11<0x023, "global_load_d16_hi_b16">;3349defm GLOBAL_STORE_BYTE_D16_HI : GLOBAL_Real_AllAddr_gfx11<0x024, "global_store_d16_hi_b8">;3350defm GLOBAL_STORE_SHORT_D16_HI : GLOBAL_Real_AllAddr_gfx11<0x025, "global_store_d16_hi_b16">;3351defm GLOBAL_LOAD_DWORD_ADDTID : GLOBAL_Real_AllAddr_gfx11<0x028, "global_load_addtid_b32">;3352defm GLOBAL_STORE_DWORD_ADDTID : GLOBAL_Real_AllAddr_gfx11<0x029, "global_store_addtid_b32">;3353defm GLOBAL_ATOMIC_SWAP : GLOBAL_Real_Atomics_gfx11<0x033, "global_atomic_swap_b32">;3354defm GLOBAL_ATOMIC_CMPSWAP : GLOBAL_Real_Atomics_gfx11<0x034, "global_atomic_cmpswap_b32">;3355defm GLOBAL_ATOMIC_ADD : GLOBAL_Real_Atomics_gfx11<0x035, "global_atomic_add_u32">;3356defm GLOBAL_ATOMIC_SUB : GLOBAL_Real_Atomics_gfx11<0x036, "global_atomic_sub_u32">;3357defm GLOBAL_ATOMIC_CSUB : GLOBAL_Real_Atomics_gfx11<0x037, "global_atomic_csub_u32">;3358defm GLOBAL_ATOMIC_SMIN : GLOBAL_Real_Atomics_gfx11<0x038, "global_atomic_min_i32">;3359defm GLOBAL_ATOMIC_UMIN : GLOBAL_Real_Atomics_gfx11<0x039, "global_atomic_min_u32">;3360defm GLOBAL_ATOMIC_SMAX : GLOBAL_Real_Atomics_gfx11<0x03a, "global_atomic_max_i32">;3361defm GLOBAL_ATOMIC_UMAX : GLOBAL_Real_Atomics_gfx11<0x03b, "global_atomic_max_u32">;3362defm GLOBAL_ATOMIC_AND : GLOBAL_Real_Atomics_gfx11<0x03c, "global_atomic_and_b32">;3363defm GLOBAL_ATOMIC_OR : GLOBAL_Real_Atomics_gfx11<0x03d, "global_atomic_or_b32">;3364defm GLOBAL_ATOMIC_XOR : GLOBAL_Real_Atomics_gfx11<0x03e, "global_atomic_xor_b32">;3365defm GLOBAL_ATOMIC_INC : GLOBAL_Real_Atomics_gfx11<0x03f, "global_atomic_inc_u32">;3366defm GLOBAL_ATOMIC_DEC : GLOBAL_Real_Atomics_gfx11<0x040, "global_atomic_dec_u32">;3367defm GLOBAL_ATOMIC_SWAP_X2 : GLOBAL_Real_Atomics_gfx11<0x041, "global_atomic_swap_b64">;3368defm GLOBAL_ATOMIC_CMPSWAP_X2 : GLOBAL_Real_Atomics_gfx11<0x042, "global_atomic_cmpswap_b64">;3369defm GLOBAL_ATOMIC_ADD_X2 : GLOBAL_Real_Atomics_gfx11<0x043, "global_atomic_add_u64">;3370defm GLOBAL_ATOMIC_SUB_X2 : GLOBAL_Real_Atomics_gfx11<0x044, "global_atomic_sub_u64">;3371defm GLOBAL_ATOMIC_SMIN_X2 : GLOBAL_Real_Atomics_gfx11<0x045, "global_atomic_min_i64">;3372defm GLOBAL_ATOMIC_UMIN_X2 : GLOBAL_Real_Atomics_gfx11<0x046, "global_atomic_min_u64">;3373defm GLOBAL_ATOMIC_SMAX_X2 : GLOBAL_Real_Atomics_gfx11<0x047, "global_atomic_max_i64">;3374defm GLOBAL_ATOMIC_UMAX_X2 : GLOBAL_Real_Atomics_gfx11<0x048, "global_atomic_max_u64">;3375defm GLOBAL_ATOMIC_AND_X2 : GLOBAL_Real_Atomics_gfx11<0x049, "global_atomic_and_b64">;3376defm GLOBAL_ATOMIC_OR_X2 : GLOBAL_Real_Atomics_gfx11<0x04a, "global_atomic_or_b64">;3377defm GLOBAL_ATOMIC_XOR_X2 : GLOBAL_Real_Atomics_gfx11<0x04b, "global_atomic_xor_b64">;3378defm GLOBAL_ATOMIC_INC_X2 : GLOBAL_Real_Atomics_gfx11<0x04c, "global_atomic_inc_u64">;3379defm GLOBAL_ATOMIC_DEC_X2 : GLOBAL_Real_Atomics_gfx11<0x04d, "global_atomic_dec_u64">;3380defm GLOBAL_ATOMIC_FCMPSWAP : GLOBAL_Real_Atomics_gfx11<0x050, "global_atomic_cmpswap_f32">;3381defm GLOBAL_ATOMIC_FMIN : GLOBAL_Real_Atomics_gfx11<0x051, "global_atomic_min_f32">;3382defm GLOBAL_ATOMIC_FMAX : GLOBAL_Real_Atomics_gfx11<0x052, "global_atomic_max_f32">;3383defm GLOBAL_ATOMIC_ADD_F32 : GLOBAL_Real_Atomics_gfx11<0x056>;3384 3385// ENC_FLAT_SCRATCH.3386defm SCRATCH_LOAD_UBYTE : SCRATCH_Real_AllAddr_gfx11<0x10, "scratch_load_u8">;3387defm SCRATCH_LOAD_SBYTE : SCRATCH_Real_AllAddr_gfx11<0x11, "scratch_load_i8">;3388defm SCRATCH_LOAD_USHORT : SCRATCH_Real_AllAddr_gfx11<0x12, "scratch_load_u16">;3389defm SCRATCH_LOAD_SSHORT : SCRATCH_Real_AllAddr_gfx11<0x13, "scratch_load_i16">;3390defm SCRATCH_LOAD_DWORD : SCRATCH_Real_AllAddr_gfx11<0x14, "scratch_load_b32">;3391defm SCRATCH_LOAD_DWORDX2 : SCRATCH_Real_AllAddr_gfx11<0x15, "scratch_load_b64">;3392defm SCRATCH_LOAD_DWORDX3 : SCRATCH_Real_AllAddr_gfx11<0x16, "scratch_load_b96">;3393defm SCRATCH_LOAD_DWORDX4 : SCRATCH_Real_AllAddr_gfx11<0x17, "scratch_load_b128">;3394defm SCRATCH_STORE_BYTE : SCRATCH_Real_AllAddr_gfx11<0x18, "scratch_store_b8">;3395defm SCRATCH_STORE_SHORT : SCRATCH_Real_AllAddr_gfx11<0x19, "scratch_store_b16">;3396defm SCRATCH_STORE_DWORD : SCRATCH_Real_AllAddr_gfx11<0x1a, "scratch_store_b32">;3397defm SCRATCH_STORE_DWORDX2 : SCRATCH_Real_AllAddr_gfx11<0x1b, "scratch_store_b64">;3398defm SCRATCH_STORE_DWORDX3 : SCRATCH_Real_AllAddr_gfx11<0x1c, "scratch_store_b96">;3399defm SCRATCH_STORE_DWORDX4 : SCRATCH_Real_AllAddr_gfx11<0x1d, "scratch_store_b128">;3400defm SCRATCH_LOAD_UBYTE_D16 : SCRATCH_Real_AllAddr_gfx11<0x1e, "scratch_load_d16_u8">;3401defm SCRATCH_LOAD_SBYTE_D16 : SCRATCH_Real_AllAddr_gfx11<0x1f, "scratch_load_d16_i8">;3402defm SCRATCH_LOAD_SHORT_D16 : SCRATCH_Real_AllAddr_gfx11<0x20, "scratch_load_d16_b16">;3403defm SCRATCH_LOAD_UBYTE_D16_HI : SCRATCH_Real_AllAddr_gfx11<0x21, "scratch_load_d16_hi_u8">;3404defm SCRATCH_LOAD_SBYTE_D16_HI : SCRATCH_Real_AllAddr_gfx11<0x22, "scratch_load_d16_hi_i8">;3405defm SCRATCH_LOAD_SHORT_D16_HI : SCRATCH_Real_AllAddr_gfx11<0x23, "scratch_load_d16_hi_b16">;3406defm SCRATCH_STORE_BYTE_D16_HI : SCRATCH_Real_AllAddr_gfx11<0x24, "scratch_store_d16_hi_b8">;3407defm SCRATCH_STORE_SHORT_D16_HI : SCRATCH_Real_AllAddr_gfx11<0x25, "scratch_store_d16_hi_b16">;3408 3409//===----------------------------------------------------------------------===//3410// GFX123411//===----------------------------------------------------------------------===//3412 3413multiclass VFLAT_Real_gfx12 <bits<8> op, string name = get_FLAT_ps<NAME>.Mnemonic> {3414 defvar ps = !cast<FLAT_Pseudo>(NAME);3415 def _gfx12 : VFLAT_Real <op, ps, name>,3416 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.GFX12> {3417 let AssemblerPredicate = isGFX12Only;3418 let DecoderNamespace = "GFX12";3419 3420 let Inst{25-24} = {ps.is_flat_global, ps.is_flat_scratch};3421 let Inst{48} = cpol{CPolBit.SCAL}; // scale offset3422 }3423}3424 3425multiclass VFLAT_Aliases_gfx12<string name, string alias = name> {3426 defvar ps = get_FLAT_ps<NAME>;3427 let AssemblerPredicate = isGFX12Only in {3428 if !ne(ps.Mnemonic, name) then3429 def : AMDGPUMnemonicAlias<ps.Mnemonic, name>;3430 if !ne(alias, name) then3431 def : AMDGPUMnemonicAlias<alias, name>;3432 }3433}3434 3435multiclass VFLAT_Real_Base_gfx12<bits<8> op,3436 string name = get_FLAT_ps<NAME>.Mnemonic,3437 string alias = name> :3438 VFLAT_Aliases_gfx12<name, alias>,3439 VFLAT_Real_gfx12<op, name>;3440 3441multiclass VFLAT_Real_AllAddr_gfx12<bits<8> op,3442 string name = get_FLAT_ps<NAME>.Mnemonic,3443 string alias = name> :3444 VFLAT_Real_Base_gfx12<op, name, alias> {3445 defm _SADDR : VFLAT_Real_gfx12<op, name>;3446}3447 3448multiclass VGLOBAL_Real_AllAddr_gfx1200<bits<8> op> {3449 let AssemblerPredicate = isGFX12Not12_50 in {3450 defm "" : VFLAT_Real_gfx12<op>;3451 defm _SADDR : VFLAT_Real_gfx12<op>;3452 }3453}3454 3455multiclass VFLAT_Real_AllAddr_gfx12_w64<bits<8> op,3456 string name = get_FLAT_ps<NAME>.Mnemonic> :3457 VFLAT_Aliases_gfx12<name> {3458 let DecoderNamespace = "GFX12W64" in {3459 defm "" : VFLAT_Real_gfx12<op, name>;3460 defm _SADDR : VFLAT_Real_gfx12<op, name>;3461 }3462}3463 3464multiclass VFLAT_Real_Atomics_gfx12<bits<8> op,3465 string name = get_FLAT_ps<NAME>.Mnemonic,3466 string alias = name> :3467 VFLAT_Real_AllAddr_gfx12<op, name, alias> {3468 defm _RTN : VFLAT_Real_gfx12<op, name>;3469 defm _SADDR_RTN : VFLAT_Real_gfx12<op, name>;3470}3471 3472multiclass VSCRATCH_Real_AllAddr_gfx12<bits<8> op,3473 string name = get_FLAT_ps<NAME>.Mnemonic> :3474 VFLAT_Real_Base_gfx12<op, name> {3475 defm _SADDR : VFLAT_Real_gfx12<op, name>;3476 defm _ST : VFLAT_Real_gfx12<op, name>;3477 defm _SVS : VFLAT_Real_gfx12<op, name>;3478}3479 3480// ENC_VFLAT.3481defm FLAT_LOAD_UBYTE : VFLAT_Real_AllAddr_gfx12<0x010, "flat_load_u8">;3482defm FLAT_LOAD_SBYTE : VFLAT_Real_AllAddr_gfx12<0x011, "flat_load_i8">;3483defm FLAT_LOAD_USHORT : VFLAT_Real_AllAddr_gfx12<0x012, "flat_load_u16">;3484defm FLAT_LOAD_SSHORT : VFLAT_Real_AllAddr_gfx12<0x013, "flat_load_i16">;3485defm FLAT_LOAD_DWORD : VFLAT_Real_AllAddr_gfx12<0x014, "flat_load_b32">;3486defm FLAT_LOAD_DWORDX2 : VFLAT_Real_AllAddr_gfx12<0x015, "flat_load_b64">;3487defm FLAT_LOAD_DWORDX3 : VFLAT_Real_AllAddr_gfx12<0x016, "flat_load_b96">;3488defm FLAT_LOAD_DWORDX4 : VFLAT_Real_AllAddr_gfx12<0x017, "flat_load_b128">;3489defm FLAT_STORE_BYTE : VFLAT_Real_AllAddr_gfx12<0x018, "flat_store_b8">;3490defm FLAT_STORE_SHORT : VFLAT_Real_AllAddr_gfx12<0x019, "flat_store_b16">;3491defm FLAT_STORE_DWORD : VFLAT_Real_AllAddr_gfx12<0x01a, "flat_store_b32">;3492defm FLAT_STORE_DWORDX2 : VFLAT_Real_AllAddr_gfx12<0x01b, "flat_store_b64">;3493defm FLAT_STORE_DWORDX3 : VFLAT_Real_AllAddr_gfx12<0x01c, "flat_store_b96">;3494defm FLAT_STORE_DWORDX4 : VFLAT_Real_AllAddr_gfx12<0x01d, "flat_store_b128">;3495defm FLAT_LOAD_UBYTE_D16 : VFLAT_Real_AllAddr_gfx12<0x01e, "flat_load_d16_u8">;3496defm FLAT_LOAD_SBYTE_D16 : VFLAT_Real_AllAddr_gfx12<0x01f, "flat_load_d16_i8">;3497defm FLAT_LOAD_SHORT_D16 : VFLAT_Real_AllAddr_gfx12<0x020, "flat_load_d16_b16">;3498defm FLAT_LOAD_UBYTE_D16_HI : VFLAT_Real_AllAddr_gfx12<0x021, "flat_load_d16_hi_u8">;3499defm FLAT_LOAD_SBYTE_D16_HI : VFLAT_Real_AllAddr_gfx12<0x022, "flat_load_d16_hi_i8">;3500defm FLAT_LOAD_SHORT_D16_HI : VFLAT_Real_AllAddr_gfx12<0x023, "flat_load_d16_hi_b16">;3501defm FLAT_STORE_BYTE_D16_HI : VFLAT_Real_AllAddr_gfx12<0x024, "flat_store_d16_hi_b8">;3502defm FLAT_STORE_SHORT_D16_HI : VFLAT_Real_AllAddr_gfx12<0x025, "flat_store_d16_hi_b16">;3503defm FLAT_ATOMIC_SWAP : VFLAT_Real_Atomics_gfx12<0x033, "flat_atomic_swap_b32">;3504defm FLAT_ATOMIC_CMPSWAP : VFLAT_Real_Atomics_gfx12<0x034, "flat_atomic_cmpswap_b32">;3505defm FLAT_ATOMIC_ADD : VFLAT_Real_Atomics_gfx12<0x035, "flat_atomic_add_u32">;3506defm FLAT_ATOMIC_SUB : VFLAT_Real_Atomics_gfx12<0x036, "flat_atomic_sub_u32">;3507defm FLAT_ATOMIC_CSUB_U32 : VFLAT_Real_Atomics_gfx12<0x037, "flat_atomic_sub_clamp_u32">;3508defm FLAT_ATOMIC_SMIN : VFLAT_Real_Atomics_gfx12<0x038, "flat_atomic_min_i32">;3509defm FLAT_ATOMIC_UMIN : VFLAT_Real_Atomics_gfx12<0x039, "flat_atomic_min_u32">;3510defm FLAT_ATOMIC_SMAX : VFLAT_Real_Atomics_gfx12<0x03a, "flat_atomic_max_i32">;3511defm FLAT_ATOMIC_UMAX : VFLAT_Real_Atomics_gfx12<0x03b, "flat_atomic_max_u32">;3512defm FLAT_ATOMIC_AND : VFLAT_Real_Atomics_gfx12<0x03c, "flat_atomic_and_b32">;3513defm FLAT_ATOMIC_OR : VFLAT_Real_Atomics_gfx12<0x03d, "flat_atomic_or_b32">;3514defm FLAT_ATOMIC_XOR : VFLAT_Real_Atomics_gfx12<0x03e, "flat_atomic_xor_b32">;3515defm FLAT_ATOMIC_INC : VFLAT_Real_Atomics_gfx12<0x03f, "flat_atomic_inc_u32">;3516defm FLAT_ATOMIC_DEC : VFLAT_Real_Atomics_gfx12<0x040, "flat_atomic_dec_u32">;3517defm FLAT_ATOMIC_SWAP_X2 : VFLAT_Real_Atomics_gfx12<0x041, "flat_atomic_swap_b64">;3518defm FLAT_ATOMIC_CMPSWAP_X2 : VFLAT_Real_Atomics_gfx12<0x042, "flat_atomic_cmpswap_b64">;3519defm FLAT_ATOMIC_ADD_X2 : VFLAT_Real_Atomics_gfx12<0x043, "flat_atomic_add_u64">;3520defm FLAT_ATOMIC_SUB_X2 : VFLAT_Real_Atomics_gfx12<0x044, "flat_atomic_sub_u64">;3521defm FLAT_ATOMIC_SMIN_X2 : VFLAT_Real_Atomics_gfx12<0x045, "flat_atomic_min_i64">;3522defm FLAT_ATOMIC_UMIN_X2 : VFLAT_Real_Atomics_gfx12<0x046, "flat_atomic_min_u64">;3523defm FLAT_ATOMIC_SMAX_X2 : VFLAT_Real_Atomics_gfx12<0x047, "flat_atomic_max_i64">;3524defm FLAT_ATOMIC_UMAX_X2 : VFLAT_Real_Atomics_gfx12<0x048, "flat_atomic_max_u64">;3525defm FLAT_ATOMIC_AND_X2 : VFLAT_Real_Atomics_gfx12<0x049, "flat_atomic_and_b64">;3526defm FLAT_ATOMIC_OR_X2 : VFLAT_Real_Atomics_gfx12<0x04a, "flat_atomic_or_b64">;3527defm FLAT_ATOMIC_XOR_X2 : VFLAT_Real_Atomics_gfx12<0x04b, "flat_atomic_xor_b64">;3528defm FLAT_ATOMIC_INC_X2 : VFLAT_Real_Atomics_gfx12<0x04c, "flat_atomic_inc_u64">;3529defm FLAT_ATOMIC_DEC_X2 : VFLAT_Real_Atomics_gfx12<0x04d, "flat_atomic_dec_u64">;3530defm FLAT_ATOMIC_COND_SUB_U32 : VFLAT_Real_Atomics_gfx12<0x050>;3531defm FLAT_ATOMIC_FMIN : VFLAT_Real_Atomics_gfx12<0x051, "flat_atomic_min_num_f32", "flat_atomic_min_f32">;3532defm FLAT_ATOMIC_FMAX : VFLAT_Real_Atomics_gfx12<0x052, "flat_atomic_max_num_f32", "flat_atomic_max_f32">;3533defm FLAT_ATOMIC_ADD_F32 : VFLAT_Real_Atomics_gfx12<0x056>;3534defm FLAT_ATOMIC_PK_ADD_F16 : VFLAT_Real_Atomics_gfx12<0x059>;3535defm FLAT_ATOMIC_PK_ADD_BF16 : VFLAT_Real_Atomics_gfx12<0x05a>;3536 3537// ENC_VGLOBAL.3538defm GLOBAL_LOAD_UBYTE : VFLAT_Real_AllAddr_gfx12<0x010, "global_load_u8">;3539defm GLOBAL_LOAD_SBYTE : VFLAT_Real_AllAddr_gfx12<0x011, "global_load_i8">;3540defm GLOBAL_LOAD_USHORT : VFLAT_Real_AllAddr_gfx12<0x012, "global_load_u16">;3541defm GLOBAL_LOAD_SSHORT : VFLAT_Real_AllAddr_gfx12<0x013, "global_load_i16">;3542defm GLOBAL_LOAD_DWORD : VFLAT_Real_AllAddr_gfx12<0x014, "global_load_b32">;3543defm GLOBAL_LOAD_DWORDX2 : VFLAT_Real_AllAddr_gfx12<0x015, "global_load_b64">;3544defm GLOBAL_LOAD_DWORDX3 : VFLAT_Real_AllAddr_gfx12<0x016, "global_load_b96">;3545defm GLOBAL_LOAD_DWORDX4 : VFLAT_Real_AllAddr_gfx12<0x017, "global_load_b128">;3546defm GLOBAL_STORE_BYTE : VFLAT_Real_AllAddr_gfx12<0x018, "global_store_b8">;3547defm GLOBAL_STORE_SHORT : VFLAT_Real_AllAddr_gfx12<0x019, "global_store_b16">;3548defm GLOBAL_STORE_DWORD : VFLAT_Real_AllAddr_gfx12<0x01a, "global_store_b32">;3549defm GLOBAL_STORE_DWORDX2 : VFLAT_Real_AllAddr_gfx12<0x01b, "global_store_b64">;3550defm GLOBAL_STORE_DWORDX3 : VFLAT_Real_AllAddr_gfx12<0x01c, "global_store_b96">;3551defm GLOBAL_STORE_DWORDX4 : VFLAT_Real_AllAddr_gfx12<0x01d, "global_store_b128">;3552defm GLOBAL_LOAD_UBYTE_D16 : VFLAT_Real_AllAddr_gfx12<0x01e, "global_load_d16_u8">;3553defm GLOBAL_LOAD_SBYTE_D16 : VFLAT_Real_AllAddr_gfx12<0x01f, "global_load_d16_i8">;3554defm GLOBAL_LOAD_SHORT_D16 : VFLAT_Real_AllAddr_gfx12<0x020, "global_load_d16_b16">;3555defm GLOBAL_LOAD_UBYTE_D16_HI : VFLAT_Real_AllAddr_gfx12<0x021, "global_load_d16_hi_u8">;3556defm GLOBAL_LOAD_SBYTE_D16_HI : VFLAT_Real_AllAddr_gfx12<0x022, "global_load_d16_hi_i8">;3557defm GLOBAL_LOAD_SHORT_D16_HI : VFLAT_Real_AllAddr_gfx12<0x023, "global_load_d16_hi_b16">;3558defm GLOBAL_STORE_BYTE_D16_HI : VFLAT_Real_AllAddr_gfx12<0x024, "global_store_d16_hi_b8">;3559defm GLOBAL_STORE_SHORT_D16_HI : VFLAT_Real_AllAddr_gfx12<0x025, "global_store_d16_hi_b16">;3560defm GLOBAL_LOAD_DWORD_ADDTID : VFLAT_Real_AllAddr_gfx12<0x028, "global_load_addtid_b32">;3561defm GLOBAL_STORE_DWORD_ADDTID : VFLAT_Real_AllAddr_gfx12<0x029, "global_store_addtid_b32">;3562defm GLOBAL_LOAD_BLOCK : VFLAT_Real_AllAddr_gfx12<0x053>;3563defm GLOBAL_STORE_BLOCK : VFLAT_Real_AllAddr_gfx12<0x054>;3564 3565defm GLOBAL_ATOMIC_SWAP : VFLAT_Real_Atomics_gfx12<0x033, "global_atomic_swap_b32">;3566defm GLOBAL_ATOMIC_CMPSWAP : VFLAT_Real_Atomics_gfx12<0x034, "global_atomic_cmpswap_b32">;3567defm GLOBAL_ATOMIC_ADD : VFLAT_Real_Atomics_gfx12<0x035, "global_atomic_add_u32">;3568defm GLOBAL_ATOMIC_SUB : VFLAT_Real_Atomics_gfx12<0x036, "global_atomic_sub_u32">;3569defm GLOBAL_ATOMIC_CSUB : VFLAT_Real_Atomics_gfx12<0x037, "global_atomic_sub_clamp_u32", "global_atomic_csub_u32">;3570defm GLOBAL_ATOMIC_SMIN : VFLAT_Real_Atomics_gfx12<0x038, "global_atomic_min_i32">;3571defm GLOBAL_ATOMIC_UMIN : VFLAT_Real_Atomics_gfx12<0x039, "global_atomic_min_u32">;3572defm GLOBAL_ATOMIC_SMAX : VFLAT_Real_Atomics_gfx12<0x03a, "global_atomic_max_i32">;3573defm GLOBAL_ATOMIC_UMAX : VFLAT_Real_Atomics_gfx12<0x03b, "global_atomic_max_u32">;3574defm GLOBAL_ATOMIC_AND : VFLAT_Real_Atomics_gfx12<0x03c, "global_atomic_and_b32">;3575defm GLOBAL_ATOMIC_OR : VFLAT_Real_Atomics_gfx12<0x03d, "global_atomic_or_b32">;3576defm GLOBAL_ATOMIC_XOR : VFLAT_Real_Atomics_gfx12<0x03e, "global_atomic_xor_b32">;3577defm GLOBAL_ATOMIC_INC : VFLAT_Real_Atomics_gfx12<0x03f, "global_atomic_inc_u32">;3578defm GLOBAL_ATOMIC_DEC : VFLAT_Real_Atomics_gfx12<0x040, "global_atomic_dec_u32">;3579defm GLOBAL_ATOMIC_SWAP_X2 : VFLAT_Real_Atomics_gfx12<0x041, "global_atomic_swap_b64">;3580defm GLOBAL_ATOMIC_CMPSWAP_X2 : VFLAT_Real_Atomics_gfx12<0x042, "global_atomic_cmpswap_b64">;3581defm GLOBAL_ATOMIC_ADD_X2 : VFLAT_Real_Atomics_gfx12<0x043, "global_atomic_add_u64">;3582defm GLOBAL_ATOMIC_SUB_X2 : VFLAT_Real_Atomics_gfx12<0x044, "global_atomic_sub_u64">;3583defm GLOBAL_ATOMIC_SMIN_X2 : VFLAT_Real_Atomics_gfx12<0x045, "global_atomic_min_i64">;3584defm GLOBAL_ATOMIC_UMIN_X2 : VFLAT_Real_Atomics_gfx12<0x046, "global_atomic_min_u64">;3585defm GLOBAL_ATOMIC_SMAX_X2 : VFLAT_Real_Atomics_gfx12<0x047, "global_atomic_max_i64">;3586defm GLOBAL_ATOMIC_UMAX_X2 : VFLAT_Real_Atomics_gfx12<0x048, "global_atomic_max_u64">;3587defm GLOBAL_ATOMIC_AND_X2 : VFLAT_Real_Atomics_gfx12<0x049, "global_atomic_and_b64">;3588defm GLOBAL_ATOMIC_OR_X2 : VFLAT_Real_Atomics_gfx12<0x04a, "global_atomic_or_b64">;3589defm GLOBAL_ATOMIC_XOR_X2 : VFLAT_Real_Atomics_gfx12<0x04b, "global_atomic_xor_b64">;3590defm GLOBAL_ATOMIC_INC_X2 : VFLAT_Real_Atomics_gfx12<0x04c, "global_atomic_inc_u64">;3591defm GLOBAL_ATOMIC_DEC_X2 : VFLAT_Real_Atomics_gfx12<0x04d, "global_atomic_dec_u64">;3592defm GLOBAL_ATOMIC_COND_SUB_U32 : VFLAT_Real_Atomics_gfx12<0x050>;3593defm GLOBAL_ATOMIC_FMIN : VFLAT_Real_Atomics_gfx12<0x051, "global_atomic_min_num_f32", "global_atomic_min_f32">;3594defm GLOBAL_ATOMIC_FMAX : VFLAT_Real_Atomics_gfx12<0x052, "global_atomic_max_num_f32", "global_atomic_max_f32">;3595defm GLOBAL_ATOMIC_ADD_F32 : VFLAT_Real_Atomics_gfx12<0x056>;3596 3597defm GLOBAL_LOAD_TR_B128_w32 : VGLOBAL_Real_AllAddr_gfx1200<0x057>;3598defm GLOBAL_LOAD_TR_B64_w32 : VGLOBAL_Real_AllAddr_gfx1200<0x058>;3599 3600defm GLOBAL_LOAD_TR_B128_w64 : VFLAT_Real_AllAddr_gfx12_w64<0x057>;3601defm GLOBAL_LOAD_TR_B64_w64 : VFLAT_Real_AllAddr_gfx12_w64<0x058>;3602 3603defm GLOBAL_ATOMIC_ORDERED_ADD_B64 : VFLAT_Real_Atomics_gfx12<0x073>;3604defm GLOBAL_ATOMIC_PK_ADD_F16 : VFLAT_Real_Atomics_gfx12<0x059>;3605defm GLOBAL_ATOMIC_PK_ADD_BF16 : VFLAT_Real_Atomics_gfx12<0x05a>;3606 3607defm GLOBAL_INV : VFLAT_Real_Base_gfx12<0x02b>;3608defm GLOBAL_WB : VFLAT_Real_Base_gfx12<0x02c>;3609defm GLOBAL_WBINV : VFLAT_Real_Base_gfx12<0x04f>;3610 3611// ENC_VSCRATCH.3612defm SCRATCH_LOAD_UBYTE : VSCRATCH_Real_AllAddr_gfx12<0x10, "scratch_load_u8">;3613defm SCRATCH_LOAD_SBYTE : VSCRATCH_Real_AllAddr_gfx12<0x11, "scratch_load_i8">;3614defm SCRATCH_LOAD_USHORT : VSCRATCH_Real_AllAddr_gfx12<0x12, "scratch_load_u16">;3615defm SCRATCH_LOAD_SSHORT : VSCRATCH_Real_AllAddr_gfx12<0x13, "scratch_load_i16">;3616defm SCRATCH_LOAD_DWORD : VSCRATCH_Real_AllAddr_gfx12<0x14, "scratch_load_b32">;3617defm SCRATCH_LOAD_DWORDX2 : VSCRATCH_Real_AllAddr_gfx12<0x15, "scratch_load_b64">;3618defm SCRATCH_LOAD_DWORDX3 : VSCRATCH_Real_AllAddr_gfx12<0x16, "scratch_load_b96">;3619defm SCRATCH_LOAD_DWORDX4 : VSCRATCH_Real_AllAddr_gfx12<0x17, "scratch_load_b128">;3620defm SCRATCH_STORE_BYTE : VSCRATCH_Real_AllAddr_gfx12<0x18, "scratch_store_b8">;3621defm SCRATCH_STORE_SHORT : VSCRATCH_Real_AllAddr_gfx12<0x19, "scratch_store_b16">;3622defm SCRATCH_STORE_DWORD : VSCRATCH_Real_AllAddr_gfx12<0x1a, "scratch_store_b32">;3623defm SCRATCH_STORE_DWORDX2 : VSCRATCH_Real_AllAddr_gfx12<0x1b, "scratch_store_b64">;3624defm SCRATCH_STORE_DWORDX3 : VSCRATCH_Real_AllAddr_gfx12<0x1c, "scratch_store_b96">;3625defm SCRATCH_STORE_DWORDX4 : VSCRATCH_Real_AllAddr_gfx12<0x1d, "scratch_store_b128">;3626defm SCRATCH_LOAD_UBYTE_D16 : VSCRATCH_Real_AllAddr_gfx12<0x1e, "scratch_load_d16_u8">;3627defm SCRATCH_LOAD_SBYTE_D16 : VSCRATCH_Real_AllAddr_gfx12<0x1f, "scratch_load_d16_i8">;3628defm SCRATCH_LOAD_SHORT_D16 : VSCRATCH_Real_AllAddr_gfx12<0x20, "scratch_load_d16_b16">;3629defm SCRATCH_LOAD_UBYTE_D16_HI : VSCRATCH_Real_AllAddr_gfx12<0x21, "scratch_load_d16_hi_u8">;3630defm SCRATCH_LOAD_SBYTE_D16_HI : VSCRATCH_Real_AllAddr_gfx12<0x22, "scratch_load_d16_hi_i8">;3631defm SCRATCH_LOAD_SHORT_D16_HI : VSCRATCH_Real_AllAddr_gfx12<0x23, "scratch_load_d16_hi_b16">;3632defm SCRATCH_STORE_BYTE_D16_HI : VSCRATCH_Real_AllAddr_gfx12<0x24, "scratch_store_d16_hi_b8">;3633defm SCRATCH_STORE_SHORT_D16_HI : VSCRATCH_Real_AllAddr_gfx12<0x25, "scratch_store_d16_hi_b16">;3634 3635defm SCRATCH_LOAD_BLOCK : VSCRATCH_Real_AllAddr_gfx12<0x53>;3636defm SCRATCH_STORE_BLOCK : VSCRATCH_Real_AllAddr_gfx12<0x54>;3637 3638//===----------------------------------------------------------------------===//3639// GFX12503640//===----------------------------------------------------------------------===//3641 3642multiclass VFLAT_Real_gfx1250<bits<8> op,3643 string name = get_FLAT_ps<NAME>.Mnemonic> {3644 defvar ps = !cast<FLAT_Pseudo>(NAME);3645 def _gfx1250 : VFLAT_Real<op, ps, name>,3646 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX1250> {3647 let AssemblerPredicate = isGFX125xOnly;3648 let DecoderNamespace = "GFX1250";3649 3650 let Inst{25-24} = {ps.is_flat_global, ps.is_flat_scratch};3651 let Inst{48} = cpol{CPolBit.SCAL}; // scale offset3652 }3653}3654 3655multiclass VFLAT_Aliases_gfx1250<string name> {3656 defvar ps = get_FLAT_ps<NAME>;3657 if !ne(ps.Mnemonic, name) then3658 def : MnemonicAlias<ps.Mnemonic, name>, Requires<[isGFX125xOnly]>;3659}3660 3661multiclass VFLAT_Real_Base_gfx1250<bits<8> op, string name = get_FLAT_ps<NAME>.Mnemonic> :3662 VFLAT_Aliases_gfx1250<name> {3663 defm "" : VFLAT_Real_gfx1250<op, name>;3664}3665 3666multiclass VFLAT_Real_RTN_gfx1250<bits<8> op, string name> {3667 defm _RTN : VFLAT_Real_gfx1250<op, name>;3668}3669 3670multiclass VFLAT_Real_SADDR_gfx1250<bits<8> op, string name> {3671 defm _SADDR : VFLAT_Real_gfx1250<op, name>;3672}3673 3674multiclass VFLAT_Real_SADDR_RTN_gfx1250<bits<8> op, string name> {3675 defm _SADDR_RTN : VFLAT_Real_gfx1250<op, name>;3676}3677 3678multiclass VFLAT_Real_AllAddr_gfx1250<bits<8> op, string name = get_FLAT_ps<NAME>.Mnemonic> :3679 VFLAT_Real_Base_gfx1250<op, name>,3680 VFLAT_Real_SADDR_gfx1250<op, name>;3681 3682multiclass VFLAT_Real_Atomics_gfx1250<bits<8> op, string name = get_FLAT_ps<NAME>.Mnemonic> :3683 VFLAT_Real_AllAddr_gfx1250<op, name>,3684 VFLAT_Real_RTN_gfx1250<op, name>,3685 VFLAT_Real_SADDR_RTN_gfx1250<op, name>;3686 3687defm TENSOR_SAVE : VFLAT_Real_gfx1250<0x06e>;3688defm TENSOR_STOP : VFLAT_Real_gfx1250<0x06f>;3689 3690defm FLAT_PREFETCH_B8 : VFLAT_Real_AllAddr_gfx1250<0x05d>;3691defm GLOBAL_PREFETCH_B8 : VFLAT_Real_AllAddr_gfx1250<0x05d>;3692 3693defm FLAT_LOAD_MONITOR_B32 : VFLAT_Real_AllAddr_gfx1250<0x070>;3694defm FLAT_LOAD_MONITOR_B64 : VFLAT_Real_AllAddr_gfx1250<0x071>;3695defm FLAT_LOAD_MONITOR_B128 : VFLAT_Real_AllAddr_gfx1250<0x072>;3696 3697defm GLOBAL_LOAD_MONITOR_B32 : VFLAT_Real_AllAddr_gfx1250<0x070>;3698defm GLOBAL_LOAD_MONITOR_B64 : VFLAT_Real_AllAddr_gfx1250<0x071>;3699defm GLOBAL_LOAD_MONITOR_B128 : VFLAT_Real_AllAddr_gfx1250<0x072>;3700 3701defm CLUSTER_LOAD_B32 : VFLAT_Real_AllAddr_gfx1250<0x067>;3702defm CLUSTER_LOAD_B64 : VFLAT_Real_AllAddr_gfx1250<0x068>;3703defm CLUSTER_LOAD_B128 : VFLAT_Real_AllAddr_gfx1250<0x069>;3704 3705defm CLUSTER_LOAD_ASYNC_TO_LDS_B8 : VFLAT_Real_AllAddr_gfx1250<0x6a>;3706defm CLUSTER_LOAD_ASYNC_TO_LDS_B32 : VFLAT_Real_AllAddr_gfx1250<0x6b>;3707defm CLUSTER_LOAD_ASYNC_TO_LDS_B64 : VFLAT_Real_AllAddr_gfx1250<0x6c>;3708defm CLUSTER_LOAD_ASYNC_TO_LDS_B128 : VFLAT_Real_AllAddr_gfx1250<0x6d>;3709defm GLOBAL_LOAD_ASYNC_TO_LDS_B8 : VFLAT_Real_AllAddr_gfx1250<0x5f>;3710defm GLOBAL_LOAD_ASYNC_TO_LDS_B32 : VFLAT_Real_AllAddr_gfx1250<0x60>;3711defm GLOBAL_LOAD_ASYNC_TO_LDS_B64 : VFLAT_Real_AllAddr_gfx1250<0x61>;3712defm GLOBAL_LOAD_ASYNC_TO_LDS_B128 : VFLAT_Real_AllAddr_gfx1250<0x62>;3713defm GLOBAL_STORE_ASYNC_FROM_LDS_B8 : VFLAT_Real_AllAddr_gfx1250<0x63>;3714defm GLOBAL_STORE_ASYNC_FROM_LDS_B32 : VFLAT_Real_AllAddr_gfx1250<0x64>;3715defm GLOBAL_STORE_ASYNC_FROM_LDS_B64 : VFLAT_Real_AllAddr_gfx1250<0x65>;3716defm GLOBAL_STORE_ASYNC_FROM_LDS_B128 : VFLAT_Real_AllAddr_gfx1250<0x66>;3717 3718defm GLOBAL_LOAD_TR_B128_w32 : VFLAT_Real_AllAddr_gfx1250<0x057, "global_load_tr16_b128">;3719defm GLOBAL_LOAD_TR_B64_w32 : VFLAT_Real_AllAddr_gfx1250<0x058, "global_load_tr8_b64">;3720 3721defm GLOBAL_LOAD_TR4_B64 : VFLAT_Real_AllAddr_gfx1250<0x073>;3722defm GLOBAL_LOAD_TR6_B96 : VFLAT_Real_AllAddr_gfx1250<0x074>;3723 3724// Additional aliases for global load transpose instructions.3725def : MnemonicAlias<"global_load_b128_tr_b16", "global_load_tr16_b128">, Requires<[isGFX125xOnly]>;3726def : MnemonicAlias<"global_load_b64_tr_b8", "global_load_tr8_b64">, Requires<[isGFX125xOnly]>;3727def : MnemonicAlias<"global_load_b64_tr_b4", "global_load_tr4_b64">, Requires<[isGFX125xOnly]>;3728def : MnemonicAlias<"global_load_b96_tr_b6", "global_load_tr6_b96">, Requires<[isGFX125xOnly]>;3729 3730defm FLAT_ATOMIC_ADD_F64 : VFLAT_Real_Atomics_gfx1250<0x055>;3731defm FLAT_ATOMIC_MIN_F64 : VFLAT_Real_Atomics_gfx1250<0x05b, "flat_atomic_min_num_f64">;3732defm FLAT_ATOMIC_MAX_F64 : VFLAT_Real_Atomics_gfx1250<0x05c, "flat_atomic_max_num_f64">;3733 3734defm GLOBAL_ATOMIC_ADD_F64 : VFLAT_Real_Atomics_gfx1250<0x055>;3735defm GLOBAL_ATOMIC_MIN_F64 : VFLAT_Real_Atomics_gfx1250<0x05b, "global_atomic_min_num_f64">;3736defm GLOBAL_ATOMIC_MAX_F64 : VFLAT_Real_Atomics_gfx1250<0x05c, "global_atomic_max_num_f64">;3737 3738def True16D16Table : GenericTable {3739 let FilterClass = "True16D16Table";3740 let CppTypeName = "True16D16Info";3741 let Fields = ["T16Op", "HiOp", "LoOp"];3742 let PrimaryKey = ["T16Op"];3743 let PrimaryKeyName = "getT16D16Helper";3744}3745