1540 lines · plain
1//===---- SMInstructions.td - Scalar Memory Instruction Definitions -------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9def smrd_offset_8 : ImmOperand<i32, "SMRDOffset8", 1>;10 11let EncoderMethod = "getSMEMOffsetEncoding",12 DecoderMethod = "decodeSMEMOffset" in {13def SMEMOffset : ImmOperand<i32, "SMEMOffset", 1>;14def SMEMOffsetMod : NamedIntOperand<"offset", 0> {15 let AlwaysPrint = 1;16 let PrintInHex = 1;17}18def OptSMEMOffsetMod : NamedIntOperand<"offset"> {19 let ImmTy = SMEMOffsetMod.ImmTy;20 let PredicateMethod = SMEMOffsetMod.PredicateMethod;21 let PrintMethod = SMEMOffsetMod.PrintMethod;22}23}24 25//===----------------------------------------------------------------------===//26// Scalar Memory classes27//===----------------------------------------------------------------------===//28 29class SM_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> :30 InstSI <outs, ins, "", pattern>,31 SIMCInstr<NAME, SIEncodingFamily.NONE> {32 let isPseudo = 1;33 let isCodeGenOnly = 1;34 35 let LGKM_CNT = 1;36 let SMRD = 1;37 let mayStore = 0;38 let mayLoad = 1;39 let hasSideEffects = 0;40 let maybeAtomic = 0;41 let UseNamedOperandTable = 1;42 let SchedRW = [WriteSMEM];43 44 string Mnemonic = opName;45 string AsmOperands = asmOps;46 47 bits<1> has_sbase = 1;48 bits<1> has_sdst = 1;49 bit has_glc = 0;50 bit has_dlc = 0;51 bit has_offset = 0;52 bit has_soffset = 0;53 bit is_buffer = 0;54}55 56class SM_Real <SM_Pseudo ps, string opName = ps.Mnemonic>57 : InstSI<ps.OutOperandList, ps.InOperandList, opName # ps.AsmOperands> {58 59 let isPseudo = 0;60 let isCodeGenOnly = 0;61 62 Instruction Opcode = !cast<Instruction>(NAME);63 64 // copy relevant pseudo op flags65 let LGKM_CNT = ps.LGKM_CNT;66 let SMRD = ps.SMRD;67 let mayStore = ps.mayStore;68 let mayLoad = ps.mayLoad;69 let hasSideEffects = ps.hasSideEffects;70 let UseNamedOperandTable = ps.UseNamedOperandTable;71 let SchedRW = ps.SchedRW;72 let SubtargetPredicate = ps.SubtargetPredicate;73 let OtherPredicates = ps.OtherPredicates;74 let AsmMatchConverter = ps.AsmMatchConverter;75 let IsAtomicRet = ps.IsAtomicRet;76 let IsAtomicNoRet = ps.IsAtomicNoRet;77 let Uses = ps.Uses;78 let Defs = ps.Defs;79 let isConvergent = ps.isConvergent;80 81 let TSFlags = ps.TSFlags;82 83 bit is_buffer = ps.is_buffer;84 85 // encoding86 bits<7> sbase;87 bits<7> sdst;88 bits<32> offset;89 bits<8> soffset;90 bits<12> cpol;91}92 93class OffsetMode<bit hasOffset, bit hasSOffset, string variant,94 dag ins, string asm> {95 bit HasOffset = hasOffset;96 bit HasSOffset = hasSOffset;97 string Variant = variant;98 dag Ins = ins;99 string Asm = asm;100}101 102def IMM_Offset : OffsetMode<1, 0, "_IMM", (ins SMEMOffset:$offset), "$offset">;103def SGPR_Offset : OffsetMode<0, 1, "_SGPR", (ins SReg_32:$soffset), "$soffset">;104def SGPR_IMM_Offset : OffsetMode<1, 1, "_SGPR_IMM",105 (ins SReg_32:$soffset, SMEMOffsetMod:$offset),106 "$soffset$offset">;107def SGPR_IMM_OptOffset : OffsetMode<1, 1, "_SGPR_IMM",108 (ins SReg_32:$soffset, OptSMEMOffsetMod:$offset),109 "$soffset$offset">;110 111class SM_Probe_Pseudo <string opName, RegisterClass baseClass, OffsetMode offsets>112 : SM_Pseudo<opName, (outs),113 !con((ins i8imm:$sdata, baseClass:$sbase), offsets.Ins),114 " $sdata, $sbase, " # offsets.Asm> {115 let mayLoad = 0;116 let mayStore = 0;117 let has_glc = 0;118 let LGKM_CNT = 0;119 let ScalarStore = 0;120 let hasSideEffects = 1;121 let has_offset = offsets.HasOffset;122 let has_soffset = offsets.HasSOffset;123}124 125class SM_Load_Pseudo <string opName, RegisterClass baseClass,126 RegisterClass dstClass, OffsetMode offsets>127 : SM_Pseudo<opName, (outs dstClass:$sdst),128 !con((ins baseClass:$sbase), offsets.Ins, (ins CPol:$cpol)),129 " $sdst, $sbase, " # offsets.Asm # "$cpol", []> {130 RegisterClass BaseClass = baseClass;131 let mayLoad = 1;132 let isReMaterializable = 1;133 let mayStore = 0;134 let has_glc = 1;135 let has_dlc = 1;136 let has_offset = offsets.HasOffset;137 let has_soffset = offsets.HasSOffset;138}139 140class SM_Store_Pseudo <string opName, RegisterClass baseClass,141 RegisterClass srcClass, OffsetMode offsets>142 : SM_Pseudo<opName, (outs), !con((ins srcClass:$sdata, baseClass:$sbase),143 offsets.Ins, (ins CPol:$cpol)),144 " $sdata, $sbase, " # offsets.Asm # "$cpol"> {145 RegisterClass BaseClass = baseClass;146 let mayLoad = 0;147 let mayStore = 1;148 let has_glc = 1;149 let has_dlc = 1;150 let has_offset = offsets.HasOffset;151 let has_soffset = offsets.HasSOffset;152 let ScalarStore = 1;153}154 155class SM_Discard_Pseudo <string opName, OffsetMode offsets>156 : SM_Pseudo<opName, (outs), !con((ins SReg_64:$sbase), offsets.Ins),157 " $sbase, " # offsets.Asm> {158 let mayLoad = 0;159 let mayStore = 0;160 let has_glc = 0;161 let has_sdst = 0;162 let ScalarStore = 0;163 let hasSideEffects = 1;164 let has_offset = offsets.HasOffset;165 let has_soffset = offsets.HasSOffset;166}167 168multiclass SM_Load_Pseudos<string op, RegisterClass baseClass,169 RegisterClass dstClass, OffsetMode offsets> {170 defvar opName = !tolower(op);171 def "" : SM_Load_Pseudo <opName, baseClass, dstClass, offsets>;172 173 // The constrained multi-dword load equivalents with early clobber flag at174 // the dst operands. They are needed only for codegen and there is no need175 // for their real opcodes.176 if !gt(dstClass.RegTypes[0].Size, 32) then177 let Constraints = "@earlyclobber $sdst",178 PseudoInstr = op # offsets.Variant in179 def "" # _ec : SM_Load_Pseudo <opName, baseClass, dstClass, offsets>;180}181 182multiclass SM_Pseudo_Loads<RegisterClass baseClass,183 RegisterClass dstClass> {184 defm _IMM : SM_Load_Pseudos <NAME, baseClass, dstClass, IMM_Offset>;185 defm _SGPR : SM_Load_Pseudos <NAME, baseClass, dstClass, SGPR_Offset>;186 defm _SGPR_IMM : SM_Load_Pseudos <NAME, baseClass, dstClass, SGPR_IMM_Offset>;187}188 189multiclass SM_Pseudo_Stores<RegisterClass baseClass,190 RegisterClass srcClass> {191 defvar opName = !tolower(NAME);192 def _IMM : SM_Store_Pseudo <opName, baseClass, srcClass, IMM_Offset>;193 def _SGPR : SM_Store_Pseudo <opName, baseClass, srcClass, SGPR_Offset>;194 def _SGPR_IMM : SM_Store_Pseudo <opName, baseClass, srcClass, SGPR_IMM_Offset>;195}196 197multiclass SM_Pseudo_Discards {198 defvar opName = !tolower(NAME);199 def _IMM : SM_Discard_Pseudo <opName, IMM_Offset>;200 def _SGPR : SM_Discard_Pseudo <opName, SGPR_Offset>;201 def _SGPR_IMM : SM_Discard_Pseudo <opName, SGPR_IMM_Offset>;202}203 204class SM_Time_Pseudo<string opName, SDPatternOperator node = null_frag> : SM_Pseudo<205 opName, (outs SReg_64_XEXEC:$sdst), (ins),206 " $sdst", [(set i64:$sdst, (node))]> {207 let hasSideEffects = 1;208 209 let mayStore = 0;210 let mayLoad = 0;211 let has_sbase = 0;212}213 214class SM_Inval_Pseudo <string opName, SDPatternOperator node = null_frag> : SM_Pseudo<215 opName, (outs), (ins), "", [(node)]> {216 let hasSideEffects = 1;217 let mayLoad = 0;218 let mayStore = 0;219 let has_sdst = 0;220 let has_sbase = 0;221}222 223multiclass SM_Pseudo_Probe<RegisterClass baseClass> {224 defvar opName = !tolower(NAME);225 def _IMM : SM_Probe_Pseudo <opName, baseClass, IMM_Offset>;226 def _SGPR : SM_Probe_Pseudo <opName, baseClass, SGPR_Offset>;227 def _SGPR_IMM : SM_Probe_Pseudo <opName, baseClass, SGPR_IMM_Offset>;228 def _SGPR_OPT_IMM : SM_Probe_Pseudo <opName, baseClass, SGPR_IMM_OptOffset>;229}230 231class SM_WaveId_Pseudo<string opName, SDPatternOperator node> : SM_Pseudo<232 opName, (outs SReg_32_XM0_XEXEC:$sdst), (ins),233 " $sdst", [(set i32:$sdst, (node))]> {234 let hasSideEffects = 1;235 let mayStore = 0;236 let mayLoad = 0;237 let has_sbase = 0;238}239 240class SM_Prefetch_Pseudo <string opName, RegisterClass baseClass, bit hasSBase>241 : SM_Pseudo<opName, (outs), !con(!if(hasSBase, (ins baseClass:$sbase), (ins)),242 (ins SMEMOffset:$offset, SReg_32:$soffset, i8imm:$sdata)),243 !if(hasSBase, " $sbase,", "") # " $offset, $soffset, $sdata"> {244 // Mark prefetches as both load and store to prevent reordering with loads245 // and stores. This is also needed for pattern to match prefetch intrinsic.246 let mayLoad = 1;247 let mayStore = 1;248 let has_glc = 0;249 let LGKM_CNT = 0;250 let has_sbase = hasSBase;251 let ScalarStore = 0;252 let has_offset = 1;253 let has_soffset = 1;254}255 256//===----------------------------------------------------------------------===//257// Scalar Atomic Memory Classes258//===----------------------------------------------------------------------===//259 260class SM_Atomic_Pseudo <string opName,261 dag outs, dag ins, string asmOps, bit isRet>262 : SM_Pseudo<opName, outs, ins, asmOps, []> {263 264 bit glc = isRet;265 266 let mayLoad = 1;267 let mayStore = 1;268 let has_glc = 1;269 let has_dlc = 1;270 let has_soffset = 1;271 272 // Should these be set?273 let ScalarStore = 1;274 let hasSideEffects = 1;275 let maybeAtomic = 1;276 277 let IsAtomicNoRet = !not(isRet);278 let IsAtomicRet = isRet;279}280 281class SM_Pseudo_Atomic<string opName,282 RegisterClass baseClass,283 RegisterClass dataClass,284 OffsetMode offsets,285 bit isRet,286 Operand CPolTy = !if(isRet, CPol_GLC, CPol_NonGLC)> :287 SM_Atomic_Pseudo<opName,288 !if(isRet, (outs dataClass:$sdst), (outs)),289 !con((ins dataClass:$sdata, baseClass:$sbase), offsets.Ins,290 (ins CPolTy:$cpol)),291 !if(isRet, " $sdst", " $sdata") #292 ", $sbase, " # offsets.Asm # "$cpol",293 isRet> {294 let has_offset = offsets.HasOffset;295 let has_soffset = offsets.HasSOffset;296 297 let Constraints = !if(isRet, "$sdst = $sdata", "");298}299 300multiclass SM_Pseudo_Atomics<RegisterClass baseClass,301 RegisterClass dataClass> {302 defvar opName = !tolower(NAME);303 def _IMM : SM_Pseudo_Atomic <opName, baseClass, dataClass, IMM_Offset, 0>;304 def _SGPR : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_Offset, 0>;305 def _SGPR_IMM : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_IMM_Offset, 0>;306 def _IMM_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, IMM_Offset, 1>;307 def _SGPR_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_Offset, 1>;308 def _SGPR_IMM_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_IMM_Offset, 1>;309}310 311//===----------------------------------------------------------------------===//312// Scalar Memory Instructions313//===----------------------------------------------------------------------===//314 315// We are using the SReg_32_XM0 and not the SReg_32 register class for 32-bit316// SMRD instructions, because the SReg_32_XM0 register class does not include M0317// and writing to M0 from an SMRD instruction will hang the GPU.318 319// XXX - SMEM instructions do not allow exec for data operand, but320// does sdst for SMRD on SI/CI?321defm S_LOAD_DWORD : SM_Pseudo_Loads <SReg_64, SReg_32_XM0_XEXEC>;322defm S_LOAD_DWORDX2 : SM_Pseudo_Loads <SReg_64, SReg_64_XEXEC>;323let SubtargetPredicate = HasScalarDwordx3Loads in324 defm S_LOAD_DWORDX3 : SM_Pseudo_Loads <SReg_64, SReg_96>;325defm S_LOAD_DWORDX4 : SM_Pseudo_Loads <SReg_64, SReg_128>;326defm S_LOAD_DWORDX8 : SM_Pseudo_Loads <SReg_64, SReg_256>;327defm S_LOAD_DWORDX16 : SM_Pseudo_Loads <SReg_64, SReg_512>;328defm S_LOAD_I8 : SM_Pseudo_Loads <SReg_64, SReg_32_XM0_XEXEC>;329defm S_LOAD_U8 : SM_Pseudo_Loads <SReg_64, SReg_32_XM0_XEXEC>;330defm S_LOAD_I16 : SM_Pseudo_Loads <SReg_64, SReg_32_XM0_XEXEC>;331defm S_LOAD_U16 : SM_Pseudo_Loads <SReg_64, SReg_32_XM0_XEXEC>;332 333let is_buffer = 1 in {334defm S_BUFFER_LOAD_DWORD : SM_Pseudo_Loads <SReg_128_XNULL, SReg_32_XM0_XEXEC>;335// FIXME: exec_lo/exec_hi appear to be allowed for SMRD loads on336// SI/CI, bit disallowed for SMEM on VI.337defm S_BUFFER_LOAD_DWORDX2 : SM_Pseudo_Loads <SReg_128_XNULL, SReg_64_XEXEC>;338let SubtargetPredicate = HasScalarDwordx3Loads in339 defm S_BUFFER_LOAD_DWORDX3 : SM_Pseudo_Loads <SReg_128_XNULL, SReg_96>;340defm S_BUFFER_LOAD_DWORDX4 : SM_Pseudo_Loads <SReg_128_XNULL, SReg_128>;341defm S_BUFFER_LOAD_DWORDX8 : SM_Pseudo_Loads <SReg_128_XNULL, SReg_256>;342defm S_BUFFER_LOAD_DWORDX16 : SM_Pseudo_Loads <SReg_128_XNULL, SReg_512>;343defm S_BUFFER_LOAD_I8 : SM_Pseudo_Loads <SReg_128_XNULL, SReg_32_XM0_XEXEC>;344defm S_BUFFER_LOAD_U8 : SM_Pseudo_Loads <SReg_128_XNULL, SReg_32_XM0_XEXEC>;345defm S_BUFFER_LOAD_I16 : SM_Pseudo_Loads <SReg_128_XNULL, SReg_32_XM0_XEXEC>;346defm S_BUFFER_LOAD_U16 : SM_Pseudo_Loads <SReg_128_XNULL, SReg_32_XM0_XEXEC>;347}348 349let SubtargetPredicate = HasScalarStores in {350defm S_STORE_DWORD : SM_Pseudo_Stores <SReg_64, SReg_32_XM0_XEXEC>;351defm S_STORE_DWORDX2 : SM_Pseudo_Stores <SReg_64, SReg_64_XEXEC>;352defm S_STORE_DWORDX4 : SM_Pseudo_Stores <SReg_64, SReg_128>;353 354let is_buffer = 1 in {355defm S_BUFFER_STORE_DWORD : SM_Pseudo_Stores <SReg_128_XNULL, SReg_32_XM0_XEXEC>;356defm S_BUFFER_STORE_DWORDX2 : SM_Pseudo_Stores <SReg_128_XNULL, SReg_64_XEXEC>;357defm S_BUFFER_STORE_DWORDX4 : SM_Pseudo_Stores <SReg_128_XNULL, SReg_128>;358}359} // End SubtargetPredicate = HasScalarStores360 361let SubtargetPredicate = HasSMemTimeInst in362def S_MEMTIME : SM_Time_Pseudo <"s_memtime", int_amdgcn_s_memtime>;363def S_DCACHE_INV : SM_Inval_Pseudo <"s_dcache_inv", int_amdgcn_s_dcache_inv>;364 365let SubtargetPredicate = isGFX7GFX8GFX9 in {366def S_DCACHE_INV_VOL : SM_Inval_Pseudo <"s_dcache_inv_vol", int_amdgcn_s_dcache_inv_vol>;367} // let SubtargetPredicate = isGFX7GFX8GFX9368 369let SubtargetPredicate = isGFX8Plus in {370let OtherPredicates = [HasScalarStores] in {371def S_DCACHE_WB : SM_Inval_Pseudo <"s_dcache_wb", int_amdgcn_s_dcache_wb>;372def S_DCACHE_WB_VOL : SM_Inval_Pseudo <"s_dcache_wb_vol", int_amdgcn_s_dcache_wb_vol>;373} // End OtherPredicates = [HasScalarStores]374 375defm S_ATC_PROBE : SM_Pseudo_Probe <SReg_64>;376let is_buffer = 1 in {377defm S_ATC_PROBE_BUFFER : SM_Pseudo_Probe <SReg_128_XNULL>;378}379} // SubtargetPredicate = isGFX8Plus380 381let SubtargetPredicate = HasSMemRealTime in382def S_MEMREALTIME : SM_Time_Pseudo <"s_memrealtime", int_amdgcn_s_memrealtime>;383 384let SubtargetPredicate = isGFX10Plus in385def S_GL1_INV : SM_Inval_Pseudo<"s_gl1_inv">;386let SubtargetPredicate = HasGetWaveIdInst in387def S_GET_WAVEID_IN_WORKGROUP : SM_WaveId_Pseudo <"s_get_waveid_in_workgroup", int_amdgcn_s_get_waveid_in_workgroup>;388 389 390let SubtargetPredicate = HasScalarFlatScratchInsts, Uses = [FLAT_SCR] in {391defm S_SCRATCH_LOAD_DWORD : SM_Pseudo_Loads <SReg_64, SReg_32_XM0_XEXEC>;392defm S_SCRATCH_LOAD_DWORDX2 : SM_Pseudo_Loads <SReg_64, SReg_64_XEXEC>;393defm S_SCRATCH_LOAD_DWORDX4 : SM_Pseudo_Loads <SReg_64, SReg_128>;394 395defm S_SCRATCH_STORE_DWORD : SM_Pseudo_Stores <SReg_64, SReg_32_XM0_XEXEC>;396defm S_SCRATCH_STORE_DWORDX2 : SM_Pseudo_Stores <SReg_64, SReg_64_XEXEC>;397defm S_SCRATCH_STORE_DWORDX4 : SM_Pseudo_Stores <SReg_64, SReg_128>;398} // SubtargetPredicate = HasScalarFlatScratchInsts399 400let SubtargetPredicate = HasScalarAtomics in {401 402let is_buffer = 1 in {403defm S_BUFFER_ATOMIC_SWAP : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>;404defm S_BUFFER_ATOMIC_CMPSWAP : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>;405defm S_BUFFER_ATOMIC_ADD : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>;406defm S_BUFFER_ATOMIC_SUB : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>;407defm S_BUFFER_ATOMIC_SMIN : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>;408defm S_BUFFER_ATOMIC_UMIN : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>;409defm S_BUFFER_ATOMIC_SMAX : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>;410defm S_BUFFER_ATOMIC_UMAX : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>;411defm S_BUFFER_ATOMIC_AND : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>;412defm S_BUFFER_ATOMIC_OR : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>;413defm S_BUFFER_ATOMIC_XOR : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>;414defm S_BUFFER_ATOMIC_INC : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>;415defm S_BUFFER_ATOMIC_DEC : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_32_XM0_XEXEC>;416 417defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>;418defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_128>;419defm S_BUFFER_ATOMIC_ADD_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>;420defm S_BUFFER_ATOMIC_SUB_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>;421defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>;422defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>;423defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>;424defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>;425defm S_BUFFER_ATOMIC_AND_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>;426defm S_BUFFER_ATOMIC_OR_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>;427defm S_BUFFER_ATOMIC_XOR_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>;428defm S_BUFFER_ATOMIC_INC_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>;429defm S_BUFFER_ATOMIC_DEC_X2 : SM_Pseudo_Atomics <SReg_128_XNULL, SReg_64_XEXEC>;430}431 432defm S_ATOMIC_SWAP : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;433defm S_ATOMIC_CMPSWAP : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;434defm S_ATOMIC_ADD : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;435defm S_ATOMIC_SUB : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;436defm S_ATOMIC_SMIN : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;437defm S_ATOMIC_UMIN : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;438defm S_ATOMIC_SMAX : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;439defm S_ATOMIC_UMAX : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;440defm S_ATOMIC_AND : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;441defm S_ATOMIC_OR : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;442defm S_ATOMIC_XOR : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;443defm S_ATOMIC_INC : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;444defm S_ATOMIC_DEC : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>;445 446defm S_ATOMIC_SWAP_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;447defm S_ATOMIC_CMPSWAP_X2 : SM_Pseudo_Atomics <SReg_64, SReg_128>;448defm S_ATOMIC_ADD_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;449defm S_ATOMIC_SUB_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;450defm S_ATOMIC_SMIN_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;451defm S_ATOMIC_UMIN_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;452defm S_ATOMIC_SMAX_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;453defm S_ATOMIC_UMAX_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;454defm S_ATOMIC_AND_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;455defm S_ATOMIC_OR_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;456defm S_ATOMIC_XOR_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;457defm S_ATOMIC_INC_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;458defm S_ATOMIC_DEC_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>;459 460} // let SubtargetPredicate = HasScalarAtomics461 462let SubtargetPredicate = HasScalarAtomics in {463defm S_DCACHE_DISCARD : SM_Pseudo_Discards;464defm S_DCACHE_DISCARD_X2 : SM_Pseudo_Discards;465}466 467let SubtargetPredicate = isGFX12Plus in {468def S_PREFETCH_INST : SM_Prefetch_Pseudo <"s_prefetch_inst", SReg_64, 1>;469def S_PREFETCH_INST_PC_REL : SM_Prefetch_Pseudo <"s_prefetch_inst_pc_rel", SReg_64, 0>;470def S_PREFETCH_DATA : SM_Prefetch_Pseudo <"s_prefetch_data", SReg_64, 1>;471def S_PREFETCH_DATA_PC_REL : SM_Prefetch_Pseudo <"s_prefetch_data_pc_rel", SReg_64, 0>;472def S_BUFFER_PREFETCH_DATA : SM_Prefetch_Pseudo <"s_buffer_prefetch_data", SReg_128_XNULL, 1> {473 let is_buffer = 1;474}475} // end let SubtargetPredicate = isGFX12Plus476 477//===----------------------------------------------------------------------===//478// Targets479//===----------------------------------------------------------------------===//480 481//===----------------------------------------------------------------------===//482// SI483//===----------------------------------------------------------------------===//484 485class SMRD_Real_si <bits<5> op, SM_Pseudo ps>486 : SM_Real<ps>487 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI>488 , Enc32 {489 490 let AssemblerPredicate = isGFX6GFX7;491 let DecoderNamespace = "GFX6GFX7";492 493 let Inst{7-0} = !if(ps.has_offset, offset{7-0}, !if(ps.has_soffset, soffset, ?));494 let Inst{8} = ps.has_offset;495 let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?);496 let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?);497 let Inst{26-22} = op;498 let Inst{31-27} = 0x18; //encoding499}500 501multiclass SM_Real_Loads_si<bits<5> op> {502 defvar ps = NAME;503 defvar immPs = !cast<SM_Load_Pseudo>(ps#_IMM);504 def _IMM_si : SMRD_Real_si <op, immPs> {505 let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset_8:$offset, CPol:$cpol);506 }507 508 defvar sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR);509 def _SGPR_si : SMRD_Real_si <op, sgprPs>;510}511 512defm S_LOAD_DWORD : SM_Real_Loads_si <0x00>;513defm S_LOAD_DWORDX2 : SM_Real_Loads_si <0x01>;514defm S_LOAD_DWORDX4 : SM_Real_Loads_si <0x02>;515defm S_LOAD_DWORDX8 : SM_Real_Loads_si <0x03>;516defm S_LOAD_DWORDX16 : SM_Real_Loads_si <0x04>;517defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_si <0x08>;518defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_si <0x09>;519defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_si <0x0a>;520defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_si <0x0b>;521defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_si <0x0c>;522 523def S_MEMTIME_si : SMRD_Real_si <0x1e, S_MEMTIME>;524def S_DCACHE_INV_si : SMRD_Real_si <0x1f, S_DCACHE_INV>;525 526 527//===----------------------------------------------------------------------===//528// VI and GFX9.529//===----------------------------------------------------------------------===//530 531class SMEM_Real_vi <bits<8> op, SM_Pseudo ps>532 : SM_Real<ps>533 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI>534 , Enc64 {535 field bit IsGFX9SpecificEncoding = false;536 let AssemblerPredicate = !if(IsGFX9SpecificEncoding, isGFX9Only, isGFX8GFX9);537 let DecoderNamespace = "GFX8";538 539 let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?);540 let Inst{12-6} = !if(ps.has_sdst, sdst{6-0}, ?);541 542 // Note that for GFX9 instructions with immediate offsets, soffset_en543 // must be defined, whereas in GFX8 it's undefined in all cases,544 // meaning GFX9 is not perfectly backward-compatible with GFX8, despite545 // documentation suggesting otherwise.546 field bit SOffsetEn = !if(IsGFX9SpecificEncoding,547 !if(ps.has_offset, ps.has_soffset, !if(ps.has_soffset, 0, ?)),548 ?);549 let Inst{14} = SOffsetEn;550 551 let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ?);552 553 // imm554 // TODO: Shall not be defined if the instruction has no offset nor555 // soffset.556 let Inst{17} = ps.has_offset;557 558 let Inst{25-18} = op;559 let Inst{31-26} = 0x30; //encoding560 561 // VI supports 20-bit unsigned offsets while GFX9+ supports 21-bit signed.562 // Offset value is corrected accordingly when offset is encoded/decoded.563 // TODO: Forbid non-M0 register offsets for GFX8 stores and atomics.564 field bits<21> Offset;565 let Offset{6-0} = !if(ps.has_offset, offset{6-0},566 !if(ps.has_soffset, soffset{6-0}, ?));567 let Offset{20-7} = !if(ps.has_offset, offset{20-7}, ?);568 let Inst{52-32} = Offset;569 570 // soffset571 let Inst{63-57} = !if(!and(IsGFX9SpecificEncoding, ps.has_soffset),572 soffset{6-0}, ?);573}574 575class SMEM_Real_Load_vi<bits<8> op, string ps>576 : SMEM_Real_vi<op, !cast<SM_Pseudo>(ps)>;577 578// The alternative GFX9 SGPR encoding using soffset to encode the579// offset register. Not available in assembler and goes to the GFX9580// encoding family to avoid conflicts with the primary SGPR variant.581class SMEM_Real_SGPR_alt_gfx9 {582 bit IsGFX9SpecificEncoding = true;583 bit SOffsetEn = 1;584 bit Offset = ?;585 int Subtarget = SIEncodingFamily.GFX9;586 string AsmVariantName = "NonParsable";587}588 589multiclass SM_Real_Loads_vi<bits<8> op> {590 defvar ps = NAME;591 def _IMM_vi : SMEM_Real_Load_vi <op, ps#"_IMM">;592 def _SGPR_vi : SMEM_Real_Load_vi <op, ps#"_SGPR">;593 def _SGPR_alt_gfx9 : SMEM_Real_Load_vi <op, ps#"_SGPR">,594 SMEM_Real_SGPR_alt_gfx9;595 let IsGFX9SpecificEncoding = true in596 def _SGPR_IMM_gfx9 : SMEM_Real_Load_vi <op, ps#"_SGPR_IMM">;597}598 599class SMEM_Real_Store_Base_vi <bits<8> op, SM_Pseudo ps> : SMEM_Real_vi <op, ps> {600 // encoding601 bits<7> sdata;602 603 let sdst = ?;604 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?);605}606 607class SMEM_Real_Store_vi <bits<8> op, string ps>608 : SMEM_Real_Store_Base_vi <op, !cast<SM_Pseudo>(ps)>;609 610multiclass SM_Real_Stores_vi<bits<8> op> {611 defvar ps = NAME;612 def _IMM_vi : SMEM_Real_Store_vi <op, ps#"_IMM">;613 def _SGPR_vi : SMEM_Real_Store_vi <op, ps#"_SGPR">;614 def _SGPR_alt_gfx9 : SMEM_Real_Store_vi <op, ps#"_SGPR">,615 SMEM_Real_SGPR_alt_gfx9;616 let IsGFX9SpecificEncoding = true in617 def _SGPR_IMM_gfx9 : SMEM_Real_Store_vi <op, ps#"_SGPR_IMM">;618}619 620multiclass SM_Real_Probe_vi<bits<8> op> {621 defvar ps = NAME;622 def _IMM_vi : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_IMM)>;623 def _SGPR_vi : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>;624 def _SGPR_alt_gfx9625 : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>,626 SMEM_Real_SGPR_alt_gfx9;627 let IsGFX9SpecificEncoding = true in628 def _SGPR_IMM_gfx9629 : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR_IMM)>;630}631 632defm S_LOAD_DWORD : SM_Real_Loads_vi <0x00>;633defm S_LOAD_DWORDX2 : SM_Real_Loads_vi <0x01>;634defm S_LOAD_DWORDX4 : SM_Real_Loads_vi <0x02>;635defm S_LOAD_DWORDX8 : SM_Real_Loads_vi <0x03>;636defm S_LOAD_DWORDX16 : SM_Real_Loads_vi <0x04>;637defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_vi <0x08>;638defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_vi <0x09>;639defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_vi <0x0a>;640defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_vi <0x0b>;641defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_vi <0x0c>;642 643defm S_STORE_DWORD : SM_Real_Stores_vi <0x10>;644defm S_STORE_DWORDX2 : SM_Real_Stores_vi <0x11>;645defm S_STORE_DWORDX4 : SM_Real_Stores_vi <0x12>;646 647defm S_BUFFER_STORE_DWORD : SM_Real_Stores_vi <0x18>;648defm S_BUFFER_STORE_DWORDX2 : SM_Real_Stores_vi <0x19>;649defm S_BUFFER_STORE_DWORDX4 : SM_Real_Stores_vi <0x1a>;650 651// These instructions use same encoding652def S_DCACHE_INV_vi : SMEM_Real_vi <0x20, S_DCACHE_INV>;653def S_DCACHE_WB_vi : SMEM_Real_vi <0x21, S_DCACHE_WB>;654def S_DCACHE_INV_VOL_vi : SMEM_Real_vi <0x22, S_DCACHE_INV_VOL>;655def S_DCACHE_WB_VOL_vi : SMEM_Real_vi <0x23, S_DCACHE_WB_VOL>;656def S_MEMTIME_vi : SMEM_Real_vi <0x24, S_MEMTIME>;657def S_MEMREALTIME_vi : SMEM_Real_vi <0x25, S_MEMREALTIME>;658 659defm S_SCRATCH_LOAD_DWORD : SM_Real_Loads_vi <0x05>;660defm S_SCRATCH_LOAD_DWORDX2 : SM_Real_Loads_vi <0x06>;661defm S_SCRATCH_LOAD_DWORDX4 : SM_Real_Loads_vi <0x07>;662 663defm S_SCRATCH_STORE_DWORD : SM_Real_Stores_vi <0x15>;664defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_vi <0x16>;665defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_vi <0x17>;666 667defm S_ATC_PROBE : SM_Real_Probe_vi <0x26>;668defm S_ATC_PROBE_BUFFER : SM_Real_Probe_vi <0x27>;669 670//===----------------------------------------------------------------------===//671// GFX9672//===----------------------------------------------------------------------===//673 674class SMEM_Atomic_Real_vi <bits<8> op, SM_Atomic_Pseudo ps>675 : SMEM_Real_vi <op, ps> {676 677 bits<7> sdata;678 679 let Constraints = ps.Constraints;680 681 let cpol{CPolBit.GLC} = ps.glc;682 let Inst{12-6} = !if(ps.glc, sdst{6-0}, sdata{6-0});683}684 685multiclass SM_Real_Atomics_vi<bits<8> op> {686 defvar ps = NAME;687 def _IMM_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>;688 def _SGPR_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>;689 def _SGPR_alt_gfx9690 : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>,691 SMEM_Real_SGPR_alt_gfx9;692 let IsGFX9SpecificEncoding = true in693 def _SGPR_IMM_gfx9694 : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM)>;695 def _IMM_RTN_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>;696 def _SGPR_RTN_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>;697 def _SGPR_RTN_alt_gfx9698 : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>,699 SMEM_Real_SGPR_alt_gfx9;700 let IsGFX9SpecificEncoding = true in701 def _SGPR_IMM_RTN_gfx9702 : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM_RTN)>;703}704 705defm S_BUFFER_ATOMIC_SWAP : SM_Real_Atomics_vi <0x40>;706defm S_BUFFER_ATOMIC_CMPSWAP : SM_Real_Atomics_vi <0x41>;707defm S_BUFFER_ATOMIC_ADD : SM_Real_Atomics_vi <0x42>;708defm S_BUFFER_ATOMIC_SUB : SM_Real_Atomics_vi <0x43>;709defm S_BUFFER_ATOMIC_SMIN : SM_Real_Atomics_vi <0x44>;710defm S_BUFFER_ATOMIC_UMIN : SM_Real_Atomics_vi <0x45>;711defm S_BUFFER_ATOMIC_SMAX : SM_Real_Atomics_vi <0x46>;712defm S_BUFFER_ATOMIC_UMAX : SM_Real_Atomics_vi <0x47>;713defm S_BUFFER_ATOMIC_AND : SM_Real_Atomics_vi <0x48>;714defm S_BUFFER_ATOMIC_OR : SM_Real_Atomics_vi <0x49>;715defm S_BUFFER_ATOMIC_XOR : SM_Real_Atomics_vi <0x4a>;716defm S_BUFFER_ATOMIC_INC : SM_Real_Atomics_vi <0x4b>;717defm S_BUFFER_ATOMIC_DEC : SM_Real_Atomics_vi <0x4c>;718 719defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Real_Atomics_vi <0x60>;720defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_vi <0x61>;721defm S_BUFFER_ATOMIC_ADD_X2 : SM_Real_Atomics_vi <0x62>;722defm S_BUFFER_ATOMIC_SUB_X2 : SM_Real_Atomics_vi <0x63>;723defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Real_Atomics_vi <0x64>;724defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Real_Atomics_vi <0x65>;725defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Real_Atomics_vi <0x66>;726defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Real_Atomics_vi <0x67>;727defm S_BUFFER_ATOMIC_AND_X2 : SM_Real_Atomics_vi <0x68>;728defm S_BUFFER_ATOMIC_OR_X2 : SM_Real_Atomics_vi <0x69>;729defm S_BUFFER_ATOMIC_XOR_X2 : SM_Real_Atomics_vi <0x6a>;730defm S_BUFFER_ATOMIC_INC_X2 : SM_Real_Atomics_vi <0x6b>;731defm S_BUFFER_ATOMIC_DEC_X2 : SM_Real_Atomics_vi <0x6c>;732 733defm S_ATOMIC_SWAP : SM_Real_Atomics_vi <0x80>;734defm S_ATOMIC_CMPSWAP : SM_Real_Atomics_vi <0x81>;735defm S_ATOMIC_ADD : SM_Real_Atomics_vi <0x82>;736defm S_ATOMIC_SUB : SM_Real_Atomics_vi <0x83>;737defm S_ATOMIC_SMIN : SM_Real_Atomics_vi <0x84>;738defm S_ATOMIC_UMIN : SM_Real_Atomics_vi <0x85>;739defm S_ATOMIC_SMAX : SM_Real_Atomics_vi <0x86>;740defm S_ATOMIC_UMAX : SM_Real_Atomics_vi <0x87>;741defm S_ATOMIC_AND : SM_Real_Atomics_vi <0x88>;742defm S_ATOMIC_OR : SM_Real_Atomics_vi <0x89>;743defm S_ATOMIC_XOR : SM_Real_Atomics_vi <0x8a>;744defm S_ATOMIC_INC : SM_Real_Atomics_vi <0x8b>;745defm S_ATOMIC_DEC : SM_Real_Atomics_vi <0x8c>;746 747defm S_ATOMIC_SWAP_X2 : SM_Real_Atomics_vi <0xa0>;748defm S_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_vi <0xa1>;749defm S_ATOMIC_ADD_X2 : SM_Real_Atomics_vi <0xa2>;750defm S_ATOMIC_SUB_X2 : SM_Real_Atomics_vi <0xa3>;751defm S_ATOMIC_SMIN_X2 : SM_Real_Atomics_vi <0xa4>;752defm S_ATOMIC_UMIN_X2 : SM_Real_Atomics_vi <0xa5>;753defm S_ATOMIC_SMAX_X2 : SM_Real_Atomics_vi <0xa6>;754defm S_ATOMIC_UMAX_X2 : SM_Real_Atomics_vi <0xa7>;755defm S_ATOMIC_AND_X2 : SM_Real_Atomics_vi <0xa8>;756defm S_ATOMIC_OR_X2 : SM_Real_Atomics_vi <0xa9>;757defm S_ATOMIC_XOR_X2 : SM_Real_Atomics_vi <0xaa>;758defm S_ATOMIC_INC_X2 : SM_Real_Atomics_vi <0xab>;759defm S_ATOMIC_DEC_X2 : SM_Real_Atomics_vi <0xac>;760 761multiclass SM_Real_Discard_vi<bits<8> op> {762 defvar ps = NAME;763 def _IMM_vi : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_IMM)>;764 def _SGPR_vi : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR)>;765 def _SGPR_alt_gfx9 : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR)>,766 SMEM_Real_SGPR_alt_gfx9;767 let IsGFX9SpecificEncoding = true in768 def _SGPR_IMM_gfx9 : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR_IMM)>;769}770 771defm S_DCACHE_DISCARD : SM_Real_Discard_vi <0x28>;772defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_vi <0x29>;773 774//===----------------------------------------------------------------------===//775// CI776//===----------------------------------------------------------------------===//777 778def smrd_literal_offset : ImmOperand<i32, "SMRDLiteralOffset">;779 780class SMRD_Real_Load_IMM_ci <bits<5> op, SM_Load_Pseudo ps> :781 SM_Real<ps>,782 Enc64 {783 784 let AssemblerPredicate = isGFX7Only;785 let DecoderNamespace = "GFX7";786 let InOperandList = (ins ps.BaseClass:$sbase, smrd_literal_offset:$offset, CPol:$cpol);787 788 let Inst{7-0} = 0xff;789 let Inst{8} = 0;790 let Inst{14-9} = sbase{6-1};791 let Inst{21-15} = sdst{6-0};792 let Inst{26-22} = op;793 let Inst{31-27} = 0x18; //encoding794 let Inst{63-32} = offset{31-0};795}796 797def S_LOAD_DWORD_IMM_ci : SMRD_Real_Load_IMM_ci <0x00, S_LOAD_DWORD_IMM>;798def S_LOAD_DWORDX2_IMM_ci : SMRD_Real_Load_IMM_ci <0x01, S_LOAD_DWORDX2_IMM>;799def S_LOAD_DWORDX4_IMM_ci : SMRD_Real_Load_IMM_ci <0x02, S_LOAD_DWORDX4_IMM>;800def S_LOAD_DWORDX8_IMM_ci : SMRD_Real_Load_IMM_ci <0x03, S_LOAD_DWORDX8_IMM>;801def S_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x04, S_LOAD_DWORDX16_IMM>;802def S_BUFFER_LOAD_DWORD_IMM_ci : SMRD_Real_Load_IMM_ci <0x08, S_BUFFER_LOAD_DWORD_IMM>;803def S_BUFFER_LOAD_DWORDX2_IMM_ci : SMRD_Real_Load_IMM_ci <0x09, S_BUFFER_LOAD_DWORDX2_IMM>;804def S_BUFFER_LOAD_DWORDX4_IMM_ci : SMRD_Real_Load_IMM_ci <0x0a, S_BUFFER_LOAD_DWORDX4_IMM>;805def S_BUFFER_LOAD_DWORDX8_IMM_ci : SMRD_Real_Load_IMM_ci <0x0b, S_BUFFER_LOAD_DWORDX8_IMM>;806def S_BUFFER_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x0c, S_BUFFER_LOAD_DWORDX16_IMM>;807 808class SMRD_Real_ci <bits<5> op, SM_Pseudo ps>809 : SM_Real<ps>810 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI>811 , Enc32 {812 813 let AssemblerPredicate = isGFX7Only;814 let DecoderNamespace = "GFX7";815 816 let Inst{7-0} = !if(ps.has_offset, offset{7-0}, !if(ps.has_soffset, soffset, ?));817 let Inst{8} = ps.has_offset;818 let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?);819 let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?);820 let Inst{26-22} = op;821 let Inst{31-27} = 0x18; //encoding822}823 824def S_DCACHE_INV_VOL_ci : SMRD_Real_ci <0x1d, S_DCACHE_INV_VOL>;825 826//===----------------------------------------------------------------------===//827// Scalar Memory Patterns828//===----------------------------------------------------------------------===//829 830class SMRDLoadPat<PatFrag Op> : PatFrag <(ops node:$ptr), (Op node:$ptr), [{ return isUniformLoad(N);}]> {831 let GISelPredicateCode = [{832 if (!MI.hasOneMemOperand())833 return false;834 if (!isInstrUniform(MI))835 return false;836 837 // FIXME: We should probably be caching this.838 SmallVector<GEPInfo, 4> AddrInfo;839 getAddrModeInfo(MI, MRI, AddrInfo);840 841 if (hasVgprParts(AddrInfo))842 return false;843 return true;844 }];845}846 847def smrd_load : SMRDLoadPat<load>;848def smrd_extloadi8 : SMRDLoadPat<extloadi8>;849def smrd_zextloadi8 : SMRDLoadPat<zextloadi8>;850def smrd_sextloadi8 : SMRDLoadPat<sextloadi8>;851def smrd_extloadi16 : SMRDLoadPat<extloadi16>;852def smrd_zextloadi16 : SMRDLoadPat<zextloadi16>;853def smrd_sextloadi16 : SMRDLoadPat<sextloadi16>;854 855def smrd_prefetch : PatFrag <(ops node:$ptr, node:$rw, node:$loc, node:$type),856 (prefetch node:$ptr, node:$rw, node:$loc, node:$type),857 [{ return !N->getOperand(1)->isDivergent() && Subtarget->hasSafeSmemPrefetch();}]> {858 let GISelPredicateCode = [{859 return isInstrUniform(MI) && Subtarget->hasSafeSmemPrefetch();860 }];861}862 863def SMRDImm : ComplexPattern<iPTR, 2, "SelectSMRDImm">;864def SMRDImm32 : ComplexPattern<iPTR, 2, "SelectSMRDImm32">;865let WantsRoot = true in {866 def SMRDSgpr : ComplexPattern<iPTR, 3, "SelectSMRDSgpr", [], [], -3>;867 def SMRDSgprImm : ComplexPattern<iPTR, 4, "SelectSMRDSgprImm", [], []>;868}869def SMRDBufferImm : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm">;870def SMRDBufferImm32 : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm32">;871def SMRDBufferSgprImm : ComplexPattern<iPTR, 2, "SelectSMRDBufferSgprImm">;872 873class SMRDAlignedLoadPat<PatFrag Op> : PatFrag <(ops node:$ptr), (Op node:$ptr), [{874 // Returns true if it is a single dword load or naturally aligned multi-dword load.875 LoadSDNode *Ld = cast<LoadSDNode>(N);876 unsigned Size = Ld->getMemoryVT().getStoreSize();877 return Size <= 4 || Ld->getAlign().value() >= Size;878}]> {879 let GISelPredicateCode = [{880 auto &Ld = cast<GLoad>(MI);881 TypeSize Size = Ld.getMMO().getSize().getValue();882 return Size <= 4 || Ld.getMMO().getAlign().value() >= Size;883 }];884}885 886def aligned_smrd_load : SMRDAlignedLoadPat<smrd_load>;887 888multiclass SMRD_Patterns <string Instr, ValueType vt, PatFrag frag,889 bit immci = true, string suffix = ""> {890 // 1. IMM offset891 def : GCNPat <892 (frag (SMRDImm i64:$sbase, i32:$offset)),893 (vt (!cast<SM_Pseudo>(Instr#"_IMM"#suffix) $sbase, $offset, 0))>;894 895 // 2. 32-bit IMM offset on CI896 if immci then def : GCNPat <897 (frag (SMRDImm32 i64:$sbase, i32:$offset)),898 (vt (!cast<InstSI>(Instr#"_IMM_ci"#suffix) $sbase, $offset, 0))> {899 let SubtargetPredicate = isGFX7Only;900 }901 902 // 3. SGPR offset903 def : GCNPat <904 (frag (SMRDSgpr i64:$sbase, i32:$soffset)),905 (vt (!cast<SM_Pseudo>(Instr#"_SGPR"#suffix) $sbase, $soffset, 0))> {906 let SubtargetPredicate = isNotGFX9Plus;907 }908 def : GCNPat <909 (frag (SMRDSgpr i64:$sbase, i32:$soffset, CPol:$cpol)),910 (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM"#suffix) $sbase, $soffset, 0, $cpol))> {911 let SubtargetPredicate = isGFX9Plus;912 }913 914 // 4. SGPR+IMM offset915 def : GCNPat <916 (frag (SMRDSgprImm i64:$sbase, i32:$soffset, i32:$offset, CPol:$cpol)),917 (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM"#suffix) $sbase, $soffset, $offset, $cpol))> {918 let SubtargetPredicate = isGFX9Plus;919 }920 921 // 5. No offset922 def : GCNPat <923 (vt (frag (i64 SReg_64:$sbase))),924 (vt (!cast<SM_Pseudo>(Instr#"_IMM"#suffix) i64:$sbase, 0, 0))>;925}926 927multiclass SMRD_Pattern <string Instr, ValueType vt, bit immci = true> {928 // High priority when XNACK is enabled and the load was naturally aligned.929 let OtherPredicates = [HasXNACKEnabled], AddedComplexity = 102 in930 defm: SMRD_Patterns <Instr, vt, aligned_smrd_load, immci>;931 932 // XNACK is enabled and the load wasn't naturally aligned. The constrained sload variant.933 if !gt(vt.Size, 32) then {934 let OtherPredicates = [HasXNACKEnabled], AddedComplexity = 101 in935 defm: SMRD_Patterns <Instr, vt, smrd_load, /*immci=*/false, /*suffix=*/"_ec">;936 }937 938 // XNACK is disabled.939 let AddedComplexity = 100 in940 defm: SMRD_Patterns <Instr, vt, smrd_load, immci>;941}942 943multiclass SMLoad_Pattern <string Instr, ValueType vt, bit immci = true> {944 // 1. Offset as an immediate945 def : GCNPat <946 (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm i32:$offset), timm:$cachepolicy),947 (vt (!cast<SM_Pseudo>(Instr#"_IMM") SReg_128:$sbase, i32imm:$offset, (extract_cpol $cachepolicy)))> {948 let AddedComplexity = 2;949 }950 951 // 2. 32-bit IMM offset on CI952 if immci then def : GCNPat <953 (vt (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm32 i32:$offset), timm:$cachepolicy)),954 (!cast<InstSI>(Instr#"_IMM_ci") SReg_128:$sbase, smrd_literal_offset:$offset,955 (extract_cpol $cachepolicy))> {956 let SubtargetPredicate = isGFX7Only;957 let AddedComplexity = 1;958 }959 960 // 3. Offset loaded in an 32bit SGPR961 def : GCNPat <962 (SIsbuffer_load v4i32:$sbase, i32:$soffset, timm:$cachepolicy),963 (vt (!cast<SM_Pseudo>(Instr#"_SGPR") SReg_128:$sbase, SReg_32:$soffset, (extract_cpol $cachepolicy)))> {964 let SubtargetPredicate = isNotGFX9Plus;965 }966 def : GCNPat <967 (SIsbuffer_load v4i32:$sbase, i32:$soffset, timm:$cachepolicy),968 (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") SReg_128:$sbase, SReg_32:$soffset, 0, (extract_cpol $cachepolicy)))> {969 let SubtargetPredicate = isGFX9Plus;970 }971 972 // 4. Offset as an 32-bit SGPR + immediate973 def : GCNPat <974 (SIsbuffer_load v4i32:$sbase, (SMRDBufferSgprImm i32:$soffset, i32:$offset),975 timm:$cachepolicy),976 (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") SReg_128:$sbase, SReg_32:$soffset, i32imm:$offset,977 (extract_cpol $cachepolicy)))> {978 let SubtargetPredicate = isGFX9Plus;979 }980}981 982multiclass ScalarLoadWithExtensionPat <string Instr, SDPatternOperator node, ValueType vt> {983 // 1. IMM offset984 def : GCNPat <985 (node (SMRDImm i64:$sbase, i32:$offset)),986 (vt (!cast<SM_Pseudo>(Instr#"_IMM") $sbase, $offset, 0))>{987 let SubtargetPredicate = isGFX12Plus;988 }989 990 // 2. SGPR offset991 def : GCNPat <992 (node (SMRDSgpr i64:$sbase, i32:$soffset, CPol:$cpol)),993 (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") $sbase, $soffset, 0, $cpol))>{994 let SubtargetPredicate = isGFX12Plus;995 }996 997 // 3. SGPR+IMM offset998 def : GCNPat <999 (node (SMRDSgprImm i64:$sbase, i32:$soffset, i32:$offset, CPol:$cpol)),1000 (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") $sbase, $soffset, $offset, $cpol))>{1001 let SubtargetPredicate = isGFX12Plus;1002 }1003 1004 // 4. No offset1005 def : GCNPat <1006 (vt (node (i64 SReg_64:$sbase))),1007 (vt (!cast<SM_Pseudo>(Instr#"_IMM") i64:$sbase, 0, 0))>{1008 let SubtargetPredicate = isGFX12Plus;1009 }1010}1011 1012multiclass ScalarBufferLoadIntrinsicPat <SDPatternOperator name, string Instr> {1013 1014 // 1. Offset as an immediate1015 def : GCNPat <1016 (name v4i32:$sbase, (SMRDBufferImm i32:$offset), timm:$cachepolicy),1017 (i32 (!cast<SM_Pseudo>(Instr#"_IMM") SReg_128:$sbase, i32imm:$offset, (extract_cpol $cachepolicy)))> {1018 let SubtargetPredicate = isGFX12Plus;1019 }1020 1021 // 2. Offset as an 32-bit SGPR1022 def : GCNPat <1023 (name v4i32:$sbase, i32:$soffset, timm:$cachepolicy),1024 (i32 (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") SReg_128:$sbase, SReg_32:$soffset, 0, (extract_cpol $cachepolicy)))> {1025 let SubtargetPredicate = isGFX12Plus;1026 }1027 1028 // 3. Offset as an 32-bit SGPR + immediate1029 def : GCNPat <1030 (name v4i32:$sbase, (SMRDBufferSgprImm i32:$soffset, i32:$offset),1031 timm:$cachepolicy),1032 (i32 (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") SReg_128:$sbase, SReg_32:$soffset, i32imm:$offset,1033 (extract_cpol $cachepolicy)))> {1034 let SubtargetPredicate = isGFX12Plus;1035 }1036}1037 1038// Global and constant loads can be selected to either MUBUF or SMRD1039// instructions, but SMRD instructions are faster so we want the instruction1040// selector to prefer those.1041let AddedComplexity = 100 in {1042 1043defm : ScalarLoadWithExtensionPat <"S_LOAD_U8", smrd_extloadi8, i32>;1044defm : ScalarLoadWithExtensionPat <"S_LOAD_U8", smrd_zextloadi8, i32>;1045defm : ScalarLoadWithExtensionPat <"S_LOAD_I8", smrd_sextloadi8, i32>;1046defm : ScalarLoadWithExtensionPat <"S_LOAD_U16", smrd_extloadi16, i32>;1047defm : ScalarLoadWithExtensionPat <"S_LOAD_U16", smrd_zextloadi16, i32>;1048defm : ScalarLoadWithExtensionPat <"S_LOAD_I16", smrd_sextloadi16, i32>;1049defm : ScalarBufferLoadIntrinsicPat <SIsbuffer_load_byte, "S_BUFFER_LOAD_I8">;1050defm : ScalarBufferLoadIntrinsicPat <SIsbuffer_load_ubyte, "S_BUFFER_LOAD_U8">;1051defm : ScalarBufferLoadIntrinsicPat <SIsbuffer_load_short, "S_BUFFER_LOAD_I16">;1052defm : ScalarBufferLoadIntrinsicPat <SIsbuffer_load_ushort, "S_BUFFER_LOAD_U16">;1053 1054} // End let AddedComplexity = 1001055 1056foreach vt = Reg32Types.types in {1057defm : SMRD_Pattern <"S_LOAD_DWORD", vt>;1058}1059 1060foreach vt = SReg_64.RegTypes in {1061defm : SMRD_Pattern <"S_LOAD_DWORDX2", vt>;1062}1063 1064foreach vt = SReg_96.RegTypes in {1065defm : SMRD_Pattern <"S_LOAD_DWORDX3", vt, false>;1066}1067 1068foreach vt = SReg_128.RegTypes in {1069defm : SMRD_Pattern <"S_LOAD_DWORDX4", vt>;1070}1071 1072foreach vt = SReg_256.RegTypes in {1073defm : SMRD_Pattern <"S_LOAD_DWORDX8", vt>;1074}1075 1076foreach vt = SReg_512.RegTypes in {1077defm : SMRD_Pattern <"S_LOAD_DWORDX16", vt>;1078}1079 1080 1081defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD", i32>;1082defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2", v2i32>;1083defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX3", v3i32, false>;1084defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4", v4i32>;1085defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8", v8i32>;1086defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16", v16i32>;1087 1088defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD", f32>;1089defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2", v2f32>;1090defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX3", v3f32, false>;1091defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4", v4f32>;1092defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8", v8f32>;1093defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16", v16f32>;1094 1095let OtherPredicates = [HasSMemTimeInst] in {1096def : GCNPat <1097 (i64 (readcyclecounter)),1098 (S_MEMTIME)1099>;1100} // let OtherPredicates = [HasSMemTimeInst]1101 1102let OtherPredicates = [HasShaderCyclesRegister] in {1103def : GCNPat <1104 (i64 (readcyclecounter)),1105 (REG_SEQUENCE SReg_64,1106 (S_GETREG_B32 getHwRegImm<HWREG.SHADER_CYCLES, 0, -12>.ret), sub0,1107 (S_MOV_B32 (i32 0)), sub1)> {1108}1109} // let OtherPredicates = [HasShaderCyclesRegister]1110 1111let OtherPredicates = [HasSMemRealTime] in {1112def : GCNPat <1113 (i64 (readsteadycounter)),1114 (S_MEMREALTIME)1115>;1116} // let OtherPredicates = [HasSMemRealTime]1117 1118let SubtargetPredicate = isGFX11Plus in {1119def : GCNPat <1120 (i64 (readsteadycounter)),1121 (S_SENDMSG_RTN_B64 (i32 /*MSG_RTN_GET_REALTIME=*/0x83))1122>;1123} // let SubtargetPredicate = [isGFX11Plus]1124 1125def i32imm_zero : TImmLeaf <i32, [{1126 return Imm == 0;1127}]>;1128 1129def i32imm_one : TImmLeaf <i32, [{1130 return Imm == 1;1131}]>;1132 1133multiclass SMPrefetchPat<string type, TImmLeaf cache_type> {1134 def : GCNPat <1135 (smrd_prefetch (SMRDImm i64:$sbase, i32:$offset), timm, timm, cache_type),1136 (!cast<SM_Prefetch_Pseudo>("S_PREFETCH_"#type) $sbase, $offset, (i32 SGPR_NULL), (i8 0))1137 >;1138 1139 def : GCNPat <1140 (smrd_prefetch (i64 SReg_64:$sbase), timm, timm, cache_type),1141 (!cast<SM_Prefetch_Pseudo>("S_PREFETCH_"#type) $sbase, 0, (i32 SGPR_NULL), (i8 0))1142 >;1143 1144 def : GCNPat <1145 (smrd_prefetch (i32 SReg_32:$sbase), timm, timm, cache_type),1146 (!cast<SM_Prefetch_Pseudo>("S_PREFETCH_"#type)1147 (i64 (REG_SEQUENCE SReg_64, $sbase, sub0, (i32 (S_MOV_B32 (i32 0))), sub1)),1148 0, (i32 SGPR_NULL), (i8 0))1149 >;1150}1151 1152defm : SMPrefetchPat<"INST", i32imm_zero>;1153let AddedComplexity = 12 in // Prefer scalar prefetch over global for r/o case.1154defm : SMPrefetchPat<"DATA", i32imm_one>;1155 1156let SubtargetPredicate = isGFX12Plus in {1157 def : GCNPat <1158 (int_amdgcn_s_prefetch_data (SMRDImm i64:$sbase, i32:$offset), (i32 SReg_32:$len)),1159 (S_PREFETCH_DATA $sbase, $offset, $len, 0)1160 >;1161 1162 def : GCNPat <1163 (int_amdgcn_s_prefetch_data (i64 SReg_64:$sbase), (i32 SReg_32:$len)),1164 (S_PREFETCH_DATA $sbase, 0, $len, 0)1165 >;1166 1167 def : GCNPat <1168 (int_amdgcn_s_prefetch_data (SMRDImm i64:$sbase, i32:$offset), imm:$len),1169 (S_PREFETCH_DATA $sbase, $offset, (i32 SGPR_NULL), (as_i8timm $len))1170 >;1171 1172 def : GCNPat <1173 (int_amdgcn_s_prefetch_data (i64 SReg_64:$sbase), imm:$len),1174 (S_PREFETCH_DATA $sbase, 0, (i32 SGPR_NULL), (as_i8timm $len))1175 1176 >;1177 1178 def : GCNPat <1179 (SIsbuffer_prefetch v4i32:$sbase, (SMRDBufferImm i32:$offset), (i32 SReg_32:$len)),1180 (S_BUFFER_PREFETCH_DATA SReg_128:$sbase, i32imm:$offset, $len, 0)1181 >;1182 1183 def : GCNPat <1184 (SIsbuffer_prefetch v4i32:$sbase, (SMRDBufferImm i32:$offset), imm:$len),1185 (S_BUFFER_PREFETCH_DATA SReg_128:$sbase, i32imm:$offset, (i32 SGPR_NULL), (as_i8timm $len))1186 >;1187} // End let SubtargetPredicate = isGFX12Plus1188 1189//===----------------------------------------------------------------------===//1190// GFX10.1191//===----------------------------------------------------------------------===//1192 1193class SMEM_Real_10Plus_common<bits<8> op, SM_Pseudo ps, string opName,1194 int subtarget, RegisterWithSubRegs sgpr_null> :1195 SM_Real<ps, opName>, SIMCInstr<ps.PseudoInstr, subtarget>, Enc64 {1196 let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?);1197 let Inst{12-6} = !if(ps.has_sdst, sdst{6-0}, ?);1198 let Inst{25-18} = op;1199 let Inst{31-26} = 0x3d;1200 // There are SMEM instructions that do not employ any of the offset1201 // fields, in which case we need them to remain undefined.1202 let Inst{52-32} = !if(ps.has_offset, offset{20-0}, !if(ps.has_soffset, 0, ?));1203 let Inst{63-57} = !if(ps.has_soffset, soffset{6-0},1204 !if(ps.has_offset, sgpr_null.HWEncoding{6-0}, ?));1205}1206 1207class SMEM_Real_gfx10<bits<8> op, SM_Pseudo ps>1208 : SMEM_Real_10Plus_common<op, ps, ps.Mnemonic, SIEncodingFamily.GFX10,1209 SGPR_NULL_gfxpre11> {1210 let AssemblerPredicate = isGFX10Only;1211 let DecoderNamespace = "GFX10";1212 let Inst{14} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ?);1213 let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ?);1214}1215 1216class SMEM_Real_Load_gfx10<bits<8> op, string ps>1217 : SMEM_Real_gfx10<op, !cast<SM_Pseudo>(ps)>;1218 1219multiclass SM_Real_Loads_gfx10<bits<8> op> {1220 defvar ps = NAME;1221 def _IMM_gfx10 : SMEM_Real_Load_gfx10<op, ps#"_IMM">;1222 def _SGPR_gfx10 : SMEM_Real_Load_gfx10<op, ps#"_SGPR">;1223 def _SGPR_IMM_gfx10 : SMEM_Real_Load_gfx10<op, ps#"_SGPR_IMM">;1224}1225 1226class SMEM_Real_Store_gfx10<bits<8> op, SM_Pseudo ps> : SMEM_Real_gfx10<op, ps> {1227 bits<7> sdata;1228 1229 let sdst = ?;1230 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?);1231}1232 1233multiclass SM_Real_Stores_gfx10<bits<8> op> {1234 defvar ps = NAME;1235 defvar immPs = !cast<SM_Store_Pseudo>(ps#_IMM);1236 def _IMM_gfx10 : SMEM_Real_Store_gfx10 <op, immPs>;1237 1238 defvar sgprPs = !cast<SM_Store_Pseudo>(ps#_SGPR);1239 def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, sgprPs>;1240 1241 defvar sgprImmPs = !cast<SM_Store_Pseudo>(ps#_SGPR_IMM);1242 def _SGPR_IMM_gfx10 : SMEM_Real_Store_gfx10 <op, sgprImmPs>;1243}1244 1245defm S_LOAD_DWORD : SM_Real_Loads_gfx10<0x000>;1246defm S_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x001>;1247defm S_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x002>;1248defm S_LOAD_DWORDX8 : SM_Real_Loads_gfx10<0x003>;1249defm S_LOAD_DWORDX16 : SM_Real_Loads_gfx10<0x004>;1250 1251defm S_SCRATCH_LOAD_DWORD : SM_Real_Loads_gfx10<0x005>;1252defm S_SCRATCH_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x006>;1253defm S_SCRATCH_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x007>;1254 1255defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_gfx10<0x008>;1256defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x009>;1257defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x00a>;1258defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_gfx10<0x00b>;1259defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_gfx10<0x00c>;1260 1261defm S_STORE_DWORD : SM_Real_Stores_gfx10<0x010>;1262defm S_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x011>;1263defm S_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x012>;1264defm S_SCRATCH_STORE_DWORD : SM_Real_Stores_gfx10<0x015>;1265defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x016>;1266defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x017>;1267defm S_BUFFER_STORE_DWORD : SM_Real_Stores_gfx10<0x018>;1268defm S_BUFFER_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x019>;1269defm S_BUFFER_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x01a>;1270 1271def S_MEMREALTIME_gfx10 : SMEM_Real_gfx10<0x025, S_MEMREALTIME>;1272def S_MEMTIME_gfx10 : SMEM_Real_gfx10<0x024, S_MEMTIME>;1273def S_GL1_INV_gfx10 : SMEM_Real_gfx10<0x01f, S_GL1_INV>;1274def S_GET_WAVEID_IN_WORKGROUP_gfx10 : SMEM_Real_gfx10<0x02a, S_GET_WAVEID_IN_WORKGROUP>;1275def S_DCACHE_INV_gfx10 : SMEM_Real_gfx10<0x020, S_DCACHE_INV>;1276 1277def S_DCACHE_WB_gfx10 : SMEM_Real_gfx10<0x021, S_DCACHE_WB>;1278 1279multiclass SM_Real_Probe_gfx10<bits<8> op> {1280 defvar ps = NAME;1281 def _IMM_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_IMM)>;1282 def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR)>;1283 def _SGPR_IMM_gfx101284 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR_IMM)>;1285}1286 1287defm S_ATC_PROBE : SM_Real_Probe_gfx10 <0x26>;1288defm S_ATC_PROBE_BUFFER : SM_Real_Probe_gfx10 <0x27>;1289 1290class SMEM_Atomic_Real_gfx10 <bits<8> op, SM_Atomic_Pseudo ps>1291 : SMEM_Real_gfx10 <op, ps> {1292 1293 bits<7> sdata;1294 1295 let Constraints = ps.Constraints;1296 1297 let cpol{CPolBit.GLC} = ps.glc;1298 1299 let Inst{14} = !if(ps.has_dlc, cpol{CPolBit.DLC}, 0);1300 let Inst{12-6} = !if(ps.glc, sdst{6-0}, sdata{6-0});1301}1302 1303multiclass SM_Real_Atomics_gfx10<bits<8> op> {1304 defvar ps = NAME;1305 def _IMM_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>;1306 def _SGPR_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>;1307 def _SGPR_IMM_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM)>;1308 def _IMM_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>;1309 def _SGPR_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>;1310 def _SGPR_IMM_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM_RTN)>;1311}1312 1313defm S_BUFFER_ATOMIC_SWAP : SM_Real_Atomics_gfx10 <0x40>;1314defm S_BUFFER_ATOMIC_CMPSWAP : SM_Real_Atomics_gfx10 <0x41>;1315defm S_BUFFER_ATOMIC_ADD : SM_Real_Atomics_gfx10 <0x42>;1316defm S_BUFFER_ATOMIC_SUB : SM_Real_Atomics_gfx10 <0x43>;1317defm S_BUFFER_ATOMIC_SMIN : SM_Real_Atomics_gfx10 <0x44>;1318defm S_BUFFER_ATOMIC_UMIN : SM_Real_Atomics_gfx10 <0x45>;1319defm S_BUFFER_ATOMIC_SMAX : SM_Real_Atomics_gfx10 <0x46>;1320defm S_BUFFER_ATOMIC_UMAX : SM_Real_Atomics_gfx10 <0x47>;1321defm S_BUFFER_ATOMIC_AND : SM_Real_Atomics_gfx10 <0x48>;1322defm S_BUFFER_ATOMIC_OR : SM_Real_Atomics_gfx10 <0x49>;1323defm S_BUFFER_ATOMIC_XOR : SM_Real_Atomics_gfx10 <0x4a>;1324defm S_BUFFER_ATOMIC_INC : SM_Real_Atomics_gfx10 <0x4b>;1325defm S_BUFFER_ATOMIC_DEC : SM_Real_Atomics_gfx10 <0x4c>;1326 1327defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Real_Atomics_gfx10 <0x60>;1328defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_gfx10 <0x61>;1329defm S_BUFFER_ATOMIC_ADD_X2 : SM_Real_Atomics_gfx10 <0x62>;1330defm S_BUFFER_ATOMIC_SUB_X2 : SM_Real_Atomics_gfx10 <0x63>;1331defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Real_Atomics_gfx10 <0x64>;1332defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Real_Atomics_gfx10 <0x65>;1333defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Real_Atomics_gfx10 <0x66>;1334defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Real_Atomics_gfx10 <0x67>;1335defm S_BUFFER_ATOMIC_AND_X2 : SM_Real_Atomics_gfx10 <0x68>;1336defm S_BUFFER_ATOMIC_OR_X2 : SM_Real_Atomics_gfx10 <0x69>;1337defm S_BUFFER_ATOMIC_XOR_X2 : SM_Real_Atomics_gfx10 <0x6a>;1338defm S_BUFFER_ATOMIC_INC_X2 : SM_Real_Atomics_gfx10 <0x6b>;1339defm S_BUFFER_ATOMIC_DEC_X2 : SM_Real_Atomics_gfx10 <0x6c>;1340 1341defm S_ATOMIC_SWAP : SM_Real_Atomics_gfx10 <0x80>;1342defm S_ATOMIC_CMPSWAP : SM_Real_Atomics_gfx10 <0x81>;1343defm S_ATOMIC_ADD : SM_Real_Atomics_gfx10 <0x82>;1344defm S_ATOMIC_SUB : SM_Real_Atomics_gfx10 <0x83>;1345defm S_ATOMIC_SMIN : SM_Real_Atomics_gfx10 <0x84>;1346defm S_ATOMIC_UMIN : SM_Real_Atomics_gfx10 <0x85>;1347defm S_ATOMIC_SMAX : SM_Real_Atomics_gfx10 <0x86>;1348defm S_ATOMIC_UMAX : SM_Real_Atomics_gfx10 <0x87>;1349defm S_ATOMIC_AND : SM_Real_Atomics_gfx10 <0x88>;1350defm S_ATOMIC_OR : SM_Real_Atomics_gfx10 <0x89>;1351defm S_ATOMIC_XOR : SM_Real_Atomics_gfx10 <0x8a>;1352defm S_ATOMIC_INC : SM_Real_Atomics_gfx10 <0x8b>;1353defm S_ATOMIC_DEC : SM_Real_Atomics_gfx10 <0x8c>;1354 1355defm S_ATOMIC_SWAP_X2 : SM_Real_Atomics_gfx10 <0xa0>;1356defm S_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_gfx10 <0xa1>;1357defm S_ATOMIC_ADD_X2 : SM_Real_Atomics_gfx10 <0xa2>;1358defm S_ATOMIC_SUB_X2 : SM_Real_Atomics_gfx10 <0xa3>;1359defm S_ATOMIC_SMIN_X2 : SM_Real_Atomics_gfx10 <0xa4>;1360defm S_ATOMIC_UMIN_X2 : SM_Real_Atomics_gfx10 <0xa5>;1361defm S_ATOMIC_SMAX_X2 : SM_Real_Atomics_gfx10 <0xa6>;1362defm S_ATOMIC_UMAX_X2 : SM_Real_Atomics_gfx10 <0xa7>;1363defm S_ATOMIC_AND_X2 : SM_Real_Atomics_gfx10 <0xa8>;1364defm S_ATOMIC_OR_X2 : SM_Real_Atomics_gfx10 <0xa9>;1365defm S_ATOMIC_XOR_X2 : SM_Real_Atomics_gfx10 <0xaa>;1366defm S_ATOMIC_INC_X2 : SM_Real_Atomics_gfx10 <0xab>;1367defm S_ATOMIC_DEC_X2 : SM_Real_Atomics_gfx10 <0xac>;1368 1369multiclass SM_Real_Discard_gfx10<bits<8> op> {1370 defvar ps = NAME;1371 def _IMM_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_IMM)>;1372 def _SGPR_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR)>;1373 def _SGPR_IMM_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR_IMM)>;1374}1375 1376defm S_DCACHE_DISCARD : SM_Real_Discard_gfx10 <0x28>;1377defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_gfx10 <0x29>;1378 1379def SMInfoTable : GenericTable {1380 let FilterClass = "SM_Real";1381 let CppTypeName = "SMInfo";1382 let Fields = ["Opcode", "is_buffer"];1383 1384 let PrimaryKey = ["Opcode"];1385 let PrimaryKeyName = "getSMEMOpcodeHelper";1386}1387 1388//===----------------------------------------------------------------------===//1389// GFX11.1390//===----------------------------------------------------------------------===//1391 1392class SMEM_Real_gfx11<bits<8> op, SM_Pseudo ps, string opName = ps.Mnemonic> :1393 SMEM_Real_10Plus_common<op, ps, opName, SIEncodingFamily.GFX11,1394 SGPR_NULL_gfx11plus> {1395 let AssemblerPredicate = isGFX11Only;1396 let DecoderNamespace = "GFX11";1397 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, 0);1398 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, 0);1399}1400 1401class SMEM_Real_Load_gfx11<bits<8> op, string ps, string opName> :1402 SMEM_Real_gfx11<op, !cast<SM_Pseudo>(ps), opName>;1403 1404multiclass SM_Real_Loads_gfx11<bits<8> op, string ps> {1405 defvar opName = !tolower(NAME);1406 def _IMM_gfx11 : SMEM_Real_Load_gfx11<op, ps#"_IMM", opName>;1407 def _SGPR_gfx11 : SMEM_Real_Load_gfx11<op, ps#"_SGPR", opName>;1408 def _SGPR_IMM_gfx11 : SMEM_Real_Load_gfx11<op, ps#"_SGPR_IMM", opName>;1409 def : AMDGPUMnemonicAlias<!cast<SM_Pseudo>(ps#"_IMM").Mnemonic, opName> {1410 let AssemblerPredicate = isGFX11Plus;1411 }1412}1413 1414defm S_LOAD_B32 : SM_Real_Loads_gfx11<0x000, "S_LOAD_DWORD">;1415defm S_LOAD_B64 : SM_Real_Loads_gfx11<0x001, "S_LOAD_DWORDX2">;1416defm S_LOAD_B128 : SM_Real_Loads_gfx11<0x002, "S_LOAD_DWORDX4">;1417defm S_LOAD_B256 : SM_Real_Loads_gfx11<0x003, "S_LOAD_DWORDX8">;1418defm S_LOAD_B512 : SM_Real_Loads_gfx11<0x004, "S_LOAD_DWORDX16">;1419 1420defm S_BUFFER_LOAD_B32 : SM_Real_Loads_gfx11<0x008, "S_BUFFER_LOAD_DWORD">;1421defm S_BUFFER_LOAD_B64 : SM_Real_Loads_gfx11<0x009, "S_BUFFER_LOAD_DWORDX2">;1422defm S_BUFFER_LOAD_B128 : SM_Real_Loads_gfx11<0x00a, "S_BUFFER_LOAD_DWORDX4">;1423defm S_BUFFER_LOAD_B256 : SM_Real_Loads_gfx11<0x00b, "S_BUFFER_LOAD_DWORDX8">;1424defm S_BUFFER_LOAD_B512 : SM_Real_Loads_gfx11<0x00c, "S_BUFFER_LOAD_DWORDX16">;1425 1426def S_GL1_INV_gfx11 : SMEM_Real_gfx11<0x020, S_GL1_INV>;1427def S_DCACHE_INV_gfx11 : SMEM_Real_gfx11<0x021, S_DCACHE_INV>;1428 1429class SMEM_Real_Store_gfx11 <bits<8> op, SM_Pseudo ps> : SMEM_Real_gfx11<op, ps> {1430 // encoding1431 bits<7> sdata;1432 1433 let sdst = ?;1434 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?);1435}1436 1437multiclass SM_Real_Probe_gfx11<bits<8> op> {1438 defvar ps = NAME;1439 def _IMM_gfx11 : SMEM_Real_Store_gfx11 <op, !cast<SM_Probe_Pseudo>(ps#_IMM)>;1440 def _SGPR_gfx11 : SMEM_Real_Store_gfx11 <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>;1441 def _SGPR_IMM_gfx111442 : SMEM_Real_Store_gfx11 <op, !cast<SM_Probe_Pseudo>(ps#_SGPR_IMM)>;1443}1444 1445defm S_ATC_PROBE : SM_Real_Probe_gfx11 <0x22>;1446defm S_ATC_PROBE_BUFFER : SM_Real_Probe_gfx11 <0x23>;1447 1448//===----------------------------------------------------------------------===//1449// GFX12.1450//===----------------------------------------------------------------------===//1451 1452class SMEM_Real_gfx12Plus<bits<6> op, SM_Pseudo ps, string opName,1453 int subtarget, RegisterWithSubRegs sgpr_null> :1454 SM_Real<ps, opName>, SIMCInstr<ps.PseudoInstr, subtarget>, Enc64 {1455 1456 let Inst{18-13} = op;1457 let Inst{31-26} = 0x3d;1458 1459 let Inst{55-32} = !if(ps.has_offset, offset{23-0}, !if(ps.has_soffset, 0, ?));1460 let Inst{63-57} = !if(ps.has_soffset, soffset{6-0},1461 !if(ps.has_offset, sgpr_null.HWEncoding{6-0}, ?));1462}1463 1464class SMEM_Real_gfx12<bits<6> op, SM_Pseudo ps, string opName = ps.Mnemonic> :1465 SMEM_Real_gfx12Plus<op, ps, opName, SIEncodingFamily.GFX12,1466 SGPR_NULL_gfx11plus> {1467 let AssemblerPredicate = isGFX12Plus;1468 let DecoderNamespace = "GFX12";1469 1470 let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?);1471 let Inst{12-6} = !if(ps.has_sdst, sdst{6-0}, ?);1472}1473 1474class SMEM_Real_Prefetch_gfx12<bits<6> op, SM_Pseudo ps> :1475 SMEM_Real_gfx12<op, ps> {1476 bits<7> sdata; // Only 5 bits of sdata are supported.1477 1478 let sdst = ?;1479 let Inst{12-11} = 0; // Unused sdata bits.1480 let Inst{10-6} = !if(ps.has_sdst, sdata{4-0}, ?);1481}1482 1483class SMEM_Real_Load_gfx12<bits<6> op, string ps, string opName, OffsetMode offsets> :1484 SMEM_Real_gfx12<op, !cast<SM_Pseudo>(ps # offsets.Variant), opName> {1485 RegisterClass BaseClass = !cast<SM_Load_Pseudo>(ps # offsets.Variant).BaseClass;1486 let InOperandList = !con((ins BaseClass:$sbase), offsets.Ins, (ins CPol:$cpol));1487 1488 let Inst{20} = cpol{CPolBit.NV}; // non-volatile1489 let Inst{22-21} = cpol{4-3}; // scope1490 let Inst{24-23} = cpol{1-0}; // th - only lower 2 bits are supported1491 let Inst{56} = cpol{CPolBit.SCAL}; // scale offset1492}1493 1494multiclass SM_Real_Loads_gfx12<bits<6> op, string ps = NAME> {1495 defvar opName = !tolower(NAME);1496 def _IMM_gfx12 : SMEM_Real_Load_gfx12<op, ps, opName, IMM_Offset>;1497 def _SGPR_IMM_gfx12 : SMEM_Real_Load_gfx12<op, ps, opName, SGPR_IMM_OptOffset>;1498}1499 1500defm S_LOAD_B32 : SM_Real_Loads_gfx12<0x00, "S_LOAD_DWORD">;1501defm S_LOAD_B64 : SM_Real_Loads_gfx12<0x01, "S_LOAD_DWORDX2">;1502defm S_LOAD_B96 : SM_Real_Loads_gfx12<0x05, "S_LOAD_DWORDX3">;1503defm S_LOAD_B128 : SM_Real_Loads_gfx12<0x02, "S_LOAD_DWORDX4">;1504defm S_LOAD_B256 : SM_Real_Loads_gfx12<0x03, "S_LOAD_DWORDX8">;1505defm S_LOAD_B512 : SM_Real_Loads_gfx12<0x04, "S_LOAD_DWORDX16">;1506 1507defm S_LOAD_I8 : SM_Real_Loads_gfx12<0x08>;1508defm S_LOAD_U8 : SM_Real_Loads_gfx12<0x09>;1509defm S_LOAD_I16 : SM_Real_Loads_gfx12<0x0a>;1510defm S_LOAD_U16 : SM_Real_Loads_gfx12<0x0b>;1511 1512defm S_BUFFER_LOAD_B32 : SM_Real_Loads_gfx12<0x10, "S_BUFFER_LOAD_DWORD">;1513defm S_BUFFER_LOAD_B64 : SM_Real_Loads_gfx12<0x11, "S_BUFFER_LOAD_DWORDX2">;1514defm S_BUFFER_LOAD_B96 : SM_Real_Loads_gfx12<0x15, "S_BUFFER_LOAD_DWORDX3">;1515defm S_BUFFER_LOAD_B128 : SM_Real_Loads_gfx12<0x12, "S_BUFFER_LOAD_DWORDX4">;1516defm S_BUFFER_LOAD_B256 : SM_Real_Loads_gfx12<0x13, "S_BUFFER_LOAD_DWORDX8">;1517defm S_BUFFER_LOAD_B512 : SM_Real_Loads_gfx12<0x14, "S_BUFFER_LOAD_DWORDX16">;1518 1519defm S_BUFFER_LOAD_I8 : SM_Real_Loads_gfx12<0x18>;1520defm S_BUFFER_LOAD_U8 : SM_Real_Loads_gfx12<0x19>;1521defm S_BUFFER_LOAD_I16 : SM_Real_Loads_gfx12<0x1a>;1522defm S_BUFFER_LOAD_U16 : SM_Real_Loads_gfx12<0x1b>;1523 1524def S_DCACHE_INV_gfx12 : SMEM_Real_gfx12<0x021, S_DCACHE_INV>;1525 1526def S_PREFETCH_INST_gfx12 : SMEM_Real_Prefetch_gfx12<0x24, S_PREFETCH_INST>;1527def S_PREFETCH_INST_PC_REL_gfx12 : SMEM_Real_Prefetch_gfx12<0x25, S_PREFETCH_INST_PC_REL>;1528def S_PREFETCH_DATA_gfx12 : SMEM_Real_Prefetch_gfx12<0x26, S_PREFETCH_DATA>;1529def S_BUFFER_PREFETCH_DATA_gfx12 : SMEM_Real_Prefetch_gfx12<0x27, S_BUFFER_PREFETCH_DATA>;1530def S_PREFETCH_DATA_PC_REL_gfx12 : SMEM_Real_Prefetch_gfx12<0x28, S_PREFETCH_DATA_PC_REL>;1531 1532multiclass SMEM_Real_Probe_gfx12<bits<6> op> {1533 defvar ps = NAME;1534 def _IMM_gfx12 : SMEM_Real_Prefetch_gfx12<op, !cast<SM_Probe_Pseudo>(ps#_IMM)>;1535 def _SGPR_IMM_gfx12 : SMEM_Real_Prefetch_gfx12<op, !cast<SM_Probe_Pseudo>(ps#_SGPR_OPT_IMM)>;1536}1537 1538defm S_ATC_PROBE : SMEM_Real_Probe_gfx12<0x22>;1539defm S_ATC_PROBE_BUFFER : SMEM_Real_Probe_gfx12<0x23>;1540