6399 lines · plain
1//=-- SMEInstrFormats.td - AArch64 SME Instruction classes -*- tablegen -*--=//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// AArch64 Scalable Matrix Extension (SME) Instruction Class Definitions.10//11//===----------------------------------------------------------------------===//12 13def imm_to_tile8 : ComplexPattern<i32, 1, "ImmToReg<AArch64::ZAB0, 0>", []>;14def imm_to_tile16 : ComplexPattern<i32, 1, "ImmToReg<AArch64::ZAH0, 1>", []>;15def imm_to_tile32 : ComplexPattern<i32, 1, "ImmToReg<AArch64::ZAS0, 3>", []>;16def imm_to_tile64 : ComplexPattern<i32, 1, "ImmToReg<AArch64::ZAD0, 7>", []>;17def imm_to_tile128 : ComplexPattern<i32, 1, "ImmToReg<AArch64::ZAQ0, 15>", []>;18def imm_to_zt : ComplexPattern<i32, 1, "ImmToReg<AArch64::ZT0, 0>", []>;19 20def tileslice8 : ComplexPattern<i32 , 2, "SelectSMETileSlice<15, 1>", []>;21def tileslice16 : ComplexPattern<i32 , 2, "SelectSMETileSlice<7, 1>", []>;22def tileslice32 : ComplexPattern<i32 , 2, "SelectSMETileSlice<3, 1>", []>;23def tileslice64 : ComplexPattern<i32 , 2, "SelectSMETileSlice<1, 1>", []>;24def tileslice128 : ComplexPattern<i32 , 2, "SelectSMETileSlice<0, 1>", []>; // nop25 26def tileslicerange3s2 : ComplexPattern<i32, 2, "SelectSMETileSlice<14, 2>", []>;27def tileslicerange2s2 : ComplexPattern<i32, 2, "SelectSMETileSlice<6, 2>", []>;28def tileslicerange1s2 : ComplexPattern<i32, 2, "SelectSMETileSlice<2, 2>", []>;29def tileslicerange0s2 : ComplexPattern<i32, 2, "SelectSMETileSlice<0, 2>", []>;30 31def tileslicerange2s4 : ComplexPattern<i32, 2, "SelectSMETileSlice<12, 4>", []>;32def tileslicerange1s4 : ComplexPattern<i32, 2, "SelectSMETileSlice<4, 4>", []>;33def tileslicerange0s4 : ComplexPattern<i32, 2, "SelectSMETileSlice<0, 4>", []>;34 35let WantsRoot = true in36def am_sme_indexed_b4 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<0, 15>">;37 38// The FORM_TRANSPOSED_REG_TUPLE pseudos defined below are intended to39// improve register allocation for intrinsics which use strided and40// contiguous multi-vector registers, avoiding unnecessary copies.41// The SMEPeepholeOpt pass will replace a REG_SEQUENCE instruction with the42// FORM_TRANSPOSED_REG_TUPLE pseudo if the operands are copies where the43// source register is in the StridedOrContiguous class. The operands in the44// sequence must all have the same subreg index.45// The pseudo is then used to provide a hint to the register allocator46// suggesting a contigious multi-vector register which matches the47// subregister sequence used by the operands.48 49def FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO :50 Pseudo<(outs ZPR2:$tup),51 (ins ZPR:$zn0, ZPR:$zn1), []>, Sched<[]>{52 let hasSideEffects = 0;53}54 55def FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO :56 Pseudo<(outs ZPR4:$tup),57 (ins ZPR:$zn0, ZPR:$zn1, ZPR:$zn2, ZPR:$zn3), []>, Sched<[]>{58 let hasSideEffects = 0;59}60 61def SDTZALoadStore : SDTypeProfile<0, 3, [SDTCisInt<0>, SDTCisPtrTy<1>, SDTCisInt<2>]>;62// SME ZA loads and stores63def AArch64SMELdr : SDNode<"AArch64ISD::SME_ZA_LDR", SDTZALoadStore,64 [SDNPHasChain, SDNPSideEffect, SDNPMayLoad]>;65def AArch64SMEStr : SDNode<"AArch64ISD::SME_ZA_STR", SDTZALoadStore,66 [SDNPHasChain, SDNPSideEffect, SDNPMayStore]>;67 68//===----------------------------------------------------------------------===//69// SME Pseudo Classes70//===----------------------------------------------------------------------===//71 72def getSMEPseudoMap : InstrMapping {73 let FilterClass = "SMEPseudo2Instr";74 let RowFields = ["PseudoName"];75 let ColFields = ["IsInstr"];76 let KeyCol = ["0"];77 let ValueCols = [["1"]];78}79 80class SMEPseudo2Instr<string name, bit instr> {81 string PseudoName = name;82 bit IsInstr = instr;83}84 85class sme_outer_product_pseudo<ZPRRegOp zpr_ty, SMEMatrixTypeEnum za_flag>86 : Pseudo<(outs), (ins i32imm:$tile, PPR3bAny:$pn, PPR3bAny:$pm,87 zpr_ty:$zn, zpr_ty:$zm), []>,88 Sched<[]> {89 // Translated to the actual instructions in AArch64ISelLowering.cpp90 let SMEMatrixType = za_flag;91 let usesCustomInserter = 1;92 let mayLoad = 1;93 let mayStore = 1;94}95 96class sme_sparse_outer_product_pseudo<RegisterOperand zn_ty, RegisterOperand zm_ty, SMEMatrixTypeEnum za_flag>97 : Pseudo<(outs), (ins i32imm:$tile, zn_ty:$zn, zm_ty:$zm, ZK:$zk, i32imm:$idx), []>,98 Sched<[]> {99 // Translated to the actual instructions in AArch64ISelLowering.cpp100 let SMEMatrixType = za_flag;101 let usesCustomInserter = 1;102 let mayLoad = 1;103 let mayStore = 1;104}105 106class sme2_quarter_tile_outer_product_pseudo<RegisterOperand zn_ty, RegisterOperand zm_ty, SMEMatrixTypeEnum za_flag>107 : Pseudo<(outs), (ins i32imm:$tile,108 zn_ty:$zn, zm_ty:$zm), []>,109 Sched<[]> {110 // Translated to the actual instructions in AArch64ISelLowering.cpp111 let SMEMatrixType = za_flag;112 let usesCustomInserter = 1;113}114 115class sme2_za_array_2op_multi_single_pseudo<string name, Operand index_ty, RegisterOperand multi_vector_ty,116 ZPRRegOp zpr_ty, SMEMatrixTypeEnum za_flag>117 : SMEPseudo2Instr<name, 0>,118 Pseudo<(outs), (ins MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), []> {119 let SMEMatrixType = za_flag;120 let usesCustomInserter = 1;121 let mayLoad = 1;122 let mayStore = 1;123}124 125class sme2_za_array_2op_multi_multi_pseudo<string name, Operand index_ty, RegisterOperand multi_vector_ty,126 SMEMatrixTypeEnum za_flag>127 : SMEPseudo2Instr<name, 0>,128 Pseudo<(outs), (ins MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), []> {129 let SMEMatrixType = za_flag;130 let usesCustomInserter = 1;131 let mayLoad = 1;132 let mayStore = 1;133}134 135class sme2_za_array_2op_multi_index_pseudo<string name, Operand index_ty, RegisterOperand multi_vector_ty,136 ZPRRegOp zpr_ty, Operand imm_ty, SMEMatrixTypeEnum za_flag>137 : SMEPseudo2Instr<name, 0>,138 Pseudo<(outs), (ins MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm, imm_ty:$i), []> {139 let SMEMatrixType = za_flag;140 let usesCustomInserter = 1;141 let mayLoad = 1;142 let mayStore = 1;143}144 145class sme2_move_to_za_pseudo<string name, Operand imm_ty, RegisterOperand multi_vector_ty, SMEMatrixTypeEnum za_flag>146 : SMEPseudo2Instr<name, 0>,147 Pseudo<(outs), (ins MatrixIndexGPR32Op8_11:$Rs, imm_ty:$imm, multi_vector_ty:$Zn), []> {148 let SMEMatrixType = za_flag;149 let usesCustomInserter = 1;150 let mayStore = 1;151}152 153class sme2_move_to_tile_pseudo<string name, Operand tile_imm, Operand imm_ty, RegisterOperand multi_vector_ty, SMEMatrixTypeEnum za_flag>154 : SMEPseudo2Instr<name, 0>,155 Pseudo<(outs), (ins tile_imm:$tile, MatrixIndexGPR32Op12_15:$Rs, imm_ty:$imm, multi_vector_ty:$Zn), []> {156 let SMEMatrixType = za_flag;157 let usesCustomInserter = 1;158 let mayStore = 1;159}160 161class sem2p1_zero_matrix_pseudo<string name, Operand index_ty, SMEMatrixTypeEnum za_flag>162 : SMEPseudo2Instr<name, 0>,163 Pseudo<(outs), (ins MatrixIndexGPR32Op8_11:$Rs, index_ty:$imm), []> {164 let SMEMatrixType = za_flag;165 let usesCustomInserter = 1;166 let mayStore = 1;167}168 169class sme2_movez_to_tile_pseudo<string name, Operand tile_imm, Operand imm_ty, RegisterOperand vector_ty, SMEMatrixTypeEnum za_flag>170 : SMEPseudo2Instr<name, 0>,171 Pseudo<(outs vector_ty:$Zn), (ins tile_imm:$tile, MatrixIndexGPR32Op12_15:$Rs, imm_ty:$imm), []> {172 let SMEMatrixType = za_flag;173 let usesCustomInserter = 1;174 let mayLoad = 1;175 let mayStore = 1;176}177 178class sme2_movaz_array_to_tile_pseudo<string name, Operand index_ty, RegisterOperand multi_vector_ty,179 SMEMatrixTypeEnum za_flag>180 : SMEPseudo2Instr<name, 0>,181 Pseudo<(outs multi_vector_ty:$Zd), (ins MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm3), []> {182 let SMEMatrixType = za_flag;183 let usesCustomInserter = 1;184 let mayLoad = 1;185 let mayStore = 1;186}187 188//===----------------------------------------------------------------------===//189// SME pattern match helpers.190//===----------------------------------------------------------------------===//191 192class SME2_ZA_TwoOp_Multi_Single_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty,193 ValueType vt, ComplexPattern tileslice>194 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn, vt:$Zm),195 (!cast<Instruction>(name # _PSEUDO) $base, $offset, vt:$Zn, zpr_ty:$Zm)>;196 197 198class SME2_ZA_TwoOp_VG2_Multi_Single_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty,199 ValueType vt, ComplexPattern tileslice>200 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zm),201 (!cast<Instruction>(name # _PSEUDO) $base, $offset, (REG_SEQUENCE ZPR2, vt:$Zn1, zsub0, vt:$Zn2, zsub1),202 zpr_ty:$Zm)>;203class SME2_ZA_TwoOp_VG4_Multi_Single_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty,204 ValueType vt, ComplexPattern tileslice>205 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)),206 vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4, vt:$Zm),207 (!cast<Instruction>(name # _PSEUDO) $base, $offset,208 (REG_SEQUENCE ZPR4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3),209 zpr_ty:$Zm)>;210 211class SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ValueType vt, ComplexPattern tileslice>212 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zm1, vt:$Zm2),213 (!cast<Instruction>(name # _PSEUDO) $base, $offset,214 (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1),215 (REG_SEQUENCE ZPR2Mul2, vt:$Zm1, zsub0, vt:$Zm2, zsub1))>;216 217class SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ValueType vt, ComplexPattern tileslice>218 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)),219 vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4, vt:$Zm1, vt:$Zm2, vt:$Zm3, vt:$Zm4),220 (!cast<Instruction>(name # _PSEUDO) $base, $offset,221 (REG_SEQUENCE ZPR4Mul4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3),222 (REG_SEQUENCE ZPR4Mul4, vt:$Zm1, zsub0, vt:$Zm2, zsub1, vt:$Zm3, zsub2, vt:$Zm4, zsub3))>;223 224class SME2_ZA_TwoOp_Multi_Index_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, ValueType vt,225 Operand imm_ty, ComplexPattern tileslice>226 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn, vt:$Zm, (i32 imm_ty:$i)),227 (!cast<Instruction>(name # _PSEUDO) $base, $offset, vt:$Zn, zpr_ty:$Zm, (i32 imm_ty:$i))>;228 229 230class SME2_ZA_TwoOp_VG2_Multi_Index_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, ValueType vt,231 Operand imm_ty, ComplexPattern tileslice>232 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zm, (i32 imm_ty:$i)),233 (!cast<Instruction>(name # _PSEUDO) $base, $offset,234 (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), zpr_ty:$Zm, imm_ty:$i)>;235 236class SME2_ZA_TwoOp_VG4_Multi_Index_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, ValueType vt,237 Operand imm_ty, ComplexPattern tileslice>238 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)),239 vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4, vt:$Zm, (i32 imm_ty:$i)),240 (!cast<Instruction>(name # _PSEUDO) $base, $offset,241 (REG_SEQUENCE ZPR4Mul4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3),242 zpr_ty:$Zm, imm_ty:$i)>;243 244class SME2_Sat_Shift_VG2_Pat<string name, SDPatternOperator intrinsic, ValueType out_vt, ValueType in_vt, Operand imm_ty>245 : Pat<(out_vt (intrinsic in_vt:$Zn1, in_vt:$Zn2, (i32 imm_ty:$i))),246 (!cast<Instruction>(name) (REG_SEQUENCE ZPR2Mul2, in_vt:$Zn1, zsub0, in_vt:$Zn2, zsub1), imm_ty:$i)>;247 248class SME2_Sat_Shift_VG4_Pat<string name, SDPatternOperator intrinsic, ValueType out_vt, ValueType in_vt, Operand imm_ty>249 : Pat<(out_vt (intrinsic in_vt:$Zn1, in_vt:$Zn2, in_vt:$Zn3, in_vt:$Zn4, (i32 imm_ty:$i))),250 (!cast<Instruction>(name) (REG_SEQUENCE ZPR4Mul4, in_vt:$Zn1, zsub0, in_vt:$Zn2, zsub1, in_vt:$Zn3, zsub2, in_vt:$Zn4, zsub3),251 imm_ty:$i)>;252 253class SME2_Cvt_VG4_Pat<string name, SDPatternOperator intrinsic, ValueType out_vt, ValueType in_vt>254 : Pat<(out_vt (intrinsic in_vt:$Zn1, in_vt:$Zn2, in_vt:$Zn3, in_vt:$Zn4)),255 (!cast<Instruction>(name) (REG_SEQUENCE ZPR4Mul4, in_vt:$Zn1, zsub0, in_vt:$Zn2, zsub1, in_vt:$Zn3, zsub2, in_vt:$Zn4, zsub3))>;256 257class SME2_ZA_VG1x2_Multi_Pat<string name, SDPatternOperator intrinsic, ValueType vt, Operand index_ty, ComplexPattern tileslice>258 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2),259 (!cast<Instruction>(name # _PSEUDO) $base, $offset, (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1))>;260 261class SME2_ZA_VG1x4_Multi_Pat<string name, SDPatternOperator intrinsic, ValueType vt, Operand index_ty, ComplexPattern tileslice>262 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4),263 (!cast<Instruction>(name # _PSEUDO) $base, $offset, (REG_SEQUENCE ZPR4Mul4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3))>;264 265class SME2_Tile_VG2_Multi_Pat<string name, SDPatternOperator intrinsic, Operand tile_imm, ValueType vt, Operand index_ty, ComplexPattern tileslice>266 : Pat<(intrinsic tile_imm:$tile, (i32 (tileslice MatrixIndexGPR32Op12_15:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2),267 (!cast<Instruction>(name # _PSEUDO) $tile, $base, $offset, (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1))>;268 269class SME2_Tile_VG4_Multi_Pat<string name, SDPatternOperator intrinsic, Operand tile_imm, ValueType vt, Operand index_ty, ComplexPattern tileslice>270 : Pat<(intrinsic tile_imm:$tile, (i32 (tileslice MatrixIndexGPR32Op12_15:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4),271 (!cast<Instruction>(name # _PSEUDO) $tile, $base, $offset, (REG_SEQUENCE ZPR4Mul4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3))>;272 273class SME2_Zero_Matrix_Pat<string name, SDPatternOperator intrinsic, Operand offset_ty, ComplexPattern tileslice>274 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, offset_ty:$offset))),275 (!cast<Instruction>(name) $base, $offset)>;276 277class SME2_Tile_Movaz_Pat<string name, SDPatternOperator intrinsic, ValueType out_vt, Operand tile_imm, Operand index_ty, ComplexPattern tileslice>278 : Pat<(out_vt (intrinsic tile_imm:$tile, (i32 (tileslice MatrixIndexGPR32Op12_15:$base, index_ty:$offset)))),279 (!cast<Instruction>(name # _PSEUDO) $tile, $base, $offset)>;280 281class SME2_ZA_Tile_Vec_Single_Single_Pat<string name, SDPatternOperator intrinsic, Operand imm_ty, ValueType vt>282 : Pat<(intrinsic imm_ty:$tile, vt:$Zn, vt:$Zm),283 (!cast<Instruction>(name # _PSEUDO) $tile, $Zn, $Zm)>;284 285class SME2_ZA_Tile_Vec_Multi_Pat<string name, SDPatternOperator intrinsic, Operand imm_ty, ValueType vt>286 : Pat<(intrinsic imm_ty:$tile, vt:$Zn, vt:$Zm1, vt:$Zm2),287 (!cast<Instruction>(name # _PSEUDO) $tile, $Zn, (REG_SEQUENCE ZPR2Mul2, vt:$Zm1, zsub0, vt:$Zm2, zsub1))>;288 289class SME2_ZA_Tile_Vec_Multi_Single_Pat<string name, SDPatternOperator intrinsic, Operand imm_ty, ValueType vt>290 : Pat<(intrinsic imm_ty:$tile, vt:$Zn1, vt:$Zn2, vt:$Zm),291 (!cast<Instruction>(name # _PSEUDO) $tile, (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), $Zm)>;292 293class SME2_ZA_Tile_Vec_Multi_Multi_Pat<string name, SDPatternOperator intrinsic, Operand imm_ty, ValueType vt>294 : Pat<(intrinsic imm_ty:$tile, vt:$Zn1, vt:$Zn2, vt:$Zm1, vt:$Zm2),295 (!cast<Instruction>(name # _PSEUDO) $tile, (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), (REG_SEQUENCE ZPR2Mul2, vt:$Zm1, zsub0, vt:$Zm2, zsub1))>;296 297class SME2_ZA_TMOP_Pat<string name, SDPatternOperator intrinsic, Operand tile_imm, ValueType vt>298 : Pat<(intrinsic tile_imm:$tile, vt:$Zn1, vt:$Zn2, vt:$Zm, nxv16i8:$Zk, timm32_0_3:$idx),299 (!cast<Instruction>(name # _PSEUDO) $tile, (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), $Zm, $Zk, $idx)>;300 301 302//===----------------------------------------------------------------------===//303// SME pattern match helpers.304//===----------------------------------------------------------------------===//305 306class SME_ZA_Tile_TwoPred_TwoVec_Pat<string name, SDPatternOperator intrinsic, Operand imm_ty, ValueType pg_ty, ValueType vt>307 : Pat<(intrinsic imm_ty:$tile, (pg_ty PPR3bAny:$Pn), (pg_ty PPR3bAny:$Pm), vt:$Zn, vt:$Zm),308 (!cast<Instruction>(name # _PSEUDO) $tile, $Pn, $Pm, $Zn, $Zm)>;309 310 311//===----------------------------------------------------------------------===//312// SME smstart/smstop313//===----------------------------------------------------------------------===//314 315// SME defines three pstate fields to set or clear PSTATE.SM, PSTATE.ZA, or316// both fields:317//318// MSR SVCRSM, #<imm1>319// MSR SVCRZA, #<imm1>320// MSR SVCRSMZA, #<imm1>321//322// It's tricky to using the existing pstate operand defined in323// AArch64SystemOperands.td since it only encodes 5 bits including op1;op2,324// when these fields are also encoded in CRm[3:1].325def MSRpstatesvcrImm1326 : PstateWriteSimple<(ins svcr_op:$pstatefield, timm0_1:$imm), "msr",327 "\t$pstatefield, $imm">,328 Sched<[WriteSys]> {329 bits<3> pstatefield;330 bit imm;331 let Inst{18-16} = 0b011; // op1332 let Inst{11-9} = pstatefield;333 let Inst{8} = imm;334 let Inst{7-5} = 0b011; // op2335 let hasPostISelHook = 1;336}337 338def : InstAlias<"smstart", (MSRpstatesvcrImm1 0b011, 0b1)>;339def : InstAlias<"smstart sm", (MSRpstatesvcrImm1 0b001, 0b1)>;340def : InstAlias<"smstart za", (MSRpstatesvcrImm1 0b010, 0b1)>;341 342def : InstAlias<"smstop", (MSRpstatesvcrImm1 0b011, 0b0)>;343def : InstAlias<"smstop sm", (MSRpstatesvcrImm1 0b001, 0b0)>;344def : InstAlias<"smstop za", (MSRpstatesvcrImm1 0b010, 0b0)>;345 346 347//===----------------------------------------------------------------------===//348// SME Outer Products349//===----------------------------------------------------------------------===//350 351class sme_fp_outer_product_inst<bit S, bits<2> sz, bits<2> op, MatrixTileOperand za_ty,352 ZPRRegOp zpr_ty, string mnemonic>353 : I<(outs za_ty:$ZAda),354 (ins za_ty:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn, zpr_ty:$Zm),355 mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn, $Zm",356 "", []>,357 Sched<[]> {358 bits<5> Zm;359 bits<3> Pm;360 bits<3> Pn;361 bits<5> Zn;362 let Inst{31-25} = 0b1000000;363 let Inst{24} = op{1};364 let Inst{23} = 0b1;365 let Inst{22-21} = sz;366 let Inst{20-16} = Zm;367 let Inst{15-13} = Pm;368 let Inst{12-10} = Pn;369 let Inst{9-5} = Zn;370 let Inst{4} = S;371 let Inst{3} = op{0};372 373 let Constraints = "$ZAda = $_ZAda";374}375 376multiclass sme_outer_product_fp32<bit S, bits<2> sz, ZPRRegOp zpr_ty, string mnemonic, SDPatternOperator op> {377 def NAME : sme_fp_outer_product_inst<S, sz, 0b00, TileOp32, zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1> {378 bits<2> ZAda;379 let Inst{1-0} = ZAda;380 let Inst{2} = 0b0;381 }382 383 def NAME # _PSEUDO : sme_outer_product_pseudo<zpr_ty, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>;384 385 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_3, nxv4i1, nxv4f32>;386}387 388multiclass sme2_fp8_fmopa_za32<string mnemonic, SDPatternOperator intrinsic> {389 def NAME : sme_fp_outer_product_inst<0, 0b01, 0b00, TileOp32, ZPR8, mnemonic>, SMEPseudo2Instr<NAME, 1> {390 bits<2> ZAda;391 let Inst{1-0} = ZAda;392 let Inst{2} = 0b0;393 394 let Uses = [FPMR, FPCR];395 }396 397 let mayStore = 1, mayLoad = 1 in398 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR8, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>;399 400 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, intrinsic, timm32_0_3, nxv16i1, nxv16i8>;401}402 403multiclass sme_outer_product_fp64<bit S, string mnemonic, SDPatternOperator op> {404 def NAME : sme_fp_outer_product_inst<S, 0b10, 0b00, TileOp64, ZPR64, mnemonic>, SMEPseudo2Instr<NAME, 1> {405 bits<3> ZAda;406 let Inst{2-0} = ZAda;407 }408 409 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR64, SMEMatrixTileD>, SMEPseudo2Instr<NAME, 0>;410 411 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_7, nxv2i1, nxv2f64>;412}413 414multiclass sme2_fp8_fmopa_za16<string mnemonic, SDPatternOperator intrinsic> {415 def NAME : sme_fp_outer_product_inst<0, {0, 0b1}, 0b01, TileOp16, ZPR8, mnemonic>, SMEPseudo2Instr<NAME, 1> {416 bits<1> ZAda;417 let Inst{2-1} = 0b00;418 let Inst{0} = ZAda;419 420 let Uses = [FPMR, FPCR];421 }422 423 let mayStore = 1, mayLoad = 1 in424 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR8, SMEMatrixTileH>, SMEPseudo2Instr<NAME, 0>;425 426 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, intrinsic, timm32_0_1, nxv16i1, nxv16i8>;427}428 429multiclass sme2p1_fmop_tile_fp16<string mnemonic, bit bf, bit s, ValueType vt, SDPatternOperator intrinsic = null_frag> {430 def NAME : sme_fp_outer_product_inst<s, {0,bf}, 0b11, TileOp16, ZPR16, mnemonic>, SMEPseudo2Instr<NAME, 1> {431 bits<1> ZAda;432 let Inst{2-1} = 0b00;433 let Inst{0} = ZAda;434 }435 436 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR16, SMEMatrixTileH>, SMEPseudo2Instr<NAME, 0>;437 438 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, intrinsic, timm32_0_1, nxv8i1, vt>;439}440 441class sme_int_outer_product_inst<bits<3> opc, bit sz, bit sme2,442 MatrixTileOperand za_ty, ZPRRegOp zpr_ty,443 string mnemonic>444 : I<(outs za_ty:$ZAda),445 (ins za_ty:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn, zpr_ty:$Zm),446 mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn, $Zm",447 "", []>,448 Sched<[]> {449 bits<5> Zm;450 bits<3> Pm;451 bits<3> Pn;452 bits<5> Zn;453 let Inst{31-25} = 0b1010000;454 let Inst{24} = opc{2}; // u0455 let Inst{23} = 0b1;456 let Inst{22} = sz;457 let Inst{21} = opc{1}; // u1458 let Inst{20-16} = Zm;459 let Inst{15-13} = Pm;460 let Inst{12-10} = Pn;461 let Inst{9-5} = Zn;462 let Inst{4} = opc{0}; //S;463 let Inst{3} = sme2;464 465 let Constraints = "$ZAda = $_ZAda";466}467 468multiclass sme_int_outer_product_i32<bits<3> opc, string mnemonic,469 SDPatternOperator op> {470 def NAME : sme_int_outer_product_inst<opc, 0b0, 0b0, TileOp32,471 ZPR8, mnemonic>, SMEPseudo2Instr<NAME, 1> {472 bits<2> ZAda;473 let Inst{1-0} = ZAda;474 let Inst{2} = 0b0;475 }476 477 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR8, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>;478 479 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_3, nxv16i1, nxv16i8>;480}481 482multiclass sme_int_outer_product_i64<bits<3> opc, string mnemonic,483 SDPatternOperator op> {484 def NAME : sme_int_outer_product_inst<opc, 0b1, 0b0, TileOp64,485 ZPR16, mnemonic>, SMEPseudo2Instr<NAME, 1> {486 bits<3> ZAda;487 let Inst{2-0} = ZAda;488 }489 490 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR16, SMEMatrixTileD>, SMEPseudo2Instr<NAME, 0>;491 492 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_7, nxv8i1, nxv8i16>;493}494 495class sme_outer_product_widening_inst<bits<3> opc, ZPRRegOp zpr_ty, string mnemonic>496 : I<(outs TileOp32:$ZAda),497 (ins TileOp32:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn, zpr_ty:$Zm),498 mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn, $Zm",499 "", []>,500 Sched<[]> {501 bits<5> Zm;502 bits<3> Pm;503 bits<3> Pn;504 bits<5> Zn;505 bits<2> ZAda;506 let Inst{31-25} = 0b1000000;507 let Inst{24} = !if(opc{2}, 0, 1);508 let Inst{23-22} = 0b10;509 let Inst{21} = opc{1};510 let Inst{20-16} = Zm;511 let Inst{15-13} = Pm;512 let Inst{12-10} = Pn;513 let Inst{9-5} = Zn;514 let Inst{4} = opc{0};515 let Inst{3} = opc{2};516 let Inst{2} = 0b0;517 let Inst{1-0} = ZAda;518 519 let Constraints = "$ZAda = $_ZAda";520}521 522multiclass sme_bf16_outer_product<bits<3> opc, string mnemonic, SDPatternOperator op> {523 def NAME : sme_outer_product_widening_inst<opc, ZPR16, mnemonic>, SMEPseudo2Instr<NAME, 1>;524 525 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR16, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>;526 527 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_3, nxv8i1, nxv8bf16>;528}529 530multiclass sme_f16_outer_product<bits<3> opc, string mnemonic, SDPatternOperator op> {531 def NAME : sme_outer_product_widening_inst<opc, ZPR16, mnemonic>, SMEPseudo2Instr<NAME, 1>;532 533 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR16, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>;534 535 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_3, nxv8i1, nxv8f16>;536}537 538class sme_quarter_outer_product_i64<bits<2> zn_u_pair, bits<2> zm_u_pair, bit subtr, RegisterOperand zn_ty, RegisterOperand zm_ty, string mnemonic>539 : I<(outs TileOp64:$ZAda),540 (ins TileOp64:$_ZAda, zn_ty:$Zn, zm_ty:$Zm),541 mnemonic, "\t$ZAda, $Zn, $Zm",542 "", []>,543 Sched<[]> {544 bits<3> ZAda;545 bits<3> Zn;546 bits<3> Zm;547 let Inst{31-25} = 0b1010000;548 let Inst{24} = zn_u_pair{1}; // u0549 let Inst{23-22} = 0b11;550 let Inst{21} = zm_u_pair{1}; // u1551 let Inst{20} = zm_u_pair{0}; // M552 let Inst{19-17} = Zm;553 let Inst{16-10} = 0b0000000;554 let Inst{9} = zn_u_pair{0}; // N555 let Inst{8-6} = Zn;556 let Inst{5} = 0;557 let Inst{4} = subtr;558 let Inst{3} = 0b1;559 let Inst{2-0} = ZAda;560 561 let Constraints = "$ZAda = $_ZAda";562}563 564class sme_quarter_outer_product_i8_i32<bits<2> zn_u_pair, bits<2> zm_u_pair, bit subtr, RegisterOperand zn_ty, RegisterOperand zm_ty, string mnemonic>565 : I<(outs TileOp32:$ZAda),566 (ins TileOp32:$_ZAda, zn_ty:$Zn, zm_ty:$Zm),567 mnemonic, "\t$ZAda, $Zn, $Zm",568 "", []>,569 Sched<[]> {570 bits<2> ZAda;571 bits<3> Zn;572 bits<3> Zm;573 let Inst{31-25} = 0b1000000;574 let Inst{24} = zn_u_pair{1}; // u0575 let Inst{23-22} = 0b00;576 let Inst{21} = zm_u_pair{1}; // u1577 let Inst{20} = zm_u_pair{0}; // M578 let Inst{19-17} = Zm;579 let Inst{16-10} = 0b0100000;580 let Inst{9} = zn_u_pair{0}; // N581 let Inst{8-6} = Zn;582 let Inst{5} = 0;583 let Inst{4} = subtr;584 let Inst{3-2} = 0b00;585 let Inst{1-0} = ZAda;586 587 let Constraints = "$ZAda = $_ZAda";588}589 590class sme_quarter_outer_product_i16_i32<bit u0, bit N, bit M, bit subtr, RegisterOperand zn_ty, RegisterOperand zm_ty, string mnemonic>591 : I<(outs TileOp32:$ZAda),592 (ins TileOp32:$_ZAda, zn_ty:$Zn, zm_ty:$Zm),593 mnemonic, "\t$ZAda, $Zn, $Zm",594 "", []>,595 Sched<[]> {596 bits<2> ZAda;597 bits<3> Zn;598 bits<3> Zm;599 let Inst{31-25} = 0b1000000;600 let Inst{24} = u0;601 let Inst{23-21} = 0b000;602 let Inst{20} = M;603 let Inst{19-17} = Zm;604 let Inst{16-10} = 0b0100000;605 let Inst{9} = N;606 let Inst{8-6} = Zn;607 let Inst{5} = 0;608 let Inst{4} = subtr;609 let Inst{3-2} = 0b10;610 let Inst{1-0} = ZAda;611 612 let Constraints = "$ZAda = $_ZAda";613}614 615multiclass sme_quarter_outer_product_i8_i32<bit zn_u, bit zm_u, bit subtr, string mnemonic, string op>{616 // Single vectors617 def _MZZ_BToS : sme_quarter_outer_product_i8_i32<{zn_u, 0}, {zm_u, 0}, subtr,618 ZPR8Mul2_Lo, ZPR8Mul2_Hi, mnemonic>, SMEPseudo2Instr<NAME # _MZZ_BToS, 1>;619 620 def NAME # _MZZ_BToS_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZPR8Mul2_Lo, ZPR8Mul2_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _MZZ_BToS, 0>;621 622 def : SME2_ZA_Tile_Vec_Single_Single_Pat<NAME # _MZZ_BToS, !cast<SDPatternOperator>(op # "_1x1"), timm32_0_3, nxv16i8>;623 624 // Multiple and single vectors625 def _M2ZZ_BToS : sme_quarter_outer_product_i8_i32<{zn_u, 1}, {zm_u, 0}, subtr,626 ZZ_b_mul_r_Lo, ZPR8Mul2_Hi, mnemonic>, SMEPseudo2Instr<NAME # _M2ZZ_BToS, 1>;627 628 def NAME # _M2ZZ_BToS_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZZ_b_mul_r_Lo, ZPR8Mul2_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _M2ZZ_BToS, 0>;629 630 def : SME2_ZA_Tile_Vec_Multi_Single_Pat<NAME # _M2ZZ_BToS, !cast<SDPatternOperator>(op # "_2x1"), timm32_0_3, nxv16i8>;631 632 // Single and multiple vectors633 def _MZ2Z_BToS : sme_quarter_outer_product_i8_i32<{zn_u, 0}, {zm_u, 1}, subtr,634 ZPR8Mul2_Lo, ZZ_b_mul_r_Hi, mnemonic>, SMEPseudo2Instr<NAME # _MZ2Z_BToS, 1>;635 636 def NAME # _MZ2Z_BToS_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZPR8Mul2_Lo, ZZ_b_mul_r_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _MZ2Z_BToS, 0>;637 638 def : SME2_ZA_Tile_Vec_Multi_Pat<NAME # _MZ2Z_BToS, !cast<SDPatternOperator>(op # "_1x2"), timm32_0_3, nxv16i8>;639 640 // Multiple vectors641 def _M2Z2Z_BToS : sme_quarter_outer_product_i8_i32<{zn_u, 1}, {zm_u, 1}, subtr,642 ZZ_b_mul_r_Lo, ZZ_b_mul_r_Hi, mnemonic>, SMEPseudo2Instr<NAME # _M2Z2Z_BToS, 1>;643 644 def NAME # _M2Z2Z_BToS_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZZ_b_mul_r_Lo, ZZ_b_mul_r_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _M2Z2Z_BToS, 0>;645 646 def : SME2_ZA_Tile_Vec_Multi_Multi_Pat<NAME # _M2Z2Z_BToS, !cast<SDPatternOperator>(op # "_2x2"), timm32_0_3, nxv16i8>;647}648 649multiclass sme_quarter_outer_product_i16_i32<bit unsigned, bit subtr, string mnemonic, string op>{650 // Single vectors651 def _MZZ_HToS : sme_quarter_outer_product_i16_i32<unsigned, 0b0, 0b0, subtr,652 ZPR16Mul2_Lo, ZPR16Mul2_Hi, mnemonic>, SMEPseudo2Instr<NAME # _MZZ_HToS, 1>;653 654 def NAME # _MZZ_HToS_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZPR16Mul2_Lo, ZPR16Mul2_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _MZZ_HToS, 0>;655 656 def : SME2_ZA_Tile_Vec_Single_Single_Pat<NAME # _MZZ_HToS, !cast<SDPatternOperator>(op # "_1x1"), timm32_0_3, nxv8i16>;657 658 // Multiple and single vectors659 def _M2ZZ_HToS : sme_quarter_outer_product_i16_i32<unsigned, 0b1, 0b0, subtr,660 ZZ_h_mul_r_Lo, ZPR16Mul2_Hi, mnemonic>, SMEPseudo2Instr<NAME # _M2ZZ_HToS, 1>;661 662 def NAME # _M2ZZ_HToS_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZZ_h_mul_r_Lo, ZPR16Mul2_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _M2ZZ_HToS, 0>;663 664 def : SME2_ZA_Tile_Vec_Multi_Single_Pat<NAME # _M2ZZ_HToS, !cast<SDPatternOperator>(op # "_2x1"), timm32_0_3, nxv8i16>;665 666 // Single and multiple vectors667 def _MZ2Z_HToS : sme_quarter_outer_product_i16_i32<unsigned, 0b0, 0b1, subtr,668 ZPR16Mul2_Lo, ZZ_h_mul_r_Hi, mnemonic>, SMEPseudo2Instr<NAME # _MZ2Z_HToS, 1>;669 670 def NAME # _MZ2Z_HToS_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZPR16Mul2_Lo, ZZ_h_mul_r_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _MZ2Z_HToS, 0>;671 672 def : SME2_ZA_Tile_Vec_Multi_Pat<NAME # _MZ2Z_HToS, !cast<SDPatternOperator>(op # "_1x2"), timm32_0_3, nxv8i16>;673 674 // Multiple vectors675 def _M2Z2Z_HToS : sme_quarter_outer_product_i16_i32<unsigned, 0b1, 0b1, subtr,676 ZZ_h_mul_r_Lo, ZZ_h_mul_r_Hi, mnemonic>, SMEPseudo2Instr<NAME # _M2Z2Z_HToS, 1>;677 678 def NAME # _M2Z2Z_HToS_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZZ_h_mul_r_Lo, ZZ_h_mul_r_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _M2Z2Z_HToS, 0>;679 680 def : SME2_ZA_Tile_Vec_Multi_Multi_Pat<NAME # _M2Z2Z_HToS, !cast<SDPatternOperator>(op # "_2x2"), timm32_0_3, nxv8i16>;681}682 683multiclass sme_quarter_outer_product_i64<bit zn_u, bit zm_u, bit subtr, string mnemonic, string op>{684 // Single vectors685 def _MZZ_HtoD : sme_quarter_outer_product_i64<{zn_u, 0}, {zm_u, 0}, subtr,686 ZPR16Mul2_Lo, ZPR16Mul2_Hi, mnemonic>, SMEPseudo2Instr<NAME # _MZZ_HtoD, 1>;687 688 def NAME # _MZZ_HtoD_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZPR16Mul2_Lo, ZPR16Mul2_Hi, SMEMatrixTileD>, SMEPseudo2Instr<NAME # _MZZ_HtoD, 0>;689 690 def : SME2_ZA_Tile_Vec_Single_Single_Pat<NAME # _MZZ_HtoD, !cast<SDPatternOperator>(op # "_1x1"), timm32_0_7, nxv8i16>;691 692 // Multiple and single vectors693 def _M2ZZ_HtoD : sme_quarter_outer_product_i64<{zn_u, 1}, {zm_u, 0}, subtr,694 ZZ_h_mul_r_Lo, ZPR16Mul2_Hi, mnemonic>, SMEPseudo2Instr<NAME # _M2ZZ_HtoD, 1>;695 696 def NAME # _M2ZZ_HtoD_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZZ_h_mul_r_Lo, ZPR16Mul2_Hi, SMEMatrixTileD>, SMEPseudo2Instr<NAME # _M2ZZ_HtoD, 0>;697 698 def : SME2_ZA_Tile_Vec_Multi_Single_Pat<NAME # _M2ZZ_HtoD, !cast<SDPatternOperator>(op # "_2x1"), timm32_0_7, nxv8i16>;699 700 // Single and multiple vectors701 def _MZ2Z_HtoD : sme_quarter_outer_product_i64<{zn_u, 0}, {zm_u, 1}, subtr,702 ZPR16Mul2_Lo, ZZ_h_mul_r_Hi, mnemonic>, SMEPseudo2Instr<NAME # _MZ2Z_HtoD, 1>;703 704 def NAME # _MZ2Z_HtoD_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZPR16Mul2_Lo, ZZ_h_mul_r_Hi, SMEMatrixTileD>, SMEPseudo2Instr<NAME # _MZ2Z_HtoD, 0>;705 706 def : SME2_ZA_Tile_Vec_Multi_Pat<NAME # _MZ2Z_HtoD, !cast<SDPatternOperator>(op # "_1x2"), timm32_0_7, nxv8i16>;707 708 // Multiple vectors709 def _M2Z2Z_HtoD : sme_quarter_outer_product_i64<{zn_u, 1}, {zm_u, 1}, subtr,710 ZZ_h_mul_r_Lo, ZZ_h_mul_r_Hi, mnemonic>, SMEPseudo2Instr<NAME # _M2Z2Z_HtoD, 1>;711 712 def NAME # _M2Z2Z_HtoD_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZZ_h_mul_r_Lo, ZZ_h_mul_r_Hi, SMEMatrixTileD>, SMEPseudo2Instr<NAME # _M2Z2Z_HtoD, 0>;713 714 def : SME2_ZA_Tile_Vec_Multi_Multi_Pat<NAME # _M2Z2Z_HtoD, !cast<SDPatternOperator>(op # "_2x2"), timm32_0_7, nxv8i16>;715}716 717//===----------------------------------------------------------------------===//718// SME Add Vector to Tile719//===----------------------------------------------------------------------===//720 721class sme_add_vector_to_tile_inst<bit op, bit V, MatrixTileOperand tile_ty,722 ZPRRegOp zpr_ty, string mnemonic>723 : I<(outs tile_ty:$ZAda),724 (ins tile_ty:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn),725 mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn",726 "", []>, Sched<[]> {727 bits<3> Pm;728 bits<3> Pn;729 bits<5> Zn;730 let Inst{31-23} = 0b110000001;731 let Inst{22} = op;732 let Inst{21-17} = 0b01000;733 let Inst{16} = V;734 let Inst{15-13} = Pm;735 let Inst{12-10} = Pn;736 let Inst{9-5} = Zn;737 let Inst{4-3} = 0b00;738 739 let Constraints = "$ZAda = $_ZAda";740}741 742class sme_add_vector_to_tile_pseudo<ZPRRegOp zpr_ty, SMEMatrixTypeEnum za_flag>743 : Pseudo<(outs),744 (ins i32imm:$tile, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn), []>,745 Sched<[]> {746 // Translated to the actual instructions in AArch64ISelLowering.cpp747 let SMEMatrixType = za_flag;748 let usesCustomInserter = 1;749 let mayLoad = 1;750 let mayStore = 1;751}752 753multiclass sme_add_vector_to_tile_u32<bit V, string mnemonic, SDPatternOperator op> {754 def NAME : sme_add_vector_to_tile_inst<0b0, V, TileOp32, ZPR32, mnemonic>, SMEPseudo2Instr<NAME, 1> {755 bits<2> ZAda;756 let Inst{2} = 0b0;757 let Inst{1-0} = ZAda;758 }759 760 def _PSEUDO_S : sme_add_vector_to_tile_pseudo<ZPR32, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>;761 762 def : Pat<(op timm32_0_3:$tile, (nxv4i1 PPR3bAny:$pn), (nxv4i1 PPR3bAny:$pm),763 (nxv4i32 ZPR32:$zn)),764 (!cast<Instruction>(NAME # _PSEUDO_S) timm32_0_3:$tile, $pn, $pm, $zn)>;765}766 767multiclass sme_add_vector_to_tile_u64<bit V, string mnemonic, SDPatternOperator op> {768 def NAME : sme_add_vector_to_tile_inst<0b1, V, TileOp64, ZPR64, mnemonic>, SMEPseudo2Instr<NAME, 1> {769 bits<3> ZAda;770 let Inst{2-0} = ZAda;771 }772 773 def _PSEUDO_D : sme_add_vector_to_tile_pseudo<ZPR64, SMEMatrixTileD>, SMEPseudo2Instr<NAME, 0>;774 775 let Predicates = [HasSMEI16I64] in {776 def : Pat<(op timm32_0_7:$tile, (nxv2i1 PPR3bAny:$pn), (nxv2i1 PPR3bAny:$pm),777 (nxv2i64 ZPR64:$zn)),778 (!cast<Instruction>(NAME # _PSEUDO_D) timm32_0_7:$tile, $pn, $pm, $zn)>;779 }780}781 782//===----------------------------------------------------------------------===//783// SME Contiguous Loads784//===----------------------------------------------------------------------===//785 786class sme_mem_ld_ss_base<bit Q, bit V, bits<2> msz, dag outs, dag ins,787 string mnemonic, string argstr>788 : I<outs, ins, mnemonic, argstr, "", []>, Sched<[]> {789 bits<5> Rm;790 bits<2> Rv;791 bits<3> Pg;792 bits<5> Rn;793 let Inst{31-25} = 0b1110000;794 let Inst{24} = Q;795 let Inst{23-22} = msz;796 let Inst{21} = 0b0;797 let Inst{20-16} = Rm;798 let Inst{15} = V;799 let Inst{14-13} = Rv;800 let Inst{12-10} = Pg;801 let Inst{9-5} = Rn;802 let Inst{4} = 0b0;803 804 let mayLoad = 1;805}806 807class sme_mem_ld_ss_inst<bit Q, bits<2> msz, string mnemonic,808 MatrixTileVectorOperand tile_ty, bit is_col,809 Operand imm_ty, RegisterOperand gpr_ty>810 : sme_mem_ld_ss_base<811 Q, is_col, msz, (outs tile_ty:$ZAt),812 (ins MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn,813 gpr_ty:$Rm),814 mnemonic, "\t\\{$ZAt[$Rv, $imm]\\}, $Pg/z, [$Rn, $Rm]">;815 816multiclass sme_mem_ss_aliases_base<string mnemonic, Instruction inst,817 MatrixTileVectorOperand tile_ty,818 Operand imm_ty, RegisterOperand gpr_ty,819 string pg_suffix=""> {820 def : InstAlias<mnemonic # "\t$ZAt[$Rv, $imm], $Pg" # pg_suffix # ", [$Rn, $Rm]",821 (inst tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn, gpr_ty:$Rm), 0>;822 // Default XZR offset aliases823 def : InstAlias<mnemonic # "\t\\{$ZAt[$Rv, $imm]\\}, $Pg" # pg_suffix # ", [$Rn]",824 (inst tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 1>;825 def : InstAlias<mnemonic # "\t$ZAt[$Rv, $imm], $Pg" # pg_suffix # ", [$Rn]",826 (inst tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 0>;827}828 829multiclass sme_mem_ss_aliases<string mnemonic, string inst, bit is_col,830 string pg_suffix=""> {831 defm : sme_mem_ss_aliases_base<mnemonic # "b", !cast<Instruction>(inst # _B),832 !if(is_col, TileVectorOpV8, TileVectorOpH8),833 sme_elm_idx0_15, GPR64shifted8, pg_suffix>;834 defm : sme_mem_ss_aliases_base<mnemonic # "h", !cast<Instruction>(inst # _H),835 !if(is_col, TileVectorOpV16, TileVectorOpH16),836 sme_elm_idx0_7, GPR64shifted16, pg_suffix>;837 defm : sme_mem_ss_aliases_base<mnemonic # "w", !cast<Instruction>(inst # _S),838 !if(is_col, TileVectorOpV32, TileVectorOpH32),839 sme_elm_idx0_3, GPR64shifted32, pg_suffix>;840 defm : sme_mem_ss_aliases_base<mnemonic # "d", !cast<Instruction>(inst # _D),841 !if(is_col, TileVectorOpV64, TileVectorOpH64),842 sme_elm_idx0_1, GPR64shifted64, pg_suffix>;843 defm : sme_mem_ss_aliases_base<mnemonic # "q", !cast<Instruction>(inst # _Q),844 !if(is_col, TileVectorOpV128, TileVectorOpH128),845 sme_elm_idx0_0, GPR64shifted128, pg_suffix>;846}847 848multiclass sme_mem_ld_ss_aliases<string inst, bit is_col> {849 defm NAME : sme_mem_ss_aliases<"ld1", inst, is_col, "/z">;850}851 852multiclass sme_mem_ld_ss_patterns<Instruction Inst, SDPatternOperator Load,853 Operand tile_ty, Operand offset_ty,854 ComplexPattern addr,855 ComplexPattern tileslice> {856 // base, tileslice857 def : Pat<(Load PPR3bAny:$pg, GPR64sp:$base, tile_ty:$tile,858 (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, offset_ty:$imm))),859 (Inst tile_ty:$tile, $idx, $imm, $pg, $base, XZR)>;860 861 // reg + reg, tileslice862 let AddedComplexity = 1 in {863 def : Pat<(Load PPR3bAny:$pg, (addr GPR64sp:$base, GPR64:$offset),864 tile_ty:$tile, (i32 (tileslice MatrixIndexGPR32Op12_15:$idx,865 offset_ty:$imm))),866 (Inst tile_ty:$tile, $idx, $imm, $pg, $base, $offset)>;867 }868}869 870class sme_load_pseudo871 : Pseudo<(outs), (ins i32imm:$tile, MatrixIndexGPR32Op12_15:$idx,872 i32imm:$imm, PPR3bAny:$pg, GPR64sp:$base, GPR64:$offset), []>,873 Sched<[]> {874 // Translated to the actual instructions in AArch64ISelLowering.cpp875 let usesCustomInserter = 1;876 let mayLoad = 1;877}878 879multiclass sme_mem_ld_v_ss<string mnemonic, bit is_col> {880 def _B : sme_mem_ld_ss_inst<0b0, 0b00, mnemonic # "b",881 !if(is_col, TileVectorOpV8, TileVectorOpH8),882 is_col, sme_elm_idx0_15, GPR64shifted8> {883 bits<0> ZAt;884 bits<4> imm;885 let Inst{3-0} = imm;886 }887 def _H : sme_mem_ld_ss_inst<0b0, 0b01, mnemonic # "h",888 !if(is_col, TileVectorOpV16, TileVectorOpH16),889 is_col, sme_elm_idx0_7, GPR64shifted16> {890 bits<1> ZAt;891 bits<3> imm;892 let Inst{3} = ZAt;893 let Inst{2-0} = imm;894 }895 def _S : sme_mem_ld_ss_inst<0b0, 0b10, mnemonic # "w",896 !if(is_col, TileVectorOpV32, TileVectorOpH32),897 is_col, sme_elm_idx0_3, GPR64shifted32> {898 bits<2> ZAt;899 bits<2> imm;900 let Inst{3-2} = ZAt;901 let Inst{1-0} = imm;902 }903 def _D : sme_mem_ld_ss_inst<0b0, 0b11, mnemonic # "d",904 !if(is_col, TileVectorOpV64, TileVectorOpH64),905 is_col, sme_elm_idx0_1, GPR64shifted64> {906 bits<3> ZAt;907 bits<1> imm;908 let Inst{3-1} = ZAt;909 let Inst{0} = imm;910 }911 def _Q : sme_mem_ld_ss_inst<0b1, 0b11, mnemonic # "q",912 !if(is_col, TileVectorOpV128, TileVectorOpH128),913 is_col, sme_elm_idx0_0, GPR64shifted128> {914 bits<4> ZAt;915 bits<0> imm;916 let Inst{3-0} = ZAt;917 }918 919 defm : sme_mem_ld_ss_aliases<NAME, is_col>;920 921 // Pseudo instructions for lowering intrinsics, using immediates instead of922 // tile registers.923 def _PSEUDO_B : sme_load_pseudo;924 def _PSEUDO_H : sme_load_pseudo;925 def _PSEUDO_S : sme_load_pseudo;926 def _PSEUDO_D : sme_load_pseudo;927 def _PSEUDO_Q : sme_load_pseudo;928 929 defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_B),930 !if(is_col, int_aarch64_sme_ld1b_vert,931 int_aarch64_sme_ld1b_horiz),932 sme_elm_idx0_0, timm32_0_15, am_sve_regreg_lsl0,933 tileslice8>;934 defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_H),935 !if(is_col, int_aarch64_sme_ld1h_vert,936 int_aarch64_sme_ld1h_horiz),937 timm32_0_1, timm32_0_7, am_sve_regreg_lsl1,938 tileslice16>;939 defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_S),940 !if(is_col, int_aarch64_sme_ld1w_vert,941 int_aarch64_sme_ld1w_horiz),942 timm32_0_3, timm32_0_3, am_sve_regreg_lsl2,943 tileslice32>;944 defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_D),945 !if(is_col, int_aarch64_sme_ld1d_vert,946 int_aarch64_sme_ld1d_horiz),947 timm32_0_7, timm32_0_1, am_sve_regreg_lsl3,948 tileslice64>;949 defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_Q),950 !if(is_col, int_aarch64_sme_ld1q_vert,951 int_aarch64_sme_ld1q_horiz),952 timm32_0_15, sme_elm_idx0_0, am_sve_regreg_lsl4,953 tileslice128>;954}955 956multiclass sme_mem_ld_ss<string mnemonic> {957 defm _H : sme_mem_ld_v_ss<mnemonic, /*is_col=*/0b0>;958 defm _V : sme_mem_ld_v_ss<mnemonic, /*is_col=*/0b1>;959}960 961//===----------------------------------------------------------------------===//962// SME Contiguous Stores963//===----------------------------------------------------------------------===//964 965class sme_mem_st_ss_base<bit Q, bit V, bits<2> msz, dag ins,966 string mnemonic, string argstr>967 : I<(outs), ins, mnemonic, argstr, "", []>, Sched<[]> {968 bits<5> Rm;969 bits<2> Rv;970 bits<3> Pg;971 bits<5> Rn;972 let Inst{31-25} = 0b1110000;973 let Inst{24} = Q;974 let Inst{23-22} = msz;975 let Inst{21} = 0b1;976 let Inst{20-16} = Rm;977 let Inst{15} = V;978 let Inst{14-13} = Rv;979 let Inst{12-10} = Pg;980 let Inst{9-5} = Rn;981 let Inst{4} = 0b0;982 983 let mayStore = 1;984 let hasSideEffects = 1;985}986 987class sme_mem_st_ss_inst<bit Q, bits<2> msz, string mnemonic,988 MatrixTileVectorOperand tile_ty, bit is_col,989 Operand imm_ty, RegisterOperand gpr_ty>990 : sme_mem_st_ss_base<991 Q, is_col, msz,992 (ins tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg,993 GPR64sp:$Rn, gpr_ty:$Rm),994 mnemonic, "\t\\{$ZAt[$Rv, $imm]\\}, $Pg, [$Rn, $Rm]">;995 996multiclass sme_mem_st_ss_aliases<string inst, bit is_col> {997 defm NAME : sme_mem_ss_aliases<"st1", inst, is_col>;998}999 1000multiclass sme_mem_st_ss_patterns<Instruction Inst, SDPatternOperator Store,1001 Operand offset_ty,1002 ComplexPattern imm2tile,1003 ComplexPattern addr,1004 ComplexPattern tileslice> {1005 // base, tileslice1006 def : Pat<(Store PPR3bAny:$pg, GPR64sp:$base, (imm2tile untyped:$tile),1007 (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, offset_ty:$imm))),1008 (Inst $tile, $idx, $imm, $pg, $base, XZR)>;1009 1010 // reg + reg, tileslice1011 let AddedComplexity = 1 in {1012 def : Pat<(Store PPR3bAny:$pg, (addr GPR64sp:$base, GPR64:$offset),1013 (imm2tile untyped:$tile),1014 (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, offset_ty:$imm))),1015 (Inst $tile, $idx, $imm, $pg, $base, $offset)>;1016 }1017}1018 1019multiclass sme_mem_st_v_ss<string mnemonic, bit is_col> {1020 def _B : sme_mem_st_ss_inst<0b0, 0b00, mnemonic # "b",1021 !if(is_col, TileVectorOpV8, TileVectorOpH8),1022 is_col, sme_elm_idx0_15, GPR64shifted8> {1023 bits<0> ZAt;1024 bits<4> imm;1025 let Inst{3-0} = imm;1026 }1027 def _H : sme_mem_st_ss_inst<0b0, 0b01, mnemonic # "h",1028 !if(is_col, TileVectorOpV16, TileVectorOpH16),1029 is_col, sme_elm_idx0_7, GPR64shifted16> {1030 bits<1> ZAt;1031 bits<3> imm;1032 let Inst{3} = ZAt;1033 let Inst{2-0} = imm;1034 }1035 def _S : sme_mem_st_ss_inst<0b0, 0b10, mnemonic # "w",1036 !if(is_col, TileVectorOpV32, TileVectorOpH32),1037 is_col, sme_elm_idx0_3, GPR64shifted32> {1038 bits<2> ZAt;1039 bits<2> imm;1040 let Inst{3-2} = ZAt;1041 let Inst{1-0} = imm;1042 }1043 def _D : sme_mem_st_ss_inst<0b0, 0b11, mnemonic # "d",1044 !if(is_col, TileVectorOpV64, TileVectorOpH64),1045 is_col, sme_elm_idx0_1, GPR64shifted64> {1046 bits<3> ZAt;1047 bits<1> imm;1048 let Inst{3-1} = ZAt;1049 let Inst{0} = imm;1050 }1051 def _Q : sme_mem_st_ss_inst<0b1, 0b11, mnemonic # "q",1052 !if(is_col, TileVectorOpV128, TileVectorOpH128),1053 is_col, sme_elm_idx0_0, GPR64shifted128> {1054 bits<4> ZAt;1055 bits<0> imm;1056 let Inst{3-0} = ZAt;1057 }1058 1059 defm : sme_mem_st_ss_aliases<NAME, is_col>;1060 1061 defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _B),1062 !if(is_col, int_aarch64_sme_st1b_vert,1063 int_aarch64_sme_st1b_horiz),1064 timm32_0_15, imm_to_tile8, am_sve_regreg_lsl0,1065 tileslice8>;1066 defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _H),1067 !if(is_col, int_aarch64_sme_st1h_vert,1068 int_aarch64_sme_st1h_horiz),1069 timm32_0_7, imm_to_tile16, am_sve_regreg_lsl1,1070 tileslice16>;1071 defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _S),1072 !if(is_col, int_aarch64_sme_st1w_vert,1073 int_aarch64_sme_st1w_horiz),1074 timm32_0_3, imm_to_tile32, am_sve_regreg_lsl2,1075 tileslice32>;1076 defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _D),1077 !if(is_col, int_aarch64_sme_st1d_vert,1078 int_aarch64_sme_st1d_horiz),1079 timm32_0_1, imm_to_tile64, am_sve_regreg_lsl3,1080 tileslice64>;1081 defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _Q),1082 !if(is_col, int_aarch64_sme_st1q_vert,1083 int_aarch64_sme_st1q_horiz),1084 sme_elm_idx0_0, imm_to_tile128,1085 am_sve_regreg_lsl4, tileslice128>;1086}1087 1088multiclass sme_mem_st_ss<string mnemonic> {1089 defm _H : sme_mem_st_v_ss<mnemonic, /*is_col=*/0b0>;1090 defm _V : sme_mem_st_v_ss<mnemonic, /*is_col=*/0b1>;1091}1092 1093//===----------------------------------------------------------------------===//1094// SME Save and Restore Array1095//===----------------------------------------------------------------------===//1096 1097class sme_spill_fill_base<bit isStore, dag outs, dag ins, string opcodestr>1098 : I<outs, ins, opcodestr, "\t$ZAt[$Rv, $imm4], [$Rn, $offset, mul vl]", "",1099 []>,1100 Sched<[]> {1101 // 'offset' operand is encoded in the same bits as 'imm4'. There is currently1102 // no way to tell TableGen about this.1103 let DecoderMethod = "DecodeSMESpillFillInstruction";1104 bits<0> ZAt;1105 bits<2> Rv;1106 bits<5> Rn;1107 bits<4> imm4;1108 let Inst{31-22} = 0b1110000100;1109 let Inst{21} = isStore;1110 let Inst{20-15} = 0b000000;1111 let Inst{14-13} = Rv;1112 let Inst{12-10} = 0b000;1113 let Inst{9-5} = Rn;1114 let Inst{4} = 0b0;1115 let Inst{3-0} = imm4;1116}1117 1118let mayStore = 1 in1119class sme_spill_inst<string opcodestr>1120 : sme_spill_fill_base<0b1, (outs),1121 (ins MatrixOp:$ZAt, MatrixIndexGPR32Op12_15:$Rv,1122 sme_elm_idx0_15:$imm4, GPR64sp:$Rn,1123 imm32_0_15:$offset),1124 opcodestr>;1125let mayLoad = 1 in1126class sme_fill_inst<string opcodestr>1127 : sme_spill_fill_base<0b0, (outs MatrixOp:$ZAt),1128 (ins MatrixIndexGPR32Op12_15:$Rv,1129 sme_elm_idx0_15:$imm4, GPR64sp:$Rn,1130 imm32_0_15:$offset),1131 opcodestr>;1132multiclass sme_spill<string opcodestr> {1133 def NAME : sme_spill_inst<opcodestr>;1134 def : InstAlias<opcodestr # "\t$ZAt[$Rv, $imm4], [$Rn]",1135 (!cast<Instruction>(NAME) MatrixOp:$ZAt,1136 MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_15:$imm4, GPR64sp:$Rn, 0), 1>;1137 1138 def : Pat<(AArch64SMEStr (i32 MatrixIndexGPR32Op12_15:$slice), (i64 GPR64sp:$base), (i32 sme_elm_idx0_15:$imm)),1139 (!cast<Instruction>(NAME) ZA, MatrixIndexGPR32Op12_15:$slice, sme_elm_idx0_15:$imm, GPR64sp:$base, imm32_0_15:$imm)>;1140}1141 1142multiclass sme_fill<string opcodestr> {1143 def NAME : sme_fill_inst<opcodestr>;1144 def : InstAlias<opcodestr # "\t$ZAt[$Rv, $imm4], [$Rn]",1145 (!cast<Instruction>(NAME) MatrixOp:$ZAt,1146 MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_15:$imm4, GPR64sp:$Rn, 0), 1>;1147 def NAME # _PSEUDO1148 : Pseudo<(outs),1149 (ins MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_15:$imm4,1150 GPR64sp:$base), []>,1151 Sched<[]> {1152 // Translated to actual instruction in AArch64ISelLowering.cpp1153 let usesCustomInserter = 1;1154 let mayLoad = 1;1155 }1156 def : Pat<(AArch64SMELdr MatrixIndexGPR32Op12_15:$slice, GPR64sp:$base, sme_elm_idx0_15:$imm),1157 (!cast<Instruction>(NAME # _PSEUDO) MatrixIndexGPR32Op12_15:$slice, sme_elm_idx0_15:$imm, GPR64sp:$base)>;1158}1159 1160//===----------------------------------------------------------------------===//1161// Move instructions1162//===----------------------------------------------------------------------===//1163 1164class sme_vector_to_tile_base<bit Q, bit V, bits<2> sz, dag outs, dag ins,1165 string mnemonic, string argstr>1166 : I<outs, ins, mnemonic, argstr, "", []>, Sched<[]> {1167 bits<2> Rv;1168 bits<3> Pg;1169 bits<5> Zn;1170 let Inst{31-24} = 0b11000000;1171 let Inst{23-22} = sz;1172 let Inst{21-17} = 0b00000;1173 let Inst{16} = Q;1174 let Inst{15} = V;1175 let Inst{14-13} = Rv;1176 let Inst{12-10} = Pg;1177 let Inst{9-5} = Zn;1178 let Inst{4} = 0b0;1179}1180 1181class sme_vector_to_tile_inst<bit Q, bits<2> sz, MatrixTileVectorOperand tile_ty,1182 bit is_col, Operand imm_ty, ZPRRegOp zpr_ty,1183 string mnemonic>1184 : sme_vector_to_tile_base<Q, is_col, sz, (outs tile_ty:$ZAd),1185 (ins tile_ty:$_ZAd, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, zpr_ty:$Zn),1186 mnemonic, "\t$ZAd[$Rv, $imm], $Pg/m, $Zn">{1187 1188 let Constraints = "$ZAd = $_ZAd";1189}1190 1191 1192multiclass sme_vector_to_tile_aliases<Instruction inst,1193 MatrixTileVectorOperand tile_ty,1194 ZPRRegOp zpr_ty, Operand imm_ty> {1195 def : InstAlias<"mov\t$ZAd[$Rv, $imm], $Pg/m, $Zn",1196 (inst tile_ty:$ZAd, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, zpr_ty:$Zn), 1>;1197}1198 1199multiclass sme_vector_to_tile_patterns<Instruction inst, ValueType zpr_vt,1200 ValueType ppr_vt, Operand imm_ty,1201 Operand offset_ty,1202 SDPatternOperator op,1203 ComplexPattern tileslice> {1204 def : Pat<(op imm_ty:$tile, (i32 (tileslice MatrixIndexGPR32Op12_15:$idx,1205 offset_ty:$imm)),1206 (ppr_vt PPR3bAny:$pg), (zpr_vt ZPRAny:$zn)),1207 (inst imm_ty:$tile, $idx, $imm, $pg, $zn)>;1208}1209 1210class sme_mova_insert_pseudo<SMEMatrixTypeEnum za_flag>1211 : Pseudo<(outs), (ins i32imm:$tile, MatrixIndexGPR32Op12_15:$idx,1212 i32imm:$imm, PPR3bAny:$pg, ZPRAny:$zn), []>,1213 Sched<[]> {1214 // Translated to the actual instructions in AArch64ISelLowering.cpp1215 let SMEMatrixType = za_flag;1216 let usesCustomInserter = 1;1217 let mayStore = 1;1218}1219 1220multiclass sme_vector_v_to_tile<string mnemonic, bit is_col> {1221 def _B : sme_vector_to_tile_inst<0b0, 0b00, !if(is_col, TileVectorOpV8,1222 TileVectorOpH8),1223 is_col, sme_elm_idx0_15, ZPR8, mnemonic>,1224 SMEPseudo2Instr<NAME # _B, 1> {1225 bits<0> ZAd;1226 bits<4> imm;1227 let Inst{3-0} = imm;1228 }1229 def _H : sme_vector_to_tile_inst<0b0, 0b01, !if(is_col, TileVectorOpV16,1230 TileVectorOpH16),1231 is_col, sme_elm_idx0_7, ZPR16, mnemonic>,1232 SMEPseudo2Instr<NAME # _H, 1> {1233 bits<1> ZAd;1234 bits<3> imm;1235 let Inst{3} = ZAd;1236 let Inst{2-0} = imm;1237 }1238 def _S : sme_vector_to_tile_inst<0b0, 0b10, !if(is_col, TileVectorOpV32,1239 TileVectorOpH32),1240 is_col, sme_elm_idx0_3, ZPR32, mnemonic>,1241 SMEPseudo2Instr<NAME # _S, 1> {1242 bits<2> ZAd;1243 bits<2> imm;1244 let Inst{3-2} = ZAd;1245 let Inst{1-0} = imm;1246 }1247 def _D : sme_vector_to_tile_inst<0b0, 0b11, !if(is_col, TileVectorOpV64,1248 TileVectorOpH64),1249 is_col, sme_elm_idx0_1, ZPR64, mnemonic>,1250 SMEPseudo2Instr<NAME # _D, 1> {1251 bits<3> ZAd;1252 bits<1> imm;1253 let Inst{3-1} = ZAd;1254 let Inst{0} = imm;1255 }1256 def _Q : sme_vector_to_tile_inst<0b1, 0b11, !if(is_col, TileVectorOpV128,1257 TileVectorOpH128),1258 is_col, sme_elm_idx0_0, ZPR128, mnemonic>,1259 SMEPseudo2Instr<NAME # _Q, 1> {1260 bits<4> ZAd;1261 bits<0> imm;1262 let Inst{3-0} = ZAd;1263 }1264 1265 // Pseudo instructions for lowering intrinsics, using immediates instead of1266 // tile registers.1267 def _PSEUDO_B : sme_mova_insert_pseudo<SMEMatrixTileB>, SMEPseudo2Instr<NAME # _B, 0>;1268 def _PSEUDO_H : sme_mova_insert_pseudo<SMEMatrixTileH>, SMEPseudo2Instr<NAME # _H, 0>;1269 def _PSEUDO_S : sme_mova_insert_pseudo<SMEMatrixTileS>, SMEPseudo2Instr<NAME # _S, 0>;1270 def _PSEUDO_D : sme_mova_insert_pseudo<SMEMatrixTileD>, SMEPseudo2Instr<NAME # _D, 0>;1271 def _PSEUDO_Q : sme_mova_insert_pseudo<SMEMatrixTileQ>, SMEPseudo2Instr<NAME # _Q, 0>;1272 1273 defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _B),1274 !if(is_col, TileVectorOpV8,1275 TileVectorOpH8),1276 ZPR8, sme_elm_idx0_15>;1277 defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _H),1278 !if(is_col, TileVectorOpV16,1279 TileVectorOpH16),1280 ZPR16, sme_elm_idx0_7>;1281 defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _S),1282 !if(is_col, TileVectorOpV32,1283 TileVectorOpH32),1284 ZPR32, sme_elm_idx0_3>;1285 defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _D),1286 !if(is_col, TileVectorOpV64,1287 TileVectorOpH64),1288 ZPR64, sme_elm_idx0_1>;1289 defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _Q),1290 !if(is_col, TileVectorOpV128,1291 TileVectorOpH128),1292 ZPR128, sme_elm_idx0_0>;1293 1294 defvar op = !if(is_col, int_aarch64_sme_write_vert,1295 int_aarch64_sme_write_horiz);1296 1297 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_B),1298 nxv16i8, nxv16i1, sme_elm_idx0_0, sme_elm_idx0_15,1299 op, tileslice8>;1300 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_H),1301 nxv8i16, nxv8i1, sme_elm_idx0_1, sme_elm_idx0_7,1302 op, tileslice16>;1303 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_H),1304 nxv8f16, nxv8i1, sme_elm_idx0_1, sme_elm_idx0_7,1305 op, tileslice16>;1306 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_H),1307 nxv8bf16, nxv8i1, sme_elm_idx0_1, sme_elm_idx0_7,1308 op, tileslice16>;1309 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_S),1310 nxv4i32, nxv4i1, sme_elm_idx0_3, sme_elm_idx0_3,1311 op, tileslice32>;1312 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_S),1313 nxv4f32, nxv4i1, sme_elm_idx0_3, sme_elm_idx0_3,1314 op, tileslice32>;1315 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_D),1316 nxv2i64, nxv2i1, sme_elm_idx0_7, sme_elm_idx0_1,1317 op, tileslice64>;1318 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_D),1319 nxv2f64, nxv2i1, sme_elm_idx0_7, sme_elm_idx0_1,1320 op, tileslice64>;1321 1322 defvar opq = !if(is_col, int_aarch64_sme_writeq_vert,1323 int_aarch64_sme_writeq_horiz);1324 1325 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q),1326 nxv16i8, nxv16i1, sme_elm_idx0_15,1327 sme_elm_idx0_0, opq, tileslice128>;1328 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q),1329 nxv8i16, nxv8i1, sme_elm_idx0_15,1330 sme_elm_idx0_0, opq, tileslice128>;1331 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q),1332 nxv8f16, nxv8i1, sme_elm_idx0_15,1333 sme_elm_idx0_0, opq, tileslice128>;1334 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q),1335 nxv8bf16, nxv8i1, sme_elm_idx0_15,1336 sme_elm_idx0_0, opq, tileslice128>;1337 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q),1338 nxv4i32, nxv4i1, sme_elm_idx0_15,1339 sme_elm_idx0_0, opq, tileslice128>;1340 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q),1341 nxv4f32, nxv4i1, sme_elm_idx0_15,1342 sme_elm_idx0_0, opq, tileslice128>;1343 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q),1344 nxv2i64, nxv2i1, sme_elm_idx0_15,1345 sme_elm_idx0_0, opq, tileslice128>;1346 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q),1347 nxv2f64, nxv2i1, sme_elm_idx0_15,1348 sme_elm_idx0_0, opq, tileslice128>;1349}1350 1351multiclass sme_vector_to_tile<string mnemonic> {1352 defm _H : sme_vector_v_to_tile<mnemonic, /*is_col=*/0b0>;1353 defm _V : sme_vector_v_to_tile<mnemonic, /*is_col=*/0b1>;1354}1355 1356class sme_tile_to_vector_base<bit Q, bit V, bits<2> sz, dag outs, dag ins,1357 string mnemonic, string argstr>1358 : I<outs, ins, mnemonic, argstr, "", []>, Sched<[]> {1359 bits<2> Rv;1360 bits<3> Pg;1361 bits<5> Zd;1362 let Inst{31-24} = 0b11000000;1363 let Inst{23-22} = sz;1364 let Inst{21-17} = 0b00001;1365 let Inst{16} = Q;1366 let Inst{15} = V;1367 let Inst{14-13} = Rv;1368 let Inst{12-10} = Pg;1369 let Inst{9} = 0b0;1370 let Inst{4-0} = Zd;1371}1372 1373class sme_tile_to_vector_inst<bit Q, bits<2> sz, ZPRRegOp zpr_ty,1374 MatrixTileVectorOperand tile_ty,1375 bit is_col, Operand imm_ty, string mnemonic>1376 : sme_tile_to_vector_base<Q, is_col, sz, (outs zpr_ty:$Zd),1377 (ins zpr_ty:$_Zd, PPR3bAny:$Pg, tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm),1378 mnemonic, "\t$Zd, $Pg/m, $ZAn[$Rv, $imm]"> {1379 1380 let Constraints = "$Zd = $_Zd";1381}1382 1383multiclass sme_tile_to_vector_aliases<Instruction inst, ZPRRegOp zpr_ty,1384 MatrixTileVectorOperand tile_ty,1385 Operand imm_ty > {1386 def : InstAlias<"mov\t$Zd, $Pg/m, $ZAn[$Rv, $imm]",1387 (inst zpr_ty:$Zd, PPR3bAny:$Pg, tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm), 1>;1388}1389 1390multiclass sme_tile_to_vector_patterns<Instruction inst, ValueType zpr_vt,1391 ValueType ppr_vt, Operand offset_ty,1392 ComplexPattern imm2tile,1393 ComplexPattern tileslice,1394 SDPatternOperator op> {1395 def : Pat<(zpr_vt (op (zpr_vt ZPRAny:$passthru), (ppr_vt PPR3bAny:$pg),1396 (imm2tile untyped:$tile), MatrixIndexGPR32Op12_15:$idx)),1397 (inst $passthru, $pg, $tile, $idx, 0)>;1398 let AddedComplexity = 1 in {1399 def : Pat<(zpr_vt (op (zpr_vt ZPRAny:$passthru), (ppr_vt PPR3bAny:$pg),1400 (imm2tile untyped:$tile),1401 (i32 (tileslice MatrixIndexGPR32Op12_15:$idx,1402 offset_ty:$imm)))),1403 (inst $passthru, $pg, $tile, $idx, $imm)>;1404 }1405}1406 1407multiclass sme_tile_to_vector_v<string mnemonic, bit is_col> {1408 def _B : sme_tile_to_vector_inst<0b0, 0b00, ZPR8, !if(is_col, TileVectorOpV8,1409 TileVectorOpH8),1410 is_col, sme_elm_idx0_15, mnemonic> {1411 bits<0> ZAn;1412 bits<4> imm;1413 let Inst{8-5} = imm;1414 let mayLoad = 1;1415 }1416 def _H : sme_tile_to_vector_inst<0b0, 0b01, ZPR16, !if(is_col, TileVectorOpV16,1417 TileVectorOpH16),1418 is_col, sme_elm_idx0_7, mnemonic> {1419 bits<1> ZAn;1420 bits<3> imm;1421 let Inst{8} = ZAn;1422 let Inst{7-5} = imm;1423 let mayLoad = 1;1424 }1425 def _S : sme_tile_to_vector_inst<0b0, 0b10, ZPR32, !if(is_col, TileVectorOpV32,1426 TileVectorOpH32),1427 is_col, sme_elm_idx0_3, mnemonic> {1428 bits<2> ZAn;1429 bits<2> imm;1430 let Inst{8-7} = ZAn;1431 let Inst{6-5} = imm;1432 let mayLoad = 1;1433 }1434 def _D : sme_tile_to_vector_inst<0b0, 0b11, ZPR64, !if(is_col, TileVectorOpV64,1435 TileVectorOpH64),1436 is_col, sme_elm_idx0_1, mnemonic> {1437 bits<3> ZAn;1438 bits<1> imm;1439 let Inst{8-6} = ZAn;1440 let Inst{5} = imm;1441 let mayLoad = 1;1442 }1443 def _Q : sme_tile_to_vector_inst<0b1, 0b11, ZPR128, !if(is_col, TileVectorOpV128,1444 TileVectorOpH128),1445 is_col, sme_elm_idx0_0, mnemonic> {1446 bits<4> ZAn;1447 bits<0> imm;1448 let Inst{8-5} = ZAn;1449 let mayLoad = 1;1450 }1451 1452 defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _B), ZPR8,1453 !if(is_col, TileVectorOpV8,1454 TileVectorOpH8), sme_elm_idx0_15>;1455 defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _H), ZPR16,1456 !if(is_col, TileVectorOpV16,1457 TileVectorOpH16), sme_elm_idx0_7>;1458 defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _S), ZPR32,1459 !if(is_col, TileVectorOpV32,1460 TileVectorOpH32), sme_elm_idx0_3>;1461 defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _D), ZPR64,1462 !if(is_col, TileVectorOpV64,1463 TileVectorOpH64), sme_elm_idx0_1>;1464 defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _Q), ZPR128,1465 !if(is_col, TileVectorOpV128,1466 TileVectorOpH128), sme_elm_idx0_0>;1467 1468 defvar op = !if(is_col, int_aarch64_sme_read_vert,1469 int_aarch64_sme_read_horiz);1470 1471 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _B),1472 nxv16i8, nxv16i1, sme_elm_idx0_15,1473 imm_to_tile8, tileslice8, op>;1474 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _H),1475 nxv8i16, nxv8i1, sme_elm_idx0_7,1476 imm_to_tile16, tileslice16, op>;1477 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _H),1478 nxv8f16, nxv8i1, sme_elm_idx0_7,1479 imm_to_tile16, tileslice16, op>;1480 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _H),1481 nxv8bf16, nxv8i1, sme_elm_idx0_7,1482 imm_to_tile16, tileslice16, op>;1483 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _S),1484 nxv4i32, nxv4i1, sme_elm_idx0_3,1485 imm_to_tile32, tileslice32, op>;1486 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _S),1487 nxv4f32, nxv4i1, sme_elm_idx0_3,1488 imm_to_tile32, tileslice32, op>;1489 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _D),1490 nxv2i64, nxv2i1, sme_elm_idx0_1,1491 imm_to_tile64, tileslice64, op>;1492 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _D),1493 nxv2f64, nxv2i1, sme_elm_idx0_1,1494 imm_to_tile64, tileslice64, op>;1495 1496 defvar opq = !if(is_col, int_aarch64_sme_readq_vert,1497 int_aarch64_sme_readq_horiz);1498 1499 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q),1500 nxv16i8, nxv16i1, sme_elm_idx0_0,1501 imm_to_tile128, tileslice128, opq>;1502 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q),1503 nxv8i16, nxv8i1, sme_elm_idx0_0,1504 imm_to_tile128, tileslice128, opq>;1505 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q),1506 nxv8f16, nxv8i1, sme_elm_idx0_0,1507 imm_to_tile128, tileslice128, opq>;1508 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q),1509 nxv8bf16, nxv8i1, sme_elm_idx0_0,1510 imm_to_tile128, tileslice128, opq>;1511 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q),1512 nxv4i32, nxv4i1, sme_elm_idx0_0,1513 imm_to_tile128, tileslice128, opq>;1514 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q),1515 nxv4f32, nxv4i1, sme_elm_idx0_0,1516 imm_to_tile128, tileslice128, opq>;1517 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q),1518 nxv2i64, nxv2i1, sme_elm_idx0_0,1519 imm_to_tile128, tileslice128, opq>;1520 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q),1521 nxv2f64, nxv2i1, sme_elm_idx0_0,1522 imm_to_tile128, tileslice128, opq>;1523}1524 1525multiclass sme_tile_to_vector<string mnemonic> {1526 defm _H : sme_tile_to_vector_v<mnemonic, /*is_col=*/0b0>;1527 defm _V : sme_tile_to_vector_v<mnemonic, /*is_col=*/0b1>;1528}1529 1530//===----------------------------------------------------------------------===//1531// SME Zero1532//===----------------------------------------------------------------------===//1533 1534// NOTE: This definition isn't really correct because there are outputs, i.e.1535// the tile registers being zeroed. We fix this up in a custom inserter that1536// marks the appropriate registers as being implicitly defined.1537class sme_zero_inst<string mnemonic>1538 : I<(outs), (ins MatrixTileList:$imm),1539 mnemonic, "\t$imm", "", []>, Sched<[]> {1540 bits<8> imm;1541 let Inst{31-8} = 0b110000000000100000000000;1542 let Inst{7-0} = imm;1543}1544 1545multiclass sme_zero<string mnemonic> {1546 def NAME : sme_zero_inst<mnemonic>;1547 1548 def : InstAlias<"zero\t\\{za\\}", (!cast<Instruction>(NAME) 0b11111111), 1>;1549 def : InstAlias<"zero\t\\{za0.h\\}", (!cast<Instruction>(NAME) 0b01010101), 1>;1550 def : InstAlias<"zero\t\\{za1.h\\}", (!cast<Instruction>(NAME) 0b10101010), 1>;1551 def : InstAlias<"zero\t\\{za0.s\\}", (!cast<Instruction>(NAME) 0b00010001), 1>;1552 def : InstAlias<"zero\t\\{za1.s\\}", (!cast<Instruction>(NAME) 0b00100010), 1>;1553 def : InstAlias<"zero\t\\{za2.s\\}", (!cast<Instruction>(NAME) 0b01000100), 1>;1554 def : InstAlias<"zero\t\\{za3.s\\}", (!cast<Instruction>(NAME) 0b10001000), 1>;1555 def : InstAlias<"zero\t\\{za0.s,za1.s\\}", (!cast<Instruction>(NAME) 0b00110011), 1>;1556 def : InstAlias<"zero\t\\{za0.s,za3.s\\}", (!cast<Instruction>(NAME) 0b10011001), 1>;1557 def : InstAlias<"zero\t\\{za1.s,za2.s\\}", (!cast<Instruction>(NAME) 0b01100110), 1>;1558 def : InstAlias<"zero\t\\{za2.s,za3.s\\}", (!cast<Instruction>(NAME) 0b11001100), 1>;1559 def : InstAlias<"zero\t\\{za0.s,za1.s,za2.s\\}", (!cast<Instruction>(NAME) 0b01110111), 1>;1560 def : InstAlias<"zero\t\\{za0.s,za1.s,za3.s\\}", (!cast<Instruction>(NAME) 0b10111011), 1>;1561 def : InstAlias<"zero\t\\{za0.s,za2.s,za3.s\\}", (!cast<Instruction>(NAME) 0b11011101), 1>;1562 def : InstAlias<"zero\t\\{za1.s,za2.s,za3.s\\}", (!cast<Instruction>(NAME) 0b11101110), 1>;1563 1564 def NAME # _PSEUDO : Pseudo<(outs), (ins i32imm:$tilelist), []>,1565 Sched<[]> {1566 // Translated to the actual instructions in AArch64ISelLowering.cpp1567 let usesCustomInserter = 1;1568 }1569 1570 def : Pat<(int_aarch64_sme_zero timm32_0_255:$imm),1571 (!cast<Instruction>(NAME # _PSEUDO) timm32_0_255:$imm)>;1572}1573 1574//===----------------------------------------------------------------------===//1575// SVE2 Instructions1576//===----------------------------------------------------------------------===//1577 1578class sve2_int_perm_revd<string asm>1579 : I<(outs ZPR128:$Zd), (ins ZPR128:$_Zd, PPR3bAny:$Pg, ZPR128:$Zn),1580 asm, "\t$Zd, $Pg/m, $Zn", "", []>,1581 Sched<[]> {1582 bits<5> Zd;1583 bits<3> Pg;1584 bits<5> Zn;1585 let Inst{31-24} = 0b00000101;1586 let Inst{23-22} = 0b00; // size1587 let Inst{21-13} = 0b101110100;1588 let Inst{12-10} = Pg;1589 let Inst{9-5} = Zn;1590 let Inst{4-0} = Zd;1591 1592 let Constraints = "$Zd = $_Zd";1593}1594 1595multiclass sve2_int_perm_revd<string asm, SDPatternOperator op> {1596 def NAME : sve2_int_perm_revd<asm>;1597 1598 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME)>;1599 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME)>;1600 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME)>;1601 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME)>;1602 1603 def : SVE_1_Op_Passthru_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, !cast<Instruction>(NAME)>;1604 def : SVE_1_Op_Passthru_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME)>;1605 def : SVE_1_Op_Passthru_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME)>;1606 def : SVE_1_Op_Passthru_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME)>;1607 1608}1609 1610class sve2_clamp<string asm, bits<2> sz, bit U, ZPRRegOp zpr_ty>1611 : I<(outs zpr_ty:$Zd), (ins zpr_ty:$_Zd, zpr_ty:$Zn, zpr_ty:$Zm),1612 asm, "\t$Zd, $Zn, $Zm", "", []>,1613 Sched<[]> {1614 bits<5> Zm;1615 bits<5> Zn;1616 bits<5> Zd;1617 let Inst{31-24} = 0b01000100;1618 let Inst{23-22} = sz;1619 let Inst{21} = 0b0;1620 let Inst{20-16} = Zm;1621 let Inst{15-11} = 0b11000;1622 let Inst{10} = U;1623 let Inst{9-5} = Zn;1624 let Inst{4-0} = Zd;1625 1626 let Constraints = "$Zd = $_Zd";1627 let DestructiveInstType = DestructiveOther;1628 let ElementSize = zpr_ty.ElementSize;1629}1630 1631multiclass sve2_clamp<string asm, bit U, SDPatternOperator op> {1632 def _B : sve2_clamp<asm, 0b00, U, ZPR8>;1633 def _H : sve2_clamp<asm, 0b01, U, ZPR16>;1634 def _S : sve2_clamp<asm, 0b10, U, ZPR32>;1635 def _D : sve2_clamp<asm, 0b11, U, ZPR64>;1636 1637 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;1638 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;1639 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;1640 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;1641}1642 1643class sve2_int_perm_sel_p<string asm, PPRRegOp ppr_ty, Operand imm_ty>1644 : I<(outs PPRorPNRAny:$Pd), (ins PPRorPNRAny:$Pn, ppr_ty:$Pm,1645 MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm),1646 asm, "\t$Pd, $Pn, $Pm[$Rv, $imm]", "", []>,1647 Sched<[]> {1648 bits<2> Rv;1649 bits<4> Pn;1650 bits<4> Pm;1651 bits<4> Pd;1652 let Inst{31-24} = 0b00100101;1653 let Inst{21} = 0b1;1654 let Inst{17-16} = Rv;1655 let Inst{15-14} = 0b01;1656 let Inst{13-10} = Pn;1657 let Inst{9} = 0b0;1658 let Inst{8-5} = Pm;1659 let Inst{4} = 0b0;1660 let Inst{3-0} = Pd;1661}1662 1663multiclass sve2_int_perm_sel_p<string asm, SDPatternOperator op> {1664 def _B : sve2_int_perm_sel_p<asm, PPR8, sme_elm_idx0_15> {1665 bits<4> imm;1666 let Inst{23-22} = imm{3-2};1667 let Inst{20-19} = imm{1-0};1668 let Inst{18} = 0b1;1669 }1670 def _H : sve2_int_perm_sel_p<asm, PPR16, sme_elm_idx0_7> {1671 bits<3> imm;1672 let Inst{23-22} = imm{2-1};1673 let Inst{20} = imm{0};1674 let Inst{19-18} = 0b10;1675 }1676 def _S : sve2_int_perm_sel_p<asm, PPR32, sme_elm_idx0_3> {1677 bits<2> imm;1678 let Inst{23-22} = imm{1-0};1679 let Inst{20-18} = 0b100;1680 }1681 def _D : sve2_int_perm_sel_p<asm, PPR64, sme_elm_idx0_1> {1682 bits<1> imm;1683 let Inst{23} = imm;1684 let Inst{22} = 0b1;1685 let Inst{20-18} = 0b000;1686 }1687 1688 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv16i1 PPR8:$Pm),1689 MatrixIndexGPR32Op12_15:$idx)),1690 (!cast<Instruction>(NAME # _B) $Pn, $Pm, $idx, 0)>;1691 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv8i1 PPR16:$Pm),1692 MatrixIndexGPR32Op12_15:$idx)),1693 (!cast<Instruction>(NAME # _H) $Pn, $Pm, $idx, 0)>;1694 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv4i1 PPR32:$Pm),1695 MatrixIndexGPR32Op12_15:$idx)),1696 (!cast<Instruction>(NAME # _S) $Pn, $Pm, $idx, 0)>;1697 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv2i1 PPR64:$Pm),1698 MatrixIndexGPR32Op12_15:$idx)),1699 (!cast<Instruction>(NAME # _D) $Pn, $Pm, $idx, 0)>;1700 1701 let AddedComplexity = 1 in {1702 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv16i1 PPR8:$Pm),1703 (i32 (tileslice8 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_15:$imm)))),1704 (!cast<Instruction>(NAME # _B) $Pn, $Pm, $idx, $imm)>;1705 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv8i1 PPR16:$Pm),1706 (i32 (tileslice16 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_7:$imm)))),1707 (!cast<Instruction>(NAME # _H) $Pn, $Pm, $idx, $imm)>;1708 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv4i1 PPR32:$Pm),1709 (i32 (tileslice32 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_3:$imm)))),1710 (!cast<Instruction>(NAME # _S) $Pn, $Pm, $idx, $imm)>;1711 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv2i1 PPR64:$Pm),1712 (i32 (tileslice64 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_1:$imm)))),1713 (!cast<Instruction>(NAME # _D) $Pn, $Pm, $idx, $imm)>;1714 }1715}1716 1717//===----------------------------------------------------------------------===//1718// SME2 Instructions1719//===----------------------------------------------------------------------===//1720 1721//===----------------------------------------------------------------------===//1722// SME2 single-multi ternary int/fp, two/four registers1723 1724class sme2_dot_mla_add_sub_array_vg24_single<bits<7> op,1725 MatrixOperand matrix_ty,1726 RegisterOperand multi_vector_ty,1727 ZPRRegOp zpr_ty,1728 string mnemonic>1729 : I<(outs matrix_ty:$ZAd),1730 (ins matrix_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rv,1731 sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm),1732 mnemonic,"\t$ZAd[$Rv, $imm3, " # !if(op{5}, "vgx4", "vgx2") # "], $Zn, $Zm",1733 "", []> , Sched<[]> {1734 bits<0> ZAd;1735 bits<4> Zm;1736 bits<5> Zn;1737 bits<2> Rv;1738 bits<3> imm3;1739 let Inst{31-23} = 0b110000010;1740 let Inst{22} = op{6}; //sz1741 let Inst{21} = 0b1;1742 let Inst{20} = op{5}; //vgx41743 let Inst{19-16} = Zm;1744 let Inst{15} = 0b0;1745 let Inst{14-13} = Rv;1746 let Inst{12-10} = op{4-2};1747 let Inst{9-5} = Zn;1748 let Inst{4-3} = op{1-0};1749 let Inst{2-0} = imm3;1750 let Constraints = "$ZAd = $_ZAd";1751}1752 1753multiclass sme2_dot_mla_add_sub_array_vg24_single<string mnemonic, bits<7> op,1754 MatrixOperand matrix_ty,1755 RegisterOperand multi_vector_ty,1756 ZPRRegOp zpr_ty>{1757 def NAME: sme2_dot_mla_add_sub_array_vg24_single<op, matrix_ty, multi_vector_ty, zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>;1758 1759 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm",1760 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>;1761}1762 1763multiclass sme2_dot_mla_add_sub_array_vg2_single<string mnemonic, bits<7> op,1764 MatrixOperand matrix_ty,1765 RegisterOperand multi_vector_ty,1766 ZPRRegOp zpr_ty, ValueType vty, SDPatternOperator intrinsic>{1767 def NAME: sme2_dot_mla_add_sub_array_vg24_single<op, matrix_ty, multi_vector_ty, zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>;1768 1769 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm",1770 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>;1771 1772 def _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, zpr_ty, SMEMatrixArray>;1773 1774 def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, vty, tileslice16>;1775}1776 1777multiclass sme2_dot_mla_add_sub_array_vg4_single<string mnemonic, bits<7> op,1778 MatrixOperand matrix_ty,1779 RegisterOperand multi_vector_ty,1780 ZPRRegOp zpr_ty, ValueType vty, SDPatternOperator intrinsic>{1781 def NAME: sme2_dot_mla_add_sub_array_vg24_single<op, matrix_ty, multi_vector_ty, zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>;1782 1783 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm",1784 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>;1785 1786 def _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, zpr_ty, SMEMatrixArray>;1787 1788 def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, vty, tileslice16>;1789}1790 1791//===----------------------------------------------------------------------===//1792// SME2 multiple vectors ternary INT/FP two and four registers1793class sme2_dot_mla_add_sub_array_vg2_multi<bits<7> op,1794 MatrixOperand matrix_ty,1795 RegisterOperand multi_vector_ty,1796 string mnemonic>1797 : I<(outs matrix_ty:$ZAd),1798 (ins matrix_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rv,1799 sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm),1800 mnemonic, "\t$ZAd[$Rv, $imm3, vgx2], $Zn, $Zm",1801 "", []>, Sched<[]>{1802 bits<0> ZAd;1803 bits<4> Zm;1804 bits<4> Zn;1805 bits<2> Rv;1806 bits<3> imm3;1807 let Inst{31-23} = 0b110000011;1808 let Inst{22} = op{6}; //sz1809 let Inst{21} = 0b1;1810 let Inst{20-17} = Zm;1811 let Inst{16-15} = 0b00;1812 let Inst{14-13} = Rv;1813 let Inst{12-10} = op{5-3};1814 let Inst{9-6} = Zn;1815 let Inst{5-3} = op{2-0};1816 let Inst{2-0} = imm3;1817 let Constraints = "$ZAd = $_ZAd";1818}1819 1820multiclass sme2_dot_mla_add_sub_array_vg2_multi<string mnemonic, bits<7> op,1821 MatrixOperand matrix_ty,1822 RegisterOperand multi_vector_ty, ValueType zpr_ty,1823 SDPatternOperator intrinsic> {1824 def NAME : sme2_dot_mla_add_sub_array_vg2_multi<op, matrix_ty, multi_vector_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>;1825 1826 def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, SMEMatrixArray>;1827 1828 def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, tileslice16>;1829 1830 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm",1831 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 0>;1832}1833 1834class sme2_dot_mla_add_sub_array_vg4_multi<bits<7> op,1835 MatrixOperand matrix_ty,1836 RegisterOperand multi_vector_ty,1837 string mnemonic>1838 : I<(outs matrix_ty:$ZAd),1839 (ins matrix_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rv,1840 sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm),1841 mnemonic, "\t$ZAd[$Rv, $imm3, vgx4], $Zn, $Zm",1842 "", []>, Sched<[]>{1843 bits<0> ZAd;1844 bits<3> Zm;1845 bits<3> Zn;1846 bits<2> Rv;1847 bits<3> imm3;1848 let Inst{31-23} = 0b110000011;1849 let Inst{22} = op{6}; //sz1850 let Inst{21} = 0b1;1851 let Inst{20-18} = Zm;1852 let Inst{17-15} = 0b010;1853 let Inst{14-13} = Rv;1854 let Inst{12-10} = op{5-3};1855 let Inst{9-7} = Zn;1856 let Inst{6} = 0b0;1857 let Inst{5-3} = op{2-0};1858 let Inst{2-0} = imm3;1859 let Constraints = "$ZAd = $_ZAd";1860}1861 1862multiclass sme2_dot_mla_add_sub_array_vg4_multi<string mnemonic, bits<7> op,1863 MatrixOperand matrix_ty,1864 RegisterOperand multi_vector_ty,1865 ValueType zpr_ty, SDPatternOperator intrinsic>{1866 def NAME : sme2_dot_mla_add_sub_array_vg4_multi<op, matrix_ty, multi_vector_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>;1867 1868 def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, SMEMatrixArray>;1869 1870 def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, tileslice16>;1871 1872 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm",1873 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 0>;1874}1875 1876//===----------------------------------------------------------------------===//1877// SME2 multiple vectors binary two or four registers1878 1879class sme2_multivec_accum_add_sub<string mnemonic, bit sz, bit vg4, bits<3> op,1880 MatrixOperand matrix_ty,1881 RegisterOperand vector_ty>1882 : I<(outs matrix_ty:$ZAdn),1883 (ins matrix_ty:$_ZAdn, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, vector_ty:$Zm),1884 mnemonic, "\t$ZAdn[$Rv, $imm3, " # !if(vg4, "vgx4", "vgx2") # "], $Zm",1885 "", []>, Sched<[]> {1886 bits<0> ZAdn;1887 bits<2> Rv;1888 bits<3> imm3;1889 let Inst{31-23} = 0b110000011;1890 let Inst{22} = sz;1891 let Inst{21-19} = 0b100;1892 let Inst{18} = op{2};1893 let Inst{17} = 0b0;1894 let Inst{16} = vg4;1895 let Inst{15} = 0b0;1896 let Inst{14-13} = Rv;1897 let Inst{12-10} = 0b111;1898 let Inst{5} = 0b0;1899 let Inst{4-3} = op{1-0};1900 let Inst{2-0} = imm3;1901 1902 let Constraints = "$ZAdn = $_ZAdn";1903}1904 1905class sme2_multivec_accum_add_sub_vg2<string mnemonic, bit sz, bits<3> op,1906 MatrixOperand matrix_ty,1907 RegisterOperand vector_ty>1908 : sme2_multivec_accum_add_sub<mnemonic, sz, 0b0, op, matrix_ty, vector_ty> {1909 bits<4> Zm;1910 let Inst{9-6} = Zm;1911}1912 1913 1914multiclass sme2_multivec_accum_add_sub_vg2<string mnemonic, bits<4> op,1915 MatrixOperand matrix_ty,1916 RegisterOperand vector_ty,1917 ValueType vty,1918 SDPatternOperator intrinsic> {1919 def NAME : sme2_multivec_accum_add_sub_vg2<mnemonic, op{3}, op{2-0}, matrix_ty, vector_ty>,1920 SMEPseudo2Instr<NAME, 1>;1921 def : InstAlias<mnemonic # "\t$ZAdn[$Rv, $imm3], $Zm",1922 (!cast<Instruction>(NAME) matrix_ty:$ZAdn, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, vector_ty:$Zm), 0>;1923 1924 def _PSEUDO : sme2_move_to_za_pseudo<NAME, sme_elm_idx0_7, vector_ty, SMEMatrixArray>{1925 let mayLoad = 1;1926 }1927 def : SME2_ZA_VG1x2_Multi_Pat<NAME, intrinsic, vty, sme_elm_idx0_7, tileslice16>;1928}1929 1930class sme2_multivec_accum_add_sub_vg4<string mnemonic, bit sz, bits<3> op,1931 MatrixOperand matrix_ty,1932 RegisterOperand vector_ty>1933 : sme2_multivec_accum_add_sub<mnemonic, sz, 0b1, op, matrix_ty, vector_ty> {1934 bits<3> Zm;1935 let Inst{9-7} = Zm;1936 let Inst{6} = 0b0;1937}1938 1939multiclass sme2_multivec_accum_add_sub_vg4<string mnemonic, bits<4> op,1940 MatrixOperand matrix_ty,1941 RegisterOperand vector_ty,1942 ValueType vty,1943 SDPatternOperator intrinsic> {1944 def NAME : sme2_multivec_accum_add_sub_vg4<mnemonic, op{3}, op{2-0}, matrix_ty, vector_ty>,1945 SMEPseudo2Instr<NAME, 1>;1946 def : InstAlias<mnemonic # "\t$ZAdn[$Rv, $imm3], $Zm",1947 (!cast<Instruction>(NAME) matrix_ty:$ZAdn, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, vector_ty:$Zm), 0>;1948 1949 def _PSEUDO : sme2_move_to_za_pseudo<NAME, sme_elm_idx0_7, vector_ty, SMEMatrixArray>{1950 let mayLoad = 1;1951 }1952 def : SME2_ZA_VG1x4_Multi_Pat<NAME, intrinsic, vty, sme_elm_idx0_7, tileslice16>;1953}1954 1955//===----------------------------------------------------------------------===//1956// SME2 Multi-vector - Multiple and Single SVE Destructive1957// Two and Four registers1958 1959class sme2_sve_destructive_vector_vg2_single<bits<2> sz, bits<7> op,1960 RegisterOperand vector_ty,1961 ZPRRegOp zpr_ty,1962 string mnemonic>1963 : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, zpr_ty:$Zm),1964 mnemonic, "\t$Zdn, $_Zdn, $Zm",1965 "", []>, Sched<[]> {1966 bits<4> Zm;1967 bits<4> Zdn;1968 let Inst{31-24} = 0b11000001;1969 let Inst{23-22} = sz;1970 let Inst{21-20} = 0b10;1971 let Inst{19-16} = Zm;1972 let Inst{15-11} = 0b10100;1973 let Inst{10-5} = op{6-1};1974 let Inst{4-1} = Zdn;1975 let Inst{0} = op{0};1976 1977 let Constraints = "$Zdn = $_Zdn";1978}1979 1980multiclass sme2_fp_sve_destructive_vector_vg2_single<string mnemonic, bits<7> op> {1981 def _H : sme2_sve_destructive_vector_vg2_single<0b01, op, ZZ_h_mul_r, ZPR4b16, mnemonic>;1982 def _S : sme2_sve_destructive_vector_vg2_single<0b10, op, ZZ_s_mul_r, ZPR4b32, mnemonic>;1983 def _D : sme2_sve_destructive_vector_vg2_single<0b11, op, ZZ_d_mul_r, ZPR4b64, mnemonic>;1984}1985 1986multiclass sme2_int_sve_destructive_vector_vg2_single<string mnemonic, bits<7> op> {1987 def _B : sme2_sve_destructive_vector_vg2_single<0b00, op, ZZ_b_mul_r, ZPR4b8, mnemonic>;1988 def _H : sme2_sve_destructive_vector_vg2_single<0b01, op, ZZ_h_mul_r, ZPR4b16, mnemonic>;1989 def _S : sme2_sve_destructive_vector_vg2_single<0b10, op, ZZ_s_mul_r, ZPR4b32, mnemonic>;1990 def _D : sme2_sve_destructive_vector_vg2_single<0b11, op, ZZ_d_mul_r, ZPR4b64, mnemonic>;1991}1992 1993// SME2.1 fmax/fmin instructions.1994multiclass sme2p1_bf_max_min_vector_vg2_single<string mnemonic, bits<7>op> {1995 def _H : sme2_sve_destructive_vector_vg2_single<0b00, op, ZZ_h_mul_r,1996 ZPR4b16, mnemonic>;1997}1998 1999class sme2_sve_destructive_vector_vg4_single<bits<2> sz, bits<7> op,2000 RegisterOperand vector_ty,2001 ZPRRegOp zpr_ty,2002 string mnemonic>2003 : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, zpr_ty:$Zm),2004 mnemonic, "\t$Zdn, $_Zdn, $Zm",2005 "", []>, Sched<[]> {2006 bits<4> Zm;2007 bits<3> Zdn;2008 let Inst{31-24} = 0b11000001;2009 let Inst{23-22} = sz;2010 let Inst{21-20} = 0b10;2011 let Inst{19-16} = Zm;2012 let Inst{15-11} = 0b10101;2013 let Inst{10-5} = op{6-1};2014 let Inst{4-2} = Zdn;2015 let Inst{1} = 0b0;2016 let Inst{0} = op{0};2017 2018 let Constraints = "$Zdn = $_Zdn";2019}2020 2021multiclass sme2_fp_sve_destructive_vector_vg4_single<string mnemonic, bits<7> op> {2022 def _H : sme2_sve_destructive_vector_vg4_single<0b01, op, ZZZZ_h_mul_r, ZPR4b16, mnemonic>;2023 def _S : sme2_sve_destructive_vector_vg4_single<0b10, op, ZZZZ_s_mul_r, ZPR4b32, mnemonic>;2024 def _D : sme2_sve_destructive_vector_vg4_single<0b11, op, ZZZZ_d_mul_r, ZPR4b64, mnemonic>;2025}2026 2027multiclass sme2_int_sve_destructive_vector_vg4_single<string mnemonic, bits<7> op> {2028 def _B : sme2_sve_destructive_vector_vg4_single<0b00, op, ZZZZ_b_mul_r, ZPR4b8, mnemonic>;2029 def _H : sme2_sve_destructive_vector_vg4_single<0b01, op, ZZZZ_h_mul_r, ZPR4b16, mnemonic>;2030 def _S : sme2_sve_destructive_vector_vg4_single<0b10, op, ZZZZ_s_mul_r, ZPR4b32, mnemonic>;2031 def _D : sme2_sve_destructive_vector_vg4_single<0b11, op, ZZZZ_d_mul_r, ZPR4b64, mnemonic>;2032}2033 2034// SME2.1 fmax/fmin instructions.2035multiclass sme2p1_bf_max_min_vector_vg4_single<string mnemonic, bits<7>op> {2036 def _H : sme2_sve_destructive_vector_vg4_single<0b00, op, ZZZZ_h_mul_r,2037 ZPR4b16, mnemonic>;2038}2039 2040class sme2_sve_destructive_vector_vg2_multi<bits<2> sz, bits<7> op,2041 RegisterOperand vector_ty,2042 string mnemonic>2043 : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, vector_ty:$Zm),2044 mnemonic, "\t$Zdn, $_Zdn, $Zm",2045 "", []>, Sched<[]> {2046 bits<4> Zm;2047 bits<4> Zdn;2048 let Inst{31-24} = 0b11000001;2049 let Inst{23-22} = sz;2050 let Inst{21} = 0b1;2051 let Inst{20-17} = Zm;2052 let Inst{16-11} = 0b010110;2053 let Inst{10-5} = op{6-1};2054 let Inst{4-1} = Zdn;2055 let Inst{0} = op{0};2056 2057 let Constraints = "$Zdn = $_Zdn";2058}2059 2060multiclass sme2_fp_sve_destructive_vector_vg2_multi<string mnemonic, bits<7> op> {2061 def _H : sme2_sve_destructive_vector_vg2_multi<0b01, op, ZZ_h_mul_r, mnemonic>;2062 def _S : sme2_sve_destructive_vector_vg2_multi<0b10, op, ZZ_s_mul_r, mnemonic>;2063 def _D : sme2_sve_destructive_vector_vg2_multi<0b11, op, ZZ_d_mul_r, mnemonic>;2064}2065 2066multiclass sme2_int_sve_destructive_vector_vg2_multi<string mnemonic, bits<7> op> {2067 def _B : sme2_sve_destructive_vector_vg2_multi<0b00, op, ZZ_b_mul_r, mnemonic>;2068 def _H : sme2_sve_destructive_vector_vg2_multi<0b01, op, ZZ_h_mul_r, mnemonic>;2069 def _S : sme2_sve_destructive_vector_vg2_multi<0b10, op, ZZ_s_mul_r, mnemonic>;2070 def _D : sme2_sve_destructive_vector_vg2_multi<0b11, op, ZZ_d_mul_r, mnemonic>;2071}2072 2073// SME2.1 fmax/fmin instructions.2074multiclass sme2p1_bf_max_min_vector_vg2_multi<string mnemonic, bits<7>op> {2075 def _H : sme2_sve_destructive_vector_vg2_multi<0b00, op, ZZ_h_mul_r,2076 mnemonic>;2077}2078 2079class sme2_sve_destructive_vector_vg4_multi<bits<2> sz, bits<7> op,2080 RegisterOperand vector_ty,2081 string mnemonic>2082 : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, vector_ty:$Zm),2083 mnemonic, "\t$Zdn, $_Zdn, $Zm",2084 "", []>, Sched<[]> {2085 bits<3> Zm;2086 bits<3> Zdn;2087 let Inst{31-24} = 0b11000001;2088 let Inst{23-22} = sz;2089 let Inst{21} = 0b1;2090 let Inst{20-18} = Zm;2091 let Inst{17-11} = 0b0010111;2092 let Inst{10-5} = op{6-1};2093 let Inst{4-2} = Zdn;2094 let Inst{1} = 0b0;2095 let Inst{0} = op{0};2096 2097 let Constraints = "$Zdn = $_Zdn";2098}2099 2100multiclass sme2_fp_sve_destructive_vector_vg4_multi<string mnemonic, bits<7> op> {2101 def _H : sme2_sve_destructive_vector_vg4_multi<0b01, op, ZZZZ_h_mul_r, mnemonic>;2102 def _S : sme2_sve_destructive_vector_vg4_multi<0b10, op, ZZZZ_s_mul_r, mnemonic>;2103 def _D : sme2_sve_destructive_vector_vg4_multi<0b11, op, ZZZZ_d_mul_r, mnemonic>;2104}2105 2106multiclass sme2_int_sve_destructive_vector_vg4_multi<string mnemonic, bits<7> op> {2107 def _B : sme2_sve_destructive_vector_vg4_multi<0b00, op, ZZZZ_b_mul_r, mnemonic>;2108 def _H : sme2_sve_destructive_vector_vg4_multi<0b01, op, ZZZZ_h_mul_r, mnemonic>;2109 def _S : sme2_sve_destructive_vector_vg4_multi<0b10, op, ZZZZ_s_mul_r, mnemonic>;2110 def _D : sme2_sve_destructive_vector_vg4_multi<0b11, op, ZZZZ_d_mul_r, mnemonic>;2111}2112 2113// SME2.1 fmax/fmin instructions.2114multiclass sme2p1_bf_max_min_vector_vg4_multi<string mnemonic, bits<7>op> {2115 def _H : sme2_sve_destructive_vector_vg4_multi<0b00, op, ZZZZ_h_mul_r,2116 mnemonic>;2117}2118 2119//===----------------------------------------------------------------------===//2120// SME2 Multi-vector - Index/Single/Multi Array Vectors FMA sources2121 2122class sme2_mla_long_array_index_base<bits<2> op0, bits<2> op, Operand index_ty,2123 RegisterOperand multi_vector_ty,2124 string mnemonic, string vg_acronym="">2125 : I<(outs MatrixOp32:$ZAda),2126 (ins MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm, multi_vector_ty:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3),2127 mnemonic, "\t$ZAda[$Rv, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "], $Zn, $Zm$i3",2128 "", []>, Sched<[]> {2129 bits<0> ZAda;2130 bits<4> Zm;2131 bits<2> Rv;2132 let Inst{31-24} = 0b11000001;2133 let Inst{23-22} = op0;2134 let Inst{21} = 0b0;2135 let Inst{20} = !if(!eq(vg_acronym, ""), 0, 1);2136 let Inst{19-16} = Zm;2137 let Inst{14-13} = Rv;2138 let Inst{12} = 0b1;2139 let Inst{4-3} = op;2140 2141 let Constraints = "$ZAda = $_ZAda";2142}2143 2144multiclass sme2_mla_long_array_index<string mnemonic, bits<2> op0, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> {2145 def _HtoS : sme2_mla_long_array_index_base<op0, op, uimm3s2range, ZPR16,2146 mnemonic>, SMEPseudo2Instr<NAME # _HtoS, 1> {2147 bits<3> i3;2148 bits<5> Zn;2149 bits<3> imm;2150 let Inst{15} = i3{2};2151 let Inst{11-10} = i3{1-0};2152 let Inst{9-5} = Zn;2153 let Inst{2-0} = imm;2154 }2155 2156 def _HtoS_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _HtoS, uimm3s2range, ZPR16, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>;2157 2158 def : SME2_ZA_TwoOp_Multi_Index_Pat<NAME # _HtoS, intrinsic, uimm3s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange3s2>;2159}2160 2161class sme2_mla_long_array_vg2_index<string mnemonic, bits<2> op0, bits<2> op>2162 : sme2_mla_long_array_index_base<op0, op, uimm2s2range, ZZ_h_mul_r,2163 mnemonic, "vgx2"> {2164 bits<3> i3;2165 bits<4> Zn;2166 bits<2> imm;2167 let Inst{15} = 0b0;2168 let Inst{11-10} = i3{2-1};2169 let Inst{9-6} = Zn;2170 let Inst{5} = 0b0;2171 let Inst{2} = i3{0};2172 let Inst{1-0} = imm;2173}2174 2175multiclass sme2_fp_mla_long_array_vg2_index<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> {2176 def _HtoS : sme2_mla_long_array_vg2_index<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _HtoS, 1>;2177 2178 def _HtoS_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _HtoS, uimm2s2range, ZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>;2179 2180 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME # _HtoS, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange2s2>;2181 2182 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3",2183 (!cast<Instruction>(NAME #_HtoS) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>;2184}2185 2186multiclass sme2_int_mla_long_array_vg2_index<string mnemonic, bits<2> op, SDPatternOperator intrinsic> {2187 def _S : sme2_mla_long_array_vg2_index<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _S, 1>;2188 2189 def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm2s2range, ZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>;2190 2191 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange2s2>;2192 2193 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3",2194 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>;2195}2196 2197class sme2_mla_long_array_vg4_index<string mnemonic, bits<2> op0, bits<2> op>2198 : sme2_mla_long_array_index_base<op0, op, uimm2s2range, ZZZZ_h_mul_r,2199 mnemonic, "vgx4"> {2200 bits<3> i3;2201 bits<3> Zn;2202 bits<2> imm;2203 let Inst{15} = 0b1;2204 let Inst{11-10} = i3{2-1};2205 let Inst{9-7} = Zn;2206 let Inst{6-5} = 0b00;2207 let Inst{2} = i3{0};2208 let Inst{1-0} = imm;2209}2210 2211multiclass sme2_fp_mla_long_array_vg4_index<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> {2212 def _HtoS : sme2_mla_long_array_vg4_index<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _HtoS, 1>;2213 2214 def _HtoS_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _HtoS, uimm2s2range, ZZZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>;2215 2216 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME # _HtoS, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange2s2>;2217 2218 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3",2219 (!cast<Instruction>(NAME #_HtoS) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>;2220}2221 2222multiclass sme2_int_mla_long_array_vg4_index<string mnemonic, bits<2> op, SDPatternOperator intrinsic> {2223 def _HtoS : sme2_mla_long_array_vg4_index<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _HtoS, 1>;2224 2225 def _HtoS_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _HtoS, uimm2s2range, ZZZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>;2226 2227 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME # _HtoS, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange2s2>;2228 2229 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3",2230 (!cast<Instruction>(NAME #_HtoS) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>;2231}2232 2233class sme2_mla_long_array<bits<2>op0, bits<2> op,2234 MatrixOperand matrix_ty,2235 Operand index_ty,2236 RegisterOperand first_vector_ty,2237 RegisterOperand second_vector_ty,2238 string mnemonic, string vg_acronym="">2239 : I<(outs matrix_ty:$ZAda),2240 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv,2241 index_ty:$imm, first_vector_ty:$Zn, second_vector_ty:$Zm),2242 mnemonic,"\t$ZAda[$Rv, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "], $Zn, $Zm",2243 "", []> , Sched<[]> {2244 bits<0> ZAda;2245 bits<2> Rv;2246 let Inst{31-24} = 0b11000001;2247 let Inst{23-22} = op0;2248 let Inst{21} = 0b1;2249 let Inst{15} = 0b0;2250 let Inst{14-13} = Rv;2251 let Inst{12-11} = 0b01;2252 let Inst{10} = !if(!eq(vg_acronym, ""), 1, 0);2253 let Inst{4-3} = op;2254 2255 let Constraints = "$ZAda = $_ZAda";2256}2257 2258multiclass sme2_mla_long_array_single<string mnemonic, bits<2> op0, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> {2259 def _HtoS : sme2_mla_long_array<op0, op, MatrixOp32, uimm3s2range, ZPR16, ZPR4b16,2260 mnemonic> , SMEPseudo2Instr<NAME # _HtoS, 1>{2261 bits<4> Zm;2262 bits<5> Zn;2263 bits<3> imm;2264 let Inst{20} = 0b0;2265 let Inst{19-16} = Zm;2266 let Inst{9-5} = Zn;2267 let Inst{2-0} = imm;2268 }2269 2270 def _HtoS_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _HtoS, uimm3s2range, ZPR16, ZPR4b16, SMEMatrixArray>;2271 2272 def : SME2_ZA_TwoOp_Multi_Single_Pat<NAME # _HtoS, intrinsic, uimm3s2range, ZPR4b16, zpr_ty, tileslicerange3s2>;2273}2274 2275class sme2_mla_long_array_single_16b<string mnemonic>2276 : sme2_mla_long_array<0b00, 0b00, MatrixOp16, uimm3s2range, ZPR8, ZPR4b8, mnemonic> {2277 bits<4> Zm;2278 bits<5> Zn;2279 bits<3> imm;2280 let Inst{20} = 0b1;2281 let Inst{19-16} = Zm;2282 let Inst{9-5} = Zn;2283 let Inst{2-0} = imm;2284 let Uses = [FPMR, FPCR];2285}2286 2287multiclass sme2_fp8_fmlal_single_za16<string mnemonic, SDPatternOperator intrinsic> {2288 def NAME : sme2_mla_long_array_single_16b<mnemonic>, SMEPseudo2Instr<NAME, 1>;2289 2290 def _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, uimm3s2range, ZPR8, ZPR4b8, SMEMatrixArray>;2291 2292 def: SME2_ZA_TwoOp_Multi_Single_Pat<NAME, intrinsic, uimm3s2range, ZPR4b8, nxv16i8, tileslicerange3s2>;2293}2294 2295class sme2_mla_long_array_vg24_single<bits<2> op0, bit vg4, bits<2> op, bit o2,2296 MatrixOperand matrix_ty, RegisterOperand multi_vector_ty,2297 ZPRRegOp zpr_ty, string mnemonic, string vg_acronym>2298 : sme2_mla_long_array<op0, op, matrix_ty, uimm2s2range, multi_vector_ty, zpr_ty,2299 mnemonic, vg_acronym> {2300 bits<4> Zm;2301 bits<5> Zn;2302 bits<2> imm;2303 let Inst{20} = vg4;2304 let Inst{19-16} = Zm;2305 let Inst{9-5} = Zn;2306 let Inst{2} = o2;2307 let Inst{1-0} = imm;2308}2309 2310multiclass sme2_fp_mla_long_array_vg2_single<string mnemonic, bits<3> op, MatrixOperand matrix_ty,2311 RegisterOperand multi_vector_ty, ZPRRegOp vector_ty,2312 ValueType zpr_ty, SDPatternOperator intrinsic, list<Register> uses=[]> {2313 def NAME : sme2_mla_long_array_vg24_single<0b00, 0b0, op{2-1}, op{0}, matrix_ty, multi_vector_ty,2314 vector_ty, mnemonic, "vgx2">, SMEPseudo2Instr<NAME, 1> {2315 let Uses = uses;2316 }2317 2318 def _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, uimm2s2range, multi_vector_ty,2319 vector_ty, SMEMatrixArray>;2320 2321 def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME, intrinsic, uimm2s2range, vector_ty, zpr_ty,2322 tileslicerange2s2>;2323 2324 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm",2325 (!cast<Instruction>(NAME) matrix_ty:$ZAda, MatrixIndexGPR32Op8_11:$Rv,2326 uimm2s2range:$imm, multi_vector_ty:$Zn, vector_ty:$Zm), 0>;2327}2328 2329multiclass sme2_int_mla_long_array_vg2_single<string mnemonic, bits<2> op, SDPatternOperator intrinsic> {2330 def _HtoS : sme2_mla_long_array_vg24_single<0b01, 0b0, op, 0b0, MatrixOp32, ZZ_h, ZPR4b16, mnemonic,2331 "vgx2">, SMEPseudo2Instr<NAME # _HtoS, 1>;2332 2333 def _HtoS_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _HtoS, uimm2s2range, ZZ_h, ZPR4b16, SMEMatrixArray>;2334 2335 def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME # _HtoS, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, tileslicerange2s2>;2336 2337 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm",2338 (!cast<Instruction>(NAME #_HtoS) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h:$Zn, ZPR4b16:$Zm), 0>;2339}2340 2341multiclass sme2_fp_mla_long_array_vg4_single<string mnemonic, bits<3> op, MatrixOperand matrix_ty,2342 RegisterOperand multi_vector_ty, ZPRRegOp vector_ty,2343 ValueType zpr_ty, SDPatternOperator intrinsic, list<Register> uses=[]> {2344 def NAME : sme2_mla_long_array_vg24_single<0b00, 0b1, op{2-1}, op{0}, matrix_ty, multi_vector_ty,2345 vector_ty, mnemonic, "vgx4">, SMEPseudo2Instr<NAME, 1> {2346 let Uses = uses;2347 }2348 2349 def _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, uimm2s2range, multi_vector_ty, vector_ty,2350 SMEMatrixArray>;2351 2352 def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME, intrinsic, uimm2s2range, vector_ty, zpr_ty,2353 tileslicerange2s2>;2354 2355 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm",2356 (!cast<Instruction>(NAME) matrix_ty:$ZAda, MatrixIndexGPR32Op8_11:$Rv,2357 uimm2s2range:$imm, multi_vector_ty:$Zn, vector_ty:$Zm), 0>;2358}2359 2360multiclass sme2_int_mla_long_array_vg4_single<string mnemonic, bits<2> op, SDPatternOperator intrinsic> {2361 def _HtoS : sme2_mla_long_array_vg24_single<0b01, 0b1, op, 0b0, MatrixOp32, ZZZZ_h, ZPR4b16, mnemonic,2362 "vgx4">, SMEPseudo2Instr<NAME # _HtoS, 1>;2363 2364 def _HtoS_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _HtoS, uimm2s2range, ZZZZ_h, ZPR4b16, SMEMatrixArray>;2365 2366 def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME # _HtoS, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, tileslicerange2s2>;2367 2368 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm",2369 (!cast<Instruction>(NAME #_HtoS) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h:$Zn, ZPR4b16:$Zm), 0>;2370}2371 2372class sme2_mla_long_array_vg2_multi<string mnemonic, bits<2> op0, bits<3> op,2373 MatrixOperand matrix_ty, RegisterOperand multi_vector_ty>2374 : sme2_mla_long_array<op0, op{1-0}, matrix_ty, uimm2s2range, multi_vector_ty, multi_vector_ty,2375 mnemonic, "vgx2"> {2376 bits<4> Zm;2377 bits<4> Zn;2378 bits<2> imm;2379 let Inst{20-17} = Zm;2380 let Inst{16} = 0b0;2381 let Inst{9-6} = Zn;2382 let Inst{5} = op{2}; // fp82383 let Inst{2} = 0b0;2384 let Inst{1-0} = imm;2385}2386 2387multiclass sme2_fp_mla_long_array_vg2_multi<string mnemonic, bits<3> op, MatrixOperand matrix_ty,2388 RegisterOperand multi_vector_ty, ValueType zpr_ty,2389 SDPatternOperator intrinsic, list<Register> uses=[]> {2390 def NAME : sme2_mla_long_array_vg2_multi<mnemonic, 0b10, op, matrix_ty, multi_vector_ty>,2391 SMEPseudo2Instr<NAME, 1> {2392 let Uses = uses;2393 }2394 2395 def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, uimm2s2range, multi_vector_ty, SMEMatrixArray>;2396 2397 def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME, intrinsic, uimm2s2range, zpr_ty, tileslicerange2s2>;2398 2399 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm",2400 (!cast<Instruction>(NAME) matrix_ty:$ZAda, MatrixIndexGPR32Op8_11:$Rv,2401 uimm2s2range:$imm, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 0>;2402}2403 2404multiclass sme2_int_mla_long_array_vg2_multi<string mnemonic, bits<2> op, SDPatternOperator intrinsic> {2405 def _HtoS : sme2_mla_long_array_vg2_multi<mnemonic, 0b11, {0b0, op}, MatrixOp32, ZZ_h_mul_r>,2406 SMEPseudo2Instr<NAME # _HtoS, 1>;2407 2408 def _HtoS_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _HtoS, uimm2s2range, ZZ_h_mul_r, SMEMatrixArray>;2409 2410 def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME # _HtoS, intrinsic, uimm2s2range, nxv8i16, tileslicerange2s2>;2411 2412 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm2], $Zn, $Zm",2413 (!cast<Instruction>(NAME #_HtoS) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm2, ZZ_h_mul_r:$Zn, ZZ_h_mul_r:$Zm), 0>;2414}2415 2416class sme2_mla_long_array_vg4_multi<string mnemonic, bits<2> op0, bits<3> op,2417 MatrixOperand matrix_ty,2418 RegisterOperand multi_vector_ty>2419 : sme2_mla_long_array<op0, op{1-0}, matrix_ty, uimm2s2range, multi_vector_ty, multi_vector_ty,2420 mnemonic, "vgx4"> {2421 bits<3> Zm;2422 bits<3> Zn;2423 bits<2> imm;2424 let Inst{20-18} = Zm;2425 let Inst{17} = 0b0;2426 let Inst{16} = 0b1;2427 let Inst{9-7} = Zn;2428 let Inst{6} = 0b0;2429 let Inst{5} = op{2}; //fp82430 let Inst{2} = 0b0;2431 let Inst{1-0} = imm;2432}2433 2434multiclass sme2_fp_mla_long_array_vg4_multi<string mnemonic, bits<3> op, MatrixOperand matrix_ty,2435 RegisterOperand multi_vector_ty, ValueType zpr_ty,2436 SDPatternOperator intrinsic, list<Register> uses=[]> {2437 def NAME : sme2_mla_long_array_vg4_multi<mnemonic, 0b10, op, matrix_ty, multi_vector_ty>,2438 SMEPseudo2Instr<NAME, 1> {2439 let Uses = uses;2440 }2441 2442 def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, uimm2s2range, multi_vector_ty, SMEMatrixArray>;2443 2444 def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME, intrinsic, uimm2s2range, zpr_ty, tileslicerange2s2>;2445 2446 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm",2447 (!cast<Instruction>(NAME) matrix_ty:$ZAda, MatrixIndexGPR32Op8_11:$Rv,2448 uimm2s2range:$imm, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 0>;2449}2450 2451multiclass sme2_int_mla_long_array_vg4_multi<string mnemonic, bits<2> op, SDPatternOperator intrinsic> {2452 def _HtoS : sme2_mla_long_array_vg4_multi<mnemonic, 0b11, {0b0, op}, MatrixOp32, ZZZZ_h_mul_r>,2453 SMEPseudo2Instr<NAME # _HtoS, 1>;2454 2455 def _HtoS_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _HtoS, uimm2s2range, ZZZZ_h_mul_r, SMEMatrixArray>;2456 2457 def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME # _HtoS, intrinsic, uimm2s2range, nxv8i16, tileslicerange2s2>;2458 2459 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm2], $Zn, $Zm",2460 (!cast<Instruction>(NAME #_HtoS) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm2, ZZZZ_h_mul_r:$Zn, ZZZZ_h_mul_r:$Zm), 0>;2461}2462 2463//===----------------------------------------------------------------------===//2464class sme2_frint_cvt_vg2_multi<bits<2>sz, bits<5>op, RegisterOperand first_ty,2465 RegisterOperand second_ty, string mnemonic>2466 : I<(outs first_ty:$Zd), (ins second_ty:$Zn),2467 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> {2468 bits<4> Zn;2469 bits<4> Zd;2470 let Inst{31-24} = 0b11000001;2471 let Inst{23-22} = sz;2472 let Inst{21-20} = 0b10;2473 let Inst{19-16} = op{4-1};2474 let Inst{15-10} = 0b111000;2475 let Inst{9-6} = Zn;2476 let Inst{5} = op{0};2477 let Inst{4-1} = Zd;2478 let Inst{0} = 0b0;2479}2480 2481// SME2 multi-vec FP to int convert two registers2482// SME2 multi-vec int to FP two registers2483multiclass sme2_fp_cvt_vg2_multi<string mnemonic, bits<5> op> {2484 def NAME : sme2_frint_cvt_vg2_multi<0b00, op, ZZ_s_mul_r, ZZ_s_mul_r, mnemonic>;2485}2486 2487// SME2 multi-vec FRINT two registers2488multiclass sme2_frint_vector_vg2_multi<string mnemonic, bits<5> op> {2489 def _S : sme2_frint_cvt_vg2_multi<0b10, op, ZZ_s_mul_r, ZZ_s_mul_r, mnemonic>;2490}2491 2492class sme2_frint_zip_cvt_vg4_multi<bits<2>sz, bits<7>op, RegisterOperand first_ty,2493 RegisterOperand second_ty, string mnemonic>2494 : I<(outs first_ty:$Zd), (ins second_ty:$Zn),2495 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> {2496 bits<3> Zn;2497 bits<3> Zd;2498 let Inst{31-24} = 0b11000001;2499 let Inst{23-22} = sz;2500 let Inst{21-20} = 0b11;2501 let Inst{19-16} = op{6-3};2502 let Inst{15-10} = 0b111000;2503 let Inst{9-7} = Zn;2504 let Inst{6-5} = op{2-1};2505 let Inst{4-2} = Zd;2506 let Inst{1} = op{0};2507 let Inst{0} = 0b0;2508}2509 2510// SME2 multi-vec FP to int convert four registers2511// SME2 multi-vec int to FP four registers2512multiclass sme2_fp_cvt_vg4_multi<string mnemonic, bits<7> op> {2513 def NAME : sme2_frint_zip_cvt_vg4_multi<0b00, op, ZZZZ_s_mul_r, ZZZZ_s_mul_r, mnemonic>;2514}2515 2516// SME2 multi-vec quadwords ZIP four registers2517multiclass sme2_zip_vector_vg4<string mnemonic, bits<7> op> {2518 def _B : sme2_frint_zip_cvt_vg4_multi<0b00, op, ZZZZ_b_mul_r, ZZZZ_b_mul_r,2519 mnemonic>;2520 def _H : sme2_frint_zip_cvt_vg4_multi<0b01, op, ZZZZ_h_mul_r, ZZZZ_h_mul_r,2521 mnemonic>;2522 def _S : sme2_frint_zip_cvt_vg4_multi<0b10, op, ZZZZ_s_mul_r, ZZZZ_s_mul_r,2523 mnemonic>;2524 def _D : sme2_frint_zip_cvt_vg4_multi<0b11, op, ZZZZ_d_mul_r, ZZZZ_d_mul_r,2525 mnemonic>;2526}2527 2528// SME2 multi-vec quadwords ZIP four registers2529multiclass sme2_zip_vector_vg4_Q<string mnemonic, bits<7> op> {2530 def NAME: sme2_frint_zip_cvt_vg4_multi<0b00, op, ZZZZ_q_mul_r, ZZZZ_q_mul_r,2531 mnemonic>;2532}2533 2534// SME2 multi-vec FRINT four registers2535multiclass sme2_frint_vector_vg4_multi<string mnemonic, bits<7> op> {2536 def _S : sme2_frint_zip_cvt_vg4_multi<0b10, op, ZZZZ_s_mul_r, ZZZZ_s_mul_r,2537 mnemonic>;2538}2539 2540class sme2_cvt_vg2_single<string mnemonic, bits<5> op,2541 RegisterOperand first_ty, RegisterOperand second_ty>2542 : I<(outs first_ty:$Zd), (ins second_ty:$Zn),2543 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> {2544 bits<4> Zn;2545 bits<5> Zd;2546 let Inst{31-23} = 0b110000010;2547 let Inst{22} = op{4};2548 let Inst{21-19} = 0b100;2549 let Inst{18-16} = op{3-1};2550 let Inst{15-10} = 0b111000;2551 let Inst{9-6} = Zn;2552 let Inst{5} = op{0};2553 let Inst{4-0} = Zd;2554}2555 2556// SME2 multi-vec FP down convert two registers2557// SME2 multi-vec int down convert two registers2558multiclass sme2_cvt_vg2_single<string mnemonic, bits<5> op, ValueType out_vt,2559 ValueType in_vt, SDPatternOperator intrinsic> {2560 def NAME : sme2_cvt_vg2_single<mnemonic, op, ZPR16, ZZ_s_mul_r>;2561 def : SVE2p1_Cvt_VG2_Pat<NAME, intrinsic, out_vt, in_vt>;2562}2563 2564// SME2 multi-vec FP8 down convert two registers2565multiclass sme2_fp8_cvt_vg2_single<string mnemonic, bit op, ValueType in_vt, SDPatternOperator intrinsic> {2566 def NAME : sme2_cvt_vg2_single<mnemonic, {op, 0b1000}, ZPR8, ZZ_h_mul_r>{2567 let mayLoad = 1;2568 let mayStore = 0;2569 let Uses = [FPMR, FPCR];2570 }2571 def : Pat<(nxv16i8 (intrinsic in_vt:$Zn1, in_vt:$Zn2)),2572 (!cast<Instruction>(NAME) (REG_SEQUENCE ZPR2Mul2, in_vt:$Zn1, zsub0, in_vt:$Zn2, zsub1))>;2573}2574 2575class sme2_cvt_unpk_vector_vg2<bits<2>sz, bits<3> op, bit u, RegisterOperand first_ty,2576 RegisterOperand second_ty, string mnemonic>2577 : I<(outs first_ty:$Zd), (ins second_ty:$Zn),2578 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> {2579 bits<5> Zn;2580 bits<4> Zd;2581 let Inst{31-24} = 0b11000001;2582 let Inst{23-22} = sz;2583 let Inst{21-19} = 0b100;2584 let Inst{18-16} = op;2585 let Inst{15-10} = 0b111000;2586 let Inst{9-5} = Zn;2587 let Inst{4-1} = Zd;2588 let Inst{0} = u;2589}2590 2591// SME2 multi-vec unpack two registers2592multiclass sme2_unpk_vector_vg2<string mnemonic, bit u> {2593 def _H : sme2_cvt_unpk_vector_vg2<0b01, 0b101, u, ZZ_h_mul_r, ZPR8, mnemonic>;2594 def _S : sme2_cvt_unpk_vector_vg2<0b10, 0b101, u, ZZ_s_mul_r, ZPR16, mnemonic>;2595 def _D : sme2_cvt_unpk_vector_vg2<0b11, 0b101, u, ZZ_d_mul_r, ZPR32, mnemonic>;2596}2597 2598// SME2.1 multi-vec convert two registers2599multiclass sme2p1_fp_cvt_vector_vg2_single<string mnemonic, bit l> {2600 def _S : sme2_cvt_unpk_vector_vg2<0b10, 0b000, l, ZZ_s_mul_r, ZPR16, mnemonic>;2601}2602 2603// SME2 multi-vec FP8 up convert two registers2604multiclass sme2p1_fp8_cvt_vector_vg2_single<string mnemonic, bits<2> opc, bit L> {2605 def NAME : sme2_cvt_unpk_vector_vg2<opc, 0b110, L, ZZ_h_mul_r, ZPR8, mnemonic>{2606 let Uses = [FPMR, FPCR];2607 }2608}2609 2610 2611class sme2_cvt_vg4_single<bit sz, bits<3> op, bits<4>op2, RegisterOperand first_ty,2612 RegisterOperand second_ty, string mnemonic>2613 : I<(outs first_ty:$Zd), (ins second_ty:$Zn),2614 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> {2615 bits<3> Zn;2616 bits<5> Zd;2617 let Inst{31-24} = 0b11000001;2618 let Inst{23} = sz;2619 let Inst{22} = op{2};2620 let Inst{21-20} = 0b11;2621 let Inst{19-16} = op2;2622 let Inst{15-10} = 0b111000;2623 let Inst{9-7} = Zn;2624 let Inst{6-5} = op{1-0};2625 let Inst{4-0} = Zd;2626}2627 2628// SME2 multi-vec int down convert four registers2629multiclass sme2_int_cvt_vg4_single<string mnemonic, bits<3> op, SDPatternOperator intrinsic> {2630 def _StoB : sme2_cvt_vg4_single<0, op, 0b0011, ZPR8, ZZZZ_s_mul_r, mnemonic>;2631 def _DtoH : sme2_cvt_vg4_single<1, op, 0b0011, ZPR16, ZZZZ_d_mul_r, mnemonic>;2632 2633 def : SME2_Cvt_VG4_Pat<NAME # _StoB, intrinsic, nxv16i8, nxv4i32>;2634 def : SME2_Cvt_VG4_Pat<NAME # _DtoH, intrinsic, nxv8i16, nxv2i64>;2635}2636 2637//SME2 multi-vec FP8 down convert four registers2638multiclass sme2_fp8_cvt_vg4_single<string mnemonic, bit N, SDPatternOperator intrinsic> {2639 def NAME : sme2_cvt_vg4_single<0b0, {0b00, N}, 0b0100, ZPR8, ZZZZ_s_mul_r, mnemonic> {2640 let mayLoad = 1;2641 let mayStore = 0;2642 let Uses = [FPMR, FPCR];2643 }2644 def : SME2_Cvt_VG4_Pat<NAME, intrinsic, nxv16i8, nxv4f32>;2645}2646 2647class sme2_unpk_vector_vg4<bits<2>sz, bit u, RegisterOperand first_ty,2648 RegisterOperand second_ty, string mnemonic>2649 : I<(outs first_ty:$Zd), (ins second_ty:$Zn),2650 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> {2651 bits<4> Zn;2652 bits<3> Zd;2653 let Inst{31-24} = 0b11000001;2654 let Inst{23-22} = sz;2655 let Inst{21-10} = 0b110101111000;2656 let Inst{9-6} = Zn;2657 let Inst{5} = 0b0;2658 let Inst{4-2} = Zd;2659 let Inst{1} = 0b0;2660 let Inst{0} = u;2661}2662 2663// SME2 multi-vec UNPK four registers2664multiclass sme2_unpk_vector_vg4<string mnemonic, bit u> {2665 def _H : sme2_unpk_vector_vg4<0b01, u, ZZZZ_h_mul_r, ZZ_b_mul_r, mnemonic>;2666 def _S : sme2_unpk_vector_vg4<0b10, u, ZZZZ_s_mul_r, ZZ_h_mul_r, mnemonic>;2667 def _D : sme2_unpk_vector_vg4<0b11, u, ZZZZ_d_mul_r, ZZ_s_mul_r, mnemonic>;2668}2669 2670//===----------------------------------------------------------------------===//2671// SME2 multi-vec CLAMP registers2672 2673class sme2_clamp_vector_vg24_multi<bits<2> sz, bits<3> op1, bit u,2674 RegisterOperand multi_vector_ty,2675 ZPRRegOp vector_ty, string mnemonic>2676 : I<(outs multi_vector_ty:$Zd),2677 (ins multi_vector_ty:$_Zd, vector_ty:$Zn, vector_ty:$Zm),2678 mnemonic, "\t$Zd, $Zn, $Zm",2679 "", []>, Sched<[]>{2680 bits<5> Zm;2681 bits<5> Zn;2682 let Inst{31-24} = 0b11000001;2683 let Inst{23-22} = sz;2684 let Inst{21} = 0b1;2685 let Inst{20-16} = Zm;2686 let Inst{15-13} = 0b110;2687 let Inst{12-10} = op1;2688 let Inst{9-5} = Zn;2689 let Inst{0} = u;2690 2691 let Constraints = "$Zd = $_Zd";2692}2693 2694class sme2_clamp_vector_vg2_multi<bits<2> sz, bits<3> op1, bit u,2695 RegisterOperand multi_vector_ty,2696 ZPRRegOp vector_ty, string mnemonic>2697 : sme2_clamp_vector_vg24_multi<sz, op1, u, multi_vector_ty, vector_ty,2698 mnemonic>{2699 bits<4> Zd;2700 let Inst{4-1} = Zd;2701}2702 2703multiclass sme2_fp_clamp_vector_vg2_multi<string mnemonic>{2704 def _H : sme2_clamp_vector_vg2_multi<0b01, 0b000, 0b0, ZZ_h_mul_r, ZPR16, mnemonic>;2705 def _S : sme2_clamp_vector_vg2_multi<0b10, 0b000, 0b0, ZZ_s_mul_r, ZPR32, mnemonic>;2706 def _D : sme2_clamp_vector_vg2_multi<0b11, 0b000, 0b0, ZZ_d_mul_r, ZPR64, mnemonic>;2707}2708 2709multiclass sme2_int_clamp_vector_vg2_multi<string mnemonic, bit u>{2710 def _B : sme2_clamp_vector_vg2_multi<0b00, 0b001, u, ZZ_b_mul_r, ZPR8, mnemonic>;2711 def _H : sme2_clamp_vector_vg2_multi<0b01, 0b001, u, ZZ_h_mul_r, ZPR16, mnemonic>;2712 def _S : sme2_clamp_vector_vg2_multi<0b10, 0b001, u, ZZ_s_mul_r, ZPR32, mnemonic>;2713 def _D : sme2_clamp_vector_vg2_multi<0b11, 0b001, u, ZZ_d_mul_r, ZPR64, mnemonic>;2714}2715 2716// SME2.1 multi-vec FCLAMP two registers2717multiclass sme2p1_bfclamp_vector_vg2_multi<string mnemonic> {2718 def _H : sme2_clamp_vector_vg2_multi<0b00, 0b000, 0b0, ZZ_h_mul_r, ZPR16,2719 mnemonic>;2720}2721 2722class sme2_clamp_vector_vg4_multi<bits<2> sz, bits<3> op1, bit u,2723 RegisterOperand multi_vector_ty,2724 ZPRRegOp vector_ty, string mnemonic>2725 : sme2_clamp_vector_vg24_multi<sz, op1, u, multi_vector_ty, vector_ty,2726 mnemonic>{2727 bits<3> Zd;2728 let Inst{4-2} = Zd;2729 let Inst{1} = 0b0;2730}2731 2732multiclass sme2_fp_clamp_vector_vg4_multi<string mnemonic>{2733 def _H : sme2_clamp_vector_vg4_multi<0b01, 0b010, 0b0, ZZZZ_h_mul_r, ZPR16, mnemonic>;2734 def _S : sme2_clamp_vector_vg4_multi<0b10, 0b010, 0b0, ZZZZ_s_mul_r, ZPR32, mnemonic>;2735 def _D : sme2_clamp_vector_vg4_multi<0b11, 0b010, 0b0, ZZZZ_d_mul_r, ZPR64, mnemonic>;2736}2737 2738multiclass sme2_int_clamp_vector_vg4_multi<string mnemonic, bit u>{2739 def _B : sme2_clamp_vector_vg4_multi<0b00, 0b011, u, ZZZZ_b_mul_r, ZPR8, mnemonic>;2740 def _H : sme2_clamp_vector_vg4_multi<0b01, 0b011, u, ZZZZ_h_mul_r, ZPR16, mnemonic>;2741 def _S : sme2_clamp_vector_vg4_multi<0b10, 0b011, u, ZZZZ_s_mul_r, ZPR32, mnemonic>;2742 def _D : sme2_clamp_vector_vg4_multi<0b11, 0b011, u, ZZZZ_d_mul_r, ZPR64, mnemonic>;2743}2744 2745// SME2.1 multi-vec FCLAMP four registers2746multiclass sme2p1_bfclamp_vector_vg4_multi<string mnemonic> {2747 def _H : sme2_clamp_vector_vg4_multi<0b00, 0b010, 0b0, ZZZZ_h_mul_r, ZPR16,2748 mnemonic>;2749}2750 2751// SME2 multi-vec ZIP two registers2752class sme2_zip_vector_vg2<bits<2> sz, bit q, bit u,2753 RegisterOperand multi_vector_ty,2754 ZPRRegOp vector_ty, string mnemonic>2755 : I<(outs multi_vector_ty:$Zd), (ins vector_ty:$Zn, vector_ty:$Zm),2756 mnemonic, "\t$Zd, $Zn, $Zm",2757 "", []>, Sched<[]>{2758 bits<4> Zd;2759 bits<5> Zm;2760 bits<5> Zn;2761 let Inst{31-24} = 0b11000001;2762 let Inst{23-22} = sz;2763 let Inst{21} = 0b1;2764 let Inst{20-16} = Zm;2765 let Inst{15-11} = 0b11010;2766 let Inst{10} = q;2767 let Inst{9-5} = Zn;2768 let Inst{4-1} = Zd;2769 let Inst{0} = u;2770}2771 2772multiclass sme2_zip_vector_vg2<string mnemonic, bit op> {2773 def _B : sme2_zip_vector_vg2<0b00, 0b0, op, ZZ_b_mul_r, ZPR8, mnemonic>;2774 def _H : sme2_zip_vector_vg2<0b01, 0b0, op, ZZ_h_mul_r, ZPR16, mnemonic>;2775 def _S : sme2_zip_vector_vg2<0b10, 0b0, op, ZZ_s_mul_r, ZPR32, mnemonic>;2776 def _D : sme2_zip_vector_vg2<0b11, 0b0, op, ZZ_d_mul_r, ZPR64, mnemonic>;2777 def _Q : sme2_zip_vector_vg2<0b00, 0b1, op, ZZ_q_mul_r, ZPR128, mnemonic>;2778}2779 2780//===----------------------------------------------------------------------===//2781// SME2 Dot Products and MLA2782class sme2_multi_vec_array_vg2_index<bits<2> sz, bits<6> op, MatrixOperand matrix_ty,2783 RegisterOperand multi_vector_ty,2784 ZPRRegOp vector_ty, Operand index_ty,2785 string mnemonic>2786 : I<(outs matrix_ty:$ZAda),2787 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,2788 multi_vector_ty:$Zn, vector_ty:$Zm, index_ty:$i),2789 mnemonic, "\t$ZAda[$Rv, $imm3, vgx2], $Zn, $Zm$i",2790 "", []>, Sched<[]> {2791 bits<0> ZAda;2792 bits<4> Zm;2793 bits<2> Rv;2794 bits<4> Zn;2795 bits<3> imm3;2796 let Inst{31-24} = 0b11000001;2797 let Inst{23-22} = sz;2798 let Inst{21-20} = 0b01;2799 let Inst{19-16} = Zm;2800 let Inst{15} = 0b0;2801 let Inst{14-13} = Rv;2802 let Inst{12-10} = op{5-3};2803 let Inst{9-6} = Zn;2804 let Inst{5-3} = op{2-0};2805 let Inst{2-0} = imm3;2806 2807 let Constraints = "$ZAda = $_ZAda";2808}2809 2810// SME2 multi-vec ternary indexed two registers 32-bit2811multiclass sme2_multi_vec_array_vg2_index_32b<string mnemonic, bits<2> sz, bits<4> op,2812 RegisterOperand multi_vector_ty,2813 ZPRRegOp vector_ty, ValueType vt,2814 SDPatternOperator intrinsic> {2815 def NAME : sme2_multi_vec_array_vg2_index<sz, {op{3},?,?,op{2-0}}, MatrixOp32, multi_vector_ty, vector_ty,2816 VectorIndexS32b_timm, mnemonic>, SMEPseudo2Instr<NAME, 1> {2817 bits<2> i;2818 let Inst{11-10} = i;2819 }2820 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexS32b_timm, SMEMatrixArray>;2821 2822 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vt, VectorIndexS32b_timm, tileslice16>;2823 2824 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i",2825 (!cast<Instruction>(NAME) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,2826 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexS32b_timm:$i), 0>;2827}2828 2829// SME2.1 multi-vec ternary indexed two registers 16-bit2830multiclass sme2p1_multi_vec_array_vg2_index_16b<string mnemonic, bits<2> sz, bits<3> op,2831 RegisterOperand multi_vector_ty, ZPRRegOp vector_ty,2832 ValueType vt, SDPatternOperator intrinsic> {2833 def NAME : sme2_multi_vec_array_vg2_index<sz, {op{2},?,?,op{1-0},?}, MatrixOp16,2834 multi_vector_ty, vector_ty,2835 VectorIndexH, mnemonic>, SMEPseudo2Instr<NAME, 1> {2836 bits<3> i;2837 let Inst{11-10} = i{2-1};2838 let Inst{3} = i{0};2839 }2840 2841 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexH32b, SMEMatrixArray>;2842 2843 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vt, VectorIndexH32b_timm, tileslice16>;2844 2845 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i",2846 (!cast<Instruction>(NAME) MatrixOp16:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,2847 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexH:$i), 0>;2848}2849 2850// SME2 multi-vec indexed FP8 two-way dot product to FP16 two registers2851multiclass sme2p1_multi_vec_array_vg2_index_f8f16<string mnemonic, bits<2> sz, bits<3> op,2852 RegisterOperand multi_vector_ty, ZPRRegOp zpr_ty> {2853 def NAME : sme2_multi_vec_array_vg2_index<sz, {op{2},?,?,op{1-0},?}, MatrixOp16,2854 multi_vector_ty, zpr_ty,2855 VectorIndexH, mnemonic> {2856 bits<3> i;2857 let Inst{11-10} = i{2-1};2858 let Inst{3} = i{0};2859 }2860 2861 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i",2862 (!cast<Instruction>(NAME) MatrixOp16:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,2863 multi_vector_ty:$Zn, zpr_ty:$Zm, VectorIndexH:$i), 0>;2864}2865 2866// SME2 multi-vec indexed FP8 two-way vertical dot product to single precision2867// two registers2868class sme2_fp8_multi_vec_array_vg4_index<string mnemonic, bit T>2869 : sme2_multi_vec_array_vg2_index<0b11, {0b01,?,0b0, T,?}, MatrixOp32,2870 ZZ_b_mul_r, ZPR4b8, VectorIndexS, mnemonic> {2871 2872 bits<2> i;2873 let Inst{10} = i{1};2874 let Inst{3} = i{0};2875 let AsmString = !strconcat(mnemonic, "{\t$ZAda[$Rv, $imm3, vgx4], $Zn, $Zm$i}");2876 let Uses = [FPMR, FPCR];2877}2878 2879// SME2 multi-vec ternary indexed two registers 64-bit2880 2881class sme2_multi_vec_array_vg2_index_64b<bits<2> op,2882 RegisterOperand multi_vector_ty,2883 ZPRRegOp vector_ty,2884 string mnemonic>2885 : I<(outs MatrixOp64:$ZAda),2886 (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,2887 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1),2888 mnemonic, "\t$ZAda[$Rv, $imm3, vgx2], $Zn, $Zm$i1",2889 "", []>, Sched<[]> {2890 bits<0> ZAda;2891 bits<4> Zm;2892 bits<2> Rv;2893 bits<1> i1;2894 bits<4> Zn;2895 bits<3> imm3;2896 let Inst{31-20} = 0b110000011101;2897 let Inst{19-16} = Zm;2898 let Inst{15} = 0b0;2899 let Inst{14-13} = Rv;2900 let Inst{12-11} = 0b00;2901 let Inst{10} = i1;2902 let Inst{9-6} = Zn;2903 let Inst{5} = 0b0;2904 let Inst{4-3} = op;2905 let Inst{2-0} = imm3;2906 2907 let Constraints = "$ZAda = $_ZAda";2908}2909 2910multiclass sme2_multi_vec_array_vg2_index_64b<string mnemonic, bits<2> op,2911 RegisterOperand multi_vector_ty,2912 ZPRRegOp vector_ty, ValueType vt,2913 SDPatternOperator intrinsic> {2914 def NAME : sme2_multi_vec_array_vg2_index_64b<op, multi_vector_ty, vector_ty,2915 mnemonic>, SMEPseudo2Instr<NAME, 1>;2916 2917 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexD32b_timm, SMEMatrixArray>;2918 2919 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vt, VectorIndexD32b_timm, tileslice16>;2920 2921 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i1",2922 (!cast<Instruction>(NAME) MatrixOp64:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,2923 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1), 0>;2924}2925 2926class sme2_multi_vec_array_vg4_index<bit sz, bits<7> op, MatrixOperand matrix_ty,2927 RegisterOperand multi_vector_ty,2928 ZPRRegOp vector_ty, Operand index_ty,2929 string mnemonic>2930 : I<(outs matrix_ty:$ZAda),2931 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,2932 multi_vector_ty:$Zn, vector_ty:$Zm, index_ty:$i),2933 mnemonic, "\t$ZAda[$Rv, $imm3, vgx4], $Zn, $Zm$i",2934 "", []>, Sched<[]> {2935 bits<0> ZAda;2936 bits<4> Zm;2937 bits<2> Rv;2938 bits<3> Zn;2939 bits<3> imm3;2940 let Inst{31-23} = 0b110000010;2941 let Inst{22} = sz;2942 let Inst{21-20} = 0b01;2943 let Inst{19-16} = Zm;2944 let Inst{15} = 0b1;2945 let Inst{14-13} = Rv;2946 let Inst{12-10} = op{6-4};2947 let Inst{9-7} = Zn;2948 let Inst{6-3} = op{3-0};2949 let Inst{2-0} = imm3;2950 2951 let Constraints = "$ZAda = $_ZAda";2952}2953 2954// SME2 multi-vec ternary indexed four registers 32-bit2955multiclass sme2_multi_vec_array_vg4_index_32b<string mnemonic, bits<4> op,2956 RegisterOperand multi_vector_ty,2957 ZPRRegOp vector_ty, ValueType vt,2958 SDPatternOperator intrinsic> {2959 def NAME : sme2_multi_vec_array_vg4_index<0b1, {op{3},?,?,0b0, op{2-0}}, MatrixOp32, multi_vector_ty,2960 vector_ty, VectorIndexS32b_timm, mnemonic>, SMEPseudo2Instr<NAME, 1> {2961 bits<2> i;2962 let Inst{11-10} = i;2963 }2964 2965 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexS32b_timm, SMEMatrixArray>;2966 2967 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vt, VectorIndexS32b_timm, tileslice16>;2968 2969 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i",2970 (!cast<Instruction>(NAME) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,2971 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexS32b_timm:$i), 0>;2972}2973 2974// SME2.1 multi-vec ternary indexed four registers 16-bit (FP8)2975multiclass sme2p1_multi_vec_array_vg4_index_f8f16<string mnemonic, bits<3> op,2976 RegisterOperand multi_vector_ty,2977 ZPRRegOp zpr_ty> {2978 def NAME : sme2_multi_vec_array_vg4_index<0b0,{0b1,?,?,op,?}, MatrixOp16,2979 multi_vector_ty, zpr_ty,2980 VectorIndexH, mnemonic>{2981 bits<3> i;2982 let Inst{11-10} = i{2-1};2983 let Inst{3} = i{0};2984 }2985 2986 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i",2987 (!cast<Instruction>(NAME) MatrixOp16:$ZAda, MatrixIndexGPR32Op8_11:$Rv,2988 sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm, VectorIndexH:$i), 0>;2989}2990 2991// SME2.1 multi-vec ternary indexed four registers 16-bit2992multiclass sme2p1_multi_vec_array_vg4_index_16b<string mnemonic, bits<3> op,2993 RegisterOperand multi_vector_ty,2994 ZPRRegOp vector_ty, ValueType vt,2995 SDPatternOperator intrinsic> {2996 def NAME : sme2_multi_vec_array_vg4_index<0b0,{0b1,?,?,op,?}, MatrixOp16,2997 multi_vector_ty, vector_ty,2998 VectorIndexH, mnemonic>, SMEPseudo2Instr<NAME, 1> {2999 bits<3> i;3000 let Inst{11-10} = i{2-1};3001 let Inst{3} = i{0};3002 }3003 3004 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexH32b_timm, SMEMatrixArray>;3005 3006 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vt, VectorIndexH32b_timm, tileslice16>;3007 3008 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i",3009 (!cast<Instruction>(NAME) MatrixOp16:$ZAda, MatrixIndexGPR32Op8_11:$Rv,3010 sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexH:$i), 0>;3011}3012 3013// SME2 multi-vec ternary indexed four registers 64-bit3014class sme2_multi_vec_array_vg4_index_64b<bits<3> op,3015 RegisterOperand multi_vector_ty,3016 ZPRRegOp vector_ty,3017 string mnemonic>3018 : I<(outs MatrixOp64:$ZAda),3019 (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,3020 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1),3021 mnemonic, "\t$ZAda[$Rv, $imm3, vgx4], $Zn, $Zm$i1",3022 "", []>, Sched<[]> {3023 bits<0> ZAda;3024 bits<4> Zm;3025 bits<2> Rv;3026 bits<1> i1;3027 bits<3> Zn;3028 bits<3> imm3;3029 let Inst{31-20} = 0b110000011101;3030 let Inst{19-16} = Zm;3031 let Inst{15} = 0b1;3032 let Inst{14-13} = Rv;3033 let Inst{12} = 0b0;3034 let Inst{11} = op{2};3035 let Inst{10} = i1;3036 let Inst{9-7} = Zn;3037 let Inst{6-5} = 0b00;3038 let Inst{4-3} = op{1-0};3039 let Inst{2-0} = imm3;3040 3041 let Constraints = "$ZAda = $_ZAda";3042}3043 3044multiclass sme2_multi_vec_array_vg4_index_64b<string mnemonic, bits<3> op,3045 RegisterOperand multi_vector_ty,3046 ZPRRegOp vector_ty, ValueType vty,3047 SDPatternOperator intrinsic> {3048 def NAME : sme2_multi_vec_array_vg4_index_64b<op, multi_vector_ty, vector_ty,3049 mnemonic>, SMEPseudo2Instr<NAME, 1>;3050 3051 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexD32b_timm, SMEMatrixArray>;3052 3053 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vty, VectorIndexD32b_timm, tileslice16>;3054 3055 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i1",3056 (!cast<Instruction>(NAME) MatrixOp64:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,3057 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1), 0>;3058}3059 3060// FMLAL (multiple and indexed vector, FP8 to FP16)3061class sme2_fp8_fmlal_vg24_index_za16<bits<2> sz, bit vg4, bits<3> op,3062 RegisterOperand multi_vector_ty, string mnemonic>3063 : I<(outs MatrixOp16:$ZAda),3064 (ins MatrixOp16:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm2,3065 multi_vector_ty:$Zn, ZPR4b8:$Zm, VectorIndexB32b_timm:$i),3066 mnemonic, "\t$ZAda[$Rv, $imm2, " # !if(vg4, "vgx4", "vgx2") # "], $Zn, $Zm$i",3067 "", []>, Sched<[]> {3068 bits<0> ZAda;3069 bits<4> Zm;3070 bits<2> Rv;3071 bits<4> i;3072 bits<2> imm2;3073 let Inst{31-24} = 0b11000001;3074 let Inst{23-22} = sz;3075 let Inst{21-20} = 0b01;3076 let Inst{19-16} = Zm;3077 let Inst{15} = vg4;3078 let Inst{14-13} = Rv;3079 let Inst{12} = op{2};3080 let Inst{11-10} = i{3-2};3081 let Inst{5-4} = op{1-0};3082 let Inst{3-2} = i{1-0};3083 let Inst{1-0} = imm2;3084 3085 let Uses = [FPMR, FPCR];3086 let Constraints = "$ZAda = $_ZAda";3087}3088 3089multiclass sme2_fp8_fmlal_index_za16_vgx2<string mnemonic, SDPatternOperator intrinsic> {3090 def NAME : sme2_fp8_fmlal_vg24_index_za16<0b10, 0b0, 0b111, ZZ_b_mul_r, mnemonic>, SMEPseudo2Instr<NAME, 1> {3091 bits<4> Zn;3092 let Inst{9-6} = Zn;3093 }3094 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, uimm2s2range, ZZ_b_mul_r, ZPR4b8, VectorIndexB32b_timm, SMEMatrixArray>;3095 3096 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, uimm2s2range, ZPR4b8, nxv16i8, VectorIndexB32b_timm, tileslicerange2s2>;3097 3098 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm2], $Zn, $Zm$i",3099 (!cast<Instruction>(NAME) MatrixOp16:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm2,3100 ZZ_b_mul_r:$Zn, ZPR4b8:$Zm, VectorIndexB32b_timm:$i), 0>;3101}3102 3103multiclass sme2_fp8_fmlal_index_za16_vgx4<string mnemonic, SDPatternOperator intrinsic> {3104 def NAME: sme2_fp8_fmlal_vg24_index_za16<0b10, 0b1, 0b110, ZZZZ_b_mul_r, mnemonic>, SMEPseudo2Instr<NAME, 1> {3105 bits<3> Zn;3106 let Inst{9-7} = Zn;3107 let Inst{6} = 0b0;3108 }3109 3110 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, uimm2s2range, ZZZZ_b_mul_r, ZPR4b8, VectorIndexB32b_timm, SMEMatrixArray>;3111 3112 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, uimm2s2range, ZPR4b8, nxv16i8, VectorIndexB32b_timm, tileslicerange2s2>;3113 3114 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i",3115 (!cast<Instruction>(NAME) MatrixOp16:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm,3116 ZZZZ_b_mul_r:$Zn, ZPR4b8:$Zm, VectorIndexB32b_timm:$i), 0>;3117}3118 3119//===----------------------------------------------------------------------===//3120// FMLAL (single and indexed vector, FP8 to FP16)3121class sme2_fp8_fmlal_index_za16<string mnemonic, bits<2> sz,bits<2> op>3122 : I<(outs MatrixOp16:$ZAda),3123 (ins MatrixOp16:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm3s2range:$imm3, ZPR8:$Zn, ZPR4b8:$Zm, VectorIndexB32b_timm:$i),3124 mnemonic, "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i",3125 "", []>, Sched<[]> {3126 bits<0> ZAda;3127 bits<4> Zm;3128 bits<2> Rv;3129 bits<4> i;3130 bits<5> Zn;3131 bits<3> imm3;3132 let Inst{31-24} = 0b11000001;3133 let Inst{23-22} = sz;3134 let Inst{21-20} = 0b00;3135 let Inst{19-16} = Zm;3136 let Inst{15} = i{3};3137 let Inst{14-13} = Rv;3138 let Inst{12} = op{1};3139 let Inst{11-10} = i{2-1};3140 let Inst{9-5} = Zn;3141 let Inst{4} = op{0};3142 let Inst{3} = i{0};3143 let Inst{2-0} = imm3;3144 3145 let Uses = [FPMR, FPCR];3146 let Constraints = "$ZAda = $_ZAda";3147}3148 3149multiclass sme2_fp8_fmlal_index_za16<string mnemonic, SDPatternOperator intrinsic> {3150 def NAME : sme2_fp8_fmlal_index_za16<mnemonic, 0b11, 0b00>, SMEPseudo2Instr<NAME, 1>;3151 3152 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, uimm3s2range, ZPR8, ZPR4b8, VectorIndexB32b_timm, SMEMatrixArray>;3153 3154 def : SME2_ZA_TwoOp_Multi_Index_Pat<NAME, intrinsic, uimm3s2range, ZPR4b8, nxv16i8, VectorIndexB32b_timm, tileslicerange3s2>;3155}3156 3157// SME2 multi-vec indexed long long MLA one source 32-bit3158class sme2_mla_ll_array_index_32b<string mnemonic, bits<2> sz, bits<3> op>3159 : I<(outs MatrixOp32:$ZAda),3160 (ins MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s4range:$imm2, ZPR8:$Zn, ZPR4b8:$Zm, VectorIndexB32b_timm:$i),3161 mnemonic, "\t$ZAda[$Rv, $imm2], $Zn, $Zm$i",3162 "", []>, Sched<[]> {3163 bits<0> ZAda;3164 bits<4> Zm;3165 bits<2> Rv;3166 bits<4> i;3167 bits<5> Zn;3168 bits<2> imm2;3169 let Inst{31-24} = 0b11000001;3170 let Inst{23-22} = sz;3171 let Inst{21-20} = 0b00;3172 let Inst{19-16} = Zm;3173 let Inst{15} = i{3};3174 let Inst{14-13} = Rv;3175 let Inst{12-10} = i{2-0};3176 let Inst{9-5} = Zn;3177 let Inst{4-2} = op;3178 let Inst{1-0} = imm2;3179 3180 let Constraints = "$ZAda = $_ZAda";3181}3182 3183multiclass sme2_mla_ll_array_index_32b<string mnemonic, bits<2> sz, bits<3> op, SDPatternOperator intrinsic, list<Register> uses=[]> {3184 def NAME : sme2_mla_ll_array_index_32b<mnemonic, sz, op>, SMEPseudo2Instr<NAME, 1> {3185 let Uses = uses;3186 }3187 3188 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, uimm2s4range, ZPR8, ZPR4b8, VectorIndexB32b_timm, SMEMatrixArray>;3189 3190 def : SME2_ZA_TwoOp_Multi_Index_Pat<NAME, intrinsic, uimm2s4range, ZPR4b8, nxv16i8, VectorIndexB32b_timm, tileslicerange2s4>;3191}3192 3193// SME2 multi-vec indexed long long MLA one source 64-bit3194 3195class sme2_mla_ll_array_index_64b<string mnemonic, bits<2> op>3196 : I<(outs MatrixOp64:$ZAda),3197 (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s4range:$imm2, ZPR16:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i),3198 mnemonic, "\t$ZAda[$Rv, $imm2], $Zn, $Zm$i",3199 "", []>, Sched<[]> {3200 bits<0> ZAda;3201 bits<4> Zm;3202 bits<2> Rv;3203 bits<3> i;3204 bits<5> Zn;3205 bits<2> imm2;3206 let Inst{31-20} = 0b110000011000;3207 let Inst{19-16} = Zm;3208 let Inst{15} = i{2};3209 let Inst{14-13} = Rv;3210 let Inst{12} = 0b0;3211 let Inst{11-10} = i{1-0};3212 let Inst{9-5} = Zn;3213 let Inst{4-3} = op;3214 let Inst{2} = 0b0;3215 let Inst{1-0} = imm2;3216 3217 let Constraints = "$ZAda = $_ZAda";3218}3219 3220multiclass sme2_mla_ll_array_index_64b<string mnemonic, bits<2> op, SDPatternOperator intrinsic> {3221 def NAME : sme2_mla_ll_array_index_64b<mnemonic, op>, SMEPseudo2Instr<NAME, 1>;3222 3223 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, uimm2s4range, ZPR16, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>;3224 3225 def : SME2_ZA_TwoOp_Multi_Index_Pat<NAME, intrinsic, uimm2s4range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange2s4>;3226}3227 3228class sme2_mla_ll_array_vg24_index_32b<bits<2> sz, bit vg4, bits<3> op,3229 RegisterOperand vector_ty,3230 string mnemonic>3231 : I<(outs MatrixOp32:$ZAda),3232 (ins MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm,3233 vector_ty:$Zn, ZPR4b8:$Zm, VectorIndexB32b_timm:$i),3234 mnemonic, "\t$ZAda[$Rv, $imm, " # !if(vg4, "vgx4", "vgx2") # "], $Zn, $Zm$i",3235 "", []>, Sched<[]> {3236 bits<0> ZAda;3237 bits<4> Zm;3238 bits<2> Rv;3239 bits<4> i;3240 bit imm;3241 let Inst{31-24} = 0b11000001;3242 let Inst{23-22} = sz;3243 let Inst{21-20} = 0b01;3244 let Inst{19-16} = Zm;3245 let Inst{15} = vg4;3246 let Inst{14-13} = Rv;3247 let Inst{12} = 0b0;3248 let Inst{11-10} = i{3-2};3249 let Inst{5-3} = op;3250 let Inst{2-1} = i{1-0};3251 let Inst{0} = imm;3252 3253 let Constraints = "$ZAda = $_ZAda";3254}3255 3256//SME2 multi-vec indexed long long MLA two sources 32-bit3257 3258multiclass sme2_mla_ll_array_vg2_index_32b<string mnemonic, bits<2> sz, bits<3> op, SDPatternOperator intrinsic, list<Register> uses=[]> {3259 def NAME: sme2_mla_ll_array_vg24_index_32b<sz, 0b0, op, ZZ_b_mul_r, mnemonic>, SMEPseudo2Instr<NAME, 1> {3260 bits<4> Zn;3261 let Inst{9-6} = Zn;3262 let Uses = uses;3263 }3264 3265 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, uimm1s4range, ZZ_b_mul_r, ZPR4b8, VectorIndexB32b_timm, SMEMatrixArray>;3266 3267 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, uimm1s4range, ZPR4b8, nxv16i8, VectorIndexB32b_timm, tileslicerange1s4>;3268 3269 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i",3270 (!cast<Instruction>(NAME) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZ_b_mul_r:$Zn, ZPR4b8:$Zm, VectorIndexB32b_timm:$i), 0>;3271}3272 3273// SME2 multi-vec indexed long long MLA four sources 32-bit3274 3275multiclass sme2_mla_ll_array_vg4_index_32b<string mnemonic, bits<2> sz, bits<4> op, SDPatternOperator intrinsic, list<Register> uses=[]> {3276 def NAME: sme2_mla_ll_array_vg24_index_32b<sz, 0b1, op{2-0}, ZZZZ_b_mul_r, mnemonic>, SMEPseudo2Instr<NAME, 1> {3277 bits<3> Zn;3278 let Inst{9-7} = Zn;3279 let Inst{6} = op{3};3280 let Uses = uses;3281 }3282 3283 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, uimm1s4range, ZZZZ_b_mul_r, ZPR4b8, VectorIndexB32b_timm, SMEMatrixArray>;3284 3285 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, uimm1s4range, ZPR4b8, nxv16i8, VectorIndexB32b_timm, tileslicerange1s4>;3286 3287 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i",3288 (!cast<Instruction>(NAME) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZZZ_b_mul_r:$Zn, ZPR4b8:$Zm, VectorIndexB32b_timm:$i), 0>;3289}3290class sme2_mla_ll_array_vg24_index_64b<bit vg4, bits<2> op,3291 RegisterOperand vector_ty,3292 string mnemonic>3293 : I<(outs MatrixOp64:$ZAda),3294 (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm,3295 vector_ty:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i),3296 mnemonic, "\t$ZAda[$Rv, $imm, " # !if(vg4, "vgx4", "vgx2") # "], $Zn, $Zm$i",3297 "", []>, Sched<[]> {3298 bits<0> ZAda;3299 bits<4> Zm;3300 bits<2> Rv;3301 bits<3> i;3302 bit imm;3303 let Inst{31-20} = 0b110000011001;3304 let Inst{19-16} = Zm;3305 let Inst{15} = vg4;3306 let Inst{14-13} = Rv;3307 let Inst{12-11} = 0b00;3308 let Inst{10} = i{2};3309 let Inst{5} = 0b0;3310 let Inst{4-3} = op;3311 let Inst{2-1} = i{1-0};3312 let Inst{0} = imm;3313 3314 let Constraints = "$ZAda = $_ZAda";3315}3316 3317// SME2 multi-vec indexed long long MLA two sources 64-bit3318 3319multiclass sme2_mla_ll_array_vg2_index_64b<string mnemonic, bits<2> op, SDPatternOperator intrinsic> {3320 def NAME: sme2_mla_ll_array_vg24_index_64b<0b0, op, ZZ_h_mul_r, mnemonic>, SMEPseudo2Instr<NAME, 1> {3321 bits<4> Zn;3322 let Inst{9-6} = Zn;3323 }3324 3325 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, uimm1s4range, ZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>;3326 3327 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, uimm1s4range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange1s4>;3328 3329 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i",3330 (!cast<Instruction>(NAME) MatrixOp64:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i), 0>;3331}3332 3333// SME2 multi-vec indexed long long MLA four sources 64-bit3334 3335multiclass sme2_mla_ll_array_vg4_index_64b<string mnemonic, bits<2> op, SDPatternOperator intrinsic> {3336 def NAME: sme2_mla_ll_array_vg24_index_64b<0b1, op, ZZZZ_h_mul_r, mnemonic>, SMEPseudo2Instr<NAME, 1> {3337 bits<3> Zn;3338 let Inst{9-7} = Zn;3339 let Inst{6} = 0b0;3340 }3341 3342 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, uimm1s4range, ZZZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>;3343 3344 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, uimm1s4range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange1s4>;3345 3346 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i",3347 (!cast<Instruction>(NAME) MatrixOp64:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i), 0>;3348}3349 3350 3351//SME2 multiple and single vector long long FMA one source3352 3353class sme2_mla_ll_array_single<string mnemonic, bits<5> op,3354 MatrixOperand matrix_ty, ZPRRegOp vector_ty,3355 ZPRRegOp zpr_ty>3356 : I<(outs matrix_ty:$ZAda),3357 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s4range:$imm,3358 vector_ty:$Zn, zpr_ty:$Zm),3359 mnemonic, "\t$ZAda[$Rv, $imm], $Zn, $Zm",3360 "", []>, Sched<[]> {3361 bits<0> ZAda;3362 bits<4> Zm;3363 bits<2> Rv;3364 bits<5> Zn;3365 bits<2> imm;3366 let Inst{31-23} = 0b110000010;3367 let Inst{22} = op{4}; //sz3368 let Inst{21} = 0b1;3369 let Inst{20} = op{3}; //fp83370 let Inst{19-16} = Zm;3371 let Inst{15} = 0b0;3372 let Inst{14-13} = Rv;3373 let Inst{12-10} = 0b001;3374 let Inst{9-5} = Zn;3375 let Inst{4-2} = op{2-0};3376 let Inst{1-0} = imm;3377 3378 let Constraints = "$ZAda = $_ZAda";3379}3380 3381multiclass sme2_mla_ll_array_single<string mnemonic, bits<5> op, MatrixOperand matrix_ty, ZPRRegOp vector_ty,3382 ZPRRegOp zpr_ty, ValueType vt, SDPatternOperator intrinsic, list<Register> uses=[]> {3383 def NAME : sme2_mla_ll_array_single<mnemonic, op, matrix_ty, vector_ty, zpr_ty>, SMEPseudo2Instr<NAME, 1> {3384 let Uses = uses;3385 }3386 3387 def NAME # _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, uimm2s4range, vector_ty, zpr_ty, SMEMatrixArray>;3388 3389 def : SME2_ZA_TwoOp_Multi_Single_Pat<NAME, intrinsic, uimm2s4range, zpr_ty, vt, tileslicerange2s4>;3390}3391 3392class sme2_mla_ll_array_vg24_single<bits<6> op, MatrixOperand matrix_ty,3393 RegisterOperand vector_ty, ZPRRegOp zpr_ty,3394 string mnemonic>3395 : I<(outs matrix_ty:$ZAda),3396 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm,3397 vector_ty:$Zn, zpr_ty:$Zm),3398 mnemonic, "\t$ZAda[$Rv, $imm, " # !if(op{4}, "vgx4", "vgx2") # "], $Zn, $Zm",3399 "", []>, Sched<[]> {3400 bits<0> ZAda;3401 bits<4> Zm;3402 bits<2> Rv;3403 bits<5> Zn;3404 bit imm;3405 let Inst{31-23} = 0b110000010;3406 let Inst{22} = op{5}; //sz3407 let Inst{21} = 0b1;3408 let Inst{20} = op{4}; //vg43409 let Inst{19-16} = Zm;3410 let Inst{15} = 0b0;3411 let Inst{14-13} = Rv;3412 let Inst{12-10} = 0b000;3413 let Inst{9-5} = Zn;3414 let Inst{4-1} = op{3-0};3415 let Inst{0} = imm;3416 3417 let Constraints = "$ZAda = $_ZAda";3418}3419 3420//SME2 single-multi long long MLA two and four sources3421 3422multiclass sme2_mla_ll_array_vg24_single<string mnemonic, bits<6> op,3423 MatrixOperand matrix_ty,3424 RegisterOperand multi_vector_ty,3425 ZPRRegOp zpr_ty, list<Register> uses> {3426 def NAME: sme2_mla_ll_array_vg24_single<op, matrix_ty, multi_vector_ty, zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1> {3427 let Uses = uses;3428 }3429 3430 def NAME # _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, uimm1s4range, multi_vector_ty, zpr_ty, SMEMatrixArray>;3431 3432 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm], $Zn, $Zm",3433 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>;3434}3435 3436multiclass sme2_mla_ll_array_vg2_single<string mnemonic, bits<6> op,3437 MatrixOperand matrix_ty, RegisterOperand multi_vector_ty,3438 ZPRRegOp zpr_ty, ValueType vt,SDPatternOperator intrinsic, list<Register> uses=[]> {3439 defm NAME: sme2_mla_ll_array_vg24_single<mnemonic, op, matrix_ty, multi_vector_ty, zpr_ty, uses>;3440 3441 def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME, intrinsic, uimm1s4range, zpr_ty, vt, tileslicerange1s4>;3442}3443 3444multiclass sme2_mla_ll_array_vg4_single<string mnemonic, bits<6> op,3445 MatrixOperand matrix_ty, RegisterOperand multi_vector_ty,3446 ZPRRegOp zpr_ty, ValueType vt, SDPatternOperator intrinsic, list<Register> uses=[]> {3447 defm NAME: sme2_mla_ll_array_vg24_single<mnemonic, op, matrix_ty, multi_vector_ty, zpr_ty, uses>;3448 3449 def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME, intrinsic, uimm1s4range, zpr_ty, vt, tileslicerange1s4>;3450}3451 3452// SME2 multiple vectors long long MLA two sources3453 3454class sme2_mla_ll_array_vg2_multi<bits<5> op, MatrixOperand matrix_ty,3455 RegisterOperand vector_ty,string mnemonic>3456 : I<(outs matrix_ty:$ZAda),3457 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm,3458 vector_ty:$Zn, vector_ty:$Zm),3459 mnemonic, "\t$ZAda[$Rv, $imm, vgx2], $Zn, $Zm",3460 "", []>, Sched<[]> {3461 bits<0> ZAda;3462 bits<4> Zm;3463 bits<2> Rv;3464 bits<4> Zn;3465 bit imm;3466 let Inst{31-23} = 0b110000011;3467 let Inst{22} = op{4}; // sz3468 let Inst{21} = 0b1;3469 let Inst{20-17} = Zm;3470 let Inst{16-15} = 0b00;3471 let Inst{14-13} = Rv;3472 let Inst{12-10} = 0b000;3473 let Inst{9-6} = Zn;3474 let Inst{5-2} = op{3-0};3475 let Inst{1} = 0b0;3476 let Inst{0} = imm;3477 3478 let Constraints = "$ZAda = $_ZAda";3479}3480 3481multiclass sme2_mla_ll_array_vg2_multi<string mnemonic, bits<5> op,3482 MatrixOperand matrix_ty,3483 RegisterOperand vector_ty, ValueType vt,3484 SDPatternOperator intrinsic, list<Register> uses=[]> {3485 def NAME : sme2_mla_ll_array_vg2_multi<op, matrix_ty, vector_ty, mnemonic>, SMEPseudo2Instr<NAME, 1> {3486 let Uses = uses;3487 }3488 3489 def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, uimm1s4range, vector_ty, SMEMatrixArray>;3490 3491 def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME, intrinsic, uimm1s4range, vt, tileslicerange1s4>;3492 3493 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm",3494 (!cast<Instruction>(NAME) matrix_ty:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, vector_ty:$Zn, vector_ty:$Zm), 0>;3495}3496 3497// SME2 multiple vectors long long MLA four sources3498 3499class sme2_mla_ll_array_vg4_multi<bits<5> op,MatrixOperand matrix_ty,3500 RegisterOperand vector_ty,3501 string mnemonic>3502 : I<(outs matrix_ty:$ZAda),3503 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm,3504 vector_ty:$Zn, vector_ty:$Zm),3505 mnemonic, "\t$ZAda[$Rv, $imm, vgx4], $Zn, $Zm",3506 "", []>, Sched<[]> {3507 bits<0> ZAda;3508 bits<3> Zm;3509 bits<2> Rv;3510 bits<3> Zn;3511 bit imm;3512 let Inst{31-23} = 0b110000011;3513 let Inst{22} = op{4}; // sz3514 let Inst{21} = 0b1;3515 let Inst{20-18} = Zm;3516 let Inst{17-15} = 0b010;3517 let Inst{14-13} = Rv;3518 let Inst{12-10} = 0b000;3519 let Inst{9-7} = Zn;3520 let Inst{6} = 0b0;3521 let Inst{5-2} = op{3-0};3522 let Inst{1} = 0b0;3523 let Inst{0} = imm;3524 3525 let Constraints = "$ZAda = $_ZAda";3526}3527 3528multiclass sme2_mla_ll_array_vg4_multi<string mnemonic, bits<5> op,3529 MatrixOperand matrix_ty,3530 RegisterOperand vector_ty, ValueType vt,3531 SDPatternOperator intrinsic, list<Register> uses=[]> {3532 def NAME : sme2_mla_ll_array_vg4_multi<op, matrix_ty, vector_ty, mnemonic>, SMEPseudo2Instr<NAME, 1> {3533 let Uses = uses;3534 }3535 3536 def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, uimm1s4range, vector_ty, SMEMatrixArray>;3537 3538 def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME, intrinsic, uimm1s4range, vt, tileslicerange1s4>;3539 3540 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm",3541 (!cast<Instruction>(NAME) matrix_ty:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, vector_ty:$Zn, vector_ty:$Zm), 0>;3542}3543 3544//===----------------------------------------------------------------------===//3545// SME2 Outer Product and Accumulate3546 3547multiclass sme2_int_mopx_tile<string mnemonic, bits<3> op, SDPatternOperator intrinsic> {3548 def NAME : sme_int_outer_product_inst<op, 0b0, 0b1, TileOp32, ZPR16, mnemonic>, SMEPseudo2Instr<NAME, 1> {3549 bits<2> ZAda;3550 let Inst{1-0} = ZAda;3551 let Inst{2} = 0b0;3552 }3553 3554 def _PSEUDO : sme_outer_product_pseudo<ZPR16, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>;3555 3556 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, intrinsic, timm32_0_3, nxv8i1, nxv8i16>;3557}3558 3559multiclass sme2_int_bmopx_tile<string mnemonic, bits<3> op, SDPatternOperator intrinsic> {3560 def NAME : sme_outer_product_widening_inst<op, ZPR32, mnemonic>, SMEPseudo2Instr<NAME, 1>;3561 3562 def _PSEUDO : sme_outer_product_pseudo<ZPR32, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>;3563 3564 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, intrinsic, timm32_0_3, nxv4i1, nxv4i32>;3565}3566 3567//===----------------------------------------------------------------------===//3568// SME2 Sparse Outer Product and Accumulate3569 3570class sme_int_sparse_outer_product_i16<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, string mnemonic>3571 : I<(outs TileOp16:$ZAda),3572 (ins TileOp16:$_ZAda, zn_ty:$Zn, zm_ty:$Zm, ZK:$Zk, VectorIndexS32b:$imm),3573 mnemonic, "\t$ZAda, $Zn, $Zm, $Zk$imm",3574 "", []>,3575 Sched<[]> {3576 bit ZAda;3577 bits<4> Zn;3578 bits<5> Zm;3579 bits<3> Zk;3580 bits<2> imm;3581 let Inst{31-25} = 0b1000000;3582 let Inst{24} = opc{4};3583 let Inst{23-22} = 0b01;3584 let Inst{21} = opc{3};3585 let Inst{20-16} = Zm;3586 let Inst{15} = opc{2};3587 let Inst{14} = 0b0;3588 let Inst{13} = opc{1};3589 let Inst{12-10} = Zk;3590 let Inst{9-6} = Zn;3591 let Inst{5-4} = imm;3592 let Inst{3} = opc{0};3593 let Inst{2-1} = 0b00;3594 let Inst{0} = ZAda;3595 3596 let Constraints = "$ZAda = $_ZAda";3597}3598 3599class sme_int_sparse_outer_product_i32<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, string mnemonic>3600 : I<(outs TileOp32:$ZAda),3601 (ins TileOp32:$_ZAda, zn_ty:$Zn, zm_ty:$Zm, ZK:$Zk, VectorIndexS32b:$imm),3602 mnemonic, "\t$ZAda, $Zn, $Zm, $Zk$imm",3603 "", []>,3604 Sched<[]> {3605 bits<2> ZAda;3606 bits<4> Zn;3607 bits<5> Zm;3608 bits<3> Zk;3609 bits<2> imm;3610 let Inst{31-25} = 0b1000000;3611 let Inst{24} = opc{4};3612 let Inst{23-22} = 0b01;3613 let Inst{21} = opc{3};3614 let Inst{20-16} = Zm;3615 let Inst{15} = opc{2};3616 let Inst{14} = 0b0;3617 let Inst{13} = opc{1};3618 let Inst{12-10} = Zk;3619 let Inst{9-6} = Zn;3620 let Inst{5-4} = imm;3621 let Inst{3} = opc{0};3622 let Inst{2} = 0b0;3623 let Inst{1-0} = ZAda;3624 3625 let Constraints = "$ZAda = $_ZAda";3626}3627 3628multiclass sme_tmopa_16b<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, ValueType vt, string mnemonic, SDPatternOperator intrinsic, list<Register> uses=[]> {3629 def NAME : sme_int_sparse_outer_product_i16<opc, zn_ty, zm_ty, mnemonic>, SMEPseudo2Instr<NAME, 1> {3630 let Uses = uses;3631 }3632 3633 def NAME # _PSEUDO : sme_sparse_outer_product_pseudo<zn_ty, zm_ty, SMEMatrixTileH>, SMEPseudo2Instr<NAME, 0>;3634 3635 def : SME2_ZA_TMOP_Pat<NAME, intrinsic, timm32_0_1, vt>;3636}3637 3638multiclass sme_tmopa_32b<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, ValueType vt, string mnemonic, SDPatternOperator intrinsic, list<Register> uses=[]> {3639 def NAME : sme_int_sparse_outer_product_i32<opc, zn_ty, zm_ty, mnemonic>, SMEPseudo2Instr<NAME, 1> {3640 let Uses = uses;3641 }3642 3643 def NAME # _PSEUDO : sme_sparse_outer_product_pseudo<zn_ty, zm_ty, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>;3644 3645 def : SME2_ZA_TMOP_Pat<NAME, intrinsic, timm32_0_3, vt>;3646}3647 3648//===----------------------------------------------------------------------===///3649// SME2 Zero Lookup Table.3650class sme2_zero_zt<string mnemonic, bits<4> opc>3651 : I<(outs ZTR:$ZT), (ins ),3652 mnemonic, "\t\\{ $ZT \\}",3653 "", []>, Sched<[]> {3654 bits<0> ZT;3655 let Inst{31-4} = 0b1100000001001000000000000000;3656 let Inst{3-0} = opc;3657}3658 3659multiclass sme2_zero_zt<string mnemonic, bits<4> opc> {3660 def NAME : sme2_zero_zt<mnemonic, opc>;3661 def NAME # _PSEUDO3662 : Pseudo<(outs), (ins ZTR:$ZT), []>, Sched<[]> {3663 // Translated to actual instruction in AArch64ISelLowering.cpp3664 let usesCustomInserter = 1;3665 }3666 def : Pat<(int_aarch64_sme_zero_zt (imm_to_zt untyped:$zt)),3667 (!cast<Instruction>(NAME # _PSEUDO) $zt)>;3668}3669 3670//===----------------------------------------------------------------------===//3671// SME2 lookup table load/store3672class sme2_spill_fill_vector<string mnemonic, bits<8> opc>3673 : I<!if(opc{7}, (outs ), (outs ZTR:$ZTt)),3674 !if(opc{7}, (ins ZTR:$ZTt, GPR64sp:$Rn), (ins GPR64sp:$Rn)),3675 mnemonic, "\t$ZTt, [$Rn]",3676 "", []>, Sched<[]> {3677 bits<0> ZTt;3678 bits<5> Rn;3679 let Inst{31-22} = 0b1110000100;3680 let Inst{21-16} = opc{7-2};3681 let Inst{15-10} = 0b100000;3682 let Inst{9-5} = Rn;3683 let Inst{4-2} = 0b000;3684 let Inst{1-0} = opc{1-0};3685 3686 let mayLoad = !not(opc{7});3687 let mayStore = opc{7};3688}3689 3690 3691multiclass sme2_spill_fill_vector<string mnemonic, bits<8> opc, SDPatternOperator op> {3692 def NAME : sme2_spill_fill_vector<mnemonic, opc>;3693 def NAME # _PSEUDO3694 : Pseudo<(outs), (ins ZTR:$ZTt, GPR64sp:$base), []>, Sched<[]> {3695 // Translated to actual instruction in AArch64ISelLowering.cpp3696 let usesCustomInserter = 1;3697 }3698 def : Pat<(op (imm_to_zt untyped:$tile), GPR64sp:$base),3699 (!cast<Instruction>(NAME # _PSEUDO) $tile, $base)>;3700}3701 3702//===----------------------------------------------------------------------===///3703// SME2 move to/from lookup table3704class sme2_movt_zt_to_scalar<string mnemonic, bits<7> opc>3705 : I<(outs GPR64:$Rt), (ins ZTR:$ZTt, uimm3s8:$imm3),3706 mnemonic, "\t$Rt, $ZTt[$imm3]",3707 "", []>, Sched<[]> {3708 bits<0> ZTt;3709 bits<3> imm3;3710 bits<5> Rt;3711 let Inst{31-15} = 0b11000000010011000;3712 let Inst{14-12} = imm3;3713 let Inst{11-5} = opc;3714 let Inst{4-0} = Rt;3715}3716 3717class sme2_movt_scalar_to_zt<string mnemonic, bits<7> opc>3718 : I<(outs ZTR:$ZTt), (ins uimm3s8:$imm3, GPR64:$Rt),3719 mnemonic, "\t$ZTt[$imm3], $Rt",3720 "", []>, Sched<[]> {3721 bits<0> ZTt;3722 bits<3> imm3;3723 bits<5> Rt;3724 let Inst{31-15} = 0b11000000010011100;3725 let Inst{14-12} = imm3;3726 let Inst{11-5} = opc;3727 let Inst{4-0} = Rt;3728}3729 3730// SME2 move vector to lookup table3731class sme2_movt_zt_to_zt<string mnemonic, bits<7> opc>3732 : I<(outs ZTR:$ZTt), (ins sme_elm_idx0_3:$off2, ZPRAny:$Zt),3733 mnemonic, "\t$ZTt[$off2, mul vl], $Zt",3734 "", []>, Sched<[]> {3735 bits<0> ZTt;3736 bits<5> Zt;3737 bits<2> off2;3738 let Inst{31-14} = 0b110000000100111100;3739 let Inst{13-12} = off2;3740 let Inst{11-5} = opc;3741 let Inst{4-0} = Zt;3742}3743 3744multiclass sme2_movt_zt_to_zt<string mnemonic, bits<7> opc, SDPatternOperator intrinsic_lane, SDPatternOperator intrinsic> {3745 def NAME : sme2_movt_zt_to_zt<mnemonic, opc>;3746 def NAME # _PSEUDO3747 : Pseudo<(outs), (ins ZTR:$ZT, sme_elm_idx0_3:$off2, ZPRAny:$Zt), []>, Sched<[]> {3748 let usesCustomInserter = 1;3749 }3750 def : InstAlias<mnemonic # "\t$ZTt, $Zt",3751 (!cast<Instruction>(NAME) ZTR:$ZTt, 0, ZPRAny:$Zt), 1>;3752 3753 foreach vt = [nxv16i8, nxv8i16, nxv4i32, nxv2i64, nxv8f16, nxv4f32, nxv2f64, nxv8bf16] in {3754 def : Pat<(intrinsic_lane (imm_to_zt untyped:$zt), vt:$zn, sme_elm_idx0_3:$imm),3755 (!cast<Instruction>(NAME # _PSEUDO) $zt, $imm, $zn)>;3756 def : Pat<(intrinsic (imm_to_zt untyped:$zt), vt:$zn),3757 (!cast<Instruction>(NAME # _PSEUDO) $zt, 0, $zn)>;3758 }3759}3760 3761//===----------------------------------------------------------------------===//3762// SME2 lookup table expand one register3763class sme2_luti_vector_index<bits<2> sz, bits<7> opc, RegisterOperand vector_ty,3764 AsmVectorIndexOpnd index_ty, string mnemonic>3765 : I<(outs vector_ty:$Zd),3766 (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i),3767 mnemonic, "\t$Zd, $ZTt, $Zn$i",3768 "", []>, Sched<[]> {3769 bits<0> ZTt;3770 bits<5> Zn;3771 bits<5> Zd;3772 let Inst{31-19} = 0b1100000011001;3773 let Inst{18-14} = opc{6-2};3774 let Inst{13-12} = sz;3775 let Inst{11-10} = opc{1-0};3776 let Inst{9-5} = Zn;3777 let Inst{4-0} = Zd;3778}3779 3780class sme2_luti2_vector_index<bits<2> sz, RegisterOperand vector_ty,3781 string mnemonic>3782 : sme2_luti_vector_index<sz, {1,?,?,?,?,0,0}, vector_ty, VectorIndexB32b_timm, mnemonic> {3783 bits<4> i;3784 let Inst{17-14} = i;3785}3786 3787multiclass sme2_luti2_vector_index<string mnemonic, SDPatternOperator intrinsic> {3788 def _B : sme2_luti2_vector_index<0b00, ZPR8, mnemonic>;3789 def _H : sme2_luti2_vector_index<0b01, ZPR16, mnemonic>;3790 def _S : sme2_luti2_vector_index<0b10, ZPR32, mnemonic>;3791 3792 def : Pat<(nxv16i8 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))),3793 (!cast<Instruction>(NAME # _B) $zt, nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))>;3794 def : Pat<(nxv8i16 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))),3795 (!cast<Instruction>(NAME # _H) $zt, nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))>;3796 def : Pat<(nxv4i32 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))),3797 (!cast<Instruction>(NAME # _S) $zt, nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))>;3798 def : Pat<(nxv8f16 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))),3799 (!cast<Instruction>(NAME # _H) $zt, nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))>;3800 def : Pat<(nxv8bf16 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))),3801 (!cast<Instruction>(NAME # _H) $zt, nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))>;3802 def : Pat<(nxv4f32 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))),3803 (!cast<Instruction>(NAME # _S) $zt, nxv16i8:$zn, (i32 VectorIndexB32b_timm:$imm))>;3804}3805 3806class sme2_luti4_vector_index<bits<2> sz, RegisterOperand vector_ty,3807 string mnemonic>3808 : sme2_luti_vector_index<sz, {0,1,?,?,?,0,0}, vector_ty, VectorIndexH32b_timm, mnemonic> {3809 bits<3> i;3810 let Inst{16-14} = i;3811}3812 3813multiclass sme2_luti4_vector_index<string mnemonic, SDPatternOperator intrinsic> {3814 def _B : sme2_luti4_vector_index<0b00, ZPR8, mnemonic>;3815 def _H : sme2_luti4_vector_index<0b01, ZPR16, mnemonic>;3816 def _S : sme2_luti4_vector_index<0b10, ZPR32, mnemonic>;3817 3818 def : Pat<(nxv16i8 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))),3819 (!cast<Instruction>(NAME # _B) $zt, nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))>;3820 def : Pat<(nxv8i16 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))),3821 (!cast<Instruction>(NAME # _H) $zt, nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))>;3822 def : Pat<(nxv4i32 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))),3823 (!cast<Instruction>(NAME # _S) $zt, nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))>;3824 def : Pat<(nxv8f16 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))),3825 (!cast<Instruction>(NAME # _H) $zt, nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))>;3826 def : Pat<(nxv8bf16 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))),3827 (!cast<Instruction>(NAME # _H) $zt, nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))>;3828 def : Pat<(nxv4f32 (intrinsic (imm_to_zt untyped:$zt), nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))),3829 (!cast<Instruction>(NAME # _S) $zt, nxv16i8:$zn, (i32 VectorIndexH32b_timm:$imm))>;3830}3831 3832// SME2 lookup table expand two contiguous registers3833class sme2_luti_vector_vg2_index<bits<2> sz, bits<6> opc, RegisterOperand vector_ty,3834 AsmVectorIndexOpnd index_ty, string mnemonic>3835 : I<(outs vector_ty:$Zd),3836 (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i),3837 mnemonic, "\t$Zd, $ZTt, $Zn$i",3838 "", []>, Sched<[]> {3839 bits<0> ZTt;3840 bits<5> Zn;3841 bits<4> Zd;3842 let Inst{31-19} = 0b1100000010001;3843 let Inst{18-15} = opc{5-2};3844 let Inst{14} = 0b1;3845 let Inst{13-12} = sz;3846 let Inst{11-10} = opc{1-0};3847 let Inst{9-5} = Zn;3848 let Inst{4-1} = Zd;3849 let Inst{0} = 0b0;3850}3851 3852class sme2_luti2_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty,3853 string mnemonic>3854 : sme2_luti_vector_vg2_index<sz, {1,?,?,?,0,0}, vector_ty, VectorIndexH, mnemonic> {3855 bits<3> i;3856 let Inst{17-15} = i;3857}3858 3859multiclass sme2_luti2_vector_vg2_index<string mnemonic> {3860 def _B : sme2_luti2_vector_vg2_index<0b00, ZZ_b_mul_r, mnemonic>;3861 def _H : sme2_luti2_vector_vg2_index<0b01, ZZ_h_mul_r, mnemonic>;3862 def _S : sme2_luti2_vector_vg2_index<0b10, ZZ_s_mul_r, mnemonic>;3863}3864 3865class sme2_luti4_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty,3866 string mnemonic>3867 : sme2_luti_vector_vg2_index<sz, {0,1,?,?,0,0}, vector_ty, VectorIndexS, mnemonic> {3868 bits<2> i;3869 let Inst{16-15} = i;3870}3871 3872multiclass sme2_luti4_vector_vg2_index<string mnemonic> {3873 def _B : sme2_luti4_vector_vg2_index<0b00, ZZ_b_mul_r, mnemonic>;3874 def _H : sme2_luti4_vector_vg2_index<0b01, ZZ_h_mul_r, mnemonic>;3875 def _S : sme2_luti4_vector_vg2_index<0b10, ZZ_s_mul_r, mnemonic>;3876}3877 3878// SME2 lookup table expand four contiguous registers3879class sme2_luti_vector_vg4_index<bits<2> sz, bits<5>opc, RegisterOperand vector_ty,3880 AsmVectorIndexOpnd index_ty, string mnemonic>3881 : I<(outs vector_ty:$Zd),3882 (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i),3883 mnemonic, "\t$Zd, $ZTt, $Zn$i",3884 "", []>, Sched<[]> {3885 bits<0> ZTt;3886 bits<5> Zn;3887 bits<3> Zd;3888 let Inst{31-19} = 0b1100000010001;3889 let Inst{18-16} = opc{4-2};3890 let Inst{15-14} = 0b10;3891 let Inst{13-12} = sz;3892 let Inst{11-10} = opc{1-0};3893 let Inst{9-5} = Zn;3894 let Inst{4-2} = Zd;3895 let Inst{1-0} = 0b00;3896}3897 3898class sme2_luti2_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty,3899 string mnemonic>3900 : sme2_luti_vector_vg4_index<sz, {1,?,?,0,0}, vector_ty, VectorIndexS, mnemonic> {3901 bits<2> i;3902 let Inst{17-16} = i;3903}3904 3905multiclass sme2_luti2_vector_vg4_index<string mnemonic> {3906 def _B : sme2_luti2_vector_vg4_index<0b00, ZZZZ_b_mul_r, mnemonic>;3907 def _H : sme2_luti2_vector_vg4_index<0b01, ZZZZ_h_mul_r, mnemonic>;3908 def _S : sme2_luti2_vector_vg4_index<0b10, ZZZZ_s_mul_r, mnemonic>;3909}3910 3911class sme2_luti4_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty,3912 string mnemonic>3913 : sme2_luti_vector_vg4_index<sz, {0,1,?,0,0}, vector_ty, VectorIndexD, mnemonic> {3914 bits<1> i;3915 let Inst{16} = i;3916}3917 3918multiclass sme2_luti4_vector_vg4_index<string mnemonic> {3919 def _H : sme2_luti4_vector_vg4_index<0b01, ZZZZ_h_mul_r, mnemonic>;3920 def _S : sme2_luti4_vector_vg4_index<0b10, ZZZZ_s_mul_r, mnemonic>;3921}3922 3923// 8-bit Look up table3924class sme2_lut_single<string asm>3925 : I<(outs ZPR8:$Zd), (ins ZTR:$ZTt, ZPRAny:$Zn),3926 asm, "\t$Zd, $ZTt, $Zn", "", []>, Sched<[]> {3927 bits<0> ZTt;3928 bits<5> Zd;3929 bits<5> Zn;3930 let Inst{31-10} = 0b1100000011001000010000;3931 let Inst{9-5} = Zn;3932 let Inst{4-0} = Zd;3933}3934 3935//===----------------------------------------------------------------------===//3936// Lookup table read with 6-bit indices (8-bit)3937class sme2_luti6_zt_base<RegisterOperand zd_ty, string asm>3938 : I<(outs zd_ty:$Zd), (ins ZTR:$ZTt, ZZZ_Any:$Zn),3939 asm, "\t$Zd, $ZTt, $Zn", "", []>, Sched<[]> {3940 bits<0> ZTt;3941 bits<3> Zd;3942 bits<3> Zn;3943 let Inst{31-21} = 0b11000000100;3944 let Inst{19-10} = 0b1010000000;3945 let Inst{9-7} = Zn;3946 let Inst{6-5} = 0b00;3947}3948 3949class sme2_luti6_zt_consecutive<string asm>3950 : sme2_luti6_zt_base<ZZZZ_b_mul_r, asm> {3951 let Inst{20} = 0;3952 let Inst{4-2} = Zd;3953 let Inst{1-0} = 0b00;3954}3955 3956class sme2_luti6_zt_strided<string asm>3957 : sme2_luti6_zt_base<ZZZZ_b_strided, asm> {3958 let Inst{20} = 1;3959 let Inst{4} = Zd{2};3960 let Inst{3-2} = 0b00;3961 let Inst{1-0} = Zd{1-0};3962}3963 3964//===----------------------------------------------------------------------===//3965// Lookup table read with 6-bit indices (8-bit)3966class sme2_luti6_vector_vg4_base<RegisterOperand zd_ty, string asm>3967 : I<(outs zd_ty:$Zd), (ins ZZ_h:$Zn, ZZ_Any:$Zm, VectorIndexD:$i1),3968 asm, "\t$Zd, $Zn, $Zm$i1", "", []>, Sched<[]> {3969 bits<3> Zd;3970 bits<5> Zn;3971 bits<5> Zm;3972 bits<1> i1;3973 let Inst{31-23} = 0b110000010;3974 let Inst{22} = i1;3975 let Inst{21} = 0b1;3976 let Inst{20-16} = Zm;3977 let Inst{9-5} = Zn;3978}3979 3980class sme2_luti6_vector_vg4_consecutive<string asm>3981 : sme2_luti6_vector_vg4_base<ZZZZ_h_mul_r, asm> {3982 let Inst{15-10} = 0b111101;3983 let Inst{4-2} = Zd;3984 let Inst{1-0} = 0b00;3985}3986 3987class sme2_luti6_vector_vg4_strided<string asm>3988 : sme2_luti6_vector_vg4_base<ZZZZ_h_strided, asm> {3989 let Inst{15-10} = 0b111111;3990 let Inst{4} = Zd{2};3991 let Inst{3-2} = 0b00;3992 let Inst{1-0} = Zd{1-0};3993}3994 3995//===----------------------------------------------------------------------===//3996// SME2 MOV3997class sme2_mova_vec_to_tile_vg2_multi_base<bits<2> sz, bit v,3998 RegisterOperand tile_ty,3999 Operand index_ty,4000 RegisterOperand vector_ty,4001 string mnemonic>4002 : I<(outs tile_ty:$ZAd),4003 (ins tile_ty:$_ZAd, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm, vector_ty:$Zn),4004 mnemonic, "\t$ZAd[$Rs, $imm], $Zn",4005 "", []>, Sched<[]> {4006 bits<2> Rs;4007 bits<4> Zn;4008 let Inst{31-24} = 0b11000000;4009 let Inst{23-22} = sz;4010 let Inst{21-16} = 0b000100;4011 let Inst{15} = v;4012 let Inst{14-13} = Rs;4013 let Inst{12-10} = 0b000;4014 let Inst{9-6} = Zn;4015 let Inst{5-3} = 0b000;4016 4017 let Constraints = "$ZAd = $_ZAd";4018}4019 4020multiclass sme2_mova_vec_to_tile_or_array_aliases<int prefer, Instruction inst,4021 RegisterOperand tile_or_array_ty,4022 RegisterOperand rv_ty,4023 Operand index_ty,4024 RegisterOperand vector_ty,4025 string mnemonic,4026 string vg_acronym=""> {4027 def : InstAlias<mnemonic # "\t$ZAd[$Rs, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "], $Zn",4028 (inst tile_or_array_ty:$ZAd, rv_ty:$Rs, index_ty:$imm, vector_ty:$Zn), prefer>;4029 4030}4031 4032// SME2 move vector to tile, two registers4033multiclass sme2_mova_vec_to_tile_vg2_multi_base<bit v, string mnemonic, SDPatternOperator intrinsic> {4034 4035 def _B : sme2_mova_vec_to_tile_vg2_multi_base<0b00, v,4036 !if(v, TileVectorOpV8,4037 TileVectorOpH8),4038 uimm3s2range, ZZ_b_mul_r,4039 mnemonic>, SMEPseudo2Instr<NAME # _B, 1> {4040 bits<0> ZAd;4041 bits<3> imm;4042 let Inst{2-0} = imm;4043 }4044 4045 def _H : sme2_mova_vec_to_tile_vg2_multi_base<0b01, v,4046 !if(v, TileVectorOpV16,4047 TileVectorOpH16),4048 uimm2s2range, ZZ_h_mul_r,4049 mnemonic>, SMEPseudo2Instr<NAME # _H, 1> {4050 bits<1> ZAd;4051 bits<2> imm;4052 let Inst{2} = ZAd;4053 let Inst{1-0} = imm;4054 }4055 4056 def _S : sme2_mova_vec_to_tile_vg2_multi_base<0b10, v,4057 !if(v, TileVectorOpV32,4058 TileVectorOpH32),4059 uimm1s2range, ZZ_s_mul_r,4060 mnemonic>, SMEPseudo2Instr<NAME # _S, 1> {4061 bits<2> ZAd;4062 bits<1> imm;4063 let Inst{2-1} = ZAd;4064 let Inst{0} = imm;4065 }4066 4067 def _D : sme2_mova_vec_to_tile_vg2_multi_base<0b11, v,4068 !if(v, TileVectorOpV64,4069 TileVectorOpH64),4070 uimm0s2range, ZZ_d_mul_r,4071 mnemonic>, SMEPseudo2Instr<NAME # _D, 1> {4072 bits<3> ZAd;4073 bits<0> imm;4074 let Inst{2-0} = ZAd;4075 }4076 4077 def NAME # _B_PSEUDO : sme2_move_to_tile_pseudo<NAME # _B, sme_elm_idx0_0, uimm3s2range, ZZ_b_mul_r, SMEMatrixTileB>;4078 def NAME # _H_PSEUDO : sme2_move_to_tile_pseudo<NAME # _H, sme_elm_idx0_1, uimm2s2range, ZZ_h_mul_r, SMEMatrixTileH>;4079 def NAME # _S_PSEUDO : sme2_move_to_tile_pseudo<NAME # _S, sme_elm_idx0_3, uimm1s2range, ZZ_s_mul_r, SMEMatrixTileS>;4080 def NAME # _D_PSEUDO : sme2_move_to_tile_pseudo<NAME # _D, sme_elm_idx0_7, uimm0s2range, ZZ_d_mul_r, SMEMatrixTileD>;4081 4082 def : SME2_Tile_VG2_Multi_Pat<NAME # _B, intrinsic, sme_elm_idx0_0, nxv16i8, uimm3s2range, tileslicerange3s2>;4083 def : SME2_Tile_VG2_Multi_Pat<NAME # _H, intrinsic, sme_elm_idx0_1, nxv8i16, uimm2s2range, tileslicerange2s2>;4084 def : SME2_Tile_VG2_Multi_Pat<NAME # _H, intrinsic, sme_elm_idx0_1, nxv8f16, uimm2s2range, tileslicerange2s2>;4085 def : SME2_Tile_VG2_Multi_Pat<NAME # _H, intrinsic, sme_elm_idx0_1, nxv8bf16, uimm2s2range, tileslicerange2s2>;4086 def : SME2_Tile_VG2_Multi_Pat<NAME # _S, intrinsic, sme_elm_idx0_3, nxv4i32, uimm1s2range, tileslicerange1s2>;4087 def : SME2_Tile_VG2_Multi_Pat<NAME # _S, intrinsic, sme_elm_idx0_3, nxv4f32, uimm1s2range, tileslicerange1s2>;4088 def : SME2_Tile_VG2_Multi_Pat<NAME # _D, intrinsic, sme_elm_idx0_7, nxv2i64, uimm0s2range, tileslicerange0s2>;4089 def : SME2_Tile_VG2_Multi_Pat<NAME # _D, intrinsic, sme_elm_idx0_7, nxv2f64, uimm0s2range, tileslicerange0s2>;4090 4091 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _B),4092 !if(v, TileVectorOpV8,4093 TileVectorOpH8),4094 MatrixIndexGPR32Op12_15,4095 uimm3s2range, ZZ_b_mul_r,4096 "mov">;4097 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _H),4098 !if(v, TileVectorOpV16,4099 TileVectorOpH16),4100 MatrixIndexGPR32Op12_15,4101 uimm2s2range, ZZ_h_mul_r,4102 "mov">;4103 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _S),4104 !if(v, TileVectorOpV32,4105 TileVectorOpH32),4106 MatrixIndexGPR32Op12_15,4107 uimm1s2range, ZZ_s_mul_r,4108 "mov">;4109 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _D),4110 !if(v, TileVectorOpV64,4111 TileVectorOpH64),4112 MatrixIndexGPR32Op12_15,4113 uimm0s2range, ZZ_d_mul_r,4114 "mov">;4115 4116 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _B),4117 !if(v, TileVectorOpV8,4118 TileVectorOpH8),4119 MatrixIndexGPR32Op12_15,4120 uimm3s2range, ZZ_b_mul_r,4121 "mova">;4122 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _H),4123 !if(v, TileVectorOpV16,4124 TileVectorOpH16),4125 MatrixIndexGPR32Op12_15,4126 uimm2s2range, ZZ_h_mul_r,4127 "mova">;4128 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _S),4129 !if(v, TileVectorOpV32,4130 TileVectorOpH32),4131 MatrixIndexGPR32Op12_15,4132 uimm1s2range, ZZ_s_mul_r,4133 "mova">;4134 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _D),4135 !if(v, TileVectorOpV64,4136 TileVectorOpH64),4137 MatrixIndexGPR32Op12_15,4138 uimm0s2range, ZZ_d_mul_r,4139 "mova">;4140 4141 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _B),4142 !if(v, TileVectorOpV8,4143 TileVectorOpH8),4144 MatrixIndexGPR32Op12_15,4145 uimm3s2range, ZZ_b_mul_r,4146 "mova">;4147 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _H),4148 !if(v, TileVectorOpV16,4149 TileVectorOpH16),4150 MatrixIndexGPR32Op12_15,4151 uimm2s2range, ZZ_h_mul_r,4152 "mova">;4153 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _S),4154 !if(v, TileVectorOpV32,4155 TileVectorOpH32),4156 MatrixIndexGPR32Op12_15,4157 uimm1s2range, ZZ_s_mul_r,4158 "mova">;4159 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _D),4160 !if(v, TileVectorOpV64,4161 TileVectorOpH64),4162 MatrixIndexGPR32Op12_15,4163 uimm0s2range, ZZ_d_mul_r,4164 "mova">;4165}4166 4167multiclass sme2_mova_vec_to_tile_vg2_multi<string mnemonic,4168 SDPatternOperator int_h, SDPatternOperator int_v>{4169 defm _H : sme2_mova_vec_to_tile_vg2_multi_base<0b0, mnemonic, int_h>;4170 defm _V : sme2_mova_vec_to_tile_vg2_multi_base<0b1, mnemonic, int_v>;4171}4172 4173class sme2_mova_vec_to_tile_vg4_multi_base<bits<2> sz, bit v, bits<3> op,4174 RegisterOperand tile_ty,4175 Operand index_ty,4176 RegisterOperand vector_ty,4177 string mnemonic>4178 : I<(outs tile_ty:$ZAd),4179 (ins tile_ty:$_ZAd, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm,4180 vector_ty:$Zn),4181 mnemonic,4182 "\t$ZAd[$Rs, $imm], $Zn",4183 "", []>, Sched<[]> {4184 bits<2> Rs;4185 bits<3> Zn;4186 let Inst{31-24} = 0b11000000;4187 let Inst{23-22} = sz;4188 let Inst{21-16} = 0b000100;4189 let Inst{15} = v;4190 let Inst{14-13} = Rs;4191 let Inst{12-10} = 0b001;4192 let Inst{9-7} = Zn;4193 let Inst{6-3} = 0b0000;4194 let Inst{2-0} = op;4195 let Constraints = "$ZAd = $_ZAd";4196}4197 4198// SME2 move vector to tile, four registers4199multiclass sme2_mova_vec_to_tile_vg4_multi_base<bit v, string mnemonic, SDPatternOperator intrinsic> {4200 4201 def _B : sme2_mova_vec_to_tile_vg4_multi_base<0b00, v, {0,?,?},4202 !if(v, TileVectorOpV8,4203 TileVectorOpH8),4204 uimm2s4range, ZZZZ_b_mul_r,4205 mnemonic>, SMEPseudo2Instr<NAME # _B, 1> {4206 bits<0> ZAd;4207 bits<2> imm;4208 let Inst{1-0} = imm;4209 }4210 4211 def _H : sme2_mova_vec_to_tile_vg4_multi_base<0b01, v, {0,?,?},4212 !if(v, TileVectorOpV16,4213 TileVectorOpH16),4214 uimm1s4range, ZZZZ_h_mul_r,4215 mnemonic>, SMEPseudo2Instr<NAME # _H, 1> {4216 bits<1> ZAd;4217 bits<1> imm;4218 let Inst{1} = ZAd;4219 let Inst{0} = imm;4220 }4221 4222 def _S : sme2_mova_vec_to_tile_vg4_multi_base<0b10, v, {0,?,?},4223 !if(v, TileVectorOpV32,4224 TileVectorOpH32),4225 uimm0s4range, ZZZZ_s_mul_r,4226 mnemonic>, SMEPseudo2Instr<NAME # _S, 1> {4227 bits<2> ZAd;4228 bits<0> imm;4229 let Inst{1-0} = ZAd;4230 }4231 4232 def _D : sme2_mova_vec_to_tile_vg4_multi_base<0b11, v, {?,?,?},4233 !if(v, TileVectorOpV64,4234 TileVectorOpH64),4235 uimm0s4range, ZZZZ_d_mul_r,4236 mnemonic>, SMEPseudo2Instr<NAME # _D, 1> {4237 bits<3> ZAd;4238 bits<0> imm;4239 let Inst{2-0} = ZAd;4240 }4241 4242 def NAME # _B_PSEUDO : sme2_move_to_tile_pseudo<NAME # _B, sme_elm_idx0_0, uimm2s4range, ZZZZ_b_mul_r, SMEMatrixTileB>;4243 def NAME # _H_PSEUDO : sme2_move_to_tile_pseudo<NAME # _H, sme_elm_idx0_1, uimm1s4range, ZZZZ_h_mul_r, SMEMatrixTileH>;4244 def NAME # _S_PSEUDO : sme2_move_to_tile_pseudo<NAME # _S, sme_elm_idx0_3, uimm0s4range, ZZZZ_s_mul_r, SMEMatrixTileS>;4245 def NAME # _D_PSEUDO : sme2_move_to_tile_pseudo<NAME # _D, sme_elm_idx0_7, uimm0s4range, ZZZZ_d_mul_r, SMEMatrixTileD>;4246 4247 def : SME2_Tile_VG4_Multi_Pat<NAME # _B, intrinsic, sme_elm_idx0_0, nxv16i8, uimm2s4range, tileslicerange2s4>;4248 def : SME2_Tile_VG4_Multi_Pat<NAME # _H, intrinsic, sme_elm_idx0_1, nxv8i16, uimm1s4range, tileslicerange1s4>;4249 def : SME2_Tile_VG4_Multi_Pat<NAME # _H, intrinsic, sme_elm_idx0_1, nxv8f16, uimm1s4range, tileslicerange1s4>;4250 def : SME2_Tile_VG4_Multi_Pat<NAME # _H, intrinsic, sme_elm_idx0_1, nxv8bf16, uimm1s4range, tileslicerange1s4>;4251 def : SME2_Tile_VG4_Multi_Pat<NAME # _S, intrinsic, sme_elm_idx0_3, nxv4i32, uimm0s4range, tileslicerange0s4>;4252 def : SME2_Tile_VG4_Multi_Pat<NAME # _S, intrinsic, sme_elm_idx0_3, nxv4f32, uimm0s4range, tileslicerange0s4>;4253 def : SME2_Tile_VG4_Multi_Pat<NAME # _D, intrinsic, sme_elm_idx0_7, nxv2i64, uimm0s4range, tileslicerange0s4>;4254 def : SME2_Tile_VG4_Multi_Pat<NAME # _D, intrinsic, sme_elm_idx0_7, nxv2f64, uimm0s4range, tileslicerange0s4>;4255 4256 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _B),4257 !if(v, TileVectorOpV8,4258 TileVectorOpH8),4259 MatrixIndexGPR32Op12_15,4260 uimm2s4range, ZZZZ_b_mul_r,4261 "mov">;4262 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _H),4263 !if(v, TileVectorOpV16,4264 TileVectorOpH16),4265 MatrixIndexGPR32Op12_15,4266 uimm1s4range, ZZZZ_h_mul_r,4267 "mov">;4268 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _S),4269 !if(v, TileVectorOpV32,4270 TileVectorOpH32),4271 MatrixIndexGPR32Op12_15,4272 uimm0s4range, ZZZZ_s_mul_r,4273 "mov">;4274 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _D),4275 !if(v, TileVectorOpV64,4276 TileVectorOpH64),4277 MatrixIndexGPR32Op12_15,4278 uimm0s4range, ZZZZ_d_mul_r,4279 "mov">;4280 4281 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _B),4282 !if(v, TileVectorOpV8,4283 TileVectorOpH8),4284 MatrixIndexGPR32Op12_15,4285 uimm2s4range, ZZZZ_b_mul_r,4286 "mova">;4287 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _H),4288 !if(v, TileVectorOpV16,4289 TileVectorOpH16),4290 MatrixIndexGPR32Op12_15,4291 uimm1s4range, ZZZZ_h_mul_r,4292 "mova">;4293 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _S),4294 !if(v, TileVectorOpV32,4295 TileVectorOpH32),4296 MatrixIndexGPR32Op12_15,4297 uimm0s4range, ZZZZ_s_mul_r,4298 "mova">;4299 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _D),4300 !if(v, TileVectorOpV64,4301 TileVectorOpH64),4302 MatrixIndexGPR32Op12_15,4303 uimm0s4range, ZZZZ_d_mul_r,4304 "mova">;4305 4306}4307 4308multiclass sme2_mova_vec_to_tile_vg4_multi<string mnemonic,4309 SDPatternOperator int_h, SDPatternOperator int_v>{4310 defm _H : sme2_mova_vec_to_tile_vg4_multi_base<0b0, mnemonic, int_h>;4311 defm _V : sme2_mova_vec_to_tile_vg4_multi_base<0b1, mnemonic, int_v>;4312}4313 4314// SME Move into Array4315class sme2_mova_vec_to_array_vg24_multi< bits<5> op, RegisterOperand array_ty,4316 RegisterOperand vector_ty,4317 string mnemonic,4318 string vg_acronym="">4319 : I<(outs array_ty:$ZAd),4320 (ins array_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rs, sme_elm_idx0_7:$imm,4321 vector_ty:$Zn),4322 mnemonic, "\t$ZAd[$Rs, $imm, " # vg_acronym # "], $Zn",4323 "", []>, Sched<[]> {4324 bits<0> ZAd;4325 bits<2> Rs;4326 bits<3> imm;4327 let Inst{31-15} = 0b11000000000001000;4328 let Inst{14-13} = Rs;4329 let Inst{12-11} = 0b01;4330 let Inst{10-6} = op;4331 let Inst{5-3} = 0b000;4332 let Inst{2-0} = imm;4333 4334 let Constraints = "$ZAd = $_ZAd";4335}4336 4337// MOVA (vector to array, two registers)4338multiclass sme2_mova_vec_to_array_vg2_multi<string mnemonic, SDPatternOperator intrinsic> {4339 def NAME : sme2_mova_vec_to_array_vg24_multi<{0,?,?,?,?}, MatrixOp64,4340 ZZ_d_mul_r, mnemonic, "vgx2">, SMEPseudo2Instr<NAME, 1> {4341 bits<4> Zn;4342 let Inst{9-6} = Zn;4343 }4344 4345 def NAME # _PSEUDO : sme2_move_to_za_pseudo<NAME, sme_elm_idx0_7, ZZ_d_mul_r, SMEMatrixArray>;4346 4347 def : SME2_ZA_VG1x2_Multi_Pat<NAME, intrinsic, nxv16i8, sme_elm_idx0_7, tileslice16>;4348 def : SME2_ZA_VG1x2_Multi_Pat<NAME, intrinsic, nxv8i16, sme_elm_idx0_7, tileslice16>;4349 def : SME2_ZA_VG1x2_Multi_Pat<NAME, intrinsic, nxv8f16, sme_elm_idx0_7, tileslice16>;4350 def : SME2_ZA_VG1x2_Multi_Pat<NAME, intrinsic, nxv8bf16, sme_elm_idx0_7, tileslice16>;4351 def : SME2_ZA_VG1x2_Multi_Pat<NAME, intrinsic, nxv4i32, sme_elm_idx0_7, tileslice16>;4352 def : SME2_ZA_VG1x2_Multi_Pat<NAME, intrinsic, nxv4f32, sme_elm_idx0_7, tileslice16>;4353 def : SME2_ZA_VG1x2_Multi_Pat<NAME, intrinsic, nxv2i64, sme_elm_idx0_7, tileslice16>;4354 def : SME2_ZA_VG1x2_Multi_Pat<NAME, intrinsic, nxv2f64, sme_elm_idx0_7, tileslice16>;4355 4356 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4357 MatrixOp8,4358 MatrixIndexGPR32Op8_11,4359 sme_elm_idx0_7, ZZ_b_mul_r,4360 "mova">;4361 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4362 MatrixOp16,4363 MatrixIndexGPR32Op8_11,4364 sme_elm_idx0_7, ZZ_h_mul_r,4365 "mova">;4366 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4367 MatrixOp32,4368 MatrixIndexGPR32Op8_11,4369 sme_elm_idx0_7, ZZ_s_mul_r,4370 "mova">;4371 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4372 MatrixOp64,4373 MatrixIndexGPR32Op8_11,4374 sme_elm_idx0_7, ZZ_d_mul_r,4375 "mova">;4376 4377 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4378 MatrixOp8,4379 MatrixIndexGPR32Op8_11,4380 sme_elm_idx0_7, ZZ_b_mul_r,4381 "mova", "vgx2">;4382 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4383 MatrixOp16,4384 MatrixIndexGPR32Op8_11,4385 sme_elm_idx0_7, ZZ_h_mul_r,4386 "mova", "vgx2">;4387 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4388 MatrixOp32,4389 MatrixIndexGPR32Op8_11,4390 sme_elm_idx0_7, ZZ_s_mul_r,4391 "mova", "vgx2">;4392 4393 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4394 MatrixOp8,4395 MatrixIndexGPR32Op8_11,4396 sme_elm_idx0_7, ZZ_b_mul_r,4397 "mov">;4398 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4399 MatrixOp16,4400 MatrixIndexGPR32Op8_11,4401 sme_elm_idx0_7, ZZ_h_mul_r,4402 "mov">;4403 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4404 MatrixOp32,4405 MatrixIndexGPR32Op8_11,4406 sme_elm_idx0_7, ZZ_s_mul_r,4407 "mov">;4408 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4409 MatrixOp64,4410 MatrixIndexGPR32Op8_11,4411 sme_elm_idx0_7, ZZ_d_mul_r,4412 "mov">;4413 4414 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4415 MatrixOp8,4416 MatrixIndexGPR32Op8_11,4417 sme_elm_idx0_7, ZZ_b_mul_r,4418 "mov", "vgx2">;4419 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4420 MatrixOp16,4421 MatrixIndexGPR32Op8_11,4422 sme_elm_idx0_7, ZZ_h_mul_r,4423 "mov", "vgx2">;4424 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4425 MatrixOp32,4426 MatrixIndexGPR32Op8_11,4427 sme_elm_idx0_7, ZZ_s_mul_r,4428 "mov", "vgx2">;4429 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME),4430 MatrixOp64,4431 MatrixIndexGPR32Op8_11,4432 sme_elm_idx0_7, ZZ_d_mul_r,4433 "mov", "vgx2">;4434}4435 4436// MOVA (vector to array, four registers)4437multiclass sme2_mova_vec_to_array_vg4_multi<string mnemonic, SDPatternOperator intrinsic> {4438 def NAME : sme2_mova_vec_to_array_vg24_multi<{1,?,?,?,0}, MatrixOp64,4439 ZZZZ_d_mul_r, mnemonic, "vgx4">, SMEPseudo2Instr<NAME, 1> {4440 bits<3> Zn;4441 let Inst{9-7} = Zn;4442 }4443 4444 def NAME # _PSEUDO : sme2_move_to_za_pseudo<NAME, sme_elm_idx0_7, ZZZZ_d_mul_r, SMEMatrixArray>;4445 4446 def : SME2_ZA_VG1x4_Multi_Pat<NAME, intrinsic, nxv16i8, sme_elm_idx0_7, tileslice16>;4447 def : SME2_ZA_VG1x4_Multi_Pat<NAME, intrinsic, nxv8i16, sme_elm_idx0_7, tileslice16>;4448 def : SME2_ZA_VG1x4_Multi_Pat<NAME, intrinsic, nxv8f16, sme_elm_idx0_7, tileslice16>;4449 def : SME2_ZA_VG1x4_Multi_Pat<NAME, intrinsic, nxv8bf16, sme_elm_idx0_7, tileslice16>;4450 def : SME2_ZA_VG1x4_Multi_Pat<NAME, intrinsic, nxv4i32, sme_elm_idx0_7, tileslice16>;4451 def : SME2_ZA_VG1x4_Multi_Pat<NAME, intrinsic, nxv4f32, sme_elm_idx0_7, tileslice16>;4452 def : SME2_ZA_VG1x4_Multi_Pat<NAME, intrinsic, nxv2i64, sme_elm_idx0_7, tileslice16>;4453 def : SME2_ZA_VG1x4_Multi_Pat<NAME, intrinsic, nxv2f64, sme_elm_idx0_7, tileslice16>;4454 4455 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4456 MatrixOp8,4457 MatrixIndexGPR32Op8_11,4458 sme_elm_idx0_7, ZZZZ_b_mul_r,4459 "mova">;4460 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4461 MatrixOp16,4462 MatrixIndexGPR32Op8_11,4463 sme_elm_idx0_7, ZZZZ_h_mul_r,4464 "mova">;4465 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4466 MatrixOp32,4467 MatrixIndexGPR32Op8_11,4468 sme_elm_idx0_7, ZZZZ_s_mul_r,4469 "mova">;4470 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4471 MatrixOp64,4472 MatrixIndexGPR32Op8_11,4473 sme_elm_idx0_7, ZZZZ_d_mul_r,4474 "mova">;4475 4476 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4477 MatrixOp8,4478 MatrixIndexGPR32Op8_11,4479 sme_elm_idx0_7, ZZZZ_b_mul_r,4480 "mova", "vgx4">;4481 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4482 MatrixOp16,4483 MatrixIndexGPR32Op8_11,4484 sme_elm_idx0_7, ZZZZ_h_mul_r,4485 "mova", "vgx4">;4486 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4487 MatrixOp32,4488 MatrixIndexGPR32Op8_11,4489 sme_elm_idx0_7, ZZZZ_s_mul_r,4490 "mova", "vgx4">;4491 4492 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4493 MatrixOp8,4494 MatrixIndexGPR32Op8_11,4495 sme_elm_idx0_7, ZZZZ_b_mul_r,4496 "mov">;4497 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4498 MatrixOp16,4499 MatrixIndexGPR32Op8_11,4500 sme_elm_idx0_7, ZZZZ_h_mul_r,4501 "mov">;4502 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4503 MatrixOp32,4504 MatrixIndexGPR32Op8_11,4505 sme_elm_idx0_7, ZZZZ_s_mul_r,4506 "mov">;4507 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4508 MatrixOp64,4509 MatrixIndexGPR32Op8_11,4510 sme_elm_idx0_7, ZZZZ_d_mul_r,4511 "mov">;4512 4513 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4514 MatrixOp8,4515 MatrixIndexGPR32Op8_11,4516 sme_elm_idx0_7, ZZZZ_b_mul_r,4517 "mov", "vgx4">;4518 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4519 MatrixOp16,4520 MatrixIndexGPR32Op8_11,4521 sme_elm_idx0_7, ZZZZ_h_mul_r,4522 "mov", "vgx4">;4523 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME),4524 MatrixOp32,4525 MatrixIndexGPR32Op8_11,4526 sme_elm_idx0_7, ZZZZ_s_mul_r,4527 "mov", "vgx4">;4528 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME),4529 MatrixOp64,4530 MatrixIndexGPR32Op8_11,4531 sme_elm_idx0_7, ZZZZ_d_mul_r,4532 "mov", "vgx4">;4533 4534}4535 4536class sme2_mova_tile_to_vec_vg2_multi_base<bits<2> sz, bit v, bits<3> op,4537 RegisterOperand vector_ty,4538 RegisterOperand tile_ty,4539 Operand index_ty,4540 string mnemonic>4541 : I<!if(op{1}, (outs vector_ty:$Zd, tile_ty:$_ZAn), (outs vector_ty:$Zd)),4542 (ins tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm),4543 mnemonic,4544 "\t$Zd, $ZAn[$Rs, $imm]",4545 "", []>, Sched<[]> {4546 bits<4> Zd;4547 bits<2> Rs;4548 let Inst{31-24} = 0b11000000;4549 let Inst{23-22} = sz;4550 let Inst{21-16} = 0b000110;4551 let Inst{15} = v;4552 let Inst{14-13} = Rs;4553 let Inst{12-11} = 0b00;4554 let Inst{10-8} = op;4555 let Inst{4-1} = Zd;4556 let Inst{0} = 0b0;4557 4558 let Constraints = !if(op{1}, "$ZAn = $_ZAn", "");4559}4560 4561multiclass sme2_mova_tile_or_array_to_vec_aliases<int op, Instruction inst,4562 RegisterOperand vector_ty,4563 RegisterOperand tile_or_array_ty,4564 RegisterOperand rv_ty,4565 Operand index_ty,4566 string mnemonic,4567 string vg_acronym=""> {4568def : InstAlias<mnemonic # "\t$Zd, $ZAn[$Rs, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "]",4569 (inst vector_ty:$Zd, tile_or_array_ty:$ZAn, rv_ty:$Rs, index_ty:$imm), op>;4570 4571}4572 4573multiclass sme2_mova_tile_to_vec_vg2_multi_inst<bit v, bits<3> opc, string mnemonic> {4574 4575 def _B : sme2_mova_tile_to_vec_vg2_multi_base<0b00, v, opc, ZZ_b_mul_r,4576 !if(v, TileVectorOpV8,4577 TileVectorOpH8),4578 uimm3s2range, mnemonic>, SMEPseudo2Instr<NAME # _B, 1> {4579 bits<0> ZAn;4580 bits<3> imm;4581 let Inst{7-5} = imm;4582 let mayLoad = 1;4583 }4584 4585 def _H : sme2_mova_tile_to_vec_vg2_multi_base<0b01, v, opc, ZZ_h_mul_r,4586 !if(v, TileVectorOpV16,4587 TileVectorOpH16),4588 uimm2s2range, mnemonic>, SMEPseudo2Instr<NAME # _H, 1> {4589 bits<1> ZAn;4590 bits<2> imm;4591 let Inst{7} = ZAn;4592 let Inst{6-5} = imm;4593 let mayLoad = 1;4594 }4595 4596 def _S : sme2_mova_tile_to_vec_vg2_multi_base<0b10, v, opc, ZZ_s_mul_r,4597 !if(v, TileVectorOpV32,4598 TileVectorOpH32),4599 uimm1s2range, mnemonic>, SMEPseudo2Instr<NAME # _S, 1> {4600 bits<2> ZAn;4601 bits<1> imm;4602 let Inst{7-6} = ZAn;4603 let Inst{5} = imm;4604 let mayLoad = 1;4605 }4606 4607 def _D : sme2_mova_tile_to_vec_vg2_multi_base<0b11, v, opc, ZZ_d_mul_r,4608 !if(v, TileVectorOpV64,4609 TileVectorOpH64),4610 uimm0s2range, mnemonic>, SMEPseudo2Instr<NAME # _D, 1> {4611 bits<3> ZAn;4612 bits<0> imm;4613 let Inst{7-5} = ZAn;4614 let mayLoad = 1;4615 }4616 4617 if !eq(mnemonic, "mova") then {4618 defm : sme2_mova_tile_or_array_to_vec_aliases<1,!cast<Instruction>(NAME # _B),4619 ZZ_b_mul_r,4620 !if(v, TileVectorOpV8,4621 TileVectorOpH8),4622 MatrixIndexGPR32Op12_15,4623 uimm3s2range, "mov">;4624 defm : sme2_mova_tile_or_array_to_vec_aliases<1,!cast<Instruction>(NAME # _H),4625 ZZ_h_mul_r,4626 !if(v, TileVectorOpV16,4627 TileVectorOpH16),4628 MatrixIndexGPR32Op12_15,4629 uimm2s2range, "mov">;4630 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _S),4631 ZZ_s_mul_r,4632 !if(v, TileVectorOpV32,4633 TileVectorOpH32),4634 MatrixIndexGPR32Op12_15,4635 uimm1s2range, "mov">;4636 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _D),4637 ZZ_d_mul_r,4638 !if(v, TileVectorOpV64,4639 TileVectorOpH64),4640 MatrixIndexGPR32Op12_15,4641 uimm0s2range, "mov">;4642 }4643 4644 defm : sme2_mova_tile_or_array_to_vec_aliases<0,!cast<Instruction>(NAME # _B),4645 ZZ_b_mul_r,4646 !if(v, TileVectorOpV8,4647 TileVectorOpH8),4648 MatrixIndexGPR32Op12_15,4649 uimm3s2range, mnemonic>;4650 defm : sme2_mova_tile_or_array_to_vec_aliases<0,!cast<Instruction>(NAME # _H),4651 ZZ_h_mul_r,4652 !if(v, TileVectorOpV16,4653 TileVectorOpH16),4654 MatrixIndexGPR32Op12_15,4655 uimm2s2range, mnemonic>;4656 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _S),4657 ZZ_s_mul_r,4658 !if(v, TileVectorOpV32,4659 TileVectorOpH32),4660 MatrixIndexGPR32Op12_15,4661 uimm1s2range, mnemonic>;4662 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _D),4663 ZZ_d_mul_r,4664 !if(v, TileVectorOpV64,4665 TileVectorOpH64),4666 MatrixIndexGPR32Op12_15,4667 uimm0s2range, mnemonic>;4668 4669}4670 4671// SME2 move tile to vector, two registers4672multiclass sme2_mova_tile_to_vec_vg2_multi<string mnemonic>{4673 defm _H : sme2_mova_tile_to_vec_vg2_multi_inst<0b0, 0b000, mnemonic>;4674 defm _V : sme2_mova_tile_to_vec_vg2_multi_inst<0b1, 0b000, mnemonic>;4675}4676 4677 4678// SME2p1 move tile to vector and zero tile, two registers4679multiclass sme2p1_movaz_tile_to_vec_vg2<string mnemonic>{4680 defm _H : sme2_mova_tile_to_vec_vg2_multi_inst<0b0, 0b010, mnemonic>;4681 defm _V : sme2_mova_tile_to_vec_vg2_multi_inst<0b1, 0b010, mnemonic>;4682 4683 4684 def NAME # _H_B_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _H_B, sme_elm_idx0_0, uimm3s2range, ZZ_b_mul_r, SMEMatrixTileB>;4685 def NAME # _H_H_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _H_H, sme_elm_idx0_1, uimm2s2range, ZZ_h_mul_r, SMEMatrixTileH>;4686 def NAME # _H_S_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _H_S, sme_elm_idx0_3, uimm1s2range, ZZ_s_mul_r, SMEMatrixTileS>;4687 def NAME # _H_D_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _H_D, sme_elm_idx0_7, uimm0s2range, ZZ_d_mul_r, SMEMatrixTileD>;4688 4689 def NAME # _V_B_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _V_B, sme_elm_idx0_0, uimm3s2range, ZZ_b_mul_r, SMEMatrixTileB>;4690 def NAME # _V_H_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _V_H, sme_elm_idx0_1, uimm2s2range, ZZ_h_mul_r, SMEMatrixTileH>;4691 def NAME # _V_S_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _V_S, sme_elm_idx0_3, uimm1s2range, ZZ_s_mul_r, SMEMatrixTileS>;4692 def NAME # _V_D_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _V_D, sme_elm_idx0_7, uimm0s2range, ZZ_d_mul_r, SMEMatrixTileD>;4693}4694 4695class sme2_mova_tile_to_vec_vg4_multi_base<bits<2> sz, bit v, bits<6> op,4696 RegisterOperand vector_ty,4697 RegisterOperand tile_ty,4698 Operand index_ty,4699 string mnemonic>4700 : I<!if(op{4}, (outs vector_ty:$Zd, tile_ty:$_ZAn), (outs vector_ty:$Zd)),4701 (ins tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm),4702 mnemonic,4703 "\t$Zd, $ZAn[$Rs, $imm]",4704 "", []>, Sched<[]> {4705 bits<3> Zd;4706 bits<2> Rs;4707 let Inst{31-24} = 0b11000000;4708 let Inst{23-22} = sz;4709 let Inst{21-16} = 0b000110;4710 let Inst{15} = v;4711 let Inst{14-13} = Rs;4712 let Inst{12-11} = 0b00;4713 let Inst{10-5} = op{5-0};4714 let Inst{4-2} = Zd;4715 let Inst{1-0} = 0b00;4716 4717 let Constraints = !if(op{4}, "$ZAn = $_ZAn", "");4718}4719 4720multiclass sme2_mova_tile_to_vec_vg4_multi_base<bit v, bits<3> opc, string mnemonic> {4721 4722 def _B : sme2_mova_tile_to_vec_vg4_multi_base<0b00, v, {opc,0,?,?},4723 ZZZZ_b_mul_r,4724 !if(v, TileVectorOpV8,4725 TileVectorOpH8),4726 uimm2s4range, mnemonic>, SMEPseudo2Instr<NAME # _B, 1> {4727 bits<0> ZAn;4728 bits<2> imm;4729 let Inst{6-5} = imm;4730 let mayLoad = 1;4731 }4732 4733 def _H : sme2_mova_tile_to_vec_vg4_multi_base<0b01, v, {opc,0,?,?},4734 ZZZZ_h_mul_r,4735 !if(v, TileVectorOpV16,4736 TileVectorOpH16),4737 uimm1s4range, mnemonic>, SMEPseudo2Instr<NAME # _H, 1> {4738 bits<1> ZAn;4739 bits<1> imm;4740 let Inst{6} = ZAn;4741 let Inst{5} = imm;4742 let mayLoad = 1;4743 }4744 4745 def _S : sme2_mova_tile_to_vec_vg4_multi_base<0b10, v, {opc,0,?,?},4746 ZZZZ_s_mul_r,4747 !if(v, TileVectorOpV32,4748 TileVectorOpH32),4749 uimm0s4range, mnemonic>, SMEPseudo2Instr<NAME # _S, 1> {4750 bits<2> ZAn;4751 bits<0> imm;4752 let Inst{6-5} = ZAn;4753 let mayLoad = 1;4754 }4755 4756 def _D : sme2_mova_tile_to_vec_vg4_multi_base<0b11, v, {opc,?,?,?},4757 ZZZZ_d_mul_r,4758 !if(v, TileVectorOpV64,4759 TileVectorOpH64),4760 uimm0s4range, mnemonic>, SMEPseudo2Instr<NAME # _D, 1> {4761 bits<3> ZAn;4762 bits<0> imm;4763 let Inst{7-5} = ZAn;4764 let mayLoad = 1;4765 }4766 4767 if !eq(mnemonic, "mova") then {4768 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _B),4769 ZZZZ_b_mul_r,4770 !if(v, TileVectorOpV8,4771 TileVectorOpH8),4772 MatrixIndexGPR32Op12_15,4773 uimm2s4range, "mov">;4774 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _H),4775 ZZZZ_h_mul_r,4776 !if(v, TileVectorOpV16,4777 TileVectorOpH16),4778 MatrixIndexGPR32Op12_15,4779 uimm1s4range, "mov">;4780 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _S),4781 ZZZZ_s_mul_r,4782 !if(v, TileVectorOpV32,4783 TileVectorOpH32),4784 MatrixIndexGPR32Op12_15,4785 uimm0s4range, "mov">;4786 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _D),4787 ZZZZ_d_mul_r,4788 !if(v, TileVectorOpV64,4789 TileVectorOpH64),4790 MatrixIndexGPR32Op12_15,4791 uimm0s4range, "mov">;4792 }4793 4794 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _B),4795 ZZZZ_b_mul_r,4796 !if(v, TileVectorOpV8,4797 TileVectorOpH8),4798 MatrixIndexGPR32Op12_15,4799 uimm2s4range, mnemonic>;4800 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _H),4801 ZZZZ_h_mul_r,4802 !if(v, TileVectorOpV16,4803 TileVectorOpH16),4804 MatrixIndexGPR32Op12_15,4805 uimm1s4range, mnemonic>;4806 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _S),4807 ZZZZ_s_mul_r,4808 !if(v, TileVectorOpV32,4809 TileVectorOpH32),4810 MatrixIndexGPR32Op12_15,4811 uimm0s4range, mnemonic>;4812 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _D),4813 ZZZZ_d_mul_r,4814 !if(v, TileVectorOpV64,4815 TileVectorOpH64),4816 MatrixIndexGPR32Op12_15,4817 uimm0s4range, mnemonic>;4818 4819}4820 4821// SME2 move tile to vector, four registers4822multiclass sme2_mova_tile_to_vec_vg4_multi<string mnemonic>{4823 defm _H : sme2_mova_tile_to_vec_vg4_multi_base<0b0, 0b100, mnemonic>;4824 defm _V : sme2_mova_tile_to_vec_vg4_multi_base<0b1, 0b100, mnemonic>;4825}4826 4827// SME2p1 move tile to vector and zero tile, four registers4828multiclass sme2p1_movaz_tile_to_vec_vg4<string mnemonic>{4829 defm _H : sme2_mova_tile_to_vec_vg4_multi_base<0b0, 0b110, mnemonic>;4830 defm _V : sme2_mova_tile_to_vec_vg4_multi_base<0b1, 0b110, mnemonic>;4831 4832 def NAME # _H_B_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _H_B, sme_elm_idx0_0, uimm2s4range, ZZZZ_b_mul_r, SMEMatrixTileB>;4833 def NAME # _H_H_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _H_H, sme_elm_idx0_1, uimm1s4range, ZZZZ_h_mul_r, SMEMatrixTileH>;4834 def NAME # _H_S_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _H_S, sme_elm_idx0_3, uimm0s4range, ZZZZ_s_mul_r, SMEMatrixTileS>;4835 def NAME # _H_D_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _H_D, sme_elm_idx0_7, uimm0s4range, ZZZZ_d_mul_r, SMEMatrixTileD>;4836 4837 def NAME # _V_B_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _V_B, sme_elm_idx0_0, uimm2s4range, ZZZZ_b_mul_r, SMEMatrixTileB>;4838 def NAME # _V_H_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _V_H, sme_elm_idx0_1, uimm1s4range, ZZZZ_h_mul_r, SMEMatrixTileH>;4839 def NAME # _V_S_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _V_S, sme_elm_idx0_3, uimm0s4range, ZZZZ_s_mul_r, SMEMatrixTileS>;4840 def NAME # _V_D_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _V_D, sme_elm_idx0_7, uimm0s4range, ZZZZ_d_mul_r, SMEMatrixTileD>;4841}4842 4843 4844class sme2_mova_array_to_vec_vg24_multi<bits<4>op, RegisterOperand vector_ty,4845 RegisterOperand array_ty,4846 string mnemonic, string vg_acronym>4847 : I<!if(op{2}, (outs vector_ty:$Zd, array_ty:$_ZAn), (outs vector_ty:$Zd)),4848 (ins array_ty:$ZAn, MatrixIndexGPR32Op8_11:$Rs, sme_elm_idx0_7:$imm),4849 mnemonic,4850 "\t$Zd, $ZAn[$Rs, $imm, " # vg_acronym # "]",4851 "", []>, Sched<[]> {4852 bits<0> ZAn;4853 bits<2> Rs;4854 bits<3> imm;4855 let Inst{31-15} = 0b11000000000001100;4856 let Inst{14-13} = Rs;4857 let Inst{12-11} = 0b01;4858 let Inst{10-8} = op{3-1};4859 let Inst{7-5} = imm;4860 let Inst{1} = op{0};4861 let Inst{0} = 0b0;4862 let Constraints = !if(op{2}, "$ZAn = $_ZAn", "");4863}4864 4865// move array to vector, two registers.4866multiclass sme2_mova_array_to_vec_vg2_multi<bits<3> opc, string mnemonic> {4867 def NAME : sme2_mova_array_to_vec_vg24_multi<{opc,?}, ZZ_d_mul_r, MatrixOp64,4868 mnemonic, "vgx2">, SMEPseudo2Instr<NAME, 1>{4869 bits<4> Zd;4870 let Inst{4-1} = Zd;4871 }4872 4873 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4874 ZZ_b_mul_r, MatrixOp8,4875 MatrixIndexGPR32Op8_11,4876 sme_elm_idx0_7, mnemonic>;4877 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4878 ZZ_h_mul_r, MatrixOp16,4879 MatrixIndexGPR32Op8_11,4880 sme_elm_idx0_7, mnemonic>;4881 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4882 ZZ_s_mul_r, MatrixOp32,4883 MatrixIndexGPR32Op8_11,4884 sme_elm_idx0_7, mnemonic>;4885 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4886 ZZ_d_mul_r, MatrixOp64,4887 MatrixIndexGPR32Op8_11,4888 sme_elm_idx0_7, mnemonic>;4889 4890 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4891 ZZ_b_mul_r, MatrixOp8,4892 MatrixIndexGPR32Op8_11,4893 sme_elm_idx0_7, mnemonic, "vgx2">;4894 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4895 ZZ_h_mul_r, MatrixOp16,4896 MatrixIndexGPR32Op8_11,4897 sme_elm_idx0_7, mnemonic, "vgx2">;4898 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4899 ZZ_s_mul_r, MatrixOp32,4900 MatrixIndexGPR32Op8_11,4901 sme_elm_idx0_7, mnemonic, "vgx2">;4902 4903 if !eq(mnemonic, "mova") then {4904 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4905 ZZ_b_mul_r, MatrixOp8,4906 MatrixIndexGPR32Op8_11,4907 sme_elm_idx0_7, "mov">;4908 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4909 ZZ_h_mul_r, MatrixOp16,4910 MatrixIndexGPR32Op8_11,4911 sme_elm_idx0_7, "mov">;4912 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4913 ZZ_s_mul_r, MatrixOp32,4914 MatrixIndexGPR32Op8_11,4915 sme_elm_idx0_7, "mov">;4916 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4917 ZZ_d_mul_r, MatrixOp64,4918 MatrixIndexGPR32Op8_11,4919 sme_elm_idx0_7, "mov">;4920 4921 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4922 ZZ_b_mul_r, MatrixOp8,4923 MatrixIndexGPR32Op8_11,4924 sme_elm_idx0_7, "mov", "vgx2">;4925 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4926 ZZ_h_mul_r, MatrixOp16,4927 MatrixIndexGPR32Op8_11,4928 sme_elm_idx0_7, "mov", "vgx2">;4929 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4930 ZZ_s_mul_r, MatrixOp32,4931 MatrixIndexGPR32Op8_11,4932 sme_elm_idx0_7, "mov", "vgx2">;4933 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME),4934 ZZ_d_mul_r, MatrixOp64,4935 MatrixIndexGPR32Op8_11,4936 sme_elm_idx0_7, "mov", "vgx2">;4937 }4938}4939 4940multiclass sme2_movaz_array_to_vec_vg2_multi<string mnemonic> {4941 defm NAME : sme2_mova_array_to_vec_vg2_multi<0b010, mnemonic>;4942 def NAME # _PSEUDO : sme2_movaz_array_to_tile_pseudo<NAME, sme_elm_idx0_7, ZZ_d_mul_r, SMEMatrixArray>;4943}4944 4945// move array to vector, four registers4946multiclass sme2_mova_array_to_vec_vg4_multi<bits<4> opc, string mnemonic> {4947 def NAME : sme2_mova_array_to_vec_vg24_multi<opc, ZZZZ_d_mul_r, MatrixOp64,4948 mnemonic, "vgx4">, SMEPseudo2Instr<NAME, 1> {4949 bits<3> Zd;4950 let Inst{4-2} = Zd;4951 }4952 4953 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4954 ZZZZ_b_mul_r, MatrixOp8,4955 MatrixIndexGPR32Op8_11,4956 sme_elm_idx0_7, mnemonic>;4957 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4958 ZZZZ_h_mul_r, MatrixOp16,4959 MatrixIndexGPR32Op8_11,4960 sme_elm_idx0_7, mnemonic>;4961 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4962 ZZZZ_s_mul_r, MatrixOp32,4963 MatrixIndexGPR32Op8_11,4964 sme_elm_idx0_7, mnemonic>;4965 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4966 ZZZZ_d_mul_r, MatrixOp64,4967 MatrixIndexGPR32Op8_11,4968 sme_elm_idx0_7, mnemonic>;4969 4970 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4971 ZZZZ_b_mul_r, MatrixOp8,4972 MatrixIndexGPR32Op8_11,4973 sme_elm_idx0_7, mnemonic, "vgx4">;4974 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4975 ZZZZ_h_mul_r, MatrixOp16,4976 MatrixIndexGPR32Op8_11,4977 sme_elm_idx0_7, mnemonic, "vgx4">;4978 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4979 ZZZZ_s_mul_r, MatrixOp32,4980 MatrixIndexGPR32Op8_11,4981 sme_elm_idx0_7, mnemonic, "vgx4">;4982 4983 if !eq(mnemonic, "mova") then {4984 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4985 ZZZZ_b_mul_r, MatrixOp8,4986 MatrixIndexGPR32Op8_11,4987 sme_elm_idx0_7, "mov">;4988 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4989 ZZZZ_h_mul_r, MatrixOp16,4990 MatrixIndexGPR32Op8_11,4991 sme_elm_idx0_7, "mov">;4992 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4993 ZZZZ_s_mul_r, MatrixOp32,4994 MatrixIndexGPR32Op8_11,4995 sme_elm_idx0_7, "mov">;4996 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),4997 ZZZZ_d_mul_r, MatrixOp64,4998 MatrixIndexGPR32Op8_11,4999 sme_elm_idx0_7, "mov">;5000 5001 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),5002 ZZZZ_b_mul_r, MatrixOp8,5003 MatrixIndexGPR32Op8_11,5004 sme_elm_idx0_7, "mov", "vgx4">;5005 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),5006 ZZZZ_h_mul_r, MatrixOp16,5007 MatrixIndexGPR32Op8_11,5008 sme_elm_idx0_7, "mov", "vgx4">;5009 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME),5010 ZZZZ_s_mul_r, MatrixOp32,5011 MatrixIndexGPR32Op8_11,5012 sme_elm_idx0_7, "mov", "vgx4">;5013 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME),5014 ZZZZ_d_mul_r, MatrixOp64,5015 MatrixIndexGPR32Op8_11,5016 sme_elm_idx0_7, "mov", "vgx4">;5017 }5018}5019 5020multiclass sme2_movaz_array_to_vec_vg4_multi<string mnemonic> {5021 defm NAME : sme2_mova_array_to_vec_vg4_multi<0b1100, mnemonic>;5022 def NAME # _PSEUDO : sme2_movaz_array_to_tile_pseudo<NAME, sme_elm_idx0_7, ZZZZ_d_mul_r, SMEMatrixArray>;5023}5024 5025//===----------------------------------------------------------------------===//5026// SME2 multi-vec saturating shift right narrow5027class sme2_sat_shift_vector_vg2<string mnemonic, bit op, bit u>5028 : I<(outs ZPR16:$Zd), (ins ZZ_s_mul_r:$Zn, vecshiftR16:$imm4),5029 mnemonic, "\t$Zd, $Zn, $imm4",5030 "", []>, Sched<[]> {5031 bits<4> imm4;5032 bits<4> Zn;5033 bits<5> Zd;5034 let Inst{31-21} = 0b11000001111;5035 let Inst{20} = op;5036 let Inst{19-16} = imm4;5037 let Inst{15-10} = 0b110101;5038 let Inst{9-6} = Zn;5039 let Inst{5} = u;5040 let Inst{4-0} = Zd;5041}5042 5043multiclass sme2_sat_shift_vector_vg2<string mnemonic, bit op, bit u, SDPatternOperator intrinsic> {5044 def _H : sme2_sat_shift_vector_vg2<mnemonic, op, u>;5045 5046 def : SME2_Sat_Shift_VG2_Pat<NAME # _H, intrinsic, nxv8i16, nxv4i32, vecshiftR16>;5047}5048 5049class sme2_sat_shift_vector_vg4<bits<2> sz, bits<3> op, ZPRRegOp zpr_ty,5050 RegisterOperand vector_ty, Operand imm_ty,5051 string mnemonic>5052 : I<(outs zpr_ty:$Zd), (ins vector_ty:$Zn, imm_ty:$imm),5053 mnemonic, "\t$Zd, $Zn, $imm",5054 "", []>, Sched<[]> {5055 bits<3> Zn;5056 bits<5> Zd;5057 let Inst{31-24} = 0b11000001;5058 let Inst{23-22} = sz;5059 let Inst{21} = 0b1;5060 // Inst{20-16} = imm5;5061 let Inst{15-11} = 0b11011;5062 let Inst{10} = op{2};5063 let Inst{9-7} = Zn;5064 let Inst{6-5} = op{1-0};5065 let Inst{4-0} = Zd;5066}5067 5068multiclass sme2_sat_shift_vector_vg4<string mnemonic, bits<3> op, SDPatternOperator intrinsic> {5069 def _B : sme2_sat_shift_vector_vg4<{0,1}, op, ZPR8, ZZZZ_s_mul_r, vecshiftR32,5070 mnemonic>{5071 bits<5> imm;5072 let Inst{20-16} = imm;5073 }5074 def _H : sme2_sat_shift_vector_vg4<{1,?}, op, ZPR16, ZZZZ_d_mul_r, vecshiftR64,5075 mnemonic> {5076 bits<6> imm;5077 let Inst{22} = imm{5};5078 let Inst{20-16} = imm{4-0};5079 }5080 5081 def : SME2_Sat_Shift_VG4_Pat<NAME # _B, intrinsic, nxv16i8, nxv4i32, vecshiftR32>;5082 def : SME2_Sat_Shift_VG4_Pat<NAME # _H, intrinsic, nxv8i16, nxv2i64, vecshiftR64>;5083}5084 5085//===----------------------------------------------------------------------===//5086// SME2 Multi-vector - SVE Select5087class sme2_sel_vector_vg24<bits<2> sz, bits<4> op, RegisterOperand vector_ty,5088 string mnemonic>5089 : I<(outs vector_ty:$Zd),5090 (ins PNRAny_p8to15:$PNg, vector_ty:$Zn, vector_ty:$Zm),5091 mnemonic, "\t$Zd, $PNg, $Zn, $Zm",5092 "", []>, Sched<[]> {5093 bits<3> PNg;5094 let Inst{31-24} = 0b11000001;5095 let Inst{23-22} = sz;5096 let Inst{21} = 0b1;5097 let Inst{17-16} = op{3-2};5098 let Inst{15-13} = 0b100;5099 let Inst{12-10} = PNg;5100 let Inst{6} = op{1};5101 let Inst{5} = 0b0;5102 let Inst{1} = op{0};5103 let Inst{0} = 0b0;5104}5105 5106class sme2_sel_vector_vg2<bits<2> sz, RegisterOperand vector_ty,5107 string mnemonic>5108 : sme2_sel_vector_vg24<sz, {?,0,?,?}, vector_ty, mnemonic> {5109 bits<4> Zm;5110 bits<4> Zn;5111 bits<4> Zd;5112 let Inst{20-17} = Zm;5113 let Inst{9-6} = Zn;5114 let Inst{4-1} = Zd;5115}5116 5117multiclass sme2_sel_vector_vg2<string mnemonic>{5118 def _B : sme2_sel_vector_vg2<0b00, ZZ_b_mul_r, mnemonic>;5119 def _H : sme2_sel_vector_vg2<0b01, ZZ_h_mul_r, mnemonic>;5120 def _S : sme2_sel_vector_vg2<0b10, ZZ_s_mul_r, mnemonic>;5121 def _D : sme2_sel_vector_vg2<0b11, ZZ_d_mul_r, mnemonic>;5122}5123class sme2_sel_vector_vg4<bits<2> sz, RegisterOperand vector_ty,5124 string mnemonic>5125 : sme2_sel_vector_vg24<sz, 0b0100, vector_ty, mnemonic> {5126 bits<3> Zm;5127 bits<3> Zn;5128 bits<3> Zd;5129 let Inst{20-18} = Zm;5130 let Inst{9-7} = Zn;5131 let Inst{4-2} = Zd;5132}5133multiclass sme2_sel_vector_vg4<string mnemonic> {5134 def _B : sme2_sel_vector_vg4<0b00, ZZZZ_b_mul_r, mnemonic>;5135 def _H : sme2_sel_vector_vg4<0b01, ZZZZ_h_mul_r, mnemonic>;5136 def _S : sme2_sel_vector_vg4<0b10, ZZZZ_s_mul_r, mnemonic>;5137 def _D : sme2_sel_vector_vg4<0b11, ZZZZ_d_mul_r, mnemonic>;5138}5139 5140//===----------------------------------------------------------------------===//5141// Non contiguous Load and Store5142 5143class sme2_ld_vector_vg2_multi_scalar_scalar<bits<2> msz, bit n,5144 RegisterOperand multi_vector_ty,5145 RegisterOperand gpr_ty,5146 string mnemonic>5147 : I<(outs multi_vector_ty:$Zt),5148 (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm),5149 mnemonic, "\t$Zt, $PNg/z, [$Rn, $Rm]",5150 "", []>, Sched<[]> {5151 bits<5> Rm;5152 bits<3> PNg;5153 bits<5> Rn;5154 bits<4> Zt;5155 let Inst{31-21} = 0b10100001000;5156 let Inst{20-16} = Rm;5157 let Inst{15} = 0b0;5158 let Inst{14-13} = msz;5159 let Inst{12-10} = PNg;5160 let Inst{9-5} = Rn;5161 let Inst{4} = Zt{3};5162 let Inst{3} = n;5163 let Inst{2-0} = Zt{2-0};5164 5165 let mayLoad = 1;5166}5167 5168class sme2_ld_vector_vg4_multi_scalar_scalar<bits<2> msz, bit n,5169 RegisterOperand multi_vector_ty,5170 RegisterOperand gpr_ty,5171 string mnemonic>5172 : I<(outs multi_vector_ty:$Zt),5173 (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm),5174 mnemonic, "\t$Zt, $PNg/z, [$Rn, $Rm]",5175 "", []>, Sched<[]> {5176 bits<5> Rm;5177 bits<3> PNg;5178 bits<5> Rn;5179 bits<3> Zt;5180 let Inst{31-21} = 0b10100001000;5181 let Inst{20-16} = Rm;5182 let Inst{15} = 0b1;5183 let Inst{14-13} = msz;5184 let Inst{12-10} = PNg;5185 let Inst{9-5} = Rn;5186 let Inst{4} = Zt{2};5187 let Inst{3} = n;5188 let Inst{2} = 0b0;5189 let Inst{1-0} = Zt{1-0};5190 5191 let mayLoad = 1;5192}5193 5194class sme2_ld_vector_vg24_multi_scalar_immediate<bits<2> msz, bit n, bits<2> op,5195 RegisterOperand multi_vector_ty,5196 Operand index_ty,5197 string mnemonic>5198 : I<(outs multi_vector_ty:$Zt),5199 (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, index_ty:$imm4),5200 mnemonic, "\t$Zt, $PNg/z, [$Rn, $imm4, mul vl]",5201 "", []>, Sched<[]> {5202 bits<4> imm4;5203 bits<3> PNg;5204 bits<5> Rn;5205 let Inst{31-20} = 0b101000010100;5206 let Inst{19-16} = imm4;5207 let Inst{15} = op{1};5208 let Inst{14-13} = msz;5209 let Inst{12-10} = PNg;5210 let Inst{9-5} = Rn;5211 let Inst{3} = n;5212 let Inst{2} = op{0};5213 5214 let mayLoad = 1;5215}5216 5217multiclass sme2_ld_vector_vg2_multi_scalar_immediate<bits<2> msz, bit n,5218 RegisterOperand multi_vector_ty,5219 Operand index_ty,5220 string mnemonic>{5221 def NAME : sme2_ld_vector_vg24_multi_scalar_immediate<msz, n, {0,?},5222 multi_vector_ty,5223 index_ty, mnemonic> {5224 bits<4> Zt;5225 let Inst{4} = Zt{3};5226 let Inst{2-0} = Zt{2-0};5227 }5228 5229 def : InstAlias<mnemonic # "\t$Zt, $PNg/z, [$Rn]",5230 (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, 0), 1>;5231}5232 5233multiclass sme2_ld_vector_vg4_multi_scalar_immediate<bits<2> msz, bit n,5234 RegisterOperand multi_vector_ty,5235 Operand index_ty,5236 string mnemonic> {5237 def NAME : sme2_ld_vector_vg24_multi_scalar_immediate<msz, n, 0b10,5238 multi_vector_ty,5239 index_ty, mnemonic> {5240 bits<3> Zt;5241 let Inst{4} = Zt{2};5242 let Inst{1-0} = Zt{1-0};5243 }5244 5245 def : InstAlias<mnemonic # "\t$Zt, $PNg/z, [$Rn]",5246 (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, 0), 1>;5247}5248 5249//===----------------------------------------------------------------------===//5250// SME2 Non-Contiguous Store5251class sme2_st_vector_vg2_multi_scalar_scalar<bits<2> msz, bit n,5252 RegisterOperand multi_vector_ty,5253 RegisterOperand gpr_ty,5254 string mnemonic>5255 : I<(outs ),5256 (ins multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm),5257 mnemonic, "\t$Zt, $PNg, [$Rn, $Rm]",5258 "", []>, Sched<[]> {5259 bits<5> Rm;5260 bits<3> PNg;5261 bits<5> Rn;5262 bits<4> Zt;5263 let Inst{31-21} = 0b10100001001;5264 let Inst{20-16} = Rm;5265 let Inst{15} = 0b0;5266 let Inst{14-13} = msz;5267 let Inst{12-10} = PNg;5268 let Inst{9-5} = Rn;5269 let Inst{4} = Zt{3};5270 let Inst{3} = n;5271 let Inst{2-0} = Zt{2-0};5272 5273 let mayStore = 1;5274}5275 5276class sme2_st_vector_vg4_multi_scalar_scalar<bits<2> msz, bit n,5277 RegisterOperand multi_vector_ty,5278 RegisterOperand gpr_ty,5279 string mnemonic>5280 : I<(outs ),5281 (ins multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm),5282 mnemonic, "\t$Zt, $PNg, [$Rn, $Rm]",5283 "", []>, Sched<[]> {5284 bits<5> Rm;5285 bits<3> PNg;5286 bits<5> Rn;5287 bits<3> Zt;5288 let Inst{31-21} = 0b10100001001;5289 let Inst{20-16} = Rm;5290 let Inst{15} = 0b1;5291 let Inst{14-13} = msz;5292 let Inst{12-10} = PNg;5293 let Inst{9-5} = Rn;5294 let Inst{4} = Zt{2};5295 let Inst{3} = n;5296 let Inst{2} = 0b0;5297 let Inst{1-0} = Zt{1-0};5298 5299 let mayStore = 1;5300}5301 5302class sme2_st_vector_vg24_multi_scalar_immediate<bits<2> msz, bit n, bits<2> op,5303 RegisterOperand multi_vector_ty,5304 Operand index_ty,5305 string mnemonic>5306 : I<(outs ),5307 (ins multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, index_ty:$imm4),5308 mnemonic, "\t$Zt, $PNg, [$Rn, $imm4, mul vl]",5309 "", []>, Sched<[]> {5310 bits<4> imm4;5311 bits<3> PNg;5312 bits<5> Rn;5313 let Inst{31-20} = 0b101000010110;5314 let Inst{19-16} = imm4;5315 let Inst{15} = op{1};5316 let Inst{14-13} = msz;5317 let Inst{12-10} = PNg;5318 let Inst{9-5} = Rn;5319 let Inst{3} = n;5320 let Inst{2} = op{0};5321 5322 let mayStore = 1;5323}5324 5325 5326multiclass sme2_st_vector_vg2_multi_scalar_immediate<bits<2> msz, bit n,5327 RegisterOperand multi_vector_ty,5328 Operand index_ty,5329 string mnemonic> {5330 def NAME: sme2_st_vector_vg24_multi_scalar_immediate<msz, n, {0,?},5331 multi_vector_ty,5332 index_ty, mnemonic> {5333 bits<4> Zt;5334 let Inst{4} = Zt{3};5335 let Inst{2-0} = Zt{2-0};5336 }5337 5338 def : InstAlias<mnemonic # "\t$Zt, $PNg, [$Rn]",5339 (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn,0), 1>;5340}5341 5342multiclass sme2_st_vector_vg4_multi_scalar_immediate<bits<2> msz, bit n,5343 RegisterOperand multi_vector_ty,5344 Operand index_ty,5345 string mnemonic> {5346 def NAME : sme2_st_vector_vg24_multi_scalar_immediate<msz, n, 0b10,5347 multi_vector_ty,5348 index_ty, mnemonic> {5349 bits<3> Zt;5350 let Inst{4} = Zt{2};5351 let Inst{1-0} = Zt{1-0};5352 }5353 5354 def : InstAlias<mnemonic # "\t$Zt, $PNg, [$Rn]",5355 (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn,0), 1>;5356}5357 5358//===----------------------------------------------------------------------===//5359// SME2.15360//===----------------------------------------------------------------------===//5361// SME zeroing move array to vector5362class sme2p1_movaz_tile_to_vec_base<bits<2> sz, bit q, bit v, ZPRRegOp vector_ty,5363 RegisterOperand tile_ty, Operand index_ty,5364 string mnemonic>5365 : I<(outs vector_ty:$Zd, tile_ty:$ZAn),5366 (ins tile_ty:$_ZAn, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm),5367 mnemonic, "\t$Zd, $ZAn[$Rs, $imm]",5368 "", []>, Sched<[]> {5369 bits<2> Rs;5370 bits<5> Zd;5371 let Inst{31-24} = 0b11000000;5372 let Inst{23-22} = sz;5373 let Inst{21-17} = 0b00001;5374 let Inst{16} = q;5375 let Inst{15} = v;5376 let Inst{14-13} = Rs;5377 let Inst{12-9} = 0b0001;5378 let Inst{4-0} = Zd;5379 let Constraints = "$ZAn = $_ZAn";5380}5381 5382multiclass sme2p1_movaz_tile_to_vec_base<bit v, string mnemonic> {5383 def _B : sme2p1_movaz_tile_to_vec_base<0b00, 0b0, v, ZPR8,5384 !if(v, TileVectorOpV8, TileVectorOpH8),5385 sme_elm_idx0_15, mnemonic>, SMEPseudo2Instr<NAME # _B, 1> {5386 bits<0> ZAn;5387 bits<4> imm;5388 let Inst{8-5} = imm;5389 }5390 5391 def _H : sme2p1_movaz_tile_to_vec_base<0b01, 0b0, v, ZPR16,5392 !if(v, TileVectorOpV16, TileVectorOpH16),5393 sme_elm_idx0_7, mnemonic>, SMEPseudo2Instr<NAME # _H, 1> {5394 bits<1> ZAn;5395 bits<3> imm;5396 let Inst{8} = ZAn;5397 let Inst{7-5} = imm;5398 }5399 5400 def _S : sme2p1_movaz_tile_to_vec_base<0b10, 0b0, v, ZPR32,5401 !if(v, TileVectorOpV32, TileVectorOpH32),5402 sme_elm_idx0_3, mnemonic>, SMEPseudo2Instr<NAME # _S, 1> {5403 bits<2> ZAn;5404 bits<2> imm;5405 let Inst{8-7} = ZAn;5406 let Inst{6-5} = imm;5407 }5408 5409 def _D : sme2p1_movaz_tile_to_vec_base<0b11, 0b0, v, ZPR64,5410 !if(v, TileVectorOpV64, TileVectorOpH64),5411 sme_elm_idx0_1, mnemonic>, SMEPseudo2Instr<NAME # _D, 1> {5412 bits<3> ZAn;5413 bits<1> imm;5414 let Inst{8-6} = ZAn;5415 let Inst{5} = imm;5416 }5417 5418 def _Q : sme2p1_movaz_tile_to_vec_base<0b11, 0b1, v, ZPR128,5419 !if(v, TileVectorOpV128, TileVectorOpH128),5420 sme_elm_idx0_0, mnemonic>, SMEPseudo2Instr<NAME # _Q, 1> {5421 bits<4> ZAn;5422 bits<0> imm;5423 let Inst{8-5} = ZAn;5424 }5425}5426 5427multiclass sme2p1_movaz_tile_to_vec<string mnemonic, SDPatternOperator intrinsic_horiz, SDPatternOperator intrinsic_vert,5428 SDPatternOperator intrinsic_horiz_q, SDPatternOperator intrinsic_vert_q>{5429 defm _H : sme2p1_movaz_tile_to_vec_base<0b0, mnemonic>;5430 defm _V : sme2p1_movaz_tile_to_vec_base<0b1, mnemonic>;5431 5432 def NAME # _H_B_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _H_B, sme_elm_idx0_0, sme_elm_idx0_15, ZPR8, SMEMatrixTileB>;5433 def NAME # _H_H_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _H_H, sme_elm_idx0_1, sme_elm_idx0_7, ZPR16, SMEMatrixTileH>;5434 def NAME # _H_S_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _H_S, sme_elm_idx0_3, sme_elm_idx0_3, ZPR32, SMEMatrixTileS>;5435 def NAME # _H_D_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _H_D, sme_elm_idx0_7, sme_elm_idx0_1, ZPR64, SMEMatrixTileD>;5436 def NAME # _H_Q_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _H_Q, sme_elm_idx0_15, sme_elm_idx0_0, ZPR128, SMEMatrixTileQ>;5437 5438 def NAME # _V_B_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _V_B, sme_elm_idx0_0, sme_elm_idx0_15, ZPR8, SMEMatrixTileB>;5439 def NAME # _V_H_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _V_H, sme_elm_idx0_1, sme_elm_idx0_7, ZPR16, SMEMatrixTileH>;5440 def NAME # _V_S_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _V_S, sme_elm_idx0_3, sme_elm_idx0_3, ZPR32, SMEMatrixTileS>;5441 def NAME # _V_D_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _V_D, sme_elm_idx0_7, sme_elm_idx0_1, ZPR64, SMEMatrixTileD>;5442 def NAME # _V_Q_PSEUDO : sme2_movez_to_tile_pseudo<NAME # _V_Q, sme_elm_idx0_15, sme_elm_idx0_0, ZPR128, SMEMatrixTileQ>;5443 5444 def : SME2_Tile_Movaz_Pat<NAME # _H_B, intrinsic_horiz, nxv16i8,sme_elm_idx0_0, sme_elm_idx0_15, tileslice8>;5445 def : SME2_Tile_Movaz_Pat<NAME # _H_H, intrinsic_horiz, nxv8i16, sme_elm_idx0_1, sme_elm_idx0_7, tileslice16>;5446 def : SME2_Tile_Movaz_Pat<NAME # _H_S, intrinsic_horiz, nxv4i32, sme_elm_idx0_3, sme_elm_idx0_3, tileslice32>;5447 def : SME2_Tile_Movaz_Pat<NAME # _H_D, intrinsic_horiz, nxv2i64, sme_elm_idx0_7, sme_elm_idx0_1, tileslice64>;5448 def : SME2_Tile_Movaz_Pat<NAME # _H_H, intrinsic_horiz, nxv8bf16, sme_elm_idx0_1, sme_elm_idx0_7, tileslice16>;5449 def : SME2_Tile_Movaz_Pat<NAME # _H_H, intrinsic_horiz, nxv8f16, sme_elm_idx0_1, sme_elm_idx0_7, tileslice16>;5450 def : SME2_Tile_Movaz_Pat<NAME # _H_S, intrinsic_horiz, nxv4f32, sme_elm_idx0_3, sme_elm_idx0_3, tileslice32>;5451 def : SME2_Tile_Movaz_Pat<NAME # _H_D, intrinsic_horiz, nxv2f64, sme_elm_idx0_7, sme_elm_idx0_1, tileslice64>;5452 5453 def : SME2_Tile_Movaz_Pat<NAME # _V_B, intrinsic_vert, nxv16i8, sme_elm_idx0_0, sme_elm_idx0_15, tileslice8>;5454 def : SME2_Tile_Movaz_Pat<NAME # _V_H, intrinsic_vert, nxv8i16, sme_elm_idx0_1, sme_elm_idx0_7, tileslice16>;5455 def : SME2_Tile_Movaz_Pat<NAME # _V_S, intrinsic_vert, nxv4i32, sme_elm_idx0_3, sme_elm_idx0_3, tileslice32>;5456 def : SME2_Tile_Movaz_Pat<NAME # _V_D, intrinsic_vert, nxv2i64, sme_elm_idx0_7, sme_elm_idx0_1, tileslice64>;5457 def : SME2_Tile_Movaz_Pat<NAME # _V_H, intrinsic_vert, nxv8bf16, sme_elm_idx0_1, sme_elm_idx0_7, tileslice16>;5458 def : SME2_Tile_Movaz_Pat<NAME # _V_H, intrinsic_vert, nxv8f16, sme_elm_idx0_1, sme_elm_idx0_7, tileslice16>;5459 def : SME2_Tile_Movaz_Pat<NAME # _V_S, intrinsic_vert, nxv4f32, sme_elm_idx0_3, sme_elm_idx0_3, tileslice32>;5460 def : SME2_Tile_Movaz_Pat<NAME # _V_D, intrinsic_vert, nxv2f64, sme_elm_idx0_7, sme_elm_idx0_1, tileslice64>;5461 5462 // H_Q5463 def : SME2_Tile_Movaz_Pat<NAME # _H_Q, intrinsic_horiz_q, nxv16i8, sme_elm_idx0_15, sme_elm_idx0_0, tileslice128>;5464 def : SME2_Tile_Movaz_Pat<NAME # _H_Q, intrinsic_horiz_q, nxv8i16, sme_elm_idx0_15, sme_elm_idx0_0, tileslice128>;5465 def : SME2_Tile_Movaz_Pat<NAME # _H_Q, intrinsic_horiz_q, nxv4i32, sme_elm_idx0_15, sme_elm_idx0_0, tileslice128>;5466 def : SME2_Tile_Movaz_Pat<NAME # _H_Q, intrinsic_horiz_q, nxv2i64, sme_elm_idx0_15, sme_elm_idx0_0, tileslice128>;5467 def : SME2_Tile_Movaz_Pat<NAME # _H_Q, intrinsic_horiz_q, nxv8bf16, sme_elm_idx0_15, sme_elm_idx0_0, tileslice128>;5468 def : SME2_Tile_Movaz_Pat<NAME # _H_Q, intrinsic_horiz_q, nxv8f16, sme_elm_idx0_15, sme_elm_idx0_0, tileslice128>;5469 def : SME2_Tile_Movaz_Pat<NAME # _H_Q, intrinsic_horiz_q, nxv4f32, sme_elm_idx0_15, sme_elm_idx0_0, tileslice128>;5470 def : SME2_Tile_Movaz_Pat<NAME # _H_Q, intrinsic_horiz_q, nxv2f64, sme_elm_idx0_15, sme_elm_idx0_0, tileslice128>;5471 5472 // _V_Q5473 def : SME2_Tile_Movaz_Pat<NAME # _V_Q, intrinsic_vert_q, nxv16i8, sme_elm_idx0_15, sme_elm_idx0_0, tileslice128>;5474 def : SME2_Tile_Movaz_Pat<NAME # _V_Q, intrinsic_vert_q, nxv8i16, sme_elm_idx0_15, sme_elm_idx0_0, tileslice128>;5475 def : SME2_Tile_Movaz_Pat<NAME # _V_Q, intrinsic_vert_q, nxv4i32, sme_elm_idx0_15, sme_elm_idx0_0, tileslice128>;5476 def : SME2_Tile_Movaz_Pat<NAME # _V_Q, intrinsic_vert_q, nxv2i64, sme_elm_idx0_15, sme_elm_idx0_0, tileslice128>;5477 def : SME2_Tile_Movaz_Pat<NAME # _V_Q, intrinsic_vert_q, nxv8bf16, sme_elm_idx0_15, sme_elm_idx0_0, tileslice128>;5478 def : SME2_Tile_Movaz_Pat<NAME # _V_Q, intrinsic_vert_q, nxv8f16, sme_elm_idx0_15, sme_elm_idx0_0, tileslice128>;5479 def : SME2_Tile_Movaz_Pat<NAME # _V_Q, intrinsic_vert_q, nxv4f32, sme_elm_idx0_15, sme_elm_idx0_0, tileslice128>;5480 def : SME2_Tile_Movaz_Pat<NAME # _V_Q, intrinsic_vert_q, nxv2f64, sme_elm_idx0_15, sme_elm_idx0_0, tileslice128>;5481}5482 5483//===----------------------------------------------------------------------===//5484// SME2.1 multiple vectors zero array5485 5486class sme2p1_zero_matrix<bits<6> opc, Operand index_ty, string mnemonic,5487 string vg_acronym="">5488 : I<(outs MatrixOp64:$ZAd),5489 (ins MatrixOp64:$_ZAd, MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm),5490 mnemonic, "\t$ZAd[$Rv, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "]",5491 "", []>, Sched<[]> {5492 bits<0> ZAd;5493 bits<2> Rv;5494 let Inst{31-18} = 0b11000000000011;5495 let Inst{17-15} = opc{5-3};5496 let Inst{14-13} = Rv;5497 let Inst{12-3} = 0b0000000000;5498 let Inst{2-0} = opc{2-0};5499 let Constraints = "$ZAd = $_ZAd";5500}5501 5502multiclass sme2p1_zero_matrix<string mnemonic> {5503 def _VG2_Z : sme2p1_zero_matrix<{0b000,?,?,?}, sme_elm_idx0_7, mnemonic, "vgx2">, SMEPseudo2Instr<NAME # _VG2_Z , 1> {5504 bits<3> imm;5505 let Inst{2-0} = imm;5506 }5507 def _2Z : sme2p1_zero_matrix<{0b001,?,?,?}, uimm3s2range, mnemonic>, SMEPseudo2Instr<NAME # _2Z, 1> {5508 bits<3> imm;5509 let Inst{2-0} = imm;5510 }5511 def _VG2_2Z : sme2p1_zero_matrix<{0b0100,?,?}, uimm2s2range, mnemonic, "vgx2">, SMEPseudo2Instr<NAME # _VG2_2Z, 1> {5512 bits<2> imm;5513 let Inst{1-0} = imm;5514 }5515 def _VG4_2Z : sme2p1_zero_matrix<{0b0110,?,?}, uimm2s2range, mnemonic, "vgx4">, SMEPseudo2Instr<NAME # _VG4_2Z, 1> {5516 bits<2> imm;5517 let Inst{1-0} = imm;5518 }5519 def _VG4_Z : sme2p1_zero_matrix<{0b100,?,?,?}, sme_elm_idx0_7, mnemonic, "vgx4">, SMEPseudo2Instr<NAME # _VG4_Z, 1> {5520 bits<3> imm;5521 let Inst{2-0} = imm;5522 }5523 def _4Z : sme2p1_zero_matrix<{0b1010,?,?}, uimm2s4range, mnemonic>, SMEPseudo2Instr<NAME # _4Z, 1> {5524 bits<2> imm;5525 let Inst{1-0} = imm;5526 }5527 def _VG2_4Z : sme2p1_zero_matrix<{0b11000,?}, uimm1s4range, mnemonic, "vgx2">, SMEPseudo2Instr<NAME # _VG2_4Z, 1> {5528 bits<1> imm;5529 let Inst{0} = imm;5530 }5531 def _VG4_4Z : sme2p1_zero_matrix<{0b11100,?}, uimm1s4range, mnemonic, "vgx4">, SMEPseudo2Instr<NAME # _VG4_4Z, 1> {5532 bits<1> imm;5533 let Inst{0} = imm;5534 }5535 5536 def NAME # _VG2_Z_PSEUDO : sem2p1_zero_matrix_pseudo<NAME # _VG2_Z, sme_elm_idx0_7, SMEMatrixArray>;5537 def NAME # _VG4_Z_PSEUDO : sem2p1_zero_matrix_pseudo<NAME # _VG4_Z, sme_elm_idx0_7, SMEMatrixArray>;5538 def NAME # _2Z_PSEUDO : sem2p1_zero_matrix_pseudo<NAME # _2Z, uimm2s2range, SMEMatrixArray>;5539 def NAME # _VG2_2Z_PSEUDO : sem2p1_zero_matrix_pseudo<NAME # _VG2_2Z, uimm1s2range, SMEMatrixArray>;5540 def NAME # _VG4_2Z_PSEUDO : sem2p1_zero_matrix_pseudo<NAME # _VG4_2Z, uimm1s2range, SMEMatrixArray>;5541 def NAME # _4Z_PSEUDO : sem2p1_zero_matrix_pseudo<NAME # _4Z, uimm1s4range, SMEMatrixArray>;5542 def NAME # _VG2_4Z_PSEUDO : sem2p1_zero_matrix_pseudo<NAME # _VG2_4Z, uimm0s4range, SMEMatrixArray>;5543 def NAME # _VG4_4Z_PSEUDO : sem2p1_zero_matrix_pseudo<NAME # _VG4_4Z, uimm0s4range, SMEMatrixArray>;5544 5545 def : SME2_Zero_Matrix_Pat<NAME # _VG2_Z_PSEUDO, int_aarch64_sme_zero_za64_vg1x2, sme_elm_idx0_7, tileslice16>;5546 def : SME2_Zero_Matrix_Pat<NAME # _VG4_Z_PSEUDO, int_aarch64_sme_zero_za64_vg1x4, sme_elm_idx0_7, tileslice16>;5547 def : SME2_Zero_Matrix_Pat<NAME # _2Z_PSEUDO, int_aarch64_sme_zero_za64_vg2x1, uimm2s2range, tileslicerange2s2>;5548 def : SME2_Zero_Matrix_Pat<NAME # _VG2_2Z_PSEUDO, int_aarch64_sme_zero_za64_vg2x2, uimm1s2range, tileslicerange1s2>;5549 def : SME2_Zero_Matrix_Pat<NAME # _VG4_2Z_PSEUDO, int_aarch64_sme_zero_za64_vg2x4, uimm1s2range, tileslicerange1s2>;5550 def : SME2_Zero_Matrix_Pat<NAME # _4Z_PSEUDO, int_aarch64_sme_zero_za64_vg4x1, uimm1s4range, tileslicerange1s4>;5551 def : SME2_Zero_Matrix_Pat<NAME # _VG2_4Z_PSEUDO, int_aarch64_sme_zero_za64_vg4x2, uimm0s4range, tileslicerange0s4>;5552 def : SME2_Zero_Matrix_Pat<NAME # _VG4_4Z_PSEUDO, int_aarch64_sme_zero_za64_vg4x4, uimm0s4range, tileslicerange0s4>;5553}5554 5555//===----------------------------------------------------------------------===//5556// SME2.1 lookup table expand two non-contiguous registers5557 5558class sme2p1_luti_vector_vg2_index<bits<4> op, bits<2> sz, RegisterOperand vector_ty,5559 AsmVectorIndexOpnd index_ty,5560 string mnemonic>5561 : I<(outs vector_ty:$Zd), (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i),5562 mnemonic, "\t$Zd, $ZTt, $Zn$i",5563 "", []>, Sched<[]> {5564 bits<0> ZTt;5565 bits<5> Zn;5566 bits<4> Zd;5567 let Inst{31-19} = 0b1100000010011;5568 let Inst{18-15} = op;5569 let Inst{14} = 0b1;5570 let Inst{13-12} = sz;5571 let Inst{11-10} = 0b00;5572 let Inst{9-5} = Zn;5573 let Inst{4} = Zd{3};5574 let Inst{3} = 0b0;5575 let Inst{2-0} = Zd{2-0};5576}5577 5578class sme2p1_luti2_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty,5579 AsmVectorIndexOpnd index_ty,5580 string mnemonic>5581 : sme2p1_luti_vector_vg2_index<{1,?,?,?}, sz, vector_ty, index_ty, mnemonic> {5582 bits<3> i;5583 let Inst{17-15} = i;5584}5585 5586multiclass sme2p1_luti2_vector_vg2_index<string mnemonic> {5587 def _B : sme2p1_luti2_vector_vg2_index<0b00, ZZ_b_strided, VectorIndexH,5588 mnemonic>;5589 def _H : sme2p1_luti2_vector_vg2_index<0b01, ZZ_h_strided, VectorIndexH,5590 mnemonic>;5591}5592 5593class sme2p1_luti4_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty,5594 AsmVectorIndexOpnd index_ty,5595 string mnemonic>5596 : sme2p1_luti_vector_vg2_index<{0b01,?,?}, sz, vector_ty, index_ty, mnemonic> {5597 bits<2> i;5598 let Inst{16-15} = i;5599}5600multiclass sme2p1_luti4_vector_vg2_index<string mnemonic> {5601 def _B : sme2p1_luti4_vector_vg2_index<0b00, ZZ_b_strided, VectorIndexS,5602 mnemonic>;5603 def _H : sme2p1_luti4_vector_vg2_index<0b01, ZZ_h_strided, VectorIndexS,5604 mnemonic>;5605}5606 5607// SME2.1 lookup table expand four non-contiguous registers5608class sme2p1_luti_vector_vg4_index<bits<3> op, bits<2> sz, RegisterOperand vector_ty,5609 AsmVectorIndexOpnd index_ty,5610 string mnemonic>5611 : I<(outs vector_ty:$Zd), (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i),5612 mnemonic, "\t$Zd, $ZTt, $Zn$i",5613 "", []>, Sched<[]> {5614 bits<0> ZTt;5615 bits<5> Zn;5616 bits<3> Zd;5617 let Inst{31-19} = 0b1100000010011;5618 let Inst{18-16} = op;5619 let Inst{15-14} = 0b10;5620 let Inst{13-12} = sz;5621 let Inst{11-10} = 0b00;5622 let Inst{9-5} = Zn;5623 let Inst{4} = Zd{2};5624 let Inst{3-2} = 0b00;5625 let Inst{1-0} = Zd{1-0};5626}5627 5628class sme2p1_luti2_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty,5629 AsmVectorIndexOpnd index_ty,5630 string mnemonic>5631 : sme2p1_luti_vector_vg4_index<{1,?,?}, sz, vector_ty, index_ty, mnemonic> {5632 bits<2> i;5633 let Inst{17-16} = i;5634}5635 5636multiclass sme2p1_luti2_vector_vg4_index<string mnemonic> {5637 def _B : sme2p1_luti2_vector_vg4_index<0b00, ZZZZ_b_strided, VectorIndexS,5638 mnemonic>;5639 def _H : sme2p1_luti2_vector_vg4_index<0b01, ZZZZ_h_strided, VectorIndexS,5640 mnemonic>;5641}5642 5643class sme2p1_luti4_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty,5644 AsmVectorIndexOpnd index_ty,5645 string mnemonic>5646 : sme2p1_luti_vector_vg4_index<{0b01,?}, sz, vector_ty, index_ty, mnemonic> {5647 bit i;5648 let Inst{16} = i;5649}5650 5651multiclass sme2p1_luti4_vector_vg4_index<string mnemonic> {5652 def _H: sme2p1_luti4_vector_vg4_index<0b01, ZZZZ_h_strided, VectorIndexD, mnemonic>;5653}5654 5655// SME2 lookup table two source registers expand to four contiguous destination registers5656class sme2_luti4_vector_vg4<bits<2> sz, bits<2> op, string mnemonic>5657 : I<(outs ZZZZ_b_mul_r:$Zd), (ins ZTR:$ZTt, ZZ_mul_r:$Zn),5658 mnemonic, "\t$Zd, $ZTt, $Zn",5659 "", []>, Sched<[]> {5660 bits<0> ZTt;5661 bits<4> Zn;5662 bits<3> Zd;5663 let Inst{31-14} = 0b110000001000101100;5664 let Inst{13-12} = sz;5665 let Inst{11-10} = op;5666 let Inst{9-6} = Zn;5667 let Inst{5} = 0b0;5668 let Inst{4-2} = Zd;5669 let Inst{1-0} = 0b00;5670}5671 5672// SME2 lookup table two source registers expand to four non-contiguous destination registers5673class sme2_luti4_vector_vg4_strided<bits<2> sz, bits<2> op, string mnemonic>5674 : I<(outs ZZZZ_b_strided:$Zd), (ins ZTR:$ZTt, ZZ_mul_r:$Zn),5675 mnemonic, "\t$Zd, $ZTt, $Zn",5676 "", []>, Sched<[]> {5677 bits<0> ZTt;5678 bits<4> Zn;5679 bits<3> Zd;5680 let Inst{31-14} = 0b110000001001101100;5681 let Inst{13-12} = sz;5682 let Inst{11-10} = op;5683 let Inst{9-6} = Zn;5684 let Inst{5} = 0b0;5685 let Inst{4} = Zd{2};5686 let Inst{3-2} = 0b00;5687 let Inst{1-0} = Zd{1-0};5688}5689 5690multiclass sme2_bfscale_single<string mnemonic> {5691 def _2ZZ : sme2_sve_destructive_vector_vg2_single<0b00, 0b0011000, ZZ_h_mul_r, ZPR4b16, mnemonic>;5692 def _4ZZ : sme2_sve_destructive_vector_vg4_single<0b00, 0b0011000, ZZZZ_h_mul_r, ZPR4b16, mnemonic>;5693}5694 5695multiclass sme2_bfscale_multi<string mnemonic> {5696 def _2Z2Z : sme2_sve_destructive_vector_vg2_multi<0b00, 0b0011000, ZZ_h_mul_r, mnemonic>;5697 def _4Z4Z : sme2_sve_destructive_vector_vg4_multi<0b00, 0b0011000, ZZZZ_h_mul_r, mnemonic>;5698}5699 5700class sme2_bf16_fp32_quarter_tile_outer_product<bit M, bit N, bit S, string mnemonic, RegisterOperand zn_ty, RegisterOperand zm_ty>5701 : I<(outs TileOp32:$ZAda),5702 (ins TileOp32:$_ZAda, zn_ty:$Zn, zm_ty:$Zm),5703 mnemonic, "\t$ZAda, $Zn, $Zm",5704 "", []>, Sched<[]> {5705 bits<2> ZAda;5706 bits<3> Zn;5707 bits<3> Zm;5708 5709 let Inst{31-21} = 0b10000001000;5710 let Inst{20} = M;5711 let Inst{19-17} = Zm;5712 let Inst{16-10} = 0b0000000;5713 let Inst{9} = N;5714 let Inst{8-6} = Zn;5715 let Inst{5} = 0;5716 let Inst{4} = S;5717 let Inst{3-2} = 0b00;5718 let Inst{1-0} = ZAda;5719 5720 let Constraints = "$ZAda = $_ZAda";5721 let Uses = [FPCR];5722}5723 5724multiclass sme2_bfmop4as_widening<bit S, string mnemonic, string op> {5725 // Single vectors5726 def _MZZ_S : sme2_bf16_fp32_quarter_tile_outer_product<0, 0, S, mnemonic, ZPR16Mul2_Lo, ZPR16Mul2_Hi>, SMEPseudo2Instr<NAME # _MZZ_S, 1>;5727 5728 def NAME # _MZZ_S_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZPR16Mul2_Lo, ZPR16Mul2_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _MZZ_S, 0>;5729 5730 def : SME2_ZA_Tile_Vec_Single_Single_Pat<NAME # _MZZ_S, !cast<SDPatternOperator>(op # "_1x1"), timm32_0_3, nxv8bf16>;5731 5732 // Multiple and single vectors5733 def _M2ZZ_S : sme2_bf16_fp32_quarter_tile_outer_product<0, 1, S, mnemonic, ZZ_h_mul_r_Lo, ZPR16Mul2_Hi>, SMEPseudo2Instr<NAME # _M2ZZ_S, 1>;5734 5735 def NAME # _M2ZZ_S_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZZ_h_mul_r_Lo, ZPR16Mul2_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _M2ZZ_S, 0>;5736 5737 def : SME2_ZA_Tile_Vec_Multi_Single_Pat<NAME # _M2ZZ_S, !cast<SDPatternOperator>(op # "_2x1"), timm32_0_3, nxv8bf16>;5738 5739 // Single and multiple vectors5740 def _MZ2Z_S : sme2_bf16_fp32_quarter_tile_outer_product<1, 0, S, mnemonic, ZPR16Mul2_Lo, ZZ_h_mul_r_Hi>, SMEPseudo2Instr<NAME # _MZ2Z_S, 1>;5741 5742 def NAME # _MZ2Z_S_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZPR16Mul2_Lo, ZZ_h_mul_r_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _MZ2Z_S, 0>;5743 5744 def : SME2_ZA_Tile_Vec_Multi_Pat<NAME # _MZ2Z_S, !cast<SDPatternOperator>(op # "_1x2"), timm32_0_3, nxv8bf16>;5745 5746 // Multiple vectors5747 def _M2Z2Z_S : sme2_bf16_fp32_quarter_tile_outer_product<1, 1, S, mnemonic, ZZ_h_mul_r_Lo, ZZ_h_mul_r_Hi>, SMEPseudo2Instr<NAME # _M2Z2Z_S, 1>;5748 5749 def NAME # _M2Z2Z_S_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZZ_h_mul_r_Lo, ZZ_h_mul_r_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _M2Z2Z_S, 0>;5750 5751 def : SME2_ZA_Tile_Vec_Multi_Multi_Pat<NAME # _M2Z2Z_S, !cast<SDPatternOperator>(op # "_2x2"), timm32_0_3, nxv8bf16>;5752}5753 5754class sme2_multi2_fmul_sm<bits<2> size, string mnemonic, RegisterOperand vector_ty, RegisterOperand zpr_ty>5755 : I<(outs vector_ty:$Zd), (ins vector_ty:$Zn, zpr_ty:$Zm),5756 mnemonic, "\t$Zd, $Zn, $Zm",5757 "", []>, Sched<[]> {5758 bits<4> Zd;5759 bits<4> Zn;5760 bits<4> Zm;5761 5762 let Inst{31-24} = 0b11000001;5763 let Inst{23-22} = size;5764 let Inst{21} = 0b1;5765 let Inst{20-17} = Zm;5766 let Inst{16-10} = 0b0111010;5767 let Inst{9-6} = Zn;5768 let Inst{5} = 0b0;5769 let Inst{4-1} = Zd;5770 let Inst{0} = 0b0;5771}5772 5773multiclass sme2_multi2_fmul_sm<string mnemonic> {5774 def _H : sme2_multi2_fmul_sm<0b01, mnemonic, ZZ_h_mul_r, ZPR4b16>;5775 def _S : sme2_multi2_fmul_sm<0b10, mnemonic, ZZ_s_mul_r, ZPR4b32>;5776 def _D : sme2_multi2_fmul_sm<0b11, mnemonic, ZZ_d_mul_r, ZPR4b64>;5777}5778 5779class sme2_multi4_fmul_sm<bits<2> size, string mnemonic, RegisterOperand vector_ty, RegisterOperand zpr_ty>5780 : I<(outs vector_ty:$Zd), (ins vector_ty:$Zn, zpr_ty:$Zm),5781 mnemonic, "\t$Zd, $Zn, $Zm",5782 "", []>, Sched<[]> {5783 bits<3> Zd;5784 bits<3> Zn;5785 bits<4> Zm;5786 5787 let Inst{31-24} = 0b11000001;5788 let Inst{23-22} = size;5789 let Inst{21} = 0b1;5790 let Inst{20-17} = Zm;5791 let Inst{16-10} = 0b1111010;5792 let Inst{9-7} = Zn;5793 let Inst{6-5} = 0b00;5794 let Inst{4-2} = Zd;5795 let Inst{1-0} = 0b00;5796}5797 5798multiclass sme2_multi4_fmul_sm<string mnemonic> {5799 def _H : sme2_multi4_fmul_sm<0b01, mnemonic, ZZZZ_h_mul_r, ZPR4b16>;5800 def _S : sme2_multi4_fmul_sm<0b10, mnemonic, ZZZZ_s_mul_r, ZPR4b32>;5801 def _D : sme2_multi4_fmul_sm<0b11, mnemonic, ZZZZ_d_mul_r, ZPR4b64>;5802}5803 5804multiclass sme2_bfmul_single<string mnemonic> {5805 def _2ZZ : sme2_multi2_fmul_sm<0b00, mnemonic, ZZ_h_mul_r, ZPR4b16>;5806 def _4ZZ : sme2_multi4_fmul_sm<0b00, mnemonic, ZZZZ_h_mul_r, ZPR4b16>;5807}5808 5809class sme2_multi2_fmul_mm<bits<2> size, string mnemonic, RegisterOperand vector_ty>5810 : I<(outs vector_ty:$Zd), (ins vector_ty:$Zn, vector_ty:$Zm),5811 mnemonic, "\t$Zd, $Zn, $Zm",5812 "", []>, Sched<[]> {5813 bits<4> Zd;5814 bits<4> Zn;5815 bits<4> Zm;5816 5817 let Inst{31-24} = 0b11000001;5818 let Inst{23-22} = size;5819 let Inst{21} = 0b1;5820 let Inst{20-17} = Zm;5821 let Inst{16-10} = 0b0111001;5822 let Inst{9-6} = Zn;5823 let Inst{5} = 0b0;5824 let Inst{4-1} = Zd;5825 let Inst{0} = 0b0;5826}5827 5828multiclass sme2_multi2_fmul_mm<string mnemonic> {5829 def _H : sme2_multi2_fmul_mm<0b01, mnemonic, ZZ_h_mul_r>;5830 def _S : sme2_multi2_fmul_mm<0b10, mnemonic, ZZ_s_mul_r>;5831 def _D : sme2_multi2_fmul_mm<0b11, mnemonic, ZZ_d_mul_r>;5832}5833 5834class sme2_multi4_fmul_mm<bits<2> size, string mnemonic, RegisterOperand vector_ty>5835 : I<(outs vector_ty:$Zd), (ins vector_ty:$Zn, vector_ty:$Zm),5836 mnemonic, "\t$Zd, $Zn, $Zm",5837 "", []>, Sched<[]> {5838 bits<3> Zd;5839 bits<3> Zn;5840 bits<3> Zm;5841 5842 let Inst{31-24} = 0b11000001;5843 let Inst{23-22} = size;5844 let Inst{21} = 0b1;5845 let Inst{20-18} = Zm;5846 let Inst{17-10} = 0b01111001;5847 let Inst{9-7} = Zn;5848 let Inst{6-5} = 0b00;5849 let Inst{4-2} = Zd;5850 let Inst{1-0} = 0b00;5851}5852 5853multiclass sme2_multi4_fmul_mm<string mnemonic> {5854 def _H : sme2_multi4_fmul_mm<0b01, mnemonic, ZZZZ_h_mul_r>;5855 def _S : sme2_multi4_fmul_mm<0b10, mnemonic, ZZZZ_s_mul_r>;5856 def _D : sme2_multi4_fmul_mm<0b11, mnemonic, ZZZZ_d_mul_r>;5857}5858 5859multiclass sme2_bfmul_multi<string mnemonic> {5860 def _2Z2Z : sme2_multi2_fmul_mm<0b00, mnemonic, ZZ_h_mul_r>;5861 def _4Z4Z : sme2_multi4_fmul_mm<0b00, mnemonic, ZZZZ_h_mul_r>;5862}5863 5864class sme2_fp16_quarter_tile_outer_product<bit M, bit N, bit S, string mnemonic, RegisterOperand zn_ty, RegisterOperand zm_ty>5865 : I<(outs TileOp16:$ZAda),5866 (ins TileOp16:$_ZAda, zn_ty:$Zn, zm_ty:$Zm),5867 mnemonic, "\t$ZAda, $Zn, $Zm",5868 "", []>, Sched<[]> {5869 bit ZAda;5870 bits<3> Zn;5871 bits<3> Zm;5872 5873 let Inst{31-21} = 0b10000001000;5874 let Inst{20} = M;5875 let Inst{19-17} = Zm;5876 let Inst{16-10} = 0b0000000;5877 let Inst{9} = N;5878 let Inst{8-6} = Zn;5879 let Inst{5} = 0;5880 let Inst{4} = S;5881 let Inst{3-1} = 0b100;5882 let Inst{0} = ZAda;5883 5884 let Constraints = "$ZAda = $_ZAda";5885 let Uses = [FPCR];5886}5887 5888multiclass sme2_fmop4as_fp16_non_widening<bit S, string mnemonic, string op> {5889 // Single vectors5890 def _MZZ_H : sme2_fp16_quarter_tile_outer_product<0, 0, S, mnemonic, ZPR16Mul2_Lo, ZPR16Mul2_Hi>, SMEPseudo2Instr<NAME # _MZZ_H, 1>;5891 5892 def NAME # _MZZ_H_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZPR16Mul2_Lo, ZPR16Mul2_Hi, SMEMatrixTileH>, SMEPseudo2Instr<NAME # _MZZ_H, 0>;5893 5894 def : SME2_ZA_Tile_Vec_Single_Single_Pat<NAME # _MZZ_H, !cast<SDPatternOperator>(op # "_1x1"), timm32_0_1, nxv8f16>;5895 5896 // Multiple and single vectors5897 def _M2ZZ_H : sme2_fp16_quarter_tile_outer_product<0, 1, S, mnemonic, ZZ_h_mul_r_Lo, ZPR16Mul2_Hi>, SMEPseudo2Instr<NAME # _M2ZZ_H, 1>;5898 5899 def NAME # _M2ZZ_H_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZZ_h_mul_r_Lo, ZPR16Mul2_Hi, SMEMatrixTileH>, SMEPseudo2Instr<NAME # _M2ZZ_H, 0>;5900 5901 def : SME2_ZA_Tile_Vec_Multi_Single_Pat<NAME # _M2ZZ_H, !cast<SDPatternOperator>(op # "_2x1"), timm32_0_1, nxv8f16>;5902 5903 // Single and multiple vectors5904 def _MZ2Z_H : sme2_fp16_quarter_tile_outer_product<1, 0, S, mnemonic, ZPR16Mul2_Lo, ZZ_h_mul_r_Hi>, SMEPseudo2Instr<NAME # _MZ2Z_H, 1>;5905 5906 def NAME # _MZ2Z_H_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZPR16Mul2_Lo, ZZ_h_mul_r_Hi, SMEMatrixTileH>, SMEPseudo2Instr<NAME # _MZ2Z_H, 0>;5907 5908 def : SME2_ZA_Tile_Vec_Multi_Pat<NAME # _MZ2Z_H, !cast<SDPatternOperator>(op # "_1x2"), timm32_0_1, nxv8f16>;5909 5910 // Multiple vectors5911 def _M2Z2Z_H : sme2_fp16_quarter_tile_outer_product<1, 1, S, mnemonic, ZZ_h_mul_r_Lo, ZZ_h_mul_r_Hi>, SMEPseudo2Instr<NAME # _M2Z2Z_H, 1>;5912 5913 def NAME # _M2Z2Z_H_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZZ_h_mul_r_Lo, ZZ_h_mul_r_Hi, SMEMatrixTileH>, SMEPseudo2Instr<NAME # _M2Z2Z_H, 0>;5914 5915 def : SME2_ZA_Tile_Vec_Multi_Multi_Pat<NAME # _M2Z2Z_H, !cast<SDPatternOperator>(op # "_2x2"), timm32_0_1, nxv8f16>;5916}5917 5918class sme2_fp8_fp32_quarter_tile_outer_product<bit M, bit N, string mnemonic, RegisterOperand zn_ty, RegisterOperand zm_ty>5919 : I<(outs TileOp32:$ZAda),5920 (ins TileOp32:$_ZAda, zn_ty:$Zn, zm_ty:$Zm),5921 mnemonic, "\t$ZAda, $Zn, $Zm",5922 "", []>, Sched<[]> {5923 bits<2> ZAda;5924 bits<3> Zn;5925 bits<3> Zm;5926 5927 let Inst{31-21} = 0b10000000001;5928 let Inst{20} = M;5929 let Inst{19-17} = Zm;5930 let Inst{16-10} = 0b0000000;5931 let Inst{9} = N;5932 let Inst{8-6} = Zn;5933 let Inst{5-2} = 0b0000;5934 let Inst{1-0} = ZAda;5935 5936 let Constraints = "$ZAda = $_ZAda";5937 let Uses = [FPMR, FPCR];5938}5939 5940multiclass sme2_fmop4a_fp8_fp32_4way<string mnemonic, string op> {5941 // Single vectors5942 def _MZZ_BtoS : sme2_fp8_fp32_quarter_tile_outer_product<0, 0, mnemonic, ZPR8Mul2_Lo, ZPR8Mul2_Hi>, SMEPseudo2Instr<NAME # _MZZ_BtoS, 1>;5943 5944 def NAME # _MZZ_BtoS_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZPR8Mul2_Lo, ZPR8Mul2_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _MZZ_BtoS, 0>;5945 5946 def : SME2_ZA_Tile_Vec_Single_Single_Pat<NAME # _MZZ_BtoS, !cast<SDPatternOperator>(op # "_1x1"), timm32_0_3, nxv16i8>;5947 5948 // Multiple and single vectors5949 def _M2ZZ_BtoS : sme2_fp8_fp32_quarter_tile_outer_product<0, 1, mnemonic, ZZ_b_mul_r_Lo, ZPR8Mul2_Hi>, SMEPseudo2Instr<NAME # _M2ZZ_BtoS, 1>;5950 5951 def NAME # _M2ZZ_BtoS_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZZ_b_mul_r_Lo, ZPR8Mul2_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _M2ZZ_BtoS, 0>;5952 5953 def : SME2_ZA_Tile_Vec_Multi_Single_Pat<NAME # _M2ZZ_BtoS, !cast<SDPatternOperator>(op # "_2x1"), timm32_0_3, nxv16i8>;5954 5955 // Single and multiple vectors5956 def _MZ2Z_BtoS : sme2_fp8_fp32_quarter_tile_outer_product<1, 0, mnemonic, ZPR8Mul2_Lo, ZZ_b_mul_r_Hi>, SMEPseudo2Instr<NAME # _MZ2Z_BtoS, 1>;5957 5958 def NAME # _MZ2Z_BtoS_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZPR8Mul2_Lo, ZZ_b_mul_r_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _MZ2Z_BtoS, 0>;5959 5960 def : SME2_ZA_Tile_Vec_Multi_Pat<NAME # _MZ2Z_BtoS, !cast<SDPatternOperator>(op # "_1x2"), timm32_0_3, nxv16i8>;5961 5962 // Multiple vectors5963 def _M2Z2Z_BtoS : sme2_fp8_fp32_quarter_tile_outer_product<1, 1, mnemonic, ZZ_b_mul_r_Lo, ZZ_b_mul_r_Hi>, SMEPseudo2Instr<NAME # _M2Z2Z_BtoS, 1>;5964 5965 def NAME # _M2Z2Z_BtoS_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZZ_b_mul_r_Lo, ZZ_b_mul_r_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _M2Z2Z_BtoS, 0>;5966 5967 def : SME2_ZA_Tile_Vec_Multi_Multi_Pat<NAME # _M2Z2Z_BtoS, !cast<SDPatternOperator>(op # "_2x2"), timm32_0_3, nxv16i8>;5968}5969 5970class sme2_bf16_fp16_quarter_tile_outer_product<bit M, bit N, bit S, string mnemonic, RegisterOperand zn_ty, RegisterOperand zm_ty>5971 : I<(outs TileOp16:$ZAda),5972 (ins TileOp16:$_ZAda, zn_ty:$Zn, zm_ty:$Zm),5973 mnemonic, "\t$ZAda, $Zn, $Zm",5974 "", []>, Sched<[]> {5975 bit ZAda;5976 bits<3> Zn;5977 bits<3> Zm;5978 5979 let Inst{31-21} = 0b10000001001;5980 let Inst{20} = M;5981 let Inst{19-17} = Zm;5982 let Inst{16-10} = 0b0000000;5983 let Inst{9} = N;5984 let Inst{8-6} = Zn;5985 let Inst{5} = 0;5986 let Inst{4} = S;5987 let Inst{3-1} = 0b100;5988 let Inst{0} = ZAda;5989 5990 let Constraints = "$ZAda = $_ZAda";5991 let Uses = [FPCR];5992}5993 5994multiclass sme2_bfmop4as_non_widening<bit S, string mnemonic, string op> {5995 // Single vectors5996 def _MZZ_H : sme2_bf16_fp16_quarter_tile_outer_product<0, 0, S, mnemonic, ZPR16Mul2_Lo, ZPR16Mul2_Hi>, SMEPseudo2Instr<NAME # _MZZ_H, 1>;5997 5998 def NAME # _MZZ_H_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZPR16Mul2_Lo, ZPR16Mul2_Hi, SMEMatrixTileH>, SMEPseudo2Instr<NAME # _MZZ_H, 0>;5999 6000 def : SME2_ZA_Tile_Vec_Single_Single_Pat<NAME # _MZZ_H, !cast<SDPatternOperator>(op # "_1x1"), timm32_0_1, nxv8bf16>;6001 6002 // Multiple and single vectors6003 def _M2ZZ_H : sme2_bf16_fp16_quarter_tile_outer_product<0, 1, S, mnemonic, ZZ_h_mul_r_Lo, ZPR16Mul2_Hi>, SMEPseudo2Instr<NAME # _M2ZZ_H, 1>;6004 6005 def NAME # _M2ZZ_H_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZZ_h_mul_r_Lo, ZPR16Mul2_Hi, SMEMatrixTileH>, SMEPseudo2Instr<NAME # _M2ZZ_H, 0>;6006 6007 def : SME2_ZA_Tile_Vec_Multi_Single_Pat<NAME # _M2ZZ_H, !cast<SDPatternOperator>(op # "_2x1"), timm32_0_1, nxv8bf16>;6008 6009 // Single and multiple vectors6010 def _MZ2Z_H : sme2_bf16_fp16_quarter_tile_outer_product<1, 0, S, mnemonic, ZPR16Mul2_Lo, ZZ_h_mul_r_Hi>, SMEPseudo2Instr<NAME # _MZ2Z_H, 1>;6011 6012 def NAME # _MZ2Z_H_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZPR16Mul2_Lo, ZZ_h_mul_r_Hi, SMEMatrixTileH>, SMEPseudo2Instr<NAME # _MZ2Z_H, 0>;6013 6014 def : SME2_ZA_Tile_Vec_Multi_Pat<NAME # _MZ2Z_H, !cast<SDPatternOperator>(op # "_1x2"), timm32_0_1, nxv8bf16>;6015 6016 // Multiple vectors6017 def _M2Z2Z_H : sme2_bf16_fp16_quarter_tile_outer_product<1, 1, S, mnemonic, ZZ_h_mul_r_Lo, ZZ_h_mul_r_Hi>, SMEPseudo2Instr<NAME # _M2Z2Z_H, 1>;6018 6019 def NAME # _M2Z2Z_H_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZZ_h_mul_r_Lo, ZZ_h_mul_r_Hi, SMEMatrixTileH>, SMEPseudo2Instr<NAME # _M2Z2Z_H, 0>;6020 6021 def : SME2_ZA_Tile_Vec_Multi_Multi_Pat<NAME # _M2Z2Z_H, !cast<SDPatternOperator>(op # "_2x2"), timm32_0_1, nxv8bf16>;6022}6023 6024class sme2_fp32_quarter_tile_outer_product<bit M, bit N, bit S, string mnemonic, RegisterOperand zn_ty, RegisterOperand zm_ty>6025 : I<(outs TileOp32:$ZAda),6026 (ins TileOp32:$_ZAda, zn_ty:$Zn, zm_ty:$Zm),6027 mnemonic, "\t$ZAda, $Zn, $Zm",6028 "", []>, Sched<[]> {6029 bits<2> ZAda;6030 bits<3> Zn;6031 bits<3> Zm;6032 6033 let Inst{31-21} = 0b10000000000;6034 let Inst{20} = M;6035 let Inst{19-17} = Zm;6036 let Inst{16-10} = 0b0000000;6037 let Inst{9} = N;6038 let Inst{8-6} = Zn;6039 let Inst{5} = 0;6040 let Inst{4} = S;6041 let Inst{3-2} = 0b00;6042 let Inst{1-0} = ZAda;6043 6044 let Constraints = "$ZAda = $_ZAda";6045 let Uses = [FPCR];6046}6047 6048multiclass sme2_fmop4as_fp32_non_widening<bit S, string mnemonic, string op> {6049 // Single vectors6050 def _MZZ_S : sme2_fp32_quarter_tile_outer_product<0, 0, S, mnemonic, ZPR32Mul2_Lo, ZPR32Mul2_Hi>, SMEPseudo2Instr<NAME # _MZZ_S, 1>;6051 6052 def NAME # _MZZ_S_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZPR16Mul2_Lo, ZPR16Mul2_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _MZZ_S, 0>;6053 6054 def : SME2_ZA_Tile_Vec_Single_Single_Pat<NAME # _MZZ_S, !cast<SDPatternOperator>(op # "_1x1"), timm32_0_3, nxv4f32>;6055 6056 // Multiple and single vectors6057 def _M2ZZ_S : sme2_fp32_quarter_tile_outer_product<0, 1, S, mnemonic, ZZ_s_mul_r_Lo, ZPR32Mul2_Hi>, SMEPseudo2Instr<NAME # _M2ZZ_S, 1>;6058 6059 def NAME # _M2ZZ_S_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZZ_s_mul_r_Lo, ZPR32Mul2_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _M2ZZ_S, 0>;6060 6061 def : SME2_ZA_Tile_Vec_Multi_Single_Pat<NAME # _M2ZZ_S, !cast<SDPatternOperator>(op # "_2x1"), timm32_0_3, nxv4f32>;6062 6063 // Single and multiple vectors6064 def _MZ2Z_S : sme2_fp32_quarter_tile_outer_product<1, 0, S, mnemonic, ZPR32Mul2_Lo, ZZ_s_mul_r_Hi>, SMEPseudo2Instr<NAME # _MZ2Z_S, 1>;6065 6066 def NAME # _MZ2Z_S_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZPR32Mul2_Lo, ZZ_s_mul_r_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _MZ2Z_S, 0>;6067 6068 def : SME2_ZA_Tile_Vec_Multi_Pat<NAME # _MZ2Z_S, !cast<SDPatternOperator>(op # "_1x2"), timm32_0_3, nxv4f32>;6069 6070 // Multiple vectors6071 def _M2Z2Z_S : sme2_fp32_quarter_tile_outer_product<1, 1, S, mnemonic, ZZ_s_mul_r_Lo, ZZ_s_mul_r_Hi>, SMEPseudo2Instr<NAME # _M2Z2Z_S, 1>;6072 6073 def NAME # _M2Z2Z_S_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZZ_s_mul_r_Lo, ZZ_s_mul_r_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _M2Z2Z_S, 0>;6074 6075 def : SME2_ZA_Tile_Vec_Multi_Multi_Pat<NAME # _M2Z2Z_S, !cast<SDPatternOperator>(op # "_2x2"), timm32_0_3, nxv4f32>;6076}6077 6078class sme2_fp64_quarter_tile_outer_product<bit M, bit N, bit S, string mnemonic, RegisterOperand zn_ty, RegisterOperand zm_ty>6079 : I<(outs TileOp64:$ZAda),6080 (ins TileOp64:$_ZAda, zn_ty:$Zn, zm_ty:$Zm),6081 mnemonic, "\t$ZAda, $Zn, $Zm",6082 "", []>, Sched<[]> {6083 bits<3> ZAda;6084 bits<3> Zn;6085 bits<3> Zm;6086 6087 let Inst{31-21} = 0b10000000110;6088 let Inst{20} = M;6089 let Inst{19-17} = Zm;6090 let Inst{16-10} = 0b0000000;6091 let Inst{9} = N;6092 let Inst{8-6} = Zn;6093 let Inst{5} = 0;6094 let Inst{4} = S;6095 let Inst{3} = 0b1;6096 let Inst{2-0} = ZAda;6097 6098 let Constraints = "$ZAda = $_ZAda";6099 let Uses = [FPCR];6100}6101 6102multiclass sme2_fmop4as_fp64_non_widening<bit S, string mnemonic, string op> {6103 // Single vectors6104 def _MZZ_D : sme2_fp64_quarter_tile_outer_product<0, 0, S, mnemonic, ZPR64Mul2_Lo, ZPR64Mul2_Hi>, SMEPseudo2Instr<NAME # _MZZ_D, 1>;6105 6106 def NAME # _MZZ_D_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZPR64Mul2_Lo, ZPR64Mul2_Hi, SMEMatrixTileD>, SMEPseudo2Instr<NAME # _MZZ_D, 0>;6107 6108 def : SME2_ZA_Tile_Vec_Single_Single_Pat<NAME # _MZZ_D, !cast<SDPatternOperator>(op # "_1x1"), timm32_0_7, nxv2f64>;6109 6110 // Multiple and single vectors6111 def _M2ZZ_D : sme2_fp64_quarter_tile_outer_product<0, 1, S, mnemonic, ZZ_d_mul_r_Lo, ZPR64Mul2_Hi>, SMEPseudo2Instr<NAME # _M2ZZ_D, 1>;6112 6113 def NAME # _M2ZZ_D_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZZ_d_mul_r_Lo, ZPR64Mul2_Hi, SMEMatrixTileD>, SMEPseudo2Instr<NAME # _M2ZZ_D, 0>;6114 6115 def : SME2_ZA_Tile_Vec_Multi_Single_Pat<NAME # _M2ZZ_D, !cast<SDPatternOperator>(op # "_2x1"), timm32_0_7, nxv2f64>;6116 6117 // Single and multiple vectors6118 def _MZ2Z_D : sme2_fp64_quarter_tile_outer_product<1, 0, S, mnemonic, ZPR64Mul2_Lo, ZZ_d_mul_r_Hi>, SMEPseudo2Instr<NAME # _MZ2Z_D, 1>;6119 6120 def NAME # _MZ2Z_D_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZPR64Mul2_Lo, ZZ_d_mul_r_Hi, SMEMatrixTileD>, SMEPseudo2Instr<NAME # _MZ2Z_D, 0>;6121 6122 def : SME2_ZA_Tile_Vec_Multi_Pat<NAME # _MZ2Z_D, !cast<SDPatternOperator>(op # "_1x2"), timm32_0_7, nxv2f64>;6123 6124 // Multiple vectors6125 def _M2Z2Z_D : sme2_fp64_quarter_tile_outer_product<1, 1, S, mnemonic, ZZ_d_mul_r_Lo, ZZ_d_mul_r_Hi>, SMEPseudo2Instr<NAME # _M2Z2Z_D, 1>;6126 6127 def NAME # _M2Z2Z_D_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZZ_d_mul_r_Lo, ZZ_d_mul_r_Hi, SMEMatrixTileD>, SMEPseudo2Instr<NAME # _M2Z2Z_D, 0>;6128 6129 def : SME2_ZA_Tile_Vec_Multi_Multi_Pat<NAME # _M2Z2Z_D, !cast<SDPatternOperator>(op # "_2x2"), timm32_0_7, nxv2f64>;6130}6131 6132class sme2_fp16_fp32_quarter_tile_outer_product<bit M, bit N, bit S, string mnemonic, RegisterOperand zn_ty, RegisterOperand zm_ty>6133 : I<(outs TileOp32:$ZAda),6134 (ins TileOp32:$_ZAda, zn_ty:$Zn, zm_ty:$Zm),6135 mnemonic, "\t$ZAda, $Zn, $Zm",6136 "", []>, Sched<[]> {6137 bits<2> ZAda;6138 bits<3> Zn;6139 bits<3> Zm;6140 6141 let Inst{31-21} = 0b10000001001;6142 let Inst{20} = M;6143 let Inst{19-17} = Zm;6144 let Inst{16-10} = 0b0000000;6145 let Inst{9} = N;6146 let Inst{8-6} = Zn;6147 let Inst{5} = 0;6148 let Inst{4} = S;6149 let Inst{3-2} = 0b00;6150 let Inst{1-0} = ZAda;6151 6152 let Constraints = "$ZAda = $_ZAda";6153 let Uses = [FPCR];6154}6155 6156multiclass sme2_fmop4as_fp16_fp32_widening<bit S, string mnemonic, string op> {6157 // Single vectors6158 def _MZZ_HtoS : sme2_fp16_fp32_quarter_tile_outer_product<0, 0, S, mnemonic, ZPR16Mul2_Lo, ZPR16Mul2_Hi>, SMEPseudo2Instr<NAME # _MZZ_HtoS, 1>;6159 6160 def NAME # _MZZ_HtoS_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZPR16Mul2_Lo, ZPR16Mul2_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _MZZ_HtoS, 0>;6161 6162 def : SME2_ZA_Tile_Vec_Single_Single_Pat<NAME # _MZZ_HtoS, !cast<SDPatternOperator>(op # "_1x1"), timm32_0_3, nxv8f16>;6163 6164 // Multiple and single vectors6165 def _M2ZZ_HtoS : sme2_fp16_fp32_quarter_tile_outer_product<0, 1, S, mnemonic, ZZ_h_mul_r_Lo, ZPR16Mul2_Hi>, SMEPseudo2Instr<NAME # _M2ZZ_HtoS, 1>;6166 6167 def NAME # _M2ZZ_HtoS_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZZ_h_mul_r_Lo, ZPR16Mul2_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _M2ZZ_HtoS, 0>;6168 6169 def : SME2_ZA_Tile_Vec_Multi_Single_Pat<NAME # _M2ZZ_HtoS, !cast<SDPatternOperator>(op # "_2x1"), timm32_0_3, nxv8f16>;6170 6171 // Single and multiple vectors6172 def _MZ2Z_HtoS : sme2_fp16_fp32_quarter_tile_outer_product<1, 0, S, mnemonic, ZPR16Mul2_Lo, ZZ_h_mul_r_Hi>, SMEPseudo2Instr<NAME # _MZ2Z_HtoS, 1>;6173 6174 def NAME # _MZ2Z_HtoS_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZPR16Mul2_Lo, ZZ_h_mul_r_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _MZ2Z_HtoS, 0>;6175 6176 def : SME2_ZA_Tile_Vec_Multi_Pat<NAME # _MZ2Z_HtoS, !cast<SDPatternOperator>(op # "_1x2"), timm32_0_3, nxv8f16>;6177 6178 // Multiple vectors6179 def _M2Z2Z_HtoS : sme2_fp16_fp32_quarter_tile_outer_product<1, 1, S, mnemonic, ZZ_h_mul_r_Lo, ZZ_h_mul_r_Hi>, SMEPseudo2Instr<NAME # _M2Z2Z_HtoS, 1>;6180 6181 def NAME # _M2Z2Z_HtoS_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZZ_h_mul_r_Lo, ZZ_h_mul_r_Hi, SMEMatrixTileS>, SMEPseudo2Instr<NAME # _M2Z2Z_HtoS, 0>;6182 6183 def : SME2_ZA_Tile_Vec_Multi_Multi_Pat<NAME # _M2Z2Z_HtoS, !cast<SDPatternOperator>(op # "_2x2"), timm32_0_3, nxv8f16>;6184}6185 6186class sme2_fp8_fp16_quarter_tile_outer_product<bit M, bit N, string mnemonic, RegisterOperand zn_ty, RegisterOperand zm_ty>6187 : I<(outs TileOp16:$ZAda),6188 (ins TileOp16:$_ZAda, zn_ty:$Zn, zm_ty:$Zm),6189 mnemonic, "\t$ZAda, $Zn, $Zm",6190 "", []>, Sched<[]> {6191 bit ZAda;6192 bits<3> Zn;6193 bits<3> Zm;6194 6195 let Inst{31-21} = 0b10000000001;6196 let Inst{20} = M;6197 let Inst{19-17} = Zm;6198 let Inst{16-10} = 0b0000000;6199 let Inst{9} = N;6200 let Inst{8-6} = Zn;6201 let Inst{5-1} = 0b00100;6202 let Inst{0} = ZAda;6203 6204 let Constraints = "$ZAda = $_ZAda";6205 let Uses = [FPMR, FPCR];6206}6207 6208multiclass sme2_fmop4a_fp8_fp16_2way<string mnemonic, string op> {6209 6210 // Single vectors6211 def _MZZ_BtoH : sme2_fp8_fp16_quarter_tile_outer_product<0b0, 0b0, mnemonic, ZPR8Mul2_Lo, ZPR8Mul2_Hi>, SMEPseudo2Instr<NAME # _MZZ_BtoH, 1>;6212 6213 def NAME # _MZZ_BtoH_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZPR8Mul2_Lo, ZPR8Mul2_Hi, SMEMatrixTileH>, SMEPseudo2Instr<NAME # _MZZ_BtoH, 0>;6214 6215 def : SME2_ZA_Tile_Vec_Single_Single_Pat<NAME # _MZZ_BtoH, !cast<SDPatternOperator>(op # "_1x1"), timm32_0_1, nxv16i8>;6216 6217 // Multiple and single vectors6218 def _M2ZZ_BtoH : sme2_fp8_fp16_quarter_tile_outer_product<0b0, 0b1, mnemonic, ZZ_b_mul_r_Lo, ZPR8Mul2_Hi>, SMEPseudo2Instr<NAME # _M2ZZ_BtoH, 1>;6219 6220 def NAME # _M2ZZ_BtoH_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZZ_b_mul_r_Lo, ZPR8Mul2_Hi, SMEMatrixTileH>, SMEPseudo2Instr<NAME # _M2ZZ_BtoH, 0>;6221 6222 def : SME2_ZA_Tile_Vec_Multi_Single_Pat<NAME # _M2ZZ_BtoH, !cast<SDPatternOperator>(op # "_2x1"), timm32_0_1, nxv16i8>;6223 6224 // Single and multiple vectors6225 def _MZ2Z_BtoH : sme2_fp8_fp16_quarter_tile_outer_product<0b1, 0b0, mnemonic, ZPR8Mul2_Lo, ZZ_b_mul_r_Hi>, SMEPseudo2Instr<NAME # _MZ2Z_BtoH, 1>;6226 6227 def NAME # _MZ2Z_BtoH_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZPR8Mul2_Lo, ZZ_b_mul_r_Hi, SMEMatrixTileH>, SMEPseudo2Instr<NAME # _MZ2Z_BtoH, 0>;6228 6229 def : SME2_ZA_Tile_Vec_Multi_Pat<NAME # _MZ2Z_BtoH, !cast<SDPatternOperator>(op # "_1x2"), timm32_0_1, nxv16i8>;6230 6231 // Multiple vectors6232 def _M2Z2Z_BtoH : sme2_fp8_fp16_quarter_tile_outer_product<0b1, 0b1, mnemonic, ZZ_b_mul_r_Lo, ZZ_b_mul_r_Hi>, SMEPseudo2Instr<NAME # _M2Z2Z_BtoH, 1>;6233 6234 def NAME # _M2Z2Z_BtoH_PSEUDO : sme2_quarter_tile_outer_product_pseudo<ZZ_b_mul_r_Lo, ZZ_b_mul_r_Hi, SMEMatrixTileH>, SMEPseudo2Instr<NAME # _M2Z2Z_BtoH, 0>;6235 6236 def : SME2_ZA_Tile_Vec_Multi_Multi_Pat<NAME # _M2Z2Z_BtoH, !cast<SDPatternOperator>(op # "_2x2"), timm32_0_1, nxv16i8>;6237}6238 6239// FP8 SME FDOT instructions6240 6241multiclass sme2_fp8_fdot_index_za16_vg1x2<string mnemonic, bits<3> op,6242 SDPatternOperator intrinsic> {6243 def NAME : sme2_multi_vec_array_vg2_index<0b11, {op{2},?,?,op{1-0},?}, MatrixOp16,6244 ZZ_b_mul_r, ZPR4b8,6245 VectorIndexH32b_timm, mnemonic>,6246 SMEPseudo2Instr<NAME, 1>{6247 let Uses=[FPMR, FPCR];6248 6249 bits<3> i;6250 let Inst{11-10} = i{2-1};6251 let Inst{3} = i{0};6252 }6253 6254 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i",6255 (!cast<Instruction>(NAME) MatrixOp16:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,6256 ZZ_b_mul_r:$Zn, ZPR4b8:$Zm, VectorIndexH32b_timm:$i), 0>;6257 6258 6259 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, ZZ_b_mul_r, ZPR4b8, VectorIndexH32b_timm, SMEMatrixArray>;6260 6261 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, ZPR4b8, nxv16i8, VectorIndexH32b_timm, tileslice16>;6262}6263 6264multiclass sme2_fp8_fdot_index_za16_vg1x4<string mnemonic,6265 SDPatternOperator intrinsic> {6266 def NAME : sme2_multi_vec_array_vg4_index<0b0, {0b1,?,?,0b100,?}, MatrixOp16,6267 ZZZZ_b_mul_r, ZPR4b8,6268 VectorIndexH32b_timm, mnemonic>,6269 SMEPseudo2Instr<NAME, 1> {6270 let Uses=[FPMR, FPCR];6271 6272 bits<3> i;6273 let Inst{11-10} = i{2-1};6274 let Inst{3} = i{0};6275 }6276 6277 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i",6278 (!cast<Instruction>(NAME) MatrixOp16:$ZAda, MatrixIndexGPR32Op8_11:$Rv,6279 sme_elm_idx0_7:$imm3, ZZZZ_b_mul_r:$Zn, ZPR4b8:$Zm, VectorIndexH32b_timm:$i), 0>;6280 6281 6282 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, ZZZZ_b_mul_r, ZPR4b8, VectorIndexH32b_timm, SMEMatrixArray>;6283 6284 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, ZPR4b8, nxv16i8, VectorIndexH32b_timm, tileslice16>;6285}6286 6287multiclass sme2_fp8_fdot_index_za32_vg1x2<string mnemonic,6288 SDPatternOperator intrinsic> {6289 def NAME : sme2_multi_vec_array_vg2_index<0b01, {0b0,?,?,0b111}, MatrixOp32, ZZ_b_mul_r, ZPR4b8,6290 VectorIndexS32b_timm, mnemonic>,6291 SMEPseudo2Instr<NAME, 1> {6292 let Uses=[FPMR, FPCR];6293 6294 bits<2> i;6295 let Inst{11-10} = i;6296 }6297 6298 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i",6299 (!cast<Instruction>(NAME) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,6300 ZZ_b_mul_r:$Zn, ZPR4b8:$Zm, VectorIndexS32b_timm:$i), 0>;6301 6302 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, ZZ_b_mul_r, ZPR4b8, VectorIndexS32b_timm, SMEMatrixArray>;6303 6304 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, ZPR4b8, nxv16i8, VectorIndexS32b_timm, tileslice16>;6305}6306 6307multiclass sme2_fp8_fdot_index_za32_vg1x4<string mnemonic,6308 SDPatternOperator intrinsic> {6309 def NAME : sme2_multi_vec_array_vg4_index<0b1, {0b0,?,?,0b0,0b001}, MatrixOp32, ZZZZ_b_mul_r,6310 ZPR4b8, VectorIndexS32b_timm, mnemonic>,6311 SMEPseudo2Instr<NAME, 1> {6312 let Uses=[FPMR, FPCR];6313 6314 bits<2> i;6315 let Inst{11-10} = i;6316 }6317 6318 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i",6319 (!cast<Instruction>(NAME) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,6320 ZZZZ_b_mul_r:$Zn, ZPR4b8:$Zm, VectorIndexS32b_timm:$i), 0>;6321 6322 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, ZZZZ_b_mul_r, ZPR4b8, VectorIndexS32b_timm, SMEMatrixArray>;6323 6324 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, ZPR4b8, nxv16i8, VectorIndexS32b_timm, tileslice16>;6325}6326 6327multiclass sme2_fp8_fdotv_index_za32_vg1x4<string mnemonic, bit T, SDPatternOperator intrinsic> {6328 def NAME : sme2_fp8_multi_vec_array_vg4_index<mnemonic, T>,6329 SMEPseudo2Instr<NAME, 1>;6330 6331 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, ZZ_b_mul_r, ZPR4b8, VectorIndexS32b_timm, SMEMatrixArray>;6332 6333 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, ZPR4b8, nxv16i8, VectorIndexS32b_timm, tileslice16>;6334}6335 6336multiclass sme2_fp8_fdot_single_vg1x2<string mnemonic, bits<7> op,6337 MatrixOperand matrix_op,6338 SDPatternOperator intrinsic> {6339 def NAME: sme2_dot_mla_add_sub_array_vg24_single<op, matrix_op, ZZ_b, ZPR4b8, mnemonic>,6340 SMEPseudo2Instr<NAME, 1> {6341 let Uses=[FPMR, FPCR];6342 }6343 6344 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm",6345 (!cast<Instruction>(NAME) matrix_op:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, ZZ_b:$Zn, ZPR4b8:$Zm), 0>;6346 6347 def _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, sme_elm_idx0_7, ZZ_b, ZPR4b8, SMEMatrixArray>;6348 6349 def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME, intrinsic, sme_elm_idx0_7, ZPR4b8, nxv16i8, tileslice16>;6350}6351 6352multiclass sme2_fp8_fdot_single_vg1x4<string mnemonic, bits<7> op,6353 MatrixOperand matrix_op,6354 SDPatternOperator intrinsic> {6355 def NAME: sme2_dot_mla_add_sub_array_vg24_single<op, matrix_op, ZZZZ_b, ZPR4b8, mnemonic>,6356 SMEPseudo2Instr<NAME, 1> {6357 let Uses=[FPMR, FPCR];6358 }6359 6360 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm",6361 (!cast<Instruction>(NAME) matrix_op:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, ZZZZ_b:$Zn, ZPR4b8:$Zm), 0>;6362 6363 def _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, sme_elm_idx0_7, ZZZZ_b, ZPR4b8, SMEMatrixArray>;6364 6365 def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME, intrinsic, sme_elm_idx0_7, ZPR4b8, nxv16i8, tileslice16>;6366}6367 6368multiclass sme2_fp8_fdot_multi_vg1x2<string mnemonic, bits<7> op,6369 MatrixOperand matrix_op,6370 SDPatternOperator intrinsic> {6371 def NAME : sme2_dot_mla_add_sub_array_vg2_multi<op, matrix_op, ZZ_b_mul_r, mnemonic>,6372 SMEPseudo2Instr<NAME, 1> {6373 let Uses=[FPMR, FPCR];6374 }6375 6376 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm",6377 (!cast<Instruction>(NAME) matrix_op:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, ZZ_b_mul_r:$Zn, ZZ_b_mul_r:$Zm), 0>;6378 6379 def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, sme_elm_idx0_7, ZZ_b_mul_r, SMEMatrixArray>;6380 6381 def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME, intrinsic, sme_elm_idx0_7, nxv16i8, tileslice16>;6382}6383 6384multiclass sme2_fp8_fdot_multi_vg1x4<string mnemonic, bits<7> op,6385 MatrixOperand matrix_op,6386 SDPatternOperator intrinsic> {6387 def NAME : sme2_dot_mla_add_sub_array_vg4_multi<op, matrix_op, ZZZZ_b_mul_r, mnemonic>,6388 SMEPseudo2Instr<NAME, 1> {6389 let Uses=[FPMR, FPCR];6390 }6391 6392 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm",6393 (!cast<Instruction>(NAME) matrix_op:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, ZZZZ_b_mul_r:$Zn, ZZZZ_b_mul_r:$Zm), 0>;6394 6395 def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, sme_elm_idx0_7, ZZZZ_b_mul_r, SMEMatrixArray>;6396 6397 def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME, intrinsic, sme_elm_idx0_7, nxv16i8, tileslice16>;6398}6399