11403 lines · plain
1//=-- SVEInstrFormats.td - AArch64 SVE Instruction classes -*- tablegen -*--=//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// AArch64 Scalable Vector Extension (SVE) Instruction Class Definitions.10//11//===----------------------------------------------------------------------===//12 13// Helper class to hold conversions of legal fixed-length vector types.14class NEONType<ValueType VT> {15 // The largest legal scalable vector type that can hold VT.16 ValueType SVEContainer = !cond(17 !eq(VT, v8i8): nxv16i8,18 !eq(VT, v16i8): nxv16i8,19 !eq(VT, v4i16): nxv8i16,20 !eq(VT, v8i16): nxv8i16,21 !eq(VT, v2i32): nxv4i32,22 !eq(VT, v4i32): nxv4i32,23 !eq(VT, v1i64): nxv2i64,24 !eq(VT, v2i64): nxv2i64,25 !eq(VT, v4f16): nxv8f16,26 !eq(VT, v8f16): nxv8f16,27 !eq(VT, v2f32): nxv4f32,28 !eq(VT, v4f32): nxv4f32,29 !eq(VT, v1f64): nxv2f64,30 !eq(VT, v2f64): nxv2f64,31 !eq(VT, v4bf16): nxv8bf16,32 !eq(VT, v8bf16): nxv8bf16,33 true : untyped);34}35 36// Helper class to hold conversions of legal scalable vector types.37class SVEType<ValueType VT> {38 // The largest legal scalable vector type that can hold VT.39 // Non-matches return VT because only packed types remain.40 ValueType Packed = !cond(41 !eq(VT, nxv2f16): nxv8f16,42 !eq(VT, nxv4f16): nxv8f16,43 !eq(VT, nxv2f32): nxv4f32,44 !eq(VT, nxv2bf16): nxv8bf16,45 !eq(VT, nxv4bf16): nxv8bf16,46 true : VT);47 48 // The legal scalable vector that is half the length of VT.49 ValueType HalfLength = !cond(50 !eq(VT, nxv8f16): nxv4f16,51 !eq(VT, nxv4f16): nxv2f16,52 !eq(VT, nxv4f32): nxv2f32,53 !eq(VT, nxv8bf16): nxv4bf16,54 !eq(VT, nxv4bf16): nxv2bf16,55 true : untyped);56 57 // The legal scalable vector that is quarter the length of VT.58 ValueType QuarterLength = !cond(59 !eq(VT, nxv8f16): nxv2f16,60 !eq(VT, nxv8bf16): nxv2bf16,61 true : untyped);62 63 // The 64-bit vector subreg of VT.64 ValueType DSub = !cond(65 !eq(VT, nxv16i8): v8i8,66 !eq(VT, nxv8i16): v4i16,67 !eq(VT, nxv4i32): v2i32,68 !eq(VT, nxv2i64): v1i64,69 !eq(VT, nxv2f16): v4f16,70 !eq(VT, nxv4f16): v4f16,71 !eq(VT, nxv8f16): v4f16,72 !eq(VT, nxv2f32): v2f32,73 !eq(VT, nxv4f32): v2f32,74 !eq(VT, nxv2f64): v1f64,75 !eq(VT, nxv2bf16): v4bf16,76 !eq(VT, nxv4bf16): v4bf16,77 !eq(VT, nxv8bf16): v4bf16,78 true : untyped);79 80 // The 128-bit vector subreg of VT.81 ValueType ZSub = !cond(82 !eq(VT, nxv16i8): v16i8,83 !eq(VT, nxv8i16): v8i16,84 !eq(VT, nxv4i32): v4i32,85 !eq(VT, nxv2i64): v2i64,86 !eq(VT, nxv2f16): v8f16,87 !eq(VT, nxv4f16): v8f16,88 !eq(VT, nxv8f16): v8f16,89 !eq(VT, nxv2f32): v4f32,90 !eq(VT, nxv4f32): v4f32,91 !eq(VT, nxv2f64): v2f64,92 !eq(VT, nxv2bf16): v8bf16,93 !eq(VT, nxv4bf16): v8bf16,94 !eq(VT, nxv8bf16): v8bf16,95 true : untyped);96 97 // The legal scalar used to hold a vector element.98 ValueType EltAsScalar = !cond(99 !eq(VT, nxv16i8): i32,100 !eq(VT, nxv8i16): i32,101 !eq(VT, nxv4i32): i32,102 !eq(VT, nxv2i64): i64,103 !eq(VT, nxv2f16): f16,104 !eq(VT, nxv4f16): f16,105 !eq(VT, nxv8f16): f16,106 !eq(VT, nxv2f32): f32,107 !eq(VT, nxv4f32): f32,108 !eq(VT, nxv2f64): f64,109 !eq(VT, nxv2bf16): bf16,110 !eq(VT, nxv4bf16): bf16,111 !eq(VT, nxv8bf16): bf16,112 true : untyped);113}114 115def SDT_AArch64Setcc : SDTypeProfile<1, 4, [116 SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVec<3>,117 SDTCVecEltisVT<0, i1>, SDTCVecEltisVT<1, i1>, SDTCisSameAs<2, 3>,118 SDTCisVT<4, OtherVT>119]>;120 121def AArch64setcc_z : SDNode<"AArch64ISD::SETCC_MERGE_ZERO", SDT_AArch64Setcc>;122let HasOneUse = 1 in123def AArch64setcc_z_oneuse : PatFrag<(ops node:$pg, node:$op1, node:$op2, node:$cc),124 (AArch64setcc_z node:$pg, node:$op1, node:$op2, node:$cc)>;125 126def SVEPatternOperand : AsmOperandClass {127 let Name = "SVEPattern";128 let ParserMethod = "tryParseSVEPattern";129 let PredicateMethod = "isSVEPattern";130 let RenderMethod = "addImmOperands";131 let DiagnosticType = "InvalidSVEPattern";132}133 134def sve_pred_enum : Operand<i32>, TImmLeaf<i32, [{135 return (((uint32_t)Imm) < 32);136 }]> {137 138 let PrintMethod = "printSVEPattern";139 let ParserMatchClass = SVEPatternOperand;140}141 142def SVEVecLenSpecifierOperand : AsmOperandClass {143 let Name = "SVEVecLenSpecifier";144 let ParserMethod = "tryParseSVEVecLenSpecifier";145 let PredicateMethod = "isSVEVecLenSpecifier";146 let RenderMethod = "addImmOperands";147 let DiagnosticType = "InvalidSVEVecLenSpecifier";148}149 150def sve_vec_len_specifier_enum : Operand<i32>, TImmLeaf<i32, [{151 return (((uint32_t)Imm) < 2);152 }]> {153 154 let PrintMethod = "printSVEVecLenSpecifier";155 let ParserMatchClass = SVEVecLenSpecifierOperand;156}157 158def SVEPrefetchOperand : AsmOperandClass {159 let Name = "SVEPrefetch";160 let ParserMethod = "tryParsePrefetch<true>";161 let PredicateMethod = "isPrefetch";162 let RenderMethod = "addPrefetchOperands";163}164 165def sve_prfop : Operand<i32>, TImmLeaf<i32, [{166 return (((uint32_t)Imm) <= 15);167 }]> {168 let PrintMethod = "printPrefetchOp<true>";169 let ParserMatchClass = SVEPrefetchOperand;170}171 172class SVELogicalImmOperand<int Width> : AsmOperandClass {173 let Name = "SVELogicalImm" # Width;174 let DiagnosticType = "LogicalSecondSource";175 let PredicateMethod = "isLogicalImm<int" # Width # "_t>";176 let RenderMethod = "addLogicalImmOperands<int" # Width # "_t>";177}178 179def sve_logical_imm8 : Operand<i64> {180 let ParserMatchClass = SVELogicalImmOperand<8>;181 let PrintMethod = "printLogicalImm<int8_t>";182 183 let MCOperandPredicate = [{184 if (!MCOp.isImm())185 return false;186 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64);187 return AArch64_AM::isSVEMaskOfIdenticalElements<int8_t>(Val);188 }];189}190 191def sve_logical_imm16 : Operand<i64> {192 let ParserMatchClass = SVELogicalImmOperand<16>;193 let PrintMethod = "printLogicalImm<int16_t>";194 195 let MCOperandPredicate = [{196 if (!MCOp.isImm())197 return false;198 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64);199 return AArch64_AM::isSVEMaskOfIdenticalElements<int16_t>(Val);200 }];201}202 203def sve_logical_imm32 : Operand<i64> {204 let ParserMatchClass = SVELogicalImmOperand<32>;205 let PrintMethod = "printLogicalImm<int32_t>";206 207 let MCOperandPredicate = [{208 if (!MCOp.isImm())209 return false;210 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64);211 return AArch64_AM::isSVEMaskOfIdenticalElements<int32_t>(Val);212 }];213}214 215class SVEPreferredLogicalImmOperand<int Width> : AsmOperandClass {216 let Name = "SVEPreferredLogicalImm" # Width;217 let PredicateMethod = "isSVEPreferredLogicalImm<int" # Width # "_t>";218 let RenderMethod = "addLogicalImmOperands<int" # Width # "_t>";219}220 221def sve_preferred_logical_imm16 : Operand<i64> {222 let ParserMatchClass = SVEPreferredLogicalImmOperand<16>;223 let PrintMethod = "printSVELogicalImm<int16_t>";224 225 let MCOperandPredicate = [{226 if (!MCOp.isImm())227 return false;228 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64);229 return AArch64_AM::isSVEMaskOfIdenticalElements<int16_t>(Val) &&230 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val);231 }];232}233 234def sve_preferred_logical_imm32 : Operand<i64> {235 let ParserMatchClass = SVEPreferredLogicalImmOperand<32>;236 let PrintMethod = "printSVELogicalImm<int32_t>";237 238 let MCOperandPredicate = [{239 if (!MCOp.isImm())240 return false;241 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64);242 return AArch64_AM::isSVEMaskOfIdenticalElements<int32_t>(Val) &&243 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val);244 }];245}246 247def sve_preferred_logical_imm64 : Operand<i64> {248 let ParserMatchClass = SVEPreferredLogicalImmOperand<64>;249 let PrintMethod = "printSVELogicalImm<int64_t>";250 251 let MCOperandPredicate = [{252 if (!MCOp.isImm())253 return false;254 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64);255 return AArch64_AM::isSVEMaskOfIdenticalElements<int64_t>(Val) &&256 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val);257 }];258}259 260class SVELogicalImmNotOperand<int Width> : AsmOperandClass {261 let Name = "SVELogicalImm" # Width # "Not";262 let DiagnosticType = "LogicalSecondSource";263 let PredicateMethod = "isLogicalImm<int" # Width # "_t>";264 let RenderMethod = "addLogicalImmNotOperands<int" # Width # "_t>";265}266 267def sve_logical_imm8_not : Operand<i64> {268 let ParserMatchClass = SVELogicalImmNotOperand<8>;269}270 271def sve_logical_imm16_not : Operand<i64> {272 let ParserMatchClass = SVELogicalImmNotOperand<16>;273}274 275def sve_logical_imm32_not : Operand<i64> {276 let ParserMatchClass = SVELogicalImmNotOperand<32>;277}278 279class SVEShiftedImmOperand<int ElementWidth, string Infix, string Predicate>280 : AsmOperandClass {281 let Name = "SVE" # Infix # "Imm" # ElementWidth;282 let DiagnosticType = "Invalid" # Name;283 let RenderMethod = "addImmWithOptionalShiftOperands<8>";284 let ParserMethod = "tryParseImmWithOptionalShift";285 let PredicateMethod = Predicate;286}287 288def SVECpyImmOperand8 : SVEShiftedImmOperand<8, "Cpy", "isSVECpyImm<int8_t>">;289def SVECpyImmOperand16 : SVEShiftedImmOperand<16, "Cpy", "isSVECpyImm<int16_t>">;290def SVECpyImmOperand32 : SVEShiftedImmOperand<32, "Cpy", "isSVECpyImm<int32_t>">;291def SVECpyImmOperand64 : SVEShiftedImmOperand<64, "Cpy", "isSVECpyImm<int64_t>">;292 293def SVEAddSubImmOperand8 : SVEShiftedImmOperand<8, "AddSub", "isSVEAddSubImm<int8_t>">;294def SVEAddSubImmOperand16 : SVEShiftedImmOperand<16, "AddSub", "isSVEAddSubImm<int16_t>">;295def SVEAddSubImmOperand32 : SVEShiftedImmOperand<32, "AddSub", "isSVEAddSubImm<int32_t>">;296def SVEAddSubImmOperand64 : SVEShiftedImmOperand<64, "AddSub", "isSVEAddSubImm<int64_t>">;297 298class imm8_opt_lsl<int ElementWidth, string printType,299 AsmOperandClass OpndClass>300 : Operand<i32> {301 let EncoderMethod = "getImm8OptLsl";302 let DecoderMethod = "DecodeImm8OptLsl<" # ElementWidth # ">";303 let PrintMethod = "printImm8OptLsl<" # printType # ">";304 let ParserMatchClass = OpndClass;305 let MIOperandInfo = (ops i32imm, i32imm);306}307 308def cpy_imm8_opt_lsl_i8 : imm8_opt_lsl<8, "int8_t", SVECpyImmOperand8>;309def cpy_imm8_opt_lsl_i16 : imm8_opt_lsl<16, "int16_t", SVECpyImmOperand16>;310def cpy_imm8_opt_lsl_i32 : imm8_opt_lsl<32, "int32_t", SVECpyImmOperand32>;311def cpy_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "int64_t", SVECpyImmOperand64>;312 313def addsub_imm8_opt_lsl_i8 : imm8_opt_lsl<8, "uint8_t", SVEAddSubImmOperand8>;314def addsub_imm8_opt_lsl_i16 : imm8_opt_lsl<16, "uint16_t", SVEAddSubImmOperand16>;315def addsub_imm8_opt_lsl_i32 : imm8_opt_lsl<32, "uint32_t", SVEAddSubImmOperand32>;316def addsub_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "uint64_t", SVEAddSubImmOperand64>;317 318let Complexity = 1 in {319def SVEAddSubImm8Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i8, false>", []>;320def SVEAddSubImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i16, false>", []>;321def SVEAddSubImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i32, false>", []>;322def SVEAddSubImm64Pat : ComplexPattern<i64, 2, "SelectSVEAddSubImm<MVT::i64, false>", []>;323 324def SVEAddSubNegImm8Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i8, true>", []>;325def SVEAddSubNegImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i16, true>", []>;326def SVEAddSubNegImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i32, true>", []>;327def SVEAddSubNegImm64Pat : ComplexPattern<i64, 2, "SelectSVEAddSubImm<MVT::i64, true>", []>;328 329def SVEAddSubSSatNegImm8Pat : ComplexPattern<i32, 2, "SelectSVEAddSubSSatImm<MVT::i8, true>", []>;330def SVEAddSubSSatNegImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubSSatImm<MVT::i16, true>", []>;331def SVEAddSubSSatNegImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubSSatImm<MVT::i32, true>", []>;332def SVEAddSubSSatNegImm64Pat : ComplexPattern<i64, 2, "SelectSVEAddSubSSatImm<MVT::i64, true>", []>;333 334def SVEAddSubSSatPosImm8Pat : ComplexPattern<i32, 2, "SelectSVEAddSubSSatImm<MVT::i8, false>", []>;335def SVEAddSubSSatPosImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubSSatImm<MVT::i16, false>", []>;336def SVEAddSubSSatPosImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubSSatImm<MVT::i32, false>", []>;337def SVEAddSubSSatPosImm64Pat : ComplexPattern<i64, 2, "SelectSVEAddSubSSatImm<MVT::i64, false>", []>;338} // Complexity = 1339 340def SVECpyDupImm8Pat : ComplexPattern<i32, 2, "SelectSVECpyDupImm<MVT::i8>", []>;341def SVECpyDupImm16Pat : ComplexPattern<i32, 2, "SelectSVECpyDupImm<MVT::i16>", []>;342def SVECpyDupImm32Pat : ComplexPattern<i32, 2, "SelectSVECpyDupImm<MVT::i32>", []>;343def SVECpyDupImm64Pat : ComplexPattern<i64, 2, "SelectSVECpyDupImm<MVT::i64>", []>;344 345def SVELogicalImm8Pat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i8>", []>;346def SVELogicalImm16Pat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i16>", []>;347def SVELogicalImm32Pat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i32>", []>;348def SVELogicalImm64Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64>", []>;349 350def SVELogicalFPImm16Pat : ComplexPattern<f16, 1, "SelectSVELogicalImm<MVT::i16>", []>;351def SVELogicalFPImm32Pat : ComplexPattern<f32, 1, "SelectSVELogicalImm<MVT::i32>", []>;352def SVELogicalFPImm64Pat : ComplexPattern<f64, 1, "SelectSVELogicalImm<MVT::i64>", []>;353def SVELogicalBFPImmPat : ComplexPattern<bf16, 1, "SelectSVELogicalImm<MVT::i16>", []>;354 355def SVELogicalImm8NotPat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i8, true>", []>;356def SVELogicalImm16NotPat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i16, true>", []>;357def SVELogicalImm32NotPat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i32, true>", []>;358def SVELogicalImm64NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64, true>", []>;359 360def SVEArithUImm8Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i8>", []>;361def SVEArithUImm16Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i16>", []>;362def SVEArithUImm32Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i32>", []>;363def SVEArithUImm64Pat : ComplexPattern<i64, 1, "SelectSVEArithImm<MVT::i64>", []>;364 365def SVEArithSImmPat32 : ComplexPattern<i32, 1, "SelectSVESignedArithImm", []>;366def SVEArithSImmPat64 : ComplexPattern<i64, 1, "SelectSVESignedArithImm", []>;367 368def SVEShiftImmL8 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 7>", []>;369def SVEShiftImmL16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 15>", []>;370def SVEShiftImmL32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 31>", []>;371def SVEShiftImmL64 : ComplexPattern<i64, 1, "SelectSVEShiftImm<0, 63>", []>;372def SVEShiftImmR8 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 8, true>", []>;373def SVEShiftImmR16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 16, true>", []>;374def SVEShiftImmR32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 32, true>", []>;375def SVEShiftImmR64 : ComplexPattern<i64, 1, "SelectSVEShiftImm<1, 64, true>", []>;376 377def SVEShiftSplatImmR : ComplexPattern<iAny, 1, "SelectSVEShiftSplatImmR", []>;378 379def SVEAllActive : ComplexPattern<untyped, 0, "SelectAllActivePredicate", []>;380def SVEAnyPredicate : ComplexPattern<untyped, 0, "SelectAnyPredicate", []>;381 382class SVEExactFPImm<string Suffix, string ValA, string ValB> : AsmOperandClass {383 let Name = "SVEExactFPImmOperand" # Suffix;384 let DiagnosticType = "Invalid" # Name;385 let ParserMethod = "tryParseFPImm<false>";386 let PredicateMethod = "isExactFPImm<" # ValA # ", " # ValB # ">";387 let RenderMethod = "addExactFPImmOperands<" # ValA # ", " # ValB # ">";388}389 390class SVEExactFPImmOperand<string Suffix, string ValA, string ValB> : Operand<i32> {391 let PrintMethod = "printExactFPImm<" # ValA # ", " # ValB # ">";392 let ParserMatchClass = SVEExactFPImm<Suffix, ValA, ValB>;393}394 395def sve_fpimm_half_one396 : SVEExactFPImmOperand<"HalfOne", "AArch64ExactFPImm::half",397 "AArch64ExactFPImm::one">;398def sve_fpimm_half_two399 : SVEExactFPImmOperand<"HalfTwo", "AArch64ExactFPImm::half",400 "AArch64ExactFPImm::two">;401def sve_fpimm_zero_one402 : SVEExactFPImmOperand<"ZeroOne", "AArch64ExactFPImm::zero",403 "AArch64ExactFPImm::one">;404 405def sve_incdec_imm : Operand<i32>, TImmLeaf<i32, [{406 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 17);407}]> {408 let ParserMatchClass = Imm1_16Operand;409 let EncoderMethod = "getSVEIncDecImm";410 let DecoderMethod = "DecodeSVEIncDecImm";411}412 413// This allows i32 immediate extraction from i64 based arithmetic.414def sve_cnt_mul_imm_i32 : ComplexPattern<i32, 1, "SelectCntImm<1, 16, 1, false>">;415def sve_cnt_mul_imm_i64 : ComplexPattern<i64, 1, "SelectCntImm<1, 16, 1, false>">;416def sve_cnt_shl_imm : ComplexPattern<i64, 1, "SelectCntImm<1, 16, 1, true>">;417 418def sve_ext_imm_0_31 : ComplexPattern<i64, 1, "SelectEXTImm<31, 8>">;419def sve_ext_imm_0_63 : ComplexPattern<i64, 1, "SelectEXTImm<63, 4>">;420def sve_ext_imm_0_127 : ComplexPattern<i64, 1, "SelectEXTImm<127, 2>">;421def sve_ext_imm_0_255 : ComplexPattern<i64, 1, "SelectEXTImm<255, 1>">;422 423let HasOneUse = 1 in424def int_aarch64_sve_cntp_oneuse : PatFrag<(ops node:$pred, node:$src2),425 (int_aarch64_sve_cntp node:$pred, node:$src2)>;426let HasOneUse = 1 in427def step_vector_oneuse : PatFrag<(ops node:$idx),428 (step_vector node:$idx)>;429 430 431//===----------------------------------------------------------------------===//432// SVE PTrue - These are used extensively throughout the pattern matching so433// it's important we define them first.434//===----------------------------------------------------------------------===//435 436class sve_int_ptrue<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty,437 ValueType vt, SDPatternOperator op>438: I<(outs pprty:$Pd), (ins sve_pred_enum:$pattern),439 asm, "\t$Pd, $pattern",440 "",441 [(set (vt pprty:$Pd), (op sve_pred_enum:$pattern))]>, Sched<[]> {442 bits<4> Pd;443 bits<5> pattern;444 let Inst{31-24} = 0b00100101;445 let Inst{23-22} = sz8_64;446 let Inst{21-19} = 0b011;447 let Inst{18-17} = opc{2-1};448 let Inst{16} = opc{0};449 let Inst{15-10} = 0b111000;450 let Inst{9-5} = pattern;451 let Inst{4} = 0b0;452 let Inst{3-0} = Pd;453 454 let Defs = !if(!eq (opc{0}, 1), [NZCV], []);455 let ElementSize = pprty.ElementSize;456 let hasSideEffects = 0;457 let isReMaterializable = 1;458 let Uses = [VG];459}460 461multiclass sve_int_ptrue<bits<3> opc, string asm, SDPatternOperator op> {462 def _B : sve_int_ptrue<0b00, opc, asm, PPR8, nxv16i1, op>;463 def _H : sve_int_ptrue<0b01, opc, asm, PPR16, nxv8i1, op>;464 def _S : sve_int_ptrue<0b10, opc, asm, PPR32, nxv4i1, op>;465 def _D : sve_int_ptrue<0b11, opc, asm, PPR64, nxv2i1, op>;466 467 def : InstAlias<asm # "\t$Pd",468 (!cast<Instruction>(NAME # _B) PPR8:$Pd, 0b11111), 1>;469 def : InstAlias<asm # "\t$Pd",470 (!cast<Instruction>(NAME # _H) PPR16:$Pd, 0b11111), 1>;471 def : InstAlias<asm # "\t$Pd",472 (!cast<Instruction>(NAME # _S) PPR32:$Pd, 0b11111), 1>;473 def : InstAlias<asm # "\t$Pd",474 (!cast<Instruction>(NAME # _D) PPR64:$Pd, 0b11111), 1>;475}476 477def SDT_AArch64PTrue : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVT<1, i32>]>;478def AArch64ptrue : SDNode<"AArch64ISD::PTRUE", SDT_AArch64PTrue>;479 480let Predicates = [HasSVE_or_SME] in {481 defm PTRUE : sve_int_ptrue<0b000, "ptrue", AArch64ptrue>;482 defm PTRUES : sve_int_ptrue<0b001, "ptrues", null_frag>;483 484 def : Pat<(nxv16i1 immAllOnesV), (PTRUE_B 31)>;485 def : Pat<(nxv8i1 immAllOnesV), (PTRUE_H 31)>;486 def : Pat<(nxv4i1 immAllOnesV), (PTRUE_S 31)>;487 def : Pat<(nxv2i1 immAllOnesV), (PTRUE_D 31)>;488}489 490//===----------------------------------------------------------------------===//491// SVE pattern match helpers.492//===----------------------------------------------------------------------===//493def SVEDup0 : ComplexPattern<vAny, 0, "SelectDupZero", []>;494def SVEDup0Undef : ComplexPattern<vAny, 0, "SelectDupZeroOrUndef", []>;495def SVEAny : ComplexPattern<vAny, 0, "SelectAny", []>;496 497class SVE_1_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,498 Instruction inst>499: Pat<(vtd (op vt1:$Op1)),500 (inst $Op1)>;501 502class SVE_1_Op_Passthru_Pat<ValueType vtd, SDPatternOperator op, ValueType pg,503 ValueType vts, Instruction inst>504: Pat<(vtd (op pg:$Op1, vts:$Op2, vtd:$Op3)),505 (inst $Op3, $Op1, $Op2)>;506 507 508multiclass SVE_1_Op_PassthruUndef_Pat<ValueType vtd, SDPatternOperator op, ValueType pg,509 ValueType vts, Instruction inst> {510 def : Pat<(vtd (op pg:$Op1, vts:$Op2, (vtd undef))),511 (inst (IMPLICIT_DEF), $Op1, $Op2)>;512 def : Pat<(vtd (op (pg (SVEAllActive:$Op1)), vts:$Op2, vtd:$Op3)),513 (inst $Op3, $Op1, $Op2)>;514}515 516multiclass SVE_1_Op_PassthruUndefZero_Pat<ValueType vtd, SDPatternOperator op, ValueType pg,517 ValueType vts, Instruction inst> {518 let AddedComplexity = 1 in {519 def : Pat<(vtd (op pg:$Op1, vts:$Op2, (vtd (SVEDup0Undef)))),520 (inst $Op1, $Op2)>;521 def : Pat<(vtd (op (pg (SVEAllActive:$Op1)), vts:$Op2, (vtd (SVEAny)))),522 (inst $Op1, $Op2)>;523 }524}525 526// Used to match FP_ROUND_MERGE_PASSTHRU, which has an additional flag for the527// type of rounding. This is matched by timm0_1 in pattern below and ignored.528class SVE_1_Op_Passthru_Round_Pat<ValueType vtd, SDPatternOperator op, ValueType pg,529 ValueType vts, Instruction inst>530: Pat<(vtd (op pg:$Op1, vts:$Op2, (i64 timm0_1), vtd:$Op3)),531 (inst $Op3, $Op1, $Op2)>;532 533multiclass SVE_1_Op_PassthruUndef_Round_Pat<ValueType vtd, SDPatternOperator op, ValueType pg,534 ValueType vts, Instruction inst>{535 def : Pat<(vtd (op pg:$Op1, vts:$Op2, (i64 timm0_1), (vtd undef))),536 (inst (IMPLICIT_DEF), $Op1, $Op2)>;537 def : Pat<(vtd (op (pg (SVEAllActive:$Op1)), vts:$Op2, (i64 timm0_1), vtd:$Op3)),538 (inst $Op3, $Op1, $Op2)>;539}540 541class SVE_1_Op_PassthruZero_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,542 ValueType vt2, Instruction inst>543 : Pat<(vtd (op (vtd (SVEDup0)), vt1:$Op1, vt2:$Op2)),544 (inst (IMPLICIT_DEF), $Op1, $Op2)>;545 546class SVE_1_Op_Imm_OptLsl_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty,547 ValueType it, ComplexPattern cpx, Instruction inst>548 : Pat<(vt (op (vt zprty:$Op1), (vt (splat_vector (it (cpx i32:$imm, i32:$shift)))))),549 (inst $Op1, i32:$imm, i32:$shift)>;550 551class SVE_1_Op_Imm_Arith_Any_Predicate<ValueType vt, ValueType pt,552 SDPatternOperator op, ZPRRegOp zprty,553 ValueType it, ComplexPattern cpx,554 Instruction inst>555 : Pat<(vt (op (pt (SVEAnyPredicate)), (vt zprty:$Op1), (vt (splat_vector (it (cpx i32:$imm)))))),556 (inst $Op1, i32:$imm)>;557 558class SVE_1_Op_Imm_Log_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty,559 ValueType it, ComplexPattern cpx, Instruction inst>560 : Pat<(vt (op (vt zprty:$Op1), (vt (splat_vector (it (cpx i64:$imm)))))),561 (inst $Op1, i64:$imm)>;562 563class SVE_2_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,564 ValueType vt2, Instruction inst>565: Pat<(vtd (op vt1:$Op1, vt2:$Op2)),566 (inst $Op1, $Op2)>;567 568class SVE_2_Op_Pred_All_Active<ValueType vtd, SDPatternOperator op,569 ValueType pt, ValueType vt1, ValueType vt2,570 Instruction inst>571: Pat<(vtd (op (pt (SVEAllActive)), vt1:$Op1, vt2:$Op2)),572 (inst $Op1, $Op2)>;573 574class SVE_2_Op_Pred_All_Active_Pt<ValueType vtd, SDPatternOperator op,575 ValueType pt, ValueType vt1, ValueType vt2,576 Instruction inst>577: Pat<(vtd (op (pt (SVEAllActive:$Op1)), vt1:$Op2, vt2:$Op3)),578 (inst $Op1, $Op2, $Op3)>;579 580class SVE_3_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,581 ValueType vt2, ValueType vt3, Instruction inst>582: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3)),583 (inst $Op1, $Op2, $Op3)>;584 585multiclass SVE_3_Op_Undef_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,586 ValueType vt2, ValueType vt3, Instruction inst> {587 def : Pat<(vtd (op (vt1 undef), vt2:$Op1, vt3:$Op2)),588 (inst (IMPLICIT_DEF), $Op1, $Op2)>;589 def : Pat<(vtd (op vt1:$Op1, (vt2 (SVEAllActive:$Op2)), vt3:$Op3)),590 (inst $Op1, $Op2, $Op3)>;591}592 593multiclass SVE_3_Op_UndefZero_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,594 ValueType vt2, ValueType vt3, Instruction inst> {595 let AddedComplexity = 1 in {596 def : Pat<(vtd (op (vt1 (SVEDup0Undef)), vt2:$Op1, vt3:$Op2)),597 (inst $Op1, $Op2)>;598 def : Pat<(vtd (op (vt1 (SVEAny)), (vt2 (SVEAllActive:$Op2)), vt3:$Op3)),599 (inst $Op2, $Op3)>;600 }601}602 603class SVE_4_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,604 ValueType vt2, ValueType vt3, ValueType vt4,605 Instruction inst>606: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, vt4:$Op4)),607 (inst $Op1, $Op2, $Op3, $Op4)>;608 609class SVE_2_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,610 ValueType vt2, Operand ImmTy, Instruction inst>611: Pat<(vtd (op vt1:$Op1, (vt2 ImmTy:$Op2))),612 (inst $Op1, ImmTy:$Op2)>;613 614multiclass SVE2p1_Cntp_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,615 Instruction inst> {616 def : Pat<(vtd (op vt1:$Op1, (i32 2))), (inst $Op1, 0)>;617 def : Pat<(vtd (op vt1:$Op1, (i32 4))), (inst $Op1, 1)>;618}619 620multiclass SVE2p1_While_PN_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,621 Instruction inst> {622 def : Pat<(vtd (op vt1:$Op1, vt1:$Op2, (i32 2))), (inst $Op1, $Op2, 0)>;623 def : Pat<(vtd (op vt1:$Op1, vt1:$Op2, (i32 4))), (inst $Op1, $Op2, 1)>;624}625 626class SVE_3_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,627 ValueType vt2, ValueType vt3, Operand ImmTy,628 Instruction inst>629: Pat<(vtd (op vt1:$Op1, vt2:$Op2, (vt3 ImmTy:$Op3))),630 (inst $Op1, $Op2, ImmTy:$Op3)>;631 632class SVE_4_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,633 ValueType vt2, ValueType vt3, ValueType vt4,634 Operand ImmTy, Instruction inst>635: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, (vt4 ImmTy:$Op4))),636 (inst $Op1, $Op2, $Op3, ImmTy:$Op4)>;637 638let AddedComplexity = 1 in {639class SVE_3_Op_Pat_SelZero<ValueType vtd, SDPatternOperator op, ValueType vt1,640 ValueType vt2, ValueType vt3, Instruction inst>641: Pat<(vtd (vtd (op vt1:$Op1, (vselect vt1:$Op1, vt2:$Op2, (SVEDup0)), vt3:$Op3))),642 (inst $Op1, $Op2, $Op3)>;643 644class SVE_3_Op_Pat_Shift_Imm_SelZero<ValueType vtd, SDPatternOperator op,645 ValueType vt1, ValueType vt2,646 Operand vt3, Instruction inst>647: Pat<(vtd (op vt1:$Op1, (vselect vt1:$Op1, vt2:$Op2, (SVEDup0)), (i32 (vt3:$Op3)))),648 (inst $Op1, $Op2, vt3:$Op3)>;649}650 651//652// Common but less generic patterns.653//654 655class SVE_2_Op_AllActive_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,656 ValueType vt2, Instruction inst, Instruction ptrue>657: Pat<(vtd (op vt1:$Op1, vt2:$Op2)),658 (inst (ptrue 31), $Op1, $Op2)>;659 660class SVE_InReg_Extend<ValueType vt, SDPatternOperator op, ValueType pt,661 ValueType inreg_vt, Instruction inst>662: Pat<(vt (op pt:$Pg, vt:$Src, inreg_vt, vt:$PassThru)),663 (inst $PassThru, $Pg, $Src)>;664 665multiclass SVE_InReg_Extend_PassthruUndef<ValueType vt, SDPatternOperator op, ValueType pt,666 ValueType inreg_vt, Instruction inst> {667 def : Pat<(vt (op pt:$Pg, vt:$Src, inreg_vt, (vt undef))),668 (inst (IMPLICIT_DEF), $Pg, $Src)>;669 def : Pat<(vt (op (pt (SVEAllActive:$Pg)), vt:$Src, inreg_vt, vt:$PassThru)),670 (inst $PassThru, $Pg, $Src)>;671}672 673multiclass SVE_InReg_Extend_PassthruUndefZero<ValueType vt, SDPatternOperator op, ValueType pt,674 ValueType inreg_vt, Instruction inst> {675 let AddedComplexity = 1 in {676 def : Pat<(vt (op pt:$Pg, vt:$Src, inreg_vt, (vt (SVEDup0Undef)))),677 (inst $Pg, $Src)>;678 679 def : Pat<(vt (op (pt (SVEAllActive:$Pg)), vt:$Src, inreg_vt, (vt (SVEAny)))),680 (inst $Pg, $Src)>;681 }682}683 684class SVE_Shift_DupImm_Pred_Pat<ValueType vt, SDPatternOperator op,685 ValueType pt, ValueType it,686 ComplexPattern cast, Instruction inst>687: Pat<(vt (op pt:$Pg, vt:$Rn, (vt (splat_vector (it (cast i32:$imm)))))),688 (inst $Pg, $Rn, i32:$imm)>;689 690class SVE_Shift_DupImm_Any_Predicate_Pat<ValueType vt, SDPatternOperator op,691 ValueType pt, ValueType it,692 ComplexPattern cast, Instruction inst>693: Pat<(vt (op (pt (SVEAnyPredicate)), vt:$Rn, (vt (splat_vector (it (cast i32:$imm)))))),694 (inst $Rn, i32:$imm)>;695 696class SVE_2_Op_Imm_Pat_Zero<ValueType vt, SDPatternOperator op, ValueType pt,697 ValueType it, ComplexPattern cpx, Instruction inst>698: Pat<(vt (op pt:$Pg, (vselect pt:$Pg, vt:$Op1, (SVEDup0)),699 (vt (splat_vector (it (cpx i32:$imm)))))),700 (inst $Pg, $Op1, i32:$imm)>;701 702class SVE_2_Op_Fp_Imm_Pat<ValueType vt, SDPatternOperator op,703 ValueType pt, ValueType it,704 FPImmLeaf immL, int imm,705 Instruction inst>706: Pat<(vt (op (pt PPR_3b:$Pg), (vt ZPR:$Zs1), (vt (splat_vector (it immL))))),707 (inst $Pg, $Zs1, imm)>;708 709class SVE_2_Op_Fp_Imm_Pat_Zero<ValueType vt, SDPatternOperator op,710 ValueType pt, ValueType it,711 FPImmLeaf immL, int imm,712 Instruction inst>713: Pat<(vt (op pt:$Pg, (vselect pt:$Pg, vt:$Zs1, (SVEDup0)),714 (vt (splat_vector (it immL))))),715 (inst $Pg, $Zs1, imm)>;716 717class SVE_Shift_Add_All_Active_Pat<ValueType vtd, SDPatternOperator op, ValueType pt,718 ValueType vt1, ValueType vt2, ValueType vt3,719 Instruction inst>720: Pat<(vtd (add vt1:$Op1, (op (pt (SVEAllActive)), vt2:$Op2, vt3:$Op3))),721 (inst $Op1, $Op2, $Op3)>;722 723class SVE2p1_Sat_Shift_VG2_Pat<string name, SDPatternOperator intrinsic, ValueType out_vt, ValueType in_vt, Operand imm_ty>724 : Pat<(out_vt (intrinsic in_vt:$Zn1, in_vt:$Zn2, (i32 imm_ty:$i))),725 (!cast<Instruction>(name) (REG_SEQUENCE ZPR2Mul2, in_vt:$Zn1, zsub0, in_vt:$Zn2, zsub1), imm_ty:$i)>;726 727class SVE2p1_Cvt_VG2_Pat<string name, SDPatternOperator intrinsic, ValueType out_vt, ValueType in_vt>728 : Pat<(out_vt (intrinsic in_vt:$Zn1, in_vt:$Zn2)),729 (!cast<Instruction>(name) (REG_SEQUENCE ZPR2Mul2, in_vt:$Zn1, zsub0, in_vt:$Zn2, zsub1))>;730 731//===----------------------------------------------------------------------===//732// SVE pattern match helpers.733//===----------------------------------------------------------------------===//734 735// Matches either an intrinsic, or a predicated operation with an all active predicate736class VSelectPredOrPassthruPatFrags<SDPatternOperator intrinsic, SDPatternOperator sdnode>737: PatFrags<(ops node:$Pg, node:$Op1, node:$Op2), [738 (intrinsic node:$Pg, node:$Op1, node:$Op2),739 (vselect node:$Pg, (sdnode (SVEAllActive), node:$Op1, node:$Op2), node:$Op1),740 ], [{741 return N->getOpcode() != ISD::VSELECT || N->getOperand(1).hasOneUse();742 }]>;743// Same as above with a commutative operation744class VSelectCommPredOrPassthruPatFrags<SDPatternOperator intrinsic, SDPatternOperator sdnode>745: PatFrags<(ops node:$Pg, node:$Op1, node:$Op2), [746 (intrinsic node:$Pg, node:$Op1, node:$Op2),747 (vselect node:$Pg, (sdnode (SVEAllActive), node:$Op1, node:$Op2), node:$Op1),748 (vselect node:$Pg, (sdnode (SVEAllActive), node:$Op2, node:$Op1), node:$Op1),749 ], [{750 return N->getOpcode() != ISD::VSELECT || N->getOperand(1).hasOneUse();751 }]>;752// Similarly matches either an intrinsic, or an unpredicated operation with a select753class VSelectUnpredOrPassthruPatFrags<SDPatternOperator intrinsic, SDPatternOperator sdnode>754: PatFrags<(ops node:$Pg, node:$Op1, node:$Op2), [755 (intrinsic node:$Pg, node:$Op1, node:$Op2),756 (vselect node:$Pg, (sdnode node:$Op1, node:$Op2), node:$Op1),757 ], [{758 return N->getOpcode() != ISD::VSELECT || N->getOperand(1).hasOneUse();759 }]>;760 761//762// Pseudo -> Instruction mappings763//764def getSVEPseudoMap : InstrMapping {765 let FilterClass = "SVEPseudo2Instr";766 let RowFields = ["PseudoName"];767 let ColFields = ["IsInstr"];768 let KeyCol = ["0"];769 let ValueCols = [["1"]];770}771 772class SVEPseudo2Instr<string name, bit instr> {773 string PseudoName = name;774 bit IsInstr = instr;775}776 777// Lookup e.g. DIV -> DIVR778def getSVERevInstr : InstrMapping {779 let FilterClass = "SVEInstr2Rev";780 let RowFields = ["InstrName"];781 let ColFields = ["isReverseInstr"];782 let KeyCol = ["0"];783 let ValueCols = [["1"]];784}785 786// Lookup e.g. DIVR -> DIV787def getSVENonRevInstr : InstrMapping {788 let FilterClass = "SVEInstr2Rev";789 let RowFields = ["InstrName"];790 let ColFields = ["isReverseInstr"];791 let KeyCol = ["1"];792 let ValueCols = [["0"]];793}794 795class SVEInstr2Rev<string name1, string name2, bit name1IsReverseInstr> {796 string InstrName = !if(name1IsReverseInstr, name1, name2);797 bit isReverseInstr = name1IsReverseInstr;798}799 800//801// Pseudos for destructive operands802//803let hasNoSchedulingInfo = 1 in {804 class PredTwoOpPseudo<string name, ZPRRegOp zprty,805 FalseLanesEnum flags = FalseLanesNone>806 : SVEPseudo2Instr<name, 0>,807 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, zprty:$Zs2), []> {808 let FalseLanes = flags;809 }810 811 class PredTwoOpImmPseudo<string name, ZPRRegOp zprty, Operand immty,812 FalseLanesEnum flags = FalseLanesNone>813 : SVEPseudo2Instr<name, 0>,814 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, immty:$imm), []> {815 let FalseLanes = flags;816 }817 818 class PredThreeOpPseudo<string name, ZPRRegOp zprty,819 FalseLanesEnum flags = FalseLanesNone>820 : SVEPseudo2Instr<name, 0>,821 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, zprty:$Zs2, zprty:$Zs3), []> {822 let FalseLanes = flags;823 }824 825 class UnpredRegImmPseudo<ZPRRegOp zprty, Operand immty>826 : SVEPseudo2Instr<NAME, 0>,827 Pseudo<(outs zprty:$Zd), (ins zprty:$Zs, immty:$imm), []> {828 }829}830 831//832// Pseudos for passthru operands833//834let hasNoSchedulingInfo = 1 in {835 class PredOneOpPassthruPseudo<string name, ZPRRegOp zprty,836 FalseLanesEnum flags = FalseLanesNone>837 : SVEPseudo2Instr<name, 0>,838 Pseudo<(outs zprty:$Zd), (ins zprty:$Passthru, PPR3bAny:$Pg, zprty:$Zs), []> {839 let FalseLanes = flags;840 let Constraints = !if(!eq(flags, FalseLanesZero), "$Zd = $Passthru,@earlyclobber $Zd", "");841 }842}843 844//===----------------------------------------------------------------------===//845// SVE Predicate Misc Group846//===----------------------------------------------------------------------===//847 848class sve_int_pfalse<bits<6> opc, string asm>849: I<(outs PPRorPNR8:$Pd), (ins),850 asm, "\t$Pd",851 "",852 []>, Sched<[]> {853 bits<4> Pd;854 let Inst{31-24} = 0b00100101;855 let Inst{23-22} = opc{5-4};856 let Inst{21-19} = 0b011;857 let Inst{18-16} = opc{3-1};858 let Inst{15-10} = 0b111001;859 let Inst{9} = opc{0};860 let Inst{8-4} = 0b00000;861 let Inst{3-0} = Pd;862 863 let hasSideEffects = 0;864 let isReMaterializable = 1;865 let Uses = [VG];866}867 868multiclass sve_int_pfalse<bits<6> opc, string asm> {869 def NAME : sve_int_pfalse<opc, asm>;870 871 def : Pat<(nxv16i1 immAllZerosV), (!cast<Instruction>(NAME))>;872 def : Pat<(nxv8i1 immAllZerosV), (!cast<Instruction>(NAME))>;873 def : Pat<(nxv4i1 immAllZerosV), (!cast<Instruction>(NAME))>;874 def : Pat<(nxv2i1 immAllZerosV), (!cast<Instruction>(NAME))>;875 def : Pat<(nxv1i1 immAllZerosV), (!cast<Instruction>(NAME))>;876}877 878class sve_int_ptest<bits<6> opc, string asm, SDPatternOperator op>879: I<(outs), (ins PPRAny:$Pg, PPR8:$Pn),880 asm, "\t$Pg, $Pn",881 "",882 [(set NZCV, (op (nxv16i1 PPRAny:$Pg), (nxv16i1 PPR8:$Pn)))]>, Sched<[]> {883 bits<4> Pg;884 bits<4> Pn;885 let Inst{31-24} = 0b00100101;886 let Inst{23-22} = opc{5-4};887 let Inst{21-19} = 0b010;888 let Inst{18-16} = opc{3-1};889 let Inst{15-14} = 0b11;890 let Inst{13-10} = Pg;891 let Inst{9} = opc{0};892 let Inst{8-5} = Pn;893 let Inst{4-0} = 0b00000;894 895 let Defs = [NZCV];896 let hasSideEffects = 0;897 let isCompare = 1;898}899 900multiclass sve_int_ptest<bits<6> opc, string asm, SDPatternOperator op,901 SDPatternOperator op_any, SDPatternOperator op_first> {902 def NAME : sve_int_ptest<opc, asm, op>;903 904 let hasNoSchedulingInfo = 1, isCompare = 1, Defs = [NZCV] in {905 def _ANY : Pseudo<(outs), (ins PPRAny:$Pg, PPR8:$Pn),906 [(set NZCV, (op_any (nxv16i1 PPRAny:$Pg), (nxv16i1 PPR8:$Pn)))]>,907 PseudoInstExpansion<(!cast<Instruction>(NAME) PPRAny:$Pg, PPR8:$Pn)>;908 909 def _FIRST : Pseudo<(outs), (ins PPRAny:$Pg, PPR8:$Pn),910 [(set NZCV, (op_first (nxv16i1 PPRAny:$Pg), (nxv16i1 PPR8:$Pn)))]>,911 PseudoInstExpansion<(!cast<Instruction>(NAME) PPRAny:$Pg, PPR8:$Pn)>;912 }913}914 915class sve_int_pfirst_next<bits<2> sz8_64, bits<5> opc, string asm,916 PPRRegOp pprty>917: I<(outs pprty:$Pdn), (ins PPRAny:$Pg, pprty:$_Pdn),918 asm, "\t$Pdn, $Pg, $_Pdn",919 "",920 []>, Sched<[]> {921 bits<4> Pdn;922 bits<4> Pg;923 let Inst{31-24} = 0b00100101;924 let Inst{23-22} = sz8_64;925 let Inst{21-19} = 0b011;926 let Inst{18-16} = opc{4-2};927 let Inst{15-11} = 0b11000;928 let Inst{10-9} = opc{1-0};929 let Inst{8-5} = Pg;930 let Inst{4} = 0;931 let Inst{3-0} = Pdn;932 933 let Constraints = "$Pdn = $_Pdn";934 let Defs = [NZCV];935 let ElementSize = pprty.ElementSize;936 let hasSideEffects = 0;937 let isPTestLike = 1;938}939 940multiclass sve_int_pfirst<bits<5> opc, string asm, SDPatternOperator op> {941 def _B : sve_int_pfirst_next<0b01, opc, asm, PPR8>;942 943 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>;944}945 946multiclass sve_int_pnext<bits<5> opc, string asm, SDPatternOperator op> {947 def _B : sve_int_pfirst_next<0b00, opc, asm, PPR8>;948 def _H : sve_int_pfirst_next<0b01, opc, asm, PPR16>;949 def _S : sve_int_pfirst_next<0b10, opc, asm, PPR32>;950 def _D : sve_int_pfirst_next<0b11, opc, asm, PPR64>;951 952 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>;953 def : SVE_2_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>;954 def : SVE_2_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>;955 def : SVE_2_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>;956}957 958//===----------------------------------------------------------------------===//959// SVE Predicate Count Group960//===----------------------------------------------------------------------===//961 962class sve_int_count_r<bits<2> sz8_64, bits<5> opc, string asm,963 RegisterOperand dty, PPRRegOp pprty, RegisterOperand sty>964: I<(outs dty:$Rdn), (ins pprty:$Pg, sty:$_Rdn),965 asm, "\t$Rdn, $Pg",966 "",967 []>, Sched<[]> {968 bits<5> Rdn;969 bits<4> Pg;970 let Inst{31-24} = 0b00100101;971 let Inst{23-22} = sz8_64;972 let Inst{21-19} = 0b101;973 let Inst{18-16} = opc{4-2};974 let Inst{15-11} = 0b10001;975 let Inst{10-9} = opc{1-0};976 let Inst{8-5} = Pg;977 let Inst{4-0} = Rdn;978 979 // Signed 32bit forms require their GPR operand printed.980 let AsmString = !if(!eq(opc{4,2-0}, 0b0000),981 !strconcat(asm, "\t$Rdn, $Pg, $_Rdn"),982 !strconcat(asm, "\t$Rdn, $Pg"));983 let Constraints = "$Rdn = $_Rdn";984 let hasSideEffects = 0;985}986 987multiclass sve_int_count_r_s32<bits<5> opc, string asm,988 SDPatternOperator op> {989 def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64as32>;990 def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64as32>;991 def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64as32>;992 def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64as32>;993 994 def : Pat<(i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))),995 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _B) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>;996 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))))),997 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>;998 999 def : Pat<(i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))),1000 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _H) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>;1001 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))))),1002 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>;1003 1004 def : Pat<(i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))),1005 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _S) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>;1006 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))))),1007 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>;1008 1009 def : Pat<(i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))),1010 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _D) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>;1011 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))))),1012 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>;1013}1014 1015multiclass sve_int_count_r_u32<bits<5> opc, string asm,1016 SDPatternOperator op> {1017 def _B : sve_int_count_r<0b00, opc, asm, GPR32z, PPR8, GPR32z>;1018 def _H : sve_int_count_r<0b01, opc, asm, GPR32z, PPR16, GPR32z>;1019 def _S : sve_int_count_r<0b10, opc, asm, GPR32z, PPR32, GPR32z>;1020 def _D : sve_int_count_r<0b11, opc, asm, GPR32z, PPR64, GPR32z>;1021 1022 def : Pat<(i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))),1023 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, $Rn)>;1024 def : Pat<(i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))),1025 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, $Rn)>;1026 def : Pat<(i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))),1027 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, $Rn)>;1028 def : Pat<(i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))),1029 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, $Rn)>;1030}1031 1032multiclass sve_int_count_r_x64<bits<5> opc, string asm,1033 SDPatternOperator op,1034 SDPatternOperator combine_op = null_frag> {1035 def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64z>;1036 def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64z>;1037 def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64z>;1038 def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64z>;1039 1040 def : Pat<(i64 (op GPR64:$Rn, (nxv16i1 PPRAny:$Pg))),1041 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, $Rn)>;1042 def : Pat<(i64 (op GPR64:$Rn, (nxv8i1 PPRAny:$Pg))),1043 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, $Rn)>;1044 def : Pat<(i64 (op GPR64:$Rn, (nxv4i1 PPRAny:$Pg))),1045 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, $Rn)>;1046 def : Pat<(i64 (op GPR64:$Rn, (nxv2i1 PPRAny:$Pg))),1047 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, $Rn)>;1048 1049 // combine_op(x, cntp(all_active, p)) ==> inst p, x1050 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv16i1 (SVEAllActive)), (nxv16i1 PPRAny:$pred)))),1051 (!cast<Instruction>(NAME # _B) PPRAny:$pred, $Rn)>;1052 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv8i1 (SVEAllActive)), (nxv8i1 PPRAny:$pred)))),1053 (!cast<Instruction>(NAME # _H) PPRAny:$pred, $Rn)>;1054 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv4i1 (SVEAllActive)), (nxv4i1 PPRAny:$pred)))),1055 (!cast<Instruction>(NAME # _S) PPRAny:$pred, $Rn)>;1056 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv2i1 (SVEAllActive)), (nxv2i1 PPRAny:$pred)))),1057 (!cast<Instruction>(NAME # _D) PPRAny:$pred, $Rn)>;1058 1059 // combine_op(x, cntp(p, p)) ==> inst p, x1060 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv16i1 PPRAny:$pred), (nxv16i1 PPRAny:$pred)))),1061 (!cast<Instruction>(NAME # _B) PPRAny:$pred, $Rn)>;1062 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv8i1 PPRAny:$pred), (nxv8i1 PPRAny:$pred)))),1063 (!cast<Instruction>(NAME # _H) PPRAny:$pred, $Rn)>;1064 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv4i1 PPRAny:$pred), (nxv4i1 PPRAny:$pred)))),1065 (!cast<Instruction>(NAME # _S) PPRAny:$pred, $Rn)>;1066 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv2i1 PPRAny:$pred), (nxv2i1 PPRAny:$pred)))),1067 (!cast<Instruction>(NAME # _D) PPRAny:$pred, $Rn)>;1068 1069 // combine_op(x, trunc(cntp(all_active, p))) ==> inst p, x1070 def : Pat<(i32 (combine_op GPR32:$Rn, (trunc (int_aarch64_sve_cntp_oneuse (nxv16i1 (SVEAllActive)), (nxv16i1 PPRAny:$pred))))),1071 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _B) PPRAny:$pred,1072 (INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Rn, sub_32)),1073 sub_32)>;1074 def : Pat<(i32 (combine_op GPR32:$Rn, (trunc (int_aarch64_sve_cntp_oneuse (nxv8i1 (SVEAllActive)), (nxv8i1 PPRAny:$pred))))),1075 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _H) PPRAny:$pred,1076 (INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Rn, sub_32)),1077 sub_32)>;1078 def : Pat<(i32 (combine_op GPR32:$Rn, (trunc (int_aarch64_sve_cntp_oneuse (nxv4i1 (SVEAllActive)), (nxv4i1 PPRAny:$pred))))),1079 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _S) PPRAny:$pred,1080 (INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Rn, sub_32)),1081 sub_32)>;1082 def : Pat<(i32 (combine_op GPR32:$Rn, (trunc (int_aarch64_sve_cntp_oneuse (nxv2i1 (SVEAllActive)), (nxv2i1 PPRAny:$pred))))),1083 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _D) PPRAny:$pred,1084 (INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Rn, sub_32)),1085 sub_32)>;1086 1087 // combine_op(x, trunc(cntp(p, p))) ==> inst p, x1088 def : Pat<(i32 (combine_op GPR32:$Rn, (trunc (int_aarch64_sve_cntp_oneuse (nxv16i1 PPRAny:$pred), (nxv16i1 PPRAny:$pred))))),1089 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _B) PPRAny:$pred,1090 (INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Rn, sub_32)),1091 sub_32)>;1092 def : Pat<(i32 (combine_op GPR32:$Rn, (trunc (int_aarch64_sve_cntp_oneuse (nxv8i1 PPRAny:$pred), (nxv8i1 PPRAny:$pred))))),1093 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _H) PPRAny:$pred,1094 (INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Rn, sub_32)),1095 sub_32)>;1096 def : Pat<(i32 (combine_op GPR32:$Rn, (trunc (int_aarch64_sve_cntp_oneuse (nxv4i1 PPRAny:$pred), (nxv4i1 PPRAny:$pred))))),1097 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _S) PPRAny:$pred,1098 (INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Rn, sub_32)),1099 sub_32)>;1100 def : Pat<(i32 (combine_op GPR32:$Rn, (trunc (int_aarch64_sve_cntp_oneuse (nxv2i1 PPRAny:$pred), (nxv2i1 PPRAny:$pred))))),1101 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _D) PPRAny:$pred,1102 (INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Rn, sub_32)),1103 sub_32)>;1104}1105 1106class sve_int_count_v<bits<2> sz8_64, bits<5> opc, string asm,1107 ZPRRegOp zprty, PPRRegOp pprty>1108: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, pprty:$Pm),1109 asm, "\t$Zdn, $Pm",1110 "",1111 []>, Sched<[]> {1112 bits<4> Pm;1113 bits<5> Zdn;1114 let Inst{31-24} = 0b00100101;1115 let Inst{23-22} = sz8_64;1116 let Inst{21-19} = 0b101;1117 let Inst{18-16} = opc{4-2};1118 let Inst{15-11} = 0b10000;1119 let Inst{10-9} = opc{1-0};1120 let Inst{8-5} = Pm;1121 let Inst{4-0} = Zdn;1122 1123 let Constraints = "$Zdn = $_Zdn";1124 let DestructiveInstType = DestructiveOther;1125 let ElementSize = ElementSizeNone;1126 let hasSideEffects = 0;1127}1128 1129multiclass sve_int_count_v<bits<5> opc, string asm,1130 SDPatternOperator op = null_frag> {1131 def _H : sve_int_count_v<0b01, opc, asm, ZPR16, PPR16>;1132 def _S : sve_int_count_v<0b10, opc, asm, ZPR32, PPR32>;1133 def _D : sve_int_count_v<0b11, opc, asm, ZPR64, PPR64>;1134 1135 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, !cast<Instruction>(NAME # _H)>;1136 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, !cast<Instruction>(NAME # _S)>;1137 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, !cast<Instruction>(NAME # _D)>;1138 1139 def : InstAlias<asm # "\t$Zdn, $Pm",1140 (!cast<Instruction>(NAME # "_H") ZPR16:$Zdn, PPRAny:$Pm), 0>;1141 def : InstAlias<asm # "\t$Zdn, $Pm",1142 (!cast<Instruction>(NAME # "_S") ZPR32:$Zdn, PPRAny:$Pm), 0>;1143 def : InstAlias<asm # "\t$Zdn, $Pm",1144 (!cast<Instruction>(NAME # "_D") ZPR64:$Zdn, PPRAny:$Pm), 0>;1145}1146 1147class sve_int_pcount_pred<bits<2> sz8_64, bits<3> opc, string asm,1148 PPRRegOp pprty>1149: I<(outs GPR64:$Rd), (ins PPRAny:$Pg, pprty:$Pn),1150 asm, "\t$Rd, $Pg, $Pn",1151 "",1152 []>, Sched<[]> {1153 bits<4> Pg;1154 bits<4> Pn;1155 bits<5> Rd;1156 let Inst{31-24} = 0b00100101;1157 let Inst{23-22} = sz8_64;1158 let Inst{21-19} = 0b100;1159 let Inst{18-16} = opc{2-0};1160 let Inst{15-14} = 0b10;1161 let Inst{13-10} = Pg;1162 let Inst{9} = 0b0;1163 let Inst{8-5} = Pn;1164 let Inst{4-0} = Rd;1165 1166 let hasSideEffects = 0;1167}1168 1169multiclass sve_int_pcount_pred<bits<3> opc, string asm,1170 SDPatternOperator int_op> {1171 def _B : sve_int_pcount_pred<0b00, opc, asm, PPR8>;1172 def _H : sve_int_pcount_pred<0b01, opc, asm, PPR16>;1173 def _S : sve_int_pcount_pred<0b10, opc, asm, PPR32>;1174 def _D : sve_int_pcount_pred<0b11, opc, asm, PPR64>;1175 1176 def : SVE_2_Op_Pat<i64, int_op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>;1177 def : SVE_2_Op_Pat<i64, int_op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>;1178 def : SVE_2_Op_Pat<i64, int_op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>;1179 def : SVE_2_Op_Pat<i64, int_op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>;1180}1181 1182multiclass sve_int_pcount_pred_tmp<bits<3> opc, string asm> {1183 def _B : sve_int_pcount_pred<0b00, opc, asm, PPR8>;1184 def _H : sve_int_pcount_pred<0b01, opc, asm, PPR16>;1185 def _S : sve_int_pcount_pred<0b10, opc, asm, PPR32>;1186 def _D : sve_int_pcount_pred<0b11, opc, asm, PPR64>;1187}1188//===----------------------------------------------------------------------===//1189// SVE Element Count Group1190//===----------------------------------------------------------------------===//1191 1192class sve_int_count<bits<3> opc, string asm>1193: I<(outs GPR64:$Rd), (ins sve_pred_enum:$pattern, sve_incdec_imm:$imm4),1194 asm, "\t$Rd, $pattern, mul $imm4",1195 "",1196 []>, Sched<[]> {1197 bits<5> Rd;1198 bits<4> imm4;1199 bits<5> pattern;1200 let Inst{31-24} = 0b00000100;1201 let Inst{23-22} = opc{2-1};1202 let Inst{21-20} = 0b10;1203 let Inst{19-16} = imm4;1204 let Inst{15-11} = 0b11100;1205 let Inst{10} = opc{0};1206 let Inst{9-5} = pattern;1207 let Inst{4-0} = Rd;1208 1209 let hasSideEffects = 0;1210 let isReMaterializable = 1;1211 let Uses = [VG];1212}1213 1214multiclass sve_int_count<bits<3> opc, string asm, SDPatternOperator op> {1215 def NAME : sve_int_count<opc, asm>;1216 1217 def : InstAlias<asm # "\t$Rd, $pattern",1218 (!cast<Instruction>(NAME) GPR64:$Rd, sve_pred_enum:$pattern, 1), 1>;1219 def : InstAlias<asm # "\t$Rd",1220 (!cast<Instruction>(NAME) GPR64:$Rd, 0b11111, 1), 2>;1221 1222 def : Pat<(i64 (mul (op sve_pred_enum:$pattern), (sve_cnt_mul_imm_i64 i32:$imm))),1223 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>;1224 1225 def : Pat<(i64 (shl (op sve_pred_enum:$pattern), (sve_cnt_shl_imm i32:$imm))),1226 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>;1227 1228 def : Pat<(i64 (op sve_pred_enum:$pattern)),1229 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, 1)>;1230}1231 1232class sve_int_countvlv<bits<5> opc, string asm, ZPRRegOp zprty>1233: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4),1234 asm, "\t$Zdn, $pattern, mul $imm4",1235 "",1236 []>, Sched<[]> {1237 bits<5> Zdn;1238 bits<5> pattern;1239 bits<4> imm4;1240 let Inst{31-24} = 0b00000100;1241 let Inst{23-22} = opc{4-3};1242 let Inst{21} = 0b1;1243 let Inst{20} = opc{2};1244 let Inst{19-16} = imm4;1245 let Inst{15-12} = 0b1100;1246 let Inst{11-10} = opc{1-0};1247 let Inst{9-5} = pattern;1248 let Inst{4-0} = Zdn;1249 1250 let Constraints = "$Zdn = $_Zdn";1251 let DestructiveInstType = DestructiveOther;1252 let ElementSize = ElementSizeNone;1253 let hasSideEffects = 0;1254}1255 1256multiclass sve_int_countvlv<bits<5> opc, string asm, ZPRRegOp zprty,1257 SDPatternOperator op = null_frag,1258 ValueType vt = OtherVT> {1259 def NAME : sve_int_countvlv<opc, asm, zprty>;1260 1261 def : InstAlias<asm # "\t$Zdn, $pattern",1262 (!cast<Instruction>(NAME) zprty:$Zdn, sve_pred_enum:$pattern, 1), 1>;1263 def : InstAlias<asm # "\t$Zdn",1264 (!cast<Instruction>(NAME) zprty:$Zdn, 0b11111, 1), 2>;1265 1266 def : Pat<(vt (op (vt zprty:$Zn), (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))),1267 (!cast<Instruction>(NAME) $Zn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>;1268}1269 1270class sve_int_pred_pattern_a<bits<3> opc, string asm>1271: I<(outs GPR64:$Rdn), (ins GPR64:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4),1272 asm, "\t$Rdn, $pattern, mul $imm4",1273 "",1274 []>, Sched<[]> {1275 bits<5> Rdn;1276 bits<5> pattern;1277 bits<4> imm4;1278 let Inst{31-24} = 0b00000100;1279 let Inst{23-22} = opc{2-1};1280 let Inst{21-20} = 0b11;1281 let Inst{19-16} = imm4;1282 let Inst{15-11} = 0b11100;1283 let Inst{10} = opc{0};1284 let Inst{9-5} = pattern;1285 let Inst{4-0} = Rdn;1286 1287 let Constraints = "$Rdn = $_Rdn";1288 let hasSideEffects = 0;1289}1290 1291multiclass sve_int_pred_pattern_a<bits<3> opc, string asm,1292 SDPatternOperator op,1293 SDPatternOperator opcnt> {1294 let Predicates = [HasSVE_or_SME] in {1295 def NAME : sve_int_pred_pattern_a<opc, asm>;1296 1297 def : InstAlias<asm # "\t$Rdn, $pattern",1298 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, 1), 1>;1299 def : InstAlias<asm # "\t$Rdn",1300 (!cast<Instruction>(NAME) GPR64:$Rdn, 0b11111, 1), 2>;1301 }1302 1303 let Predicates = [HasSVE_or_SME, UseScalarIncVL] in {1304 def : Pat<(i64 (op GPR64:$Rdn, (opcnt sve_pred_enum:$pattern))),1305 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, 1)>;1306 1307 def : Pat<(i64 (op GPR64:$Rdn, (mul (opcnt sve_pred_enum:$pattern), (sve_cnt_mul_imm_i64 i32:$imm)))),1308 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, $imm)>;1309 1310 def : Pat<(i64 (op GPR64:$Rdn, (shl (opcnt sve_pred_enum:$pattern), (sve_cnt_shl_imm i32:$imm)))),1311 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, $imm)>;1312 1313 def : Pat<(i32 (op GPR32:$Rdn, (i32 (trunc (opcnt (sve_pred_enum:$pattern)))))),1314 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF),1315 GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, 1),1316 sub_32)>;1317 1318 def : Pat<(i32 (op GPR32:$Rdn, (mul (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (sve_cnt_mul_imm_i32 i32:$imm)))),1319 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF),1320 GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, $imm),1321 sub_32)>;1322 1323 def : Pat<(i32 (op GPR32:$Rdn, (shl (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (sve_cnt_shl_imm i32:$imm)))),1324 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF),1325 GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, $imm),1326 sub_32)>;1327 }1328}1329 1330class sve_int_pred_pattern_b<bits<5> opc, string asm, RegisterOperand dt,1331 RegisterOperand st>1332: I<(outs dt:$Rdn), (ins st:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4),1333 asm, "\t$Rdn, $pattern, mul $imm4",1334 "",1335 []>, Sched<[]> {1336 bits<5> Rdn;1337 bits<5> pattern;1338 bits<4> imm4;1339 let Inst{31-24} = 0b00000100;1340 let Inst{23-22} = opc{4-3};1341 let Inst{21} = 0b1;1342 let Inst{20} = opc{2};1343 let Inst{19-16} = imm4;1344 let Inst{15-12} = 0b1111;1345 let Inst{11-10} = opc{1-0};1346 let Inst{9-5} = pattern;1347 let Inst{4-0} = Rdn;1348 1349 // Signed 32bit forms require their GPR operand printed.1350 let AsmString = !if(!eq(opc{2,0}, 0b00),1351 !strconcat(asm, "\t$Rdn, $_Rdn, $pattern, mul $imm4"),1352 !strconcat(asm, "\t$Rdn, $pattern, mul $imm4"));1353 1354 let Constraints = "$Rdn = $_Rdn";1355 let hasSideEffects = 0;1356}1357 1358multiclass sve_int_pred_pattern_b_s32<bits<5> opc, string asm,1359 SDPatternOperator op> {1360 def NAME : sve_int_pred_pattern_b<opc, asm, GPR64z, GPR64as32>;1361 1362 def : InstAlias<asm # "\t$Rd, $Rn, $pattern",1363 (!cast<Instruction>(NAME) GPR64z:$Rd, GPR64as32:$Rn, sve_pred_enum:$pattern, 1), 1>;1364 def : InstAlias<asm # "\t$Rd, $Rn",1365 (!cast<Instruction>(NAME) GPR64z:$Rd, GPR64as32:$Rn, 0b11111, 1), 2>;1366 1367 // NOTE: Register allocation doesn't like tied operands of differing register1368 // class, hence the extra INSERT_SUBREG complication.1369 1370 def : Pat<(i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))),1371 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32), sve_pred_enum:$pattern, sve_incdec_imm:$imm4), sub_32)>;1372 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))))),1373 (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32), sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>;1374}1375 1376multiclass sve_int_pred_pattern_b_u32<bits<5> opc, string asm,1377 SDPatternOperator op> {1378 def NAME : sve_int_pred_pattern_b<opc, asm, GPR32z, GPR32z>;1379 1380 def : InstAlias<asm # "\t$Rdn, $pattern",1381 (!cast<Instruction>(NAME) GPR32z:$Rdn, sve_pred_enum:$pattern, 1), 1>;1382 def : InstAlias<asm # "\t$Rdn",1383 (!cast<Instruction>(NAME) GPR32z:$Rdn, 0b11111, 1), 2>;1384 1385 def : Pat<(i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))),1386 (!cast<Instruction>(NAME) $Rn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>;1387}1388 1389multiclass sve_int_pred_pattern_b_x64<bits<5> opc, string asm,1390 SDPatternOperator op> {1391 def NAME : sve_int_pred_pattern_b<opc, asm, GPR64z, GPR64z>;1392 1393 def : InstAlias<asm # "\t$Rdn, $pattern",1394 (!cast<Instruction>(NAME) GPR64z:$Rdn, sve_pred_enum:$pattern, 1), 1>;1395 def : InstAlias<asm # "\t$Rdn",1396 (!cast<Instruction>(NAME) GPR64z:$Rdn, 0b11111, 1), 2>;1397 1398 def : Pat<(i64 (op GPR64:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))),1399 (!cast<Instruction>(NAME) $Rn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>;1400}1401 1402 1403//===----------------------------------------------------------------------===//1404// SVE Permute - Cross Lane Group1405//===----------------------------------------------------------------------===//1406 1407class sve_int_perm_dup_r<bits<2> sz8_64, string asm, ZPRRegOp zprty,1408 ValueType vt, RegisterClass srcRegType,1409 SDPatternOperator op>1410: I<(outs zprty:$Zd), (ins srcRegType:$Rn),1411 asm, "\t$Zd, $Rn",1412 "",1413 [(set (vt zprty:$Zd), (op srcRegType:$Rn))]>, Sched<[]> {1414 bits<5> Rn;1415 bits<5> Zd;1416 let Inst{31-24} = 0b00000101;1417 let Inst{23-22} = sz8_64;1418 let Inst{21-10} = 0b100000001110;1419 let Inst{9-5} = Rn;1420 let Inst{4-0} = Zd;1421 1422 let hasSideEffects = 0;1423}1424 1425multiclass sve_int_perm_dup_r<string asm, SDPatternOperator op> {1426 def _B : sve_int_perm_dup_r<0b00, asm, ZPR8, nxv16i8, GPR32sp, op>;1427 def _H : sve_int_perm_dup_r<0b01, asm, ZPR16, nxv8i16, GPR32sp, op>;1428 def _S : sve_int_perm_dup_r<0b10, asm, ZPR32, nxv4i32, GPR32sp, op>;1429 def _D : sve_int_perm_dup_r<0b11, asm, ZPR64, nxv2i64, GPR64sp, op>;1430 1431 def : InstAlias<"mov $Zd, $Rn",1432 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, GPR32sp:$Rn), 1>;1433 def : InstAlias<"mov $Zd, $Rn",1434 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, GPR32sp:$Rn), 1>;1435 def : InstAlias<"mov $Zd, $Rn",1436 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, GPR32sp:$Rn), 1>;1437 def : InstAlias<"mov $Zd, $Rn",1438 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, GPR64sp:$Rn), 1>;1439}1440 1441class sve_int_perm_dup_i<bits<5> tsz, Operand immtype, string asm,1442 ZPRRegOp zprty>1443: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$idx),1444 asm, "\t$Zd, $Zn$idx",1445 "",1446 []>, Sched<[]> {1447 bits<5> Zd;1448 bits<5> Zn;1449 bits<7> idx;1450 let Inst{31-24} = 0b00000101;1451 let Inst{23-22} = {?,?}; // imm3h1452 let Inst{21} = 0b1;1453 let Inst{20-16} = tsz;1454 let Inst{15-10} = 0b001000;1455 let Inst{9-5} = Zn;1456 let Inst{4-0} = Zd;1457 1458 let hasSideEffects = 0;1459}1460 1461multiclass sve_int_perm_dup_i<string asm> {1462 def _B : sve_int_perm_dup_i<{?,?,?,?,1}, sve_elm_idx_extdup_b, asm, ZPR8> {1463 let Inst{23-22} = idx{5-4};1464 let Inst{20-17} = idx{3-0};1465 }1466 def _H : sve_int_perm_dup_i<{?,?,?,1,0}, sve_elm_idx_extdup_h, asm, ZPR16> {1467 let Inst{23-22} = idx{4-3};1468 let Inst{20-18} = idx{2-0};1469 }1470 def _S : sve_int_perm_dup_i<{?,?,1,0,0}, sve_elm_idx_extdup_s, asm, ZPR32> {1471 let Inst{23-22} = idx{3-2};1472 let Inst{20-19} = idx{1-0};1473 }1474 def _D : sve_int_perm_dup_i<{?,1,0,0,0}, sve_elm_idx_extdup_d, asm, ZPR64> {1475 let Inst{23-22} = idx{2-1};1476 let Inst{20} = idx{0};1477 }1478 def _Q : sve_int_perm_dup_i<{1,0,0,0,0}, sve_elm_idx_extdup_q, asm, ZPR128> {1479 let Inst{23-22} = idx{1-0};1480 }1481 1482 def : InstAlias<"mov $Zd, $Zn$idx",1483 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, ZPR8:$Zn, sve_elm_idx_extdup_b:$idx), 1>;1484 def : InstAlias<"mov $Zd, $Zn$idx",1485 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, ZPR16:$Zn, sve_elm_idx_extdup_h:$idx), 1>;1486 def : InstAlias<"mov $Zd, $Zn$idx",1487 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, ZPR32:$Zn, sve_elm_idx_extdup_s:$idx), 1>;1488 def : InstAlias<"mov $Zd, $Zn$idx",1489 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, ZPR64:$Zn, sve_elm_idx_extdup_d:$idx), 1>;1490 def : InstAlias<"mov $Zd, $Zn$idx",1491 (!cast<Instruction>(NAME # _Q) ZPR128:$Zd, ZPR128:$Zn, sve_elm_idx_extdup_q:$idx), 1>;1492 def : InstAlias<"mov $Zd, $Bn",1493 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, FPR8asZPR:$Bn, 0), 2>;1494 def : InstAlias<"mov $Zd, $Hn",1495 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, FPR16asZPR:$Hn, 0), 2>;1496 def : InstAlias<"mov $Zd, $Sn",1497 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, FPR32asZPR:$Sn, 0), 2>;1498 def : InstAlias<"mov $Zd, $Dn",1499 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, FPR64asZPR:$Dn, 0), 2>;1500 def : InstAlias<"mov $Zd, $Qn",1501 (!cast<Instruction>(NAME # _Q) ZPR128:$Zd, FPR128asZPR:$Qn, 0), 2>;1502 1503 // Duplicate an extracted vector element across a vector.1504 1505 def : Pat<(nxv16i8 (splat_vector (i32 (vector_extract (nxv16i8 ZPR:$vec), sve_elm_idx_extdup_b:$index)))),1506 (!cast<Instruction>(NAME # _B) ZPR:$vec, sve_elm_idx_extdup_b:$index)>;1507 def : Pat<(nxv16i8 (splat_vector (i32 (vector_extract (v16i8 V128:$vec), sve_elm_idx_extdup_b:$index)))),1508 (!cast<Instruction>(NAME # _B) (SUBREG_TO_REG (i64 0), $vec, zsub), sve_elm_idx_extdup_b:$index)>;1509 def : Pat<(nxv16i8 (splat_vector (i32 (vector_extract (v8i8 V64:$vec), sve_elm_idx_extdup_b:$index)))),1510 (!cast<Instruction>(NAME # _B) (SUBREG_TO_REG (i64 0), $vec, dsub), sve_elm_idx_extdup_b:$index)>;1511 1512 foreach VT = [nxv8i16, nxv2f16, nxv4f16, nxv8f16, nxv2bf16, nxv4bf16, nxv8bf16] in {1513 def : Pat<(VT (splat_vector (SVEType<VT>.EltAsScalar (vector_extract (SVEType<VT>.Packed ZPR:$vec), sve_elm_idx_extdup_h:$index)))),1514 (!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>;1515 def : Pat<(VT (splat_vector (SVEType<VT>.EltAsScalar (vector_extract (SVEType<VT>.ZSub V128:$vec), sve_elm_idx_extdup_h:$index)))),1516 (!cast<Instruction>(NAME # _H) (SUBREG_TO_REG (i64 0), $vec, zsub), sve_elm_idx_extdup_h:$index)>;1517 def : Pat<(VT (splat_vector (SVEType<VT>.EltAsScalar (vector_extract (SVEType<VT>.DSub V64:$vec), sve_elm_idx_extdup_h:$index)))),1518 (!cast<Instruction>(NAME # _H) (SUBREG_TO_REG (i64 0), $vec, dsub), sve_elm_idx_extdup_h:$index)>;1519 }1520 1521 foreach VT = [nxv4i32, nxv2f32, nxv4f32 ] in {1522 def : Pat<(VT (splat_vector (SVEType<VT>.EltAsScalar (vector_extract (SVEType<VT>.Packed ZPR:$vec), sve_elm_idx_extdup_s:$index)))),1523 (!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>;1524 def : Pat<(VT (splat_vector (SVEType<VT>.EltAsScalar (vector_extract (SVEType<VT>.ZSub V128:$vec), sve_elm_idx_extdup_s:$index)))),1525 (!cast<Instruction>(NAME # _S) (SUBREG_TO_REG (i64 0), $vec, zsub), sve_elm_idx_extdup_s:$index)>;1526 def : Pat<(VT (splat_vector (SVEType<VT>.EltAsScalar (vector_extract (SVEType<VT>.DSub V64:$vec), sve_elm_idx_extdup_s:$index)))),1527 (!cast<Instruction>(NAME # _S) (SUBREG_TO_REG (i64 0), $vec, dsub), sve_elm_idx_extdup_s:$index)>;1528 }1529 1530 foreach VT = [nxv2i64, nxv2f64] in {1531 def : Pat<(VT (splat_vector (SVEType<VT>.EltAsScalar (vector_extract (VT ZPR:$vec), sve_elm_idx_extdup_d:$index)))),1532 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>;1533 def : Pat<(VT (splat_vector (SVEType<VT>.EltAsScalar (vector_extract (SVEType<VT>.ZSub V128:$vec), sve_elm_idx_extdup_d:$index)))),1534 (!cast<Instruction>(NAME # _D) (SUBREG_TO_REG (i64 0), $vec, zsub), sve_elm_idx_extdup_d:$index)>;1535 def : Pat<(VT (splat_vector (SVEType<VT>.EltAsScalar (vector_extract (SVEType<VT>.DSub V64:$vec), sve_elm_idx_extdup_d:$index)))),1536 (!cast<Instruction>(NAME # _D) (SUBREG_TO_REG (i64 0), $vec, dsub), sve_elm_idx_extdup_d:$index)>;1537 }1538 1539 // When extracting from an unpacked vector the index must be scaled to account1540 // for the "holes" in the underlying packed vector type. We get the scaling1541 // for free by "promoting" the element type to one whose underlying vector1542 // type is packed. This is only valid when extracting from a vector whose1543 // length is the same or bigger than the result of the splat.1544 1545 foreach VT = [nxv4f16, nxv4bf16] in {1546 def : Pat<(SVEType<VT>.HalfLength (splat_vector (SVEType<VT>.EltAsScalar (vector_extract (VT ZPR:$vec), sve_elm_idx_extdup_s:$index)))),1547 (!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>;1548 def : Pat<(VT (splat_vector (SVEType<VT>.EltAsScalar (vector_extract (VT ZPR:$vec), sve_elm_idx_extdup_s:$index)))),1549 (!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>;1550 }1551 1552 foreach VT = [nxv2f16, nxv2f32, nxv2bf16] in {1553 def : Pat<(VT (splat_vector (SVEType<VT>.EltAsScalar (vector_extract (VT ZPR:$vec), sve_elm_idx_extdup_d:$index)))),1554 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>;1555 }1556 1557 // Duplicate an indexed 128-bit segment across a vector.1558 1559 def : Pat<(nxv16i8 (AArch64duplane128 nxv16i8:$Op1, i64:$imm)),1560 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>;1561 def : Pat<(nxv8i16 (AArch64duplane128 nxv8i16:$Op1, i64:$imm)),1562 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>;1563 def : Pat<(nxv4i32 (AArch64duplane128 nxv4i32:$Op1, i64:$imm)),1564 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>;1565 def : Pat<(nxv2i64 (AArch64duplane128 nxv2i64:$Op1, i64:$imm)),1566 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>;1567 def : Pat<(nxv8f16 (AArch64duplane128 nxv8f16:$Op1, i64:$imm)),1568 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>;1569 def : Pat<(nxv4f32 (AArch64duplane128 nxv4f32:$Op1, i64:$imm)),1570 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>;1571 def : Pat<(nxv2f64 (AArch64duplane128 nxv2f64:$Op1, i64:$imm)),1572 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>;1573 def : Pat<(nxv8bf16 (AArch64duplane128 nxv8bf16:$Op1, i64:$imm)),1574 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>;1575}1576 1577class sve_int_perm_tbl<bits<2> sz8_64, bits<2> opc, string asm, ZPRRegOp zprty,1578 RegisterOperand VecList>1579: I<(outs zprty:$Zd), (ins VecList:$Zn, zprty:$Zm),1580 asm, "\t$Zd, $Zn, $Zm",1581 "",1582 []>, Sched<[]> {1583 bits<5> Zd;1584 bits<5> Zm;1585 bits<5> Zn;1586 let Inst{31-24} = 0b00000101;1587 let Inst{23-22} = sz8_64;1588 let Inst{21} = 0b1;1589 let Inst{20-16} = Zm;1590 let Inst{15-13} = 0b001;1591 let Inst{12-11} = opc;1592 let Inst{10} = 0b0;1593 let Inst{9-5} = Zn;1594 let Inst{4-0} = Zd;1595 1596 let hasSideEffects = 0;1597}1598 1599multiclass sve_int_perm_tbl<string asm, SDPatternOperator op> {1600 def _B : sve_int_perm_tbl<0b00, 0b10, asm, ZPR8, Z_b>;1601 def _H : sve_int_perm_tbl<0b01, 0b10, asm, ZPR16, Z_h>;1602 def _S : sve_int_perm_tbl<0b10, 0b10, asm, ZPR32, Z_s>;1603 def _D : sve_int_perm_tbl<0b11, 0b10, asm, ZPR64, Z_d>;1604 1605 def : InstAlias<asm # "\t$Zd, $Zn, $Zm",1606 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, ZPR8:$Zn, ZPR8:$Zm), 0>;1607 def : InstAlias<asm # "\t$Zd, $Zn, $Zm",1608 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, ZPR16:$Zn, ZPR16:$Zm), 0>;1609 def : InstAlias<asm # "\t$Zd, $Zn, $Zm",1610 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, ZPR32:$Zn, ZPR32:$Zm), 0>;1611 def : InstAlias<asm # "\t$Zd, $Zn, $Zm",1612 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, ZPR64:$Zn, ZPR64:$Zm), 0>;1613 1614 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;1615 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;1616 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;1617 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;1618 1619 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>;1620 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>;1621 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>;1622 1623 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i16, !cast<Instruction>(NAME # _H)>;1624}1625 1626multiclass sve2_int_perm_tbl<string asm, SDPatternOperator op> {1627 def _B : sve_int_perm_tbl<0b00, 0b01, asm, ZPR8, ZZ_b>;1628 def _H : sve_int_perm_tbl<0b01, 0b01, asm, ZPR16, ZZ_h>;1629 def _S : sve_int_perm_tbl<0b10, 0b01, asm, ZPR32, ZZ_s>;1630 def _D : sve_int_perm_tbl<0b11, 0b01, asm, ZPR64, ZZ_d>;1631 1632 def : Pat<(nxv16i8 (op nxv16i8:$Op1, nxv16i8:$Op2, nxv16i8:$Op3)),1633 (nxv16i8 (!cast<Instruction>(NAME # _B) (REG_SEQUENCE ZPR2, nxv16i8:$Op1, zsub0,1634 nxv16i8:$Op2, zsub1),1635 nxv16i8:$Op3))>;1636 1637 def : Pat<(nxv8i16 (op nxv8i16:$Op1, nxv8i16:$Op2, nxv8i16:$Op3)),1638 (nxv8i16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8i16:$Op1, zsub0,1639 nxv8i16:$Op2, zsub1),1640 nxv8i16:$Op3))>;1641 1642 def : Pat<(nxv4i32 (op nxv4i32:$Op1, nxv4i32:$Op2, nxv4i32:$Op3)),1643 (nxv4i32 (!cast<Instruction>(NAME # _S) (REG_SEQUENCE ZPR2, nxv4i32:$Op1, zsub0,1644 nxv4i32:$Op2, zsub1),1645 nxv4i32:$Op3))>;1646 1647 def : Pat<(nxv2i64 (op nxv2i64:$Op1, nxv2i64:$Op2, nxv2i64:$Op3)),1648 (nxv2i64 (!cast<Instruction>(NAME # _D) (REG_SEQUENCE ZPR2, nxv2i64:$Op1, zsub0,1649 nxv2i64:$Op2, zsub1),1650 nxv2i64:$Op3))>;1651 1652 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8i16:$Op3)),1653 (nxv8f16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8f16:$Op1, zsub0,1654 nxv8f16:$Op2, zsub1),1655 nxv8i16:$Op3))>;1656 1657 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4i32:$Op3)),1658 (nxv4f32 (!cast<Instruction>(NAME # _S) (REG_SEQUENCE ZPR2, nxv4f32:$Op1, zsub0,1659 nxv4f32:$Op2, zsub1),1660 nxv4i32:$Op3))>;1661 1662 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, nxv2i64:$Op3)),1663 (nxv2f64 (!cast<Instruction>(NAME # _D) (REG_SEQUENCE ZPR2, nxv2f64:$Op1, zsub0,1664 nxv2f64:$Op2, zsub1),1665 nxv2i64:$Op3))>;1666 1667 def : Pat<(nxv8bf16 (op nxv8bf16:$Op1, nxv8bf16:$Op2, nxv8i16:$Op3)),1668 (nxv8bf16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8bf16:$Op1, zsub0,1669 nxv8bf16:$Op2, zsub1),1670 nxv8i16:$Op3))>;1671}1672 1673class sve2_int_perm_tbx<bits<2> sz8_64, bits<2> opc, string asm, ZPRRegOp zprty>1674: I<(outs zprty:$Zd), (ins zprty:$_Zd, zprty:$Zn, zprty:$Zm),1675 asm, "\t$Zd, $Zn, $Zm",1676 "",1677 []>, Sched<[]> {1678 bits<5> Zd;1679 bits<5> Zm;1680 bits<5> Zn;1681 let Inst{31-24} = 0b00000101;1682 let Inst{23-22} = sz8_64;1683 let Inst{21} = 0b1;1684 let Inst{20-16} = Zm;1685 let Inst{15-13} = 0b001;1686 let Inst{12-11} = opc;1687 let Inst{10} = 0b1;1688 let Inst{9-5} = Zn;1689 let Inst{4-0} = Zd;1690 1691 let Constraints = "$Zd = $_Zd";1692 let hasSideEffects = 0;1693}1694 1695multiclass sve2_int_perm_tbx<string asm, bits<2> opc, SDPatternOperator op> {1696 def _B : sve2_int_perm_tbx<0b00, opc, asm, ZPR8>;1697 def _H : sve2_int_perm_tbx<0b01, opc, asm, ZPR16>;1698 def _S : sve2_int_perm_tbx<0b10, opc, asm, ZPR32>;1699 def _D : sve2_int_perm_tbx<0b11, opc, asm, ZPR64>;1700 1701 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;1702 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;1703 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;1704 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;1705 1706 def : SVE_3_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>;1707 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>;1708 def : SVE_3_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>;1709 1710 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, nxv8i16, !cast<Instruction>(NAME # _H)>;1711}1712 1713class sve_int_perm_reverse_z<bits<2> sz8_64, string asm, ZPRRegOp zprty>1714: I<(outs zprty:$Zd), (ins zprty:$Zn),1715 asm, "\t$Zd, $Zn",1716 "",1717 []>, Sched<[]> {1718 bits<5> Zd;1719 bits<5> Zn;1720 let Inst{31-24} = 0b00000101;1721 let Inst{23-22} = sz8_64;1722 let Inst{21-10} = 0b111000001110;1723 let Inst{9-5} = Zn;1724 let Inst{4-0} = Zd;1725 1726 let hasSideEffects = 0;1727}1728 1729multiclass sve_int_perm_reverse_z<string asm, SDPatternOperator op> {1730 def _B : sve_int_perm_reverse_z<0b00, asm, ZPR8>;1731 def _H : sve_int_perm_reverse_z<0b01, asm, ZPR16>;1732 def _S : sve_int_perm_reverse_z<0b10, asm, ZPR32>;1733 def _D : sve_int_perm_reverse_z<0b11, asm, ZPR64>;1734 1735 def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME # _B)>;1736 def : SVE_1_Op_Pat<nxv8i16, op, nxv8i16, !cast<Instruction>(NAME # _H)>;1737 def : SVE_1_Op_Pat<nxv4i32, op, nxv4i32, !cast<Instruction>(NAME # _S)>;1738 def : SVE_1_Op_Pat<nxv2i64, op, nxv2i64, !cast<Instruction>(NAME # _D)>;1739 1740 def : SVE_1_Op_Pat<nxv2f16, op, nxv2f16, !cast<Instruction>(NAME # _D)>;1741 def : SVE_1_Op_Pat<nxv4f16, op, nxv4f16, !cast<Instruction>(NAME # _S)>;1742 def : SVE_1_Op_Pat<nxv8f16, op, nxv8f16, !cast<Instruction>(NAME # _H)>;1743 def : SVE_1_Op_Pat<nxv2f32, op, nxv2f32, !cast<Instruction>(NAME # _D)>;1744 def : SVE_1_Op_Pat<nxv4f32, op, nxv4f32, !cast<Instruction>(NAME # _S)>;1745 def : SVE_1_Op_Pat<nxv2f64, op, nxv2f64, !cast<Instruction>(NAME # _D)>;1746 1747 def : SVE_1_Op_Pat<nxv2bf16, op, nxv2bf16, !cast<Instruction>(NAME # _D)>;1748 def : SVE_1_Op_Pat<nxv4bf16, op, nxv4bf16, !cast<Instruction>(NAME # _S)>;1749 def : SVE_1_Op_Pat<nxv8bf16, op, nxv8bf16, !cast<Instruction>(NAME # _H)>;1750}1751 1752class sve_int_perm_reverse_p<bits<2> sz8_64, string asm, PPRRegOp pprty,1753 SDPatternOperator op>1754: I<(outs pprty:$Pd), (ins pprty:$Pn),1755 asm, "\t$Pd, $Pn",1756 "",1757 [(set nxv16i1:$Pd, (op nxv16i1:$Pn))]>, Sched<[]> {1758 bits<4> Pd;1759 bits<4> Pn;1760 let Inst{31-24} = 0b00000101;1761 let Inst{23-22} = sz8_64;1762 let Inst{21-9} = 0b1101000100000;1763 let Inst{8-5} = Pn;1764 let Inst{4} = 0b0;1765 let Inst{3-0} = Pd;1766 1767 let hasSideEffects = 0;1768}1769 1770multiclass sve_int_perm_reverse_p<string asm, SDPatternOperator ir_op,1771 SDPatternOperator op_b16,1772 SDPatternOperator op_b32,1773 SDPatternOperator op_b64> {1774 def _B : sve_int_perm_reverse_p<0b00, asm, PPR8, ir_op>;1775 def _H : sve_int_perm_reverse_p<0b01, asm, PPR16, op_b16>;1776 def _S : sve_int_perm_reverse_p<0b10, asm, PPR32, op_b32>;1777 def _D : sve_int_perm_reverse_p<0b11, asm, PPR64, op_b64>;1778 1779 def : SVE_1_Op_Pat<nxv8i1, ir_op, nxv8i1, !cast<Instruction>(NAME # _H)>;1780 def : SVE_1_Op_Pat<nxv4i1, ir_op, nxv4i1, !cast<Instruction>(NAME # _S)>;1781 def : SVE_1_Op_Pat<nxv2i1, ir_op, nxv2i1, !cast<Instruction>(NAME # _D)>;1782}1783 1784class sve_int_perm_unpk<bits<2> sz16_64, bits<2> opc, string asm,1785 ZPRRegOp zprty1, ZPRRegOp zprty2>1786: I<(outs zprty1:$Zd), (ins zprty2:$Zn),1787 asm, "\t$Zd, $Zn",1788 "", []>, Sched<[]> {1789 bits<5> Zd;1790 bits<5> Zn;1791 let Inst{31-24} = 0b00000101;1792 let Inst{23-22} = sz16_64;1793 let Inst{21-18} = 0b1100;1794 let Inst{17-16} = opc;1795 let Inst{15-10} = 0b001110;1796 let Inst{9-5} = Zn;1797 let Inst{4-0} = Zd;1798 1799 let hasSideEffects = 0;1800}1801 1802multiclass sve_int_perm_unpk<bits<2> opc, string asm, SDPatternOperator op> {1803 def _H : sve_int_perm_unpk<0b01, opc, asm, ZPR16, ZPR8>;1804 def _S : sve_int_perm_unpk<0b10, opc, asm, ZPR32, ZPR16>;1805 def _D : sve_int_perm_unpk<0b11, opc, asm, ZPR64, ZPR32>;1806 1807 def : SVE_1_Op_Pat<nxv8i16, op, nxv16i8, !cast<Instruction>(NAME # _H)>;1808 def : SVE_1_Op_Pat<nxv4i32, op, nxv8i16, !cast<Instruction>(NAME # _S)>;1809 def : SVE_1_Op_Pat<nxv2i64, op, nxv4i32, !cast<Instruction>(NAME # _D)>;1810}1811 1812class sve_int_perm_insrs<bits<2> sz8_64, string asm, ZPRRegOp zprty,1813 RegisterClass srcRegType>1814: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcRegType:$Rm),1815 asm, "\t$Zdn, $Rm",1816 "",1817 []>, Sched<[]> {1818 bits<5> Rm;1819 bits<5> Zdn;1820 let Inst{31-24} = 0b00000101;1821 let Inst{23-22} = sz8_64;1822 let Inst{21-10} = 0b100100001110;1823 let Inst{9-5} = Rm;1824 let Inst{4-0} = Zdn;1825 1826 let Constraints = "$Zdn = $_Zdn";1827 let DestructiveInstType = DestructiveOther;1828 let hasSideEffects = 0;1829}1830 1831multiclass sve_int_perm_insrs<string asm, SDPatternOperator op> {1832 def _B : sve_int_perm_insrs<0b00, asm, ZPR8, GPR32>;1833 def _H : sve_int_perm_insrs<0b01, asm, ZPR16, GPR32>;1834 def _S : sve_int_perm_insrs<0b10, asm, ZPR32, GPR32>;1835 def _D : sve_int_perm_insrs<0b11, asm, ZPR64, GPR64>;1836 1837 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, i32, !cast<Instruction>(NAME # _B)>;1838 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, i32, !cast<Instruction>(NAME # _H)>;1839 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, i32, !cast<Instruction>(NAME # _S)>;1840 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, i64, !cast<Instruction>(NAME # _D)>;1841}1842 1843class sve_int_perm_insrv<bits<2> sz8_64, string asm, ZPRRegOp zprty,1844 FPRasZPROperand srcOpType>1845: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcOpType:$Vm),1846 asm, "\t$Zdn, $Vm",1847 "",1848 []>, Sched<[]> {1849 bits<5> Vm;1850 bits<5> Zdn;1851 let Inst{31-24} = 0b00000101;1852 let Inst{23-22} = sz8_64;1853 let Inst{21-10} = 0b110100001110;1854 let Inst{9-5} = Vm;1855 let Inst{4-0} = Zdn;1856 1857 let Constraints = "$Zdn = $_Zdn";1858 let DestructiveInstType = DestructiveOther;1859 let hasSideEffects = 0;1860}1861 1862multiclass sve_int_perm_insrv<string asm, SDPatternOperator op> {1863 def _B : sve_int_perm_insrv<0b00, asm, ZPR8, FPR8asZPR>;1864 def _H : sve_int_perm_insrv<0b01, asm, ZPR16, FPR16asZPR>;1865 def _S : sve_int_perm_insrv<0b10, asm, ZPR32, FPR32asZPR>;1866 def _D : sve_int_perm_insrv<0b11, asm, ZPR64, FPR64asZPR>;1867 1868 def : Pat<(nxv8f16 (op nxv8f16:$Zn, f16:$Vm)),1869 (!cast<Instruction>(NAME # _H) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, hsub))>;1870 def : Pat<(nxv4f32 (op nxv4f32:$Zn, f32:$Vm)),1871 (!cast<Instruction>(NAME # _S) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, ssub))>;1872 def : Pat<(nxv2f64 (op nxv2f64:$Zn, f64:$Vm)),1873 (!cast<Instruction>(NAME # _D) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, dsub))>;1874 1875 def : Pat<(nxv8bf16 (op nxv8bf16:$Zn, bf16:$Vm)),1876 (!cast<Instruction>(NAME # _H) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, hsub))>;1877 1878 // Keep integer insertions within the vector unit.1879 def : Pat<(nxv16i8 (op (nxv16i8 ZPR:$Zn), (i32 (vector_extract (nxv16i8 ZPR:$Vm), 0)))),1880 (!cast<Instruction>(NAME # _B) $Zn, ZPR:$Vm)>;1881 def : Pat<(nxv8i16 (op (nxv8i16 ZPR:$Zn), (i32 (vector_extract (nxv8i16 ZPR:$Vm), 0)))),1882 (!cast<Instruction>(NAME # _H) $Zn, ZPR:$Vm)>;1883 def : Pat<(nxv4i32 (op (nxv4i32 ZPR:$Zn), (i32 (vector_extract (nxv4i32 ZPR:$Vm), 0)))),1884 (!cast<Instruction>(NAME # _S) $Zn, ZPR: $Vm)>;1885 def : Pat<(nxv2i64 (op (nxv2i64 ZPR:$Zn), (i64 (vector_extract (nxv2i64 ZPR:$Vm), 0)))),1886 (!cast<Instruction>(NAME # _D) $Zn, ZPR:$Vm)>;1887 1888}1889 1890//===----------------------------------------------------------------------===//1891// SVE Permute - Extract Group1892//===----------------------------------------------------------------------===//1893 1894class sve_int_perm_extract_i<string asm>1895: I<(outs ZPR8:$Zdn), (ins ZPR8:$_Zdn, ZPR8:$Zm, imm0_255:$imm8),1896 asm, "\t$Zdn, $_Zdn, $Zm, $imm8",1897 "", []>, Sched<[]> {1898 bits<5> Zdn;1899 bits<5> Zm;1900 bits<8> imm8;1901 let Inst{31-21} = 0b00000101001;1902 let Inst{20-16} = imm8{7-3};1903 let Inst{15-13} = 0b000;1904 let Inst{12-10} = imm8{2-0};1905 let Inst{9-5} = Zm;1906 let Inst{4-0} = Zdn;1907 1908 let Constraints = "$Zdn = $_Zdn";1909 let DestructiveInstType = Destructive2xRegImmUnpred;1910 let ElementSize = ElementSizeNone;1911 let hasSideEffects = 0;1912}1913 1914multiclass sve_int_perm_extract_i<string asm, SDPatternOperator op, string Ps> {1915 def NAME : sve_int_perm_extract_i<asm>,1916 SVEPseudo2Instr<Ps, 1>;1917 1918 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, imm0_255,1919 !cast<Instruction>(NAME)>;1920}1921 1922class sve2_int_perm_extract_i_cons<string asm>1923: I<(outs ZPR8:$Zd), (ins ZZ_b:$Zn, imm0_255:$imm8),1924 asm, "\t$Zd, $Zn, $imm8",1925 "", []>, Sched<[]> {1926 bits<5> Zd;1927 bits<5> Zn;1928 bits<8> imm8;1929 let Inst{31-21} = 0b00000101011;1930 let Inst{20-16} = imm8{7-3};1931 let Inst{15-13} = 0b000;1932 let Inst{12-10} = imm8{2-0};1933 let Inst{9-5} = Zn;1934 let Inst{4-0} = Zd;1935 1936 let hasSideEffects = 0;1937}1938 1939//===----------------------------------------------------------------------===//1940// SVE Vector Select Group1941//===----------------------------------------------------------------------===//1942 1943class sve_int_sel_vvv<bits<2> sz8_64, string asm, ZPRRegOp zprty>1944: I<(outs zprty:$Zd), (ins PPRAny:$Pg, zprty:$Zn, zprty:$Zm),1945 asm, "\t$Zd, $Pg, $Zn, $Zm",1946 "",1947 []>, Sched<[]> {1948 bits<4> Pg;1949 bits<5> Zd;1950 bits<5> Zm;1951 bits<5> Zn;1952 let Inst{31-24} = 0b00000101;1953 let Inst{23-22} = sz8_64;1954 let Inst{21} = 0b1;1955 let Inst{20-16} = Zm;1956 let Inst{15-14} = 0b11;1957 let Inst{13-10} = Pg;1958 let Inst{9-5} = Zn;1959 let Inst{4-0} = Zd;1960 1961 let hasSideEffects = 0;1962}1963 1964multiclass sve_int_sel_vvv<string asm, SDPatternOperator op> {1965 def _B : sve_int_sel_vvv<0b00, asm, ZPR8>;1966 def _H : sve_int_sel_vvv<0b01, asm, ZPR16>;1967 def _S : sve_int_sel_vvv<0b10, asm, ZPR32>;1968 def _D : sve_int_sel_vvv<0b11, asm, ZPR64>;1969 1970 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;1971 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;1972 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;1973 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;1974 1975 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;1976 def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _S)>;1977 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;1978 def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _D)>;1979 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _D)>;1980 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;1981 1982 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>;1983 def : SVE_3_Op_Pat<nxv4bf16, op, nxv4i1, nxv4bf16, nxv4bf16, !cast<Instruction>(NAME # _S)>;1984 def : SVE_3_Op_Pat<nxv2bf16, op, nxv2i1, nxv2bf16, nxv2bf16, !cast<Instruction>(NAME # _D)>;1985 1986 def : InstAlias<"mov $Zd, $Pg/m, $Zn",1987 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPRAny:$Pg, ZPR8:$Zn, ZPR8:$Zd), 1>;1988 def : InstAlias<"mov $Zd, $Pg/m, $Zn",1989 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, ZPR16:$Zn, ZPR16:$Zd), 1>;1990 def : InstAlias<"mov $Zd, $Pg/m, $Zn",1991 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, ZPR32:$Zn, ZPR32:$Zd), 1>;1992 def : InstAlias<"mov $Zd, $Pg/m, $Zn",1993 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, ZPR64:$Zn, ZPR64:$Zd), 1>;1994}1995 1996 1997//===----------------------------------------------------------------------===//1998// SVE Predicate Logical Operations Group1999//===----------------------------------------------------------------------===//2000 2001class sve_int_pred_log<bits<4> opc, string asm>2002: I<(outs PPRorPNR8:$Pd), (ins PPRorPNRAny:$Pg, PPRorPNR8:$Pn, PPRorPNR8:$Pm),2003 asm, "\t$Pd, $Pg/z, $Pn, $Pm",2004 "",2005 []>, Sched<[]> {2006 bits<4> Pd;2007 bits<4> Pg;2008 bits<4> Pm;2009 bits<4> Pn;2010 let Inst{31-24} = 0b00100101;2011 let Inst{23-22} = opc{3-2};2012 let Inst{21-20} = 0b00;2013 let Inst{19-16} = Pm;2014 let Inst{15-14} = 0b01;2015 let Inst{13-10} = Pg;2016 let Inst{9} = opc{1};2017 let Inst{8-5} = Pn;2018 let Inst{4} = opc{0};2019 let Inst{3-0} = Pd;2020 2021 // SEL has no predication qualifier.2022 let AsmString = !if(!eq(opc, 0b0011),2023 !strconcat(asm, "\t$Pd, $Pg, $Pn, $Pm"),2024 !strconcat(asm, "\t$Pd, $Pg/z, $Pn, $Pm"));2025 2026 let Defs = !if(!eq (opc{2}, 1), [NZCV], []);2027 let hasSideEffects = 0;2028}2029 2030multiclass sve_int_pred_log<bits<4> opc, string asm, SDPatternOperator op,2031 SDPatternOperator op_nopred = null_frag> {2032 def NAME : sve_int_pred_log<opc, asm>;2033 2034 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>;2035 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, nxv8i1, !cast<Instruction>(NAME)>;2036 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, nxv4i1, !cast<Instruction>(NAME)>;2037 def : SVE_3_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, nxv2i1, !cast<Instruction>(NAME)>;2038 def : SVE_3_Op_Pat<nxv1i1, op, nxv1i1, nxv1i1, nxv1i1, !cast<Instruction>(NAME)>;2039 def : SVE_2_Op_AllActive_Pat<nxv16i1, op_nopred, nxv16i1, nxv16i1,2040 !cast<Instruction>(NAME), PTRUE_B>;2041 def : SVE_2_Op_AllActive_Pat<nxv8i1, op_nopred, nxv8i1, nxv8i1,2042 !cast<Instruction>(NAME), PTRUE_H>;2043 def : SVE_2_Op_AllActive_Pat<nxv4i1, op_nopred, nxv4i1, nxv4i1,2044 !cast<Instruction>(NAME), PTRUE_S>;2045 def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2i1, nxv2i1,2046 !cast<Instruction>(NAME), PTRUE_D>;2047 // Emulate .Q operation using a PTRUE_D when the other lanes don't matter.2048 def : SVE_2_Op_AllActive_Pat<nxv1i1, op_nopred, nxv1i1, nxv1i1,2049 !cast<Instruction>(NAME), PTRUE_D>;2050}2051 2052// An instance of sve_int_pred_log_and but uses op_nopred's first operand as the2053// general predicate.2054multiclass sve_int_pred_log_v2<bits<4> opc, string asm, SDPatternOperator op,2055 SDPatternOperator op_nopred> :2056 sve_int_pred_log<opc, asm, op> {2057 def : Pat<(nxv16i1 (op_nopred nxv16i1:$Op1, nxv16i1:$Op2)),2058 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>;2059 def : Pat<(nxv8i1 (op_nopred nxv8i1:$Op1, nxv8i1:$Op2)),2060 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>;2061 def : Pat<(nxv4i1 (op_nopred nxv4i1:$Op1, nxv4i1:$Op2)),2062 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>;2063 def : Pat<(nxv2i1 (op_nopred nxv2i1:$Op1, nxv2i1:$Op2)),2064 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>;2065 // Emulate .Q operation using a PTRUE_D when the other lanes don't matter.2066 def : Pat<(nxv1i1 (op_nopred nxv1i1:$Op1, nxv1i1:$Op2)),2067 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>;2068}2069 2070//===----------------------------------------------------------------------===//2071// SVE Logical Mask Immediate Group2072//===----------------------------------------------------------------------===//2073 2074class sve_int_log_imm<bits<2> opc, string asm>2075: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, logical_imm64:$imms13),2076 asm, "\t$Zdn, $_Zdn, $imms13",2077 "", []>, Sched<[]> {2078 bits<5> Zdn;2079 bits<13> imms13;2080 let Inst{31-24} = 0b00000101;2081 let Inst{23-22} = opc;2082 let Inst{21-18} = 0b0000;2083 let Inst{17-5} = imms13;2084 let Inst{4-0} = Zdn;2085 2086 let Constraints = "$Zdn = $_Zdn";2087 let DecoderMethod = "DecodeSVELogicalImmInstruction";2088 let DestructiveInstType = DestructiveOther;2089 let ElementSize = ElementSizeNone;2090 let hasSideEffects = 0;2091}2092 2093multiclass sve_int_log_imm<bits<2> opc, string asm, string alias, SDPatternOperator op> {2094 def NAME : sve_int_log_imm<opc, asm>;2095 2096 def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8Pat, !cast<Instruction>(NAME)>;2097 def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16Pat, !cast<Instruction>(NAME)>;2098 def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32Pat, !cast<Instruction>(NAME)>;2099 def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64Pat, !cast<Instruction>(NAME)>;2100 2101 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm",2102 (!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8:$imm), 4>;2103 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm",2104 (!cast<Instruction>(NAME) ZPR16:$Zdn, sve_logical_imm16:$imm), 3>;2105 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm",2106 (!cast<Instruction>(NAME) ZPR32:$Zdn, sve_logical_imm32:$imm), 2>;2107 2108 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm",2109 (!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8_not:$imm), 0>;2110 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm",2111 (!cast<Instruction>(NAME) ZPR16:$Zdn, sve_logical_imm16_not:$imm), 0>;2112 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm",2113 (!cast<Instruction>(NAME) ZPR32:$Zdn, sve_logical_imm32_not:$imm), 0>;2114 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm",2115 (!cast<Instruction>(NAME) ZPR64:$Zdn, logical_imm64_not:$imm), 0>;2116}2117 2118multiclass sve_int_log_imm_bic<SDPatternOperator op> {2119 def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8NotPat, !cast<Instruction>("AND_ZI")>;2120 def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16NotPat, !cast<Instruction>("AND_ZI")>;2121 def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32NotPat, !cast<Instruction>("AND_ZI")>;2122 def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64NotPat, !cast<Instruction>("AND_ZI")>;2123}2124 2125class sve_int_dup_mask_imm<string asm>2126: I<(outs ZPR64:$Zd), (ins logical_imm64:$imms),2127 asm, "\t$Zd, $imms",2128 "",2129 []>, Sched<[]> {2130 bits<5> Zd;2131 bits<13> imms;2132 let Inst{31-18} = 0b00000101110000;2133 let Inst{17-5} = imms;2134 let Inst{4-0} = Zd;2135 2136 let DecoderMethod = "DecodeSVELogicalImmInstruction";2137 let hasSideEffects = 0;2138 let isAsCheapAsAMove = 1;2139 let isReMaterializable = 1;2140 let Uses = [VG];2141}2142 2143multiclass sve_int_dup_mask_imm<string asm> {2144 def NAME : sve_int_dup_mask_imm<asm>;2145 2146 def : InstAlias<"dupm $Zd, $imm",2147 (!cast<Instruction>(NAME) ZPR8:$Zd, sve_logical_imm8:$imm), 4>;2148 def : InstAlias<"dupm $Zd, $imm",2149 (!cast<Instruction>(NAME) ZPR16:$Zd, sve_logical_imm16:$imm), 3>;2150 def : InstAlias<"dupm $Zd, $imm",2151 (!cast<Instruction>(NAME) ZPR32:$Zd, sve_logical_imm32:$imm), 2>;2152 2153 // All Zd.b forms have a CPY/DUP equivalent, hence no byte alias here.2154 def : InstAlias<"mov $Zd, $imm",2155 (!cast<Instruction>(NAME) ZPR16:$Zd, sve_preferred_logical_imm16:$imm), 7>;2156 def : InstAlias<"mov $Zd, $imm",2157 (!cast<Instruction>(NAME) ZPR32:$Zd, sve_preferred_logical_imm32:$imm), 6>;2158 def : InstAlias<"mov $Zd, $imm",2159 (!cast<Instruction>(NAME) ZPR64:$Zd, sve_preferred_logical_imm64:$imm), 5>;2160 2161 // NOTE: No pattern for nxv16i8 because DUP has full coverage.2162 def : Pat<(nxv8i16 (splat_vector (i32 (SVELogicalImm16Pat i64:$imm)))),2163 (!cast<Instruction>(NAME) i64:$imm)>;2164 def : Pat<(nxv4i32 (splat_vector (i32 (SVELogicalImm32Pat i64:$imm)))),2165 (!cast<Instruction>(NAME) i64:$imm)>;2166 def : Pat<(nxv2i64 (splat_vector (i64 (SVELogicalImm64Pat i64:$imm)))),2167 (!cast<Instruction>(NAME) i64:$imm)>;2168 2169 def : Pat<(nxv8f16 (splat_vector (f16 (SVELogicalFPImm16Pat i64:$imm)))),2170 (!cast<Instruction>(NAME) i64:$imm)>;2171 def : Pat<(nxv4f16 (splat_vector (f16 (SVELogicalFPImm16Pat i64:$imm)))),2172 (!cast<Instruction>(NAME) i64:$imm)>;2173 def : Pat<(nxv2f16 (splat_vector (f16 (SVELogicalFPImm16Pat i64:$imm)))),2174 (!cast<Instruction>(NAME) i64:$imm)>;2175 def : Pat<(nxv4f32 (splat_vector (f32 (SVELogicalFPImm32Pat i64:$imm)))),2176 (!cast<Instruction>(NAME) i64:$imm)>;2177 def : Pat<(nxv2f32 (splat_vector (f32 (SVELogicalFPImm32Pat i64:$imm)))),2178 (!cast<Instruction>(NAME) i64:$imm)>;2179 def : Pat<(nxv2f64 (splat_vector (f64 (SVELogicalFPImm64Pat i64:$imm)))),2180 (!cast<Instruction>(NAME) i64:$imm)>;2181 2182 def : Pat<(nxv8bf16 (splat_vector (bf16 (SVELogicalBFPImmPat i64:$imm)))),2183 (!cast<Instruction>(NAME) i64:$imm)>;2184 def : Pat<(nxv4bf16 (splat_vector (bf16 (SVELogicalBFPImmPat i64:$imm)))),2185 (!cast<Instruction>(NAME) i64:$imm)>;2186 def : Pat<(nxv2bf16 (splat_vector (bf16 (SVELogicalBFPImmPat i64:$imm)))),2187 (!cast<Instruction>(NAME) i64:$imm)>;2188}2189 2190//===----------------------------------------------------------------------===//2191// SVE Integer Arithmetic - Unpredicated Group.2192//===----------------------------------------------------------------------===//2193 2194class sve_int_bin_cons_arit_0<bits<2> sz8_64, bits<3> opc, string asm,2195 ZPRRegOp zprty>2196: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm),2197 asm, "\t$Zd, $Zn, $Zm",2198 "", []>, Sched<[]> {2199 bits<5> Zd;2200 bits<5> Zm;2201 bits<5> Zn;2202 let Inst{31-24} = 0b00000100;2203 let Inst{23-22} = sz8_64;2204 let Inst{21} = 0b1;2205 let Inst{20-16} = Zm;2206 let Inst{15-13} = 0b000;2207 let Inst{12-10} = opc;2208 let Inst{9-5} = Zn;2209 let Inst{4-0} = Zd;2210 2211 let hasSideEffects = 0;2212}2213 2214multiclass sve_int_bin_cons_arit_0<bits<3> opc, string asm, SDPatternOperator op> {2215 def _B : sve_int_bin_cons_arit_0<0b00, opc, asm, ZPR8>;2216 def _H : sve_int_bin_cons_arit_0<0b01, opc, asm, ZPR16>;2217 def _S : sve_int_bin_cons_arit_0<0b10, opc, asm, ZPR32>;2218 def _D : sve_int_bin_cons_arit_0<0b11, opc, asm, ZPR64>;2219 2220 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;2221 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;2222 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;2223 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;2224}2225 2226//===----------------------------------------------------------------------===//2227// SVE Floating Point Arithmetic - Predicated Group2228//===----------------------------------------------------------------------===//2229 2230class sve_fp_2op_i_p_zds<bits<2> sz, bits<3> opc, string asm,2231 ZPRRegOp zprty,2232 Operand imm_ty>2233: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, imm_ty:$i1),2234 asm, "\t$Zdn, $Pg/m, $_Zdn, $i1",2235 "",2236 []>, Sched<[]> {2237 bits<3> Pg;2238 bits<5> Zdn;2239 bit i1;2240 let Inst{31-24} = 0b01100101;2241 let Inst{23-22} = sz;2242 let Inst{21-19} = 0b011;2243 let Inst{18-16} = opc;2244 let Inst{15-13} = 0b100;2245 let Inst{12-10} = Pg;2246 let Inst{9-6} = 0b0000;2247 let Inst{5} = i1;2248 let Inst{4-0} = Zdn;2249 2250 let Constraints = "$Zdn = $_Zdn";2251 let DestructiveInstType = DestructiveOther;2252 let ElementSize = zprty.ElementSize;2253 let hasSideEffects = 0;2254 let mayRaiseFPException = 1;2255}2256 2257multiclass sve_fp_2op_i_p_zds<bits<3> opc, string asm, string Ps, Operand imm_ty, FPImmLeaf A, FPImmLeaf B, SDPatternOperator op> {2258 let DestructiveInstType = DestructiveBinaryImm in {2259 def _H : SVEPseudo2Instr<Ps # _H, 1>, sve_fp_2op_i_p_zds<0b01, opc, asm, ZPR16, imm_ty>;2260 def _S : SVEPseudo2Instr<Ps # _S, 1>, sve_fp_2op_i_p_zds<0b10, opc, asm, ZPR32, imm_ty>;2261 def _D : SVEPseudo2Instr<Ps # _D, 1>, sve_fp_2op_i_p_zds<0b11, opc, asm, ZPR64, imm_ty>;2262 }2263 2264 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, op, nxv8i1, f16, A, 0, !cast<Instruction>(NAME # "_H")>;2265 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, op, nxv8i1, f16, B, 1, !cast<Instruction>(NAME # "_H")>;2266 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, op, nxv4i1, f32, A, 0, !cast<Instruction>(NAME # "_S")>;2267 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, op, nxv4i1, f32, B, 1, !cast<Instruction>(NAME # "_S")>;2268 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, op, nxv2i1, f64, A, 0, !cast<Instruction>(NAME # "_D")>;2269 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, op, nxv2i1, f64, B, 1, !cast<Instruction>(NAME # "_D")>;2270}2271 2272class sve_fp_2op_p_zds<bits<2> sz, bits<4> opc, string asm,2273 ZPRRegOp zprty>2274: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm),2275 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm",2276 "",2277 []>, Sched<[]> {2278 bits<3> Pg;2279 bits<5> Zdn;2280 bits<5> Zm;2281 let Inst{31-24} = 0b01100101;2282 let Inst{23-22} = sz;2283 let Inst{21-20} = 0b00;2284 let Inst{19-16} = opc;2285 let Inst{15-13} = 0b100;2286 let Inst{12-10} = Pg;2287 let Inst{9-5} = Zm;2288 let Inst{4-0} = Zdn;2289 2290 let Constraints = "$Zdn = $_Zdn";2291 let DestructiveInstType = DestructiveOther;2292 let ElementSize = zprty.ElementSize;2293 let hasSideEffects = 0;2294 let mayRaiseFPException = 1;2295}2296 2297multiclass sve_fp_2op_p_zds<bits<4> opc, string asm, string Ps,2298 SDPatternOperator op, DestructiveInstTypeEnum flags,2299 string revname="", bit isReverseInstr=0> {2300 let DestructiveInstType = flags in {2301 def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>,2302 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>;2303 def _S : sve_fp_2op_p_zds<0b10, opc, asm, ZPR32>,2304 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>;2305 def _D : sve_fp_2op_p_zds<0b11, opc, asm, ZPR64>,2306 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>;2307 }2308 2309 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;2310 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;2311 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;2312}2313 2314multiclass sve_fp_2op_p_zds_fscale<bits<4> opc, string asm,2315 SDPatternOperator op> {2316 def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>;2317 def _S : sve_fp_2op_p_zds<0b10, opc, asm, ZPR32>;2318 def _D : sve_fp_2op_p_zds<0b11, opc, asm, ZPR64>;2319 2320 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>;2321 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>;2322 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>;2323}2324 2325multiclass sve_fp_2op_p_zds_bfloat<bits<4> opc, string asm, string Ps,2326 SDPatternOperator op,2327 DestructiveInstTypeEnum flags,2328 string revname="", bit isReverseInstr=0> {2329 let DestructiveInstType = flags in {2330 def NAME : sve_fp_2op_p_zds<0b00, opc, asm, ZPR16>,2331 SVEPseudo2Instr<Ps, 1>, SVEInstr2Rev<NAME , revname , isReverseInstr>;2332 }2333 2334 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME)>;2335}2336 2337class sve_fp_2op_p_zds_bfscale<bits<4> opc, string asm, DestructiveInstTypeEnum flags>2338: sve_fp_2op_p_zds<0b00, opc, asm, ZPR16>{2339 let DestructiveInstType = flags;2340}2341 2342multiclass sve_fp_2op_p_zds_zeroing_hsd<SDPatternOperator op> {2343 def _H_ZERO : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesZero>;2344 def _S_ZERO : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>;2345 def _D_ZERO : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>;2346 2347 def : SVE_3_Op_Pat_SelZero<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _H_ZERO)>;2348 def : SVE_3_Op_Pat_SelZero<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _S_ZERO)>;2349 def : SVE_3_Op_Pat_SelZero<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _D_ZERO)>;2350}2351 2352multiclass sve_fp_2op_p_zds_zeroing_bfloat<SDPatternOperator op> {2353 def _ZERO : PredTwoOpPseudo<NAME, ZPR16, FalseLanesZero>;2354 2355 def : SVE_3_Op_Pat_SelZero<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Pseudo>(NAME # _ZERO)>;2356}2357 2358class sve_fp_ftmad<bits<2> sz, string asm, ZPRRegOp zprty>2359: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, timm32_0_7:$imm3),2360 asm, "\t$Zdn, $_Zdn, $Zm, $imm3",2361 "",2362 []>, Sched<[]> {2363 bits<5> Zdn;2364 bits<5> Zm;2365 bits<3> imm3;2366 let Inst{31-24} = 0b01100101;2367 let Inst{23-22} = sz;2368 let Inst{21-19} = 0b010;2369 let Inst{18-16} = imm3;2370 let Inst{15-10} = 0b100000;2371 let Inst{9-5} = Zm;2372 let Inst{4-0} = Zdn;2373 2374 let Constraints = "$Zdn = $_Zdn";2375 let DestructiveInstType = DestructiveOther;2376 let ElementSize = ElementSizeNone;2377 let hasSideEffects = 0;2378 let mayRaiseFPException = 1;2379}2380 2381multiclass sve_fp_ftmad<string asm, SDPatternOperator op> {2382 def _H : sve_fp_ftmad<0b01, asm, ZPR16>;2383 def _S : sve_fp_ftmad<0b10, asm, ZPR32>;2384 def _D : sve_fp_ftmad<0b11, asm, ZPR64>;2385 2386 def : Pat<(nxv8f16 (op (nxv8f16 ZPR16:$Zn), (nxv8f16 ZPR16:$Zm), (i32 timm32_0_7:$imm))),2387 (!cast<Instruction>(NAME # _H) ZPR16:$Zn, ZPR16:$Zm, timm32_0_7:$imm)>;2388 def : Pat<(nxv4f32 (op (nxv4f32 ZPR32:$Zn), (nxv4f32 ZPR32:$Zm), (i32 timm32_0_7:$imm))),2389 (!cast<Instruction>(NAME # _S) ZPR32:$Zn, ZPR32:$Zm, timm32_0_7:$imm)>;2390 def : Pat<(nxv2f64 (op (nxv2f64 ZPR64:$Zn), (nxv2f64 ZPR64:$Zm), (i32 timm32_0_7:$imm))),2391 (!cast<Instruction>(NAME # _D) ZPR64:$Zn, ZPR64:$Zm, timm32_0_7:$imm)>;2392}2393 2394multiclass sve_fp_2op_i_p_zds_hfd<Operand imm_ty, FPImmLeaf A, FPImmLeaf B, SDPatternOperator ir_op = null_frag> {2395 def _H_UNDEF : PredTwoOpImmPseudo<NAME # _H, ZPR16, imm_ty, FalseLanesUndef>;2396 def _S_UNDEF : PredTwoOpImmPseudo<NAME # _S, ZPR32, imm_ty, FalseLanesUndef>;2397 def _D_UNDEF : PredTwoOpImmPseudo<NAME # _D, ZPR64, imm_ty, FalseLanesUndef>;2398 2399 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, ir_op, nxv8i1, f16, A, 0, !cast<Instruction>(NAME # "_H_UNDEF")>;2400 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, ir_op, nxv8i1, f16, B, 1, !cast<Instruction>(NAME # "_H_UNDEF")>;2401 def : SVE_2_Op_Fp_Imm_Pat<nxv4f16, ir_op, nxv4i1, f16, A, 0, !cast<Instruction>(NAME # "_H_UNDEF")>;2402 def : SVE_2_Op_Fp_Imm_Pat<nxv4f16, ir_op, nxv4i1, f16, B, 1, !cast<Instruction>(NAME # "_H_UNDEF")>;2403 def : SVE_2_Op_Fp_Imm_Pat<nxv2f16, ir_op, nxv2i1, f16, A, 0, !cast<Instruction>(NAME # "_H_UNDEF")>;2404 def : SVE_2_Op_Fp_Imm_Pat<nxv2f16, ir_op, nxv2i1, f16, B, 1, !cast<Instruction>(NAME # "_H_UNDEF")>;2405 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, ir_op, nxv4i1, f32, A, 0, !cast<Instruction>(NAME # "_S_UNDEF")>;2406 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, ir_op, nxv4i1, f32, B, 1, !cast<Instruction>(NAME # "_S_UNDEF")>;2407 def : SVE_2_Op_Fp_Imm_Pat<nxv2f32, ir_op, nxv2i1, f32, A, 0, !cast<Instruction>(NAME # "_S_UNDEF")>;2408 def : SVE_2_Op_Fp_Imm_Pat<nxv2f32, ir_op, nxv2i1, f32, B, 1, !cast<Instruction>(NAME # "_S_UNDEF")>;2409 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, ir_op, nxv2i1, f64, A, 0, !cast<Instruction>(NAME # "_D_UNDEF")>;2410 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, ir_op, nxv2i1, f64, B, 1, !cast<Instruction>(NAME # "_D_UNDEF")>;2411}2412 2413multiclass sve_fp_2op_i_p_zds_zeroing_hfd<Operand imm_ty, FPImmLeaf A, FPImmLeaf B, SDPatternOperator op> {2414 def _H_ZERO : PredTwoOpImmPseudo<NAME # _H, ZPR16, imm_ty, FalseLanesZero>;2415 def _S_ZERO : PredTwoOpImmPseudo<NAME # _S, ZPR32, imm_ty, FalseLanesZero>;2416 def _D_ZERO : PredTwoOpImmPseudo<NAME # _D, ZPR64, imm_ty, FalseLanesZero>;2417 2418 let AddedComplexity = 2 in {2419 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv8f16, op, nxv8i1, f16, A, 0, !cast<Instruction>(NAME # "_H_ZERO")>;2420 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv8f16, op, nxv8i1, f16, B, 1, !cast<Instruction>(NAME # "_H_ZERO")>;2421 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv4f32, op, nxv4i1, f32, A, 0, !cast<Instruction>(NAME # "_S_ZERO")>;2422 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv4f32, op, nxv4i1, f32, B, 1, !cast<Instruction>(NAME # "_S_ZERO")>;2423 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv2f64, op, nxv2i1, f64, A, 0, !cast<Instruction>(NAME # "_D_ZERO")>;2424 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv2f64, op, nxv2i1, f64, B, 1, !cast<Instruction>(NAME # "_D_ZERO")>;2425 }2426}2427 2428//===----------------------------------------------------------------------===//2429// SVE Floating Point Arithmetic - Unpredicated Group2430//===----------------------------------------------------------------------===//2431 2432class sve_fp_3op_u_zd<bits<2> sz, bits<3> opc, string asm, ZPRRegOp zprty>2433: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm),2434 asm, "\t$Zd, $Zn, $Zm",2435 "",2436 []>, Sched<[]> {2437 bits<5> Zd;2438 bits<5> Zm;2439 bits<5> Zn;2440 let Inst{31-24} = 0b01100101;2441 let Inst{23-22} = sz;2442 let Inst{21} = 0b0;2443 let Inst{20-16} = Zm;2444 let Inst{15-13} = 0b000;2445 let Inst{12-10} = opc;2446 let Inst{9-5} = Zn;2447 let Inst{4-0} = Zd;2448 2449 let hasSideEffects = 0;2450 let mayRaiseFPException = 1;2451}2452 2453multiclass sve_fp_3op_u_zd<bits<3> opc, string asm, SDPatternOperator op> {2454 def _H : sve_fp_3op_u_zd<0b01, opc, asm, ZPR16>;2455 def _S : sve_fp_3op_u_zd<0b10, opc, asm, ZPR32>;2456 def _D : sve_fp_3op_u_zd<0b11, opc, asm, ZPR64>;2457 2458 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;2459 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;2460 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;2461}2462 2463multiclass sve_fp_3op_u_zd_bfloat<bits<3> opc, string asm, SDPatternOperator op> {2464 def NAME : sve_fp_3op_u_zd<0b00, opc, asm, ZPR16>;2465 2466 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME)>;2467}2468 2469multiclass sve_fp_3op_u_zd_ftsmul<bits<3> opc, string asm, SDPatternOperator op> {2470 def _H : sve_fp_3op_u_zd<0b01, opc, asm, ZPR16>;2471 def _S : sve_fp_3op_u_zd<0b10, opc, asm, ZPR32>;2472 def _D : sve_fp_3op_u_zd<0b11, opc, asm, ZPR64>;2473 2474 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>;2475 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>;2476 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>;2477}2478 2479//===----------------------------------------------------------------------===//2480// SVE Floating Point Fused Multiply-Add Group2481//===----------------------------------------------------------------------===//2482 2483class sve_fp_3op_p_zds_a<bits<2> sz, bits<2> opc, string asm, ZPRRegOp zprty>2484: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm),2485 asm, "\t$Zda, $Pg/m, $Zn, $Zm",2486 "",2487 []>, Sched<[]> {2488 bits<3> Pg;2489 bits<5> Zda;2490 bits<5> Zm;2491 bits<5> Zn;2492 let Inst{31-24} = 0b01100101;2493 let Inst{23-22} = sz;2494 let Inst{21} = 0b1;2495 let Inst{20-16} = Zm;2496 let Inst{15} = 0b0;2497 let Inst{14-13} = opc;2498 let Inst{12-10} = Pg;2499 let Inst{9-5} = Zn;2500 let Inst{4-0} = Zda;2501 2502 let Constraints = "$Zda = $_Zda";2503 let ElementSize = zprty.ElementSize;2504 let DestructiveInstType = DestructiveTernaryCommWithRev;2505 let hasSideEffects = 0;2506 let mayRaiseFPException = 1;2507}2508 2509multiclass sve_fp_3op_p_zds_a<bits<2> opc, string asm, string Ps,2510 SDPatternOperator op, string revname,2511 bit isReverseInstr=0> {2512 def _H : sve_fp_3op_p_zds_a<0b01, opc, asm, ZPR16>,2513 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>;2514 def _S : sve_fp_3op_p_zds_a<0b10, opc, asm, ZPR32>,2515 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>;2516 def _D : sve_fp_3op_p_zds_a<0b11, opc, asm, ZPR64>,2517 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>;2518 2519 def : SVE_4_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;2520 def : SVE_4_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _H)>;2521 def : SVE_4_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _H)>;2522 def : SVE_4_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;2523 def : SVE_4_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _S)>;2524 def : SVE_4_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;2525}2526 2527multiclass sve_fp_3op_p_zds_a_bfloat<bits<2> opc, string asm, string Ps,2528 SDPatternOperator op> {2529 def NAME : sve_fp_3op_p_zds_a<0b00, opc, asm, ZPR16>,2530 SVEPseudo2Instr<Ps, 1>, SVEInstr2Rev<NAME, "", 0>;2531 2532 def : SVE_4_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME)>;2533 def : SVE_4_Op_Pat<nxv4bf16, op, nxv4i1, nxv4bf16, nxv4bf16, nxv4bf16, !cast<Instruction>(NAME)>;2534 def : SVE_4_Op_Pat<nxv2bf16, op, nxv2i1, nxv2bf16, nxv2bf16, nxv2bf16, !cast<Instruction>(NAME)>;2535}2536 2537class sve_fp_3op_p_zds_b<bits<2> sz, bits<2> opc, string asm,2538 ZPRRegOp zprty>2539: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, zprty:$Za),2540 asm, "\t$Zdn, $Pg/m, $Zm, $Za",2541 "",2542 []>, Sched<[]> {2543 bits<3> Pg;2544 bits<5> Za;2545 bits<5> Zdn;2546 bits<5> Zm;2547 let Inst{31-24} = 0b01100101;2548 let Inst{23-22} = sz;2549 let Inst{21} = 0b1;2550 let Inst{20-16} = Za;2551 let Inst{15} = 0b1;2552 let Inst{14-13} = opc;2553 let Inst{12-10} = Pg;2554 let Inst{9-5} = Zm;2555 let Inst{4-0} = Zdn;2556 2557 let Constraints = "$Zdn = $_Zdn";2558 let DestructiveInstType = DestructiveOther;2559 let ElementSize = zprty.ElementSize;2560 let hasSideEffects = 0;2561 let mayRaiseFPException = 1;2562}2563 2564multiclass sve_fp_3op_p_zds_b<bits<2> opc, string asm, SDPatternOperator op,2565 string revname, bit isReverseInstr> {2566 def _H : sve_fp_3op_p_zds_b<0b01, opc, asm, ZPR16>,2567 SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>;2568 def _S : sve_fp_3op_p_zds_b<0b10, opc, asm, ZPR32>,2569 SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>;2570 def _D : sve_fp_3op_p_zds_b<0b11, opc, asm, ZPR64>,2571 SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>;2572 2573 def : SVE_4_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;2574 def : SVE_4_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;2575 def : SVE_4_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;2576}2577 2578//===----------------------------------------------------------------------===//2579// SVE Floating Point Multiply-Add - Indexed Group2580//===----------------------------------------------------------------------===//2581 2582class sve_fp_fma_by_indexed_elem<bits<2> sz, bits<2> opc, string asm,2583 ZPRRegOp zprty1,2584 ZPRRegOp zprty2, Operand itype>2585: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty1:$Zn, zprty2:$Zm, itype:$iop),2586 asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> {2587 bits<5> Zda;2588 bits<5> Zn;2589 let Inst{31-24} = 0b01100100;2590 let Inst{23-22} = sz;2591 let Inst{21} = 0b1;2592 let Inst{15-12} = 0b0000;2593 let Inst{11-10} = opc;2594 let Inst{9-5} = Zn;2595 let Inst{4-0} = Zda;2596 2597 let Constraints = "$Zda = $_Zda";2598 let DestructiveInstType = DestructiveOther;2599 let ElementSize = ElementSizeNone;2600 let hasSideEffects = 0;2601 let mayRaiseFPException = 1;2602}2603 2604multiclass sve_fp_fma_by_indexed_elem<bits<2> opc, string asm,2605 SDPatternOperator op> {2606 def _H : sve_fp_fma_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR3b16, VectorIndexH32b> {2607 bits<3> Zm;2608 bits<3> iop;2609 let Inst{22} = iop{2};2610 let Inst{20-19} = iop{1-0};2611 let Inst{18-16} = Zm;2612 }2613 def _S : sve_fp_fma_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR3b32, VectorIndexS32b> {2614 bits<3> Zm;2615 bits<2> iop;2616 let Inst{20-19} = iop;2617 let Inst{18-16} = Zm;2618 }2619 def _D : sve_fp_fma_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR4b64, VectorIndexD32b> {2620 bits<4> Zm;2621 bit iop;2622 let Inst{20} = iop;2623 let Inst{19-16} = Zm;2624 }2625 2626 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexH32b_timm:$idx))),2627 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, VectorIndexH32b_timm:$idx)>;2628 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexS32b_timm:$idx))),2629 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx)>;2630 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, (i32 VectorIndexD32b_timm:$idx))),2631 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx)>;2632}2633 2634multiclass sve_fp_fma_by_indexed_elem_bfloat<string asm, bits<2> opc,2635 SDPatternOperator op> {2636 def NAME : sve_fp_fma_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR3b16, VectorIndexH32b> {2637 bits<3> Zm;2638 bits<3> iop;2639 let Inst{22} = iop{2};2640 let Inst{20-19} = iop{1-0};2641 let Inst{18-16} = Zm;2642 }2643 2644 def : Pat<(nxv8bf16 (op nxv8bf16:$op1, nxv8bf16:$op2, nxv8bf16:$op3, (i32 VectorIndexH32b_timm:$idx))),2645 (!cast<Instruction>(NAME) $op1, $op2, $op3, VectorIndexH32b_timm:$idx)>;2646}2647 2648//===----------------------------------------------------------------------===//2649// SVE Floating Point Multiply - Indexed Group2650//===----------------------------------------------------------------------===//2651 2652class sve_fp_fmul_by_indexed_elem<bits<2> sz, bit o2, string asm, ZPRRegOp zprty,2653 ZPRRegOp zprty2, Operand itype>2654: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty2:$Zm, itype:$iop),2655 asm, "\t$Zd, $Zn, $Zm$iop", "", []>, Sched<[]> {2656 bits<5> Zd;2657 bits<5> Zn;2658 let Inst{31-24} = 0b01100100;2659 let Inst{23-22} = sz;2660 let Inst{21} = 0b1;2661 let Inst{15-12} = 0b0010;2662 let Inst{11} = o2;2663 let Inst{10} = 0b0;2664 let Inst{9-5} = Zn;2665 let Inst{4-0} = Zd;2666 2667 let hasSideEffects = 0;2668 let mayRaiseFPException = 1;2669}2670 2671multiclass sve_fp_fmul_by_indexed_elem<string asm, SDPatternOperator op> {2672 def _H : sve_fp_fmul_by_indexed_elem<{0, ?}, 0b0, asm, ZPR16, ZPR3b16, VectorIndexH32b> {2673 bits<3> Zm;2674 bits<3> iop;2675 let Inst{22} = iop{2};2676 let Inst{20-19} = iop{1-0};2677 let Inst{18-16} = Zm;2678 }2679 def _S : sve_fp_fmul_by_indexed_elem<0b10, 0b0, asm, ZPR32, ZPR3b32, VectorIndexS32b> {2680 bits<3> Zm;2681 bits<2> iop;2682 let Inst{20-19} = iop;2683 let Inst{18-16} = Zm;2684 }2685 def _D : sve_fp_fmul_by_indexed_elem<0b11, 0b0, asm, ZPR64, ZPR4b64, VectorIndexD32b> {2686 bits<4> Zm;2687 bit iop;2688 let Inst{20} = iop;2689 let Inst{19-16} = Zm;2690 }2691 2692 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, (i32 VectorIndexH32b_timm:$idx))),2693 (!cast<Instruction>(NAME # _H) $Op1, $Op2, VectorIndexH32b_timm:$idx)>;2694 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, (i32 VectorIndexS32b_timm:$idx))),2695 (!cast<Instruction>(NAME # _S) $Op1, $Op2, VectorIndexS32b_timm:$idx)>;2696 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, (i32 VectorIndexD32b_timm:$idx))),2697 (!cast<Instruction>(NAME # _D) $Op1, $Op2, VectorIndexD32b_timm:$idx)>;2698}2699 2700multiclass sve_fp_fmul_by_indexed_elem_bfloat<string asm,2701 SDPatternOperator op> {2702 def NAME : sve_fp_fmul_by_indexed_elem<{0, ?}, 0b1, asm, ZPR16, ZPR3b16, VectorIndexH32b> {2703 bits<3> Zm;2704 bits<3> iop;2705 let Inst{22} = iop{2};2706 let Inst{20-19} = iop{1-0};2707 let Inst{18-16} = Zm;2708 }2709 def : Pat <(nxv8bf16 (op nxv8bf16:$Op1, nxv8bf16:$Op2, (i32 VectorIndexH32b_timm:$idx))),2710 (!cast<Instruction>(NAME) $Op1, $Op2, VectorIndexH32b_timm:$idx)>;2711}2712 2713//===----------------------------------------------------------------------===//2714// SVE Floating Point Complex Multiply-Add Group2715//===----------------------------------------------------------------------===//2716 2717class sve_fp_fcmla<bits<2> sz, string asm, ZPRRegOp zprty>2718: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm,2719 complexrotateop:$imm),2720 asm, "\t$Zda, $Pg/m, $Zn, $Zm, $imm",2721 "", []>, Sched<[]> {2722 bits<5> Zda;2723 bits<3> Pg;2724 bits<5> Zn;2725 bits<5> Zm;2726 bits<2> imm;2727 let Inst{31-24} = 0b01100100;2728 let Inst{23-22} = sz;2729 let Inst{21} = 0;2730 let Inst{20-16} = Zm;2731 let Inst{15} = 0;2732 let Inst{14-13} = imm;2733 let Inst{12-10} = Pg;2734 let Inst{9-5} = Zn;2735 let Inst{4-0} = Zda;2736 2737 let Constraints = "$Zda = $_Zda";2738 let DestructiveInstType = DestructiveOther;2739 let ElementSize = zprty.ElementSize;2740 let hasSideEffects = 0;2741 let mayRaiseFPException = 1;2742}2743 2744multiclass sve_fp_fcmla<string asm, SDPatternOperator op> {2745 def _H : sve_fp_fcmla<0b01, asm, ZPR16>;2746 def _S : sve_fp_fcmla<0b10, asm, ZPR32>;2747 def _D : sve_fp_fcmla<0b11, asm, ZPR64>;2748 2749 def : Pat<(nxv8f16 (op nxv8i1:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, nxv8f16:$Op4, (i32 complexrotateop:$imm))),2750 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>;2751 def : Pat<(nxv4f32 (op nxv4i1:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, nxv4f32:$Op4, (i32 complexrotateop:$imm))),2752 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>;2753 def : Pat<(nxv2f64 (op nxv2i1:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, nxv2f64:$Op4, (i32 complexrotateop:$imm))),2754 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>;2755}2756 2757//===----------------------------------------------------------------------===//2758// SVE Floating Point Complex Multiply-Add - Indexed Group2759//===----------------------------------------------------------------------===//2760 2761class sve_fp_fcmla_by_indexed_elem<bits<2> sz, string asm,2762 ZPRRegOp zprty,2763 ZPRRegOp zprty2, Operand itype>2764: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, zprty2:$Zm, itype:$iop,2765 complexrotateop:$imm),2766 asm, "\t$Zda, $Zn, $Zm$iop, $imm",2767 "", []>, Sched<[]> {2768 bits<5> Zda;2769 bits<5> Zn;2770 bits<2> imm;2771 let Inst{31-24} = 0b01100100;2772 let Inst{23-22} = sz;2773 let Inst{21} = 0b1;2774 let Inst{15-12} = 0b0001;2775 let Inst{11-10} = imm;2776 let Inst{9-5} = Zn;2777 let Inst{4-0} = Zda;2778 2779 let Constraints = "$Zda = $_Zda";2780 let DestructiveInstType = DestructiveOther;2781 let ElementSize = ElementSizeNone;2782 let hasSideEffects = 0;2783 let mayRaiseFPException = 1;2784}2785 2786multiclass sve_fp_fcmla_by_indexed_elem<string asm, SDPatternOperator op> {2787 def _H : sve_fp_fcmla_by_indexed_elem<0b10, asm, ZPR16, ZPR3b16, VectorIndexS32b> {2788 bits<3> Zm;2789 bits<2> iop;2790 let Inst{20-19} = iop;2791 let Inst{18-16} = Zm;2792 }2793 def _S : sve_fp_fcmla_by_indexed_elem<0b11, asm, ZPR32, ZPR4b32, VectorIndexD32b> {2794 bits<4> Zm;2795 bits<1> iop;2796 let Inst{20} = iop;2797 let Inst{19-16} = Zm;2798 }2799 2800 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))),2801 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>;2802 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))),2803 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>;2804}2805 2806//===----------------------------------------------------------------------===//2807// SVE Floating Point Complex Addition Group2808//===----------------------------------------------------------------------===//2809 2810class sve_fp_fcadd<bits<2> sz, string asm, ZPRRegOp zprty>2811: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm,2812 complexrotateopodd:$imm),2813 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm, $imm",2814 "",2815 []>, Sched<[]> {2816 bits<5> Zdn;2817 bits<5> Zm;2818 bits<3> Pg;2819 bit imm;2820 let Inst{31-24} = 0b01100100;2821 let Inst{23-22} = sz;2822 let Inst{21-17} = 0;2823 let Inst{16} = imm;2824 let Inst{15-13} = 0b100;2825 let Inst{12-10} = Pg;2826 let Inst{9-5} = Zm;2827 let Inst{4-0} = Zdn;2828 2829 let Constraints = "$Zdn = $_Zdn";2830 let DestructiveInstType = DestructiveOther;2831 let ElementSize = zprty.ElementSize;2832 let hasSideEffects = 0;2833 let mayRaiseFPException = 1;2834}2835 2836multiclass sve_fp_fcadd<string asm, SDPatternOperator op> {2837 def _H : sve_fp_fcadd<0b01, asm, ZPR16>;2838 def _S : sve_fp_fcadd<0b10, asm, ZPR32>;2839 def _D : sve_fp_fcadd<0b11, asm, ZPR64>;2840 2841 def : Pat<(nxv8f16 (op nxv8i1:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 complexrotateopodd:$imm))),2842 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>;2843 def : Pat<(nxv4f32 (op nxv4i1:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 complexrotateopodd:$imm))),2844 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>;2845 def : Pat<(nxv2f64 (op nxv2i1:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, (i32 complexrotateopodd:$imm))),2846 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>;2847}2848 2849//===----------------------------------------------------------------------===//2850// SVE2 Floating Point Convert Group2851//===----------------------------------------------------------------------===//2852 2853class sve2_fp_convert_precision<bits<4> opc, bit merging, string asm,2854 ZPRRegOp zprty1, ZPRRegOp zprty2, bit destructive=merging>2855: I<(outs zprty1:$Zd),2856 !if(destructive, (ins zprty1:$_Zd, PPR3bAny:$Pg, zprty2:$Zn),2857 (ins PPR3bAny:$Pg, zprty2:$Zn)),2858 asm, "\t$Zd, " # !if(merging, "$Pg/m", "$Pg/z") # ", $Zn",2859 "",2860 []>, Sched<[]> {2861 bits<5> Zd;2862 bits<5> Zn;2863 bits<3> Pg;2864 let Inst{31-24} = 0b01100100;2865 let Inst{23-22} = opc{3-2};2866 let Inst{21-20} = 0b00;2867 let Inst{19} = merging;2868 let Inst{18} = 0b0;2869 let Inst{17-16} = opc{1-0};2870 let Inst{15-13} = 0b101;2871 let Inst{12-10} = Pg;2872 let Inst{9-5} = Zn;2873 let Inst{4-0} = Zd;2874 2875 let Constraints = !if(destructive, "$Zd = $_Zd", "");2876 let hasSideEffects = 0;2877 let mayRaiseFPException = 1;2878}2879 2880multiclass sve2_fp_convert_down_narrow<string asm, string op> {2881 def _StoH : sve2_fp_convert_precision<0b1000, 0b1, asm, ZPR16, ZPR32>;2882 def _DtoS : sve2_fp_convert_precision<0b1110, 0b1, asm, ZPR32, ZPR64>;2883 2884 def : SVE_3_Op_Pat<nxv8f16, !cast<SDPatternOperator>(op # _f16f32), nxv8f16, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _StoH)>;2885 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>;2886}2887 2888multiclass sve2_fp_convert_up_long<string asm, string op> {2889 def _HtoS : sve2_fp_convert_precision<0b1001, 0b1, asm, ZPR32, ZPR16>;2890 def _StoD : sve2_fp_convert_precision<0b1111, 0b1, asm, ZPR64, ZPR32>;2891 2892 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f16), nxv4f32, nxv4i1, nxv8f16, !cast<Instruction>(NAME # _HtoS)>;2893 def : SVE_3_Op_Pat<nxv2f64, !cast<SDPatternOperator>(op # _f64f32), nxv2f64, nxv2i1, nxv4f32, !cast<Instruction>(NAME # _StoD)>;2894}2895 2896multiclass sve2_fp_convert_down_odd_rounding_top<string asm, string op> {2897 def _DtoS : sve2_fp_convert_precision<0b0010, 0b1, asm, ZPR32, ZPR64>;2898 2899 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>;2900}2901 2902multiclass sve2_fp_convert_up_long_z<string asm, string op> {2903 def _HtoS : sve2_fp_convert_precision<0b1001, 0b0, asm, ZPR32, ZPR16>;2904 def _StoD : sve2_fp_convert_precision<0b1111, 0b0, asm, ZPR64, ZPR32>;2905 2906 defm : SVE_3_Op_UndefZero_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f16), nxv4f32, nxv4i1, nxv8f16, !cast<Instruction>(NAME # _HtoS)>;2907 defm : SVE_3_Op_UndefZero_Pat<nxv2f64, !cast<SDPatternOperator>(op # _f64f32), nxv2f64, nxv2i1, nxv4f32, !cast<Instruction>(NAME # _StoD)>;2908}2909 2910multiclass sve2_fp_convert_down_narrow_z<string asm> {2911 def _StoH : sve2_fp_convert_precision<0b1000, 0b0, asm, ZPR16, ZPR32, /*destructive*/ true>;2912 def _DtoS : sve2_fp_convert_precision<0b1110, 0b0, asm, ZPR32, ZPR64, /*destructive*/ true>;2913}2914 2915//===----------------------------------------------------------------------===//2916// SVE2 Floating Point Pairwise Group2917//===----------------------------------------------------------------------===//2918 2919class sve2_fp_pairwise_pred<bits<2> sz, bits<3> opc, string asm,2920 ZPRRegOp zprty>2921: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm),2922 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm",2923 "",2924 []>, Sched<[]> {2925 bits<3> Pg;2926 bits<5> Zm;2927 bits<5> Zdn;2928 let Inst{31-24} = 0b01100100;2929 let Inst{23-22} = sz;2930 let Inst{21-19} = 0b010;2931 let Inst{18-16} = opc;2932 let Inst{15-13} = 0b100;2933 let Inst{12-10} = Pg;2934 let Inst{9-5} = Zm;2935 let Inst{4-0} = Zdn;2936 2937 let Constraints = "$Zdn = $_Zdn";2938 let DestructiveInstType = DestructiveOther;2939 let ElementSize = zprty.ElementSize;2940 let hasSideEffects = 0;2941 let mayRaiseFPException = 1;2942}2943 2944multiclass sve2_fp_pairwise_pred<bits<3> opc, string asm,2945 SDPatternOperator op> {2946 def _H : sve2_fp_pairwise_pred<0b01, opc, asm, ZPR16>;2947 def _S : sve2_fp_pairwise_pred<0b10, opc, asm, ZPR32>;2948 def _D : sve2_fp_pairwise_pred<0b11, opc, asm, ZPR64>;2949 2950 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;2951 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;2952 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;2953}2954 2955//===----------------------------------------------------------------------===//2956// SVE2 Floating Point Widening Multiply-Add - Indexed Group2957//===----------------------------------------------------------------------===//2958 2959class sve2_fp_mla_long_by_indexed_elem<bits<3> opc, string asm>2960: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm,2961 VectorIndexH32b:$iop),2962 asm, "\t$Zda, $Zn, $Zm$iop",2963 "",2964 []>, Sched<[]> {2965 bits<5> Zda;2966 bits<5> Zn;2967 bits<3> Zm;2968 bits<3> iop;2969 let Inst{31-23} = 0b011001001;2970 let Inst{22} = opc{2};2971 let Inst{21} = 0b1;2972 let Inst{20-19} = iop{2-1};2973 let Inst{18-16} = Zm;2974 let Inst{15-14} = 0b01;2975 let Inst{13} = opc{1};2976 let Inst{12} = 0b0;2977 let Inst{11} = iop{0};2978 let Inst{10} = opc{0};2979 let Inst{9-5} = Zn;2980 let Inst{4-0} = Zda;2981 2982 let Constraints = "$Zda = $_Zda";2983 let DestructiveInstType = DestructiveOther;2984 let ElementSize = ElementSizeNone;2985 let hasSideEffects = 0;2986 let mayRaiseFPException = 1;2987}2988 2989multiclass sve2_fp_mla_long_by_indexed_elem<bits<3> opc, string asm,2990 ValueType OutVT, ValueType InVT,2991 SDPatternOperator op> {2992 def NAME : sve2_fp_mla_long_by_indexed_elem<opc, asm>;2993 def : SVE_4_Op_Imm_Pat<OutVT, op, OutVT, InVT, InVT, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME)>;2994}2995 2996//===----------------------------------------------------------------------===//2997// SVE2 Floating Point Widening Multiply-Add Group2998//===----------------------------------------------------------------------===//2999 3000class sve2_fp_mla_long<bits<3> opc, string asm>3001: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm),3002 asm, "\t$Zda, $Zn, $Zm",3003 "",3004 []>, Sched<[]> {3005 bits<5> Zda;3006 bits<5> Zn;3007 bits<5> Zm;3008 let Inst{31-23} = 0b011001001;3009 let Inst{22} = opc{2};3010 let Inst{21} = 0b1;3011 let Inst{20-16} = Zm;3012 let Inst{15-14} = 0b10;3013 let Inst{13} = opc{1};3014 let Inst{12-11} = 0b00;3015 let Inst{10} = opc{0};3016 let Inst{9-5} = Zn;3017 let Inst{4-0} = Zda;3018 3019 let Constraints = "$Zda = $_Zda";3020 let DestructiveInstType = DestructiveOther;3021 let ElementSize = ElementSizeNone;3022 let hasSideEffects = 0;3023 let mayRaiseFPException = 1;3024}3025 3026multiclass sve2_fp_mla_long<bits<3> opc, string asm, ValueType OutVT,3027 ValueType InVT, SDPatternOperator op> {3028 def NAME : sve2_fp_mla_long<opc, asm>;3029 def : SVE_3_Op_Pat<OutVT, op, OutVT, InVT, InVT, !cast<Instruction>(NAME)>;3030}3031 3032//===----------------------------------------------------------------------===//3033// SVE Stack Allocation Group3034//===----------------------------------------------------------------------===//3035 3036class sve_int_arith_vl<bit opc, string asm, bit streaming_sve = 0b0>3037: I<(outs GPR64sp:$Rd), (ins GPR64sp:$Rn, simm6_32b:$imm6),3038 asm, "\t$Rd, $Rn, $imm6",3039 "",3040 []>, Sched<[]> {3041 bits<5> Rd;3042 bits<5> Rn;3043 bits<6> imm6;3044 let Inst{31-23} = 0b000001000;3045 let Inst{22} = opc;3046 let Inst{21} = 0b1;3047 let Inst{20-16} = Rn;3048 let Inst{15-12} = 0b0101;3049 let Inst{11} = streaming_sve;3050 let Inst{10-5} = imm6;3051 let Inst{4-0} = Rd;3052 3053 let hasSideEffects = 0;3054 let Uses = [VG];3055}3056 3057class sve_int_read_vl_a<bit op, bits<5> opc2, string asm, bit streaming_sve = 0b0>3058: I<(outs GPR64:$Rd), (ins simm6_32b:$imm6),3059 asm, "\t$Rd, $imm6",3060 "",3061 []>, Sched<[]> {3062 bits<5> Rd;3063 bits<6> imm6;3064 let Inst{31-23} = 0b000001001;3065 let Inst{22} = op;3066 let Inst{21} = 0b1;3067 let Inst{20-16} = opc2{4-0};3068 let Inst{15-12} = 0b0101;3069 let Inst{11} = streaming_sve;3070 let Inst{10-5} = imm6;3071 let Inst{4-0} = Rd;3072 3073 let hasSideEffects = 0;3074 let isReMaterializable = 1;3075 let Uses = [VG];3076}3077 3078//===----------------------------------------------------------------------===//3079// SVE Permute - In Lane Group3080//===----------------------------------------------------------------------===//3081 3082class sve_int_perm_bin_perm_zz<bits<3> opc, bits<2> sz8_64, string asm,3083 ZPRRegOp zprty>3084: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm),3085 asm, "\t$Zd, $Zn, $Zm",3086 "",3087 []>, Sched<[]> {3088 bits<5> Zd;3089 bits<5> Zm;3090 bits<5> Zn;3091 let Inst{31-24} = 0b00000101;3092 let Inst{23-22} = sz8_64;3093 let Inst{21} = 0b1;3094 let Inst{20-16} = Zm;3095 let Inst{15-13} = 0b011;3096 let Inst{12-10} = opc;3097 let Inst{9-5} = Zn;3098 let Inst{4-0} = Zd;3099 3100 let hasSideEffects = 0;3101}3102 3103multiclass sve_int_perm_bin_perm_zz<bits<3> opc, string asm,3104 SDPatternOperator op> {3105 def _B : sve_int_perm_bin_perm_zz<opc, 0b00, asm, ZPR8>;3106 def _H : sve_int_perm_bin_perm_zz<opc, 0b01, asm, ZPR16>;3107 def _S : sve_int_perm_bin_perm_zz<opc, 0b10, asm, ZPR32>;3108 def _D : sve_int_perm_bin_perm_zz<opc, 0b11, asm, ZPR64>;3109 3110 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;3111 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;3112 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;3113 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;3114 3115 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;3116 def : SVE_2_Op_Pat<nxv4f16, op, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _S)>;3117 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;3118 def : SVE_2_Op_Pat<nxv2f16, op, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _D)>;3119 def : SVE_2_Op_Pat<nxv2f32, op, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _D)>;3120 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;3121 3122 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>;3123 def : SVE_2_Op_Pat<nxv4bf16, op, nxv4bf16, nxv4bf16, !cast<Instruction>(NAME # _S)>;3124 def : SVE_2_Op_Pat<nxv2bf16, op, nxv2bf16, nxv2bf16, !cast<Instruction>(NAME # _D)>;3125}3126 3127//===----------------------------------------------------------------------===//3128// SVE Floating Point Unary Operations Group3129//===----------------------------------------------------------------------===//3130 3131class sve_fp_2op_p_zd<bits<7> opc, string asm, RegisterOperand i_zprtype,3132 RegisterOperand o_zprtype, ElementSizeEnum Sz>3133: I<(outs o_zprtype:$Zd), (ins i_zprtype:$_Zd, PPR3bAny:$Pg, i_zprtype:$Zn),3134 asm, "\t$Zd, $Pg/m, $Zn",3135 "",3136 []>, Sched<[]> {3137 bits<3> Pg;3138 bits<5> Zd;3139 bits<5> Zn;3140 let Inst{31-24} = 0b01100101;3141 let Inst{23-22} = opc{6-5};3142 let Inst{21} = 0b0;3143 let Inst{20-16} = opc{4-0};3144 let Inst{15-13} = 0b101;3145 let Inst{12-10} = Pg;3146 let Inst{9-5} = Zn;3147 let Inst{4-0} = Zd;3148 3149 let Constraints = "$Zd = $_Zd";3150 let DestructiveInstType = DestructiveUnaryPassthru;3151 let ElementSize = Sz;3152 let hasSideEffects = 0;3153 let mayRaiseFPException = 1;3154}3155 3156multiclass sve_fp_2op_p_zd<bits<7> opc, string asm,3157 RegisterOperand i_zprtype,3158 RegisterOperand o_zprtype,3159 SDPatternOperator int_op,3160 SDPatternOperator ir_op, ValueType vt1,3161 ValueType vt2, ValueType vt3, ElementSizeEnum Sz> {3162 def NAME : sve_fp_2op_p_zd<opc, asm, i_zprtype, o_zprtype, Sz>,3163 SVEPseudo2Instr<NAME, 1>;3164 // convert vt1 to a packed type for the intrinsic patterns3165 defvar packedvt1 = SVEType<vt1>.Packed;3166 3167 // convert vt3 to a packed type for the intrinsic patterns3168 defvar packedvt3 = SVEType<vt3>.Packed;3169 3170 def : SVE_3_Op_Pat<packedvt1, int_op, packedvt1, vt2, packedvt3, !cast<Instruction>(NAME)>;3171 def : SVE_1_Op_Passthru_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME)>;3172 3173 def _UNDEF : PredOneOpPassthruPseudo<NAME, !cast<ZPRRegOp>(i_zprtype)>;3174 3175 defm : SVE_1_Op_PassthruUndef_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME # _UNDEF)>;3176}3177 3178multiclass sve_fp_2op_p_zdr<bits<7> opc, string asm,3179 RegisterOperand i_zprtype,3180 RegisterOperand o_zprtype,3181 SDPatternOperator int_op,3182 SDPatternOperator ir_op, ValueType vt1,3183 ValueType vt2, ValueType vt3, ElementSizeEnum Sz> {3184 def NAME : sve_fp_2op_p_zd<opc, asm, i_zprtype, o_zprtype, Sz>,3185 SVEPseudo2Instr<NAME, 1>;3186 3187 // convert vt1 to a packed type for the intrinsic patterns3188 defvar packedvt1 = SVEType<vt1>.Packed;3189 3190 def : SVE_3_Op_Pat<packedvt1, int_op, packedvt1, vt2, vt3, !cast<Instruction>(NAME)>;3191 def : SVE_1_Op_Passthru_Round_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME)>;3192 3193 def _UNDEF : PredOneOpPassthruPseudo<NAME, !cast<ZPRRegOp>(i_zprtype)>;3194 3195 defm : SVE_1_Op_PassthruUndef_Round_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME # _UNDEF)>;3196}3197 3198multiclass sve_fp_2op_p_zd_HSD<bits<5> opc, string asm, SDPatternOperator op> {3199 def _H : sve_fp_2op_p_zd<{ 0b01, opc }, asm, ZPR16, ZPR16, ElementSizeH>,3200 SVEPseudo2Instr<NAME # _H, 1>;3201 def _S : sve_fp_2op_p_zd<{ 0b10, opc }, asm, ZPR32, ZPR32, ElementSizeS>,3202 SVEPseudo2Instr<NAME # _S, 1>;3203 def _D : sve_fp_2op_p_zd<{ 0b11, opc }, asm, ZPR64, ZPR64, ElementSizeD>,3204 SVEPseudo2Instr<NAME # _D, 1>;3205 3206 def : SVE_1_Op_Passthru_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;3207 def : SVE_1_Op_Passthru_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>;3208 def : SVE_1_Op_Passthru_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>;3209 def : SVE_1_Op_Passthru_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;3210 def : SVE_1_Op_Passthru_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>;3211 def : SVE_1_Op_Passthru_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;3212 3213 def _H_UNDEF : PredOneOpPassthruPseudo<NAME # _H, ZPR16>;3214 def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32>;3215 def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64>;3216 3217 defm : SVE_1_Op_PassthruUndef_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H_UNDEF)>;3218 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H_UNDEF)>;3219 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H_UNDEF)>;3220 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S_UNDEF)>;3221 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S_UNDEF)>;3222 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D_UNDEF)>;3223}3224 3225multiclass sve2_fp_flogb<string asm, string Ps, SDPatternOperator op> {3226 def _H : sve_fp_2op_p_zd<0b0011010, asm, ZPR16, ZPR16, ElementSizeH>,3227 SVEPseudo2Instr<Ps # _H, 1>;3228 def _S : sve_fp_2op_p_zd<0b0011100, asm, ZPR32, ZPR32, ElementSizeS>,3229 SVEPseudo2Instr<Ps # _S, 1>;3230 def _D : sve_fp_2op_p_zd<0b0011110, asm, ZPR64, ZPR64, ElementSizeD>,3231 SVEPseudo2Instr<Ps # _D, 1>;3232 3233 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;3234 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;3235 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;3236}3237 3238multiclass sve2_fp_un_pred_zeroing_hsd<SDPatternOperator op> {3239 def _H_ZERO : PredOneOpPassthruPseudo<NAME # _H, ZPR16, FalseLanesZero>;3240 def _S_ZERO : PredOneOpPassthruPseudo<NAME # _S, ZPR32, FalseLanesZero>;3241 def _D_ZERO : PredOneOpPassthruPseudo<NAME # _D, ZPR64, FalseLanesZero>;3242 3243 def : SVE_1_Op_PassthruZero_Pat<nxv8i16, op, nxv8i1, nxv8f16, !cast<Pseudo>(NAME # _H_ZERO)>;3244 def : SVE_1_Op_PassthruZero_Pat<nxv4i32, op, nxv4i1, nxv4f32, !cast<Pseudo>(NAME # _S_ZERO)>;3245 def : SVE_1_Op_PassthruZero_Pat<nxv2i64, op, nxv2i1, nxv2f64, !cast<Pseudo>(NAME # _D_ZERO)>;3246}3247 3248multiclass sve2_fp_convert_down_odd_rounding<string asm, string op, SDPatternOperator ir_op = null_frag> {3249 def _DtoS : sve_fp_2op_p_zd<0b0001010, asm, ZPR64, ZPR32, ElementSizeD>;3250 3251 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>;3252 def : SVE_1_Op_Passthru_Pat<nxv2f32, ir_op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>;3253}3254 3255multiclass sve_fp_2op_p_zd_frint<bits<2> opc, string asm> {3256 def _S : sve_fp_2op_p_zd<{ 0b0010, opc{1}, 0, opc{0} }, asm, ZPR32, ZPR32, ElementSizeS>;3257 def _D : sve_fp_2op_p_zd<{ 0b0010, opc{1}, 1, opc{0} }, asm, ZPR64, ZPR64, ElementSizeD>;3258}3259 3260//===----------------------------------------------------------------------===//3261// SVE Floating Point Unary Operations - Unpredicated Group3262//===----------------------------------------------------------------------===//3263 3264class sve_fp_2op_u_zd<bits<2> sz, bits<3> opc, string asm,3265 ZPRRegOp zprty>3266: I<(outs zprty:$Zd), (ins zprty:$Zn),3267 asm, "\t$Zd, $Zn",3268 "",3269 []>, Sched<[]> {3270 bits<5> Zd;3271 bits<5> Zn;3272 let Inst{31-24} = 0b01100101;3273 let Inst{23-22} = sz;3274 let Inst{21-19} = 0b001;3275 let Inst{18-16} = opc;3276 let Inst{15-10} = 0b001100;3277 let Inst{9-5} = Zn;3278 let Inst{4-0} = Zd;3279 3280 let hasSideEffects = 0;3281 let mayRaiseFPException = 1;3282}3283 3284multiclass sve_fp_2op_u_zd<bits<3> opc, string asm, SDPatternOperator op> {3285 def _H : sve_fp_2op_u_zd<0b01, opc, asm, ZPR16>;3286 def _S : sve_fp_2op_u_zd<0b10, opc, asm, ZPR32>;3287 def _D : sve_fp_2op_u_zd<0b11, opc, asm, ZPR64>;3288 3289 def : SVE_1_Op_Pat<nxv8f16, op, nxv8f16, !cast<Instruction>(NAME # _H)>;3290 def : SVE_1_Op_Pat<nxv4f32, op, nxv4f32, !cast<Instruction>(NAME # _S)>;3291 def : SVE_1_Op_Pat<nxv2f64, op, nxv2f64, !cast<Instruction>(NAME # _D)>;3292}3293 3294//===----------------------------------------------------------------------===//3295// SVE Floating Point Unary Operations - Zeroing Predicate Group3296//===----------------------------------------------------------------------===//3297 3298class sve_fp_z2op_p_zd<bits<7> opc,string asm, RegisterOperand i_zprtype,3299 RegisterOperand o_zprtype>3300: I<(outs o_zprtype:$Zd), (ins PPR3bAny:$Pg, i_zprtype:$Zn),3301 asm, "\t$Zd, $Pg/z, $Zn",3302 "",3303 []>, Sched<[]> {3304 bits<3> Pg;3305 bits<5> Zd;3306 bits<5> Zn;3307 let Inst{31-24} = 0b01100100;3308 let Inst{23-22} = opc{6-5};3309 let Inst{21-19} = 0b011;3310 let Inst{18-16} = opc{4-2};3311 let Inst{15} = 0b1;3312 let Inst{14-13} = opc{1-0};3313 let Inst{12-10} = Pg;3314 let Inst{9-5} = Zn;3315 let Inst{4-0} = Zd;3316 3317 let hasSideEffects = 0;3318 let mayRaiseFPException = 1;3319}3320 3321multiclass sve_fp_z2op_p_zd<string asm, SDPatternOperator op> {3322 def _DtoS : sve_fp_z2op_p_zd<0b0001010, asm, ZPR64, ZPR32>;3323 3324 defm : SVE_3_Op_UndefZero_Pat<nxv4f32, op, nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>;3325}3326 3327multiclass sve_fp_z2op_p_zd_hsd<bits<5> opc, string asm, SDPatternOperator op> {3328 def _H : sve_fp_z2op_p_zd<{ 0b01, opc }, asm, ZPR16, ZPR16>;3329 def _S : sve_fp_z2op_p_zd<{ 0b10, opc }, asm, ZPR32, ZPR32>;3330 def _D : sve_fp_z2op_p_zd<{ 0b11, opc }, asm, ZPR64, ZPR64>;3331 3332 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;3333 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>;3334 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>;3335 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;3336 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>;3337 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;3338}3339 3340multiclass sve_fp_z2op_p_zd_frint<bits<2> opc, string asm> {3341 def _S : sve_fp_z2op_p_zd<{ 0b0010, opc{1}, 0, opc{0} }, asm, ZPR32, ZPR32>;3342 def _D : sve_fp_z2op_p_zd<{ 0b0010, opc{1}, 1, opc{0} }, asm, ZPR64, ZPR64>;3343}3344 3345multiclass sve_fp_z2op_p_zd_bfcvt<string asm, SDPatternOperator op> {3346 def NAME : sve_fp_z2op_p_zd<0b1001010, asm, ZPR32, ZPR16>;3347 3348 defm : SVE_3_Op_UndefZero_Pat<nxv8bf16, op, nxv8bf16, nxv4i1, nxv4f32, !cast<Instruction>(NAME)>;3349}3350 3351multiclass sve_fp_z2op_p_zd_d<bit U, string asm, string int_op, SDPatternOperator ir_op> {3352 def _HtoH : sve_fp_z2op_p_zd<{ 0b011101, U }, asm, ZPR16, ZPR16>;3353 def _HtoS : sve_fp_z2op_p_zd<{ 0b011110, U }, asm, ZPR16, ZPR32>;3354 def _HtoD : sve_fp_z2op_p_zd<{ 0b011111, U }, asm, ZPR16, ZPR64>;3355 def _StoS : sve_fp_z2op_p_zd<{ 0b101110, U }, asm, ZPR32, ZPR32>;3356 def _StoD : sve_fp_z2op_p_zd<{ 0b111110, U }, asm, ZPR32, ZPR64>;3357 def _DtoS : sve_fp_z2op_p_zd<{ 0b111100, U }, asm, ZPR64, ZPR32>;3358 def _DtoD : sve_fp_z2op_p_zd<{ 0b111111, U }, asm, ZPR64, ZPR64>;3359 3360 defm : SVE_3_Op_UndefZero_Pat<nxv4i32, !cast<SDPatternOperator>(int_op # _i32f64), nxv4i32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>;3361 defm : SVE_3_Op_UndefZero_Pat<nxv2i64, !cast<SDPatternOperator>(int_op # _i64f32), nxv2i64, nxv2i1, nxv4f32, !cast<Instruction>(NAME # _StoD)>;3362 defm : SVE_3_Op_UndefZero_Pat<nxv4i32, !cast<SDPatternOperator>(int_op # _i32f16), nxv4i32, nxv4i1, nxv8f16, !cast<Instruction>(NAME # _HtoS)>;3363 defm : SVE_3_Op_UndefZero_Pat<nxv2i64, !cast<SDPatternOperator>(int_op # _i64f16), nxv2i64, nxv2i1, nxv8f16, !cast<Instruction>(NAME # _HtoD)>;3364 3365 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv8i16, ir_op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _HtoH)>;3366 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv4i32, ir_op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _StoS)>;3367 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2i64, ir_op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoD)>;3368}3369 3370multiclass sve_fp_z2op_p_zd_c<bit U, string asm, string int_op, SDPatternOperator ir_op> {3371 def _HtoH : sve_fp_z2op_p_zd<{ 0b011001, U }, asm, ZPR16, ZPR16>;3372 def _StoH : sve_fp_z2op_p_zd<{ 0b011010, U }, asm, ZPR32, ZPR16>;3373 def _StoS : sve_fp_z2op_p_zd<{ 0b101010, U }, asm, ZPR32, ZPR32>;3374 def _StoD : sve_fp_z2op_p_zd<{ 0b111000, U }, asm, ZPR32, ZPR64>;3375 def _DtoS : sve_fp_z2op_p_zd<{ 0b111010, U }, asm, ZPR64, ZPR32>;3376 def _DtoH : sve_fp_z2op_p_zd<{ 0b011011, U }, asm, ZPR64, ZPR16>;3377 def _DtoD : sve_fp_z2op_p_zd<{ 0b111011, U }, asm, ZPR64, ZPR64>;3378 3379 defm : SVE_3_Op_UndefZero_Pat<nxv4f32, !cast<SDPatternOperator>(int_op # _f32i64), nxv4f32, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _DtoS)>;3380 defm : SVE_3_Op_UndefZero_Pat<nxv2f64, !cast<SDPatternOperator>(int_op # _f64i32), nxv2f64, nxv2i1, nxv4i32, !cast<Instruction>(NAME # _StoD)>;3381 defm : SVE_3_Op_UndefZero_Pat<nxv8f16, !cast<SDPatternOperator>(int_op # _f16i32), nxv8f16, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _StoH)>;3382 defm : SVE_3_Op_UndefZero_Pat<nxv8f16, !cast<SDPatternOperator>(int_op # _f16i64), nxv8f16, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _DtoH)>;3383 3384 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv8f16, ir_op, nxv8i1,nxv8i16, !cast<Instruction>(NAME # _HtoH)>;3385 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv4f32, ir_op, nxv4i1,nxv4i32, !cast<Instruction>(NAME # _StoS)>;3386 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2f64, ir_op, nxv2i1,nxv2i64, !cast<Instruction>(NAME # _DtoD)>;3387}3388 3389multiclass sve_fp_z2op_p_zd_d_flogb<string asm, SDPatternOperator op> {3390 def _H : sve_fp_z2op_p_zd<0b0011001, asm, ZPR16, ZPR16>;3391 def _S : sve_fp_z2op_p_zd<0b0011010, asm, ZPR32, ZPR32>;3392 def _D : sve_fp_z2op_p_zd<0b0011011, asm, ZPR64, ZPR64>;3393 3394 defm : SVE_3_Op_UndefZero_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;3395 defm : SVE_3_Op_UndefZero_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;3396 defm : SVE_3_Op_UndefZero_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;3397}3398 3399multiclass sve_fp_z2op_p_zd_b_0<string asm, string op> {3400 def _StoH : sve_fp_z2op_p_zd<0b1001000, asm, ZPR32, ZPR16>;3401 def _HtoS : sve_fp_z2op_p_zd<0b1001001, asm, ZPR16, ZPR32>;3402 def _DtoH : sve_fp_z2op_p_zd<0b1101000, asm, ZPR64, ZPR16>;3403 def _HtoD : sve_fp_z2op_p_zd<0b1101001, asm, ZPR16, ZPR64>;3404 def _DtoS : sve_fp_z2op_p_zd<0b1101010, asm, ZPR64, ZPR32>;3405 def _StoD : sve_fp_z2op_p_zd<0b1101011, asm, ZPR32, ZPR64>;3406 3407 defm : SVE_3_Op_UndefZero_Pat<nxv8f16, !cast<SDPatternOperator>(op # _f16f32), nxv8f16, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _StoH)>;3408 defm : SVE_3_Op_UndefZero_Pat<nxv8f16, !cast<SDPatternOperator>(op # _f16f64), nxv8f16, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoH)>;3409 defm : SVE_3_Op_UndefZero_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>;3410 defm : SVE_3_Op_UndefZero_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f16), nxv4f32, nxv4i1, nxv8f16, !cast<Instruction>(NAME # _HtoS)>;3411 defm : SVE_3_Op_UndefZero_Pat<nxv2f64, !cast<SDPatternOperator>(op # _f64f16), nxv2f64, nxv2i1, nxv8f16, !cast<Instruction>(NAME # _HtoD)>;3412 defm : SVE_3_Op_UndefZero_Pat<nxv2f64, !cast<SDPatternOperator>(op # _f64f32), nxv2f64, nxv2i1, nxv4f32, !cast<Instruction>(NAME # _StoD)>;3413}3414 3415//===----------------------------------------------------------------------===//3416// SVE Integer Arithmetic - Binary Predicated Group3417//===----------------------------------------------------------------------===//3418 3419class sve_int_bin_pred_arit_log<bits<2> sz8_64, bits<2> fmt, bits<3> opc,3420 string asm, ZPRRegOp zprty>3421: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm),3422 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", "", []>, Sched<[]> {3423 bits<3> Pg;3424 bits<5> Zdn;3425 bits<5> Zm;3426 let Inst{31-24} = 0b00000100;3427 let Inst{23-22} = sz8_64;3428 let Inst{21} = 0b0;3429 let Inst{20-19} = fmt;3430 let Inst{18-16} = opc;3431 let Inst{15-13} = 0b000;3432 let Inst{12-10} = Pg;3433 let Inst{9-5} = Zm;3434 let Inst{4-0} = Zdn;3435 3436 let Constraints = "$Zdn = $_Zdn";3437 let DestructiveInstType = DestructiveOther;3438 let ElementSize = zprty.ElementSize;3439 let hasSideEffects = 0;3440}3441 3442multiclass sve_int_bin_pred_log<bits<3> opc, string asm, string Ps,3443 SDPatternOperator op,3444 DestructiveInstTypeEnum flags> {3445 let DestructiveInstType = flags in {3446 def _B : sve_int_bin_pred_arit_log<0b00, 0b11, opc, asm, ZPR8>,3447 SVEPseudo2Instr<Ps # _B, 1>;3448 def _H : sve_int_bin_pred_arit_log<0b01, 0b11, opc, asm, ZPR16>,3449 SVEPseudo2Instr<Ps # _H, 1>;3450 def _S : sve_int_bin_pred_arit_log<0b10, 0b11, opc, asm, ZPR32>,3451 SVEPseudo2Instr<Ps # _S, 1>;3452 def _D : sve_int_bin_pred_arit_log<0b11, 0b11, opc, asm, ZPR64>,3453 SVEPseudo2Instr<Ps # _D, 1>;3454 }3455 3456 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;3457 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;3458 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;3459 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;3460}3461 3462multiclass sve_int_bin_pred_arit_0<bits<3> opc, string asm, string Ps,3463 SDPatternOperator op,3464 DestructiveInstTypeEnum flags,3465 string revname="", bit isReverseInstr=0> {3466 let DestructiveInstType = flags in {3467 def _B : sve_int_bin_pred_arit_log<0b00, 0b00, opc, asm, ZPR8>,3468 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>;3469 def _H : sve_int_bin_pred_arit_log<0b01, 0b00, opc, asm, ZPR16>,3470 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>;3471 def _S : sve_int_bin_pred_arit_log<0b10, 0b00, opc, asm, ZPR32>,3472 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>;3473 def _D : sve_int_bin_pred_arit_log<0b11, 0b00, opc, asm, ZPR64>,3474 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>;3475 }3476 3477 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;3478 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;3479 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;3480 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;3481}3482 3483multiclass sve_int_bin_pred_arit_1<bits<3> opc, string asm, string Ps,3484 SDPatternOperator op,3485 DestructiveInstTypeEnum flags> {3486 let DestructiveInstType = flags in {3487 def _B : sve_int_bin_pred_arit_log<0b00, 0b01, opc, asm, ZPR8>,3488 SVEPseudo2Instr<Ps # _B, 1>;3489 def _H : sve_int_bin_pred_arit_log<0b01, 0b01, opc, asm, ZPR16>,3490 SVEPseudo2Instr<Ps # _H, 1>;3491 def _S : sve_int_bin_pred_arit_log<0b10, 0b01, opc, asm, ZPR32>,3492 SVEPseudo2Instr<Ps # _S, 1>;3493 def _D : sve_int_bin_pred_arit_log<0b11, 0b01, opc, asm, ZPR64>,3494 SVEPseudo2Instr<Ps # _D, 1>;3495 }3496 3497 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;3498 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;3499 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;3500 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;3501}3502 3503multiclass sve_int_bin_pred_arit_2<bits<3> opc, string asm, string Ps,3504 SDPatternOperator op,3505 DestructiveInstTypeEnum flags> {3506 let DestructiveInstType = flags in {3507 def _B : sve_int_bin_pred_arit_log<0b00, 0b10, opc, asm, ZPR8>,3508 SVEPseudo2Instr<Ps # _B, 1>;3509 def _H : sve_int_bin_pred_arit_log<0b01, 0b10, opc, asm, ZPR16>,3510 SVEPseudo2Instr<Ps # _H, 1>;3511 def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>,3512 SVEPseudo2Instr<Ps # _S, 1>;3513 def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>,3514 SVEPseudo2Instr<Ps # _D, 1>;3515 }3516 3517 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;3518 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;3519 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;3520 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;3521}3522 3523// Special case for divides which are not defined for 8b/16b elements.3524multiclass sve_int_bin_pred_arit_2_div<bits<3> opc, string asm, string Ps,3525 SDPatternOperator op,3526 DestructiveInstTypeEnum flags,3527 string revname="", bit isReverseInstr=0> {3528 let DestructiveInstType = flags in {3529 def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>,3530 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>;3531 def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>,3532 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>;3533 }3534 3535 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;3536 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;3537}3538 3539//===----------------------------------------------------------------------===//3540// SVE Integer Multiply-Add Group3541//===----------------------------------------------------------------------===//3542 3543class sve_int_mladdsub_vvv_pred<bits<2> sz8_64, bits<1> opc, string asm,3544 ZPRRegOp zprty>3545: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, zprty:$Za),3546 asm, "\t$Zdn, $Pg/m, $Zm, $Za",3547 "",3548 []>, Sched<[]> {3549 bits<3> Pg;3550 bits<5> Zdn;3551 bits<5> Za;3552 bits<5> Zm;3553 let Inst{31-24} = 0b00000100;3554 let Inst{23-22} = sz8_64;3555 let Inst{21} = 0b0;3556 let Inst{20-16} = Zm;3557 let Inst{15-14} = 0b11;3558 let Inst{13} = opc;3559 let Inst{12-10} = Pg;3560 let Inst{9-5} = Za;3561 let Inst{4-0} = Zdn;3562 3563 let Constraints = "$Zdn = $_Zdn";3564 let DestructiveInstType = DestructiveOther;3565 let ElementSize = zprty.ElementSize;3566 let hasSideEffects = 0;3567}3568 3569multiclass sve_int_mladdsub_vvv_pred<bits<1> opc, string asm, SDPatternOperator op,3570 string revname, bit isReverseInstr=0> {3571 def _B : sve_int_mladdsub_vvv_pred<0b00, opc, asm, ZPR8>,3572 SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>;3573 def _H : sve_int_mladdsub_vvv_pred<0b01, opc, asm, ZPR16>,3574 SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>;3575 def _S : sve_int_mladdsub_vvv_pred<0b10, opc, asm, ZPR32>,3576 SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>;3577 def _D : sve_int_mladdsub_vvv_pred<0b11, opc, asm, ZPR64>,3578 SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>;3579 3580 def : SVE_4_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;3581 def : SVE_4_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;3582 def : SVE_4_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;3583 def : SVE_4_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;3584}3585 3586class sve_int_mlas_vvv_pred<bits<2> sz8_64, bits<1> opc, string asm,3587 ZPRRegOp zprty>3588: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm),3589 asm, "\t$Zda, $Pg/m, $Zn, $Zm",3590 "",3591 []>, Sched<[]> {3592 bits<3> Pg;3593 bits<5> Zda;3594 bits<5> Zm;3595 bits<5> Zn;3596 let Inst{31-24} = 0b00000100;3597 let Inst{23-22} = sz8_64;3598 let Inst{21} = 0b0;3599 let Inst{20-16} = Zm;3600 let Inst{15-14} = 0b01;3601 let Inst{13} = opc;3602 let Inst{12-10} = Pg;3603 let Inst{9-5} = Zn;3604 let Inst{4-0} = Zda;3605 3606 let Constraints = "$Zda = $_Zda";3607 let DestructiveInstType = DestructiveTernaryCommWithRev;3608 let ElementSize = zprty.ElementSize;3609 let hasSideEffects = 0;3610}3611 3612multiclass sve_int_mlas_vvv_pred<bits<1> opc, string asm, SDPatternOperator op,3613 string Ps, string revname, bit isReverseInstr=0> {3614 def _B : sve_int_mlas_vvv_pred<0b00, opc, asm, ZPR8>,3615 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>;3616 def _H : sve_int_mlas_vvv_pred<0b01, opc, asm, ZPR16>,3617 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>;3618 def _S : sve_int_mlas_vvv_pred<0b10, opc, asm, ZPR32>,3619 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>;3620 def _D : sve_int_mlas_vvv_pred<0b11, opc, asm, ZPR64>,3621 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>;3622 3623 def : SVE_4_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;3624 def : SVE_4_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;3625 def : SVE_4_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;3626 def : SVE_4_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;3627}3628 3629//class for generating pseudo for SVE MLA/MAD/MLS/MSB3630multiclass sve_int_3op_p_mladdsub<SDPatternOperator op> {3631 def _B_UNDEF : PredThreeOpPseudo<NAME # _B, ZPR8, FalseLanesUndef>;3632 def _H_UNDEF : PredThreeOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>;3633 def _S_UNDEF : PredThreeOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>;3634 def _D_UNDEF : PredThreeOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>;3635 3636 let AddedComplexity = 9 in {3637 def : SVE_4_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B_UNDEF)>;3638 def : SVE_4_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H_UNDEF)>;3639 def : SVE_4_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S_UNDEF)>;3640 def : SVE_4_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D_UNDEF)>;3641 }3642}3643 3644//===----------------------------------------------------------------------===//3645// SVE2 Integer Multiply-Add - Unpredicated Group3646//===----------------------------------------------------------------------===//3647 3648class sve2_int_mla<bits<2> sz, bits<5> opc, string asm,3649 ZPRRegOp zprty1, ZPRRegOp zprty2>3650: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm),3651 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> {3652 bits<5> Zda;3653 bits<5> Zn;3654 bits<5> Zm;3655 let Inst{31-24} = 0b01000100;3656 let Inst{23-22} = sz;3657 let Inst{21} = 0b0;3658 let Inst{20-16} = Zm;3659 let Inst{15} = 0b0;3660 let Inst{14-10} = opc;3661 let Inst{9-5} = Zn;3662 let Inst{4-0} = Zda;3663 3664 let Constraints = "$Zda = $_Zda";3665 let DestructiveInstType = DestructiveOther;3666 let ElementSize = ElementSizeNone;3667 let hasSideEffects = 0;3668}3669 3670multiclass sve2_int_mla<bit S, string asm, SDPatternOperator op> {3671 def _B : sve2_int_mla<0b00, { 0b1110, S }, asm, ZPR8, ZPR8>;3672 def _H : sve2_int_mla<0b01, { 0b1110, S }, asm, ZPR16, ZPR16>;3673 def _S : sve2_int_mla<0b10, { 0b1110, S }, asm, ZPR32, ZPR32>;3674 def _D : sve2_int_mla<0b11, { 0b1110, S }, asm, ZPR64, ZPR64>;3675 3676 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;3677 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;3678 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;3679 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;3680}3681 3682multiclass sve2_int_mla_long<bits<5> opc, string asm, SDPatternOperator op> {3683 def _H : sve2_int_mla<0b01, opc, asm, ZPR16, ZPR8>;3684 def _S : sve2_int_mla<0b10, opc, asm, ZPR32, ZPR16>;3685 def _D : sve2_int_mla<0b11, opc, asm, ZPR64, ZPR32>;3686 3687 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>;3688 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>;3689 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>;3690}3691 3692//===----------------------------------------------------------------------===//3693// SVE2 Integer Multiply-Add - Indexed Group3694//===----------------------------------------------------------------------===//3695 3696class sve2_int_mla_by_indexed_elem<bits<2> sz, bits<6> opc, string asm,3697 ZPRRegOp zprty1, ZPRRegOp zprty2,3698 ZPRRegOp zprty3, Operand itype>3699: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop),3700 asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> {3701 bits<5> Zda;3702 bits<5> Zn;3703 let Inst{31-24} = 0b01000100;3704 let Inst{23-22} = sz;3705 let Inst{21} = 0b1;3706 let Inst{15-10} = opc;3707 let Inst{9-5} = Zn;3708 let Inst{4-0} = Zda;3709 3710 let Constraints = "$Zda = $_Zda";3711 let DestructiveInstType = DestructiveOther;3712 let ElementSize = ElementSizeNone;3713 let hasSideEffects = 0;3714}3715 3716multiclass sve2_int_mla_by_indexed_elem<bits<2> opc, bit S, string asm,3717 SDPatternOperator op> {3718 def _H : sve2_int_mla_by_indexed_elem<{0, ?}, { 0b000, opc, S }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> {3719 bits<3> Zm;3720 bits<3> iop;3721 let Inst{22} = iop{2};3722 let Inst{20-19} = iop{1-0};3723 let Inst{18-16} = Zm;3724 }3725 def _S : sve2_int_mla_by_indexed_elem<0b10, { 0b000, opc, S }, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> {3726 bits<3> Zm;3727 bits<2> iop;3728 let Inst{20-19} = iop;3729 let Inst{18-16} = Zm;3730 }3731 def _D : sve2_int_mla_by_indexed_elem<0b11, { 0b000, opc, S }, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> {3732 bits<4> Zm;3733 bit iop;3734 let Inst{20} = iop;3735 let Inst{19-16} = Zm;3736 }3737 3738 def : SVE_4_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>;3739 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>;3740 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>;3741}3742 3743//===----------------------------------------------------------------------===//3744// SVE2 Integer Multiply-Add Long - Indexed Group3745//===----------------------------------------------------------------------===//3746 3747multiclass sve2_int_mla_long_by_indexed_elem<bits<4> opc, string asm,3748 SDPatternOperator op> {3749 def _S : sve2_int_mla_by_indexed_elem<0b10, { opc{3}, 0b0, opc{2-1}, ?, opc{0} },3750 asm, ZPR32, ZPR16, ZPR3b16, VectorIndexH32b> {3751 bits<3> Zm;3752 bits<3> iop;3753 let Inst{20-19} = iop{2-1};3754 let Inst{18-16} = Zm;3755 let Inst{11} = iop{0};3756 }3757 def _D : sve2_int_mla_by_indexed_elem<0b11, { opc{3}, 0b0, opc{2-1}, ?, opc{0} },3758 asm, ZPR64, ZPR32, ZPR4b32, VectorIndexS32b> {3759 bits<4> Zm;3760 bits<2> iop;3761 let Inst{20} = iop{1};3762 let Inst{19-16} = Zm;3763 let Inst{11} = iop{0};3764 }3765 3766 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _S)>;3767 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _D)>;3768}3769 3770//===----------------------------------------------------------------------===//3771// SVE Integer Dot Product Group3772//===----------------------------------------------------------------------===//3773 3774class sve_intx_dot<bits<2> sz, bits<5> op5, bit U, string asm,3775 ZPRRegOp zprty1, ZPRRegOp zprty2>3776: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), asm,3777 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> {3778 bits<5> Zda;3779 bits<5> Zn;3780 bits<5> Zm;3781 let Inst{31-24} = 0b01000100;3782 let Inst{23-22} = sz;3783 let Inst{21} = 0;3784 let Inst{20-16} = Zm;3785 let Inst{15-11} = op5;3786 let Inst{10} = U;3787 let Inst{9-5} = Zn;3788 let Inst{4-0} = Zda;3789 3790 let Constraints = "$Zda = $_Zda";3791 let DestructiveInstType = DestructiveOther;3792 let hasSideEffects = 0;3793}3794 3795multiclass sve_intx_dot<bit opc, string asm, SDPatternOperator op> {3796 def _BtoS : sve_intx_dot<0b10, 0b00000, opc, asm, ZPR32, ZPR8>;3797 def _HtoD : sve_intx_dot<0b11, 0b00000, opc, asm, ZPR64, ZPR16>;3798 3799 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _BtoS)>;3800 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _HtoD)>;3801}3802 3803multiclass sve2p1_two_way_dot_vv<string mnemonic, bit u, SDPatternOperator intrinsic> {3804 def NAME : sve_intx_dot<0b00, 0b11001, u, mnemonic, ZPR32, ZPR16>;3805 3806 def : SVE_3_Op_Pat<nxv4i32, intrinsic, nxv4i32, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>;3807}3808 3809//===----------------------------------------------------------------------===//3810// SVE Integer Dot Product Group - Indexed Group3811//===----------------------------------------------------------------------===//3812 3813class sve_intx_dot_by_indexed_elem<bit U, string asm,3814 ZPRRegOp zprty1, ZPRRegOp zprty2,3815 ZPRRegOp zprty3, Operand itype>3816: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop),3817 asm, "\t$Zda, $Zn, $Zm$iop",3818 "", []>, Sched<[]> {3819 bits<5> Zda;3820 bits<5> Zn;3821 let Inst{31-24} = 0b01000100;3822 let Inst{21} = 0b1;3823 let Inst{15-11} = 0;3824 let Inst{10} = U;3825 let Inst{9-5} = Zn;3826 let Inst{4-0} = Zda;3827 3828 let Constraints = "$Zda = $_Zda";3829 let DestructiveInstType = DestructiveOther;3830 let hasSideEffects = 0;3831}3832 3833multiclass sve_intx_dot_by_indexed_elem<bit opc, string asm,3834 SDPatternOperator op> {3835 def _BtoS : sve_intx_dot_by_indexed_elem<opc, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b_timm> {3836 bits<2> iop;3837 bits<3> Zm;3838 let Inst{23-22} = 0b10;3839 let Inst{20-19} = iop;3840 let Inst{18-16} = Zm;3841 }3842 def _HtoD : sve_intx_dot_by_indexed_elem<opc, asm, ZPR64, ZPR16, ZPR4b16, VectorIndexD32b_timm> {3843 bits<1> iop;3844 bits<4> Zm;3845 let Inst{23-22} = 0b11;3846 let Inst{20} = iop;3847 let Inst{19-16} = Zm;3848 }3849 3850 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _BtoS)>;3851 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv8i16, nxv8i16, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _HtoD)>;3852}3853 3854class sve_intx_dot_by_indexed_elem_x<bit opc, string asm>3855: sve_intx_dot_by_indexed_elem<opc, asm, ZPR16, ZPR8, ZPR3b8, VectorIndexH32b_timm> {3856 bits<3> iop;3857 bits<3> Zm;3858 let Inst{23} = 0b0;3859 let Inst{22} = iop{2};3860 let Inst{20-19} = iop{1-0};3861 let Inst{18-16} = Zm;3862}3863 3864//===----------------------------------------------------------------------===//3865// SVE2 Complex Integer Dot Product Group3866//===----------------------------------------------------------------------===//3867 3868class sve2_complex_int_arith<bits<2> sz, bits<4> opc, string asm,3869 ZPRRegOp zprty1, ZPRRegOp zprty2>3870: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm,3871 complexrotateop:$rot),3872 asm, "\t$Zda, $Zn, $Zm, $rot", "", []>, Sched<[]> {3873 bits<5> Zda;3874 bits<5> Zn;3875 bits<5> Zm;3876 bits<2> rot;3877 let Inst{31-24} = 0b01000100;3878 let Inst{23-22} = sz;3879 let Inst{21} = 0b0;3880 let Inst{20-16} = Zm;3881 let Inst{15-12} = opc;3882 let Inst{11-10} = rot;3883 let Inst{9-5} = Zn;3884 let Inst{4-0} = Zda;3885 3886 let Constraints = "$Zda = $_Zda";3887 let DestructiveInstType = DestructiveOther;3888 let ElementSize = ElementSizeNone;3889 let hasSideEffects = 0;3890}3891 3892multiclass sve2_cintx_dot<string asm, SDPatternOperator op> {3893 def _S : sve2_complex_int_arith<0b10, 0b0001, asm, ZPR32, ZPR8>;3894 def _D : sve2_complex_int_arith<0b11, 0b0001, asm, ZPR64, ZPR16>;3895 3896 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv16i8 ZPR8:$Op2), (nxv16i8 ZPR8:$Op3),3897 (i32 complexrotateop:$imm))),3898 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR8:$Op2, ZPR8:$Op3, complexrotateop:$imm)>;3899 def : Pat<(nxv2i64 (op (nxv2i64 ZPR64:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3),3900 (i32 complexrotateop:$imm))),3901 (!cast<Instruction>(NAME # "_D") ZPR64:$Op1, ZPR16:$Op2, ZPR16:$Op3, complexrotateop:$imm)>;3902}3903 3904//===----------------------------------------------------------------------===//3905// SVE2 Complex Multiply-Add Group3906//===----------------------------------------------------------------------===//3907 3908multiclass sve2_int_cmla<bit opc, string asm, SDPatternOperator op> {3909 def _B : sve2_complex_int_arith<0b00, { 0b001, opc }, asm, ZPR8, ZPR8>;3910 def _H : sve2_complex_int_arith<0b01, { 0b001, opc }, asm, ZPR16, ZPR16>;3911 def _S : sve2_complex_int_arith<0b10, { 0b001, opc }, asm, ZPR32, ZPR32>;3912 def _D : sve2_complex_int_arith<0b11, { 0b001, opc }, asm, ZPR64, ZPR64>;3913 3914 def : SVE_4_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, i32, complexrotateop, !cast<Instruction>(NAME # _B)>;3915 def : SVE_4_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, i32, complexrotateop, !cast<Instruction>(NAME # _H)>;3916 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, i32, complexrotateop, !cast<Instruction>(NAME # _S)>;3917 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, i32, complexrotateop, !cast<Instruction>(NAME # _D)>;3918}3919 3920//===----------------------------------------------------------------------===//3921// SVE2 Complex Integer Dot Product - Indexed Group3922//===----------------------------------------------------------------------===//3923 3924class sve2_complex_int_arith_indexed<bits<2> sz, bits<4> opc, string asm,3925 ZPRRegOp zprty1, ZPRRegOp zprty2,3926 ZPRRegOp zprty3, Operand itype>3927: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop,3928 complexrotateop:$rot),3929 asm, "\t$Zda, $Zn, $Zm$iop, $rot", "", []>, Sched<[]> {3930 bits<5> Zda;3931 bits<5> Zn;3932 bits<2> rot;3933 let Inst{31-24} = 0b01000100;3934 let Inst{23-22} = sz;3935 let Inst{21} = 0b1;3936 let Inst{15-12} = opc;3937 let Inst{11-10} = rot;3938 let Inst{9-5} = Zn;3939 let Inst{4-0} = Zda;3940 3941 let Constraints = "$Zda = $_Zda";3942 let DestructiveInstType = DestructiveOther;3943 let ElementSize = ElementSizeNone;3944 let hasSideEffects = 0;3945}3946 3947multiclass sve2_cintx_dot_by_indexed_elem<string asm, SDPatternOperator op> {3948 def _S : sve2_complex_int_arith_indexed<0b10, 0b0100, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b> {3949 bits<2> iop;3950 bits<3> Zm;3951 let Inst{20-19} = iop;3952 let Inst{18-16} = Zm;3953 }3954 def _D : sve2_complex_int_arith_indexed<0b11, 0b0100, asm, ZPR64, ZPR16, ZPR4b16, VectorIndexD32b> {3955 bit iop;3956 bits<4> Zm;3957 let Inst{20} = iop;3958 let Inst{19-16} = Zm;3959 }3960 3961 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv16i8 ZPR8:$Op2), (nxv16i8 ZPR8:$Op3),3962 (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))),3963 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR8:$Op2, ZPR8:$Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>;3964 def : Pat<(nxv2i64 (op (nxv2i64 ZPR64:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3),3965 (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))),3966 (!cast<Instruction>(NAME # "_D") ZPR64:$Op1, ZPR16:$Op2, ZPR16:$Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>;3967}3968 3969//===----------------------------------------------------------------------===//3970// SVE2 Complex Multiply-Add - Indexed Group3971//===----------------------------------------------------------------------===//3972 3973multiclass sve2_cmla_by_indexed_elem<bit opc, string asm,3974 SDPatternOperator op> {3975 def _H : sve2_complex_int_arith_indexed<0b10, { 0b011, opc }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexS32b> {3976 bits<2> iop;3977 bits<3> Zm;3978 let Inst{20-19} = iop;3979 let Inst{18-16} = Zm;3980 }3981 def _S : sve2_complex_int_arith_indexed<0b11, { 0b011, opc }, asm, ZPR32, ZPR32, ZPR4b32, VectorIndexD32b> {3982 bit iop;3983 bits<4> Zm;3984 let Inst{20} = iop;3985 let Inst{19-16} = Zm;3986 }3987 3988 def : Pat<(nxv8i16 (op (nxv8i16 ZPR16:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3),3989 (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))),3990 (!cast<Instruction>(NAME # "_H") ZPR16:$Op1, ZPR16:$Op2, ZPR16:$Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>;3991 3992 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv4i32 ZPR32:$Op2), (nxv4i32 ZPR32:$Op3),3993 (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))),3994 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR32:$Op2, ZPR32:$Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>;3995}3996 3997//===----------------------------------------------------------------------===//3998// SVE2 Integer Multiply - Unpredicated Group3999//===----------------------------------------------------------------------===//4000 4001class sve2_int_mul<bits<2> sz, bits<3> opc, string asm, ZPRRegOp zprty>4002: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm),4003 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> {4004 bits<5> Zd;4005 bits<5> Zm;4006 bits<5> Zn;4007 let Inst{31-24} = 0b00000100;4008 let Inst{23-22} = sz;4009 let Inst{21} = 0b1;4010 let Inst{20-16} = Zm;4011 let Inst{15-13} = 0b011;4012 let Inst{12-10} = opc;4013 let Inst{9-5} = Zn;4014 let Inst{4-0} = Zd;4015 4016 let hasSideEffects = 0;4017}4018 4019multiclass sve2_int_mul<bits<3> opc, string asm, SDPatternOperator op> {4020 def _B : sve2_int_mul<0b00, opc, asm, ZPR8>;4021 def _H : sve2_int_mul<0b01, opc, asm, ZPR16>;4022 def _S : sve2_int_mul<0b10, opc, asm, ZPR32>;4023 def _D : sve2_int_mul<0b11, opc, asm, ZPR64>;4024 4025 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;4026 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;4027 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;4028 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;4029}4030 4031multiclass sve2_int_mul_single<bits<3> opc, string asm, SDPatternOperator op> {4032 def _B : sve2_int_mul<0b00, opc, asm, ZPR8>;4033 4034 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;4035}4036 4037//===----------------------------------------------------------------------===//4038// SVE2 Integer Multiply - Indexed Group4039//===----------------------------------------------------------------------===//4040 4041class sve2_int_mul_by_indexed_elem<bits<2> sz, bits<4> opc, string asm,4042 ZPRRegOp zprty1, ZPRRegOp zprty2,4043 ZPRRegOp zprty3, Operand itype>4044: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm, itype:$iop),4045 asm, "\t$Zd, $Zn, $Zm$iop", "", []>, Sched<[]> {4046 bits<5> Zd;4047 bits<5> Zn;4048 let Inst{31-24} = 0b01000100;4049 let Inst{23-22} = sz;4050 let Inst{21} = 0b1;4051 let Inst{15-14} = 0b11;4052 let Inst{13-10} = opc;4053 let Inst{9-5} = Zn;4054 let Inst{4-0} = Zd;4055 4056 let hasSideEffects = 0;4057}4058 4059multiclass sve2_int_mul_by_indexed_elem<bits<4> opc, string asm,4060 SDPatternOperator op> {4061 def _H : sve2_int_mul_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> {4062 bits<3> Zm;4063 bits<3> iop;4064 let Inst{22} = iop{2};4065 let Inst{20-19} = iop{1-0};4066 let Inst{18-16} = Zm;4067 }4068 def _S : sve2_int_mul_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> {4069 bits<3> Zm;4070 bits<2> iop;4071 let Inst{20-19} = iop;4072 let Inst{18-16} = Zm;4073 }4074 def _D : sve2_int_mul_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> {4075 bits<4> Zm;4076 bit iop;4077 let Inst{20} = iop;4078 let Inst{19-16} = Zm;4079 }4080 4081 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>;4082 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>;4083 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>;4084}4085 4086multiclass sve2_int_mul_long_by_indexed_elem<bits<3> opc, string asm,4087 SDPatternOperator op> {4088 def _S : sve2_int_mul_by_indexed_elem<0b10, { opc{2-1}, ?, opc{0} }, asm,4089 ZPR32, ZPR16, ZPR3b16, VectorIndexH32b> {4090 bits<3> Zm;4091 bits<3> iop;4092 let Inst{20-19} = iop{2-1};4093 let Inst{18-16} = Zm;4094 let Inst{11} = iop{0};4095 }4096 def _D : sve2_int_mul_by_indexed_elem<0b11, { opc{2-1}, ?, opc{0} }, asm,4097 ZPR64, ZPR32, ZPR4b32, VectorIndexS32b> {4098 bits<4> Zm;4099 bits<2> iop;4100 let Inst{20} = iop{1};4101 let Inst{19-16} = Zm;4102 let Inst{11} = iop{0};4103 }4104 4105 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _S)>;4106 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _D)>;4107}4108 4109//===----------------------------------------------------------------------===//4110// SVE2 Integer - Predicated Group4111//===----------------------------------------------------------------------===//4112 4113class sve2_int_arith_pred<bits<2> sz, bits<6> opc, string asm,4114 ZPRRegOp zprty>4115: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm),4116 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", "", []>, Sched<[]> {4117 bits<3> Pg;4118 bits<5> Zm;4119 bits<5> Zdn;4120 let Inst{31-24} = 0b01000100;4121 let Inst{23-22} = sz;4122 let Inst{21} = 0b0;4123 let Inst{20-16} = opc{5-1};4124 let Inst{15-14} = 0b10;4125 let Inst{13} = opc{0};4126 let Inst{12-10} = Pg;4127 let Inst{9-5} = Zm;4128 let Inst{4-0} = Zdn;4129 4130 let Constraints = "$Zdn = $_Zdn";4131 let DestructiveInstType = DestructiveOther;4132 let ElementSize = zprty.ElementSize;4133 let hasSideEffects = 0;4134}4135 4136multiclass sve2_int_arith_pred<bits<6> opc, string asm, SDPatternOperator op,4137 string Ps = "",4138 DestructiveInstTypeEnum flags=DestructiveOther,4139 string revname="", bit isReverseInstr=0> {4140 let DestructiveInstType = flags in {4141 def _B : sve2_int_arith_pred<0b00, opc, asm, ZPR8>,4142 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>;4143 def _H : sve2_int_arith_pred<0b01, opc, asm, ZPR16>,4144 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>;4145 def _S : sve2_int_arith_pred<0b10, opc, asm, ZPR32>,4146 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>;4147 def _D : sve2_int_arith_pred<0b11, opc, asm, ZPR64>,4148 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>;4149 }4150 4151 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;4152 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;4153 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;4154 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;4155}4156 4157class sve2_int_sadd_long_accum_pairwise<bits<2> sz, bit U, string asm,4158 ZPRRegOp zprty1, ZPRRegOp zprty2>4159: I<(outs zprty1:$Zda), (ins PPR3bAny:$Pg, zprty1:$_Zda, zprty2:$Zn),4160 asm, "\t$Zda, $Pg/m, $Zn", "", []>, Sched<[]> {4161 bits<3> Pg;4162 bits<5> Zn;4163 bits<5> Zda;4164 let Inst{31-24} = 0b01000100;4165 let Inst{23-22} = sz;4166 let Inst{21-17} = 0b00010;4167 let Inst{16} = U;4168 let Inst{15-13} = 0b101;4169 let Inst{12-10} = Pg;4170 let Inst{9-5} = Zn;4171 let Inst{4-0} = Zda;4172 4173 let Constraints = "$Zda = $_Zda";4174 let DestructiveInstType = DestructiveOther;4175 let ElementSize = zprty1.ElementSize;4176 let hasSideEffects = 0;4177}4178 4179multiclass sve2_int_sadd_long_accum_pairwise<bit U, string asm, SDPatternOperator op> {4180 def _H : sve2_int_sadd_long_accum_pairwise<0b01, U, asm, ZPR16, ZPR8>;4181 def _S : sve2_int_sadd_long_accum_pairwise<0b10, U, asm, ZPR32, ZPR16>;4182 def _D : sve2_int_sadd_long_accum_pairwise<0b11, U, asm, ZPR64, ZPR32>;4183 4184 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>;4185 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv8i16, !cast<Instruction>(NAME # _S)>;4186 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>;4187}4188 4189class sve2_int_un_pred_arit<bits<2> sz, bits<2> opc,4190 string asm, ZPRRegOp zprty>4191: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn),4192 asm, "\t$Zd, $Pg/m, $Zn",4193 "",4194 []>, Sched<[]> {4195 bits<3> Pg;4196 bits<5> Zd;4197 bits<5> Zn;4198 let Inst{31-24} = 0b01000100;4199 let Inst{23-22} = sz;4200 let Inst{21-20} = 0b00;4201 let Inst{19} = opc{1};4202 let Inst{18-17} = 0b00;4203 let Inst{16} = opc{0};4204 let Inst{15-13} = 0b101;4205 let Inst{12-10} = Pg;4206 let Inst{9-5} = Zn;4207 let Inst{4-0} = Zd;4208 let Constraints = "$Zd = $_Zd";4209 let DestructiveInstType = DestructiveUnaryPassthru;4210 let ElementSize = zprty.ElementSize;4211 let hasSideEffects = 0;4212}4213 4214class sve2_int_un_pred_arit_z<bits<2> sz, bits<2> opc,4215 string asm, ZPRRegOp zprty>4216: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn),4217 asm, "\t$Zd, $Pg/z, $Zn",4218 "",4219 []>, Sched<[]> {4220 bits<3> Pg;4221 bits<5> Zd;4222 bits<5> Zn;4223 let Inst{31-24} = 0b01000100;4224 let Inst{23-22} = sz;4225 let Inst{21-20} = 0b00;4226 let Inst{19} = opc{1};4227 let Inst{18-17} = 0b01;4228 let Inst{16} = opc{0};4229 let Inst{15-13} = 0b101;4230 let Inst{12-10} = Pg;4231 let Inst{9-5} = Zn;4232 let Inst{4-0} = Zd;4233 let hasSideEffects = 0;4234}4235 4236multiclass sve2_int_un_pred_arit_s<bits<2> opc, string asm,4237 SDPatternOperator op> {4238 def _S : sve2_int_un_pred_arit<0b10, opc, asm, ZPR32>,4239 SVEPseudo2Instr<NAME # _S, 1>;4240 4241 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;4242 4243 def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32>;4244 4245 defm : SVE_3_Op_Undef_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _S_UNDEF)>;4246}4247 4248multiclass sve2_int_un_pred_arit<bits<2> opc, string asm, SDPatternOperator op> {4249 def _B : sve2_int_un_pred_arit<0b00, opc, asm, ZPR8>,4250 SVEPseudo2Instr<NAME # _B, 1>;4251 def _H : sve2_int_un_pred_arit<0b01, opc, asm, ZPR16>,4252 SVEPseudo2Instr<NAME # _H, 1>;4253 def _S : sve2_int_un_pred_arit<0b10, opc, asm, ZPR32>,4254 SVEPseudo2Instr<NAME # _S, 1>;4255 def _D : sve2_int_un_pred_arit<0b11, opc, asm, ZPR64>,4256 SVEPseudo2Instr<NAME # _D, 1>;4257 4258 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;4259 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>;4260 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;4261 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;4262 4263 def _B_UNDEF : PredOneOpPassthruPseudo<NAME # _B, ZPR8>;4264 def _H_UNDEF : PredOneOpPassthruPseudo<NAME # _H, ZPR16>;4265 def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32>;4266 def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64>;4267 4268 defm : SVE_3_Op_Undef_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _B_UNDEF)>;4269 defm : SVE_3_Op_Undef_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _H_UNDEF)>;4270 defm : SVE_3_Op_Undef_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _S_UNDEF)>;4271 defm : SVE_3_Op_Undef_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _D_UNDEF)>;4272}4273 4274multiclass sve2_int_un_pred_arit_z_S<bits<2> opc, string asm, SDPatternOperator op> {4275 def _S : sve2_int_un_pred_arit_z<0b10, opc, asm, ZPR32>;4276 4277 defm : SVE_3_Op_UndefZero_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;4278}4279 4280multiclass sve2_int_un_pred_arit_z<bits<2> opc, string asm, SDPatternOperator op> {4281 def _B : sve2_int_un_pred_arit_z<0b00, opc, asm, ZPR8>;4282 def _H : sve2_int_un_pred_arit_z<0b01, opc, asm, ZPR16>;4283 def _S : sve2_int_un_pred_arit_z<0b10, opc, asm, ZPR32>;4284 def _D : sve2_int_un_pred_arit_z<0b11, opc, asm, ZPR64>;4285 4286 defm : SVE_3_Op_UndefZero_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;4287 defm : SVE_3_Op_UndefZero_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>;4288 defm : SVE_3_Op_UndefZero_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;4289 defm : SVE_3_Op_UndefZero_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;4290}4291 4292//===----------------------------------------------------------------------===//4293// SVE2 Widening Integer Arithmetic Group4294//===----------------------------------------------------------------------===//4295 4296class sve2_wide_int_arith<bits<2> sz, bits<5> opc, string asm,4297 ZPRRegOp zprty1, ZPRRegOp zprty2, ZPRRegOp zprty3>4298: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm),4299 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> {4300 bits<5> Zd;4301 bits<5> Zn;4302 bits<5> Zm;4303 let Inst{31-24} = 0b01000101;4304 let Inst{23-22} = sz;4305 let Inst{21} = 0b0;4306 let Inst{20-16} = Zm;4307 let Inst{15} = 0b0;4308 let Inst{14-10} = opc;4309 let Inst{9-5} = Zn;4310 let Inst{4-0} = Zd;4311 4312 let hasSideEffects = 0;4313}4314 4315multiclass sve2_wide_int_arith_long<bits<5> opc, string asm,4316 SDPatternOperator op> {4317 def _H : sve2_wide_int_arith<0b01, opc, asm, ZPR16, ZPR8, ZPR8>;4318 def _S : sve2_wide_int_arith<0b10, opc, asm, ZPR32, ZPR16, ZPR16>;4319 def _D : sve2_wide_int_arith<0b11, opc, asm, ZPR64, ZPR32, ZPR32>;4320 4321 def : SVE_2_Op_Pat<nxv8i16, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>;4322 def : SVE_2_Op_Pat<nxv4i32, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>;4323 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>;4324}4325 4326multiclass sve2_wide_int_arith_wide<bits<3> opc, string asm,4327 SDPatternOperator op> {4328 def _H : sve2_wide_int_arith<0b01, { 0b10, opc }, asm, ZPR16, ZPR16, ZPR8>;4329 def _S : sve2_wide_int_arith<0b10, { 0b10, opc }, asm, ZPR32, ZPR32, ZPR16>;4330 def _D : sve2_wide_int_arith<0b11, { 0b10, opc }, asm, ZPR64, ZPR64, ZPR32>;4331 4332 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>;4333 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, !cast<Instruction>(NAME # _S)>;4334 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>;4335}4336 4337multiclass sve2_wide_int_arith_pmul<bits<2> sz, bits<5> opc, string asm,4338 SDPatternOperator op> {4339 def NAME : sve2_wide_int_arith<sz, opc, asm, ZPR128, ZPR64, ZPR64>;4340 4341 // To avoid using 128 bit elements in the IR, the pattern below works with4342 // llvm intrinsics with the _pair suffix, to reflect that4343 // _Q is implemented as a pair of _D.4344 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>;4345}4346 4347multiclass sve2_pmul_long<bits<1> opc, string asm, SDPatternOperator op> {4348 def _H : sve2_wide_int_arith<0b01, {0b1101, opc}, asm, ZPR16, ZPR8, ZPR8>;4349 def _D : sve2_wide_int_arith<0b11, {0b1101, opc}, asm, ZPR64, ZPR32, ZPR32>;4350 4351 // To avoid using 128 bit elements in the IR, the patterns below work with4352 // llvm intrinsics with the _pair suffix, to reflect that4353 // _H is implemented as a pair of _B and _D is implemented as a pair of _S.4354 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>;4355 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>;4356}4357 4358//===----------------------------------------------------------------------===//4359// SVE2 Misc Group4360//===----------------------------------------------------------------------===//4361 4362class sve2_misc<bits<2> sz, bits<4> opc, string asm,4363 ZPRRegOp zprty1, ZPRRegOp zprty2>4364: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm),4365 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> {4366 bits<5> Zd;4367 bits<5> Zn;4368 bits<5> Zm;4369 let Inst{31-24} = 0b01000101;4370 let Inst{23-22} = sz;4371 let Inst{21} = 0b0;4372 let Inst{20-16} = Zm;4373 let Inst{15-14} = 0b10;4374 let Inst{13-10} = opc;4375 let Inst{9-5} = Zn;4376 let Inst{4-0} = Zd;4377 4378 let hasSideEffects = 0;4379}4380 4381multiclass sve2_misc_bitwise<bits<4> opc, string asm, SDPatternOperator op> {4382 def _B : sve2_misc<0b00, opc, asm, ZPR8, ZPR8>;4383 def _H : sve2_misc<0b01, opc, asm, ZPR16, ZPR16>;4384 def _S : sve2_misc<0b10, opc, asm, ZPR32, ZPR32>;4385 def _D : sve2_misc<0b11, opc, asm, ZPR64, ZPR64>;4386 4387 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;4388 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;4389 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;4390 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;4391}4392 4393multiclass sve2_misc_int_addsub_long_interleaved<bits<2> opc, string asm,4394 SDPatternOperator op> {4395 def _H : sve2_misc<0b01, { 0b00, opc }, asm, ZPR16, ZPR8>;4396 def _S : sve2_misc<0b10, { 0b00, opc }, asm, ZPR32, ZPR16>;4397 def _D : sve2_misc<0b11, { 0b00, opc }, asm, ZPR64, ZPR32>;4398 4399 def : SVE_2_Op_Pat<nxv8i16, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>;4400 def : SVE_2_Op_Pat<nxv4i32, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>;4401 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>;4402}4403 4404class sve2_bitwise_xor_interleaved<bits<2> sz, bits<1> opc, string asm,4405 ZPRRegOp zprty1, ZPRRegOp zprty2>4406: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, zprty2:$Zm),4407 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> {4408 bits<5> Zd;4409 bits<5> Zn;4410 bits<5> Zm;4411 let Inst{31-24} = 0b01000101;4412 let Inst{23-22} = sz;4413 let Inst{21} = 0b0;4414 let Inst{20-16} = Zm;4415 let Inst{15-11} = 0b10010;4416 let Inst{10} = opc;4417 let Inst{9-5} = Zn;4418 let Inst{4-0} = Zd;4419 4420 let Constraints = "$Zd = $_Zd";4421 let DestructiveInstType = DestructiveOther;4422 let ElementSize = ElementSizeNone;4423 let hasSideEffects = 0;4424}4425 4426multiclass sve2_bitwise_xor_interleaved<bit opc, string asm,4427 SDPatternOperator op> {4428 def _B : sve2_bitwise_xor_interleaved<0b00, opc, asm, ZPR8, ZPR8>;4429 def _H : sve2_bitwise_xor_interleaved<0b01, opc, asm, ZPR16, ZPR16>;4430 def _S : sve2_bitwise_xor_interleaved<0b10, opc, asm, ZPR32, ZPR32>;4431 def _D : sve2_bitwise_xor_interleaved<0b11, opc, asm, ZPR64, ZPR64>;4432 4433 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;4434 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;4435 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;4436 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;4437}4438 4439class sve2_bitwise_shift_left_long<bits<3> tsz8_64, bits<2> opc, string asm,4440 ZPRRegOp zprty1, ZPRRegOp zprty2,4441 Operand immtype>4442: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm),4443 asm, "\t$Zd, $Zn, $imm",4444 "", []>, Sched<[]> {4445 bits<5> Zd;4446 bits<5> Zn;4447 bits<5> imm;4448 let Inst{31-23} = 0b010001010;4449 let Inst{22} = tsz8_64{2};4450 let Inst{21} = 0b0;4451 let Inst{20-19} = tsz8_64{1-0};4452 let Inst{18-16} = imm{2-0}; // imm34453 let Inst{15-12} = 0b1010;4454 let Inst{11-10} = opc;4455 let Inst{9-5} = Zn;4456 let Inst{4-0} = Zd;4457 4458 let hasSideEffects = 0;4459}4460 4461multiclass sve2_bitwise_shift_left_long<bits<2> opc, string asm,4462 SDPatternOperator op> {4463 def _H : sve2_bitwise_shift_left_long<{0,0,1}, opc, asm,4464 ZPR16, ZPR8, vecshiftL8>;4465 def _S : sve2_bitwise_shift_left_long<{0,1,?}, opc, asm,4466 ZPR32, ZPR16, vecshiftL16> {4467 let Inst{19} = imm{3};4468 }4469 def _D : sve2_bitwise_shift_left_long<{1,?,?}, opc, asm,4470 ZPR64, ZPR32, vecshiftL32> {4471 let Inst{20-19} = imm{4-3};4472 }4473 def : SVE_2_Op_Imm_Pat<nxv8i16, op, nxv16i8, i32, vecshiftL8, !cast<Instruction>(NAME # _H)>;4474 def : SVE_2_Op_Imm_Pat<nxv4i32, op, nxv8i16, i32, vecshiftL16, !cast<Instruction>(NAME # _S)>;4475 def : SVE_2_Op_Imm_Pat<nxv2i64, op, nxv4i32, i32, vecshiftL32, !cast<Instruction>(NAME # _D)>;4476}4477 4478//===----------------------------------------------------------------------===//4479// SVE2 Accumulate Group4480//===----------------------------------------------------------------------===//4481 4482class sve2_int_bin_shift_imm<bits<4> tsz8_64, bit opc, string asm,4483 ZPRRegOp zprty, Operand immtype>4484: I<(outs zprty:$Zd), (ins zprty:$_Zd, zprty:$Zn, immtype:$imm),4485 asm, "\t$Zd, $Zn, $imm",4486 "", []>, Sched<[]> {4487 bits<5> Zd;4488 bits<5> Zn;4489 bits<6> imm;4490 let Inst{31-24} = 0b01000101;4491 let Inst{23-22} = tsz8_64{3-2};4492 let Inst{21} = 0b0;4493 let Inst{20-19} = tsz8_64{1-0};4494 let Inst{18-16} = imm{2-0}; // imm34495 let Inst{15-11} = 0b11110;4496 let Inst{10} = opc;4497 let Inst{9-5} = Zn;4498 let Inst{4-0} = Zd;4499 4500 let Constraints = "$Zd = $_Zd";4501 let hasSideEffects = 0;4502}4503 4504multiclass sve2_int_bin_shift_imm_left<bit opc, string asm,4505 SDPatternOperator op> {4506 def _B : sve2_int_bin_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>;4507 def _H : sve2_int_bin_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> {4508 let Inst{19} = imm{3};4509 }4510 def _S : sve2_int_bin_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> {4511 let Inst{20-19} = imm{4-3};4512 }4513 def _D : sve2_int_bin_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> {4514 let Inst{22} = imm{5};4515 let Inst{20-19} = imm{4-3};4516 }4517 4518 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, vecshiftL8, !cast<Instruction>(NAME # _B)>;4519 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, vecshiftL16, !cast<Instruction>(NAME # _H)>;4520 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, vecshiftL32, !cast<Instruction>(NAME # _S)>;4521 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, vecshiftL64, !cast<Instruction>(NAME # _D)>;4522}4523 4524multiclass sve2_int_bin_shift_imm_right<bit opc, string asm,4525 SDPatternOperator op> {4526 def _B : sve2_int_bin_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>;4527 def _H : sve2_int_bin_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> {4528 let Inst{19} = imm{3};4529 }4530 def _S : sve2_int_bin_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> {4531 let Inst{20-19} = imm{4-3};4532 }4533 def _D : sve2_int_bin_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> {4534 let Inst{22} = imm{5};4535 let Inst{20-19} = imm{4-3};4536 }4537 4538 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, vecshiftR8, !cast<Instruction>(NAME # _B)>;4539 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, vecshiftR16, !cast<Instruction>(NAME # _H)>;4540 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, vecshiftR32, !cast<Instruction>(NAME # _S)>;4541 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, vecshiftR64, !cast<Instruction>(NAME # _D)>;4542}4543 4544class sve2_int_bin_accum_shift_imm<bits<4> tsz8_64, bits<2> opc, string asm,4545 ZPRRegOp zprty, Operand immtype>4546: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, immtype:$imm),4547 asm, "\t$Zda, $Zn, $imm",4548 "", []>, Sched<[]> {4549 bits<5> Zda;4550 bits<5> Zn;4551 bits<6> imm;4552 let Inst{31-24} = 0b01000101;4553 let Inst{23-22} = tsz8_64{3-2};4554 let Inst{21} = 0b0;4555 let Inst{20-19} = tsz8_64{1-0};4556 let Inst{18-16} = imm{2-0}; // imm34557 let Inst{15-12} = 0b1110;4558 let Inst{11-10} = opc;4559 let Inst{9-5} = Zn;4560 let Inst{4-0} = Zda;4561 4562 let Constraints = "$Zda = $_Zda";4563 let DestructiveInstType = DestructiveOther;4564 let ElementSize = ElementSizeNone;4565 let hasSideEffects = 0;4566}4567 4568multiclass sve2_int_bin_accum_shift_imm_right<bits<2> opc, string asm,4569 SDPatternOperator op,4570 SDPatternOperator shift_op = null_frag> {4571 def _B : sve2_int_bin_accum_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>;4572 def _H : sve2_int_bin_accum_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> {4573 let Inst{19} = imm{3};4574 }4575 def _S : sve2_int_bin_accum_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> {4576 let Inst{20-19} = imm{4-3};4577 }4578 def _D : sve2_int_bin_accum_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> {4579 let Inst{22} = imm{5};4580 let Inst{20-19} = imm{4-3};4581 }4582 4583 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, vecshiftR8, !cast<Instruction>(NAME # _B)>;4584 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, vecshiftR16, !cast<Instruction>(NAME # _H)>;4585 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, vecshiftR32, !cast<Instruction>(NAME # _S)>;4586 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, vecshiftR64, !cast<Instruction>(NAME # _D)>;4587 4588 def : SVE_Shift_Add_All_Active_Pat<nxv16i8, shift_op, nxv16i1, nxv16i8, nxv16i8, i32, !cast<Instruction>(NAME # _B)>;4589 def : SVE_Shift_Add_All_Active_Pat<nxv8i16, shift_op, nxv8i1, nxv8i16, nxv8i16, i32, !cast<Instruction>(NAME # _H)>;4590 def : SVE_Shift_Add_All_Active_Pat<nxv4i32, shift_op, nxv4i1, nxv4i32, nxv4i32, i32, !cast<Instruction>(NAME # _S)>;4591 def : SVE_Shift_Add_All_Active_Pat<nxv2i64, shift_op, nxv2i1, nxv2i64, nxv2i64, i32, !cast<Instruction>(NAME # _D)>;4592}4593 4594class sve2_int_cadd<bits<2> sz, bit opc, string asm, ZPRRegOp zprty>4595: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, complexrotateopodd:$rot),4596 asm, "\t$Zdn, $_Zdn, $Zm, $rot", "", []>, Sched<[]> {4597 bits<5> Zdn;4598 bits<5> Zm;4599 bit rot;4600 let Inst{31-24} = 0b01000101;4601 let Inst{23-22} = sz;4602 let Inst{21-17} = 0b00000;4603 let Inst{16} = opc;4604 let Inst{15-11} = 0b11011;4605 let Inst{10} = rot;4606 let Inst{9-5} = Zm;4607 let Inst{4-0} = Zdn;4608 4609 let Constraints = "$Zdn = $_Zdn";4610 let DestructiveInstType = DestructiveOther;4611 let ElementSize = ElementSizeNone;4612 let hasSideEffects = 0;4613}4614 4615multiclass sve2_int_cadd<bit opc, string asm, SDPatternOperator op> {4616 def _B : sve2_int_cadd<0b00, opc, asm, ZPR8>;4617 def _H : sve2_int_cadd<0b01, opc, asm, ZPR16>;4618 def _S : sve2_int_cadd<0b10, opc, asm, ZPR32>;4619 def _D : sve2_int_cadd<0b11, opc, asm, ZPR64>;4620 4621 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, complexrotateopodd, !cast<Instruction>(NAME # _B)>;4622 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, complexrotateopodd, !cast<Instruction>(NAME # _H)>;4623 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, complexrotateopodd, !cast<Instruction>(NAME # _S)>;4624 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, complexrotateopodd, !cast<Instruction>(NAME # _D)>;4625}4626 4627class sve2_int_absdiff_accum<bits<3> sz, bits<4> opc, string asm,4628 ZPRRegOp zprty1, ZPRRegOp zprty2>4629: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm),4630 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> {4631 bits<5> Zda;4632 bits<5> Zn;4633 bits<5> Zm;4634 let Inst{31-25} = 0b0100010;4635 let Inst{24-22} = sz;4636 let Inst{21} = 0b0;4637 let Inst{20-16} = Zm;4638 let Inst{15-14} = 0b11;4639 let Inst{13-10} = opc;4640 let Inst{9-5} = Zn;4641 let Inst{4-0} = Zda;4642 4643 let Constraints = "$Zda = $_Zda";4644 let DestructiveInstType = DestructiveOther;4645 let ElementSize = ElementSizeNone;4646 let hasSideEffects = 0;4647}4648 4649multiclass sve2_int_absdiff_accum<bit opc, string asm, SDPatternOperator op> {4650 def _B : sve2_int_absdiff_accum<0b100, { 0b111, opc }, asm, ZPR8, ZPR8>;4651 def _H : sve2_int_absdiff_accum<0b101, { 0b111, opc }, asm, ZPR16, ZPR16>;4652 def _S : sve2_int_absdiff_accum<0b110, { 0b111, opc }, asm, ZPR32, ZPR32>;4653 def _D : sve2_int_absdiff_accum<0b111, { 0b111, opc }, asm, ZPR64, ZPR64>;4654 4655 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;4656 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;4657 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;4658 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;4659}4660 4661multiclass sve2_int_absdiff_accum_long<bits<2> opc, string asm,4662 SDPatternOperator op> {4663 def _H : sve2_int_absdiff_accum<0b101, { 0b00, opc }, asm, ZPR16, ZPR8>;4664 def _S : sve2_int_absdiff_accum<0b110, { 0b00, opc }, asm, ZPR32, ZPR16>;4665 def _D : sve2_int_absdiff_accum<0b111, { 0b00, opc }, asm, ZPR64, ZPR32>;4666 4667 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>;4668 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>;4669 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>;4670}4671 4672multiclass sve2_int_two_way_absdiff_accum_long<bit U, string asm> {4673 def _BtoH : sve2_int_absdiff_accum<0b001, { 0b01, U, 0b1 }, asm, ZPR16, ZPR8>;4674 def _HtoS : sve2_int_absdiff_accum<0b010, { 0b01, U, 0b1 }, asm, ZPR32, ZPR16>;4675 def _StoD : sve2_int_absdiff_accum<0b011, { 0b01, U, 0b1 }, asm, ZPR64, ZPR32>;4676}4677 4678multiclass sve2_int_addsub_long_carry<bits<2> opc, string asm,4679 SDPatternOperator op> {4680 def _S : sve2_int_absdiff_accum<{ 0b1, opc{1}, 0b0 }, { 0b010, opc{0} }, asm,4681 ZPR32, ZPR32>;4682 def _D : sve2_int_absdiff_accum<{ 0b1, opc{1}, 0b1 }, { 0b010, opc{0} }, asm,4683 ZPR64, ZPR64>;4684 4685 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;4686 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;4687}4688 4689//===----------------------------------------------------------------------===//4690// SVE2 Narrowing Group4691//===----------------------------------------------------------------------===//4692 4693class sve2_int_bin_shift_imm_narrow_bottom<bits<3> tsz8_64, bits<3> opc,4694 string asm, ZPRRegOp zprty1,4695 ZPRRegOp zprty2, Operand immtype>4696: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm),4697 asm, "\t$Zd, $Zn, $imm",4698 "", []>, Sched<[]> {4699 bits<5> Zd;4700 bits<5> Zn;4701 bits<5> imm;4702 let Inst{31-23} = 0b010001010;4703 let Inst{22} = tsz8_64{2};4704 let Inst{21} = 0b1;4705 let Inst{20-19} = tsz8_64{1-0};4706 let Inst{18-16} = imm{2-0}; // imm34707 let Inst{15-14} = 0b00;4708 let Inst{13-11} = opc;4709 let Inst{10} = 0b0;4710 let Inst{9-5} = Zn;4711 let Inst{4-0} = Zd;4712 4713 let hasSideEffects = 0;4714}4715 4716multiclass sve2_int_bin_shift_imm_right_narrow_bottom<bits<3> opc, string asm,4717 SDPatternOperator op> {4718 def _B : sve2_int_bin_shift_imm_narrow_bottom<{0,0,1}, opc, asm, ZPR8, ZPR16,4719 vecshiftR8>;4720 def _H : sve2_int_bin_shift_imm_narrow_bottom<{0,1,?}, opc, asm, ZPR16, ZPR32,4721 vecshiftR16> {4722 let Inst{19} = imm{3};4723 }4724 def _S : sve2_int_bin_shift_imm_narrow_bottom<{1,?,?}, opc, asm, ZPR32, ZPR64,4725 vecshiftR32> {4726 let Inst{20-19} = imm{4-3};4727 }4728 def : SVE_2_Op_Imm_Pat<nxv16i8, op, nxv8i16, i32, vecshiftR8, !cast<Instruction>(NAME # _B)>;4729 def : SVE_2_Op_Imm_Pat<nxv8i16, op, nxv4i32, i32, vecshiftR16, !cast<Instruction>(NAME # _H)>;4730 def : SVE_2_Op_Imm_Pat<nxv4i32, op, nxv2i64, i32, vecshiftR32, !cast<Instruction>(NAME # _S)>;4731}4732 4733class sve2_int_bin_shift_imm_narrow_top<bits<3> tsz8_64, bits<3> opc,4734 string asm, ZPRRegOp zprty1,4735 ZPRRegOp zprty2, Operand immtype>4736: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, immtype:$imm),4737 asm, "\t$Zd, $Zn, $imm",4738 "", []>, Sched<[]> {4739 bits<5> Zd;4740 bits<5> Zn;4741 bits<5> imm;4742 let Inst{31-23} = 0b010001010;4743 let Inst{22} = tsz8_64{2};4744 let Inst{21} = 0b1;4745 let Inst{20-19} = tsz8_64{1-0};4746 let Inst{18-16} = imm{2-0}; // imm34747 let Inst{15-14} = 0b00;4748 let Inst{13-11} = opc;4749 let Inst{10} = 0b1;4750 let Inst{9-5} = Zn;4751 let Inst{4-0} = Zd;4752 4753 let Constraints = "$Zd = $_Zd";4754 let hasSideEffects = 0;4755}4756 4757multiclass sve2_int_bin_shift_imm_right_narrow_top<bits<3> opc, string asm,4758 SDPatternOperator op> {4759 def _B : sve2_int_bin_shift_imm_narrow_top<{0,0,1}, opc, asm, ZPR8, ZPR16,4760 vecshiftR8>;4761 def _H : sve2_int_bin_shift_imm_narrow_top<{0,1,?}, opc, asm, ZPR16, ZPR32,4762 vecshiftR16> {4763 let Inst{19} = imm{3};4764 }4765 def _S : sve2_int_bin_shift_imm_narrow_top<{1,?,?}, opc, asm, ZPR32, ZPR64,4766 vecshiftR32> {4767 let Inst{20-19} = imm{4-3};4768 }4769 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv8i16, i32, vecshiftR8, !cast<Instruction>(NAME # _B)>;4770 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv4i32, i32, vecshiftR16, !cast<Instruction>(NAME # _H)>;4771 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv2i64, i32, vecshiftR32, !cast<Instruction>(NAME # _S)>;4772}4773 4774class sve2_int_addsub_narrow_high_bottom<bits<2> sz, bits<2> opc, string asm,4775 ZPRRegOp zprty1, ZPRRegOp zprty2>4776: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm),4777 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> {4778 bits<5> Zd;4779 bits<5> Zn;4780 bits<5> Zm;4781 let Inst{31-24} = 0b01000101;4782 let Inst{23-22} = sz;4783 let Inst{21} = 0b1;4784 let Inst{20-16} = Zm;4785 let Inst{15-13} = 0b011;4786 let Inst{12-11} = opc; // S, R4787 let Inst{10} = 0b0; // Top4788 let Inst{9-5} = Zn;4789 let Inst{4-0} = Zd;4790 4791 let hasSideEffects = 0;4792}4793 4794multiclass sve2_int_addsub_narrow_high_bottom<bits<2> opc, string asm,4795 SDPatternOperator op> {4796 def _B : sve2_int_addsub_narrow_high_bottom<0b01, opc, asm, ZPR8, ZPR16>;4797 def _H : sve2_int_addsub_narrow_high_bottom<0b10, opc, asm, ZPR16, ZPR32>;4798 def _S : sve2_int_addsub_narrow_high_bottom<0b11, opc, asm, ZPR32, ZPR64>;4799 4800 def : SVE_2_Op_Pat<nxv16i8, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _B)>;4801 def : SVE_2_Op_Pat<nxv8i16, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _H)>;4802 def : SVE_2_Op_Pat<nxv4i32, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _S)>;4803}4804 4805class sve2_int_addsub_narrow_high_top<bits<2> sz, bits<2> opc, string asm,4806 ZPRRegOp zprty1, ZPRRegOp zprty2>4807: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, zprty2:$Zm),4808 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> {4809 bits<5> Zd;4810 bits<5> Zn;4811 bits<5> Zm;4812 let Inst{31-24} = 0b01000101;4813 let Inst{23-22} = sz;4814 let Inst{21} = 0b1;4815 let Inst{20-16} = Zm;4816 let Inst{15-13} = 0b011;4817 let Inst{12-11} = opc; // S, R4818 let Inst{10} = 0b1; // Top4819 let Inst{9-5} = Zn;4820 let Inst{4-0} = Zd;4821 4822 let Constraints = "$Zd = $_Zd";4823 let hasSideEffects = 0;4824}4825 4826multiclass sve2_int_addsub_narrow_high_top<bits<2> opc, string asm,4827 SDPatternOperator op> {4828 def _B : sve2_int_addsub_narrow_high_top<0b01, opc, asm, ZPR8, ZPR16>;4829 def _H : sve2_int_addsub_narrow_high_top<0b10, opc, asm, ZPR16, ZPR32>;4830 def _S : sve2_int_addsub_narrow_high_top<0b11, opc, asm, ZPR32, ZPR64>;4831 4832 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _B)>;4833 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _H)>;4834 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _S)>;4835}4836 4837class sve2_int_sat_extract_narrow_bottom<bits<3> tsz8_64, bits<2> opc, string asm,4838 ZPRRegOp zprty1, ZPRRegOp zprty2>4839: I<(outs zprty1:$Zd), (ins zprty2:$Zn),4840 asm, "\t$Zd, $Zn", "", []>, Sched<[]> {4841 bits<5> Zd;4842 bits<5> Zn;4843 let Inst{31-23} = 0b010001010;4844 let Inst{22} = tsz8_64{2};4845 let Inst{21} = 0b1;4846 let Inst{20-19} = tsz8_64{1-0};4847 let Inst{18-13} = 0b000010;4848 let Inst{12-11} = opc;4849 let Inst{10} = 0b0;4850 let Inst{9-5} = Zn;4851 let Inst{4-0} = Zd;4852 4853 let hasSideEffects = 0;4854}4855 4856multiclass sve2_int_sat_extract_narrow_bottom<bits<2> opc, string asm,4857 SDPatternOperator op> {4858 def _B : sve2_int_sat_extract_narrow_bottom<0b001, opc, asm, ZPR8, ZPR16>;4859 def _H : sve2_int_sat_extract_narrow_bottom<0b010, opc, asm, ZPR16, ZPR32>;4860 def _S : sve2_int_sat_extract_narrow_bottom<0b100, opc, asm, ZPR32, ZPR64>;4861 4862 def : SVE_1_Op_Pat<nxv16i8, op, nxv8i16, !cast<Instruction>(NAME # _B)>;4863 def : SVE_1_Op_Pat<nxv8i16, op, nxv4i32, !cast<Instruction>(NAME # _H)>;4864 def : SVE_1_Op_Pat<nxv4i32, op, nxv2i64, !cast<Instruction>(NAME # _S)>;4865}4866 4867class sve2_int_sat_extract_narrow_top<bits<3> tsz8_64, bits<2> opc, string asm,4868 ZPRRegOp zprty1, ZPRRegOp zprty2>4869: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn),4870 asm, "\t$Zd, $Zn", "", []>, Sched<[]> {4871 bits<5> Zd;4872 bits<5> Zn;4873 let Inst{31-23} = 0b010001010;4874 let Inst{22} = tsz8_64{2};4875 let Inst{21} = 0b1;4876 let Inst{20-19} = tsz8_64{1-0};4877 let Inst{18-13} = 0b000010;4878 let Inst{12-11} = opc;4879 let Inst{10} = 0b1;4880 let Inst{9-5} = Zn;4881 let Inst{4-0} = Zd;4882 4883 let Constraints = "$Zd = $_Zd";4884 let hasSideEffects = 0;4885}4886 4887multiclass sve2_int_sat_extract_narrow_top<bits<2> opc, string asm,4888 SDPatternOperator op> {4889 def _B : sve2_int_sat_extract_narrow_top<0b001, opc, asm, ZPR8, ZPR16>;4890 def _H : sve2_int_sat_extract_narrow_top<0b010, opc, asm, ZPR16, ZPR32>;4891 def _S : sve2_int_sat_extract_narrow_top<0b100, opc, asm, ZPR32, ZPR64>;4892 4893 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv8i16, !cast<Instruction>(NAME # _B)>;4894 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv4i32, !cast<Instruction>(NAME # _H)>;4895 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>;4896}4897 4898//===----------------------------------------------------------------------===//4899// SVE Integer Arithmetic - Unary Predicated Group4900//===----------------------------------------------------------------------===//4901 4902class sve_int_un_pred_arit<bits<2> sz8_64, bits<4> opc,4903 string asm, ZPRRegOp zprty>4904: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn),4905 asm, "\t$Zd, $Pg/m, $Zn",4906 "",4907 []>, Sched<[]> {4908 bits<3> Pg;4909 bits<5> Zd;4910 bits<5> Zn;4911 let Inst{31-24} = 0b00000100;4912 let Inst{23-22} = sz8_64;4913 let Inst{21-20} = 0b01;4914 let Inst{19} = opc{0};4915 let Inst{18-16} = opc{3-1};4916 let Inst{15-13} = 0b101;4917 let Inst{12-10} = Pg;4918 let Inst{9-5} = Zn;4919 let Inst{4-0} = Zd;4920 4921 let Constraints = "$Zd = $_Zd";4922 let DestructiveInstType = DestructiveUnaryPassthru;4923 let ElementSize = zprty.ElementSize;4924 let hasSideEffects = 0;4925}4926 4927class sve_int_un_pred_arit_z<bits<2> sz8_64, bits<4> opc,4928 string asm, ZPRRegOp zprty>4929: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn),4930 asm, "\t$Zd, $Pg/z, $Zn",4931 "",4932 []>, Sched<[]> {4933 bits<3> Pg;4934 bits<5> Zd;4935 bits<5> Zn;4936 let Inst{31-24} = 0b00000100;4937 let Inst{23-22} = sz8_64;4938 let Inst{21-20} = 0b00;4939 let Inst{19} = opc{0};4940 let Inst{18-16} = opc{3-1};4941 let Inst{15-13} = 0b101;4942 let Inst{12-10} = Pg;4943 let Inst{9-5} = Zn;4944 let Inst{4-0} = Zd;4945 4946 let hasSideEffects = 0;4947}4948 4949multiclass sve_int_un_pred_arit<bits<3> opc, string asm,4950 SDPatternOperator op> {4951 def _B : sve_int_un_pred_arit<0b00, { opc, 0b0 }, asm, ZPR8>,4952 SVEPseudo2Instr<NAME # _B, 1>;4953 def _H : sve_int_un_pred_arit<0b01, { opc, 0b0 }, asm, ZPR16>,4954 SVEPseudo2Instr<NAME # _H, 1>;4955 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>,4956 SVEPseudo2Instr<NAME # _S, 1>;4957 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>,4958 SVEPseudo2Instr<NAME # _D, 1>;4959 4960 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;4961 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>;4962 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;4963 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;4964 4965 def _B_UNDEF : PredOneOpPassthruPseudo<NAME # _B, ZPR8>;4966 def _H_UNDEF : PredOneOpPassthruPseudo<NAME # _H, ZPR16>;4967 def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32>;4968 def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64>;4969 4970 defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _B_UNDEF)>;4971 defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _H_UNDEF)>;4972 defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _S_UNDEF)>;4973 defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _D_UNDEF)>;4974}4975 4976multiclass sve_int_un_pred_arit_z<bits<3> opc, string asm, SDPatternOperator op> {4977 def _B : sve_int_un_pred_arit_z<0b00, { opc, 0b0 }, asm, ZPR8>;4978 def _H : sve_int_un_pred_arit_z<0b01, { opc, 0b0 }, asm, ZPR16>;4979 def _S : sve_int_un_pred_arit_z<0b10, { opc, 0b0 }, asm, ZPR32>;4980 def _D : sve_int_un_pred_arit_z<0b11, { opc, 0b0 }, asm, ZPR64>;4981 4982 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;4983 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>;4984 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;4985 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;4986}4987 4988multiclass sve_int_un_pred_arit_h<bits<3> opc, string asm,4989 SDPatternOperator op> {4990 def _H : sve_int_un_pred_arit<0b01, { opc, 0b0 }, asm, ZPR16>,4991 SVEPseudo2Instr<NAME # _H, 1>;4992 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>,4993 SVEPseudo2Instr<NAME # _S, 1>;4994 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>,4995 SVEPseudo2Instr<NAME # _D, 1>;4996 4997 def : SVE_InReg_Extend<nxv8i16, op, nxv8i1, nxv8i8, !cast<Instruction>(NAME # _H)>;4998 def : SVE_InReg_Extend<nxv4i32, op, nxv4i1, nxv4i8, !cast<Instruction>(NAME # _S)>;4999 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i8, !cast<Instruction>(NAME # _D)>;5000 5001 def _H_UNDEF : PredOneOpPassthruPseudo<NAME # _H, ZPR16>;5002 def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32>;5003 def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64>;5004 5005 defm : SVE_InReg_Extend_PassthruUndef<nxv8i16, op, nxv8i1, nxv8i8, !cast<Pseudo>(NAME # _H_UNDEF)>;5006 defm : SVE_InReg_Extend_PassthruUndef<nxv4i32, op, nxv4i1, nxv4i8, !cast<Pseudo>(NAME # _S_UNDEF)>;5007 defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i8, !cast<Pseudo>(NAME # _D_UNDEF)>;5008}5009 5010multiclass sve_int_un_pred_arit_h_z<bits<3> opc, string asm, SDPatternOperator op> {5011 def _H : sve_int_un_pred_arit_z<0b01, { opc, 0b0 }, asm, ZPR16>;5012 def _S : sve_int_un_pred_arit_z<0b10, { opc, 0b0 }, asm, ZPR32>;5013 def _D : sve_int_un_pred_arit_z<0b11, { opc, 0b0 }, asm, ZPR64>;5014 5015 defm : SVE_InReg_Extend_PassthruUndefZero<nxv8i16, op, nxv8i1, nxv8i8, !cast<Instruction>(NAME # _H)>;5016 defm : SVE_InReg_Extend_PassthruUndefZero<nxv4i32, op, nxv4i1, nxv4i8, !cast<Instruction>(NAME # _S)>;5017 defm : SVE_InReg_Extend_PassthruUndefZero<nxv2i64, op, nxv2i1, nxv2i8, !cast<Instruction>(NAME # _D)>;5018}5019 5020multiclass sve_int_un_pred_arit_w<bits<3> opc, string asm,5021 SDPatternOperator op> {5022 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>,5023 SVEPseudo2Instr<NAME # _S, 1>;5024 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>,5025 SVEPseudo2Instr<NAME # _D, 1>;5026 5027 def : SVE_InReg_Extend<nxv4i32, op, nxv4i1, nxv4i16, !cast<Instruction>(NAME # _S)>;5028 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i16, !cast<Instruction>(NAME # _D)>;5029 5030 def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32>;5031 def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64>;5032 5033 defm : SVE_InReg_Extend_PassthruUndef<nxv4i32, op, nxv4i1, nxv4i16, !cast<Pseudo>(NAME # _S_UNDEF)>;5034 defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i16, !cast<Pseudo>(NAME # _D_UNDEF)>;5035}5036 5037multiclass sve_int_un_pred_arit_w_z<bits<3> opc, string asm, SDPatternOperator op> {5038 def _S : sve_int_un_pred_arit_z<0b10, { opc, 0b0 }, asm, ZPR32>;5039 def _D : sve_int_un_pred_arit_z<0b11, { opc, 0b0 }, asm, ZPR64>;5040 5041 defm : SVE_InReg_Extend_PassthruUndefZero<nxv4i32, op, nxv4i1, nxv4i16, !cast<Instruction>(NAME # _S)>;5042 defm : SVE_InReg_Extend_PassthruUndefZero<nxv2i64, op, nxv2i1, nxv2i16, !cast<Instruction>(NAME # _D)>;5043}5044 5045multiclass sve_int_un_pred_arit_d<bits<3> opc, string asm,5046 SDPatternOperator op> {5047 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>,5048 SVEPseudo2Instr<NAME # _D, 1>;5049 5050 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i32, !cast<Instruction>(NAME # _D)>;5051 5052 def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64>;5053 5054 defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i32, !cast<Pseudo>(NAME # _D_UNDEF)>;5055}5056 5057multiclass sve_int_un_pred_arit_d_z<bits<3> opc, string asm, SDPatternOperator op> {5058 def _D : sve_int_un_pred_arit_z<0b11, {opc, 0b0}, asm, ZPR64>;5059 5060 defm : SVE_InReg_Extend_PassthruUndefZero<nxv2i64, op, nxv2i1, nxv2i32, !cast<Instruction>(NAME # _D)>;5061}5062 5063multiclass sve_int_un_pred_arit_bitwise<bits<3> opc, string asm,5064 SDPatternOperator op> {5065 def _B : sve_int_un_pred_arit<0b00, { opc, 0b1 }, asm, ZPR8>,5066 SVEPseudo2Instr<NAME # _B, 1>;5067 def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>,5068 SVEPseudo2Instr<NAME # _H, 1>;5069 def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>,5070 SVEPseudo2Instr<NAME # _S, 1>;5071 def _D : sve_int_un_pred_arit<0b11, { opc, 0b1 }, asm, ZPR64>,5072 SVEPseudo2Instr<NAME # _D, 1>;5073 5074 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;5075 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>;5076 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;5077 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;5078 5079 def _B_UNDEF : PredOneOpPassthruPseudo<NAME # _B, ZPR8>;5080 def _H_UNDEF : PredOneOpPassthruPseudo<NAME # _H, ZPR16>;5081 def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32>;5082 def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64>;5083 5084 defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _B_UNDEF)>;5085 defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _H_UNDEF)>;5086 defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _S_UNDEF)>;5087 defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _D_UNDEF)>;5088}5089 5090multiclass sve_int_un_pred_arit_bitwise_z<bits<3> opc, string asm, SDPatternOperator op> {5091 def _B : sve_int_un_pred_arit_z<0b00, { opc, 0b1 }, asm, ZPR8>;5092 def _H : sve_int_un_pred_arit_z<0b01, { opc, 0b1 }, asm, ZPR16>;5093 def _S : sve_int_un_pred_arit_z<0b10, { opc, 0b1 }, asm, ZPR32>;5094 def _D : sve_int_un_pred_arit_z<0b11, { opc, 0b1 }, asm, ZPR64>;5095 5096 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;5097 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>;5098 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;5099 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;5100}5101 5102multiclass sve_int_un_pred_arit_bitwise_fp<bits<3> opc, string asm,5103 SDPatternOperator op> {5104 def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>,5105 SVEPseudo2Instr<NAME # _H, 1>;5106 def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>,5107 SVEPseudo2Instr<NAME # _S, 1>;5108 def _D : sve_int_un_pred_arit<0b11, { opc, 0b1 }, asm, ZPR64>,5109 SVEPseudo2Instr<NAME # _D, 1>;5110 5111 def : SVE_1_Op_Passthru_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;5112 def : SVE_1_Op_Passthru_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>;5113 def : SVE_1_Op_Passthru_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>;5114 def : SVE_1_Op_Passthru_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;5115 def : SVE_1_Op_Passthru_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>;5116 def : SVE_1_Op_Passthru_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;5117 def : SVE_1_Op_Passthru_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, !cast<Instruction>(NAME # _H)>;5118 def : SVE_1_Op_Passthru_Pat<nxv4bf16, op, nxv4i1, nxv4bf16, !cast<Instruction>(NAME # _H)>;5119 def : SVE_1_Op_Passthru_Pat<nxv2bf16, op, nxv2i1, nxv2bf16, !cast<Instruction>(NAME # _H)>;5120 5121 def _H_UNDEF : PredOneOpPassthruPseudo<NAME # _H, ZPR16>;5122 def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32>;5123 def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64>;5124 5125 defm : SVE_1_Op_PassthruUndef_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Pseudo>(NAME # _H_UNDEF)>;5126 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Pseudo>(NAME # _H_UNDEF)>;5127 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Pseudo>(NAME # _H_UNDEF)>;5128 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Pseudo>(NAME # _S_UNDEF)>;5129 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Pseudo>(NAME # _S_UNDEF)>;5130 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Pseudo>(NAME # _D_UNDEF)>;5131 defm : SVE_1_Op_PassthruUndef_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, !cast<Pseudo>(NAME # _H_UNDEF)>;5132 defm : SVE_1_Op_PassthruUndef_Pat<nxv4bf16, op, nxv4i1, nxv4bf16, !cast<Pseudo>(NAME # _H_UNDEF)>;5133 defm : SVE_1_Op_PassthruUndef_Pat<nxv2bf16, op, nxv2i1, nxv2bf16, !cast<Pseudo>(NAME # _H_UNDEF)>;5134}5135 5136multiclass sve_int_un_pred_arit_bitwise_fp_z<bits<3> opc, string asm, SDPatternOperator op> {5137 def _H : sve_int_un_pred_arit_z<0b01, { opc, 0b1 }, asm, ZPR16>;5138 def _S : sve_int_un_pred_arit_z<0b10, { opc, 0b1 }, asm, ZPR32>;5139 def _D : sve_int_un_pred_arit_z<0b11, { opc, 0b1 }, asm, ZPR64>;5140 5141 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;5142 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>;5143 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>;5144 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;5145 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>;5146 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;5147}5148 5149multiclass sve_fp_un_pred_arit_hsd<SDPatternOperator op> {5150 def _H_UNDEF : PredOneOpPassthruPseudo<NAME # _H, ZPR16>;5151 def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32>;5152 def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64>;5153 5154 defm : SVE_1_Op_PassthruUndef_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Pseudo>(NAME # _H_UNDEF)>;5155 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Pseudo>(NAME # _H_UNDEF)>;5156 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Pseudo>(NAME # _H_UNDEF)>;5157 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Pseudo>(NAME # _S_UNDEF)>;5158 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Pseudo>(NAME # _S_UNDEF)>;5159 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Pseudo>(NAME # _D_UNDEF)>;5160}5161 5162multiclass sve_int_un_pred_arit_bhsd<SDPatternOperator op> {5163 def _B_UNDEF : PredOneOpPassthruPseudo<NAME # _B, ZPR8>;5164 def _H_UNDEF : PredOneOpPassthruPseudo<NAME # _H, ZPR16>;5165 def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32>;5166 def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64>;5167 5168 defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _B_UNDEF)>;5169 defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _H_UNDEF)>;5170 defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _S_UNDEF)>;5171 defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _D_UNDEF)>;5172}5173 5174//===----------------------------------------------------------------------===//5175// SVE Integer Wide Immediate - Unpredicated Group5176//===----------------------------------------------------------------------===//5177class sve_int_dup_imm<bits<2> sz8_64, string asm,5178 ZPRRegOp zprty, Operand immtype>5179: I<(outs zprty:$Zd), (ins immtype:$imm),5180 asm, "\t$Zd, $imm",5181 "",5182 []>, Sched<[]> {5183 bits<5> Zd;5184 bits<9> imm;5185 let Inst{31-24} = 0b00100101;5186 let Inst{23-22} = sz8_64;5187 let Inst{21-14} = 0b11100011;5188 let Inst{13} = imm{8}; // sh5189 let Inst{12-5} = imm{7-0}; // imm85190 let Inst{4-0} = Zd;5191 5192 let hasSideEffects = 0;5193 let isAsCheapAsAMove = 1;5194 let isReMaterializable = 1;5195 let Uses = [VG];5196}5197 5198multiclass sve_int_dup_imm<string asm> {5199 def _B : sve_int_dup_imm<0b00, asm, ZPR8, cpy_imm8_opt_lsl_i8>;5200 def _H : sve_int_dup_imm<0b01, asm, ZPR16, cpy_imm8_opt_lsl_i16>;5201 def _S : sve_int_dup_imm<0b10, asm, ZPR32, cpy_imm8_opt_lsl_i32>;5202 def _D : sve_int_dup_imm<0b11, asm, ZPR64, cpy_imm8_opt_lsl_i64>;5203 5204 def : InstAlias<"mov $Zd, $imm",5205 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, cpy_imm8_opt_lsl_i8:$imm), 1>;5206 def : InstAlias<"mov $Zd, $imm",5207 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, cpy_imm8_opt_lsl_i16:$imm), 1>;5208 def : InstAlias<"mov $Zd, $imm",5209 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, cpy_imm8_opt_lsl_i32:$imm), 1>;5210 def : InstAlias<"mov $Zd, $imm",5211 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, cpy_imm8_opt_lsl_i64:$imm), 1>;5212 5213 def : InstAlias<"fmov $Zd, #0.0",5214 (!cast<Instruction>(NAME # _H) ZPR16:$Zd,5215 (cpy_imm8_opt_lsl_i16 0, 0)), 1>;5216 def : InstAlias<"fmov $Zd, #0.0",5217 (!cast<Instruction>(NAME # _S) ZPR32:$Zd,5218 (cpy_imm8_opt_lsl_i32 0, 0)), 1>;5219 def : InstAlias<"fmov $Zd, #0.0",5220 (!cast<Instruction>(NAME # _D) ZPR64:$Zd,5221 (cpy_imm8_opt_lsl_i64 0, 0)), 1>;5222}5223 5224class sve_int_dup_fpimm<bits<2> sz8_64, Operand fpimmtype,5225 string asm, ZPRRegOp zprty>5226: I<(outs zprty:$Zd), (ins fpimmtype:$imm8),5227 asm, "\t$Zd, $imm8",5228 "",5229 []>, Sched<[]> {5230 bits<5> Zd;5231 bits<8> imm8;5232 let Inst{31-24} = 0b00100101;5233 let Inst{23-22} = sz8_64;5234 let Inst{21-14} = 0b11100111;5235 let Inst{13} = 0b0;5236 let Inst{12-5} = imm8;5237 let Inst{4-0} = Zd;5238 5239 let hasSideEffects = 0;5240 let isAsCheapAsAMove = 1;5241 let isReMaterializable = 1;5242 let Uses = [VG];5243}5244 5245multiclass sve_int_dup_fpimm<string asm> {5246 def _H : sve_int_dup_fpimm<0b01, fpimm16, asm, ZPR16>;5247 def _S : sve_int_dup_fpimm<0b10, fpimm32, asm, ZPR32>;5248 def _D : sve_int_dup_fpimm<0b11, fpimm64, asm, ZPR64>;5249 5250 def : InstAlias<"fmov $Zd, $imm8",5251 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, fpimm16:$imm8), 1>;5252 def : InstAlias<"fmov $Zd, $imm8",5253 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, fpimm32:$imm8), 1>;5254 def : InstAlias<"fmov $Zd, $imm8",5255 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, fpimm64:$imm8), 1>;5256}5257 5258class sve_int_arith_imm0<bits<2> sz8_64, bits<3> opc, string asm,5259 ZPRRegOp zprty, Operand immtype>5260: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm),5261 asm, "\t$Zdn, $_Zdn, $imm",5262 "",5263 []>, Sched<[]> {5264 bits<5> Zdn;5265 bits<9> imm;5266 let Inst{31-24} = 0b00100101;5267 let Inst{23-22} = sz8_64;5268 let Inst{21-19} = 0b100;5269 let Inst{18-16} = opc;5270 let Inst{15-14} = 0b11;5271 let Inst{13} = imm{8}; // sh5272 let Inst{12-5} = imm{7-0}; // imm85273 let Inst{4-0} = Zdn;5274 5275 let Constraints = "$Zdn = $_Zdn";5276 let DestructiveInstType = DestructiveOther;5277 let ElementSize = ElementSizeNone;5278 let hasSideEffects = 0;5279}5280 5281multiclass sve_int_arith_imm0<bits<3> opc, string asm, SDPatternOperator op,5282 SDPatternOperator inv_op = null_frag> {5283 def _B : sve_int_arith_imm0<0b00, opc, asm, ZPR8, addsub_imm8_opt_lsl_i8>;5284 def _H : sve_int_arith_imm0<0b01, opc, asm, ZPR16, addsub_imm8_opt_lsl_i16>;5285 def _S : sve_int_arith_imm0<0b10, opc, asm, ZPR32, addsub_imm8_opt_lsl_i32>;5286 def _D : sve_int_arith_imm0<0b11, opc, asm, ZPR64, addsub_imm8_opt_lsl_i64>;5287 5288 def : SVE_1_Op_Imm_OptLsl_Pat<nxv16i8, op, ZPR8, i32, SVEAddSubImm8Pat, !cast<Instruction>(NAME # _B)>;5289 def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, op, ZPR16, i32, SVEAddSubImm16Pat, !cast<Instruction>(NAME # _H)>;5290 def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, op, ZPR32, i32, SVEAddSubImm32Pat, !cast<Instruction>(NAME # _S)>;5291 def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, op, ZPR64, i64, SVEAddSubImm64Pat, !cast<Instruction>(NAME # _D)>;5292 5293 // Extra patterns for add(x, splat(-ve)) -> sub(x, +ve). There is no i85294 // pattern as all i8 constants can be handled by an add.5295 def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, inv_op, ZPR16, i32, SVEAddSubNegImm16Pat, !cast<Instruction>(NAME # _H)>;5296 def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, inv_op, ZPR32, i32, SVEAddSubNegImm32Pat, !cast<Instruction>(NAME # _S)>;5297 def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, inv_op, ZPR64, i64, SVEAddSubNegImm64Pat, !cast<Instruction>(NAME # _D)>;5298}5299 5300multiclass sve_int_arith_imm0_ssat<bits<3> opc, string asm, SDPatternOperator op,5301 SDPatternOperator inv_op> {5302 def _B : sve_int_arith_imm0<0b00, opc, asm, ZPR8, addsub_imm8_opt_lsl_i8>;5303 def _H : sve_int_arith_imm0<0b01, opc, asm, ZPR16, addsub_imm8_opt_lsl_i16>;5304 def _S : sve_int_arith_imm0<0b10, opc, asm, ZPR32, addsub_imm8_opt_lsl_i32>;5305 def _D : sve_int_arith_imm0<0b11, opc, asm, ZPR64, addsub_imm8_opt_lsl_i64>;5306 5307 def : SVE_1_Op_Imm_OptLsl_Pat<nxv16i8, op, ZPR8, i32, SVEAddSubSSatPosImm8Pat, !cast<Instruction>(NAME # _B)>;5308 def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, op, ZPR16, i32, SVEAddSubSSatPosImm16Pat, !cast<Instruction>(NAME # _H)>;5309 def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, op, ZPR32, i32, SVEAddSubSSatPosImm32Pat, !cast<Instruction>(NAME # _S)>;5310 def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, op, ZPR64, i64, SVEAddSubSSatPosImm64Pat, !cast<Instruction>(NAME # _D)>;5311 5312 def : SVE_1_Op_Imm_OptLsl_Pat<nxv16i8, inv_op, ZPR8, i32, SVEAddSubSSatNegImm8Pat, !cast<Instruction>(NAME # _B)>;5313 def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, inv_op, ZPR16, i32, SVEAddSubSSatNegImm16Pat, !cast<Instruction>(NAME # _H)>;5314 def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, inv_op, ZPR32, i32, SVEAddSubSSatNegImm32Pat, !cast<Instruction>(NAME # _S)>;5315 def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, inv_op, ZPR64, i64, SVEAddSubSSatNegImm64Pat, !cast<Instruction>(NAME # _D)>;5316}5317 5318class sve_int_arith_imm<bits<2> sz8_64, bits<6> opc, string asm,5319 ZPRRegOp zprty, Operand immtype>5320: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm),5321 asm, "\t$Zdn, $_Zdn, $imm",5322 "",5323 []>, Sched<[]> {5324 bits<5> Zdn;5325 bits<8> imm;5326 let Inst{31-24} = 0b00100101;5327 let Inst{23-22} = sz8_64;5328 let Inst{21-16} = opc;5329 let Inst{15-13} = 0b110;5330 let Inst{12-5} = imm;5331 let Inst{4-0} = Zdn;5332 5333 let Constraints = "$Zdn = $_Zdn";5334 let DestructiveInstType = DestructiveOther;5335 let ElementSize = ElementSizeNone;5336 let hasSideEffects = 0;5337}5338 5339multiclass sve_int_arith_imm1<bits<2> opc, string asm, SDPatternOperator op> {5340 def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, simm8_32b>;5341 def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, simm8_32b>;5342 def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, simm8_32b>;5343 def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, simm8_32b>;5344 5345 def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _B)>;5346 def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _H)>;5347 def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _S)>;5348 def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat64, !cast<Instruction>(NAME # _D)>;5349}5350 5351multiclass sve_int_arith_imm1_unsigned<bits<2> opc, string asm, SDPatternOperator op> {5352 def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, imm0_255>;5353 def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, imm0_255>;5354 def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, imm0_255>;5355 def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, imm0_255>;5356 5357 def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithUImm8Pat, !cast<Instruction>(NAME # _B)>;5358 def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithUImm16Pat, !cast<Instruction>(NAME # _H)>;5359 def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithUImm32Pat, !cast<Instruction>(NAME # _S)>;5360 def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithUImm64Pat, !cast<Instruction>(NAME # _D)>;5361}5362 5363multiclass sve_int_arith_imm2<string asm, SDPatternOperator op> {5364 def _B : sve_int_arith_imm<0b00, 0b110000, asm, ZPR8, simm8_32b>;5365 def _H : sve_int_arith_imm<0b01, 0b110000, asm, ZPR16, simm8_32b>;5366 def _S : sve_int_arith_imm<0b10, 0b110000, asm, ZPR32, simm8_32b>;5367 def _D : sve_int_arith_imm<0b11, 0b110000, asm, ZPR64, simm8_32b>;5368 5369 def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _B)>;5370 def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _H)>;5371 def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _S)>;5372 def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat64, !cast<Instruction>(NAME # _D)>;5373}5374 5375//===----------------------------------------------------------------------===//5376// SVE Bitwise Logical - Unpredicated Group5377//===----------------------------------------------------------------------===//5378 5379class sve_int_bin_cons_log<bits<2> opc, string asm>5380: I<(outs ZPR64:$Zd), (ins ZPR64:$Zn, ZPR64:$Zm),5381 asm, "\t$Zd, $Zn, $Zm",5382 "",5383 []>, Sched<[]> {5384 bits<5> Zd;5385 bits<5> Zm;5386 bits<5> Zn;5387 let Inst{31-24} = 0b00000100;5388 let Inst{23-22} = opc{1-0};5389 let Inst{21} = 0b1;5390 let Inst{20-16} = Zm;5391 let Inst{15-10} = 0b001100;5392 let Inst{9-5} = Zn;5393 let Inst{4-0} = Zd;5394 5395 let hasSideEffects = 0;5396}5397 5398multiclass sve_int_bin_cons_log<bits<2> opc, string asm, SDPatternOperator op> {5399 def NAME : sve_int_bin_cons_log<opc, asm>;5400 5401 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>;5402 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>;5403 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>;5404 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>;5405 5406 def : InstAlias<asm # "\t$Zd, $Zn, $Zm",5407 (!cast<Instruction>(NAME) ZPR8:$Zd, ZPR8:$Zn, ZPR8:$Zm), 1>;5408 def : InstAlias<asm # "\t$Zd, $Zn, $Zm",5409 (!cast<Instruction>(NAME) ZPR16:$Zd, ZPR16:$Zn, ZPR16:$Zm), 1>;5410 def : InstAlias<asm # "\t$Zd, $Zn, $Zm",5411 (!cast<Instruction>(NAME) ZPR32:$Zd, ZPR32:$Zn, ZPR32:$Zm), 1>;5412}5413 5414class sve2_int_bitwise_ternary_op_d<bits<3> opc, string asm>5415: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, ZPR64:$Zm, ZPR64:$Zk),5416 asm, "\t$Zdn, $_Zdn, $Zm, $Zk",5417 "",5418 []>, Sched<[]> {5419 bits<5> Zdn;5420 bits<5> Zk;5421 bits<5> Zm;5422 let Inst{31-24} = 0b00000100;5423 let Inst{23-22} = opc{2-1};5424 let Inst{21} = 0b1;5425 let Inst{20-16} = Zm;5426 let Inst{15-11} = 0b00111;5427 let Inst{10} = opc{0};5428 let Inst{9-5} = Zk;5429 let Inst{4-0} = Zdn;5430 5431 let Constraints = "$Zdn = $_Zdn";5432 let DestructiveInstType = DestructiveOther;5433 let ElementSize = ElementSizeNone;5434 let hasSideEffects = 0;5435}5436 5437multiclass sve2_int_bitwise_ternary_op<bits<3> opc, string asm,5438 SDPatternOperator op> {5439 def NAME : sve2_int_bitwise_ternary_op_d<opc, asm>;5440 5441 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk",5442 (!cast<Instruction>(NAME) ZPR8:$Zdn, ZPR8:$Zm, ZPR8:$Zk), 1>;5443 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk",5444 (!cast<Instruction>(NAME) ZPR16:$Zdn, ZPR16:$Zm, ZPR16:$Zk), 1>;5445 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk",5446 (!cast<Instruction>(NAME) ZPR32:$Zdn, ZPR32:$Zm, ZPR32:$Zk), 1>;5447 5448 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>;5449 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>;5450 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>;5451 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>;5452 5453 // Allow selecting SVE2 ternary ops with Neon types.5454 foreach VT = [nxv16i8, nxv8i16, nxv4i32, nxv2i64] in {5455 def : Pat<(SVEType<VT>.DSub (op V64:$op1, V64:$op2, V64:$op3)),5456 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $op1, dsub),5457 (INSERT_SUBREG (IMPLICIT_DEF), $op2, dsub),5458 (INSERT_SUBREG (IMPLICIT_DEF), $op3, dsub)), dsub)>;5459 5460 def : Pat<(SVEType<VT>.ZSub (op V128:$op1, V128:$op2, V128:$op3)),5461 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $op1, zsub),5462 (INSERT_SUBREG (IMPLICIT_DEF), $op2, zsub),5463 (INSERT_SUBREG (IMPLICIT_DEF), $op3, zsub)), zsub)>;5464 }5465}5466 5467class sve2_int_rotate_right_imm<bits<4> tsz8_64, string asm,5468 ZPRRegOp zprty, Operand immtype>5469: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, immtype:$imm),5470 asm, "\t$Zdn, $_Zdn, $Zm, $imm",5471 "",5472 []>, Sched<[]> {5473 bits<5> Zdn;5474 bits<5> Zm;5475 bits<6> imm;5476 let Inst{31-24} = 0b00000100;5477 let Inst{23-22} = tsz8_64{3-2};5478 let Inst{21} = 0b1;5479 let Inst{20-19} = tsz8_64{1-0};5480 let Inst{18-16} = imm{2-0}; // imm35481 let Inst{15-10} = 0b001101;5482 let Inst{9-5} = Zm;5483 let Inst{4-0} = Zdn;5484 5485 let Constraints = "$Zdn = $_Zdn";5486 let DestructiveInstType = DestructiveOther;5487 let ElementSize = ElementSizeNone;5488 let hasSideEffects = 0;5489}5490 5491multiclass sve2_int_rotate_right_imm<string asm, SDPatternOperator op> {5492 def _B : sve2_int_rotate_right_imm<{0,0,0,1}, asm, ZPR8, vecshiftR8>;5493 def _H : sve2_int_rotate_right_imm<{0,0,1,?}, asm, ZPR16, vecshiftR16> {5494 let Inst{19} = imm{3};5495 }5496 def _S : sve2_int_rotate_right_imm<{0,1,?,?}, asm, ZPR32, vecshiftR32> {5497 let Inst{20-19} = imm{4-3};5498 }5499 def _D : sve2_int_rotate_right_imm<{1,?,?,?}, asm, ZPR64, vecshiftR64> {5500 let Inst{22} = imm{5};5501 let Inst{20-19} = imm{4-3};5502 }5503 5504 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, vecshiftR8, !cast<Instruction>(NAME # _B)>;5505 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, vecshiftR16, !cast<Instruction>(NAME # _H)>;5506 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, vecshiftR32, !cast<Instruction>(NAME # _S)>;5507 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, vecshiftR64, !cast<Instruction>(NAME # _D)>;5508}5509 5510//===----------------------------------------------------------------------===//5511// SVE Integer Wide Immediate - Predicated Group5512//===----------------------------------------------------------------------===//5513 5514class sve_int_dup_fpimm_pred<bits<2> sz, Operand fpimmtype,5515 string asm, ZPRRegOp zprty>5516: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPRAny:$Pg, fpimmtype:$imm8),5517 asm, "\t$Zd, $Pg/m, $imm8",5518 "",5519 []>, Sched<[]> {5520 bits<4> Pg;5521 bits<5> Zd;5522 bits<8> imm8;5523 let Inst{31-24} = 0b00000101;5524 let Inst{23-22} = sz;5525 let Inst{21-20} = 0b01;5526 let Inst{19-16} = Pg;5527 let Inst{15-13} = 0b110;5528 let Inst{12-5} = imm8;5529 let Inst{4-0} = Zd;5530 5531 let Constraints = "$Zd = $_Zd";5532 let DestructiveInstType = DestructiveOther;5533 let ElementSize = zprty.ElementSize;5534 let hasSideEffects = 0;5535}5536 5537multiclass sve_int_dup_fpimm_pred<string asm> {5538 def _H : sve_int_dup_fpimm_pred<0b01, fpimm16, asm, ZPR16>;5539 def _S : sve_int_dup_fpimm_pred<0b10, fpimm32, asm, ZPR32>;5540 def _D : sve_int_dup_fpimm_pred<0b11, fpimm64, asm, ZPR64>;5541 5542 def : InstAlias<"fmov $Zd, $Pg/m, $imm8",5543 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, fpimm16:$imm8), 1>;5544 def : InstAlias<"fmov $Zd, $Pg/m, $imm8",5545 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, fpimm32:$imm8), 1>;5546 def : InstAlias<"fmov $Zd, $Pg/m, $imm8",5547 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, fpimm64:$imm8), 1>;5548 5549 def : Pat<(nxv8f16 (vselect nxv8i1:$pg, (splat_vector fpimm16:$imm8), nxv8f16:$zd)),5550 (!cast<Instruction>(NAME # _H) $zd, $pg, fpimm16:$imm8)>;5551 def : Pat<(nxv4f16 (vselect nxv4i1:$pg, (splat_vector fpimm16:$imm8), nxv4f16:$zd)),5552 (!cast<Instruction>(NAME # _H) $zd, $pg, fpimm16:$imm8)>;5553 def : Pat<(nxv2f16 (vselect nxv2i1:$pg, (splat_vector fpimm16:$imm8), nxv2f16:$zd)),5554 (!cast<Instruction>(NAME # _H) $zd, $pg, fpimm16:$imm8)>;5555 def : Pat<(nxv4f32 (vselect nxv4i1:$pg, (splat_vector fpimm32:$imm8), nxv4f32:$zd)),5556 (!cast<Instruction>(NAME # _S) $zd, $pg, fpimm32:$imm8)>;5557 def : Pat<(nxv2f32 (vselect nxv2i1:$pg, (splat_vector fpimm32:$imm8), nxv2f32:$zd)),5558 (!cast<Instruction>(NAME # _S) $zd, $pg, fpimm32:$imm8)>;5559 def : Pat<(nxv2f64 (vselect nxv2i1:$pg, (splat_vector fpimm64:$imm8), nxv2f64:$zd)),5560 (!cast<Instruction>(NAME # _D) $zd, $pg, fpimm64:$imm8)>;5561 5562 // Some half precision immediates alias with bfloat (e.g. f16(1.875) == bf16(1.0)).5563 def : Pat<(nxv8bf16 (vselect nxv8i1:$pg, (splat_vector fpimmbf16:$imm8), nxv8bf16:$zd)),5564 (!cast<Instruction>(NAME # _H) $zd, $pg, (fpimm16XForm bf16:$imm8))>;5565 def : Pat<(nxv4bf16 (vselect nxv4i1:$pg, (splat_vector fpimmbf16:$imm8), nxv4bf16:$zd)),5566 (!cast<Instruction>(NAME # _H) $zd, $pg, (fpimm16XForm bf16:$imm8))>;5567 def : Pat<(nxv2bf16 (vselect nxv2i1:$pg, (splat_vector fpimmbf16:$imm8), nxv2bf16:$zd)),5568 (!cast<Instruction>(NAME # _H) $zd, $pg, (fpimm16XForm bf16:$imm8))>;5569}5570 5571class sve_int_dup_imm_pred<bits<2> sz8_64, bit m, string asm,5572 ZPRRegOp zprty, string pred_qual, dag iops>5573: I<(outs zprty:$Zd), iops,5574 asm, "\t$Zd, $Pg"#pred_qual#", $imm",5575 "", []>, Sched<[]> {5576 bits<5> Zd;5577 bits<4> Pg;5578 bits<9> imm;5579 let Inst{31-24} = 0b00000101;5580 let Inst{23-22} = sz8_64;5581 let Inst{21-20} = 0b01;5582 let Inst{19-16} = Pg;5583 let Inst{15} = 0b0;5584 let Inst{14} = m;5585 let Inst{13} = imm{8}; // sh5586 let Inst{12-5} = imm{7-0}; // imm85587 let Inst{4-0} = Zd;5588 5589 let DestructiveInstType = DestructiveOther;5590 let ElementSize = zprty.ElementSize;5591 let hasSideEffects = 0;5592}5593 5594multiclass sve_int_dup_imm_pred_merge_inst<5595 bits<2> sz8_64, string asm, ZPRRegOp zprty, imm8_opt_lsl cpyimm,5596 ValueType intty, ValueType predty, ValueType scalarty, ComplexPattern cpx> {5597 let Constraints = "$Zd = $_Zd" in5598 def NAME : sve_int_dup_imm_pred<sz8_64, 1, asm, zprty, "/m",5599 (ins zprty:$_Zd, PPRAny:$Pg, cpyimm:$imm)>;5600 def : InstAlias<"mov $Zd, $Pg/m, $imm",5601 (!cast<Instruction>(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>;5602 def : Pat<(vselect predty:$Pg,5603 (intty (splat_vector (scalarty (cpx i32:$imm, i32:$shift)))),5604 ZPR:$Zd),5605 (!cast<Instruction>(NAME) $Zd, $Pg, $imm, $shift)>;5606}5607 5608multiclass sve_int_dup_imm_pred_merge<string asm, SDPatternOperator op> {5609 defm _B : sve_int_dup_imm_pred_merge_inst<0b00, asm, ZPR8, cpy_imm8_opt_lsl_i8,5610 nxv16i8, nxv16i1, i32, SVECpyDupImm8Pat>;5611 defm _H : sve_int_dup_imm_pred_merge_inst<0b01, asm, ZPR16, cpy_imm8_opt_lsl_i16,5612 nxv8i16, nxv8i1, i32, SVECpyDupImm16Pat>;5613 defm _S : sve_int_dup_imm_pred_merge_inst<0b10, asm, ZPR32, cpy_imm8_opt_lsl_i32,5614 nxv4i32, nxv4i1, i32, SVECpyDupImm32Pat>;5615 defm _D : sve_int_dup_imm_pred_merge_inst<0b11, asm, ZPR64, cpy_imm8_opt_lsl_i64,5616 nxv2i64, nxv2i1, i64, SVECpyDupImm64Pat>;5617 5618 def : InstAlias<"fmov $Zd, $Pg/m, #0.0",5619 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg,5620 (cpy_imm8_opt_lsl_i16 0, 0)), 0>;5621 def : InstAlias<"fmov $Zd, $Pg/m, #0.0",5622 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg,5623 (cpy_imm8_opt_lsl_i32 0, 0)), 0>;5624 def : InstAlias<"fmov $Zd, $Pg/m, #0.0",5625 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg,5626 (cpy_imm8_opt_lsl_i64 0, 0)), 0>;5627 5628 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv8f16 ZPR:$Zd)),5629 (!cast<Instruction>(NAME # _H) $Zd, $Pg, 0, 0)>;5630 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv4f16 ZPR:$Zd)),5631 (!cast<Instruction>(NAME # _S) $Zd, $Pg, 0, 0)>;5632 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv2f16 ZPR:$Zd)),5633 (!cast<Instruction>(NAME # _D) $Zd, $Pg, 0, 0)>;5634 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv4f32 ZPR:$Zd)),5635 (!cast<Instruction>(NAME # _S) $Zd, $Pg, 0, 0)>;5636 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv2f32 ZPR:$Zd)),5637 (!cast<Instruction>(NAME # _D) $Zd, $Pg, 0, 0)>;5638 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv2f64 ZPR:$Zd)),5639 (!cast<Instruction>(NAME # _D) $Zd, $Pg, 0, 0)>;5640 5641 def : Pat<(nxv16i8 (op nxv16i1:$pg, (i32 (SVECpyDupImm8Pat i32:$a, i32:$b)), nxv16i8:$zd)),5642 (!cast<Instruction>(NAME # _B) $zd, $pg, $a, $b)>;5643 def : Pat<(nxv8i16 (op nxv8i1:$pg, (i32 (SVECpyDupImm16Pat i32:$a, i32:$b)), nxv8i16:$zd)),5644 (!cast<Instruction>(NAME # _H) $zd, $pg, $a, $b)>;5645 def : Pat<(nxv4i32 (op nxv4i1:$pg, (i32 (SVECpyDupImm32Pat i32:$a, i32:$b)), nxv4i32:$zd)),5646 (!cast<Instruction>(NAME # _S) $zd, $pg, $a, $b)>;5647 def : Pat<(nxv2i64 (op nxv2i1:$pg, (i64 (SVECpyDupImm64Pat i32:$a, i32:$b)), nxv2i64:$zd)),5648 (!cast<Instruction>(NAME # _D) $zd, $pg, $a, $b)>;5649}5650 5651multiclass sve_int_dup_imm_pred_zero_inst<5652 bits<2> sz8_64, string asm, ZPRRegOp zprty, imm8_opt_lsl cpyimm,5653 ValueType intty, ValueType predty, ValueType scalarty, ComplexPattern cpx> {5654 def NAME : sve_int_dup_imm_pred<sz8_64, 0, asm, zprty, "/z",5655 (ins PPRAny:$Pg, cpyimm:$imm)>;5656 def : InstAlias<"mov $Zd, $Pg/z, $imm",5657 (!cast<Instruction>(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>;5658 def : Pat<(intty (zext (predty PPRAny:$Ps1))),5659 (!cast<Instruction>(NAME) PPRAny:$Ps1, 1, 0)>;5660 def : Pat<(intty (sext (predty PPRAny:$Ps1))),5661 (!cast<Instruction>(NAME) PPRAny:$Ps1, -1, 0)>;5662 def : Pat<(intty (anyext (predty PPRAny:$Ps1))),5663 (!cast<Instruction>(NAME) PPRAny:$Ps1, 1, 0)>;5664 def : Pat<(vselect predty:$Pg,5665 (intty (splat_vector (scalarty (cpx i32:$imm, i32:$shift)))),5666 (intty (splat_vector (scalarty 0)))),5667 (!cast<Instruction>(NAME) $Pg, $imm, $shift)>;5668}5669 5670multiclass sve_int_dup_imm_pred_zero<string asm, SDPatternOperator op> {5671 defm _B : sve_int_dup_imm_pred_zero_inst<0b00, asm, ZPR8, cpy_imm8_opt_lsl_i8,5672 nxv16i8, nxv16i1, i32, SVECpyDupImm8Pat>;5673 defm _H : sve_int_dup_imm_pred_zero_inst<0b01, asm, ZPR16, cpy_imm8_opt_lsl_i16,5674 nxv8i16, nxv8i1, i32, SVECpyDupImm16Pat>;5675 defm _S : sve_int_dup_imm_pred_zero_inst<0b10, asm, ZPR32, cpy_imm8_opt_lsl_i32,5676 nxv4i32, nxv4i1, i32, SVECpyDupImm32Pat>;5677 defm _D : sve_int_dup_imm_pred_zero_inst<0b11, asm, ZPR64, cpy_imm8_opt_lsl_i64,5678 nxv2i64, nxv2i1, i64, SVECpyDupImm64Pat>;5679 5680 def : Pat<(nxv16i8 (op nxv16i1:$pg, (i32 (SVECpyDupImm8Pat i32:$a, i32:$b)), (SVEDup0))),5681 (!cast<Instruction>(NAME # _B) $pg, $a, $b)>;5682 def : Pat<(nxv8i16 (op nxv8i1:$pg, (i32 (SVECpyDupImm16Pat i32:$a, i32:$b)), (SVEDup0))),5683 (!cast<Instruction>(NAME # _H) $pg, $a, $b)>;5684 def : Pat<(nxv4i32 (op nxv4i1:$pg, (i32 (SVECpyDupImm32Pat i32:$a, i32:$b)), (SVEDup0))),5685 (!cast<Instruction>(NAME # _S) $pg, $a, $b)>;5686 def : Pat<(nxv2i64 (op nxv2i1:$pg, (i64 (SVECpyDupImm64Pat i32:$a, i32:$b)), (SVEDup0))),5687 (!cast<Instruction>(NAME # _D) $pg, $a, $b)>;5688}5689 5690//===----------------------------------------------------------------------===//5691// SVE Integer Compare - Vectors Group5692//===----------------------------------------------------------------------===//5693 5694class sve_int_cmp<bit cmp_1, bits<2> sz8_64, bits<3> opc, string asm,5695 PPRRegOp pprty, ZPRRegOp zprty1, ZPRRegOp zprty2>5696: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty1:$Zn, zprty2:$Zm),5697 asm, "\t$Pd, $Pg/z, $Zn, $Zm",5698 "",5699 []>, Sched<[]> {5700 bits<4> Pd;5701 bits<3> Pg;5702 bits<5> Zm;5703 bits<5> Zn;5704 let Inst{31-24} = 0b00100100;5705 let Inst{23-22} = sz8_64;5706 let Inst{21} = 0b0;5707 let Inst{20-16} = Zm;5708 let Inst{15} = opc{2};5709 let Inst{14} = cmp_1;5710 let Inst{13} = opc{1};5711 let Inst{12-10} = Pg;5712 let Inst{9-5} = Zn;5713 let Inst{4} = opc{0};5714 let Inst{3-0} = Pd;5715 5716 let Defs = [NZCV];5717 let ElementSize = pprty.ElementSize;5718 let hasSideEffects = 0;5719 let isPTestLike = 1;5720}5721 5722multiclass SVE_SETCC_Pat<CondCode cc, CondCode invcc, ValueType predvt,5723 ValueType intvt, Instruction cmp> {5724 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, cc)),5725 (cmp $Op1, $Op2, $Op3)>;5726 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, invcc)),5727 (cmp $Op1, $Op3, $Op2)>;5728 def : Pat<(predvt (and predvt:$Pg, (AArch64setcc_z_oneuse (predvt (SVEAllActive)), intvt:$Op2, intvt:$Op3, cc))),5729 (cmp $Pg, $Op2, $Op3)>;5730 def : Pat<(predvt (and predvt:$Pg, (AArch64setcc_z_oneuse (predvt (SVEAllActive)), intvt:$Op2, intvt:$Op3, invcc))),5731 (cmp $Pg, $Op3, $Op2)>;5732}5733 5734multiclass SVE_SETCC_Pat_With_Zero<CondCode cc, CondCode invcc, ValueType predvt,5735 ValueType intvt, Instruction cmp> {5736 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, (SVEDup0), cc)),5737 (cmp $Op1, $Op2)>;5738 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, (SVEDup0), intvt:$Op2, invcc)),5739 (cmp $Op1, $Op2)>;5740 def : Pat<(predvt (and predvt:$Pg, (AArch64setcc_z_oneuse (predvt (SVEAllActive)), intvt:$Op1, (SVEDup0), cc))),5741 (cmp $Pg, $Op1)>;5742 def : Pat<(predvt (and predvt:$Pg, (AArch64setcc_z_oneuse (predvt (SVEAllActive)), (SVEDup0), intvt:$Op1, invcc))),5743 (cmp $Pg, $Op1)>;5744}5745 5746multiclass sve_int_cmp_0<bits<3> opc, string asm, CondCode cc, CondCode invcc> {5747 def _B : sve_int_cmp<0b0, 0b00, opc, asm, PPR8, ZPR8, ZPR8>;5748 def _H : sve_int_cmp<0b0, 0b01, opc, asm, PPR16, ZPR16, ZPR16>;5749 def _S : sve_int_cmp<0b0, 0b10, opc, asm, PPR32, ZPR32, ZPR32>;5750 def _D : sve_int_cmp<0b0, 0b11, opc, asm, PPR64, ZPR64, ZPR64>;5751 5752 defm : SVE_SETCC_Pat<cc, invcc, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;5753 defm : SVE_SETCC_Pat<cc, invcc, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>;5754 defm : SVE_SETCC_Pat<cc, invcc, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;5755 defm : SVE_SETCC_Pat<cc, invcc, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;5756}5757 5758multiclass sve_int_cmp_0_wide<bits<3> opc, string asm, SDPatternOperator op> {5759 def _B : sve_int_cmp<0b0, 0b00, opc, asm, PPR8, ZPR8, ZPR64>;5760 def _H : sve_int_cmp<0b0, 0b01, opc, asm, PPR16, ZPR16, ZPR64>;5761 def _S : sve_int_cmp<0b0, 0b10, opc, asm, PPR32, ZPR32, ZPR64>;5762 5763 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>;5764 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>;5765 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>;5766}5767 5768multiclass sve_int_cmp_1_wide<bits<3> opc, string asm, SDPatternOperator op> {5769 def _B : sve_int_cmp<0b1, 0b00, opc, asm, PPR8, ZPR8, ZPR64>;5770 def _H : sve_int_cmp<0b1, 0b01, opc, asm, PPR16, ZPR16, ZPR64>;5771 def _S : sve_int_cmp<0b1, 0b10, opc, asm, PPR32, ZPR32, ZPR64>;5772 5773 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>;5774 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>;5775 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>;5776}5777 5778 5779//===----------------------------------------------------------------------===//5780// SVE Integer Compare - Signed Immediate Group5781//===----------------------------------------------------------------------===//5782 5783class sve_int_scmp_vi<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty,5784 ZPRRegOp zprty,5785 Operand immtype>5786: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm5),5787 asm, "\t$Pd, $Pg/z, $Zn, $imm5",5788 "",5789 []>, Sched<[]> {5790 bits<4> Pd;5791 bits<3> Pg;5792 bits<5> Zn;5793 bits<5> imm5;5794 let Inst{31-24} = 0b00100101;5795 let Inst{23-22} = sz8_64;5796 let Inst{21} = 0b0;5797 let Inst{20-16} = imm5;5798 let Inst{15} = opc{2};5799 let Inst{14} = 0b0;5800 let Inst{13} = opc{1};5801 let Inst{12-10} = Pg;5802 let Inst{9-5} = Zn;5803 let Inst{4} = opc{0};5804 let Inst{3-0} = Pd;5805 5806 let Defs = [NZCV];5807 let ElementSize = pprty.ElementSize;5808 let hasSideEffects = 0;5809 let isPTestLike = 1;5810}5811 5812multiclass SVE_SETCC_Imm_Pat<CondCode cc, CondCode commuted_cc,5813 ValueType predvt, ValueType intvt,5814 Operand immtype, Instruction cmp> {5815 def : Pat<(predvt (AArch64setcc_z (predvt PPR_3b:$Pg),5816 (intvt ZPR:$Zs1),5817 (intvt (splat_vector (immtype:$imm))),5818 cc)),5819 (cmp $Pg, $Zs1, immtype:$imm)>;5820 def : Pat<(predvt (AArch64setcc_z (predvt PPR_3b:$Pg),5821 (intvt (splat_vector (immtype:$imm))),5822 (intvt ZPR:$Zs1),5823 commuted_cc)),5824 (cmp $Pg, $Zs1, immtype:$imm)>;5825 def : Pat<(predvt (and predvt:$Pg,5826 (AArch64setcc_z_oneuse (predvt (SVEAllActive)),5827 (intvt ZPR:$Zs1),5828 (intvt (splat_vector (immtype:$imm))),5829 cc))),5830 (cmp $Pg, $Zs1, immtype:$imm)>;5831 def : Pat<(predvt (and predvt:$Pg,5832 (AArch64setcc_z_oneuse (predvt (SVEAllActive)),5833 (intvt (splat_vector (immtype:$imm))),5834 (intvt ZPR:$Zs1),5835 commuted_cc))),5836 (cmp $Pg, $Zs1, immtype:$imm)>;5837}5838 5839multiclass sve_int_scmp_vi<bits<3> opc, string asm, CondCode cc, CondCode commuted_cc> {5840 def _B : sve_int_scmp_vi<0b00, opc, asm, PPR8, ZPR8, simm5_32b>;5841 def _H : sve_int_scmp_vi<0b01, opc, asm, PPR16, ZPR16, simm5_32b>;5842 def _S : sve_int_scmp_vi<0b10, opc, asm, PPR32, ZPR32, simm5_32b>;5843 def _D : sve_int_scmp_vi<0b11, opc, asm, PPR64, ZPR64, simm5_64b>;5844 5845 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv16i1, nxv16i8, simm5_32b,5846 !cast<Instruction>(NAME # _B)>;5847 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv8i1, nxv8i16, simm5_32b,5848 !cast<Instruction>(NAME # _H)>;5849 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv4i1, nxv4i32, simm5_32b,5850 !cast<Instruction>(NAME # _S)>;5851 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv2i1, nxv2i64, simm5_64b,5852 !cast<Instruction>(NAME # _D)>;5853}5854 5855 5856//===----------------------------------------------------------------------===//5857// SVE Integer Compare - Unsigned Immediate Group5858//===----------------------------------------------------------------------===//5859 5860class sve_int_ucmp_vi<bits<2> sz8_64, bits<2> opc, string asm, PPRRegOp pprty,5861 ZPRRegOp zprty, Operand immtype>5862: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm7),5863 asm, "\t$Pd, $Pg/z, $Zn, $imm7",5864 "",5865 []>, Sched<[]> {5866 bits<4> Pd;5867 bits<3> Pg;5868 bits<5> Zn;5869 bits<7> imm7;5870 let Inst{31-24} = 0b00100100;5871 let Inst{23-22} = sz8_64;5872 let Inst{21} = 1;5873 let Inst{20-14} = imm7;5874 let Inst{13} = opc{1};5875 let Inst{12-10} = Pg;5876 let Inst{9-5} = Zn;5877 let Inst{4} = opc{0};5878 let Inst{3-0} = Pd;5879 5880 let Defs = [NZCV];5881 let ElementSize = pprty.ElementSize;5882 let hasSideEffects = 0;5883 let isPTestLike = 1;5884}5885 5886multiclass sve_int_ucmp_vi<bits<2> opc, string asm, CondCode cc,5887 CondCode commuted_cc> {5888 def _B : sve_int_ucmp_vi<0b00, opc, asm, PPR8, ZPR8, imm0_127>;5889 def _H : sve_int_ucmp_vi<0b01, opc, asm, PPR16, ZPR16, imm0_127>;5890 def _S : sve_int_ucmp_vi<0b10, opc, asm, PPR32, ZPR32, imm0_127>;5891 def _D : sve_int_ucmp_vi<0b11, opc, asm, PPR64, ZPR64, imm0_127_64b>;5892 5893 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv16i1, nxv16i8, imm0_127,5894 !cast<Instruction>(NAME # _B)>;5895 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv8i1, nxv8i16, imm0_127,5896 !cast<Instruction>(NAME # _H)>;5897 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv4i1, nxv4i32, imm0_127,5898 !cast<Instruction>(NAME # _S)>;5899 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv2i1, nxv2i64, imm0_127_64b,5900 !cast<Instruction>(NAME # _D)>;5901}5902 5903 5904//===----------------------------------------------------------------------===//5905// SVE Integer Compare - Scalars Group5906//===----------------------------------------------------------------------===//5907 5908class sve_int_cterm<bit sz, bit opc, string asm, RegisterClass rt>5909: I<(outs), (ins rt:$Rn, rt:$Rm),5910 asm, "\t$Rn, $Rm",5911 "",5912 []>, Sched<[]> {5913 bits<5> Rm;5914 bits<5> Rn;5915 let Inst{31-23} = 0b001001011;5916 let Inst{22} = sz;5917 let Inst{21} = 0b1;5918 let Inst{20-16} = Rm;5919 let Inst{15-10} = 0b001000;5920 let Inst{9-5} = Rn;5921 let Inst{4} = opc;5922 let Inst{3-0} = 0b0000;5923 5924 let Defs = [NZCV];5925 let hasSideEffects = 0;5926}5927 5928class sve_int_while_rr<bits<2> sz8_64, bits<4> opc, string asm,5929 RegisterClass gprty, PPRRegOp pprty>5930: I<(outs pprty:$Pd), (ins gprty:$Rn, gprty:$Rm),5931 asm, "\t$Pd, $Rn, $Rm",5932 "", []>, Sched<[]> {5933 bits<4> Pd;5934 bits<5> Rm;5935 bits<5> Rn;5936 let Inst{31-24} = 0b00100101;5937 let Inst{23-22} = sz8_64;5938 let Inst{21} = 0b1;5939 let Inst{20-16} = Rm;5940 let Inst{15-13} = 0b000;5941 let Inst{12-10} = opc{3-1};5942 let Inst{9-5} = Rn;5943 let Inst{4} = opc{0};5944 let Inst{3-0} = Pd;5945 5946 let Defs = [NZCV];5947 let ElementSize = pprty.ElementSize;5948 let hasSideEffects = 0;5949 let isWhile = 1;5950}5951 5952multiclass sve_int_while4_rr<bits<3> opc, string asm, SDPatternOperator op,5953 SDPatternOperator rev_op> {5954 def _B : sve_int_while_rr<0b00, { 0, opc }, asm, GPR32, PPR8>;5955 def _H : sve_int_while_rr<0b01, { 0, opc }, asm, GPR32, PPR16>;5956 def _S : sve_int_while_rr<0b10, { 0, opc }, asm, GPR32, PPR32>;5957 def _D : sve_int_while_rr<0b11, { 0, opc }, asm, GPR32, PPR64>;5958 5959 def : SVE_2_Op_Pat<nxv16i1, op, i32, i32, !cast<Instruction>(NAME # _B)>;5960 def : SVE_2_Op_Pat<nxv8i1, op, i32, i32, !cast<Instruction>(NAME # _H)>;5961 def : SVE_2_Op_Pat<nxv4i1, op, i32, i32, !cast<Instruction>(NAME # _S)>;5962 def : SVE_2_Op_Pat<nxv2i1, op, i32, i32, !cast<Instruction>(NAME # _D)>;5963 5964 def : Pat<(nxv16i1 (vector_reverse (rev_op i32:$op2, i32:$op1))),5965 (!cast<Instruction>(NAME # "_B") $op1, $op2)>;5966 def : Pat<(nxv8i1 (vector_reverse (rev_op i32:$op2, i32:$op1))),5967 (!cast<Instruction>(NAME # "_H") $op1, $op2)>;5968 def : Pat<(nxv4i1 (vector_reverse (rev_op i32:$op2, i32:$op1))),5969 (!cast<Instruction>(NAME # "_S") $op1, $op2)>;5970 def : Pat<(nxv2i1 (vector_reverse (rev_op i32:$op2, i32:$op1))),5971 (!cast<Instruction>(NAME # "_D") $op1, $op2)>;5972}5973 5974multiclass sve_int_while8_rr<bits<3> opc, string asm, SDPatternOperator op,5975 SDPatternOperator rev_op> {5976 def _B : sve_int_while_rr<0b00, { 1, opc }, asm, GPR64, PPR8>;5977 def _H : sve_int_while_rr<0b01, { 1, opc }, asm, GPR64, PPR16>;5978 def _S : sve_int_while_rr<0b10, { 1, opc }, asm, GPR64, PPR32>;5979 def _D : sve_int_while_rr<0b11, { 1, opc }, asm, GPR64, PPR64>;5980 5981 def : SVE_2_Op_Pat<nxv16i1, op, i64, i64, !cast<Instruction>(NAME # _B)>;5982 def : SVE_2_Op_Pat<nxv8i1, op, i64, i64, !cast<Instruction>(NAME # _H)>;5983 def : SVE_2_Op_Pat<nxv4i1, op, i64, i64, !cast<Instruction>(NAME # _S)>;5984 def : SVE_2_Op_Pat<nxv2i1, op, i64, i64, !cast<Instruction>(NAME # _D)>;5985 5986 def : Pat<(nxv16i1 (vector_reverse (rev_op i64:$op2, i64:$op1))),5987 (!cast<Instruction>(NAME # "_B") $op1, $op2)>;5988 def : Pat<(nxv8i1 (vector_reverse (rev_op i64:$op2, i64:$op1))),5989 (!cast<Instruction>(NAME # "_H") $op1, $op2)>;5990 def : Pat<(nxv4i1 (vector_reverse (rev_op i64:$op2, i64:$op1))),5991 (!cast<Instruction>(NAME # "_S") $op1, $op2)>;5992 def : Pat<(nxv2i1 (vector_reverse (rev_op i64:$op2, i64:$op1))),5993 (!cast<Instruction>(NAME # "_D") $op1, $op2)>;5994}5995 5996class sve2_int_while_rr<bits<2> sz8_64, bits<1> rw, string asm,5997 PPRRegOp pprty>5998: I<(outs pprty:$Pd), (ins GPR64:$Rn, GPR64:$Rm),5999 asm, "\t$Pd, $Rn, $Rm",6000 "", []>, Sched<[]> {6001 bits<4> Pd;6002 bits<5> Rm;6003 bits<5> Rn;6004 let Inst{31-24} = 0b00100101;6005 let Inst{23-22} = sz8_64;6006 let Inst{21} = 0b1;6007 let Inst{20-16} = Rm;6008 let Inst{15-10} = 0b001100;6009 let Inst{9-5} = Rn;6010 let Inst{4} = rw;6011 let Inst{3-0} = Pd;6012 6013 let Defs = [NZCV];6014 let ElementSize = pprty.ElementSize;6015 let hasSideEffects = 0;6016 let isWhile = 1;6017}6018 6019multiclass sve2_int_while_rr<bits<1> rw, string asm, SDPatternOperator op> {6020 def _B : sve2_int_while_rr<0b00, rw, asm, PPR8>;6021 def _H : sve2_int_while_rr<0b01, rw, asm, PPR16>;6022 def _S : sve2_int_while_rr<0b10, rw, asm, PPR32>;6023 def _D : sve2_int_while_rr<0b11, rw, asm, PPR64>;6024 6025 def : Pat<(nxv16i1 (op i64:$Op1, i64:$Op2, (i64 1))),6026 (!cast<Instruction>(NAME # _B) $Op1, $Op2)>;6027 def : Pat<(nxv8i1 (op i64:$Op1, i64:$Op2, (i64 2))),6028 (!cast<Instruction>(NAME # _H) $Op1, $Op2)>;6029 def : Pat<(nxv4i1 (op i64:$Op1, i64:$Op2, (i64 4))),6030 (!cast<Instruction>(NAME # _S) $Op1, $Op2)>;6031 def : Pat<(nxv2i1 (op i64:$Op1, i64:$Op2, (i64 8))),6032 (!cast<Instruction>(NAME # _D) $Op1, $Op2)>;6033}6034 6035//===----------------------------------------------------------------------===//6036// SVE Floating Point Fast Reduction Group6037//===----------------------------------------------------------------------===//6038 6039class sve_fp_fast_red<bits<2> sz, bits<3> opc, string asm,6040 ZPRRegOp zprty, FPRasZPROperand dstOpType>6041: I<(outs dstOpType:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn),6042 asm, "\t$Vd, $Pg, $Zn",6043 "",6044 []>, Sched<[]> {6045 bits<5> Zn;6046 bits<5> Vd;6047 bits<3> Pg;6048 let Inst{31-24} = 0b01100101;6049 let Inst{23-22} = sz;6050 let Inst{21-19} = 0b000;6051 let Inst{18-16} = opc;6052 let Inst{15-13} = 0b001;6053 let Inst{12-10} = Pg;6054 let Inst{9-5} = Zn;6055 let Inst{4-0} = Vd;6056 6057 let hasSideEffects = 0;6058 let mayRaiseFPException = 1;6059}6060 6061multiclass sve_fp_fast_red<bits<3> opc, string asm, SDPatternOperator op> {6062 def _H : sve_fp_fast_red<0b01, opc, asm, ZPR16, FPR16asZPR>;6063 def _S : sve_fp_fast_red<0b10, opc, asm, ZPR32, FPR32asZPR>;6064 def _D : sve_fp_fast_red<0b11, opc, asm, ZPR64, FPR64asZPR>;6065 6066 def : SVE_2_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>;6067 def : SVE_2_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>;6068 def : SVE_2_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;6069 def : SVE_2_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>;6070 def : SVE_2_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;6071 def : SVE_2_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;6072}6073 6074//===----------------------------------------------------------------------===//6075// SVE Floating Point Accumulating Reduction Group6076//===----------------------------------------------------------------------===//6077 6078class sve_fp_2op_p_vd<bits<2> sz, bits<3> opc, string asm,6079 ZPRRegOp zprty, FPRasZPROperand dstOpType>6080: I<(outs dstOpType:$Vdn), (ins PPR3bAny:$Pg, dstOpType:$_Vdn, zprty:$Zm),6081 asm, "\t$Vdn, $Pg, $_Vdn, $Zm",6082 "",6083 []>,6084 Sched<[]> {6085 bits<3> Pg;6086 bits<5> Vdn;6087 bits<5> Zm;6088 let Inst{31-24} = 0b01100101;6089 let Inst{23-22} = sz;6090 let Inst{21-19} = 0b011;6091 let Inst{18-16} = opc;6092 let Inst{15-13} = 0b001;6093 let Inst{12-10} = Pg;6094 let Inst{9-5} = Zm;6095 let Inst{4-0} = Vdn;6096 6097 let Constraints = "$Vdn = $_Vdn";6098 let hasSideEffects = 0;6099 let mayRaiseFPException = 1;6100}6101 6102multiclass sve_fp_2op_p_vd<bits<3> opc, string asm, SDPatternOperator op> {6103 def _H : sve_fp_2op_p_vd<0b01, opc, asm, ZPR16, FPR16asZPR>;6104 def _S : sve_fp_2op_p_vd<0b10, opc, asm, ZPR32, FPR32asZPR>;6105 def _D : sve_fp_2op_p_vd<0b11, opc, asm, ZPR64, FPR64asZPR>;6106 6107 def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _H)>;6108 def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _H)>;6109 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;6110 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _S)>;6111 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;6112 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;6113}6114 6115//===----------------------------------------------------------------------===//6116// SVE Floating Point Compare - Vectors Group6117//===----------------------------------------------------------------------===//6118 6119class sve_fp_3op_p_pd<bits<2> sz, bits<3> opc, string asm, PPRRegOp pprty,6120 ZPRRegOp zprty>6121: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm),6122 asm, "\t$Pd, $Pg/z, $Zn, $Zm",6123 "",6124 []>, Sched<[]> {6125 bits<4> Pd;6126 bits<3> Pg;6127 bits<5> Zm;6128 bits<5> Zn;6129 let Inst{31-24} = 0b01100101;6130 let Inst{23-22} = sz;6131 let Inst{21} = 0b0;6132 let Inst{20-16} = Zm;6133 let Inst{15} = opc{2};6134 let Inst{14} = 0b1;6135 let Inst{13} = opc{1};6136 let Inst{12-10} = Pg;6137 let Inst{9-5} = Zn;6138 let Inst{4} = opc{0};6139 let Inst{3-0} = Pd;6140 6141 let hasSideEffects = 0;6142 let mayRaiseFPException = 1;6143}6144 6145multiclass sve_fp_3op_p_pd<bits<3> opc, string asm, SDPatternOperator op> {6146 def _H : sve_fp_3op_p_pd<0b01, opc, asm, PPR16, ZPR16>;6147 def _S : sve_fp_3op_p_pd<0b10, opc, asm, PPR32, ZPR32>;6148 def _D : sve_fp_3op_p_pd<0b11, opc, asm, PPR64, ZPR64>;6149 6150 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;6151 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;6152 def : SVE_3_Op_Pat<nxv2i1, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;6153}6154 6155multiclass sve_fp_3op_p_pd_cc<bits<3> opc, string asm,6156 CondCode cc1, CondCode cc2,6157 CondCode invcc1, CondCode invcc2> {6158 def _H : sve_fp_3op_p_pd<0b01, opc, asm, PPR16, ZPR16>;6159 def _S : sve_fp_3op_p_pd<0b10, opc, asm, PPR32, ZPR32>;6160 def _D : sve_fp_3op_p_pd<0b11, opc, asm, PPR64, ZPR64>;6161 6162 defm : SVE_SETCC_Pat<cc1, invcc1, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;6163 defm : SVE_SETCC_Pat<cc1, invcc1, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>;6164 defm : SVE_SETCC_Pat<cc1, invcc1, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>;6165 defm : SVE_SETCC_Pat<cc1, invcc1, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;6166 defm : SVE_SETCC_Pat<cc1, invcc1, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>;6167 defm : SVE_SETCC_Pat<cc1, invcc1, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;6168 6169 defm : SVE_SETCC_Pat<cc2, invcc2, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;6170 defm : SVE_SETCC_Pat<cc2, invcc2, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>;6171 defm : SVE_SETCC_Pat<cc2, invcc2, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>;6172 defm : SVE_SETCC_Pat<cc2, invcc2, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;6173 defm : SVE_SETCC_Pat<cc2, invcc2, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>;6174 defm : SVE_SETCC_Pat<cc2, invcc2, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;6175}6176 6177//===----------------------------------------------------------------------===//6178// SVE Floating Point Compare - with Zero Group6179//===----------------------------------------------------------------------===//6180 6181class sve_fp_2op_p_pd<bits<2> sz, bits<3> opc, string asm, PPRRegOp pprty,6182 ZPRRegOp zprty>6183: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn),6184 asm, "\t$Pd, $Pg/z, $Zn, #0.0",6185 "",6186 []>, Sched<[]> {6187 bits<4> Pd;6188 bits<3> Pg;6189 bits<5> Zn;6190 let Inst{31-24} = 0b01100101;6191 let Inst{23-22} = sz;6192 let Inst{21-18} = 0b0100;6193 let Inst{17-16} = opc{2-1};6194 let Inst{15-13} = 0b001;6195 let Inst{12-10} = Pg;6196 let Inst{9-5} = Zn;6197 let Inst{4} = opc{0};6198 let Inst{3-0} = Pd;6199 6200 let hasSideEffects = 0;6201 let mayRaiseFPException = 1;6202}6203 6204multiclass sve_fp_2op_p_pd<bits<3> opc, string asm,6205 CondCode cc1, CondCode cc2,6206 CondCode invcc1, CondCode invcc2> {6207 def _H : sve_fp_2op_p_pd<0b01, opc, asm, PPR16, ZPR16>;6208 def _S : sve_fp_2op_p_pd<0b10, opc, asm, PPR32, ZPR32>;6209 def _D : sve_fp_2op_p_pd<0b11, opc, asm, PPR64, ZPR64>;6210 6211 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;6212 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>;6213 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>;6214 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;6215 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>;6216 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;6217 6218 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;6219 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>;6220 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>;6221 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;6222 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>;6223 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;6224}6225 6226 6227//===----------------------------------------------------------------------===//6228//SVE Index Generation Group6229//===----------------------------------------------------------------------===//6230 6231def simm5_8b_tgt : TImmLeaf<i8, [{ return (int8_t)Imm >= -16 && (int8_t)Imm < 16; }]>;6232def simm5_16b_tgt : TImmLeaf<i16, [{ return (int16_t)Imm >= -16 && (int16_t)Imm < 16; }]>;6233def simm5_32b_tgt : TImmLeaf<i32, [{ return (int32_t)Imm >= -16 && (int32_t)Imm < 16; }]>;6234def simm5_64b_tgt : TImmLeaf<i64, [{ return (int64_t)Imm >= -16 && (int64_t)Imm < 16; }]>;6235def i64imm_32bit_tgt : TImmLeaf<i64, [{6236 return (Imm & 0xffffffffULL) == static_cast<uint64_t>(Imm);6237}]>;6238 6239class sve_int_index_ii<bits<2> sz8_64, string asm, ZPRRegOp zprty,6240 Operand imm_ty>6241: I<(outs zprty:$Zd), (ins imm_ty:$imm5, imm_ty:$imm5b),6242 asm, "\t$Zd, $imm5, $imm5b",6243 "", []>, Sched<[]> {6244 bits<5> Zd;6245 bits<5> imm5;6246 bits<5> imm5b;6247 let Inst{31-24} = 0b00000100;6248 let Inst{23-22} = sz8_64;6249 let Inst{21} = 0b1;6250 let Inst{20-16} = imm5b;6251 let Inst{15-10} = 0b010000;6252 let Inst{9-5} = imm5;6253 let Inst{4-0} = Zd;6254 6255 let hasSideEffects = 0;6256 let isReMaterializable = 1;6257 let Uses = [VG];6258}6259 6260multiclass sve_int_index_ii<string asm> {6261 def _B : sve_int_index_ii<0b00, asm, ZPR8, simm5_8b>;6262 def _H : sve_int_index_ii<0b01, asm, ZPR16, simm5_16b>;6263 def _S : sve_int_index_ii<0b10, asm, ZPR32, simm5_32b>;6264 def _D : sve_int_index_ii<0b11, asm, ZPR64, simm5_64b>;6265 6266 def : Pat<(nxv16i8 (step_vector simm5_8b_tgt:$imm5b)),6267 (!cast<Instruction>(NAME # "_B") (i32 0), (!cast<SDNodeXForm>("trunc_imm") $imm5b))>;6268 def : Pat<(nxv8i16 (step_vector simm5_16b_tgt:$imm5b)),6269 (!cast<Instruction>(NAME # "_H") (i32 0), (!cast<SDNodeXForm>("trunc_imm") $imm5b))>;6270 def : Pat<(nxv4i32 (step_vector simm5_32b_tgt:$imm5b)),6271 (!cast<Instruction>(NAME # "_S") (i32 0), simm5_32b:$imm5b)>;6272 def : Pat<(nxv2i64 (step_vector simm5_64b_tgt:$imm5b)),6273 (!cast<Instruction>(NAME # "_D") (i64 0), simm5_64b:$imm5b)>;6274 6275 // add(step_vector(step), dup(X)) -> index(X, step).6276 def : Pat<(add (nxv16i8 (step_vector_oneuse simm5_8b_tgt:$imm5b)), (nxv16i8 (splat_vector(simm5_8b:$imm5)))),6277 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, (!cast<SDNodeXForm>("trunc_imm") $imm5b))>;6278 def : Pat<(add (nxv8i16 (step_vector_oneuse simm5_16b_tgt:$imm5b)), (nxv8i16 (splat_vector(simm5_16b:$imm5)))),6279 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, (!cast<SDNodeXForm>("trunc_imm") $imm5b))>;6280 def : Pat<(add (nxv4i32 (step_vector_oneuse simm5_32b_tgt:$imm5b)), (nxv4i32 (splat_vector(simm5_32b:$imm5)))),6281 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, simm5_32b:$imm5b)>;6282 def : Pat<(add (nxv2i64 (step_vector_oneuse simm5_64b_tgt:$imm5b)), (nxv2i64 (splat_vector(simm5_64b:$imm5)))),6283 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, simm5_64b:$imm5b)>;6284}6285 6286class sve_int_index_ir<bits<2> sz8_64, string asm, ZPRRegOp zprty,6287 RegisterClass srcRegType, Operand imm_ty>6288: I<(outs zprty:$Zd), (ins imm_ty:$imm5, srcRegType:$Rm),6289 asm, "\t$Zd, $imm5, $Rm",6290 "", []>, Sched<[]> {6291 bits<5> Rm;6292 bits<5> Zd;6293 bits<5> imm5;6294 let Inst{31-24} = 0b00000100;6295 let Inst{23-22} = sz8_64;6296 let Inst{21} = 0b1;6297 let Inst{20-16} = Rm;6298 let Inst{15-10} = 0b010010;6299 let Inst{9-5} = imm5;6300 let Inst{4-0} = Zd;6301 6302 let hasSideEffects = 0;6303}6304 6305multiclass sve_int_index_ir<string asm, SDPatternOperator mulop, SDPatternOperator muloneuseop> {6306 def _B : sve_int_index_ir<0b00, asm, ZPR8, GPR32, simm5_8b>;6307 def _H : sve_int_index_ir<0b01, asm, ZPR16, GPR32, simm5_16b>;6308 def _S : sve_int_index_ir<0b10, asm, ZPR32, GPR32, simm5_32b>;6309 def _D : sve_int_index_ir<0b11, asm, ZPR64, GPR64, simm5_64b>;6310 6311 def : Pat<(nxv16i8 (step_vector i8:$imm)),6312 (!cast<Instruction>(NAME # "_B") (i32 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>;6313 def : Pat<(nxv8i16 (step_vector i16:$imm)),6314 (!cast<Instruction>(NAME # "_H") (i32 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>;6315 def : Pat<(nxv4i32 (step_vector i32:$imm)),6316 (!cast<Instruction>(NAME # "_S") (i32 0), (!cast<Instruction>("MOVi32imm") $imm))>;6317 def : Pat<(nxv2i64 (step_vector i64:$imm)),6318 (!cast<Instruction>(NAME # "_D") (i64 0), (!cast<Instruction>("MOVi64imm") $imm))>;6319 def : Pat<(nxv2i64 (step_vector i64imm_32bit_tgt:$imm)),6320 (!cast<Instruction>(NAME # "_D") (i64 0), (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>;6321 6322 // add(step_vector(step), dup(X)) -> index(X, step).6323 def : Pat<(add (nxv16i8 (step_vector_oneuse i8:$imm)), (nxv16i8 (splat_vector(simm5_8b:$imm5)))),6324 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>;6325 def : Pat<(add (nxv8i16 (step_vector_oneuse i16:$imm)), (nxv8i16 (splat_vector(simm5_16b:$imm5)))),6326 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>;6327 def : Pat<(add (nxv4i32 (step_vector_oneuse i32:$imm)), (nxv4i32 (splat_vector(simm5_32b:$imm5)))),6328 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, (!cast<Instruction>("MOVi32imm") $imm))>;6329 def : Pat<(add (nxv2i64 (step_vector_oneuse i64:$imm)), (nxv2i64 (splat_vector(simm5_64b:$imm5)))),6330 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, (!cast<Instruction>("MOVi64imm") $imm))>;6331 def : Pat<(add (nxv2i64 (step_vector_oneuse i64imm_32bit_tgt:$imm)), (nxv2i64 (splat_vector(simm5_64b:$imm5)))),6332 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>;6333 6334 // mul(step_vector(1), dup(Y)) -> index(0, Y).6335 def : Pat<(mulop (nxv16i1 (SVEAllActive)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (splat_vector(i32 GPR32:$Rm)))),6336 (!cast<Instruction>(NAME # "_B") (i32 0), GPR32:$Rm)>;6337 def : Pat<(mulop (nxv8i1 (SVEAllActive)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (splat_vector(i32 GPR32:$Rm)))),6338 (!cast<Instruction>(NAME # "_H") (i32 0), GPR32:$Rm)>;6339 def : Pat<(mulop (nxv4i1 (SVEAllActive)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (splat_vector(i32 GPR32:$Rm)))),6340 (!cast<Instruction>(NAME # "_S") (i32 0), GPR32:$Rm)>;6341 def : Pat<(mulop (nxv2i1 (SVEAllActive)), (nxv2i64 (step_vector_oneuse (i64 1))), (nxv2i64 (splat_vector(i64 GPR64:$Rm)))),6342 (!cast<Instruction>(NAME # "_D") (i64 0), GPR64:$Rm)>;6343 6344 // add(mul(step_vector(1), dup(Y)), dup(X)) -> index(X, Y).6345 def : Pat<(add (muloneuseop (nxv16i1 (SVEAllActive)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (splat_vector(i32 GPR32:$Rm)))), (nxv16i8 (splat_vector(simm5_8b:$imm5)))),6346 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, GPR32:$Rm)>;6347 def : Pat<(add (muloneuseop (nxv8i1 (SVEAllActive)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (splat_vector(i32 GPR32:$Rm)))), (nxv8i16 (splat_vector(simm5_16b:$imm5)))),6348 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, GPR32:$Rm)>;6349 def : Pat<(add (muloneuseop (nxv4i1 (SVEAllActive)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (splat_vector(i32 GPR32:$Rm)))), (nxv4i32 (splat_vector(simm5_32b:$imm5)))),6350 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, GPR32:$Rm)>;6351 def : Pat<(add (muloneuseop (nxv2i1 (SVEAllActive)), (nxv2i64 (step_vector_oneuse (i64 1))), (nxv2i64 (splat_vector(i64 GPR64:$Rm)))), (nxv2i64 (splat_vector(simm5_64b:$imm5)))),6352 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, GPR64:$Rm)>;6353}6354 6355class sve_int_index_ri<bits<2> sz8_64, string asm, ZPRRegOp zprty,6356 RegisterClass srcRegType, Operand imm_ty>6357: I<(outs zprty:$Zd), (ins srcRegType:$Rn, imm_ty:$imm5),6358 asm, "\t$Zd, $Rn, $imm5",6359 "", []>, Sched<[]> {6360 bits<5> Rn;6361 bits<5> Zd;6362 bits<5> imm5;6363 let Inst{31-24} = 0b00000100;6364 let Inst{23-22} = sz8_64;6365 let Inst{21} = 0b1;6366 let Inst{20-16} = imm5;6367 let Inst{15-10} = 0b010001;6368 let Inst{9-5} = Rn;6369 let Inst{4-0} = Zd;6370 6371 let hasSideEffects = 0;6372}6373 6374multiclass sve_int_index_ri<string asm> {6375 def _B : sve_int_index_ri<0b00, asm, ZPR8, GPR32, simm5_8b>;6376 def _H : sve_int_index_ri<0b01, asm, ZPR16, GPR32, simm5_16b>;6377 def _S : sve_int_index_ri<0b10, asm, ZPR32, GPR32, simm5_32b>;6378 def _D : sve_int_index_ri<0b11, asm, ZPR64, GPR64, simm5_64b>;6379 6380 // add(step_vector(step), dup(X)) -> index(X, step).6381 def : Pat<(add (nxv16i8 (step_vector_oneuse simm5_8b_tgt:$imm5)), (nxv16i8 (splat_vector(i32 GPR32:$Rm)))),6382 (!cast<Instruction>(NAME # "_B") GPR32:$Rm, (!cast<SDNodeXForm>("trunc_imm") $imm5))>;6383 def : Pat<(add (nxv8i16 (step_vector_oneuse simm5_16b_tgt:$imm5)), (nxv8i16 (splat_vector(i32 GPR32:$Rm)))),6384 (!cast<Instruction>(NAME # "_H") GPR32:$Rm, (!cast<SDNodeXForm>("trunc_imm") $imm5))>;6385 def : Pat<(add (nxv4i32 (step_vector_oneuse simm5_32b_tgt:$imm5)), (nxv4i32 (splat_vector(i32 GPR32:$Rm)))),6386 (!cast<Instruction>(NAME # "_S") GPR32:$Rm, simm5_32b:$imm5)>;6387 def : Pat<(add (nxv2i64 (step_vector_oneuse simm5_64b_tgt:$imm5)), (nxv2i64 (splat_vector(i64 GPR64:$Rm)))),6388 (!cast<Instruction>(NAME # "_D") GPR64:$Rm, simm5_64b:$imm5)>;6389}6390 6391class sve_int_index_rr<bits<2> sz8_64, string asm, ZPRRegOp zprty,6392 RegisterClass srcRegType>6393: I<(outs zprty:$Zd), (ins srcRegType:$Rn, srcRegType:$Rm),6394 asm, "\t$Zd, $Rn, $Rm",6395 "", []>, Sched<[]> {6396 bits<5> Zd;6397 bits<5> Rm;6398 bits<5> Rn;6399 let Inst{31-24} = 0b00000100;6400 let Inst{23-22} = sz8_64;6401 let Inst{21} = 0b1;6402 let Inst{20-16} = Rm;6403 let Inst{15-10} = 0b010011;6404 let Inst{9-5} = Rn;6405 let Inst{4-0} = Zd;6406 6407 let hasSideEffects = 0;6408}6409 6410multiclass sve_int_index_rr<string asm, SDPatternOperator mulop> {6411 def _B : sve_int_index_rr<0b00, asm, ZPR8, GPR32>;6412 def _H : sve_int_index_rr<0b01, asm, ZPR16, GPR32>;6413 def _S : sve_int_index_rr<0b10, asm, ZPR32, GPR32>;6414 def _D : sve_int_index_rr<0b11, asm, ZPR64, GPR64>;6415 6416 // add(step_vector(step), dup(X)) -> index(X, step).6417 def : Pat<(add (nxv16i8 (step_vector_oneuse i8:$imm)), (nxv16i8 (splat_vector(i32 GPR32:$Rn)))),6418 (!cast<Instruction>(NAME # "_B") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>;6419 def : Pat<(add (nxv8i16 (step_vector_oneuse i16:$imm)), (nxv8i16 (splat_vector(i32 GPR32:$Rn)))),6420 (!cast<Instruction>(NAME # "_H") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>;6421 def : Pat<(add (nxv4i32 (step_vector_oneuse i32:$imm)), (nxv4i32 (splat_vector(i32 GPR32:$Rn)))),6422 (!cast<Instruction>(NAME # "_S") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") $imm))>;6423 def : Pat<(add (nxv2i64 (step_vector_oneuse i64:$imm)), (nxv2i64 (splat_vector(i64 GPR64:$Rn)))),6424 (!cast<Instruction>(NAME # "_D") GPR64:$Rn, (!cast<Instruction>("MOVi64imm") $imm))>;6425 def : Pat<(add (nxv2i64 (step_vector_oneuse i64imm_32bit_tgt:$imm)), (nxv2i64 (splat_vector(i64 GPR64:$Rn)))),6426 (!cast<Instruction>(NAME # "_D") GPR64:$Rn, (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>;6427 6428 // add(mul(step_vector(1), dup(Y)), dup(X)) -> index(X, Y).6429 def : Pat<(add (mulop (nxv16i1 (SVEAllActive)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (splat_vector(i32 GPR32:$Rm)))), (nxv16i8 (splat_vector(i32 GPR32:$Rn)))),6430 (!cast<Instruction>(NAME # "_B") GPR32:$Rn, GPR32:$Rm)>;6431 def : Pat<(add (mulop (nxv8i1 (SVEAllActive)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (splat_vector(i32 GPR32:$Rm)))),(nxv8i16 (splat_vector(i32 GPR32:$Rn)))),6432 (!cast<Instruction>(NAME # "_H") GPR32:$Rn, GPR32:$Rm)>;6433 def : Pat<(add (mulop (nxv4i1 (SVEAllActive)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (splat_vector(i32 GPR32:$Rm)))),(nxv4i32 (splat_vector(i32 GPR32:$Rn)))),6434 (!cast<Instruction>(NAME # "_S") GPR32:$Rn, GPR32:$Rm)>;6435 def : Pat<(add (mulop (nxv2i1 (SVEAllActive)), (nxv2i64 (step_vector_oneuse (i64 1))), (nxv2i64 (splat_vector(i64 GPR64:$Rm)))),(nxv2i64 (splat_vector(i64 GPR64:$Rn)))),6436 (!cast<Instruction>(NAME # "_D") GPR64:$Rn, GPR64:$Rm)>;6437}6438 6439//===----------------------------------------------------------------------===//6440// SVE Bitwise Shift - Predicated Group6441//===----------------------------------------------------------------------===//6442 6443class sve_int_bin_pred_shift_imm<bits<4> tsz8_64, bits<4> opc, string asm,6444 ZPRRegOp zprty, Operand immtype>6445: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, immtype:$imm),6446 asm, "\t$Zdn, $Pg/m, $_Zdn, $imm",6447 "",6448 []>, Sched<[]> {6449 bits<3> Pg;6450 bits<5> Zdn;6451 bits<6> imm;6452 let Inst{31-24} = 0b00000100;6453 let Inst{23-22} = tsz8_64{3-2};6454 let Inst{21-20} = 0b00;6455 let Inst{19-16} = opc;6456 let Inst{15-13} = 0b100;6457 let Inst{12-10} = Pg;6458 let Inst{9-8} = tsz8_64{1-0};6459 let Inst{7-5} = imm{2-0}; // imm36460 let Inst{4-0} = Zdn;6461 6462 let Constraints = "$Zdn = $_Zdn";6463 let DestructiveInstType = DestructiveBinaryImm;6464 let ElementSize = zprty.ElementSize;6465 let hasSideEffects = 0;6466}6467 6468multiclass sve_int_bin_pred_shift_imm_left<bits<4> opc, string asm, string Ps,6469 SDPatternOperator op = null_frag> {6470 def _B : SVEPseudo2Instr<Ps # _B, 1>,6471 sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>;6472 def _H : SVEPseudo2Instr<Ps # _H, 1>,6473 sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> {6474 let Inst{8} = imm{3};6475 }6476 def _S : SVEPseudo2Instr<Ps # _S, 1>,6477 sve_int_bin_pred_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> {6478 let Inst{9-8} = imm{4-3};6479 }6480 def _D : SVEPseudo2Instr<Ps # _D, 1>,6481 sve_int_bin_pred_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> {6482 let Inst{22} = imm{5};6483 let Inst{9-8} = imm{4-3};6484 }6485 6486 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i1, nxv16i8, i32, vecshiftL8, !cast<Instruction>(NAME # _B)>;6487 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i1, nxv8i16, i32, vecshiftL16, !cast<Instruction>(NAME # _H)>;6488 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i1, nxv4i32, i32, vecshiftL32, !cast<Instruction>(NAME # _S)>;6489 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i1, nxv2i64, i32, vecshiftL64, !cast<Instruction>(NAME # _D)>;6490}6491 6492// As above but shift amount takes the form of a "vector immediate".6493multiclass sve_int_bin_pred_shift_imm_left_dup<bits<4> opc, string asm,6494 string Ps, SDPatternOperator op>6495: sve_int_bin_pred_shift_imm_left<opc, asm, Ps, null_frag> {6496 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmL8, !cast<Instruction>(NAME # _B)>;6497 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmL16, !cast<Instruction>(NAME # _H)>;6498 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmL32, !cast<Instruction>(NAME # _S)>;6499 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmL64, !cast<Instruction>(NAME # _D)>;6500}6501 6502multiclass sve_int_bin_pred_shift_imm_left_zeroing_bhsd<SDPatternOperator op> {6503 def _B_ZERO : PredTwoOpImmPseudo<NAME # _B, ZPR8, vecshiftL8, FalseLanesZero>;6504 def _H_ZERO : PredTwoOpImmPseudo<NAME # _H, ZPR16, vecshiftL16, FalseLanesZero>;6505 def _S_ZERO : PredTwoOpImmPseudo<NAME # _S, ZPR32, vecshiftL32, FalseLanesZero>;6506 def _D_ZERO : PredTwoOpImmPseudo<NAME # _D, ZPR64, vecshiftL64, FalseLanesZero>;6507 6508 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, vecshiftL8, !cast<Pseudo>(NAME # _B_ZERO)>;6509 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1, nxv8i16, vecshiftL16, !cast<Pseudo>(NAME # _H_ZERO)>;6510 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1, nxv4i32, vecshiftL32, !cast<Pseudo>(NAME # _S_ZERO)>;6511 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1, nxv2i64, vecshiftL64, !cast<Pseudo>(NAME # _D_ZERO)>;6512}6513 6514multiclass sve_int_bin_pred_shift_imm_right<bits<4> opc, string asm, string Ps,6515 SDPatternOperator op = null_frag> {6516 def _B : SVEPseudo2Instr<Ps # _B, 1>,6517 sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>;6518 def _H : SVEPseudo2Instr<Ps # _H, 1>,6519 sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> {6520 let Inst{8} = imm{3};6521 }6522 def _S : SVEPseudo2Instr<Ps # _S, 1>,6523 sve_int_bin_pred_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> {6524 let Inst{9-8} = imm{4-3};6525 }6526 def _D : SVEPseudo2Instr<Ps # _D, 1>,6527 sve_int_bin_pred_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> {6528 let Inst{22} = imm{5};6529 let Inst{9-8} = imm{4-3};6530 }6531 6532 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i1, nxv16i8, i32, vecshiftR8, !cast<Instruction>(NAME # _B)>;6533 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i1, nxv8i16, i32, vecshiftR16, !cast<Instruction>(NAME # _H)>;6534 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i1, nxv4i32, i32, vecshiftR32, !cast<Instruction>(NAME # _S)>;6535 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i1, nxv2i64, i32, vecshiftR64, !cast<Instruction>(NAME # _D)>;6536}6537 6538// As above but shift amount takes the form of a "vector immediate".6539multiclass sve_int_bin_pred_shift_imm_right_dup<bits<4> opc, string asm,6540 string Ps, SDPatternOperator op>6541: sve_int_bin_pred_shift_imm_right<opc, asm, Ps, null_frag> {6542 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmR8, !cast<Instruction>(NAME # _B)>;6543 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmR16, !cast<Instruction>(NAME # _H)>;6544 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmR32, !cast<Instruction>(NAME # _S)>;6545 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmR64, !cast<Instruction>(NAME # _D)>;6546}6547 6548multiclass sve_int_bin_pred_shift_imm_right_zeroing_bhsd<SDPatternOperator op = null_frag> {6549 def _B_ZERO : PredTwoOpImmPseudo<NAME # _B, ZPR8, vecshiftR8, FalseLanesZero>;6550 def _H_ZERO : PredTwoOpImmPseudo<NAME # _H, ZPR16, vecshiftR16, FalseLanesZero>;6551 def _S_ZERO : PredTwoOpImmPseudo<NAME # _S, ZPR32, vecshiftR32, FalseLanesZero>;6552 def _D_ZERO : PredTwoOpImmPseudo<NAME # _D, ZPR64, vecshiftR64, FalseLanesZero>;6553 6554 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, vecshiftR8, !cast<Pseudo>(NAME # _B_ZERO)>;6555 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1, nxv8i16, vecshiftR16, !cast<Pseudo>(NAME # _H_ZERO)>;6556 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1, nxv4i32, vecshiftR32, !cast<Pseudo>(NAME # _S_ZERO)>;6557 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1, nxv2i64, vecshiftR64, !cast<Pseudo>(NAME # _D_ZERO)>;6558}6559 6560class sve_int_bin_pred_shift<bits<2> sz8_64, bit wide, bits<3> opc,6561 string asm, ZPRRegOp zprty, ZPRRegOp zprty2>6562: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty2:$Zm),6563 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm",6564 "",6565 []>, Sched<[]> {6566 bits<3> Pg;6567 bits<5> Zdn;6568 bits<5> Zm;6569 let Inst{31-24} = 0b00000100;6570 let Inst{23-22} = sz8_64;6571 let Inst{21-20} = 0b01;6572 let Inst{19} = wide;6573 let Inst{18-16} = opc;6574 let Inst{15-13} = 0b100;6575 let Inst{12-10} = Pg;6576 let Inst{9-5} = Zm;6577 let Inst{4-0} = Zdn;6578 6579 let Constraints = "$Zdn = $_Zdn";6580 let DestructiveInstType = DestructiveOther;6581 let ElementSize = zprty.ElementSize;6582 let hasSideEffects = 0;6583}6584 6585multiclass sve_int_bin_pred_shift<bits<3> opc, string asm, string Ps,6586 SDPatternOperator op, string revname, bit isReverseInstr = 0> {6587 let DestructiveInstType = DestructiveBinaryCommWithRev in {6588 def _B : sve_int_bin_pred_shift<0b00, 0b0, opc, asm, ZPR8, ZPR8>,6589 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>;6590 def _H : sve_int_bin_pred_shift<0b01, 0b0, opc, asm, ZPR16, ZPR16>,6591 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>;6592 def _S : sve_int_bin_pred_shift<0b10, 0b0, opc, asm, ZPR32, ZPR32>,6593 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>;6594 def _D : sve_int_bin_pred_shift<0b11, 0b0, opc, asm, ZPR64, ZPR64>,6595 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>;6596 }6597 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;6598 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;6599 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;6600 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;6601}6602 6603multiclass sve_int_bin_pred_zeroing_bhsd<SDPatternOperator op> {6604 def _B_ZERO : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesZero>;6605 def _H_ZERO : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesZero>;6606 def _S_ZERO : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>;6607 def _D_ZERO : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>;6608 6609 def : SVE_3_Op_Pat_SelZero<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _B_ZERO)>;6610 def : SVE_3_Op_Pat_SelZero<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _H_ZERO)>;6611 def : SVE_3_Op_Pat_SelZero<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _S_ZERO)>;6612 def : SVE_3_Op_Pat_SelZero<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _D_ZERO)>;6613}6614 6615multiclass sve_int_bin_pred_imm_zeroing_bhsd<SDPatternOperator op,6616 ComplexPattern imm_b, ComplexPattern imm_h,6617 ComplexPattern imm_s, ComplexPattern imm_d> {6618 def _B_ZERO : PredTwoOpImmPseudo<NAME # _B, ZPR8, Operand<i32>, FalseLanesZero>;6619 def _H_ZERO : PredTwoOpImmPseudo<NAME # _H, ZPR16, Operand<i32>, FalseLanesZero>;6620 def _S_ZERO : PredTwoOpImmPseudo<NAME # _S, ZPR32, Operand<i32>, FalseLanesZero>;6621 def _D_ZERO : PredTwoOpImmPseudo<NAME # _D, ZPR64, Operand<i32>, FalseLanesZero>;6622 6623 def : SVE_2_Op_Imm_Pat_Zero<nxv16i8, op, nxv16i1, i32, imm_b, !cast<Pseudo>(NAME # _B_ZERO)>;6624 def : SVE_2_Op_Imm_Pat_Zero<nxv8i16, op, nxv8i1, i32, imm_h, !cast<Pseudo>(NAME # _H_ZERO)>;6625 def : SVE_2_Op_Imm_Pat_Zero<nxv4i32, op, nxv4i1, i32, imm_s, !cast<Pseudo>(NAME # _S_ZERO)>;6626 def : SVE_2_Op_Imm_Pat_Zero<nxv2i64, op, nxv2i1, i64, imm_d, !cast<Pseudo>(NAME # _D_ZERO)>;6627}6628 6629multiclass sve_int_bin_pred_shift_wide<bits<3> opc, string asm,6630 SDPatternOperator op> {6631 def _B : sve_int_bin_pred_shift<0b00, 0b1, opc, asm, ZPR8, ZPR64>;6632 def _H : sve_int_bin_pred_shift<0b01, 0b1, opc, asm, ZPR16, ZPR64>;6633 def _S : sve_int_bin_pred_shift<0b10, 0b1, opc, asm, ZPR32, ZPR64>;6634 6635 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>;6636 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>;6637 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>;6638}6639 6640//===----------------------------------------------------------------------===//6641// SVE Shift - Unpredicated Group6642//===----------------------------------------------------------------------===//6643 6644class sve_int_bin_cons_shift_wide<bits<2> sz8_64, bits<2> opc, string asm,6645 ZPRRegOp zprty>6646: I<(outs zprty:$Zd), (ins zprty:$Zn, ZPR64:$Zm),6647 asm, "\t$Zd, $Zn, $Zm",6648 "",6649 []>, Sched<[]> {6650 bits<5> Zd;6651 bits<5> Zm;6652 bits<5> Zn;6653 let Inst{31-24} = 0b00000100;6654 let Inst{23-22} = sz8_64;6655 let Inst{21} = 0b1;6656 let Inst{20-16} = Zm;6657 let Inst{15-12} = 0b1000;6658 let Inst{11-10} = opc;6659 let Inst{9-5} = Zn;6660 let Inst{4-0} = Zd;6661 6662 let hasSideEffects = 0;6663}6664 6665multiclass sve_int_bin_cons_shift_wide<bits<2> opc, string asm, SDPatternOperator op> {6666 def _B : sve_int_bin_cons_shift_wide<0b00, opc, asm, ZPR8>;6667 def _H : sve_int_bin_cons_shift_wide<0b01, opc, asm, ZPR16>;6668 def _S : sve_int_bin_cons_shift_wide<0b10, opc, asm, ZPR32>;6669 6670 def : SVE_2_Op_Pred_All_Active<nxv16i8, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>;6671 def : SVE_2_Op_Pred_All_Active<nxv8i16, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>;6672 def : SVE_2_Op_Pred_All_Active<nxv4i32, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>;6673}6674 6675class sve_int_bin_cons_shift_imm<bits<4> tsz8_64, bits<2> opc, string asm,6676 ZPRRegOp zprty, Operand immtype>6677: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$imm),6678 asm, "\t$Zd, $Zn, $imm",6679 "",6680 []>, Sched<[]> {6681 bits<5> Zd;6682 bits<5> Zn;6683 bits<6> imm;6684 let Inst{31-24} = 0b00000100;6685 let Inst{23-22} = tsz8_64{3-2};6686 let Inst{21} = 0b1;6687 let Inst{20-19} = tsz8_64{1-0};6688 let Inst{18-16} = imm{2-0}; // imm36689 let Inst{15-12} = 0b1001;6690 let Inst{11-10} = opc;6691 let Inst{9-5} = Zn;6692 let Inst{4-0} = Zd;6693 6694 let hasSideEffects = 0;6695}6696 6697multiclass sve_int_bin_cons_shift_imm_left<bits<2> opc, string asm,6698 SDPatternOperator op> {6699 def _B : sve_int_bin_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>;6700 def _H : sve_int_bin_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> {6701 let Inst{19} = imm{3};6702 }6703 def _S : sve_int_bin_cons_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> {6704 let Inst{20-19} = imm{4-3};6705 }6706 def _D : sve_int_bin_cons_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> {6707 let Inst{22} = imm{5};6708 let Inst{20-19} = imm{4-3};6709 }6710 6711 def : SVE_Shift_DupImm_Any_Predicate_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmL8, !cast<Instruction>(NAME # _B)>;6712 def : SVE_Shift_DupImm_Any_Predicate_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmL16, !cast<Instruction>(NAME # _H)>;6713 def : SVE_Shift_DupImm_Any_Predicate_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmL32, !cast<Instruction>(NAME # _S)>;6714 def : SVE_Shift_DupImm_Any_Predicate_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmL64, !cast<Instruction>(NAME # _D)>;6715}6716 6717multiclass sve_int_bin_cons_shift_imm_right<bits<2> opc, string asm,6718 SDPatternOperator op> {6719 def _B : sve_int_bin_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>;6720 def _H : sve_int_bin_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> {6721 let Inst{19} = imm{3};6722 }6723 def _S : sve_int_bin_cons_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> {6724 let Inst{20-19} = imm{4-3};6725 }6726 def _D : sve_int_bin_cons_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> {6727 let Inst{22} = imm{5};6728 let Inst{20-19} = imm{4-3};6729 }6730 6731 def : SVE_Shift_DupImm_Any_Predicate_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmR8, !cast<Instruction>(NAME # _B)>;6732 def : SVE_Shift_DupImm_Any_Predicate_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmR16, !cast<Instruction>(NAME # _H)>;6733 def : SVE_Shift_DupImm_Any_Predicate_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmR32, !cast<Instruction>(NAME # _S)>;6734 def : SVE_Shift_DupImm_Any_Predicate_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmR64, !cast<Instruction>(NAME # _D)>;6735}6736 6737//===----------------------------------------------------------------------===//6738// SVE Memory - Store Group6739//===----------------------------------------------------------------------===//6740 6741class sve_mem_cst_si<bits<2> msz, bits<2> esz, string asm,6742 RegisterOperand VecList>6743: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4),6744 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]",6745 "",6746 []>, Sched<[]> {6747 bits<3> Pg;6748 bits<5> Rn;6749 bits<5> Zt;6750 bits<4> imm4;6751 let Inst{31-25} = 0b1110010;6752 let Inst{24-23} = msz;6753 let Inst{22-21} = esz;6754 let Inst{20} = 0;6755 let Inst{19-16} = imm4;6756 let Inst{15-13} = 0b111;6757 let Inst{12-10} = Pg;6758 let Inst{9-5} = Rn;6759 let Inst{4-0} = Zt;6760 6761 let hasSideEffects = 0;6762 let mayStore = 1;6763}6764 6765multiclass sve_mem_cst_si<bits<2> msz, bits<2> esz, string asm,6766 RegisterOperand listty, ZPRRegOp zprty>6767{6768 def NAME : sve_mem_cst_si<msz, esz, asm, listty>;6769 6770 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $imm4, mul vl]",6771 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>;6772 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]",6773 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>;6774 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]",6775 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;6776}6777 6778class sve_mem_est_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList,6779 string asm, Operand immtype>6780: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4),6781 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]",6782 "",6783 []>, Sched<[]> {6784 bits<3> Pg;6785 bits<5> Rn;6786 bits<5> Zt;6787 bits<4> imm4;6788 let Inst{31-25} = 0b1110010;6789 let Inst{24-23} = sz;6790 let Inst{22-21} = nregs;6791 let Inst{20} = 1;6792 let Inst{19-16} = imm4;6793 let Inst{15-13} = 0b111;6794 let Inst{12-10} = Pg;6795 let Inst{9-5} = Rn;6796 let Inst{4-0} = Zt;6797 6798 let hasSideEffects = 0;6799 let mayStore = 1;6800}6801 6802multiclass sve_mem_est_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList,6803 string asm, Operand immtype> {6804 def NAME : sve_mem_est_si<sz, nregs, VecList, asm, immtype>;6805 6806 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]",6807 (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;6808}6809 6810 6811// SVE store multiple structures (quadwords, scalar plus immediate)6812class sve_mem_128b_est_si<bits<2> nregs, RegisterOperand VecList,6813 string asm, Operand immtype>6814 : I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4),6815 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]",6816 "", []>, Sched<[]> {6817 bits<5> Zt;6818 bits<5> Rn;6819 bits<3> Pg;6820 bits<4> imm4;6821 let Inst{31-24} = 0b11100100;6822 let Inst{23-22} = nregs;6823 let Inst{21-20} = 0b00;6824 let Inst{19-16} = imm4;6825 let Inst{15-13} = 0b000;6826 let Inst{12-10} = Pg;6827 let Inst{9-5} = Rn;6828 let Inst{4-0} = Zt;6829 6830 let hasSideEffects = 0;6831 let mayStore = 1;6832}6833 6834multiclass sve_mem_128b_est_si<bits<2> nregs, RegisterOperand VecList,6835 string asm, Operand immtype> {6836 def NAME : sve_mem_128b_est_si<nregs, VecList, asm, immtype>;6837 6838 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]",6839 (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;6840}6841 6842 6843class sve_mem_est_ss<bits<2> sz, bits<2> nregs, RegisterOperand VecList,6844 string asm, RegisterOperand gprty>6845: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),6846 asm, "\t$Zt, $Pg, [$Rn, $Rm]",6847 "",6848 []>, Sched<[]> {6849 bits<3> Pg;6850 bits<5> Rm;6851 bits<5> Rn;6852 bits<5> Zt;6853 let Inst{31-25} = 0b1110010;6854 let Inst{24-23} = sz;6855 let Inst{22-21} = nregs;6856 let Inst{20-16} = Rm;6857 let Inst{15-13} = 0b011;6858 let Inst{12-10} = Pg;6859 let Inst{9-5} = Rn;6860 let Inst{4-0} = Zt;6861 6862 let hasSideEffects = 0;6863 let mayStore = 1;6864}6865 6866 6867// SVE store multiple structures (quadwords, scalar plus scalar)6868class sve_mem_128b_est_ss<bits<2> nregs, RegisterOperand VecList,6869 string asm, RegisterOperand gprty>6870 : I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),6871 asm, "\t$Zt, $Pg, [$Rn, $Rm]",6872 "", []>, Sched<[]> {6873 bits<5> Zt;6874 bits<5> Rn;6875 bits<3> Pg;6876 bits<5> Rm;6877 let Inst{31-24} = 0b11100100;6878 let Inst{23-22} = nregs;6879 let Inst{21} = 0b1;6880 let Inst{20-16} = Rm;6881 let Inst{15-13} = 0b000;6882 let Inst{12-10} = Pg;6883 let Inst{9-5} = Rn;6884 let Inst{4-0} = Zt;6885 6886 let hasSideEffects = 0;6887 let mayStore = 1;6888}6889 6890 6891class sve_mem_cst_ss_base<bits<4> dtype, string asm,6892 RegisterOperand listty, RegisterOperand gprty>6893: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),6894 asm, "\t$Zt, $Pg, [$Rn, $Rm]",6895 "",6896 []>, Sched<[]> {6897 bits<3> Pg;6898 bits<5> Rm;6899 bits<5> Rn;6900 bits<5> Zt;6901 let Inst{31-25} = 0b1110010;6902 let Inst{24-21} = dtype;6903 let Inst{20-16} = Rm;6904 let Inst{15-13} = 0b010;6905 let Inst{12-10} = Pg;6906 let Inst{9-5} = Rn;6907 let Inst{4-0} = Zt;6908 6909 let hasSideEffects = 0;6910 let mayStore = 1;6911}6912 6913multiclass sve_mem_cst_ss<bits<4> dtype, string asm,6914 RegisterOperand listty, ZPRRegOp zprty,6915 RegisterOperand gprty> {6916 def NAME : sve_mem_cst_ss_base<dtype, asm, listty, gprty>;6917 6918 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Rm]",6919 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>;6920}6921 6922class sve_mem_cstnt_si<bits<2> msz, string asm, RegisterOperand VecList>6923: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4),6924 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]",6925 "",6926 []>, Sched<[]> {6927 bits<3> Pg;6928 bits<5> Rn;6929 bits<5> Zt;6930 bits<4> imm4;6931 let Inst{31-25} = 0b1110010;6932 let Inst{24-23} = msz;6933 let Inst{22-20} = 0b001;6934 let Inst{19-16} = imm4;6935 let Inst{15-13} = 0b111;6936 let Inst{12-10} = Pg;6937 let Inst{9-5} = Rn;6938 let Inst{4-0} = Zt;6939 6940 let hasSideEffects = 0;6941 let mayStore = 1;6942}6943 6944multiclass sve_mem_cstnt_si<bits<2> msz, string asm, RegisterOperand listty,6945 ZPRRegOp zprty> {6946 def NAME : sve_mem_cstnt_si<msz, asm, listty>;6947 6948 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]",6949 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>;6950 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $imm4, mul vl]",6951 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>;6952 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]",6953 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;6954}6955 6956class sve_mem_cstnt_ss_base<bits<2> msz, string asm, RegisterOperand listty,6957 RegisterOperand gprty>6958: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),6959 asm, "\t$Zt, $Pg, [$Rn, $Rm]",6960 "",6961 []>, Sched<[]> {6962 bits<3> Pg;6963 bits<5> Rm;6964 bits<5> Rn;6965 bits<5> Zt;6966 let Inst{31-25} = 0b1110010;6967 let Inst{24-23} = msz;6968 let Inst{22-21} = 0b00;6969 let Inst{20-16} = Rm;6970 let Inst{15-13} = 0b011;6971 let Inst{12-10} = Pg;6972 let Inst{9-5} = Rn;6973 let Inst{4-0} = Zt;6974 6975 let hasSideEffects = 0;6976 let mayStore = 1;6977}6978 6979multiclass sve_mem_cstnt_ss<bits<2> msz, string asm, RegisterOperand listty,6980 ZPRRegOp zprty, RegisterOperand gprty> {6981 def NAME : sve_mem_cstnt_ss_base<msz, asm, listty, gprty>;6982 6983 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Rm]",6984 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>;6985}6986 6987class sve2_mem_sstnt_vs_base<bits<3> opc, string asm,6988 RegisterOperand listty, ZPRRegOp zprty>6989: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, zprty:$Zn, GPR64:$Rm),6990 asm, "\t$Zt, $Pg, [$Zn, $Rm]",6991 "",6992 []>, Sched<[]> {6993 bits<3> Pg;6994 bits<5> Rm;6995 bits<5> Zn;6996 bits<5> Zt;6997 let Inst{31-25} = 0b1110010;6998 let Inst{24-22} = opc;6999 let Inst{21} = 0b0;7000 let Inst{20-16} = Rm;7001 let Inst{15-13} = 0b001;7002 let Inst{12-10} = Pg;7003 let Inst{9-5} = Zn;7004 let Inst{4-0} = Zt;7005 7006 let hasSideEffects = 0;7007 let mayStore = 1;7008}7009 7010multiclass sve2_mem_sstnt_vs_32_ptrs<bits<3> opc, string asm,7011 SDPatternOperator op,7012 ValueType vt> {7013 def NAME : sve2_mem_sstnt_vs_base<opc, asm, Z_s, ZPR32>;7014 7015 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $Rm]",7016 (!cast<Instruction>(NAME) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>;7017 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]",7018 (!cast<Instruction>(NAME) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>;7019 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]",7020 (!cast<Instruction>(NAME) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>;7021 7022 def : Pat <(op (nxv4i32 ZPR32:$Zt), (nxv4i1 PPR3bAny:$Pg), (nxv4i32 ZPR32:$Zn), (i64 GPR64:$Rm), vt),7023 (!cast<Instruction>(NAME) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm)>;7024}7025 7026multiclass sve2_mem_sstnt_vs_64_ptrs<bits<3> opc, string asm,7027 SDPatternOperator op,7028 ValueType vt> {7029 def NAME : sve2_mem_sstnt_vs_base<opc, asm, Z_d, ZPR64>;7030 7031 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $Rm]",7032 (!cast<Instruction>(NAME) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>;7033 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]",7034 (!cast<Instruction>(NAME) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>;7035 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]",7036 (!cast<Instruction>(NAME) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>;7037 7038 def : Pat <(op (nxv2i64 ZPR64:$Zt), (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zn), (i64 GPR64:$Rm), vt),7039 (!cast<Instruction>(NAME) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm)>;7040}7041 7042class sve_mem_sst_sv<bits<3> opc, bit xs, bit scaled, string asm,7043 RegisterOperand VecList, RegisterOperand zprext>7044: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm),7045 asm, "\t$Zt, $Pg, [$Rn, $Zm]",7046 "",7047 []>, Sched<[]> {7048 bits<3> Pg;7049 bits<5> Rn;7050 bits<5> Zm;7051 bits<5> Zt;7052 let Inst{31-25} = 0b1110010;7053 let Inst{24-22} = opc;7054 let Inst{21} = scaled;7055 let Inst{20-16} = Zm;7056 let Inst{15} = 0b1;7057 let Inst{14} = xs;7058 let Inst{13} = 0;7059 let Inst{12-10} = Pg;7060 let Inst{9-5} = Rn;7061 let Inst{4-0} = Zt;7062 7063 let hasSideEffects = 0;7064 let mayStore = 1;7065}7066 7067multiclass sve_mem_32b_sst_sv_32_scaled<bits<3> opc, string asm,7068 SDPatternOperator sxtw_op,7069 SDPatternOperator uxtw_op,7070 RegisterOperand sxtw_opnd,7071 RegisterOperand uxtw_opnd,7072 ValueType vt > {7073 def _UXTW_SCALED : sve_mem_sst_sv<opc, 0, 1, asm, Z_s, uxtw_opnd>;7074 def _SXTW_SCALED : sve_mem_sst_sv<opc, 1, 1, asm, Z_s, sxtw_opnd>;7075 7076 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]",7077 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>;7078 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]",7079 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>;7080 7081 def : Pat<(uxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt),7082 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;7083 def : Pat<(sxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt),7084 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;7085}7086 7087multiclass sve_mem_64b_sst_sv_32_scaled<bits<3> opc, string asm,7088 SDPatternOperator sxtw_op,7089 SDPatternOperator uxtw_op,7090 RegisterOperand sxtw_opnd,7091 RegisterOperand uxtw_opnd,7092 ValueType vt > {7093 def _UXTW_SCALED : sve_mem_sst_sv<opc, 0, 1, asm, Z_d, uxtw_opnd>;7094 def _SXTW_SCALED : sve_mem_sst_sv<opc, 1, 1, asm, Z_d, sxtw_opnd>;7095 7096 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]",7097 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>;7098 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]",7099 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>;7100 7101 def : Pat<(uxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt),7102 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;7103 def : Pat<(sxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt),7104 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;7105}7106 7107multiclass sve_mem_64b_sst_sv_32_unscaled<bits<3> opc, string asm,7108 SDPatternOperator sxtw_op,7109 SDPatternOperator uxtw_op,7110 RegisterOperand sxtw_opnd,7111 RegisterOperand uxtw_opnd,7112 ValueType vt> {7113 def _UXTW : sve_mem_sst_sv<opc, 0, 0, asm, Z_d, uxtw_opnd>;7114 def _SXTW : sve_mem_sst_sv<opc, 1, 0, asm, Z_d, sxtw_opnd>;7115 7116 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]",7117 (!cast<Instruction>(NAME # _UXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>;7118 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]",7119 (!cast<Instruction>(NAME # _SXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>;7120 7121 def : Pat<(uxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt),7122 (!cast<Instruction>(NAME # _UXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;7123 def : Pat<(sxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt),7124 (!cast<Instruction>(NAME # _SXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;7125}7126 7127multiclass sve_mem_32b_sst_sv_32_unscaled<bits<3> opc, string asm,7128 SDPatternOperator sxtw_op,7129 SDPatternOperator uxtw_op,7130 RegisterOperand sxtw_opnd,7131 RegisterOperand uxtw_opnd,7132 ValueType vt> {7133 def _UXTW : sve_mem_sst_sv<opc, 0, 0, asm, Z_s, uxtw_opnd>;7134 def _SXTW : sve_mem_sst_sv<opc, 1, 0, asm, Z_s, sxtw_opnd>;7135 7136 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]",7137 (!cast<Instruction>(NAME # _UXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>;7138 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]",7139 (!cast<Instruction>(NAME # _SXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>;7140 7141 def : Pat<(uxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt),7142 (!cast<Instruction>(NAME # _UXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;7143 def : Pat<(sxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt),7144 (!cast<Instruction>(NAME # _SXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;7145}7146 7147class sve_mem_sst_sv2<bits<2> msz, bit scaled, string asm,7148 RegisterOperand zprext>7149: I<(outs), (ins Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm),7150 asm, "\t$Zt, $Pg, [$Rn, $Zm]",7151 "",7152 []>, Sched<[]> {7153 bits<3> Pg;7154 bits<5> Rn;7155 bits<5> Zm;7156 bits<5> Zt;7157 let Inst{31-25} = 0b1110010;7158 let Inst{24-23} = msz;7159 let Inst{22} = 0b0;7160 let Inst{21} = scaled;7161 let Inst{20-16} = Zm;7162 let Inst{15-13} = 0b101;7163 let Inst{12-10} = Pg;7164 let Inst{9-5} = Rn;7165 let Inst{4-0} = Zt;7166 7167 let hasSideEffects = 0;7168 let mayStore = 1;7169}7170 7171multiclass sve_mem_sst_sv_64_scaled<bits<2> msz, string asm,7172 SDPatternOperator op,7173 RegisterOperand zprext,7174 ValueType vt> {7175 def _SCALED : sve_mem_sst_sv2<msz, 1, asm, zprext>;7176 7177 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]",7178 (!cast<Instruction>(NAME # _SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 0>;7179 7180 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt),7181 (!cast<Instruction>(NAME # _SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$indices)>;7182}7183 7184multiclass sve_mem_sst_sv_64_unscaled<bits<2> msz, string asm,7185 SDPatternOperator op,7186 ValueType vt> {7187 def NAME : sve_mem_sst_sv2<msz, 0, asm, ZPR64ExtLSL8>;7188 7189 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]",7190 (!cast<Instruction>(NAME) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), 0>;7191 7192 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt),7193 (!cast<Instruction>(NAME) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;7194}7195 7196class sve_mem_sst_vi<bits<3> opc, string asm, ZPRRegOp zprty,7197 RegisterOperand VecList, Operand imm_ty>7198: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, zprty:$Zn, imm_ty:$imm5),7199 asm, "\t$Zt, $Pg, [$Zn, $imm5]",7200 "",7201 []>, Sched<[]> {7202 bits<3> Pg;7203 bits<5> imm5;7204 bits<5> Zn;7205 bits<5> Zt;7206 let Inst{31-25} = 0b1110010;7207 let Inst{24-23} = opc{2-1};7208 let Inst{22} = 0b1;7209 let Inst{21} = opc{0};7210 let Inst{20-16} = imm5;7211 let Inst{15-13} = 0b101;7212 let Inst{12-10} = Pg;7213 let Inst{9-5} = Zn;7214 let Inst{4-0} = Zt;7215 7216 let hasSideEffects = 0;7217 let mayStore = 1;7218}7219 7220multiclass sve_mem_32b_sst_vi_ptrs<bits<3> opc, string asm,7221 Operand imm_ty,7222 SDPatternOperator op,7223 ValueType vt> {7224 def _IMM : sve_mem_sst_vi<opc, asm, ZPR32, Z_s, imm_ty>;7225 7226 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]",7227 (!cast<Instruction>(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 0>;7228 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $imm5]",7229 (!cast<Instruction>(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>;7230 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]",7231 (!cast<Instruction>(NAME # _IMM) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>;7232 7233 def : Pat<(op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), (nxv4i32 ZPR:$ptrs), imm_ty:$index, vt),7234 (!cast<Instruction>(NAME # _IMM) ZPR:$data, PPR:$gp, ZPR:$ptrs, imm_ty:$index)>;7235}7236 7237multiclass sve_mem_64b_sst_vi_ptrs<bits<3> opc, string asm,7238 Operand imm_ty,7239 SDPatternOperator op,7240 ValueType vt> {7241 def _IMM : sve_mem_sst_vi<opc, asm, ZPR64, Z_d, imm_ty>;7242 7243 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]",7244 (!cast<Instruction>(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 0>;7245 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $imm5]",7246 (!cast<Instruction>(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>;7247 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]",7248 (!cast<Instruction>(NAME # _IMM) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>;7249 7250 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), (nxv2i64 ZPR:$ptrs), imm_ty:$index, vt),7251 (!cast<Instruction>(NAME # _IMM) ZPR:$data, PPR:$gp, ZPR:$ptrs, imm_ty:$index)>;7252}7253 7254class sve_mem_z_spill<string asm>7255: I<(outs), (ins ZPRAny:$Zt, GPR64sp:$Rn, simm9:$imm9),7256 asm, "\t$Zt, [$Rn, $imm9, mul vl]",7257 "",7258 []>, Sched<[]> {7259 bits<5> Rn;7260 bits<5> Zt;7261 bits<9> imm9;7262 let Inst{31-22} = 0b1110010110;7263 let Inst{21-16} = imm9{8-3};7264 let Inst{15-13} = 0b010;7265 let Inst{12-10} = imm9{2-0};7266 let Inst{9-5} = Rn;7267 let Inst{4-0} = Zt;7268 7269 let hasSideEffects = 0;7270 let mayStore = 1;7271}7272 7273multiclass sve_mem_z_spill<string asm> {7274 def NAME : sve_mem_z_spill<asm>;7275 7276 def : InstAlias<asm # "\t$Zt, [$Rn]",7277 (!cast<Instruction>(NAME) ZPRAny:$Zt, GPR64sp:$Rn, 0), 1>;7278}7279 7280class sve_mem_p_spill<string asm>7281: I<(outs), (ins PPRorPNRAny:$Pt, GPR64sp:$Rn, simm9:$imm9),7282 asm, "\t$Pt, [$Rn, $imm9, mul vl]",7283 "",7284 []>, Sched<[]> {7285 bits<4> Pt;7286 bits<5> Rn;7287 bits<9> imm9;7288 let Inst{31-22} = 0b1110010110;7289 let Inst{21-16} = imm9{8-3};7290 let Inst{15-13} = 0b000;7291 let Inst{12-10} = imm9{2-0};7292 let Inst{9-5} = Rn;7293 let Inst{4} = 0b0;7294 let Inst{3-0} = Pt;7295 7296 let hasSideEffects = 0;7297 let mayStore = 1;7298}7299 7300multiclass sve_mem_p_spill<string asm> {7301 def NAME : sve_mem_p_spill<asm>;7302 7303 def : InstAlias<asm # "\t$Pt, [$Rn]",7304 (!cast<Instruction>(NAME) PPRorPNRAny:$Pt, GPR64sp:$Rn, 0), 1>;7305}7306 7307//===----------------------------------------------------------------------===//7308// SVE Permute - Predicates Group7309//===----------------------------------------------------------------------===//7310 7311class sve_int_perm_bin_perm_pp<bits<3> opc, bits<2> sz8_64, string asm,7312 PPRRegOp pprty, SDPatternOperator op>7313: I<(outs pprty:$Pd), (ins pprty:$Pn, pprty:$Pm),7314 asm, "\t$Pd, $Pn, $Pm",7315 "",7316 [(set nxv16i1:$Pd, (op nxv16i1:$Pn, nxv16i1:$Pm))]>, Sched<[]> {7317 bits<4> Pd;7318 bits<4> Pm;7319 bits<4> Pn;7320 let Inst{31-24} = 0b00000101;7321 let Inst{23-22} = sz8_64;7322 let Inst{21-20} = 0b10;7323 let Inst{19-16} = Pm;7324 let Inst{15-13} = 0b010;7325 let Inst{12-10} = opc;7326 let Inst{9} = 0b0;7327 let Inst{8-5} = Pn;7328 let Inst{4} = 0b0;7329 let Inst{3-0} = Pd;7330 7331 let hasSideEffects = 0;7332}7333 7334multiclass sve_int_perm_bin_perm_pp<bits<3> opc, string asm,7335 SDPatternOperator ir_op,7336 SDPatternOperator op_b16,7337 SDPatternOperator op_b32,7338 SDPatternOperator op_b64> {7339 def _B : sve_int_perm_bin_perm_pp<opc, 0b00, asm, PPR8, ir_op>;7340 def _H : sve_int_perm_bin_perm_pp<opc, 0b01, asm, PPR16, op_b16>;7341 def _S : sve_int_perm_bin_perm_pp<opc, 0b10, asm, PPR32, op_b32>;7342 def _D : sve_int_perm_bin_perm_pp<opc, 0b11, asm, PPR64, op_b64>;7343 7344 def : SVE_2_Op_Pat<nxv8i1, ir_op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>;7345 def : SVE_2_Op_Pat<nxv4i1, ir_op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>;7346 def : SVE_2_Op_Pat<nxv2i1, ir_op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>;7347}7348 7349class sve_int_perm_punpk<bit opc, string asm>7350: I<(outs PPR16:$Pd), (ins PPR8:$Pn),7351 asm, "\t$Pd, $Pn",7352 "",7353 []>, Sched<[]> {7354 bits<4> Pd;7355 bits<4> Pn;7356 let Inst{31-17} = 0b000001010011000;7357 let Inst{16} = opc;7358 let Inst{15-9} = 0b0100000;7359 let Inst{8-5} = Pn;7360 let Inst{4} = 0b0;7361 let Inst{3-0} = Pd;7362 7363 let hasSideEffects = 0;7364}7365 7366multiclass sve_int_perm_punpk<bit opc, string asm, SDPatternOperator op> {7367 def NAME : sve_int_perm_punpk<opc, asm>;7368 7369 def : SVE_1_Op_Pat<nxv8i1, op, nxv16i1, !cast<Instruction>(NAME)>;7370 def : SVE_1_Op_Pat<nxv4i1, op, nxv8i1, !cast<Instruction>(NAME)>;7371 def : SVE_1_Op_Pat<nxv2i1, op, nxv4i1, !cast<Instruction>(NAME)>;7372}7373 7374class sve_int_rdffr_pred<bit s, string asm, SDPatternOperator op = null_frag>7375: I<(outs PPR8:$Pd), (ins PPRAny:$Pg),7376 asm, "\t$Pd, $Pg/z",7377 "",7378 [(set (nxv16i1 PPR8:$Pd), (op (nxv16i1 PPRAny:$Pg)))]>, Sched<[]> {7379 bits<4> Pd;7380 bits<4> Pg;7381 let Inst{31-23} = 0b001001010;7382 let Inst{22} = s;7383 let Inst{21-9} = 0b0110001111000;7384 let Inst{8-5} = Pg;7385 let Inst{4} = 0;7386 let Inst{3-0} = Pd;7387 7388 let Defs = !if(s, [NZCV], []);7389 let Uses = [FFR];7390 let hasSideEffects = 1;7391}7392 7393class sve_int_rdffr_unpred<string asm, SDPatternOperator op> : I<7394 (outs PPR8:$Pd), (ins),7395 asm, "\t$Pd",7396 "",7397 [(set (nxv16i1 PPR8:$Pd), (op))]>, Sched<[]> {7398 bits<4> Pd;7399 let Inst{31-4} = 0b0010010100011001111100000000;7400 let Inst{3-0} = Pd;7401 7402 let Uses = [FFR];7403 let hasSideEffects = 1;7404}7405 7406class sve_int_wrffr<string asm, SDPatternOperator op>7407: I<(outs), (ins PPR8:$Pn),7408 asm, "\t$Pn",7409 "",7410 [(op (nxv16i1 PPR8:$Pn))]>, Sched<[]> {7411 bits<4> Pn;7412 let Inst{31-9} = 0b00100101001010001001000;7413 let Inst{8-5} = Pn;7414 let Inst{4-0} = 0b00000;7415 7416 let Defs = [FFR];7417 let hasSideEffects = 1;7418}7419 7420class sve_int_setffr<string asm, SDPatternOperator op>7421: I<(outs), (ins),7422 asm, "",7423 "",7424 [(op)]>, Sched<[]> {7425 let Inst{31-0} = 0b00100101001011001001000000000000;7426 7427 let Defs = [FFR];7428 let hasSideEffects = 1;7429}7430 7431//===----------------------------------------------------------------------===//7432// SVE Permute Vector - Predicated Group7433//===----------------------------------------------------------------------===//7434 7435class sve_int_perm_clast_rz<bits<2> sz8_64, bit ab, string asm,7436 ZPRRegOp zprty, RegisterClass rt>7437: I<(outs rt:$Rdn), (ins PPR3bAny:$Pg, rt:$_Rdn, zprty:$Zm),7438 asm, "\t$Rdn, $Pg, $_Rdn, $Zm",7439 "",7440 []>, Sched<[]> {7441 bits<3> Pg;7442 bits<5> Rdn;7443 bits<5> Zm;7444 let Inst{31-24} = 0b00000101;7445 let Inst{23-22} = sz8_64;7446 let Inst{21-17} = 0b11000;7447 let Inst{16} = ab;7448 let Inst{15-13} = 0b101;7449 let Inst{12-10} = Pg;7450 let Inst{9-5} = Zm;7451 let Inst{4-0} = Rdn;7452 7453 let Constraints = "$Rdn = $_Rdn";7454 let hasSideEffects = 0;7455}7456 7457multiclass sve_int_perm_clast_rz<bit ab, string asm, SDPatternOperator op> {7458 def _B : sve_int_perm_clast_rz<0b00, ab, asm, ZPR8, GPR32>;7459 def _H : sve_int_perm_clast_rz<0b01, ab, asm, ZPR16, GPR32>;7460 def _S : sve_int_perm_clast_rz<0b10, ab, asm, ZPR32, GPR32>;7461 def _D : sve_int_perm_clast_rz<0b11, ab, asm, ZPR64, GPR64>;7462 7463 def : SVE_3_Op_Pat<i32, op, nxv16i1, i32, nxv16i8, !cast<Instruction>(NAME # _B)>;7464 def : SVE_3_Op_Pat<i32, op, nxv8i1, i32, nxv8i16, !cast<Instruction>(NAME # _H)>;7465 def : SVE_3_Op_Pat<i32, op, nxv4i1, i32, nxv4i32, !cast<Instruction>(NAME # _S)>;7466 def : SVE_3_Op_Pat<i64, op, nxv2i1, i64, nxv2i64, !cast<Instruction>(NAME # _D)>;7467}7468 7469class sve_int_perm_clast_vz<bits<2> sz8_64, bit ab, string asm,7470 ZPRRegOp zprty, RegisterClass rt>7471: I<(outs rt:$Vdn), (ins PPR3bAny:$Pg, rt:$_Vdn, zprty:$Zm),7472 asm, "\t$Vdn, $Pg, $_Vdn, $Zm",7473 "",7474 []>, Sched<[]> {7475 bits<3> Pg;7476 bits<5> Vdn;7477 bits<5> Zm;7478 let Inst{31-24} = 0b00000101;7479 let Inst{23-22} = sz8_64;7480 let Inst{21-17} = 0b10101;7481 let Inst{16} = ab;7482 let Inst{15-13} = 0b100;7483 let Inst{12-10} = Pg;7484 let Inst{9-5} = Zm;7485 let Inst{4-0} = Vdn;7486 7487 let Constraints = "$Vdn = $_Vdn";7488 let hasSideEffects = 0;7489}7490 7491multiclass sve_int_perm_clast_vz<bit ab, string asm, SDPatternOperator op> {7492 def _B : sve_int_perm_clast_vz<0b00, ab, asm, ZPR8, FPR8>;7493 def _H : sve_int_perm_clast_vz<0b01, ab, asm, ZPR16, FPR16>;7494 def _S : sve_int_perm_clast_vz<0b10, ab, asm, ZPR32, FPR32>;7495 def _D : sve_int_perm_clast_vz<0b11, ab, asm, ZPR64, FPR64>;7496 7497 def : SVE_3_Op_Pat<f16, op, nxv8i1, f16, nxv8f16, !cast<Instruction>(NAME # _H)>;7498 def : SVE_3_Op_Pat<f32, op, nxv4i1, f32, nxv4f32, !cast<Instruction>(NAME # _S)>;7499 def : SVE_3_Op_Pat<f64, op, nxv2i1, f64, nxv2f64, !cast<Instruction>(NAME # _D)>;7500 7501 def : SVE_3_Op_Pat<bf16, op, nxv8i1, bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>;7502}7503 7504class sve_int_perm_clast_zz<bits<2> sz8_64, bit ab, string asm,7505 ZPRRegOp zprty>7506: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm),7507 asm, "\t$Zdn, $Pg, $_Zdn, $Zm",7508 "",7509 []>, Sched<[]> {7510 bits<3> Pg;7511 bits<5> Zdn;7512 bits<5> Zm;7513 let Inst{31-24} = 0b00000101;7514 let Inst{23-22} = sz8_64;7515 let Inst{21-17} = 0b10100;7516 let Inst{16} = ab;7517 let Inst{15-13} = 0b100;7518 let Inst{12-10} = Pg;7519 let Inst{9-5} = Zm;7520 let Inst{4-0} = Zdn;7521 7522 let Constraints = "$Zdn = $_Zdn";7523 let DestructiveInstType = DestructiveOther;7524 let ElementSize = ElementSizeNone;7525 let hasSideEffects = 0;7526}7527 7528multiclass sve_int_perm_clast_zz<bit ab, string asm, SDPatternOperator op> {7529 def _B : sve_int_perm_clast_zz<0b00, ab, asm, ZPR8>;7530 def _H : sve_int_perm_clast_zz<0b01, ab, asm, ZPR16>;7531 def _S : sve_int_perm_clast_zz<0b10, ab, asm, ZPR32>;7532 def _D : sve_int_perm_clast_zz<0b11, ab, asm, ZPR64>;7533 7534 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;7535 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;7536 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;7537 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;7538 7539 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;7540 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;7541 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;7542 7543 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>;7544}7545 7546class sve_int_perm_last_r<bits<2> sz8_64, bit ab, string asm,7547 ZPRRegOp zprty, RegisterClass resultRegType>7548: I<(outs resultRegType:$Rd), (ins PPR3bAny:$Pg, zprty:$Zn),7549 asm, "\t$Rd, $Pg, $Zn",7550 "",7551 []>, Sched<[]> {7552 bits<3> Pg;7553 bits<5> Rd;7554 bits<5> Zn;7555 let Inst{31-24} = 0b00000101;7556 let Inst{23-22} = sz8_64;7557 let Inst{21-17} = 0b10000;7558 let Inst{16} = ab;7559 let Inst{15-13} = 0b101;7560 let Inst{12-10} = Pg;7561 let Inst{9-5} = Zn;7562 let Inst{4-0} = Rd;7563 7564 let hasSideEffects = 0;7565}7566 7567multiclass sve_int_perm_last_r<bit ab, string asm, SDPatternOperator op> {7568 def _B : sve_int_perm_last_r<0b00, ab, asm, ZPR8, GPR32>;7569 def _H : sve_int_perm_last_r<0b01, ab, asm, ZPR16, GPR32>;7570 def _S : sve_int_perm_last_r<0b10, ab, asm, ZPR32, GPR32>;7571 def _D : sve_int_perm_last_r<0b11, ab, asm, ZPR64, GPR64>;7572 7573 def : SVE_2_Op_Pat<i32, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;7574 def : SVE_2_Op_Pat<i32, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>;7575 def : SVE_2_Op_Pat<i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;7576 def : SVE_2_Op_Pat<i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;7577}7578 7579class sve_int_perm_last_v<bits<2> sz8_64, bit ab, string asm,7580 ZPRRegOp zprty, RegisterClass dstRegtype>7581: I<(outs dstRegtype:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn),7582 asm, "\t$Vd, $Pg, $Zn",7583 "",7584 []>, Sched<[]> {7585 bits<3> Pg;7586 bits<5> Vd;7587 bits<5> Zn;7588 let Inst{31-24} = 0b00000101;7589 let Inst{23-22} = sz8_64;7590 let Inst{21-17} = 0b10001;7591 let Inst{16} = ab;7592 let Inst{15-13} = 0b100;7593 let Inst{12-10} = Pg;7594 let Inst{9-5} = Zn;7595 let Inst{4-0} = Vd;7596 7597 let hasSideEffects = 0;7598}7599 7600multiclass sve_int_perm_last_v<bit ab, string asm, SDPatternOperator op> {7601 def _B : sve_int_perm_last_v<0b00, ab, asm, ZPR8, FPR8>;7602 def _H : sve_int_perm_last_v<0b01, ab, asm, ZPR16, FPR16>;7603 def _S : sve_int_perm_last_v<0b10, ab, asm, ZPR32, FPR32>;7604 def _D : sve_int_perm_last_v<0b11, ab, asm, ZPR64, FPR64>;7605 7606 def : SVE_2_Op_Pat<f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;7607 def : SVE_2_Op_Pat<f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;7608 def : SVE_2_Op_Pat<f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>;7609 def : SVE_2_Op_Pat<f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;7610 7611 def : SVE_2_Op_Pat<bf16, op, nxv8i1, nxv8bf16, !cast<Instruction>(NAME # _H)>;7612}7613 7614class sve_int_perm_splice<bits<2> sz8_64, string asm, ZPRRegOp zprty>7615: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm),7616 asm, "\t$Zdn, $Pg, $_Zdn, $Zm",7617 "",7618 []>, Sched<[]> {7619 bits<3> Pg;7620 bits<5> Zdn;7621 bits<5> Zm;7622 let Inst{31-24} = 0b00000101;7623 let Inst{23-22} = sz8_64;7624 let Inst{21-13} = 0b101100100;7625 let Inst{12-10} = Pg;7626 let Inst{9-5} = Zm;7627 let Inst{4-0} = Zdn;7628 7629 let Constraints = "$Zdn = $_Zdn";7630 let DestructiveInstType = DestructiveOther;7631 let ElementSize = ElementSizeNone;7632 let hasSideEffects = 0;7633}7634 7635multiclass sve_int_perm_splice<string asm, SDPatternOperator op> {7636 def _B : sve_int_perm_splice<0b00, asm, ZPR8>;7637 def _H : sve_int_perm_splice<0b01, asm, ZPR16>;7638 def _S : sve_int_perm_splice<0b10, asm, ZPR32>;7639 def _D : sve_int_perm_splice<0b11, asm, ZPR64>;7640 7641 foreach VT = [nxv16i8] in7642 def : SVE_3_Op_Pat<VT, op, nxv16i1, VT, VT, !cast<Instruction>(NAME # _B)>;7643 7644 foreach VT = [nxv8i16, nxv8f16, nxv8bf16] in7645 def : SVE_3_Op_Pat<VT, op, nxv8i1, VT, VT, !cast<Instruction>(NAME # _H)>;7646 7647 foreach VT = [nxv4i32, nxv4f16, nxv4f32, nxv4bf16] in7648 def : SVE_3_Op_Pat<VT, op, nxv4i1, VT, VT, !cast<Instruction>(NAME # _S)>;7649 7650 foreach VT = [nxv2i64, nxv2f16, nxv2f32, nxv2f64, nxv2bf16] in7651 def : SVE_3_Op_Pat<VT, op, nxv2i1, VT, VT, !cast<Instruction>(NAME # _D)>;7652}7653 7654class sve2_int_perm_splice_cons<bits<2> sz8_64, string asm,7655 ZPRRegOp zprty, RegisterOperand VecList>7656: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, VecList:$Zn),7657 asm, "\t$Zd, $Pg, $Zn",7658 "",7659 []>, Sched<[]> {7660 bits<3> Pg;7661 bits<5> Zn;7662 bits<5> Zd;7663 let Inst{31-24} = 0b00000101;7664 let Inst{23-22} = sz8_64;7665 let Inst{21-13} = 0b101101100;7666 let Inst{12-10} = Pg;7667 let Inst{9-5} = Zn;7668 let Inst{4-0} = Zd;7669 7670 let hasSideEffects = 0;7671}7672 7673multiclass sve2_int_perm_splice_cons<string asm, SDPatternOperator op> {7674 def _B : sve2_int_perm_splice_cons<0b00, asm, ZPR8, ZZ_b>;7675 def _H : sve2_int_perm_splice_cons<0b01, asm, ZPR16, ZZ_h>;7676 def _S : sve2_int_perm_splice_cons<0b10, asm, ZPR32, ZZ_s>;7677 def _D : sve2_int_perm_splice_cons<0b11, asm, ZPR64, ZZ_d>;7678 7679 let AddedComplexity = 2 in {7680 foreach VT = [nxv16i8] in7681 def : Pat<(VT (op nxv16i1:$pred, VT:$zn1, VT:$zn2)),7682 (!cast<Instruction>(NAME # _B)7683 nxv16i1:$pred, (REG_SEQUENCE ZPR2, VT:$zn1, zsub0, VT:$zn2, zsub1))>;7684 7685 foreach VT = [nxv8i16, nxv8f16, nxv8bf16] in7686 def : Pat<(VT (op nxv8i1:$pred, VT:$zn1, VT:$zn2)),7687 (!cast<Instruction>(NAME # _H)7688 nxv8i1:$pred, (REG_SEQUENCE ZPR2, VT:$zn1, zsub0, VT:$zn2, zsub1))>;7689 7690 foreach VT = [nxv4i32, nxv4f16, nxv4f32, nxv4bf16] in7691 def : Pat<(VT (op nxv4i1:$pred, VT:$zn1, VT:$zn2)),7692 (!cast<Instruction>(NAME # _S)7693 nxv4i1:$pred, (REG_SEQUENCE ZPR2, VT:$zn1, zsub0, VT:$zn2, zsub1))>;7694 7695 foreach VT = [nxv2i64, nxv2f16, nxv2f32, nxv2f64, nxv2bf16] in7696 def : Pat<(VT (op nxv2i1:$pred, VT:$zn1, VT:$zn2)),7697 (!cast<Instruction>(NAME # _D)7698 nxv2i1:$pred, (REG_SEQUENCE ZPR2, VT:$zn1, zsub0, VT:$zn2, zsub1))>;7699 }7700}7701 7702class sve2_int_perm_expand<bits<2> sz, string asm,7703 ZPRRegOp zprty>7704: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn),7705 asm, "\t$Zd, $Pg, $Zn",7706 "",7707 []>, Sched<[]> {7708 bits<3> Pg;7709 bits<5> Zn;7710 bits<5> Zd;7711 let Inst{31-24} = 0b00000101;7712 let Inst{23-22} = sz;7713 let Inst{21-13} = 0b110001100;7714 let Inst{12-10} = Pg;7715 let Inst{9-5} = Zn;7716 let Inst{4-0} = Zd;7717 7718 let hasSideEffects = 0;7719}7720 7721multiclass sve2_int_perm_expand<string asm> {7722 def _B : sve2_int_perm_expand<0b00, asm, ZPR8>;7723 def _H : sve2_int_perm_expand<0b01, asm, ZPR16>;7724 def _S : sve2_int_perm_expand<0b10, asm, ZPR32>;7725 def _D : sve2_int_perm_expand<0b11, asm, ZPR64>;7726}7727 7728class sve_int_perm_rev<bits<2> sz8_64, bits<2> opc, string asm,7729 ZPRRegOp zprty>7730: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn),7731 asm, "\t$Zd, $Pg/m, $Zn",7732 "",7733 []>, Sched<[]> {7734 bits<5> Zd;7735 bits<3> Pg;7736 bits<5> Zn;7737 let Inst{31-24} = 0b00000101;7738 let Inst{23-22} = sz8_64;7739 let Inst{21-18} = 0b1001;7740 let Inst{17-16} = opc;7741 let Inst{15-13} = 0b100;7742 let Inst{12-10} = Pg;7743 let Inst{9-5} = Zn;7744 let Inst{4-0} = Zd;7745 7746 let Constraints = "$Zd = $_Zd";7747 let DestructiveInstType = DestructiveOther;7748 let ElementSize = zprty.ElementSize;7749 let hasSideEffects = 0;7750}7751 7752multiclass sve_int_perm_rev_rbit<string asm, SDPatternOperator op> {7753 def _B : sve_int_perm_rev<0b00, 0b11, asm, ZPR8>;7754 def _H : sve_int_perm_rev<0b01, 0b11, asm, ZPR16>;7755 def _S : sve_int_perm_rev<0b10, 0b11, asm, ZPR32>;7756 def _D : sve_int_perm_rev<0b11, 0b11, asm, ZPR64>;7757 7758 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;7759 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>;7760 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;7761 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;7762}7763 7764multiclass sve_int_perm_rev_revb<string asm, SDPatternOperator op> {7765 def _H : sve_int_perm_rev<0b01, 0b00, asm, ZPR16>;7766 def _S : sve_int_perm_rev<0b10, 0b00, asm, ZPR32>;7767 def _D : sve_int_perm_rev<0b11, 0b00, asm, ZPR64>;7768 7769 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>;7770 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;7771 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;7772}7773 7774multiclass sve_int_perm_rev_revh<string asm, SDPatternOperator op> {7775 def _S : sve_int_perm_rev<0b10, 0b01, asm, ZPR32>;7776 def _D : sve_int_perm_rev<0b11, 0b01, asm, ZPR64>;7777 7778 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;7779 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;7780}7781 7782multiclass sve_int_perm_rev_revw<string asm, SDPatternOperator op> {7783 def _D : sve_int_perm_rev<0b11, 0b10, asm, ZPR64>;7784 7785 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;7786}7787 7788class sve_int_perm_rev_z<bits<2> sz, bits<4> opc, string asm,7789 ZPRRegOp zprty>7790: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn),7791 asm, "\t$Zd, $Pg/z, $Zn",7792 "",7793 []>, Sched<[]> {7794 bits<5> Zd;7795 bits<3> Pg;7796 bits<5> Zn;7797 let Inst{31-24} = 0b00000101;7798 let Inst{23-22} = sz;7799 let Inst{21-20} = 0b10;7800 let Inst{19-16} = opc;7801 let Inst{15-13} = 0b101;7802 let Inst{12-10} = Pg;7803 let Inst{9-5} = Zn;7804 let Inst{4-0} = Zd;7805 7806 let hasSideEffects = 0;7807}7808 7809multiclass sve_int_perm_rev_rbit_z<string asm, SDPatternOperator op> {7810 def _B : sve_int_perm_rev_z<0b00, 0b0111, asm, ZPR8>;7811 def _H : sve_int_perm_rev_z<0b01, 0b0111, asm, ZPR16>;7812 def _S : sve_int_perm_rev_z<0b10, 0b0111, asm, ZPR32>;7813 def _D : sve_int_perm_rev_z<0b11, 0b0111, asm, ZPR64>;7814 7815 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;7816 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>;7817 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;7818 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;7819}7820 7821multiclass sve_int_perm_rev_revb_z<string asm, SDPatternOperator op> {7822 def _H : sve_int_perm_rev_z<0b01, 0b0100, asm, ZPR16>;7823 def _S : sve_int_perm_rev_z<0b10, 0b0100, asm, ZPR32>;7824 def _D : sve_int_perm_rev_z<0b11, 0b0100, asm, ZPR64>;7825 7826 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>;7827 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;7828 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;7829}7830 7831multiclass sve_int_perm_rev_revh_z<string asm, SDPatternOperator op> {7832 def _S : sve_int_perm_rev_z<0b10, 0b0101, asm, ZPR32>;7833 def _D : sve_int_perm_rev_z<0b11, 0b0101, asm, ZPR64>;7834 7835 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;7836 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;7837}7838 7839multiclass sve_int_perm_rev_revw_z<string asm, SDPatternOperator op> {7840 def _D : sve_int_perm_rev_z<0b11, 0b0110, asm, ZPR64>;7841 7842 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;7843}7844 7845multiclass sve_int_perm_rev_revd_z<string asm, SDPatternOperator op> {7846 def NAME : sve_int_perm_rev_z<0b00, 0b1110, asm, ZPR128>;7847 7848 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME)>;7849 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME)>;7850 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME)>;7851 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME)>;7852 7853 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, !cast<Instruction>(NAME)>;7854 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME)>;7855 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME)>;7856 defm : SVE_1_Op_PassthruUndefZero_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME)>;7857}7858 7859class sve_int_perm_cpy_r<bits<2> sz8_64, string asm, ZPRRegOp zprty,7860 RegisterClass srcRegType>7861: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, srcRegType:$Rn),7862 asm, "\t$Zd, $Pg/m, $Rn",7863 "",7864 []>, Sched<[]> {7865 bits<3> Pg;7866 bits<5> Rn;7867 bits<5> Zd;7868 let Inst{31-24} = 0b00000101;7869 let Inst{23-22} = sz8_64;7870 let Inst{21-13} = 0b101000101;7871 let Inst{12-10} = Pg;7872 let Inst{9-5} = Rn;7873 let Inst{4-0} = Zd;7874 7875 let Constraints = "$Zd = $_Zd";7876 let DestructiveInstType = DestructiveOther;7877 let ElementSize = zprty.ElementSize;7878 let hasSideEffects = 0;7879}7880 7881multiclass sve_int_perm_cpy_r<string asm, SDPatternOperator op> {7882 def _B : sve_int_perm_cpy_r<0b00, asm, ZPR8, GPR32sp>;7883 def _H : sve_int_perm_cpy_r<0b01, asm, ZPR16, GPR32sp>;7884 def _S : sve_int_perm_cpy_r<0b10, asm, ZPR32, GPR32sp>;7885 def _D : sve_int_perm_cpy_r<0b11, asm, ZPR64, GPR64sp>;7886 7887 def : InstAlias<"mov $Zd, $Pg/m, $Rn",7888 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>;7889 def : InstAlias<"mov $Zd, $Pg/m, $Rn",7890 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>;7891 def : InstAlias<"mov $Zd, $Pg/m, $Rn",7892 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>;7893 def : InstAlias<"mov $Zd, $Pg/m, $Rn",7894 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPR3bAny:$Pg, GPR64sp:$Rn), 1>;7895 7896 def : Pat<(nxv16i8 (op nxv16i1:$pg, i32:$splat, nxv16i8:$passthru)),7897 (!cast<Instruction>(NAME # _B) $passthru, $pg, $splat)>;7898 def : Pat<(nxv8i16 (op nxv8i1:$pg, i32:$splat, nxv8i16:$passthru)),7899 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>;7900 def : Pat<(nxv4i32 (op nxv4i1:$pg, i32:$splat, nxv4i32:$passthru)),7901 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>;7902 def : Pat<(nxv2i64 (op nxv2i1:$pg, i64:$splat, nxv2i64:$passthru)),7903 (!cast<Instruction>(NAME # _D) $passthru, $pg, $splat)>;7904}7905 7906class sve_int_perm_cpy_v<bits<2> sz8_64, string asm, ZPRRegOp zprty,7907 RegisterClass srcRegtype>7908: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, srcRegtype:$Vn),7909 asm, "\t$Zd, $Pg/m, $Vn",7910 "",7911 []>, Sched<[]> {7912 bits<3> Pg;7913 bits<5> Vn;7914 bits<5> Zd;7915 let Inst{31-24} = 0b00000101;7916 let Inst{23-22} = sz8_64;7917 let Inst{21-13} = 0b100000100;7918 let Inst{12-10} = Pg;7919 let Inst{9-5} = Vn;7920 let Inst{4-0} = Zd;7921 7922 let Constraints = "$Zd = $_Zd";7923 let DestructiveInstType = DestructiveOther;7924 let ElementSize = zprty.ElementSize;7925 let hasSideEffects = 0;7926}7927 7928multiclass sve_int_perm_cpy_v<string asm, SDPatternOperator op> {7929 def _B : sve_int_perm_cpy_v<0b00, asm, ZPR8, FPR8>;7930 def _H : sve_int_perm_cpy_v<0b01, asm, ZPR16, FPR16>;7931 def _S : sve_int_perm_cpy_v<0b10, asm, ZPR32, FPR32>;7932 def _D : sve_int_perm_cpy_v<0b11, asm, ZPR64, FPR64>;7933 7934 def : InstAlias<"mov $Zd, $Pg/m, $Vn",7935 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPR3bAny:$Pg, FPR8:$Vn), 1>;7936 def : InstAlias<"mov $Zd, $Pg/m, $Vn",7937 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPR3bAny:$Pg, FPR16:$Vn), 1>;7938 def : InstAlias<"mov $Zd, $Pg/m, $Vn",7939 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPR3bAny:$Pg, FPR32:$Vn), 1>;7940 def : InstAlias<"mov $Zd, $Pg/m, $Vn",7941 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPR3bAny:$Pg, FPR64:$Vn), 1>;7942 7943 def : Pat<(nxv8f16 (op nxv8i1:$pg, f16:$splat, nxv8f16:$passthru)),7944 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>;7945 def : Pat<(nxv4f16 (op nxv4i1:$pg, f16:$splat, nxv4f16:$passthru)),7946 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>;7947 def : Pat<(nxv2f16 (op nxv2i1:$pg, f16:$splat, nxv2f16:$passthru)),7948 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>;7949 def : Pat<(nxv2f32 (op nxv2i1:$pg, f32:$splat, nxv2f32:$passthru)),7950 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>;7951 def : Pat<(nxv4f32 (op nxv4i1:$pg, f32:$splat, nxv4f32:$passthru)),7952 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>;7953 def : Pat<(nxv2f64 (op nxv2i1:$pg, f64:$splat, nxv2f64:$passthru)),7954 (!cast<Instruction>(NAME # _D) $passthru, $pg, $splat)>;7955 7956 def : Pat<(nxv8bf16 (op nxv8i1:$pg, bf16:$splat, nxv8bf16:$passthru)),7957 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>;7958}7959 7960class sve_int_perm_compact<bits<2> sz, string asm, ZPRRegOp zprty>7961: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn),7962 asm, "\t$Zd, $Pg, $Zn",7963 "",7964 []>, Sched<[]> {7965 bits<3> Pg;7966 bits<5> Zd;7967 bits<5> Zn;7968 let Inst{31-24} = 0b00000101;7969 let Inst{23-22} = sz;7970 let Inst{21-13} = 0b100001100;7971 let Inst{12-10} = Pg;7972 let Inst{9-5} = Zn;7973 let Inst{4-0} = Zd;7974 7975 let hasSideEffects = 0;7976}7977 7978multiclass sve_int_perm_compact_sd<string asm, SDPatternOperator op> {7979 def _S : sve_int_perm_compact<0b10, asm, ZPR32>;7980 def _D : sve_int_perm_compact<0b11, asm, ZPR64>;7981 7982 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;7983 def : SVE_2_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;7984 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;7985 def : SVE_2_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;7986}7987 7988multiclass sve_int_perm_compact_bh<string asm, SDPatternOperator op> {7989 def _B : sve_int_perm_compact<0b00, asm, ZPR8>;7990 def _H : sve_int_perm_compact<0b01, asm, ZPR16>;7991 7992 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;7993 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>;7994 def : SVE_2_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;7995 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, !cast<Instruction>(NAME # _H)>;7996}7997 7998//===----------------------------------------------------------------------===//7999// SVE Memory - Contiguous Load Group8000//===----------------------------------------------------------------------===//8001 8002class sve_mem_cld_si_base<bits<4> dtype, bit nf, string asm,8003 RegisterOperand VecList>8004: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4),8005 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]",8006 "",8007 []>, Sched<[]> {8008 bits<3> Pg;8009 bits<5> Rn;8010 bits<5> Zt;8011 bits<4> imm4;8012 let Inst{31-25} = 0b1010010;8013 let Inst{24-21} = dtype;8014 let Inst{20} = nf;8015 let Inst{19-16} = imm4;8016 let Inst{15-13} = 0b101;8017 let Inst{12-10} = Pg;8018 let Inst{9-5} = Rn;8019 let Inst{4-0} = Zt;8020 8021 let Defs = !if(nf, [FFR], []);8022 let Uses = !if(nf, [FFR], []);8023 let hasSideEffects = nf;8024 let mayLoad = 1;8025}8026 8027multiclass sve_mem_cld_si_base<bits<4> dtype, bit nf, string asm,8028 RegisterOperand listty, ZPRRegOp zprty> {8029 def NAME : sve_mem_cld_si_base<dtype, nf, asm, listty>;8030 8031 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",8032 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>;8033 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]",8034 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>;8035 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",8036 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;8037}8038 8039multiclass sve_mem_cld_si<bits<4> dtype, string asm, RegisterOperand listty,8040 ZPRRegOp zprty>8041: sve_mem_cld_si_base<dtype, 0, asm, listty, zprty>;8042 8043multiclass sve_mem_cldnf_si<bits<4> dtype, string asm, RegisterOperand listty,8044 ZPRRegOp zprty>8045: sve_mem_cld_si_base<dtype, 1, asm, listty, zprty>;8046 8047class sve_mem_cldnt_si_base<bits<2> msz, string asm, RegisterOperand VecList>8048: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4),8049 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]",8050 "",8051 []>, Sched<[]> {8052 bits<5> Zt;8053 bits<3> Pg;8054 bits<5> Rn;8055 bits<4> imm4;8056 let Inst{31-25} = 0b1010010;8057 let Inst{24-23} = msz;8058 let Inst{22-20} = 0b000;8059 let Inst{19-16} = imm4;8060 let Inst{15-13} = 0b111;8061 let Inst{12-10} = Pg;8062 let Inst{9-5} = Rn;8063 let Inst{4-0} = Zt;8064 8065 let hasSideEffects = 0;8066 let mayLoad = 1;8067}8068 8069multiclass sve_mem_cldnt_si<bits<2> msz, string asm, RegisterOperand listty,8070 ZPRRegOp zprty> {8071 def NAME : sve_mem_cldnt_si_base<msz, asm, listty>;8072 8073 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",8074 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>;8075 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]",8076 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>;8077 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",8078 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;8079}8080 8081class sve_mem_cldnt_ss_base<bits<2> msz, string asm, RegisterOperand VecList,8082 RegisterOperand gprty>8083: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),8084 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]",8085 "",8086 []>, Sched<[]> {8087 bits<3> Pg;8088 bits<5> Rm;8089 bits<5> Rn;8090 bits<5> Zt;8091 let Inst{31-25} = 0b1010010;8092 let Inst{24-23} = msz;8093 let Inst{22-21} = 0b00;8094 let Inst{20-16} = Rm;8095 let Inst{15-13} = 0b110;8096 let Inst{12-10} = Pg;8097 let Inst{9-5} = Rn;8098 let Inst{4-0} = Zt;8099 8100 let hasSideEffects = 0;8101 let mayLoad = 1;8102}8103 8104multiclass sve_mem_cldnt_ss<bits<2> msz, string asm, RegisterOperand listty,8105 ZPRRegOp zprty, RegisterOperand gprty> {8106 def NAME : sve_mem_cldnt_ss_base<msz, asm, listty, gprty>;8107 8108 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]",8109 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>;8110}8111 8112class sve_mem_ldqr_si<bits<2> sz, string asm, RegisterOperand VecList>8113: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s16:$imm4),8114 asm, "\t$Zt, $Pg/z, [$Rn, $imm4]", "", []>, Sched<[]> {8115 bits<5> Zt;8116 bits<5> Rn;8117 bits<3> Pg;8118 bits<4> imm4;8119 let Inst{31-25} = 0b1010010;8120 let Inst{24-23} = sz;8121 let Inst{22-20} = 0;8122 let Inst{19-16} = imm4;8123 let Inst{15-13} = 0b001;8124 let Inst{12-10} = Pg;8125 let Inst{9-5} = Rn;8126 let Inst{4-0} = Zt;8127 8128 let hasSideEffects = 0;8129 let mayLoad = 1;8130}8131 8132multiclass sve_mem_ldqr_si<bits<2> sz, string asm, RegisterOperand listty,8133 ZPRRegOp zprty> {8134 def NAME : sve_mem_ldqr_si<sz, asm, listty>;8135 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",8136 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;8137 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",8138 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>;8139 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4]",8140 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s16:$imm4), 0>;8141}8142 8143class sve_mem_ldqr_ss<bits<2> sz, string asm, RegisterOperand VecList,8144 RegisterOperand gprty>8145: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),8146 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", "", []>, Sched<[]> {8147 bits<5> Zt;8148 bits<3> Pg;8149 bits<5> Rn;8150 bits<5> Rm;8151 let Inst{31-25} = 0b1010010;8152 let Inst{24-23} = sz;8153 let Inst{22-21} = 0;8154 let Inst{20-16} = Rm;8155 let Inst{15-13} = 0;8156 let Inst{12-10} = Pg;8157 let Inst{9-5} = Rn;8158 let Inst{4-0} = Zt;8159 8160 let hasSideEffects = 0;8161 let mayLoad = 1;8162}8163 8164multiclass sve_mem_ldqr_ss<bits<2> sz, string asm, RegisterOperand listty,8165 ZPRRegOp zprty, RegisterOperand gprty> {8166 def NAME : sve_mem_ldqr_ss<sz, asm, listty, gprty>;8167 8168 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]",8169 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>;8170}8171 8172class sve_mem_ld_dup<bits<2> dtypeh, bits<2> dtypel, string asm,8173 RegisterOperand VecList, Operand immtype>8174: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm6),8175 asm, "\t$Zt, $Pg/z, [$Rn, $imm6]",8176 "",8177 []>, Sched<[]> {8178 bits<3> Pg;8179 bits<5> Rn;8180 bits<5> Zt;8181 bits<6> imm6;8182 let Inst{31-25} = 0b1000010;8183 let Inst{24-23} = dtypeh;8184 let Inst{22} = 1;8185 let Inst{21-16} = imm6;8186 let Inst{15} = 0b1;8187 let Inst{14-13} = dtypel;8188 let Inst{12-10} = Pg;8189 let Inst{9-5} = Rn;8190 let Inst{4-0} = Zt;8191 8192 let hasSideEffects = 0;8193 let mayLoad = 1;8194}8195 8196multiclass sve_mem_ld_dup<bits<2> dtypeh, bits<2> dtypel, string asm,8197 RegisterOperand zlistty, ZPRRegOp zprty, Operand immtype> {8198 def NAME : sve_mem_ld_dup<dtypeh, dtypel, asm, zlistty, immtype>;8199 8200 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",8201 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>;8202 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm6]",8203 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm6), 0>;8204 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",8205 (!cast<Instruction>(NAME) zlistty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;8206}8207 8208class sve_mem_cld_ss_base<bits<4> dtype, bit ff, dag iops, string asm,8209 RegisterOperand VecList>8210: I<(outs VecList:$Zt), iops,8211 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]",8212 "",8213 []>, Sched<[]> {8214 bits<5> Zt;8215 bits<3> Pg;8216 bits<5> Rm;8217 bits<5> Rn;8218 let Inst{31-25} = 0b1010010;8219 let Inst{24-21} = dtype;8220 let Inst{20-16} = Rm;8221 let Inst{15-14} = 0b01;8222 let Inst{13} = ff;8223 let Inst{12-10} = Pg;8224 let Inst{9-5} = Rn;8225 let Inst{4-0} = Zt;8226 8227 let Defs = !if(ff, [FFR], []);8228 let Uses = !if(ff, [FFR], []);8229 let hasSideEffects = ff;8230 let mayLoad = 1;8231}8232 8233multiclass sve_mem_cld_ss<bits<4> dtype, string asm, RegisterOperand listty,8234 ZPRRegOp zprty, RegisterOperand gprty> {8235 def NAME : sve_mem_cld_ss_base<dtype, 0, (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),8236 asm, listty>;8237 8238 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]",8239 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>;8240}8241 8242multiclass sve_mem_cldff_ss<bits<4> dtype, string asm, RegisterOperand listty,8243 ZPRRegOp zprty, RegisterOperand gprty> {8244 def NAME : sve_mem_cld_ss_base<dtype, 1, (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), asm, listty>;8245 8246 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]",8247 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>;8248 8249 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",8250 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 1>;8251 8252 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",8253 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 0>;8254}8255 8256class sve_mem_eld_si<bits<2> sz, bits<3> nregs, RegisterOperand VecList,8257 string asm, Operand immtype>8258: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4),8259 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]",8260 "",8261 []>, Sched<[]> {8262 bits<5> Zt;8263 bits<3> Pg;8264 bits<5> Rn;8265 bits<4> imm4;8266 let Inst{31-25} = 0b1010010;8267 let Inst{24-23} = sz;8268 let Inst{22-21} = nregs{1-0};8269 let Inst{20} = nregs{2};8270 let Inst{19-16} = imm4;8271 let Inst{15-13} = 0b111;8272 let Inst{12-10} = Pg;8273 let Inst{9-5} = Rn;8274 let Inst{4-0} = Zt;8275 8276 let hasSideEffects = 0;8277 let mayLoad = 1;8278}8279 8280multiclass sve_mem_eld_si<bits<2> sz, bits<3> nregs, RegisterOperand VecList,8281 string asm, Operand immtype> {8282 def NAME : sve_mem_eld_si<sz, nregs, VecList, asm, immtype>;8283 8284 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",8285 (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;8286}8287 8288 8289class sve_mem_eld_ss<bits<2> sz, bits<3> nregs, RegisterOperand VecList,8290 string asm, RegisterOperand gprty>8291: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),8292 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]",8293 "",8294 []>, Sched<[]> {8295 bits<3> Pg;8296 bits<5> Rm;8297 bits<5> Rn;8298 bits<5> Zt;8299 let Inst{31-25} = 0b1010010;8300 let Inst{24-23} = sz;8301 let Inst{22-21} = nregs{1-0};8302 let Inst{20-16} = Rm;8303 let Inst{15} = 0b1;8304 let Inst{14} = nregs{2};8305 let Inst{13} = 0b0;8306 let Inst{12-10} = Pg;8307 let Inst{9-5} = Rn;8308 let Inst{4-0} = Zt;8309 8310 let hasSideEffects = 0;8311 let mayLoad = 1;8312}8313 8314//===----------------------------------------------------------------------===//8315// SVE Memory - 32-bit Gather and Unsized Contiguous Group8316//===----------------------------------------------------------------------===//8317 8318// bit xs is '1' if offsets are signed8319// bit scaled is '1' if the offsets are scaled8320class sve_mem_32b_gld_sv<bits<4> opc, bit xs, bit scaled, string asm,8321 RegisterOperand zprext>8322: I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm),8323 asm, "\t$Zt, $Pg/z, [$Rn, $Zm]",8324 "",8325 []>, Sched<[]> {8326 bits<3> Pg;8327 bits<5> Rn;8328 bits<5> Zm;8329 bits<5> Zt;8330 let Inst{31-25} = 0b1000010;8331 let Inst{24-23} = opc{3-2};8332 let Inst{22} = xs;8333 let Inst{21} = scaled;8334 let Inst{20-16} = Zm;8335 let Inst{15} = 0b0;8336 let Inst{14-13} = opc{1-0};8337 let Inst{12-10} = Pg;8338 let Inst{9-5} = Rn;8339 let Inst{4-0} = Zt;8340 8341 8342 let Defs = !if(!eq(opc{0}, 1), [FFR], []);8343 let Uses = !if(!eq(opc{0}, 1), [FFR], []);8344 let hasSideEffects = opc{0};8345 let mayLoad = 1;8346}8347 8348multiclass sve_mem_32b_gld_sv_32_scaled<bits<4> opc, string asm,8349 SDPatternOperator sxtw_op,8350 SDPatternOperator uxtw_op,8351 RegisterOperand sxtw_opnd,8352 RegisterOperand uxtw_opnd,8353 ValueType vt> {8354 def _UXTW_SCALED : sve_mem_32b_gld_sv<opc, 0, 1, asm, uxtw_opnd>;8355 def _SXTW_SCALED : sve_mem_32b_gld_sv<opc, 1, 1, asm, sxtw_opnd>;8356 8357 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",8358 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>;8359 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",8360 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>;8361 8362 def : Pat<(nxv4i32 (uxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$indices), vt)),8363 (!cast<Instruction>(NAME # _UXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>;8364 def : Pat<(nxv4i32 (sxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$indices), vt)),8365 (!cast<Instruction>(NAME # _SXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>;8366}8367 8368multiclass sve_mem_32b_gld_vs_32_unscaled<bits<4> opc, string asm,8369 SDPatternOperator sxtw_op,8370 SDPatternOperator uxtw_op,8371 RegisterOperand sxtw_opnd,8372 RegisterOperand uxtw_opnd,8373 ValueType vt> {8374 def _UXTW : sve_mem_32b_gld_sv<opc, 0, 0, asm, uxtw_opnd>;8375 def _SXTW : sve_mem_32b_gld_sv<opc, 1, 0, asm, sxtw_opnd>;8376 8377 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",8378 (!cast<Instruction>(NAME # _UXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>;8379 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",8380 (!cast<Instruction>(NAME # _SXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>;8381 8382 def : Pat<(nxv4i32 (uxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt)),8383 (!cast<Instruction>(NAME # _UXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;8384 def : Pat<(nxv4i32 (sxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt)),8385 (!cast<Instruction>(NAME # _SXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;8386}8387 8388 8389class sve_mem_32b_gld_vi<bits<4> opc, string asm, Operand imm_ty>8390: I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5),8391 asm, "\t$Zt, $Pg/z, [$Zn, $imm5]",8392 "",8393 []>, Sched<[]> {8394 bits<3> Pg;8395 bits<5> Zn;8396 bits<5> Zt;8397 bits<5> imm5;8398 let Inst{31-25} = 0b1000010;8399 let Inst{24-23} = opc{3-2};8400 let Inst{22-21} = 0b01;8401 let Inst{20-16} = imm5;8402 let Inst{15} = 0b1;8403 let Inst{14-13} = opc{1-0};8404 let Inst{12-10} = Pg;8405 let Inst{9-5} = Zn;8406 let Inst{4-0} = Zt;8407 8408 8409 let Defs = !if(!eq(opc{0}, 1), [FFR], []);8410 let Uses = !if(!eq(opc{0}, 1), [FFR], []);8411 let hasSideEffects = opc{0};8412 let mayLoad = 1;8413}8414 8415multiclass sve_mem_32b_gld_vi_32_ptrs<bits<4> opc, string asm, Operand imm_ty,8416 SDPatternOperator op, ValueType vt> {8417 def _IMM : sve_mem_32b_gld_vi<opc, asm, imm_ty>;8418 8419 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]",8420 (!cast<Instruction>(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 0>;8421 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $imm5]",8422 (!cast<Instruction>(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>;8423 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]",8424 (!cast<Instruction>(NAME # _IMM) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>;8425 8426 def : Pat<(nxv4i32 (op (nxv4i1 PPR:$gp), (nxv4i32 ZPR:$ptrs), imm_ty:$index, vt)),8427 (!cast<Instruction>(NAME # _IMM) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>;8428}8429 8430class sve_mem_prfm_si<bits<2> msz, string asm>8431: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, simm6s1:$imm6),8432 asm, "\t$prfop, $Pg, [$Rn, $imm6, mul vl]",8433 "",8434 []>, Sched<[]> {8435 bits<5> Rn;8436 bits<3> Pg;8437 bits<6> imm6;8438 bits<4> prfop;8439 let Inst{31-22} = 0b1000010111;8440 let Inst{21-16} = imm6;8441 let Inst{15} = 0b0;8442 let Inst{14-13} = msz;8443 let Inst{12-10} = Pg;8444 let Inst{9-5} = Rn;8445 let Inst{4} = 0b0;8446 let Inst{3-0} = prfop;8447 8448 let hasSideEffects = 1;8449}8450 8451multiclass sve_mem_prfm_si<bits<2> msz, string asm> {8452 def NAME : sve_mem_prfm_si<msz, asm>;8453 8454 def : InstAlias<asm # "\t$prfop, $Pg, [$Rn]",8455 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;8456}8457 8458class sve_mem_prfm_ss<bits<3> opc, string asm, RegisterOperand gprty>8459: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),8460 asm, "\t$prfop, $Pg, [$Rn, $Rm]",8461 "",8462 []>, Sched<[]> {8463 bits<5> Rm;8464 bits<5> Rn;8465 bits<3> Pg;8466 bits<4> prfop;8467 let Inst{31-25} = 0b1000010;8468 let Inst{24-23} = opc{2-1};8469 let Inst{22-21} = 0b00;8470 let Inst{20-16} = Rm;8471 let Inst{15} = 0b1;8472 let Inst{14} = opc{0};8473 let Inst{13} = 0b0;8474 let Inst{12-10} = Pg;8475 let Inst{9-5} = Rn;8476 let Inst{4} = 0b0;8477 let Inst{3-0} = prfop;8478 8479 let hasSideEffects = 1;8480}8481 8482class sve_mem_32b_prfm_sv<bits<2> msz, bit xs, string asm,8483 RegisterOperand zprext>8484: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm),8485 asm, "\t$prfop, $Pg, [$Rn, $Zm]",8486 "",8487 []>, Sched<[]> {8488 bits<3> Pg;8489 bits<5> Rn;8490 bits<5> Zm;8491 bits<4> prfop;8492 let Inst{31-23} = 0b100001000;8493 let Inst{22} = xs;8494 let Inst{21} = 0b1;8495 let Inst{20-16} = Zm;8496 let Inst{15} = 0b0;8497 let Inst{14-13} = msz;8498 let Inst{12-10} = Pg;8499 let Inst{9-5} = Rn;8500 let Inst{4} = 0b0;8501 let Inst{3-0} = prfop;8502 8503 let hasSideEffects = 1;8504}8505 8506multiclass sve_mem_32b_prfm_sv_scaled<bits<2> msz, string asm,8507 RegisterOperand sxtw_opnd,8508 RegisterOperand uxtw_opnd,8509 SDPatternOperator op_sxtw,8510 SDPatternOperator op_uxtw> {8511 def _UXTW_SCALED : sve_mem_32b_prfm_sv<msz, 0, asm, uxtw_opnd>;8512 def _SXTW_SCALED : sve_mem_32b_prfm_sv<msz, 1, asm, sxtw_opnd>;8513 8514 def : Pat<(op_uxtw (nxv4i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv4i32 uxtw_opnd:$Zm), (i32 sve_prfop:$prfop)),8515 (!cast<Instruction>(NAME # _UXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>;8516 8517 def : Pat<(op_sxtw (nxv4i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv4i32 sxtw_opnd:$Zm), (i32 sve_prfop:$prfop)),8518 (!cast<Instruction>(NAME # _SXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>;8519}8520 8521class sve_mem_32b_prfm_vi<bits<2> msz, string asm, Operand imm_ty>8522: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5),8523 asm, "\t$prfop, $Pg, [$Zn, $imm5]",8524 "",8525 []>, Sched<[]> {8526 bits<3> Pg;8527 bits<5> Zn;8528 bits<5> imm5;8529 bits<4> prfop;8530 let Inst{31-25} = 0b1000010;8531 let Inst{24-23} = msz;8532 let Inst{22-21} = 0b00;8533 let Inst{20-16} = imm5;8534 let Inst{15-13} = 0b111;8535 let Inst{12-10} = Pg;8536 let Inst{9-5} = Zn;8537 let Inst{4} = 0b0;8538 let Inst{3-0} = prfop;8539 8540 let hasSideEffects = 1;8541}8542 8543multiclass sve_mem_32b_prfm_vi<bits<2> msz, string asm, Operand imm_ty, SDPatternOperator op> {8544 def NAME : sve_mem_32b_prfm_vi<msz, asm, imm_ty>;8545 8546 def : InstAlias<asm # "\t$prfop, $Pg, [$Zn]",8547 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>;8548 8549 def : Pat<(op (nxv4i1 PPR_3b:$Pg), (nxv4i32 ZPR32:$Zn), (i64 imm_ty:$imm), (i32 sve_prfop:$prfop)),8550 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR_3b:$Pg, ZPR32:$Zn, imm_ty:$imm)>;8551}8552 8553class sve_mem_z_fill<string asm>8554: I<(outs ZPRAny:$Zt), (ins GPR64sp:$Rn, simm9:$imm9),8555 asm, "\t$Zt, [$Rn, $imm9, mul vl]",8556 "",8557 []>, Sched<[]> {8558 bits<5> Rn;8559 bits<5> Zt;8560 bits<9> imm9;8561 let Inst{31-22} = 0b1000010110;8562 let Inst{21-16} = imm9{8-3};8563 let Inst{15-13} = 0b010;8564 let Inst{12-10} = imm9{2-0};8565 let Inst{9-5} = Rn;8566 let Inst{4-0} = Zt;8567 8568 let hasSideEffects = 0;8569 let mayLoad = 1;8570}8571 8572multiclass sve_mem_z_fill<string asm> {8573 def NAME : sve_mem_z_fill<asm>;8574 8575 def : InstAlias<asm # "\t$Zt, [$Rn]",8576 (!cast<Instruction>(NAME) ZPRAny:$Zt, GPR64sp:$Rn, 0), 1>;8577}8578 8579class sve_mem_p_fill<string asm>8580: I<(outs PPRorPNRAny:$Pt), (ins GPR64sp:$Rn, simm9:$imm9),8581 asm, "\t$Pt, [$Rn, $imm9, mul vl]",8582 "",8583 []>, Sched<[]> {8584 bits<4> Pt;8585 bits<5> Rn;8586 bits<9> imm9;8587 let Inst{31-22} = 0b1000010110;8588 let Inst{21-16} = imm9{8-3};8589 let Inst{15-13} = 0b000;8590 let Inst{12-10} = imm9{2-0};8591 let Inst{9-5} = Rn;8592 let Inst{4} = 0b0;8593 let Inst{3-0} = Pt;8594 8595 let hasSideEffects = 0;8596 let mayLoad = 1;8597}8598 8599multiclass sve_mem_p_fill<string asm> {8600 def NAME : sve_mem_p_fill<asm>;8601 8602 def : InstAlias<asm # "\t$Pt, [$Rn]",8603 (!cast<Instruction>(NAME) PPRorPNRAny:$Pt, GPR64sp:$Rn, 0), 1>;8604}8605 8606class sve2_mem_gldnt_vs_base<bits<5> opc, dag iops, string asm,8607 RegisterOperand VecList>8608: I<(outs VecList:$Zt), iops,8609 asm, "\t$Zt, $Pg/z, [$Zn, $Rm]",8610 "",8611 []>, Sched<[]> {8612 bits<3> Pg;8613 bits<5> Rm;8614 bits<5> Zn;8615 bits<5> Zt;8616 let Inst{31} = 0b1;8617 let Inst{30} = opc{4};8618 let Inst{29-25} = 0b00010;8619 let Inst{24-23} = opc{3-2};8620 let Inst{22-21} = 0b00;8621 let Inst{20-16} = Rm;8622 let Inst{15} = 0b1;8623 let Inst{14-13} = opc{1-0};8624 let Inst{12-10} = Pg;8625 let Inst{9-5} = Zn;8626 let Inst{4-0} = Zt;8627 8628 let hasSideEffects = 0;8629 let mayLoad = 1;8630}8631 8632multiclass sve2_mem_gldnt_vs_32_ptrs<bits<5> opc, string asm,8633 SDPatternOperator op,8634 ValueType vt> {8635 def NAME : sve2_mem_gldnt_vs_base<opc, (ins PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), asm, Z_s>;8636 8637 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $Rm]",8638 (!cast<Instruction>(NAME) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>;8639 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]",8640 (!cast<Instruction>(NAME) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>;8641 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]",8642 (!cast<Instruction>(NAME) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>;8643 8644 def : Pat <(nxv4i32 (op (nxv4i1 PPR3bAny:$Pg), (nxv4i32 ZPR32:$Zd), (i64 GPR64:$Rm), vt)),8645 (!cast<Instruction>(NAME) PPR3bAny:$Pg, ZPR32:$Zd, GPR64:$Rm)>;8646}8647 8648multiclass sve2_mem_gldnt_vs_64_ptrs<bits<5> opc, string asm,8649 SDPatternOperator op,8650 ValueType vt> {8651 def NAME : sve2_mem_gldnt_vs_base<opc, (ins PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), asm, Z_d>;8652 8653 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $Rm]",8654 (!cast<Instruction>(NAME) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>;8655 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]",8656 (!cast<Instruction>(NAME) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>;8657 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]",8658 (!cast<Instruction>(NAME) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>;8659 8660 def : Pat <(nxv2i64 (op (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zd), (i64 GPR64:$Rm), vt)),8661 (!cast<Instruction>(NAME) PPR3bAny:$Pg, ZPR64:$Zd, GPR64:$Rm)>;8662}8663 8664//===----------------------------------------------------------------------===//8665// SVE Memory - 64-bit Gather Group8666//===----------------------------------------------------------------------===//8667 8668// bit xs is '1' if offsets are signed8669// bit scaled is '1' if the offsets are scaled8670// bit lsl is '0' if the offsets are extended (uxtw/sxtw), '1' if shifted (lsl)8671class sve_mem_64b_gld_sv<bits<4> opc, bit xs, bit scaled, bit lsl, string asm,8672 RegisterOperand zprext>8673: I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm),8674 asm, "\t$Zt, $Pg/z, [$Rn, $Zm]",8675 "",8676 []>, Sched<[]> {8677 bits<3> Pg;8678 bits<5> Rn;8679 bits<5> Zm;8680 bits<5> Zt;8681 let Inst{31-25} = 0b1100010;8682 let Inst{24-23} = opc{3-2};8683 let Inst{22} = xs;8684 let Inst{21} = scaled;8685 let Inst{20-16} = Zm;8686 let Inst{15} = lsl;8687 let Inst{14-13} = opc{1-0};8688 let Inst{12-10} = Pg;8689 let Inst{9-5} = Rn;8690 let Inst{4-0} = Zt;8691 8692 8693 let Defs = !if(!eq(opc{0}, 1), [FFR], []);8694 let Uses = !if(!eq(opc{0}, 1), [FFR], []);8695 let hasSideEffects = opc{0};8696 let mayLoad = 1;8697}8698 8699multiclass sve_mem_64b_gld_sv_32_scaled<bits<4> opc, string asm,8700 SDPatternOperator sxtw_op,8701 SDPatternOperator uxtw_op,8702 RegisterOperand sxtw_opnd,8703 RegisterOperand uxtw_opnd,8704 ValueType vt> {8705 def _UXTW_SCALED : sve_mem_64b_gld_sv<opc, 0, 1, 0, asm, uxtw_opnd>;8706 def _SXTW_SCALED : sve_mem_64b_gld_sv<opc, 1, 1, 0, asm, sxtw_opnd>;8707 8708 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",8709 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>;8710 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",8711 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>;8712 8713 def : Pat<(nxv2i64 (uxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)),8714 (!cast<Instruction>(NAME # _UXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>;8715 def : Pat<(nxv2i64 (sxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)),8716 (!cast<Instruction>(NAME # _SXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>;8717}8718 8719multiclass sve_mem_64b_gld_vs_32_unscaled<bits<4> opc, string asm,8720 SDPatternOperator sxtw_op,8721 SDPatternOperator uxtw_op,8722 RegisterOperand sxtw_opnd,8723 RegisterOperand uxtw_opnd,8724 ValueType vt> {8725 def _UXTW : sve_mem_64b_gld_sv<opc, 0, 0, 0, asm, uxtw_opnd>;8726 def _SXTW : sve_mem_64b_gld_sv<opc, 1, 0, 0, asm, sxtw_opnd>;8727 8728 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",8729 (!cast<Instruction>(NAME # _UXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>;8730 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",8731 (!cast<Instruction>(NAME # _SXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>;8732 8733 def : Pat<(nxv2i64 (uxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)),8734 (!cast<Instruction>(NAME # _UXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;8735 def : Pat<(nxv2i64 (sxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)),8736 (!cast<Instruction>(NAME # _SXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;8737}8738 8739multiclass sve_mem_64b_gld_sv2_64_scaled<bits<4> opc, string asm,8740 SDPatternOperator op,8741 RegisterOperand zprext, ValueType vt> {8742 def _SCALED : sve_mem_64b_gld_sv<opc, 1, 1, 1, asm, zprext>;8743 8744 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",8745 (!cast<Instruction>(NAME # _SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 0>;8746 8747 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)),8748 (!cast<Instruction>(NAME # _SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>;8749}8750 8751multiclass sve_mem_64b_gld_vs2_64_unscaled<bits<4> opc, string asm,8752 SDPatternOperator op, ValueType vt> {8753 def NAME : sve_mem_64b_gld_sv<opc, 1, 0, 1, asm, ZPR64ExtLSL8>;8754 8755 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",8756 (!cast<Instruction>(NAME) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), 0>;8757 8758 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)),8759 (!cast<Instruction>(NAME) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;8760}8761 8762class sve_mem_64b_gld_vi<bits<4> opc, string asm, Operand imm_ty>8763: I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5),8764 asm, "\t$Zt, $Pg/z, [$Zn, $imm5]",8765 "",8766 []>, Sched<[]> {8767 bits<3> Pg;8768 bits<5> Zn;8769 bits<5> Zt;8770 bits<5> imm5;8771 let Inst{31-25} = 0b1100010;8772 let Inst{24-23} = opc{3-2};8773 let Inst{22-21} = 0b01;8774 let Inst{20-16} = imm5;8775 let Inst{15} = 0b1;8776 let Inst{14-13} = opc{1-0};8777 let Inst{12-10} = Pg;8778 let Inst{9-5} = Zn;8779 let Inst{4-0} = Zt;8780 8781 let Defs = !if(!eq(opc{0}, 1), [FFR], []);8782 let Uses = !if(!eq(opc{0}, 1), [FFR], []);8783 let hasSideEffects = opc{0};8784 let mayLoad = 1;8785}8786 8787multiclass sve_mem_64b_gld_vi_64_ptrs<bits<4> opc, string asm, Operand imm_ty,8788 SDPatternOperator op, ValueType vt> {8789 def _IMM : sve_mem_64b_gld_vi<opc, asm, imm_ty>;8790 8791 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]",8792 (!cast<Instruction>(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 0>;8793 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $imm5]",8794 (!cast<Instruction>(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>;8795 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]",8796 (!cast<Instruction>(NAME # _IMM) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>;8797 8798 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), (nxv2i64 ZPR:$ptrs), imm_ty:$index, vt)),8799 (!cast<Instruction>(NAME # _IMM) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>;8800}8801 8802// bit lsl is '0' if the offsets are extended (uxtw/sxtw), '1' if shifted (lsl)8803class sve_mem_64b_prfm_sv<bits<2> msz, bit xs, bit lsl, string asm,8804 RegisterOperand zprext>8805: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm),8806 asm, "\t$prfop, $Pg, [$Rn, $Zm]",8807 "",8808 []>, Sched<[]> {8809 bits<3> Pg;8810 bits<5> Rn;8811 bits<5> Zm;8812 bits<4> prfop;8813 let Inst{31-23} = 0b110001000;8814 let Inst{22} = xs;8815 let Inst{21} = 0b1;8816 let Inst{20-16} = Zm;8817 let Inst{15} = lsl;8818 let Inst{14-13} = msz;8819 let Inst{12-10} = Pg;8820 let Inst{9-5} = Rn;8821 let Inst{4} = 0b0;8822 let Inst{3-0} = prfop;8823 8824 let hasSideEffects = 1;8825}8826 8827multiclass sve_mem_64b_prfm_sv_ext_scaled<bits<2> msz, string asm,8828 RegisterOperand sxtw_opnd,8829 RegisterOperand uxtw_opnd,8830 SDPatternOperator op_sxtw,8831 SDPatternOperator op_uxtw> {8832 def _UXTW_SCALED : sve_mem_64b_prfm_sv<msz, 0, 0, asm, uxtw_opnd>;8833 def _SXTW_SCALED : sve_mem_64b_prfm_sv<msz, 1, 0, asm, sxtw_opnd>;8834 8835 def : Pat<(op_uxtw (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 uxtw_opnd:$Zm), (i32 sve_prfop:$prfop)),8836 (!cast<Instruction>(NAME # _UXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>;8837 8838 def : Pat<(op_sxtw (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 sxtw_opnd:$Zm), (i32 sve_prfop:$prfop)),8839 (!cast<Instruction>(NAME # _SXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>;8840 8841}8842 8843multiclass sve_mem_64b_prfm_sv_lsl_scaled<bits<2> msz, string asm,8844 RegisterOperand zprext, SDPatternOperator frag> {8845 def NAME : sve_mem_64b_prfm_sv<msz, 1, 1, asm, zprext>;8846 8847 def : Pat<(frag (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 zprext:$Zm), (i32 sve_prfop:$prfop)),8848 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm)>;8849 8850}8851 8852class sve_mem_64b_prfm_vi<bits<2> msz, string asm, Operand imm_ty>8853: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5),8854 asm, "\t$prfop, $Pg, [$Zn, $imm5]",8855 "",8856 []>, Sched<[]> {8857 bits<3> Pg;8858 bits<5> Zn;8859 bits<5> imm5;8860 bits<4> prfop;8861 let Inst{31-25} = 0b1100010;8862 let Inst{24-23} = msz;8863 let Inst{22-21} = 0b00;8864 let Inst{20-16} = imm5;8865 let Inst{15-13} = 0b111;8866 let Inst{12-10} = Pg;8867 let Inst{9-5} = Zn;8868 let Inst{4} = 0b0;8869 let Inst{3-0} = prfop;8870 8871 let hasSideEffects = 1;8872}8873 8874multiclass sve_mem_64b_prfm_vi<bits<2> msz, string asm, Operand imm_ty, SDPatternOperator op> {8875 def NAME : sve_mem_64b_prfm_vi<msz, asm, imm_ty>;8876 8877 def : InstAlias<asm # "\t$prfop, $Pg, [$Zn]",8878 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>;8879 8880 def : Pat<(op (nxv2i1 PPR_3b:$Pg), (nxv2i64 ZPR32:$Zn), (i64 imm_ty:$imm), (i32 sve_prfop:$prfop)),8881 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR_3b:$Pg, ZPR32:$Zn, imm_ty:$imm)>;8882}8883 8884//===----------------------------------------------------------------------===//8885// SVE Compute Vector Address Group8886//===----------------------------------------------------------------------===//8887 8888class sve_int_bin_cons_misc_0_a<bits<2> opc, bits<2> msz, string asm,8889 ZPRRegOp zprty, RegisterOperand zprext>8890: I<(outs zprty:$Zd), (ins zprty:$Zn, zprext:$Zm),8891 asm, "\t$Zd, [$Zn, $Zm]",8892 "",8893 []>, Sched<[]> {8894 bits<5> Zd;8895 bits<5> Zn;8896 bits<5> Zm;8897 let Inst{31-24} = 0b00000100;8898 let Inst{23-22} = opc;8899 let Inst{21} = 0b1;8900 let Inst{20-16} = Zm;8901 let Inst{15-12} = 0b1010;8902 let Inst{11-10} = msz;8903 let Inst{9-5} = Zn;8904 let Inst{4-0} = Zd;8905 8906 let hasSideEffects = 0;8907}8908 8909multiclass sve_int_bin_cons_misc_0_a_uxtw<bits<2> opc, string asm> {8910 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtUXTW8>;8911 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtUXTW16>;8912 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtUXTW32>;8913 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtUXTW64>;8914}8915 8916multiclass sve_int_bin_cons_misc_0_a_sxtw<bits<2> opc, string asm> {8917 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtSXTW8>;8918 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtSXTW16>;8919 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtSXTW32>;8920 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtSXTW64>;8921}8922 8923multiclass sve_int_bin_cons_misc_0_a_32_lsl<bits<2> opc, string asm> {8924 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR32, ZPR32ExtLSL8>;8925 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR32, ZPR32ExtLSL16>;8926 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR32, ZPR32ExtLSL32>;8927 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR32, ZPR32ExtLSL64>;8928}8929 8930multiclass sve_int_bin_cons_misc_0_a_64_lsl<bits<2> opc, string asm> {8931 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtLSL8>;8932 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtLSL16>;8933 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtLSL32>;8934 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtLSL64>;8935}8936 8937//===----------------------------------------------------------------------===//8938// SVE Integer Misc - Unpredicated Group8939//===----------------------------------------------------------------------===//8940 8941class sve_int_bin_cons_misc_0_b<bits<2> sz, string asm, ZPRRegOp zprty>8942: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm),8943 asm, "\t$Zd, $Zn, $Zm",8944 "",8945 []>, Sched<[]> {8946 bits<5> Zd;8947 bits<5> Zm;8948 bits<5> Zn;8949 let Inst{31-24} = 0b00000100;8950 let Inst{23-22} = sz;8951 let Inst{21} = 0b1;8952 let Inst{20-16} = Zm;8953 let Inst{15-10} = 0b101100;8954 let Inst{9-5} = Zn;8955 let Inst{4-0} = Zd;8956 8957 let hasSideEffects = 0;8958}8959 8960multiclass sve_int_bin_cons_misc_0_b<string asm, SDPatternOperator op> {8961 def _H : sve_int_bin_cons_misc_0_b<0b01, asm, ZPR16>;8962 def _S : sve_int_bin_cons_misc_0_b<0b10, asm, ZPR32>;8963 def _D : sve_int_bin_cons_misc_0_b<0b11, asm, ZPR64>;8964 8965 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>;8966 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>;8967 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>;8968}8969 8970class sve_int_bin_cons_misc_0_c<bits<8> opc, string asm, ZPRRegOp zprty>8971: I<(outs zprty:$Zd), (ins zprty:$Zn),8972 asm, "\t$Zd, $Zn",8973 "",8974 []>, Sched<[]> {8975 bits<5> Zd;8976 bits<5> Zn;8977 let Inst{31-24} = 0b00000100;8978 let Inst{23-22} = opc{7-6};8979 let Inst{21} = 0b1;8980 let Inst{20-16} = opc{5-1};8981 let Inst{15-11} = 0b10111;8982 let Inst{10} = opc{0};8983 let Inst{9-5} = Zn;8984 let Inst{4-0} = Zd;8985 8986 let hasSideEffects = 0;8987}8988 8989multiclass sve_int_bin_cons_misc_0_c_fexpa<string asm, SDPatternOperator op> {8990 def _H : sve_int_bin_cons_misc_0_c<0b01000000, asm, ZPR16>;8991 def _S : sve_int_bin_cons_misc_0_c<0b10000000, asm, ZPR32>;8992 def _D : sve_int_bin_cons_misc_0_c<0b11000000, asm, ZPR64>;8993 8994 def : SVE_1_Op_Pat<nxv8f16, op, nxv8i16, !cast<Instruction>(NAME # _H)>;8995 def : SVE_1_Op_Pat<nxv4f32, op, nxv4i32, !cast<Instruction>(NAME # _S)>;8996 def : SVE_1_Op_Pat<nxv2f64, op, nxv2i64, !cast<Instruction>(NAME # _D)>;8997}8998 8999//===----------------------------------------------------------------------===//9000// SVE Integer Reduction Group9001//===----------------------------------------------------------------------===//9002 9003class sve_int_reduce<bits<2> sz8_32, bits<2> fmt, bits<3> opc, string asm,9004 ZPRRegOp zprty, FPRasZPROperand dstOpType>9005: I<(outs dstOpType:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn),9006 asm, "\t$Vd, $Pg, $Zn",9007 "",9008 []>, Sched<[]> {9009 bits<3> Pg;9010 bits<5> Vd;9011 bits<5> Zn;9012 let Inst{31-24} = 0b00000100;9013 let Inst{23-22} = sz8_32;9014 let Inst{21} = 0b0;9015 let Inst{20-19} = fmt;9016 let Inst{18-16} = opc;9017 let Inst{15-13} = 0b001;9018 let Inst{12-10} = Pg;9019 let Inst{9-5} = Zn;9020 let Inst{4-0} = Vd;9021 9022 let hasSideEffects = 0;9023}9024 9025multiclass sve_int_reduce_0_saddv<bits<3> opc, string asm,9026 SDPatternOperator op> {9027 def _B : sve_int_reduce<0b00, 0b00, opc, asm, ZPR8, FPR64asZPR>;9028 def _H : sve_int_reduce<0b01, 0b00, opc, asm, ZPR16, FPR64asZPR>;9029 def _S : sve_int_reduce<0b10, 0b00, opc, asm, ZPR32, FPR64asZPR>;9030 9031 def : SVE_2_Op_Pat<nxv2i64, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;9032 def : SVE_2_Op_Pat<nxv2i64, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>;9033 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;9034}9035 9036multiclass sve_int_reduce_0_uaddv<bits<3> opc, string asm,9037 SDPatternOperator op> {9038 def _B : sve_int_reduce<0b00, 0b00, opc, asm, ZPR8, FPR64asZPR>;9039 def _H : sve_int_reduce<0b01, 0b00, opc, asm, ZPR16, FPR64asZPR>;9040 def _S : sve_int_reduce<0b10, 0b00, opc, asm, ZPR32, FPR64asZPR>;9041 def _D : sve_int_reduce<0b11, 0b00, opc, asm, ZPR64, FPR64asZPR>;9042 9043 def : SVE_2_Op_Pat<nxv2i64, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;9044 def : SVE_2_Op_Pat<nxv2i64, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>;9045 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;9046 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;9047}9048 9049multiclass sve_int_reduce_1<bits<3> opc, string asm,9050 SDPatternOperator op> {9051 def _B : sve_int_reduce<0b00, 0b01, opc, asm, ZPR8, FPR8asZPR>;9052 def _H : sve_int_reduce<0b01, 0b01, opc, asm, ZPR16, FPR16asZPR>;9053 def _S : sve_int_reduce<0b10, 0b01, opc, asm, ZPR32, FPR32asZPR>;9054 def _D : sve_int_reduce<0b11, 0b01, opc, asm, ZPR64, FPR64asZPR>;9055 9056 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;9057 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>;9058 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;9059 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;9060}9061 9062multiclass sve_int_reduce_2<bits<3> opc, string asm,9063 SDPatternOperator op> {9064 def _B : sve_int_reduce<0b00, 0b11, opc, asm, ZPR8, FPR8asZPR>;9065 def _H : sve_int_reduce<0b01, 0b11, opc, asm, ZPR16, FPR16asZPR>;9066 def _S : sve_int_reduce<0b10, 0b11, opc, asm, ZPR32, FPR32asZPR>;9067 def _D : sve_int_reduce<0b11, 0b11, opc, asm, ZPR64, FPR64asZPR>;9068 9069 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;9070 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>;9071 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;9072 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;9073}9074 9075class sve_int_movprfx_pred<bits<2> sz8_32, bits<3> opc, string asm,9076 ZPRRegOp zprty, string pg_suffix, dag iops>9077: I<(outs zprty:$Zd), iops,9078 asm, "\t$Zd, $Pg"#pg_suffix#", $Zn",9079 "",9080 []>, Sched<[]> {9081 bits<3> Pg;9082 bits<5> Zd;9083 bits<5> Zn;9084 let Inst{31-24} = 0b00000100;9085 let Inst{23-22} = sz8_32;9086 let Inst{21-19} = 0b010;9087 let Inst{18-16} = opc;9088 let Inst{15-13} = 0b001;9089 let Inst{12-10} = Pg;9090 let Inst{9-5} = Zn;9091 let Inst{4-0} = Zd;9092 9093 let ElementSize = zprty.ElementSize;9094 let hasSideEffects = 0;9095}9096 9097multiclass sve_int_movprfx_pred_merge<bits<3> opc, string asm> {9098let Constraints = "$Zd = $_Zd" in {9099 def _B : sve_int_movprfx_pred<0b00, opc, asm, ZPR8, "/m",9100 (ins ZPR8:$_Zd, PPR3bAny:$Pg, ZPR8:$Zn)>;9101 def _H : sve_int_movprfx_pred<0b01, opc, asm, ZPR16, "/m",9102 (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR16:$Zn)>;9103 def _S : sve_int_movprfx_pred<0b10, opc, asm, ZPR32, "/m",9104 (ins ZPR32:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn)>;9105 def _D : sve_int_movprfx_pred<0b11, opc, asm, ZPR64, "/m",9106 (ins ZPR64:$_Zd, PPR3bAny:$Pg, ZPR64:$Zn)>;9107}9108}9109 9110multiclass sve_int_movprfx_pred_zero<bits<3> opc, string asm> {9111 def _B : sve_int_movprfx_pred<0b00, opc, asm, ZPR8, "/z",9112 (ins PPR3bAny:$Pg, ZPR8:$Zn)>;9113 def _H : sve_int_movprfx_pred<0b01, opc, asm, ZPR16, "/z",9114 (ins PPR3bAny:$Pg, ZPR16:$Zn)>;9115 def _S : sve_int_movprfx_pred<0b10, opc, asm, ZPR32, "/z",9116 (ins PPR3bAny:$Pg, ZPR32:$Zn)>;9117 def _D : sve_int_movprfx_pred<0b11, opc, asm, ZPR64, "/z",9118 (ins PPR3bAny:$Pg, ZPR64:$Zn)>;9119}9120 9121//===----------------------------------------------------------------------===//9122// SVE Propagate Break Group9123//===----------------------------------------------------------------------===//9124 9125class sve_int_brkp<bits<2> opc, string asm>9126: I<(outs PPR8:$Pd), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$Pm),9127 asm, "\t$Pd, $Pg/z, $Pn, $Pm",9128 "",9129 []>, Sched<[]> {9130 bits<4> Pd;9131 bits<4> Pg;9132 bits<4> Pm;9133 bits<4> Pn;9134 let Inst{31-24} = 0b00100101;9135 let Inst{23} = 0b0;9136 let Inst{22} = opc{1};9137 let Inst{21-20} = 0b00;9138 let Inst{19-16} = Pm;9139 let Inst{15-14} = 0b11;9140 let Inst{13-10} = Pg;9141 let Inst{9} = 0b0;9142 let Inst{8-5} = Pn;9143 let Inst{4} = opc{0};9144 let Inst{3-0} = Pd;9145 9146 let Defs = !if(!eq (opc{1}, 1), [NZCV], []);9147 let hasSideEffects = 0;9148}9149 9150multiclass sve_int_brkp<bits<2> opc, string asm, SDPatternOperator op> {9151 def NAME : sve_int_brkp<opc, asm>;9152 9153 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>;9154}9155 9156 9157//===----------------------------------------------------------------------===//9158// SVE Partition Break Group9159//===----------------------------------------------------------------------===//9160 9161class sve_int_brkn<bit S, string asm>9162: I<(outs PPR8:$Pdm), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$_Pdm),9163 asm, "\t$Pdm, $Pg/z, $Pn, $_Pdm",9164 "",9165 []>, Sched<[]> {9166 bits<4> Pdm;9167 bits<4> Pg;9168 bits<4> Pn;9169 let Inst{31-23} = 0b001001010;9170 let Inst{22} = S;9171 let Inst{21-14} = 0b01100001;9172 let Inst{13-10} = Pg;9173 let Inst{9} = 0b0;9174 let Inst{8-5} = Pn;9175 let Inst{4} = 0b0;9176 let Inst{3-0} = Pdm;9177 9178 let Constraints = "$Pdm = $_Pdm";9179 let Defs = !if(S, [NZCV], []);9180 let ElementSize = ElementSizeB;9181 let hasSideEffects = 0;9182}9183 9184multiclass sve_int_brkn<bits<1> opc, string asm, SDPatternOperator op> {9185 def NAME : sve_int_brkn<opc, asm>;9186 9187 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>;9188}9189 9190class sve_int_break<bits<3> opc, string asm, string suffix, dag iops>9191: I<(outs PPR8:$Pd), iops,9192 asm, "\t$Pd, $Pg"#suffix#", $Pn",9193 "",9194 []>, Sched<[]> {9195 bits<4> Pd;9196 bits<4> Pg;9197 bits<4> Pn;9198 let Inst{31-24} = 0b00100101;9199 let Inst{23-22} = opc{2-1};9200 let Inst{21-14} = 0b01000001;9201 let Inst{13-10} = Pg;9202 let Inst{9} = 0b0;9203 let Inst{8-5} = Pn;9204 let Inst{4} = opc{0};9205 let Inst{3-0} = Pd;9206 9207 let Constraints = !if(!eq (opc{0}, 1), "$Pd = $_Pd", "");9208 let Defs = !if(!eq (opc{1}, 1), [NZCV], []);9209 let hasSideEffects = 0;9210}9211 9212multiclass sve_int_break_m<bits<3> opc, string asm, SDPatternOperator op> {9213 def NAME : sve_int_break<opc, asm, "/m", (ins PPR8:$_Pd, PPRAny:$Pg, PPR8:$Pn)>;9214 9215 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>;9216}9217 9218multiclass sve_int_break_z<bits<3> opc, string asm, SDPatternOperator op> {9219 def NAME : sve_int_break<opc, asm, "/z", (ins PPRAny:$Pg, PPR8:$Pn)>;9220 9221 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>;9222}9223 9224//===----------------------------------------------------------------------===//9225// SVE2 String Processing Group9226//===----------------------------------------------------------------------===//9227 9228class sve2_char_match<bit sz, bit opc, string asm,9229 PPRRegOp pprty, ZPRRegOp zprty>9230: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm),9231 asm, "\t$Pd, $Pg/z, $Zn, $Zm",9232 "",9233 []>, Sched<[]> {9234 bits<4> Pd;9235 bits<3> Pg;9236 bits<5> Zm;9237 bits<5> Zn;9238 let Inst{31-23} = 0b010001010;9239 let Inst{22} = sz;9240 let Inst{21} = 0b1;9241 let Inst{20-16} = Zm;9242 let Inst{15-13} = 0b100;9243 let Inst{12-10} = Pg;9244 let Inst{9-5} = Zn;9245 let Inst{4} = opc;9246 let Inst{3-0} = Pd;9247 9248 let Defs = [NZCV];9249 let ElementSize = pprty.ElementSize;9250 let hasSideEffects = 0;9251 let isPTestLike = 1;9252}9253 9254multiclass sve2_char_match<bit opc, string asm, SDPatternOperator op> {9255 def _B : sve2_char_match<0b0, opc, asm, PPR8, ZPR8>;9256 def _H : sve2_char_match<0b1, opc, asm, PPR16, ZPR16>;9257 9258 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;9259 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;9260}9261 9262//===----------------------------------------------------------------------===//9263// SVE2 Histogram Computation - Segment Group9264//===----------------------------------------------------------------------===//9265 9266class sve2_hist_gen_segment<string asm, SDPatternOperator op>9267: I<(outs ZPR8:$Zd), (ins ZPR8:$Zn, ZPR8:$Zm),9268 asm, "\t$Zd, $Zn, $Zm",9269 "",9270 [(set nxv16i8:$Zd, (op nxv16i8:$Zn, nxv16i8:$Zm))]>, Sched<[]> {9271 bits<5> Zd;9272 bits<5> Zn;9273 bits<5> Zm;9274 let Inst{31-21} = 0b01000101001;9275 let Inst{20-16} = Zm;9276 let Inst{15-10} = 0b101000;9277 let Inst{9-5} = Zn;9278 let Inst{4-0} = Zd;9279 9280 let hasSideEffects = 0;9281}9282 9283//===----------------------------------------------------------------------===//9284// SVE2 Histogram Computation - Vector Group9285//===----------------------------------------------------------------------===//9286 9287class sve2_hist_gen_vector<bit sz, string asm, ZPRRegOp zprty>9288: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm),9289 asm, "\t$Zd, $Pg/z, $Zn, $Zm",9290 "",9291 []>, Sched<[]> {9292 bits<5> Zd;9293 bits<5> Zn;9294 bits<3> Pg;9295 bits<5> Zm;9296 let Inst{31-23} = 0b010001011;9297 let Inst{22} = sz;9298 let Inst{21} = 0b1;9299 let Inst{20-16} = Zm;9300 let Inst{15-13} = 0b110;9301 let Inst{12-10} = Pg;9302 let Inst{9-5} = Zn;9303 let Inst{4-0} = Zd;9304 9305 let hasSideEffects = 0;9306}9307 9308multiclass sve2_hist_gen_vector<string asm, SDPatternOperator op> {9309 def _S : sve2_hist_gen_vector<0b0, asm, ZPR32>;9310 def _D : sve2_hist_gen_vector<0b1, asm, ZPR64>;9311 9312 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;9313 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;9314}9315 9316//===----------------------------------------------------------------------===//9317// SVE2 Crypto Extensions Group9318//===----------------------------------------------------------------------===//9319 9320class sve2_crypto_cons_bin_op<bit opc, string asm, ZPRRegOp zprty>9321: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm),9322 asm, "\t$Zd, $Zn, $Zm",9323 "",9324 []>, Sched<[]> {9325 bits<5> Zd;9326 bits<5> Zn;9327 bits<5> Zm;9328 let Inst{31-21} = 0b01000101001;9329 let Inst{20-16} = Zm;9330 let Inst{15-11} = 0b11110;9331 let Inst{10} = opc;9332 let Inst{9-5} = Zn;9333 let Inst{4-0} = Zd;9334 9335 let hasSideEffects = 0;9336}9337 9338multiclass sve2_crypto_cons_bin_op<bit opc, string asm, ZPRRegOp zprty,9339 SDPatternOperator op, ValueType vt> {9340 def NAME : sve2_crypto_cons_bin_op<opc, asm, zprty>;9341 def : SVE_2_Op_Pat<vt, op, vt, vt, !cast<Instruction>(NAME)>;9342}9343 9344class sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty,9345 SDPatternOperator op, ValueType vt>9346: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm),9347 asm, "\t$Zdn, $_Zdn, $Zm",9348 "",9349 [(set (vt zprty:$Zdn), (op (vt zprty:$_Zdn), (vt zprty:$Zm)))]>, Sched<[]> {9350 bits<5> Zdn;9351 bits<5> Zm;9352 let Inst{31-17} = 0b010001010010001;9353 let Inst{16} = opc{1};9354 let Inst{15-11} = 0b11100;9355 let Inst{10} = opc{0};9356 let Inst{9-5} = Zm;9357 let Inst{4-0} = Zdn;9358 9359 let Constraints = "$Zdn = $_Zdn";9360 let hasSideEffects = 0;9361}9362 9363class sve2_crypto_unary_op<bit opc, string asm, ZPRRegOp zprty>9364: I<(outs zprty:$Zdn), (ins zprty:$_Zdn),9365 asm, "\t$Zdn, $_Zdn",9366 "",9367 []>, Sched<[]> {9368 bits<5> Zdn;9369 let Inst{31-11} = 0b010001010010000011100;9370 let Inst{10} = opc;9371 let Inst{9-5} = 0b00000;9372 let Inst{4-0} = Zdn;9373 9374 let Constraints = "$Zdn = $_Zdn";9375 let hasSideEffects = 0;9376}9377 9378multiclass sve2_crypto_unary_op<bit opc, string asm, SDPatternOperator op> {9379 def NAME : sve2_crypto_unary_op<opc, asm, ZPR8>;9380 def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME)>;9381}9382 9383class sve_crypto_binary_multi2<bits<3> opc, string asm>9384: I<(outs ZZ_b_mul_r:$Zdn),9385 (ins ZZ_b_mul_r:$_Zdn, ZPR128:$Zm, VectorIndexS32b_timm:$imm2),9386 asm,9387 "\t$Zdn, $_Zdn, $Zm$imm2",9388 "",9389 []>, Sched<[]> {9390 bits<5> Zm;9391 bits<4> Zdn;9392 bits<2> imm2;9393 let Inst{31-21} = 0b01000101001;9394 let Inst{20-19} = imm2;9395 let Inst{18-17} = 0b01;9396 let Inst{16} = opc{2};9397 let Inst{15-11} = 0b11101;9398 let Inst{10} = opc{1};9399 let Inst{9-5} = Zm;9400 let Inst{4-1} = Zdn;9401 let Inst{0} = opc{0};9402 9403 let Constraints = "$Zdn = $_Zdn";9404 let hasSideEffects = 0;9405}9406 9407class sve_crypto_binary_multi4<bits<4> opc, string asm>9408: I<(outs ZZZZ_b_mul_r:$Zdn),9409 (ins ZZZZ_b_mul_r:$_Zdn, ZPR128:$Zm, VectorIndexS32b_timm:$imm2),9410 asm,9411 "\t$Zdn, $_Zdn, $Zm$imm2",9412 "",9413 []>, Sched<[]> {9414 bits<5> Zm;9415 bits<3> Zdn;9416 bits<2> imm2;9417 let Inst{31-21} = 0b01000101001;9418 let Inst{20-19} = imm2;9419 let Inst{18-17} = 0b11;9420 let Inst{16} = opc{3};9421 let Inst{15-11} = 0b11101;9422 let Inst{10} = opc{2};9423 let Inst{9-5} = Zm;9424 let Inst{4-2} = Zdn;9425 let Inst{1-0} = opc{1-0};9426 9427 let Constraints = "$Zdn = $_Zdn";9428 let hasSideEffects = 0;9429}9430 9431class sve_crypto_pmlal_multi<string asm>9432: I<(outs ZZ_q_mul_r:$Zda),9433 (ins ZZ_q_mul_r:$_Zda, ZPR64:$Zn, ZPR64:$Zm),9434 asm,9435 "\t$Zda, $Zn, $Zm",9436 "",9437 []>, Sched<[]> {9438 bits<5> Zm;9439 bits<5> Zn;9440 bits<4> Zda;9441 let Inst{31-21} = 0b01000101001;9442 let Inst{20-16} = Zm;9443 let Inst{15-10} = 0b111111;9444 let Inst{9-5} = Zn;9445 let Inst{4-1} = Zda;9446 let Inst{0} = 0b0;9447 9448 let Constraints = "$Zda = $_Zda";9449 let hasSideEffects = 0;9450}9451 9452class sve_crypto_pmull_multi<string asm>9453: I<(outs ZZ_q_mul_r:$Zd),9454 (ins ZPR64:$Zn, ZPR64:$Zm),9455 asm,9456 "\t$Zd, $Zn, $Zm",9457 "",9458 []>, Sched<[]> {9459 bits<5> Zm;9460 bits<5> Zn;9461 bits<4> Zd;9462 let Inst{31-21} = 0b01000101001;9463 let Inst{20-16} = Zm;9464 let Inst{15-10} = 0b111110;9465 let Inst{9-5} = Zn;9466 let Inst{4-1} = Zd;9467 let Inst{0} = 0b0;9468 let hasSideEffects = 0;9469}9470 9471//===----------------------------------------------------------------------===//9472// SVE BFloat16 Group9473//===----------------------------------------------------------------------===//9474 9475class sve_float_dot<bit bf, bit o2, ZPRRegOp dst_ty, ZPRRegOp src_ty, string asm>9476: I<(outs dst_ty:$Zda), (ins dst_ty:$_Zda, src_ty:$Zn, src_ty:$Zm),9477 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> {9478 bits<5> Zda;9479 bits<5> Zn;9480 bits<5> Zm;9481 let Inst{31-23} = 0b011001000;9482 let Inst{22} = bf;9483 let Inst{21} = 0b1;9484 let Inst{20-16} = Zm;9485 let Inst{15-11} = 0b10000;9486 let Inst{10} = o2;9487 let Inst{9-5} = Zn;9488 let Inst{4-0} = Zda;9489 9490 let Constraints = "$Zda = $_Zda";9491 let DestructiveInstType = DestructiveOther;9492 let hasSideEffects = 0;9493 let mayRaiseFPException = 1;9494}9495 9496multiclass sve_float_dot<bit bf, bit o2, ZPRRegOp dst_ty, ZPRRegOp src_ty,9497 string asm, ValueType InVT, SDPatternOperator op> {9498 def NAME : sve_float_dot<bf, o2, dst_ty, src_ty, asm>;9499 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, InVT, InVT, !cast<Instruction>(NAME)>;9500}9501 9502multiclass sve_fp8_dot<bit bf, ZPRRegOp dstrc, string asm, ValueType vt,9503 SDPatternOperator op> {9504 def NAME : sve_float_dot<bf, 0b1, dstrc, ZPR8, asm> {9505 let Uses = [FPMR, FPCR];9506 9507 let mayLoad = 1;9508 let mayStore = 0;9509 }9510 9511 def : SVE_3_Op_Pat<vt, op, vt, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>;9512}9513 9514class sve_float_dot_indexed<bit bf, ZPRRegOp dst_ty, ZPRRegOp src1_ty,9515 ZPRRegOp src2_ty, Operand iop_ty, string asm>9516: I<(outs dst_ty:$Zda), (ins dst_ty:$_Zda, src1_ty:$Zn, src2_ty:$Zm, iop_ty:$iop),9517 asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> {9518 bits<5> Zda;9519 bits<5> Zn;9520 bits<3> Zm;9521 let Inst{31-23} = 0b011001000;9522 let Inst{22} = bf;9523 let Inst{21} = 0b1;9524 let Inst{18-16} = Zm;9525 let Inst{15-12} = 0b0100;9526 let Inst{9-5} = Zn;9527 let Inst{4-0} = Zda;9528 9529 let Constraints = "$Zda = $_Zda";9530 let DestructiveInstType = DestructiveOther;9531 let hasSideEffects = 0;9532 let mayRaiseFPException = 1;9533}9534 9535multiclass sve_float_dot_indexed<bit bf, bits<2> opc, ZPRRegOp src1_ty,9536 ZPRRegOp src2_ty, string asm, ValueType InVT,9537 SDPatternOperator op> {9538 def NAME : sve_float_dot_indexed<bf, ZPR32, src1_ty, src2_ty, VectorIndexS32b, asm> {9539 bits<2> iop;9540 let Inst{20-19} = iop;9541 let Inst{11-10} = opc;9542 }9543 def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, InVT, InVT, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME)>;9544}9545 9546multiclass sve_bfloat_convert<string asm, SDPatternOperator op, SDPatternOperator ir_op> {9547 def NAME : sve_fp_2op_p_zd<0b1001010, asm, ZPR32, ZPR16, ElementSizeS>;9548 9549 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv4i1, nxv4f32, !cast<Instruction>(NAME)>;9550 def : SVE_1_Op_Passthru_Round_Pat<nxv4bf16, ir_op, nxv4i1, nxv4f32, !cast<Instruction>(NAME)>;9551 def : SVE_1_Op_Passthru_Round_Pat<nxv2bf16, ir_op, nxv2i1, nxv2f32, !cast<Instruction>(NAME)>;9552}9553 9554multiclass sve_bfloat_convert_top<string asm, SDPatternOperator op> {9555 def NAME : sve2_fp_convert_precision<0b1010, 0b1, asm, ZPR16, ZPR32>;9556 9557 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv4i1, nxv4f32, !cast<Instruction>(NAME)>;9558}9559 9560//===----------------------------------------------------------------------===//9561// SVE Integer Matrix Multiply Group9562//===----------------------------------------------------------------------===//9563 9564class sve_int_matmul<bits<2> uns, string asm>9565: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR8:$Zm), asm,9566 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> {9567 bits<5> Zda;9568 bits<5> Zn;9569 bits<5> Zm;9570 let Inst{31-24} = 0b01000101;9571 let Inst{23-22} = uns;9572 let Inst{21} = 0;9573 let Inst{20-16} = Zm;9574 let Inst{15-10} = 0b100110;9575 let Inst{9-5} = Zn;9576 let Inst{4-0} = Zda;9577 9578 let Constraints = "$Zda = $_Zda";9579 let DestructiveInstType = DestructiveOther;9580 let ElementSize = ZPR32.ElementSize;9581 let hasSideEffects = 0;9582}9583 9584multiclass sve_int_matmul<bits<2> uns, string asm, SDPatternOperator op> {9585 def NAME : sve_int_matmul<uns, asm>;9586 9587 def : SVE_3_Op_Pat<nxv4i32, op , nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>;9588}9589 9590//===----------------------------------------------------------------------===//9591// SVE Integer Dot Product Mixed Sign Group9592//===----------------------------------------------------------------------===//9593 9594class sve_int_dot_mixed<string asm>9595: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR8:$Zm), asm,9596 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> {9597 bits<5> Zda;9598 bits<5> Zn;9599 bits<5> Zm;9600 let Inst{31-21} = 0b01000100100;9601 let Inst{20-16} = Zm;9602 let Inst{15-10} = 0b011110;9603 let Inst{9-5} = Zn;9604 let Inst{4-0} = Zda;9605 9606 let Constraints = "$Zda = $_Zda";9607 let DestructiveInstType = DestructiveOther;9608 let ElementSize = ZPR32.ElementSize;9609 let hasSideEffects = 0;9610}9611 9612multiclass sve_int_dot_mixed<string asm, SDPatternOperator op> {9613 def NAME : sve_int_dot_mixed<asm>;9614 9615 def : SVE_3_Op_Pat<nxv4i32, op , nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>;9616}9617 9618//===----------------------------------------------------------------------===//9619// SVE Integer Dot Product Mixed Sign - Indexed Group9620//===----------------------------------------------------------------------===//9621 9622class sve_int_dot_mixed_indexed<bit U, string asm>9623: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR3b8:$Zm, VectorIndexS32b:$idx),9624 asm, "\t$Zda, $Zn, $Zm$idx", "", []>, Sched<[]> {9625 bits<5> Zda;9626 bits<5> Zn;9627 bits<3> Zm;9628 bits<2> idx;9629 let Inst{31-21} = 0b01000100101;9630 let Inst{20-19} = idx;9631 let Inst{18-16} = Zm;9632 let Inst{15-11} = 0b00011;9633 let Inst{10} = U;9634 let Inst{9-5} = Zn;9635 let Inst{4-0} = Zda;9636 9637 let Constraints = "$Zda = $_Zda";9638 let DestructiveInstType = DestructiveOther;9639 let ElementSize = ZPR32.ElementSize;9640 let hasSideEffects = 0;9641}9642 9643multiclass sve_int_dot_mixed_indexed<bit U, string asm, SDPatternOperator op> {9644 def NAME : sve_int_dot_mixed_indexed<U, asm>;9645 9646 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME)>;9647}9648 9649//===----------------------------------------------------------------------===//9650// SVE Floating Point Matrix Multiply Accumulate Group9651//===----------------------------------------------------------------------===//9652 9653class sve_fp_matrix_mla<bits<3> opc, string asm, ZPRRegOp zda_ty, ZPRRegOp reg_ty>9654: I<(outs zda_ty:$Zda), (ins zda_ty:$_Zda, reg_ty:$Zn, reg_ty:$Zm),9655 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> {9656 bits<5> Zda;9657 bits<5> Zn;9658 bits<5> Zm;9659 let Inst{31-24} = 0b01100100;9660 let Inst{23-22} = opc{2-1};9661 let Inst{21} = 1;9662 let Inst{20-16} = Zm;9663 let Inst{15-11} = 0b11100;9664 let Inst{10} = opc{0};9665 let Inst{9-5} = Zn;9666 let Inst{4-0} = Zda;9667 9668 let Constraints = "$Zda = $_Zda";9669 let DestructiveInstType = DestructiveOther;9670 let hasSideEffects = 0;9671 let mayRaiseFPException = 1;9672}9673 9674multiclass sve_fp_matrix_mla<bits<3> opc, string asm, ZPRRegOp zda_ty,9675 ZPRRegOp reg_ty, SDPatternOperator op,9676 ValueType zda_vt, ValueType reg_vt> {9677 def NAME : sve_fp_matrix_mla<opc, asm, zda_ty, reg_ty>;9678 9679 def : SVE_3_Op_Pat<zda_vt, op, zda_vt, reg_vt, reg_vt, !cast<Instruction>(NAME)>;9680}9681 9682//===----------------------------------------------------------------------===//9683// SVE Memory - Contiguous Load And Replicate 256-bit Group9684//===----------------------------------------------------------------------===//9685 9686class sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand VecList>9687: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s32:$imm4),9688 asm, "\t$Zt, $Pg/z, [$Rn, $imm4]", "", []>, Sched<[]> {9689 bits<5> Zt;9690 bits<5> Rn;9691 bits<3> Pg;9692 bits<4> imm4;9693 let Inst{31-25} = 0b1010010;9694 let Inst{24-23} = sz;9695 let Inst{22-20} = 0b010;9696 let Inst{19-16} = imm4;9697 let Inst{15-13} = 0b001;9698 let Inst{12-10} = Pg;9699 let Inst{9-5} = Rn;9700 let Inst{4-0} = Zt;9701 9702 let hasSideEffects = 0;9703 let mayLoad = 1;9704}9705 9706multiclass sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand listty,9707 ZPRRegOp zprty, ValueType Ty, ValueType PredTy, SDNode Ld1ro> {9708 def NAME : sve_mem_ldor_si<sz, asm, listty>;9709 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",9710 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;9711 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",9712 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>;9713 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4]",9714 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s32:$imm4), 0>;9715 9716 // Base addressing mode9717 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$Pg), GPR64sp:$base)),9718 (!cast<Instruction>(NAME) PPR3bAny:$Pg, GPR64sp:$base, (i64 0))>;9719 let AddedComplexity = 2 in {9720 // Reg + Imm addressing mode9721 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$Pg), (add GPR64:$base, (i64 simm4s32:$imm)))),9722 (!cast<Instruction>(NAME) $Pg, $base, simm4s32:$imm)>;9723 }9724}9725 9726class sve_mem_ldor_ss<bits<2> sz, string asm, RegisterOperand VecList,9727 RegisterOperand gprty>9728: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),9729 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", "", []>, Sched<[]> {9730 bits<5> Zt;9731 bits<3> Pg;9732 bits<5> Rn;9733 bits<5> Rm;9734 let Inst{31-25} = 0b1010010;9735 let Inst{24-23} = sz;9736 let Inst{22-21} = 0b01;9737 let Inst{20-16} = Rm;9738 let Inst{15-13} = 0;9739 let Inst{12-10} = Pg;9740 let Inst{9-5} = Rn;9741 let Inst{4-0} = Zt;9742 9743 let hasSideEffects = 0;9744 let mayLoad = 1;9745}9746 9747multiclass sve_mem_ldor_ss<bits<2> sz, string asm, RegisterOperand listty,9748 ZPRRegOp zprty, RegisterOperand gprty, ValueType Ty,9749 ValueType PredTy, SDNode Ld1ro, ComplexPattern AddrCP> {9750 def NAME : sve_mem_ldor_ss<sz, asm, listty, gprty>;9751 9752 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]",9753 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>;9754 9755 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$gp), (AddrCP GPR64sp:$base, gprty:$offset))),9756 (!cast<Instruction>(NAME) PPR3bAny:$gp, GPR64sp:$base, gprty:$offset)>;9757}9758 9759//===----------------------------------------------------------------------===//9760// SVE Interleave 128-bit Elements Group9761//===----------------------------------------------------------------------===//9762 9763class sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm>9764: I<(outs ZPR128:$Zd), (ins ZPR128:$Zn, ZPR128:$Zm),9765 asm, "\t$Zd, $Zn, $Zm",9766 "",9767 []>, Sched<[]> {9768 bits<5> Zd;9769 bits<5> Zm;9770 bits<5> Zn;9771 let Inst{31-21} = 0b00000101101;9772 let Inst{20-16} = Zm;9773 let Inst{15-13} = 0b000;9774 let Inst{12-11} = opc;9775 let Inst{10} = P;9776 let Inst{9-5} = Zn;9777 let Inst{4-0} = Zd;9778 9779 let hasSideEffects = 0;9780}9781 9782multiclass sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm, SDPatternOperator op> {9783 def NAME : sve_int_perm_bin_perm_128_zz<opc, P, asm>;9784 9785 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>;9786 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>;9787 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME)>;9788 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>;9789 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME)>;9790 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>;9791 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME)>;9792 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME)>;9793}9794 9795/// Addressing modes9796let WantsRoot = true in {9797 def am_sve_indexed_s4 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<-8, 7>">;9798 def am_sve_indexed_s6 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<-32, 31>">;9799 def am_sve_indexed_s9 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<-256, 255>">;9800}9801 9802def am_sve_regreg_lsl0 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<0>", []>;9803def am_sve_regreg_lsl1 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<1>", []>;9804def am_sve_regreg_lsl2 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<2>", []>;9805def am_sve_regreg_lsl3 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<3>", []>;9806def am_sve_regreg_lsl4 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<4>", []>;9807 9808// Predicated pseudo floating point two operand instructions.9809multiclass sve_fp_bin_pred_hfd<SDPatternOperator op> {9810 def _H_UNDEF : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>;9811 def _S_UNDEF : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>;9812 def _D_UNDEF : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>;9813 9814 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _H_UNDEF)>;9815 def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Pseudo>(NAME # _H_UNDEF)>;9816 def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Pseudo>(NAME # _H_UNDEF)>;9817 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _S_UNDEF)>;9818 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Pseudo>(NAME # _S_UNDEF)>;9819 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _D_UNDEF)>;9820}9821 9822// Predicated pseudo floating point (BFloat) two operand instructions.9823multiclass sve_fp_bin_pred_bfloat<SDPatternOperator op> {9824 def _UNDEF : PredTwoOpPseudo<NAME, ZPR16, FalseLanesUndef>;9825 9826 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Pseudo>(NAME # _UNDEF)>;9827 def : SVE_3_Op_Pat<nxv4bf16, op, nxv4i1, nxv4bf16, nxv4bf16, !cast<Pseudo>(NAME # _UNDEF)>;9828 def : SVE_3_Op_Pat<nxv2bf16, op, nxv2i1, nxv2bf16, nxv2bf16, !cast<Pseudo>(NAME # _UNDEF)>;9829}9830 9831// Predicated pseudo floating point three operand instructions.9832multiclass sve_fp_3op_pred_hfd<SDPatternOperator op> {9833 def _H_UNDEF : PredThreeOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>;9834 def _S_UNDEF : PredThreeOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>;9835 def _D_UNDEF : PredThreeOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>;9836 9837 def : SVE_4_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H_UNDEF)>;9838 def : SVE_4_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _H_UNDEF)>;9839 def : SVE_4_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _H_UNDEF)>;9840 def : SVE_4_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S_UNDEF)>;9841 def : SVE_4_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _S_UNDEF)>;9842 def : SVE_4_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D_UNDEF)>;9843}9844 9845// Predicated pseudo floating point (BFloat) three operand instructions.9846multiclass sve_fp_3op_pred_bfloat<SDPatternOperator op> {9847 def _UNDEF : PredThreeOpPseudo<NAME, ZPR16, FalseLanesUndef>;9848 9849 def : SVE_4_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _UNDEF)>;9850 def : SVE_4_Op_Pat<nxv4bf16, op, nxv4i1, nxv4bf16, nxv4bf16, nxv4bf16, !cast<Instruction>(NAME # _UNDEF)>;9851 def : SVE_4_Op_Pat<nxv2bf16, op, nxv2i1, nxv2bf16, nxv2bf16, nxv2bf16, !cast<Instruction>(NAME # _UNDEF)>;9852}9853 9854// Predicated pseudo integer two operand instructions.9855multiclass sve_int_bin_pred_bhsd<SDPatternOperator op> {9856 def _B_UNDEF : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesUndef>;9857 def _H_UNDEF : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>;9858 def _S_UNDEF : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>;9859 def _D_UNDEF : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>;9860 9861 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _B_UNDEF)>;9862 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _H_UNDEF)>;9863 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _S_UNDEF)>;9864 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _D_UNDEF)>;9865}9866 9867// As sve_int_bin_pred but when only i32 and i64 vector types are required.9868multiclass sve_int_bin_pred_sd<SDPatternOperator op> {9869 def _S_UNDEF : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>;9870 def _D_UNDEF : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>;9871 9872 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _S_UNDEF)>;9873 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _D_UNDEF)>;9874}9875 9876// Predicated pseudo integer two operand instructions. Second operand is an9877// immediate specified by imm_[bhsd].9878multiclass sve_int_shift_pred_bhsd<SDPatternOperator op,9879 ComplexPattern imm_b, ComplexPattern imm_h,9880 ComplexPattern imm_s, ComplexPattern imm_d> {9881 def _B_UNDEF : PredTwoOpImmPseudo<NAME # _B, ZPR8, Operand<i32>, FalseLanesUndef>;9882 def _H_UNDEF : PredTwoOpImmPseudo<NAME # _H, ZPR16, Operand<i32>, FalseLanesUndef>;9883 def _S_UNDEF : PredTwoOpImmPseudo<NAME # _S, ZPR32, Operand<i32>, FalseLanesUndef>;9884 def _D_UNDEF : PredTwoOpImmPseudo<NAME # _D, ZPR64, Operand<i32>, FalseLanesUndef>;9885 9886 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, imm_b, !cast<Instruction>(NAME # _B_UNDEF)>;9887 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, imm_h, !cast<Instruction>(NAME # _H_UNDEF)>;9888 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, imm_s, !cast<Instruction>(NAME # _S_UNDEF)>;9889 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, imm_d, !cast<Instruction>(NAME # _D_UNDEF)>;9890}9891 9892multiclass sve_int_bin_pred_all_active_bhsd<SDPatternOperator op> {9893 def _B_UNDEF : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesUndef>;9894 def _H_UNDEF : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>;9895 def _S_UNDEF : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>;9896 def _D_UNDEF : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>;9897 9898 def : SVE_2_Op_Pred_All_Active_Pt<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _B_UNDEF)>;9899 def : SVE_2_Op_Pred_All_Active_Pt<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _H_UNDEF)>;9900 def : SVE_2_Op_Pred_All_Active_Pt<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _S_UNDEF)>;9901 def : SVE_2_Op_Pred_All_Active_Pt<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _D_UNDEF)>;9902}9903 9904//===----------------------------------------------------------------------===//9905// SME2 or SVE2.1 Instructions9906//===----------------------------------------------------------------------===//9907 9908class sve_fp_clamp<string asm, bits<2> sz, ZPRRegOp zpr_ty>9909 : I<(outs zpr_ty:$Zd), (ins zpr_ty:$_Zd, zpr_ty:$Zn, zpr_ty:$Zm),9910 asm, "\t$Zd, $Zn, $Zm", "", []>,9911 Sched<[]> {9912 bits<5> Zm;9913 bits<5> Zn;9914 bits<5> Zd;9915 let Inst{31-24} = 0b01100100;9916 let Inst{23-22} = sz;9917 let Inst{21} = 0b1;9918 let Inst{20-16} = Zm;9919 let Inst{15-10} = 0b001001;9920 let Inst{9-5} = Zn;9921 let Inst{4-0} = Zd;9922 9923 let Constraints = "$Zd = $_Zd";9924 let DestructiveInstType = DestructiveOther;9925 let ElementSize = zpr_ty.ElementSize;9926 let hasSideEffects = 0;9927}9928 9929multiclass sve_fp_clamp<string asm, SDPatternOperator op> {9930 def _H : sve_fp_clamp<asm, 0b01, ZPR16>;9931 def _S : sve_fp_clamp<asm, 0b10, ZPR32>;9932 def _D : sve_fp_clamp<asm, 0b11, ZPR64>;9933 9934 def : SVE_3_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;9935 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;9936 def : SVE_3_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;9937}9938 9939multiclass sve_fp_clamp_bfloat<string asm, SDPatternOperator op> {9940 def NAME : sve_fp_clamp<asm, 0b00, ZPR16>;9941 9942 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME)>;9943}9944 9945// SVE two-way dot product (indexed)9946class sve2p1_two_way_dot_vvi<string mnemonic, bit u>9947 : I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexS32b:$i2),9948 mnemonic, "\t$Zda, $Zn, $Zm$i2",9949 "", []>, Sched<[]> {9950 bits<5> Zda;9951 bits<5> Zn;9952 bits<3> Zm;9953 bits<2> i2;9954 let Inst{31-21} = 0b01000100100;9955 let Inst{20-19} = i2;9956 let Inst{18-16} = Zm;9957 let Inst{15-11} = 0b11001;9958 let Inst{10} = u;9959 let Inst{9-5} = Zn;9960 let Inst{4-0} = Zda;9961 9962 let Constraints = "$Zda = $_Zda";9963 let DestructiveInstType = DestructiveOther;9964 let hasSideEffects = 0;9965}9966 9967multiclass sve2p1_two_way_dot_vvi<string mnemonic, bit u, SDPatternOperator intrinsic> {9968 def NAME : sve2p1_two_way_dot_vvi<mnemonic, u>;9969 9970 def : SVE_4_Op_Imm_Pat<nxv4i32, intrinsic, nxv4i32, nxv8i16, nxv8i16, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME)>;9971}9972 9973class sve2p1_ptrue_pn<string mnemonic, bits<2> sz, PNRP8to15RegOp pnrty, SDPatternOperator op>9974 : I<(outs pnrty:$PNd), (ins ), mnemonic, "\t$PNd",9975 "", [(set pnrty:$PNd, (op))]>, Sched<[]> {9976 bits<3> PNd;9977 let Inst{31-24} = 0b00100101;9978 let Inst{23-22} = sz;9979 let Inst{21-3} = 0b1000000111100000010;9980 let Inst{2-0} = PNd;9981 9982 let hasSideEffects = 0;9983 let isReMaterializable = 1;9984 let Uses = [VG];9985}9986 9987 9988multiclass sve2p1_ptrue_pn<string mnemonic> {9989 def _B : sve2p1_ptrue_pn<mnemonic, 0b00, PNR8_p8to15, int_aarch64_sve_ptrue_c8>;9990 def _H : sve2p1_ptrue_pn<mnemonic, 0b01, PNR16_p8to15, int_aarch64_sve_ptrue_c16>;9991 def _S : sve2p1_ptrue_pn<mnemonic, 0b10, PNR32_p8to15, int_aarch64_sve_ptrue_c32>;9992 def _D : sve2p1_ptrue_pn<mnemonic, 0b11, PNR64_p8to15, int_aarch64_sve_ptrue_c64>;9993}9994 9995 9996// SVE extract mask predicate from predicate-as-counter9997class sve2p1_pred_as_ctr_to_mask_base<string mnemonic, bits<2> sz, bits<3> opc,9998 RegisterOperand pprty, Operand idxty>9999 : I<(outs pprty:$Pd), (ins PNRAny_p8to15:$PNn, idxty:$index),10000 mnemonic, "\t$Pd, $PNn$index",10001 "", []>, Sched<[]> {10002 bits<4> Pd;10003 bits<3> PNn;10004 bits<2> imm2;10005 let Inst{31-24} = 0b00100101;10006 let Inst{23-22} = sz;10007 let Inst{21-11} = 0b10000001110;10008 let Inst{10-8} = opc;10009 let Inst{7-5} = PNn;10010 let Inst{4} = 0b1;10011 let Inst{3-0} = Pd;10012 10013 let hasSideEffects = 0;10014}10015 10016class sve2p1_pred_as_ctr_to_mask<string mnemonic, bits<2> sz, PPRRegOp pprty>10017 : sve2p1_pred_as_ctr_to_mask_base<mnemonic, sz, {0, ?, ?}, pprty, VectorIndexS32b_timm> {10018 bits<2> index;10019 let Inst{9-8} = index;10020}10021 10022multiclass sve2p1_pred_as_ctr_to_mask<string mnemonic, SDPatternOperator op> {10023 def _B : sve2p1_pred_as_ctr_to_mask<mnemonic, 0b00, PPR8>;10024 def _H : sve2p1_pred_as_ctr_to_mask<mnemonic, 0b01, PPR16>;10025 def _S : sve2p1_pred_as_ctr_to_mask<mnemonic, 0b10, PPR32>;10026 def _D : sve2p1_pred_as_ctr_to_mask<mnemonic, 0b11, PPR64>;10027 10028 def : SVE_2_Op_Imm_Pat<nxv16i1, op, aarch64svcount, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _B)>;10029 def : SVE_2_Op_Imm_Pat<nxv8i1, op, aarch64svcount, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _H)>;10030 def : SVE_2_Op_Imm_Pat<nxv4i1, op, aarch64svcount, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>;10031 def : SVE_2_Op_Imm_Pat<nxv2i1, op, aarch64svcount, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _D)>;10032}10033 10034 10035class sve2p1_pred_as_ctr_to_mask_pair<string mnemonic, bits<2> sz, RegisterOperand pprty>10036 : sve2p1_pred_as_ctr_to_mask_base<mnemonic, sz, {1, 0, ?}, pprty, VectorIndexD> {10037 bit index;10038 let Inst{8} = index;10039}10040 10041multiclass sve2p1_pred_as_ctr_to_mask_pair<string mnemonic> {10042 def _B : sve2p1_pred_as_ctr_to_mask_pair<mnemonic, 0b00, PP_b>;10043 def _H : sve2p1_pred_as_ctr_to_mask_pair<mnemonic, 0b01, PP_h>;10044 def _S : sve2p1_pred_as_ctr_to_mask_pair<mnemonic, 0b10, PP_s>;10045 def _D : sve2p1_pred_as_ctr_to_mask_pair<mnemonic, 0b11, PP_d>;10046}10047 10048 10049// SME2 multi-vec extract narrow10050class sve2p1_multi_vec_extract_narrow<string mnemonic, bits<2> opc, bits<3> tsz>10051 : I<(outs ZPR16:$Zd), (ins ZZ_s_mul_r:$Zn),10052 mnemonic, "\t$Zd, $Zn",10053 "", []>, Sched<[]> {10054 bits<5> Zd;10055 bits<4> Zn;10056 let Inst{31-23} = 0b010001010;10057 let Inst{22} = tsz{2};10058 let Inst{21} = 0b1;10059 let Inst{20-19} = tsz{1-0};10060 let Inst{18-13} = 0b001010;10061 let Inst{12-11} = opc;10062 let Inst{10} = 0b0;10063 let Inst{9-6} = Zn;10064 let Inst{5} = 0b0;10065 let Inst{4-0} = Zd;10066 10067 let hasSideEffects = 0;10068}10069 10070multiclass sve2p1_multi_vec_extract_narrow<string mnemonic, bits<2> opc, SDPatternOperator intrinsic> {10071 def NAME : sve2p1_multi_vec_extract_narrow<mnemonic, opc, 0b010>;10072 def : SVE2p1_Cvt_VG2_Pat<NAME, intrinsic, nxv8i16, nxv4i32>;10073}10074 10075// SVE2 multi-vec shift narrow10076class sve2p1_multi_vec_shift_narrow<string mnemonic, ZPRRegOp ZdRC, RegisterOperand ZSrcOp,10077 Operand immtype, bits<3> opc, bits<2> tsz>10078 : I<(outs ZdRC:$Zd), (ins ZSrcOp:$Zn, immtype:$imm),10079 mnemonic, "\t$Zd, $Zn, $imm",10080 "", []>, Sched<[]> {10081 bits<5> Zd;10082 bits<4> Zn;10083 bits<4> imm;10084 let Inst{31-23} = 0b010001011;10085 let Inst{22} = tsz{1};10086 let Inst{21} = 0b1;10087 let Inst{20} = tsz{0};10088 let Inst{18-16} = imm{2-0}; // imm310089 let Inst{15-14} = 0b00;10090 let Inst{13-11} = opc;10091 let Inst{10} = 0b0;10092 let Inst{9-6} = Zn;10093 let Inst{5} = 0b0;10094 let Inst{4-0} = Zd;10095 10096 let hasSideEffects = 0;10097}10098 10099multiclass sve_multi_vec_shift_narrow<string mnemonic, bits<3> opc, SDPatternOperator intrinsic> {10100 def NAME : sve2p1_multi_vec_shift_narrow<mnemonic, ZPR16, ZZ_s_mul_r, vecshiftR16, opc, 0b01> {10101 let Inst{19} = imm{3}; // imm410102 }10103 10104 def : SVE2p1_Sat_Shift_VG2_Pat<NAME, intrinsic, nxv8i16, nxv4i32, vecshiftR16>;10105}10106 10107multiclass sve_multi_vec_round_shift_narrow<string mnemonic, bits<3> opc> {10108 def NAME : sve2p1_multi_vec_shift_narrow<mnemonic, ZPR8, ZZ_h_mul_r, vecshiftR8, opc, 0b00> {10109 let Inst{19} = 0b1; // always 1 for imm3 version10110 }10111}10112 10113// SME2 multi-vec contiguous load (scalar plus scalar, two registers)10114class sve2p1_mem_cld_ss_2z<string mnemonic, bits<2> msz, bit n,10115 RegisterOperand vector_ty, RegisterOperand gpr_ty>10116 : I<(outs vector_ty:$Zt),10117 (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm),10118 mnemonic, "\t$Zt, $PNg/z, [$Rn, $Rm]",10119 "", []>, Sched<[]> {10120 bits<4> Zt;10121 bits<5> Rm;10122 bits<5> Rn;10123 bits<3> PNg;10124 let Inst{31-21} = 0b10100000000;10125 let Inst{20-16} = Rm;10126 let Inst{15} = 0b0;10127 let Inst{14-13} = msz;10128 let Inst{12-10} = PNg;10129 let Inst{9-5} = Rn;10130 let Inst{4-1} = Zt;10131 let Inst{0} = n;10132 10133 let hasSideEffects = 0;10134 let mayLoad = 1;10135}10136 10137multiclass sve2p1_mem_cld_ss_2z<string mnemonic, bits<2> msz, bit n,10138 RegisterOperand vector_ty, RegisterOperand gpr_ty, RegisterOperand vector_pseudo_ty> {10139 def NAME # _PSEUDO : Pseudo<(outs vector_pseudo_ty:$Zt), (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm), []>;10140 def NAME : sve2p1_mem_cld_ss_2z<mnemonic, msz, n, vector_ty, gpr_ty>;10141}10142 10143// SME2 multi-vec contiguous load (scalar plus immediate, two registers)10144class sve2p1_mem_cld_si_2z<string mnemonic, bits<2> msz, bit n,10145 RegisterOperand vector_ty>10146 : I<(outs vector_ty:$Zt),10147 (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, simm4s2:$imm4),10148 mnemonic, "\t$Zt, $PNg/z, [$Rn, $imm4, mul vl]",10149 "", []>, Sched<[]> {10150 bits<4> Zt;10151 bits<5> Rn;10152 bits<3> PNg;10153 bits<4> imm4;10154 let Inst{31-20} = 0b101000000100;10155 let Inst{19-16} = imm4;10156 let Inst{15} = 0b0;10157 let Inst{14-13} = msz;10158 let Inst{12-10} = PNg;10159 let Inst{9-5} = Rn;10160 let Inst{4-1} = Zt;10161 let Inst{0} = n;10162 10163 let hasSideEffects = 0;10164 let mayLoad = 1;10165}10166 10167multiclass sve2p1_mem_cld_si_2z<string mnemonic, bits<2> msz, bit n,10168 RegisterOperand vector_ty, RegisterOperand vector_pseudo_ty> {10169 def NAME : sve2p1_mem_cld_si_2z<mnemonic, msz, n, vector_ty>;10170 def : InstAlias<mnemonic # " $Zt, $PNg/z, [$Rn]",10171 (!cast<Instruction>(NAME) vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, 0), 1>;10172 def NAME # _PSEUDO : Pseudo<(outs vector_pseudo_ty:$Zt), (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, simm4s2:$imm4), []>;10173}10174 10175// SME2 multi-vec contiguous load (scalar plus scalar, four registers)10176class sve2p1_mem_cld_ss_4z<string mnemonic, bits<2> msz, bit n,10177 RegisterOperand vector_ty, RegisterOperand gpr_ty>10178 : I<(outs vector_ty:$Zt),10179 (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm),10180 mnemonic, "\t$Zt, $PNg/z, [$Rn, $Rm]",10181 "", []>, Sched<[]> {10182 bits<3> Zt;10183 bits<5> Rm;10184 bits<5> Rn;10185 bits<3> PNg;10186 let Inst{31-21} = 0b10100000000;10187 let Inst{20-16} = Rm;10188 let Inst{15} = 0b1;10189 let Inst{14-13} = msz;10190 let Inst{12-10} = PNg;10191 let Inst{9-5} = Rn;10192 let Inst{4-2} = Zt;10193 let Inst{1} = 0b0;10194 let Inst{0} = n;10195 10196 let hasSideEffects = 0;10197 let mayLoad = 1;10198}10199 10200multiclass sve2p1_mem_cld_ss_4z<string mnemonic, bits<2> msz, bit n,10201 RegisterOperand vector_ty, RegisterOperand gpr_ty, RegisterOperand vector_pseudo_ty> {10202 def NAME # _PSEUDO : Pseudo<(outs vector_pseudo_ty:$Zt), (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm), []>;10203 def NAME : sve2p1_mem_cld_ss_4z<mnemonic, msz, n, vector_ty, gpr_ty>;10204}10205 10206// SME2 multi-vec contiguous load (scalar plus immediate, four registers)10207class sve2p1_mem_cld_si_4z<string mnemonic, bits<2> msz, bit n,10208 RegisterOperand vector_ty>10209 : I<(outs vector_ty:$Zt),10210 (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, simm4s4:$imm4),10211 mnemonic, "\t$Zt, $PNg/z, [$Rn, $imm4, mul vl]",10212 "", []>, Sched<[]> {10213 bits<3> Zt;10214 bits<5> Rn;10215 bits<3> PNg;10216 bits<4> imm4;10217 let Inst{31-20} = 0b101000000100;10218 let Inst{19-16} = imm4;10219 let Inst{15} = 0b1;10220 let Inst{14-13} = msz;10221 let Inst{12-10} = PNg;10222 let Inst{9-5} = Rn;10223 let Inst{4-2} = Zt;10224 let Inst{1} = 0b0;10225 let Inst{0} = n;10226 10227 let hasSideEffects = 0;10228 let mayLoad = 1;10229}10230 10231multiclass sve2p1_mem_cld_si_4z<string mnemonic, bits<2> msz, bit n,10232 RegisterOperand vector_ty, RegisterOperand vector_pseudo_ty> {10233 def NAME : sve2p1_mem_cld_si_4z<mnemonic, msz, n, vector_ty>;10234 def : InstAlias<mnemonic # " $Zt, $PNg/z, [$Rn]",10235 (!cast<Instruction>(NAME) vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, 0), 1>;10236 def NAME # _PSEUDO : Pseudo<(outs vector_pseudo_ty:$Zt), (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, simm4s4:$imm4), []>;10237}10238 10239// SME2 multi-vec contiguous store (scalar plus scalar, two registers)10240class sve2p1_mem_cst_ss_2z<string mnemonic, bits<2> msz, bit n,10241 RegisterOperand vector_ty, RegisterOperand gpr_ty>10242 : I<(outs ),10243 (ins vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm),10244 mnemonic, "\t$Zt, $PNg, [$Rn, $Rm]",10245 "", []>, Sched<[]> {10246 bits<4> Zt;10247 bits<5> Rm;10248 bits<5> Rn;10249 bits<3> PNg;10250 let Inst{31-21} = 0b10100000001;10251 let Inst{20-16} = Rm;10252 let Inst{15} = 0b0;10253 let Inst{14-13} = msz;10254 let Inst{12-10} = PNg;10255 let Inst{9-5} = Rn;10256 let Inst{4-1} = Zt;10257 let Inst{0} = n;10258 10259 let hasSideEffects = 0;10260 let mayStore = 1;10261}10262 10263 10264// SME2 multi-vec contiguous store (scalar plus immediate, two registers)10265class sve2p1_mem_cst_si_2z<string mnemonic, bits<2> msz, bit n,10266 RegisterOperand vector_ty>10267 : I<(outs ),10268 (ins vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, simm4s2:$imm4),10269 mnemonic, "\t$Zt, $PNg, [$Rn, $imm4, mul vl]",10270 "", []>, Sched<[]> {10271 bits<4> Zt;10272 bits<5> Rn;10273 bits<3> PNg;10274 bits<4> imm4;10275 let Inst{31-20} = 0b101000000110;10276 let Inst{19-16} = imm4;10277 let Inst{15} = 0b0;10278 let Inst{14-13} = msz;10279 let Inst{12-10} = PNg;10280 let Inst{9-5} = Rn;10281 let Inst{4-1} = Zt;10282 let Inst{0} = n;10283 10284 let hasSideEffects = 0;10285 let mayStore = 1;10286}10287 10288 10289multiclass sve2p1_mem_cst_si_2z<string mnemonic, bits<2> msz, bit n,10290 RegisterOperand vector_ty> {10291 def NAME : sve2p1_mem_cst_si_2z<mnemonic, msz, n, vector_ty>;10292 10293 def : InstAlias<mnemonic # " $Zt, $PNg, [$Rn]",10294 (!cast<Instruction>(NAME) vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, 0), 1>;10295}10296 10297 10298// SME2 multi-vec contiguous store (scalar plus scalar, four registers)10299class sve2p1_mem_cst_ss_4z<string mnemonic, bits<2> msz, bit n,10300 RegisterOperand vector_ty, RegisterOperand gpr_ty>10301 : I<(outs ),10302 (ins vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm),10303 mnemonic, "\t$Zt, $PNg, [$Rn, $Rm]",10304 "", []>, Sched<[]> {10305 bits<3> Zt;10306 bits<5> Rm;10307 bits<5> Rn;10308 bits<3> PNg;10309 let Inst{31-21} = 0b10100000001;10310 let Inst{20-16} = Rm;10311 let Inst{15} = 0b1;10312 let Inst{14-13} = msz;10313 let Inst{12-10} = PNg;10314 let Inst{9-5} = Rn;10315 let Inst{4-2} = Zt;10316 let Inst{1} = 0b0;10317 let Inst{0} = n;10318 10319 let mayStore = 1;10320}10321 10322 10323// SME2 multi-vec contiguous store (scalar plus immediate, four registers)10324class sve2p1_mem_cst_si_4z<string mnemonic, bits<2> msz, bit n,10325 RegisterOperand vector_ty>10326 : I<(outs ),10327 (ins vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, simm4s4:$imm4),10328 mnemonic, "\t$Zt, $PNg, [$Rn, $imm4, mul vl]",10329 "", []>, Sched<[]> {10330 bits<3> Zt;10331 bits<5> Rn;10332 bits<3> PNg;10333 bits<4> imm4;10334 let Inst{31-20} = 0b101000000110;10335 let Inst{19-16} = imm4;10336 let Inst{15} = 0b1;10337 let Inst{14-13} = msz;10338 let Inst{12-10} = PNg;10339 let Inst{9-5} = Rn;10340 let Inst{4-2} = Zt;10341 let Inst{1} = 0b0;10342 let Inst{0} = n;10343 10344 let hasSideEffects = 0;10345 let mayStore = 1;10346}10347 10348 10349multiclass sve2p1_mem_cst_si_4z<string mnemonic, bits<2> msz, bit n,10350 RegisterOperand vector_ty> {10351 def NAME : sve2p1_mem_cst_si_4z<mnemonic, msz, n, vector_ty>;10352 10353 def : InstAlias<mnemonic # " $Zt, $PNg, [$Rn]",10354 (!cast<Instruction>(NAME) vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn,0), 1>;10355}10356 10357// SVE predicate count (predicate-as-counter)10358class sve2p1_pcount_pn<string mnemonic, bits<3> opc, bits<2> sz, PNRRegOp pnrty>10359 : I<(outs GPR64:$Rd),10360 (ins pnrty:$PNn, sve_vec_len_specifier_enum:$vl),10361 mnemonic, "\t$Rd, $PNn, $vl",10362 "", []>, Sched<[]> {10363 bits<5> Rd;10364 bits<4> PNn;10365 bits<1> vl;10366 let Inst{31-24} = 0b00100101;10367 let Inst{23-22} = sz;10368 let Inst{21-19} = 0b100;10369 let Inst{18-16} = opc;10370 let Inst{15-11} = 0b10000;10371 let Inst{10} = vl;10372 let Inst{9} = 0b1;10373 let Inst{8-5} = PNn;10374 let Inst{4-0} = Rd;10375 10376 let hasSideEffects = 0;10377}10378 10379multiclass sve2p1_pcount_pn<string mnemonic, bits<3> opc> {10380 def _B : sve2p1_pcount_pn<mnemonic, opc, 0b00, PNR8>;10381 def _H : sve2p1_pcount_pn<mnemonic, opc, 0b01, PNR16>;10382 def _S : sve2p1_pcount_pn<mnemonic, opc, 0b10, PNR32>;10383 def _D : sve2p1_pcount_pn<mnemonic, opc, 0b11, PNR64>;10384 10385 defm : SVE2p1_Cntp_Pat<i64, int_aarch64_sve_cntp_c8, aarch64svcount, !cast<Instruction>(NAME # _B)>;10386 defm : SVE2p1_Cntp_Pat<i64, int_aarch64_sve_cntp_c16, aarch64svcount, !cast<Instruction>(NAME # _H)>;10387 defm : SVE2p1_Cntp_Pat<i64, int_aarch64_sve_cntp_c32, aarch64svcount, !cast<Instruction>(NAME # _S)>;10388 defm : SVE2p1_Cntp_Pat<i64, int_aarch64_sve_cntp_c64, aarch64svcount, !cast<Instruction>(NAME # _D)>;10389}10390 10391 10392// SVE integer compare scalar count and limit (predicate-as-counter)10393class sve2p1_int_while_rr_pn<string mnemonic, bits<2> sz, bits<3> opc,10394 PNRP8to15RegOp pnrty>10395 : I<(outs pnrty:$PNd), (ins GPR64:$Rn, GPR64:$Rm, sve_vec_len_specifier_enum:$vl),10396 mnemonic, "\t$PNd, $Rn, $Rm, $vl",10397 "", []>, Sched<[]> {10398 bits<3> PNd;10399 bits<5> Rn;10400 bits<1> vl;10401 bits<5> Rm;10402 let Inst{31-24} = 0b00100101;10403 let Inst{23-22} = sz;10404 let Inst{21} = 0b1;10405 let Inst{20-16} = Rm;10406 let Inst{15-14} = 0b01;10407 let Inst{13} = vl;10408 let Inst{12} = 0b0;10409 let Inst{11-10} = opc{2-1};10410 let Inst{9-5} = Rn;10411 let Inst{4} = 0b1;10412 let Inst{3} = opc{0};10413 let Inst{2-0} = PNd;10414 10415 let Defs = [NZCV];10416 let hasSideEffects = 0;10417}10418 10419 10420multiclass sve2p1_int_while_rr_pn<string mnemonic, bits<3> opc> {10421 def _B : sve2p1_int_while_rr_pn<mnemonic, 0b00, opc, PNR8_p8to15>;10422 def _H : sve2p1_int_while_rr_pn<mnemonic, 0b01, opc, PNR16_p8to15>;10423 def _S : sve2p1_int_while_rr_pn<mnemonic, 0b10, opc, PNR32_p8to15>;10424 def _D : sve2p1_int_while_rr_pn<mnemonic, 0b11, opc, PNR64_p8to15>;10425 10426 defm : SVE2p1_While_PN_Pat<aarch64svcount, !cast<SDPatternOperator>("int_aarch64_sve_" # mnemonic # "_c8"),10427 i64, !cast<Instruction>(NAME # _B)>;10428 defm : SVE2p1_While_PN_Pat<aarch64svcount, !cast<SDPatternOperator>("int_aarch64_sve_" # mnemonic # "_c16"),10429 i64, !cast<Instruction>(NAME # _H)>;10430 defm : SVE2p1_While_PN_Pat<aarch64svcount, !cast<SDPatternOperator>("int_aarch64_sve_" # mnemonic # "_c32"),10431 i64, !cast<Instruction>(NAME # _S)>;10432 defm : SVE2p1_While_PN_Pat<aarch64svcount, !cast<SDPatternOperator>("int_aarch64_sve_" # mnemonic # "_c64"),10433 i64, !cast<Instruction>(NAME # _D)>;10434}10435 10436 10437// SVE integer compare scalar count and limit (predicate pair)10438class sve2p1_int_while_rr_pair<string mnemonic, bits<2> sz, bits<3> opc,10439 PPR2MulRegOp ppr_ty>10440 : I<(outs ppr_ty:$Pd), (ins GPR64:$Rn, GPR64:$Rm),10441 mnemonic, "\t$Pd, $Rn, $Rm",10442 "", []>, Sched<[]> {10443 bits<3> Pd;10444 bits<5> Rn;10445 bits<5> Rm;10446 let Inst{31-24} = 0b00100101;10447 let Inst{23-22} = sz;10448 let Inst{21} = 0b1;10449 let Inst{20-16} = Rm;10450 let Inst{15-12} = 0b0101;10451 let Inst{11-10} = opc{2-1};10452 let Inst{9-5} = Rn;10453 let Inst{4} = 0b1;10454 let Inst{3-1} = Pd;10455 let Inst{0} = opc{0};10456 10457 let Defs = [NZCV];10458 let hasSideEffects = 0;10459 let ElementSize = ppr_ty.ElementSize;10460 let isWhile = 1;10461}10462 10463 10464multiclass sve2p1_int_while_rr_pair<string mnemonic, bits<3> opc> {10465 def _B : sve2p1_int_while_rr_pair<mnemonic, 0b00, opc, PP_b_mul_r>;10466 def _H : sve2p1_int_while_rr_pair<mnemonic, 0b01, opc, PP_h_mul_r>;10467 def _S : sve2p1_int_while_rr_pair<mnemonic, 0b10, opc, PP_s_mul_r>;10468 def _D : sve2p1_int_while_rr_pair<mnemonic, 0b11, opc, PP_d_mul_r>;10469}10470 10471 10472class sve_mem_128b_gld_64_unscaled<string mnemonic>10473 : I<(outs Z_q:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm),10474 mnemonic, "\t$Zt, $Pg/z, [$Zn, $Rm]",10475 "", []>, Sched<[]> {10476 bits<5> Zt;10477 bits<5> Zn;10478 bits<3> Pg;10479 bits<5> Rm;10480 let Inst{31-21} = 0b11000100000;10481 let Inst{20-16} = Rm;10482 let Inst{15-13} = 0b101;10483 let Inst{12-10} = Pg;10484 let Inst{9-5} = Zn;10485 let Inst{4-0} = Zt;10486 10487 let hasSideEffects = 0;10488 let mayLoad = 1;10489}10490 10491 10492multiclass sve_mem_128b_gld_64_unscaled<string mnemonic, SDPatternOperator op> {10493 def NAME : sve_mem_128b_gld_64_unscaled<mnemonic>;10494 10495 def : InstAlias<mnemonic # " $Zt, $Pg/z, [$Zn]",10496 (!cast<Instruction>(NAME) Z_q:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>;10497 10498 10499 def : Pat<(nxv2i64 (op (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zn), (i64 GPR64sp:$Rm), nxv2i64)),10500 (!cast<Instruction>(NAME) PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm)>;10501 def : Pat<(nxv4i32 (op (nxv4i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zn), (i64 GPR64sp:$Rm), nxv4i32)),10502 (!cast<Instruction>(NAME) PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm)>;10503 def : Pat<(nxv8i16 (op (nxv8i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zn), (i64 GPR64sp:$Rm), nxv8i16)),10504 (!cast<Instruction>(NAME) PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm)>;10505 def : Pat<(nxv16i8 (op (nxv16i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zn), (i64 GPR64sp:$Rm), nxv16i8)),10506 (!cast<Instruction>(NAME) PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm)>;10507 10508 def : Pat<(nxv2f64 (op (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zn), (i64 GPR64sp:$Rm), nxv2f64)),10509 (!cast<Instruction>(NAME) PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm)>;10510 def : Pat<(nxv4f32 (op (nxv4i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zn), (i64 GPR64sp:$Rm), nxv4f32)),10511 (!cast<Instruction>(NAME) PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm)>;10512 def : Pat<(nxv8f16 (op (nxv8i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zn), (i64 GPR64sp:$Rm), nxv8f16)),10513 (!cast<Instruction>(NAME) PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm)>;10514 def : Pat<(nxv8bf16 (op (nxv8i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zn), (i64 GPR64sp:$Rm), nxv8bf16)),10515 (!cast<Instruction>(NAME) PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm)>;10516}10517 10518class sve_mem_sst_128b_64_unscaled<string mnemonic>10519 : I<(outs ), (ins Z_q:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm),10520 mnemonic, "\t$Zt, $Pg, [$Zn, $Rm]",10521 "", []>, Sched<[]> {10522 bits<5> Zt;10523 bits<5> Zn;10524 bits<3> Pg;10525 bits<5> Rm;10526 let Inst{31-21} = 0b11100100001;10527 let Inst{20-16} = Rm;10528 let Inst{15-13} = 0b001;10529 let Inst{12-10} = Pg;10530 let Inst{9-5} = Zn;10531 let Inst{4-0} = Zt;10532 10533 let hasSideEffects = 0;10534 let mayStore = 1;10535}10536 10537 10538multiclass sve_mem_sst_128b_64_unscaled<string mnemonic, SDPatternOperator op> {10539 def NAME : sve_mem_sst_128b_64_unscaled<mnemonic>;10540 10541 def : InstAlias<mnemonic # " $Zt, $Pg, [$Zn]",10542 (!cast<Instruction>(NAME) Z_q:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>;10543 10544 def : Pat<(op (nxv2i64 Z_q:$Zt), (nxv2i1 PPR3bAny:$gp), (nxv2i64 ZPR64:$Zn), (i64 GPR64sp:$Rm), nxv2i64),10545 (!cast<Instruction>(NAME) Z_q:$Zt, PPR3bAny:$gp, ZPR64:$Zn, GPR64:$Rm)>;10546 def : Pat<(op (nxv4i32 Z_q:$Zt), (nxv4i1 PPR3bAny:$gp), (nxv2i64 ZPR64:$Zn), (i64 GPR64sp:$Rm), nxv4i32),10547 (!cast<Instruction>(NAME) Z_q:$Zt, PPR3bAny:$gp, ZPR64:$Zn, GPR64:$Rm)>;10548 def : Pat<(op (nxv8i16 Z_q:$Zt), (nxv8i1 PPR3bAny:$gp), (nxv2i64 ZPR64:$Zn), (i64 GPR64sp:$Rm), nxv8i16),10549 (!cast<Instruction>(NAME) Z_q:$Zt, PPR3bAny:$gp,ZPR64:$Zn, GPR64:$Rm)>;10550 def : Pat<(op (nxv16i8 Z_q:$Zt), (nxv16i1 PPR3bAny:$gp), (nxv2i64 ZPR64:$Zn), (i64 GPR64sp:$Rm), nxv16i8),10551 (!cast<Instruction>(NAME) Z_q:$Zt, PPR3bAny:$gp, ZPR64:$Zn, GPR64:$Rm)>;10552 10553 def : Pat<(op (nxv2f64 Z_q:$Zt), (nxv2i1 PPR3bAny:$gp), (nxv2i64 ZPR64:$Zn), (i64 GPR64sp:$Rm), nxv2f64),10554 (!cast<Instruction>(NAME) Z_q:$Zt, PPR3bAny:$gp, ZPR64:$Zn, GPR64:$Rm)>;10555 def : Pat<(op (nxv4f32 Z_q:$Zt), (nxv4i1 PPR3bAny:$gp), (nxv2i64 ZPR64:$Zn), (i64 GPR64sp:$Rm), nxv4f32),10556 (!cast<Instruction>(NAME) Z_q:$Zt, PPR3bAny:$gp, ZPR64:$Zn, GPR64:$Rm)>;10557 def : Pat<(op (nxv8f16 Z_q:$Zt), (nxv8i1 PPR3bAny:$gp), (nxv2i64 ZPR64:$Zn), (i64 GPR64sp:$Rm), nxv8f16),10558 (!cast<Instruction>(NAME) Z_q:$Zt, PPR3bAny:$gp, ZPR64:$Zn, GPR64:$Rm)>;10559 def : Pat<(op (nxv8bf16 Z_q:$Zt), (nxv8i1 PPR3bAny:$gp), (nxv2i64 ZPR64:$Zn), (i64 GPR64sp:$Rm), nxv8bf16),10560 (!cast<Instruction>(NAME) Z_q:$Zt, PPR3bAny:$gp, ZPR64:$Zn, GPR64:$Rm)>;10561}10562 10563 10564// SVE contiguous load (quadwords, scalar plus immediate)10565class sve_mem_128b_cld_si<bits<2> dtype, string mnemonic>10566 : I<(outs Z_q:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4),10567 mnemonic, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]",10568 "", []>, Sched<[]> {10569 bits<5> Zt;10570 bits<5> Rn;10571 bits<3> Pg;10572 bits<4> imm4;10573 let Inst{31-25} = 0b1010010;10574 let Inst{24-23} = dtype;10575 let Inst{22-20} = 0b001;10576 let Inst{19-16} = imm4;10577 let Inst{15-13} = 0b001;10578 let Inst{12-10} = Pg;10579 let Inst{9-5} = Rn;10580 let Inst{4-0} = Zt;10581 10582 let hasSideEffects = 0;10583 let mayLoad = 1;10584}10585 10586multiclass sve_mem_128b_cld_si<bits<2> dtype, string mnemonic> {10587 def NAME : sve_mem_128b_cld_si<dtype, mnemonic>;10588 10589 def : InstAlias<mnemonic # " $Zt, $Pg/z, [$Rn]",10590 (!cast<Instruction>(NAME) Z_q:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;10591 def : InstAlias<mnemonic # " $Zt, $Pg/z, [$Rn]",10592 (!cast<Instruction>(NAME) ZPR128:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>;10593 def : InstAlias<mnemonic # " $Zt, $Pg/z, [$Rn, $imm4, mul vl]",10594 (!cast<Instruction>(NAME) ZPR128:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>;10595}10596 10597 10598// SVE contiguous load (quadwords, scalar plus scalar)10599class sve_mem_128b_cld_ss<bits<2> dtype, string mnemonic, RegisterOperand gprsh_ty>10600 : I<(outs Z_q:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprsh_ty:$Rm),10601 mnemonic, "\t$Zt, $Pg/z, [$Rn, $Rm]", "",10602 []>, Sched<[]> {10603 bits<5> Zt;10604 bits<5> Rn;10605 bits<3> Pg;10606 bits<5> Rm;10607 let Inst{31-25} = 0b1010010;10608 let Inst{24-23} = dtype;10609 let Inst{22-21} = 0b00;10610 let Inst{20-16} = Rm;10611 let Inst{15-13} = 0b100;10612 let Inst{12-10} = Pg;10613 let Inst{9-5} = Rn;10614 let Inst{4-0} = Zt;10615 10616 let hasSideEffects = 0;10617 let mayLoad = 1;10618}10619 10620multiclass sve_mem_128b_cld_ss<bits<2> dtype, string mnemonic, RegisterOperand gprsh_ty> {10621 def NAME : sve_mem_128b_cld_ss<dtype, mnemonic, gprsh_ty>;10622 10623 def : InstAlias<mnemonic # " $Zt, $Pg/z, [$Rn, $Rm]",10624 (!cast<Instruction>(NAME) ZPR128:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprsh_ty:$Rm), 0>;10625}10626 10627 10628// SVE floating-point recursive reduction (quadwords)10629class sve2p1_fp_reduction_q<bits<2> sz, bits<3> opc, string mnemonic,10630 RegisterOperand zpr_ty, string vec_sfx>10631 : I<(outs V128:$Vd), (ins PPR3bAny:$Pg, zpr_ty:$Zn),10632 mnemonic, "\t$Vd." # vec_sfx # ", $Pg, $Zn",10633 "", []>, Sched<[]> {10634 bits<5> Vd;10635 bits<5> Zn;10636 bits<3> Pg;10637 let Inst{31-24} = 0b01100100;10638 let Inst{23-22} = sz;10639 let Inst{21-19} = 0b010;10640 let Inst{18-16} = opc;10641 let Inst{15-13} = 0b101;10642 let Inst{12-10} = Pg;10643 let Inst{9-5} = Zn;10644 let Inst{4-0} = Vd;10645 10646 let hasSideEffects = 0;10647 let mayRaiseFPException = 1;10648}10649 10650multiclass sve2p1_fp_reduction_q<bits<3> opc, string mnemonic, SDPatternOperator op> {10651 def _H : sve2p1_fp_reduction_q<0b01, opc, mnemonic, ZPR16, "8h">;10652 def _S : sve2p1_fp_reduction_q<0b10, opc, mnemonic, ZPR32, "4s">;10653 def _D : sve2p1_fp_reduction_q<0b11, opc, mnemonic, ZPR64, "2d">;10654 10655 def : SVE_2_Op_Pat<v8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;10656 def : SVE_2_Op_Pat<v4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;10657 def : SVE_2_Op_Pat<v2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;10658}10659 10660 10661// SVE Permute Vector - Quadwords (DUPQ)10662class sve2p1_dupq<bits<5> ind_tsz, string mnemonic, ZPRRegOp zprty, Operand itype>10663 : I<(outs zprty:$Zd), (ins zprty:$Zn, itype:$index),10664 mnemonic, "\t$Zd, $Zn$index",10665 "", []>, Sched<[]> {10666 bits<5> Zd;10667 bits<5> Zn;10668 let Inst{31-21} = 0b00000101001;10669 let Inst{20-16} = ind_tsz;10670 let Inst{15-10} = 0b001001;10671 let Inst{9-5} = Zn;10672 let Inst{4-0} = Zd;10673 10674 let hasSideEffects = 0;10675}10676 10677multiclass sve2p1_dupq<string mnemonic, SDPatternOperator Op> {10678 def _B : sve2p1_dupq<{?, ?, ?, ?, 1}, mnemonic, ZPR8, VectorIndexB32b_timm> {10679 bits<4> index;10680 let Inst{20-17} = index;10681 }10682 def _H : sve2p1_dupq<{?, ?, ?, 1, 0}, mnemonic, ZPR16, VectorIndexH32b_timm> {10683 bits<3> index;10684 let Inst{20-18} = index;10685 }10686 def _S : sve2p1_dupq<{?, ?, 1, 0, 0}, mnemonic, ZPR32, VectorIndexS32b_timm> {10687 bits<2> index;10688 let Inst{20-19} = index;10689 }10690 def _D : sve2p1_dupq<{?, 1, 0, 0, 0}, mnemonic, ZPR64, VectorIndexD32b_timm> {10691 bits<1> index;10692 let Inst{20} = index;10693 }10694 10695 def : SVE_2_Op_Imm_Pat<nxv16i8, Op, nxv16i8, i32, VectorIndexB32b_timm, !cast<Instruction>(NAME # _B)>;10696 def : SVE_2_Op_Imm_Pat<nxv8i16, Op, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>;10697 def : SVE_2_Op_Imm_Pat<nxv4i32, Op, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>;10698 def : SVE_2_Op_Imm_Pat<nxv2i64, Op, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>;10699 10700 def : SVE_2_Op_Imm_Pat<nxv8f16, Op, nxv8f16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>;10701 def : SVE_2_Op_Imm_Pat<nxv4f32, Op, nxv4f32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>;10702 def : SVE_2_Op_Imm_Pat<nxv2f64, Op, nxv2f64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>;10703 def : SVE_2_Op_Imm_Pat<nxv8bf16, Op, nxv8bf16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>;10704}10705 10706 10707// SVE Permute Vector - Quadwords (EXTQ)10708class sve2p1_extq<string mnemonic>10709 : I<(outs ZPR8:$Zdn), (ins ZPR8:$_Zdn, ZPR8:$Zm, timm32_0_15:$imm4),10710 mnemonic, "\t$Zdn, $_Zdn, $Zm, $imm4",10711 "", []>, Sched<[]> {10712 bits<5> Zdn;10713 bits<5> Zm;10714 bits<4> imm4;10715 let Inst{31-20} = 0b000001010110;10716 let Inst{19-16} = imm4;10717 let Inst{15-10} = 0b001001;10718 let Inst{9-5} = Zm;10719 let Inst{4-0} = Zdn;10720 10721 let Constraints = "$Zdn = $_Zdn";10722 let DestructiveInstType = DestructiveOther;10723 let ElementSize = ZPR8.ElementSize;10724 let hasSideEffects = 0;10725}10726 10727multiclass sve2p1_extq<string mnemonic, SDPatternOperator Op> {10728 def NAME : sve2p1_extq<mnemonic>;10729 def : SVE_3_Op_Imm_Pat<nxv16i8, Op, nxv16i8, nxv16i8, i32, timm32_0_15, !cast<Instruction>(NAME)>;10730 def : SVE_3_Op_Imm_Pat<nxv8i16, Op, nxv8i16, nxv8i16, i32, extq_timm32_0_7m2, !cast<Instruction>(NAME)>;10731 def : SVE_3_Op_Imm_Pat<nxv4i32, Op, nxv4i32, nxv4i32, i32, extq_timm32_0_3m4, !cast<Instruction>(NAME)>;10732 def : SVE_3_Op_Imm_Pat<nxv2i64, Op, nxv2i64, nxv2i64, i32, extq_timm32_0_1m8, !cast<Instruction>(NAME)>;10733 10734 def : SVE_3_Op_Imm_Pat<nxv8f16, Op, nxv8f16, nxv8f16, i32, extq_timm32_0_7m2, !cast<Instruction>(NAME)>;10735 def : SVE_3_Op_Imm_Pat<nxv4f32, Op, nxv4f32, nxv4f32, i32, extq_timm32_0_3m4, !cast<Instruction>(NAME)>;10736 def : SVE_3_Op_Imm_Pat<nxv2f64, Op, nxv2f64, nxv2f64, i32, extq_timm32_0_1m8, !cast<Instruction>(NAME)>;10737 def : SVE_3_Op_Imm_Pat<nxv8bf16, Op, nxv8bf16, nxv8bf16, i32, extq_timm32_0_7m2, !cast<Instruction>(NAME)>;10738}10739 10740// SVE move predicate from vector10741class sve2p1_vector_to_pred<bits<4> opc, string mnemonic,10742 PPRRegOp ppr_ty, Operand itype>10743 : I<(outs ppr_ty:$Pd), (ins ZPRAny:$Zn, itype:$index),10744 mnemonic, "\t$Pd, $Zn$index",10745 "", []>, Sched<[]> {10746 bits<4> Pd;10747 bits<5> Zn;10748 let Inst{31-24} = 0b00000101;10749 let Inst{23-22} = opc{3-2};10750 let Inst{21-19} = 0b101;10751 let Inst{18-17} = opc{1-0};10752 let Inst{16-10} = 0b0001110;10753 let Inst{9-5} = Zn;10754 let Inst{4} = 0b0;10755 let Inst{3-0} = Pd;10756 10757 let hasSideEffects = 0;10758}10759 10760multiclass sve2p1_vector_to_pred<string mnemonic, SDPatternOperator Op_lane, SDPatternOperator Op> {10761 def _B : sve2p1_vector_to_pred<{0, 0, 0, 1}, mnemonic, PPR8, VectorIndex032b> {10762 bits<0> index;10763 }10764 def _H : sve2p1_vector_to_pred<{0, 0, 1, ?}, mnemonic, PPR16, VectorIndexD32b> {10765 bits<1> index;10766 let Inst{17} = index;10767 }10768 def _S : sve2p1_vector_to_pred<{0, 1, ?, ?}, mnemonic, PPR32, VectorIndexS32b> {10769 bits<2> index;10770 let Inst{18-17} = index;10771 }10772 def _D : sve2p1_vector_to_pred<{1, ?, ?, ?}, mnemonic, PPR64, VectorIndexH32b> {10773 bits<3> index;10774 let Inst{22} = index{2};10775 let Inst{18-17} = index{1-0};10776 }10777 10778 def : InstAlias<mnemonic # "\t$Pd, $Zn",10779 (!cast<Instruction>(NAME # _B) PPR8:$Pd, ZPRAny:$Zn, 0), 1>;10780 def : InstAlias<mnemonic # "\t$Pd, $Zn",10781 (!cast<Instruction>(NAME # _H) PPR16:$Pd, ZPRAny:$Zn, 0), 0>;10782 def : InstAlias<mnemonic # "\t$Pd, $Zn",10783 (!cast<Instruction>(NAME # _S) PPR32:$Pd, ZPRAny:$Zn, 0), 0>;10784 def : InstAlias<mnemonic # "\t$Pd, $Zn",10785 (!cast<Instruction>(NAME # _D) PPR64:$Pd, ZPRAny:$Zn, 0), 0>;10786 10787 // any_lane10788 def : Pat<(nxv16i1 (Op_lane (nxv16i8 ZPRAny:$Zn), (i32 timm32_0_0:$Idx))),10789 (!cast<Instruction>(NAME # _B) ZPRAny:$Zn, timm32_0_0:$Idx)>;10790 def : Pat<(nxv8i1 (Op_lane (nxv8i16 ZPRAny:$Zn), (i32 timm32_0_1:$Idx))),10791 (!cast<Instruction>(NAME # _H) ZPRAny:$Zn, timm32_0_1:$Idx)>;10792 def : Pat<(nxv4i1 (Op_lane (nxv4i32 ZPRAny:$Zn), (i32 timm32_0_3:$Idx))),10793 (!cast<Instruction>(NAME # _S) ZPRAny:$Zn, timm32_0_3:$Idx)>;10794 def : Pat<(nxv2i1 (Op_lane (nxv2i64 ZPRAny:$Zn), (i32 timm32_0_7:$Idx))),10795 (!cast<Instruction>(NAME # _D) ZPRAny:$Zn, timm32_0_7:$Idx)>;10796 // lane_010797 def : Pat<(nxv16i1 (Op (nxv16i8 ZPRAny:$Zn))),10798 (!cast<Instruction>(NAME # _B) ZPRAny:$Zn, 0)>;10799 def : Pat<(nxv8i1 (Op (nxv8i16 ZPRAny:$Zn))),10800 (!cast<Instruction>(NAME # _H) ZPRAny:$Zn, 0)>;10801 def : Pat<(nxv4i1 (Op (nxv4i32 ZPRAny:$Zn))),10802 (!cast<Instruction>(NAME # _S) ZPRAny:$Zn, 0)>;10803 def : Pat<(nxv2i1 (Op (nxv2i64 ZPRAny:$Zn))),10804 (!cast<Instruction>(NAME # _D) ZPRAny:$Zn, 0)>;10805}10806 10807 10808// SVE move predicate into vector10809class sve2p1_pred_to_vector<bits<4> opc, string mnemonic,10810 PPRRegOp ppr_ty, Operand itype>10811 : I<(outs ZPRAny:$Zd), (ins ZPRAny:$_Zd, itype:$index, ppr_ty:$Pn),10812 mnemonic, "\t$Zd$index, $Pn",10813 "", []>, Sched<[]> {10814 bits<5> Zd;10815 bits<4> Pn;10816 let Inst{31-24} = 0b00000101;10817 let Inst{23-22} = opc{3-2};10818 let Inst{21-19} = 0b101;10819 let Inst{18-17} = opc{1-0};10820 let Inst{16-9} = 0b10011100;10821 let Inst{8-5} = Pn;10822 let Inst{4-0} = Zd;10823 10824 let Constraints = "$Zd = $_Zd";10825 let hasSideEffects = 0;10826}10827 10828multiclass sve2p1_pred_to_vector<string mnemonic, SDPatternOperator MergeOp,10829 SDPatternOperator ZeroOp> {10830 def _B : sve2p1_pred_to_vector<{0, 0, 0, 1}, mnemonic, PPR8, VectorIndex0> {10831 bits<0> index;10832 }10833 def _H : sve2p1_pred_to_vector<{0, 0, 1, ?}, mnemonic, PPR16, VectorIndexD32b> {10834 bits<1> index;10835 let Inst{17} = index;10836 }10837 def _S : sve2p1_pred_to_vector<{0, 1, ?, ?}, mnemonic, PPR32, VectorIndexS32b> {10838 bits<2> index;10839 let Inst{18-17} = index;10840 }10841 def _D : sve2p1_pred_to_vector<{1, ?, ?, ?}, mnemonic, PPR64, VectorIndexH32b> {10842 bits<3> index;10843 let Inst{22} = index{2};10844 let Inst{18-17} = index{1-0};10845 }10846 10847 def : InstAlias<mnemonic # "\t$Zd, $Pn",10848 (!cast<Instruction>(NAME # _B) ZPRAny:$Zd, 0, PPR8:$Pn), 1>;10849 def : InstAlias<mnemonic # "\t$Zd, $Pn",10850 (!cast<Instruction>(NAME # _H) ZPRAny:$Zd, 0, PPR16:$Pn), 0>;10851 def : InstAlias<mnemonic # "\t$Zd, $Pn",10852 (!cast<Instruction>(NAME # _S) ZPRAny:$Zd, 0, PPR32:$Pn), 0>;10853 def : InstAlias<mnemonic # "\t$Zd, $Pn",10854 (!cast<Instruction>(NAME # _D) ZPRAny:$Zd, 0, PPR64:$Pn), 0>;10855 10856 // Merge10857 def : Pat<(nxv8i16 (MergeOp (nxv8i16 ZPRAny:$Zd), (nxv8i1 PPR16:$Pn), (i32 timm32_1_1:$Idx))),10858 (!cast<Instruction>(NAME # _H) ZPRAny:$Zd, timm32_1_1:$Idx, PPR16:$Pn)>;10859 def : Pat<(nxv4i32 (MergeOp (nxv4i32 ZPRAny:$Zd), (nxv4i1 PPR32:$Pn), (i32 timm32_1_3:$Idx))),10860 (!cast<Instruction>(NAME # _S) ZPRAny:$Zd, timm32_1_3:$Idx, PPR32:$Pn)>;10861 def : Pat<(nxv2i64 (MergeOp (nxv2i64 ZPRAny:$Zd), (nxv2i1 PPR64:$Pn), (i32 timm32_1_7:$Idx))),10862 (!cast<Instruction>(NAME # _D) ZPRAny:$Zd, timm32_1_7:$Idx, PPR64:$Pn)>;10863 10864 // Zero10865 def : Pat<(nxv16i8 (ZeroOp (nxv16i1 PPR8:$Pn))),10866 (!cast<Instruction>(NAME # _B) (IMPLICIT_DEF), 0, PPR8:$Pn)>;10867 def : Pat<(nxv8i16 (ZeroOp (nxv8i1 PPR16:$Pn))),10868 (!cast<Instruction>(NAME # _H) (IMPLICIT_DEF), 0, PPR16:$Pn)>;10869 def : Pat<(nxv4i32 (ZeroOp (nxv4i1 PPR32:$Pn))),10870 (!cast<Instruction>(NAME # _S) (IMPLICIT_DEF), 0, PPR32:$Pn)>;10871 def : Pat<(nxv2i64 (ZeroOp (nxv2i1 PPR64:$Pn))),10872 (!cast<Instruction>(NAME # _D) (IMPLICIT_DEF), 0, PPR64:$Pn)>;10873}10874 10875 10876// SVE bitwise logical/add/min/max reductions (quadwords)10877class sve2p1_int_reduce_q<bits<2> sz, bits<4> opc, string mnemonic,10878 RegisterOperand zpr_ty, string vec_sfx>10879 : I<(outs V128:$Vd), (ins PPR3bAny:$Pg, zpr_ty:$Zn),10880 mnemonic, "\t$Vd." # vec_sfx # ", $Pg, $Zn",10881 "", []>, Sched<[]> {10882 bits<5> Vd;10883 bits<5> Zn;10884 bits<3> Pg;10885 let Inst{31-24} = 0b00000100;10886 let Inst{23-22} = sz;10887 let Inst{21} = 0b0;10888 let Inst{20-19} = opc{3-2};10889 let Inst{18} = 0b1;10890 let Inst{17-16} = opc{1-0};10891 let Inst{15-13} = 0b001;10892 let Inst{12-10} = Pg;10893 let Inst{9-5} = Zn;10894 let Inst{4-0} = Vd;10895 10896 let hasSideEffects = 0;10897}10898 10899multiclass sve2p1_int_reduce_q<bits<4> opc, string mnemonic, SDPatternOperator op> {10900 def _B : sve2p1_int_reduce_q<0b00, opc, mnemonic, ZPR8, "16b">;10901 def _H : sve2p1_int_reduce_q<0b01, opc, mnemonic, ZPR16, "8h">;10902 def _S : sve2p1_int_reduce_q<0b10, opc, mnemonic, ZPR32, "4s">;10903 def _D : sve2p1_int_reduce_q<0b11, opc, mnemonic, ZPR64, "2d">;10904 10905 def : SVE_2_Op_Pat<v16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;10906 def : SVE_2_Op_Pat<v8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>;10907 def : SVE_2_Op_Pat<v4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;10908 def : SVE_2_Op_Pat<v2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;10909}10910 10911 10912// SVE permute vector elements (quadwords)10913class sve2p1_permute_vec_elems_q<bits<2> sz, bits<3> opc, string mnemonic,10914 ZPRRegOp zpr_ty, RegisterOperand src1_ty>10915 : I<(outs zpr_ty:$Zd), (ins src1_ty:$Zn, zpr_ty:$Zm),10916 mnemonic, "\t$Zd, $Zn, $Zm",10917 "", []>, Sched<[]> {10918 bits<5> Zd;10919 bits<5> Zn;10920 bits<5> Zm;10921 let Inst{31-24} = 0b01000100;10922 let Inst{23-22} = sz;10923 let Inst{21} = 0b0;10924 let Inst{20-16} = Zm;10925 let Inst{15-13} = 0b111;10926 let Inst{12-10} = opc;10927 let Inst{9-5} = Zn;10928 let Inst{4-0} = Zd;10929 10930 let hasSideEffects = 0;10931}10932 10933multiclass sve2p1_permute_vec_elems_q<bits<3> opc, string mnemonic,10934 SDPatternOperator op> {10935 def _B : sve2p1_permute_vec_elems_q<0b00, opc, mnemonic, ZPR8, ZPR8>;10936 def _H : sve2p1_permute_vec_elems_q<0b01, opc, mnemonic, ZPR16, ZPR16>;10937 def _S : sve2p1_permute_vec_elems_q<0b10, opc, mnemonic, ZPR32, ZPR32>;10938 def _D : sve2p1_permute_vec_elems_q<0b11, opc, mnemonic, ZPR64, ZPR64>;10939 10940 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;10941 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;10942 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;10943 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;10944 10945 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;10946 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;10947 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;10948 10949 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>;10950}10951 10952multiclass sve2p1_tblq<string mnemonic, SDPatternOperator op> {10953 def _B : sve2p1_permute_vec_elems_q<0b00, 0b110, mnemonic, ZPR8, Z_b>;10954 def _H : sve2p1_permute_vec_elems_q<0b01, 0b110, mnemonic, ZPR16, Z_h>;10955 def _S : sve2p1_permute_vec_elems_q<0b10, 0b110, mnemonic, ZPR32, Z_s>;10956 def _D : sve2p1_permute_vec_elems_q<0b11, 0b110, mnemonic, ZPR64, Z_d>;10957 10958 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;10959 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;10960 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;10961 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;10962 10963 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>;10964 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>;10965 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>;10966 10967 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i16, !cast<Instruction>(NAME # _H)>;10968}10969 10970//===----------------------------------------------------------------------===//10971// SVE2 FP8 Instructions10972//===----------------------------------------------------------------------===//10973 10974// FP8 upconvert10975class sve2_fp8_cvt_single<bit L, bits<2> opc, string mnemonic,10976 ZPRRegOp dst_ty, ZPRRegOp src_ty>10977 : I<(outs dst_ty:$Zd), (ins src_ty:$Zn),10978 mnemonic, "\t$Zd, $Zn",10979 "", []>, Sched<[]>{10980 bits<5> Zd;10981 bits<5> Zn;10982 let Inst{31-17} = 0b011001010000100;10983 let Inst{16} = L;10984 let Inst{15-12} = 0b0011;10985 let Inst{11-10} = opc;10986 let Inst{9-5} = Zn;10987 let Inst{4-0} = Zd;10988 let Uses = [FPMR, FPCR];10989 10990 let mayLoad = 1;10991 let mayStore = 0;10992}10993 10994multiclass sve2_fp8_cvt_single<bit L, bits<2> opc, string mnemonic, ValueType vtd, SDPatternOperator op> {10995 def _BtoH : sve2_fp8_cvt_single<L, opc, mnemonic, ZPR16, ZPR8>;10996 10997 def : SVE_1_Op_Pat<vtd, op, nxv16i8, !cast<Instruction>(NAME # _BtoH)>;10998}10999 11000// FP8 downconvert11001class sve2_fp8_down_cvt_single<bits<2> opc, string mnemonic,11002 ZPRRegOp dst_ty, RegisterOperand src_ty>11003 : I<(outs dst_ty:$Zd), (ins src_ty:$Zn),11004 mnemonic, "\t$Zd, $Zn",11005 "", []>, Sched<[]>{11006 bits<5> Zd;11007 bits<4> Zn;11008 let Inst{31-12} = 0b01100101000010100011;11009 let Inst{11-10} = opc;11010 let Inst{9-6} = Zn;11011 let Inst{5} = 0b0;11012 let Inst{4-0} = Zd;11013 let Uses = [FPMR, FPCR];11014 11015 let mayLoad = 1;11016 let mayStore = 0;11017}11018 11019multiclass sve2_fp8_down_cvt_single<bits<2> opc, string mnemonic, RegisterOperand src,11020 ValueType ty, SDPatternOperator op> {11021 def NAME : sve2_fp8_down_cvt_single<opc, mnemonic, ZPR8, src>;11022 11023 def : Pat<(nxv16i8 (op ty:$Zn1, ty:$Zn2)),11024 (!cast<Instruction>(NAME) (REG_SEQUENCE ZPR2Mul2, $Zn1, zsub0, $Zn2, zsub1))>;11025}11026 11027class sve2_fp8_down_cvt_single_top<bits<2> opc, string mnemonic, RegisterOperand src_ty>11028 : I<(outs ZPR8:$Zd), (ins ZPR8:$_Zd, src_ty:$Zn), mnemonic, "\t$Zd, $Zn","", []>, Sched<[]> {11029 bits<5> Zd;11030 bits<4> Zn;11031 11032 let Inst{31-12} = 0b01100101000010100011;11033 let Inst{11-10} = opc;11034 let Inst{9-6} = Zn;11035 let Inst{5} = 0b0;11036 let Inst{4-0} = Zd;11037 11038 let Constraints = "$Zd = $_Zd";11039 let DestructiveInstType = DestructiveOther;11040 let ElementSize = ZPR8.ElementSize;11041 11042 let Uses = [FPMR, FPCR];11043 let mayLoad = 1;11044 let mayStore = 0;11045}11046 11047multiclass sve2_fp8_down_cvt_single_top<bits<2> opc, string mnemonic, RegisterOperand src_ty,11048 ValueType ty, SDPatternOperator op> {11049 def NAME : sve2_fp8_down_cvt_single_top<opc, mnemonic, src_ty>;11050 11051 def : Pat<(nxv16i8 (op nxv16i8:$Zd, ty:$Zn1, ty:$Zn2)),11052 (!cast<Instruction>(NAME) $Zd, (REG_SEQUENCE ZPR2Mul2, $Zn1, zsub0, $Zn2, zsub1))>;11053}11054 11055// FP8 Widening Multiply-Add Long - Indexed Group11056class sve2_fp8_mla_long_by_indexed_elem<bit T, string mnemonic>11057 : I<(outs ZPR16:$Zda),11058 (ins ZPR16:$_Zda, ZPR8:$Zn, ZPR3b8:$Zm, VectorIndexB32b:$imm4),11059 mnemonic, "\t$Zda, $Zn, $Zm$imm4",11060 "", []>, Sched<[]>{11061 bits<5> Zda;11062 bits<5> Zn;11063 bits<3> Zm;11064 bits<4> imm4;11065 let Inst{31-24} = 0b01100100;11066 let Inst{23} = T;11067 let Inst{22-21} = 0b01;11068 let Inst{20-19} = imm4{3-2};11069 let Inst{18-16} = Zm;11070 let Inst{15-12} = 0b0101;11071 let Inst{11-10} = imm4{1-0};11072 let Inst{9-5} = Zn;11073 let Inst{4-0} = Zda;11074 let Constraints = "$Zda = $_Zda";11075 let DestructiveInstType = DestructiveOther;11076 let ElementSize = ZPR16.ElementSize;11077 let Uses = [FPMR, FPCR];11078}11079 11080multiclass sve2_fp8_mla_long_by_indexed_elem<bit T, string mnemonic, SDPatternOperator op> {11081 def NAME : sve2_fp8_mla_long_by_indexed_elem<T, mnemonic>;11082 11083 def : SVE_4_Op_Pat<nxv8f16, op, nxv8f16, nxv16i8, nxv16i8, i32, !cast<Instruction>(NAME)>;11084}11085 11086// FP8 Widening Multiply-Add (Long)/(Long Long) Group11087class sve2_fp8_mla<bits<3>opc, ZPRRegOp dst_ty, string mnemonic>11088 : I<(outs dst_ty:$Zda),11089 (ins dst_ty:$_Zda, ZPR8:$Zn, ZPR8:$Zm),11090 mnemonic, "\t$Zda, $Zn, $Zm",11091 "", []>, Sched<[]>{11092 bits<5> Zda;11093 bits<5> Zn;11094 bits<5> Zm;11095 let Inst{31-24} = 0b01100100;11096 let Inst{23} = opc{2};11097 let Inst{22-21} = 0b01;11098 let Inst{20-16} = Zm;11099 let Inst{15-14} = 0b10;11100 let Inst{13-12} = opc{1-0};11101 let Inst{11-10} = 0b10;11102 let Inst{9-5} = Zn;11103 let Inst{4-0} = Zda;11104 let Constraints = "$Zda = $_Zda";11105 let DestructiveInstType = DestructiveOther;11106 let ElementSize = dst_ty.ElementSize;11107 let Uses = [FPMR, FPCR];11108}11109 11110multiclass sve2_fp8_mla<bits<3> opc, ZPRRegOp dst_ty, string mnemonic, ValueType vta, SDPatternOperator op> {11111 def NAME : sve2_fp8_mla<opc, dst_ty, mnemonic>;11112 11113 def : SVE_3_Op_Pat<vta, op, vta, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>;11114}11115 11116// FP8 Widening Multiply-Add Long Long - Indexed Group11117class sve2_fp8_mla_long_long_by_indexed_elem<bits<2> TT, string mnemonic>11118 : I<(outs ZPR32:$Zda),11119 (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR3b8:$Zm, VectorIndexB32b:$imm4),11120 mnemonic, "\t$Zda, $Zn, $Zm$imm4",11121 "", []>, Sched<[]>{11122 bits<5> Zda;11123 bits<5> Zn;11124 bits<3> Zm;11125 bits<4> imm4;11126 let Inst{31-24} = 0b01100100;11127 let Inst{23-22} = TT;11128 let Inst{21} = 0b1;11129 let Inst{20-19} = imm4{3-2};11130 let Inst{18-16} = Zm;11131 let Inst{15-12} = 0b1100;11132 let Inst{11-10} = imm4{1-0};11133 let Inst{9-5} = Zn;11134 let Inst{4-0} = Zda;11135 let Constraints = "$Zda = $_Zda";11136 let DestructiveInstType = DestructiveOther;11137 let ElementSize = ZPR32.ElementSize;11138 let Uses = [FPMR, FPCR];11139}11140 11141multiclass sve2_fp8_mla_long_long_by_indexed_elem<bits<2> TT, string mnemonic, SDPatternOperator op> {11142 def NAME : sve2_fp8_mla_long_long_by_indexed_elem<TT, mnemonic>;11143 11144 def : SVE_4_Op_Pat<nxv4f32, op, nxv4f32, nxv16i8, nxv16i8, i32, !cast<Instruction>(NAME)>;11145}11146 11147// FP8 Matrix Multiply-accumulate Group11148class sve2_fp8_mmla<bit opc, ZPRRegOp dst_ty, string mnemonic>11149 : I<(outs dst_ty:$Zda),11150 (ins dst_ty:$_Zda, ZPR8:$Zn, ZPR8:$Zm),11151 mnemonic, "\t$Zda, $Zn, $Zm",11152 "", []>, Sched<[]>{11153 bits<5> Zda;11154 bits<5> Zn;11155 bits<5> Zm;11156 let Inst{31-23} = 0b011001000;11157 let Inst{22} = opc;11158 let Inst{21} = 0b1;11159 let Inst{20-16} = Zm;11160 let Inst{15-10} = 0b111000;11161 let Inst{9-5} = Zn;11162 let Inst{4-0} = Zda;11163 let Constraints = "$Zda = $_Zda";11164 let DestructiveInstType = DestructiveOther;11165 let ElementSize = dst_ty.ElementSize;11166 let Uses = [FPMR, FPCR];11167}11168 11169class sve_fp8_dot_indexed<bits<4> opc, ZPRRegOp dst_ty, Operand iop_ty, string mnemonic>11170: I<(outs dst_ty:$Zda), (ins dst_ty:$_Zda, ZPR8:$Zn, ZPR3b8:$Zm, iop_ty:$iop),11171 mnemonic, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> {11172 bits<5> Zda;11173 bits<5> Zn;11174 bits<3> Zm;11175 let Inst{31-23} = 0b011001000;11176 let Inst{22} = opc{3};11177 let Inst{21} = 0b1;11178 let Inst{20-19} = opc{2-1};11179 let Inst{18-16} = Zm;11180 let Inst{15-12} = 0b0100;11181 let Inst{11} = opc{0};11182 let Inst{10} = 0b1;11183 let Inst{9-5} = Zn;11184 let Inst{4-0} = Zda;11185 11186 let Uses = [FPMR, FPCR];11187 let Constraints = "$Zda = $_Zda";11188 let DestructiveInstType = DestructiveOther;11189 let hasSideEffects = 0;11190 let mayRaiseFPException = 1;11191 11192 let mayLoad = 1;11193 let mayStore = 0;11194}11195 11196// FP8 Widening Dot-Product - Indexed Group11197multiclass sve2_fp8_dot_indexed_h<string asm, SDPatternOperator op> {11198 def NAME : sve_fp8_dot_indexed<{0, ?, ?, ?}, ZPR16, VectorIndexH32b, asm> {11199 bits<3> iop;11200 11201 let Inst{20-19} = iop{2-1};11202 let Inst{11} = iop{0};11203 }11204 11205 def : SVE_4_Op_Pat<nxv8f16, op, nxv8f16, nxv16i8, nxv16i8, i32, !cast<Instruction>(NAME)>;11206}11207 11208multiclass sve2_fp8_dot_indexed_s<string asm, SDPatternOperator op> {11209 def NAME : sve_fp8_dot_indexed<{1, ?, ?, 0}, ZPR32, VectorIndexS32b, asm> {11210 bits<2> iop;11211 11212 let Inst{20-19} = iop{1-0};11213 }11214 11215 def : SVE_4_Op_Pat<nxv4f32, op, nxv4f32, nxv16i8, nxv16i8, i32, !cast<Instruction>(NAME)>;11216}11217 11218// Look up table11219class sve2_lut_vector_index<ZPRRegOp zd_ty, RegisterOperand zn_ty,11220 Operand idx_ty, bits<4>opc, string mnemonic>11221 : I<(outs zd_ty:$Zd), (ins zn_ty:$Zn, ZPRAny:$Zm, idx_ty:$idx),11222 mnemonic, "\t$Zd, $Zn, $Zm$idx",11223 "", []>, Sched<[]> {11224 bits<5> Zd;11225 bits<5> Zn;11226 bits<5> Zm;11227 let Inst{31-24} = 0b01000101;11228 let Inst{22} = opc{3};11229 let Inst{21} = 0b1;11230 let Inst{20-16} = Zm;11231 let Inst{15-13} = 0b101;11232 let Inst{12-10} = opc{2-0};11233 let Inst{9-5} = Zn;11234 let Inst{4-0} = Zd;11235}11236 11237// Look up table read with 2-bit indices11238multiclass sve2_luti2_vector_index<string mnemonic> {11239 def _B : sve2_lut_vector_index<ZPR8, Z_b, VectorIndexS32b, {?, 0b100}, mnemonic> {11240 bits<2> idx;11241 let Inst{23-22} = idx;11242 }11243 def _H : sve2_lut_vector_index<ZPR16, Z_h, VectorIndexH32b, {?,?,0b10}, mnemonic> {11244 bits<3> idx;11245 let Inst{23-22} = idx{2-1};11246 let Inst{12} = idx{0};11247 }11248 11249 def : SVE_3_Op_Imm_Pat<nxv16i8, int_aarch64_sve_luti2_lane, nxv16i8, nxv16i8,11250 i32, timm32_0_3, !cast<Instruction>(NAME # _B)>;11251 def : SVE_3_Op_Imm_Pat<nxv8i16, int_aarch64_sve_luti2_lane, nxv8i16, nxv16i8,11252 i32, timm32_0_7, !cast<Instruction>(NAME # _H)>;11253 def : SVE_3_Op_Imm_Pat<nxv8f16, int_aarch64_sve_luti2_lane, nxv8f16, nxv16i8,11254 i32, timm32_0_7, !cast<Instruction>(NAME # _H)>;11255 def : SVE_3_Op_Imm_Pat<nxv8bf16, int_aarch64_sve_luti2_lane, nxv8bf16, nxv16i8,11256 i32, timm32_0_7, !cast<Instruction>(NAME # _H)>;11257}11258 11259// Look up table read with 4-bit indices11260multiclass sve2_luti4_vector_index<string mnemonic> {11261 def _B : sve2_lut_vector_index<ZPR8, Z_b, VectorIndexD32b, 0b1001, mnemonic> {11262 bit idx;11263 let Inst{23} = idx;11264 }11265 def _H : sve2_lut_vector_index<ZPR16, Z_h, VectorIndexS32b, {?, 0b111}, mnemonic> {11266 bits<2> idx;11267 let Inst{23-22} = idx;11268 }11269 11270 def : SVE_3_Op_Imm_Pat<nxv16i8, int_aarch64_sve_luti4_lane, nxv16i8, nxv16i8,11271 i32, timm32_0_1, !cast<Instruction>(NAME # _B)>;11272 def : SVE_3_Op_Imm_Pat<nxv8i16, int_aarch64_sve_luti4_lane, nxv8i16, nxv16i8,11273 i32, timm32_0_3, !cast<Instruction>(NAME # _H)>;11274 def : SVE_3_Op_Imm_Pat<nxv8f16, int_aarch64_sve_luti4_lane, nxv8f16, nxv16i8,11275 i32, timm32_0_3, !cast<Instruction>(NAME # _H)>;11276 def : SVE_3_Op_Imm_Pat<nxv8bf16, int_aarch64_sve_luti4_lane, nxv8bf16, nxv16i8,11277 i32, timm32_0_3, !cast<Instruction>(NAME # _H)>;11278}11279 11280// Look up table read with 4-bit indices (two contiguous registers)11281multiclass sve2_luti4_vector_vg2_index<string mnemonic> {11282 def NAME : sve2_lut_vector_index<ZPR16, ZZ_h, VectorIndexS32b, {?, 0b101}, mnemonic> {11283 bits<2> idx;11284 let Inst{23-22} = idx;11285 }11286 11287 def : Pat<(nxv8i16 (int_aarch64_sve_luti4_lane_x2 nxv8i16:$Op1, nxv8i16:$Op2,11288 nxv16i8:$Op3, (i32 timm32_0_3:$Op4))),11289 (nxv8i16 (!cast<Instruction>(NAME) (REG_SEQUENCE ZPR2, nxv8i16:$Op1, zsub0,11290 nxv8i16:$Op2, zsub1),11291 nxv16i8:$Op3, timm32_0_3:$Op4))>;11292 def : Pat<(nxv8f16 (int_aarch64_sve_luti4_lane_x2 nxv8f16:$Op1, nxv8f16:$Op2,11293 nxv16i8:$Op3, (i32 timm32_0_3:$Op4))),11294 (nxv8f16 (!cast<Instruction>(NAME) (REG_SEQUENCE ZPR2, nxv8f16:$Op1, zsub0,11295 nxv8f16:$Op2, zsub1),11296 nxv16i8:$Op3, timm32_0_3:$Op4))>;11297 def : Pat<(nxv8bf16 (int_aarch64_sve_luti4_lane_x2 nxv8bf16:$Op1, nxv8bf16:$Op2,11298 nxv16i8:$Op3, (i32 timm32_0_3:$Op4))),11299 (nxv8bf16 (!cast<Instruction>(NAME) (REG_SEQUENCE ZPR2, nxv8bf16:$Op1, zsub0,11300 nxv8bf16:$Op2, zsub1),11301 nxv16i8:$Op3, timm32_0_3:$Op4))>;11302}11303 11304// Look up table read with 6-bit indices11305multiclass sve2_luti6_vector_index<string mnemonic> {11306 def _H : sve2_lut_vector_index<ZPR16, ZZ_h, VectorIndexD32b, 0b1011, mnemonic> {11307 bit idx;11308 let Inst{23} = idx;11309 }11310}11311 11312// Look up table11313class sve2_luti6_vector<string mnemonic>11314 : I<(outs ZPR8:$Zd), (ins ZZ_b:$Zn, ZPRAny:$Zm),11315 mnemonic, "\t$Zd, $Zn, $Zm",11316 "", []>, Sched<[]> {11317 bits<5> Zd;11318 bits<5> Zn;11319 bits<5> Zm;11320 let Inst{31-21} = 0b01000101001;11321 let Inst{20-16} = Zm;11322 let Inst{15-10} = 0b101011;11323 let Inst{9-5} = Zn;11324 let Inst{4-0} = Zd;11325}11326 11327//===----------------------------------------------------------------------===//11328// Checked Pointer Arithmetic (FEAT_CPA)11329//===----------------------------------------------------------------------===//11330class sve_int_mad_cpa<string asm>11331 : I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, ZPR64:$Zm, ZPR64:$Za),11332 asm, "\t$Zdn, $Zm, $Za", "", []>, Sched<[]> {11333 bits<5> Zdn;11334 bits<5> Zm;11335 bits<5> Za;11336 let Inst{31-24} = 0b01000100;11337 let Inst{23-22} = 0b11; // sz11338 let Inst{21} = 0b0;11339 let Inst{20-16} = Zm;11340 let Inst{15} = 0b1;11341 let Inst{14-10} = 0b10110; // opc11342 let Inst{9-5} = Za;11343 let Inst{4-0} = Zdn;11344 11345 let Constraints = "$Zdn = $_Zdn";11346 let DestructiveInstType = DestructiveOther;11347 let ElementSize = ZPR64.ElementSize;11348 let hasSideEffects = 0;11349}11350 11351class sve_int_mla_cpa<string asm>11352 : sve2_int_mla<0b11, 0b10100, asm, ZPR64, ZPR64> {11353 let Inst{15} = 0b1;11354 11355 let ElementSize = ZPR64.ElementSize;11356}11357 11358//===----------------------------------------------------------------------===//11359// FP to Int down-converts11360//===----------------------------------------------------------------------===//11361class sve2_fp_to_int_downcvt<string asm, ZPRRegOp ZdRC, RegisterOperand ZSrcOp, bits<2> size, bit U>11362 : I<(outs ZdRC:$Zd), (ins ZSrcOp:$Zn),11363 asm, "\t$Zd, $Zn", "", []>, Sched<[]> {11364 bits<5> Zd;11365 bits<4> Zn;11366 let Inst{31-24} = 0b01100101;11367 let Inst{23-22} = size;11368 let Inst{21-11} = 0b00110100110;11369 let Inst{10} = U;11370 let Inst{9-6} = Zn;11371 let Inst{5} = 0b0;11372 let Inst{4-0} = Zd;11373}11374 11375multiclass sve2_fp_to_int_downcvt<string asm, bit U> {11376 def _HtoB : sve2_fp_to_int_downcvt<asm, ZPR8, ZZ_h_mul_r, 0b01, U>;11377 def _StoH : sve2_fp_to_int_downcvt<asm, ZPR16, ZZ_s_mul_r, 0b10, U>;11378 def _DtoS : sve2_fp_to_int_downcvt<asm, ZPR32, ZZ_d_mul_r, 0b11, U>;11379}11380 11381//===----------------------------------------------------------------------===//11382// Int to FP up-converts11383//===----------------------------------------------------------------------===//11384class sve2_int_to_fp_upcvt<string asm, ZPRRegOp ZdRC, ZPRRegOp ZnRC,11385 bits<2> size, bits<2> U>11386 : I<(outs ZdRC:$Zd), (ins ZnRC:$Zn),11387 asm, "\t$Zd, $Zn", "", []>, Sched<[]> {11388 bits<5> Zd;11389 bits<5> Zn;11390 let Inst{31-24} = 0b01100101;11391 let Inst{23-22} = size;11392 let Inst{21-12} = 0b0011000011;11393 let Inst{11-10} = U;11394 let Inst{9-5} = Zn;11395 let Inst{4-0} = Zd;11396}11397 11398multiclass sve2_int_to_fp_upcvt<string asm, bits<2> U> {11399 def _BtoH : sve2_int_to_fp_upcvt<asm, ZPR16, ZPR8, 0b01, U>;11400 def _HtoS : sve2_int_to_fp_upcvt<asm, ZPR32, ZPR16, 0b10, U>;11401 def _StoD : sve2_int_to_fp_upcvt<asm, ZPR64, ZPR32, 0b11, U>;11402}11403