7554 lines · plain
1//===-- ARMInstrMVE.td - MVE support for ARM ---------------*- tablegen -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file describes the ARM MVE instruction set.10//11//===----------------------------------------------------------------------===//12 13// VPT condition mask14def vpt_mask : Operand<i32> {15 let PrintMethod = "printVPTMask";16 let ParserMatchClass = it_mask_asmoperand;17 let EncoderMethod = "getVPTMaskOpValue";18 let DecoderMethod = "DecodeVPTMaskOperand";19}20 21// VPT/VCMP restricted predicate for sign invariant types22def pred_restricted_i_asmoperand : AsmOperandClass {23 let Name = "CondCodeRestrictedI";24 let RenderMethod = "addITCondCodeOperands";25 let PredicateMethod = "isITCondCodeRestrictedI";26 let ParserMethod = "parseITCondCode";27 let DiagnosticString = "condition code for sign-independent integer "#28 "comparison must be EQ or NE";29}30 31// VPT/VCMP restricted predicate for signed types32def pred_restricted_s_asmoperand : AsmOperandClass {33 let Name = "CondCodeRestrictedS";34 let RenderMethod = "addITCondCodeOperands";35 let PredicateMethod = "isITCondCodeRestrictedS";36 let ParserMethod = "parseITCondCode";37 let DiagnosticString = "condition code for signed integer "#38 "comparison must be EQ, NE, LT, GT, LE or GE";39}40 41// VPT/VCMP restricted predicate for unsigned types42def pred_restricted_u_asmoperand : AsmOperandClass {43 let Name = "CondCodeRestrictedU";44 let RenderMethod = "addITCondCodeOperands";45 let PredicateMethod = "isITCondCodeRestrictedU";46 let ParserMethod = "parseITCondCode";47 let DiagnosticString = "condition code for unsigned integer "#48 "comparison must be EQ, NE, HS or HI";49}50 51// VPT/VCMP restricted predicate for floating point52def pred_restricted_fp_asmoperand : AsmOperandClass {53 let Name = "CondCodeRestrictedFP";54 let RenderMethod = "addITCondCodeOperands";55 let PredicateMethod = "isITCondCodeRestrictedFP";56 let ParserMethod = "parseITCondCode";57 let DiagnosticString = "condition code for floating-point "#58 "comparison must be EQ, NE, LT, GT, LE or GE";59}60 61class VCMPPredicateOperand : Operand<i32>;62 63def pred_basic_i : VCMPPredicateOperand {64 let PrintMethod = "printMandatoryRestrictedPredicateOperand";65 let ParserMatchClass = pred_restricted_i_asmoperand;66 let DecoderMethod = "DecodeRestrictedIPredicateOperand";67 let EncoderMethod = "getRestrictedCondCodeOpValue";68}69 70def pred_basic_u : VCMPPredicateOperand {71 let PrintMethod = "printMandatoryRestrictedPredicateOperand";72 let ParserMatchClass = pred_restricted_u_asmoperand;73 let DecoderMethod = "DecodeRestrictedUPredicateOperand";74 let EncoderMethod = "getRestrictedCondCodeOpValue";75}76 77def pred_basic_s : VCMPPredicateOperand {78 let PrintMethod = "printMandatoryRestrictedPredicateOperand";79 let ParserMatchClass = pred_restricted_s_asmoperand;80 let DecoderMethod = "DecodeRestrictedSPredicateOperand";81 let EncoderMethod = "getRestrictedCondCodeOpValue";82}83 84def pred_basic_fp : VCMPPredicateOperand {85 let PrintMethod = "printMandatoryRestrictedPredicateOperand";86 let ParserMatchClass = pred_restricted_fp_asmoperand;87 let DecoderMethod = "DecodeRestrictedFPPredicateOperand";88 let EncoderMethod = "getRestrictedCondCodeOpValue";89}90 91// Register list operands for interleaving load/stores92def VecList2QAsmOperand : AsmOperandClass {93 let Name = "VecListTwoMQ";94 let ParserMethod = "parseVectorList";95 let RenderMethod = "addMVEVecListOperands";96 let DiagnosticString = "operand must be a list of two consecutive "#97 "q-registers in range [q0,q7]";98}99 100def VecList2Q : RegisterOperand<MQQPR, "printMVEVectorListTwoQ"> {101 let ParserMatchClass = VecList2QAsmOperand;102 let PrintMethod = "printMVEVectorList<2>";103}104 105def VecList4QAsmOperand : AsmOperandClass {106 let Name = "VecListFourMQ";107 let ParserMethod = "parseVectorList";108 let RenderMethod = "addMVEVecListOperands";109 let DiagnosticString = "operand must be a list of four consecutive "#110 "q-registers in range [q0,q7]";111}112 113def VecList4Q : RegisterOperand<MQQQQPR, "printMVEVectorListFourQ"> {114 let ParserMatchClass = VecList4QAsmOperand;115 let PrintMethod = "printMVEVectorList<4>";116}117 118// taddrmode_imm7 := reg[r0-r7] +/- (imm7 << shift)119class TMemImm7ShiftOffsetAsmOperand<int shift> : AsmOperandClass {120 let Name = "TMemImm7Shift"#shift#"Offset";121 let PredicateMethod = "isMemImm7ShiftedOffset<"#shift#",ARM::tGPRRegClassID>";122 let RenderMethod = "addMemImmOffsetOperands";123}124 125class taddrmode_imm7<int shift> : MemOperand,126 ComplexPattern<i32, 2, "SelectTAddrModeImm7<"#shift#">", []> {127 let ParserMatchClass = TMemImm7ShiftOffsetAsmOperand<shift>;128 // They are printed the same way as the T2 imm8 version129 let PrintMethod = "printT2AddrModeImm8Operand<false>";130 // This can also be the same as the T2 version.131 let EncoderMethod = "getT2AddrModeImmOpValue<7,"#shift#">";132 let DecoderMethod = "DecodeTAddrModeImm7<"#shift#">";133 let MIOperandInfo = (ops tGPR:$base, i32imm:$offsimm);134}135 136// t2addrmode_imm7 := reg +/- (imm7)137class MemImm7ShiftOffsetAsmOperand<int shift> : AsmOperandClass {138 let Name = "MemImm7Shift"#shift#"Offset";139 let PredicateMethod = "isMemImm7ShiftedOffset<" # shift #140 ",ARM::GPRnopcRegClassID>";141 let RenderMethod = "addMemImmOffsetOperands";142}143 144def MemImm7Shift0OffsetAsmOperand : MemImm7ShiftOffsetAsmOperand<0>;145def MemImm7Shift1OffsetAsmOperand : MemImm7ShiftOffsetAsmOperand<1>;146def MemImm7Shift2OffsetAsmOperand : MemImm7ShiftOffsetAsmOperand<2>;147class T2AddrMode_Imm7<int shift> : MemOperand,148 ComplexPattern<i32, 2, "SelectT2AddrModeImm7<"#shift#">", []> {149 let EncoderMethod = "getT2AddrModeImmOpValue<7,"#shift#">";150 let DecoderMethod = "DecodeT2AddrModeImm7<"#shift#", 0>";151 let ParserMatchClass =152 !cast<AsmOperandClass>("MemImm7Shift"#shift#"OffsetAsmOperand");153 let MIOperandInfo = (ops GPRnopc:$base, i32imm:$offsimm);154}155 156class t2addrmode_imm7<int shift> : T2AddrMode_Imm7<shift> {157 // They are printed the same way as the imm8 version158 let PrintMethod = "printT2AddrModeImm8Operand<false>";159}160 161class MemImm7ShiftOffsetWBAsmOperand<int shift> : AsmOperandClass {162 let Name = "MemImm7Shift"#shift#"OffsetWB";163 let PredicateMethod = "isMemImm7ShiftedOffset<" # shift #164 ",ARM::rGPRRegClassID>";165 let RenderMethod = "addMemImmOffsetOperands";166}167 168def MemImm7Shift0OffsetWBAsmOperand : MemImm7ShiftOffsetWBAsmOperand<0>;169def MemImm7Shift1OffsetWBAsmOperand : MemImm7ShiftOffsetWBAsmOperand<1>;170def MemImm7Shift2OffsetWBAsmOperand : MemImm7ShiftOffsetWBAsmOperand<2>;171 172class t2addrmode_imm7_pre<int shift> : T2AddrMode_Imm7<shift> {173 // They are printed the same way as the imm8 version174 let PrintMethod = "printT2AddrModeImm8Operand<true>";175 let ParserMatchClass =176 !cast<AsmOperandClass>("MemImm7Shift"#shift#"OffsetWBAsmOperand");177 let DecoderMethod = "DecodeT2AddrModeImm7<"#shift#", 1>";178 let MIOperandInfo = (ops rGPR:$base, i32imm:$offsim);179}180 181class t2am_imm7shiftOffsetAsmOperand<int shift>182 : AsmOperandClass { let Name = "Imm7Shift"#shift; }183def t2am_imm7shift0OffsetAsmOperand : t2am_imm7shiftOffsetAsmOperand<0>;184def t2am_imm7shift1OffsetAsmOperand : t2am_imm7shiftOffsetAsmOperand<1>;185def t2am_imm7shift2OffsetAsmOperand : t2am_imm7shiftOffsetAsmOperand<2>;186 187class t2am_imm7_offset<int shift>188 : MemOperand,189 ComplexPattern<i32, 1, "SelectT2AddrModeImm7Offset<"#shift#">"> {190 // They are printed the same way as the imm8 version191 let PrintMethod = "printT2AddrModeImm8OffsetOperand";192 let ParserMatchClass =193 !cast<AsmOperandClass>("t2am_imm7shift"#shift#"OffsetAsmOperand");194 let EncoderMethod = "getT2ScaledImmOpValue<7,"#shift#">";195 let DecoderMethod = "DecodeT2Imm7<"#shift#">";196 let WantsRoot = true;197}198 199// Operands for gather/scatter loads of the form [Rbase, Qoffsets]200class MemRegRQOffsetAsmOperand<int shift> : AsmOperandClass {201 let Name = "MemRegRQS"#shift#"Offset";202 let PredicateMethod = "isMemRegRQOffset<"#shift#">";203 let RenderMethod = "addMemRegRQOffsetOperands";204}205 206def MemRegRQS0OffsetAsmOperand : MemRegRQOffsetAsmOperand<0>;207def MemRegRQS1OffsetAsmOperand : MemRegRQOffsetAsmOperand<1>;208def MemRegRQS2OffsetAsmOperand : MemRegRQOffsetAsmOperand<2>;209def MemRegRQS3OffsetAsmOperand : MemRegRQOffsetAsmOperand<3>;210 211// mve_addr_rq_shift := reg + vreg{ << UXTW #shift}212class mve_addr_rq_shift<int shift> : MemOperand {213 let EncoderMethod = "getMveAddrModeRQOpValue";214 let PrintMethod = "printMveAddrModeRQOperand<"#shift#">";215 let ParserMatchClass =216 !cast<AsmOperandClass>("MemRegRQS"#shift#"OffsetAsmOperand");217 let DecoderMethod = "DecodeMveAddrModeRQ";218 let MIOperandInfo = (ops GPRnopc:$base, MQPR:$offsreg);219}220 221class MemRegQOffsetAsmOperand<int shift> : AsmOperandClass {222 let Name = "MemRegQS"#shift#"Offset";223 let PredicateMethod = "isMemRegQOffset<"#shift#">";224 let RenderMethod = "addMemImmOffsetOperands";225}226 227def MemRegQS2OffsetAsmOperand : MemRegQOffsetAsmOperand<2>;228def MemRegQS3OffsetAsmOperand : MemRegQOffsetAsmOperand<3>;229 230// mve_addr_q_shift := vreg {+ #imm7s2/4}231class mve_addr_q_shift<int shift> : MemOperand {232 let EncoderMethod = "getMveAddrModeQOpValue<"#shift#">";233 // Can be printed same way as other reg + imm operands234 let PrintMethod = "printT2AddrModeImm8Operand<false>";235 let ParserMatchClass =236 !cast<AsmOperandClass>("MemRegQS"#shift#"OffsetAsmOperand");237 let DecoderMethod = "DecodeMveAddrModeQ<"#shift#">";238 let MIOperandInfo = (ops MQPR:$base, i32imm:$imm);239}240 241// A family of classes wrapping up information about the vector types242// used by MVE.243class MVEVectorVTInfo<ValueType vec, ValueType dblvec,244 ValueType pred, ValueType dblpred,245 bits<2> size, string suffixletter, bit unsigned> {246 // The LLVM ValueType representing the vector, so we can use it in247 // ISel patterns.248 ValueType Vec = vec;249 250 // The LLVM ValueType representing a vector with elements double the size251 // of those in Vec, so we can use it in ISel patterns. It is up to the252 // invoker of this class to ensure that this is a correct choice.253 ValueType DblVec = dblvec;254 255 // An LLVM ValueType representing a corresponding vector of256 // predicate bits, for use in ISel patterns that handle an IR257 // intrinsic describing the predicated form of the instruction.258 ValueType Pred = pred;259 260 // Same as Pred but for DblVec rather than Vec.261 ValueType DblPred = dblpred;262 263 // The most common representation of the vector element size in MVE264 // instruction encodings: a 2-bit value V representing an (8<<V)-bit265 // vector element.266 bits<2> Size = size;267 268 // For vectors explicitly mentioning a signedness of integers: 0 for269 // signed and 1 for unsigned. For anything else, undefined.270 bit Unsigned = unsigned;271 272 // The number of bits in a vector element, in integer form.273 int LaneBits = !shl(8, Size);274 275 // The suffix used in assembly language on an instruction operating276 // on this lane if it only cares about number of bits.277 string BitsSuffix = !if(!eq(suffixletter, "p"),278 !if(!eq(unsigned, 0b0), "8", "16"),279 !cast<string>(LaneBits));280 281 // The suffix used on an instruction that mentions the whole type.282 string Suffix = suffixletter # BitsSuffix;283 284 // The letter part of the suffix only.285 string SuffixLetter = suffixletter;286}287 288// Integer vector types that don't treat signed and unsigned differently.289def MVE_v16i8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b00, "i", ?>;290def MVE_v8i16 : MVEVectorVTInfo<v8i16, v4i32, v8i1, v4i1, 0b01, "i", ?>;291def MVE_v4i32 : MVEVectorVTInfo<v4i32, v2i64, v4i1, v2i1, 0b10, "i", ?>;292def MVE_v2i64 : MVEVectorVTInfo<v2i64, ?, v2i1, ?, 0b11, "i", ?>;293 294// Explicitly signed and unsigned integer vectors. They map to the295// same set of LLVM ValueTypes as above, but are represented296// differently in assembly and instruction encodings.297def MVE_v16s8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b00, "s", 0b0>;298def MVE_v8s16 : MVEVectorVTInfo<v8i16, v4i32, v8i1, v4i1, 0b01, "s", 0b0>;299def MVE_v4s32 : MVEVectorVTInfo<v4i32, v2i64, v4i1, v2i1, 0b10, "s", 0b0>;300def MVE_v2s64 : MVEVectorVTInfo<v2i64, ?, v2i1, ?, 0b11, "s", 0b0>;301def MVE_v16u8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b00, "u", 0b1>;302def MVE_v8u16 : MVEVectorVTInfo<v8i16, v4i32, v8i1, v4i1, 0b01, "u", 0b1>;303def MVE_v4u32 : MVEVectorVTInfo<v4i32, v2i64, v4i1, v2i1, 0b10, "u", 0b1>;304def MVE_v2u64 : MVEVectorVTInfo<v2i64, ?, v2i1, ?, 0b11, "u", 0b1>;305 306// FP vector types.307def MVE_v8f16 : MVEVectorVTInfo<v8f16, v4f32, v8i1, v4i1, 0b01, "f", ?>;308def MVE_v4f32 : MVEVectorVTInfo<v4f32, v2f64, v4i1, v2i1, 0b10, "f", ?>;309def MVE_v2f64 : MVEVectorVTInfo<v2f64, ?, v2i1, ?, 0b11, "f", ?>;310 311// Polynomial vector types.312def MVE_v16p8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b11, "p", 0b0>;313def MVE_v8p16 : MVEVectorVTInfo<v8i16, v4i32, v8i1, v4i1, 0b11, "p", 0b1>;314 315multiclass MVE_TwoOpPattern<MVEVectorVTInfo VTI, SDPatternOperator Op, Intrinsic PredInt,316 dag PredOperands, Instruction Inst,317 SDPatternOperator IdentityVec = null_frag> {318 // Unpredicated319 def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),320 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;321 322 // Predicated with select323 if !ne(VTI.Size, 0b11) then {324 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$mask),325 (VTI.Vec (Op (VTI.Vec MQPR:$Qm),326 (VTI.Vec MQPR:$Qn))),327 (VTI.Vec MQPR:$inactive))),328 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),329 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,330 (VTI.Vec MQPR:$inactive)))>;331 332 // Optionally with the select folded through the op333 def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm),334 (VTI.Vec (vselect (VTI.Pred VCCR:$mask),335 (VTI.Vec MQPR:$Qn),336 (VTI.Vec IdentityVec))))),337 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),338 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,339 (VTI.Vec MQPR:$Qm)))>;340 }341 342 // Predicated with intrinsic343 def : Pat<(VTI.Vec !con((PredInt (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)),344 PredOperands,345 (? (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive)))),346 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),347 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,348 (VTI.Vec MQPR:$inactive)))>;349}350 351multiclass MVE_TwoOpPatternDup<MVEVectorVTInfo VTI, SDPatternOperator Op, Intrinsic PredInt,352 dag PredOperands, Instruction Inst,353 SDPatternOperator IdentityVec = null_frag> {354 // Unpredicated355 def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm), (VTI.Vec (ARMvdup rGPR:$Rn)))),356 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn))>;357 358 // Predicated with select359 if !ne(VTI.Size, 0b11) then {360 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$mask),361 (VTI.Vec (Op (VTI.Vec MQPR:$Qm),362 (VTI.Vec (ARMvdup rGPR:$Rn)))),363 (VTI.Vec MQPR:$inactive))),364 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn,365 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,366 (VTI.Vec MQPR:$inactive)))>;367 368 // Optionally with the select folded through the op369 def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm),370 (VTI.Vec (vselect (VTI.Pred VCCR:$mask),371 (ARMvdup rGPR:$Rn),372 (VTI.Vec IdentityVec))))),373 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn,374 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,375 (VTI.Vec MQPR:$Qm)))>;376 }377 378 // Predicated with intrinsic379 def : Pat<(VTI.Vec !con((PredInt (VTI.Vec MQPR:$Qm), (VTI.Vec (ARMvdup rGPR:$Rn))),380 PredOperands,381 (? (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive)))),382 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn,383 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,384 (VTI.Vec MQPR:$inactive)))>;385}386 387def vadd : PatFrags<(ops node:$lhs, node:$rhs),388 [(fadd node:$lhs, node:$rhs),389 (int_arm_mve_vadd node:$lhs, node:$rhs)]>;390def vsub : PatFrags<(ops node:$lhs, node:$rhs),391 [(fsub node:$lhs, node:$rhs),392 (int_arm_mve_vsub node:$lhs, node:$rhs)]>;393def vmul : PatFrags<(ops node:$lhs, node:$rhs),394 [(fmul node:$lhs, node:$rhs),395 (int_arm_mve_vmul node:$lhs, node:$rhs)]>;396 397// --------- Start of base classes for the instructions themselves398 399class MVE_MI<dag oops, dag iops, InstrItinClass itin, string asm,400 string ops, string cstr, bits<2> vecsize, list<dag> pattern>401 : Thumb2XI<oops, iops, AddrModeNone, 4, itin, !strconcat(asm, "\t", ops), cstr,402 pattern>,403 Requires<[HasMVEInt]> {404 let D = MVEDomain;405 let DecoderNamespace = "MVE";406 let VecSize = vecsize;407}408 409// MVE_p is used for most predicated instructions, to add the cluster410// of input operands that provides the VPT suffix (none, T or E) and411// the input predicate register.412class MVE_p<dag oops, dag iops, InstrItinClass itin, string iname,413 string suffix, string ops, vpred_ops vpred, string cstr,414 bits<2> vecsize, list<dag> pattern=[]>415 : MVE_MI<oops, !con(iops, (ins vpred:$vp)), itin,416 // If the instruction has a suffix, like vadd.f32, then the417 // VPT predication suffix goes before the dot, so the full418 // name has to be "vadd${vp}.f32".419 !strconcat(iname, "${vp}",420 !if(!eq(suffix, ""), "", !strconcat(".", suffix))),421 ops, !strconcat(cstr, vpred.vpred_constraint), vecsize, pattern> {422 bits<0> vp;423 let Inst{31-29} = 0b111;424 let Inst{27-26} = 0b11;425}426 427class MVE_f<dag oops, dag iops, InstrItinClass itin, string iname,428 string suffix, string ops, vpred_ops vpred, string cstr,429 bits<2> vecsize, list<dag> pattern=[]>430 : MVE_p<oops, iops, itin, iname, suffix, ops, vpred, cstr, vecsize, pattern> {431 let Predicates = [HasMVEFloat];432}433 434class MVE_MI_with_pred<dag oops, dag iops, InstrItinClass itin, string asm,435 string ops, string cstr, list<dag> pattern>436 : Thumb2I<oops, iops, AddrModeNone, 4, itin, asm, !strconcat("\t", ops), cstr,437 pattern>,438 Requires<[HasV8_1MMainline, HasMVEInt]> {439 let D = MVEDomain;440 let DecoderNamespace = "MVE";441}442 443class MVE_VMOV_lane_base<dag oops, dag iops, InstrItinClass itin, string asm,444 string suffix, string ops, string cstr,445 list<dag> pattern>446 : Thumb2I<oops, iops, AddrModeNone, 4, itin, asm,447 !if(!eq(suffix, ""), "", "." # suffix) # "\t" # ops,448 cstr, pattern>,449 Requires<[HasV8_1MMainline, HasMVEInt]> {450 let D = MVEDomain;451 let DecoderNamespace = "MVE";452}453 454class MVE_ScalarShift<string iname, dag oops, dag iops, string asm, string cstr,455 list<dag> pattern=[]>456 : MVE_MI_with_pred<oops, iops, NoItinerary, iname, asm, cstr, pattern> {457 let Inst{31-20} = 0b111010100101;458 let Inst{8} = 0b1;459 let validForTailPredication=1;460}461 462class MVE_ScalarShiftSingleReg<string iname, dag iops, string asm, string cstr,463 list<dag> pattern=[]>464 : MVE_ScalarShift<iname, (outs rGPR:$RdaDest), iops, asm, cstr, pattern> {465 bits<4> RdaDest;466 467 let Inst{19-16} = RdaDest{3-0};468}469 470class MVE_ScalarShiftSRegImm<string iname, bits<2> op5_4>471 : MVE_ScalarShiftSingleReg<iname, (ins rGPR:$RdaSrc, long_shift:$imm),472 "$RdaSrc, $imm", "$RdaDest = $RdaSrc",473 [(set rGPR:$RdaDest,474 (i32 (!cast<Intrinsic>("int_arm_mve_" # iname)475 (i32 rGPR:$RdaSrc), (i32 imm:$imm))))]> {476 bits<5> imm;477 478 let Inst{15} = 0b0;479 let Inst{14-12} = imm{4-2};480 let Inst{11-8} = 0b1111;481 let Inst{7-6} = imm{1-0};482 let Inst{5-4} = op5_4{1-0};483 let Inst{3-0} = 0b1111;484}485 486def MVE_SQSHL : MVE_ScalarShiftSRegImm<"sqshl", 0b11>;487def MVE_SRSHR : MVE_ScalarShiftSRegImm<"srshr", 0b10>;488def MVE_UQSHL : MVE_ScalarShiftSRegImm<"uqshl", 0b00>;489def MVE_URSHR : MVE_ScalarShiftSRegImm<"urshr", 0b01>;490 491class MVE_ScalarShiftSRegReg<string iname, bits<2> op5_4>492 : MVE_ScalarShiftSingleReg<iname, (ins rGPR:$RdaSrc, rGPR:$Rm),493 "$RdaSrc, $Rm", "@earlyclobber $RdaDest,$RdaDest = $RdaSrc",494 [(set rGPR:$RdaDest,495 (i32 (!cast<Intrinsic>("int_arm_mve_" # iname)496 (i32 rGPR:$RdaSrc), (i32 rGPR:$Rm))))]> {497 bits<4> Rm;498 499 let Inst{15-12} = Rm{3-0};500 let Inst{11-8} = 0b1111;501 let Inst{7-6} = 0b00;502 let Inst{5-4} = op5_4{1-0};503 let Inst{3-0} = 0b1101;504 505 let Unpredictable{8-6} = 0b111;506}507 508def MVE_SQRSHR : MVE_ScalarShiftSRegReg<"sqrshr", 0b10>;509def MVE_UQRSHL : MVE_ScalarShiftSRegReg<"uqrshl", 0b00>;510 511class MVE_ScalarShiftDoubleReg<string iname, dag iops, string asm,512 string cstr, list<dag> pattern=[]>513 : MVE_ScalarShift<iname, (outs tGPREven:$RdaLo, tGPROdd:$RdaHi),514 iops, asm, cstr, pattern> {515 bits<4> RdaLo;516 bits<4> RdaHi;517 518 let Inst{19-17} = RdaLo{3-1};519 let Inst{11-9} = RdaHi{3-1};520 521 let hasSideEffects = 0;522}523 524class MVE_ScalarShiftDRegImm<string iname, bits<2> op5_4, bit op16,525 list<dag> pattern=[]>526 : MVE_ScalarShiftDoubleReg<527 iname, (ins tGPREven:$RdaLo_src, tGPROdd:$RdaHi_src, long_shift:$imm),528 "$RdaLo, $RdaHi, $imm", "$RdaLo = $RdaLo_src,$RdaHi = $RdaHi_src",529 pattern> {530 bits<5> imm;531 532 let Inst{16} = op16;533 let Inst{15} = 0b0;534 let Inst{14-12} = imm{4-2};535 let Inst{7-6} = imm{1-0};536 let Inst{5-4} = op5_4{1-0};537 let Inst{3-0} = 0b1111;538}539 540class MVE_ScalarShiftDRegRegBase<string iname, dag iops, string asm,541 bit op5, bit op16, list<dag> pattern=[]>542 : MVE_ScalarShiftDoubleReg<543 iname, iops, asm, "@earlyclobber $RdaHi,@earlyclobber $RdaLo,"544 "$RdaLo = $RdaLo_src,$RdaHi = $RdaHi_src",545 pattern> {546 bits<4> Rm;547 548 let Inst{16} = op16;549 let Inst{15-12} = Rm{3-0};550 let Inst{6} = 0b0;551 let Inst{5} = op5;552 let Inst{4} = 0b0;553 let Inst{3-0} = 0b1101;554 555 // Custom decoder method because of the following overlapping encodings:556 // ASRL and SQRSHR557 // LSLL and UQRSHL558 // SQRSHRL and SQRSHR559 // UQRSHLL and UQRSHL560 let DecoderMethod = "DecodeMVEOverlappingLongShift";561}562 563class MVE_ScalarShiftDRegReg<string iname, bit op5, list<dag> pattern=[]>564 : MVE_ScalarShiftDRegRegBase<565 iname, (ins tGPREven:$RdaLo_src, tGPROdd:$RdaHi_src, rGPR:$Rm),566 "$RdaLo, $RdaHi, $Rm", op5, 0b0, pattern> {567 568 let Inst{7} = 0b0;569}570 571class MVE_ScalarShiftDRegRegWithSat<string iname, bit op5, list<dag> pattern=[]>572 : MVE_ScalarShiftDRegRegBase<573 iname, (ins tGPREven:$RdaLo_src, tGPROdd:$RdaHi_src, rGPR:$Rm, saturateop:$sat),574 "$RdaLo, $RdaHi, $sat, $Rm", op5, 0b1, pattern> {575 bit sat;576 577 let Inst{7} = sat;578}579 580def MVE_ASRLr : MVE_ScalarShiftDRegReg<"asrl", 0b1, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,581 (ARMasrl tGPREven:$RdaLo_src,582 tGPROdd:$RdaHi_src, rGPR:$Rm))]>;583def MVE_ASRLi : MVE_ScalarShiftDRegImm<"asrl", 0b10, ?, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,584 (ARMasrl tGPREven:$RdaLo_src,585 tGPROdd:$RdaHi_src, (i32 long_shift:$imm)))]>;586def MVE_LSLLr : MVE_ScalarShiftDRegReg<"lsll", 0b0, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,587 (ARMlsll tGPREven:$RdaLo_src,588 tGPROdd:$RdaHi_src, rGPR:$Rm))]>;589def MVE_LSLLi : MVE_ScalarShiftDRegImm<"lsll", 0b00, ?, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,590 (ARMlsll tGPREven:$RdaLo_src,591 tGPROdd:$RdaHi_src, (i32 long_shift:$imm)))]>;592def MVE_LSRL : MVE_ScalarShiftDRegImm<"lsrl", 0b01, ?, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,593 (ARMlsrl tGPREven:$RdaLo_src,594 tGPROdd:$RdaHi_src, (i32 long_shift:$imm)))]>;595 596def MVE_SQRSHRL : MVE_ScalarShiftDRegRegWithSat<"sqrshrl", 0b1>;597def MVE_SQSHLL : MVE_ScalarShiftDRegImm<"sqshll", 0b11, 0b1>;598def MVE_SRSHRL : MVE_ScalarShiftDRegImm<"srshrl", 0b10, 0b1>;599 600def MVE_UQRSHLL : MVE_ScalarShiftDRegRegWithSat<"uqrshll", 0b0>;601def MVE_UQSHLL : MVE_ScalarShiftDRegImm<"uqshll", 0b00, 0b1>;602def MVE_URSHRL : MVE_ScalarShiftDRegImm<"urshrl", 0b01, 0b1>;603 604// start of mve_rDest instructions605 606class MVE_rDest<dag oops, dag iops, InstrItinClass itin,607 string iname, string suffix,608 string ops, string cstr, bits<2> vecsize, list<dag> pattern=[]>609// Always use vpred_n and not vpred_r: with the output register being610// a GPR and not a vector register, there can't be any question of611// what to put in its inactive lanes.612 : MVE_p<oops, iops, itin, iname, suffix, ops, vpred_n, cstr, vecsize, pattern> {613 614 let Inst{25-23} = 0b101;615 let Inst{11-9} = 0b111;616 let Inst{4} = 0b0;617}618 619class MVE_VABAV<string suffix, bit U, bits<2> size>620 : MVE_rDest<(outs rGPR:$Rda), (ins rGPR:$Rda_src, MQPR:$Qn, MQPR:$Qm),621 NoItinerary, "vabav", suffix, "$Rda, $Qn, $Qm", "$Rda = $Rda_src",622 size, []> {623 bits<4> Qm;624 bits<4> Qn;625 bits<4> Rda;626 627 let Inst{28} = U;628 let Inst{22} = 0b0;629 let Inst{21-20} = size{1-0};630 let Inst{19-17} = Qn{2-0};631 let Inst{16} = 0b0;632 let Inst{15-12} = Rda{3-0};633 let Inst{8} = 0b1;634 let Inst{7} = Qn{3};635 let Inst{6} = 0b0;636 let Inst{5} = Qm{3};637 let Inst{3-1} = Qm{2-0};638 let Inst{0} = 0b1;639 let horizontalReduction = 1;640}641 642multiclass MVE_VABAV_m<MVEVectorVTInfo VTI> {643 def "" : MVE_VABAV<VTI.Suffix, VTI.Unsigned, VTI.Size>;644 defvar Inst = !cast<Instruction>(NAME);645 646 let Predicates = [HasMVEInt] in {647 def : Pat<(i32 (int_arm_mve_vabav648 (i32 VTI.Unsigned),649 (i32 rGPR:$Rda_src),650 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),651 (i32 (Inst (i32 rGPR:$Rda_src),652 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm)))>;653 654 def : Pat<(i32 (int_arm_mve_vabav_predicated655 (i32 VTI.Unsigned),656 (i32 rGPR:$Rda_src),657 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),658 (VTI.Pred VCCR:$mask))),659 (i32 (Inst (i32 rGPR:$Rda_src),660 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),661 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;662 }663}664 665defm MVE_VABAVs8 : MVE_VABAV_m<MVE_v16s8>;666defm MVE_VABAVs16 : MVE_VABAV_m<MVE_v8s16>;667defm MVE_VABAVs32 : MVE_VABAV_m<MVE_v4s32>;668defm MVE_VABAVu8 : MVE_VABAV_m<MVE_v16u8>;669defm MVE_VABAVu16 : MVE_VABAV_m<MVE_v8u16>;670defm MVE_VABAVu32 : MVE_VABAV_m<MVE_v4u32>;671 672class MVE_VADDV<string iname, string suffix, dag iops, string cstr,673 bit A, bit U, bits<2> size, list<dag> pattern=[]>674 : MVE_rDest<(outs tGPREven:$Rda), iops, NoItinerary,675 iname, suffix, "$Rda, $Qm", cstr, size, pattern> {676 bits<3> Qm;677 bits<4> Rda;678 679 let Inst{28} = U;680 let Inst{22-20} = 0b111;681 let Inst{19-18} = size{1-0};682 let Inst{17-16} = 0b01;683 let Inst{15-13} = Rda{3-1};684 let Inst{12} = 0b0;685 let Inst{8-6} = 0b100;686 let Inst{5} = A;687 let Inst{3-1} = Qm{2-0};688 let Inst{0} = 0b0;689 let horizontalReduction = 1;690 let validForTailPredication = 1;691}692 693def SDTVecReduceP : SDTypeProfile<1, 2, [ // VADDLVp694 SDTCisInt<0>, SDTCisVec<1>, SDTCisVec<2>695]>;696 697// sign- or zero-extend the elements of a vector to i32,698// add them all together, and return an i32 of their sum699def ARMVADDVs : SDNode<"ARMISD::VADDVs", SDTVecReduce>;700def ARMVADDVu : SDNode<"ARMISD::VADDVu", SDTVecReduce>;701 702// Same as VADDV[su] but with a v4i1 predicate mask703def ARMVADDVps : SDNode<"ARMISD::VADDVps", SDTVecReduceP>;704def ARMVADDVpu : SDNode<"ARMISD::VADDVpu", SDTVecReduceP>;705 706multiclass MVE_VADDV_A<MVEVectorVTInfo VTI> {707 def acc : MVE_VADDV<"vaddva", VTI.Suffix,708 (ins tGPREven:$Rda_src, MQPR:$Qm), "$Rda = $Rda_src",709 0b1, VTI.Unsigned, VTI.Size>;710 def no_acc : MVE_VADDV<"vaddv", VTI.Suffix,711 (ins MQPR:$Qm), "",712 0b0, VTI.Unsigned, VTI.Size>;713 714 defvar InstA = !cast<Instruction>(NAME # "acc");715 defvar InstN = !cast<Instruction>(NAME # "no_acc");716 717 let Predicates = [HasMVEInt] in {718 if VTI.Unsigned then {719 def : Pat<(i32 (vecreduce_add (VTI.Vec MQPR:$vec))),720 (i32 (InstN $vec))>;721 def : Pat<(i32 (vecreduce_add (VTI.Vec (vselect (VTI.Pred VCCR:$pred),722 (VTI.Vec MQPR:$vec),723 (VTI.Vec ARMimmAllZerosV))))),724 (i32 (InstN $vec, ARMVCCThen, $pred, zero_reg))>;725 def : Pat<(i32 (ARMVADDVu (VTI.Vec MQPR:$vec))),726 (i32 (InstN $vec))>;727 def : Pat<(i32 (ARMVADDVpu (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),728 (i32 (InstN $vec, ARMVCCThen, $pred, zero_reg))>;729 def : Pat<(i32 (add (i32 (vecreduce_add (VTI.Vec MQPR:$vec))),730 (i32 tGPREven:$acc))),731 (i32 (InstA $acc, $vec))>;732 def : Pat<(i32 (add (i32 (vecreduce_add (VTI.Vec (vselect (VTI.Pred VCCR:$pred),733 (VTI.Vec MQPR:$vec),734 (VTI.Vec ARMimmAllZerosV))))),735 (i32 tGPREven:$acc))),736 (i32 (InstA $acc, $vec, ARMVCCThen, $pred, zero_reg))>;737 def : Pat<(i32 (add (i32 (ARMVADDVu (VTI.Vec MQPR:$vec))),738 (i32 tGPREven:$acc))),739 (i32 (InstA $acc, $vec))>;740 def : Pat<(i32 (add (i32 (ARMVADDVpu (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),741 (i32 tGPREven:$acc))),742 (i32 (InstA $acc, $vec, ARMVCCThen, $pred, zero_reg))>;743 } else {744 def : Pat<(i32 (ARMVADDVs (VTI.Vec MQPR:$vec))),745 (i32 (InstN $vec))>;746 def : Pat<(i32 (add (i32 (ARMVADDVs (VTI.Vec MQPR:$vec))),747 (i32 tGPREven:$acc))),748 (i32 (InstA $acc, $vec))>;749 def : Pat<(i32 (ARMVADDVps (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),750 (i32 (InstN $vec, ARMVCCThen, $pred, zero_reg))>;751 def : Pat<(i32 (add (i32 (ARMVADDVps (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),752 (i32 tGPREven:$acc))),753 (i32 (InstA $acc, $vec, ARMVCCThen, $pred, zero_reg))>;754 }755 756 def : Pat<(i32 (int_arm_mve_addv_predicated (VTI.Vec MQPR:$vec),757 (i32 VTI.Unsigned),758 (VTI.Pred VCCR:$pred))),759 (i32 (InstN $vec, ARMVCCThen, $pred, zero_reg))>;760 def : Pat<(i32 (add (int_arm_mve_addv_predicated (VTI.Vec MQPR:$vec),761 (i32 VTI.Unsigned),762 (VTI.Pred VCCR:$pred)),763 (i32 tGPREven:$acc))),764 (i32 (InstA $acc, $vec, ARMVCCThen, $pred, zero_reg))>;765 }766}767 768defm MVE_VADDVs8 : MVE_VADDV_A<MVE_v16s8>;769defm MVE_VADDVs16 : MVE_VADDV_A<MVE_v8s16>;770defm MVE_VADDVs32 : MVE_VADDV_A<MVE_v4s32>;771defm MVE_VADDVu8 : MVE_VADDV_A<MVE_v16u8>;772defm MVE_VADDVu16 : MVE_VADDV_A<MVE_v8u16>;773defm MVE_VADDVu32 : MVE_VADDV_A<MVE_v4u32>;774 775class MVE_VADDLV<string iname, string suffix, dag iops, string cstr,776 bit A, bit U, list<dag> pattern=[]>777 : MVE_rDest<(outs tGPREven:$RdaLo, tGPROdd:$RdaHi), iops, NoItinerary, iname,778 suffix, "$RdaLo, $RdaHi, $Qm", cstr, 0b10, pattern> {779 bits<3> Qm;780 bits<4> RdaLo;781 bits<4> RdaHi;782 783 let Inst{28} = U;784 let Inst{22-20} = RdaHi{3-1};785 let Inst{19-18} = 0b10;786 let Inst{17-16} = 0b01;787 let Inst{15-13} = RdaLo{3-1};788 let Inst{12} = 0b0;789 let Inst{8-6} = 0b100;790 let Inst{5} = A;791 let Inst{3-1} = Qm{2-0};792 let Inst{0} = 0b0;793 let horizontalReduction = 1;794}795 796def SDTVecReduceL : SDTypeProfile<2, 1, [ // VADDLV797 SDTCisInt<0>, SDTCisInt<1>, SDTCisVec<2>798]>;799def SDTVecReduceLA : SDTypeProfile<2, 3, [ // VADDLVA800 SDTCisInt<0>, SDTCisInt<1>, SDTCisInt<2>, SDTCisInt<3>,801 SDTCisVec<4>802]>;803def SDTVecReduceLP : SDTypeProfile<2, 2, [ // VADDLVp804 SDTCisInt<0>, SDTCisInt<1>, SDTCisVec<2>, SDTCisVec<2>805]>;806def SDTVecReduceLPA : SDTypeProfile<2, 4, [ // VADDLVAp807 SDTCisInt<0>, SDTCisInt<1>, SDTCisInt<2>, SDTCisInt<3>,808 SDTCisVec<4>, SDTCisVec<5>809]>;810 811multiclass MVE_VADDLV_A<MVEVectorVTInfo VTI> {812 def acc : MVE_VADDLV<"vaddlva", VTI.Suffix,813 (ins tGPREven:$RdaLo_src, tGPROdd:$RdaHi_src, MQPR:$Qm),814 "$RdaLo = $RdaLo_src,$RdaHi = $RdaHi_src",815 0b1, VTI.Unsigned>;816 def no_acc : MVE_VADDLV<"vaddlv", VTI.Suffix,817 (ins MQPR:$Qm), "",818 0b0, VTI.Unsigned>;819 820 defvar InstA = !cast<Instruction>(NAME # "acc");821 defvar InstN = !cast<Instruction>(NAME # "no_acc");822 823 defvar letter = VTI.SuffixLetter;824 825 // sign- or zero-extend elements to i64 and sum, returning826 // the low and high 32-bit halves of the sum827 defvar ARMVADDLV = SDNode<"ARMISD::VADDLV" # letter, SDTVecReduceL>;828 829 // Same as VADDLV[su] but also add an input accumulator830 // provided as low and high halves831 defvar ARMVADDLVA = SDNode<"ARMISD::VADDLVA" # letter, SDTVecReduceLA>;832 833 // Same as VADDLV[su] but with a v4i1 predicate mask834 defvar ARMVADDLVp = SDNode<"ARMISD::VADDLVp" # letter, SDTVecReduceLP>;835 836 // Same as VADDLVp[su] but with a v4i1 predicate mask837 defvar ARMVADDLVAp = SDNode<"ARMISD::VADDLVAp" # letter, SDTVecReduceLPA>;838 839 let Predicates = [HasMVEInt] in {840 def : Pat<(ARMVADDLV (v4i32 MQPR:$vec)),841 (InstN (v4i32 MQPR:$vec))>;842 def : Pat<(ARMVADDLVA tGPREven:$acclo, tGPROdd:$acchi, (v4i32 MQPR:$vec)),843 (InstA tGPREven:$acclo, tGPROdd:$acchi, (v4i32 MQPR:$vec))>;844 def : Pat<(ARMVADDLVp (v4i32 MQPR:$vec), (VTI.Pred VCCR:$pred)),845 (InstN (v4i32 MQPR:$vec), ARMVCCThen, (VTI.Pred VCCR:$pred), zero_reg)>;846 def : Pat<(ARMVADDLVAp tGPREven:$acclo, tGPROdd:$acchi, (v4i32 MQPR:$vec),847 (VTI.Pred VCCR:$pred)),848 (InstA tGPREven:$acclo, tGPROdd:$acchi, (v4i32 MQPR:$vec),849 ARMVCCThen, (VTI.Pred VCCR:$pred), zero_reg)>;850 }851}852 853defm MVE_VADDLVs32 : MVE_VADDLV_A<MVE_v4s32>;854defm MVE_VADDLVu32 : MVE_VADDLV_A<MVE_v4u32>;855 856class MVE_VMINMAXNMV<string iname, string suffix, bit sz,857 bit bit_17, bit bit_7, list<dag> pattern=[]>858 : MVE_rDest<(outs rGPR:$RdaDest), (ins rGPR:$RdaSrc, MQPR:$Qm),859 NoItinerary, iname, suffix, "$RdaSrc, $Qm",860 "$RdaDest = $RdaSrc", !if(sz, 0b01, 0b10), pattern> {861 bits<3> Qm;862 bits<4> RdaDest;863 864 let Inst{28} = sz;865 let Inst{22-20} = 0b110;866 let Inst{19-18} = 0b11;867 let Inst{17} = bit_17;868 let Inst{16} = 0b0;869 let Inst{15-12} = RdaDest{3-0};870 let Inst{8} = 0b1;871 let Inst{7} = bit_7;872 let Inst{6-5} = 0b00;873 let Inst{3-1} = Qm{2-0};874 let Inst{0} = 0b0;875 let horizontalReduction = 1;876 877 let Predicates = [HasMVEFloat];878 let hasSideEffects = 0;879}880 881multiclass MVE_VMINMAXNMV_p<string iname, bit notAbs, bit isMin,882 MVEVectorVTInfo VTI, string intrBaseName,883 ValueType Scalar, RegisterClass ScalarReg> {884 def "": MVE_VMINMAXNMV<iname, VTI.Suffix, VTI.Size{0}, notAbs, isMin>;885 defvar Inst = !cast<Instruction>(NAME);886 defvar unpred_intr = !cast<Intrinsic>(intrBaseName);887 defvar pred_intr = !cast<Intrinsic>(intrBaseName#"_predicated");888 889 let Predicates = [HasMVEFloat] in {890 def : Pat<(Scalar (unpred_intr (Scalar ScalarReg:$prev),891 (VTI.Vec MQPR:$vec))),892 (COPY_TO_REGCLASS (Inst (COPY_TO_REGCLASS ScalarReg:$prev, rGPR),893 (VTI.Vec MQPR:$vec)),894 ScalarReg)>;895 def : Pat<(Scalar (pred_intr (Scalar ScalarReg:$prev),896 (VTI.Vec MQPR:$vec),897 (VTI.Pred VCCR:$pred))),898 (COPY_TO_REGCLASS (Inst (COPY_TO_REGCLASS ScalarReg:$prev, rGPR),899 (VTI.Vec MQPR:$vec),900 ARMVCCThen, (VTI.Pred VCCR:$pred), zero_reg),901 ScalarReg)>;902 }903}904 905multiclass MVE_VMINMAXNMV_fty<string iname, bit notAbs, bit isMin,906 string intrBase> {907 defm f32 : MVE_VMINMAXNMV_p<iname, notAbs, isMin, MVE_v4f32, intrBase,908 f32, SPR>;909 defm f16 : MVE_VMINMAXNMV_p<iname, notAbs, isMin, MVE_v8f16, intrBase,910 f16, HPR>;911}912 913defm MVE_VMINNMV : MVE_VMINMAXNMV_fty<"vminnmv", 1, 1, "int_arm_mve_minnmv">;914defm MVE_VMAXNMV : MVE_VMINMAXNMV_fty<"vmaxnmv", 1, 0, "int_arm_mve_maxnmv">;915defm MVE_VMINNMAV: MVE_VMINMAXNMV_fty<"vminnmav", 0, 1, "int_arm_mve_minnmav">;916defm MVE_VMAXNMAV: MVE_VMINMAXNMV_fty<"vmaxnmav", 0, 0, "int_arm_mve_maxnmav">;917 918class MVE_VMINMAXV<string iname, string suffix, bit U, bits<2> size,919 bit bit_17, bit bit_7, list<dag> pattern=[]>920 : MVE_rDest<(outs rGPR:$RdaDest), (ins rGPR:$RdaSrc, MQPR:$Qm), NoItinerary,921 iname, suffix, "$RdaSrc, $Qm", "$RdaDest = $RdaSrc", size, pattern> {922 bits<3> Qm;923 bits<4> RdaDest;924 925 let Inst{28} = U;926 let Inst{22-20} = 0b110;927 let Inst{19-18} = size{1-0};928 let Inst{17} = bit_17;929 let Inst{16} = 0b0;930 let Inst{15-12} = RdaDest{3-0};931 let Inst{8} = 0b1;932 let Inst{7} = bit_7;933 let Inst{6-5} = 0b00;934 let Inst{3-1} = Qm{2-0};935 let Inst{0} = 0b0;936 let horizontalReduction = 1;937}938 939multiclass MVE_VMINMAXV_p<string iname, bit notAbs, bit isMin,940 MVEVectorVTInfo VTI, string intrBaseName> {941 def "": MVE_VMINMAXV<iname, VTI.Suffix, VTI.Unsigned, VTI.Size,942 notAbs, isMin>;943 defvar Inst = !cast<Instruction>(NAME);944 defvar unpred_intr = !cast<Intrinsic>(intrBaseName);945 defvar pred_intr = !cast<Intrinsic>(intrBaseName#"_predicated");946 defvar base_args = (? (i32 rGPR:$prev), (VTI.Vec MQPR:$vec));947 defvar args = !if(notAbs, !con(base_args, (? (i32 VTI.Unsigned))),948 base_args);949 950 let Predicates = [HasMVEInt] in {951 def : Pat<(i32 !con(args, (unpred_intr))),952 (i32 (Inst (i32 rGPR:$prev), (VTI.Vec MQPR:$vec)))>;953 def : Pat<(i32 !con(args, (pred_intr (VTI.Pred VCCR:$pred)))),954 (i32 (Inst (i32 rGPR:$prev), (VTI.Vec MQPR:$vec),955 ARMVCCThen, (VTI.Pred VCCR:$pred), zero_reg))>;956 }957}958 959multiclass MVE_VMINMAXV_ty<string iname, bit isMin, string intrBaseName> {960 defm s8 : MVE_VMINMAXV_p<iname, 1, isMin, MVE_v16s8, intrBaseName>;961 defm s16: MVE_VMINMAXV_p<iname, 1, isMin, MVE_v8s16, intrBaseName>;962 defm s32: MVE_VMINMAXV_p<iname, 1, isMin, MVE_v4s32, intrBaseName>;963 defm u8 : MVE_VMINMAXV_p<iname, 1, isMin, MVE_v16u8, intrBaseName>;964 defm u16: MVE_VMINMAXV_p<iname, 1, isMin, MVE_v8u16, intrBaseName>;965 defm u32: MVE_VMINMAXV_p<iname, 1, isMin, MVE_v4u32, intrBaseName>;966}967 968def SDTVecReduceR : SDTypeProfile<1, 2, [ // Reduction of an integer and vector into an integer969 SDTCisInt<0>, SDTCisInt<1>, SDTCisVec<2>970]>;971 972// Find minimum unsigned value of a vector and register973def ARMVMINVu : SDNode<"ARMISD::VMINVu", SDTVecReduceR>;974 975// Find minimum signed value of a vector and register976def ARMVMINVs : SDNode<"ARMISD::VMINVs", SDTVecReduceR>;977 978// Find maximum unsigned value of a vector and register979def ARMVMAXVu : SDNode<"ARMISD::VMAXVu", SDTVecReduceR>;980 981// Find maximum signed value of a vector and register982def ARMVMAXVs : SDNode<"ARMISD::VMAXVs", SDTVecReduceR>;983 984defm MVE_VMINV : MVE_VMINMAXV_ty<"vminv", 1, "int_arm_mve_minv">;985defm MVE_VMAXV : MVE_VMINMAXV_ty<"vmaxv", 0, "int_arm_mve_maxv">;986 987let Predicates = [HasMVEInt] in {988 def : Pat<(i32 (vecreduce_smax (v16i8 MQPR:$src))),989 (i32 (MVE_VMAXVs8 (t2MVNi (i32 127)), $src))>;990 def : Pat<(i32 (vecreduce_smax (v8i16 MQPR:$src))),991 (i32 (MVE_VMAXVs16 (t2MOVi32imm (i32 -32768)), $src))>;992 def : Pat<(i32 (vecreduce_smax (v4i32 MQPR:$src))),993 (i32 (MVE_VMAXVs32 (t2MOVi (i32 -2147483648)), $src))>;994 def : Pat<(i32 (vecreduce_umax (v16i8 MQPR:$src))),995 (i32 (MVE_VMAXVu8 (t2MOVi (i32 0)), $src))>;996 def : Pat<(i32 (vecreduce_umax (v8i16 MQPR:$src))),997 (i32 (MVE_VMAXVu16 (t2MOVi (i32 0)), $src))>;998 def : Pat<(i32 (vecreduce_umax (v4i32 MQPR:$src))),999 (i32 (MVE_VMAXVu32 (t2MOVi (i32 0)), $src))>;1000 1001 def : Pat<(i32 (vecreduce_smin (v16i8 MQPR:$src))),1002 (i32 (MVE_VMINVs8 (t2MOVi (i32 127)), $src))>;1003 def : Pat<(i32 (vecreduce_smin (v8i16 MQPR:$src))),1004 (i32 (MVE_VMINVs16 (t2MOVi16 (i32 32767)), $src))>;1005 def : Pat<(i32 (vecreduce_smin (v4i32 MQPR:$src))),1006 (i32 (MVE_VMINVs32 (t2MVNi (i32 -2147483648)), $src))>;1007 def : Pat<(i32 (vecreduce_umin (v16i8 MQPR:$src))),1008 (i32 (MVE_VMINVu8 (t2MOVi (i32 255)), $src))>;1009 def : Pat<(i32 (vecreduce_umin (v8i16 MQPR:$src))),1010 (i32 (MVE_VMINVu16 (t2MOVi16 (i32 65535)), $src))>;1011 def : Pat<(i32 (vecreduce_umin (v4i32 MQPR:$src))),1012 (i32 (MVE_VMINVu32 (t2MOVi (i32 4294967295)), $src))>;1013 1014 def : Pat<(i32 (ARMVMINVu (i32 rGPR:$x), (v16i8 MQPR:$src))),1015 (i32 (MVE_VMINVu8 $x, $src))>;1016 def : Pat<(i32 (ARMVMINVu (i32 rGPR:$x), (v8i16 MQPR:$src))),1017 (i32 (MVE_VMINVu16 $x, $src))>;1018 def : Pat<(i32 (ARMVMINVu (i32 rGPR:$x), (v4i32 MQPR:$src))),1019 (i32 (MVE_VMINVu32 $x, $src))>;1020 def : Pat<(i32 (ARMVMINVs (i32 rGPR:$x), (v16i8 MQPR:$src))),1021 (i32 (MVE_VMINVs8 $x, $src))>;1022 def : Pat<(i32 (ARMVMINVs (i32 rGPR:$x), (v8i16 MQPR:$src))),1023 (i32 (MVE_VMINVs16 $x, $src))>;1024 def : Pat<(i32 (ARMVMINVs (i32 rGPR:$x), (v4i32 MQPR:$src))),1025 (i32 (MVE_VMINVs32 $x, $src))>;1026 1027 def : Pat<(i32 (ARMVMAXVu (i32 rGPR:$x), (v16i8 MQPR:$src))),1028 (i32 (MVE_VMAXVu8 $x, $src))>;1029 def : Pat<(i32 (ARMVMAXVu (i32 rGPR:$x), (v8i16 MQPR:$src))),1030 (i32 (MVE_VMAXVu16 $x, $src))>;1031 def : Pat<(i32 (ARMVMAXVu (i32 rGPR:$x), (v4i32 MQPR:$src))),1032 (i32 (MVE_VMAXVu32 $x, $src))>;1033 def : Pat<(i32 (ARMVMAXVs (i32 rGPR:$x), (v16i8 MQPR:$src))),1034 (i32 (MVE_VMAXVs8 $x, $src))>;1035 def : Pat<(i32 (ARMVMAXVs (i32 rGPR:$x), (v8i16 MQPR:$src))),1036 (i32 (MVE_VMAXVs16 $x, $src))>;1037 def : Pat<(i32 (ARMVMAXVs (i32 rGPR:$x), (v4i32 MQPR:$src))),1038 (i32 (MVE_VMAXVs32 $x, $src))>;1039 1040}1041 1042multiclass MVE_VMINMAXAV_ty<string iname, bit isMin, string intrBaseName> {1043 defm s8 : MVE_VMINMAXV_p<iname, 0, isMin, MVE_v16s8, intrBaseName>;1044 defm s16: MVE_VMINMAXV_p<iname, 0, isMin, MVE_v8s16, intrBaseName>;1045 defm s32: MVE_VMINMAXV_p<iname, 0, isMin, MVE_v4s32, intrBaseName>;1046}1047 1048defm MVE_VMINAV : MVE_VMINMAXAV_ty<"vminav", 1, "int_arm_mve_minav">;1049defm MVE_VMAXAV : MVE_VMINMAXAV_ty<"vmaxav", 0, "int_arm_mve_maxav">;1050 1051class MVE_VMLAMLSDAV<string iname, string suffix, dag iops, string cstr,1052 bit sz, bit bit_28, bit A, bit X, bit bit_8, bit bit_0,1053 bits<2> vecsize>1054 : MVE_rDest<(outs tGPREven:$RdaDest), iops, NoItinerary, iname, suffix,1055 "$RdaDest, $Qn, $Qm", cstr, vecsize, []> {1056 bits<4> RdaDest;1057 bits<3> Qm;1058 bits<3> Qn;1059 1060 let Inst{28} = bit_28;1061 let Inst{22-20} = 0b111;1062 let Inst{19-17} = Qn{2-0};1063 let Inst{16} = sz;1064 let Inst{15-13} = RdaDest{3-1};1065 let Inst{12} = X;1066 let Inst{8} = bit_8;1067 let Inst{7-6} = 0b00;1068 let Inst{5} = A;1069 let Inst{3-1} = Qm{2-0};1070 let Inst{0} = bit_0;1071 let horizontalReduction = 1;1072 // Allow tail predication for non-exchanging versions. As this is also a1073 // horizontalReduction, ARMLowOverheadLoops will also have to check that1074 // the vector operands contain zeros in their false lanes for the instruction1075 // to be properly valid.1076 let validForTailPredication = !eq(X, 0);1077}1078 1079multiclass MVE_VMLAMLSDAV_A<string iname, string x, MVEVectorVTInfo VTI,1080 bit sz, bit bit_28, bit X, bit bit_8, bit bit_0> {1081 def ""#x#VTI.Suffix : MVE_VMLAMLSDAV<iname # x, VTI.Suffix,1082 (ins MQPR:$Qn, MQPR:$Qm), "",1083 sz, bit_28, 0b0, X, bit_8, bit_0, VTI.Size>;1084 def "a"#x#VTI.Suffix : MVE_VMLAMLSDAV<iname # "a" # x, VTI.Suffix,1085 (ins tGPREven:$RdaSrc, MQPR:$Qn, MQPR:$Qm),1086 "$RdaDest = $RdaSrc",1087 sz, bit_28, 0b1, X, bit_8, bit_0, VTI.Size>;1088 let Predicates = [HasMVEInt] in {1089 def : Pat<(i32 (int_arm_mve_vmldava1090 (i32 VTI.Unsigned),1091 (i32 bit_0) /* subtract */,1092 (i32 X) /* exchange */,1093 (i32 0) /* accumulator */,1094 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),1095 (i32 (!cast<Instruction>(NAME # x # VTI.Suffix)1096 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm)))>;1097 1098 def : Pat<(i32 (int_arm_mve_vmldava_predicated1099 (i32 VTI.Unsigned),1100 (i32 bit_0) /* subtract */,1101 (i32 X) /* exchange */,1102 (i32 0) /* accumulator */,1103 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),1104 (VTI.Pred VCCR:$mask))),1105 (i32 (!cast<Instruction>(NAME # x # VTI.Suffix)1106 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),1107 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;1108 1109 def : Pat<(i32 (int_arm_mve_vmldava1110 (i32 VTI.Unsigned),1111 (i32 bit_0) /* subtract */,1112 (i32 X) /* exchange */,1113 (i32 tGPREven:$RdaSrc),1114 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),1115 (i32 (!cast<Instruction>(NAME # "a" # x # VTI.Suffix)1116 (i32 tGPREven:$RdaSrc),1117 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm)))>;1118 1119 def : Pat<(i32 (int_arm_mve_vmldava_predicated1120 (i32 VTI.Unsigned),1121 (i32 bit_0) /* subtract */,1122 (i32 X) /* exchange */,1123 (i32 tGPREven:$RdaSrc),1124 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),1125 (VTI.Pred VCCR:$mask))),1126 (i32 (!cast<Instruction>(NAME # "a" # x # VTI.Suffix)1127 (i32 tGPREven:$RdaSrc),1128 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),1129 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;1130 }1131}1132 1133multiclass MVE_VMLAMLSDAV_AX<string iname, MVEVectorVTInfo VTI, bit sz,1134 bit bit_28, bit bit_8, bit bit_0> {1135 defm "" : MVE_VMLAMLSDAV_A<iname, "", VTI, sz, bit_28,1136 0b0, bit_8, bit_0>;1137 defm "" : MVE_VMLAMLSDAV_A<iname, "x", VTI, sz, bit_28,1138 0b1, bit_8, bit_0>;1139}1140 1141multiclass MVE_VMLADAV_multi<MVEVectorVTInfo SVTI, MVEVectorVTInfo UVTI,1142 bit sz, bit bit_8> {1143 defm "" : MVE_VMLAMLSDAV_AX<"vmladav", SVTI,1144 sz, 0b0, bit_8, 0b0>;1145 defm "" : MVE_VMLAMLSDAV_A<"vmladav", "", UVTI,1146 sz, 0b1, 0b0, bit_8, 0b0>;1147}1148 1149multiclass MVE_VMLSDAV_multi<MVEVectorVTInfo VTI, bit sz, bit bit_28> {1150 defm "" : MVE_VMLAMLSDAV_AX<"vmlsdav", VTI,1151 sz, bit_28, 0b0, 0b1>;1152}1153 1154defm MVE_VMLADAV : MVE_VMLADAV_multi<MVE_v16s8, MVE_v16u8, 0b0, 0b1>;1155defm MVE_VMLADAV : MVE_VMLADAV_multi<MVE_v8s16, MVE_v8u16, 0b0, 0b0>;1156defm MVE_VMLADAV : MVE_VMLADAV_multi<MVE_v4s32, MVE_v4u32, 0b1, 0b0>;1157 1158defm MVE_VMLSDAV : MVE_VMLSDAV_multi<MVE_v16s8, 0b0, 0b1>;1159defm MVE_VMLSDAV : MVE_VMLSDAV_multi<MVE_v8s16, 0b0, 0b0>;1160defm MVE_VMLSDAV : MVE_VMLSDAV_multi<MVE_v4s32, 0b1, 0b0>;1161 1162def SDTVecReduce2 : SDTypeProfile<1, 2, [ // VMLAV1163 SDTCisInt<0>, SDTCisVec<1>, SDTCisVec<2>1164]>;1165def SDTVecReduce2L : SDTypeProfile<2, 2, [ // VMLALV1166 SDTCisInt<0>, SDTCisInt<1>, SDTCisVec<2>, SDTCisVec<3>1167]>;1168def SDTVecReduce2LA : SDTypeProfile<2, 4, [ // VMLALVA1169 SDTCisInt<0>, SDTCisInt<1>, SDTCisInt<2>, SDTCisInt<3>,1170 SDTCisVec<4>, SDTCisVec<5>1171]>;1172def SDTVecReduce2P : SDTypeProfile<1, 3, [ // VMLAV1173 SDTCisInt<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVec<3>1174]>;1175def SDTVecReduce2LP : SDTypeProfile<2, 3, [ // VMLALV1176 SDTCisInt<0>, SDTCisInt<1>, SDTCisVec<2>, SDTCisVec<3>, SDTCisVec<4>1177]>;1178def SDTVecReduce2LAP : SDTypeProfile<2, 5, [ // VMLALVA1179 SDTCisInt<0>, SDTCisInt<1>, SDTCisInt<2>, SDTCisInt<3>,1180 SDTCisVec<4>, SDTCisVec<5>, SDTCisVec<6>1181]>;1182 1183// sign- or zero-extend the elements of two vectors to i32, multiply1184// them and add the results together, returning an i32 of the sum1185def ARMVMLAVs : SDNode<"ARMISD::VMLAVs", SDTVecReduce2>;1186def ARMVMLAVu : SDNode<"ARMISD::VMLAVu", SDTVecReduce2>;1187 1188// Same as VMLAV but with i64, returning the low and1189// high 32-bit halves of the sum1190def ARMVMLALVs : SDNode<"ARMISD::VMLALVs", SDTVecReduce2L>;1191def ARMVMLALVu : SDNode<"ARMISD::VMLALVu", SDTVecReduce2L>;1192 1193// Same as VMLALV but also add an input accumulator1194// provided as low and high halves1195def ARMVMLALVAs : SDNode<"ARMISD::VMLALVAs", SDTVecReduce2LA>;1196def ARMVMLALVAu : SDNode<"ARMISD::VMLALVAu", SDTVecReduce2LA>;1197 1198// Same as VMLAV[su] with a v4i1 predicate mask1199def ARMVMLAVps : SDNode<"ARMISD::VMLAVps", SDTVecReduce2P>;1200def ARMVMLAVpu : SDNode<"ARMISD::VMLAVpu", SDTVecReduce2P>;1201 1202// Same as VMLALV[su] with a v4i1 predicate mask1203def ARMVMLALVps : SDNode<"ARMISD::VMLALVps", SDTVecReduce2LP>;1204def ARMVMLALVpu : SDNode<"ARMISD::VMLALVpu", SDTVecReduce2LP>;1205 1206// Same as VMLALVA[su] with a v4i1 predicate mask1207def ARMVMLALVAps : SDNode<"ARMISD::VMLALVAps", SDTVecReduce2LAP>;1208def ARMVMLALVApu : SDNode<"ARMISD::VMLALVApu", SDTVecReduce2LAP>;1209 1210let Predicates = [HasMVEInt] in {1211 def : Pat<(i32 (vecreduce_add (mul (v4i32 MQPR:$src1), (v4i32 MQPR:$src2)))),1212 (i32 (MVE_VMLADAVu32 $src1, $src2))>;1213 def : Pat<(i32 (vecreduce_add (mul (v8i16 MQPR:$src1), (v8i16 MQPR:$src2)))),1214 (i32 (MVE_VMLADAVu16 $src1, $src2))>;1215 def : Pat<(i32 (ARMVMLAVs (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))),1216 (i32 (MVE_VMLADAVs16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;1217 def : Pat<(i32 (ARMVMLAVu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))),1218 (i32 (MVE_VMLADAVu16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;1219 def : Pat<(i32 (vecreduce_add (mul (v16i8 MQPR:$src1), (v16i8 MQPR:$src2)))),1220 (i32 (MVE_VMLADAVu8 $src1, $src2))>;1221 def : Pat<(i32 (ARMVMLAVs (v16i8 MQPR:$val1), (v16i8 MQPR:$val2))),1222 (i32 (MVE_VMLADAVs8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;1223 def : Pat<(i32 (ARMVMLAVu (v16i8 MQPR:$val1), (v16i8 MQPR:$val2))),1224 (i32 (MVE_VMLADAVu8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;1225 1226 def : Pat<(i32 (add (i32 (vecreduce_add (mul (v4i32 MQPR:$src1), (v4i32 MQPR:$src2)))),1227 (i32 tGPREven:$src3))),1228 (i32 (MVE_VMLADAVau32 $src3, $src1, $src2))>;1229 def : Pat<(i32 (add (i32 (vecreduce_add (mul (v8i16 MQPR:$src1), (v8i16 MQPR:$src2)))),1230 (i32 tGPREven:$src3))),1231 (i32 (MVE_VMLADAVau16 $src3, $src1, $src2))>;1232 def : Pat<(i32 (add (ARMVMLAVs (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)), tGPREven:$Rd)),1233 (i32 (MVE_VMLADAVas16 tGPREven:$Rd, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;1234 def : Pat<(i32 (add (ARMVMLAVu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)), tGPREven:$Rd)),1235 (i32 (MVE_VMLADAVau16 tGPREven:$Rd, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;1236 def : Pat<(i32 (add (i32 (vecreduce_add (mul (v16i8 MQPR:$src1), (v16i8 MQPR:$src2)))),1237 (i32 tGPREven:$src3))),1238 (i32 (MVE_VMLADAVau8 $src3, $src1, $src2))>;1239 def : Pat<(i32 (add (ARMVMLAVs (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)), tGPREven:$Rd)),1240 (i32 (MVE_VMLADAVas8 tGPREven:$Rd, (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;1241 def : Pat<(i32 (add (ARMVMLAVu (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)), tGPREven:$Rd)),1242 (i32 (MVE_VMLADAVau8 tGPREven:$Rd, (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;1243 1244 // Predicated1245 def : Pat<(i32 (vecreduce_add (vselect (v4i1 VCCR:$pred),1246 (mul (v4i32 MQPR:$src1), (v4i32 MQPR:$src2)),1247 (v4i32 ARMimmAllZerosV)))),1248 (i32 (MVE_VMLADAVu32 $src1, $src2, ARMVCCThen, $pred, zero_reg))>;1249 def : Pat<(i32 (vecreduce_add (vselect (v8i1 VCCR:$pred),1250 (mul (v8i16 MQPR:$src1), (v8i16 MQPR:$src2)),1251 (v8i16 ARMimmAllZerosV)))),1252 (i32 (MVE_VMLADAVu16 $src1, $src2, ARMVCCThen, $pred, zero_reg))>;1253 def : Pat<(i32 (ARMVMLAVps (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred))),1254 (i32 (MVE_VMLADAVs16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;1255 def : Pat<(i32 (ARMVMLAVpu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred))),1256 (i32 (MVE_VMLADAVu16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;1257 def : Pat<(i32 (vecreduce_add (vselect (v16i1 VCCR:$pred),1258 (mul (v16i8 MQPR:$src1), (v16i8 MQPR:$src2)),1259 (v16i8 ARMimmAllZerosV)))),1260 (i32 (MVE_VMLADAVu8 $src1, $src2, ARMVCCThen, $pred, zero_reg))>;1261 def : Pat<(i32 (ARMVMLAVps (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), (v16i1 VCCR:$pred))),1262 (i32 (MVE_VMLADAVs8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;1263 def : Pat<(i32 (ARMVMLAVpu (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), (v16i1 VCCR:$pred))),1264 (i32 (MVE_VMLADAVu8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;1265 1266 def : Pat<(i32 (add (i32 (vecreduce_add (vselect (v4i1 VCCR:$pred),1267 (mul (v4i32 MQPR:$src1), (v4i32 MQPR:$src2)),1268 (v4i32 ARMimmAllZerosV)))),1269 (i32 tGPREven:$src3))),1270 (i32 (MVE_VMLADAVau32 $src3, $src1, $src2, ARMVCCThen, $pred, zero_reg))>;1271 def : Pat<(i32 (add (i32 (vecreduce_add (vselect (v8i1 VCCR:$pred),1272 (mul (v8i16 MQPR:$src1), (v8i16 MQPR:$src2)),1273 (v8i16 ARMimmAllZerosV)))),1274 (i32 tGPREven:$src3))),1275 (i32 (MVE_VMLADAVau16 $src3, $src1, $src2, ARMVCCThen, $pred, zero_reg))>;1276 def : Pat<(i32 (add (ARMVMLAVps (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)), tGPREven:$Rd)),1277 (i32 (MVE_VMLADAVas16 tGPREven:$Rd, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;1278 def : Pat<(i32 (add (ARMVMLAVpu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)), tGPREven:$Rd)),1279 (i32 (MVE_VMLADAVau16 tGPREven:$Rd, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;1280 def : Pat<(i32 (add (i32 (vecreduce_add (vselect (v16i1 VCCR:$pred),1281 (mul (v16i8 MQPR:$src1), (v16i8 MQPR:$src2)),1282 (v16i8 ARMimmAllZerosV)))),1283 (i32 tGPREven:$src3))),1284 (i32 (MVE_VMLADAVau8 $src3, $src1, $src2, ARMVCCThen, $pred, zero_reg))>;1285 def : Pat<(i32 (add (ARMVMLAVps (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), (v16i1 VCCR:$pred)), tGPREven:$Rd)),1286 (i32 (MVE_VMLADAVas8 tGPREven:$Rd, (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;1287 def : Pat<(i32 (add (ARMVMLAVpu (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), (v16i1 VCCR:$pred)), tGPREven:$Rd)),1288 (i32 (MVE_VMLADAVau8 tGPREven:$Rd, (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;1289}1290 1291// vmlav aliases vmladav1292foreach acc = ["", "a"] in {1293 foreach suffix = ["s8", "s16", "s32", "u8", "u16", "u32"] in {1294 def : MVEInstAlias<"vmlav"#acc#"${vp}."#suffix#"\t$RdaDest, $Qn, $Qm",1295 (!cast<Instruction>("MVE_VMLADAV"#acc#suffix)1296 tGPREven:$RdaDest, MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;1297 }1298}1299 1300// Base class for VMLALDAV and VMLSLDAV, VRMLALDAVH, VRMLSLDAVH1301class MVE_VMLALDAVBase<string iname, string suffix, dag iops, string cstr,1302 bit sz, bit bit_28, bit A, bit X, bit bit_8, bit bit_0,1303 bits<2> vecsize, list<dag> pattern=[]>1304 : MVE_rDest<(outs tGPREven:$RdaLoDest, tGPROdd:$RdaHiDest), iops, NoItinerary,1305 iname, suffix, "$RdaLoDest, $RdaHiDest, $Qn, $Qm", cstr, vecsize, pattern> {1306 bits<4> RdaLoDest;1307 bits<4> RdaHiDest;1308 bits<3> Qm;1309 bits<3> Qn;1310 1311 let Inst{28} = bit_28;1312 let Inst{22-20} = RdaHiDest{3-1};1313 let Inst{19-17} = Qn{2-0};1314 let Inst{16} = sz;1315 let Inst{15-13} = RdaLoDest{3-1};1316 let Inst{12} = X;1317 let Inst{8} = bit_8;1318 let Inst{7-6} = 0b00;1319 let Inst{5} = A;1320 let Inst{3-1} = Qm{2-0};1321 let Inst{0} = bit_0;1322 let horizontalReduction = 1;1323 // Allow tail predication for non-exchanging versions. As this is also a1324 // horizontalReduction, ARMLowOverheadLoops will also have to check that1325 // the vector operands contain zeros in their false lanes for the instruction1326 // to be properly valid.1327 let validForTailPredication = !eq(X, 0);1328 1329 let hasSideEffects = 0;1330}1331 1332multiclass MVE_VMLALDAVBase_A<string iname, string x, string suffix,1333 bit sz, bit bit_28, bit X, bit bit_8, bit bit_0,1334 bits<2> vecsize, list<dag> pattern=[]> {1335 def ""#x#suffix : MVE_VMLALDAVBase<1336 iname # x, suffix, (ins MQPR:$Qn, MQPR:$Qm), "",1337 sz, bit_28, 0b0, X, bit_8, bit_0, vecsize, pattern>;1338 def "a"#x#suffix : MVE_VMLALDAVBase<1339 iname # "a" # x, suffix,1340 (ins tGPREven:$RdaLoSrc, tGPROdd:$RdaHiSrc, MQPR:$Qn, MQPR:$Qm),1341 "$RdaLoDest = $RdaLoSrc,$RdaHiDest = $RdaHiSrc",1342 sz, bit_28, 0b1, X, bit_8, bit_0, vecsize, pattern>;1343}1344 1345 1346multiclass MVE_VMLALDAVBase_AX<string iname, string suffix, bit sz, bit bit_28,1347 bit bit_8, bit bit_0, bits<2> vecsize, list<dag> pattern=[]> {1348 defm "" : MVE_VMLALDAVBase_A<iname, "", suffix, sz,1349 bit_28, 0b0, bit_8, bit_0, vecsize, pattern>;1350 defm "" : MVE_VMLALDAVBase_A<iname, "x", suffix, sz,1351 bit_28, 0b1, bit_8, bit_0, vecsize, pattern>;1352}1353 1354multiclass MVE_VRMLALDAVH_multi<MVEVectorVTInfo VTI, list<dag> pattern=[]> {1355 defm "" : MVE_VMLALDAVBase_AX<"vrmlaldavh", "s"#VTI.BitsSuffix,1356 0b0, 0b0, 0b1, 0b0, VTI.Size, pattern>;1357 defm "" : MVE_VMLALDAVBase_A<"vrmlaldavh", "", "u"#VTI.BitsSuffix,1358 0b0, 0b1, 0b0, 0b1, 0b0, VTI.Size, pattern>;1359}1360 1361defm MVE_VRMLALDAVH : MVE_VRMLALDAVH_multi<MVE_v4i32>;1362 1363// vrmlalvh aliases for vrmlaldavh1364def : MVEInstAlias<"vrmlalvh${vp}.s32\t$RdaLo, $RdaHi, $Qn, $Qm",1365 (MVE_VRMLALDAVHs321366 tGPREven:$RdaLo, tGPROdd:$RdaHi,1367 MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;1368def : MVEInstAlias<"vrmlalvha${vp}.s32\t$RdaLo, $RdaHi, $Qn, $Qm",1369 (MVE_VRMLALDAVHas321370 tGPREven:$RdaLo, tGPROdd:$RdaHi,1371 MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;1372def : MVEInstAlias<"vrmlalvh${vp}.u32\t$RdaLo, $RdaHi, $Qn, $Qm",1373 (MVE_VRMLALDAVHu321374 tGPREven:$RdaLo, tGPROdd:$RdaHi,1375 MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;1376def : MVEInstAlias<"vrmlalvha${vp}.u32\t$RdaLo, $RdaHi, $Qn, $Qm",1377 (MVE_VRMLALDAVHau321378 tGPREven:$RdaLo, tGPROdd:$RdaHi,1379 MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;1380 1381multiclass MVE_VMLALDAV_multi<MVEVectorVTInfo VTI, list<dag> pattern=[]> {1382 defm "" : MVE_VMLALDAVBase_AX<"vmlaldav", "s"#VTI.BitsSuffix,1383 VTI.Size{1}, 0b0, 0b0, 0b0, VTI.Size, pattern>;1384 defm "" : MVE_VMLALDAVBase_A<"vmlaldav", "", "u"#VTI.BitsSuffix,1385 VTI.Size{1}, 0b1, 0b0, 0b0, 0b0, VTI.Size, pattern>;1386}1387 1388defm MVE_VMLALDAV : MVE_VMLALDAV_multi<MVE_v8i16>;1389defm MVE_VMLALDAV : MVE_VMLALDAV_multi<MVE_v4i32>;1390 1391let Predicates = [HasMVEInt] in {1392 def : Pat<(ARMVMLALVs (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)),1393 (MVE_VMLALDAVs32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))>;1394 def : Pat<(ARMVMLALVu (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)),1395 (MVE_VMLALDAVu32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))>;1396 def : Pat<(ARMVMLALVs (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)),1397 (MVE_VMLALDAVs16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))>;1398 def : Pat<(ARMVMLALVu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)),1399 (MVE_VMLALDAVu16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))>;1400 1401 def : Pat<(ARMVMLALVAs tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)),1402 (MVE_VMLALDAVas32 tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))>;1403 def : Pat<(ARMVMLALVAu tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)),1404 (MVE_VMLALDAVau32 tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))>;1405 def : Pat<(ARMVMLALVAs tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)),1406 (MVE_VMLALDAVas16 tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))>;1407 def : Pat<(ARMVMLALVAu tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)),1408 (MVE_VMLALDAVau16 tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))>;1409 1410 // Predicated1411 def : Pat<(ARMVMLALVps (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), (v4i1 VCCR:$pred)),1412 (MVE_VMLALDAVs32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;1413 def : Pat<(ARMVMLALVpu (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), (v4i1 VCCR:$pred)),1414 (MVE_VMLALDAVu32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;1415 def : Pat<(ARMVMLALVps (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)),1416 (MVE_VMLALDAVs16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;1417 def : Pat<(ARMVMLALVpu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)),1418 (MVE_VMLALDAVu16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;1419 1420 def : Pat<(ARMVMLALVAps tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), (v4i1 VCCR:$pred)),1421 (MVE_VMLALDAVas32 tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;1422 def : Pat<(ARMVMLALVApu tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), (v4i1 VCCR:$pred)),1423 (MVE_VMLALDAVau32 tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;1424 def : Pat<(ARMVMLALVAps tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)),1425 (MVE_VMLALDAVas16 tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;1426 def : Pat<(ARMVMLALVApu tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)),1427 (MVE_VMLALDAVau16 tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;1428}1429 1430// vmlalv aliases vmlaldav1431foreach acc = ["", "a"] in {1432 foreach suffix = ["s16", "s32", "u16", "u32"] in {1433 def : MVEInstAlias<"vmlalv" # acc # "${vp}." # suffix #1434 "\t$RdaLoDest, $RdaHiDest, $Qn, $Qm",1435 (!cast<Instruction>("MVE_VMLALDAV"#acc#suffix)1436 tGPREven:$RdaLoDest, tGPROdd:$RdaHiDest,1437 MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;1438 }1439}1440 1441multiclass MVE_VMLSLDAV_multi<string iname, string suffix, bit sz,1442 bit bit_28, bits<2> vecsize, list<dag> pattern=[]> {1443 defm "" : MVE_VMLALDAVBase_AX<iname, suffix, sz, bit_28, 0b0, 0b1, vecsize, pattern>;1444}1445 1446defm MVE_VMLSLDAV : MVE_VMLSLDAV_multi<"vmlsldav", "s16", 0b0, 0b0, 0b01>;1447defm MVE_VMLSLDAV : MVE_VMLSLDAV_multi<"vmlsldav", "s32", 0b1, 0b0, 0b10>;1448defm MVE_VRMLSLDAVH : MVE_VMLSLDAV_multi<"vrmlsldavh", "s32", 0b0, 0b1, 0b10>;1449 1450// end of mve_rDest instructions1451 1452// start of mve_comp instructions1453 1454class MVE_comp<InstrItinClass itin, string iname, string suffix,1455 string cstr, bits<2> vecsize, list<dag> pattern=[]>1456 : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm), itin, iname, suffix,1457 "$Qd, $Qn, $Qm", vpred_r, cstr, vecsize, pattern> {1458 bits<4> Qd;1459 bits<4> Qn;1460 bits<4> Qm;1461 1462 let Inst{22} = Qd{3};1463 let Inst{19-17} = Qn{2-0};1464 let Inst{16} = 0b0;1465 let Inst{15-13} = Qd{2-0};1466 let Inst{12} = 0b0;1467 let Inst{10-9} = 0b11;1468 let Inst{7} = Qn{3};1469 let Inst{5} = Qm{3};1470 let Inst{3-1} = Qm{2-0};1471 let Inst{0} = 0b0;1472}1473 1474class MVE_VMINMAXNM<string iname, string suffix, bits<2> sz, bit bit_21,1475 list<dag> pattern=[]>1476 : MVE_comp<NoItinerary, iname, suffix, "", sz, pattern> {1477 1478 let Inst{28} = 0b1;1479 let Inst{25-24} = 0b11;1480 let Inst{23} = 0b0;1481 let Inst{21} = bit_21;1482 let Inst{20} = sz{0};1483 let Inst{11} = 0b1;1484 let Inst{8} = 0b1;1485 let Inst{6} = 0b1;1486 let Inst{4} = 0b1;1487 1488 let Predicates = [HasMVEFloat];1489 let validForTailPredication = 1;1490}1491 1492multiclass MVE_VMINMAXNM_m<string iname, bit bit_4, MVEVectorVTInfo VTI, SDNode Op, Intrinsic PredInt> {1493 def "" : MVE_VMINMAXNM<iname, VTI.Suffix, VTI.Size, bit_4>;1494 1495 let Predicates = [HasMVEFloat] in {1496 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 0)), !cast<Instruction>(NAME)>;1497 }1498}1499 1500defm MVE_VMAXNMf32 : MVE_VMINMAXNM_m<"vmaxnm", 0b0, MVE_v4f32, fmaxnum, int_arm_mve_max_predicated>;1501defm MVE_VMAXNMf16 : MVE_VMINMAXNM_m<"vmaxnm", 0b0, MVE_v8f16, fmaxnum, int_arm_mve_max_predicated>;1502defm MVE_VMINNMf32 : MVE_VMINMAXNM_m<"vminnm", 0b1, MVE_v4f32, fminnum, int_arm_mve_min_predicated>;1503defm MVE_VMINNMf16 : MVE_VMINMAXNM_m<"vminnm", 0b1, MVE_v8f16, fminnum, int_arm_mve_min_predicated>;1504 1505 1506class MVE_VMINMAX<string iname, string suffix, bit U, bits<2> size,1507 bit bit_4, list<dag> pattern=[]>1508 : MVE_comp<NoItinerary, iname, suffix, "", size, pattern> {1509 1510 let Inst{28} = U;1511 let Inst{25-24} = 0b11;1512 let Inst{23} = 0b0;1513 let Inst{21-20} = size{1-0};1514 let Inst{11} = 0b0;1515 let Inst{8} = 0b0;1516 let Inst{6} = 0b1;1517 let Inst{4} = bit_4;1518 let validForTailPredication = 1;1519}1520 1521multiclass MVE_VMINMAX_m<string iname, bit bit_4, MVEVectorVTInfo VTI,1522 SDNode Op, Intrinsic PredInt> {1523 def "" : MVE_VMINMAX<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, bit_4>;1524 1525 let Predicates = [HasMVEInt] in {1526 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)), !cast<Instruction>(NAME)>;1527 }1528}1529 1530multiclass MVE_VMAX<MVEVectorVTInfo VTI>1531 : MVE_VMINMAX_m<"vmax", 0b0, VTI, !if(VTI.Unsigned, umax, smax), int_arm_mve_max_predicated>;1532multiclass MVE_VMIN<MVEVectorVTInfo VTI>1533 : MVE_VMINMAX_m<"vmin", 0b1, VTI, !if(VTI.Unsigned, umin, smin), int_arm_mve_min_predicated>;1534 1535defm MVE_VMINs8 : MVE_VMIN<MVE_v16s8>;1536defm MVE_VMINs16 : MVE_VMIN<MVE_v8s16>;1537defm MVE_VMINs32 : MVE_VMIN<MVE_v4s32>;1538defm MVE_VMINu8 : MVE_VMIN<MVE_v16u8>;1539defm MVE_VMINu16 : MVE_VMIN<MVE_v8u16>;1540defm MVE_VMINu32 : MVE_VMIN<MVE_v4u32>;1541 1542defm MVE_VMAXs8 : MVE_VMAX<MVE_v16s8>;1543defm MVE_VMAXs16 : MVE_VMAX<MVE_v8s16>;1544defm MVE_VMAXs32 : MVE_VMAX<MVE_v4s32>;1545defm MVE_VMAXu8 : MVE_VMAX<MVE_v16u8>;1546defm MVE_VMAXu16 : MVE_VMAX<MVE_v8u16>;1547defm MVE_VMAXu32 : MVE_VMAX<MVE_v4u32>;1548 1549// end of mve_comp instructions1550 1551// start of mve_bit instructions1552 1553class MVE_bit_arith<dag oops, dag iops, string iname, string suffix,1554 string ops, string cstr, bits<2> vecsize, list<dag> pattern=[]>1555 : MVE_p<oops, iops, NoItinerary, iname, suffix, ops, vpred_r, cstr, vecsize, pattern> {1556 bits<4> Qd;1557 bits<4> Qm;1558 1559 let Inst{22} = Qd{3};1560 let Inst{15-13} = Qd{2-0};1561 let Inst{5} = Qm{3};1562 let Inst{3-1} = Qm{2-0};1563}1564 1565def MVE_VBIC : MVE_bit_arith<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm),1566 "vbic", "", "$Qd, $Qn, $Qm", "", 0b00> {1567 bits<4> Qn;1568 1569 let Inst{28} = 0b0;1570 let Inst{25-23} = 0b110;1571 let Inst{21-20} = 0b01;1572 let Inst{19-17} = Qn{2-0};1573 let Inst{16} = 0b0;1574 let Inst{12-8} = 0b00001;1575 let Inst{7} = Qn{3};1576 let Inst{6} = 0b1;1577 let Inst{4} = 0b1;1578 let Inst{0} = 0b0;1579 let validForTailPredication = 1;1580}1581 1582class MVE_VREV<string iname, string suffix, bits<2> size, bits<2> bit_8_7,1583 bits<2> vecsize, string cstr="">1584 : MVE_bit_arith<(outs MQPR:$Qd), (ins MQPR:$Qm), iname,1585 suffix, "$Qd, $Qm", cstr, vecsize> {1586 1587 let Inst{28} = 0b1;1588 let Inst{25-23} = 0b111;1589 let Inst{21-20} = 0b11;1590 let Inst{19-18} = size;1591 let Inst{17-16} = 0b00;1592 let Inst{12-9} = 0b0000;1593 let Inst{8-7} = bit_8_7;1594 let Inst{6} = 0b1;1595 let Inst{4} = 0b0;1596 let Inst{0} = 0b0;1597}1598 1599def MVE_VREV64_8 : MVE_VREV<"vrev64", "8", 0b00, 0b00, 0b11, "@earlyclobber $Qd">;1600def MVE_VREV64_16 : MVE_VREV<"vrev64", "16", 0b01, 0b00, 0b11, "@earlyclobber $Qd">;1601def MVE_VREV64_32 : MVE_VREV<"vrev64", "32", 0b10, 0b00, 0b11, "@earlyclobber $Qd">;1602 1603def MVE_VREV32_8 : MVE_VREV<"vrev32", "8", 0b00, 0b01, 0b10>;1604def MVE_VREV32_16 : MVE_VREV<"vrev32", "16", 0b01, 0b01, 0b10>;1605 1606def MVE_VREV16_8 : MVE_VREV<"vrev16", "8", 0b00, 0b10, 0b01>;1607 1608let Predicates = [HasMVEInt] in {1609 def : Pat<(v8i16 (bswap (v8i16 MQPR:$src))),1610 (v8i16 (MVE_VREV16_8 (v8i16 MQPR:$src)))>;1611 def : Pat<(v4i32 (bswap (v4i32 MQPR:$src))),1612 (v4i32 (MVE_VREV32_8 (v4i32 MQPR:$src)))>;1613}1614 1615multiclass MVE_VREV_basic_patterns<int revbits, list<MVEVectorVTInfo> VTIs,1616 Instruction Inst> {1617 defvar unpred_op = !cast<SDNode>("ARMvrev" # revbits);1618 1619 foreach VTI = VTIs in {1620 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$src))),1621 (VTI.Vec (Inst (VTI.Vec MQPR:$src)))>;1622 def : Pat<(VTI.Vec (int_arm_mve_vrev_predicated (VTI.Vec MQPR:$src),1623 revbits, (VTI.Pred VCCR:$pred), (VTI.Vec MQPR:$inactive))),1624 (VTI.Vec (Inst (VTI.Vec MQPR:$src), ARMVCCThen,1625 (VTI.Pred VCCR:$pred), zero_reg, (VTI.Vec MQPR:$inactive)))>;1626 }1627}1628 1629let Predicates = [HasMVEInt] in {1630 defm: MVE_VREV_basic_patterns<64, [MVE_v4i32, MVE_v4f32], MVE_VREV64_32>;1631 defm: MVE_VREV_basic_patterns<64, [MVE_v8i16, MVE_v8f16], MVE_VREV64_16>;1632 defm: MVE_VREV_basic_patterns<64, [MVE_v16i8 ], MVE_VREV64_8>;1633 1634 defm: MVE_VREV_basic_patterns<32, [MVE_v8i16, MVE_v8f16], MVE_VREV32_16>;1635 defm: MVE_VREV_basic_patterns<32, [MVE_v16i8 ], MVE_VREV32_8>;1636 1637 defm: MVE_VREV_basic_patterns<16, [MVE_v16i8 ], MVE_VREV16_8>;1638}1639 1640def MVE_VMVN : MVE_bit_arith<(outs MQPR:$Qd), (ins MQPR:$Qm),1641 "vmvn", "", "$Qd, $Qm", "", 0b00> {1642 let Inst{28} = 0b1;1643 let Inst{25-23} = 0b111;1644 let Inst{21-16} = 0b110000;1645 let Inst{12-6} = 0b0010111;1646 let Inst{4} = 0b0;1647 let Inst{0} = 0b0;1648 let validForTailPredication = 1;1649}1650 1651let Predicates = [HasMVEInt] in {1652 foreach VTI = [ MVE_v16i8, MVE_v8i16, MVE_v4i32, MVE_v2i64 ] in {1653 def : Pat<(VTI.Vec (vnotq (VTI.Vec MQPR:$val1))),1654 (VTI.Vec (MVE_VMVN (VTI.Vec MQPR:$val1)))>;1655 def : Pat<(VTI.Vec (int_arm_mve_mvn_predicated (VTI.Vec MQPR:$val1),1656 (VTI.Pred VCCR:$pred), (VTI.Vec MQPR:$inactive))),1657 (VTI.Vec (MVE_VMVN (VTI.Vec MQPR:$val1), ARMVCCThen,1658 (VTI.Pred VCCR:$pred), zero_reg, (VTI.Vec MQPR:$inactive)))>;1659 }1660}1661 1662class MVE_bit_ops<string iname, bits<2> bit_21_20, bit bit_28>1663 : MVE_bit_arith<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm),1664 iname, "", "$Qd, $Qn, $Qm", "", 0b00> {1665 bits<4> Qn;1666 1667 let Inst{28} = bit_28;1668 let Inst{25-23} = 0b110;1669 let Inst{21-20} = bit_21_20;1670 let Inst{19-17} = Qn{2-0};1671 let Inst{16} = 0b0;1672 let Inst{12-8} = 0b00001;1673 let Inst{7} = Qn{3};1674 let Inst{6} = 0b1;1675 let Inst{4} = 0b1;1676 let Inst{0} = 0b0;1677 let validForTailPredication = 1;1678}1679 1680def MVE_VEOR : MVE_bit_ops<"veor", 0b00, 0b1>;1681def MVE_VORN : MVE_bit_ops<"vorn", 0b11, 0b0>;1682def MVE_VORR : MVE_bit_ops<"vorr", 0b10, 0b0>;1683def MVE_VAND : MVE_bit_ops<"vand", 0b00, 0b0>;1684 1685// add ignored suffixes as aliases1686 1687foreach s=["s8", "s16", "s32", "u8", "u16", "u32", "i8", "i16", "i32", "f16", "f32"] in {1688 def : MVEInstAlias<"vbic${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",1689 (MVE_VBIC MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;1690 def : MVEInstAlias<"veor${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",1691 (MVE_VEOR MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;1692 def : MVEInstAlias<"vorn${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",1693 (MVE_VORN MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;1694 def : MVEInstAlias<"vorr${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",1695 (MVE_VORR MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;1696 def : MVEInstAlias<"vand${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",1697 (MVE_VAND MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;1698}1699 1700let Predicates = [HasMVEInt] in {1701 defm : MVE_TwoOpPattern<MVE_v16i8, and, int_arm_mve_and_predicated, (? ), MVE_VAND, ARMimmAllOnesV>;1702 defm : MVE_TwoOpPattern<MVE_v8i16, and, int_arm_mve_and_predicated, (? ), MVE_VAND, ARMimmAllOnesV>;1703 defm : MVE_TwoOpPattern<MVE_v4i32, and, int_arm_mve_and_predicated, (? ), MVE_VAND, ARMimmAllOnesV>;1704 defm : MVE_TwoOpPattern<MVE_v2i64, and, int_arm_mve_and_predicated, (? ), MVE_VAND, ARMimmAllOnesV>;1705 1706 defm : MVE_TwoOpPattern<MVE_v16i8, or, int_arm_mve_orr_predicated, (? ), MVE_VORR, ARMimmAllZerosV>;1707 defm : MVE_TwoOpPattern<MVE_v8i16, or, int_arm_mve_orr_predicated, (? ), MVE_VORR, ARMimmAllZerosV>;1708 defm : MVE_TwoOpPattern<MVE_v4i32, or, int_arm_mve_orr_predicated, (? ), MVE_VORR, ARMimmAllZerosV>;1709 defm : MVE_TwoOpPattern<MVE_v2i64, or, int_arm_mve_orr_predicated, (? ), MVE_VORR, ARMimmAllZerosV>;1710 1711 defm : MVE_TwoOpPattern<MVE_v16i8, xor, int_arm_mve_eor_predicated, (? ), MVE_VEOR, ARMimmAllZerosV>;1712 defm : MVE_TwoOpPattern<MVE_v8i16, xor, int_arm_mve_eor_predicated, (? ), MVE_VEOR, ARMimmAllZerosV>;1713 defm : MVE_TwoOpPattern<MVE_v4i32, xor, int_arm_mve_eor_predicated, (? ), MVE_VEOR, ARMimmAllZerosV>;1714 defm : MVE_TwoOpPattern<MVE_v2i64, xor, int_arm_mve_eor_predicated, (? ), MVE_VEOR, ARMimmAllZerosV>;1715 1716 defm : MVE_TwoOpPattern<MVE_v16i8, BinOpFrag<(and node:$LHS, (vnotq node:$RHS))>,1717 int_arm_mve_bic_predicated, (? ), MVE_VBIC>;1718 defm : MVE_TwoOpPattern<MVE_v8i16, BinOpFrag<(and node:$LHS, (vnotq node:$RHS))>,1719 int_arm_mve_bic_predicated, (? ), MVE_VBIC>;1720 defm : MVE_TwoOpPattern<MVE_v4i32, BinOpFrag<(and node:$LHS, (vnotq node:$RHS))>,1721 int_arm_mve_bic_predicated, (? ), MVE_VBIC>;1722 defm : MVE_TwoOpPattern<MVE_v2i64, BinOpFrag<(and node:$LHS, (vnotq node:$RHS))>,1723 int_arm_mve_bic_predicated, (? ), MVE_VBIC>;1724 1725 defm : MVE_TwoOpPattern<MVE_v16i8, BinOpFrag<(or node:$LHS, (vnotq node:$RHS))>,1726 int_arm_mve_orn_predicated, (? ), MVE_VORN>;1727 defm : MVE_TwoOpPattern<MVE_v8i16, BinOpFrag<(or node:$LHS, (vnotq node:$RHS))>,1728 int_arm_mve_orn_predicated, (? ), MVE_VORN>;1729 defm : MVE_TwoOpPattern<MVE_v4i32, BinOpFrag<(or node:$LHS, (vnotq node:$RHS))>,1730 int_arm_mve_orn_predicated, (? ), MVE_VORN>;1731 defm : MVE_TwoOpPattern<MVE_v2i64, BinOpFrag<(or node:$LHS, (vnotq node:$RHS))>,1732 int_arm_mve_orn_predicated, (? ), MVE_VORN>;1733}1734 1735class MVE_bit_cmode<string iname, string suffix, bit halfword, dag inOps, bits<2> vecsize>1736 : MVE_p<(outs MQPR:$Qd), inOps, NoItinerary,1737 iname, suffix, "$Qd, $imm", vpred_n, "$Qd = $Qd_src", vecsize> {1738 bits<12> imm;1739 bits<4> Qd;1740 1741 let Inst{28} = imm{7};1742 let Inst{27-23} = 0b11111;1743 let Inst{22} = Qd{3};1744 let Inst{21-19} = 0b000;1745 let Inst{18-16} = imm{6-4};1746 let Inst{15-13} = Qd{2-0};1747 let Inst{12} = 0b0;1748 let Inst{11} = halfword;1749 let Inst{10} = !if(halfword, 0, imm{10});1750 let Inst{9} = imm{9};1751 let Inst{8} = 0b1;1752 let Inst{7-6} = 0b01;1753 let Inst{4} = 0b1;1754 let Inst{3-0} = imm{3-0};1755}1756 1757multiclass MVE_bit_cmode_p<string iname, bit opcode,1758 MVEVectorVTInfo VTI, Operand imm_type, SDNode op> {1759 def "" : MVE_bit_cmode<iname, VTI.Suffix, VTI.Size{0},1760 (ins MQPR:$Qd_src, imm_type:$imm), VTI.Size> {1761 let Inst{5} = opcode;1762 let validForTailPredication = 1;1763 }1764 1765 defvar Inst = !cast<Instruction>(NAME);1766 defvar UnpredPat = (VTI.Vec (op (VTI.Vec MQPR:$src), timm:$simm));1767 1768 let Predicates = [HasMVEInt] in {1769 def : Pat<UnpredPat,1770 (VTI.Vec (Inst (VTI.Vec MQPR:$src), imm_type:$simm))>;1771 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),1772 UnpredPat, (VTI.Vec MQPR:$src))),1773 (VTI.Vec (Inst (VTI.Vec MQPR:$src), imm_type:$simm,1774 ARMVCCThen, (VTI.Pred VCCR:$pred), zero_reg))>;1775 }1776}1777 1778multiclass MVE_VORRimm<MVEVectorVTInfo VTI, Operand imm_type> {1779 defm "": MVE_bit_cmode_p<"vorr", 0, VTI, imm_type, ARMvorrImm>;1780}1781multiclass MVE_VBICimm<MVEVectorVTInfo VTI, Operand imm_type> {1782 defm "": MVE_bit_cmode_p<"vbic", 1, VTI, imm_type, ARMvbicImm>;1783}1784 1785defm MVE_VORRimmi16 : MVE_VORRimm<MVE_v8i16, nImmSplatI16>;1786defm MVE_VORRimmi32 : MVE_VORRimm<MVE_v4i32, nImmSplatI32>;1787defm MVE_VBICimmi16 : MVE_VBICimm<MVE_v8i16, nImmSplatI16>;1788defm MVE_VBICimmi32 : MVE_VBICimm<MVE_v4i32, nImmSplatI32>;1789 1790def MVE_VORNimmi16 : MVEInstAlias<"vorn${vp}.i16\t$Qd, $imm",1791 (MVE_VORRimmi16 MQPR:$Qd, nImmSplatNotI16:$imm, vpred_n:$vp), 0>;1792def MVE_VORNimmi32 : MVEInstAlias<"vorn${vp}.i32\t$Qd, $imm",1793 (MVE_VORRimmi32 MQPR:$Qd, nImmSplatNotI32:$imm, vpred_n:$vp), 0>;1794 1795def MVE_VANDimmi16 : MVEInstAlias<"vand${vp}.i16\t$Qd, $imm",1796 (MVE_VBICimmi16 MQPR:$Qd, nImmSplatNotI16:$imm, vpred_n:$vp), 0>;1797def MVE_VANDimmi32 : MVEInstAlias<"vand${vp}.i32\t$Qd, $imm",1798 (MVE_VBICimmi32 MQPR:$Qd, nImmSplatNotI32:$imm, vpred_n:$vp), 0>;1799 1800def MVE_VMOV : MVEInstAlias<"vmov${vp}\t$Qd, $Qm",1801 (MVE_VORR MQPR:$Qd, MQPR:$Qm, MQPR:$Qm, vpred_r:$vp)>;1802 1803class MVE_VMOV_lane_direction {1804 bit bit_20;1805 dag oops;1806 dag iops;1807 string ops;1808 string cstr;1809}1810def MVE_VMOV_from_lane : MVE_VMOV_lane_direction {1811 let bit_20 = 0b1;1812 let oops = (outs rGPR:$Rt);1813 let iops = (ins MQPR:$Qd);1814 let ops = "$Rt, $Qd$Idx";1815 let cstr = "";1816}1817def MVE_VMOV_to_lane : MVE_VMOV_lane_direction {1818 let bit_20 = 0b0;1819 let oops = (outs MQPR:$Qd);1820 let iops = (ins MQPR:$Qd_src, rGPR:$Rt);1821 let ops = "$Qd$Idx, $Rt";1822 let cstr = "$Qd = $Qd_src";1823}1824 1825class MVE_VMOV_lane<string suffix, bit U, dag indexop,1826 MVE_VMOV_lane_direction dir>1827 : MVE_VMOV_lane_base<dir.oops, !con(dir.iops, indexop), NoItinerary,1828 "vmov", suffix, dir.ops, dir.cstr, []> {1829 bits<4> Qd;1830 bits<4> Rt;1831 1832 let Inst{31-24} = 0b11101110;1833 let Inst{23} = U;1834 let Inst{20} = dir.bit_20;1835 let Inst{19-17} = Qd{2-0};1836 let Inst{15-12} = Rt{3-0};1837 let Inst{11-8} = 0b1011;1838 let Inst{7} = Qd{3};1839 let Inst{4-0} = 0b10000;1840 1841 let hasSideEffects = 0;1842}1843 1844class MVE_VMOV_lane_32<MVE_VMOV_lane_direction dir>1845 : MVE_VMOV_lane<"32", 0b0, (ins MVEVectorIndex<4>:$Idx), dir> {1846 bits<2> Idx;1847 let Inst{22} = 0b0;1848 let Inst{6-5} = 0b00;1849 let Inst{16} = Idx{1};1850 let Inst{21} = Idx{0};1851 1852 let VecSize = 0b10;1853 let Predicates = [HasFPRegsV8_1M];1854}1855 1856class MVE_VMOV_lane_16<string suffix, bit U, MVE_VMOV_lane_direction dir>1857 : MVE_VMOV_lane<suffix, U, (ins MVEVectorIndex<8>:$Idx), dir> {1858 bits<3> Idx;1859 let Inst{22} = 0b0;1860 let Inst{5} = 0b1;1861 let Inst{16} = Idx{2};1862 let Inst{21} = Idx{1};1863 let Inst{6} = Idx{0};1864 1865 let VecSize = 0b01;1866}1867 1868class MVE_VMOV_lane_8<string suffix, bit U, MVE_VMOV_lane_direction dir>1869 : MVE_VMOV_lane<suffix, U, (ins MVEVectorIndex<16>:$Idx), dir> {1870 bits<4> Idx;1871 let Inst{22} = 0b1;1872 let Inst{16} = Idx{3};1873 let Inst{21} = Idx{2};1874 let Inst{6} = Idx{1};1875 let Inst{5} = Idx{0};1876 1877 let VecSize = 0b00;1878}1879 1880def MVE_VMOV_from_lane_32 : MVE_VMOV_lane_32< MVE_VMOV_from_lane>;1881def MVE_VMOV_from_lane_s16 : MVE_VMOV_lane_16<"s16", 0b0, MVE_VMOV_from_lane>;1882def MVE_VMOV_from_lane_u16 : MVE_VMOV_lane_16<"u16", 0b1, MVE_VMOV_from_lane>;1883def MVE_VMOV_from_lane_s8 : MVE_VMOV_lane_8 < "s8", 0b0, MVE_VMOV_from_lane>;1884def MVE_VMOV_from_lane_u8 : MVE_VMOV_lane_8 < "u8", 0b1, MVE_VMOV_from_lane>;1885let isInsertSubreg = 1 in1886def MVE_VMOV_to_lane_32 : MVE_VMOV_lane_32< MVE_VMOV_to_lane>;1887def MVE_VMOV_to_lane_16 : MVE_VMOV_lane_16< "16", 0b0, MVE_VMOV_to_lane>;1888def MVE_VMOV_to_lane_8 : MVE_VMOV_lane_8 < "8", 0b0, MVE_VMOV_to_lane>;1889 1890// This is the same as insertelt but allows the inserted value to be an i32 as1891// will be used when it is the only legal type.1892def ARMVecInsert : SDTypeProfile<1, 3, [1893 SDTCisVT<2, i32>, SDTCisSameAs<0, 1>, SDTCisPtrTy<3>1894]>;1895def ARMinsertelt : SDNode<"ISD::INSERT_VECTOR_ELT", ARMVecInsert>;1896 1897let Predicates = [HasMVEInt] in {1898 def : Pat<(extractelt (v2f64 MQPR:$src), imm:$lane),1899 (f64 (EXTRACT_SUBREG MQPR:$src, (DSubReg_f64_reg imm:$lane)))>;1900 def : Pat<(insertelt (v2f64 MQPR:$src1), DPR:$src2, imm:$lane),1901 (INSERT_SUBREG (v2f64 (COPY_TO_REGCLASS MQPR:$src1, MQPR)), DPR:$src2, (DSubReg_f64_reg imm:$lane))>;1902 1903 def : Pat<(extractelt (v4i32 MQPR:$src), imm:$lane),1904 (COPY_TO_REGCLASS1905 (i32 (EXTRACT_SUBREG MQPR:$src, (SSubReg_f32_reg imm:$lane))), rGPR)>;1906 def : Pat<(insertelt (v4i32 MQPR:$src1), rGPR:$src2, imm:$lane),1907 (MVE_VMOV_to_lane_32 MQPR:$src1, rGPR:$src2, imm:$lane)>;1908 // This tries to copy from one lane to another, without going via GPR regs1909 def : Pat<(insertelt (v4i32 MQPR:$src1), (extractelt (v4i32 MQPR:$src2), imm:$extlane), imm:$inslane),1910 (v4i32 (COPY_TO_REGCLASS1911 (INSERT_SUBREG (v4f32 (COPY_TO_REGCLASS (v4i32 MQPR:$src1), MQPR)),1912 (f32 (EXTRACT_SUBREG (v4f32 (COPY_TO_REGCLASS (v4i32 MQPR:$src2), MQPR)),1913 (SSubReg_f32_reg imm:$extlane))),1914 (SSubReg_f32_reg imm:$inslane)),1915 MQPR))>;1916 1917 def : Pat<(vector_insert (v16i8 MQPR:$src1), rGPR:$src2, imm:$lane),1918 (MVE_VMOV_to_lane_8 MQPR:$src1, rGPR:$src2, imm:$lane)>;1919 def : Pat<(vector_insert (v8i16 MQPR:$src1), rGPR:$src2, imm:$lane),1920 (MVE_VMOV_to_lane_16 MQPR:$src1, rGPR:$src2, imm:$lane)>;1921 1922 def : Pat<(ARMvgetlanes (v16i8 MQPR:$src), imm:$lane),1923 (MVE_VMOV_from_lane_s8 MQPR:$src, imm:$lane)>;1924 def : Pat<(ARMvgetlanes (v8i16 MQPR:$src), imm:$lane),1925 (MVE_VMOV_from_lane_s16 MQPR:$src, imm:$lane)>;1926 def : Pat<(ARMvgetlanes (v8f16 MQPR:$src), imm:$lane),1927 (MVE_VMOV_from_lane_s16 MQPR:$src, imm:$lane)>;1928 def : Pat<(ARMvgetlaneu (v16i8 MQPR:$src), imm:$lane),1929 (MVE_VMOV_from_lane_u8 MQPR:$src, imm:$lane)>;1930 def : Pat<(ARMvgetlaneu (v8i16 MQPR:$src), imm:$lane),1931 (MVE_VMOV_from_lane_u16 MQPR:$src, imm:$lane)>;1932 def : Pat<(ARMvgetlaneu (v8f16 MQPR:$src), imm:$lane),1933 (MVE_VMOV_from_lane_u16 MQPR:$src, imm:$lane)>;1934 // For i16's inserts being extracted from low lanes, then may use VINS.1935 let Predicates = [HasFullFP16] in {1936 def : Pat<(ARMinsertelt (v8i16 MQPR:$src1),1937 (ARMvgetlaneu (v8i16 MQPR:$src2), imm_even:$extlane),1938 imm_odd:$inslane),1939 (COPY_TO_REGCLASS (INSERT_SUBREG (v4f32 (COPY_TO_REGCLASS MQPR:$src1, MQPR)),1940 (VINSH (EXTRACT_SUBREG MQPR:$src1, (SSubReg_f16_reg imm_odd:$inslane)),1941 (EXTRACT_SUBREG MQPR:$src2, (SSubReg_f16_reg imm_even:$extlane))),1942 (SSubReg_f16_reg imm_odd:$inslane)), MQPR)>;1943 }1944 1945 def : Pat<(v16i8 (scalar_to_vector GPR:$src)),1946 (MVE_VMOV_to_lane_8 (v16i8 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;1947 def : Pat<(v8i16 (scalar_to_vector GPR:$src)),1948 (MVE_VMOV_to_lane_16 (v8i16 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;1949 def : Pat<(v4i32 (scalar_to_vector GPR:$src)),1950 (MVE_VMOV_to_lane_32 (v4i32 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;1951 1952 // Floating point patterns, still enabled under HasMVEInt1953 def : Pat<(extractelt (v4f32 MQPR:$src), imm:$lane),1954 (COPY_TO_REGCLASS (f32 (EXTRACT_SUBREG MQPR:$src, (SSubReg_f32_reg imm:$lane))), SPR)>;1955 def : Pat<(insertelt (v4f32 MQPR:$src1), (f32 SPR:$src2), imm:$lane),1956 (INSERT_SUBREG (v4f32 (COPY_TO_REGCLASS MQPR:$src1, MQPR)), SPR:$src2, (SSubReg_f32_reg imm:$lane))>;1957 1958 def : Pat<(insertelt (v8f16 MQPR:$src1), (f16 HPR:$src2), imm_even:$lane),1959 (MVE_VMOV_to_lane_16 MQPR:$src1, (COPY_TO_REGCLASS (f16 HPR:$src2), rGPR), imm:$lane)>;1960 let Predicates = [HasFullFP16] in {1961 def : Pat<(insertelt (v8f16 MQPR:$src1), (f16 HPR:$src2), imm_odd:$lane),1962 (COPY_TO_REGCLASS (INSERT_SUBREG (v4f32 (COPY_TO_REGCLASS MQPR:$src1, MQPR)),1963 (VINSH (EXTRACT_SUBREG MQPR:$src1, (SSubReg_f16_reg imm_odd:$lane)),1964 (COPY_TO_REGCLASS HPR:$src2, SPR)),1965 (SSubReg_f16_reg imm_odd:$lane)), MQPR)>;1966 }1967 def : Pat<(extractelt (v8f16 MQPR:$src), imm_even:$lane),1968 (EXTRACT_SUBREG MQPR:$src, (SSubReg_f16_reg imm_even:$lane))>;1969 let Predicates = [HasFullFP16] in {1970 def : Pat<(extractelt (v8f16 MQPR:$src), imm_odd:$lane),1971 (COPY_TO_REGCLASS1972 (VMOVH (EXTRACT_SUBREG MQPR:$src, (SSubReg_f16_reg imm_odd:$lane))),1973 HPR)>;1974 }1975 1976 def : Pat<(v2f64 (scalar_to_vector (f64 DPR:$src))),1977 (INSERT_SUBREG (v2f64 (IMPLICIT_DEF)), DPR:$src, dsub_0)>;1978 def : Pat<(v4f32 (scalar_to_vector SPR:$src)),1979 (INSERT_SUBREG (v4f32 (IMPLICIT_DEF)), SPR:$src, ssub_0)>;1980 def : Pat<(v4f32 (scalar_to_vector GPR:$src)),1981 (MVE_VMOV_to_lane_32 (v4f32 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;1982 def : Pat<(v8f16 (scalar_to_vector (f16 HPR:$src))),1983 (INSERT_SUBREG (v8f16 (IMPLICIT_DEF)), (f16 HPR:$src), ssub_0)>;1984 def : Pat<(v8f16 (scalar_to_vector GPR:$src)),1985 (MVE_VMOV_to_lane_16 (v8f16 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;1986}1987 1988// end of mve_bit instructions1989 1990// start of MVE Integer instructions1991 1992class MVE_int<string iname, string suffix, bits<2> size, list<dag> pattern=[]>1993 : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm), NoItinerary,1994 iname, suffix, "$Qd, $Qn, $Qm", vpred_r, "", size, pattern> {1995 bits<4> Qd;1996 bits<4> Qn;1997 bits<4> Qm;1998 1999 let Inst{22} = Qd{3};2000 let Inst{21-20} = size;2001 let Inst{19-17} = Qn{2-0};2002 let Inst{15-13} = Qd{2-0};2003 let Inst{7} = Qn{3};2004 let Inst{6} = 0b1;2005 let Inst{5} = Qm{3};2006 let Inst{3-1} = Qm{2-0};2007}2008 2009class MVE_VMULt1<string iname, string suffix, bits<2> size,2010 list<dag> pattern=[]>2011 : MVE_int<iname, suffix, size, pattern> {2012 2013 let Inst{28} = 0b0;2014 let Inst{25-23} = 0b110;2015 let Inst{16} = 0b0;2016 let Inst{12-8} = 0b01001;2017 let Inst{4} = 0b1;2018 let Inst{0} = 0b0;2019 let validForTailPredication = 1;2020}2021 2022multiclass MVE_VMUL_m<MVEVectorVTInfo VTI> {2023 def "" : MVE_VMULt1<"vmul", VTI.Suffix, VTI.Size>;2024 2025 let Predicates = [HasMVEInt] in {2026 defm : MVE_TwoOpPattern<VTI, mul, int_arm_mve_mul_predicated, (? ),2027 !cast<Instruction>(NAME), ARMimmOneV>;2028 }2029}2030 2031defm MVE_VMULi8 : MVE_VMUL_m<MVE_v16i8>;2032defm MVE_VMULi16 : MVE_VMUL_m<MVE_v8i16>;2033defm MVE_VMULi32 : MVE_VMUL_m<MVE_v4i32>;2034 2035class MVE_VQxDMULH_Base<string iname, string suffix, bits<2> size, bit rounding,2036 list<dag> pattern=[]>2037 : MVE_int<iname, suffix, size, pattern> {2038 2039 let Inst{28} = rounding;2040 let Inst{25-23} = 0b110;2041 let Inst{16} = 0b0;2042 let Inst{12-8} = 0b01011;2043 let Inst{4} = 0b0;2044 let Inst{0} = 0b0;2045 let validForTailPredication = 1;2046}2047 2048// MVE vqdmulh instruction2049def MVEvqdmulh : SDNode<"ARMISD::VQDMULH", SDTIntBinOp>;2050 2051multiclass MVE_VQxDMULH_m<string iname, MVEVectorVTInfo VTI,2052 SDPatternOperator Op, Intrinsic unpred_int, Intrinsic pred_int,2053 bit rounding> {2054 def "" : MVE_VQxDMULH_Base<iname, VTI.Suffix, VTI.Size, rounding>;2055 defvar Inst = !cast<Instruction>(NAME);2056 2057 let Predicates = [HasMVEInt] in {2058 defm : MVE_TwoOpPattern<VTI, Op, pred_int, (? ), Inst>;2059 2060 // Extra unpredicated multiply intrinsic patterns2061 def : Pat<(VTI.Vec (unpred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),2062 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;2063 }2064}2065 2066multiclass MVE_VQxDMULH<string iname, MVEVectorVTInfo VTI, bit rounding>2067 : MVE_VQxDMULH_m<iname, VTI, !if(rounding, null_frag,2068 MVEvqdmulh),2069 !if(rounding, int_arm_mve_vqrdmulh,2070 int_arm_mve_vqdmulh),2071 !if(rounding, int_arm_mve_qrdmulh_predicated,2072 int_arm_mve_qdmulh_predicated),2073 rounding>;2074 2075defm MVE_VQDMULHi8 : MVE_VQxDMULH<"vqdmulh", MVE_v16s8, 0b0>;2076defm MVE_VQDMULHi16 : MVE_VQxDMULH<"vqdmulh", MVE_v8s16, 0b0>;2077defm MVE_VQDMULHi32 : MVE_VQxDMULH<"vqdmulh", MVE_v4s32, 0b0>;2078 2079defm MVE_VQRDMULHi8 : MVE_VQxDMULH<"vqrdmulh", MVE_v16s8, 0b1>;2080defm MVE_VQRDMULHi16 : MVE_VQxDMULH<"vqrdmulh", MVE_v8s16, 0b1>;2081defm MVE_VQRDMULHi32 : MVE_VQxDMULH<"vqrdmulh", MVE_v4s32, 0b1>;2082 2083class MVE_VADDSUB<string iname, string suffix, bits<2> size, bit subtract,2084 list<dag> pattern=[]>2085 : MVE_int<iname, suffix, size, pattern> {2086 2087 let Inst{28} = subtract;2088 let Inst{25-23} = 0b110;2089 let Inst{16} = 0b0;2090 let Inst{12-8} = 0b01000;2091 let Inst{4} = 0b0;2092 let Inst{0} = 0b0;2093 let validForTailPredication = 1;2094}2095 2096multiclass MVE_VADDSUB_m<string iname, MVEVectorVTInfo VTI, bit subtract,2097 SDNode Op, Intrinsic PredInt> {2098 def "" : MVE_VADDSUB<iname, VTI.Suffix, VTI.Size, subtract>;2099 defvar Inst = !cast<Instruction>(NAME);2100 2101 let Predicates = [HasMVEInt] in {2102 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? ), !cast<Instruction>(NAME), ARMimmAllZerosV>;2103 }2104}2105 2106multiclass MVE_VADD<MVEVectorVTInfo VTI>2107 : MVE_VADDSUB_m<"vadd", VTI, 0b0, add, int_arm_mve_add_predicated>;2108multiclass MVE_VSUB<MVEVectorVTInfo VTI>2109 : MVE_VADDSUB_m<"vsub", VTI, 0b1, sub, int_arm_mve_sub_predicated>;2110 2111defm MVE_VADDi8 : MVE_VADD<MVE_v16i8>;2112defm MVE_VADDi16 : MVE_VADD<MVE_v8i16>;2113defm MVE_VADDi32 : MVE_VADD<MVE_v4i32>;2114 2115defm MVE_VSUBi8 : MVE_VSUB<MVE_v16i8>;2116defm MVE_VSUBi16 : MVE_VSUB<MVE_v8i16>;2117defm MVE_VSUBi32 : MVE_VSUB<MVE_v4i32>;2118 2119class MVE_VQADDSUB<string iname, string suffix, bit U, bit subtract,2120 bits<2> size>2121 : MVE_int<iname, suffix, size, []> {2122 2123 let Inst{28} = U;2124 let Inst{25-23} = 0b110;2125 let Inst{16} = 0b0;2126 let Inst{12-10} = 0b000;2127 let Inst{9} = subtract;2128 let Inst{8} = 0b0;2129 let Inst{4} = 0b1;2130 let Inst{0} = 0b0;2131 let validForTailPredication = 1;2132}2133 2134class MVE_VQADD_<string suffix, bit U, bits<2> size>2135 : MVE_VQADDSUB<"vqadd", suffix, U, 0b0, size>;2136class MVE_VQSUB_<string suffix, bit U, bits<2> size>2137 : MVE_VQADDSUB<"vqsub", suffix, U, 0b1, size>;2138 2139multiclass MVE_VQADD_m<MVEVectorVTInfo VTI,2140 SDNode Op, Intrinsic PredInt> {2141 def "" : MVE_VQADD_<VTI.Suffix, VTI.Unsigned, VTI.Size>;2142 defvar Inst = !cast<Instruction>(NAME);2143 2144 let Predicates = [HasMVEInt] in {2145 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)),2146 !cast<Instruction>(NAME)>;2147 }2148}2149 2150multiclass MVE_VQADD<MVEVectorVTInfo VTI, SDNode unpred_op>2151 : MVE_VQADD_m<VTI, unpred_op, int_arm_mve_qadd_predicated>;2152 2153defm MVE_VQADDs8 : MVE_VQADD<MVE_v16s8, saddsat>;2154defm MVE_VQADDs16 : MVE_VQADD<MVE_v8s16, saddsat>;2155defm MVE_VQADDs32 : MVE_VQADD<MVE_v4s32, saddsat>;2156defm MVE_VQADDu8 : MVE_VQADD<MVE_v16u8, uaddsat>;2157defm MVE_VQADDu16 : MVE_VQADD<MVE_v8u16, uaddsat>;2158defm MVE_VQADDu32 : MVE_VQADD<MVE_v4u32, uaddsat>;2159 2160multiclass MVE_VQSUB_m<MVEVectorVTInfo VTI,2161 SDNode Op, Intrinsic PredInt> {2162 def "" : MVE_VQSUB_<VTI.Suffix, VTI.Unsigned, VTI.Size>;2163 defvar Inst = !cast<Instruction>(NAME);2164 2165 let Predicates = [HasMVEInt] in {2166 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)),2167 !cast<Instruction>(NAME)>;2168 }2169}2170 2171multiclass MVE_VQSUB<MVEVectorVTInfo VTI, SDNode unpred_op>2172 : MVE_VQSUB_m<VTI, unpred_op, int_arm_mve_qsub_predicated>;2173 2174defm MVE_VQSUBs8 : MVE_VQSUB<MVE_v16s8, ssubsat>;2175defm MVE_VQSUBs16 : MVE_VQSUB<MVE_v8s16, ssubsat>;2176defm MVE_VQSUBs32 : MVE_VQSUB<MVE_v4s32, ssubsat>;2177defm MVE_VQSUBu8 : MVE_VQSUB<MVE_v16u8, usubsat>;2178defm MVE_VQSUBu16 : MVE_VQSUB<MVE_v8u16, usubsat>;2179defm MVE_VQSUBu32 : MVE_VQSUB<MVE_v4u32, usubsat>;2180 2181class MVE_VABD_int<string suffix, bit U, bits<2> size,2182 list<dag> pattern=[]>2183 : MVE_int<"vabd", suffix, size, pattern> {2184 2185 let Inst{28} = U;2186 let Inst{25-23} = 0b110;2187 let Inst{16} = 0b0;2188 let Inst{12-8} = 0b00111;2189 let Inst{4} = 0b0;2190 let Inst{0} = 0b0;2191 let validForTailPredication = 1;2192}2193 2194multiclass MVE_VABD_m<MVEVectorVTInfo VTI, SDNode Op,2195 Intrinsic unpred_int, Intrinsic PredInt> {2196 def "" : MVE_VABD_int<VTI.Suffix, VTI.Unsigned, VTI.Size>;2197 defvar Inst = !cast<Instruction>(NAME);2198 2199 let Predicates = [HasMVEInt] in {2200 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)),2201 !cast<Instruction>(NAME)>;2202 2203 // Unpredicated absolute difference2204 def : Pat<(VTI.Vec (unpred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),2205 (i32 VTI.Unsigned))),2206 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;2207 }2208}2209 2210multiclass MVE_VABD<MVEVectorVTInfo VTI, SDNode Op>2211 : MVE_VABD_m<VTI, Op, int_arm_mve_vabd, int_arm_mve_abd_predicated>;2212 2213defm MVE_VABDs8 : MVE_VABD<MVE_v16s8, abds>;2214defm MVE_VABDs16 : MVE_VABD<MVE_v8s16, abds>;2215defm MVE_VABDs32 : MVE_VABD<MVE_v4s32, abds>;2216defm MVE_VABDu8 : MVE_VABD<MVE_v16u8, abdu>;2217defm MVE_VABDu16 : MVE_VABD<MVE_v8u16, abdu>;2218defm MVE_VABDu32 : MVE_VABD<MVE_v4u32, abdu>;2219 2220class MVE_VRHADD_Base<string suffix, bit U, bits<2> size, list<dag> pattern=[]>2221 : MVE_int<"vrhadd", suffix, size, pattern> {2222 2223 let Inst{28} = U;2224 let Inst{25-23} = 0b110;2225 let Inst{16} = 0b0;2226 let Inst{12-8} = 0b00001;2227 let Inst{4} = 0b0;2228 let Inst{0} = 0b0;2229 let validForTailPredication = 1;2230}2231 2232def addnuw : PatFrag<(ops node:$lhs, node:$rhs),2233 (add node:$lhs, node:$rhs), [{2234 return N->getFlags().hasNoUnsignedWrap();2235}]>;2236 2237def addnsw : PatFrag<(ops node:$lhs, node:$rhs),2238 (add node:$lhs, node:$rhs), [{2239 return N->getFlags().hasNoSignedWrap();2240}]>;2241 2242def subnuw : PatFrag<(ops node:$lhs, node:$rhs),2243 (sub node:$lhs, node:$rhs), [{2244 return N->getFlags().hasNoUnsignedWrap();2245}]>;2246 2247def subnsw : PatFrag<(ops node:$lhs, node:$rhs),2248 (sub node:$lhs, node:$rhs), [{2249 return N->getFlags().hasNoSignedWrap();2250}]>;2251 2252multiclass MVE_VRHADD_m<MVEVectorVTInfo VTI, SDNode Op,2253 SDPatternOperator unpred_op, Intrinsic PredInt> {2254 def "" : MVE_VRHADD_Base<VTI.Suffix, VTI.Unsigned, VTI.Size>;2255 defvar Inst = !cast<Instruction>(NAME);2256 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)), !cast<Instruction>(NAME)>;2257 2258 let Predicates = [HasMVEInt] in {2259 // Unpredicated rounding add-with-divide-by-two intrinsic2260 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),2261 (i32 VTI.Unsigned))),2262 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;2263 }2264}2265 2266multiclass MVE_VRHADD<MVEVectorVTInfo VTI, SDNode rhadd>2267 : MVE_VRHADD_m<VTI, rhadd, int_arm_mve_vrhadd, int_arm_mve_rhadd_predicated>;2268 2269defm MVE_VRHADDs8 : MVE_VRHADD<MVE_v16s8, avgceils>;2270defm MVE_VRHADDs16 : MVE_VRHADD<MVE_v8s16, avgceils>;2271defm MVE_VRHADDs32 : MVE_VRHADD<MVE_v4s32, avgceils>;2272defm MVE_VRHADDu8 : MVE_VRHADD<MVE_v16u8, avgceilu>;2273defm MVE_VRHADDu16 : MVE_VRHADD<MVE_v8u16, avgceilu>;2274defm MVE_VRHADDu32 : MVE_VRHADD<MVE_v4u32, avgceilu>;2275 2276class MVE_VHADDSUB<string iname, string suffix, bit U, bit subtract,2277 bits<2> size, list<dag> pattern=[]>2278 : MVE_int<iname, suffix, size, pattern> {2279 2280 let Inst{28} = U;2281 let Inst{25-23} = 0b110;2282 let Inst{16} = 0b0;2283 let Inst{12-10} = 0b000;2284 let Inst{9} = subtract;2285 let Inst{8} = 0b0;2286 let Inst{4} = 0b0;2287 let Inst{0} = 0b0;2288 let validForTailPredication = 1;2289}2290 2291class MVE_VHADD_<string suffix, bit U, bits<2> size,2292 list<dag> pattern=[]>2293 : MVE_VHADDSUB<"vhadd", suffix, U, 0b0, size, pattern>;2294class MVE_VHSUB_<string suffix, bit U, bits<2> size,2295 list<dag> pattern=[]>2296 : MVE_VHADDSUB<"vhsub", suffix, U, 0b1, size, pattern>;2297 2298multiclass MVE_VHADD_m<MVEVectorVTInfo VTI, SDNode Op,2299 SDPatternOperator unpred_op, Intrinsic PredInt> {2300 def "" : MVE_VHADD_<VTI.Suffix, VTI.Unsigned, VTI.Size>;2301 defvar Inst = !cast<Instruction>(NAME);2302 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)), !cast<Instruction>(NAME)>;2303 2304 let Predicates = [HasMVEInt] in {2305 // Unpredicated add-and-divide-by-two2306 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn), (i32 VTI.Unsigned))),2307 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;2308 }2309}2310 2311multiclass MVE_VHADD<MVEVectorVTInfo VTI, SDNode Op>2312 : MVE_VHADD_m<VTI, Op, int_arm_mve_vhadd, int_arm_mve_hadd_predicated>;2313 2314defm MVE_VHADDs8 : MVE_VHADD<MVE_v16s8, avgfloors>;2315defm MVE_VHADDs16 : MVE_VHADD<MVE_v8s16, avgfloors>;2316defm MVE_VHADDs32 : MVE_VHADD<MVE_v4s32, avgfloors>;2317defm MVE_VHADDu8 : MVE_VHADD<MVE_v16u8, avgflooru>;2318defm MVE_VHADDu16 : MVE_VHADD<MVE_v8u16, avgflooru>;2319defm MVE_VHADDu32 : MVE_VHADD<MVE_v4u32, avgflooru>;2320 2321multiclass MVE_VHSUB_m<MVEVectorVTInfo VTI,2322 SDPatternOperator unpred_op, Intrinsic pred_int, PatFrag sub_op,2323 SDNode shift_op> {2324 def "" : MVE_VHSUB_<VTI.Suffix, VTI.Unsigned, VTI.Size>;2325 defvar Inst = !cast<Instruction>(NAME);2326 2327 let Predicates = [HasMVEInt] in {2328 // Unpredicated subtract-and-divide-by-two2329 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),2330 (i32 VTI.Unsigned))),2331 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;2332 2333 def : Pat<(VTI.Vec (shift_op (sub_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)), (i32 1))),2334 (Inst MQPR:$Qm, MQPR:$Qn)>;2335 2336 2337 // Predicated subtract-and-divide-by-two2338 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),2339 (i32 VTI.Unsigned), (VTI.Pred VCCR:$mask),2340 (VTI.Vec MQPR:$inactive))),2341 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),2342 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,2343 (VTI.Vec MQPR:$inactive)))>;2344 }2345}2346 2347multiclass MVE_VHSUB<MVEVectorVTInfo VTI, PatFrag sub_op, SDNode shift_op>2348 : MVE_VHSUB_m<VTI, int_arm_mve_vhsub, int_arm_mve_hsub_predicated, sub_op,2349 shift_op>;2350 2351defm MVE_VHSUBs8 : MVE_VHSUB<MVE_v16s8, subnsw, ARMvshrsImm>;2352defm MVE_VHSUBs16 : MVE_VHSUB<MVE_v8s16, subnsw, ARMvshrsImm>;2353defm MVE_VHSUBs32 : MVE_VHSUB<MVE_v4s32, subnsw, ARMvshrsImm>;2354defm MVE_VHSUBu8 : MVE_VHSUB<MVE_v16u8, subnuw, ARMvshruImm>;2355defm MVE_VHSUBu16 : MVE_VHSUB<MVE_v8u16, subnuw, ARMvshruImm>;2356defm MVE_VHSUBu32 : MVE_VHSUB<MVE_v4u32, subnuw, ARMvshruImm>;2357 2358class MVE_VDUP<string suffix, bit B, bit E, bits<2> vecsize, list<dag> pattern=[]>2359 : MVE_p<(outs MQPR:$Qd), (ins rGPR:$Rt), NoItinerary,2360 "vdup", suffix, "$Qd, $Rt", vpred_r, "", vecsize, pattern> {2361 bits<4> Qd;2362 bits<4> Rt;2363 2364 let Inst{28} = 0b0;2365 let Inst{25-23} = 0b101;2366 let Inst{22} = B;2367 let Inst{21-20} = 0b10;2368 let Inst{19-17} = Qd{2-0};2369 let Inst{16} = 0b0;2370 let Inst{15-12} = Rt;2371 let Inst{11-8} = 0b1011;2372 let Inst{7} = Qd{3};2373 let Inst{6} = 0b0;2374 let Inst{5} = E;2375 let Inst{4-0} = 0b10000;2376 let validForTailPredication = 1;2377}2378 2379def MVE_VDUP32 : MVE_VDUP<"32", 0b0, 0b0, 0b10>;2380def MVE_VDUP16 : MVE_VDUP<"16", 0b0, 0b1, 0b01>;2381def MVE_VDUP8 : MVE_VDUP<"8", 0b1, 0b0, 0b00>;2382 2383let Predicates = [HasMVEInt] in {2384 def : Pat<(v16i8 (ARMvdup (i32 rGPR:$elem))),2385 (MVE_VDUP8 rGPR:$elem)>;2386 def : Pat<(v8i16 (ARMvdup (i32 rGPR:$elem))),2387 (MVE_VDUP16 rGPR:$elem)>;2388 def : Pat<(v4i32 (ARMvdup (i32 rGPR:$elem))),2389 (MVE_VDUP32 rGPR:$elem)>;2390 2391 def : Pat<(v8f16 (ARMvdup (i32 rGPR:$elem))),2392 (MVE_VDUP16 rGPR:$elem)>;2393 def : Pat<(v4f32 (ARMvdup (i32 rGPR:$elem))),2394 (MVE_VDUP32 rGPR:$elem)>;2395 2396 // Match a vselect with an ARMvdup as a predicated MVE_VDUP2397 def : Pat<(v16i8 (vselect (v16i1 VCCR:$pred),2398 (v16i8 (ARMvdup (i32 rGPR:$elem))),2399 (v16i8 MQPR:$inactive))),2400 (MVE_VDUP8 rGPR:$elem, ARMVCCThen, (v16i1 VCCR:$pred), zero_reg,2401 (v16i8 MQPR:$inactive))>;2402 def : Pat<(v8i16 (vselect (v8i1 VCCR:$pred),2403 (v8i16 (ARMvdup (i32 rGPR:$elem))),2404 (v8i16 MQPR:$inactive))),2405 (MVE_VDUP16 rGPR:$elem, ARMVCCThen, (v8i1 VCCR:$pred), zero_reg,2406 (v8i16 MQPR:$inactive))>;2407 def : Pat<(v4i32 (vselect (v4i1 VCCR:$pred),2408 (v4i32 (ARMvdup (i32 rGPR:$elem))),2409 (v4i32 MQPR:$inactive))),2410 (MVE_VDUP32 rGPR:$elem, ARMVCCThen, (v4i1 VCCR:$pred), zero_reg,2411 (v4i32 MQPR:$inactive))>;2412 def : Pat<(v4f32 (vselect (v4i1 VCCR:$pred),2413 (v4f32 (ARMvdup (i32 rGPR:$elem))),2414 (v4f32 MQPR:$inactive))),2415 (MVE_VDUP32 rGPR:$elem, ARMVCCThen, (v4i1 VCCR:$pred), zero_reg,2416 (v4f32 MQPR:$inactive))>;2417 def : Pat<(v8f16 (vselect (v8i1 VCCR:$pred),2418 (v8f16 (ARMvdup (i32 rGPR:$elem))),2419 (v8f16 MQPR:$inactive))),2420 (MVE_VDUP16 rGPR:$elem, ARMVCCThen, (v8i1 VCCR:$pred), zero_reg,2421 (v8f16 MQPR:$inactive))>;2422}2423 2424 2425class MVEIntSingleSrc<string iname, string suffix, bits<2> size,2426 list<dag> pattern=[]>2427 : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qm), NoItinerary,2428 iname, suffix, "$Qd, $Qm", vpred_r, "", size, pattern> {2429 bits<4> Qd;2430 bits<4> Qm;2431 2432 let Inst{22} = Qd{3};2433 let Inst{19-18} = size{1-0};2434 let Inst{15-13} = Qd{2-0};2435 let Inst{5} = Qm{3};2436 let Inst{3-1} = Qm{2-0};2437}2438 2439class MVE_VCLSCLZ<string iname, string suffix, bits<2> size,2440 bit count_zeroes, list<dag> pattern=[]>2441 : MVEIntSingleSrc<iname, suffix, size, pattern> {2442 2443 let Inst{28} = 0b1;2444 let Inst{25-23} = 0b111;2445 let Inst{21-20} = 0b11;2446 let Inst{17-16} = 0b00;2447 let Inst{12-8} = 0b00100;2448 let Inst{7} = count_zeroes;2449 let Inst{6} = 0b1;2450 let Inst{4} = 0b0;2451 let Inst{0} = 0b0;2452 let validForTailPredication = 1;2453}2454 2455multiclass MVE_VCLSCLZ_p<string opname, bit opcode, MVEVectorVTInfo VTI,2456 SDPatternOperator unpred_op> {2457 def "": MVE_VCLSCLZ<"v"#opname, VTI.Suffix, VTI.Size, opcode>;2458 2459 defvar Inst = !cast<Instruction>(NAME);2460 defvar pred_int = !cast<Intrinsic>("int_arm_mve_"#opname#"_predicated");2461 2462 let Predicates = [HasMVEInt] in {2463 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$val))),2464 (VTI.Vec (Inst (VTI.Vec MQPR:$val)))>;2465 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$val), (VTI.Pred VCCR:$pred),2466 (VTI.Vec MQPR:$inactive))),2467 (VTI.Vec (Inst (VTI.Vec MQPR:$val), ARMVCCThen,2468 (VTI.Pred VCCR:$pred), zero_reg, (VTI.Vec MQPR:$inactive)))>;2469 }2470}2471 2472defm MVE_VCLSs8 : MVE_VCLSCLZ_p<"cls", 0, MVE_v16s8, int_arm_mve_vcls>;2473defm MVE_VCLSs16 : MVE_VCLSCLZ_p<"cls", 0, MVE_v8s16, int_arm_mve_vcls>;2474defm MVE_VCLSs32 : MVE_VCLSCLZ_p<"cls", 0, MVE_v4s32, int_arm_mve_vcls>;2475 2476defm MVE_VCLZs8 : MVE_VCLSCLZ_p<"clz", 1, MVE_v16i8, ctlz>;2477defm MVE_VCLZs16 : MVE_VCLSCLZ_p<"clz", 1, MVE_v8i16, ctlz>;2478defm MVE_VCLZs32 : MVE_VCLSCLZ_p<"clz", 1, MVE_v4i32, ctlz>;2479 2480class MVE_VABSNEG_int<string iname, string suffix, bits<2> size, bit negate,2481 bit saturate, list<dag> pattern=[]>2482 : MVEIntSingleSrc<iname, suffix, size, pattern> {2483 2484 let Inst{28} = 0b1;2485 let Inst{25-23} = 0b111;2486 let Inst{21-20} = 0b11;2487 let Inst{17} = 0b0;2488 let Inst{16} = !eq(saturate, 0);2489 let Inst{12-11} = 0b00;2490 let Inst{10} = saturate;2491 let Inst{9-8} = 0b11;2492 let Inst{7} = negate;2493 let Inst{6} = 0b1;2494 let Inst{4} = 0b0;2495 let Inst{0} = 0b0;2496 let validForTailPredication = 1;2497}2498 2499multiclass MVE_VABSNEG_int_m<string iname, bit negate, bit saturate,2500 SDPatternOperator unpred_op, Intrinsic pred_int,2501 MVEVectorVTInfo VTI> {2502 def "" : MVE_VABSNEG_int<iname, VTI.Suffix, VTI.Size, negate, saturate>;2503 defvar Inst = !cast<Instruction>(NAME);2504 2505 let Predicates = [HasMVEInt] in {2506 // VQABS and VQNEG have more difficult isel patterns defined elsewhere2507 if !not(saturate) then {2508 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$v))),2509 (VTI.Vec (Inst $v))>;2510 }2511 2512 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$v), (VTI.Pred VCCR:$mask),2513 (VTI.Vec MQPR:$inactive))),2514 (VTI.Vec (Inst $v, ARMVCCThen, $mask, zero_reg, $inactive))>;2515 }2516}2517 2518foreach VTI = [ MVE_v16s8, MVE_v8s16, MVE_v4s32 ] in {2519 defm "MVE_VABS" # VTI.Suffix : MVE_VABSNEG_int_m<2520 "vabs", 0, 0, abs, int_arm_mve_abs_predicated, VTI>;2521 defm "MVE_VQABS" # VTI.Suffix : MVE_VABSNEG_int_m<2522 "vqabs", 0, 1, ?, int_arm_mve_qabs_predicated, VTI>;2523 defm "MVE_VNEG" # VTI.Suffix : MVE_VABSNEG_int_m<2524 "vneg", 1, 0, vnegq, int_arm_mve_neg_predicated, VTI>;2525 defm "MVE_VQNEG" # VTI.Suffix : MVE_VABSNEG_int_m<2526 "vqneg", 1, 1, ?, int_arm_mve_qneg_predicated, VTI>;2527}2528 2529// int_min/int_max: vector containing INT_MIN/INT_MAX VTI.Size times2530// zero_vec: v4i32-initialized zero vector, potentially wrapped in a bitconvert2531multiclass vqabsneg_pattern<MVEVectorVTInfo VTI, dag int_min, dag int_max,2532 dag zero_vec, MVE_VABSNEG_int vqabs_instruction,2533 MVE_VABSNEG_int vqneg_instruction> {2534 let Predicates = [HasMVEInt] in {2535 // The below tree can be replaced by a vqabs instruction, as it represents2536 // the following vectorized expression (r being the value in $reg):2537 // r > 0 ? r : (r == INT_MIN ? INT_MAX : -r)2538 def : Pat<(VTI.Vec (vselect2539 (VTI.Pred (ARMvcmpz (VTI.Vec MQPR:$reg), ARMCCgt)),2540 (VTI.Vec MQPR:$reg),2541 (VTI.Vec (vselect2542 (VTI.Pred (ARMvcmp (VTI.Vec MQPR:$reg), int_min, ARMCCeq)),2543 int_max,2544 (sub (VTI.Vec zero_vec), (VTI.Vec MQPR:$reg)))))),2545 (VTI.Vec (vqabs_instruction (VTI.Vec MQPR:$reg)))>;2546 // Similarly, this tree represents vqneg, i.e. the following vectorized expression:2547 // r == INT_MIN ? INT_MAX : -r2548 def : Pat<(VTI.Vec (vselect2549 (VTI.Pred (ARMvcmp (VTI.Vec MQPR:$reg), int_min, ARMCCeq)),2550 int_max,2551 (sub (VTI.Vec zero_vec), (VTI.Vec MQPR:$reg)))),2552 (VTI.Vec (vqneg_instruction (VTI.Vec MQPR:$reg)))>;2553 }2554}2555 2556defm MVE_VQABSNEG_Ps8 : vqabsneg_pattern<MVE_v16i8,2557 (v16i8 (ARMvmovImm (i32 3712))),2558 (v16i8 (ARMvmovImm (i32 3711))),2559 (bitconvert (v4i32 (ARMvmovImm (i32 0)))),2560 MVE_VQABSs8, MVE_VQNEGs8>;2561defm MVE_VQABSNEG_Ps16 : vqabsneg_pattern<MVE_v8i16,2562 (v8i16 (ARMvmovImm (i32 2688))),2563 (v8i16 (ARMvmvnImm (i32 2688))),2564 (bitconvert (v4i32 (ARMvmovImm (i32 0)))),2565 MVE_VQABSs16, MVE_VQNEGs16>;2566defm MVE_VQABSNEG_Ps32 : vqabsneg_pattern<MVE_v4i32,2567 (v4i32 (ARMvmovImm (i32 1664))),2568 (v4i32 (ARMvmvnImm (i32 1664))),2569 (ARMvmovImm (i32 0)),2570 MVE_VQABSs32, MVE_VQNEGs32>;2571 2572class MVE_mod_imm<string iname, string suffix, bits<4> cmode, bit op,2573 dag iops, bits<2> vecsize, list<dag> pattern=[]>2574 : MVE_p<(outs MQPR:$Qd), iops, NoItinerary, iname, suffix, "$Qd, $imm",2575 vpred_r, "", vecsize, pattern> {2576 bits<13> imm;2577 bits<4> Qd;2578 2579 let Inst{28} = imm{7};2580 let Inst{25-23} = 0b111;2581 let Inst{22} = Qd{3};2582 let Inst{21-19} = 0b000;2583 let Inst{18-16} = imm{6-4};2584 let Inst{15-13} = Qd{2-0};2585 let Inst{12} = 0b0;2586 let Inst{11-8} = cmode{3-0};2587 let Inst{7-6} = 0b01;2588 let Inst{5} = op;2589 let Inst{4} = 0b1;2590 let Inst{3-0} = imm{3-0};2591 2592 let DecoderMethod = "DecodeMVEModImmInstruction";2593 let validForTailPredication = 1;2594}2595 2596let isReMaterializable = 1 in {2597let isAsCheapAsAMove = 1 in {2598def MVE_VMOVimmi8 : MVE_mod_imm<"vmov", "i8", {1,1,1,0}, 0b0, (ins nImmSplatI8:$imm), 0b00>;2599def MVE_VMOVimmi16 : MVE_mod_imm<"vmov", "i16", {1,0,?,0}, 0b0, (ins nImmSplatI16:$imm), 0b01> {2600 let Inst{9} = imm{9};2601}2602def MVE_VMOVimmi32 : MVE_mod_imm<"vmov", "i32", {?,?,?,?}, 0b0, (ins nImmVMOVI32:$imm), 0b10> {2603 let Inst{11-8} = imm{11-8};2604}2605def MVE_VMOVimmi64 : MVE_mod_imm<"vmov", "i64", {1,1,1,0}, 0b1, (ins nImmSplatI64:$imm), 0b11>;2606def MVE_VMOVimmf32 : MVE_mod_imm<"vmov", "f32", {1,1,1,1}, 0b0, (ins nImmVMOVF32:$imm), 0b10>;2607} // let isAsCheapAsAMove = 12608 2609def MVE_VMVNimmi16 : MVE_mod_imm<"vmvn", "i16", {1,0,?,0}, 0b1, (ins nImmSplatI16:$imm), 0b01> {2610 let Inst{9} = imm{9};2611}2612def MVE_VMVNimmi32 : MVE_mod_imm<"vmvn", "i32", {?,?,?,?}, 0b1, (ins nImmVMOVI32:$imm), 0b10> {2613 let Inst{11-8} = imm{11-8};2614}2615} // let isReMaterializable = 12616 2617let Predicates = [HasMVEInt] in {2618 def : Pat<(v16i8 (ARMvmovImm timm:$simm)),2619 (v16i8 (MVE_VMOVimmi8 nImmSplatI8:$simm))>;2620 def : Pat<(v8i16 (ARMvmovImm timm:$simm)),2621 (v8i16 (MVE_VMOVimmi16 nImmSplatI16:$simm))>;2622 def : Pat<(v4i32 (ARMvmovImm timm:$simm)),2623 (v4i32 (MVE_VMOVimmi32 nImmVMOVI32:$simm))>;2624 def : Pat<(v2i64 (ARMvmovImm timm:$simm)),2625 (v2i64 (MVE_VMOVimmi64 nImmSplatI64:$simm))>;2626 2627 def : Pat<(v8i16 (ARMvmvnImm timm:$simm)),2628 (v8i16 (MVE_VMVNimmi16 nImmSplatI16:$simm))>;2629 def : Pat<(v4i32 (ARMvmvnImm timm:$simm)),2630 (v4i32 (MVE_VMVNimmi32 nImmVMOVI32:$simm))>;2631 2632 def : Pat<(v4f32 (ARMvmovFPImm timm:$simm)),2633 (v4f32 (MVE_VMOVimmf32 nImmVMOVF32:$simm))>;2634 2635 def : Pat<(v8i16 (vselect (v8i1 VCCR:$pred), (ARMvmvnImm timm:$simm),2636 MQPR:$inactive)),2637 (v8i16 (MVE_VMVNimmi16 nImmSplatI16:$simm,2638 ARMVCCThen, VCCR:$pred, zero_reg, MQPR:$inactive))>;2639 def : Pat<(v4i32 (vselect (v4i1 VCCR:$pred), (ARMvmvnImm timm:$simm),2640 MQPR:$inactive)),2641 (v4i32 (MVE_VMVNimmi32 nImmSplatI32:$simm,2642 ARMVCCThen, VCCR:$pred, zero_reg, MQPR:$inactive))>;2643}2644 2645class MVE_VMINMAXA<string iname, string suffix, bits<2> size,2646 bit bit_12, list<dag> pattern=[]>2647 : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qd_src, MQPR:$Qm),2648 NoItinerary, iname, suffix, "$Qd, $Qm", vpred_n, "$Qd = $Qd_src",2649 size, pattern> {2650 bits<4> Qd;2651 bits<4> Qm;2652 2653 let Inst{28} = 0b0;2654 let Inst{25-23} = 0b100;2655 let Inst{22} = Qd{3};2656 let Inst{21-20} = 0b11;2657 let Inst{19-18} = size;2658 let Inst{17-16} = 0b11;2659 let Inst{15-13} = Qd{2-0};2660 let Inst{12} = bit_12;2661 let Inst{11-6} = 0b111010;2662 let Inst{5} = Qm{3};2663 let Inst{4} = 0b0;2664 let Inst{3-1} = Qm{2-0};2665 let Inst{0} = 0b1;2666 let validForTailPredication = 1;2667}2668 2669multiclass MVE_VMINMAXA_m<string iname, MVEVectorVTInfo VTI,2670 SDNode unpred_op, Intrinsic pred_int, bit bit_12> {2671 def "" : MVE_VMINMAXA<iname, VTI.Suffix, VTI.Size, bit_12>;2672 defvar Inst = !cast<Instruction>(NAME);2673 2674 let Predicates = [HasMVEInt] in {2675 // Unpredicated v(min|max)a2676 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qd), (abs (VTI.Vec MQPR:$Qm)))),2677 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm)))>;2678 2679 // Predicated v(min|max)a2680 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm),2681 (VTI.Pred VCCR:$mask))),2682 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm),2683 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;2684 }2685}2686 2687multiclass MVE_VMINA<MVEVectorVTInfo VTI>2688 : MVE_VMINMAXA_m<"vmina", VTI, umin, int_arm_mve_vmina_predicated, 0b1>;2689 2690defm MVE_VMINAs8 : MVE_VMINA<MVE_v16s8>;2691defm MVE_VMINAs16 : MVE_VMINA<MVE_v8s16>;2692defm MVE_VMINAs32 : MVE_VMINA<MVE_v4s32>;2693 2694multiclass MVE_VMAXA<MVEVectorVTInfo VTI>2695 : MVE_VMINMAXA_m<"vmaxa", VTI, umax, int_arm_mve_vmaxa_predicated, 0b0>;2696 2697defm MVE_VMAXAs8 : MVE_VMAXA<MVE_v16s8>;2698defm MVE_VMAXAs16 : MVE_VMAXA<MVE_v8s16>;2699defm MVE_VMAXAs32 : MVE_VMAXA<MVE_v4s32>;2700 2701// end of MVE Integer instructions2702 2703// start of mve_imm_shift instructions2704 2705def MVE_VSHLC : MVE_p<(outs rGPR:$RdmDest, MQPR:$Qd),2706 (ins MQPR:$QdSrc, rGPR:$RdmSrc, long_shift:$imm),2707 NoItinerary, "vshlc", "", "$QdSrc, $RdmSrc, $imm",2708 vpred_n, "$RdmDest = $RdmSrc,$Qd = $QdSrc", 0b10> {2709 bits<5> imm;2710 bits<4> Qd;2711 bits<4> RdmDest;2712 2713 let Inst{28} = 0b0;2714 let Inst{25-23} = 0b101;2715 let Inst{22} = Qd{3};2716 let Inst{21} = 0b1;2717 let Inst{20-16} = imm{4-0};2718 let Inst{15-13} = Qd{2-0};2719 let Inst{12-4} = 0b011111100;2720 let Inst{3-0} = RdmDest{3-0};2721}2722 2723class MVE_shift_imm<dag oops, dag iops, string iname, string suffix,2724 string ops, vpred_ops vpred, string cstr,2725 bits<2> vecsize, list<dag> pattern=[]>2726 : MVE_p<oops, iops, NoItinerary, iname, suffix, ops, vpred, cstr, vecsize, pattern> {2727 bits<4> Qd;2728 bits<4> Qm;2729 2730 let Inst{22} = Qd{3};2731 let Inst{15-13} = Qd{2-0};2732 let Inst{5} = Qm{3};2733 let Inst{3-1} = Qm{2-0};2734}2735 2736class MVE_VMOVL<string iname, string suffix, bits<2> sz, bit U, bit top,2737 list<dag> pattern=[]>2738 : MVE_shift_imm<(outs MQPR:$Qd), (ins MQPR:$Qm),2739 iname, suffix, "$Qd, $Qm", vpred_r, "",2740 sz, pattern> {2741 let Inst{28} = U;2742 let Inst{25-23} = 0b101;2743 let Inst{21} = 0b1;2744 let Inst{20-19} = sz{1-0};2745 let Inst{18-16} = 0b000;2746 let Inst{12} = top;2747 let Inst{11-6} = 0b111101;2748 let Inst{4} = 0b0;2749 let Inst{0} = 0b0;2750 let doubleWidthResult = 1;2751}2752 2753multiclass MVE_VMOVL_m<bit top, string chr, MVEVectorVTInfo OutVTI,2754 MVEVectorVTInfo InVTI> {2755 def "": MVE_VMOVL<"vmovl" # chr, InVTI.Suffix, OutVTI.Size,2756 InVTI.Unsigned, top>;2757 defvar Inst = !cast<Instruction>(NAME);2758 2759 def : Pat<(OutVTI.Vec (int_arm_mve_vmovl_predicated (InVTI.Vec MQPR:$src),2760 (i32 InVTI.Unsigned), (i32 top),2761 (OutVTI.Pred VCCR:$pred),2762 (OutVTI.Vec MQPR:$inactive))),2763 (OutVTI.Vec (Inst (InVTI.Vec MQPR:$src), ARMVCCThen,2764 (OutVTI.Pred VCCR:$pred), zero_reg,2765 (OutVTI.Vec MQPR:$inactive)))>;2766}2767 2768defm MVE_VMOVLs8bh : MVE_VMOVL_m<0, "b", MVE_v8s16, MVE_v16s8>;2769defm MVE_VMOVLs8th : MVE_VMOVL_m<1, "t", MVE_v8s16, MVE_v16s8>;2770defm MVE_VMOVLu8bh : MVE_VMOVL_m<0, "b", MVE_v8u16, MVE_v16u8>;2771defm MVE_VMOVLu8th : MVE_VMOVL_m<1, "t", MVE_v8u16, MVE_v16u8>;2772defm MVE_VMOVLs16bh : MVE_VMOVL_m<0, "b", MVE_v4s32, MVE_v8s16>;2773defm MVE_VMOVLs16th : MVE_VMOVL_m<1, "t", MVE_v4s32, MVE_v8s16>;2774defm MVE_VMOVLu16bh : MVE_VMOVL_m<0, "b", MVE_v4s32, MVE_v8u16>;2775defm MVE_VMOVLu16th : MVE_VMOVL_m<1, "t", MVE_v4s32, MVE_v8u16>;2776 2777let Predicates = [HasMVEInt] in {2778 def : Pat<(sext_inreg (v4i32 MQPR:$src), v4i16),2779 (MVE_VMOVLs16bh MQPR:$src)>;2780 def : Pat<(sext_inreg (v8i16 MQPR:$src), v8i8),2781 (MVE_VMOVLs8bh MQPR:$src)>;2782 def : Pat<(sext_inreg (v4i32 MQPR:$src), v4i8),2783 (MVE_VMOVLs16bh (MVE_VMOVLs8bh MQPR:$src))>;2784 2785 def : Pat<(sext_inreg (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src)))), v8i8),2786 (MVE_VMOVLs8th MQPR:$src)>;2787 def : Pat<(sext_inreg (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src)))), v4i16),2788 (MVE_VMOVLs16th MQPR:$src)>;2789 2790 // zext_inreg 8 -> 162791 def : Pat<(ARMvbicImm (v8i16 MQPR:$src), (i32 0xAFF)),2792 (MVE_VMOVLu8bh MQPR:$src)>;2793 // zext_inreg 16 -> 322794 def : Pat<(and (v4i32 MQPR:$src), (v4i32 (ARMvmovImm (i32 0xCFF)))),2795 (MVE_VMOVLu16bh MQPR:$src)>;2796 // Same zext_inreg with vrevs, picking the top half2797 def : Pat<(ARMvbicImm (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src)))), (i32 0xAFF)),2798 (MVE_VMOVLu8th MQPR:$src)>;2799 def : Pat<(and (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src)))),2800 (v4i32 (ARMvmovImm (i32 0xCFF)))),2801 (MVE_VMOVLu16th MQPR:$src)>;2802}2803 2804 2805class MVE_VSHLL_imm<string iname, string suffix, bit U, bit th,2806 Operand immtype, bits<2> vecsize, list<dag> pattern=[]>2807 : MVE_shift_imm<(outs MQPR:$Qd), (ins MQPR:$Qm, immtype:$imm),2808 iname, suffix, "$Qd, $Qm, $imm", vpred_r, "", vecsize, pattern> {2809 let Inst{28} = U;2810 let Inst{25-23} = 0b101;2811 let Inst{21} = 0b1;2812 let Inst{12} = th;2813 let Inst{11-6} = 0b111101;2814 let Inst{4} = 0b0;2815 let Inst{0} = 0b0;2816 2817 // For the MVE_VSHLL_patterns multiclass to refer to2818 Operand immediateType = immtype;2819 2820 let doubleWidthResult = 1;2821}2822 2823// The immediate VSHLL instructions accept shift counts from 1 up to2824// the lane width (8 or 16), but the full-width shifts have an2825// entirely separate encoding, given below with 'lw' in the name.2826 2827class MVE_VSHLL_imm8<string iname, string suffix,2828 bit U, bit th, list<dag> pattern=[]>2829 : MVE_VSHLL_imm<iname, suffix, U, th, mve_shift_imm1_7, 0b01, pattern> {2830 bits<3> imm;2831 let Inst{20-19} = 0b01;2832 let Inst{18-16} = imm;2833}2834 2835class MVE_VSHLL_imm16<string iname, string suffix,2836 bit U, bit th, list<dag> pattern=[]>2837 : MVE_VSHLL_imm<iname, suffix, U, th, mve_shift_imm1_15, 0b10, pattern> {2838 bits<4> imm;2839 let Inst{20} = 0b1;2840 let Inst{19-16} = imm;2841}2842 2843def MVE_VSHLL_imms8bh : MVE_VSHLL_imm8 <"vshllb", "s8", 0b0, 0b0>;2844def MVE_VSHLL_imms8th : MVE_VSHLL_imm8 <"vshllt", "s8", 0b0, 0b1>;2845def MVE_VSHLL_immu8bh : MVE_VSHLL_imm8 <"vshllb", "u8", 0b1, 0b0>;2846def MVE_VSHLL_immu8th : MVE_VSHLL_imm8 <"vshllt", "u8", 0b1, 0b1>;2847def MVE_VSHLL_imms16bh : MVE_VSHLL_imm16<"vshllb", "s16", 0b0, 0b0>;2848def MVE_VSHLL_imms16th : MVE_VSHLL_imm16<"vshllt", "s16", 0b0, 0b1>;2849def MVE_VSHLL_immu16bh : MVE_VSHLL_imm16<"vshllb", "u16", 0b1, 0b0>;2850def MVE_VSHLL_immu16th : MVE_VSHLL_imm16<"vshllt", "u16", 0b1, 0b1>;2851 2852class MVE_VSHLL_by_lane_width<string iname, string suffix, bits<2> size,2853 bit U, string ops, list<dag> pattern=[]>2854 : MVE_shift_imm<(outs MQPR:$Qd), (ins MQPR:$Qm),2855 iname, suffix, ops, vpred_r, "", !if(size, 0b10, 0b01), pattern> {2856 let Inst{28} = U;2857 let Inst{25-23} = 0b100;2858 let Inst{21-20} = 0b11;2859 let Inst{19-18} = size{1-0};2860 let Inst{17-16} = 0b01;2861 let Inst{11-6} = 0b111000;2862 let Inst{4} = 0b0;2863 let Inst{0} = 0b1;2864 let doubleWidthResult = 1;2865}2866 2867multiclass MVE_VSHLL_lw<string iname, string suffix, bits<2> sz, bit U,2868 string ops, list<dag> pattern=[]> {2869 def bh : MVE_VSHLL_by_lane_width<iname#"b", suffix, sz, U, ops, pattern> {2870 let Inst{12} = 0b0;2871 }2872 def th : MVE_VSHLL_by_lane_width<iname#"t", suffix, sz, U, ops, pattern> {2873 let Inst{12} = 0b1;2874 }2875}2876 2877defm MVE_VSHLL_lws8 : MVE_VSHLL_lw<"vshll", "s8", 0b00, 0b0, "$Qd, $Qm, #8">;2878defm MVE_VSHLL_lws16 : MVE_VSHLL_lw<"vshll", "s16", 0b01, 0b0, "$Qd, $Qm, #16">;2879defm MVE_VSHLL_lwu8 : MVE_VSHLL_lw<"vshll", "u8", 0b00, 0b1, "$Qd, $Qm, #8">;2880defm MVE_VSHLL_lwu16 : MVE_VSHLL_lw<"vshll", "u16", 0b01, 0b1, "$Qd, $Qm, #16">;2881 2882multiclass MVE_VSHLL_patterns<MVEVectorVTInfo VTI, int top> {2883 defvar suffix = !strconcat(VTI.Suffix, !if(top, "th", "bh"));2884 defvar inst_imm = !cast<MVE_VSHLL_imm>("MVE_VSHLL_imm" # suffix);2885 defvar inst_lw = !cast<MVE_VSHLL_by_lane_width>("MVE_VSHLL_lw" # suffix);2886 defvar unpred_int = int_arm_mve_vshll_imm;2887 defvar pred_int = int_arm_mve_vshll_imm_predicated;2888 defvar imm = inst_imm.immediateType;2889 2890 def : Pat<(VTI.DblVec (unpred_int (VTI.Vec MQPR:$src), imm:$imm,2891 (i32 VTI.Unsigned), (i32 top))),2892 (VTI.DblVec (inst_imm (VTI.Vec MQPR:$src), imm:$imm))>;2893 def : Pat<(VTI.DblVec (unpred_int (VTI.Vec MQPR:$src), (i32 VTI.LaneBits),2894 (i32 VTI.Unsigned), (i32 top))),2895 (VTI.DblVec (inst_lw (VTI.Vec MQPR:$src)))>;2896 2897 def : Pat<(VTI.DblVec (pred_int (VTI.Vec MQPR:$src), imm:$imm,2898 (i32 VTI.Unsigned), (i32 top),2899 (VTI.DblPred VCCR:$mask),2900 (VTI.DblVec MQPR:$inactive))),2901 (VTI.DblVec (inst_imm (VTI.Vec MQPR:$src), imm:$imm,2902 ARMVCCThen, (VTI.DblPred VCCR:$mask), zero_reg,2903 (VTI.DblVec MQPR:$inactive)))>;2904 def : Pat<(VTI.DblVec (pred_int (VTI.Vec MQPR:$src), (i32 VTI.LaneBits),2905 (i32 VTI.Unsigned), (i32 top),2906 (VTI.DblPred VCCR:$mask),2907 (VTI.DblVec MQPR:$inactive))),2908 (VTI.DblVec (inst_lw (VTI.Vec MQPR:$src), ARMVCCThen,2909 (VTI.DblPred VCCR:$mask), zero_reg,2910 (VTI.DblVec MQPR:$inactive)))>;2911}2912 2913foreach VTI = [MVE_v16s8, MVE_v8s16, MVE_v16u8, MVE_v8u16] in2914 foreach top = [0, 1] in2915 defm : MVE_VSHLL_patterns<VTI, top>;2916 2917class MVE_shift_imm_partial<Operand imm, string iname, string suffix, bits<2> vecsize>2918 : MVE_shift_imm<(outs MQPR:$Qd), (ins MQPR:$QdSrc, MQPR:$Qm, imm:$imm),2919 iname, suffix, "$Qd, $Qm, $imm", vpred_n, "$Qd = $QdSrc", vecsize> {2920 Operand immediateType = imm;2921}2922 2923class MVE_VxSHRN<string iname, string suffix, bit bit_12, bit bit_28,2924 Operand imm, bits<2> vecsize>2925 : MVE_shift_imm_partial<imm, iname, suffix, vecsize> {2926 bits<5> imm;2927 2928 let Inst{28} = bit_28;2929 let Inst{25-23} = 0b101;2930 let Inst{21} = 0b0;2931 let Inst{20-16} = imm{4-0};2932 let Inst{12} = bit_12;2933 let Inst{11-6} = 0b111111;2934 let Inst{4} = 0b0;2935 let Inst{0} = 0b1;2936 let validForTailPredication = 1;2937 let retainsPreviousHalfElement = 1;2938}2939 2940def MVE_VRSHRNi16bh : MVE_VxSHRN<"vrshrnb", "i16", 0b0, 0b1, shr_imm8, 0b01> {2941 let Inst{20-19} = 0b01;2942}2943def MVE_VRSHRNi16th : MVE_VxSHRN<"vrshrnt", "i16", 0b1, 0b1, shr_imm8, 0b01> {2944 let Inst{20-19} = 0b01;2945}2946def MVE_VRSHRNi32bh : MVE_VxSHRN<"vrshrnb", "i32", 0b0, 0b1, shr_imm16, 0b10> {2947 let Inst{20} = 0b1;2948}2949def MVE_VRSHRNi32th : MVE_VxSHRN<"vrshrnt", "i32", 0b1, 0b1, shr_imm16, 0b10> {2950 let Inst{20} = 0b1;2951}2952 2953def MVE_VSHRNi16bh : MVE_VxSHRN<"vshrnb", "i16", 0b0, 0b0, shr_imm8, 0b01> {2954 let Inst{20-19} = 0b01;2955}2956def MVE_VSHRNi16th : MVE_VxSHRN<"vshrnt", "i16", 0b1, 0b0, shr_imm8, 0b01> {2957 let Inst{20-19} = 0b01;2958}2959def MVE_VSHRNi32bh : MVE_VxSHRN<"vshrnb", "i32", 0b0, 0b0, shr_imm16, 0b10> {2960 let Inst{20} = 0b1;2961}2962def MVE_VSHRNi32th : MVE_VxSHRN<"vshrnt", "i32", 0b1, 0b0, shr_imm16, 0b10> {2963 let Inst{20} = 0b1;2964}2965 2966class MVE_VxQRSHRUN<string iname, string suffix, bit bit_28, bit bit_12,2967 Operand imm, bits<2> vecsize>2968 : MVE_shift_imm_partial<imm, iname, suffix, vecsize> {2969 bits<5> imm;2970 2971 let Inst{28} = bit_28;2972 let Inst{25-23} = 0b101;2973 let Inst{21} = 0b0;2974 let Inst{20-16} = imm{4-0};2975 let Inst{12} = bit_12;2976 let Inst{11-6} = 0b111111;2977 let Inst{4} = 0b0;2978 let Inst{0} = 0b0;2979 let validForTailPredication = 1;2980 let retainsPreviousHalfElement = 1;2981}2982 2983def MVE_VQRSHRUNs16bh : MVE_VxQRSHRUN<2984 "vqrshrunb", "s16", 0b1, 0b0, shr_imm8, 0b01> {2985 let Inst{20-19} = 0b01;2986}2987def MVE_VQRSHRUNs16th : MVE_VxQRSHRUN<2988 "vqrshrunt", "s16", 0b1, 0b1, shr_imm8, 0b01> {2989 let Inst{20-19} = 0b01;2990}2991def MVE_VQRSHRUNs32bh : MVE_VxQRSHRUN<2992 "vqrshrunb", "s32", 0b1, 0b0, shr_imm16, 0b10> {2993 let Inst{20} = 0b1;2994}2995def MVE_VQRSHRUNs32th : MVE_VxQRSHRUN<2996 "vqrshrunt", "s32", 0b1, 0b1, shr_imm16, 0b10> {2997 let Inst{20} = 0b1;2998}2999 3000def MVE_VQSHRUNs16bh : MVE_VxQRSHRUN<3001 "vqshrunb", "s16", 0b0, 0b0, shr_imm8, 0b01> {3002 let Inst{20-19} = 0b01;3003}3004def MVE_VQSHRUNs16th : MVE_VxQRSHRUN<3005 "vqshrunt", "s16", 0b0, 0b1, shr_imm8, 0b01> {3006 let Inst{20-19} = 0b01;3007}3008def MVE_VQSHRUNs32bh : MVE_VxQRSHRUN<3009 "vqshrunb", "s32", 0b0, 0b0, shr_imm16, 0b10> {3010 let Inst{20} = 0b1;3011}3012def MVE_VQSHRUNs32th : MVE_VxQRSHRUN<3013 "vqshrunt", "s32", 0b0, 0b1, shr_imm16, 0b10> {3014 let Inst{20} = 0b1;3015}3016 3017class MVE_VxQRSHRN<string iname, string suffix, bit bit_0, bit bit_12,3018 Operand imm, bits<2> vecsize>3019 : MVE_shift_imm_partial<imm, iname, suffix, vecsize> {3020 bits<5> imm;3021 3022 let Inst{25-23} = 0b101;3023 let Inst{21} = 0b0;3024 let Inst{20-16} = imm{4-0};3025 let Inst{12} = bit_12;3026 let Inst{11-6} = 0b111101;3027 let Inst{4} = 0b0;3028 let Inst{0} = bit_0;3029 let validForTailPredication = 1;3030 let retainsPreviousHalfElement = 1;3031}3032 3033multiclass MVE_VxQRSHRN_types<string iname, bit bit_0, bit bit_12> {3034 def s16 : MVE_VxQRSHRN<iname, "s16", bit_0, bit_12, shr_imm8, 0b01> {3035 let Inst{28} = 0b0;3036 let Inst{20-19} = 0b01;3037 }3038 def u16 : MVE_VxQRSHRN<iname, "u16", bit_0, bit_12, shr_imm8, 0b01> {3039 let Inst{28} = 0b1;3040 let Inst{20-19} = 0b01;3041 }3042 def s32 : MVE_VxQRSHRN<iname, "s32", bit_0, bit_12, shr_imm16, 0b10> {3043 let Inst{28} = 0b0;3044 let Inst{20} = 0b1;3045 }3046 def u32 : MVE_VxQRSHRN<iname, "u32", bit_0, bit_12, shr_imm16, 0b10> {3047 let Inst{28} = 0b1;3048 let Inst{20} = 0b1;3049 }3050}3051 3052defm MVE_VQRSHRNbh : MVE_VxQRSHRN_types<"vqrshrnb", 0b1, 0b0>;3053defm MVE_VQRSHRNth : MVE_VxQRSHRN_types<"vqrshrnt", 0b1, 0b1>;3054defm MVE_VQSHRNbh : MVE_VxQRSHRN_types<"vqshrnb", 0b0, 0b0>;3055defm MVE_VQSHRNth : MVE_VxQRSHRN_types<"vqshrnt", 0b0, 0b1>;3056 3057multiclass MVE_VSHRN_patterns<MVE_shift_imm_partial inst,3058 MVEVectorVTInfo OutVTI, MVEVectorVTInfo InVTI,3059 bit q, bit r, bit top> {3060 defvar inparams = (? (OutVTI.Vec MQPR:$QdSrc), (InVTI.Vec MQPR:$Qm),3061 (inst.immediateType:$imm), (i32 q), (i32 r),3062 (i32 OutVTI.Unsigned), (i32 InVTI.Unsigned), (i32 top));3063 defvar outparams = (inst (OutVTI.Vec MQPR:$QdSrc), (InVTI.Vec MQPR:$Qm),3064 (imm:$imm));3065 3066 def : Pat<(OutVTI.Vec !setdagop(inparams, int_arm_mve_vshrn)),3067 (OutVTI.Vec outparams)>;3068 def : Pat<(OutVTI.Vec !con(inparams, (int_arm_mve_vshrn_predicated3069 (InVTI.Pred VCCR:$pred)))),3070 (OutVTI.Vec !con(outparams, (? ARMVCCThen, VCCR:$pred, zero_reg)))>;3071}3072 3073defm : MVE_VSHRN_patterns<MVE_VSHRNi16bh, MVE_v16s8, MVE_v8s16, 0,0,0>;3074defm : MVE_VSHRN_patterns<MVE_VSHRNi16th, MVE_v16s8, MVE_v8s16, 0,0,1>;3075defm : MVE_VSHRN_patterns<MVE_VSHRNi32bh, MVE_v8s16, MVE_v4s32, 0,0,0>;3076defm : MVE_VSHRN_patterns<MVE_VSHRNi32th, MVE_v8s16, MVE_v4s32, 0,0,1>;3077defm : MVE_VSHRN_patterns<MVE_VSHRNi16bh, MVE_v16u8, MVE_v8u16, 0,0,0>;3078defm : MVE_VSHRN_patterns<MVE_VSHRNi16th, MVE_v16u8, MVE_v8u16, 0,0,1>;3079defm : MVE_VSHRN_patterns<MVE_VSHRNi32bh, MVE_v8u16, MVE_v4u32, 0,0,0>;3080defm : MVE_VSHRN_patterns<MVE_VSHRNi32th, MVE_v8u16, MVE_v4u32, 0,0,1>;3081defm : MVE_VSHRN_patterns<MVE_VRSHRNi16bh, MVE_v16s8, MVE_v8s16, 0,1,0>;3082defm : MVE_VSHRN_patterns<MVE_VRSHRNi16th, MVE_v16s8, MVE_v8s16, 0,1,1>;3083defm : MVE_VSHRN_patterns<MVE_VRSHRNi32bh, MVE_v8s16, MVE_v4s32, 0,1,0>;3084defm : MVE_VSHRN_patterns<MVE_VRSHRNi32th, MVE_v8s16, MVE_v4s32, 0,1,1>;3085defm : MVE_VSHRN_patterns<MVE_VRSHRNi16bh, MVE_v16u8, MVE_v8u16, 0,1,0>;3086defm : MVE_VSHRN_patterns<MVE_VRSHRNi16th, MVE_v16u8, MVE_v8u16, 0,1,1>;3087defm : MVE_VSHRN_patterns<MVE_VRSHRNi32bh, MVE_v8u16, MVE_v4u32, 0,1,0>;3088defm : MVE_VSHRN_patterns<MVE_VRSHRNi32th, MVE_v8u16, MVE_v4u32, 0,1,1>;3089defm : MVE_VSHRN_patterns<MVE_VQSHRNbhs16, MVE_v16s8, MVE_v8s16, 1,0,0>;3090defm : MVE_VSHRN_patterns<MVE_VQSHRNths16, MVE_v16s8, MVE_v8s16, 1,0,1>;3091defm : MVE_VSHRN_patterns<MVE_VQSHRNbhs32, MVE_v8s16, MVE_v4s32, 1,0,0>;3092defm : MVE_VSHRN_patterns<MVE_VQSHRNths32, MVE_v8s16, MVE_v4s32, 1,0,1>;3093defm : MVE_VSHRN_patterns<MVE_VQSHRNbhu16, MVE_v16u8, MVE_v8u16, 1,0,0>;3094defm : MVE_VSHRN_patterns<MVE_VQSHRNthu16, MVE_v16u8, MVE_v8u16, 1,0,1>;3095defm : MVE_VSHRN_patterns<MVE_VQSHRNbhu32, MVE_v8u16, MVE_v4u32, 1,0,0>;3096defm : MVE_VSHRN_patterns<MVE_VQSHRNthu32, MVE_v8u16, MVE_v4u32, 1,0,1>;3097defm : MVE_VSHRN_patterns<MVE_VQRSHRNbhs16, MVE_v16s8, MVE_v8s16, 1,1,0>;3098defm : MVE_VSHRN_patterns<MVE_VQRSHRNths16, MVE_v16s8, MVE_v8s16, 1,1,1>;3099defm : MVE_VSHRN_patterns<MVE_VQRSHRNbhs32, MVE_v8s16, MVE_v4s32, 1,1,0>;3100defm : MVE_VSHRN_patterns<MVE_VQRSHRNths32, MVE_v8s16, MVE_v4s32, 1,1,1>;3101defm : MVE_VSHRN_patterns<MVE_VQRSHRNbhu16, MVE_v16u8, MVE_v8u16, 1,1,0>;3102defm : MVE_VSHRN_patterns<MVE_VQRSHRNthu16, MVE_v16u8, MVE_v8u16, 1,1,1>;3103defm : MVE_VSHRN_patterns<MVE_VQRSHRNbhu32, MVE_v8u16, MVE_v4u32, 1,1,0>;3104defm : MVE_VSHRN_patterns<MVE_VQRSHRNthu32, MVE_v8u16, MVE_v4u32, 1,1,1>;3105defm : MVE_VSHRN_patterns<MVE_VQSHRUNs16bh, MVE_v16u8, MVE_v8s16, 1,0,0>;3106defm : MVE_VSHRN_patterns<MVE_VQSHRUNs16th, MVE_v16u8, MVE_v8s16, 1,0,1>;3107defm : MVE_VSHRN_patterns<MVE_VQSHRUNs32bh, MVE_v8u16, MVE_v4s32, 1,0,0>;3108defm : MVE_VSHRN_patterns<MVE_VQSHRUNs32th, MVE_v8u16, MVE_v4s32, 1,0,1>;3109defm : MVE_VSHRN_patterns<MVE_VQRSHRUNs16bh, MVE_v16u8, MVE_v8s16, 1,1,0>;3110defm : MVE_VSHRN_patterns<MVE_VQRSHRUNs16th, MVE_v16u8, MVE_v8s16, 1,1,1>;3111defm : MVE_VSHRN_patterns<MVE_VQRSHRUNs32bh, MVE_v8u16, MVE_v4s32, 1,1,0>;3112defm : MVE_VSHRN_patterns<MVE_VQRSHRUNs32th, MVE_v8u16, MVE_v4s32, 1,1,1>;3113 3114// end of mve_imm_shift instructions3115 3116// start of mve_shift instructions3117 3118class MVE_shift_by_vec<string iname, string suffix, bit U,3119 bits<2> size, bit bit_4, bit bit_8>3120 : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qm, MQPR:$Qn), NoItinerary,3121 iname, suffix, "$Qd, $Qm, $Qn", vpred_r, "", size, []> {3122 // Shift instructions which take a vector of shift counts3123 bits<4> Qd;3124 bits<4> Qm;3125 bits<4> Qn;3126 3127 let Inst{28} = U;3128 let Inst{25-24} = 0b11;3129 let Inst{23} = 0b0;3130 let Inst{22} = Qd{3};3131 let Inst{21-20} = size;3132 let Inst{19-17} = Qn{2-0};3133 let Inst{16} = 0b0;3134 let Inst{15-13} = Qd{2-0};3135 let Inst{12-9} = 0b0010;3136 let Inst{8} = bit_8;3137 let Inst{7} = Qn{3};3138 let Inst{6} = 0b1;3139 let Inst{5} = Qm{3};3140 let Inst{4} = bit_4;3141 let Inst{3-1} = Qm{2-0};3142 let Inst{0} = 0b0;3143 let validForTailPredication = 1;3144}3145 3146multiclass MVE_shift_by_vec_p<string iname, MVEVectorVTInfo VTI, bit q, bit r> {3147 def "" : MVE_shift_by_vec<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, q, r>;3148 defvar Inst = !cast<Instruction>(NAME);3149 3150 def : Pat<(VTI.Vec (int_arm_mve_vshl_vector3151 (VTI.Vec MQPR:$in), (VTI.Vec MQPR:$sh),3152 (i32 q), (i32 r), (i32 VTI.Unsigned))),3153 (VTI.Vec (Inst (VTI.Vec MQPR:$in), (VTI.Vec MQPR:$sh)))>;3154 3155 def : Pat<(VTI.Vec (int_arm_mve_vshl_vector_predicated3156 (VTI.Vec MQPR:$in), (VTI.Vec MQPR:$sh),3157 (i32 q), (i32 r), (i32 VTI.Unsigned),3158 (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive))),3159 (VTI.Vec (Inst (VTI.Vec MQPR:$in), (VTI.Vec MQPR:$sh),3160 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,3161 (VTI.Vec MQPR:$inactive)))>;3162}3163 3164multiclass mve_shift_by_vec_multi<string iname, bit bit_4, bit bit_8> {3165 defm s8 : MVE_shift_by_vec_p<iname, MVE_v16s8, bit_4, bit_8>;3166 defm s16 : MVE_shift_by_vec_p<iname, MVE_v8s16, bit_4, bit_8>;3167 defm s32 : MVE_shift_by_vec_p<iname, MVE_v4s32, bit_4, bit_8>;3168 defm u8 : MVE_shift_by_vec_p<iname, MVE_v16u8, bit_4, bit_8>;3169 defm u16 : MVE_shift_by_vec_p<iname, MVE_v8u16, bit_4, bit_8>;3170 defm u32 : MVE_shift_by_vec_p<iname, MVE_v4u32, bit_4, bit_8>;3171}3172 3173defm MVE_VSHL_by_vec : mve_shift_by_vec_multi<"vshl", 0b0, 0b0>;3174defm MVE_VQSHL_by_vec : mve_shift_by_vec_multi<"vqshl", 0b1, 0b0>;3175defm MVE_VQRSHL_by_vec : mve_shift_by_vec_multi<"vqrshl", 0b1, 0b1>;3176defm MVE_VRSHL_by_vec : mve_shift_by_vec_multi<"vrshl", 0b0, 0b1>;3177 3178let Predicates = [HasMVEInt] in {3179 defm : MVE_TwoOpPattern<MVE_v16i8, ARMvshlu, int_arm_mve_vshl_vector_predicated,3180 (? (i32 0), (i32 0), (i32 1)), MVE_VSHL_by_vecu8, null_frag>;3181 defm : MVE_TwoOpPattern<MVE_v8i16, ARMvshlu, int_arm_mve_vshl_vector_predicated,3182 (? (i32 0), (i32 0), (i32 1)), MVE_VSHL_by_vecu16, null_frag>;3183 defm : MVE_TwoOpPattern<MVE_v4i32, ARMvshlu, int_arm_mve_vshl_vector_predicated,3184 (? (i32 0), (i32 0), (i32 1)), MVE_VSHL_by_vecu32, null_frag>;3185 defm : MVE_TwoOpPattern<MVE_v16i8, ARMvshls, int_arm_mve_vshl_vector_predicated,3186 (? (i32 0), (i32 0), (i32 0)), MVE_VSHL_by_vecs8, null_frag>;3187 defm : MVE_TwoOpPattern<MVE_v8i16, ARMvshls, int_arm_mve_vshl_vector_predicated,3188 (? (i32 0), (i32 0), (i32 0)), MVE_VSHL_by_vecs16, null_frag>;3189 defm : MVE_TwoOpPattern<MVE_v4i32, ARMvshls, int_arm_mve_vshl_vector_predicated,3190 (? (i32 0), (i32 0), (i32 0)), MVE_VSHL_by_vecs32, null_frag>;3191}3192 3193class MVE_shift_with_imm<string iname, string suffix, dag oops, dag iops,3194 string ops, vpred_ops vpred, string cstr,3195 bits<2> vecsize, list<dag> pattern=[]>3196 : MVE_p<oops, iops, NoItinerary, iname, suffix, ops, vpred, cstr, vecsize, pattern> {3197 bits<4> Qd;3198 bits<4> Qm;3199 3200 let Inst{23} = 0b1;3201 let Inst{22} = Qd{3};3202 let Inst{15-13} = Qd{2-0};3203 let Inst{12-11} = 0b00;3204 let Inst{7-6} = 0b01;3205 let Inst{5} = Qm{3};3206 let Inst{4} = 0b1;3207 let Inst{3-1} = Qm{2-0};3208 let Inst{0} = 0b0;3209 let validForTailPredication = 1;3210 3211 // For the MVE_shift_imm_patterns multiclass to refer to3212 MVEVectorVTInfo VTI;3213 Operand immediateType;3214 Intrinsic unpred_int;3215 Intrinsic pred_int;3216 dag unsignedFlag = (?);3217}3218 3219class MVE_VSxI_imm<string iname, string suffix, bit bit_8, Operand immType, bits<2> vecsize>3220 : MVE_shift_with_imm<iname, suffix, (outs MQPR:$Qd),3221 (ins MQPR:$Qd_src, MQPR:$Qm, immType:$imm),3222 "$Qd, $Qm, $imm", vpred_n, "$Qd = $Qd_src", vecsize> {3223 bits<6> imm;3224 let Inst{28} = 0b1;3225 let Inst{25-24} = 0b11;3226 let Inst{21-16} = imm;3227 let Inst{10-9} = 0b10;3228 let Inst{8} = bit_8;3229 let validForTailPredication = 1;3230 3231 Operand immediateType = immType;3232}3233 3234def MVE_VSRIimm8 : MVE_VSxI_imm<"vsri", "8", 0b0, shr_imm8, 0b00> {3235 let Inst{21-19} = 0b001;3236}3237 3238def MVE_VSRIimm16 : MVE_VSxI_imm<"vsri", "16", 0b0, shr_imm16, 0b01> {3239 let Inst{21-20} = 0b01;3240}3241 3242def MVE_VSRIimm32 : MVE_VSxI_imm<"vsri", "32", 0b0, shr_imm32, 0b10> {3243 let Inst{21} = 0b1;3244}3245 3246def MVE_VSLIimm8 : MVE_VSxI_imm<"vsli", "8", 0b1, imm0_7, 0b00> {3247 let Inst{21-19} = 0b001;3248}3249 3250def MVE_VSLIimm16 : MVE_VSxI_imm<"vsli", "16", 0b1, imm0_15, 0b01> {3251 let Inst{21-20} = 0b01;3252}3253 3254def MVE_VSLIimm32 : MVE_VSxI_imm<"vsli", "32", 0b1,imm0_31, 0b10> {3255 let Inst{21} = 0b1;3256}3257 3258multiclass MVE_VSxI_patterns<MVE_VSxI_imm inst, string name,3259 MVEVectorVTInfo VTI> {3260 defvar inparams = (? (VTI.Vec MQPR:$QdSrc), (VTI.Vec MQPR:$Qm),3261 (inst.immediateType:$imm));3262 defvar outparams = (inst (VTI.Vec MQPR:$QdSrc), (VTI.Vec MQPR:$Qm),3263 (inst.immediateType:$imm));3264 defvar unpred_int = !cast<Intrinsic>("int_arm_mve_" # name);3265 defvar pred_int = !cast<Intrinsic>("int_arm_mve_" # name # "_predicated");3266 3267 def : Pat<(VTI.Vec !setdagop(inparams, unpred_int)),3268 (VTI.Vec outparams)>;3269 def : Pat<(VTI.Vec !con(inparams, (pred_int (VTI.Pred VCCR:$pred)))),3270 (VTI.Vec !con(outparams, (? ARMVCCThen, VCCR:$pred, zero_reg)))>;3271}3272 3273defm : MVE_VSxI_patterns<MVE_VSLIimm8, "vsli", MVE_v16i8>;3274defm : MVE_VSxI_patterns<MVE_VSLIimm16, "vsli", MVE_v8i16>;3275defm : MVE_VSxI_patterns<MVE_VSLIimm32, "vsli", MVE_v4i32>;3276defm : MVE_VSxI_patterns<MVE_VSRIimm8, "vsri", MVE_v16i8>;3277defm : MVE_VSxI_patterns<MVE_VSRIimm16, "vsri", MVE_v8i16>;3278defm : MVE_VSxI_patterns<MVE_VSRIimm32, "vsri", MVE_v4i32>;3279 3280class MVE_VQSHL_imm<MVEVectorVTInfo VTI_, Operand immType>3281 : MVE_shift_with_imm<"vqshl", VTI_.Suffix, (outs MQPR:$Qd),3282 (ins MQPR:$Qm, immType:$imm), "$Qd, $Qm, $imm",3283 vpred_r, "", VTI_.Size> {3284 bits<6> imm;3285 3286 let Inst{28} = VTI_.Unsigned;3287 let Inst{25-24} = 0b11;3288 let Inst{21-16} = imm;3289 let Inst{10-8} = 0b111;3290 3291 let VTI = VTI_;3292 let immediateType = immType;3293 let unsignedFlag = (? (i32 VTI.Unsigned));3294}3295 3296let unpred_int = int_arm_mve_vqshl_imm,3297 pred_int = int_arm_mve_vqshl_imm_predicated in {3298 def MVE_VQSHLimms8 : MVE_VQSHL_imm<MVE_v16s8, imm0_7> {3299 let Inst{21-19} = 0b001;3300 }3301 def MVE_VQSHLimmu8 : MVE_VQSHL_imm<MVE_v16u8, imm0_7> {3302 let Inst{21-19} = 0b001;3303 }3304 3305 def MVE_VQSHLimms16 : MVE_VQSHL_imm<MVE_v8s16, imm0_15> {3306 let Inst{21-20} = 0b01;3307 }3308 def MVE_VQSHLimmu16 : MVE_VQSHL_imm<MVE_v8u16, imm0_15> {3309 let Inst{21-20} = 0b01;3310 }3311 3312 def MVE_VQSHLimms32 : MVE_VQSHL_imm<MVE_v4s32, imm0_31> {3313 let Inst{21} = 0b1;3314 }3315 def MVE_VQSHLimmu32 : MVE_VQSHL_imm<MVE_v4u32, imm0_31> {3316 let Inst{21} = 0b1;3317 }3318}3319 3320class MVE_VQSHLU_imm<MVEVectorVTInfo VTI_, Operand immType>3321 : MVE_shift_with_imm<"vqshlu", VTI_.Suffix, (outs MQPR:$Qd),3322 (ins MQPR:$Qm, immType:$imm), "$Qd, $Qm, $imm",3323 vpred_r, "", VTI_.Size> {3324 bits<6> imm;3325 3326 let Inst{28} = 0b1;3327 let Inst{25-24} = 0b11;3328 let Inst{21-16} = imm;3329 let Inst{10-8} = 0b110;3330 3331 let VTI = VTI_;3332 let immediateType = immType;3333}3334 3335let unpred_int = int_arm_mve_vqshlu_imm,3336 pred_int = int_arm_mve_vqshlu_imm_predicated in {3337 def MVE_VQSHLU_imms8 : MVE_VQSHLU_imm<MVE_v16s8, imm0_7> {3338 let Inst{21-19} = 0b001;3339 }3340 3341 def MVE_VQSHLU_imms16 : MVE_VQSHLU_imm<MVE_v8s16, imm0_15> {3342 let Inst{21-20} = 0b01;3343 }3344 3345 def MVE_VQSHLU_imms32 : MVE_VQSHLU_imm<MVE_v4s32, imm0_31> {3346 let Inst{21} = 0b1;3347 }3348}3349 3350class MVE_VRSHR_imm<MVEVectorVTInfo VTI_, Operand immType>3351 : MVE_shift_with_imm<"vrshr", VTI_.Suffix, (outs MQPR:$Qd),3352 (ins MQPR:$Qm, immType:$imm), "$Qd, $Qm, $imm",3353 vpred_r, "", VTI_.Size> {3354 bits<6> imm;3355 3356 let Inst{28} = VTI_.Unsigned;3357 let Inst{25-24} = 0b11;3358 let Inst{21-16} = imm;3359 let Inst{10-8} = 0b010;3360 3361 let VTI = VTI_;3362 let immediateType = immType;3363 let unsignedFlag = (? (i32 VTI.Unsigned));3364}3365 3366let unpred_int = int_arm_mve_vrshr_imm,3367 pred_int = int_arm_mve_vrshr_imm_predicated in {3368 def MVE_VRSHR_imms8 : MVE_VRSHR_imm<MVE_v16s8, shr_imm8> {3369 let Inst{21-19} = 0b001;3370 }3371 3372 def MVE_VRSHR_immu8 : MVE_VRSHR_imm<MVE_v16u8, shr_imm8> {3373 let Inst{21-19} = 0b001;3374 }3375 3376 def MVE_VRSHR_imms16 : MVE_VRSHR_imm<MVE_v8s16, shr_imm16> {3377 let Inst{21-20} = 0b01;3378 }3379 3380 def MVE_VRSHR_immu16 : MVE_VRSHR_imm<MVE_v8u16, shr_imm16> {3381 let Inst{21-20} = 0b01;3382 }3383 3384 def MVE_VRSHR_imms32 : MVE_VRSHR_imm<MVE_v4s32, shr_imm32> {3385 let Inst{21} = 0b1;3386 }3387 3388 def MVE_VRSHR_immu32 : MVE_VRSHR_imm<MVE_v4u32, shr_imm32> {3389 let Inst{21} = 0b1;3390 }3391}3392 3393multiclass MVE_shift_imm_patterns<MVE_shift_with_imm inst> {3394 def : Pat<(inst.VTI.Vec !con((inst.unpred_int (inst.VTI.Vec MQPR:$src),3395 inst.immediateType:$imm),3396 inst.unsignedFlag)),3397 (inst.VTI.Vec (inst (inst.VTI.Vec MQPR:$src),3398 inst.immediateType:$imm))>;3399 3400 def : Pat<(inst.VTI.Vec !con((inst.pred_int (inst.VTI.Vec MQPR:$src),3401 inst.immediateType:$imm),3402 inst.unsignedFlag,3403 (? (inst.VTI.Pred VCCR:$mask),3404 (inst.VTI.Vec MQPR:$inactive)))),3405 (inst.VTI.Vec (inst (inst.VTI.Vec MQPR:$src),3406 inst.immediateType:$imm,3407 ARMVCCThen, (inst.VTI.Pred VCCR:$mask), zero_reg,3408 (inst.VTI.Vec MQPR:$inactive)))>;3409}3410 3411defm : MVE_shift_imm_patterns<MVE_VQSHLimms8>;3412defm : MVE_shift_imm_patterns<MVE_VQSHLimmu8>;3413defm : MVE_shift_imm_patterns<MVE_VQSHLimms16>;3414defm : MVE_shift_imm_patterns<MVE_VQSHLimmu16>;3415defm : MVE_shift_imm_patterns<MVE_VQSHLimms32>;3416defm : MVE_shift_imm_patterns<MVE_VQSHLimmu32>;3417defm : MVE_shift_imm_patterns<MVE_VQSHLU_imms8>;3418defm : MVE_shift_imm_patterns<MVE_VQSHLU_imms16>;3419defm : MVE_shift_imm_patterns<MVE_VQSHLU_imms32>;3420defm : MVE_shift_imm_patterns<MVE_VRSHR_imms8>;3421defm : MVE_shift_imm_patterns<MVE_VRSHR_immu8>;3422defm : MVE_shift_imm_patterns<MVE_VRSHR_imms16>;3423defm : MVE_shift_imm_patterns<MVE_VRSHR_immu16>;3424defm : MVE_shift_imm_patterns<MVE_VRSHR_imms32>;3425defm : MVE_shift_imm_patterns<MVE_VRSHR_immu32>;3426 3427class MVE_VSHR_imm<string suffix, dag imm, bits<2> vecsize>3428 : MVE_shift_with_imm<"vshr", suffix, (outs MQPR:$Qd),3429 !con((ins MQPR:$Qm), imm), "$Qd, $Qm, $imm",3430 vpred_r, "", vecsize> {3431 bits<6> imm;3432 3433 let Inst{25-24} = 0b11;3434 let Inst{21-16} = imm;3435 let Inst{10-8} = 0b000;3436}3437 3438def MVE_VSHR_imms8 : MVE_VSHR_imm<"s8", (ins shr_imm8:$imm), 0b00> {3439 let Inst{28} = 0b0;3440 let Inst{21-19} = 0b001;3441}3442 3443def MVE_VSHR_immu8 : MVE_VSHR_imm<"u8", (ins shr_imm8:$imm), 0b00> {3444 let Inst{28} = 0b1;3445 let Inst{21-19} = 0b001;3446}3447 3448def MVE_VSHR_imms16 : MVE_VSHR_imm<"s16", (ins shr_imm16:$imm), 0b01> {3449 let Inst{28} = 0b0;3450 let Inst{21-20} = 0b01;3451}3452 3453def MVE_VSHR_immu16 : MVE_VSHR_imm<"u16", (ins shr_imm16:$imm), 0b01> {3454 let Inst{28} = 0b1;3455 let Inst{21-20} = 0b01;3456}3457 3458def MVE_VSHR_imms32 : MVE_VSHR_imm<"s32", (ins shr_imm32:$imm), 0b10> {3459 let Inst{28} = 0b0;3460 let Inst{21} = 0b1;3461}3462 3463def MVE_VSHR_immu32 : MVE_VSHR_imm<"u32", (ins shr_imm32:$imm), 0b10> {3464 let Inst{28} = 0b1;3465 let Inst{21} = 0b1;3466}3467 3468class MVE_VSHL_imm<string suffix, dag imm, bits<2> vecsize>3469 : MVE_shift_with_imm<"vshl", suffix, (outs MQPR:$Qd),3470 !con((ins MQPR:$Qm), imm), "$Qd, $Qm, $imm",3471 vpred_r, "", vecsize> {3472 bits<6> imm;3473 3474 let Inst{28} = 0b0;3475 let Inst{25-24} = 0b11;3476 let Inst{21-16} = imm;3477 let Inst{10-8} = 0b101;3478}3479 3480def MVE_VSHL_immi8 : MVE_VSHL_imm<"i8", (ins imm0_7:$imm), 0b00> {3481 let Inst{21-19} = 0b001;3482}3483 3484def MVE_VSHL_immi16 : MVE_VSHL_imm<"i16", (ins imm0_15:$imm), 0b01> {3485 let Inst{21-20} = 0b01;3486}3487 3488def MVE_VSHL_immi32 : MVE_VSHL_imm<"i32", (ins imm0_31:$imm), 0b10> {3489 let Inst{21} = 0b1;3490}3491 3492multiclass MVE_immediate_shift_patterns_inner<3493 MVEVectorVTInfo VTI, Operand imm_operand_type, SDNode unpred_op,3494 Intrinsic pred_int, Instruction inst, list<int> unsignedFlag = []> {3495 3496 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$src), imm_operand_type:$imm)),3497 (VTI.Vec (inst (VTI.Vec MQPR:$src), imm_operand_type:$imm))>;3498 3499 def : Pat<(VTI.Vec !con((pred_int (VTI.Vec MQPR:$src), imm_operand_type:$imm),3500 !dag(pred_int, unsignedFlag, ?),3501 (pred_int (VTI.Pred VCCR:$mask),3502 (VTI.Vec MQPR:$inactive)))),3503 (VTI.Vec (inst (VTI.Vec MQPR:$src), imm_operand_type:$imm,3504 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,3505 (VTI.Vec MQPR:$inactive)))>;3506}3507 3508multiclass MVE_immediate_shift_patterns<MVEVectorVTInfo VTI,3509 Operand imm_operand_type> {3510 defm : MVE_immediate_shift_patterns_inner<VTI, imm_operand_type,3511 ARMvshlImm, int_arm_mve_shl_imm_predicated,3512 !cast<Instruction>("MVE_VSHL_immi" # VTI.BitsSuffix)>;3513 defm : MVE_immediate_shift_patterns_inner<VTI, imm_operand_type,3514 ARMvshruImm, int_arm_mve_shr_imm_predicated,3515 !cast<Instruction>("MVE_VSHR_immu" # VTI.BitsSuffix), [1]>;3516 defm : MVE_immediate_shift_patterns_inner<VTI, imm_operand_type,3517 ARMvshrsImm, int_arm_mve_shr_imm_predicated,3518 !cast<Instruction>("MVE_VSHR_imms" # VTI.BitsSuffix), [0]>;3519}3520 3521let Predicates = [HasMVEInt] in {3522 defm : MVE_immediate_shift_patterns<MVE_v16i8, imm0_7>;3523 defm : MVE_immediate_shift_patterns<MVE_v8i16, imm0_15>;3524 defm : MVE_immediate_shift_patterns<MVE_v4i32, imm0_31>;3525}3526 3527// end of mve_shift instructions3528 3529// start of MVE Floating Point instructions3530 3531class MVE_float<string iname, string suffix, dag oops, dag iops, string ops,3532 vpred_ops vpred, string cstr, bits<2> vecsize, list<dag> pattern=[]>3533 : MVE_f<oops, iops, NoItinerary, iname, suffix, ops, vpred, cstr, vecsize, pattern> {3534 bits<4> Qm;3535 3536 let Inst{12} = 0b0;3537 let Inst{6} = 0b1;3538 let Inst{5} = Qm{3};3539 let Inst{3-1} = Qm{2-0};3540 let Inst{0} = 0b0;3541}3542 3543class MVE_VRINT<string rmode, bits<3> op, string suffix, bits<2> size,3544 list<dag> pattern=[]>3545 : MVE_float<!strconcat("vrint", rmode), suffix, (outs MQPR:$Qd),3546 (ins MQPR:$Qm), "$Qd, $Qm", vpred_r, "", size, pattern> {3547 bits<4> Qd;3548 3549 let Inst{28} = 0b1;3550 let Inst{25-23} = 0b111;3551 let Inst{22} = Qd{3};3552 let Inst{21-20} = 0b11;3553 let Inst{19-18} = size;3554 let Inst{17-16} = 0b10;3555 let Inst{15-13} = Qd{2-0};3556 let Inst{11-10} = 0b01;3557 let Inst{9-7} = op{2-0};3558 let Inst{4} = 0b0;3559 let validForTailPredication = 1;3560 3561}3562 3563multiclass MVE_VRINT_m<MVEVectorVTInfo VTI, string suffix, bits<3> opcode,3564 SDPatternOperator unpred_op> {3565 def "": MVE_VRINT<suffix, opcode, VTI.Suffix, VTI.Size>;3566 defvar Inst = !cast<Instruction>(NAME);3567 defvar pred_int = !cast<Intrinsic>("int_arm_mve_vrint"#suffix#"_predicated");3568 3569 let Predicates = [HasMVEFloat] in {3570 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$val))),3571 (VTI.Vec (Inst (VTI.Vec MQPR:$val)))>;3572 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$val), (VTI.Pred VCCR:$pred),3573 (VTI.Vec MQPR:$inactive))),3574 (VTI.Vec (Inst (VTI.Vec MQPR:$val), ARMVCCThen,3575 (VTI.Pred VCCR:$pred), zero_reg, (VTI.Vec MQPR:$inactive)))>;3576 }3577}3578 3579multiclass MVE_VRINT_ops<MVEVectorVTInfo VTI> {3580 defm N : MVE_VRINT_m<VTI, "n", 0b000, froundeven>;3581 defm X : MVE_VRINT_m<VTI, "x", 0b001, frint>;3582 defm A : MVE_VRINT_m<VTI, "a", 0b010, fround>;3583 defm Z : MVE_VRINT_m<VTI, "z", 0b011, ftrunc>;3584 defm M : MVE_VRINT_m<VTI, "m", 0b101, ffloor>;3585 defm P : MVE_VRINT_m<VTI, "p", 0b111, fceil>;3586}3587 3588defm MVE_VRINTf16 : MVE_VRINT_ops<MVE_v8f16>;3589defm MVE_VRINTf32 : MVE_VRINT_ops<MVE_v4f32>;3590 3591class MVEFloatArithNeon<string iname, string suffix, bit size,3592 dag oops, dag iops, string ops,3593 vpred_ops vpred, string cstr, bits<2> vecsize, list<dag> pattern=[]>3594 : MVE_float<iname, suffix, oops, iops, ops, vpred, cstr, vecsize, pattern> {3595 let Inst{20} = size;3596 let Inst{16} = 0b0;3597}3598 3599class MVE_VMUL_fp<string iname, string suffix, bits<2> size, list<dag> pattern=[]>3600 : MVEFloatArithNeon<iname, suffix, size{0}, (outs MQPR:$Qd),3601 (ins MQPR:$Qn, MQPR:$Qm), "$Qd, $Qn, $Qm", vpred_r, "",3602 size, pattern> {3603 bits<4> Qd;3604 bits<4> Qn;3605 3606 let Inst{28} = 0b1;3607 let Inst{25-23} = 0b110;3608 let Inst{22} = Qd{3};3609 let Inst{21} = 0b0;3610 let Inst{19-17} = Qn{2-0};3611 let Inst{15-13} = Qd{2-0};3612 let Inst{12-8} = 0b01101;3613 let Inst{7} = Qn{3};3614 let Inst{4} = 0b1;3615 let validForTailPredication = 1;3616}3617 3618multiclass MVE_VMULT_fp_m<string iname, MVEVectorVTInfo VTI, SDPatternOperator Op,3619 Intrinsic PredInt, SDPatternOperator IdentityVec> {3620 def "" : MVE_VMUL_fp<iname, VTI.Suffix, VTI.Size>;3621 defvar Inst = !cast<Instruction>(NAME);3622 3623 let Predicates = [HasMVEFloat] in {3624 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? ), !cast<Instruction>(NAME), IdentityVec>;3625 }3626}3627 3628multiclass MVE_VMUL_fp_m<MVEVectorVTInfo VTI, SDPatternOperator IdentityVec>3629 : MVE_VMULT_fp_m<"vmul", VTI, vmul, int_arm_mve_mul_predicated, IdentityVec>;3630 3631def ARMimmOneF: PatLeaf<(bitconvert (v4f32 (ARMvmovFPImm (i32 112))))>; // 1.0 float3632def ARMimmOneH: PatLeaf<(bitconvert (v8i16 (ARMvmovImm (i32 2620))))>; // 1.0 half3633 3634defm MVE_VMULf32 : MVE_VMUL_fp_m<MVE_v4f32, ARMimmOneF>;3635defm MVE_VMULf16 : MVE_VMUL_fp_m<MVE_v8f16, ARMimmOneH>;3636 3637class MVE_VCMLA<string suffix, bits<2> size, string cstr>3638 : MVEFloatArithNeon<"vcmla", suffix, size{1}, (outs MQPR:$Qd),3639 (ins MQPR:$Qd_src, MQPR:$Qn, MQPR:$Qm, complexrotateop:$rot),3640 "$Qd, $Qn, $Qm, $rot", vpred_n, "$Qd = $Qd_src"#cstr, size, []> {3641 bits<4> Qd;3642 bits<4> Qn;3643 bits<2> rot;3644 3645 let Inst{28} = 0b1;3646 let Inst{25} = 0b0;3647 let Inst{24-23} = rot;3648 let Inst{22} = Qd{3};3649 let Inst{21} = 0b1;3650 let Inst{19-17} = Qn{2-0};3651 let Inst{15-13} = Qd{2-0};3652 let Inst{12-8} = 0b01000;3653 let Inst{7} = Qn{3};3654 let Inst{4} = 0b0;3655}3656 3657multiclass MVE_VCMLA_m<MVEVectorVTInfo VTI, string cstr=""> {3658 def "" : MVE_VCMLA<VTI.Suffix, VTI.Size, cstr>;3659 defvar Inst = !cast<Instruction>(NAME);3660 3661 let Predicates = [HasMVEFloat] in {3662 def : Pat<(VTI.Vec (int_arm_mve_vcmlaq3663 imm:$rot, (VTI.Vec MQPR:$Qd_src),3664 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),3665 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src),3666 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),3667 imm:$rot))>;3668 3669 def: Pat<(VTI.Vec (fadd_contract MQPR:$Qd_src,3670 (int_arm_mve_vcmulq imm:$rot,3671 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm)))),3672 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src),3673 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),3674 imm:$rot))>;3675 3676 def : Pat<(VTI.Vec (int_arm_mve_vcmlaq_predicated3677 imm:$rot, (VTI.Vec MQPR:$Qd_src),3678 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),3679 (VTI.Pred VCCR:$mask))),3680 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src), (VTI.Vec MQPR:$Qn),3681 (VTI.Vec MQPR:$Qm), imm:$rot,3682 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;3683 }3684}3685 3686defm MVE_VCMLAf16 : MVE_VCMLA_m<MVE_v8f16>;3687defm MVE_VCMLAf32 : MVE_VCMLA_m<MVE_v4f32, ",@earlyclobber $Qd">;3688 3689class MVE_VADDSUBFMA_fp<string iname, string suffix, bits<2> size, bit bit_4,3690 bit bit_8, bit bit_21, dag iops=(ins),3691 vpred_ops vpred=vpred_r, string cstr="",3692 list<dag> pattern=[]>3693 : MVEFloatArithNeon<iname, suffix, size{0}, (outs MQPR:$Qd),3694 !con(iops, (ins MQPR:$Qn, MQPR:$Qm)), "$Qd, $Qn, $Qm",3695 vpred, cstr, size, pattern> {3696 bits<4> Qd;3697 bits<4> Qn;3698 3699 let Inst{28} = 0b0;3700 let Inst{25-23} = 0b110;3701 let Inst{22} = Qd{3};3702 let Inst{21} = bit_21;3703 let Inst{19-17} = Qn{2-0};3704 let Inst{15-13} = Qd{2-0};3705 let Inst{11-9} = 0b110;3706 let Inst{8} = bit_8;3707 let Inst{7} = Qn{3};3708 let Inst{4} = bit_4;3709 let validForTailPredication = 1;3710}3711 3712multiclass MVE_VFMA_fp_multi<string iname, bit fms, MVEVectorVTInfo VTI> {3713 def "" : MVE_VADDSUBFMA_fp<iname, VTI.Suffix, VTI.Size, 0b1, 0b0, fms,3714 (ins MQPR:$Qd_src), vpred_n, "$Qd = $Qd_src">;3715 defvar Inst = !cast<Instruction>(NAME);3716 defvar pred_int = int_arm_mve_fma_predicated;3717 defvar m1 = (VTI.Vec MQPR:$m1);3718 defvar m2 = (VTI.Vec MQPR:$m2);3719 defvar add = (VTI.Vec MQPR:$add);3720 defvar pred = (VTI.Pred VCCR:$pred);3721 3722 let Predicates = [HasMVEFloat] in {3723 if fms then {3724 def : Pat<(VTI.Vec (fma (fneg m1), m2, add)),3725 (Inst $add, $m1, $m2)>;3726 def : Pat<(VTI.Vec (int_arm_mve_fma (fneg m1), m2, add)),3727 (Inst $add, $m1, $m2)>;3728 def : Pat<(VTI.Vec (int_arm_mve_fma m1, (fneg m2), add)),3729 (Inst $add, $m1, $m2)>;3730 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),3731 (VTI.Vec (fma (fneg m1), m2, add)),3732 add)),3733 (Inst $add, $m1, $m2, ARMVCCThen, $pred, zero_reg)>;3734 def : Pat<(VTI.Vec (pred_int (fneg m1), m2, add, pred)),3735 (Inst $add, $m1, $m2, ARMVCCThen, $pred, zero_reg)>;3736 def : Pat<(VTI.Vec (pred_int m1, (fneg m2), add, pred)),3737 (Inst $add, $m1, $m2, ARMVCCThen, $pred, zero_reg)>;3738 } else {3739 def : Pat<(VTI.Vec (fma m1, m2, add)),3740 (Inst $add, $m1, $m2)>;3741 def : Pat<(VTI.Vec (int_arm_mve_fma m1, m2, add)),3742 (Inst $add, $m1, $m2)>;3743 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),3744 (VTI.Vec (fma m1, m2, add)),3745 add)),3746 (Inst $add, $m1, $m2, ARMVCCThen, $pred, zero_reg)>;3747 def : Pat<(VTI.Vec (pred_int m1, m2, add, pred)),3748 (Inst $add, $m1, $m2, ARMVCCThen, $pred, zero_reg)>;3749 }3750 }3751}3752 3753defm MVE_VFMAf32 : MVE_VFMA_fp_multi<"vfma", 0, MVE_v4f32>;3754defm MVE_VFMAf16 : MVE_VFMA_fp_multi<"vfma", 0, MVE_v8f16>;3755defm MVE_VFMSf32 : MVE_VFMA_fp_multi<"vfms", 1, MVE_v4f32>;3756defm MVE_VFMSf16 : MVE_VFMA_fp_multi<"vfms", 1, MVE_v8f16>;3757 3758multiclass MVE_VADDSUB_fp_m<string iname, bit bit_21, MVEVectorVTInfo VTI,3759 SDPatternOperator Op, Intrinsic PredInt, SDPatternOperator IdentityVec> {3760 def "" : MVE_VADDSUBFMA_fp<iname, VTI.Suffix, VTI.Size, 0, 1, bit_21> {3761 let validForTailPredication = 1;3762 }3763 defvar Inst = !cast<Instruction>(NAME);3764 3765 let Predicates = [HasMVEFloat] in {3766 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? ), !cast<Instruction>(NAME), IdentityVec>;3767 }3768}3769 3770multiclass MVE_VADD_fp_m<MVEVectorVTInfo VTI, SDPatternOperator IdentityVec>3771 : MVE_VADDSUB_fp_m<"vadd", 0, VTI, vadd, int_arm_mve_add_predicated, IdentityVec>;3772multiclass MVE_VSUB_fp_m<MVEVectorVTInfo VTI, SDPatternOperator IdentityVec>3773 : MVE_VADDSUB_fp_m<"vsub", 1, VTI, vsub, int_arm_mve_sub_predicated, IdentityVec>;3774 3775def ARMimmMinusZeroF: PatLeaf<(bitconvert (v4i32 (ARMvmovImm (i32 1664))))>; // -0.0 float3776def ARMimmMinusZeroH: PatLeaf<(bitconvert (v8i16 (ARMvmovImm (i32 2688))))>; // -0.0 half3777 3778defm MVE_VADDf32 : MVE_VADD_fp_m<MVE_v4f32, ARMimmMinusZeroF>;3779defm MVE_VADDf16 : MVE_VADD_fp_m<MVE_v8f16, ARMimmMinusZeroH>;3780 3781defm MVE_VSUBf32 : MVE_VSUB_fp_m<MVE_v4f32, ARMimmAllZerosV>;3782defm MVE_VSUBf16 : MVE_VSUB_fp_m<MVE_v8f16, ARMimmAllZerosV>;3783 3784class MVE_VCADD<string suffix, bits<2> size, string cstr="">3785 : MVEFloatArithNeon<"vcadd", suffix, size{1}, (outs MQPR:$Qd),3786 (ins MQPR:$Qn, MQPR:$Qm, complexrotateopodd:$rot),3787 "$Qd, $Qn, $Qm, $rot", vpred_r, cstr, size, []> {3788 bits<4> Qd;3789 bits<4> Qn;3790 bit rot;3791 3792 let Inst{28} = 0b1;3793 let Inst{25} = 0b0;3794 let Inst{24} = rot;3795 let Inst{23} = 0b1;3796 let Inst{22} = Qd{3};3797 let Inst{21} = 0b0;3798 let Inst{19-17} = Qn{2-0};3799 let Inst{15-13} = Qd{2-0};3800 let Inst{12-8} = 0b01000;3801 let Inst{7} = Qn{3};3802 let Inst{4} = 0b0;3803}3804 3805multiclass MVE_VCADD_m<MVEVectorVTInfo VTI, string cstr=""> {3806 def "" : MVE_VCADD<VTI.Suffix, VTI.Size, cstr>;3807 defvar Inst = !cast<Instruction>(NAME);3808 3809 let Predicates = [HasMVEFloat] in {3810 def : Pat<(VTI.Vec (int_arm_mve_vcaddq (i32 1),3811 imm:$rot, (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),3812 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),3813 imm:$rot))>;3814 3815 def : Pat<(VTI.Vec (int_arm_mve_vcaddq_predicated (i32 1),3816 imm:$rot, (VTI.Vec MQPR:$inactive),3817 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),3818 (VTI.Pred VCCR:$mask))),3819 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),3820 imm:$rot, ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,3821 (VTI.Vec MQPR:$inactive)))>;3822 3823 }3824}3825 3826defm MVE_VCADDf16 : MVE_VCADD_m<MVE_v8f16>;3827defm MVE_VCADDf32 : MVE_VCADD_m<MVE_v4f32, "@earlyclobber $Qd">;3828 3829class MVE_VABD_fp<string suffix, bits<2> size>3830 : MVE_float<"vabd", suffix, (outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm),3831 "$Qd, $Qn, $Qm", vpred_r, "", size> {3832 bits<4> Qd;3833 bits<4> Qn;3834 3835 let Inst{28} = 0b1;3836 let Inst{25-23} = 0b110;3837 let Inst{22} = Qd{3};3838 let Inst{21} = 0b1;3839 let Inst{20} = size{0};3840 let Inst{19-17} = Qn{2-0};3841 let Inst{16} = 0b0;3842 let Inst{15-13} = Qd{2-0};3843 let Inst{11-8} = 0b1101;3844 let Inst{7} = Qn{3};3845 let Inst{4} = 0b0;3846 let validForTailPredication = 1;3847}3848 3849multiclass MVE_VABDT_fp_m<MVEVectorVTInfo VTI,3850 Intrinsic unpred_int, Intrinsic pred_int> {3851 def "" : MVE_VABD_fp<VTI.Suffix, VTI.Size>;3852 defvar Inst = !cast<Instruction>(NAME);3853 3854 let Predicates = [HasMVEFloat] in {3855 def : Pat<(VTI.Vec (unpred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),3856 (i32 0))),3857 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;3858 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),3859 (i32 0), (VTI.Pred VCCR:$mask),3860 (VTI.Vec MQPR:$inactive))),3861 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),3862 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,3863 (VTI.Vec MQPR:$inactive)))>;3864 }3865}3866 3867multiclass MVE_VABD_fp_m<MVEVectorVTInfo VTI>3868 : MVE_VABDT_fp_m<VTI, int_arm_mve_vabd, int_arm_mve_abd_predicated>;3869 3870defm MVE_VABDf32 : MVE_VABD_fp_m<MVE_v4f32>;3871defm MVE_VABDf16 : MVE_VABD_fp_m<MVE_v8f16>;3872 3873let Predicates = [HasMVEFloat] in {3874 def : Pat<(v8f16 (fabs (fsub (v8f16 MQPR:$Qm), (v8f16 MQPR:$Qn)))),3875 (MVE_VABDf16 MQPR:$Qm, MQPR:$Qn)>;3876 def : Pat<(v4f32 (fabs (fsub (v4f32 MQPR:$Qm), (v4f32 MQPR:$Qn)))),3877 (MVE_VABDf32 MQPR:$Qm, MQPR:$Qn)>;3878}3879 3880class MVE_VCVT_fix<string suffix, bit fsi, bit U, bit op,3881 Operand imm_operand_type>3882 : MVE_float<"vcvt", suffix,3883 (outs MQPR:$Qd), (ins MQPR:$Qm, imm_operand_type:$imm6),3884 "$Qd, $Qm, $imm6", vpred_r, "", !if(fsi, 0b10, 0b01), []> {3885 bits<4> Qd;3886 bits<6> imm6;3887 3888 let Inst{28} = U;3889 let Inst{25-23} = 0b111;3890 let Inst{22} = Qd{3};3891 let Inst{21} = 0b1;3892 let Inst{19-16} = imm6{3-0};3893 let Inst{15-13} = Qd{2-0};3894 let Inst{11-10} = 0b11;3895 let Inst{9} = fsi;3896 let Inst{8} = op;3897 let Inst{7} = 0b0;3898 let Inst{4} = 0b1;3899 3900 let DecoderMethod = "DecodeMVEVCVTt1fp";3901 let validForTailPredication = 1;3902}3903 3904class MVE_VCVT_imm_asmop<int Bits> : AsmOperandClass {3905 let PredicateMethod = "isImmediate<1," # Bits # ">";3906 let DiagnosticString =3907 "MVE fixed-point immediate operand must be between 1 and " # Bits;3908 let Name = "MVEVcvtImm" # Bits;3909 let RenderMethod = "addImmOperands";3910}3911class MVE_VCVT_imm<int Bits>: Operand<i32> {3912 let ParserMatchClass = MVE_VCVT_imm_asmop<Bits>;3913 let EncoderMethod = "getNEONVcvtImm32OpValue";3914 let DecoderMethod = "DecodeVCVTImmOperand";3915}3916 3917class MVE_VCVT_fix_f32<string suffix, bit U, bit op>3918 : MVE_VCVT_fix<suffix, 0b1, U, op, MVE_VCVT_imm<32>> {3919 let Inst{20} = imm6{4};3920}3921class MVE_VCVT_fix_f16<string suffix, bit U, bit op>3922 : MVE_VCVT_fix<suffix, 0b0, U, op, MVE_VCVT_imm<16>> {3923 let Inst{20} = 0b1;3924}3925 3926multiclass MVE_VCVT_fix_patterns<Instruction Inst, bit U, MVEVectorVTInfo DestVTI,3927 MVEVectorVTInfo SrcVTI> {3928 let Predicates = [HasMVEFloat] in {3929 def : Pat<(DestVTI.Vec (int_arm_mve_vcvt_fix3930 (i32 U), (SrcVTI.Vec MQPR:$Qm), imm:$scale)),3931 (DestVTI.Vec (Inst (SrcVTI.Vec MQPR:$Qm), imm:$scale))>;3932 def : Pat<(DestVTI.Vec (int_arm_mve_vcvt_fix_predicated (i32 U),3933 (DestVTI.Vec MQPR:$inactive),3934 (SrcVTI.Vec MQPR:$Qm),3935 imm:$scale,3936 (DestVTI.Pred VCCR:$mask))),3937 (DestVTI.Vec (Inst (SrcVTI.Vec MQPR:$Qm), imm:$scale,3938 ARMVCCThen, (DestVTI.Pred VCCR:$mask), zero_reg,3939 (DestVTI.Vec MQPR:$inactive)))>;3940 }3941}3942 3943multiclass MVE_VCVT_fix_f32_m<bit U, bit op,3944 MVEVectorVTInfo DestVTI, MVEVectorVTInfo SrcVTI> {3945 def "" : MVE_VCVT_fix_f32<DestVTI.Suffix#"."#SrcVTI.Suffix, U, op>;3946 defm : MVE_VCVT_fix_patterns<!cast<Instruction>(NAME), U, DestVTI, SrcVTI>;3947}3948 3949multiclass MVE_VCVT_fix_f16_m<bit U, bit op,3950 MVEVectorVTInfo DestVTI, MVEVectorVTInfo SrcVTI> {3951 def "" : MVE_VCVT_fix_f16<DestVTI.Suffix#"."#SrcVTI.Suffix, U, op>;3952 defm : MVE_VCVT_fix_patterns<!cast<Instruction>(NAME), U, DestVTI, SrcVTI>;3953}3954 3955defm MVE_VCVTf16s16_fix : MVE_VCVT_fix_f16_m<0b0, 0b0, MVE_v8f16, MVE_v8s16>;3956defm MVE_VCVTs16f16_fix : MVE_VCVT_fix_f16_m<0b0, 0b1, MVE_v8s16, MVE_v8f16>;3957defm MVE_VCVTf16u16_fix : MVE_VCVT_fix_f16_m<0b1, 0b0, MVE_v8f16, MVE_v8u16>;3958defm MVE_VCVTu16f16_fix : MVE_VCVT_fix_f16_m<0b1, 0b1, MVE_v8u16, MVE_v8f16>;3959defm MVE_VCVTf32s32_fix : MVE_VCVT_fix_f32_m<0b0, 0b0, MVE_v4f32, MVE_v4s32>;3960defm MVE_VCVTs32f32_fix : MVE_VCVT_fix_f32_m<0b0, 0b1, MVE_v4s32, MVE_v4f32>;3961defm MVE_VCVTf32u32_fix : MVE_VCVT_fix_f32_m<0b1, 0b0, MVE_v4f32, MVE_v4u32>;3962defm MVE_VCVTu32f32_fix : MVE_VCVT_fix_f32_m<0b1, 0b1, MVE_v4u32, MVE_v4f32>;3963 3964class MVE_VCVT_fp_int_anpm<string suffix, bits<2> size, bit op, string anpm,3965 bits<2> rm, list<dag> pattern=[]>3966 : MVE_float<!strconcat("vcvt", anpm), suffix, (outs MQPR:$Qd),3967 (ins MQPR:$Qm), "$Qd, $Qm", vpred_r, "", size, pattern> {3968 bits<4> Qd;3969 3970 let Inst{28} = 0b1;3971 let Inst{25-23} = 0b111;3972 let Inst{22} = Qd{3};3973 let Inst{21-20} = 0b11;3974 let Inst{19-18} = size;3975 let Inst{17-16} = 0b11;3976 let Inst{15-13} = Qd{2-0};3977 let Inst{12-10} = 0b000;3978 let Inst{9-8} = rm;3979 let Inst{7} = op;3980 let Inst{4} = 0b0;3981 let validForTailPredication = 1;3982}3983 3984multiclass MVE_VCVT_fp_int_anpm_inner<MVEVectorVTInfo Int, MVEVectorVTInfo Flt,3985 string anpm, bits<2> rm> {3986 def "": MVE_VCVT_fp_int_anpm<Int.Suffix # "." # Flt.Suffix, Int.Size,3987 Int.Unsigned, anpm, rm>;3988 3989 defvar Inst = !cast<Instruction>(NAME);3990 defvar IntrBaseName = "int_arm_mve_vcvt" # anpm;3991 defvar UnpredIntr = !cast<Intrinsic>(IntrBaseName);3992 defvar PredIntr = !cast<Intrinsic>(IntrBaseName # "_predicated");3993 3994 let Predicates = [HasMVEFloat] in {3995 def : Pat<(Int.Vec (UnpredIntr (i32 Int.Unsigned), (Flt.Vec MQPR:$in))),3996 (Int.Vec (Inst (Flt.Vec MQPR:$in)))>;3997 3998 def : Pat<(Int.Vec (PredIntr (i32 Int.Unsigned), (Int.Vec MQPR:$inactive),3999 (Flt.Vec MQPR:$in), (Flt.Pred VCCR:$pred))),4000 (Int.Vec (Inst (Flt.Vec MQPR:$in), ARMVCCThen,4001 (Flt.Pred VCCR:$pred), zero_reg, (Int.Vec MQPR:$inactive)))>;4002 }4003}4004 4005multiclass MVE_VCVT_fp_int_anpm_outer<MVEVectorVTInfo Int,4006 MVEVectorVTInfo Flt> {4007 defm a : MVE_VCVT_fp_int_anpm_inner<Int, Flt, "a", 0b00>;4008 defm n : MVE_VCVT_fp_int_anpm_inner<Int, Flt, "n", 0b01>;4009 defm p : MVE_VCVT_fp_int_anpm_inner<Int, Flt, "p", 0b10>;4010 defm m : MVE_VCVT_fp_int_anpm_inner<Int, Flt, "m", 0b11>;4011}4012 4013// This defines instructions such as MVE_VCVTu16f16a, with an explicit4014// rounding-mode suffix on the mnemonic. The class below will define4015// the bare MVE_VCVTu16f16 (with implied rounding toward zero).4016defm MVE_VCVTs16f16 : MVE_VCVT_fp_int_anpm_outer<MVE_v8s16, MVE_v8f16>;4017defm MVE_VCVTu16f16 : MVE_VCVT_fp_int_anpm_outer<MVE_v8u16, MVE_v8f16>;4018defm MVE_VCVTs32f32 : MVE_VCVT_fp_int_anpm_outer<MVE_v4s32, MVE_v4f32>;4019defm MVE_VCVTu32f32 : MVE_VCVT_fp_int_anpm_outer<MVE_v4u32, MVE_v4f32>;4020 4021class MVE_VCVT_fp_int<string suffix, bits<2> size, bit toint, bit unsigned,4022 list<dag> pattern=[]>4023 : MVE_float<"vcvt", suffix, (outs MQPR:$Qd),4024 (ins MQPR:$Qm), "$Qd, $Qm", vpred_r, "", size, pattern> {4025 bits<4> Qd;4026 4027 let Inst{28} = 0b1;4028 let Inst{25-23} = 0b111;4029 let Inst{22} = Qd{3};4030 let Inst{21-20} = 0b11;4031 let Inst{19-18} = size;4032 let Inst{17-16} = 0b11;4033 let Inst{15-13} = Qd{2-0};4034 let Inst{12-9} = 0b0011;4035 let Inst{8} = toint;4036 let Inst{7} = unsigned;4037 let Inst{4} = 0b0;4038 let validForTailPredication = 1;4039}4040 4041multiclass MVE_VCVT_fp_int_m<MVEVectorVTInfo Dest, MVEVectorVTInfo Src,4042 SDNode unpred_op> {4043 defvar Unsigned = !or(!eq(Dest.SuffixLetter,"u"), !eq(Src.SuffixLetter,"u"));4044 defvar ToInt = !eq(Src.SuffixLetter,"f");4045 4046 def "" : MVE_VCVT_fp_int<Dest.Suffix # "." # Src.Suffix, Dest.Size,4047 ToInt, Unsigned>;4048 defvar Inst = !cast<Instruction>(NAME);4049 4050 let Predicates = [HasMVEFloat] in {4051 def : Pat<(Dest.Vec (unpred_op (Src.Vec MQPR:$src))),4052 (Dest.Vec (Inst (Src.Vec MQPR:$src)))>;4053 def : Pat<(Dest.Vec (int_arm_mve_vcvt_fp_int_predicated4054 (Src.Vec MQPR:$src), (i32 Unsigned),4055 (Src.Pred VCCR:$mask), (Dest.Vec MQPR:$inactive))),4056 (Dest.Vec (Inst (Src.Vec MQPR:$src), ARMVCCThen,4057 (Src.Pred VCCR:$mask), zero_reg,4058 (Dest.Vec MQPR:$inactive)))>;4059 }4060}4061// The unsuffixed VCVT for float->int implicitly rounds toward zero,4062// which I reflect here in the llvm instruction names4063defm MVE_VCVTs16f16z : MVE_VCVT_fp_int_m<MVE_v8s16, MVE_v8f16, fp_to_sint>;4064defm MVE_VCVTu16f16z : MVE_VCVT_fp_int_m<MVE_v8u16, MVE_v8f16, fp_to_uint>;4065defm MVE_VCVTs32f32z : MVE_VCVT_fp_int_m<MVE_v4s32, MVE_v4f32, fp_to_sint>;4066defm MVE_VCVTu32f32z : MVE_VCVT_fp_int_m<MVE_v4u32, MVE_v4f32, fp_to_uint>;4067// Whereas VCVT for int->float rounds to nearest4068defm MVE_VCVTf16s16n : MVE_VCVT_fp_int_m<MVE_v8f16, MVE_v8s16, sint_to_fp>;4069defm MVE_VCVTf16u16n : MVE_VCVT_fp_int_m<MVE_v8f16, MVE_v8u16, uint_to_fp>;4070defm MVE_VCVTf32s32n : MVE_VCVT_fp_int_m<MVE_v4f32, MVE_v4s32, sint_to_fp>;4071defm MVE_VCVTf32u32n : MVE_VCVT_fp_int_m<MVE_v4f32, MVE_v4u32, uint_to_fp>;4072 4073let Predicates = [HasMVEFloat] in {4074 def : Pat<(v4i32 (fp_to_sint_sat v4f32:$src, i32)),4075 (MVE_VCVTs32f32z v4f32:$src)>;4076 def : Pat<(v4i32 (fp_to_uint_sat v4f32:$src, i32)),4077 (MVE_VCVTu32f32z v4f32:$src)>;4078 def : Pat<(v8i16 (fp_to_sint_sat v8f16:$src, i16)),4079 (MVE_VCVTs16f16z v8f16:$src)>;4080 def : Pat<(v8i16 (fp_to_uint_sat v8f16:$src, i16)),4081 (MVE_VCVTu16f16z v8f16:$src)>;4082}4083 4084class MVE_VABSNEG_fp<string iname, string suffix, bits<2> size, bit negate,4085 list<dag> pattern=[]>4086 : MVE_float<iname, suffix, (outs MQPR:$Qd),4087 (ins MQPR:$Qm), "$Qd, $Qm", vpred_r, "", size, pattern> {4088 bits<4> Qd;4089 4090 let Inst{28} = 0b1;4091 let Inst{25-23} = 0b111;4092 let Inst{22} = Qd{3};4093 let Inst{21-20} = 0b11;4094 let Inst{19-18} = size;4095 let Inst{17-16} = 0b01;4096 let Inst{15-13} = Qd{2-0};4097 let Inst{11-8} = 0b0111;4098 let Inst{7} = negate;4099 let Inst{4} = 0b0;4100 let validForTailPredication = 1;4101}4102 4103multiclass MVE_VABSNEG_fp_m<string iname, SDNode unpred_op, Intrinsic pred_int,4104 MVEVectorVTInfo VTI, bit opcode> {4105 def "" : MVE_VABSNEG_fp<iname, VTI.Suffix, VTI.Size, opcode>;4106 defvar Inst = !cast<Instruction>(NAME);4107 4108 let Predicates = [HasMVEInt] in {4109 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$v))),4110 (VTI.Vec (Inst $v))>;4111 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$v), (VTI.Pred VCCR:$mask),4112 (VTI.Vec MQPR:$inactive))),4113 (VTI.Vec (Inst $v, ARMVCCThen, $mask, zero_reg, $inactive))>;4114 }4115}4116 4117defm MVE_VABSf16 : MVE_VABSNEG_fp_m<"vabs", fabs, int_arm_mve_abs_predicated,4118 MVE_v8f16, 0>;4119defm MVE_VABSf32 : MVE_VABSNEG_fp_m<"vabs", fabs, int_arm_mve_abs_predicated,4120 MVE_v4f32, 0>;4121defm MVE_VNEGf16 : MVE_VABSNEG_fp_m<"vneg", fneg, int_arm_mve_neg_predicated,4122 MVE_v8f16, 1>;4123defm MVE_VNEGf32 : MVE_VABSNEG_fp_m<"vneg", fneg, int_arm_mve_neg_predicated,4124 MVE_v4f32, 1>;4125 4126class MVE_VMAXMINNMA<string iname, string suffix, bits<2> size, bit bit_12,4127 list<dag> pattern=[]>4128 : MVE_f<(outs MQPR:$Qd), (ins MQPR:$Qd_src, MQPR:$Qm),4129 NoItinerary, iname, suffix, "$Qd, $Qm", vpred_n, "$Qd = $Qd_src",4130 size, pattern> {4131 bits<4> Qd;4132 bits<4> Qm;4133 4134 let Inst{28} = size{0};4135 let Inst{25-23} = 0b100;4136 let Inst{22} = Qd{3};4137 let Inst{21-16} = 0b111111;4138 let Inst{15-13} = Qd{2-0};4139 let Inst{12} = bit_12;4140 let Inst{11-6} = 0b111010;4141 let Inst{5} = Qm{3};4142 let Inst{4} = 0b0;4143 let Inst{3-1} = Qm{2-0};4144 let Inst{0} = 0b1;4145 4146 let isCommutable = 1;4147 let validForTailPredication = 1;4148}4149 4150multiclass MVE_VMAXMINNMA_m<string iname, MVEVectorVTInfo VTI,4151 SDNode unpred_op, Intrinsic pred_int,4152 bit bit_12> {4153 def "" : MVE_VMAXMINNMA<iname, VTI.Suffix, VTI.Size, bit_12>;4154 defvar Inst = !cast<Instruction>(NAME);4155 4156 let Predicates = [HasMVEInt] in {4157 // Unpredicated v(max|min)nma4158 def : Pat<(VTI.Vec (unpred_op (fabs (VTI.Vec MQPR:$Qd)),4159 (fabs (VTI.Vec MQPR:$Qm)))),4160 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm)))>;4161 4162 // Predicated v(max|min)nma4163 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm),4164 (VTI.Pred VCCR:$mask))),4165 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm),4166 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;4167 }4168}4169 4170multiclass MVE_VMAXNMA<MVEVectorVTInfo VTI, bit bit_12>4171 : MVE_VMAXMINNMA_m<"vmaxnma", VTI, fmaxnum, int_arm_mve_vmaxnma_predicated, bit_12>;4172 4173defm MVE_VMAXNMAf32 : MVE_VMAXNMA<MVE_v4f32, 0b0>;4174defm MVE_VMAXNMAf16 : MVE_VMAXNMA<MVE_v8f16, 0b0>;4175 4176multiclass MVE_VMINNMA<MVEVectorVTInfo VTI, bit bit_12>4177 : MVE_VMAXMINNMA_m<"vminnma", VTI, fminnum, int_arm_mve_vminnma_predicated, bit_12>;4178 4179defm MVE_VMINNMAf32 : MVE_VMINNMA<MVE_v4f32, 0b1>;4180defm MVE_VMINNMAf16 : MVE_VMINNMA<MVE_v8f16, 0b1>;4181 4182// end of MVE Floating Point instructions4183 4184// start of MVE compares4185 4186class MVE_VCMPqq<string suffix, bit bit_28, bits<2> bits_21_20,4187 VCMPPredicateOperand predtype, bits<2> vecsize, list<dag> pattern=[]>4188 : MVE_p<(outs VCCR:$P0), (ins MQPR:$Qn, MQPR:$Qm, predtype:$fc),4189 NoItinerary, "vcmp", suffix, "$fc, $Qn, $Qm", vpred_n, "", vecsize, pattern> {4190 // Base class for comparing two vector registers4191 bits<3> fc;4192 bits<4> Qn;4193 bits<4> Qm;4194 4195 let Inst{28} = bit_28;4196 let Inst{25-22} = 0b1000;4197 let Inst{21-20} = bits_21_20;4198 let Inst{19-17} = Qn{2-0};4199 let Inst{16-13} = 0b1000;4200 let Inst{12} = fc{2};4201 let Inst{11-8} = 0b1111;4202 let Inst{7} = fc{0};4203 let Inst{6} = 0b0;4204 let Inst{5} = Qm{3};4205 let Inst{4} = 0b0;4206 let Inst{3-1} = Qm{2-0};4207 let Inst{0} = fc{1};4208 4209 let Constraints = "";4210 4211 // We need a custom decoder method for these instructions because of4212 // the output VCCR operand, which isn't encoded in the instruction4213 // bits anywhere (there is only one choice for it) but has to be4214 // included in the MC operands so that codegen will be able to track4215 // its data flow between instructions, spill/reload it when4216 // necessary, etc. There seems to be no way to get the Tablegen4217 // decoder to emit an operand that isn't affected by any instruction4218 // bit.4219 let DecoderMethod = "DecodeMVEVCMP<false," # predtype.DecoderMethod # ">";4220 let validForTailPredication = 1;4221}4222 4223class MVE_VCMPqqf<string suffix, bit size>4224 : MVE_VCMPqq<suffix, size, 0b11, pred_basic_fp, !if(size, 0b01, 0b10)> {4225 let Predicates = [HasMVEFloat];4226}4227 4228class MVE_VCMPqqi<string suffix, bits<2> size>4229 : MVE_VCMPqq<suffix, 0b1, size, pred_basic_i, size> {4230 let Inst{12} = 0b0;4231 let Inst{0} = 0b0;4232}4233 4234class MVE_VCMPqqu<string suffix, bits<2> size>4235 : MVE_VCMPqq<suffix, 0b1, size, pred_basic_u, size> {4236 let Inst{12} = 0b0;4237 let Inst{0} = 0b1;4238}4239 4240class MVE_VCMPqqs<string suffix, bits<2> size>4241 : MVE_VCMPqq<suffix, 0b1, size, pred_basic_s, size> {4242 let Inst{12} = 0b1;4243}4244 4245def MVE_VCMPf32 : MVE_VCMPqqf<"f32", 0b0>;4246def MVE_VCMPf16 : MVE_VCMPqqf<"f16", 0b1>;4247 4248def MVE_VCMPi8 : MVE_VCMPqqi<"i8", 0b00>;4249def MVE_VCMPi16 : MVE_VCMPqqi<"i16", 0b01>;4250def MVE_VCMPi32 : MVE_VCMPqqi<"i32", 0b10>;4251 4252def MVE_VCMPu8 : MVE_VCMPqqu<"u8", 0b00>;4253def MVE_VCMPu16 : MVE_VCMPqqu<"u16", 0b01>;4254def MVE_VCMPu32 : MVE_VCMPqqu<"u32", 0b10>;4255 4256def MVE_VCMPs8 : MVE_VCMPqqs<"s8", 0b00>;4257def MVE_VCMPs16 : MVE_VCMPqqs<"s16", 0b01>;4258def MVE_VCMPs32 : MVE_VCMPqqs<"s32", 0b10>;4259 4260class MVE_VCMPqr<string suffix, bit bit_28, bits<2> bits_21_20,4261 VCMPPredicateOperand predtype, bits<2> vecsize, list<dag> pattern=[]>4262 : MVE_p<(outs VCCR:$P0), (ins MQPR:$Qn, GPRwithZR:$Rm, predtype:$fc),4263 NoItinerary, "vcmp", suffix, "$fc, $Qn, $Rm", vpred_n, "", vecsize, pattern> {4264 // Base class for comparing a vector register with a scalar4265 bits<3> fc;4266 bits<4> Qn;4267 bits<4> Rm;4268 4269 let Inst{28} = bit_28;4270 let Inst{25-22} = 0b1000;4271 let Inst{21-20} = bits_21_20;4272 let Inst{19-17} = Qn{2-0};4273 let Inst{16-13} = 0b1000;4274 let Inst{12} = fc{2};4275 let Inst{11-8} = 0b1111;4276 let Inst{7} = fc{0};4277 let Inst{6} = 0b1;4278 let Inst{5} = fc{1};4279 let Inst{4} = 0b0;4280 let Inst{3-0} = Rm{3-0};4281 4282 let Constraints = "";4283 // Custom decoder method, for the same reason as MVE_VCMPqq4284 let DecoderMethod = "DecodeMVEVCMP<true," # predtype.DecoderMethod # ">";4285 let validForTailPredication = 1;4286}4287 4288class MVE_VCMPqrf<string suffix, bit size>4289 : MVE_VCMPqr<suffix, size, 0b11, pred_basic_fp, !if(size, 0b01, 0b10)> {4290 let Predicates = [HasMVEFloat];4291}4292 4293class MVE_VCMPqri<string suffix, bits<2> size>4294 : MVE_VCMPqr<suffix, 0b1, size, pred_basic_i, size> {4295 let Inst{12} = 0b0;4296 let Inst{5} = 0b0;4297}4298 4299class MVE_VCMPqru<string suffix, bits<2> size>4300 : MVE_VCMPqr<suffix, 0b1, size, pred_basic_u, size> {4301 let Inst{12} = 0b0;4302 let Inst{5} = 0b1;4303}4304 4305class MVE_VCMPqrs<string suffix, bits<2> size>4306 : MVE_VCMPqr<suffix, 0b1, size, pred_basic_s, size> {4307 let Inst{12} = 0b1;4308}4309 4310def MVE_VCMPf32r : MVE_VCMPqrf<"f32", 0b0>;4311def MVE_VCMPf16r : MVE_VCMPqrf<"f16", 0b1>;4312 4313def MVE_VCMPi8r : MVE_VCMPqri<"i8", 0b00>;4314def MVE_VCMPi16r : MVE_VCMPqri<"i16", 0b01>;4315def MVE_VCMPi32r : MVE_VCMPqri<"i32", 0b10>;4316 4317def MVE_VCMPu8r : MVE_VCMPqru<"u8", 0b00>;4318def MVE_VCMPu16r : MVE_VCMPqru<"u16", 0b01>;4319def MVE_VCMPu32r : MVE_VCMPqru<"u32", 0b10>;4320 4321def MVE_VCMPs8r : MVE_VCMPqrs<"s8", 0b00>;4322def MVE_VCMPs16r : MVE_VCMPqrs<"s16", 0b01>;4323def MVE_VCMPs32r : MVE_VCMPqrs<"s32", 0b10>;4324 4325multiclass unpred_vcmp_z<string suffix, PatLeaf fc> {4326 def i8 : Pat<(v16i1 (ARMvcmpz (v16i8 MQPR:$v1), fc)),4327 (v16i1 (!cast<Instruction>("MVE_VCMP"#suffix#"8r") (v16i8 MQPR:$v1), ZR, fc))>;4328 def i16 : Pat<(v8i1 (ARMvcmpz (v8i16 MQPR:$v1), fc)),4329 (v8i1 (!cast<Instruction>("MVE_VCMP"#suffix#"16r") (v8i16 MQPR:$v1), ZR, fc))>;4330 def i32 : Pat<(v4i1 (ARMvcmpz (v4i32 MQPR:$v1), fc)),4331 (v4i1 (!cast<Instruction>("MVE_VCMP"#suffix#"32r") (v4i32 MQPR:$v1), ZR, fc))>;4332 4333 def : Pat<(v16i1 (and (v16i1 VCCR:$p1), (v16i1 (ARMvcmpz (v16i8 MQPR:$v1), fc)))),4334 (v16i1 (!cast<Instruction>("MVE_VCMP"#suffix#"8r") (v16i8 MQPR:$v1), ZR, fc, ARMVCCThen, VCCR:$p1, zero_reg))>;4335 def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (ARMvcmpz (v8i16 MQPR:$v1), fc)))),4336 (v8i1 (!cast<Instruction>("MVE_VCMP"#suffix#"16r") (v8i16 MQPR:$v1), ZR, fc, ARMVCCThen, VCCR:$p1, zero_reg))>;4337 def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (ARMvcmpz (v4i32 MQPR:$v1), fc)))),4338 (v4i1 (!cast<Instruction>("MVE_VCMP"#suffix#"32r") (v4i32 MQPR:$v1), ZR, fc, ARMVCCThen, VCCR:$p1, zero_reg))>;4339}4340 4341multiclass unpred_vcmp_r<string suffix, PatLeaf fc> {4342 def i8 : Pat<(v16i1 (ARMvcmp (v16i8 MQPR:$v1), (v16i8 MQPR:$v2), fc)),4343 (v16i1 (!cast<Instruction>("MVE_VCMP"#suffix#"8") (v16i8 MQPR:$v1), (v16i8 MQPR:$v2), fc))>;4344 def i16 : Pat<(v8i1 (ARMvcmp (v8i16 MQPR:$v1), (v8i16 MQPR:$v2), fc)),4345 (v8i1 (!cast<Instruction>("MVE_VCMP"#suffix#"16") (v8i16 MQPR:$v1), (v8i16 MQPR:$v2), fc))>;4346 def i32 : Pat<(v4i1 (ARMvcmp (v4i32 MQPR:$v1), (v4i32 MQPR:$v2), fc)),4347 (v4i1 (!cast<Instruction>("MVE_VCMP"#suffix#"32") (v4i32 MQPR:$v1), (v4i32 MQPR:$v2), fc))>;4348 4349 def i8r : Pat<(v16i1 (ARMvcmp (v16i8 MQPR:$v1), (v16i8 (ARMvdup rGPR:$v2)), fc)),4350 (v16i1 (!cast<Instruction>("MVE_VCMP"#suffix#"8r") (v16i8 MQPR:$v1), (i32 rGPR:$v2), fc))>;4351 def i16r : Pat<(v8i1 (ARMvcmp (v8i16 MQPR:$v1), (v8i16 (ARMvdup rGPR:$v2)), fc)),4352 (v8i1 (!cast<Instruction>("MVE_VCMP"#suffix#"16r") (v8i16 MQPR:$v1), (i32 rGPR:$v2), fc))>;4353 def i32r : Pat<(v4i1 (ARMvcmp (v4i32 MQPR:$v1), (v4i32 (ARMvdup rGPR:$v2)), fc)),4354 (v4i1 (!cast<Instruction>("MVE_VCMP"#suffix#"32r") (v4i32 MQPR:$v1), (i32 rGPR:$v2), fc))>;4355 4356 def : Pat<(v16i1 (and (v16i1 VCCR:$p1), (v16i1 (ARMvcmp (v16i8 MQPR:$v1), (v16i8 MQPR:$v2), fc)))),4357 (v16i1 (!cast<Instruction>("MVE_VCMP"#suffix#"8") (v16i8 MQPR:$v1), (v16i8 MQPR:$v2), fc, ARMVCCThen, VCCR:$p1, zero_reg))>;4358 def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (ARMvcmp (v8i16 MQPR:$v1), (v8i16 MQPR:$v2), fc)))),4359 (v8i1 (!cast<Instruction>("MVE_VCMP"#suffix#"16") (v8i16 MQPR:$v1), (v8i16 MQPR:$v2), fc, ARMVCCThen, VCCR:$p1, zero_reg))>;4360 def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (ARMvcmp (v4i32 MQPR:$v1), (v4i32 MQPR:$v2), fc)))),4361 (v4i1 (!cast<Instruction>("MVE_VCMP"#suffix#"32") (v4i32 MQPR:$v1), (v4i32 MQPR:$v2), fc, ARMVCCThen, VCCR:$p1, zero_reg))>;4362 4363 def : Pat<(v16i1 (and (v16i1 VCCR:$p1), (v16i1 (ARMvcmp (v16i8 MQPR:$v1), (v16i8 (ARMvdup rGPR:$v2)), fc)))),4364 (v16i1 (!cast<Instruction>("MVE_VCMP"#suffix#"8r") (v16i8 MQPR:$v1), (i32 rGPR:$v2), fc, ARMVCCThen, VCCR:$p1, zero_reg))>;4365 def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (ARMvcmp (v8i16 MQPR:$v1), (v8i16 (ARMvdup rGPR:$v2)), fc)))),4366 (v8i1 (!cast<Instruction>("MVE_VCMP"#suffix#"16r") (v8i16 MQPR:$v1), (i32 rGPR:$v2), fc, ARMVCCThen, VCCR:$p1, zero_reg))>;4367 def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (ARMvcmp (v4i32 MQPR:$v1), (v4i32 (ARMvdup rGPR:$v2)), fc)))),4368 (v4i1 (!cast<Instruction>("MVE_VCMP"#suffix#"32r") (v4i32 MQPR:$v1), (i32 rGPR:$v2), fc, ARMVCCThen, VCCR:$p1, zero_reg))>;4369}4370 4371multiclass unpred_vcmpf_z<PatLeaf fc> {4372 def f16 : Pat<(v8i1 (ARMvcmpz (v8f16 MQPR:$v1), fc)),4373 (v8i1 (MVE_VCMPf16r (v8f16 MQPR:$v1), ZR, fc))>;4374 def f32 : Pat<(v4i1 (ARMvcmpz (v4f32 MQPR:$v1), fc)),4375 (v4i1 (MVE_VCMPf32r (v4f32 MQPR:$v1), ZR, fc))>;4376 4377 def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (ARMvcmpz (v8f16 MQPR:$v1), fc)))),4378 (v8i1 (MVE_VCMPf16r (v8f16 MQPR:$v1), ZR, fc, ARMVCCThen, VCCR:$p1, zero_reg))>;4379 def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (ARMvcmpz (v4f32 MQPR:$v1), fc)))),4380 (v4i1 (MVE_VCMPf32r (v4f32 MQPR:$v1), ZR, fc, ARMVCCThen, VCCR:$p1, zero_reg))>;4381}4382 4383multiclass unpred_vcmpf_r<PatLeaf fc> {4384 def : Pat<(v8i1 (ARMvcmp (v8f16 MQPR:$v1), (v8f16 MQPR:$v2), fc)),4385 (v8i1 (MVE_VCMPf16 (v8f16 MQPR:$v1), (v8f16 MQPR:$v2), fc))>;4386 def : Pat<(v4i1 (ARMvcmp (v4f32 MQPR:$v1), (v4f32 MQPR:$v2), fc)),4387 (v4i1 (MVE_VCMPf32 (v4f32 MQPR:$v1), (v4f32 MQPR:$v2), fc))>;4388 4389 def : Pat<(v8i1 (ARMvcmp (v8f16 MQPR:$v1), (v8f16 (ARMvdup rGPR:$v2)), fc)),4390 (v8i1 (MVE_VCMPf16r (v8f16 MQPR:$v1), (i32 rGPR:$v2), fc))>;4391 def : Pat<(v4i1 (ARMvcmp (v4f32 MQPR:$v1), (v4f32 (ARMvdup rGPR:$v2)), fc)),4392 (v4i1 (MVE_VCMPf32r (v4f32 MQPR:$v1), (i32 rGPR:$v2), fc))>;4393 4394 def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (ARMvcmp (v8f16 MQPR:$v1), (v8f16 MQPR:$v2), fc)))),4395 (v8i1 (MVE_VCMPf16 (v8f16 MQPR:$v1), (v8f16 MQPR:$v2), fc, ARMVCCThen, VCCR:$p1, zero_reg))>;4396 def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (ARMvcmp (v4f32 MQPR:$v1), (v4f32 MQPR:$v2), fc)))),4397 (v4i1 (MVE_VCMPf32 (v4f32 MQPR:$v1), (v4f32 MQPR:$v2), fc, ARMVCCThen, VCCR:$p1, zero_reg))>;4398 4399 def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (ARMvcmp (v8f16 MQPR:$v1), (v8f16 (ARMvdup rGPR:$v2)), fc)))),4400 (v8i1 (MVE_VCMPf16r (v8f16 MQPR:$v1), (i32 rGPR:$v2), fc, ARMVCCThen, VCCR:$p1, zero_reg))>;4401 def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (ARMvcmp (v4f32 MQPR:$v1), (v4f32 (ARMvdup rGPR:$v2)), fc)))),4402 (v4i1 (MVE_VCMPf32r (v4f32 MQPR:$v1), (i32 rGPR:$v2), fc, ARMVCCThen, VCCR:$p1, zero_reg))>;4403}4404 4405let Predicates = [HasMVEInt] in {4406 defm MVE_VCEQZ : unpred_vcmp_z<"i", ARMCCeq>;4407 defm MVE_VCNEZ : unpred_vcmp_z<"i", ARMCCne>;4408 defm MVE_VCGEZ : unpred_vcmp_z<"s", ARMCCge>;4409 defm MVE_VCLTZ : unpred_vcmp_z<"s", ARMCClt>;4410 defm MVE_VCGTZ : unpred_vcmp_z<"s", ARMCCgt>;4411 defm MVE_VCLEZ : unpred_vcmp_z<"s", ARMCCle>;4412 defm MVE_VCGTUZ : unpred_vcmp_z<"u", ARMCChi>;4413 defm MVE_VCGEUZ : unpred_vcmp_z<"u", ARMCChs>;4414 4415 defm MVE_VCEQ : unpred_vcmp_r<"i", ARMCCeq>;4416 defm MVE_VCNE : unpred_vcmp_r<"i", ARMCCne>;4417 defm MVE_VCGE : unpred_vcmp_r<"s", ARMCCge>;4418 defm MVE_VCLT : unpred_vcmp_r<"s", ARMCClt>;4419 defm MVE_VCGT : unpred_vcmp_r<"s", ARMCCgt>;4420 defm MVE_VCLE : unpred_vcmp_r<"s", ARMCCle>;4421 defm MVE_VCGTU : unpred_vcmp_r<"u", ARMCChi>;4422 defm MVE_VCGEU : unpred_vcmp_r<"u", ARMCChs>;4423}4424 4425let Predicates = [HasMVEFloat] in {4426 defm MVE_VFCEQZ : unpred_vcmpf_z<ARMCCeq>;4427 defm MVE_VFCNEZ : unpred_vcmpf_z<ARMCCne>;4428 defm MVE_VFCGEZ : unpred_vcmpf_z<ARMCCge>;4429 defm MVE_VFCLTZ : unpred_vcmpf_z<ARMCClt>;4430 defm MVE_VFCGTZ : unpred_vcmpf_z<ARMCCgt>;4431 defm MVE_VFCLEZ : unpred_vcmpf_z<ARMCCle>;4432 4433 defm MVE_VFCEQ : unpred_vcmpf_r<ARMCCeq>;4434 defm MVE_VFCNE : unpred_vcmpf_r<ARMCCne>;4435 defm MVE_VFCGE : unpred_vcmpf_r<ARMCCge>;4436 defm MVE_VFCLT : unpred_vcmpf_r<ARMCClt>;4437 defm MVE_VFCGT : unpred_vcmpf_r<ARMCCgt>;4438 defm MVE_VFCLE : unpred_vcmpf_r<ARMCCle>;4439}4440 4441 4442// Extra "worst case" and/or/xor patterns, going into and out of GRP4443multiclass two_predops<SDPatternOperator opnode, Instruction insn> {4444 def v16i1 : Pat<(v16i1 (opnode (v16i1 VCCR:$p1), (v16i1 VCCR:$p2))),4445 (v16i1 (COPY_TO_REGCLASS4446 (insn (i32 (COPY_TO_REGCLASS (v16i1 VCCR:$p1), rGPR)),4447 (i32 (COPY_TO_REGCLASS (v16i1 VCCR:$p2), rGPR))),4448 VCCR))>;4449 def v8i1 : Pat<(v8i1 (opnode (v8i1 VCCR:$p1), (v8i1 VCCR:$p2))),4450 (v8i1 (COPY_TO_REGCLASS4451 (insn (i32 (COPY_TO_REGCLASS (v8i1 VCCR:$p1), rGPR)),4452 (i32 (COPY_TO_REGCLASS (v8i1 VCCR:$p2), rGPR))),4453 VCCR))>;4454 def v4i1 : Pat<(v4i1 (opnode (v4i1 VCCR:$p1), (v4i1 VCCR:$p2))),4455 (v4i1 (COPY_TO_REGCLASS4456 (insn (i32 (COPY_TO_REGCLASS (v4i1 VCCR:$p1), rGPR)),4457 (i32 (COPY_TO_REGCLASS (v4i1 VCCR:$p2), rGPR))),4458 VCCR))>;4459 def v2i1 : Pat<(v2i1 (opnode (v2i1 VCCR:$p1), (v2i1 VCCR:$p2))),4460 (v2i1 (COPY_TO_REGCLASS4461 (insn (i32 (COPY_TO_REGCLASS (v2i1 VCCR:$p1), rGPR)),4462 (i32 (COPY_TO_REGCLASS (v2i1 VCCR:$p2), rGPR))),4463 VCCR))>;4464}4465 4466let Predicates = [HasMVEInt] in {4467 defm POR : two_predops<or, t2ORRrr>;4468 defm PAND : two_predops<and, t2ANDrr>;4469 defm PEOR : two_predops<xor, t2EORrr>;4470}4471 4472// Predicate cast for MVE i1 types4473// Occasionally we need to cast between a i32 and a boolean vector, for4474// example when moving between rGPR and VPR.P0 as part of predicate vector4475// shuffles. We also sometimes need to cast between different predicate4476// vector types (v4i1<>v8i1, etc.) also as part of lowering vector shuffles.4477def predicate_cast : SDNode<"ARMISD::PREDICATE_CAST", SDTUnaryOp>;4478 4479def load_align4 : PatFrag<(ops node:$ptr), (load node:$ptr), [{4480 return cast<LoadSDNode>(N)->getAlign() >= 4;4481}]>;4482 4483let Predicates = [HasMVEInt] in {4484 foreach VT = [ v2i1, v4i1, v8i1, v16i1 ] in {4485 def : Pat<(i32 (predicate_cast (VT VCCR:$src))),4486 (i32 (COPY_TO_REGCLASS (VT VCCR:$src), VCCR))>;4487 def : Pat<(VT (predicate_cast (i32 VCCR:$src))),4488 (VT (COPY_TO_REGCLASS (i32 VCCR:$src), VCCR))>;4489 4490 foreach VT2 = [ v2i1, v4i1, v8i1, v16i1 ] in4491 def : Pat<(VT (predicate_cast (VT2 VCCR:$src))),4492 (VT (COPY_TO_REGCLASS (VT2 VCCR:$src), VCCR))>;4493 }4494 4495 // If we happen to be casting from a load we can convert that straight4496 // into a predicate load, so long as the load is of the correct type.4497 foreach VT = [ v2i1, v4i1, v8i1, v16i1 ] in {4498 def : Pat<(VT (predicate_cast (i32 (load_align4 taddrmode_imm7<2>:$addr)))),4499 (VT (VLDR_P0_off taddrmode_imm7<2>:$addr))>;4500 }4501 4502 // Here we match the specific SDNode type 'ARMVectorRegCastImpl'4503 // rather than the more general 'ARMVectorRegCast' which would also4504 // match some bitconverts. If we use the latter in cases where the4505 // input and output types are the same, the bitconvert gets elided4506 // and we end up generating a nonsense match of nothing.4507 4508 foreach VT = [ v16i8, v8i16, v8f16, v4i32, v4f32, v2i64, v2f64 ] in4509 foreach VT2 = [ v16i8, v8i16, v8f16, v4i32, v4f32, v2i64, v2f64 ] in4510 def : Pat<(VT (ARMVectorRegCastImpl (VT2 MQPR:$src))),4511 (VT MQPR:$src)>;4512}4513 4514// end of MVE compares4515 4516// start of MVE_qDest_qSrc4517 4518class MVE_qDest_qSrc<string iname, string suffix, dag oops, dag iops,4519 string ops, vpred_ops vpred, string cstr,4520 bits<2> vecsize, list<dag> pattern=[]>4521 : MVE_p<oops, iops, NoItinerary, iname, suffix,4522 ops, vpred, cstr, vecsize, pattern> {4523 bits<4> Qd;4524 bits<4> Qm;4525 4526 let Inst{25-23} = 0b100;4527 let Inst{22} = Qd{3};4528 let Inst{15-13} = Qd{2-0};4529 let Inst{11-9} = 0b111;4530 let Inst{6} = 0b0;4531 let Inst{5} = Qm{3};4532 let Inst{4} = 0b0;4533 let Inst{3-1} = Qm{2-0};4534}4535 4536class MVE_VQxDMLxDH<string iname, bit exch, bit round, bit subtract,4537 string suffix, bits<2> size, string cstr="",4538 list<dag> pattern=[]>4539 : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),4540 (ins MQPR:$Qd_src, MQPR:$Qn, MQPR:$Qm), "$Qd, $Qn, $Qm",4541 vpred_n, "$Qd = $Qd_src"#cstr, size, pattern> {4542 bits<4> Qn;4543 4544 let Inst{28} = subtract;4545 let Inst{21-20} = size;4546 let Inst{19-17} = Qn{2-0};4547 let Inst{16} = 0b0;4548 let Inst{12} = exch;4549 let Inst{8} = 0b0;4550 let Inst{7} = Qn{3};4551 let Inst{0} = round;4552}4553 4554multiclass MVE_VQxDMLxDH_p<string iname, bit exch, bit round, bit subtract,4555 MVEVectorVTInfo VTI> {4556 def "": MVE_VQxDMLxDH<iname, exch, round, subtract, VTI.Suffix, VTI.Size,4557 !if(!eq(VTI.LaneBits, 32), ",@earlyclobber $Qd", "")>;4558 defvar Inst = !cast<Instruction>(NAME);4559 defvar ConstParams = (? (i32 exch), (i32 round), (i32 subtract));4560 defvar unpred_intr = int_arm_mve_vqdmlad;4561 defvar pred_intr = int_arm_mve_vqdmlad_predicated;4562 4563 def : Pat<(VTI.Vec !con((unpred_intr (VTI.Vec MQPR:$a), (VTI.Vec MQPR:$b),4564 (VTI.Vec MQPR:$c)), ConstParams)),4565 (VTI.Vec (Inst (VTI.Vec MQPR:$a), (VTI.Vec MQPR:$b),4566 (VTI.Vec MQPR:$c)))>;4567 def : Pat<(VTI.Vec !con((pred_intr (VTI.Vec MQPR:$a), (VTI.Vec MQPR:$b),4568 (VTI.Vec MQPR:$c)), ConstParams,4569 (? (VTI.Pred VCCR:$pred)))),4570 (VTI.Vec (Inst (VTI.Vec MQPR:$a), (VTI.Vec MQPR:$b),4571 (VTI.Vec MQPR:$c),4572 ARMVCCThen, (VTI.Pred VCCR:$pred), zero_reg))>;4573}4574 4575multiclass MVE_VQxDMLxDH_multi<string iname, bit exch,4576 bit round, bit subtract> {4577 defm s8 : MVE_VQxDMLxDH_p<iname, exch, round, subtract, MVE_v16s8>;4578 defm s16 : MVE_VQxDMLxDH_p<iname, exch, round, subtract, MVE_v8s16>;4579 defm s32 : MVE_VQxDMLxDH_p<iname, exch, round, subtract, MVE_v4s32>;4580}4581 4582defm MVE_VQDMLADH : MVE_VQxDMLxDH_multi<"vqdmladh", 0b0, 0b0, 0b0>;4583defm MVE_VQDMLADHX : MVE_VQxDMLxDH_multi<"vqdmladhx", 0b1, 0b0, 0b0>;4584defm MVE_VQRDMLADH : MVE_VQxDMLxDH_multi<"vqrdmladh", 0b0, 0b1, 0b0>;4585defm MVE_VQRDMLADHX : MVE_VQxDMLxDH_multi<"vqrdmladhx", 0b1, 0b1, 0b0>;4586defm MVE_VQDMLSDH : MVE_VQxDMLxDH_multi<"vqdmlsdh", 0b0, 0b0, 0b1>;4587defm MVE_VQDMLSDHX : MVE_VQxDMLxDH_multi<"vqdmlsdhx", 0b1, 0b0, 0b1>;4588defm MVE_VQRDMLSDH : MVE_VQxDMLxDH_multi<"vqrdmlsdh", 0b0, 0b1, 0b1>;4589defm MVE_VQRDMLSDHX : MVE_VQxDMLxDH_multi<"vqrdmlsdhx", 0b1, 0b1, 0b1>;4590 4591class MVE_VCMUL<string iname, string suffix, bits<2> size, string cstr="">4592 : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),4593 (ins MQPR:$Qn, MQPR:$Qm, complexrotateop:$rot),4594 "$Qd, $Qn, $Qm, $rot", vpred_r, cstr, size,4595 []> {4596 bits<4> Qn;4597 bits<2> rot;4598 4599 let Inst{28} = size{1};4600 let Inst{21-20} = 0b11;4601 let Inst{19-17} = Qn{2-0};4602 let Inst{16} = 0b0;4603 let Inst{12} = rot{1};4604 let Inst{8} = 0b0;4605 let Inst{7} = Qn{3};4606 let Inst{0} = rot{0};4607 4608 let Predicates = [HasMVEFloat];4609}4610 4611multiclass MVE_VCMUL_m<string iname, MVEVectorVTInfo VTI,4612 string cstr=""> {4613 def "" : MVE_VCMUL<iname, VTI.Suffix, VTI.Size, cstr>;4614 defvar Inst = !cast<Instruction>(NAME);4615 4616 let Predicates = [HasMVEFloat] in {4617 def : Pat<(VTI.Vec (int_arm_mve_vcmulq4618 imm:$rot, (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),4619 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),4620 imm:$rot))>;4621 4622 def : Pat<(VTI.Vec (int_arm_mve_vcmulq_predicated4623 imm:$rot, (VTI.Vec MQPR:$inactive),4624 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),4625 (VTI.Pred VCCR:$mask))),4626 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),4627 imm:$rot, ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,4628 (VTI.Vec MQPR:$inactive)))>;4629 4630 }4631}4632 4633defm MVE_VCMULf16 : MVE_VCMUL_m<"vcmul", MVE_v8f16>;4634defm MVE_VCMULf32 : MVE_VCMUL_m<"vcmul", MVE_v4f32, "@earlyclobber $Qd">;4635 4636class MVE_VMULL<string iname, string suffix, bit bit_28, bits<2> bits_21_20,4637 bit T, string cstr, bits<2> vecsize, list<dag> pattern=[]>4638 : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),4639 (ins MQPR:$Qn, MQPR:$Qm), "$Qd, $Qn, $Qm",4640 vpred_r, cstr, vecsize, pattern> {4641 bits<4> Qd;4642 bits<4> Qn;4643 bits<4> Qm;4644 4645 let Inst{28} = bit_28;4646 let Inst{21-20} = bits_21_20;4647 let Inst{19-17} = Qn{2-0};4648 let Inst{16} = 0b1;4649 let Inst{12} = T;4650 let Inst{8} = 0b0;4651 let Inst{7} = Qn{3};4652 let Inst{0} = 0b0;4653 let validForTailPredication = 1;4654 let doubleWidthResult = 1;4655}4656 4657multiclass MVE_VMULL_m<MVEVectorVTInfo VTI,4658 SDPatternOperator unpred_op, Intrinsic pred_int,4659 bit Top, bits<2> vecsize, string cstr=""> {4660 def "" : MVE_VMULL<"vmull" # !if(Top, "t", "b"), VTI.Suffix, VTI.Unsigned,4661 VTI.Size, Top, cstr, vecsize>;4662 defvar Inst = !cast<Instruction>(NAME);4663 4664 let Predicates = [HasMVEInt] in {4665 defvar uflag = !if(!eq(VTI.SuffixLetter, "p"), (?), (? (i32 VTI.Unsigned)));4666 4667 // Unpredicated multiply4668 def : Pat<(VTI.DblVec !con((unpred_op (VTI.Vec MQPR:$Qm),4669 (VTI.Vec MQPR:$Qn)),4670 uflag, (? (i32 Top)))),4671 (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;4672 4673 // Predicated multiply4674 def : Pat<(VTI.DblVec !con((pred_int (VTI.Vec MQPR:$Qm),4675 (VTI.Vec MQPR:$Qn)),4676 uflag, (? (i32 Top), (VTI.DblPred VCCR:$mask),4677 (VTI.DblVec MQPR:$inactive)))),4678 (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),4679 ARMVCCThen, (VTI.DblPred VCCR:$mask), zero_reg,4680 (VTI.DblVec MQPR:$inactive)))>;4681 }4682}4683 4684// For polynomial multiplies, the size bits take the unused value 0b11, and4685// the unsigned bit switches to encoding the size.4686 4687defm MVE_VMULLBs8 : MVE_VMULL_m<MVE_v16s8, int_arm_mve_vmull,4688 int_arm_mve_mull_int_predicated, 0b0, 0b01>;4689defm MVE_VMULLTs8 : MVE_VMULL_m<MVE_v16s8, int_arm_mve_vmull,4690 int_arm_mve_mull_int_predicated, 0b1, 0b01>;4691defm MVE_VMULLBs16 : MVE_VMULL_m<MVE_v8s16, int_arm_mve_vmull,4692 int_arm_mve_mull_int_predicated, 0b0, 0b10>;4693defm MVE_VMULLTs16 : MVE_VMULL_m<MVE_v8s16, int_arm_mve_vmull,4694 int_arm_mve_mull_int_predicated, 0b1, 0b10>;4695defm MVE_VMULLBs32 : MVE_VMULL_m<MVE_v4s32, int_arm_mve_vmull,4696 int_arm_mve_mull_int_predicated, 0b0, 0b11,4697 "@earlyclobber $Qd">;4698defm MVE_VMULLTs32 : MVE_VMULL_m<MVE_v4s32, int_arm_mve_vmull,4699 int_arm_mve_mull_int_predicated, 0b1, 0b11,4700 "@earlyclobber $Qd">;4701 4702defm MVE_VMULLBu8 : MVE_VMULL_m<MVE_v16u8, int_arm_mve_vmull,4703 int_arm_mve_mull_int_predicated, 0b0, 0b01>;4704defm MVE_VMULLTu8 : MVE_VMULL_m<MVE_v16u8, int_arm_mve_vmull,4705 int_arm_mve_mull_int_predicated, 0b1, 0b01>;4706defm MVE_VMULLBu16 : MVE_VMULL_m<MVE_v8u16, int_arm_mve_vmull,4707 int_arm_mve_mull_int_predicated, 0b0, 0b10>;4708defm MVE_VMULLTu16 : MVE_VMULL_m<MVE_v8u16, int_arm_mve_vmull,4709 int_arm_mve_mull_int_predicated, 0b1, 0b10>;4710defm MVE_VMULLBu32 : MVE_VMULL_m<MVE_v4u32, int_arm_mve_vmull,4711 int_arm_mve_mull_int_predicated, 0b0, 0b11,4712 "@earlyclobber $Qd">;4713defm MVE_VMULLTu32 : MVE_VMULL_m<MVE_v4u32, int_arm_mve_vmull,4714 int_arm_mve_mull_int_predicated, 0b1, 0b11,4715 "@earlyclobber $Qd">;4716 4717defm MVE_VMULLBp8 : MVE_VMULL_m<MVE_v16p8, int_arm_mve_vmull_poly,4718 int_arm_mve_mull_poly_predicated, 0b0, 0b01>;4719defm MVE_VMULLTp8 : MVE_VMULL_m<MVE_v16p8, int_arm_mve_vmull_poly,4720 int_arm_mve_mull_poly_predicated, 0b1, 0b01>;4721defm MVE_VMULLBp16 : MVE_VMULL_m<MVE_v8p16, int_arm_mve_vmull_poly,4722 int_arm_mve_mull_poly_predicated, 0b0, 0b10>;4723defm MVE_VMULLTp16 : MVE_VMULL_m<MVE_v8p16, int_arm_mve_vmull_poly,4724 int_arm_mve_mull_poly_predicated, 0b1, 0b10>;4725 4726let Predicates = [HasMVEInt] in {4727 def : Pat<(v2i64 (ARMvmulls (v4i32 MQPR:$src1), (v4i32 MQPR:$src2))),4728 (MVE_VMULLBs32 MQPR:$src1, MQPR:$src2)>;4729 def : Pat<(v2i64 (ARMvmulls (v4i32 (ARMvrev64 (v4i32 MQPR:$src1))),4730 (v4i32 (ARMvrev64 (v4i32 MQPR:$src2))))),4731 (MVE_VMULLTs32 MQPR:$src1, MQPR:$src2)>;4732 4733 def : Pat<(mul (sext_inreg (v4i32 MQPR:$src1), v4i16),4734 (sext_inreg (v4i32 MQPR:$src2), v4i16)),4735 (MVE_VMULLBs16 MQPR:$src1, MQPR:$src2)>;4736 def : Pat<(mul (sext_inreg (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src1)))), v4i16),4737 (sext_inreg (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src2)))), v4i16)),4738 (MVE_VMULLTs16 MQPR:$src1, MQPR:$src2)>;4739 4740 def : Pat<(mul (sext_inreg (v8i16 MQPR:$src1), v8i8),4741 (sext_inreg (v8i16 MQPR:$src2), v8i8)),4742 (MVE_VMULLBs8 MQPR:$src1, MQPR:$src2)>;4743 def : Pat<(mul (sext_inreg (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src1)))), v8i8),4744 (sext_inreg (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src2)))), v8i8)),4745 (MVE_VMULLTs8 MQPR:$src1, MQPR:$src2)>;4746 4747 def : Pat<(v2i64 (ARMvmullu (v4i32 MQPR:$src1), (v4i32 MQPR:$src2))),4748 (MVE_VMULLBu32 MQPR:$src1, MQPR:$src2)>;4749 def : Pat<(v2i64 (ARMvmullu (v4i32 (ARMvrev64 (v4i32 MQPR:$src1))),4750 (v4i32 (ARMvrev64 (v4i32 MQPR:$src2))))),4751 (MVE_VMULLTu32 MQPR:$src1, MQPR:$src2)>;4752 4753 def : Pat<(mul (and (v4i32 MQPR:$src1), (v4i32 (ARMvmovImm (i32 0xCFF)))),4754 (and (v4i32 MQPR:$src2), (v4i32 (ARMvmovImm (i32 0xCFF))))),4755 (MVE_VMULLBu16 MQPR:$src1, MQPR:$src2)>;4756 def : Pat<(mul (and (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src1)))),4757 (v4i32 (ARMvmovImm (i32 0xCFF)))),4758 (and (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src2)))),4759 (v4i32 (ARMvmovImm (i32 0xCFF))))),4760 (MVE_VMULLTu16 MQPR:$src1, MQPR:$src2)>;4761 4762 def : Pat<(mul (ARMvbicImm (v8i16 MQPR:$src1), (i32 0xAFF)),4763 (ARMvbicImm (v8i16 MQPR:$src2), (i32 0xAFF))),4764 (MVE_VMULLBu8 MQPR:$src1, MQPR:$src2)>;4765 def : Pat<(mul (ARMvbicImm (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src1)))), (i32 0xAFF)),4766 (ARMvbicImm (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src2)))), (i32 0xAFF))),4767 (MVE_VMULLTu8 MQPR:$src1, MQPR:$src2)>;4768}4769 4770class MVE_VxMULH<string iname, string suffix, bit U, bits<2> size, bit round,4771 list<dag> pattern=[]>4772 : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),4773 (ins MQPR:$Qn, MQPR:$Qm), "$Qd, $Qn, $Qm",4774 vpred_r, "", size, pattern> {4775 bits<4> Qn;4776 4777 let Inst{28} = U;4778 let Inst{21-20} = size;4779 let Inst{19-17} = Qn{2-0};4780 let Inst{16} = 0b1;4781 let Inst{12} = round;4782 let Inst{8} = 0b0;4783 let Inst{7} = Qn{3};4784 let Inst{0} = 0b1;4785 let validForTailPredication = 1;4786}4787 4788multiclass MVE_VxMULH_m<string iname, MVEVectorVTInfo VTI, SDPatternOperator unpred_op,4789 Intrinsic PredInt, bit round> {4790 def "" : MVE_VxMULH<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, round>;4791 defvar Inst = !cast<Instruction>(NAME);4792 4793 let Predicates = [HasMVEInt] in {4794 if !eq(round, 0b0) then {4795 defvar mulh = !if(VTI.Unsigned, mulhu, mulhs);4796 defm : MVE_TwoOpPattern<VTI, mulh, PredInt, (? (i32 VTI.Unsigned)),4797 !cast<Instruction>(NAME)>;4798 } else {4799 // Predicated multiply returning high bits4800 def : Pat<(VTI.Vec (PredInt (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),4801 (i32 VTI.Unsigned), (VTI.Pred VCCR:$mask),4802 (VTI.Vec MQPR:$inactive))),4803 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),4804 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,4805 (VTI.Vec MQPR:$inactive)))>;4806 }4807 4808 // Unpredicated intrinsic4809 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),4810 (i32 VTI.Unsigned))),4811 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;4812 }4813}4814 4815multiclass MVE_VMULT<string iname, MVEVectorVTInfo VTI, bit round>4816 : MVE_VxMULH_m<iname, VTI, !if(round, int_arm_mve_vrmulh, int_arm_mve_vmulh),4817 !if(round, int_arm_mve_rmulh_predicated,4818 int_arm_mve_mulh_predicated),4819 round>;4820 4821defm MVE_VMULHs8 : MVE_VMULT<"vmulh", MVE_v16s8, 0b0>;4822defm MVE_VMULHs16 : MVE_VMULT<"vmulh", MVE_v8s16, 0b0>;4823defm MVE_VMULHs32 : MVE_VMULT<"vmulh", MVE_v4s32, 0b0>;4824defm MVE_VMULHu8 : MVE_VMULT<"vmulh", MVE_v16u8, 0b0>;4825defm MVE_VMULHu16 : MVE_VMULT<"vmulh", MVE_v8u16, 0b0>;4826defm MVE_VMULHu32 : MVE_VMULT<"vmulh", MVE_v4u32, 0b0>;4827 4828defm MVE_VRMULHs8 : MVE_VMULT<"vrmulh", MVE_v16s8, 0b1>;4829defm MVE_VRMULHs16 : MVE_VMULT<"vrmulh", MVE_v8s16, 0b1>;4830defm MVE_VRMULHs32 : MVE_VMULT<"vrmulh", MVE_v4s32, 0b1>;4831defm MVE_VRMULHu8 : MVE_VMULT<"vrmulh", MVE_v16u8, 0b1>;4832defm MVE_VRMULHu16 : MVE_VMULT<"vrmulh", MVE_v8u16, 0b1>;4833defm MVE_VRMULHu32 : MVE_VMULT<"vrmulh", MVE_v4u32, 0b1>;4834 4835class MVE_VxMOVxN<string iname, string suffix, bit bit_28, bit bit_17,4836 bits<2> size, bit T, list<dag> pattern=[]>4837 : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),4838 (ins MQPR:$Qd_src, MQPR:$Qm), "$Qd, $Qm",4839 vpred_n, "$Qd = $Qd_src", !if(size, 0b10, 0b01), pattern> {4840 4841 let Inst{28} = bit_28;4842 let Inst{21-20} = 0b11;4843 let Inst{19-18} = size;4844 let Inst{17} = bit_17;4845 let Inst{16} = 0b1;4846 let Inst{12} = T;4847 let Inst{8} = 0b0;4848 let Inst{7} = !not(bit_17);4849 let Inst{0} = 0b1;4850 let validForTailPredication = 1;4851 let retainsPreviousHalfElement = 1;4852}4853 4854multiclass MVE_VxMOVxN_halves<string iname, string suffix,4855 bit bit_28, bit bit_17, bits<2> size> {4856 def bh : MVE_VxMOVxN<iname # "b", suffix, bit_28, bit_17, size, 0b0>;4857 def th : MVE_VxMOVxN<iname # "t", suffix, bit_28, bit_17, size, 0b1>;4858}4859 4860defm MVE_VMOVNi16 : MVE_VxMOVxN_halves<"vmovn", "i16", 0b1, 0b0, 0b00>;4861defm MVE_VMOVNi32 : MVE_VxMOVxN_halves<"vmovn", "i32", 0b1, 0b0, 0b01>;4862defm MVE_VQMOVNs16 : MVE_VxMOVxN_halves<"vqmovn", "s16", 0b0, 0b1, 0b00>;4863defm MVE_VQMOVNs32 : MVE_VxMOVxN_halves<"vqmovn", "s32", 0b0, 0b1, 0b01>;4864defm MVE_VQMOVNu16 : MVE_VxMOVxN_halves<"vqmovn", "u16", 0b1, 0b1, 0b00>;4865defm MVE_VQMOVNu32 : MVE_VxMOVxN_halves<"vqmovn", "u32", 0b1, 0b1, 0b01>;4866defm MVE_VQMOVUNs16 : MVE_VxMOVxN_halves<"vqmovun", "s16", 0b0, 0b0, 0b00>;4867defm MVE_VQMOVUNs32 : MVE_VxMOVxN_halves<"vqmovun", "s32", 0b0, 0b0, 0b01>;4868 4869// MVE vmovn4870def MVEvmovn : SDNode<"ARMISD::VMOVN", SDTARMVEXT>;4871 4872multiclass MVE_VMOVN_p<Instruction Inst, bit top,4873 MVEVectorVTInfo VTI, MVEVectorVTInfo InVTI> {4874 // Match the most obvious MVEvmovn(a,b,t), which overwrites the odd or even4875 // lanes of a (depending on t) with the even lanes of b.4876 def : Pat<(VTI.Vec (MVEvmovn (VTI.Vec MQPR:$Qd_src),4877 (VTI.Vec MQPR:$Qm), (i32 top))),4878 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src), (VTI.Vec MQPR:$Qm)))>;4879 4880 if !not(top) then {4881 // If we see MVEvmovn(a,ARMvrev(b),1), that wants to overwrite the odd4882 // lanes of a with the odd lanes of b. In other words, the lanes we're4883 // _keeping_ from a are the even ones. So we can flip it round and say that4884 // this is the same as overwriting the even lanes of b with the even lanes4885 // of a, i.e. it's a VMOVNB with the operands reversed.4886 defvar vrev = !cast<SDNode>("ARMvrev" # InVTI.LaneBits);4887 def : Pat<(VTI.Vec (MVEvmovn (VTI.Vec MQPR:$Qm),4888 (VTI.Vec (vrev MQPR:$Qd_src)), (i32 1))),4889 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src), (VTI.Vec MQPR:$Qm)))>;4890 }4891 4892 // Match the IR intrinsic for a predicated VMOVN. This regards the Qm input4893 // as having wider lanes that we're narrowing, instead of already-narrow4894 // lanes that we're taking every other one of.4895 def : Pat<(VTI.Vec (int_arm_mve_vmovn_predicated (VTI.Vec MQPR:$Qd_src),4896 (InVTI.Vec MQPR:$Qm), (i32 top),4897 (InVTI.Pred VCCR:$pred))),4898 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src),4899 (InVTI.Vec MQPR:$Qm),4900 ARMVCCThen, (InVTI.Pred VCCR:$pred), zero_reg))>;4901}4902 4903defm : MVE_VMOVN_p<MVE_VMOVNi32bh, 0, MVE_v8i16, MVE_v4i32>;4904defm : MVE_VMOVN_p<MVE_VMOVNi32th, 1, MVE_v8i16, MVE_v4i32>;4905defm : MVE_VMOVN_p<MVE_VMOVNi16bh, 0, MVE_v16i8, MVE_v8i16>;4906defm : MVE_VMOVN_p<MVE_VMOVNi16th, 1, MVE_v16i8, MVE_v8i16>;4907 4908multiclass MVE_VQMOVN_p<Instruction Inst, bit outU, bit inU, bit top,4909 MVEVectorVTInfo VTI, MVEVectorVTInfo InVTI> {4910 def : Pat<(VTI.Vec (int_arm_mve_vqmovn (VTI.Vec MQPR:$Qd_src),4911 (InVTI.Vec MQPR:$Qm),4912 (i32 outU), (i32 inU), (i32 top))),4913 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src),4914 (InVTI.Vec MQPR:$Qm)))>;4915 4916 def : Pat<(VTI.Vec (int_arm_mve_vqmovn_predicated (VTI.Vec MQPR:$Qd_src),4917 (InVTI.Vec MQPR:$Qm),4918 (i32 outU), (i32 inU), (i32 top),4919 (InVTI.Pred VCCR:$pred))),4920 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src),4921 (InVTI.Vec MQPR:$Qm),4922 ARMVCCThen, (InVTI.Pred VCCR:$pred), zero_reg))>;4923}4924 4925defm : MVE_VQMOVN_p<MVE_VQMOVNs32bh, 0, 0, 0, MVE_v8i16, MVE_v4i32>;4926defm : MVE_VQMOVN_p<MVE_VQMOVNs32th, 0, 0, 1, MVE_v8i16, MVE_v4i32>;4927defm : MVE_VQMOVN_p<MVE_VQMOVNs16bh, 0, 0, 0, MVE_v16i8, MVE_v8i16>;4928defm : MVE_VQMOVN_p<MVE_VQMOVNs16th, 0, 0, 1, MVE_v16i8, MVE_v8i16>;4929defm : MVE_VQMOVN_p<MVE_VQMOVNu32bh, 1, 1, 0, MVE_v8i16, MVE_v4i32>;4930defm : MVE_VQMOVN_p<MVE_VQMOVNu32th, 1, 1, 1, MVE_v8i16, MVE_v4i32>;4931defm : MVE_VQMOVN_p<MVE_VQMOVNu16bh, 1, 1, 0, MVE_v16i8, MVE_v8i16>;4932defm : MVE_VQMOVN_p<MVE_VQMOVNu16th, 1, 1, 1, MVE_v16i8, MVE_v8i16>;4933defm : MVE_VQMOVN_p<MVE_VQMOVUNs32bh, 1, 0, 0, MVE_v8i16, MVE_v4i32>;4934defm : MVE_VQMOVN_p<MVE_VQMOVUNs32th, 1, 0, 1, MVE_v8i16, MVE_v4i32>;4935defm : MVE_VQMOVN_p<MVE_VQMOVUNs16bh, 1, 0, 0, MVE_v16i8, MVE_v8i16>;4936defm : MVE_VQMOVN_p<MVE_VQMOVUNs16th, 1, 0, 1, MVE_v16i8, MVE_v8i16>;4937 4938def SDTARMVMOVNQ : SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisSameAs<0, 1>,4939 SDTCisVec<2>, SDTCisVT<3, i32>]>;4940 4941// Vector (V) Saturating (Q) Move and Narrow (N), signed (s)4942def MVEvqmovns : SDNode<"ARMISD::VQMOVNs", SDTARMVMOVNQ>;4943 4944// Vector (V) Saturating (Q) Move and Narrow (N), unsigned (u)4945def MVEvqmovnu : SDNode<"ARMISD::VQMOVNu", SDTARMVMOVNQ>;4946 4947let Predicates = [HasMVEInt] in {4948 def : Pat<(v8i16 (MVEvqmovns (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), (i32 0))),4949 (v8i16 (MVE_VQMOVNs32bh (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm)))>;4950 def : Pat<(v8i16 (MVEvqmovns (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), (i32 1))),4951 (v8i16 (MVE_VQMOVNs32th (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm)))>;4952 def : Pat<(v16i8 (MVEvqmovns (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), (i32 0))),4953 (v16i8 (MVE_VQMOVNs16bh (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm)))>;4954 def : Pat<(v16i8 (MVEvqmovns (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), (i32 1))),4955 (v16i8 (MVE_VQMOVNs16th (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm)))>;4956 4957 def : Pat<(v8i16 (MVEvqmovnu (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), (i32 0))),4958 (v8i16 (MVE_VQMOVNu32bh (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm)))>;4959 def : Pat<(v8i16 (MVEvqmovnu (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), (i32 1))),4960 (v8i16 (MVE_VQMOVNu32th (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm)))>;4961 def : Pat<(v16i8 (MVEvqmovnu (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), (i32 0))),4962 (v16i8 (MVE_VQMOVNu16bh (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm)))>;4963 def : Pat<(v16i8 (MVEvqmovnu (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), (i32 1))),4964 (v16i8 (MVE_VQMOVNu16th (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm)))>;4965 4966 def : Pat<(v8i16 (MVEvqmovns (v8i16 MQPR:$Qd_src), (v4i32 (ARMvshrsImm (v4i32 MQPR:$Qm), imm0_31:$imm)), (i32 0))),4967 (v8i16 (MVE_VQSHRNbhs32 (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), imm0_31:$imm))>;4968 def : Pat<(v16i8 (MVEvqmovns (v16i8 MQPR:$Qd_src), (v8i16 (ARMvshrsImm (v8i16 MQPR:$Qm), imm0_15:$imm)), (i32 0))),4969 (v16i8 (MVE_VQSHRNbhs16 (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), imm0_15:$imm))>;4970 def : Pat<(v8i16 (MVEvqmovns (v8i16 MQPR:$Qd_src), (v4i32 (ARMvshrsImm (v4i32 MQPR:$Qm), imm0_31:$imm)), (i32 1))),4971 (v8i16 (MVE_VQSHRNths32 (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), imm0_31:$imm))>;4972 def : Pat<(v16i8 (MVEvqmovns (v16i8 MQPR:$Qd_src), (v8i16 (ARMvshrsImm (v8i16 MQPR:$Qm), imm0_15:$imm)), (i32 1))),4973 (v16i8 (MVE_VQSHRNths16 (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), imm0_15:$imm))>;4974 4975 def : Pat<(v8i16 (MVEvqmovnu (v8i16 MQPR:$Qd_src), (v4i32 (ARMvshruImm (v4i32 MQPR:$Qm), imm0_31:$imm)), (i32 0))),4976 (v8i16 (MVE_VQSHRNbhu32 (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), imm0_31:$imm))>;4977 def : Pat<(v16i8 (MVEvqmovnu (v16i8 MQPR:$Qd_src), (v8i16 (ARMvshruImm (v8i16 MQPR:$Qm), imm0_15:$imm)), (i32 0))),4978 (v16i8 (MVE_VQSHRNbhu16 (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), imm0_15:$imm))>;4979 def : Pat<(v8i16 (MVEvqmovnu (v8i16 MQPR:$Qd_src), (v4i32 (ARMvshruImm (v4i32 MQPR:$Qm), imm0_31:$imm)), (i32 1))),4980 (v8i16 (MVE_VQSHRNthu32 (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), imm0_31:$imm))>;4981 def : Pat<(v16i8 (MVEvqmovnu (v16i8 MQPR:$Qd_src), (v8i16 (ARMvshruImm (v8i16 MQPR:$Qm), imm0_15:$imm)), (i32 1))),4982 (v16i8 (MVE_VQSHRNthu16 (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), imm0_15:$imm))>;4983}4984 4985class MVE_VCVT_ff<string iname, string suffix, bit op, bit T,4986 dag iops_extra, vpred_ops vpred, string cstr>4987 : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),4988 !con(iops_extra, (ins MQPR:$Qm)), "$Qd, $Qm",4989 vpred, cstr, 0b10, []> {4990 let Inst{28} = op;4991 let Inst{21-16} = 0b111111;4992 let Inst{12} = T;4993 let Inst{8-7} = 0b00;4994 let Inst{0} = 0b1;4995 4996 let Predicates = [HasMVEFloat];4997 let retainsPreviousHalfElement = 1;4998}4999 5000def SDTARMVCVTL : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>,5001 SDTCisVT<2, i32>]>;5002 5003// MVE vcvt f32 -> f16, truncating into either the bottom or top lanes5004def MVEvcvtn : SDNode<"ARMISD::VCVTN", SDTARMVMOVNQ>;5005 5006// MVE vcvt f16 -> f32, extending from either the bottom or top lanes5007def MVEvcvtl : SDNode<"ARMISD::VCVTL", SDTARMVCVTL>;5008 5009multiclass MVE_VCVT_f2h_m<string iname, int half> {5010 def "": MVE_VCVT_ff<iname, "f16.f32", 0b0, half,5011 (ins MQPR:$Qd_src), vpred_n, "$Qd = $Qd_src">;5012 defvar Inst = !cast<Instruction>(NAME);5013 5014 let Predicates = [HasMVEFloat] in {5015 def : Pat<(v8f16 (int_arm_mve_vcvt_narrow5016 (v8f16 MQPR:$Qd_src), (v4f32 MQPR:$Qm), (i32 half))),5017 (v8f16 (Inst (v8f16 MQPR:$Qd_src), (v4f32 MQPR:$Qm)))>;5018 def : Pat<(v8f16 (int_arm_mve_vcvt_narrow_predicated5019 (v8f16 MQPR:$Qd_src), (v4f32 MQPR:$Qm), (i32 half),5020 (v4i1 VCCR:$mask))),5021 (v8f16 (Inst (v8f16 MQPR:$Qd_src), (v4f32 MQPR:$Qm),5022 ARMVCCThen, (v4i1 VCCR:$mask), zero_reg))>;5023 5024 def : Pat<(v8f16 (MVEvcvtn (v8f16 MQPR:$Qd_src), (v4f32 MQPR:$Qm), (i32 half))),5025 (v8f16 (Inst (v8f16 MQPR:$Qd_src), (v4f32 MQPR:$Qm)))>;5026 }5027}5028 5029multiclass MVE_VCVT_h2f_m<string iname, int half> {5030 def "": MVE_VCVT_ff<iname, "f32.f16", 0b1, half, (ins), vpred_r, "">;5031 defvar Inst = !cast<Instruction>(NAME);5032 5033 let Predicates = [HasMVEFloat] in {5034 def : Pat<(v4f32 (int_arm_mve_vcvt_widen (v8f16 MQPR:$Qm), (i32 half))),5035 (v4f32 (Inst (v8f16 MQPR:$Qm)))>;5036 def : Pat<(v4f32 (int_arm_mve_vcvt_widen_predicated5037 (v4f32 MQPR:$inactive), (v8f16 MQPR:$Qm), (i32 half),5038 (v4i1 VCCR:$mask))),5039 (v4f32 (Inst (v8f16 MQPR:$Qm), ARMVCCThen,5040 (v4i1 VCCR:$mask), zero_reg, (v4f32 MQPR:$inactive)))>;5041 5042 def : Pat<(v4f32 (MVEvcvtl (v8f16 MQPR:$Qm), (i32 half))),5043 (v4f32 (Inst (v8f16 MQPR:$Qm)))>;5044 }5045}5046 5047defm MVE_VCVTf16f32bh : MVE_VCVT_f2h_m<"vcvtb", 0b0>;5048defm MVE_VCVTf16f32th : MVE_VCVT_f2h_m<"vcvtt", 0b1>;5049defm MVE_VCVTf32f16bh : MVE_VCVT_h2f_m<"vcvtb", 0b0>;5050defm MVE_VCVTf32f16th : MVE_VCVT_h2f_m<"vcvtt", 0b1>;5051 5052class MVE_VxCADD<string iname, string suffix, bits<2> size, bit halve,5053 string cstr="">5054 : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),5055 (ins MQPR:$Qn, MQPR:$Qm, complexrotateopodd:$rot),5056 "$Qd, $Qn, $Qm, $rot", vpred_r, cstr, size, []> {5057 bits<4> Qn;5058 bit rot;5059 5060 let Inst{28} = halve;5061 let Inst{21-20} = size;5062 let Inst{19-17} = Qn{2-0};5063 let Inst{16} = 0b0;5064 let Inst{12} = rot;5065 let Inst{8} = 0b1;5066 let Inst{7} = Qn{3};5067 let Inst{0} = 0b0;5068}5069 5070multiclass MVE_VxCADD_m<string iname, MVEVectorVTInfo VTI,5071 bit halve, string cstr=""> {5072 def "" : MVE_VxCADD<iname, VTI.Suffix, VTI.Size, halve, cstr>;5073 defvar Inst = !cast<Instruction>(NAME);5074 5075 let Predicates = [HasMVEInt] in {5076 def : Pat<(VTI.Vec (int_arm_mve_vcaddq halve,5077 imm:$rot, (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),5078 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),5079 imm:$rot))>;5080 5081 def : Pat<(VTI.Vec (int_arm_mve_vcaddq_predicated halve,5082 imm:$rot, (VTI.Vec MQPR:$inactive),5083 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),5084 (VTI.Pred VCCR:$mask))),5085 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),5086 imm:$rot, ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,5087 (VTI.Vec MQPR:$inactive)))>;5088 5089 }5090}5091 5092defm MVE_VCADDi8 : MVE_VxCADD_m<"vcadd", MVE_v16i8, 0b1>;5093defm MVE_VCADDi16 : MVE_VxCADD_m<"vcadd", MVE_v8i16, 0b1>;5094defm MVE_VCADDi32 : MVE_VxCADD_m<"vcadd", MVE_v4i32, 0b1, "@earlyclobber $Qd">;5095 5096defm MVE_VHCADDs8 : MVE_VxCADD_m<"vhcadd", MVE_v16s8, 0b0>;5097defm MVE_VHCADDs16 : MVE_VxCADD_m<"vhcadd", MVE_v8s16, 0b0>;5098defm MVE_VHCADDs32 : MVE_VxCADD_m<"vhcadd", MVE_v4s32, 0b0, "@earlyclobber $Qd">;5099 5100class MVE_VADCSBC<string iname, bit I, bit subtract,5101 dag carryin, list<dag> pattern=[]>5102 : MVE_qDest_qSrc<iname, "i32", (outs MQPR:$Qd, cl_FPSCR_NZCV:$carryout),5103 !con((ins MQPR:$Qn, MQPR:$Qm), carryin),5104 "$Qd, $Qn, $Qm", vpred_r, "", 0b10, pattern> {5105 bits<4> Qn;5106 5107 let Inst{28} = subtract;5108 let Inst{21-20} = 0b11;5109 let Inst{19-17} = Qn{2-0};5110 let Inst{16} = 0b0;5111 let Inst{12} = I;5112 let Inst{8} = 0b1;5113 let Inst{7} = Qn{3};5114 let Inst{0} = 0b0;5115 5116 // Custom decoder method in order to add the FPSCR operand(s), which5117 // Tablegen won't do right5118 let DecoderMethod = "DecodeMVEVADCInstruction";5119}5120 5121def MVE_VADC : MVE_VADCSBC<"vadc", 0b0, 0b0, (ins cl_FPSCR_NZCV:$carryin)>;5122def MVE_VADCI : MVE_VADCSBC<"vadci", 0b1, 0b0, (ins)>;5123 5124def MVE_VSBC : MVE_VADCSBC<"vsbc", 0b0, 0b1, (ins cl_FPSCR_NZCV:$carryin)>;5125def MVE_VSBCI : MVE_VADCSBC<"vsbci", 0b1, 0b1, (ins)>;5126 5127class MVE_VQDMULL<string iname, string suffix, bit size, bit T,5128 string cstr="", list<dag> pattern=[]>5129 : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),5130 (ins MQPR:$Qn, MQPR:$Qm), "$Qd, $Qn, $Qm",5131 vpred_r, cstr, !if(size, 0b10, 0b01), pattern> {5132 bits<4> Qn;5133 5134 let Inst{28} = size;5135 let Inst{21-20} = 0b11;5136 let Inst{19-17} = Qn{2-0};5137 let Inst{16} = 0b0;5138 let Inst{12} = T;5139 let Inst{8} = 0b1;5140 let Inst{7} = Qn{3};5141 let Inst{0} = 0b1;5142 let validForTailPredication = 1;5143 let doubleWidthResult = 1;5144}5145 5146multiclass MVE_VQDMULL_m<string iname, MVEVectorVTInfo VTI, bit size, bit T,5147 string cstr> {5148 def "" : MVE_VQDMULL<iname, VTI.Suffix, size, T, cstr>;5149 defvar Inst = !cast<Instruction>(NAME);5150 5151 let Predicates = [HasMVEInt] in {5152 // Unpredicated saturating multiply5153 def : Pat<(VTI.DblVec (int_arm_mve_vqdmull (VTI.Vec MQPR:$Qm),5154 (VTI.Vec MQPR:$Qn), (i32 T))),5155 (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;5156 // Predicated saturating multiply5157 def : Pat<(VTI.DblVec (int_arm_mve_vqdmull_predicated5158 (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),5159 (i32 T), (VTI.DblPred VCCR:$mask),5160 (VTI.DblVec MQPR:$inactive))),5161 (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),5162 ARMVCCThen, (VTI.DblPred VCCR:$mask), zero_reg,5163 (VTI.DblVec MQPR:$inactive)))>;5164 }5165}5166 5167multiclass MVE_VQDMULL_halves<MVEVectorVTInfo VTI, bit size, string cstr=""> {5168 defm bh : MVE_VQDMULL_m<"vqdmullb", VTI, size, 0b0, cstr>;5169 defm th : MVE_VQDMULL_m<"vqdmullt", VTI, size, 0b1, cstr>;5170}5171 5172defm MVE_VQDMULLs16 : MVE_VQDMULL_halves<MVE_v8s16, 0b0>;5173defm MVE_VQDMULLs32 : MVE_VQDMULL_halves<MVE_v4s32, 0b1, "@earlyclobber $Qd">;5174 5175// end of mve_qDest_qSrc5176 5177// start of mve_qDest_rSrc5178 5179class MVE_qr_base<dag oops, dag iops, string iname, string suffix, string ops,5180 vpred_ops vpred, string cstr, bits<2> vecsize, list<dag> pattern=[]>5181 : MVE_p<oops, iops, NoItinerary, iname, suffix, ops, vpred, cstr, vecsize, pattern> {5182 bits<4> Qd;5183 bits<4> Qn;5184 bits<4> Rm;5185 5186 let Inst{25-23} = 0b100;5187 let Inst{22} = Qd{3};5188 let Inst{19-17} = Qn{2-0};5189 let Inst{15-13} = Qd{2-0};5190 let Inst{11-9} = 0b111;5191 let Inst{7} = Qn{3};5192 let Inst{6} = 0b1;5193 let Inst{4} = 0b0;5194 let Inst{3-0} = Rm{3-0};5195}5196 5197class MVE_qDest_rSrc<string iname, string suffix, string cstr="", bits<2> vecsize, list<dag> pattern=[]>5198 : MVE_qr_base<(outs MQPR:$Qd), (ins MQPR:$Qn, rGPR:$Rm),5199 iname, suffix, "$Qd, $Qn, $Rm", vpred_r, cstr,5200 vecsize, pattern>;5201 5202class MVE_qDestSrc_rSrc<string iname, string suffix, bits<2> vecsize, list<dag> pattern=[]>5203 : MVE_qr_base<(outs MQPR:$Qd), (ins MQPR:$Qd_src, MQPR:$Qn, rGPR:$Rm),5204 iname, suffix, "$Qd, $Qn, $Rm", vpred_n, "$Qd = $Qd_src",5205 vecsize, pattern>;5206 5207class MVE_qDest_single_rSrc<string iname, string suffix, bits<2> vecsize, list<dag> pattern=[]>5208 : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qd_src, rGPR:$Rm), NoItinerary, iname,5209 suffix, "$Qd, $Rm", vpred_n, "$Qd = $Qd_src", vecsize, pattern> {5210 bits<4> Qd;5211 bits<4> Rm;5212 5213 let Inst{22} = Qd{3};5214 let Inst{15-13} = Qd{2-0};5215 let Inst{3-0} = Rm{3-0};5216}5217 5218// Patterns for vector-scalar instructions with integer operands5219multiclass MVE_vec_scalar_int_pat_m<Instruction inst, MVEVectorVTInfo VTI,5220 SDPatternOperator unpred_op,5221 SDPatternOperator pred_op,5222 bit unpred_has_sign = 0,5223 bit pred_has_sign = 0> {5224 defvar UnpredSign = !if(unpred_has_sign, (? (i32 VTI.Unsigned)), (?));5225 defvar PredSign = !if(pred_has_sign, (? (i32 VTI.Unsigned)), (?));5226 5227 let Predicates = [HasMVEInt] in {5228 // Unpredicated version5229 def : Pat<(VTI.Vec !con((unpred_op (VTI.Vec MQPR:$Qm),5230 (VTI.Vec (ARMvdup rGPR:$val))),5231 UnpredSign)),5232 (VTI.Vec (inst (VTI.Vec MQPR:$Qm), (i32 rGPR:$val)))>;5233 // Predicated version5234 def : Pat<(VTI.Vec !con((pred_op (VTI.Vec MQPR:$Qm),5235 (VTI.Vec (ARMvdup rGPR:$val))),5236 PredSign,5237 (pred_op (VTI.Pred VCCR:$mask),5238 (VTI.Vec MQPR:$inactive)))),5239 (VTI.Vec (inst (VTI.Vec MQPR:$Qm), (i32 rGPR:$val),5240 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,5241 (VTI.Vec MQPR:$inactive)))>;5242 }5243}5244 5245class MVE_VADDSUB_qr<string iname, string suffix, bits<2> size,5246 bit bit_5, bit bit_12, bit bit_16, bit bit_28>5247 : MVE_qDest_rSrc<iname, suffix, "", size> {5248 5249 let Inst{28} = bit_28;5250 let Inst{21-20} = size;5251 let Inst{16} = bit_16;5252 let Inst{12} = bit_12;5253 let Inst{8} = 0b1;5254 let Inst{5} = bit_5;5255 let validForTailPredication = 1;5256}5257 5258// Vector-scalar add/sub5259multiclass MVE_VADDSUB_qr_m<string iname, MVEVectorVTInfo VTI, bit subtract,5260 SDNode Op, Intrinsic PredInt> {5261 def "" : MVE_VADDSUB_qr<iname, VTI.Suffix, VTI.Size, 0b0, subtract, 0b1, 0b0>;5262 let Predicates = [HasMVEInt] in {5263 defm : MVE_TwoOpPatternDup<VTI, Op, PredInt, (? ), !cast<Instruction>(NAME), ARMimmAllZerosV>;5264 }5265}5266 5267multiclass MVE_VADD_qr_m<MVEVectorVTInfo VTI>5268 : MVE_VADDSUB_qr_m<"vadd", VTI, 0b0, add, int_arm_mve_add_predicated>;5269 5270multiclass MVE_VSUB_qr_m<MVEVectorVTInfo VTI>5271 : MVE_VADDSUB_qr_m<"vsub", VTI, 0b1, sub, int_arm_mve_sub_predicated>;5272 5273defm MVE_VADD_qr_i8 : MVE_VADD_qr_m<MVE_v16i8>;5274defm MVE_VADD_qr_i16 : MVE_VADD_qr_m<MVE_v8i16>;5275defm MVE_VADD_qr_i32 : MVE_VADD_qr_m<MVE_v4i32>;5276 5277defm MVE_VSUB_qr_i8 : MVE_VSUB_qr_m<MVE_v16i8>;5278defm MVE_VSUB_qr_i16 : MVE_VSUB_qr_m<MVE_v8i16>;5279defm MVE_VSUB_qr_i32 : MVE_VSUB_qr_m<MVE_v4i32>;5280 5281// Vector-scalar saturating add/sub5282multiclass MVE_VQADDSUB_qr_m<string iname, MVEVectorVTInfo VTI, bit subtract,5283 SDNode Op, Intrinsic PredInt> {5284 def "" : MVE_VADDSUB_qr<iname, VTI.Suffix, VTI.Size, 0b1, subtract,5285 0b0, VTI.Unsigned>;5286 5287 let Predicates = [HasMVEInt] in {5288 defm : MVE_TwoOpPatternDup<VTI, Op, PredInt, (? (i32 VTI.Unsigned)),5289 !cast<Instruction>(NAME)>;5290 }5291}5292 5293multiclass MVE_VQADD_qr_m<MVEVectorVTInfo VTI, SDNode Op>5294 : MVE_VQADDSUB_qr_m<"vqadd", VTI, 0b0, Op, int_arm_mve_qadd_predicated>;5295 5296multiclass MVE_VQSUB_qr_m<MVEVectorVTInfo VTI, SDNode Op>5297 : MVE_VQADDSUB_qr_m<"vqsub", VTI, 0b1, Op, int_arm_mve_qsub_predicated>;5298 5299defm MVE_VQADD_qr_s8 : MVE_VQADD_qr_m<MVE_v16s8, saddsat>;5300defm MVE_VQADD_qr_s16 : MVE_VQADD_qr_m<MVE_v8s16, saddsat>;5301defm MVE_VQADD_qr_s32 : MVE_VQADD_qr_m<MVE_v4s32, saddsat>;5302defm MVE_VQADD_qr_u8 : MVE_VQADD_qr_m<MVE_v16u8, uaddsat>;5303defm MVE_VQADD_qr_u16 : MVE_VQADD_qr_m<MVE_v8u16, uaddsat>;5304defm MVE_VQADD_qr_u32 : MVE_VQADD_qr_m<MVE_v4u32, uaddsat>;5305 5306defm MVE_VQSUB_qr_s8 : MVE_VQSUB_qr_m<MVE_v16s8, ssubsat>;5307defm MVE_VQSUB_qr_s16 : MVE_VQSUB_qr_m<MVE_v8s16, ssubsat>;5308defm MVE_VQSUB_qr_s32 : MVE_VQSUB_qr_m<MVE_v4s32, ssubsat>;5309defm MVE_VQSUB_qr_u8 : MVE_VQSUB_qr_m<MVE_v16u8, usubsat>;5310defm MVE_VQSUB_qr_u16 : MVE_VQSUB_qr_m<MVE_v8u16, usubsat>;5311defm MVE_VQSUB_qr_u32 : MVE_VQSUB_qr_m<MVE_v4u32, usubsat>;5312 5313class MVE_VQDMULL_qr<string iname, string suffix, bit size,5314 bit T, string cstr="", list<dag> pattern=[]>5315 : MVE_qDest_rSrc<iname, suffix, cstr, !if(size, 0b10, 0b01), pattern> {5316 5317 let Inst{28} = size;5318 let Inst{21-20} = 0b11;5319 let Inst{16} = 0b0;5320 let Inst{12} = T;5321 let Inst{8} = 0b1;5322 let Inst{5} = 0b1;5323 let validForTailPredication = 1;5324 let doubleWidthResult = 1;5325}5326 5327multiclass MVE_VQDMULL_qr_m<string iname, MVEVectorVTInfo VTI, bit size,5328 bit T, string cstr> {5329 def "" : MVE_VQDMULL_qr<iname, VTI.Suffix, size, T, cstr>;5330 defvar Inst = !cast<Instruction>(NAME);5331 5332 let Predicates = [HasMVEInt] in {5333 // Unpredicated saturating multiply5334 def : Pat<(VTI.DblVec (int_arm_mve_vqdmull (VTI.Vec MQPR:$Qm),5335 (VTI.Vec (ARMvdup rGPR:$val)),5336 (i32 T))),5337 (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (i32 rGPR:$val)))>;5338 // Predicated saturating multiply5339 def : Pat<(VTI.DblVec (int_arm_mve_vqdmull_predicated5340 (VTI.Vec MQPR:$Qm),5341 (VTI.Vec (ARMvdup rGPR:$val)),5342 (i32 T),5343 (VTI.DblPred VCCR:$mask),5344 (VTI.DblVec MQPR:$inactive))),5345 (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (i32 rGPR:$val),5346 ARMVCCThen, (VTI.DblPred VCCR:$mask), zero_reg,5347 (VTI.DblVec MQPR:$inactive)))>;5348 }5349}5350 5351multiclass MVE_VQDMULL_qr_halves<MVEVectorVTInfo VTI, bit size, string cstr=""> {5352 defm bh : MVE_VQDMULL_qr_m<"vqdmullb", VTI, size, 0b0, cstr>;5353 defm th : MVE_VQDMULL_qr_m<"vqdmullt", VTI, size, 0b1, cstr>;5354}5355 5356defm MVE_VQDMULL_qr_s16 : MVE_VQDMULL_qr_halves<MVE_v8s16, 0b0>;5357defm MVE_VQDMULL_qr_s32 : MVE_VQDMULL_qr_halves<MVE_v4s32, 0b1, "@earlyclobber $Qd">;5358 5359class MVE_VxADDSUB_qr<string iname, string suffix,5360 bit bit_28, bits<2> size, bit subtract,5361 bits<2> vecsize, list<dag> pattern=[]>5362 : MVE_qDest_rSrc<iname, suffix, "", vecsize, pattern> {5363 5364 let Inst{28} = bit_28;5365 let Inst{21-20} = size;5366 let Inst{16} = 0b0;5367 let Inst{12} = subtract;5368 let Inst{8} = 0b1;5369 let Inst{5} = 0b0;5370 let validForTailPredication = 1;5371}5372 5373multiclass MVE_VHADDSUB_qr_m<string iname, MVEVectorVTInfo VTI, bit subtract, SDPatternOperator Op,5374 Intrinsic unpred_int, Intrinsic pred_int, PatFrag add_op, SDNode shift_op> {5375 def "" : MVE_VxADDSUB_qr<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, subtract, VTI.Size>;5376 defm : MVE_TwoOpPatternDup<VTI, Op, pred_int, (? (i32 VTI.Unsigned)), !cast<Instruction>(NAME)>;5377 defm : MVE_vec_scalar_int_pat_m<!cast<Instruction>(NAME),5378 VTI, unpred_int, pred_int, 1, 1>;5379 defvar Inst = !cast<Instruction>(NAME);5380 5381 let Predicates = [HasMVEInt] in {5382 def : Pat<(VTI.Vec (shift_op (add_op (VTI.Vec MQPR:$Qm), (VTI.Vec (ARMvdup rGPR:$Rn))), (i32 1))),5383 (Inst MQPR:$Qm, rGPR:$Rn)>;5384 }5385}5386 5387multiclass MVE_VHADD_qr_m<MVEVectorVTInfo VTI, PatFrag add_op, SDNode shift_op, SDNode Op> :5388 MVE_VHADDSUB_qr_m<"vhadd", VTI, 0b0, Op, int_arm_mve_vhadd,5389 int_arm_mve_hadd_predicated, add_op, shift_op>;5390 5391multiclass MVE_VHSUB_qr_m<MVEVectorVTInfo VTI, PatFrag add_op, SDNode shift_op> :5392 MVE_VHADDSUB_qr_m<"vhsub", VTI, 0b1, null_frag, int_arm_mve_vhsub,5393 int_arm_mve_hsub_predicated, add_op, shift_op>;5394 5395defm MVE_VHADD_qr_s8 : MVE_VHADD_qr_m<MVE_v16s8, addnsw, ARMvshrsImm, avgfloors>;5396defm MVE_VHADD_qr_s16 : MVE_VHADD_qr_m<MVE_v8s16, addnsw, ARMvshrsImm, avgfloors>;5397defm MVE_VHADD_qr_s32 : MVE_VHADD_qr_m<MVE_v4s32, addnsw, ARMvshrsImm, avgfloors>;5398defm MVE_VHADD_qr_u8 : MVE_VHADD_qr_m<MVE_v16u8, addnuw, ARMvshruImm, avgflooru>;5399defm MVE_VHADD_qr_u16 : MVE_VHADD_qr_m<MVE_v8u16, addnuw, ARMvshruImm, avgflooru>;5400defm MVE_VHADD_qr_u32 : MVE_VHADD_qr_m<MVE_v4u32, addnuw, ARMvshruImm, avgflooru>;5401 5402defm MVE_VHSUB_qr_s8 : MVE_VHSUB_qr_m<MVE_v16s8, subnsw, ARMvshrsImm>;5403defm MVE_VHSUB_qr_s16 : MVE_VHSUB_qr_m<MVE_v8s16, subnsw, ARMvshrsImm>;5404defm MVE_VHSUB_qr_s32 : MVE_VHSUB_qr_m<MVE_v4s32, subnsw, ARMvshrsImm>;5405defm MVE_VHSUB_qr_u8 : MVE_VHSUB_qr_m<MVE_v16u8, subnuw, ARMvshruImm>;5406defm MVE_VHSUB_qr_u16 : MVE_VHSUB_qr_m<MVE_v8u16, subnuw, ARMvshruImm>;5407defm MVE_VHSUB_qr_u32 : MVE_VHSUB_qr_m<MVE_v4u32, subnuw, ARMvshruImm>;5408 5409multiclass MVE_VADDSUB_qr_f<string iname, MVEVectorVTInfo VTI, bit subtract,5410 SDPatternOperator Op, Intrinsic PredInt,5411 SDPatternOperator IdentityVec> {5412 def "" : MVE_VxADDSUB_qr<iname, VTI.Suffix, VTI.Size{0}, 0b11, subtract, VTI.Size>;5413 defm : MVE_TwoOpPatternDup<VTI, Op, PredInt, (? ),5414 !cast<Instruction>(NAME), IdentityVec>;5415}5416 5417let Predicates = [HasMVEFloat] in {5418 defm MVE_VADD_qr_f32 : MVE_VADDSUB_qr_f<"vadd", MVE_v4f32, 0b0, vadd,5419 int_arm_mve_add_predicated, ARMimmMinusZeroF>;5420 defm MVE_VADD_qr_f16 : MVE_VADDSUB_qr_f<"vadd", MVE_v8f16, 0b0, vadd,5421 int_arm_mve_add_predicated, ARMimmMinusZeroH>;5422 5423 defm MVE_VSUB_qr_f32 : MVE_VADDSUB_qr_f<"vsub", MVE_v4f32, 0b1, vsub,5424 int_arm_mve_sub_predicated, ARMimmAllZerosV>;5425 defm MVE_VSUB_qr_f16 : MVE_VADDSUB_qr_f<"vsub", MVE_v8f16, 0b1, vsub,5426 int_arm_mve_sub_predicated, ARMimmAllZerosV>;5427}5428 5429class MVE_VxSHL_qr<string iname, string suffix, bit U, bits<2> size,5430 bit bit_7, bit bit_17, list<dag> pattern=[]>5431 : MVE_qDest_single_rSrc<iname, suffix, size, pattern> {5432 5433 let Inst{28} = U;5434 let Inst{25-23} = 0b100;5435 let Inst{21-20} = 0b11;5436 let Inst{19-18} = size;5437 let Inst{17} = bit_17;5438 let Inst{16} = 0b1;5439 let Inst{12-8} = 0b11110;5440 let Inst{7} = bit_7;5441 let Inst{6-4} = 0b110;5442 let validForTailPredication = 1;5443}5444 5445multiclass MVE_VxSHL_qr_p<string iname, MVEVectorVTInfo VTI, bit q, bit r> {5446 def "" : MVE_VxSHL_qr<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, q, r>;5447 defvar Inst = !cast<Instruction>(NAME);5448 5449 def : Pat<(VTI.Vec (int_arm_mve_vshl_scalar5450 (VTI.Vec MQPR:$in), (i32 rGPR:$sh),5451 (i32 q), (i32 r), (i32 VTI.Unsigned))),5452 (VTI.Vec (Inst (VTI.Vec MQPR:$in), (i32 rGPR:$sh)))>;5453 5454 def : Pat<(VTI.Vec (int_arm_mve_vshl_scalar_predicated5455 (VTI.Vec MQPR:$in), (i32 rGPR:$sh),5456 (i32 q), (i32 r), (i32 VTI.Unsigned),5457 (VTI.Pred VCCR:$mask))),5458 (VTI.Vec (Inst (VTI.Vec MQPR:$in), (i32 rGPR:$sh),5459 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;5460}5461 5462multiclass MVE_VxSHL_qr_types<string iname, bit bit_7, bit bit_17> {5463 defm s8 : MVE_VxSHL_qr_p<iname, MVE_v16s8, bit_7, bit_17>;5464 defm s16 : MVE_VxSHL_qr_p<iname, MVE_v8s16, bit_7, bit_17>;5465 defm s32 : MVE_VxSHL_qr_p<iname, MVE_v4s32, bit_7, bit_17>;5466 defm u8 : MVE_VxSHL_qr_p<iname, MVE_v16u8, bit_7, bit_17>;5467 defm u16 : MVE_VxSHL_qr_p<iname, MVE_v8u16, bit_7, bit_17>;5468 defm u32 : MVE_VxSHL_qr_p<iname, MVE_v4u32, bit_7, bit_17>;5469}5470 5471defm MVE_VSHL_qr : MVE_VxSHL_qr_types<"vshl", 0b0, 0b0>;5472defm MVE_VRSHL_qr : MVE_VxSHL_qr_types<"vrshl", 0b0, 0b1>;5473defm MVE_VQSHL_qr : MVE_VxSHL_qr_types<"vqshl", 0b1, 0b0>;5474defm MVE_VQRSHL_qr : MVE_VxSHL_qr_types<"vqrshl", 0b1, 0b1>;5475 5476let Predicates = [HasMVEInt] in {5477 def : Pat<(v4i32 (ARMvshlu (v4i32 MQPR:$Qm), (v4i32 (ARMvdup rGPR:$Rm)))),5478 (v4i32 (MVE_VSHL_qru32 (v4i32 MQPR:$Qm), rGPR:$Rm))>;5479 def : Pat<(v8i16 (ARMvshlu (v8i16 MQPR:$Qm), (v8i16 (ARMvdup rGPR:$Rm)))),5480 (v8i16 (MVE_VSHL_qru16 (v8i16 MQPR:$Qm), rGPR:$Rm))>;5481 def : Pat<(v16i8 (ARMvshlu (v16i8 MQPR:$Qm), (v16i8 (ARMvdup rGPR:$Rm)))),5482 (v16i8 (MVE_VSHL_qru8 (v16i8 MQPR:$Qm), rGPR:$Rm))>;5483 5484 def : Pat<(v4i32 (ARMvshls (v4i32 MQPR:$Qm), (v4i32 (ARMvdup rGPR:$Rm)))),5485 (v4i32 (MVE_VSHL_qrs32 (v4i32 MQPR:$Qm), rGPR:$Rm))>;5486 def : Pat<(v8i16 (ARMvshls (v8i16 MQPR:$Qm), (v8i16 (ARMvdup rGPR:$Rm)))),5487 (v8i16 (MVE_VSHL_qrs16 (v8i16 MQPR:$Qm), rGPR:$Rm))>;5488 def : Pat<(v16i8 (ARMvshls (v16i8 MQPR:$Qm), (v16i8 (ARMvdup rGPR:$Rm)))),5489 (v16i8 (MVE_VSHL_qrs8 (v16i8 MQPR:$Qm), rGPR:$Rm))>;5490}5491 5492class MVE_VBRSR<string iname, string suffix, bits<2> size, list<dag> pattern=[]>5493 : MVE_qDest_rSrc<iname, suffix, "", size, pattern> {5494 5495 let Inst{28} = 0b1;5496 let Inst{21-20} = size;5497 let Inst{16} = 0b1;5498 let Inst{12} = 0b1;5499 let Inst{8} = 0b0;5500 let Inst{5} = 0b1;5501 let validForTailPredication = 1;5502}5503 5504def MVE_VBRSR8 : MVE_VBRSR<"vbrsr", "8", 0b00>;5505def MVE_VBRSR16 : MVE_VBRSR<"vbrsr", "16", 0b01>;5506def MVE_VBRSR32 : MVE_VBRSR<"vbrsr", "32", 0b10>;5507 5508multiclass MVE_VBRSR_pat_m<MVEVectorVTInfo VTI, Instruction Inst> {5509 // Unpredicated5510 def : Pat<(VTI.Vec (int_arm_mve_vbrsr (VTI.Vec MQPR:$Qn), (i32 rGPR:$Rm))),5511 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (i32 rGPR:$Rm)))>;5512 // Predicated5513 def : Pat<(VTI.Vec (int_arm_mve_vbrsr_predicated5514 (VTI.Vec MQPR:$inactive),5515 (VTI.Vec MQPR:$Qn), (i32 rGPR:$Rm),5516 (VTI.Pred VCCR:$mask))),5517 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (i32 rGPR:$Rm),5518 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,5519 (VTI.Vec MQPR:$inactive)))>;5520}5521 5522let Predicates = [HasMVEInt] in {5523 def : Pat<(v16i8 ( bitreverse (v16i8 MQPR:$val1))),5524 (v16i8 ( MVE_VBRSR8 (v16i8 MQPR:$val1), (t2MOVi (i32 8)) ))>;5525 5526 def : Pat<(v4i32 ( bitreverse (v4i32 MQPR:$val1))),5527 (v4i32 ( MVE_VBRSR32 (v4i32 MQPR:$val1), (t2MOVi (i32 32)) ))>;5528 5529 def : Pat<(v8i16 ( bitreverse (v8i16 MQPR:$val1))),5530 (v8i16 ( MVE_VBRSR16 (v8i16 MQPR:$val1), (t2MOVi (i32 16)) ))>;5531 5532 defm : MVE_VBRSR_pat_m<MVE_v16i8, MVE_VBRSR8>;5533 defm : MVE_VBRSR_pat_m<MVE_v8i16, MVE_VBRSR16>;5534 defm : MVE_VBRSR_pat_m<MVE_v4i32, MVE_VBRSR32>;5535}5536 5537let Predicates = [HasMVEFloat] in {5538 defm : MVE_VBRSR_pat_m<MVE_v8f16, MVE_VBRSR16>;5539 defm : MVE_VBRSR_pat_m<MVE_v4f32, MVE_VBRSR32>;5540}5541 5542class MVE_VMUL_qr_int<string iname, string suffix, bits<2> size>5543 : MVE_qDest_rSrc<iname, suffix, "", size> {5544 5545 let Inst{28} = 0b0;5546 let Inst{21-20} = size;5547 let Inst{16} = 0b1;5548 let Inst{12} = 0b1;5549 let Inst{8} = 0b0;5550 let Inst{5} = 0b1;5551 let validForTailPredication = 1;5552}5553 5554multiclass MVE_VMUL_qr_int_m<MVEVectorVTInfo VTI> {5555 def "" : MVE_VMUL_qr_int<"vmul", VTI.Suffix, VTI.Size>;5556 let Predicates = [HasMVEInt] in {5557 defm : MVE_TwoOpPatternDup<VTI, mul, int_arm_mve_mul_predicated, (? ),5558 !cast<Instruction>(NAME), ARMimmOneV>;5559 }5560}5561 5562defm MVE_VMUL_qr_i8 : MVE_VMUL_qr_int_m<MVE_v16i8>;5563defm MVE_VMUL_qr_i16 : MVE_VMUL_qr_int_m<MVE_v8i16>;5564defm MVE_VMUL_qr_i32 : MVE_VMUL_qr_int_m<MVE_v4i32>;5565 5566class MVE_VxxMUL_qr<string iname, string suffix,5567 bit bit_28, bits<2> size, bits<2> vecsize, list<dag> pattern=[]>5568 : MVE_qDest_rSrc<iname, suffix, "", vecsize, pattern> {5569 5570 let Inst{28} = bit_28;5571 let Inst{21-20} = size;5572 let Inst{16} = 0b1;5573 let Inst{12} = 0b0;5574 let Inst{8} = 0b0;5575 let Inst{5} = 0b1;5576 let validForTailPredication = 1;5577}5578 5579multiclass MVE_VxxMUL_qr_m<string iname, MVEVectorVTInfo VTI, bit bit_28,5580 SDPatternOperator Op, Intrinsic int_unpred, Intrinsic int_pred> {5581 def "" : MVE_VxxMUL_qr<iname, VTI.Suffix, bit_28, VTI.Size, VTI.Size>;5582 5583 let Predicates = [HasMVEInt] in {5584 defm : MVE_TwoOpPatternDup<VTI, Op, int_pred, (? ), !cast<Instruction>(NAME)>;5585 }5586 defm : MVE_vec_scalar_int_pat_m<!cast<Instruction>(NAME), VTI, int_unpred, int_pred>;5587}5588 5589multiclass MVE_VQDMULH_qr_m<MVEVectorVTInfo VTI> :5590 MVE_VxxMUL_qr_m<"vqdmulh", VTI, 0b0, MVEvqdmulh,5591 int_arm_mve_vqdmulh, int_arm_mve_qdmulh_predicated>;5592 5593multiclass MVE_VQRDMULH_qr_m<MVEVectorVTInfo VTI> :5594 MVE_VxxMUL_qr_m<"vqrdmulh", VTI, 0b1, null_frag,5595 int_arm_mve_vqrdmulh, int_arm_mve_qrdmulh_predicated>;5596 5597defm MVE_VQDMULH_qr_s8 : MVE_VQDMULH_qr_m<MVE_v16s8>;5598defm MVE_VQDMULH_qr_s16 : MVE_VQDMULH_qr_m<MVE_v8s16>;5599defm MVE_VQDMULH_qr_s32 : MVE_VQDMULH_qr_m<MVE_v4s32>;5600 5601defm MVE_VQRDMULH_qr_s8 : MVE_VQRDMULH_qr_m<MVE_v16s8>;5602defm MVE_VQRDMULH_qr_s16 : MVE_VQRDMULH_qr_m<MVE_v8s16>;5603defm MVE_VQRDMULH_qr_s32 : MVE_VQRDMULH_qr_m<MVE_v4s32>;5604 5605multiclass MVE_VxxMUL_qr_f_m<MVEVectorVTInfo VTI, SDPatternOperator IdentityVec> {5606 let validForTailPredication = 1 in5607 def "" : MVE_VxxMUL_qr<"vmul", VTI.Suffix, VTI.Size{0}, 0b11, VTI.Size>;5608 defm : MVE_TwoOpPatternDup<VTI, vmul, int_arm_mve_mul_predicated, (? ),5609 !cast<Instruction>(NAME), IdentityVec>;5610}5611 5612let Predicates = [HasMVEFloat] in {5613 defm MVE_VMUL_qr_f16 : MVE_VxxMUL_qr_f_m<MVE_v8f16, ARMimmOneH>;5614 defm MVE_VMUL_qr_f32 : MVE_VxxMUL_qr_f_m<MVE_v4f32, ARMimmOneF>;5615}5616 5617class MVE_VFMAMLA_qr<string iname, string suffix,5618 bit bit_28, bits<2> bits_21_20, bit S,5619 bits<2> vecsize, list<dag> pattern=[]>5620 : MVE_qDestSrc_rSrc<iname, suffix, vecsize, pattern> {5621 5622 let Inst{28} = bit_28;5623 let Inst{21-20} = bits_21_20;5624 let Inst{16} = 0b1;5625 let Inst{12} = S;5626 let Inst{8} = 0b0;5627 let Inst{5} = 0b0;5628 let validForTailPredication = 1;5629 let hasSideEffects = 0;5630}5631 5632multiclass MVE_VMLA_qr_multi<string iname, MVEVectorVTInfo VTI,5633 bit scalar_addend> {5634 def "": MVE_VFMAMLA_qr<iname, VTI.Suffix, 0b0, VTI.Size,5635 scalar_addend, VTI.Size>;5636 defvar Inst = !cast<Instruction>(NAME);5637 defvar pred_int = !cast<Intrinsic>("int_arm_mve_" # iname # "_n_predicated");5638 defvar v1 = (VTI.Vec MQPR:$v1);5639 defvar v2 = (VTI.Vec MQPR:$v2);5640 defvar vs = (VTI.Vec (ARMvdup rGPR:$s));5641 defvar s = (i32 rGPR:$s);5642 defvar pred = (VTI.Pred VCCR:$pred);5643 5644 let Predicates = [HasMVEInt] in {5645 if scalar_addend then {5646 def : Pat<(VTI.Vec (add (mul v1, v2), vs)),5647 (VTI.Vec (Inst v1, v2, s))>;5648 } else {5649 def : Pat<(VTI.Vec (add (mul v2, vs), v1)),5650 (VTI.Vec (Inst v1, v2, s))>;5651 }5652 5653 def : Pat<(VTI.Vec (pred_int v1, v2, s, pred)),5654 (VTI.Vec (Inst v1, v2, s, ARMVCCThen, pred, zero_reg))>;5655 }5656}5657 5658defm MVE_VMLA_qr_i8 : MVE_VMLA_qr_multi<"vmla", MVE_v16i8, 0b0>;5659defm MVE_VMLA_qr_i16 : MVE_VMLA_qr_multi<"vmla", MVE_v8i16, 0b0>;5660defm MVE_VMLA_qr_i32 : MVE_VMLA_qr_multi<"vmla", MVE_v4i32, 0b0>;5661 5662defm MVE_VMLAS_qr_i8 : MVE_VMLA_qr_multi<"vmlas", MVE_v16i8, 0b1>;5663defm MVE_VMLAS_qr_i16 : MVE_VMLA_qr_multi<"vmlas", MVE_v8i16, 0b1>;5664defm MVE_VMLAS_qr_i32 : MVE_VMLA_qr_multi<"vmlas", MVE_v4i32, 0b1>;5665 5666multiclass MVE_VFMA_qr_multi<string iname, MVEVectorVTInfo VTI,5667 bit scalar_addend> {5668 def "": MVE_VFMAMLA_qr<iname, VTI.Suffix, VTI.Size{0}, 0b11, scalar_addend, VTI.Size>;5669 defvar Inst = !cast<Instruction>(NAME);5670 defvar pred_int = int_arm_mve_fma_predicated;5671 defvar v1 = (VTI.Vec MQPR:$v1);5672 defvar v2 = (VTI.Vec MQPR:$v2);5673 defvar vs = (VTI.Vec (ARMvdup (i32 rGPR:$s)));5674 defvar is = (i32 rGPR:$s);5675 defvar pred = (VTI.Pred VCCR:$pred);5676 5677 let Predicates = [HasMVEFloat] in {5678 if scalar_addend then {5679 def : Pat<(VTI.Vec (fma v1, v2, vs)),5680 (VTI.Vec (Inst v1, v2, is))>;5681 def : Pat<(VTI.Vec (int_arm_mve_fma v1, v2, vs)),5682 (VTI.Vec (Inst v1, v2, is))>;5683 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),5684 (VTI.Vec (fma v1, v2, vs)),5685 v1)),5686 (VTI.Vec (Inst v1, v2, is, ARMVCCThen, $pred, zero_reg))>;5687 } else {5688 def : Pat<(VTI.Vec (fma v1, vs, v2)),5689 (VTI.Vec (Inst v2, v1, is))>;5690 def : Pat<(VTI.Vec (fma vs, v1, v2)),5691 (VTI.Vec (Inst v2, v1, is))>;5692 def : Pat<(VTI.Vec (int_arm_mve_fma v1, vs, v2)),5693 (VTI.Vec (Inst v2, v1, is))>;5694 def : Pat<(VTI.Vec (int_arm_mve_fma vs, v1, v2)),5695 (VTI.Vec (Inst v2, v1, is))>;5696 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),5697 (VTI.Vec (fma vs, v2, v1)),5698 v1)),5699 (VTI.Vec (Inst v1, v2, is, ARMVCCThen, $pred, zero_reg))>;5700 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),5701 (VTI.Vec (fma v2, vs, v1)),5702 v1)),5703 (VTI.Vec (Inst v1, v2, is, ARMVCCThen, $pred, zero_reg))>;5704 def : Pat<(VTI.Vec (pred_int v1, vs, v2, pred)),5705 (VTI.Vec (Inst v2, v1, is, ARMVCCThen, pred, zero_reg))>;5706 def : Pat<(VTI.Vec (pred_int vs, v1, v2, pred)),5707 (VTI.Vec (Inst v2, v1, is, ARMVCCThen, pred, zero_reg))>;5708 }5709 }5710}5711 5712let Predicates = [HasMVEFloat] in {5713 defm MVE_VFMA_qr_f16 : MVE_VFMA_qr_multi<"vfma", MVE_v8f16, 0>;5714 defm MVE_VFMA_qr_f32 : MVE_VFMA_qr_multi<"vfma", MVE_v4f32, 0>;5715 defm MVE_VFMA_qr_Sf16 : MVE_VFMA_qr_multi<"vfmas", MVE_v8f16, 1>;5716 defm MVE_VFMA_qr_Sf32 : MVE_VFMA_qr_multi<"vfmas", MVE_v4f32, 1>;5717}5718 5719class MVE_VQDMLAH_qr<string iname, string suffix, bit U, bits<2> size,5720 bit bit_5, bit bit_12, list<dag> pattern=[]>5721 : MVE_qDestSrc_rSrc<iname, suffix, size, pattern> {5722 5723 let Inst{28} = U;5724 let Inst{21-20} = size;5725 let Inst{16} = 0b0;5726 let Inst{12} = bit_12;5727 let Inst{8} = 0b0;5728 let Inst{5} = bit_5;5729}5730 5731multiclass MVE_VQDMLAH_qr_multi<string iname, MVEVectorVTInfo VTI,5732 bit bit_5, bit bit_12> {5733 def "": MVE_VQDMLAH_qr<iname, VTI.Suffix, 0b0, VTI.Size, bit_5, bit_12>;5734 defvar Inst = !cast<Instruction>(NAME);5735 defvar unpred_int = !cast<Intrinsic>("int_arm_mve_" # iname);5736 defvar pred_int = !cast<Intrinsic>("int_arm_mve_" # iname # "_predicated");5737 5738 let Predicates = [HasMVEInt] in {5739 def : Pat<(VTI.Vec (unpred_int (VTI.Vec MQPR:$v1), (VTI.Vec MQPR:$v2),5740 (i32 rGPR:$s))),5741 (VTI.Vec (Inst (VTI.Vec MQPR:$v1), (VTI.Vec MQPR:$v2),5742 (i32 rGPR:$s)))>;5743 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$v1), (VTI.Vec MQPR:$v2),5744 (i32 rGPR:$s), (VTI.Pred VCCR:$pred))),5745 (VTI.Vec (Inst (VTI.Vec MQPR:$v1), (VTI.Vec MQPR:$v2),5746 (i32 rGPR:$s), ARMVCCThen,5747 (VTI.Pred VCCR:$pred), zero_reg))>;5748 }5749}5750 5751multiclass MVE_VQDMLAH_qr_types<string iname, bit bit_5, bit bit_12> {5752 defm s8 : MVE_VQDMLAH_qr_multi<iname, MVE_v16s8, bit_5, bit_12>;5753 defm s16 : MVE_VQDMLAH_qr_multi<iname, MVE_v8s16, bit_5, bit_12>;5754 defm s32 : MVE_VQDMLAH_qr_multi<iname, MVE_v4s32, bit_5, bit_12>;5755}5756 5757defm MVE_VQDMLAH_qr : MVE_VQDMLAH_qr_types<"vqdmlah", 0b1, 0b0>;5758defm MVE_VQRDMLAH_qr : MVE_VQDMLAH_qr_types<"vqrdmlah", 0b0, 0b0>;5759defm MVE_VQDMLASH_qr : MVE_VQDMLAH_qr_types<"vqdmlash", 0b1, 0b1>;5760defm MVE_VQRDMLASH_qr : MVE_VQDMLAH_qr_types<"vqrdmlash", 0b0, 0b1>;5761 5762class MVE_VxDUP<string iname, string suffix, bits<2> size, bit bit_12,5763 ValueType VT, SDPatternOperator vxdup>5764 : MVE_p<(outs MQPR:$Qd, tGPREven:$Rn),5765 (ins tGPREven:$Rn_src, MVE_VIDUP_imm:$imm), NoItinerary,5766 iname, suffix, "$Qd, $Rn, $imm", vpred_r, "$Rn = $Rn_src", size,5767 [(set (VT MQPR:$Qd), (i32 tGPREven:$Rn),5768 (vxdup (i32 tGPREven:$Rn_src), (i32 imm:$imm)))]> {5769 bits<4> Qd;5770 bits<4> Rn;5771 bits<2> imm;5772 5773 let Inst{28} = 0b0;5774 let Inst{25-23} = 0b100;5775 let Inst{22} = Qd{3};5776 let Inst{21-20} = size;5777 let Inst{19-17} = Rn{3-1};5778 let Inst{16} = 0b1;5779 let Inst{15-13} = Qd{2-0};5780 let Inst{12} = bit_12;5781 let Inst{11-8} = 0b1111;5782 let Inst{7} = imm{1};5783 let Inst{6-1} = 0b110111;5784 let Inst{0} = imm{0};5785 let validForTailPredication = 1;5786 let hasSideEffects = 0;5787}5788 5789def MVE_VIDUPu8 : MVE_VxDUP<"vidup", "u8", 0b00, 0b0, v16i8, ARMvidup>;5790def MVE_VIDUPu16 : MVE_VxDUP<"vidup", "u16", 0b01, 0b0, v8i16, ARMvidup>;5791def MVE_VIDUPu32 : MVE_VxDUP<"vidup", "u32", 0b10, 0b0, v4i32, ARMvidup>;5792 5793def MVE_VDDUPu8 : MVE_VxDUP<"vddup", "u8", 0b00, 0b1, v16i8, null_frag>;5794def MVE_VDDUPu16 : MVE_VxDUP<"vddup", "u16", 0b01, 0b1, v8i16, null_frag>;5795def MVE_VDDUPu32 : MVE_VxDUP<"vddup", "u32", 0b10, 0b1, v4i32, null_frag>;5796 5797class MVE_VxWDUP<string iname, string suffix, bits<2> size, bit bit_12,5798 list<dag> pattern=[]>5799 : MVE_p<(outs MQPR:$Qd, tGPREven:$Rn),5800 (ins tGPREven:$Rn_src, tGPROdd:$Rm, MVE_VIDUP_imm:$imm), NoItinerary,5801 iname, suffix, "$Qd, $Rn, $Rm, $imm", vpred_r, "$Rn = $Rn_src", size,5802 pattern> {5803 bits<4> Qd;5804 bits<4> Rm;5805 bits<4> Rn;5806 bits<2> imm;5807 5808 let Inst{28} = 0b0;5809 let Inst{25-23} = 0b100;5810 let Inst{22} = Qd{3};5811 let Inst{21-20} = size;5812 let Inst{19-17} = Rn{3-1};5813 let Inst{16} = 0b1;5814 let Inst{15-13} = Qd{2-0};5815 let Inst{12} = bit_12;5816 let Inst{11-8} = 0b1111;5817 let Inst{7} = imm{1};5818 let Inst{6-4} = 0b110;5819 let Inst{3-1} = Rm{3-1};5820 let Inst{0} = imm{0};5821 let validForTailPredication = 1;5822 let hasSideEffects = 0;5823}5824 5825def MVE_VIWDUPu8 : MVE_VxWDUP<"viwdup", "u8", 0b00, 0b0>;5826def MVE_VIWDUPu16 : MVE_VxWDUP<"viwdup", "u16", 0b01, 0b0>;5827def MVE_VIWDUPu32 : MVE_VxWDUP<"viwdup", "u32", 0b10, 0b0>;5828 5829def MVE_VDWDUPu8 : MVE_VxWDUP<"vdwdup", "u8", 0b00, 0b1>;5830def MVE_VDWDUPu16 : MVE_VxWDUP<"vdwdup", "u16", 0b01, 0b1>;5831def MVE_VDWDUPu32 : MVE_VxWDUP<"vdwdup", "u32", 0b10, 0b1>;5832 5833let isReMaterializable = 1 in5834class MVE_VCTPInst<string suffix, bits<2> size, list<dag> pattern=[]>5835 : MVE_p<(outs VCCR:$P0), (ins rGPR:$Rn), NoItinerary, "vctp", suffix,5836 "$Rn", vpred_n, "", size, pattern> {5837 bits<4> Rn;5838 5839 let Inst{28-27} = 0b10;5840 let Inst{26-22} = 0b00000;5841 let Inst{21-20} = size;5842 let Inst{19-16} = Rn{3-0};5843 let Inst{15-11} = 0b11101;5844 let Inst{10-0} = 0b00000000001;5845 let Unpredictable{10-0} = 0b11111111111;5846 5847 let Constraints = "";5848 let DecoderMethod = "DecodeMveVCTP";5849 let validForTailPredication = 1;5850}5851 5852multiclass MVE_VCTP<MVEVectorVTInfo VTI, Intrinsic intr> {5853 def "": MVE_VCTPInst<VTI.BitsSuffix, VTI.Size>;5854 defvar Inst = !cast<Instruction>(NAME);5855 5856 let Predicates = [HasMVEInt] in {5857 def : Pat<(intr rGPR:$Rn),5858 (VTI.Pred (Inst rGPR:$Rn))>;5859 def : Pat<(and (intr rGPR:$Rn), (VTI.Pred VCCR:$mask)),5860 (VTI.Pred (Inst rGPR:$Rn, ARMVCCThen, VCCR:$mask, zero_reg))>;5861 }5862}5863 5864defm MVE_VCTP8 : MVE_VCTP<MVE_v16i8, int_arm_mve_vctp8>;5865defm MVE_VCTP16 : MVE_VCTP<MVE_v8i16, int_arm_mve_vctp16>;5866defm MVE_VCTP32 : MVE_VCTP<MVE_v4i32, int_arm_mve_vctp32>;5867defm MVE_VCTP64 : MVE_VCTP<MVE_v2i64, int_arm_mve_vctp64>;5868 5869// end of mve_qDest_rSrc5870 5871// start of coproc mov5872 5873class MVE_VMOV_64bit<dag oops, dag iops, bit to_qreg, string ops, string cstr>5874 : MVE_VMOV_lane_base<oops, !con(iops, (ins MVEPairVectorIndex2:$idx,5875 MVEPairVectorIndex0:$idx2)),5876 NoItinerary, "vmov", "", ops, cstr, []> {5877 bits<5> Rt;5878 bits<5> Rt2;5879 bits<4> Qd;5880 bit idx;5881 bit idx2;5882 5883 let Inst{31-23} = 0b111011000;5884 let Inst{22} = Qd{3};5885 let Inst{21} = 0b0;5886 let Inst{20} = to_qreg;5887 let Inst{19-16} = Rt2{3-0};5888 let Inst{15-13} = Qd{2-0};5889 let Inst{12-5} = 0b01111000;5890 let Inst{4} = idx2;5891 let Inst{3-0} = Rt{3-0};5892 5893 let VecSize = 0b10;5894 let hasSideEffects = 0;5895}5896 5897// The assembly syntax for these instructions mentions the vector5898// register name twice, e.g.5899//5900// vmov q2[2], q2[0], r0, r15901// vmov r0, r1, q2[2], q2[0]5902//5903// which needs a bit of juggling with MC operand handling.5904//5905// For the move _into_ a vector register, the MC operand list also has5906// to mention the register name twice: once as the output, and once as5907// an extra input to represent where the unchanged half of the output5908// register comes from (when this instruction is used in code5909// generation). So we arrange that the first mention of the vector reg5910// in the instruction is considered by the AsmMatcher to be the output5911// ($Qd), and the second one is the input ($QdSrc). Binding them5912// together with the existing 'tie' constraint is enough to enforce at5913// register allocation time that they have to be the same register.5914//5915// For the move _from_ a vector register, there's no way to get round5916// the fact that both instances of that register name have to be5917// inputs. They have to be the same register again, but this time, we5918// can't use a tie constraint, because that has to be between an5919// output and an input operand. So this time, we have to arrange that5920// the q-reg appears just once in the MC operand list, in spite of5921// being mentioned twice in the asm syntax - which needs a custom5922// AsmMatchConverter.5923 5924def MVE_VMOV_q_rr : MVE_VMOV_64bit<(outs MQPR:$Qd),5925 (ins MQPR:$QdSrc, rGPR:$Rt, rGPR:$Rt2),5926 0b1, "$Qd$idx, $QdSrc$idx2, $Rt, $Rt2",5927 "$Qd = $QdSrc"> {5928 let DecoderMethod = "DecodeMVEVMOVDRegtoQ";5929}5930 5931def MVE_VMOV_rr_q : MVE_VMOV_64bit<(outs rGPR:$Rt, rGPR:$Rt2), (ins MQPR:$Qd),5932 0b0, "$Rt, $Rt2, $Qd$idx, $Qd$idx2", ""> {5933 let DecoderMethod = "DecodeMVEVMOVQtoDReg";5934 let AsmMatchConverter = "cvtMVEVMOVQtoDReg";5935}5936 5937let Predicates = [HasMVEInt] in {5938 // Double lane moves. There are a number of patterns here. We know that the5939 // insertelt's will be in descending order by index, and need to match the 55940 // patterns that might contain 2-0 or 3-1 pairs. These are:5941 // 3 2 1 0 -> vmovqrr 31; vmovqrr 205942 // 3 2 1 -> vmovqrr 31; vmov 25943 // 3 1 -> vmovqrr 315944 // 2 1 0 -> vmovqrr 20; vmov 15945 // 2 0 -> vmovqrr 205946 // The other potential patterns will be handled by single lane inserts.5947 def : Pat<(insertelt (insertelt (insertelt (insertelt (v4i32 MQPR:$src1),5948 rGPR:$srcA, (i32 0)),5949 rGPR:$srcB, (i32 1)),5950 rGPR:$srcC, (i32 2)),5951 rGPR:$srcD, (i32 3)),5952 (MVE_VMOV_q_rr (MVE_VMOV_q_rr MQPR:$src1, rGPR:$srcA, rGPR:$srcC, (i32 2), (i32 0)),5953 rGPR:$srcB, rGPR:$srcD, (i32 3), (i32 1))>;5954 def : Pat<(insertelt (insertelt (insertelt (v4i32 MQPR:$src1),5955 rGPR:$srcB, (i32 1)),5956 rGPR:$srcC, (i32 2)),5957 rGPR:$srcD, (i32 3)),5958 (MVE_VMOV_q_rr (MVE_VMOV_to_lane_32 MQPR:$src1, rGPR:$srcC, (i32 2)),5959 rGPR:$srcB, rGPR:$srcD, (i32 3), (i32 1))>;5960 def : Pat<(insertelt (insertelt (v4i32 MQPR:$src1), rGPR:$srcA, (i32 1)), rGPR:$srcB, (i32 3)),5961 (MVE_VMOV_q_rr MQPR:$src1, rGPR:$srcA, rGPR:$srcB, (i32 3), (i32 1))>;5962 def : Pat<(insertelt (insertelt (insertelt (v4i32 MQPR:$src1),5963 rGPR:$srcB, (i32 0)),5964 rGPR:$srcC, (i32 1)),5965 rGPR:$srcD, (i32 2)),5966 (MVE_VMOV_q_rr (MVE_VMOV_to_lane_32 MQPR:$src1, rGPR:$srcC, (i32 1)),5967 rGPR:$srcB, rGPR:$srcD, (i32 2), (i32 0))>;5968 def : Pat<(insertelt (insertelt (v4i32 MQPR:$src1), rGPR:$srcA, (i32 0)), rGPR:$srcB, (i32 2)),5969 (MVE_VMOV_q_rr MQPR:$src1, rGPR:$srcA, rGPR:$srcB, (i32 2), (i32 0))>;5970}5971 5972// end of coproc mov5973 5974// start of MVE interleaving load/store5975 5976// Base class for the family of interleaving/deinterleaving5977// load/stores with names like VLD20.8 and VST43.32.5978class MVE_vldst24_base<bit writeback, bit fourregs, bits<2> stage, bits<2> size,5979 bit load, dag Oops, dag loadIops, dag wbIops,5980 string iname, string ops,5981 string cstr, list<dag> pattern=[]>5982 : MVE_MI<Oops, !con(loadIops, wbIops), NoItinerary, iname, ops, cstr, size, pattern> {5983 bits<4> VQd;5984 bits<4> Rn;5985 5986 let Inst{31-22} = 0b1111110010;5987 let Inst{21} = writeback;5988 let Inst{20} = load;5989 let Inst{19-16} = Rn;5990 let Inst{15-13} = VQd{2-0};5991 let Inst{12-9} = 0b1111;5992 let Inst{8-7} = size;5993 let Inst{6-5} = stage;5994 let Inst{4-1} = 0b0000;5995 let Inst{0} = fourregs;5996 5997 let mayLoad = load;5998 let mayStore = !eq(load,0);5999 let hasSideEffects = 0;6000 let validForTailPredication = load;6001}6002 6003// A parameter class used to encapsulate all the ways the writeback6004// variants of VLD20 and friends differ from the non-writeback ones.6005class MVE_vldst24_writeback<bit b, dag Oo, dag Io,6006 string sy="", string c="", string n=""> {6007 bit writeback = b;6008 dag Oops = Oo;6009 dag Iops = Io;6010 string syntax = sy;6011 string cstr = c;6012 string id_suffix = n;6013}6014 6015// Another parameter class that encapsulates the differences between VLD2x6016// and VLD4x.6017class MVE_vldst24_nvecs<int n, list<int> s, bit b, RegisterOperand vl> {6018 int nvecs = n;6019 list<int> stages = s;6020 bit bit0 = b;6021 RegisterOperand VecList = vl;6022}6023 6024// A third parameter class that distinguishes VLDnn.8 from .16 from .32.6025class MVE_vldst24_lanesize<int i, bits<2> b> {6026 int lanesize = i;6027 bits<2> sizebits = b;6028}6029 6030// A base class for each direction of transfer: one for load, one for6031// store. I can't make these a fourth independent parametric tuple6032// class, because they have to take the nvecs tuple class as a6033// parameter, in order to find the right VecList operand type.6034 6035class MVE_vld24_base<MVE_vldst24_nvecs n, bits<2> pat, bits<2> size,6036 MVE_vldst24_writeback wb, string iname,6037 list<dag> pattern=[]>6038 : MVE_vldst24_base<wb.writeback, n.bit0, pat, size, 1,6039 !con((outs n.VecList:$VQd), wb.Oops),6040 (ins n.VecList:$VQdSrc), wb.Iops,6041 iname, "$VQd, $Rn" # wb.syntax,6042 wb.cstr # ",$VQdSrc = $VQd", pattern>;6043 6044class MVE_vst24_base<MVE_vldst24_nvecs n, bits<2> pat, bits<2> size,6045 MVE_vldst24_writeback wb, string iname,6046 list<dag> pattern=[]>6047 : MVE_vldst24_base<wb.writeback, n.bit0, pat, size, 0,6048 wb.Oops, (ins n.VecList:$VQd), wb.Iops,6049 iname, "$VQd, $Rn" # wb.syntax,6050 wb.cstr, pattern>;6051 6052// Actually define all the interleaving loads and stores, by a series6053// of nested foreaches over number of vectors (VLD2/VLD4); stage6054// within one of those series (VLDx0/VLDx1/VLDx2/VLDx3); size of6055// vector lane; writeback or no writeback.6056foreach n = [MVE_vldst24_nvecs<2, [0,1], 0, VecList2Q>,6057 MVE_vldst24_nvecs<4, [0,1,2,3], 1, VecList4Q>] in6058foreach stage = n.stages in6059foreach s = [MVE_vldst24_lanesize< 8, 0b00>,6060 MVE_vldst24_lanesize<16, 0b01>,6061 MVE_vldst24_lanesize<32, 0b10>] in6062foreach wb = [MVE_vldst24_writeback<6063 1, (outs rGPR:$wb), (ins t2_nosp_addr_offset_none:$Rn),6064 "!", "$Rn.base = $wb", "_wb">,6065 MVE_vldst24_writeback<0, (outs), (ins t2_addr_offset_none:$Rn)>] in {6066 6067 // For each case within all of those foreaches, define the actual6068 // instructions. The def names are made by gluing together pieces6069 // from all the parameter classes, and will end up being things like6070 // MVE_VLD20_8 and MVE_VST43_16_wb.6071 6072 def "MVE_VLD" # n.nvecs # stage # "_" # s.lanesize # wb.id_suffix6073 : MVE_vld24_base<n, stage, s.sizebits, wb,6074 "vld" # n.nvecs # stage # "." # s.lanesize>;6075 6076 def "MVE_VST" # n.nvecs # stage # "_" # s.lanesize # wb.id_suffix6077 : MVE_vst24_base<n, stage, s.sizebits, wb,6078 "vst" # n.nvecs # stage # "." # s.lanesize>;6079}6080 6081def SDTARMVST2 : SDTypeProfile<1, 5, [SDTCisPtrTy<0>, SDTCisPtrTy<1>, SDTCisVT<2, i32>, SDTCisVec<3>,6082 SDTCisSameAs<3, 4>, SDTCisVT<5, i32>]>;6083def SDTARMVST4 : SDTypeProfile<1, 7, [SDTCisPtrTy<0>, SDTCisPtrTy<1>, SDTCisVT<2, i32>, SDTCisVec<3>,6084 SDTCisSameAs<3, 4>, SDTCisSameAs<3, 5>,6085 SDTCisSameAs<3, 6>, SDTCisVT<7, i32>]>;6086def MVEVST2UPD : SDNode<"ARMISD::VST2_UPD", SDTARMVST2, [SDNPHasChain, SDNPMemOperand]>;6087def MVEVST4UPD : SDNode<"ARMISD::VST4_UPD", SDTARMVST4, [SDNPHasChain, SDNPMemOperand]>;6088 6089multiclass MVE_vst24_patterns<int lanesize, ValueType VT> {6090 foreach stage = [0,1] in6091 def : Pat<(int_arm_mve_vst2q i32:$addr,6092 (VT MQPR:$v0), (VT MQPR:$v1), (i32 stage)),6093 (!cast<Instruction>("MVE_VST2"#stage#"_"#lanesize)6094 (REG_SEQUENCE MQQPR, VT:$v0, qsub_0, VT:$v1, qsub_1),6095 t2_addr_offset_none:$addr)>;6096 foreach stage = [0,1] in6097 def : Pat<(i32 (MVEVST2UPD i32:$addr, (i32 32),6098 (VT MQPR:$v0), (VT MQPR:$v1), (i32 stage))),6099 (i32 (!cast<Instruction>("MVE_VST2"#stage#"_"#lanesize#_wb)6100 (REG_SEQUENCE MQQPR, VT:$v0, qsub_0, VT:$v1, qsub_1),6101 t2_addr_offset_none:$addr))>;6102 6103 foreach stage = [0,1,2,3] in6104 def : Pat<(int_arm_mve_vst4q i32:$addr,6105 (VT MQPR:$v0), (VT MQPR:$v1),6106 (VT MQPR:$v2), (VT MQPR:$v3), (i32 stage)),6107 (!cast<Instruction>("MVE_VST4"#stage#"_"#lanesize)6108 (REG_SEQUENCE MQQQQPR, VT:$v0, qsub_0, VT:$v1, qsub_1,6109 VT:$v2, qsub_2, VT:$v3, qsub_3),6110 t2_addr_offset_none:$addr)>;6111 foreach stage = [0,1,2,3] in6112 def : Pat<(i32 (MVEVST4UPD i32:$addr, (i32 64),6113 (VT MQPR:$v0), (VT MQPR:$v1),6114 (VT MQPR:$v2), (VT MQPR:$v3), (i32 stage))),6115 (i32 (!cast<Instruction>("MVE_VST4"#stage#"_"#lanesize#_wb)6116 (REG_SEQUENCE MQQQQPR, VT:$v0, qsub_0, VT:$v1, qsub_1,6117 VT:$v2, qsub_2, VT:$v3, qsub_3),6118 t2_addr_offset_none:$addr))>;6119}6120defm : MVE_vst24_patterns<8, v16i8>;6121defm : MVE_vst24_patterns<16, v8i16>;6122defm : MVE_vst24_patterns<32, v4i32>;6123defm : MVE_vst24_patterns<16, v8f16>;6124defm : MVE_vst24_patterns<32, v4f32>;6125 6126// end of MVE interleaving load/store6127 6128// start of MVE predicable load/store6129 6130// A parameter class for the direction of transfer.6131class MVE_ldst_direction<bit b, dag Oo, dag Io, string c=""> {6132 bit load = b;6133 dag Oops = Oo;6134 dag Iops = Io;6135 string cstr = c;6136}6137def MVE_ld: MVE_ldst_direction<1, (outs MQPR:$Qd), (ins), ",@earlyclobber $Qd">;6138def MVE_st: MVE_ldst_direction<0, (outs), (ins MQPR:$Qd)>;6139 6140// A parameter class for the size of memory access in a load.6141class MVE_memsz<bits<2> e, int s, AddrMode m, string mn, list<string> types> {6142 bits<2> encoding = e; // opcode bit(s) for encoding6143 int shift = s; // shift applied to immediate load offset6144 AddrMode AM = m;6145 6146 // For instruction aliases: define the complete list of type6147 // suffixes at this size, and the canonical ones for loads and6148 // stores.6149 string MnemonicLetter = mn;6150 int TypeBits = !shl(8, s);6151 string CanonLoadSuffix = ".u" # TypeBits;6152 string CanonStoreSuffix = "." # TypeBits;6153 list<string> suffixes = !foreach(letter, types, "." # letter # TypeBits);6154}6155 6156// Instances of MVE_memsz.6157//6158// (memD doesn't need an AddrMode, because those are only for6159// contiguous loads, and memD is only used by gather/scatters.)6160def MVE_memB: MVE_memsz<0b00, 0, AddrModeT2_i7, "b", ["", "u", "s"]>;6161def MVE_memH: MVE_memsz<0b01, 1, AddrModeT2_i7s2, "h", ["", "u", "s", "f"]>;6162def MVE_memW: MVE_memsz<0b10, 2, AddrModeT2_i7s4, "w", ["", "u", "s", "f"]>;6163def MVE_memD: MVE_memsz<0b11, 3, ?, "d", ["", "u", "s", "f"]>;6164 6165// This is the base class for all the MVE loads and stores other than6166// the interleaving ones. All the non-interleaving loads/stores share6167// the characteristic that they operate on just one vector register,6168// so they are VPT-predicable.6169//6170// The predication operand is vpred_n, for both loads and stores. For6171// store instructions, the reason is obvious: if there is no output6172// register, there can't be a need for an input parameter giving the6173// output register's previous value. Load instructions also don't need6174// that input parameter, because unlike MVE data processing6175// instructions, predicated loads are defined to set the inactive6176// lanes of the output register to zero, instead of preserving their6177// input values.6178class MVE_VLDRSTR_base<MVE_ldst_direction dir, bit U, bit P, bit W, bit opc,6179 dag oops, dag iops, string asm, string suffix,6180 string ops, string cstr, bits<2> vecsize, list<dag> pattern=[]>6181 : MVE_p<oops, iops, NoItinerary, asm, suffix, ops, vpred_n, cstr, vecsize, pattern> {6182 bits<3> Qd;6183 6184 let Inst{28} = U;6185 let Inst{25} = 0b0;6186 let Inst{24} = P;6187 let Inst{22} = 0b0;6188 let Inst{21} = W;6189 let Inst{20} = dir.load;6190 let Inst{15-13} = Qd{2-0};6191 let Inst{12} = opc;6192 let Inst{11-9} = 0b111;6193 6194 let mayLoad = dir.load;6195 let mayStore = !eq(dir.load,0);6196 let hasSideEffects = 0;6197 let validForTailPredication = 1;6198}6199 6200// Contiguous load and store instructions. These come in two main6201// categories: same-size loads/stores in which 128 bits of vector6202// register is transferred to or from 128 bits of memory in the most6203// obvious way, and widening loads / narrowing stores, in which the6204// size of memory accessed is less than the size of a vector register,6205// so the load instructions sign- or zero-extend each memory value6206// into a wider vector lane, and the store instructions truncate6207// correspondingly.6208//6209// The instruction mnemonics for these two classes look reasonably6210// similar, but the actual encodings are different enough to need two6211// separate base classes.6212 6213// Contiguous, same size6214class MVE_VLDRSTR_cs<MVE_ldst_direction dir, MVE_memsz memsz, bit P, bit W,6215 dag oops, dag iops, string asm, string suffix,6216 IndexMode im, string ops, string cstr>6217 : MVE_VLDRSTR_base<dir, 0, P, W, 1, oops, iops, asm, suffix, ops, cstr, memsz.encoding> {6218 bits<12> addr;6219 let Inst{23} = addr{7};6220 let Inst{19-16} = addr{11-8};6221 let Inst{8-7} = memsz.encoding;6222 let Inst{6-0} = addr{6-0};6223 6224 let IM = im;6225}6226 6227// Contiguous, widening/narrowing6228class MVE_VLDRSTR_cw<MVE_ldst_direction dir, MVE_memsz memsz, bit U,6229 bit P, bit W, bits<2> size, dag oops, dag iops,6230 string asm, string suffix, IndexMode im,6231 string ops, string cstr>6232 : MVE_VLDRSTR_base<dir, U, P, W, 0, oops, iops, asm, suffix, ops, cstr, size> {6233 bits<11> addr;6234 let Inst{23} = addr{7};6235 let Inst{19} = memsz.encoding{0}; // enough to tell 16- from 32-bit6236 let Inst{18-16} = addr{10-8};6237 let Inst{8-7} = size;6238 let Inst{6-0} = addr{6-0};6239 6240 let IM = im;6241}6242 6243// Multiclass wrapper on each of the _cw and _cs base classes, to6244// generate three writeback modes (none, preindex, postindex).6245 6246multiclass MVE_VLDRSTR_cw_m<MVE_ldst_direction dir, MVE_memsz memsz,6247 string asm, string suffix, bit U, bits<2> size> {6248 let AM = memsz.AM in {6249 def "" : MVE_VLDRSTR_cw<6250 dir, memsz, U, 1, 0, size,6251 dir.Oops, !con(dir.Iops, (ins taddrmode_imm7<memsz.shift>:$addr)),6252 asm, suffix, IndexModeNone, "$Qd, $addr", "">;6253 6254 def _pre : MVE_VLDRSTR_cw<6255 dir, memsz, U, 1, 1, size,6256 !con((outs tGPR:$wb), dir.Oops),6257 !con(dir.Iops, (ins taddrmode_imm7<memsz.shift>:$addr)),6258 asm, suffix, IndexModePre, "$Qd, $addr!", "$addr.base = $wb"> {6259 let DecoderMethod = "DecodeMVE_MEM_1_pre<"#memsz.shift#">";6260 }6261 6262 def _post : MVE_VLDRSTR_cw<6263 dir, memsz, U, 0, 1, size,6264 !con((outs tGPR:$wb), dir.Oops),6265 !con(dir.Iops, (ins t_addr_offset_none:$Rn,6266 t2am_imm7_offset<memsz.shift>:$addr)),6267 asm, suffix, IndexModePost, "$Qd, $Rn$addr", "$Rn.base = $wb"> {6268 bits<4> Rn;6269 let Inst{18-16} = Rn{2-0};6270 }6271 }6272}6273 6274multiclass MVE_VLDRSTR_cs_m<MVE_ldst_direction dir, MVE_memsz memsz,6275 string asm, string suffix> {6276 let AM = memsz.AM in {6277 def "" : MVE_VLDRSTR_cs<6278 dir, memsz, 1, 0,6279 dir.Oops, !con(dir.Iops, (ins t2addrmode_imm7<memsz.shift>:$addr)),6280 asm, suffix, IndexModeNone, "$Qd, $addr", "">;6281 6282 def _pre : MVE_VLDRSTR_cs<6283 dir, memsz, 1, 1,6284 !con((outs rGPR:$wb), dir.Oops),6285 !con(dir.Iops, (ins t2addrmode_imm7_pre<memsz.shift>:$addr)),6286 asm, suffix, IndexModePre, "$Qd, $addr!", "$addr.base = $wb"> {6287 let DecoderMethod = "DecodeMVE_MEM_2_pre<"#memsz.shift#">";6288 }6289 6290 def _post : MVE_VLDRSTR_cs<6291 dir, memsz, 0, 1,6292 !con((outs rGPR:$wb), dir.Oops),6293 !con(dir.Iops, (ins t2_nosp_addr_offset_none:$Rn,6294 t2am_imm7_offset<memsz.shift>:$addr)),6295 asm, suffix, IndexModePost, "$Qd, $Rn$addr", "$Rn.base = $wb"> {6296 bits<4> Rn;6297 let Inst{19-16} = Rn{3-0};6298 }6299 }6300}6301 6302// Now actually declare all the contiguous load/stores, via those6303// multiclasses. The instruction ids coming out of this are the bare6304// names shown in the defm, with _pre or _post appended for writeback,6305// e.g. MVE_VLDRBS16, MVE_VSTRB16_pre, MVE_VSTRHU16_post.6306 6307defm MVE_VLDRBS16: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memB, "vldrb", "s16", 0, 0b01>;6308defm MVE_VLDRBS32: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memB, "vldrb", "s32", 0, 0b10>;6309defm MVE_VLDRBU16: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memB, "vldrb", "u16", 1, 0b01>;6310defm MVE_VLDRBU32: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memB, "vldrb", "u32", 1, 0b10>;6311defm MVE_VLDRHS32: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memH, "vldrh", "s32", 0, 0b10>;6312defm MVE_VLDRHU32: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memH, "vldrh", "u32", 1, 0b10>;6313 6314defm MVE_VLDRBU8: MVE_VLDRSTR_cs_m<MVE_ld, MVE_memB, "vldrb", "u8">;6315defm MVE_VLDRHU16: MVE_VLDRSTR_cs_m<MVE_ld, MVE_memH, "vldrh", "u16">;6316defm MVE_VLDRWU32: MVE_VLDRSTR_cs_m<MVE_ld, MVE_memW, "vldrw", "u32">;6317 6318defm MVE_VSTRB16: MVE_VLDRSTR_cw_m<MVE_st, MVE_memB, "vstrb", "16", 0, 0b01>;6319defm MVE_VSTRB32: MVE_VLDRSTR_cw_m<MVE_st, MVE_memB, "vstrb", "32", 0, 0b10>;6320defm MVE_VSTRH32: MVE_VLDRSTR_cw_m<MVE_st, MVE_memH, "vstrh", "32", 0, 0b10>;6321 6322defm MVE_VSTRBU8 : MVE_VLDRSTR_cs_m<MVE_st, MVE_memB, "vstrb", "8">;6323defm MVE_VSTRHU16: MVE_VLDRSTR_cs_m<MVE_st, MVE_memH, "vstrh", "16">;6324defm MVE_VSTRWU32: MVE_VLDRSTR_cs_m<MVE_st, MVE_memW, "vstrw", "32">;6325 6326// Gather loads / scatter stores whose address operand is of the form6327// [Rn,Qm], i.e. a single GPR as the common base address, plus a6328// vector of offset from it. ('Load/store this sequence of elements of6329// the same array.')6330//6331// Like the contiguous family, these loads and stores can widen the6332// loaded values / truncate the stored ones, or they can just6333// load/store the same size of memory and vector lane. But unlike the6334// contiguous family, there's no particular difference in encoding6335// between those two cases.6336//6337// This family also comes with the option to scale the offset values6338// in Qm by the size of the loaded memory (i.e. to treat them as array6339// indices), or not to scale them (to treat them as plain byte offsets6340// in memory, so that perhaps the loaded values are unaligned). The6341// scaled instructions' address operand in assembly looks like6342// [Rn,Qm,UXTW #2] or similar.6343 6344// Base class.6345class MVE_VLDRSTR_rq<MVE_ldst_direction dir, MVE_memsz memsz, bit U,6346 bits<2> size, bit os, string asm, string suffix, int shift>6347 : MVE_VLDRSTR_base<dir, U, 0b0, 0b0, 0, dir.Oops,6348 !con(dir.Iops, (ins mve_addr_rq_shift<shift>:$addr)),6349 asm, suffix, "$Qd, $addr", dir.cstr, size> {6350 bits<7> addr;6351 let Inst{23} = 0b1;6352 let Inst{19-16} = addr{6-3};6353 let Inst{8-7} = size;6354 let Inst{6} = memsz.encoding{1};6355 let Inst{5} = 0;6356 let Inst{4} = memsz.encoding{0};6357 let Inst{3-1} = addr{2-0};6358 let Inst{0} = os;6359}6360 6361// Multiclass that defines the scaled and unscaled versions of an6362// instruction, when the memory size is wider than a byte. The scaled6363// version gets the default name like MVE_VLDRBU16_rq; the unscaled /6364// potentially unaligned version gets a "_u" suffix, e.g.6365// MVE_VLDRBU16_rq_u.6366multiclass MVE_VLDRSTR_rq_w<MVE_ldst_direction dir, MVE_memsz memsz,6367 string asm, string suffix, bit U, bits<2> size> {6368 def _u : MVE_VLDRSTR_rq<dir, memsz, U, size, 0, asm, suffix, 0>;6369 def "" : MVE_VLDRSTR_rq<dir, memsz, U, size, 1, asm, suffix, memsz.shift>;6370}6371 6372// Subclass of MVE_VLDRSTR_rq with the same API as that multiclass,6373// for use when the memory size is one byte, so there's no 'scaled'6374// version of the instruction at all. (This is encoded as if it were6375// unscaled, but named in the default way with no _u suffix.)6376class MVE_VLDRSTR_rq_b<MVE_ldst_direction dir, MVE_memsz memsz,6377 string asm, string suffix, bit U, bits<2> size>6378 : MVE_VLDRSTR_rq<dir, memsz, U, size, 0, asm, suffix, 0>;6379 6380// Multiclasses wrapping that to add ISel patterns for intrinsics.6381multiclass MVE_VLDR_rq_w<MVE_memsz memsz, list<MVEVectorVTInfo> VTIs> {6382 defm "": MVE_VLDRSTR_rq_w<MVE_ld, memsz, "vldr" # memsz.MnemonicLetter,6383 VTIs[0].Suffix, VTIs[0].Unsigned, VTIs[0].Size>;6384 defvar Inst = !cast<Instruction>(NAME);6385 defvar InstU = !cast<Instruction>(NAME # "_u");6386 6387 foreach VTI = VTIs in6388 foreach UnsignedFlag = !if(!eq(VTI.Size, memsz.encoding),6389 [0,1], [VTI.Unsigned]) in {6390 def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), memsz.TypeBits, 0, UnsignedFlag)),6391 (VTI.Vec (InstU GPR:$base, MQPR:$offsets))>;6392 def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), memsz.TypeBits, memsz.shift, UnsignedFlag)),6393 (VTI.Vec (Inst GPR:$base, MQPR:$offsets))>;6394 def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), memsz.TypeBits, 0, UnsignedFlag, (VTI.Pred VCCR:$pred))),6395 (VTI.Vec (InstU GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred, zero_reg))>;6396 def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), memsz.TypeBits, memsz.shift, UnsignedFlag, (VTI.Pred VCCR:$pred))),6397 (VTI.Vec (Inst GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred, zero_reg))>;6398 }6399}6400multiclass MVE_VLDR_rq_b<list<MVEVectorVTInfo> VTIs> {6401 def "": MVE_VLDRSTR_rq_b<MVE_ld, MVE_memB, "vldrb",6402 VTIs[0].Suffix, VTIs[0].Unsigned, VTIs[0].Size>;6403 defvar Inst = !cast<Instruction>(NAME);6404 6405 foreach VTI = VTIs in {6406 def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), 8, 0, VTI.Unsigned)),6407 (VTI.Vec (Inst GPR:$base, MQPR:$offsets))>;6408 def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), 8, 0, VTI.Unsigned, (VTI.Pred VCCR:$pred))),6409 (VTI.Vec (Inst GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred, zero_reg))>;6410 }6411}6412multiclass MVE_VSTR_rq_w<MVE_memsz memsz, list<MVEVectorVTInfo> VTIs> {6413 defm "": MVE_VLDRSTR_rq_w<MVE_st, memsz, "vstr" # memsz.MnemonicLetter,6414 VTIs[0].BitsSuffix, 0, VTIs[0].Size>;6415 defvar Inst = !cast<Instruction>(NAME);6416 defvar InstU = !cast<Instruction>(NAME # "_u");6417 6418 foreach VTI = VTIs in {6419 def : Pat<(int_arm_mve_vstr_scatter_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), memsz.TypeBits, 0),6420 (InstU MQPR:$data, GPR:$base, MQPR:$offsets)>;6421 def : Pat<(int_arm_mve_vstr_scatter_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), memsz.TypeBits, memsz.shift),6422 (Inst MQPR:$data, GPR:$base, MQPR:$offsets)>;6423 def : Pat<(int_arm_mve_vstr_scatter_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), memsz.TypeBits, 0, (VTI.Pred VCCR:$pred)),6424 (InstU MQPR:$data, GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred, zero_reg)>;6425 def : Pat<(int_arm_mve_vstr_scatter_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), memsz.TypeBits, memsz.shift, (VTI.Pred VCCR:$pred)),6426 (Inst MQPR:$data, GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred, zero_reg)>;6427 }6428}6429multiclass MVE_VSTR_rq_b<list<MVEVectorVTInfo> VTIs> {6430 def "": MVE_VLDRSTR_rq_b<MVE_st, MVE_memB, "vstrb",6431 VTIs[0].BitsSuffix, 0, VTIs[0].Size>;6432 defvar Inst = !cast<Instruction>(NAME);6433 6434 foreach VTI = VTIs in {6435 def : Pat<(int_arm_mve_vstr_scatter_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), 8, 0),6436 (Inst MQPR:$data, GPR:$base, MQPR:$offsets)>;6437 def : Pat<(int_arm_mve_vstr_scatter_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), 8, 0, (VTI.Pred VCCR:$pred)),6438 (Inst MQPR:$data, GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred, zero_reg)>;6439 }6440}6441 6442// Actually define all the loads and stores in this family.6443 6444defm MVE_VLDRBU8_rq : MVE_VLDR_rq_b<[MVE_v16u8,MVE_v16s8]>;6445defm MVE_VLDRBU16_rq: MVE_VLDR_rq_b<[MVE_v8u16]>;6446defm MVE_VLDRBS16_rq: MVE_VLDR_rq_b<[MVE_v8s16]>;6447defm MVE_VLDRBU32_rq: MVE_VLDR_rq_b<[MVE_v4u32]>;6448defm MVE_VLDRBS32_rq: MVE_VLDR_rq_b<[MVE_v4s32]>;6449 6450defm MVE_VLDRHU16_rq: MVE_VLDR_rq_w<MVE_memH, [MVE_v8u16,MVE_v8s16,MVE_v8f16]>;6451defm MVE_VLDRHU32_rq: MVE_VLDR_rq_w<MVE_memH, [MVE_v4u32]>;6452defm MVE_VLDRHS32_rq: MVE_VLDR_rq_w<MVE_memH, [MVE_v4s32]>;6453defm MVE_VLDRWU32_rq: MVE_VLDR_rq_w<MVE_memW, [MVE_v4u32,MVE_v4s32,MVE_v4f32]>;6454defm MVE_VLDRDU64_rq: MVE_VLDR_rq_w<MVE_memD, [MVE_v2u64,MVE_v2s64]>;6455 6456defm MVE_VSTRB8_rq : MVE_VSTR_rq_b<[MVE_v16i8]>;6457defm MVE_VSTRB16_rq : MVE_VSTR_rq_b<[MVE_v8i16]>;6458defm MVE_VSTRB32_rq : MVE_VSTR_rq_b<[MVE_v4i32]>;6459 6460defm MVE_VSTRH16_rq : MVE_VSTR_rq_w<MVE_memH, [MVE_v8i16,MVE_v8f16]>;6461defm MVE_VSTRH32_rq : MVE_VSTR_rq_w<MVE_memH, [MVE_v4i32]>;6462defm MVE_VSTRW32_rq : MVE_VSTR_rq_w<MVE_memW, [MVE_v4i32,MVE_v4f32]>;6463defm MVE_VSTRD64_rq : MVE_VSTR_rq_w<MVE_memD, [MVE_v2i64]>;6464 6465// Gather loads / scatter stores whose address operand is of the form6466// [Qm,#imm], i.e. a vector containing a full base address for each6467// loaded item, plus an immediate offset applied consistently to all6468// of them. ('Load/store the same field from this vector of pointers6469// to a structure type.')6470//6471// This family requires the vector lane size to be at least 32 bits6472// (so there's room for an address in each lane at all). It has no6473// widening/narrowing variants. But it does support preindex6474// writeback, in which the address vector is updated to hold the6475// addresses actually loaded from.6476 6477// Base class.6478class MVE_VLDRSTR_qi<MVE_ldst_direction dir, MVE_memsz memsz, bit W, dag wbops,6479 string asm, string wbAsm, string suffix, string cstr = "">6480 : MVE_VLDRSTR_base<dir, 1, 1, W, 1, !con(wbops, dir.Oops),6481 !con(dir.Iops, (ins mve_addr_q_shift<memsz.shift>:$addr)),6482 asm, suffix, "$Qd, $addr" # wbAsm, cstr # dir.cstr, memsz.encoding> {6483 bits<11> addr;6484 let Inst{23} = addr{7};6485 let Inst{19-17} = addr{10-8};6486 let Inst{16} = 0;6487 let Inst{8} = memsz.encoding{0}; // enough to distinguish 32- from 64-bit6488 let Inst{7} = 0;6489 let Inst{6-0} = addr{6-0};6490}6491 6492// Multiclass that generates the non-writeback and writeback variants.6493multiclass MVE_VLDRSTR_qi_m<MVE_ldst_direction dir, MVE_memsz memsz,6494 string asm, string suffix> {6495 def "" : MVE_VLDRSTR_qi<dir, memsz, 0, (outs), asm, "", suffix>;6496 def _pre : MVE_VLDRSTR_qi<dir, memsz, 1, (outs MQPR:$wb), asm, "!", suffix,6497 "$addr.base = $wb"> {6498 let DecoderMethod="DecodeMVE_MEM_3_pre<"#memsz.shift#">";6499 }6500}6501 6502// Multiclasses wrapping that one, adding selection patterns for the6503// non-writeback loads and all the stores. (The writeback loads must6504// deliver multiple output values, so they have to be selected by C++6505// code.)6506multiclass MVE_VLDR_qi<MVE_memsz memsz, MVEVectorVTInfo AVTI,6507 list<MVEVectorVTInfo> DVTIs> {6508 defm "" : MVE_VLDRSTR_qi_m<MVE_ld, memsz, "vldr" # memsz.MnemonicLetter,6509 "u" # memsz.TypeBits>;6510 defvar Inst = !cast<Instruction>(NAME);6511 6512 foreach DVTI = DVTIs in {6513 def : Pat<(DVTI.Vec (int_arm_mve_vldr_gather_base6514 (AVTI.Vec MQPR:$addr), (i32 imm:$offset))),6515 (DVTI.Vec (Inst (AVTI.Vec MQPR:$addr), (i32 imm:$offset)))>;6516 def : Pat<(DVTI.Vec (int_arm_mve_vldr_gather_base_predicated6517 (AVTI.Vec MQPR:$addr), (i32 imm:$offset), (AVTI.Pred VCCR:$pred))),6518 (DVTI.Vec (Inst (AVTI.Vec MQPR:$addr), (i32 imm:$offset),6519 ARMVCCThen, VCCR:$pred, zero_reg))>;6520 }6521}6522multiclass MVE_VSTR_qi<MVE_memsz memsz, MVEVectorVTInfo AVTI,6523 list<MVEVectorVTInfo> DVTIs> {6524 defm "" : MVE_VLDRSTR_qi_m<MVE_st, memsz, "vstr" # memsz.MnemonicLetter,6525 !cast<string>(memsz.TypeBits)>;6526 defvar Inst = !cast<Instruction>(NAME);6527 defvar InstPre = !cast<Instruction>(NAME # "_pre");6528 6529 foreach DVTI = DVTIs in {6530 def : Pat<(int_arm_mve_vstr_scatter_base6531 (AVTI.Vec MQPR:$addr), (i32 imm:$offset), (DVTI.Vec MQPR:$data)),6532 (Inst (DVTI.Vec MQPR:$data), (AVTI.Vec MQPR:$addr),6533 (i32 imm:$offset))>;6534 def : Pat<(int_arm_mve_vstr_scatter_base_predicated6535 (AVTI.Vec MQPR:$addr), (i32 imm:$offset), (DVTI.Vec MQPR:$data), (AVTI.Pred VCCR:$pred)),6536 (Inst (DVTI.Vec MQPR:$data), (AVTI.Vec MQPR:$addr),6537 (i32 imm:$offset), ARMVCCThen, VCCR:$pred, zero_reg)>;6538 def : Pat<(AVTI.Vec (int_arm_mve_vstr_scatter_base_wb6539 (AVTI.Vec MQPR:$addr), (i32 imm:$offset), (DVTI.Vec MQPR:$data))),6540 (AVTI.Vec (InstPre (DVTI.Vec MQPR:$data), (AVTI.Vec MQPR:$addr),6541 (i32 imm:$offset)))>;6542 def : Pat<(AVTI.Vec (int_arm_mve_vstr_scatter_base_wb_predicated6543 (AVTI.Vec MQPR:$addr), (i32 imm:$offset), (DVTI.Vec MQPR:$data), (AVTI.Pred VCCR:$pred))),6544 (AVTI.Vec (InstPre (DVTI.Vec MQPR:$data), (AVTI.Vec MQPR:$addr),6545 (i32 imm:$offset), ARMVCCThen, VCCR:$pred, zero_reg))>;6546 }6547}6548 6549// Actual instruction definitions.6550defm MVE_VLDRWU32_qi: MVE_VLDR_qi<MVE_memW, MVE_v4i32, [MVE_v4i32,MVE_v4f32]>;6551defm MVE_VLDRDU64_qi: MVE_VLDR_qi<MVE_memD, MVE_v2i64, [MVE_v2i64,MVE_v2f64]>;6552defm MVE_VSTRW32_qi: MVE_VSTR_qi<MVE_memW, MVE_v4i32, [MVE_v4i32,MVE_v4f32]>;6553defm MVE_VSTRD64_qi: MVE_VSTR_qi<MVE_memD, MVE_v2i64, [MVE_v2i64,MVE_v2f64]>;6554 6555// Define aliases for all the instructions where memory size and6556// vector lane size are the same. These are mnemonic aliases, so they6557// apply consistently across all of the above families - contiguous6558// loads, and both the rq and qi types of gather/scatter.6559//6560// Rationale: As long as you're loading (for example) 16-bit memory6561// values into 16-bit vector lanes, you can think of them as signed or6562// unsigned integers, fp16 or just raw 16-bit blobs and it makes no6563// difference. So we permit all of vldrh.16, vldrh.u16, vldrh.s16,6564// vldrh.f16 and treat them all as equivalent to the canonical6565// spelling (which happens to be .u16 for loads, and just .16 for6566// stores).6567 6568foreach vpt_cond = ["", "t", "e"] in6569foreach memsz = [MVE_memB, MVE_memH, MVE_memW, MVE_memD] in6570foreach suffix = memsz.suffixes in {6571 // Define an alias with every suffix in the list, except for the one6572 // used by the real Instruction record (i.e. the one that all the6573 // rest are aliases *for*).6574 6575 if !ne(suffix, memsz.CanonLoadSuffix) then {6576 def : MnemonicAlias<6577 "vldr" # memsz.MnemonicLetter # vpt_cond # suffix,6578 "vldr" # memsz.MnemonicLetter # vpt_cond # memsz.CanonLoadSuffix>;6579 }6580 6581 if !ne(suffix, memsz.CanonStoreSuffix) then {6582 def : MnemonicAlias<6583 "vstr" # memsz.MnemonicLetter # vpt_cond # suffix,6584 "vstr" # memsz.MnemonicLetter # vpt_cond # memsz.CanonStoreSuffix>;6585 }6586}6587 6588// end of MVE predicable load/store6589 6590class MVE_VPT<string suffix, bits<2> size, dag iops, string asm, list<dag> pattern=[]>6591 : MVE_MI<(outs ), iops, NoItinerary, !strconcat("vpt", "${Mk}", ".", suffix), asm, "", size, pattern> {6592 bits<3> fc;6593 bits<4> Mk;6594 bits<3> Qn;6595 6596 let Inst{31-23} = 0b111111100;6597 let Inst{22} = Mk{3};6598 let Inst{21-20} = size;6599 let Inst{19-17} = Qn{2-0};6600 let Inst{16} = 0b1;6601 let Inst{15-13} = Mk{2-0};6602 let Inst{12} = fc{2};6603 let Inst{11-8} = 0b1111;6604 let Inst{7} = fc{0};6605 let Inst{4} = 0b0;6606 6607 let Defs = [VPR];6608 let validForTailPredication=1;6609}6610 6611class MVE_VPTt1<string suffix, bits<2> size, dag iops>6612 : MVE_VPT<suffix, size, iops, "$fc, $Qn, $Qm"> {6613 bits<4> Qm;6614 bits<4> Mk;6615 6616 let Inst{6} = 0b0;6617 let Inst{5} = Qm{3};6618 let Inst{3-1} = Qm{2-0};6619 let Inst{0} = fc{1};6620}6621 6622class MVE_VPTt1i<string suffix, bits<2> size>6623 : MVE_VPTt1<suffix, size,6624 (ins vpt_mask:$Mk, MQPR:$Qn, MQPR:$Qm, pred_basic_i:$fc)> {6625 let Inst{12} = 0b0;6626 let Inst{0} = 0b0;6627}6628 6629def MVE_VPTv4i32 : MVE_VPTt1i<"i32", 0b10>;6630def MVE_VPTv8i16 : MVE_VPTt1i<"i16", 0b01>;6631def MVE_VPTv16i8 : MVE_VPTt1i<"i8", 0b00>;6632 6633class MVE_VPTt1u<string suffix, bits<2> size>6634 : MVE_VPTt1<suffix, size,6635 (ins vpt_mask:$Mk, MQPR:$Qn, MQPR:$Qm, pred_basic_u:$fc)> {6636 let Inst{12} = 0b0;6637 let Inst{0} = 0b1;6638}6639 6640def MVE_VPTv4u32 : MVE_VPTt1u<"u32", 0b10>;6641def MVE_VPTv8u16 : MVE_VPTt1u<"u16", 0b01>;6642def MVE_VPTv16u8 : MVE_VPTt1u<"u8", 0b00>;6643 6644class MVE_VPTt1s<string suffix, bits<2> size>6645 : MVE_VPTt1<suffix, size,6646 (ins vpt_mask:$Mk, MQPR:$Qn, MQPR:$Qm, pred_basic_s:$fc)> {6647 let Inst{12} = 0b1;6648}6649 6650def MVE_VPTv4s32 : MVE_VPTt1s<"s32", 0b10>;6651def MVE_VPTv8s16 : MVE_VPTt1s<"s16", 0b01>;6652def MVE_VPTv16s8 : MVE_VPTt1s<"s8", 0b00>;6653 6654class MVE_VPTt2<string suffix, bits<2> size, dag iops>6655 : MVE_VPT<suffix, size, iops,6656 "$fc, $Qn, $Rm"> {6657 bits<4> Rm;6658 bits<3> fc;6659 bits<4> Mk;6660 6661 let Inst{6} = 0b1;6662 let Inst{5} = fc{1};6663 let Inst{3-0} = Rm{3-0};6664}6665 6666class MVE_VPTt2i<string suffix, bits<2> size>6667 : MVE_VPTt2<suffix, size,6668 (ins vpt_mask:$Mk, MQPR:$Qn, GPRwithZR:$Rm, pred_basic_i:$fc)> {6669 let Inst{12} = 0b0;6670 let Inst{5} = 0b0;6671}6672 6673def MVE_VPTv4i32r : MVE_VPTt2i<"i32", 0b10>;6674def MVE_VPTv8i16r : MVE_VPTt2i<"i16", 0b01>;6675def MVE_VPTv16i8r : MVE_VPTt2i<"i8", 0b00>;6676 6677class MVE_VPTt2u<string suffix, bits<2> size>6678 : MVE_VPTt2<suffix, size,6679 (ins vpt_mask:$Mk, MQPR:$Qn, GPRwithZR:$Rm, pred_basic_u:$fc)> {6680 let Inst{12} = 0b0;6681 let Inst{5} = 0b1;6682}6683 6684def MVE_VPTv4u32r : MVE_VPTt2u<"u32", 0b10>;6685def MVE_VPTv8u16r : MVE_VPTt2u<"u16", 0b01>;6686def MVE_VPTv16u8r : MVE_VPTt2u<"u8", 0b00>;6687 6688class MVE_VPTt2s<string suffix, bits<2> size>6689 : MVE_VPTt2<suffix, size,6690 (ins vpt_mask:$Mk, MQPR:$Qn, GPRwithZR:$Rm, pred_basic_s:$fc)> {6691 let Inst{12} = 0b1;6692}6693 6694def MVE_VPTv4s32r : MVE_VPTt2s<"s32", 0b10>;6695def MVE_VPTv8s16r : MVE_VPTt2s<"s16", 0b01>;6696def MVE_VPTv16s8r : MVE_VPTt2s<"s8", 0b00>;6697 6698 6699class MVE_VPTf<string suffix, bit size, dag iops, string asm, list<dag> pattern=[]>6700 : MVE_MI<(outs ), iops, NoItinerary, !strconcat("vpt", "${Mk}", ".", suffix), asm,6701 "", !if(size, 0b01, 0b10), pattern> {6702 bits<3> fc;6703 bits<4> Mk;6704 bits<3> Qn;6705 6706 let Inst{31-29} = 0b111;6707 let Inst{28} = size;6708 let Inst{27-23} = 0b11100;6709 let Inst{22} = Mk{3};6710 let Inst{21-20} = 0b11;6711 let Inst{19-17} = Qn{2-0};6712 let Inst{16} = 0b1;6713 let Inst{15-13} = Mk{2-0};6714 let Inst{12} = fc{2};6715 let Inst{11-8} = 0b1111;6716 let Inst{7} = fc{0};6717 let Inst{4} = 0b0;6718 6719 let Defs = [VPR];6720 let Predicates = [HasMVEFloat];6721 let validForTailPredication=1;6722}6723 6724class MVE_VPTft1<string suffix, bit size>6725 : MVE_VPTf<suffix, size, (ins vpt_mask:$Mk, MQPR:$Qn, MQPR:$Qm, pred_basic_fp:$fc),6726 "$fc, $Qn, $Qm"> {6727 bits<3> fc;6728 bits<4> Qm;6729 6730 let Inst{6} = 0b0;6731 let Inst{5} = Qm{3};6732 let Inst{3-1} = Qm{2-0};6733 let Inst{0} = fc{1};6734}6735 6736def MVE_VPTv4f32 : MVE_VPTft1<"f32", 0b0>;6737def MVE_VPTv8f16 : MVE_VPTft1<"f16", 0b1>;6738 6739class MVE_VPTft2<string suffix, bit size>6740 : MVE_VPTf<suffix, size, (ins vpt_mask:$Mk, MQPR:$Qn, GPRwithZR:$Rm, pred_basic_fp:$fc),6741 "$fc, $Qn, $Rm"> {6742 bits<3> fc;6743 bits<4> Rm;6744 6745 let Inst{6} = 0b1;6746 let Inst{5} = fc{1};6747 let Inst{3-0} = Rm{3-0};6748}6749 6750def MVE_VPTv4f32r : MVE_VPTft2<"f32", 0b0>;6751def MVE_VPTv8f16r : MVE_VPTft2<"f16", 0b1>;6752 6753def MVE_VPST : MVE_MI<(outs ), (ins vpt_mask:$Mk), NoItinerary,6754 !strconcat("vpst", "${Mk}"), "", "", 0b00, []> {6755 bits<4> Mk;6756 6757 let Inst{31-23} = 0b111111100;6758 let Inst{22} = Mk{3};6759 let Inst{21-16} = 0b110001;6760 let Inst{15-13} = Mk{2-0};6761 let Inst{12-0} = 0b0111101001101;6762 let Unpredictable{12} = 0b1;6763 let Unpredictable{7} = 0b1;6764 let Unpredictable{5} = 0b1;6765 6766 let Uses = [VPR];6767 let validForTailPredication = 1;6768}6769 6770def MVE_VPSEL : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm), NoItinerary,6771 "vpsel", "", "$Qd, $Qn, $Qm", vpred_n, "", 0b00, []> {6772 bits<4> Qn;6773 bits<4> Qd;6774 bits<4> Qm;6775 6776 let Inst{28} = 0b1;6777 let Inst{25-23} = 0b100;6778 let Inst{22} = Qd{3};6779 let Inst{21-20} = 0b11;6780 let Inst{19-17} = Qn{2-0};6781 let Inst{16} = 0b1;6782 let Inst{15-13} = Qd{2-0};6783 let Inst{12-9} = 0b0111;6784 let Inst{8} = 0b1;6785 let Inst{7} = Qn{3};6786 let Inst{6} = 0b0;6787 let Inst{5} = Qm{3};6788 let Inst{4} = 0b0;6789 let Inst{3-1} = Qm{2-0};6790 let Inst{0} = 0b1;6791}6792 6793foreach suffix = ["s8", "s16", "s32", "u8", "u16", "u32",6794 "i8", "i16", "i32", "f16", "f32"] in6795def : MVEInstAlias<"vpsel${vp}." # suffix # "\t$Qd, $Qn, $Qm",6796 (MVE_VPSEL MQPR:$Qd, MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;6797 6798let Predicates = [HasMVEInt] in {6799 def : Pat<(v16i8 (vselect (v16i1 VCCR:$pred), (v16i8 MQPR:$v1), (v16i8 MQPR:$v2))),6800 (v16i8 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>;6801 def : Pat<(v8i16 (vselect (v8i1 VCCR:$pred), (v8i16 MQPR:$v1), (v8i16 MQPR:$v2))),6802 (v8i16 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>;6803 def : Pat<(v4i32 (vselect (v4i1 VCCR:$pred), (v4i32 MQPR:$v1), (v4i32 MQPR:$v2))),6804 (v4i32 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>;6805 def : Pat<(v2i64 (vselect (v2i1 VCCR:$pred), (v2i64 MQPR:$v1), (v2i64 MQPR:$v2))),6806 (v2i64 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>;6807 6808 def : Pat<(v8f16 (vselect (v8i1 VCCR:$pred), (v8f16 MQPR:$v1), (v8f16 MQPR:$v2))),6809 (v8f16 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>;6810 def : Pat<(v4f32 (vselect (v4i1 VCCR:$pred), (v4f32 MQPR:$v1), (v4f32 MQPR:$v2))),6811 (v4f32 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>;6812 def : Pat<(v2f64 (vselect (v2i1 VCCR:$pred), (v2f64 MQPR:$v1), (v2f64 MQPR:$v2))),6813 (v2f64 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>;6814 6815 def : Pat<(v16i8 (vselect (v16i8 MQPR:$pred), (v16i8 MQPR:$v1), (v16i8 MQPR:$v2))),6816 (v16i8 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone,6817 (MVE_VCMPi8 (v16i8 MQPR:$pred), (MVE_VMOVimmi8 0), ARMCCne), zero_reg))>;6818 def : Pat<(v8i16 (vselect (v8i16 MQPR:$pred), (v8i16 MQPR:$v1), (v8i16 MQPR:$v2))),6819 (v8i16 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone,6820 (MVE_VCMPi16 (v8i16 MQPR:$pred), (MVE_VMOVimmi16 0), ARMCCne), zero_reg))>;6821 def : Pat<(v4i32 (vselect (v4i32 MQPR:$pred), (v4i32 MQPR:$v1), (v4i32 MQPR:$v2))),6822 (v4i32 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone,6823 (MVE_VCMPi32 (v4i32 MQPR:$pred), (MVE_VMOVimmi32 0), ARMCCne), zero_reg))>;6824 6825 def : Pat<(v8f16 (vselect (v8i16 MQPR:$pred), (v8f16 MQPR:$v1), (v8f16 MQPR:$v2))),6826 (v8f16 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone,6827 (MVE_VCMPi16 (v8i16 MQPR:$pred), (MVE_VMOVimmi16 0), ARMCCne), zero_reg))>;6828 def : Pat<(v4f32 (vselect (v4i32 MQPR:$pred), (v4f32 MQPR:$v1), (v4f32 MQPR:$v2))),6829 (v4f32 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone,6830 (MVE_VCMPi32 (v4i32 MQPR:$pred), (MVE_VMOVimmi32 0), ARMCCne), zero_reg))>;6831 6832 // Pred <-> Int6833 def : Pat<(v16i8 (zext (v16i1 VCCR:$pred))),6834 (v16i8 (MVE_VPSEL (MVE_VMOVimmi8 1), (MVE_VMOVimmi8 0), ARMVCCNone, VCCR:$pred, zero_reg))>;6835 def : Pat<(v8i16 (zext (v8i1 VCCR:$pred))),6836 (v8i16 (MVE_VPSEL (MVE_VMOVimmi16 1), (MVE_VMOVimmi16 0), ARMVCCNone, VCCR:$pred, zero_reg))>;6837 def : Pat<(v4i32 (zext (v4i1 VCCR:$pred))),6838 (v4i32 (MVE_VPSEL (MVE_VMOVimmi32 1), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>;6839 def : Pat<(v2i64 (zext (v2i1 VCCR:$pred))),6840 (v2i64 (MVE_VPSEL (MVE_VMOVimmi64 1), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>;6841 6842 def : Pat<(v16i8 (sext (v16i1 VCCR:$pred))),6843 (v16i8 (MVE_VPSEL (MVE_VMOVimmi8 255), (MVE_VMOVimmi8 0), ARMVCCNone, VCCR:$pred, zero_reg))>;6844 def : Pat<(v8i16 (sext (v8i1 VCCR:$pred))),6845 (v8i16 (MVE_VPSEL (MVE_VMOVimmi8 255), (MVE_VMOVimmi16 0), ARMVCCNone, VCCR:$pred, zero_reg))>;6846 def : Pat<(v4i32 (sext (v4i1 VCCR:$pred))),6847 (v4i32 (MVE_VPSEL (MVE_VMOVimmi8 255), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>;6848 def : Pat<(v2i64 (sext (v2i1 VCCR:$pred))),6849 (v2i64 (MVE_VPSEL (MVE_VMOVimmi8 255), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>;6850 6851 def : Pat<(v16i8 (anyext (v16i1 VCCR:$pred))),6852 (v16i8 (MVE_VPSEL (MVE_VMOVimmi8 1), (MVE_VMOVimmi8 0), ARMVCCNone, VCCR:$pred, zero_reg))>;6853 def : Pat<(v8i16 (anyext (v8i1 VCCR:$pred))),6854 (v8i16 (MVE_VPSEL (MVE_VMOVimmi16 1), (MVE_VMOVimmi16 0), ARMVCCNone, VCCR:$pred, zero_reg))>;6855 def : Pat<(v4i32 (anyext (v4i1 VCCR:$pred))),6856 (v4i32 (MVE_VPSEL (MVE_VMOVimmi32 1), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>;6857 def : Pat<(v2i64 (anyext (v2i1 VCCR:$pred))),6858 (v2i64 (MVE_VPSEL (MVE_VMOVimmi64 1), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>;6859}6860 6861let Predicates = [HasMVEFloat] in {6862 // Pred <-> Float6863 // 112 is 1.0 in float6864 def : Pat<(v4f32 (uint_to_fp (v4i1 VCCR:$pred))),6865 (v4f32 (MVE_VPSEL (v4f32 (MVE_VMOVimmf32 112)), (v4f32 (MVE_VMOVimmi32 0)), ARMVCCNone, VCCR:$pred, zero_reg))>;6866 // 2620 in 1.0 in half6867 def : Pat<(v8f16 (uint_to_fp (v8i1 VCCR:$pred))),6868 (v8f16 (MVE_VPSEL (v8f16 (MVE_VMOVimmi16 2620)), (v8f16 (MVE_VMOVimmi16 0)), ARMVCCNone, VCCR:$pred, zero_reg))>;6869 // 240 is -1.0 in float6870 def : Pat<(v4f32 (sint_to_fp (v4i1 VCCR:$pred))),6871 (v4f32 (MVE_VPSEL (v4f32 (MVE_VMOVimmf32 240)), (v4f32 (MVE_VMOVimmi32 0)), ARMVCCNone, VCCR:$pred, zero_reg))>;6872 // 2748 is -1.0 in half6873 def : Pat<(v8f16 (sint_to_fp (v8i1 VCCR:$pred))),6874 (v8f16 (MVE_VPSEL (v8f16 (MVE_VMOVimmi16 2748)), (v8f16 (MVE_VMOVimmi16 0)), ARMVCCNone, VCCR:$pred, zero_reg))>;6875 6876 def : Pat<(v4i1 (fp_to_uint (v4f32 MQPR:$v1))),6877 (v4i1 (MVE_VCMPf32r (v4f32 MQPR:$v1), ZR, ARMCCne))>;6878 def : Pat<(v8i1 (fp_to_uint (v8f16 MQPR:$v1))),6879 (v8i1 (MVE_VCMPf16r (v8f16 MQPR:$v1), ZR, ARMCCne))>;6880 def : Pat<(v4i1 (fp_to_sint (v4f32 MQPR:$v1))),6881 (v4i1 (MVE_VCMPf32r (v4f32 MQPR:$v1), ZR, ARMCCne))>;6882 def : Pat<(v8i1 (fp_to_sint (v8f16 MQPR:$v1))),6883 (v8i1 (MVE_VCMPf16r (v8f16 MQPR:$v1), ZR, ARMCCne))>;6884}6885 6886def MVE_VPNOT : MVE_p<(outs VCCR:$P0), (ins VCCR:$P0_in), NoItinerary,6887 "vpnot", "", "", vpred_n, "", 0b00, []> {6888 let Inst{31-0} = 0b11111110001100010000111101001101;6889 let Unpredictable{19-17} = 0b111;6890 let Unpredictable{12} = 0b1;6891 let Unpredictable{7} = 0b1;6892 let Unpredictable{5} = 0b1;6893 6894 let Constraints = "";6895 let DecoderMethod = "DecodeMVEVPNOT";6896}6897 6898let Predicates = [HasMVEInt] in {6899 def : Pat<(v2i1 (xor (v2i1 VCCR:$pred), (v2i1 (predicate_cast (i32 65535))))),6900 (v2i1 (MVE_VPNOT (v2i1 VCCR:$pred)))>;6901 def : Pat<(v4i1 (xor (v4i1 VCCR:$pred), (v4i1 (predicate_cast (i32 65535))))),6902 (v4i1 (MVE_VPNOT (v4i1 VCCR:$pred)))>;6903 def : Pat<(v8i1 (xor (v8i1 VCCR:$pred), (v8i1 (predicate_cast (i32 65535))))),6904 (v8i1 (MVE_VPNOT (v8i1 VCCR:$pred)))>;6905 def : Pat<(v16i1 (xor (v16i1 VCCR:$pred), (v16i1 (predicate_cast (i32 65535))))),6906 (v16i1 (MVE_VPNOT (v16i1 VCCR:$pred)))>;6907}6908 6909 6910class MVE_loltp_start<dag iops, string asm, string ops, bits<2> size>6911 : t2LOL<(outs GPRlr:$LR), iops, asm, ops> {6912 bits<4> Rn;6913 let Predicates = [HasMVEInt];6914 let Inst{22} = 0b0;6915 let Inst{21-20} = size;6916 let Inst{19-16} = Rn{3-0};6917 let Inst{12} = 0b0;6918}6919 6920class MVE_DLSTP<string asm, bits<2> size>6921 : MVE_loltp_start<(ins rGPR:$Rn), asm, "$LR, $Rn", size> {6922 let Inst{13} = 0b1;6923 let Inst{11-1} = 0b00000000000;6924 let Unpredictable{10-1} = 0b1111111111;6925}6926 6927class MVE_WLSTP<string asm, bits<2> size>6928 : MVE_loltp_start<(ins rGPR:$Rn, wlslabel_u11:$label),6929 asm, "$LR, $Rn, $label", size> {6930 bits<11> label;6931 let Inst{13} = 0b0;6932 let Inst{11} = label{0};6933 let Inst{10-1} = label{10-1};6934 let isBranch = 1;6935 let isTerminator = 1;6936}6937 6938def SDT_MVEMEMCPYLOOPNODE6939 : SDTypeProfile<0, 3, [SDTCisPtrTy<0>, SDTCisPtrTy<1>, SDTCisVT<2, i32>]>;6940 6941// Pseudo-instruction representing a memory copy using a tail predicated6942// loop6943def MVE_MEMCPYLOOPNODE : SDNode<"ARMISD::MEMCPYLOOP", SDT_MVEMEMCPYLOOPNODE,6944 [SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;6945 6946let usesCustomInserter = 1, hasNoSchedulingInfo = 1, Defs = [CPSR] in {6947 def MVE_MEMCPYLOOPINST : PseudoInst<(outs),6948 (ins rGPR:$dst, rGPR:$src, rGPR:$sz),6949 NoItinerary,6950 [(MVE_MEMCPYLOOPNODE rGPR:$dst, rGPR:$src, rGPR:$sz)]>;6951}6952 6953def SDT_MVEMEMSETLOOPNODE6954 : SDTypeProfile<0, 3, [SDTCisPtrTy<0>, SDTCisVT<1, v16i8>, SDTCisVT<2, i32>]>;6955 6956// Pseudo-instruction representing a memset using a tail predicated6957// loop6958def MVE_MEMSETLOOPNODE : SDNode<"ARMISD::MEMSETLOOP", SDT_MVEMEMSETLOOPNODE,6959 [SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;6960 6961let usesCustomInserter = 1, hasNoSchedulingInfo = 1, Defs = [CPSR] in {6962 def MVE_MEMSETLOOPINST : PseudoInst<(outs),6963 (ins rGPR:$dst, MQPR:$src, rGPR:$sz),6964 NoItinerary,6965 [(MVE_MEMSETLOOPNODE rGPR:$dst, MQPR:$src, rGPR:$sz)]>;6966}6967 6968def MVE_DLSTP_8 : MVE_DLSTP<"dlstp.8", 0b00>;6969def MVE_DLSTP_16 : MVE_DLSTP<"dlstp.16", 0b01>;6970def MVE_DLSTP_32 : MVE_DLSTP<"dlstp.32", 0b10>;6971def MVE_DLSTP_64 : MVE_DLSTP<"dlstp.64", 0b11>;6972 6973def MVE_WLSTP_8 : MVE_WLSTP<"wlstp.8", 0b00>;6974def MVE_WLSTP_16 : MVE_WLSTP<"wlstp.16", 0b01>;6975def MVE_WLSTP_32 : MVE_WLSTP<"wlstp.32", 0b10>;6976def MVE_WLSTP_64 : MVE_WLSTP<"wlstp.64", 0b11>;6977 6978class MVE_loltp_end<dag oops, dag iops, string asm, string ops>6979 : t2LOL<oops, iops, asm, ops> {6980 let Predicates = [HasMVEInt];6981 let Inst{22-21} = 0b00;6982 let Inst{19-16} = 0b1111;6983 let Inst{12} = 0b0;6984}6985 6986def MVE_LETP : MVE_loltp_end<(outs GPRlr:$LRout),6987 (ins GPRlr:$LRin, lelabel_u11:$label),6988 "letp", "$LRin, $label"> {6989 bits<11> label;6990 let Inst{20} = 0b1;6991 let Inst{13} = 0b0;6992 let Inst{11} = label{0};6993 let Inst{10-1} = label{10-1};6994 let isBranch = 1;6995 let isTerminator = 1;6996}6997 6998def MVE_LCTP : MVE_loltp_end<(outs), (ins pred:$p), "lctp${p}", ""> {6999 let Inst{20} = 0b0;7000 let Inst{13} = 0b1;7001 let Inst{11-1} = 0b00000000000;7002 let Unpredictable{21-20} = 0b11;7003 let Unpredictable{11-1} = 0b11111111111;7004}7005 7006 7007// Pseudo instructions for lowering MQQPR and MQQQQPR stack spills and reloads.7008// They are equivalent to VLDMDIA/VSTMDIA with a single reg, as opposed to multiple7009// dreg subregs.7010 7011let Predicates = [HasMVEInt], AM = AddrMode4 in {7012let mayStore = 1, hasSideEffects = 0 in {7013 def MQQPRStore : t2PseudoInst<(outs), (ins MQQPR:$val, GPRnopc:$ptr),7014 4, NoItinerary, []>;7015 def MQQQQPRStore : t2PseudoInst<(outs), (ins MQQQQPR:$val, GPRnopc:$ptr),7016 4, NoItinerary, []>;7017}7018let mayLoad = 1, hasSideEffects = 0 in {7019 def MQQPRLoad : t2PseudoInst<(outs MQQPR:$val), (ins GPRnopc:$ptr),7020 4, NoItinerary, []>;7021 def MQQQQPRLoad : t2PseudoInst<(outs MQQQQPR:$val), (ins GPRnopc:$ptr),7022 4, NoItinerary, []>;7023}7024}7025 7026// Pseudo for lowering MVE Q register COPYs. These will usually get converted7027// to a "MVE_VORR dst, src, src", but may behave differently in tail predicated7028// loops to ensure the whole register is copied, not a subset from a7029// tail-predicated MVE_VORR. In the event we cannot prove a MVE_VORR is valid,7030// it will become a pair of VMOVD instructions for each half of the Q register.7031let Predicates = [HasMVEInt], hasSideEffects = 0, isMoveReg = 1,7032 D = MVEDomain in {7033 def MQPRCopy : t2PseudoInst<(outs MQPR:$dst), (ins MQPR:$src),7034 8, NoItinerary, []>;7035}7036 7037 7038//===----------------------------------------------------------------------===//7039// Patterns7040//===----------------------------------------------------------------------===//7041 7042// PatFrags for loads and stores. Often trying to keep semi-consistent names.7043 7044def aligned32_pre_store : PatFrag<(ops node:$val, node:$ptr, node:$offset),7045 (pre_store node:$val, node:$ptr, node:$offset), [{7046 return cast<StoreSDNode>(N)->getAlign() >= 4;7047}]>;7048def aligned32_post_store : PatFrag<(ops node:$val, node:$ptr, node:$offset),7049 (post_store node:$val, node:$ptr, node:$offset), [{7050 return cast<StoreSDNode>(N)->getAlign() >= 4;7051}]>;7052def aligned16_pre_store : PatFrag<(ops node:$val, node:$ptr, node:$offset),7053 (pre_store node:$val, node:$ptr, node:$offset), [{7054 return cast<StoreSDNode>(N)->getAlign() >= 2;7055}]>;7056def aligned16_post_store : PatFrag<(ops node:$val, node:$ptr, node:$offset),7057 (post_store node:$val, node:$ptr, node:$offset), [{7058 return cast<StoreSDNode>(N)->getAlign() >= 2;7059}]>;7060 7061 7062def aligned_maskedloadvi8 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),7063 (masked_ld node:$ptr, undef, node:$pred, node:$passthru), [{7064 auto *Ld = cast<MaskedLoadSDNode>(N);7065 return Ld->getMemoryVT().getScalarType() == MVT::i8;7066}]>;7067def aligned_sextmaskedloadvi8 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),7068 (aligned_maskedloadvi8 node:$ptr, node:$pred, node:$passthru), [{7069 return cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;7070}]>;7071def aligned_zextmaskedloadvi8 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),7072 (aligned_maskedloadvi8 node:$ptr, node:$pred, node:$passthru), [{7073 return cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;7074}]>;7075def aligned_extmaskedloadvi8 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),7076 (aligned_maskedloadvi8 node:$ptr, node:$pred, node:$passthru), [{7077 auto *Ld = cast<MaskedLoadSDNode>(N);7078 EVT ScalarVT = Ld->getMemoryVT().getScalarType();7079 return ScalarVT.isInteger() && Ld->getExtensionType() == ISD::EXTLOAD;7080}]>;7081def aligned_maskedloadvi16: PatFrag<(ops node:$ptr, node:$pred, node:$passthru),7082 (masked_ld node:$ptr, undef, node:$pred, node:$passthru), [{7083 auto *Ld = cast<MaskedLoadSDNode>(N);7084 EVT ScalarVT = Ld->getMemoryVT().getScalarType();7085 return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && Ld->getAlign() >= 2;7086}]>;7087def aligned_sextmaskedloadvi16 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),7088 (aligned_maskedloadvi16 node:$ptr, node:$pred, node:$passthru), [{7089 return cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;7090}]>;7091def aligned_zextmaskedloadvi16 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),7092 (aligned_maskedloadvi16 node:$ptr, node:$pred, node:$passthru), [{7093 return cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;7094}]>;7095def aligned_extmaskedloadvi16 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),7096 (aligned_maskedloadvi16 node:$ptr, node:$pred, node:$passthru), [{7097 auto *Ld = cast<MaskedLoadSDNode>(N);7098 EVT ScalarVT = Ld->getMemoryVT().getScalarType();7099 return ScalarVT.isInteger() && Ld->getExtensionType() == ISD::EXTLOAD;7100}]>;7101def aligned_maskedloadvi32: PatFrag<(ops node:$ptr, node:$pred, node:$passthru),7102 (masked_ld node:$ptr, undef, node:$pred, node:$passthru), [{7103 auto *Ld = cast<MaskedLoadSDNode>(N);7104 EVT ScalarVT = Ld->getMemoryVT().getScalarType();7105 return (ScalarVT == MVT::i32 || ScalarVT == MVT::f32) && Ld->getAlign() >= 4;7106}]>;7107 7108def aligned_maskedstvi8 : PatFrag<(ops node:$val, node:$ptr, node:$pred),7109 (masked_st node:$val, node:$ptr, undef, node:$pred), [{7110 return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;7111}]>;7112def aligned_maskedstvi16 : PatFrag<(ops node:$val, node:$ptr, node:$pred),7113 (masked_st node:$val, node:$ptr, undef, node:$pred), [{7114 auto *St = cast<MaskedStoreSDNode>(N);7115 EVT ScalarVT = St->getMemoryVT().getScalarType();7116 return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlign() >= 2;7117}]>;7118def aligned_maskedstvi32 : PatFrag<(ops node:$val, node:$ptr, node:$pred),7119 (masked_st node:$val, node:$ptr, undef, node:$pred), [{7120 auto *St = cast<MaskedStoreSDNode>(N);7121 EVT ScalarVT = St->getMemoryVT().getScalarType();7122 return (ScalarVT == MVT::i32 || ScalarVT == MVT::f32) && St->getAlign() >= 4;7123}]>;7124 7125def pre_maskedstore : PatFrag<(ops node:$val, node:$base, node:$offset, node:$mask),7126 (masked_st node:$val, node:$base, node:$offset, node:$mask), [{7127 ISD::MemIndexedMode AM = cast<MaskedStoreSDNode>(N)->getAddressingMode();7128 return AM == ISD::PRE_INC || AM == ISD::PRE_DEC;7129}]>;7130def post_maskedstore : PatFrag<(ops node:$val, node:$base, node:$offset, node:$mask),7131 (masked_st node:$val, node:$base, node:$offset, node:$mask), [{7132 ISD::MemIndexedMode AM = cast<MaskedStoreSDNode>(N)->getAddressingMode();7133 return AM == ISD::POST_INC || AM == ISD::POST_DEC;7134}]>;7135def aligned_pre_maskedstorevi8 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask),7136 (pre_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{7137 return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;7138}]>;7139def aligned_post_maskedstorevi8 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask),7140 (post_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{7141 return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;7142}]>;7143def aligned_pre_maskedstorevi16 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask),7144 (pre_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{7145 auto *St = cast<MaskedStoreSDNode>(N);7146 EVT ScalarVT = St->getMemoryVT().getScalarType();7147 return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlign() >= 2;7148}]>;7149def aligned_post_maskedstorevi16 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask),7150 (post_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{7151 auto *St = cast<MaskedStoreSDNode>(N);7152 EVT ScalarVT = St->getMemoryVT().getScalarType();7153 return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlign() >= 2;7154}]>;7155def aligned_pre_maskedstorevi32 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask),7156 (pre_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{7157 auto *St = cast<MaskedStoreSDNode>(N);7158 EVT ScalarVT = St->getMemoryVT().getScalarType();7159 return (ScalarVT == MVT::i32 || ScalarVT == MVT::f32) && St->getAlign() >= 4;7160}]>;7161def aligned_post_maskedstorevi32 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask),7162 (post_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{7163 auto *St = cast<MaskedStoreSDNode>(N);7164 EVT ScalarVT = St->getMemoryVT().getScalarType();7165 return (ScalarVT == MVT::i32 || ScalarVT == MVT::f32) && St->getAlign() >= 4;7166}]>;7167 7168 7169// PatFrags for "Aligned" extending / truncating7170 7171def aligned_extloadvi8 : PatFrag<(ops node:$ptr), (extloadvi8 node:$ptr)>;7172def aligned_sextloadvi8 : PatFrag<(ops node:$ptr), (sextloadvi8 node:$ptr)>;7173def aligned_zextloadvi8 : PatFrag<(ops node:$ptr), (zextloadvi8 node:$ptr)>;7174 7175def aligned_truncstvi8 : PatFrag<(ops node:$val, node:$ptr),7176 (truncstorevi8 node:$val, node:$ptr)>;7177def aligned_post_truncstvi8 : PatFrag<(ops node:$val, node:$base, node:$offset),7178 (post_truncstvi8 node:$val, node:$base, node:$offset)>;7179def aligned_pre_truncstvi8 : PatFrag<(ops node:$val, node:$base, node:$offset),7180 (pre_truncstvi8 node:$val, node:$base, node:$offset)>;7181 7182let MinAlignment = 2 in {7183 def aligned_extloadvi16 : PatFrag<(ops node:$ptr), (extloadvi16 node:$ptr)>;7184 def aligned_sextloadvi16 : PatFrag<(ops node:$ptr), (sextloadvi16 node:$ptr)>;7185 def aligned_zextloadvi16 : PatFrag<(ops node:$ptr), (zextloadvi16 node:$ptr)>;7186 7187 def aligned_truncstvi16 : PatFrag<(ops node:$val, node:$ptr),7188 (truncstorevi16 node:$val, node:$ptr)>;7189 def aligned_post_truncstvi16 : PatFrag<(ops node:$val, node:$base, node:$offset),7190 (post_truncstvi16 node:$val, node:$base, node:$offset)>;7191 def aligned_pre_truncstvi16 : PatFrag<(ops node:$val, node:$base, node:$offset),7192 (pre_truncstvi16 node:$val, node:$base, node:$offset)>;7193}7194 7195def truncmaskedst : PatFrag<(ops node:$val, node:$base, node:$pred),7196 (masked_st node:$val, node:$base, undef, node:$pred), [{7197 return cast<MaskedStoreSDNode>(N)->isTruncatingStore();7198}]>;7199def aligned_truncmaskedstvi8 : PatFrag<(ops node:$val, node:$base, node:$pred),7200 (truncmaskedst node:$val, node:$base, node:$pred), [{7201 return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;7202}]>;7203def aligned_truncmaskedstvi16 : PatFrag<(ops node:$val, node:$base, node:$pred),7204 (truncmaskedst node:$val, node:$base, node:$pred), [{7205 auto *St = cast<MaskedStoreSDNode>(N);7206 EVT ScalarVT = St->getMemoryVT().getScalarType();7207 return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlign() >= 2;7208}]>;7209def pre_truncmaskedst : PatFrag<(ops node:$val, node:$base, node:$offset, node:$pred),7210 (masked_st node:$val, node:$base, node:$offset, node:$pred), [{7211 ISD::MemIndexedMode AM = cast<MaskedStoreSDNode>(N)->getAddressingMode();7212 return cast<MaskedStoreSDNode>(N)->isTruncatingStore() && (AM == ISD::PRE_INC || AM == ISD::PRE_DEC);7213}]>;7214def aligned_pre_truncmaskedstvi8 : PatFrag<(ops node:$val, node:$base, node:$offset, node:$pred),7215 (pre_truncmaskedst node:$val, node:$base, node:$offset, node:$pred), [{7216 return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;7217}]>;7218def aligned_pre_truncmaskedstvi16 : PatFrag<(ops node:$val, node:$base, node:$offset, node:$pred),7219 (pre_truncmaskedst node:$val, node:$base, node:$offset, node:$pred), [{7220 auto *St = cast<MaskedStoreSDNode>(N);7221 EVT ScalarVT = St->getMemoryVT().getScalarType();7222 return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlign() >= 2;7223}]>;7224def post_truncmaskedst : PatFrag<(ops node:$val, node:$base, node:$offset, node:$postd),7225 (masked_st node:$val, node:$base, node:$offset, node:$postd), [{7226 ISD::MemIndexedMode AM = cast<MaskedStoreSDNode>(N)->getAddressingMode();7227 return cast<MaskedStoreSDNode>(N)->isTruncatingStore() && (AM == ISD::POST_INC || AM == ISD::POST_DEC);7228}]>;7229def aligned_post_truncmaskedstvi8 : PatFrag<(ops node:$val, node:$base, node:$offset, node:$postd),7230 (post_truncmaskedst node:$val, node:$base, node:$offset, node:$postd), [{7231 return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;7232}]>;7233def aligned_post_truncmaskedstvi16 : PatFrag<(ops node:$val, node:$base, node:$offset, node:$postd),7234 (post_truncmaskedst node:$val, node:$base, node:$offset, node:$postd), [{7235 auto *St = cast<MaskedStoreSDNode>(N);7236 EVT ScalarVT = St->getMemoryVT().getScalarType();7237 return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlign() >= 2;7238}]>;7239 7240// Load/store patterns7241 7242class MVE_vector_store_typed<ValueType Ty, Instruction RegImmInst,7243 PatFrag StoreKind, int shift>7244 : Pat<(StoreKind (Ty MQPR:$val), t2addrmode_imm7<shift>:$addr),7245 (RegImmInst (Ty MQPR:$val), t2addrmode_imm7<shift>:$addr)>;7246 7247class MVE_vector_maskedstore_typed<ValueType Ty, Instruction RegImmInst,7248 PatFrag StoreKind, int shift>7249 : Pat<(StoreKind (Ty MQPR:$val), t2addrmode_imm7<shift>:$addr, VCCR:$pred),7250 (RegImmInst (Ty MQPR:$val), t2addrmode_imm7<shift>:$addr, ARMVCCThen, VCCR:$pred, zero_reg)>;7251 7252multiclass MVE_vector_store<Instruction RegImmInst, PatFrag StoreKind,7253 int shift> {7254 def : MVE_vector_store_typed<v16i8, RegImmInst, StoreKind, shift>;7255 def : MVE_vector_store_typed<v8i16, RegImmInst, StoreKind, shift>;7256 def : MVE_vector_store_typed<v8f16, RegImmInst, StoreKind, shift>;7257 def : MVE_vector_store_typed<v4i32, RegImmInst, StoreKind, shift>;7258 def : MVE_vector_store_typed<v4f32, RegImmInst, StoreKind, shift>;7259 def : MVE_vector_store_typed<v2i64, RegImmInst, StoreKind, shift>;7260 def : MVE_vector_store_typed<v2f64, RegImmInst, StoreKind, shift>;7261}7262 7263class MVE_vector_load_typed<ValueType Ty, Instruction RegImmInst,7264 PatFrag LoadKind, int shift>7265 : Pat<(Ty (LoadKind t2addrmode_imm7<shift>:$addr)),7266 (Ty (RegImmInst t2addrmode_imm7<shift>:$addr))>;7267 7268class MVE_vector_maskedload_typed<ValueType Ty, Instruction RegImmInst,7269 PatFrag LoadKind, int shift>7270 : Pat<(Ty (LoadKind t2addrmode_imm7<shift>:$addr, VCCR:$pred, (Ty (ARMvmovImm (i32 0))))),7271 (Ty (RegImmInst t2addrmode_imm7<shift>:$addr, ARMVCCThen, VCCR:$pred, zero_reg))>;7272 7273multiclass MVE_vector_load<Instruction RegImmInst, PatFrag LoadKind,7274 int shift> {7275 def : MVE_vector_load_typed<v16i8, RegImmInst, LoadKind, shift>;7276 def : MVE_vector_load_typed<v8i16, RegImmInst, LoadKind, shift>;7277 def : MVE_vector_load_typed<v8f16, RegImmInst, LoadKind, shift>;7278 def : MVE_vector_load_typed<v4i32, RegImmInst, LoadKind, shift>;7279 def : MVE_vector_load_typed<v4f32, RegImmInst, LoadKind, shift>;7280 def : MVE_vector_load_typed<v2i64, RegImmInst, LoadKind, shift>;7281 def : MVE_vector_load_typed<v2f64, RegImmInst, LoadKind, shift>;7282}7283 7284class MVE_vector_offset_store_typed<ValueType Ty, Instruction Opcode,7285 PatFrag StoreKind, int shift>7286 : Pat<(StoreKind (Ty MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<shift>:$addr),7287 (Opcode MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<shift>:$addr)>;7288 7289class MVE_vector_offset_maskedstore_typed<ValueType Ty, Instruction Opcode,7290 PatFrag StoreKind, int shift>7291 : Pat<(StoreKind (Ty MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<shift>:$addr, VCCR:$pred),7292 (Opcode MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<shift>:$addr, ARMVCCThen, VCCR:$pred, zero_reg)>;7293 7294multiclass MVE_vector_offset_store<Instruction RegImmInst, PatFrag StoreKind,7295 int shift> {7296 def : MVE_vector_offset_store_typed<v16i8, RegImmInst, StoreKind, shift>;7297 def : MVE_vector_offset_store_typed<v8i16, RegImmInst, StoreKind, shift>;7298 def : MVE_vector_offset_store_typed<v8f16, RegImmInst, StoreKind, shift>;7299 def : MVE_vector_offset_store_typed<v4i32, RegImmInst, StoreKind, shift>;7300 def : MVE_vector_offset_store_typed<v4f32, RegImmInst, StoreKind, shift>;7301 def : MVE_vector_offset_store_typed<v2i64, RegImmInst, StoreKind, shift>;7302 def : MVE_vector_offset_store_typed<v2f64, RegImmInst, StoreKind, shift>;7303}7304 7305 7306let Predicates = [HasMVEInt, IsLE] in {7307 // Stores7308 defm : MVE_vector_store<MVE_VSTRBU8, byte_alignedstore, 0>;7309 defm : MVE_vector_store<MVE_VSTRHU16, hword_alignedstore, 1>;7310 defm : MVE_vector_store<MVE_VSTRWU32, alignedstore32, 2>;7311 7312 // Loads7313 defm : MVE_vector_load<MVE_VLDRBU8, byte_alignedload, 0>;7314 defm : MVE_vector_load<MVE_VLDRHU16, hword_alignedload, 1>;7315 defm : MVE_vector_load<MVE_VLDRWU32, alignedload32, 2>;7316 7317 // Pre/post inc stores7318 defm : MVE_vector_offset_store<MVE_VSTRBU8_pre, pre_store, 0>;7319 defm : MVE_vector_offset_store<MVE_VSTRBU8_post, post_store, 0>;7320 defm : MVE_vector_offset_store<MVE_VSTRHU16_pre, aligned16_pre_store, 1>;7321 defm : MVE_vector_offset_store<MVE_VSTRHU16_post, aligned16_post_store, 1>;7322 defm : MVE_vector_offset_store<MVE_VSTRWU32_pre, aligned32_pre_store, 2>;7323 defm : MVE_vector_offset_store<MVE_VSTRWU32_post, aligned32_post_store, 2>;7324}7325 7326let Predicates = [HasMVEInt, IsBE] in {7327 // Aligned Stores7328 def : MVE_vector_store_typed<v16i8, MVE_VSTRBU8, store, 0>;7329 def : MVE_vector_store_typed<v8i16, MVE_VSTRHU16, alignedstore16, 1>;7330 def : MVE_vector_store_typed<v8f16, MVE_VSTRHU16, alignedstore16, 1>;7331 def : MVE_vector_store_typed<v4i32, MVE_VSTRWU32, alignedstore32, 2>;7332 def : MVE_vector_store_typed<v4f32, MVE_VSTRWU32, alignedstore32, 2>;7333 7334 // Aligned Loads7335 def : MVE_vector_load_typed<v16i8, MVE_VLDRBU8, load, 0>;7336 def : MVE_vector_load_typed<v8i16, MVE_VLDRHU16, alignedload16, 1>;7337 def : MVE_vector_load_typed<v8f16, MVE_VLDRHU16, alignedload16, 1>;7338 def : MVE_vector_load_typed<v4i32, MVE_VLDRWU32, alignedload32, 2>;7339 def : MVE_vector_load_typed<v4f32, MVE_VLDRWU32, alignedload32, 2>;7340 7341 // Other unaligned loads/stores need to go though a VREV7342 def : Pat<(v2f64 (load t2addrmode_imm7<0>:$addr)),7343 (v2f64 (MVE_VREV64_8 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr)))>;7344 def : Pat<(v2i64 (load t2addrmode_imm7<0>:$addr)),7345 (v2i64 (MVE_VREV64_8 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr)))>;7346 def : Pat<(v4i32 (load t2addrmode_imm7<0>:$addr)),7347 (v4i32 (MVE_VREV32_8 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr)))>;7348 def : Pat<(v4f32 (load t2addrmode_imm7<0>:$addr)),7349 (v4f32 (MVE_VREV32_8 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr)))>;7350 def : Pat<(v8i16 (load t2addrmode_imm7<0>:$addr)),7351 (v8i16 (MVE_VREV16_8 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr)))>;7352 def : Pat<(v8f16 (load t2addrmode_imm7<0>:$addr)),7353 (v8f16 (MVE_VREV16_8 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr)))>;7354 def : Pat<(store (v2f64 MQPR:$val), t2addrmode_imm7<0>:$addr),7355 (MVE_VSTRBU8 (MVE_VREV64_8 MQPR:$val), t2addrmode_imm7<0>:$addr)>;7356 def : Pat<(store (v2i64 MQPR:$val), t2addrmode_imm7<0>:$addr),7357 (MVE_VSTRBU8 (MVE_VREV64_8 MQPR:$val), t2addrmode_imm7<0>:$addr)>;7358 def : Pat<(store (v4i32 MQPR:$val), t2addrmode_imm7<0>:$addr),7359 (MVE_VSTRBU8 (MVE_VREV32_8 MQPR:$val), t2addrmode_imm7<0>:$addr)>;7360 def : Pat<(store (v4f32 MQPR:$val), t2addrmode_imm7<0>:$addr),7361 (MVE_VSTRBU8 (MVE_VREV32_8 MQPR:$val), t2addrmode_imm7<0>:$addr)>;7362 def : Pat<(store (v8i16 MQPR:$val), t2addrmode_imm7<0>:$addr),7363 (MVE_VSTRBU8 (MVE_VREV16_8 MQPR:$val), t2addrmode_imm7<0>:$addr)>;7364 def : Pat<(store (v8f16 MQPR:$val), t2addrmode_imm7<0>:$addr),7365 (MVE_VSTRBU8 (MVE_VREV16_8 MQPR:$val), t2addrmode_imm7<0>:$addr)>;7366 7367 // Pre/Post inc stores7368 def : MVE_vector_offset_store_typed<v16i8, MVE_VSTRBU8_pre, pre_store, 0>;7369 def : MVE_vector_offset_store_typed<v16i8, MVE_VSTRBU8_post, post_store, 0>;7370 def : MVE_vector_offset_store_typed<v8i16, MVE_VSTRHU16_pre, aligned16_pre_store, 1>;7371 def : MVE_vector_offset_store_typed<v8i16, MVE_VSTRHU16_post, aligned16_post_store, 1>;7372 def : MVE_vector_offset_store_typed<v8f16, MVE_VSTRHU16_pre, aligned16_pre_store, 1>;7373 def : MVE_vector_offset_store_typed<v8f16, MVE_VSTRHU16_post, aligned16_post_store, 1>;7374 def : MVE_vector_offset_store_typed<v4i32, MVE_VSTRWU32_pre, aligned32_pre_store, 2>;7375 def : MVE_vector_offset_store_typed<v4i32, MVE_VSTRWU32_post, aligned32_post_store, 2>;7376 def : MVE_vector_offset_store_typed<v4f32, MVE_VSTRWU32_pre, aligned32_pre_store, 2>;7377 def : MVE_vector_offset_store_typed<v4f32, MVE_VSTRWU32_post, aligned32_post_store, 2>;7378}7379 7380let Predicates = [HasMVEInt] in {7381 // Aligned masked store, shared between LE and BE7382 def : MVE_vector_maskedstore_typed<v16i8, MVE_VSTRBU8, aligned_maskedstvi8, 0>;7383 def : MVE_vector_maskedstore_typed<v8i16, MVE_VSTRHU16, aligned_maskedstvi16, 1>;7384 def : MVE_vector_maskedstore_typed<v8f16, MVE_VSTRHU16, aligned_maskedstvi16, 1>;7385 def : MVE_vector_maskedstore_typed<v4i32, MVE_VSTRWU32, aligned_maskedstvi32, 2>;7386 def : MVE_vector_maskedstore_typed<v4f32, MVE_VSTRWU32, aligned_maskedstvi32, 2>;7387 7388 // Pre/Post inc masked stores7389 def : MVE_vector_offset_maskedstore_typed<v16i8, MVE_VSTRBU8_pre, aligned_pre_maskedstorevi8, 0>;7390 def : MVE_vector_offset_maskedstore_typed<v16i8, MVE_VSTRBU8_post, aligned_post_maskedstorevi8, 0>;7391 def : MVE_vector_offset_maskedstore_typed<v8i16, MVE_VSTRHU16_pre, aligned_pre_maskedstorevi16, 1>;7392 def : MVE_vector_offset_maskedstore_typed<v8i16, MVE_VSTRHU16_post, aligned_post_maskedstorevi16, 1>;7393 def : MVE_vector_offset_maskedstore_typed<v8f16, MVE_VSTRHU16_pre, aligned_pre_maskedstorevi16, 1>;7394 def : MVE_vector_offset_maskedstore_typed<v8f16, MVE_VSTRHU16_post, aligned_post_maskedstorevi16, 1>;7395 def : MVE_vector_offset_maskedstore_typed<v4i32, MVE_VSTRWU32_pre, aligned_pre_maskedstorevi32, 2>;7396 def : MVE_vector_offset_maskedstore_typed<v4i32, MVE_VSTRWU32_post, aligned_post_maskedstorevi32, 2>;7397 def : MVE_vector_offset_maskedstore_typed<v4f32, MVE_VSTRWU32_pre, aligned_pre_maskedstorevi32, 2>;7398 def : MVE_vector_offset_maskedstore_typed<v4f32, MVE_VSTRWU32_post, aligned_post_maskedstorevi32, 2>;7399 7400 // Aligned masked loads7401 def : MVE_vector_maskedload_typed<v16i8, MVE_VLDRBU8, aligned_maskedloadvi8, 0>;7402 def : MVE_vector_maskedload_typed<v8i16, MVE_VLDRHU16, aligned_maskedloadvi16, 1>;7403 def : MVE_vector_maskedload_typed<v8f16, MVE_VLDRHU16, aligned_maskedloadvi16, 1>;7404 def : MVE_vector_maskedload_typed<v4i32, MVE_VLDRWU32, aligned_maskedloadvi32, 2>;7405 def : MVE_vector_maskedload_typed<v4f32, MVE_VLDRWU32, aligned_maskedloadvi32, 2>;7406}7407 7408// Widening/Narrowing Loads/Stores7409 7410multiclass MVEExtLoadStore<Instruction LoadSInst, Instruction LoadUInst, string StoreInst,7411 string Amble, ValueType VT, int Shift> {7412 // Trunc stores7413 def : Pat<(!cast<PatFrag>("aligned_truncst"#Amble) (VT MQPR:$val), taddrmode_imm7<Shift>:$addr),7414 (!cast<Instruction>(StoreInst) MQPR:$val, taddrmode_imm7<Shift>:$addr)>;7415 def : Pat<(!cast<PatFrag>("aligned_post_truncst"#Amble) (VT MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<Shift>:$addr),7416 (!cast<Instruction>(StoreInst#"_post") MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<Shift>:$addr)>;7417 def : Pat<(!cast<PatFrag>("aligned_pre_truncst"#Amble) (VT MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<Shift>:$addr),7418 (!cast<Instruction>(StoreInst#"_pre") MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<Shift>:$addr)>;7419 7420 // Masked trunc stores7421 def : Pat<(!cast<PatFrag>("aligned_truncmaskedst"#Amble) (VT MQPR:$val), taddrmode_imm7<Shift>:$addr, VCCR:$pred),7422 (!cast<Instruction>(StoreInst) MQPR:$val, taddrmode_imm7<Shift>:$addr, ARMVCCThen, VCCR:$pred, zero_reg)>;7423 def : Pat<(!cast<PatFrag>("aligned_post_truncmaskedst"#Amble) (VT MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<Shift>:$addr, VCCR:$pred),7424 (!cast<Instruction>(StoreInst#"_post") MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<Shift>:$addr, ARMVCCThen, VCCR:$pred, zero_reg)>;7425 def : Pat<(!cast<PatFrag>("aligned_pre_truncmaskedst"#Amble) (VT MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<Shift>:$addr, VCCR:$pred),7426 (!cast<Instruction>(StoreInst#"_pre") MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<Shift>:$addr, ARMVCCThen, VCCR:$pred, zero_reg)>;7427 7428 // Ext loads7429 def : Pat<(VT (!cast<PatFrag>("aligned_extload"#Amble) taddrmode_imm7<Shift>:$addr)),7430 (VT (LoadUInst taddrmode_imm7<Shift>:$addr))>;7431 def : Pat<(VT (!cast<PatFrag>("aligned_sextload"#Amble) taddrmode_imm7<Shift>:$addr)),7432 (VT (LoadSInst taddrmode_imm7<Shift>:$addr))>;7433 def : Pat<(VT (!cast<PatFrag>("aligned_zextload"#Amble) taddrmode_imm7<Shift>:$addr)),7434 (VT (LoadUInst taddrmode_imm7<Shift>:$addr))>;7435 7436 // Masked ext loads7437 def : Pat<(VT (!cast<PatFrag>("aligned_extmaskedload"#Amble) taddrmode_imm7<Shift>:$addr, VCCR:$pred, (VT (ARMvmovImm (i32 0))))),7438 (VT (LoadUInst taddrmode_imm7<Shift>:$addr, ARMVCCThen, VCCR:$pred, zero_reg))>;7439 def : Pat<(VT (!cast<PatFrag>("aligned_sextmaskedload"#Amble) taddrmode_imm7<Shift>:$addr, VCCR:$pred, (VT (ARMvmovImm (i32 0))))),7440 (VT (LoadSInst taddrmode_imm7<Shift>:$addr, ARMVCCThen, VCCR:$pred, zero_reg))>;7441 def : Pat<(VT (!cast<PatFrag>("aligned_zextmaskedload"#Amble) taddrmode_imm7<Shift>:$addr, VCCR:$pred, (VT (ARMvmovImm (i32 0))))),7442 (VT (LoadUInst taddrmode_imm7<Shift>:$addr, ARMVCCThen, VCCR:$pred, zero_reg))>;7443}7444 7445let Predicates = [HasMVEInt] in {7446 defm : MVEExtLoadStore<MVE_VLDRBS16, MVE_VLDRBU16, "MVE_VSTRB16", "vi8", v8i16, 0>;7447 defm : MVEExtLoadStore<MVE_VLDRBS32, MVE_VLDRBU32, "MVE_VSTRB32", "vi8", v4i32, 0>;7448 defm : MVEExtLoadStore<MVE_VLDRHS32, MVE_VLDRHU32, "MVE_VSTRH32", "vi16", v4i32, 1>;7449}7450 7451 7452// Bit convert patterns7453 7454let Predicates = [HasMVEInt] in {7455 def : Pat<(v2f64 (bitconvert (v2i64 MQPR:$src))), (v2f64 MQPR:$src)>;7456 def : Pat<(v2i64 (bitconvert (v2f64 MQPR:$src))), (v2i64 MQPR:$src)>;7457 7458 def : Pat<(v4i32 (bitconvert (v4f32 MQPR:$src))), (v4i32 MQPR:$src)>;7459 def : Pat<(v4f32 (bitconvert (v4i32 MQPR:$src))), (v4f32 MQPR:$src)>;7460 7461 def : Pat<(v8i16 (bitconvert (v8f16 MQPR:$src))), (v8i16 MQPR:$src)>;7462 def : Pat<(v8f16 (bitconvert (v8i16 MQPR:$src))), (v8f16 MQPR:$src)>;7463}7464 7465let Predicates = [IsLE,HasMVEInt] in {7466 def : Pat<(v2f64 (bitconvert (v4f32 MQPR:$src))), (v2f64 MQPR:$src)>;7467 def : Pat<(v2f64 (bitconvert (v4i32 MQPR:$src))), (v2f64 MQPR:$src)>;7468 def : Pat<(v2f64 (bitconvert (v8f16 MQPR:$src))), (v2f64 MQPR:$src)>;7469 def : Pat<(v2f64 (bitconvert (v8i16 MQPR:$src))), (v2f64 MQPR:$src)>;7470 def : Pat<(v2f64 (bitconvert (v16i8 MQPR:$src))), (v2f64 MQPR:$src)>;7471 7472 def : Pat<(v2i64 (bitconvert (v4f32 MQPR:$src))), (v2i64 MQPR:$src)>;7473 def : Pat<(v2i64 (bitconvert (v4i32 MQPR:$src))), (v2i64 MQPR:$src)>;7474 def : Pat<(v2i64 (bitconvert (v8f16 MQPR:$src))), (v2i64 MQPR:$src)>;7475 def : Pat<(v2i64 (bitconvert (v8i16 MQPR:$src))), (v2i64 MQPR:$src)>;7476 def : Pat<(v2i64 (bitconvert (v16i8 MQPR:$src))), (v2i64 MQPR:$src)>;7477 7478 def : Pat<(v4f32 (bitconvert (v2f64 MQPR:$src))), (v4f32 MQPR:$src)>;7479 def : Pat<(v4f32 (bitconvert (v2i64 MQPR:$src))), (v4f32 MQPR:$src)>;7480 def : Pat<(v4f32 (bitconvert (v8f16 MQPR:$src))), (v4f32 MQPR:$src)>;7481 def : Pat<(v4f32 (bitconvert (v8i16 MQPR:$src))), (v4f32 MQPR:$src)>;7482 def : Pat<(v4f32 (bitconvert (v16i8 MQPR:$src))), (v4f32 MQPR:$src)>;7483 7484 def : Pat<(v4i32 (bitconvert (v2f64 MQPR:$src))), (v4i32 MQPR:$src)>;7485 def : Pat<(v4i32 (bitconvert (v2i64 MQPR:$src))), (v4i32 MQPR:$src)>;7486 def : Pat<(v4i32 (bitconvert (v8f16 MQPR:$src))), (v4i32 MQPR:$src)>;7487 def : Pat<(v4i32 (bitconvert (v8i16 MQPR:$src))), (v4i32 MQPR:$src)>;7488 def : Pat<(v4i32 (bitconvert (v16i8 MQPR:$src))), (v4i32 MQPR:$src)>;7489 7490 def : Pat<(v8f16 (bitconvert (v2f64 MQPR:$src))), (v8f16 MQPR:$src)>;7491 def : Pat<(v8f16 (bitconvert (v2i64 MQPR:$src))), (v8f16 MQPR:$src)>;7492 def : Pat<(v8f16 (bitconvert (v4f32 MQPR:$src))), (v8f16 MQPR:$src)>;7493 def : Pat<(v8f16 (bitconvert (v4i32 MQPR:$src))), (v8f16 MQPR:$src)>;7494 def : Pat<(v8f16 (bitconvert (v16i8 MQPR:$src))), (v8f16 MQPR:$src)>;7495 7496 def : Pat<(v8i16 (bitconvert (v2f64 MQPR:$src))), (v8i16 MQPR:$src)>;7497 def : Pat<(v8i16 (bitconvert (v2i64 MQPR:$src))), (v8i16 MQPR:$src)>;7498 def : Pat<(v8i16 (bitconvert (v4f32 MQPR:$src))), (v8i16 MQPR:$src)>;7499 def : Pat<(v8i16 (bitconvert (v4i32 MQPR:$src))), (v8i16 MQPR:$src)>;7500 def : Pat<(v8i16 (bitconvert (v16i8 MQPR:$src))), (v8i16 MQPR:$src)>;7501 7502 def : Pat<(v16i8 (bitconvert (v2f64 MQPR:$src))), (v16i8 MQPR:$src)>;7503 def : Pat<(v16i8 (bitconvert (v2i64 MQPR:$src))), (v16i8 MQPR:$src)>;7504 def : Pat<(v16i8 (bitconvert (v4f32 MQPR:$src))), (v16i8 MQPR:$src)>;7505 def : Pat<(v16i8 (bitconvert (v4i32 MQPR:$src))), (v16i8 MQPR:$src)>;7506 def : Pat<(v16i8 (bitconvert (v8f16 MQPR:$src))), (v16i8 MQPR:$src)>;7507 def : Pat<(v16i8 (bitconvert (v8i16 MQPR:$src))), (v16i8 MQPR:$src)>;7508}7509 7510let Predicates = [IsBE,HasMVEInt] in {7511 def : Pat<(v2f64 (bitconvert (v4f32 MQPR:$src))), (v2f64 (MVE_VREV64_32 MQPR:$src))>;7512 def : Pat<(v2f64 (bitconvert (v4i32 MQPR:$src))), (v2f64 (MVE_VREV64_32 MQPR:$src))>;7513 def : Pat<(v2f64 (bitconvert (v8f16 MQPR:$src))), (v2f64 (MVE_VREV64_16 MQPR:$src))>;7514 def : Pat<(v2f64 (bitconvert (v8i16 MQPR:$src))), (v2f64 (MVE_VREV64_16 MQPR:$src))>;7515 def : Pat<(v2f64 (bitconvert (v16i8 MQPR:$src))), (v2f64 (MVE_VREV64_8 MQPR:$src))>;7516 7517 def : Pat<(v2i64 (bitconvert (v4f32 MQPR:$src))), (v2i64 (MVE_VREV64_32 MQPR:$src))>;7518 def : Pat<(v2i64 (bitconvert (v4i32 MQPR:$src))), (v2i64 (MVE_VREV64_32 MQPR:$src))>;7519 def : Pat<(v2i64 (bitconvert (v8f16 MQPR:$src))), (v2i64 (MVE_VREV64_16 MQPR:$src))>;7520 def : Pat<(v2i64 (bitconvert (v8i16 MQPR:$src))), (v2i64 (MVE_VREV64_16 MQPR:$src))>;7521 def : Pat<(v2i64 (bitconvert (v16i8 MQPR:$src))), (v2i64 (MVE_VREV64_8 MQPR:$src))>;7522 7523 def : Pat<(v4f32 (bitconvert (v2f64 MQPR:$src))), (v4f32 (MVE_VREV64_32 MQPR:$src))>;7524 def : Pat<(v4f32 (bitconvert (v2i64 MQPR:$src))), (v4f32 (MVE_VREV64_32 MQPR:$src))>;7525 def : Pat<(v4f32 (bitconvert (v8f16 MQPR:$src))), (v4f32 (MVE_VREV32_16 MQPR:$src))>;7526 def : Pat<(v4f32 (bitconvert (v8i16 MQPR:$src))), (v4f32 (MVE_VREV32_16 MQPR:$src))>;7527 def : Pat<(v4f32 (bitconvert (v16i8 MQPR:$src))), (v4f32 (MVE_VREV32_8 MQPR:$src))>;7528 7529 def : Pat<(v4i32 (bitconvert (v2f64 MQPR:$src))), (v4i32 (MVE_VREV64_32 MQPR:$src))>;7530 def : Pat<(v4i32 (bitconvert (v2i64 MQPR:$src))), (v4i32 (MVE_VREV64_32 MQPR:$src))>;7531 def : Pat<(v4i32 (bitconvert (v8f16 MQPR:$src))), (v4i32 (MVE_VREV32_16 MQPR:$src))>;7532 def : Pat<(v4i32 (bitconvert (v8i16 MQPR:$src))), (v4i32 (MVE_VREV32_16 MQPR:$src))>;7533 def : Pat<(v4i32 (bitconvert (v16i8 MQPR:$src))), (v4i32 (MVE_VREV32_8 MQPR:$src))>;7534 7535 def : Pat<(v8f16 (bitconvert (v2f64 MQPR:$src))), (v8f16 (MVE_VREV64_16 MQPR:$src))>;7536 def : Pat<(v8f16 (bitconvert (v2i64 MQPR:$src))), (v8f16 (MVE_VREV64_16 MQPR:$src))>;7537 def : Pat<(v8f16 (bitconvert (v4f32 MQPR:$src))), (v8f16 (MVE_VREV32_16 MQPR:$src))>;7538 def : Pat<(v8f16 (bitconvert (v4i32 MQPR:$src))), (v8f16 (MVE_VREV32_16 MQPR:$src))>;7539 def : Pat<(v8f16 (bitconvert (v16i8 MQPR:$src))), (v8f16 (MVE_VREV16_8 MQPR:$src))>;7540 7541 def : Pat<(v8i16 (bitconvert (v2f64 MQPR:$src))), (v8i16 (MVE_VREV64_16 MQPR:$src))>;7542 def : Pat<(v8i16 (bitconvert (v2i64 MQPR:$src))), (v8i16 (MVE_VREV64_16 MQPR:$src))>;7543 def : Pat<(v8i16 (bitconvert (v4f32 MQPR:$src))), (v8i16 (MVE_VREV32_16 MQPR:$src))>;7544 def : Pat<(v8i16 (bitconvert (v4i32 MQPR:$src))), (v8i16 (MVE_VREV32_16 MQPR:$src))>;7545 def : Pat<(v8i16 (bitconvert (v16i8 MQPR:$src))), (v8i16 (MVE_VREV16_8 MQPR:$src))>;7546 7547 def : Pat<(v16i8 (bitconvert (v2f64 MQPR:$src))), (v16i8 (MVE_VREV64_8 MQPR:$src))>;7548 def : Pat<(v16i8 (bitconvert (v2i64 MQPR:$src))), (v16i8 (MVE_VREV64_8 MQPR:$src))>;7549 def : Pat<(v16i8 (bitconvert (v4f32 MQPR:$src))), (v16i8 (MVE_VREV32_8 MQPR:$src))>;7550 def : Pat<(v16i8 (bitconvert (v4i32 MQPR:$src))), (v16i8 (MVE_VREV32_8 MQPR:$src))>;7551 def : Pat<(v16i8 (bitconvert (v8f16 MQPR:$src))), (v16i8 (MVE_VREV16_8 MQPR:$src))>;7552 def : Pat<(v16i8 (bitconvert (v8i16 MQPR:$src))), (v16i8 (MVE_VREV16_8 MQPR:$src))>;7553}7554