13748 lines · plain
1//===-- X86InstrAVX512.td - AVX512 Instruction Set ---------*- tablegen -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file describes the X86 AVX512 instruction set, defining the10// instructions, and properties of the instructions which are needed for code11// generation, machine code emission, and analysis.12//13//===----------------------------------------------------------------------===//14 15// This multiclass generates the masking variants from the non-masking16// variant. It only provides the assembly pieces for the masking variants.17// It assumes custom ISel patterns for masking which can be provided as18// template arguments.19multiclass AVX512_maskable_custom<bits<8> O, Format F,20 dag Outs,21 dag Ins, dag MaskingIns, dag ZeroMaskingIns,22 string OpcodeStr,23 string AttSrcAsm, string IntelSrcAsm,24 list<dag> Pattern,25 list<dag> MaskingPattern,26 list<dag> ZeroMaskingPattern,27 string MaskingConstraint = "",28 bit IsCommutable = 0,29 bit IsKCommutable = 0,30 bit IsKZCommutable = IsCommutable,31 string ClobberConstraint = "",32 string Suffix = ""> {33 let isCommutable = IsCommutable, Constraints = ClobberConstraint in34 def Suffix: AVX512<O, F, Outs, Ins,35 OpcodeStr#"\t{"#AttSrcAsm#", $dst|"#36 "$dst, "#IntelSrcAsm#"}",37 Pattern>;38 39 // Prefer over VMOV*rrk Pat<>40 let isCommutable = IsKCommutable in41 def k#Suffix: AVX512<O, F, Outs, MaskingIns,42 OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}}|"#43 "$dst {${mask}}, "#IntelSrcAsm#"}",44 MaskingPattern>,45 EVEX_K {46 // In case of the 3src subclass this is overridden with a let.47 string Constraints = !if(!eq(ClobberConstraint, ""), MaskingConstraint,48 !if(!eq(MaskingConstraint, ""), ClobberConstraint,49 !strconcat(ClobberConstraint, ", ", MaskingConstraint)));50 }51 52 // Zero mask does not add any restrictions to commute operands transformation.53 // So, it is Ok to use IsCommutable instead of IsKCommutable.54 let isCommutable = IsKZCommutable, // Prefer over VMOV*rrkz Pat<>55 Constraints = ClobberConstraint in56 def kz#Suffix: AVX512<O, F, Outs, ZeroMaskingIns,57 OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}} {z}|"#58 "$dst {${mask}} {z}, "#IntelSrcAsm#"}",59 ZeroMaskingPattern>,60 EVEX_KZ;61}62 63 64// Common base class of AVX512_maskable and AVX512_maskable_3src.65multiclass AVX512_maskable_common<bits<8> O, Format F, X86VectorVTInfo _,66 dag Outs,67 dag Ins, dag MaskingIns, dag ZeroMaskingIns,68 string OpcodeStr,69 string AttSrcAsm, string IntelSrcAsm,70 dag RHS, dag MaskingRHS,71 SDPatternOperator Select = vselect_mask,72 string MaskingConstraint = "",73 bit IsCommutable = 0,74 bit IsKCommutable = 0,75 bit IsKZCommutable = IsCommutable,76 string ClobberConstraint = "",77 string Suffix = ""> :78 AVX512_maskable_custom<O, F, Outs, Ins, MaskingIns, ZeroMaskingIns, OpcodeStr,79 AttSrcAsm, IntelSrcAsm,80 [(set _.RC:$dst, RHS)],81 [(set _.RC:$dst, MaskingRHS)],82 [(set _.RC:$dst,83 (Select _.KRCWM:$mask, RHS, _.ImmAllZerosV))],84 MaskingConstraint, IsCommutable,85 IsKCommutable, IsKZCommutable, ClobberConstraint,86 Suffix>;87 88// This multiclass generates the unconditional/non-masking, the masking and89// the zero-masking variant of the vector instruction. In the masking case, the90// preserved vector elements come from a new dummy input operand tied to $dst.91// This version uses a separate dag for non-masking and masking.92multiclass AVX512_maskable_split<bits<8> O, Format F, X86VectorVTInfo _,93 dag Outs, dag Ins, string OpcodeStr,94 string AttSrcAsm, string IntelSrcAsm,95 dag RHS, dag MaskRHS,96 string ClobberConstraint = "",97 bit IsCommutable = 0, bit IsKCommutable = 0,98 bit IsKZCommutable = IsCommutable> :99 AVX512_maskable_custom<O, F, Outs, Ins,100 !con((ins _.RC:$src0, _.KRCWM:$mask), Ins),101 !con((ins _.KRCWM:$mask), Ins),102 OpcodeStr, AttSrcAsm, IntelSrcAsm,103 [(set _.RC:$dst, RHS)],104 [(set _.RC:$dst,105 (vselect_mask _.KRCWM:$mask, MaskRHS, _.RC:$src0))],106 [(set _.RC:$dst,107 (vselect_mask _.KRCWM:$mask, MaskRHS, _.ImmAllZerosV))],108 "$src0 = $dst", IsCommutable, IsKCommutable,109 IsKZCommutable, ClobberConstraint>;110 111// This multiclass generates the unconditional/non-masking, the masking and112// the zero-masking variant of the vector instruction. In the masking case, the113// preserved vector elements come from a new dummy input operand tied to $dst.114multiclass AVX512_maskable<bits<8> O, Format F, X86VectorVTInfo _,115 dag Outs, dag Ins, string OpcodeStr,116 string AttSrcAsm, string IntelSrcAsm,117 dag RHS,118 bit IsCommutable = 0, bit IsKCommutable = 0,119 bit IsKZCommutable = IsCommutable,120 SDPatternOperator Select = vselect_mask,121 string ClobberConstraint = "",122 string Suffix = ""> :123 AVX512_maskable_common<O, F, _, Outs, Ins,124 !con((ins _.RC:$src0, _.KRCWM:$mask), Ins),125 !con((ins _.KRCWM:$mask), Ins),126 OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS,127 (Select _.KRCWM:$mask, RHS, _.RC:$src0),128 Select, "$src0 = $dst", IsCommutable, IsKCommutable,129 IsKZCommutable, ClobberConstraint, Suffix>;130 131// This multiclass generates the unconditional/non-masking, the masking and132// the zero-masking variant of the scalar instruction.133multiclass AVX512_maskable_scalar<bits<8> O, Format F, X86VectorVTInfo _,134 dag Outs, dag Ins, string OpcodeStr,135 string AttSrcAsm, string IntelSrcAsm,136 dag RHS, string Suffix = ""> :137 AVX512_maskable<O, F, _, Outs, Ins, OpcodeStr, AttSrcAsm, IntelSrcAsm,138 RHS, 0, 0, 0, X86selects_mask, "", Suffix>;139 140// Similar to AVX512_maskable but in this case one of the source operands141// ($src1) is already tied to $dst so we just use that for the preserved142// vector elements. NOTE that the NonTiedIns (the ins dag) should exclude143// $src1.144multiclass AVX512_maskable_3src<bits<8> O, Format F, X86VectorVTInfo _,145 dag Outs, dag NonTiedIns, string OpcodeStr,146 string AttSrcAsm, string IntelSrcAsm,147 dag RHS,148 bit IsCommutable = 0,149 bit IsKCommutable = 0,150 SDPatternOperator Select = vselect_mask,151 bit MaskOnly = 0, string Suffix = ""> :152 AVX512_maskable_common<O, F, _, Outs,153 !con((ins _.RC:$src1), NonTiedIns),154 !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns),155 !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns),156 OpcodeStr, AttSrcAsm, IntelSrcAsm,157 !if(MaskOnly, (null_frag), RHS),158 (Select _.KRCWM:$mask, RHS, _.RC:$src1),159 Select, "", IsCommutable, IsKCommutable,160 IsCommutable, "", Suffix>;161 162// Similar to AVX512_maskable_3src but in this case the input VT for the tied163// operand differs from the output VT. This requires a bitconvert on164// the preserved vector going into the vselect.165// NOTE: The unmasked pattern is disabled.166multiclass AVX512_maskable_3src_cast<bits<8> O, Format F, X86VectorVTInfo OutVT,167 X86VectorVTInfo InVT,168 dag Outs, dag NonTiedIns, string OpcodeStr,169 string AttSrcAsm, string IntelSrcAsm,170 dag RHS, bit IsCommutable = 0> :171 AVX512_maskable_common<O, F, OutVT, Outs,172 !con((ins InVT.RC:$src1), NonTiedIns),173 !con((ins InVT.RC:$src1, InVT.KRCWM:$mask), NonTiedIns),174 !con((ins InVT.RC:$src1, InVT.KRCWM:$mask), NonTiedIns),175 OpcodeStr, AttSrcAsm, IntelSrcAsm, (null_frag),176 (vselect_mask InVT.KRCWM:$mask, RHS,177 (bitconvert InVT.RC:$src1)),178 vselect_mask, "", IsCommutable>;179 180multiclass AVX512_maskable_3src_scalar<bits<8> O, Format F, X86VectorVTInfo _,181 dag Outs, dag NonTiedIns, string OpcodeStr,182 string AttSrcAsm, string IntelSrcAsm,183 dag RHS,184 bit IsCommutable = 0,185 bit IsKCommutable = 0,186 bit MaskOnly = 0, string Suffix = ""> :187 AVX512_maskable_3src<O, F, _, Outs, NonTiedIns, OpcodeStr, AttSrcAsm,188 IntelSrcAsm, RHS, IsCommutable, IsKCommutable,189 X86selects_mask, MaskOnly, Suffix>;190 191multiclass AVX512_maskable_in_asm<bits<8> O, Format F, X86VectorVTInfo _,192 dag Outs, dag Ins,193 string OpcodeStr,194 string AttSrcAsm, string IntelSrcAsm,195 list<dag> Pattern> :196 AVX512_maskable_custom<O, F, Outs, Ins,197 !con((ins _.RC:$src0, _.KRCWM:$mask), Ins),198 !con((ins _.KRCWM:$mask), Ins),199 OpcodeStr, AttSrcAsm, IntelSrcAsm, Pattern, [], [],200 "$src0 = $dst">;201 202multiclass AVX512_maskable_3src_in_asm<bits<8> O, Format F, X86VectorVTInfo _,203 dag Outs, dag NonTiedIns,204 string OpcodeStr,205 string AttSrcAsm, string IntelSrcAsm,206 list<dag> Pattern> :207 AVX512_maskable_custom<O, F, Outs,208 !con((ins _.RC:$src1), NonTiedIns),209 !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns),210 !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns),211 OpcodeStr, AttSrcAsm, IntelSrcAsm, Pattern, [], [],212 "">;213 214// Instruction with mask that puts result in mask register,215// like "compare" and "vptest"216multiclass AVX512_maskable_custom_cmp<bits<8> O, Format F,217 dag Outs,218 dag Ins, dag MaskingIns,219 string OpcodeStr,220 string AttSrcAsm, string IntelSrcAsm,221 list<dag> Pattern,222 list<dag> MaskingPattern,223 bit IsCommutable = 0,224 string Suffix = ""> {225 let isCommutable = IsCommutable in {226 def Suffix: AVX512<O, F, Outs, Ins,227 OpcodeStr#"\t{"#AttSrcAsm#", $dst|"#228 "$dst, "#IntelSrcAsm#"}",229 Pattern>;230 231 def k#Suffix: AVX512<O, F, Outs, MaskingIns,232 OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}}|"#233 "$dst {${mask}}, "#IntelSrcAsm#"}",234 MaskingPattern>, EVEX_K;235 }236}237 238multiclass AVX512_maskable_common_cmp<bits<8> O, Format F, X86VectorVTInfo _,239 dag Outs,240 dag Ins, dag MaskingIns,241 string OpcodeStr,242 string AttSrcAsm, string IntelSrcAsm,243 dag RHS, dag MaskingRHS,244 bit IsCommutable = 0,245 string Suffix = ""> :246 AVX512_maskable_custom_cmp<O, F, Outs, Ins, MaskingIns, OpcodeStr,247 AttSrcAsm, IntelSrcAsm,248 [(set _.KRC:$dst, RHS)],249 [(set _.KRC:$dst, MaskingRHS)], IsCommutable, Suffix>;250 251multiclass AVX512_maskable_cmp<bits<8> O, Format F, X86VectorVTInfo _,252 dag Outs, dag Ins, string OpcodeStr,253 string AttSrcAsm, string IntelSrcAsm,254 dag RHS, dag RHS_su, bit IsCommutable = 0,255 string Suffix = ""> :256 AVX512_maskable_common_cmp<O, F, _, Outs, Ins,257 !con((ins _.KRCWM:$mask), Ins),258 OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS,259 (and _.KRCWM:$mask, RHS_su), IsCommutable, Suffix>;260 261// Used by conversion instructions.262multiclass AVX512_maskable_cvt<bits<8> O, Format F, X86VectorVTInfo _,263 dag Outs,264 dag Ins, dag MaskingIns, dag ZeroMaskingIns,265 string OpcodeStr,266 string AttSrcAsm, string IntelSrcAsm,267 dag RHS, dag MaskingRHS, dag ZeroMaskingRHS> :268 AVX512_maskable_custom<O, F, Outs, Ins, MaskingIns, ZeroMaskingIns, OpcodeStr,269 AttSrcAsm, IntelSrcAsm,270 [(set _.RC:$dst, RHS)],271 [(set _.RC:$dst, MaskingRHS)],272 [(set _.RC:$dst, ZeroMaskingRHS)],273 "$src0 = $dst">;274 275multiclass AVX512_maskable_fma<bits<8> O, Format F, X86VectorVTInfo _,276 dag Outs, dag NonTiedIns, string OpcodeStr,277 string AttSrcAsm, string IntelSrcAsm,278 dag RHS, dag MaskingRHS, bit IsCommutable,279 bit IsKCommutable> :280 AVX512_maskable_custom<O, F, Outs,281 !con((ins _.RC:$src1), NonTiedIns),282 !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns),283 !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns),284 OpcodeStr, AttSrcAsm, IntelSrcAsm,285 [(set _.RC:$dst, RHS)],286 [(set _.RC:$dst,287 (vselect_mask _.KRCWM:$mask, MaskingRHS, _.RC:$src1))],288 [(set _.RC:$dst,289 (vselect_mask _.KRCWM:$mask, MaskingRHS, _.ImmAllZerosV))],290 "", IsCommutable, IsKCommutable>;291 292// Alias instruction that maps zero vector to pxor / xorp* for AVX-512.293// This is expanded by ExpandPostRAPseudos to an xorps / vxorps, and then294// swizzled by ExecutionDomainFix to pxor.295// We set canFoldAsLoad because this can be converted to a constant-pool296// load of an all-zeros value if folding it would be beneficial.297let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,298 isPseudo = 1, Predicates = [HasAVX512], SchedRW = [WriteZero] in {299def AVX512_512_SET0 : I<0, Pseudo, (outs VR512:$dst), (ins), "",300 [(set VR512:$dst, (v16i32 immAllZerosV))]>;301def AVX512_512_SETALLONES : I<0, Pseudo, (outs VR512:$dst), (ins), "",302 [(set VR512:$dst, (v16i32 immAllOnesV))]>;303let AddedComplexity = 1, Predicates = [HasVLX] in {304 def AVX512_128_SETALLONES : I<0, Pseudo, (outs VR128X:$dst), (ins),305 "", [(set VR128X:$dst, (v4i32 immAllOnesV))]>;306 def AVX512_256_SETALLONES : I<0, Pseudo, (outs VR256X:$dst), (ins),307 "", [(set VR256X:$dst, (v8i32 immAllOnesV))]>;308}309}310 311let Predicates = [HasAVX512] in {312def : Pat<(v64i8 immAllZerosV), (AVX512_512_SET0)>;313def : Pat<(v32i16 immAllZerosV), (AVX512_512_SET0)>;314def : Pat<(v8i64 immAllZerosV), (AVX512_512_SET0)>;315def : Pat<(v32f16 immAllZerosV), (AVX512_512_SET0)>;316def : Pat<(v16f32 immAllZerosV), (AVX512_512_SET0)>;317def : Pat<(v8f64 immAllZerosV), (AVX512_512_SET0)>;318}319 320// Alias instructions that allow VPTERNLOG to be used with a mask to create321// a mix of all ones and all zeros elements. This is done this way to force322// the same register to be used as input for all three sources.323let isPseudo = 1, Predicates = [HasAVX512], SchedRW = [WriteVecALU] in {324def AVX512_512_SEXT_MASK_32 : I<0, Pseudo, (outs VR512:$dst),325 (ins VK16WM:$mask), "",326 [(set VR512:$dst, (vselect (v16i1 VK16WM:$mask),327 (v16i32 immAllOnesV),328 (v16i32 immAllZerosV)))]>;329def AVX512_512_SEXT_MASK_64 : I<0, Pseudo, (outs VR512:$dst),330 (ins VK8WM:$mask), "",331 [(set VR512:$dst, (vselect (v8i1 VK8WM:$mask),332 (v8i64 immAllOnesV),333 (v8i64 immAllZerosV)))]>;334}335 336let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,337 isPseudo = 1, Predicates = [HasAVX512], SchedRW = [WriteZero] in {338def AVX512_128_SET0 : I<0, Pseudo, (outs VR128X:$dst), (ins), "",339 [(set VR128X:$dst, (v4i32 immAllZerosV))]>;340def AVX512_256_SET0 : I<0, Pseudo, (outs VR256X:$dst), (ins), "",341 [(set VR256X:$dst, (v8i32 immAllZerosV))]>;342}343 344let Predicates = [HasAVX512] in {345def : Pat<(v8i16 immAllZerosV), (AVX512_128_SET0)>;346def : Pat<(v16i8 immAllZerosV), (AVX512_128_SET0)>;347def : Pat<(v2i64 immAllZerosV), (AVX512_128_SET0)>;348def : Pat<(v8f16 immAllZerosV), (AVX512_128_SET0)>;349def : Pat<(v4f32 immAllZerosV), (AVX512_128_SET0)>;350def : Pat<(v2f64 immAllZerosV), (AVX512_128_SET0)>;351def : Pat<(v32i8 immAllZerosV), (AVX512_256_SET0)>;352def : Pat<(v16i16 immAllZerosV), (AVX512_256_SET0)>;353def : Pat<(v4i64 immAllZerosV), (AVX512_256_SET0)>;354def : Pat<(v16f16 immAllZerosV), (AVX512_256_SET0)>;355def : Pat<(v8f32 immAllZerosV), (AVX512_256_SET0)>;356def : Pat<(v4f64 immAllZerosV), (AVX512_256_SET0)>;357}358 359// Alias instructions that map fld0 to xorps for sse or vxorps for avx.360// This is expanded by ExpandPostRAPseudos.361let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,362 isPseudo = 1, SchedRW = [WriteZero], Predicates = [HasAVX512] in {363 def AVX512_FsFLD0SH : I<0, Pseudo, (outs FR16X:$dst), (ins), "",364 [(set FR16X:$dst, fp16imm0)]>;365 def AVX512_FsFLD0SS : I<0, Pseudo, (outs FR32X:$dst), (ins), "",366 [(set FR32X:$dst, fp32imm0)]>;367 def AVX512_FsFLD0SD : I<0, Pseudo, (outs FR64X:$dst), (ins), "",368 [(set FR64X:$dst, fp64imm0)]>;369 def AVX512_FsFLD0F128 : I<0, Pseudo, (outs VR128X:$dst), (ins), "",370 [(set VR128X:$dst, fp128imm0)]>;371}372 373//===----------------------------------------------------------------------===//374// AVX-512 - VECTOR INSERT375//376 377// Supports two different pattern operators for mask and unmasked ops. Allows378// null_frag to be passed for one.379multiclass vinsert_for_size_split<int Opcode, X86VectorVTInfo From,380 X86VectorVTInfo To,381 SDPatternOperator vinsert_insert,382 SDPatternOperator vinsert_for_mask,383 X86FoldableSchedWrite sched> {384 let hasSideEffects = 0, ExeDomain = To.ExeDomain in {385 defm rri : AVX512_maskable_split<Opcode, MRMSrcReg, To, (outs To.RC:$dst),386 (ins To.RC:$src1, From.RC:$src2, u8imm:$src3),387 "vinsert" # From.EltTypeName # "x" # From.NumElts,388 "$src3, $src2, $src1", "$src1, $src2, $src3",389 (vinsert_insert:$src3 (To.VT To.RC:$src1),390 (From.VT From.RC:$src2),391 (iPTR imm)),392 (vinsert_for_mask:$src3 (To.VT To.RC:$src1),393 (From.VT From.RC:$src2),394 (iPTR imm))>,395 AVX512AIi8Base, EVEX, VVVV, Sched<[sched]>;396 let mayLoad = 1 in397 defm rmi : AVX512_maskable_split<Opcode, MRMSrcMem, To, (outs To.RC:$dst),398 (ins To.RC:$src1, From.MemOp:$src2, u8imm:$src3),399 "vinsert" # From.EltTypeName # "x" # From.NumElts,400 "$src3, $src2, $src1", "$src1, $src2, $src3",401 (vinsert_insert:$src3 (To.VT To.RC:$src1),402 (From.VT (From.LdFrag addr:$src2)),403 (iPTR imm)),404 (vinsert_for_mask:$src3 (To.VT To.RC:$src1),405 (From.VT (From.LdFrag addr:$src2)),406 (iPTR imm))>, AVX512AIi8Base, EVEX, VVVV,407 EVEX_CD8<From.EltSize, From.CD8TupleForm>,408 Sched<[sched.Folded, sched.ReadAfterFold]>;409 }410}411 412// Passes the same pattern operator for masked and unmasked ops.413multiclass vinsert_for_size<int Opcode, X86VectorVTInfo From,414 X86VectorVTInfo To,415 SDPatternOperator vinsert_insert,416 X86FoldableSchedWrite sched> :417 vinsert_for_size_split<Opcode, From, To, vinsert_insert, vinsert_insert, sched>;418 419multiclass vinsert_for_size_lowering<string InstrStr, X86VectorVTInfo From,420 X86VectorVTInfo To, PatFrag vinsert_insert,421 SDNodeXForm INSERT_get_vinsert_imm , list<Predicate> p> {422 let Predicates = p in {423 def : Pat<(vinsert_insert:$ins424 (To.VT To.RC:$src1), (From.VT From.RC:$src2), (iPTR imm)),425 (To.VT (!cast<Instruction>(InstrStr#"rri")426 To.RC:$src1, From.RC:$src2,427 (INSERT_get_vinsert_imm To.RC:$ins)))>;428 429 def : Pat<(vinsert_insert:$ins430 (To.VT To.RC:$src1),431 (From.VT (From.LdFrag addr:$src2)),432 (iPTR imm)),433 (To.VT (!cast<Instruction>(InstrStr#"rmi")434 To.RC:$src1, addr:$src2,435 (INSERT_get_vinsert_imm To.RC:$ins)))>;436 }437}438 439multiclass vinsert_for_type<ValueType EltVT32, int Opcode128,440 ValueType EltVT64, int Opcode256,441 X86FoldableSchedWrite sched> {442 443 let Predicates = [HasVLX] in444 defm NAME # "32X4Z256" : vinsert_for_size<Opcode128,445 X86VectorVTInfo< 4, EltVT32, VR128X>,446 X86VectorVTInfo< 8, EltVT32, VR256X>,447 vinsert128_insert, sched>, EVEX_V256;448 449 defm NAME # "32X4Z" : vinsert_for_size<Opcode128,450 X86VectorVTInfo< 4, EltVT32, VR128X>,451 X86VectorVTInfo<16, EltVT32, VR512>,452 vinsert128_insert, sched>, EVEX_V512;453 454 defm NAME # "64X4Z" : vinsert_for_size<Opcode256,455 X86VectorVTInfo< 4, EltVT64, VR256X>,456 X86VectorVTInfo< 8, EltVT64, VR512>,457 vinsert256_insert, sched>, REX_W, EVEX_V512;458 459 // Even with DQI we'd like to only use these instructions for masking.460 let Predicates = [HasVLX, HasDQI] in461 defm NAME # "64X2Z256" : vinsert_for_size_split<Opcode128,462 X86VectorVTInfo< 2, EltVT64, VR128X>,463 X86VectorVTInfo< 4, EltVT64, VR256X>,464 null_frag, vinsert128_insert, sched>,465 EVEX_V256, REX_W;466 467 // Even with DQI we'd like to only use these instructions for masking.468 let Predicates = [HasDQI] in {469 defm NAME # "64X2Z" : vinsert_for_size_split<Opcode128,470 X86VectorVTInfo< 2, EltVT64, VR128X>,471 X86VectorVTInfo< 8, EltVT64, VR512>,472 null_frag, vinsert128_insert, sched>,473 REX_W, EVEX_V512;474 475 defm NAME # "32X8Z" : vinsert_for_size_split<Opcode256,476 X86VectorVTInfo< 8, EltVT32, VR256X>,477 X86VectorVTInfo<16, EltVT32, VR512>,478 null_frag, vinsert256_insert, sched>,479 EVEX_V512;480 }481}482 483// FIXME: Is there a better scheduler class for VINSERTF/VINSERTI?484defm VINSERTF : vinsert_for_type<f32, 0x18, f64, 0x1a, WriteFShuffle256>;485defm VINSERTI : vinsert_for_type<i32, 0x38, i64, 0x3a, WriteShuffle256>;486 487// Codegen pattern with the alternative types,488// Even with AVX512DQ we'll still use these for unmasked operations.489defm : vinsert_for_size_lowering<"VINSERTF32X4Z256", v2f64x_info, v4f64x_info,490 vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX]>;491defm : vinsert_for_size_lowering<"VINSERTI32X4Z256", v2i64x_info, v4i64x_info,492 vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX]>;493 494defm : vinsert_for_size_lowering<"VINSERTF32X4Z", v2f64x_info, v8f64_info,495 vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512]>;496defm : vinsert_for_size_lowering<"VINSERTI32X4Z", v2i64x_info, v8i64_info,497 vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512]>;498 499defm : vinsert_for_size_lowering<"VINSERTF64X4Z", v8f32x_info, v16f32_info,500 vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512]>;501defm : vinsert_for_size_lowering<"VINSERTI64X4Z", v8i32x_info, v16i32_info,502 vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512]>;503 504// Codegen pattern with the alternative types insert VEC128 into VEC256505defm : vinsert_for_size_lowering<"VINSERTI32X4Z256", v8i16x_info, v16i16x_info,506 vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX]>;507defm : vinsert_for_size_lowering<"VINSERTI32X4Z256", v16i8x_info, v32i8x_info,508 vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX]>;509defm : vinsert_for_size_lowering<"VINSERTF32X4Z256", v8f16x_info, v16f16x_info,510 vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX]>;511defm : vinsert_for_size_lowering<"VINSERTF32X4Z256", v8bf16x_info, v16bf16x_info,512 vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX]>;513// Codegen pattern with the alternative types insert VEC128 into VEC512514defm : vinsert_for_size_lowering<"VINSERTI32X4Z", v8i16x_info, v32i16_info,515 vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512]>;516defm : vinsert_for_size_lowering<"VINSERTI32X4Z", v16i8x_info, v64i8_info,517 vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512]>;518defm : vinsert_for_size_lowering<"VINSERTF32X4Z", v8f16x_info, v32f16_info,519 vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512]>;520defm : vinsert_for_size_lowering<"VINSERTF32X4Z", v8bf16x_info, v32bf16_info,521 vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512]>;522// Codegen pattern with the alternative types insert VEC256 into VEC512523defm : vinsert_for_size_lowering<"VINSERTI64X4Z", v16i16x_info, v32i16_info,524 vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512]>;525defm : vinsert_for_size_lowering<"VINSERTI64X4Z", v32i8x_info, v64i8_info,526 vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512]>;527defm : vinsert_for_size_lowering<"VINSERTF64X4Z", v16f16x_info, v32f16_info,528 vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512]>;529defm : vinsert_for_size_lowering<"VINSERTF64X4Z", v16bf16x_info, v32bf16_info,530 vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512]>;531 532 533multiclass vinsert_for_mask_cast<string InstrStr, X86VectorVTInfo From,534 X86VectorVTInfo To, X86VectorVTInfo Cast,535 PatFrag vinsert_insert,536 SDNodeXForm INSERT_get_vinsert_imm,537 list<Predicate> p> {538let Predicates = p in {539 def : Pat<(Cast.VT540 (vselect_mask Cast.KRCWM:$mask,541 (bitconvert542 (vinsert_insert:$ins (To.VT To.RC:$src1),543 (From.VT From.RC:$src2),544 (iPTR imm))),545 Cast.RC:$src0)),546 (!cast<Instruction>(InstrStr#"rrik")547 Cast.RC:$src0, Cast.KRCWM:$mask, To.RC:$src1, From.RC:$src2,548 (INSERT_get_vinsert_imm To.RC:$ins))>;549 def : Pat<(Cast.VT550 (vselect_mask Cast.KRCWM:$mask,551 (bitconvert552 (vinsert_insert:$ins (To.VT To.RC:$src1),553 (From.VT554 (bitconvert555 (From.LdFrag addr:$src2))),556 (iPTR imm))),557 Cast.RC:$src0)),558 (!cast<Instruction>(InstrStr#"rmik")559 Cast.RC:$src0, Cast.KRCWM:$mask, To.RC:$src1, addr:$src2,560 (INSERT_get_vinsert_imm To.RC:$ins))>;561 562 def : Pat<(Cast.VT563 (vselect_mask Cast.KRCWM:$mask,564 (bitconvert565 (vinsert_insert:$ins (To.VT To.RC:$src1),566 (From.VT From.RC:$src2),567 (iPTR imm))),568 Cast.ImmAllZerosV)),569 (!cast<Instruction>(InstrStr#"rrikz")570 Cast.KRCWM:$mask, To.RC:$src1, From.RC:$src2,571 (INSERT_get_vinsert_imm To.RC:$ins))>;572 def : Pat<(Cast.VT573 (vselect_mask Cast.KRCWM:$mask,574 (bitconvert575 (vinsert_insert:$ins (To.VT To.RC:$src1),576 (From.VT (From.LdFrag addr:$src2)),577 (iPTR imm))),578 Cast.ImmAllZerosV)),579 (!cast<Instruction>(InstrStr#"rmikz")580 Cast.KRCWM:$mask, To.RC:$src1, addr:$src2,581 (INSERT_get_vinsert_imm To.RC:$ins))>;582}583}584 585defm : vinsert_for_mask_cast<"VINSERTF32X4Z256", v2f64x_info, v4f64x_info,586 v8f32x_info, vinsert128_insert,587 INSERT_get_vinsert128_imm, [HasVLX]>;588defm : vinsert_for_mask_cast<"VINSERTF64X2Z256", v4f32x_info, v8f32x_info,589 v4f64x_info, vinsert128_insert,590 INSERT_get_vinsert128_imm, [HasDQI, HasVLX]>;591 592defm : vinsert_for_mask_cast<"VINSERTI32X4Z256", v2i64x_info, v4i64x_info,593 v8i32x_info, vinsert128_insert,594 INSERT_get_vinsert128_imm, [HasVLX]>;595defm : vinsert_for_mask_cast<"VINSERTI32X4Z256", v8i16x_info, v16i16x_info,596 v8i32x_info, vinsert128_insert,597 INSERT_get_vinsert128_imm, [HasVLX]>;598defm : vinsert_for_mask_cast<"VINSERTI32X4Z256", v16i8x_info, v32i8x_info,599 v8i32x_info, vinsert128_insert,600 INSERT_get_vinsert128_imm, [HasVLX]>;601defm : vinsert_for_mask_cast<"VINSERTF64X2Z256", v4i32x_info, v8i32x_info,602 v4i64x_info, vinsert128_insert,603 INSERT_get_vinsert128_imm, [HasDQI, HasVLX]>;604defm : vinsert_for_mask_cast<"VINSERTF64X2Z256", v8i16x_info, v16i16x_info,605 v4i64x_info, vinsert128_insert,606 INSERT_get_vinsert128_imm, [HasDQI, HasVLX]>;607defm : vinsert_for_mask_cast<"VINSERTF64X2Z256", v16i8x_info, v32i8x_info,608 v4i64x_info, vinsert128_insert,609 INSERT_get_vinsert128_imm, [HasDQI, HasVLX]>;610 611defm : vinsert_for_mask_cast<"VINSERTF32X4Z", v2f64x_info, v8f64_info,612 v16f32_info, vinsert128_insert,613 INSERT_get_vinsert128_imm, [HasAVX512]>;614defm : vinsert_for_mask_cast<"VINSERTF64X2Z", v4f32x_info, v16f32_info,615 v8f64_info, vinsert128_insert,616 INSERT_get_vinsert128_imm, [HasDQI]>;617 618defm : vinsert_for_mask_cast<"VINSERTI32X4Z", v2i64x_info, v8i64_info,619 v16i32_info, vinsert128_insert,620 INSERT_get_vinsert128_imm, [HasAVX512]>;621defm : vinsert_for_mask_cast<"VINSERTI32X4Z", v8i16x_info, v32i16_info,622 v16i32_info, vinsert128_insert,623 INSERT_get_vinsert128_imm, [HasAVX512]>;624defm : vinsert_for_mask_cast<"VINSERTI32X4Z", v16i8x_info, v64i8_info,625 v16i32_info, vinsert128_insert,626 INSERT_get_vinsert128_imm, [HasAVX512]>;627defm : vinsert_for_mask_cast<"VINSERTI64X2Z", v4i32x_info, v16i32_info,628 v8i64_info, vinsert128_insert,629 INSERT_get_vinsert128_imm, [HasDQI]>;630defm : vinsert_for_mask_cast<"VINSERTI64X2Z", v8i16x_info, v32i16_info,631 v8i64_info, vinsert128_insert,632 INSERT_get_vinsert128_imm, [HasDQI]>;633defm : vinsert_for_mask_cast<"VINSERTI64X2Z", v16i8x_info, v64i8_info,634 v8i64_info, vinsert128_insert,635 INSERT_get_vinsert128_imm, [HasDQI]>;636 637defm : vinsert_for_mask_cast<"VINSERTF32X8Z", v4f64x_info, v8f64_info,638 v16f32_info, vinsert256_insert,639 INSERT_get_vinsert256_imm, [HasDQI]>;640defm : vinsert_for_mask_cast<"VINSERTF64X4Z", v8f32x_info, v16f32_info,641 v8f64_info, vinsert256_insert,642 INSERT_get_vinsert256_imm, [HasAVX512]>;643 644defm : vinsert_for_mask_cast<"VINSERTI32X8Z", v4i64x_info, v8i64_info,645 v16i32_info, vinsert256_insert,646 INSERT_get_vinsert256_imm, [HasDQI]>;647defm : vinsert_for_mask_cast<"VINSERTI32X8Z", v16i16x_info, v32i16_info,648 v16i32_info, vinsert256_insert,649 INSERT_get_vinsert256_imm, [HasDQI]>;650defm : vinsert_for_mask_cast<"VINSERTI32X8Z", v32i8x_info, v64i8_info,651 v16i32_info, vinsert256_insert,652 INSERT_get_vinsert256_imm, [HasDQI]>;653defm : vinsert_for_mask_cast<"VINSERTI64X4Z", v8i32x_info, v16i32_info,654 v8i64_info, vinsert256_insert,655 INSERT_get_vinsert256_imm, [HasAVX512]>;656defm : vinsert_for_mask_cast<"VINSERTI64X4Z", v16i16x_info, v32i16_info,657 v8i64_info, vinsert256_insert,658 INSERT_get_vinsert256_imm, [HasAVX512]>;659defm : vinsert_for_mask_cast<"VINSERTI64X4Z", v32i8x_info, v64i8_info,660 v8i64_info, vinsert256_insert,661 INSERT_get_vinsert256_imm, [HasAVX512]>;662 663// vinsertps - insert f32 to XMM664let ExeDomain = SSEPackedSingle in {665let isCommutable = 1 in666def VINSERTPSZrri : AVX512AIi8<0x21, MRMSrcReg, (outs VR128X:$dst),667 (ins VR128X:$src1, VR128X:$src2, u8imm:$src3),668 "vinsertps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",669 [(set VR128X:$dst, (X86insertps VR128X:$src1, VR128X:$src2, timm:$src3))]>,670 EVEX, VVVV, Sched<[SchedWriteFShuffle.XMM]>;671let mayLoad = 1 in672def VINSERTPSZrmi : AVX512AIi8<0x21, MRMSrcMem, (outs VR128X:$dst),673 (ins VR128X:$src1, f32mem:$src2, u8imm:$src3),674 "vinsertps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",675 [(set VR128X:$dst, (X86insertps VR128X:$src1,676 (v4f32 (scalar_to_vector (loadf32 addr:$src2))),677 timm:$src3))]>,678 EVEX, VVVV, EVEX_CD8<32, CD8VT1>,679 Sched<[SchedWriteFShuffle.XMM.Folded, SchedWriteFShuffle.XMM.ReadAfterFold]>;680}681 682//===----------------------------------------------------------------------===//683// AVX-512 VECTOR EXTRACT684//---685 686// Supports two different pattern operators for mask and unmasked ops. Allows687// null_frag to be passed for one.688multiclass vextract_for_size_split<int Opcode,689 X86VectorVTInfo From, X86VectorVTInfo To,690 SDPatternOperator vextract_extract,691 SDPatternOperator vextract_for_mask,692 SchedWrite SchedRR, SchedWrite SchedMR> {693 694 let hasSideEffects = 0, ExeDomain = To.ExeDomain in {695 defm rri : AVX512_maskable_split<Opcode, MRMDestReg, To, (outs To.RC:$dst),696 (ins From.RC:$src1, u8imm:$idx),697 "vextract" # To.EltTypeName # "x" # To.NumElts,698 "$idx, $src1", "$src1, $idx",699 (vextract_extract:$idx (From.VT From.RC:$src1), (iPTR imm)),700 (vextract_for_mask:$idx (From.VT From.RC:$src1), (iPTR imm))>,701 AVX512AIi8Base, EVEX, Sched<[SchedRR]>;702 703 def mri : AVX512AIi8<Opcode, MRMDestMem, (outs),704 (ins To.MemOp:$dst, From.RC:$src1, u8imm:$idx),705 "vextract" # To.EltTypeName # "x" # To.NumElts #706 "\t{$idx, $src1, $dst|$dst, $src1, $idx}",707 [(store (To.VT (vextract_extract:$idx708 (From.VT From.RC:$src1), (iPTR imm))),709 addr:$dst)]>, EVEX,710 Sched<[SchedMR]>;711 712 let mayStore = 1, hasSideEffects = 0 in713 def mrik : AVX512AIi8<Opcode, MRMDestMem, (outs),714 (ins To.MemOp:$dst, To.KRCWM:$mask,715 From.RC:$src1, u8imm:$idx),716 "vextract" # To.EltTypeName # "x" # To.NumElts #717 "\t{$idx, $src1, $dst {${mask}}|"718 "$dst {${mask}}, $src1, $idx}", []>,719 EVEX_K, EVEX, Sched<[SchedMR]>;720 }721}722 723// Passes the same pattern operator for masked and unmasked ops.724multiclass vextract_for_size<int Opcode, X86VectorVTInfo From,725 X86VectorVTInfo To,726 SDPatternOperator vextract_extract,727 SchedWrite SchedRR, SchedWrite SchedMR> :728 vextract_for_size_split<Opcode, From, To, vextract_extract, vextract_extract, SchedRR, SchedMR>;729 730// Codegen pattern for the alternative types731multiclass vextract_for_size_lowering<string InstrStr, X86VectorVTInfo From,732 X86VectorVTInfo To, PatFrag vextract_extract,733 SDNodeXForm EXTRACT_get_vextract_imm, list<Predicate> p> {734 let Predicates = p in {735 def : Pat<(vextract_extract:$ext (From.VT From.RC:$src1), (iPTR imm)),736 (To.VT (!cast<Instruction>(InstrStr#"rri")737 From.RC:$src1,738 (EXTRACT_get_vextract_imm To.RC:$ext)))>;739 def : Pat<(store (To.VT (vextract_extract:$ext (From.VT From.RC:$src1),740 (iPTR imm))), addr:$dst),741 (!cast<Instruction>(InstrStr#"mri") addr:$dst, From.RC:$src1,742 (EXTRACT_get_vextract_imm To.RC:$ext))>;743 }744}745 746multiclass vextract_for_type<ValueType EltVT32, int Opcode128,747 ValueType EltVT64, int Opcode256,748 SchedWrite SchedRR, SchedWrite SchedMR> {749 let Predicates = [HasAVX512] in {750 defm NAME # "32X4Z" : vextract_for_size<Opcode128,751 X86VectorVTInfo<16, EltVT32, VR512>,752 X86VectorVTInfo< 4, EltVT32, VR128X>,753 vextract128_extract, SchedRR, SchedMR>,754 EVEX_V512, EVEX_CD8<32, CD8VT4>;755 defm NAME # "64X4Z" : vextract_for_size<Opcode256,756 X86VectorVTInfo< 8, EltVT64, VR512>,757 X86VectorVTInfo< 4, EltVT64, VR256X>,758 vextract256_extract, SchedRR, SchedMR>,759 REX_W, EVEX_V512, EVEX_CD8<64, CD8VT4>;760 }761 let Predicates = [HasVLX] in762 defm NAME # "32X4Z256" : vextract_for_size<Opcode128,763 X86VectorVTInfo< 8, EltVT32, VR256X>,764 X86VectorVTInfo< 4, EltVT32, VR128X>,765 vextract128_extract, SchedRR, SchedMR>,766 EVEX_V256, EVEX_CD8<32, CD8VT4>;767 768 // Even with DQI we'd like to only use these instructions for masking.769 let Predicates = [HasVLX, HasDQI] in770 defm NAME # "64X2Z256" : vextract_for_size_split<Opcode128,771 X86VectorVTInfo< 4, EltVT64, VR256X>,772 X86VectorVTInfo< 2, EltVT64, VR128X>,773 null_frag, vextract128_extract, SchedRR, SchedMR>,774 EVEX_V256, EVEX_CD8<64, CD8VT2>, REX_W;775 776 // Even with DQI we'd like to only use these instructions for masking.777 let Predicates = [HasDQI] in {778 defm NAME # "64X2Z" : vextract_for_size_split<Opcode128,779 X86VectorVTInfo< 8, EltVT64, VR512>,780 X86VectorVTInfo< 2, EltVT64, VR128X>,781 null_frag, vextract128_extract, SchedRR, SchedMR>,782 REX_W, EVEX_V512, EVEX_CD8<64, CD8VT2>;783 defm NAME # "32X8Z" : vextract_for_size_split<Opcode256,784 X86VectorVTInfo<16, EltVT32, VR512>,785 X86VectorVTInfo< 8, EltVT32, VR256X>,786 null_frag, vextract256_extract, SchedRR, SchedMR>,787 EVEX_V512, EVEX_CD8<32, CD8VT8>;788 }789}790 791// TODO - replace WriteFStore/WriteVecStore with X86SchedWriteMoveLSWidths types.792defm VEXTRACTF : vextract_for_type<f32, 0x19, f64, 0x1b, WriteFShuffle256, WriteFStore>;793defm VEXTRACTI : vextract_for_type<i32, 0x39, i64, 0x3b, WriteShuffle256, WriteVecStore>;794 795// extract_subvector codegen patterns with the alternative types.796// Even with AVX512DQ we'll still use these for unmasked operations.797defm : vextract_for_size_lowering<"VEXTRACTF32X4Z", v8f64_info, v2f64x_info,798 vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512]>;799defm : vextract_for_size_lowering<"VEXTRACTI32X4Z", v8i64_info, v2i64x_info,800 vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512]>;801 802defm : vextract_for_size_lowering<"VEXTRACTF64X4Z", v16f32_info, v8f32x_info,803 vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512]>;804defm : vextract_for_size_lowering<"VEXTRACTI64X4Z", v16i32_info, v8i32x_info,805 vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512]>;806 807defm : vextract_for_size_lowering<"VEXTRACTF32X4Z256", v4f64x_info, v2f64x_info,808 vextract128_extract, EXTRACT_get_vextract128_imm, [HasVLX]>;809defm : vextract_for_size_lowering<"VEXTRACTI32X4Z256", v4i64x_info, v2i64x_info,810 vextract128_extract, EXTRACT_get_vextract128_imm, [HasVLX]>;811 812// Codegen pattern with the alternative types extract VEC128 from VEC256813defm : vextract_for_size_lowering<"VEXTRACTI32X4Z256", v16i16x_info, v8i16x_info,814 vextract128_extract, EXTRACT_get_vextract128_imm, [HasVLX]>;815defm : vextract_for_size_lowering<"VEXTRACTI32X4Z256", v32i8x_info, v16i8x_info,816 vextract128_extract, EXTRACT_get_vextract128_imm, [HasVLX]>;817defm : vextract_for_size_lowering<"VEXTRACTF32X4Z256", v16f16x_info, v8f16x_info,818 vextract128_extract, EXTRACT_get_vextract128_imm, [HasVLX]>;819defm : vextract_for_size_lowering<"VEXTRACTF32X4Z256", v16bf16x_info, v8bf16x_info,820 vextract128_extract, EXTRACT_get_vextract128_imm, [HasVLX]>;821 822// Codegen pattern with the alternative types extract VEC128 from VEC512823defm : vextract_for_size_lowering<"VEXTRACTI32X4Z", v32i16_info, v8i16x_info,824 vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512]>;825defm : vextract_for_size_lowering<"VEXTRACTI32X4Z", v64i8_info, v16i8x_info,826 vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512]>;827defm : vextract_for_size_lowering<"VEXTRACTF32X4Z", v32f16_info, v8f16x_info,828 vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512]>;829defm : vextract_for_size_lowering<"VEXTRACTF32X4Z", v32bf16_info, v8bf16x_info,830 vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512]>;831// Codegen pattern with the alternative types extract VEC256 from VEC512832defm : vextract_for_size_lowering<"VEXTRACTI64X4Z", v32i16_info, v16i16x_info,833 vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512]>;834defm : vextract_for_size_lowering<"VEXTRACTI64X4Z", v64i8_info, v32i8x_info,835 vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512]>;836defm : vextract_for_size_lowering<"VEXTRACTF64X4Z", v32f16_info, v16f16x_info,837 vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512]>;838defm : vextract_for_size_lowering<"VEXTRACTF64X4Z", v32bf16_info, v16bf16x_info,839 vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512]>;840 841 842// A 128-bit extract from bits [255:128] of a 512-bit vector should use a843// smaller extract to enable EVEX->VEX.844let Predicates = [NoVLX] in {845def : Pat<(v2i64 (extract_subvector (v8i64 VR512:$src), (iPTR 2))),846 (v2i64 (VEXTRACTI128rri847 (v4i64 (EXTRACT_SUBREG (v8i64 VR512:$src), sub_ymm)),848 (iPTR 1)))>;849def : Pat<(v2f64 (extract_subvector (v8f64 VR512:$src), (iPTR 2))),850 (v2f64 (VEXTRACTF128rri851 (v4f64 (EXTRACT_SUBREG (v8f64 VR512:$src), sub_ymm)),852 (iPTR 1)))>;853def : Pat<(v4i32 (extract_subvector (v16i32 VR512:$src), (iPTR 4))),854 (v4i32 (VEXTRACTI128rri855 (v8i32 (EXTRACT_SUBREG (v16i32 VR512:$src), sub_ymm)),856 (iPTR 1)))>;857def : Pat<(v4f32 (extract_subvector (v16f32 VR512:$src), (iPTR 4))),858 (v4f32 (VEXTRACTF128rri859 (v8f32 (EXTRACT_SUBREG (v16f32 VR512:$src), sub_ymm)),860 (iPTR 1)))>;861def : Pat<(v8i16 (extract_subvector (v32i16 VR512:$src), (iPTR 8))),862 (v8i16 (VEXTRACTI128rri863 (v16i16 (EXTRACT_SUBREG (v32i16 VR512:$src), sub_ymm)),864 (iPTR 1)))>;865def : Pat<(v8f16 (extract_subvector (v32f16 VR512:$src), (iPTR 8))),866 (v8f16 (VEXTRACTF128rri867 (v16f16 (EXTRACT_SUBREG (v32f16 VR512:$src), sub_ymm)),868 (iPTR 1)))>;869def : Pat<(v16i8 (extract_subvector (v64i8 VR512:$src), (iPTR 16))),870 (v16i8 (VEXTRACTI128rri871 (v32i8 (EXTRACT_SUBREG (v64i8 VR512:$src), sub_ymm)),872 (iPTR 1)))>;873}874 875// A 128-bit extract from bits [255:128] of a 512-bit vector should use a876// smaller extract to enable EVEX->VEX.877let Predicates = [HasVLX] in {878def : Pat<(v2i64 (extract_subvector (v8i64 VR512:$src), (iPTR 2))),879 (v2i64 (VEXTRACTI32X4Z256rri880 (v4i64 (EXTRACT_SUBREG (v8i64 VR512:$src), sub_ymm)),881 (iPTR 1)))>;882def : Pat<(v2f64 (extract_subvector (v8f64 VR512:$src), (iPTR 2))),883 (v2f64 (VEXTRACTF32X4Z256rri884 (v4f64 (EXTRACT_SUBREG (v8f64 VR512:$src), sub_ymm)),885 (iPTR 1)))>;886def : Pat<(v4i32 (extract_subvector (v16i32 VR512:$src), (iPTR 4))),887 (v4i32 (VEXTRACTI32X4Z256rri888 (v8i32 (EXTRACT_SUBREG (v16i32 VR512:$src), sub_ymm)),889 (iPTR 1)))>;890def : Pat<(v4f32 (extract_subvector (v16f32 VR512:$src), (iPTR 4))),891 (v4f32 (VEXTRACTF32X4Z256rri892 (v8f32 (EXTRACT_SUBREG (v16f32 VR512:$src), sub_ymm)),893 (iPTR 1)))>;894def : Pat<(v8i16 (extract_subvector (v32i16 VR512:$src), (iPTR 8))),895 (v8i16 (VEXTRACTI32X4Z256rri896 (v16i16 (EXTRACT_SUBREG (v32i16 VR512:$src), sub_ymm)),897 (iPTR 1)))>;898def : Pat<(v8f16 (extract_subvector (v32f16 VR512:$src), (iPTR 8))),899 (v8f16 (VEXTRACTF32X4Z256rri900 (v16f16 (EXTRACT_SUBREG (v32f16 VR512:$src), sub_ymm)),901 (iPTR 1)))>;902def : Pat<(v16i8 (extract_subvector (v64i8 VR512:$src), (iPTR 16))),903 (v16i8 (VEXTRACTI32X4Z256rri904 (v32i8 (EXTRACT_SUBREG (v64i8 VR512:$src), sub_ymm)),905 (iPTR 1)))>;906}907 908 909// Additional patterns for handling a bitcast between the vselect and the910// extract_subvector.911multiclass vextract_for_mask_cast<string InstrStr, X86VectorVTInfo From,912 X86VectorVTInfo To, X86VectorVTInfo Cast,913 PatFrag vextract_extract,914 SDNodeXForm EXTRACT_get_vextract_imm,915 list<Predicate> p> {916let Predicates = p in {917 def : Pat<(Cast.VT (vselect_mask Cast.KRCWM:$mask,918 (bitconvert919 (To.VT (vextract_extract:$ext920 (From.VT From.RC:$src), (iPTR imm)))),921 To.RC:$src0)),922 (Cast.VT (!cast<Instruction>(InstrStr#"rrik")923 Cast.RC:$src0, Cast.KRCWM:$mask, From.RC:$src,924 (EXTRACT_get_vextract_imm To.RC:$ext)))>;925 926 def : Pat<(Cast.VT (vselect_mask Cast.KRCWM:$mask,927 (bitconvert928 (To.VT (vextract_extract:$ext929 (From.VT From.RC:$src), (iPTR imm)))),930 Cast.ImmAllZerosV)),931 (Cast.VT (!cast<Instruction>(InstrStr#"rrikz")932 Cast.KRCWM:$mask, From.RC:$src,933 (EXTRACT_get_vextract_imm To.RC:$ext)))>;934}935}936 937defm : vextract_for_mask_cast<"VEXTRACTF32X4Z256", v4f64x_info, v2f64x_info,938 v4f32x_info, vextract128_extract,939 EXTRACT_get_vextract128_imm, [HasVLX]>;940defm : vextract_for_mask_cast<"VEXTRACTF64X2Z256", v8f32x_info, v4f32x_info,941 v2f64x_info, vextract128_extract,942 EXTRACT_get_vextract128_imm, [HasDQI, HasVLX]>;943 944defm : vextract_for_mask_cast<"VEXTRACTI32X4Z256", v4i64x_info, v2i64x_info,945 v4i32x_info, vextract128_extract,946 EXTRACT_get_vextract128_imm, [HasVLX]>;947defm : vextract_for_mask_cast<"VEXTRACTI32X4Z256", v16i16x_info, v8i16x_info,948 v4i32x_info, vextract128_extract,949 EXTRACT_get_vextract128_imm, [HasVLX]>;950defm : vextract_for_mask_cast<"VEXTRACTI32X4Z256", v32i8x_info, v16i8x_info,951 v4i32x_info, vextract128_extract,952 EXTRACT_get_vextract128_imm, [HasVLX]>;953defm : vextract_for_mask_cast<"VEXTRACTI64X2Z256", v8i32x_info, v4i32x_info,954 v2i64x_info, vextract128_extract,955 EXTRACT_get_vextract128_imm, [HasDQI, HasVLX]>;956defm : vextract_for_mask_cast<"VEXTRACTI64X2Z256", v16i16x_info, v8i16x_info,957 v2i64x_info, vextract128_extract,958 EXTRACT_get_vextract128_imm, [HasDQI, HasVLX]>;959defm : vextract_for_mask_cast<"VEXTRACTI64X2Z256", v32i8x_info, v16i8x_info,960 v2i64x_info, vextract128_extract,961 EXTRACT_get_vextract128_imm, [HasDQI, HasVLX]>;962 963defm : vextract_for_mask_cast<"VEXTRACTF32X4Z", v8f64_info, v2f64x_info,964 v4f32x_info, vextract128_extract,965 EXTRACT_get_vextract128_imm, [HasAVX512]>;966defm : vextract_for_mask_cast<"VEXTRACTF64X2Z", v16f32_info, v4f32x_info,967 v2f64x_info, vextract128_extract,968 EXTRACT_get_vextract128_imm, [HasDQI]>;969 970defm : vextract_for_mask_cast<"VEXTRACTI32X4Z", v8i64_info, v2i64x_info,971 v4i32x_info, vextract128_extract,972 EXTRACT_get_vextract128_imm, [HasAVX512]>;973defm : vextract_for_mask_cast<"VEXTRACTI32X4Z", v32i16_info, v8i16x_info,974 v4i32x_info, vextract128_extract,975 EXTRACT_get_vextract128_imm, [HasAVX512]>;976defm : vextract_for_mask_cast<"VEXTRACTI32X4Z", v64i8_info, v16i8x_info,977 v4i32x_info, vextract128_extract,978 EXTRACT_get_vextract128_imm, [HasAVX512]>;979defm : vextract_for_mask_cast<"VEXTRACTI64X2Z", v16i32_info, v4i32x_info,980 v2i64x_info, vextract128_extract,981 EXTRACT_get_vextract128_imm, [HasDQI]>;982defm : vextract_for_mask_cast<"VEXTRACTI64X2Z", v32i16_info, v8i16x_info,983 v2i64x_info, vextract128_extract,984 EXTRACT_get_vextract128_imm, [HasDQI]>;985defm : vextract_for_mask_cast<"VEXTRACTI64X2Z", v64i8_info, v16i8x_info,986 v2i64x_info, vextract128_extract,987 EXTRACT_get_vextract128_imm, [HasDQI]>;988 989defm : vextract_for_mask_cast<"VEXTRACTF32X8Z", v8f64_info, v4f64x_info,990 v8f32x_info, vextract256_extract,991 EXTRACT_get_vextract256_imm, [HasDQI]>;992defm : vextract_for_mask_cast<"VEXTRACTF64X4Z", v16f32_info, v8f32x_info,993 v4f64x_info, vextract256_extract,994 EXTRACT_get_vextract256_imm, [HasAVX512]>;995 996defm : vextract_for_mask_cast<"VEXTRACTI32X8Z", v8i64_info, v4i64x_info,997 v8i32x_info, vextract256_extract,998 EXTRACT_get_vextract256_imm, [HasDQI]>;999defm : vextract_for_mask_cast<"VEXTRACTI32X8Z", v32i16_info, v16i16x_info,1000 v8i32x_info, vextract256_extract,1001 EXTRACT_get_vextract256_imm, [HasDQI]>;1002defm : vextract_for_mask_cast<"VEXTRACTI32X8Z", v64i8_info, v32i8x_info,1003 v8i32x_info, vextract256_extract,1004 EXTRACT_get_vextract256_imm, [HasDQI]>;1005defm : vextract_for_mask_cast<"VEXTRACTI64X4Z", v16i32_info, v8i32x_info,1006 v4i64x_info, vextract256_extract,1007 EXTRACT_get_vextract256_imm, [HasAVX512]>;1008defm : vextract_for_mask_cast<"VEXTRACTI64X4Z", v32i16_info, v16i16x_info,1009 v4i64x_info, vextract256_extract,1010 EXTRACT_get_vextract256_imm, [HasAVX512]>;1011defm : vextract_for_mask_cast<"VEXTRACTI64X4Z", v64i8_info, v32i8x_info,1012 v4i64x_info, vextract256_extract,1013 EXTRACT_get_vextract256_imm, [HasAVX512]>;1014 1015// vextractps - extract 32 bits from XMM1016def VEXTRACTPSZrri : AVX512AIi8<0x17, MRMDestReg, (outs GR32orGR64:$dst),1017 (ins VR128X:$src1, u8imm:$src2),1018 "vextractps\t{$src2, $src1, $dst|$dst, $src1, $src2}",1019 [(set GR32orGR64:$dst, (extractelt (bc_v4i32 (v4f32 VR128X:$src1)), imm:$src2))]>,1020 EVEX, WIG, Sched<[WriteVecExtract]>;1021 1022def VEXTRACTPSZmri : AVX512AIi8<0x17, MRMDestMem, (outs),1023 (ins f32mem:$dst, VR128X:$src1, u8imm:$src2),1024 "vextractps\t{$src2, $src1, $dst|$dst, $src1, $src2}",1025 [(store (extractelt (bc_v4i32 (v4f32 VR128X:$src1)), imm:$src2),1026 addr:$dst)]>,1027 EVEX, WIG, EVEX_CD8<32, CD8VT1>, Sched<[WriteVecExtractSt]>;1028 1029//===---------------------------------------------------------------------===//1030// AVX-512 BROADCAST1031//---1032// broadcast with a scalar argument.1033multiclass avx512_broadcast_scalar<string Name, X86VectorVTInfo DestInfo,1034 X86VectorVTInfo SrcInfo> {1035 def : Pat<(DestInfo.VT (X86VBroadcast SrcInfo.FRC:$src)),1036 (!cast<Instruction>(Name#DestInfo.ZSuffix#rr)1037 (SrcInfo.VT (COPY_TO_REGCLASS SrcInfo.FRC:$src, SrcInfo.RC)))>;1038 def : Pat<(DestInfo.VT (vselect_mask DestInfo.KRCWM:$mask,1039 (X86VBroadcast SrcInfo.FRC:$src),1040 DestInfo.RC:$src0)),1041 (!cast<Instruction>(Name#DestInfo.ZSuffix#rrk)1042 DestInfo.RC:$src0, DestInfo.KRCWM:$mask,1043 (SrcInfo.VT (COPY_TO_REGCLASS SrcInfo.FRC:$src, SrcInfo.RC)))>;1044 def : Pat<(DestInfo.VT (vselect_mask DestInfo.KRCWM:$mask,1045 (X86VBroadcast SrcInfo.FRC:$src),1046 DestInfo.ImmAllZerosV)),1047 (!cast<Instruction>(Name#DestInfo.ZSuffix#rrkz)1048 DestInfo.KRCWM:$mask, (SrcInfo.VT (COPY_TO_REGCLASS SrcInfo.FRC:$src, SrcInfo.RC)))>;1049}1050 1051// Split version to allow mask and broadcast node to be different types. This1052// helps support the 32x2 broadcasts.1053multiclass avx512_broadcast_rm_split<bits<8> opc, string OpcodeStr,1054 SchedWrite SchedRR, SchedWrite SchedRM,1055 X86VectorVTInfo MaskInfo,1056 X86VectorVTInfo DestInfo,1057 X86VectorVTInfo SrcInfo,1058 bit IsConvertibleToThreeAddress,1059 SDPatternOperator UnmaskedOp = X86VBroadcast,1060 SDPatternOperator UnmaskedBcastOp = SrcInfo.BroadcastLdFrag> {1061 let hasSideEffects = 0 in1062 def rr : AVX512PI<opc, MRMSrcReg, (outs MaskInfo.RC:$dst), (ins SrcInfo.RC:$src),1063 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),1064 [(set MaskInfo.RC:$dst,1065 (MaskInfo.VT1066 (bitconvert1067 (DestInfo.VT1068 (UnmaskedOp (SrcInfo.VT SrcInfo.RC:$src))))))],1069 DestInfo.ExeDomain>, T8, PD, EVEX, Sched<[SchedRR]>;1070 def rrkz : AVX512PI<opc, MRMSrcReg, (outs MaskInfo.RC:$dst),1071 (ins MaskInfo.KRCWM:$mask, SrcInfo.RC:$src),1072 !strconcat(OpcodeStr, "\t{$src, ${dst} {${mask}} {z}|",1073 "${dst} {${mask}} {z}, $src}"),1074 [(set MaskInfo.RC:$dst,1075 (vselect_mask MaskInfo.KRCWM:$mask,1076 (MaskInfo.VT1077 (bitconvert1078 (DestInfo.VT1079 (X86VBroadcast (SrcInfo.VT SrcInfo.RC:$src))))),1080 MaskInfo.ImmAllZerosV))],1081 DestInfo.ExeDomain>, T8, PD, EVEX, EVEX_KZ, Sched<[SchedRR]>;1082 let Constraints = "$src0 = $dst" in1083 def rrk : AVX512PI<opc, MRMSrcReg, (outs MaskInfo.RC:$dst),1084 (ins MaskInfo.RC:$src0, MaskInfo.KRCWM:$mask,1085 SrcInfo.RC:$src),1086 !strconcat(OpcodeStr, "\t{$src, ${dst} {${mask}}|",1087 "${dst} {${mask}}, $src}"),1088 [(set MaskInfo.RC:$dst,1089 (vselect_mask MaskInfo.KRCWM:$mask,1090 (MaskInfo.VT1091 (bitconvert1092 (DestInfo.VT1093 (X86VBroadcast (SrcInfo.VT SrcInfo.RC:$src))))),1094 MaskInfo.RC:$src0))],1095 DestInfo.ExeDomain>, T8, PD, EVEX, EVEX_K, Sched<[SchedRR]>;1096 1097 let hasSideEffects = 0, mayLoad = 1, isReMaterializable = 1, canFoldAsLoad = 1 in1098 def rm : AVX512PI<opc, MRMSrcMem, (outs MaskInfo.RC:$dst),1099 (ins SrcInfo.ScalarMemOp:$src),1100 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),1101 [(set MaskInfo.RC:$dst,1102 (MaskInfo.VT1103 (bitconvert1104 (DestInfo.VT1105 (UnmaskedBcastOp addr:$src)))))],1106 DestInfo.ExeDomain>, T8, PD, EVEX,1107 EVEX_CD8<SrcInfo.EltSize, CD8VT1>, Sched<[SchedRM]>;1108 1109 def rmkz : AVX512PI<opc, MRMSrcMem, (outs MaskInfo.RC:$dst),1110 (ins MaskInfo.KRCWM:$mask, SrcInfo.ScalarMemOp:$src),1111 !strconcat(OpcodeStr, "\t{$src, ${dst} {${mask}} {z}|",1112 "${dst} {${mask}} {z}, $src}"),1113 [(set MaskInfo.RC:$dst,1114 (vselect_mask MaskInfo.KRCWM:$mask,1115 (MaskInfo.VT1116 (bitconvert1117 (DestInfo.VT1118 (SrcInfo.BroadcastLdFrag addr:$src)))),1119 MaskInfo.ImmAllZerosV))],1120 DestInfo.ExeDomain>, T8, PD, EVEX, EVEX_KZ,1121 EVEX_CD8<SrcInfo.EltSize, CD8VT1>, Sched<[SchedRM]>;1122 1123 let Constraints = "$src0 = $dst",1124 isConvertibleToThreeAddress = IsConvertibleToThreeAddress in1125 def rmk : AVX512PI<opc, MRMSrcMem, (outs MaskInfo.RC:$dst),1126 (ins MaskInfo.RC:$src0, MaskInfo.KRCWM:$mask,1127 SrcInfo.ScalarMemOp:$src),1128 !strconcat(OpcodeStr, "\t{$src, ${dst} {${mask}}|",1129 "${dst} {${mask}}, $src}"),1130 [(set MaskInfo.RC:$dst,1131 (vselect_mask MaskInfo.KRCWM:$mask,1132 (MaskInfo.VT1133 (bitconvert1134 (DestInfo.VT1135 (SrcInfo.BroadcastLdFrag addr:$src)))),1136 MaskInfo.RC:$src0))],1137 DestInfo.ExeDomain>, T8, PD, EVEX, EVEX_K,1138 EVEX_CD8<SrcInfo.EltSize, CD8VT1>, Sched<[SchedRM]>;1139}1140 1141// Helper class to force mask and broadcast result to same type.1142multiclass avx512_broadcast_rm<bits<8> opc, string OpcodeStr,1143 SchedWrite SchedRR, SchedWrite SchedRM,1144 X86VectorVTInfo DestInfo,1145 X86VectorVTInfo SrcInfo,1146 bit IsConvertibleToThreeAddress> :1147 avx512_broadcast_rm_split<opc, OpcodeStr, SchedRR, SchedRM,1148 DestInfo, DestInfo, SrcInfo,1149 IsConvertibleToThreeAddress>;1150 1151multiclass avx512_fp_broadcast_sd<bits<8> opc, string OpcodeStr,1152 AVX512VLVectorVTInfo _> {1153 let Predicates = [HasAVX512] in {1154 defm Z : avx512_broadcast_rm<opc, OpcodeStr, WriteFShuffle256,1155 WriteFShuffle256Ld, _.info512, _.info128, 1>,1156 avx512_broadcast_scalar<NAME, _.info512, _.info128>,1157 EVEX_V512;1158 }1159 1160 let Predicates = [HasVLX] in {1161 defm Z256 : avx512_broadcast_rm<opc, OpcodeStr, WriteFShuffle256,1162 WriteFShuffle256Ld, _.info256, _.info128, 1>,1163 avx512_broadcast_scalar<NAME, _.info256, _.info128>,1164 EVEX_V256;1165 }1166}1167 1168multiclass avx512_fp_broadcast_ss<bits<8> opc, string OpcodeStr,1169 AVX512VLVectorVTInfo _> {1170 let Predicates = [HasAVX512] in {1171 defm Z : avx512_broadcast_rm<opc, OpcodeStr, WriteFShuffle256,1172 WriteFShuffle256Ld, _.info512, _.info128, 1>,1173 avx512_broadcast_scalar<NAME, _.info512, _.info128>,1174 EVEX_V512;1175 }1176 1177 let Predicates = [HasVLX] in {1178 defm Z256 : avx512_broadcast_rm<opc, OpcodeStr, WriteFShuffle256,1179 WriteFShuffle256Ld, _.info256, _.info128, 1>,1180 avx512_broadcast_scalar<NAME, _.info256, _.info128>,1181 EVEX_V256;1182 defm Z128 : avx512_broadcast_rm<opc, OpcodeStr, WriteFShuffle256,1183 WriteFShuffle256Ld, _.info128, _.info128, 1>,1184 avx512_broadcast_scalar<NAME, _.info128, _.info128>,1185 EVEX_V128;1186 }1187}1188defm VBROADCASTSS : avx512_fp_broadcast_ss<0x18, "vbroadcastss",1189 avx512vl_f32_info>;1190defm VBROADCASTSD : avx512_fp_broadcast_sd<0x19, "vbroadcastsd",1191 avx512vl_f64_info>, REX_W;1192 1193multiclass avx512_int_broadcast_reg<bits<8> opc, SchedWrite SchedRR,1194 X86VectorVTInfo _, SDPatternOperator OpNode,1195 RegisterClass SrcRC> {1196 // Fold with a mask even if it has multiple uses since it is cheap.1197 let ExeDomain = _.ExeDomain in1198 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),1199 (ins SrcRC:$src),1200 "vpbroadcast"#_.Suffix, "$src", "$src",1201 (_.VT (OpNode SrcRC:$src)), /*IsCommutable*/0,1202 /*IsKCommutable*/0, /*IsKZCommutable*/0, vselect>,1203 T8, PD, EVEX, Sched<[SchedRR]>;1204}1205 1206multiclass avx512_int_broadcastbw_reg<bits<8> opc, string Name, SchedWrite SchedRR,1207 X86VectorVTInfo _, SDPatternOperator OpNode,1208 RegisterClass SrcRC, SubRegIndex Subreg> {1209 let hasSideEffects = 0, ExeDomain = _.ExeDomain in1210 defm rr : AVX512_maskable_custom<opc, MRMSrcReg,1211 (outs _.RC:$dst), (ins GR32:$src),1212 !con((ins _.RC:$src0, _.KRCWM:$mask), (ins GR32:$src)),1213 !con((ins _.KRCWM:$mask), (ins GR32:$src)),1214 "vpbroadcast"#_.Suffix, "$src", "$src", [], [], [],1215 "$src0 = $dst">, T8, PD, EVEX, Sched<[SchedRR]>;1216 1217 def : Pat <(_.VT (OpNode SrcRC:$src)),1218 (!cast<Instruction>(Name#rr)1219 (i32 (INSERT_SUBREG (i32 (IMPLICIT_DEF)), SrcRC:$src, Subreg)))>;1220 1221 // Fold with a mask even if it has multiple uses since it is cheap.1222 def : Pat <(vselect _.KRCWM:$mask, (_.VT (OpNode SrcRC:$src)), _.RC:$src0),1223 (!cast<Instruction>(Name#rrk) _.RC:$src0, _.KRCWM:$mask,1224 (i32 (INSERT_SUBREG (i32 (IMPLICIT_DEF)), SrcRC:$src, Subreg)))>;1225 1226 def : Pat <(vselect _.KRCWM:$mask, (_.VT (OpNode SrcRC:$src)), _.ImmAllZerosV),1227 (!cast<Instruction>(Name#rrkz) _.KRCWM:$mask,1228 (i32 (INSERT_SUBREG (i32 (IMPLICIT_DEF)), SrcRC:$src, Subreg)))>;1229}1230 1231multiclass avx512_int_broadcastbw_reg_vl<bits<8> opc, string Name,1232 AVX512VLVectorVTInfo _, SDPatternOperator OpNode,1233 RegisterClass SrcRC, SubRegIndex Subreg, Predicate prd> {1234 let Predicates = [prd] in1235 defm Z : avx512_int_broadcastbw_reg<opc, Name#Z, WriteShuffle256, _.info512,1236 OpNode, SrcRC, Subreg>, EVEX_V512;1237 let Predicates = [prd, HasVLX] in {1238 defm Z256 : avx512_int_broadcastbw_reg<opc, Name#Z256, WriteShuffle256,1239 _.info256, OpNode, SrcRC, Subreg>, EVEX_V256;1240 defm Z128 : avx512_int_broadcastbw_reg<opc, Name#Z128, WriteShuffle,1241 _.info128, OpNode, SrcRC, Subreg>, EVEX_V128;1242 }1243}1244 1245multiclass avx512_int_broadcast_reg_vl<bits<8> opc, AVX512VLVectorVTInfo _,1246 SDPatternOperator OpNode,1247 RegisterClass SrcRC, Predicate prd> {1248 let Predicates = [prd] in1249 defm Z : avx512_int_broadcast_reg<opc, WriteShuffle256, _.info512, OpNode,1250 SrcRC>, EVEX_V512;1251 let Predicates = [prd, HasVLX] in {1252 defm Z256 : avx512_int_broadcast_reg<opc, WriteShuffle256, _.info256, OpNode,1253 SrcRC>, EVEX_V256;1254 defm Z128 : avx512_int_broadcast_reg<opc, WriteShuffle, _.info128, OpNode,1255 SrcRC>, EVEX_V128;1256 }1257}1258 1259defm VPBROADCASTBr : avx512_int_broadcastbw_reg_vl<0x7A, "VPBROADCASTBr",1260 avx512vl_i8_info, X86VBroadcast, GR8, sub_8bit, HasBWI>;1261defm VPBROADCASTWr : avx512_int_broadcastbw_reg_vl<0x7B, "VPBROADCASTWr",1262 avx512vl_i16_info, X86VBroadcast, GR16, sub_16bit,1263 HasBWI>;1264defm VPBROADCASTDr : avx512_int_broadcast_reg_vl<0x7C, avx512vl_i32_info,1265 X86VBroadcast, GR32, HasAVX512>;1266defm VPBROADCASTQr : avx512_int_broadcast_reg_vl<0x7C, avx512vl_i64_info,1267 X86VBroadcast, GR64, HasAVX512>, REX_W;1268 1269multiclass avx512_int_broadcast_rm_vl<bits<8> opc, string OpcodeStr,1270 AVX512VLVectorVTInfo _, Predicate prd,1271 bit IsConvertibleToThreeAddress> {1272 let Predicates = [prd] in {1273 defm Z : avx512_broadcast_rm<opc, OpcodeStr, WriteShuffle256,1274 WriteShuffle256Ld, _.info512, _.info128,1275 IsConvertibleToThreeAddress>,1276 EVEX_V512;1277 }1278 let Predicates = [prd, HasVLX] in {1279 defm Z256 : avx512_broadcast_rm<opc, OpcodeStr, WriteShuffle256,1280 WriteShuffle256Ld, _.info256, _.info128,1281 IsConvertibleToThreeAddress>,1282 EVEX_V256;1283 defm Z128 : avx512_broadcast_rm<opc, OpcodeStr, WriteShuffle,1284 WriteShuffleXLd, _.info128, _.info128,1285 IsConvertibleToThreeAddress>,1286 EVEX_V128;1287 }1288}1289 1290defm VPBROADCASTB : avx512_int_broadcast_rm_vl<0x78, "vpbroadcastb",1291 avx512vl_i8_info, HasBWI, 0>;1292defm VPBROADCASTW : avx512_int_broadcast_rm_vl<0x79, "vpbroadcastw",1293 avx512vl_i16_info, HasBWI, 0>;1294defm VPBROADCASTD : avx512_int_broadcast_rm_vl<0x58, "vpbroadcastd",1295 avx512vl_i32_info, HasAVX512, 1>;1296defm VPBROADCASTQ : avx512_int_broadcast_rm_vl<0x59, "vpbroadcastq",1297 avx512vl_i64_info, HasAVX512, 1>, REX_W;1298 1299multiclass avx512_subvec_broadcast_rm<bits<8> opc, string OpcodeStr,1300 SDPatternOperator OpNode,1301 X86VectorVTInfo _Dst,1302 X86VectorVTInfo _Src> {1303 let hasSideEffects = 0, mayLoad = 1 in1304 defm rm : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),1305 (ins _Src.MemOp:$src), OpcodeStr, "$src", "$src",1306 (_Dst.VT (OpNode addr:$src))>,1307 Sched<[SchedWriteShuffle.YMM.Folded]>,1308 AVX5128IBase, EVEX;1309}1310 1311// This should be used for the AVX512DQ broadcast instructions. It disables1312// the unmasked patterns so that we only use the DQ instructions when masking1313// is requested.1314multiclass avx512_subvec_broadcast_rm_dq<bits<8> opc, string OpcodeStr,1315 SDPatternOperator OpNode,1316 X86VectorVTInfo _Dst,1317 X86VectorVTInfo _Src> {1318 let hasSideEffects = 0, mayLoad = 1 in1319 defm rm : AVX512_maskable_split<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),1320 (ins _Src.MemOp:$src), OpcodeStr, "$src", "$src",1321 (null_frag),1322 (_Dst.VT (OpNode addr:$src))>,1323 Sched<[SchedWriteShuffle.YMM.Folded]>,1324 AVX5128IBase, EVEX;1325}1326let Predicates = [HasBWI] in {1327 def : Pat<(v32f16 (X86VBroadcastld16 addr:$src)),1328 (VPBROADCASTWZrm addr:$src)>;1329 1330 def : Pat<(v32f16 (X86VBroadcast (v8f16 VR128X:$src))),1331 (VPBROADCASTWZrr VR128X:$src)>;1332 def : Pat<(v32f16 (X86VBroadcast (f16 FR16X:$src))),1333 (VPBROADCASTWZrr (COPY_TO_REGCLASS FR16X:$src, VR128X))>;1334}1335let Predicates = [HasVLX, HasBWI] in {1336 def : Pat<(v8f16 (X86VBroadcastld16 addr:$src)),1337 (VPBROADCASTWZ128rm addr:$src)>;1338 def : Pat<(v16f16 (X86VBroadcastld16 addr:$src)),1339 (VPBROADCASTWZ256rm addr:$src)>;1340 1341 def : Pat<(v8f16 (X86VBroadcast (v8f16 VR128X:$src))),1342 (VPBROADCASTWZ128rr VR128X:$src)>;1343 def : Pat<(v16f16 (X86VBroadcast (v8f16 VR128X:$src))),1344 (VPBROADCASTWZ256rr VR128X:$src)>;1345 1346 def : Pat<(v8f16 (X86VBroadcast (f16 FR16X:$src))),1347 (VPBROADCASTWZ128rr (COPY_TO_REGCLASS FR16X:$src, VR128X))>;1348 def : Pat<(v16f16 (X86VBroadcast (f16 FR16X:$src))),1349 (VPBROADCASTWZ256rr (COPY_TO_REGCLASS FR16X:$src, VR128X))>;1350}1351 1352//===----------------------------------------------------------------------===//1353// AVX-512 BROADCAST SUBVECTORS1354//1355 1356defm VBROADCASTI32X4Z : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti32x4",1357 X86SubVBroadcastld128, v16i32_info, v4i32x_info>,1358 EVEX_V512, EVEX_CD8<32, CD8VT4>;1359defm VBROADCASTF32X4Z : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf32x4",1360 X86SubVBroadcastld128, v16f32_info, v4f32x_info>,1361 EVEX_V512, EVEX_CD8<32, CD8VT4>;1362defm VBROADCASTI64X4Z : avx512_subvec_broadcast_rm<0x5b, "vbroadcasti64x4",1363 X86SubVBroadcastld256, v8i64_info, v4i64x_info>, REX_W,1364 EVEX_V512, EVEX_CD8<64, CD8VT4>;1365defm VBROADCASTF64X4Z : avx512_subvec_broadcast_rm<0x1b, "vbroadcastf64x4",1366 X86SubVBroadcastld256, v8f64_info, v4f64x_info>, REX_W,1367 EVEX_V512, EVEX_CD8<64, CD8VT4>;1368 1369let Predicates = [HasAVX512] in {1370def : Pat<(v8f64 (X86SubVBroadcastld256 addr:$src)),1371 (VBROADCASTF64X4Zrm addr:$src)>;1372def : Pat<(v16f32 (X86SubVBroadcastld256 addr:$src)),1373 (VBROADCASTF64X4Zrm addr:$src)>;1374def : Pat<(v32f16 (X86SubVBroadcastld256 addr:$src)),1375 (VBROADCASTF64X4Zrm addr:$src)>;1376def : Pat<(v8i64 (X86SubVBroadcastld256 addr:$src)),1377 (VBROADCASTI64X4Zrm addr:$src)>;1378def : Pat<(v16i32 (X86SubVBroadcastld256 addr:$src)),1379 (VBROADCASTI64X4Zrm addr:$src)>;1380def : Pat<(v32i16 (X86SubVBroadcastld256 addr:$src)),1381 (VBROADCASTI64X4Zrm addr:$src)>;1382def : Pat<(v64i8 (X86SubVBroadcastld256 addr:$src)),1383 (VBROADCASTI64X4Zrm addr:$src)>;1384 1385def : Pat<(v8f64 (X86SubVBroadcastld128 addr:$src)),1386 (VBROADCASTF32X4Zrm addr:$src)>;1387def : Pat<(v16f32 (X86SubVBroadcastld128 addr:$src)),1388 (VBROADCASTF32X4Zrm addr:$src)>;1389def : Pat<(v32f16 (X86SubVBroadcastld128 addr:$src)),1390 (VBROADCASTF32X4Zrm addr:$src)>;1391def : Pat<(v8i64 (X86SubVBroadcastld128 addr:$src)),1392 (VBROADCASTI32X4Zrm addr:$src)>;1393def : Pat<(v16i32 (X86SubVBroadcastld128 addr:$src)),1394 (VBROADCASTI32X4Zrm addr:$src)>;1395def : Pat<(v32i16 (X86SubVBroadcastld128 addr:$src)),1396 (VBROADCASTI32X4Zrm addr:$src)>;1397def : Pat<(v64i8 (X86SubVBroadcastld128 addr:$src)),1398 (VBROADCASTI32X4Zrm addr:$src)>;1399 1400// Patterns for selects of bitcasted operations.1401def : Pat<(vselect_mask VK16WM:$mask,1402 (bc_v16f32 (v8f64 (X86SubVBroadcastld128 addr:$src))),1403 (v16f32 immAllZerosV)),1404 (VBROADCASTF32X4Zrmkz VK16WM:$mask, addr:$src)>;1405def : Pat<(vselect_mask VK16WM:$mask,1406 (bc_v16f32 (v8f64 (X86SubVBroadcastld128 addr:$src))),1407 VR512:$src0),1408 (VBROADCASTF32X4Zrmk VR512:$src0, VK16WM:$mask, addr:$src)>;1409def : Pat<(vselect_mask VK16WM:$mask,1410 (bc_v16i32 (v8i64 (X86SubVBroadcastld128 addr:$src))),1411 (v16i32 immAllZerosV)),1412 (VBROADCASTI32X4Zrmkz VK16WM:$mask, addr:$src)>;1413def : Pat<(vselect_mask VK16WM:$mask,1414 (bc_v16i32 (v8i64 (X86SubVBroadcastld128 addr:$src))),1415 VR512:$src0),1416 (VBROADCASTI32X4Zrmk VR512:$src0, VK16WM:$mask, addr:$src)>;1417 1418def : Pat<(vselect_mask VK8WM:$mask,1419 (bc_v8f64 (v16f32 (X86SubVBroadcastld256 addr:$src))),1420 (v8f64 immAllZerosV)),1421 (VBROADCASTF64X4Zrmkz VK8WM:$mask, addr:$src)>;1422def : Pat<(vselect_mask VK8WM:$mask,1423 (bc_v8f64 (v16f32 (X86SubVBroadcastld256 addr:$src))),1424 VR512:$src0),1425 (VBROADCASTF64X4Zrmk VR512:$src0, VK8WM:$mask, addr:$src)>;1426def : Pat<(vselect_mask VK8WM:$mask,1427 (bc_v8i64 (v16i32 (X86SubVBroadcastld256 addr:$src))),1428 (v8i64 immAllZerosV)),1429 (VBROADCASTI64X4Zrmkz VK8WM:$mask, addr:$src)>;1430def : Pat<(vselect_mask VK8WM:$mask,1431 (bc_v8i64 (v16i32 (X86SubVBroadcastld256 addr:$src))),1432 VR512:$src0),1433 (VBROADCASTI64X4Zrmk VR512:$src0, VK8WM:$mask, addr:$src)>;1434}1435 1436let Predicates = [HasVLX] in {1437defm VBROADCASTI32X4Z256 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti32x4",1438 X86SubVBroadcastld128, v8i32x_info, v4i32x_info>,1439 EVEX_V256, EVEX_CD8<32, CD8VT4>;1440defm VBROADCASTF32X4Z256 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf32x4",1441 X86SubVBroadcastld128, v8f32x_info, v4f32x_info>,1442 EVEX_V256, EVEX_CD8<32, CD8VT4>;1443 1444def : Pat<(v4f64 (X86SubVBroadcastld128 addr:$src)),1445 (VBROADCASTF32X4Z256rm addr:$src)>;1446def : Pat<(v8f32 (X86SubVBroadcastld128 addr:$src)),1447 (VBROADCASTF32X4Z256rm addr:$src)>;1448def : Pat<(v16f16 (X86SubVBroadcastld128 addr:$src)),1449 (VBROADCASTF32X4Z256rm addr:$src)>;1450def : Pat<(v4i64 (X86SubVBroadcastld128 addr:$src)),1451 (VBROADCASTI32X4Z256rm addr:$src)>;1452def : Pat<(v8i32 (X86SubVBroadcastld128 addr:$src)),1453 (VBROADCASTI32X4Z256rm addr:$src)>;1454def : Pat<(v16i16 (X86SubVBroadcastld128 addr:$src)),1455 (VBROADCASTI32X4Z256rm addr:$src)>;1456def : Pat<(v32i8 (X86SubVBroadcastld128 addr:$src)),1457 (VBROADCASTI32X4Z256rm addr:$src)>;1458 1459// Patterns for selects of bitcasted operations.1460def : Pat<(vselect_mask VK8WM:$mask,1461 (bc_v8f32 (v4f64 (X86SubVBroadcastld128 addr:$src))),1462 (v8f32 immAllZerosV)),1463 (VBROADCASTF32X4Z256rmkz VK8WM:$mask, addr:$src)>;1464def : Pat<(vselect_mask VK8WM:$mask,1465 (bc_v8f32 (v4f64 (X86SubVBroadcastld128 addr:$src))),1466 VR256X:$src0),1467 (VBROADCASTF32X4Z256rmk VR256X:$src0, VK8WM:$mask, addr:$src)>;1468def : Pat<(vselect_mask VK8WM:$mask,1469 (bc_v8i32 (v4i64 (X86SubVBroadcastld128 addr:$src))),1470 (v8i32 immAllZerosV)),1471 (VBROADCASTI32X4Z256rmkz VK8WM:$mask, addr:$src)>;1472def : Pat<(vselect_mask VK8WM:$mask,1473 (bc_v8i32 (v4i64 (X86SubVBroadcastld128 addr:$src))),1474 VR256X:$src0),1475 (VBROADCASTI32X4Z256rmk VR256X:$src0, VK8WM:$mask, addr:$src)>;1476}1477 1478let Predicates = [HasBF16] in {1479 def : Pat<(v32bf16 (X86SubVBroadcastld256 addr:$src)),1480 (VBROADCASTF64X4Zrm addr:$src)>;1481 def : Pat<(v32bf16 (X86SubVBroadcastld128 addr:$src)),1482 (VBROADCASTF32X4Zrm addr:$src)>;1483}1484 1485let Predicates = [HasBF16, HasVLX] in1486 def : Pat<(v16bf16 (X86SubVBroadcastld128 addr:$src)),1487 (VBROADCASTF32X4Z256rm addr:$src)>;1488 1489let Predicates = [HasVLX, HasDQI] in {1490defm VBROADCASTI64X2Z256 : avx512_subvec_broadcast_rm_dq<0x5a, "vbroadcasti64x2",1491 X86SubVBroadcastld128, v4i64x_info, v2i64x_info>,1492 EVEX_V256, EVEX_CD8<64, CD8VT2>, REX_W;1493defm VBROADCASTF64X2Z256 : avx512_subvec_broadcast_rm_dq<0x1a, "vbroadcastf64x2",1494 X86SubVBroadcastld128, v4f64x_info, v2f64x_info>,1495 EVEX_V256, EVEX_CD8<64, CD8VT2>, REX_W;1496 1497// Patterns for selects of bitcasted operations.1498def : Pat<(vselect_mask VK4WM:$mask,1499 (bc_v4f64 (v8f32 (X86SubVBroadcastld128 addr:$src))),1500 (v4f64 immAllZerosV)),1501 (VBROADCASTF64X2Z256rmkz VK4WM:$mask, addr:$src)>;1502def : Pat<(vselect_mask VK4WM:$mask,1503 (bc_v4f64 (v8f32 (X86SubVBroadcastld128 addr:$src))),1504 VR256X:$src0),1505 (VBROADCASTF64X2Z256rmk VR256X:$src0, VK4WM:$mask, addr:$src)>;1506def : Pat<(vselect_mask VK4WM:$mask,1507 (bc_v4i64 (v8i32 (X86SubVBroadcastld128 addr:$src))),1508 (v4i64 immAllZerosV)),1509 (VBROADCASTI64X2Z256rmkz VK4WM:$mask, addr:$src)>;1510def : Pat<(vselect_mask VK4WM:$mask,1511 (bc_v4i64 (v8i32 (X86SubVBroadcastld128 addr:$src))),1512 VR256X:$src0),1513 (VBROADCASTI64X2Z256rmk VR256X:$src0, VK4WM:$mask, addr:$src)>;1514}1515 1516let Predicates = [HasDQI] in {1517defm VBROADCASTI64X2Z : avx512_subvec_broadcast_rm_dq<0x5a, "vbroadcasti64x2",1518 X86SubVBroadcastld128, v8i64_info, v2i64x_info>, REX_W,1519 EVEX_V512, EVEX_CD8<64, CD8VT2>;1520defm VBROADCASTI32X8Z : avx512_subvec_broadcast_rm_dq<0x5b, "vbroadcasti32x8",1521 X86SubVBroadcastld256, v16i32_info, v8i32x_info>,1522 EVEX_V512, EVEX_CD8<32, CD8VT8>;1523defm VBROADCASTF64X2Z : avx512_subvec_broadcast_rm_dq<0x1a, "vbroadcastf64x2",1524 X86SubVBroadcastld128, v8f64_info, v2f64x_info>, REX_W,1525 EVEX_V512, EVEX_CD8<64, CD8VT2>;1526defm VBROADCASTF32X8Z : avx512_subvec_broadcast_rm_dq<0x1b, "vbroadcastf32x8",1527 X86SubVBroadcastld256, v16f32_info, v8f32x_info>,1528 EVEX_V512, EVEX_CD8<32, CD8VT8>;1529 1530// Patterns for selects of bitcasted operations.1531def : Pat<(vselect_mask VK16WM:$mask,1532 (bc_v16f32 (v8f64 (X86SubVBroadcastld256 addr:$src))),1533 (v16f32 immAllZerosV)),1534 (VBROADCASTF32X8Zrmkz VK16WM:$mask, addr:$src)>;1535def : Pat<(vselect_mask VK16WM:$mask,1536 (bc_v16f32 (v8f64 (X86SubVBroadcastld256 addr:$src))),1537 VR512:$src0),1538 (VBROADCASTF32X8Zrmk VR512:$src0, VK16WM:$mask, addr:$src)>;1539def : Pat<(vselect_mask VK16WM:$mask,1540 (bc_v16i32 (v8i64 (X86SubVBroadcastld256 addr:$src))),1541 (v16i32 immAllZerosV)),1542 (VBROADCASTI32X8Zrmkz VK16WM:$mask, addr:$src)>;1543def : Pat<(vselect_mask VK16WM:$mask,1544 (bc_v16i32 (v8i64 (X86SubVBroadcastld256 addr:$src))),1545 VR512:$src0),1546 (VBROADCASTI32X8Zrmk VR512:$src0, VK16WM:$mask, addr:$src)>;1547 1548def : Pat<(vselect_mask VK8WM:$mask,1549 (bc_v8f64 (v16f32 (X86SubVBroadcastld128 addr:$src))),1550 (v8f64 immAllZerosV)),1551 (VBROADCASTF64X2Zrmkz VK8WM:$mask, addr:$src)>;1552def : Pat<(vselect_mask VK8WM:$mask,1553 (bc_v8f64 (v16f32 (X86SubVBroadcastld128 addr:$src))),1554 VR512:$src0),1555 (VBROADCASTF64X2Zrmk VR512:$src0, VK8WM:$mask, addr:$src)>;1556def : Pat<(vselect_mask VK8WM:$mask,1557 (bc_v8i64 (v16i32 (X86SubVBroadcastld128 addr:$src))),1558 (v8i64 immAllZerosV)),1559 (VBROADCASTI64X2Zrmkz VK8WM:$mask, addr:$src)>;1560def : Pat<(vselect_mask VK8WM:$mask,1561 (bc_v8i64 (v16i32 (X86SubVBroadcastld128 addr:$src))),1562 VR512:$src0),1563 (VBROADCASTI64X2Zrmk VR512:$src0, VK8WM:$mask, addr:$src)>;1564}1565 1566multiclass avx512_common_broadcast_32x2<bits<8> opc, string OpcodeStr,1567 AVX512VLVectorVTInfo _Dst,1568 AVX512VLVectorVTInfo _Src> {1569 let Predicates = [HasDQI] in1570 defm Z : avx512_broadcast_rm_split<opc, OpcodeStr, WriteShuffle256,1571 WriteShuffle256Ld, _Dst.info512,1572 _Src.info512, _Src.info128, 0, null_frag, null_frag>,1573 EVEX_V512;1574 let Predicates = [HasDQI, HasVLX] in1575 defm Z256 : avx512_broadcast_rm_split<opc, OpcodeStr, WriteShuffle256,1576 WriteShuffle256Ld, _Dst.info256,1577 _Src.info256, _Src.info128, 0, null_frag, null_frag>,1578 EVEX_V256;1579}1580 1581multiclass avx512_common_broadcast_i32x2<bits<8> opc, string OpcodeStr,1582 AVX512VLVectorVTInfo _Dst,1583 AVX512VLVectorVTInfo _Src> :1584 avx512_common_broadcast_32x2<opc, OpcodeStr, _Dst, _Src> {1585 1586 let Predicates = [HasDQI, HasVLX] in1587 defm Z128 : avx512_broadcast_rm_split<opc, OpcodeStr, WriteShuffle,1588 WriteShuffleXLd, _Dst.info128,1589 _Src.info128, _Src.info128, 0, null_frag, null_frag>,1590 EVEX_V128;1591}1592 1593defm VBROADCASTI32X2 : avx512_common_broadcast_i32x2<0x59, "vbroadcasti32x2",1594 avx512vl_i32_info, avx512vl_i64_info>;1595defm VBROADCASTF32X2 : avx512_common_broadcast_32x2<0x19, "vbroadcastf32x2",1596 avx512vl_f32_info, avx512vl_f64_info>;1597 1598//===----------------------------------------------------------------------===//1599// AVX-512 BROADCAST MASK TO VECTOR REGISTER1600//---1601multiclass avx512_mask_broadcastm<bits<8> opc, string OpcodeStr,1602 X86VectorVTInfo _, RegisterClass KRC> {1603 def rr : AVX512XS8I<opc, MRMSrcReg, (outs _.RC:$dst), (ins KRC:$src),1604 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),1605 [(set _.RC:$dst, (_.VT (X86VBroadcastm KRC:$src)))]>,1606 EVEX, Sched<[WriteShuffle]>;1607}1608 1609multiclass avx512_mask_broadcast<bits<8> opc, string OpcodeStr,1610 AVX512VLVectorVTInfo VTInfo, RegisterClass KRC> {1611 let Predicates = [HasCDI] in1612 defm Z : avx512_mask_broadcastm<opc, OpcodeStr, VTInfo.info512, KRC>, EVEX_V512;1613 let Predicates = [HasCDI, HasVLX] in {1614 defm Z256 : avx512_mask_broadcastm<opc, OpcodeStr, VTInfo.info256, KRC>, EVEX_V256;1615 defm Z128 : avx512_mask_broadcastm<opc, OpcodeStr, VTInfo.info128, KRC>, EVEX_V128;1616 }1617}1618 1619defm VPBROADCASTMW2D : avx512_mask_broadcast<0x3A, "vpbroadcastmw2d",1620 avx512vl_i32_info, VK16>;1621defm VPBROADCASTMB2Q : avx512_mask_broadcast<0x2A, "vpbroadcastmb2q",1622 avx512vl_i64_info, VK8>, REX_W;1623 1624//===----------------------------------------------------------------------===//1625// -- VPERMI2 - 3 source operands form --1626multiclass avx512_perm_i<bits<8> opc, string OpcodeStr,1627 X86FoldableSchedWrite sched,1628 X86VectorVTInfo _, X86VectorVTInfo IdxVT> {1629let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain,1630 hasSideEffects = 0 in {1631 defm rr: AVX512_maskable_3src_cast<opc, MRMSrcReg, _, IdxVT, (outs _.RC:$dst),1632 (ins _.RC:$src2, _.RC:$src3),1633 OpcodeStr, "$src3, $src2", "$src2, $src3",1634 (_.VT (X86VPermt2 _.RC:$src2, IdxVT.RC:$src1, _.RC:$src3)), 1>,1635 EVEX, VVVV, AVX5128IBase, Sched<[sched]>;1636 1637 let mayLoad = 1 in1638 defm rm: AVX512_maskable_3src_cast<opc, MRMSrcMem, _, IdxVT, (outs _.RC:$dst),1639 (ins _.RC:$src2, _.MemOp:$src3),1640 OpcodeStr, "$src3, $src2", "$src2, $src3",1641 (_.VT (X86VPermt2 _.RC:$src2, IdxVT.RC:$src1,1642 (_.VT (_.LdFrag addr:$src3)))), 1>,1643 EVEX, VVVV, AVX5128IBase, Sched<[sched.Folded, sched.ReadAfterFold]>;1644 }1645}1646 1647multiclass avx512_perm_i_mb<bits<8> opc, string OpcodeStr,1648 X86FoldableSchedWrite sched,1649 X86VectorVTInfo _, X86VectorVTInfo IdxVT> {1650 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain,1651 hasSideEffects = 0, mayLoad = 1 in1652 defm rmb: AVX512_maskable_3src_cast<opc, MRMSrcMem, _, IdxVT, (outs _.RC:$dst),1653 (ins _.RC:$src2, _.ScalarMemOp:$src3),1654 OpcodeStr, !strconcat("${src3}", _.BroadcastStr,", $src2"),1655 !strconcat("$src2, ${src3}", _.BroadcastStr ),1656 (_.VT (X86VPermt2 _.RC:$src2,1657 IdxVT.RC:$src1,(_.VT (_.BroadcastLdFrag addr:$src3)))), 1>,1658 AVX5128IBase, EVEX, VVVV, EVEX_B,1659 Sched<[sched.Folded, sched.ReadAfterFold]>;1660}1661 1662multiclass avx512_perm_i_sizes<bits<8> opc, string OpcodeStr,1663 X86FoldableSchedWrite sched,1664 AVX512VLVectorVTInfo VTInfo,1665 AVX512VLVectorVTInfo ShuffleMask> {1666 defm NAME#Z: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info512,1667 ShuffleMask.info512>,1668 avx512_perm_i_mb<opc, OpcodeStr, sched, VTInfo.info512,1669 ShuffleMask.info512>, EVEX_V512;1670 let Predicates = [HasVLX] in {1671 defm NAME#Z128: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info128,1672 ShuffleMask.info128>,1673 avx512_perm_i_mb<opc, OpcodeStr, sched, VTInfo.info128,1674 ShuffleMask.info128>, EVEX_V128;1675 defm NAME#Z256: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info256,1676 ShuffleMask.info256>,1677 avx512_perm_i_mb<opc, OpcodeStr, sched, VTInfo.info256,1678 ShuffleMask.info256>, EVEX_V256;1679 }1680}1681 1682multiclass avx512_perm_i_sizes_bw<bits<8> opc, string OpcodeStr,1683 X86FoldableSchedWrite sched,1684 AVX512VLVectorVTInfo VTInfo,1685 AVX512VLVectorVTInfo Idx,1686 Predicate Prd> {1687 let Predicates = [Prd] in1688 defm NAME#Z: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info512,1689 Idx.info512>, EVEX_V512;1690 let Predicates = [Prd, HasVLX] in {1691 defm NAME#Z128: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info128,1692 Idx.info128>, EVEX_V128;1693 defm NAME#Z256: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info256,1694 Idx.info256>, EVEX_V256;1695 }1696}1697 1698defm VPERMI2D : avx512_perm_i_sizes<0x76, "vpermi2d", WriteVarShuffle256,1699 avx512vl_i32_info, avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;1700defm VPERMI2Q : avx512_perm_i_sizes<0x76, "vpermi2q", WriteVarShuffle256,1701 avx512vl_i64_info, avx512vl_i64_info>, REX_W, EVEX_CD8<64, CD8VF>;1702defm VPERMI2W : avx512_perm_i_sizes_bw<0x75, "vpermi2w", WriteVarShuffle256,1703 avx512vl_i16_info, avx512vl_i16_info, HasBWI>,1704 REX_W, EVEX_CD8<16, CD8VF>;1705defm VPERMI2B : avx512_perm_i_sizes_bw<0x75, "vpermi2b", WriteVarShuffle256,1706 avx512vl_i8_info, avx512vl_i8_info, HasVBMI>,1707 EVEX_CD8<8, CD8VF>;1708defm VPERMI2PS : avx512_perm_i_sizes<0x77, "vpermi2ps", WriteFVarShuffle256,1709 avx512vl_f32_info, avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;1710defm VPERMI2PD : avx512_perm_i_sizes<0x77, "vpermi2pd", WriteFVarShuffle256,1711 avx512vl_f64_info, avx512vl_i64_info>, REX_W, EVEX_CD8<64, CD8VF>;1712 1713// Extra patterns to deal with extra bitcasts due to passthru and index being1714// different types on the fp versions.1715multiclass avx512_perm_i_lowering<string InstrStr, X86VectorVTInfo _,1716 X86VectorVTInfo IdxVT,1717 X86VectorVTInfo CastVT> {1718 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,1719 (X86VPermt2 (_.VT _.RC:$src2),1720 (IdxVT.VT (bitconvert1721 (CastVT.VT _.RC:$src1))),1722 _.RC:$src3),1723 (_.VT (bitconvert (CastVT.VT _.RC:$src1))))),1724 (!cast<Instruction>(InstrStr#"rrk") _.RC:$src1, _.KRCWM:$mask,1725 _.RC:$src2, _.RC:$src3)>;1726 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,1727 (X86VPermt2 _.RC:$src2,1728 (IdxVT.VT (bitconvert1729 (CastVT.VT _.RC:$src1))),1730 (_.LdFrag addr:$src3)),1731 (_.VT (bitconvert (CastVT.VT _.RC:$src1))))),1732 (!cast<Instruction>(InstrStr#"rmk") _.RC:$src1, _.KRCWM:$mask,1733 _.RC:$src2, addr:$src3)>;1734 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,1735 (X86VPermt2 _.RC:$src2,1736 (IdxVT.VT (bitconvert (CastVT.VT _.RC:$src1))),1737 (_.BroadcastLdFrag addr:$src3)),1738 (_.VT (bitconvert (CastVT.VT _.RC:$src1))))),1739 (!cast<Instruction>(InstrStr#"rmbk") _.RC:$src1, _.KRCWM:$mask,1740 _.RC:$src2, addr:$src3)>;1741}1742 1743// TODO: Should we add more casts? The vXi64 case is common due to ABI.1744defm : avx512_perm_i_lowering<"VPERMI2PSZ", v16f32_info, v16i32_info, v8i64_info>;1745defm : avx512_perm_i_lowering<"VPERMI2PSZ256", v8f32x_info, v8i32x_info, v4i64x_info>;1746defm : avx512_perm_i_lowering<"VPERMI2PSZ128", v4f32x_info, v4i32x_info, v2i64x_info>;1747 1748// VPERMT21749multiclass avx512_perm_t<bits<8> opc, string OpcodeStr,1750 X86FoldableSchedWrite sched,1751 X86VectorVTInfo _, X86VectorVTInfo IdxVT> {1752let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain in {1753 defm rr: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),1754 (ins IdxVT.RC:$src2, _.RC:$src3),1755 OpcodeStr, "$src3, $src2", "$src2, $src3",1756 (_.VT (X86VPermt2 _.RC:$src1, IdxVT.RC:$src2, _.RC:$src3)), 1>,1757 EVEX, VVVV, AVX5128IBase, Sched<[sched]>;1758 1759 let hasSideEffects = 0, mayLoad = 1 in1760 defm rm: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),1761 (ins IdxVT.RC:$src2, _.MemOp:$src3),1762 OpcodeStr, "$src3, $src2", "$src2, $src3",1763 (_.VT (X86VPermt2 _.RC:$src1, IdxVT.RC:$src2,1764 (_.LdFrag addr:$src3))), 1>,1765 EVEX, VVVV, AVX5128IBase, Sched<[sched.Folded, sched.ReadAfterFold]>;1766 }1767}1768multiclass avx512_perm_t_mb<bits<8> opc, string OpcodeStr,1769 X86FoldableSchedWrite sched,1770 X86VectorVTInfo _, X86VectorVTInfo IdxVT> {1771 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain, hasSideEffects = 0, mayLoad = 1 in1772 defm rmb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),1773 (ins IdxVT.RC:$src2, _.ScalarMemOp:$src3),1774 OpcodeStr, !strconcat("${src3}", _.BroadcastStr,", $src2"),1775 !strconcat("$src2, ${src3}", _.BroadcastStr ),1776 (_.VT (X86VPermt2 _.RC:$src1,1777 IdxVT.RC:$src2,(_.VT (_.BroadcastLdFrag addr:$src3)))), 1>,1778 AVX5128IBase, EVEX, VVVV, EVEX_B,1779 Sched<[sched.Folded, sched.ReadAfterFold]>;1780}1781 1782multiclass avx512_perm_t_sizes<bits<8> opc, string OpcodeStr,1783 X86FoldableSchedWrite sched,1784 AVX512VLVectorVTInfo VTInfo,1785 AVX512VLVectorVTInfo ShuffleMask> {1786 defm NAME#Z: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info512,1787 ShuffleMask.info512>,1788 avx512_perm_t_mb<opc, OpcodeStr, sched, VTInfo.info512,1789 ShuffleMask.info512>, EVEX_V512;1790 let Predicates = [HasVLX] in {1791 defm NAME#Z128: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info128,1792 ShuffleMask.info128>,1793 avx512_perm_t_mb<opc, OpcodeStr, sched, VTInfo.info128,1794 ShuffleMask.info128>, EVEX_V128;1795 defm NAME#Z256: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info256,1796 ShuffleMask.info256>,1797 avx512_perm_t_mb<opc, OpcodeStr, sched, VTInfo.info256,1798 ShuffleMask.info256>, EVEX_V256;1799 }1800}1801 1802multiclass avx512_perm_t_sizes_bw<bits<8> opc, string OpcodeStr,1803 X86FoldableSchedWrite sched,1804 AVX512VLVectorVTInfo VTInfo,1805 AVX512VLVectorVTInfo Idx, Predicate Prd> {1806 let Predicates = [Prd] in1807 defm NAME#Z: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info512,1808 Idx.info512>, EVEX_V512;1809 let Predicates = [Prd, HasVLX] in {1810 defm NAME#Z128: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info128,1811 Idx.info128>, EVEX_V128;1812 defm NAME#Z256: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info256,1813 Idx.info256>, EVEX_V256;1814 }1815}1816 1817defm VPERMT2D : avx512_perm_t_sizes<0x7E, "vpermt2d", WriteVarShuffle256,1818 avx512vl_i32_info, avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;1819defm VPERMT2Q : avx512_perm_t_sizes<0x7E, "vpermt2q", WriteVarShuffle256,1820 avx512vl_i64_info, avx512vl_i64_info>, REX_W, EVEX_CD8<64, CD8VF>;1821defm VPERMT2W : avx512_perm_t_sizes_bw<0x7D, "vpermt2w", WriteVarShuffle256,1822 avx512vl_i16_info, avx512vl_i16_info, HasBWI>,1823 REX_W, EVEX_CD8<16, CD8VF>;1824defm VPERMT2B : avx512_perm_t_sizes_bw<0x7D, "vpermt2b", WriteVarShuffle256,1825 avx512vl_i8_info, avx512vl_i8_info, HasVBMI>,1826 EVEX_CD8<8, CD8VF>;1827defm VPERMT2PS : avx512_perm_t_sizes<0x7F, "vpermt2ps", WriteFVarShuffle256,1828 avx512vl_f32_info, avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;1829defm VPERMT2PD : avx512_perm_t_sizes<0x7F, "vpermt2pd", WriteFVarShuffle256,1830 avx512vl_f64_info, avx512vl_i64_info>, REX_W, EVEX_CD8<64, CD8VF>;1831 1832//===----------------------------------------------------------------------===//1833// AVX-512 - BLEND using mask1834//1835 1836multiclass WriteFVarBlendask<bits<8> opc, string OpcodeStr,1837 X86FoldableSchedWrite sched, X86VectorVTInfo _> {1838 let ExeDomain = _.ExeDomain, hasSideEffects = 0 in {1839 def rr : AVX5128I<opc, MRMSrcReg, (outs _.RC:$dst),1840 (ins _.RC:$src1, _.RC:$src2),1841 !strconcat(OpcodeStr,1842 "\t{$src2, $src1, ${dst}|${dst}, $src1, $src2}"), []>,1843 EVEX, VVVV, Sched<[sched]>;1844 def rrk : AVX5128I<opc, MRMSrcReg, (outs _.RC:$dst),1845 (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2),1846 !strconcat(OpcodeStr,1847 "\t{$src2, $src1, ${dst} {${mask}}|${dst} {${mask}}, $src1, $src2}"),1848 []>, EVEX, VVVV, EVEX_K, Sched<[sched]>;1849 def rrkz : AVX5128I<opc, MRMSrcReg, (outs _.RC:$dst),1850 (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2),1851 !strconcat(OpcodeStr,1852 "\t{$src2, $src1, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src1, $src2}"),1853 []>, EVEX, VVVV, EVEX_KZ, Sched<[sched]>;1854 let mayLoad = 1 in {1855 def rm : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),1856 (ins _.RC:$src1, _.MemOp:$src2),1857 !strconcat(OpcodeStr,1858 "\t{$src2, $src1, ${dst}|${dst}, $src1, $src2}"),1859 []>, EVEX, VVVV, EVEX_CD8<_.EltSize, CD8VF>,1860 Sched<[sched.Folded, sched.ReadAfterFold]>;1861 def rmk : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),1862 (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2),1863 !strconcat(OpcodeStr,1864 "\t{$src2, $src1, ${dst} {${mask}}|${dst} {${mask}}, $src1, $src2}"),1865 []>, EVEX, VVVV, EVEX_K, EVEX_CD8<_.EltSize, CD8VF>,1866 Sched<[sched.Folded, sched.ReadAfterFold]>;1867 def rmkz : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),1868 (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2),1869 !strconcat(OpcodeStr,1870 "\t{$src2, $src1, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src1, $src2}"),1871 []>, EVEX, VVVV, EVEX_KZ, EVEX_CD8<_.EltSize, CD8VF>,1872 Sched<[sched.Folded, sched.ReadAfterFold]>;1873 }1874 }1875}1876multiclass WriteFVarBlendask_rmb<bits<8> opc, string OpcodeStr,1877 X86FoldableSchedWrite sched, X86VectorVTInfo _> {1878 let ExeDomain = _.ExeDomain, mayLoad = 1, hasSideEffects = 0 in {1879 def rmbk : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),1880 (ins _.KRCWM:$mask, _.RC:$src1, _.ScalarMemOp:$src2),1881 !strconcat(OpcodeStr,1882 "\t{${src2}", _.BroadcastStr, ", $src1, $dst {${mask}}|",1883 "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, "}"), []>,1884 EVEX, VVVV, EVEX_K, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>,1885 Sched<[sched.Folded, sched.ReadAfterFold]>;1886 1887 def rmbkz : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),1888 (ins _.KRCWM:$mask, _.RC:$src1, _.ScalarMemOp:$src2),1889 !strconcat(OpcodeStr,1890 "\t{${src2}", _.BroadcastStr, ", $src1, $dst {${mask}} {z}|",1891 "$dst {${mask}} {z}, $src1, ${src2}", _.BroadcastStr, "}"), []>,1892 EVEX, VVVV, EVEX_KZ, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>,1893 Sched<[sched.Folded, sched.ReadAfterFold]>;1894 1895 def rmb : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),1896 (ins _.RC:$src1, _.ScalarMemOp:$src2),1897 !strconcat(OpcodeStr,1898 "\t{${src2}", _.BroadcastStr, ", $src1, $dst|",1899 "$dst, $src1, ${src2}", _.BroadcastStr, "}"), []>,1900 EVEX, VVVV, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>,1901 Sched<[sched.Folded, sched.ReadAfterFold]>;1902 }1903}1904 1905multiclass blendmask_dq<bits<8> opc, string OpcodeStr, X86SchedWriteWidths sched,1906 AVX512VLVectorVTInfo VTInfo> {1907 defm Z : WriteFVarBlendask<opc, OpcodeStr, sched.ZMM, VTInfo.info512>,1908 WriteFVarBlendask_rmb<opc, OpcodeStr, sched.ZMM, VTInfo.info512>,1909 EVEX_V512;1910 1911 let Predicates = [HasVLX] in {1912 defm Z256 : WriteFVarBlendask<opc, OpcodeStr, sched.YMM, VTInfo.info256>,1913 WriteFVarBlendask_rmb<opc, OpcodeStr, sched.YMM, VTInfo.info256>,1914 EVEX_V256;1915 defm Z128 : WriteFVarBlendask<opc, OpcodeStr, sched.XMM, VTInfo.info128>,1916 WriteFVarBlendask_rmb<opc, OpcodeStr, sched.XMM, VTInfo.info128>,1917 EVEX_V128;1918 }1919}1920 1921multiclass blendmask_bw<bits<8> opc, string OpcodeStr, X86SchedWriteWidths sched,1922 AVX512VLVectorVTInfo VTInfo> {1923 let Predicates = [HasBWI] in1924 defm Z : WriteFVarBlendask<opc, OpcodeStr, sched.ZMM, VTInfo.info512>,1925 EVEX_V512;1926 1927 let Predicates = [HasBWI, HasVLX] in {1928 defm Z256 : WriteFVarBlendask<opc, OpcodeStr, sched.YMM, VTInfo.info256>,1929 EVEX_V256;1930 defm Z128 : WriteFVarBlendask<opc, OpcodeStr, sched.XMM, VTInfo.info128>,1931 EVEX_V128;1932 }1933}1934 1935defm VBLENDMPS : blendmask_dq<0x65, "vblendmps", SchedWriteFVarBlend,1936 avx512vl_f32_info>;1937defm VBLENDMPD : blendmask_dq<0x65, "vblendmpd", SchedWriteFVarBlend,1938 avx512vl_f64_info>, REX_W;1939defm VPBLENDMD : blendmask_dq<0x64, "vpblendmd", SchedWriteVarBlend,1940 avx512vl_i32_info>;1941defm VPBLENDMQ : blendmask_dq<0x64, "vpblendmq", SchedWriteVarBlend,1942 avx512vl_i64_info>, REX_W;1943defm VPBLENDMB : blendmask_bw<0x66, "vpblendmb", SchedWriteVarBlend,1944 avx512vl_i8_info>;1945defm VPBLENDMW : blendmask_bw<0x66, "vpblendmw", SchedWriteVarBlend,1946 avx512vl_i16_info>, REX_W;1947 1948//===----------------------------------------------------------------------===//1949// Compare Instructions1950//===----------------------------------------------------------------------===//1951 1952// avx512_cmp_scalar - AVX512 CMPSS and CMPSD1953 1954multiclass avx512_cmp_scalar<X86VectorVTInfo _, SDNode OpNode, SDNode OpNodeSAE,1955 PatFrag OpNode_su, PatFrag OpNodeSAE_su,1956 X86FoldableSchedWrite sched> {1957 defm rri : AVX512_maskable_cmp<0xC2, MRMSrcReg, _,1958 (outs _.KRC:$dst),1959 (ins _.RC:$src1, _.RC:$src2, u8imm:$cc),1960 "vcmp"#_.Suffix,1961 "$cc, $src2, $src1", "$src1, $src2, $cc",1962 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), timm:$cc),1963 (OpNode_su (_.VT _.RC:$src1), (_.VT _.RC:$src2), timm:$cc), 0, "_Int">,1964 EVEX, VVVV, VEX_LIG, Sched<[sched]>, SIMD_EXC;1965 let mayLoad = 1 in1966 defm rmi : AVX512_maskable_cmp<0xC2, MRMSrcMem, _,1967 (outs _.KRC:$dst),1968 (ins _.RC:$src1, _.IntScalarMemOp:$src2, u8imm:$cc),1969 "vcmp"#_.Suffix,1970 "$cc, $src2, $src1", "$src1, $src2, $cc",1971 (OpNode (_.VT _.RC:$src1), (_.ScalarIntMemFrags addr:$src2),1972 timm:$cc),1973 (OpNode_su (_.VT _.RC:$src1), (_.ScalarIntMemFrags addr:$src2),1974 timm:$cc), 0, "_Int">, EVEX, VVVV, VEX_LIG, EVEX_CD8<_.EltSize, CD8VT1>,1975 Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC;1976 1977 let Uses = [MXCSR] in1978 defm rrib : AVX512_maskable_cmp<0xC2, MRMSrcReg, _,1979 (outs _.KRC:$dst),1980 (ins _.RC:$src1, _.RC:$src2, u8imm:$cc),1981 "vcmp"#_.Suffix,1982 "$cc, {sae}, $src2, $src1","$src1, $src2, {sae}, $cc",1983 (OpNodeSAE (_.VT _.RC:$src1), (_.VT _.RC:$src2),1984 timm:$cc),1985 (OpNodeSAE_su (_.VT _.RC:$src1), (_.VT _.RC:$src2),1986 timm:$cc), 0, "_Int">,1987 EVEX, VVVV, VEX_LIG, EVEX_B, Sched<[sched]>;1988 1989 let isCodeGenOnly = 1 in {1990 let isCommutable = 1 in1991 def rri : AVX512Ii8<0xC2, MRMSrcReg,1992 (outs _.KRC:$dst), (ins _.FRC:$src1, _.FRC:$src2, u8imm:$cc),1993 !strconcat("vcmp", _.Suffix,1994 "\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}"),1995 [(set _.KRC:$dst, (OpNode _.FRC:$src1,1996 _.FRC:$src2,1997 timm:$cc))]>,1998 EVEX, VVVV, VEX_LIG, Sched<[sched]>, SIMD_EXC;1999 let mayLoad = 1 in2000 def rmi : AVX512Ii8<0xC2, MRMSrcMem,2001 (outs _.KRC:$dst),2002 (ins _.FRC:$src1, _.ScalarMemOp:$src2, u8imm:$cc),2003 !strconcat("vcmp", _.Suffix,2004 "\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}"),2005 [(set _.KRC:$dst, (OpNode _.FRC:$src1,2006 (_.ScalarLdFrag addr:$src2),2007 timm:$cc))]>,2008 EVEX, VVVV, VEX_LIG, EVEX_CD8<_.EltSize, CD8VT1>,2009 Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC;2010 }2011}2012 2013let Predicates = [HasAVX512] in {2014 let ExeDomain = SSEPackedSingle in2015 defm VCMPSSZ : avx512_cmp_scalar<f32x_info, X86cmpms, X86cmpmsSAE,2016 X86cmpms_su, X86cmpmsSAE_su,2017 SchedWriteFCmp.Scl>, AVX512XSIi8Base;2018 let ExeDomain = SSEPackedDouble in2019 defm VCMPSDZ : avx512_cmp_scalar<f64x_info, X86cmpms, X86cmpmsSAE,2020 X86cmpms_su, X86cmpmsSAE_su,2021 SchedWriteFCmp.Scl>, AVX512XDIi8Base, REX_W;2022}2023let Predicates = [HasFP16], ExeDomain = SSEPackedSingle in2024 defm VCMPSHZ : avx512_cmp_scalar<f16x_info, X86cmpms, X86cmpmsSAE,2025 X86cmpms_su, X86cmpmsSAE_su,2026 SchedWriteFCmp.Scl>, AVX512XSIi8Base, TA;2027 2028multiclass avx512_icmp_packed<bits<8> opc, string OpcodeStr,2029 X86FoldableSchedWrite sched,2030 X86VectorVTInfo _, bit IsCommutable> {2031 let isCommutable = IsCommutable, hasSideEffects = 0 in2032 def rr : AVX512BI<opc, MRMSrcReg,2033 (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2),2034 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),2035 []>, EVEX, VVVV, Sched<[sched]>;2036 let mayLoad = 1, hasSideEffects = 0 in2037 def rm : AVX512BI<opc, MRMSrcMem,2038 (outs _.KRC:$dst), (ins _.RC:$src1, _.MemOp:$src2),2039 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),2040 []>, EVEX, VVVV, Sched<[sched.Folded, sched.ReadAfterFold]>;2041 let isCommutable = IsCommutable, hasSideEffects = 0 in2042 def rrk : AVX512BI<opc, MRMSrcReg,2043 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2),2044 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst {${mask}}|",2045 "$dst {${mask}}, $src1, $src2}"),2046 []>, EVEX, VVVV, EVEX_K, Sched<[sched]>;2047 let mayLoad = 1, hasSideEffects = 0 in2048 def rmk : AVX512BI<opc, MRMSrcMem,2049 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2),2050 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst {${mask}}|",2051 "$dst {${mask}}, $src1, $src2}"),2052 []>, EVEX, VVVV, EVEX_K, Sched<[sched.Folded, sched.ReadAfterFold]>;2053}2054 2055multiclass avx512_icmp_packed_rmb<bits<8> opc, string OpcodeStr,2056 X86FoldableSchedWrite sched, X86VectorVTInfo _,2057 bit IsCommutable> :2058 avx512_icmp_packed<opc, OpcodeStr, sched, _, IsCommutable> {2059 let mayLoad = 1, hasSideEffects = 0 in {2060 def rmb : AVX512BI<opc, MRMSrcMem,2061 (outs _.KRC:$dst), (ins _.RC:$src1, _.ScalarMemOp:$src2),2062 !strconcat(OpcodeStr, "\t{${src2}", _.BroadcastStr, ", $src1, $dst",2063 "|$dst, $src1, ${src2}", _.BroadcastStr, "}"),2064 []>, EVEX, VVVV, EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;2065 def rmbk : AVX512BI<opc, MRMSrcMem,2066 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1,2067 _.ScalarMemOp:$src2),2068 !strconcat(OpcodeStr,2069 "\t{${src2}", _.BroadcastStr, ", $src1, $dst {${mask}}|",2070 "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, "}"),2071 []>, EVEX, VVVV, EVEX_K, EVEX_B,2072 Sched<[sched.Folded, sched.ReadAfterFold]>;2073 }2074}2075 2076multiclass avx512_icmp_packed_vl<bits<8> opc, string OpcodeStr,2077 X86SchedWriteWidths sched,2078 AVX512VLVectorVTInfo VTInfo, Predicate prd,2079 bit IsCommutable = 0> {2080 let Predicates = [prd] in2081 defm Z : avx512_icmp_packed<opc, OpcodeStr, sched.ZMM,2082 VTInfo.info512, IsCommutable>, EVEX_V512;2083 2084 let Predicates = [prd, HasVLX] in {2085 defm Z256 : avx512_icmp_packed<opc, OpcodeStr, sched.YMM,2086 VTInfo.info256, IsCommutable>, EVEX_V256;2087 defm Z128 : avx512_icmp_packed<opc, OpcodeStr, sched.XMM,2088 VTInfo.info128, IsCommutable>, EVEX_V128;2089 }2090}2091 2092multiclass avx512_icmp_packed_rmb_vl<bits<8> opc, string OpcodeStr,2093 X86SchedWriteWidths sched,2094 AVX512VLVectorVTInfo VTInfo,2095 Predicate prd, bit IsCommutable = 0> {2096 let Predicates = [prd] in2097 defm Z : avx512_icmp_packed_rmb<opc, OpcodeStr, sched.ZMM,2098 VTInfo.info512, IsCommutable>, EVEX_V512;2099 2100 let Predicates = [prd, HasVLX] in {2101 defm Z256 : avx512_icmp_packed_rmb<opc, OpcodeStr, sched.YMM,2102 VTInfo.info256, IsCommutable>, EVEX_V256;2103 defm Z128 : avx512_icmp_packed_rmb<opc, OpcodeStr, sched.XMM,2104 VTInfo.info128, IsCommutable>, EVEX_V128;2105 }2106}2107 2108// AddedComplexity is needed because the explicit SETEQ/SETGT CondCode doesn't2109// increase the pattern complexity the way an immediate would.2110let AddedComplexity = 2 in {2111// FIXME: Is there a better scheduler class for VPCMP?2112defm VPCMPEQB : avx512_icmp_packed_vl<0x74, "vpcmpeqb",2113 SchedWriteVecALU, avx512vl_i8_info, HasBWI, 1>,2114 EVEX_CD8<8, CD8VF>, WIG;2115 2116defm VPCMPEQW : avx512_icmp_packed_vl<0x75, "vpcmpeqw",2117 SchedWriteVecALU, avx512vl_i16_info, HasBWI, 1>,2118 EVEX_CD8<16, CD8VF>, WIG;2119 2120defm VPCMPEQD : avx512_icmp_packed_rmb_vl<0x76, "vpcmpeqd",2121 SchedWriteVecALU, avx512vl_i32_info, HasAVX512, 1>,2122 EVEX_CD8<32, CD8VF>;2123 2124defm VPCMPEQQ : avx512_icmp_packed_rmb_vl<0x29, "vpcmpeqq",2125 SchedWriteVecALU, avx512vl_i64_info, HasAVX512, 1>,2126 T8, REX_W, EVEX_CD8<64, CD8VF>;2127 2128defm VPCMPGTB : avx512_icmp_packed_vl<0x64, "vpcmpgtb",2129 SchedWriteVecALU, avx512vl_i8_info, HasBWI>,2130 EVEX_CD8<8, CD8VF>, WIG;2131 2132defm VPCMPGTW : avx512_icmp_packed_vl<0x65, "vpcmpgtw",2133 SchedWriteVecALU, avx512vl_i16_info, HasBWI>,2134 EVEX_CD8<16, CD8VF>, WIG;2135 2136defm VPCMPGTD : avx512_icmp_packed_rmb_vl<0x66, "vpcmpgtd",2137 SchedWriteVecALU, avx512vl_i32_info, HasAVX512>,2138 EVEX_CD8<32, CD8VF>;2139 2140defm VPCMPGTQ : avx512_icmp_packed_rmb_vl<0x37, "vpcmpgtq",2141 SchedWriteVecALU, avx512vl_i64_info, HasAVX512>,2142 T8, REX_W, EVEX_CD8<64, CD8VF>;2143}2144 2145multiclass avx512_icmp_cc<bits<8> opc, string Suffix, PatFrag Frag,2146 PatFrag Frag_su,2147 X86FoldableSchedWrite sched,2148 X86VectorVTInfo _, string Name> {2149 let isCommutable = 1 in2150 def rri : AVX512AIi8<opc, MRMSrcReg,2151 (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2, u8imm:$cc),2152 !strconcat("vpcmp", Suffix,2153 "\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}"),2154 [(set _.KRC:$dst, (_.KVT (Frag:$cc (_.VT _.RC:$src1),2155 (_.VT _.RC:$src2),2156 cond)))]>,2157 EVEX, VVVV, Sched<[sched]>;2158 let mayLoad = 1 in2159 def rmi : AVX512AIi8<opc, MRMSrcMem,2160 (outs _.KRC:$dst), (ins _.RC:$src1, _.MemOp:$src2, u8imm:$cc),2161 !strconcat("vpcmp", Suffix,2162 "\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}"),2163 [(set _.KRC:$dst, (_.KVT2164 (Frag:$cc2165 (_.VT _.RC:$src1),2166 (_.VT (_.LdFrag addr:$src2)),2167 cond)))]>,2168 EVEX, VVVV, Sched<[sched.Folded, sched.ReadAfterFold]>;2169 let isCommutable = 1 in2170 def rrik : AVX512AIi8<opc, MRMSrcReg,2171 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2,2172 u8imm:$cc),2173 !strconcat("vpcmp", Suffix,2174 "\t{$cc, $src2, $src1, $dst {${mask}}|",2175 "$dst {${mask}}, $src1, $src2, $cc}"),2176 [(set _.KRC:$dst, (and _.KRCWM:$mask,2177 (_.KVT (Frag_su:$cc (_.VT _.RC:$src1),2178 (_.VT _.RC:$src2),2179 cond))))]>,2180 EVEX, VVVV, EVEX_K, Sched<[sched]>;2181 let mayLoad = 1 in2182 def rmik : AVX512AIi8<opc, MRMSrcMem,2183 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2,2184 u8imm:$cc),2185 !strconcat("vpcmp", Suffix,2186 "\t{$cc, $src2, $src1, $dst {${mask}}|",2187 "$dst {${mask}}, $src1, $src2, $cc}"),2188 [(set _.KRC:$dst, (and _.KRCWM:$mask,2189 (_.KVT2190 (Frag_su:$cc2191 (_.VT _.RC:$src1),2192 (_.VT (_.LdFrag addr:$src2)),2193 cond))))]>,2194 EVEX, VVVV, EVEX_K, Sched<[sched.Folded, sched.ReadAfterFold]>;2195 2196 def : Pat<(_.KVT (Frag:$cc (_.LdFrag addr:$src2),2197 (_.VT _.RC:$src1), cond)),2198 (!cast<Instruction>(Name#_.ZSuffix#"rmi")2199 _.RC:$src1, addr:$src2, (X86pcmpm_imm_commute $cc))>;2200 2201 def : Pat<(and _.KRCWM:$mask,2202 (_.KVT (Frag_su:$cc (_.LdFrag addr:$src2),2203 (_.VT _.RC:$src1), cond))),2204 (!cast<Instruction>(Name#_.ZSuffix#"rmik")2205 _.KRCWM:$mask, _.RC:$src1, addr:$src2,2206 (X86pcmpm_imm_commute $cc))>;2207}2208 2209multiclass avx512_icmp_cc_rmb<bits<8> opc, string Suffix, PatFrag Frag,2210 PatFrag Frag_su, X86FoldableSchedWrite sched,2211 X86VectorVTInfo _, string Name> :2212 avx512_icmp_cc<opc, Suffix, Frag, Frag_su, sched, _, Name> {2213 let mayLoad = 1 in {2214 def rmbi : AVX512AIi8<opc, MRMSrcMem,2215 (outs _.KRC:$dst), (ins _.RC:$src1, _.ScalarMemOp:$src2,2216 u8imm:$cc),2217 !strconcat("vpcmp", Suffix,2218 "\t{$cc, ${src2}", _.BroadcastStr, ", $src1, $dst|",2219 "$dst, $src1, ${src2}", _.BroadcastStr, ", $cc}"),2220 [(set _.KRC:$dst, (_.KVT (Frag:$cc2221 (_.VT _.RC:$src1),2222 (_.BroadcastLdFrag addr:$src2),2223 cond)))]>,2224 EVEX, VVVV, EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;2225 def rmbik : AVX512AIi8<opc, MRMSrcMem,2226 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1,2227 _.ScalarMemOp:$src2, u8imm:$cc),2228 !strconcat("vpcmp", Suffix,2229 "\t{$cc, ${src2}", _.BroadcastStr, ", $src1, $dst {${mask}}|",2230 "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, ", $cc}"),2231 [(set _.KRC:$dst, (and _.KRCWM:$mask,2232 (_.KVT (Frag_su:$cc2233 (_.VT _.RC:$src1),2234 (_.BroadcastLdFrag addr:$src2),2235 cond))))]>,2236 EVEX, VVVV, EVEX_K, EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;2237 }2238 2239 def : Pat<(_.KVT (Frag:$cc (_.BroadcastLdFrag addr:$src2),2240 (_.VT _.RC:$src1), cond)),2241 (!cast<Instruction>(Name#_.ZSuffix#"rmbi")2242 _.RC:$src1, addr:$src2, (X86pcmpm_imm_commute $cc))>;2243 2244 def : Pat<(and _.KRCWM:$mask,2245 (_.KVT (Frag_su:$cc (_.BroadcastLdFrag addr:$src2),2246 (_.VT _.RC:$src1), cond))),2247 (!cast<Instruction>(Name#_.ZSuffix#"rmbik")2248 _.KRCWM:$mask, _.RC:$src1, addr:$src2,2249 (X86pcmpm_imm_commute $cc))>;2250}2251 2252multiclass avx512_icmp_cc_vl<bits<8> opc, string Suffix, PatFrag Frag,2253 PatFrag Frag_su, X86SchedWriteWidths sched,2254 AVX512VLVectorVTInfo VTInfo, Predicate prd> {2255 let Predicates = [prd] in2256 defm Z : avx512_icmp_cc<opc, Suffix, Frag, Frag_su,2257 sched.ZMM, VTInfo.info512, NAME>, EVEX_V512;2258 2259 let Predicates = [prd, HasVLX] in {2260 defm Z256 : avx512_icmp_cc<opc, Suffix, Frag, Frag_su,2261 sched.YMM, VTInfo.info256, NAME>, EVEX_V256;2262 defm Z128 : avx512_icmp_cc<opc, Suffix, Frag, Frag_su,2263 sched.XMM, VTInfo.info128, NAME>, EVEX_V128;2264 }2265}2266 2267multiclass avx512_icmp_cc_rmb_vl<bits<8> opc, string Suffix, PatFrag Frag,2268 PatFrag Frag_su, X86SchedWriteWidths sched,2269 AVX512VLVectorVTInfo VTInfo, Predicate prd> {2270 let Predicates = [prd] in2271 defm Z : avx512_icmp_cc_rmb<opc, Suffix, Frag, Frag_su,2272 sched.ZMM, VTInfo.info512, NAME>, EVEX_V512;2273 2274 let Predicates = [prd, HasVLX] in {2275 defm Z256 : avx512_icmp_cc_rmb<opc, Suffix, Frag, Frag_su,2276 sched.YMM, VTInfo.info256, NAME>, EVEX_V256;2277 defm Z128 : avx512_icmp_cc_rmb<opc, Suffix, Frag, Frag_su,2278 sched.XMM, VTInfo.info128, NAME>, EVEX_V128;2279 }2280}2281 2282// FIXME: Is there a better scheduler class for VPCMP/VPCMPU?2283defm VPCMPB : avx512_icmp_cc_vl<0x3F, "b", X86pcmpm, X86pcmpm_su,2284 SchedWriteVecALU, avx512vl_i8_info, HasBWI>,2285 EVEX_CD8<8, CD8VF>;2286defm VPCMPUB : avx512_icmp_cc_vl<0x3E, "ub", X86pcmpum, X86pcmpum_su,2287 SchedWriteVecALU, avx512vl_i8_info, HasBWI>,2288 EVEX_CD8<8, CD8VF>;2289 2290defm VPCMPW : avx512_icmp_cc_vl<0x3F, "w", X86pcmpm, X86pcmpm_su,2291 SchedWriteVecALU, avx512vl_i16_info, HasBWI>,2292 REX_W, EVEX_CD8<16, CD8VF>;2293defm VPCMPUW : avx512_icmp_cc_vl<0x3E, "uw", X86pcmpum, X86pcmpum_su,2294 SchedWriteVecALU, avx512vl_i16_info, HasBWI>,2295 REX_W, EVEX_CD8<16, CD8VF>;2296 2297defm VPCMPD : avx512_icmp_cc_rmb_vl<0x1F, "d", X86pcmpm, X86pcmpm_su,2298 SchedWriteVecALU, avx512vl_i32_info,2299 HasAVX512>, EVEX_CD8<32, CD8VF>;2300defm VPCMPUD : avx512_icmp_cc_rmb_vl<0x1E, "ud", X86pcmpum, X86pcmpum_su,2301 SchedWriteVecALU, avx512vl_i32_info,2302 HasAVX512>, EVEX_CD8<32, CD8VF>;2303 2304defm VPCMPQ : avx512_icmp_cc_rmb_vl<0x1F, "q", X86pcmpm, X86pcmpm_su,2305 SchedWriteVecALU, avx512vl_i64_info,2306 HasAVX512>, REX_W, EVEX_CD8<64, CD8VF>;2307defm VPCMPUQ : avx512_icmp_cc_rmb_vl<0x1E, "uq", X86pcmpum, X86pcmpum_su,2308 SchedWriteVecALU, avx512vl_i64_info,2309 HasAVX512>, REX_W, EVEX_CD8<64, CD8VF>;2310 2311multiclass avx512_vcmp_common<X86FoldableSchedWrite sched, X86VectorVTInfo _,2312 string Name> {2313let Uses = [MXCSR], mayRaiseFPException = 1 in {2314 defm rri : AVX512_maskable_cmp<0xC2, MRMSrcReg, _,2315 (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2,u8imm:$cc),2316 "vcmp"#_.Suffix,2317 "$cc, $src2, $src1", "$src1, $src2, $cc",2318 (X86any_cmpm (_.VT _.RC:$src1), (_.VT _.RC:$src2), timm:$cc),2319 (X86cmpm_su (_.VT _.RC:$src1), (_.VT _.RC:$src2), timm:$cc),2320 1>, Sched<[sched]>;2321 2322 let mayLoad = 1 in {2323 defm rmi : AVX512_maskable_cmp<0xC2, MRMSrcMem, _,2324 (outs _.KRC:$dst),(ins _.RC:$src1, _.MemOp:$src2, u8imm:$cc),2325 "vcmp"#_.Suffix,2326 "$cc, $src2, $src1", "$src1, $src2, $cc",2327 (X86any_cmpm (_.VT _.RC:$src1), (_.VT (_.LdFrag addr:$src2)),2328 timm:$cc),2329 (X86cmpm_su (_.VT _.RC:$src1), (_.VT (_.LdFrag addr:$src2)),2330 timm:$cc)>,2331 Sched<[sched.Folded, sched.ReadAfterFold]>;2332 2333 defm rmbi : AVX512_maskable_cmp<0xC2, MRMSrcMem, _,2334 (outs _.KRC:$dst),2335 (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$cc),2336 "vcmp"#_.Suffix,2337 "$cc, ${src2}"#_.BroadcastStr#", $src1",2338 "$src1, ${src2}"#_.BroadcastStr#", $cc",2339 (X86any_cmpm (_.VT _.RC:$src1),2340 (_.VT (_.BroadcastLdFrag addr:$src2)),2341 timm:$cc),2342 (X86cmpm_su (_.VT _.RC:$src1),2343 (_.VT (_.BroadcastLdFrag addr:$src2)),2344 timm:$cc)>,2345 EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;2346 }2347 }2348 2349 // Patterns for selecting with loads in other operand.2350 def : Pat<(X86any_cmpm (_.LdFrag addr:$src2), (_.VT _.RC:$src1),2351 timm:$cc),2352 (!cast<Instruction>(Name#_.ZSuffix#"rmi") _.RC:$src1, addr:$src2,2353 (X86cmpm_imm_commute timm:$cc))>;2354 2355 def : Pat<(and _.KRCWM:$mask, (X86cmpm_su (_.LdFrag addr:$src2),2356 (_.VT _.RC:$src1),2357 timm:$cc)),2358 (!cast<Instruction>(Name#_.ZSuffix#"rmik") _.KRCWM:$mask,2359 _.RC:$src1, addr:$src2,2360 (X86cmpm_imm_commute timm:$cc))>;2361 2362 def : Pat<(X86any_cmpm (_.BroadcastLdFrag addr:$src2),2363 (_.VT _.RC:$src1), timm:$cc),2364 (!cast<Instruction>(Name#_.ZSuffix#"rmbi") _.RC:$src1, addr:$src2,2365 (X86cmpm_imm_commute timm:$cc))>;2366 2367 def : Pat<(and _.KRCWM:$mask, (X86cmpm_su (_.BroadcastLdFrag addr:$src2),2368 (_.VT _.RC:$src1),2369 timm:$cc)),2370 (!cast<Instruction>(Name#_.ZSuffix#"rmbik") _.KRCWM:$mask,2371 _.RC:$src1, addr:$src2,2372 (X86cmpm_imm_commute timm:$cc))>;2373 2374 // Patterns for mask intrinsics.2375 def : Pat<(X86cmpmm (_.VT _.RC:$src1), (_.VT _.RC:$src2), timm:$cc,2376 (_.KVT immAllOnesV)),2377 (!cast<Instruction>(Name#_.ZSuffix#"rri") _.RC:$src1, _.RC:$src2, timm:$cc)>;2378 2379 def : Pat<(X86cmpmm (_.VT _.RC:$src1), (_.VT _.RC:$src2), timm:$cc, _.KRCWM:$mask),2380 (!cast<Instruction>(Name#_.ZSuffix#"rrik") _.KRCWM:$mask, _.RC:$src1,2381 _.RC:$src2, timm:$cc)>;2382 2383 def : Pat<(X86cmpmm (_.VT _.RC:$src1), (_.VT (_.LdFrag addr:$src2)), timm:$cc,2384 (_.KVT immAllOnesV)),2385 (!cast<Instruction>(Name#_.ZSuffix#"rmi") _.RC:$src1, addr:$src2, timm:$cc)>;2386 2387 def : Pat<(X86cmpmm (_.VT _.RC:$src1), (_.VT (_.LdFrag addr:$src2)), timm:$cc,2388 _.KRCWM:$mask),2389 (!cast<Instruction>(Name#_.ZSuffix#"rmik") _.KRCWM:$mask, _.RC:$src1,2390 addr:$src2, timm:$cc)>;2391 2392 def : Pat<(X86cmpmm (_.VT _.RC:$src1), (_.VT (_.BroadcastLdFrag addr:$src2)), timm:$cc,2393 (_.KVT immAllOnesV)),2394 (!cast<Instruction>(Name#_.ZSuffix#"rmbi") _.RC:$src1, addr:$src2, timm:$cc)>;2395 2396 def : Pat<(X86cmpmm (_.VT _.RC:$src1), (_.VT (_.BroadcastLdFrag addr:$src2)), timm:$cc,2397 _.KRCWM:$mask),2398 (!cast<Instruction>(Name#_.ZSuffix#"rmbik") _.KRCWM:$mask, _.RC:$src1,2399 addr:$src2, timm:$cc)>;2400 2401 // Patterns for mask intrinsics with loads in other operand.2402 def : Pat<(X86cmpmm (_.VT (_.LdFrag addr:$src2)), (_.VT _.RC:$src1), timm:$cc,2403 (_.KVT immAllOnesV)),2404 (!cast<Instruction>(Name#_.ZSuffix#"rmi") _.RC:$src1, addr:$src2,2405 (X86cmpm_imm_commute timm:$cc))>;2406 2407 def : Pat<(X86cmpmm (_.VT (_.LdFrag addr:$src2)), (_.VT _.RC:$src1), timm:$cc,2408 _.KRCWM:$mask),2409 (!cast<Instruction>(Name#_.ZSuffix#"rmik") _.KRCWM:$mask,2410 _.RC:$src1, addr:$src2,2411 (X86cmpm_imm_commute timm:$cc))>;2412 2413 def : Pat<(X86cmpmm (_.VT (_.BroadcastLdFrag addr:$src2)), (_.VT _.RC:$src1), timm:$cc,2414 (_.KVT immAllOnesV)),2415 (!cast<Instruction>(Name#_.ZSuffix#"rmbi") _.RC:$src1, addr:$src2,2416 (X86cmpm_imm_commute timm:$cc))>;2417 2418 def : Pat<(X86cmpmm (_.VT (_.BroadcastLdFrag addr:$src2)), (_.VT _.RC:$src1), timm:$cc,2419 _.KRCWM:$mask),2420 (!cast<Instruction>(Name#_.ZSuffix#"rmbik") _.KRCWM:$mask,2421 _.RC:$src1, addr:$src2,2422 (X86cmpm_imm_commute timm:$cc))>;2423}2424 2425multiclass avx512_vcmp_sae<X86FoldableSchedWrite sched, X86VectorVTInfo _> {2426 // comparison code form (VCMP[EQ/LT/LE/...]2427 let Uses = [MXCSR] in2428 defm rrib : AVX512_maskable_custom_cmp<0xC2, MRMSrcReg, (outs _.KRC:$dst),2429 (ins _.RC:$src1, _.RC:$src2, u8imm:$cc),2430 (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2, u8imm:$cc),2431 "vcmp"#_.Suffix,2432 "$cc, {sae}, $src2, $src1",2433 "$src1, $src2, {sae}, $cc",2434 [(set _.KRC:$dst, (X86cmpmmSAE (_.VT _.RC:$src1),2435 (_.VT _.RC:$src2), timm:$cc, (_.KVT immAllOnesV)))],2436 [(set _.KRC:$dst, (X86cmpmmSAE (_.VT _.RC:$src1),2437 (_.VT _.RC:$src2), timm:$cc, _.KRCWM:$mask))]>,2438 EVEX_B, Sched<[sched]>;2439}2440 2441multiclass avx512_vcmp<X86SchedWriteWidths sched, AVX512VLVectorVTInfo _,2442 Predicate Pred = HasAVX512> {2443 let Predicates = [Pred] in {2444 defm Z : avx512_vcmp_common<sched.ZMM, _.info512, NAME>,2445 avx512_vcmp_sae<sched.ZMM, _.info512>, EVEX_V512;2446 2447 }2448 let Predicates = [Pred,HasVLX] in {2449 defm Z128 : avx512_vcmp_common<sched.XMM, _.info128, NAME>, EVEX_V128;2450 defm Z256 : avx512_vcmp_common<sched.YMM, _.info256, NAME>, EVEX_V256;2451 }2452}2453 2454defm VCMPPD : avx512_vcmp<SchedWriteFCmp, avx512vl_f64_info>,2455 AVX512PDIi8Base, EVEX, VVVV, EVEX_CD8<64, CD8VF>, REX_W;2456defm VCMPPS : avx512_vcmp<SchedWriteFCmp, avx512vl_f32_info>,2457 AVX512PSIi8Base, EVEX, VVVV, EVEX_CD8<32, CD8VF>;2458defm VCMPPH : avx512_vcmp<SchedWriteFCmp, avx512vl_f16_info, HasFP16>,2459 AVX512PSIi8Base, EVEX, VVVV, EVEX_CD8<16, CD8VF>, TA;2460 2461// Patterns to select fp compares with load as first operand.2462let Predicates = [HasAVX512] in {2463 def : Pat<(v1i1 (X86cmpms (loadf64 addr:$src2), FR64X:$src1, timm:$cc)),2464 (VCMPSDZrmi FR64X:$src1, addr:$src2, (X86cmpm_imm_commute timm:$cc))>;2465 2466 def : Pat<(v1i1 (X86cmpms (loadf32 addr:$src2), FR32X:$src1, timm:$cc)),2467 (VCMPSSZrmi FR32X:$src1, addr:$src2, (X86cmpm_imm_commute timm:$cc))>;2468}2469 2470let Predicates = [HasFP16] in {2471 def : Pat<(v1i1 (X86cmpms (loadf16 addr:$src2), FR16X:$src1, timm:$cc)),2472 (VCMPSHZrmi FR16X:$src1, addr:$src2, (X86cmpm_imm_commute timm:$cc))>;2473}2474 2475// ----------------------------------------------------------------2476// FPClass2477 2478//handle fpclass instruction mask = op(reg_scalar,imm)2479// op(mem_scalar,imm)2480multiclass avx512_scalar_fpclass<bits<8> opc, string OpcodeStr,2481 X86FoldableSchedWrite sched, X86VectorVTInfo _,2482 Predicate prd> {2483 let Predicates = [prd], ExeDomain = _.ExeDomain, Uses = [MXCSR] in {2484 def ri : AVX512<opc, MRMSrcReg, (outs _.KRC:$dst),2485 (ins _.RC:$src1, i32u8imm:$src2),2486 OpcodeStr#_.Suffix#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",2487 [(set _.KRC:$dst,(X86Vfpclasss (_.VT _.RC:$src1),2488 (i32 timm:$src2)))]>,2489 Sched<[sched]>;2490 def rik : AVX512<opc, MRMSrcReg, (outs _.KRC:$dst),2491 (ins _.KRCWM:$mask, _.RC:$src1, i32u8imm:$src2),2492 OpcodeStr#_.Suffix#2493 "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",2494 [(set _.KRC:$dst,(and _.KRCWM:$mask,2495 (X86Vfpclasss_su (_.VT _.RC:$src1),2496 (i32 timm:$src2))))]>,2497 EVEX_K, Sched<[sched]>;2498 def mi : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),2499 (ins _.IntScalarMemOp:$src1, i32u8imm:$src2),2500 OpcodeStr#_.Suffix#2501 "\t{$src2, $src1, $dst|$dst, $src1, $src2}",2502 [(set _.KRC:$dst,2503 (X86Vfpclasss (_.ScalarIntMemFrags addr:$src1),2504 (i32 timm:$src2)))]>,2505 Sched<[sched.Folded, sched.ReadAfterFold]>;2506 def mik : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),2507 (ins _.KRCWM:$mask, _.IntScalarMemOp:$src1, i32u8imm:$src2),2508 OpcodeStr#_.Suffix#2509 "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",2510 [(set _.KRC:$dst,(and _.KRCWM:$mask,2511 (X86Vfpclasss_su (_.ScalarIntMemFrags addr:$src1),2512 (i32 timm:$src2))))]>,2513 EVEX_K, Sched<[sched.Folded, sched.ReadAfterFold]>;2514 }2515}2516 2517//handle fpclass instruction mask = fpclass(reg_vec, reg_vec, imm)2518// fpclass(reg_vec, mem_vec, imm)2519// fpclass(reg_vec, broadcast(eltVt), imm)2520multiclass avx512_vector_fpclass<bits<8> opc, string OpcodeStr,2521 X86FoldableSchedWrite sched, X86VectorVTInfo _,2522 string mem, list<Register> _Uses = [MXCSR]>{2523 let ExeDomain = _.ExeDomain, Uses = _Uses in {2524 def ri : AVX512<opc, MRMSrcReg, (outs _.KRC:$dst),2525 (ins _.RC:$src1, i32u8imm:$src2),2526 OpcodeStr#_.Suffix#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",2527 [(set _.KRC:$dst,(X86Vfpclass (_.VT _.RC:$src1),2528 (i32 timm:$src2)))]>,2529 Sched<[sched]>;2530 def rik : AVX512<opc, MRMSrcReg, (outs _.KRC:$dst),2531 (ins _.KRCWM:$mask, _.RC:$src1, i32u8imm:$src2),2532 OpcodeStr#_.Suffix#2533 "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",2534 [(set _.KRC:$dst,(and _.KRCWM:$mask,2535 (X86Vfpclass_su (_.VT _.RC:$src1),2536 (i32 timm:$src2))))]>,2537 EVEX_K, Sched<[sched]>;2538 def mi : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),2539 (ins _.MemOp:$src1, i32u8imm:$src2),2540 OpcodeStr#_.Suffix#"{"#mem#"}"#2541 "\t{$src2, $src1, $dst|$dst, $src1, $src2}",2542 [(set _.KRC:$dst,(X86Vfpclass2543 (_.VT (_.LdFrag addr:$src1)),2544 (i32 timm:$src2)))]>,2545 Sched<[sched.Folded, sched.ReadAfterFold]>;2546 def mik : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),2547 (ins _.KRCWM:$mask, _.MemOp:$src1, i32u8imm:$src2),2548 OpcodeStr#_.Suffix#"{"#mem#"}"#2549 "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",2550 [(set _.KRC:$dst, (and _.KRCWM:$mask, (X86Vfpclass_su2551 (_.VT (_.LdFrag addr:$src1)),2552 (i32 timm:$src2))))]>,2553 EVEX_K, Sched<[sched.Folded, sched.ReadAfterFold]>;2554 def mbi : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),2555 (ins _.ScalarMemOp:$src1, i32u8imm:$src2),2556 OpcodeStr#_.Suffix#"\t{$src2, ${src1}"#2557 _.BroadcastStr#", $dst|$dst, ${src1}"2558 #_.BroadcastStr#", $src2}",2559 [(set _.KRC:$dst,(X86Vfpclass2560 (_.VT (_.BroadcastLdFrag addr:$src1)),2561 (i32 timm:$src2)))]>,2562 EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;2563 def mbik : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),2564 (ins _.KRCWM:$mask, _.ScalarMemOp:$src1, i32u8imm:$src2),2565 OpcodeStr#_.Suffix#"\t{$src2, ${src1}"#2566 _.BroadcastStr#", $dst {${mask}}|$dst {${mask}}, ${src1}"#2567 _.BroadcastStr#", $src2}",2568 [(set _.KRC:$dst,(and _.KRCWM:$mask, (X86Vfpclass_su2569 (_.VT (_.BroadcastLdFrag addr:$src1)),2570 (i32 timm:$src2))))]>,2571 EVEX_B, EVEX_K, Sched<[sched.Folded, sched.ReadAfterFold]>;2572 }2573 2574 // Allow registers or broadcast with the x, y, z suffix we use to disambiguate2575 // the memory form.2576 def : InstAlias<OpcodeStr#_.Suffix#mem#2577 "\t{$src2, $src1, $dst|$dst, $src1, $src2}",2578 (!cast<Instruction>(NAME#"ri")2579 _.KRC:$dst, _.RC:$src1, i32u8imm:$src2), 0, "att">;2580 def : InstAlias<OpcodeStr#_.Suffix#mem#2581 "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",2582 (!cast<Instruction>(NAME#"rik")2583 _.KRC:$dst, _.KRCWM:$mask, _.RC:$src1, i32u8imm:$src2), 0, "att">;2584 def : InstAlias<OpcodeStr#_.Suffix#mem#2585 "\t{$src2, ${src1}"#_.BroadcastStr#", $dst|$dst, ${src1}"#2586 _.BroadcastStr#", $src2}",2587 (!cast<Instruction>(NAME#"mbi")2588 _.KRC:$dst, _.ScalarMemOp:$src1, i32u8imm:$src2), 0, "att">;2589 def : InstAlias<OpcodeStr#_.Suffix#mem#2590 "\t{$src2, ${src1}"#_.BroadcastStr#", $dst {${mask}}|"2591 "$dst {${mask}}, ${src1}"#_.BroadcastStr#", $src2}",2592 (!cast<Instruction>(NAME#"mbik")2593 _.KRC:$dst, _.KRCWM:$mask, _.ScalarMemOp:$src1, i32u8imm:$src2), 0, "att">;2594}2595 2596multiclass avx512_vector_fpclass_all<string OpcodeStr, AVX512VLVectorVTInfo _,2597 bits<8> opc, X86SchedWriteWidths sched,2598 Predicate prd>{2599 let Predicates = [prd] in {2600 defm Z : avx512_vector_fpclass<opc, OpcodeStr, sched.ZMM,2601 _.info512, "z">, EVEX_V512;2602 }2603 let Predicates = [prd, HasVLX] in {2604 defm Z128 : avx512_vector_fpclass<opc, OpcodeStr, sched.XMM,2605 _.info128, "x">, EVEX_V128;2606 defm Z256 : avx512_vector_fpclass<opc, OpcodeStr, sched.YMM,2607 _.info256, "y">, EVEX_V256;2608 }2609}2610 2611multiclass avx512_fp_fpclass_all<string OpcodeStr, bits<8> opcVec,2612 bits<8> opcScalar, X86SchedWriteWidths sched> {2613 defm PH : avx512_vector_fpclass_all<OpcodeStr, avx512vl_f16_info, opcVec,2614 sched, HasFP16>,2615 EVEX_CD8<16, CD8VF>, AVX512PSIi8Base, TA;2616 defm SHZ : avx512_scalar_fpclass<opcScalar, OpcodeStr,2617 sched.Scl, f16x_info, HasFP16>,2618 EVEX_CD8<16, CD8VT1>, AVX512PSIi8Base, TA;2619 defm PS : avx512_vector_fpclass_all<OpcodeStr, avx512vl_f32_info, opcVec,2620 sched, HasDQI>,2621 EVEX_CD8<32, CD8VF>, AVX512AIi8Base;2622 defm PD : avx512_vector_fpclass_all<OpcodeStr, avx512vl_f64_info, opcVec,2623 sched, HasDQI>,2624 EVEX_CD8<64, CD8VF>, AVX512AIi8Base, REX_W;2625 defm SSZ : avx512_scalar_fpclass<opcScalar, OpcodeStr,2626 sched.Scl, f32x_info, HasDQI>, VEX_LIG,2627 EVEX_CD8<32, CD8VT1>, AVX512AIi8Base;2628 defm SDZ : avx512_scalar_fpclass<opcScalar, OpcodeStr,2629 sched.Scl, f64x_info, HasDQI>, VEX_LIG,2630 EVEX_CD8<64, CD8VT1>, AVX512AIi8Base, REX_W;2631}2632 2633defm VFPCLASS : avx512_fp_fpclass_all<"vfpclass", 0x66, 0x67, SchedWriteFCmp>, EVEX;2634 2635//-----------------------------------------------------------------2636// Mask register copy, including2637// - copy between mask registers2638// - load/store mask registers2639// - copy from GPR to mask register and vice versa2640//2641multiclass avx512_mask_mov<bits<8> opc_kk, bits<8> opc_km, bits<8> opc_mk,2642 string OpcodeStr, RegisterClass KRC, ValueType vvt,2643 X86MemOperand x86memop, string Suffix = ""> {2644 let isMoveReg = 1, hasSideEffects = 0, SchedRW = [WriteMove],2645 explicitOpPrefix = !if(!eq(Suffix, ""), NoExplicitOpPrefix, ExplicitEVEX) in2646 def kk#Suffix : I<opc_kk, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src),2647 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>,2648 Sched<[WriteMove]>;2649 def km#Suffix : I<opc_km, MRMSrcMem, (outs KRC:$dst), (ins x86memop:$src),2650 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),2651 [(set KRC:$dst, (vvt (load addr:$src)))]>,2652 Sched<[WriteLoad]>, NoCD8;2653 def mk#Suffix : I<opc_mk, MRMDestMem, (outs), (ins x86memop:$dst, KRC:$src),2654 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),2655 [(store KRC:$src, addr:$dst)]>,2656 Sched<[WriteStore]>, NoCD8;2657}2658 2659multiclass avx512_mask_mov_gpr<bits<8> opc_kr, bits<8> opc_rk,2660 string OpcodeStr, RegisterClass KRC,2661 RegisterClass GRC, string Suffix = ""> {2662 let hasSideEffects = 0 in {2663 def kr#Suffix : I<opc_kr, MRMSrcReg, (outs KRC:$dst), (ins GRC:$src),2664 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>,2665 Sched<[WriteMove]>;2666 def rk#Suffix : I<opc_rk, MRMSrcReg, (outs GRC:$dst), (ins KRC:$src),2667 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>,2668 Sched<[WriteMove]>;2669 }2670}2671 2672let Predicates = [HasDQI, NoEGPR] in2673 defm KMOVB : avx512_mask_mov<0x90, 0x90, 0x91, "kmovb", VK8, v8i1, i8mem>,2674 avx512_mask_mov_gpr<0x92, 0x93, "kmovb", VK8, GR32>,2675 VEX, TB, PD;2676let Predicates = [HasDQI, HasEGPR, In64BitMode] in2677 defm KMOVB : avx512_mask_mov<0x90, 0x90, 0x91, "kmovb", VK8, v8i1, i8mem, "_EVEX">,2678 avx512_mask_mov_gpr<0x92, 0x93, "kmovb", VK8, GR32, "_EVEX">,2679 EVEX, TB, PD;2680 2681let Predicates = [HasAVX512, NoEGPR] in2682 defm KMOVW : avx512_mask_mov<0x90, 0x90, 0x91, "kmovw", VK16, v16i1, i16mem>,2683 avx512_mask_mov_gpr<0x92, 0x93, "kmovw", VK16, GR32>,2684 VEX, TB;2685let Predicates = [HasAVX512, HasEGPR, In64BitMode] in2686 defm KMOVW : avx512_mask_mov<0x90, 0x90, 0x91, "kmovw", VK16, v16i1, i16mem, "_EVEX">,2687 avx512_mask_mov_gpr<0x92, 0x93, "kmovw", VK16, GR32, "_EVEX">,2688 EVEX, TB;2689 2690let Predicates = [HasBWI, NoEGPR] in {2691 defm KMOVD : avx512_mask_mov<0x90, 0x90, 0x91, "kmovd", VK32, v32i1,i32mem>,2692 VEX, TB, PD, REX_W;2693 defm KMOVD : avx512_mask_mov_gpr<0x92, 0x93, "kmovd", VK32, GR32>,2694 VEX, TB, XD;2695 defm KMOVQ : avx512_mask_mov<0x90, 0x90, 0x91, "kmovq", VK64, v64i1, i64mem>,2696 VEX, TB, REX_W;2697 defm KMOVQ : avx512_mask_mov_gpr<0x92, 0x93, "kmovq", VK64, GR64>,2698 VEX, TB, XD, REX_W;2699}2700let Predicates = [HasBWI, HasEGPR, In64BitMode] in {2701 defm KMOVD : avx512_mask_mov<0x90, 0x90, 0x91, "kmovd", VK32, v32i1,i32mem, "_EVEX">,2702 EVEX, TB, PD, REX_W;2703 defm KMOVD : avx512_mask_mov_gpr<0x92, 0x93, "kmovd", VK32, GR32, "_EVEX">,2704 EVEX, TB, XD;2705 defm KMOVQ : avx512_mask_mov<0x90, 0x90, 0x91, "kmovq", VK64, v64i1, i64mem, "_EVEX">,2706 EVEX, TB, REX_W;2707 defm KMOVQ : avx512_mask_mov_gpr<0x92, 0x93, "kmovq", VK64, GR64, "_EVEX">,2708 EVEX, TB, XD, REX_W;2709}2710 2711// GR from/to mask register2712def : Pat<(v16i1 (bitconvert (i16 GR16:$src))),2713 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), GR16:$src, sub_16bit)), VK16)>;2714def : Pat<(i16 (bitconvert (v16i1 VK16:$src))),2715 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS VK16:$src, GR32)), sub_16bit)>;2716def : Pat<(i8 (trunc (i16 (bitconvert (v16i1 VK16:$src))))),2717 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS VK16:$src, GR32)), sub_8bit)>;2718 2719def : Pat<(v8i1 (bitconvert (i8 GR8:$src))),2720 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), GR8:$src, sub_8bit)), VK8)>;2721def : Pat<(i8 (bitconvert (v8i1 VK8:$src))),2722 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS VK8:$src, GR32)), sub_8bit)>;2723 2724def : Pat<(i32 (zext (i16 (bitconvert (v16i1 VK16:$src))))),2725 (KMOVWrk VK16:$src)>;2726def : Pat<(i64 (zext (i16 (bitconvert (v16i1 VK16:$src))))),2727 (SUBREG_TO_REG (i64 0), (KMOVWrk VK16:$src), sub_32bit)>;2728def : Pat<(i32 (anyext (i16 (bitconvert (v16i1 VK16:$src))))),2729 (COPY_TO_REGCLASS VK16:$src, GR32)>;2730def : Pat<(i64 (anyext (i16 (bitconvert (v16i1 VK16:$src))))),2731 (INSERT_SUBREG (IMPLICIT_DEF), (COPY_TO_REGCLASS VK16:$src, GR32), sub_32bit)>;2732 2733def : Pat<(i32 (zext (i8 (bitconvert (v8i1 VK8:$src))))),2734 (KMOVBrk VK8:$src)>, Requires<[HasDQI]>;2735def : Pat<(i64 (zext (i8 (bitconvert (v8i1 VK8:$src))))),2736 (SUBREG_TO_REG (i64 0), (KMOVBrk VK8:$src), sub_32bit)>, Requires<[HasDQI]>;2737def : Pat<(i32 (anyext (i8 (bitconvert (v8i1 VK8:$src))))),2738 (COPY_TO_REGCLASS VK8:$src, GR32)>;2739def : Pat<(i64 (anyext (i8 (bitconvert (v8i1 VK8:$src))))),2740 (INSERT_SUBREG (IMPLICIT_DEF), (COPY_TO_REGCLASS VK8:$src, GR32), sub_32bit)>;2741 2742def : Pat<(v32i1 (bitconvert (i32 GR32:$src))),2743 (COPY_TO_REGCLASS GR32:$src, VK32)>;2744def : Pat<(i32 (bitconvert (v32i1 VK32:$src))),2745 (COPY_TO_REGCLASS VK32:$src, GR32)>;2746def : Pat<(v64i1 (bitconvert (i64 GR64:$src))),2747 (COPY_TO_REGCLASS GR64:$src, VK64)>;2748def : Pat<(i64 (bitconvert (v64i1 VK64:$src))),2749 (COPY_TO_REGCLASS VK64:$src, GR64)>;2750 2751// Load/store kreg2752let Predicates = [HasDQI] in {2753 def : Pat<(v1i1 (load addr:$src)),2754 (COPY_TO_REGCLASS (KMOVBkm addr:$src), VK1)>;2755 def : Pat<(v2i1 (load addr:$src)),2756 (COPY_TO_REGCLASS (KMOVBkm addr:$src), VK2)>;2757 def : Pat<(v4i1 (load addr:$src)),2758 (COPY_TO_REGCLASS (KMOVBkm addr:$src), VK4)>;2759}2760 2761let Predicates = [HasAVX512] in {2762 def : Pat<(v8i1 (bitconvert (i8 (load addr:$src)))),2763 (COPY_TO_REGCLASS (MOVZX32rm8 addr:$src), VK8)>;2764 def : Pat<(v16i1 (bitconvert (loadi16 addr:$src))),2765 (KMOVWkm addr:$src)>;2766}2767 2768def X86kextract : SDNode<"ISD::EXTRACT_VECTOR_ELT",2769 SDTypeProfile<1, 2, [SDTCisVT<0, i8>,2770 SDTCVecEltisVT<1, i1>,2771 SDTCisPtrTy<2>]>>;2772 2773let Predicates = [HasAVX512] in {2774 multiclass operation_gpr_mask_copy_lowering<RegisterClass maskRC, ValueType maskVT> {2775 def : Pat<(maskVT (scalar_to_vector GR32:$src)),2776 (COPY_TO_REGCLASS GR32:$src, maskRC)>;2777 2778 def : Pat<(maskVT (scalar_to_vector GR8:$src)),2779 (COPY_TO_REGCLASS (INSERT_SUBREG (i32 (IMPLICIT_DEF)), GR8:$src, sub_8bit), maskRC)>;2780 2781 def : Pat<(i8 (X86kextract maskRC:$src, (iPTR 0))),2782 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS maskRC:$src, GR32)), sub_8bit)>;2783 2784 def : Pat<(i32 (anyext (i8 (X86kextract maskRC:$src, (iPTR 0))))),2785 (i32 (COPY_TO_REGCLASS maskRC:$src, GR32))>;2786 }2787 2788 defm : operation_gpr_mask_copy_lowering<VK1, v1i1>;2789 defm : operation_gpr_mask_copy_lowering<VK2, v2i1>;2790 defm : operation_gpr_mask_copy_lowering<VK4, v4i1>;2791 defm : operation_gpr_mask_copy_lowering<VK8, v8i1>;2792 defm : operation_gpr_mask_copy_lowering<VK16, v16i1>;2793 defm : operation_gpr_mask_copy_lowering<VK32, v32i1>;2794 defm : operation_gpr_mask_copy_lowering<VK64, v64i1>;2795 2796 def : Pat<(insert_subvector (v16i1 immAllZerosV),2797 (v1i1 (scalar_to_vector GR8:$src)), (iPTR 0)),2798 (KMOVWkr (AND32ri2799 (INSERT_SUBREG (i32 (IMPLICIT_DEF)), GR8:$src, sub_8bit),2800 (i32 1)))>;2801}2802 2803// Mask unary operation2804// - KNOT2805multiclass avx512_mask_unop<bits<8> opc, string OpcodeStr,2806 RegisterClass KRC, SDPatternOperator OpNode,2807 X86FoldableSchedWrite sched, Predicate prd> {2808 let Predicates = [prd] in2809 def kk : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src),2810 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),2811 [(set KRC:$dst, (OpNode KRC:$src))]>,2812 Sched<[sched]>;2813}2814 2815multiclass avx512_mask_unop_all<bits<8> opc, string OpcodeStr,2816 SDPatternOperator OpNode,2817 X86FoldableSchedWrite sched> {2818 defm B : avx512_mask_unop<opc, !strconcat(OpcodeStr, "b"), VK8, OpNode,2819 sched, HasDQI>, VEX, TB, PD;2820 defm W : avx512_mask_unop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode,2821 sched, HasAVX512>, VEX, TB;2822 defm D : avx512_mask_unop<opc, !strconcat(OpcodeStr, "d"), VK32, OpNode,2823 sched, HasBWI>, VEX, TB, PD, REX_W;2824 defm Q : avx512_mask_unop<opc, !strconcat(OpcodeStr, "q"), VK64, OpNode,2825 sched, HasBWI>, VEX, TB, REX_W;2826}2827 2828// TODO - do we need a X86SchedWriteWidths::KMASK type?2829defm KNOT : avx512_mask_unop_all<0x44, "knot", vnot, SchedWriteVecLogic.XMM>;2830 2831// KNL does not support KMOVB, 8-bit mask is promoted to 16-bit2832let Predicates = [HasAVX512, NoDQI] in2833def : Pat<(vnot VK8:$src),2834 (COPY_TO_REGCLASS (KNOTWkk (COPY_TO_REGCLASS VK8:$src, VK16)), VK8)>;2835 2836def : Pat<(vnot VK4:$src),2837 (COPY_TO_REGCLASS (KNOTWkk (COPY_TO_REGCLASS VK4:$src, VK16)), VK4)>;2838def : Pat<(vnot VK2:$src),2839 (COPY_TO_REGCLASS (KNOTWkk (COPY_TO_REGCLASS VK2:$src, VK16)), VK2)>;2840def : Pat<(vnot VK1:$src),2841 (COPY_TO_REGCLASS (KNOTWkk (COPY_TO_REGCLASS VK1:$src, VK16)), VK2)>;2842 2843// Mask binary operation2844// - KAND, KANDN, KOR, KXNOR, KXOR2845multiclass avx512_mask_binop<bits<8> opc, string OpcodeStr,2846 RegisterClass KRC, SDPatternOperator OpNode,2847 X86FoldableSchedWrite sched, Predicate prd,2848 bit IsCommutable> {2849 let Predicates = [prd], isCommutable = IsCommutable in2850 def kk : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src1, KRC:$src2),2851 !strconcat(OpcodeStr,2852 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),2853 [(set KRC:$dst, (OpNode KRC:$src1, KRC:$src2))]>,2854 Sched<[sched]>;2855}2856 2857multiclass avx512_mask_binop_all<bits<8> opc, string OpcodeStr,2858 SDPatternOperator OpNode,2859 X86FoldableSchedWrite sched, bit IsCommutable,2860 Predicate prdW = HasAVX512> {2861 defm B : avx512_mask_binop<opc, !strconcat(OpcodeStr, "b"), VK8, OpNode,2862 sched, HasDQI, IsCommutable>, VEX, VVVV, VEX_L, TB, PD;2863 defm W : avx512_mask_binop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode,2864 sched, prdW, IsCommutable>, VEX, VVVV, VEX_L, TB;2865 defm D : avx512_mask_binop<opc, !strconcat(OpcodeStr, "d"), VK32, OpNode,2866 sched, HasBWI, IsCommutable>, VEX, VVVV, VEX_L, REX_W, TB, PD;2867 defm Q : avx512_mask_binop<opc, !strconcat(OpcodeStr, "q"), VK64, OpNode,2868 sched, HasBWI, IsCommutable>, VEX, VVVV, VEX_L, REX_W, TB;2869}2870 2871// TODO - do we need a X86SchedWriteWidths::KMASK type?2872defm KAND : avx512_mask_binop_all<0x41, "kand", and, SchedWriteVecLogic.XMM, 1>;2873defm KOR : avx512_mask_binop_all<0x45, "kor", or, SchedWriteVecLogic.XMM, 1>;2874defm KXNOR : avx512_mask_binop_all<0x46, "kxnor", vxnor, SchedWriteVecLogic.XMM, 1>;2875defm KXOR : avx512_mask_binop_all<0x47, "kxor", xor, SchedWriteVecLogic.XMM, 1>;2876defm KANDN : avx512_mask_binop_all<0x42, "kandn", vandn, SchedWriteVecLogic.XMM, 0>;2877defm KADD : avx512_mask_binop_all<0x4A, "kadd", X86kadd, SchedWriteVecLogic.XMM, 1, HasDQI>;2878 2879multiclass avx512_binop_pat<SDPatternOperator VOpNode,2880 Instruction Inst> {2881 // With AVX512F, 8-bit mask is promoted to 16-bit mask,2882 // for the DQI set, this type is legal and KxxxB instruction is used2883 let Predicates = [NoDQI] in2884 def : Pat<(VOpNode VK8:$src1, VK8:$src2),2885 (COPY_TO_REGCLASS2886 (Inst (COPY_TO_REGCLASS VK8:$src1, VK16),2887 (COPY_TO_REGCLASS VK8:$src2, VK16)), VK8)>;2888 2889 // All types smaller than 8 bits require conversion anyway2890 def : Pat<(VOpNode VK1:$src1, VK1:$src2),2891 (COPY_TO_REGCLASS (Inst2892 (COPY_TO_REGCLASS VK1:$src1, VK16),2893 (COPY_TO_REGCLASS VK1:$src2, VK16)), VK1)>;2894 def : Pat<(VOpNode VK2:$src1, VK2:$src2),2895 (COPY_TO_REGCLASS (Inst2896 (COPY_TO_REGCLASS VK2:$src1, VK16),2897 (COPY_TO_REGCLASS VK2:$src2, VK16)), VK2)>;2898 def : Pat<(VOpNode VK4:$src1, VK4:$src2),2899 (COPY_TO_REGCLASS (Inst2900 (COPY_TO_REGCLASS VK4:$src1, VK16),2901 (COPY_TO_REGCLASS VK4:$src2, VK16)), VK4)>;2902}2903 2904defm : avx512_binop_pat<and, KANDWkk>;2905defm : avx512_binop_pat<vandn, KANDNWkk>;2906defm : avx512_binop_pat<or, KORWkk>;2907defm : avx512_binop_pat<vxnor, KXNORWkk>;2908defm : avx512_binop_pat<xor, KXORWkk>;2909 2910// Mask unpacking2911multiclass avx512_mask_unpck<string Suffix, X86KVectorVTInfo Dst,2912 X86KVectorVTInfo Src, X86FoldableSchedWrite sched,2913 Predicate prd> {2914 let Predicates = [prd] in {2915 let hasSideEffects = 0 in2916 def kk : I<0x4b, MRMSrcReg, (outs Dst.KRC:$dst),2917 (ins Src.KRC:$src1, Src.KRC:$src2),2918 "kunpck"#Suffix#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,2919 VEX, VVVV, VEX_L, Sched<[sched]>;2920 2921 def : Pat<(Dst.KVT (concat_vectors Src.KRC:$src1, Src.KRC:$src2)),2922 (!cast<Instruction>(NAME#kk) Src.KRC:$src2, Src.KRC:$src1)>;2923 }2924}2925 2926defm KUNPCKBW : avx512_mask_unpck<"bw", v16i1_info, v8i1_info, WriteShuffle, HasAVX512>, TB, PD;2927defm KUNPCKWD : avx512_mask_unpck<"wd", v32i1_info, v16i1_info, WriteShuffle, HasBWI>, TB;2928defm KUNPCKDQ : avx512_mask_unpck<"dq", v64i1_info, v32i1_info, WriteShuffle, HasBWI>, TB, REX_W;2929 2930// Mask bit testing2931multiclass avx512_mask_testop<bits<8> opc, string OpcodeStr, RegisterClass KRC,2932 SDNode OpNode, X86FoldableSchedWrite sched,2933 Predicate prd> {2934 let Predicates = [prd], Defs = [EFLAGS] in2935 def kk : I<opc, MRMSrcReg, (outs), (ins KRC:$src1, KRC:$src2),2936 !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"),2937 [(set EFLAGS, (OpNode KRC:$src1, KRC:$src2))]>,2938 Sched<[sched]>;2939}2940 2941multiclass avx512_mask_testop_w<bits<8> opc, string OpcodeStr, SDNode OpNode,2942 X86FoldableSchedWrite sched,2943 Predicate prdW = HasAVX512> {2944 defm B : avx512_mask_testop<opc, OpcodeStr#"b", VK8, OpNode, sched, HasDQI>,2945 VEX, TB, PD;2946 defm W : avx512_mask_testop<opc, OpcodeStr#"w", VK16, OpNode, sched, prdW>,2947 VEX, TB;2948 defm Q : avx512_mask_testop<opc, OpcodeStr#"q", VK64, OpNode, sched, HasBWI>,2949 VEX, TB, REX_W;2950 defm D : avx512_mask_testop<opc, OpcodeStr#"d", VK32, OpNode, sched, HasBWI>,2951 VEX, TB, PD, REX_W;2952}2953 2954// TODO - do we need a X86SchedWriteWidths::KMASK type?2955defm KORTEST : avx512_mask_testop_w<0x98, "kortest", X86kortest, SchedWriteVecLogic.XMM>;2956defm KTEST : avx512_mask_testop_w<0x99, "ktest", X86ktest, SchedWriteVecLogic.XMM, HasDQI>;2957 2958// Mask shift2959multiclass avx512_mask_shiftop<bits<8> opc, string OpcodeStr, RegisterClass KRC,2960 SDNode OpNode, X86FoldableSchedWrite sched> {2961 let Predicates = [HasAVX512] in2962 def ki : Ii8<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src, u8imm:$imm),2963 !strconcat(OpcodeStr,2964 "\t{$imm, $src, $dst|$dst, $src, $imm}"),2965 [(set KRC:$dst, (OpNode KRC:$src, (i8 timm:$imm)))]>,2966 Sched<[sched]>;2967}2968 2969multiclass avx512_mask_shiftop_w<bits<8> opc1, bits<8> opc2, string OpcodeStr,2970 SDNode OpNode, X86FoldableSchedWrite sched> {2971 defm W : avx512_mask_shiftop<opc1, !strconcat(OpcodeStr, "w"), VK16, OpNode,2972 sched>, VEX, TA, PD, REX_W;2973 let Predicates = [HasDQI] in2974 defm B : avx512_mask_shiftop<opc1, !strconcat(OpcodeStr, "b"), VK8, OpNode,2975 sched>, VEX, TA, PD;2976 let Predicates = [HasBWI] in {2977 defm Q : avx512_mask_shiftop<opc2, !strconcat(OpcodeStr, "q"), VK64, OpNode,2978 sched>, VEX, TA, PD, REX_W;2979 defm D : avx512_mask_shiftop<opc2, !strconcat(OpcodeStr, "d"), VK32, OpNode,2980 sched>, VEX, TA, PD;2981 }2982}2983 2984defm KSHIFTL : avx512_mask_shiftop_w<0x32, 0x33, "kshiftl", X86kshiftl, WriteShuffle>;2985defm KSHIFTR : avx512_mask_shiftop_w<0x30, 0x31, "kshiftr", X86kshiftr, WriteShuffle>;2986 2987// Patterns for comparing 128/256-bit integer vectors using 512-bit instruction.2988multiclass axv512_icmp_packed_cc_no_vlx_lowering<PatFrag Frag, PatFrag Frag_su,2989 string InstStr,2990 X86VectorVTInfo Narrow,2991 X86VectorVTInfo Wide> {2992def : Pat<(Narrow.KVT (Frag:$cc (Narrow.VT Narrow.RC:$src1),2993 (Narrow.VT Narrow.RC:$src2), cond)),2994 (COPY_TO_REGCLASS2995 (!cast<Instruction>(InstStr#"Zrri")2996 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),2997 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src2, Narrow.SubRegIdx)),2998 (X86pcmpm_imm $cc)), Narrow.KRC)>;2999 3000def : Pat<(Narrow.KVT (and Narrow.KRC:$mask,3001 (Narrow.KVT (Frag_su:$cc (Narrow.VT Narrow.RC:$src1),3002 (Narrow.VT Narrow.RC:$src2),3003 cond)))),3004 (COPY_TO_REGCLASS (!cast<Instruction>(InstStr#"Zrrik")3005 (COPY_TO_REGCLASS Narrow.KRC:$mask, Wide.KRC),3006 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),3007 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src2, Narrow.SubRegIdx)),3008 (X86pcmpm_imm $cc)), Narrow.KRC)>;3009}3010 3011multiclass axv512_icmp_packed_cc_rmb_no_vlx_lowering<PatFrag Frag, PatFrag Frag_su,3012 string InstStr,3013 X86VectorVTInfo Narrow,3014 X86VectorVTInfo Wide> {3015// Broadcast load.3016def : Pat<(Narrow.KVT (Frag:$cc (Narrow.VT Narrow.RC:$src1),3017 (Narrow.BroadcastLdFrag addr:$src2), cond)),3018 (COPY_TO_REGCLASS3019 (!cast<Instruction>(InstStr#"Zrmbi")3020 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),3021 addr:$src2, (X86pcmpm_imm $cc)), Narrow.KRC)>;3022 3023def : Pat<(Narrow.KVT (and Narrow.KRC:$mask,3024 (Narrow.KVT3025 (Frag_su:$cc (Narrow.VT Narrow.RC:$src1),3026 (Narrow.BroadcastLdFrag addr:$src2),3027 cond)))),3028 (COPY_TO_REGCLASS (!cast<Instruction>(InstStr#"Zrmbik")3029 (COPY_TO_REGCLASS Narrow.KRC:$mask, Wide.KRC),3030 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),3031 addr:$src2, (X86pcmpm_imm $cc)), Narrow.KRC)>;3032 3033// Commuted with broadcast load.3034def : Pat<(Narrow.KVT (Frag:$cc (Narrow.BroadcastLdFrag addr:$src2),3035 (Narrow.VT Narrow.RC:$src1),3036 cond)),3037 (COPY_TO_REGCLASS3038 (!cast<Instruction>(InstStr#"Zrmbi")3039 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),3040 addr:$src2, (X86pcmpm_imm_commute $cc)), Narrow.KRC)>;3041 3042def : Pat<(Narrow.KVT (and Narrow.KRC:$mask,3043 (Narrow.KVT3044 (Frag_su:$cc (Narrow.BroadcastLdFrag addr:$src2),3045 (Narrow.VT Narrow.RC:$src1),3046 cond)))),3047 (COPY_TO_REGCLASS (!cast<Instruction>(InstStr#"Zrmbik")3048 (COPY_TO_REGCLASS Narrow.KRC:$mask, Wide.KRC),3049 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),3050 addr:$src2, (X86pcmpm_imm_commute $cc)), Narrow.KRC)>;3051}3052 3053// Same as above, but for fp types which don't use PatFrags.3054multiclass axv512_cmp_packed_cc_no_vlx_lowering<string InstStr,3055 X86VectorVTInfo Narrow,3056 X86VectorVTInfo Wide> {3057def : Pat<(Narrow.KVT (X86cmpm (Narrow.VT Narrow.RC:$src1),3058 (Narrow.VT Narrow.RC:$src2), timm:$cc)),3059 (COPY_TO_REGCLASS3060 (!cast<Instruction>(InstStr#"Zrri")3061 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),3062 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src2, Narrow.SubRegIdx)),3063 timm:$cc), Narrow.KRC)>;3064 3065def : Pat<(Narrow.KVT (and Narrow.KRC:$mask,3066 (X86cmpm_su (Narrow.VT Narrow.RC:$src1),3067 (Narrow.VT Narrow.RC:$src2), timm:$cc))),3068 (COPY_TO_REGCLASS (!cast<Instruction>(InstStr#"Zrrik")3069 (COPY_TO_REGCLASS Narrow.KRC:$mask, Wide.KRC),3070 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),3071 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src2, Narrow.SubRegIdx)),3072 timm:$cc), Narrow.KRC)>;3073 3074// Broadcast load.3075def : Pat<(Narrow.KVT (X86cmpm (Narrow.VT Narrow.RC:$src1),3076 (Narrow.VT (Narrow.BroadcastLdFrag addr:$src2)), timm:$cc)),3077 (COPY_TO_REGCLASS3078 (!cast<Instruction>(InstStr#"Zrmbi")3079 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),3080 addr:$src2, timm:$cc), Narrow.KRC)>;3081 3082def : Pat<(Narrow.KVT (and Narrow.KRC:$mask,3083 (X86cmpm_su (Narrow.VT Narrow.RC:$src1),3084 (Narrow.VT (Narrow.BroadcastLdFrag addr:$src2)), timm:$cc))),3085 (COPY_TO_REGCLASS (!cast<Instruction>(InstStr#"Zrmbik")3086 (COPY_TO_REGCLASS Narrow.KRC:$mask, Wide.KRC),3087 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),3088 addr:$src2, timm:$cc), Narrow.KRC)>;3089 3090// Commuted with broadcast load.3091def : Pat<(Narrow.KVT (X86cmpm (Narrow.VT (Narrow.BroadcastLdFrag addr:$src2)),3092 (Narrow.VT Narrow.RC:$src1), timm:$cc)),3093 (COPY_TO_REGCLASS3094 (!cast<Instruction>(InstStr#"Zrmbi")3095 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),3096 addr:$src2, (X86cmpm_imm_commute timm:$cc)), Narrow.KRC)>;3097 3098def : Pat<(Narrow.KVT (and Narrow.KRC:$mask,3099 (X86cmpm_su (Narrow.VT (Narrow.BroadcastLdFrag addr:$src2)),3100 (Narrow.VT Narrow.RC:$src1), timm:$cc))),3101 (COPY_TO_REGCLASS (!cast<Instruction>(InstStr#"Zrmbik")3102 (COPY_TO_REGCLASS Narrow.KRC:$mask, Wide.KRC),3103 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),3104 addr:$src2, (X86cmpm_imm_commute timm:$cc)), Narrow.KRC)>;3105}3106 3107let Predicates = [HasAVX512, NoVLX] in {3108 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpm, X86pcmpm_su, "VPCMPD", v8i32x_info, v16i32_info>;3109 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpum, X86pcmpum_su, "VPCMPUD", v8i32x_info, v16i32_info>;3110 3111 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpm, X86pcmpm_su, "VPCMPD", v4i32x_info, v16i32_info>;3112 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpum, X86pcmpum_su, "VPCMPUD", v4i32x_info, v16i32_info>;3113 3114 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpm, X86pcmpm_su, "VPCMPQ", v4i64x_info, v8i64_info>;3115 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpum, X86pcmpum_su, "VPCMPUQ", v4i64x_info, v8i64_info>;3116 3117 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpm, X86pcmpm_su, "VPCMPQ", v2i64x_info, v8i64_info>;3118 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpum, X86pcmpum_su, "VPCMPUQ", v2i64x_info, v8i64_info>;3119 3120 defm : axv512_icmp_packed_cc_rmb_no_vlx_lowering<X86pcmpm, X86pcmpm_su, "VPCMPD", v8i32x_info, v16i32_info>;3121 defm : axv512_icmp_packed_cc_rmb_no_vlx_lowering<X86pcmpum, X86pcmpum_su, "VPCMPUD", v8i32x_info, v16i32_info>;3122 3123 defm : axv512_icmp_packed_cc_rmb_no_vlx_lowering<X86pcmpm, X86pcmpm_su, "VPCMPD", v4i32x_info, v16i32_info>;3124 defm : axv512_icmp_packed_cc_rmb_no_vlx_lowering<X86pcmpum, X86pcmpum_su, "VPCMPUD", v4i32x_info, v16i32_info>;3125 3126 defm : axv512_icmp_packed_cc_rmb_no_vlx_lowering<X86pcmpm, X86pcmpm_su, "VPCMPQ", v4i64x_info, v8i64_info>;3127 defm : axv512_icmp_packed_cc_rmb_no_vlx_lowering<X86pcmpum, X86pcmpum_su, "VPCMPUQ", v4i64x_info, v8i64_info>;3128 3129 defm : axv512_icmp_packed_cc_rmb_no_vlx_lowering<X86pcmpm, X86pcmpm_su, "VPCMPQ", v2i64x_info, v8i64_info>;3130 defm : axv512_icmp_packed_cc_rmb_no_vlx_lowering<X86pcmpum, X86pcmpum_su, "VPCMPUQ", v2i64x_info, v8i64_info>;3131 3132 defm : axv512_cmp_packed_cc_no_vlx_lowering<"VCMPPS", v8f32x_info, v16f32_info>;3133 defm : axv512_cmp_packed_cc_no_vlx_lowering<"VCMPPS", v4f32x_info, v16f32_info>;3134 defm : axv512_cmp_packed_cc_no_vlx_lowering<"VCMPPD", v4f64x_info, v8f64_info>;3135 defm : axv512_cmp_packed_cc_no_vlx_lowering<"VCMPPD", v2f64x_info, v8f64_info>;3136}3137 3138let Predicates = [HasBWI, NoVLX] in {3139 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpm, X86pcmpm_su, "VPCMPB", v32i8x_info, v64i8_info>;3140 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpum, X86pcmpum_su, "VPCMPUB", v32i8x_info, v64i8_info>;3141 3142 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpm, X86pcmpm_su, "VPCMPB", v16i8x_info, v64i8_info>;3143 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpum, X86pcmpum_su, "VPCMPUB", v16i8x_info, v64i8_info>;3144 3145 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpm, X86pcmpm_su, "VPCMPW", v16i16x_info, v32i16_info>;3146 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpum, X86pcmpum_su, "VPCMPUW", v16i16x_info, v32i16_info>;3147 3148 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpm, X86pcmpm_su, "VPCMPW", v8i16x_info, v32i16_info>;3149 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpum, X86pcmpum_su, "VPCMPUW", v8i16x_info, v32i16_info>;3150}3151 3152// Mask setting all 0s or 1s3153multiclass avx512_mask_setop<RegisterClass KRC, ValueType VT, SDPatternOperator Val> {3154 let Predicates = [HasAVX512] in3155 let isReMaterializable = 1, isAsCheapAsAMove = 1, isPseudo = 1,3156 SchedRW = [WriteZero] in3157 def NAME# : I<0, Pseudo, (outs KRC:$dst), (ins), "",3158 [(set KRC:$dst, (VT Val))]>;3159}3160 3161multiclass avx512_mask_setop_w<SDPatternOperator Val> {3162 defm W : avx512_mask_setop<VK16, v16i1, Val>;3163 defm D : avx512_mask_setop<VK32, v32i1, Val>;3164 defm Q : avx512_mask_setop<VK64, v64i1, Val>;3165}3166 3167defm KSET0 : avx512_mask_setop_w<immAllZerosV>;3168defm KSET1 : avx512_mask_setop_w<immAllOnesV>;3169 3170// 8-bit mask set operations for AVX512DQ3171let Predicates = [HasDQI] in {3172 defm KSET0B : avx512_mask_setop<VK8, v8i1, immAllZerosV>;3173 defm KSET1B : avx512_mask_setop<VK8, v8i1, immAllOnesV>;3174}3175 3176// With AVX-512 only, 8-bit mask is promoted to 16-bit mask.3177let Predicates = [HasAVX512] in {3178 def : Pat<(v8i1 immAllZerosV), (COPY_TO_REGCLASS (KSET0W), VK8)>;3179 def : Pat<(v4i1 immAllZerosV), (COPY_TO_REGCLASS (KSET0W), VK4)>;3180 def : Pat<(v2i1 immAllZerosV), (COPY_TO_REGCLASS (KSET0W), VK2)>;3181 def : Pat<(v1i1 immAllZerosV), (COPY_TO_REGCLASS (KSET0W), VK1)>;3182 def : Pat<(v8i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK8)>;3183 def : Pat<(v4i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK4)>;3184 def : Pat<(v2i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK2)>;3185 def : Pat<(v1i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK1)>;3186}3187 3188// With AVX512DQ, use 8-bit operations for 8-bit masks to avoid setting upper3189// bits3190let Predicates = [HasDQI] in {3191 def : Pat<(v8i1 immAllZerosV), (KSET0B)>;3192 def : Pat<(v8i1 immAllOnesV), (KSET1B)>;3193}3194 3195// Optimize bitconvert of all-ones constants to use kxnor instructions3196let Predicates = [HasDQI] in {3197 def : Pat<(v8i1(bitconvert(i8 255))), (KSET1B)>;3198 def : Pat<(v16i1(bitconvert(i16 255))), (COPY_TO_REGCLASS(KSET1B), VK16)>;3199}3200let Predicates = [HasBWI] in {3201 def : Pat<(v32i1(bitconvert(i32 -1))), (KSET1D)>;3202 def : Pat<(v64i1(bitconvert(i64 -1))), (KSET1Q)>;3203}3204// Submask patterns: lower N bits set in larger mask registers3205let Predicates = [HasBWI, HasDQI] in {3206 // v32i1 submasks3207 def : Pat<(v32i1(bitconvert(i32 255))), (COPY_TO_REGCLASS(KSET1B), VK32)>;3208 def : Pat<(v32i1(bitconvert(i32 65535))), (COPY_TO_REGCLASS(KSET1W), VK32)>;3209 // v64i1 submasks3210 def : Pat<(v64i1(bitconvert(i64 255))), (COPY_TO_REGCLASS(KSET1B), VK64)>;3211 def : Pat<(v64i1(bitconvert(i64 65535))), (COPY_TO_REGCLASS(KSET1W), VK64)>;3212 def : Pat<(v64i1(bitconvert(i64 4294967295))), (COPY_TO_REGCLASS(KSET1D),3213 VK64)>;3214}3215 3216// Patterns for kmask insert_subvector/extract_subvector to/from index=03217multiclass operation_subvector_mask_lowering<RegisterClass subRC, ValueType subVT,3218 RegisterClass RC, ValueType VT> {3219 def : Pat<(subVT (extract_subvector (VT RC:$src), (iPTR 0))),3220 (subVT (COPY_TO_REGCLASS RC:$src, subRC))>;3221 3222 def : Pat<(VT (insert_subvector undef, subRC:$src, (iPTR 0))),3223 (VT (COPY_TO_REGCLASS subRC:$src, RC))>;3224}3225defm : operation_subvector_mask_lowering<VK1, v1i1, VK2, v2i1>;3226defm : operation_subvector_mask_lowering<VK1, v1i1, VK4, v4i1>;3227defm : operation_subvector_mask_lowering<VK1, v1i1, VK8, v8i1>;3228defm : operation_subvector_mask_lowering<VK1, v1i1, VK16, v16i1>;3229defm : operation_subvector_mask_lowering<VK1, v1i1, VK32, v32i1>;3230defm : operation_subvector_mask_lowering<VK1, v1i1, VK64, v64i1>;3231 3232defm : operation_subvector_mask_lowering<VK2, v2i1, VK4, v4i1>;3233defm : operation_subvector_mask_lowering<VK2, v2i1, VK8, v8i1>;3234defm : operation_subvector_mask_lowering<VK2, v2i1, VK16, v16i1>;3235defm : operation_subvector_mask_lowering<VK2, v2i1, VK32, v32i1>;3236defm : operation_subvector_mask_lowering<VK2, v2i1, VK64, v64i1>;3237 3238defm : operation_subvector_mask_lowering<VK4, v4i1, VK8, v8i1>;3239defm : operation_subvector_mask_lowering<VK4, v4i1, VK16, v16i1>;3240defm : operation_subvector_mask_lowering<VK4, v4i1, VK32, v32i1>;3241defm : operation_subvector_mask_lowering<VK4, v4i1, VK64, v64i1>;3242 3243defm : operation_subvector_mask_lowering<VK8, v8i1, VK16, v16i1>;3244defm : operation_subvector_mask_lowering<VK8, v8i1, VK32, v32i1>;3245defm : operation_subvector_mask_lowering<VK8, v8i1, VK64, v64i1>;3246 3247defm : operation_subvector_mask_lowering<VK16, v16i1, VK32, v32i1>;3248defm : operation_subvector_mask_lowering<VK16, v16i1, VK64, v64i1>;3249 3250defm : operation_subvector_mask_lowering<VK32, v32i1, VK64, v64i1>;3251 3252//===----------------------------------------------------------------------===//3253// AVX-512 - Aligned and unaligned load and store3254//3255 3256multiclass avx512_load<bits<8> opc, string OpcodeStr, string Name,3257 X86VectorVTInfo _, PatFrag ld_frag, PatFrag mload,3258 X86SchedWriteMoveLS Sched, bit NoRMPattern = 0,3259 SDPatternOperator SelectOprr = vselect> {3260 let hasSideEffects = 0 in {3261 let isMoveReg = 1 in3262 def rr : AVX512PI<opc, MRMSrcReg, (outs _.RC:$dst), (ins _.RC:$src),3263 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), [],3264 _.ExeDomain>, EVEX, Sched<[Sched.RR]>;3265 def rrkz : AVX512PI<opc, MRMSrcReg, (outs _.RC:$dst),3266 (ins _.KRCWM:$mask, _.RC:$src),3267 !strconcat(OpcodeStr, "\t{$src, ${dst} {${mask}} {z}|",3268 "${dst} {${mask}} {z}, $src}"),3269 [(set _.RC:$dst, (_.VT (SelectOprr _.KRCWM:$mask,3270 (_.VT _.RC:$src),3271 _.ImmAllZerosV)))], _.ExeDomain>,3272 EVEX, EVEX_KZ, Sched<[Sched.RR]>;3273 3274 let mayLoad = 1, canFoldAsLoad = 1, isReMaterializable = 1 in3275 def rm : AVX512PI<opc, MRMSrcMem, (outs _.RC:$dst), (ins _.MemOp:$src),3276 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),3277 !if(NoRMPattern, [],3278 [(set _.RC:$dst,3279 (_.VT (ld_frag addr:$src)))]),3280 _.ExeDomain>, EVEX, Sched<[Sched.RM]>;3281 3282 let Constraints = "$src0 = $dst", isConvertibleToThreeAddress = 1 in {3283 def rrk : AVX512PI<opc, MRMSrcReg, (outs _.RC:$dst),3284 (ins _.RC:$src0, _.KRCWM:$mask, _.RC:$src1),3285 !strconcat(OpcodeStr, "\t{$src1, ${dst} {${mask}}|",3286 "${dst} {${mask}}, $src1}"),3287 [(set _.RC:$dst, (_.VT (SelectOprr _.KRCWM:$mask,3288 (_.VT _.RC:$src1),3289 (_.VT _.RC:$src0))))], _.ExeDomain>,3290 EVEX, EVEX_K, Sched<[Sched.RR]>;3291 let mayLoad = 1, canFoldAsLoad = 1 in3292 def rmk : AVX512PI<opc, MRMSrcMem, (outs _.RC:$dst),3293 (ins _.RC:$src0, _.KRCWM:$mask, _.MemOp:$src1),3294 !strconcat(OpcodeStr, "\t{$src1, ${dst} {${mask}}|",3295 "${dst} {${mask}}, $src1}"),3296 [(set _.RC:$dst, (_.VT3297 (vselect_mask _.KRCWM:$mask,3298 (_.VT (ld_frag addr:$src1)),3299 (_.VT _.RC:$src0))))], _.ExeDomain>,3300 EVEX, EVEX_K, Sched<[Sched.RM]>;3301 }3302 let mayLoad = 1, canFoldAsLoad = 1 in3303 def rmkz : AVX512PI<opc, MRMSrcMem, (outs _.RC:$dst),3304 (ins _.KRCWM:$mask, _.MemOp:$src),3305 OpcodeStr #"\t{$src, ${dst} {${mask}} {z}|"#3306 "${dst} {${mask}} {z}, $src}",3307 [(set _.RC:$dst, (_.VT (vselect_mask _.KRCWM:$mask,3308 (_.VT (ld_frag addr:$src)), _.ImmAllZerosV)))],3309 _.ExeDomain>, EVEX, EVEX_KZ, Sched<[Sched.RM]>;3310 }3311 def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, undef)),3312 (!cast<Instruction>(Name#_.ZSuffix#rmkz) _.KRCWM:$mask, addr:$ptr)>;3313 3314 def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, _.ImmAllZerosV)),3315 (!cast<Instruction>(Name#_.ZSuffix#rmkz) _.KRCWM:$mask, addr:$ptr)>;3316 3317 def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, (_.VT _.RC:$src0))),3318 (!cast<Instruction>(Name#_.ZSuffix#rmk) _.RC:$src0,3319 _.KRCWM:$mask, addr:$ptr)>;3320}3321 3322multiclass avx512_alignedload_vl<bits<8> opc, string OpcodeStr,3323 AVX512VLVectorVTInfo _, Predicate prd,3324 X86SchedWriteMoveLSWidths Sched,3325 bit NoRMPattern = 0> {3326 let Predicates = [prd] in3327 defm Z : avx512_load<opc, OpcodeStr, NAME, _.info512,3328 _.info512.AlignedLdFrag, masked_load_aligned,3329 Sched.ZMM, NoRMPattern>, EVEX_V512;3330 3331 let Predicates = [prd, HasVLX] in {3332 defm Z256 : avx512_load<opc, OpcodeStr, NAME, _.info256,3333 _.info256.AlignedLdFrag, masked_load_aligned,3334 Sched.YMM, NoRMPattern>, EVEX_V256;3335 defm Z128 : avx512_load<opc, OpcodeStr, NAME, _.info128,3336 _.info128.AlignedLdFrag, masked_load_aligned,3337 Sched.XMM, NoRMPattern>, EVEX_V128;3338 }3339}3340 3341multiclass avx512_load_vl<bits<8> opc, string OpcodeStr,3342 AVX512VLVectorVTInfo _, Predicate prd,3343 X86SchedWriteMoveLSWidths Sched,3344 bit NoRMPattern = 0,3345 SDPatternOperator SelectOprr = vselect> {3346 let Predicates = [prd] in3347 defm Z : avx512_load<opc, OpcodeStr, NAME, _.info512, _.info512.LdFrag,3348 masked_load, Sched.ZMM, NoRMPattern, SelectOprr>, EVEX_V512;3349 3350 let Predicates = [prd, HasVLX] in {3351 defm Z256 : avx512_load<opc, OpcodeStr, NAME, _.info256, _.info256.LdFrag,3352 masked_load, Sched.YMM, NoRMPattern, SelectOprr>, EVEX_V256;3353 defm Z128 : avx512_load<opc, OpcodeStr, NAME, _.info128, _.info128.LdFrag,3354 masked_load, Sched.XMM, NoRMPattern, SelectOprr>, EVEX_V128;3355 }3356}3357 3358multiclass avx512_store<bits<8> opc, string OpcodeStr, string BaseName,3359 X86VectorVTInfo _, PatFrag st_frag, PatFrag mstore,3360 X86SchedWriteMoveLS Sched, bit NoMRPattern = 0> {3361 let hasSideEffects = 0, isCodeGenOnly = 1, ForceDisassemble = 1 in {3362 let isMoveReg = 1 in3363 def rr_REV : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst), (ins _.RC:$src),3364 OpcodeStr # "\t{$src, $dst|$dst, $src}",3365 [], _.ExeDomain>, EVEX,3366 Sched<[Sched.RR]>;3367 def rrk_REV : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst),3368 (ins _.KRCWM:$mask, _.RC:$src),3369 OpcodeStr # "\t{$src, ${dst} {${mask}}|"#3370 "${dst} {${mask}}, $src}",3371 [], _.ExeDomain>, EVEX, EVEX_K,3372 Sched<[Sched.RR]>;3373 def rrkz_REV : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst),3374 (ins _.KRCWM:$mask, _.RC:$src),3375 OpcodeStr # "\t{$src, ${dst} {${mask}} {z}|" #3376 "${dst} {${mask}} {z}, $src}",3377 [], _.ExeDomain>, EVEX, EVEX_KZ,3378 Sched<[Sched.RR]>;3379 }3380 3381 let hasSideEffects = 0, mayStore = 1 in3382 def mr : AVX512PI<opc, MRMDestMem, (outs), (ins _.MemOp:$dst, _.RC:$src),3383 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),3384 !if(NoMRPattern, [],3385 [(st_frag (_.VT _.RC:$src), addr:$dst)]),3386 _.ExeDomain>, EVEX, Sched<[Sched.MR]>;3387 def mrk : AVX512PI<opc, MRMDestMem, (outs),3388 (ins _.MemOp:$dst, _.KRCWM:$mask, _.RC:$src),3389 OpcodeStr # "\t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}",3390 [], _.ExeDomain>, EVEX, EVEX_K, Sched<[Sched.MR]>;3391 3392 def: Pat<(mstore (_.VT _.RC:$src), addr:$ptr, _.KRCWM:$mask),3393 (!cast<Instruction>(BaseName#_.ZSuffix#mrk) addr:$ptr,3394 _.KRCWM:$mask, _.RC:$src)>;3395 3396 def : InstAlias<OpcodeStr#".s\t{$src, $dst|$dst, $src}",3397 (!cast<Instruction>(BaseName#_.ZSuffix#"rr_REV")3398 _.RC:$dst, _.RC:$src), 0>;3399 def : InstAlias<OpcodeStr#".s\t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}",3400 (!cast<Instruction>(BaseName#_.ZSuffix#"rrk_REV")3401 _.RC:$dst, _.KRCWM:$mask, _.RC:$src), 0>;3402 def : InstAlias<OpcodeStr#".s\t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}",3403 (!cast<Instruction>(BaseName#_.ZSuffix#"rrkz_REV")3404 _.RC:$dst, _.KRCWM:$mask, _.RC:$src), 0>;3405}3406 3407multiclass avx512_store_vl< bits<8> opc, string OpcodeStr,3408 AVX512VLVectorVTInfo _, Predicate prd,3409 X86SchedWriteMoveLSWidths Sched,3410 bit NoMRPattern = 0> {3411 let Predicates = [prd] in3412 defm Z : avx512_store<opc, OpcodeStr, NAME, _.info512, store,3413 masked_store, Sched.ZMM, NoMRPattern>, EVEX_V512;3414 let Predicates = [prd, HasVLX] in {3415 defm Z256 : avx512_store<opc, OpcodeStr, NAME, _.info256, store,3416 masked_store, Sched.YMM, NoMRPattern>, EVEX_V256;3417 defm Z128 : avx512_store<opc, OpcodeStr, NAME, _.info128, store,3418 masked_store, Sched.XMM, NoMRPattern>, EVEX_V128;3419 }3420}3421 3422multiclass avx512_alignedstore_vl<bits<8> opc, string OpcodeStr,3423 AVX512VLVectorVTInfo _, Predicate prd,3424 X86SchedWriteMoveLSWidths Sched,3425 bit NoMRPattern = 0> {3426 let Predicates = [prd] in3427 defm Z : avx512_store<opc, OpcodeStr, NAME, _.info512, alignedstore,3428 masked_store_aligned, Sched.ZMM, NoMRPattern>, EVEX_V512;3429 3430 let Predicates = [prd, HasVLX] in {3431 defm Z256 : avx512_store<opc, OpcodeStr, NAME, _.info256, alignedstore,3432 masked_store_aligned, Sched.YMM, NoMRPattern>, EVEX_V256;3433 defm Z128 : avx512_store<opc, OpcodeStr, NAME, _.info128, alignedstore,3434 masked_store_aligned, Sched.XMM, NoMRPattern>, EVEX_V128;3435 }3436}3437 3438defm VMOVAPS : avx512_alignedload_vl<0x28, "vmovaps", avx512vl_f32_info,3439 HasAVX512, SchedWriteFMoveLS>,3440 avx512_alignedstore_vl<0x29, "vmovaps", avx512vl_f32_info,3441 HasAVX512, SchedWriteFMoveLS>,3442 TB, EVEX_CD8<32, CD8VF>;3443 3444defm VMOVAPD : avx512_alignedload_vl<0x28, "vmovapd", avx512vl_f64_info,3445 HasAVX512, SchedWriteFMoveLS>,3446 avx512_alignedstore_vl<0x29, "vmovapd", avx512vl_f64_info,3447 HasAVX512, SchedWriteFMoveLS>,3448 TB, PD, REX_W, EVEX_CD8<64, CD8VF>;3449 3450defm VMOVUPS : avx512_load_vl<0x10, "vmovups", avx512vl_f32_info, HasAVX512,3451 SchedWriteFMoveLS, 0, null_frag>,3452 avx512_store_vl<0x11, "vmovups", avx512vl_f32_info, HasAVX512,3453 SchedWriteFMoveLS>,3454 TB, EVEX_CD8<32, CD8VF>;3455 3456defm VMOVUPD : avx512_load_vl<0x10, "vmovupd", avx512vl_f64_info, HasAVX512,3457 SchedWriteFMoveLS, 0, null_frag>,3458 avx512_store_vl<0x11, "vmovupd", avx512vl_f64_info, HasAVX512,3459 SchedWriteFMoveLS>,3460 TB, PD, REX_W, EVEX_CD8<64, CD8VF>;3461 3462defm VMOVDQA32 : avx512_alignedload_vl<0x6F, "vmovdqa32", avx512vl_i32_info,3463 HasAVX512, SchedWriteVecMoveLS, 1>,3464 avx512_alignedstore_vl<0x7F, "vmovdqa32", avx512vl_i32_info,3465 HasAVX512, SchedWriteVecMoveLS, 1>,3466 TB, PD, EVEX_CD8<32, CD8VF>;3467 3468defm VMOVDQA64 : avx512_alignedload_vl<0x6F, "vmovdqa64", avx512vl_i64_info,3469 HasAVX512, SchedWriteVecMoveLS>,3470 avx512_alignedstore_vl<0x7F, "vmovdqa64", avx512vl_i64_info,3471 HasAVX512, SchedWriteVecMoveLS>,3472 TB, PD, REX_W, EVEX_CD8<64, CD8VF>;3473 3474defm VMOVDQU8 : avx512_load_vl<0x6F, "vmovdqu8", avx512vl_i8_info, HasBWI,3475 SchedWriteVecMoveLS, 1>,3476 avx512_store_vl<0x7F, "vmovdqu8", avx512vl_i8_info, HasBWI,3477 SchedWriteVecMoveLS, 1>,3478 TB, XD, EVEX_CD8<8, CD8VF>;3479 3480defm VMOVDQU16 : avx512_load_vl<0x6F, "vmovdqu16", avx512vl_i16_info, HasBWI,3481 SchedWriteVecMoveLS, 1>,3482 avx512_store_vl<0x7F, "vmovdqu16", avx512vl_i16_info, HasBWI,3483 SchedWriteVecMoveLS, 1>,3484 TB, XD, REX_W, EVEX_CD8<16, CD8VF>;3485 3486defm VMOVDQU32 : avx512_load_vl<0x6F, "vmovdqu32", avx512vl_i32_info, HasAVX512,3487 SchedWriteVecMoveLS, 1, null_frag>,3488 avx512_store_vl<0x7F, "vmovdqu32", avx512vl_i32_info, HasAVX512,3489 SchedWriteVecMoveLS, 1>,3490 TB, XS, EVEX_CD8<32, CD8VF>;3491 3492defm VMOVDQU64 : avx512_load_vl<0x6F, "vmovdqu64", avx512vl_i64_info, HasAVX512,3493 SchedWriteVecMoveLS, 0, null_frag>,3494 avx512_store_vl<0x7F, "vmovdqu64", avx512vl_i64_info, HasAVX512,3495 SchedWriteVecMoveLS>,3496 TB, XS, REX_W, EVEX_CD8<64, CD8VF>;3497 3498// Special instructions to help with spilling when we don't have VLX. We need3499// to load or store from a ZMM register instead. These are converted in3500// expandPostRAPseudos.3501let isReMaterializable = 1, canFoldAsLoad = 1,3502 isPseudo = 1, mayLoad = 1, hasSideEffects = 0 in {3503def VMOVAPSZ128rm_NOVLX : I<0, Pseudo, (outs VR128X:$dst), (ins f128mem:$src),3504 "", []>, Sched<[WriteFLoadX]>;3505def VMOVAPSZ256rm_NOVLX : I<0, Pseudo, (outs VR256X:$dst), (ins f256mem:$src),3506 "", []>, Sched<[WriteFLoadY]>;3507def VMOVUPSZ128rm_NOVLX : I<0, Pseudo, (outs VR128X:$dst), (ins f128mem:$src),3508 "", []>, Sched<[WriteFLoadX]>;3509def VMOVUPSZ256rm_NOVLX : I<0, Pseudo, (outs VR256X:$dst), (ins f256mem:$src),3510 "", []>, Sched<[WriteFLoadY]>;3511}3512 3513let isPseudo = 1, mayStore = 1, hasSideEffects = 0 in {3514def VMOVAPSZ128mr_NOVLX : I<0, Pseudo, (outs), (ins f128mem:$dst, VR128X:$src),3515 "", []>, Sched<[WriteFStoreX]>;3516def VMOVAPSZ256mr_NOVLX : I<0, Pseudo, (outs), (ins f256mem:$dst, VR256X:$src),3517 "", []>, Sched<[WriteFStoreY]>;3518def VMOVUPSZ128mr_NOVLX : I<0, Pseudo, (outs), (ins f128mem:$dst, VR128X:$src),3519 "", []>, Sched<[WriteFStoreX]>;3520def VMOVUPSZ256mr_NOVLX : I<0, Pseudo, (outs), (ins f256mem:$dst, VR256X:$src),3521 "", []>, Sched<[WriteFStoreY]>;3522}3523 3524def : Pat<(v8i64 (vselect VK8WM:$mask, (v8i64 immAllZerosV),3525 (v8i64 VR512:$src))),3526 (VMOVDQA64Zrrkz (COPY_TO_REGCLASS (KNOTWkk (COPY_TO_REGCLASS VK8:$mask, VK16)),3527 VK8), VR512:$src)>;3528 3529def : Pat<(v16i32 (vselect VK16WM:$mask, (v16i32 immAllZerosV),3530 (v16i32 VR512:$src))),3531 (VMOVDQA32Zrrkz (KNOTWkk VK16WM:$mask), VR512:$src)>;3532 3533// These patterns exist to prevent the above patterns from introducing a second3534// mask inversion when one already exists.3535def : Pat<(v8i64 (vselect (v8i1 (vnot VK8:$mask)),3536 (v8i64 immAllZerosV),3537 (v8i64 VR512:$src))),3538 (VMOVDQA64Zrrkz VK8:$mask, VR512:$src)>;3539def : Pat<(v16i32 (vselect (v16i1 (vnot VK16:$mask)),3540 (v16i32 immAllZerosV),3541 (v16i32 VR512:$src))),3542 (VMOVDQA32Zrrkz VK16WM:$mask, VR512:$src)>;3543 3544multiclass mask_move_lowering<string InstrStr, X86VectorVTInfo Narrow,3545 X86VectorVTInfo Wide> {3546 def : Pat<(Narrow.VT (vselect (Narrow.KVT Narrow.KRCWM:$mask),3547 Narrow.RC:$src1, Narrow.RC:$src0)),3548 (EXTRACT_SUBREG3549 (Wide.VT3550 (!cast<Instruction>(InstrStr#"rrk")3551 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src0, Narrow.SubRegIdx)),3552 (COPY_TO_REGCLASS Narrow.KRCWM:$mask, Wide.KRCWM),3553 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)))),3554 Narrow.SubRegIdx)>;3555 3556 def : Pat<(Narrow.VT (vselect (Narrow.KVT Narrow.KRCWM:$mask),3557 Narrow.RC:$src1, Narrow.ImmAllZerosV)),3558 (EXTRACT_SUBREG3559 (Wide.VT3560 (!cast<Instruction>(InstrStr#"rrkz")3561 (COPY_TO_REGCLASS Narrow.KRCWM:$mask, Wide.KRCWM),3562 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)))),3563 Narrow.SubRegIdx)>;3564}3565 3566// Patterns for handling v8i1 selects of 256-bit vectors when VLX isn't3567// available. Use a 512-bit operation and extract.3568let Predicates = [HasAVX512, NoVLX] in {3569 defm : mask_move_lowering<"VMOVAPSZ", v4f32x_info, v16f32_info>;3570 defm : mask_move_lowering<"VMOVDQA32Z", v4i32x_info, v16i32_info>;3571 defm : mask_move_lowering<"VMOVAPSZ", v8f32x_info, v16f32_info>;3572 defm : mask_move_lowering<"VMOVDQA32Z", v8i32x_info, v16i32_info>;3573 3574 defm : mask_move_lowering<"VMOVAPDZ", v2f64x_info, v8f64_info>;3575 defm : mask_move_lowering<"VMOVDQA64Z", v2i64x_info, v8i64_info>;3576 defm : mask_move_lowering<"VMOVAPDZ", v4f64x_info, v8f64_info>;3577 defm : mask_move_lowering<"VMOVDQA64Z", v4i64x_info, v8i64_info>;3578}3579 3580let Predicates = [HasBWI, NoVLX] in {3581 defm : mask_move_lowering<"VMOVDQU8Z", v16i8x_info, v64i8_info>;3582 defm : mask_move_lowering<"VMOVDQU8Z", v32i8x_info, v64i8_info>;3583 3584 defm : mask_move_lowering<"VMOVDQU16Z", v8i16x_info, v32i16_info>;3585 defm : mask_move_lowering<"VMOVDQU16Z", v16i16x_info, v32i16_info>;3586 3587 defm : mask_move_lowering<"VMOVDQU16Z", v8f16x_info, v32f16_info>;3588 defm : mask_move_lowering<"VMOVDQU16Z", v16f16x_info, v32f16_info>;3589 3590 defm : mask_move_lowering<"VMOVDQU16Z", v8bf16x_info, v32bf16_info>;3591 defm : mask_move_lowering<"VMOVDQU16Z", v16bf16x_info, v32bf16_info>;3592}3593 3594let Predicates = [HasAVX512] in {3595 // 512-bit load.3596 def : Pat<(alignedloadv16i32 addr:$src),3597 (VMOVDQA64Zrm addr:$src)>;3598 def : Pat<(alignedloadv32i16 addr:$src),3599 (VMOVDQA64Zrm addr:$src)>;3600 def : Pat<(alignedloadv32f16 addr:$src),3601 (VMOVAPSZrm addr:$src)>;3602 def : Pat<(alignedloadv32bf16 addr:$src),3603 (VMOVAPSZrm addr:$src)>;3604 def : Pat<(alignedloadv64i8 addr:$src),3605 (VMOVDQA64Zrm addr:$src)>;3606 def : Pat<(loadv16i32 addr:$src),3607 (VMOVDQU64Zrm addr:$src)>;3608 def : Pat<(loadv32i16 addr:$src),3609 (VMOVDQU64Zrm addr:$src)>;3610 def : Pat<(loadv32f16 addr:$src),3611 (VMOVUPSZrm addr:$src)>;3612 def : Pat<(loadv32bf16 addr:$src),3613 (VMOVUPSZrm addr:$src)>;3614 def : Pat<(loadv64i8 addr:$src),3615 (VMOVDQU64Zrm addr:$src)>;3616 3617 // 512-bit store.3618 def : Pat<(alignedstore (v16i32 VR512:$src), addr:$dst),3619 (VMOVDQA64Zmr addr:$dst, VR512:$src)>;3620 def : Pat<(alignedstore (v32i16 VR512:$src), addr:$dst),3621 (VMOVDQA64Zmr addr:$dst, VR512:$src)>;3622 def : Pat<(alignedstore (v32f16 VR512:$src), addr:$dst),3623 (VMOVAPSZmr addr:$dst, VR512:$src)>;3624 def : Pat<(alignedstore (v32bf16 VR512:$src), addr:$dst),3625 (VMOVAPSZmr addr:$dst, VR512:$src)>;3626 def : Pat<(alignedstore (v64i8 VR512:$src), addr:$dst),3627 (VMOVDQA64Zmr addr:$dst, VR512:$src)>;3628 def : Pat<(store (v16i32 VR512:$src), addr:$dst),3629 (VMOVDQU64Zmr addr:$dst, VR512:$src)>;3630 def : Pat<(store (v32i16 VR512:$src), addr:$dst),3631 (VMOVDQU64Zmr addr:$dst, VR512:$src)>;3632 def : Pat<(store (v32f16 VR512:$src), addr:$dst),3633 (VMOVUPSZmr addr:$dst, VR512:$src)>;3634 def : Pat<(store (v32bf16 VR512:$src), addr:$dst),3635 (VMOVUPSZmr addr:$dst, VR512:$src)>;3636 def : Pat<(store (v64i8 VR512:$src), addr:$dst),3637 (VMOVDQU64Zmr addr:$dst, VR512:$src)>;3638}3639 3640let Predicates = [HasVLX] in {3641 // 128-bit load.3642 def : Pat<(alignedloadv4i32 addr:$src),3643 (VMOVDQA64Z128rm addr:$src)>;3644 def : Pat<(alignedloadv8i16 addr:$src),3645 (VMOVDQA64Z128rm addr:$src)>;3646 def : Pat<(alignedloadv8f16 addr:$src),3647 (VMOVAPSZ128rm addr:$src)>;3648 def : Pat<(alignedloadv8bf16 addr:$src),3649 (VMOVAPSZ128rm addr:$src)>;3650 def : Pat<(alignedloadv16i8 addr:$src),3651 (VMOVDQA64Z128rm addr:$src)>;3652 def : Pat<(loadv4i32 addr:$src),3653 (VMOVDQU64Z128rm addr:$src)>;3654 def : Pat<(loadv8i16 addr:$src),3655 (VMOVDQU64Z128rm addr:$src)>;3656 def : Pat<(loadv8f16 addr:$src),3657 (VMOVUPSZ128rm addr:$src)>;3658 def : Pat<(loadv8bf16 addr:$src),3659 (VMOVUPSZ128rm addr:$src)>;3660 def : Pat<(loadv16i8 addr:$src),3661 (VMOVDQU64Z128rm addr:$src)>;3662 3663 // 128-bit store.3664 def : Pat<(alignedstore (v4i32 VR128X:$src), addr:$dst),3665 (VMOVDQA64Z128mr addr:$dst, VR128X:$src)>;3666 def : Pat<(alignedstore (v8i16 VR128X:$src), addr:$dst),3667 (VMOVDQA64Z128mr addr:$dst, VR128X:$src)>;3668 def : Pat<(alignedstore (v8f16 VR128X:$src), addr:$dst),3669 (VMOVAPSZ128mr addr:$dst, VR128X:$src)>;3670 def : Pat<(alignedstore (v8bf16 VR128X:$src), addr:$dst),3671 (VMOVAPSZ128mr addr:$dst, VR128X:$src)>;3672 def : Pat<(alignedstore (v16i8 VR128X:$src), addr:$dst),3673 (VMOVDQA64Z128mr addr:$dst, VR128X:$src)>;3674 def : Pat<(store (v4i32 VR128X:$src), addr:$dst),3675 (VMOVDQU64Z128mr addr:$dst, VR128X:$src)>;3676 def : Pat<(store (v8i16 VR128X:$src), addr:$dst),3677 (VMOVDQU64Z128mr addr:$dst, VR128X:$src)>;3678 def : Pat<(store (v8f16 VR128X:$src), addr:$dst),3679 (VMOVUPSZ128mr addr:$dst, VR128X:$src)>;3680 def : Pat<(store (v8bf16 VR128X:$src), addr:$dst),3681 (VMOVUPSZ128mr addr:$dst, VR128X:$src)>;3682 def : Pat<(store (v16i8 VR128X:$src), addr:$dst),3683 (VMOVDQU64Z128mr addr:$dst, VR128X:$src)>;3684 3685 // 256-bit load.3686 def : Pat<(alignedloadv8i32 addr:$src),3687 (VMOVDQA64Z256rm addr:$src)>;3688 def : Pat<(alignedloadv16i16 addr:$src),3689 (VMOVDQA64Z256rm addr:$src)>;3690 def : Pat<(alignedloadv16f16 addr:$src),3691 (VMOVAPSZ256rm addr:$src)>;3692 def : Pat<(alignedloadv16bf16 addr:$src),3693 (VMOVAPSZ256rm addr:$src)>;3694 def : Pat<(alignedloadv32i8 addr:$src),3695 (VMOVDQA64Z256rm addr:$src)>;3696 def : Pat<(loadv8i32 addr:$src),3697 (VMOVDQU64Z256rm addr:$src)>;3698 def : Pat<(loadv16i16 addr:$src),3699 (VMOVDQU64Z256rm addr:$src)>;3700 def : Pat<(loadv16f16 addr:$src),3701 (VMOVUPSZ256rm addr:$src)>;3702 def : Pat<(loadv16bf16 addr:$src),3703 (VMOVUPSZ256rm addr:$src)>;3704 def : Pat<(loadv32i8 addr:$src),3705 (VMOVDQU64Z256rm addr:$src)>;3706 3707 // 256-bit store.3708 def : Pat<(alignedstore (v8i32 VR256X:$src), addr:$dst),3709 (VMOVDQA64Z256mr addr:$dst, VR256X:$src)>;3710 def : Pat<(alignedstore (v16i16 VR256X:$src), addr:$dst),3711 (VMOVDQA64Z256mr addr:$dst, VR256X:$src)>;3712 def : Pat<(alignedstore (v16f16 VR256X:$src), addr:$dst),3713 (VMOVAPSZ256mr addr:$dst, VR256X:$src)>;3714 def : Pat<(alignedstore (v16bf16 VR256X:$src), addr:$dst),3715 (VMOVAPSZ256mr addr:$dst, VR256X:$src)>;3716 def : Pat<(alignedstore (v32i8 VR256X:$src), addr:$dst),3717 (VMOVDQA64Z256mr addr:$dst, VR256X:$src)>;3718 def : Pat<(store (v8i32 VR256X:$src), addr:$dst),3719 (VMOVDQU64Z256mr addr:$dst, VR256X:$src)>;3720 def : Pat<(store (v16i16 VR256X:$src), addr:$dst),3721 (VMOVDQU64Z256mr addr:$dst, VR256X:$src)>;3722 def : Pat<(store (v16f16 VR256X:$src), addr:$dst),3723 (VMOVUPSZ256mr addr:$dst, VR256X:$src)>;3724 def : Pat<(store (v16bf16 VR256X:$src), addr:$dst),3725 (VMOVUPSZ256mr addr:$dst, VR256X:$src)>;3726 def : Pat<(store (v32i8 VR256X:$src), addr:$dst),3727 (VMOVDQU64Z256mr addr:$dst, VR256X:$src)>;3728}3729 3730multiclass mask_move_lowering_f16_bf16<AVX512VLVectorVTInfo _> {3731let Predicates = [HasBWI] in {3732 def : Pat<(_.info512.VT (vselect VK32WM:$mask, (_.info512.VT VR512:$src1), (_.info512.VT VR512:$src0))),3733 (VMOVDQU16Zrrk VR512:$src0, VK32WM:$mask, VR512:$src1)>;3734 def : Pat<(_.info512.VT (vselect VK32WM:$mask, (_.info512.VT VR512:$src1), _.info512.ImmAllZerosV)),3735 (VMOVDQU16Zrrkz VK32WM:$mask, VR512:$src1)>;3736 def : Pat<(_.info512.VT (vselect VK32WM:$mask,3737 (_.info512.VT (_.info512.AlignedLdFrag addr:$src)), (_.info512.VT VR512:$src0))),3738 (VMOVDQU16Zrmk VR512:$src0, VK32WM:$mask, addr:$src)>;3739 def : Pat<(_.info512.VT (vselect VK32WM:$mask,3740 (_.info512.VT (_.info512.AlignedLdFrag addr:$src)), _.info512.ImmAllZerosV)),3741 (VMOVDQU16Zrmkz VK32WM:$mask, addr:$src)>;3742 def : Pat<(_.info512.VT (vselect VK32WM:$mask,3743 (_.info512.VT (_.info512.LdFrag addr:$src)), (_.info512.VT VR512:$src0))),3744 (VMOVDQU16Zrmk VR512:$src0, VK32WM:$mask, addr:$src)>;3745 def : Pat<(_.info512.VT (vselect VK32WM:$mask,3746 (_.info512.VT (_.info512.LdFrag addr:$src)), _.info512.ImmAllZerosV)),3747 (VMOVDQU16Zrmkz VK32WM:$mask, addr:$src)>;3748 def : Pat<(_.info512.VT (masked_load addr:$src, VK32WM:$mask, (_.info512.VT VR512:$src0))),3749 (VMOVDQU16Zrmk VR512:$src0, VK32WM:$mask, addr:$src)>;3750 def : Pat<(_.info512.VT (masked_load addr:$src, VK32WM:$mask, undef)),3751 (VMOVDQU16Zrmkz VK32WM:$mask, addr:$src)>;3752 def : Pat<(_.info512.VT (masked_load addr:$src, VK32WM:$mask, _.info512.ImmAllZerosV)),3753 (VMOVDQU16Zrmkz VK32WM:$mask, addr:$src)>;3754 3755 def : Pat<(masked_store (_.info512.VT VR512:$src), addr:$dst, VK32WM:$mask),3756 (VMOVDQU16Zmrk addr:$dst, VK32WM:$mask, VR512:$src)>;3757}3758let Predicates = [HasBWI, HasVLX] in {3759 def : Pat<(_.info256.VT (vselect VK16WM:$mask, (_.info256.VT VR256X:$src1), (_.info256.VT VR256X:$src0))),3760 (VMOVDQU16Z256rrk VR256X:$src0, VK16WM:$mask, VR256X:$src1)>;3761 def : Pat<(_.info256.VT (vselect VK16WM:$mask, (_.info256.VT VR256X:$src1), _.info256.ImmAllZerosV)),3762 (VMOVDQU16Z256rrkz VK16WM:$mask, VR256X:$src1)>;3763 def : Pat<(_.info256.VT (vselect VK16WM:$mask,3764 (_.info256.VT (_.info256.AlignedLdFrag addr:$src)), (_.info256.VT VR256X:$src0))),3765 (VMOVDQU16Z256rmk VR256X:$src0, VK16WM:$mask, addr:$src)>;3766 def : Pat<(_.info256.VT (vselect VK16WM:$mask,3767 (_.info256.VT (_.info256.AlignedLdFrag addr:$src)), _.info256.ImmAllZerosV)),3768 (VMOVDQU16Z256rmkz VK16WM:$mask, addr:$src)>;3769 def : Pat<(_.info256.VT (vselect VK16WM:$mask,3770 (_.info256.VT (_.info256.LdFrag addr:$src)), (_.info256.VT VR256X:$src0))),3771 (VMOVDQU16Z256rmk VR256X:$src0, VK16WM:$mask, addr:$src)>;3772 def : Pat<(_.info256.VT (vselect VK16WM:$mask,3773 (_.info256.VT (_.info256.LdFrag addr:$src)), _.info256.ImmAllZerosV)),3774 (VMOVDQU16Z256rmkz VK16WM:$mask, addr:$src)>;3775 def : Pat<(_.info256.VT (masked_load addr:$src, VK16WM:$mask, (_.info256.VT VR256X:$src0))),3776 (VMOVDQU16Z256rmk VR256X:$src0, VK16WM:$mask, addr:$src)>;3777 def : Pat<(_.info256.VT (masked_load addr:$src, VK16WM:$mask, undef)),3778 (VMOVDQU16Z256rmkz VK16WM:$mask, addr:$src)>;3779 def : Pat<(_.info256.VT (masked_load addr:$src, VK16WM:$mask, _.info256.ImmAllZerosV)),3780 (VMOVDQU16Z256rmkz VK16WM:$mask, addr:$src)>;3781 3782 def : Pat<(masked_store (_.info256.VT VR256X:$src), addr:$dst, VK16WM:$mask),3783 (VMOVDQU16Z256mrk addr:$dst, VK16WM:$mask, VR256X:$src)>;3784 3785 def : Pat<(_.info128.VT (vselect VK8WM:$mask, (_.info128.VT VR128X:$src1), (_.info128.VT VR128X:$src0))),3786 (VMOVDQU16Z128rrk VR128X:$src0, VK8WM:$mask, VR128X:$src1)>;3787 def : Pat<(_.info128.VT (vselect VK8WM:$mask, (_.info128.VT VR128X:$src1), _.info128.ImmAllZerosV)),3788 (VMOVDQU16Z128rrkz VK8WM:$mask, VR128X:$src1)>;3789 def : Pat<(_.info128.VT (vselect VK8WM:$mask,3790 (_.info128.VT (_.info128.AlignedLdFrag addr:$src)), (_.info128.VT VR128X:$src0))),3791 (VMOVDQU16Z128rmk VR128X:$src0, VK8WM:$mask, addr:$src)>;3792 def : Pat<(_.info128.VT (vselect VK8WM:$mask,3793 (_.info128.VT (_.info128.AlignedLdFrag addr:$src)), _.info128.ImmAllZerosV)),3794 (VMOVDQU16Z128rmkz VK8WM:$mask, addr:$src)>;3795 def : Pat<(_.info128.VT (vselect VK8WM:$mask,3796 (_.info128.VT (_.info128.LdFrag addr:$src)), (_.info128.VT VR128X:$src0))),3797 (VMOVDQU16Z128rmk VR128X:$src0, VK8WM:$mask, addr:$src)>;3798 def : Pat<(_.info128.VT (vselect VK8WM:$mask,3799 (_.info128.VT (_.info128.LdFrag addr:$src)), _.info128.ImmAllZerosV)),3800 (VMOVDQU16Z128rmkz VK8WM:$mask, addr:$src)>;3801 def : Pat<(_.info128.VT (masked_load addr:$src, VK8WM:$mask, (_.info128.VT VR128X:$src0))),3802 (VMOVDQU16Z128rmk VR128X:$src0, VK8WM:$mask, addr:$src)>;3803 def : Pat<(_.info128.VT (masked_load addr:$src, VK8WM:$mask, undef)),3804 (VMOVDQU16Z128rmkz VK8WM:$mask, addr:$src)>;3805 def : Pat<(_.info128.VT (masked_load addr:$src, VK8WM:$mask, _.info128.ImmAllZerosV)),3806 (VMOVDQU16Z128rmkz VK8WM:$mask, addr:$src)>;3807 3808 def : Pat<(masked_store (_.info128.VT VR128X:$src), addr:$dst, VK8WM:$mask),3809 (VMOVDQU16Z128mrk addr:$dst, VK8WM:$mask, VR128X:$src)>;3810}3811}3812 3813defm : mask_move_lowering_f16_bf16<avx512vl_f16_info>;3814defm : mask_move_lowering_f16_bf16<avx512vl_bf16_info>;3815 3816// Move Int Doubleword to Packed Double Int3817//3818let ExeDomain = SSEPackedInt in {3819def VMOVDI2PDIZrr : AVX512BI<0x6E, MRMSrcReg, (outs VR128X:$dst), (ins GR32:$src),3820 "vmovd\t{$src, $dst|$dst, $src}",3821 [(set VR128X:$dst,3822 (v4i32 (scalar_to_vector GR32:$src)))]>,3823 EVEX, Sched<[WriteVecMoveFromGpr]>;3824let mayLoad = 1 in3825def VMOVDI2PDIZrm : AVX512BI<0x6E, MRMSrcMem, (outs VR128X:$dst), (ins i32mem:$src),3826 "vmovd\t{$src, $dst|$dst, $src}",3827 [(set VR128X:$dst,3828 (v4i32 (scalar_to_vector (loadi32 addr:$src))))]>,3829 EVEX, EVEX_CD8<32, CD8VT1>, Sched<[WriteVecLoad]>;3830def VMOV64toPQIZrr : AVX512BI<0x6E, MRMSrcReg, (outs VR128X:$dst), (ins GR64:$src),3831 "vmovq\t{$src, $dst|$dst, $src}",3832 [(set VR128X:$dst,3833 (v2i64 (scalar_to_vector GR64:$src)))]>,3834 EVEX, REX_W, Sched<[WriteVecMoveFromGpr]>;3835let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, mayLoad = 1 in3836def VMOV64toPQIZrm : AVX512BI<0x6E, MRMSrcMem, (outs VR128X:$dst),3837 (ins i64mem:$src),3838 "vmovq\t{$src, $dst|$dst, $src}", []>,3839 EVEX, REX_W, EVEX_CD8<64, CD8VT1>, Sched<[WriteVecLoad]>;3840let isCodeGenOnly = 1 in {3841def VMOV64toSDZrr : AVX512BI<0x6E, MRMSrcReg, (outs FR64X:$dst), (ins GR64:$src),3842 "vmovq\t{$src, $dst|$dst, $src}",3843 [(set FR64X:$dst, (bitconvert GR64:$src))]>,3844 EVEX, REX_W, Sched<[WriteVecMoveFromGpr]>;3845def VMOVSDto64Zrr : AVX512BI<0x7E, MRMDestReg, (outs GR64:$dst), (ins FR64X:$src),3846 "vmovq\t{$src, $dst|$dst, $src}",3847 [(set GR64:$dst, (bitconvert FR64X:$src))]>,3848 EVEX, REX_W, Sched<[WriteVecMoveFromGpr]>;3849}3850} // ExeDomain = SSEPackedInt3851 3852// Move Int Doubleword to Single Scalar3853//3854let ExeDomain = SSEPackedInt, isCodeGenOnly = 1 in {3855def VMOVDI2SSZrr : AVX512BI<0x6E, MRMSrcReg, (outs FR32X:$dst), (ins GR32:$src),3856 "vmovd\t{$src, $dst|$dst, $src}",3857 [(set FR32X:$dst, (bitconvert GR32:$src))]>,3858 EVEX, Sched<[WriteVecMoveFromGpr]>;3859} // ExeDomain = SSEPackedInt, isCodeGenOnly = 13860 3861// Move doubleword from xmm register to r/m323862//3863let ExeDomain = SSEPackedInt in {3864def VMOVPDI2DIZrr : AVX512BI<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128X:$src),3865 "vmovd\t{$src, $dst|$dst, $src}",3866 [(set GR32:$dst, (extractelt (v4i32 VR128X:$src),3867 (iPTR 0)))]>,3868 EVEX, Sched<[WriteVecMoveToGpr]>;3869def VMOVPDI2DIZmr : AVX512BI<0x7E, MRMDestMem, (outs),3870 (ins i32mem:$dst, VR128X:$src),3871 "vmovd\t{$src, $dst|$dst, $src}",3872 [(store (i32 (extractelt (v4i32 VR128X:$src),3873 (iPTR 0))), addr:$dst)]>,3874 EVEX, EVEX_CD8<32, CD8VT1>, Sched<[WriteVecStore]>;3875} // ExeDomain = SSEPackedInt3876 3877// Move quadword from xmm1 register to r/m643878//3879let ExeDomain = SSEPackedInt in {3880def VMOVPQIto64Zrr : I<0x7E, MRMDestReg, (outs GR64:$dst), (ins VR128X:$src),3881 "vmovq\t{$src, $dst|$dst, $src}",3882 [(set GR64:$dst, (extractelt (v2i64 VR128X:$src),3883 (iPTR 0)))]>,3884 TB, PD, EVEX, REX_W, Sched<[WriteVecMoveToGpr]>,3885 Requires<[HasAVX512]>;3886 3887let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, mayStore = 1 in3888def VMOVPQIto64Zmr : I<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, VR128X:$src),3889 "vmovq\t{$src, $dst|$dst, $src}", []>, TB, PD,3890 EVEX, REX_W, EVEX_CD8<64, CD8VT1>, Sched<[WriteVecStore]>,3891 Requires<[HasAVX512, In64BitMode]>;3892 3893def VMOVPQI2QIZmr : I<0xD6, MRMDestMem, (outs),3894 (ins i64mem:$dst, VR128X:$src),3895 "vmovq\t{$src, $dst|$dst, $src}",3896 [(store (extractelt (v2i64 VR128X:$src), (iPTR 0)),3897 addr:$dst)]>,3898 EVEX, TB, PD, REX_W, EVEX_CD8<64, CD8VT1>,3899 Sched<[WriteVecStore]>, Requires<[HasAVX512]>;3900 3901let hasSideEffects = 0, isCodeGenOnly = 1, ForceDisassemble = 1 in3902def VMOVPQI2QIZrr : AVX512BI<0xD6, MRMDestReg, (outs VR128X:$dst),3903 (ins VR128X:$src),3904 "vmovq\t{$src, $dst|$dst, $src}", []>,3905 EVEX, REX_W, Sched<[SchedWriteVecLogic.XMM]>;3906} // ExeDomain = SSEPackedInt3907 3908def : InstAlias<"vmovq.s\t{$src, $dst|$dst, $src}",3909 (VMOVPQI2QIZrr VR128X:$dst, VR128X:$src), 0>;3910 3911let Predicates = [HasAVX512] in {3912 def : Pat<(X86vextractstore64 (v2i64 VR128X:$src), addr:$dst),3913 (VMOVPQI2QIZmr addr:$dst, VR128X:$src)>;3914}3915 3916// Move Scalar Single to Double Int3917//3918let ExeDomain = SSEPackedInt, isCodeGenOnly = 1 in {3919def VMOVSS2DIZrr : AVX512BI<0x7E, MRMDestReg, (outs GR32:$dst),3920 (ins FR32X:$src),3921 "vmovd\t{$src, $dst|$dst, $src}",3922 [(set GR32:$dst, (bitconvert FR32X:$src))]>,3923 EVEX, Sched<[WriteVecMoveToGpr]>;3924} // ExeDomain = SSEPackedInt, isCodeGenOnly = 13925 3926// Move Quadword Int to Packed Quadword Int3927//3928let ExeDomain = SSEPackedInt, mayLoad = 1, hasSideEffects = 0 in {3929def VMOVQI2PQIZrm : AVX512XSI<0x7E, MRMSrcMem, (outs VR128X:$dst),3930 (ins i64mem:$src),3931 "vmovq\t{$src, $dst|$dst, $src}",3932 [(set VR128X:$dst,3933 (v2i64 (scalar_to_vector (loadi64 addr:$src))))]>,3934 EVEX, REX_W, EVEX_CD8<8, CD8VT8>, Sched<[WriteVecLoad]>;3935} // ExeDomain = SSEPackedInt3936 3937// Allow "vmovd" but print "vmovq".3938def : InstAlias<"vmovd\t{$src, $dst|$dst, $src}",3939 (VMOV64toPQIZrr VR128X:$dst, GR64:$src), 0>;3940def : InstAlias<"vmovd\t{$src, $dst|$dst, $src}",3941 (VMOVPQIto64Zrr GR64:$dst, VR128X:$src), 0>;3942 3943// Conversions between masks and scalar fp.3944def : Pat<(v32i1 (bitconvert FR32X:$src)),3945 (KMOVDkr (VMOVSS2DIZrr FR32X:$src))>;3946def : Pat<(f32 (bitconvert VK32:$src)),3947 (VMOVDI2SSZrr (KMOVDrk VK32:$src))>;3948 3949def : Pat<(v64i1 (bitconvert FR64X:$src)),3950 (KMOVQkr (VMOVSDto64Zrr FR64X:$src))>;3951def : Pat<(f64 (bitconvert VK64:$src)),3952 (VMOV64toSDZrr (KMOVQrk VK64:$src))>;3953 3954//===----------------------------------------------------------------------===//3955// AVX-512 MOVSH, MOVSS, MOVSD3956//===----------------------------------------------------------------------===//3957 3958multiclass avx512_move_scalar<string asm, SDNode OpNode, PatFrag vzload_frag,3959 X86VectorVTInfo _, Predicate prd = HasAVX512> {3960 let Predicates = !if (!eq (prd, HasFP16), [HasFP16], [prd, OptForSize]) in3961 def rr : AVX512PI<0x10, MRMSrcReg, (outs _.RC:$dst),3962 (ins _.RC:$src1, _.RC:$src2),3963 !strconcat(asm, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),3964 [(set _.RC:$dst, (_.VT (OpNode _.RC:$src1, _.RC:$src2)))],3965 _.ExeDomain>, EVEX, VVVV, Sched<[SchedWriteFShuffle.XMM]>;3966 let Predicates = [prd] in {3967 def rrkz : AVX512PI<0x10, MRMSrcReg, (outs _.RC:$dst),3968 (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2),3969 !strconcat(asm, "\t{$src2, $src1, $dst {${mask}} {z}|",3970 "$dst {${mask}} {z}, $src1, $src2}"),3971 [(set _.RC:$dst, (_.VT (X86selects _.KRCWM:$mask,3972 (_.VT (OpNode _.RC:$src1, _.RC:$src2)),3973 _.ImmAllZerosV)))],3974 _.ExeDomain>, EVEX, VVVV, EVEX_KZ, Sched<[SchedWriteFShuffle.XMM]>;3975 let Constraints = "$src0 = $dst" in3976 def rrk : AVX512PI<0x10, MRMSrcReg, (outs _.RC:$dst),3977 (ins _.RC:$src0, _.KRCWM:$mask, _.RC:$src1, _.RC:$src2),3978 !strconcat(asm, "\t{$src2, $src1, $dst {${mask}}|",3979 "$dst {${mask}}, $src1, $src2}"),3980 [(set _.RC:$dst, (_.VT (X86selects _.KRCWM:$mask,3981 (_.VT (OpNode _.RC:$src1, _.RC:$src2)),3982 (_.VT _.RC:$src0))))],3983 _.ExeDomain>, EVEX, VVVV, EVEX_K, Sched<[SchedWriteFShuffle.XMM]>;3984 let canFoldAsLoad = 1, isReMaterializable = 1, mayLoad = 1, hasSideEffects = 0 in {3985 def rm : AVX512PI<0x10, MRMSrcMem, (outs _.RC:$dst), (ins _.ScalarMemOp:$src),3986 !strconcat(asm, "\t{$src, $dst|$dst, $src}"),3987 [(set _.RC:$dst, (_.VT (vzload_frag addr:$src)))],3988 _.ExeDomain>, EVEX, Sched<[WriteFLoad]>;3989 // _alt version uses FR32/FR64 register class.3990 let isCodeGenOnly = 1, mayLoad = 1, hasSideEffects = 0 in3991 def rm_alt : AVX512PI<0x10, MRMSrcMem, (outs _.FRC:$dst), (ins _.ScalarMemOp:$src),3992 !strconcat(asm, "\t{$src, $dst|$dst, $src}"),3993 [(set _.FRC:$dst, (_.ScalarLdFrag addr:$src))],3994 _.ExeDomain>, EVEX, Sched<[WriteFLoad]>;3995 }3996 let mayLoad = 1, hasSideEffects = 0 in {3997 let Constraints = "$src0 = $dst" in3998 def rmk : AVX512PI<0x10, MRMSrcMem, (outs _.RC:$dst),3999 (ins _.RC:$src0, _.KRCWM:$mask, _.ScalarMemOp:$src),4000 !strconcat(asm, "\t{$src, $dst {${mask}}|",4001 "$dst {${mask}}, $src}"),4002 [], _.ExeDomain>, EVEX, EVEX_K, Sched<[WriteFLoad]>;4003 def rmkz : AVX512PI<0x10, MRMSrcMem, (outs _.RC:$dst),4004 (ins _.KRCWM:$mask, _.ScalarMemOp:$src),4005 !strconcat(asm, "\t{$src, $dst {${mask}} {z}|",4006 "$dst {${mask}} {z}, $src}"),4007 [], _.ExeDomain>, EVEX, EVEX_KZ, Sched<[WriteFLoad]>;4008 }4009 def mr: AVX512PI<0x11, MRMDestMem, (outs), (ins _.ScalarMemOp:$dst, _.FRC:$src),4010 !strconcat(asm, "\t{$src, $dst|$dst, $src}"),4011 [(store _.FRC:$src, addr:$dst)], _.ExeDomain>,4012 EVEX, Sched<[WriteFStore]>;4013 let mayStore = 1, hasSideEffects = 0 in4014 def mrk: AVX512PI<0x11, MRMDestMem, (outs),4015 (ins _.ScalarMemOp:$dst, VK1WM:$mask, _.RC:$src),4016 !strconcat(asm, "\t{$src, $dst {${mask}}|$dst {${mask}}, $src}"),4017 [], _.ExeDomain>, EVEX, EVEX_K, Sched<[WriteFStore]>;4018 }4019}4020 4021defm VMOVSSZ : avx512_move_scalar<"vmovss", X86Movss, X86vzload32, f32x_info>,4022 VEX_LIG, TB, XS, EVEX_CD8<32, CD8VT1>;4023 4024defm VMOVSDZ : avx512_move_scalar<"vmovsd", X86Movsd, X86vzload64, f64x_info>,4025 VEX_LIG, TB, XD, REX_W, EVEX_CD8<64, CD8VT1>;4026 4027defm VMOVSHZ : avx512_move_scalar<"vmovsh", X86Movsh, X86vzload16, f16x_info,4028 HasFP16>,4029 VEX_LIG, T_MAP5, XS, EVEX_CD8<16, CD8VT1>;4030 4031multiclass avx512_move_scalar_lowering<string InstrStr, SDNode OpNode,4032 PatLeaf ZeroFP, X86VectorVTInfo _> {4033 4034def : Pat<(_.VT (OpNode _.RC:$src0,4035 (_.VT (scalar_to_vector4036 (_.EltVT (X86selects VK1WM:$mask,4037 (_.EltVT _.FRC:$src1),4038 (_.EltVT _.FRC:$src2))))))),4039 (!cast<Instruction>(InstrStr#rrk)4040 (_.VT (COPY_TO_REGCLASS _.FRC:$src2, _.RC)),4041 VK1WM:$mask,4042 (_.VT _.RC:$src0),4043 (_.VT (COPY_TO_REGCLASS _.FRC:$src1, _.RC)))>;4044 4045def : Pat<(_.VT (OpNode _.RC:$src0,4046 (_.VT (scalar_to_vector4047 (_.EltVT (X86selects VK1WM:$mask,4048 (_.EltVT _.FRC:$src1),4049 (_.EltVT ZeroFP))))))),4050 (!cast<Instruction>(InstrStr#rrkz)4051 VK1WM:$mask,4052 (_.VT _.RC:$src0),4053 (_.VT (COPY_TO_REGCLASS _.FRC:$src1, _.RC)))>;4054}4055 4056multiclass avx512_store_scalar_lowering<string InstrStr, AVX512VLVectorVTInfo _,4057 dag Mask, RegisterClass MaskRC> {4058 4059def : Pat<(masked_store4060 (_.info512.VT (insert_subvector undef,4061 (_.info128.VT _.info128.RC:$src),4062 (iPTR 0))), addr:$dst, Mask),4063 (!cast<Instruction>(InstrStr#mrk) addr:$dst,4064 (COPY_TO_REGCLASS MaskRC:$mask, VK1WM),4065 _.info128.RC:$src)>;4066 4067}4068 4069multiclass avx512_store_scalar_lowering_subreg<string InstrStr,4070 AVX512VLVectorVTInfo _,4071 dag Mask, RegisterClass MaskRC,4072 SubRegIndex subreg> {4073 4074def : Pat<(masked_store4075 (_.info512.VT (insert_subvector undef,4076 (_.info128.VT _.info128.RC:$src),4077 (iPTR 0))), addr:$dst, Mask),4078 (!cast<Instruction>(InstrStr#mrk) addr:$dst,4079 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), MaskRC:$mask, subreg)), VK1WM),4080 _.info128.RC:$src)>;4081 4082}4083 4084// This matches the more recent codegen from clang that avoids emitting a 5124085// bit masked store directly. Codegen will widen 128-bit masked store to 5124086// bits on AVX512F only targets.4087multiclass avx512_store_scalar_lowering_subreg2<string InstrStr,4088 AVX512VLVectorVTInfo _,4089 dag Mask512, dag Mask128,4090 RegisterClass MaskRC,4091 SubRegIndex subreg> {4092 4093// AVX512F pattern.4094def : Pat<(masked_store4095 (_.info512.VT (insert_subvector undef,4096 (_.info128.VT _.info128.RC:$src),4097 (iPTR 0))), addr:$dst, Mask512),4098 (!cast<Instruction>(InstrStr#mrk) addr:$dst,4099 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), MaskRC:$mask, subreg)), VK1WM),4100 _.info128.RC:$src)>;4101 4102// AVX512VL pattern.4103def : Pat<(masked_store (_.info128.VT _.info128.RC:$src), addr:$dst, Mask128),4104 (!cast<Instruction>(InstrStr#mrk) addr:$dst,4105 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), MaskRC:$mask, subreg)), VK1WM),4106 _.info128.RC:$src)>;4107}4108 4109multiclass avx512_load_scalar_lowering<string InstrStr, AVX512VLVectorVTInfo _,4110 dag Mask, RegisterClass MaskRC> {4111 4112def : Pat<(_.info128.VT (extract_subvector4113 (_.info512.VT (masked_load addr:$srcAddr, Mask,4114 _.info512.ImmAllZerosV)),4115 (iPTR 0))),4116 (!cast<Instruction>(InstrStr#rmkz)4117 (COPY_TO_REGCLASS MaskRC:$mask, VK1WM),4118 addr:$srcAddr)>;4119 4120def : Pat<(_.info128.VT (extract_subvector4121 (_.info512.VT (masked_load addr:$srcAddr, Mask,4122 (_.info512.VT (insert_subvector undef,4123 (_.info128.VT (X86vzmovl _.info128.RC:$src)),4124 (iPTR 0))))),4125 (iPTR 0))),4126 (!cast<Instruction>(InstrStr#rmk) _.info128.RC:$src,4127 (COPY_TO_REGCLASS MaskRC:$mask, VK1WM),4128 addr:$srcAddr)>;4129 4130}4131 4132multiclass avx512_load_scalar_lowering_subreg<string InstrStr,4133 AVX512VLVectorVTInfo _,4134 dag Mask, RegisterClass MaskRC,4135 SubRegIndex subreg> {4136 4137def : Pat<(_.info128.VT (extract_subvector4138 (_.info512.VT (masked_load addr:$srcAddr, Mask,4139 _.info512.ImmAllZerosV)),4140 (iPTR 0))),4141 (!cast<Instruction>(InstrStr#rmkz)4142 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), MaskRC:$mask, subreg)), VK1WM),4143 addr:$srcAddr)>;4144 4145def : Pat<(_.info128.VT (extract_subvector4146 (_.info512.VT (masked_load addr:$srcAddr, Mask,4147 (_.info512.VT (insert_subvector undef,4148 (_.info128.VT (X86vzmovl _.info128.RC:$src)),4149 (iPTR 0))))),4150 (iPTR 0))),4151 (!cast<Instruction>(InstrStr#rmk) _.info128.RC:$src,4152 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), MaskRC:$mask, subreg)), VK1WM),4153 addr:$srcAddr)>;4154 4155}4156 4157// This matches the more recent codegen from clang that avoids emitting a 5124158// bit masked load directly. Codegen will widen 128-bit masked load to 5124159// bits on AVX512F only targets.4160multiclass avx512_load_scalar_lowering_subreg2<string InstrStr,4161 AVX512VLVectorVTInfo _,4162 dag Mask512, dag Mask128,4163 RegisterClass MaskRC,4164 SubRegIndex subreg> {4165// AVX512F patterns.4166def : Pat<(_.info128.VT (extract_subvector4167 (_.info512.VT (masked_load addr:$srcAddr, Mask512,4168 _.info512.ImmAllZerosV)),4169 (iPTR 0))),4170 (!cast<Instruction>(InstrStr#rmkz)4171 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), MaskRC:$mask, subreg)), VK1WM),4172 addr:$srcAddr)>;4173 4174def : Pat<(_.info128.VT (extract_subvector4175 (_.info512.VT (masked_load addr:$srcAddr, Mask512,4176 (_.info512.VT (insert_subvector undef,4177 (_.info128.VT (X86vzmovl _.info128.RC:$src)),4178 (iPTR 0))))),4179 (iPTR 0))),4180 (!cast<Instruction>(InstrStr#rmk) _.info128.RC:$src,4181 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), MaskRC:$mask, subreg)), VK1WM),4182 addr:$srcAddr)>;4183 4184// AVX512Vl patterns.4185def : Pat<(_.info128.VT (masked_load addr:$srcAddr, Mask128,4186 _.info128.ImmAllZerosV)),4187 (!cast<Instruction>(InstrStr#rmkz)4188 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), MaskRC:$mask, subreg)), VK1WM),4189 addr:$srcAddr)>;4190 4191def : Pat<(_.info128.VT (masked_load addr:$srcAddr, Mask128,4192 (_.info128.VT (X86vzmovl _.info128.RC:$src)))),4193 (!cast<Instruction>(InstrStr#rmk) _.info128.RC:$src,4194 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), MaskRC:$mask, subreg)), VK1WM),4195 addr:$srcAddr)>;4196}4197 4198defm : avx512_move_scalar_lowering<"VMOVSSZ", X86Movss, fp32imm0, v4f32x_info>;4199defm : avx512_move_scalar_lowering<"VMOVSDZ", X86Movsd, fp64imm0, v2f64x_info>;4200 4201defm : avx512_store_scalar_lowering<"VMOVSSZ", avx512vl_f32_info,4202 (v16i1 (bitconvert (i16 (trunc (and GR32:$mask, (i32 1)))))), GR32>;4203defm : avx512_store_scalar_lowering_subreg<"VMOVSSZ", avx512vl_f32_info,4204 (v16i1 (bitconvert (i16 (and GR16:$mask, (i16 1))))), GR16, sub_16bit>;4205defm : avx512_store_scalar_lowering_subreg<"VMOVSDZ", avx512vl_f64_info,4206 (v8i1 (bitconvert (i8 (and GR8:$mask, (i8 1))))), GR8, sub_8bit>;4207 4208let Predicates = [HasFP16] in {4209defm : avx512_move_scalar_lowering<"VMOVSHZ", X86Movsh, fp16imm0, v8f16x_info>;4210defm : avx512_store_scalar_lowering<"VMOVSHZ", avx512vl_f16_info,4211 (v32i1 (bitconvert (and GR32:$mask, (i32 1)))), GR32>;4212defm : avx512_store_scalar_lowering_subreg<"VMOVSHZ", avx512vl_f16_info,4213 (v32i1 (bitconvert (and GR32:$mask, (i32 1)))), GR32, sub_32bit>;4214defm : avx512_store_scalar_lowering_subreg2<"VMOVSHZ", avx512vl_f16_info,4215 (v32i1 (insert_subvector4216 (v32i1 immAllZerosV),4217 (v8i1 (bitconvert (and GR8:$mask, (i8 1)))),4218 (iPTR 0))),4219 (v8i1 (bitconvert (and GR8:$mask, (i8 1)))),4220 GR8, sub_8bit>;4221 4222defm : avx512_load_scalar_lowering<"VMOVSHZ", avx512vl_f16_info,4223 (v32i1 (bitconvert (and GR32:$mask, (i32 1)))), GR32>;4224defm : avx512_load_scalar_lowering_subreg<"VMOVSHZ", avx512vl_f16_info,4225 (v32i1 (bitconvert (and GR32:$mask, (i32 1)))), GR32, sub_32bit>;4226defm : avx512_load_scalar_lowering_subreg2<"VMOVSHZ", avx512vl_f16_info,4227 (v32i1 (insert_subvector4228 (v32i1 immAllZerosV),4229 (v8i1 (bitconvert (and GR8:$mask, (i8 1)))),4230 (iPTR 0))),4231 (v8i1 (bitconvert (and GR8:$mask, (i8 1)))),4232 GR8, sub_8bit>;4233 4234def : Pat<(f16 (X86selects VK1WM:$mask, (f16 FR16X:$src1), (f16 FR16X:$src2))),4235 (COPY_TO_REGCLASS (v8f16 (VMOVSHZrrk4236 (v8f16 (COPY_TO_REGCLASS FR16X:$src2, VR128X)),4237 VK1WM:$mask, (v8f16 (IMPLICIT_DEF)),4238 (v8f16 (COPY_TO_REGCLASS FR16X:$src1, VR128X)))), FR16X)>;4239 4240def : Pat<(f16 (X86selects VK1WM:$mask, (f16 FR16X:$src1), fp16imm0)),4241 (COPY_TO_REGCLASS (v8f16 (VMOVSHZrrkz VK1WM:$mask, (v8f16 (IMPLICIT_DEF)),4242 (v8f16 (COPY_TO_REGCLASS FR16X:$src1, VR128X)))), FR16X)>;4243}4244 4245defm : avx512_store_scalar_lowering_subreg2<"VMOVSSZ", avx512vl_f32_info,4246 (v16i1 (insert_subvector4247 (v16i1 immAllZerosV),4248 (v4i1 (extract_subvector4249 (v8i1 (bitconvert (and GR8:$mask, (i8 1)))),4250 (iPTR 0))),4251 (iPTR 0))),4252 (v4i1 (extract_subvector4253 (v8i1 (bitconvert (and GR8:$mask, (i8 1)))),4254 (iPTR 0))), GR8, sub_8bit>;4255defm : avx512_store_scalar_lowering_subreg2<"VMOVSDZ", avx512vl_f64_info,4256 (v8i14257 (extract_subvector4258 (v16i14259 (insert_subvector4260 (v16i1 immAllZerosV),4261 (v2i1 (extract_subvector4262 (v8i1 (bitconvert (i8 (and GR8:$mask, (i8 1))))),4263 (iPTR 0))),4264 (iPTR 0))),4265 (iPTR 0))),4266 (v2i1 (extract_subvector4267 (v8i1 (bitconvert (i8 (and GR8:$mask, (i8 1))))),4268 (iPTR 0))), GR8, sub_8bit>;4269 4270defm : avx512_load_scalar_lowering<"VMOVSSZ", avx512vl_f32_info,4271 (v16i1 (bitconvert (i16 (trunc (and GR32:$mask, (i32 1)))))), GR32>;4272defm : avx512_load_scalar_lowering_subreg<"VMOVSSZ", avx512vl_f32_info,4273 (v16i1 (bitconvert (i16 (and GR16:$mask, (i16 1))))), GR16, sub_16bit>;4274defm : avx512_load_scalar_lowering_subreg<"VMOVSDZ", avx512vl_f64_info,4275 (v8i1 (bitconvert (i8 (and GR8:$mask, (i8 1))))), GR8, sub_8bit>;4276 4277defm : avx512_load_scalar_lowering_subreg2<"VMOVSSZ", avx512vl_f32_info,4278 (v16i1 (insert_subvector4279 (v16i1 immAllZerosV),4280 (v4i1 (extract_subvector4281 (v8i1 (bitconvert (and GR8:$mask, (i8 1)))),4282 (iPTR 0))),4283 (iPTR 0))),4284 (v4i1 (extract_subvector4285 (v8i1 (bitconvert (and GR8:$mask, (i8 1)))),4286 (iPTR 0))), GR8, sub_8bit>;4287defm : avx512_load_scalar_lowering_subreg2<"VMOVSDZ", avx512vl_f64_info,4288 (v8i14289 (extract_subvector4290 (v16i14291 (insert_subvector4292 (v16i1 immAllZerosV),4293 (v2i1 (extract_subvector4294 (v8i1 (bitconvert (i8 (and GR8:$mask, (i8 1))))),4295 (iPTR 0))),4296 (iPTR 0))),4297 (iPTR 0))),4298 (v2i1 (extract_subvector4299 (v8i1 (bitconvert (i8 (and GR8:$mask, (i8 1))))),4300 (iPTR 0))), GR8, sub_8bit>;4301 4302def : Pat<(f32 (X86selects VK1WM:$mask, (f32 FR32X:$src1), (f32 FR32X:$src2))),4303 (COPY_TO_REGCLASS (v4f32 (VMOVSSZrrk4304 (v4f32 (COPY_TO_REGCLASS FR32X:$src2, VR128X)),4305 VK1WM:$mask, (v4f32 (IMPLICIT_DEF)),4306 (v4f32 (COPY_TO_REGCLASS FR32X:$src1, VR128X)))), FR32X)>;4307 4308def : Pat<(f32 (X86selects VK1WM:$mask, (f32 FR32X:$src1), fp32imm0)),4309 (COPY_TO_REGCLASS (v4f32 (VMOVSSZrrkz VK1WM:$mask, (v4f32 (IMPLICIT_DEF)),4310 (v4f32 (COPY_TO_REGCLASS FR32X:$src1, VR128X)))), FR32X)>;4311 4312def : Pat<(f32 (X86selects VK1WM:$mask, (loadf32 addr:$src), (f32 FR32X:$src0))),4313 (COPY_TO_REGCLASS4314 (v4f32 (VMOVSSZrmk (v4f32 (COPY_TO_REGCLASS FR32X:$src0, VR128X)),4315 VK1WM:$mask, addr:$src)),4316 FR32X)>;4317def : Pat<(f32 (X86selects VK1WM:$mask, (loadf32 addr:$src), fp32imm0)),4318 (COPY_TO_REGCLASS (v4f32 (VMOVSSZrmkz VK1WM:$mask, addr:$src)), FR32X)>;4319 4320def : Pat<(f64 (X86selects VK1WM:$mask, (f64 FR64X:$src1), (f64 FR64X:$src2))),4321 (COPY_TO_REGCLASS (v2f64 (VMOVSDZrrk4322 (v2f64 (COPY_TO_REGCLASS FR64X:$src2, VR128X)),4323 VK1WM:$mask, (v2f64 (IMPLICIT_DEF)),4324 (v2f64 (COPY_TO_REGCLASS FR64X:$src1, VR128X)))), FR64X)>;4325 4326def : Pat<(f64 (X86selects VK1WM:$mask, (f64 FR64X:$src1), fp64imm0)),4327 (COPY_TO_REGCLASS (v2f64 (VMOVSDZrrkz VK1WM:$mask, (v2f64 (IMPLICIT_DEF)),4328 (v2f64 (COPY_TO_REGCLASS FR64X:$src1, VR128X)))), FR64X)>;4329 4330def : Pat<(f64 (X86selects VK1WM:$mask, (loadf64 addr:$src), (f64 FR64X:$src0))),4331 (COPY_TO_REGCLASS4332 (v2f64 (VMOVSDZrmk (v2f64 (COPY_TO_REGCLASS FR64X:$src0, VR128X)),4333 VK1WM:$mask, addr:$src)),4334 FR64X)>;4335def : Pat<(f64 (X86selects VK1WM:$mask, (loadf64 addr:$src), fp64imm0)),4336 (COPY_TO_REGCLASS (v2f64 (VMOVSDZrmkz VK1WM:$mask, addr:$src)), FR64X)>;4337 4338 4339def : Pat<(v4f32 (X86selects VK1WM:$mask, (v4f32 VR128X:$src1), (v4f32 VR128X:$src2))),4340 (VMOVSSZrrk VR128X:$src2, VK1WM:$mask, VR128X:$src1, VR128X:$src1)>;4341def : Pat<(v2f64 (X86selects VK1WM:$mask, (v2f64 VR128X:$src1), (v2f64 VR128X:$src2))),4342 (VMOVSDZrrk VR128X:$src2, VK1WM:$mask, VR128X:$src1, VR128X:$src1)>;4343 4344def : Pat<(v4f32 (X86selects VK1WM:$mask, (v4f32 VR128X:$src1), (v4f32 immAllZerosV))),4345 (VMOVSSZrrkz VK1WM:$mask, VR128X:$src1, VR128X:$src1)>;4346def : Pat<(v2f64 (X86selects VK1WM:$mask, (v2f64 VR128X:$src1), (v2f64 immAllZerosV))),4347 (VMOVSDZrrkz VK1WM:$mask, VR128X:$src1, VR128X:$src1)>;4348 4349let hasSideEffects = 0, isCodeGenOnly = 1, ForceDisassemble = 1 in {4350 let Predicates = [HasFP16] in {4351 def VMOVSHZrr_REV: AVX512<0x11, MRMDestReg, (outs VR128X:$dst),4352 (ins VR128X:$src1, VR128X:$src2),4353 "vmovsh\t{$src2, $src1, $dst|$dst, $src1, $src2}",4354 []>, T_MAP5, XS, EVEX, VVVV, VEX_LIG,4355 Sched<[SchedWriteFShuffle.XMM]>;4356 4357 let Constraints = "$src0 = $dst" in4358 def VMOVSHZrrk_REV: AVX512<0x11, MRMDestReg, (outs VR128X:$dst),4359 (ins f16x_info.RC:$src0, f16x_info.KRCWM:$mask,4360 VR128X:$src1, VR128X:$src2),4361 "vmovsh\t{$src2, $src1, $dst {${mask}}|"#4362 "$dst {${mask}}, $src1, $src2}",4363 []>, T_MAP5, XS, EVEX_K, EVEX, VVVV, VEX_LIG,4364 Sched<[SchedWriteFShuffle.XMM]>;4365 4366 def VMOVSHZrrkz_REV: AVX512<0x11, MRMDestReg, (outs VR128X:$dst),4367 (ins f16x_info.KRCWM:$mask, VR128X:$src1, VR128X:$src2),4368 "vmovsh\t{$src2, $src1, $dst {${mask}} {z}|"#4369 "$dst {${mask}} {z}, $src1, $src2}",4370 []>, EVEX_KZ, T_MAP5, XS, EVEX, VVVV, VEX_LIG,4371 Sched<[SchedWriteFShuffle.XMM]>;4372 }4373 def VMOVSSZrr_REV: AVX512<0x11, MRMDestReg, (outs VR128X:$dst),4374 (ins VR128X:$src1, VR128X:$src2),4375 "vmovss\t{$src2, $src1, $dst|$dst, $src1, $src2}",4376 []>, TB, XS, EVEX, VVVV, VEX_LIG,4377 Sched<[SchedWriteFShuffle.XMM]>;4378 4379 let Constraints = "$src0 = $dst" in4380 def VMOVSSZrrk_REV: AVX512<0x11, MRMDestReg, (outs VR128X:$dst),4381 (ins f32x_info.RC:$src0, f32x_info.KRCWM:$mask,4382 VR128X:$src1, VR128X:$src2),4383 "vmovss\t{$src2, $src1, $dst {${mask}}|"#4384 "$dst {${mask}}, $src1, $src2}",4385 []>, EVEX_K, TB, XS, EVEX, VVVV, VEX_LIG,4386 Sched<[SchedWriteFShuffle.XMM]>;4387 4388 def VMOVSSZrrkz_REV: AVX512<0x11, MRMDestReg, (outs VR128X:$dst),4389 (ins f32x_info.KRCWM:$mask, VR128X:$src1, VR128X:$src2),4390 "vmovss\t{$src2, $src1, $dst {${mask}} {z}|"#4391 "$dst {${mask}} {z}, $src1, $src2}",4392 []>, EVEX_KZ, TB, XS, EVEX, VVVV, VEX_LIG,4393 Sched<[SchedWriteFShuffle.XMM]>;4394 4395 def VMOVSDZrr_REV: AVX512<0x11, MRMDestReg, (outs VR128X:$dst),4396 (ins VR128X:$src1, VR128X:$src2),4397 "vmovsd\t{$src2, $src1, $dst|$dst, $src1, $src2}",4398 []>, TB, XD, EVEX, VVVV, VEX_LIG, REX_W,4399 Sched<[SchedWriteFShuffle.XMM]>;4400 4401 let Constraints = "$src0 = $dst" in4402 def VMOVSDZrrk_REV: AVX512<0x11, MRMDestReg, (outs VR128X:$dst),4403 (ins f64x_info.RC:$src0, f64x_info.KRCWM:$mask,4404 VR128X:$src1, VR128X:$src2),4405 "vmovsd\t{$src2, $src1, $dst {${mask}}|"#4406 "$dst {${mask}}, $src1, $src2}",4407 []>, EVEX_K, TB, XD, EVEX, VVVV, VEX_LIG,4408 REX_W, Sched<[SchedWriteFShuffle.XMM]>;4409 4410 def VMOVSDZrrkz_REV: AVX512<0x11, MRMDestReg, (outs VR128X:$dst),4411 (ins f64x_info.KRCWM:$mask, VR128X:$src1,4412 VR128X:$src2),4413 "vmovsd\t{$src2, $src1, $dst {${mask}} {z}|"#4414 "$dst {${mask}} {z}, $src1, $src2}",4415 []>, EVEX_KZ, TB, XD, EVEX, VVVV, VEX_LIG,4416 REX_W, Sched<[SchedWriteFShuffle.XMM]>;4417}4418 4419def : InstAlias<"vmovsh.s\t{$src2, $src1, $dst|$dst, $src1, $src2}",4420 (VMOVSHZrr_REV VR128X:$dst, VR128X:$src1, VR128X:$src2), 0>;4421def : InstAlias<"vmovsh.s\t{$src2, $src1, $dst {${mask}}|"#4422 "$dst {${mask}}, $src1, $src2}",4423 (VMOVSHZrrk_REV VR128X:$dst, VK1WM:$mask,4424 VR128X:$src1, VR128X:$src2), 0>;4425def : InstAlias<"vmovsh.s\t{$src2, $src1, $dst {${mask}} {z}|"#4426 "$dst {${mask}} {z}, $src1, $src2}",4427 (VMOVSHZrrkz_REV VR128X:$dst, VK1WM:$mask,4428 VR128X:$src1, VR128X:$src2), 0>;4429def : InstAlias<"vmovss.s\t{$src2, $src1, $dst|$dst, $src1, $src2}",4430 (VMOVSSZrr_REV VR128X:$dst, VR128X:$src1, VR128X:$src2), 0>;4431def : InstAlias<"vmovss.s\t{$src2, $src1, $dst {${mask}}|"#4432 "$dst {${mask}}, $src1, $src2}",4433 (VMOVSSZrrk_REV VR128X:$dst, VK1WM:$mask,4434 VR128X:$src1, VR128X:$src2), 0>;4435def : InstAlias<"vmovss.s\t{$src2, $src1, $dst {${mask}} {z}|"#4436 "$dst {${mask}} {z}, $src1, $src2}",4437 (VMOVSSZrrkz_REV VR128X:$dst, VK1WM:$mask,4438 VR128X:$src1, VR128X:$src2), 0>;4439def : InstAlias<"vmovsd.s\t{$src2, $src1, $dst|$dst, $src1, $src2}",4440 (VMOVSDZrr_REV VR128X:$dst, VR128X:$src1, VR128X:$src2), 0>;4441def : InstAlias<"vmovsd.s\t{$src2, $src1, $dst {${mask}}|"#4442 "$dst {${mask}}, $src1, $src2}",4443 (VMOVSDZrrk_REV VR128X:$dst, VK1WM:$mask,4444 VR128X:$src1, VR128X:$src2), 0>;4445def : InstAlias<"vmovsd.s\t{$src2, $src1, $dst {${mask}} {z}|"#4446 "$dst {${mask}} {z}, $src1, $src2}",4447 (VMOVSDZrrkz_REV VR128X:$dst, VK1WM:$mask,4448 VR128X:$src1, VR128X:$src2), 0>;4449 4450let Predicates = [HasAVX512, OptForSize] in {4451 def : Pat<(v4f32 (X86vzmovl (v4f32 VR128X:$src))),4452 (VMOVSSZrr (v4f32 (AVX512_128_SET0)), VR128X:$src)>;4453 def : Pat<(v4i32 (X86vzmovl (v4i32 VR128X:$src))),4454 (VMOVSSZrr (v4i32 (AVX512_128_SET0)), VR128X:$src)>;4455 4456 // Move low f32 and clear high bits.4457 def : Pat<(v8f32 (X86vzmovl (v8f32 VR256X:$src))),4458 (SUBREG_TO_REG (i32 0),4459 (v4f32 (VMOVSSZrr (v4f32 (AVX512_128_SET0)),4460 (v4f32 (EXTRACT_SUBREG (v8f32 VR256X:$src), sub_xmm)))), sub_xmm)>;4461 def : Pat<(v8i32 (X86vzmovl (v8i32 VR256X:$src))),4462 (SUBREG_TO_REG (i32 0),4463 (v4i32 (VMOVSSZrr (v4i32 (AVX512_128_SET0)),4464 (v4i32 (EXTRACT_SUBREG (v8i32 VR256X:$src), sub_xmm)))), sub_xmm)>;4465 4466 def : Pat<(v16f32 (X86vzmovl (v16f32 VR512:$src))),4467 (SUBREG_TO_REG (i32 0),4468 (v4f32 (VMOVSSZrr (v4f32 (AVX512_128_SET0)),4469 (v4f32 (EXTRACT_SUBREG (v16f32 VR512:$src), sub_xmm)))), sub_xmm)>;4470 def : Pat<(v16i32 (X86vzmovl (v16i32 VR512:$src))),4471 (SUBREG_TO_REG (i32 0),4472 (v4i32 (VMOVSSZrr (v4i32 (AVX512_128_SET0)),4473 (v4i32 (EXTRACT_SUBREG (v16i32 VR512:$src), sub_xmm)))), sub_xmm)>;4474}4475 4476// Use 128-bit blends for OptForSpeed since BLENDs have better throughput than4477// VMOVSS/SD. Unfortunately, loses the ability to use XMM16-31.4478let Predicates = [HasAVX512, OptForSpeed] in {4479 def : Pat<(v16f32 (X86vzmovl (v16f32 VR512:$src))),4480 (SUBREG_TO_REG (i32 0),4481 (v4f32 (VBLENDPSrri (v4f32 (V_SET0)),4482 (v4f32 (EXTRACT_SUBREG (v16f32 VR512:$src), sub_xmm)),4483 (i8 1))), sub_xmm)>;4484 def : Pat<(v16i32 (X86vzmovl (v16i32 VR512:$src))),4485 (SUBREG_TO_REG (i32 0),4486 (v4i32 (VPBLENDWrri (v4i32 (V_SET0)),4487 (v4i32 (EXTRACT_SUBREG (v16i32 VR512:$src), sub_xmm)),4488 (i8 3))), sub_xmm)>;4489}4490 4491let Predicates = [HasAVX512] in {4492 def : Pat<(v4f32 (scalar_to_vector (loadf32 addr:$src))),4493 (VMOVSSZrm addr:$src)>;4494 def : Pat<(v2f64 (scalar_to_vector (loadf64 addr:$src))),4495 (VMOVSDZrm addr:$src)>;4496 4497 // Represent the same patterns above but in the form they appear for4498 // 256-bit types4499 def : Pat<(v8f32 (X86vzload32 addr:$src)),4500 (SUBREG_TO_REG (i32 0), (VMOVSSZrm addr:$src), sub_xmm)>;4501 def : Pat<(v4f64 (X86vzload64 addr:$src)),4502 (SUBREG_TO_REG (i32 0), (VMOVSDZrm addr:$src), sub_xmm)>;4503 4504 // Represent the same patterns above but in the form they appear for4505 // 512-bit types4506 def : Pat<(v16f32 (X86vzload32 addr:$src)),4507 (SUBREG_TO_REG (i32 0), (VMOVSSZrm addr:$src), sub_xmm)>;4508 def : Pat<(v8f64 (X86vzload64 addr:$src)),4509 (SUBREG_TO_REG (i32 0), (VMOVSDZrm addr:$src), sub_xmm)>;4510}4511let Predicates = [HasFP16] in {4512 def : Pat<(v8f16 (X86vzmovl (v8f16 VR128X:$src))),4513 (VMOVSHZrr (v8f16 (AVX512_128_SET0)), VR128X:$src)>;4514 def : Pat<(v8i16 (X86vzmovl (v8i16 VR128X:$src))),4515 (VMOVSHZrr (v8i16 (AVX512_128_SET0)), VR128X:$src)>;4516 4517 // FIXME we need better canonicalization in dag combine4518 def : Pat<(v16f16 (X86vzmovl (v16f16 VR256X:$src))),4519 (SUBREG_TO_REG (i32 0),4520 (v8f16 (VMOVSHZrr (v8f16 (AVX512_128_SET0)),4521 (v8f16 (EXTRACT_SUBREG (v16f16 VR256X:$src), sub_xmm)))), sub_xmm)>;4522 def : Pat<(v16i16 (X86vzmovl (v16i16 VR256X:$src))),4523 (SUBREG_TO_REG (i32 0),4524 (v8i16 (VMOVSHZrr (v8i16 (AVX512_128_SET0)),4525 (v8i16 (EXTRACT_SUBREG (v16i16 VR256X:$src), sub_xmm)))), sub_xmm)>;4526 4527 // FIXME we need better canonicalization in dag combine4528 def : Pat<(v32f16 (X86vzmovl (v32f16 VR512:$src))),4529 (SUBREG_TO_REG (i32 0),4530 (v8f16 (VMOVSHZrr (v8f16 (AVX512_128_SET0)),4531 (v8f16 (EXTRACT_SUBREG (v32f16 VR512:$src), sub_xmm)))), sub_xmm)>;4532 def : Pat<(v32i16 (X86vzmovl (v32i16 VR512:$src))),4533 (SUBREG_TO_REG (i32 0),4534 (v8i16 (VMOVSHZrr (v8i16 (AVX512_128_SET0)),4535 (v8i16 (EXTRACT_SUBREG (v32i16 VR512:$src), sub_xmm)))), sub_xmm)>;4536 4537 def : Pat<(v8f16 (X86vzload16 addr:$src)),4538 (VMOVSHZrm addr:$src)>;4539 4540 def : Pat<(v16f16 (X86vzload16 addr:$src)),4541 (SUBREG_TO_REG (i32 0), (VMOVSHZrm addr:$src), sub_xmm)>;4542 4543 def : Pat<(v32f16 (X86vzload16 addr:$src)),4544 (SUBREG_TO_REG (i32 0), (VMOVSHZrm addr:$src), sub_xmm)>;4545}4546 4547let ExeDomain = SSEPackedInt, SchedRW = [SchedWriteVecLogic.XMM] in {4548def VMOVZPQILo2PQIZrr : AVX512XSI<0x7E, MRMSrcReg, (outs VR128X:$dst),4549 (ins VR128X:$src),4550 "vmovq\t{$src, $dst|$dst, $src}",4551 [(set VR128X:$dst, (v2i64 (X86vzmovl4552 (v2i64 VR128X:$src))))]>,4553 EVEX, REX_W;4554}4555 4556let Predicates = [HasAVX512] in {4557 def : Pat<(v4i32 (scalar_to_vector (i32 (anyext GR8:$src)))),4558 (VMOVDI2PDIZrr (i32 (INSERT_SUBREG (i32 (IMPLICIT_DEF)),4559 GR8:$src, sub_8bit)))>;4560 def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector GR32:$src)))),4561 (VMOVDI2PDIZrr GR32:$src)>;4562 4563 def : Pat<(v2i64 (X86vzmovl (v2i64 (scalar_to_vector GR64:$src)))),4564 (VMOV64toPQIZrr GR64:$src)>;4565 4566 // AVX 128-bit movd/movq instruction write zeros in the high 128-bit part.4567 def : Pat<(v4i32 (X86vzload32 addr:$src)),4568 (VMOVDI2PDIZrm addr:$src)>;4569 def : Pat<(v8i32 (X86vzload32 addr:$src)),4570 (SUBREG_TO_REG (i32 0), (v4i32 (VMOVDI2PDIZrm addr:$src)), sub_xmm)>;4571 def : Pat<(v2f64 (X86vzmovl (v2f64 VR128X:$src))),4572 (VMOVZPQILo2PQIZrr VR128X:$src)>;4573 def : Pat<(v2i64 (X86vzload64 addr:$src)),4574 (VMOVQI2PQIZrm addr:$src)>;4575 def : Pat<(v4i64 (X86vzload64 addr:$src)),4576 (SUBREG_TO_REG (i64 0), (v2i64 (VMOVQI2PQIZrm addr:$src)), sub_xmm)>;4577 4578 // Use regular 128-bit instructions to match 512-bit scalar_to_vec+zext.4579 def : Pat<(v16i32 (X86vzload32 addr:$src)),4580 (SUBREG_TO_REG (i32 0), (v4i32 (VMOVDI2PDIZrm addr:$src)), sub_xmm)>;4581 def : Pat<(v8i64 (X86vzload64 addr:$src)),4582 (SUBREG_TO_REG (i64 0), (v2i64 (VMOVQI2PQIZrm addr:$src)), sub_xmm)>;4583 4584 def : Pat<(v4f64 (X86vzmovl (v4f64 VR256X:$src))),4585 (SUBREG_TO_REG (i32 0),4586 (v2f64 (VMOVZPQILo2PQIZrr4587 (v2f64 (EXTRACT_SUBREG (v4f64 VR256X:$src), sub_xmm)))),4588 sub_xmm)>;4589 def : Pat<(v4i64 (X86vzmovl (v4i64 VR256X:$src))),4590 (SUBREG_TO_REG (i32 0),4591 (v2i64 (VMOVZPQILo2PQIZrr4592 (v2i64 (EXTRACT_SUBREG (v4i64 VR256X:$src), sub_xmm)))),4593 sub_xmm)>;4594 4595 def : Pat<(v8f64 (X86vzmovl (v8f64 VR512:$src))),4596 (SUBREG_TO_REG (i32 0),4597 (v2f64 (VMOVZPQILo2PQIZrr4598 (v2f64 (EXTRACT_SUBREG (v8f64 VR512:$src), sub_xmm)))),4599 sub_xmm)>;4600 def : Pat<(v8i64 (X86vzmovl (v8i64 VR512:$src))),4601 (SUBREG_TO_REG (i32 0),4602 (v2i64 (VMOVZPQILo2PQIZrr4603 (v2i64 (EXTRACT_SUBREG (v8i64 VR512:$src), sub_xmm)))),4604 sub_xmm)>;4605}4606 4607//===----------------------------------------------------------------------===//4608// AVX-512 - Non-temporals4609//===----------------------------------------------------------------------===//4610 4611let mayLoad = 1, hasSideEffects = 0 in {4612def VMOVNTDQAZrm : AVX512PI<0x2A, MRMSrcMem, (outs VR512:$dst),4613 (ins i512mem:$src), "vmovntdqa\t{$src, $dst|$dst, $src}",4614 [], SSEPackedInt>, Sched<[SchedWriteVecMoveLSNT.ZMM.RM]>,4615 EVEX, T8, PD, EVEX_V512, EVEX_CD8<64, CD8VF>;4616 4617let Predicates = [HasVLX] in {4618 def VMOVNTDQAZ256rm : AVX512PI<0x2A, MRMSrcMem, (outs VR256X:$dst),4619 (ins i256mem:$src),4620 "vmovntdqa\t{$src, $dst|$dst, $src}",4621 [], SSEPackedInt>, Sched<[SchedWriteVecMoveLSNT.YMM.RM]>,4622 EVEX, T8, PD, EVEX_V256, EVEX_CD8<64, CD8VF>;4623 4624 def VMOVNTDQAZ128rm : AVX512PI<0x2A, MRMSrcMem, (outs VR128X:$dst),4625 (ins i128mem:$src),4626 "vmovntdqa\t{$src, $dst|$dst, $src}",4627 [], SSEPackedInt>, Sched<[SchedWriteVecMoveLSNT.XMM.RM]>,4628 EVEX, T8, PD, EVEX_V128, EVEX_CD8<64, CD8VF>;4629}4630}4631 4632multiclass avx512_movnt<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,4633 X86SchedWriteMoveLS Sched,4634 PatFrag st_frag = alignednontemporalstore> {4635 let mayStore = 1, SchedRW = [Sched.MR], AddedComplexity = 400 in4636 def mr : AVX512PI<opc, MRMDestMem, (outs), (ins _.MemOp:$dst, _.RC:$src),4637 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),4638 [(st_frag (_.VT _.RC:$src), addr:$dst)],4639 _.ExeDomain>, EVEX, EVEX_CD8<_.EltSize, CD8VF>;4640}4641 4642multiclass avx512_movnt_vl<bits<8> opc, string OpcodeStr,4643 AVX512VLVectorVTInfo VTInfo,4644 X86SchedWriteMoveLSWidths Sched> {4645 let Predicates = [HasAVX512] in4646 defm Z : avx512_movnt<opc, OpcodeStr, VTInfo.info512, Sched.ZMM>, EVEX_V512;4647 4648 let Predicates = [HasAVX512, HasVLX] in {4649 defm Z256 : avx512_movnt<opc, OpcodeStr, VTInfo.info256, Sched.YMM>, EVEX_V256;4650 defm Z128 : avx512_movnt<opc, OpcodeStr, VTInfo.info128, Sched.XMM>, EVEX_V128;4651 }4652}4653 4654defm VMOVNTDQ : avx512_movnt_vl<0xE7, "vmovntdq", avx512vl_i64_info,4655 SchedWriteVecMoveLSNT>, TB, PD;4656defm VMOVNTPD : avx512_movnt_vl<0x2B, "vmovntpd", avx512vl_f64_info,4657 SchedWriteFMoveLSNT>, TB, PD, REX_W;4658defm VMOVNTPS : avx512_movnt_vl<0x2B, "vmovntps", avx512vl_f32_info,4659 SchedWriteFMoveLSNT>, TB;4660 4661let Predicates = [HasAVX512], AddedComplexity = 400 in {4662 def : Pat<(alignednontemporalstore (v16i32 VR512:$src), addr:$dst),4663 (VMOVNTDQZmr addr:$dst, VR512:$src)>;4664 def : Pat<(alignednontemporalstore (v32i16 VR512:$src), addr:$dst),4665 (VMOVNTDQZmr addr:$dst, VR512:$src)>;4666 def : Pat<(alignednontemporalstore (v64i8 VR512:$src), addr:$dst),4667 (VMOVNTDQZmr addr:$dst, VR512:$src)>;4668 4669 def : Pat<(v8f64 (alignednontemporalload addr:$src)),4670 (VMOVNTDQAZrm addr:$src)>;4671 def : Pat<(v16f32 (alignednontemporalload addr:$src)),4672 (VMOVNTDQAZrm addr:$src)>;4673 def : Pat<(v8i64 (alignednontemporalload addr:$src)),4674 (VMOVNTDQAZrm addr:$src)>;4675 def : Pat<(v16i32 (alignednontemporalload addr:$src)),4676 (VMOVNTDQAZrm addr:$src)>;4677 def : Pat<(v32i16 (alignednontemporalload addr:$src)),4678 (VMOVNTDQAZrm addr:$src)>;4679 def : Pat<(v64i8 (alignednontemporalload addr:$src)),4680 (VMOVNTDQAZrm addr:$src)>;4681}4682 4683let Predicates = [HasVLX], AddedComplexity = 400 in {4684 def : Pat<(alignednontemporalstore (v8i32 VR256X:$src), addr:$dst),4685 (VMOVNTDQZ256mr addr:$dst, VR256X:$src)>;4686 def : Pat<(alignednontemporalstore (v16i16 VR256X:$src), addr:$dst),4687 (VMOVNTDQZ256mr addr:$dst, VR256X:$src)>;4688 def : Pat<(alignednontemporalstore (v32i8 VR256X:$src), addr:$dst),4689 (VMOVNTDQZ256mr addr:$dst, VR256X:$src)>;4690 4691 def : Pat<(v4f64 (alignednontemporalload addr:$src)),4692 (VMOVNTDQAZ256rm addr:$src)>;4693 def : Pat<(v8f32 (alignednontemporalload addr:$src)),4694 (VMOVNTDQAZ256rm addr:$src)>;4695 def : Pat<(v4i64 (alignednontemporalload addr:$src)),4696 (VMOVNTDQAZ256rm addr:$src)>;4697 def : Pat<(v8i32 (alignednontemporalload addr:$src)),4698 (VMOVNTDQAZ256rm addr:$src)>;4699 def : Pat<(v16i16 (alignednontemporalload addr:$src)),4700 (VMOVNTDQAZ256rm addr:$src)>;4701 def : Pat<(v32i8 (alignednontemporalload addr:$src)),4702 (VMOVNTDQAZ256rm addr:$src)>;4703 4704 def : Pat<(alignednontemporalstore (v4i32 VR128X:$src), addr:$dst),4705 (VMOVNTDQZ128mr addr:$dst, VR128X:$src)>;4706 def : Pat<(alignednontemporalstore (v8i16 VR128X:$src), addr:$dst),4707 (VMOVNTDQZ128mr addr:$dst, VR128X:$src)>;4708 def : Pat<(alignednontemporalstore (v16i8 VR128X:$src), addr:$dst),4709 (VMOVNTDQZ128mr addr:$dst, VR128X:$src)>;4710 4711 def : Pat<(v2f64 (alignednontemporalload addr:$src)),4712 (VMOVNTDQAZ128rm addr:$src)>;4713 def : Pat<(v4f32 (alignednontemporalload addr:$src)),4714 (VMOVNTDQAZ128rm addr:$src)>;4715 def : Pat<(v2i64 (alignednontemporalload addr:$src)),4716 (VMOVNTDQAZ128rm addr:$src)>;4717 def : Pat<(v4i32 (alignednontemporalload addr:$src)),4718 (VMOVNTDQAZ128rm addr:$src)>;4719 def : Pat<(v8i16 (alignednontemporalload addr:$src)),4720 (VMOVNTDQAZ128rm addr:$src)>;4721 def : Pat<(v16i8 (alignednontemporalload addr:$src)),4722 (VMOVNTDQAZ128rm addr:$src)>;4723}4724 4725//===----------------------------------------------------------------------===//4726// AVX-512 - Integer arithmetic4727//4728multiclass avx512_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,4729 X86VectorVTInfo _, X86FoldableSchedWrite sched,4730 bit IsCommutable = 0> {4731 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),4732 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,4733 "$src2, $src1", "$src1, $src2",4734 (_.VT (OpNode _.RC:$src1, _.RC:$src2)),4735 IsCommutable, IsCommutable>, AVX512BIBase, EVEX, VVVV,4736 Sched<[sched]>;4737 4738 let mayLoad = 1, hasSideEffects = 0 in4739 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),4740 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr,4741 "$src2, $src1", "$src1, $src2",4742 (_.VT (OpNode _.RC:$src1, (_.LdFrag addr:$src2)))>,4743 AVX512BIBase, EVEX, VVVV,4744 Sched<[sched.Folded, sched.ReadAfterFold]>;4745}4746 4747multiclass avx512_binop_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,4748 X86VectorVTInfo _, X86FoldableSchedWrite sched,4749 bit IsCommutable = 0> :4750 avx512_binop_rm<opc, OpcodeStr, OpNode, _, sched, IsCommutable> {4751 let mayLoad = 1, hasSideEffects = 0 in4752 defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),4753 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,4754 "${src2}"#_.BroadcastStr#", $src1",4755 "$src1, ${src2}"#_.BroadcastStr,4756 (_.VT (OpNode _.RC:$src1,4757 (_.BroadcastLdFrag addr:$src2)))>,4758 AVX512BIBase, EVEX, VVVV, EVEX_B,4759 Sched<[sched.Folded, sched.ReadAfterFold]>;4760}4761 4762multiclass avx512_binop_rm_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,4763 AVX512VLVectorVTInfo VTInfo,4764 X86SchedWriteWidths sched, Predicate prd,4765 bit IsCommutable = 0> {4766 let Predicates = [prd] in4767 defm Z : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info512, sched.ZMM,4768 IsCommutable>, EVEX_V512;4769 4770 let Predicates = [prd, HasVLX] in {4771 defm Z256 : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info256,4772 sched.YMM, IsCommutable>, EVEX_V256;4773 defm Z128 : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info128,4774 sched.XMM, IsCommutable>, EVEX_V128;4775 }4776}4777 4778multiclass avx512_binop_rmb_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,4779 AVX512VLVectorVTInfo VTInfo,4780 X86SchedWriteWidths sched, Predicate prd,4781 bit IsCommutable = 0> {4782 let Predicates = [prd] in4783 defm Z : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info512, sched.ZMM,4784 IsCommutable>, EVEX_V512;4785 4786 let Predicates = [prd, HasVLX] in {4787 defm Z256 : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info256,4788 sched.YMM, IsCommutable>, EVEX_V256;4789 defm Z128 : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info128,4790 sched.XMM, IsCommutable>, EVEX_V128;4791 }4792}4793 4794multiclass avx512_binop_rm_vl_q<bits<8> opc, string OpcodeStr, SDNode OpNode,4795 X86SchedWriteWidths sched, Predicate prd,4796 bit IsCommutable = 0> {4797 defm NAME : avx512_binop_rmb_vl<opc, OpcodeStr, OpNode, avx512vl_i64_info,4798 sched, prd, IsCommutable>,4799 REX_W, EVEX_CD8<64, CD8VF>;4800}4801 4802multiclass avx512_binop_rm_vl_d<bits<8> opc, string OpcodeStr, SDNode OpNode,4803 X86SchedWriteWidths sched, Predicate prd,4804 bit IsCommutable = 0> {4805 defm NAME : avx512_binop_rmb_vl<opc, OpcodeStr, OpNode, avx512vl_i32_info,4806 sched, prd, IsCommutable>, EVEX_CD8<32, CD8VF>;4807}4808 4809multiclass avx512_binop_rm_vl_w<bits<8> opc, string OpcodeStr, SDNode OpNode,4810 X86SchedWriteWidths sched, Predicate prd,4811 bit IsCommutable = 0> {4812 defm NAME : avx512_binop_rm_vl<opc, OpcodeStr, OpNode, avx512vl_i16_info,4813 sched, prd, IsCommutable>, EVEX_CD8<16, CD8VF>,4814 WIG;4815}4816 4817multiclass avx512_binop_rm_vl_b<bits<8> opc, string OpcodeStr, SDNode OpNode,4818 X86SchedWriteWidths sched, Predicate prd,4819 bit IsCommutable = 0> {4820 defm NAME : avx512_binop_rm_vl<opc, OpcodeStr, OpNode, avx512vl_i8_info,4821 sched, prd, IsCommutable>, EVEX_CD8<8, CD8VF>,4822 WIG;4823}4824 4825multiclass avx512_binop_rm_vl_dq<bits<8> opc_d, bits<8> opc_q, string OpcodeStr,4826 SDNode OpNode, X86SchedWriteWidths sched,4827 Predicate prd, bit IsCommutable = 0> {4828 defm Q : avx512_binop_rm_vl_q<opc_q, OpcodeStr#"q", OpNode, sched, prd,4829 IsCommutable>;4830 4831 defm D : avx512_binop_rm_vl_d<opc_d, OpcodeStr#"d", OpNode, sched, prd,4832 IsCommutable>;4833}4834 4835multiclass avx512_binop_rm_vl_bw<bits<8> opc_b, bits<8> opc_w, string OpcodeStr,4836 SDNode OpNode, X86SchedWriteWidths sched,4837 Predicate prd, bit IsCommutable = 0> {4838 defm W : avx512_binop_rm_vl_w<opc_w, OpcodeStr#"w", OpNode, sched, prd,4839 IsCommutable>;4840 4841 defm B : avx512_binop_rm_vl_b<opc_b, OpcodeStr#"b", OpNode, sched, prd,4842 IsCommutable>;4843}4844 4845multiclass avx512_binop_rm_vl_all<bits<8> opc_b, bits<8> opc_w,4846 bits<8> opc_d, bits<8> opc_q,4847 string OpcodeStr, SDNode OpNode,4848 X86SchedWriteWidths sched,4849 bit IsCommutable = 0> {4850 defm NAME : avx512_binop_rm_vl_dq<opc_d, opc_q, OpcodeStr, OpNode,4851 sched, HasAVX512, IsCommutable>,4852 avx512_binop_rm_vl_bw<opc_b, opc_w, OpcodeStr, OpNode,4853 sched, HasBWI, IsCommutable>;4854}4855 4856multiclass avx512_binop_rm2<bits<8> opc, string OpcodeStr,4857 X86FoldableSchedWrite sched,4858 SDNode OpNode,X86VectorVTInfo _Src,4859 X86VectorVTInfo _Dst, X86VectorVTInfo _Brdct,4860 bit IsCommutable = 0> {4861 defm rr : AVX512_maskable<opc, MRMSrcReg, _Dst, (outs _Dst.RC:$dst),4862 (ins _Src.RC:$src1, _Src.RC:$src2), OpcodeStr,4863 "$src2, $src1","$src1, $src2",4864 (_Dst.VT (OpNode4865 (_Src.VT _Src.RC:$src1),4866 (_Src.VT _Src.RC:$src2))),4867 IsCommutable>,4868 AVX512BIBase, EVEX, VVVV, Sched<[sched]>;4869 let mayLoad = 1, hasSideEffects = 0 in {4870 defm rm : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),4871 (ins _Src.RC:$src1, _Src.MemOp:$src2), OpcodeStr,4872 "$src2, $src1", "$src1, $src2",4873 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1),4874 (_Src.LdFrag addr:$src2)))>,4875 AVX512BIBase, EVEX, VVVV,4876 Sched<[sched.Folded, sched.ReadAfterFold]>;4877 4878 defm rmb : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),4879 (ins _Src.RC:$src1, _Brdct.ScalarMemOp:$src2),4880 OpcodeStr,4881 "${src2}"#_Brdct.BroadcastStr#", $src1",4882 "$src1, ${src2}"#_Brdct.BroadcastStr,4883 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), (bitconvert4884 (_Brdct.VT (_Brdct.BroadcastLdFrag addr:$src2)))))>,4885 AVX512BIBase, EVEX, VVVV, EVEX_B,4886 Sched<[sched.Folded, sched.ReadAfterFold]>;4887 }4888}4889 4890defm VPADD : avx512_binop_rm_vl_all<0xFC, 0xFD, 0xFE, 0xD4, "vpadd", add,4891 SchedWriteVecALU, 1>;4892defm VPSUB : avx512_binop_rm_vl_all<0xF8, 0xF9, 0xFA, 0xFB, "vpsub", sub,4893 SchedWriteVecALU, 0>;4894defm VPADDS : avx512_binop_rm_vl_bw<0xEC, 0xED, "vpadds", saddsat,4895 SchedWriteVecALU, HasBWI, 1>;4896defm VPSUBS : avx512_binop_rm_vl_bw<0xE8, 0xE9, "vpsubs", ssubsat,4897 SchedWriteVecALU, HasBWI, 0>;4898defm VPADDUS : avx512_binop_rm_vl_bw<0xDC, 0xDD, "vpaddus", uaddsat,4899 SchedWriteVecALU, HasBWI, 1>;4900defm VPSUBUS : avx512_binop_rm_vl_bw<0xD8, 0xD9, "vpsubus", usubsat,4901 SchedWriteVecALU, HasBWI, 0>;4902defm VPMULLD : avx512_binop_rm_vl_d<0x40, "vpmulld", mul,4903 SchedWritePMULLD, HasAVX512, 1>, T8;4904defm VPMULLW : avx512_binop_rm_vl_w<0xD5, "vpmullw", mul,4905 SchedWriteVecIMul, HasBWI, 1>;4906defm VPMULLQ : avx512_binop_rm_vl_q<0x40, "vpmullq", mul,4907 SchedWriteVecIMul, HasDQI, 1>, T8;4908defm VPMULHW : avx512_binop_rm_vl_w<0xE5, "vpmulhw", mulhs, SchedWriteVecIMul,4909 HasBWI, 1>;4910defm VPMULHUW : avx512_binop_rm_vl_w<0xE4, "vpmulhuw", mulhu, SchedWriteVecIMul,4911 HasBWI, 1>;4912defm VPMULHRSW : avx512_binop_rm_vl_w<0x0B, "vpmulhrsw", X86mulhrs,4913 SchedWriteVecIMul, HasBWI, 1>, T8;4914defm VPAVG : avx512_binop_rm_vl_bw<0xE0, 0xE3, "vpavg", avgceilu,4915 SchedWriteVecALU, HasBWI, 1>;4916defm VPMULDQ : avx512_binop_rm_vl_q<0x28, "vpmuldq", X86pmuldq,4917 SchedWriteVecIMul, HasAVX512, 1>, T8;4918defm VPMULUDQ : avx512_binop_rm_vl_q<0xF4, "vpmuludq", X86pmuludq,4919 SchedWriteVecIMul, HasAVX512, 1>;4920 4921multiclass avx512_binop_all<bits<8> opc, string OpcodeStr,4922 X86SchedWriteWidths sched,4923 AVX512VLVectorVTInfo _SrcVTInfo,4924 AVX512VLVectorVTInfo _DstVTInfo,4925 SDNode OpNode, list<Predicate> prds512,4926 list<Predicate> prds,4927 X86VectorVTInfo _VTInfo512 = _SrcVTInfo.info512,4928 X86VectorVTInfo _VTInfo256 = _SrcVTInfo.info256,4929 X86VectorVTInfo _VTInfo128 = _SrcVTInfo.info128> {4930 let Predicates = prds512 in4931 defm NAME#Z : avx512_binop_rm2<opc, OpcodeStr, sched.ZMM, OpNode,4932 _SrcVTInfo.info512, _DstVTInfo.info512,4933 _VTInfo512>, EVEX_V512;4934 let Predicates = prds in {4935 defm NAME#Z256 : avx512_binop_rm2<opc, OpcodeStr, sched.YMM, OpNode,4936 _SrcVTInfo.info256, _DstVTInfo.info256,4937 _VTInfo256>, EVEX_V256;4938 defm NAME#Z128 : avx512_binop_rm2<opc, OpcodeStr, sched.XMM, OpNode,4939 _SrcVTInfo.info128, _DstVTInfo.info128,4940 _VTInfo128>, EVEX_V128;4941 }4942}4943 4944defm VPMULTISHIFTQB : avx512_binop_all<0x83, "vpmultishiftqb", SchedWriteVecALU,4945 avx512vl_i8_info, avx512vl_i8_info,4946 X86multishift, [HasVBMI], [HasVLX, HasVBMI],4947 v8i64_info, v4i64x_info, v2i64x_info>, T8,4948 EVEX_CD8<64, CD8VF>, REX_W;4949 4950multiclass avx512_packs_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,4951 X86VectorVTInfo _Src, X86VectorVTInfo _Dst,4952 X86FoldableSchedWrite sched> {4953 let mayLoad = 1, hasSideEffects = 0 in4954 defm rmb : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),4955 (ins _Src.RC:$src1, _Src.ScalarMemOp:$src2),4956 OpcodeStr,4957 "${src2}"#_Src.BroadcastStr#", $src1",4958 "$src1, ${src2}"#_Src.BroadcastStr,4959 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), (bitconvert4960 (_Src.VT (_Src.BroadcastLdFrag addr:$src2)))))>,4961 EVEX, VVVV, EVEX_B, EVEX_CD8<_Src.EltSize, CD8VF>,4962 Sched<[sched.Folded, sched.ReadAfterFold]>;4963}4964 4965multiclass avx512_packs_rm<bits<8> opc, string OpcodeStr,4966 SDNode OpNode,X86VectorVTInfo _Src,4967 X86VectorVTInfo _Dst, X86FoldableSchedWrite sched,4968 bit IsCommutable = 0> {4969 defm rr : AVX512_maskable<opc, MRMSrcReg, _Dst, (outs _Dst.RC:$dst),4970 (ins _Src.RC:$src1, _Src.RC:$src2), OpcodeStr,4971 "$src2, $src1","$src1, $src2",4972 (_Dst.VT (OpNode4973 (_Src.VT _Src.RC:$src1),4974 (_Src.VT _Src.RC:$src2))),4975 IsCommutable, IsCommutable>,4976 EVEX_CD8<_Src.EltSize, CD8VF>, EVEX, VVVV, Sched<[sched]>;4977 let mayLoad = 1, hasSideEffects = 0 in4978 defm rm : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),4979 (ins _Src.RC:$src1, _Src.MemOp:$src2), OpcodeStr,4980 "$src2, $src1", "$src1, $src2",4981 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1),4982 (_Src.LdFrag addr:$src2)))>,4983 EVEX, VVVV, EVEX_CD8<_Src.EltSize, CD8VF>,4984 Sched<[sched.Folded, sched.ReadAfterFold]>;4985}4986 4987multiclass avx512_packs_all_i32_i16<bits<8> opc, string OpcodeStr,4988 SDNode OpNode> {4989 let Predicates = [HasBWI] in4990 defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, v16i32_info,4991 v32i16_info, SchedWriteShuffle.ZMM>,4992 avx512_packs_rmb<opc, OpcodeStr, OpNode, v16i32_info,4993 v32i16_info, SchedWriteShuffle.ZMM>, EVEX_V512;4994 let Predicates = [HasBWI, HasVLX] in {4995 defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, v8i32x_info,4996 v16i16x_info, SchedWriteShuffle.YMM>,4997 avx512_packs_rmb<opc, OpcodeStr, OpNode, v8i32x_info,4998 v16i16x_info, SchedWriteShuffle.YMM>,4999 EVEX_V256;5000 defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, v4i32x_info,5001 v8i16x_info, SchedWriteShuffle.XMM>,5002 avx512_packs_rmb<opc, OpcodeStr, OpNode, v4i32x_info,5003 v8i16x_info, SchedWriteShuffle.XMM>,5004 EVEX_V128;5005 }5006}5007multiclass avx512_packs_all_i16_i8<bits<8> opc, string OpcodeStr,5008 SDNode OpNode> {5009 let Predicates = [HasBWI] in5010 defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, v32i16_info, v64i8_info,5011 SchedWriteShuffle.ZMM>, EVEX_V512, WIG;5012 let Predicates = [HasBWI, HasVLX] in {5013 defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, v16i16x_info,5014 v32i8x_info, SchedWriteShuffle.YMM>,5015 EVEX_V256, WIG;5016 defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, v8i16x_info,5017 v16i8x_info, SchedWriteShuffle.XMM>,5018 EVEX_V128, WIG;5019 }5020}5021 5022multiclass avx512_vpmadd<bits<8> opc, string OpcodeStr,5023 SDNode OpNode, AVX512VLVectorVTInfo _Src,5024 AVX512VLVectorVTInfo _Dst, bit IsCommutable = 0> {5025 let Predicates = [HasBWI] in5026 defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info512,5027 _Dst.info512, SchedWriteVecIMul.ZMM,5028 IsCommutable>, EVEX_V512;5029 let Predicates = [HasBWI, HasVLX] in {5030 defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info256,5031 _Dst.info256, SchedWriteVecIMul.YMM,5032 IsCommutable>, EVEX_V256;5033 defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info128,5034 _Dst.info128, SchedWriteVecIMul.XMM,5035 IsCommutable>, EVEX_V128;5036 }5037}5038 5039defm VPACKSSDW : avx512_packs_all_i32_i16<0x6B, "vpackssdw", X86Packss>, AVX512BIBase;5040defm VPACKUSDW : avx512_packs_all_i32_i16<0x2b, "vpackusdw", X86Packus>, AVX5128IBase;5041defm VPACKSSWB : avx512_packs_all_i16_i8 <0x63, "vpacksswb", X86Packss>, AVX512BIBase;5042defm VPACKUSWB : avx512_packs_all_i16_i8 <0x67, "vpackuswb", X86Packus>, AVX512BIBase;5043 5044defm VPMADDUBSW : avx512_vpmadd<0x04, "vpmaddubsw", X86vpmaddubsw,5045 avx512vl_i8_info, avx512vl_i16_info>, AVX512BIBase, T8, WIG;5046defm VPMADDWD : avx512_vpmadd<0xF5, "vpmaddwd", X86vpmaddwd,5047 avx512vl_i16_info, avx512vl_i32_info, 1>, AVX512BIBase, WIG;5048 5049defm VPMAXSB : avx512_binop_rm_vl_b<0x3C, "vpmaxsb", smax,5050 SchedWriteVecALU, HasBWI, 1>, T8;5051defm VPMAXSW : avx512_binop_rm_vl_w<0xEE, "vpmaxsw", smax,5052 SchedWriteVecALU, HasBWI, 1>;5053defm VPMAXSD : avx512_binop_rm_vl_d<0x3D, "vpmaxsd", smax,5054 SchedWriteVecALU, HasAVX512, 1>, T8;5055defm VPMAXSQ : avx512_binop_rm_vl_q<0x3D, "vpmaxsq", smax,5056 SchedWriteVecALU, HasAVX512, 1>, T8;5057 5058defm VPMAXUB : avx512_binop_rm_vl_b<0xDE, "vpmaxub", umax,5059 SchedWriteVecALU, HasBWI, 1>;5060defm VPMAXUW : avx512_binop_rm_vl_w<0x3E, "vpmaxuw", umax,5061 SchedWriteVecALU, HasBWI, 1>, T8;5062defm VPMAXUD : avx512_binop_rm_vl_d<0x3F, "vpmaxud", umax,5063 SchedWriteVecALU, HasAVX512, 1>, T8;5064defm VPMAXUQ : avx512_binop_rm_vl_q<0x3F, "vpmaxuq", umax,5065 SchedWriteVecALU, HasAVX512, 1>, T8;5066 5067defm VPMINSB : avx512_binop_rm_vl_b<0x38, "vpminsb", smin,5068 SchedWriteVecALU, HasBWI, 1>, T8;5069defm VPMINSW : avx512_binop_rm_vl_w<0xEA, "vpminsw", smin,5070 SchedWriteVecALU, HasBWI, 1>;5071defm VPMINSD : avx512_binop_rm_vl_d<0x39, "vpminsd", smin,5072 SchedWriteVecALU, HasAVX512, 1>, T8;5073defm VPMINSQ : avx512_binop_rm_vl_q<0x39, "vpminsq", smin,5074 SchedWriteVecALU, HasAVX512, 1>, T8;5075 5076defm VPMINUB : avx512_binop_rm_vl_b<0xDA, "vpminub", umin,5077 SchedWriteVecALU, HasBWI, 1>;5078defm VPMINUW : avx512_binop_rm_vl_w<0x3A, "vpminuw", umin,5079 SchedWriteVecALU, HasBWI, 1>, T8;5080defm VPMINUD : avx512_binop_rm_vl_d<0x3B, "vpminud", umin,5081 SchedWriteVecALU, HasAVX512, 1>, T8;5082defm VPMINUQ : avx512_binop_rm_vl_q<0x3B, "vpminuq", umin,5083 SchedWriteVecALU, HasAVX512, 1>, T8;5084 5085// PMULLQ: Use 512bit version to implement 128/256 bit in case NoVLX.5086let Predicates = [HasDQI, NoVLX] in {5087 def : Pat<(v4i64 (mul (v4i64 VR256X:$src1), (v4i64 VR256X:$src2))),5088 (EXTRACT_SUBREG5089 (VPMULLQZrr5090 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src1, sub_ymm),5091 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src2, sub_ymm)),5092 sub_ymm)>;5093 def : Pat<(v4i64 (mul (v4i64 VR256X:$src1), (v4i64 (X86VBroadcastld64 addr:$src2)))),5094 (EXTRACT_SUBREG5095 (VPMULLQZrmb5096 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src1, sub_ymm),5097 addr:$src2),5098 sub_ymm)>;5099 5100 def : Pat<(v2i64 (mul (v2i64 VR128X:$src1), (v2i64 VR128X:$src2))),5101 (EXTRACT_SUBREG5102 (VPMULLQZrr5103 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src1, sub_xmm),5104 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src2, sub_xmm)),5105 sub_xmm)>;5106 def : Pat<(v2i64 (mul (v2i64 VR128X:$src1), (v2i64 (X86VBroadcastld64 addr:$src2)))),5107 (EXTRACT_SUBREG5108 (VPMULLQZrmb5109 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src1, sub_xmm),5110 addr:$src2),5111 sub_xmm)>;5112}5113 5114multiclass avx512_min_max_lowering<string Instr, SDNode OpNode> {5115 def : Pat<(v4i64 (OpNode VR256X:$src1, VR256X:$src2)),5116 (EXTRACT_SUBREG5117 (!cast<Instruction>(Instr#"rr")5118 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src1, sub_ymm),5119 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src2, sub_ymm)),5120 sub_ymm)>;5121 def : Pat<(v4i64 (OpNode (v4i64 VR256X:$src1), (v4i64 (X86VBroadcastld64 addr:$src2)))),5122 (EXTRACT_SUBREG5123 (!cast<Instruction>(Instr#"rmb")5124 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src1, sub_ymm),5125 addr:$src2),5126 sub_ymm)>;5127 5128 def : Pat<(v2i64 (OpNode VR128X:$src1, VR128X:$src2)),5129 (EXTRACT_SUBREG5130 (!cast<Instruction>(Instr#"rr")5131 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src1, sub_xmm),5132 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src2, sub_xmm)),5133 sub_xmm)>;5134 def : Pat<(v2i64 (OpNode (v2i64 VR128X:$src1), (v2i64 (X86VBroadcastld64 addr:$src2)))),5135 (EXTRACT_SUBREG5136 (!cast<Instruction>(Instr#"rmb")5137 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src1, sub_xmm),5138 addr:$src2),5139 sub_xmm)>;5140}5141 5142let Predicates = [HasAVX512, NoVLX] in {5143 defm : avx512_min_max_lowering<"VPMAXUQZ", umax>;5144 defm : avx512_min_max_lowering<"VPMINUQZ", umin>;5145 defm : avx512_min_max_lowering<"VPMAXSQZ", smax>;5146 defm : avx512_min_max_lowering<"VPMINSQZ", smin>;5147}5148 5149//===----------------------------------------------------------------------===//5150// AVX-512 Logical Instructions5151//===----------------------------------------------------------------------===//5152 5153defm VPAND : avx512_binop_rm_vl_dq<0xDB, 0xDB, "vpand", and,5154 SchedWriteVecLogic, HasAVX512, 1>;5155defm VPOR : avx512_binop_rm_vl_dq<0xEB, 0xEB, "vpor", or,5156 SchedWriteVecLogic, HasAVX512, 1>;5157defm VPXOR : avx512_binop_rm_vl_dq<0xEF, 0xEF, "vpxor", xor,5158 SchedWriteVecLogic, HasAVX512, 1>;5159defm VPANDN : avx512_binop_rm_vl_dq<0xDF, 0xDF, "vpandn", X86andnp,5160 SchedWriteVecLogic, HasAVX512>;5161 5162let Predicates = [HasVLX] in {5163 def : Pat<(v16i8 (and VR128X:$src1, VR128X:$src2)),5164 (VPANDQZ128rr VR128X:$src1, VR128X:$src2)>;5165 def : Pat<(v8i16 (and VR128X:$src1, VR128X:$src2)),5166 (VPANDQZ128rr VR128X:$src1, VR128X:$src2)>;5167 5168 def : Pat<(v16i8 (or VR128X:$src1, VR128X:$src2)),5169 (VPORQZ128rr VR128X:$src1, VR128X:$src2)>;5170 def : Pat<(v8i16 (or VR128X:$src1, VR128X:$src2)),5171 (VPORQZ128rr VR128X:$src1, VR128X:$src2)>;5172 5173 def : Pat<(v16i8 (xor VR128X:$src1, VR128X:$src2)),5174 (VPXORQZ128rr VR128X:$src1, VR128X:$src2)>;5175 def : Pat<(v8i16 (xor VR128X:$src1, VR128X:$src2)),5176 (VPXORQZ128rr VR128X:$src1, VR128X:$src2)>;5177 5178 def : Pat<(v16i8 (X86andnp VR128X:$src1, VR128X:$src2)),5179 (VPANDNQZ128rr VR128X:$src1, VR128X:$src2)>;5180 def : Pat<(v8i16 (X86andnp VR128X:$src1, VR128X:$src2)),5181 (VPANDNQZ128rr VR128X:$src1, VR128X:$src2)>;5182 5183 def : Pat<(and VR128X:$src1, (loadv16i8 addr:$src2)),5184 (VPANDQZ128rm VR128X:$src1, addr:$src2)>;5185 def : Pat<(and VR128X:$src1, (loadv8i16 addr:$src2)),5186 (VPANDQZ128rm VR128X:$src1, addr:$src2)>;5187 5188 def : Pat<(or VR128X:$src1, (loadv16i8 addr:$src2)),5189 (VPORQZ128rm VR128X:$src1, addr:$src2)>;5190 def : Pat<(or VR128X:$src1, (loadv8i16 addr:$src2)),5191 (VPORQZ128rm VR128X:$src1, addr:$src2)>;5192 5193 def : Pat<(xor VR128X:$src1, (loadv16i8 addr:$src2)),5194 (VPXORQZ128rm VR128X:$src1, addr:$src2)>;5195 def : Pat<(xor VR128X:$src1, (loadv8i16 addr:$src2)),5196 (VPXORQZ128rm VR128X:$src1, addr:$src2)>;5197 5198 def : Pat<(X86andnp VR128X:$src1, (loadv16i8 addr:$src2)),5199 (VPANDNQZ128rm VR128X:$src1, addr:$src2)>;5200 def : Pat<(X86andnp VR128X:$src1, (loadv8i16 addr:$src2)),5201 (VPANDNQZ128rm VR128X:$src1, addr:$src2)>;5202 5203 def : Pat<(v32i8 (and VR256X:$src1, VR256X:$src2)),5204 (VPANDQZ256rr VR256X:$src1, VR256X:$src2)>;5205 def : Pat<(v16i16 (and VR256X:$src1, VR256X:$src2)),5206 (VPANDQZ256rr VR256X:$src1, VR256X:$src2)>;5207 5208 def : Pat<(v32i8 (or VR256X:$src1, VR256X:$src2)),5209 (VPORQZ256rr VR256X:$src1, VR256X:$src2)>;5210 def : Pat<(v16i16 (or VR256X:$src1, VR256X:$src2)),5211 (VPORQZ256rr VR256X:$src1, VR256X:$src2)>;5212 5213 def : Pat<(v32i8 (xor VR256X:$src1, VR256X:$src2)),5214 (VPXORQZ256rr VR256X:$src1, VR256X:$src2)>;5215 def : Pat<(v16i16 (xor VR256X:$src1, VR256X:$src2)),5216 (VPXORQZ256rr VR256X:$src1, VR256X:$src2)>;5217 5218 def : Pat<(v32i8 (X86andnp VR256X:$src1, VR256X:$src2)),5219 (VPANDNQZ256rr VR256X:$src1, VR256X:$src2)>;5220 def : Pat<(v16i16 (X86andnp VR256X:$src1, VR256X:$src2)),5221 (VPANDNQZ256rr VR256X:$src1, VR256X:$src2)>;5222 5223 def : Pat<(and VR256X:$src1, (loadv32i8 addr:$src2)),5224 (VPANDQZ256rm VR256X:$src1, addr:$src2)>;5225 def : Pat<(and VR256X:$src1, (loadv16i16 addr:$src2)),5226 (VPANDQZ256rm VR256X:$src1, addr:$src2)>;5227 5228 def : Pat<(or VR256X:$src1, (loadv32i8 addr:$src2)),5229 (VPORQZ256rm VR256X:$src1, addr:$src2)>;5230 def : Pat<(or VR256X:$src1, (loadv16i16 addr:$src2)),5231 (VPORQZ256rm VR256X:$src1, addr:$src2)>;5232 5233 def : Pat<(xor VR256X:$src1, (loadv32i8 addr:$src2)),5234 (VPXORQZ256rm VR256X:$src1, addr:$src2)>;5235 def : Pat<(xor VR256X:$src1, (loadv16i16 addr:$src2)),5236 (VPXORQZ256rm VR256X:$src1, addr:$src2)>;5237 5238 def : Pat<(X86andnp VR256X:$src1, (loadv32i8 addr:$src2)),5239 (VPANDNQZ256rm VR256X:$src1, addr:$src2)>;5240 def : Pat<(X86andnp VR256X:$src1, (loadv16i16 addr:$src2)),5241 (VPANDNQZ256rm VR256X:$src1, addr:$src2)>;5242}5243 5244let Predicates = [HasAVX512] in {5245 def : Pat<(v64i8 (and VR512:$src1, VR512:$src2)),5246 (VPANDQZrr VR512:$src1, VR512:$src2)>;5247 def : Pat<(v32i16 (and VR512:$src1, VR512:$src2)),5248 (VPANDQZrr VR512:$src1, VR512:$src2)>;5249 5250 def : Pat<(v64i8 (or VR512:$src1, VR512:$src2)),5251 (VPORQZrr VR512:$src1, VR512:$src2)>;5252 def : Pat<(v32i16 (or VR512:$src1, VR512:$src2)),5253 (VPORQZrr VR512:$src1, VR512:$src2)>;5254 5255 def : Pat<(v64i8 (xor VR512:$src1, VR512:$src2)),5256 (VPXORQZrr VR512:$src1, VR512:$src2)>;5257 def : Pat<(v32i16 (xor VR512:$src1, VR512:$src2)),5258 (VPXORQZrr VR512:$src1, VR512:$src2)>;5259 5260 def : Pat<(v64i8 (X86andnp VR512:$src1, VR512:$src2)),5261 (VPANDNQZrr VR512:$src1, VR512:$src2)>;5262 def : Pat<(v32i16 (X86andnp VR512:$src1, VR512:$src2)),5263 (VPANDNQZrr VR512:$src1, VR512:$src2)>;5264 5265 def : Pat<(and VR512:$src1, (loadv64i8 addr:$src2)),5266 (VPANDQZrm VR512:$src1, addr:$src2)>;5267 def : Pat<(and VR512:$src1, (loadv32i16 addr:$src2)),5268 (VPANDQZrm VR512:$src1, addr:$src2)>;5269 5270 def : Pat<(or VR512:$src1, (loadv64i8 addr:$src2)),5271 (VPORQZrm VR512:$src1, addr:$src2)>;5272 def : Pat<(or VR512:$src1, (loadv32i16 addr:$src2)),5273 (VPORQZrm VR512:$src1, addr:$src2)>;5274 5275 def : Pat<(xor VR512:$src1, (loadv64i8 addr:$src2)),5276 (VPXORQZrm VR512:$src1, addr:$src2)>;5277 def : Pat<(xor VR512:$src1, (loadv32i16 addr:$src2)),5278 (VPXORQZrm VR512:$src1, addr:$src2)>;5279 5280 def : Pat<(X86andnp VR512:$src1, (loadv64i8 addr:$src2)),5281 (VPANDNQZrm VR512:$src1, addr:$src2)>;5282 def : Pat<(X86andnp VR512:$src1, (loadv32i16 addr:$src2)),5283 (VPANDNQZrm VR512:$src1, addr:$src2)>;5284}5285 5286// Patterns to catch vselect with different type than logic op.5287multiclass avx512_logical_lowering<string InstrStr, SDNode OpNode,5288 X86VectorVTInfo _,5289 X86VectorVTInfo IntInfo> {5290 // Masked register-register logical operations.5291 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,5292 (bitconvert (IntInfo.VT (OpNode _.RC:$src1, _.RC:$src2))),5293 _.RC:$src0)),5294 (!cast<Instruction>(InstrStr#rrk) _.RC:$src0, _.KRCWM:$mask,5295 _.RC:$src1, _.RC:$src2)>;5296 5297 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,5298 (bitconvert (IntInfo.VT (OpNode _.RC:$src1, _.RC:$src2))),5299 _.ImmAllZerosV)),5300 (!cast<Instruction>(InstrStr#rrkz) _.KRCWM:$mask, _.RC:$src1,5301 _.RC:$src2)>;5302 5303 // Masked register-memory logical operations.5304 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,5305 (bitconvert (IntInfo.VT (OpNode _.RC:$src1,5306 (load addr:$src2)))),5307 _.RC:$src0)),5308 (!cast<Instruction>(InstrStr#rmk) _.RC:$src0, _.KRCWM:$mask,5309 _.RC:$src1, addr:$src2)>;5310 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,5311 (bitconvert (IntInfo.VT (OpNode _.RC:$src1,5312 (load addr:$src2)))),5313 _.ImmAllZerosV)),5314 (!cast<Instruction>(InstrStr#rmkz) _.KRCWM:$mask, _.RC:$src1,5315 addr:$src2)>;5316}5317 5318multiclass avx512_logical_lowering_bcast<string InstrStr, SDNode OpNode,5319 X86VectorVTInfo _,5320 X86VectorVTInfo IntInfo> {5321 // Register-broadcast logical operations.5322 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,5323 (bitconvert5324 (IntInfo.VT (OpNode _.RC:$src1,5325 (IntInfo.VT (IntInfo.BroadcastLdFrag addr:$src2))))),5326 _.RC:$src0)),5327 (!cast<Instruction>(InstrStr#rmbk) _.RC:$src0, _.KRCWM:$mask,5328 _.RC:$src1, addr:$src2)>;5329 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,5330 (bitconvert5331 (IntInfo.VT (OpNode _.RC:$src1,5332 (IntInfo.VT (IntInfo.BroadcastLdFrag addr:$src2))))),5333 _.ImmAllZerosV)),5334 (!cast<Instruction>(InstrStr#rmbkz) _.KRCWM:$mask,5335 _.RC:$src1, addr:$src2)>;5336}5337 5338multiclass avx512_logical_lowering_sizes<string InstrStr, SDNode OpNode,5339 AVX512VLVectorVTInfo SelectInfo,5340 AVX512VLVectorVTInfo IntInfo> {5341let Predicates = [HasVLX] in {5342 defm : avx512_logical_lowering<InstrStr#"Z128", OpNode, SelectInfo.info128,5343 IntInfo.info128>;5344 defm : avx512_logical_lowering<InstrStr#"Z256", OpNode, SelectInfo.info256,5345 IntInfo.info256>;5346}5347let Predicates = [HasAVX512] in {5348 defm : avx512_logical_lowering<InstrStr#"Z", OpNode, SelectInfo.info512,5349 IntInfo.info512>;5350}5351}5352 5353multiclass avx512_logical_lowering_sizes_bcast<string InstrStr, SDNode OpNode,5354 AVX512VLVectorVTInfo SelectInfo,5355 AVX512VLVectorVTInfo IntInfo> {5356let Predicates = [HasVLX] in {5357 defm : avx512_logical_lowering_bcast<InstrStr#"Z128", OpNode,5358 SelectInfo.info128, IntInfo.info128>;5359 defm : avx512_logical_lowering_bcast<InstrStr#"Z256", OpNode,5360 SelectInfo.info256, IntInfo.info256>;5361}5362let Predicates = [HasAVX512] in {5363 defm : avx512_logical_lowering_bcast<InstrStr#"Z", OpNode,5364 SelectInfo.info512, IntInfo.info512>;5365}5366}5367 5368multiclass avx512_logical_lowering_types<string InstrStr, SDNode OpNode> {5369 // i64 vselect with i32/i16/i8 logic op5370 defm : avx512_logical_lowering_sizes<InstrStr#"Q", OpNode, avx512vl_i64_info,5371 avx512vl_i32_info>;5372 defm : avx512_logical_lowering_sizes<InstrStr#"Q", OpNode, avx512vl_i64_info,5373 avx512vl_i16_info>;5374 defm : avx512_logical_lowering_sizes<InstrStr#"Q", OpNode, avx512vl_i64_info,5375 avx512vl_i8_info>;5376 5377 // i32 vselect with i64/i16/i8 logic op5378 defm : avx512_logical_lowering_sizes<InstrStr#"D", OpNode, avx512vl_i32_info,5379 avx512vl_i64_info>;5380 defm : avx512_logical_lowering_sizes<InstrStr#"D", OpNode, avx512vl_i32_info,5381 avx512vl_i16_info>;5382 defm : avx512_logical_lowering_sizes<InstrStr#"D", OpNode, avx512vl_i32_info,5383 avx512vl_i8_info>;5384 5385 // f32 vselect with i64/i32/i16/i8 logic op5386 defm : avx512_logical_lowering_sizes<InstrStr#"D", OpNode, avx512vl_f32_info,5387 avx512vl_i64_info>;5388 defm : avx512_logical_lowering_sizes<InstrStr#"D", OpNode, avx512vl_f32_info,5389 avx512vl_i32_info>;5390 defm : avx512_logical_lowering_sizes<InstrStr#"D", OpNode, avx512vl_f32_info,5391 avx512vl_i16_info>;5392 defm : avx512_logical_lowering_sizes<InstrStr#"D", OpNode, avx512vl_f32_info,5393 avx512vl_i8_info>;5394 5395 // f64 vselect with i64/i32/i16/i8 logic op5396 defm : avx512_logical_lowering_sizes<InstrStr#"Q", OpNode, avx512vl_f64_info,5397 avx512vl_i64_info>;5398 defm : avx512_logical_lowering_sizes<InstrStr#"Q", OpNode, avx512vl_f64_info,5399 avx512vl_i32_info>;5400 defm : avx512_logical_lowering_sizes<InstrStr#"Q", OpNode, avx512vl_f64_info,5401 avx512vl_i16_info>;5402 defm : avx512_logical_lowering_sizes<InstrStr#"Q", OpNode, avx512vl_f64_info,5403 avx512vl_i8_info>;5404 5405 defm : avx512_logical_lowering_sizes_bcast<InstrStr#"D", OpNode,5406 avx512vl_f32_info,5407 avx512vl_i32_info>;5408 defm : avx512_logical_lowering_sizes_bcast<InstrStr#"Q", OpNode,5409 avx512vl_f64_info,5410 avx512vl_i64_info>;5411}5412 5413defm : avx512_logical_lowering_types<"VPAND", and>;5414defm : avx512_logical_lowering_types<"VPOR", or>;5415defm : avx512_logical_lowering_types<"VPXOR", xor>;5416defm : avx512_logical_lowering_types<"VPANDN", X86andnp>;5417 5418//===----------------------------------------------------------------------===//5419// AVX-512 FP arithmetic5420//===----------------------------------------------------------------------===//5421 5422multiclass avx512_fp_scalar<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,5423 SDPatternOperator OpNode, SDNode VecNode,5424 X86FoldableSchedWrite sched, bit IsCommutable> {5425 let ExeDomain = _.ExeDomain, Uses = [MXCSR], mayRaiseFPException = 1 in {5426 defm rr : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),5427 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,5428 "$src2, $src1", "$src1, $src2",5429 (_.VT (VecNode _.RC:$src1, _.RC:$src2)), "_Int">,5430 Sched<[sched]>;5431 5432 let mayLoad = 1 in5433 defm rm : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),5434 (ins _.RC:$src1, _.IntScalarMemOp:$src2), OpcodeStr,5435 "$src2, $src1", "$src1, $src2",5436 (_.VT (VecNode _.RC:$src1,5437 (_.ScalarIntMemFrags addr:$src2))), "_Int">,5438 Sched<[sched.Folded, sched.ReadAfterFold]>;5439 let isCodeGenOnly = 1, Predicates = [HasAVX512] in {5440 def rr : I< opc, MRMSrcReg, (outs _.FRC:$dst),5441 (ins _.FRC:$src1, _.FRC:$src2),5442 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",5443 [(set _.FRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2))]>,5444 Sched<[sched]> {5445 let isCommutable = IsCommutable;5446 }5447 let mayLoad = 1 in5448 def rm : I< opc, MRMSrcMem, (outs _.FRC:$dst),5449 (ins _.FRC:$src1, _.ScalarMemOp:$src2),5450 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",5451 [(set _.FRC:$dst, (OpNode _.FRC:$src1,5452 (_.ScalarLdFrag addr:$src2)))]>,5453 Sched<[sched.Folded, sched.ReadAfterFold]>;5454 }5455 }5456}5457 5458multiclass avx512_fp_scalar_round<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,5459 SDNode VecNode, X86FoldableSchedWrite sched> {5460 let ExeDomain = _.ExeDomain, Uses = [MXCSR] in5461 defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),5462 (ins _.RC:$src1, _.RC:$src2, AVX512RC:$rc), OpcodeStr,5463 "$rc, $src2, $src1", "$src1, $src2, $rc",5464 (VecNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),5465 (i32 timm:$rc)), "_Int">,5466 EVEX_B, EVEX_RC, Sched<[sched]>;5467}5468multiclass avx512_fp_scalar_sae<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,5469 SDPatternOperator OpNode, SDNode VecNode, SDNode SaeNode,5470 X86FoldableSchedWrite sched, bit IsCommutable> {5471 let ExeDomain = _.ExeDomain in {5472 defm rr : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),5473 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,5474 "$src2, $src1", "$src1, $src2",5475 (_.VT (VecNode _.RC:$src1, _.RC:$src2)), "_Int">,5476 Sched<[sched]>, SIMD_EXC;5477 5478 let mayLoad = 1 in5479 defm rm : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),5480 (ins _.RC:$src1, _.IntScalarMemOp:$src2), OpcodeStr,5481 "$src2, $src1", "$src1, $src2",5482 (_.VT (VecNode _.RC:$src1,5483 (_.ScalarIntMemFrags addr:$src2))), "_Int">,5484 Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC;5485 5486 let isCodeGenOnly = 1, Predicates = [HasAVX512],5487 Uses = [MXCSR], mayRaiseFPException = 1 in {5488 def rr : I< opc, MRMSrcReg, (outs _.FRC:$dst),5489 (ins _.FRC:$src1, _.FRC:$src2),5490 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",5491 [(set _.FRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2))]>,5492 Sched<[sched]> {5493 let isCommutable = IsCommutable;5494 }5495 let mayLoad = 1 in5496 def rm : I< opc, MRMSrcMem, (outs _.FRC:$dst),5497 (ins _.FRC:$src1, _.ScalarMemOp:$src2),5498 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",5499 [(set _.FRC:$dst, (OpNode _.FRC:$src1,5500 (_.ScalarLdFrag addr:$src2)))]>,5501 Sched<[sched.Folded, sched.ReadAfterFold]>;5502 }5503 5504 let Uses = [MXCSR] in5505 defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),5506 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,5507 "{sae}, $src2, $src1", "$src1, $src2, {sae}",5508 (SaeNode (_.VT _.RC:$src1), (_.VT _.RC:$src2)), "_Int">,5509 EVEX_B, Sched<[sched]>;5510 }5511}5512 5513multiclass avx512_binop_s_round<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,5514 SDNode VecNode, SDNode RndNode,5515 X86SchedWriteSizes sched, bit IsCommutable> {5516 defm SSZ : avx512_fp_scalar<opc, OpcodeStr#"ss", f32x_info, OpNode, VecNode,5517 sched.PS.Scl, IsCommutable>,5518 avx512_fp_scalar_round<opc, OpcodeStr#"ss", f32x_info, RndNode,5519 sched.PS.Scl>,5520 TB, XS, EVEX, VVVV, VEX_LIG, EVEX_CD8<32, CD8VT1>;5521 defm SDZ : avx512_fp_scalar<opc, OpcodeStr#"sd", f64x_info, OpNode, VecNode,5522 sched.PD.Scl, IsCommutable>,5523 avx512_fp_scalar_round<opc, OpcodeStr#"sd", f64x_info, RndNode,5524 sched.PD.Scl>,5525 TB, XD, REX_W, EVEX, VVVV, VEX_LIG, EVEX_CD8<64, CD8VT1>;5526 let Predicates = [HasFP16] in5527 defm SHZ : avx512_fp_scalar<opc, OpcodeStr#"sh", f16x_info, OpNode,5528 VecNode, sched.PH.Scl, IsCommutable>,5529 avx512_fp_scalar_round<opc, OpcodeStr#"sh", f16x_info, RndNode,5530 sched.PH.Scl>,5531 T_MAP5, XS, EVEX, VVVV, VEX_LIG, EVEX_CD8<16, CD8VT1>;5532}5533 5534multiclass avx512_binop_s_sae<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,5535 SDNode VecNode, SDNode SaeNode,5536 X86SchedWriteSizes sched, bit IsCommutable> {5537 defm SSZ : avx512_fp_scalar_sae<opc, OpcodeStr#"ss", f32x_info, OpNode,5538 VecNode, SaeNode, sched.PS.Scl, IsCommutable>,5539 TB, XS, EVEX, VVVV, VEX_LIG, EVEX_CD8<32, CD8VT1>;5540 defm SDZ : avx512_fp_scalar_sae<opc, OpcodeStr#"sd", f64x_info, OpNode,5541 VecNode, SaeNode, sched.PD.Scl, IsCommutable>,5542 TB, XD, REX_W, EVEX, VVVV, VEX_LIG, EVEX_CD8<64, CD8VT1>;5543 let Predicates = [HasFP16] in {5544 defm SHZ : avx512_fp_scalar_sae<opc, OpcodeStr#"sh", f16x_info, OpNode,5545 VecNode, SaeNode, sched.PH.Scl, IsCommutable>,5546 T_MAP5, XS, EVEX, VVVV, VEX_LIG, EVEX_CD8<16, CD8VT1>;5547 }5548}5549defm VADD : avx512_binop_s_round<0x58, "vadd", any_fadd, X86fadds, X86faddRnds,5550 SchedWriteFAddSizes, 1>;5551defm VMUL : avx512_binop_s_round<0x59, "vmul", any_fmul, X86fmuls, X86fmulRnds,5552 SchedWriteFMulSizes, 1>;5553defm VSUB : avx512_binop_s_round<0x5C, "vsub", any_fsub, X86fsubs, X86fsubRnds,5554 SchedWriteFAddSizes, 0>;5555defm VDIV : avx512_binop_s_round<0x5E, "vdiv", any_fdiv, X86fdivs, X86fdivRnds,5556 SchedWriteFDivSizes, 0>;5557defm VMIN : avx512_binop_s_sae<0x5D, "vmin", X86any_fmin, X86fmins, X86fminSAEs,5558 SchedWriteFCmpSizes, 0>;5559defm VMAX : avx512_binop_s_sae<0x5F, "vmax", X86any_fmax, X86fmaxs, X86fmaxSAEs,5560 SchedWriteFCmpSizes, 0>;5561 5562// MIN/MAX nodes are commutable under (nnan + ninf). In this case we use5563// X86fminc and X86fmaxc instead of X86fmin and X86fmax5564multiclass avx512_comutable_binop_s<bits<8> opc, string OpcodeStr,5565 X86VectorVTInfo _, SDNode OpNode,5566 X86FoldableSchedWrite sched> {5567 let isCodeGenOnly = 1, Predicates = [HasAVX512], ExeDomain = _.ExeDomain in {5568 def rr : I< opc, MRMSrcReg, (outs _.FRC:$dst),5569 (ins _.FRC:$src1, _.FRC:$src2),5570 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",5571 [(set _.FRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2))]>,5572 Sched<[sched]> {5573 let isCommutable = 1;5574 }5575 let mayLoad = 1 in5576 def rm : I< opc, MRMSrcMem, (outs _.FRC:$dst),5577 (ins _.FRC:$src1, _.ScalarMemOp:$src2),5578 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",5579 [(set _.FRC:$dst, (OpNode _.FRC:$src1,5580 (_.ScalarLdFrag addr:$src2)))]>,5581 Sched<[sched.Folded, sched.ReadAfterFold]>;5582 }5583}5584defm VMINCSSZ : avx512_comutable_binop_s<0x5D, "vminss", f32x_info, X86fminc,5585 SchedWriteFCmp.Scl>, TB, XS,5586 EVEX, VVVV, VEX_LIG, EVEX_CD8<32, CD8VT1>, SIMD_EXC;5587 5588defm VMINCSDZ : avx512_comutable_binop_s<0x5D, "vminsd", f64x_info, X86fminc,5589 SchedWriteFCmp.Scl>, TB, XD,5590 REX_W, EVEX, VVVV, VEX_LIG,5591 EVEX_CD8<64, CD8VT1>, SIMD_EXC;5592 5593defm VMAXCSSZ : avx512_comutable_binop_s<0x5F, "vmaxss", f32x_info, X86fmaxc,5594 SchedWriteFCmp.Scl>, TB, XS,5595 EVEX, VVVV, VEX_LIG, EVEX_CD8<32, CD8VT1>, SIMD_EXC;5596 5597defm VMAXCSDZ : avx512_comutable_binop_s<0x5F, "vmaxsd", f64x_info, X86fmaxc,5598 SchedWriteFCmp.Scl>, TB, XD,5599 REX_W, EVEX, VVVV, VEX_LIG,5600 EVEX_CD8<64, CD8VT1>, SIMD_EXC;5601 5602defm VMINCSHZ : avx512_comutable_binop_s<0x5D, "vminsh", f16x_info, X86fminc,5603 SchedWriteFCmp.Scl>, T_MAP5, XS,5604 EVEX, VVVV, VEX_LIG, EVEX_CD8<16, CD8VT1>, SIMD_EXC;5605 5606defm VMAXCSHZ : avx512_comutable_binop_s<0x5F, "vmaxsh", f16x_info, X86fmaxc,5607 SchedWriteFCmp.Scl>, T_MAP5, XS,5608 EVEX, VVVV, VEX_LIG, EVEX_CD8<16, CD8VT1>, SIMD_EXC;5609 5610multiclass avx512_fp_packed<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,5611 SDPatternOperator MaskOpNode,5612 X86VectorVTInfo _, X86FoldableSchedWrite sched,5613 bit IsCommutable,5614 bit IsKCommutable = IsCommutable,5615 string suffix = _.Suffix,5616 string ClobberConstraint = "",5617 bit MayRaiseFPException = 1> {5618 let ExeDomain = _.ExeDomain, hasSideEffects = 0,5619 Uses = [MXCSR], mayRaiseFPException = MayRaiseFPException in {5620 defm rr: AVX512_maskable_split<opc, MRMSrcReg, _, (outs _.RC:$dst),5621 (ins _.RC:$src1, _.RC:$src2), OpcodeStr#suffix,5622 "$src2, $src1", "$src1, $src2",5623 (_.VT (OpNode _.RC:$src1, _.RC:$src2)),5624 (_.VT (MaskOpNode _.RC:$src1, _.RC:$src2)), ClobberConstraint,5625 IsCommutable, IsKCommutable, IsKCommutable>, EVEX, VVVV, Sched<[sched]>;5626 let mayLoad = 1 in {5627 defm rm: AVX512_maskable_split<opc, MRMSrcMem, _, (outs _.RC:$dst),5628 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr#suffix,5629 "$src2, $src1", "$src1, $src2",5630 (OpNode _.RC:$src1, (_.LdFrag addr:$src2)),5631 (MaskOpNode _.RC:$src1, (_.LdFrag addr:$src2)),5632 ClobberConstraint>, EVEX, VVVV, Sched<[sched.Folded, sched.ReadAfterFold]>;5633 defm rmb: AVX512_maskable_split<opc, MRMSrcMem, _, (outs _.RC:$dst),5634 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr#suffix,5635 "${src2}"#_.BroadcastStr#", $src1",5636 "$src1, ${src2}"#_.BroadcastStr,5637 (OpNode _.RC:$src1, (_.VT (_.BroadcastLdFrag addr:$src2))),5638 (MaskOpNode _.RC:$src1, (_.VT (_.BroadcastLdFrag addr:$src2))),5639 ClobberConstraint>, EVEX, VVVV, EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;5640 }5641 }5642}5643 5644multiclass avx512_fp_round_packed<bits<8> opc, string OpcodeStr,5645 SDPatternOperator OpNodeRnd,5646 X86FoldableSchedWrite sched, X86VectorVTInfo _,5647 string suffix = _.Suffix,5648 string ClobberConstraint = ""> {5649 let ExeDomain = _.ExeDomain, Uses = [MXCSR] in5650 defm rrb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),5651 (ins _.RC:$src1, _.RC:$src2, AVX512RC:$rc), OpcodeStr#suffix,5652 "$rc, $src2, $src1", "$src1, $src2, $rc",5653 (_.VT (OpNodeRnd _.RC:$src1, _.RC:$src2, (i32 timm:$rc))),5654 0, 0, 0, vselect_mask, ClobberConstraint>,5655 EVEX, VVVV, EVEX_B, EVEX_RC, Sched<[sched]>;5656}5657 5658multiclass avx512_fp_sae_packed<bits<8> opc, string OpcodeStr,5659 SDPatternOperator OpNodeSAE,5660 X86FoldableSchedWrite sched, X86VectorVTInfo _> {5661 let ExeDomain = _.ExeDomain, Uses = [MXCSR] in5662 defm rrb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),5663 (ins _.RC:$src1, _.RC:$src2), OpcodeStr#_.Suffix,5664 "{sae}, $src2, $src1", "$src1, $src2, {sae}",5665 (_.VT (OpNodeSAE _.RC:$src1, _.RC:$src2))>,5666 EVEX, VVVV, EVEX_B, Sched<[sched]>;5667}5668 5669multiclass avx512_fp_binop_p<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,5670 SDPatternOperator MaskOpNode,5671 Predicate prd, X86SchedWriteSizes sched,5672 bit IsCommutable = 0,5673 bit IsPD128Commutable = IsCommutable> {5674 let Predicates = [prd] in {5675 defm PSZ : avx512_fp_packed<opc, OpcodeStr, OpNode, MaskOpNode, v16f32_info,5676 sched.PS.ZMM, IsCommutable>, EVEX_V512, TB,5677 EVEX_CD8<32, CD8VF>;5678 defm PDZ : avx512_fp_packed<opc, OpcodeStr, OpNode, MaskOpNode, v8f64_info,5679 sched.PD.ZMM, IsCommutable>, EVEX_V512, TB, PD, REX_W,5680 EVEX_CD8<64, CD8VF>;5681 }5682 5683 // Define only if AVX512VL feature is present.5684 let Predicates = [prd, HasVLX] in {5685 defm PSZ128 : avx512_fp_packed<opc, OpcodeStr, OpNode, MaskOpNode, v4f32x_info,5686 sched.PS.XMM, IsCommutable>, EVEX_V128, TB,5687 EVEX_CD8<32, CD8VF>;5688 defm PSZ256 : avx512_fp_packed<opc, OpcodeStr, OpNode, MaskOpNode, v8f32x_info,5689 sched.PS.YMM, IsCommutable>, EVEX_V256, TB,5690 EVEX_CD8<32, CD8VF>;5691 defm PDZ128 : avx512_fp_packed<opc, OpcodeStr, OpNode, MaskOpNode, v2f64x_info,5692 sched.PD.XMM, IsPD128Commutable,5693 IsCommutable>, EVEX_V128, TB, PD, REX_W,5694 EVEX_CD8<64, CD8VF>;5695 defm PDZ256 : avx512_fp_packed<opc, OpcodeStr, OpNode, MaskOpNode, v4f64x_info,5696 sched.PD.YMM, IsCommutable>, EVEX_V256, TB, PD, REX_W,5697 EVEX_CD8<64, CD8VF>;5698 }5699}5700 5701multiclass avx512_fp_binop_ph<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,5702 SDPatternOperator MaskOpNode,5703 X86SchedWriteSizes sched, bit IsCommutable = 0> {5704 let Predicates = [HasFP16] in {5705 defm PHZ : avx512_fp_packed<opc, OpcodeStr, OpNode, MaskOpNode, v32f16_info,5706 sched.PH.ZMM, IsCommutable>, EVEX_V512, T_MAP5,5707 EVEX_CD8<16, CD8VF>;5708 }5709 let Predicates = [HasVLX, HasFP16] in {5710 defm PHZ128 : avx512_fp_packed<opc, OpcodeStr, OpNode, MaskOpNode, v8f16x_info,5711 sched.PH.XMM, IsCommutable>, EVEX_V128, T_MAP5,5712 EVEX_CD8<16, CD8VF>;5713 defm PHZ256 : avx512_fp_packed<opc, OpcodeStr, OpNode, MaskOpNode, v16f16x_info,5714 sched.PH.YMM, IsCommutable>, EVEX_V256, T_MAP5,5715 EVEX_CD8<16, CD8VF>;5716 }5717}5718 5719let Uses = [MXCSR] in5720multiclass avx512_fp_binop_p_round<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd,5721 X86SchedWriteSizes sched> {5722 let Predicates = [HasFP16] in {5723 defm PHZ : avx512_fp_round_packed<opc, OpcodeStr, OpNodeRnd, sched.PH.ZMM,5724 v32f16_info>,5725 EVEX_V512, T_MAP5, EVEX_CD8<16, CD8VF>;5726 }5727 defm PSZ : avx512_fp_round_packed<opc, OpcodeStr, OpNodeRnd, sched.PS.ZMM,5728 v16f32_info>,5729 EVEX_V512, TB, EVEX_CD8<32, CD8VF>;5730 defm PDZ : avx512_fp_round_packed<opc, OpcodeStr, OpNodeRnd, sched.PD.ZMM,5731 v8f64_info>,5732 EVEX_V512, TB, PD, REX_W,EVEX_CD8<64, CD8VF>;5733}5734 5735let Uses = [MXCSR] in5736multiclass avx512_fp_binop_p_sae<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd,5737 X86SchedWriteSizes sched> {5738 let Predicates = [HasFP16] in {5739 defm PHZ : avx512_fp_sae_packed<opc, OpcodeStr, OpNodeRnd, sched.PH.ZMM,5740 v32f16_info>,5741 EVEX_V512, T_MAP5, EVEX_CD8<16, CD8VF>;5742 }5743 defm PSZ : avx512_fp_sae_packed<opc, OpcodeStr, OpNodeRnd, sched.PS.ZMM,5744 v16f32_info>,5745 EVEX_V512, TB, EVEX_CD8<32, CD8VF>;5746 defm PDZ : avx512_fp_sae_packed<opc, OpcodeStr, OpNodeRnd, sched.PD.ZMM,5747 v8f64_info>,5748 EVEX_V512, TB, PD, REX_W,EVEX_CD8<64, CD8VF>;5749}5750 5751defm VADD : avx512_fp_binop_p<0x58, "vadd", any_fadd, fadd, HasAVX512,5752 SchedWriteFAddSizes, 1>,5753 avx512_fp_binop_ph<0x58, "vadd", any_fadd, fadd, SchedWriteFAddSizes, 1>,5754 avx512_fp_binop_p_round<0x58, "vadd", X86faddRnd, SchedWriteFAddSizes>;5755defm VMUL : avx512_fp_binop_p<0x59, "vmul", any_fmul, fmul, HasAVX512,5756 SchedWriteFMulSizes, 1>,5757 avx512_fp_binop_ph<0x59, "vmul", any_fmul, fmul, SchedWriteFMulSizes, 1>,5758 avx512_fp_binop_p_round<0x59, "vmul", X86fmulRnd, SchedWriteFMulSizes>;5759defm VSUB : avx512_fp_binop_p<0x5C, "vsub", any_fsub, fsub, HasAVX512,5760 SchedWriteFAddSizes>,5761 avx512_fp_binop_ph<0x5C, "vsub", any_fsub, fsub, SchedWriteFAddSizes>,5762 avx512_fp_binop_p_round<0x5C, "vsub", X86fsubRnd, SchedWriteFAddSizes>;5763defm VDIV : avx512_fp_binop_p<0x5E, "vdiv", any_fdiv, fdiv, HasAVX512,5764 SchedWriteFDivSizes>,5765 avx512_fp_binop_ph<0x5E, "vdiv", any_fdiv, fdiv, SchedWriteFDivSizes>,5766 avx512_fp_binop_p_round<0x5E, "vdiv", X86fdivRnd, SchedWriteFDivSizes>;5767defm VMIN : avx512_fp_binop_p<0x5D, "vmin", X86any_fmin, X86fmin, HasAVX512,5768 SchedWriteFCmpSizes, 0>,5769 avx512_fp_binop_ph<0x5D, "vmin", X86any_fmin, X86fmin, SchedWriteFCmpSizes, 0>,5770 avx512_fp_binop_p_sae<0x5D, "vmin", X86fminSAE, SchedWriteFCmpSizes>;5771defm VMAX : avx512_fp_binop_p<0x5F, "vmax", X86any_fmax, X86fmax, HasAVX512,5772 SchedWriteFCmpSizes, 0>,5773 avx512_fp_binop_ph<0x5F, "vmax", X86any_fmax, X86fmax, SchedWriteFCmpSizes, 0>,5774 avx512_fp_binop_p_sae<0x5F, "vmax", X86fmaxSAE, SchedWriteFCmpSizes>;5775let isCodeGenOnly = 1 in {5776 defm VMINC : avx512_fp_binop_p<0x5D, "vmin", X86fminc, X86fminc, HasAVX512,5777 SchedWriteFCmpSizes, 1>,5778 avx512_fp_binop_ph<0x5D, "vmin", X86fminc, X86fminc,5779 SchedWriteFCmpSizes, 1>;5780 defm VMAXC : avx512_fp_binop_p<0x5F, "vmax", X86fmaxc, X86fmaxc, HasAVX512,5781 SchedWriteFCmpSizes, 1>,5782 avx512_fp_binop_ph<0x5F, "vmax", X86fmaxc, X86fmaxc,5783 SchedWriteFCmpSizes, 1>;5784}5785let Uses = []<Register>, mayRaiseFPException = 0 in {5786defm VAND : avx512_fp_binop_p<0x54, "vand", null_frag, null_frag, HasDQI,5787 SchedWriteFLogicSizes, 1>;5788defm VANDN : avx512_fp_binop_p<0x55, "vandn", null_frag, null_frag, HasDQI,5789 SchedWriteFLogicSizes, 0>;5790defm VOR : avx512_fp_binop_p<0x56, "vor", null_frag, null_frag, HasDQI,5791 SchedWriteFLogicSizes, 1>;5792defm VXOR : avx512_fp_binop_p<0x57, "vxor", null_frag, null_frag, HasDQI,5793 SchedWriteFLogicSizes, 1>;5794}5795 5796multiclass avx512_fp_scalef_p<bits<8> opc, string OpcodeStr, SDNode OpNode,5797 X86FoldableSchedWrite sched, X86VectorVTInfo _> {5798 let ExeDomain = _.ExeDomain, Uses = [MXCSR], mayRaiseFPException = 1 in {5799 defm rr: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),5800 (ins _.RC:$src1, _.RC:$src2), OpcodeStr#_.Suffix,5801 "$src2, $src1", "$src1, $src2",5802 (_.VT (OpNode _.RC:$src1, _.RC:$src2))>,5803 EVEX, VVVV, Sched<[sched]>;5804 let mayLoad = 1 in {5805 defm rm: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),5806 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr#_.Suffix,5807 "$src2, $src1", "$src1, $src2",5808 (OpNode _.RC:$src1, (_.LdFrag addr:$src2))>,5809 EVEX, VVVV, Sched<[sched.Folded, sched.ReadAfterFold]>;5810 defm rmb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),5811 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr#_.Suffix,5812 "${src2}"#_.BroadcastStr#", $src1",5813 "$src1, ${src2}"#_.BroadcastStr,5814 (OpNode _.RC:$src1, (_.VT (_.BroadcastLdFrag addr:$src2)))>,5815 EVEX, VVVV, EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;5816 }5817 }5818}5819 5820multiclass avx512_fp_scalef_scalar<bits<8> opc, string OpcodeStr, SDNode OpNode,5821 X86FoldableSchedWrite sched, X86VectorVTInfo _> {5822 let ExeDomain = _.ExeDomain, Uses = [MXCSR], mayRaiseFPException = 1 in {5823 defm rr: AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),5824 (ins _.RC:$src1, _.RC:$src2), OpcodeStr#_.Suffix,5825 "$src2, $src1", "$src1, $src2",5826 (_.VT (OpNode _.RC:$src1, _.RC:$src2))>,5827 Sched<[sched]>;5828 let mayLoad = 1 in5829 defm rm: AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),5830 (ins _.RC:$src1, _.IntScalarMemOp:$src2), OpcodeStr#_.Suffix,5831 "$src2, $src1", "$src1, $src2",5832 (OpNode _.RC:$src1, (_.ScalarIntMemFrags addr:$src2))>,5833 Sched<[sched.Folded, sched.ReadAfterFold]>;5834 }5835}5836 5837multiclass avx512_fp_scalef_all<bits<8> opc, bits<8> opcScaler, string OpcodeStr,5838 X86SchedWriteWidths sched> {5839 let Predicates = [HasFP16] in {5840 defm PHZ : avx512_fp_scalef_p<opc, OpcodeStr, X86scalef, sched.ZMM, v32f16_info>,5841 avx512_fp_round_packed<opc, OpcodeStr, X86scalefRnd, sched.ZMM, v32f16_info>,5842 EVEX_V512, T_MAP6, PD, EVEX_CD8<16, CD8VF>;5843 defm SHZ : avx512_fp_scalef_scalar<opcScaler, OpcodeStr, X86scalefs, sched.Scl, f16x_info>,5844 avx512_fp_scalar_round<opcScaler, OpcodeStr#"sh", f16x_info, X86scalefsRnd, sched.Scl>,5845 EVEX, VVVV, T_MAP6, PD, EVEX_CD8<16, CD8VT1>;5846 }5847 defm PSZ : avx512_fp_scalef_p<opc, OpcodeStr, X86scalef, sched.ZMM, v16f32_info>,5848 avx512_fp_round_packed<opc, OpcodeStr, X86scalefRnd, sched.ZMM, v16f32_info>,5849 EVEX_V512, EVEX_CD8<32, CD8VF>, T8, PD;5850 defm PDZ : avx512_fp_scalef_p<opc, OpcodeStr, X86scalef, sched.ZMM, v8f64_info>,5851 avx512_fp_round_packed<opc, OpcodeStr, X86scalefRnd, sched.ZMM, v8f64_info>,5852 EVEX_V512, REX_W, EVEX_CD8<64, CD8VF>, T8, PD;5853 defm SSZ : avx512_fp_scalef_scalar<opcScaler, OpcodeStr, X86scalefs, sched.Scl, f32x_info>,5854 avx512_fp_scalar_round<opcScaler, OpcodeStr#"ss", f32x_info,5855 X86scalefsRnd, sched.Scl>,5856 EVEX, VVVV, VEX_LIG, EVEX_CD8<32, CD8VT1>, T8, PD;5857 defm SDZ : avx512_fp_scalef_scalar<opcScaler, OpcodeStr, X86scalefs, sched.Scl, f64x_info>,5858 avx512_fp_scalar_round<opcScaler, OpcodeStr#"sd", f64x_info,5859 X86scalefsRnd, sched.Scl>,5860 EVEX, VVVV, VEX_LIG, EVEX_CD8<64, CD8VT1>, REX_W, T8, PD;5861 5862 // Define only if AVX512VL feature is present.5863 let Predicates = [HasVLX] in {5864 defm PSZ128 : avx512_fp_scalef_p<opc, OpcodeStr, X86scalef, sched.XMM, v4f32x_info>,5865 EVEX_V128, EVEX_CD8<32, CD8VF>, T8, PD;5866 defm PSZ256 : avx512_fp_scalef_p<opc, OpcodeStr, X86scalef, sched.YMM, v8f32x_info>,5867 EVEX_V256, EVEX_CD8<32, CD8VF>, T8, PD;5868 defm PDZ128 : avx512_fp_scalef_p<opc, OpcodeStr, X86scalef, sched.XMM, v2f64x_info>,5869 EVEX_V128, REX_W, EVEX_CD8<64, CD8VF>, T8, PD;5870 defm PDZ256 : avx512_fp_scalef_p<opc, OpcodeStr, X86scalef, sched.YMM, v4f64x_info>,5871 EVEX_V256, REX_W, EVEX_CD8<64, CD8VF>, T8, PD;5872 }5873 5874 let Predicates = [HasFP16, HasVLX] in {5875 defm PHZ128 : avx512_fp_scalef_p<opc, OpcodeStr, X86scalef, sched.XMM, v8f16x_info>,5876 EVEX_V128, EVEX_CD8<16, CD8VF>, T_MAP6, PD;5877 defm PHZ256 : avx512_fp_scalef_p<opc, OpcodeStr, X86scalef, sched.YMM, v16f16x_info>,5878 EVEX_V256, EVEX_CD8<16, CD8VF>, T_MAP6, PD;5879 }5880}5881defm VSCALEF : avx512_fp_scalef_all<0x2C, 0x2D, "vscalef", SchedWriteFAdd>;5882 5883//===----------------------------------------------------------------------===//5884// AVX-512 VPTESTM instructions5885//===----------------------------------------------------------------------===//5886 5887multiclass avx512_vptest<bits<8> opc, string OpcodeStr,5888 X86FoldableSchedWrite sched, X86VectorVTInfo _> {5889 // NOTE: Patterns are omitted in favor of manual selection in X86ISelDAGToDAG.5890 // There are just too many permutations due to commutability and bitcasts.5891 let ExeDomain = _.ExeDomain, hasSideEffects = 0 in {5892 defm rr : AVX512_maskable_cmp<opc, MRMSrcReg, _, (outs _.KRC:$dst),5893 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,5894 "$src2, $src1", "$src1, $src2",5895 (null_frag), (null_frag), 1>,5896 EVEX, VVVV, Sched<[sched]>;5897 let mayLoad = 1 in5898 defm rm : AVX512_maskable_cmp<opc, MRMSrcMem, _, (outs _.KRC:$dst),5899 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr,5900 "$src2, $src1", "$src1, $src2",5901 (null_frag), (null_frag)>,5902 EVEX, VVVV, EVEX_CD8<_.EltSize, CD8VF>,5903 Sched<[sched.Folded, sched.ReadAfterFold]>;5904 }5905}5906 5907multiclass avx512_vptest_mb<bits<8> opc, string OpcodeStr,5908 X86FoldableSchedWrite sched, X86VectorVTInfo _> {5909 let ExeDomain = _.ExeDomain, mayLoad = 1, hasSideEffects = 0 in5910 defm rmb : AVX512_maskable_cmp<opc, MRMSrcMem, _, (outs _.KRC:$dst),5911 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,5912 "${src2}"#_.BroadcastStr#", $src1",5913 "$src1, ${src2}"#_.BroadcastStr,5914 (null_frag), (null_frag)>,5915 EVEX_B, EVEX, VVVV, EVEX_CD8<_.EltSize, CD8VF>,5916 Sched<[sched.Folded, sched.ReadAfterFold]>;5917}5918 5919multiclass avx512_vptest_dq_sizes<bits<8> opc, string OpcodeStr,5920 X86SchedWriteWidths sched,5921 AVX512VLVectorVTInfo _> {5922 let Predicates = [HasAVX512] in5923 defm Z : avx512_vptest<opc, OpcodeStr, sched.ZMM, _.info512>,5924 avx512_vptest_mb<opc, OpcodeStr, sched.ZMM, _.info512>, EVEX_V512;5925 5926 let Predicates = [HasAVX512, HasVLX] in {5927 defm Z256 : avx512_vptest<opc, OpcodeStr, sched.YMM, _.info256>,5928 avx512_vptest_mb<opc, OpcodeStr, sched.YMM, _.info256>, EVEX_V256;5929 defm Z128 : avx512_vptest<opc, OpcodeStr, sched.XMM, _.info128>,5930 avx512_vptest_mb<opc, OpcodeStr, sched.XMM, _.info128>, EVEX_V128;5931 }5932}5933 5934multiclass avx512_vptest_dq<bits<8> opc, string OpcodeStr,5935 X86SchedWriteWidths sched> {5936 defm D : avx512_vptest_dq_sizes<opc, OpcodeStr#"d", sched,5937 avx512vl_i32_info>;5938 defm Q : avx512_vptest_dq_sizes<opc, OpcodeStr#"q", sched,5939 avx512vl_i64_info>, REX_W;5940}5941 5942multiclass avx512_vptest_wb<bits<8> opc, string OpcodeStr,5943 X86SchedWriteWidths sched> {5944 let Predicates = [HasBWI] in {5945 defm WZ: avx512_vptest<opc, OpcodeStr#"w", sched.ZMM,5946 v32i16_info>, EVEX_V512, REX_W;5947 defm BZ: avx512_vptest<opc, OpcodeStr#"b", sched.ZMM,5948 v64i8_info>, EVEX_V512;5949 }5950 5951 let Predicates = [HasVLX, HasBWI] in {5952 defm WZ256: avx512_vptest<opc, OpcodeStr#"w", sched.YMM,5953 v16i16x_info>, EVEX_V256, REX_W;5954 defm WZ128: avx512_vptest<opc, OpcodeStr#"w", sched.XMM,5955 v8i16x_info>, EVEX_V128, REX_W;5956 defm BZ256: avx512_vptest<opc, OpcodeStr#"b", sched.YMM,5957 v32i8x_info>, EVEX_V256;5958 defm BZ128: avx512_vptest<opc, OpcodeStr#"b", sched.XMM,5959 v16i8x_info>, EVEX_V128;5960 }5961}5962 5963multiclass avx512_vptest_all_forms<bits<8> opc_wb, bits<8> opc_dq, string OpcodeStr,5964 X86SchedWriteWidths sched> :5965 avx512_vptest_wb<opc_wb, OpcodeStr, sched>,5966 avx512_vptest_dq<opc_dq, OpcodeStr, sched>;5967 5968defm VPTESTM : avx512_vptest_all_forms<0x26, 0x27, "vptestm",5969 SchedWriteVecLogic>, T8, PD;5970defm VPTESTNM : avx512_vptest_all_forms<0x26, 0x27, "vptestnm",5971 SchedWriteVecLogic>, T8, XS;5972 5973//===----------------------------------------------------------------------===//5974// AVX-512 Shift instructions5975//===----------------------------------------------------------------------===//5976 5977multiclass avx512_shift_rmi<bits<8> opc, Format ImmFormR, Format ImmFormM,5978 string OpcodeStr, SDNode OpNode,5979 X86FoldableSchedWrite sched, X86VectorVTInfo _> {5980 let ExeDomain = _.ExeDomain in {5981 defm ri : AVX512_maskable<opc, ImmFormR, _, (outs _.RC:$dst),5982 (ins _.RC:$src1, u8imm:$src2), OpcodeStr,5983 "$src2, $src1", "$src1, $src2",5984 (_.VT (OpNode _.RC:$src1, (i8 timm:$src2)))>,5985 Sched<[sched]>;5986 let mayLoad = 1 in5987 defm mi : AVX512_maskable<opc, ImmFormM, _, (outs _.RC:$dst),5988 (ins _.MemOp:$src1, u8imm:$src2), OpcodeStr,5989 "$src2, $src1", "$src1, $src2",5990 (_.VT (OpNode (_.VT (_.LdFrag addr:$src1)),5991 (i8 timm:$src2)))>,5992 Sched<[sched.Folded]>;5993 }5994}5995 5996multiclass avx512_shift_rmbi<bits<8> opc, Format ImmFormM,5997 string OpcodeStr, SDNode OpNode,5998 X86FoldableSchedWrite sched, X86VectorVTInfo _> {5999 let ExeDomain = _.ExeDomain, mayLoad = 1 in6000 defm mbi : AVX512_maskable<opc, ImmFormM, _, (outs _.RC:$dst),6001 (ins _.ScalarMemOp:$src1, u8imm:$src2), OpcodeStr,6002 "$src2, ${src1}"#_.BroadcastStr, "${src1}"#_.BroadcastStr#", $src2",6003 (_.VT (OpNode (_.BroadcastLdFrag addr:$src1), (i8 timm:$src2)))>,6004 EVEX_B, Sched<[sched.Folded]>;6005}6006 6007multiclass avx512_shift_rrm<bits<8> opc, string OpcodeStr, SDNode OpNode,6008 X86FoldableSchedWrite sched, ValueType SrcVT,6009 X86VectorVTInfo _> {6010 // src2 is always 128-bit6011 let ExeDomain = _.ExeDomain in {6012 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),6013 (ins _.RC:$src1, VR128X:$src2), OpcodeStr,6014 "$src2, $src1", "$src1, $src2",6015 (_.VT (OpNode _.RC:$src1, (SrcVT VR128X:$src2)))>,6016 AVX512BIBase, EVEX, VVVV, Sched<[sched]>;6017 let mayLoad = 1 in6018 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),6019 (ins _.RC:$src1, i128mem:$src2), OpcodeStr,6020 "$src2, $src1", "$src1, $src2",6021 (_.VT (OpNode _.RC:$src1, (SrcVT (load addr:$src2))))>,6022 AVX512BIBase,6023 EVEX, VVVV, Sched<[sched.Folded, sched.ReadAfterFold]>;6024 }6025}6026 6027multiclass avx512_shift_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,6028 X86SchedWriteWidths sched, ValueType SrcVT,6029 AVX512VLVectorVTInfo VTInfo,6030 Predicate prd> {6031 let Predicates = [prd] in6032 defm Z : avx512_shift_rrm<opc, OpcodeStr, OpNode, sched.ZMM, SrcVT,6033 VTInfo.info512>, EVEX_V512,6034 EVEX_CD8<VTInfo.info512.EltSize, CD8VQ> ;6035 let Predicates = [prd, HasVLX] in {6036 defm Z256 : avx512_shift_rrm<opc, OpcodeStr, OpNode, sched.YMM, SrcVT,6037 VTInfo.info256>, EVEX_V256,6038 EVEX_CD8<VTInfo.info256.EltSize, CD8VH>;6039 defm Z128 : avx512_shift_rrm<opc, OpcodeStr, OpNode, sched.XMM, SrcVT,6040 VTInfo.info128>, EVEX_V128,6041 EVEX_CD8<VTInfo.info128.EltSize, CD8VF>;6042 }6043}6044 6045multiclass avx512_shift_types<bits<8> opcd, bits<8> opcq, bits<8> opcw,6046 string OpcodeStr, SDNode OpNode,6047 X86SchedWriteWidths sched> {6048 defm D : avx512_shift_sizes<opcd, OpcodeStr#"d", OpNode, sched, v4i32,6049 avx512vl_i32_info, HasAVX512>;6050 defm Q : avx512_shift_sizes<opcq, OpcodeStr#"q", OpNode, sched, v2i64,6051 avx512vl_i64_info, HasAVX512>, REX_W;6052 defm W : avx512_shift_sizes<opcw, OpcodeStr#"w", OpNode, sched, v8i16,6053 avx512vl_i16_info, HasBWI>;6054}6055 6056multiclass avx512_shift_rmi_sizes<bits<8> opc, Format ImmFormR, Format ImmFormM,6057 string OpcodeStr, SDNode OpNode,6058 X86SchedWriteWidths sched,6059 AVX512VLVectorVTInfo VTInfo> {6060 let Predicates = [HasAVX512] in6061 defm Z: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,6062 sched.ZMM, VTInfo.info512>,6063 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode, sched.ZMM,6064 VTInfo.info512>, EVEX_V512;6065 let Predicates = [HasAVX512, HasVLX] in {6066 defm Z256: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,6067 sched.YMM, VTInfo.info256>,6068 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode, sched.YMM,6069 VTInfo.info256>, EVEX_V256;6070 defm Z128: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,6071 sched.XMM, VTInfo.info128>,6072 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode, sched.XMM,6073 VTInfo.info128>, EVEX_V128;6074 }6075}6076 6077multiclass avx512_shift_rmi_w<bits<8> opcw, Format ImmFormR, Format ImmFormM,6078 string OpcodeStr, SDNode OpNode,6079 X86SchedWriteWidths sched> {6080 let Predicates = [HasBWI] in6081 defm WZ: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode,6082 sched.ZMM, v32i16_info>, EVEX_V512, WIG;6083 let Predicates = [HasVLX, HasBWI] in {6084 defm WZ256: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode,6085 sched.YMM, v16i16x_info>, EVEX_V256, WIG;6086 defm WZ128: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode,6087 sched.XMM, v8i16x_info>, EVEX_V128, WIG;6088 }6089}6090 6091multiclass avx512_shift_rmi_dq<bits<8> opcd, bits<8> opcq,6092 Format ImmFormR, Format ImmFormM,6093 string OpcodeStr, SDNode OpNode,6094 X86SchedWriteWidths sched> {6095 defm D: avx512_shift_rmi_sizes<opcd, ImmFormR, ImmFormM, OpcodeStr#"d", OpNode,6096 sched, avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;6097 defm Q: avx512_shift_rmi_sizes<opcq, ImmFormR, ImmFormM, OpcodeStr#"q", OpNode,6098 sched, avx512vl_i64_info>, EVEX_CD8<64, CD8VF>, REX_W;6099}6100 6101defm VPSRL : avx512_shift_rmi_dq<0x72, 0x73, MRM2r, MRM2m, "vpsrl", X86vsrli,6102 SchedWriteVecShiftImm>,6103 avx512_shift_rmi_w<0x71, MRM2r, MRM2m, "vpsrlw", X86vsrli,6104 SchedWriteVecShiftImm>, AVX512BIi8Base, EVEX, VVVV;6105 6106defm VPSLL : avx512_shift_rmi_dq<0x72, 0x73, MRM6r, MRM6m, "vpsll", X86vshli,6107 SchedWriteVecShiftImm>,6108 avx512_shift_rmi_w<0x71, MRM6r, MRM6m, "vpsllw", X86vshli,6109 SchedWriteVecShiftImm>, AVX512BIi8Base, EVEX, VVVV;6110 6111defm VPSRA : avx512_shift_rmi_dq<0x72, 0x72, MRM4r, MRM4m, "vpsra", X86vsrai,6112 SchedWriteVecShiftImm>,6113 avx512_shift_rmi_w<0x71, MRM4r, MRM4m, "vpsraw", X86vsrai,6114 SchedWriteVecShiftImm>, AVX512BIi8Base, EVEX, VVVV;6115 6116defm VPROR : avx512_shift_rmi_dq<0x72, 0x72, MRM0r, MRM0m, "vpror", X86vrotri,6117 SchedWriteVecShiftImm>, AVX512BIi8Base, EVEX, VVVV;6118defm VPROL : avx512_shift_rmi_dq<0x72, 0x72, MRM1r, MRM1m, "vprol", X86vrotli,6119 SchedWriteVecShiftImm>, AVX512BIi8Base, EVEX, VVVV;6120 6121defm VPSLL : avx512_shift_types<0xF2, 0xF3, 0xF1, "vpsll", X86vshl,6122 SchedWriteVecShift>;6123defm VPSRA : avx512_shift_types<0xE2, 0xE2, 0xE1, "vpsra", X86vsra,6124 SchedWriteVecShift>;6125defm VPSRL : avx512_shift_types<0xD2, 0xD3, 0xD1, "vpsrl", X86vsrl,6126 SchedWriteVecShift>;6127 6128// Use 512bit VPSRA/VPSRAI version to implement v2i64/v4i64 in case NoVLX.6129let Predicates = [HasAVX512, NoVLX] in {6130 def : Pat<(v4i64 (X86vsra (v4i64 VR256X:$src1), (v2i64 VR128X:$src2))),6131 (EXTRACT_SUBREG (v8i646132 (VPSRAQZrr6133 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),6134 VR128X:$src2)), sub_ymm)>;6135 6136 def : Pat<(v2i64 (X86vsra (v2i64 VR128X:$src1), (v2i64 VR128X:$src2))),6137 (EXTRACT_SUBREG (v8i646138 (VPSRAQZrr6139 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),6140 VR128X:$src2)), sub_xmm)>;6141 6142 def : Pat<(v4i64 (X86vsrai (v4i64 VR256X:$src1), (i8 timm:$src2))),6143 (EXTRACT_SUBREG (v8i646144 (VPSRAQZri6145 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),6146 timm:$src2)), sub_ymm)>;6147 6148 def : Pat<(v2i64 (X86vsrai (v2i64 VR128X:$src1), (i8 timm:$src2))),6149 (EXTRACT_SUBREG (v8i646150 (VPSRAQZri6151 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),6152 timm:$src2)), sub_xmm)>;6153}6154 6155//===-------------------------------------------------------------------===//6156// Variable Bit Shifts6157//===-------------------------------------------------------------------===//6158 6159multiclass avx512_var_shift<bits<8> opc, string OpcodeStr, SDNode OpNode,6160 X86FoldableSchedWrite sched, X86VectorVTInfo _> {6161 let ExeDomain = _.ExeDomain in {6162 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),6163 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,6164 "$src2, $src1", "$src1, $src2",6165 (_.VT (OpNode _.RC:$src1, (_.VT _.RC:$src2)))>,6166 AVX5128IBase, EVEX, VVVV, Sched<[sched]>;6167 let mayLoad = 1 in6168 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),6169 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr,6170 "$src2, $src1", "$src1, $src2",6171 (_.VT (OpNode _.RC:$src1,6172 (_.VT (_.LdFrag addr:$src2))))>,6173 AVX5128IBase, EVEX, VVVV, EVEX_CD8<_.EltSize, CD8VF>,6174 Sched<[sched.Folded, sched.ReadAfterFold]>;6175 }6176}6177 6178multiclass avx512_var_shift_mb<bits<8> opc, string OpcodeStr, SDNode OpNode,6179 X86FoldableSchedWrite sched, X86VectorVTInfo _> {6180 let ExeDomain = _.ExeDomain, mayLoad = 1 in6181 defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),6182 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,6183 "${src2}"#_.BroadcastStr#", $src1",6184 "$src1, ${src2}"#_.BroadcastStr,6185 (_.VT (OpNode _.RC:$src1, (_.VT (_.BroadcastLdFrag addr:$src2))))>,6186 AVX5128IBase, EVEX_B, EVEX, VVVV, EVEX_CD8<_.EltSize, CD8VF>,6187 Sched<[sched.Folded, sched.ReadAfterFold]>;6188}6189 6190multiclass avx512_var_shift_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,6191 X86SchedWriteWidths sched, AVX512VLVectorVTInfo _> {6192 let Predicates = [HasAVX512] in6193 defm Z : avx512_var_shift<opc, OpcodeStr, OpNode, sched.ZMM, _.info512>,6194 avx512_var_shift_mb<opc, OpcodeStr, OpNode, sched.ZMM, _.info512>, EVEX_V512;6195 6196 let Predicates = [HasAVX512, HasVLX] in {6197 defm Z256 : avx512_var_shift<opc, OpcodeStr, OpNode, sched.YMM, _.info256>,6198 avx512_var_shift_mb<opc, OpcodeStr, OpNode, sched.YMM, _.info256>, EVEX_V256;6199 defm Z128 : avx512_var_shift<opc, OpcodeStr, OpNode, sched.XMM, _.info128>,6200 avx512_var_shift_mb<opc, OpcodeStr, OpNode, sched.XMM, _.info128>, EVEX_V128;6201 }6202}6203 6204multiclass avx512_var_shift_types<bits<8> opc, string OpcodeStr,6205 SDNode OpNode, X86SchedWriteWidths sched> {6206 defm D : avx512_var_shift_sizes<opc, OpcodeStr#"d", OpNode, sched,6207 avx512vl_i32_info>;6208 defm Q : avx512_var_shift_sizes<opc, OpcodeStr#"q", OpNode, sched,6209 avx512vl_i64_info>, REX_W;6210}6211 6212// Use 512bit version to implement 128/256 bit in case NoVLX.6213multiclass avx512_var_shift_lowering<AVX512VLVectorVTInfo _, string OpcodeStr,6214 SDNode OpNode, list<Predicate> p> {6215 let Predicates = p in {6216 def : Pat<(_.info256.VT (OpNode (_.info256.VT _.info256.RC:$src1),6217 (_.info256.VT _.info256.RC:$src2))),6218 (EXTRACT_SUBREG6219 (!cast<Instruction>(OpcodeStr#"Zrr")6220 (INSERT_SUBREG (_.info512.VT (IMPLICIT_DEF)), VR256X:$src1, sub_ymm),6221 (INSERT_SUBREG (_.info512.VT (IMPLICIT_DEF)), VR256X:$src2, sub_ymm)),6222 sub_ymm)>;6223 6224 def : Pat<(_.info128.VT (OpNode (_.info128.VT _.info128.RC:$src1),6225 (_.info128.VT _.info128.RC:$src2))),6226 (EXTRACT_SUBREG6227 (!cast<Instruction>(OpcodeStr#"Zrr")6228 (INSERT_SUBREG (_.info512.VT (IMPLICIT_DEF)), VR128X:$src1, sub_xmm),6229 (INSERT_SUBREG (_.info512.VT (IMPLICIT_DEF)), VR128X:$src2, sub_xmm)),6230 sub_xmm)>;6231 }6232}6233multiclass avx512_var_shift_w<bits<8> opc, string OpcodeStr,6234 SDNode OpNode, X86SchedWriteWidths sched> {6235 let Predicates = [HasBWI] in6236 defm WZ: avx512_var_shift<opc, OpcodeStr, OpNode, sched.ZMM, v32i16_info>,6237 EVEX_V512, REX_W;6238 let Predicates = [HasVLX, HasBWI] in {6239 6240 defm WZ256: avx512_var_shift<opc, OpcodeStr, OpNode, sched.YMM, v16i16x_info>,6241 EVEX_V256, REX_W;6242 defm WZ128: avx512_var_shift<opc, OpcodeStr, OpNode, sched.XMM, v8i16x_info>,6243 EVEX_V128, REX_W;6244 }6245}6246 6247defm VPSLLV : avx512_var_shift_types<0x47, "vpsllv", X86vshlv, SchedWriteVarVecShift>,6248 avx512_var_shift_w<0x12, "vpsllvw", X86vshlv, SchedWriteVarVecShift>;6249 6250defm VPSRAV : avx512_var_shift_types<0x46, "vpsrav", X86vsrav, SchedWriteVarVecShift>,6251 avx512_var_shift_w<0x11, "vpsravw", X86vsrav, SchedWriteVarVecShift>;6252 6253defm VPSRLV : avx512_var_shift_types<0x45, "vpsrlv", X86vsrlv, SchedWriteVarVecShift>,6254 avx512_var_shift_w<0x10, "vpsrlvw", X86vsrlv, SchedWriteVarVecShift>;6255 6256defm VPRORV : avx512_var_shift_types<0x14, "vprorv", rotr, SchedWriteVarVecShift>;6257defm VPROLV : avx512_var_shift_types<0x15, "vprolv", rotl, SchedWriteVarVecShift>;6258 6259defm : avx512_var_shift_lowering<avx512vl_i64_info, "VPSRAVQ", X86vsrav, [HasAVX512, NoVLX]>;6260defm : avx512_var_shift_lowering<avx512vl_i16_info, "VPSLLVW", X86vshlv, [HasBWI, NoVLX]>;6261defm : avx512_var_shift_lowering<avx512vl_i16_info, "VPSRAVW", X86vsrav, [HasBWI, NoVLX]>;6262defm : avx512_var_shift_lowering<avx512vl_i16_info, "VPSRLVW", X86vsrlv, [HasBWI, NoVLX]>;6263 6264 6265// Use 512bit VPROL/VPROLI version to implement v2i64/v4i64 + v4i32/v8i32 in case NoVLX.6266let Predicates = [HasAVX512, NoVLX] in {6267 def : Pat<(v2i64 (rotl (v2i64 VR128X:$src1), (v2i64 VR128X:$src2))),6268 (EXTRACT_SUBREG (v8i646269 (VPROLVQZrr6270 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),6271 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src2, sub_xmm)))),6272 sub_xmm)>;6273 def : Pat<(v4i64 (rotl (v4i64 VR256X:$src1), (v4i64 VR256X:$src2))),6274 (EXTRACT_SUBREG (v8i646275 (VPROLVQZrr6276 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),6277 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src2, sub_ymm)))),6278 sub_ymm)>;6279 6280 def : Pat<(v4i32 (rotl (v4i32 VR128X:$src1), (v4i32 VR128X:$src2))),6281 (EXTRACT_SUBREG (v16i326282 (VPROLVDZrr6283 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),6284 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src2, sub_xmm)))),6285 sub_xmm)>;6286 def : Pat<(v8i32 (rotl (v8i32 VR256X:$src1), (v8i32 VR256X:$src2))),6287 (EXTRACT_SUBREG (v16i326288 (VPROLVDZrr6289 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),6290 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src2, sub_ymm)))),6291 sub_ymm)>;6292 6293 def : Pat<(v2i64 (X86vrotli (v2i64 VR128X:$src1), (i8 timm:$src2))),6294 (EXTRACT_SUBREG (v8i646295 (VPROLQZri6296 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),6297 timm:$src2)), sub_xmm)>;6298 def : Pat<(v4i64 (X86vrotli (v4i64 VR256X:$src1), (i8 timm:$src2))),6299 (EXTRACT_SUBREG (v8i646300 (VPROLQZri6301 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),6302 timm:$src2)), sub_ymm)>;6303 6304 def : Pat<(v4i32 (X86vrotli (v4i32 VR128X:$src1), (i8 timm:$src2))),6305 (EXTRACT_SUBREG (v16i326306 (VPROLDZri6307 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),6308 timm:$src2)), sub_xmm)>;6309 def : Pat<(v8i32 (X86vrotli (v8i32 VR256X:$src1), (i8 timm:$src2))),6310 (EXTRACT_SUBREG (v16i326311 (VPROLDZri6312 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),6313 timm:$src2)), sub_ymm)>;6314}6315 6316// Use 512bit VPROR/VPRORI version to implement v2i64/v4i64 + v4i32/v8i32 in case NoVLX.6317let Predicates = [HasAVX512, NoVLX] in {6318 def : Pat<(v2i64 (rotr (v2i64 VR128X:$src1), (v2i64 VR128X:$src2))),6319 (EXTRACT_SUBREG (v8i646320 (VPRORVQZrr6321 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),6322 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src2, sub_xmm)))),6323 sub_xmm)>;6324 def : Pat<(v4i64 (rotr (v4i64 VR256X:$src1), (v4i64 VR256X:$src2))),6325 (EXTRACT_SUBREG (v8i646326 (VPRORVQZrr6327 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),6328 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src2, sub_ymm)))),6329 sub_ymm)>;6330 6331 def : Pat<(v4i32 (rotr (v4i32 VR128X:$src1), (v4i32 VR128X:$src2))),6332 (EXTRACT_SUBREG (v16i326333 (VPRORVDZrr6334 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),6335 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src2, sub_xmm)))),6336 sub_xmm)>;6337 def : Pat<(v8i32 (rotr (v8i32 VR256X:$src1), (v8i32 VR256X:$src2))),6338 (EXTRACT_SUBREG (v16i326339 (VPRORVDZrr6340 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),6341 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src2, sub_ymm)))),6342 sub_ymm)>;6343 6344 def : Pat<(v2i64 (X86vrotri (v2i64 VR128X:$src1), (i8 timm:$src2))),6345 (EXTRACT_SUBREG (v8i646346 (VPRORQZri6347 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),6348 timm:$src2)), sub_xmm)>;6349 def : Pat<(v4i64 (X86vrotri (v4i64 VR256X:$src1), (i8 timm:$src2))),6350 (EXTRACT_SUBREG (v8i646351 (VPRORQZri6352 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),6353 timm:$src2)), sub_ymm)>;6354 6355 def : Pat<(v4i32 (X86vrotri (v4i32 VR128X:$src1), (i8 timm:$src2))),6356 (EXTRACT_SUBREG (v16i326357 (VPRORDZri6358 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),6359 timm:$src2)), sub_xmm)>;6360 def : Pat<(v8i32 (X86vrotri (v8i32 VR256X:$src1), (i8 timm:$src2))),6361 (EXTRACT_SUBREG (v16i326362 (VPRORDZri6363 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),6364 timm:$src2)), sub_ymm)>;6365}6366 6367//===-------------------------------------------------------------------===//6368// 1-src variable permutation VPERMW/D/Q6369//===-------------------------------------------------------------------===//6370 6371multiclass avx512_vperm_dq_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,6372 X86FoldableSchedWrite sched, AVX512VLVectorVTInfo _> {6373 let Predicates = [HasAVX512] in6374 defm Z : avx512_var_shift<opc, OpcodeStr, OpNode, sched, _.info512>,6375 avx512_var_shift_mb<opc, OpcodeStr, OpNode, sched, _.info512>, EVEX_V512;6376 6377 let Predicates = [HasAVX512, HasVLX] in6378 defm Z256 : avx512_var_shift<opc, OpcodeStr, OpNode, sched, _.info256>,6379 avx512_var_shift_mb<opc, OpcodeStr, OpNode, sched, _.info256>, EVEX_V256;6380}6381 6382multiclass avx512_vpermi_dq_sizes<bits<8> opc, Format ImmFormR, Format ImmFormM,6383 string OpcodeStr, SDNode OpNode,6384 X86FoldableSchedWrite sched, AVX512VLVectorVTInfo VTInfo> {6385 let Predicates = [HasAVX512] in6386 defm Z: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,6387 sched, VTInfo.info512>,6388 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,6389 sched, VTInfo.info512>, EVEX_V512;6390 let Predicates = [HasAVX512, HasVLX] in6391 defm Z256: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,6392 sched, VTInfo.info256>,6393 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,6394 sched, VTInfo.info256>, EVEX_V256;6395}6396 6397multiclass avx512_vperm_bw<bits<8> opc, string OpcodeStr,6398 Predicate prd, SDNode OpNode,6399 X86FoldableSchedWrite sched, AVX512VLVectorVTInfo _> {6400 let Predicates = [prd] in6401 defm Z: avx512_var_shift<opc, OpcodeStr, OpNode, sched, _.info512>,6402 EVEX_V512 ;6403 let Predicates = [HasVLX, prd] in {6404 defm Z256: avx512_var_shift<opc, OpcodeStr, OpNode, sched, _.info256>,6405 EVEX_V256 ;6406 defm Z128: avx512_var_shift<opc, OpcodeStr, OpNode, sched, _.info128>,6407 EVEX_V128 ;6408 }6409}6410 6411defm VPERMW : avx512_vperm_bw<0x8D, "vpermw", HasBWI, X86VPermv,6412 WriteVarShuffle256, avx512vl_i16_info>, REX_W;6413defm VPERMB : avx512_vperm_bw<0x8D, "vpermb", HasVBMI, X86VPermv,6414 WriteVarShuffle256, avx512vl_i8_info>;6415 6416defm VPERMD : avx512_vperm_dq_sizes<0x36, "vpermd", X86VPermv,6417 WriteVarShuffle256, avx512vl_i32_info>;6418defm VPERMQ : avx512_vperm_dq_sizes<0x36, "vpermq", X86VPermv,6419 WriteVarShuffle256, avx512vl_i64_info>, REX_W;6420defm VPERMPS : avx512_vperm_dq_sizes<0x16, "vpermps", X86VPermv,6421 WriteFVarShuffle256, avx512vl_f32_info>;6422defm VPERMPD : avx512_vperm_dq_sizes<0x16, "vpermpd", X86VPermv,6423 WriteFVarShuffle256, avx512vl_f64_info>, REX_W;6424 6425defm VPERMQ : avx512_vpermi_dq_sizes<0x00, MRMSrcReg, MRMSrcMem, "vpermq",6426 X86VPermi, WriteShuffle256, avx512vl_i64_info>,6427 EVEX, AVX512AIi8Base, EVEX_CD8<64, CD8VF>, REX_W;6428defm VPERMPD : avx512_vpermi_dq_sizes<0x01, MRMSrcReg, MRMSrcMem, "vpermpd",6429 X86VPermi, WriteFShuffle256, avx512vl_f64_info>,6430 EVEX, AVX512AIi8Base, EVEX_CD8<64, CD8VF>, REX_W;6431 6432//===----------------------------------------------------------------------===//6433// AVX-512 - VPERMIL6434//===----------------------------------------------------------------------===//6435 6436multiclass avx512_permil_vec<bits<8> OpcVar, string OpcodeStr, SDNode OpNode,6437 X86FoldableSchedWrite sched, X86VectorVTInfo _,6438 X86VectorVTInfo Ctrl> {6439 defm rr: AVX512_maskable<OpcVar, MRMSrcReg, _, (outs _.RC:$dst),6440 (ins _.RC:$src1, Ctrl.RC:$src2), OpcodeStr,6441 "$src2, $src1", "$src1, $src2",6442 (_.VT (OpNode _.RC:$src1,6443 (Ctrl.VT Ctrl.RC:$src2)))>,6444 T8, PD, EVEX, VVVV, Sched<[sched]>;6445 let mayLoad = 1 in {6446 defm rm: AVX512_maskable<OpcVar, MRMSrcMem, _, (outs _.RC:$dst),6447 (ins _.RC:$src1, Ctrl.MemOp:$src2), OpcodeStr,6448 "$src2, $src1", "$src1, $src2",6449 (_.VT (OpNode6450 _.RC:$src1,6451 (Ctrl.VT (Ctrl.LdFrag addr:$src2))))>,6452 T8, PD, EVEX, VVVV, EVEX_CD8<_.EltSize, CD8VF>,6453 Sched<[sched.Folded, sched.ReadAfterFold]>;6454 defm rmb: AVX512_maskable<OpcVar, MRMSrcMem, _, (outs _.RC:$dst),6455 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,6456 "${src2}"#_.BroadcastStr#", $src1",6457 "$src1, ${src2}"#_.BroadcastStr,6458 (_.VT (OpNode6459 _.RC:$src1,6460 (Ctrl.VT (Ctrl.BroadcastLdFrag addr:$src2))))>,6461 T8, PD, EVEX, VVVV, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>,6462 Sched<[sched.Folded, sched.ReadAfterFold]>;6463 }6464}6465 6466multiclass avx512_permil_vec_common<string OpcodeStr, bits<8> OpcVar,6467 X86SchedWriteWidths sched,6468 AVX512VLVectorVTInfo _,6469 AVX512VLVectorVTInfo Ctrl> {6470 let Predicates = [HasAVX512] in {6471 defm Z : avx512_permil_vec<OpcVar, OpcodeStr, X86VPermilpv, sched.ZMM,6472 _.info512, Ctrl.info512>, EVEX_V512;6473 }6474 let Predicates = [HasAVX512, HasVLX] in {6475 defm Z128 : avx512_permil_vec<OpcVar, OpcodeStr, X86VPermilpv, sched.XMM,6476 _.info128, Ctrl.info128>, EVEX_V128;6477 defm Z256 : avx512_permil_vec<OpcVar, OpcodeStr, X86VPermilpv, sched.YMM,6478 _.info256, Ctrl.info256>, EVEX_V256;6479 }6480}6481 6482multiclass avx512_permil<string OpcodeStr, bits<8> OpcImm, bits<8> OpcVar,6483 AVX512VLVectorVTInfo _, AVX512VLVectorVTInfo Ctrl>{6484 defm NAME: avx512_permil_vec_common<OpcodeStr, OpcVar, SchedWriteFVarShuffle,6485 _, Ctrl>;6486 defm NAME: avx512_shift_rmi_sizes<OpcImm, MRMSrcReg, MRMSrcMem, OpcodeStr,6487 X86VPermilpi, SchedWriteFShuffle, _>,6488 EVEX, AVX512AIi8Base, EVEX_CD8<_.info128.EltSize, CD8VF>;6489}6490 6491let ExeDomain = SSEPackedSingle in6492defm VPERMILPS : avx512_permil<"vpermilps", 0x04, 0x0C, avx512vl_f32_info,6493 avx512vl_i32_info>;6494let ExeDomain = SSEPackedDouble in6495defm VPERMILPD : avx512_permil<"vpermilpd", 0x05, 0x0D, avx512vl_f64_info,6496 avx512vl_i64_info>, REX_W;6497 6498//===----------------------------------------------------------------------===//6499// AVX-512 - VPSHUFD, VPSHUFLW, VPSHUFHW6500//===----------------------------------------------------------------------===//6501 6502defm VPSHUFD : avx512_shift_rmi_sizes<0x70, MRMSrcReg, MRMSrcMem, "vpshufd",6503 X86PShufd, SchedWriteShuffle, avx512vl_i32_info>,6504 EVEX, AVX512BIi8Base, EVEX_CD8<32, CD8VF>;6505defm VPSHUFH : avx512_shift_rmi_w<0x70, MRMSrcReg, MRMSrcMem, "vpshufhw",6506 X86PShufhw, SchedWriteShuffle>,6507 EVEX, AVX512XSIi8Base;6508defm VPSHUFL : avx512_shift_rmi_w<0x70, MRMSrcReg, MRMSrcMem, "vpshuflw",6509 X86PShuflw, SchedWriteShuffle>,6510 EVEX, AVX512XDIi8Base;6511 6512//===----------------------------------------------------------------------===//6513// AVX-512 - VPSHUFB6514//===----------------------------------------------------------------------===//6515 6516multiclass avx512_pshufb_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,6517 X86SchedWriteWidths sched> {6518 let Predicates = [HasBWI] in6519 defm Z: avx512_var_shift<opc, OpcodeStr, OpNode, sched.ZMM, v64i8_info>,6520 EVEX_V512;6521 6522 let Predicates = [HasVLX, HasBWI] in {6523 defm Z256: avx512_var_shift<opc, OpcodeStr, OpNode, sched.YMM, v32i8x_info>,6524 EVEX_V256;6525 defm Z128: avx512_var_shift<opc, OpcodeStr, OpNode, sched.XMM, v16i8x_info>,6526 EVEX_V128;6527 }6528}6529 6530defm VPSHUFB: avx512_pshufb_sizes<0x00, "vpshufb", X86pshufb,6531 SchedWriteVarShuffle>, WIG;6532 6533//===----------------------------------------------------------------------===//6534// Move Low to High and High to Low packed FP Instructions6535//===----------------------------------------------------------------------===//6536 6537def VMOVLHPSZrr : AVX512PSI<0x16, MRMSrcReg, (outs VR128X:$dst),6538 (ins VR128X:$src1, VR128X:$src2),6539 "vmovlhps\t{$src2, $src1, $dst|$dst, $src1, $src2}",6540 [(set VR128X:$dst, (v4f32 (X86Movlhps VR128X:$src1, VR128X:$src2)))]>,6541 Sched<[SchedWriteFShuffle.XMM]>, EVEX, VVVV;6542let isCommutable = 1 in6543def VMOVHLPSZrr : AVX512PSI<0x12, MRMSrcReg, (outs VR128X:$dst),6544 (ins VR128X:$src1, VR128X:$src2),6545 "vmovhlps\t{$src2, $src1, $dst|$dst, $src1, $src2}",6546 [(set VR128X:$dst, (v4f32 (X86Movhlps VR128X:$src1, VR128X:$src2)))]>,6547 Sched<[SchedWriteFShuffle.XMM]>, EVEX, VVVV;6548 6549//===----------------------------------------------------------------------===//6550// VMOVHPS/PD VMOVLPS Instructions6551// All patterns was taken from SSS implementation.6552//===----------------------------------------------------------------------===//6553 6554multiclass avx512_mov_hilo_packed<bits<8> opc, string OpcodeStr,6555 SDPatternOperator OpNode,6556 X86VectorVTInfo _> {6557 let hasSideEffects = 0, mayLoad = 1, ExeDomain = _.ExeDomain in6558 def rm : AVX512<opc, MRMSrcMem, (outs _.RC:$dst),6559 (ins _.RC:$src1, f64mem:$src2),6560 !strconcat(OpcodeStr,6561 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),6562 [(set _.RC:$dst,6563 (OpNode _.RC:$src1,6564 (_.VT (bitconvert6565 (v2f64 (scalar_to_vector (loadf64 addr:$src2)))))))]>,6566 Sched<[SchedWriteFShuffle.XMM.Folded, SchedWriteFShuffle.XMM.ReadAfterFold]>, EVEX, VVVV;6567}6568 6569// No patterns for MOVLPS/MOVHPS as the Movlhps node should only be created in6570// SSE1. And MOVLPS pattern is even more complex.6571defm VMOVHPSZ128 : avx512_mov_hilo_packed<0x16, "vmovhps", null_frag,6572 v4f32x_info>, EVEX_CD8<32, CD8VT2>, TB;6573defm VMOVHPDZ128 : avx512_mov_hilo_packed<0x16, "vmovhpd", X86Unpckl,6574 v2f64x_info>, EVEX_CD8<64, CD8VT1>, TB, PD, REX_W;6575defm VMOVLPSZ128 : avx512_mov_hilo_packed<0x12, "vmovlps", null_frag,6576 v4f32x_info>, EVEX_CD8<32, CD8VT2>, TB;6577defm VMOVLPDZ128 : avx512_mov_hilo_packed<0x12, "vmovlpd", X86Movsd,6578 v2f64x_info>, EVEX_CD8<64, CD8VT1>, TB, PD, REX_W;6579 6580let Predicates = [HasAVX512] in {6581 // VMOVHPD patterns6582 def : Pat<(v2f64 (X86Unpckl VR128X:$src1, (X86vzload64 addr:$src2))),6583 (VMOVHPDZ128rm VR128X:$src1, addr:$src2)>;6584 6585 // VMOVLPD patterns6586 def : Pat<(v2f64 (X86Movsd VR128X:$src1, (X86vzload64 addr:$src2))),6587 (VMOVLPDZ128rm VR128X:$src1, addr:$src2)>;6588}6589 6590let SchedRW = [WriteFStore] in {6591let mayStore = 1, hasSideEffects = 0 in6592def VMOVHPSZ128mr : AVX512PSI<0x17, MRMDestMem, (outs),6593 (ins f64mem:$dst, VR128X:$src),6594 "vmovhps\t{$src, $dst|$dst, $src}",6595 []>, EVEX, EVEX_CD8<32, CD8VT2>;6596def VMOVHPDZ128mr : AVX512PDI<0x17, MRMDestMem, (outs),6597 (ins f64mem:$dst, VR128X:$src),6598 "vmovhpd\t{$src, $dst|$dst, $src}",6599 [(store (f64 (extractelt6600 (v2f64 (X86Unpckh VR128X:$src, VR128X:$src)),6601 (iPTR 0))), addr:$dst)]>,6602 EVEX, EVEX_CD8<64, CD8VT1>, REX_W;6603let mayStore = 1, hasSideEffects = 0 in6604def VMOVLPSZ128mr : AVX512PSI<0x13, MRMDestMem, (outs),6605 (ins f64mem:$dst, VR128X:$src),6606 "vmovlps\t{$src, $dst|$dst, $src}",6607 []>, EVEX, EVEX_CD8<32, CD8VT2>;6608def VMOVLPDZ128mr : AVX512PDI<0x13, MRMDestMem, (outs),6609 (ins f64mem:$dst, VR128X:$src),6610 "vmovlpd\t{$src, $dst|$dst, $src}",6611 [(store (f64 (extractelt (v2f64 VR128X:$src),6612 (iPTR 0))), addr:$dst)]>,6613 EVEX, EVEX_CD8<64, CD8VT1>, REX_W;6614} // SchedRW6615 6616let Predicates = [HasAVX512] in {6617 // VMOVHPD patterns6618 def : Pat<(store (f64 (extractelt6619 (v2f64 (X86VPermilpi VR128X:$src, (i8 1))),6620 (iPTR 0))), addr:$dst),6621 (VMOVHPDZ128mr addr:$dst, VR128X:$src)>;6622}6623//===----------------------------------------------------------------------===//6624// FMA - Fused Multiply Operations6625//6626 6627multiclass avx512_fma3p_213_rm<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,6628 SDNode MaskOpNode, X86FoldableSchedWrite sched,6629 X86VectorVTInfo _> {6630 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain, hasSideEffects = 0,6631 Uses = [MXCSR], mayRaiseFPException = 1 in {6632 defm r: AVX512_maskable_fma<opc, MRMSrcReg, _, (outs _.RC:$dst),6633 (ins _.RC:$src2, _.RC:$src3),6634 OpcodeStr, "$src3, $src2", "$src2, $src3",6635 (_.VT (OpNode _.RC:$src2, _.RC:$src1, _.RC:$src3)),6636 (_.VT (MaskOpNode _.RC:$src2, _.RC:$src1, _.RC:$src3)), 1, 1>,6637 EVEX, VVVV, Sched<[sched]>;6638 6639 defm m: AVX512_maskable_fma<opc, MRMSrcMem, _, (outs _.RC:$dst),6640 (ins _.RC:$src2, _.MemOp:$src3),6641 OpcodeStr, "$src3, $src2", "$src2, $src3",6642 (_.VT (OpNode _.RC:$src2, _.RC:$src1, (_.LdFrag addr:$src3))),6643 (_.VT (MaskOpNode _.RC:$src2, _.RC:$src1, (_.LdFrag addr:$src3))), 1, 0>,6644 EVEX, VVVV, Sched<[sched.Folded, sched.ReadAfterFold,6645 sched.ReadAfterFold]>;6646 6647 defm mb: AVX512_maskable_fma<opc, MRMSrcMem, _, (outs _.RC:$dst),6648 (ins _.RC:$src2, _.ScalarMemOp:$src3),6649 OpcodeStr, !strconcat("${src3}", _.BroadcastStr,", $src2"),6650 !strconcat("$src2, ${src3}", _.BroadcastStr ),6651 (OpNode _.RC:$src2,6652 _.RC:$src1,(_.VT (_.BroadcastLdFrag addr:$src3))),6653 (MaskOpNode _.RC:$src2,6654 _.RC:$src1,(_.VT (_.BroadcastLdFrag addr:$src3))), 1, 0>,6655 EVEX, VVVV, EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold,6656 sched.ReadAfterFold]>;6657 }6658}6659 6660multiclass avx512_fma3_213_round<bits<8> opc, string OpcodeStr, SDNode OpNode,6661 X86FoldableSchedWrite sched,6662 X86VectorVTInfo _> {6663 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain, hasSideEffects = 0,6664 Uses = [MXCSR] in6665 defm rb: AVX512_maskable_fma<opc, MRMSrcReg, _, (outs _.RC:$dst),6666 (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc),6667 OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc",6668 (_.VT ( OpNode _.RC:$src2, _.RC:$src1, _.RC:$src3, (i32 timm:$rc))),6669 (_.VT ( OpNode _.RC:$src2, _.RC:$src1, _.RC:$src3, (i32 timm:$rc))), 1, 1>,6670 EVEX, VVVV, EVEX_B, EVEX_RC, Sched<[sched]>;6671}6672 6673multiclass avx512_fma3p_213_common<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,6674 SDNode MaskOpNode, SDNode OpNodeRnd,6675 X86SchedWriteWidths sched,6676 AVX512VLVectorVTInfo _,6677 Predicate prd = HasAVX512> {6678 let Predicates = [prd] in {6679 defm Z : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, MaskOpNode,6680 sched.ZMM, _.info512>,6681 avx512_fma3_213_round<opc, OpcodeStr, OpNodeRnd, sched.ZMM,6682 _.info512>,6683 EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>;6684 }6685 let Predicates = [HasVLX, prd] in {6686 defm Z256 : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, MaskOpNode,6687 sched.YMM, _.info256>,6688 EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>;6689 defm Z128 : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, MaskOpNode,6690 sched.XMM, _.info128>,6691 EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>;6692 }6693}6694 6695multiclass avx512_fma3p_213_f<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,6696 SDNode MaskOpNode, SDNode OpNodeRnd> {6697 defm PH : avx512_fma3p_213_common<opc, OpcodeStr#"ph", OpNode, MaskOpNode,6698 OpNodeRnd, SchedWriteFMA,6699 avx512vl_f16_info, HasFP16>, T_MAP6, PD;6700 defm PS : avx512_fma3p_213_common<opc, OpcodeStr#"ps", OpNode, MaskOpNode,6701 OpNodeRnd, SchedWriteFMA,6702 avx512vl_f32_info>, T8, PD;6703 defm PD : avx512_fma3p_213_common<opc, OpcodeStr#"pd", OpNode, MaskOpNode,6704 OpNodeRnd, SchedWriteFMA,6705 avx512vl_f64_info>, T8, PD, REX_W;6706}6707 6708defm VFMADD213 : avx512_fma3p_213_f<0xA8, "vfmadd213", any_fma,6709 fma, X86FmaddRnd>;6710defm VFMSUB213 : avx512_fma3p_213_f<0xAA, "vfmsub213", X86any_Fmsub,6711 X86Fmsub, X86FmsubRnd>;6712defm VFMADDSUB213 : avx512_fma3p_213_f<0xA6, "vfmaddsub213", X86Fmaddsub,6713 X86Fmaddsub, X86FmaddsubRnd>;6714defm VFMSUBADD213 : avx512_fma3p_213_f<0xA7, "vfmsubadd213", X86Fmsubadd,6715 X86Fmsubadd, X86FmsubaddRnd>;6716defm VFNMADD213 : avx512_fma3p_213_f<0xAC, "vfnmadd213", X86any_Fnmadd,6717 X86Fnmadd, X86FnmaddRnd>;6718defm VFNMSUB213 : avx512_fma3p_213_f<0xAE, "vfnmsub213", X86any_Fnmsub,6719 X86Fnmsub, X86FnmsubRnd>;6720 6721 6722multiclass avx512_fma3p_231_rm<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,6723 SDNode MaskOpNode, X86FoldableSchedWrite sched,6724 X86VectorVTInfo _> {6725 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain, hasSideEffects = 0,6726 Uses = [MXCSR], mayRaiseFPException = 1 in {6727 defm r: AVX512_maskable_fma<opc, MRMSrcReg, _, (outs _.RC:$dst),6728 (ins _.RC:$src2, _.RC:$src3),6729 OpcodeStr, "$src3, $src2", "$src2, $src3",6730 (null_frag),6731 (_.VT (MaskOpNode _.RC:$src2, _.RC:$src3, _.RC:$src1)), 1, 1>,6732 EVEX, VVVV, Sched<[sched]>;6733 6734 defm m: AVX512_maskable_fma<opc, MRMSrcMem, _, (outs _.RC:$dst),6735 (ins _.RC:$src2, _.MemOp:$src3),6736 OpcodeStr, "$src3, $src2", "$src2, $src3",6737 (_.VT (OpNode _.RC:$src2, (_.LdFrag addr:$src3), _.RC:$src1)),6738 (_.VT (MaskOpNode _.RC:$src2, (_.LdFrag addr:$src3), _.RC:$src1)), 1, 0>,6739 EVEX, VVVV, Sched<[sched.Folded, sched.ReadAfterFold,6740 sched.ReadAfterFold]>;6741 6742 defm mb: AVX512_maskable_fma<opc, MRMSrcMem, _, (outs _.RC:$dst),6743 (ins _.RC:$src2, _.ScalarMemOp:$src3),6744 OpcodeStr, "${src3}"#_.BroadcastStr#", $src2",6745 "$src2, ${src3}"#_.BroadcastStr,6746 (_.VT (OpNode _.RC:$src2,6747 (_.VT (_.BroadcastLdFrag addr:$src3)),6748 _.RC:$src1)),6749 (_.VT (MaskOpNode _.RC:$src2,6750 (_.VT (_.BroadcastLdFrag addr:$src3)),6751 _.RC:$src1)), 1, 0>, EVEX, VVVV, EVEX_B,6752 Sched<[sched.Folded, sched.ReadAfterFold,6753 sched.ReadAfterFold]>;6754 }6755}6756 6757multiclass avx512_fma3_231_round<bits<8> opc, string OpcodeStr, SDNode OpNode,6758 X86FoldableSchedWrite sched,6759 X86VectorVTInfo _> {6760 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain, hasSideEffects = 0,6761 Uses = [MXCSR] in6762 defm rb: AVX512_maskable_fma<opc, MRMSrcReg, _, (outs _.RC:$dst),6763 (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc),6764 OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc",6765 (null_frag),6766 (_.VT (OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1, (i32 timm:$rc))),6767 1, 1>, EVEX, VVVV, EVEX_B, EVEX_RC, Sched<[sched]>;6768}6769 6770multiclass avx512_fma3p_231_common<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,6771 SDNode MaskOpNode, SDNode OpNodeRnd,6772 X86SchedWriteWidths sched,6773 AVX512VLVectorVTInfo _,6774 Predicate prd = HasAVX512> {6775 let Predicates = [prd] in {6776 defm Z : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, MaskOpNode,6777 sched.ZMM, _.info512>,6778 avx512_fma3_231_round<opc, OpcodeStr, OpNodeRnd, sched.ZMM,6779 _.info512>,6780 EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>;6781 }6782 let Predicates = [HasVLX, prd] in {6783 defm Z256 : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, MaskOpNode,6784 sched.YMM, _.info256>,6785 EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>;6786 defm Z128 : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, MaskOpNode,6787 sched.XMM, _.info128>,6788 EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>;6789 }6790}6791 6792multiclass avx512_fma3p_231_f<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,6793 SDNode MaskOpNode, SDNode OpNodeRnd > {6794 defm PH : avx512_fma3p_231_common<opc, OpcodeStr#"ph", OpNode, MaskOpNode,6795 OpNodeRnd, SchedWriteFMA,6796 avx512vl_f16_info, HasFP16>, T_MAP6, PD;6797 defm PS : avx512_fma3p_231_common<opc, OpcodeStr#"ps", OpNode, MaskOpNode,6798 OpNodeRnd, SchedWriteFMA,6799 avx512vl_f32_info>, T8, PD;6800 defm PD : avx512_fma3p_231_common<opc, OpcodeStr#"pd", OpNode, MaskOpNode,6801 OpNodeRnd, SchedWriteFMA,6802 avx512vl_f64_info>, T8, PD, REX_W;6803}6804 6805defm VFMADD231 : avx512_fma3p_231_f<0xB8, "vfmadd231", any_fma,6806 fma, X86FmaddRnd>;6807defm VFMSUB231 : avx512_fma3p_231_f<0xBA, "vfmsub231", X86any_Fmsub,6808 X86Fmsub, X86FmsubRnd>;6809defm VFMADDSUB231 : avx512_fma3p_231_f<0xB6, "vfmaddsub231", X86Fmaddsub,6810 X86Fmaddsub, X86FmaddsubRnd>;6811defm VFMSUBADD231 : avx512_fma3p_231_f<0xB7, "vfmsubadd231", X86Fmsubadd,6812 X86Fmsubadd, X86FmsubaddRnd>;6813defm VFNMADD231 : avx512_fma3p_231_f<0xBC, "vfnmadd231", X86any_Fnmadd,6814 X86Fnmadd, X86FnmaddRnd>;6815defm VFNMSUB231 : avx512_fma3p_231_f<0xBE, "vfnmsub231", X86any_Fnmsub,6816 X86Fnmsub, X86FnmsubRnd>;6817 6818multiclass avx512_fma3p_132_rm<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,6819 SDNode MaskOpNode, X86FoldableSchedWrite sched,6820 X86VectorVTInfo _> {6821 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain, hasSideEffects = 0,6822 Uses = [MXCSR], mayRaiseFPException = 1 in {6823 defm r: AVX512_maskable_fma<opc, MRMSrcReg, _, (outs _.RC:$dst),6824 (ins _.RC:$src2, _.RC:$src3),6825 OpcodeStr, "$src3, $src2", "$src2, $src3",6826 (null_frag),6827 (_.VT (MaskOpNode _.RC:$src1, _.RC:$src3, _.RC:$src2)), 1, 1>,6828 EVEX, VVVV, Sched<[sched]>;6829 6830 // Pattern is 312 order so that the load is in a different place from the6831 // 213 and 231 patterns this helps tablegen's duplicate pattern detection.6832 defm m: AVX512_maskable_fma<opc, MRMSrcMem, _, (outs _.RC:$dst),6833 (ins _.RC:$src2, _.MemOp:$src3),6834 OpcodeStr, "$src3, $src2", "$src2, $src3",6835 (_.VT (OpNode (_.LdFrag addr:$src3), _.RC:$src1, _.RC:$src2)),6836 (_.VT (MaskOpNode (_.LdFrag addr:$src3), _.RC:$src1, _.RC:$src2)), 1, 0>,6837 EVEX, VVVV, Sched<[sched.Folded, sched.ReadAfterFold,6838 sched.ReadAfterFold]>;6839 6840 // Pattern is 312 order so that the load is in a different place from the6841 // 213 and 231 patterns this helps tablegen's duplicate pattern detection.6842 defm mb: AVX512_maskable_fma<opc, MRMSrcMem, _, (outs _.RC:$dst),6843 (ins _.RC:$src2, _.ScalarMemOp:$src3),6844 OpcodeStr, "${src3}"#_.BroadcastStr#", $src2",6845 "$src2, ${src3}"#_.BroadcastStr,6846 (_.VT (OpNode (_.VT (_.BroadcastLdFrag addr:$src3)),6847 _.RC:$src1, _.RC:$src2)),6848 (_.VT (MaskOpNode (_.VT (_.BroadcastLdFrag addr:$src3)),6849 _.RC:$src1, _.RC:$src2)), 1, 0>,6850 EVEX, VVVV, EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold,6851 sched.ReadAfterFold]>;6852 }6853}6854 6855multiclass avx512_fma3_132_round<bits<8> opc, string OpcodeStr, SDNode OpNode,6856 X86FoldableSchedWrite sched,6857 X86VectorVTInfo _> {6858 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain, hasSideEffects = 0,6859 Uses = [MXCSR] in6860 defm rb: AVX512_maskable_fma<opc, MRMSrcReg, _, (outs _.RC:$dst),6861 (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc),6862 OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc",6863 (null_frag),6864 (_.VT (OpNode _.RC:$src1, _.RC:$src3, _.RC:$src2, (i32 timm:$rc))),6865 1, 1>, EVEX, VVVV, EVEX_B, EVEX_RC, Sched<[sched]>;6866}6867 6868multiclass avx512_fma3p_132_common<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,6869 SDNode MaskOpNode, SDNode OpNodeRnd,6870 X86SchedWriteWidths sched,6871 AVX512VLVectorVTInfo _,6872 Predicate prd = HasAVX512> {6873 let Predicates = [prd] in {6874 defm Z : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, MaskOpNode,6875 sched.ZMM, _.info512>,6876 avx512_fma3_132_round<opc, OpcodeStr, OpNodeRnd, sched.ZMM,6877 _.info512>,6878 EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>;6879 }6880 let Predicates = [HasVLX, prd] in {6881 defm Z256 : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, MaskOpNode,6882 sched.YMM, _.info256>,6883 EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>;6884 defm Z128 : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, MaskOpNode,6885 sched.XMM, _.info128>,6886 EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>;6887 }6888}6889 6890multiclass avx512_fma3p_132_f<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,6891 SDNode MaskOpNode, SDNode OpNodeRnd > {6892 defm PH : avx512_fma3p_132_common<opc, OpcodeStr#"ph", OpNode, MaskOpNode,6893 OpNodeRnd, SchedWriteFMA,6894 avx512vl_f16_info, HasFP16>, T_MAP6, PD;6895 defm PS : avx512_fma3p_132_common<opc, OpcodeStr#"ps", OpNode, MaskOpNode,6896 OpNodeRnd, SchedWriteFMA,6897 avx512vl_f32_info>, T8, PD;6898 defm PD : avx512_fma3p_132_common<opc, OpcodeStr#"pd", OpNode, MaskOpNode,6899 OpNodeRnd, SchedWriteFMA,6900 avx512vl_f64_info>, T8, PD, REX_W;6901}6902 6903defm VFMADD132 : avx512_fma3p_132_f<0x98, "vfmadd132", any_fma,6904 fma, X86FmaddRnd>;6905defm VFMSUB132 : avx512_fma3p_132_f<0x9A, "vfmsub132", X86any_Fmsub,6906 X86Fmsub, X86FmsubRnd>;6907defm VFMADDSUB132 : avx512_fma3p_132_f<0x96, "vfmaddsub132", X86Fmaddsub,6908 X86Fmaddsub, X86FmaddsubRnd>;6909defm VFMSUBADD132 : avx512_fma3p_132_f<0x97, "vfmsubadd132", X86Fmsubadd,6910 X86Fmsubadd, X86FmsubaddRnd>;6911defm VFNMADD132 : avx512_fma3p_132_f<0x9C, "vfnmadd132", X86any_Fnmadd,6912 X86Fnmadd, X86FnmaddRnd>;6913defm VFNMSUB132 : avx512_fma3p_132_f<0x9E, "vfnmsub132", X86any_Fnmsub,6914 X86Fnmsub, X86FnmsubRnd>;6915 6916// Scalar FMA6917multiclass avx512_fma3s_common<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,6918 dag RHS_r, dag RHS_m, dag RHS_b, bit MaskOnlyReg> {6919let Constraints = "$src1 = $dst", hasSideEffects = 0 in {6920 defm r: AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),6921 (ins _.RC:$src2, _.RC:$src3), OpcodeStr,6922 "$src3, $src2", "$src2, $src3", (null_frag), 1, 1, 0, "_Int">,6923 EVEX, VVVV, Sched<[SchedWriteFMA.Scl]>, SIMD_EXC;6924 6925 let mayLoad = 1 in6926 defm m: AVX512_maskable_3src_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),6927 (ins _.RC:$src2, _.IntScalarMemOp:$src3), OpcodeStr,6928 "$src3, $src2", "$src2, $src3", (null_frag), 1, 1, 0, "_Int">,6929 EVEX, VVVV, Sched<[SchedWriteFMA.Scl.Folded, SchedWriteFMA.Scl.ReadAfterFold,6930 SchedWriteFMA.Scl.ReadAfterFold]>, SIMD_EXC;6931 6932 let Uses = [MXCSR] in6933 defm rb: AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),6934 (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc),6935 OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc", (null_frag), 1, 1, 0, "_Int">,6936 EVEX, VVVV, EVEX_B, EVEX_RC, Sched<[SchedWriteFMA.Scl]>;6937 6938 let isCodeGenOnly = 1, isCommutable = 1 in {6939 def r : AVX512<opc, MRMSrcReg, (outs _.FRC:$dst),6940 (ins _.FRC:$src1, _.FRC:$src2, _.FRC:$src3),6941 !strconcat(OpcodeStr,6942 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),6943 !if(MaskOnlyReg, [], [RHS_r])>, Sched<[SchedWriteFMA.Scl]>, EVEX, VVVV, SIMD_EXC;6944 def m : AVX512<opc, MRMSrcMem, (outs _.FRC:$dst),6945 (ins _.FRC:$src1, _.FRC:$src2, _.ScalarMemOp:$src3),6946 !strconcat(OpcodeStr,6947 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),6948 [RHS_m]>, Sched<[SchedWriteFMA.Scl.Folded, SchedWriteFMA.Scl.ReadAfterFold,6949 SchedWriteFMA.Scl.ReadAfterFold]>, EVEX, VVVV, SIMD_EXC;6950 6951 let Uses = [MXCSR] in6952 def rb : AVX512<opc, MRMSrcReg, (outs _.FRC:$dst),6953 (ins _.FRC:$src1, _.FRC:$src2, _.FRC:$src3, AVX512RC:$rc),6954 !strconcat(OpcodeStr,6955 "\t{$rc, $src3, $src2, $dst|$dst, $src2, $src3, $rc}"),6956 !if(MaskOnlyReg, [], [RHS_b])>, EVEX_B, EVEX_RC,6957 Sched<[SchedWriteFMA.Scl]>, EVEX, VVVV;6958 }// isCodeGenOnly = 16959}// Constraints = "$src1 = $dst"6960}6961 6962multiclass avx512_fma3s_all<bits<8> opc213, bits<8> opc231, bits<8> opc132,6963 string OpcodeStr, SDPatternOperator OpNode, SDNode OpNodeRnd,6964 X86VectorVTInfo _, string SUFF> {6965 let ExeDomain = _.ExeDomain in {6966 defm NAME#213#SUFF#Z: avx512_fma3s_common<opc213, OpcodeStr#"213"#_.Suffix, _,6967 // Operands for intrinsic are in 123 order to preserve passthu6968 // semantics.6969 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src1,6970 _.FRC:$src3))),6971 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src1,6972 (_.ScalarLdFrag addr:$src3)))),6973 (set _.FRC:$dst, (_.EltVT (OpNodeRnd _.FRC:$src2, _.FRC:$src1,6974 _.FRC:$src3, (i32 timm:$rc)))), 0>;6975 6976 defm NAME#231#SUFF#Z: avx512_fma3s_common<opc231, OpcodeStr#"231"#_.Suffix, _,6977 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src3,6978 _.FRC:$src1))),6979 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2,6980 (_.ScalarLdFrag addr:$src3), _.FRC:$src1))),6981 (set _.FRC:$dst, (_.EltVT (OpNodeRnd _.FRC:$src2, _.FRC:$src3,6982 _.FRC:$src1, (i32 timm:$rc)))), 1>;6983 6984 // One pattern is 312 order so that the load is in a different place from the6985 // 213 and 231 patterns this helps tablegen's duplicate pattern detection.6986 defm NAME#132#SUFF#Z: avx512_fma3s_common<opc132, OpcodeStr#"132"#_.Suffix, _,6987 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src1, _.FRC:$src3,6988 _.FRC:$src2))),6989 (set _.FRC:$dst, (_.EltVT (OpNode (_.ScalarLdFrag addr:$src3),6990 _.FRC:$src1, _.FRC:$src2))),6991 (set _.FRC:$dst, (_.EltVT (OpNodeRnd _.FRC:$src1, _.FRC:$src3,6992 _.FRC:$src2, (i32 timm:$rc)))), 1>;6993 }6994}6995 6996multiclass avx512_fma3s<bits<8> opc213, bits<8> opc231, bits<8> opc132,6997 string OpcodeStr, SDPatternOperator OpNode, SDNode OpNodeRnd> {6998 let Predicates = [HasAVX512] in {6999 defm NAME : avx512_fma3s_all<opc213, opc231, opc132, OpcodeStr, OpNode,7000 OpNodeRnd, f32x_info, "SS">,7001 EVEX_CD8<32, CD8VT1>, VEX_LIG, T8, PD;7002 defm NAME : avx512_fma3s_all<opc213, opc231, opc132, OpcodeStr, OpNode,7003 OpNodeRnd, f64x_info, "SD">,7004 EVEX_CD8<64, CD8VT1>, VEX_LIG, REX_W, T8, PD;7005 }7006 let Predicates = [HasFP16] in {7007 defm NAME : avx512_fma3s_all<opc213, opc231, opc132, OpcodeStr, OpNode,7008 OpNodeRnd, f16x_info, "SH">,7009 EVEX_CD8<16, CD8VT1>, VEX_LIG, T_MAP6, PD;7010 }7011}7012 7013defm VFMADD : avx512_fma3s<0xA9, 0xB9, 0x99, "vfmadd", any_fma, X86FmaddRnd>;7014defm VFMSUB : avx512_fma3s<0xAB, 0xBB, 0x9B, "vfmsub", X86any_Fmsub, X86FmsubRnd>;7015defm VFNMADD : avx512_fma3s<0xAD, 0xBD, 0x9D, "vfnmadd", X86any_Fnmadd, X86FnmaddRnd>;7016defm VFNMSUB : avx512_fma3s<0xAF, 0xBF, 0x9F, "vfnmsub", X86any_Fnmsub, X86FnmsubRnd>;7017 7018multiclass avx512_scalar_fma_patterns<SDPatternOperator Op, SDNode MaskedOp,7019 SDNode RndOp, string Prefix,7020 string Suffix, SDNode Move,7021 X86VectorVTInfo _, PatLeaf ZeroFP,7022 Predicate prd = HasAVX512> {7023 let Predicates = [prd] in {7024 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector7025 (Op _.FRC:$src2,7026 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),7027 _.FRC:$src3))))),7028 (!cast<I>(Prefix#"213"#Suffix#"Zr_Int")7029 VR128X:$src1, (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)),7030 (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)))>;7031 7032 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector7033 (Op _.FRC:$src2, _.FRC:$src3,7034 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))))))),7035 (!cast<I>(Prefix#"231"#Suffix#"Zr_Int")7036 VR128X:$src1, (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)),7037 (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)))>;7038 7039 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector7040 (Op _.FRC:$src2,7041 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),7042 (_.ScalarLdFrag addr:$src3)))))),7043 (!cast<I>(Prefix#"213"#Suffix#"Zm_Int")7044 VR128X:$src1, (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)),7045 addr:$src3)>;7046 7047 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector7048 (Op (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),7049 (_.ScalarLdFrag addr:$src3), _.FRC:$src2))))),7050 (!cast<I>(Prefix#"132"#Suffix#"Zm_Int")7051 VR128X:$src1, (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)),7052 addr:$src3)>;7053 7054 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector7055 (Op _.FRC:$src2, (_.ScalarLdFrag addr:$src3),7056 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))))))),7057 (!cast<I>(Prefix#"231"#Suffix#"Zm_Int")7058 VR128X:$src1, (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)),7059 addr:$src3)>;7060 7061 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector7062 (X86selects_mask VK1WM:$mask,7063 (MaskedOp _.FRC:$src2,7064 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),7065 _.FRC:$src3),7066 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))))))),7067 (!cast<I>(Prefix#"213"#Suffix#"Zrk_Int")7068 VR128X:$src1, VK1WM:$mask,7069 (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)),7070 (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)))>;7071 7072 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector7073 (X86selects_mask VK1WM:$mask,7074 (MaskedOp _.FRC:$src2,7075 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),7076 (_.ScalarLdFrag addr:$src3)),7077 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))))))),7078 (!cast<I>(Prefix#"213"#Suffix#"Zmk_Int")7079 VR128X:$src1, VK1WM:$mask,7080 (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), addr:$src3)>;7081 7082 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector7083 (X86selects_mask VK1WM:$mask,7084 (MaskedOp (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),7085 (_.ScalarLdFrag addr:$src3), _.FRC:$src2),7086 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))))))),7087 (!cast<I>(Prefix#"132"#Suffix#"Zmk_Int")7088 VR128X:$src1, VK1WM:$mask,7089 (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), addr:$src3)>;7090 7091 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector7092 (X86selects_mask VK1WM:$mask,7093 (MaskedOp _.FRC:$src2, _.FRC:$src3,7094 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))),7095 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))))))),7096 (!cast<I>(Prefix#"231"#Suffix#"Zrk_Int")7097 VR128X:$src1, VK1WM:$mask,7098 (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)),7099 (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)))>;7100 7101 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector7102 (X86selects_mask VK1WM:$mask,7103 (MaskedOp _.FRC:$src2, (_.ScalarLdFrag addr:$src3),7104 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))),7105 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))))))),7106 (!cast<I>(Prefix#"231"#Suffix#"Zmk_Int")7107 VR128X:$src1, VK1WM:$mask,7108 (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), addr:$src3)>;7109 7110 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector7111 (X86selects_mask VK1WM:$mask,7112 (MaskedOp _.FRC:$src2,7113 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),7114 _.FRC:$src3),7115 (_.EltVT ZeroFP)))))),7116 (!cast<I>(Prefix#"213"#Suffix#"Zrkz_Int")7117 VR128X:$src1, VK1WM:$mask,7118 (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)),7119 (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)))>;7120 7121 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector7122 (X86selects_mask VK1WM:$mask,7123 (MaskedOp _.FRC:$src2, _.FRC:$src3,7124 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))),7125 (_.EltVT ZeroFP)))))),7126 (!cast<I>(Prefix#"231"#Suffix#"Zrkz_Int")7127 VR128X:$src1, VK1WM:$mask,7128 (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)),7129 (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)))>;7130 7131 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector7132 (X86selects_mask VK1WM:$mask,7133 (MaskedOp _.FRC:$src2,7134 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),7135 (_.ScalarLdFrag addr:$src3)),7136 (_.EltVT ZeroFP)))))),7137 (!cast<I>(Prefix#"213"#Suffix#"Zmkz_Int")7138 VR128X:$src1, VK1WM:$mask,7139 (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), addr:$src3)>;7140 7141 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector7142 (X86selects_mask VK1WM:$mask,7143 (MaskedOp (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),7144 _.FRC:$src2, (_.ScalarLdFrag addr:$src3)),7145 (_.EltVT ZeroFP)))))),7146 (!cast<I>(Prefix#"132"#Suffix#"Zmkz_Int")7147 VR128X:$src1, VK1WM:$mask,7148 (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), addr:$src3)>;7149 7150 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector7151 (X86selects_mask VK1WM:$mask,7152 (MaskedOp _.FRC:$src2, (_.ScalarLdFrag addr:$src3),7153 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))),7154 (_.EltVT ZeroFP)))))),7155 (!cast<I>(Prefix#"231"#Suffix#"Zmkz_Int")7156 VR128X:$src1, VK1WM:$mask,7157 (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), addr:$src3)>;7158 7159 // Patterns with rounding mode.7160 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector7161 (RndOp _.FRC:$src2,7162 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),7163 _.FRC:$src3, (i32 timm:$rc)))))),7164 (!cast<I>(Prefix#"213"#Suffix#"Zrb_Int")7165 VR128X:$src1, (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)),7166 (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)), AVX512RC:$rc)>;7167 7168 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector7169 (RndOp _.FRC:$src2, _.FRC:$src3,7170 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),7171 (i32 timm:$rc)))))),7172 (!cast<I>(Prefix#"231"#Suffix#"Zrb_Int")7173 VR128X:$src1, (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)),7174 (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)), AVX512RC:$rc)>;7175 7176 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector7177 (X86selects_mask VK1WM:$mask,7178 (RndOp _.FRC:$src2,7179 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),7180 _.FRC:$src3, (i32 timm:$rc)),7181 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))))))),7182 (!cast<I>(Prefix#"213"#Suffix#"Zrbk_Int")7183 VR128X:$src1, VK1WM:$mask,7184 (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)),7185 (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)), AVX512RC:$rc)>;7186 7187 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector7188 (X86selects_mask VK1WM:$mask,7189 (RndOp _.FRC:$src2, _.FRC:$src3,7190 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),7191 (i32 timm:$rc)),7192 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))))))),7193 (!cast<I>(Prefix#"231"#Suffix#"Zrbk_Int")7194 VR128X:$src1, VK1WM:$mask,7195 (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)),7196 (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)), AVX512RC:$rc)>;7197 7198 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector7199 (X86selects_mask VK1WM:$mask,7200 (RndOp _.FRC:$src2,7201 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),7202 _.FRC:$src3, (i32 timm:$rc)),7203 (_.EltVT ZeroFP)))))),7204 (!cast<I>(Prefix#"213"#Suffix#"Zrbkz_Int")7205 VR128X:$src1, VK1WM:$mask,7206 (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)),7207 (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)), AVX512RC:$rc)>;7208 7209 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector7210 (X86selects_mask VK1WM:$mask,7211 (RndOp _.FRC:$src2, _.FRC:$src3,7212 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),7213 (i32 timm:$rc)),7214 (_.EltVT ZeroFP)))))),7215 (!cast<I>(Prefix#"231"#Suffix#"Zrbkz_Int")7216 VR128X:$src1, VK1WM:$mask,7217 (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)),7218 (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)), AVX512RC:$rc)>;7219 }7220}7221defm : avx512_scalar_fma_patterns<any_fma, fma, X86FmaddRnd, "VFMADD", "SH",7222 X86Movsh, v8f16x_info, fp16imm0, HasFP16>;7223defm : avx512_scalar_fma_patterns<X86any_Fmsub, X86Fmsub, X86FmsubRnd, "VFMSUB", "SH",7224 X86Movsh, v8f16x_info, fp16imm0, HasFP16>;7225defm : avx512_scalar_fma_patterns<X86any_Fnmadd, X86Fnmadd, X86FnmaddRnd, "VFNMADD", "SH",7226 X86Movsh, v8f16x_info, fp16imm0, HasFP16>;7227defm : avx512_scalar_fma_patterns<X86any_Fnmsub, X86Fnmsub, X86FnmsubRnd, "VFNMSUB", "SH",7228 X86Movsh, v8f16x_info, fp16imm0, HasFP16>;7229 7230defm : avx512_scalar_fma_patterns<any_fma, fma, X86FmaddRnd, "VFMADD",7231 "SS", X86Movss, v4f32x_info, fp32imm0>;7232defm : avx512_scalar_fma_patterns<X86any_Fmsub, X86Fmsub, X86FmsubRnd, "VFMSUB",7233 "SS", X86Movss, v4f32x_info, fp32imm0>;7234defm : avx512_scalar_fma_patterns<X86any_Fnmadd, X86Fnmadd, X86FnmaddRnd, "VFNMADD",7235 "SS", X86Movss, v4f32x_info, fp32imm0>;7236defm : avx512_scalar_fma_patterns<X86any_Fnmsub, X86Fnmsub, X86FnmsubRnd, "VFNMSUB",7237 "SS", X86Movss, v4f32x_info, fp32imm0>;7238 7239defm : avx512_scalar_fma_patterns<any_fma, fma, X86FmaddRnd, "VFMADD",7240 "SD", X86Movsd, v2f64x_info, fp64imm0>;7241defm : avx512_scalar_fma_patterns<X86any_Fmsub, X86Fmsub, X86FmsubRnd, "VFMSUB",7242 "SD", X86Movsd, v2f64x_info, fp64imm0>;7243defm : avx512_scalar_fma_patterns<X86any_Fnmadd, X86Fnmadd, X86FnmaddRnd, "VFNMADD",7244 "SD", X86Movsd, v2f64x_info, fp64imm0>;7245defm : avx512_scalar_fma_patterns<X86any_Fnmsub, X86Fnmsub, X86FnmsubRnd, "VFNMSUB",7246 "SD", X86Movsd, v2f64x_info, fp64imm0>;7247 7248//===----------------------------------------------------------------------===//7249// AVX-512 Packed Multiply of Unsigned 52-bit Integers and Add the Low 52-bit IFMA7250//===----------------------------------------------------------------------===//7251let Constraints = "$src1 = $dst" in {7252multiclass avx512_pmadd52_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,7253 X86FoldableSchedWrite sched, X86VectorVTInfo _> {7254 // NOTE: The SDNode have the multiply operands first with the add last.7255 // This enables commuted load patterns to be autogenerated by tablegen.7256 let ExeDomain = _.ExeDomain in {7257 defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),7258 (ins _.RC:$src2, _.RC:$src3),7259 OpcodeStr, "$src3, $src2", "$src2, $src3",7260 (_.VT (OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1)), 1, 1>,7261 T8, PD, EVEX, VVVV, Sched<[sched]>;7262 7263 defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),7264 (ins _.RC:$src2, _.MemOp:$src3),7265 OpcodeStr, "$src3, $src2", "$src2, $src3",7266 (_.VT (OpNode _.RC:$src2, (_.LdFrag addr:$src3), _.RC:$src1))>,7267 T8, PD, EVEX, VVVV, Sched<[sched.Folded, sched.ReadAfterFold,7268 sched.ReadAfterFold]>;7269 7270 defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),7271 (ins _.RC:$src2, _.ScalarMemOp:$src3),7272 OpcodeStr, !strconcat("${src3}", _.BroadcastStr,", $src2"),7273 !strconcat("$src2, ${src3}", _.BroadcastStr ),7274 (OpNode _.RC:$src2,7275 (_.VT (_.BroadcastLdFrag addr:$src3)),7276 _.RC:$src1)>,7277 T8, PD, EVEX, VVVV, EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold,7278 sched.ReadAfterFold]>;7279 }7280}7281} // Constraints = "$src1 = $dst"7282 7283multiclass avx512_pmadd52_common<bits<8> opc, string OpcodeStr, SDNode OpNode,7284 X86SchedWriteWidths sched, AVX512VLVectorVTInfo _> {7285 let Predicates = [HasIFMA] in {7286 defm Z : avx512_pmadd52_rm<opc, OpcodeStr, OpNode, sched.ZMM, _.info512>,7287 EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>;7288 }7289 let Predicates = [HasVLX, HasIFMA] in {7290 defm Z256 : avx512_pmadd52_rm<opc, OpcodeStr, OpNode, sched.YMM, _.info256>,7291 EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>;7292 defm Z128 : avx512_pmadd52_rm<opc, OpcodeStr, OpNode, sched.XMM, _.info128>,7293 EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>;7294 }7295}7296 7297defm VPMADD52LUQ : avx512_pmadd52_common<0xb4, "vpmadd52luq", x86vpmadd52l,7298 SchedWriteVecIMul, avx512vl_i64_info>,7299 REX_W;7300defm VPMADD52HUQ : avx512_pmadd52_common<0xb5, "vpmadd52huq", x86vpmadd52h,7301 SchedWriteVecIMul, avx512vl_i64_info>,7302 REX_W;7303 7304//===----------------------------------------------------------------------===//7305// AVX-512 Scalar convert from sign integer to float/double7306//===----------------------------------------------------------------------===//7307 7308multiclass avx512_vcvtsi<bits<8> opc, SDPatternOperator OpNode, X86FoldableSchedWrite sched,7309 RegisterClass SrcRC, X86VectorVTInfo DstVT,7310 X86MemOperand x86memop, PatFrag ld_frag, string asm,7311 string mem, list<Register> _Uses = [MXCSR],7312 bit _mayRaiseFPException = 1> {7313let ExeDomain = DstVT.ExeDomain, Uses = _Uses,7314 mayRaiseFPException = _mayRaiseFPException in {7315 let hasSideEffects = 0, isCodeGenOnly = 1 in {7316 def rr : SI<opc, MRMSrcReg, (outs DstVT.FRC:$dst),7317 (ins DstVT.FRC:$src1, SrcRC:$src),7318 !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>,7319 EVEX, VVVV, Sched<[sched, ReadDefault, ReadInt2Fpu]>;7320 let mayLoad = 1 in7321 def rm : SI<opc, MRMSrcMem, (outs DstVT.FRC:$dst),7322 (ins DstVT.FRC:$src1, x86memop:$src),7323 asm#"{"#mem#"}\t{$src, $src1, $dst|$dst, $src1, $src}", []>,7324 EVEX, VVVV, Sched<[sched.Folded, sched.ReadAfterFold]>;7325 } // hasSideEffects = 07326 def rr_Int : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst),7327 (ins DstVT.RC:$src1, SrcRC:$src2),7328 !strconcat(asm,"\t{$src2, $src1, $dst|$dst, $src1, $src2}"),7329 [(set DstVT.RC:$dst,7330 (OpNode (DstVT.VT DstVT.RC:$src1), SrcRC:$src2))]>,7331 EVEX, VVVV, Sched<[sched, ReadDefault, ReadInt2Fpu]>;7332 7333 let mayLoad = 1 in7334 def rm_Int : SI<opc, MRMSrcMem, (outs DstVT.RC:$dst),7335 (ins DstVT.RC:$src1, x86memop:$src2),7336 asm#"{"#mem#"}\t{$src2, $src1, $dst|$dst, $src1, $src2}",7337 [(set DstVT.RC:$dst,7338 (OpNode (DstVT.VT DstVT.RC:$src1),7339 (ld_frag addr:$src2)))]>,7340 EVEX, VVVV, Sched<[sched.Folded, sched.ReadAfterFold]>;7341}7342 def : InstAlias<"v"#asm#mem#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",7343 (!cast<Instruction>(NAME#"rr_Int") DstVT.RC:$dst,7344 DstVT.RC:$src1, SrcRC:$src2), 0, "att">;7345}7346 7347multiclass avx512_vcvtsi_round<bits<8> opc, SDNode OpNode,7348 X86FoldableSchedWrite sched, RegisterClass SrcRC,7349 X86VectorVTInfo DstVT, string asm,7350 string mem> {7351 let ExeDomain = DstVT.ExeDomain, Uses = [MXCSR] in7352 def rrb_Int : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst),7353 (ins DstVT.RC:$src1, SrcRC:$src2, AVX512RC:$rc),7354 !strconcat(asm,7355 "\t{$src2, $rc, $src1, $dst|$dst, $src1, $rc, $src2}"),7356 [(set DstVT.RC:$dst,7357 (OpNode (DstVT.VT DstVT.RC:$src1),7358 SrcRC:$src2,7359 (i32 timm:$rc)))]>,7360 EVEX, VVVV, EVEX_B, EVEX_RC, Sched<[sched, ReadDefault, ReadInt2Fpu]>;7361 def : InstAlias<"v"#asm#mem#"\t{$src2, $rc, $src1, $dst|$dst, $src1, $rc, $src2}",7362 (!cast<Instruction>(NAME#"rrb_Int") DstVT.RC:$dst,7363 DstVT.RC:$src1, SrcRC:$src2, AVX512RC:$rc), 0, "att">;7364}7365 7366multiclass avx512_vcvtsi_common<bits<8> opc, SDNode OpNode, SDNode OpNodeRnd,7367 X86FoldableSchedWrite sched,7368 RegisterClass SrcRC, X86VectorVTInfo DstVT,7369 X86MemOperand x86memop, PatFrag ld_frag,7370 string asm, string mem> {7371 defm NAME : avx512_vcvtsi_round<opc, OpNodeRnd, sched, SrcRC, DstVT, asm, mem>,7372 avx512_vcvtsi<opc, OpNode, sched, SrcRC, DstVT, x86memop,7373 ld_frag, asm, mem>, VEX_LIG;7374}7375 7376let Predicates = [HasAVX512] in {7377defm VCVTSI2SSZ : avx512_vcvtsi_common<0x2A, X86SintToFp, X86SintToFpRnd,7378 WriteCvtI2SS, GR32,7379 v4f32x_info, i32mem, loadi32, "cvtsi2ss", "l">,7380 TB, XS, EVEX_CD8<32, CD8VT1>;7381defm VCVTSI642SSZ: avx512_vcvtsi_common<0x2A, X86SintToFp, X86SintToFpRnd,7382 WriteCvtI2SS, GR64,7383 v4f32x_info, i64mem, loadi64, "cvtsi2ss", "q">,7384 TB, XS, REX_W, EVEX_CD8<64, CD8VT1>;7385defm VCVTSI2SDZ : avx512_vcvtsi<0x2A, null_frag, WriteCvtI2SD, GR32,7386 v2f64x_info, i32mem, loadi32, "cvtsi2sd", "l", [], 0>,7387 TB, XD, VEX_LIG, EVEX_CD8<32, CD8VT1>;7388defm VCVTSI642SDZ: avx512_vcvtsi_common<0x2A, X86SintToFp, X86SintToFpRnd,7389 WriteCvtI2SD, GR64,7390 v2f64x_info, i64mem, loadi64, "cvtsi2sd", "q">,7391 TB, XD, REX_W, EVEX_CD8<64, CD8VT1>;7392 7393def : InstAlias<"vcvtsi2ss\t{$src, $src1, $dst|$dst, $src1, $src}",7394 (VCVTSI2SSZrm_Int VR128X:$dst, VR128X:$src1, i32mem:$src), 0, "att">;7395def : InstAlias<"vcvtsi2sd\t{$src, $src1, $dst|$dst, $src1, $src}",7396 (VCVTSI2SDZrm_Int VR128X:$dst, VR128X:$src1, i32mem:$src), 0, "att">;7397 7398def : Pat<(f32 (any_sint_to_fp (loadi32 addr:$src))),7399 (VCVTSI2SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>;7400def : Pat<(f32 (any_sint_to_fp (loadi64 addr:$src))),7401 (VCVTSI642SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>;7402def : Pat<(f64 (any_sint_to_fp (loadi32 addr:$src))),7403 (VCVTSI2SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>;7404def : Pat<(f64 (any_sint_to_fp (loadi64 addr:$src))),7405 (VCVTSI642SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>;7406 7407def : Pat<(f32 (any_sint_to_fp GR32:$src)),7408 (VCVTSI2SSZrr (f32 (IMPLICIT_DEF)), GR32:$src)>;7409def : Pat<(f32 (any_sint_to_fp GR64:$src)),7410 (VCVTSI642SSZrr (f32 (IMPLICIT_DEF)), GR64:$src)>;7411def : Pat<(f64 (any_sint_to_fp GR32:$src)),7412 (VCVTSI2SDZrr (f64 (IMPLICIT_DEF)), GR32:$src)>;7413def : Pat<(f64 (any_sint_to_fp GR64:$src)),7414 (VCVTSI642SDZrr (f64 (IMPLICIT_DEF)), GR64:$src)>;7415 7416defm VCVTUSI2SSZ : avx512_vcvtsi_common<0x7B, X86UintToFp, X86UintToFpRnd,7417 WriteCvtI2SS, GR32,7418 v4f32x_info, i32mem, loadi32,7419 "cvtusi2ss", "l">, TB, XS, EVEX_CD8<32, CD8VT1>;7420defm VCVTUSI642SSZ : avx512_vcvtsi_common<0x7B, X86UintToFp, X86UintToFpRnd,7421 WriteCvtI2SS, GR64,7422 v4f32x_info, i64mem, loadi64, "cvtusi2ss", "q">,7423 TB, XS, REX_W, EVEX_CD8<64, CD8VT1>;7424defm VCVTUSI2SDZ : avx512_vcvtsi<0x7B, null_frag, WriteCvtI2SD, GR32, v2f64x_info,7425 i32mem, loadi32, "cvtusi2sd", "l", [], 0>,7426 TB, XD, VEX_LIG, EVEX_CD8<32, CD8VT1>;7427defm VCVTUSI642SDZ : avx512_vcvtsi_common<0x7B, X86UintToFp, X86UintToFpRnd,7428 WriteCvtI2SD, GR64,7429 v2f64x_info, i64mem, loadi64, "cvtusi2sd", "q">,7430 TB, XD, REX_W, EVEX_CD8<64, CD8VT1>;7431 7432def : InstAlias<"vcvtusi2ss\t{$src, $src1, $dst|$dst, $src1, $src}",7433 (VCVTUSI2SSZrm_Int VR128X:$dst, VR128X:$src1, i32mem:$src), 0, "att">;7434def : InstAlias<"vcvtusi2sd\t{$src, $src1, $dst|$dst, $src1, $src}",7435 (VCVTUSI2SDZrm_Int VR128X:$dst, VR128X:$src1, i32mem:$src), 0, "att">;7436 7437def : Pat<(f32 (any_uint_to_fp (loadi32 addr:$src))),7438 (VCVTUSI2SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>;7439def : Pat<(f32 (any_uint_to_fp (loadi64 addr:$src))),7440 (VCVTUSI642SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>;7441def : Pat<(f64 (any_uint_to_fp (loadi32 addr:$src))),7442 (VCVTUSI2SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>;7443def : Pat<(f64 (any_uint_to_fp (loadi64 addr:$src))),7444 (VCVTUSI642SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>;7445 7446def : Pat<(f32 (any_uint_to_fp GR32:$src)),7447 (VCVTUSI2SSZrr (f32 (IMPLICIT_DEF)), GR32:$src)>;7448def : Pat<(f32 (any_uint_to_fp GR64:$src)),7449 (VCVTUSI642SSZrr (f32 (IMPLICIT_DEF)), GR64:$src)>;7450def : Pat<(f64 (any_uint_to_fp GR32:$src)),7451 (VCVTUSI2SDZrr (f64 (IMPLICIT_DEF)), GR32:$src)>;7452def : Pat<(f64 (any_uint_to_fp GR64:$src)),7453 (VCVTUSI642SDZrr (f64 (IMPLICIT_DEF)), GR64:$src)>;7454}7455 7456//===----------------------------------------------------------------------===//7457// AVX-512 Scalar convert from float/double to integer7458//===----------------------------------------------------------------------===//7459 7460multiclass avx512_cvt_s_int_round<bits<8> opc, X86VectorVTInfo SrcVT,7461 X86VectorVTInfo DstVT, SDNode OpNode,7462 SDNode OpNodeRnd,7463 X86FoldableSchedWrite sched, string asm,7464 string aliasStr, Predicate prd = HasAVX512> {7465 let Predicates = [prd], ExeDomain = SrcVT.ExeDomain in {7466 def rr_Int : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst), (ins SrcVT.RC:$src),7467 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),7468 [(set DstVT.RC:$dst, (OpNode (SrcVT.VT SrcVT.RC:$src)))]>,7469 EVEX, VEX_LIG, Sched<[sched]>, SIMD_EXC;7470 let Uses = [MXCSR] in7471 def rrb_Int : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst), (ins SrcVT.RC:$src, AVX512RC:$rc),7472 !strconcat(asm,"\t{$rc, $src, $dst|$dst, $src, $rc}"),7473 [(set DstVT.RC:$dst, (OpNodeRnd (SrcVT.VT SrcVT.RC:$src),(i32 timm:$rc)))]>,7474 EVEX, VEX_LIG, EVEX_B, EVEX_RC,7475 Sched<[sched]>;7476 let mayLoad = 1 in7477 def rm_Int : SI<opc, MRMSrcMem, (outs DstVT.RC:$dst), (ins SrcVT.IntScalarMemOp:$src),7478 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),7479 [(set DstVT.RC:$dst, (OpNode7480 (SrcVT.ScalarIntMemFrags addr:$src)))]>,7481 EVEX, VEX_LIG, Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC;7482 } // Predicates = [prd]7483 7484 def : InstAlias<"v" # asm # aliasStr # "\t{$src, $dst|$dst, $src}",7485 (!cast<Instruction>(NAME # "rr_Int") DstVT.RC:$dst, SrcVT.RC:$src), 0, "att">;7486 def : InstAlias<"v" # asm # aliasStr # "\t{$rc, $src, $dst|$dst, $src, $rc}",7487 (!cast<Instruction>(NAME # "rrb_Int") DstVT.RC:$dst, SrcVT.RC:$src, AVX512RC:$rc), 0, "att">;7488 def : InstAlias<"v" # asm # aliasStr # "\t{$src, $dst|$dst, $src}",7489 (!cast<Instruction>(NAME # "rm_Int") DstVT.RC:$dst,7490 SrcVT.IntScalarMemOp:$src), 0, "att">;7491}7492 7493// Convert float/double to signed/unsigned int 32/647494defm VCVTSS2SIZ: avx512_cvt_s_int_round<0x2D, f32x_info, i32x_info,X86cvts2si,7495 X86cvts2siRnd, WriteCvtSS2I, "cvtss2si", "{l}">,7496 TB, XS, EVEX_CD8<32, CD8VT1>;7497defm VCVTSS2SI64Z: avx512_cvt_s_int_round<0x2D, f32x_info, i64x_info, X86cvts2si,7498 X86cvts2siRnd, WriteCvtSS2I, "cvtss2si", "{q}">,7499 TB, XS, REX_W, EVEX_CD8<32, CD8VT1>;7500defm VCVTSS2USIZ: avx512_cvt_s_int_round<0x79, f32x_info, i32x_info, X86cvts2usi,7501 X86cvts2usiRnd, WriteCvtSS2I, "cvtss2usi", "{l}">,7502 TB, XS, EVEX_CD8<32, CD8VT1>;7503defm VCVTSS2USI64Z: avx512_cvt_s_int_round<0x79, f32x_info, i64x_info, X86cvts2usi,7504 X86cvts2usiRnd, WriteCvtSS2I, "cvtss2usi", "{q}">,7505 TB, XS, REX_W, EVEX_CD8<32, CD8VT1>;7506defm VCVTSD2SIZ: avx512_cvt_s_int_round<0x2D, f64x_info, i32x_info, X86cvts2si,7507 X86cvts2siRnd, WriteCvtSD2I, "cvtsd2si", "{l}">,7508 TB, XD, EVEX_CD8<64, CD8VT1>;7509defm VCVTSD2SI64Z: avx512_cvt_s_int_round<0x2D, f64x_info, i64x_info, X86cvts2si,7510 X86cvts2siRnd, WriteCvtSD2I, "cvtsd2si", "{q}">,7511 TB, XD, REX_W, EVEX_CD8<64, CD8VT1>;7512defm VCVTSD2USIZ: avx512_cvt_s_int_round<0x79, f64x_info, i32x_info, X86cvts2usi,7513 X86cvts2usiRnd, WriteCvtSD2I, "cvtsd2usi", "{l}">,7514 TB, XD, EVEX_CD8<64, CD8VT1>;7515defm VCVTSD2USI64Z: avx512_cvt_s_int_round<0x79, f64x_info, i64x_info, X86cvts2usi,7516 X86cvts2usiRnd, WriteCvtSD2I, "cvtsd2usi", "{q}">,7517 TB, XD, REX_W, EVEX_CD8<64, CD8VT1>;7518 7519multiclass avx512_cvt_s<bits<8> opc, string asm, X86VectorVTInfo SrcVT,7520 X86VectorVTInfo DstVT, SDNode OpNode,7521 X86FoldableSchedWrite sched> {7522 let Predicates = [HasAVX512], ExeDomain = SrcVT.ExeDomain in {7523 let isCodeGenOnly = 1 in {7524 def rr : AVX512<opc, MRMSrcReg, (outs DstVT.RC:$dst), (ins SrcVT.FRC:$src),7525 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),7526 [(set DstVT.RC:$dst, (OpNode SrcVT.FRC:$src))]>,7527 EVEX, VEX_LIG, Sched<[sched]>, SIMD_EXC;7528 let mayLoad = 1 in7529 def rm : AVX512<opc, MRMSrcMem, (outs DstVT.RC:$dst), (ins SrcVT.ScalarMemOp:$src),7530 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),7531 [(set DstVT.RC:$dst, (OpNode (SrcVT.ScalarLdFrag addr:$src)))]>,7532 EVEX, VEX_LIG, Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC;7533 }7534 } // Predicates = [HasAVX512]7535}7536 7537defm VCVTSS2SIZ: avx512_cvt_s<0x2D, "vcvtss2si", f32x_info, i32x_info,7538 lrint, WriteCvtSS2I>, TB, XS, EVEX_CD8<32, CD8VT1>;7539defm VCVTSS2SI64Z: avx512_cvt_s<0x2D, "vcvtss2si", f32x_info, i64x_info,7540 llrint, WriteCvtSS2I>, REX_W, TB, XS, EVEX_CD8<32, CD8VT1>;7541defm VCVTSD2SIZ: avx512_cvt_s<0x2D, "vcvtsd2si", f64x_info, i32x_info,7542 lrint, WriteCvtSD2I>, TB, XD, EVEX_CD8<64, CD8VT1>;7543defm VCVTSD2SI64Z: avx512_cvt_s<0x2D, "vcvtsd2si", f64x_info, i64x_info,7544 llrint, WriteCvtSD2I>, REX_W, TB, XD, EVEX_CD8<64, CD8VT1>;7545 7546let Predicates = [HasAVX512] in {7547 def : Pat<(i64 (lrint FR32:$src)), (VCVTSS2SI64Zrr FR32:$src)>;7548 def : Pat<(i64 (lrint (loadf32 addr:$src))), (VCVTSS2SI64Zrm addr:$src)>;7549 7550 def : Pat<(i64 (lrint FR64:$src)), (VCVTSD2SI64Zrr FR64:$src)>;7551 def : Pat<(i64 (lrint (loadf64 addr:$src))), (VCVTSD2SI64Zrm addr:$src)>;7552}7553 7554// Patterns used for matching vcvtsi2s{s,d} intrinsic sequences from clang7555// which produce unnecessary vmovs{s,d} instructions7556let Predicates = [HasAVX512] in {7557def : Pat<(v4f32 (X86Movss7558 (v4f32 VR128X:$dst),7559 (v4f32 (scalar_to_vector (f32 (any_sint_to_fp GR64:$src)))))),7560 (VCVTSI642SSZrr_Int VR128X:$dst, GR64:$src)>;7561 7562def : Pat<(v4f32 (X86Movss7563 (v4f32 VR128X:$dst),7564 (v4f32 (scalar_to_vector (f32 (any_sint_to_fp (loadi64 addr:$src))))))),7565 (VCVTSI642SSZrm_Int VR128X:$dst, addr:$src)>;7566 7567def : Pat<(v4f32 (X86Movss7568 (v4f32 VR128X:$dst),7569 (v4f32 (scalar_to_vector (f32 (any_sint_to_fp GR32:$src)))))),7570 (VCVTSI2SSZrr_Int VR128X:$dst, GR32:$src)>;7571 7572def : Pat<(v4f32 (X86Movss7573 (v4f32 VR128X:$dst),7574 (v4f32 (scalar_to_vector (f32 (any_sint_to_fp (loadi32 addr:$src))))))),7575 (VCVTSI2SSZrm_Int VR128X:$dst, addr:$src)>;7576 7577def : Pat<(v2f64 (X86Movsd7578 (v2f64 VR128X:$dst),7579 (v2f64 (scalar_to_vector (f64 (any_sint_to_fp GR64:$src)))))),7580 (VCVTSI642SDZrr_Int VR128X:$dst, GR64:$src)>;7581 7582def : Pat<(v2f64 (X86Movsd7583 (v2f64 VR128X:$dst),7584 (v2f64 (scalar_to_vector (f64 (any_sint_to_fp (loadi64 addr:$src))))))),7585 (VCVTSI642SDZrm_Int VR128X:$dst, addr:$src)>;7586 7587def : Pat<(v2f64 (X86Movsd7588 (v2f64 VR128X:$dst),7589 (v2f64 (scalar_to_vector (f64 (any_sint_to_fp GR32:$src)))))),7590 (VCVTSI2SDZrr_Int VR128X:$dst, GR32:$src)>;7591 7592def : Pat<(v2f64 (X86Movsd7593 (v2f64 VR128X:$dst),7594 (v2f64 (scalar_to_vector (f64 (any_sint_to_fp (loadi32 addr:$src))))))),7595 (VCVTSI2SDZrm_Int VR128X:$dst, addr:$src)>;7596 7597def : Pat<(v4f32 (X86Movss7598 (v4f32 VR128X:$dst),7599 (v4f32 (scalar_to_vector (f32 (any_uint_to_fp GR64:$src)))))),7600 (VCVTUSI642SSZrr_Int VR128X:$dst, GR64:$src)>;7601 7602def : Pat<(v4f32 (X86Movss7603 (v4f32 VR128X:$dst),7604 (v4f32 (scalar_to_vector (f32 (any_uint_to_fp (loadi64 addr:$src))))))),7605 (VCVTUSI642SSZrm_Int VR128X:$dst, addr:$src)>;7606 7607def : Pat<(v4f32 (X86Movss7608 (v4f32 VR128X:$dst),7609 (v4f32 (scalar_to_vector (f32 (any_uint_to_fp GR32:$src)))))),7610 (VCVTUSI2SSZrr_Int VR128X:$dst, GR32:$src)>;7611 7612def : Pat<(v4f32 (X86Movss7613 (v4f32 VR128X:$dst),7614 (v4f32 (scalar_to_vector (f32 (any_uint_to_fp (loadi32 addr:$src))))))),7615 (VCVTUSI2SSZrm_Int VR128X:$dst, addr:$src)>;7616 7617def : Pat<(v2f64 (X86Movsd7618 (v2f64 VR128X:$dst),7619 (v2f64 (scalar_to_vector (f64 (any_uint_to_fp GR64:$src)))))),7620 (VCVTUSI642SDZrr_Int VR128X:$dst, GR64:$src)>;7621 7622def : Pat<(v2f64 (X86Movsd7623 (v2f64 VR128X:$dst),7624 (v2f64 (scalar_to_vector (f64 (any_uint_to_fp (loadi64 addr:$src))))))),7625 (VCVTUSI642SDZrm_Int VR128X:$dst, addr:$src)>;7626 7627def : Pat<(v2f64 (X86Movsd7628 (v2f64 VR128X:$dst),7629 (v2f64 (scalar_to_vector (f64 (any_uint_to_fp GR32:$src)))))),7630 (VCVTUSI2SDZrr_Int VR128X:$dst, GR32:$src)>;7631 7632def : Pat<(v2f64 (X86Movsd7633 (v2f64 VR128X:$dst),7634 (v2f64 (scalar_to_vector (f64 (any_uint_to_fp (loadi32 addr:$src))))))),7635 (VCVTUSI2SDZrm_Int VR128X:$dst, addr:$src)>;7636} // Predicates = [HasAVX512]7637 7638// Convert float/double to signed/unsigned int 32/64 with truncation7639multiclass avx512_cvt_s_all<bits<8> opc, string asm, X86VectorVTInfo _SrcRC,7640 X86VectorVTInfo _DstRC, SDPatternOperator OpNode,7641 SDNode OpNodeInt, SDNode OpNodeSAE,7642 X86FoldableSchedWrite sched, string aliasStr,7643 Predicate prd = HasAVX512> {7644let Predicates = [prd], ExeDomain = _SrcRC.ExeDomain in {7645 let isCodeGenOnly = 1 in {7646 def rr : AVX512<opc, MRMSrcReg, (outs _DstRC.RC:$dst), (ins _SrcRC.FRC:$src),7647 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),7648 [(set _DstRC.RC:$dst, (OpNode _SrcRC.FRC:$src))]>,7649 EVEX, VEX_LIG, Sched<[sched]>, SIMD_EXC;7650 let mayLoad = 1 in7651 def rm : AVX512<opc, MRMSrcMem, (outs _DstRC.RC:$dst), (ins _SrcRC.ScalarMemOp:$src),7652 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),7653 [(set _DstRC.RC:$dst, (OpNode (_SrcRC.ScalarLdFrag addr:$src)))]>,7654 EVEX, VEX_LIG, Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC;7655 }7656 7657 def rr_Int : AVX512<opc, MRMSrcReg, (outs _DstRC.RC:$dst), (ins _SrcRC.RC:$src),7658 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),7659 [(set _DstRC.RC:$dst, (OpNodeInt (_SrcRC.VT _SrcRC.RC:$src)))]>,7660 EVEX, VEX_LIG, Sched<[sched]>, SIMD_EXC;7661 let Uses = [MXCSR] in7662 def rrb_Int : AVX512<opc, MRMSrcReg, (outs _DstRC.RC:$dst), (ins _SrcRC.RC:$src),7663 !strconcat(asm,"\t{{sae}, $src, $dst|$dst, $src, {sae}}"),7664 [(set _DstRC.RC:$dst, (OpNodeSAE (_SrcRC.VT _SrcRC.RC:$src)))]>,7665 EVEX, VEX_LIG, EVEX_B, Sched<[sched]>;7666 let mayLoad = 1 in7667 def rm_Int : AVX512<opc, MRMSrcMem, (outs _DstRC.RC:$dst),7668 (ins _SrcRC.IntScalarMemOp:$src),7669 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),7670 [(set _DstRC.RC:$dst,7671 (OpNodeInt (_SrcRC.ScalarIntMemFrags addr:$src)))]>,7672 EVEX, VEX_LIG, Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC;7673} // Predicates = [prd]7674 7675 def : InstAlias<asm # aliasStr # "\t{$src, $dst|$dst, $src}",7676 (!cast<Instruction>(NAME # "rr_Int") _DstRC.RC:$dst, _SrcRC.RC:$src), 0, "att">;7677 def : InstAlias<asm # aliasStr # "\t{{sae}, $src, $dst|$dst, $src, {sae}}",7678 (!cast<Instruction>(NAME # "rrb_Int") _DstRC.RC:$dst, _SrcRC.RC:$src), 0, "att">;7679 def : InstAlias<asm # aliasStr # "\t{$src, $dst|$dst, $src}",7680 (!cast<Instruction>(NAME # "rm_Int") _DstRC.RC:$dst,7681 _SrcRC.IntScalarMemOp:$src), 0, "att">;7682}7683 7684defm VCVTTSS2SIZ: avx512_cvt_s_all<0x2C, "vcvttss2si", f32x_info, i32x_info,7685 any_fp_to_sint, X86cvtts2Int, X86cvtts2IntSAE, WriteCvtSS2I,7686 "{l}">, TB, XS, EVEX_CD8<32, CD8VT1>;7687defm VCVTTSS2SI64Z: avx512_cvt_s_all<0x2C, "vcvttss2si", f32x_info, i64x_info,7688 any_fp_to_sint, X86cvtts2Int, X86cvtts2IntSAE, WriteCvtSS2I,7689 "{q}">, REX_W, TB, XS, EVEX_CD8<32, CD8VT1>;7690defm VCVTTSD2SIZ: avx512_cvt_s_all<0x2C, "vcvttsd2si", f64x_info, i32x_info,7691 any_fp_to_sint, X86cvtts2Int, X86cvtts2IntSAE, WriteCvtSD2I,7692 "{l}">, TB, XD, EVEX_CD8<64, CD8VT1>;7693defm VCVTTSD2SI64Z: avx512_cvt_s_all<0x2C, "vcvttsd2si", f64x_info, i64x_info,7694 any_fp_to_sint, X86cvtts2Int, X86cvtts2IntSAE, WriteCvtSD2I,7695 "{q}">, REX_W, TB, XD, EVEX_CD8<64, CD8VT1>;7696 7697defm VCVTTSS2USIZ: avx512_cvt_s_all<0x78, "vcvttss2usi", f32x_info, i32x_info,7698 any_fp_to_uint, X86cvtts2UInt, X86cvtts2UIntSAE, WriteCvtSS2I,7699 "{l}">, TB, XS, EVEX_CD8<32, CD8VT1>;7700defm VCVTTSS2USI64Z: avx512_cvt_s_all<0x78, "vcvttss2usi", f32x_info, i64x_info,7701 any_fp_to_uint, X86cvtts2UInt, X86cvtts2UIntSAE, WriteCvtSS2I,7702 "{q}">, TB, XS,REX_W, EVEX_CD8<32, CD8VT1>;7703defm VCVTTSD2USIZ: avx512_cvt_s_all<0x78, "vcvttsd2usi", f64x_info, i32x_info,7704 any_fp_to_uint, X86cvtts2UInt, X86cvtts2UIntSAE, WriteCvtSD2I,7705 "{l}">, TB, XD, EVEX_CD8<64, CD8VT1>;7706defm VCVTTSD2USI64Z: avx512_cvt_s_all<0x78, "vcvttsd2usi", f64x_info, i64x_info,7707 any_fp_to_uint, X86cvtts2UInt, X86cvtts2UIntSAE, WriteCvtSD2I,7708 "{q}">, TB, XD, REX_W, EVEX_CD8<64, CD8VT1>;7709 7710//===----------------------------------------------------------------------===//7711// AVX-512 Convert form float to double and back7712//===----------------------------------------------------------------------===//7713 7714let Uses = [MXCSR], mayRaiseFPException = 1 in7715multiclass avx512_cvt_fp_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,7716 X86VectorVTInfo _Src, SDNode OpNode,7717 X86FoldableSchedWrite sched> {7718 defm rr : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),7719 (ins _.RC:$src1, _Src.RC:$src2), OpcodeStr,7720 "$src2, $src1", "$src1, $src2",7721 (_.VT (OpNode (_.VT _.RC:$src1),7722 (_Src.VT _Src.RC:$src2))), "_Int">,7723 EVEX, VVVV, VEX_LIG, Sched<[sched]>;7724 let mayLoad = 1 in7725 defm rm : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),7726 (ins _.RC:$src1, _Src.IntScalarMemOp:$src2), OpcodeStr,7727 "$src2, $src1", "$src1, $src2",7728 (_.VT (OpNode (_.VT _.RC:$src1),7729 (_Src.ScalarIntMemFrags addr:$src2))), "_Int">,7730 EVEX, VVVV, VEX_LIG,7731 Sched<[sched.Folded, sched.ReadAfterFold]>;7732 7733 let isCodeGenOnly = 1, hasSideEffects = 0 in {7734 def rr : I<opc, MRMSrcReg, (outs _.FRC:$dst),7735 (ins _.FRC:$src1, _Src.FRC:$src2),7736 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,7737 EVEX, VVVV, VEX_LIG, Sched<[sched]>;7738 let mayLoad = 1 in7739 def rm : I<opc, MRMSrcMem, (outs _.FRC:$dst),7740 (ins _.FRC:$src1, _Src.ScalarMemOp:$src2),7741 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,7742 EVEX, VVVV, VEX_LIG, Sched<[sched.Folded, sched.ReadAfterFold]>;7743 }7744}7745 7746// Scalar Conversion with SAE - suppress all exceptions7747multiclass avx512_cvt_fp_sae_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,7748 X86VectorVTInfo _Src, SDNode OpNodeSAE,7749 X86FoldableSchedWrite sched> {7750 let Uses = [MXCSR] in7751 defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),7752 (ins _.RC:$src1, _Src.RC:$src2), OpcodeStr,7753 "{sae}, $src2, $src1", "$src1, $src2, {sae}",7754 (_.VT (OpNodeSAE (_.VT _.RC:$src1),7755 (_Src.VT _Src.RC:$src2))), "_Int">,7756 EVEX, VVVV, VEX_LIG, EVEX_B, Sched<[sched]>;7757}7758 7759// Scalar Conversion with rounding control (RC)7760multiclass avx512_cvt_fp_rc_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,7761 X86VectorVTInfo _Src, SDNode OpNodeRnd,7762 X86FoldableSchedWrite sched> {7763 let Uses = [MXCSR] in7764 defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),7765 (ins _.RC:$src1, _Src.RC:$src2, AVX512RC:$rc), OpcodeStr,7766 "$rc, $src2, $src1", "$src1, $src2, $rc",7767 (_.VT (OpNodeRnd (_.VT _.RC:$src1),7768 (_Src.VT _Src.RC:$src2), (i32 timm:$rc))), "_Int">,7769 EVEX, VVVV, VEX_LIG, Sched<[sched]>,7770 EVEX_B, EVEX_RC;7771}7772multiclass avx512_cvt_fp_scalar_trunc<bits<8> opc, string OpcodeStr,7773 SDNode OpNode, SDNode OpNodeRnd,7774 X86FoldableSchedWrite sched,7775 X86VectorVTInfo _src, X86VectorVTInfo _dst,7776 Predicate prd = HasAVX512> {7777 let Predicates = [prd], ExeDomain = SSEPackedSingle in {7778 defm Z : avx512_cvt_fp_scalar<opc, OpcodeStr, _dst, _src, OpNode, sched>,7779 avx512_cvt_fp_rc_scalar<opc, OpcodeStr, _dst, _src,7780 OpNodeRnd, sched>, EVEX_CD8<_src.EltSize, CD8VT1>;7781 }7782}7783 7784multiclass avx512_cvt_fp_scalar_extend<bits<8> opc, string OpcodeStr,7785 SDNode OpNode, SDNode OpNodeSAE,7786 X86FoldableSchedWrite sched,7787 X86VectorVTInfo _src, X86VectorVTInfo _dst,7788 Predicate prd = HasAVX512> {7789 let Predicates = [prd], ExeDomain = SSEPackedSingle in {7790 defm Z : avx512_cvt_fp_scalar<opc, OpcodeStr, _dst, _src, OpNode, sched>,7791 avx512_cvt_fp_sae_scalar<opc, OpcodeStr, _dst, _src, OpNodeSAE, sched>,7792 EVEX_CD8<_src.EltSize, CD8VT1>;7793 }7794}7795defm VCVTSD2SS : avx512_cvt_fp_scalar_trunc<0x5A, "vcvtsd2ss", X86frounds,7796 X86froundsRnd, WriteCvtSD2SS, f64x_info,7797 f32x_info>, TB, XD, REX_W;7798defm VCVTSS2SD : avx512_cvt_fp_scalar_extend<0x5A, "vcvtss2sd", X86fpexts,7799 X86fpextsSAE, WriteCvtSS2SD, f32x_info,7800 f64x_info>, TB, XS;7801defm VCVTSD2SH : avx512_cvt_fp_scalar_trunc<0x5A, "vcvtsd2sh", X86frounds,7802 X86froundsRnd, WriteCvtSD2SS, f64x_info,7803 f16x_info, HasFP16>, T_MAP5, XD, REX_W;7804defm VCVTSH2SD : avx512_cvt_fp_scalar_extend<0x5A, "vcvtsh2sd", X86fpexts,7805 X86fpextsSAE, WriteCvtSS2SD, f16x_info,7806 f64x_info, HasFP16>, T_MAP5, XS;7807defm VCVTSS2SH : avx512_cvt_fp_scalar_trunc<0x1D, "vcvtss2sh", X86frounds,7808 X86froundsRnd, WriteCvtSD2SS, f32x_info,7809 f16x_info, HasFP16>, T_MAP5;7810defm VCVTSH2SS : avx512_cvt_fp_scalar_extend<0x13, "vcvtsh2ss", X86fpexts,7811 X86fpextsSAE, WriteCvtSS2SD, f16x_info,7812 f32x_info, HasFP16>, T_MAP6;7813 7814def : Pat<(f64 (any_fpextend FR32X:$src)),7815 (VCVTSS2SDZrr (f64 (IMPLICIT_DEF)), FR32X:$src)>,7816 Requires<[HasAVX512]>;7817def : Pat<(f64 (any_fpextend (loadf32 addr:$src))),7818 (VCVTSS2SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>,7819 Requires<[HasAVX512, OptForSize]>;7820 7821def : Pat<(f32 (any_fpround FR64X:$src)),7822 (VCVTSD2SSZrr (f32 (IMPLICIT_DEF)), FR64X:$src)>,7823 Requires<[HasAVX512]>;7824 7825def : Pat<(f32 (any_fpextend FR16X:$src)),7826 (VCVTSH2SSZrr (f32 (IMPLICIT_DEF)), FR16X:$src)>,7827 Requires<[HasFP16]>;7828def : Pat<(f32 (any_fpextend (loadf16 addr:$src))),7829 (VCVTSH2SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>,7830 Requires<[HasFP16, OptForSize]>;7831 7832def : Pat<(f64 (any_fpextend FR16X:$src)),7833 (VCVTSH2SDZrr (f64 (IMPLICIT_DEF)), FR16X:$src)>,7834 Requires<[HasFP16]>;7835def : Pat<(f64 (any_fpextend (loadf16 addr:$src))),7836 (VCVTSH2SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>,7837 Requires<[HasFP16, OptForSize]>;7838 7839def : Pat<(f16 (any_fpround FR32X:$src)),7840 (VCVTSS2SHZrr (f16 (IMPLICIT_DEF)), FR32X:$src)>,7841 Requires<[HasFP16]>;7842def : Pat<(f16 (any_fpround FR64X:$src)),7843 (VCVTSD2SHZrr (f16 (IMPLICIT_DEF)), FR64X:$src)>,7844 Requires<[HasFP16]>;7845 7846def : Pat<(v4f32 (X86Movss7847 (v4f32 VR128X:$dst),7848 (v4f32 (scalar_to_vector7849 (f32 (any_fpround (f64 (extractelt VR128X:$src, (iPTR 0))))))))),7850 (VCVTSD2SSZrr_Int VR128X:$dst, VR128X:$src)>,7851 Requires<[HasAVX512]>;7852 7853def : Pat<(v2f64 (X86Movsd7854 (v2f64 VR128X:$dst),7855 (v2f64 (scalar_to_vector7856 (f64 (any_fpextend (f32 (extractelt VR128X:$src, (iPTR 0))))))))),7857 (VCVTSS2SDZrr_Int VR128X:$dst, VR128X:$src)>,7858 Requires<[HasAVX512]>;7859 7860//===----------------------------------------------------------------------===//7861// AVX-512 Vector convert from signed/unsigned integer to float/double7862// and from float/double to signed/unsigned integer7863//===----------------------------------------------------------------------===//7864 7865multiclass avx512_vcvt_fp<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,7866 X86VectorVTInfo _Src, SDPatternOperator OpNode, SDPatternOperator MaskOpNode,7867 X86FoldableSchedWrite sched,7868 string Broadcast = _.BroadcastStr,7869 string Alias = "", X86MemOperand MemOp = _Src.MemOp,7870 RegisterClass MaskRC = _.KRCWM,7871 dag LdDAG = (_.VT (OpNode (_Src.VT (_Src.LdFrag addr:$src)))),7872 dag MaskLdDAG = (_.VT (MaskOpNode (_Src.VT (_Src.LdFrag addr:$src))))> {7873let Uses = [MXCSR], mayRaiseFPException = 1 in {7874 defm rr : AVX512_maskable_cvt<opc, MRMSrcReg, _, (outs _.RC:$dst),7875 (ins _Src.RC:$src),7876 (ins _.RC:$src0, MaskRC:$mask, _Src.RC:$src),7877 (ins MaskRC:$mask, _Src.RC:$src),7878 OpcodeStr, "$src", "$src",7879 (_.VT (OpNode (_Src.VT _Src.RC:$src))),7880 (vselect_mask MaskRC:$mask,7881 (_.VT (MaskOpNode (_Src.VT _Src.RC:$src))),7882 _.RC:$src0),7883 (vselect_mask MaskRC:$mask,7884 (_.VT (MaskOpNode (_Src.VT _Src.RC:$src))),7885 _.ImmAllZerosV)>,7886 EVEX, Sched<[sched]>;7887 7888 let mayLoad = 1 in {7889 defm rm : AVX512_maskable_cvt<opc, MRMSrcMem, _, (outs _.RC:$dst),7890 (ins MemOp:$src),7891 (ins _.RC:$src0, MaskRC:$mask, MemOp:$src),7892 (ins MaskRC:$mask, MemOp:$src),7893 OpcodeStr#Alias, "$src", "$src",7894 LdDAG,7895 (vselect_mask MaskRC:$mask, MaskLdDAG, _.RC:$src0),7896 (vselect_mask MaskRC:$mask, MaskLdDAG, _.ImmAllZerosV)>,7897 EVEX, Sched<[sched.Folded]>;7898 7899 defm rmb : AVX512_maskable_cvt<opc, MRMSrcMem, _, (outs _.RC:$dst),7900 (ins _Src.ScalarMemOp:$src),7901 (ins _.RC:$src0, MaskRC:$mask, _Src.ScalarMemOp:$src),7902 (ins MaskRC:$mask, _Src.ScalarMemOp:$src),7903 OpcodeStr,7904 "${src}"#Broadcast, "${src}"#Broadcast,7905 (_.VT (OpNode (_Src.VT7906 (_Src.BroadcastLdFrag addr:$src))7907 )),7908 (vselect_mask MaskRC:$mask,7909 (_.VT7910 (MaskOpNode7911 (_Src.VT7912 (_Src.BroadcastLdFrag addr:$src)))),7913 _.RC:$src0),7914 (vselect_mask MaskRC:$mask,7915 (_.VT7916 (MaskOpNode7917 (_Src.VT7918 (_Src.BroadcastLdFrag addr:$src)))),7919 _.ImmAllZerosV)>,7920 EVEX, EVEX_B, Sched<[sched.Folded]>;7921 }7922 }7923}7924// Conversion with SAE - suppress all exceptions7925multiclass avx512_vcvt_fp_sae<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,7926 X86VectorVTInfo _Src, SDNode OpNodeSAE,7927 X86FoldableSchedWrite sched> {7928 let Uses = [MXCSR] in7929 defm rrb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),7930 (ins _Src.RC:$src), OpcodeStr,7931 "{sae}, $src", "$src, {sae}",7932 (_.VT (OpNodeSAE (_Src.VT _Src.RC:$src)))>,7933 EVEX, EVEX_B, Sched<[sched]>;7934}7935 7936// Conversion with rounding control (RC)7937multiclass avx512_vcvt_fp_rc<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,7938 X86VectorVTInfo _Src, SDPatternOperator OpNodeRnd,7939 X86FoldableSchedWrite sched> {7940 let Uses = [MXCSR] in7941 defm rrb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),7942 (ins _Src.RC:$src, AVX512RC:$rc), OpcodeStr,7943 "$rc, $src", "$src, $rc",7944 (_.VT (OpNodeRnd (_Src.VT _Src.RC:$src), (i32 timm:$rc)))>,7945 EVEX, EVEX_B, EVEX_RC, Sched<[sched]>;7946}7947 7948// Similar to avx512_vcvt_fp, but uses an extload for the memory form.7949multiclass avx512_vcvt_fpextend<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,7950 X86VectorVTInfo _Src, SDPatternOperator OpNode,7951 SDNode MaskOpNode,7952 X86FoldableSchedWrite sched,7953 string Broadcast = _.BroadcastStr,7954 string Alias = "", X86MemOperand MemOp = _Src.MemOp,7955 RegisterClass MaskRC = _.KRCWM>7956 : avx512_vcvt_fp<opc, OpcodeStr, _, _Src, OpNode, MaskOpNode, sched, Broadcast,7957 Alias, MemOp, MaskRC,7958 (_.VT (!cast<PatFrag>("extload"#_Src.VTName) addr:$src)),7959 (_.VT (!cast<PatFrag>("extload"#_Src.VTName) addr:$src))>;7960 7961// Extend [Float to Double, Half to Float]7962multiclass avx512_cvt_extend<bits<8> opc, string OpcodeStr,7963 AVX512VLVectorVTInfo _dst, AVX512VLVectorVTInfo _src,7964 X86SchedWriteWidths sched, Predicate prd = HasAVX512> {7965 let Predicates = [prd] in {7966 defm Z : avx512_vcvt_fpextend<opc, OpcodeStr, _dst.info512, _src.info256,7967 any_fpextend, fpextend, sched.ZMM>,7968 avx512_vcvt_fp_sae<opc, OpcodeStr, _dst.info512, _src.info256,7969 X86vfpextSAE, sched.ZMM>, EVEX_V512;7970 }7971 let Predicates = [prd, HasVLX] in {7972 defm Z128 : avx512_vcvt_fpextend<opc, OpcodeStr, _dst.info128, _src.info128,7973 X86any_vfpext, X86vfpext, sched.XMM,7974 _dst.info128.BroadcastStr,7975 "", f64mem>, EVEX_V128;7976 defm Z256 : avx512_vcvt_fpextend<opc, OpcodeStr, _dst.info256, _src.info128,7977 any_fpextend, fpextend, sched.YMM>, EVEX_V256;7978 }7979}7980 7981// Truncate [Double to Float, Float to Half]7982multiclass avx512_cvt_trunc<bits<8> opc, string OpcodeStr,7983 AVX512VLVectorVTInfo _dst, AVX512VLVectorVTInfo _src,7984 X86SchedWriteWidths sched, Predicate prd = HasAVX512,7985 PatFrag bcast128 = _src.info128.BroadcastLdFrag,7986 PatFrag loadVT128 = _src.info128.LdFrag,7987 RegisterClass maskRC128 = _src.info128.KRCWM> {7988 let Predicates = [prd] in {7989 defm Z : avx512_vcvt_fp<opc, OpcodeStr, _dst.info256, _src.info512,7990 X86any_vfpround, X86vfpround, sched.ZMM>,7991 avx512_vcvt_fp_rc<opc, OpcodeStr, _dst.info256, _src.info512,7992 X86vfproundRnd, sched.ZMM>, EVEX_V512;7993 }7994 let Predicates = [prd, HasVLX] in {7995 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, _dst.info128, _src.info128,7996 null_frag, null_frag, sched.XMM,7997 _src.info128.BroadcastStr, "{x}",7998 f128mem, maskRC128>, EVEX_V128;7999 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, _dst.info128, _src.info256,8000 X86any_vfpround, X86vfpround,8001 sched.YMM, _src.info256.BroadcastStr, "{y}">, EVEX_V256;8002 8003 // Special patterns to allow use of X86vmfpround for masking. Instruction8004 // patterns have been disabled with null_frag.8005 def : Pat<(_dst.info128.VT (X86any_vfpround (_src.info128.VT VR128X:$src))),8006 (!cast<Instruction>(NAME # "Z128rr") VR128X:$src)>;8007 def : Pat<(X86vmfpround (_src.info128.VT VR128X:$src), (_dst.info128.VT VR128X:$src0),8008 maskRC128:$mask),8009 (!cast<Instruction>(NAME # "Z128rrk") VR128X:$src0, maskRC128:$mask, VR128X:$src)>;8010 def : Pat<(X86vmfpround (_src.info128.VT VR128X:$src), _dst.info128.ImmAllZerosV,8011 maskRC128:$mask),8012 (!cast<Instruction>(NAME # "Z128rrkz") maskRC128:$mask, VR128X:$src)>;8013 8014 def : Pat<(_dst.info128.VT (X86any_vfpround (loadVT128 addr:$src))),8015 (!cast<Instruction>(NAME # "Z128rm") addr:$src)>;8016 def : Pat<(X86vmfpround (loadVT128 addr:$src), (_dst.info128.VT VR128X:$src0),8017 maskRC128:$mask),8018 (!cast<Instruction>(NAME # "Z128rmk") VR128X:$src0, maskRC128:$mask, addr:$src)>;8019 def : Pat<(X86vmfpround (loadVT128 addr:$src), _dst.info128.ImmAllZerosV,8020 maskRC128:$mask),8021 (!cast<Instruction>(NAME # "Z128rmkz") maskRC128:$mask, addr:$src)>;8022 8023 def : Pat<(_dst.info128.VT (X86any_vfpround (_src.info128.VT (bcast128 addr:$src)))),8024 (!cast<Instruction>(NAME # "Z128rmb") addr:$src)>;8025 def : Pat<(X86vmfpround (_src.info128.VT (bcast128 addr:$src)),8026 (_dst.info128.VT VR128X:$src0), maskRC128:$mask),8027 (!cast<Instruction>(NAME # "Z128rmbk") VR128X:$src0, maskRC128:$mask, addr:$src)>;8028 def : Pat<(X86vmfpround (_src.info128.VT (bcast128 addr:$src)),8029 _dst.info128.ImmAllZerosV, maskRC128:$mask),8030 (!cast<Instruction>(NAME # "Z128rmbkz") maskRC128:$mask, addr:$src)>;8031 }8032 8033 def : InstAlias<OpcodeStr#"x\t{$src, $dst|$dst, $src}",8034 (!cast<Instruction>(NAME # "Z128rr") VR128X:$dst, VR128X:$src), 0, "att">;8035 def : InstAlias<OpcodeStr#"x\t{$src, $dst {${mask}}|$dst {${mask}}, $src}",8036 (!cast<Instruction>(NAME # "Z128rrk") VR128X:$dst,8037 VK2WM:$mask, VR128X:$src), 0, "att">;8038 def : InstAlias<OpcodeStr#"x\t{$src, $dst {${mask}} {z}|"8039 "$dst {${mask}} {z}, $src}",8040 (!cast<Instruction>(NAME # "Z128rrkz") VR128X:$dst,8041 VK2WM:$mask, VR128X:$src), 0, "att">;8042 def : InstAlias<OpcodeStr#"x\t{${src}{1to2}, $dst|$dst, ${src}{1to2}}",8043 (!cast<Instruction>(NAME # "Z128rmb") VR128X:$dst, f64mem:$src), 0, "att">;8044 def : InstAlias<OpcodeStr#"x\t{${src}{1to2}, $dst {${mask}}|"8045 "$dst {${mask}}, ${src}{1to2}}",8046 (!cast<Instruction>(NAME # "Z128rmbk") VR128X:$dst,8047 VK2WM:$mask, f64mem:$src), 0, "att">;8048 def : InstAlias<OpcodeStr#"x\t{${src}{1to2}, $dst {${mask}} {z}|"8049 "$dst {${mask}} {z}, ${src}{1to2}}",8050 (!cast<Instruction>(NAME # "Z128rmbkz") VR128X:$dst,8051 VK2WM:$mask, f64mem:$src), 0, "att">;8052 8053 def : InstAlias<OpcodeStr#"y\t{$src, $dst|$dst, $src}",8054 (!cast<Instruction>(NAME # "Z256rr") VR128X:$dst, VR256X:$src), 0, "att">;8055 def : InstAlias<OpcodeStr#"y\t{$src, $dst {${mask}}|$dst {${mask}}, $src}",8056 (!cast<Instruction>(NAME # "Z256rrk") VR128X:$dst,8057 VK4WM:$mask, VR256X:$src), 0, "att">;8058 def : InstAlias<OpcodeStr#"y\t{$src, $dst {${mask}} {z}|"8059 "$dst {${mask}} {z}, $src}",8060 (!cast<Instruction>(NAME # "Z256rrkz") VR128X:$dst,8061 VK4WM:$mask, VR256X:$src), 0, "att">;8062 def : InstAlias<OpcodeStr#"y\t{${src}{1to4}, $dst|$dst, ${src}{1to4}}",8063 (!cast<Instruction>(NAME # "Z256rmb") VR128X:$dst, f64mem:$src), 0, "att">;8064 def : InstAlias<OpcodeStr#"y\t{${src}{1to4}, $dst {${mask}}|"8065 "$dst {${mask}}, ${src}{1to4}}",8066 (!cast<Instruction>(NAME # "Z256rmbk") VR128X:$dst,8067 VK4WM:$mask, f64mem:$src), 0, "att">;8068 def : InstAlias<OpcodeStr#"y\t{${src}{1to4}, $dst {${mask}} {z}|"8069 "$dst {${mask}} {z}, ${src}{1to4}}",8070 (!cast<Instruction>(NAME # "Z256rmbkz") VR128X:$dst,8071 VK4WM:$mask, f64mem:$src), 0, "att">;8072}8073 8074defm VCVTPD2PS : avx512_cvt_trunc<0x5A, "vcvtpd2ps",8075 avx512vl_f32_info, avx512vl_f64_info, SchedWriteCvtPD2PS>,8076 REX_W, TB, PD, EVEX_CD8<64, CD8VF>;8077defm VCVTPS2PD : avx512_cvt_extend<0x5A, "vcvtps2pd",8078 avx512vl_f64_info, avx512vl_f32_info, SchedWriteCvtPS2PD>,8079 TB, EVEX_CD8<32, CD8VH>;8080 8081// Extend Half to Double8082multiclass avx512_cvtph2pd<bits<8> opc, string OpcodeStr,8083 X86SchedWriteWidths sched> {8084 let Predicates = [HasFP16] in {8085 defm Z : avx512_vcvt_fpextend<opc, OpcodeStr, v8f64_info, v8f16x_info,8086 any_fpextend, fpextend, sched.ZMM>,8087 avx512_vcvt_fp_sae<opc, OpcodeStr, v8f64_info, v8f16x_info,8088 X86vfpextSAE, sched.ZMM>, EVEX_V512;8089 def : Pat<(v8f64 (extloadv8f16 addr:$src)),8090 (!cast<Instruction>(NAME # "Zrm") addr:$src)>;8091 }8092 let Predicates = [HasFP16, HasVLX] in {8093 defm Z128 : avx512_vcvt_fpextend<opc, OpcodeStr, v2f64x_info, v8f16x_info,8094 X86any_vfpext, X86vfpext, sched.XMM, "{1to2}", "",8095 f32mem>, EVEX_V128;8096 defm Z256 : avx512_vcvt_fpextend<opc, OpcodeStr, v4f64x_info, v8f16x_info,8097 X86any_vfpext, X86vfpext, sched.YMM, "{1to4}", "",8098 f64mem>, EVEX_V256;8099 }8100}8101 8102// Truncate Double to Half8103multiclass avx512_cvtpd2ph<bits<8> opc, string OpcodeStr, X86SchedWriteWidths sched> {8104 let Predicates = [HasFP16] in {8105 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f16x_info, v8f64_info,8106 X86any_vfpround, X86vfpround, sched.ZMM, "{1to8}", "{z}">,8107 avx512_vcvt_fp_rc<opc, OpcodeStr, v8f16x_info, v8f64_info,8108 X86vfproundRnd, sched.ZMM>, EVEX_V512;8109 }8110 let Predicates = [HasFP16, HasVLX] in {8111 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v8f16x_info, v2f64x_info, null_frag,8112 null_frag, sched.XMM, "{1to2}", "{x}", f128mem,8113 VK2WM>, EVEX_V128;8114 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8f16x_info, v4f64x_info, null_frag,8115 null_frag, sched.YMM, "{1to4}", "{y}", f256mem,8116 VK4WM>, EVEX_V256;8117 }8118 def : InstAlias<OpcodeStr#"x\t{$src, $dst|$dst, $src}",8119 (!cast<Instruction>(NAME # "Z128rr") VR128X:$dst,8120 VR128X:$src), 0, "att">;8121 def : InstAlias<OpcodeStr#"x\t{$src, $dst {${mask}}|$dst {${mask}}, $src}",8122 (!cast<Instruction>(NAME # "Z128rrk") VR128X:$dst,8123 VK2WM:$mask, VR128X:$src), 0, "att">;8124 def : InstAlias<OpcodeStr#"x\t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}",8125 (!cast<Instruction>(NAME # "Z128rrkz") VR128X:$dst,8126 VK2WM:$mask, VR128X:$src), 0, "att">;8127 def : InstAlias<OpcodeStr#"x\t{${src}{1to2}, $dst|$dst, ${src}{1to2}}",8128 (!cast<Instruction>(NAME # "Z128rmb") VR128X:$dst,8129 i64mem:$src), 0, "att">;8130 def : InstAlias<OpcodeStr#"x\t{${src}{1to2}, $dst {${mask}}|"8131 "$dst {${mask}}, ${src}{1to2}}",8132 (!cast<Instruction>(NAME # "Z128rmbk") VR128X:$dst,8133 VK2WM:$mask, i64mem:$src), 0, "att">;8134 def : InstAlias<OpcodeStr#"x\t{${src}{1to2}, $dst {${mask}} {z}|"8135 "$dst {${mask}} {z}, ${src}{1to2}}",8136 (!cast<Instruction>(NAME # "Z128rmbkz") VR128X:$dst,8137 VK2WM:$mask, i64mem:$src), 0, "att">;8138 8139 def : InstAlias<OpcodeStr#"y\t{$src, $dst|$dst, $src}",8140 (!cast<Instruction>(NAME # "Z256rr") VR128X:$dst,8141 VR256X:$src), 0, "att">;8142 def : InstAlias<OpcodeStr#"y\t{$src, $dst {${mask}}|"8143 "$dst {${mask}}, $src}",8144 (!cast<Instruction>(NAME # "Z256rrk") VR128X:$dst,8145 VK4WM:$mask, VR256X:$src), 0, "att">;8146 def : InstAlias<OpcodeStr#"y\t{$src, $dst {${mask}} {z}|"8147 "$dst {${mask}} {z}, $src}",8148 (!cast<Instruction>(NAME # "Z256rrkz") VR128X:$dst,8149 VK4WM:$mask, VR256X:$src), 0, "att">;8150 def : InstAlias<OpcodeStr#"y\t{${src}{1to4}, $dst|$dst, ${src}{1to4}}",8151 (!cast<Instruction>(NAME # "Z256rmb") VR128X:$dst,8152 i64mem:$src), 0, "att">;8153 def : InstAlias<OpcodeStr#"y\t{${src}{1to4}, $dst {${mask}}|"8154 "$dst {${mask}}, ${src}{1to4}}",8155 (!cast<Instruction>(NAME # "Z256rmbk") VR128X:$dst,8156 VK4WM:$mask, i64mem:$src), 0, "att">;8157 def : InstAlias<OpcodeStr#"y\t{${src}{1to4}, $dst {${mask}} {z}|"8158 "$dst {${mask}} {z}, ${src}{1to4}}",8159 (!cast<Instruction>(NAME # "Z256rmbkz") VR128X:$dst,8160 VK4WM:$mask, i64mem:$src), 0, "att">;8161 8162 def : InstAlias<OpcodeStr#"z\t{$src, $dst|$dst, $src}",8163 (!cast<Instruction>(NAME # "Zrr") VR128X:$dst,8164 VR512:$src), 0, "att">;8165 def : InstAlias<OpcodeStr#"z\t{$src, $dst {${mask}}|"8166 "$dst {${mask}}, $src}",8167 (!cast<Instruction>(NAME # "Zrrk") VR128X:$dst,8168 VK8WM:$mask, VR512:$src), 0, "att">;8169 def : InstAlias<OpcodeStr#"z\t{$src, $dst {${mask}} {z}|"8170 "$dst {${mask}} {z}, $src}",8171 (!cast<Instruction>(NAME # "Zrrkz") VR128X:$dst,8172 VK8WM:$mask, VR512:$src), 0, "att">;8173 def : InstAlias<OpcodeStr#"z\t{${src}{1to8}, $dst|$dst, ${src}{1to8}}",8174 (!cast<Instruction>(NAME # "Zrmb") VR128X:$dst,8175 i64mem:$src), 0, "att">;8176 def : InstAlias<OpcodeStr#"z\t{${src}{1to8}, $dst {${mask}}|"8177 "$dst {${mask}}, ${src}{1to8}}",8178 (!cast<Instruction>(NAME # "Zrmbk") VR128X:$dst,8179 VK8WM:$mask, i64mem:$src), 0, "att">;8180 def : InstAlias<OpcodeStr#"z\t{${src}{1to8}, $dst {${mask}} {z}|"8181 "$dst {${mask}} {z}, ${src}{1to8}}",8182 (!cast<Instruction>(NAME # "Zrmbkz") VR128X:$dst,8183 VK8WM:$mask, i64mem:$src), 0, "att">;8184}8185 8186defm VCVTPS2PHX : avx512_cvt_trunc<0x1D, "vcvtps2phx", avx512vl_f16_info,8187 avx512vl_f32_info, SchedWriteCvtPD2PS,8188 HasFP16>, T_MAP5, PD, EVEX_CD8<32, CD8VF>;8189defm VCVTPH2PSX : avx512_cvt_extend<0x13, "vcvtph2psx", avx512vl_f32_info,8190 avx512vl_f16_info, SchedWriteCvtPS2PD,8191 HasFP16>, T_MAP6, PD, EVEX_CD8<16, CD8VH>;8192defm VCVTPD2PH : avx512_cvtpd2ph<0x5A, "vcvtpd2ph", SchedWriteCvtPD2PS>,8193 REX_W, T_MAP5, PD, EVEX_CD8<64, CD8VF>;8194defm VCVTPH2PD : avx512_cvtph2pd<0x5A, "vcvtph2pd", SchedWriteCvtPS2PD>,8195 T_MAP5, EVEX_CD8<16, CD8VQ>;8196 8197let Predicates = [HasFP16, HasVLX] in {8198 // Special patterns to allow use of X86vmfpround for masking. Instruction8199 // patterns have been disabled with null_frag.8200 def : Pat<(v8f16 (X86any_vfpround (v4f64 VR256X:$src))),8201 (VCVTPD2PHZ256rr VR256X:$src)>;8202 def : Pat<(v8f16 (X86vmfpround (v4f64 VR256X:$src), (v8f16 VR128X:$src0),8203 VK4WM:$mask)),8204 (VCVTPD2PHZ256rrk VR128X:$src0, VK4WM:$mask, VR256X:$src)>;8205 def : Pat<(X86vmfpround (v4f64 VR256X:$src), v8f16x_info.ImmAllZerosV,8206 VK4WM:$mask),8207 (VCVTPD2PHZ256rrkz VK4WM:$mask, VR256X:$src)>;8208 8209 def : Pat<(v8f16 (X86any_vfpround (loadv4f64 addr:$src))),8210 (VCVTPD2PHZ256rm addr:$src)>;8211 def : Pat<(X86vmfpround (loadv4f64 addr:$src), (v8f16 VR128X:$src0),8212 VK4WM:$mask),8213 (VCVTPD2PHZ256rmk VR128X:$src0, VK4WM:$mask, addr:$src)>;8214 def : Pat<(X86vmfpround (loadv4f64 addr:$src), v8f16x_info.ImmAllZerosV,8215 VK4WM:$mask),8216 (VCVTPD2PHZ256rmkz VK4WM:$mask, addr:$src)>;8217 8218 def : Pat<(v8f16 (X86any_vfpround (v4f64 (X86VBroadcastld64 addr:$src)))),8219 (VCVTPD2PHZ256rmb addr:$src)>;8220 def : Pat<(X86vmfpround (v4f64 (X86VBroadcastld64 addr:$src)),8221 (v8f16 VR128X:$src0), VK4WM:$mask),8222 (VCVTPD2PHZ256rmbk VR128X:$src0, VK4WM:$mask, addr:$src)>;8223 def : Pat<(X86vmfpround (v4f64 (X86VBroadcastld64 addr:$src)),8224 v8f16x_info.ImmAllZerosV, VK4WM:$mask),8225 (VCVTPD2PHZ256rmbkz VK4WM:$mask, addr:$src)>;8226 8227 def : Pat<(v8f16 (X86any_vfpround (v2f64 VR128X:$src))),8228 (VCVTPD2PHZ128rr VR128X:$src)>;8229 def : Pat<(X86vmfpround (v2f64 VR128X:$src), (v8f16 VR128X:$src0),8230 VK2WM:$mask),8231 (VCVTPD2PHZ128rrk VR128X:$src0, VK2WM:$mask, VR128X:$src)>;8232 def : Pat<(X86vmfpround (v2f64 VR128X:$src), v8f16x_info.ImmAllZerosV,8233 VK2WM:$mask),8234 (VCVTPD2PHZ128rrkz VK2WM:$mask, VR128X:$src)>;8235 8236 def : Pat<(v8f16 (X86any_vfpround (loadv2f64 addr:$src))),8237 (VCVTPD2PHZ128rm addr:$src)>;8238 def : Pat<(X86vmfpround (loadv2f64 addr:$src), (v8f16 VR128X:$src0),8239 VK2WM:$mask),8240 (VCVTPD2PHZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;8241 def : Pat<(X86vmfpround (loadv2f64 addr:$src), v8f16x_info.ImmAllZerosV,8242 VK2WM:$mask),8243 (VCVTPD2PHZ128rmkz VK2WM:$mask, addr:$src)>;8244 8245 def : Pat<(v8f16 (X86any_vfpround (v2f64 (X86VBroadcastld64 addr:$src)))),8246 (VCVTPD2PHZ128rmb addr:$src)>;8247 def : Pat<(X86vmfpround (v2f64 (X86VBroadcastld64 addr:$src)),8248 (v8f16 VR128X:$src0), VK2WM:$mask),8249 (VCVTPD2PHZ128rmbk VR128X:$src0, VK2WM:$mask, addr:$src)>;8250 def : Pat<(X86vmfpround (v2f64 (X86VBroadcastld64 addr:$src)),8251 v8f16x_info.ImmAllZerosV, VK2WM:$mask),8252 (VCVTPD2PHZ128rmbkz VK2WM:$mask, addr:$src)>;8253}8254 8255// Convert Signed/Unsigned Doubleword to Double8256let Uses = []<Register>, mayRaiseFPException = 0 in8257multiclass avx512_cvtdq2pd<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,8258 SDNode MaskOpNode, SDPatternOperator OpNode128,8259 SDNode MaskOpNode128,8260 X86SchedWriteWidths sched> {8261 // No rounding in this op8262 let Predicates = [HasAVX512] in8263 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8i32x_info, OpNode,8264 MaskOpNode, sched.ZMM>, EVEX_V512;8265 8266 let Predicates = [HasVLX] in {8267 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v4i32x_info,8268 OpNode128, MaskOpNode128, sched.XMM, "{1to2}",8269 "", i64mem, VK2WM,8270 (v2f64 (OpNode128 (bc_v4i328271 (v2i648272 (scalar_to_vector (loadi64 addr:$src)))))),8273 (v2f64 (MaskOpNode128 (bc_v4i328274 (v2i648275 (scalar_to_vector (loadi64 addr:$src))))))>,8276 EVEX_V128;8277 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4i32x_info, OpNode,8278 MaskOpNode, sched.YMM>, EVEX_V256;8279 }8280}8281 8282// Convert Signed/Unsigned Doubleword to Float8283multiclass avx512_cvtdq2ps<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,8284 SDNode MaskOpNode, SDNode OpNodeRnd,8285 X86SchedWriteWidths sched> {8286 let Predicates = [HasAVX512] in8287 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16f32_info, v16i32_info, OpNode,8288 MaskOpNode, sched.ZMM>,8289 avx512_vcvt_fp_rc<opc, OpcodeStr, v16f32_info, v16i32_info,8290 OpNodeRnd, sched.ZMM>, EVEX_V512;8291 8292 let Predicates = [HasVLX] in {8293 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4i32x_info, OpNode,8294 MaskOpNode, sched.XMM>, EVEX_V128;8295 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8i32x_info, OpNode,8296 MaskOpNode, sched.YMM>, EVEX_V256;8297 }8298}8299 8300// Convert Float to Signed/Unsigned Doubleword with truncation8301multiclass avx512_cvttps2dq<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,8302 SDNode MaskOpNode,8303 SDNode OpNodeSAE, X86SchedWriteWidths sched> {8304 let Predicates = [HasAVX512] in {8305 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16i32_info, v16f32_info, OpNode,8306 MaskOpNode, sched.ZMM>,8307 avx512_vcvt_fp_sae<opc, OpcodeStr, v16i32_info, v16f32_info,8308 OpNodeSAE, sched.ZMM>, EVEX_V512;8309 }8310 let Predicates = [HasVLX] in {8311 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f32x_info, OpNode,8312 MaskOpNode, sched.XMM>, EVEX_V128;8313 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f32x_info, OpNode,8314 MaskOpNode, sched.YMM>, EVEX_V256;8315 }8316}8317 8318// Convert Float to Signed/Unsigned Doubleword8319multiclass avx512_cvtps2dq<bits<8> opc, string OpcodeStr, SDNode OpNode,8320 SDNode MaskOpNode, SDNode OpNodeRnd,8321 X86SchedWriteWidths sched> {8322 let Predicates = [HasAVX512] in {8323 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16i32_info, v16f32_info, OpNode,8324 MaskOpNode, sched.ZMM>,8325 avx512_vcvt_fp_rc<opc, OpcodeStr, v16i32_info, v16f32_info,8326 OpNodeRnd, sched.ZMM>, EVEX_V512;8327 }8328 let Predicates = [HasVLX] in {8329 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f32x_info, OpNode,8330 MaskOpNode, sched.XMM>, EVEX_V128;8331 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f32x_info, OpNode,8332 MaskOpNode, sched.YMM>, EVEX_V256;8333 }8334}8335 8336// Convert Double to Signed/Unsigned Doubleword with truncation8337multiclass avx512_cvttpd2dq<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,8338 SDNode MaskOpNode, SDNode OpNodeSAE,8339 X86SchedWriteWidths sched> {8340 let Predicates = [HasAVX512] in {8341 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f64_info, OpNode,8342 MaskOpNode, sched.ZMM>,8343 avx512_vcvt_fp_sae<opc, OpcodeStr, v8i32x_info, v8f64_info,8344 OpNodeSAE, sched.ZMM>, EVEX_V512;8345 }8346 let Predicates = [HasVLX] in {8347 // we need "x"/"y" suffixes in order to distinguish between 128 and 2568348 // memory forms of these instructions in Asm Parser. They have the same8349 // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly8350 // due to the same reason.8351 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v2f64x_info,8352 null_frag, null_frag, sched.XMM, "{1to2}", "{x}", f128mem,8353 VK2WM>, EVEX_V128;8354 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f64x_info, OpNode,8355 MaskOpNode, sched.YMM, "{1to4}", "{y}">, EVEX_V256;8356 }8357 8358 def : InstAlias<OpcodeStr#"x\t{$src, $dst|$dst, $src}",8359 (!cast<Instruction>(NAME # "Z128rr") VR128X:$dst,8360 VR128X:$src), 0, "att">;8361 def : InstAlias<OpcodeStr#"x\t{$src, $dst {${mask}}|$dst {${mask}}, $src}",8362 (!cast<Instruction>(NAME # "Z128rrk") VR128X:$dst,8363 VK2WM:$mask, VR128X:$src), 0, "att">;8364 def : InstAlias<OpcodeStr#"x\t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}",8365 (!cast<Instruction>(NAME # "Z128rrkz") VR128X:$dst,8366 VK2WM:$mask, VR128X:$src), 0, "att">;8367 def : InstAlias<OpcodeStr#"x\t{${src}{1to2}, $dst|$dst, ${src}{1to2}}",8368 (!cast<Instruction>(NAME # "Z128rmb") VR128X:$dst,8369 f64mem:$src), 0, "att">;8370 def : InstAlias<OpcodeStr#"x\t{${src}{1to2}, $dst {${mask}}|"8371 "$dst {${mask}}, ${src}{1to2}}",8372 (!cast<Instruction>(NAME # "Z128rmbk") VR128X:$dst,8373 VK2WM:$mask, f64mem:$src), 0, "att">;8374 def : InstAlias<OpcodeStr#"x\t{${src}{1to2}, $dst {${mask}} {z}|"8375 "$dst {${mask}} {z}, ${src}{1to2}}",8376 (!cast<Instruction>(NAME # "Z128rmbkz") VR128X:$dst,8377 VK2WM:$mask, f64mem:$src), 0, "att">;8378 8379 def : InstAlias<OpcodeStr#"y\t{$src, $dst|$dst, $src}",8380 (!cast<Instruction>(NAME # "Z256rr") VR128X:$dst,8381 VR256X:$src), 0, "att">;8382 def : InstAlias<OpcodeStr#"y\t{$src, $dst {${mask}}|$dst {${mask}}, $src}",8383 (!cast<Instruction>(NAME # "Z256rrk") VR128X:$dst,8384 VK4WM:$mask, VR256X:$src), 0, "att">;8385 def : InstAlias<OpcodeStr#"y\t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}",8386 (!cast<Instruction>(NAME # "Z256rrkz") VR128X:$dst,8387 VK4WM:$mask, VR256X:$src), 0, "att">;8388 def : InstAlias<OpcodeStr#"y\t{${src}{1to4}, $dst|$dst, ${src}{1to4}}",8389 (!cast<Instruction>(NAME # "Z256rmb") VR128X:$dst,8390 f64mem:$src), 0, "att">;8391 def : InstAlias<OpcodeStr#"y\t{${src}{1to4}, $dst {${mask}}|"8392 "$dst {${mask}}, ${src}{1to4}}",8393 (!cast<Instruction>(NAME # "Z256rmbk") VR128X:$dst,8394 VK4WM:$mask, f64mem:$src), 0, "att">;8395 def : InstAlias<OpcodeStr#"y\t{${src}{1to4}, $dst {${mask}} {z}|"8396 "$dst {${mask}} {z}, ${src}{1to4}}",8397 (!cast<Instruction>(NAME # "Z256rmbkz") VR128X:$dst,8398 VK4WM:$mask, f64mem:$src), 0, "att">;8399}8400 8401// Convert Double to Signed/Unsigned Doubleword8402multiclass avx512_cvtpd2dq<bits<8> opc, string OpcodeStr, SDNode OpNode,8403 SDNode MaskOpNode, SDNode OpNodeRnd,8404 X86SchedWriteWidths sched> {8405 let Predicates = [HasAVX512] in {8406 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f64_info, OpNode,8407 MaskOpNode, sched.ZMM>,8408 avx512_vcvt_fp_rc<opc, OpcodeStr, v8i32x_info, v8f64_info,8409 OpNodeRnd, sched.ZMM>, EVEX_V512;8410 }8411 let Predicates = [HasVLX] in {8412 // we need "x"/"y" suffixes in order to distinguish between 128 and 2568413 // memory forms of these instructions in Asm Parcer. They have the same8414 // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly8415 // due to the same reason.8416 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v2f64x_info,8417 null_frag, null_frag, sched.XMM, "{1to2}", "{x}", f128mem,8418 VK2WM>, EVEX_V128;8419 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f64x_info, OpNode,8420 MaskOpNode, sched.YMM, "{1to4}", "{y}">, EVEX_V256;8421 }8422 8423 def : InstAlias<OpcodeStr#"x\t{$src, $dst|$dst, $src}",8424 (!cast<Instruction>(NAME # "Z128rr") VR128X:$dst, VR128X:$src), 0, "att">;8425 def : InstAlias<OpcodeStr#"x\t{$src, $dst {${mask}}|$dst {${mask}}, $src}",8426 (!cast<Instruction>(NAME # "Z128rrk") VR128X:$dst,8427 VK2WM:$mask, VR128X:$src), 0, "att">;8428 def : InstAlias<OpcodeStr#"x\t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}",8429 (!cast<Instruction>(NAME # "Z128rrkz") VR128X:$dst,8430 VK2WM:$mask, VR128X:$src), 0, "att">;8431 def : InstAlias<OpcodeStr#"x\t{${src}{1to2}, $dst|$dst, ${src}{1to2}}",8432 (!cast<Instruction>(NAME # "Z128rmb") VR128X:$dst,8433 f64mem:$src), 0, "att">;8434 def : InstAlias<OpcodeStr#"x\t{${src}{1to2}, $dst {${mask}}|"8435 "$dst {${mask}}, ${src}{1to2}}",8436 (!cast<Instruction>(NAME # "Z128rmbk") VR128X:$dst,8437 VK2WM:$mask, f64mem:$src), 0, "att">;8438 def : InstAlias<OpcodeStr#"x\t{${src}{1to2}, $dst {${mask}} {z}|"8439 "$dst {${mask}} {z}, ${src}{1to2}}",8440 (!cast<Instruction>(NAME # "Z128rmbkz") VR128X:$dst,8441 VK2WM:$mask, f64mem:$src), 0, "att">;8442 8443 def : InstAlias<OpcodeStr#"y\t{$src, $dst|$dst, $src}",8444 (!cast<Instruction>(NAME # "Z256rr") VR128X:$dst, VR256X:$src), 0, "att">;8445 def : InstAlias<OpcodeStr#"y\t{$src, $dst {${mask}}|$dst {${mask}}, $src}",8446 (!cast<Instruction>(NAME # "Z256rrk") VR128X:$dst,8447 VK4WM:$mask, VR256X:$src), 0, "att">;8448 def : InstAlias<OpcodeStr#"y\t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}",8449 (!cast<Instruction>(NAME # "Z256rrkz") VR128X:$dst,8450 VK4WM:$mask, VR256X:$src), 0, "att">;8451 def : InstAlias<OpcodeStr#"y\t{${src}{1to4}, $dst|$dst, ${src}{1to4}}",8452 (!cast<Instruction>(NAME # "Z256rmb") VR128X:$dst,8453 f64mem:$src), 0, "att">;8454 def : InstAlias<OpcodeStr#"y\t{${src}{1to4}, $dst {${mask}}|"8455 "$dst {${mask}}, ${src}{1to4}}",8456 (!cast<Instruction>(NAME # "Z256rmbk") VR128X:$dst,8457 VK4WM:$mask, f64mem:$src), 0, "att">;8458 def : InstAlias<OpcodeStr#"y\t{${src}{1to4}, $dst {${mask}} {z}|"8459 "$dst {${mask}} {z}, ${src}{1to4}}",8460 (!cast<Instruction>(NAME # "Z256rmbkz") VR128X:$dst,8461 VK4WM:$mask, f64mem:$src), 0, "att">;8462}8463 8464// Convert Double to Signed/Unsigned Quardword8465multiclass avx512_cvtpd2qq<bits<8> opc, string OpcodeStr, SDNode OpNode,8466 SDNode MaskOpNode, SDNode OpNodeRnd,8467 X86SchedWriteWidths sched> {8468 let Predicates = [HasDQI] in {8469 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f64_info, OpNode,8470 MaskOpNode, sched.ZMM>,8471 avx512_vcvt_fp_rc<opc, OpcodeStr, v8i64_info, v8f64_info,8472 OpNodeRnd, sched.ZMM>, EVEX_V512;8473 }8474 let Predicates = [HasDQI, HasVLX] in {8475 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v2f64x_info, OpNode,8476 MaskOpNode, sched.XMM>, EVEX_V128;8477 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f64x_info, OpNode,8478 MaskOpNode, sched.YMM>, EVEX_V256;8479 }8480}8481 8482// Convert Double to Signed/Unsigned Quardword with truncation8483multiclass avx512_cvttpd2qq<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,8484 SDNode MaskOpNode, SDNode OpNodeRnd,8485 X86SchedWriteWidths sched> {8486 let Predicates = [HasDQI] in {8487 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f64_info, OpNode,8488 MaskOpNode, sched.ZMM>,8489 avx512_vcvt_fp_sae<opc, OpcodeStr, v8i64_info, v8f64_info,8490 OpNodeRnd, sched.ZMM>, EVEX_V512;8491 }8492 let Predicates = [HasDQI, HasVLX] in {8493 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v2f64x_info, OpNode,8494 MaskOpNode, sched.XMM>, EVEX_V128;8495 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f64x_info, OpNode,8496 MaskOpNode, sched.YMM>, EVEX_V256;8497 }8498}8499 8500// Convert Signed/Unsigned Quardword to Double8501multiclass avx512_cvtqq2pd<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,8502 SDNode MaskOpNode, SDNode OpNodeRnd,8503 X86SchedWriteWidths sched> {8504 let Predicates = [HasDQI] in {8505 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8i64_info, OpNode,8506 MaskOpNode, sched.ZMM>,8507 avx512_vcvt_fp_rc<opc, OpcodeStr, v8f64_info, v8i64_info,8508 OpNodeRnd, sched.ZMM>, EVEX_V512;8509 }8510 let Predicates = [HasDQI, HasVLX] in {8511 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v2i64x_info, OpNode,8512 MaskOpNode, sched.XMM>, EVEX_V128;8513 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4i64x_info, OpNode,8514 MaskOpNode, sched.YMM>, EVEX_V256;8515 }8516}8517 8518// Convert Float to Signed/Unsigned Quardword8519multiclass avx512_cvtps2qq<bits<8> opc, string OpcodeStr, SDNode OpNode,8520 SDNode MaskOpNode, SDNode OpNodeRnd,8521 X86SchedWriteWidths sched> {8522 let Predicates = [HasDQI] in {8523 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f32x_info, OpNode,8524 MaskOpNode, sched.ZMM>,8525 avx512_vcvt_fp_rc<opc, OpcodeStr, v8i64_info, v8f32x_info,8526 OpNodeRnd, sched.ZMM>, EVEX_V512;8527 }8528 let Predicates = [HasDQI, HasVLX] in {8529 // Explicitly specified broadcast string, since we take only 2 elements8530 // from v4f32x_info source8531 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v4f32x_info, OpNode,8532 MaskOpNode, sched.XMM, "{1to2}", "", f64mem, VK2WM,8533 (v2i64 (OpNode (bc_v4f328534 (v2f648535 (scalar_to_vector (loadf64 addr:$src)))))),8536 (v2i64 (MaskOpNode (bc_v4f328537 (v2f648538 (scalar_to_vector (loadf64 addr:$src))))))>,8539 EVEX_V128;8540 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f32x_info, OpNode,8541 MaskOpNode, sched.YMM>, EVEX_V256;8542 }8543}8544 8545// Convert Float to Signed/Unsigned Quardword with truncation8546multiclass avx512_cvttps2qq<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,8547 SDNode MaskOpNode, SDNode OpNodeRnd,8548 X86SchedWriteWidths sched> {8549 let Predicates = [HasDQI] in {8550 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f32x_info, OpNode,8551 MaskOpNode, sched.ZMM>,8552 avx512_vcvt_fp_sae<opc, OpcodeStr, v8i64_info, v8f32x_info,8553 OpNodeRnd, sched.ZMM>, EVEX_V512;8554 }8555 let Predicates = [HasDQI, HasVLX] in {8556 // Explicitly specified broadcast string, since we take only 2 elements8557 // from v4f32x_info source8558 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v4f32x_info, OpNode,8559 MaskOpNode, sched.XMM, "{1to2}", "", f64mem, VK2WM,8560 (v2i64 (OpNode (bc_v4f328561 (v2f648562 (scalar_to_vector (loadf64 addr:$src)))))),8563 (v2i64 (MaskOpNode (bc_v4f328564 (v2f648565 (scalar_to_vector (loadf64 addr:$src))))))>,8566 EVEX_V128;8567 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f32x_info, OpNode,8568 MaskOpNode, sched.YMM>, EVEX_V256;8569 }8570}8571 8572// Convert Signed/Unsigned Quardword to Float8573// Also Convert Signed/Unsigned Doubleword to Half8574multiclass avx512_cvtqq2ps_dq2ph<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,8575 SDPatternOperator MaskOpNode, SDPatternOperator OpNode128,8576 SDPatternOperator OpNode128M, SDPatternOperator OpNodeRnd,8577 AVX512VLVectorVTInfo _dst, AVX512VLVectorVTInfo _src,8578 X86SchedWriteWidths sched, Predicate prd = HasDQI> {8579 let Predicates = [prd] in {8580 defm Z : avx512_vcvt_fp<opc, OpcodeStr, _dst.info256, _src.info512, OpNode,8581 MaskOpNode, sched.ZMM>,8582 avx512_vcvt_fp_rc<opc, OpcodeStr, _dst.info256, _src.info512,8583 OpNodeRnd, sched.ZMM>, EVEX_V512;8584 }8585 let Predicates = [prd, HasVLX] in {8586 // we need "x"/"y" suffixes in order to distinguish between 128 and 2568587 // memory forms of these instructions in Asm Parcer. They have the same8588 // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly8589 // due to the same reason.8590 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, _dst.info128, _src.info128, null_frag,8591 null_frag, sched.XMM, _src.info128.BroadcastStr,8592 "{x}", i128mem, _src.info128.KRCWM>,8593 EVEX_V128;8594 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, _dst.info128, _src.info256, OpNode,8595 MaskOpNode, sched.YMM, _src.info256.BroadcastStr,8596 "{y}">, EVEX_V256;8597 8598 // Special patterns to allow use of X86VM[SU]intToFP for masking. Instruction8599 // patterns have been disabled with null_frag.8600 def : Pat<(_dst.info128.VT (OpNode128 (_src.info128.VT VR128X:$src))),8601 (!cast<Instruction>(NAME # "Z128rr") VR128X:$src)>;8602 def : Pat<(OpNode128M (_src.info128.VT VR128X:$src), (_dst.info128.VT VR128X:$src0),8603 _src.info128.KRCWM:$mask),8604 (!cast<Instruction>(NAME # "Z128rrk") VR128X:$src0, _src.info128.KRCWM:$mask, VR128X:$src)>;8605 def : Pat<(OpNode128M (_src.info128.VT VR128X:$src), _dst.info128.ImmAllZerosV,8606 _src.info128.KRCWM:$mask),8607 (!cast<Instruction>(NAME # "Z128rrkz") _src.info128.KRCWM:$mask, VR128X:$src)>;8608 8609 def : Pat<(_dst.info128.VT (OpNode128 (_src.info128.LdFrag addr:$src))),8610 (!cast<Instruction>(NAME # "Z128rm") addr:$src)>;8611 def : Pat<(OpNode128M (_src.info128.LdFrag addr:$src), (_dst.info128.VT VR128X:$src0),8612 _src.info128.KRCWM:$mask),8613 (!cast<Instruction>(NAME # "Z128rmk") VR128X:$src0, _src.info128.KRCWM:$mask, addr:$src)>;8614 def : Pat<(OpNode128M (_src.info128.LdFrag addr:$src), _dst.info128.ImmAllZerosV,8615 _src.info128.KRCWM:$mask),8616 (!cast<Instruction>(NAME # "Z128rmkz") _src.info128.KRCWM:$mask, addr:$src)>;8617 8618 def : Pat<(_dst.info128.VT (OpNode128 (_src.info128.VT (X86VBroadcastld64 addr:$src)))),8619 (!cast<Instruction>(NAME # "Z128rmb") addr:$src)>;8620 def : Pat<(OpNode128M (_src.info128.VT (X86VBroadcastld64 addr:$src)),8621 (_dst.info128.VT VR128X:$src0), _src.info128.KRCWM:$mask),8622 (!cast<Instruction>(NAME # "Z128rmbk") VR128X:$src0, _src.info128.KRCWM:$mask, addr:$src)>;8623 def : Pat<(OpNode128M (_src.info128.VT (X86VBroadcastld64 addr:$src)),8624 _dst.info128.ImmAllZerosV, _src.info128.KRCWM:$mask),8625 (!cast<Instruction>(NAME # "Z128rmbkz") _src.info128.KRCWM:$mask, addr:$src)>;8626 }8627 8628 def : InstAlias<OpcodeStr#"x\t{$src, $dst|$dst, $src}",8629 (!cast<Instruction>(NAME # "Z128rr") VR128X:$dst,8630 VR128X:$src), 0, "att">;8631 def : InstAlias<OpcodeStr#"x\t{$src, $dst {${mask}}|$dst {${mask}}, $src}",8632 (!cast<Instruction>(NAME # "Z128rrk") VR128X:$dst,8633 VK2WM:$mask, VR128X:$src), 0, "att">;8634 def : InstAlias<OpcodeStr#"x\t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}",8635 (!cast<Instruction>(NAME # "Z128rrkz") VR128X:$dst,8636 VK2WM:$mask, VR128X:$src), 0, "att">;8637 def : InstAlias<OpcodeStr#"x\t{${src}{1to2}, $dst|$dst, ${src}{1to2}}",8638 (!cast<Instruction>(NAME # "Z128rmb") VR128X:$dst,8639 i64mem:$src), 0, "att">;8640 def : InstAlias<OpcodeStr#"x\t{${src}{1to2}, $dst {${mask}}|"8641 "$dst {${mask}}, ${src}{1to2}}",8642 (!cast<Instruction>(NAME # "Z128rmbk") VR128X:$dst,8643 VK2WM:$mask, i64mem:$src), 0, "att">;8644 def : InstAlias<OpcodeStr#"x\t{${src}{1to2}, $dst {${mask}} {z}|"8645 "$dst {${mask}} {z}, ${src}{1to2}}",8646 (!cast<Instruction>(NAME # "Z128rmbkz") VR128X:$dst,8647 VK2WM:$mask, i64mem:$src), 0, "att">;8648 8649 def : InstAlias<OpcodeStr#"y\t{$src, $dst|$dst, $src}",8650 (!cast<Instruction>(NAME # "Z256rr") VR128X:$dst,8651 VR256X:$src), 0, "att">;8652 def : InstAlias<OpcodeStr#"y\t{$src, $dst {${mask}}|"8653 "$dst {${mask}}, $src}",8654 (!cast<Instruction>(NAME # "Z256rrk") VR128X:$dst,8655 VK4WM:$mask, VR256X:$src), 0, "att">;8656 def : InstAlias<OpcodeStr#"y\t{$src, $dst {${mask}} {z}|"8657 "$dst {${mask}} {z}, $src}",8658 (!cast<Instruction>(NAME # "Z256rrkz") VR128X:$dst,8659 VK4WM:$mask, VR256X:$src), 0, "att">;8660 def : InstAlias<OpcodeStr#"y\t{${src}{1to4}, $dst|$dst, ${src}{1to4}}",8661 (!cast<Instruction>(NAME # "Z256rmb") VR128X:$dst,8662 i64mem:$src), 0, "att">;8663 def : InstAlias<OpcodeStr#"y\t{${src}{1to4}, $dst {${mask}}|"8664 "$dst {${mask}}, ${src}{1to4}}",8665 (!cast<Instruction>(NAME # "Z256rmbk") VR128X:$dst,8666 VK4WM:$mask, i64mem:$src), 0, "att">;8667 def : InstAlias<OpcodeStr#"y\t{${src}{1to4}, $dst {${mask}} {z}|"8668 "$dst {${mask}} {z}, ${src}{1to4}}",8669 (!cast<Instruction>(NAME # "Z256rmbkz") VR128X:$dst,8670 VK4WM:$mask, i64mem:$src), 0, "att">;8671}8672 8673defm VCVTDQ2PD : avx512_cvtdq2pd<0xE6, "vcvtdq2pd", any_sint_to_fp, sint_to_fp,8674 X86any_VSintToFP, X86VSintToFP,8675 SchedWriteCvtDQ2PD>, TB, XS, EVEX_CD8<32, CD8VH>;8676 8677defm VCVTDQ2PS : avx512_cvtdq2ps<0x5B, "vcvtdq2ps", any_sint_to_fp, sint_to_fp,8678 X86VSintToFpRnd, SchedWriteCvtDQ2PS>,8679 TB, EVEX_CD8<32, CD8VF>;8680 8681defm VCVTTPS2DQ : avx512_cvttps2dq<0x5B, "vcvttps2dq", X86any_cvttp2si,8682 X86cvttp2si, X86cvttp2siSAE,8683 SchedWriteCvtPS2DQ>, TB, XS, EVEX_CD8<32, CD8VF>;8684 8685defm VCVTTPD2DQ : avx512_cvttpd2dq<0xE6, "vcvttpd2dq", X86any_cvttp2si,8686 X86cvttp2si, X86cvttp2siSAE,8687 SchedWriteCvtPD2DQ>,8688 TB, PD, REX_W, EVEX_CD8<64, CD8VF>;8689 8690defm VCVTTPS2UDQ : avx512_cvttps2dq<0x78, "vcvttps2udq", X86any_cvttp2ui,8691 X86cvttp2ui, X86cvttp2uiSAE,8692 SchedWriteCvtPS2DQ>, TB, EVEX_CD8<32, CD8VF>;8693 8694defm VCVTTPD2UDQ : avx512_cvttpd2dq<0x78, "vcvttpd2udq", X86any_cvttp2ui,8695 X86cvttp2ui, X86cvttp2uiSAE,8696 SchedWriteCvtPD2DQ>,8697 TB, REX_W, EVEX_CD8<64, CD8VF>;8698 8699defm VCVTUDQ2PD : avx512_cvtdq2pd<0x7A, "vcvtudq2pd", any_uint_to_fp,8700 uint_to_fp, X86any_VUintToFP, X86VUintToFP,8701 SchedWriteCvtDQ2PD>, TB, XS, EVEX_CD8<32, CD8VH>;8702 8703defm VCVTUDQ2PS : avx512_cvtdq2ps<0x7A, "vcvtudq2ps", any_uint_to_fp,8704 uint_to_fp, X86VUintToFpRnd,8705 SchedWriteCvtDQ2PS>, TB, XD, EVEX_CD8<32, CD8VF>;8706 8707defm VCVTPS2DQ : avx512_cvtps2dq<0x5B, "vcvtps2dq", X86cvtp2Int, X86cvtp2Int,8708 X86cvtp2IntRnd, SchedWriteCvtPS2DQ>, TB, PD,8709 EVEX_CD8<32, CD8VF>;8710 8711defm VCVTPD2DQ : avx512_cvtpd2dq<0xE6, "vcvtpd2dq", X86cvtp2Int, X86cvtp2Int,8712 X86cvtp2IntRnd, SchedWriteCvtPD2DQ>, TB, XD,8713 REX_W, EVEX_CD8<64, CD8VF>;8714 8715defm VCVTPS2UDQ : avx512_cvtps2dq<0x79, "vcvtps2udq", X86cvtp2UInt, X86cvtp2UInt,8716 X86cvtp2UIntRnd, SchedWriteCvtPS2DQ>,8717 TB, EVEX_CD8<32, CD8VF>;8718 8719defm VCVTPD2UDQ : avx512_cvtpd2dq<0x79, "vcvtpd2udq", X86cvtp2UInt, X86cvtp2UInt,8720 X86cvtp2UIntRnd, SchedWriteCvtPD2DQ>, REX_W,8721 TB, EVEX_CD8<64, CD8VF>;8722 8723defm VCVTPD2QQ : avx512_cvtpd2qq<0x7B, "vcvtpd2qq", X86cvtp2Int, X86cvtp2Int,8724 X86cvtp2IntRnd, SchedWriteCvtPD2DQ>, REX_W,8725 TB, PD, EVEX_CD8<64, CD8VF>;8726 8727defm VCVTPS2QQ : avx512_cvtps2qq<0x7B, "vcvtps2qq", X86cvtp2Int, X86cvtp2Int,8728 X86cvtp2IntRnd, SchedWriteCvtPS2DQ>, TB, PD,8729 EVEX_CD8<32, CD8VH>;8730 8731defm VCVTPD2UQQ : avx512_cvtpd2qq<0x79, "vcvtpd2uqq", X86cvtp2UInt, X86cvtp2UInt,8732 X86cvtp2UIntRnd, SchedWriteCvtPD2DQ>, REX_W,8733 TB, PD, EVEX_CD8<64, CD8VF>;8734 8735defm VCVTPS2UQQ : avx512_cvtps2qq<0x79, "vcvtps2uqq", X86cvtp2UInt, X86cvtp2UInt,8736 X86cvtp2UIntRnd, SchedWriteCvtPS2DQ>, TB, PD,8737 EVEX_CD8<32, CD8VH>;8738 8739defm VCVTTPD2QQ : avx512_cvttpd2qq<0x7A, "vcvttpd2qq", X86any_cvttp2si,8740 X86cvttp2si, X86cvttp2siSAE,8741 SchedWriteCvtPD2DQ>, REX_W,8742 TB, PD, EVEX_CD8<64, CD8VF>;8743 8744defm VCVTTPS2QQ : avx512_cvttps2qq<0x7A, "vcvttps2qq", X86any_cvttp2si,8745 X86cvttp2si, X86cvttp2siSAE,8746 SchedWriteCvtPS2DQ>, TB, PD,8747 EVEX_CD8<32, CD8VH>;8748 8749defm VCVTTPD2UQQ : avx512_cvttpd2qq<0x78, "vcvttpd2uqq", X86any_cvttp2ui,8750 X86cvttp2ui, X86cvttp2uiSAE,8751 SchedWriteCvtPD2DQ>, REX_W,8752 TB, PD, EVEX_CD8<64, CD8VF>;8753 8754defm VCVTTPS2UQQ : avx512_cvttps2qq<0x78, "vcvttps2uqq", X86any_cvttp2ui,8755 X86cvttp2ui, X86cvttp2uiSAE,8756 SchedWriteCvtPS2DQ>, TB, PD,8757 EVEX_CD8<32, CD8VH>;8758 8759defm VCVTQQ2PD : avx512_cvtqq2pd<0xE6, "vcvtqq2pd", any_sint_to_fp,8760 sint_to_fp, X86VSintToFpRnd,8761 SchedWriteCvtDQ2PD>, REX_W, TB, XS, EVEX_CD8<64, CD8VF>;8762 8763defm VCVTUQQ2PD : avx512_cvtqq2pd<0x7A, "vcvtuqq2pd", any_uint_to_fp,8764 uint_to_fp, X86VUintToFpRnd, SchedWriteCvtDQ2PD>,8765 REX_W, TB, XS, EVEX_CD8<64, CD8VF>;8766 8767defm VCVTDQ2PH : avx512_cvtqq2ps_dq2ph<0x5B, "vcvtdq2ph", any_sint_to_fp, sint_to_fp,8768 X86any_VSintToFP, X86VMSintToFP,8769 X86VSintToFpRnd, avx512vl_f16_info, avx512vl_i32_info,8770 SchedWriteCvtDQ2PS, HasFP16>,8771 T_MAP5, EVEX_CD8<32, CD8VF>;8772 8773defm VCVTUDQ2PH : avx512_cvtqq2ps_dq2ph<0x7A, "vcvtudq2ph", any_uint_to_fp, uint_to_fp,8774 X86any_VUintToFP, X86VMUintToFP,8775 X86VUintToFpRnd, avx512vl_f16_info, avx512vl_i32_info,8776 SchedWriteCvtDQ2PS, HasFP16>, T_MAP5, XD,8777 EVEX_CD8<32, CD8VF>;8778 8779defm VCVTQQ2PS : avx512_cvtqq2ps_dq2ph<0x5B, "vcvtqq2ps", any_sint_to_fp, sint_to_fp,8780 X86any_VSintToFP, X86VMSintToFP,8781 X86VSintToFpRnd, avx512vl_f32_info, avx512vl_i64_info,8782 SchedWriteCvtDQ2PS>, REX_W, TB,8783 EVEX_CD8<64, CD8VF>;8784 8785defm VCVTUQQ2PS : avx512_cvtqq2ps_dq2ph<0x7A, "vcvtuqq2ps", any_uint_to_fp, uint_to_fp,8786 X86any_VUintToFP, X86VMUintToFP,8787 X86VUintToFpRnd, avx512vl_f32_info, avx512vl_i64_info,8788 SchedWriteCvtDQ2PS>, REX_W, TB, XD,8789 EVEX_CD8<64, CD8VF>;8790 8791let Predicates = [HasVLX] in {8792 // Special patterns to allow use of X86mcvtp2Int for masking. Instruction8793 // patterns have been disabled with null_frag.8794 def : Pat<(v4i32 (X86cvtp2Int (v2f64 VR128X:$src))),8795 (VCVTPD2DQZ128rr VR128X:$src)>;8796 def : Pat<(X86mcvtp2Int (v2f64 VR128X:$src), (v4i32 VR128X:$src0),8797 VK2WM:$mask),8798 (VCVTPD2DQZ128rrk VR128X:$src0, VK2WM:$mask, VR128X:$src)>;8799 def : Pat<(X86mcvtp2Int (v2f64 VR128X:$src), v4i32x_info.ImmAllZerosV,8800 VK2WM:$mask),8801 (VCVTPD2DQZ128rrkz VK2WM:$mask, VR128X:$src)>;8802 8803 def : Pat<(v4i32 (X86cvtp2Int (loadv2f64 addr:$src))),8804 (VCVTPD2DQZ128rm addr:$src)>;8805 def : Pat<(X86mcvtp2Int (loadv2f64 addr:$src), (v4i32 VR128X:$src0),8806 VK2WM:$mask),8807 (VCVTPD2DQZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;8808 def : Pat<(X86mcvtp2Int (loadv2f64 addr:$src), v4i32x_info.ImmAllZerosV,8809 VK2WM:$mask),8810 (VCVTPD2DQZ128rmkz VK2WM:$mask, addr:$src)>;8811 8812 def : Pat<(v4i32 (X86cvtp2Int (v2f64 (X86VBroadcastld64 addr:$src)))),8813 (VCVTPD2DQZ128rmb addr:$src)>;8814 def : Pat<(X86mcvtp2Int (v2f64 (X86VBroadcastld64 addr:$src)),8815 (v4i32 VR128X:$src0), VK2WM:$mask),8816 (VCVTPD2DQZ128rmbk VR128X:$src0, VK2WM:$mask, addr:$src)>;8817 def : Pat<(X86mcvtp2Int (v2f64 (X86VBroadcastld64 addr:$src)),8818 v4i32x_info.ImmAllZerosV, VK2WM:$mask),8819 (VCVTPD2DQZ128rmbkz VK2WM:$mask, addr:$src)>;8820 8821 // Special patterns to allow use of X86mcvttp2si for masking. Instruction8822 // patterns have been disabled with null_frag.8823 def : Pat<(v4i32 (X86any_cvttp2si (v2f64 VR128X:$src))),8824 (VCVTTPD2DQZ128rr VR128X:$src)>;8825 def : Pat<(X86mcvttp2si (v2f64 VR128X:$src), (v4i32 VR128X:$src0),8826 VK2WM:$mask),8827 (VCVTTPD2DQZ128rrk VR128X:$src0, VK2WM:$mask, VR128X:$src)>;8828 def : Pat<(X86mcvttp2si (v2f64 VR128X:$src), v4i32x_info.ImmAllZerosV,8829 VK2WM:$mask),8830 (VCVTTPD2DQZ128rrkz VK2WM:$mask, VR128X:$src)>;8831 8832 def : Pat<(v4i32 (X86any_cvttp2si (loadv2f64 addr:$src))),8833 (VCVTTPD2DQZ128rm addr:$src)>;8834 def : Pat<(X86mcvttp2si (loadv2f64 addr:$src), (v4i32 VR128X:$src0),8835 VK2WM:$mask),8836 (VCVTTPD2DQZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;8837 def : Pat<(X86mcvttp2si (loadv2f64 addr:$src), v4i32x_info.ImmAllZerosV,8838 VK2WM:$mask),8839 (VCVTTPD2DQZ128rmkz VK2WM:$mask, addr:$src)>;8840 8841 def : Pat<(v4i32 (X86any_cvttp2si (v2f64 (X86VBroadcastld64 addr:$src)))),8842 (VCVTTPD2DQZ128rmb addr:$src)>;8843 def : Pat<(X86mcvttp2si (v2f64 (X86VBroadcastld64 addr:$src)),8844 (v4i32 VR128X:$src0), VK2WM:$mask),8845 (VCVTTPD2DQZ128rmbk VR128X:$src0, VK2WM:$mask, addr:$src)>;8846 def : Pat<(X86mcvttp2si (v2f64 (X86VBroadcastld64 addr:$src)),8847 v4i32x_info.ImmAllZerosV, VK2WM:$mask),8848 (VCVTTPD2DQZ128rmbkz VK2WM:$mask, addr:$src)>;8849 8850 // Special patterns to allow use of X86mcvtp2UInt for masking. Instruction8851 // patterns have been disabled with null_frag.8852 def : Pat<(v4i32 (X86cvtp2UInt (v2f64 VR128X:$src))),8853 (VCVTPD2UDQZ128rr VR128X:$src)>;8854 def : Pat<(X86mcvtp2UInt (v2f64 VR128X:$src), (v4i32 VR128X:$src0),8855 VK2WM:$mask),8856 (VCVTPD2UDQZ128rrk VR128X:$src0, VK2WM:$mask, VR128X:$src)>;8857 def : Pat<(X86mcvtp2UInt (v2f64 VR128X:$src), v4i32x_info.ImmAllZerosV,8858 VK2WM:$mask),8859 (VCVTPD2UDQZ128rrkz VK2WM:$mask, VR128X:$src)>;8860 8861 def : Pat<(v4i32 (X86cvtp2UInt (loadv2f64 addr:$src))),8862 (VCVTPD2UDQZ128rm addr:$src)>;8863 def : Pat<(X86mcvtp2UInt (loadv2f64 addr:$src), (v4i32 VR128X:$src0),8864 VK2WM:$mask),8865 (VCVTPD2UDQZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;8866 def : Pat<(X86mcvtp2UInt (loadv2f64 addr:$src), v4i32x_info.ImmAllZerosV,8867 VK2WM:$mask),8868 (VCVTPD2UDQZ128rmkz VK2WM:$mask, addr:$src)>;8869 8870 def : Pat<(v4i32 (X86cvtp2UInt (v2f64 (X86VBroadcastld64 addr:$src)))),8871 (VCVTPD2UDQZ128rmb addr:$src)>;8872 def : Pat<(X86mcvtp2UInt (v2f64 (X86VBroadcastld64 addr:$src)),8873 (v4i32 VR128X:$src0), VK2WM:$mask),8874 (VCVTPD2UDQZ128rmbk VR128X:$src0, VK2WM:$mask, addr:$src)>;8875 def : Pat<(X86mcvtp2UInt (v2f64 (X86VBroadcastld64 addr:$src)),8876 v4i32x_info.ImmAllZerosV, VK2WM:$mask),8877 (VCVTPD2UDQZ128rmbkz VK2WM:$mask, addr:$src)>;8878 8879 // Special patterns to allow use of X86mcvtp2UInt for masking. Instruction8880 // patterns have been disabled with null_frag.8881 def : Pat<(v4i32 (X86any_cvttp2ui (v2f64 VR128X:$src))),8882 (VCVTTPD2UDQZ128rr VR128X:$src)>;8883 def : Pat<(X86mcvttp2ui (v2f64 VR128X:$src), (v4i32 VR128X:$src0),8884 VK2WM:$mask),8885 (VCVTTPD2UDQZ128rrk VR128X:$src0, VK2WM:$mask, VR128X:$src)>;8886 def : Pat<(X86mcvttp2ui (v2f64 VR128X:$src), v4i32x_info.ImmAllZerosV,8887 VK2WM:$mask),8888 (VCVTTPD2UDQZ128rrkz VK2WM:$mask, VR128X:$src)>;8889 8890 def : Pat<(v4i32 (X86any_cvttp2ui (loadv2f64 addr:$src))),8891 (VCVTTPD2UDQZ128rm addr:$src)>;8892 def : Pat<(X86mcvttp2ui (loadv2f64 addr:$src), (v4i32 VR128X:$src0),8893 VK2WM:$mask),8894 (VCVTTPD2UDQZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;8895 def : Pat<(X86mcvttp2ui (loadv2f64 addr:$src), v4i32x_info.ImmAllZerosV,8896 VK2WM:$mask),8897 (VCVTTPD2UDQZ128rmkz VK2WM:$mask, addr:$src)>;8898 8899 def : Pat<(v4i32 (X86any_cvttp2ui (v2f64 (X86VBroadcastld64 addr:$src)))),8900 (VCVTTPD2UDQZ128rmb addr:$src)>;8901 def : Pat<(X86mcvttp2ui (v2f64 (X86VBroadcastld64 addr:$src)),8902 (v4i32 VR128X:$src0), VK2WM:$mask),8903 (VCVTTPD2UDQZ128rmbk VR128X:$src0, VK2WM:$mask, addr:$src)>;8904 def : Pat<(X86mcvttp2ui (v2f64 (X86VBroadcastld64 addr:$src)),8905 v4i32x_info.ImmAllZerosV, VK2WM:$mask),8906 (VCVTTPD2UDQZ128rmbkz VK2WM:$mask, addr:$src)>;8907 8908 def : Pat<(v4i32 (lrint VR128X:$src)), (VCVTPS2DQZ128rr VR128X:$src)>;8909 def : Pat<(v4i32 (lrint (loadv4f32 addr:$src))), (VCVTPS2DQZ128rm addr:$src)>;8910 def : Pat<(v8i32 (lrint VR256X:$src)), (VCVTPS2DQZ256rr VR256X:$src)>;8911 def : Pat<(v8i32 (lrint (loadv8f32 addr:$src))), (VCVTPS2DQZ256rm addr:$src)>;8912 def : Pat<(v4i32 (lrint VR256X:$src)), (VCVTPD2DQZ256rr VR256X:$src)>;8913 def : Pat<(v4i32 (lrint (loadv4f64 addr:$src))), (VCVTPD2DQZ256rm addr:$src)>;8914}8915def : Pat<(v16i32 (lrint VR512:$src)), (VCVTPS2DQZrr VR512:$src)>;8916def : Pat<(v16i32 (lrint (loadv16f32 addr:$src))), (VCVTPS2DQZrm addr:$src)>;8917def : Pat<(v8i32 (lrint VR512:$src)), (VCVTPD2DQZrr VR512:$src)>;8918def : Pat<(v8i32 (lrint (loadv8f64 addr:$src))), (VCVTPD2DQZrm addr:$src)>;8919 8920let Predicates = [HasDQI, HasVLX] in {8921 def : Pat<(v2i64 (X86cvtp2Int (bc_v4f32 (v2f64 (X86vzload64 addr:$src))))),8922 (VCVTPS2QQZ128rm addr:$src)>;8923 def : Pat<(v2i64 (vselect_mask VK2WM:$mask,8924 (X86cvtp2Int (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),8925 VR128X:$src0)),8926 (VCVTPS2QQZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;8927 def : Pat<(v2i64 (vselect_mask VK2WM:$mask,8928 (X86cvtp2Int (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),8929 v2i64x_info.ImmAllZerosV)),8930 (VCVTPS2QQZ128rmkz VK2WM:$mask, addr:$src)>;8931 8932 def : Pat<(v2i64 (X86cvtp2UInt (bc_v4f32 (v2f64 (X86vzload64 addr:$src))))),8933 (VCVTPS2UQQZ128rm addr:$src)>;8934 def : Pat<(v2i64 (vselect_mask VK2WM:$mask,8935 (X86cvtp2UInt (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),8936 VR128X:$src0)),8937 (VCVTPS2UQQZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;8938 def : Pat<(v2i64 (vselect_mask VK2WM:$mask,8939 (X86cvtp2UInt (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),8940 v2i64x_info.ImmAllZerosV)),8941 (VCVTPS2UQQZ128rmkz VK2WM:$mask, addr:$src)>;8942 8943 def : Pat<(v2i64 (X86any_cvttp2si (bc_v4f32 (v2f64 (X86vzload64 addr:$src))))),8944 (VCVTTPS2QQZ128rm addr:$src)>;8945 def : Pat<(v2i64 (vselect_mask VK2WM:$mask,8946 (X86cvttp2si (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),8947 VR128X:$src0)),8948 (VCVTTPS2QQZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;8949 def : Pat<(v2i64 (vselect_mask VK2WM:$mask,8950 (X86cvttp2si (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),8951 v2i64x_info.ImmAllZerosV)),8952 (VCVTTPS2QQZ128rmkz VK2WM:$mask, addr:$src)>;8953 8954 def : Pat<(v2i64 (X86any_cvttp2ui (bc_v4f32 (v2f64 (X86vzload64 addr:$src))))),8955 (VCVTTPS2UQQZ128rm addr:$src)>;8956 def : Pat<(v2i64 (vselect_mask VK2WM:$mask,8957 (X86cvttp2ui (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),8958 VR128X:$src0)),8959 (VCVTTPS2UQQZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;8960 def : Pat<(v2i64 (vselect_mask VK2WM:$mask,8961 (X86cvttp2ui (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),8962 v2i64x_info.ImmAllZerosV)),8963 (VCVTTPS2UQQZ128rmkz VK2WM:$mask, addr:$src)>;8964 8965 def : Pat<(v4i64 (lrint VR128X:$src)), (VCVTPS2QQZ256rr VR128X:$src)>;8966 def : Pat<(v4i64 (lrint (loadv4f32 addr:$src))), (VCVTPS2QQZ256rm addr:$src)>;8967 def : Pat<(v4i64 (llrint VR128X:$src)), (VCVTPS2QQZ256rr VR128X:$src)>;8968 def : Pat<(v4i64 (llrint (loadv4f32 addr:$src))), (VCVTPS2QQZ256rm addr:$src)>;8969 def : Pat<(v2i64 (lrint VR128X:$src)), (VCVTPD2QQZ128rr VR128X:$src)>;8970 def : Pat<(v2i64 (lrint (loadv2f64 addr:$src))), (VCVTPD2QQZ128rm addr:$src)>;8971 def : Pat<(v4i64 (lrint VR256X:$src)), (VCVTPD2QQZ256rr VR256X:$src)>;8972 def : Pat<(v4i64 (lrint (loadv4f64 addr:$src))), (VCVTPD2QQZ256rm addr:$src)>;8973 def : Pat<(v2i64 (llrint VR128X:$src)), (VCVTPD2QQZ128rr VR128X:$src)>;8974 def : Pat<(v2i64 (llrint (loadv2f64 addr:$src))), (VCVTPD2QQZ128rm addr:$src)>;8975 def : Pat<(v4i64 (llrint VR256X:$src)), (VCVTPD2QQZ256rr VR256X:$src)>;8976 def : Pat<(v4i64 (llrint (loadv4f64 addr:$src))), (VCVTPD2QQZ256rm addr:$src)>;8977}8978 8979let Predicates = [HasDQI] in {8980 def : Pat<(v8i64 (lrint VR256X:$src)), (VCVTPS2QQZrr VR256X:$src)>;8981 def : Pat<(v8i64 (lrint (loadv8f32 addr:$src))), (VCVTPS2QQZrm addr:$src)>;8982 def : Pat<(v8i64 (llrint VR256X:$src)), (VCVTPS2QQZrr VR256X:$src)>;8983 def : Pat<(v8i64 (llrint (loadv8f32 addr:$src))), (VCVTPS2QQZrm addr:$src)>;8984 def : Pat<(v8i64 (lrint VR512:$src)), (VCVTPD2QQZrr VR512:$src)>;8985 def : Pat<(v8i64 (lrint (loadv8f64 addr:$src))), (VCVTPD2QQZrm addr:$src)>;8986 def : Pat<(v8i64 (llrint VR512:$src)), (VCVTPD2QQZrr VR512:$src)>;8987 def : Pat<(v8i64 (llrint (loadv8f64 addr:$src))), (VCVTPD2QQZrm addr:$src)>;8988}8989 8990let Predicates = [HasVLX] in {8991 def : Pat<(v2f64 (X86any_VSintToFP (bc_v4i32 (v2i64 (X86vzload64 addr:$src))))),8992 (VCVTDQ2PDZ128rm addr:$src)>;8993 def : Pat<(v2f64 (vselect_mask VK2WM:$mask,8994 (X86VSintToFP (bc_v4i32 (v2i64 (X86vzload64 addr:$src)))),8995 VR128X:$src0)),8996 (VCVTDQ2PDZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;8997 def : Pat<(v2f64 (vselect_mask VK2WM:$mask,8998 (X86VSintToFP (bc_v4i32 (v2i64 (X86vzload64 addr:$src)))),8999 v2f64x_info.ImmAllZerosV)),9000 (VCVTDQ2PDZ128rmkz VK2WM:$mask, addr:$src)>;9001 9002 def : Pat<(v2f64 (X86any_VUintToFP (bc_v4i32 (v2i64 (X86vzload64 addr:$src))))),9003 (VCVTUDQ2PDZ128rm addr:$src)>;9004 def : Pat<(v2f64 (vselect_mask VK2WM:$mask,9005 (X86VUintToFP (bc_v4i32 (v2i64 (X86vzload64 addr:$src)))),9006 VR128X:$src0)),9007 (VCVTUDQ2PDZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;9008 def : Pat<(v2f64 (vselect_mask VK2WM:$mask,9009 (X86VUintToFP (bc_v4i32 (v2i64 (X86vzload64 addr:$src)))),9010 v2f64x_info.ImmAllZerosV)),9011 (VCVTUDQ2PDZ128rmkz VK2WM:$mask, addr:$src)>;9012}9013 9014//===----------------------------------------------------------------------===//9015// Half precision conversion instructions9016//===----------------------------------------------------------------------===//9017 9018let Uses = [MXCSR], mayRaiseFPException = 1 in9019multiclass avx512_cvtph2ps<X86VectorVTInfo _dest, X86VectorVTInfo _src,9020 X86MemOperand x86memop, dag ld_dag,9021 X86FoldableSchedWrite sched> {9022 defm rr : AVX512_maskable_split<0x13, MRMSrcReg, _dest ,(outs _dest.RC:$dst),9023 (ins _src.RC:$src), "vcvtph2ps", "$src", "$src",9024 (X86any_cvtph2ps (_src.VT _src.RC:$src)),9025 (X86cvtph2ps (_src.VT _src.RC:$src))>,9026 T8, PD, Sched<[sched]>;9027 let mayLoad = 1 in9028 defm rm : AVX512_maskable_split<0x13, MRMSrcMem, _dest, (outs _dest.RC:$dst),9029 (ins x86memop:$src), "vcvtph2ps", "$src", "$src",9030 (X86any_cvtph2ps (_src.VT ld_dag)),9031 (X86cvtph2ps (_src.VT ld_dag))>,9032 T8, PD, Sched<[sched.Folded]>;9033}9034 9035multiclass avx512_cvtph2ps_sae<X86VectorVTInfo _dest, X86VectorVTInfo _src,9036 X86FoldableSchedWrite sched> {9037 let Uses = [MXCSR] in9038 defm rrb : AVX512_maskable<0x13, MRMSrcReg, _dest, (outs _dest.RC:$dst),9039 (ins _src.RC:$src), "vcvtph2ps",9040 "{sae}, $src", "$src, {sae}",9041 (X86cvtph2psSAE (_src.VT _src.RC:$src))>,9042 T8, PD, EVEX_B, Sched<[sched]>;9043}9044 9045let Predicates = [HasAVX512] in9046 defm VCVTPH2PSZ : avx512_cvtph2ps<v16f32_info, v16i16x_info, f256mem,9047 (load addr:$src), WriteCvtPH2PSZ>,9048 avx512_cvtph2ps_sae<v16f32_info, v16i16x_info, WriteCvtPH2PSZ>,9049 EVEX, EVEX_V512, EVEX_CD8<32, CD8VH>;9050 9051let Predicates = [HasVLX] in {9052 defm VCVTPH2PSZ256 : avx512_cvtph2ps<v8f32x_info, v8i16x_info, f128mem,9053 (load addr:$src), WriteCvtPH2PSY>, EVEX, EVEX_V256,9054 EVEX_CD8<32, CD8VH>;9055 defm VCVTPH2PSZ128 : avx512_cvtph2ps<v4f32x_info, v8i16x_info, f64mem,9056 (bitconvert (v2i64 (X86vzload64 addr:$src))),9057 WriteCvtPH2PS>, EVEX, EVEX_V128,9058 EVEX_CD8<32, CD8VH>;9059 9060 // Pattern match vcvtph2ps of a scalar i64 load.9061 def : Pat<(v4f32 (X86any_cvtph2ps (v8i16 (bitconvert9062 (v2i64 (scalar_to_vector (loadi64 addr:$src))))))),9063 (VCVTPH2PSZ128rm addr:$src)>;9064}9065 9066multiclass avx512_cvtps2ph<X86VectorVTInfo _dest, X86VectorVTInfo _src,9067 X86MemOperand x86memop, SchedWrite RR, SchedWrite MR> {9068let ExeDomain = GenericDomain, Uses = [MXCSR], mayRaiseFPException = 1 in {9069 def rr : AVX512AIi8<0x1D, MRMDestReg, (outs _dest.RC:$dst),9070 (ins _src.RC:$src1, i32u8imm:$src2),9071 "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}",9072 [(set _dest.RC:$dst,9073 (X86any_cvtps2ph (_src.VT _src.RC:$src1), (i32 timm:$src2)))]>,9074 Sched<[RR]>;9075 let Constraints = "$src0 = $dst" in9076 def rrk : AVX512AIi8<0x1D, MRMDestReg, (outs _dest.RC:$dst),9077 (ins _dest.RC:$src0, _src.KRCWM:$mask, _src.RC:$src1, i32u8imm:$src2),9078 "vcvtps2ph\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",9079 [(set _dest.RC:$dst,9080 (X86mcvtps2ph (_src.VT _src.RC:$src1), (i32 timm:$src2),9081 _dest.RC:$src0, _src.KRCWM:$mask))]>,9082 Sched<[RR]>, EVEX_K;9083 def rrkz : AVX512AIi8<0x1D, MRMDestReg, (outs _dest.RC:$dst),9084 (ins _src.KRCWM:$mask, _src.RC:$src1, i32u8imm:$src2),9085 "vcvtps2ph\t{$src2, $src1, $dst {${mask}} {z}|$dst {${mask}} {z}, $src1, $src2}",9086 [(set _dest.RC:$dst,9087 (X86mcvtps2ph (_src.VT _src.RC:$src1), (i32 timm:$src2),9088 _dest.ImmAllZerosV, _src.KRCWM:$mask))]>,9089 Sched<[RR]>, EVEX_KZ;9090 let hasSideEffects = 0, mayStore = 1 in {9091 def mr : AVX512AIi8<0x1D, MRMDestMem, (outs),9092 (ins x86memop:$dst, _src.RC:$src1, i32u8imm:$src2),9093 "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,9094 Sched<[MR]>;9095 def mrk : AVX512AIi8<0x1D, MRMDestMem, (outs),9096 (ins x86memop:$dst, _dest.KRCWM:$mask, _src.RC:$src1, i32u8imm:$src2),9097 "vcvtps2ph\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}", []>,9098 EVEX_K, Sched<[MR]>;9099 }9100}9101}9102 9103multiclass avx512_cvtps2ph_sae<X86VectorVTInfo _dest, X86VectorVTInfo _src,9104 SchedWrite Sched> {9105 let hasSideEffects = 0, Uses = [MXCSR] in {9106 def rrb : AVX512AIi8<0x1D, MRMDestReg, (outs _dest.RC:$dst),9107 (ins _src.RC:$src1, i32u8imm:$src2),9108 "vcvtps2ph\t{$src2, {sae}, $src1, $dst|$dst, $src1, {sae}, $src2}",9109 [(set _dest.RC:$dst,9110 (X86cvtps2phSAE (_src.VT _src.RC:$src1), (i32 timm:$src2)))]>,9111 EVEX_B, Sched<[Sched]>;9112 let Constraints = "$src0 = $dst" in9113 def rrbk : AVX512AIi8<0x1D, MRMDestReg, (outs _dest.RC:$dst),9114 (ins _dest.RC:$src0, _src.KRCWM:$mask, _src.RC:$src1, i32u8imm:$src2),9115 "vcvtps2ph\t{$src2, {sae}, $src1, $dst {${mask}}|$dst {${mask}}, $src1, {sae}, $src2}",9116 [(set _dest.RC:$dst,9117 (X86mcvtps2phSAE (_src.VT _src.RC:$src1), (i32 timm:$src2),9118 _dest.RC:$src0, _src.KRCWM:$mask))]>,9119 EVEX_B, Sched<[Sched]>, EVEX_K;9120 def rrbkz : AVX512AIi8<0x1D, MRMDestReg, (outs _dest.RC:$dst),9121 (ins _src.KRCWM:$mask, _src.RC:$src1, i32u8imm:$src2),9122 "vcvtps2ph\t{$src2, {sae}, $src1, $dst {${mask}} {z}|$dst {${mask}} {z}, $src1, {sae}, $src2}",9123 [(set _dest.RC:$dst,9124 (X86mcvtps2phSAE (_src.VT _src.RC:$src1), (i32 timm:$src2),9125 _dest.ImmAllZerosV, _src.KRCWM:$mask))]>,9126 EVEX_B, Sched<[Sched]>, EVEX_KZ;9127}9128}9129 9130let Predicates = [HasAVX512] in {9131 defm VCVTPS2PHZ : avx512_cvtps2ph<v16i16x_info, v16f32_info, f256mem,9132 WriteCvtPS2PHZ, WriteCvtPS2PHZSt>,9133 avx512_cvtps2ph_sae<v16i16x_info, v16f32_info, WriteCvtPS2PHZ>,9134 EVEX, EVEX_V512, EVEX_CD8<32, CD8VH>;9135 9136 def : Pat<(store (v16i16 (X86any_cvtps2ph VR512:$src1, timm:$src2)), addr:$dst),9137 (VCVTPS2PHZmr addr:$dst, VR512:$src1, timm:$src2)>;9138}9139 9140let Predicates = [HasVLX] in {9141 defm VCVTPS2PHZ256 : avx512_cvtps2ph<v8i16x_info, v8f32x_info, f128mem,9142 WriteCvtPS2PHY, WriteCvtPS2PHYSt>,9143 EVEX, EVEX_V256, EVEX_CD8<32, CD8VH>;9144 defm VCVTPS2PHZ128 : avx512_cvtps2ph<v8i16x_info, v4f32x_info, f64mem,9145 WriteCvtPS2PH, WriteCvtPS2PHSt>,9146 EVEX, EVEX_V128, EVEX_CD8<32, CD8VH>;9147 9148 def : Pat<(store (f64 (extractelt9149 (bc_v2f64 (v8i16 (X86any_cvtps2ph VR128X:$src1, timm:$src2))),9150 (iPTR 0))), addr:$dst),9151 (VCVTPS2PHZ128mr addr:$dst, VR128X:$src1, timm:$src2)>;9152 def : Pat<(store (i64 (extractelt9153 (bc_v2i64 (v8i16 (X86any_cvtps2ph VR128X:$src1, timm:$src2))),9154 (iPTR 0))), addr:$dst),9155 (VCVTPS2PHZ128mr addr:$dst, VR128X:$src1, timm:$src2)>;9156 def : Pat<(store (v8i16 (X86any_cvtps2ph VR256X:$src1, timm:$src2)), addr:$dst),9157 (VCVTPS2PHZ256mr addr:$dst, VR256X:$src1, timm:$src2)>;9158}9159 9160// Unordered/Ordered scalar fp compare with Sae and set EFLAGS9161multiclass avx512_ord_cmp_sae<bits<8> opc, X86VectorVTInfo _,9162 string OpcodeStr, Domain d,9163 X86FoldableSchedWrite sched = WriteFComX> {9164 let ExeDomain = d, hasSideEffects = 0, Uses = [MXCSR] in9165 def rrb: AVX512<opc, MRMSrcReg, (outs), (ins _.RC:$src1, _.RC:$src2),9166 !strconcat(OpcodeStr, "\t{{sae}, $src2, $src1|$src1, $src2, {sae}}"), []>,9167 EVEX, EVEX_B, VEX_LIG, EVEX_V128, Sched<[sched]>;9168}9169 9170let Defs = [EFLAGS], Predicates = [HasAVX512] in {9171 defm VUCOMISSZ : avx512_ord_cmp_sae<0x2E, v4f32x_info, "vucomiss", SSEPackedSingle>,9172 AVX512PSIi8Base, EVEX_CD8<32, CD8VT1>;9173 defm VUCOMISDZ : avx512_ord_cmp_sae<0x2E, v2f64x_info, "vucomisd", SSEPackedDouble>,9174 AVX512PDIi8Base, REX_W, EVEX_CD8<64, CD8VT1>;9175 defm VCOMISSZ : avx512_ord_cmp_sae<0x2F, v4f32x_info, "vcomiss", SSEPackedSingle>,9176 AVX512PSIi8Base, EVEX_CD8<32, CD8VT1>;9177 defm VCOMISDZ : avx512_ord_cmp_sae<0x2F, v2f64x_info, "vcomisd", SSEPackedDouble>,9178 AVX512PDIi8Base, REX_W, EVEX_CD8<64, CD8VT1>;9179}9180 9181let Defs = [EFLAGS], Predicates = [HasAVX512] in {9182 defm VUCOMISSZ : sse12_ord_cmp<0x2E, FR32X, X86any_fcmp, f32, f32mem, loadf32,9183 "ucomiss", SSEPackedSingle>, TB, EVEX, VEX_LIG,9184 EVEX_CD8<32, CD8VT1>;9185 defm VUCOMISDZ : sse12_ord_cmp<0x2E, FR64X, X86any_fcmp, f64, f64mem, loadf64,9186 "ucomisd", SSEPackedDouble>, TB, PD, EVEX,9187 VEX_LIG, REX_W, EVEX_CD8<64, CD8VT1>;9188 defm VCOMISSZ : sse12_ord_cmp<0x2F, FR32X, X86strict_fcmps, f32, f32mem, loadf32,9189 "comiss", SSEPackedSingle>, TB, EVEX, VEX_LIG,9190 EVEX_CD8<32, CD8VT1>;9191 defm VCOMISDZ : sse12_ord_cmp<0x2F, FR64X, X86strict_fcmps, f64, f64mem, loadf64,9192 "comisd", SSEPackedDouble>, TB, PD, EVEX,9193 VEX_LIG, REX_W, EVEX_CD8<64, CD8VT1>;9194 let isCodeGenOnly = 1 in {9195 defm VUCOMISSZ : sse12_ord_cmp_int<0x2E, VR128X, X86ucomi, v4f32, ssmem,9196 sse_load_f32, "ucomiss", SSEPackedSingle>, TB, EVEX, VEX_LIG,9197 EVEX_CD8<32, CD8VT1>;9198 defm VUCOMISDZ : sse12_ord_cmp_int<0x2E, VR128X, X86ucomi, v2f64, sdmem,9199 sse_load_f64, "ucomisd", SSEPackedDouble>, TB, PD, EVEX,9200 VEX_LIG, REX_W, EVEX_CD8<64, CD8VT1>;9201 9202 defm VCOMISSZ : sse12_ord_cmp_int<0x2F, VR128X, X86comi, v4f32, ssmem,9203 sse_load_f32, "comiss", SSEPackedSingle>, TB, EVEX, VEX_LIG,9204 EVEX_CD8<32, CD8VT1>;9205 defm VCOMISDZ : sse12_ord_cmp_int<0x2F, VR128X, X86comi, v2f64, sdmem,9206 sse_load_f64, "comisd", SSEPackedDouble>, TB, PD, EVEX,9207 VEX_LIG, REX_W, EVEX_CD8<64, CD8VT1>;9208 }9209}9210 9211let Defs = [EFLAGS], Predicates = [HasFP16] in {9212 defm VUCOMISHZ : avx512_ord_cmp_sae<0x2E, v8f16x_info, "vucomish",9213 SSEPackedSingle>, AVX512PSIi8Base, T_MAP5,9214 EVEX_CD8<16, CD8VT1>;9215 defm VCOMISHZ : avx512_ord_cmp_sae<0x2F, v8f16x_info, "vcomish",9216 SSEPackedSingle>, AVX512PSIi8Base, T_MAP5,9217 EVEX_CD8<16, CD8VT1>;9218 defm VUCOMISHZ : sse12_ord_cmp<0x2E, FR16X, X86any_fcmp, f16, f16mem, loadf16,9219 "ucomish", SSEPackedSingle>, T_MAP5, EVEX,9220 VEX_LIG, EVEX_CD8<16, CD8VT1>;9221 defm VCOMISHZ : sse12_ord_cmp<0x2F, FR16X, X86strict_fcmps, f16, f16mem, loadf16,9222 "comish", SSEPackedSingle>, T_MAP5, EVEX,9223 VEX_LIG, EVEX_CD8<16, CD8VT1>;9224 let isCodeGenOnly = 1 in {9225 defm VUCOMISHZ : sse12_ord_cmp_int<0x2E, VR128X, X86ucomi, v8f16, shmem,9226 sse_load_f16, "ucomish", SSEPackedSingle>,9227 T_MAP5, EVEX, VEX_LIG, EVEX_CD8<16, CD8VT1>;9228 9229 defm VCOMISHZ : sse12_ord_cmp_int<0x2F, VR128X, X86comi, v8f16, shmem,9230 sse_load_f16, "comish", SSEPackedSingle>,9231 T_MAP5, EVEX, VEX_LIG, EVEX_CD8<16, CD8VT1>;9232 }9233}9234 9235/// avx512_fp14_s rcp14ss, rcp14sd, rsqrt14ss, rsqrt14sd, rcpsh, rsqrtsh9236multiclass avx512_fp14_s<bits<8> opc, string OpcodeStr, SDNode OpNode,9237 X86FoldableSchedWrite sched, X86VectorVTInfo _,9238 Predicate prd = HasAVX512> {9239 let Predicates = [prd], ExeDomain = _.ExeDomain in {9240 defm rr : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),9241 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,9242 "$src2, $src1", "$src1, $src2",9243 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))>,9244 EVEX, VVVV, VEX_LIG, Sched<[sched]>;9245 let mayLoad = 1 in9246 defm rm : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),9247 (ins _.RC:$src1, _.IntScalarMemOp:$src2), OpcodeStr,9248 "$src2, $src1", "$src1, $src2",9249 (OpNode (_.VT _.RC:$src1),9250 (_.ScalarIntMemFrags addr:$src2))>, EVEX, VVVV, VEX_LIG,9251 Sched<[sched.Folded, sched.ReadAfterFold]>;9252}9253}9254 9255defm VRCPSHZ : avx512_fp14_s<0x4D, "vrcpsh", X86rcp14s, SchedWriteFRcp.Scl,9256 f16x_info, HasFP16>, EVEX_CD8<16, CD8VT1>,9257 T_MAP6, PD;9258defm VRSQRTSHZ : avx512_fp14_s<0x4F, "vrsqrtsh", X86rsqrt14s,9259 SchedWriteFRsqrt.Scl, f16x_info, HasFP16>,9260 EVEX_CD8<16, CD8VT1>, T_MAP6, PD;9261let Uses = [MXCSR] in {9262defm VRCP14SSZ : avx512_fp14_s<0x4D, "vrcp14ss", X86rcp14s, SchedWriteFRcp.Scl,9263 f32x_info>, EVEX_CD8<32, CD8VT1>,9264 T8, PD;9265defm VRCP14SDZ : avx512_fp14_s<0x4D, "vrcp14sd", X86rcp14s, SchedWriteFRcp.Scl,9266 f64x_info>, REX_W, EVEX_CD8<64, CD8VT1>,9267 T8, PD;9268defm VRSQRT14SSZ : avx512_fp14_s<0x4F, "vrsqrt14ss", X86rsqrt14s,9269 SchedWriteFRsqrt.Scl, f32x_info>,9270 EVEX_CD8<32, CD8VT1>, T8, PD;9271defm VRSQRT14SDZ : avx512_fp14_s<0x4F, "vrsqrt14sd", X86rsqrt14s,9272 SchedWriteFRsqrt.Scl, f64x_info>, REX_W,9273 EVEX_CD8<64, CD8VT1>, T8, PD;9274}9275 9276/// avx512_fp14_p rcp14ps, rcp14pd, rsqrt14ps, rsqrt14pd9277multiclass avx512_fp14_p<bits<8> opc, string OpcodeStr, SDNode OpNode,9278 X86FoldableSchedWrite sched, X86VectorVTInfo _> {9279 let ExeDomain = _.ExeDomain in {9280 defm r: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),9281 (ins _.RC:$src), OpcodeStr, "$src", "$src",9282 (_.VT (OpNode _.RC:$src))>, EVEX, T8, PD,9283 Sched<[sched]>;9284 defm m: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),9285 (ins _.MemOp:$src), OpcodeStr, "$src", "$src",9286 (OpNode (_.VT9287 (bitconvert (_.LdFrag addr:$src))))>, EVEX, T8, PD,9288 Sched<[sched.Folded, sched.ReadAfterFold]>;9289 defm mb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),9290 (ins _.ScalarMemOp:$src), OpcodeStr,9291 "${src}"#_.BroadcastStr, "${src}"#_.BroadcastStr,9292 (OpNode (_.VT9293 (_.BroadcastLdFrag addr:$src)))>,9294 EVEX, T8, PD, EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;9295 }9296}9297 9298multiclass avx512_fp14_p_vl_all<bits<8> opc, string OpcodeStr, SDNode OpNode,9299 X86SchedWriteWidths sched> {9300 let Uses = [MXCSR] in {9301 defm 14PSZ : avx512_fp14_p<opc, !strconcat(OpcodeStr, "14ps"), OpNode, sched.ZMM,9302 v16f32_info>, EVEX_V512, EVEX_CD8<32, CD8VF>;9303 defm 14PDZ : avx512_fp14_p<opc, !strconcat(OpcodeStr, "14pd"), OpNode, sched.ZMM,9304 v8f64_info>, EVEX_V512, REX_W, EVEX_CD8<64, CD8VF>;9305 }9306 let Predicates = [HasFP16] in9307 defm PHZ : avx512_fp14_p<opc, !strconcat(OpcodeStr, "ph"), OpNode, sched.ZMM,9308 v32f16_info>, EVEX_V512, T_MAP6, EVEX_CD8<16, CD8VF>;9309 9310 // Define only if AVX512VL feature is present.9311 let Predicates = [HasVLX], Uses = [MXCSR] in {9312 defm 14PSZ128 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "14ps"),9313 OpNode, sched.XMM, v4f32x_info>,9314 EVEX_V128, EVEX_CD8<32, CD8VF>;9315 defm 14PSZ256 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "14ps"),9316 OpNode, sched.YMM, v8f32x_info>,9317 EVEX_V256, EVEX_CD8<32, CD8VF>;9318 defm 14PDZ128 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "14pd"),9319 OpNode, sched.XMM, v2f64x_info>,9320 EVEX_V128, REX_W, EVEX_CD8<64, CD8VF>;9321 defm 14PDZ256 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "14pd"),9322 OpNode, sched.YMM, v4f64x_info>,9323 EVEX_V256, REX_W, EVEX_CD8<64, CD8VF>;9324 }9325 let Predicates = [HasFP16, HasVLX] in {9326 defm PHZ128 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "ph"),9327 OpNode, sched.XMM, v8f16x_info>,9328 EVEX_V128, T_MAP6, EVEX_CD8<16, CD8VF>;9329 defm PHZ256 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "ph"),9330 OpNode, sched.YMM, v16f16x_info>,9331 EVEX_V256, T_MAP6, EVEX_CD8<16, CD8VF>;9332 }9333}9334 9335defm VRSQRT : avx512_fp14_p_vl_all<0x4E, "vrsqrt", X86rsqrt14, SchedWriteFRsqrt>;9336defm VRCP : avx512_fp14_p_vl_all<0x4C, "vrcp", X86rcp14, SchedWriteFRcp>;9337 9338/// avx512_fp28_s rcp28ss, rcp28sd, rsqrt28ss, rsqrt28sd9339multiclass avx512_fp28_s<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,9340 SDNode OpNode, SDNode OpNodeSAE,9341 X86FoldableSchedWrite sched> {9342 let ExeDomain = _.ExeDomain, Uses = [MXCSR] in {9343 defm r : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),9344 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,9345 "$src2, $src1", "$src1, $src2",9346 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))>,9347 Sched<[sched]>, SIMD_EXC;9348 9349 defm rb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),9350 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,9351 "{sae}, $src2, $src1", "$src1, $src2, {sae}",9352 (OpNodeSAE (_.VT _.RC:$src1), (_.VT _.RC:$src2))>,9353 EVEX_B, Sched<[sched]>;9354 9355 defm m : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),9356 (ins _.RC:$src1, _.IntScalarMemOp:$src2), OpcodeStr,9357 "$src2, $src1", "$src1, $src2",9358 (OpNode (_.VT _.RC:$src1), (_.ScalarIntMemFrags addr:$src2))>,9359 Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC;9360 }9361}9362 9363multiclass avx512_fp28_s_ass<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,9364 X86FoldableSchedWrite sched> {9365 let ExeDomain = _.ExeDomain, Uses = [MXCSR], hasSideEffects = 0 in {9366 defm r : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),9367 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,9368 "$src2, $src1", "$src1, $src2",9369 (null_frag)>, Sched<[sched]>, SIMD_EXC;9370 defm rb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),9371 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,9372 "{sae}, $src2, $src1", "$src1, $src2, {sae}",9373 (null_frag)>, EVEX_B, Sched<[sched]>;9374 let mayLoad = 1 in9375 defm m : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),9376 (ins _.RC:$src1, _.IntScalarMemOp:$src2), OpcodeStr,9377 "$src2, $src1", "$src1, $src2",9378 (null_frag)>,9379 Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC;9380 }9381}9382 9383multiclass avx512_eri_s_ass<bits<8> opc, string OpcodeStr,9384 X86FoldableSchedWrite sched> {9385 defm SSZ : avx512_fp28_s_ass<opc, OpcodeStr#"ss", f32x_info, sched>,9386 EVEX_CD8<32, CD8VT1>, VEX_LIG, T8, PD, EVEX, VVVV;9387 defm SDZ : avx512_fp28_s_ass<opc, OpcodeStr#"sd", f64x_info, sched>,9388 EVEX_CD8<64, CD8VT1>, VEX_LIG, REX_W, T8, PD, EVEX, VVVV;9389}9390 9391defm VRCP28 : avx512_eri_s_ass<0xCB, "vrcp28", SchedWriteFRcp.Scl>;9392defm VRSQRT28 : avx512_eri_s_ass<0xCD, "vrsqrt28", SchedWriteFRsqrt.Scl>;9393 9394multiclass avx512_eri_s<bits<8> opc, string OpcodeStr, SDNode OpNode,9395 SDNode OpNodeSAE, X86FoldableSchedWrite sched> {9396 defm SSZ : avx512_fp28_s<opc, OpcodeStr#"ss", f32x_info, OpNode, OpNodeSAE,9397 sched>, EVEX_CD8<32, CD8VT1>, VEX_LIG, T8, PD, EVEX, VVVV;9398 defm SDZ : avx512_fp28_s<opc, OpcodeStr#"sd", f64x_info, OpNode, OpNodeSAE,9399 sched>, EVEX_CD8<64, CD8VT1>, VEX_LIG, REX_W, T8, PD, EVEX, VVVV;9400}9401 9402multiclass avx512_vgetexpsh<bits<8> opc, string OpcodeStr, SDNode OpNode,9403 SDNode OpNodeSAE, X86FoldableSchedWrite sched> {9404 let Predicates = [HasFP16] in9405 defm SHZ : avx512_fp28_s<opc, OpcodeStr#"sh", f16x_info, OpNode, OpNodeSAE, sched>,9406 EVEX_CD8<16, CD8VT1>, T_MAP6, PD, EVEX, VVVV;9407}9408 9409defm VGETEXP : avx512_eri_s<0x43, "vgetexp", X86fgetexps, X86fgetexpSAEs,9410 SchedWriteFRnd.Scl>,9411 avx512_vgetexpsh<0x43, "vgetexp", X86fgetexps, X86fgetexpSAEs,9412 SchedWriteFRnd.Scl>;9413/// avx512_fp28_p rcp28ps, rcp28pd, rsqrt28ps, rsqrt28pd9414 9415multiclass avx512_fp28_p<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,9416 SDNode OpNode, X86FoldableSchedWrite sched> {9417 let ExeDomain = _.ExeDomain, Uses = [MXCSR], mayRaiseFPException = 1 in {9418 defm r : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),9419 (ins _.RC:$src), OpcodeStr, "$src", "$src",9420 (OpNode (_.VT _.RC:$src))>,9421 Sched<[sched]>;9422 9423 defm m : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),9424 (ins _.MemOp:$src), OpcodeStr, "$src", "$src",9425 (OpNode (_.VT9426 (bitconvert (_.LdFrag addr:$src))))>,9427 Sched<[sched.Folded, sched.ReadAfterFold]>;9428 9429 defm mb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),9430 (ins _.ScalarMemOp:$src), OpcodeStr,9431 "${src}"#_.BroadcastStr, "${src}"#_.BroadcastStr,9432 (OpNode (_.VT9433 (_.BroadcastLdFrag addr:$src)))>,9434 EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;9435 }9436}9437multiclass avx512_fp28_p_sae<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,9438 SDNode OpNode, X86FoldableSchedWrite sched> {9439 let ExeDomain = _.ExeDomain, Uses = [MXCSR] in9440 defm rb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),9441 (ins _.RC:$src), OpcodeStr,9442 "{sae}, $src", "$src, {sae}",9443 (OpNode (_.VT _.RC:$src))>,9444 EVEX_B, Sched<[sched]>;9445}9446 9447multiclass avx512_fp28_p_ass<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,9448 X86FoldableSchedWrite sched> {9449 let ExeDomain = _.ExeDomain, Uses = [MXCSR], mayRaiseFPException = 1,9450 hasSideEffects = 0 in {9451 defm r : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),9452 (ins _.RC:$src), OpcodeStr, "$src", "$src",9453 (null_frag)>, Sched<[sched]>;9454 let mayLoad = 1 in9455 defm m : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),9456 (ins _.MemOp:$src), OpcodeStr, "$src", "$src",9457 (null_frag)>,9458 Sched<[sched.Folded, sched.ReadAfterFold]>;9459 let mayLoad = 1 in9460 defm mb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),9461 (ins _.ScalarMemOp:$src), OpcodeStr,9462 "${src}"#_.BroadcastStr, "${src}"#_.BroadcastStr,9463 (null_frag)>,9464 EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;9465 }9466}9467multiclass avx512_fp28_p_sae_ass<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,9468 X86FoldableSchedWrite sched> {9469 let ExeDomain = _.ExeDomain, Uses = [MXCSR], hasSideEffects = 0 in9470 defm rb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),9471 (ins _.RC:$src), OpcodeStr,9472 "{sae}, $src", "$src, {sae}",9473 (null_frag)>, Sched<[sched]>, EVEX_B;9474}9475 9476multiclass avx512_eri_ass<bits<8> opc, string OpcodeStr,9477 X86SchedWriteWidths sched> {9478 defm PSZ : avx512_fp28_p_ass<opc, OpcodeStr#"ps", v16f32_info, sched.ZMM>,9479 avx512_fp28_p_sae_ass<opc, OpcodeStr#"ps", v16f32_info, sched.ZMM>,9480 T8, PD, EVEX_V512, EVEX_CD8<32, CD8VF>;9481 defm PDZ : avx512_fp28_p_ass<opc, OpcodeStr#"pd", v8f64_info, sched.ZMM>,9482 avx512_fp28_p_sae_ass<opc, OpcodeStr#"pd", v8f64_info, sched.ZMM>,9483 T8, PD, EVEX_V512, REX_W, EVEX_CD8<64, CD8VF>;9484}9485 9486defm VRSQRT28 : avx512_eri_ass<0xCC, "vrsqrt28", SchedWriteFRsqrt>, EVEX;9487defm VRCP28 : avx512_eri_ass<0xCA, "vrcp28", SchedWriteFRcp>, EVEX;9488defm VEXP2 : avx512_eri_ass<0xC8, "vexp2", SchedWriteFAdd>, EVEX;9489 9490multiclass avx512_eri<bits<8> opc, string OpcodeStr, SDNode OpNode,9491 SDNode OpNodeSAE, X86SchedWriteWidths sched> {9492 defm PSZ : avx512_fp28_p<opc, OpcodeStr#"ps", v16f32_info, OpNode, sched.ZMM>,9493 avx512_fp28_p_sae<opc, OpcodeStr#"ps", v16f32_info, OpNodeSAE, sched.ZMM>,9494 T8, PD, EVEX_V512, EVEX_CD8<32, CD8VF>;9495 defm PDZ : avx512_fp28_p<opc, OpcodeStr#"pd", v8f64_info, OpNode, sched.ZMM>,9496 avx512_fp28_p_sae<opc, OpcodeStr#"pd", v8f64_info, OpNodeSAE, sched.ZMM>,9497 T8, PD, EVEX_V512, REX_W, EVEX_CD8<64, CD8VF>;9498}9499 9500multiclass avx512_fp_unaryop_packed<bits<8> opc, string OpcodeStr,9501 SDNode OpNode, X86SchedWriteWidths sched> {9502 // Define only if AVX512VL feature is present.9503 let Predicates = [HasVLX] in {9504 defm PSZ128 : avx512_fp28_p<opc, OpcodeStr#"ps", v4f32x_info, OpNode,9505 sched.XMM>,9506 EVEX_V128, T8, PD, EVEX_CD8<32, CD8VF>;9507 defm PSZ256 : avx512_fp28_p<opc, OpcodeStr#"ps", v8f32x_info, OpNode,9508 sched.YMM>,9509 EVEX_V256, T8, PD, EVEX_CD8<32, CD8VF>;9510 defm PDZ128 : avx512_fp28_p<opc, OpcodeStr#"pd", v2f64x_info, OpNode,9511 sched.XMM>,9512 EVEX_V128, REX_W, T8, PD, EVEX_CD8<64, CD8VF>;9513 defm PDZ256 : avx512_fp28_p<opc, OpcodeStr#"pd", v4f64x_info, OpNode,9514 sched.YMM>,9515 EVEX_V256, REX_W, T8, PD, EVEX_CD8<64, CD8VF>;9516 }9517}9518 9519multiclass avx512_vgetexp_fp16<bits<8> opc, string OpcodeStr, SDNode OpNode,9520 SDNode OpNodeSAE, X86SchedWriteWidths sched> {9521 let Predicates = [HasFP16] in9522 defm PHZ : avx512_fp28_p<opc, OpcodeStr#"ph", v32f16_info, OpNode, sched.ZMM>,9523 avx512_fp28_p_sae<opc, OpcodeStr#"ph", v32f16_info, OpNodeSAE, sched.ZMM>,9524 T_MAP6, PD, EVEX_V512, EVEX_CD8<16, CD8VF>;9525 let Predicates = [HasFP16, HasVLX] in {9526 defm PHZ128 : avx512_fp28_p<opc, OpcodeStr#"ph", v8f16x_info, OpNode, sched.XMM>,9527 EVEX_V128, T_MAP6, PD, EVEX_CD8<16, CD8VF>;9528 defm PHZ256 : avx512_fp28_p<opc, OpcodeStr#"ph", v16f16x_info, OpNode, sched.YMM>,9529 EVEX_V256, T_MAP6, PD, EVEX_CD8<16, CD8VF>;9530 }9531}9532defm VGETEXP : avx512_eri<0x42, "vgetexp", X86fgetexp, X86fgetexpSAE,9533 SchedWriteFRnd>,9534 avx512_vgetexp_fp16<0x42, "vgetexp", X86fgetexp, X86fgetexpSAE,9535 SchedWriteFRnd>,9536 avx512_fp_unaryop_packed<0x42, "vgetexp", X86fgetexp,9537 SchedWriteFRnd>, EVEX;9538 9539multiclass avx512_sqrt_packed_round<bits<8> opc, string OpcodeStr,9540 X86FoldableSchedWrite sched, X86VectorVTInfo _>{9541 let ExeDomain = _.ExeDomain in9542 defm rb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),9543 (ins _.RC:$src, AVX512RC:$rc), OpcodeStr, "$rc, $src", "$src, $rc",9544 (_.VT (X86fsqrtRnd _.RC:$src, (i32 timm:$rc)))>,9545 EVEX, EVEX_B, EVEX_RC, Sched<[sched]>;9546}9547 9548multiclass avx512_sqrt_packed<bits<8> opc, string OpcodeStr,9549 X86FoldableSchedWrite sched, X86VectorVTInfo _>{9550 let ExeDomain = _.ExeDomain, Uses = [MXCSR], mayRaiseFPException = 1 in {9551 defm r: AVX512_maskable_split<opc, MRMSrcReg, _, (outs _.RC:$dst),9552 (ins _.RC:$src), OpcodeStr, "$src", "$src",9553 (_.VT (any_fsqrt _.RC:$src)),9554 (_.VT (fsqrt _.RC:$src))>, EVEX,9555 Sched<[sched]>;9556 defm m: AVX512_maskable_split<opc, MRMSrcMem, _, (outs _.RC:$dst),9557 (ins _.MemOp:$src), OpcodeStr, "$src", "$src",9558 (any_fsqrt (_.VT (_.LdFrag addr:$src))),9559 (fsqrt (_.VT (_.LdFrag addr:$src)))>, EVEX,9560 Sched<[sched.Folded, sched.ReadAfterFold]>;9561 defm mb: AVX512_maskable_split<opc, MRMSrcMem, _, (outs _.RC:$dst),9562 (ins _.ScalarMemOp:$src), OpcodeStr,9563 "${src}"#_.BroadcastStr, "${src}"#_.BroadcastStr,9564 (any_fsqrt (_.VT (_.BroadcastLdFrag addr:$src))),9565 (fsqrt (_.VT (_.BroadcastLdFrag addr:$src)))>,9566 EVEX, EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;9567 }9568}9569 9570let Uses = [MXCSR], mayRaiseFPException = 1 in9571multiclass avx512_sqrt_packed_all<bits<8> opc, string OpcodeStr,9572 X86SchedWriteSizes sched> {9573 let Predicates = [HasFP16] in9574 defm PHZ : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ph"),9575 sched.PH.ZMM, v32f16_info>,9576 EVEX_V512, T_MAP5, EVEX_CD8<16, CD8VF>;9577 let Predicates = [HasFP16, HasVLX] in {9578 defm PHZ128 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ph"),9579 sched.PH.XMM, v8f16x_info>,9580 EVEX_V128, T_MAP5, EVEX_CD8<16, CD8VF>;9581 defm PHZ256 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ph"),9582 sched.PH.YMM, v16f16x_info>,9583 EVEX_V256, T_MAP5, EVEX_CD8<16, CD8VF>;9584 }9585 defm PSZ : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ps"),9586 sched.PS.ZMM, v16f32_info>,9587 EVEX_V512, TB, EVEX_CD8<32, CD8VF>;9588 defm PDZ : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "pd"),9589 sched.PD.ZMM, v8f64_info>,9590 EVEX_V512, REX_W, TB, PD, EVEX_CD8<64, CD8VF>;9591 // Define only if AVX512VL feature is present.9592 let Predicates = [HasVLX] in {9593 defm PSZ128 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ps"),9594 sched.PS.XMM, v4f32x_info>,9595 EVEX_V128, TB, EVEX_CD8<32, CD8VF>;9596 defm PSZ256 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ps"),9597 sched.PS.YMM, v8f32x_info>,9598 EVEX_V256, TB, EVEX_CD8<32, CD8VF>;9599 defm PDZ128 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "pd"),9600 sched.PD.XMM, v2f64x_info>,9601 EVEX_V128, REX_W, TB, PD, EVEX_CD8<64, CD8VF>;9602 defm PDZ256 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "pd"),9603 sched.PD.YMM, v4f64x_info>,9604 EVEX_V256, REX_W, TB, PD, EVEX_CD8<64, CD8VF>;9605 }9606}9607 9608let Uses = [MXCSR] in9609multiclass avx512_sqrt_packed_all_round<bits<8> opc, string OpcodeStr,9610 X86SchedWriteSizes sched> {9611 let Predicates = [HasFP16] in9612 defm PHZ : avx512_sqrt_packed_round<opc, !strconcat(OpcodeStr, "ph"),9613 sched.PH.ZMM, v32f16_info>,9614 EVEX_V512, T_MAP5, EVEX_CD8<16, CD8VF>;9615 defm PSZ : avx512_sqrt_packed_round<opc, !strconcat(OpcodeStr, "ps"),9616 sched.PS.ZMM, v16f32_info>,9617 EVEX_V512, TB, EVEX_CD8<32, CD8VF>;9618 defm PDZ : avx512_sqrt_packed_round<opc, !strconcat(OpcodeStr, "pd"),9619 sched.PD.ZMM, v8f64_info>,9620 EVEX_V512, REX_W, TB, PD, EVEX_CD8<64, CD8VF>;9621}9622 9623multiclass avx512_sqrt_scalar<bits<8> opc, string OpcodeStr, X86FoldableSchedWrite sched,9624 X86VectorVTInfo _, string Name, Predicate prd = HasAVX512> {9625 let ExeDomain = _.ExeDomain, Predicates = [prd] in {9626 defm r : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),9627 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,9628 "$src2, $src1", "$src1, $src2",9629 (X86fsqrts (_.VT _.RC:$src1),9630 (_.VT _.RC:$src2)), "_Int">,9631 Sched<[sched]>, SIMD_EXC;9632 defm m : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),9633 (ins _.RC:$src1, _.IntScalarMemOp:$src2), OpcodeStr,9634 "$src2, $src1", "$src1, $src2",9635 (X86fsqrts (_.VT _.RC:$src1),9636 (_.ScalarIntMemFrags addr:$src2)), "_Int">,9637 Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC;9638 let Uses = [MXCSR] in9639 defm rb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),9640 (ins _.RC:$src1, _.RC:$src2, AVX512RC:$rc), OpcodeStr,9641 "$rc, $src2, $src1", "$src1, $src2, $rc",9642 (X86fsqrtRnds (_.VT _.RC:$src1),9643 (_.VT _.RC:$src2),9644 (i32 timm:$rc)), "_Int">,9645 EVEX_B, EVEX_RC, Sched<[sched]>;9646 9647 let isCodeGenOnly = 1, hasSideEffects = 0 in {9648 def r : I<opc, MRMSrcReg, (outs _.FRC:$dst),9649 (ins _.FRC:$src1, _.FRC:$src2),9650 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,9651 Sched<[sched]>, SIMD_EXC;9652 let mayLoad = 1 in9653 def m : I<opc, MRMSrcMem, (outs _.FRC:$dst),9654 (ins _.FRC:$src1, _.ScalarMemOp:$src2),9655 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,9656 Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC;9657 }9658 }9659 9660 let Predicates = [prd] in {9661 def : Pat<(_.EltVT (any_fsqrt _.FRC:$src)),9662 (!cast<Instruction>(Name#Zr)9663 (_.EltVT (IMPLICIT_DEF)), _.FRC:$src)>;9664 }9665 9666 let Predicates = [prd, OptForSize] in {9667 def : Pat<(_.EltVT (any_fsqrt (load addr:$src))),9668 (!cast<Instruction>(Name#Zm)9669 (_.EltVT (IMPLICIT_DEF)), addr:$src)>;9670 }9671}9672 9673multiclass avx512_sqrt_scalar_all<bits<8> opc, string OpcodeStr,9674 X86SchedWriteSizes sched> {9675 defm SHZ : avx512_sqrt_scalar<opc, OpcodeStr#"sh", sched.PH.Scl, f16x_info, NAME#"SH", HasFP16>,9676 EVEX_CD8<16, CD8VT1>, EVEX, VVVV, T_MAP5, XS;9677 defm SSZ : avx512_sqrt_scalar<opc, OpcodeStr#"ss", sched.PS.Scl, f32x_info, NAME#"SS">,9678 EVEX_CD8<32, CD8VT1>, EVEX, VVVV, TB, XS;9679 defm SDZ : avx512_sqrt_scalar<opc, OpcodeStr#"sd", sched.PD.Scl, f64x_info, NAME#"SD">,9680 EVEX_CD8<64, CD8VT1>, EVEX, VVVV, TB, XD, REX_W;9681}9682 9683defm VSQRT : avx512_sqrt_packed_all<0x51, "vsqrt", SchedWriteFSqrtSizes>,9684 avx512_sqrt_packed_all_round<0x51, "vsqrt", SchedWriteFSqrtSizes>;9685 9686defm VSQRT : avx512_sqrt_scalar_all<0x51, "vsqrt", SchedWriteFSqrtSizes>, VEX_LIG;9687 9688multiclass avx512_rndscale_scalar<bits<8> opc, string OpcodeStr,9689 X86FoldableSchedWrite sched, X86VectorVTInfo _> {9690 let ExeDomain = _.ExeDomain in {9691 defm rri : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),9692 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3), OpcodeStr,9693 "$src3, $src2, $src1", "$src1, $src2, $src3",9694 (_.VT (X86RndScales (_.VT _.RC:$src1), (_.VT _.RC:$src2),9695 (i32 timm:$src3))), "_Int">,9696 Sched<[sched]>, SIMD_EXC;9697 9698 let Uses = [MXCSR] in9699 defm rrib : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),9700 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3), OpcodeStr,9701 "$src3, {sae}, $src2, $src1", "$src1, $src2, {sae}, $src3",9702 (_.VT (X86RndScalesSAE (_.VT _.RC:$src1), (_.VT _.RC:$src2),9703 (i32 timm:$src3))), "_Int">, EVEX_B,9704 Sched<[sched]>;9705 9706 let mayLoad = 1 in9707 defm rmi : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),9708 (ins _.RC:$src1, _.IntScalarMemOp:$src2, i32u8imm:$src3),9709 OpcodeStr,9710 "$src3, $src2, $src1", "$src1, $src2, $src3",9711 (_.VT (X86RndScales _.RC:$src1,9712 (_.ScalarIntMemFrags addr:$src2), (i32 timm:$src3))), "_Int">,9713 Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC;9714 9715 let isCodeGenOnly = 1, hasSideEffects = 0, Predicates = [HasAVX512] in {9716 def rri : I<opc, MRMSrcReg, (outs _.FRC:$dst),9717 (ins _.FRC:$src1, _.FRC:$src2, i32u8imm:$src3),9718 OpcodeStr#"\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",9719 []>, Sched<[sched]>, SIMD_EXC;9720 9721 let mayLoad = 1 in9722 def rmi : I<opc, MRMSrcMem, (outs _.FRC:$dst),9723 (ins _.FRC:$src1, _.ScalarMemOp:$src2, i32u8imm:$src3),9724 OpcodeStr#"\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",9725 []>, Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC;9726 }9727 }9728 9729 let Predicates = [HasAVX512] in {9730 def : Pat<(X86any_VRndScale _.FRC:$src1, timm:$src2),9731 (_.EltVT (!cast<Instruction>(NAME#rri) (_.EltVT (IMPLICIT_DEF)),9732 _.FRC:$src1, timm:$src2))>;9733 }9734 9735 let Predicates = [HasAVX512, OptForSize] in {9736 def : Pat<(X86any_VRndScale (_.ScalarLdFrag addr:$src1), timm:$src2),9737 (_.EltVT (!cast<Instruction>(NAME#rmi) (_.EltVT (IMPLICIT_DEF)),9738 addr:$src1, timm:$src2))>;9739 }9740}9741 9742let Predicates = [HasFP16] in9743defm VRNDSCALESHZ : avx512_rndscale_scalar<0x0A, "vrndscalesh",9744 SchedWriteFRnd.Scl, f16x_info>,9745 AVX512PSIi8Base, TA, EVEX, VVVV,9746 EVEX_CD8<16, CD8VT1>;9747 9748defm VRNDSCALESSZ : avx512_rndscale_scalar<0x0A, "vrndscaless",9749 SchedWriteFRnd.Scl, f32x_info>,9750 AVX512AIi8Base, EVEX, VVVV, VEX_LIG,9751 EVEX_CD8<32, CD8VT1>;9752 9753defm VRNDSCALESDZ : avx512_rndscale_scalar<0x0B, "vrndscalesd",9754 SchedWriteFRnd.Scl, f64x_info>,9755 REX_W, AVX512AIi8Base, EVEX, VVVV, VEX_LIG,9756 EVEX_CD8<64, CD8VT1>;9757 9758multiclass avx512_masked_scalar<SDNode OpNode, string OpcPrefix, SDNode Move,9759 dag Mask, X86VectorVTInfo _, PatLeaf ZeroFP,9760 dag OutMask, Predicate BasePredicate> {9761 let Predicates = [BasePredicate] in {9762 def : Pat<(Move _.VT:$src1, (scalar_to_vector (X86selects_mask Mask,9763 (OpNode (extractelt _.VT:$src2, (iPTR 0))),9764 (extractelt _.VT:$dst, (iPTR 0))))),9765 (!cast<Instruction>("V"#OpcPrefix#rk_Int)9766 _.VT:$dst, OutMask, _.VT:$src2, _.VT:$src1)>;9767 9768 def : Pat<(Move _.VT:$src1, (scalar_to_vector (X86selects_mask Mask,9769 (OpNode (extractelt _.VT:$src2, (iPTR 0))),9770 ZeroFP))),9771 (!cast<Instruction>("V"#OpcPrefix#rkz_Int)9772 OutMask, _.VT:$src2, _.VT:$src1)>;9773 }9774}9775 9776defm : avx512_masked_scalar<fsqrt, "SQRTSHZ", X86Movsh,9777 (v1i1 (scalar_to_vector (i8 (trunc (i32 GR32:$mask))))), v8f16x_info,9778 fp16imm0, (COPY_TO_REGCLASS $mask, VK1WM), HasFP16>;9779defm : avx512_masked_scalar<fsqrt, "SQRTSSZ", X86Movss,9780 (v1i1 (scalar_to_vector (i8 (trunc (i32 GR32:$mask))))), v4f32x_info,9781 fp32imm0, (COPY_TO_REGCLASS $mask, VK1WM), HasAVX512>;9782defm : avx512_masked_scalar<fsqrt, "SQRTSDZ", X86Movsd,9783 (v1i1 (scalar_to_vector (i8 (trunc (i32 GR32:$mask))))), v2f64x_info,9784 fp64imm0, (COPY_TO_REGCLASS $mask, VK1WM), HasAVX512>;9785 9786 9787//-------------------------------------------------9788// Integer truncate and extend operations9789//-------------------------------------------------9790 9791multiclass avx512_trunc_common<bits<8> opc, string OpcodeStr, SDNode OpNode,9792 SDPatternOperator MaskNode,9793 X86FoldableSchedWrite sched, X86VectorVTInfo SrcInfo,9794 X86VectorVTInfo DestInfo, X86MemOperand x86memop> {9795 let ExeDomain = DestInfo.ExeDomain in {9796 def rr : AVX512XS8I<opc, MRMDestReg, (outs DestInfo.RC:$dst),9797 (ins SrcInfo.RC:$src),9798 OpcodeStr # "\t{$src, $dst|$dst, $src}",9799 [(set DestInfo.RC:$dst,9800 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src))))]>,9801 EVEX, Sched<[sched]>;9802 let Constraints = "$src0 = $dst" in9803 def rrk : AVX512XS8I<opc, MRMDestReg, (outs DestInfo.RC:$dst),9804 (ins DestInfo.RC:$src0, SrcInfo.KRCWM:$mask, SrcInfo.RC:$src),9805 OpcodeStr # "\t{$src, $dst {${mask}}|$dst {${mask}}, $src}",9806 [(set DestInfo.RC:$dst,9807 (MaskNode (SrcInfo.VT SrcInfo.RC:$src),9808 (DestInfo.VT DestInfo.RC:$src0),9809 SrcInfo.KRCWM:$mask))]>,9810 EVEX, EVEX_K, Sched<[sched]>;9811 def rrkz : AVX512XS8I<opc, MRMDestReg, (outs DestInfo.RC:$dst),9812 (ins SrcInfo.KRCWM:$mask, SrcInfo.RC:$src),9813 OpcodeStr # "\t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}",9814 [(set DestInfo.RC:$dst,9815 (DestInfo.VT (MaskNode (SrcInfo.VT SrcInfo.RC:$src),9816 DestInfo.ImmAllZerosV, SrcInfo.KRCWM:$mask)))]>,9817 EVEX, EVEX_KZ, Sched<[sched]>;9818 }9819 9820 let mayStore = 1, hasSideEffects = 0, ExeDomain = DestInfo.ExeDomain in {9821 def mr : AVX512XS8I<opc, MRMDestMem, (outs),9822 (ins x86memop:$dst, SrcInfo.RC:$src),9823 OpcodeStr # "\t{$src, $dst|$dst, $src}", []>,9824 EVEX, Sched<[sched.Folded]>;9825 9826 def mrk : AVX512XS8I<opc, MRMDestMem, (outs),9827 (ins x86memop:$dst, SrcInfo.KRCWM:$mask, SrcInfo.RC:$src),9828 OpcodeStr # "\t{$src, $dst {${mask}}|$dst {${mask}}, $src}", []>,9829 EVEX, EVEX_K, Sched<[sched.Folded]>;9830 }//mayStore = 1, hasSideEffects = 09831}9832 9833multiclass avx512_trunc_mr_lowering<X86VectorVTInfo SrcInfo,9834 PatFrag truncFrag, PatFrag mtruncFrag,9835 string Name> {9836 9837 def : Pat<(truncFrag (SrcInfo.VT SrcInfo.RC:$src), addr:$dst),9838 (!cast<Instruction>(Name#SrcInfo.ZSuffix#mr)9839 addr:$dst, SrcInfo.RC:$src)>;9840 9841 def : Pat<(mtruncFrag (SrcInfo.VT SrcInfo.RC:$src), addr:$dst,9842 SrcInfo.KRCWM:$mask),9843 (!cast<Instruction>(Name#SrcInfo.ZSuffix#mrk)9844 addr:$dst, SrcInfo.KRCWM:$mask, SrcInfo.RC:$src)>;9845}9846 9847multiclass avx512_trunc<bits<8> opc, string OpcodeStr, SDNode OpNode128,9848 SDNode OpNode256, SDNode OpNode512,9849 SDPatternOperator MaskNode128,9850 SDPatternOperator MaskNode256,9851 SDPatternOperator MaskNode512,9852 X86SchedWriteWidths sched,9853 AVX512VLVectorVTInfo VTSrcInfo,9854 X86VectorVTInfo DestInfoZ128,9855 X86VectorVTInfo DestInfoZ256, X86VectorVTInfo DestInfoZ,9856 X86MemOperand x86memopZ128, X86MemOperand x86memopZ256,9857 X86MemOperand x86memopZ, PatFrag truncFrag,9858 PatFrag mtruncFrag, Predicate prd = HasAVX512>{9859 9860 let Predicates = [HasVLX, prd] in {9861 defm Z128: avx512_trunc_common<opc, OpcodeStr, OpNode128, MaskNode128, sched.XMM,9862 VTSrcInfo.info128, DestInfoZ128, x86memopZ128>,9863 avx512_trunc_mr_lowering<VTSrcInfo.info128, truncFrag,9864 mtruncFrag, NAME>, EVEX_V128;9865 9866 defm Z256: avx512_trunc_common<opc, OpcodeStr, OpNode256, MaskNode256, sched.YMM,9867 VTSrcInfo.info256, DestInfoZ256, x86memopZ256>,9868 avx512_trunc_mr_lowering<VTSrcInfo.info256, truncFrag,9869 mtruncFrag, NAME>, EVEX_V256;9870 }9871 let Predicates = [prd] in9872 defm Z: avx512_trunc_common<opc, OpcodeStr, OpNode512, MaskNode512, sched.ZMM,9873 VTSrcInfo.info512, DestInfoZ, x86memopZ>,9874 avx512_trunc_mr_lowering<VTSrcInfo.info512, truncFrag,9875 mtruncFrag, NAME>, EVEX_V512;9876}9877 9878multiclass avx512_trunc_qb<bits<8> opc, string OpcodeStr,9879 X86SchedWriteWidths sched, PatFrag StoreNode,9880 PatFrag MaskedStoreNode, SDNode InVecNode,9881 SDPatternOperator InVecMaskNode> {9882 defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, InVecNode, InVecNode,9883 InVecMaskNode, InVecMaskNode, InVecMaskNode, sched,9884 avx512vl_i64_info, v16i8x_info, v16i8x_info,9885 v16i8x_info, i16mem, i32mem, i64mem, StoreNode,9886 MaskedStoreNode>, EVEX_CD8<8, CD8VO>;9887}9888 9889multiclass avx512_trunc_qw<bits<8> opc, string OpcodeStr, SDNode OpNode,9890 SDPatternOperator MaskNode,9891 X86SchedWriteWidths sched, PatFrag StoreNode,9892 PatFrag MaskedStoreNode, SDNode InVecNode,9893 SDPatternOperator InVecMaskNode> {9894 defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, InVecNode, OpNode,9895 InVecMaskNode, InVecMaskNode, MaskNode, sched,9896 avx512vl_i64_info, v8i16x_info, v8i16x_info,9897 v8i16x_info, i32mem, i64mem, i128mem, StoreNode,9898 MaskedStoreNode>, EVEX_CD8<16, CD8VQ>;9899}9900 9901multiclass avx512_trunc_qd<bits<8> opc, string OpcodeStr, SDNode OpNode,9902 SDPatternOperator MaskNode,9903 X86SchedWriteWidths sched, PatFrag StoreNode,9904 PatFrag MaskedStoreNode, SDNode InVecNode,9905 SDPatternOperator InVecMaskNode> {9906 defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, OpNode, OpNode,9907 InVecMaskNode, MaskNode, MaskNode, sched,9908 avx512vl_i64_info, v4i32x_info, v4i32x_info,9909 v8i32x_info, i64mem, i128mem, i256mem, StoreNode,9910 MaskedStoreNode>, EVEX_CD8<32, CD8VH>;9911}9912 9913multiclass avx512_trunc_db<bits<8> opc, string OpcodeStr, SDNode OpNode,9914 SDPatternOperator MaskNode,9915 X86SchedWriteWidths sched, PatFrag StoreNode,9916 PatFrag MaskedStoreNode, SDNode InVecNode,9917 SDPatternOperator InVecMaskNode> {9918 defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, InVecNode, OpNode,9919 InVecMaskNode, InVecMaskNode, MaskNode, sched,9920 avx512vl_i32_info, v16i8x_info, v16i8x_info,9921 v16i8x_info, i32mem, i64mem, i128mem, StoreNode,9922 MaskedStoreNode>, EVEX_CD8<8, CD8VQ>;9923}9924 9925multiclass avx512_trunc_dw<bits<8> opc, string OpcodeStr, SDNode OpNode,9926 SDPatternOperator MaskNode,9927 X86SchedWriteWidths sched, PatFrag StoreNode,9928 PatFrag MaskedStoreNode, SDNode InVecNode,9929 SDPatternOperator InVecMaskNode> {9930 defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, OpNode, OpNode,9931 InVecMaskNode, MaskNode, MaskNode, sched,9932 avx512vl_i32_info, v8i16x_info, v8i16x_info,9933 v16i16x_info, i64mem, i128mem, i256mem, StoreNode,9934 MaskedStoreNode>, EVEX_CD8<16, CD8VH>;9935}9936 9937multiclass avx512_trunc_wb<bits<8> opc, string OpcodeStr, SDNode OpNode,9938 SDPatternOperator MaskNode,9939 X86SchedWriteWidths sched, PatFrag StoreNode,9940 PatFrag MaskedStoreNode, SDNode InVecNode,9941 SDPatternOperator InVecMaskNode> {9942 defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, OpNode, OpNode,9943 InVecMaskNode, MaskNode, MaskNode, sched,9944 avx512vl_i16_info, v16i8x_info, v16i8x_info,9945 v32i8x_info, i64mem, i128mem, i256mem, StoreNode,9946 MaskedStoreNode, HasBWI>, EVEX_CD8<16, CD8VH>;9947}9948 9949defm VPMOVQB : avx512_trunc_qb<0x32, "vpmovqb",9950 SchedWriteVecTruncate, truncstorevi8,9951 masked_truncstorevi8, X86vtrunc, X86vmtrunc>;9952defm VPMOVSQB : avx512_trunc_qb<0x22, "vpmovsqb",9953 SchedWriteVecTruncate, truncstore_s_vi8,9954 masked_truncstore_s_vi8, X86vtruncs,9955 X86vmtruncs>;9956defm VPMOVUSQB : avx512_trunc_qb<0x12, "vpmovusqb",9957 SchedWriteVecTruncate, truncstore_us_vi8,9958 masked_truncstore_us_vi8, X86vtruncus, X86vmtruncus>;9959 9960defm VPMOVQW : avx512_trunc_qw<0x34, "vpmovqw", trunc, select_trunc,9961 SchedWriteVecTruncate, truncstorevi16,9962 masked_truncstorevi16, X86vtrunc, X86vmtrunc>;9963defm VPMOVSQW : avx512_trunc_qw<0x24, "vpmovsqw", X86vtruncs, select_truncs,9964 SchedWriteVecTruncate, truncstore_s_vi16,9965 masked_truncstore_s_vi16, X86vtruncs,9966 X86vmtruncs>;9967defm VPMOVUSQW : avx512_trunc_qw<0x14, "vpmovusqw", X86vtruncus,9968 select_truncus, SchedWriteVecTruncate,9969 truncstore_us_vi16, masked_truncstore_us_vi16,9970 X86vtruncus, X86vmtruncus>;9971 9972defm VPMOVQD : avx512_trunc_qd<0x35, "vpmovqd", trunc, select_trunc,9973 SchedWriteVecTruncate, truncstorevi32,9974 masked_truncstorevi32, X86vtrunc, X86vmtrunc>;9975defm VPMOVSQD : avx512_trunc_qd<0x25, "vpmovsqd", X86vtruncs, select_truncs,9976 SchedWriteVecTruncate, truncstore_s_vi32,9977 masked_truncstore_s_vi32, X86vtruncs,9978 X86vmtruncs>;9979defm VPMOVUSQD : avx512_trunc_qd<0x15, "vpmovusqd", X86vtruncus,9980 select_truncus, SchedWriteVecTruncate,9981 truncstore_us_vi32, masked_truncstore_us_vi32,9982 X86vtruncus, X86vmtruncus>;9983 9984defm VPMOVDB : avx512_trunc_db<0x31, "vpmovdb", trunc, select_trunc,9985 SchedWriteVecTruncate, truncstorevi8,9986 masked_truncstorevi8, X86vtrunc, X86vmtrunc>;9987defm VPMOVSDB : avx512_trunc_db<0x21, "vpmovsdb", X86vtruncs, select_truncs,9988 SchedWriteVecTruncate, truncstore_s_vi8,9989 masked_truncstore_s_vi8, X86vtruncs,9990 X86vmtruncs>;9991defm VPMOVUSDB : avx512_trunc_db<0x11, "vpmovusdb", X86vtruncus,9992 select_truncus, SchedWriteVecTruncate,9993 truncstore_us_vi8, masked_truncstore_us_vi8,9994 X86vtruncus, X86vmtruncus>;9995 9996defm VPMOVDW : avx512_trunc_dw<0x33, "vpmovdw", trunc, select_trunc,9997 SchedWriteVecTruncate, truncstorevi16,9998 masked_truncstorevi16, X86vtrunc, X86vmtrunc>;9999defm VPMOVSDW : avx512_trunc_dw<0x23, "vpmovsdw", X86vtruncs, select_truncs,10000 SchedWriteVecTruncate, truncstore_s_vi16,10001 masked_truncstore_s_vi16, X86vtruncs,10002 X86vmtruncs>;10003defm VPMOVUSDW : avx512_trunc_dw<0x13, "vpmovusdw", X86vtruncus,10004 select_truncus, SchedWriteVecTruncate,10005 truncstore_us_vi16, masked_truncstore_us_vi16,10006 X86vtruncus, X86vmtruncus>;10007 10008defm VPMOVWB : avx512_trunc_wb<0x30, "vpmovwb", trunc, select_trunc,10009 SchedWriteVecTruncate, truncstorevi8,10010 masked_truncstorevi8, X86vtrunc,10011 X86vmtrunc>;10012defm VPMOVSWB : avx512_trunc_wb<0x20, "vpmovswb", X86vtruncs, select_truncs,10013 SchedWriteVecTruncate, truncstore_s_vi8,10014 masked_truncstore_s_vi8, X86vtruncs,10015 X86vmtruncs>;10016defm VPMOVUSWB : avx512_trunc_wb<0x10, "vpmovuswb", X86vtruncus,10017 select_truncus, SchedWriteVecTruncate,10018 truncstore_us_vi8, masked_truncstore_us_vi8,10019 X86vtruncus, X86vmtruncus>;10020 10021let Predicates = [HasAVX512, NoVLX] in {10022def: Pat<(v8i16 (trunc (v8i32 VR256X:$src))),10023 (v8i16 (EXTRACT_SUBREG10024 (v16i16 (VPMOVDWZrr (v16i32 (INSERT_SUBREG (IMPLICIT_DEF),10025 VR256X:$src, sub_ymm)))), sub_xmm))>;10026def: Pat<(v4i32 (trunc (v4i64 VR256X:$src))),10027 (v4i32 (EXTRACT_SUBREG10028 (v8i32 (VPMOVQDZrr (v8i64 (INSERT_SUBREG (IMPLICIT_DEF),10029 VR256X:$src, sub_ymm)))), sub_xmm))>;10030}10031 10032let Predicates = [HasBWI, NoVLX] in {10033def: Pat<(v16i8 (trunc (v16i16 VR256X:$src))),10034 (v16i8 (EXTRACT_SUBREG (VPMOVWBZrr (v32i16 (INSERT_SUBREG (IMPLICIT_DEF),10035 VR256X:$src, sub_ymm))), sub_xmm))>;10036}10037 10038// Without BWI we can't use vXi16/vXi8 vselect so we have to use vmtrunc nodes.10039multiclass mtrunc_lowering<string InstrName, SDNode OpNode,10040 X86VectorVTInfo DestInfo,10041 X86VectorVTInfo SrcInfo> {10042 def : Pat<(DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src),10043 DestInfo.RC:$src0,10044 SrcInfo.KRCWM:$mask)),10045 (!cast<Instruction>(InstrName#"rrk") DestInfo.RC:$src0,10046 SrcInfo.KRCWM:$mask,10047 SrcInfo.RC:$src)>;10048 10049 def : Pat<(DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src),10050 DestInfo.ImmAllZerosV,10051 SrcInfo.KRCWM:$mask)),10052 (!cast<Instruction>(InstrName#"rrkz") SrcInfo.KRCWM:$mask,10053 SrcInfo.RC:$src)>;10054}10055 10056let Predicates = [HasVLX] in {10057defm : mtrunc_lowering<"VPMOVDWZ256", X86vmtrunc, v8i16x_info, v8i32x_info>;10058defm : mtrunc_lowering<"VPMOVSDWZ256", X86vmtruncs, v8i16x_info, v8i32x_info>;10059defm : mtrunc_lowering<"VPMOVUSDWZ256", X86vmtruncus, v8i16x_info, v8i32x_info>;10060}10061 10062let Predicates = [HasAVX512] in {10063defm : mtrunc_lowering<"VPMOVDWZ", X86vmtrunc, v16i16x_info, v16i32_info>;10064defm : mtrunc_lowering<"VPMOVSDWZ", X86vmtruncs, v16i16x_info, v16i32_info>;10065defm : mtrunc_lowering<"VPMOVUSDWZ", X86vmtruncus, v16i16x_info, v16i32_info>;10066 10067defm : mtrunc_lowering<"VPMOVDBZ", X86vmtrunc, v16i8x_info, v16i32_info>;10068defm : mtrunc_lowering<"VPMOVSDBZ", X86vmtruncs, v16i8x_info, v16i32_info>;10069defm : mtrunc_lowering<"VPMOVUSDBZ", X86vmtruncus, v16i8x_info, v16i32_info>;10070 10071defm : mtrunc_lowering<"VPMOVQWZ", X86vmtrunc, v8i16x_info, v8i64_info>;10072defm : mtrunc_lowering<"VPMOVSQWZ", X86vmtruncs, v8i16x_info, v8i64_info>;10073defm : mtrunc_lowering<"VPMOVUSQWZ", X86vmtruncus, v8i16x_info, v8i64_info>;10074}10075 10076multiclass avx512_pmovx_common<bits<8> opc, string OpcodeStr, X86FoldableSchedWrite sched,10077 X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo,10078 X86MemOperand x86memop, PatFrag LdFrag, SDNode OpNode>{10079 let ExeDomain = DestInfo.ExeDomain in {10080 defm rr : AVX512_maskable<opc, MRMSrcReg, DestInfo, (outs DestInfo.RC:$dst),10081 (ins SrcInfo.RC:$src), OpcodeStr ,"$src", "$src",10082 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src)))>,10083 EVEX, Sched<[sched]>;10084 10085 let mayLoad = 1 in10086 defm rm : AVX512_maskable<opc, MRMSrcMem, DestInfo, (outs DestInfo.RC:$dst),10087 (ins x86memop:$src), OpcodeStr ,"$src", "$src",10088 (DestInfo.VT (LdFrag addr:$src))>,10089 EVEX, Sched<[sched.Folded]>;10090 }10091}10092 10093multiclass avx512_pmovx_bw<bits<8> opc, string OpcodeStr,10094 SDNode OpNode, SDNode InVecNode, string ExtTy,10095 X86SchedWriteWidths sched,10096 PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> {10097 let Predicates = [HasVLX, HasBWI] in {10098 defm Z128: avx512_pmovx_common<opc, OpcodeStr, sched.XMM, v8i16x_info,10099 v16i8x_info, i64mem, LdFrag, InVecNode>,10100 EVEX_CD8<8, CD8VH>, T8, PD, EVEX_V128, WIG;10101 10102 defm Z256: avx512_pmovx_common<opc, OpcodeStr, sched.YMM, v16i16x_info,10103 v16i8x_info, i128mem, LdFrag, OpNode>,10104 EVEX_CD8<8, CD8VH>, T8, PD, EVEX_V256, WIG;10105 }10106 let Predicates = [HasBWI] in {10107 defm Z : avx512_pmovx_common<opc, OpcodeStr, sched.ZMM, v32i16_info,10108 v32i8x_info, i256mem, LdFrag, OpNode>,10109 EVEX_CD8<8, CD8VH>, T8, PD, EVEX_V512, WIG;10110 }10111}10112 10113multiclass avx512_pmovx_bd<bits<8> opc, string OpcodeStr,10114 SDNode OpNode, SDNode InVecNode, string ExtTy,10115 X86SchedWriteWidths sched,10116 PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> {10117 let Predicates = [HasVLX, HasAVX512] in {10118 defm Z128: avx512_pmovx_common<opc, OpcodeStr, sched.XMM, v4i32x_info,10119 v16i8x_info, i32mem, LdFrag, InVecNode>,10120 EVEX_CD8<8, CD8VQ>, T8, PD, EVEX_V128, WIG;10121 10122 defm Z256: avx512_pmovx_common<opc, OpcodeStr, sched.YMM, v8i32x_info,10123 v16i8x_info, i64mem, LdFrag, InVecNode>,10124 EVEX_CD8<8, CD8VQ>, T8, PD, EVEX_V256, WIG;10125 }10126 let Predicates = [HasAVX512] in {10127 defm Z : avx512_pmovx_common<opc, OpcodeStr, sched.ZMM, v16i32_info,10128 v16i8x_info, i128mem, LdFrag, OpNode>,10129 EVEX_CD8<8, CD8VQ>, T8, PD, EVEX_V512, WIG;10130 }10131}10132 10133multiclass avx512_pmovx_bq<bits<8> opc, string OpcodeStr,10134 SDNode InVecNode, string ExtTy,10135 X86SchedWriteWidths sched,10136 PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> {10137 let Predicates = [HasVLX, HasAVX512] in {10138 defm Z128: avx512_pmovx_common<opc, OpcodeStr, sched.XMM, v2i64x_info,10139 v16i8x_info, i16mem, LdFrag, InVecNode>,10140 EVEX_CD8<8, CD8VO>, T8, PD, EVEX_V128, WIG;10141 10142 defm Z256: avx512_pmovx_common<opc, OpcodeStr, sched.YMM, v4i64x_info,10143 v16i8x_info, i32mem, LdFrag, InVecNode>,10144 EVEX_CD8<8, CD8VO>, T8, PD, EVEX_V256, WIG;10145 }10146 let Predicates = [HasAVX512] in {10147 defm Z : avx512_pmovx_common<opc, OpcodeStr, sched.ZMM, v8i64_info,10148 v16i8x_info, i64mem, LdFrag, InVecNode>,10149 EVEX_CD8<8, CD8VO>, T8, PD, EVEX_V512, WIG;10150 }10151}10152 10153multiclass avx512_pmovx_wd<bits<8> opc, string OpcodeStr,10154 SDNode OpNode, SDNode InVecNode, string ExtTy,10155 X86SchedWriteWidths sched,10156 PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi16")> {10157 let Predicates = [HasVLX, HasAVX512] in {10158 defm Z128: avx512_pmovx_common<opc, OpcodeStr, sched.XMM, v4i32x_info,10159 v8i16x_info, i64mem, LdFrag, InVecNode>,10160 EVEX_CD8<16, CD8VH>, T8, PD, EVEX_V128, WIG;10161 10162 defm Z256: avx512_pmovx_common<opc, OpcodeStr, sched.YMM, v8i32x_info,10163 v8i16x_info, i128mem, LdFrag, OpNode>,10164 EVEX_CD8<16, CD8VH>, T8, PD, EVEX_V256, WIG;10165 }10166 let Predicates = [HasAVX512] in {10167 defm Z : avx512_pmovx_common<opc, OpcodeStr, sched.ZMM, v16i32_info,10168 v16i16x_info, i256mem, LdFrag, OpNode>,10169 EVEX_CD8<16, CD8VH>, T8, PD, EVEX_V512, WIG;10170 }10171}10172 10173multiclass avx512_pmovx_wq<bits<8> opc, string OpcodeStr,10174 SDNode OpNode, SDNode InVecNode, string ExtTy,10175 X86SchedWriteWidths sched,10176 PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi16")> {10177 let Predicates = [HasVLX, HasAVX512] in {10178 defm Z128: avx512_pmovx_common<opc, OpcodeStr, sched.XMM, v2i64x_info,10179 v8i16x_info, i32mem, LdFrag, InVecNode>,10180 EVEX_CD8<16, CD8VQ>, T8, PD, EVEX_V128, WIG;10181 10182 defm Z256: avx512_pmovx_common<opc, OpcodeStr, sched.YMM, v4i64x_info,10183 v8i16x_info, i64mem, LdFrag, InVecNode>,10184 EVEX_CD8<16, CD8VQ>, T8, PD, EVEX_V256, WIG;10185 }10186 let Predicates = [HasAVX512] in {10187 defm Z : avx512_pmovx_common<opc, OpcodeStr, sched.ZMM, v8i64_info,10188 v8i16x_info, i128mem, LdFrag, OpNode>,10189 EVEX_CD8<16, CD8VQ>, T8, PD, EVEX_V512, WIG;10190 }10191}10192 10193multiclass avx512_pmovx_dq<bits<8> opc, string OpcodeStr,10194 SDNode OpNode, SDNode InVecNode, string ExtTy,10195 X86SchedWriteWidths sched,10196 PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi32")> {10197 10198 let Predicates = [HasVLX, HasAVX512] in {10199 defm Z128: avx512_pmovx_common<opc, OpcodeStr, sched.XMM, v2i64x_info,10200 v4i32x_info, i64mem, LdFrag, InVecNode>,10201 EVEX_CD8<32, CD8VH>, T8, PD, EVEX_V128;10202 10203 defm Z256: avx512_pmovx_common<opc, OpcodeStr, sched.YMM, v4i64x_info,10204 v4i32x_info, i128mem, LdFrag, OpNode>,10205 EVEX_CD8<32, CD8VH>, T8, PD, EVEX_V256;10206 }10207 let Predicates = [HasAVX512] in {10208 defm Z : avx512_pmovx_common<opc, OpcodeStr, sched.ZMM, v8i64_info,10209 v8i32x_info, i256mem, LdFrag, OpNode>,10210 EVEX_CD8<32, CD8VH>, T8, PD, EVEX_V512;10211 }10212}10213 10214defm VPMOVZXBW : avx512_pmovx_bw<0x30, "vpmovzxbw", zext, zext_invec, "z", SchedWriteVecExtend>;10215defm VPMOVZXBD : avx512_pmovx_bd<0x31, "vpmovzxbd", zext, zext_invec, "z", SchedWriteVecExtend>;10216defm VPMOVZXBQ : avx512_pmovx_bq<0x32, "vpmovzxbq", zext_invec, "z", SchedWriteVecExtend>;10217defm VPMOVZXWD : avx512_pmovx_wd<0x33, "vpmovzxwd", zext, zext_invec, "z", SchedWriteVecExtend>;10218defm VPMOVZXWQ : avx512_pmovx_wq<0x34, "vpmovzxwq", zext, zext_invec, "z", SchedWriteVecExtend>;10219defm VPMOVZXDQ : avx512_pmovx_dq<0x35, "vpmovzxdq", zext, zext_invec, "z", SchedWriteVecExtend>;10220 10221defm VPMOVSXBW: avx512_pmovx_bw<0x20, "vpmovsxbw", sext, sext_invec, "s", SchedWriteVecExtend>;10222defm VPMOVSXBD: avx512_pmovx_bd<0x21, "vpmovsxbd", sext, sext_invec, "s", SchedWriteVecExtend>;10223defm VPMOVSXBQ: avx512_pmovx_bq<0x22, "vpmovsxbq", sext_invec, "s", SchedWriteVecExtend>;10224defm VPMOVSXWD: avx512_pmovx_wd<0x23, "vpmovsxwd", sext, sext_invec, "s", SchedWriteVecExtend>;10225defm VPMOVSXWQ: avx512_pmovx_wq<0x24, "vpmovsxwq", sext, sext_invec, "s", SchedWriteVecExtend>;10226defm VPMOVSXDQ: avx512_pmovx_dq<0x25, "vpmovsxdq", sext, sext_invec, "s", SchedWriteVecExtend>;10227 10228 10229// Patterns that we also need any extend versions of. aext_vector_inreg10230// is currently legalized to zext_vector_inreg.10231multiclass AVX512_pmovx_patterns_base<string OpcPrefix, SDNode ExtOp> {10232 // 256-bit patterns10233 let Predicates = [HasVLX, HasBWI] in {10234 def : Pat<(v16i16 (ExtOp (loadv16i8 addr:$src))),10235 (!cast<I>(OpcPrefix#BWZ256rm) addr:$src)>;10236 }10237 10238 let Predicates = [HasVLX] in {10239 def : Pat<(v8i32 (ExtOp (loadv8i16 addr:$src))),10240 (!cast<I>(OpcPrefix#WDZ256rm) addr:$src)>;10241 10242 def : Pat<(v4i64 (ExtOp (loadv4i32 addr:$src))),10243 (!cast<I>(OpcPrefix#DQZ256rm) addr:$src)>;10244 }10245 10246 // 512-bit patterns10247 let Predicates = [HasBWI] in {10248 def : Pat<(v32i16 (ExtOp (loadv32i8 addr:$src))),10249 (!cast<I>(OpcPrefix#BWZrm) addr:$src)>;10250 }10251 let Predicates = [HasAVX512] in {10252 def : Pat<(v16i32 (ExtOp (loadv16i8 addr:$src))),10253 (!cast<I>(OpcPrefix#BDZrm) addr:$src)>;10254 def : Pat<(v16i32 (ExtOp (loadv16i16 addr:$src))),10255 (!cast<I>(OpcPrefix#WDZrm) addr:$src)>;10256 10257 def : Pat<(v8i64 (ExtOp (loadv8i16 addr:$src))),10258 (!cast<I>(OpcPrefix#WQZrm) addr:$src)>;10259 10260 def : Pat<(v8i64 (ExtOp (loadv8i32 addr:$src))),10261 (!cast<I>(OpcPrefix#DQZrm) addr:$src)>;10262 }10263}10264 10265multiclass AVX512_pmovx_patterns<string OpcPrefix, SDNode ExtOp,10266 SDNode InVecOp> :10267 AVX512_pmovx_patterns_base<OpcPrefix, ExtOp> {10268 // 128-bit patterns10269 let Predicates = [HasVLX, HasBWI] in {10270 def : Pat<(v8i16 (InVecOp (bc_v16i8 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),10271 (!cast<I>(OpcPrefix#BWZ128rm) addr:$src)>;10272 def : Pat<(v8i16 (InVecOp (bc_v16i8 (v2f64 (scalar_to_vector (loadf64 addr:$src)))))),10273 (!cast<I>(OpcPrefix#BWZ128rm) addr:$src)>;10274 def : Pat<(v8i16 (InVecOp (bc_v16i8 (v2i64 (X86vzload64 addr:$src))))),10275 (!cast<I>(OpcPrefix#BWZ128rm) addr:$src)>;10276 }10277 let Predicates = [HasVLX] in {10278 def : Pat<(v4i32 (InVecOp (bc_v16i8 (v4i32 (scalar_to_vector (loadi32 addr:$src)))))),10279 (!cast<I>(OpcPrefix#BDZ128rm) addr:$src)>;10280 def : Pat<(v4i32 (InVecOp (bc_v16i8 (v4i32 (X86vzload32 addr:$src))))),10281 (!cast<I>(OpcPrefix#BDZ128rm) addr:$src)>;10282 10283 def : Pat<(v2i64 (InVecOp (bc_v16i8 (v4i32 (scalar_to_vector (extloadi32i16 addr:$src)))))),10284 (!cast<I>(OpcPrefix#BQZ128rm) addr:$src)>;10285 10286 def : Pat<(v4i32 (InVecOp (bc_v8i16 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),10287 (!cast<I>(OpcPrefix#WDZ128rm) addr:$src)>;10288 def : Pat<(v4i32 (InVecOp (bc_v8i16 (v2f64 (scalar_to_vector (loadf64 addr:$src)))))),10289 (!cast<I>(OpcPrefix#WDZ128rm) addr:$src)>;10290 def : Pat<(v4i32 (InVecOp (bc_v8i16 (v2i64 (X86vzload64 addr:$src))))),10291 (!cast<I>(OpcPrefix#WDZ128rm) addr:$src)>;10292 10293 def : Pat<(v2i64 (InVecOp (bc_v8i16 (v4i32 (scalar_to_vector (loadi32 addr:$src)))))),10294 (!cast<I>(OpcPrefix#WQZ128rm) addr:$src)>;10295 def : Pat<(v2i64 (InVecOp (bc_v8i16 (v4i32 (X86vzload32 addr:$src))))),10296 (!cast<I>(OpcPrefix#WQZ128rm) addr:$src)>;10297 10298 def : Pat<(v2i64 (InVecOp (bc_v4i32 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),10299 (!cast<I>(OpcPrefix#DQZ128rm) addr:$src)>;10300 def : Pat<(v2i64 (InVecOp (bc_v4i32 (v2f64 (scalar_to_vector (loadf64 addr:$src)))))),10301 (!cast<I>(OpcPrefix#DQZ128rm) addr:$src)>;10302 def : Pat<(v2i64 (InVecOp (bc_v4i32 (v2i64 (X86vzload64 addr:$src))))),10303 (!cast<I>(OpcPrefix#DQZ128rm) addr:$src)>;10304 }10305 let Predicates = [HasVLX] in {10306 def : Pat<(v8i32 (InVecOp (bc_v16i8 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),10307 (!cast<I>(OpcPrefix#BDZ256rm) addr:$src)>;10308 def : Pat<(v8i32 (InVecOp (bc_v16i8 (v2i64 (scalar_to_vector (loadf64 addr:$src)))))),10309 (!cast<I>(OpcPrefix#BDZ256rm) addr:$src)>;10310 def : Pat<(v8i32 (InVecOp (bc_v16i8 (v2i64 (X86vzload64 addr:$src))))),10311 (!cast<I>(OpcPrefix#BDZ256rm) addr:$src)>;10312 10313 def : Pat<(v4i64 (InVecOp (bc_v16i8 (v4i32 (scalar_to_vector (loadi32 addr:$src)))))),10314 (!cast<I>(OpcPrefix#BQZ256rm) addr:$src)>;10315 def : Pat<(v4i64 (InVecOp (bc_v16i8 (v4i32 (X86vzload32 addr:$src))))),10316 (!cast<I>(OpcPrefix#BQZ256rm) addr:$src)>;10317 10318 def : Pat<(v4i64 (InVecOp (bc_v8i16 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),10319 (!cast<I>(OpcPrefix#WQZ256rm) addr:$src)>;10320 def : Pat<(v4i64 (InVecOp (bc_v8i16 (v2i64 (scalar_to_vector (loadf64 addr:$src)))))),10321 (!cast<I>(OpcPrefix#WQZ256rm) addr:$src)>;10322 def : Pat<(v4i64 (InVecOp (bc_v8i16 (v2i64 (X86vzload64 addr:$src))))),10323 (!cast<I>(OpcPrefix#WQZ256rm) addr:$src)>;10324 }10325 // 512-bit patterns10326 let Predicates = [HasAVX512] in {10327 def : Pat<(v8i64 (InVecOp (bc_v16i8 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),10328 (!cast<I>(OpcPrefix#BQZrm) addr:$src)>;10329 def : Pat<(v8i64 (InVecOp (bc_v16i8 (v2f64 (scalar_to_vector (loadf64 addr:$src)))))),10330 (!cast<I>(OpcPrefix#BQZrm) addr:$src)>;10331 def : Pat<(v8i64 (InVecOp (bc_v16i8 (v2i64 (X86vzload64 addr:$src))))),10332 (!cast<I>(OpcPrefix#BQZrm) addr:$src)>;10333 }10334}10335 10336defm : AVX512_pmovx_patterns<"VPMOVSX", sext, sext_invec>;10337defm : AVX512_pmovx_patterns<"VPMOVZX", zext, zext_invec>;10338 10339// Without BWI we can't do a trunc from v16i16 to v16i8. DAG combine can merge10340// ext+trunc aggressively making it impossible to legalize the DAG to this10341// pattern directly.10342let Predicates = [HasAVX512, NoBWI] in {10343def: Pat<(v16i8 (trunc (v16i16 VR256X:$src))),10344 (VPMOVDBZrr (v16i32 (VPMOVZXWDZrr VR256X:$src)))>;10345def: Pat<(v16i8 (trunc (loadv16i16 addr:$src))),10346 (VPMOVDBZrr (v16i32 (VPMOVZXWDZrm addr:$src)))>;10347}10348 10349//===----------------------------------------------------------------------===//10350// GATHER - SCATTER Operations10351 10352// FIXME: Improve scheduling of gather/scatter instructions.10353multiclass avx512_gather<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,10354 X86MemOperand memop, RegisterClass MaskRC = _.KRCWM> {10355 let Constraints = "@earlyclobber $dst, $src1 = $dst, $mask = $mask_wb",10356 ExeDomain = _.ExeDomain, mayLoad = 1, hasSideEffects = 0 in10357 def rm : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst, MaskRC:$mask_wb),10358 (ins _.RC:$src1, MaskRC:$mask, memop:$src2),10359 !strconcat(OpcodeStr#_.Suffix,10360 "\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"),10361 []>, EVEX, EVEX_K, EVEX_CD8<_.EltSize, CD8VT1>,10362 Sched<[WriteLoad, WriteVecMaskedGatherWriteback]>;10363}10364 10365multiclass avx512_gather_q_pd<bits<8> dopc, bits<8> qopc,10366 AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> {10367 defm NAME#D#SUFF#Z: avx512_gather<dopc, OpcodeStr#"d", _.info512,10368 vy64xmem>, EVEX_V512, REX_W;10369 defm NAME#Q#SUFF#Z: avx512_gather<qopc, OpcodeStr#"q", _.info512,10370 vz64mem>, EVEX_V512, REX_W;10371let Predicates = [HasVLX] in {10372 defm NAME#D#SUFF#Z256: avx512_gather<dopc, OpcodeStr#"d", _.info256,10373 vx64xmem>, EVEX_V256, REX_W;10374 defm NAME#Q#SUFF#Z256: avx512_gather<qopc, OpcodeStr#"q", _.info256,10375 vy64xmem>, EVEX_V256, REX_W;10376 defm NAME#D#SUFF#Z128: avx512_gather<dopc, OpcodeStr#"d", _.info128,10377 vx64xmem>, EVEX_V128, REX_W;10378 defm NAME#Q#SUFF#Z128: avx512_gather<qopc, OpcodeStr#"q", _.info128,10379 vx64xmem>, EVEX_V128, REX_W;10380}10381}10382 10383multiclass avx512_gather_d_ps<bits<8> dopc, bits<8> qopc,10384 AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> {10385 defm NAME#D#SUFF#Z: avx512_gather<dopc, OpcodeStr#"d", _.info512, vz32mem>,10386 EVEX_V512;10387 defm NAME#Q#SUFF#Z: avx512_gather<qopc, OpcodeStr#"q", _.info256, vz32mem>,10388 EVEX_V512;10389let Predicates = [HasVLX] in {10390 defm NAME#D#SUFF#Z256: avx512_gather<dopc, OpcodeStr#"d", _.info256,10391 vy32xmem>, EVEX_V256;10392 defm NAME#Q#SUFF#Z256: avx512_gather<qopc, OpcodeStr#"q", _.info128,10393 vy32xmem>, EVEX_V256;10394 defm NAME#D#SUFF#Z128: avx512_gather<dopc, OpcodeStr#"d", _.info128,10395 vx32xmem>, EVEX_V128;10396 defm NAME#Q#SUFF#Z128: avx512_gather<qopc, OpcodeStr#"q", _.info128,10397 vx32xmem, VK2WM>, EVEX_V128;10398}10399}10400 10401 10402defm VGATHER : avx512_gather_q_pd<0x92, 0x93, avx512vl_f64_info, "vgather", "PD">,10403 avx512_gather_d_ps<0x92, 0x93, avx512vl_f32_info, "vgather", "PS">;10404 10405defm VPGATHER : avx512_gather_q_pd<0x90, 0x91, avx512vl_i64_info, "vpgather", "Q">,10406 avx512_gather_d_ps<0x90, 0x91, avx512vl_i32_info, "vpgather", "D">;10407 10408multiclass avx512_scatter<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,10409 X86MemOperand memop, RegisterClass MaskRC = _.KRCWM> {10410 10411let mayStore = 1, Constraints = "$mask = $mask_wb", ExeDomain = _.ExeDomain,10412 hasSideEffects = 0 in10413 10414 def mr : AVX5128I<opc, MRMDestMem, (outs MaskRC:$mask_wb),10415 (ins memop:$dst, MaskRC:$mask, _.RC:$src),10416 !strconcat(OpcodeStr#_.Suffix,10417 "\t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}"),10418 []>, EVEX, EVEX_K, EVEX_CD8<_.EltSize, CD8VT1>,10419 Sched<[WriteStore]>;10420}10421 10422multiclass avx512_scatter_q_pd<bits<8> dopc, bits<8> qopc,10423 AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> {10424 defm NAME#D#SUFF#Z: avx512_scatter<dopc, OpcodeStr#"d", _.info512,10425 vy64xmem>, EVEX_V512, REX_W;10426 defm NAME#Q#SUFF#Z: avx512_scatter<qopc, OpcodeStr#"q", _.info512,10427 vz64mem>, EVEX_V512, REX_W;10428let Predicates = [HasVLX] in {10429 defm NAME#D#SUFF#Z256: avx512_scatter<dopc, OpcodeStr#"d", _.info256,10430 vx64xmem>, EVEX_V256, REX_W;10431 defm NAME#Q#SUFF#Z256: avx512_scatter<qopc, OpcodeStr#"q", _.info256,10432 vy64xmem>, EVEX_V256, REX_W;10433 defm NAME#D#SUFF#Z128: avx512_scatter<dopc, OpcodeStr#"d", _.info128,10434 vx64xmem>, EVEX_V128, REX_W;10435 defm NAME#Q#SUFF#Z128: avx512_scatter<qopc, OpcodeStr#"q", _.info128,10436 vx64xmem>, EVEX_V128, REX_W;10437}10438}10439 10440multiclass avx512_scatter_d_ps<bits<8> dopc, bits<8> qopc,10441 AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> {10442 defm NAME#D#SUFF#Z: avx512_scatter<dopc, OpcodeStr#"d", _.info512, vz32mem>,10443 EVEX_V512;10444 defm NAME#Q#SUFF#Z: avx512_scatter<qopc, OpcodeStr#"q", _.info256, vz32mem>,10445 EVEX_V512;10446let Predicates = [HasVLX] in {10447 defm NAME#D#SUFF#Z256: avx512_scatter<dopc, OpcodeStr#"d", _.info256,10448 vy32xmem>, EVEX_V256;10449 defm NAME#Q#SUFF#Z256: avx512_scatter<qopc, OpcodeStr#"q", _.info128,10450 vy32xmem>, EVEX_V256;10451 defm NAME#D#SUFF#Z128: avx512_scatter<dopc, OpcodeStr#"d", _.info128,10452 vx32xmem>, EVEX_V128;10453 defm NAME#Q#SUFF#Z128: avx512_scatter<qopc, OpcodeStr#"q", _.info128,10454 vx32xmem, VK2WM>, EVEX_V128;10455}10456}10457 10458defm VSCATTER : avx512_scatter_q_pd<0xA2, 0xA3, avx512vl_f64_info, "vscatter", "PD">,10459 avx512_scatter_d_ps<0xA2, 0xA3, avx512vl_f32_info, "vscatter", "PS">;10460 10461defm VPSCATTER : avx512_scatter_q_pd<0xA0, 0xA1, avx512vl_i64_info, "vpscatter", "Q">,10462 avx512_scatter_d_ps<0xA0, 0xA1, avx512vl_i32_info, "vpscatter", "D">;10463 10464// prefetch10465multiclass avx512_gather_scatter_prefetch<bits<8> opc, Format F, string OpcodeStr,10466 RegisterClass KRC, X86MemOperand memop> {10467 let mayLoad = 1, mayStore = 1 in10468 def m : AVX5128I<opc, F, (outs), (ins KRC:$mask, memop:$src),10469 !strconcat(OpcodeStr, "\t{$src {${mask}}|{${mask}}, $src}"), []>,10470 EVEX, EVEX_K, Sched<[WriteLoad]>;10471}10472 10473defm VGATHERPF0DPS: avx512_gather_scatter_prefetch<0xC6, MRM1m, "vgatherpf0dps",10474 VK16WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;10475 10476defm VGATHERPF0QPS: avx512_gather_scatter_prefetch<0xC7, MRM1m, "vgatherpf0qps",10477 VK8WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;10478 10479defm VGATHERPF0DPD: avx512_gather_scatter_prefetch<0xC6, MRM1m, "vgatherpf0dpd",10480 VK8WM, vy64xmem>, EVEX_V512, REX_W, EVEX_CD8<64, CD8VT1>;10481 10482defm VGATHERPF0QPD: avx512_gather_scatter_prefetch<0xC7, MRM1m, "vgatherpf0qpd",10483 VK8WM, vz64mem>, EVEX_V512, REX_W, EVEX_CD8<64, CD8VT1>;10484 10485defm VGATHERPF1DPS: avx512_gather_scatter_prefetch<0xC6, MRM2m, "vgatherpf1dps",10486 VK16WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;10487 10488defm VGATHERPF1QPS: avx512_gather_scatter_prefetch<0xC7, MRM2m, "vgatherpf1qps",10489 VK8WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;10490 10491defm VGATHERPF1DPD: avx512_gather_scatter_prefetch<0xC6, MRM2m, "vgatherpf1dpd",10492 VK8WM, vy64xmem>, EVEX_V512, REX_W, EVEX_CD8<64, CD8VT1>;10493 10494defm VGATHERPF1QPD: avx512_gather_scatter_prefetch<0xC7, MRM2m, "vgatherpf1qpd",10495 VK8WM, vz64mem>, EVEX_V512, REX_W, EVEX_CD8<64, CD8VT1>;10496 10497defm VSCATTERPF0DPS: avx512_gather_scatter_prefetch<0xC6, MRM5m, "vscatterpf0dps",10498 VK16WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;10499 10500defm VSCATTERPF0QPS: avx512_gather_scatter_prefetch<0xC7, MRM5m, "vscatterpf0qps",10501 VK8WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;10502 10503defm VSCATTERPF0DPD: avx512_gather_scatter_prefetch<0xC6, MRM5m, "vscatterpf0dpd",10504 VK8WM, vy64xmem>, EVEX_V512, REX_W, EVEX_CD8<64, CD8VT1>;10505 10506defm VSCATTERPF0QPD: avx512_gather_scatter_prefetch<0xC7, MRM5m, "vscatterpf0qpd",10507 VK8WM, vz64mem>, EVEX_V512, REX_W, EVEX_CD8<64, CD8VT1>;10508 10509defm VSCATTERPF1DPS: avx512_gather_scatter_prefetch<0xC6, MRM6m, "vscatterpf1dps",10510 VK16WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;10511 10512defm VSCATTERPF1QPS: avx512_gather_scatter_prefetch<0xC7, MRM6m, "vscatterpf1qps",10513 VK8WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;10514 10515defm VSCATTERPF1DPD: avx512_gather_scatter_prefetch<0xC6, MRM6m, "vscatterpf1dpd",10516 VK8WM, vy64xmem>, EVEX_V512, REX_W, EVEX_CD8<64, CD8VT1>;10517 10518defm VSCATTERPF1QPD: avx512_gather_scatter_prefetch<0xC7, MRM6m, "vscatterpf1qpd",10519 VK8WM, vz64mem>, EVEX_V512, REX_W, EVEX_CD8<64, CD8VT1>;10520 10521multiclass cvt_by_vec_width<bits<8> opc, X86VectorVTInfo Vec, string OpcodeStr, SchedWrite Sched> {10522def rk : AVX512XS8I<opc, MRMSrcReg, (outs Vec.RC:$dst), (ins Vec.KRC:$src),10523 !strconcat(OpcodeStr#Vec.Suffix, "\t{$src, $dst|$dst, $src}"),10524 [(set Vec.RC:$dst, (Vec.VT (sext Vec.KRC:$src)))]>,10525 EVEX, Sched<[Sched]>;10526}10527 10528multiclass cvt_mask_by_elt_width<bits<8> opc, AVX512VLVectorVTInfo VTInfo,10529 string OpcodeStr, Predicate prd> {10530let Predicates = [prd] in10531 defm Z : cvt_by_vec_width<opc, VTInfo.info512, OpcodeStr, WriteVecMoveZ>, EVEX_V512;10532 10533 let Predicates = [prd, HasVLX] in {10534 defm Z256 : cvt_by_vec_width<opc, VTInfo.info256, OpcodeStr, WriteVecMoveY>, EVEX_V256;10535 defm Z128 : cvt_by_vec_width<opc, VTInfo.info128, OpcodeStr, WriteVecMoveX>, EVEX_V128;10536 }10537}10538 10539defm VPMOVM2B : cvt_mask_by_elt_width<0x28, avx512vl_i8_info, "vpmovm2" , HasBWI>;10540defm VPMOVM2W : cvt_mask_by_elt_width<0x28, avx512vl_i16_info, "vpmovm2", HasBWI> , REX_W;10541defm VPMOVM2D : cvt_mask_by_elt_width<0x38, avx512vl_i32_info, "vpmovm2", HasDQI>;10542defm VPMOVM2Q : cvt_mask_by_elt_width<0x38, avx512vl_i64_info, "vpmovm2", HasDQI> , REX_W;10543 10544multiclass convert_vector_to_mask_common<bits<8> opc, X86VectorVTInfo _, string OpcodeStr > {10545 def kr : AVX512XS8I<opc, MRMSrcReg, (outs _.KRC:$dst), (ins _.RC:$src),10546 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),10547 [(set _.KRC:$dst, (X86pcmpgtm _.ImmAllZerosV, (_.VT _.RC:$src)))]>,10548 EVEX, Sched<[WriteMove]>;10549}10550 10551// Use 512bit version to implement 128/256 bit in case NoVLX.10552multiclass convert_vector_to_mask_lowering<X86VectorVTInfo ExtendInfo,10553 X86VectorVTInfo _,10554 string Name> {10555 10556 def : Pat<(_.KVT (X86pcmpgtm _.ImmAllZerosV, (_.VT _.RC:$src))),10557 (_.KVT (COPY_TO_REGCLASS10558 (!cast<Instruction>(Name#"Zkr")10559 (INSERT_SUBREG (ExtendInfo.VT (IMPLICIT_DEF)),10560 _.RC:$src, _.SubRegIdx)),10561 _.KRC))>;10562}10563 10564multiclass avx512_convert_vector_to_mask<bits<8> opc, string OpcodeStr,10565 AVX512VLVectorVTInfo VTInfo, Predicate prd> {10566 let Predicates = [prd] in10567 defm Z : convert_vector_to_mask_common <opc, VTInfo.info512, OpcodeStr>,10568 EVEX_V512;10569 10570 let Predicates = [prd, HasVLX] in {10571 defm Z256 : convert_vector_to_mask_common<opc, VTInfo.info256, OpcodeStr>,10572 EVEX_V256;10573 defm Z128 : convert_vector_to_mask_common<opc, VTInfo.info128, OpcodeStr>,10574 EVEX_V128;10575 }10576 let Predicates = [prd, NoVLX] in {10577 defm Z256_Alt : convert_vector_to_mask_lowering<VTInfo.info512, VTInfo.info256, NAME>;10578 defm Z128_Alt : convert_vector_to_mask_lowering<VTInfo.info512, VTInfo.info128, NAME>;10579 }10580}10581 10582defm VPMOVB2M : avx512_convert_vector_to_mask<0x29, "vpmovb2m",10583 avx512vl_i8_info, HasBWI>;10584defm VPMOVW2M : avx512_convert_vector_to_mask<0x29, "vpmovw2m",10585 avx512vl_i16_info, HasBWI>, REX_W;10586defm VPMOVD2M : avx512_convert_vector_to_mask<0x39, "vpmovd2m",10587 avx512vl_i32_info, HasDQI>;10588defm VPMOVQ2M : avx512_convert_vector_to_mask<0x39, "vpmovq2m",10589 avx512vl_i64_info, HasDQI>, REX_W;10590 10591// Patterns for handling sext from a mask register to v16i8/v16i16 when DQI10592// is available, but BWI is not. We can't handle this in lowering because10593// a target independent DAG combine likes to combine sext and trunc.10594let Predicates = [HasDQI, NoBWI] in {10595 def : Pat<(v16i8 (sext (v16i1 VK16:$src))),10596 (VPMOVDBZrr (v16i32 (VPMOVM2DZrk VK16:$src)))>;10597 def : Pat<(v16i16 (sext (v16i1 VK16:$src))),10598 (VPMOVDWZrr (v16i32 (VPMOVM2DZrk VK16:$src)))>;10599}10600 10601let Predicates = [HasDQI, NoBWI, HasVLX] in {10602 def : Pat<(v8i16 (sext (v8i1 VK8:$src))),10603 (VPMOVDWZ256rr (v8i32 (VPMOVM2DZ256rk VK8:$src)))>;10604}10605 10606//===----------------------------------------------------------------------===//10607// AVX-512 - COMPRESS and EXPAND10608//10609 10610multiclass compress_by_vec_width_common<bits<8> opc, X86VectorVTInfo _,10611 string OpcodeStr, X86FoldableSchedWrite sched> {10612 defm rr : AVX512_maskable<opc, MRMDestReg, _, (outs _.RC:$dst),10613 (ins _.RC:$src1), OpcodeStr, "$src1", "$src1",10614 (null_frag)>, AVX5128IBase,10615 Sched<[sched]>;10616 10617 let mayStore = 1, hasSideEffects = 0 in10618 def mr : AVX5128I<opc, MRMDestMem, (outs),10619 (ins _.MemOp:$dst, _.RC:$src),10620 OpcodeStr # "\t{$src, $dst|$dst, $src}",10621 []>, EVEX_CD8<_.EltSize, CD8VT1>,10622 Sched<[sched.Folded]>;10623 10624 def mrk : AVX5128I<opc, MRMDestMem, (outs),10625 (ins _.MemOp:$dst, _.KRCWM:$mask, _.RC:$src),10626 OpcodeStr # "\t{$src, $dst {${mask}}|$dst {${mask}}, $src}",10627 []>,10628 EVEX_K, EVEX_CD8<_.EltSize, CD8VT1>,10629 Sched<[sched.Folded]>;10630}10631 10632multiclass compress_by_vec_width_lowering<X86VectorVTInfo _, string Name> {10633 def : Pat<(X86mCompressingStore (_.VT _.RC:$src), addr:$dst, _.KRCWM:$mask),10634 (!cast<Instruction>(Name#_.ZSuffix#mrk)10635 addr:$dst, _.KRCWM:$mask, _.RC:$src)>;10636 10637 def : Pat<(X86compress (_.VT _.RC:$src), _.RC:$src0, _.KRCWM:$mask),10638 (!cast<Instruction>(Name#_.ZSuffix#rrk)10639 _.RC:$src0, _.KRCWM:$mask, _.RC:$src)>;10640 def : Pat<(X86compress (_.VT _.RC:$src), _.ImmAllZerosV, _.KRCWM:$mask),10641 (!cast<Instruction>(Name#_.ZSuffix#rrkz)10642 _.KRCWM:$mask, _.RC:$src)>;10643 def : Pat<(_.VT (vector_compress _.RC:$src, _.KRCWM:$mask, undef)),10644 (!cast<Instruction>(Name#_.ZSuffix#rrkz)10645 _.KRCWM:$mask, _.RC:$src)>;10646 def : Pat<(_.VT (vector_compress _.RC:$src, _.KRCWM:$mask, _.ImmAllZerosV)),10647 (!cast<Instruction>(Name#_.ZSuffix#rrkz)10648 _.KRCWM:$mask, _.RC:$src)>;10649 def : Pat<(_.VT (vector_compress _.RC:$src, _.KRCWM:$mask, _.RC:$passthru)),10650 (!cast<Instruction>(Name#_.ZSuffix#rrk)10651 _.RC:$passthru, _.KRCWM:$mask, _.RC:$src)>;10652}10653 10654multiclass compress_by_elt_width<bits<8> opc, string OpcodeStr,10655 X86FoldableSchedWrite sched,10656 AVX512VLVectorVTInfo VTInfo,10657 Predicate Pred = HasAVX512> {10658 let Predicates = [Pred] in10659 defm Z : compress_by_vec_width_common<opc, VTInfo.info512, OpcodeStr, sched>,10660 compress_by_vec_width_lowering<VTInfo.info512, NAME>, EVEX_V512;10661 10662 let Predicates = [Pred, HasVLX] in {10663 defm Z256 : compress_by_vec_width_common<opc, VTInfo.info256, OpcodeStr, sched>,10664 compress_by_vec_width_lowering<VTInfo.info256, NAME>, EVEX_V256;10665 defm Z128 : compress_by_vec_width_common<opc, VTInfo.info128, OpcodeStr, sched>,10666 compress_by_vec_width_lowering<VTInfo.info128, NAME>, EVEX_V128;10667 }10668}10669 10670// FIXME: Is there a better scheduler class for VPCOMPRESS?10671defm VPCOMPRESSD : compress_by_elt_width <0x8B, "vpcompressd", WriteVarShuffle256,10672 avx512vl_i32_info>, EVEX;10673defm VPCOMPRESSQ : compress_by_elt_width <0x8B, "vpcompressq", WriteVarShuffle256,10674 avx512vl_i64_info>, EVEX, REX_W;10675defm VCOMPRESSPS : compress_by_elt_width <0x8A, "vcompressps", WriteVarShuffle256,10676 avx512vl_f32_info>, EVEX;10677defm VCOMPRESSPD : compress_by_elt_width <0x8A, "vcompresspd", WriteVarShuffle256,10678 avx512vl_f64_info>, EVEX, REX_W;10679 10680// expand10681multiclass expand_by_vec_width<bits<8> opc, X86VectorVTInfo _,10682 string OpcodeStr, X86FoldableSchedWrite sched> {10683 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),10684 (ins _.RC:$src1), OpcodeStr, "$src1", "$src1",10685 (null_frag)>, AVX5128IBase,10686 Sched<[sched]>;10687 10688 let mayLoad = 1 in10689 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),10690 (ins _.MemOp:$src1), OpcodeStr, "$src1", "$src1",10691 (null_frag)>,10692 AVX5128IBase, EVEX_CD8<_.EltSize, CD8VT1>,10693 Sched<[sched.Folded, sched.ReadAfterFold]>;10694}10695 10696multiclass expand_by_vec_width_lowering<X86VectorVTInfo _, string Name> {10697 10698 def : Pat<(_.VT (X86mExpandingLoad addr:$src, _.KRCWM:$mask, undef)),10699 (!cast<Instruction>(Name#_.ZSuffix#rmkz)10700 _.KRCWM:$mask, addr:$src)>;10701 10702 def : Pat<(_.VT (X86mExpandingLoad addr:$src, _.KRCWM:$mask, _.ImmAllZerosV)),10703 (!cast<Instruction>(Name#_.ZSuffix#rmkz)10704 _.KRCWM:$mask, addr:$src)>;10705 10706 def : Pat<(_.VT (X86mExpandingLoad addr:$src, _.KRCWM:$mask,10707 (_.VT _.RC:$src0))),10708 (!cast<Instruction>(Name#_.ZSuffix#rmk)10709 _.RC:$src0, _.KRCWM:$mask, addr:$src)>;10710 10711 def : Pat<(X86expand (_.VT _.RC:$src), _.RC:$src0, _.KRCWM:$mask),10712 (!cast<Instruction>(Name#_.ZSuffix#rrk)10713 _.RC:$src0, _.KRCWM:$mask, _.RC:$src)>;10714 def : Pat<(X86expand (_.VT _.RC:$src), _.ImmAllZerosV, _.KRCWM:$mask),10715 (!cast<Instruction>(Name#_.ZSuffix#rrkz)10716 _.KRCWM:$mask, _.RC:$src)>;10717}10718 10719multiclass expand_by_elt_width<bits<8> opc, string OpcodeStr,10720 X86FoldableSchedWrite sched,10721 AVX512VLVectorVTInfo VTInfo,10722 Predicate Pred = HasAVX512> {10723 let Predicates = [Pred] in10724 defm Z : expand_by_vec_width<opc, VTInfo.info512, OpcodeStr, sched>,10725 expand_by_vec_width_lowering<VTInfo.info512, NAME>, EVEX_V512;10726 10727 let Predicates = [Pred, HasVLX] in {10728 defm Z256 : expand_by_vec_width<opc, VTInfo.info256, OpcodeStr, sched>,10729 expand_by_vec_width_lowering<VTInfo.info256, NAME>, EVEX_V256;10730 defm Z128 : expand_by_vec_width<opc, VTInfo.info128, OpcodeStr, sched>,10731 expand_by_vec_width_lowering<VTInfo.info128, NAME>, EVEX_V128;10732 }10733}10734 10735// FIXME: Is there a better scheduler class for VPEXPAND?10736defm VPEXPANDD : expand_by_elt_width <0x89, "vpexpandd", WriteVarShuffle256,10737 avx512vl_i32_info>, EVEX;10738defm VPEXPANDQ : expand_by_elt_width <0x89, "vpexpandq", WriteVarShuffle256,10739 avx512vl_i64_info>, EVEX, REX_W;10740defm VEXPANDPS : expand_by_elt_width <0x88, "vexpandps", WriteVarShuffle256,10741 avx512vl_f32_info>, EVEX;10742defm VEXPANDPD : expand_by_elt_width <0x88, "vexpandpd", WriteVarShuffle256,10743 avx512vl_f64_info>, EVEX, REX_W;10744 10745//handle instruction reg_vec1 = op(reg_vec,imm)10746// op(mem_vec,imm)10747// op(broadcast(eltVt),imm)10748//all instruction created with FROUND_CURRENT10749multiclass avx512_unary_fp_packed_imm<bits<8> opc, string OpcodeStr,10750 SDPatternOperator OpNode,10751 SDPatternOperator MaskOpNode,10752 X86FoldableSchedWrite sched,10753 X86VectorVTInfo _> {10754 let ExeDomain = _.ExeDomain, Uses = [MXCSR], mayRaiseFPException = 1 in {10755 defm rri : AVX512_maskable_split<opc, MRMSrcReg, _, (outs _.RC:$dst),10756 (ins _.RC:$src1, i32u8imm:$src2),10757 OpcodeStr#_.Suffix, "$src2, $src1", "$src1, $src2",10758 (OpNode (_.VT _.RC:$src1), (i32 timm:$src2)),10759 (MaskOpNode (_.VT _.RC:$src1), (i32 timm:$src2))>,10760 Sched<[sched]>;10761 let mayLoad = 1 in {10762 defm rmi : AVX512_maskable_split<opc, MRMSrcMem, _, (outs _.RC:$dst),10763 (ins _.MemOp:$src1, i32u8imm:$src2),10764 OpcodeStr#_.Suffix, "$src2, $src1", "$src1, $src2",10765 (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1))),10766 (i32 timm:$src2)),10767 (MaskOpNode (_.VT (bitconvert (_.LdFrag addr:$src1))),10768 (i32 timm:$src2))>,10769 Sched<[sched.Folded, sched.ReadAfterFold]>;10770 defm rmbi : AVX512_maskable_split<opc, MRMSrcMem, _, (outs _.RC:$dst),10771 (ins _.ScalarMemOp:$src1, i32u8imm:$src2),10772 OpcodeStr#_.Suffix, "$src2, ${src1}"#_.BroadcastStr,10773 "${src1}"#_.BroadcastStr#", $src2",10774 (OpNode (_.VT (_.BroadcastLdFrag addr:$src1)),10775 (i32 timm:$src2)),10776 (MaskOpNode (_.VT (_.BroadcastLdFrag addr:$src1)),10777 (i32 timm:$src2))>, EVEX_B,10778 Sched<[sched.Folded, sched.ReadAfterFold]>;10779 }10780 }10781}10782 10783//handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm),{sae}10784multiclass avx512_unary_fp_sae_packed_imm<bits<8> opc, string OpcodeStr,10785 SDNode OpNode, X86FoldableSchedWrite sched,10786 X86VectorVTInfo _> {10787 let ExeDomain = _.ExeDomain, Uses = [MXCSR] in10788 defm rrib : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),10789 (ins _.RC:$src1, i32u8imm:$src2),10790 OpcodeStr#_.Suffix, "$src2, {sae}, $src1",10791 "$src1, {sae}, $src2",10792 (OpNode (_.VT _.RC:$src1),10793 (i32 timm:$src2))>,10794 EVEX_B, Sched<[sched]>;10795}10796 10797multiclass avx512_common_unary_fp_sae_packed_imm<string OpcodeStr,10798 AVX512VLVectorVTInfo _, bits<8> opc, SDPatternOperator OpNode,10799 SDPatternOperator MaskOpNode, SDNode OpNodeSAE, X86SchedWriteWidths sched,10800 Predicate prd>{10801 let Predicates = [prd] in {10802 defm Z : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, MaskOpNode,10803 sched.ZMM, _.info512>,10804 avx512_unary_fp_sae_packed_imm<opc, OpcodeStr, OpNodeSAE,10805 sched.ZMM, _.info512>, EVEX_V512;10806 }10807 let Predicates = [prd, HasVLX] in {10808 defm Z128 : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, MaskOpNode,10809 sched.XMM, _.info128>, EVEX_V128;10810 defm Z256 : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, MaskOpNode,10811 sched.YMM, _.info256>, EVEX_V256;10812 }10813}10814 10815//handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm)10816// op(reg_vec2,mem_vec,imm)10817// op(reg_vec2,broadcast(eltVt),imm)10818//all instruction created with FROUND_CURRENT10819multiclass avx512_fp_packed_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,10820 X86FoldableSchedWrite sched, X86VectorVTInfo _>{10821 let ExeDomain = _.ExeDomain, Uses = [MXCSR], mayRaiseFPException = 1 in {10822 defm rri : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),10823 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3),10824 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",10825 (OpNode (_.VT _.RC:$src1),10826 (_.VT _.RC:$src2),10827 (i32 timm:$src3))>,10828 Sched<[sched]>;10829 let mayLoad = 1 in {10830 defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),10831 (ins _.RC:$src1, _.MemOp:$src2, i32u8imm:$src3),10832 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",10833 (OpNode (_.VT _.RC:$src1),10834 (_.VT (bitconvert (_.LdFrag addr:$src2))),10835 (i32 timm:$src3))>,10836 Sched<[sched.Folded, sched.ReadAfterFold]>;10837 defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),10838 (ins _.RC:$src1, _.ScalarMemOp:$src2, i32u8imm:$src3),10839 OpcodeStr, "$src3, ${src2}"#_.BroadcastStr#", $src1",10840 "$src1, ${src2}"#_.BroadcastStr#", $src3",10841 (OpNode (_.VT _.RC:$src1),10842 (_.VT (_.BroadcastLdFrag addr:$src2)),10843 (i32 timm:$src3))>, EVEX_B,10844 Sched<[sched.Folded, sched.ReadAfterFold]>;10845 }10846 }10847}10848 10849//handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm)10850// op(reg_vec2,mem_vec,imm)10851multiclass avx512_3Op_rm_imm8<bits<8> opc, string OpcodeStr, SDNode OpNode,10852 X86FoldableSchedWrite sched, X86VectorVTInfo DestInfo,10853 X86VectorVTInfo SrcInfo>{10854 let ExeDomain = DestInfo.ExeDomain, ImmT = Imm8 in {10855 defm rri : AVX512_maskable<opc, MRMSrcReg, DestInfo, (outs DestInfo.RC:$dst),10856 (ins SrcInfo.RC:$src1, SrcInfo.RC:$src2, u8imm:$src3),10857 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",10858 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1),10859 (SrcInfo.VT SrcInfo.RC:$src2),10860 (i8 timm:$src3)))>,10861 Sched<[sched]>;10862 let mayLoad = 1 in10863 defm rmi : AVX512_maskable<opc, MRMSrcMem, DestInfo, (outs DestInfo.RC:$dst),10864 (ins SrcInfo.RC:$src1, SrcInfo.MemOp:$src2, u8imm:$src3),10865 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",10866 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1),10867 (SrcInfo.VT (bitconvert10868 (SrcInfo.LdFrag addr:$src2))),10869 (i8 timm:$src3)))>,10870 Sched<[sched.Folded, sched.ReadAfterFold]>;10871 }10872}10873 10874//handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm)10875// op(reg_vec2,mem_vec,imm)10876// op(reg_vec2,broadcast(eltVt),imm)10877multiclass avx512_3Op_imm8<bits<8> opc, string OpcodeStr, SDNode OpNode,10878 X86FoldableSchedWrite sched, X86VectorVTInfo _>:10879 avx512_3Op_rm_imm8<opc, OpcodeStr, OpNode, sched, _, _>{10880 10881 let ExeDomain = _.ExeDomain, ImmT = Imm8, mayLoad = 1 in10882 defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),10883 (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$src3),10884 OpcodeStr, "$src3, ${src2}"#_.BroadcastStr#", $src1",10885 "$src1, ${src2}"#_.BroadcastStr#", $src3",10886 (OpNode (_.VT _.RC:$src1),10887 (_.VT (_.BroadcastLdFrag addr:$src2)),10888 (i8 timm:$src3))>, EVEX_B,10889 Sched<[sched.Folded, sched.ReadAfterFold]>;10890}10891 10892//handle scalar instruction reg_vec1 = op(reg_vec2,reg_vec3,imm)10893// op(reg_vec2,mem_scalar,imm)10894multiclass avx512_fp_scalar_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,10895 X86FoldableSchedWrite sched, X86VectorVTInfo _> {10896 let ExeDomain = _.ExeDomain, Uses = [MXCSR], mayRaiseFPException = 1 in {10897 defm rri : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),10898 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3),10899 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",10900 (OpNode (_.VT _.RC:$src1),10901 (_.VT _.RC:$src2),10902 (i32 timm:$src3))>,10903 Sched<[sched]>;10904 let mayLoad = 1 in10905 defm rmi : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),10906 (ins _.RC:$src1, _.IntScalarMemOp:$src2, i32u8imm:$src3),10907 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",10908 (OpNode (_.VT _.RC:$src1),10909 (_.ScalarIntMemFrags addr:$src2),10910 (i32 timm:$src3))>,10911 Sched<[sched.Folded, sched.ReadAfterFold]>;10912 }10913}10914 10915//handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm),{sae}10916multiclass avx512_fp_sae_packed_imm<bits<8> opc, string OpcodeStr,10917 SDNode OpNode, X86FoldableSchedWrite sched,10918 X86VectorVTInfo _> {10919 let ExeDomain = _.ExeDomain, Uses = [MXCSR] in10920 defm rrib : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),10921 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3),10922 OpcodeStr, "$src3, {sae}, $src2, $src1",10923 "$src1, $src2, {sae}, $src3",10924 (OpNode (_.VT _.RC:$src1),10925 (_.VT _.RC:$src2),10926 (i32 timm:$src3))>,10927 EVEX_B, Sched<[sched]>;10928}10929 10930//handle scalar instruction reg_vec1 = op(reg_vec2,reg_vec3,imm),{sae}10931multiclass avx512_fp_sae_scalar_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,10932 X86FoldableSchedWrite sched, X86VectorVTInfo _> {10933 let ExeDomain = _.ExeDomain, Uses = [MXCSR] in10934 defm NAME#rrib : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),10935 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3),10936 OpcodeStr, "$src3, {sae}, $src2, $src1",10937 "$src1, $src2, {sae}, $src3",10938 (OpNode (_.VT _.RC:$src1),10939 (_.VT _.RC:$src2),10940 (i32 timm:$src3))>,10941 EVEX_B, Sched<[sched]>;10942}10943 10944multiclass avx512_common_fp_sae_packed_imm<string OpcodeStr,10945 AVX512VLVectorVTInfo _, bits<8> opc, SDNode OpNode,10946 SDNode OpNodeSAE, X86SchedWriteWidths sched, Predicate prd>{10947 let Predicates = [prd] in {10948 defm Z : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, sched.ZMM, _.info512>,10949 avx512_fp_sae_packed_imm<opc, OpcodeStr, OpNodeSAE, sched.ZMM, _.info512>,10950 EVEX_V512;10951 10952 }10953 let Predicates = [prd, HasVLX] in {10954 defm Z128 : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, sched.XMM, _.info128>,10955 EVEX_V128;10956 defm Z256 : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, sched.YMM, _.info256>,10957 EVEX_V256;10958 }10959}10960 10961multiclass avx512_common_3Op_rm_imm8<bits<8> opc, SDNode OpNode, string OpStr,10962 X86SchedWriteWidths sched, AVX512VLVectorVTInfo DestInfo,10963 AVX512VLVectorVTInfo SrcInfo, Predicate Pred = HasBWI> {10964 let Predicates = [Pred] in {10965 defm Z : avx512_3Op_rm_imm8<opc, OpStr, OpNode, sched.ZMM, DestInfo.info512,10966 SrcInfo.info512>, EVEX_V512, AVX512AIi8Base, EVEX, VVVV;10967 }10968 let Predicates = [Pred, HasVLX] in {10969 defm Z128 : avx512_3Op_rm_imm8<opc, OpStr, OpNode, sched.XMM, DestInfo.info128,10970 SrcInfo.info128>, EVEX_V128, AVX512AIi8Base, EVEX, VVVV;10971 defm Z256 : avx512_3Op_rm_imm8<opc, OpStr, OpNode, sched.YMM, DestInfo.info256,10972 SrcInfo.info256>, EVEX_V256, AVX512AIi8Base, EVEX, VVVV;10973 }10974}10975 10976multiclass avx512_common_3Op_imm8<string OpcodeStr, AVX512VLVectorVTInfo _,10977 bits<8> opc, SDNode OpNode, X86SchedWriteWidths sched,10978 Predicate Pred = HasAVX512> {10979 let Predicates = [Pred] in {10980 defm Z : avx512_3Op_imm8<opc, OpcodeStr, OpNode, sched.ZMM, _.info512>,10981 EVEX_V512;10982 }10983 let Predicates = [Pred, HasVLX] in {10984 defm Z128 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, sched.XMM, _.info128>,10985 EVEX_V128;10986 defm Z256 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, sched.YMM, _.info256>,10987 EVEX_V256;10988 }10989}10990 10991multiclass avx512_common_fp_sae_scalar_imm<string OpcodeStr,10992 X86VectorVTInfo _, bits<8> opc, SDNode OpNode,10993 SDNode OpNodeSAE, X86SchedWriteWidths sched, Predicate prd> {10994 let Predicates = [prd] in {10995 defm Z : avx512_fp_scalar_imm<opc, OpcodeStr, OpNode, sched.XMM, _>,10996 avx512_fp_sae_scalar_imm<opc, OpcodeStr, OpNodeSAE, sched.XMM, _>;10997 }10998}10999 11000multiclass avx512_common_unary_fp_sae_packed_imm_all<string OpcodeStr,11001 bits<8> opcPs, bits<8> opcPd, SDPatternOperator OpNode,11002 SDPatternOperator MaskOpNode, SDNode OpNodeSAE,11003 X86SchedWriteWidths sched, Predicate prd>{11004 defm PH : avx512_common_unary_fp_sae_packed_imm<OpcodeStr, avx512vl_f16_info,11005 opcPs, OpNode, MaskOpNode, OpNodeSAE, sched, HasFP16>,11006 AVX512PSIi8Base, TA, EVEX, EVEX_CD8<16, CD8VF>;11007 defm PS : avx512_common_unary_fp_sae_packed_imm<OpcodeStr, avx512vl_f32_info,11008 opcPs, OpNode, MaskOpNode, OpNodeSAE, sched, prd>,11009 AVX512AIi8Base, EVEX, EVEX_CD8<32, CD8VF>;11010 defm PD : avx512_common_unary_fp_sae_packed_imm<OpcodeStr, avx512vl_f64_info,11011 opcPd, OpNode, MaskOpNode, OpNodeSAE, sched, prd>,11012 AVX512AIi8Base, EVEX, EVEX_CD8<64, CD8VF>, REX_W;11013}11014 11015defm VREDUCE : avx512_common_unary_fp_sae_packed_imm_all<"vreduce", 0x56, 0x56,11016 X86VReduce, X86VReduce, X86VReduceSAE,11017 SchedWriteFRnd, HasDQI>;11018defm VRNDSCALE : avx512_common_unary_fp_sae_packed_imm_all<"vrndscale", 0x08, 0x09,11019 X86any_VRndScale, X86VRndScale, X86VRndScaleSAE,11020 SchedWriteFRnd, HasAVX512>;11021defm VGETMANT : avx512_common_unary_fp_sae_packed_imm_all<"vgetmant", 0x26, 0x26,11022 X86VGetMant, X86VGetMant, X86VGetMantSAE,11023 SchedWriteFRnd, HasAVX512>;11024 11025defm VRANGEPD : avx512_common_fp_sae_packed_imm<"vrangepd", avx512vl_f64_info,11026 0x50, X86VRange, X86VRangeSAE,11027 SchedWriteFAdd, HasDQI>,11028 AVX512AIi8Base, EVEX, VVVV, EVEX_CD8<64, CD8VF>, REX_W;11029defm VRANGEPS : avx512_common_fp_sae_packed_imm<"vrangeps", avx512vl_f32_info,11030 0x50, X86VRange, X86VRangeSAE,11031 SchedWriteFAdd, HasDQI>,11032 AVX512AIi8Base, EVEX, VVVV, EVEX_CD8<32, CD8VF>;11033 11034defm VRANGESD: avx512_common_fp_sae_scalar_imm<"vrangesd",11035 f64x_info, 0x51, X86Ranges, X86RangesSAE, SchedWriteFAdd, HasDQI>,11036 AVX512AIi8Base, VEX_LIG, EVEX, VVVV, EVEX_CD8<64, CD8VT1>, REX_W;11037defm VRANGESS: avx512_common_fp_sae_scalar_imm<"vrangess", f32x_info,11038 0x51, X86Ranges, X86RangesSAE, SchedWriteFAdd, HasDQI>,11039 AVX512AIi8Base, VEX_LIG, EVEX, VVVV, EVEX_CD8<32, CD8VT1>;11040 11041defm VREDUCESD: avx512_common_fp_sae_scalar_imm<"vreducesd", f64x_info,11042 0x57, X86Reduces, X86ReducesSAE, SchedWriteFRnd, HasDQI>,11043 AVX512AIi8Base, VEX_LIG, EVEX, VVVV, EVEX_CD8<64, CD8VT1>, REX_W;11044defm VREDUCESS: avx512_common_fp_sae_scalar_imm<"vreducess", f32x_info,11045 0x57, X86Reduces, X86ReducesSAE, SchedWriteFRnd, HasDQI>,11046 AVX512AIi8Base, VEX_LIG, EVEX, VVVV, EVEX_CD8<32, CD8VT1>;11047defm VREDUCESH: avx512_common_fp_sae_scalar_imm<"vreducesh", f16x_info,11048 0x57, X86Reduces, X86ReducesSAE, SchedWriteFRnd, HasFP16>,11049 AVX512PSIi8Base, TA, VEX_LIG, EVEX, VVVV, EVEX_CD8<16, CD8VT1>;11050 11051defm VGETMANTSD: avx512_common_fp_sae_scalar_imm<"vgetmantsd", f64x_info,11052 0x27, X86GetMants, X86GetMantsSAE, SchedWriteFRnd, HasAVX512>,11053 AVX512AIi8Base, VEX_LIG, EVEX, VVVV, EVEX_CD8<64, CD8VT1>, REX_W;11054defm VGETMANTSS: avx512_common_fp_sae_scalar_imm<"vgetmantss", f32x_info,11055 0x27, X86GetMants, X86GetMantsSAE, SchedWriteFRnd, HasAVX512>,11056 AVX512AIi8Base, VEX_LIG, EVEX, VVVV, EVEX_CD8<32, CD8VT1>;11057defm VGETMANTSH: avx512_common_fp_sae_scalar_imm<"vgetmantsh", f16x_info,11058 0x27, X86GetMants, X86GetMantsSAE, SchedWriteFRnd, HasFP16>,11059 AVX512PSIi8Base, TA, VEX_LIG, EVEX, VVVV, EVEX_CD8<16, CD8VT1>;11060 11061multiclass avx512_shuff_packed_128_common<bits<8> opc, string OpcodeStr,11062 X86FoldableSchedWrite sched,11063 X86VectorVTInfo _,11064 X86VectorVTInfo CastInfo> {11065 let ExeDomain = _.ExeDomain in {11066 defm rri : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),11067 (ins _.RC:$src1, _.RC:$src2, u8imm:$src3),11068 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",11069 (_.VT (bitconvert11070 (CastInfo.VT (X86Shuf128 _.RC:$src1, _.RC:$src2,11071 (i8 timm:$src3)))))>,11072 Sched<[sched]>;11073 let mayLoad = 1 in {11074 defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),11075 (ins _.RC:$src1, _.MemOp:$src2, u8imm:$src3),11076 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",11077 (_.VT11078 (bitconvert11079 (CastInfo.VT (X86Shuf128 _.RC:$src1,11080 (CastInfo.LdFrag addr:$src2),11081 (i8 timm:$src3)))))>,11082 Sched<[sched.Folded, sched.ReadAfterFold]>;11083 defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),11084 (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$src3),11085 OpcodeStr, "$src3, ${src2}"#_.BroadcastStr#", $src1",11086 "$src1, ${src2}"#_.BroadcastStr#", $src3",11087 (_.VT11088 (bitconvert11089 (CastInfo.VT11090 (X86Shuf128 _.RC:$src1,11091 (_.BroadcastLdFrag addr:$src2),11092 (i8 timm:$src3)))))>, EVEX_B,11093 Sched<[sched.Folded, sched.ReadAfterFold]>;11094 }11095 }11096}11097 11098multiclass avx512_shuff_packed_128<string OpcodeStr, X86FoldableSchedWrite sched,11099 AVX512VLVectorVTInfo _,11100 AVX512VLVectorVTInfo CastInfo, bits<8> opc>{11101 let Predicates = [HasAVX512] in11102 defm Z : avx512_shuff_packed_128_common<opc, OpcodeStr, sched,11103 _.info512, CastInfo.info512>, EVEX_V512;11104 11105 let Predicates = [HasAVX512, HasVLX] in11106 defm Z256 : avx512_shuff_packed_128_common<opc, OpcodeStr, sched,11107 _.info256, CastInfo.info256>, EVEX_V256;11108}11109 11110defm VSHUFF32X4 : avx512_shuff_packed_128<"vshuff32x4", WriteFShuffle256,11111 avx512vl_f32_info, avx512vl_f64_info, 0x23>, AVX512AIi8Base, EVEX, VVVV, EVEX_CD8<32, CD8VF>;11112defm VSHUFF64X2 : avx512_shuff_packed_128<"vshuff64x2", WriteFShuffle256,11113 avx512vl_f64_info, avx512vl_f64_info, 0x23>, AVX512AIi8Base, EVEX, VVVV, EVEX_CD8<64, CD8VF>, REX_W;11114defm VSHUFI32X4 : avx512_shuff_packed_128<"vshufi32x4", WriteFShuffle256,11115 avx512vl_i32_info, avx512vl_i64_info, 0x43>, AVX512AIi8Base, EVEX, VVVV, EVEX_CD8<32, CD8VF>;11116defm VSHUFI64X2 : avx512_shuff_packed_128<"vshufi64x2", WriteFShuffle256,11117 avx512vl_i64_info, avx512vl_i64_info, 0x43>, AVX512AIi8Base, EVEX, VVVV, EVEX_CD8<64, CD8VF>, REX_W;11118 11119multiclass avx512_valign<bits<8> opc, string OpcodeStr,11120 X86FoldableSchedWrite sched, X86VectorVTInfo _>{11121 let ExeDomain = _.ExeDomain in {11122 defm rri : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),11123 (ins _.RC:$src1, _.RC:$src2, u8imm:$src3),11124 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",11125 (_.VT (X86VAlign _.RC:$src1, _.RC:$src2, (i8 timm:$src3)))>,11126 Sched<[sched]>;11127 let mayLoad = 1 in {11128 defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),11129 (ins _.RC:$src1, _.MemOp:$src2, u8imm:$src3),11130 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",11131 (_.VT (X86VAlign _.RC:$src1,11132 (bitconvert (_.LdFrag addr:$src2)),11133 (i8 timm:$src3)))>,11134 Sched<[sched.Folded, sched.ReadAfterFold]>;11135 11136 defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),11137 (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$src3),11138 OpcodeStr, "$src3, ${src2}"#_.BroadcastStr#", $src1",11139 "$src1, ${src2}"#_.BroadcastStr#", $src3",11140 (X86VAlign _.RC:$src1,11141 (_.VT (_.BroadcastLdFrag addr:$src2)),11142 (i8 timm:$src3))>, EVEX_B,11143 Sched<[sched.Folded, sched.ReadAfterFold]>;11144 }11145 }11146}11147 11148multiclass avx512_valign_common<string OpcodeStr, X86SchedWriteWidths sched,11149 AVX512VLVectorVTInfo _> {11150 let Predicates = [HasAVX512] in {11151 defm Z : avx512_valign<0x03, OpcodeStr, sched.ZMM, _.info512>,11152 AVX512AIi8Base, EVEX, VVVV, EVEX_V512;11153 }11154 let Predicates = [HasAVX512, HasVLX] in {11155 defm Z128 : avx512_valign<0x03, OpcodeStr, sched.XMM, _.info128>,11156 AVX512AIi8Base, EVEX, VVVV, EVEX_V128;11157 // We can't really override the 256-bit version so change it back to unset.11158 defm Z256 : avx512_valign<0x03, OpcodeStr, sched.YMM, _.info256>,11159 AVX512AIi8Base, EVEX, VVVV, EVEX_V256;11160 }11161}11162 11163defm VALIGND: avx512_valign_common<"valignd", SchedWriteShuffle,11164 avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;11165defm VALIGNQ: avx512_valign_common<"valignq", SchedWriteShuffle,11166 avx512vl_i64_info>, EVEX_CD8<64, CD8VF>,11167 REX_W;11168 11169defm VPALIGNR: avx512_common_3Op_rm_imm8<0x0F, X86PAlignr, "vpalignr",11170 SchedWriteShuffle, avx512vl_i8_info,11171 avx512vl_i8_info>, EVEX_CD8<8, CD8VF>;11172 11173// Fragments to help convert valignq into masked valignd. Or valignq/valignd11174// into vpalignr.11175def ValignqImm32XForm : SDNodeXForm<timm, [{11176 return getI8Imm(N->getZExtValue() * 2, SDLoc(N));11177}]>;11178def ValignqImm8XForm : SDNodeXForm<timm, [{11179 return getI8Imm(N->getZExtValue() * 8, SDLoc(N));11180}]>;11181def ValigndImm8XForm : SDNodeXForm<timm, [{11182 return getI8Imm(N->getZExtValue() * 4, SDLoc(N));11183}]>;11184 11185multiclass avx512_vpalign_mask_lowering<string OpcodeStr, SDNode OpNode,11186 X86VectorVTInfo From, X86VectorVTInfo To,11187 SDNodeXForm ImmXForm> {11188 def : Pat<(To.VT (vselect_mask To.KRCWM:$mask,11189 (bitconvert11190 (From.VT (OpNode From.RC:$src1, From.RC:$src2,11191 timm:$src3))),11192 To.RC:$src0)),11193 (!cast<Instruction>(OpcodeStr#"rrik") To.RC:$src0, To.KRCWM:$mask,11194 To.RC:$src1, To.RC:$src2,11195 (ImmXForm timm:$src3))>;11196 11197 def : Pat<(To.VT (vselect_mask To.KRCWM:$mask,11198 (bitconvert11199 (From.VT (OpNode From.RC:$src1, From.RC:$src2,11200 timm:$src3))),11201 To.ImmAllZerosV)),11202 (!cast<Instruction>(OpcodeStr#"rrikz") To.KRCWM:$mask,11203 To.RC:$src1, To.RC:$src2,11204 (ImmXForm timm:$src3))>;11205 11206 def : Pat<(To.VT (vselect_mask To.KRCWM:$mask,11207 (bitconvert11208 (From.VT (OpNode From.RC:$src1,11209 (From.LdFrag addr:$src2),11210 timm:$src3))),11211 To.RC:$src0)),11212 (!cast<Instruction>(OpcodeStr#"rmik") To.RC:$src0, To.KRCWM:$mask,11213 To.RC:$src1, addr:$src2,11214 (ImmXForm timm:$src3))>;11215 11216 def : Pat<(To.VT (vselect_mask To.KRCWM:$mask,11217 (bitconvert11218 (From.VT (OpNode From.RC:$src1,11219 (From.LdFrag addr:$src2),11220 timm:$src3))),11221 To.ImmAllZerosV)),11222 (!cast<Instruction>(OpcodeStr#"rmikz") To.KRCWM:$mask,11223 To.RC:$src1, addr:$src2,11224 (ImmXForm timm:$src3))>;11225}11226 11227multiclass avx512_vpalign_mask_lowering_mb<string OpcodeStr, SDNode OpNode,11228 X86VectorVTInfo From,11229 X86VectorVTInfo To,11230 SDNodeXForm ImmXForm> :11231 avx512_vpalign_mask_lowering<OpcodeStr, OpNode, From, To, ImmXForm> {11232 def : Pat<(From.VT (OpNode From.RC:$src1,11233 (bitconvert (To.VT (To.BroadcastLdFrag addr:$src2))),11234 timm:$src3)),11235 (!cast<Instruction>(OpcodeStr#"rmbi") To.RC:$src1, addr:$src2,11236 (ImmXForm timm:$src3))>;11237 11238 def : Pat<(To.VT (vselect_mask To.KRCWM:$mask,11239 (bitconvert11240 (From.VT (OpNode From.RC:$src1,11241 (bitconvert11242 (To.VT (To.BroadcastLdFrag addr:$src2))),11243 timm:$src3))),11244 To.RC:$src0)),11245 (!cast<Instruction>(OpcodeStr#"rmbik") To.RC:$src0, To.KRCWM:$mask,11246 To.RC:$src1, addr:$src2,11247 (ImmXForm timm:$src3))>;11248 11249 def : Pat<(To.VT (vselect_mask To.KRCWM:$mask,11250 (bitconvert11251 (From.VT (OpNode From.RC:$src1,11252 (bitconvert11253 (To.VT (To.BroadcastLdFrag addr:$src2))),11254 timm:$src3))),11255 To.ImmAllZerosV)),11256 (!cast<Instruction>(OpcodeStr#"rmbikz") To.KRCWM:$mask,11257 To.RC:$src1, addr:$src2,11258 (ImmXForm timm:$src3))>;11259}11260 11261let Predicates = [HasAVX512] in {11262 // For 512-bit we lower to the widest element type we can. So we only need11263 // to handle converting valignq to valignd.11264 defm : avx512_vpalign_mask_lowering_mb<"VALIGNDZ", X86VAlign, v8i64_info,11265 v16i32_info, ValignqImm32XForm>;11266}11267 11268let Predicates = [HasVLX] in {11269 // For 128-bit we lower to the widest element type we can. So we only need11270 // to handle converting valignq to valignd.11271 defm : avx512_vpalign_mask_lowering_mb<"VALIGNDZ128", X86VAlign, v2i64x_info,11272 v4i32x_info, ValignqImm32XForm>;11273 // For 256-bit we lower to the widest element type we can. So we only need11274 // to handle converting valignq to valignd.11275 defm : avx512_vpalign_mask_lowering_mb<"VALIGNDZ256", X86VAlign, v4i64x_info,11276 v8i32x_info, ValignqImm32XForm>;11277}11278 11279let Predicates = [HasVLX, HasBWI] in {11280 // We can turn 128 and 256 bit VALIGND/VALIGNQ into VPALIGNR.11281 defm : avx512_vpalign_mask_lowering<"VPALIGNRZ128", X86VAlign, v2i64x_info,11282 v16i8x_info, ValignqImm8XForm>;11283 defm : avx512_vpalign_mask_lowering<"VPALIGNRZ128", X86VAlign, v4i32x_info,11284 v16i8x_info, ValigndImm8XForm>;11285}11286 11287defm VDBPSADBW: avx512_common_3Op_rm_imm8<0x42, X86dbpsadbw, "vdbpsadbw",11288 SchedWritePSADBW, avx512vl_i16_info, avx512vl_i8_info>,11289 EVEX_CD8<8, CD8VF>;11290 11291multiclass avx512_unary_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,11292 X86FoldableSchedWrite sched, X86VectorVTInfo _> {11293 let ExeDomain = _.ExeDomain in {11294 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),11295 (ins _.RC:$src1), OpcodeStr,11296 "$src1", "$src1",11297 (_.VT (OpNode (_.VT _.RC:$src1)))>, EVEX, AVX5128IBase,11298 Sched<[sched]>;11299 11300 let mayLoad = 1 in11301 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),11302 (ins _.MemOp:$src1), OpcodeStr,11303 "$src1", "$src1",11304 (_.VT (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1)))))>,11305 EVEX, AVX5128IBase, EVEX_CD8<_.EltSize, CD8VF>,11306 Sched<[sched.Folded]>;11307 }11308}11309 11310multiclass avx512_unary_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,11311 X86FoldableSchedWrite sched, X86VectorVTInfo _> :11312 avx512_unary_rm<opc, OpcodeStr, OpNode, sched, _> {11313 let mayLoad = 1 in11314 defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),11315 (ins _.ScalarMemOp:$src1), OpcodeStr,11316 "${src1}"#_.BroadcastStr,11317 "${src1}"#_.BroadcastStr,11318 (_.VT (OpNode (_.VT (_.BroadcastLdFrag addr:$src1))))>,11319 EVEX, AVX5128IBase, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>,11320 Sched<[sched.Folded]>;11321}11322 11323multiclass avx512_unary_rm_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,11324 X86SchedWriteWidths sched,11325 AVX512VLVectorVTInfo VTInfo, Predicate prd> {11326 let Predicates = [prd] in11327 defm Z : avx512_unary_rm<opc, OpcodeStr, OpNode, sched.ZMM, VTInfo.info512>,11328 EVEX_V512;11329 11330 let Predicates = [prd, HasVLX] in {11331 defm Z256 : avx512_unary_rm<opc, OpcodeStr, OpNode, sched.YMM, VTInfo.info256>,11332 EVEX_V256;11333 defm Z128 : avx512_unary_rm<opc, OpcodeStr, OpNode, sched.XMM, VTInfo.info128>,11334 EVEX_V128;11335 }11336}11337 11338multiclass avx512_unary_rmb_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,11339 X86SchedWriteWidths sched, AVX512VLVectorVTInfo VTInfo,11340 Predicate prd> {11341 let Predicates = [prd] in11342 defm Z : avx512_unary_rmb<opc, OpcodeStr, OpNode, sched.ZMM, VTInfo.info512>,11343 EVEX_V512;11344 11345 let Predicates = [prd, HasVLX] in {11346 defm Z256 : avx512_unary_rmb<opc, OpcodeStr, OpNode, sched.YMM, VTInfo.info256>,11347 EVEX_V256;11348 defm Z128 : avx512_unary_rmb<opc, OpcodeStr, OpNode, sched.XMM, VTInfo.info128>,11349 EVEX_V128;11350 }11351}11352 11353multiclass avx512_unary_rm_vl_dq<bits<8> opc_d, bits<8> opc_q, string OpcodeStr,11354 SDNode OpNode, X86SchedWriteWidths sched,11355 Predicate prd> {11356 defm Q : avx512_unary_rmb_vl<opc_q, OpcodeStr#"q", OpNode, sched,11357 avx512vl_i64_info, prd>, REX_W;11358 defm D : avx512_unary_rmb_vl<opc_d, OpcodeStr#"d", OpNode, sched,11359 avx512vl_i32_info, prd>;11360}11361 11362multiclass avx512_unary_rm_vl_bw<bits<8> opc_b, bits<8> opc_w, string OpcodeStr,11363 SDNode OpNode, X86SchedWriteWidths sched,11364 Predicate prd> {11365 defm W : avx512_unary_rm_vl<opc_w, OpcodeStr#"w", OpNode, sched,11366 avx512vl_i16_info, prd>, WIG;11367 defm B : avx512_unary_rm_vl<opc_b, OpcodeStr#"b", OpNode, sched,11368 avx512vl_i8_info, prd>, WIG;11369}11370 11371multiclass avx512_unary_rm_vl_all<bits<8> opc_b, bits<8> opc_w,11372 bits<8> opc_d, bits<8> opc_q,11373 string OpcodeStr, SDNode OpNode,11374 X86SchedWriteWidths sched> {11375 defm NAME : avx512_unary_rm_vl_dq<opc_d, opc_q, OpcodeStr, OpNode, sched,11376 HasAVX512>,11377 avx512_unary_rm_vl_bw<opc_b, opc_w, OpcodeStr, OpNode, sched,11378 HasBWI>;11379}11380 11381defm VPABS : avx512_unary_rm_vl_all<0x1C, 0x1D, 0x1E, 0x1F, "vpabs", abs,11382 SchedWriteVecALU>;11383 11384// VPABS: Use 512bit version to implement 128/256 bit in case NoVLX.11385let Predicates = [HasAVX512, NoVLX] in {11386 def : Pat<(v4i64 (abs VR256X:$src)),11387 (EXTRACT_SUBREG11388 (VPABSQZrr11389 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)),11390 sub_ymm)>;11391 def : Pat<(v2i64 (abs VR128X:$src)),11392 (EXTRACT_SUBREG11393 (VPABSQZrr11394 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm)),11395 sub_xmm)>;11396}11397 11398// Use 512bit version to implement 128/256 bit.11399multiclass avx512_unary_lowering<string InstrStr, SDNode OpNode,11400 AVX512VLVectorVTInfo _, Predicate prd> {11401 let Predicates = [prd, NoVLX] in {11402 def : Pat<(_.info256.VT (OpNode (_.info256.VT _.info256.RC:$src1))),11403 (EXTRACT_SUBREG11404 (!cast<Instruction>(InstrStr # "Zrr")11405 (INSERT_SUBREG(_.info512.VT(IMPLICIT_DEF)),11406 _.info256.RC:$src1,11407 _.info256.SubRegIdx)),11408 _.info256.SubRegIdx)>;11409 11410 def : Pat<(_.info128.VT (OpNode (_.info128.VT _.info128.RC:$src1))),11411 (EXTRACT_SUBREG11412 (!cast<Instruction>(InstrStr # "Zrr")11413 (INSERT_SUBREG(_.info512.VT(IMPLICIT_DEF)),11414 _.info128.RC:$src1,11415 _.info128.SubRegIdx)),11416 _.info128.SubRegIdx)>;11417 }11418}11419 11420defm VPLZCNT : avx512_unary_rm_vl_dq<0x44, 0x44, "vplzcnt", ctlz,11421 SchedWriteVecIMul, HasCDI>;11422 11423// FIXME: Is there a better scheduler class for VPCONFLICT?11424defm VPCONFLICT : avx512_unary_rm_vl_dq<0xC4, 0xC4, "vpconflict", X86Conflict,11425 SchedWriteVecALU, HasCDI>;11426 11427// VPLZCNT: Use 512bit version to implement 128/256 bit in case NoVLX.11428defm : avx512_unary_lowering<"VPLZCNTQ", ctlz, avx512vl_i64_info, HasCDI>;11429defm : avx512_unary_lowering<"VPLZCNTD", ctlz, avx512vl_i32_info, HasCDI>;11430 11431//===---------------------------------------------------------------------===//11432// Counts number of ones - VPOPCNTD and VPOPCNTQ11433//===---------------------------------------------------------------------===//11434 11435// FIXME: Is there a better scheduler class for VPOPCNTD/VPOPCNTQ?11436defm VPOPCNT : avx512_unary_rm_vl_dq<0x55, 0x55, "vpopcnt", ctpop,11437 SchedWriteVecALU, HasVPOPCNTDQ>;11438 11439defm : avx512_unary_lowering<"VPOPCNTQ", ctpop, avx512vl_i64_info, HasVPOPCNTDQ>;11440defm : avx512_unary_lowering<"VPOPCNTD", ctpop, avx512vl_i32_info, HasVPOPCNTDQ>;11441 11442//===---------------------------------------------------------------------===//11443// Replicate Single FP - MOVSHDUP and MOVSLDUP11444//===---------------------------------------------------------------------===//11445 11446multiclass avx512_replicate<bits<8> opc, string OpcodeStr, SDNode OpNode,11447 X86SchedWriteWidths sched> {11448 defm NAME: avx512_unary_rm_vl<opc, OpcodeStr, OpNode, sched,11449 avx512vl_f32_info, HasAVX512>, TB, XS;11450}11451 11452defm VMOVSHDUP : avx512_replicate<0x16, "vmovshdup", X86Movshdup,11453 SchedWriteFShuffle>;11454defm VMOVSLDUP : avx512_replicate<0x12, "vmovsldup", X86Movsldup,11455 SchedWriteFShuffle>;11456 11457//===----------------------------------------------------------------------===//11458// AVX-512 - MOVDDUP11459//===----------------------------------------------------------------------===//11460 11461multiclass avx512_movddup_128<bits<8> opc, string OpcodeStr,11462 X86FoldableSchedWrite sched, X86VectorVTInfo _> {11463 let ExeDomain = _.ExeDomain in {11464 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),11465 (ins _.RC:$src), OpcodeStr, "$src", "$src",11466 (_.VT (X86VBroadcast (_.VT _.RC:$src)))>, EVEX,11467 Sched<[sched]>;11468 let mayLoad = 1 in11469 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),11470 (ins _.ScalarMemOp:$src), OpcodeStr, "$src", "$src",11471 (_.VT (_.BroadcastLdFrag addr:$src))>,11472 EVEX, EVEX_CD8<_.EltSize, CD8VH>,11473 Sched<[sched.Folded]>;11474 }11475}11476 11477multiclass avx512_movddup_common<bits<8> opc, string OpcodeStr,11478 X86SchedWriteWidths sched, AVX512VLVectorVTInfo VTInfo> {11479 defm Z : avx512_unary_rm<opc, OpcodeStr, X86Movddup, sched.ZMM,11480 VTInfo.info512>, EVEX_V512;11481 11482 let Predicates = [HasAVX512, HasVLX] in {11483 defm Z256 : avx512_unary_rm<opc, OpcodeStr, X86Movddup, sched.YMM,11484 VTInfo.info256>, EVEX_V256;11485 defm Z128 : avx512_movddup_128<opc, OpcodeStr, sched.XMM,11486 VTInfo.info128>, EVEX_V128;11487 }11488}11489 11490multiclass avx512_movddup<bits<8> opc, string OpcodeStr,11491 X86SchedWriteWidths sched> {11492 defm NAME: avx512_movddup_common<opc, OpcodeStr, sched,11493 avx512vl_f64_info>, TB, XD, REX_W;11494}11495 11496defm VMOVDDUP : avx512_movddup<0x12, "vmovddup", SchedWriteFShuffle>;11497 11498let Predicates = [HasVLX] in {11499def : Pat<(v2f64 (X86VBroadcast f64:$src)),11500 (VMOVDDUPZ128rr (v2f64 (COPY_TO_REGCLASS FR64X:$src, VR128X)))>;11501 11502def : Pat<(vselect_mask (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcast f64:$src)),11503 (v2f64 VR128X:$src0)),11504 (VMOVDDUPZ128rrk VR128X:$src0, VK2WM:$mask,11505 (v2f64 (COPY_TO_REGCLASS FR64X:$src, VR128X)))>;11506def : Pat<(vselect_mask (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcast f64:$src)),11507 immAllZerosV),11508 (VMOVDDUPZ128rrkz VK2WM:$mask, (v2f64 (COPY_TO_REGCLASS FR64X:$src, VR128X)))>;11509}11510 11511//===----------------------------------------------------------------------===//11512// AVX-512 - Unpack Instructions11513//===----------------------------------------------------------------------===//11514 11515let Uses = []<Register>, mayRaiseFPException = 0 in {11516defm VUNPCKH : avx512_fp_binop_p<0x15, "vunpckh", X86Unpckh, X86Unpckh, HasAVX512,11517 SchedWriteFShuffleSizes, 0, 1>;11518defm VUNPCKL : avx512_fp_binop_p<0x14, "vunpckl", X86Unpckl, X86Unpckl, HasAVX512,11519 SchedWriteFShuffleSizes>;11520}11521 11522defm VPUNPCKLBW : avx512_binop_rm_vl_b<0x60, "vpunpcklbw", X86Unpckl,11523 SchedWriteShuffle, HasBWI>;11524defm VPUNPCKHBW : avx512_binop_rm_vl_b<0x68, "vpunpckhbw", X86Unpckh,11525 SchedWriteShuffle, HasBWI>;11526defm VPUNPCKLWD : avx512_binop_rm_vl_w<0x61, "vpunpcklwd", X86Unpckl,11527 SchedWriteShuffle, HasBWI>;11528defm VPUNPCKHWD : avx512_binop_rm_vl_w<0x69, "vpunpckhwd", X86Unpckh,11529 SchedWriteShuffle, HasBWI>;11530 11531defm VPUNPCKLDQ : avx512_binop_rm_vl_d<0x62, "vpunpckldq", X86Unpckl,11532 SchedWriteShuffle, HasAVX512>;11533defm VPUNPCKHDQ : avx512_binop_rm_vl_d<0x6A, "vpunpckhdq", X86Unpckh,11534 SchedWriteShuffle, HasAVX512>;11535defm VPUNPCKLQDQ : avx512_binop_rm_vl_q<0x6C, "vpunpcklqdq", X86Unpckl,11536 SchedWriteShuffle, HasAVX512>;11537defm VPUNPCKHQDQ : avx512_binop_rm_vl_q<0x6D, "vpunpckhqdq", X86Unpckh,11538 SchedWriteShuffle, HasAVX512>;11539 11540//===----------------------------------------------------------------------===//11541// AVX-512 - Extract & Insert Integer Instructions11542//===----------------------------------------------------------------------===//11543 11544multiclass avx512_extract_elt_bw_m<bits<8> opc, string OpcodeStr, SDNode OpNode,11545 X86VectorVTInfo _> {11546 def mri : AVX512Ii8<opc, MRMDestMem, (outs),11547 (ins _.ScalarMemOp:$dst, _.RC:$src1, u8imm:$src2),11548 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",11549 [(store (_.EltVT (trunc (OpNode (_.VT _.RC:$src1), timm:$src2))),11550 addr:$dst)]>,11551 EVEX, EVEX_CD8<_.EltSize, CD8VT1>, Sched<[WriteVecExtractSt]>;11552}11553 11554multiclass avx512_extract_elt_b<string OpcodeStr, X86VectorVTInfo _> {11555 let Predicates = [HasBWI] in {11556 def rri : AVX512Ii8<0x14, MRMDestReg, (outs GR32orGR64:$dst),11557 (ins _.RC:$src1, u8imm:$src2),11558 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",11559 [(set GR32orGR64:$dst,11560 (X86pextrb (_.VT _.RC:$src1), timm:$src2))]>,11561 EVEX, TA, PD, Sched<[WriteVecExtract]>;11562 11563 defm NAME : avx512_extract_elt_bw_m<0x14, OpcodeStr, X86pextrb, _>, TA, PD;11564 }11565}11566 11567multiclass avx512_extract_elt_w<string OpcodeStr, X86VectorVTInfo _> {11568 let Predicates = [HasBWI] in {11569 def rri : AVX512Ii8<0xC5, MRMSrcReg, (outs GR32orGR64:$dst),11570 (ins _.RC:$src1, u8imm:$src2),11571 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",11572 [(set GR32orGR64:$dst,11573 (X86pextrw (_.VT _.RC:$src1), timm:$src2))]>,11574 EVEX, TB, PD, Sched<[WriteVecExtract]>;11575 11576 let hasSideEffects = 0, isCodeGenOnly = 1, ForceDisassemble = 1 in11577 def rri_REV : AVX512Ii8<0x15, MRMDestReg, (outs GR32orGR64:$dst),11578 (ins _.RC:$src1, u8imm:$src2),11579 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,11580 EVEX, TA, PD, Sched<[WriteVecExtract]>;11581 11582 defm NAME : avx512_extract_elt_bw_m<0x15, OpcodeStr, X86pextrw, _>, TA, PD;11583 }11584}11585 11586multiclass avx512_extract_elt_dq<string OpcodeStr, X86VectorVTInfo _,11587 RegisterClass GRC> {11588 let Predicates = [HasDQI] in {11589 def rri : AVX512Ii8<0x16, MRMDestReg, (outs GRC:$dst),11590 (ins _.RC:$src1, u8imm:$src2),11591 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",11592 [(set GRC:$dst,11593 (extractelt (_.VT _.RC:$src1), imm:$src2))]>,11594 EVEX, TA, PD, Sched<[WriteVecExtract]>;11595 11596 def mri : AVX512Ii8<0x16, MRMDestMem, (outs),11597 (ins _.ScalarMemOp:$dst, _.RC:$src1, u8imm:$src2),11598 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",11599 [(store (extractelt (_.VT _.RC:$src1),11600 imm:$src2),addr:$dst)]>,11601 EVEX, EVEX_CD8<_.EltSize, CD8VT1>, TA, PD,11602 Sched<[WriteVecExtractSt]>;11603 }11604}11605 11606defm VPEXTRBZ : avx512_extract_elt_b<"vpextrb", v16i8x_info>, WIG;11607defm VPEXTRWZ : avx512_extract_elt_w<"vpextrw", v8i16x_info>, WIG;11608defm VPEXTRDZ : avx512_extract_elt_dq<"vpextrd", v4i32x_info, GR32>;11609defm VPEXTRQZ : avx512_extract_elt_dq<"vpextrq", v2i64x_info, GR64>, REX_W;11610 11611multiclass avx512_insert_elt_m<bits<8> opc, string OpcodeStr, SDNode OpNode,11612 X86VectorVTInfo _, PatFrag LdFrag,11613 SDPatternOperator immoperator> {11614 let mayLoad = 1 in11615 def rmi : AVX512Ii8<opc, MRMSrcMem, (outs _.RC:$dst),11616 (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$src3),11617 OpcodeStr#"\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",11618 [(set _.RC:$dst,11619 (_.VT (OpNode _.RC:$src1, (LdFrag addr:$src2), immoperator:$src3)))]>,11620 EVEX, VVVV, EVEX_CD8<_.EltSize, CD8VT1>, Sched<[WriteVecInsert.Folded, WriteVecInsert.ReadAfterFold]>;11621}11622 11623multiclass avx512_insert_elt_bw<bits<8> opc, string OpcodeStr, SDNode OpNode,11624 X86VectorVTInfo _, PatFrag LdFrag> {11625 let Predicates = [HasBWI] in {11626 def rri : AVX512Ii8<opc, MRMSrcReg, (outs _.RC:$dst),11627 (ins _.RC:$src1, GR32orGR64:$src2, u8imm:$src3),11628 OpcodeStr#"\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",11629 [(set _.RC:$dst,11630 (OpNode _.RC:$src1, GR32orGR64:$src2, timm:$src3))]>, EVEX, VVVV,11631 Sched<[WriteVecInsert]>;11632 11633 defm NAME : avx512_insert_elt_m<opc, OpcodeStr, OpNode, _, LdFrag, timm>;11634 }11635}11636 11637multiclass avx512_insert_elt_dq<bits<8> opc, string OpcodeStr,11638 X86VectorVTInfo _, RegisterClass GRC> {11639 let Predicates = [HasDQI] in {11640 def rri : AVX512Ii8<opc, MRMSrcReg, (outs _.RC:$dst),11641 (ins _.RC:$src1, GRC:$src2, u8imm:$src3),11642 OpcodeStr#"\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",11643 [(set _.RC:$dst,11644 (_.VT (insertelt _.RC:$src1, GRC:$src2, imm:$src3)))]>,11645 EVEX, VVVV, TA, PD, Sched<[WriteVecInsert]>;11646 11647 defm NAME : avx512_insert_elt_m<opc, OpcodeStr, insertelt, _,11648 _.ScalarLdFrag, imm>, TA, PD;11649 }11650}11651 11652defm VPINSRBZ : avx512_insert_elt_bw<0x20, "vpinsrb", X86pinsrb, v16i8x_info,11653 extloadi8>, TA, PD, WIG;11654defm VPINSRWZ : avx512_insert_elt_bw<0xC4, "vpinsrw", X86pinsrw, v8i16x_info,11655 extloadi16>, TB, PD, WIG;11656defm VPINSRDZ : avx512_insert_elt_dq<0x22, "vpinsrd", v4i32x_info, GR32>;11657defm VPINSRQZ : avx512_insert_elt_dq<0x22, "vpinsrq", v2i64x_info, GR64>, REX_W;11658 11659let Predicates = [HasAVX512, NoBWI] in {11660 def : Pat<(X86pinsrb VR128:$src1,11661 (i32 (anyext (i8 (bitconvert v8i1:$src2)))),11662 timm:$src3),11663 (VPINSRBrri VR128:$src1, (i32 (COPY_TO_REGCLASS VK8:$src2, GR32)),11664 timm:$src3)>;11665}11666 11667let Predicates = [HasBWI] in {11668 def : Pat<(X86pinsrb VR128:$src1, (i32 (anyext (i8 GR8:$src2))), timm:$src3),11669 (VPINSRBZrri VR128:$src1, (INSERT_SUBREG (i32 (IMPLICIT_DEF)),11670 GR8:$src2, sub_8bit), timm:$src3)>;11671 def : Pat<(X86pinsrb VR128:$src1,11672 (i32 (anyext (i8 (bitconvert v8i1:$src2)))),11673 timm:$src3),11674 (VPINSRBZrri VR128:$src1, (i32 (COPY_TO_REGCLASS VK8:$src2, GR32)),11675 timm:$src3)>;11676}11677 11678// Always select FP16 instructions if available.11679let Predicates = [HasBWI], AddedComplexity = -10 in {11680 def : Pat<(f16 (load addr:$src)), (COPY_TO_REGCLASS (VPINSRWZrmi (v8i16 (IMPLICIT_DEF)), addr:$src, 0), FR16X)>;11681 def : Pat<(store f16:$src, addr:$dst), (VPEXTRWZmri addr:$dst, (v8i16 (COPY_TO_REGCLASS FR16:$src, VR128)), 0)>;11682 def : Pat<(i16 (bitconvert f16:$src)), (EXTRACT_SUBREG (VPEXTRWZrri (v8i16 (COPY_TO_REGCLASS FR16X:$src, VR128X)), 0), sub_16bit)>;11683 def : Pat<(f16 (bitconvert i16:$src)), (COPY_TO_REGCLASS (VPINSRWZrri (v8i16 (IMPLICIT_DEF)), (INSERT_SUBREG (IMPLICIT_DEF), GR16:$src, sub_16bit), 0), FR16X)>;11684}11685 11686//===----------------------------------------------------------------------===//11687// VSHUFPS - VSHUFPD Operations11688//===----------------------------------------------------------------------===//11689 11690multiclass avx512_shufp<string OpcodeStr, AVX512VLVectorVTInfo VTInfo_FP>{11691 defm NAME: avx512_common_3Op_imm8<OpcodeStr, VTInfo_FP, 0xC6, X86Shufp,11692 SchedWriteFShuffle>,11693 EVEX_CD8<VTInfo_FP.info512.EltSize, CD8VF>,11694 TA, EVEX, VVVV;11695}11696 11697defm VSHUFPS: avx512_shufp<"vshufps", avx512vl_f32_info>, TB;11698defm VSHUFPD: avx512_shufp<"vshufpd", avx512vl_f64_info>, TB, PD, REX_W;11699 11700//===----------------------------------------------------------------------===//11701// AVX-512 - Byte shift Left/Right11702//===----------------------------------------------------------------------===//11703 11704multiclass avx512_shift_packed<bits<8> opc, SDNode OpNode, Format MRMr,11705 Format MRMm, string OpcodeStr,11706 X86FoldableSchedWrite sched, X86VectorVTInfo _>{11707 def ri : AVX512<opc, MRMr,11708 (outs _.RC:$dst), (ins _.RC:$src1, u8imm:$src2),11709 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),11710 [(set _.RC:$dst,(_.VT (OpNode _.RC:$src1, (i8 timm:$src2))))]>,11711 Sched<[sched]>;11712 def mi : AVX512<opc, MRMm,11713 (outs _.RC:$dst), (ins _.MemOp:$src1, u8imm:$src2),11714 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),11715 [(set _.RC:$dst,(_.VT (OpNode11716 (_.VT (bitconvert (_.LdFrag addr:$src1))),11717 (i8 timm:$src2))))]>,11718 Sched<[sched.Folded, sched.ReadAfterFold]>;11719}11720 11721multiclass avx512_shift_packed_all<bits<8> opc, SDNode OpNode, Format MRMr,11722 Format MRMm, string OpcodeStr,11723 X86SchedWriteWidths sched, Predicate prd>{11724 let Predicates = [prd] in11725 defm Z : avx512_shift_packed<opc, OpNode, MRMr, MRMm, OpcodeStr,11726 sched.ZMM, v64i8_info>, EVEX_V512;11727 let Predicates = [prd, HasVLX] in {11728 defm Z256 : avx512_shift_packed<opc, OpNode, MRMr, MRMm, OpcodeStr,11729 sched.YMM, v32i8x_info>, EVEX_V256;11730 defm Z128 : avx512_shift_packed<opc, OpNode, MRMr, MRMm, OpcodeStr,11731 sched.XMM, v16i8x_info>, EVEX_V128;11732 }11733}11734defm VPSLLDQ : avx512_shift_packed_all<0x73, X86vshldq, MRM7r, MRM7m, "vpslldq",11735 SchedWriteShuffle, HasBWI>,11736 AVX512PDIi8Base, EVEX, VVVV, WIG;11737defm VPSRLDQ : avx512_shift_packed_all<0x73, X86vshrdq, MRM3r, MRM3m, "vpsrldq",11738 SchedWriteShuffle, HasBWI>,11739 AVX512PDIi8Base, EVEX, VVVV, WIG;11740 11741multiclass avx512_psadbw_packed<bits<8> opc, SDNode OpNode,11742 string OpcodeStr, X86FoldableSchedWrite sched,11743 X86VectorVTInfo _dst, X86VectorVTInfo _src> {11744 let isCommutable = 1 in11745 def rr : AVX512BI<opc, MRMSrcReg,11746 (outs _dst.RC:$dst), (ins _src.RC:$src1, _src.RC:$src2),11747 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),11748 [(set _dst.RC:$dst,(_dst.VT11749 (OpNode (_src.VT _src.RC:$src1),11750 (_src.VT _src.RC:$src2))))]>,11751 Sched<[sched]>;11752 let mayLoad = 1 in11753 def rm : AVX512BI<opc, MRMSrcMem,11754 (outs _dst.RC:$dst), (ins _src.RC:$src1, _src.MemOp:$src2),11755 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),11756 [(set _dst.RC:$dst,(_dst.VT11757 (OpNode (_src.VT _src.RC:$src1),11758 (_src.VT (bitconvert11759 (_src.LdFrag addr:$src2))))))]>,11760 Sched<[sched.Folded, sched.ReadAfterFold]>;11761}11762 11763multiclass avx512_psadbw_packed_all<bits<8> opc, SDNode OpNode,11764 string OpcodeStr, X86SchedWriteWidths sched,11765 Predicate prd> {11766 let Predicates = [prd] in11767 defm Z : avx512_psadbw_packed<opc, OpNode, OpcodeStr, sched.ZMM,11768 v8i64_info, v64i8_info>, EVEX_V512;11769 let Predicates = [prd, HasVLX] in {11770 defm Z256 : avx512_psadbw_packed<opc, OpNode, OpcodeStr, sched.YMM,11771 v4i64x_info, v32i8x_info>, EVEX_V256;11772 defm Z128 : avx512_psadbw_packed<opc, OpNode, OpcodeStr, sched.XMM,11773 v2i64x_info, v16i8x_info>, EVEX_V128;11774 }11775}11776 11777defm VPSADBW : avx512_psadbw_packed_all<0xf6, X86psadbw, "vpsadbw",11778 SchedWritePSADBW, HasBWI>, EVEX, VVVV, WIG;11779 11780// Transforms to swizzle an immediate to enable better matching when11781// memory operand isn't in the right place.11782def VPTERNLOG321_imm8 : SDNodeXForm<timm, [{11783 // Convert a VPTERNLOG immediate by swapping operand 0 and operand 2.11784 uint8_t Imm = N->getZExtValue();11785 // Swap bits 1/4 and 3/6.11786 uint8_t NewImm = Imm & 0xa5;11787 if (Imm & 0x02) NewImm |= 0x10;11788 if (Imm & 0x10) NewImm |= 0x02;11789 if (Imm & 0x08) NewImm |= 0x40;11790 if (Imm & 0x40) NewImm |= 0x08;11791 return getI8Imm(NewImm, SDLoc(N));11792}]>;11793def VPTERNLOG213_imm8 : SDNodeXForm<timm, [{11794 // Convert a VPTERNLOG immediate by swapping operand 1 and operand 2.11795 uint8_t Imm = N->getZExtValue();11796 // Swap bits 2/4 and 3/5.11797 uint8_t NewImm = Imm & 0xc3;11798 if (Imm & 0x04) NewImm |= 0x10;11799 if (Imm & 0x10) NewImm |= 0x04;11800 if (Imm & 0x08) NewImm |= 0x20;11801 if (Imm & 0x20) NewImm |= 0x08;11802 return getI8Imm(NewImm, SDLoc(N));11803}]>;11804def VPTERNLOG132_imm8 : SDNodeXForm<timm, [{11805 // Convert a VPTERNLOG immediate by swapping operand 1 and operand 2.11806 uint8_t Imm = N->getZExtValue();11807 // Swap bits 1/2 and 5/6.11808 uint8_t NewImm = Imm & 0x99;11809 if (Imm & 0x02) NewImm |= 0x04;11810 if (Imm & 0x04) NewImm |= 0x02;11811 if (Imm & 0x20) NewImm |= 0x40;11812 if (Imm & 0x40) NewImm |= 0x20;11813 return getI8Imm(NewImm, SDLoc(N));11814}]>;11815def VPTERNLOG231_imm8 : SDNodeXForm<timm, [{11816 // Convert a VPTERNLOG immediate by moving operand 1 to the end.11817 uint8_t Imm = N->getZExtValue();11818 // Move bits 1->2, 2->4, 3->6, 4->1, 5->3, 6->511819 uint8_t NewImm = Imm & 0x81;11820 if (Imm & 0x02) NewImm |= 0x04;11821 if (Imm & 0x04) NewImm |= 0x10;11822 if (Imm & 0x08) NewImm |= 0x40;11823 if (Imm & 0x10) NewImm |= 0x02;11824 if (Imm & 0x20) NewImm |= 0x08;11825 if (Imm & 0x40) NewImm |= 0x20;11826 return getI8Imm(NewImm, SDLoc(N));11827}]>;11828def VPTERNLOG312_imm8 : SDNodeXForm<timm, [{11829 // Convert a VPTERNLOG immediate by moving operand 2 to the beginning.11830 uint8_t Imm = N->getZExtValue();11831 // Move bits 1->4, 2->1, 3->5, 4->2, 5->6, 6->311832 uint8_t NewImm = Imm & 0x81;11833 if (Imm & 0x02) NewImm |= 0x10;11834 if (Imm & 0x04) NewImm |= 0x02;11835 if (Imm & 0x08) NewImm |= 0x20;11836 if (Imm & 0x10) NewImm |= 0x04;11837 if (Imm & 0x20) NewImm |= 0x40;11838 if (Imm & 0x40) NewImm |= 0x08;11839 return getI8Imm(NewImm, SDLoc(N));11840}]>;11841 11842multiclass avx512_ternlog<bits<8> opc, string OpcodeStr, SDNode OpNode,11843 X86FoldableSchedWrite sched, X86VectorVTInfo _,11844 string Name>{11845 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain in {11846 defm rri : AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),11847 (ins _.RC:$src2, _.RC:$src3, u8imm:$src4),11848 OpcodeStr, "$src4, $src3, $src2", "$src2, $src3, $src4",11849 (OpNode (_.VT _.RC:$src1),11850 (_.VT _.RC:$src2),11851 (_.VT _.RC:$src3),11852 (i8 timm:$src4)), 1, 1>,11853 AVX512AIi8Base, EVEX, VVVV, Sched<[sched]>;11854 let mayLoad = 1 in {11855 defm rmi : AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),11856 (ins _.RC:$src2, _.MemOp:$src3, u8imm:$src4),11857 OpcodeStr, "$src4, $src3, $src2", "$src2, $src3, $src4",11858 (OpNode (_.VT _.RC:$src1),11859 (_.VT _.RC:$src2),11860 (_.VT (bitconvert (_.LdFrag addr:$src3))),11861 (i8 timm:$src4)), 1, 0>,11862 AVX512AIi8Base, EVEX, VVVV, EVEX_CD8<_.EltSize, CD8VF>,11863 Sched<[sched.Folded, sched.ReadAfterFold]>;11864 defm rmbi : AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),11865 (ins _.RC:$src2, _.ScalarMemOp:$src3, u8imm:$src4),11866 OpcodeStr, "$src4, ${src3}"#_.BroadcastStr#", $src2",11867 "$src2, ${src3}"#_.BroadcastStr#", $src4",11868 (OpNode (_.VT _.RC:$src1),11869 (_.VT _.RC:$src2),11870 (_.VT (_.BroadcastLdFrag addr:$src3)),11871 (i8 timm:$src4)), 1, 0>, EVEX_B,11872 AVX512AIi8Base, EVEX, VVVV, EVEX_CD8<_.EltSize, CD8VF>,11873 Sched<[sched.Folded, sched.ReadAfterFold]>;11874 }11875 }// Constraints = "$src1 = $dst"11876 11877 // Additional patterns for matching passthru operand in other positions.11878 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,11879 (OpNode _.RC:$src3, _.RC:$src2, _.RC:$src1, (i8 timm:$src4)),11880 _.RC:$src1)),11881 (!cast<Instruction>(Name#_.ZSuffix#rrik) _.RC:$src1, _.KRCWM:$mask,11882 _.RC:$src2, _.RC:$src3, (VPTERNLOG321_imm8 timm:$src4))>;11883 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,11884 (OpNode _.RC:$src2, _.RC:$src1, _.RC:$src3, (i8 timm:$src4)),11885 _.RC:$src1)),11886 (!cast<Instruction>(Name#_.ZSuffix#rrik) _.RC:$src1, _.KRCWM:$mask,11887 _.RC:$src2, _.RC:$src3, (VPTERNLOG213_imm8 timm:$src4))>;11888 11889 // Additional patterns for matching zero masking with loads in other11890 // positions.11891 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,11892 (OpNode (bitconvert (_.LdFrag addr:$src3)),11893 _.RC:$src2, _.RC:$src1, (i8 timm:$src4)),11894 _.ImmAllZerosV)),11895 (!cast<Instruction>(Name#_.ZSuffix#rmikz) _.RC:$src1, _.KRCWM:$mask,11896 _.RC:$src2, addr:$src3, (VPTERNLOG321_imm8 timm:$src4))>;11897 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,11898 (OpNode _.RC:$src1, (bitconvert (_.LdFrag addr:$src3)),11899 _.RC:$src2, (i8 timm:$src4)),11900 _.ImmAllZerosV)),11901 (!cast<Instruction>(Name#_.ZSuffix#rmikz) _.RC:$src1, _.KRCWM:$mask,11902 _.RC:$src2, addr:$src3, (VPTERNLOG132_imm8 timm:$src4))>;11903 11904 // Additional patterns for matching masked loads with different11905 // operand orders.11906 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,11907 (OpNode _.RC:$src1, (bitconvert (_.LdFrag addr:$src3)),11908 _.RC:$src2, (i8 timm:$src4)),11909 _.RC:$src1)),11910 (!cast<Instruction>(Name#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,11911 _.RC:$src2, addr:$src3, (VPTERNLOG132_imm8 timm:$src4))>;11912 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,11913 (OpNode (bitconvert (_.LdFrag addr:$src3)),11914 _.RC:$src2, _.RC:$src1, (i8 timm:$src4)),11915 _.RC:$src1)),11916 (!cast<Instruction>(Name#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,11917 _.RC:$src2, addr:$src3, (VPTERNLOG321_imm8 timm:$src4))>;11918 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,11919 (OpNode _.RC:$src2, _.RC:$src1,11920 (bitconvert (_.LdFrag addr:$src3)), (i8 timm:$src4)),11921 _.RC:$src1)),11922 (!cast<Instruction>(Name#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,11923 _.RC:$src2, addr:$src3, (VPTERNLOG213_imm8 timm:$src4))>;11924 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,11925 (OpNode _.RC:$src2, (bitconvert (_.LdFrag addr:$src3)),11926 _.RC:$src1, (i8 timm:$src4)),11927 _.RC:$src1)),11928 (!cast<Instruction>(Name#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,11929 _.RC:$src2, addr:$src3, (VPTERNLOG231_imm8 timm:$src4))>;11930 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,11931 (OpNode (bitconvert (_.LdFrag addr:$src3)),11932 _.RC:$src1, _.RC:$src2, (i8 timm:$src4)),11933 _.RC:$src1)),11934 (!cast<Instruction>(Name#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,11935 _.RC:$src2, addr:$src3, (VPTERNLOG312_imm8 timm:$src4))>;11936 11937 // Additional patterns for matching zero masking with broadcasts in other11938 // positions.11939 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,11940 (OpNode (_.BroadcastLdFrag addr:$src3),11941 _.RC:$src2, _.RC:$src1, (i8 timm:$src4)),11942 _.ImmAllZerosV)),11943 (!cast<Instruction>(Name#_.ZSuffix#rmbikz) _.RC:$src1,11944 _.KRCWM:$mask, _.RC:$src2, addr:$src3,11945 (VPTERNLOG321_imm8 timm:$src4))>;11946 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,11947 (OpNode _.RC:$src1,11948 (_.BroadcastLdFrag addr:$src3),11949 _.RC:$src2, (i8 timm:$src4)),11950 _.ImmAllZerosV)),11951 (!cast<Instruction>(Name#_.ZSuffix#rmbikz) _.RC:$src1,11952 _.KRCWM:$mask, _.RC:$src2, addr:$src3,11953 (VPTERNLOG132_imm8 timm:$src4))>;11954 11955 // Additional patterns for matching masked broadcasts with different11956 // operand orders.11957 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,11958 (OpNode _.RC:$src1, (_.BroadcastLdFrag addr:$src3),11959 _.RC:$src2, (i8 timm:$src4)),11960 _.RC:$src1)),11961 (!cast<Instruction>(Name#_.ZSuffix#rmbik) _.RC:$src1, _.KRCWM:$mask,11962 _.RC:$src2, addr:$src3, (VPTERNLOG132_imm8 timm:$src4))>;11963 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,11964 (OpNode (_.BroadcastLdFrag addr:$src3),11965 _.RC:$src2, _.RC:$src1, (i8 timm:$src4)),11966 _.RC:$src1)),11967 (!cast<Instruction>(Name#_.ZSuffix#rmbik) _.RC:$src1, _.KRCWM:$mask,11968 _.RC:$src2, addr:$src3, (VPTERNLOG321_imm8 timm:$src4))>;11969 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,11970 (OpNode _.RC:$src2, _.RC:$src1,11971 (_.BroadcastLdFrag addr:$src3),11972 (i8 timm:$src4)), _.RC:$src1)),11973 (!cast<Instruction>(Name#_.ZSuffix#rmbik) _.RC:$src1, _.KRCWM:$mask,11974 _.RC:$src2, addr:$src3, (VPTERNLOG213_imm8 timm:$src4))>;11975 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,11976 (OpNode _.RC:$src2,11977 (_.BroadcastLdFrag addr:$src3),11978 _.RC:$src1, (i8 timm:$src4)),11979 _.RC:$src1)),11980 (!cast<Instruction>(Name#_.ZSuffix#rmbik) _.RC:$src1, _.KRCWM:$mask,11981 _.RC:$src2, addr:$src3, (VPTERNLOG231_imm8 timm:$src4))>;11982 def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,11983 (OpNode (_.BroadcastLdFrag addr:$src3),11984 _.RC:$src1, _.RC:$src2, (i8 timm:$src4)),11985 _.RC:$src1)),11986 (!cast<Instruction>(Name#_.ZSuffix#rmbik) _.RC:$src1, _.KRCWM:$mask,11987 _.RC:$src2, addr:$src3, (VPTERNLOG312_imm8 timm:$src4))>;11988}11989 11990multiclass avx512_common_ternlog<string OpcodeStr, X86SchedWriteWidths sched,11991 AVX512VLVectorVTInfo _> {11992 let Predicates = [HasAVX512] in11993 defm Z : avx512_ternlog<0x25, OpcodeStr, X86vpternlog, sched.ZMM,11994 _.info512, NAME>, EVEX_V512;11995 let Predicates = [HasAVX512, HasVLX] in {11996 defm Z128 : avx512_ternlog<0x25, OpcodeStr, X86vpternlog, sched.XMM,11997 _.info128, NAME>, EVEX_V128;11998 defm Z256 : avx512_ternlog<0x25, OpcodeStr, X86vpternlog, sched.YMM,11999 _.info256, NAME>, EVEX_V256;12000 }12001}12002 12003defm VPTERNLOGD : avx512_common_ternlog<"vpternlogd", SchedWriteVecALU,12004 avx512vl_i32_info>;12005defm VPTERNLOGQ : avx512_common_ternlog<"vpternlogq", SchedWriteVecALU,12006 avx512vl_i64_info>, REX_W;12007 12008// Patterns to implement vnot using vpternlog instead of creating all ones12009// using pcmpeq or vpternlog and then xoring with that. The value 15 is chosen12010// so that the result is only dependent on src0. But we use the same source12011// for all operands to prevent a false dependency.12012// TODO: We should maybe have a more generalized algorithm for folding to12013// vpternlog.12014let Predicates = [HasAVX512] in {12015 def : Pat<(v64i8 (vnot VR512:$src)),12016 (VPTERNLOGQZrri VR512:$src, VR512:$src, VR512:$src, (i8 15))>;12017 def : Pat<(v32i16 (vnot VR512:$src)),12018 (VPTERNLOGQZrri VR512:$src, VR512:$src, VR512:$src, (i8 15))>;12019 def : Pat<(v16i32 (vnot VR512:$src)),12020 (VPTERNLOGQZrri VR512:$src, VR512:$src, VR512:$src, (i8 15))>;12021 def : Pat<(v8i64 (vnot VR512:$src)),12022 (VPTERNLOGQZrri VR512:$src, VR512:$src, VR512:$src, (i8 15))>;12023}12024 12025let Predicates = [HasAVX512, NoVLX] in {12026 def : Pat<(v16i8 (vnot VR128X:$src)),12027 (EXTRACT_SUBREG12028 (VPTERNLOGQZrri12029 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),12030 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),12031 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),12032 (i8 15)), sub_xmm)>;12033 def : Pat<(v8i16 (vnot VR128X:$src)),12034 (EXTRACT_SUBREG12035 (VPTERNLOGQZrri12036 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),12037 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),12038 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),12039 (i8 15)), sub_xmm)>;12040 def : Pat<(v4i32 (vnot VR128X:$src)),12041 (EXTRACT_SUBREG12042 (VPTERNLOGQZrri12043 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),12044 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),12045 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),12046 (i8 15)), sub_xmm)>;12047 def : Pat<(v2i64 (vnot VR128X:$src)),12048 (EXTRACT_SUBREG12049 (VPTERNLOGQZrri12050 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),12051 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),12052 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),12053 (i8 15)), sub_xmm)>;12054 12055 def : Pat<(v32i8 (vnot VR256X:$src)),12056 (EXTRACT_SUBREG12057 (VPTERNLOGQZrri12058 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),12059 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),12060 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),12061 (i8 15)), sub_ymm)>;12062 def : Pat<(v16i16 (vnot VR256X:$src)),12063 (EXTRACT_SUBREG12064 (VPTERNLOGQZrri12065 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),12066 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),12067 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),12068 (i8 15)), sub_ymm)>;12069 def : Pat<(v8i32 (vnot VR256X:$src)),12070 (EXTRACT_SUBREG12071 (VPTERNLOGQZrri12072 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),12073 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),12074 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),12075 (i8 15)), sub_ymm)>;12076 def : Pat<(v4i64 (vnot VR256X:$src)),12077 (EXTRACT_SUBREG12078 (VPTERNLOGQZrri12079 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),12080 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),12081 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),12082 (i8 15)), sub_ymm)>;12083}12084 12085let Predicates = [HasVLX] in {12086 def : Pat<(v16i8 (vnot VR128X:$src)),12087 (VPTERNLOGQZ128rri VR128X:$src, VR128X:$src, VR128X:$src, (i8 15))>;12088 def : Pat<(v8i16 (vnot VR128X:$src)),12089 (VPTERNLOGQZ128rri VR128X:$src, VR128X:$src, VR128X:$src, (i8 15))>;12090 def : Pat<(v4i32 (vnot VR128X:$src)),12091 (VPTERNLOGQZ128rri VR128X:$src, VR128X:$src, VR128X:$src, (i8 15))>;12092 def : Pat<(v2i64 (vnot VR128X:$src)),12093 (VPTERNLOGQZ128rri VR128X:$src, VR128X:$src, VR128X:$src, (i8 15))>;12094 12095 def : Pat<(v32i8 (vnot VR256X:$src)),12096 (VPTERNLOGQZ256rri VR256X:$src, VR256X:$src, VR256X:$src, (i8 15))>;12097 def : Pat<(v16i16 (vnot VR256X:$src)),12098 (VPTERNLOGQZ256rri VR256X:$src, VR256X:$src, VR256X:$src, (i8 15))>;12099 def : Pat<(v8i32 (vnot VR256X:$src)),12100 (VPTERNLOGQZ256rri VR256X:$src, VR256X:$src, VR256X:$src, (i8 15))>;12101 def : Pat<(v4i64 (vnot VR256X:$src)),12102 (VPTERNLOGQZ256rri VR256X:$src, VR256X:$src, VR256X:$src, (i8 15))>;12103}12104 12105//===----------------------------------------------------------------------===//12106// AVX-512 - FixupImm12107//===----------------------------------------------------------------------===//12108 12109multiclass avx512_fixupimm_packed<bits<8> opc, string OpcodeStr,12110 X86FoldableSchedWrite sched, X86VectorVTInfo _,12111 X86VectorVTInfo TblVT>{12112 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain,12113 Uses = [MXCSR], mayRaiseFPException = 1 in {12114 defm rri : AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),12115 (ins _.RC:$src2, _.RC:$src3, i32u8imm:$src4),12116 OpcodeStr#_.Suffix, "$src4, $src3, $src2", "$src2, $src3, $src4",12117 (X86VFixupimm (_.VT _.RC:$src1),12118 (_.VT _.RC:$src2),12119 (TblVT.VT _.RC:$src3),12120 (i32 timm:$src4))>, Sched<[sched]>;12121 let mayLoad = 1 in {12122 defm rmi : AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),12123 (ins _.RC:$src2, _.MemOp:$src3, i32u8imm:$src4),12124 OpcodeStr#_.Suffix, "$src4, $src3, $src2", "$src2, $src3, $src4",12125 (X86VFixupimm (_.VT _.RC:$src1),12126 (_.VT _.RC:$src2),12127 (TblVT.VT (bitconvert (TblVT.LdFrag addr:$src3))),12128 (i32 timm:$src4))>,12129 Sched<[sched.Folded, sched.ReadAfterFold]>;12130 defm rmbi : AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),12131 (ins _.RC:$src2, _.ScalarMemOp:$src3, i32u8imm:$src4),12132 OpcodeStr#_.Suffix, "$src4, ${src3}"#_.BroadcastStr#", $src2",12133 "$src2, ${src3}"#_.BroadcastStr#", $src4",12134 (X86VFixupimm (_.VT _.RC:$src1),12135 (_.VT _.RC:$src2),12136 (TblVT.VT (TblVT.BroadcastLdFrag addr:$src3)),12137 (i32 timm:$src4))>,12138 EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;12139 }12140 } // Constraints = "$src1 = $dst"12141}12142 12143multiclass avx512_fixupimm_packed_sae<bits<8> opc, string OpcodeStr,12144 X86FoldableSchedWrite sched,12145 X86VectorVTInfo _, X86VectorVTInfo TblVT>12146 : avx512_fixupimm_packed<opc, OpcodeStr, sched, _, TblVT> {12147let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain, Uses = [MXCSR] in {12148 defm rrib : AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),12149 (ins _.RC:$src2, _.RC:$src3, i32u8imm:$src4),12150 OpcodeStr#_.Suffix, "$src4, {sae}, $src3, $src2",12151 "$src2, $src3, {sae}, $src4",12152 (X86VFixupimmSAE (_.VT _.RC:$src1),12153 (_.VT _.RC:$src2),12154 (TblVT.VT _.RC:$src3),12155 (i32 timm:$src4))>,12156 EVEX_B, Sched<[sched]>;12157 }12158}12159 12160multiclass avx512_fixupimm_scalar<bits<8> opc, string OpcodeStr,12161 X86FoldableSchedWrite sched, X86VectorVTInfo _,12162 X86VectorVTInfo _src3VT> {12163 let Constraints = "$src1 = $dst" , Predicates = [HasAVX512],12164 ExeDomain = _.ExeDomain in {12165 defm rri : AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),12166 (ins _.RC:$src2, _.RC:$src3, i32u8imm:$src4),12167 OpcodeStr#_.Suffix, "$src4, $src3, $src2", "$src2, $src3, $src4",12168 (X86VFixupimms (_.VT _.RC:$src1),12169 (_.VT _.RC:$src2),12170 (_src3VT.VT _src3VT.RC:$src3),12171 (i32 timm:$src4))>, Sched<[sched]>, SIMD_EXC;12172 let Uses = [MXCSR] in12173 defm rrib : AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),12174 (ins _.RC:$src2, _.RC:$src3, i32u8imm:$src4),12175 OpcodeStr#_.Suffix, "$src4, {sae}, $src3, $src2",12176 "$src2, $src3, {sae}, $src4",12177 (X86VFixupimmSAEs (_.VT _.RC:$src1),12178 (_.VT _.RC:$src2),12179 (_src3VT.VT _src3VT.RC:$src3),12180 (i32 timm:$src4))>,12181 EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;12182 let mayLoad = 1 in12183 defm rmi : AVX512_maskable_3src_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),12184 (ins _.RC:$src2, _.ScalarMemOp:$src3, i32u8imm:$src4),12185 OpcodeStr#_.Suffix, "$src4, $src3, $src2", "$src2, $src3, $src4",12186 (X86VFixupimms (_.VT _.RC:$src1),12187 (_.VT _.RC:$src2),12188 (_src3VT.VT (scalar_to_vector12189 (_src3VT.ScalarLdFrag addr:$src3))),12190 (i32 timm:$src4))>,12191 Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC;12192 }12193}12194 12195multiclass avx512_fixupimm_packed_all<X86SchedWriteWidths sched,12196 AVX512VLVectorVTInfo _Vec,12197 AVX512VLVectorVTInfo _Tbl> {12198 let Predicates = [HasAVX512] in12199 defm Z : avx512_fixupimm_packed_sae<0x54, "vfixupimm", sched.ZMM,12200 _Vec.info512, _Tbl.info512>, AVX512AIi8Base,12201 EVEX, VVVV, EVEX_V512;12202 let Predicates = [HasAVX512, HasVLX] in {12203 defm Z128 : avx512_fixupimm_packed<0x54, "vfixupimm", sched.XMM,12204 _Vec.info128, _Tbl.info128>, AVX512AIi8Base,12205 EVEX, VVVV, EVEX_V128;12206 defm Z256 : avx512_fixupimm_packed<0x54, "vfixupimm", sched.YMM,12207 _Vec.info256, _Tbl.info256>, AVX512AIi8Base,12208 EVEX, VVVV, EVEX_V256;12209 }12210}12211 12212defm VFIXUPIMMSSZ : avx512_fixupimm_scalar<0x55, "vfixupimm",12213 SchedWriteFAdd.Scl, f32x_info, v4i32x_info>,12214 AVX512AIi8Base, VEX_LIG, EVEX, VVVV, EVEX_CD8<32, CD8VT1>;12215defm VFIXUPIMMSDZ : avx512_fixupimm_scalar<0x55, "vfixupimm",12216 SchedWriteFAdd.Scl, f64x_info, v2i64x_info>,12217 AVX512AIi8Base, VEX_LIG, EVEX, VVVV, EVEX_CD8<64, CD8VT1>, REX_W;12218defm VFIXUPIMMPS : avx512_fixupimm_packed_all<SchedWriteFAdd, avx512vl_f32_info,12219 avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;12220defm VFIXUPIMMPD : avx512_fixupimm_packed_all<SchedWriteFAdd, avx512vl_f64_info,12221 avx512vl_i64_info>, EVEX_CD8<64, CD8VF>, REX_W;12222 12223// Patterns used to select SSE scalar fp arithmetic instructions from12224// either:12225//12226// (1) a scalar fp operation followed by a blend12227//12228// The effect is that the backend no longer emits unnecessary vector12229// insert instructions immediately after SSE scalar fp instructions12230// like addss or mulss.12231//12232// For example, given the following code:12233// __m128 foo(__m128 A, __m128 B) {12234// A[0] += B[0];12235// return A;12236// }12237//12238// Previously we generated:12239// addss %xmm0, %xmm112240// movss %xmm1, %xmm012241//12242// We now generate:12243// addss %xmm1, %xmm012244//12245// (2) a vector packed single/double fp operation followed by a vector insert12246//12247// The effect is that the backend converts the packed fp instruction12248// followed by a vector insert into a single SSE scalar fp instruction.12249//12250// For example, given the following code:12251// __m128 foo(__m128 A, __m128 B) {12252// __m128 C = A + B;12253// return (__m128) {c[0], a[1], a[2], a[3]};12254// }12255//12256// Previously we generated:12257// addps %xmm0, %xmm112258// movss %xmm1, %xmm012259//12260// We now generate:12261// addss %xmm1, %xmm012262 12263// TODO: Some canonicalization in lowering would simplify the number of12264// patterns we have to try to match.12265multiclass AVX512_scalar_math_fp_patterns<SDPatternOperator Op, SDNode MaskedOp,12266 string OpcPrefix, SDNode MoveNode,12267 X86VectorVTInfo _, PatLeaf ZeroFP> {12268 let Predicates = [HasAVX512] in {12269 // extracted scalar math op with insert via movss12270 def : Pat<(MoveNode12271 (_.VT VR128X:$dst),12272 (_.VT (scalar_to_vector12273 (Op (_.EltVT (extractelt (_.VT VR128X:$dst), (iPTR 0))),12274 _.FRC:$src)))),12275 (!cast<Instruction>("V"#OpcPrefix#"Zrr_Int") _.VT:$dst,12276 (_.VT (COPY_TO_REGCLASS _.FRC:$src, VR128X)))>;12277 def : Pat<(MoveNode12278 (_.VT VR128X:$dst),12279 (_.VT (scalar_to_vector12280 (Op (_.EltVT (extractelt (_.VT VR128X:$dst), (iPTR 0))),12281 (_.ScalarLdFrag addr:$src))))),12282 (!cast<Instruction>("V"#OpcPrefix#"Zrm_Int") _.VT:$dst, addr:$src)>;12283 12284 // extracted masked scalar math op with insert via movss12285 def : Pat<(MoveNode (_.VT VR128X:$src1),12286 (scalar_to_vector12287 (X86selects_mask VK1WM:$mask,12288 (MaskedOp (_.EltVT12289 (extractelt (_.VT VR128X:$src1), (iPTR 0))),12290 _.FRC:$src2),12291 _.FRC:$src0))),12292 (!cast<Instruction>("V"#OpcPrefix#"Zrrk_Int")12293 (_.VT (COPY_TO_REGCLASS _.FRC:$src0, VR128X)),12294 VK1WM:$mask, _.VT:$src1,12295 (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)))>;12296 def : Pat<(MoveNode (_.VT VR128X:$src1),12297 (scalar_to_vector12298 (X86selects_mask VK1WM:$mask,12299 (MaskedOp (_.EltVT12300 (extractelt (_.VT VR128X:$src1), (iPTR 0))),12301 (_.ScalarLdFrag addr:$src2)),12302 _.FRC:$src0))),12303 (!cast<Instruction>("V"#OpcPrefix#"Zrmk_Int")12304 (_.VT (COPY_TO_REGCLASS _.FRC:$src0, VR128X)),12305 VK1WM:$mask, _.VT:$src1, addr:$src2)>;12306 12307 // extracted masked scalar math op with insert via movss12308 def : Pat<(MoveNode (_.VT VR128X:$src1),12309 (scalar_to_vector12310 (X86selects_mask VK1WM:$mask,12311 (MaskedOp (_.EltVT12312 (extractelt (_.VT VR128X:$src1), (iPTR 0))),12313 _.FRC:$src2), (_.EltVT ZeroFP)))),12314 (!cast<I>("V"#OpcPrefix#"Zrrkz_Int")12315 VK1WM:$mask, _.VT:$src1,12316 (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)))>;12317 def : Pat<(MoveNode (_.VT VR128X:$src1),12318 (scalar_to_vector12319 (X86selects_mask VK1WM:$mask,12320 (MaskedOp (_.EltVT12321 (extractelt (_.VT VR128X:$src1), (iPTR 0))),12322 (_.ScalarLdFrag addr:$src2)), (_.EltVT ZeroFP)))),12323 (!cast<I>("V"#OpcPrefix#"Zrmkz_Int") VK1WM:$mask, _.VT:$src1, addr:$src2)>;12324 }12325}12326 12327defm : AVX512_scalar_math_fp_patterns<any_fadd, fadd, "ADDSS", X86Movss, v4f32x_info, fp32imm0>;12328defm : AVX512_scalar_math_fp_patterns<any_fsub, fsub, "SUBSS", X86Movss, v4f32x_info, fp32imm0>;12329defm : AVX512_scalar_math_fp_patterns<any_fmul, fmul, "MULSS", X86Movss, v4f32x_info, fp32imm0>;12330defm : AVX512_scalar_math_fp_patterns<any_fdiv, fdiv, "DIVSS", X86Movss, v4f32x_info, fp32imm0>;12331 12332defm : AVX512_scalar_math_fp_patterns<any_fadd, fadd, "ADDSD", X86Movsd, v2f64x_info, fp64imm0>;12333defm : AVX512_scalar_math_fp_patterns<any_fsub, fsub, "SUBSD", X86Movsd, v2f64x_info, fp64imm0>;12334defm : AVX512_scalar_math_fp_patterns<any_fmul, fmul, "MULSD", X86Movsd, v2f64x_info, fp64imm0>;12335defm : AVX512_scalar_math_fp_patterns<any_fdiv, fdiv, "DIVSD", X86Movsd, v2f64x_info, fp64imm0>;12336 12337defm : AVX512_scalar_math_fp_patterns<any_fadd, fadd, "ADDSH", X86Movsh, v8f16x_info, fp16imm0>;12338defm : AVX512_scalar_math_fp_patterns<any_fsub, fsub, "SUBSH", X86Movsh, v8f16x_info, fp16imm0>;12339defm : AVX512_scalar_math_fp_patterns<any_fmul, fmul, "MULSH", X86Movsh, v8f16x_info, fp16imm0>;12340defm : AVX512_scalar_math_fp_patterns<any_fdiv, fdiv, "DIVSH", X86Movsh, v8f16x_info, fp16imm0>;12341 12342multiclass AVX512_scalar_unary_math_patterns<SDPatternOperator OpNode, string OpcPrefix,12343 SDNode Move, X86VectorVTInfo _> {12344 let Predicates = [HasAVX512] in {12345 def : Pat<(_.VT (Move _.VT:$dst,12346 (scalar_to_vector (OpNode (extractelt _.VT:$src, 0))))),12347 (!cast<Instruction>("V"#OpcPrefix#"Zr_Int") _.VT:$dst, _.VT:$src)>;12348 }12349}12350 12351defm : AVX512_scalar_unary_math_patterns<any_fsqrt, "SQRTSS", X86Movss, v4f32x_info>;12352defm : AVX512_scalar_unary_math_patterns<any_fsqrt, "SQRTSD", X86Movsd, v2f64x_info>;12353defm : AVX512_scalar_unary_math_patterns<any_fsqrt, "SQRTSH", X86Movsh, v8f16x_info>;12354 12355//===----------------------------------------------------------------------===//12356// AES instructions12357//===----------------------------------------------------------------------===//12358 12359multiclass avx512_vaes<bits<8> Op, string OpStr, string IntPrefix> {12360 let Predicates = [HasVLX, HasVAES] in {12361 defm Z128 : AESI_binop_rm_int<Op, OpStr,12362 !cast<Intrinsic>(IntPrefix),12363 loadv2i64, 0, VR128X, i128mem>,12364 EVEX, VVVV, EVEX_CD8<64, CD8VF>, EVEX_V128, WIG;12365 defm Z256 : AESI_binop_rm_int<Op, OpStr,12366 !cast<Intrinsic>(IntPrefix#"_256"),12367 loadv4i64, 0, VR256X, i256mem>,12368 EVEX, VVVV, EVEX_CD8<64, CD8VF>, EVEX_V256, WIG;12369 }12370 let Predicates = [HasAVX512, HasVAES] in12371 defm Z : AESI_binop_rm_int<Op, OpStr,12372 !cast<Intrinsic>(IntPrefix#"_512"),12373 loadv8i64, 0, VR512, i512mem>,12374 EVEX, VVVV, EVEX_CD8<64, CD8VF>, EVEX_V512, WIG;12375}12376 12377defm VAESENC : avx512_vaes<0xDC, "vaesenc", "int_x86_aesni_aesenc">;12378defm VAESENCLAST : avx512_vaes<0xDD, "vaesenclast", "int_x86_aesni_aesenclast">;12379defm VAESDEC : avx512_vaes<0xDE, "vaesdec", "int_x86_aesni_aesdec">;12380defm VAESDECLAST : avx512_vaes<0xDF, "vaesdeclast", "int_x86_aesni_aesdeclast">;12381 12382//===----------------------------------------------------------------------===//12383// PCLMUL instructions - Carry less multiplication12384//===----------------------------------------------------------------------===//12385 12386let Predicates = [HasAVX512, HasVPCLMULQDQ] in12387defm VPCLMULQDQZ : vpclmulqdq<VR512, i512mem, loadv8i64, int_x86_pclmulqdq_512>,12388 EVEX, VVVV, EVEX_V512, EVEX_CD8<64, CD8VF>, WIG;12389 12390let Predicates = [HasVLX, HasVPCLMULQDQ] in {12391defm VPCLMULQDQZ128 : vpclmulqdq<VR128X, i128mem, loadv2i64, int_x86_pclmulqdq>,12392 EVEX, VVVV, EVEX_V128, EVEX_CD8<64, CD8VF>, WIG;12393 12394defm VPCLMULQDQZ256: vpclmulqdq<VR256X, i256mem, loadv4i64,12395 int_x86_pclmulqdq_256>, EVEX, VVVV, EVEX_V256,12396 EVEX_CD8<64, CD8VF>, WIG;12397}12398 12399// Aliases12400defm : vpclmulqdq_aliases<"VPCLMULQDQZ", VR512, i512mem>;12401defm : vpclmulqdq_aliases<"VPCLMULQDQZ128", VR128X, i128mem>;12402defm : vpclmulqdq_aliases<"VPCLMULQDQZ256", VR256X, i256mem>;12403 12404//===----------------------------------------------------------------------===//12405// VBMI212406//===----------------------------------------------------------------------===//12407 12408multiclass VBMI2_shift_var_rm<bits<8> Op, string OpStr, SDNode OpNode, bit SwapLR,12409 X86FoldableSchedWrite sched, X86VectorVTInfo VTI> {12410 let Constraints = "$src1 = $dst",12411 ExeDomain = VTI.ExeDomain in {12412 defm r: AVX512_maskable_3src<Op, MRMSrcReg, VTI, (outs VTI.RC:$dst),12413 (ins VTI.RC:$src2, VTI.RC:$src3), OpStr,12414 "$src3, $src2", "$src2, $src3",12415 !if(SwapLR,12416 (VTI.VT (OpNode (VTI.VT VTI.RC:$src2), (VTI.VT VTI.RC:$src1), (VTI.VT VTI.RC:$src3))),12417 (VTI.VT (OpNode (VTI.VT VTI.RC:$src1), (VTI.VT VTI.RC:$src2), (VTI.VT VTI.RC:$src3))))>,12418 T8, PD, EVEX, VVVV, Sched<[sched]>;12419 defm m: AVX512_maskable_3src<Op, MRMSrcMem, VTI, (outs VTI.RC:$dst),12420 (ins VTI.RC:$src2, VTI.MemOp:$src3), OpStr,12421 "$src3, $src2", "$src2, $src3",12422 !if(SwapLR,12423 (VTI.VT (OpNode (VTI.VT VTI.RC:$src2), (VTI.VT VTI.RC:$src1), (VTI.VT (VTI.LdFrag addr:$src3)))),12424 (VTI.VT (OpNode (VTI.VT VTI.RC:$src1), (VTI.VT VTI.RC:$src2), (VTI.VT (VTI.LdFrag addr:$src3)))))>,12425 T8, PD, EVEX, VVVV,12426 Sched<[sched.Folded, sched.ReadAfterFold]>;12427 }12428}12429 12430multiclass VBMI2_shift_var_rmb<bits<8> Op, string OpStr, SDNode OpNode, bit SwapLR,12431 X86FoldableSchedWrite sched, X86VectorVTInfo VTI>12432 : VBMI2_shift_var_rm<Op, OpStr, OpNode, SwapLR, sched, VTI> {12433 let Constraints = "$src1 = $dst",12434 ExeDomain = VTI.ExeDomain in12435 defm mb: AVX512_maskable_3src<Op, MRMSrcMem, VTI, (outs VTI.RC:$dst),12436 (ins VTI.RC:$src2, VTI.ScalarMemOp:$src3), OpStr,12437 "${src3}"#VTI.BroadcastStr#", $src2",12438 "$src2, ${src3}"#VTI.BroadcastStr,12439 !if(SwapLR,12440 (OpNode (VTI.VT VTI.RC:$src2), (VTI.VT VTI.RC:$src1), (VTI.VT (VTI.BroadcastLdFrag addr:$src3))),12441 (OpNode (VTI.VT VTI.RC:$src1), (VTI.VT VTI.RC:$src2), (VTI.VT (VTI.BroadcastLdFrag addr:$src3))))>,12442 T8, PD, EVEX, VVVV, EVEX_B,12443 Sched<[sched.Folded, sched.ReadAfterFold]>;12444}12445 12446multiclass VBMI2_shift_var_rm_common<bits<8> Op, string OpStr, SDNode OpNode, bit SwapLR,12447 X86SchedWriteWidths sched, AVX512VLVectorVTInfo VTI> {12448 let Predicates = [HasVBMI2] in12449 defm Z : VBMI2_shift_var_rm<Op, OpStr, OpNode, SwapLR, sched.ZMM, VTI.info512>,12450 EVEX_V512;12451 let Predicates = [HasVBMI2, HasVLX] in {12452 defm Z256 : VBMI2_shift_var_rm<Op, OpStr, OpNode, SwapLR, sched.YMM, VTI.info256>,12453 EVEX_V256;12454 defm Z128 : VBMI2_shift_var_rm<Op, OpStr, OpNode, SwapLR, sched.XMM, VTI.info128>,12455 EVEX_V128;12456 }12457}12458 12459multiclass VBMI2_shift_var_rmb_common<bits<8> Op, string OpStr, SDNode OpNode, bit SwapLR,12460 X86SchedWriteWidths sched, AVX512VLVectorVTInfo VTI> {12461 let Predicates = [HasVBMI2] in12462 defm Z : VBMI2_shift_var_rmb<Op, OpStr, OpNode, SwapLR, sched.ZMM, VTI.info512>,12463 EVEX_V512;12464 let Predicates = [HasVBMI2, HasVLX] in {12465 defm Z256 : VBMI2_shift_var_rmb<Op, OpStr, OpNode, SwapLR, sched.YMM, VTI.info256>,12466 EVEX_V256;12467 defm Z128 : VBMI2_shift_var_rmb<Op, OpStr, OpNode, SwapLR, sched.XMM, VTI.info128>,12468 EVEX_V128;12469 }12470}12471multiclass VBMI2_shift_var<bits<8> wOp, bits<8> dqOp, string Prefix,12472 SDNode OpNode, bit SwapLR, X86SchedWriteWidths sched> {12473 defm W : VBMI2_shift_var_rm_common<wOp, Prefix#"w", OpNode, SwapLR, sched,12474 avx512vl_i16_info>, REX_W, EVEX_CD8<16, CD8VF>;12475 defm D : VBMI2_shift_var_rmb_common<dqOp, Prefix#"d", OpNode, SwapLR, sched,12476 avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;12477 defm Q : VBMI2_shift_var_rmb_common<dqOp, Prefix#"q", OpNode, SwapLR, sched,12478 avx512vl_i64_info>, REX_W, EVEX_CD8<64, CD8VF>;12479}12480 12481multiclass VBMI2_shift_imm<bits<8> wOp, bits<8> dqOp, string Prefix,12482 SDNode OpNode, X86SchedWriteWidths sched> {12483 defm W : avx512_common_3Op_rm_imm8<wOp, OpNode, Prefix#"w", sched,12484 avx512vl_i16_info, avx512vl_i16_info, HasVBMI2>,12485 REX_W, EVEX_CD8<16, CD8VF>;12486 defm D : avx512_common_3Op_imm8<Prefix#"d", avx512vl_i32_info, dqOp,12487 OpNode, sched, HasVBMI2>, AVX512AIi8Base, EVEX, VVVV, EVEX_CD8<32, CD8VF>;12488 defm Q : avx512_common_3Op_imm8<Prefix#"q", avx512vl_i64_info, dqOp, OpNode,12489 sched, HasVBMI2>, AVX512AIi8Base, EVEX, VVVV, EVEX_CD8<64, CD8VF>, REX_W;12490}12491 12492// Concat & Shift12493defm VPSHLDV : VBMI2_shift_var<0x70, 0x71, "vpshldv", fshl, 0, SchedWriteVecIMul>;12494defm VPSHRDV : VBMI2_shift_var<0x72, 0x73, "vpshrdv", fshr, 1, SchedWriteVecIMul>;12495defm VPSHLD : VBMI2_shift_imm<0x70, 0x71, "vpshld", X86VShld, SchedWriteVecIMul>;12496defm VPSHRD : VBMI2_shift_imm<0x72, 0x73, "vpshrd", X86VShrd, SchedWriteVecIMul>;12497 12498// Compress12499defm VPCOMPRESSB : compress_by_elt_width<0x63, "vpcompressb", WriteVarShuffle256,12500 avx512vl_i8_info, HasVBMI2>, EVEX;12501defm VPCOMPRESSW : compress_by_elt_width <0x63, "vpcompressw", WriteVarShuffle256,12502 avx512vl_i16_info, HasVBMI2>, EVEX, REX_W;12503// Expand12504defm VPEXPANDB : expand_by_elt_width <0x62, "vpexpandb", WriteVarShuffle256,12505 avx512vl_i8_info, HasVBMI2>, EVEX;12506defm VPEXPANDW : expand_by_elt_width <0x62, "vpexpandw", WriteVarShuffle256,12507 avx512vl_i16_info, HasVBMI2>, EVEX, REX_W;12508 12509//===----------------------------------------------------------------------===//12510// VNNI12511//===----------------------------------------------------------------------===//12512 12513let Constraints = "$src1 = $dst" in12514multiclass VNNI_rmb<bits<8> Op, string OpStr, SDNode OpNode,12515 X86FoldableSchedWrite sched, X86VectorVTInfo VTI,12516 bit IsCommutable> {12517 let ExeDomain = VTI.ExeDomain in {12518 defm rr : AVX512_maskable_3src<Op, MRMSrcReg, VTI, (outs VTI.RC:$dst),12519 (ins VTI.RC:$src2, VTI.RC:$src3), OpStr,12520 "$src3, $src2", "$src2, $src3",12521 (VTI.VT (OpNode VTI.RC:$src1,12522 VTI.RC:$src2, VTI.RC:$src3)),12523 IsCommutable, IsCommutable>,12524 EVEX, VVVV, T8, Sched<[sched]>;12525 let mayLoad = 1 in {12526 defm rm : AVX512_maskable_3src<Op, MRMSrcMem, VTI, (outs VTI.RC:$dst),12527 (ins VTI.RC:$src2, VTI.MemOp:$src3), OpStr,12528 "$src3, $src2", "$src2, $src3",12529 (VTI.VT (OpNode VTI.RC:$src1, VTI.RC:$src2,12530 (VTI.VT (VTI.LdFrag addr:$src3))))>,12531 EVEX, VVVV, EVEX_CD8<32, CD8VF>, T8,12532 Sched<[sched.Folded, sched.ReadAfterFold,12533 sched.ReadAfterFold]>;12534 defm rmb : AVX512_maskable_3src<Op, MRMSrcMem, VTI, (outs VTI.RC:$dst),12535 (ins VTI.RC:$src2, VTI.ScalarMemOp:$src3),12536 OpStr, "${src3}"#VTI.BroadcastStr#", $src2",12537 "$src2, ${src3}"#VTI.BroadcastStr,12538 (OpNode VTI.RC:$src1, VTI.RC:$src2,12539 (VTI.VT (VTI.BroadcastLdFrag addr:$src3)))>,12540 EVEX, VVVV, EVEX_CD8<32, CD8VF>, EVEX_B,12541 T8, Sched<[sched.Folded, sched.ReadAfterFold,12542 sched.ReadAfterFold]>;12543 }12544 }12545}12546 12547multiclass VNNI_common<bits<8> Op, string OpStr, SDNode OpNode,12548 X86SchedWriteWidths sched, bit IsCommutable,12549 list<Predicate> prds, list<Predicate> prds512> {12550 let Predicates = prds512 in12551 defm Z : VNNI_rmb<Op, OpStr, OpNode, sched.ZMM, v16i32_info,12552 IsCommutable>, EVEX_V512;12553 let Predicates = prds in {12554 defm Z256 : VNNI_rmb<Op, OpStr, OpNode, sched.YMM, v8i32x_info,12555 IsCommutable>, EVEX_V256;12556 defm Z128 : VNNI_rmb<Op, OpStr, OpNode, sched.XMM, v4i32x_info,12557 IsCommutable>, EVEX_V128;12558 }12559}12560 12561// FIXME: Is there a better scheduler class for VPDP?12562defm VPDPBUSD : VNNI_common<0x50, "vpdpbusd", X86Vpdpbusd, SchedWriteVecIMul, 0,12563 [HasVNNI, HasVLX], [HasVNNI]>, PD;12564defm VPDPBUSDS : VNNI_common<0x51, "vpdpbusds", X86Vpdpbusds, SchedWriteVecIMul, 0,12565 [HasVNNI, HasVLX], [HasVNNI]>, PD;12566defm VPDPWSSD : VNNI_common<0x52, "vpdpwssd", X86Vpdpwssd, SchedWriteVecIMul, 1,12567 [HasVNNI, HasVLX], [HasVNNI]>, PD;12568defm VPDPWSSDS : VNNI_common<0x53, "vpdpwssds", X86Vpdpwssds, SchedWriteVecIMul, 1,12569 [HasVNNI, HasVLX], [HasVNNI]>, PD;12570 12571// Patterns to match VPDPWSSD from existing instructions/intrinsics.12572let Predicates = [HasVNNI] in {12573 def : Pat<(v16i32 (add VR512:$src1,12574 (X86vpmaddwd_su VR512:$src2, VR512:$src3))),12575 (VPDPWSSDZrr VR512:$src1, VR512:$src2, VR512:$src3)>;12576 def : Pat<(v16i32 (add VR512:$src1,12577 (X86vpmaddwd_su VR512:$src2, (load addr:$src3)))),12578 (VPDPWSSDZrm VR512:$src1, VR512:$src2, addr:$src3)>;12579}12580let Predicates = [HasVNNI,HasVLX] in {12581 def : Pat<(v8i32 (add VR256X:$src1,12582 (X86vpmaddwd_su VR256X:$src2, VR256X:$src3))),12583 (VPDPWSSDZ256rr VR256X:$src1, VR256X:$src2, VR256X:$src3)>;12584 def : Pat<(v8i32 (add VR256X:$src1,12585 (X86vpmaddwd_su VR256X:$src2, (load addr:$src3)))),12586 (VPDPWSSDZ256rm VR256X:$src1, VR256X:$src2, addr:$src3)>;12587 def : Pat<(v4i32 (add VR128X:$src1,12588 (X86vpmaddwd_su VR128X:$src2, VR128X:$src3))),12589 (VPDPWSSDZ128rr VR128X:$src1, VR128X:$src2, VR128X:$src3)>;12590 def : Pat<(v4i32 (add VR128X:$src1,12591 (X86vpmaddwd_su VR128X:$src2, (load addr:$src3)))),12592 (VPDPWSSDZ128rm VR128X:$src1, VR128X:$src2, addr:$src3)>;12593}12594 12595//===----------------------------------------------------------------------===//12596// Bit Algorithms12597//===----------------------------------------------------------------------===//12598 12599// FIXME: Is there a better scheduler class for VPOPCNTB/VPOPCNTW?12600defm VPOPCNTB : avx512_unary_rm_vl<0x54, "vpopcntb", ctpop, SchedWriteVecALU,12601 avx512vl_i8_info, HasBITALG>;12602defm VPOPCNTW : avx512_unary_rm_vl<0x54, "vpopcntw", ctpop, SchedWriteVecALU,12603 avx512vl_i16_info, HasBITALG>, REX_W;12604 12605defm : avx512_unary_lowering<"VPOPCNTB", ctpop, avx512vl_i8_info, HasBITALG>;12606defm : avx512_unary_lowering<"VPOPCNTW", ctpop, avx512vl_i16_info, HasBITALG>;12607 12608multiclass VPSHUFBITQMB_rm<X86FoldableSchedWrite sched, X86VectorVTInfo VTI> {12609 defm rr : AVX512_maskable_cmp<0x8F, MRMSrcReg, VTI, (outs VTI.KRC:$dst),12610 (ins VTI.RC:$src1, VTI.RC:$src2),12611 "vpshufbitqmb",12612 "$src2, $src1", "$src1, $src2",12613 (X86Vpshufbitqmb (VTI.VT VTI.RC:$src1),12614 (VTI.VT VTI.RC:$src2)),12615 (X86Vpshufbitqmb_su (VTI.VT VTI.RC:$src1),12616 (VTI.VT VTI.RC:$src2))>, EVEX, VVVV, T8, PD,12617 Sched<[sched]>;12618 let mayLoad = 1 in12619 defm rm : AVX512_maskable_cmp<0x8F, MRMSrcMem, VTI, (outs VTI.KRC:$dst),12620 (ins VTI.RC:$src1, VTI.MemOp:$src2),12621 "vpshufbitqmb",12622 "$src2, $src1", "$src1, $src2",12623 (X86Vpshufbitqmb (VTI.VT VTI.RC:$src1),12624 (VTI.VT (VTI.LdFrag addr:$src2))),12625 (X86Vpshufbitqmb_su (VTI.VT VTI.RC:$src1),12626 (VTI.VT (VTI.LdFrag addr:$src2)))>,12627 EVEX, VVVV, EVEX_CD8<8, CD8VF>, T8, PD,12628 Sched<[sched.Folded, sched.ReadAfterFold]>;12629}12630 12631multiclass VPSHUFBITQMB_common<X86SchedWriteWidths sched, AVX512VLVectorVTInfo VTI> {12632 let Predicates = [HasBITALG] in12633 defm Z : VPSHUFBITQMB_rm<sched.ZMM, VTI.info512>, EVEX_V512;12634 let Predicates = [HasBITALG, HasVLX] in {12635 defm Z256 : VPSHUFBITQMB_rm<sched.YMM, VTI.info256>, EVEX_V256;12636 defm Z128 : VPSHUFBITQMB_rm<sched.XMM, VTI.info128>, EVEX_V128;12637 }12638}12639 12640// FIXME: Is there a better scheduler class for VPSHUFBITQMB?12641defm VPSHUFBITQMB : VPSHUFBITQMB_common<SchedWriteVecIMul, avx512vl_i8_info>;12642 12643//===----------------------------------------------------------------------===//12644// GFNI12645//===----------------------------------------------------------------------===//12646 12647multiclass GF2P8MULB_avx512_common<bits<8> Op, string OpStr, SDNode OpNode,12648 X86SchedWriteWidths sched> {12649 let Predicates = [HasGFNI, HasAVX512] in12650 defm Z : avx512_binop_rm<Op, OpStr, OpNode, v64i8_info, sched.ZMM, 1>,12651 EVEX_V512;12652 let Predicates = [HasGFNI, HasVLX] in {12653 defm Z256 : avx512_binop_rm<Op, OpStr, OpNode, v32i8x_info, sched.YMM, 1>,12654 EVEX_V256;12655 defm Z128 : avx512_binop_rm<Op, OpStr, OpNode, v16i8x_info, sched.XMM, 1>,12656 EVEX_V128;12657 }12658}12659 12660defm VGF2P8MULB : GF2P8MULB_avx512_common<0xCF, "vgf2p8mulb", X86GF2P8mulb,12661 SchedWriteVecALU>,12662 EVEX_CD8<8, CD8VF>, T8;12663 12664multiclass GF2P8AFFINE_avx512_rmb_imm<bits<8> Op, string OpStr, SDNode OpNode,12665 X86FoldableSchedWrite sched, X86VectorVTInfo VTI,12666 X86VectorVTInfo BcstVTI>12667 : avx512_3Op_rm_imm8<Op, OpStr, OpNode, sched, VTI, VTI> {12668 let ExeDomain = VTI.ExeDomain, mayLoad = 1 in12669 defm rmbi : AVX512_maskable<Op, MRMSrcMem, VTI, (outs VTI.RC:$dst),12670 (ins VTI.RC:$src1, BcstVTI.ScalarMemOp:$src2, u8imm:$src3),12671 OpStr, "$src3, ${src2}"#BcstVTI.BroadcastStr#", $src1",12672 "$src1, ${src2}"#BcstVTI.BroadcastStr#", $src3",12673 (OpNode (VTI.VT VTI.RC:$src1),12674 (bitconvert (BcstVTI.VT (X86VBroadcastld64 addr:$src2))),12675 (i8 timm:$src3))>, EVEX_B,12676 Sched<[sched.Folded, sched.ReadAfterFold]>;12677}12678 12679multiclass GF2P8AFFINE_avx512_common<bits<8> Op, string OpStr, SDNode OpNode,12680 X86SchedWriteWidths sched> {12681 let Predicates = [HasGFNI, HasAVX512] in12682 defm Z : GF2P8AFFINE_avx512_rmb_imm<Op, OpStr, OpNode, sched.ZMM,12683 v64i8_info, v8i64_info>, EVEX_V512;12684 let Predicates = [HasGFNI, HasVLX] in {12685 defm Z256 : GF2P8AFFINE_avx512_rmb_imm<Op, OpStr, OpNode, sched.YMM,12686 v32i8x_info, v4i64x_info>, EVEX_V256;12687 defm Z128 : GF2P8AFFINE_avx512_rmb_imm<Op, OpStr, OpNode, sched.XMM,12688 v16i8x_info, v2i64x_info>, EVEX_V128;12689 }12690}12691 12692defm VGF2P8AFFINEINVQB : GF2P8AFFINE_avx512_common<0xCF, "vgf2p8affineinvqb",12693 X86GF2P8affineinvqb, SchedWriteVecIMul>,12694 EVEX, VVVV, EVEX_CD8<64, CD8VF>, REX_W, AVX512AIi8Base;12695defm VGF2P8AFFINEQB : GF2P8AFFINE_avx512_common<0xCE, "vgf2p8affineqb",12696 X86GF2P8affineqb, SchedWriteVecIMul>,12697 EVEX, VVVV, EVEX_CD8<64, CD8VF>, REX_W, AVX512AIi8Base;12698 12699 12700//===----------------------------------------------------------------------===//12701// AVX5124FMAPS12702//===----------------------------------------------------------------------===//12703 12704let hasSideEffects = 0, mayLoad = 1, ExeDomain = SSEPackedSingle,12705 Constraints = "$src1 = $dst", Uses = [MXCSR], mayRaiseFPException = 1 in {12706defm V4FMADDPSrm : AVX512_maskable_3src_in_asm<0x9A, MRMSrcMem, v16f32_info,12707 (outs VR512:$dst), (ins VR512:$src2, f128mem:$src3),12708 "v4fmaddps", "$src3, $src2", "$src2, $src3",12709 []>, EVEX_V512, EVEX, VVVV, T8, XD, EVEX_CD8<32, CD8VQ>,12710 Sched<[SchedWriteFMA.ZMM.Folded]>;12711 12712defm V4FNMADDPSrm : AVX512_maskable_3src_in_asm<0xAA, MRMSrcMem, v16f32_info,12713 (outs VR512:$dst), (ins VR512:$src2, f128mem:$src3),12714 "v4fnmaddps", "$src3, $src2", "$src2, $src3",12715 []>, EVEX_V512, EVEX, VVVV, T8, XD, EVEX_CD8<32, CD8VQ>,12716 Sched<[SchedWriteFMA.ZMM.Folded]>;12717 12718defm V4FMADDSSrm : AVX512_maskable_3src_in_asm<0x9B, MRMSrcMem, f32x_info,12719 (outs VR128X:$dst), (ins VR128X:$src2, f128mem:$src3),12720 "v4fmaddss", "$src3, $src2", "$src2, $src3",12721 []>, VEX_LIG, EVEX, VVVV, T8, XD, EVEX_CD8<32, CD8VF>,12722 Sched<[SchedWriteFMA.Scl.Folded]>;12723 12724defm V4FNMADDSSrm : AVX512_maskable_3src_in_asm<0xAB, MRMSrcMem, f32x_info,12725 (outs VR128X:$dst), (ins VR128X:$src2, f128mem:$src3),12726 "v4fnmaddss", "$src3, $src2", "$src2, $src3",12727 []>, VEX_LIG, EVEX, VVVV, T8, XD, EVEX_CD8<32, CD8VF>,12728 Sched<[SchedWriteFMA.Scl.Folded]>;12729}12730 12731//===----------------------------------------------------------------------===//12732// AVX5124VNNIW12733//===----------------------------------------------------------------------===//12734 12735let hasSideEffects = 0, mayLoad = 1, ExeDomain = SSEPackedInt,12736 Constraints = "$src1 = $dst" in {12737defm VP4DPWSSDrm : AVX512_maskable_3src_in_asm<0x52, MRMSrcMem, v16i32_info,12738 (outs VR512:$dst), (ins VR512:$src2, f128mem:$src3),12739 "vp4dpwssd", "$src3, $src2", "$src2, $src3",12740 []>, EVEX_V512, EVEX, VVVV, T8, XD, EVEX_CD8<32, CD8VQ>,12741 Sched<[SchedWriteFMA.ZMM.Folded]>;12742 12743defm VP4DPWSSDSrm : AVX512_maskable_3src_in_asm<0x53, MRMSrcMem, v16i32_info,12744 (outs VR512:$dst), (ins VR512:$src2, f128mem:$src3),12745 "vp4dpwssds", "$src3, $src2", "$src2, $src3",12746 []>, EVEX_V512, EVEX, VVVV, T8, XD, EVEX_CD8<32, CD8VQ>,12747 Sched<[SchedWriteFMA.ZMM.Folded]>;12748}12749 12750let hasSideEffects = 0 in {12751 let mayStore = 1, SchedRW = [WriteFStoreX] in12752 def MASKPAIR16STORE : PseudoI<(outs), (ins anymem:$dst, VK16PAIR:$src), []>;12753 let mayLoad = 1, SchedRW = [WriteFLoadX] in12754 def MASKPAIR16LOAD : PseudoI<(outs VK16PAIR:$dst), (ins anymem:$src), []>;12755}12756 12757//===----------------------------------------------------------------------===//12758// VP2INTERSECT12759//===----------------------------------------------------------------------===//12760 12761multiclass avx512_vp2intersect_modes<X86FoldableSchedWrite sched, X86VectorVTInfo _> {12762 def rr : I<0x68, MRMSrcReg,12763 (outs _.KRPC:$dst),12764 (ins _.RC:$src1, _.RC:$src2),12765 !strconcat("vp2intersect", _.Suffix,12766 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),12767 [(set _.KRPC:$dst, (X86vp2intersect12768 _.RC:$src1, (_.VT _.RC:$src2)))]>,12769 EVEX, VVVV, T8, XD, Sched<[sched]>;12770 12771 let mayLoad = 1 in {12772 def rm : I<0x68, MRMSrcMem,12773 (outs _.KRPC:$dst),12774 (ins _.RC:$src1, _.MemOp:$src2),12775 !strconcat("vp2intersect", _.Suffix,12776 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),12777 [(set _.KRPC:$dst, (X86vp2intersect12778 _.RC:$src1, (_.VT (bitconvert (_.LdFrag addr:$src2)))))]>,12779 EVEX, VVVV, T8, XD, EVEX_CD8<_.EltSize, CD8VF>,12780 Sched<[sched.Folded, sched.ReadAfterFold]>;12781 12782 def rmb : I<0x68, MRMSrcMem,12783 (outs _.KRPC:$dst),12784 (ins _.RC:$src1, _.ScalarMemOp:$src2),12785 !strconcat("vp2intersect", _.Suffix, "\t{${src2}", _.BroadcastStr,12786 ", $src1, $dst|$dst, $src1, ${src2}", _.BroadcastStr ,"}"),12787 [(set _.KRPC:$dst, (X86vp2intersect12788 _.RC:$src1, (_.VT (_.BroadcastLdFrag addr:$src2))))]>,12789 EVEX, VVVV, T8, XD, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>,12790 Sched<[sched.Folded, sched.ReadAfterFold]>;12791 }12792}12793 12794multiclass avx512_vp2intersect<X86SchedWriteWidths sched, AVX512VLVectorVTInfo _> {12795 let Predicates = [HasAVX512, HasVP2INTERSECT] in12796 defm Z : avx512_vp2intersect_modes<sched.ZMM, _.info512>, EVEX_V512;12797 12798 let Predicates = [HasAVX512, HasVP2INTERSECT, HasVLX] in {12799 defm Z256 : avx512_vp2intersect_modes<sched.YMM, _.info256>, EVEX_V256;12800 defm Z128 : avx512_vp2intersect_modes<sched.XMM, _.info128>, EVEX_V128;12801 }12802}12803 12804let ExeDomain = SSEPackedInt in {12805defm VP2INTERSECTD : avx512_vp2intersect<SchedWriteVecALU, avx512vl_i32_info>;12806defm VP2INTERSECTQ : avx512_vp2intersect<SchedWriteVecALU, avx512vl_i64_info>, REX_W;12807}12808 12809let ExeDomain = SSEPackedSingle in12810defm VCVTNE2PS2BF16 : avx512_binop_all<0x72, "vcvtne2ps2bf16",12811 SchedWriteCvtPD2PS, //FIXME: Should be SchedWriteCvtPS2BF12812 avx512vl_f32_info, avx512vl_bf16_info,12813 X86vfpround2, [HasBF16], [HasVLX, HasBF16]>, T8, XD,12814 EVEX_CD8<32, CD8VF>;12815 12816// Truncate Float to BFloat16, Float16 to BF8/HF8[,S]12817multiclass avx512_cvt_trunc_ne<bits<8> opc, string OpcodeStr,12818 AVX512VLVectorVTInfo vt_dst,12819 AVX512VLVectorVTInfo vt_src,12820 X86SchedWriteWidths sched,12821 SDPatternOperator OpNode,12822 SDPatternOperator MaskOpNode,12823 list<Predicate> prds, list<Predicate> prds512,12824 PatFrag bcast128 = vt_src.info128.BroadcastLdFrag,12825 PatFrag loadVT128 = vt_src.info128.LdFrag,12826 RegisterClass maskRC128 = vt_src.info128.KRCWM> {12827 let ExeDomain = SSEPackedSingle in {12828 let Predicates = prds512, Uses = []<Register>, mayRaiseFPException = 0 in {12829 defm Z : avx512_vcvt_fp<opc, OpcodeStr, vt_dst.info256, vt_src.info512,12830 OpNode, OpNode, sched.ZMM>, EVEX_V512;12831 }12832 let Predicates = prds in {12833 let Uses = []<Register>, mayRaiseFPException = 0 in {12834 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, vt_dst.info128, vt_src.info128,12835 null_frag, null_frag, sched.XMM, vt_src.info128.BroadcastStr, "{x}", f128mem,12836 vt_src.info128.KRCWM>, EVEX_V128;12837 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, vt_dst.info128, vt_src.info256,12838 OpNode, OpNode,12839 sched.YMM, vt_src.info256.BroadcastStr, "{y}">, EVEX_V256;12840 }12841 } // Predicates = prds12842 } // ExeDomain = SSEPackedSingle12843 12844 def : InstAlias<OpcodeStr#"x\t{$src, $dst|$dst, $src}",12845 (!cast<Instruction>(NAME # "Z128rr") VR128X:$dst,12846 VR128X:$src), 0>;12847 def : InstAlias<OpcodeStr#"x\t{$src, $dst|$dst, $src}",12848 (!cast<Instruction>(NAME # "Z128rm") VR128X:$dst,12849 f128mem:$src), 0, "intel">;12850 def : InstAlias<OpcodeStr#"y\t{$src, $dst|$dst, $src}",12851 (!cast<Instruction>(NAME # "Z256rr") VR128X:$dst,12852 VR256X:$src), 0>;12853 def : InstAlias<OpcodeStr#"y\t{$src, $dst|$dst, $src}",12854 (!cast<Instruction>(NAME # "Z256rm") VR128X:$dst,12855 f256mem:$src), 0, "intel">;12856 12857 let Predicates = prds in {12858 // Special patterns to allow use of MaskOpNode for masking 128 version. Instruction12859 // patterns have been disabled with null_frag.12860 def : Pat<(vt_dst.info128.VT (OpNode (vt_src.info128.VT VR128X:$src))),12861 (!cast<Instruction>(NAME # "Z128rr") VR128X:$src)>;12862 def : Pat<(MaskOpNode (vt_src.info128.VT VR128X:$src), (vt_dst.info128.VT VR128X:$src0),12863 maskRC128:$mask),12864 (!cast<Instruction>(NAME # "Z128rrk") VR128X:$src0, maskRC128:$mask, VR128X:$src)>;12865 def : Pat<(MaskOpNode (vt_src.info128.VT VR128X:$src), vt_dst.info128.ImmAllZerosV,12866 maskRC128:$mask),12867 (!cast<Instruction>(NAME # "Z128rrkz") maskRC128:$mask, VR128X:$src)>;12868 12869 def : Pat<(vt_dst.info128.VT (OpNode (loadVT128 addr:$src))),12870 (!cast<Instruction>(NAME # "Z128rm") addr:$src)>;12871 def : Pat<(MaskOpNode (loadVT128 addr:$src), (vt_dst.info128.VT VR128X:$src0),12872 maskRC128:$mask),12873 (!cast<Instruction>(NAME # "Z128rmk") VR128X:$src0, maskRC128:$mask, addr:$src)>;12874 def : Pat<(MaskOpNode (loadVT128 addr:$src), vt_dst.info128.ImmAllZerosV,12875 maskRC128:$mask),12876 (!cast<Instruction>(NAME # "Z128rmkz") maskRC128:$mask, addr:$src)>;12877 12878 def : Pat<(vt_dst.info128.VT (OpNode (vt_src.info128.VT (bcast128 addr:$src)))),12879 (!cast<Instruction>(NAME # "Z128rmb") addr:$src)>;12880 def : Pat<(MaskOpNode (vt_src.info128.VT (bcast128 addr:$src)),12881 (vt_dst.info128.VT VR128X:$src0), maskRC128:$mask),12882 (!cast<Instruction>(NAME # "Z128rmbk") VR128X:$src0, maskRC128:$mask, addr:$src)>;12883 def : Pat<(MaskOpNode (vt_src.info128.VT (bcast128 addr:$src)),12884 vt_dst.info128.ImmAllZerosV, maskRC128:$mask),12885 (!cast<Instruction>(NAME # "Z128rmbkz") maskRC128:$mask, addr:$src)>;12886 }12887}12888 12889defm VCVTNEPS2BF16 : avx512_cvt_trunc_ne<0x72, "vcvtneps2bf16",12890 avx512vl_bf16_info, avx512vl_f32_info,12891 SchedWriteCvtPD2PS, X86cvtneps2bf16,12892 X86mcvtneps2bf16, [HasBF16, HasVLX],12893 [HasBF16]>, T8, XS, EVEX_CD8<32, CD8VF>;12894 12895let Predicates = [HasBF16, HasVLX] in {12896 def : Pat<(v8bf16 (int_x86_vcvtneps2bf16128 (v4f32 VR128X:$src))),12897 (VCVTNEPS2BF16Z128rr VR128X:$src)>;12898 def : Pat<(v8bf16 (int_x86_vcvtneps2bf16128 (loadv4f32 addr:$src))),12899 (VCVTNEPS2BF16Z128rm addr:$src)>;12900 12901 def : Pat<(v8bf16 (int_x86_vcvtneps2bf16256 (v8f32 VR256X:$src))),12902 (VCVTNEPS2BF16Z256rr VR256X:$src)>;12903 def : Pat<(v8bf16 (int_x86_vcvtneps2bf16256 (loadv8f32 addr:$src))),12904 (VCVTNEPS2BF16Z256rm addr:$src)>;12905 12906 def : Pat<(v8bf16 (X86VBroadcastld16 addr:$src)),12907 (VPBROADCASTWZ128rm addr:$src)>;12908 def : Pat<(v16bf16 (X86VBroadcastld16 addr:$src)),12909 (VPBROADCASTWZ256rm addr:$src)>;12910 12911 def : Pat<(v8bf16 (X86VBroadcast (v8bf16 VR128X:$src))),12912 (VPBROADCASTWZ128rr VR128X:$src)>;12913 def : Pat<(v16bf16 (X86VBroadcast (v8bf16 VR128X:$src))),12914 (VPBROADCASTWZ256rr VR128X:$src)>;12915 12916 def : Pat<(v8bf16 (X86vfpround (v8f32 VR256X:$src))),12917 (VCVTNEPS2BF16Z256rr VR256X:$src)>;12918 def : Pat<(v8bf16 (X86vfpround (loadv8f32 addr:$src))),12919 (VCVTNEPS2BF16Z256rm addr:$src)>;12920 12921 // TODO: No scalar broadcast due to we don't support legal scalar bf16 so far.12922}12923 12924let Predicates = [HasBF16] in {12925 def : Pat<(v32bf16 (X86VBroadcastld16 addr:$src)),12926 (VPBROADCASTWZrm addr:$src)>;12927 12928 def : Pat<(v32bf16 (X86VBroadcast (v8bf16 VR128X:$src))),12929 (VPBROADCASTWZrr VR128X:$src)>;12930 12931 def : Pat<(v16bf16 (X86vfpround (v16f32 VR512:$src))),12932 (VCVTNEPS2BF16Zrr VR512:$src)>;12933 def : Pat<(v16bf16 (X86vfpround (loadv16f32 addr:$src))),12934 (VCVTNEPS2BF16Zrm addr:$src)>;12935 // TODO: No scalar broadcast due to we don't support legal scalar bf16 so far.12936}12937 12938let Constraints = "$src1 = $dst" in {12939multiclass avx512_dpf16ps_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,12940 X86FoldableSchedWrite sched,12941 X86VectorVTInfo _, X86VectorVTInfo src_v> {12942 defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),12943 (ins src_v.RC:$src2, src_v.RC:$src3),12944 OpcodeStr, "$src3, $src2", "$src2, $src3",12945 (_.VT (OpNode _.RC:$src1, src_v.RC:$src2, src_v.RC:$src3))>,12946 EVEX, VVVV, Sched<[sched]>;12947 12948 defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),12949 (ins src_v.RC:$src2, src_v.MemOp:$src3),12950 OpcodeStr, "$src3, $src2", "$src2, $src3",12951 (_.VT (OpNode _.RC:$src1, src_v.RC:$src2,12952 (src_v.LdFrag addr:$src3)))>, EVEX, VVVV,12953 Sched<[sched.Folded, sched.ReadAfterFold]>;12954 12955 let mayLoad = 1, hasSideEffects = 0 in12956 defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),12957 (ins src_v.RC:$src2, f32mem:$src3),12958 OpcodeStr,12959 !strconcat("${src3}", _.BroadcastStr,", $src2"),12960 !strconcat("$src2, ${src3}", _.BroadcastStr),12961 (null_frag)>,12962 EVEX_B, EVEX, VVVV, Sched<[sched.Folded, sched.ReadAfterFold]>;12963 12964}12965} // Constraints = "$src1 = $dst"12966 12967multiclass avx512_dpf16ps_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,12968 AVX512VLVectorVTInfo _, list<Predicate> prds,12969 list<Predicate> prds512> {12970 let Predicates = prds512 in {12971 defm Z : avx512_dpf16ps_rm<opc, OpcodeStr, OpNode, WriteFMAZ,12972 avx512vl_f32_info.info512, _.info512>, EVEX_V512;12973 }12974 let Predicates = prds in {12975 defm Z256 : avx512_dpf16ps_rm<opc, OpcodeStr, OpNode, WriteFMAY,12976 v8f32x_info, _.info256>, EVEX_V256;12977 defm Z128 : avx512_dpf16ps_rm<opc, OpcodeStr, OpNode, WriteFMAX,12978 v4f32x_info, _.info128>, EVEX_V128;12979 }12980}12981 12982let ExeDomain = SSEPackedSingle in12983defm VDPBF16PS : avx512_dpf16ps_sizes<0x52, "vdpbf16ps", X86dpbf16ps, avx512vl_bf16_info,12984 [HasVLX, HasBF16], [HasBF16]>,12985 T8, XS, EVEX_CD8<32, CD8VF>;12986 12987//===----------------------------------------------------------------------===//12988// AVX512FP1612989//===----------------------------------------------------------------------===//12990 12991let Predicates = [HasFP16] in {12992// Move word ( r/m16) to Packed word12993def VMOVW2SHrr : AVX512<0x6E, MRMSrcReg, (outs VR128X:$dst), (ins GR32:$src),12994 "vmovw\t{$src, $dst|$dst, $src}", []>, T_MAP5, PD, EVEX, Sched<[WriteVecMoveFromGpr]>;12995let mayLoad = 1 in12996def VMOVWrm : AVX512<0x6E, MRMSrcMem, (outs VR128X:$dst), (ins i16mem:$src),12997 "vmovw\t{$src, $dst|$dst, $src}",12998 [(set VR128X:$dst,12999 (v8i16 (scalar_to_vector (loadi16 addr:$src))))]>,13000 T_MAP5, PD, EVEX, EVEX_CD8<16, CD8VT1>, Sched<[WriteFLoad]>;13001 13002def : Pat<(f16 (bitconvert GR16:$src)),13003 (f16 (COPY_TO_REGCLASS13004 (VMOVW2SHrr13005 (INSERT_SUBREG (IMPLICIT_DEF), GR16:$src, sub_16bit)),13006 FR16X))>;13007def : Pat<(v8i16 (scalar_to_vector (i16 GR16:$src))),13008 (VMOVW2SHrr (INSERT_SUBREG (IMPLICIT_DEF), GR16:$src, sub_16bit))>;13009def : Pat<(v4i32 (X86vzmovl (scalar_to_vector (and GR32:$src, 0xffff)))),13010 (VMOVW2SHrr GR32:$src)>;13011// FIXME: We should really find a way to improve these patterns.13012def : Pat<(v8i32 (X86vzmovl13013 (insert_subvector undef,13014 (v4i32 (scalar_to_vector13015 (and GR32:$src, 0xffff))),13016 (iPTR 0)))),13017 (SUBREG_TO_REG (i32 0), (VMOVW2SHrr GR32:$src), sub_xmm)>;13018def : Pat<(v16i32 (X86vzmovl13019 (insert_subvector undef,13020 (v4i32 (scalar_to_vector13021 (and GR32:$src, 0xffff))),13022 (iPTR 0)))),13023 (SUBREG_TO_REG (i32 0), (VMOVW2SHrr GR32:$src), sub_xmm)>;13024 13025def : Pat<(v8i16 (X86vzmovl (scalar_to_vector (i16 GR16:$src)))),13026 (VMOVW2SHrr (INSERT_SUBREG (IMPLICIT_DEF), GR16:$src, sub_16bit))>;13027 13028// AVX 128-bit movw instruction write zeros in the high 128-bit part.13029def : Pat<(v8i16 (X86vzload16 addr:$src)),13030 (VMOVWrm addr:$src)>;13031def : Pat<(v16i16 (X86vzload16 addr:$src)),13032 (SUBREG_TO_REG (i32 0), (v8i16 (VMOVWrm addr:$src)), sub_xmm)>;13033 13034// Use regular 128-bit instructions to match 512-bit scalar_to_vec+zext.13035def : Pat<(v32i16 (X86vzload16 addr:$src)),13036 (SUBREG_TO_REG (i32 0), (v8i16 (VMOVWrm addr:$src)), sub_xmm)>;13037 13038def : Pat<(v4i32 (scalar_to_vector (i32 (extloadi16 addr:$src)))),13039 (VMOVWrm addr:$src)>;13040def : Pat<(v4i32 (X86vzmovl (scalar_to_vector (i32 (zextloadi16 addr:$src))))),13041 (VMOVWrm addr:$src)>;13042def : Pat<(v8i32 (X86vzmovl13043 (insert_subvector undef,13044 (v4i32 (scalar_to_vector13045 (i32 (zextloadi16 addr:$src)))),13046 (iPTR 0)))),13047 (SUBREG_TO_REG (i32 0), (VMOVWrm addr:$src), sub_xmm)>;13048def : Pat<(v16i32 (X86vzmovl13049 (insert_subvector undef,13050 (v4i32 (scalar_to_vector13051 (i32 (zextloadi16 addr:$src)))),13052 (iPTR 0)))),13053 (SUBREG_TO_REG (i32 0), (VMOVWrm addr:$src), sub_xmm)>;13054 13055// Move word from xmm register to r/m1613056def VMOVSH2Wrr : AVX512<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128X:$src),13057 "vmovw\t{$src, $dst|$dst, $src}", []>, T_MAP5, PD, EVEX, Sched<[WriteVecMoveToGpr]>;13058def VMOVWmr : AVX512<0x7E, MRMDestMem, (outs),13059 (ins i16mem:$dst, VR128X:$src),13060 "vmovw\t{$src, $dst|$dst, $src}",13061 [(store (i16 (extractelt (v8i16 VR128X:$src),13062 (iPTR 0))), addr:$dst)]>,13063 T_MAP5, PD, EVEX, EVEX_CD8<16, CD8VT1>, Sched<[WriteFStore]>;13064 13065def : Pat<(i16 (bitconvert FR16X:$src)),13066 (i16 (EXTRACT_SUBREG13067 (VMOVSH2Wrr (COPY_TO_REGCLASS FR16X:$src, VR128X)),13068 sub_16bit))>;13069def : Pat<(i16 (extractelt (v8i16 VR128X:$src), (iPTR 0))),13070 (i16 (EXTRACT_SUBREG (VMOVSH2Wrr VR128X:$src), sub_16bit))>;13071 13072// Allow "vmovw" to use GR6413073let hasSideEffects = 0 in {13074 def VMOVW64toSHrr : AVX512<0x6E, MRMSrcReg, (outs VR128X:$dst), (ins GR64:$src),13075 "vmovw\t{$src, $dst|$dst, $src}", []>, T_MAP5, PD, EVEX, REX_W, Sched<[WriteVecMoveFromGpr]>;13076 def VMOVSHtoW64rr : AVX512<0x7E, MRMDestReg, (outs GR64:$dst), (ins VR128X:$src),13077 "vmovw\t{$src, $dst|$dst, $src}", []>, T_MAP5, PD, EVEX, REX_W, Sched<[WriteVecMoveToGpr]>;13078}13079}13080 13081// Convert 16-bit float to i16/u1613082multiclass avx512_cvtph2w<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,13083 SDPatternOperator MaskOpNode, SDNode OpNodeRnd,13084 AVX512VLVectorVTInfo _Dst,13085 AVX512VLVectorVTInfo _Src,13086 X86SchedWriteWidths sched> {13087 let Predicates = [HasFP16] in {13088 defm Z : avx512_vcvt_fp<opc, OpcodeStr, _Dst.info512, _Src.info512,13089 OpNode, MaskOpNode, sched.ZMM>,13090 avx512_vcvt_fp_rc<opc, OpcodeStr, _Dst.info512, _Src.info512,13091 OpNodeRnd, sched.ZMM>, EVEX_V512;13092 }13093 let Predicates = [HasFP16, HasVLX] in {13094 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, _Dst.info128, _Src.info128,13095 OpNode, MaskOpNode, sched.XMM>, EVEX_V128;13096 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, _Dst.info256, _Src.info256,13097 OpNode, MaskOpNode, sched.YMM>, EVEX_V256;13098 }13099}13100 13101// Convert 16-bit float to i16/u16 truncate13102multiclass avx512_cvttph2w<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,13103 SDPatternOperator MaskOpNode, SDNode OpNodeRnd,13104 AVX512VLVectorVTInfo _Dst, AVX512VLVectorVTInfo _Src,13105 X86SchedWriteWidths sched> {13106 let Predicates = [HasFP16] in {13107 defm Z : avx512_vcvt_fp<opc, OpcodeStr, _Dst.info512, _Src.info512,13108 OpNode, MaskOpNode, sched.ZMM>,13109 avx512_vcvt_fp_sae<opc, OpcodeStr, _Dst.info512, _Src.info512,13110 OpNodeRnd, sched.ZMM>, EVEX_V512;13111 }13112 let Predicates = [HasFP16, HasVLX] in {13113 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, _Dst.info128, _Src.info128,13114 OpNode, MaskOpNode, sched.XMM>, EVEX_V128;13115 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, _Dst.info256, _Src.info256,13116 OpNode, MaskOpNode, sched.YMM>, EVEX_V256;13117 }13118}13119 13120defm VCVTPH2UW : avx512_cvtph2w<0x7D, "vcvtph2uw", X86cvtp2UInt, X86cvtp2UInt,13121 X86cvtp2UIntRnd, avx512vl_i16_info,13122 avx512vl_f16_info, SchedWriteCvtPD2DQ>,13123 T_MAP5, EVEX_CD8<16, CD8VF>;13124defm VCVTUW2PH : avx512_cvtph2w<0x7D, "vcvtuw2ph", any_uint_to_fp, uint_to_fp,13125 X86VUintToFpRnd, avx512vl_f16_info,13126 avx512vl_i16_info, SchedWriteCvtPD2DQ>,13127 T_MAP5, XD, EVEX_CD8<16, CD8VF>;13128defm VCVTTPH2W : avx512_cvttph2w<0x7C, "vcvttph2w", X86any_cvttp2si,13129 X86cvttp2si, X86cvttp2siSAE,13130 avx512vl_i16_info, avx512vl_f16_info,13131 SchedWriteCvtPD2DQ>, T_MAP5, PD, EVEX_CD8<16, CD8VF>;13132defm VCVTTPH2UW : avx512_cvttph2w<0x7C, "vcvttph2uw", X86any_cvttp2ui,13133 X86cvttp2ui, X86cvttp2uiSAE,13134 avx512vl_i16_info, avx512vl_f16_info,13135 SchedWriteCvtPD2DQ>, T_MAP5, EVEX_CD8<16, CD8VF>;13136defm VCVTPH2W : avx512_cvtph2w<0x7D, "vcvtph2w", X86cvtp2Int, X86cvtp2Int,13137 X86cvtp2IntRnd, avx512vl_i16_info,13138 avx512vl_f16_info, SchedWriteCvtPD2DQ>,13139 T_MAP5, PD, EVEX_CD8<16, CD8VF>;13140defm VCVTW2PH : avx512_cvtph2w<0x7D, "vcvtw2ph", any_sint_to_fp, sint_to_fp,13141 X86VSintToFpRnd, avx512vl_f16_info,13142 avx512vl_i16_info, SchedWriteCvtPD2DQ>,13143 T_MAP5, XS, EVEX_CD8<16, CD8VF>;13144 13145// Convert Half to Signed/Unsigned Doubleword13146multiclass avx512_cvtph2dq<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,13147 SDPatternOperator MaskOpNode, SDNode OpNodeRnd,13148 X86SchedWriteWidths sched> {13149 let Predicates = [HasFP16] in {13150 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16i32_info, v16f16x_info, OpNode,13151 MaskOpNode, sched.ZMM>,13152 avx512_vcvt_fp_rc<opc, OpcodeStr, v16i32_info, v16f16x_info,13153 OpNodeRnd, sched.ZMM>, EVEX_V512;13154 }13155 let Predicates = [HasFP16, HasVLX] in {13156 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v8f16x_info, OpNode,13157 MaskOpNode, sched.XMM, "{1to4}", "", f64mem>, EVEX_V128;13158 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f16x_info, OpNode,13159 MaskOpNode, sched.YMM>, EVEX_V256;13160 }13161}13162 13163// Convert Half to Signed/Unsigned Doubleword with truncation13164multiclass avx512_cvttph2dq<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,13165 SDPatternOperator MaskOpNode, SDNode OpNodeRnd,13166 X86SchedWriteWidths sched> {13167 let Predicates = [HasFP16] in {13168 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16i32_info, v16f16x_info, OpNode,13169 MaskOpNode, sched.ZMM>,13170 avx512_vcvt_fp_sae<opc, OpcodeStr, v16i32_info, v16f16x_info,13171 OpNodeRnd, sched.ZMM>, EVEX_V512;13172 }13173 let Predicates = [HasFP16, HasVLX] in {13174 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v8f16x_info, OpNode,13175 MaskOpNode, sched.XMM, "{1to4}", "", f64mem>, EVEX_V128;13176 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f16x_info, OpNode,13177 MaskOpNode, sched.YMM>, EVEX_V256;13178 }13179}13180 13181 13182defm VCVTPH2DQ : avx512_cvtph2dq<0x5B, "vcvtph2dq", X86cvtp2Int, X86cvtp2Int,13183 X86cvtp2IntRnd, SchedWriteCvtPS2DQ>, T_MAP5, PD,13184 EVEX_CD8<16, CD8VH>;13185defm VCVTPH2UDQ : avx512_cvtph2dq<0x79, "vcvtph2udq", X86cvtp2UInt, X86cvtp2UInt,13186 X86cvtp2UIntRnd, SchedWriteCvtPS2DQ>, T_MAP5,13187 EVEX_CD8<16, CD8VH>;13188 13189defm VCVTTPH2DQ : avx512_cvttph2dq<0x5B, "vcvttph2dq", X86any_cvttp2si,13190 X86cvttp2si, X86cvttp2siSAE,13191 SchedWriteCvtPS2DQ>, T_MAP5, XS,13192 EVEX_CD8<16, CD8VH>;13193 13194defm VCVTTPH2UDQ : avx512_cvttph2dq<0x78, "vcvttph2udq", X86any_cvttp2ui,13195 X86cvttp2ui, X86cvttp2uiSAE,13196 SchedWriteCvtPS2DQ>, T_MAP5,13197 EVEX_CD8<16, CD8VH>;13198 13199// Convert Half to Signed/Unsigned Quardword13200multiclass avx512_cvtph2qq<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,13201 SDPatternOperator MaskOpNode, SDNode OpNodeRnd,13202 X86SchedWriteWidths sched> {13203 let Predicates = [HasFP16] in {13204 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f16x_info, OpNode,13205 MaskOpNode, sched.ZMM>,13206 avx512_vcvt_fp_rc<opc, OpcodeStr, v8i64_info, v8f16x_info,13207 OpNodeRnd, sched.ZMM>, EVEX_V512;13208 }13209 let Predicates = [HasFP16, HasVLX] in {13210 // Explicitly specified broadcast string, since we take only 2 elements13211 // from v8f16x_info source13212 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v8f16x_info, OpNode,13213 MaskOpNode, sched.XMM, "{1to2}", "", f32mem>,13214 EVEX_V128;13215 // Explicitly specified broadcast string, since we take only 4 elements13216 // from v8f16x_info source13217 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v8f16x_info, OpNode,13218 MaskOpNode, sched.YMM, "{1to4}", "", f64mem>,13219 EVEX_V256;13220 }13221}13222 13223// Convert Half to Signed/Unsigned Quardword with truncation13224multiclass avx512_cvttph2qq<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,13225 SDPatternOperator MaskOpNode, SDNode OpNodeRnd,13226 X86SchedWriteWidths sched> {13227 let Predicates = [HasFP16] in {13228 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f16x_info, OpNode,13229 MaskOpNode, sched.ZMM>,13230 avx512_vcvt_fp_sae<opc, OpcodeStr, v8i64_info, v8f16x_info,13231 OpNodeRnd, sched.ZMM>, EVEX_V512;13232 }13233 let Predicates = [HasFP16, HasVLX] in {13234 // Explicitly specified broadcast string, since we take only 2 elements13235 // from v8f16x_info source13236 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v8f16x_info, OpNode,13237 MaskOpNode, sched.XMM, "{1to2}", "", f32mem>, EVEX_V128;13238 // Explicitly specified broadcast string, since we take only 4 elements13239 // from v8f16x_info source13240 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v8f16x_info, OpNode,13241 MaskOpNode, sched.YMM, "{1to4}", "", f64mem>, EVEX_V256;13242 }13243}13244 13245defm VCVTPH2QQ : avx512_cvtph2qq<0x7B, "vcvtph2qq", X86cvtp2Int, X86cvtp2Int,13246 X86cvtp2IntRnd, SchedWriteCvtPS2DQ>, T_MAP5, PD,13247 EVEX_CD8<16, CD8VQ>;13248 13249defm VCVTPH2UQQ : avx512_cvtph2qq<0x79, "vcvtph2uqq", X86cvtp2UInt, X86cvtp2UInt,13250 X86cvtp2UIntRnd, SchedWriteCvtPS2DQ>, T_MAP5, PD,13251 EVEX_CD8<16, CD8VQ>;13252 13253defm VCVTTPH2QQ : avx512_cvttph2qq<0x7A, "vcvttph2qq", X86any_cvttp2si,13254 X86cvttp2si, X86cvttp2siSAE,13255 SchedWriteCvtPS2DQ>, T_MAP5, PD,13256 EVEX_CD8<16, CD8VQ>;13257 13258defm VCVTTPH2UQQ : avx512_cvttph2qq<0x78, "vcvttph2uqq", X86any_cvttp2ui,13259 X86cvttp2ui, X86cvttp2uiSAE,13260 SchedWriteCvtPS2DQ>, T_MAP5, PD,13261 EVEX_CD8<16, CD8VQ>;13262 13263let Predicates = [HasFP16, HasVLX] in {13264 def : Pat<(v8i16 (lrint (v8f16 VR128X:$src))), (VCVTPH2WZ128rr VR128X:$src)>;13265 def : Pat<(v8i16 (lrint (loadv8f16 addr:$src))), (VCVTPH2WZ128rm addr:$src)>;13266 def : Pat<(v16i16 (lrint (v16f16 VR256X:$src))), (VCVTPH2WZ256rr VR256X:$src)>;13267 def : Pat<(v16i16 (lrint (loadv16f16 addr:$src))), (VCVTPH2WZ256rm addr:$src)>;13268 def : Pat<(v8i32 (lrint (v8f16 VR128X:$src))), (VCVTPH2DQZ256rr VR128X:$src)>;13269 def : Pat<(v8i32 (lrint (loadv8f16 addr:$src))), (VCVTPH2DQZ256rm addr:$src)>;13270}13271 13272let Predicates = [HasFP16] in {13273 def : Pat<(v32i16 (lrint (v32f16 VR512:$src))), (VCVTPH2WZrr VR512:$src)>;13274 def : Pat<(v32i16 (lrint (loadv32f16 addr:$src))), (VCVTPH2WZrm addr:$src)>;13275 def : Pat<(v16i32 (lrint (v16f16 VR256X:$src))), (VCVTPH2DQZrr VR256X:$src)>;13276 def : Pat<(v16i32 (lrint (loadv16f16 addr:$src))), (VCVTPH2DQZrm addr:$src)>;13277 def : Pat<(v8i64 (lrint (v8f16 VR128X:$src))), (VCVTPH2QQZrr VR128X:$src)>;13278 def : Pat<(v8i64 (lrint (loadv8f16 addr:$src))), (VCVTPH2QQZrm addr:$src)>;13279 def : Pat<(v8i64 (llrint (v8f16 VR128X:$src))), (VCVTPH2QQZrr VR128X:$src)>;13280 def : Pat<(v8i64 (llrint (loadv8f16 addr:$src))), (VCVTPH2QQZrm addr:$src)>;13281}13282 13283// Convert Signed/Unsigned Quardword to Half13284multiclass avx512_cvtqq2ph<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,13285 SDPatternOperator MaskOpNode, SDNode OpNodeRnd,13286 X86SchedWriteWidths sched> {13287 // we need "x"/"y"/"z" suffixes in order to distinguish between 128, 256 and13288 // 512 memory forms of these instructions in Asm Parcer. They have the same13289 // dest type - 'v8f16x_info'. We also specify the broadcast string explicitly13290 // due to the same reason.13291 let Predicates = [HasFP16] in {13292 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f16x_info, v8i64_info, OpNode,13293 MaskOpNode, sched.ZMM, "{1to8}", "{z}">,13294 avx512_vcvt_fp_rc<opc, OpcodeStr, v8f16x_info, v8i64_info,13295 OpNodeRnd, sched.ZMM>, EVEX_V512;13296 }13297 let Predicates = [HasFP16, HasVLX] in {13298 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v8f16x_info, v2i64x_info,13299 null_frag, null_frag, sched.XMM, "{1to2}", "{x}",13300 i128mem, VK2WM>, EVEX_V128;13301 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8f16x_info, v4i64x_info,13302 null_frag, null_frag, sched.YMM, "{1to4}", "{y}",13303 i256mem, VK4WM>, EVEX_V256;13304 }13305 13306 def : InstAlias<OpcodeStr#"x\t{$src, $dst|$dst, $src}",13307 (!cast<Instruction>(NAME # "Z128rr") VR128X:$dst,13308 VR128X:$src), 0, "att">;13309 def : InstAlias<OpcodeStr#"x\t{$src, $dst {${mask}}|$dst {${mask}}, $src}",13310 (!cast<Instruction>(NAME # "Z128rrk") VR128X:$dst,13311 VK2WM:$mask, VR128X:$src), 0, "att">;13312 def : InstAlias<OpcodeStr#"x\t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}",13313 (!cast<Instruction>(NAME # "Z128rrkz") VR128X:$dst,13314 VK2WM:$mask, VR128X:$src), 0, "att">;13315 def : InstAlias<OpcodeStr#"x\t{${src}{1to2}, $dst|$dst, ${src}{1to2}}",13316 (!cast<Instruction>(NAME # "Z128rmb") VR128X:$dst,13317 i64mem:$src), 0, "att">;13318 def : InstAlias<OpcodeStr#"x\t{${src}{1to2}, $dst {${mask}}|"13319 "$dst {${mask}}, ${src}{1to2}}",13320 (!cast<Instruction>(NAME # "Z128rmbk") VR128X:$dst,13321 VK2WM:$mask, i64mem:$src), 0, "att">;13322 def : InstAlias<OpcodeStr#"x\t{${src}{1to2}, $dst {${mask}} {z}|"13323 "$dst {${mask}} {z}, ${src}{1to2}}",13324 (!cast<Instruction>(NAME # "Z128rmbkz") VR128X:$dst,13325 VK2WM:$mask, i64mem:$src), 0, "att">;13326 13327 def : InstAlias<OpcodeStr#"y\t{$src, $dst|$dst, $src}",13328 (!cast<Instruction>(NAME # "Z256rr") VR128X:$dst,13329 VR256X:$src), 0, "att">;13330 def : InstAlias<OpcodeStr#"y\t{$src, $dst {${mask}}|"13331 "$dst {${mask}}, $src}",13332 (!cast<Instruction>(NAME # "Z256rrk") VR128X:$dst,13333 VK4WM:$mask, VR256X:$src), 0, "att">;13334 def : InstAlias<OpcodeStr#"y\t{$src, $dst {${mask}} {z}|"13335 "$dst {${mask}} {z}, $src}",13336 (!cast<Instruction>(NAME # "Z256rrkz") VR128X:$dst,13337 VK4WM:$mask, VR256X:$src), 0, "att">;13338 def : InstAlias<OpcodeStr#"y\t{${src}{1to4}, $dst|$dst, ${src}{1to4}}",13339 (!cast<Instruction>(NAME # "Z256rmb") VR128X:$dst,13340 i64mem:$src), 0, "att">;13341 def : InstAlias<OpcodeStr#"y\t{${src}{1to4}, $dst {${mask}}|"13342 "$dst {${mask}}, ${src}{1to4}}",13343 (!cast<Instruction>(NAME # "Z256rmbk") VR128X:$dst,13344 VK4WM:$mask, i64mem:$src), 0, "att">;13345 def : InstAlias<OpcodeStr#"y\t{${src}{1to4}, $dst {${mask}} {z}|"13346 "$dst {${mask}} {z}, ${src}{1to4}}",13347 (!cast<Instruction>(NAME # "Z256rmbkz") VR128X:$dst,13348 VK4WM:$mask, i64mem:$src), 0, "att">;13349 13350 def : InstAlias<OpcodeStr#"z\t{$src, $dst|$dst, $src}",13351 (!cast<Instruction>(NAME # "Zrr") VR128X:$dst,13352 VR512:$src), 0, "att">;13353 def : InstAlias<OpcodeStr#"z\t{$src, $dst {${mask}}|"13354 "$dst {${mask}}, $src}",13355 (!cast<Instruction>(NAME # "Zrrk") VR128X:$dst,13356 VK8WM:$mask, VR512:$src), 0, "att">;13357 def : InstAlias<OpcodeStr#"z\t{$src, $dst {${mask}} {z}|"13358 "$dst {${mask}} {z}, $src}",13359 (!cast<Instruction>(NAME # "Zrrkz") VR128X:$dst,13360 VK8WM:$mask, VR512:$src), 0, "att">;13361 def : InstAlias<OpcodeStr#"z\t{${src}{1to8}, $dst|$dst, ${src}{1to8}}",13362 (!cast<Instruction>(NAME # "Zrmb") VR128X:$dst,13363 i64mem:$src), 0, "att">;13364 def : InstAlias<OpcodeStr#"z\t{${src}{1to8}, $dst {${mask}}|"13365 "$dst {${mask}}, ${src}{1to8}}",13366 (!cast<Instruction>(NAME # "Zrmbk") VR128X:$dst,13367 VK8WM:$mask, i64mem:$src), 0, "att">;13368 def : InstAlias<OpcodeStr#"z\t{${src}{1to8}, $dst {${mask}} {z}|"13369 "$dst {${mask}} {z}, ${src}{1to8}}",13370 (!cast<Instruction>(NAME # "Zrmbkz") VR128X:$dst,13371 VK8WM:$mask, i64mem:$src), 0, "att">;13372}13373 13374defm VCVTQQ2PH : avx512_cvtqq2ph<0x5B, "vcvtqq2ph", any_sint_to_fp, sint_to_fp,13375 X86VSintToFpRnd, SchedWriteCvtDQ2PS>, REX_W, T_MAP5,13376 EVEX_CD8<64, CD8VF>;13377 13378defm VCVTUQQ2PH : avx512_cvtqq2ph<0x7A, "vcvtuqq2ph", any_uint_to_fp, uint_to_fp,13379 X86VUintToFpRnd, SchedWriteCvtDQ2PS>, REX_W, T_MAP5, XD,13380 EVEX_CD8<64, CD8VF>;13381 13382// Convert half to signed/unsigned int 32/6413383defm VCVTSH2SIZ: avx512_cvt_s_int_round<0x2D, f16x_info, i32x_info, X86cvts2si,13384 X86cvts2siRnd, WriteCvtSS2I, "cvtsh2si", "{l}", HasFP16>,13385 T_MAP5, XS, EVEX_CD8<16, CD8VT1>;13386defm VCVTSH2SI64Z: avx512_cvt_s_int_round<0x2D, f16x_info, i64x_info, X86cvts2si,13387 X86cvts2siRnd, WriteCvtSS2I, "cvtsh2si", "{q}", HasFP16>,13388 T_MAP5, XS, REX_W, EVEX_CD8<16, CD8VT1>;13389defm VCVTSH2USIZ: avx512_cvt_s_int_round<0x79, f16x_info, i32x_info, X86cvts2usi,13390 X86cvts2usiRnd, WriteCvtSS2I, "cvtsh2usi", "{l}", HasFP16>,13391 T_MAP5, XS, EVEX_CD8<16, CD8VT1>;13392defm VCVTSH2USI64Z: avx512_cvt_s_int_round<0x79, f16x_info, i64x_info, X86cvts2usi,13393 X86cvts2usiRnd, WriteCvtSS2I, "cvtsh2usi", "{q}", HasFP16>,13394 T_MAP5, XS, REX_W, EVEX_CD8<16, CD8VT1>;13395 13396defm VCVTTSH2SIZ: avx512_cvt_s_all<0x2C, "vcvttsh2si", f16x_info, i32x_info,13397 any_fp_to_sint, X86cvtts2Int, X86cvtts2IntSAE, WriteCvtSS2I,13398 "{l}", HasFP16>, T_MAP5, XS, EVEX_CD8<16, CD8VT1>;13399defm VCVTTSH2SI64Z: avx512_cvt_s_all<0x2C, "vcvttsh2si", f16x_info, i64x_info,13400 any_fp_to_sint, X86cvtts2Int, X86cvtts2IntSAE, WriteCvtSS2I,13401 "{q}", HasFP16>, REX_W, T_MAP5, XS, EVEX_CD8<16, CD8VT1>;13402defm VCVTTSH2USIZ: avx512_cvt_s_all<0x78, "vcvttsh2usi", f16x_info, i32x_info,13403 any_fp_to_uint, X86cvtts2UInt, X86cvtts2UIntSAE, WriteCvtSS2I,13404 "{l}", HasFP16>, T_MAP5, XS, EVEX_CD8<16, CD8VT1>;13405defm VCVTTSH2USI64Z: avx512_cvt_s_all<0x78, "vcvttsh2usi", f16x_info, i64x_info,13406 any_fp_to_uint, X86cvtts2UInt, X86cvtts2UIntSAE, WriteCvtSS2I,13407 "{q}", HasFP16>, T_MAP5, XS, REX_W, EVEX_CD8<16, CD8VT1>;13408 13409let Predicates = [HasFP16] in {13410 def : Pat<(i16 (lrint FR16:$src)), (EXTRACT_SUBREG (VCVTTSH2SIZrr FR16:$src), sub_16bit)>;13411 def : Pat<(i32 (lrint FR16:$src)), (VCVTTSH2SIZrr FR16:$src)>;13412 def : Pat<(i32 (lrint (loadf16 addr:$src))), (VCVTTSH2SIZrm addr:$src)>;13413}13414 13415let Predicates = [HasFP16, In64BitMode] in {13416 def : Pat<(i64 (lrint FR16:$src)), (VCVTTSH2SI64Zrr FR16:$src)>;13417 def : Pat<(i64 (lrint (loadf16 addr:$src))), (VCVTTSH2SI64Zrm addr:$src)>;13418 def : Pat<(i64 (llrint FR16:$src)), (VCVTTSH2SI64Zrr FR16:$src)>;13419 def : Pat<(i64 (llrint (loadf16 addr:$src))), (VCVTTSH2SI64Zrm addr:$src)>;13420}13421 13422let Predicates = [HasFP16] in {13423 defm VCVTSI2SHZ : avx512_vcvtsi_common<0x2A, X86SintToFp, X86SintToFpRnd, WriteCvtI2SS, GR32,13424 v8f16x_info, i32mem, loadi32, "cvtsi2sh", "l">,13425 T_MAP5, XS, EVEX_CD8<32, CD8VT1>;13426 defm VCVTSI642SHZ: avx512_vcvtsi_common<0x2A, X86SintToFp, X86SintToFpRnd, WriteCvtI2SS, GR64,13427 v8f16x_info, i64mem, loadi64, "cvtsi2sh","q">,13428 T_MAP5, XS, REX_W, EVEX_CD8<64, CD8VT1>;13429 defm VCVTUSI2SHZ : avx512_vcvtsi_common<0x7B, X86UintToFp, X86UintToFpRnd, WriteCvtI2SS, GR32,13430 v8f16x_info, i32mem, loadi32,13431 "cvtusi2sh","l">, T_MAP5, XS, EVEX_CD8<32, CD8VT1>;13432 defm VCVTUSI642SHZ : avx512_vcvtsi_common<0x7B, X86UintToFp, X86UintToFpRnd, WriteCvtI2SS, GR64,13433 v8f16x_info, i64mem, loadi64, "cvtusi2sh", "q">,13434 T_MAP5, XS, REX_W, EVEX_CD8<64, CD8VT1>;13435 def : InstAlias<"vcvtsi2sh\t{$src, $src1, $dst|$dst, $src1, $src}",13436 (VCVTSI2SHZrm_Int VR128X:$dst, VR128X:$src1, i32mem:$src), 0, "att">;13437 13438 def : InstAlias<"vcvtusi2sh\t{$src, $src1, $dst|$dst, $src1, $src}",13439 (VCVTUSI2SHZrm_Int VR128X:$dst, VR128X:$src1, i32mem:$src), 0, "att">;13440 13441 13442 def : Pat<(f16 (any_sint_to_fp (loadi32 addr:$src))),13443 (VCVTSI2SHZrm (f16 (IMPLICIT_DEF)), addr:$src)>;13444 def : Pat<(f16 (any_sint_to_fp (loadi64 addr:$src))),13445 (VCVTSI642SHZrm (f16 (IMPLICIT_DEF)), addr:$src)>;13446 13447 def : Pat<(f16 (any_sint_to_fp GR32:$src)),13448 (VCVTSI2SHZrr (f16 (IMPLICIT_DEF)), GR32:$src)>;13449 def : Pat<(f16 (any_sint_to_fp GR64:$src)),13450 (VCVTSI642SHZrr (f16 (IMPLICIT_DEF)), GR64:$src)>;13451 13452 def : Pat<(f16 (any_uint_to_fp (loadi32 addr:$src))),13453 (VCVTUSI2SHZrm (f16 (IMPLICIT_DEF)), addr:$src)>;13454 def : Pat<(f16 (any_uint_to_fp (loadi64 addr:$src))),13455 (VCVTUSI642SHZrm (f16 (IMPLICIT_DEF)), addr:$src)>;13456 13457 def : Pat<(f16 (any_uint_to_fp GR32:$src)),13458 (VCVTUSI2SHZrr (f16 (IMPLICIT_DEF)), GR32:$src)>;13459 def : Pat<(f16 (any_uint_to_fp GR64:$src)),13460 (VCVTUSI642SHZrr (f16 (IMPLICIT_DEF)), GR64:$src)>;13461 13462 // Patterns used for matching vcvtsi2sh intrinsic sequences from clang13463 // which produce unnecessary vmovsh instructions13464 def : Pat<(v8f16 (X86Movsh13465 (v8f16 VR128X:$dst),13466 (v8f16 (scalar_to_vector (f16 (any_sint_to_fp GR64:$src)))))),13467 (VCVTSI642SHZrr_Int VR128X:$dst, GR64:$src)>;13468 13469 def : Pat<(v8f16 (X86Movsh13470 (v8f16 VR128X:$dst),13471 (v8f16 (scalar_to_vector (f16 (any_sint_to_fp (loadi64 addr:$src))))))),13472 (VCVTSI642SHZrm_Int VR128X:$dst, addr:$src)>;13473 13474 def : Pat<(v8f16 (X86Movsh13475 (v8f16 VR128X:$dst),13476 (v8f16 (scalar_to_vector (f16 (any_sint_to_fp GR32:$src)))))),13477 (VCVTSI2SHZrr_Int VR128X:$dst, GR32:$src)>;13478 13479 def : Pat<(v8f16 (X86Movsh13480 (v8f16 VR128X:$dst),13481 (v8f16 (scalar_to_vector (f16 (any_sint_to_fp (loadi32 addr:$src))))))),13482 (VCVTSI2SHZrm_Int VR128X:$dst, addr:$src)>;13483 13484 def : Pat<(v8f16 (X86Movsh13485 (v8f16 VR128X:$dst),13486 (v8f16 (scalar_to_vector (f16 (any_uint_to_fp GR64:$src)))))),13487 (VCVTUSI642SHZrr_Int VR128X:$dst, GR64:$src)>;13488 13489 def : Pat<(v8f16 (X86Movsh13490 (v8f16 VR128X:$dst),13491 (v8f16 (scalar_to_vector (f16 (any_uint_to_fp (loadi64 addr:$src))))))),13492 (VCVTUSI642SHZrm_Int VR128X:$dst, addr:$src)>;13493 13494 def : Pat<(v8f16 (X86Movsh13495 (v8f16 VR128X:$dst),13496 (v8f16 (scalar_to_vector (f16 (any_uint_to_fp GR32:$src)))))),13497 (VCVTUSI2SHZrr_Int VR128X:$dst, GR32:$src)>;13498 13499 def : Pat<(v8f16 (X86Movsh13500 (v8f16 VR128X:$dst),13501 (v8f16 (scalar_to_vector (f16 (any_uint_to_fp (loadi32 addr:$src))))))),13502 (VCVTUSI2SHZrm_Int VR128X:$dst, addr:$src)>;13503} // Predicates = [HasFP16]13504 13505let Predicates = [HasFP16, HasVLX] in {13506 // Special patterns to allow use of X86VMSintToFP for masking. Instruction13507 // patterns have been disabled with null_frag.13508 def : Pat<(v8f16 (X86any_VSintToFP (v4i64 VR256X:$src))),13509 (VCVTQQ2PHZ256rr VR256X:$src)>;13510 def : Pat<(X86VMSintToFP (v4i64 VR256X:$src), (v8f16 VR128X:$src0),13511 VK4WM:$mask),13512 (VCVTQQ2PHZ256rrk VR128X:$src0, VK4WM:$mask, VR256X:$src)>;13513 def : Pat<(X86VMSintToFP (v4i64 VR256X:$src), v8f16x_info.ImmAllZerosV,13514 VK4WM:$mask),13515 (VCVTQQ2PHZ256rrkz VK4WM:$mask, VR256X:$src)>;13516 13517 def : Pat<(v8f16 (X86any_VSintToFP (loadv4i64 addr:$src))),13518 (VCVTQQ2PHZ256rm addr:$src)>;13519 def : Pat<(X86VMSintToFP (loadv4i64 addr:$src), (v8f16 VR128X:$src0),13520 VK4WM:$mask),13521 (VCVTQQ2PHZ256rmk VR128X:$src0, VK4WM:$mask, addr:$src)>;13522 def : Pat<(X86VMSintToFP (loadv4i64 addr:$src), v8f16x_info.ImmAllZerosV,13523 VK4WM:$mask),13524 (VCVTQQ2PHZ256rmkz VK4WM:$mask, addr:$src)>;13525 13526 def : Pat<(v8f16 (X86any_VSintToFP (v4i64 (X86VBroadcastld64 addr:$src)))),13527 (VCVTQQ2PHZ256rmb addr:$src)>;13528 def : Pat<(X86VMSintToFP (v4i64 (X86VBroadcastld64 addr:$src)),13529 (v8f16 VR128X:$src0), VK4WM:$mask),13530 (VCVTQQ2PHZ256rmbk VR128X:$src0, VK4WM:$mask, addr:$src)>;13531 def : Pat<(X86VMSintToFP (v4i64 (X86VBroadcastld64 addr:$src)),13532 v8f16x_info.ImmAllZerosV, VK4WM:$mask),13533 (VCVTQQ2PHZ256rmbkz VK4WM:$mask, addr:$src)>;13534 13535 def : Pat<(v8f16 (X86any_VSintToFP (v2i64 VR128X:$src))),13536 (VCVTQQ2PHZ128rr VR128X:$src)>;13537 def : Pat<(X86VMSintToFP (v2i64 VR128X:$src), (v8f16 VR128X:$src0),13538 VK2WM:$mask),13539 (VCVTQQ2PHZ128rrk VR128X:$src0, VK2WM:$mask, VR128X:$src)>;13540 def : Pat<(X86VMSintToFP (v2i64 VR128X:$src), v8f16x_info.ImmAllZerosV,13541 VK2WM:$mask),13542 (VCVTQQ2PHZ128rrkz VK2WM:$mask, VR128X:$src)>;13543 13544 def : Pat<(v8f16 (X86any_VSintToFP (loadv2i64 addr:$src))),13545 (VCVTQQ2PHZ128rm addr:$src)>;13546 def : Pat<(X86VMSintToFP (loadv2i64 addr:$src), (v8f16 VR128X:$src0),13547 VK2WM:$mask),13548 (VCVTQQ2PHZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;13549 def : Pat<(X86VMSintToFP (loadv2i64 addr:$src), v8f16x_info.ImmAllZerosV,13550 VK2WM:$mask),13551 (VCVTQQ2PHZ128rmkz VK2WM:$mask, addr:$src)>;13552 13553 def : Pat<(v8f16 (X86any_VSintToFP (v2i64 (X86VBroadcastld64 addr:$src)))),13554 (VCVTQQ2PHZ128rmb addr:$src)>;13555 def : Pat<(X86VMSintToFP (v2i64 (X86VBroadcastld64 addr:$src)),13556 (v8f16 VR128X:$src0), VK2WM:$mask),13557 (VCVTQQ2PHZ128rmbk VR128X:$src0, VK2WM:$mask, addr:$src)>;13558 def : Pat<(X86VMSintToFP (v2i64 (X86VBroadcastld64 addr:$src)),13559 v8f16x_info.ImmAllZerosV, VK2WM:$mask),13560 (VCVTQQ2PHZ128rmbkz VK2WM:$mask, addr:$src)>;13561 13562 // Special patterns to allow use of X86VMUintToFP for masking. Instruction13563 // patterns have been disabled with null_frag.13564 def : Pat<(v8f16 (X86any_VUintToFP (v4i64 VR256X:$src))),13565 (VCVTUQQ2PHZ256rr VR256X:$src)>;13566 def : Pat<(X86VMUintToFP (v4i64 VR256X:$src), (v8f16 VR128X:$src0),13567 VK4WM:$mask),13568 (VCVTUQQ2PHZ256rrk VR128X:$src0, VK4WM:$mask, VR256X:$src)>;13569 def : Pat<(X86VMUintToFP (v4i64 VR256X:$src), v8f16x_info.ImmAllZerosV,13570 VK4WM:$mask),13571 (VCVTUQQ2PHZ256rrkz VK4WM:$mask, VR256X:$src)>;13572 13573 def : Pat<(v8f16 (X86any_VUintToFP (loadv4i64 addr:$src))),13574 (VCVTUQQ2PHZ256rm addr:$src)>;13575 def : Pat<(X86VMUintToFP (loadv4i64 addr:$src), (v8f16 VR128X:$src0),13576 VK4WM:$mask),13577 (VCVTUQQ2PHZ256rmk VR128X:$src0, VK4WM:$mask, addr:$src)>;13578 def : Pat<(X86VMUintToFP (loadv4i64 addr:$src), v8f16x_info.ImmAllZerosV,13579 VK4WM:$mask),13580 (VCVTUQQ2PHZ256rmkz VK4WM:$mask, addr:$src)>;13581 13582 def : Pat<(v8f16 (X86any_VUintToFP (v4i64 (X86VBroadcastld64 addr:$src)))),13583 (VCVTUQQ2PHZ256rmb addr:$src)>;13584 def : Pat<(X86VMUintToFP (v4i64 (X86VBroadcastld64 addr:$src)),13585 (v8f16 VR128X:$src0), VK4WM:$mask),13586 (VCVTUQQ2PHZ256rmbk VR128X:$src0, VK4WM:$mask, addr:$src)>;13587 def : Pat<(X86VMUintToFP (v4i64 (X86VBroadcastld64 addr:$src)),13588 v8f16x_info.ImmAllZerosV, VK4WM:$mask),13589 (VCVTUQQ2PHZ256rmbkz VK4WM:$mask, addr:$src)>;13590 13591 def : Pat<(v8f16 (X86any_VUintToFP (v2i64 VR128X:$src))),13592 (VCVTUQQ2PHZ128rr VR128X:$src)>;13593 def : Pat<(X86VMUintToFP (v2i64 VR128X:$src), (v8f16 VR128X:$src0),13594 VK2WM:$mask),13595 (VCVTUQQ2PHZ128rrk VR128X:$src0, VK2WM:$mask, VR128X:$src)>;13596 def : Pat<(X86VMUintToFP (v2i64 VR128X:$src), v8f16x_info.ImmAllZerosV,13597 VK2WM:$mask),13598 (VCVTUQQ2PHZ128rrkz VK2WM:$mask, VR128X:$src)>;13599 13600 def : Pat<(v8f16 (X86any_VUintToFP (loadv2i64 addr:$src))),13601 (VCVTUQQ2PHZ128rm addr:$src)>;13602 def : Pat<(X86VMUintToFP (loadv2i64 addr:$src), (v8f16 VR128X:$src0),13603 VK2WM:$mask),13604 (VCVTUQQ2PHZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;13605 def : Pat<(X86VMUintToFP (loadv2i64 addr:$src), v8f16x_info.ImmAllZerosV,13606 VK2WM:$mask),13607 (VCVTUQQ2PHZ128rmkz VK2WM:$mask, addr:$src)>;13608 13609 def : Pat<(v8f16 (X86any_VUintToFP (v2i64 (X86VBroadcastld64 addr:$src)))),13610 (VCVTUQQ2PHZ128rmb addr:$src)>;13611 def : Pat<(X86VMUintToFP (v2i64 (X86VBroadcastld64 addr:$src)),13612 (v8f16 VR128X:$src0), VK2WM:$mask),13613 (VCVTUQQ2PHZ128rmbk VR128X:$src0, VK2WM:$mask, addr:$src)>;13614 def : Pat<(X86VMUintToFP (v2i64 (X86VBroadcastld64 addr:$src)),13615 v8f16x_info.ImmAllZerosV, VK2WM:$mask),13616 (VCVTUQQ2PHZ128rmbkz VK2WM:$mask, addr:$src)>;13617}13618 13619let Constraints = "@earlyclobber $dst, $src1 = $dst" in {13620 multiclass avx512_cfmaop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _, bit IsCommutable> {13621 defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),13622 (ins _.RC:$src2, _.RC:$src3),13623 OpcodeStr, "$src3, $src2", "$src2, $src3",13624 (_.VT (OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1)), IsCommutable>, EVEX, VVVV;13625 13626 defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),13627 (ins _.RC:$src2, _.MemOp:$src3),13628 OpcodeStr, "$src3, $src2", "$src2, $src3",13629 (_.VT (OpNode _.RC:$src2, (_.LdFrag addr:$src3), _.RC:$src1))>, EVEX, VVVV;13630 13631 defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),13632 (ins _.RC:$src2, _.ScalarMemOp:$src3),13633 OpcodeStr, !strconcat("${src3}", _.BroadcastStr,", $src2"), !strconcat("$src2, ${src3}", _.BroadcastStr),13634 (_.VT (OpNode _.RC:$src2, (_.VT (_.BroadcastLdFrag addr:$src3)), _.RC:$src1))>, EVEX_B, EVEX, VVVV;13635 }13636} // Constraints = "@earlyclobber $dst, $src1 = $dst"13637 13638multiclass avx512_cfmaop_round<bits<8> opc, string OpcodeStr, SDNode OpNode,13639 X86VectorVTInfo _> {13640 let Constraints = "@earlyclobber $dst, $src1 = $dst" in13641 defm rb: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),13642 (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc),13643 OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc",13644 (_.VT (OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1, (i32 timm:$rc)))>,13645 EVEX, VVVV, EVEX_B, EVEX_RC;13646}13647 13648 13649multiclass avx512_cfmaop_common<bits<8> opc, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd, bit IsCommutable> {13650 let Predicates = [HasFP16] in {13651 defm Z : avx512_cfmaop_rm<opc, OpcodeStr, OpNode, v16f32_info, IsCommutable>,13652 avx512_cfmaop_round<opc, OpcodeStr, OpNodeRnd, v16f32_info>,13653 EVEX_V512, Sched<[WriteFMAZ]>;13654 }13655 let Predicates = [HasVLX, HasFP16] in {13656 defm Z256 : avx512_cfmaop_rm<opc, OpcodeStr, OpNode, v8f32x_info, IsCommutable>, EVEX_V256, Sched<[WriteFMAY]>;13657 defm Z128 : avx512_cfmaop_rm<opc, OpcodeStr, OpNode, v4f32x_info, IsCommutable>, EVEX_V128, Sched<[WriteFMAX]>;13658 }13659}13660 13661multiclass avx512_cfmulop_common<bits<8> opc, string OpcodeStr, SDNode OpNode,13662 SDNode MaskOpNode, SDNode OpNodeRnd, bit IsCommutable> {13663 let Predicates = [HasFP16] in {13664 defm Z : avx512_fp_packed<opc, OpcodeStr, OpNode, MaskOpNode, v16f32_info,13665 WriteFMAZ, IsCommutable, IsCommutable, "", "@earlyclobber $dst", 0>,13666 avx512_fp_round_packed<opc, OpcodeStr, OpNodeRnd, WriteFMAZ, v16f32_info,13667 "", "@earlyclobber $dst">, EVEX_V512;13668 }13669 let Predicates = [HasVLX, HasFP16] in {13670 defm Z256 : avx512_fp_packed<opc, OpcodeStr, OpNode, MaskOpNode, v8f32x_info,13671 WriteFMAY, IsCommutable, IsCommutable, "", "@earlyclobber $dst", 0>, EVEX_V256;13672 defm Z128 : avx512_fp_packed<opc, OpcodeStr, OpNode, MaskOpNode, v4f32x_info,13673 WriteFMAX, IsCommutable, IsCommutable, "", "@earlyclobber $dst", 0>, EVEX_V128;13674 }13675}13676 13677 13678let Uses = [MXCSR] in {13679 defm VFMADDCPH : avx512_cfmaop_common<0x56, "vfmaddcph", x86vfmaddc, x86vfmaddcRnd, 1>,13680 T_MAP6, XS, EVEX_CD8<32, CD8VF>;13681 defm VFCMADDCPH : avx512_cfmaop_common<0x56, "vfcmaddcph", x86vfcmaddc, x86vfcmaddcRnd, 0>,13682 T_MAP6, XD, EVEX_CD8<32, CD8VF>;13683 13684 defm VFMULCPH : avx512_cfmulop_common<0xD6, "vfmulcph", x86vfmulc, x86vfmulc,13685 x86vfmulcRnd, 1>, T_MAP6, XS, EVEX_CD8<32, CD8VF>;13686 defm VFCMULCPH : avx512_cfmulop_common<0xD6, "vfcmulcph", x86vfcmulc,13687 x86vfcmulc, x86vfcmulcRnd, 0>, T_MAP6, XD, EVEX_CD8<32, CD8VF>;13688}13689 13690 13691multiclass avx512_cfmaop_sh_common<bits<8> opc, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd,13692 bit IsCommutable> {13693 let Predicates = [HasFP16], Constraints = "@earlyclobber $dst, $src1 = $dst" in {13694 defm r : AVX512_maskable_3src_scalar<opc, MRMSrcReg, f32x_info, (outs VR128X:$dst),13695 (ins VR128X:$src2, VR128X:$src3), OpcodeStr,13696 "$src3, $src2", "$src2, $src3",13697 (v4f32 (OpNode VR128X:$src2, VR128X:$src3, VR128X:$src1)), IsCommutable>,13698 Sched<[WriteFMAX]>;13699 defm m : AVX512_maskable_3src_scalar<opc, MRMSrcMem, f32x_info, (outs VR128X:$dst),13700 (ins VR128X:$src2, ssmem:$src3), OpcodeStr,13701 "$src3, $src2", "$src2, $src3",13702 (v4f32 (OpNode VR128X:$src2, (sse_load_f32 addr:$src3), VR128X:$src1))>,13703 Sched<[WriteFMAX.Folded, WriteFMAX.ReadAfterFold]>;13704 defm rb : AVX512_maskable_3src_scalar<opc, MRMSrcReg, f32x_info, (outs VR128X:$dst),13705 (ins VR128X:$src2, VR128X:$src3, AVX512RC:$rc), OpcodeStr,13706 "$rc, $src3, $src2", "$src2, $src3, $rc",13707 (v4f32 (OpNodeRnd VR128X:$src2, VR128X:$src3, VR128X:$src1, (i32 timm:$rc)))>,13708 EVEX_B, EVEX_RC, Sched<[WriteFMAX]>;13709 }13710}13711 13712multiclass avx512_cfmbinop_sh_common<bits<8> opc, string OpcodeStr, SDNode OpNode,13713 SDNode OpNodeRnd, bit IsCommutable> {13714 let Predicates = [HasFP16] in {13715 defm rr : AVX512_maskable<opc, MRMSrcReg, f32x_info, (outs VR128X:$dst),13716 (ins VR128X:$src1, VR128X:$src2), OpcodeStr,13717 "$src2, $src1", "$src1, $src2",13718 (v4f32 (OpNode VR128X:$src1, VR128X:$src2)),13719 IsCommutable, IsCommutable, IsCommutable,13720 X86selects, "@earlyclobber $dst">, Sched<[WriteFMAX]>;13721 let mayLoad = 1 in13722 defm rm : AVX512_maskable<opc, MRMSrcMem, f32x_info, (outs VR128X:$dst),13723 (ins VR128X:$src1, ssmem:$src2), OpcodeStr,13724 "$src2, $src1", "$src1, $src2",13725 (v4f32 (OpNode VR128X:$src1, (sse_load_f32 addr:$src2))),13726 0, 0, 0, X86selects, "@earlyclobber $dst">,13727 Sched<[WriteFMAX.Folded, WriteFMAX.ReadAfterFold]>;13728 defm rrb : AVX512_maskable<opc, MRMSrcReg, f32x_info, (outs VR128X:$dst),13729 (ins VR128X:$src1, VR128X:$src2, AVX512RC:$rc), OpcodeStr,13730 "$rc, $src2, $src1", "$src1, $src2, $rc",13731 (OpNodeRnd (v4f32 VR128X:$src1), (v4f32 VR128X:$src2), (i32 timm:$rc)),13732 0, 0, 0, X86selects, "@earlyclobber $dst">,13733 EVEX_B, EVEX_RC, Sched<[WriteFMAX]>;13734 }13735}13736 13737let Uses = [MXCSR] in {13738 defm VFMADDCSHZ : avx512_cfmaop_sh_common<0x57, "vfmaddcsh", x86vfmaddcSh, x86vfmaddcShRnd, 1>,13739 T_MAP6, XS, EVEX_CD8<32, CD8VT1>, EVEX_V128, EVEX, VVVV;13740 defm VFCMADDCSHZ : avx512_cfmaop_sh_common<0x57, "vfcmaddcsh", x86vfcmaddcSh, x86vfcmaddcShRnd, 0>,13741 T_MAP6, XD, EVEX_CD8<32, CD8VT1>, EVEX_V128, EVEX, VVVV;13742 13743 defm VFMULCSHZ : avx512_cfmbinop_sh_common<0xD7, "vfmulcsh", x86vfmulcSh, x86vfmulcShRnd, 1>,13744 T_MAP6, XS, EVEX_CD8<32, CD8VT1>, EVEX_V128, VEX_LIG, EVEX, VVVV;13745 defm VFCMULCSHZ : avx512_cfmbinop_sh_common<0xD7, "vfcmulcsh", x86vfcmulcSh, x86vfcmulcShRnd, 0>,13746 T_MAP6, XD, EVEX_CD8<32, CD8VT1>, EVEX_V128, VEX_LIG, EVEX, VVVV;13747}13748