8480 lines · plain
1//===-- X86InstrSSE.td - SSE Instruction Set ---------------*- tablegen -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file describes the X86 SSE instruction set, defining the instructions,10// and properties of the instructions which are needed for code generation,11// machine code emission, and analysis.12//13//===----------------------------------------------------------------------===//14 15//===----------------------------------------------------------------------===//16// SSE 1 & 2 Instructions Classes17//===----------------------------------------------------------------------===//18 19/// sse12_fp_scalar - SSE 1 & 2 scalar instructions class20multiclass sse12_fp_scalar<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,21 RegisterClass RC, X86MemOperand x86memop,22 Domain d, X86FoldableSchedWrite sched,23 bit Is2Addr = 1> {24let isCodeGenOnly = 1 in {25 let isCommutable = 1 in {26 def rr : SI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2),27 !if(Is2Addr,28 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),29 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),30 [(set RC:$dst, (OpNode RC:$src1, RC:$src2))], d>,31 Sched<[sched]>;32 }33 def rm : SI<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2),34 !if(Is2Addr,35 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),36 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),37 [(set RC:$dst, (OpNode RC:$src1, (load addr:$src2)))], d>,38 Sched<[sched.Folded, sched.ReadAfterFold]>;39}40}41 42/// sse12_fp_scalar_int - SSE 1 & 2 scalar instructions intrinsics class43multiclass sse12_fp_scalar_int<bits<8> opc,44 SDPatternOperator OpNode, RegisterClass RC,45 ValueType VT, string asm, Operand memopr,46 PatFrags mem_frags, Domain d,47 X86FoldableSchedWrite sched, bit Is2Addr = 1> {48let hasSideEffects = 0 in {49 def rr_Int : SI_Int<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2),50 !if(Is2Addr,51 !strconcat(asm, "\t{$src2, $dst|$dst, $src2}"),52 !strconcat(asm, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),53 [(set RC:$dst, (VT (OpNode RC:$src1, RC:$src2)))], d>,54 Sched<[sched]>;55 let mayLoad = 1 in56 def rm_Int : SI_Int<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, memopr:$src2),57 !if(Is2Addr,58 !strconcat(asm, "\t{$src2, $dst|$dst, $src2}"),59 !strconcat(asm, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),60 [(set RC:$dst, (VT (OpNode RC:$src1, (mem_frags addr:$src2))))], d>,61 Sched<[sched.Folded, sched.ReadAfterFold]>;62}63}64 65/// sse12_fp_packed - SSE 1 & 2 packed instructions class66multiclass sse12_fp_packed<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,67 RegisterClass RC, ValueType vt,68 X86MemOperand x86memop, PatFrag mem_frag,69 Domain d, X86FoldableSchedWrite sched,70 bit Is2Addr = 1> {71 let isCommutable = 1 in72 def rr : PI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2),73 !if(Is2Addr,74 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),75 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),76 [(set RC:$dst, (vt (OpNode RC:$src1, RC:$src2)))], d>,77 Sched<[sched]>;78 let mayLoad = 1 in79 def rm : PI<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2),80 !if(Is2Addr,81 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),82 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),83 [(set RC:$dst, (OpNode RC:$src1, (mem_frag addr:$src2)))],84 d>,85 Sched<[sched.Folded, sched.ReadAfterFold]>;86}87 88/// sse12_fp_packed_logical_rm - SSE 1 & 2 packed instructions class89multiclass sse12_fp_packed_logical_rm<bits<8> opc, RegisterClass RC, Domain d,90 string OpcodeStr, X86MemOperand x86memop,91 X86FoldableSchedWrite sched,92 list<dag> pat_rr, list<dag> pat_rm,93 bit Is2Addr = 1> {94 let isCommutable = 1, hasSideEffects = 0 in95 def rr : PI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2),96 !if(Is2Addr,97 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),98 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),99 pat_rr, d>,100 Sched<[sched]>;101 let hasSideEffects = 0, mayLoad = 1 in102 def rm : PI<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2),103 !if(Is2Addr,104 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),105 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),106 pat_rm, d>,107 Sched<[sched.Folded, sched.ReadAfterFold]>;108}109 110 111// Alias instructions that map fld0 to xorps for sse or vxorps for avx.112// This is expanded by ExpandPostRAPseudos.113let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,114 isPseudo = 1, SchedRW = [WriteZero] in {115 def FsFLD0SH : I<0, Pseudo, (outs FR16:$dst), (ins), "",116 [(set FR16:$dst, fp16imm0)]>, Requires<[HasSSE2, NoAVX512]>;117 def FsFLD0SS : I<0, Pseudo, (outs FR32:$dst), (ins), "",118 [(set FR32:$dst, fp32imm0)]>, Requires<[HasSSE1, NoAVX512]>;119 def FsFLD0SD : I<0, Pseudo, (outs FR64:$dst), (ins), "",120 [(set FR64:$dst, fp64imm0)]>, Requires<[HasSSE2, NoAVX512]>;121 def FsFLD0F128 : I<0, Pseudo, (outs VR128:$dst), (ins), "",122 [(set VR128:$dst, fp128imm0)]>, Requires<[HasSSE1, NoAVX512]>;123}124 125//===----------------------------------------------------------------------===//126// AVX & SSE - Zero/One Vectors127//===----------------------------------------------------------------------===//128 129// Alias instruction that maps zero vector to pxor / xorp* for sse.130// This is expanded by ExpandPostRAPseudos to an xorps / vxorps, and then131// swizzled by ExecutionDomainFix to pxor.132// We set canFoldAsLoad because this can be converted to a constant-pool133// load of an all-zeros value if folding it would be beneficial.134let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,135 isPseudo = 1, Predicates = [NoAVX512], SchedRW = [WriteZero] in {136def V_SET0 : I<0, Pseudo, (outs VR128:$dst), (ins), "",137 [(set VR128:$dst, (v4f32 immAllZerosV))]>;138}139 140let Predicates = [NoAVX512] in {141def : Pat<(v16i8 immAllZerosV), (V_SET0)>;142def : Pat<(v8i16 immAllZerosV), (V_SET0)>;143def : Pat<(v8f16 immAllZerosV), (V_SET0)>;144def : Pat<(v4i32 immAllZerosV), (V_SET0)>;145def : Pat<(v2i64 immAllZerosV), (V_SET0)>;146def : Pat<(v2f64 immAllZerosV), (V_SET0)>;147}148 149 150// The same as done above but for AVX. The 256-bit AVX1 ISA doesn't support PI,151// and doesn't need it because on sandy bridge the register is set to zero152// at the rename stage without using any execution unit, so SET0PSY153// and SET0PDY can be used for vector int instructions without penalty154let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,155 isPseudo = 1, Predicates = [NoAVX512], SchedRW = [WriteZero] in {156def AVX_SET0 : I<0, Pseudo, (outs VR256:$dst), (ins), "",157 [(set VR256:$dst, (v8i32 immAllZerosV))]>;158}159 160let Predicates = [NoAVX512] in {161def : Pat<(v32i8 immAllZerosV), (AVX_SET0)>;162def : Pat<(v16i16 immAllZerosV), (AVX_SET0)>;163def : Pat<(v16f16 immAllZerosV), (AVX_SET0)>;164def : Pat<(v4i64 immAllZerosV), (AVX_SET0)>;165def : Pat<(v8f32 immAllZerosV), (AVX_SET0)>;166def : Pat<(v4f64 immAllZerosV), (AVX_SET0)>;167}168 169// We set canFoldAsLoad because this can be converted to a constant-pool170// load of an all-ones value if folding it would be beneficial.171let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,172 isPseudo = 1, SchedRW = [WriteZero] in {173 def V_SETALLONES : I<0, Pseudo, (outs VR128:$dst), (ins), "",174 [(set VR128:$dst, (v4i32 immAllOnesV))]>;175 let Predicates = [HasAVX1Only, OptForMinSize] in {176 def AVX1_SETALLONES: I<0, Pseudo, (outs VR256:$dst), (ins), "",177 [(set VR256:$dst, (v8i32 immAllOnesV))]>;178 }179 let Predicates = [HasAVX2] in180 def AVX2_SETALLONES : I<0, Pseudo, (outs VR256:$dst), (ins), "",181 [(set VR256:$dst, (v8i32 immAllOnesV))]>;182}183 184//===----------------------------------------------------------------------===//185// SSE 1 & 2 - Move FP Scalar Instructions186//187// Move Instructions. Register-to-register movss/movsd is not used for FR32/64188// register copies because it's a partial register update; Register-to-register189// movss/movsd is not modeled as an INSERT_SUBREG because INSERT_SUBREG requires190// that the insert be implementable in terms of a copy, and just mentioned, we191// don't use movss/movsd for copies.192//===----------------------------------------------------------------------===//193 194multiclass sse12_move_rr<SDNode OpNode, ValueType vt, string base_opc,195 string asm_opr, Domain d> {196 let isCommutable = 1 in197 def rr : SI<0x10, MRMSrcReg, (outs VR128:$dst),198 (ins VR128:$src1, VR128:$src2),199 !strconcat(base_opc, asm_opr),200 [(set VR128:$dst, (vt (OpNode VR128:$src1, VR128:$src2)))], d>,201 Sched<[SchedWriteFShuffle.XMM]>;202 203 // For the disassembler204 let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in205 def rr_REV : SI<0x11, MRMDestReg, (outs VR128:$dst),206 (ins VR128:$src1, VR128:$src2),207 !strconcat(base_opc, asm_opr), []>,208 Sched<[SchedWriteFShuffle.XMM]>;209}210 211multiclass sse12_move<RegisterClass RC, SDNode OpNode, ValueType vt,212 X86MemOperand x86memop, string OpcodeStr,213 Domain d, Predicate pred> {214 // AVX215 let Predicates = [UseAVX, OptForSize] in216 defm V#NAME : sse12_move_rr<OpNode, vt, OpcodeStr,217 "\t{$src2, $src1, $dst|$dst, $src1, $src2}", d>,218 VEX, VVVV, VEX_LIG, WIG;219 220 def V#NAME#mr : SI<0x11, MRMDestMem, (outs), (ins x86memop:$dst, RC:$src),221 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),222 [(store RC:$src, addr:$dst)], d>,223 VEX, VEX_LIG, Sched<[WriteFStore]>, WIG;224 // SSE1 & 2225 let Constraints = "$src1 = $dst" in {226 let Predicates = [pred, NoSSE41_Or_OptForSize] in227 defm NAME : sse12_move_rr<OpNode, vt, OpcodeStr,228 "\t{$src2, $dst|$dst, $src2}", d>;229 }230 231 def NAME#mr : SI<0x11, MRMDestMem, (outs), (ins x86memop:$dst, RC:$src),232 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),233 [(store RC:$src, addr:$dst)], d>,234 Sched<[WriteFStore]>;235 236 def : InstAlias<"v"#OpcodeStr#".s\t{$src2, $src1, $dst|$dst, $src1, $src2}",237 (!cast<Instruction>("V"#NAME#"rr_REV")238 VR128:$dst, VR128:$src1, VR128:$src2), 0>;239 def : InstAlias<OpcodeStr#".s\t{$src2, $dst|$dst, $src2}",240 (!cast<Instruction>(NAME#"rr_REV")241 VR128:$dst, VR128:$src2), 0>;242}243 244// Loading from memory automatically zeroing upper bits.245multiclass sse12_move_rm<RegisterClass RC, ValueType vt, X86MemOperand x86memop,246 PatFrag mem_pat, PatFrag vzloadfrag, string OpcodeStr,247 Domain d> {248 def V#NAME#rm : SI<0x10, MRMSrcMem, (outs VR128:$dst), (ins x86memop:$src),249 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),250 [(set VR128:$dst, (vt (vzloadfrag addr:$src)))], d>,251 VEX, VEX_LIG, Sched<[WriteFLoad]>, WIG;252 def NAME#rm : SI<0x10, MRMSrcMem, (outs VR128:$dst), (ins x86memop:$src),253 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),254 [(set VR128:$dst, (vt (vzloadfrag addr:$src)))], d>,255 Sched<[WriteFLoad]>;256 257 // _alt version uses FR32/FR64 register class.258 let isCodeGenOnly = 1 in {259 def V#NAME#rm_alt : SI<0x10, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src),260 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),261 [(set RC:$dst, (mem_pat addr:$src))], d>,262 VEX, VEX_LIG, Sched<[WriteFLoad]>, WIG;263 def NAME#rm_alt : SI<0x10, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src),264 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),265 [(set RC:$dst, (mem_pat addr:$src))], d>,266 Sched<[WriteFLoad]>;267 }268}269 270// pseudo instruction for fp16 spilling.271let isPseudo = 1, Predicates = [HasSSE2] in {272 let mayStore = 1 in273 def MOVSHPmr : I<0, Pseudo, (outs), (ins f32mem:$dst, FR16X:$src), "",274 [], SSEPackedSingle>,275 Sched<[WriteFStore]>;276 let mayLoad = 1 in277 def MOVSHPrm : I<0, Pseudo, (outs FR16X:$dst), (ins f32mem:$src), "",278 [], SSEPackedSingle>,279 Sched<[WriteFLoad]>;280}281 282defm MOVSS : sse12_move<FR32, X86Movss, v4f32, f32mem, "movss",283 SSEPackedSingle, UseSSE1>, TB, XS;284defm MOVSD : sse12_move<FR64, X86Movsd, v2f64, f64mem, "movsd",285 SSEPackedDouble, UseSSE2>, TB, XD;286 287let canFoldAsLoad = 1, isReMaterializable = 1 in {288 defm MOVSS : sse12_move_rm<FR32, v4f32, f32mem, loadf32, X86vzload32, "movss",289 SSEPackedSingle>, TB, XS;290 defm MOVSD : sse12_move_rm<FR64, v2f64, f64mem, loadf64, X86vzload64, "movsd",291 SSEPackedDouble>, TB, XD;292}293 294// Patterns295let Predicates = [UseAVX] in {296 def : Pat<(v4f32 (scalar_to_vector (loadf32 addr:$src))),297 (VMOVSSrm addr:$src)>;298 def : Pat<(v2f64 (scalar_to_vector (loadf64 addr:$src))),299 (VMOVSDrm addr:$src)>;300 301 // Represent the same patterns above but in the form they appear for302 // 256-bit types303 def : Pat<(v8f32 (X86vzload32 addr:$src)),304 (SUBREG_TO_REG (i32 0), (VMOVSSrm addr:$src), sub_xmm)>;305 def : Pat<(v4f64 (X86vzload64 addr:$src)),306 (SUBREG_TO_REG (i32 0), (VMOVSDrm addr:$src), sub_xmm)>;307}308 309let Predicates = [UseAVX, OptForSize] in {310 // Move scalar to XMM zero-extended, zeroing a VR128 then do a311 // MOVSS to the lower bits.312 def : Pat<(v4f32 (X86vzmovl (v4f32 VR128:$src))),313 (VMOVSSrr (v4f32 (V_SET0)), VR128:$src)>;314 def : Pat<(v4i32 (X86vzmovl (v4i32 VR128:$src))),315 (VMOVSSrr (v4i32 (V_SET0)), VR128:$src)>;316 317 // Move low f32 and clear high bits.318 def : Pat<(v8f32 (X86vzmovl (v8f32 VR256:$src))),319 (SUBREG_TO_REG (i32 0),320 (v4f32 (VMOVSSrr (v4f32 (V_SET0)),321 (v4f32 (EXTRACT_SUBREG (v8f32 VR256:$src), sub_xmm)))), sub_xmm)>;322 def : Pat<(v8i32 (X86vzmovl (v8i32 VR256:$src))),323 (SUBREG_TO_REG (i32 0),324 (v4i32 (VMOVSSrr (v4i32 (V_SET0)),325 (v4i32 (EXTRACT_SUBREG (v8i32 VR256:$src), sub_xmm)))), sub_xmm)>;326}327 328let Predicates = [UseSSE1, NoSSE41_Or_OptForSize] in {329// Move scalar to XMM zero-extended, zeroing a VR128 then do a330// MOVSS to the lower bits.331def : Pat<(v4f32 (X86vzmovl (v4f32 VR128:$src))),332 (MOVSSrr (v4f32 (V_SET0)), VR128:$src)>;333def : Pat<(v4i32 (X86vzmovl (v4i32 VR128:$src))),334 (MOVSSrr (v4i32 (V_SET0)), VR128:$src)>;335}336 337let Predicates = [UseSSE2] in338def : Pat<(v2f64 (scalar_to_vector (loadf64 addr:$src))),339 (MOVSDrm addr:$src)>;340 341let Predicates = [UseSSE1] in342def : Pat<(v4f32 (scalar_to_vector (loadf32 addr:$src))),343 (MOVSSrm addr:$src)>;344 345//===----------------------------------------------------------------------===//346// SSE 1 & 2 - Move Aligned/Unaligned FP Instructions347//===----------------------------------------------------------------------===//348 349multiclass sse12_mov_packed<bits<8> opc, RegisterClass RC,350 X86MemOperand x86memop, PatFrag ld_frag,351 string asm, Domain d,352 X86SchedWriteMoveLS sched> {353let hasSideEffects = 0, isMoveReg = 1 in354 def rr : PI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src),355 !strconcat(asm, "\t{$src, $dst|$dst, $src}"), [], d>,356 Sched<[sched.RR]>;357let canFoldAsLoad = 1, isReMaterializable = 1 in358 def rm : PI<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src),359 !strconcat(asm, "\t{$src, $dst|$dst, $src}"),360 [(set RC:$dst, (ld_frag addr:$src))], d>,361 Sched<[sched.RM]>;362}363 364let Predicates = [HasAVX, NoVLX] in {365defm VMOVAPS : sse12_mov_packed<0x28, VR128, f128mem, alignedloadv4f32, "movaps",366 SSEPackedSingle, SchedWriteFMoveLS.XMM>,367 TB, VEX, WIG;368defm VMOVAPD : sse12_mov_packed<0x28, VR128, f128mem, alignedloadv2f64, "movapd",369 SSEPackedDouble, SchedWriteFMoveLS.XMM>,370 TB, PD, VEX, WIG;371defm VMOVUPS : sse12_mov_packed<0x10, VR128, f128mem, loadv4f32, "movups",372 SSEPackedSingle, SchedWriteFMoveLS.XMM>,373 TB, VEX, WIG;374defm VMOVUPD : sse12_mov_packed<0x10, VR128, f128mem, loadv2f64, "movupd",375 SSEPackedDouble, SchedWriteFMoveLS.XMM>,376 TB, PD, VEX, WIG;377 378defm VMOVAPSY : sse12_mov_packed<0x28, VR256, f256mem, alignedloadv8f32, "movaps",379 SSEPackedSingle, SchedWriteFMoveLS.YMM>,380 TB, VEX, VEX_L, WIG;381defm VMOVAPDY : sse12_mov_packed<0x28, VR256, f256mem, alignedloadv4f64, "movapd",382 SSEPackedDouble, SchedWriteFMoveLS.YMM>,383 TB, PD, VEX, VEX_L, WIG;384defm VMOVUPSY : sse12_mov_packed<0x10, VR256, f256mem, loadv8f32, "movups",385 SSEPackedSingle, SchedWriteFMoveLS.YMM>,386 TB, VEX, VEX_L, WIG;387defm VMOVUPDY : sse12_mov_packed<0x10, VR256, f256mem, loadv4f64, "movupd",388 SSEPackedDouble, SchedWriteFMoveLS.YMM>,389 TB, PD, VEX, VEX_L, WIG;390}391 392let Predicates = [UseSSE1] in {393defm MOVAPS : sse12_mov_packed<0x28, VR128, f128mem, alignedloadv4f32, "movaps",394 SSEPackedSingle, SchedWriteFMoveLS.XMM>,395 TB;396defm MOVUPS : sse12_mov_packed<0x10, VR128, f128mem, loadv4f32, "movups",397 SSEPackedSingle, SchedWriteFMoveLS.XMM>,398 TB;399}400let Predicates = [UseSSE2] in {401defm MOVAPD : sse12_mov_packed<0x28, VR128, f128mem, alignedloadv2f64, "movapd",402 SSEPackedDouble, SchedWriteFMoveLS.XMM>,403 TB, PD;404defm MOVUPD : sse12_mov_packed<0x10, VR128, f128mem, loadv2f64, "movupd",405 SSEPackedDouble, SchedWriteFMoveLS.XMM>,406 TB, PD;407}408 409let Predicates = [HasAVX, NoVLX] in {410let SchedRW = [SchedWriteFMoveLS.XMM.MR] in {411def VMOVAPSmr : VPSI<0x29, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src),412 "movaps\t{$src, $dst|$dst, $src}",413 [(alignedstore (v4f32 VR128:$src), addr:$dst)]>,414 VEX, WIG;415def VMOVAPDmr : VPDI<0x29, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src),416 "movapd\t{$src, $dst|$dst, $src}",417 [(alignedstore (v2f64 VR128:$src), addr:$dst)]>,418 VEX, WIG;419def VMOVUPSmr : VPSI<0x11, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src),420 "movups\t{$src, $dst|$dst, $src}",421 [(store (v4f32 VR128:$src), addr:$dst)]>,422 VEX, WIG;423def VMOVUPDmr : VPDI<0x11, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src),424 "movupd\t{$src, $dst|$dst, $src}",425 [(store (v2f64 VR128:$src), addr:$dst)]>,426 VEX, WIG;427} // SchedRW428 429let SchedRW = [SchedWriteFMoveLS.YMM.MR] in {430def VMOVAPSYmr : VPSI<0x29, MRMDestMem, (outs), (ins f256mem:$dst, VR256:$src),431 "movaps\t{$src, $dst|$dst, $src}",432 [(alignedstore (v8f32 VR256:$src), addr:$dst)]>,433 VEX, VEX_L, WIG;434def VMOVAPDYmr : VPDI<0x29, MRMDestMem, (outs), (ins f256mem:$dst, VR256:$src),435 "movapd\t{$src, $dst|$dst, $src}",436 [(alignedstore (v4f64 VR256:$src), addr:$dst)]>,437 VEX, VEX_L, WIG;438def VMOVUPSYmr : VPSI<0x11, MRMDestMem, (outs), (ins f256mem:$dst, VR256:$src),439 "movups\t{$src, $dst|$dst, $src}",440 [(store (v8f32 VR256:$src), addr:$dst)]>,441 VEX, VEX_L, WIG;442def VMOVUPDYmr : VPDI<0x11, MRMDestMem, (outs), (ins f256mem:$dst, VR256:$src),443 "movupd\t{$src, $dst|$dst, $src}",444 [(store (v4f64 VR256:$src), addr:$dst)]>,445 VEX, VEX_L, WIG;446} // SchedRW447} // Predicate448 449// For disassembler450let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0,451 isMoveReg = 1 in {452let SchedRW = [SchedWriteFMoveLS.XMM.RR] in {453 def VMOVAPSrr_REV : VPSI<0x29, MRMDestReg, (outs VR128:$dst),454 (ins VR128:$src),455 "movaps\t{$src, $dst|$dst, $src}", []>,456 VEX, WIG;457 def VMOVAPDrr_REV : VPDI<0x29, MRMDestReg, (outs VR128:$dst),458 (ins VR128:$src),459 "movapd\t{$src, $dst|$dst, $src}", []>,460 VEX, WIG;461 def VMOVUPSrr_REV : VPSI<0x11, MRMDestReg, (outs VR128:$dst),462 (ins VR128:$src),463 "movups\t{$src, $dst|$dst, $src}", []>,464 VEX, WIG;465 def VMOVUPDrr_REV : VPDI<0x11, MRMDestReg, (outs VR128:$dst),466 (ins VR128:$src),467 "movupd\t{$src, $dst|$dst, $src}", []>,468 VEX, WIG;469} // SchedRW470 471let SchedRW = [SchedWriteFMoveLS.YMM.RR] in {472 def VMOVAPSYrr_REV : VPSI<0x29, MRMDestReg, (outs VR256:$dst),473 (ins VR256:$src),474 "movaps\t{$src, $dst|$dst, $src}", []>,475 VEX, VEX_L, WIG;476 def VMOVAPDYrr_REV : VPDI<0x29, MRMDestReg, (outs VR256:$dst),477 (ins VR256:$src),478 "movapd\t{$src, $dst|$dst, $src}", []>,479 VEX, VEX_L, WIG;480 def VMOVUPSYrr_REV : VPSI<0x11, MRMDestReg, (outs VR256:$dst),481 (ins VR256:$src),482 "movups\t{$src, $dst|$dst, $src}", []>,483 VEX, VEX_L, WIG;484 def VMOVUPDYrr_REV : VPDI<0x11, MRMDestReg, (outs VR256:$dst),485 (ins VR256:$src),486 "movupd\t{$src, $dst|$dst, $src}", []>,487 VEX, VEX_L, WIG;488} // SchedRW489} // Predicate490 491// Reversed version with ".s" suffix for GAS compatibility.492def : InstAlias<"vmovaps.s\t{$src, $dst|$dst, $src}",493 (VMOVAPSrr_REV VR128:$dst, VR128:$src), 0>;494def : InstAlias<"vmovapd.s\t{$src, $dst|$dst, $src}",495 (VMOVAPDrr_REV VR128:$dst, VR128:$src), 0>;496def : InstAlias<"vmovups.s\t{$src, $dst|$dst, $src}",497 (VMOVUPSrr_REV VR128:$dst, VR128:$src), 0>;498def : InstAlias<"vmovupd.s\t{$src, $dst|$dst, $src}",499 (VMOVUPDrr_REV VR128:$dst, VR128:$src), 0>;500def : InstAlias<"vmovaps.s\t{$src, $dst|$dst, $src}",501 (VMOVAPSYrr_REV VR256:$dst, VR256:$src), 0>;502def : InstAlias<"vmovapd.s\t{$src, $dst|$dst, $src}",503 (VMOVAPDYrr_REV VR256:$dst, VR256:$src), 0>;504def : InstAlias<"vmovups.s\t{$src, $dst|$dst, $src}",505 (VMOVUPSYrr_REV VR256:$dst, VR256:$src), 0>;506def : InstAlias<"vmovupd.s\t{$src, $dst|$dst, $src}",507 (VMOVUPDYrr_REV VR256:$dst, VR256:$src), 0>;508 509let SchedRW = [SchedWriteFMoveLS.XMM.MR] in {510def MOVAPSmr : PSI<0x29, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src),511 "movaps\t{$src, $dst|$dst, $src}",512 [(alignedstore (v4f32 VR128:$src), addr:$dst)]>;513def MOVAPDmr : PDI<0x29, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src),514 "movapd\t{$src, $dst|$dst, $src}",515 [(alignedstore (v2f64 VR128:$src), addr:$dst)]>;516def MOVUPSmr : PSI<0x11, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src),517 "movups\t{$src, $dst|$dst, $src}",518 [(store (v4f32 VR128:$src), addr:$dst)]>;519def MOVUPDmr : PDI<0x11, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src),520 "movupd\t{$src, $dst|$dst, $src}",521 [(store (v2f64 VR128:$src), addr:$dst)]>;522} // SchedRW523 524// For disassembler525let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0,526 isMoveReg = 1, SchedRW = [SchedWriteFMoveLS.XMM.RR] in {527 def MOVAPSrr_REV : PSI<0x29, MRMDestReg, (outs VR128:$dst), (ins VR128:$src),528 "movaps\t{$src, $dst|$dst, $src}", []>;529 def MOVAPDrr_REV : PDI<0x29, MRMDestReg, (outs VR128:$dst), (ins VR128:$src),530 "movapd\t{$src, $dst|$dst, $src}", []>;531 def MOVUPSrr_REV : PSI<0x11, MRMDestReg, (outs VR128:$dst), (ins VR128:$src),532 "movups\t{$src, $dst|$dst, $src}", []>;533 def MOVUPDrr_REV : PDI<0x11, MRMDestReg, (outs VR128:$dst), (ins VR128:$src),534 "movupd\t{$src, $dst|$dst, $src}", []>;535}536 537// Reversed version with ".s" suffix for GAS compatibility.538def : InstAlias<"movaps.s\t{$src, $dst|$dst, $src}",539 (MOVAPSrr_REV VR128:$dst, VR128:$src), 0>;540def : InstAlias<"movapd.s\t{$src, $dst|$dst, $src}",541 (MOVAPDrr_REV VR128:$dst, VR128:$src), 0>;542def : InstAlias<"movups.s\t{$src, $dst|$dst, $src}",543 (MOVUPSrr_REV VR128:$dst, VR128:$src), 0>;544def : InstAlias<"movupd.s\t{$src, $dst|$dst, $src}",545 (MOVUPDrr_REV VR128:$dst, VR128:$src), 0>;546 547let Predicates = [HasAVX, NoVLX] in {548 // 256-bit load/store need to use floating point load/store in case we don't549 // have AVX2. Execution domain fixing will convert to integer if AVX2 is550 // available and changing the domain is beneficial.551 def : Pat<(alignedloadv4i64 addr:$src),552 (VMOVAPSYrm addr:$src)>;553 def : Pat<(alignedloadv8i32 addr:$src),554 (VMOVAPSYrm addr:$src)>;555 def : Pat<(alignedloadv16i16 addr:$src),556 (VMOVAPSYrm addr:$src)>;557 def : Pat<(alignedloadv32i8 addr:$src),558 (VMOVAPSYrm addr:$src)>;559 def : Pat<(loadv4i64 addr:$src),560 (VMOVUPSYrm addr:$src)>;561 def : Pat<(loadv8i32 addr:$src),562 (VMOVUPSYrm addr:$src)>;563 def : Pat<(loadv16i16 addr:$src),564 (VMOVUPSYrm addr:$src)>;565 def : Pat<(loadv32i8 addr:$src),566 (VMOVUPSYrm addr:$src)>;567 568 def : Pat<(alignedstore (v4i64 VR256:$src), addr:$dst),569 (VMOVAPSYmr addr:$dst, VR256:$src)>;570 def : Pat<(alignedstore (v8i32 VR256:$src), addr:$dst),571 (VMOVAPSYmr addr:$dst, VR256:$src)>;572 def : Pat<(alignedstore (v16i16 VR256:$src), addr:$dst),573 (VMOVAPSYmr addr:$dst, VR256:$src)>;574 def : Pat<(alignedstore (v32i8 VR256:$src), addr:$dst),575 (VMOVAPSYmr addr:$dst, VR256:$src)>;576 def : Pat<(store (v4i64 VR256:$src), addr:$dst),577 (VMOVUPSYmr addr:$dst, VR256:$src)>;578 def : Pat<(store (v8i32 VR256:$src), addr:$dst),579 (VMOVUPSYmr addr:$dst, VR256:$src)>;580 def : Pat<(store (v16i16 VR256:$src), addr:$dst),581 (VMOVUPSYmr addr:$dst, VR256:$src)>;582 def : Pat<(store (v32i8 VR256:$src), addr:$dst),583 (VMOVUPSYmr addr:$dst, VR256:$src)>;584 585 def : Pat<(alignedloadv8f16 addr:$src),586 (VMOVAPSrm addr:$src)>;587 def : Pat<(alignedloadv8bf16 addr:$src),588 (VMOVAPSrm addr:$src)>;589 def : Pat<(loadv8f16 addr:$src),590 (VMOVUPSrm addr:$src)>;591 def : Pat<(loadv8bf16 addr:$src),592 (VMOVUPSrm addr:$src)>;593 def : Pat<(alignedstore (v8f16 VR128:$src), addr:$dst),594 (VMOVAPSmr addr:$dst, VR128:$src)>;595 def : Pat<(alignedstore (v8bf16 VR128:$src), addr:$dst),596 (VMOVAPSmr addr:$dst, VR128:$src)>;597 def : Pat<(store (v8f16 VR128:$src), addr:$dst),598 (VMOVUPSmr addr:$dst, VR128:$src)>;599 def : Pat<(store (v8bf16 VR128:$src), addr:$dst),600 (VMOVUPSmr addr:$dst, VR128:$src)>;601 602 def : Pat<(alignedloadv16f16 addr:$src),603 (VMOVAPSYrm addr:$src)>;604 def : Pat<(alignedloadv16bf16 addr:$src),605 (VMOVAPSYrm addr:$src)>;606 def : Pat<(loadv16f16 addr:$src),607 (VMOVUPSYrm addr:$src)>;608 def : Pat<(loadv16bf16 addr:$src),609 (VMOVUPSYrm addr:$src)>;610 def : Pat<(alignedstore (v16f16 VR256:$src), addr:$dst),611 (VMOVAPSYmr addr:$dst, VR256:$src)>;612 def : Pat<(alignedstore (v16bf16 VR256:$src), addr:$dst),613 (VMOVAPSYmr addr:$dst, VR256:$src)>;614 def : Pat<(store (v16f16 VR256:$src), addr:$dst),615 (VMOVUPSYmr addr:$dst, VR256:$src)>;616 def : Pat<(store (v16bf16 VR256:$src), addr:$dst),617 (VMOVUPSYmr addr:$dst, VR256:$src)>;618}619 620// Use movaps / movups for SSE integer load / store (one byte shorter).621// The instructions selected below are then converted to MOVDQA/MOVDQU622// during the SSE domain pass.623let Predicates = [UseSSE1] in {624 def : Pat<(alignedloadv2i64 addr:$src),625 (MOVAPSrm addr:$src)>;626 def : Pat<(alignedloadv4i32 addr:$src),627 (MOVAPSrm addr:$src)>;628 def : Pat<(alignedloadv8i16 addr:$src),629 (MOVAPSrm addr:$src)>;630 def : Pat<(alignedloadv16i8 addr:$src),631 (MOVAPSrm addr:$src)>;632 def : Pat<(loadv2i64 addr:$src),633 (MOVUPSrm addr:$src)>;634 def : Pat<(loadv4i32 addr:$src),635 (MOVUPSrm addr:$src)>;636 def : Pat<(loadv8i16 addr:$src),637 (MOVUPSrm addr:$src)>;638 def : Pat<(loadv16i8 addr:$src),639 (MOVUPSrm addr:$src)>;640 641 def : Pat<(alignedstore (v2i64 VR128:$src), addr:$dst),642 (MOVAPSmr addr:$dst, VR128:$src)>;643 def : Pat<(alignedstore (v4i32 VR128:$src), addr:$dst),644 (MOVAPSmr addr:$dst, VR128:$src)>;645 def : Pat<(alignedstore (v8i16 VR128:$src), addr:$dst),646 (MOVAPSmr addr:$dst, VR128:$src)>;647 def : Pat<(alignedstore (v16i8 VR128:$src), addr:$dst),648 (MOVAPSmr addr:$dst, VR128:$src)>;649 def : Pat<(store (v2i64 VR128:$src), addr:$dst),650 (MOVUPSmr addr:$dst, VR128:$src)>;651 def : Pat<(store (v4i32 VR128:$src), addr:$dst),652 (MOVUPSmr addr:$dst, VR128:$src)>;653 def : Pat<(store (v8i16 VR128:$src), addr:$dst),654 (MOVUPSmr addr:$dst, VR128:$src)>;655 def : Pat<(store (v16i8 VR128:$src), addr:$dst),656 (MOVUPSmr addr:$dst, VR128:$src)>;657}658 659let Predicates = [UseSSE2] in {660 def : Pat<(alignedloadv8f16 addr:$src),661 (MOVAPSrm addr:$src)>;662 def : Pat<(loadv8f16 addr:$src),663 (MOVUPSrm addr:$src)>;664 def : Pat<(alignedstore (v8f16 VR128:$src), addr:$dst),665 (MOVAPSmr addr:$dst, VR128:$src)>;666 def : Pat<(store (v8f16 VR128:$src), addr:$dst),667 (MOVUPSmr addr:$dst, VR128:$src)>;668}669 670//===----------------------------------------------------------------------===//671// SSE 1 & 2 - Move Low packed FP Instructions672//===----------------------------------------------------------------------===//673 674multiclass sse12_mov_hilo_packed_base<bits<8>opc, SDPatternOperator pdnode,675 string base_opc, string asm_opr> {676 // No pattern as they need be special cased between high and low.677 let hasSideEffects = 0, mayLoad = 1 in678 def PSrm : PI<opc, MRMSrcMem,679 (outs VR128:$dst), (ins VR128:$src1, f64mem:$src2),680 !strconcat(base_opc, "s", asm_opr),681 [], SSEPackedSingle>, TB,682 Sched<[SchedWriteFShuffle.XMM.Folded, SchedWriteFShuffle.XMM.ReadAfterFold]>;683 684 def PDrm : PI<opc, MRMSrcMem,685 (outs VR128:$dst), (ins VR128:$src1, f64mem:$src2),686 !strconcat(base_opc, "d", asm_opr),687 [(set VR128:$dst, (v2f64 (pdnode VR128:$src1,688 (scalar_to_vector (loadf64 addr:$src2)))))],689 SSEPackedDouble>, TB, PD,690 Sched<[SchedWriteFShuffle.XMM.Folded, SchedWriteFShuffle.XMM.ReadAfterFold]>;691}692 693multiclass sse12_mov_hilo_packed<bits<8>opc, SDPatternOperator pdnode,694 string base_opc> {695 let Predicates = [UseAVX] in696 defm V#NAME : sse12_mov_hilo_packed_base<opc, pdnode, base_opc,697 "\t{$src2, $src1, $dst|$dst, $src1, $src2}">,698 VEX, VVVV, WIG;699 700 let Constraints = "$src1 = $dst" in701 defm NAME : sse12_mov_hilo_packed_base<opc, pdnode, base_opc,702 "\t{$src2, $dst|$dst, $src2}">;703}704 705defm MOVL : sse12_mov_hilo_packed<0x12, X86Movsd, "movlp">;706 707let SchedRW = [WriteFStore] in {708let Predicates = [UseAVX] in {709let mayStore = 1, hasSideEffects = 0 in710def VMOVLPSmr : VPSI<0x13, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src),711 "movlps\t{$src, $dst|$dst, $src}",712 []>,713 VEX, WIG;714def VMOVLPDmr : VPDI<0x13, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src),715 "movlpd\t{$src, $dst|$dst, $src}",716 [(store (f64 (extractelt (v2f64 VR128:$src),717 (iPTR 0))), addr:$dst)]>,718 VEX, WIG;719}// UseAVX720let mayStore = 1, hasSideEffects = 0 in721def MOVLPSmr : PSI<0x13, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src),722 "movlps\t{$src, $dst|$dst, $src}",723 []>;724def MOVLPDmr : PDI<0x13, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src),725 "movlpd\t{$src, $dst|$dst, $src}",726 [(store (f64 (extractelt (v2f64 VR128:$src),727 (iPTR 0))), addr:$dst)]>;728} // SchedRW729 730let Predicates = [UseSSE1] in {731 // This pattern helps select MOVLPS on SSE1 only targets. With SSE2 we'll732 // end up with a movsd or blend instead of shufp.733 // No need for aligned load, we're only loading 64-bits.734 def : Pat<(X86Shufp (v4f32 (simple_load addr:$src2)), VR128:$src1,735 (i8 -28)),736 (MOVLPSrm VR128:$src1, addr:$src2)>;737 def : Pat<(X86Shufp (v4f32 (X86vzload64 addr:$src2)), VR128:$src1, (i8 -28)),738 (MOVLPSrm VR128:$src1, addr:$src2)>;739 740 def : Pat<(v4f32 (X86vzload64 addr:$src)),741 (MOVLPSrm (v4f32 (V_SET0)), addr:$src)>;742 def : Pat<(X86vextractstore64 (v4f32 VR128:$src), addr:$dst),743 (MOVLPSmr addr:$dst, VR128:$src)>;744}745 746//===----------------------------------------------------------------------===//747// SSE 1 & 2 - Move Hi packed FP Instructions748//===----------------------------------------------------------------------===//749 750defm MOVH : sse12_mov_hilo_packed<0x16, X86Unpckl, "movhp">;751 752let SchedRW = [WriteFStore] in {753// v2f64 extract element 1 is always custom lowered to unpack high to low754// and extract element 0 so the non-store version isn't too horrible.755let Predicates = [UseAVX] in {756let mayStore = 1, hasSideEffects = 0 in757def VMOVHPSmr : VPSI<0x17, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src),758 "movhps\t{$src, $dst|$dst, $src}",759 []>, VEX, WIG;760def VMOVHPDmr : VPDI<0x17, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src),761 "movhpd\t{$src, $dst|$dst, $src}",762 [(store (f64 (extractelt763 (v2f64 (X86Unpckh VR128:$src, VR128:$src)),764 (iPTR 0))), addr:$dst)]>, VEX, WIG;765} // UseAVX766let mayStore = 1, hasSideEffects = 0 in767def MOVHPSmr : PSI<0x17, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src),768 "movhps\t{$src, $dst|$dst, $src}",769 []>;770def MOVHPDmr : PDI<0x17, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src),771 "movhpd\t{$src, $dst|$dst, $src}",772 [(store (f64 (extractelt773 (v2f64 (X86Unpckh VR128:$src, VR128:$src)),774 (iPTR 0))), addr:$dst)]>;775} // SchedRW776 777let Predicates = [UseAVX] in {778 // MOVHPD patterns779 def : Pat<(v2f64 (X86Unpckl VR128:$src1, (X86vzload64 addr:$src2))),780 (VMOVHPDrm VR128:$src1, addr:$src2)>;781 782 def : Pat<(store (f64 (extractelt783 (v2f64 (X86VPermilpi VR128:$src, (i8 1))),784 (iPTR 0))), addr:$dst),785 (VMOVHPDmr addr:$dst, VR128:$src)>;786 787 // MOVLPD patterns788 def : Pat<(v2f64 (X86Movsd VR128:$src1, (X86vzload64 addr:$src2))),789 (VMOVLPDrm VR128:$src1, addr:$src2)>;790}791 792let Predicates = [UseSSE1] in {793 // This pattern helps select MOVHPS on SSE1 only targets. With SSE2 we'll794 // end up with a movsd or blend instead of shufp.795 // No need for aligned load, we're only loading 64-bits.796 def : Pat<(X86Movlhps VR128:$src1, (v4f32 (simple_load addr:$src2))),797 (MOVHPSrm VR128:$src1, addr:$src2)>;798 def : Pat<(X86Movlhps VR128:$src1, (v4f32 (X86vzload64 addr:$src2))),799 (MOVHPSrm VR128:$src1, addr:$src2)>;800 801 def : Pat<(X86vextractstore64 (v4f32 (X86Movhlps VR128:$src, VR128:$src)),802 addr:$dst),803 (MOVHPSmr addr:$dst, VR128:$src)>;804}805 806let Predicates = [UseSSE2] in {807 // MOVHPD patterns808 def : Pat<(v2f64 (X86Unpckl VR128:$src1, (X86vzload64 addr:$src2))),809 (MOVHPDrm VR128:$src1, addr:$src2)>;810 811 def : Pat<(store (f64 (extractelt812 (v2f64 (X86Shufp VR128:$src, VR128:$src, (i8 1))),813 (iPTR 0))), addr:$dst),814 (MOVHPDmr addr:$dst, VR128:$src)>;815 816 // MOVLPD patterns817 def : Pat<(v2f64 (X86Movsd VR128:$src1, (X86vzload64 addr:$src2))),818 (MOVLPDrm VR128:$src1, addr:$src2)>;819}820 821let Predicates = [UseSSE2, NoSSE41_Or_OptForSize] in {822 // Use MOVLPD to load into the low bits from a full vector unless we can use823 // BLENDPD.824 def : Pat<(X86Movsd VR128:$src1, (v2f64 (simple_load addr:$src2))),825 (MOVLPDrm VR128:$src1, addr:$src2)>;826}827 828//===----------------------------------------------------------------------===//829// SSE 1 & 2 - Move Low to High and High to Low packed FP Instructions830//===----------------------------------------------------------------------===//831 832let Predicates = [UseAVX] in {833 def VMOVLHPSrr : VPSI<0x16, MRMSrcReg, (outs VR128:$dst),834 (ins VR128:$src1, VR128:$src2),835 "movlhps\t{$src2, $src1, $dst|$dst, $src1, $src2}",836 [(set VR128:$dst,837 (v4f32 (X86Movlhps VR128:$src1, VR128:$src2)))]>,838 VEX, VVVV, Sched<[SchedWriteFShuffle.XMM]>, WIG;839 let isCommutable = 1 in840 def VMOVHLPSrr : VPSI<0x12, MRMSrcReg, (outs VR128:$dst),841 (ins VR128:$src1, VR128:$src2),842 "movhlps\t{$src2, $src1, $dst|$dst, $src1, $src2}",843 [(set VR128:$dst,844 (v4f32 (X86Movhlps VR128:$src1, VR128:$src2)))]>,845 VEX, VVVV, Sched<[SchedWriteFShuffle.XMM]>, WIG;846}847let Constraints = "$src1 = $dst" in {848 def MOVLHPSrr : PSI<0x16, MRMSrcReg, (outs VR128:$dst),849 (ins VR128:$src1, VR128:$src2),850 "movlhps\t{$src2, $dst|$dst, $src2}",851 [(set VR128:$dst,852 (v4f32 (X86Movlhps VR128:$src1, VR128:$src2)))]>,853 Sched<[SchedWriteFShuffle.XMM]>;854 let isCommutable = 1 in855 def MOVHLPSrr : PSI<0x12, MRMSrcReg, (outs VR128:$dst),856 (ins VR128:$src1, VR128:$src2),857 "movhlps\t{$src2, $dst|$dst, $src2}",858 [(set VR128:$dst,859 (v4f32 (X86Movhlps VR128:$src1, VR128:$src2)))]>,860 Sched<[SchedWriteFShuffle.XMM]>;861}862 863//===----------------------------------------------------------------------===//864// SSE 1 & 2 - Conversion Instructions865//===----------------------------------------------------------------------===//866 867multiclass sse12_cvt_s<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC,868 SDPatternOperator OpNode, X86MemOperand x86memop, PatFrag ld_frag,869 string asm, string mem, X86FoldableSchedWrite sched,870 Domain d,871 SchedRead Int2Fpu = ReadDefault> {872 let ExeDomain = d in {873 def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src),874 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),875 [(set DstRC:$dst, (OpNode SrcRC:$src))]>,876 Sched<[sched, Int2Fpu]>;877 def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src),878 mem#"\t{$src, $dst|$dst, $src}",879 [(set DstRC:$dst, (OpNode (ld_frag addr:$src)))]>,880 Sched<[sched.Folded]>;881 }882}883 884multiclass sse12_cvt_p<bits<8> opc, RegisterClass RC, X86MemOperand x86memop,885 ValueType DstTy, ValueType SrcTy, PatFrag ld_frag,886 string asm, Domain d, X86FoldableSchedWrite sched> {887let hasSideEffects = 0, Uses = [MXCSR], mayRaiseFPException = 1 in {888 def rr : I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src), asm,889 [(set RC:$dst, (DstTy (any_sint_to_fp (SrcTy RC:$src))))], d>,890 Sched<[sched]>;891 let mayLoad = 1 in892 def rm : I<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), asm,893 [(set RC:$dst, (DstTy (any_sint_to_fp894 (SrcTy (ld_frag addr:$src)))))], d>,895 Sched<[sched.Folded]>;896}897}898 899multiclass sse12_vcvt_avx<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC,900 X86MemOperand x86memop, string asm, string mem,901 X86FoldableSchedWrite sched, Domain d> {902let hasSideEffects = 0, Predicates = [UseAVX], ExeDomain = d in {903 def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins DstRC:$src1, SrcRC:$src),904 !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>,905 Sched<[sched, ReadDefault, ReadInt2Fpu]>;906 let mayLoad = 1 in907 def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst),908 (ins DstRC:$src1, x86memop:$src),909 asm#"{"#mem#"}\t{$src, $src1, $dst|$dst, $src1, $src}", []>,910 Sched<[sched.Folded, sched.ReadAfterFold]>;911} // hasSideEffects = 0912}913 914let isCodeGenOnly = 1, Predicates = [UseAVX], Uses = [MXCSR], mayRaiseFPException = 1 in {915defm VCVTTSS2SI : sse12_cvt_s<0x2C, FR32, GR32, any_fp_to_sint, f32mem, loadf32,916 "cvttss2si", "cvttss2si",917 WriteCvtSS2I, SSEPackedSingle>,918 TB, XS, VEX, VEX_LIG;919defm VCVTTSS2SI64 : sse12_cvt_s<0x2C, FR32, GR64, any_fp_to_sint, f32mem, loadf32,920 "cvttss2si", "cvttss2si",921 WriteCvtSS2I, SSEPackedSingle>,922 TB, XS, VEX, REX_W, VEX_LIG;923defm VCVTTSD2SI : sse12_cvt_s<0x2C, FR64, GR32, any_fp_to_sint, f64mem, loadf64,924 "cvttsd2si", "cvttsd2si",925 WriteCvtSD2I, SSEPackedDouble>,926 TB, XD, VEX, VEX_LIG;927defm VCVTTSD2SI64 : sse12_cvt_s<0x2C, FR64, GR64, any_fp_to_sint, f64mem, loadf64,928 "cvttsd2si", "cvttsd2si",929 WriteCvtSD2I, SSEPackedDouble>,930 TB, XD, VEX, REX_W, VEX_LIG;931 932defm VCVTSS2SI : sse12_cvt_s<0x2D, FR32, GR32, lrint, f32mem, loadf32,933 "cvtss2si", "cvtss2si",934 WriteCvtSS2I, SSEPackedSingle>,935 TB, XS, VEX, VEX_LIG;936defm VCVTSS2SI64 : sse12_cvt_s<0x2D, FR32, GR64, llrint, f32mem, loadf32,937 "cvtss2si", "cvtss2si",938 WriteCvtSS2I, SSEPackedSingle>,939 TB, XS, VEX, REX_W, VEX_LIG;940defm VCVTSD2SI : sse12_cvt_s<0x2D, FR64, GR32, lrint, f64mem, loadf64,941 "cvtsd2si", "cvtsd2si",942 WriteCvtSD2I, SSEPackedDouble>,943 TB, XD, VEX, VEX_LIG;944defm VCVTSD2SI64 : sse12_cvt_s<0x2D, FR64, GR64, llrint, f64mem, loadf64,945 "cvtsd2si", "cvtsd2si",946 WriteCvtSD2I, SSEPackedDouble>,947 TB, XD, VEX, REX_W, VEX_LIG;948}949 950// The assembler can recognize rr 64-bit instructions by seeing a rxx951// register, but the same isn't true when only using memory operands,952// provide other assembly "l" and "q" forms to address this explicitly953// where appropriate to do so.954let isCodeGenOnly = 1 in {955defm VCVTSI2SS : sse12_vcvt_avx<0x2A, GR32, FR32, i32mem, "cvtsi2ss", "l",956 WriteCvtI2SS, SSEPackedSingle>, TB, XS, VEX, VVVV,957 VEX_LIG, SIMD_EXC;958defm VCVTSI642SS : sse12_vcvt_avx<0x2A, GR64, FR32, i64mem, "cvtsi2ss", "q",959 WriteCvtI2SS, SSEPackedSingle>, TB, XS, VEX, VVVV,960 REX_W, VEX_LIG, SIMD_EXC;961defm VCVTSI2SD : sse12_vcvt_avx<0x2A, GR32, FR64, i32mem, "cvtsi2sd", "l",962 WriteCvtI2SD, SSEPackedDouble>, TB, XD, VEX, VVVV,963 VEX_LIG;964defm VCVTSI642SD : sse12_vcvt_avx<0x2A, GR64, FR64, i64mem, "cvtsi2sd", "q",965 WriteCvtI2SD, SSEPackedDouble>, TB, XD, VEX, VVVV,966 REX_W, VEX_LIG, SIMD_EXC;967} // isCodeGenOnly = 1968 969let Predicates = [UseAVX] in {970 def : Pat<(f32 (any_sint_to_fp (loadi32 addr:$src))),971 (VCVTSI2SSrm (f32 (IMPLICIT_DEF)), addr:$src)>;972 def : Pat<(f32 (any_sint_to_fp (loadi64 addr:$src))),973 (VCVTSI642SSrm (f32 (IMPLICIT_DEF)), addr:$src)>;974 def : Pat<(f64 (any_sint_to_fp (loadi32 addr:$src))),975 (VCVTSI2SDrm (f64 (IMPLICIT_DEF)), addr:$src)>;976 def : Pat<(f64 (any_sint_to_fp (loadi64 addr:$src))),977 (VCVTSI642SDrm (f64 (IMPLICIT_DEF)), addr:$src)>;978 979 def : Pat<(f32 (any_sint_to_fp GR32:$src)),980 (VCVTSI2SSrr (f32 (IMPLICIT_DEF)), GR32:$src)>;981 def : Pat<(f32 (any_sint_to_fp GR64:$src)),982 (VCVTSI642SSrr (f32 (IMPLICIT_DEF)), GR64:$src)>;983 def : Pat<(f64 (any_sint_to_fp GR32:$src)),984 (VCVTSI2SDrr (f64 (IMPLICIT_DEF)), GR32:$src)>;985 def : Pat<(f64 (any_sint_to_fp GR64:$src)),986 (VCVTSI642SDrr (f64 (IMPLICIT_DEF)), GR64:$src)>;987 988 def : Pat<(i64 (lrint FR32:$src)), (VCVTSS2SI64rr FR32:$src)>;989 def : Pat<(i64 (lrint (loadf32 addr:$src))), (VCVTSS2SI64rm addr:$src)>;990 991 def : Pat<(i64 (lrint FR64:$src)), (VCVTSD2SI64rr FR64:$src)>;992 def : Pat<(i64 (lrint (loadf64 addr:$src))), (VCVTSD2SI64rm addr:$src)>;993}994 995let isCodeGenOnly = 1 in {996defm CVTTSS2SI : sse12_cvt_s<0x2C, FR32, GR32, any_fp_to_sint, f32mem, loadf32,997 "cvttss2si", "cvttss2si",998 WriteCvtSS2I, SSEPackedSingle>, TB, XS, SIMD_EXC;999defm CVTTSS2SI64 : sse12_cvt_s<0x2C, FR32, GR64, any_fp_to_sint, f32mem, loadf32,1000 "cvttss2si", "cvttss2si",1001 WriteCvtSS2I, SSEPackedSingle>, TB, XS, REX_W, SIMD_EXC;1002defm CVTTSD2SI : sse12_cvt_s<0x2C, FR64, GR32, any_fp_to_sint, f64mem, loadf64,1003 "cvttsd2si", "cvttsd2si",1004 WriteCvtSD2I, SSEPackedDouble>, TB, XD, SIMD_EXC;1005defm CVTTSD2SI64 : sse12_cvt_s<0x2C, FR64, GR64, any_fp_to_sint, f64mem, loadf64,1006 "cvttsd2si", "cvttsd2si",1007 WriteCvtSD2I, SSEPackedDouble>, TB, XD, REX_W, SIMD_EXC;1008 1009defm CVTSS2SI : sse12_cvt_s<0x2D, FR32, GR32, lrint, f32mem, loadf32,1010 "cvtss2si", "cvtss2si",1011 WriteCvtSS2I, SSEPackedSingle>, TB, XS, SIMD_EXC;1012defm CVTSS2SI64 : sse12_cvt_s<0x2D, FR32, GR64, llrint, f32mem, loadf32,1013 "cvtss2si", "cvtss2si",1014 WriteCvtSS2I, SSEPackedSingle>, TB, XS, REX_W, SIMD_EXC;1015defm CVTSD2SI : sse12_cvt_s<0x2D, FR64, GR32, lrint, f64mem, loadf64,1016 "cvtsd2si", "cvtsd2si",1017 WriteCvtSD2I, SSEPackedDouble>, TB, XD, SIMD_EXC;1018defm CVTSD2SI64 : sse12_cvt_s<0x2D, FR64, GR64, llrint, f64mem, loadf64,1019 "cvtsd2si", "cvtsd2si",1020 WriteCvtSD2I, SSEPackedDouble>, TB, XD, REX_W, SIMD_EXC;1021 1022defm CVTSI2SS : sse12_cvt_s<0x2A, GR32, FR32, any_sint_to_fp, i32mem, loadi32,1023 "cvtsi2ss", "cvtsi2ss{l}",1024 WriteCvtI2SS, SSEPackedSingle, ReadInt2Fpu>, TB, XS, SIMD_EXC;1025defm CVTSI642SS : sse12_cvt_s<0x2A, GR64, FR32, any_sint_to_fp, i64mem, loadi64,1026 "cvtsi2ss", "cvtsi2ss{q}",1027 WriteCvtI2SS, SSEPackedSingle, ReadInt2Fpu>, TB, XS, REX_W, SIMD_EXC;1028defm CVTSI2SD : sse12_cvt_s<0x2A, GR32, FR64, any_sint_to_fp, i32mem, loadi32,1029 "cvtsi2sd", "cvtsi2sd{l}",1030 WriteCvtI2SD, SSEPackedDouble, ReadInt2Fpu>, TB, XD;1031defm CVTSI642SD : sse12_cvt_s<0x2A, GR64, FR64, any_sint_to_fp, i64mem, loadi64,1032 "cvtsi2sd", "cvtsi2sd{q}",1033 WriteCvtI2SD, SSEPackedDouble, ReadInt2Fpu>, TB, XD, REX_W, SIMD_EXC;1034} // isCodeGenOnly = 11035 1036let Predicates = [UseSSE1] in {1037 def : Pat<(i64 (lrint FR32:$src)), (CVTSS2SI64rr FR32:$src)>;1038 def : Pat<(i64 (lrint (loadf32 addr:$src))), (CVTSS2SI64rm addr:$src)>;1039}1040 1041let Predicates = [UseSSE2] in {1042 def : Pat<(i64 (lrint FR64:$src)), (CVTSD2SI64rr FR64:$src)>;1043 def : Pat<(i64 (lrint (loadf64 addr:$src))), (CVTSD2SI64rm addr:$src)>;1044}1045 1046// Conversion Instructions Intrinsics - Match intrinsics which expect MM1047// and/or XMM operand(s).1048 1049multiclass sse12_cvt_sint<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC,1050 ValueType DstVT, ValueType SrcVT, SDNode OpNode,1051 Operand memop, PatFrags mem_frags, string asm,1052 X86FoldableSchedWrite sched, Domain d> {1053let ExeDomain = d in {1054 def rr_Int : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src),1055 !strconcat(asm, "\t{$src, $dst|$dst, $src}"),1056 [(set DstRC:$dst, (DstVT (OpNode (SrcVT SrcRC:$src))))]>,1057 Sched<[sched]>;1058 def rm_Int : SI<opc, MRMSrcMem, (outs DstRC:$dst), (ins memop:$src),1059 !strconcat(asm, "\t{$src, $dst|$dst, $src}"),1060 [(set DstRC:$dst, (DstVT (OpNode (SrcVT (mem_frags addr:$src)))))]>,1061 Sched<[sched.Folded]>;1062}1063}1064 1065multiclass sse12_cvt_sint_3addr<bits<8> opc, RegisterClass SrcRC,1066 RegisterClass DstRC, X86MemOperand x86memop,1067 string asm, string mem, X86FoldableSchedWrite sched,1068 Domain d, bit Is2Addr = 1> {1069let hasSideEffects = 0, ExeDomain = d in {1070 def rr_Int : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins DstRC:$src1, SrcRC:$src2),1071 !if(Is2Addr,1072 !strconcat(asm, "\t{$src2, $dst|$dst, $src2}"),1073 !strconcat(asm, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),1074 []>, Sched<[sched, ReadDefault, ReadInt2Fpu]>;1075 let mayLoad = 1 in1076 def rm_Int : SI<opc, MRMSrcMem, (outs DstRC:$dst),1077 (ins DstRC:$src1, x86memop:$src2),1078 !if(Is2Addr,1079 asm#"{"#mem#"}\t{$src2, $dst|$dst, $src2}",1080 asm#"{"#mem#"}\t{$src2, $src1, $dst|$dst, $src1, $src2}"),1081 []>, Sched<[sched.Folded, sched.ReadAfterFold]>;1082}1083}1084 1085let Uses = [MXCSR], mayRaiseFPException = 1 in {1086let Predicates = [UseAVX] in {1087defm VCVTSD2SI : sse12_cvt_sint<0x2D, VR128, GR32, i32, v2f64,1088 X86cvts2si, sdmem, sse_load_f64, "cvtsd2si",1089 WriteCvtSD2I, SSEPackedDouble>, TB, XD, VEX, VEX_LIG;1090defm VCVTSD2SI64 : sse12_cvt_sint<0x2D, VR128, GR64, i64, v2f64,1091 X86cvts2si, sdmem, sse_load_f64, "cvtsd2si",1092 WriteCvtSD2I, SSEPackedDouble>, TB, XD, VEX, REX_W, VEX_LIG;1093}1094defm CVTSD2SI : sse12_cvt_sint<0x2D, VR128, GR32, i32, v2f64, X86cvts2si,1095 sdmem, sse_load_f64, "cvtsd2si", WriteCvtSD2I,1096 SSEPackedDouble>, TB, XD;1097defm CVTSD2SI64 : sse12_cvt_sint<0x2D, VR128, GR64, i64, v2f64, X86cvts2si,1098 sdmem, sse_load_f64, "cvtsd2si", WriteCvtSD2I,1099 SSEPackedDouble>, TB, XD, REX_W;1100}1101 1102let Predicates = [UseAVX] in {1103defm VCVTSI2SS : sse12_cvt_sint_3addr<0x2A, GR32, VR128,1104 i32mem, "cvtsi2ss", "l", WriteCvtI2SS, SSEPackedSingle, 0>,1105 TB, XS, VEX, VVVV, VEX_LIG, SIMD_EXC;1106defm VCVTSI642SS : sse12_cvt_sint_3addr<0x2A, GR64, VR128,1107 i64mem, "cvtsi2ss", "q", WriteCvtI2SS, SSEPackedSingle, 0>,1108 TB, XS, VEX, VVVV, VEX_LIG, REX_W, SIMD_EXC;1109defm VCVTSI2SD : sse12_cvt_sint_3addr<0x2A, GR32, VR128,1110 i32mem, "cvtsi2sd", "l", WriteCvtI2SD, SSEPackedDouble, 0>,1111 TB, XD, VEX, VVVV, VEX_LIG;1112defm VCVTSI642SD : sse12_cvt_sint_3addr<0x2A, GR64, VR128,1113 i64mem, "cvtsi2sd", "q", WriteCvtI2SD, SSEPackedDouble, 0>,1114 TB, XD, VEX, VVVV, VEX_LIG, REX_W, SIMD_EXC;1115}1116let Constraints = "$src1 = $dst" in {1117 defm CVTSI2SS : sse12_cvt_sint_3addr<0x2A, GR32, VR128,1118 i32mem, "cvtsi2ss", "l", WriteCvtI2SS, SSEPackedSingle>,1119 TB, XS, SIMD_EXC;1120 defm CVTSI642SS : sse12_cvt_sint_3addr<0x2A, GR64, VR128,1121 i64mem, "cvtsi2ss", "q", WriteCvtI2SS, SSEPackedSingle>,1122 TB, XS, REX_W, SIMD_EXC;1123 defm CVTSI2SD : sse12_cvt_sint_3addr<0x2A, GR32, VR128,1124 i32mem, "cvtsi2sd", "l", WriteCvtI2SD, SSEPackedDouble>,1125 TB, XD;1126 defm CVTSI642SD : sse12_cvt_sint_3addr<0x2A, GR64, VR128,1127 i64mem, "cvtsi2sd", "q", WriteCvtI2SD, SSEPackedDouble>,1128 TB, XD, REX_W, SIMD_EXC;1129}1130 1131def : InstAlias<"vcvtsi2ss{l}\t{$src2, $src1, $dst|$dst, $src1, $src2}",1132 (VCVTSI2SSrr_Int VR128:$dst, VR128:$src1, GR32:$src2), 0, "att">;1133def : InstAlias<"vcvtsi2ss{q}\t{$src2, $src1, $dst|$dst, $src1, $src2}",1134 (VCVTSI642SSrr_Int VR128:$dst, VR128:$src1, GR64:$src2), 0, "att">;1135def : InstAlias<"vcvtsi2sd{l}\t{$src2, $src1, $dst|$dst, $src1, $src2}",1136 (VCVTSI2SDrr_Int VR128:$dst, VR128:$src1, GR32:$src2), 0, "att">;1137def : InstAlias<"vcvtsi2sd{q}\t{$src2, $src1, $dst|$dst, $src1, $src2}",1138 (VCVTSI642SDrr_Int VR128:$dst, VR128:$src1, GR64:$src2), 0, "att">;1139 1140def : InstAlias<"vcvtsi2ss\t{$src, $src1, $dst|$dst, $src1, $src}",1141 (VCVTSI2SSrm_Int VR128:$dst, VR128:$src1, i32mem:$src), 0, "att">;1142def : InstAlias<"vcvtsi2sd\t{$src, $src1, $dst|$dst, $src1, $src}",1143 (VCVTSI2SDrm_Int VR128:$dst, VR128:$src1, i32mem:$src), 0, "att">;1144 1145def : InstAlias<"cvtsi2ss{l}\t{$src, $dst|$dst, $src}",1146 (CVTSI2SSrr_Int VR128:$dst, GR32:$src), 0, "att">;1147def : InstAlias<"cvtsi2ss{q}\t{$src, $dst|$dst, $src}",1148 (CVTSI642SSrr_Int VR128:$dst, GR64:$src), 0, "att">;1149def : InstAlias<"cvtsi2sd{l}\t{$src, $dst|$dst, $src}",1150 (CVTSI2SDrr_Int VR128:$dst, GR32:$src), 0, "att">;1151def : InstAlias<"cvtsi2sd{q}\t{$src, $dst|$dst, $src}",1152 (CVTSI642SDrr_Int VR128:$dst, GR64:$src), 0, "att">;1153 1154def : InstAlias<"cvtsi2ss\t{$src, $dst|$dst, $src}",1155 (CVTSI2SSrm_Int VR128:$dst, i32mem:$src), 0, "att">;1156def : InstAlias<"cvtsi2sd\t{$src, $dst|$dst, $src}",1157 (CVTSI2SDrm_Int VR128:$dst, i32mem:$src), 0, "att">;1158 1159/// SSE 1 Only1160 1161// Aliases for intrinsics1162let Predicates = [UseAVX], Uses = [MXCSR], mayRaiseFPException = 1 in {1163defm VCVTTSS2SI : sse12_cvt_sint<0x2C, VR128, GR32, i32, v4f32, X86cvtts2Int,1164 ssmem, sse_load_f32, "cvttss2si",1165 WriteCvtSS2I, SSEPackedSingle>, TB, XS, VEX, VEX_LIG;1166defm VCVTTSS2SI64 : sse12_cvt_sint<0x2C, VR128, GR64, i64, v4f32,1167 X86cvtts2Int, ssmem, sse_load_f32,1168 "cvttss2si", WriteCvtSS2I, SSEPackedSingle>,1169 TB, XS, VEX, VEX_LIG, REX_W;1170defm VCVTTSD2SI : sse12_cvt_sint<0x2C, VR128, GR32, i32, v2f64, X86cvtts2Int,1171 sdmem, sse_load_f64, "cvttsd2si",1172 WriteCvtSS2I, SSEPackedDouble>, TB, XD, VEX, VEX_LIG;1173defm VCVTTSD2SI64 : sse12_cvt_sint<0x2C, VR128, GR64, i64, v2f64,1174 X86cvtts2Int, sdmem, sse_load_f64,1175 "cvttsd2si", WriteCvtSS2I, SSEPackedDouble>,1176 TB, XD, VEX, VEX_LIG, REX_W;1177}1178let Uses = [MXCSR], mayRaiseFPException = 1 in {1179defm CVTTSS2SI : sse12_cvt_sint<0x2C, VR128, GR32, i32, v4f32, X86cvtts2Int,1180 ssmem, sse_load_f32, "cvttss2si",1181 WriteCvtSS2I, SSEPackedSingle>, TB, XS;1182defm CVTTSS2SI64 : sse12_cvt_sint<0x2C, VR128, GR64, i64, v4f32,1183 X86cvtts2Int, ssmem, sse_load_f32,1184 "cvttss2si", WriteCvtSS2I, SSEPackedSingle>,1185 TB, XS, REX_W;1186defm CVTTSD2SI : sse12_cvt_sint<0x2C, VR128, GR32, i32, v2f64, X86cvtts2Int,1187 sdmem, sse_load_f64, "cvttsd2si",1188 WriteCvtSD2I, SSEPackedDouble>, TB, XD;1189defm CVTTSD2SI64 : sse12_cvt_sint<0x2C, VR128, GR64, i64, v2f64,1190 X86cvtts2Int, sdmem, sse_load_f64,1191 "cvttsd2si", WriteCvtSD2I, SSEPackedDouble>,1192 TB, XD, REX_W;1193}1194 1195def : InstAlias<"vcvttss2si{l}\t{$src, $dst|$dst, $src}",1196 (VCVTTSS2SIrr_Int GR32:$dst, VR128:$src), 0, "att">;1197def : InstAlias<"vcvttss2si{l}\t{$src, $dst|$dst, $src}",1198 (VCVTTSS2SIrm_Int GR32:$dst, f32mem:$src), 0, "att">;1199def : InstAlias<"vcvttsd2si{l}\t{$src, $dst|$dst, $src}",1200 (VCVTTSD2SIrr_Int GR32:$dst, VR128:$src), 0, "att">;1201def : InstAlias<"vcvttsd2si{l}\t{$src, $dst|$dst, $src}",1202 (VCVTTSD2SIrm_Int GR32:$dst, f64mem:$src), 0, "att">;1203def : InstAlias<"vcvttss2si{q}\t{$src, $dst|$dst, $src}",1204 (VCVTTSS2SI64rr_Int GR64:$dst, VR128:$src), 0, "att">;1205def : InstAlias<"vcvttss2si{q}\t{$src, $dst|$dst, $src}",1206 (VCVTTSS2SI64rm_Int GR64:$dst, f32mem:$src), 0, "att">;1207def : InstAlias<"vcvttsd2si{q}\t{$src, $dst|$dst, $src}",1208 (VCVTTSD2SI64rr_Int GR64:$dst, VR128:$src), 0, "att">;1209def : InstAlias<"vcvttsd2si{q}\t{$src, $dst|$dst, $src}",1210 (VCVTTSD2SI64rm_Int GR64:$dst, f64mem:$src), 0, "att">;1211 1212def : InstAlias<"cvttss2si{l}\t{$src, $dst|$dst, $src}",1213 (CVTTSS2SIrr_Int GR32:$dst, VR128:$src), 0, "att">;1214def : InstAlias<"cvttss2si{l}\t{$src, $dst|$dst, $src}",1215 (CVTTSS2SIrm_Int GR32:$dst, f32mem:$src), 0, "att">;1216def : InstAlias<"cvttsd2si{l}\t{$src, $dst|$dst, $src}",1217 (CVTTSD2SIrr_Int GR32:$dst, VR128:$src), 0, "att">;1218def : InstAlias<"cvttsd2si{l}\t{$src, $dst|$dst, $src}",1219 (CVTTSD2SIrm_Int GR32:$dst, f64mem:$src), 0, "att">;1220def : InstAlias<"cvttss2si{q}\t{$src, $dst|$dst, $src}",1221 (CVTTSS2SI64rr_Int GR64:$dst, VR128:$src), 0, "att">;1222def : InstAlias<"cvttss2si{q}\t{$src, $dst|$dst, $src}",1223 (CVTTSS2SI64rm_Int GR64:$dst, f32mem:$src), 0, "att">;1224def : InstAlias<"cvttsd2si{q}\t{$src, $dst|$dst, $src}",1225 (CVTTSD2SI64rr_Int GR64:$dst, VR128:$src), 0, "att">;1226def : InstAlias<"cvttsd2si{q}\t{$src, $dst|$dst, $src}",1227 (CVTTSD2SI64rm_Int GR64:$dst, f64mem:$src), 0, "att">;1228 1229let Predicates = [UseAVX], Uses = [MXCSR], mayRaiseFPException = 1 in {1230defm VCVTSS2SI : sse12_cvt_sint<0x2D, VR128, GR32, i32, v4f32, X86cvts2si,1231 ssmem, sse_load_f32, "cvtss2si",1232 WriteCvtSS2I, SSEPackedSingle>, TB, XS, VEX, VEX_LIG;1233defm VCVTSS2SI64 : sse12_cvt_sint<0x2D, VR128, GR64, i64, v4f32, X86cvts2si,1234 ssmem, sse_load_f32, "cvtss2si",1235 WriteCvtSS2I, SSEPackedSingle>, TB, XS, VEX, REX_W, VEX_LIG;1236}1237let Uses = [MXCSR], mayRaiseFPException = 1 in {1238defm CVTSS2SI : sse12_cvt_sint<0x2D, VR128, GR32, i32, v4f32, X86cvts2si,1239 ssmem, sse_load_f32, "cvtss2si",1240 WriteCvtSS2I, SSEPackedSingle>, TB, XS;1241defm CVTSS2SI64 : sse12_cvt_sint<0x2D, VR128, GR64, i64, v4f32, X86cvts2si,1242 ssmem, sse_load_f32, "cvtss2si",1243 WriteCvtSS2I, SSEPackedSingle>, TB, XS, REX_W;1244 1245defm VCVTDQ2PS : sse12_cvt_p<0x5B, VR128, i128mem, v4f32, v4i32, load,1246 "vcvtdq2ps\t{$src, $dst|$dst, $src}",1247 SSEPackedSingle, WriteCvtI2PS>,1248 TB, VEX, Requires<[HasAVX, NoVLX]>, WIG;1249defm VCVTDQ2PSY : sse12_cvt_p<0x5B, VR256, i256mem, v8f32, v8i32, load,1250 "vcvtdq2ps\t{$src, $dst|$dst, $src}",1251 SSEPackedSingle, WriteCvtI2PSY>,1252 TB, VEX, VEX_L, Requires<[HasAVX, NoVLX]>, WIG;1253 1254defm CVTDQ2PS : sse12_cvt_p<0x5B, VR128, i128mem, v4f32, v4i32, memop,1255 "cvtdq2ps\t{$src, $dst|$dst, $src}",1256 SSEPackedSingle, WriteCvtI2PS>,1257 TB, Requires<[UseSSE2]>;1258}1259 1260// AVX aliases1261def : InstAlias<"vcvtss2si{l}\t{$src, $dst|$dst, $src}",1262 (VCVTSS2SIrr_Int GR32:$dst, VR128:$src), 0, "att">;1263def : InstAlias<"vcvtss2si{l}\t{$src, $dst|$dst, $src}",1264 (VCVTSS2SIrm_Int GR32:$dst, ssmem:$src), 0, "att">;1265def : InstAlias<"vcvtsd2si{l}\t{$src, $dst|$dst, $src}",1266 (VCVTSD2SIrr_Int GR32:$dst, VR128:$src), 0, "att">;1267def : InstAlias<"vcvtsd2si{l}\t{$src, $dst|$dst, $src}",1268 (VCVTSD2SIrm_Int GR32:$dst, sdmem:$src), 0, "att">;1269def : InstAlias<"vcvtss2si{q}\t{$src, $dst|$dst, $src}",1270 (VCVTSS2SI64rr_Int GR64:$dst, VR128:$src), 0, "att">;1271def : InstAlias<"vcvtss2si{q}\t{$src, $dst|$dst, $src}",1272 (VCVTSS2SI64rm_Int GR64:$dst, ssmem:$src), 0, "att">;1273def : InstAlias<"vcvtsd2si{q}\t{$src, $dst|$dst, $src}",1274 (VCVTSD2SI64rr_Int GR64:$dst, VR128:$src), 0, "att">;1275def : InstAlias<"vcvtsd2si{q}\t{$src, $dst|$dst, $src}",1276 (VCVTSD2SI64rm_Int GR64:$dst, sdmem:$src), 0, "att">;1277 1278// SSE aliases1279def : InstAlias<"cvtss2si{l}\t{$src, $dst|$dst, $src}",1280 (CVTSS2SIrr_Int GR32:$dst, VR128:$src), 0, "att">;1281def : InstAlias<"cvtss2si{l}\t{$src, $dst|$dst, $src}",1282 (CVTSS2SIrm_Int GR32:$dst, ssmem:$src), 0, "att">;1283def : InstAlias<"cvtsd2si{l}\t{$src, $dst|$dst, $src}",1284 (CVTSD2SIrr_Int GR32:$dst, VR128:$src), 0, "att">;1285def : InstAlias<"cvtsd2si{l}\t{$src, $dst|$dst, $src}",1286 (CVTSD2SIrm_Int GR32:$dst, sdmem:$src), 0, "att">;1287def : InstAlias<"cvtss2si{q}\t{$src, $dst|$dst, $src}",1288 (CVTSS2SI64rr_Int GR64:$dst, VR128:$src), 0, "att">;1289def : InstAlias<"cvtss2si{q}\t{$src, $dst|$dst, $src}",1290 (CVTSS2SI64rm_Int GR64:$dst, ssmem:$src), 0, "att">;1291def : InstAlias<"cvtsd2si{q}\t{$src, $dst|$dst, $src}",1292 (CVTSD2SI64rr_Int GR64:$dst, VR128:$src), 0, "att">;1293def : InstAlias<"cvtsd2si{q}\t{$src, $dst|$dst, $src}",1294 (CVTSD2SI64rm_Int GR64:$dst, sdmem:$src), 0, "att">;1295 1296/// SSE 2 Only1297 1298// Convert scalar double to scalar single1299let isCodeGenOnly = 1, hasSideEffects = 0, Predicates = [UseAVX],1300 ExeDomain = SSEPackedSingle in {1301def VCVTSD2SSrr : VSDI<0x5A, MRMSrcReg, (outs FR32:$dst),1302 (ins FR32:$src1, FR64:$src2),1303 "cvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,1304 VEX, VVVV, VEX_LIG, WIG,1305 Sched<[WriteCvtSD2SS]>, SIMD_EXC;1306let mayLoad = 1 in1307def VCVTSD2SSrm : I<0x5A, MRMSrcMem, (outs FR32:$dst),1308 (ins FR32:$src1, f64mem:$src2),1309 "vcvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,1310 TB, XD, VEX, VVVV, VEX_LIG, WIG,1311 Sched<[WriteCvtSD2SS.Folded, WriteCvtSD2SS.ReadAfterFold]>, SIMD_EXC;1312}1313 1314def : Pat<(f32 (any_fpround FR64:$src)),1315 (VCVTSD2SSrr (f32 (IMPLICIT_DEF)), FR64:$src)>,1316 Requires<[UseAVX]>;1317 1318let isCodeGenOnly = 1, ExeDomain = SSEPackedSingle in {1319def CVTSD2SSrr : SDI<0x5A, MRMSrcReg, (outs FR32:$dst), (ins FR64:$src),1320 "cvtsd2ss\t{$src, $dst|$dst, $src}",1321 [(set FR32:$dst, (any_fpround FR64:$src))]>,1322 Sched<[WriteCvtSD2SS]>, SIMD_EXC;1323def CVTSD2SSrm : I<0x5A, MRMSrcMem, (outs FR32:$dst), (ins f64mem:$src),1324 "cvtsd2ss\t{$src, $dst|$dst, $src}",1325 [(set FR32:$dst, (any_fpround (loadf64 addr:$src)))]>,1326 TB, XD, Requires<[UseSSE2, OptForSize]>,1327 Sched<[WriteCvtSD2SS.Folded, WriteCvtSD2SS.ReadAfterFold]>, SIMD_EXC;1328}1329 1330let Uses = [MXCSR], mayRaiseFPException = 1, ExeDomain = SSEPackedSingle in {1331def VCVTSD2SSrr_Int: I<0x5A, MRMSrcReg,1332 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2),1333 "vcvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}",1334 [(set VR128:$dst,1335 (v4f32 (X86frounds VR128:$src1, (v2f64 VR128:$src2))))]>,1336 TB, XD, VEX, VVVV, VEX_LIG, WIG, Requires<[UseAVX]>,1337 Sched<[WriteCvtSD2SS]>;1338def VCVTSD2SSrm_Int: I<0x5A, MRMSrcMem,1339 (outs VR128:$dst), (ins VR128:$src1, sdmem:$src2),1340 "vcvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}",1341 [(set VR128:$dst,1342 (v4f32 (X86frounds VR128:$src1, (sse_load_f64 addr:$src2))))]>,1343 TB, XD, VEX, VVVV, VEX_LIG, WIG, Requires<[UseAVX]>,1344 Sched<[WriteCvtSD2SS.Folded, WriteCvtSD2SS.ReadAfterFold]>;1345let Constraints = "$src1 = $dst" in {1346def CVTSD2SSrr_Int: I<0x5A, MRMSrcReg,1347 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2),1348 "cvtsd2ss\t{$src2, $dst|$dst, $src2}",1349 [(set VR128:$dst,1350 (v4f32 (X86frounds VR128:$src1, (v2f64 VR128:$src2))))]>,1351 TB, XD, Requires<[UseSSE2]>, Sched<[WriteCvtSD2SS]>;1352def CVTSD2SSrm_Int: I<0x5A, MRMSrcMem,1353 (outs VR128:$dst), (ins VR128:$src1, sdmem:$src2),1354 "cvtsd2ss\t{$src2, $dst|$dst, $src2}",1355 [(set VR128:$dst,1356 (v4f32 (X86frounds VR128:$src1, (sse_load_f64 addr:$src2))))]>,1357 TB, XD, Requires<[UseSSE2]>,1358 Sched<[WriteCvtSD2SS.Folded, WriteCvtSD2SS.ReadAfterFold]>;1359}1360}1361 1362// Convert scalar single to scalar double1363// SSE2 instructions with XS prefix1364let isCodeGenOnly = 1, hasSideEffects = 0, ExeDomain = SSEPackedSingle in {1365def VCVTSS2SDrr : I<0x5A, MRMSrcReg, (outs FR64:$dst),1366 (ins FR64:$src1, FR32:$src2),1367 "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,1368 TB, XS, VEX, VVVV, VEX_LIG, WIG,1369 Sched<[WriteCvtSS2SD]>, Requires<[UseAVX]>, SIMD_EXC;1370let mayLoad = 1 in1371def VCVTSS2SDrm : I<0x5A, MRMSrcMem, (outs FR64:$dst),1372 (ins FR64:$src1, f32mem:$src2),1373 "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,1374 TB, XS, VEX, VVVV, VEX_LIG, WIG,1375 Sched<[WriteCvtSS2SD.Folded, WriteCvtSS2SD.ReadAfterFold]>,1376 Requires<[UseAVX, OptForSize]>, SIMD_EXC;1377} // isCodeGenOnly = 1, hasSideEffects = 01378 1379def : Pat<(f64 (any_fpextend FR32:$src)),1380 (VCVTSS2SDrr (f64 (IMPLICIT_DEF)), FR32:$src)>, Requires<[UseAVX]>;1381def : Pat<(any_fpextend (loadf32 addr:$src)),1382 (VCVTSS2SDrm (f64 (IMPLICIT_DEF)), addr:$src)>, Requires<[UseAVX, OptForSize]>;1383 1384let isCodeGenOnly = 1, ExeDomain = SSEPackedSingle in {1385def CVTSS2SDrr : I<0x5A, MRMSrcReg, (outs FR64:$dst), (ins FR32:$src),1386 "cvtss2sd\t{$src, $dst|$dst, $src}",1387 [(set FR64:$dst, (any_fpextend FR32:$src))]>,1388 TB, XS, Requires<[UseSSE2]>, Sched<[WriteCvtSS2SD]>, SIMD_EXC;1389def CVTSS2SDrm : I<0x5A, MRMSrcMem, (outs FR64:$dst), (ins f32mem:$src),1390 "cvtss2sd\t{$src, $dst|$dst, $src}",1391 [(set FR64:$dst, (any_fpextend (loadf32 addr:$src)))]>,1392 TB, XS, Requires<[UseSSE2, OptForSize]>,1393 Sched<[WriteCvtSS2SD.Folded, WriteCvtSS2SD.ReadAfterFold]>, SIMD_EXC;1394} // isCodeGenOnly = 11395 1396let hasSideEffects = 0, Uses = [MXCSR], mayRaiseFPException = 1,1397 ExeDomain = SSEPackedSingle in {1398def VCVTSS2SDrr_Int: I<0x5A, MRMSrcReg,1399 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2),1400 "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}",1401 []>, TB, XS, VEX, VVVV, VEX_LIG, WIG,1402 Requires<[HasAVX]>, Sched<[WriteCvtSS2SD]>;1403let mayLoad = 1 in1404def VCVTSS2SDrm_Int: I<0x5A, MRMSrcMem,1405 (outs VR128:$dst), (ins VR128:$src1, ssmem:$src2),1406 "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}",1407 []>, TB, XS, VEX, VVVV, VEX_LIG, WIG, Requires<[HasAVX]>,1408 Sched<[WriteCvtSS2SD.Folded, WriteCvtSS2SD.ReadAfterFold]>;1409let Constraints = "$src1 = $dst" in { // SSE2 instructions with XS prefix1410def CVTSS2SDrr_Int: I<0x5A, MRMSrcReg,1411 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2),1412 "cvtss2sd\t{$src2, $dst|$dst, $src2}",1413 []>, TB, XS, Requires<[UseSSE2]>,1414 Sched<[WriteCvtSS2SD]>;1415let mayLoad = 1 in1416def CVTSS2SDrm_Int: I<0x5A, MRMSrcMem,1417 (outs VR128:$dst), (ins VR128:$src1, ssmem:$src2),1418 "cvtss2sd\t{$src2, $dst|$dst, $src2}",1419 []>, TB, XS, Requires<[UseSSE2]>,1420 Sched<[WriteCvtSS2SD.Folded, WriteCvtSS2SD.ReadAfterFold]>;1421}1422} // hasSideEffects = 01423 1424// Patterns used for matching (v)cvtsi2ss, (v)cvtsi2sd, (v)cvtsd2ss and1425// (v)cvtss2sd intrinsic sequences from clang which produce unnecessary1426// vmovs{s,d} instructions1427let Predicates = [UseAVX] in {1428def : Pat<(v4f32 (X86Movss1429 (v4f32 VR128:$dst),1430 (v4f32 (scalar_to_vector1431 (f32 (any_fpround (f64 (extractelt VR128:$src, (iPTR 0))))))))),1432 (VCVTSD2SSrr_Int VR128:$dst, VR128:$src)>;1433 1434def : Pat<(v2f64 (X86Movsd1435 (v2f64 VR128:$dst),1436 (v2f64 (scalar_to_vector1437 (f64 (any_fpextend (f32 (extractelt VR128:$src, (iPTR 0))))))))),1438 (VCVTSS2SDrr_Int VR128:$dst, VR128:$src)>;1439 1440def : Pat<(v4f32 (X86Movss1441 (v4f32 VR128:$dst),1442 (v4f32 (scalar_to_vector (f32 (any_sint_to_fp GR64:$src)))))),1443 (VCVTSI642SSrr_Int VR128:$dst, GR64:$src)>;1444 1445def : Pat<(v4f32 (X86Movss1446 (v4f32 VR128:$dst),1447 (v4f32 (scalar_to_vector (f32 (any_sint_to_fp (loadi64 addr:$src))))))),1448 (VCVTSI642SSrm_Int VR128:$dst, addr:$src)>;1449 1450def : Pat<(v4f32 (X86Movss1451 (v4f32 VR128:$dst),1452 (v4f32 (scalar_to_vector (f32 (any_sint_to_fp GR32:$src)))))),1453 (VCVTSI2SSrr_Int VR128:$dst, GR32:$src)>;1454 1455def : Pat<(v4f32 (X86Movss1456 (v4f32 VR128:$dst),1457 (v4f32 (scalar_to_vector (f32 (any_sint_to_fp (loadi32 addr:$src))))))),1458 (VCVTSI2SSrm_Int VR128:$dst, addr:$src)>;1459 1460def : Pat<(v2f64 (X86Movsd1461 (v2f64 VR128:$dst),1462 (v2f64 (scalar_to_vector (f64 (any_sint_to_fp GR64:$src)))))),1463 (VCVTSI642SDrr_Int VR128:$dst, GR64:$src)>;1464 1465def : Pat<(v2f64 (X86Movsd1466 (v2f64 VR128:$dst),1467 (v2f64 (scalar_to_vector (f64 (any_sint_to_fp (loadi64 addr:$src))))))),1468 (VCVTSI642SDrm_Int VR128:$dst, addr:$src)>;1469 1470def : Pat<(v2f64 (X86Movsd1471 (v2f64 VR128:$dst),1472 (v2f64 (scalar_to_vector (f64 (any_sint_to_fp GR32:$src)))))),1473 (VCVTSI2SDrr_Int VR128:$dst, GR32:$src)>;1474 1475def : Pat<(v2f64 (X86Movsd1476 (v2f64 VR128:$dst),1477 (v2f64 (scalar_to_vector (f64 (any_sint_to_fp (loadi32 addr:$src))))))),1478 (VCVTSI2SDrm_Int VR128:$dst, addr:$src)>;1479} // Predicates = [UseAVX]1480 1481let Predicates = [UseSSE2] in {1482def : Pat<(v4f32 (X86Movss1483 (v4f32 VR128:$dst),1484 (v4f32 (scalar_to_vector1485 (f32 (any_fpround (f64 (extractelt VR128:$src, (iPTR 0))))))))),1486 (CVTSD2SSrr_Int VR128:$dst, VR128:$src)>;1487 1488def : Pat<(v2f64 (X86Movsd1489 (v2f64 VR128:$dst),1490 (v2f64 (scalar_to_vector1491 (f64 (any_fpextend (f32 (extractelt VR128:$src, (iPTR 0))))))))),1492 (CVTSS2SDrr_Int VR128:$dst, VR128:$src)>;1493 1494def : Pat<(v2f64 (X86Movsd1495 (v2f64 VR128:$dst),1496 (v2f64 (scalar_to_vector (f64 (any_sint_to_fp GR64:$src)))))),1497 (CVTSI642SDrr_Int VR128:$dst, GR64:$src)>;1498 1499def : Pat<(v2f64 (X86Movsd1500 (v2f64 VR128:$dst),1501 (v2f64 (scalar_to_vector (f64 (any_sint_to_fp (loadi64 addr:$src))))))),1502 (CVTSI642SDrm_Int VR128:$dst, addr:$src)>;1503 1504def : Pat<(v2f64 (X86Movsd1505 (v2f64 VR128:$dst),1506 (v2f64 (scalar_to_vector (f64 (any_sint_to_fp GR32:$src)))))),1507 (CVTSI2SDrr_Int VR128:$dst, GR32:$src)>;1508 1509def : Pat<(v2f64 (X86Movsd1510 (v2f64 VR128:$dst),1511 (v2f64 (scalar_to_vector (f64 (any_sint_to_fp (loadi32 addr:$src))))))),1512 (CVTSI2SDrm_Int VR128:$dst, addr:$src)>;1513} // Predicates = [UseSSE2]1514 1515let Predicates = [UseSSE1] in {1516def : Pat<(v4f32 (X86Movss1517 (v4f32 VR128:$dst),1518 (v4f32 (scalar_to_vector (f32 (any_sint_to_fp GR64:$src)))))),1519 (CVTSI642SSrr_Int VR128:$dst, GR64:$src)>;1520 1521def : Pat<(v4f32 (X86Movss1522 (v4f32 VR128:$dst),1523 (v4f32 (scalar_to_vector (f32 (any_sint_to_fp (loadi64 addr:$src))))))),1524 (CVTSI642SSrm_Int VR128:$dst, addr:$src)>;1525 1526def : Pat<(v4f32 (X86Movss1527 (v4f32 VR128:$dst),1528 (v4f32 (scalar_to_vector (f32 (any_sint_to_fp GR32:$src)))))),1529 (CVTSI2SSrr_Int VR128:$dst, GR32:$src)>;1530 1531def : Pat<(v4f32 (X86Movss1532 (v4f32 VR128:$dst),1533 (v4f32 (scalar_to_vector (f32 (any_sint_to_fp (loadi32 addr:$src))))))),1534 (CVTSI2SSrm_Int VR128:$dst, addr:$src)>;1535} // Predicates = [UseSSE1]1536 1537let Predicates = [HasAVX, NoVLX] in {1538// Convert packed single/double fp to doubleword1539def VCVTPS2DQrr : VPDI<0x5B, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),1540 "cvtps2dq\t{$src, $dst|$dst, $src}",1541 [(set VR128:$dst, (v4i32 (X86cvtp2Int (v4f32 VR128:$src))))]>,1542 VEX, Sched<[WriteCvtPS2I]>, WIG, SIMD_EXC;1543def VCVTPS2DQrm : VPDI<0x5B, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),1544 "cvtps2dq\t{$src, $dst|$dst, $src}",1545 [(set VR128:$dst,1546 (v4i32 (X86cvtp2Int (loadv4f32 addr:$src))))]>,1547 VEX, Sched<[WriteCvtPS2ILd]>, WIG, SIMD_EXC;1548def VCVTPS2DQYrr : VPDI<0x5B, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),1549 "cvtps2dq\t{$src, $dst|$dst, $src}",1550 [(set VR256:$dst,1551 (v8i32 (X86cvtp2Int (v8f32 VR256:$src))))]>,1552 VEX, VEX_L, Sched<[WriteCvtPS2IY]>, WIG, SIMD_EXC;1553def VCVTPS2DQYrm : VPDI<0x5B, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src),1554 "cvtps2dq\t{$src, $dst|$dst, $src}",1555 [(set VR256:$dst,1556 (v8i32 (X86cvtp2Int (loadv8f32 addr:$src))))]>,1557 VEX, VEX_L, Sched<[WriteCvtPS2IYLd]>, WIG, SIMD_EXC;1558}1559def CVTPS2DQrr : PDI<0x5B, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),1560 "cvtps2dq\t{$src, $dst|$dst, $src}",1561 [(set VR128:$dst, (v4i32 (X86cvtp2Int (v4f32 VR128:$src))))]>,1562 Sched<[WriteCvtPS2I]>, SIMD_EXC;1563def CVTPS2DQrm : PDI<0x5B, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),1564 "cvtps2dq\t{$src, $dst|$dst, $src}",1565 [(set VR128:$dst,1566 (v4i32 (X86cvtp2Int (memopv4f32 addr:$src))))]>,1567 Sched<[WriteCvtPS2ILd]>, SIMD_EXC;1568 1569// Convert Packed Double FP to Packed DW Integers1570let Predicates = [HasAVX, NoVLX], Uses = [MXCSR], mayRaiseFPException = 1 in {1571// The assembler can recognize rr 256-bit instructions by seeing a ymm1572// register, but the same isn't true when using memory operands instead.1573// Provide other assembly rr and rm forms to address this explicitly.1574def VCVTPD2DQrr : SDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),1575 "vcvtpd2dq\t{$src, $dst|$dst, $src}",1576 [(set VR128:$dst,1577 (v4i32 (X86cvtp2Int (v2f64 VR128:$src))))]>,1578 VEX, Sched<[WriteCvtPD2I]>, WIG;1579 1580// XMM only1581def VCVTPD2DQrm : SDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),1582 "vcvtpd2dq{x}\t{$src, $dst|$dst, $src}",1583 [(set VR128:$dst,1584 (v4i32 (X86cvtp2Int (loadv2f64 addr:$src))))]>, VEX,1585 Sched<[WriteCvtPD2ILd]>, WIG;1586 1587// YMM only1588def VCVTPD2DQYrr : SDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR256:$src),1589 "vcvtpd2dq\t{$src, $dst|$dst, $src}",1590 [(set VR128:$dst,1591 (v4i32 (X86cvtp2Int (v4f64 VR256:$src))))]>,1592 VEX, VEX_L, Sched<[WriteCvtPD2IY]>, WIG;1593def VCVTPD2DQYrm : SDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f256mem:$src),1594 "vcvtpd2dq{y}\t{$src, $dst|$dst, $src}",1595 [(set VR128:$dst,1596 (v4i32 (X86cvtp2Int (loadv4f64 addr:$src))))]>,1597 VEX, VEX_L, Sched<[WriteCvtPD2IYLd]>, WIG;1598}1599 1600let Predicates = [HasAVX] in {1601 def : Pat<(v4i32 (lrint VR128:$src)), (VCVTPS2DQrr VR128:$src)>;1602 def : Pat<(v4i32 (lrint (loadv4f32 addr:$src))), (VCVTPS2DQrm addr:$src)>;1603 def : Pat<(v8i32 (lrint VR256:$src)), (VCVTPS2DQYrr VR256:$src)>;1604 def : Pat<(v8i32 (lrint (loadv8f32 addr:$src))), (VCVTPS2DQYrm addr:$src)>;1605 def : Pat<(v4i32 (lrint VR256:$src)), (VCVTPD2DQYrr VR256:$src)>;1606 def : Pat<(v4i32 (lrint (loadv4f64 addr:$src))), (VCVTPD2DQYrm addr:$src)>;1607}1608 1609let Predicates = [UseSSE2] in {1610 def : Pat<(v4i32 (lrint VR128:$src)), (CVTPS2DQrr VR128:$src)>;1611 def : Pat<(v4i32 (lrint (loadv4f32 addr:$src))), (CVTPS2DQrm addr:$src)>;1612}1613 1614def : InstAlias<"vcvtpd2dqx\t{$src, $dst|$dst, $src}",1615 (VCVTPD2DQrr VR128:$dst, VR128:$src), 0, "att">;1616def : InstAlias<"vcvtpd2dqy\t{$src, $dst|$dst, $src}",1617 (VCVTPD2DQYrr VR128:$dst, VR256:$src), 0, "att">;1618 1619def CVTPD2DQrm : SDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),1620 "cvtpd2dq\t{$src, $dst|$dst, $src}",1621 [(set VR128:$dst,1622 (v4i32 (X86cvtp2Int (memopv2f64 addr:$src))))]>,1623 Sched<[WriteCvtPD2ILd]>, SIMD_EXC;1624def CVTPD2DQrr : SDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),1625 "cvtpd2dq\t{$src, $dst|$dst, $src}",1626 [(set VR128:$dst,1627 (v4i32 (X86cvtp2Int (v2f64 VR128:$src))))]>,1628 Sched<[WriteCvtPD2I]>, SIMD_EXC;1629 1630// Convert with truncation packed single/double fp to doubleword1631// SSE2 packed instructions with XS prefix1632let Uses = [MXCSR], mayRaiseFPException = 1 in {1633let Predicates = [HasAVX, NoVLX] in {1634def VCVTTPS2DQrr : VS2SI<0x5B, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),1635 "cvttps2dq\t{$src, $dst|$dst, $src}",1636 [(set VR128:$dst,1637 (v4i32 (X86any_cvttp2si (v4f32 VR128:$src))))]>,1638 VEX, Sched<[WriteCvtPS2I]>, WIG;1639def VCVTTPS2DQrm : VS2SI<0x5B, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),1640 "cvttps2dq\t{$src, $dst|$dst, $src}",1641 [(set VR128:$dst,1642 (v4i32 (X86any_cvttp2si (loadv4f32 addr:$src))))]>,1643 VEX, Sched<[WriteCvtPS2ILd]>, WIG;1644def VCVTTPS2DQYrr : VS2SI<0x5B, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),1645 "cvttps2dq\t{$src, $dst|$dst, $src}",1646 [(set VR256:$dst,1647 (v8i32 (X86any_cvttp2si (v8f32 VR256:$src))))]>,1648 VEX, VEX_L, Sched<[WriteCvtPS2IY]>, WIG;1649def VCVTTPS2DQYrm : VS2SI<0x5B, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src),1650 "cvttps2dq\t{$src, $dst|$dst, $src}",1651 [(set VR256:$dst,1652 (v8i32 (X86any_cvttp2si (loadv8f32 addr:$src))))]>,1653 VEX, VEX_L,1654 Sched<[WriteCvtPS2IYLd]>, WIG;1655}1656 1657def CVTTPS2DQrr : S2SI<0x5B, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),1658 "cvttps2dq\t{$src, $dst|$dst, $src}",1659 [(set VR128:$dst,1660 (v4i32 (X86any_cvttp2si (v4f32 VR128:$src))))]>,1661 Sched<[WriteCvtPS2I]>;1662def CVTTPS2DQrm : S2SI<0x5B, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),1663 "cvttps2dq\t{$src, $dst|$dst, $src}",1664 [(set VR128:$dst,1665 (v4i32 (X86any_cvttp2si (memopv4f32 addr:$src))))]>,1666 Sched<[WriteCvtPS2ILd]>;1667}1668 1669// The assembler can recognize rr 256-bit instructions by seeing a ymm1670// register, but the same isn't true when using memory operands instead.1671// Provide other assembly rr and rm forms to address this explicitly.1672let Predicates = [HasAVX, NoVLX], Uses = [MXCSR], mayRaiseFPException = 1 in {1673// XMM only1674def VCVTTPD2DQrr : VPDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),1675 "cvttpd2dq\t{$src, $dst|$dst, $src}",1676 [(set VR128:$dst,1677 (v4i32 (X86any_cvttp2si (v2f64 VR128:$src))))]>,1678 VEX, Sched<[WriteCvtPD2I]>, WIG;1679def VCVTTPD2DQrm : VPDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),1680 "cvttpd2dq{x}\t{$src, $dst|$dst, $src}",1681 [(set VR128:$dst,1682 (v4i32 (X86any_cvttp2si (loadv2f64 addr:$src))))]>,1683 VEX, Sched<[WriteCvtPD2ILd]>, WIG;1684 1685// YMM only1686def VCVTTPD2DQYrr : VPDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR256:$src),1687 "cvttpd2dq\t{$src, $dst|$dst, $src}",1688 [(set VR128:$dst,1689 (v4i32 (X86any_cvttp2si (v4f64 VR256:$src))))]>,1690 VEX, VEX_L, Sched<[WriteCvtPD2IY]>, WIG;1691def VCVTTPD2DQYrm : VPDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f256mem:$src),1692 "cvttpd2dq{y}\t{$src, $dst|$dst, $src}",1693 [(set VR128:$dst,1694 (v4i32 (X86any_cvttp2si (loadv4f64 addr:$src))))]>,1695 VEX, VEX_L, Sched<[WriteCvtPD2IYLd]>, WIG;1696} // Predicates = [HasAVX, NoVLX]1697 1698def : InstAlias<"vcvttpd2dqx\t{$src, $dst|$dst, $src}",1699 (VCVTTPD2DQrr VR128:$dst, VR128:$src), 0, "att">;1700def : InstAlias<"vcvttpd2dqy\t{$src, $dst|$dst, $src}",1701 (VCVTTPD2DQYrr VR128:$dst, VR256:$src), 0, "att">;1702 1703let Predicates = [HasAVX, NoVLX] in {1704 def : Pat<(v4i32 (any_fp_to_sint (v4f64 VR256:$src))),1705 (VCVTTPD2DQYrr VR256:$src)>;1706 def : Pat<(v4i32 (any_fp_to_sint (loadv4f64 addr:$src))),1707 (VCVTTPD2DQYrm addr:$src)>;1708}1709 1710def CVTTPD2DQrr : PDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),1711 "cvttpd2dq\t{$src, $dst|$dst, $src}",1712 [(set VR128:$dst,1713 (v4i32 (X86any_cvttp2si (v2f64 VR128:$src))))]>,1714 Sched<[WriteCvtPD2I]>, SIMD_EXC;1715def CVTTPD2DQrm : PDI<0xE6, MRMSrcMem, (outs VR128:$dst),(ins f128mem:$src),1716 "cvttpd2dq\t{$src, $dst|$dst, $src}",1717 [(set VR128:$dst,1718 (v4i32 (X86any_cvttp2si (memopv2f64 addr:$src))))]>,1719 Sched<[WriteCvtPD2ILd]>, SIMD_EXC;1720 1721// Convert packed single to packed double1722let Predicates = [HasAVX, NoVLX], Uses = [MXCSR], mayRaiseFPException = 1 in {1723 // SSE2 instructions without OpSize prefix1724def VCVTPS2PDrr : I<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),1725 "vcvtps2pd\t{$src, $dst|$dst, $src}",1726 [(set VR128:$dst, (v2f64 (X86any_vfpext (v4f32 VR128:$src))))]>,1727 TB, VEX, Sched<[WriteCvtPS2PD]>, WIG;1728def VCVTPS2PDrm : I<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f64mem:$src),1729 "vcvtps2pd\t{$src, $dst|$dst, $src}",1730 [(set VR128:$dst, (v2f64 (extloadv2f32 addr:$src)))]>,1731 TB, VEX, Sched<[WriteCvtPS2PD.Folded]>, WIG;1732def VCVTPS2PDYrr : I<0x5A, MRMSrcReg, (outs VR256:$dst), (ins VR128:$src),1733 "vcvtps2pd\t{$src, $dst|$dst, $src}",1734 [(set VR256:$dst, (v4f64 (any_fpextend (v4f32 VR128:$src))))]>,1735 TB, VEX, VEX_L, Sched<[WriteCvtPS2PDY]>, WIG;1736def VCVTPS2PDYrm : I<0x5A, MRMSrcMem, (outs VR256:$dst), (ins f128mem:$src),1737 "vcvtps2pd\t{$src, $dst|$dst, $src}",1738 [(set VR256:$dst, (v4f64 (extloadv4f32 addr:$src)))]>,1739 TB, VEX, VEX_L, Sched<[WriteCvtPS2PDY.Folded]>, WIG;1740}1741 1742let Predicates = [UseSSE2], Uses = [MXCSR], mayRaiseFPException = 1 in {1743def CVTPS2PDrr : I<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),1744 "cvtps2pd\t{$src, $dst|$dst, $src}",1745 [(set VR128:$dst, (v2f64 (X86any_vfpext (v4f32 VR128:$src))))]>,1746 TB, Sched<[WriteCvtPS2PD]>;1747def CVTPS2PDrm : I<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f64mem:$src),1748 "cvtps2pd\t{$src, $dst|$dst, $src}",1749 [(set VR128:$dst, (v2f64 (extloadv2f32 addr:$src)))]>,1750 TB, Sched<[WriteCvtPS2PD.Folded]>;1751}1752 1753// Convert Packed DW Integers to Packed Double FP1754let Predicates = [HasAVX, NoVLX] in {1755let hasSideEffects = 0, mayLoad = 1 in1756def VCVTDQ2PDrm : S2SI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src),1757 "vcvtdq2pd\t{$src, $dst|$dst, $src}",1758 [(set VR128:$dst,1759 (v2f64 (X86any_VSintToFP1760 (bc_v4i321761 (v2i64 (scalar_to_vector1762 (loadi64 addr:$src)))))))]>,1763 VEX, Sched<[WriteCvtI2PDLd]>, WIG;1764def VCVTDQ2PDrr : S2SI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),1765 "vcvtdq2pd\t{$src, $dst|$dst, $src}",1766 [(set VR128:$dst,1767 (v2f64 (X86any_VSintToFP (v4i32 VR128:$src))))]>,1768 VEX, Sched<[WriteCvtI2PD]>, WIG;1769def VCVTDQ2PDYrm : S2SI<0xE6, MRMSrcMem, (outs VR256:$dst), (ins i128mem:$src),1770 "vcvtdq2pd\t{$src, $dst|$dst, $src}",1771 [(set VR256:$dst,1772 (v4f64 (any_sint_to_fp (loadv4i32 addr:$src))))]>,1773 VEX, VEX_L, Sched<[WriteCvtI2PDYLd]>,1774 WIG;1775def VCVTDQ2PDYrr : S2SI<0xE6, MRMSrcReg, (outs VR256:$dst), (ins VR128:$src),1776 "vcvtdq2pd\t{$src, $dst|$dst, $src}",1777 [(set VR256:$dst,1778 (v4f64 (any_sint_to_fp (v4i32 VR128:$src))))]>,1779 VEX, VEX_L, Sched<[WriteCvtI2PDY]>, WIG;1780}1781 1782let hasSideEffects = 0, mayLoad = 1 in1783def CVTDQ2PDrm : S2SI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src),1784 "cvtdq2pd\t{$src, $dst|$dst, $src}",1785 [(set VR128:$dst,1786 (v2f64 (X86any_VSintToFP1787 (bc_v4i321788 (v2i64 (scalar_to_vector1789 (loadi64 addr:$src)))))))]>,1790 Sched<[WriteCvtI2PDLd]>;1791def CVTDQ2PDrr : S2SI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),1792 "cvtdq2pd\t{$src, $dst|$dst, $src}",1793 [(set VR128:$dst,1794 (v2f64 (X86any_VSintToFP (v4i32 VR128:$src))))]>,1795 Sched<[WriteCvtI2PD]>;1796 1797// AVX register conversion intrinsics1798let Predicates = [HasAVX, NoVLX] in {1799 def : Pat<(v2f64 (X86any_VSintToFP (bc_v4i32 (v2i64 (X86vzload64 addr:$src))))),1800 (VCVTDQ2PDrm addr:$src)>;1801} // Predicates = [HasAVX, NoVLX]1802 1803// SSE2 register conversion intrinsics1804let Predicates = [UseSSE2] in {1805 def : Pat<(v2f64 (X86any_VSintToFP (bc_v4i32 (v2i64 (X86vzload64 addr:$src))))),1806 (CVTDQ2PDrm addr:$src)>;1807} // Predicates = [UseSSE2]1808 1809// Convert packed double to packed single1810// The assembler can recognize rr 256-bit instructions by seeing a ymm1811// register, but the same isn't true when using memory operands instead.1812// Provide other assembly rr and rm forms to address this explicitly.1813let Predicates = [HasAVX, NoVLX], Uses = [MXCSR], mayRaiseFPException = 1 in {1814// XMM only1815def VCVTPD2PSrr : VPDI<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),1816 "cvtpd2ps\t{$src, $dst|$dst, $src}",1817 [(set VR128:$dst, (v4f32 (X86any_vfpround (v2f64 VR128:$src))))]>,1818 VEX, Sched<[WriteCvtPD2PS]>, WIG;1819def VCVTPD2PSrm : VPDI<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),1820 "cvtpd2ps{x}\t{$src, $dst|$dst, $src}",1821 [(set VR128:$dst, (v4f32 (X86any_vfpround (loadv2f64 addr:$src))))]>,1822 VEX, Sched<[WriteCvtPD2PS.Folded]>, WIG;1823 1824def VCVTPD2PSYrr : VPDI<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR256:$src),1825 "cvtpd2ps\t{$src, $dst|$dst, $src}",1826 [(set VR128:$dst, (v4f32 (X86any_vfpround (v4f64 VR256:$src))))]>,1827 VEX, VEX_L, Sched<[WriteCvtPD2PSY]>, WIG;1828def VCVTPD2PSYrm : VPDI<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f256mem:$src),1829 "cvtpd2ps{y}\t{$src, $dst|$dst, $src}",1830 [(set VR128:$dst, (v4f32 (X86any_vfpround (loadv4f64 addr:$src))))]>,1831 VEX, VEX_L, Sched<[WriteCvtPD2PSY.Folded]>, WIG;1832} // Predicates = [HasAVX, NoVLX]1833 1834def : InstAlias<"vcvtpd2psx\t{$src, $dst|$dst, $src}",1835 (VCVTPD2PSrr VR128:$dst, VR128:$src), 0, "att">;1836def : InstAlias<"vcvtpd2psy\t{$src, $dst|$dst, $src}",1837 (VCVTPD2PSYrr VR128:$dst, VR256:$src), 0, "att">;1838 1839def CVTPD2PSrr : PDI<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),1840 "cvtpd2ps\t{$src, $dst|$dst, $src}",1841 [(set VR128:$dst, (v4f32 (X86any_vfpround (v2f64 VR128:$src))))]>,1842 Sched<[WriteCvtPD2PS]>, SIMD_EXC;1843def CVTPD2PSrm : PDI<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),1844 "cvtpd2ps\t{$src, $dst|$dst, $src}",1845 [(set VR128:$dst, (v4f32 (X86any_vfpround (memopv2f64 addr:$src))))]>,1846 Sched<[WriteCvtPD2PS.Folded]>, SIMD_EXC;1847 1848//===----------------------------------------------------------------------===//1849// SSE 1 & 2 - Compare Instructions1850//===----------------------------------------------------------------------===//1851 1852// sse12_cmp_scalar - sse 1 & 2 compare scalar instructions1853multiclass sse12_cmp_scalar<RegisterClass RC, X86MemOperand x86memop,1854 Operand memop, SDNode OpNode, ValueType VT,1855 PatFrag ld_frag, string asm,1856 X86FoldableSchedWrite sched,1857 PatFrags mem_frags> {1858 def rri_Int : SIi8<0xC2, MRMSrcReg, (outs VR128:$dst),1859 (ins VR128:$src1, VR128:$src2, u8imm:$cc), asm,1860 [(set VR128:$dst, (OpNode (VT VR128:$src1),1861 VR128:$src2, timm:$cc))]>,1862 Sched<[sched]>, SIMD_EXC;1863 let mayLoad = 1 in1864 def rmi_Int : SIi8<0xC2, MRMSrcMem, (outs VR128:$dst),1865 (ins VR128:$src1, memop:$src2, u8imm:$cc), asm,1866 [(set VR128:$dst, (OpNode (VT VR128:$src1),1867 (mem_frags addr:$src2), timm:$cc))]>,1868 Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC;1869 1870 let isCodeGenOnly = 1 in {1871 let isCommutable = 1 in1872 def rri : SIi8<0xC2, MRMSrcReg,1873 (outs RC:$dst), (ins RC:$src1, RC:$src2, u8imm:$cc), asm,1874 [(set RC:$dst, (OpNode RC:$src1, RC:$src2, timm:$cc))]>,1875 Sched<[sched]>, SIMD_EXC;1876 def rmi : SIi8<0xC2, MRMSrcMem,1877 (outs RC:$dst), (ins RC:$src1, x86memop:$src2, u8imm:$cc), asm,1878 [(set RC:$dst, (OpNode RC:$src1,1879 (ld_frag addr:$src2), timm:$cc))]>,1880 Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC;1881 }1882}1883 1884let ExeDomain = SSEPackedSingle in1885defm VCMPSS : sse12_cmp_scalar<FR32, f32mem, ssmem, X86cmps, v4f32, loadf32,1886 "cmpss\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}",1887 SchedWriteFCmpSizes.PS.Scl, sse_load_f32>,1888 TB, XS, VEX, VVVV, VEX_LIG, WIG;1889let ExeDomain = SSEPackedDouble in1890defm VCMPSD : sse12_cmp_scalar<FR64, f64mem, sdmem, X86cmps, v2f64, loadf64,1891 "cmpsd\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}",1892 SchedWriteFCmpSizes.PD.Scl, sse_load_f64>,1893 TB, XD, VEX, VVVV, VEX_LIG, WIG;1894 1895let Constraints = "$src1 = $dst" in {1896 let ExeDomain = SSEPackedSingle in1897 defm CMPSS : sse12_cmp_scalar<FR32, f32mem, ssmem, X86cmps, v4f32, loadf32,1898 "cmpss\t{$cc, $src2, $dst|$dst, $src2, $cc}",1899 SchedWriteFCmpSizes.PS.Scl, sse_load_f32>, TB, XS;1900 let ExeDomain = SSEPackedDouble in1901 defm CMPSD : sse12_cmp_scalar<FR64, f64mem, sdmem, X86cmps, v2f64, loadf64,1902 "cmpsd\t{$cc, $src2, $dst|$dst, $src2, $cc}",1903 SchedWriteFCmpSizes.PD.Scl, sse_load_f64>, TB, XD;1904}1905 1906// sse12_ord_cmp - Unordered/Ordered scalar fp compare and set EFLAGS1907multiclass sse12_ord_cmp<bits<8> opc, RegisterClass RC, SDPatternOperator OpNode,1908 ValueType vt, X86MemOperand x86memop,1909 PatFrag ld_frag, string OpcodeStr, Domain d,1910 X86FoldableSchedWrite sched = WriteFComX> {1911 let ExeDomain = d in {1912 def rr: SI<opc, MRMSrcReg, (outs), (ins RC:$src1, RC:$src2),1913 !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"),1914 [(set EFLAGS, (OpNode (vt RC:$src1), RC:$src2))]>,1915 Sched<[sched]>, SIMD_EXC;1916 let mayLoad = 1 in1917 def rm: SI<opc, MRMSrcMem, (outs), (ins RC:$src1, x86memop:$src2),1918 !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"),1919 [(set EFLAGS, (OpNode (vt RC:$src1),1920 (ld_frag addr:$src2)))]>,1921 Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC;1922}1923}1924 1925// sse12_ord_cmp_int - Intrinsic version of sse12_ord_cmp1926multiclass sse12_ord_cmp_int<bits<8> opc, RegisterClass RC, SDNode OpNode,1927 ValueType vt, Operand memop,1928 PatFrags mem_frags, string OpcodeStr,1929 Domain d,1930 X86FoldableSchedWrite sched = WriteFComX> {1931let ExeDomain = d in {1932 def rr_Int: SI<opc, MRMSrcReg, (outs), (ins RC:$src1, RC:$src2),1933 !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"),1934 [(set EFLAGS, (OpNode (vt RC:$src1), RC:$src2))]>,1935 Sched<[sched]>, SIMD_EXC;1936let mayLoad = 1 in1937 def rm_Int: SI<opc, MRMSrcMem, (outs), (ins RC:$src1, memop:$src2),1938 !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"),1939 [(set EFLAGS, (OpNode (vt RC:$src1),1940 (mem_frags addr:$src2)))]>,1941 Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC;1942}1943}1944 1945let Defs = [EFLAGS] in {1946 defm VUCOMISS : sse12_ord_cmp<0x2E, FR32, X86any_fcmp, f32, f32mem, loadf32,1947 "ucomiss", SSEPackedSingle>, TB, VEX, VEX_LIG, WIG;1948 defm VUCOMISD : sse12_ord_cmp<0x2E, FR64, X86any_fcmp, f64, f64mem, loadf64,1949 "ucomisd", SSEPackedDouble>, TB, PD, VEX, VEX_LIG, WIG;1950 defm VCOMISS : sse12_ord_cmp<0x2F, FR32, X86strict_fcmps, f32, f32mem, loadf32,1951 "comiss", SSEPackedSingle>, TB, VEX, VEX_LIG, WIG;1952 defm VCOMISD : sse12_ord_cmp<0x2F, FR64, X86strict_fcmps, f64, f64mem, loadf64,1953 "comisd", SSEPackedDouble>, TB, PD, VEX, VEX_LIG, WIG;1954 1955 let isCodeGenOnly = 1 in {1956 defm VUCOMISS : sse12_ord_cmp_int<0x2E, VR128, X86ucomi, v4f32, ssmem,1957 sse_load_f32, "ucomiss", SSEPackedSingle>, TB, VEX, VEX_LIG, WIG;1958 defm VUCOMISD : sse12_ord_cmp_int<0x2E, VR128, X86ucomi, v2f64, sdmem,1959 sse_load_f64, "ucomisd", SSEPackedDouble>, TB, PD, VEX, VEX_LIG, WIG;1960 1961 defm VCOMISS : sse12_ord_cmp_int<0x2F, VR128, X86comi, v4f32, ssmem,1962 sse_load_f32, "comiss", SSEPackedSingle>, TB, VEX, VEX_LIG, WIG;1963 defm VCOMISD : sse12_ord_cmp_int<0x2F, VR128, X86comi, v2f64, sdmem,1964 sse_load_f64, "comisd", SSEPackedDouble>, TB, PD, VEX, VEX_LIG, WIG;1965 }1966 defm UCOMISS : sse12_ord_cmp<0x2E, FR32, X86any_fcmp, f32, f32mem, loadf32,1967 "ucomiss", SSEPackedSingle>, TB;1968 defm UCOMISD : sse12_ord_cmp<0x2E, FR64, X86any_fcmp, f64, f64mem, loadf64,1969 "ucomisd", SSEPackedDouble>, TB, PD;1970 defm COMISS : sse12_ord_cmp<0x2F, FR32, X86strict_fcmps, f32, f32mem, loadf32,1971 "comiss", SSEPackedSingle>, TB;1972 defm COMISD : sse12_ord_cmp<0x2F, FR64, X86strict_fcmps, f64, f64mem, loadf64,1973 "comisd", SSEPackedDouble>, TB, PD;1974 1975 let isCodeGenOnly = 1 in {1976 defm UCOMISS : sse12_ord_cmp_int<0x2E, VR128, X86ucomi, v4f32, ssmem,1977 sse_load_f32, "ucomiss", SSEPackedSingle>, TB;1978 defm UCOMISD : sse12_ord_cmp_int<0x2E, VR128, X86ucomi, v2f64, sdmem,1979 sse_load_f64, "ucomisd", SSEPackedDouble>, TB, PD;1980 1981 defm COMISS : sse12_ord_cmp_int<0x2F, VR128, X86comi, v4f32, ssmem,1982 sse_load_f32, "comiss", SSEPackedSingle>, TB;1983 defm COMISD : sse12_ord_cmp_int<0x2F, VR128, X86comi, v2f64, sdmem,1984 sse_load_f64, "comisd", SSEPackedDouble>, TB, PD;1985 }1986} // Defs = [EFLAGS]1987 1988// sse12_cmp_packed - sse 1 & 2 compare packed instructions1989multiclass sse12_cmp_packed<RegisterClass RC, X86MemOperand x86memop,1990 ValueType VT, string asm,1991 X86FoldableSchedWrite sched,1992 Domain d, PatFrag ld_frag> {1993 let isCommutable = 1 in1994 def rri : PIi8<0xC2, MRMSrcReg,1995 (outs RC:$dst), (ins RC:$src1, RC:$src2, u8imm:$cc), asm,1996 [(set RC:$dst, (VT (X86any_cmpp RC:$src1, RC:$src2, timm:$cc)))], d>,1997 Sched<[sched]>, SIMD_EXC;1998 def rmi : PIi8<0xC2, MRMSrcMem,1999 (outs RC:$dst), (ins RC:$src1, x86memop:$src2, u8imm:$cc), asm,2000 [(set RC:$dst,2001 (VT (X86any_cmpp RC:$src1, (ld_frag addr:$src2), timm:$cc)))], d>,2002 Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC;2003}2004 2005defm VCMPPS : sse12_cmp_packed<VR128, f128mem, v4f32,2006 "cmpps\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}",2007 SchedWriteFCmpSizes.PS.XMM, SSEPackedSingle, loadv4f32>, TB, VEX, VVVV, WIG;2008defm VCMPPD : sse12_cmp_packed<VR128, f128mem, v2f64,2009 "cmppd\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}",2010 SchedWriteFCmpSizes.PD.XMM, SSEPackedDouble, loadv2f64>, TB, PD, VEX, VVVV, WIG;2011defm VCMPPSY : sse12_cmp_packed<VR256, f256mem, v8f32,2012 "cmpps\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}",2013 SchedWriteFCmpSizes.PS.YMM, SSEPackedSingle, loadv8f32>, TB, VEX, VVVV, VEX_L, WIG;2014defm VCMPPDY : sse12_cmp_packed<VR256, f256mem, v4f64,2015 "cmppd\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}",2016 SchedWriteFCmpSizes.PD.YMM, SSEPackedDouble, loadv4f64>, TB, PD, VEX, VVVV, VEX_L, WIG;2017let Constraints = "$src1 = $dst" in {2018 defm CMPPS : sse12_cmp_packed<VR128, f128mem, v4f32,2019 "cmpps\t{$cc, $src2, $dst|$dst, $src2, $cc}",2020 SchedWriteFCmpSizes.PS.XMM, SSEPackedSingle, memopv4f32>, TB;2021 defm CMPPD : sse12_cmp_packed<VR128, f128mem, v2f64,2022 "cmppd\t{$cc, $src2, $dst|$dst, $src2, $cc}",2023 SchedWriteFCmpSizes.PD.XMM, SSEPackedDouble, memopv2f64>, TB, PD;2024}2025 2026def CommutableCMPCC : PatLeaf<(timm), [{2027 uint64_t Imm = N->getZExtValue() & 0x7;2028 return (Imm == 0x00 || Imm == 0x03 || Imm == 0x04 || Imm == 0x07);2029}]>;2030 2031// Patterns to select compares with loads in first operand.2032let Predicates = [HasAVX] in {2033 def : Pat<(v4f64 (X86any_cmpp (loadv4f64 addr:$src2), VR256:$src1,2034 CommutableCMPCC:$cc)),2035 (VCMPPDYrmi VR256:$src1, addr:$src2, timm:$cc)>;2036 2037 def : Pat<(v8f32 (X86any_cmpp (loadv8f32 addr:$src2), VR256:$src1,2038 CommutableCMPCC:$cc)),2039 (VCMPPSYrmi VR256:$src1, addr:$src2, timm:$cc)>;2040 2041 def : Pat<(v2f64 (X86any_cmpp (loadv2f64 addr:$src2), VR128:$src1,2042 CommutableCMPCC:$cc)),2043 (VCMPPDrmi VR128:$src1, addr:$src2, timm:$cc)>;2044 2045 def : Pat<(v4f32 (X86any_cmpp (loadv4f32 addr:$src2), VR128:$src1,2046 CommutableCMPCC:$cc)),2047 (VCMPPSrmi VR128:$src1, addr:$src2, timm:$cc)>;2048 2049 def : Pat<(f64 (X86cmps (loadf64 addr:$src2), FR64:$src1,2050 CommutableCMPCC:$cc)),2051 (VCMPSDrmi FR64:$src1, addr:$src2, timm:$cc)>;2052 2053 def : Pat<(f32 (X86cmps (loadf32 addr:$src2), FR32:$src1,2054 CommutableCMPCC:$cc)),2055 (VCMPSSrmi FR32:$src1, addr:$src2, timm:$cc)>;2056}2057 2058let Predicates = [UseSSE2] in {2059 def : Pat<(v2f64 (X86any_cmpp (memopv2f64 addr:$src2), VR128:$src1,2060 CommutableCMPCC:$cc)),2061 (CMPPDrmi VR128:$src1, addr:$src2, timm:$cc)>;2062 2063 def : Pat<(f64 (X86cmps (loadf64 addr:$src2), FR64:$src1,2064 CommutableCMPCC:$cc)),2065 (CMPSDrmi FR64:$src1, addr:$src2, timm:$cc)>;2066}2067 2068let Predicates = [UseSSE1] in {2069 def : Pat<(v4f32 (X86any_cmpp (memopv4f32 addr:$src2), VR128:$src1,2070 CommutableCMPCC:$cc)),2071 (CMPPSrmi VR128:$src1, addr:$src2, timm:$cc)>;2072 2073 def : Pat<(f32 (X86cmps (loadf32 addr:$src2), FR32:$src1,2074 CommutableCMPCC:$cc)),2075 (CMPSSrmi FR32:$src1, addr:$src2, timm:$cc)>;2076}2077 2078//===----------------------------------------------------------------------===//2079// SSE 1 & 2 - Shuffle Instructions2080//===----------------------------------------------------------------------===//2081 2082/// sse12_shuffle - sse 1 & 2 fp shuffle instructions2083multiclass sse12_shuffle<RegisterClass RC, X86MemOperand x86memop,2084 ValueType vt, string asm, PatFrag mem_frag,2085 X86FoldableSchedWrite sched, Domain d,2086 bit IsCommutable = 0> {2087 def rmi : PIi8<0xC6, MRMSrcMem, (outs RC:$dst),2088 (ins RC:$src1, x86memop:$src2, u8imm:$src3), asm,2089 [(set RC:$dst, (vt (X86Shufp RC:$src1, (mem_frag addr:$src2),2090 (i8 timm:$src3))))], d>,2091 Sched<[sched.Folded, sched.ReadAfterFold]>;2092 let isCommutable = IsCommutable in2093 def rri : PIi8<0xC6, MRMSrcReg, (outs RC:$dst),2094 (ins RC:$src1, RC:$src2, u8imm:$src3), asm,2095 [(set RC:$dst, (vt (X86Shufp RC:$src1, RC:$src2,2096 (i8 timm:$src3))))], d>,2097 Sched<[sched]>;2098}2099 2100let Predicates = [HasAVX, NoVLX] in {2101 defm VSHUFPS : sse12_shuffle<VR128, f128mem, v4f32,2102 "shufps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",2103 loadv4f32, SchedWriteFShuffle.XMM, SSEPackedSingle>,2104 TB, VEX, VVVV, WIG;2105 defm VSHUFPSY : sse12_shuffle<VR256, f256mem, v8f32,2106 "shufps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",2107 loadv8f32, SchedWriteFShuffle.YMM, SSEPackedSingle>,2108 TB, VEX, VVVV, VEX_L, WIG;2109 defm VSHUFPD : sse12_shuffle<VR128, f128mem, v2f64,2110 "shufpd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",2111 loadv2f64, SchedWriteFShuffle.XMM, SSEPackedDouble>,2112 TB, PD, VEX, VVVV, WIG;2113 defm VSHUFPDY : sse12_shuffle<VR256, f256mem, v4f64,2114 "shufpd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",2115 loadv4f64, SchedWriteFShuffle.YMM, SSEPackedDouble>,2116 TB, PD, VEX, VVVV, VEX_L, WIG;2117}2118let Constraints = "$src1 = $dst" in {2119 defm SHUFPS : sse12_shuffle<VR128, f128mem, v4f32,2120 "shufps\t{$src3, $src2, $dst|$dst, $src2, $src3}",2121 memopv4f32, SchedWriteFShuffle.XMM, SSEPackedSingle>, TB;2122 defm SHUFPD : sse12_shuffle<VR128, f128mem, v2f64,2123 "shufpd\t{$src3, $src2, $dst|$dst, $src2, $src3}",2124 memopv2f64, SchedWriteFShuffle.XMM, SSEPackedDouble, 1>, TB, PD;2125}2126 2127//===----------------------------------------------------------------------===//2128// SSE 1 & 2 - Unpack FP Instructions2129//===----------------------------------------------------------------------===//2130 2131/// sse12_unpack_interleave - sse 1 & 2 fp unpack and interleave2132multiclass sse12_unpack_interleave<bits<8> opc, SDNode OpNode, ValueType vt,2133 PatFrag mem_frag, RegisterClass RC,2134 X86MemOperand x86memop, string asm,2135 X86FoldableSchedWrite sched, Domain d,2136 bit IsCommutable = 0> {2137 let isCommutable = IsCommutable in2138 def rr : PI<opc, MRMSrcReg,2139 (outs RC:$dst), (ins RC:$src1, RC:$src2),2140 asm, [(set RC:$dst,2141 (vt (OpNode RC:$src1, RC:$src2)))], d>,2142 Sched<[sched]>;2143 def rm : PI<opc, MRMSrcMem,2144 (outs RC:$dst), (ins RC:$src1, x86memop:$src2),2145 asm, [(set RC:$dst,2146 (vt (OpNode RC:$src1,2147 (mem_frag addr:$src2))))], d>,2148 Sched<[sched.Folded, sched.ReadAfterFold]>;2149}2150 2151let Predicates = [HasAVX, NoVLX] in {2152defm VUNPCKHPS: sse12_unpack_interleave<0x15, X86Unpckh, v4f32, load,2153 VR128, f128mem, "unpckhps\t{$src2, $src1, $dst|$dst, $src1, $src2}",2154 SchedWriteFShuffle.XMM, SSEPackedSingle>, TB, VEX, VVVV, WIG;2155defm VUNPCKHPD: sse12_unpack_interleave<0x15, X86Unpckh, v2f64, load,2156 VR128, f128mem, "unpckhpd\t{$src2, $src1, $dst|$dst, $src1, $src2}",2157 SchedWriteFShuffle.XMM, SSEPackedDouble, 1>, TB, PD, VEX, VVVV, WIG;2158defm VUNPCKLPS: sse12_unpack_interleave<0x14, X86Unpckl, v4f32, load,2159 VR128, f128mem, "unpcklps\t{$src2, $src1, $dst|$dst, $src1, $src2}",2160 SchedWriteFShuffle.XMM, SSEPackedSingle>, TB, VEX, VVVV, WIG;2161defm VUNPCKLPD: sse12_unpack_interleave<0x14, X86Unpckl, v2f64, load,2162 VR128, f128mem, "unpcklpd\t{$src2, $src1, $dst|$dst, $src1, $src2}",2163 SchedWriteFShuffle.XMM, SSEPackedDouble>, TB, PD, VEX, VVVV, WIG;2164 2165defm VUNPCKHPSY: sse12_unpack_interleave<0x15, X86Unpckh, v8f32, load,2166 VR256, f256mem, "unpckhps\t{$src2, $src1, $dst|$dst, $src1, $src2}",2167 SchedWriteFShuffle.YMM, SSEPackedSingle>, TB, VEX, VVVV, VEX_L, WIG;2168defm VUNPCKHPDY: sse12_unpack_interleave<0x15, X86Unpckh, v4f64, load,2169 VR256, f256mem, "unpckhpd\t{$src2, $src1, $dst|$dst, $src1, $src2}",2170 SchedWriteFShuffle.YMM, SSEPackedDouble>, TB, PD, VEX, VVVV, VEX_L, WIG;2171defm VUNPCKLPSY: sse12_unpack_interleave<0x14, X86Unpckl, v8f32, load,2172 VR256, f256mem, "unpcklps\t{$src2, $src1, $dst|$dst, $src1, $src2}",2173 SchedWriteFShuffle.YMM, SSEPackedSingle>, TB, VEX, VVVV, VEX_L, WIG;2174defm VUNPCKLPDY: sse12_unpack_interleave<0x14, X86Unpckl, v4f64, load,2175 VR256, f256mem, "unpcklpd\t{$src2, $src1, $dst|$dst, $src1, $src2}",2176 SchedWriteFShuffle.YMM, SSEPackedDouble>, TB, PD, VEX, VVVV, VEX_L, WIG;2177}// Predicates = [HasAVX, NoVLX]2178 2179let Constraints = "$src1 = $dst" in {2180 defm UNPCKHPS: sse12_unpack_interleave<0x15, X86Unpckh, v4f32, memop,2181 VR128, f128mem, "unpckhps\t{$src2, $dst|$dst, $src2}",2182 SchedWriteFShuffle.XMM, SSEPackedSingle>, TB;2183 defm UNPCKHPD: sse12_unpack_interleave<0x15, X86Unpckh, v2f64, memop,2184 VR128, f128mem, "unpckhpd\t{$src2, $dst|$dst, $src2}",2185 SchedWriteFShuffle.XMM, SSEPackedDouble, 1>, TB, PD;2186 defm UNPCKLPS: sse12_unpack_interleave<0x14, X86Unpckl, v4f32, memop,2187 VR128, f128mem, "unpcklps\t{$src2, $dst|$dst, $src2}",2188 SchedWriteFShuffle.XMM, SSEPackedSingle>, TB;2189 defm UNPCKLPD: sse12_unpack_interleave<0x14, X86Unpckl, v2f64, memop,2190 VR128, f128mem, "unpcklpd\t{$src2, $dst|$dst, $src2}",2191 SchedWriteFShuffle.XMM, SSEPackedDouble>, TB, PD;2192} // Constraints = "$src1 = $dst"2193 2194let Predicates = [HasAVX1Only] in {2195 def : Pat<(v8i32 (X86Unpckl VR256:$src1, (loadv8i32 addr:$src2))),2196 (VUNPCKLPSYrm VR256:$src1, addr:$src2)>;2197 def : Pat<(v8i32 (X86Unpckl VR256:$src1, VR256:$src2)),2198 (VUNPCKLPSYrr VR256:$src1, VR256:$src2)>;2199 def : Pat<(v8i32 (X86Unpckh VR256:$src1, (loadv8i32 addr:$src2))),2200 (VUNPCKHPSYrm VR256:$src1, addr:$src2)>;2201 def : Pat<(v8i32 (X86Unpckh VR256:$src1, VR256:$src2)),2202 (VUNPCKHPSYrr VR256:$src1, VR256:$src2)>;2203 2204 def : Pat<(v4i64 (X86Unpckl VR256:$src1, (loadv4i64 addr:$src2))),2205 (VUNPCKLPDYrm VR256:$src1, addr:$src2)>;2206 def : Pat<(v4i64 (X86Unpckl VR256:$src1, VR256:$src2)),2207 (VUNPCKLPDYrr VR256:$src1, VR256:$src2)>;2208 def : Pat<(v4i64 (X86Unpckh VR256:$src1, (loadv4i64 addr:$src2))),2209 (VUNPCKHPDYrm VR256:$src1, addr:$src2)>;2210 def : Pat<(v4i64 (X86Unpckh VR256:$src1, VR256:$src2)),2211 (VUNPCKHPDYrr VR256:$src1, VR256:$src2)>;2212}2213 2214let Predicates = [UseSSE2] in {2215 // Use MOVHPD if the load isn't aligned enough for UNPCKLPD.2216 def : Pat<(v2f64 (X86Unpckl VR128:$src1,2217 (v2f64 (simple_load addr:$src2)))),2218 (MOVHPDrm VR128:$src1, addr:$src2)>;2219}2220 2221//===----------------------------------------------------------------------===//2222// SSE 1 & 2 - Extract Floating-Point Sign mask2223//===----------------------------------------------------------------------===//2224 2225/// sse12_extr_sign_mask - sse 1 & 2 unpack and interleave2226multiclass sse12_extr_sign_mask<RegisterClass RC, ValueType vt,2227 string asm, Domain d> {2228 def rr : PI<0x50, MRMSrcReg, (outs GR32orGR64:$dst), (ins RC:$src),2229 !strconcat(asm, "\t{$src, $dst|$dst, $src}"),2230 [(set GR32orGR64:$dst, (X86movmsk (vt RC:$src)))], d>,2231 Sched<[WriteFMOVMSK]>;2232}2233 2234let Predicates = [HasAVX] in {2235 defm VMOVMSKPS : sse12_extr_sign_mask<VR128, v4f32, "movmskps",2236 SSEPackedSingle>, TB, VEX, WIG;2237 defm VMOVMSKPD : sse12_extr_sign_mask<VR128, v2f64, "movmskpd",2238 SSEPackedDouble>, TB, PD, VEX, WIG;2239 defm VMOVMSKPSY : sse12_extr_sign_mask<VR256, v8f32, "movmskps",2240 SSEPackedSingle>, TB, VEX, VEX_L, WIG;2241 defm VMOVMSKPDY : sse12_extr_sign_mask<VR256, v4f64, "movmskpd",2242 SSEPackedDouble>, TB, PD, VEX, VEX_L, WIG;2243 2244 // Also support integer VTs to avoid a int->fp bitcast in the DAG.2245 def : Pat<(X86movmsk (v4i32 VR128:$src)),2246 (VMOVMSKPSrr VR128:$src)>;2247 def : Pat<(X86movmsk (v2i64 VR128:$src)),2248 (VMOVMSKPDrr VR128:$src)>;2249 def : Pat<(X86movmsk (v8i32 VR256:$src)),2250 (VMOVMSKPSYrr VR256:$src)>;2251 def : Pat<(X86movmsk (v4i64 VR256:$src)),2252 (VMOVMSKPDYrr VR256:$src)>;2253}2254 2255defm MOVMSKPS : sse12_extr_sign_mask<VR128, v4f32, "movmskps",2256 SSEPackedSingle>, TB;2257defm MOVMSKPD : sse12_extr_sign_mask<VR128, v2f64, "movmskpd",2258 SSEPackedDouble>, TB, PD;2259 2260let Predicates = [UseSSE2] in {2261 // Also support integer VTs to avoid a int->fp bitcast in the DAG.2262 def : Pat<(X86movmsk (v4i32 VR128:$src)),2263 (MOVMSKPSrr VR128:$src)>;2264 def : Pat<(X86movmsk (v2i64 VR128:$src)),2265 (MOVMSKPDrr VR128:$src)>;2266}2267 2268//===---------------------------------------------------------------------===//2269// SSE2 - Packed Integer Logical Instructions2270//===---------------------------------------------------------------------===//2271 2272let ExeDomain = SSEPackedInt in { // SSE integer instructions2273 2274/// PDI_binop_rm - Simple SSE2 binary operator.2275multiclass PDI_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,2276 ValueType OpVT, RegisterClass RC, PatFrag memop_frag,2277 X86MemOperand x86memop, X86FoldableSchedWrite sched,2278 bit IsCommutable, bit Is2Addr> {2279 let isCommutable = IsCommutable in2280 def rr : PDI<opc, MRMSrcReg, (outs RC:$dst),2281 (ins RC:$src1, RC:$src2),2282 !if(Is2Addr,2283 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),2284 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),2285 [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2)))]>,2286 Sched<[sched]>;2287 def rm : PDI<opc, MRMSrcMem, (outs RC:$dst),2288 (ins RC:$src1, x86memop:$src2),2289 !if(Is2Addr,2290 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),2291 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),2292 [(set RC:$dst, (OpVT (OpNode RC:$src1, (memop_frag addr:$src2))))]>,2293 Sched<[sched.Folded, sched.ReadAfterFold]>;2294}2295} // ExeDomain = SSEPackedInt2296 2297multiclass PDI_binop_all<bits<8> opc, string OpcodeStr, SDNode Opcode,2298 ValueType OpVT128, ValueType OpVT256,2299 X86SchedWriteWidths sched, bit IsCommutable,2300 Predicate prd> {2301let Predicates = [HasAVX, prd] in2302 defm V#NAME : PDI_binop_rm<opc, !strconcat("v", OpcodeStr), Opcode, OpVT128,2303 VR128, load, i128mem, sched.XMM,2304 IsCommutable, 0>, VEX, VVVV, WIG;2305 2306let Constraints = "$src1 = $dst" in2307 defm NAME : PDI_binop_rm<opc, OpcodeStr, Opcode, OpVT128, VR128,2308 memop, i128mem, sched.XMM, IsCommutable, 1>;2309 2310let Predicates = [HasAVX2, prd] in2311 defm V#NAME#Y : PDI_binop_rm<opc, !strconcat("v", OpcodeStr), Opcode,2312 OpVT256, VR256, load, i256mem, sched.YMM,2313 IsCommutable, 0>, VEX, VVVV, VEX_L, WIG;2314}2315 2316// These are ordered here for pattern ordering requirements with the fp versions2317 2318defm PAND : PDI_binop_all<0xDB, "pand", and, v2i64, v4i64,2319 SchedWriteVecLogic, 1, NoVLX>;2320defm POR : PDI_binop_all<0xEB, "por", or, v2i64, v4i64,2321 SchedWriteVecLogic, 1, NoVLX>;2322defm PXOR : PDI_binop_all<0xEF, "pxor", xor, v2i64, v4i64,2323 SchedWriteVecLogic, 1, NoVLX>;2324defm PANDN : PDI_binop_all<0xDF, "pandn", X86andnp, v2i64, v4i64,2325 SchedWriteVecLogic, 0, NoVLX>;2326 2327//===----------------------------------------------------------------------===//2328// SSE 1 & 2 - Logical Instructions2329//===----------------------------------------------------------------------===//2330 2331/// sse12_fp_packed_logical - SSE 1 & 2 packed FP logical ops2332///2333/// There are no patterns here because isel prefers integer versions for SSE22334/// and later. There are SSE1 v4f32 patterns later.2335multiclass sse12_fp_packed_logical<bits<8> opc, string OpcodeStr,2336 X86SchedWriteWidths sched> {2337 let Predicates = [HasAVX, NoVLX] in {2338 defm V#NAME#PSY : sse12_fp_packed_logical_rm<opc, VR256, SSEPackedSingle,2339 !strconcat(OpcodeStr, "ps"), f256mem, sched.YMM,2340 [], [], 0>, TB, VEX, VVVV, VEX_L, WIG;2341 2342 defm V#NAME#PDY : sse12_fp_packed_logical_rm<opc, VR256, SSEPackedDouble,2343 !strconcat(OpcodeStr, "pd"), f256mem, sched.YMM,2344 [], [], 0>, TB, PD, VEX, VVVV, VEX_L, WIG;2345 2346 defm V#NAME#PS : sse12_fp_packed_logical_rm<opc, VR128, SSEPackedSingle,2347 !strconcat(OpcodeStr, "ps"), f128mem, sched.XMM,2348 [], [], 0>, TB, VEX, VVVV, WIG;2349 2350 defm V#NAME#PD : sse12_fp_packed_logical_rm<opc, VR128, SSEPackedDouble,2351 !strconcat(OpcodeStr, "pd"), f128mem, sched.XMM,2352 [], [], 0>, TB, PD, VEX, VVVV, WIG;2353 }2354 2355 let Constraints = "$src1 = $dst" in {2356 defm PS : sse12_fp_packed_logical_rm<opc, VR128, SSEPackedSingle,2357 !strconcat(OpcodeStr, "ps"), f128mem, sched.XMM,2358 [], []>, TB;2359 2360 defm PD : sse12_fp_packed_logical_rm<opc, VR128, SSEPackedDouble,2361 !strconcat(OpcodeStr, "pd"), f128mem, sched.XMM,2362 [], []>, TB, PD;2363 }2364}2365 2366defm AND : sse12_fp_packed_logical<0x54, "and", SchedWriteFLogic>;2367defm OR : sse12_fp_packed_logical<0x56, "or", SchedWriteFLogic>;2368defm XOR : sse12_fp_packed_logical<0x57, "xor", SchedWriteFLogic>;2369let isCommutable = 0 in2370 defm ANDN : sse12_fp_packed_logical<0x55, "andn", SchedWriteFLogic>;2371 2372let Predicates = [HasAVX2, NoVLX] in {2373 def : Pat<(v32i8 (and VR256:$src1, VR256:$src2)),2374 (VPANDYrr VR256:$src1, VR256:$src2)>;2375 def : Pat<(v16i16 (and VR256:$src1, VR256:$src2)),2376 (VPANDYrr VR256:$src1, VR256:$src2)>;2377 def : Pat<(v8i32 (and VR256:$src1, VR256:$src2)),2378 (VPANDYrr VR256:$src1, VR256:$src2)>;2379 2380 def : Pat<(v32i8 (or VR256:$src1, VR256:$src2)),2381 (VPORYrr VR256:$src1, VR256:$src2)>;2382 def : Pat<(v16i16 (or VR256:$src1, VR256:$src2)),2383 (VPORYrr VR256:$src1, VR256:$src2)>;2384 def : Pat<(v8i32 (or VR256:$src1, VR256:$src2)),2385 (VPORYrr VR256:$src1, VR256:$src2)>;2386 2387 def : Pat<(v32i8 (xor VR256:$src1, VR256:$src2)),2388 (VPXORYrr VR256:$src1, VR256:$src2)>;2389 def : Pat<(v16i16 (xor VR256:$src1, VR256:$src2)),2390 (VPXORYrr VR256:$src1, VR256:$src2)>;2391 def : Pat<(v8i32 (xor VR256:$src1, VR256:$src2)),2392 (VPXORYrr VR256:$src1, VR256:$src2)>;2393 2394 def : Pat<(v32i8 (X86andnp VR256:$src1, VR256:$src2)),2395 (VPANDNYrr VR256:$src1, VR256:$src2)>;2396 def : Pat<(v16i16 (X86andnp VR256:$src1, VR256:$src2)),2397 (VPANDNYrr VR256:$src1, VR256:$src2)>;2398 def : Pat<(v8i32 (X86andnp VR256:$src1, VR256:$src2)),2399 (VPANDNYrr VR256:$src1, VR256:$src2)>;2400 2401 def : Pat<(and VR256:$src1, (loadv32i8 addr:$src2)),2402 (VPANDYrm VR256:$src1, addr:$src2)>;2403 def : Pat<(and VR256:$src1, (loadv16i16 addr:$src2)),2404 (VPANDYrm VR256:$src1, addr:$src2)>;2405 def : Pat<(and VR256:$src1, (loadv8i32 addr:$src2)),2406 (VPANDYrm VR256:$src1, addr:$src2)>;2407 2408 def : Pat<(or VR256:$src1, (loadv32i8 addr:$src2)),2409 (VPORYrm VR256:$src1, addr:$src2)>;2410 def : Pat<(or VR256:$src1, (loadv16i16 addr:$src2)),2411 (VPORYrm VR256:$src1, addr:$src2)>;2412 def : Pat<(or VR256:$src1, (loadv8i32 addr:$src2)),2413 (VPORYrm VR256:$src1, addr:$src2)>;2414 2415 def : Pat<(xor VR256:$src1, (loadv32i8 addr:$src2)),2416 (VPXORYrm VR256:$src1, addr:$src2)>;2417 def : Pat<(xor VR256:$src1, (loadv16i16 addr:$src2)),2418 (VPXORYrm VR256:$src1, addr:$src2)>;2419 def : Pat<(xor VR256:$src1, (loadv8i32 addr:$src2)),2420 (VPXORYrm VR256:$src1, addr:$src2)>;2421 2422 def : Pat<(X86andnp VR256:$src1, (loadv32i8 addr:$src2)),2423 (VPANDNYrm VR256:$src1, addr:$src2)>;2424 def : Pat<(X86andnp VR256:$src1, (loadv16i16 addr:$src2)),2425 (VPANDNYrm VR256:$src1, addr:$src2)>;2426 def : Pat<(X86andnp VR256:$src1, (loadv8i32 addr:$src2)),2427 (VPANDNYrm VR256:$src1, addr:$src2)>;2428}2429 2430// If only AVX1 is supported, we need to handle integer operations with2431// floating point instructions since the integer versions aren't available.2432let Predicates = [HasAVX1Only] in {2433 def : Pat<(v32i8 (and VR256:$src1, VR256:$src2)),2434 (VANDPSYrr VR256:$src1, VR256:$src2)>;2435 def : Pat<(v16i16 (and VR256:$src1, VR256:$src2)),2436 (VANDPSYrr VR256:$src1, VR256:$src2)>;2437 def : Pat<(v8i32 (and VR256:$src1, VR256:$src2)),2438 (VANDPSYrr VR256:$src1, VR256:$src2)>;2439 def : Pat<(v4i64 (and VR256:$src1, VR256:$src2)),2440 (VANDPSYrr VR256:$src1, VR256:$src2)>;2441 2442 def : Pat<(v32i8 (or VR256:$src1, VR256:$src2)),2443 (VORPSYrr VR256:$src1, VR256:$src2)>;2444 def : Pat<(v16i16 (or VR256:$src1, VR256:$src2)),2445 (VORPSYrr VR256:$src1, VR256:$src2)>;2446 def : Pat<(v8i32 (or VR256:$src1, VR256:$src2)),2447 (VORPSYrr VR256:$src1, VR256:$src2)>;2448 def : Pat<(v4i64 (or VR256:$src1, VR256:$src2)),2449 (VORPSYrr VR256:$src1, VR256:$src2)>;2450 2451 def : Pat<(v32i8 (xor VR256:$src1, VR256:$src2)),2452 (VXORPSYrr VR256:$src1, VR256:$src2)>;2453 def : Pat<(v16i16 (xor VR256:$src1, VR256:$src2)),2454 (VXORPSYrr VR256:$src1, VR256:$src2)>;2455 def : Pat<(v8i32 (xor VR256:$src1, VR256:$src2)),2456 (VXORPSYrr VR256:$src1, VR256:$src2)>;2457 def : Pat<(v4i64 (xor VR256:$src1, VR256:$src2)),2458 (VXORPSYrr VR256:$src1, VR256:$src2)>;2459 2460 def : Pat<(v32i8 (X86andnp VR256:$src1, VR256:$src2)),2461 (VANDNPSYrr VR256:$src1, VR256:$src2)>;2462 def : Pat<(v16i16 (X86andnp VR256:$src1, VR256:$src2)),2463 (VANDNPSYrr VR256:$src1, VR256:$src2)>;2464 def : Pat<(v8i32 (X86andnp VR256:$src1, VR256:$src2)),2465 (VANDNPSYrr VR256:$src1, VR256:$src2)>;2466 def : Pat<(v4i64 (X86andnp VR256:$src1, VR256:$src2)),2467 (VANDNPSYrr VR256:$src1, VR256:$src2)>;2468 2469 def : Pat<(and VR256:$src1, (loadv32i8 addr:$src2)),2470 (VANDPSYrm VR256:$src1, addr:$src2)>;2471 def : Pat<(and VR256:$src1, (loadv16i16 addr:$src2)),2472 (VANDPSYrm VR256:$src1, addr:$src2)>;2473 def : Pat<(and VR256:$src1, (loadv8i32 addr:$src2)),2474 (VANDPSYrm VR256:$src1, addr:$src2)>;2475 def : Pat<(and VR256:$src1, (loadv4i64 addr:$src2)),2476 (VANDPSYrm VR256:$src1, addr:$src2)>;2477 2478 def : Pat<(or VR256:$src1, (loadv32i8 addr:$src2)),2479 (VORPSYrm VR256:$src1, addr:$src2)>;2480 def : Pat<(or VR256:$src1, (loadv16i16 addr:$src2)),2481 (VORPSYrm VR256:$src1, addr:$src2)>;2482 def : Pat<(or VR256:$src1, (loadv8i32 addr:$src2)),2483 (VORPSYrm VR256:$src1, addr:$src2)>;2484 def : Pat<(or VR256:$src1, (loadv4i64 addr:$src2)),2485 (VORPSYrm VR256:$src1, addr:$src2)>;2486 2487 def : Pat<(xor VR256:$src1, (loadv32i8 addr:$src2)),2488 (VXORPSYrm VR256:$src1, addr:$src2)>;2489 def : Pat<(xor VR256:$src1, (loadv16i16 addr:$src2)),2490 (VXORPSYrm VR256:$src1, addr:$src2)>;2491 def : Pat<(xor VR256:$src1, (loadv8i32 addr:$src2)),2492 (VXORPSYrm VR256:$src1, addr:$src2)>;2493 def : Pat<(xor VR256:$src1, (loadv4i64 addr:$src2)),2494 (VXORPSYrm VR256:$src1, addr:$src2)>;2495 2496 def : Pat<(X86andnp VR256:$src1, (loadv32i8 addr:$src2)),2497 (VANDNPSYrm VR256:$src1, addr:$src2)>;2498 def : Pat<(X86andnp VR256:$src1, (loadv16i16 addr:$src2)),2499 (VANDNPSYrm VR256:$src1, addr:$src2)>;2500 def : Pat<(X86andnp VR256:$src1, (loadv8i32 addr:$src2)),2501 (VANDNPSYrm VR256:$src1, addr:$src2)>;2502 def : Pat<(X86andnp VR256:$src1, (loadv4i64 addr:$src2)),2503 (VANDNPSYrm VR256:$src1, addr:$src2)>;2504}2505 2506let Predicates = [HasAVX, NoVLX] in {2507 def : Pat<(v16i8 (and VR128:$src1, VR128:$src2)),2508 (VPANDrr VR128:$src1, VR128:$src2)>;2509 def : Pat<(v8i16 (and VR128:$src1, VR128:$src2)),2510 (VPANDrr VR128:$src1, VR128:$src2)>;2511 def : Pat<(v4i32 (and VR128:$src1, VR128:$src2)),2512 (VPANDrr VR128:$src1, VR128:$src2)>;2513 2514 def : Pat<(v16i8 (or VR128:$src1, VR128:$src2)),2515 (VPORrr VR128:$src1, VR128:$src2)>;2516 def : Pat<(v8i16 (or VR128:$src1, VR128:$src2)),2517 (VPORrr VR128:$src1, VR128:$src2)>;2518 def : Pat<(v4i32 (or VR128:$src1, VR128:$src2)),2519 (VPORrr VR128:$src1, VR128:$src2)>;2520 2521 def : Pat<(v16i8 (xor VR128:$src1, VR128:$src2)),2522 (VPXORrr VR128:$src1, VR128:$src2)>;2523 def : Pat<(v8i16 (xor VR128:$src1, VR128:$src2)),2524 (VPXORrr VR128:$src1, VR128:$src2)>;2525 def : Pat<(v4i32 (xor VR128:$src1, VR128:$src2)),2526 (VPXORrr VR128:$src1, VR128:$src2)>;2527 2528 def : Pat<(v16i8 (X86andnp VR128:$src1, VR128:$src2)),2529 (VPANDNrr VR128:$src1, VR128:$src2)>;2530 def : Pat<(v8i16 (X86andnp VR128:$src1, VR128:$src2)),2531 (VPANDNrr VR128:$src1, VR128:$src2)>;2532 def : Pat<(v4i32 (X86andnp VR128:$src1, VR128:$src2)),2533 (VPANDNrr VR128:$src1, VR128:$src2)>;2534 2535 def : Pat<(and VR128:$src1, (loadv16i8 addr:$src2)),2536 (VPANDrm VR128:$src1, addr:$src2)>;2537 def : Pat<(and VR128:$src1, (loadv8i16 addr:$src2)),2538 (VPANDrm VR128:$src1, addr:$src2)>;2539 def : Pat<(and VR128:$src1, (loadv4i32 addr:$src2)),2540 (VPANDrm VR128:$src1, addr:$src2)>;2541 2542 def : Pat<(or VR128:$src1, (loadv16i8 addr:$src2)),2543 (VPORrm VR128:$src1, addr:$src2)>;2544 def : Pat<(or VR128:$src1, (loadv8i16 addr:$src2)),2545 (VPORrm VR128:$src1, addr:$src2)>;2546 def : Pat<(or VR128:$src1, (loadv4i32 addr:$src2)),2547 (VPORrm VR128:$src1, addr:$src2)>;2548 2549 def : Pat<(xor VR128:$src1, (loadv16i8 addr:$src2)),2550 (VPXORrm VR128:$src1, addr:$src2)>;2551 def : Pat<(xor VR128:$src1, (loadv8i16 addr:$src2)),2552 (VPXORrm VR128:$src1, addr:$src2)>;2553 def : Pat<(xor VR128:$src1, (loadv4i32 addr:$src2)),2554 (VPXORrm VR128:$src1, addr:$src2)>;2555 2556 def : Pat<(X86andnp VR128:$src1, (loadv16i8 addr:$src2)),2557 (VPANDNrm VR128:$src1, addr:$src2)>;2558 def : Pat<(X86andnp VR128:$src1, (loadv8i16 addr:$src2)),2559 (VPANDNrm VR128:$src1, addr:$src2)>;2560 def : Pat<(X86andnp VR128:$src1, (loadv4i32 addr:$src2)),2561 (VPANDNrm VR128:$src1, addr:$src2)>;2562}2563 2564let Predicates = [UseSSE2] in {2565 def : Pat<(v16i8 (and VR128:$src1, VR128:$src2)),2566 (PANDrr VR128:$src1, VR128:$src2)>;2567 def : Pat<(v8i16 (and VR128:$src1, VR128:$src2)),2568 (PANDrr VR128:$src1, VR128:$src2)>;2569 def : Pat<(v4i32 (and VR128:$src1, VR128:$src2)),2570 (PANDrr VR128:$src1, VR128:$src2)>;2571 2572 def : Pat<(v16i8 (or VR128:$src1, VR128:$src2)),2573 (PORrr VR128:$src1, VR128:$src2)>;2574 def : Pat<(v8i16 (or VR128:$src1, VR128:$src2)),2575 (PORrr VR128:$src1, VR128:$src2)>;2576 def : Pat<(v4i32 (or VR128:$src1, VR128:$src2)),2577 (PORrr VR128:$src1, VR128:$src2)>;2578 2579 def : Pat<(v16i8 (xor VR128:$src1, VR128:$src2)),2580 (PXORrr VR128:$src1, VR128:$src2)>;2581 def : Pat<(v8i16 (xor VR128:$src1, VR128:$src2)),2582 (PXORrr VR128:$src1, VR128:$src2)>;2583 def : Pat<(v4i32 (xor VR128:$src1, VR128:$src2)),2584 (PXORrr VR128:$src1, VR128:$src2)>;2585 2586 def : Pat<(v16i8 (X86andnp VR128:$src1, VR128:$src2)),2587 (PANDNrr VR128:$src1, VR128:$src2)>;2588 def : Pat<(v8i16 (X86andnp VR128:$src1, VR128:$src2)),2589 (PANDNrr VR128:$src1, VR128:$src2)>;2590 def : Pat<(v4i32 (X86andnp VR128:$src1, VR128:$src2)),2591 (PANDNrr VR128:$src1, VR128:$src2)>;2592 2593 def : Pat<(and VR128:$src1, (memopv16i8 addr:$src2)),2594 (PANDrm VR128:$src1, addr:$src2)>;2595 def : Pat<(and VR128:$src1, (memopv8i16 addr:$src2)),2596 (PANDrm VR128:$src1, addr:$src2)>;2597 def : Pat<(and VR128:$src1, (memopv4i32 addr:$src2)),2598 (PANDrm VR128:$src1, addr:$src2)>;2599 2600 def : Pat<(or VR128:$src1, (memopv16i8 addr:$src2)),2601 (PORrm VR128:$src1, addr:$src2)>;2602 def : Pat<(or VR128:$src1, (memopv8i16 addr:$src2)),2603 (PORrm VR128:$src1, addr:$src2)>;2604 def : Pat<(or VR128:$src1, (memopv4i32 addr:$src2)),2605 (PORrm VR128:$src1, addr:$src2)>;2606 2607 def : Pat<(xor VR128:$src1, (memopv16i8 addr:$src2)),2608 (PXORrm VR128:$src1, addr:$src2)>;2609 def : Pat<(xor VR128:$src1, (memopv8i16 addr:$src2)),2610 (PXORrm VR128:$src1, addr:$src2)>;2611 def : Pat<(xor VR128:$src1, (memopv4i32 addr:$src2)),2612 (PXORrm VR128:$src1, addr:$src2)>;2613 2614 def : Pat<(X86andnp VR128:$src1, (memopv16i8 addr:$src2)),2615 (PANDNrm VR128:$src1, addr:$src2)>;2616 def : Pat<(X86andnp VR128:$src1, (memopv8i16 addr:$src2)),2617 (PANDNrm VR128:$src1, addr:$src2)>;2618 def : Pat<(X86andnp VR128:$src1, (memopv4i32 addr:$src2)),2619 (PANDNrm VR128:$src1, addr:$src2)>;2620}2621 2622// Patterns for packed operations when we don't have integer type available.2623def : Pat<(v4f32 (X86fand VR128:$src1, VR128:$src2)),2624 (ANDPSrr VR128:$src1, VR128:$src2)>;2625def : Pat<(v4f32 (X86for VR128:$src1, VR128:$src2)),2626 (ORPSrr VR128:$src1, VR128:$src2)>;2627def : Pat<(v4f32 (X86fxor VR128:$src1, VR128:$src2)),2628 (XORPSrr VR128:$src1, VR128:$src2)>;2629def : Pat<(v4f32 (X86fandn VR128:$src1, VR128:$src2)),2630 (ANDNPSrr VR128:$src1, VR128:$src2)>;2631 2632def : Pat<(X86fand VR128:$src1, (memopv4f32 addr:$src2)),2633 (ANDPSrm VR128:$src1, addr:$src2)>;2634def : Pat<(X86for VR128:$src1, (memopv4f32 addr:$src2)),2635 (ORPSrm VR128:$src1, addr:$src2)>;2636def : Pat<(X86fxor VR128:$src1, (memopv4f32 addr:$src2)),2637 (XORPSrm VR128:$src1, addr:$src2)>;2638def : Pat<(X86fandn VR128:$src1, (memopv4f32 addr:$src2)),2639 (ANDNPSrm VR128:$src1, addr:$src2)>;2640 2641//===----------------------------------------------------------------------===//2642// SSE 1 & 2 - Arithmetic Instructions2643//===----------------------------------------------------------------------===//2644 2645/// basic_sse12_fp_binop_xxx - SSE 1 & 2 binops come in both scalar and2646/// vector forms.2647///2648/// In addition, we also have a special variant of the scalar form here to2649/// represent the associated intrinsic operation. This form is unlike the2650/// plain scalar form, in that it takes an entire vector (instead of a scalar)2651/// and leaves the top elements unmodified (therefore these cannot be commuted).2652///2653/// These three forms can each be reg+reg or reg+mem.2654///2655 2656/// FIXME: once all 256-bit intrinsics are matched, cleanup and refactor those2657/// classes below2658multiclass basic_sse12_fp_binop_p<bits<8> opc, string OpcodeStr,2659 SDPatternOperator OpNode, X86SchedWriteSizes sched> {2660let Uses = [MXCSR], mayRaiseFPException = 1 in {2661 let Predicates = [HasAVX, NoVLX] in {2662 defm V#NAME#PS : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode,2663 VR128, v4f32, f128mem, loadv4f32,2664 SSEPackedSingle, sched.PS.XMM, 0>, TB, VEX, VVVV, WIG;2665 defm V#NAME#PD : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode,2666 VR128, v2f64, f128mem, loadv2f64,2667 SSEPackedDouble, sched.PD.XMM, 0>, TB, PD, VEX, VVVV, WIG;2668 2669 defm V#NAME#PSY : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"),2670 OpNode, VR256, v8f32, f256mem, loadv8f32,2671 SSEPackedSingle, sched.PS.YMM, 0>, TB, VEX, VVVV, VEX_L, WIG;2672 defm V#NAME#PDY : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"),2673 OpNode, VR256, v4f64, f256mem, loadv4f64,2674 SSEPackedDouble, sched.PD.YMM, 0>, TB, PD, VEX, VVVV, VEX_L, WIG;2675 }2676 2677 let Constraints = "$src1 = $dst" in {2678 defm PS : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, VR128,2679 v4f32, f128mem, memopv4f32, SSEPackedSingle,2680 sched.PS.XMM>, TB;2681 defm PD : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, VR128,2682 v2f64, f128mem, memopv2f64, SSEPackedDouble,2683 sched.PD.XMM>, TB, PD;2684 }2685}2686}2687 2688multiclass basic_sse12_fp_binop_s<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,2689 X86SchedWriteSizes sched> {2690let Uses = [MXCSR], mayRaiseFPException = 1 in {2691 defm V#NAME#SS : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "ss"),2692 OpNode, FR32, f32mem, SSEPackedSingle, sched.PS.Scl, 0>,2693 TB, XS, VEX, VVVV, VEX_LIG, WIG;2694 defm V#NAME#SD : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "sd"),2695 OpNode, FR64, f64mem, SSEPackedDouble, sched.PD.Scl, 0>,2696 TB, XD, VEX, VVVV, VEX_LIG, WIG;2697 2698 let Constraints = "$src1 = $dst" in {2699 defm SS : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "ss"),2700 OpNode, FR32, f32mem, SSEPackedSingle,2701 sched.PS.Scl>, TB, XS;2702 defm SD : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "sd"),2703 OpNode, FR64, f64mem, SSEPackedDouble,2704 sched.PD.Scl>, TB, XD;2705 }2706}2707}2708 2709multiclass basic_sse12_fp_binop_s_int<bits<8> opc, string OpcodeStr,2710 SDPatternOperator OpNode,2711 X86SchedWriteSizes sched> {2712let Uses = [MXCSR], mayRaiseFPException = 1 in {2713 defm V#NAME#SS : sse12_fp_scalar_int<opc, OpNode, VR128, v4f32,2714 !strconcat(OpcodeStr, "ss"), ssmem, sse_load_f32,2715 SSEPackedSingle, sched.PS.Scl, 0>, TB, XS, VEX, VVVV, VEX_LIG, WIG;2716 defm V#NAME#SD : sse12_fp_scalar_int<opc, OpNode, VR128, v2f64,2717 !strconcat(OpcodeStr, "sd"), sdmem, sse_load_f64,2718 SSEPackedDouble, sched.PD.Scl, 0>, TB, XD, VEX, VVVV, VEX_LIG, WIG;2719 2720 let Constraints = "$src1 = $dst" in {2721 defm SS : sse12_fp_scalar_int<opc, OpNode, VR128, v4f32,2722 !strconcat(OpcodeStr, "ss"), ssmem, sse_load_f32,2723 SSEPackedSingle, sched.PS.Scl>, TB, XS;2724 defm SD : sse12_fp_scalar_int<opc, OpNode, VR128, v2f64,2725 !strconcat(OpcodeStr, "sd"), sdmem, sse_load_f64,2726 SSEPackedDouble, sched.PD.Scl>, TB, XD;2727 }2728}2729}2730 2731// Binary Arithmetic instructions2732defm ADD : basic_sse12_fp_binop_p<0x58, "add", any_fadd, SchedWriteFAddSizes>,2733 basic_sse12_fp_binop_s<0x58, "add", any_fadd, SchedWriteFAddSizes>,2734 basic_sse12_fp_binop_s_int<0x58, "add", null_frag, SchedWriteFAddSizes>;2735defm MUL : basic_sse12_fp_binop_p<0x59, "mul", any_fmul, SchedWriteFMulSizes>,2736 basic_sse12_fp_binop_s<0x59, "mul", any_fmul, SchedWriteFMulSizes>,2737 basic_sse12_fp_binop_s_int<0x59, "mul", null_frag, SchedWriteFMulSizes>;2738let isCommutable = 0 in {2739 defm SUB : basic_sse12_fp_binop_p<0x5C, "sub", any_fsub, SchedWriteFAddSizes>,2740 basic_sse12_fp_binop_s<0x5C, "sub", any_fsub, SchedWriteFAddSizes>,2741 basic_sse12_fp_binop_s_int<0x5C, "sub", null_frag, SchedWriteFAddSizes>;2742 defm DIV : basic_sse12_fp_binop_p<0x5E, "div", any_fdiv, SchedWriteFDivSizes>,2743 basic_sse12_fp_binop_s<0x5E, "div", any_fdiv, SchedWriteFDivSizes>,2744 basic_sse12_fp_binop_s_int<0x5E, "div", null_frag, SchedWriteFDivSizes>;2745 defm MAX : basic_sse12_fp_binop_p<0x5F, "max", X86any_fmax, SchedWriteFCmpSizes>,2746 basic_sse12_fp_binop_s<0x5F, "max", X86any_fmax, SchedWriteFCmpSizes>,2747 basic_sse12_fp_binop_s_int<0x5F, "max", X86fmaxs, SchedWriteFCmpSizes>;2748 defm MIN : basic_sse12_fp_binop_p<0x5D, "min", X86any_fmin, SchedWriteFCmpSizes>,2749 basic_sse12_fp_binop_s<0x5D, "min", X86any_fmin, SchedWriteFCmpSizes>,2750 basic_sse12_fp_binop_s_int<0x5D, "min", X86fmins, SchedWriteFCmpSizes>;2751}2752 2753let isCodeGenOnly = 1 in {2754 defm MAXC: basic_sse12_fp_binop_p<0x5F, "max", X86fmaxc, SchedWriteFCmpSizes>,2755 basic_sse12_fp_binop_s<0x5F, "max", X86fmaxc, SchedWriteFCmpSizes>;2756 defm MINC: basic_sse12_fp_binop_p<0x5D, "min", X86fminc, SchedWriteFCmpSizes>,2757 basic_sse12_fp_binop_s<0x5D, "min", X86fminc, SchedWriteFCmpSizes>;2758}2759 2760// Patterns used to select SSE scalar fp arithmetic instructions from2761// either:2762//2763// (1) a scalar fp operation followed by a blend2764//2765// The effect is that the backend no longer emits unnecessary vector2766// insert instructions immediately after SSE scalar fp instructions2767// like addss or mulss.2768//2769// For example, given the following code:2770// __m128 foo(__m128 A, __m128 B) {2771// A[0] += B[0];2772// return A;2773// }2774//2775// Previously we generated:2776// addss %xmm0, %xmm12777// movss %xmm1, %xmm02778//2779// We now generate:2780// addss %xmm1, %xmm02781//2782// (2) a vector packed single/double fp operation followed by a vector insert2783//2784// The effect is that the backend converts the packed fp instruction2785// followed by a vector insert into a single SSE scalar fp instruction.2786//2787// For example, given the following code:2788// __m128 foo(__m128 A, __m128 B) {2789// __m128 C = A + B;2790// return (__m128) {c[0], a[1], a[2], a[3]};2791// }2792//2793// Previously we generated:2794// addps %xmm0, %xmm12795// movss %xmm1, %xmm02796//2797// We now generate:2798// addss %xmm1, %xmm02799 2800// TODO: Some canonicalization in lowering would simplify the number of2801// patterns we have to try to match.2802multiclass scalar_math_patterns<SDPatternOperator Op, string OpcPrefix, SDNode Move,2803 ValueType VT, ValueType EltTy,2804 RegisterClass RC, PatFrag ld_frag,2805 Predicate BasePredicate> {2806 let Predicates = [BasePredicate] in {2807 // extracted scalar math op with insert via movss/movsd2808 def : Pat<(VT (Move (VT VR128:$dst),2809 (VT (scalar_to_vector2810 (Op (EltTy (extractelt (VT VR128:$dst), (iPTR 0))),2811 RC:$src))))),2812 (!cast<Instruction>(OpcPrefix#rr_Int) VT:$dst,2813 (VT (COPY_TO_REGCLASS RC:$src, VR128)))>;2814 def : Pat<(VT (Move (VT VR128:$dst),2815 (VT (scalar_to_vector2816 (Op (EltTy (extractelt (VT VR128:$dst), (iPTR 0))),2817 (ld_frag addr:$src)))))),2818 (!cast<Instruction>(OpcPrefix#rm_Int) VT:$dst, addr:$src)>;2819 }2820 2821 // Repeat for AVX versions of the instructions.2822 let Predicates = [UseAVX] in {2823 // extracted scalar math op with insert via movss/movsd2824 def : Pat<(VT (Move (VT VR128:$dst),2825 (VT (scalar_to_vector2826 (Op (EltTy (extractelt (VT VR128:$dst), (iPTR 0))),2827 RC:$src))))),2828 (!cast<Instruction>("V"#OpcPrefix#rr_Int) VT:$dst,2829 (VT (COPY_TO_REGCLASS RC:$src, VR128)))>;2830 def : Pat<(VT (Move (VT VR128:$dst),2831 (VT (scalar_to_vector2832 (Op (EltTy (extractelt (VT VR128:$dst), (iPTR 0))),2833 (ld_frag addr:$src)))))),2834 (!cast<Instruction>("V"#OpcPrefix#rm_Int) VT:$dst, addr:$src)>;2835 }2836}2837 2838defm : scalar_math_patterns<any_fadd, "ADDSS", X86Movss, v4f32, f32, FR32, loadf32, UseSSE1>;2839defm : scalar_math_patterns<any_fsub, "SUBSS", X86Movss, v4f32, f32, FR32, loadf32, UseSSE1>;2840defm : scalar_math_patterns<any_fmul, "MULSS", X86Movss, v4f32, f32, FR32, loadf32, UseSSE1>;2841defm : scalar_math_patterns<any_fdiv, "DIVSS", X86Movss, v4f32, f32, FR32, loadf32, UseSSE1>;2842 2843defm : scalar_math_patterns<any_fadd, "ADDSD", X86Movsd, v2f64, f64, FR64, loadf64, UseSSE2>;2844defm : scalar_math_patterns<any_fsub, "SUBSD", X86Movsd, v2f64, f64, FR64, loadf64, UseSSE2>;2845defm : scalar_math_patterns<any_fmul, "MULSD", X86Movsd, v2f64, f64, FR64, loadf64, UseSSE2>;2846defm : scalar_math_patterns<any_fdiv, "DIVSD", X86Movsd, v2f64, f64, FR64, loadf64, UseSSE2>;2847 2848/// Unop Arithmetic2849/// In addition, we also have a special variant of the scalar form here to2850/// represent the associated intrinsic operation. This form is unlike the2851/// plain scalar form, in that it takes an entire vector (instead of a2852/// scalar) and leaves the top elements undefined.2853///2854/// And, we have a special variant form for a full-vector intrinsic form.2855 2856/// sse_fp_unop_s - SSE1 unops in scalar form2857/// For the non-AVX defs, we need $src1 to be tied to $dst because2858/// the HW instructions are 2 operand / destructive.2859multiclass sse_fp_unop_s<bits<8> opc, string OpcodeStr, RegisterClass RC,2860 X86MemOperand x86memop, Operand intmemop,2861 SDPatternOperator OpNode, Domain d,2862 X86FoldableSchedWrite sched, Predicate target> {2863 let isCodeGenOnly = 1, hasSideEffects = 0 in {2864 def r : I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1),2865 !strconcat(OpcodeStr, "\t{$src1, $dst|$dst, $src1}"),2866 [(set RC:$dst, (OpNode RC:$src1))], d>, Sched<[sched]>,2867 Requires<[target]>;2868 let mayLoad = 1 in2869 def m : I<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src1),2870 !strconcat(OpcodeStr, "\t{$src1, $dst|$dst, $src1}"),2871 [(set RC:$dst, (OpNode (load addr:$src1)))], d>,2872 Sched<[sched.Folded]>,2873 Requires<[target, OptForSize]>;2874 }2875 2876 let hasSideEffects = 0, Constraints = "$src1 = $dst", ExeDomain = d in {2877 def r_Int : I<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src1, VR128:$src2),2878 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), []>,2879 Sched<[sched]>;2880 let mayLoad = 1 in2881 def m_Int : I<opc, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, intmemop:$src2),2882 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), []>,2883 Sched<[sched.Folded, sched.ReadAfterFold]>;2884 }2885 2886}2887 2888multiclass sse_fp_unop_s_intr<ValueType vt, PatFrags mem_frags,2889 Intrinsic Intr, Predicate target> {2890 let Predicates = [target] in {2891 // These are unary operations, but they are modeled as having 2 source operands2892 // because the high elements of the destination are unchanged in SSE.2893 def : Pat<(Intr VR128:$src),2894 (!cast<Instruction>(NAME#r_Int) VR128:$src, VR128:$src)>;2895 }2896 // We don't want to fold scalar loads into these instructions unless2897 // optimizing for size. This is because the folded instruction will have a2898 // partial register update, while the unfolded sequence will not, e.g.2899 // movss mem, %xmm02900 // rcpss %xmm0, %xmm02901 // which has a clobber before the rcp, vs.2902 // rcpss mem, %xmm02903 let Predicates = [target, OptForSize] in {2904 def : Pat<(Intr (mem_frags addr:$src2)),2905 (!cast<Instruction>(NAME#m_Int)2906 (vt (IMPLICIT_DEF)), addr:$src2)>;2907 }2908}2909 2910multiclass avx_fp_unop_s_intr<ValueType vt, PatFrags mem_frags,2911 Intrinsic Intr, Predicate target> {2912 let Predicates = [target] in {2913 def : Pat<(Intr VR128:$src),2914 (!cast<Instruction>(NAME#r_Int) VR128:$src,2915 VR128:$src)>;2916 }2917 let Predicates = [target, OptForSize] in {2918 def : Pat<(Intr (mem_frags addr:$src2)),2919 (!cast<Instruction>(NAME#m_Int)2920 (vt (IMPLICIT_DEF)), addr:$src2)>;2921 }2922}2923 2924multiclass avx_fp_unop_s<bits<8> opc, string OpcodeStr, RegisterClass RC,2925 ValueType ScalarVT, X86MemOperand x86memop,2926 Operand intmemop, SDPatternOperator OpNode, Domain d,2927 X86FoldableSchedWrite sched, Predicate target> {2928 let isCodeGenOnly = 1, hasSideEffects = 0 in {2929 def r : I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2),2930 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),2931 [], d>, Sched<[sched]>;2932 let mayLoad = 1 in2933 def m : I<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2),2934 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),2935 [], d>, Sched<[sched.Folded, sched.ReadAfterFold]>;2936 }2937 let hasSideEffects = 0, ExeDomain = d in {2938 def r_Int : I<opc, MRMSrcReg, (outs VR128:$dst),2939 (ins VR128:$src1, VR128:$src2),2940 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),2941 []>, Sched<[sched]>;2942 let mayLoad = 1 in2943 def m_Int : I<opc, MRMSrcMem, (outs VR128:$dst),2944 (ins VR128:$src1, intmemop:$src2),2945 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),2946 []>, Sched<[sched.Folded, sched.ReadAfterFold]>;2947 }2948 2949 // We don't want to fold scalar loads into these instructions unless2950 // optimizing for size. This is because the folded instruction will have a2951 // partial register update, while the unfolded sequence will not, e.g.2952 // vmovss mem, %xmm02953 // vrcpss %xmm0, %xmm0, %xmm02954 // which has a clobber before the rcp, vs.2955 // vrcpss mem, %xmm0, %xmm02956 // TODO: In theory, we could fold the load, and avoid the stall caused by2957 // the partial register store, either in BreakFalseDeps or with smarter RA.2958 let Predicates = [target] in {2959 def : Pat<(OpNode RC:$src), (!cast<Instruction>(NAME#r)2960 (ScalarVT (IMPLICIT_DEF)), RC:$src)>;2961 }2962 let Predicates = [target, OptForSize] in {2963 def : Pat<(ScalarVT (OpNode (load addr:$src))),2964 (!cast<Instruction>(NAME#m) (ScalarVT (IMPLICIT_DEF)),2965 addr:$src)>;2966 }2967}2968 2969/// sse1_fp_unop_p - SSE1 unops in packed form.2970multiclass sse1_fp_unop_p<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,2971 X86SchedWriteWidths sched, list<Predicate> prds> {2972let Predicates = prds in {2973 def V#NAME#PSr : PSI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),2974 !strconcat("v", OpcodeStr,2975 "ps\t{$src, $dst|$dst, $src}"),2976 [(set VR128:$dst, (v4f32 (OpNode VR128:$src)))]>,2977 VEX, Sched<[sched.XMM]>, WIG;2978 def V#NAME#PSm : PSI<opc, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),2979 !strconcat("v", OpcodeStr,2980 "ps\t{$src, $dst|$dst, $src}"),2981 [(set VR128:$dst, (OpNode (loadv4f32 addr:$src)))]>,2982 VEX, Sched<[sched.XMM.Folded]>, WIG;2983 def V#NAME#PSYr : PSI<opc, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),2984 !strconcat("v", OpcodeStr,2985 "ps\t{$src, $dst|$dst, $src}"),2986 [(set VR256:$dst, (v8f32 (OpNode VR256:$src)))]>,2987 VEX, VEX_L, Sched<[sched.YMM]>, WIG;2988 def V#NAME#PSYm : PSI<opc, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src),2989 !strconcat("v", OpcodeStr,2990 "ps\t{$src, $dst|$dst, $src}"),2991 [(set VR256:$dst, (OpNode (loadv8f32 addr:$src)))]>,2992 VEX, VEX_L, Sched<[sched.YMM.Folded]>, WIG;2993}2994 2995 def PSr : PSI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),2996 !strconcat(OpcodeStr, "ps\t{$src, $dst|$dst, $src}"),2997 [(set VR128:$dst, (v4f32 (OpNode VR128:$src)))]>,2998 Sched<[sched.XMM]>;2999 def PSm : PSI<opc, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),3000 !strconcat(OpcodeStr, "ps\t{$src, $dst|$dst, $src}"),3001 [(set VR128:$dst, (OpNode (memopv4f32 addr:$src)))]>,3002 Sched<[sched.XMM.Folded]>;3003}3004 3005/// sse2_fp_unop_p - SSE2 unops in vector forms.3006multiclass sse2_fp_unop_p<bits<8> opc, string OpcodeStr,3007 SDPatternOperator OpNode, X86SchedWriteWidths sched> {3008let Predicates = [HasAVX, NoVLX] in {3009 def V#NAME#PDr : PDI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),3010 !strconcat("v", OpcodeStr,3011 "pd\t{$src, $dst|$dst, $src}"),3012 [(set VR128:$dst, (v2f64 (OpNode VR128:$src)))]>,3013 VEX, Sched<[sched.XMM]>, WIG;3014 def V#NAME#PDm : PDI<opc, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),3015 !strconcat("v", OpcodeStr,3016 "pd\t{$src, $dst|$dst, $src}"),3017 [(set VR128:$dst, (OpNode (loadv2f64 addr:$src)))]>,3018 VEX, Sched<[sched.XMM.Folded]>, WIG;3019 def V#NAME#PDYr : PDI<opc, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),3020 !strconcat("v", OpcodeStr,3021 "pd\t{$src, $dst|$dst, $src}"),3022 [(set VR256:$dst, (v4f64 (OpNode VR256:$src)))]>,3023 VEX, VEX_L, Sched<[sched.YMM]>, WIG;3024 def V#NAME#PDYm : PDI<opc, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src),3025 !strconcat("v", OpcodeStr,3026 "pd\t{$src, $dst|$dst, $src}"),3027 [(set VR256:$dst, (OpNode (loadv4f64 addr:$src)))]>,3028 VEX, VEX_L, Sched<[sched.YMM.Folded]>, WIG;3029}3030 3031 def PDr : PDI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),3032 !strconcat(OpcodeStr, "pd\t{$src, $dst|$dst, $src}"),3033 [(set VR128:$dst, (v2f64 (OpNode VR128:$src)))]>,3034 Sched<[sched.XMM]>;3035 def PDm : PDI<opc, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),3036 !strconcat(OpcodeStr, "pd\t{$src, $dst|$dst, $src}"),3037 [(set VR128:$dst, (OpNode (memopv2f64 addr:$src)))]>,3038 Sched<[sched.XMM.Folded]>;3039}3040 3041multiclass sse1_fp_unop_s_intr<string OpcodeStr, Predicate AVXTarget> {3042 defm SS : sse_fp_unop_s_intr<v4f32, sse_load_f32,3043 !cast<Intrinsic>("int_x86_sse_"#OpcodeStr#_ss),3044 UseSSE1>, TB, XS;3045 defm V#NAME#SS : avx_fp_unop_s_intr<v4f32, sse_load_f32,3046 !cast<Intrinsic>("int_x86_sse_"#OpcodeStr#_ss),3047 AVXTarget>,3048 TB, XS, VEX, VVVV, VEX_LIG, WIG;3049}3050 3051multiclass sse1_fp_unop_s<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,3052 X86SchedWriteWidths sched, Predicate AVXTarget> {3053 defm SS : sse_fp_unop_s<opc, OpcodeStr#ss, FR32, f32mem,3054 ssmem, OpNode, SSEPackedSingle, sched.Scl, UseSSE1>, TB, XS;3055 defm V#NAME#SS : avx_fp_unop_s<opc, "v"#OpcodeStr#ss, FR32, f32,3056 f32mem, ssmem, OpNode, SSEPackedSingle, sched.Scl, AVXTarget>,3057 TB, XS, VEX, VVVV, VEX_LIG, WIG;3058}3059 3060multiclass sse2_fp_unop_s<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,3061 X86SchedWriteWidths sched, Predicate AVXTarget> {3062 defm SD : sse_fp_unop_s<opc, OpcodeStr#sd, FR64, f64mem,3063 sdmem, OpNode, SSEPackedDouble, sched.Scl, UseSSE2>, TB, XD;3064 defm V#NAME#SD : avx_fp_unop_s<opc, "v"#OpcodeStr#sd, FR64, f64,3065 f64mem, sdmem, OpNode, SSEPackedDouble, sched.Scl, AVXTarget>,3066 TB, XD, VEX, VVVV, VEX_LIG, WIG;3067}3068 3069// Square root.3070defm SQRT : sse1_fp_unop_s<0x51, "sqrt", any_fsqrt, SchedWriteFSqrt, UseAVX>,3071 sse1_fp_unop_p<0x51, "sqrt", any_fsqrt, SchedWriteFSqrt, [HasAVX, NoVLX]>,3072 sse2_fp_unop_s<0x51, "sqrt", any_fsqrt, SchedWriteFSqrt64, UseAVX>,3073 sse2_fp_unop_p<0x51, "sqrt", any_fsqrt, SchedWriteFSqrt64>, SIMD_EXC;3074 3075// Reciprocal approximations. Note that these typically require refinement3076// in order to obtain suitable precision.3077defm RSQRT : sse1_fp_unop_s<0x52, "rsqrt", X86frsqrt, SchedWriteFRsqrt, HasAVX>,3078 sse1_fp_unop_s_intr<"rsqrt", HasAVX>,3079 sse1_fp_unop_p<0x52, "rsqrt", X86frsqrt, SchedWriteFRsqrt, [HasAVX]>;3080defm RCP : sse1_fp_unop_s<0x53, "rcp", X86frcp, SchedWriteFRcp, HasAVX>,3081 sse1_fp_unop_s_intr<"rcp", HasAVX>,3082 sse1_fp_unop_p<0x53, "rcp", X86frcp, SchedWriteFRcp, [HasAVX]>;3083 3084// There is no f64 version of the reciprocal approximation instructions.3085 3086multiclass scalar_unary_math_patterns<SDPatternOperator OpNode, string OpcPrefix, SDNode Move,3087 ValueType VT, Predicate BasePredicate> {3088 let Predicates = [BasePredicate] in {3089 def : Pat<(VT (Move VT:$dst, (scalar_to_vector3090 (OpNode (extractelt VT:$src, 0))))),3091 (!cast<Instruction>(OpcPrefix#r_Int) VT:$dst, VT:$src)>;3092 }3093 3094 // Repeat for AVX versions of the instructions.3095 let Predicates = [UseAVX] in {3096 def : Pat<(VT (Move VT:$dst, (scalar_to_vector3097 (OpNode (extractelt VT:$src, 0))))),3098 (!cast<Instruction>("V"#OpcPrefix#r_Int) VT:$dst, VT:$src)>;3099 }3100}3101 3102defm : scalar_unary_math_patterns<any_fsqrt, "SQRTSS", X86Movss, v4f32, UseSSE1>;3103defm : scalar_unary_math_patterns<any_fsqrt, "SQRTSD", X86Movsd, v2f64, UseSSE2>;3104 3105multiclass scalar_unary_math_intr_patterns<Intrinsic Intr, string OpcPrefix,3106 SDNode Move, ValueType VT,3107 Predicate BasePredicate> {3108 let Predicates = [BasePredicate] in {3109 def : Pat<(VT (Move VT:$dst, (Intr VT:$src))),3110 (!cast<Instruction>(OpcPrefix#r_Int) VT:$dst, VT:$src)>;3111 }3112 3113 // Repeat for AVX versions of the instructions.3114 let Predicates = [HasAVX] in {3115 def : Pat<(VT (Move VT:$dst, (Intr VT:$src))),3116 (!cast<Instruction>("V"#OpcPrefix#r_Int) VT:$dst, VT:$src)>;3117 }3118}3119 3120defm : scalar_unary_math_intr_patterns<int_x86_sse_rcp_ss, "RCPSS", X86Movss,3121 v4f32, UseSSE1>;3122defm : scalar_unary_math_intr_patterns<int_x86_sse_rsqrt_ss, "RSQRTSS", X86Movss,3123 v4f32, UseSSE1>;3124 3125 3126//===----------------------------------------------------------------------===//3127// SSE 1 & 2 - Non-temporal stores3128//===----------------------------------------------------------------------===//3129 3130let AddedComplexity = 400 in { // Prefer non-temporal versions3131let Predicates = [HasAVX, NoVLX] in {3132let SchedRW = [SchedWriteFMoveLSNT.XMM.MR] in {3133def VMOVNTPSmr : VPSI<0x2B, MRMDestMem, (outs),3134 (ins f128mem:$dst, VR128:$src),3135 "movntps\t{$src, $dst|$dst, $src}",3136 [(alignednontemporalstore (v4f32 VR128:$src),3137 addr:$dst)]>, VEX, WIG;3138def VMOVNTPDmr : VPDI<0x2B, MRMDestMem, (outs),3139 (ins f128mem:$dst, VR128:$src),3140 "movntpd\t{$src, $dst|$dst, $src}",3141 [(alignednontemporalstore (v2f64 VR128:$src),3142 addr:$dst)]>, VEX, WIG;3143} // SchedRW3144 3145let SchedRW = [SchedWriteFMoveLSNT.YMM.MR] in {3146def VMOVNTPSYmr : VPSI<0x2B, MRMDestMem, (outs),3147 (ins f256mem:$dst, VR256:$src),3148 "movntps\t{$src, $dst|$dst, $src}",3149 [(alignednontemporalstore (v8f32 VR256:$src),3150 addr:$dst)]>, VEX, VEX_L, WIG;3151def VMOVNTPDYmr : VPDI<0x2B, MRMDestMem, (outs),3152 (ins f256mem:$dst, VR256:$src),3153 "movntpd\t{$src, $dst|$dst, $src}",3154 [(alignednontemporalstore (v4f64 VR256:$src),3155 addr:$dst)]>, VEX, VEX_L, WIG;3156} // SchedRW3157 3158let ExeDomain = SSEPackedInt in {3159def VMOVNTDQmr : VPDI<0xE7, MRMDestMem, (outs),3160 (ins i128mem:$dst, VR128:$src),3161 "movntdq\t{$src, $dst|$dst, $src}",3162 [(alignednontemporalstore (v2i64 VR128:$src),3163 addr:$dst)]>, VEX, WIG,3164 Sched<[SchedWriteVecMoveLSNT.XMM.MR]>;3165def VMOVNTDQYmr : VPDI<0xE7, MRMDestMem, (outs),3166 (ins i256mem:$dst, VR256:$src),3167 "movntdq\t{$src, $dst|$dst, $src}",3168 [(alignednontemporalstore (v4i64 VR256:$src),3169 addr:$dst)]>, VEX, VEX_L, WIG,3170 Sched<[SchedWriteVecMoveLSNT.YMM.MR]>;3171} // ExeDomain3172} // Predicates3173 3174let SchedRW = [SchedWriteFMoveLSNT.XMM.MR] in {3175def MOVNTPSmr : PSI<0x2B, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src),3176 "movntps\t{$src, $dst|$dst, $src}",3177 [(alignednontemporalstore (v4f32 VR128:$src), addr:$dst)]>;3178def MOVNTPDmr : PDI<0x2B, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src),3179 "movntpd\t{$src, $dst|$dst, $src}",3180 [(alignednontemporalstore(v2f64 VR128:$src), addr:$dst)]>;3181} // SchedRW3182 3183let ExeDomain = SSEPackedInt, SchedRW = [SchedWriteVecMoveLSNT.XMM.MR] in3184def MOVNTDQmr : PDI<0xE7, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src),3185 "movntdq\t{$src, $dst|$dst, $src}",3186 [(alignednontemporalstore (v2i64 VR128:$src), addr:$dst)]>;3187 3188let SchedRW = [WriteStoreNT] in {3189// There is no AVX form for instructions below this point3190def MOVNTImr : I<0xC3, MRMDestMem, (outs), (ins i32mem:$dst, GR32:$src),3191 "movnti{l}\t{$src, $dst|$dst, $src}",3192 [(nontemporalstore (i32 GR32:$src), addr:$dst)]>,3193 TB, Requires<[HasSSE2]>;3194def MOVNTI_64mr : RI<0xC3, MRMDestMem, (outs), (ins i64mem:$dst, GR64:$src),3195 "movnti{q}\t{$src, $dst|$dst, $src}",3196 [(nontemporalstore (i64 GR64:$src), addr:$dst)]>,3197 TB, Requires<[HasSSE2]>;3198} // SchedRW = [WriteStoreNT]3199 3200let Predicates = [HasAVX, NoVLX] in {3201 def : Pat<(alignednontemporalstore (v8i32 VR256:$src), addr:$dst),3202 (VMOVNTDQYmr addr:$dst, VR256:$src)>;3203 def : Pat<(alignednontemporalstore (v16i16 VR256:$src), addr:$dst),3204 (VMOVNTDQYmr addr:$dst, VR256:$src)>;3205 def : Pat<(alignednontemporalstore (v16f16 VR256:$src), addr:$dst),3206 (VMOVNTDQYmr addr:$dst, VR256:$src)>;3207 def : Pat<(alignednontemporalstore (v32i8 VR256:$src), addr:$dst),3208 (VMOVNTDQYmr addr:$dst, VR256:$src)>;3209 3210 def : Pat<(alignednontemporalstore (v4i32 VR128:$src), addr:$dst),3211 (VMOVNTDQmr addr:$dst, VR128:$src)>;3212 def : Pat<(alignednontemporalstore (v8i16 VR128:$src), addr:$dst),3213 (VMOVNTDQmr addr:$dst, VR128:$src)>;3214 def : Pat<(alignednontemporalstore (v8f16 VR128:$src), addr:$dst),3215 (VMOVNTDQmr addr:$dst, VR128:$src)>;3216 def : Pat<(alignednontemporalstore (v16i8 VR128:$src), addr:$dst),3217 (VMOVNTDQmr addr:$dst, VR128:$src)>;3218}3219 3220let Predicates = [UseSSE2] in {3221 def : Pat<(alignednontemporalstore (v4i32 VR128:$src), addr:$dst),3222 (MOVNTDQmr addr:$dst, VR128:$src)>;3223 def : Pat<(alignednontemporalstore (v8i16 VR128:$src), addr:$dst),3224 (MOVNTDQmr addr:$dst, VR128:$src)>;3225 def : Pat<(alignednontemporalstore (v8f16 VR128:$src), addr:$dst),3226 (MOVNTDQmr addr:$dst, VR128:$src)>;3227 def : Pat<(alignednontemporalstore (v16i8 VR128:$src), addr:$dst),3228 (MOVNTDQmr addr:$dst, VR128:$src)>;3229}3230 3231} // AddedComplexity3232 3233//===----------------------------------------------------------------------===//3234// SSE 1 & 2 - Prefetch and memory fence3235//===----------------------------------------------------------------------===//3236 3237// Prefetch intrinsic.3238let Predicates = [HasSSEPrefetch], SchedRW = [WriteLoad] in {3239def PREFETCHT0 : I<0x18, MRM1m, (outs), (ins i8mem:$src),3240 "prefetcht0\t$src", [(prefetch addr:$src, timm, (i32 3), (i32 1))]>, TB;3241def PREFETCHT1 : I<0x18, MRM2m, (outs), (ins i8mem:$src),3242 "prefetcht1\t$src", [(prefetch addr:$src, timm, (i32 2), (i32 1))]>, TB;3243def PREFETCHT2 : I<0x18, MRM3m, (outs), (ins i8mem:$src),3244 "prefetcht2\t$src", [(prefetch addr:$src, timm, (i32 1), (i32 1))]>, TB;3245def PREFETCHNTA : I<0x18, MRM0m, (outs), (ins i8mem:$src),3246 "prefetchnta\t$src", [(prefetch addr:$src, timm, (i32 0), (i32 1))]>, TB;3247}3248 3249// FIXME: How should flush instruction be modeled?3250let SchedRW = [WriteLoad] in {3251// Flush cache3252def CLFLUSH : I<0xAE, MRM7m, (outs), (ins i8mem:$src),3253 "clflush\t$src", [(int_x86_sse2_clflush addr:$src)]>,3254 TB, Requires<[HasCLFLUSH]>;3255}3256 3257let SchedRW = [WriteNop] in {3258// Pause. This "instruction" is encoded as "rep; nop", so even though it3259// was introduced with SSE2, it's backward compatible.3260def PAUSE : I<0x90, RawFrm, (outs), (ins),3261 "pause", [(int_x86_sse2_pause)]>, XS;3262}3263 3264let SchedRW = [WriteFence] in {3265// Load, store, and memory fence3266// TODO: As with mfence, we may want to ease the availability of sfence/lfence3267// to include any 64-bit target.3268def SFENCE : I<0xAE, MRM7X, (outs), (ins), "sfence", [(int_x86_sse_sfence)]>,3269 TB, Requires<[HasSSE1]>;3270def LFENCE : I<0xAE, MRM5X, (outs), (ins), "lfence", [(int_x86_sse2_lfence)]>,3271 TB, Requires<[HasSSE2]>;3272def MFENCE : I<0xAE, MRM6X, (outs), (ins), "mfence", [(int_x86_sse2_mfence)]>,3273 TB, Requires<[HasMFence]>;3274} // SchedRW3275 3276def : Pat<(X86MFence), (MFENCE)>;3277 3278//===----------------------------------------------------------------------===//3279// SSE 1 & 2 - Load/Store XCSR register3280//===----------------------------------------------------------------------===//3281 3282let mayLoad=1, hasSideEffects=1, Defs=[MXCSR] in3283def VLDMXCSR : VPSI<0xAE, MRM2m, (outs), (ins i32mem:$src),3284 "ldmxcsr\t$src", [(int_x86_sse_ldmxcsr addr:$src)]>,3285 VEX, Sched<[WriteLDMXCSR]>, WIG;3286let mayStore=1, hasSideEffects=1, Uses=[MXCSR] in3287def VSTMXCSR : VPSI<0xAE, MRM3m, (outs), (ins i32mem:$dst),3288 "stmxcsr\t$dst", [(int_x86_sse_stmxcsr addr:$dst)]>,3289 VEX, Sched<[WriteSTMXCSR]>, WIG;3290 3291let mayLoad=1, hasSideEffects=1, Defs=[MXCSR] in3292def LDMXCSR : I<0xAE, MRM2m, (outs), (ins i32mem:$src),3293 "ldmxcsr\t$src", [(int_x86_sse_ldmxcsr addr:$src)]>,3294 TB, Sched<[WriteLDMXCSR]>;3295let mayStore=1, hasSideEffects=1, Uses=[MXCSR] in3296def STMXCSR : I<0xAE, MRM3m, (outs), (ins i32mem:$dst),3297 "stmxcsr\t$dst", [(int_x86_sse_stmxcsr addr:$dst)]>,3298 TB, Sched<[WriteSTMXCSR]>;3299 3300//===---------------------------------------------------------------------===//3301// SSE2 - Move Aligned/Unaligned Packed Integer Instructions3302//===---------------------------------------------------------------------===//3303 3304let ExeDomain = SSEPackedInt in { // SSE integer instructions3305 3306let hasSideEffects = 0 in {3307def VMOVDQArr : VPDI<0x6F, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),3308 "movdqa\t{$src, $dst|$dst, $src}", []>,3309 Sched<[SchedWriteVecMoveLS.XMM.RR]>, VEX, WIG;3310def VMOVDQUrr : VSSI<0x6F, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),3311 "movdqu\t{$src, $dst|$dst, $src}", []>,3312 Sched<[SchedWriteVecMoveLS.XMM.RR]>, VEX, WIG;3313def VMOVDQAYrr : VPDI<0x6F, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),3314 "movdqa\t{$src, $dst|$dst, $src}", []>,3315 Sched<[SchedWriteVecMoveLS.YMM.RR]>, VEX, VEX_L, WIG;3316def VMOVDQUYrr : VSSI<0x6F, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),3317 "movdqu\t{$src, $dst|$dst, $src}", []>,3318 Sched<[SchedWriteVecMoveLS.YMM.RR]>, VEX, VEX_L, WIG;3319}3320 3321// For Disassembler3322let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in {3323def VMOVDQArr_REV : VPDI<0x7F, MRMDestReg, (outs VR128:$dst), (ins VR128:$src),3324 "movdqa\t{$src, $dst|$dst, $src}", []>,3325 Sched<[SchedWriteVecMoveLS.XMM.RR]>,3326 VEX, WIG;3327def VMOVDQAYrr_REV : VPDI<0x7F, MRMDestReg, (outs VR256:$dst), (ins VR256:$src),3328 "movdqa\t{$src, $dst|$dst, $src}", []>,3329 Sched<[SchedWriteVecMoveLS.YMM.RR]>,3330 VEX, VEX_L, WIG;3331def VMOVDQUrr_REV : VSSI<0x7F, MRMDestReg, (outs VR128:$dst), (ins VR128:$src),3332 "movdqu\t{$src, $dst|$dst, $src}", []>,3333 Sched<[SchedWriteVecMoveLS.XMM.RR]>,3334 VEX, WIG;3335def VMOVDQUYrr_REV : VSSI<0x7F, MRMDestReg, (outs VR256:$dst), (ins VR256:$src),3336 "movdqu\t{$src, $dst|$dst, $src}", []>,3337 Sched<[SchedWriteVecMoveLS.YMM.RR]>,3338 VEX, VEX_L, WIG;3339}3340 3341let canFoldAsLoad = 1, mayLoad = 1, isReMaterializable = 1,3342 hasSideEffects = 0, Predicates = [HasAVX,NoVLX] in {3343def VMOVDQArm : VPDI<0x6F, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src),3344 "movdqa\t{$src, $dst|$dst, $src}",3345 [(set VR128:$dst, (alignedloadv2i64 addr:$src))]>,3346 Sched<[SchedWriteVecMoveLS.XMM.RM]>, VEX, WIG;3347def VMOVDQAYrm : VPDI<0x6F, MRMSrcMem, (outs VR256:$dst), (ins i256mem:$src),3348 "movdqa\t{$src, $dst|$dst, $src}", []>,3349 Sched<[SchedWriteVecMoveLS.YMM.RM]>,3350 VEX, VEX_L, WIG;3351def VMOVDQUrm : I<0x6F, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src),3352 "vmovdqu\t{$src, $dst|$dst, $src}",3353 [(set VR128:$dst, (loadv2i64 addr:$src))]>,3354 Sched<[SchedWriteVecMoveLS.XMM.RM]>,3355 TB, XS, VEX, WIG;3356def VMOVDQUYrm : I<0x6F, MRMSrcMem, (outs VR256:$dst), (ins i256mem:$src),3357 "vmovdqu\t{$src, $dst|$dst, $src}", []>,3358 Sched<[SchedWriteVecMoveLS.YMM.RM]>,3359 TB, XS, VEX, VEX_L, WIG;3360}3361 3362let mayStore = 1, hasSideEffects = 0, Predicates = [HasAVX,NoVLX] in {3363def VMOVDQAmr : VPDI<0x7F, MRMDestMem, (outs),3364 (ins i128mem:$dst, VR128:$src),3365 "movdqa\t{$src, $dst|$dst, $src}",3366 [(alignedstore (v2i64 VR128:$src), addr:$dst)]>,3367 Sched<[SchedWriteVecMoveLS.XMM.MR]>, VEX, WIG;3368def VMOVDQAYmr : VPDI<0x7F, MRMDestMem, (outs),3369 (ins i256mem:$dst, VR256:$src),3370 "movdqa\t{$src, $dst|$dst, $src}", []>,3371 Sched<[SchedWriteVecMoveLS.YMM.MR]>, VEX, VEX_L, WIG;3372def VMOVDQUmr : I<0x7F, MRMDestMem, (outs), (ins i128mem:$dst, VR128:$src),3373 "vmovdqu\t{$src, $dst|$dst, $src}",3374 [(store (v2i64 VR128:$src), addr:$dst)]>,3375 Sched<[SchedWriteVecMoveLS.XMM.MR]>, TB, XS, VEX, WIG;3376def VMOVDQUYmr : I<0x7F, MRMDestMem, (outs), (ins i256mem:$dst, VR256:$src),3377 "vmovdqu\t{$src, $dst|$dst, $src}",[]>,3378 Sched<[SchedWriteVecMoveLS.YMM.MR]>, TB, XS, VEX, VEX_L, WIG;3379}3380 3381let SchedRW = [SchedWriteVecMoveLS.XMM.RR] in {3382let hasSideEffects = 0 in {3383def MOVDQArr : PDI<0x6F, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),3384 "movdqa\t{$src, $dst|$dst, $src}", []>;3385 3386def MOVDQUrr : I<0x6F, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),3387 "movdqu\t{$src, $dst|$dst, $src}", []>,3388 TB, XS, Requires<[UseSSE2]>;3389}3390 3391// For Disassembler3392let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in {3393def MOVDQArr_REV : PDI<0x7F, MRMDestReg, (outs VR128:$dst), (ins VR128:$src),3394 "movdqa\t{$src, $dst|$dst, $src}", []>;3395 3396def MOVDQUrr_REV : I<0x7F, MRMDestReg, (outs VR128:$dst), (ins VR128:$src),3397 "movdqu\t{$src, $dst|$dst, $src}", []>,3398 TB, XS, Requires<[UseSSE2]>;3399}3400} // SchedRW3401 3402let canFoldAsLoad = 1, mayLoad = 1, isReMaterializable = 1,3403 hasSideEffects = 0, SchedRW = [SchedWriteVecMoveLS.XMM.RM] in {3404def MOVDQArm : PDI<0x6F, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src),3405 "movdqa\t{$src, $dst|$dst, $src}",3406 [/*(set VR128:$dst, (alignedloadv2i64 addr:$src))*/]>;3407def MOVDQUrm : I<0x6F, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src),3408 "movdqu\t{$src, $dst|$dst, $src}",3409 [/*(set VR128:$dst, (loadv2i64 addr:$src))*/]>,3410 TB, XS, Requires<[UseSSE2]>;3411}3412 3413let mayStore = 1, hasSideEffects = 0,3414 SchedRW = [SchedWriteVecMoveLS.XMM.MR] in {3415def MOVDQAmr : PDI<0x7F, MRMDestMem, (outs), (ins i128mem:$dst, VR128:$src),3416 "movdqa\t{$src, $dst|$dst, $src}",3417 [/*(alignedstore (v2i64 VR128:$src), addr:$dst)*/]>;3418def MOVDQUmr : I<0x7F, MRMDestMem, (outs), (ins i128mem:$dst, VR128:$src),3419 "movdqu\t{$src, $dst|$dst, $src}",3420 [/*(store (v2i64 VR128:$src), addr:$dst)*/]>,3421 TB, XS, Requires<[UseSSE2]>;3422}3423 3424} // ExeDomain = SSEPackedInt3425 3426// Reversed version with ".s" suffix for GAS compatibility.3427def : InstAlias<"vmovdqa.s\t{$src, $dst|$dst, $src}",3428 (VMOVDQArr_REV VR128:$dst, VR128:$src), 0>;3429def : InstAlias<"vmovdqa.s\t{$src, $dst|$dst, $src}",3430 (VMOVDQAYrr_REV VR256:$dst, VR256:$src), 0>;3431def : InstAlias<"vmovdqu.s\t{$src, $dst|$dst, $src}",3432 (VMOVDQUrr_REV VR128:$dst, VR128:$src), 0>;3433def : InstAlias<"vmovdqu.s\t{$src, $dst|$dst, $src}",3434 (VMOVDQUYrr_REV VR256:$dst, VR256:$src), 0>;3435 3436// Reversed version with ".s" suffix for GAS compatibility.3437def : InstAlias<"movdqa.s\t{$src, $dst|$dst, $src}",3438 (MOVDQArr_REV VR128:$dst, VR128:$src), 0>;3439def : InstAlias<"movdqu.s\t{$src, $dst|$dst, $src}",3440 (MOVDQUrr_REV VR128:$dst, VR128:$src), 0>;3441 3442let Predicates = [HasAVX, NoVLX] in {3443 // Additional patterns for other integer sizes.3444 def : Pat<(alignedloadv4i32 addr:$src),3445 (VMOVDQArm addr:$src)>;3446 def : Pat<(alignedloadv8i16 addr:$src),3447 (VMOVDQArm addr:$src)>;3448 def : Pat<(alignedloadv8f16 addr:$src),3449 (VMOVDQArm addr:$src)>;3450 def : Pat<(alignedloadv16i8 addr:$src),3451 (VMOVDQArm addr:$src)>;3452 def : Pat<(loadv4i32 addr:$src),3453 (VMOVDQUrm addr:$src)>;3454 def : Pat<(loadv8i16 addr:$src),3455 (VMOVDQUrm addr:$src)>;3456 def : Pat<(loadv8f16 addr:$src),3457 (VMOVDQUrm addr:$src)>;3458 def : Pat<(loadv16i8 addr:$src),3459 (VMOVDQUrm addr:$src)>;3460 3461 def : Pat<(alignedstore (v4i32 VR128:$src), addr:$dst),3462 (VMOVDQAmr addr:$dst, VR128:$src)>;3463 def : Pat<(alignedstore (v8i16 VR128:$src), addr:$dst),3464 (VMOVDQAmr addr:$dst, VR128:$src)>;3465 def : Pat<(alignedstore (v8f16 VR128:$src), addr:$dst),3466 (VMOVDQAmr addr:$dst, VR128:$src)>;3467 def : Pat<(alignedstore (v16i8 VR128:$src), addr:$dst),3468 (VMOVDQAmr addr:$dst, VR128:$src)>;3469 def : Pat<(store (v4i32 VR128:$src), addr:$dst),3470 (VMOVDQUmr addr:$dst, VR128:$src)>;3471 def : Pat<(store (v8i16 VR128:$src), addr:$dst),3472 (VMOVDQUmr addr:$dst, VR128:$src)>;3473 def : Pat<(store (v8f16 VR128:$src), addr:$dst),3474 (VMOVDQUmr addr:$dst, VR128:$src)>;3475 def : Pat<(store (v16i8 VR128:$src), addr:$dst),3476 (VMOVDQUmr addr:$dst, VR128:$src)>;3477}3478 3479//===---------------------------------------------------------------------===//3480// SSE2 - Packed Integer Arithmetic Instructions3481//===---------------------------------------------------------------------===//3482 3483let ExeDomain = SSEPackedInt in { // SSE integer instructions3484 3485/// PDI_binop_rm2 - Simple SSE2 binary operator with different src and dst types3486multiclass PDI_binop_rm2<bits<8> opc, string OpcodeStr, SDNode OpNode,3487 ValueType DstVT, ValueType SrcVT, RegisterClass RC,3488 PatFrag memop_frag, X86MemOperand x86memop,3489 X86FoldableSchedWrite sched, bit Is2Addr = 1> {3490 let isCommutable = 1 in3491 def rr : PDI<opc, MRMSrcReg, (outs RC:$dst),3492 (ins RC:$src1, RC:$src2),3493 !if(Is2Addr,3494 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),3495 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),3496 [(set RC:$dst, (DstVT (OpNode (SrcVT RC:$src1), RC:$src2)))]>,3497 Sched<[sched]>;3498 def rm : PDI<opc, MRMSrcMem, (outs RC:$dst),3499 (ins RC:$src1, x86memop:$src2),3500 !if(Is2Addr,3501 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),3502 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),3503 [(set RC:$dst, (DstVT (OpNode (SrcVT RC:$src1),3504 (memop_frag addr:$src2))))]>,3505 Sched<[sched.Folded, sched.ReadAfterFold]>;3506}3507} // ExeDomain = SSEPackedInt3508 3509defm PADDB : PDI_binop_all<0xFC, "paddb", add, v16i8, v32i8,3510 SchedWriteVecALU, 1, NoVLX_Or_NoBWI>;3511defm PADDW : PDI_binop_all<0xFD, "paddw", add, v8i16, v16i16,3512 SchedWriteVecALU, 1, NoVLX_Or_NoBWI>;3513defm PADDD : PDI_binop_all<0xFE, "paddd", add, v4i32, v8i32,3514 SchedWriteVecALU, 1, NoVLX>;3515defm PADDQ : PDI_binop_all<0xD4, "paddq", add, v2i64, v4i64,3516 SchedWriteVecALU, 1, NoVLX>;3517defm PADDSB : PDI_binop_all<0xEC, "paddsb", saddsat, v16i8, v32i8,3518 SchedWriteVecALU, 1, NoVLX_Or_NoBWI>;3519defm PADDSW : PDI_binop_all<0xED, "paddsw", saddsat, v8i16, v16i16,3520 SchedWriteVecALU, 1, NoVLX_Or_NoBWI>;3521defm PADDUSB : PDI_binop_all<0xDC, "paddusb", uaddsat, v16i8, v32i8,3522 SchedWriteVecALU, 1, NoVLX_Or_NoBWI>;3523defm PADDUSW : PDI_binop_all<0xDD, "paddusw", uaddsat, v8i16, v16i16,3524 SchedWriteVecALU, 1, NoVLX_Or_NoBWI>;3525defm PMULLW : PDI_binop_all<0xD5, "pmullw", mul, v8i16, v16i16,3526 SchedWriteVecIMul, 1, NoVLX_Or_NoBWI>;3527defm PMULHUW : PDI_binop_all<0xE4, "pmulhuw", mulhu, v8i16, v16i16,3528 SchedWriteVecIMul, 1, NoVLX_Or_NoBWI>;3529defm PMULHW : PDI_binop_all<0xE5, "pmulhw", mulhs, v8i16, v16i16,3530 SchedWriteVecIMul, 1, NoVLX_Or_NoBWI>;3531defm PSUBB : PDI_binop_all<0xF8, "psubb", sub, v16i8, v32i8,3532 SchedWriteVecALU, 0, NoVLX_Or_NoBWI>;3533defm PSUBW : PDI_binop_all<0xF9, "psubw", sub, v8i16, v16i16,3534 SchedWriteVecALU, 0, NoVLX_Or_NoBWI>;3535defm PSUBD : PDI_binop_all<0xFA, "psubd", sub, v4i32, v8i32,3536 SchedWriteVecALU, 0, NoVLX>;3537defm PSUBQ : PDI_binop_all<0xFB, "psubq", sub, v2i64, v4i64,3538 SchedWriteVecALU, 0, NoVLX>;3539defm PSUBSB : PDI_binop_all<0xE8, "psubsb", ssubsat, v16i8, v32i8,3540 SchedWriteVecALU, 0, NoVLX_Or_NoBWI>;3541defm PSUBSW : PDI_binop_all<0xE9, "psubsw", ssubsat, v8i16, v16i16,3542 SchedWriteVecALU, 0, NoVLX_Or_NoBWI>;3543defm PSUBUSB : PDI_binop_all<0xD8, "psubusb", usubsat, v16i8, v32i8,3544 SchedWriteVecALU, 0, NoVLX_Or_NoBWI>;3545defm PSUBUSW : PDI_binop_all<0xD9, "psubusw", usubsat, v8i16, v16i16,3546 SchedWriteVecALU, 0, NoVLX_Or_NoBWI>;3547defm PMINUB : PDI_binop_all<0xDA, "pminub", umin, v16i8, v32i8,3548 SchedWriteVecALU, 1, NoVLX_Or_NoBWI>;3549defm PMINSW : PDI_binop_all<0xEA, "pminsw", smin, v8i16, v16i16,3550 SchedWriteVecALU, 1, NoVLX_Or_NoBWI>;3551defm PMAXUB : PDI_binop_all<0xDE, "pmaxub", umax, v16i8, v32i8,3552 SchedWriteVecALU, 1, NoVLX_Or_NoBWI>;3553defm PMAXSW : PDI_binop_all<0xEE, "pmaxsw", smax, v8i16, v16i16,3554 SchedWriteVecALU, 1, NoVLX_Or_NoBWI>;3555defm PAVGB : PDI_binop_all<0xE0, "pavgb", avgceilu, v16i8, v32i8,3556 SchedWriteVecALU, 1, NoVLX_Or_NoBWI>;3557defm PAVGW : PDI_binop_all<0xE3, "pavgw", avgceilu, v8i16, v16i16,3558 SchedWriteVecALU, 1, NoVLX_Or_NoBWI>;3559defm PMULUDQ : PDI_binop_all<0xF4, "pmuludq", X86pmuludq, v2i64, v4i64,3560 SchedWriteVecIMul, 1, NoVLX>;3561 3562let Predicates = [HasAVX, NoVLX_Or_NoBWI] in3563defm VPMADDWD : PDI_binop_rm2<0xF5, "vpmaddwd", X86vpmaddwd, v4i32, v8i16, VR128,3564 load, i128mem, SchedWriteVecIMul.XMM, 0>,3565 VEX, VVVV, WIG;3566 3567let Predicates = [HasAVX2, NoVLX_Or_NoBWI] in3568defm VPMADDWDY : PDI_binop_rm2<0xF5, "vpmaddwd", X86vpmaddwd, v8i32, v16i16,3569 VR256, load, i256mem, SchedWriteVecIMul.YMM,3570 0>, VEX, VVVV, VEX_L, WIG;3571let Constraints = "$src1 = $dst" in3572defm PMADDWD : PDI_binop_rm2<0xF5, "pmaddwd", X86vpmaddwd, v4i32, v8i16, VR128,3573 memop, i128mem, SchedWriteVecIMul.XMM>;3574 3575let Predicates = [HasAVX, NoVLX_Or_NoBWI] in3576defm VPSADBW : PDI_binop_rm2<0xF6, "vpsadbw", X86psadbw, v2i64, v16i8, VR128,3577 load, i128mem, SchedWritePSADBW.XMM, 0>,3578 VEX, VVVV, WIG;3579let Predicates = [HasAVX2, NoVLX_Or_NoBWI] in3580defm VPSADBWY : PDI_binop_rm2<0xF6, "vpsadbw", X86psadbw, v4i64, v32i8, VR256,3581 load, i256mem, SchedWritePSADBW.YMM, 0>,3582 VEX, VVVV, VEX_L, WIG;3583let Constraints = "$src1 = $dst" in3584defm PSADBW : PDI_binop_rm2<0xF6, "psadbw", X86psadbw, v2i64, v16i8, VR128,3585 memop, i128mem, SchedWritePSADBW.XMM>;3586 3587//===---------------------------------------------------------------------===//3588// SSE2 - Packed Integer Logical Instructions3589//===---------------------------------------------------------------------===//3590 3591multiclass PDI_binop_rmi<bits<8> opc, bits<8> opc2, Format ImmForm,3592 string OpcodeStr, SDNode OpNode,3593 SDNode OpNode2, RegisterClass RC,3594 X86FoldableSchedWrite sched,3595 X86FoldableSchedWrite schedImm,3596 ValueType DstVT, ValueType SrcVT,3597 PatFrag ld_frag, bit Is2Addr = 1> {3598 // src2 is always 128-bit3599 def rr : PDI<opc, MRMSrcReg, (outs RC:$dst),3600 (ins RC:$src1, VR128:$src2),3601 !if(Is2Addr,3602 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),3603 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),3604 [(set RC:$dst, (DstVT (OpNode RC:$src1, (SrcVT VR128:$src2))))]>,3605 Sched<[sched]>;3606 def rm : PDI<opc, MRMSrcMem, (outs RC:$dst),3607 (ins RC:$src1, i128mem:$src2),3608 !if(Is2Addr,3609 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),3610 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),3611 [(set RC:$dst, (DstVT (OpNode RC:$src1,3612 (SrcVT (ld_frag addr:$src2)))))]>,3613 Sched<[sched.Folded, sched.ReadAfterFold]>;3614 def ri : PDIi8<opc2, ImmForm, (outs RC:$dst),3615 (ins RC:$src1, u8imm:$src2),3616 !if(Is2Addr,3617 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),3618 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),3619 [(set RC:$dst, (DstVT (OpNode2 RC:$src1, (i8 timm:$src2))))]>,3620 Sched<[schedImm]>;3621}3622 3623multiclass PDI_binop_rmi_all<bits<8> opc, bits<8> opc2, Format ImmForm,3624 string OpcodeStr, SDNode OpNode,3625 SDNode OpNode2, ValueType DstVT128,3626 ValueType DstVT256, ValueType SrcVT,3627 X86SchedWriteWidths sched,3628 X86SchedWriteWidths schedImm, Predicate prd> {3629let Predicates = [HasAVX, prd] in3630 defm V#NAME : PDI_binop_rmi<opc, opc2, ImmForm, !strconcat("v", OpcodeStr),3631 OpNode, OpNode2, VR128, sched.XMM, schedImm.XMM,3632 DstVT128, SrcVT, load, 0>, VEX, VVVV, WIG;3633let Predicates = [HasAVX2, prd] in3634 defm V#NAME#Y : PDI_binop_rmi<opc, opc2, ImmForm, !strconcat("v", OpcodeStr),3635 OpNode, OpNode2, VR256, sched.YMM, schedImm.YMM,3636 DstVT256, SrcVT, load, 0>, VEX, VVVV, VEX_L,3637 WIG;3638let Constraints = "$src1 = $dst" in3639 defm NAME : PDI_binop_rmi<opc, opc2, ImmForm, OpcodeStr, OpNode, OpNode2,3640 VR128, sched.XMM, schedImm.XMM, DstVT128, SrcVT,3641 memop>;3642}3643 3644multiclass PDI_binop_ri<bits<8> opc, Format ImmForm, string OpcodeStr,3645 SDNode OpNode, RegisterClass RC, ValueType VT,3646 X86FoldableSchedWrite sched, bit Is2Addr = 1> {3647 def ri : PDIi8<opc, ImmForm, (outs RC:$dst), (ins RC:$src1, u8imm:$src2),3648 !if(Is2Addr,3649 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),3650 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),3651 [(set RC:$dst, (VT (OpNode RC:$src1, (i8 timm:$src2))))]>,3652 Sched<[sched]>;3653}3654 3655multiclass PDI_binop_ri_all<bits<8> opc, Format ImmForm, string OpcodeStr,3656 SDNode OpNode, X86SchedWriteWidths sched> {3657let Predicates = [HasAVX, NoVLX_Or_NoBWI] in3658 defm V#NAME : PDI_binop_ri<opc, ImmForm, !strconcat("v", OpcodeStr), OpNode,3659 VR128, v16i8, sched.XMM, 0>, VEX, VVVV, WIG;3660let Predicates = [HasAVX2, NoVLX_Or_NoBWI] in3661 defm V#NAME#Y : PDI_binop_ri<opc, ImmForm, !strconcat("v", OpcodeStr), OpNode,3662 VR256, v32i8, sched.YMM, 0>,3663 VEX, VVVV, VEX_L, WIG;3664let Constraints = "$src1 = $dst" in3665 defm NAME : PDI_binop_ri<opc, ImmForm, OpcodeStr, OpNode, VR128, v16i8,3666 sched.XMM>;3667}3668 3669let ExeDomain = SSEPackedInt in {3670 defm PSLLW : PDI_binop_rmi_all<0xF1, 0x71, MRM6r, "psllw", X86vshl, X86vshli,3671 v8i16, v16i16, v8i16, SchedWriteVecShift,3672 SchedWriteVecShiftImm, NoVLX_Or_NoBWI>;3673 defm PSLLD : PDI_binop_rmi_all<0xF2, 0x72, MRM6r, "pslld", X86vshl, X86vshli,3674 v4i32, v8i32, v4i32, SchedWriteVecShift,3675 SchedWriteVecShiftImm, NoVLX>;3676 defm PSLLQ : PDI_binop_rmi_all<0xF3, 0x73, MRM6r, "psllq", X86vshl, X86vshli,3677 v2i64, v4i64, v2i64, SchedWriteVecShift,3678 SchedWriteVecShiftImm, NoVLX>;3679 3680 defm PSRLW : PDI_binop_rmi_all<0xD1, 0x71, MRM2r, "psrlw", X86vsrl, X86vsrli,3681 v8i16, v16i16, v8i16, SchedWriteVecShift,3682 SchedWriteVecShiftImm, NoVLX_Or_NoBWI>;3683 defm PSRLD : PDI_binop_rmi_all<0xD2, 0x72, MRM2r, "psrld", X86vsrl, X86vsrli,3684 v4i32, v8i32, v4i32, SchedWriteVecShift,3685 SchedWriteVecShiftImm, NoVLX>;3686 defm PSRLQ : PDI_binop_rmi_all<0xD3, 0x73, MRM2r, "psrlq", X86vsrl, X86vsrli,3687 v2i64, v4i64, v2i64, SchedWriteVecShift,3688 SchedWriteVecShiftImm, NoVLX>;3689 3690 defm PSRAW : PDI_binop_rmi_all<0xE1, 0x71, MRM4r, "psraw", X86vsra, X86vsrai,3691 v8i16, v16i16, v8i16, SchedWriteVecShift,3692 SchedWriteVecShiftImm, NoVLX_Or_NoBWI>;3693 defm PSRAD : PDI_binop_rmi_all<0xE2, 0x72, MRM4r, "psrad", X86vsra, X86vsrai,3694 v4i32, v8i32, v4i32, SchedWriteVecShift,3695 SchedWriteVecShiftImm, NoVLX>;3696 3697 defm PSLLDQ : PDI_binop_ri_all<0x73, MRM7r, "pslldq", X86vshldq,3698 SchedWriteShuffle>;3699 defm PSRLDQ : PDI_binop_ri_all<0x73, MRM3r, "psrldq", X86vshrdq,3700 SchedWriteShuffle>;3701} // ExeDomain = SSEPackedInt3702 3703//===---------------------------------------------------------------------===//3704// SSE2 - Packed Integer Comparison Instructions3705//===---------------------------------------------------------------------===//3706 3707defm PCMPEQB : PDI_binop_all<0x74, "pcmpeqb", X86pcmpeq, v16i8, v32i8,3708 SchedWriteVecALU, 1, TruePredicate>;3709defm PCMPEQW : PDI_binop_all<0x75, "pcmpeqw", X86pcmpeq, v8i16, v16i16,3710 SchedWriteVecALU, 1, TruePredicate>;3711defm PCMPEQD : PDI_binop_all<0x76, "pcmpeqd", X86pcmpeq, v4i32, v8i32,3712 SchedWriteVecALU, 1, TruePredicate>;3713defm PCMPGTB : PDI_binop_all<0x64, "pcmpgtb", X86pcmpgt, v16i8, v32i8,3714 SchedWriteVecALU, 0, TruePredicate>;3715defm PCMPGTW : PDI_binop_all<0x65, "pcmpgtw", X86pcmpgt, v8i16, v16i16,3716 SchedWriteVecALU, 0, TruePredicate>;3717defm PCMPGTD : PDI_binop_all<0x66, "pcmpgtd", X86pcmpgt, v4i32, v8i32,3718 SchedWriteVecALU, 0, TruePredicate>;3719 3720//===---------------------------------------------------------------------===//3721// SSE2 - Packed Integer Shuffle Instructions3722//===---------------------------------------------------------------------===//3723 3724let ExeDomain = SSEPackedInt in {3725multiclass sse2_pshuffle<string OpcodeStr, ValueType vt128, ValueType vt256,3726 SDNode OpNode, X86SchedWriteWidths sched,3727 Predicate prd> {3728let Predicates = [HasAVX, prd] in {3729 def V#NAME#ri : Ii8<0x70, MRMSrcReg, (outs VR128:$dst),3730 (ins VR128:$src1, u8imm:$src2),3731 !strconcat("v", OpcodeStr,3732 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),3733 [(set VR128:$dst,3734 (vt128 (OpNode VR128:$src1, (i8 timm:$src2))))]>,3735 VEX, Sched<[sched.XMM]>, WIG;3736 def V#NAME#mi : Ii8<0x70, MRMSrcMem, (outs VR128:$dst),3737 (ins i128mem:$src1, u8imm:$src2),3738 !strconcat("v", OpcodeStr,3739 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),3740 [(set VR128:$dst,3741 (vt128 (OpNode (load addr:$src1),3742 (i8 timm:$src2))))]>, VEX,3743 Sched<[sched.XMM.Folded]>, WIG;3744}3745 3746let Predicates = [HasAVX2, prd] in {3747 def V#NAME#Yri : Ii8<0x70, MRMSrcReg, (outs VR256:$dst),3748 (ins VR256:$src1, u8imm:$src2),3749 !strconcat("v", OpcodeStr,3750 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),3751 [(set VR256:$dst,3752 (vt256 (OpNode VR256:$src1, (i8 timm:$src2))))]>,3753 VEX, VEX_L, Sched<[sched.YMM]>, WIG;3754 def V#NAME#Ymi : Ii8<0x70, MRMSrcMem, (outs VR256:$dst),3755 (ins i256mem:$src1, u8imm:$src2),3756 !strconcat("v", OpcodeStr,3757 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),3758 [(set VR256:$dst,3759 (vt256 (OpNode (load addr:$src1),3760 (i8 timm:$src2))))]>, VEX, VEX_L,3761 Sched<[sched.YMM.Folded]>, WIG;3762}3763 3764let Predicates = [UseSSE2] in {3765 def ri : Ii8<0x70, MRMSrcReg,3766 (outs VR128:$dst), (ins VR128:$src1, u8imm:$src2),3767 !strconcat(OpcodeStr,3768 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),3769 [(set VR128:$dst,3770 (vt128 (OpNode VR128:$src1, (i8 timm:$src2))))]>,3771 Sched<[sched.XMM]>;3772 def mi : Ii8<0x70, MRMSrcMem,3773 (outs VR128:$dst), (ins i128mem:$src1, u8imm:$src2),3774 !strconcat(OpcodeStr,3775 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),3776 [(set VR128:$dst,3777 (vt128 (OpNode (memop addr:$src1),3778 (i8 timm:$src2))))]>,3779 Sched<[sched.XMM.Folded]>;3780}3781}3782} // ExeDomain = SSEPackedInt3783 3784defm PSHUFD : sse2_pshuffle<"pshufd", v4i32, v8i32, X86PShufd,3785 SchedWriteShuffle, NoVLX>, TB, PD;3786defm PSHUFHW : sse2_pshuffle<"pshufhw", v8i16, v16i16, X86PShufhw,3787 SchedWriteShuffle, NoVLX_Or_NoBWI>, TB, XS;3788defm PSHUFLW : sse2_pshuffle<"pshuflw", v8i16, v16i16, X86PShuflw,3789 SchedWriteShuffle, NoVLX_Or_NoBWI>, TB, XD;3790 3791//===---------------------------------------------------------------------===//3792// Packed Integer Pack Instructions (SSE & AVX)3793//===---------------------------------------------------------------------===//3794 3795let ExeDomain = SSEPackedInt in {3796multiclass sse2_pack<bits<8> opc, string OpcodeStr, ValueType OutVT,3797 ValueType ArgVT, SDNode OpNode, RegisterClass RC,3798 X86MemOperand x86memop, X86FoldableSchedWrite sched,3799 PatFrag ld_frag, bit Is2Addr = 1> {3800 def rr : PDI<opc, MRMSrcReg,3801 (outs RC:$dst), (ins RC:$src1, RC:$src2),3802 !if(Is2Addr,3803 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),3804 !strconcat(OpcodeStr,3805 "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),3806 [(set RC:$dst,3807 (OutVT (OpNode (ArgVT RC:$src1), RC:$src2)))]>,3808 Sched<[sched]>;3809 def rm : PDI<opc, MRMSrcMem,3810 (outs RC:$dst), (ins RC:$src1, x86memop:$src2),3811 !if(Is2Addr,3812 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),3813 !strconcat(OpcodeStr,3814 "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),3815 [(set RC:$dst,3816 (OutVT (OpNode (ArgVT RC:$src1),3817 (ld_frag addr:$src2))))]>,3818 Sched<[sched.Folded, sched.ReadAfterFold]>;3819}3820 3821multiclass sse4_pack<bits<8> opc, string OpcodeStr, ValueType OutVT,3822 ValueType ArgVT, SDNode OpNode, RegisterClass RC,3823 X86MemOperand x86memop, X86FoldableSchedWrite sched,3824 PatFrag ld_frag, bit Is2Addr = 1> {3825 def rr : SS48I<opc, MRMSrcReg,3826 (outs RC:$dst), (ins RC:$src1, RC:$src2),3827 !if(Is2Addr,3828 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),3829 !strconcat(OpcodeStr,3830 "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),3831 [(set RC:$dst,3832 (OutVT (OpNode (ArgVT RC:$src1), RC:$src2)))]>,3833 Sched<[sched]>;3834 def rm : SS48I<opc, MRMSrcMem,3835 (outs RC:$dst), (ins RC:$src1, x86memop:$src2),3836 !if(Is2Addr,3837 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),3838 !strconcat(OpcodeStr,3839 "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),3840 [(set RC:$dst,3841 (OutVT (OpNode (ArgVT RC:$src1),3842 (ld_frag addr:$src2))))]>,3843 Sched<[sched.Folded, sched.ReadAfterFold]>;3844}3845 3846let Predicates = [HasAVX, NoVLX_Or_NoBWI] in {3847 defm VPACKSSWB : sse2_pack<0x63, "vpacksswb", v16i8, v8i16, X86Packss, VR128,3848 i128mem, SchedWriteShuffle.XMM, load, 0>,3849 VEX, VVVV, WIG;3850 defm VPACKSSDW : sse2_pack<0x6B, "vpackssdw", v8i16, v4i32, X86Packss, VR128,3851 i128mem, SchedWriteShuffle.XMM, load, 0>,3852 VEX, VVVV, WIG;3853 3854 defm VPACKUSWB : sse2_pack<0x67, "vpackuswb", v16i8, v8i16, X86Packus, VR128,3855 i128mem, SchedWriteShuffle.XMM, load, 0>,3856 VEX, VVVV, WIG;3857 defm VPACKUSDW : sse4_pack<0x2B, "vpackusdw", v8i16, v4i32, X86Packus, VR128,3858 i128mem, SchedWriteShuffle.XMM, load, 0>,3859 VEX, VVVV, WIG;3860}3861 3862let Predicates = [HasAVX2, NoVLX_Or_NoBWI] in {3863 defm VPACKSSWBY : sse2_pack<0x63, "vpacksswb", v32i8, v16i16, X86Packss, VR256,3864 i256mem, SchedWriteShuffle.YMM, load, 0>,3865 VEX, VVVV, VEX_L, WIG;3866 defm VPACKSSDWY : sse2_pack<0x6B, "vpackssdw", v16i16, v8i32, X86Packss, VR256,3867 i256mem, SchedWriteShuffle.YMM, load, 0>,3868 VEX, VVVV, VEX_L, WIG;3869 3870 defm VPACKUSWBY : sse2_pack<0x67, "vpackuswb", v32i8, v16i16, X86Packus, VR256,3871 i256mem, SchedWriteShuffle.YMM, load, 0>,3872 VEX, VVVV, VEX_L, WIG;3873 defm VPACKUSDWY : sse4_pack<0x2B, "vpackusdw", v16i16, v8i32, X86Packus, VR256,3874 i256mem, SchedWriteShuffle.YMM, load, 0>,3875 VEX, VVVV, VEX_L, WIG;3876}3877 3878let Constraints = "$src1 = $dst" in {3879 defm PACKSSWB : sse2_pack<0x63, "packsswb", v16i8, v8i16, X86Packss, VR128,3880 i128mem, SchedWriteShuffle.XMM, memop>;3881 defm PACKSSDW : sse2_pack<0x6B, "packssdw", v8i16, v4i32, X86Packss, VR128,3882 i128mem, SchedWriteShuffle.XMM, memop>;3883 3884 defm PACKUSWB : sse2_pack<0x67, "packuswb", v16i8, v8i16, X86Packus, VR128,3885 i128mem, SchedWriteShuffle.XMM, memop>;3886 3887 defm PACKUSDW : sse4_pack<0x2B, "packusdw", v8i16, v4i32, X86Packus, VR128,3888 i128mem, SchedWriteShuffle.XMM, memop>;3889}3890} // ExeDomain = SSEPackedInt3891 3892//===---------------------------------------------------------------------===//3893// SSE2 - Packed Integer Unpack Instructions3894//===---------------------------------------------------------------------===//3895 3896let ExeDomain = SSEPackedInt in {3897multiclass sse2_unpack<bits<8> opc, string OpcodeStr, ValueType vt,3898 SDNode OpNode, RegisterClass RC, X86MemOperand x86memop,3899 X86FoldableSchedWrite sched, PatFrag ld_frag,3900 bit Is2Addr = 1> {3901 def rr : PDI<opc, MRMSrcReg,3902 (outs RC:$dst), (ins RC:$src1, RC:$src2),3903 !if(Is2Addr,3904 !strconcat(OpcodeStr,"\t{$src2, $dst|$dst, $src2}"),3905 !strconcat(OpcodeStr,"\t{$src2, $src1, $dst|$dst, $src1, $src2}")),3906 [(set RC:$dst, (vt (OpNode RC:$src1, RC:$src2)))]>,3907 Sched<[sched]>;3908 def rm : PDI<opc, MRMSrcMem,3909 (outs RC:$dst), (ins RC:$src1, x86memop:$src2),3910 !if(Is2Addr,3911 !strconcat(OpcodeStr,"\t{$src2, $dst|$dst, $src2}"),3912 !strconcat(OpcodeStr,"\t{$src2, $src1, $dst|$dst, $src1, $src2}")),3913 [(set RC:$dst, (vt (OpNode RC:$src1, (ld_frag addr:$src2))))]>,3914 Sched<[sched.Folded, sched.ReadAfterFold]>;3915}3916 3917let Predicates = [HasAVX, NoVLX_Or_NoBWI] in {3918 defm VPUNPCKLBW : sse2_unpack<0x60, "vpunpcklbw", v16i8, X86Unpckl, VR128,3919 i128mem, SchedWriteShuffle.XMM, load, 0>,3920 VEX, VVVV, WIG;3921 defm VPUNPCKLWD : sse2_unpack<0x61, "vpunpcklwd", v8i16, X86Unpckl, VR128,3922 i128mem, SchedWriteShuffle.XMM, load, 0>,3923 VEX, VVVV, WIG;3924 defm VPUNPCKHBW : sse2_unpack<0x68, "vpunpckhbw", v16i8, X86Unpckh, VR128,3925 i128mem, SchedWriteShuffle.XMM, load, 0>,3926 VEX, VVVV, WIG;3927 defm VPUNPCKHWD : sse2_unpack<0x69, "vpunpckhwd", v8i16, X86Unpckh, VR128,3928 i128mem, SchedWriteShuffle.XMM, load, 0>,3929 VEX, VVVV, WIG;3930}3931 3932let Predicates = [HasAVX, NoVLX] in {3933 defm VPUNPCKLDQ : sse2_unpack<0x62, "vpunpckldq", v4i32, X86Unpckl, VR128,3934 i128mem, SchedWriteShuffle.XMM, load, 0>,3935 VEX, VVVV, WIG;3936 defm VPUNPCKLQDQ : sse2_unpack<0x6C, "vpunpcklqdq", v2i64, X86Unpckl, VR128,3937 i128mem, SchedWriteShuffle.XMM, load, 0>,3938 VEX, VVVV, WIG;3939 defm VPUNPCKHDQ : sse2_unpack<0x6A, "vpunpckhdq", v4i32, X86Unpckh, VR128,3940 i128mem, SchedWriteShuffle.XMM, load, 0>,3941 VEX, VVVV, WIG;3942 defm VPUNPCKHQDQ : sse2_unpack<0x6D, "vpunpckhqdq", v2i64, X86Unpckh, VR128,3943 i128mem, SchedWriteShuffle.XMM, load, 0>,3944 VEX, VVVV, WIG;3945}3946 3947let Predicates = [HasAVX2, NoVLX_Or_NoBWI] in {3948 defm VPUNPCKLBWY : sse2_unpack<0x60, "vpunpcklbw", v32i8, X86Unpckl, VR256,3949 i256mem, SchedWriteShuffle.YMM, load, 0>,3950 VEX, VVVV, VEX_L, WIG;3951 defm VPUNPCKLWDY : sse2_unpack<0x61, "vpunpcklwd", v16i16, X86Unpckl, VR256,3952 i256mem, SchedWriteShuffle.YMM, load, 0>,3953 VEX, VVVV, VEX_L, WIG;3954 defm VPUNPCKHBWY : sse2_unpack<0x68, "vpunpckhbw", v32i8, X86Unpckh, VR256,3955 i256mem, SchedWriteShuffle.YMM, load, 0>,3956 VEX, VVVV, VEX_L, WIG;3957 defm VPUNPCKHWDY : sse2_unpack<0x69, "vpunpckhwd", v16i16, X86Unpckh, VR256,3958 i256mem, SchedWriteShuffle.YMM, load, 0>,3959 VEX, VVVV, VEX_L, WIG;3960}3961 3962let Predicates = [HasAVX2, NoVLX] in {3963 defm VPUNPCKLDQY : sse2_unpack<0x62, "vpunpckldq", v8i32, X86Unpckl, VR256,3964 i256mem, SchedWriteShuffle.YMM, load, 0>,3965 VEX, VVVV, VEX_L, WIG;3966 defm VPUNPCKLQDQY : sse2_unpack<0x6C, "vpunpcklqdq", v4i64, X86Unpckl, VR256,3967 i256mem, SchedWriteShuffle.YMM, load, 0>,3968 VEX, VVVV, VEX_L, WIG;3969 defm VPUNPCKHDQY : sse2_unpack<0x6A, "vpunpckhdq", v8i32, X86Unpckh, VR256,3970 i256mem, SchedWriteShuffle.YMM, load, 0>,3971 VEX, VVVV, VEX_L, WIG;3972 defm VPUNPCKHQDQY : sse2_unpack<0x6D, "vpunpckhqdq", v4i64, X86Unpckh, VR256,3973 i256mem, SchedWriteShuffle.YMM, load, 0>,3974 VEX, VVVV, VEX_L, WIG;3975}3976 3977let Constraints = "$src1 = $dst" in {3978 defm PUNPCKLBW : sse2_unpack<0x60, "punpcklbw", v16i8, X86Unpckl, VR128,3979 i128mem, SchedWriteShuffle.XMM, memop>;3980 defm PUNPCKLWD : sse2_unpack<0x61, "punpcklwd", v8i16, X86Unpckl, VR128,3981 i128mem, SchedWriteShuffle.XMM, memop>;3982 defm PUNPCKLDQ : sse2_unpack<0x62, "punpckldq", v4i32, X86Unpckl, VR128,3983 i128mem, SchedWriteShuffle.XMM, memop>;3984 defm PUNPCKLQDQ : sse2_unpack<0x6C, "punpcklqdq", v2i64, X86Unpckl, VR128,3985 i128mem, SchedWriteShuffle.XMM, memop>;3986 3987 defm PUNPCKHBW : sse2_unpack<0x68, "punpckhbw", v16i8, X86Unpckh, VR128,3988 i128mem, SchedWriteShuffle.XMM, memop>;3989 defm PUNPCKHWD : sse2_unpack<0x69, "punpckhwd", v8i16, X86Unpckh, VR128,3990 i128mem, SchedWriteShuffle.XMM, memop>;3991 defm PUNPCKHDQ : sse2_unpack<0x6A, "punpckhdq", v4i32, X86Unpckh, VR128,3992 i128mem, SchedWriteShuffle.XMM, memop>;3993 defm PUNPCKHQDQ : sse2_unpack<0x6D, "punpckhqdq", v2i64, X86Unpckh, VR128,3994 i128mem, SchedWriteShuffle.XMM, memop>;3995}3996} // ExeDomain = SSEPackedInt3997 3998//===---------------------------------------------------------------------===//3999// SSE2 - Packed Integer Extract and Insert4000//===---------------------------------------------------------------------===//4001 4002let ExeDomain = SSEPackedInt in {4003multiclass sse2_pinsrw<bit Is2Addr = 1> {4004 def rri : Ii8<0xC4, MRMSrcReg,4005 (outs VR128:$dst), (ins VR128:$src1,4006 GR32orGR64:$src2, u8imm:$src3),4007 !if(Is2Addr,4008 "pinsrw\t{$src3, $src2, $dst|$dst, $src2, $src3}",4009 "vpinsrw\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),4010 [(set VR128:$dst,4011 (X86pinsrw VR128:$src1, GR32orGR64:$src2, timm:$src3))]>,4012 Sched<[WriteVecInsert, ReadDefault, ReadInt2Fpu]>;4013 def rmi : Ii8<0xC4, MRMSrcMem,4014 (outs VR128:$dst), (ins VR128:$src1,4015 i16mem:$src2, u8imm:$src3),4016 !if(Is2Addr,4017 "pinsrw\t{$src3, $src2, $dst|$dst, $src2, $src3}",4018 "vpinsrw\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),4019 [(set VR128:$dst,4020 (X86pinsrw VR128:$src1, (extloadi16 addr:$src2),4021 timm:$src3))]>,4022 Sched<[WriteVecInsert.Folded, WriteVecInsert.ReadAfterFold]>;4023}4024 4025// Extract4026let Predicates = [HasAVX, NoBWI] in4027def VPEXTRWrri : Ii8<0xC5, MRMSrcReg,4028 (outs GR32orGR64:$dst), (ins VR128:$src1, u8imm:$src2),4029 "vpextrw\t{$src2, $src1, $dst|$dst, $src1, $src2}",4030 [(set GR32orGR64:$dst, (X86pextrw (v8i16 VR128:$src1),4031 timm:$src2))]>,4032 TB, PD, VEX, WIG, Sched<[WriteVecExtract]>;4033def PEXTRWrri : PDIi8<0xC5, MRMSrcReg,4034 (outs GR32orGR64:$dst), (ins VR128:$src1, u8imm:$src2),4035 "pextrw\t{$src2, $src1, $dst|$dst, $src1, $src2}",4036 [(set GR32orGR64:$dst, (X86pextrw (v8i16 VR128:$src1),4037 timm:$src2))]>,4038 Sched<[WriteVecExtract]>;4039 4040// Insert4041let Predicates = [HasAVX, NoBWI] in4042defm VPINSRW : sse2_pinsrw<0>, TB, PD, VEX, VVVV, WIG;4043 4044let Predicates = [UseSSE2], Constraints = "$src1 = $dst" in4045defm PINSRW : sse2_pinsrw, TB, PD;4046 4047} // ExeDomain = SSEPackedInt4048 4049// Always select FP16 instructions if available.4050let Predicates = [UseSSE2], AddedComplexity = -10 in {4051 def : Pat<(f16 (load addr:$src)), (COPY_TO_REGCLASS (PINSRWrmi (v8i16 (IMPLICIT_DEF)), addr:$src, 0), FR16)>;4052 def : Pat<(store f16:$src, addr:$dst), (MOV16mr addr:$dst, (EXTRACT_SUBREG (PEXTRWrri (v8i16 (COPY_TO_REGCLASS FR16:$src, VR128)), 0), sub_16bit))>;4053 def : Pat<(i16 (bitconvert f16:$src)), (EXTRACT_SUBREG (PEXTRWrri (v8i16 (COPY_TO_REGCLASS FR16:$src, VR128)), 0), sub_16bit)>;4054 def : Pat<(f16 (bitconvert i16:$src)), (COPY_TO_REGCLASS (PINSRWrri (v8i16 (IMPLICIT_DEF)), (INSERT_SUBREG (IMPLICIT_DEF), GR16:$src, sub_16bit), 0), FR16)>;4055}4056 4057let Predicates = [HasAVX, NoBWI] in {4058 def : Pat<(f16 (load addr:$src)), (COPY_TO_REGCLASS (VPINSRWrmi (v8i16 (IMPLICIT_DEF)), addr:$src, 0), FR16)>;4059 def : Pat<(i16 (bitconvert f16:$src)), (EXTRACT_SUBREG (VPEXTRWrri (v8i16 (COPY_TO_REGCLASS FR16:$src, VR128)), 0), sub_16bit)>;4060 def : Pat<(f16 (bitconvert i16:$src)), (COPY_TO_REGCLASS (VPINSRWrri (v8i16 (IMPLICIT_DEF)), (INSERT_SUBREG (IMPLICIT_DEF), GR16:$src, sub_16bit), 0), FR16)>;4061}4062 4063//===---------------------------------------------------------------------===//4064// SSE2 - Packed Mask Creation4065//===---------------------------------------------------------------------===//4066 4067let ExeDomain = SSEPackedInt in {4068 4069def VPMOVMSKBrr : VPDI<0xD7, MRMSrcReg, (outs GR32orGR64:$dst),4070 (ins VR128:$src),4071 "pmovmskb\t{$src, $dst|$dst, $src}",4072 [(set GR32orGR64:$dst, (X86movmsk (v16i8 VR128:$src)))]>,4073 Sched<[WriteVecMOVMSK]>, VEX, WIG;4074 4075let Predicates = [HasAVX2] in {4076def VPMOVMSKBYrr : VPDI<0xD7, MRMSrcReg, (outs GR32orGR64:$dst),4077 (ins VR256:$src),4078 "pmovmskb\t{$src, $dst|$dst, $src}",4079 [(set GR32orGR64:$dst, (X86movmsk (v32i8 VR256:$src)))]>,4080 Sched<[WriteVecMOVMSKY]>, VEX, VEX_L, WIG;4081}4082 4083def PMOVMSKBrr : PDI<0xD7, MRMSrcReg, (outs GR32orGR64:$dst), (ins VR128:$src),4084 "pmovmskb\t{$src, $dst|$dst, $src}",4085 [(set GR32orGR64:$dst, (X86movmsk (v16i8 VR128:$src)))]>,4086 Sched<[WriteVecMOVMSK]>;4087 4088} // ExeDomain = SSEPackedInt4089 4090//===---------------------------------------------------------------------===//4091// SSE2 - Conditional Store4092//===---------------------------------------------------------------------===//4093 4094let ExeDomain = SSEPackedInt, SchedRW = [SchedWriteVecMoveLS.XMM.MR] in {4095// As VEX does not have separate instruction contexts for address size4096// overrides, VMASKMOVDQU and VMASKMOVDQU64 would have a decode conflict.4097// Prefer VMASKMODDQU64.4098let Uses = [RDI], Predicates = [HasAVX,In64BitMode] in4099def VMASKMOVDQU64 : VPDI<0xF7, MRMSrcReg, (outs),4100 (ins VR128:$src, VR128:$mask),4101 "maskmovdqu\t{$mask, $src|$src, $mask}",4102 [(int_x86_sse2_maskmov_dqu VR128:$src, VR128:$mask, RDI)]>,4103 VEX, WIG;4104let Uses = [EDI], Predicates = [HasAVX], isAsmParserOnly = 1 in4105def VMASKMOVDQU : VPDI<0xF7, MRMSrcReg, (outs),4106 (ins VR128:$src, VR128:$mask),4107 "maskmovdqu\t{$mask, $src|$src, $mask}",4108 [(int_x86_sse2_maskmov_dqu VR128:$src, VR128:$mask, EDI)]>,4109 VEX, WIG;4110 4111let Uses = [RDI], Predicates = [UseSSE2,In64BitMode] in4112def MASKMOVDQU64 : PDI<0xF7, MRMSrcReg, (outs), (ins VR128:$src, VR128:$mask),4113 "maskmovdqu\t{$mask, $src|$src, $mask}",4114 [(int_x86_sse2_maskmov_dqu VR128:$src, VR128:$mask, RDI)]>;4115let Uses = [EDI], Predicates = [UseSSE2] in4116def MASKMOVDQU : PDI<0xF7, MRMSrcReg, (outs), (ins VR128:$src, VR128:$mask),4117 "maskmovdqu\t{$mask, $src|$src, $mask}",4118 [(int_x86_sse2_maskmov_dqu VR128:$src, VR128:$mask, EDI)]>;4119 4120} // ExeDomain = SSEPackedInt4121 4122//===---------------------------------------------------------------------===//4123// SSE2 - Move Doubleword/Quadword4124//===---------------------------------------------------------------------===//4125 4126//===---------------------------------------------------------------------===//4127// Move Int Doubleword to Packed Double Int4128//4129let ExeDomain = SSEPackedInt in {4130def VMOVDI2PDIrr : VS2I<0x6E, MRMSrcReg, (outs VR128:$dst), (ins GR32:$src),4131 "movd\t{$src, $dst|$dst, $src}",4132 [(set VR128:$dst,4133 (v4i32 (scalar_to_vector GR32:$src)))]>,4134 VEX, Sched<[WriteVecMoveFromGpr]>;4135def VMOVDI2PDIrm : VS2I<0x6E, MRMSrcMem, (outs VR128:$dst), (ins i32mem:$src),4136 "movd\t{$src, $dst|$dst, $src}",4137 [(set VR128:$dst,4138 (v4i32 (scalar_to_vector (loadi32 addr:$src))))]>,4139 VEX, Sched<[WriteVecLoad]>;4140def VMOV64toPQIrr : VRS2I<0x6E, MRMSrcReg, (outs VR128:$dst), (ins GR64:$src),4141 "movq\t{$src, $dst|$dst, $src}",4142 [(set VR128:$dst,4143 (v2i64 (scalar_to_vector GR64:$src)))]>,4144 VEX, Sched<[WriteVecMoveFromGpr]>;4145let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, mayLoad = 1 in4146def VMOV64toPQIrm : VRS2I<0x6E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src),4147 "movq\t{$src, $dst|$dst, $src}", []>,4148 VEX, Sched<[WriteVecLoad]>;4149let isCodeGenOnly = 1 in4150def VMOV64toSDrr : VRS2I<0x6E, MRMSrcReg, (outs FR64:$dst), (ins GR64:$src),4151 "movq\t{$src, $dst|$dst, $src}",4152 [(set FR64:$dst, (bitconvert GR64:$src))]>,4153 VEX, Sched<[WriteVecMoveFromGpr]>;4154 4155def MOVDI2PDIrr : S2I<0x6E, MRMSrcReg, (outs VR128:$dst), (ins GR32:$src),4156 "movd\t{$src, $dst|$dst, $src}",4157 [(set VR128:$dst,4158 (v4i32 (scalar_to_vector GR32:$src)))]>,4159 Sched<[WriteVecMoveFromGpr]>;4160def MOVDI2PDIrm : S2I<0x6E, MRMSrcMem, (outs VR128:$dst), (ins i32mem:$src),4161 "movd\t{$src, $dst|$dst, $src}",4162 [(set VR128:$dst,4163 (v4i32 (scalar_to_vector (loadi32 addr:$src))))]>,4164 Sched<[WriteVecLoad]>;4165def MOV64toPQIrr : RS2I<0x6E, MRMSrcReg, (outs VR128:$dst), (ins GR64:$src),4166 "movq\t{$src, $dst|$dst, $src}",4167 [(set VR128:$dst,4168 (v2i64 (scalar_to_vector GR64:$src)))]>,4169 Sched<[WriteVecMoveFromGpr]>;4170let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, mayLoad = 1 in4171def MOV64toPQIrm : RS2I<0x6E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src),4172 "movq\t{$src, $dst|$dst, $src}", []>,4173 Sched<[WriteVecLoad]>;4174let isCodeGenOnly = 1 in4175def MOV64toSDrr : RS2I<0x6E, MRMSrcReg, (outs FR64:$dst), (ins GR64:$src),4176 "movq\t{$src, $dst|$dst, $src}",4177 [(set FR64:$dst, (bitconvert GR64:$src))]>,4178 Sched<[WriteVecMoveFromGpr]>;4179} // ExeDomain = SSEPackedInt4180 4181//===---------------------------------------------------------------------===//4182// Move Int Doubleword to Single Scalar4183//4184let ExeDomain = SSEPackedInt, isCodeGenOnly = 1 in {4185 def VMOVDI2SSrr : VS2I<0x6E, MRMSrcReg, (outs FR32:$dst), (ins GR32:$src),4186 "movd\t{$src, $dst|$dst, $src}",4187 [(set FR32:$dst, (bitconvert GR32:$src))]>,4188 VEX, Sched<[WriteVecMoveFromGpr]>;4189 4190 def MOVDI2SSrr : S2I<0x6E, MRMSrcReg, (outs FR32:$dst), (ins GR32:$src),4191 "movd\t{$src, $dst|$dst, $src}",4192 [(set FR32:$dst, (bitconvert GR32:$src))]>,4193 Sched<[WriteVecMoveFromGpr]>;4194 4195} // ExeDomain = SSEPackedInt, isCodeGenOnly = 14196 4197//===---------------------------------------------------------------------===//4198// Move Packed Doubleword Int to Packed Double Int4199//4200let ExeDomain = SSEPackedInt in {4201def VMOVPDI2DIrr : VS2I<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128:$src),4202 "movd\t{$src, $dst|$dst, $src}",4203 [(set GR32:$dst, (extractelt (v4i32 VR128:$src),4204 (iPTR 0)))]>, VEX,4205 Sched<[WriteVecMoveToGpr]>;4206def VMOVPDI2DImr : VS2I<0x7E, MRMDestMem, (outs),4207 (ins i32mem:$dst, VR128:$src),4208 "movd\t{$src, $dst|$dst, $src}",4209 [(store (i32 (extractelt (v4i32 VR128:$src),4210 (iPTR 0))), addr:$dst)]>,4211 VEX, Sched<[WriteVecStore]>;4212def MOVPDI2DIrr : S2I<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128:$src),4213 "movd\t{$src, $dst|$dst, $src}",4214 [(set GR32:$dst, (extractelt (v4i32 VR128:$src),4215 (iPTR 0)))]>,4216 Sched<[WriteVecMoveToGpr]>;4217def MOVPDI2DImr : S2I<0x7E, MRMDestMem, (outs), (ins i32mem:$dst, VR128:$src),4218 "movd\t{$src, $dst|$dst, $src}",4219 [(store (i32 (extractelt (v4i32 VR128:$src),4220 (iPTR 0))), addr:$dst)]>,4221 Sched<[WriteVecStore]>;4222} // ExeDomain = SSEPackedInt4223 4224//===---------------------------------------------------------------------===//4225// Move Packed Doubleword Int first element to Doubleword Int4226//4227let ExeDomain = SSEPackedInt in {4228let SchedRW = [WriteVecMoveToGpr] in {4229def VMOVPQIto64rr : VRS2I<0x7E, MRMDestReg, (outs GR64:$dst), (ins VR128:$src),4230 "movq\t{$src, $dst|$dst, $src}",4231 [(set GR64:$dst, (extractelt (v2i64 VR128:$src),4232 (iPTR 0)))]>,4233 VEX;4234 4235def MOVPQIto64rr : RS2I<0x7E, MRMDestReg, (outs GR64:$dst), (ins VR128:$src),4236 "movq\t{$src, $dst|$dst, $src}",4237 [(set GR64:$dst, (extractelt (v2i64 VR128:$src),4238 (iPTR 0)))]>;4239} //SchedRW4240 4241let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, mayStore = 1 in4242def VMOVPQIto64mr : VRS2I<0x7E, MRMDestMem, (outs),4243 (ins i64mem:$dst, VR128:$src),4244 "movq\t{$src, $dst|$dst, $src}", []>,4245 VEX, Sched<[WriteVecStore]>;4246let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, mayStore = 1 in4247def MOVPQIto64mr : RS2I<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, VR128:$src),4248 "movq\t{$src, $dst|$dst, $src}", []>,4249 Sched<[WriteVecStore]>;4250} // ExeDomain = SSEPackedInt4251 4252//===---------------------------------------------------------------------===//4253// Bitcast FR64 <-> GR644254//4255let ExeDomain = SSEPackedInt, isCodeGenOnly = 1 in {4256 def VMOVSDto64rr : VRS2I<0x7E, MRMDestReg, (outs GR64:$dst), (ins FR64:$src),4257 "movq\t{$src, $dst|$dst, $src}",4258 [(set GR64:$dst, (bitconvert FR64:$src))]>,4259 VEX, Sched<[WriteVecMoveToGpr]>;4260 4261 def MOVSDto64rr : RS2I<0x7E, MRMDestReg, (outs GR64:$dst), (ins FR64:$src),4262 "movq\t{$src, $dst|$dst, $src}",4263 [(set GR64:$dst, (bitconvert FR64:$src))]>,4264 Sched<[WriteVecMoveToGpr]>;4265} // ExeDomain = SSEPackedInt, isCodeGenOnly = 14266 4267//===---------------------------------------------------------------------===//4268// Move Scalar Single to Double Int4269//4270let ExeDomain = SSEPackedInt, isCodeGenOnly = 1 in {4271 def VMOVSS2DIrr : VS2I<0x7E, MRMDestReg, (outs GR32:$dst), (ins FR32:$src),4272 "movd\t{$src, $dst|$dst, $src}",4273 [(set GR32:$dst, (bitconvert FR32:$src))]>,4274 VEX, Sched<[WriteVecMoveToGpr]>;4275 def MOVSS2DIrr : S2I<0x7E, MRMDestReg, (outs GR32:$dst), (ins FR32:$src),4276 "movd\t{$src, $dst|$dst, $src}",4277 [(set GR32:$dst, (bitconvert FR32:$src))]>,4278 Sched<[WriteVecMoveToGpr]>;4279} // ExeDomain = SSEPackedInt, isCodeGenOnly = 14280 4281let Predicates = [UseAVX] in {4282 def : Pat<(v4i32 (scalar_to_vector (i32 (anyext GR8:$src)))),4283 (VMOVDI2PDIrr (i32 (INSERT_SUBREG (i32 (IMPLICIT_DEF)),4284 GR8:$src, sub_8bit)))>;4285 def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector GR32:$src)))),4286 (VMOVDI2PDIrr GR32:$src)>;4287 4288 def : Pat<(v2i64 (X86vzmovl (v2i64 (scalar_to_vector GR64:$src)))),4289 (VMOV64toPQIrr GR64:$src)>;4290 4291 // AVX 128-bit movd/movq instructions write zeros in the high 128-bit part.4292 // These instructions also write zeros in the high part of a 256-bit register.4293 def : Pat<(v4i32 (X86vzload32 addr:$src)),4294 (VMOVDI2PDIrm addr:$src)>;4295 def : Pat<(v8i32 (X86vzload32 addr:$src)),4296 (SUBREG_TO_REG (i64 0), (v4i32 (VMOVDI2PDIrm addr:$src)), sub_xmm)>;4297}4298 4299let Predicates = [UseSSE2] in {4300 def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector GR32:$src)))),4301 (MOVDI2PDIrr GR32:$src)>;4302 4303 def : Pat<(v2i64 (X86vzmovl (v2i64 (scalar_to_vector GR64:$src)))),4304 (MOV64toPQIrr GR64:$src)>;4305 def : Pat<(v4i32 (X86vzload32 addr:$src)),4306 (MOVDI2PDIrm addr:$src)>;4307}4308 4309// Before the MC layer of LLVM existed, clang emitted "movd" assembly instead of4310// "movq" due to MacOS parsing limitation. In order to parse old assembly, we add4311// these aliases.4312def : InstAlias<"movd\t{$src, $dst|$dst, $src}",4313 (MOV64toPQIrr VR128:$dst, GR64:$src), 0>;4314def : InstAlias<"movd\t{$src, $dst|$dst, $src}",4315 (MOVPQIto64rr GR64:$dst, VR128:$src), 0>;4316// Allow "vmovd" but print "vmovq" since we don't need compatibility for AVX.4317def : InstAlias<"vmovd\t{$src, $dst|$dst, $src}",4318 (VMOV64toPQIrr VR128:$dst, GR64:$src), 0>;4319def : InstAlias<"vmovd\t{$src, $dst|$dst, $src}",4320 (VMOVPQIto64rr GR64:$dst, VR128:$src), 0>;4321 4322//===---------------------------------------------------------------------===//4323// SSE2 - Move Quadword4324//===---------------------------------------------------------------------===//4325 4326//===---------------------------------------------------------------------===//4327// Move Quadword Int to Packed Quadword Int4328//4329 4330let ExeDomain = SSEPackedInt, SchedRW = [WriteVecLoad] in {4331def VMOVQI2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src),4332 "vmovq\t{$src, $dst|$dst, $src}",4333 [(set VR128:$dst,4334 (v2i64 (scalar_to_vector (loadi64 addr:$src))))]>, TB, XS,4335 VEX, Requires<[UseAVX]>, WIG;4336def MOVQI2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src),4337 "movq\t{$src, $dst|$dst, $src}",4338 [(set VR128:$dst,4339 (v2i64 (scalar_to_vector (loadi64 addr:$src))))]>,4340 TB, XS, Requires<[UseSSE2]>; // SSE2 instruction with XS Prefix4341} // ExeDomain, SchedRW4342 4343//===---------------------------------------------------------------------===//4344// Move Packed Quadword Int to Quadword Int4345//4346let ExeDomain = SSEPackedInt, SchedRW = [WriteVecStore] in {4347def VMOVPQI2QImr : VS2I<0xD6, MRMDestMem, (outs), (ins i64mem:$dst, VR128:$src),4348 "movq\t{$src, $dst|$dst, $src}",4349 [(store (i64 (extractelt (v2i64 VR128:$src),4350 (iPTR 0))), addr:$dst)]>,4351 VEX, WIG;4352def MOVPQI2QImr : S2I<0xD6, MRMDestMem, (outs), (ins i64mem:$dst, VR128:$src),4353 "movq\t{$src, $dst|$dst, $src}",4354 [(store (i64 (extractelt (v2i64 VR128:$src),4355 (iPTR 0))), addr:$dst)]>;4356} // ExeDomain, SchedRW4357 4358// For disassembler only4359let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0,4360 SchedRW = [SchedWriteVecLogic.XMM] in {4361def VMOVPQI2QIrr : VS2I<0xD6, MRMDestReg, (outs VR128:$dst), (ins VR128:$src),4362 "movq\t{$src, $dst|$dst, $src}", []>, VEX, WIG;4363def MOVPQI2QIrr : S2I<0xD6, MRMDestReg, (outs VR128:$dst), (ins VR128:$src),4364 "movq\t{$src, $dst|$dst, $src}", []>;4365}4366 4367def : InstAlias<"vmovq.s\t{$src, $dst|$dst, $src}",4368 (VMOVPQI2QIrr VR128:$dst, VR128:$src), 0>;4369def : InstAlias<"movq.s\t{$src, $dst|$dst, $src}",4370 (MOVPQI2QIrr VR128:$dst, VR128:$src), 0>;4371 4372let Predicates = [UseAVX] in {4373 def : Pat<(v2i64 (X86vzload64 addr:$src)),4374 (VMOVQI2PQIrm addr:$src)>;4375 def : Pat<(v4i64 (X86vzload64 addr:$src)),4376 (SUBREG_TO_REG (i64 0), (v2i64 (VMOVQI2PQIrm addr:$src)), sub_xmm)>;4377 4378 def : Pat<(X86vextractstore64 (v2i64 VR128:$src), addr:$dst),4379 (VMOVPQI2QImr addr:$dst, VR128:$src)>;4380}4381 4382let Predicates = [UseSSE2] in {4383 def : Pat<(v2i64 (X86vzload64 addr:$src)), (MOVQI2PQIrm addr:$src)>;4384 4385 def : Pat<(X86vextractstore64 (v2i64 VR128:$src), addr:$dst),4386 (MOVPQI2QImr addr:$dst, VR128:$src)>;4387}4388 4389//===---------------------------------------------------------------------===//4390// Moving from XMM to XMM and clear upper 64 bits. Note, there is a bug in4391// IA32 document. movq xmm1, xmm2 does clear the high bits.4392//4393let ExeDomain = SSEPackedInt, SchedRW = [SchedWriteVecLogic.XMM] in {4394def VMOVZPQILo2PQIrr : I<0x7E, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),4395 "vmovq\t{$src, $dst|$dst, $src}",4396 [(set VR128:$dst, (v2i64 (X86vzmovl (v2i64 VR128:$src))))]>,4397 TB, XS, VEX, Requires<[UseAVX]>, WIG;4398def MOVZPQILo2PQIrr : I<0x7E, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),4399 "movq\t{$src, $dst|$dst, $src}",4400 [(set VR128:$dst, (v2i64 (X86vzmovl (v2i64 VR128:$src))))]>,4401 TB, XS, Requires<[UseSSE2]>;4402} // ExeDomain, SchedRW4403 4404let Predicates = [UseAVX] in {4405 def : Pat<(v2f64 (X86vzmovl (v2f64 VR128:$src))),4406 (VMOVZPQILo2PQIrr VR128:$src)>;4407}4408let Predicates = [UseSSE2] in {4409 def : Pat<(v2f64 (X86vzmovl (v2f64 VR128:$src))),4410 (MOVZPQILo2PQIrr VR128:$src)>;4411}4412 4413let Predicates = [UseAVX] in {4414 def : Pat<(v4f64 (X86vzmovl (v4f64 VR256:$src))),4415 (SUBREG_TO_REG (i32 0),4416 (v2f64 (VMOVZPQILo2PQIrr4417 (v2f64 (EXTRACT_SUBREG (v4f64 VR256:$src), sub_xmm)))),4418 sub_xmm)>;4419 def : Pat<(v4i64 (X86vzmovl (v4i64 VR256:$src))),4420 (SUBREG_TO_REG (i32 0),4421 (v2i64 (VMOVZPQILo2PQIrr4422 (v2i64 (EXTRACT_SUBREG (v4i64 VR256:$src), sub_xmm)))),4423 sub_xmm)>;4424}4425 4426//===---------------------------------------------------------------------===//4427// SSE3 - Replicate Single FP - MOVSHDUP and MOVSLDUP4428//===---------------------------------------------------------------------===//4429 4430multiclass sse3_replicate_sfp<bits<8> op, SDNode OpNode, string OpcodeStr,4431 ValueType vt, RegisterClass RC, PatFrag mem_frag,4432 X86MemOperand x86memop, X86FoldableSchedWrite sched> {4433def rr : S3SI<op, MRMSrcReg, (outs RC:$dst), (ins RC:$src),4434 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),4435 [(set RC:$dst, (vt (OpNode RC:$src)))]>,4436 Sched<[sched]>;4437def rm : S3SI<op, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src),4438 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),4439 [(set RC:$dst, (OpNode (mem_frag addr:$src)))]>,4440 Sched<[sched.Folded]>;4441}4442 4443let Predicates = [HasAVX, NoVLX] in {4444 defm VMOVSHDUP : sse3_replicate_sfp<0x16, X86Movshdup, "vmovshdup",4445 v4f32, VR128, loadv4f32, f128mem,4446 SchedWriteFShuffle.XMM>, VEX, WIG;4447 defm VMOVSLDUP : sse3_replicate_sfp<0x12, X86Movsldup, "vmovsldup",4448 v4f32, VR128, loadv4f32, f128mem,4449 SchedWriteFShuffle.XMM>, VEX, WIG;4450 defm VMOVSHDUPY : sse3_replicate_sfp<0x16, X86Movshdup, "vmovshdup",4451 v8f32, VR256, loadv8f32, f256mem,4452 SchedWriteFShuffle.YMM>, VEX, VEX_L, WIG;4453 defm VMOVSLDUPY : sse3_replicate_sfp<0x12, X86Movsldup, "vmovsldup",4454 v8f32, VR256, loadv8f32, f256mem,4455 SchedWriteFShuffle.YMM>, VEX, VEX_L, WIG;4456}4457defm MOVSHDUP : sse3_replicate_sfp<0x16, X86Movshdup, "movshdup", v4f32, VR128,4458 memopv4f32, f128mem, SchedWriteFShuffle.XMM>;4459defm MOVSLDUP : sse3_replicate_sfp<0x12, X86Movsldup, "movsldup", v4f32, VR128,4460 memopv4f32, f128mem, SchedWriteFShuffle.XMM>;4461 4462let Predicates = [HasAVX, NoVLX] in {4463 def : Pat<(v4i32 (X86Movshdup VR128:$src)),4464 (VMOVSHDUPrr VR128:$src)>;4465 def : Pat<(v4i32 (X86Movshdup (load addr:$src))),4466 (VMOVSHDUPrm addr:$src)>;4467 def : Pat<(v4i32 (X86Movsldup VR128:$src)),4468 (VMOVSLDUPrr VR128:$src)>;4469 def : Pat<(v4i32 (X86Movsldup (load addr:$src))),4470 (VMOVSLDUPrm addr:$src)>;4471 def : Pat<(v8i32 (X86Movshdup VR256:$src)),4472 (VMOVSHDUPYrr VR256:$src)>;4473 def : Pat<(v8i32 (X86Movshdup (load addr:$src))),4474 (VMOVSHDUPYrm addr:$src)>;4475 def : Pat<(v8i32 (X86Movsldup VR256:$src)),4476 (VMOVSLDUPYrr VR256:$src)>;4477 def : Pat<(v8i32 (X86Movsldup (load addr:$src))),4478 (VMOVSLDUPYrm addr:$src)>;4479}4480 4481let Predicates = [UseSSE3] in {4482 def : Pat<(v4i32 (X86Movshdup VR128:$src)),4483 (MOVSHDUPrr VR128:$src)>;4484 def : Pat<(v4i32 (X86Movshdup (memop addr:$src))),4485 (MOVSHDUPrm addr:$src)>;4486 def : Pat<(v4i32 (X86Movsldup VR128:$src)),4487 (MOVSLDUPrr VR128:$src)>;4488 def : Pat<(v4i32 (X86Movsldup (memop addr:$src))),4489 (MOVSLDUPrm addr:$src)>;4490}4491 4492//===---------------------------------------------------------------------===//4493// SSE3 - Replicate Double FP - MOVDDUP4494//===---------------------------------------------------------------------===//4495 4496multiclass sse3_replicate_dfp<string OpcodeStr, X86SchedWriteWidths sched> {4497def rr : S3DI<0x12, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),4498 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),4499 [(set VR128:$dst, (v2f64 (X86Movddup VR128:$src)))]>,4500 Sched<[sched.XMM]>;4501def rm : S3DI<0x12, MRMSrcMem, (outs VR128:$dst), (ins f64mem:$src),4502 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),4503 [(set VR128:$dst,4504 (v2f64 (X86Movddup4505 (scalar_to_vector (loadf64 addr:$src)))))]>,4506 Sched<[sched.XMM.Folded]>;4507}4508 4509// FIXME: Merge with above classes when there are patterns for the ymm version4510multiclass sse3_replicate_dfp_y<string OpcodeStr, X86SchedWriteWidths sched> {4511def rr : S3DI<0x12, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),4512 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),4513 [(set VR256:$dst, (v4f64 (X86Movddup VR256:$src)))]>,4514 Sched<[sched.YMM]>;4515def rm : S3DI<0x12, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src),4516 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),4517 [(set VR256:$dst,4518 (v4f64 (X86Movddup (loadv4f64 addr:$src))))]>,4519 Sched<[sched.YMM.Folded]>;4520}4521 4522let Predicates = [HasAVX, NoVLX] in {4523 defm VMOVDDUP : sse3_replicate_dfp<"vmovddup", SchedWriteFShuffle>,4524 VEX, WIG;4525 defm VMOVDDUPY : sse3_replicate_dfp_y<"vmovddup", SchedWriteFShuffle>,4526 VEX, VEX_L, WIG;4527}4528 4529defm MOVDDUP : sse3_replicate_dfp<"movddup", SchedWriteFShuffle>;4530 4531 4532let Predicates = [HasAVX, NoVLX] in {4533 def : Pat<(X86Movddup (v2f64 (X86vzload64 addr:$src))),4534 (VMOVDDUPrm addr:$src)>, Requires<[HasAVX]>;4535}4536 4537let Predicates = [UseSSE3] in {4538 def : Pat<(X86Movddup (v2f64 (X86vzload64 addr:$src))),4539 (MOVDDUPrm addr:$src)>;4540}4541 4542//===---------------------------------------------------------------------===//4543// SSE3 - Move Unaligned Integer4544//===---------------------------------------------------------------------===//4545 4546let Predicates = [HasAVX] in {4547 def VLDDQUrm : S3DI<0xF0, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src),4548 "vlddqu\t{$src, $dst|$dst, $src}",4549 [(set VR128:$dst, (int_x86_sse3_ldu_dq addr:$src))]>,4550 Sched<[SchedWriteVecMoveLS.XMM.RM]>, VEX, WIG;4551 def VLDDQUYrm : S3DI<0xF0, MRMSrcMem, (outs VR256:$dst), (ins i256mem:$src),4552 "vlddqu\t{$src, $dst|$dst, $src}",4553 [(set VR256:$dst, (int_x86_avx_ldu_dq_256 addr:$src))]>,4554 Sched<[SchedWriteVecMoveLS.YMM.RM]>, VEX, VEX_L, WIG;4555} // Predicates4556 4557def LDDQUrm : S3DI<0xF0, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src),4558 "lddqu\t{$src, $dst|$dst, $src}",4559 [(set VR128:$dst, (int_x86_sse3_ldu_dq addr:$src))]>,4560 Sched<[SchedWriteVecMoveLS.XMM.RM]>;4561 4562//===---------------------------------------------------------------------===//4563// SSE3 - Arithmetic4564//===---------------------------------------------------------------------===//4565 4566multiclass sse3_addsub<string OpcodeStr, ValueType vt, RegisterClass RC,4567 X86MemOperand x86memop, X86FoldableSchedWrite sched,4568 PatFrag ld_frag, bit Is2Addr = 1> {4569let Uses = [MXCSR], mayRaiseFPException = 1 in {4570 def rr : I<0xD0, MRMSrcReg,4571 (outs RC:$dst), (ins RC:$src1, RC:$src2),4572 !if(Is2Addr,4573 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),4574 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),4575 [(set RC:$dst, (vt (X86Addsub RC:$src1, RC:$src2)))]>,4576 Sched<[sched]>;4577 def rm : I<0xD0, MRMSrcMem,4578 (outs RC:$dst), (ins RC:$src1, x86memop:$src2),4579 !if(Is2Addr,4580 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),4581 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),4582 [(set RC:$dst, (vt (X86Addsub RC:$src1, (ld_frag addr:$src2))))]>,4583 Sched<[sched.Folded, sched.ReadAfterFold]>;4584}4585}4586 4587let Predicates = [HasAVX] in {4588 let ExeDomain = SSEPackedSingle in {4589 defm VADDSUBPS : sse3_addsub<"vaddsubps", v4f32, VR128, f128mem,4590 SchedWriteFAddSizes.PS.XMM, loadv4f32, 0>,4591 TB, XD, VEX, VVVV, WIG;4592 defm VADDSUBPSY : sse3_addsub<"vaddsubps", v8f32, VR256, f256mem,4593 SchedWriteFAddSizes.PS.YMM, loadv8f32, 0>,4594 TB, XD, VEX, VVVV, VEX_L, WIG;4595 }4596 let ExeDomain = SSEPackedDouble in {4597 defm VADDSUBPD : sse3_addsub<"vaddsubpd", v2f64, VR128, f128mem,4598 SchedWriteFAddSizes.PD.XMM, loadv2f64, 0>,4599 TB, PD, VEX, VVVV, WIG;4600 defm VADDSUBPDY : sse3_addsub<"vaddsubpd", v4f64, VR256, f256mem,4601 SchedWriteFAddSizes.PD.YMM, loadv4f64, 0>,4602 TB, PD, VEX, VVVV, VEX_L, WIG;4603 }4604}4605let Constraints = "$src1 = $dst", Predicates = [UseSSE3] in {4606 let ExeDomain = SSEPackedSingle in4607 defm ADDSUBPS : sse3_addsub<"addsubps", v4f32, VR128, f128mem,4608 SchedWriteFAddSizes.PS.XMM, memopv4f32>, TB, XD;4609 let ExeDomain = SSEPackedDouble in4610 defm ADDSUBPD : sse3_addsub<"addsubpd", v2f64, VR128, f128mem,4611 SchedWriteFAddSizes.PD.XMM, memopv2f64>, TB, PD;4612}4613 4614//===---------------------------------------------------------------------===//4615// SSE3 Instructions4616//===---------------------------------------------------------------------===//4617 4618// Horizontal ops4619multiclass S3D_Int<bits<8> o, string OpcodeStr, ValueType vt, RegisterClass RC,4620 X86MemOperand x86memop, SDNode OpNode,4621 X86FoldableSchedWrite sched, PatFrag ld_frag,4622 bit Is2Addr = 1> {4623let Uses = [MXCSR], mayRaiseFPException = 1 in {4624 def rr : S3DI<o, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2),4625 !if(Is2Addr,4626 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),4627 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),4628 [(set RC:$dst, (vt (OpNode RC:$src1, RC:$src2)))]>,4629 Sched<[sched]>;4630 4631 def rm : S3DI<o, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2),4632 !if(Is2Addr,4633 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),4634 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),4635 [(set RC:$dst, (vt (OpNode RC:$src1, (ld_frag addr:$src2))))]>,4636 Sched<[sched.Folded, sched.ReadAfterFold]>;4637}4638}4639multiclass S3_Int<bits<8> o, string OpcodeStr, ValueType vt, RegisterClass RC,4640 X86MemOperand x86memop, SDNode OpNode,4641 X86FoldableSchedWrite sched, PatFrag ld_frag,4642 bit Is2Addr = 1> {4643let Uses = [MXCSR], mayRaiseFPException = 1 in {4644 def rr : S3I<o, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2),4645 !if(Is2Addr,4646 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),4647 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),4648 [(set RC:$dst, (vt (OpNode RC:$src1, RC:$src2)))]>,4649 Sched<[sched]>;4650 4651 def rm : S3I<o, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2),4652 !if(Is2Addr,4653 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),4654 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),4655 [(set RC:$dst, (vt (OpNode RC:$src1, (ld_frag addr:$src2))))]>,4656 Sched<[sched.Folded, sched.ReadAfterFold]>;4657}4658}4659 4660let Predicates = [HasAVX] in {4661 let ExeDomain = SSEPackedSingle in {4662 defm VHADDPS : S3D_Int<0x7C, "vhaddps", v4f32, VR128, f128mem,4663 X86fhadd, WriteFHAdd, loadv4f32, 0>, VEX, VVVV, WIG;4664 defm VHSUBPS : S3D_Int<0x7D, "vhsubps", v4f32, VR128, f128mem,4665 X86fhsub, WriteFHAdd, loadv4f32, 0>, VEX, VVVV, WIG;4666 defm VHADDPSY : S3D_Int<0x7C, "vhaddps", v8f32, VR256, f256mem,4667 X86fhadd, WriteFHAddY, loadv8f32, 0>, VEX, VVVV, VEX_L, WIG;4668 defm VHSUBPSY : S3D_Int<0x7D, "vhsubps", v8f32, VR256, f256mem,4669 X86fhsub, WriteFHAddY, loadv8f32, 0>, VEX, VVVV, VEX_L, WIG;4670 }4671 let ExeDomain = SSEPackedDouble in {4672 defm VHADDPD : S3_Int<0x7C, "vhaddpd", v2f64, VR128, f128mem,4673 X86fhadd, WriteFHAdd, loadv2f64, 0>, VEX, VVVV, WIG;4674 defm VHSUBPD : S3_Int<0x7D, "vhsubpd", v2f64, VR128, f128mem,4675 X86fhsub, WriteFHAdd, loadv2f64, 0>, VEX, VVVV, WIG;4676 defm VHADDPDY : S3_Int<0x7C, "vhaddpd", v4f64, VR256, f256mem,4677 X86fhadd, WriteFHAddY, loadv4f64, 0>, VEX, VVVV, VEX_L, WIG;4678 defm VHSUBPDY : S3_Int<0x7D, "vhsubpd", v4f64, VR256, f256mem,4679 X86fhsub, WriteFHAddY, loadv4f64, 0>, VEX, VVVV, VEX_L, WIG;4680 }4681}4682 4683let Constraints = "$src1 = $dst" in {4684 let ExeDomain = SSEPackedSingle in {4685 defm HADDPS : S3D_Int<0x7C, "haddps", v4f32, VR128, f128mem, X86fhadd,4686 WriteFHAdd, memopv4f32>;4687 defm HSUBPS : S3D_Int<0x7D, "hsubps", v4f32, VR128, f128mem, X86fhsub,4688 WriteFHAdd, memopv4f32>;4689 }4690 let ExeDomain = SSEPackedDouble in {4691 defm HADDPD : S3_Int<0x7C, "haddpd", v2f64, VR128, f128mem, X86fhadd,4692 WriteFHAdd, memopv2f64>;4693 defm HSUBPD : S3_Int<0x7D, "hsubpd", v2f64, VR128, f128mem, X86fhsub,4694 WriteFHAdd, memopv2f64>;4695 }4696}4697 4698//===---------------------------------------------------------------------===//4699// SSSE3 - Packed Absolute Instructions4700//===---------------------------------------------------------------------===//4701 4702/// SS3I_unop_rm_int - Simple SSSE3 unary op whose type can be v*{i8,i16,i32}.4703multiclass SS3I_unop_rm<bits<8> opc, string OpcodeStr, ValueType vt,4704 SDNode OpNode, X86SchedWriteWidths sched, PatFrag ld_frag> {4705 def rr : SS38I<opc, MRMSrcReg, (outs VR128:$dst),4706 (ins VR128:$src),4707 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),4708 [(set VR128:$dst, (vt (OpNode VR128:$src)))]>,4709 Sched<[sched.XMM]>;4710 4711 def rm : SS38I<opc, MRMSrcMem, (outs VR128:$dst),4712 (ins i128mem:$src),4713 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),4714 [(set VR128:$dst,4715 (vt (OpNode (ld_frag addr:$src))))]>,4716 Sched<[sched.XMM.Folded]>;4717}4718 4719/// SS3I_unop_rm_int_y - Simple SSSE3 unary op whose type can be v*{i8,i16,i32}.4720multiclass SS3I_unop_rm_y<bits<8> opc, string OpcodeStr, ValueType vt,4721 SDNode OpNode, X86SchedWriteWidths sched> {4722 def Yrr : SS38I<opc, MRMSrcReg, (outs VR256:$dst),4723 (ins VR256:$src),4724 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),4725 [(set VR256:$dst, (vt (OpNode VR256:$src)))]>,4726 Sched<[sched.YMM]>;4727 4728 def Yrm : SS38I<opc, MRMSrcMem, (outs VR256:$dst),4729 (ins i256mem:$src),4730 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),4731 [(set VR256:$dst,4732 (vt (OpNode (load addr:$src))))]>,4733 Sched<[sched.YMM.Folded]>;4734}4735 4736let Predicates = [HasAVX, NoVLX_Or_NoBWI] in {4737 defm VPABSB : SS3I_unop_rm<0x1C, "vpabsb", v16i8, abs, SchedWriteVecALU,4738 load>, VEX, WIG;4739 defm VPABSW : SS3I_unop_rm<0x1D, "vpabsw", v8i16, abs, SchedWriteVecALU,4740 load>, VEX, WIG;4741}4742let Predicates = [HasAVX, NoVLX] in {4743 defm VPABSD : SS3I_unop_rm<0x1E, "vpabsd", v4i32, abs, SchedWriteVecALU,4744 load>, VEX, WIG;4745}4746let Predicates = [HasAVX2, NoVLX_Or_NoBWI] in {4747 defm VPABSB : SS3I_unop_rm_y<0x1C, "vpabsb", v32i8, abs, SchedWriteVecALU>,4748 VEX, VEX_L, WIG;4749 defm VPABSW : SS3I_unop_rm_y<0x1D, "vpabsw", v16i16, abs, SchedWriteVecALU>,4750 VEX, VEX_L, WIG;4751}4752let Predicates = [HasAVX2, NoVLX] in {4753 defm VPABSD : SS3I_unop_rm_y<0x1E, "vpabsd", v8i32, abs, SchedWriteVecALU>,4754 VEX, VEX_L, WIG;4755}4756 4757defm PABSB : SS3I_unop_rm<0x1C, "pabsb", v16i8, abs, SchedWriteVecALU,4758 memop>;4759defm PABSW : SS3I_unop_rm<0x1D, "pabsw", v8i16, abs, SchedWriteVecALU,4760 memop>;4761defm PABSD : SS3I_unop_rm<0x1E, "pabsd", v4i32, abs, SchedWriteVecALU,4762 memop>;4763 4764//===---------------------------------------------------------------------===//4765// SSSE3 - Packed Binary Operator Instructions4766//===---------------------------------------------------------------------===//4767 4768/// SS3I_binop_rm - Simple SSSE3 bin op4769multiclass SS3I_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,4770 ValueType DstVT, ValueType OpVT, RegisterClass RC,4771 PatFrag memop_frag, X86MemOperand x86memop,4772 X86FoldableSchedWrite sched, bit Is2Addr = 1> {4773 let isCommutable = 1 in4774 def rr : SS38I<opc, MRMSrcReg, (outs RC:$dst),4775 (ins RC:$src1, RC:$src2),4776 !if(Is2Addr,4777 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),4778 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),4779 [(set RC:$dst, (DstVT (OpNode (OpVT RC:$src1), RC:$src2)))]>,4780 Sched<[sched]>;4781 def rm : SS38I<opc, MRMSrcMem, (outs RC:$dst),4782 (ins RC:$src1, x86memop:$src2),4783 !if(Is2Addr,4784 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),4785 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),4786 [(set RC:$dst,4787 (DstVT (OpNode (OpVT RC:$src1), (memop_frag addr:$src2))))]>,4788 Sched<[sched.Folded, sched.ReadAfterFold]>;4789}4790 4791/// SS3I_binop_rm_int - Simple SSSE3 bin op whose type can be v*{i8,i16,i32}.4792multiclass SS3I_binop_rm_int<bits<8> opc, string OpcodeStr,4793 Intrinsic IntId128, X86FoldableSchedWrite sched,4794 PatFrag ld_frag, bit Is2Addr = 1> {4795 let isCommutable = 1 in4796 def rr : SS38I<opc, MRMSrcReg, (outs VR128:$dst),4797 (ins VR128:$src1, VR128:$src2),4798 !if(Is2Addr,4799 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),4800 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),4801 [(set VR128:$dst, (IntId128 VR128:$src1, VR128:$src2))]>,4802 Sched<[sched]>;4803 def rm : SS38I<opc, MRMSrcMem, (outs VR128:$dst),4804 (ins VR128:$src1, i128mem:$src2),4805 !if(Is2Addr,4806 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),4807 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),4808 [(set VR128:$dst,4809 (IntId128 VR128:$src1, (ld_frag addr:$src2)))]>,4810 Sched<[sched.Folded, sched.ReadAfterFold]>;4811}4812 4813multiclass SS3I_binop_rm_int_y<bits<8> opc, string OpcodeStr,4814 Intrinsic IntId256,4815 X86FoldableSchedWrite sched> {4816 let isCommutable = 1 in4817 def Yrr : SS38I<opc, MRMSrcReg, (outs VR256:$dst),4818 (ins VR256:$src1, VR256:$src2),4819 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),4820 [(set VR256:$dst, (IntId256 VR256:$src1, VR256:$src2))]>,4821 Sched<[sched]>;4822 def Yrm : SS38I<opc, MRMSrcMem, (outs VR256:$dst),4823 (ins VR256:$src1, i256mem:$src2),4824 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),4825 [(set VR256:$dst,4826 (IntId256 VR256:$src1, (load addr:$src2)))]>,4827 Sched<[sched.Folded, sched.ReadAfterFold]>;4828}4829 4830let ImmT = NoImm, Predicates = [HasAVX, NoVLX_Or_NoBWI] in {4831let isCommutable = 0 in {4832 defm VPSHUFB : SS3I_binop_rm<0x00, "vpshufb", X86pshufb, v16i8, v16i8,4833 VR128, load, i128mem,4834 SchedWriteVarShuffle.XMM, 0>, VEX, VVVV, WIG;4835 defm VPMADDUBSW : SS3I_binop_rm<0x04, "vpmaddubsw", X86vpmaddubsw, v8i16,4836 v16i8, VR128, load, i128mem,4837 SchedWriteVecIMul.XMM, 0>, VEX, VVVV, WIG;4838}4839defm VPMULHRSW : SS3I_binop_rm<0x0B, "vpmulhrsw", X86mulhrs, v8i16, v8i16,4840 VR128, load, i128mem,4841 SchedWriteVecIMul.XMM, 0>, VEX, VVVV, WIG;4842}4843 4844let ImmT = NoImm, Predicates = [HasAVX] in {4845let isCommutable = 0 in {4846 defm VPHADDW : SS3I_binop_rm<0x01, "vphaddw", X86hadd, v8i16, v8i16, VR128,4847 load, i128mem,4848 SchedWritePHAdd.XMM, 0>, VEX, VVVV, WIG;4849 defm VPHADDD : SS3I_binop_rm<0x02, "vphaddd", X86hadd, v4i32, v4i32, VR128,4850 load, i128mem,4851 SchedWritePHAdd.XMM, 0>, VEX, VVVV, WIG;4852 defm VPHSUBW : SS3I_binop_rm<0x05, "vphsubw", X86hsub, v8i16, v8i16, VR128,4853 load, i128mem,4854 SchedWritePHAdd.XMM, 0>, VEX, VVVV, WIG;4855 defm VPHSUBD : SS3I_binop_rm<0x06, "vphsubd", X86hsub, v4i32, v4i32, VR128,4856 load, i128mem,4857 SchedWritePHAdd.XMM, 0>, VEX, VVVV, WIG;4858 defm VPSIGNB : SS3I_binop_rm_int<0x08, "vpsignb",4859 int_x86_ssse3_psign_b_128,4860 SchedWriteVecALU.XMM, load, 0>, VEX, VVVV, WIG;4861 defm VPSIGNW : SS3I_binop_rm_int<0x09, "vpsignw",4862 int_x86_ssse3_psign_w_128,4863 SchedWriteVecALU.XMM, load, 0>, VEX, VVVV, WIG;4864 defm VPSIGND : SS3I_binop_rm_int<0x0A, "vpsignd",4865 int_x86_ssse3_psign_d_128,4866 SchedWriteVecALU.XMM, load, 0>, VEX, VVVV, WIG;4867 defm VPHADDSW : SS3I_binop_rm<0x03, "vphaddsw", X86hadds, v8i16, v8i16, VR128,4868 load, i128mem,4869 SchedWritePHAdd.XMM, 0>, VEX, VVVV, WIG;4870 defm VPHSUBSW : SS3I_binop_rm<0x07, "vphsubsw", X86hsubs, v8i16, v8i16, VR128,4871 load, i128mem,4872 SchedWritePHAdd.XMM, 0>, VEX, VVVV, WIG;4873}4874}4875 4876let ImmT = NoImm, Predicates = [HasAVX2, NoVLX_Or_NoBWI] in {4877let isCommutable = 0 in {4878 defm VPSHUFBY : SS3I_binop_rm<0x00, "vpshufb", X86pshufb, v32i8, v32i8,4879 VR256, load, i256mem,4880 SchedWriteVarShuffle.YMM, 0>, VEX, VVVV, VEX_L, WIG;4881 defm VPMADDUBSWY : SS3I_binop_rm<0x04, "vpmaddubsw", X86vpmaddubsw, v16i16,4882 v32i8, VR256, load, i256mem,4883 SchedWriteVecIMul.YMM, 0>, VEX, VVVV, VEX_L, WIG;4884}4885defm VPMULHRSWY : SS3I_binop_rm<0x0B, "vpmulhrsw", X86mulhrs, v16i16, v16i16,4886 VR256, load, i256mem,4887 SchedWriteVecIMul.YMM, 0>, VEX, VVVV, VEX_L, WIG;4888}4889 4890let ImmT = NoImm, Predicates = [HasAVX2] in {4891let isCommutable = 0 in {4892 defm VPHADDWY : SS3I_binop_rm<0x01, "vphaddw", X86hadd, v16i16, v16i16,4893 VR256, load, i256mem,4894 SchedWritePHAdd.YMM, 0>, VEX, VVVV, VEX_L, WIG;4895 defm VPHADDDY : SS3I_binop_rm<0x02, "vphaddd", X86hadd, v8i32, v8i32, VR256,4896 load, i256mem,4897 SchedWritePHAdd.YMM, 0>, VEX, VVVV, VEX_L, WIG;4898 defm VPHSUBWY : SS3I_binop_rm<0x05, "vphsubw", X86hsub, v16i16, v16i16,4899 VR256, load, i256mem,4900 SchedWritePHAdd.YMM, 0>, VEX, VVVV, VEX_L, WIG;4901 defm VPHSUBDY : SS3I_binop_rm<0x06, "vphsubd", X86hsub, v8i32, v8i32, VR256,4902 load, i256mem,4903 SchedWritePHAdd.YMM, 0>, VEX, VVVV, VEX_L, WIG;4904 defm VPSIGNB : SS3I_binop_rm_int_y<0x08, "vpsignb", int_x86_avx2_psign_b,4905 SchedWriteVecALU.YMM>, VEX, VVVV, VEX_L, WIG;4906 defm VPSIGNW : SS3I_binop_rm_int_y<0x09, "vpsignw", int_x86_avx2_psign_w,4907 SchedWriteVecALU.YMM>, VEX, VVVV, VEX_L, WIG;4908 defm VPSIGND : SS3I_binop_rm_int_y<0x0A, "vpsignd", int_x86_avx2_psign_d,4909 SchedWriteVecALU.YMM>, VEX, VVVV, VEX_L, WIG;4910 defm VPHADDSWY : SS3I_binop_rm<0x03, "vphaddsw", X86hadds, v16i16, v16i16,4911 VR256, load, i256mem,4912 SchedWritePHAdd.YMM, 0>, VEX, VVVV, VEX_L, WIG;4913 defm VPHSUBSWY : SS3I_binop_rm<0x07, "vphsubsw", X86hsubs, v16i16, v16i16,4914 VR256, load, i256mem,4915 SchedWritePHAdd.YMM, 0>, VEX, VVVV, VEX_L, WIG;4916}4917}4918 4919// None of these have i8 immediate fields.4920let ImmT = NoImm, Constraints = "$src1 = $dst" in {4921let isCommutable = 0 in {4922 defm PHADDW : SS3I_binop_rm<0x01, "phaddw", X86hadd, v8i16, v8i16, VR128,4923 memop, i128mem, SchedWritePHAdd.XMM>;4924 defm PHADDD : SS3I_binop_rm<0x02, "phaddd", X86hadd, v4i32, v4i32, VR128,4925 memop, i128mem, SchedWritePHAdd.XMM>;4926 defm PHSUBW : SS3I_binop_rm<0x05, "phsubw", X86hsub, v8i16, v8i16, VR128,4927 memop, i128mem, SchedWritePHAdd.XMM>;4928 defm PHSUBD : SS3I_binop_rm<0x06, "phsubd", X86hsub, v4i32, v4i32, VR128,4929 memop, i128mem, SchedWritePHAdd.XMM>;4930 defm PSIGNB : SS3I_binop_rm_int<0x08, "psignb", int_x86_ssse3_psign_b_128,4931 SchedWriteVecALU.XMM, memop>;4932 defm PSIGNW : SS3I_binop_rm_int<0x09, "psignw", int_x86_ssse3_psign_w_128,4933 SchedWriteVecALU.XMM, memop>;4934 defm PSIGND : SS3I_binop_rm_int<0x0A, "psignd", int_x86_ssse3_psign_d_128,4935 SchedWriteVecALU.XMM, memop>;4936 defm PSHUFB : SS3I_binop_rm<0x00, "pshufb", X86pshufb, v16i8, v16i8, VR128,4937 memop, i128mem, SchedWriteVarShuffle.XMM>;4938 defm PHADDSW : SS3I_binop_rm<0x03, "phaddsw", X86hadds, v8i16, v8i16, VR128,4939 memop, i128mem, SchedWritePHAdd.XMM>;4940 defm PHSUBSW : SS3I_binop_rm<0x07, "phsubsw", X86hsubs, v8i16, v8i16, VR128,4941 memop, i128mem, SchedWritePHAdd.XMM>;4942 defm PMADDUBSW : SS3I_binop_rm<0x04, "pmaddubsw", X86vpmaddubsw, v8i16,4943 v16i8, VR128, memop, i128mem,4944 SchedWriteVecIMul.XMM>;4945}4946defm PMULHRSW : SS3I_binop_rm<0x0B, "pmulhrsw", X86mulhrs, v8i16, v8i16,4947 VR128, memop, i128mem, SchedWriteVecIMul.XMM>;4948}4949 4950//===---------------------------------------------------------------------===//4951// SSSE3 - Packed Align Instruction Patterns4952//===---------------------------------------------------------------------===//4953 4954multiclass ssse3_palignr<string asm, ValueType VT, RegisterClass RC,4955 PatFrag memop_frag, X86MemOperand x86memop,4956 X86FoldableSchedWrite sched, bit Is2Addr = 1> {4957 let hasSideEffects = 0 in {4958 def rri : SS3AI<0x0F, MRMSrcReg, (outs RC:$dst),4959 (ins RC:$src1, RC:$src2, u8imm:$src3),4960 !if(Is2Addr,4961 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),4962 !strconcat(asm,4963 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),4964 [(set RC:$dst, (VT (X86PAlignr RC:$src1, RC:$src2, (i8 timm:$src3))))]>,4965 Sched<[sched]>;4966 let mayLoad = 1 in4967 def rmi : SS3AI<0x0F, MRMSrcMem, (outs RC:$dst),4968 (ins RC:$src1, x86memop:$src2, u8imm:$src3),4969 !if(Is2Addr,4970 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),4971 !strconcat(asm,4972 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),4973 [(set RC:$dst, (VT (X86PAlignr RC:$src1,4974 (memop_frag addr:$src2),4975 (i8 timm:$src3))))]>,4976 Sched<[sched.Folded, sched.ReadAfterFold]>;4977 }4978}4979 4980let Predicates = [HasAVX, NoVLX_Or_NoBWI] in4981 defm VPALIGNR : ssse3_palignr<"vpalignr", v16i8, VR128, load, i128mem,4982 SchedWriteShuffle.XMM, 0>, VEX, VVVV, WIG;4983let Predicates = [HasAVX2, NoVLX_Or_NoBWI] in4984 defm VPALIGNRY : ssse3_palignr<"vpalignr", v32i8, VR256, load, i256mem,4985 SchedWriteShuffle.YMM, 0>, VEX, VVVV, VEX_L, WIG;4986let Constraints = "$src1 = $dst", Predicates = [UseSSSE3] in4987 defm PALIGNR : ssse3_palignr<"palignr", v16i8, VR128, memop, i128mem,4988 SchedWriteShuffle.XMM>;4989 4990//===---------------------------------------------------------------------===//4991// SSSE3 - Thread synchronization4992//===---------------------------------------------------------------------===//4993 4994let SchedRW = [WriteSystem] in {4995let Uses = [EAX, ECX, EDX] in4996def MONITOR32rrr : I<0x01, MRM_C8, (outs), (ins), "monitor", []>,4997 TB, Requires<[HasSSE3, Not64BitMode]>;4998let Uses = [RAX, ECX, EDX] in4999def MONITOR64rrr : I<0x01, MRM_C8, (outs), (ins), "monitor", []>,5000 TB, Requires<[HasSSE3, In64BitMode]>;5001 5002let Uses = [ECX, EAX] in5003def MWAITrr : I<0x01, MRM_C9, (outs), (ins), "mwait",5004 [(int_x86_sse3_mwait ECX, EAX)]>, TB, Requires<[HasSSE3]>;5005} // SchedRW5006 5007def : InstAlias<"mwait\t{%eax, %ecx|ecx, eax}", (MWAITrr)>, Requires<[Not64BitMode]>;5008def : InstAlias<"mwait\t{%rax, %rcx|rcx, rax}", (MWAITrr)>, Requires<[In64BitMode]>;5009 5010def : InstAlias<"monitor\t{%eax, %ecx, %edx|edx, ecx, eax}", (MONITOR32rrr)>,5011 Requires<[Not64BitMode]>;5012def : InstAlias<"monitor\t{%rax, %rcx, %rdx|rdx, rcx, rax}", (MONITOR64rrr)>,5013 Requires<[In64BitMode]>;5014 5015//===----------------------------------------------------------------------===//5016// SSE4.1 - Packed Move with Sign/Zero Extend5017// NOTE: Any Extend is promoted to Zero Extend in X86ISelDAGToDAG.cpp5018//===----------------------------------------------------------------------===//5019 5020multiclass SS41I_pmovx_rrrm<bits<8> opc, string OpcodeStr, X86MemOperand MemOp,5021 RegisterClass OutRC, RegisterClass InRC,5022 X86FoldableSchedWrite sched> {5023 def rr : SS48I<opc, MRMSrcReg, (outs OutRC:$dst), (ins InRC:$src),5024 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>,5025 Sched<[sched]>;5026 5027 def rm : SS48I<opc, MRMSrcMem, (outs OutRC:$dst), (ins MemOp:$src),5028 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>,5029 Sched<[sched.Folded]>;5030}5031 5032multiclass SS41I_pmovx_rm_all<bits<8> opc, string OpcodeStr,5033 X86MemOperand MemOp, X86MemOperand MemYOp,5034 Predicate prd> {5035 defm NAME : SS41I_pmovx_rrrm<opc, OpcodeStr, MemOp, VR128, VR128,5036 SchedWriteShuffle.XMM>;5037 let Predicates = [HasAVX, prd] in5038 defm V#NAME : SS41I_pmovx_rrrm<opc, !strconcat("v", OpcodeStr), MemOp,5039 VR128, VR128, SchedWriteVecExtend.XMM>,5040 VEX, WIG;5041 let Predicates = [HasAVX2, prd] in5042 defm V#NAME#Y : SS41I_pmovx_rrrm<opc, !strconcat("v", OpcodeStr), MemYOp,5043 VR256, VR128, SchedWriteVecExtend.YMM>,5044 VEX, VEX_L, WIG;5045}5046 5047multiclass SS41I_pmovx_rm<bits<8> opc, string OpcodeStr, X86MemOperand MemOp,5048 X86MemOperand MemYOp, Predicate prd> {5049 defm PMOVSX#NAME : SS41I_pmovx_rm_all<opc, !strconcat("pmovsx", OpcodeStr),5050 MemOp, MemYOp, prd>;5051 defm PMOVZX#NAME : SS41I_pmovx_rm_all<!add(opc, 0x10),5052 !strconcat("pmovzx", OpcodeStr),5053 MemOp, MemYOp, prd>;5054}5055 5056defm BW : SS41I_pmovx_rm<0x20, "bw", i64mem, i128mem, NoVLX_Or_NoBWI>;5057defm WD : SS41I_pmovx_rm<0x23, "wd", i64mem, i128mem, NoVLX>;5058defm DQ : SS41I_pmovx_rm<0x25, "dq", i64mem, i128mem, NoVLX>;5059 5060defm BD : SS41I_pmovx_rm<0x21, "bd", i32mem, i64mem, NoVLX>;5061defm WQ : SS41I_pmovx_rm<0x24, "wq", i32mem, i64mem, NoVLX>;5062 5063defm BQ : SS41I_pmovx_rm<0x22, "bq", i16mem, i32mem, NoVLX>;5064 5065// AVX2 Patterns5066multiclass SS41I_pmovx_avx2_patterns<string OpcPrefix, string ExtTy,5067 SDNode ExtOp, SDNode InVecOp> {5068 // Register-Register patterns5069 let Predicates = [HasAVX2, NoVLX_Or_NoBWI] in {5070 def : Pat<(v16i16 (ExtOp (v16i8 VR128:$src))),5071 (!cast<I>(OpcPrefix#BWYrr) VR128:$src)>;5072 }5073 let Predicates = [HasAVX2, NoVLX] in {5074 def : Pat<(v8i32 (InVecOp (v16i8 VR128:$src))),5075 (!cast<I>(OpcPrefix#BDYrr) VR128:$src)>;5076 def : Pat<(v4i64 (InVecOp (v16i8 VR128:$src))),5077 (!cast<I>(OpcPrefix#BQYrr) VR128:$src)>;5078 5079 def : Pat<(v8i32 (ExtOp (v8i16 VR128:$src))),5080 (!cast<I>(OpcPrefix#WDYrr) VR128:$src)>;5081 def : Pat<(v4i64 (InVecOp (v8i16 VR128:$src))),5082 (!cast<I>(OpcPrefix#WQYrr) VR128:$src)>;5083 5084 def : Pat<(v4i64 (ExtOp (v4i32 VR128:$src))),5085 (!cast<I>(OpcPrefix#DQYrr) VR128:$src)>;5086 }5087 5088 // Simple Register-Memory patterns5089 let Predicates = [HasAVX2, NoVLX_Or_NoBWI] in {5090 def : Pat<(v16i16 (!cast<PatFrag>(ExtTy#"extloadvi8") addr:$src)),5091 (!cast<I>(OpcPrefix#BWYrm) addr:$src)>;5092 5093 def : Pat<(v16i16 (ExtOp (loadv16i8 addr:$src))),5094 (!cast<I>(OpcPrefix#BWYrm) addr:$src)>;5095 }5096 5097 let Predicates = [HasAVX2, NoVLX] in {5098 def : Pat<(v8i32 (!cast<PatFrag>(ExtTy#"extloadvi8") addr:$src)),5099 (!cast<I>(OpcPrefix#BDYrm) addr:$src)>;5100 def : Pat<(v4i64 (!cast<PatFrag>(ExtTy#"extloadvi8") addr:$src)),5101 (!cast<I>(OpcPrefix#BQYrm) addr:$src)>;5102 5103 def : Pat<(v8i32 (!cast<PatFrag>(ExtTy#"extloadvi16") addr:$src)),5104 (!cast<I>(OpcPrefix#WDYrm) addr:$src)>;5105 def : Pat<(v4i64 (!cast<PatFrag>(ExtTy#"extloadvi16") addr:$src)),5106 (!cast<I>(OpcPrefix#WQYrm) addr:$src)>;5107 5108 def : Pat<(v4i64 (!cast<PatFrag>(ExtTy#"extloadvi32") addr:$src)),5109 (!cast<I>(OpcPrefix#DQYrm) addr:$src)>;5110 }5111 5112 // AVX2 Register-Memory patterns5113 let Predicates = [HasAVX2, NoVLX] in {5114 def : Pat<(v8i32 (ExtOp (loadv8i16 addr:$src))),5115 (!cast<I>(OpcPrefix#WDYrm) addr:$src)>;5116 5117 def : Pat<(v8i32 (InVecOp (bc_v16i8 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),5118 (!cast<I>(OpcPrefix#BDYrm) addr:$src)>;5119 def : Pat<(v8i32 (InVecOp (bc_v16i8 (v2f64 (scalar_to_vector (loadf64 addr:$src)))))),5120 (!cast<I>(OpcPrefix#BDYrm) addr:$src)>;5121 def : Pat<(v8i32 (InVecOp (bc_v16i8 (v2i64 (X86vzload64 addr:$src))))),5122 (!cast<I>(OpcPrefix#BDYrm) addr:$src)>;5123 5124 def : Pat<(v4i64 (ExtOp (loadv4i32 addr:$src))),5125 (!cast<I>(OpcPrefix#DQYrm) addr:$src)>;5126 5127 def : Pat<(v4i64 (InVecOp (bc_v16i8 (v4i32 (scalar_to_vector (loadi32 addr:$src)))))),5128 (!cast<I>(OpcPrefix#BQYrm) addr:$src)>;5129 def : Pat<(v4i64 (InVecOp (bc_v16i8 (v2i64 (X86vzload32 addr:$src))))),5130 (!cast<I>(OpcPrefix#BQYrm) addr:$src)>;5131 5132 def : Pat<(v4i64 (InVecOp (bc_v8i16 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),5133 (!cast<I>(OpcPrefix#WQYrm) addr:$src)>;5134 def : Pat<(v4i64 (InVecOp (bc_v8i16 (v2f64 (scalar_to_vector (loadf64 addr:$src)))))),5135 (!cast<I>(OpcPrefix#WQYrm) addr:$src)>;5136 def : Pat<(v4i64 (InVecOp (bc_v8i16 (v2i64 (X86vzload64 addr:$src))))),5137 (!cast<I>(OpcPrefix#WQYrm) addr:$src)>;5138 }5139}5140 5141defm : SS41I_pmovx_avx2_patterns<"VPMOVSX", "s", sext, sext_invec>;5142defm : SS41I_pmovx_avx2_patterns<"VPMOVZX", "z", zext, zext_invec>;5143 5144// SSE4.1/AVX patterns.5145multiclass SS41I_pmovx_patterns<string OpcPrefix, string ExtTy,5146 SDNode ExtOp> {5147 let Predicates = [HasAVX, NoVLX_Or_NoBWI] in {5148 def : Pat<(v8i16 (ExtOp (v16i8 VR128:$src))),5149 (!cast<I>(OpcPrefix#BWrr) VR128:$src)>;5150 }5151 let Predicates = [HasAVX, NoVLX] in {5152 def : Pat<(v4i32 (ExtOp (v16i8 VR128:$src))),5153 (!cast<I>(OpcPrefix#BDrr) VR128:$src)>;5154 def : Pat<(v2i64 (ExtOp (v16i8 VR128:$src))),5155 (!cast<I>(OpcPrefix#BQrr) VR128:$src)>;5156 5157 def : Pat<(v4i32 (ExtOp (v8i16 VR128:$src))),5158 (!cast<I>(OpcPrefix#WDrr) VR128:$src)>;5159 def : Pat<(v2i64 (ExtOp (v8i16 VR128:$src))),5160 (!cast<I>(OpcPrefix#WQrr) VR128:$src)>;5161 5162 def : Pat<(v2i64 (ExtOp (v4i32 VR128:$src))),5163 (!cast<I>(OpcPrefix#DQrr) VR128:$src)>;5164 }5165 let Predicates = [HasAVX, NoVLX_Or_NoBWI] in {5166 def : Pat<(v8i16 (!cast<PatFrag>(ExtTy#"extloadvi8") addr:$src)),5167 (!cast<I>(OpcPrefix#BWrm) addr:$src)>;5168 }5169 let Predicates = [HasAVX, NoVLX] in {5170 def : Pat<(v4i32 (!cast<PatFrag>(ExtTy#"extloadvi8") addr:$src)),5171 (!cast<I>(OpcPrefix#BDrm) addr:$src)>;5172 def : Pat<(v2i64 (!cast<PatFrag>(ExtTy#"extloadvi8") addr:$src)),5173 (!cast<I>(OpcPrefix#BQrm) addr:$src)>;5174 5175 def : Pat<(v4i32 (!cast<PatFrag>(ExtTy#"extloadvi16") addr:$src)),5176 (!cast<I>(OpcPrefix#WDrm) addr:$src)>;5177 def : Pat<(v2i64 (!cast<PatFrag>(ExtTy#"extloadvi16") addr:$src)),5178 (!cast<I>(OpcPrefix#WQrm) addr:$src)>;5179 5180 def : Pat<(v2i64 (!cast<PatFrag>(ExtTy#"extloadvi32") addr:$src)),5181 (!cast<I>(OpcPrefix#DQrm) addr:$src)>;5182 }5183 let Predicates = [HasAVX, NoVLX_Or_NoBWI] in {5184 def : Pat<(v8i16 (ExtOp (bc_v16i8 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),5185 (!cast<I>(OpcPrefix#BWrm) addr:$src)>;5186 def : Pat<(v8i16 (ExtOp (bc_v16i8 (v2f64 (scalar_to_vector (loadf64 addr:$src)))))),5187 (!cast<I>(OpcPrefix#BWrm) addr:$src)>;5188 def : Pat<(v8i16 (ExtOp (bc_v16i8 (v2i64 (X86vzload64 addr:$src))))),5189 (!cast<I>(OpcPrefix#BWrm) addr:$src)>;5190 def : Pat<(v8i16 (ExtOp (loadv16i8 addr:$src))),5191 (!cast<I>(OpcPrefix#BWrm) addr:$src)>;5192 }5193 let Predicates = [HasAVX, NoVLX] in {5194 def : Pat<(v4i32 (ExtOp (bc_v16i8 (v4i32 (scalar_to_vector (loadi32 addr:$src)))))),5195 (!cast<I>(OpcPrefix#BDrm) addr:$src)>;5196 def : Pat<(v4i32 (ExtOp (bc_v16i8 (v4i32 (X86vzload32 addr:$src))))),5197 (!cast<I>(OpcPrefix#BDrm) addr:$src)>;5198 def : Pat<(v4i32 (ExtOp (loadv16i8 addr:$src))),5199 (!cast<I>(OpcPrefix#BDrm) addr:$src)>;5200 5201 def : Pat<(v2i64 (ExtOp (bc_v16i8 (v4i32 (scalar_to_vector (extloadi32i16 addr:$src)))))),5202 (!cast<I>(OpcPrefix#BQrm) addr:$src)>;5203 def : Pat<(v2i64 (ExtOp (loadv16i8 addr:$src))),5204 (!cast<I>(OpcPrefix#BQrm) addr:$src)>;5205 5206 def : Pat<(v4i32 (ExtOp (bc_v8i16 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),5207 (!cast<I>(OpcPrefix#WDrm) addr:$src)>;5208 def : Pat<(v4i32 (ExtOp (bc_v8i16 (v2f64 (scalar_to_vector (loadf64 addr:$src)))))),5209 (!cast<I>(OpcPrefix#WDrm) addr:$src)>;5210 def : Pat<(v4i32 (ExtOp (bc_v8i16 (v2i64 (X86vzload64 addr:$src))))),5211 (!cast<I>(OpcPrefix#WDrm) addr:$src)>;5212 def : Pat<(v4i32 (ExtOp (loadv8i16 addr:$src))),5213 (!cast<I>(OpcPrefix#WDrm) addr:$src)>;5214 5215 def : Pat<(v2i64 (ExtOp (bc_v8i16 (v4i32 (scalar_to_vector (loadi32 addr:$src)))))),5216 (!cast<I>(OpcPrefix#WQrm) addr:$src)>;5217 def : Pat<(v2i64 (ExtOp (bc_v8i16 (v4i32 (X86vzload32 addr:$src))))),5218 (!cast<I>(OpcPrefix#WQrm) addr:$src)>;5219 def : Pat<(v2i64 (ExtOp (loadv8i16 addr:$src))),5220 (!cast<I>(OpcPrefix#WQrm) addr:$src)>;5221 5222 def : Pat<(v2i64 (ExtOp (bc_v4i32 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),5223 (!cast<I>(OpcPrefix#DQrm) addr:$src)>;5224 def : Pat<(v2i64 (ExtOp (bc_v4i32 (v2f64 (scalar_to_vector (loadf64 addr:$src)))))),5225 (!cast<I>(OpcPrefix#DQrm) addr:$src)>;5226 def : Pat<(v2i64 (ExtOp (bc_v4i32 (v2i64 (X86vzload64 addr:$src))))),5227 (!cast<I>(OpcPrefix#DQrm) addr:$src)>;5228 def : Pat<(v2i64 (ExtOp (loadv4i32 addr:$src))),5229 (!cast<I>(OpcPrefix#DQrm) addr:$src)>;5230 }5231}5232 5233defm : SS41I_pmovx_patterns<"VPMOVSX", "s", sext_invec>;5234defm : SS41I_pmovx_patterns<"VPMOVZX", "z", zext_invec>;5235 5236let Predicates = [UseSSE41] in {5237 defm : SS41I_pmovx_patterns<"PMOVSX", "s", sext_invec>;5238 defm : SS41I_pmovx_patterns<"PMOVZX", "z", zext_invec>;5239}5240 5241//===----------------------------------------------------------------------===//5242// SSE4.1 - Extract Instructions5243//===----------------------------------------------------------------------===//5244 5245/// SS41I_binop_ext8 - SSE 4.1 extract 8 bits to 32 bit reg or 8 bit mem5246multiclass SS41I_extract8<bits<8> opc, string OpcodeStr> {5247 def rri : SS4AIi8<opc, MRMDestReg, (outs GR32orGR64:$dst),5248 (ins VR128:$src1, u8imm:$src2),5249 !strconcat(OpcodeStr,5250 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),5251 [(set GR32orGR64:$dst, (X86pextrb (v16i8 VR128:$src1),5252 timm:$src2))]>,5253 Sched<[WriteVecExtract]>;5254 let hasSideEffects = 0, mayStore = 1 in5255 def mri : SS4AIi8<opc, MRMDestMem, (outs),5256 (ins i8mem:$dst, VR128:$src1, u8imm:$src2),5257 !strconcat(OpcodeStr,5258 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),5259 [(store (i8 (trunc (X86pextrb (v16i8 VR128:$src1), timm:$src2))),5260 addr:$dst)]>, Sched<[WriteVecExtractSt]>;5261}5262 5263let Predicates = [HasAVX, NoBWI] in5264 defm VPEXTRB : SS41I_extract8<0x14, "vpextrb">, VEX, WIG;5265 5266defm PEXTRB : SS41I_extract8<0x14, "pextrb">;5267 5268 5269/// SS41I_extract16 - SSE 4.1 extract 16 bits to memory destination5270multiclass SS41I_extract16<bits<8> opc, string OpcodeStr> {5271 let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in5272 def rri_REV : SS4AIi8<opc, MRMDestReg, (outs GR32orGR64:$dst),5273 (ins VR128:$src1, u8imm:$src2),5274 !strconcat(OpcodeStr,5275 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>,5276 Sched<[WriteVecExtract]>;5277 5278 let hasSideEffects = 0, mayStore = 1 in5279 def mri : SS4AIi8<opc, MRMDestMem, (outs),5280 (ins i16mem:$dst, VR128:$src1, u8imm:$src2),5281 !strconcat(OpcodeStr,5282 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),5283 [(store (i16 (trunc (X86pextrw (v8i16 VR128:$src1), timm:$src2))),5284 addr:$dst)]>, Sched<[WriteVecExtractSt]>;5285}5286 5287let Predicates = [HasAVX, NoBWI] in5288 defm VPEXTRW : SS41I_extract16<0x15, "vpextrw">, VEX, WIG;5289 5290defm PEXTRW : SS41I_extract16<0x15, "pextrw">;5291 5292let Predicates = [UseSSE41] in5293 def : Pat<(store f16:$src, addr:$dst), (PEXTRWmri addr:$dst, (v8i16 (COPY_TO_REGCLASS FR16:$src, VR128)), 0)>;5294 5295let Predicates = [HasAVX, NoBWI] in5296 def : Pat<(store f16:$src, addr:$dst), (VPEXTRWmri addr:$dst, (v8i16 (COPY_TO_REGCLASS FR16:$src, VR128)), 0)>;5297 5298 5299/// SS41I_extract32 - SSE 4.1 extract 32 bits to int reg or memory destination5300multiclass SS41I_extract32<bits<8> opc, string OpcodeStr> {5301 def rri : SS4AIi8<opc, MRMDestReg, (outs GR32:$dst),5302 (ins VR128:$src1, u8imm:$src2),5303 !strconcat(OpcodeStr,5304 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),5305 [(set GR32:$dst,5306 (extractelt (v4i32 VR128:$src1), imm:$src2))]>,5307 Sched<[WriteVecExtract]>;5308 def mri : SS4AIi8<opc, MRMDestMem, (outs),5309 (ins i32mem:$dst, VR128:$src1, u8imm:$src2),5310 !strconcat(OpcodeStr,5311 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),5312 [(store (extractelt (v4i32 VR128:$src1), imm:$src2),5313 addr:$dst)]>, Sched<[WriteVecExtractSt]>;5314}5315 5316let Predicates = [HasAVX, NoDQI] in5317 defm VPEXTRD : SS41I_extract32<0x16, "vpextrd">, VEX;5318 5319defm PEXTRD : SS41I_extract32<0x16, "pextrd">;5320 5321/// SS41I_extract32 - SSE 4.1 extract 32 bits to int reg or memory destination5322multiclass SS41I_extract64<bits<8> opc, string OpcodeStr> {5323 def rri : SS4AIi8<opc, MRMDestReg, (outs GR64:$dst),5324 (ins VR128:$src1, u8imm:$src2),5325 !strconcat(OpcodeStr,5326 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),5327 [(set GR64:$dst,5328 (extractelt (v2i64 VR128:$src1), imm:$src2))]>,5329 Sched<[WriteVecExtract]>;5330 def mri : SS4AIi8<opc, MRMDestMem, (outs),5331 (ins i64mem:$dst, VR128:$src1, u8imm:$src2),5332 !strconcat(OpcodeStr,5333 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),5334 [(store (extractelt (v2i64 VR128:$src1), imm:$src2),5335 addr:$dst)]>, Sched<[WriteVecExtractSt]>;5336}5337 5338let Predicates = [HasAVX, NoDQI] in5339 defm VPEXTRQ : SS41I_extract64<0x16, "vpextrq">, VEX, REX_W;5340 5341defm PEXTRQ : SS41I_extract64<0x16, "pextrq">, REX_W;5342 5343/// SS41I_extractf32 - SSE 4.1 extract 32 bits fp value to int reg or memory5344/// destination5345multiclass SS41I_extractf32<bits<8> opc, string OpcodeStr> {5346 def rri : SS4AIi8<opc, MRMDestReg, (outs GR32orGR64:$dst),5347 (ins VR128:$src1, u8imm:$src2),5348 !strconcat(OpcodeStr,5349 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),5350 [(set GR32orGR64:$dst,5351 (extractelt (bc_v4i32 (v4f32 VR128:$src1)), imm:$src2))]>,5352 Sched<[WriteVecExtract]>;5353 def mri : SS4AIi8<opc, MRMDestMem, (outs),5354 (ins f32mem:$dst, VR128:$src1, u8imm:$src2),5355 !strconcat(OpcodeStr,5356 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),5357 [(store (extractelt (bc_v4i32 (v4f32 VR128:$src1)), imm:$src2),5358 addr:$dst)]>, Sched<[WriteVecExtractSt]>;5359}5360 5361let ExeDomain = SSEPackedSingle in {5362 let Predicates = [UseAVX] in5363 defm VEXTRACTPS : SS41I_extractf32<0x17, "vextractps">, VEX, WIG;5364 defm EXTRACTPS : SS41I_extractf32<0x17, "extractps">;5365}5366 5367//===----------------------------------------------------------------------===//5368// SSE4.1 - Insert Instructions5369//===----------------------------------------------------------------------===//5370 5371multiclass SS41I_insert8<bits<8> opc, string asm, bit Is2Addr = 1> {5372 def rri : SS4AIi8<opc, MRMSrcReg, (outs VR128:$dst),5373 (ins VR128:$src1, GR32orGR64:$src2, u8imm:$src3),5374 !if(Is2Addr,5375 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),5376 !strconcat(asm,5377 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),5378 [(set VR128:$dst,5379 (X86pinsrb VR128:$src1, GR32orGR64:$src2, timm:$src3))]>,5380 Sched<[WriteVecInsert, ReadDefault, ReadInt2Fpu]>;5381 def rmi : SS4AIi8<opc, MRMSrcMem, (outs VR128:$dst),5382 (ins VR128:$src1, i8mem:$src2, u8imm:$src3),5383 !if(Is2Addr,5384 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),5385 !strconcat(asm,5386 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),5387 [(set VR128:$dst,5388 (X86pinsrb VR128:$src1, (extloadi8 addr:$src2), timm:$src3))]>,5389 Sched<[WriteVecInsert.Folded, WriteVecInsert.ReadAfterFold]>;5390}5391 5392let Predicates = [HasAVX, NoBWI] in {5393 defm VPINSRB : SS41I_insert8<0x20, "vpinsrb", 0>, VEX, VVVV, WIG;5394 def : Pat<(X86pinsrb VR128:$src1, (i32 (anyext (i8 GR8:$src2))), timm:$src3),5395 (VPINSRBrri VR128:$src1, (INSERT_SUBREG (i32 (IMPLICIT_DEF)),5396 GR8:$src2, sub_8bit), timm:$src3)>;5397}5398 5399let Constraints = "$src1 = $dst" in5400 defm PINSRB : SS41I_insert8<0x20, "pinsrb">;5401 5402multiclass SS41I_insert32<bits<8> opc, string asm, bit Is2Addr = 1> {5403 def rri : SS4AIi8<opc, MRMSrcReg, (outs VR128:$dst),5404 (ins VR128:$src1, GR32:$src2, u8imm:$src3),5405 !if(Is2Addr,5406 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),5407 !strconcat(asm,5408 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),5409 [(set VR128:$dst,5410 (v4i32 (insertelt VR128:$src1, GR32:$src2, imm:$src3)))]>,5411 Sched<[WriteVecInsert, ReadDefault, ReadInt2Fpu]>;5412 def rmi : SS4AIi8<opc, MRMSrcMem, (outs VR128:$dst),5413 (ins VR128:$src1, i32mem:$src2, u8imm:$src3),5414 !if(Is2Addr,5415 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),5416 !strconcat(asm,5417 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),5418 [(set VR128:$dst,5419 (v4i32 (insertelt VR128:$src1, (loadi32 addr:$src2), imm:$src3)))]>,5420 Sched<[WriteVecInsert.Folded, WriteVecInsert.ReadAfterFold]>;5421}5422 5423let Predicates = [HasAVX, NoDQI] in5424 defm VPINSRD : SS41I_insert32<0x22, "vpinsrd", 0>, VEX, VVVV;5425let Constraints = "$src1 = $dst" in5426 defm PINSRD : SS41I_insert32<0x22, "pinsrd">;5427 5428multiclass SS41I_insert64<bits<8> opc, string asm, bit Is2Addr = 1> {5429 def rri : SS4AIi8<opc, MRMSrcReg, (outs VR128:$dst),5430 (ins VR128:$src1, GR64:$src2, u8imm:$src3),5431 !if(Is2Addr,5432 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),5433 !strconcat(asm,5434 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),5435 [(set VR128:$dst,5436 (v2i64 (insertelt VR128:$src1, GR64:$src2, imm:$src3)))]>,5437 Sched<[WriteVecInsert, ReadDefault, ReadInt2Fpu]>;5438 def rmi : SS4AIi8<opc, MRMSrcMem, (outs VR128:$dst),5439 (ins VR128:$src1, i64mem:$src2, u8imm:$src3),5440 !if(Is2Addr,5441 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),5442 !strconcat(asm,5443 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),5444 [(set VR128:$dst,5445 (v2i64 (insertelt VR128:$src1, (loadi64 addr:$src2), imm:$src3)))]>,5446 Sched<[WriteVecInsert.Folded, WriteVecInsert.ReadAfterFold]>;5447}5448 5449let Predicates = [HasAVX, NoDQI] in5450 defm VPINSRQ : SS41I_insert64<0x22, "vpinsrq", 0>, VEX, VVVV, REX_W;5451let Constraints = "$src1 = $dst" in5452 defm PINSRQ : SS41I_insert64<0x22, "pinsrq">, REX_W;5453 5454// insertps has a few different modes, there's the first two here below which5455// are optimized inserts that won't zero arbitrary elements in the destination5456// vector. The next one matches the intrinsic and could zero arbitrary elements5457// in the target vector.5458multiclass SS41I_insertf32<bits<8> opc, string asm, bit Is2Addr = 1> {5459 let isCommutable = 1 in5460 def rri : SS4AIi8<opc, MRMSrcReg, (outs VR128:$dst),5461 (ins VR128:$src1, VR128:$src2, u8imm:$src3),5462 !if(Is2Addr,5463 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),5464 !strconcat(asm,5465 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),5466 [(set VR128:$dst,5467 (X86insertps VR128:$src1, VR128:$src2, timm:$src3))]>,5468 Sched<[SchedWriteFShuffle.XMM]>;5469 def rmi : SS4AIi8<opc, MRMSrcMem, (outs VR128:$dst),5470 (ins VR128:$src1, f32mem:$src2, u8imm:$src3),5471 !if(Is2Addr,5472 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),5473 !strconcat(asm,5474 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),5475 [(set VR128:$dst,5476 (X86insertps VR128:$src1,5477 (v4f32 (scalar_to_vector (loadf32 addr:$src2))),5478 timm:$src3))]>,5479 Sched<[SchedWriteFShuffle.XMM.Folded, SchedWriteFShuffle.XMM.ReadAfterFold]>;5480}5481 5482let ExeDomain = SSEPackedSingle in {5483 let Predicates = [UseAVX] in5484 defm VINSERTPS : SS41I_insertf32<0x21, "vinsertps", 0>,5485 VEX, VVVV, WIG;5486 let Constraints = "$src1 = $dst" in5487 defm INSERTPS : SS41I_insertf32<0x21, "insertps", 1>;5488}5489 5490//===----------------------------------------------------------------------===//5491// SSE4.1 - Round Instructions5492//===----------------------------------------------------------------------===//5493 5494multiclass sse41_fp_unop_p<bits<8> opc, string OpcodeStr,5495 X86MemOperand x86memop, RegisterClass RC,5496 ValueType VT, PatFrag mem_frag, SDPatternOperator OpNode,5497 X86FoldableSchedWrite sched> {5498 // Intrinsic operation, reg.5499 // Vector intrinsic operation, reg5500let Uses = [MXCSR], mayRaiseFPException = 1 in {5501 def ri : SS4AIi8<opc, MRMSrcReg,5502 (outs RC:$dst), (ins RC:$src1, i32u8imm:$src2),5503 !strconcat(OpcodeStr,5504 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),5505 [(set RC:$dst, (VT (OpNode RC:$src1, timm:$src2)))]>,5506 Sched<[sched]>;5507 5508 // Vector intrinsic operation, mem5509 def mi : SS4AIi8<opc, MRMSrcMem,5510 (outs RC:$dst), (ins x86memop:$src1, i32u8imm:$src2),5511 !strconcat(OpcodeStr,5512 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),5513 [(set RC:$dst,5514 (VT (OpNode (mem_frag addr:$src1), timm:$src2)))]>,5515 Sched<[sched.Folded]>;5516}5517}5518 5519multiclass avx_fp_unop_rm<bits<8> opcss, bits<8> opcsd,5520 string OpcodeStr, X86FoldableSchedWrite sched> {5521let ExeDomain = SSEPackedSingle, hasSideEffects = 0, isCodeGenOnly = 1 in {5522 def SSri : SS4AIi8<opcss, MRMSrcReg,5523 (outs FR32:$dst), (ins FR32:$src1, FR32:$src2, i32u8imm:$src3),5524 !strconcat(OpcodeStr,5525 "ss\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),5526 []>, Sched<[sched]>;5527 5528 let mayLoad = 1 in5529 def SSmi : SS4AIi8<opcss, MRMSrcMem,5530 (outs FR32:$dst), (ins FR32:$src1, f32mem:$src2, i32u8imm:$src3),5531 !strconcat(OpcodeStr,5532 "ss\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),5533 []>, Sched<[sched.Folded, sched.ReadAfterFold]>;5534} // ExeDomain = SSEPackedSingle, hasSideEffects = 05535 5536let ExeDomain = SSEPackedDouble, hasSideEffects = 0, isCodeGenOnly = 1 in {5537 def SDri : SS4AIi8<opcsd, MRMSrcReg,5538 (outs FR64:$dst), (ins FR64:$src1, FR64:$src2, i32u8imm:$src3),5539 !strconcat(OpcodeStr,5540 "sd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),5541 []>, Sched<[sched]>;5542 5543 let mayLoad = 1 in5544 def SDmi : SS4AIi8<opcsd, MRMSrcMem,5545 (outs FR64:$dst), (ins FR64:$src1, f64mem:$src2, i32u8imm:$src3),5546 !strconcat(OpcodeStr,5547 "sd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),5548 []>, Sched<[sched.Folded, sched.ReadAfterFold]>;5549} // ExeDomain = SSEPackedDouble, hasSideEffects = 05550}5551 5552multiclass sse41_fp_unop_s<bits<8> opcss, bits<8> opcsd,5553 string OpcodeStr, X86FoldableSchedWrite sched> {5554let Uses = [MXCSR], mayRaiseFPException = 1 in {5555let ExeDomain = SSEPackedSingle, hasSideEffects = 0, isCodeGenOnly = 1 in {5556 def SSri : SS4AIi8<opcss, MRMSrcReg,5557 (outs FR32:$dst), (ins FR32:$src1, i32u8imm:$src2),5558 !strconcat(OpcodeStr,5559 "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"),5560 []>, Sched<[sched]>;5561 5562 let mayLoad = 1 in5563 def SSmi : SS4AIi8<opcss, MRMSrcMem,5564 (outs FR32:$dst), (ins f32mem:$src1, i32u8imm:$src2),5565 !strconcat(OpcodeStr,5566 "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"),5567 []>, Sched<[sched.Folded, sched.ReadAfterFold]>;5568} // ExeDomain = SSEPackedSingle, hasSideEffects = 05569 5570let ExeDomain = SSEPackedDouble, hasSideEffects = 0, isCodeGenOnly = 1 in {5571 def SDri : SS4AIi8<opcsd, MRMSrcReg,5572 (outs FR64:$dst), (ins FR64:$src1, i32u8imm:$src2),5573 !strconcat(OpcodeStr,5574 "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"),5575 []>, Sched<[sched]>;5576 5577 let mayLoad = 1 in5578 def SDmi : SS4AIi8<opcsd, MRMSrcMem,5579 (outs FR64:$dst), (ins f64mem:$src1, i32u8imm:$src2),5580 !strconcat(OpcodeStr,5581 "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"),5582 []>, Sched<[sched.Folded, sched.ReadAfterFold]>;5583} // ExeDomain = SSEPackedDouble, hasSideEffects = 05584}5585}5586 5587multiclass sse41_fp_unop_s_int<bits<8> opcss, bits<8> opcsd,5588 string OpcodeStr, X86FoldableSchedWrite sched,5589 ValueType VT32, ValueType VT64,5590 SDNode OpNode, bit Is2Addr = 1> {5591let Uses = [MXCSR], mayRaiseFPException = 1 in {5592let ExeDomain = SSEPackedSingle in {5593 def SSri_Int : SS4AIi8<opcss, MRMSrcReg,5594 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2, i32u8imm:$src3),5595 !if(Is2Addr,5596 !strconcat(OpcodeStr,5597 "ss\t{$src3, $src2, $dst|$dst, $src2, $src3}"),5598 !strconcat(OpcodeStr,5599 "ss\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),5600 [(set VR128:$dst, (VT32 (OpNode VR128:$src1, VR128:$src2, timm:$src3)))]>,5601 Sched<[sched]>;5602 5603 def SSmi_Int : SS4AIi8<opcss, MRMSrcMem,5604 (outs VR128:$dst), (ins VR128:$src1, ssmem:$src2, i32u8imm:$src3),5605 !if(Is2Addr,5606 !strconcat(OpcodeStr,5607 "ss\t{$src3, $src2, $dst|$dst, $src2, $src3}"),5608 !strconcat(OpcodeStr,5609 "ss\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),5610 [(set VR128:$dst,5611 (OpNode VR128:$src1, (sse_load_f32 addr:$src2), timm:$src3))]>,5612 Sched<[sched.Folded, sched.ReadAfterFold]>;5613} // ExeDomain = SSEPackedSingle, isCodeGenOnly = 15614 5615let ExeDomain = SSEPackedDouble in {5616 def SDri_Int : SS4AIi8<opcsd, MRMSrcReg,5617 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2, i32u8imm:$src3),5618 !if(Is2Addr,5619 !strconcat(OpcodeStr,5620 "sd\t{$src3, $src2, $dst|$dst, $src2, $src3}"),5621 !strconcat(OpcodeStr,5622 "sd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),5623 [(set VR128:$dst, (VT64 (OpNode VR128:$src1, VR128:$src2, timm:$src3)))]>,5624 Sched<[sched]>;5625 5626 def SDmi_Int : SS4AIi8<opcsd, MRMSrcMem,5627 (outs VR128:$dst), (ins VR128:$src1, sdmem:$src2, i32u8imm:$src3),5628 !if(Is2Addr,5629 !strconcat(OpcodeStr,5630 "sd\t{$src3, $src2, $dst|$dst, $src2, $src3}"),5631 !strconcat(OpcodeStr,5632 "sd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),5633 [(set VR128:$dst,5634 (OpNode VR128:$src1, (sse_load_f64 addr:$src2), timm:$src3))]>,5635 Sched<[sched.Folded, sched.ReadAfterFold]>;5636} // ExeDomain = SSEPackedDouble, isCodeGenOnly = 15637}5638}5639 5640// FP round - roundss, roundps, roundsd, roundpd5641let Predicates = [HasAVX, NoVLX] in {5642 let ExeDomain = SSEPackedSingle, Uses = [MXCSR], mayRaiseFPException = 1 in {5643 // Intrinsic form5644 defm VROUNDPS : sse41_fp_unop_p<0x08, "vroundps", f128mem, VR128, v4f32,5645 loadv4f32, X86any_VRndScale, SchedWriteFRnd.XMM>,5646 VEX, WIG;5647 defm VROUNDPSY : sse41_fp_unop_p<0x08, "vroundps", f256mem, VR256, v8f32,5648 loadv8f32, X86any_VRndScale, SchedWriteFRnd.YMM>,5649 VEX, VEX_L, WIG;5650 }5651 5652 let ExeDomain = SSEPackedDouble, Uses = [MXCSR], mayRaiseFPException = 1 in {5653 defm VROUNDPD : sse41_fp_unop_p<0x09, "vroundpd", f128mem, VR128, v2f64,5654 loadv2f64, X86any_VRndScale, SchedWriteFRnd.XMM>,5655 VEX, WIG;5656 defm VROUNDPDY : sse41_fp_unop_p<0x09, "vroundpd", f256mem, VR256, v4f64,5657 loadv4f64, X86any_VRndScale, SchedWriteFRnd.YMM>,5658 VEX, VEX_L, WIG;5659 }5660}5661let Predicates = [UseAVX] in {5662 defm VROUND : sse41_fp_unop_s_int<0x0A, 0x0B, "vround", SchedWriteFRnd.Scl,5663 v4f32, v2f64, X86RndScales, 0>,5664 VEX, VVVV, VEX_LIG, WIG, SIMD_EXC;5665 defm VROUND : avx_fp_unop_rm<0x0A, 0x0B, "vround", SchedWriteFRnd.Scl>,5666 VEX, VVVV, VEX_LIG, WIG, SIMD_EXC;5667}5668 5669let Predicates = [UseAVX] in {5670 def : Pat<(X86any_VRndScale FR32:$src1, timm:$src2),5671 (VROUNDSSri (f32 (IMPLICIT_DEF)), FR32:$src1, timm:$src2)>;5672 def : Pat<(X86any_VRndScale FR64:$src1, timm:$src2),5673 (VROUNDSDri (f64 (IMPLICIT_DEF)), FR64:$src1, timm:$src2)>;5674}5675 5676let Predicates = [UseAVX, OptForSize] in {5677 def : Pat<(X86any_VRndScale (loadf32 addr:$src1), timm:$src2),5678 (VROUNDSSmi (f32 (IMPLICIT_DEF)), addr:$src1, timm:$src2)>;5679 def : Pat<(X86any_VRndScale (loadf64 addr:$src1), timm:$src2),5680 (VROUNDSDmi (f64 (IMPLICIT_DEF)), addr:$src1, timm:$src2)>;5681}5682 5683let ExeDomain = SSEPackedSingle in5684defm ROUNDPS : sse41_fp_unop_p<0x08, "roundps", f128mem, VR128, v4f32,5685 memopv4f32, X86any_VRndScale, SchedWriteFRnd.XMM>;5686let ExeDomain = SSEPackedDouble in5687defm ROUNDPD : sse41_fp_unop_p<0x09, "roundpd", f128mem, VR128, v2f64,5688 memopv2f64, X86any_VRndScale, SchedWriteFRnd.XMM>;5689 5690defm ROUND : sse41_fp_unop_s<0x0A, 0x0B, "round", SchedWriteFRnd.Scl>;5691 5692let Constraints = "$src1 = $dst" in5693defm ROUND : sse41_fp_unop_s_int<0x0A, 0x0B, "round", SchedWriteFRnd.Scl,5694 v4f32, v2f64, X86RndScales>;5695 5696let Predicates = [UseSSE41] in {5697 def : Pat<(X86any_VRndScale FR32:$src1, timm:$src2),5698 (ROUNDSSri FR32:$src1, timm:$src2)>;5699 def : Pat<(X86any_VRndScale FR64:$src1, timm:$src2),5700 (ROUNDSDri FR64:$src1, timm:$src2)>;5701}5702 5703let Predicates = [UseSSE41, OptForSize] in {5704 def : Pat<(X86any_VRndScale (loadf32 addr:$src1), timm:$src2),5705 (ROUNDSSmi addr:$src1, timm:$src2)>;5706 def : Pat<(X86any_VRndScale (loadf64 addr:$src1), timm:$src2),5707 (ROUNDSDmi addr:$src1, timm:$src2)>;5708}5709 5710//===----------------------------------------------------------------------===//5711// SSE4.1 - Packed Bit Test5712//===----------------------------------------------------------------------===//5713 5714// ptest is commutable if only the Z flag is used. If the C flag is used,5715// commuting would change which operand is inverted.5716def X86ptest_commutable : PatFrag<(ops node:$src1, node:$src2),5717 (X86ptest node:$src1, node:$src2), [{5718 return onlyUsesZeroFlag(SDValue(N, 0));5719}]>;5720 5721// ptest instruction we'll lower to this in X86ISelLowering primarily from5722// the intel intrinsic that corresponds to this.5723let Defs = [EFLAGS], Predicates = [HasAVX] in {5724def VPTESTrr : SS48I<0x17, MRMSrcReg, (outs), (ins VR128:$src1, VR128:$src2),5725 "vptest\t{$src2, $src1|$src1, $src2}",5726 [(set EFLAGS, (X86ptest VR128:$src1, (v2i64 VR128:$src2)))]>,5727 Sched<[SchedWriteVecTest.XMM]>, VEX, WIG;5728def VPTESTrm : SS48I<0x17, MRMSrcMem, (outs), (ins VR128:$src1, f128mem:$src2),5729 "vptest\t{$src2, $src1|$src1, $src2}",5730 [(set EFLAGS,(X86ptest VR128:$src1, (loadv2i64 addr:$src2)))]>,5731 Sched<[SchedWriteVecTest.XMM.Folded, SchedWriteVecTest.XMM.ReadAfterFold]>,5732 VEX, WIG;5733 5734def VPTESTYrr : SS48I<0x17, MRMSrcReg, (outs), (ins VR256:$src1, VR256:$src2),5735 "vptest\t{$src2, $src1|$src1, $src2}",5736 [(set EFLAGS, (X86ptest VR256:$src1, (v4i64 VR256:$src2)))]>,5737 Sched<[SchedWriteVecTest.YMM]>, VEX, VEX_L, WIG;5738def VPTESTYrm : SS48I<0x17, MRMSrcMem, (outs), (ins VR256:$src1, i256mem:$src2),5739 "vptest\t{$src2, $src1|$src1, $src2}",5740 [(set EFLAGS,(X86ptest VR256:$src1, (loadv4i64 addr:$src2)))]>,5741 Sched<[SchedWriteVecTest.YMM.Folded, SchedWriteVecTest.YMM.ReadAfterFold]>,5742 VEX, VEX_L, WIG;5743}5744 5745let Defs = [EFLAGS] in {5746def PTESTrr : SS48I<0x17, MRMSrcReg, (outs), (ins VR128:$src1, VR128:$src2),5747 "ptest\t{$src2, $src1|$src1, $src2}",5748 [(set EFLAGS, (X86ptest VR128:$src1, (v2i64 VR128:$src2)))]>,5749 Sched<[SchedWriteVecTest.XMM]>;5750def PTESTrm : SS48I<0x17, MRMSrcMem, (outs), (ins VR128:$src1, f128mem:$src2),5751 "ptest\t{$src2, $src1|$src1, $src2}",5752 [(set EFLAGS, (X86ptest VR128:$src1, (memopv2i64 addr:$src2)))]>,5753 Sched<[SchedWriteVecTest.XMM.Folded, SchedWriteVecTest.XMM.ReadAfterFold]>;5754}5755 5756let Predicates = [HasAVX] in {5757 def : Pat<(X86ptest_commutable (loadv2i64 addr:$src2), VR128:$src1),5758 (VPTESTrm VR128:$src1, addr:$src2)>;5759 def : Pat<(X86ptest_commutable (loadv4i64 addr:$src2), VR256:$src1),5760 (VPTESTYrm VR256:$src1, addr:$src2)>;5761}5762let Predicates = [UseSSE41] in {5763 def : Pat<(X86ptest_commutable (memopv2i64 addr:$src2), VR128:$src1),5764 (PTESTrm VR128:$src1, addr:$src2)>;5765}5766 5767// The bit test instructions below are AVX only5768multiclass avx_bittest<bits<8> opc, string OpcodeStr, RegisterClass RC,5769 X86MemOperand x86memop, PatFrag mem_frag, ValueType vt,5770 X86FoldableSchedWrite sched> {5771 def rr : SS48I<opc, MRMSrcReg, (outs), (ins RC:$src1, RC:$src2),5772 !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"),5773 [(set EFLAGS, (X86testp RC:$src1, (vt RC:$src2)))]>,5774 Sched<[sched]>, VEX;5775 def rm : SS48I<opc, MRMSrcMem, (outs), (ins RC:$src1, x86memop:$src2),5776 !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"),5777 [(set EFLAGS, (X86testp RC:$src1, (mem_frag addr:$src2)))]>,5778 Sched<[sched.Folded, sched.ReadAfterFold]>, VEX;5779}5780 5781// testps/testpd are commutable if only the Z flag is used. If the C flag is5782// used, commuting would change which operand is inverted.5783def X86testp_commutable : PatFrag<(ops node:$src1, node:$src2),5784 (X86testp node:$src1, node:$src2), [{5785 return onlyUsesZeroFlag(SDValue(N, 0));5786}]>;5787 5788let Defs = [EFLAGS], Predicates = [HasAVX] in {5789let ExeDomain = SSEPackedSingle in {5790defm VTESTPS : avx_bittest<0x0E, "vtestps", VR128, f128mem, loadv4f32, v4f32,5791 SchedWriteFTest.XMM>;5792defm VTESTPSY : avx_bittest<0x0E, "vtestps", VR256, f256mem, loadv8f32, v8f32,5793 SchedWriteFTest.YMM>, VEX_L;5794}5795let ExeDomain = SSEPackedDouble in {5796defm VTESTPD : avx_bittest<0x0F, "vtestpd", VR128, f128mem, loadv2f64, v2f64,5797 SchedWriteFTest.XMM>;5798defm VTESTPDY : avx_bittest<0x0F, "vtestpd", VR256, f256mem, loadv4f64, v4f64,5799 SchedWriteFTest.YMM>, VEX_L;5800}5801}5802 5803let Predicates = [HasAVX] in {5804 def : Pat<(X86testp_commutable (loadv4f32 addr:$src2), VR128:$src),5805 (VTESTPSrm VR128:$src, addr:$src2)>;5806 def : Pat<(X86testp_commutable (loadv8f32 addr:$src2), VR256:$src),5807 (VTESTPSYrm VR256:$src, addr:$src2)>;5808 5809 def : Pat<(X86testp_commutable (loadv2f64 addr:$src2), VR128:$src),5810 (VTESTPDrm VR128:$src, addr:$src2)>;5811 def : Pat<(X86testp_commutable (loadv4f64 addr:$src2), VR256:$src),5812 (VTESTPDYrm VR256:$src, addr:$src2)>;5813}5814 5815//===----------------------------------------------------------------------===//5816// SSE4.1 - Misc Instructions5817//===----------------------------------------------------------------------===//5818 5819let Defs = [EFLAGS], Predicates = [HasPOPCNT] in {5820 defm POPCNT16 : Lzcnt<0xB8, "popcnt", ctpop, Xi16, WritePOPCNT, WritePOPCNT.Folded>, OpSize16, XS;5821 defm POPCNT32 : Lzcnt<0xB8, "popcnt", ctpop, Xi32, WritePOPCNT, WritePOPCNT.Folded>, OpSize32, XS;5822 defm POPCNT64 : Lzcnt<0xB8, "popcnt", ctpop, Xi64, WritePOPCNT, WritePOPCNT.Folded>, XS;5823 5824 defm POPCNT16 : Lzcnt<0x88, "popcnt", null_frag, Xi16, WritePOPCNT, WritePOPCNT.Folded, "_EVEX">, PL, PD;5825 defm POPCNT32 : Lzcnt<0x88, "popcnt", null_frag, Xi32, WritePOPCNT, WritePOPCNT.Folded, "_EVEX">, PL;5826 defm POPCNT64 : Lzcnt<0x88, "popcnt", null_frag, Xi64, WritePOPCNT, WritePOPCNT.Folded, "_EVEX">, PL;5827}5828 5829defm POPCNT16 : Lzcnt<0x88, "popcnt", null_frag, Xi16, WritePOPCNT, WritePOPCNT.Folded, "_NF">, NF, PD;5830defm POPCNT32 : Lzcnt<0x88, "popcnt", null_frag, Xi32, WritePOPCNT, WritePOPCNT.Folded, "_NF">, NF;5831defm POPCNT64 : Lzcnt<0x88, "popcnt", null_frag, Xi64, WritePOPCNT, WritePOPCNT.Folded, "_NF">, NF;5832 5833// SS41I_unop_rm_int_v16 - SSE 4.1 unary operator whose type is v8i16.5834multiclass SS41I_unop_rm_int_v16<bits<8> opc, string OpcodeStr,5835 SDNode OpNode, PatFrag ld_frag,5836 X86FoldableSchedWrite Sched> {5837 def rr : SS48I<opc, MRMSrcReg, (outs VR128:$dst),5838 (ins VR128:$src),5839 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),5840 [(set VR128:$dst, (v8i16 (OpNode (v8i16 VR128:$src))))]>,5841 Sched<[Sched]>;5842 def rm : SS48I<opc, MRMSrcMem, (outs VR128:$dst),5843 (ins i128mem:$src),5844 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),5845 [(set VR128:$dst,5846 (v8i16 (OpNode (ld_frag addr:$src))))]>,5847 Sched<[Sched.Folded]>;5848}5849 5850// PHMIN has the same profile as PSAD, thus we use the same scheduling5851// model, although the naming is misleading.5852let Predicates = [HasAVX] in5853defm VPHMINPOSUW : SS41I_unop_rm_int_v16<0x41, "vphminposuw",5854 X86phminpos, load,5855 WritePHMINPOS>, VEX, WIG;5856defm PHMINPOSUW : SS41I_unop_rm_int_v16<0x41, "phminposuw",5857 X86phminpos, memop,5858 WritePHMINPOS>;5859 5860/// SS48I_binop_rm - Simple SSE41 binary operator.5861multiclass SS48I_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,5862 ValueType OpVT, RegisterClass RC, PatFrag memop_frag,5863 X86MemOperand x86memop, X86FoldableSchedWrite sched,5864 bit Is2Addr = 1> {5865 let isCommutable = 1 in5866 def rr : SS48I<opc, MRMSrcReg, (outs RC:$dst),5867 (ins RC:$src1, RC:$src2),5868 !if(Is2Addr,5869 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),5870 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),5871 [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2)))]>,5872 Sched<[sched]>;5873 def rm : SS48I<opc, MRMSrcMem, (outs RC:$dst),5874 (ins RC:$src1, x86memop:$src2),5875 !if(Is2Addr,5876 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),5877 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),5878 [(set RC:$dst,5879 (OpVT (OpNode RC:$src1, (memop_frag addr:$src2))))]>,5880 Sched<[sched.Folded, sched.ReadAfterFold]>;5881}5882 5883let Predicates = [HasAVX, NoVLX] in {5884 defm VPMINSD : SS48I_binop_rm<0x39, "vpminsd", smin, v4i32, VR128,5885 load, i128mem, SchedWriteVecALU.XMM, 0>,5886 VEX, VVVV, WIG;5887 defm VPMINUD : SS48I_binop_rm<0x3B, "vpminud", umin, v4i32, VR128,5888 load, i128mem, SchedWriteVecALU.XMM, 0>,5889 VEX, VVVV, WIG;5890 defm VPMAXSD : SS48I_binop_rm<0x3D, "vpmaxsd", smax, v4i32, VR128,5891 load, i128mem, SchedWriteVecALU.XMM, 0>,5892 VEX, VVVV, WIG;5893 defm VPMAXUD : SS48I_binop_rm<0x3F, "vpmaxud", umax, v4i32, VR128,5894 load, i128mem, SchedWriteVecALU.XMM, 0>,5895 VEX, VVVV, WIG;5896 defm VPMULDQ : SS48I_binop_rm<0x28, "vpmuldq", X86pmuldq, v2i64, VR128,5897 load, i128mem, SchedWriteVecIMul.XMM, 0>,5898 VEX, VVVV, WIG;5899}5900let Predicates = [HasAVX, NoVLX_Or_NoBWI] in {5901 defm VPMINSB : SS48I_binop_rm<0x38, "vpminsb", smin, v16i8, VR128,5902 load, i128mem, SchedWriteVecALU.XMM, 0>,5903 VEX, VVVV, WIG;5904 defm VPMINUW : SS48I_binop_rm<0x3A, "vpminuw", umin, v8i16, VR128,5905 load, i128mem, SchedWriteVecALU.XMM, 0>,5906 VEX, VVVV, WIG;5907 defm VPMAXSB : SS48I_binop_rm<0x3C, "vpmaxsb", smax, v16i8, VR128,5908 load, i128mem, SchedWriteVecALU.XMM, 0>,5909 VEX, VVVV, WIG;5910 defm VPMAXUW : SS48I_binop_rm<0x3E, "vpmaxuw", umax, v8i16, VR128,5911 load, i128mem, SchedWriteVecALU.XMM, 0>,5912 VEX, VVVV, WIG;5913}5914 5915let Predicates = [HasAVX2, NoVLX] in {5916 defm VPMINSDY : SS48I_binop_rm<0x39, "vpminsd", smin, v8i32, VR256,5917 load, i256mem, SchedWriteVecALU.YMM, 0>,5918 VEX, VVVV, VEX_L, WIG;5919 defm VPMINUDY : SS48I_binop_rm<0x3B, "vpminud", umin, v8i32, VR256,5920 load, i256mem, SchedWriteVecALU.YMM, 0>,5921 VEX, VVVV, VEX_L, WIG;5922 defm VPMAXSDY : SS48I_binop_rm<0x3D, "vpmaxsd", smax, v8i32, VR256,5923 load, i256mem, SchedWriteVecALU.YMM, 0>,5924 VEX, VVVV, VEX_L, WIG;5925 defm VPMAXUDY : SS48I_binop_rm<0x3F, "vpmaxud", umax, v8i32, VR256,5926 load, i256mem, SchedWriteVecALU.YMM, 0>,5927 VEX, VVVV, VEX_L, WIG;5928 defm VPMULDQY : SS48I_binop_rm<0x28, "vpmuldq", X86pmuldq, v4i64, VR256,5929 load, i256mem, SchedWriteVecIMul.YMM, 0>,5930 VEX, VVVV, VEX_L, WIG;5931}5932let Predicates = [HasAVX2, NoVLX_Or_NoBWI] in {5933 defm VPMINSBY : SS48I_binop_rm<0x38, "vpminsb", smin, v32i8, VR256,5934 load, i256mem, SchedWriteVecALU.YMM, 0>,5935 VEX, VVVV, VEX_L, WIG;5936 defm VPMINUWY : SS48I_binop_rm<0x3A, "vpminuw", umin, v16i16, VR256,5937 load, i256mem, SchedWriteVecALU.YMM, 0>,5938 VEX, VVVV, VEX_L, WIG;5939 defm VPMAXSBY : SS48I_binop_rm<0x3C, "vpmaxsb", smax, v32i8, VR256,5940 load, i256mem, SchedWriteVecALU.YMM, 0>,5941 VEX, VVVV, VEX_L, WIG;5942 defm VPMAXUWY : SS48I_binop_rm<0x3E, "vpmaxuw", umax, v16i16, VR256,5943 load, i256mem, SchedWriteVecALU.YMM, 0>,5944 VEX, VVVV, VEX_L, WIG;5945}5946 5947let Constraints = "$src1 = $dst" in {5948 defm PMINSB : SS48I_binop_rm<0x38, "pminsb", smin, v16i8, VR128,5949 memop, i128mem, SchedWriteVecALU.XMM, 1>;5950 defm PMINSD : SS48I_binop_rm<0x39, "pminsd", smin, v4i32, VR128,5951 memop, i128mem, SchedWriteVecALU.XMM, 1>;5952 defm PMINUD : SS48I_binop_rm<0x3B, "pminud", umin, v4i32, VR128,5953 memop, i128mem, SchedWriteVecALU.XMM, 1>;5954 defm PMINUW : SS48I_binop_rm<0x3A, "pminuw", umin, v8i16, VR128,5955 memop, i128mem, SchedWriteVecALU.XMM, 1>;5956 defm PMAXSB : SS48I_binop_rm<0x3C, "pmaxsb", smax, v16i8, VR128,5957 memop, i128mem, SchedWriteVecALU.XMM, 1>;5958 defm PMAXSD : SS48I_binop_rm<0x3D, "pmaxsd", smax, v4i32, VR128,5959 memop, i128mem, SchedWriteVecALU.XMM, 1>;5960 defm PMAXUD : SS48I_binop_rm<0x3F, "pmaxud", umax, v4i32, VR128,5961 memop, i128mem, SchedWriteVecALU.XMM, 1>;5962 defm PMAXUW : SS48I_binop_rm<0x3E, "pmaxuw", umax, v8i16, VR128,5963 memop, i128mem, SchedWriteVecALU.XMM, 1>;5964 defm PMULDQ : SS48I_binop_rm<0x28, "pmuldq", X86pmuldq, v2i64, VR128,5965 memop, i128mem, SchedWriteVecIMul.XMM, 1>;5966}5967 5968let Predicates = [HasAVX, NoVLX] in5969 defm VPMULLD : SS48I_binop_rm<0x40, "vpmulld", mul, v4i32, VR128,5970 load, i128mem, SchedWritePMULLD.XMM, 0>,5971 VEX, VVVV, WIG;5972let Predicates = [HasAVX] in5973 defm VPCMPEQQ : SS48I_binop_rm<0x29, "vpcmpeqq", X86pcmpeq, v2i64, VR128,5974 load, i128mem, SchedWriteVecALU.XMM, 0>,5975 VEX, VVVV, WIG;5976 5977let Predicates = [HasAVX2, NoVLX] in5978 defm VPMULLDY : SS48I_binop_rm<0x40, "vpmulld", mul, v8i32, VR256,5979 load, i256mem, SchedWritePMULLD.YMM, 0>,5980 VEX, VVVV, VEX_L, WIG;5981let Predicates = [HasAVX2] in5982 defm VPCMPEQQY : SS48I_binop_rm<0x29, "vpcmpeqq", X86pcmpeq, v4i64, VR256,5983 load, i256mem, SchedWriteVecALU.YMM, 0>,5984 VEX, VVVV, VEX_L, WIG;5985 5986let Constraints = "$src1 = $dst" in {5987 defm PMULLD : SS48I_binop_rm<0x40, "pmulld", mul, v4i32, VR128,5988 memop, i128mem, SchedWritePMULLD.XMM, 1>;5989 defm PCMPEQQ : SS48I_binop_rm<0x29, "pcmpeqq", X86pcmpeq, v2i64, VR128,5990 memop, i128mem, SchedWriteVecALU.XMM, 1>;5991}5992 5993/// SS41I_binop_rmi_int - SSE 4.1 binary operator with 8-bit immediate5994multiclass SS41I_binop_rmi_int<bits<8> opc, string OpcodeStr,5995 Intrinsic IntId, RegisterClass RC, PatFrag memop_frag,5996 X86MemOperand x86memop, bit Is2Addr,5997 X86FoldableSchedWrite sched> {5998 let isCommutable = 1 in5999 def rri : SS4AIi8<opc, MRMSrcReg, (outs RC:$dst),6000 (ins RC:$src1, RC:$src2, u8imm:$src3),6001 !if(Is2Addr,6002 !strconcat(OpcodeStr,6003 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),6004 !strconcat(OpcodeStr,6005 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),6006 [(set RC:$dst, (IntId RC:$src1, RC:$src2, timm:$src3))]>,6007 Sched<[sched]>;6008 def rmi : SS4AIi8<opc, MRMSrcMem, (outs RC:$dst),6009 (ins RC:$src1, x86memop:$src2, u8imm:$src3),6010 !if(Is2Addr,6011 !strconcat(OpcodeStr,6012 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),6013 !strconcat(OpcodeStr,6014 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),6015 [(set RC:$dst,6016 (IntId RC:$src1, (memop_frag addr:$src2), timm:$src3))]>,6017 Sched<[sched.Folded, sched.ReadAfterFold]>;6018}6019 6020/// SS41I_binop_rmi - SSE 4.1 binary operator with 8-bit immediate6021multiclass SS41I_binop_rmi<bits<8> opc, string OpcodeStr, SDNode OpNode,6022 ValueType OpVT, RegisterClass RC, PatFrag memop_frag,6023 X86MemOperand x86memop, bit Is2Addr,6024 X86FoldableSchedWrite sched> {6025 let isCommutable = 1 in6026 def rri : SS4AIi8<opc, MRMSrcReg, (outs RC:$dst),6027 (ins RC:$src1, RC:$src2, u8imm:$src3),6028 !if(Is2Addr,6029 !strconcat(OpcodeStr,6030 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),6031 !strconcat(OpcodeStr,6032 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),6033 [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2, timm:$src3)))]>,6034 Sched<[sched]>;6035 def rmi : SS4AIi8<opc, MRMSrcMem, (outs RC:$dst),6036 (ins RC:$src1, x86memop:$src2, u8imm:$src3),6037 !if(Is2Addr,6038 !strconcat(OpcodeStr,6039 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),6040 !strconcat(OpcodeStr,6041 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),6042 [(set RC:$dst,6043 (OpVT (OpNode RC:$src1, (memop_frag addr:$src2), timm:$src3)))]>,6044 Sched<[sched.Folded, sched.ReadAfterFold]>;6045}6046 6047def BlendCommuteImm2 : SDNodeXForm<timm, [{6048 uint8_t Imm = N->getZExtValue() & 0x03;6049 return getI8Imm(Imm ^ 0x03, SDLoc(N));6050}]>;6051 6052def BlendCommuteImm4 : SDNodeXForm<timm, [{6053 uint8_t Imm = N->getZExtValue() & 0x0f;6054 return getI8Imm(Imm ^ 0x0f, SDLoc(N));6055}]>;6056 6057def BlendCommuteImm8 : SDNodeXForm<timm, [{6058 uint8_t Imm = N->getZExtValue() & 0xff;6059 return getI8Imm(Imm ^ 0xff, SDLoc(N));6060}]>;6061 6062// Turn a 4-bit blendi immediate to 8-bit for use with pblendw.6063def BlendScaleImm4 : SDNodeXForm<timm, [{6064 uint8_t Imm = N->getZExtValue();6065 uint8_t NewImm = 0;6066 for (unsigned i = 0; i != 4; ++i) {6067 if (Imm & (1 << i))6068 NewImm |= 0x3 << (i * 2);6069 }6070 return getI8Imm(NewImm, SDLoc(N));6071}]>;6072 6073// Turn a 2-bit blendi immediate to 8-bit for use with pblendw.6074def BlendScaleImm2 : SDNodeXForm<timm, [{6075 uint8_t Imm = N->getZExtValue();6076 uint8_t NewImm = 0;6077 for (unsigned i = 0; i != 2; ++i) {6078 if (Imm & (1 << i))6079 NewImm |= 0xf << (i * 4);6080 }6081 return getI8Imm(NewImm, SDLoc(N));6082}]>;6083 6084// Turn a 2-bit blendi immediate to 4-bit for use with pblendd.6085def BlendScaleImm2to4 : SDNodeXForm<timm, [{6086 uint8_t Imm = N->getZExtValue();6087 uint8_t NewImm = 0;6088 for (unsigned i = 0; i != 2; ++i) {6089 if (Imm & (1 << i))6090 NewImm |= 0x3 << (i * 2);6091 }6092 return getI8Imm(NewImm, SDLoc(N));6093}]>;6094 6095// Turn a 4-bit blendi immediate to 8-bit for use with pblendw and invert it.6096def BlendScaleCommuteImm4 : SDNodeXForm<timm, [{6097 uint8_t Imm = N->getZExtValue();6098 uint8_t NewImm = 0;6099 for (unsigned i = 0; i != 4; ++i) {6100 if (Imm & (1 << i))6101 NewImm |= 0x3 << (i * 2);6102 }6103 return getI8Imm(NewImm ^ 0xff, SDLoc(N));6104}]>;6105 6106// Turn a 2-bit blendi immediate to 8-bit for use with pblendw and invert it.6107def BlendScaleCommuteImm2 : SDNodeXForm<timm, [{6108 uint8_t Imm = N->getZExtValue();6109 uint8_t NewImm = 0;6110 for (unsigned i = 0; i != 2; ++i) {6111 if (Imm & (1 << i))6112 NewImm |= 0xf << (i * 4);6113 }6114 return getI8Imm(NewImm ^ 0xff, SDLoc(N));6115}]>;6116 6117// Turn a 2-bit blendi immediate to 4-bit for use with pblendd and invert it.6118def BlendScaleCommuteImm2to4 : SDNodeXForm<timm, [{6119 uint8_t Imm = N->getZExtValue();6120 uint8_t NewImm = 0;6121 for (unsigned i = 0; i != 2; ++i) {6122 if (Imm & (1 << i))6123 NewImm |= 0x3 << (i * 2);6124 }6125 return getI8Imm(NewImm ^ 0xf, SDLoc(N));6126}]>;6127 6128let Predicates = [HasAVX, NoAVX10_2] in {6129 let isCommutable = 0 in {6130 defm VMPSADBW : SS41I_binop_rmi<0x42, "vmpsadbw", X86Vmpsadbw,6131 v8i16, VR128, load, i128mem, 0,6132 SchedWriteMPSAD.XMM>, VEX, VVVV, WIG;6133 }6134}6135 6136let Predicates = [HasAVX], Uses = [MXCSR], mayRaiseFPException = 1 in {6137 let ExeDomain = SSEPackedSingle in6138 defm VDPPS : SS41I_binop_rmi_int<0x40, "vdpps", int_x86_sse41_dpps,6139 VR128, load, f128mem, 0,6140 SchedWriteDPPS.XMM>, VEX, VVVV, WIG;6141 let ExeDomain = SSEPackedDouble in6142 defm VDPPD : SS41I_binop_rmi_int<0x41, "vdppd", int_x86_sse41_dppd,6143 VR128, load, f128mem, 0,6144 SchedWriteDPPD.XMM>, VEX, VVVV, WIG;6145 let ExeDomain = SSEPackedSingle in6146 defm VDPPSY : SS41I_binop_rmi_int<0x40, "vdpps", int_x86_avx_dp_ps_256,6147 VR256, load, i256mem, 0,6148 SchedWriteDPPS.YMM>, VEX, VVVV, VEX_L, WIG;6149}6150 6151let Predicates = [HasAVX2, NoAVX10_2] in {6152 let isCommutable = 0 in {6153 defm VMPSADBWY : SS41I_binop_rmi<0x42, "vmpsadbw", X86Vmpsadbw,6154 v16i16, VR256, load, i256mem, 0,6155 SchedWriteMPSAD.YMM>, VEX, VVVV, VEX_L, WIG;6156 }6157}6158 6159let Constraints = "$src1 = $dst" in {6160 let isCommutable = 0 in {6161 defm MPSADBW : SS41I_binop_rmi<0x42, "mpsadbw", X86Vmpsadbw,6162 v8i16, VR128, memop, i128mem, 1,6163 SchedWriteMPSAD.XMM>;6164 }6165 6166 let ExeDomain = SSEPackedSingle in6167 defm DPPS : SS41I_binop_rmi_int<0x40, "dpps", int_x86_sse41_dpps,6168 VR128, memop, f128mem, 1,6169 SchedWriteDPPS.XMM>, SIMD_EXC;6170 let ExeDomain = SSEPackedDouble in6171 defm DPPD : SS41I_binop_rmi_int<0x41, "dppd", int_x86_sse41_dppd,6172 VR128, memop, f128mem, 1,6173 SchedWriteDPPD.XMM>, SIMD_EXC;6174}6175 6176/// SS41I_blend_rmi - SSE 4.1 blend with 8-bit immediate6177multiclass SS41I_blend_rmi<bits<8> opc, string OpcodeStr, SDNode OpNode,6178 ValueType OpVT, RegisterClass RC, PatFrag memop_frag,6179 X86MemOperand x86memop, bit Is2Addr, Domain d,6180 X86FoldableSchedWrite sched, SDNodeXForm commuteXForm> {6181let ExeDomain = d, Constraints = !if(Is2Addr, "$src1 = $dst", "") in {6182 let isCommutable = 1 in6183 def rri : SS4AIi8<opc, MRMSrcReg, (outs RC:$dst),6184 (ins RC:$src1, RC:$src2, u8imm:$src3),6185 !if(Is2Addr,6186 !strconcat(OpcodeStr,6187 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),6188 !strconcat(OpcodeStr,6189 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),6190 [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2, timm:$src3)))]>,6191 Sched<[sched]>;6192 def rmi : SS4AIi8<opc, MRMSrcMem, (outs RC:$dst),6193 (ins RC:$src1, x86memop:$src2, u8imm:$src3),6194 !if(Is2Addr,6195 !strconcat(OpcodeStr,6196 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),6197 !strconcat(OpcodeStr,6198 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),6199 [(set RC:$dst,6200 (OpVT (OpNode RC:$src1, (memop_frag addr:$src2), timm:$src3)))]>,6201 Sched<[sched.Folded, sched.ReadAfterFold]>;6202}6203 6204 // Pattern to commute if load is in first source.6205 def : Pat<(OpVT (OpNode (memop_frag addr:$src2), RC:$src1, timm:$src3)),6206 (!cast<Instruction>(NAME#"rmi") RC:$src1, addr:$src2,6207 (commuteXForm timm:$src3))>;6208}6209 6210let Predicates = [HasAVX] in {6211 defm VBLENDPS : SS41I_blend_rmi<0x0C, "vblendps", X86Blendi, v4f32,6212 VR128, load, f128mem, 0, SSEPackedSingle,6213 SchedWriteFBlend.XMM, BlendCommuteImm4>,6214 VEX, VVVV, WIG;6215 defm VBLENDPSY : SS41I_blend_rmi<0x0C, "vblendps", X86Blendi, v8f32,6216 VR256, load, f256mem, 0, SSEPackedSingle,6217 SchedWriteFBlend.YMM, BlendCommuteImm8>,6218 VEX, VVVV, VEX_L, WIG;6219 defm VBLENDPD : SS41I_blend_rmi<0x0D, "vblendpd", X86Blendi, v2f64,6220 VR128, load, f128mem, 0, SSEPackedDouble,6221 SchedWriteFBlend.XMM, BlendCommuteImm2>,6222 VEX, VVVV, WIG;6223 defm VBLENDPDY : SS41I_blend_rmi<0x0D, "vblendpd", X86Blendi, v4f64,6224 VR256, load, f256mem, 0, SSEPackedDouble,6225 SchedWriteFBlend.YMM, BlendCommuteImm4>,6226 VEX, VVVV, VEX_L, WIG;6227 defm VPBLENDW : SS41I_blend_rmi<0x0E, "vpblendw", X86Blendi, v8i16,6228 VR128, load, i128mem, 0, SSEPackedInt,6229 SchedWriteBlend.XMM, BlendCommuteImm8>,6230 VEX, VVVV, WIG;6231}6232 6233let Predicates = [HasAVX2] in {6234 defm VPBLENDWY : SS41I_blend_rmi<0x0E, "vpblendw", X86Blendi, v16i16,6235 VR256, load, i256mem, 0, SSEPackedInt,6236 SchedWriteBlend.YMM, BlendCommuteImm8>,6237 VEX, VVVV, VEX_L, WIG;6238}6239 6240// Emulate vXi32/vXi64 blends with vXf32/vXf64 or pblendw.6241// ExecutionDomainFixPass will cleanup domains later on.6242let Predicates = [HasAVX1Only] in {6243def : Pat<(X86Blendi (v4i64 VR256:$src1), (v4i64 VR256:$src2), timm:$src3),6244 (VBLENDPDYrri VR256:$src1, VR256:$src2, timm:$src3)>;6245def : Pat<(X86Blendi VR256:$src1, (loadv4i64 addr:$src2), timm:$src3),6246 (VBLENDPDYrmi VR256:$src1, addr:$src2, timm:$src3)>;6247def : Pat<(X86Blendi (loadv4i64 addr:$src2), VR256:$src1, timm:$src3),6248 (VBLENDPDYrmi VR256:$src1, addr:$src2, (BlendCommuteImm4 timm:$src3))>;6249 6250// Use pblendw for 128-bit integer to keep it in the integer domain and prevent6251// it from becoming movsd via commuting under optsize.6252def : Pat<(X86Blendi (v2i64 VR128:$src1), (v2i64 VR128:$src2), timm:$src3),6253 (VPBLENDWrri VR128:$src1, VR128:$src2, (BlendScaleImm2 timm:$src3))>;6254def : Pat<(X86Blendi VR128:$src1, (loadv2i64 addr:$src2), timm:$src3),6255 (VPBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleImm2 timm:$src3))>;6256def : Pat<(X86Blendi (loadv2i64 addr:$src2), VR128:$src1, timm:$src3),6257 (VPBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleCommuteImm2 timm:$src3))>;6258 6259def : Pat<(X86Blendi (v8i32 VR256:$src1), (v8i32 VR256:$src2), timm:$src3),6260 (VBLENDPSYrri VR256:$src1, VR256:$src2, timm:$src3)>;6261def : Pat<(X86Blendi VR256:$src1, (loadv8i32 addr:$src2), timm:$src3),6262 (VBLENDPSYrmi VR256:$src1, addr:$src2, timm:$src3)>;6263def : Pat<(X86Blendi (loadv8i32 addr:$src2), VR256:$src1, timm:$src3),6264 (VBLENDPSYrmi VR256:$src1, addr:$src2, (BlendCommuteImm8 timm:$src3))>;6265 6266// Use pblendw for 128-bit integer to keep it in the integer domain and prevent6267// it from becoming movss via commuting under optsize.6268def : Pat<(X86Blendi (v4i32 VR128:$src1), (v4i32 VR128:$src2), timm:$src3),6269 (VPBLENDWrri VR128:$src1, VR128:$src2, (BlendScaleImm4 timm:$src3))>;6270def : Pat<(X86Blendi VR128:$src1, (loadv4i32 addr:$src2), timm:$src3),6271 (VPBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleImm4 timm:$src3))>;6272def : Pat<(X86Blendi (loadv4i32 addr:$src2), VR128:$src1, timm:$src3),6273 (VPBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleCommuteImm4 timm:$src3))>;6274}6275 6276defm BLENDPS : SS41I_blend_rmi<0x0C, "blendps", X86Blendi, v4f32,6277 VR128, memop, f128mem, 1, SSEPackedSingle,6278 SchedWriteFBlend.XMM, BlendCommuteImm4>;6279defm BLENDPD : SS41I_blend_rmi<0x0D, "blendpd", X86Blendi, v2f64,6280 VR128, memop, f128mem, 1, SSEPackedDouble,6281 SchedWriteFBlend.XMM, BlendCommuteImm2>;6282defm PBLENDW : SS41I_blend_rmi<0x0E, "pblendw", X86Blendi, v8i16,6283 VR128, memop, i128mem, 1, SSEPackedInt,6284 SchedWriteBlend.XMM, BlendCommuteImm8>;6285 6286let Predicates = [UseSSE41] in {6287// Use pblendw for 128-bit integer to keep it in the integer domain and prevent6288// it from becoming movss via commuting under optsize.6289def : Pat<(X86Blendi (v2i64 VR128:$src1), (v2i64 VR128:$src2), timm:$src3),6290 (PBLENDWrri VR128:$src1, VR128:$src2, (BlendScaleImm2 timm:$src3))>;6291def : Pat<(X86Blendi VR128:$src1, (memopv2i64 addr:$src2), timm:$src3),6292 (PBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleImm2 timm:$src3))>;6293def : Pat<(X86Blendi (memopv2i64 addr:$src2), VR128:$src1, timm:$src3),6294 (PBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleCommuteImm2 timm:$src3))>;6295 6296def : Pat<(X86Blendi (v4i32 VR128:$src1), (v4i32 VR128:$src2), timm:$src3),6297 (PBLENDWrri VR128:$src1, VR128:$src2, (BlendScaleImm4 timm:$src3))>;6298def : Pat<(X86Blendi VR128:$src1, (memopv4i32 addr:$src2), timm:$src3),6299 (PBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleImm4 timm:$src3))>;6300def : Pat<(X86Blendi (memopv4i32 addr:$src2), VR128:$src1, timm:$src3),6301 (PBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleCommuteImm4 timm:$src3))>;6302}6303 6304// For insertion into the zero index (low half) of a 256-bit vector, it is6305// more efficient to generate a blend with immediate instead of an insert*128.6306let Predicates = [HasAVX] in {6307def : Pat<(insert_subvector (v4f64 VR256:$src1), (v2f64 VR128:$src2), (iPTR 0)),6308 (VBLENDPDYrri VR256:$src1,6309 (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)),6310 VR128:$src2, sub_xmm), 0x3)>;6311def : Pat<(insert_subvector (v8f32 VR256:$src1), (v4f32 VR128:$src2), (iPTR 0)),6312 (VBLENDPSYrri VR256:$src1,6313 (INSERT_SUBREG (v8f32 (IMPLICIT_DEF)),6314 VR128:$src2, sub_xmm), 0xf)>;6315 6316def : Pat<(insert_subvector (loadv4f64 addr:$src2), (v2f64 VR128:$src1), (iPTR 0)),6317 (VBLENDPDYrmi (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)),6318 VR128:$src1, sub_xmm), addr:$src2, 0xc)>;6319def : Pat<(insert_subvector (loadv8f32 addr:$src2), (v4f32 VR128:$src1), (iPTR 0)),6320 (VBLENDPSYrmi (INSERT_SUBREG (v8f32 (IMPLICIT_DEF)),6321 VR128:$src1, sub_xmm), addr:$src2, 0xf0)>;6322}6323 6324/// SS41I_quaternary_vx - AVX SSE 4.1 with 4 operators6325multiclass SS41I_quaternary_avx<bits<8> opc, string OpcodeStr, RegisterClass RC,6326 X86MemOperand x86memop, ValueType VT,6327 PatFrag mem_frag, SDNode OpNode,6328 X86FoldableSchedWrite sched> {6329 def rrr : Ii8Reg<opc, MRMSrcReg, (outs RC:$dst),6330 (ins RC:$src1, RC:$src2, RC:$src3),6331 !strconcat(OpcodeStr,6332 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),6333 [(set RC:$dst, (VT (OpNode RC:$src3, RC:$src2, RC:$src1)))],6334 SSEPackedInt>, TA, PD, VEX, VVVV,6335 Sched<[sched]>;6336 6337 def rmr : Ii8Reg<opc, MRMSrcMem, (outs RC:$dst),6338 (ins RC:$src1, x86memop:$src2, RC:$src3),6339 !strconcat(OpcodeStr,6340 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),6341 [(set RC:$dst,6342 (OpNode RC:$src3, (mem_frag addr:$src2),6343 RC:$src1))], SSEPackedInt>, TA, PD, VEX, VVVV,6344 Sched<[sched.Folded, sched.ReadAfterFold,6345 // x86memop:$src26346 ReadDefault, ReadDefault, ReadDefault, ReadDefault,6347 ReadDefault,6348 // RC::$src36349 sched.ReadAfterFold]>;6350}6351 6352let Predicates = [HasAVX] in {6353let ExeDomain = SSEPackedDouble in {6354defm VBLENDVPD : SS41I_quaternary_avx<0x4B, "vblendvpd", VR128, f128mem,6355 v2f64, loadv2f64, X86Blendv,6356 SchedWriteFVarBlend.XMM>;6357defm VBLENDVPDY : SS41I_quaternary_avx<0x4B, "vblendvpd", VR256, f256mem,6358 v4f64, loadv4f64, X86Blendv,6359 SchedWriteFVarBlend.YMM>, VEX_L;6360} // ExeDomain = SSEPackedDouble6361let ExeDomain = SSEPackedSingle in {6362defm VBLENDVPS : SS41I_quaternary_avx<0x4A, "vblendvps", VR128, f128mem,6363 v4f32, loadv4f32, X86Blendv,6364 SchedWriteFVarBlend.XMM>;6365defm VBLENDVPSY : SS41I_quaternary_avx<0x4A, "vblendvps", VR256, f256mem,6366 v8f32, loadv8f32, X86Blendv,6367 SchedWriteFVarBlend.YMM>, VEX_L;6368} // ExeDomain = SSEPackedSingle6369defm VPBLENDVB : SS41I_quaternary_avx<0x4C, "vpblendvb", VR128, i128mem,6370 v16i8, loadv16i8, X86Blendv,6371 SchedWriteVarBlend.XMM>;6372}6373 6374let Predicates = [HasAVX2] in {6375defm VPBLENDVBY : SS41I_quaternary_avx<0x4C, "vpblendvb", VR256, i256mem,6376 v32i8, loadv32i8, X86Blendv,6377 SchedWriteVarBlend.YMM>, VEX_L;6378}6379 6380let Predicates = [HasAVX] in {6381 def : Pat<(v4i32 (X86Blendv (v4i32 VR128:$mask), (v4i32 VR128:$src1),6382 (v4i32 VR128:$src2))),6383 (VBLENDVPSrrr VR128:$src2, VR128:$src1, VR128:$mask)>;6384 def : Pat<(v2i64 (X86Blendv (v2i64 VR128:$mask), (v2i64 VR128:$src1),6385 (v2i64 VR128:$src2))),6386 (VBLENDVPDrrr VR128:$src2, VR128:$src1, VR128:$mask)>;6387 def : Pat<(v8i32 (X86Blendv (v8i32 VR256:$mask), (v8i32 VR256:$src1),6388 (v8i32 VR256:$src2))),6389 (VBLENDVPSYrrr VR256:$src2, VR256:$src1, VR256:$mask)>;6390 def : Pat<(v4i64 (X86Blendv (v4i64 VR256:$mask), (v4i64 VR256:$src1),6391 (v4i64 VR256:$src2))),6392 (VBLENDVPDYrrr VR256:$src2, VR256:$src1, VR256:$mask)>;6393}6394 6395// Prefer a movss or movsd over a blendps when optimizing for size. these were6396// changed to use blends because blends have better throughput on sandybridge6397// and haswell, but movs[s/d] are 1-2 byte shorter instructions.6398let Predicates = [HasAVX, OptForSpeed] in {6399 def : Pat<(v4f32 (X86vzmovl (v4f32 VR128:$src))),6400 (VBLENDPSrri (v4f32 (V_SET0)), VR128:$src, (i8 1))>;6401 def : Pat<(v4i32 (X86vzmovl (v4i32 VR128:$src))),6402 (VPBLENDWrri (v4i32 (V_SET0)), VR128:$src, (i8 3))>;6403 6404 def : Pat<(v4f32 (X86Movss VR128:$src1, VR128:$src2)),6405 (VBLENDPSrri VR128:$src1, VR128:$src2, (i8 1))>;6406 def : Pat<(v4f32 (X86Movss VR128:$src1, (loadv4f32 addr:$src2))),6407 (VBLENDPSrmi VR128:$src1, addr:$src2, (i8 1))>;6408 def : Pat<(v4f32 (X86Movss (loadv4f32 addr:$src2), VR128:$src1)),6409 (VBLENDPSrmi VR128:$src1, addr:$src2, (i8 0xe))>;6410 6411 def : Pat<(v2f64 (X86Movsd VR128:$src1, VR128:$src2)),6412 (VBLENDPDrri VR128:$src1, VR128:$src2, (i8 1))>;6413 def : Pat<(v2f64 (X86Movsd VR128:$src1, (loadv2f64 addr:$src2))),6414 (VBLENDPDrmi VR128:$src1, addr:$src2, (i8 1))>;6415 def : Pat<(v2f64 (X86Movsd (loadv2f64 addr:$src2), VR128:$src1)),6416 (VBLENDPDrmi VR128:$src1, addr:$src2, (i8 2))>;6417 6418 // Move low f32 and clear high bits.6419 def : Pat<(v8f32 (X86vzmovl (v8f32 VR256:$src))),6420 (SUBREG_TO_REG (i32 0),6421 (v4f32 (VBLENDPSrri (v4f32 (V_SET0)),6422 (v4f32 (EXTRACT_SUBREG (v8f32 VR256:$src), sub_xmm)),6423 (i8 1))), sub_xmm)>;6424 def : Pat<(v8i32 (X86vzmovl (v8i32 VR256:$src))),6425 (SUBREG_TO_REG (i32 0),6426 (v4i32 (VPBLENDWrri (v4i32 (V_SET0)),6427 (v4i32 (EXTRACT_SUBREG (v8i32 VR256:$src), sub_xmm)),6428 (i8 3))), sub_xmm)>;6429}6430 6431// Prefer a movss or movsd over a blendps when optimizing for size. these were6432// changed to use blends because blends have better throughput on sandybridge6433// and haswell, but movs[s/d] are 1-2 byte shorter instructions.6434let Predicates = [UseSSE41, OptForSpeed] in {6435 // With SSE41 we can use blends for these patterns.6436 def : Pat<(v4f32 (X86vzmovl (v4f32 VR128:$src))),6437 (BLENDPSrri (v4f32 (V_SET0)), VR128:$src, (i8 1))>;6438 def : Pat<(v4i32 (X86vzmovl (v4i32 VR128:$src))),6439 (PBLENDWrri (v4i32 (V_SET0)), VR128:$src, (i8 3))>;6440 6441 def : Pat<(v4f32 (X86Movss VR128:$src1, VR128:$src2)),6442 (BLENDPSrri VR128:$src1, VR128:$src2, (i8 1))>;6443 def : Pat<(v4f32 (X86Movss VR128:$src1, (memopv4f32 addr:$src2))),6444 (BLENDPSrmi VR128:$src1, addr:$src2, (i8 1))>;6445 def : Pat<(v4f32 (X86Movss (memopv4f32 addr:$src2), VR128:$src1)),6446 (BLENDPSrmi VR128:$src1, addr:$src2, (i8 0xe))>;6447 6448 def : Pat<(v2f64 (X86Movsd VR128:$src1, VR128:$src2)),6449 (BLENDPDrri VR128:$src1, VR128:$src2, (i8 1))>;6450 def : Pat<(v2f64 (X86Movsd VR128:$src1, (memopv2f64 addr:$src2))),6451 (BLENDPDrmi VR128:$src1, addr:$src2, (i8 1))>;6452 def : Pat<(v2f64 (X86Movsd (memopv2f64 addr:$src2), VR128:$src1)),6453 (BLENDPDrmi VR128:$src1, addr:$src2, (i8 2))>;6454}6455 6456 6457/// SS41I_ternary - SSE 4.1 ternary operator6458let Uses = [XMM0], Constraints = "$src1 = $dst" in {6459 multiclass SS41I_ternary<bits<8> opc, string OpcodeStr, ValueType VT,6460 PatFrag mem_frag, X86MemOperand x86memop,6461 SDNode OpNode, X86FoldableSchedWrite sched> {6462 def rr0 : SS48I<opc, MRMSrcReg, (outs VR128:$dst),6463 (ins VR128:$src1, VR128:$src2),6464 !strconcat(OpcodeStr,6465 "\t{%xmm0, $src2, $dst|$dst, $src2, xmm0}"),6466 [(set VR128:$dst,6467 (VT (OpNode XMM0, VR128:$src2, VR128:$src1)))]>,6468 Sched<[sched]>;6469 6470 def rm0 : SS48I<opc, MRMSrcMem, (outs VR128:$dst),6471 (ins VR128:$src1, x86memop:$src2),6472 !strconcat(OpcodeStr,6473 "\t{%xmm0, $src2, $dst|$dst, $src2, xmm0}"),6474 [(set VR128:$dst,6475 (OpNode XMM0, (mem_frag addr:$src2), VR128:$src1))]>,6476 Sched<[sched.Folded, sched.ReadAfterFold]>;6477 }6478}6479 6480let ExeDomain = SSEPackedDouble in6481defm BLENDVPD : SS41I_ternary<0x15, "blendvpd", v2f64, memopv2f64, f128mem,6482 X86Blendv, SchedWriteFVarBlend.XMM>;6483let ExeDomain = SSEPackedSingle in6484defm BLENDVPS : SS41I_ternary<0x14, "blendvps", v4f32, memopv4f32, f128mem,6485 X86Blendv, SchedWriteFVarBlend.XMM>;6486defm PBLENDVB : SS41I_ternary<0x10, "pblendvb", v16i8, memopv16i8, i128mem,6487 X86Blendv, SchedWriteVarBlend.XMM>;6488 6489// Aliases with the implicit xmm0 argument6490def : InstAlias<"blendvpd\t{$src2, $dst|$dst, $src2}",6491 (BLENDVPDrr0 VR128:$dst, VR128:$src2), 0>;6492def : InstAlias<"blendvpd\t{$src2, $dst|$dst, $src2}",6493 (BLENDVPDrm0 VR128:$dst, f128mem:$src2), 0>;6494def : InstAlias<"blendvps\t{$src2, $dst|$dst, $src2}",6495 (BLENDVPSrr0 VR128:$dst, VR128:$src2), 0>;6496def : InstAlias<"blendvps\t{$src2, $dst|$dst, $src2}",6497 (BLENDVPSrm0 VR128:$dst, f128mem:$src2), 0>;6498def : InstAlias<"pblendvb\t{$src2, $dst|$dst, $src2}",6499 (PBLENDVBrr0 VR128:$dst, VR128:$src2), 0>;6500def : InstAlias<"pblendvb\t{$src2, $dst|$dst, $src2}",6501 (PBLENDVBrm0 VR128:$dst, i128mem:$src2), 0>;6502 6503let Predicates = [UseSSE41] in {6504 def : Pat<(v4i32 (X86Blendv (v4i32 XMM0), (v4i32 VR128:$src1),6505 (v4i32 VR128:$src2))),6506 (BLENDVPSrr0 VR128:$src2, VR128:$src1)>;6507 def : Pat<(v2i64 (X86Blendv (v2i64 XMM0), (v2i64 VR128:$src1),6508 (v2i64 VR128:$src2))),6509 (BLENDVPDrr0 VR128:$src2, VR128:$src1)>;6510}6511 6512let AddedComplexity = 400 in { // Prefer non-temporal versions6513 6514let Predicates = [HasAVX, NoVLX] in6515def VMOVNTDQArm : SS48I<0x2A, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src),6516 "vmovntdqa\t{$src, $dst|$dst, $src}", []>,6517 Sched<[SchedWriteVecMoveLSNT.XMM.RM]>, VEX, WIG;6518let Predicates = [HasAVX2, NoVLX] in6519def VMOVNTDQAYrm : SS48I<0x2A, MRMSrcMem, (outs VR256:$dst), (ins i256mem:$src),6520 "vmovntdqa\t{$src, $dst|$dst, $src}", []>,6521 Sched<[SchedWriteVecMoveLSNT.YMM.RM]>, VEX, VEX_L, WIG;6522def MOVNTDQArm : SS48I<0x2A, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src),6523 "movntdqa\t{$src, $dst|$dst, $src}", []>,6524 Sched<[SchedWriteVecMoveLSNT.XMM.RM]>;6525 6526let Predicates = [HasAVX2, NoVLX] in {6527 def : Pat<(v8f32 (alignednontemporalload addr:$src)),6528 (VMOVNTDQAYrm addr:$src)>;6529 def : Pat<(v4f64 (alignednontemporalload addr:$src)),6530 (VMOVNTDQAYrm addr:$src)>;6531 def : Pat<(v4i64 (alignednontemporalload addr:$src)),6532 (VMOVNTDQAYrm addr:$src)>;6533 def : Pat<(v8i32 (alignednontemporalload addr:$src)),6534 (VMOVNTDQAYrm addr:$src)>;6535 def : Pat<(v16i16 (alignednontemporalload addr:$src)),6536 (VMOVNTDQAYrm addr:$src)>;6537 def : Pat<(v16f16 (alignednontemporalload addr:$src)),6538 (VMOVNTDQAYrm addr:$src)>;6539 def : Pat<(v32i8 (alignednontemporalload addr:$src)),6540 (VMOVNTDQAYrm addr:$src)>;6541}6542 6543let Predicates = [HasAVX, NoVLX] in {6544 def : Pat<(v4f32 (alignednontemporalload addr:$src)),6545 (VMOVNTDQArm addr:$src)>;6546 def : Pat<(v2f64 (alignednontemporalload addr:$src)),6547 (VMOVNTDQArm addr:$src)>;6548 def : Pat<(v2i64 (alignednontemporalload addr:$src)),6549 (VMOVNTDQArm addr:$src)>;6550 def : Pat<(v4i32 (alignednontemporalload addr:$src)),6551 (VMOVNTDQArm addr:$src)>;6552 def : Pat<(v8i16 (alignednontemporalload addr:$src)),6553 (VMOVNTDQArm addr:$src)>;6554 def : Pat<(v8f16 (alignednontemporalload addr:$src)),6555 (VMOVNTDQArm addr:$src)>;6556 def : Pat<(v16i8 (alignednontemporalload addr:$src)),6557 (VMOVNTDQArm addr:$src)>;6558}6559 6560let Predicates = [UseSSE41] in {6561 def : Pat<(v4f32 (alignednontemporalload addr:$src)),6562 (MOVNTDQArm addr:$src)>;6563 def : Pat<(v2f64 (alignednontemporalload addr:$src)),6564 (MOVNTDQArm addr:$src)>;6565 def : Pat<(v2i64 (alignednontemporalload addr:$src)),6566 (MOVNTDQArm addr:$src)>;6567 def : Pat<(v4i32 (alignednontemporalload addr:$src)),6568 (MOVNTDQArm addr:$src)>;6569 def : Pat<(v8i16 (alignednontemporalload addr:$src)),6570 (MOVNTDQArm addr:$src)>;6571 def : Pat<(v8f16 (alignednontemporalload addr:$src)),6572 (MOVNTDQArm addr:$src)>;6573 def : Pat<(v16i8 (alignednontemporalload addr:$src)),6574 (MOVNTDQArm addr:$src)>;6575}6576 6577} // AddedComplexity6578 6579//===----------------------------------------------------------------------===//6580// SSE4.2 - Compare Instructions6581//===----------------------------------------------------------------------===//6582 6583/// SS42I_binop_rm - Simple SSE 4.2 binary operator6584multiclass SS42I_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,6585 ValueType OpVT, RegisterClass RC, PatFrag memop_frag,6586 X86MemOperand x86memop, X86FoldableSchedWrite sched,6587 bit Is2Addr = 1> {6588 def rr : SS428I<opc, MRMSrcReg, (outs RC:$dst),6589 (ins RC:$src1, RC:$src2),6590 !if(Is2Addr,6591 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),6592 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),6593 [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2)))]>,6594 Sched<[sched]>;6595 def rm : SS428I<opc, MRMSrcMem, (outs RC:$dst),6596 (ins RC:$src1, x86memop:$src2),6597 !if(Is2Addr,6598 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),6599 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),6600 [(set RC:$dst,6601 (OpVT (OpNode RC:$src1, (memop_frag addr:$src2))))]>,6602 Sched<[sched.Folded, sched.ReadAfterFold]>;6603}6604 6605let Predicates = [HasAVX] in6606 defm VPCMPGTQ : SS42I_binop_rm<0x37, "vpcmpgtq", X86pcmpgt, v2i64, VR128,6607 load, i128mem, SchedWriteVecALU.XMM, 0>,6608 VEX, VVVV, WIG;6609 6610let Predicates = [HasAVX2] in6611 defm VPCMPGTQY : SS42I_binop_rm<0x37, "vpcmpgtq", X86pcmpgt, v4i64, VR256,6612 load, i256mem, SchedWriteVecALU.YMM, 0>,6613 VEX, VVVV, VEX_L, WIG;6614 6615let Constraints = "$src1 = $dst" in6616 defm PCMPGTQ : SS42I_binop_rm<0x37, "pcmpgtq", X86pcmpgt, v2i64, VR128,6617 memop, i128mem, SchedWriteVecALU.XMM>;6618 6619//===----------------------------------------------------------------------===//6620// SSE4.2 - String/text Processing Instructions6621//===----------------------------------------------------------------------===//6622 6623multiclass pcmpistrm_SS42AI<string asm> {6624 def rri : SS42AI<0x62, MRMSrcReg, (outs),6625 (ins VR128:$src1, VR128:$src2, u8imm:$src3),6626 !strconcat(asm, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"),6627 []>, Sched<[WritePCmpIStrM]>;6628 let mayLoad = 1 in6629 def rmi :SS42AI<0x62, MRMSrcMem, (outs),6630 (ins VR128:$src1, i128mem:$src2, u8imm:$src3),6631 !strconcat(asm, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"),6632 []>, Sched<[WritePCmpIStrM.Folded, WritePCmpIStrM.ReadAfterFold]>;6633}6634 6635let Defs = [XMM0, EFLAGS], hasSideEffects = 0 in {6636 let Predicates = [HasAVX] in6637 defm VPCMPISTRM : pcmpistrm_SS42AI<"vpcmpistrm">, VEX, WIG;6638 defm PCMPISTRM : pcmpistrm_SS42AI<"pcmpistrm"> ;6639}6640 6641multiclass SS42AI_pcmpestrm<string asm> {6642 def rri : SS42AI<0x60, MRMSrcReg, (outs),6643 (ins VR128:$src1, VR128:$src3, u8imm:$src5),6644 !strconcat(asm, "\t{$src5, $src3, $src1|$src1, $src3, $src5}"),6645 []>, Sched<[WritePCmpEStrM]>;6646 let mayLoad = 1 in6647 def rmi : SS42AI<0x60, MRMSrcMem, (outs),6648 (ins VR128:$src1, i128mem:$src3, u8imm:$src5),6649 !strconcat(asm, "\t{$src5, $src3, $src1|$src1, $src3, $src5}"),6650 []>, Sched<[WritePCmpEStrM.Folded, WritePCmpEStrM.ReadAfterFold]>;6651}6652 6653let Defs = [XMM0, EFLAGS], Uses = [EAX, EDX], hasSideEffects = 0 in {6654 let Predicates = [HasAVX] in6655 defm VPCMPESTRM : SS42AI_pcmpestrm<"vpcmpestrm">, VEX, WIG;6656 defm PCMPESTRM : SS42AI_pcmpestrm<"pcmpestrm">;6657}6658 6659multiclass SS42AI_pcmpistri<string asm> {6660 def rri : SS42AI<0x63, MRMSrcReg, (outs),6661 (ins VR128:$src1, VR128:$src2, u8imm:$src3),6662 !strconcat(asm, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"),6663 []>, Sched<[WritePCmpIStrI]>;6664 let mayLoad = 1 in6665 def rmi : SS42AI<0x63, MRMSrcMem, (outs),6666 (ins VR128:$src1, i128mem:$src2, u8imm:$src3),6667 !strconcat(asm, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"),6668 []>, Sched<[WritePCmpIStrI.Folded, WritePCmpIStrI.ReadAfterFold]>;6669}6670 6671let Defs = [ECX, EFLAGS], hasSideEffects = 0 in {6672 let Predicates = [HasAVX] in6673 defm VPCMPISTRI : SS42AI_pcmpistri<"vpcmpistri">, VEX, WIG;6674 defm PCMPISTRI : SS42AI_pcmpistri<"pcmpistri">;6675}6676 6677multiclass SS42AI_pcmpestri<string asm> {6678 def rri : SS42AI<0x61, MRMSrcReg, (outs),6679 (ins VR128:$src1, VR128:$src3, u8imm:$src5),6680 !strconcat(asm, "\t{$src5, $src3, $src1|$src1, $src3, $src5}"),6681 []>, Sched<[WritePCmpEStrI]>;6682 let mayLoad = 1 in6683 def rmi : SS42AI<0x61, MRMSrcMem, (outs),6684 (ins VR128:$src1, i128mem:$src3, u8imm:$src5),6685 !strconcat(asm, "\t{$src5, $src3, $src1|$src1, $src3, $src5}"),6686 []>, Sched<[WritePCmpEStrI.Folded, WritePCmpEStrI.ReadAfterFold]>;6687}6688 6689let Defs = [ECX, EFLAGS], Uses = [EAX, EDX], hasSideEffects = 0 in {6690 let Predicates = [HasAVX] in6691 defm VPCMPESTRI : SS42AI_pcmpestri<"vpcmpestri">, VEX, WIG;6692 defm PCMPESTRI : SS42AI_pcmpestri<"pcmpestri">;6693}6694 6695//===----------------------------------------------------------------------===//6696// SSE4.2 - CRC Instructions6697//===----------------------------------------------------------------------===//6698 6699// NOTE: 'HasCRC32' is used as CRC32 instructions are GPR only and not directly6700// controlled by the SSE42 flag.6701//6702// No CRC instructions have AVX equivalents6703 6704class Crc32r<X86TypeInfo t, RegisterClass rc, SDPatternOperator node>6705 : ITy<0xF1, MRMSrcReg, t, (outs rc:$dst), (ins rc:$src1, t.RegClass:$src2),6706 "crc32", binop_args, [(set rc:$dst, (node rc:$src1, t.RegClass:$src2))]>,6707 Sched<[WriteCRC32]> {6708 let Constraints = "$src1 = $dst";6709}6710 6711class Crc32m<X86TypeInfo t, RegisterClass rc, SDPatternOperator node>6712 : ITy<0xF1, MRMSrcMem, t, (outs rc:$dst), (ins rc:$src1, t.MemOperand:$src2),6713 "crc32", binop_args, [(set rc:$dst, (node rc:$src1, (load addr:$src2)))]>,6714 Sched<[WriteCRC32.Folded, WriteCRC32.ReadAfterFold]> {6715 let Constraints = "$src1 = $dst";6716}6717 6718let Predicates = [HasCRC32, NoEGPR], OpMap = T8, OpPrefix = XD in {6719 def CRC32r32r8 : Crc32r<Xi8, GR32, int_x86_sse42_crc32_32_8>;6720 def CRC32r32m8 : Crc32m<Xi8, GR32, int_x86_sse42_crc32_32_8>;6721 def CRC32r32r16 : Crc32r<Xi16, GR32, int_x86_sse42_crc32_32_16>, OpSize16;6722 def CRC32r32m16 : Crc32m<Xi16, GR32, int_x86_sse42_crc32_32_16>, OpSize16;6723 def CRC32r32r32 : Crc32r<Xi32, GR32, int_x86_sse42_crc32_32_32>, OpSize32;6724 def CRC32r32m32 : Crc32m<Xi32, GR32, int_x86_sse42_crc32_32_32>, OpSize32;6725 def CRC32r64r64 : Crc32r<Xi64, GR64, int_x86_sse42_crc32_64_64>;6726 def CRC32r64m64 : Crc32m<Xi64, GR64, int_x86_sse42_crc32_64_64>;6727 def CRC32r64r8 : Crc32r<Xi8, GR64, null_frag>, REX_W;6728 let mayLoad = 1 in6729 def CRC32r64m8 : Crc32m<Xi8, GR64, null_frag>, REX_W;6730}6731 6732let Predicates = [HasCRC32, HasEGPR, In64BitMode], OpMap = T_MAP4, OpEnc = EncEVEX in {6733 def CRC32r32r8_EVEX : Crc32r<Xi8, GR32, int_x86_sse42_crc32_32_8>;6734 def CRC32r32m8_EVEX : Crc32m<Xi8, GR32, int_x86_sse42_crc32_32_8>;6735 def CRC32r32r16_EVEX : Crc32r<Xi16, GR32, int_x86_sse42_crc32_32_16>, PD;6736 def CRC32r32m16_EVEX : Crc32m<Xi16, GR32, int_x86_sse42_crc32_32_16>, PD;6737 def CRC32r32r32_EVEX : Crc32r<Xi32, GR32, int_x86_sse42_crc32_32_32>;6738 def CRC32r32m32_EVEX : Crc32m<Xi32, GR32, int_x86_sse42_crc32_32_32>;6739 def CRC32r64r64_EVEX : Crc32r<Xi64, GR64, int_x86_sse42_crc32_64_64>;6740 def CRC32r64m64_EVEX : Crc32m<Xi64, GR64, int_x86_sse42_crc32_64_64>;6741 def CRC32r64r8_EVEX : Crc32r<Xi8, GR64, null_frag>, REX_W;6742 let mayLoad = 1 in6743 def CRC32r64m8_EVEX : Crc32m<Xi8, GR64, null_frag>, REX_W;6744}6745 6746//===----------------------------------------------------------------------===//6747// SHA-NI Instructions6748//===----------------------------------------------------------------------===//6749 6750// FIXME: Is there a better scheduler class for SHA than WriteVecIMul?6751multiclass SHAI_binop<bits<8> Opc, string OpcodeStr, Intrinsic IntId,6752 X86FoldableSchedWrite sched, bit UsesXMM0 = 0> {6753 def rr : I<Opc, MRMSrcReg, (outs VR128:$dst),6754 (ins VR128:$src1, VR128:$src2),6755 !if(UsesXMM0,6756 !strconcat(OpcodeStr, "\t{%xmm0, $src2, $dst|$dst, $src2, xmm0}"),6757 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}")),6758 [!if(UsesXMM0,6759 (set VR128:$dst, (IntId VR128:$src1, VR128:$src2, XMM0)),6760 (set VR128:$dst, (IntId VR128:$src1, VR128:$src2)))]>,6761 T8, Sched<[sched]>;6762 6763 def rm : I<Opc, MRMSrcMem, (outs VR128:$dst),6764 (ins VR128:$src1, i128mem:$src2),6765 !if(UsesXMM0,6766 !strconcat(OpcodeStr, "\t{%xmm0, $src2, $dst|$dst, $src2, xmm0}"),6767 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}")),6768 [!if(UsesXMM0,6769 (set VR128:$dst, (IntId VR128:$src1,6770 (memop addr:$src2), XMM0)),6771 (set VR128:$dst, (IntId VR128:$src1,6772 (memop addr:$src2))))]>, T8,6773 Sched<[sched.Folded, sched.ReadAfterFold]>;6774}6775 6776let Constraints = "$src1 = $dst", Predicates = [HasSHA] in {6777 def SHA1RNDS4rri : Ii8<0xCC, MRMSrcReg, (outs VR128:$dst),6778 (ins VR128:$src1, VR128:$src2, u8imm:$src3),6779 "sha1rnds4\t{$src3, $src2, $dst|$dst, $src2, $src3}",6780 [(set VR128:$dst,6781 (int_x86_sha1rnds4 VR128:$src1, VR128:$src2,6782 (i8 timm:$src3)))]>, TA,6783 Sched<[SchedWriteVecIMul.XMM]>;6784 def SHA1RNDS4rmi : Ii8<0xCC, MRMSrcMem, (outs VR128:$dst),6785 (ins VR128:$src1, i128mem:$src2, u8imm:$src3),6786 "sha1rnds4\t{$src3, $src2, $dst|$dst, $src2, $src3}",6787 [(set VR128:$dst,6788 (int_x86_sha1rnds4 VR128:$src1,6789 (memop addr:$src2),6790 (i8 timm:$src3)))]>, TA,6791 Sched<[SchedWriteVecIMul.XMM.Folded,6792 SchedWriteVecIMul.XMM.ReadAfterFold]>;6793 6794 defm SHA1NEXTE : SHAI_binop<0xC8, "sha1nexte", int_x86_sha1nexte,6795 SchedWriteVecIMul.XMM>;6796 defm SHA1MSG1 : SHAI_binop<0xC9, "sha1msg1", int_x86_sha1msg1,6797 SchedWriteVecIMul.XMM>;6798 defm SHA1MSG2 : SHAI_binop<0xCA, "sha1msg2", int_x86_sha1msg2,6799 SchedWriteVecIMul.XMM>;6800 6801 let Uses=[XMM0] in6802 defm SHA256RNDS2 : SHAI_binop<0xCB, "sha256rnds2", int_x86_sha256rnds2,6803 SchedWriteVecIMul.XMM, 1>;6804 6805 defm SHA256MSG1 : SHAI_binop<0xCC, "sha256msg1", int_x86_sha256msg1,6806 SchedWriteVecIMul.XMM>;6807 defm SHA256MSG2 : SHAI_binop<0xCD, "sha256msg2", int_x86_sha256msg2,6808 SchedWriteVecIMul.XMM>;6809}6810 6811 6812//===----------------------------------------------------------------------===//6813// AES-NI Instructions6814//===----------------------------------------------------------------------===//6815 6816multiclass AESI_binop_rm_int<bits<8> opc, string OpcodeStr,6817 Intrinsic IntId, PatFrag ld_frag,6818 bit Is2Addr = 0, RegisterClass RC = VR128,6819 X86MemOperand MemOp = i128mem> {6820 let AsmString = OpcodeStr#6821 !if(Is2Addr, "\t{$src2, $dst|$dst, $src2}",6822 "\t{$src2, $src1, $dst|$dst, $src1, $src2}") in {6823 def rr : AES8I<opc, MRMSrcReg, (outs RC:$dst),6824 (ins RC:$src1, RC:$src2), "",6825 [(set RC:$dst, (IntId RC:$src1, RC:$src2))]>,6826 Sched<[WriteAESDecEnc]>;6827 def rm : AES8I<opc, MRMSrcMem, (outs RC:$dst),6828 (ins RC:$src1, MemOp:$src2), "",6829 [(set RC:$dst, (IntId RC:$src1, (ld_frag addr:$src2)))]>,6830 Sched<[WriteAESDecEnc.Folded, WriteAESDecEnc.ReadAfterFold]>;6831 }6832}6833 6834// Perform One Round of an AES Encryption/Decryption Flow6835let Predicates = [HasAVX, NoVLX_Or_NoVAES, HasAES] in {6836 defm VAESENC : AESI_binop_rm_int<0xDC, "vaesenc",6837 int_x86_aesni_aesenc, load>, VEX, VVVV, WIG;6838 defm VAESENCLAST : AESI_binop_rm_int<0xDD, "vaesenclast",6839 int_x86_aesni_aesenclast, load>, VEX, VVVV, WIG;6840 defm VAESDEC : AESI_binop_rm_int<0xDE, "vaesdec",6841 int_x86_aesni_aesdec, load>, VEX, VVVV, WIG;6842 defm VAESDECLAST : AESI_binop_rm_int<0xDF, "vaesdeclast",6843 int_x86_aesni_aesdeclast, load>, VEX, VVVV, WIG;6844}6845 6846let Predicates = [NoVLX, HasVAES] in {6847 defm VAESENCY : AESI_binop_rm_int<0xDC, "vaesenc",6848 int_x86_aesni_aesenc_256, load, 0, VR256,6849 i256mem>, VEX, VVVV, VEX_L, WIG;6850 defm VAESENCLASTY : AESI_binop_rm_int<0xDD, "vaesenclast",6851 int_x86_aesni_aesenclast_256, load, 0, VR256,6852 i256mem>, VEX, VVVV, VEX_L, WIG;6853 defm VAESDECY : AESI_binop_rm_int<0xDE, "vaesdec",6854 int_x86_aesni_aesdec_256, load, 0, VR256,6855 i256mem>, VEX, VVVV, VEX_L, WIG;6856 defm VAESDECLASTY : AESI_binop_rm_int<0xDF, "vaesdeclast",6857 int_x86_aesni_aesdeclast_256, load, 0, VR256,6858 i256mem>, VEX, VVVV, VEX_L, WIG;6859}6860 6861let Constraints = "$src1 = $dst" in {6862 defm AESENC : AESI_binop_rm_int<0xDC, "aesenc",6863 int_x86_aesni_aesenc, memop, 1>;6864 defm AESENCLAST : AESI_binop_rm_int<0xDD, "aesenclast",6865 int_x86_aesni_aesenclast, memop, 1>;6866 defm AESDEC : AESI_binop_rm_int<0xDE, "aesdec",6867 int_x86_aesni_aesdec, memop, 1>;6868 defm AESDECLAST : AESI_binop_rm_int<0xDF, "aesdeclast",6869 int_x86_aesni_aesdeclast, memop, 1>;6870}6871 6872// Perform the AES InvMixColumn Transformation6873let Predicates = [HasAVX, HasAES] in {6874 def VAESIMCrr : AES8I<0xDB, MRMSrcReg, (outs VR128:$dst),6875 (ins VR128:$src1),6876 "vaesimc\t{$src1, $dst|$dst, $src1}",6877 [(set VR128:$dst,6878 (int_x86_aesni_aesimc VR128:$src1))]>, Sched<[WriteAESIMC]>,6879 VEX, WIG;6880 def VAESIMCrm : AES8I<0xDB, MRMSrcMem, (outs VR128:$dst),6881 (ins i128mem:$src1),6882 "vaesimc\t{$src1, $dst|$dst, $src1}",6883 [(set VR128:$dst, (int_x86_aesni_aesimc (load addr:$src1)))]>,6884 Sched<[WriteAESIMC.Folded]>, VEX, WIG;6885}6886def AESIMCrr : AES8I<0xDB, MRMSrcReg, (outs VR128:$dst),6887 (ins VR128:$src1),6888 "aesimc\t{$src1, $dst|$dst, $src1}",6889 [(set VR128:$dst,6890 (int_x86_aesni_aesimc VR128:$src1))]>, Sched<[WriteAESIMC]>;6891def AESIMCrm : AES8I<0xDB, MRMSrcMem, (outs VR128:$dst),6892 (ins i128mem:$src1),6893 "aesimc\t{$src1, $dst|$dst, $src1}",6894 [(set VR128:$dst, (int_x86_aesni_aesimc (memop addr:$src1)))]>,6895 Sched<[WriteAESIMC.Folded]>;6896 6897// AES Round Key Generation Assist6898let Predicates = [HasAVX, HasAES] in {6899 def VAESKEYGENASSISTrri : AESAI<0xDF, MRMSrcReg, (outs VR128:$dst),6900 (ins VR128:$src1, u8imm:$src2),6901 "vaeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}",6902 [(set VR128:$dst,6903 (int_x86_aesni_aeskeygenassist VR128:$src1, timm:$src2))]>,6904 Sched<[WriteAESKeyGen]>, VEX, WIG;6905 def VAESKEYGENASSISTrmi : AESAI<0xDF, MRMSrcMem, (outs VR128:$dst),6906 (ins i128mem:$src1, u8imm:$src2),6907 "vaeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}",6908 [(set VR128:$dst,6909 (int_x86_aesni_aeskeygenassist (load addr:$src1), timm:$src2))]>,6910 Sched<[WriteAESKeyGen.Folded]>, VEX, WIG;6911}6912def AESKEYGENASSISTrri : AESAI<0xDF, MRMSrcReg, (outs VR128:$dst),6913 (ins VR128:$src1, u8imm:$src2),6914 "aeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}",6915 [(set VR128:$dst,6916 (int_x86_aesni_aeskeygenassist VR128:$src1, timm:$src2))]>,6917 Sched<[WriteAESKeyGen]>;6918def AESKEYGENASSISTrmi : AESAI<0xDF, MRMSrcMem, (outs VR128:$dst),6919 (ins i128mem:$src1, u8imm:$src2),6920 "aeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}",6921 [(set VR128:$dst,6922 (int_x86_aesni_aeskeygenassist (memop addr:$src1), timm:$src2))]>,6923 Sched<[WriteAESKeyGen.Folded]>;6924 6925//===----------------------------------------------------------------------===//6926// PCLMUL Instructions6927//===----------------------------------------------------------------------===//6928 6929// Immediate transform to help with commuting.6930def PCLMULCommuteImm : SDNodeXForm<timm, [{6931 uint8_t Imm = N->getZExtValue();6932 return getI8Imm((uint8_t)((Imm >> 4) | (Imm << 4)), SDLoc(N));6933}]>;6934 6935// SSE carry-less Multiplication instructions6936let Predicates = [NoAVX, HasPCLMUL] in {6937 let Constraints = "$src1 = $dst" in {6938 let isCommutable = 1 in6939 def PCLMULQDQrri : PCLMULIi8<0x44, MRMSrcReg, (outs VR128:$dst),6940 (ins VR128:$src1, VR128:$src2, u8imm:$src3),6941 "pclmulqdq\t{$src3, $src2, $dst|$dst, $src2, $src3}",6942 [(set VR128:$dst,6943 (int_x86_pclmulqdq VR128:$src1, VR128:$src2, timm:$src3))]>,6944 Sched<[WriteCLMul]>;6945 6946 def PCLMULQDQrmi : PCLMULIi8<0x44, MRMSrcMem, (outs VR128:$dst),6947 (ins VR128:$src1, i128mem:$src2, u8imm:$src3),6948 "pclmulqdq\t{$src3, $src2, $dst|$dst, $src2, $src3}",6949 [(set VR128:$dst,6950 (int_x86_pclmulqdq VR128:$src1, (memop addr:$src2),6951 timm:$src3))]>,6952 Sched<[WriteCLMul.Folded, WriteCLMul.ReadAfterFold]>;6953 } // Constraints = "$src1 = $dst"6954 6955 def : Pat<(int_x86_pclmulqdq (memop addr:$src2), VR128:$src1,6956 (i8 timm:$src3)),6957 (PCLMULQDQrmi VR128:$src1, addr:$src2,6958 (PCLMULCommuteImm timm:$src3))>;6959} // Predicates = [NoAVX, HasPCLMUL]6960 6961// SSE aliases6962foreach HI = ["hq","lq"] in6963foreach LO = ["hq","lq"] in {6964 def : InstAlias<"pclmul" # HI # LO # "dq\t{$src, $dst|$dst, $src}",6965 (PCLMULQDQrri VR128:$dst, VR128:$src,6966 !add(!shl(!eq(LO,"hq"),4),!eq(HI,"hq"))), 0>;6967 def : InstAlias<"pclmul" # HI # LO # "dq\t{$src, $dst|$dst, $src}",6968 (PCLMULQDQrmi VR128:$dst, i128mem:$src,6969 !add(!shl(!eq(LO,"hq"),4),!eq(HI,"hq"))), 0>;6970}6971 6972// AVX carry-less Multiplication instructions6973multiclass vpclmulqdq<RegisterClass RC, X86MemOperand MemOp,6974 PatFrag LdFrag, Intrinsic IntId> {6975 let isCommutable = 1 in6976 def rri : PCLMULIi8<0x44, MRMSrcReg, (outs RC:$dst),6977 (ins RC:$src1, RC:$src2, u8imm:$src3),6978 "vpclmulqdq\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",6979 [(set RC:$dst,6980 (IntId RC:$src1, RC:$src2, timm:$src3))]>,6981 Sched<[WriteCLMul]>;6982 6983 def rmi : PCLMULIi8<0x44, MRMSrcMem, (outs RC:$dst),6984 (ins RC:$src1, MemOp:$src2, u8imm:$src3),6985 "vpclmulqdq\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",6986 [(set RC:$dst,6987 (IntId RC:$src1, (LdFrag addr:$src2), timm:$src3))]>,6988 Sched<[WriteCLMul.Folded, WriteCLMul.ReadAfterFold]>;6989 6990 // We can commute a load in the first operand by swapping the sources and6991 // rotating the immediate.6992 def : Pat<(IntId (LdFrag addr:$src2), RC:$src1, (i8 timm:$src3)),6993 (!cast<Instruction>(NAME#"rmi") RC:$src1, addr:$src2,6994 (PCLMULCommuteImm timm:$src3))>;6995}6996 6997let Predicates = [HasAVX, NoVLX_Or_NoVPCLMULQDQ, HasPCLMUL] in6998defm VPCLMULQDQ : vpclmulqdq<VR128, i128mem, load,6999 int_x86_pclmulqdq>, VEX, VVVV, WIG;7000 7001let Predicates = [NoVLX, HasVPCLMULQDQ] in7002defm VPCLMULQDQY : vpclmulqdq<VR256, i256mem, load,7003 int_x86_pclmulqdq_256>, VEX, VVVV, VEX_L, WIG;7004 7005multiclass vpclmulqdq_aliases_impl<string InstStr, RegisterClass RC,7006 X86MemOperand MemOp, string Hi, string Lo> {7007 def : InstAlias<"vpclmul"#Hi#Lo#"dq\t{$src2, $src1, $dst|$dst, $src1, $src2}",7008 (!cast<Instruction>(InstStr # "rri") RC:$dst, RC:$src1, RC:$src2,7009 !add(!shl(!eq(Lo,"hq"),4),!eq(Hi,"hq"))), 0>;7010 def : InstAlias<"vpclmul"#Hi#Lo#"dq\t{$src2, $src1, $dst|$dst, $src1, $src2}",7011 (!cast<Instruction>(InstStr # "rmi") RC:$dst, RC:$src1, MemOp:$src2,7012 !add(!shl(!eq(Lo,"hq"),4),!eq(Hi,"hq"))), 0>;7013}7014 7015multiclass vpclmulqdq_aliases<string InstStr, RegisterClass RC,7016 X86MemOperand MemOp> {7017 defm : vpclmulqdq_aliases_impl<InstStr, RC, MemOp, "hq", "hq">;7018 defm : vpclmulqdq_aliases_impl<InstStr, RC, MemOp, "hq", "lq">;7019 defm : vpclmulqdq_aliases_impl<InstStr, RC, MemOp, "lq", "hq">;7020 defm : vpclmulqdq_aliases_impl<InstStr, RC, MemOp, "lq", "lq">;7021}7022 7023// AVX aliases7024defm : vpclmulqdq_aliases<"VPCLMULQDQ", VR128, i128mem>;7025defm : vpclmulqdq_aliases<"VPCLMULQDQY", VR256, i256mem>;7026 7027//===----------------------------------------------------------------------===//7028// SSE4A Instructions7029//===----------------------------------------------------------------------===//7030 7031let Predicates = [HasSSE4A] in {7032 7033let ExeDomain = SSEPackedInt in {7034let Constraints = "$src = $dst" in {7035def EXTRQI : Ii8<0x78, MRMXr, (outs VR128:$dst),7036 (ins VR128:$src, u8imm:$len, u8imm:$idx),7037 "extrq\t{$idx, $len, $src|$src, $len, $idx}",7038 [(set VR128:$dst, (X86extrqi VR128:$src, timm:$len,7039 timm:$idx))]>,7040 TB, PD, Sched<[SchedWriteVecALU.XMM]>;7041def EXTRQ : I<0x79, MRMSrcReg, (outs VR128:$dst),7042 (ins VR128:$src, VR128:$mask),7043 "extrq\t{$mask, $src|$src, $mask}",7044 [(set VR128:$dst, (int_x86_sse4a_extrq VR128:$src,7045 VR128:$mask))]>,7046 TB, PD, Sched<[SchedWriteVecALU.XMM]>;7047 7048def INSERTQI : Ii8<0x78, MRMSrcReg, (outs VR128:$dst),7049 (ins VR128:$src, VR128:$src2, u8imm:$len, u8imm:$idx),7050 "insertq\t{$idx, $len, $src2, $src|$src, $src2, $len, $idx}",7051 [(set VR128:$dst, (X86insertqi VR128:$src, VR128:$src2,7052 timm:$len, timm:$idx))]>,7053 TB, XD, Sched<[SchedWriteVecALU.XMM]>;7054def INSERTQ : I<0x79, MRMSrcReg, (outs VR128:$dst),7055 (ins VR128:$src, VR128:$mask),7056 "insertq\t{$mask, $src|$src, $mask}",7057 [(set VR128:$dst, (int_x86_sse4a_insertq VR128:$src,7058 VR128:$mask))]>,7059 TB, XD, Sched<[SchedWriteVecALU.XMM]>;7060}7061} // ExeDomain = SSEPackedInt7062 7063// Non-temporal (unaligned) scalar stores.7064let AddedComplexity = 400 in { // Prefer non-temporal versions7065let hasSideEffects = 0, mayStore = 1, SchedRW = [SchedWriteFMoveLSNT.Scl.MR] in {7066def MOVNTSS : I<0x2B, MRMDestMem, (outs), (ins f32mem:$dst, VR128:$src),7067 "movntss\t{$src, $dst|$dst, $src}", []>, TB, XS;7068 7069def MOVNTSD : I<0x2B, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src),7070 "movntsd\t{$src, $dst|$dst, $src}", []>, TB, XD;7071} // SchedRW7072 7073def : Pat<(nontemporalstore FR32:$src, addr:$dst),7074 (MOVNTSS addr:$dst, (v4f32 (COPY_TO_REGCLASS FR32:$src, VR128)))>;7075 7076def : Pat<(nontemporalstore FR64:$src, addr:$dst),7077 (MOVNTSD addr:$dst, (v2f64 (COPY_TO_REGCLASS FR64:$src, VR128)))>;7078 7079} // AddedComplexity7080} // HasSSE4A7081 7082//===----------------------------------------------------------------------===//7083// AVX Instructions7084//===----------------------------------------------------------------------===//7085 7086//===----------------------------------------------------------------------===//7087// VBROADCAST - Load from memory and broadcast to all elements of the7088// destination operand7089//7090class avx_broadcast_rm<bits<8> opc, string OpcodeStr, RegisterClass RC,7091 X86MemOperand x86memop, ValueType VT,7092 PatFrag bcast_frag, SchedWrite Sched> :7093 AVX8I<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src),7094 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),7095 [(set RC:$dst, (VT (bcast_frag addr:$src)))]>,7096 Sched<[Sched]>, VEX {7097 let isReMaterializable = 1;7098}7099 7100// AVX2 adds register forms7101class avx2_broadcast_rr<bits<8> opc, string OpcodeStr, RegisterClass RC,7102 ValueType ResVT, ValueType OpVT, SchedWrite Sched> :7103 AVX28I<opc, MRMSrcReg, (outs RC:$dst), (ins VR128:$src),7104 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),7105 [(set RC:$dst, (ResVT (X86VBroadcast (OpVT VR128:$src))))]>,7106 Sched<[Sched]>, VEX;7107 7108let ExeDomain = SSEPackedSingle, Predicates = [HasAVX, NoVLX] in {7109 def VBROADCASTSSrm : avx_broadcast_rm<0x18, "vbroadcastss", VR128,7110 f32mem, v4f32, X86VBroadcastld32,7111 SchedWriteFShuffle.XMM.Folded>;7112 def VBROADCASTSSYrm : avx_broadcast_rm<0x18, "vbroadcastss", VR256,7113 f32mem, v8f32, X86VBroadcastld32,7114 SchedWriteFShuffle.XMM.Folded>, VEX_L;7115}7116let ExeDomain = SSEPackedDouble, Predicates = [HasAVX, NoVLX] in7117def VBROADCASTSDYrm : avx_broadcast_rm<0x19, "vbroadcastsd", VR256, f64mem,7118 v4f64, X86VBroadcastld64,7119 SchedWriteFShuffle.XMM.Folded>, VEX_L;7120 7121let ExeDomain = SSEPackedSingle, Predicates = [HasAVX2, NoVLX] in {7122 def VBROADCASTSSrr : avx2_broadcast_rr<0x18, "vbroadcastss", VR128,7123 v4f32, v4f32, SchedWriteFShuffle.XMM>;7124 def VBROADCASTSSYrr : avx2_broadcast_rr<0x18, "vbroadcastss", VR256,7125 v8f32, v4f32, WriteFShuffle256>, VEX_L;7126}7127let ExeDomain = SSEPackedDouble, Predicates = [HasAVX2, NoVLX] in7128def VBROADCASTSDYrr : avx2_broadcast_rr<0x19, "vbroadcastsd", VR256,7129 v4f64, v2f64, WriteFShuffle256>, VEX_L;7130 7131//===----------------------------------------------------------------------===//7132// VBROADCAST*128 - Load from memory and broadcast 128-bit vector to both7133// halves of a 256-bit vector.7134//7135let mayLoad = 1, hasSideEffects = 0, Predicates = [HasAVX2] in7136def VBROADCASTI128rm : AVX8I<0x5A, MRMSrcMem, (outs VR256:$dst),7137 (ins i128mem:$src),7138 "vbroadcasti128\t{$src, $dst|$dst, $src}", []>,7139 Sched<[WriteShuffleLd]>, VEX, VEX_L;7140 7141let mayLoad = 1, hasSideEffects = 0, Predicates = [HasAVX],7142 ExeDomain = SSEPackedSingle in7143def VBROADCASTF128rm : AVX8I<0x1A, MRMSrcMem, (outs VR256:$dst),7144 (ins f128mem:$src),7145 "vbroadcastf128\t{$src, $dst|$dst, $src}", []>,7146 Sched<[SchedWriteFShuffle.XMM.Folded]>, VEX, VEX_L;7147 7148let Predicates = [HasAVX, NoVLX] in {7149def : Pat<(v4f64 (X86SubVBroadcastld128 addr:$src)),7150 (VBROADCASTF128rm addr:$src)>;7151def : Pat<(v8f32 (X86SubVBroadcastld128 addr:$src)),7152 (VBROADCASTF128rm addr:$src)>;7153// NOTE: We're using FP instructions here, but execution domain fixing can7154// convert to integer when profitable.7155def : Pat<(v4i64 (X86SubVBroadcastld128 addr:$src)),7156 (VBROADCASTF128rm addr:$src)>;7157def : Pat<(v8i32 (X86SubVBroadcastld128 addr:$src)),7158 (VBROADCASTF128rm addr:$src)>;7159def : Pat<(v16i16 (X86SubVBroadcastld128 addr:$src)),7160 (VBROADCASTF128rm addr:$src)>;7161def : Pat<(v16f16 (X86SubVBroadcastld128 addr:$src)),7162 (VBROADCASTF128rm addr:$src)>;7163def : Pat<(v32i8 (X86SubVBroadcastld128 addr:$src)),7164 (VBROADCASTF128rm addr:$src)>;7165}7166 7167let Predicates = [HasAVXNECONVERT, NoVLX] in7168 def : Pat<(v16bf16 (X86SubVBroadcastld128 addr:$src)),7169 (VBROADCASTF128rm addr:$src)>;7170 7171//===----------------------------------------------------------------------===//7172// VPERM2F128 - Permute Floating-Point Values in 128-bit chunks7173//7174 7175let ExeDomain = SSEPackedSingle in {7176let isCommutable = 1 in7177def VPERM2F128rri : AVXAIi8<0x06, MRMSrcReg, (outs VR256:$dst),7178 (ins VR256:$src1, VR256:$src2, u8imm:$src3),7179 "vperm2f128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", []>,7180 VEX, VVVV, VEX_L, Sched<[WriteFShuffle256]>;7181def VPERM2F128rmi : AVXAIi8<0x06, MRMSrcMem, (outs VR256:$dst),7182 (ins VR256:$src1, f256mem:$src2, u8imm:$src3),7183 "vperm2f128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", []>,7184 VEX, VVVV, VEX_L, Sched<[WriteFShuffle256.Folded, WriteFShuffle256.ReadAfterFold]>;7185}7186 7187// Immediate transform to help with commuting.7188def Perm2XCommuteImm : SDNodeXForm<timm, [{7189 return getI8Imm(N->getZExtValue() ^ 0x22, SDLoc(N));7190}]>;7191 7192multiclass vperm2x128_lowering<string InstrStr, ValueType VT, PatFrag memop_frag> {7193 def : Pat<(VT (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 timm:$imm))),7194 (!cast<Instruction>(InstrStr#rri) VR256:$src1, VR256:$src2, timm:$imm)>;7195 def : Pat<(VT (X86VPerm2x128 VR256:$src1, (memop_frag addr:$src2), (i8 timm:$imm))),7196 (!cast<Instruction>(InstrStr#rmi) VR256:$src1, addr:$src2, timm:$imm)>;7197 // Pattern with load in other operand.7198 def : Pat<(VT (X86VPerm2x128 (memop_frag addr:$src2), VR256:$src1, (i8 timm:$imm))),7199 (!cast<Instruction>(InstrStr#rmi) VR256:$src1, addr:$src2,7200 (Perm2XCommuteImm timm:$imm))>;7201}7202 7203let Predicates = [HasAVX] in {7204 defm : vperm2x128_lowering<"VPERM2F128", v4f64, loadv4f64>;7205 defm : vperm2x128_lowering<"VPERM2F128", v8f32, loadv8f32>;7206}7207 7208let Predicates = [HasAVX1Only] in {7209 defm : vperm2x128_lowering<"VPERM2F128", v4i64, loadv4i64>;7210 defm : vperm2x128_lowering<"VPERM2F128", v8i32, loadv8i32>;7211 defm : vperm2x128_lowering<"VPERM2F128", v16i16, loadv16i16>;7212 defm : vperm2x128_lowering<"VPERM2F128", v16f16, loadv16f16>;7213 defm : vperm2x128_lowering<"VPERM2F128", v32i8, loadv32i8>;7214}7215 7216//===----------------------------------------------------------------------===//7217// VINSERTF128 - Insert packed floating-point values7218//7219let hasSideEffects = 0, ExeDomain = SSEPackedSingle in {7220def VINSERTF128rri : AVXAIi8<0x18, MRMSrcReg, (outs VR256:$dst),7221 (ins VR256:$src1, VR128:$src2, u8imm:$src3),7222 "vinsertf128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",7223 []>, Sched<[WriteFShuffle256]>, VEX, VVVV, VEX_L;7224let mayLoad = 1 in7225def VINSERTF128rmi : AVXAIi8<0x18, MRMSrcMem, (outs VR256:$dst),7226 (ins VR256:$src1, f128mem:$src2, u8imm:$src3),7227 "vinsertf128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",7228 []>, Sched<[WriteFShuffle256.Folded, WriteFShuffle256.ReadAfterFold]>, VEX, VVVV, VEX_L;7229}7230 7231// To create a 256-bit all ones value, we should produce VCMPTRUEPS7232// with YMM register containing zero.7233// FIXME: Avoid producing vxorps to clear the fake inputs.7234let Predicates = [HasAVX1Only] in {7235def : Pat<(v8i32 immAllOnesV), (VCMPPSYrri (AVX_SET0), (AVX_SET0), 0xf)>;7236}7237 7238multiclass vinsert_lowering<string InstrStr, string PermStr,7239 ValueType From, ValueType To,7240 PatFrag frommemop_frag, PatFrag tomemop_frag> {7241 def : Pat<(vinsert128_insert:$ins (To VR256:$src1), (From VR128:$src2),7242 (iPTR imm)),7243 (!cast<Instruction>(InstrStr#rri) VR256:$src1, VR128:$src2,7244 (INSERT_get_vinsert128_imm VR256:$ins))>;7245 def : Pat<(vinsert128_insert:$ins (To VR256:$src1),7246 (From (frommemop_frag addr:$src2)),7247 (iPTR imm)),7248 (!cast<Instruction>(InstrStr#rmi) VR256:$src1, addr:$src2,7249 (INSERT_get_vinsert128_imm VR256:$ins))>;7250 // Folding "To" vector - convert to perm2x128 and commute inputs.7251 def : Pat<(vinsert128_insert:$ins (To (tomemop_frag addr:$src1)),7252 (From VR128:$src2),7253 (iPTR imm)),7254 (!cast<Instruction>(PermStr#rmi)7255 (INSERT_SUBREG (To (IMPLICIT_DEF)), VR128:$src2, sub_xmm),7256 addr:$src1, (INSERT_get_vperm2x128_commutedimm VR256:$ins))>;7257}7258 7259let Predicates = [HasAVX, NoVLX] in {7260 defm : vinsert_lowering<"VINSERTF128", "VPERM2F128", v4f32, v8f32, loadv4f32, loadv8f32>;7261 defm : vinsert_lowering<"VINSERTF128", "VPERM2F128", v2f64, v4f64, loadv2f64, loadv4f64>;7262}7263 7264let Predicates = [HasAVX1Only] in {7265 defm : vinsert_lowering<"VINSERTF128", "VPERM2F128", v2i64, v4i64, loadv2i64, loadv4i64>;7266 defm : vinsert_lowering<"VINSERTF128", "VPERM2F128", v4i32, v8i32, loadv4i32, loadv8i32>;7267 defm : vinsert_lowering<"VINSERTF128", "VPERM2F128", v8i16, v16i16, loadv8i16, loadv16i16>;7268 defm : vinsert_lowering<"VINSERTF128", "VPERM2F128", v8f16, v16f16, loadv8f16, loadv16f16>;7269 defm : vinsert_lowering<"VINSERTF128", "VPERM2F128", v16i8, v32i8, loadv16i8, loadv32i8>;7270 defm : vinsert_lowering<"VINSERTF128", "VPERM2F128", v16i8, v32i8, loadv16i8, loadv32i8>;7271}7272 7273//===----------------------------------------------------------------------===//7274// VEXTRACTF128 - Extract packed floating-point values7275//7276let hasSideEffects = 0, ExeDomain = SSEPackedSingle in {7277def VEXTRACTF128rri : AVXAIi8<0x19, MRMDestReg, (outs VR128:$dst),7278 (ins VR256:$src1, u8imm:$src2),7279 "vextractf128\t{$src2, $src1, $dst|$dst, $src1, $src2}",7280 []>, Sched<[WriteFShuffle256]>, VEX, VEX_L;7281let mayStore = 1 in7282def VEXTRACTF128mri : AVXAIi8<0x19, MRMDestMem, (outs),7283 (ins f128mem:$dst, VR256:$src1, u8imm:$src2),7284 "vextractf128\t{$src2, $src1, $dst|$dst, $src1, $src2}",7285 []>, Sched<[WriteFStoreX]>, VEX, VEX_L;7286}7287 7288multiclass vextract_lowering<string InstrStr, ValueType From, ValueType To> {7289 def : Pat<(vextract128_extract:$ext VR256:$src1, (iPTR imm)),7290 (To (!cast<Instruction>(InstrStr#rri)7291 (From VR256:$src1),7292 (EXTRACT_get_vextract128_imm VR128:$ext)))>;7293 def : Pat<(store (To (vextract128_extract:$ext (From VR256:$src1),7294 (iPTR imm))), addr:$dst),7295 (!cast<Instruction>(InstrStr#mri) addr:$dst, VR256:$src1,7296 (EXTRACT_get_vextract128_imm VR128:$ext))>;7297}7298 7299// AVX1 patterns7300let Predicates = [HasAVX, NoVLX] in {7301 defm : vextract_lowering<"VEXTRACTF128", v8f32, v4f32>;7302 defm : vextract_lowering<"VEXTRACTF128", v4f64, v2f64>;7303}7304 7305let Predicates = [HasAVX1Only] in {7306 defm : vextract_lowering<"VEXTRACTF128", v4i64, v2i64>;7307 defm : vextract_lowering<"VEXTRACTF128", v8i32, v4i32>;7308 defm : vextract_lowering<"VEXTRACTF128", v16i16, v8i16>;7309 defm : vextract_lowering<"VEXTRACTF128", v16f16, v8f16>;7310 defm : vextract_lowering<"VEXTRACTF128", v32i8, v16i8>;7311 defm : vextract_lowering<"VEXTRACTF128", v32i8, v16i8>;7312}7313 7314//===----------------------------------------------------------------------===//7315// VMASKMOV - Conditional SIMD Packed Loads and Stores7316//7317multiclass avx_movmask_rm<bits<8> opc_rm, bits<8> opc_mr, string OpcodeStr,7318 Intrinsic IntLd, Intrinsic IntLd256,7319 Intrinsic IntSt, Intrinsic IntSt256,7320 X86SchedWriteMaskMove schedX,7321 X86SchedWriteMaskMove schedY> {7322 def rm : AVX8I<opc_rm, MRMSrcMem, (outs VR128:$dst),7323 (ins VR128:$src1, f128mem:$src2),7324 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),7325 [(set VR128:$dst, (IntLd addr:$src2, VR128:$src1))]>,7326 VEX, VVVV, Sched<[schedX.RM]>;7327 def Yrm : AVX8I<opc_rm, MRMSrcMem, (outs VR256:$dst),7328 (ins VR256:$src1, f256mem:$src2),7329 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),7330 [(set VR256:$dst, (IntLd256 addr:$src2, VR256:$src1))]>,7331 VEX, VVVV, VEX_L, Sched<[schedY.RM]>;7332 def mr : AVX8I<opc_mr, MRMDestMem, (outs),7333 (ins f128mem:$dst, VR128:$src1, VR128:$src2),7334 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),7335 [(IntSt addr:$dst, VR128:$src1, VR128:$src2)]>,7336 VEX, VVVV, Sched<[schedX.MR]>;7337 def Ymr : AVX8I<opc_mr, MRMDestMem, (outs),7338 (ins f256mem:$dst, VR256:$src1, VR256:$src2),7339 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),7340 [(IntSt256 addr:$dst, VR256:$src1, VR256:$src2)]>,7341 VEX, VVVV, VEX_L, Sched<[schedY.MR]>;7342}7343 7344let ExeDomain = SSEPackedSingle in7345defm VMASKMOVPS : avx_movmask_rm<0x2C, 0x2E, "vmaskmovps",7346 int_x86_avx_maskload_ps,7347 int_x86_avx_maskload_ps_256,7348 int_x86_avx_maskstore_ps,7349 int_x86_avx_maskstore_ps_256,7350 WriteFMaskMove32, WriteFMaskMove32Y>;7351let ExeDomain = SSEPackedDouble in7352defm VMASKMOVPD : avx_movmask_rm<0x2D, 0x2F, "vmaskmovpd",7353 int_x86_avx_maskload_pd,7354 int_x86_avx_maskload_pd_256,7355 int_x86_avx_maskstore_pd,7356 int_x86_avx_maskstore_pd_256,7357 WriteFMaskMove64, WriteFMaskMove64Y>;7358 7359//===----------------------------------------------------------------------===//7360// AVX_VNNI7361//===----------------------------------------------------------------------===//7362let Predicates = [HasAVXVNNI, NoVLX_Or_NoVNNI], Constraints = "$src1 = $dst",7363 explicitOpPrefix = ExplicitVEX in7364multiclass avx_vnni_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,7365 bit IsCommutable> {7366 let isCommutable = IsCommutable in7367 def rr : AVX8I<opc, MRMSrcReg, (outs VR128:$dst),7368 (ins VR128:$src1, VR128:$src2, VR128:$src3),7369 !strconcat(OpcodeStr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),7370 [(set VR128:$dst, (v4i32 (OpNode VR128:$src1,7371 VR128:$src2, VR128:$src3)))]>,7372 VEX, VVVV, Sched<[SchedWriteVecIMul.XMM]>;7373 7374 def rm : AVX8I<opc, MRMSrcMem, (outs VR128:$dst),7375 (ins VR128:$src1, VR128:$src2, i128mem:$src3),7376 !strconcat(OpcodeStr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),7377 [(set VR128:$dst, (v4i32 (OpNode VR128:$src1, VR128:$src2,7378 (loadv4i32 addr:$src3))))]>,7379 VEX, VVVV, Sched<[SchedWriteVecIMul.XMM.Folded,7380 SchedWriteVecIMul.XMM.ReadAfterFold,7381 SchedWriteVecIMul.XMM.ReadAfterFold]>;7382 7383 let isCommutable = IsCommutable in7384 def Yrr : AVX8I<opc, MRMSrcReg, (outs VR256:$dst),7385 (ins VR256:$src1, VR256:$src2, VR256:$src3),7386 !strconcat(OpcodeStr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),7387 [(set VR256:$dst, (v8i32 (OpNode VR256:$src1,7388 VR256:$src2, VR256:$src3)))]>,7389 VEX, VVVV, VEX_L, Sched<[SchedWriteVecIMul.YMM]>;7390 7391 def Yrm : AVX8I<opc, MRMSrcMem, (outs VR256:$dst),7392 (ins VR256:$src1, VR256:$src2, i256mem:$src3),7393 !strconcat(OpcodeStr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),7394 [(set VR256:$dst, (v8i32 (OpNode VR256:$src1, VR256:$src2,7395 (loadv8i32 addr:$src3))))]>,7396 VEX, VVVV, VEX_L, Sched<[SchedWriteVecIMul.YMM.Folded,7397 SchedWriteVecIMul.YMM.ReadAfterFold,7398 SchedWriteVecIMul.YMM.ReadAfterFold]>;7399}7400 7401defm VPDPBUSD : avx_vnni_rm<0x50, "vpdpbusd", X86Vpdpbusd, 0>;7402defm VPDPBUSDS : avx_vnni_rm<0x51, "vpdpbusds", X86Vpdpbusds, 0>;7403defm VPDPWSSD : avx_vnni_rm<0x52, "vpdpwssd", X86Vpdpwssd, 1>;7404defm VPDPWSSDS : avx_vnni_rm<0x53, "vpdpwssds", X86Vpdpwssds, 1>;7405 7406let Predicates = [HasAVXVNNI, NoVLX_Or_NoVNNI] in {7407 def : Pat<(v8i32 (add VR256:$src1,7408 (X86vpmaddwd_su VR256:$src2, VR256:$src3))),7409 (VPDPWSSDYrr VR256:$src1, VR256:$src2, VR256:$src3)>;7410 def : Pat<(v8i32 (add VR256:$src1,7411 (X86vpmaddwd_su VR256:$src2, (load addr:$src3)))),7412 (VPDPWSSDYrm VR256:$src1, VR256:$src2, addr:$src3)>;7413 def : Pat<(v4i32 (add VR128:$src1,7414 (X86vpmaddwd_su VR128:$src2, VR128:$src3))),7415 (VPDPWSSDrr VR128:$src1, VR128:$src2, VR128:$src3)>;7416 def : Pat<(v4i32 (add VR128:$src1,7417 (X86vpmaddwd_su VR128:$src2, (load addr:$src3)))),7418 (VPDPWSSDrm VR128:$src1, VR128:$src2, addr:$src3)>;7419}7420 7421//===----------------------------------------------------------------------===//7422// VPERMIL - Permute Single and Double Floating-Point Values7423//7424 7425multiclass avx_permil<bits<8> opc_rm, bits<8> opc_rmi, string OpcodeStr,7426 RegisterClass RC, X86MemOperand x86memop_f,7427 X86MemOperand x86memop_i,7428 ValueType f_vt, ValueType i_vt,7429 X86FoldableSchedWrite sched,7430 X86FoldableSchedWrite varsched> {7431 let Predicates = [HasAVX, NoVLX] in {7432 def rr : AVX8I<opc_rm, MRMSrcReg, (outs RC:$dst),7433 (ins RC:$src1, RC:$src2),7434 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),7435 [(set RC:$dst, (f_vt (X86VPermilpv RC:$src1, (i_vt RC:$src2))))]>, VEX, VVVV,7436 Sched<[varsched]>;7437 def rm : AVX8I<opc_rm, MRMSrcMem, (outs RC:$dst),7438 (ins RC:$src1, x86memop_i:$src2),7439 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),7440 [(set RC:$dst, (f_vt (X86VPermilpv RC:$src1,7441 (i_vt (load addr:$src2)))))]>, VEX, VVVV,7442 Sched<[varsched.Folded, sched.ReadAfterFold]>;7443 7444 def ri : AVXAIi8<opc_rmi, MRMSrcReg, (outs RC:$dst),7445 (ins RC:$src1, u8imm:$src2),7446 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),7447 [(set RC:$dst, (f_vt (X86VPermilpi RC:$src1, (i8 timm:$src2))))]>, VEX,7448 Sched<[sched]>;7449 def mi : AVXAIi8<opc_rmi, MRMSrcMem, (outs RC:$dst),7450 (ins x86memop_f:$src1, u8imm:$src2),7451 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),7452 [(set RC:$dst,7453 (f_vt (X86VPermilpi (load addr:$src1), (i8 timm:$src2))))]>, VEX,7454 Sched<[sched.Folded]>;7455 }// Predicates = [HasAVX, NoVLX]7456}7457 7458let ExeDomain = SSEPackedSingle in {7459 defm VPERMILPS : avx_permil<0x0C, 0x04, "vpermilps", VR128, f128mem, i128mem,7460 v4f32, v4i32, SchedWriteFShuffle.XMM,7461 SchedWriteFVarShuffle.XMM>;7462 defm VPERMILPSY : avx_permil<0x0C, 0x04, "vpermilps", VR256, f256mem, i256mem,7463 v8f32, v8i32, SchedWriteFShuffle.YMM,7464 SchedWriteFVarShuffle.YMM>, VEX_L;7465}7466let ExeDomain = SSEPackedDouble in {7467 defm VPERMILPD : avx_permil<0x0D, 0x05, "vpermilpd", VR128, f128mem, i128mem,7468 v2f64, v2i64, SchedWriteFShuffle.XMM,7469 SchedWriteFVarShuffle.XMM>;7470 defm VPERMILPDY : avx_permil<0x0D, 0x05, "vpermilpd", VR256, f256mem, i256mem,7471 v4f64, v4i64, SchedWriteFShuffle.YMM,7472 SchedWriteFVarShuffle.YMM>, VEX_L;7473}7474 7475//===----------------------------------------------------------------------===//7476// VZERO - Zero YMM registers7477// Note: These instruction do not affect the YMM16-YMM31.7478//7479 7480let SchedRW = [WriteSystem] in {7481let Defs = [YMM0, YMM1, YMM2, YMM3, YMM4, YMM5, YMM6, YMM7,7482 YMM8, YMM9, YMM10, YMM11, YMM12, YMM13, YMM14, YMM15] in {7483 // Zero All YMM registers7484 def VZEROALL : I<0x77, RawFrm, (outs), (ins), "vzeroall",7485 [(int_x86_avx_vzeroall)]>, TB, VEX, VEX_L,7486 Requires<[HasAVX]>, WIG;7487 7488 // Zero Upper bits of YMM registers7489 def VZEROUPPER : I<0x77, RawFrm, (outs), (ins), "vzeroupper",7490 [(int_x86_avx_vzeroupper)]>, TB, VEX,7491 Requires<[HasAVX]>, WIG;7492} // Defs7493} // SchedRW7494 7495//===----------------------------------------------------------------------===//7496// Half precision conversion instructions7497//7498 7499multiclass f16c_ph2ps<RegisterClass RC, X86MemOperand x86memop,7500 X86FoldableSchedWrite sched> {7501 def rr : I<0x13, MRMSrcReg, (outs RC:$dst), (ins VR128:$src),7502 "vcvtph2ps\t{$src, $dst|$dst, $src}",7503 [(set RC:$dst, (X86any_cvtph2ps VR128:$src))]>,7504 T8, PD, VEX, Sched<[sched]>;7505 let hasSideEffects = 0, mayLoad = 1 in7506 def rm : I<0x13, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src),7507 "vcvtph2ps\t{$src, $dst|$dst, $src}",7508 []>, T8, PD, VEX, Sched<[sched.Folded]>;7509}7510 7511multiclass f16c_ps2ph<RegisterClass RC, X86MemOperand x86memop,7512 SchedWrite RR, SchedWrite MR> {7513 def rr : Ii8<0x1D, MRMDestReg, (outs VR128:$dst),7514 (ins RC:$src1, i32u8imm:$src2),7515 "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}",7516 [(set VR128:$dst, (X86any_cvtps2ph RC:$src1, timm:$src2))]>,7517 TA, PD, VEX, Sched<[RR]>;7518 let hasSideEffects = 0, mayStore = 1 in7519 def mr : Ii8<0x1D, MRMDestMem, (outs),7520 (ins x86memop:$dst, RC:$src1, i32u8imm:$src2),7521 "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,7522 TA, PD, VEX, Sched<[MR]>;7523}7524 7525let Predicates = [HasF16C, NoVLX] in {7526 defm VCVTPH2PS : f16c_ph2ps<VR128, f64mem, WriteCvtPH2PS>, SIMD_EXC;7527 defm VCVTPH2PSY : f16c_ph2ps<VR256, f128mem, WriteCvtPH2PSY>, VEX_L, SIMD_EXC;7528 defm VCVTPS2PH : f16c_ps2ph<VR128, f64mem, WriteCvtPS2PH,7529 WriteCvtPS2PHSt>, SIMD_EXC;7530 defm VCVTPS2PHY : f16c_ps2ph<VR256, f128mem, WriteCvtPS2PHY,7531 WriteCvtPS2PHYSt>, VEX_L, SIMD_EXC;7532 7533 // Pattern match vcvtph2ps of a scalar i64 load.7534 def : Pat<(v4f32 (X86any_cvtph2ps (bc_v8i16 (v2i64 (X86vzload64 addr:$src))))),7535 (VCVTPH2PSrm addr:$src)>;7536 def : Pat<(v4f32 (X86any_cvtph2ps (bc_v8i167537 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),7538 (VCVTPH2PSrm addr:$src)>;7539 def : Pat<(v8f32 (X86any_cvtph2ps (loadv8i16 addr:$src))),7540 (VCVTPH2PSYrm addr:$src)>;7541 7542 def : Pat<(store (f64 (extractelt7543 (bc_v2f64 (v8i16 (X86any_cvtps2ph VR128:$src1, timm:$src2))),7544 (iPTR 0))), addr:$dst),7545 (VCVTPS2PHmr addr:$dst, VR128:$src1, timm:$src2)>;7546 def : Pat<(store (i64 (extractelt7547 (bc_v2i64 (v8i16 (X86any_cvtps2ph VR128:$src1, timm:$src2))),7548 (iPTR 0))), addr:$dst),7549 (VCVTPS2PHmr addr:$dst, VR128:$src1, timm:$src2)>;7550 def : Pat<(store (v8i16 (X86any_cvtps2ph VR256:$src1, timm:$src2)), addr:$dst),7551 (VCVTPS2PHYmr addr:$dst, VR256:$src1, timm:$src2)>;7552}7553 7554//===----------------------------------------------------------------------===//7555// AVX2 Instructions7556//===----------------------------------------------------------------------===//7557 7558/// AVX2_blend_rmi - AVX2 blend with 8-bit immediate7559multiclass AVX2_blend_rmi<bits<8> opc, string OpcodeStr, SDNode OpNode,7560 ValueType OpVT, X86FoldableSchedWrite sched,7561 RegisterClass RC,7562 X86MemOperand x86memop, SDNodeXForm commuteXForm> {7563 let isCommutable = 1 in7564 def rri : AVX2AIi8<opc, MRMSrcReg, (outs RC:$dst),7565 (ins RC:$src1, RC:$src2, u8imm:$src3),7566 !strconcat(OpcodeStr,7567 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),7568 [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2, timm:$src3)))]>,7569 Sched<[sched]>, VEX, VVVV;7570 def rmi : AVX2AIi8<opc, MRMSrcMem, (outs RC:$dst),7571 (ins RC:$src1, x86memop:$src2, u8imm:$src3),7572 !strconcat(OpcodeStr,7573 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),7574 [(set RC:$dst,7575 (OpVT (OpNode RC:$src1, (load addr:$src2), timm:$src3)))]>,7576 Sched<[sched.Folded, sched.ReadAfterFold]>, VEX, VVVV;7577 7578 // Pattern to commute if load is in first source.7579 def : Pat<(OpVT (OpNode (load addr:$src2), RC:$src1, timm:$src3)),7580 (!cast<Instruction>(NAME#"rmi") RC:$src1, addr:$src2,7581 (commuteXForm timm:$src3))>;7582}7583 7584let Predicates = [HasAVX2] in {7585defm VPBLENDD : AVX2_blend_rmi<0x02, "vpblendd", X86Blendi, v4i32,7586 SchedWriteBlend.XMM, VR128, i128mem,7587 BlendCommuteImm4>;7588defm VPBLENDDY : AVX2_blend_rmi<0x02, "vpblendd", X86Blendi, v8i32,7589 SchedWriteBlend.YMM, VR256, i256mem,7590 BlendCommuteImm8>, VEX_L;7591 7592def : Pat<(X86Blendi (v4i64 VR256:$src1), (v4i64 VR256:$src2), timm:$src3),7593 (VPBLENDDYrri VR256:$src1, VR256:$src2, (BlendScaleImm4 timm:$src3))>;7594def : Pat<(X86Blendi VR256:$src1, (loadv4i64 addr:$src2), timm:$src3),7595 (VPBLENDDYrmi VR256:$src1, addr:$src2, (BlendScaleImm4 timm:$src3))>;7596def : Pat<(X86Blendi (loadv4i64 addr:$src2), VR256:$src1, timm:$src3),7597 (VPBLENDDYrmi VR256:$src1, addr:$src2, (BlendScaleCommuteImm4 timm:$src3))>;7598 7599def : Pat<(X86Blendi (v2i64 VR128:$src1), (v2i64 VR128:$src2), timm:$src3),7600 (VPBLENDDrri VR128:$src1, VR128:$src2, (BlendScaleImm2to4 timm:$src3))>;7601def : Pat<(X86Blendi VR128:$src1, (loadv2i64 addr:$src2), timm:$src3),7602 (VPBLENDDrmi VR128:$src1, addr:$src2, (BlendScaleImm2to4 timm:$src3))>;7603def : Pat<(X86Blendi (loadv2i64 addr:$src2), VR128:$src1, timm:$src3),7604 (VPBLENDDrmi VR128:$src1, addr:$src2, (BlendScaleCommuteImm2to4 timm:$src3))>;7605}7606 7607// For insertion into the zero index (low half) of a 256-bit vector, it is7608// more efficient to generate a blend with immediate instead of an insert*128.7609// NOTE: We're using FP instructions here, but execution domain fixing should7610// take care of using integer instructions when profitable.7611let Predicates = [HasAVX] in {7612def : Pat<(insert_subvector (v8i32 VR256:$src1), (v4i32 VR128:$src2), (iPTR 0)),7613 (VBLENDPSYrri VR256:$src1,7614 (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)),7615 VR128:$src2, sub_xmm), 0xf)>;7616def : Pat<(insert_subvector (v4i64 VR256:$src1), (v2i64 VR128:$src2), (iPTR 0)),7617 (VBLENDPSYrri VR256:$src1,7618 (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)),7619 VR128:$src2, sub_xmm), 0xf)>;7620def : Pat<(insert_subvector (v16i16 VR256:$src1), (v8i16 VR128:$src2), (iPTR 0)),7621 (VBLENDPSYrri VR256:$src1,7622 (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)),7623 VR128:$src2, sub_xmm), 0xf)>;7624def : Pat<(insert_subvector (v16f16 VR256:$src1), (v8f16 VR128:$src2), (iPTR 0)),7625 (VBLENDPSYrri VR256:$src1,7626 (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)),7627 VR128:$src2, sub_xmm), 0xf)>;7628def : Pat<(insert_subvector (v32i8 VR256:$src1), (v16i8 VR128:$src2), (iPTR 0)),7629 (VBLENDPSYrri VR256:$src1,7630 (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)),7631 VR128:$src2, sub_xmm), 0xf)>;7632 7633def : Pat<(insert_subvector (loadv8i32 addr:$src2), (v4i32 VR128:$src1), (iPTR 0)),7634 (VBLENDPSYrmi (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)),7635 VR128:$src1, sub_xmm), addr:$src2, 0xf0)>;7636def : Pat<(insert_subvector (loadv4i64 addr:$src2), (v2i64 VR128:$src1), (iPTR 0)),7637 (VBLENDPSYrmi (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)),7638 VR128:$src1, sub_xmm), addr:$src2, 0xf0)>;7639def : Pat<(insert_subvector (loadv16i16 addr:$src2), (v8i16 VR128:$src1), (iPTR 0)),7640 (VBLENDPSYrmi (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)),7641 VR128:$src1, sub_xmm), addr:$src2, 0xf0)>;7642def : Pat<(insert_subvector (loadv16f16 addr:$src2), (v8f16 VR128:$src1), (iPTR 0)),7643 (VBLENDPSYrmi (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)),7644 VR128:$src1, sub_xmm), addr:$src2, 0xf0)>;7645def : Pat<(insert_subvector (loadv32i8 addr:$src2), (v16i8 VR128:$src1), (iPTR 0)),7646 (VBLENDPSYrmi (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)),7647 VR128:$src1, sub_xmm), addr:$src2, 0xf0)>;7648}7649 7650//===----------------------------------------------------------------------===//7651// VPBROADCAST - Load from memory and broadcast to all elements of the7652// destination operand7653//7654multiclass avx2_broadcast<bits<8> opc, string OpcodeStr,7655 X86MemOperand x86memop, PatFrag bcast_frag,7656 ValueType OpVT128, ValueType OpVT256, Predicate prd> {7657 let Predicates = [HasAVX2, prd] in {7658 def rr : AVX28I<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),7659 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),7660 [(set VR128:$dst,7661 (OpVT128 (X86VBroadcast (OpVT128 VR128:$src))))]>,7662 Sched<[SchedWriteShuffle.XMM]>, VEX;7663 def rm : AVX28I<opc, MRMSrcMem, (outs VR128:$dst), (ins x86memop:$src),7664 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),7665 [(set VR128:$dst,7666 (OpVT128 (bcast_frag addr:$src)))]>,7667 Sched<[SchedWriteShuffle.XMM.Folded]>, VEX;7668 def Yrr : AVX28I<opc, MRMSrcReg, (outs VR256:$dst), (ins VR128:$src),7669 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),7670 [(set VR256:$dst,7671 (OpVT256 (X86VBroadcast (OpVT128 VR128:$src))))]>,7672 Sched<[WriteShuffle256]>, VEX, VEX_L;7673 def Yrm : AVX28I<opc, MRMSrcMem, (outs VR256:$dst), (ins x86memop:$src),7674 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),7675 [(set VR256:$dst,7676 (OpVT256 (bcast_frag addr:$src)))]>,7677 Sched<[SchedWriteShuffle.XMM.Folded]>, VEX, VEX_L;7678 7679 // Provide aliases for broadcast from the same register class that7680 // automatically does the extract.7681 def : Pat<(OpVT256 (X86VBroadcast (OpVT256 VR256:$src))),7682 (!cast<Instruction>(NAME#"Yrr")7683 (OpVT128 (EXTRACT_SUBREG (OpVT256 VR256:$src),sub_xmm)))>;7684 }7685}7686 7687defm VPBROADCASTB : avx2_broadcast<0x78, "vpbroadcastb", i8mem, X86VBroadcastld8,7688 v16i8, v32i8, NoVLX_Or_NoBWI>;7689defm VPBROADCASTW : avx2_broadcast<0x79, "vpbroadcastw", i16mem, X86VBroadcastld16,7690 v8i16, v16i16, NoVLX_Or_NoBWI>;7691defm VPBROADCASTD : avx2_broadcast<0x58, "vpbroadcastd", i32mem, X86VBroadcastld32,7692 v4i32, v8i32, NoVLX>;7693defm VPBROADCASTQ : avx2_broadcast<0x59, "vpbroadcastq", i64mem, X86VBroadcastld64,7694 v2i64, v4i64, NoVLX>;7695 7696let Predicates = [HasAVX2, NoVLX] in {7697 // Provide fallback in case the load node that is used in the patterns above7698 // is used by additional users, which prevents the pattern selection.7699 def : Pat<(v4f32 (X86VBroadcast FR32:$src)),7700 (VBROADCASTSSrr (v4f32 (COPY_TO_REGCLASS FR32:$src, VR128)))>;7701 def : Pat<(v8f32 (X86VBroadcast FR32:$src)),7702 (VBROADCASTSSYrr (v4f32 (COPY_TO_REGCLASS FR32:$src, VR128)))>;7703 def : Pat<(v4f64 (X86VBroadcast FR64:$src)),7704 (VBROADCASTSDYrr (v2f64 (COPY_TO_REGCLASS FR64:$src, VR128)))>;7705}7706 7707let Predicates = [HasAVX2, NoVLX_Or_NoBWI] in {7708 def : Pat<(v16i8 (X86VBroadcast GR8:$src)),7709 (VPBROADCASTBrr (VMOVDI2PDIrr7710 (i32 (INSERT_SUBREG (i32 (IMPLICIT_DEF)),7711 GR8:$src, sub_8bit))))>;7712 def : Pat<(v32i8 (X86VBroadcast GR8:$src)),7713 (VPBROADCASTBYrr (VMOVDI2PDIrr7714 (i32 (INSERT_SUBREG (i32 (IMPLICIT_DEF)),7715 GR8:$src, sub_8bit))))>;7716 7717 def : Pat<(v8i16 (X86VBroadcast GR16:$src)),7718 (VPBROADCASTWrr (VMOVDI2PDIrr7719 (i32 (INSERT_SUBREG (i32 (IMPLICIT_DEF)),7720 GR16:$src, sub_16bit))))>;7721 def : Pat<(v16i16 (X86VBroadcast GR16:$src)),7722 (VPBROADCASTWYrr (VMOVDI2PDIrr7723 (i32 (INSERT_SUBREG (i32 (IMPLICIT_DEF)),7724 GR16:$src, sub_16bit))))>;7725 7726 def : Pat<(v8f16 (X86VBroadcastld16 addr:$src)),7727 (VPBROADCASTWrm addr:$src)>;7728 def : Pat<(v16f16 (X86VBroadcastld16 addr:$src)),7729 (VPBROADCASTWYrm addr:$src)>;7730 7731 def : Pat<(v8f16 (X86VBroadcast (v8f16 VR128:$src))),7732 (VPBROADCASTWrr VR128:$src)>;7733 def : Pat<(v16f16 (X86VBroadcast (v8f16 VR128:$src))),7734 (VPBROADCASTWYrr VR128:$src)>;7735 7736 def : Pat<(v8f16 (X86VBroadcast (f16 FR16:$src))),7737 (VPBROADCASTWrr (COPY_TO_REGCLASS FR16:$src, VR128))>;7738 def : Pat<(v16f16 (X86VBroadcast (f16 FR16:$src))),7739 (VPBROADCASTWYrr (COPY_TO_REGCLASS FR16:$src, VR128))>;7740}7741let Predicates = [HasAVX2, NoVLX] in {7742 def : Pat<(v4i32 (X86VBroadcast GR32:$src)),7743 (VPBROADCASTDrr (VMOVDI2PDIrr GR32:$src))>;7744 def : Pat<(v8i32 (X86VBroadcast GR32:$src)),7745 (VPBROADCASTDYrr (VMOVDI2PDIrr GR32:$src))>;7746 def : Pat<(v2i64 (X86VBroadcast GR64:$src)),7747 (VPBROADCASTQrr (VMOV64toPQIrr GR64:$src))>;7748 def : Pat<(v4i64 (X86VBroadcast GR64:$src)),7749 (VPBROADCASTQYrr (VMOV64toPQIrr GR64:$src))>;7750}7751 7752// AVX1 broadcast patterns7753let Predicates = [HasAVX1Only] in {7754 def : Pat<(v8i32 (X86VBroadcastld32 addr:$src)),7755 (VBROADCASTSSYrm addr:$src)>;7756 def : Pat<(v4i64 (X86VBroadcastld64 addr:$src)),7757 (VBROADCASTSDYrm addr:$src)>;7758 def : Pat<(v4i32 (X86VBroadcastld32 addr:$src)),7759 (VBROADCASTSSrm addr:$src)>;7760 def : Pat<(v2i64 (X86VBroadcastld64 addr:$src)),7761 (VMOVDDUPrm addr:$src)>;7762}7763 7764 // Provide fallback in case the load node that is used in the patterns above7765 // is used by additional users, which prevents the pattern selection.7766let Predicates = [HasAVX, NoVLX] in {7767 // 128bit broadcasts:7768 def : Pat<(v2f64 (X86VBroadcast f64:$src)),7769 (VMOVDDUPrr (v2f64 (COPY_TO_REGCLASS FR64:$src, VR128)))>;7770 def : Pat<(v2f64 (X86VBroadcastld64 addr:$src)),7771 (VMOVDDUPrm addr:$src)>;7772 7773 def : Pat<(v2f64 (X86VBroadcast v2f64:$src)),7774 (VMOVDDUPrr VR128:$src)>;7775}7776 7777let Predicates = [HasAVX1Only] in {7778 def : Pat<(v4f32 (X86VBroadcast FR32:$src)),7779 (VPERMILPSri (v4f32 (COPY_TO_REGCLASS FR32:$src, VR128)), 0)>;7780 def : Pat<(v8f32 (X86VBroadcast FR32:$src)),7781 (VINSERTF128rri (INSERT_SUBREG (v8f32 (IMPLICIT_DEF)),7782 (v4f32 (VPERMILPSri (v4f32 (COPY_TO_REGCLASS FR32:$src, VR128)), 0)), sub_xmm),7783 (v4f32 (VPERMILPSri (v4f32 (COPY_TO_REGCLASS FR32:$src, VR128)), 0)), 1)>;7784 def : Pat<(v8f32 (X86VBroadcast v4f32:$src)),7785 (VINSERTF128rri (INSERT_SUBREG (v8f32 (IMPLICIT_DEF)),7786 (v4f32 (VPERMILPSri VR128:$src, 0)), sub_xmm),7787 (v4f32 (VPERMILPSri VR128:$src, 0)), 1)>;7788 def : Pat<(v4f64 (X86VBroadcast FR64:$src)),7789 (VINSERTF128rri (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)),7790 (v2f64 (VMOVDDUPrr (v2f64 (COPY_TO_REGCLASS FR64:$src, VR128)))), sub_xmm),7791 (v2f64 (VMOVDDUPrr (v2f64 (COPY_TO_REGCLASS FR64:$src, VR128)))), 1)>;7792 def : Pat<(v4f64 (X86VBroadcast v2f64:$src)),7793 (VINSERTF128rri (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)),7794 (v2f64 (VMOVDDUPrr VR128:$src)), sub_xmm),7795 (v2f64 (VMOVDDUPrr VR128:$src)), 1)>;7796 7797 def : Pat<(v4i32 (X86VBroadcast GR32:$src)),7798 (VPSHUFDri (VMOVDI2PDIrr GR32:$src), 0)>;7799 def : Pat<(v8i32 (X86VBroadcast GR32:$src)),7800 (VINSERTF128rri (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)),7801 (v4i32 (VPSHUFDri (VMOVDI2PDIrr GR32:$src), 0)), sub_xmm),7802 (v4i32 (VPSHUFDri (VMOVDI2PDIrr GR32:$src), 0)), 1)>;7803 def : Pat<(v4i64 (X86VBroadcast GR64:$src)),7804 (VINSERTF128rri (INSERT_SUBREG (v4i64 (IMPLICIT_DEF)),7805 (v4i32 (VPSHUFDri (VMOV64toPQIrr GR64:$src), 0x44)), sub_xmm),7806 (v4i32 (VPSHUFDri (VMOV64toPQIrr GR64:$src), 0x44)), 1)>;7807 7808 def : Pat<(v2i64 (X86VBroadcast i64:$src)),7809 (VPSHUFDri (VMOV64toPQIrr GR64:$src), 0x44)>;7810 def : Pat<(v4i64 (X86VBroadcast v2i64:$src)),7811 (VINSERTF128rri (INSERT_SUBREG (v4i64 (IMPLICIT_DEF)),7812 (v2i64 (VPSHUFDri VR128:$src, 0x44)), sub_xmm),7813 (v2i64 (VPSHUFDri VR128:$src, 0x44)), 1)>;7814}7815 7816//===----------------------------------------------------------------------===//7817// VPERM - Permute instructions7818//7819 7820multiclass avx2_perm<bits<8> opc, string OpcodeStr,7821 ValueType OpVT, X86FoldableSchedWrite Sched,7822 X86MemOperand memOp> {7823 let Predicates = [HasAVX2, NoVLX] in {7824 def Yrr : AVX28I<opc, MRMSrcReg, (outs VR256:$dst),7825 (ins VR256:$src1, VR256:$src2),7826 !strconcat(OpcodeStr,7827 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),7828 [(set VR256:$dst,7829 (OpVT (X86VPermv VR256:$src1, VR256:$src2)))]>,7830 Sched<[Sched]>, VEX, VVVV, VEX_L;7831 def Yrm : AVX28I<opc, MRMSrcMem, (outs VR256:$dst),7832 (ins VR256:$src1, memOp:$src2),7833 !strconcat(OpcodeStr,7834 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),7835 [(set VR256:$dst,7836 (OpVT (X86VPermv VR256:$src1,7837 (load addr:$src2))))]>,7838 Sched<[Sched.Folded, Sched.ReadAfterFold]>, VEX, VVVV, VEX_L;7839 }7840}7841 7842defm VPERMD : avx2_perm<0x36, "vpermd", v8i32, WriteVarShuffle256, i256mem>;7843let ExeDomain = SSEPackedSingle in7844defm VPERMPS : avx2_perm<0x16, "vpermps", v8f32, WriteFVarShuffle256, f256mem>;7845 7846multiclass avx2_perm_imm<bits<8> opc, string OpcodeStr, PatFrag mem_frag,7847 ValueType OpVT, X86FoldableSchedWrite Sched,7848 X86MemOperand memOp> {7849 let Predicates = [HasAVX2, NoVLX] in {7850 def Yri : AVX2AIi8<opc, MRMSrcReg, (outs VR256:$dst),7851 (ins VR256:$src1, u8imm:$src2),7852 !strconcat(OpcodeStr,7853 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),7854 [(set VR256:$dst,7855 (OpVT (X86VPermi VR256:$src1, (i8 timm:$src2))))]>,7856 Sched<[Sched]>, VEX, VEX_L;7857 def Ymi : AVX2AIi8<opc, MRMSrcMem, (outs VR256:$dst),7858 (ins memOp:$src1, u8imm:$src2),7859 !strconcat(OpcodeStr,7860 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),7861 [(set VR256:$dst,7862 (OpVT (X86VPermi (mem_frag addr:$src1),7863 (i8 timm:$src2))))]>,7864 Sched<[Sched.Folded, Sched.ReadAfterFold]>, VEX, VEX_L;7865 }7866}7867 7868defm VPERMQ : avx2_perm_imm<0x00, "vpermq", loadv4i64, v4i64,7869 WriteShuffle256, i256mem>, REX_W;7870let ExeDomain = SSEPackedDouble in7871defm VPERMPD : avx2_perm_imm<0x01, "vpermpd", loadv4f64, v4f64,7872 WriteFShuffle256, f256mem>, REX_W;7873 7874//===----------------------------------------------------------------------===//7875// VPERM2I128 - Permute Integer vector Values in 128-bit chunks7876//7877let isCommutable = 1 in7878def VPERM2I128rri : AVX2AIi8<0x46, MRMSrcReg, (outs VR256:$dst),7879 (ins VR256:$src1, VR256:$src2, u8imm:$src3),7880 "vperm2i128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", []>,7881 Sched<[WriteShuffle256]>, VEX, VVVV, VEX_L;7882def VPERM2I128rmi : AVX2AIi8<0x46, MRMSrcMem, (outs VR256:$dst),7883 (ins VR256:$src1, f256mem:$src2, u8imm:$src3),7884 "vperm2i128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", []>,7885 Sched<[WriteShuffle256.Folded, WriteShuffle256.ReadAfterFold]>, VEX, VVVV, VEX_L;7886 7887let Predicates = [HasAVX2] in {7888 defm : vperm2x128_lowering<"VPERM2I128", v4i64, loadv4i64>;7889 defm : vperm2x128_lowering<"VPERM2I128", v8i32, loadv8i32>;7890 defm : vperm2x128_lowering<"VPERM2I128", v16i16, loadv16i16>;7891 defm : vperm2x128_lowering<"VPERM2I128", v16f16, loadv16f16>;7892 defm : vperm2x128_lowering<"VPERM2I128", v32i8, loadv32i8>;7893 defm : vperm2x128_lowering<"VPERM2I128", v32i8, loadv32i8>;7894}7895 7896//===----------------------------------------------------------------------===//7897// VINSERTI128 - Insert packed integer values7898//7899let hasSideEffects = 0 in {7900def VINSERTI128rri : AVX2AIi8<0x38, MRMSrcReg, (outs VR256:$dst),7901 (ins VR256:$src1, VR128:$src2, u8imm:$src3),7902 "vinserti128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",7903 []>, Sched<[WriteShuffle256]>, VEX, VVVV, VEX_L;7904let mayLoad = 1 in7905def VINSERTI128rmi : AVX2AIi8<0x38, MRMSrcMem, (outs VR256:$dst),7906 (ins VR256:$src1, i128mem:$src2, u8imm:$src3),7907 "vinserti128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",7908 []>, Sched<[WriteShuffle256.Folded, WriteShuffle256.ReadAfterFold]>, VEX, VVVV, VEX_L;7909}7910 7911let Predicates = [HasAVX2, NoVLX] in {7912 defm : vinsert_lowering<"VINSERTI128", "VPERM2I128", v2i64, v4i64, loadv2i64, loadv4i64>;7913 defm : vinsert_lowering<"VINSERTI128", "VPERM2I128", v4i32, v8i32, loadv4i32, loadv8i32>;7914 defm : vinsert_lowering<"VINSERTI128", "VPERM2I128", v8i16, v16i16, loadv8i16, loadv16i16>;7915 defm : vinsert_lowering<"VINSERTI128", "VPERM2I128", v8f16, v16f16, loadv8f16, loadv16f16>;7916 defm : vinsert_lowering<"VINSERTI128", "VPERM2I128", v16i8, v32i8, loadv16i8, loadv32i8>;7917 defm : vinsert_lowering<"VINSERTI128", "VPERM2I128", v16i8, v32i8, loadv16i8, loadv32i8>;7918}7919 7920let Predicates = [HasAVXNECONVERT, NoVLX] in7921 defm : vinsert_lowering<"VINSERTI128", "VPERM2I128", v8bf16, v16bf16, loadv8bf16, loadv16bf16>;7922 7923//===----------------------------------------------------------------------===//7924// VEXTRACTI128 - Extract packed integer values7925//7926def VEXTRACTI128rri : AVX2AIi8<0x39, MRMDestReg, (outs VR128:$dst),7927 (ins VR256:$src1, u8imm:$src2),7928 "vextracti128\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,7929 Sched<[WriteShuffle256]>, VEX, VEX_L;7930let hasSideEffects = 0, mayStore = 1 in7931def VEXTRACTI128mri : AVX2AIi8<0x39, MRMDestMem, (outs),7932 (ins i128mem:$dst, VR256:$src1, u8imm:$src2),7933 "vextracti128\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,7934 Sched<[SchedWriteVecMoveLS.XMM.MR]>, VEX, VEX_L;7935 7936let Predicates = [HasAVX2, NoVLX] in {7937 defm : vextract_lowering<"VEXTRACTI128", v4i64, v2i64>;7938 defm : vextract_lowering<"VEXTRACTI128", v8i32, v4i32>;7939 defm : vextract_lowering<"VEXTRACTI128", v16i16, v8i16>;7940 defm : vextract_lowering<"VEXTRACTI128", v16f16, v8f16>;7941 defm : vextract_lowering<"VEXTRACTI128", v32i8, v16i8>;7942 defm : vextract_lowering<"VEXTRACTI128", v32i8, v16i8>;7943}7944 7945let Predicates = [HasAVXNECONVERT, NoVLX] in7946 defm : vextract_lowering<"VEXTRACTI128", v16bf16, v8bf16>;7947 7948//===----------------------------------------------------------------------===//7949// VPMASKMOV - Conditional SIMD Integer Packed Loads and Stores7950//7951multiclass avx2_pmovmask<string OpcodeStr,7952 Intrinsic IntLd128, Intrinsic IntLd256,7953 Intrinsic IntSt128, Intrinsic IntSt256,7954 X86SchedWriteMaskMove schedX,7955 X86SchedWriteMaskMove schedY> {7956 def rm : AVX28I<0x8c, MRMSrcMem, (outs VR128:$dst),7957 (ins VR128:$src1, i128mem:$src2),7958 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),7959 [(set VR128:$dst, (IntLd128 addr:$src2, VR128:$src1))]>,7960 VEX, VVVV, Sched<[schedX.RM]>;7961 def Yrm : AVX28I<0x8c, MRMSrcMem, (outs VR256:$dst),7962 (ins VR256:$src1, i256mem:$src2),7963 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),7964 [(set VR256:$dst, (IntLd256 addr:$src2, VR256:$src1))]>,7965 VEX, VVVV, VEX_L, Sched<[schedY.RM]>;7966 def mr : AVX28I<0x8e, MRMDestMem, (outs),7967 (ins i128mem:$dst, VR128:$src1, VR128:$src2),7968 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),7969 [(IntSt128 addr:$dst, VR128:$src1, VR128:$src2)]>,7970 VEX, VVVV, Sched<[schedX.MR]>;7971 def Ymr : AVX28I<0x8e, MRMDestMem, (outs),7972 (ins i256mem:$dst, VR256:$src1, VR256:$src2),7973 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),7974 [(IntSt256 addr:$dst, VR256:$src1, VR256:$src2)]>,7975 VEX, VVVV, VEX_L, Sched<[schedY.MR]>;7976}7977 7978defm VPMASKMOVD : avx2_pmovmask<"vpmaskmovd",7979 int_x86_avx2_maskload_d,7980 int_x86_avx2_maskload_d_256,7981 int_x86_avx2_maskstore_d,7982 int_x86_avx2_maskstore_d_256,7983 WriteVecMaskMove32, WriteVecMaskMove32Y>;7984defm VPMASKMOVQ : avx2_pmovmask<"vpmaskmovq",7985 int_x86_avx2_maskload_q,7986 int_x86_avx2_maskload_q_256,7987 int_x86_avx2_maskstore_q,7988 int_x86_avx2_maskstore_q_256,7989 WriteVecMaskMove64, WriteVecMaskMove64Y>, REX_W;7990 7991multiclass maskmov_lowering<string InstrStr, RegisterClass RC, ValueType VT,7992 ValueType MaskVT> {7993 // masked store7994 def: Pat<(masked_store (VT RC:$src), addr:$ptr, (MaskVT RC:$mask)),7995 (!cast<Instruction>(InstrStr#"mr") addr:$ptr, RC:$mask, RC:$src)>;7996 // masked load7997 def: Pat<(VT (masked_load addr:$ptr, (MaskVT RC:$mask), undef)),7998 (!cast<Instruction>(InstrStr#"rm") RC:$mask, addr:$ptr)>;7999 def: Pat<(VT (masked_load addr:$ptr, (MaskVT RC:$mask),8000 (VT immAllZerosV))),8001 (!cast<Instruction>(InstrStr#"rm") RC:$mask, addr:$ptr)>;8002}8003let Predicates = [HasAVX] in {8004 defm : maskmov_lowering<"VMASKMOVPS", VR128, v4f32, v4i32>;8005 defm : maskmov_lowering<"VMASKMOVPD", VR128, v2f64, v2i64>;8006 defm : maskmov_lowering<"VMASKMOVPSY", VR256, v8f32, v8i32>;8007 defm : maskmov_lowering<"VMASKMOVPDY", VR256, v4f64, v4i64>;8008}8009let Predicates = [HasAVX1Only] in {8010 // load/store i32/i64 not supported use ps/pd version8011 defm : maskmov_lowering<"VMASKMOVPSY", VR256, v8i32, v8i32>;8012 defm : maskmov_lowering<"VMASKMOVPDY", VR256, v4i64, v4i64>;8013 defm : maskmov_lowering<"VMASKMOVPS", VR128, v4i32, v4i32>;8014 defm : maskmov_lowering<"VMASKMOVPD", VR128, v2i64, v2i64>;8015}8016let Predicates = [HasAVX2] in {8017 defm : maskmov_lowering<"VPMASKMOVDY", VR256, v8i32, v8i32>;8018 defm : maskmov_lowering<"VPMASKMOVQY", VR256, v4i64, v4i64>;8019 defm : maskmov_lowering<"VPMASKMOVD", VR128, v4i32, v4i32>;8020 defm : maskmov_lowering<"VPMASKMOVQ", VR128, v2i64, v2i64>;8021}8022 8023//===----------------------------------------------------------------------===//8024// Variable Bit Shifts8025//8026multiclass avx2_var_shift<bits<8> opc, string OpcodeStr, SDNode OpNode,8027 ValueType vt128, ValueType vt256> {8028 def rr : AVX28I<opc, MRMSrcReg, (outs VR128:$dst),8029 (ins VR128:$src1, VR128:$src2),8030 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),8031 [(set VR128:$dst,8032 (vt128 (OpNode VR128:$src1, (vt128 VR128:$src2))))]>,8033 VEX, VVVV, Sched<[SchedWriteVarVecShift.XMM]>;8034 def rm : AVX28I<opc, MRMSrcMem, (outs VR128:$dst),8035 (ins VR128:$src1, i128mem:$src2),8036 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),8037 [(set VR128:$dst,8038 (vt128 (OpNode VR128:$src1,8039 (vt128 (load addr:$src2)))))]>,8040 VEX, VVVV, Sched<[SchedWriteVarVecShift.XMM.Folded,8041 SchedWriteVarVecShift.XMM.ReadAfterFold]>;8042 def Yrr : AVX28I<opc, MRMSrcReg, (outs VR256:$dst),8043 (ins VR256:$src1, VR256:$src2),8044 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),8045 [(set VR256:$dst,8046 (vt256 (OpNode VR256:$src1, (vt256 VR256:$src2))))]>,8047 VEX, VVVV, VEX_L, Sched<[SchedWriteVarVecShift.YMM]>;8048 def Yrm : AVX28I<opc, MRMSrcMem, (outs VR256:$dst),8049 (ins VR256:$src1, i256mem:$src2),8050 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),8051 [(set VR256:$dst,8052 (vt256 (OpNode VR256:$src1,8053 (vt256 (load addr:$src2)))))]>,8054 VEX, VVVV, VEX_L, Sched<[SchedWriteVarVecShift.YMM.Folded,8055 SchedWriteVarVecShift.YMM.ReadAfterFold]>;8056}8057 8058let Predicates = [HasAVX2, NoVLX] in {8059 defm VPSLLVD : avx2_var_shift<0x47, "vpsllvd", X86vshlv, v4i32, v8i32>;8060 defm VPSLLVQ : avx2_var_shift<0x47, "vpsllvq", X86vshlv, v2i64, v4i64>, REX_W;8061 defm VPSRLVD : avx2_var_shift<0x45, "vpsrlvd", X86vsrlv, v4i32, v8i32>;8062 defm VPSRLVQ : avx2_var_shift<0x45, "vpsrlvq", X86vsrlv, v2i64, v4i64>, REX_W;8063 defm VPSRAVD : avx2_var_shift<0x46, "vpsravd", X86vsrav, v4i32, v8i32>;8064}8065 8066//===----------------------------------------------------------------------===//8067// VGATHER - GATHER Operations8068 8069// FIXME: Improve scheduling of gather instructions.8070multiclass avx2_gather<bits<8> opc, string OpcodeStr, RegisterClass RC256,8071 X86MemOperand memop128, X86MemOperand memop256> {8072let mayLoad = 1, hasSideEffects = 0 in {8073 def rm : AVX28I<opc, MRMSrcMem4VOp3, (outs VR128:$dst, VR128:$mask_wb),8074 (ins VR128:$src1, memop128:$src2, VR128:$mask),8075 !strconcat(OpcodeStr,8076 "\t{$mask, $src2, $dst|$dst, $src2, $mask}"),8077 []>, VEX, Sched<[WriteLoad, WriteVecMaskedGatherWriteback]>;8078 def Yrm : AVX28I<opc, MRMSrcMem4VOp3, (outs RC256:$dst, RC256:$mask_wb),8079 (ins RC256:$src1, memop256:$src2, RC256:$mask),8080 !strconcat(OpcodeStr,8081 "\t{$mask, $src2, $dst|$dst, $src2, $mask}"),8082 []>, VEX, VEX_L, Sched<[WriteLoad, WriteVecMaskedGatherWriteback]>;8083}8084}8085 8086let Predicates = [HasAVX2] in {8087 let mayLoad = 1, hasSideEffects = 0, Constraints8088 = "@earlyclobber $dst,@earlyclobber $mask_wb, $src1 = $dst, $mask = $mask_wb"8089 in {8090 defm VPGATHERDQ : avx2_gather<0x90, "vpgatherdq",8091 VR256, vx64mem, vx64mem>, REX_W;8092 defm VPGATHERQQ : avx2_gather<0x91, "vpgatherqq",8093 VR256, vx64mem, vy64mem>, REX_W;8094 defm VPGATHERDD : avx2_gather<0x90, "vpgatherdd",8095 VR256, vx32mem, vy32mem>;8096 defm VPGATHERQD : avx2_gather<0x91, "vpgatherqd",8097 VR128, vx32mem, vy32mem>;8098 8099 let ExeDomain = SSEPackedDouble in {8100 defm VGATHERDPD : avx2_gather<0x92, "vgatherdpd",8101 VR256, vx64mem, vx64mem>, REX_W;8102 defm VGATHERQPD : avx2_gather<0x93, "vgatherqpd",8103 VR256, vx64mem, vy64mem>, REX_W;8104 }8105 8106 let ExeDomain = SSEPackedSingle in {8107 defm VGATHERDPS : avx2_gather<0x92, "vgatherdps",8108 VR256, vx32mem, vy32mem>;8109 defm VGATHERQPS : avx2_gather<0x93, "vgatherqps",8110 VR128, vx32mem, vy32mem>;8111 }8112 }8113}8114 8115//===----------------------------------------------------------------------===//8116// GFNI instructions8117//===----------------------------------------------------------------------===//8118 8119multiclass GF2P8MULB_rm<string OpcodeStr, ValueType OpVT,8120 RegisterClass RC, PatFrag MemOpFrag,8121 X86MemOperand X86MemOp, X86FoldableSchedWrite sched,8122 bit Is2Addr = 0> {8123 let ExeDomain = SSEPackedInt,8124 AsmString = !if(Is2Addr,8125 OpcodeStr#"\t{$src2, $dst|$dst, $src2}",8126 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}") in {8127 let isCommutable = 1 in8128 def rr : PDI<0xCF, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), "",8129 [(set RC:$dst, (OpVT (X86GF2P8mulb RC:$src1, RC:$src2)))]>,8130 Sched<[sched]>, T8;8131 8132 def rm : PDI<0xCF, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, X86MemOp:$src2), "",8133 [(set RC:$dst, (OpVT (X86GF2P8mulb RC:$src1,8134 (MemOpFrag addr:$src2))))]>,8135 Sched<[sched.Folded, sched.ReadAfterFold]>, T8;8136 }8137}8138 8139multiclass GF2P8AFFINE_rmi<bits<8> Op, string OpStr, ValueType OpVT,8140 SDNode OpNode, RegisterClass RC, PatFrag MemOpFrag,8141 X86MemOperand X86MemOp, X86FoldableSchedWrite sched,8142 bit Is2Addr = 0> {8143 let AsmString = !if(Is2Addr,8144 OpStr#"\t{$src3, $src2, $dst|$dst, $src2, $src3}",8145 OpStr#"\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}") in {8146 def rri : Ii8<Op, MRMSrcReg, (outs RC:$dst),8147 (ins RC:$src1, RC:$src2, u8imm:$src3), "",8148 [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2, timm:$src3)))],8149 SSEPackedInt>, Sched<[sched]>;8150 def rmi : Ii8<Op, MRMSrcMem, (outs RC:$dst),8151 (ins RC:$src1, X86MemOp:$src2, u8imm:$src3), "",8152 [(set RC:$dst, (OpVT (OpNode RC:$src1,8153 (MemOpFrag addr:$src2),8154 timm:$src3)))], SSEPackedInt>,8155 Sched<[sched.Folded, sched.ReadAfterFold]>;8156 }8157}8158 8159multiclass GF2P8AFFINE_common<bits<8> Op, string OpStr, SDNode OpNode> {8160 let Constraints = "$src1 = $dst",8161 Predicates = [HasGFNI, UseSSE2] in8162 defm NAME : GF2P8AFFINE_rmi<Op, OpStr, v16i8, OpNode,8163 VR128, load, i128mem, SchedWriteVecIMul.XMM, 1>;8164 let Predicates = [HasGFNI, HasAVX, NoVLX] in {8165 defm V#NAME : GF2P8AFFINE_rmi<Op, "v"#OpStr, v16i8, OpNode, VR128,8166 load, i128mem, SchedWriteVecIMul.XMM>,8167 VEX, VVVV, REX_W;8168 defm V#NAME#Y : GF2P8AFFINE_rmi<Op, "v"#OpStr, v32i8, OpNode, VR256,8169 load, i256mem, SchedWriteVecIMul.YMM>,8170 VEX, VVVV, VEX_L, REX_W;8171 }8172}8173 8174// GF2P8MULB8175let Constraints = "$src1 = $dst",8176 Predicates = [HasGFNI, UseSSE2] in8177defm GF2P8MULB : GF2P8MULB_rm<"gf2p8mulb", v16i8, VR128, memop,8178 i128mem, SchedWriteVecALU.XMM, 1>;8179let Predicates = [HasGFNI, HasAVX, NoVLX] in {8180 defm VGF2P8MULB : GF2P8MULB_rm<"vgf2p8mulb", v16i8, VR128, load,8181 i128mem, SchedWriteVecALU.XMM>, VEX, VVVV;8182 defm VGF2P8MULBY : GF2P8MULB_rm<"vgf2p8mulb", v32i8, VR256, load,8183 i256mem, SchedWriteVecALU.YMM>, VEX, VVVV, VEX_L;8184}8185// GF2P8AFFINEINVQB, GF2P8AFFINEQB8186let isCommutable = 0 in {8187 defm GF2P8AFFINEINVQB : GF2P8AFFINE_common<0xCF, "gf2p8affineinvqb",8188 X86GF2P8affineinvqb>, TA, PD;8189 defm GF2P8AFFINEQB : GF2P8AFFINE_common<0xCE, "gf2p8affineqb",8190 X86GF2P8affineqb>, TA, PD;8191}8192 8193// AVX-IFMA8194let Predicates = [HasAVXIFMA, NoVLX_Or_NoIFMA], Constraints = "$src1 = $dst" in8195multiclass avx_ifma_rm<bits<8> opc, string OpcodeStr, SDNode OpNode> {8196 // NOTE: The SDNode have the multiply operands first with the add last.8197 // This enables commuted load patterns to be autogenerated by tablegen.8198 let isCommutable = 1 in {8199 def rr : AVX8I<opc, MRMSrcReg, (outs VR128:$dst),8200 (ins VR128:$src1, VR128:$src2, VR128:$src3),8201 !strconcat(OpcodeStr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),8202 [(set VR128:$dst, (v2i64 (OpNode VR128:$src2,8203 VR128:$src3, VR128:$src1)))]>,8204 VEX, VVVV, Sched<[SchedWriteVecIMul.XMM]>;8205 }8206 def rm : AVX8I<opc, MRMSrcMem, (outs VR128:$dst),8207 (ins VR128:$src1, VR128:$src2, i128mem:$src3),8208 !strconcat(OpcodeStr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),8209 [(set VR128:$dst, (v2i64 (OpNode VR128:$src2,8210 (loadv2i64 addr:$src3), VR128:$src1)))]>,8211 VEX, VVVV, Sched<[SchedWriteVecIMul.XMM]>;8212 let isCommutable = 1 in {8213 def Yrr : AVX8I<opc, MRMSrcReg, (outs VR256:$dst),8214 (ins VR256:$src1, VR256:$src2, VR256:$src3),8215 !strconcat(OpcodeStr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),8216 [(set VR256:$dst, (v4i64 (OpNode VR256:$src2,8217 VR256:$src3, VR256:$src1)))]>,8218 VEX, VVVV, VEX_L, Sched<[SchedWriteVecIMul.YMM]>;8219 }8220 def Yrm : AVX8I<opc, MRMSrcMem, (outs VR256:$dst),8221 (ins VR256:$src1, VR256:$src2, i256mem:$src3),8222 !strconcat(OpcodeStr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),8223 [(set VR256:$dst, (v4i64 (OpNode VR256:$src2,8224 (loadv4i64 addr:$src3), VR256:$src1)))]>,8225 VEX, VVVV, VEX_L, Sched<[SchedWriteVecIMul.YMM]>;8226}8227 8228defm VPMADD52HUQ : avx_ifma_rm<0xb5, "vpmadd52huq", x86vpmadd52h>, REX_W, ExplicitVEXPrefix;8229defm VPMADD52LUQ : avx_ifma_rm<0xb4, "vpmadd52luq", x86vpmadd52l>, REX_W, ExplicitVEXPrefix;8230 8231// AVX-VNNI-INT88232let Constraints = "$src1 = $dst" in8233multiclass avx_dotprod_rm<bits<8> Opc, string OpcodeStr, ValueType OpVT,8234 RegisterClass RC, PatFrag MemOpFrag,8235 X86MemOperand X86memop, SDNode OpNode,8236 X86FoldableSchedWrite Sched,8237 bit IsCommutable> {8238 let isCommutable = IsCommutable in8239 def rr : I<Opc, MRMSrcReg, (outs RC:$dst),8240 (ins RC:$src1, RC:$src2, RC:$src3),8241 !strconcat(OpcodeStr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),8242 [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2, RC:$src3)))]>,8243 VEX, VVVV, Sched<[Sched]>;8244 def rm : I<Opc, MRMSrcMem, (outs RC:$dst),8245 (ins RC:$src1, RC:$src2, X86memop:$src3),8246 !strconcat(OpcodeStr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),8247 [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2,8248 (MemOpFrag addr:$src3))))]>,8249 VEX, VVVV, Sched<[Sched.Folded, Sched.ReadAfterFold]>;8250}8251 8252let Predicates = [HasAVXVNNIINT8] in {8253 defm VPDPBSSD : avx_dotprod_rm<0x50,"vpdpbssd", v4i32, VR128, loadv4i32,8254 i128mem, X86vpdpbssd, SchedWriteVecIMul.XMM,8255 1>, T8, XD;8256 defm VPDPBSSDY : avx_dotprod_rm<0x50,"vpdpbssd", v8i32, VR256, loadv8i32,8257 i256mem, X86vpdpbssd, SchedWriteVecIMul.YMM,8258 1>, VEX_L, T8, XD;8259 defm VPDPBUUD : avx_dotprod_rm<0x50,"vpdpbuud", v4i32, VR128, loadv4i32,8260 i128mem, X86vpdpbuud, SchedWriteVecIMul.XMM,8261 1>, T8;8262 defm VPDPBUUDY : avx_dotprod_rm<0x50,"vpdpbuud", v8i32, VR256, loadv8i32,8263 i256mem, X86vpdpbuud, SchedWriteVecIMul.YMM,8264 1>, VEX_L, T8;8265 defm VPDPBSSDS : avx_dotprod_rm<0x51,"vpdpbssds", v4i32, VR128, loadv4i32,8266 i128mem, X86vpdpbssds, SchedWriteVecIMul.XMM,8267 1>, T8, XD;8268 defm VPDPBSSDSY : avx_dotprod_rm<0x51,"vpdpbssds", v8i32, VR256, loadv8i32,8269 i256mem, X86vpdpbssds, SchedWriteVecIMul.YMM,8270 1>, VEX_L, T8, XD;8271 defm VPDPBUUDS : avx_dotprod_rm<0x51,"vpdpbuuds", v4i32, VR128, loadv4i32,8272 i128mem, X86vpdpbuuds, SchedWriteVecIMul.XMM,8273 1>, T8;8274 defm VPDPBUUDSY : avx_dotprod_rm<0x51,"vpdpbuuds", v8i32, VR256, loadv8i32,8275 i256mem, X86vpdpbuuds, SchedWriteVecIMul.YMM,8276 1>, VEX_L, T8;8277 defm VPDPBSUD : avx_dotprod_rm<0x50,"vpdpbsud", v4i32, VR128, loadv4i32,8278 i128mem, X86vpdpbsud, SchedWriteVecIMul.XMM,8279 0>, T8, XS;8280 defm VPDPBSUDY : avx_dotprod_rm<0x50,"vpdpbsud", v8i32, VR256, loadv8i32,8281 i256mem, X86vpdpbsud, SchedWriteVecIMul.YMM,8282 0>, VEX_L, T8, XS;8283 defm VPDPBSUDS : avx_dotprod_rm<0x51,"vpdpbsuds", v4i32, VR128, loadv4i32,8284 i128mem, X86vpdpbsuds, SchedWriteVecIMul.XMM,8285 0>, T8, XS;8286 defm VPDPBSUDSY : avx_dotprod_rm<0x51,"vpdpbsuds", v8i32, VR256, loadv8i32,8287 i256mem, X86vpdpbsuds, SchedWriteVecIMul.YMM,8288 0>, VEX_L, T8, XS;8289}8290 8291// AVX-NE-CONVERT8292multiclass AVX_NE_CONVERT_BASE<bits<8> Opcode, string OpcodeStr,8293 X86MemOperand MemOp128, X86MemOperand MemOp256> {8294 def rm : I<Opcode, MRMSrcMem, (outs VR128:$dst), (ins MemOp128:$src),8295 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),8296 [(set VR128:$dst,8297 (!cast<Intrinsic>("int_x86_"#OpcodeStr#"128") addr:$src))]>,8298 Sched<[WriteCvtPH2PS]>, VEX;8299 def Yrm : I<Opcode, MRMSrcMem, (outs VR256:$dst), (ins MemOp256:$src),8300 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),8301 [(set VR256:$dst,8302 (!cast<Intrinsic>("int_x86_"#OpcodeStr#"256") addr:$src))]>,8303 Sched<[WriteCvtPH2PSY]>, VEX, VEX_L;8304}8305 8306multiclass VCVTNEPS2BF16_BASE {8307 def rr : I<0x72, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),8308 "vcvtneps2bf16\t{$src, $dst|$dst, $src}",8309 [(set VR128:$dst, (int_x86_vcvtneps2bf16128 VR128:$src))]>,8310 Sched<[WriteCvtPH2PS]>;8311 def rm : I<0x72, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),8312 "vcvtneps2bf16{x}\t{$src, $dst|$dst, $src}",8313 [(set VR128:$dst, (int_x86_vcvtneps2bf16128 (loadv4f32 addr:$src)))]>,8314 Sched<[WriteCvtPH2PS]>;8315 def Yrr : I<0x72, MRMSrcReg, (outs VR128:$dst), (ins VR256:$src),8316 "vcvtneps2bf16\t{$src, $dst|$dst, $src}",8317 [(set VR128:$dst, (int_x86_vcvtneps2bf16256 VR256:$src))]>,8318 Sched<[WriteCvtPH2PSY]>, VEX_L;8319 def Yrm : I<0x72, MRMSrcMem, (outs VR128:$dst), (ins f256mem:$src),8320 "vcvtneps2bf16{y}\t{$src, $dst|$dst, $src}",8321 [(set VR128:$dst, (int_x86_vcvtneps2bf16256 (loadv8f32 addr:$src)))]>,8322 Sched<[WriteCvtPH2PSY]>, VEX_L;8323}8324 8325let Predicates = [HasAVXNECONVERT] in {8326 defm VBCSTNEBF162PS : AVX_NE_CONVERT_BASE<0xb1, "vbcstnebf162ps", f16mem,8327 f16mem>, T8, XS;8328 defm VBCSTNESH2PS : AVX_NE_CONVERT_BASE<0xb1, "vbcstnesh2ps", f16mem, f16mem>,8329 T8, PD;8330 defm VCVTNEEBF162PS : AVX_NE_CONVERT_BASE<0xb0, "vcvtneebf162ps", f128mem,8331 f256mem>, T8, XS;8332 defm VCVTNEEPH2PS : AVX_NE_CONVERT_BASE<0xb0, "vcvtneeph2ps", f128mem,8333 f256mem>, T8, PD;8334 defm VCVTNEOBF162PS : AVX_NE_CONVERT_BASE<0xb0, "vcvtneobf162ps", f128mem,8335 f256mem>, T8, XD;8336 defm VCVTNEOPH2PS : AVX_NE_CONVERT_BASE<0xb0, "vcvtneoph2ps", f128mem,8337 f256mem>, T8;8338 defm VCVTNEPS2BF16 : VCVTNEPS2BF16_BASE, VEX, T8, XS, ExplicitVEXPrefix;8339 8340 def : Pat<(v8bf16 (X86cvtneps2bf16 (v4f32 VR128:$src))),8341 (VCVTNEPS2BF16rr VR128:$src)>;8342 def : Pat<(v8bf16 (X86cvtneps2bf16 (loadv4f32 addr:$src))),8343 (VCVTNEPS2BF16rm addr:$src)>;8344 def : Pat<(v8bf16 (X86vfpround (v8f32 VR256:$src))),8345 (VCVTNEPS2BF16Yrr VR256:$src)>;8346 def : Pat<(v8bf16 (X86vfpround (loadv8f32 addr:$src))),8347 (VCVTNEPS2BF16Yrm addr:$src)>;8348}8349 8350def : InstAlias<"vcvtneps2bf16x\t{$src, $dst|$dst, $src}",8351 (VCVTNEPS2BF16rr VR128:$dst, VR128:$src), 0, "att">;8352def : InstAlias<"vcvtneps2bf16y\t{$src, $dst|$dst, $src}",8353 (VCVTNEPS2BF16Yrr VR128:$dst, VR256:$src), 0, "att">;8354 8355// FIXME: Is there a better scheduler class for SHA512 than WriteVecIMul?8356let Predicates = [HasSHA512], Constraints = "$src1 = $dst" in {8357def VSHA512MSG1rr : I<0xcc, MRMSrcReg, (outs VR256:$dst),8358 (ins VR256:$src1, VR128:$src2),8359 "vsha512msg1\t{$src2, $dst|$dst, $src2}",8360 [(set VR256:$dst,8361 (int_x86_vsha512msg1 VR256:$src1, VR128:$src2))]>, VEX_L,8362 VEX, T8, XD, Sched<[WriteVecIMul]>;8363def VSHA512MSG2rr : I<0xcd, MRMSrcReg, (outs VR256:$dst),8364 (ins VR256:$src1, VR256:$src2),8365 "vsha512msg2\t{$src2, $dst|$dst, $src2}",8366 [(set VR256:$dst,8367 (int_x86_vsha512msg2 VR256:$src1, VR256:$src2))]>, VEX_L,8368 VEX, T8, XD, Sched<[WriteVecIMul]>;8369def VSHA512RNDS2rr : I<0xcb, MRMSrcReg, (outs VR256:$dst),8370 (ins VR256:$src1, VR256:$src2, VR128:$src3),8371 "vsha512rnds2\t{$src3, $src2, $dst|$dst, $src2, $src3}",8372 [(set VR256:$dst,8373 (int_x86_vsha512rnds2 VR256:$src1, VR256:$src2, VR128:$src3))]>,8374 VEX_L, VEX, VVVV, T8, XD, Sched<[WriteVecIMul]>;8375}8376 8377// FIXME: Is there a better scheduler class for SM3 than WriteVecIMul?8378let Predicates = [HasSM3], Constraints = "$src1 = $dst" in {8379 multiclass SM3_Base<string OpStr> {8380 def rr : I<0xda, MRMSrcReg, (outs VR128:$dst),8381 (ins VR128:$src1, VR128:$src2, VR128:$src3),8382 !strconcat(OpStr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),8383 [(set VR128:$dst,8384 (!cast<Intrinsic>("int_x86_"#OpStr) VR128:$src1,8385 VR128:$src2, VR128:$src3))]>,8386 Sched<[WriteVecIMul]>, VEX, VVVV;8387 def rm : I<0xda, MRMSrcMem, (outs VR128:$dst),8388 (ins VR128:$src1, VR128:$src2, i128mem:$src3),8389 !strconcat(OpStr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),8390 [(set VR128:$dst,8391 (!cast<Intrinsic>("int_x86_"#OpStr) VR128:$src1,8392 VR128:$src2, (loadv4i32 addr:$src3)))]>,8393 Sched<[WriteVecIMul]>, VEX, VVVV;8394 }8395 8396 multiclass VSM3RNDS2_Base {8397 def rri : Ii8<0xde, MRMSrcReg, (outs VR128:$dst),8398 (ins VR128:$src1, VR128:$src2, VR128:$src3, i32u8imm:$src4),8399 "vsm3rnds2\t{$src4, $src3, $src2, $dst|$dst, $src2, $src3, $src4}",8400 [(set VR128:$dst,8401 (int_x86_vsm3rnds2 VR128:$src1,8402 VR128:$src2, VR128:$src3, timm:$src4))]>,8403 Sched<[WriteVecIMul]>;8404 def rmi : Ii8<0xde, MRMSrcMem, (outs VR128:$dst),8405 (ins VR128:$src1, VR128:$src2, i128mem:$src3, i32u8imm:$src4),8406 "vsm3rnds2\t{$src4, $src3, $src2, $dst|$dst, $src2, $src3, $src4}",8407 [(set VR128:$dst,8408 (int_x86_vsm3rnds2 VR128:$src1,8409 VR128:$src2, (loadv4i32 addr:$src3), timm:$src4))]>,8410 Sched<[WriteVecIMul]>;8411 }8412}8413 8414defm VSM3MSG1 : SM3_Base<"vsm3msg1">, T8;8415defm VSM3MSG2 : SM3_Base<"vsm3msg2">, T8, PD;8416defm VSM3RNDS2 : VSM3RNDS2_Base, VEX, VVVV, TA, PD;8417 8418// FIXME: Is there a better scheduler class for SM4 than WriteVecIMul?8419let Predicates = [HasSM4] in {8420 multiclass SM4_Base<string OpStr, RegisterClass RC, string VL,8421 PatFrag LD, X86MemOperand MemOp> {8422 def rr : I<0xda, MRMSrcReg, (outs RC:$dst),8423 (ins RC:$src1, RC:$src2),8424 !strconcat(OpStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),8425 [(set RC:$dst, (!cast<Intrinsic>("int_x86_"#OpStr#VL) RC:$src1,8426 RC:$src2))]>,8427 Sched<[WriteVecIMul]>;8428 def rm : I<0xda, MRMSrcMem, (outs RC:$dst),8429 (ins RC:$src1, MemOp:$src2),8430 !strconcat(OpStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),8431 [(set RC:$dst, (!cast<Intrinsic>("int_x86_"#OpStr#VL) RC:$src1,8432 (LD addr:$src2)))]>,8433 Sched<[WriteVecIMul]>;8434 }8435}8436 8437defm VSM4KEY4 : SM4_Base<"vsm4key4", VR128, "128", loadv4i32, i128mem>, T8, XS, VEX, VVVV;8438defm VSM4KEY4Y : SM4_Base<"vsm4key4", VR256, "256", loadv8i32, i256mem>, T8, XS, VEX_L, VEX, VVVV;8439defm VSM4RNDS4 : SM4_Base<"vsm4rnds4", VR128, "128", loadv4i32, i128mem>, T8, XD, VEX, VVVV;8440defm VSM4RNDS4Y : SM4_Base<"vsm4rnds4", VR256, "256", loadv8i32, i256mem>, T8, XD, VEX_L, VEX, VVVV;8441 8442let Predicates = [HasAVXVNNIINT16] in {8443 defm VPDPWSUD : avx_dotprod_rm<0xd2,"vpdpwsud", v4i32, VR128, loadv4i32,8444 i128mem, X86vpdpwsud, SchedWriteVecIMul.XMM,8445 0>, T8, XS;8446 defm VPDPWSUDY : avx_dotprod_rm<0xd2,"vpdpwsud", v8i32, VR256, loadv8i32,8447 i256mem, X86vpdpwsud, SchedWriteVecIMul.YMM,8448 0>, VEX_L, T8, XS;8449 defm VPDPWSUDS : avx_dotprod_rm<0xd3,"vpdpwsuds", v4i32, VR128, loadv4i32,8450 i128mem, X86vpdpwsuds, SchedWriteVecIMul.XMM,8451 0>, T8, XS;8452 defm VPDPWSUDSY : avx_dotprod_rm<0xd3,"vpdpwsuds", v8i32, VR256, loadv8i32,8453 i256mem, X86vpdpwsuds, SchedWriteVecIMul.YMM,8454 0>, VEX_L, T8, XS;8455 defm VPDPWUSD : avx_dotprod_rm<0xd2,"vpdpwusd", v4i32, VR128, loadv4i32,8456 i128mem, X86vpdpwusd, SchedWriteVecIMul.XMM,8457 0>, T8, PD;8458 defm VPDPWUSDY : avx_dotprod_rm<0xd2,"vpdpwusd", v8i32, VR256, loadv8i32,8459 i256mem, X86vpdpwusd, SchedWriteVecIMul.YMM,8460 0>, VEX_L, T8, PD;8461 defm VPDPWUSDS : avx_dotprod_rm<0xd3,"vpdpwusds", v4i32, VR128, loadv4i32,8462 i128mem, X86vpdpwusds, SchedWriteVecIMul.XMM,8463 0>, T8, PD;8464 defm VPDPWUSDSY : avx_dotprod_rm<0xd3,"vpdpwusds", v8i32, VR256, loadv8i32,8465 i256mem, X86vpdpwusds, SchedWriteVecIMul.YMM,8466 0>, VEX_L, T8, PD;8467 defm VPDPWUUD : avx_dotprod_rm<0xd2,"vpdpwuud", v4i32, VR128, loadv4i32,8468 i128mem, X86vpdpwuud, SchedWriteVecIMul.XMM,8469 1>, T8;8470 defm VPDPWUUDY : avx_dotprod_rm<0xd2,"vpdpwuud", v8i32, VR256, loadv8i32,8471 i256mem, X86vpdpwuud, SchedWriteVecIMul.YMM,8472 1>, VEX_L, T8;8473 defm VPDPWUUDS : avx_dotprod_rm<0xd3,"vpdpwuuds", v4i32, VR128, loadv4i32,8474 i128mem, X86vpdpwuuds, SchedWriteVecIMul.XMM,8475 1>, T8;8476 defm VPDPWUUDSY : avx_dotprod_rm<0xd3,"vpdpwuuds", v8i32, VR256, loadv8i32,8477 i256mem, X86vpdpwuuds, SchedWriteVecIMul.YMM,8478 1>, VEX_L, T8;8479}8480