1991 lines · c
1//===-- X86ISelLowering.h - X86 DAG Lowering Interface ----------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file defines the interfaces that X86 uses to lower LLVM code into a10// selection DAG.11//12//===----------------------------------------------------------------------===//13 14#ifndef LLVM_LIB_TARGET_X86_X86ISELLOWERING_H15#define LLVM_LIB_TARGET_X86_X86ISELLOWERING_H16 17#include "llvm/CodeGen/MachineFunction.h"18#include "llvm/CodeGen/TargetLowering.h"19 20namespace llvm {21 class X86Subtarget;22 class X86TargetMachine;23 24 namespace X86ISD {25 // X86 Specific DAG Nodes26 enum NodeType : unsigned {27 // Start the numbering where the builtin ops leave off.28 FIRST_NUMBER = ISD::BUILTIN_OP_END,29 30 /// Bit scan forward.31 BSF,32 /// Bit scan reverse.33 BSR,34 35 /// X86 funnel/double shift i16 instructions. These correspond to36 /// X86::SHLDW and X86::SHRDW instructions which have different amt37 /// modulo rules to generic funnel shifts.38 /// NOTE: The operand order matches ISD::FSHL/FSHR not SHLD/SHRD.39 FSHL,40 FSHR,41 42 /// Bitwise logical AND of floating point values. This corresponds43 /// to X86::ANDPS or X86::ANDPD.44 FAND,45 46 /// Bitwise logical OR of floating point values. This corresponds47 /// to X86::ORPS or X86::ORPD.48 FOR,49 50 /// Bitwise logical XOR of floating point values. This corresponds51 /// to X86::XORPS or X86::XORPD.52 FXOR,53 54 /// Bitwise logical ANDNOT of floating point values. This55 /// corresponds to X86::ANDNPS or X86::ANDNPD.56 FANDN,57 58 /// These operations represent an abstract X86 call59 /// instruction, which includes a bunch of information. In particular the60 /// operands of these node are:61 ///62 /// #0 - The incoming token chain63 /// #1 - The callee64 /// #2 - The number of arg bytes the caller pushes on the stack.65 /// #3 - The number of arg bytes the callee pops off the stack.66 /// #4 - The value to pass in AL/AX/EAX (optional)67 /// #5 - The value to pass in DL/DX/EDX (optional)68 ///69 /// The result values of these nodes are:70 ///71 /// #0 - The outgoing token chain72 /// #1 - The first register result value (optional)73 /// #2 - The second register result value (optional)74 ///75 CALL,76 77 /// Same as call except it adds the NoTrack prefix.78 NT_CALL,79 80 // Pseudo for a OBJC call that gets emitted together with a special81 // marker instruction.82 CALL_RVMARKER,83 84 /// The same as ISD::CopyFromReg except that this node makes it explicit85 /// that it may lower to an x87 FPU stack pop. Optimizations should be more86 /// cautious when handling this node than a normal CopyFromReg to avoid87 /// removing a required FPU stack pop. A key requirement is optimizations88 /// should not optimize any users of a chain that contains a89 /// POP_FROM_X87_REG to use a chain from a point earlier than the90 /// POP_FROM_X87_REG (which may remove a required FPU stack pop).91 POP_FROM_X87_REG,92 93 // Pseudo for a call to an imported function to ensure the correct machine94 // instruction is emitted for Import Call Optimization.95 IMP_CALL,96 97 /// X86 compare and logical compare instructions.98 CMP,99 FCMP,100 COMI,101 UCOMI,102 103 // X86 compare with Intrinsics similar to COMI.104 COMX,105 UCOMX,106 107 /// X86 bit-test instructions.108 BT,109 110 /// X86 SetCC. Operand 0 is condition code, and operand 1 is the EFLAGS111 /// operand, usually produced by a CMP instruction.112 SETCC,113 114 /// X86 Select115 SELECTS,116 117 // Same as SETCC except it's materialized with a sbb and the value is all118 // one's or all zero's.119 SETCC_CARRY, // R = carry_bit ? ~0 : 0120 121 /// X86 FP SETCC, implemented with CMP{cc}SS/CMP{cc}SD.122 /// Operands are two FP values to compare; result is a mask of123 /// 0s or 1s. Generally DTRT for C/C++ with NaNs.124 FSETCC,125 126 /// X86 FP SETCC, similar to above, but with output as an i1 mask and127 /// and a version with SAE.128 FSETCCM,129 FSETCCM_SAE,130 131 /// X86 conditional moves. Operand 0 and operand 1 are the two values132 /// to select from. Operand 2 is the condition code, and operand 3 is the133 /// flag operand produced by a CMP or TEST instruction.134 CMOV,135 136 /// X86 conditional branches. Operand 0 is the chain operand, operand 1137 /// is the block to branch if condition is true, operand 2 is the138 /// condition code, and operand 3 is the flag operand produced by a CMP139 /// or TEST instruction.140 BRCOND,141 142 /// BRIND node with NoTrack prefix. Operand 0 is the chain operand and143 /// operand 1 is the target address.144 NT_BRIND,145 146 /// Return with a glue operand. Operand 0 is the chain operand, operand147 /// 1 is the number of bytes of stack to pop.148 RET_GLUE,149 150 /// Return from interrupt. Operand 0 is the number of bytes to pop.151 IRET,152 153 /// Repeat fill, corresponds to X86::REP_STOSx.154 REP_STOS,155 156 /// Repeat move, corresponds to X86::REP_MOVSx.157 REP_MOVS,158 159 /// On Darwin, this node represents the result of the popl160 /// at function entry, used for PIC code.161 GlobalBaseReg,162 163 /// A wrapper node for TargetConstantPool, TargetJumpTable,164 /// TargetExternalSymbol, TargetGlobalAddress, TargetGlobalTLSAddress,165 /// MCSymbol and TargetBlockAddress.166 Wrapper,167 168 /// Special wrapper used under X86-64 PIC mode for RIP169 /// relative displacements.170 WrapperRIP,171 172 /// Copies a 64-bit value from an MMX vector to the low word173 /// of an XMM vector, with the high word zero filled.174 MOVQ2DQ,175 176 /// Copies a 64-bit value from the low word of an XMM vector177 /// to an MMX vector.178 MOVDQ2Q,179 180 /// Copies a 32-bit value from the low word of a MMX181 /// vector to a GPR.182 MMX_MOVD2W,183 184 /// Copies a GPR into the low 32-bit word of a MMX vector185 /// and zero out the high word.186 MMX_MOVW2D,187 188 /// Extract an 8-bit value from a vector and zero extend it to189 /// i32, corresponds to X86::PEXTRB.190 PEXTRB,191 192 /// Extract a 16-bit value from a vector and zero extend it to193 /// i32, corresponds to X86::PEXTRW.194 PEXTRW,195 196 /// Insert any element of a 4 x float vector into any element197 /// of a destination 4 x floatvector.198 INSERTPS,199 200 /// Insert the lower 8-bits of a 32-bit value to a vector,201 /// corresponds to X86::PINSRB.202 PINSRB,203 204 /// Insert the lower 16-bits of a 32-bit value to a vector,205 /// corresponds to X86::PINSRW.206 PINSRW,207 208 /// Shuffle 16 8-bit values within a vector.209 PSHUFB,210 211 /// Compute Sum of Absolute Differences.212 PSADBW,213 /// Compute Double Block Packed Sum-Absolute-Differences214 DBPSADBW,215 216 /// Bitwise Logical AND NOT of Packed FP values.217 ANDNP,218 219 /// Blend where the selector is an immediate.220 BLENDI,221 222 /// Dynamic (non-constant condition) vector blend where only the sign bits223 /// of the condition elements are used. This is used to enforce that the224 /// condition mask is not valid for generic VSELECT optimizations. This225 /// is also used to implement the intrinsics.226 /// Operands are in VSELECT order: MASK, TRUE, FALSE227 BLENDV,228 229 /// Combined add and sub on an FP vector.230 ADDSUB,231 232 // FP vector ops with rounding mode.233 FADD_RND,234 FADDS,235 FADDS_RND,236 FSUB_RND,237 FSUBS,238 FSUBS_RND,239 FMUL_RND,240 FMULS,241 FMULS_RND,242 FDIV_RND,243 FDIVS,244 FDIVS_RND,245 FMAX_SAE,246 FMAXS_SAE,247 FMIN_SAE,248 FMINS_SAE,249 FSQRT_RND,250 FSQRTS,251 FSQRTS_RND,252 253 // FP vector get exponent.254 FGETEXP,255 FGETEXP_SAE,256 FGETEXPS,257 FGETEXPS_SAE,258 // Extract Normalized Mantissas.259 VGETMANT,260 VGETMANT_SAE,261 VGETMANTS,262 VGETMANTS_SAE,263 // FP Scale.264 SCALEF,265 SCALEF_RND,266 SCALEFS,267 SCALEFS_RND,268 269 /// Integer horizontal add/sub.270 HADD,271 HSUB,272 273 /// Integer horizontal saturating add/sub.274 HADDS,275 HSUBS,276 277 /// Floating point horizontal add/sub.278 FHADD,279 FHSUB,280 281 // Detect Conflicts Within a Vector282 CONFLICT,283 284 /// Floating point max and min.285 FMAX,286 FMIN,287 288 /// Commutative FMIN and FMAX.289 FMAXC,290 FMINC,291 292 /// Scalar intrinsic floating point max and min.293 FMAXS,294 FMINS,295 296 /// Floating point reciprocal-sqrt and reciprocal approximation.297 /// Note that these typically require refinement298 /// in order to obtain suitable precision.299 FRSQRT,300 FRCP,301 302 // AVX-512 reciprocal approximations with a little more precision.303 RSQRT14,304 RSQRT14S,305 RCP14,306 RCP14S,307 308 // Thread Local Storage.309 TLSADDR,310 311 // Thread Local Storage. A call to get the start address312 // of the TLS block for the current module.313 TLSBASEADDR,314 315 // Thread Local Storage. When calling to an OS provided316 // thunk at the address from an earlier relocation.317 TLSCALL,318 319 // Thread Local Storage. A descriptor containing pointer to320 // code and to argument to get the TLS offset for the symbol.321 TLSDESC,322 323 // Exception Handling helpers.324 EH_RETURN,325 326 // SjLj exception handling setjmp.327 EH_SJLJ_SETJMP,328 329 // SjLj exception handling longjmp.330 EH_SJLJ_LONGJMP,331 332 // SjLj exception handling dispatch.333 EH_SJLJ_SETUP_DISPATCH,334 335 /// Tail call return. See X86TargetLowering::LowerCall for336 /// the list of operands.337 TC_RETURN,338 339 // Vector move to low scalar and zero higher vector elements.340 VZEXT_MOVL,341 342 // Vector integer truncate.343 VTRUNC,344 // Vector integer truncate with unsigned/signed saturation.345 VTRUNCUS,346 VTRUNCS,347 348 // Masked version of the above. Used when less than a 128-bit result is349 // produced since the mask only applies to the lower elements and can't350 // be represented by a select.351 // SRC, PASSTHRU, MASK352 VMTRUNC,353 VMTRUNCUS,354 VMTRUNCS,355 356 // Vector FP extend.357 VFPEXT,358 VFPEXT_SAE,359 VFPEXTS,360 VFPEXTS_SAE,361 362 // Vector FP round.363 VFPROUND,364 // Convert TWO packed single data to one packed data365 VFPROUND2,366 VFPROUND2_RND,367 VFPROUND_RND,368 VFPROUNDS,369 VFPROUNDS_RND,370 371 // Masked version of above. Used for v2f64->v4f32.372 // SRC, PASSTHRU, MASK373 VMFPROUND,374 375 // 128-bit vector logical left / right shift376 VSHLDQ,377 VSRLDQ,378 379 // Vector shift elements380 VSHL,381 VSRL,382 VSRA,383 384 // Vector variable shift385 VSHLV,386 VSRLV,387 VSRAV,388 389 // Vector shift elements by immediate390 VSHLI,391 VSRLI,392 VSRAI,393 394 // Shifts of mask registers.395 KSHIFTL,396 KSHIFTR,397 398 // Bit rotate by immediate399 VROTLI,400 VROTRI,401 402 // Vector packed double/float comparison.403 CMPP,404 405 // Vector integer comparisons.406 PCMPEQ,407 PCMPGT,408 409 // v8i16 Horizontal minimum and position.410 PHMINPOS,411 412 MULTISHIFT,413 414 /// Vector comparison generating mask bits for fp and415 /// integer signed and unsigned data types.416 CMPM,417 // Vector mask comparison generating mask bits for FP values.418 CMPMM,419 // Vector mask comparison with SAE for FP values.420 CMPMM_SAE,421 422 // Arithmetic operations with FLAGS results.423 ADD,424 SUB,425 ADC,426 SBB,427 SMUL,428 UMUL,429 OR,430 XOR,431 AND,432 433 // Bit field extract.434 BEXTR,435 BEXTRI,436 437 // Zero High Bits Starting with Specified Bit Position.438 BZHI,439 440 // Parallel extract and deposit.441 PDEP,442 PEXT,443 444 // X86-specific multiply by immediate.445 MUL_IMM,446 447 // Vector sign bit extraction.448 MOVMSK,449 450 // Vector bitwise comparisons.451 PTEST,452 453 // Vector packed fp sign bitwise comparisons.454 TESTP,455 456 // OR/AND test for masks.457 KORTEST,458 KTEST,459 460 // ADD for masks.461 KADD,462 463 // Several flavors of instructions with vector shuffle behaviors.464 // Saturated signed/unnsigned packing.465 PACKSS,466 PACKUS,467 // Intra-lane alignr.468 PALIGNR,469 // AVX512 inter-lane alignr.470 VALIGN,471 PSHUFD,472 PSHUFHW,473 PSHUFLW,474 SHUFP,475 // VBMI2 Concat & Shift.476 VSHLD,477 VSHRD,478 479 // Shuffle Packed Values at 128-bit granularity.480 SHUF128,481 MOVDDUP,482 MOVSHDUP,483 MOVSLDUP,484 MOVLHPS,485 MOVHLPS,486 MOVSD,487 MOVSS,488 MOVSH,489 UNPCKL,490 UNPCKH,491 VPERMILPV,492 VPERMILPI,493 VPERMI,494 VPERM2X128,495 496 // Variable Permute (VPERM).497 // Res = VPERMV MaskV, V0498 VPERMV,499 500 // 3-op Variable Permute (VPERMT2).501 // Res = VPERMV3 V0, MaskV, V1502 VPERMV3,503 504 // Bitwise ternary logic.505 VPTERNLOG,506 // Fix Up Special Packed Float32/64 values.507 VFIXUPIMM,508 VFIXUPIMM_SAE,509 VFIXUPIMMS,510 VFIXUPIMMS_SAE,511 // Range Restriction Calculation For Packed Pairs of Float32/64 values.512 VRANGE,513 VRANGE_SAE,514 VRANGES,515 VRANGES_SAE,516 // Reduce - Perform Reduction Transformation on scalar\packed FP.517 VREDUCE,518 VREDUCE_SAE,519 VREDUCES,520 VREDUCES_SAE,521 // RndScale - Round FP Values To Include A Given Number Of Fraction Bits.522 // Also used by the legacy (V)ROUND intrinsics where we mask out the523 // scaling part of the immediate.524 VRNDSCALE,525 VRNDSCALE_SAE,526 VRNDSCALES,527 VRNDSCALES_SAE,528 // Tests Types Of a FP Values for packed types.529 VFPCLASS,530 // Tests Types Of a FP Values for scalar types.531 VFPCLASSS,532 533 // Broadcast (splat) scalar or element 0 of a vector. If the operand is534 // a vector, this node may change the vector length as part of the splat.535 VBROADCAST,536 // Broadcast mask to vector.537 VBROADCASTM,538 539 /// SSE4A Extraction and Insertion.540 EXTRQI,541 INSERTQI,542 543 // XOP arithmetic/logical shifts.544 VPSHA,545 VPSHL,546 // XOP signed/unsigned integer comparisons.547 VPCOM,548 VPCOMU,549 // XOP packed permute bytes.550 VPPERM,551 // XOP two source permutation.552 VPERMIL2,553 554 // Vector multiply packed unsigned doubleword integers.555 PMULUDQ,556 // Vector multiply packed signed doubleword integers.557 PMULDQ,558 // Vector Multiply Packed UnsignedIntegers with Round and Scale.559 MULHRS,560 561 // Multiply and Add Packed Integers.562 VPMADDUBSW,563 VPMADDWD,564 565 // AVX512IFMA multiply and add.566 // NOTE: These are different than the instruction and perform567 // op0 x op1 + op2.568 VPMADD52L,569 VPMADD52H,570 571 // VNNI572 VPDPBUSD,573 VPDPBUSDS,574 VPDPWSSD,575 VPDPWSSDS,576 577 // FMA nodes.578 // We use the target independent ISD::FMA for the non-inverted case.579 FNMADD,580 FMSUB,581 FNMSUB,582 FMADDSUB,583 FMSUBADD,584 585 // FMA with rounding mode.586 FMADD_RND,587 FNMADD_RND,588 FMSUB_RND,589 FNMSUB_RND,590 FMADDSUB_RND,591 FMSUBADD_RND,592 593 // AVX512-FP16 complex addition and multiplication.594 VFMADDC,595 VFMADDC_RND,596 VFCMADDC,597 VFCMADDC_RND,598 599 VFMULC,600 VFMULC_RND,601 VFCMULC,602 VFCMULC_RND,603 604 VFMADDCSH,605 VFMADDCSH_RND,606 VFCMADDCSH,607 VFCMADDCSH_RND,608 609 VFMULCSH,610 VFMULCSH_RND,611 VFCMULCSH,612 VFCMULCSH_RND,613 614 VPDPBSUD,615 VPDPBSUDS,616 VPDPBUUD,617 VPDPBUUDS,618 VPDPBSSD,619 VPDPBSSDS,620 621 VPDPWSUD,622 VPDPWSUDS,623 VPDPWUSD,624 VPDPWUSDS,625 VPDPWUUD,626 VPDPWUUDS,627 628 VMINMAX,629 VMINMAX_SAE,630 VMINMAXS,631 VMINMAXS_SAE,632 633 CVTP2IBS,634 CVTP2IUBS,635 CVTP2IBS_RND,636 CVTP2IUBS_RND,637 CVTTP2IBS,638 CVTTP2IUBS,639 CVTTP2IBS_SAE,640 CVTTP2IUBS_SAE,641 642 MPSADBW,643 644 VCVT2PH2BF8,645 VCVT2PH2BF8S,646 VCVT2PH2HF8,647 VCVT2PH2HF8S,648 VCVTBIASPH2BF8,649 VCVTBIASPH2BF8S,650 VCVTBIASPH2HF8,651 VCVTBIASPH2HF8S,652 VCVTPH2BF8,653 VCVTPH2BF8S,654 VCVTPH2HF8,655 VCVTPH2HF8S,656 VMCVTBIASPH2BF8,657 VMCVTBIASPH2BF8S,658 VMCVTBIASPH2HF8,659 VMCVTBIASPH2HF8S,660 VMCVTPH2BF8,661 VMCVTPH2BF8S,662 VMCVTPH2HF8,663 VMCVTPH2HF8S,664 VCVTHF82PH,665 666 // Compress and expand.667 COMPRESS,668 EXPAND,669 670 // Bits shuffle671 VPSHUFBITQMB,672 673 // Convert Unsigned/Integer to Floating-Point Value with rounding mode.674 SINT_TO_FP_RND,675 UINT_TO_FP_RND,676 SCALAR_SINT_TO_FP,677 SCALAR_UINT_TO_FP,678 SCALAR_SINT_TO_FP_RND,679 SCALAR_UINT_TO_FP_RND,680 681 // Vector float/double to signed/unsigned integer.682 CVTP2SI,683 CVTP2UI,684 CVTP2SI_RND,685 CVTP2UI_RND,686 // Scalar float/double to signed/unsigned integer.687 CVTS2SI,688 CVTS2UI,689 CVTS2SI_RND,690 CVTS2UI_RND,691 692 // Vector float/double to signed/unsigned integer with truncation.693 CVTTP2SI,694 CVTTP2UI,695 CVTTP2SI_SAE,696 CVTTP2UI_SAE,697 698 // Saturation enabled Vector float/double to signed/unsigned699 // integer with truncation.700 CVTTP2SIS,701 CVTTP2UIS,702 CVTTP2SIS_SAE,703 CVTTP2UIS_SAE,704 // Masked versions of above. Used for v2f64 to v4i32.705 // SRC, PASSTHRU, MASK706 MCVTTP2SIS,707 MCVTTP2UIS,708 709 // Scalar float/double to signed/unsigned integer with truncation.710 CVTTS2SI,711 CVTTS2UI,712 CVTTS2SI_SAE,713 CVTTS2UI_SAE,714 715 // Vector signed/unsigned integer to float/double.716 CVTSI2P,717 CVTUI2P,718 719 // Scalar float/double to signed/unsigned integer with saturation.720 CVTTS2SIS,721 CVTTS2UIS,722 CVTTS2SIS_SAE,723 CVTTS2UIS_SAE,724 725 // Masked versions of above. Used for v2f64->v4f32.726 // SRC, PASSTHRU, MASK727 MCVTP2SI,728 MCVTP2UI,729 MCVTTP2SI,730 MCVTTP2UI,731 MCVTSI2P,732 MCVTUI2P,733 734 // Custom handling for FP_TO_xINT_SAT735 FP_TO_SINT_SAT,736 FP_TO_UINT_SAT,737 738 // Vector float to bfloat16.739 // Convert packed single data to packed BF16 data740 CVTNEPS2BF16,741 // Masked version of above.742 // SRC, PASSTHRU, MASK743 MCVTNEPS2BF16,744 745 // Dot product of BF16/FP16 pairs to accumulated into746 // packed single precision.747 DPBF16PS,748 DPFP16PS,749 750 // A stack checking function call. On Windows it's _chkstk call.751 DYN_ALLOCA,752 753 // For allocating variable amounts of stack space when using754 // segmented stacks. Check if the current stacklet has enough space, and755 // falls back to heap allocation if not.756 SEG_ALLOCA,757 758 // For allocating stack space when using stack clash protector.759 // Allocation is performed by block, and each block is probed.760 PROBED_ALLOCA,761 762 // Memory barriers.763 MFENCE,764 765 // Get a random integer and indicate whether it is valid in CF.766 RDRAND,767 768 // Get a NIST SP800-90B & C compliant random integer and769 // indicate whether it is valid in CF.770 RDSEED,771 772 // Protection keys773 // RDPKRU - Operand 0 is chain. Operand 1 is value for ECX.774 // WRPKRU - Operand 0 is chain. Operand 1 is value for EDX. Operand 2 is775 // value for ECX.776 RDPKRU,777 WRPKRU,778 779 // SSE42 string comparisons.780 // These nodes produce 3 results, index, mask, and flags. X86ISelDAGToDAG781 // will emit one or two instructions based on which results are used. If782 // flags and index/mask this allows us to use a single instruction since783 // we won't have to pick and opcode for flags. Instead we can rely on the784 // DAG to CSE everything and decide at isel.785 PCMPISTR,786 PCMPESTR,787 788 // Test if in transactional execution.789 XTEST,790 791 // Conversions between float and half-float.792 CVTPS2PH,793 CVTPS2PH_SAE,794 CVTPH2PS,795 CVTPH2PS_SAE,796 797 // Masked version of above.798 // SRC, RND, PASSTHRU, MASK799 MCVTPS2PH,800 MCVTPS2PH_SAE,801 802 // Galois Field Arithmetic Instructions803 GF2P8AFFINEINVQB,804 GF2P8AFFINEQB,805 GF2P8MULB,806 807 // LWP insert record.808 LWPINS,809 810 // User level wait811 UMWAIT,812 TPAUSE,813 814 // Enqueue Stores Instructions815 ENQCMD,816 ENQCMDS,817 818 // For avx512-vp2intersect819 VP2INTERSECT,820 821 // User level interrupts - testui822 TESTUI,823 824 // Perform an FP80 add after changing precision control in FPCW.825 FP80_ADD,826 827 // Conditional compare instructions828 CCMP,829 CTEST,830 831 /// X86 strict FP compare instructions.832 FIRST_STRICTFP_OPCODE,833 STRICT_FCMP = FIRST_STRICTFP_OPCODE,834 STRICT_FCMPS,835 836 // Vector packed double/float comparison.837 STRICT_CMPP,838 839 /// Vector comparison generating mask bits for fp and840 /// integer signed and unsigned data types.841 STRICT_CMPM,842 843 // Vector float/double to signed/unsigned integer with truncation.844 STRICT_CVTTP2SI,845 STRICT_CVTTP2UI,846 847 // Vector FP extend.848 STRICT_VFPEXT,849 850 // Vector FP round.851 STRICT_VFPROUND,852 853 // RndScale - Round FP Values To Include A Given Number Of Fraction Bits.854 // Also used by the legacy (V)ROUND intrinsics where we mask out the855 // scaling part of the immediate.856 STRICT_VRNDSCALE,857 858 // Vector signed/unsigned integer to float/double.859 STRICT_CVTSI2P,860 STRICT_CVTUI2P,861 862 // Strict FMA nodes.863 STRICT_FNMADD,864 STRICT_FMSUB,865 STRICT_FNMSUB,866 867 // Conversions between float and half-float.868 STRICT_CVTPS2PH,869 STRICT_CVTPH2PS,870 871 // Perform an FP80 add after changing precision control in FPCW.872 STRICT_FP80_ADD,873 874 /// Floating point max and min.875 STRICT_FMAX,876 STRICT_FMIN,877 LAST_STRICTFP_OPCODE = STRICT_FMIN,878 879 // Compare and swap.880 FIRST_MEMORY_OPCODE,881 LCMPXCHG_DAG = FIRST_MEMORY_OPCODE,882 LCMPXCHG8_DAG,883 LCMPXCHG16_DAG,884 LCMPXCHG16_SAVE_RBX_DAG,885 886 /// LOCK-prefixed arithmetic read-modify-write instructions.887 /// EFLAGS, OUTCHAIN = LADD(INCHAIN, PTR, RHS)888 LADD,889 LSUB,890 LOR,891 LXOR,892 LAND,893 LBTS,894 LBTC,895 LBTR,896 LBTS_RM,897 LBTC_RM,898 LBTR_RM,899 900 /// RAO arithmetic instructions.901 /// OUTCHAIN = AADD(INCHAIN, PTR, RHS)902 AADD,903 AOR,904 AXOR,905 AAND,906 907 // Load, scalar_to_vector, and zero extend.908 VZEXT_LOAD,909 910 // extract_vector_elt, store.911 VEXTRACT_STORE,912 913 // scalar broadcast from memory.914 VBROADCAST_LOAD,915 916 // subvector broadcast from memory.917 SUBV_BROADCAST_LOAD,918 919 // Store FP control word into i16 memory.920 FNSTCW16m,921 922 // Load FP control word from i16 memory.923 FLDCW16m,924 925 // Store x87 FPU environment into memory.926 FNSTENVm,927 928 // Load x87 FPU environment from memory.929 FLDENVm,930 931 /// This instruction implements FP_TO_SINT with the932 /// integer destination in memory and a FP reg source. This corresponds933 /// to the X86::FIST*m instructions and the rounding mode change stuff. It934 /// has two inputs (token chain and address) and two outputs (int value935 /// and token chain). Memory VT specifies the type to store to.936 FP_TO_INT_IN_MEM,937 938 /// This instruction implements SINT_TO_FP with the939 /// integer source in memory and FP reg result. This corresponds to the940 /// X86::FILD*m instructions. It has two inputs (token chain and address)941 /// and two outputs (FP value and token chain). The integer source type is942 /// specified by the memory VT.943 FILD,944 945 /// This instruction implements a fp->int store from FP stack946 /// slots. This corresponds to the fist instruction. It takes a947 /// chain operand, value to store, address, and glue. The memory VT948 /// specifies the type to store as.949 FIST,950 951 /// This instruction implements an extending load to FP stack slots.952 /// This corresponds to the X86::FLD32m / X86::FLD64m. It takes a chain953 /// operand, and ptr to load from. The memory VT specifies the type to954 /// load from.955 FLD,956 957 /// This instruction implements a truncating store from FP stack958 /// slots. This corresponds to the X86::FST32m / X86::FST64m. It takes a959 /// chain operand, value to store, address, and glue. The memory VT960 /// specifies the type to store as.961 FST,962 963 /// These instructions grab the address of the next argument964 /// from a va_list. (reads and modifies the va_list in memory)965 VAARG_64,966 VAARG_X32,967 968 // Vector truncating store with unsigned/signed saturation969 VTRUNCSTOREUS,970 VTRUNCSTORES,971 // Vector truncating masked store with unsigned/signed saturation972 VMTRUNCSTOREUS,973 VMTRUNCSTORES,974 975 // X86 specific gather and scatter976 MGATHER,977 MSCATTER,978 979 // Key locker nodes that produce flags.980 AESENC128KL,981 AESDEC128KL,982 AESENC256KL,983 AESDEC256KL,984 AESENCWIDE128KL,985 AESDECWIDE128KL,986 AESENCWIDE256KL,987 AESDECWIDE256KL,988 989 /// Compare and Add if Condition is Met. Compare value in operand 2 with990 /// value in memory of operand 1. If condition of operand 4 is met, add991 /// value operand 3 to m32 and write new value in operand 1. Operand 2 is992 /// always updated with the original value from operand 1.993 CMPCCXADD,994 995 // Save xmm argument registers to the stack, according to %al. An operator996 // is needed so that this can be expanded with control flow.997 VASTART_SAVE_XMM_REGS,998 999 // Conditional load/store instructions1000 CLOAD,1001 CSTORE,1002 LAST_MEMORY_OPCODE = CSTORE,1003 };1004 } // end namespace X86ISD1005 1006 namespace X86 {1007 /// Current rounding mode is represented in bits 11:10 of FPSR. These1008 /// values are same as corresponding constants for rounding mode used1009 /// in glibc.1010 enum RoundingMode {1011 rmInvalid = -1, // For handle Invalid rounding mode1012 rmToNearest = 0, // FE_TONEAREST1013 rmDownward = 1 << 10, // FE_DOWNWARD1014 rmUpward = 2 << 10, // FE_UPWARD1015 rmTowardZero = 3 << 10, // FE_TOWARDZERO1016 rmMask = 3 << 10 // Bit mask selecting rounding mode1017 };1018 }1019 1020 /// Define some predicates that are used for node matching.1021 namespace X86 {1022 /// Returns true if Elt is a constant zero or floating point constant +0.0.1023 bool isZeroNode(SDValue Elt);1024 1025 /// Returns true of the given offset can be1026 /// fit into displacement field of the instruction.1027 bool isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M,1028 bool hasSymbolicDisplacement);1029 1030 /// Determines whether the callee is required to pop its1031 /// own arguments. Callee pop is necessary to support tail calls.1032 bool isCalleePop(CallingConv::ID CallingConv,1033 bool is64Bit, bool IsVarArg, bool GuaranteeTCO);1034 1035 /// If Op is a constant whose elements are all the same constant or1036 /// undefined, return true and return the constant value in \p SplatVal.1037 /// If we have undef bits that don't cover an entire element, we treat these1038 /// as zero if AllowPartialUndefs is set, else we fail and return false.1039 bool isConstantSplat(SDValue Op, APInt &SplatVal,1040 bool AllowPartialUndefs = true);1041 1042 /// Check if Op is a load operation that could be folded into some other x861043 /// instruction as a memory operand. Example: vpaddd (%rdi), %xmm0, %xmm0.1044 bool mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget,1045 bool AssumeSingleUse = false);1046 1047 /// Check if Op is a load operation that could be folded into a vector splat1048 /// instruction as a memory operand. Example: vbroadcastss 16(%rdi), %xmm2.1049 bool mayFoldLoadIntoBroadcastFromMem(SDValue Op, MVT EltVT,1050 const X86Subtarget &Subtarget,1051 bool AssumeSingleUse = false);1052 1053 /// Check if Op is a value that could be used to fold a store into some1054 /// other x86 instruction as a memory operand. Ex: pextrb $0, %xmm0, (%rdi).1055 bool mayFoldIntoStore(SDValue Op);1056 1057 /// Check if Op is an operation that could be folded into a zero extend x861058 /// instruction.1059 bool mayFoldIntoZeroExtend(SDValue Op);1060 1061 /// True if the target supports the extended frame for async Swift1062 /// functions.1063 bool isExtendedSwiftAsyncFrameSupported(const X86Subtarget &Subtarget,1064 const MachineFunction &MF);1065 1066 /// Convert LLVM rounding mode to X86 rounding mode.1067 int getRoundingModeX86(unsigned RM);1068 1069 } // end namespace X861070 1071 //===--------------------------------------------------------------------===//1072 // X86 Implementation of the TargetLowering interface1073 class X86TargetLowering final : public TargetLowering {1074 public:1075 explicit X86TargetLowering(const X86TargetMachine &TM,1076 const X86Subtarget &STI);1077 1078 unsigned getJumpTableEncoding() const override;1079 bool useSoftFloat() const override;1080 1081 void markLibCallAttributes(MachineFunction *MF, unsigned CC,1082 ArgListTy &Args) const override;1083 1084 MVT getScalarShiftAmountTy(const DataLayout &, EVT VT) const override {1085 return MVT::i8;1086 }1087 1088 const MCExpr *1089 LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI,1090 const MachineBasicBlock *MBB, unsigned uid,1091 MCContext &Ctx) const override;1092 1093 /// Returns relocation base for the given PIC jumptable.1094 SDValue getPICJumpTableRelocBase(SDValue Table,1095 SelectionDAG &DAG) const override;1096 const MCExpr *1097 getPICJumpTableRelocBaseExpr(const MachineFunction *MF,1098 unsigned JTI, MCContext &Ctx) const override;1099 1100 /// Return the desired alignment for ByVal aggregate1101 /// function arguments in the caller parameter area. For X86, aggregates1102 /// that contains are placed at 16-byte boundaries while the rest are at1103 /// 4-byte boundaries.1104 Align getByValTypeAlignment(Type *Ty, const DataLayout &DL) const override;1105 1106 EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op,1107 const AttributeList &FuncAttributes) const override;1108 1109 /// Returns true if it's safe to use load / store of the1110 /// specified type to expand memcpy / memset inline. This is mostly true1111 /// for all types except for some special cases. For example, on X861112 /// targets without SSE2 f64 load / store are done with fldl / fstpl which1113 /// also does type conversion. Note the specified type doesn't have to be1114 /// legal as the hook is used before type legalization.1115 bool isSafeMemOpType(MVT VT) const override;1116 1117 bool isMemoryAccessFast(EVT VT, Align Alignment) const;1118 1119 /// Returns true if the target allows unaligned memory accesses of the1120 /// specified type. Returns whether it is "fast" in the last argument.1121 bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, Align Alignment,1122 MachineMemOperand::Flags Flags,1123 unsigned *Fast) const override;1124 1125 /// This function returns true if the memory access is aligned or if the1126 /// target allows this specific unaligned memory access. If the access is1127 /// allowed, the optional final parameter returns a relative speed of the1128 /// access (as defined by the target).1129 bool allowsMemoryAccess(1130 LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,1131 Align Alignment,1132 MachineMemOperand::Flags Flags = MachineMemOperand::MONone,1133 unsigned *Fast = nullptr) const override;1134 1135 bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT,1136 const MachineMemOperand &MMO,1137 unsigned *Fast) const {1138 return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(),1139 MMO.getAlign(), MMO.getFlags(), Fast);1140 }1141 1142 /// Provide custom lowering hooks for some operations.1143 ///1144 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;1145 1146 /// Replace the results of node with an illegal result1147 /// type with new values built out of custom code.1148 ///1149 void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results,1150 SelectionDAG &DAG) const override;1151 1152 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;1153 1154 bool preferABDSToABSWithNSW(EVT VT) const override;1155 1156 bool preferSextInRegOfTruncate(EVT TruncVT, EVT VT,1157 EVT ExtVT) const override;1158 1159 bool isXAndYEqZeroPreferableToXAndYEqY(ISD::CondCode Cond,1160 EVT VT) const override;1161 1162 /// Return true if the target has native support for1163 /// the specified value type and it is 'desirable' to use the type for the1164 /// given node type. e.g. On x86 i16 is legal, but undesirable since i161165 /// instruction encodings are longer and some i16 instructions are slow.1166 bool isTypeDesirableForOp(unsigned Opc, EVT VT) const override;1167 1168 /// Return true if the target has native support for the1169 /// specified value type and it is 'desirable' to use the type. e.g. On x861170 /// i16 is legal, but undesirable since i16 instruction encodings are longer1171 /// and some i16 instructions are slow.1172 bool IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const override;1173 1174 /// Return prefered fold type, Abs if this is a vector, AddAnd if its an1175 /// integer, None otherwise.1176 TargetLowering::AndOrSETCCFoldKind1177 isDesirableToCombineLogicOpOfSETCC(const SDNode *LogicOp,1178 const SDNode *SETCC0,1179 const SDNode *SETCC1) const override;1180 1181 /// Return the newly negated expression if the cost is not expensive and1182 /// set the cost in \p Cost to indicate that if it is cheaper or neutral to1183 /// do the negation.1184 SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG,1185 bool LegalOperations, bool ForCodeSize,1186 NegatibleCost &Cost,1187 unsigned Depth) const override;1188 1189 MachineBasicBlock *1190 EmitInstrWithCustomInserter(MachineInstr &MI,1191 MachineBasicBlock *MBB) const override;1192 1193 /// This method returns the name of a target specific DAG node.1194 const char *getTargetNodeName(unsigned Opcode) const override;1195 1196 /// Do not merge vector stores after legalization because that may conflict1197 /// with x86-specific store splitting optimizations.1198 bool mergeStoresAfterLegalization(EVT MemVT) const override {1199 return !MemVT.isVector();1200 }1201 1202 bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT,1203 const MachineFunction &MF) const override;1204 1205 bool isCheapToSpeculateCttz(Type *Ty) const override;1206 1207 bool isCheapToSpeculateCtlz(Type *Ty) const override;1208 1209 bool isCtlzFast() const override;1210 1211 bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const override {1212 // If the pair to store is a mixture of float and int values, we will1213 // save two bitwise instructions and one float-to-int instruction and1214 // increase one store instruction. There is potentially a more1215 // significant benefit because it avoids the float->int domain switch1216 // for input value. So It is more likely a win.1217 if ((LTy.isFloatingPoint() && HTy.isInteger()) ||1218 (LTy.isInteger() && HTy.isFloatingPoint()))1219 return true;1220 // If the pair only contains int values, we will save two bitwise1221 // instructions and increase one store instruction (costing one more1222 // store buffer). Since the benefit is more blurred so we leave1223 // such pair out until we get testcase to prove it is a win.1224 return false;1225 }1226 1227 bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override;1228 1229 bool hasAndNotCompare(SDValue Y) const override;1230 1231 bool hasAndNot(SDValue Y) const override;1232 1233 bool hasBitTest(SDValue X, SDValue Y) const override;1234 1235 bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(1236 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,1237 unsigned OldShiftOpcode, unsigned NewShiftOpcode,1238 SelectionDAG &DAG) const override;1239 1240 unsigned preferedOpcodeForCmpEqPiecesOfOperand(1241 EVT VT, unsigned ShiftOpc, bool MayTransformRotate,1242 const APInt &ShiftOrRotateAmt,1243 const std::optional<APInt> &AndMask) const override;1244 1245 bool preferScalarizeSplat(SDNode *N) const override;1246 1247 CondMergingParams1248 getJumpConditionMergingParams(Instruction::BinaryOps Opc, const Value *Lhs,1249 const Value *Rhs) const override;1250 1251 bool shouldFoldConstantShiftPairToMask(const SDNode *N) const override;1252 1253 bool shouldFoldMaskToVariableShiftPair(SDValue Y) const override;1254 1255 bool1256 shouldTransformSignedTruncationCheck(EVT XVT,1257 unsigned KeptBits) const override {1258 // For vectors, we don't have a preference..1259 if (XVT.isVector())1260 return false;1261 1262 auto VTIsOk = [](EVT VT) -> bool {1263 return VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 ||1264 VT == MVT::i64;1265 };1266 1267 // We are ok with KeptBitsVT being byte/word/dword, what MOVS supports.1268 // XVT will be larger than KeptBitsVT.1269 MVT KeptBitsVT = MVT::getIntegerVT(KeptBits);1270 return VTIsOk(XVT) && VTIsOk(KeptBitsVT);1271 }1272 1273 ShiftLegalizationStrategy1274 preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N,1275 unsigned ExpansionFactor) const override;1276 1277 bool shouldSplatInsEltVarIndex(EVT VT) const override;1278 1279 bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override {1280 // Converting to sat variants holds little benefit on X86 as we will just1281 // need to saturate the value back using fp arithmatic.1282 return Op != ISD::FP_TO_UINT_SAT && isOperationLegalOrCustom(Op, VT);1283 }1284 1285 bool convertSetCCLogicToBitwiseLogic(EVT VT) const override {1286 return VT.isScalarInteger();1287 }1288 1289 /// Vector-sized comparisons are fast using PCMPEQ + PMOVMSK or PTEST.1290 MVT hasFastEqualityCompare(unsigned NumBits) const override;1291 1292 /// Return the value type to use for ISD::SETCC.1293 EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,1294 EVT VT) const override;1295 1296 bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits,1297 const APInt &DemandedElts,1298 TargetLoweringOpt &TLO) const override;1299 1300 /// Determine which of the bits specified in Mask are known to be either1301 /// zero or one and return them in the KnownZero/KnownOne bitsets.1302 void computeKnownBitsForTargetNode(const SDValue Op,1303 KnownBits &Known,1304 const APInt &DemandedElts,1305 const SelectionDAG &DAG,1306 unsigned Depth = 0) const override;1307 1308 /// Determine the number of bits in the operation that are sign bits.1309 unsigned ComputeNumSignBitsForTargetNode(SDValue Op,1310 const APInt &DemandedElts,1311 const SelectionDAG &DAG,1312 unsigned Depth) const override;1313 1314 bool SimplifyDemandedVectorEltsForTargetNode(SDValue Op,1315 const APInt &DemandedElts,1316 APInt &KnownUndef,1317 APInt &KnownZero,1318 TargetLoweringOpt &TLO,1319 unsigned Depth) const override;1320 1321 bool SimplifyDemandedVectorEltsForTargetShuffle(SDValue Op,1322 const APInt &DemandedElts,1323 unsigned MaskIndex,1324 TargetLoweringOpt &TLO,1325 unsigned Depth) const;1326 1327 bool SimplifyDemandedBitsForTargetNode(SDValue Op,1328 const APInt &DemandedBits,1329 const APInt &DemandedElts,1330 KnownBits &Known,1331 TargetLoweringOpt &TLO,1332 unsigned Depth) const override;1333 1334 SDValue SimplifyMultipleUseDemandedBitsForTargetNode(1335 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,1336 SelectionDAG &DAG, unsigned Depth) const override;1337 1338 bool isGuaranteedNotToBeUndefOrPoisonForTargetNode(1339 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,1340 bool PoisonOnly, unsigned Depth) const override;1341 1342 bool canCreateUndefOrPoisonForTargetNode(1343 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,1344 bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const override;1345 1346 bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts,1347 APInt &UndefElts, const SelectionDAG &DAG,1348 unsigned Depth) const override;1349 1350 bool isTargetCanonicalConstantNode(SDValue Op) const override {1351 // Peek through bitcasts/extracts/inserts to see if we have a vector1352 // load/broadcast from memory.1353 while (Op.getOpcode() == ISD::BITCAST ||1354 Op.getOpcode() == ISD::EXTRACT_SUBVECTOR ||1355 (Op.getOpcode() == ISD::INSERT_SUBVECTOR &&1356 Op.getOperand(0).isUndef()))1357 Op = Op.getOperand(Op.getOpcode() == ISD::INSERT_SUBVECTOR ? 1 : 0);1358 1359 return Op.getOpcode() == X86ISD::VBROADCAST_LOAD ||1360 Op.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD ||1361 (Op.getOpcode() == ISD::LOAD &&1362 getTargetConstantFromLoad(cast<LoadSDNode>(Op))) ||1363 TargetLowering::isTargetCanonicalConstantNode(Op);1364 }1365 1366 bool isTargetCanonicalSelect(SDNode *N) const override;1367 1368 const Constant *getTargetConstantFromLoad(LoadSDNode *LD) const override;1369 1370 SDValue unwrapAddress(SDValue N) const override;1371 1372 SDValue getReturnAddressFrameIndex(SelectionDAG &DAG) const;1373 1374 ConstraintType getConstraintType(StringRef Constraint) const override;1375 1376 /// Examine constraint string and operand type and determine a weight value.1377 /// The operand object must already have been set up with the operand type.1378 ConstraintWeight1379 getSingleConstraintMatchWeight(AsmOperandInfo &Info,1380 const char *Constraint) const override;1381 1382 const char *LowerXConstraint(EVT ConstraintVT) const override;1383 1384 /// Lower the specified operand into the Ops vector. If it is invalid, don't1385 /// add anything to Ops. If hasMemory is true it means one of the asm1386 /// constraint of the inline asm instruction being processed is 'm'.1387 void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint,1388 std::vector<SDValue> &Ops,1389 SelectionDAG &DAG) const override;1390 1391 InlineAsm::ConstraintCode1392 getInlineAsmMemConstraint(StringRef ConstraintCode) const override {1393 if (ConstraintCode == "v")1394 return InlineAsm::ConstraintCode::v;1395 return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);1396 }1397 1398 /// Handle Lowering flag assembly outputs.1399 SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag,1400 const SDLoc &DL,1401 const AsmOperandInfo &Constraint,1402 SelectionDAG &DAG) const override;1403 1404 /// Given a physical register constraint1405 /// (e.g. {edx}), return the register number and the register class for the1406 /// register. This should only be used for C_Register constraints. On1407 /// error, this returns a register number of 0.1408 std::pair<unsigned, const TargetRegisterClass *>1409 getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,1410 StringRef Constraint, MVT VT) const override;1411 1412 /// Return true if the addressing mode represented1413 /// by AM is legal for this target, for a load/store of the specified type.1414 bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM,1415 Type *Ty, unsigned AS,1416 Instruction *I = nullptr) const override;1417 1418 bool addressingModeSupportsTLS(const GlobalValue &GV) const override;1419 1420 /// Return true if the specified immediate is legal1421 /// icmp immediate, that is the target has icmp instructions which can1422 /// compare a register against the immediate without having to materialize1423 /// the immediate into a register.1424 bool isLegalICmpImmediate(int64_t Imm) const override;1425 1426 /// Return true if the specified immediate is legal1427 /// add immediate, that is the target has add instructions which can1428 /// add a register and the immediate without having to materialize1429 /// the immediate into a register.1430 bool isLegalAddImmediate(int64_t Imm) const override;1431 1432 bool isLegalStoreImmediate(int64_t Imm) const override;1433 1434 /// Add x86-specific opcodes to the default list.1435 bool isBinOp(unsigned Opcode) const override;1436 1437 /// Returns true if the opcode is a commutative binary operation.1438 bool isCommutativeBinOp(unsigned Opcode) const override;1439 1440 /// Return true if it's free to truncate a value of1441 /// type Ty1 to type Ty2. e.g. On x86 it's free to truncate a i32 value in1442 /// register EAX to i16 by referencing its sub-register AX.1443 bool isTruncateFree(Type *Ty1, Type *Ty2) const override;1444 bool isTruncateFree(EVT VT1, EVT VT2) const override;1445 1446 bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override;1447 1448 /// Return true if any actual instruction that defines a1449 /// value of type Ty1 implicit zero-extends the value to Ty2 in the result1450 /// register. This does not necessarily include registers defined in1451 /// unknown ways, such as incoming arguments, or copies from unknown1452 /// virtual registers. Also, if isTruncateFree(Ty2, Ty1) is true, this1453 /// does not necessarily apply to truncate instructions. e.g. on x86-64,1454 /// all instructions that define 32-bit values implicit zero-extend the1455 /// result out to 64 bits.1456 bool isZExtFree(Type *Ty1, Type *Ty2) const override;1457 bool isZExtFree(EVT VT1, EVT VT2) const override;1458 bool isZExtFree(SDValue Val, EVT VT2) const override;1459 1460 bool shouldConvertPhiType(Type *From, Type *To) const override;1461 1462 /// Return true if folding a vector load into ExtVal (a sign, zero, or any1463 /// extend node) is profitable.1464 bool isVectorLoadExtDesirable(SDValue) const override;1465 1466 /// Return true if an FMA operation is faster than a pair of fmul and fadd1467 /// instructions. fmuladd intrinsics will be expanded to FMAs when this1468 /// method returns true, otherwise fmuladd is expanded to fmul + fadd.1469 bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,1470 EVT VT) const override;1471 1472 /// Return true if it's profitable to narrow operations of type SrcVT to1473 /// DestVT. e.g. on x86, it's profitable to narrow from i32 to i8 but not1474 /// from i32 to i16.1475 bool isNarrowingProfitable(SDNode *N, EVT SrcVT, EVT DestVT) const override;1476 1477 bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT,1478 unsigned SelectOpcode, SDValue X,1479 SDValue Y) const override;1480 1481 /// Given an intrinsic, checks if on the target the intrinsic will need to map1482 /// to a MemIntrinsicNode (touches memory). If this is the case, it returns1483 /// true and stores the intrinsic information into the IntrinsicInfo that was1484 /// passed to the function.1485 bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,1486 MachineFunction &MF,1487 unsigned Intrinsic) const override;1488 1489 /// Returns true if the target can instruction select the1490 /// specified FP immediate natively. If false, the legalizer will1491 /// materialize the FP immediate as a load from a constant pool.1492 bool isFPImmLegal(const APFloat &Imm, EVT VT,1493 bool ForCodeSize) const override;1494 1495 /// Targets can use this to indicate that they only support *some*1496 /// VECTOR_SHUFFLE operations, those with specific masks. By default, if a1497 /// target supports the VECTOR_SHUFFLE node, all mask values are assumed to1498 /// be legal.1499 bool isShuffleMaskLegal(ArrayRef<int> Mask, EVT VT) const override;1500 1501 /// Similar to isShuffleMaskLegal. Targets can use this to indicate if there1502 /// is a suitable VECTOR_SHUFFLE that can be used to replace a VAND with a1503 /// constant pool entry.1504 bool isVectorClearMaskLegal(ArrayRef<int> Mask, EVT VT) const override;1505 1506 /// Returns true if lowering to a jump table is allowed.1507 bool areJTsAllowed(const Function *Fn) const override;1508 1509 MVT getPreferredSwitchConditionType(LLVMContext &Context,1510 EVT ConditionVT) const override;1511 1512 /// If true, then instruction selection should1513 /// seek to shrink the FP constant of the specified type to a smaller type1514 /// in order to save space and / or reduce runtime.1515 bool ShouldShrinkFPConstant(EVT VT) const override;1516 1517 /// Return true if we believe it is correct and profitable to reduce the1518 /// load node to a smaller type.1519 bool1520 shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT,1521 std::optional<unsigned> ByteOffset) const override;1522 1523 /// Return true if the specified scalar FP type is computed in an SSE1524 /// register, not on the X87 floating point stack.1525 bool isScalarFPTypeInSSEReg(EVT VT) const;1526 1527 /// Returns true if it is beneficial to convert a load of a constant1528 /// to just the constant itself.1529 bool shouldConvertConstantLoadToIntImm(const APInt &Imm,1530 Type *Ty) const override;1531 1532 bool reduceSelectOfFPConstantLoads(EVT CmpOpVT) const override;1533 1534 bool convertSelectOfConstantsToMath(EVT VT) const override;1535 1536 bool decomposeMulByConstant(LLVMContext &Context, EVT VT,1537 SDValue C) const override;1538 1539 /// Return true if EXTRACT_SUBVECTOR is cheap for this result type1540 /// with this index.1541 bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,1542 unsigned Index) const override;1543 1544 /// Scalar ops always have equal or better analysis/performance/power than1545 /// the vector equivalent, so this always makes sense if the scalar op is1546 /// supported.1547 bool shouldScalarizeBinop(SDValue) const override;1548 1549 /// Extract of a scalar FP value from index 0 of a vector is free.1550 bool isExtractVecEltCheap(EVT VT, unsigned Index) const override {1551 EVT EltVT = VT.getScalarType();1552 return (EltVT == MVT::f32 || EltVT == MVT::f64) && Index == 0;1553 }1554 1555 /// Overflow nodes should get combined/lowered to optimal instructions1556 /// (they should allow eliminating explicit compares by getting flags from1557 /// math ops).1558 bool shouldFormOverflowOp(unsigned Opcode, EVT VT,1559 bool MathUsed) const override;1560 1561 bool storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT, unsigned NumElem,1562 unsigned AddrSpace) const override {1563 // If we can replace more than 2 scalar stores, there will be a reduction1564 // in instructions even after we add a vector constant load.1565 return IsZero || NumElem > 2;1566 }1567 1568 bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT,1569 const SelectionDAG &DAG,1570 const MachineMemOperand &MMO) const override;1571 1572 Register getRegisterByName(const char* RegName, LLT VT,1573 const MachineFunction &MF) const override;1574 1575 /// If a physical register, this returns the register that receives the1576 /// exception address on entry to an EH pad.1577 Register1578 getExceptionPointerRegister(const Constant *PersonalityFn) const override;1579 1580 /// If a physical register, this returns the register that receives the1581 /// exception typeid on entry to a landing pad.1582 Register1583 getExceptionSelectorRegister(const Constant *PersonalityFn) const override;1584 1585 bool needsFixedCatchObjects() const override;1586 1587 /// This method returns a target specific FastISel object,1588 /// or null if the target does not support "fast" ISel.1589 FastISel *createFastISel(FunctionLoweringInfo &funcInfo,1590 const TargetLibraryInfo *libInfo) const override;1591 1592 /// If the target has a standard location for the stack protector cookie,1593 /// returns the address of that location. Otherwise, returns nullptr.1594 Value *getIRStackGuard(IRBuilderBase &IRB) const override;1595 1596 bool useLoadStackGuardNode(const Module &M) const override;1597 bool useStackGuardXorFP() const override;1598 void insertSSPDeclarations(Module &M) const override;1599 SDValue emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val,1600 const SDLoc &DL) const override;1601 1602 1603 /// Return true if the target stores SafeStack pointer at a fixed offset in1604 /// some non-standard address space, and populates the address space and1605 /// offset as appropriate.1606 Value *getSafeStackPointerLocation(IRBuilderBase &IRB) const override;1607 1608 std::pair<SDValue, SDValue> BuildFILD(EVT DstVT, EVT SrcVT, const SDLoc &DL,1609 SDValue Chain, SDValue Pointer,1610 MachinePointerInfo PtrInfo,1611 Align Alignment,1612 SelectionDAG &DAG) const;1613 1614 /// Customize the preferred legalization strategy for certain types.1615 LegalizeTypeAction getPreferredVectorAction(MVT VT) const override;1616 1617 bool softPromoteHalfType() const override { return true; }1618 1619 MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC,1620 EVT VT) const override;1621 1622 unsigned getNumRegistersForCallingConv(LLVMContext &Context,1623 CallingConv::ID CC,1624 EVT VT) const override;1625 1626 unsigned getVectorTypeBreakdownForCallingConv(1627 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,1628 unsigned &NumIntermediates, MVT &RegisterVT) const override;1629 1630 bool functionArgumentNeedsConsecutiveRegisters(1631 Type *Ty, CallingConv::ID CallConv, bool isVarArg,1632 const DataLayout &DL) const override;1633 1634 bool isIntDivCheap(EVT VT, AttributeList Attr) const override;1635 1636 bool supportSwiftError() const override;1637 1638 bool supportKCFIBundles() const override { return true; }1639 1640 MachineInstr *EmitKCFICheck(MachineBasicBlock &MBB,1641 MachineBasicBlock::instr_iterator &MBBI,1642 const TargetInstrInfo *TII) const override;1643 1644 bool hasStackProbeSymbol(const MachineFunction &MF) const override;1645 bool hasInlineStackProbe(const MachineFunction &MF) const override;1646 StringRef getStackProbeSymbolName(const MachineFunction &MF) const override;1647 1648 unsigned getStackProbeSize(const MachineFunction &MF) const;1649 1650 bool hasVectorBlend() const override { return true; }1651 1652 unsigned getMaxSupportedInterleaveFactor() const override { return 4; }1653 1654 bool isInlineAsmTargetBranch(const SmallVectorImpl<StringRef> &AsmStrs,1655 unsigned OpNo) const override;1656 1657 SDValue visitMaskedLoad(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,1658 MachineMemOperand *MMO, SDValue &NewLoad,1659 SDValue Ptr, SDValue PassThru,1660 SDValue Mask) const override;1661 SDValue visitMaskedStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,1662 MachineMemOperand *MMO, SDValue Ptr, SDValue Val,1663 SDValue Mask) const override;1664 1665 /// Lower interleaved load(s) into target specific1666 /// instructions/intrinsics.1667 bool lowerInterleavedLoad(Instruction *Load, Value *Mask,1668 ArrayRef<ShuffleVectorInst *> Shuffles,1669 ArrayRef<unsigned> Indices, unsigned Factor,1670 const APInt &GapMask) const override;1671 1672 /// Lower interleaved store(s) into target specific1673 /// instructions/intrinsics.1674 bool lowerInterleavedStore(Instruction *Store, Value *Mask,1675 ShuffleVectorInst *SVI, unsigned Factor,1676 const APInt &GapMask) const override;1677 1678 SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr,1679 int JTI, SelectionDAG &DAG) const override;1680 1681 Align getPrefLoopAlignment(MachineLoop *ML) const override;1682 1683 EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const override {1684 if (VT == MVT::f80)1685 return EVT::getIntegerVT(Context, 96);1686 return TargetLoweringBase::getTypeToTransformTo(Context, VT);1687 }1688 1689 protected:1690 std::pair<const TargetRegisterClass *, uint8_t>1691 findRepresentativeClass(const TargetRegisterInfo *TRI,1692 MVT VT) const override;1693 1694 private:1695 /// Keep a reference to the X86Subtarget around so that we can1696 /// make the right decision when generating code for different targets.1697 const X86Subtarget &Subtarget;1698 1699 /// A list of legal FP immediates.1700 std::vector<APFloat> LegalFPImmediates;1701 1702 /// Indicate that this x86 target can instruction1703 /// select the specified FP immediate natively.1704 void addLegalFPImmediate(const APFloat& Imm) {1705 LegalFPImmediates.push_back(Imm);1706 }1707 1708 SDValue LowerCallResult(SDValue Chain, SDValue InGlue,1709 CallingConv::ID CallConv, bool isVarArg,1710 const SmallVectorImpl<ISD::InputArg> &Ins,1711 const SDLoc &dl, SelectionDAG &DAG,1712 SmallVectorImpl<SDValue> &InVals,1713 uint32_t *RegMask) const;1714 SDValue LowerMemArgument(SDValue Chain, CallingConv::ID CallConv,1715 const SmallVectorImpl<ISD::InputArg> &ArgInfo,1716 const SDLoc &dl, SelectionDAG &DAG,1717 const CCValAssign &VA, MachineFrameInfo &MFI,1718 unsigned i) const;1719 SDValue LowerMemOpCallTo(SDValue Chain, SDValue StackPtr, SDValue Arg,1720 const SDLoc &dl, SelectionDAG &DAG,1721 const CCValAssign &VA,1722 ISD::ArgFlagsTy Flags, bool isByval) const;1723 1724 // Call lowering helpers.1725 1726 /// Check whether the call is eligible for tail call optimization. Targets1727 /// that want to do tail call optimization should implement this function.1728 bool IsEligibleForTailCallOptimization(1729 TargetLowering::CallLoweringInfo &CLI, CCState &CCInfo,1730 SmallVectorImpl<CCValAssign> &ArgLocs, bool IsCalleePopSRet) const;1731 SDValue EmitTailCallLoadRetAddr(SelectionDAG &DAG, SDValue &OutRetAddr,1732 SDValue Chain, bool IsTailCall,1733 bool Is64Bit, int FPDiff,1734 const SDLoc &dl) const;1735 1736 unsigned GetAlignedArgumentStackSize(unsigned StackSize,1737 SelectionDAG &DAG) const;1738 1739 unsigned getAddressSpace() const;1740 1741 SDValue FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG, bool IsSigned,1742 SDValue &Chain) const;1743 SDValue LRINT_LLRINTHelper(SDNode *N, SelectionDAG &DAG) const;1744 1745 SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;1746 SDValue LowerVSELECT(SDValue Op, SelectionDAG &DAG) const;1747 SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;1748 SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;1749 1750 unsigned getGlobalWrapperKind(const GlobalValue *GV,1751 const unsigned char OpFlags) const;1752 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;1753 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;1754 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;1755 SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;1756 SDValue LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const;1757 1758 /// Creates target global address or external symbol nodes for calls or1759 /// other uses.1760 SDValue LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG, bool ForCall,1761 bool *IsImpCall) const;1762 1763 SDValue LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;1764 SDValue LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;1765 SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const;1766 SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;1767 SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const;1768 SDValue LowerLRINT_LLRINT(SDValue Op, SelectionDAG &DAG) const;1769 SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;1770 SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const;1771 SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;1772 SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) const;1773 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;1774 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;1775 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;1776 SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;1777 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;1778 SDValue LowerADDROFRETURNADDR(SDValue Op, SelectionDAG &DAG) const;1779 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;1780 SDValue LowerFRAME_TO_ARGS_OFFSET(SDValue Op, SelectionDAG &DAG) const;1781 SDValue LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const;1782 SDValue lowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const;1783 SDValue lowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const;1784 SDValue lowerEH_SJLJ_SETUP_DISPATCH(SDValue Op, SelectionDAG &DAG) const;1785 SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;1786 SDValue LowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;1787 SDValue LowerSET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;1788 SDValue LowerGET_FPENV_MEM(SDValue Op, SelectionDAG &DAG) const;1789 SDValue LowerSET_FPENV_MEM(SDValue Op, SelectionDAG &DAG) const;1790 SDValue LowerRESET_FPENV(SDValue Op, SelectionDAG &DAG) const;1791 SDValue LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const;1792 SDValue LowerWin64_FP_TO_INT128(SDValue Op, SelectionDAG &DAG,1793 SDValue &Chain) const;1794 SDValue LowerWin64_INT128_TO_FP(SDValue Op, SelectionDAG &DAG) const;1795 SDValue LowerGC_TRANSITION(SDValue Op, SelectionDAG &DAG) const;1796 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;1797 SDValue lowerFaddFsub(SDValue Op, SelectionDAG &DAG) const;1798 SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;1799 SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;1800 SDValue LowerFP_TO_BF16(SDValue Op, SelectionDAG &DAG) const;1801 1802 SDValue1803 LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,1804 const SmallVectorImpl<ISD::InputArg> &Ins,1805 const SDLoc &dl, SelectionDAG &DAG,1806 SmallVectorImpl<SDValue> &InVals) const override;1807 SDValue LowerCall(CallLoweringInfo &CLI,1808 SmallVectorImpl<SDValue> &InVals) const override;1809 1810 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,1811 const SmallVectorImpl<ISD::OutputArg> &Outs,1812 const SmallVectorImpl<SDValue> &OutVals,1813 const SDLoc &dl, SelectionDAG &DAG) const override;1814 1815 bool supportSplitCSR(MachineFunction *MF) const override {1816 return MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS &&1817 MF->getFunction().hasFnAttribute(Attribute::NoUnwind);1818 }1819 void initializeSplitCSR(MachineBasicBlock *Entry) const override;1820 void insertCopiesSplitCSR(1821 MachineBasicBlock *Entry,1822 const SmallVectorImpl<MachineBasicBlock *> &Exits) const override;1823 1824 bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override;1825 1826 bool mayBeEmittedAsTailCall(const CallInst *CI) const override;1827 1828 EVT getTypeForExtReturn(LLVMContext &Context, EVT VT,1829 ISD::NodeType ExtendKind) const override;1830 1831 bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,1832 bool isVarArg,1833 const SmallVectorImpl<ISD::OutputArg> &Outs,1834 LLVMContext &Context,1835 const Type *RetTy) const override;1836 1837 const MCPhysReg *getScratchRegisters(CallingConv::ID CC) const override;1838 ArrayRef<MCPhysReg> getRoundingControlRegisters() const override;1839 1840 TargetLoweringBase::AtomicExpansionKind1841 shouldExpandAtomicLoadInIR(LoadInst *LI) const override;1842 TargetLoweringBase::AtomicExpansionKind1843 shouldExpandAtomicStoreInIR(StoreInst *SI) const override;1844 TargetLoweringBase::AtomicExpansionKind1845 shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;1846 TargetLoweringBase::AtomicExpansionKind1847 shouldExpandLogicAtomicRMWInIR(AtomicRMWInst *AI) const;1848 void emitBitTestAtomicRMWIntrinsic(AtomicRMWInst *AI) const override;1849 void emitCmpArithAtomicRMWIntrinsic(AtomicRMWInst *AI) const override;1850 1851 LoadInst *1852 lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const override;1853 1854 bool needsCmpXchgNb(Type *MemType) const;1855 1856 void SetupEntryBlockForSjLj(MachineInstr &MI, MachineBasicBlock *MBB,1857 MachineBasicBlock *DispatchBB, int FI) const;1858 1859 // Utility function to emit the low-level va_arg code for X86-64.1860 MachineBasicBlock *1861 EmitVAARGWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const;1862 1863 /// Utility function to emit the xmm reg save portion of va_start.1864 MachineBasicBlock *EmitLoweredCascadedSelect(MachineInstr &MI1,1865 MachineInstr &MI2,1866 MachineBasicBlock *BB) const;1867 1868 MachineBasicBlock *EmitLoweredSelect(MachineInstr &I,1869 MachineBasicBlock *BB) const;1870 1871 MachineBasicBlock *EmitLoweredCatchRet(MachineInstr &MI,1872 MachineBasicBlock *BB) const;1873 1874 MachineBasicBlock *EmitLoweredSegAlloca(MachineInstr &MI,1875 MachineBasicBlock *BB) const;1876 1877 MachineBasicBlock *EmitLoweredProbedAlloca(MachineInstr &MI,1878 MachineBasicBlock *BB) const;1879 1880 MachineBasicBlock *EmitLoweredTLSCall(MachineInstr &MI,1881 MachineBasicBlock *BB) const;1882 1883 MachineBasicBlock *EmitLoweredIndirectThunk(MachineInstr &MI,1884 MachineBasicBlock *BB) const;1885 1886 MachineBasicBlock *emitEHSjLjSetJmp(MachineInstr &MI,1887 MachineBasicBlock *MBB) const;1888 1889 void emitSetJmpShadowStackFix(MachineInstr &MI,1890 MachineBasicBlock *MBB) const;1891 1892 MachineBasicBlock *emitEHSjLjLongJmp(MachineInstr &MI,1893 MachineBasicBlock *MBB) const;1894 1895 MachineBasicBlock *emitLongJmpShadowStackFix(MachineInstr &MI,1896 MachineBasicBlock *MBB) const;1897 1898 MachineBasicBlock *EmitSjLjDispatchBlock(MachineInstr &MI,1899 MachineBasicBlock *MBB) const;1900 1901 MachineBasicBlock *emitPatchableEventCall(MachineInstr &MI,1902 MachineBasicBlock *MBB) const;1903 1904 /// Emit flags for the given setcc condition and operands. Also returns the1905 /// corresponding X86 condition code constant in X86CC.1906 SDValue emitFlagsForSetcc(SDValue Op0, SDValue Op1, ISD::CondCode CC,1907 const SDLoc &dl, SelectionDAG &DAG,1908 SDValue &X86CC) const;1909 1910 bool optimizeFMulOrFDivAsShiftAddBitcast(SDNode *N, SDValue FPConst,1911 SDValue IntPow2) const override;1912 1913 /// Check if replacement of SQRT with RSQRT should be disabled.1914 bool isFsqrtCheap(SDValue Op, SelectionDAG &DAG) const override;1915 1916 /// Use rsqrt* to speed up sqrt calculations.1917 SDValue getSqrtEstimate(SDValue Op, SelectionDAG &DAG, int Enabled,1918 int &RefinementSteps, bool &UseOneConstNR,1919 bool Reciprocal) const override;1920 1921 /// Use rcp* to speed up fdiv calculations.1922 SDValue getRecipEstimate(SDValue Op, SelectionDAG &DAG, int Enabled,1923 int &RefinementSteps) const override;1924 1925 /// Reassociate floating point divisions into multiply by reciprocal.1926 unsigned combineRepeatedFPDivisors() const override;1927 1928 SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,1929 SmallVectorImpl<SDNode *> &Created) const override;1930 1931 SDValue getMOVL(SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1,1932 SDValue V2) const;1933 };1934 1935 namespace X86 {1936 FastISel *createFastISel(FunctionLoweringInfo &funcInfo,1937 const TargetLibraryInfo *libInfo);1938 } // end namespace X861939 1940 // X86 specific Gather/Scatter nodes.1941 // The class has the same order of operands as MaskedGatherScatterSDNode for1942 // convenience.1943 class X86MaskedGatherScatterSDNode : public MemIntrinsicSDNode {1944 public:1945 // This is a intended as a utility and should never be directly created.1946 X86MaskedGatherScatterSDNode() = delete;1947 ~X86MaskedGatherScatterSDNode() = delete;1948 1949 const SDValue &getBasePtr() const { return getOperand(3); }1950 const SDValue &getIndex() const { return getOperand(4); }1951 const SDValue &getMask() const { return getOperand(2); }1952 const SDValue &getScale() const { return getOperand(5); }1953 1954 static bool classof(const SDNode *N) {1955 return N->getOpcode() == X86ISD::MGATHER ||1956 N->getOpcode() == X86ISD::MSCATTER;1957 }1958 };1959 1960 class X86MaskedGatherSDNode : public X86MaskedGatherScatterSDNode {1961 public:1962 const SDValue &getPassThru() const { return getOperand(1); }1963 1964 static bool classof(const SDNode *N) {1965 return N->getOpcode() == X86ISD::MGATHER;1966 }1967 };1968 1969 class X86MaskedScatterSDNode : public X86MaskedGatherScatterSDNode {1970 public:1971 const SDValue &getValue() const { return getOperand(1); }1972 1973 static bool classof(const SDNode *N) {1974 return N->getOpcode() == X86ISD::MSCATTER;1975 }1976 };1977 1978 /// Generate unpacklo/unpackhi shuffle mask.1979 void createUnpackShuffleMask(EVT VT, SmallVectorImpl<int> &Mask, bool Lo,1980 bool Unary);1981 1982 /// Similar to unpacklo/unpackhi, but without the 128-bit lane limitation1983 /// imposed by AVX and specific to the unary pattern. Example:1984 /// v8iX Lo --> <0, 0, 1, 1, 2, 2, 3, 3>1985 /// v8iX Hi --> <4, 4, 5, 5, 6, 6, 7, 7>1986 void createSplat2ShuffleMask(MVT VT, SmallVectorImpl<int> &Mask, bool Lo);1987 1988} // end namespace llvm1989 1990#endif // LLVM_LIB_TARGET_X86_X86ISELLOWERING_H1991