938 lines · c
1//==-- AArch64ISelLowering.h - AArch64 DAG Lowering Interface ----*- C++ -*-==//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file defines the interfaces that AArch64 uses to lower LLVM code into a10// selection DAG.11//12//===----------------------------------------------------------------------===//13 14#ifndef LLVM_LIB_TARGET_AARCH64_AARCH64ISELLOWERING_H15#define LLVM_LIB_TARGET_AARCH64_AARCH64ISELLOWERING_H16 17#include "llvm/CodeGen/CallingConvLower.h"18#include "llvm/CodeGen/MachineFunction.h"19#include "llvm/CodeGen/SelectionDAG.h"20#include "llvm/CodeGen/TargetLowering.h"21#include "llvm/IR/CallingConv.h"22#include "llvm/IR/Instruction.h"23 24namespace llvm {25 26class AArch64TargetMachine;27 28namespace AArch64 {29/// Possible values of current rounding mode, which is specified in bits30/// 23:22 of FPCR.31enum Rounding {32 RN = 0, // Round to Nearest33 RP = 1, // Round towards Plus infinity34 RM = 2, // Round towards Minus infinity35 RZ = 3, // Round towards Zero36 rmMask = 3 // Bit mask selecting rounding mode37};38 39// Bit position of rounding mode bits in FPCR.40const unsigned RoundingBitsPos = 22;41 42// Reserved bits should be preserved when modifying FPCR.43const uint64_t ReservedFPControlBits = 0xfffffffff80040f8;44 45// Registers used to pass function arguments.46ArrayRef<MCPhysReg> getGPRArgRegs();47ArrayRef<MCPhysReg> getFPRArgRegs();48 49/// Maximum allowed number of unprobed bytes above SP at an ABI50/// boundary.51const unsigned StackProbeMaxUnprobedStack = 1024;52 53/// Maximum number of iterations to unroll for a constant size probing loop.54const unsigned StackProbeMaxLoopUnroll = 4;55 56} // namespace AArch6457 58namespace ARM64AS {59enum : unsigned { PTR32_SPTR = 270, PTR32_UPTR = 271, PTR64 = 272 };60}61 62class AArch64Subtarget;63 64class AArch64TargetLowering : public TargetLowering {65public:66 explicit AArch64TargetLowering(const TargetMachine &TM,67 const AArch64Subtarget &STI);68 69 const AArch64TargetMachine &getTM() const;70 71 /// Control the following reassociation of operands: (op (op x, c1), y) -> (op72 /// (op x, y), c1) where N0 is (op x, c1) and N1 is y.73 bool isReassocProfitable(SelectionDAG &DAG, SDValue N0,74 SDValue N1) const override;75 76 /// Selects the correct CCAssignFn for a given CallingConvention value.77 CCAssignFn *CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg) const;78 79 /// Selects the correct CCAssignFn for a given CallingConvention value.80 CCAssignFn *CCAssignFnForReturn(CallingConv::ID CC) const;81 82 /// Determine which of the bits specified in Mask are known to be either zero83 /// or one and return them in the KnownZero/KnownOne bitsets.84 void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known,85 const APInt &DemandedElts,86 const SelectionDAG &DAG,87 unsigned Depth = 0) const override;88 89 unsigned ComputeNumSignBitsForTargetNode(SDValue Op,90 const APInt &DemandedElts,91 const SelectionDAG &DAG,92 unsigned Depth) const override;93 94 MVT getPointerTy(const DataLayout &DL, uint32_t AS = 0) const override {95 if ((AS == ARM64AS::PTR32_SPTR) || (AS == ARM64AS::PTR32_UPTR)) {96 // These are 32-bit pointers created using the `__ptr32` extension or97 // similar. They are handled by marking them as being in a different98 // address space, and will be extended to 64-bits when used as the target99 // of a load or store operation, or cast to a 64-bit pointer type.100 return MVT::i32;101 } else {102 // Returning i64 unconditionally here (i.e. even for ILP32) means that the103 // *DAG* representation of pointers will always be 64-bits. They will be104 // truncated and extended when transferred to memory, but the 64-bit DAG105 // allows us to use AArch64's addressing modes much more easily.106 return MVT::i64;107 }108 }109 110 unsigned getVectorIdxWidth(const DataLayout &DL) const override {111 // The VectorIdx type is i64, with both normal and ilp32.112 return 64;113 }114 115 bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits,116 const APInt &DemandedElts,117 TargetLoweringOpt &TLO) const override;118 119 MVT getScalarShiftAmountTy(const DataLayout &DL, EVT) const override;120 121 /// Returns true if the target allows unaligned memory accesses of the122 /// specified type.123 bool allowsMisalignedMemoryAccesses(124 EVT VT, unsigned AddrSpace = 0, Align Alignment = Align(1),125 MachineMemOperand::Flags Flags = MachineMemOperand::MONone,126 unsigned *Fast = nullptr) const override;127 /// LLT variant.128 bool allowsMisalignedMemoryAccesses(LLT Ty, unsigned AddrSpace,129 Align Alignment,130 MachineMemOperand::Flags Flags,131 unsigned *Fast = nullptr) const override;132 133 /// Provide custom lowering hooks for some operations.134 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;135 136 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;137 138 /// This method returns a target specific FastISel object, or null if the139 /// target does not support "fast" ISel.140 FastISel *createFastISel(FunctionLoweringInfo &funcInfo,141 const TargetLibraryInfo *libInfo) const override;142 143 bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;144 145 bool isFPImmLegal(const APFloat &Imm, EVT VT,146 bool ForCodeSize) const override;147 148 /// Return true if the given shuffle mask can be codegen'd directly, or if it149 /// should be stack expanded.150 bool isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const override;151 152 /// Similar to isShuffleMaskLegal. Return true is the given 'select with zero'153 /// shuffle mask can be codegen'd directly.154 bool isVectorClearMaskLegal(ArrayRef<int> M, EVT VT) const override;155 156 /// Return the ISD::SETCC ValueType.157 EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,158 EVT VT) const override;159 160 SDValue ReconstructShuffle(SDValue Op, SelectionDAG &DAG) const;161 162 MachineBasicBlock *EmitF128CSEL(MachineInstr &MI,163 MachineBasicBlock *BB) const;164 165 MachineBasicBlock *EmitLoweredCatchRet(MachineInstr &MI,166 MachineBasicBlock *BB) const;167 168 MachineBasicBlock *EmitDynamicProbedAlloc(MachineInstr &MI,169 MachineBasicBlock *MBB) const;170 171 MachineBasicBlock *EmitCheckMatchingVL(MachineInstr &MI,172 MachineBasicBlock *MBB) const;173 174 MachineBasicBlock *EmitTileLoad(unsigned Opc, unsigned BaseReg,175 MachineInstr &MI,176 MachineBasicBlock *BB) const;177 MachineBasicBlock *EmitFill(MachineInstr &MI, MachineBasicBlock *BB) const;178 MachineBasicBlock *EmitZAInstr(unsigned Opc, unsigned BaseReg,179 MachineInstr &MI, MachineBasicBlock *BB) const;180 MachineBasicBlock *EmitZTInstr(MachineInstr &MI, MachineBasicBlock *BB,181 unsigned Opcode, bool Op0IsDef) const;182 MachineBasicBlock *EmitZero(MachineInstr &MI, MachineBasicBlock *BB) const;183 184 // Note: The following group of functions are only used as part of the old SME185 // ABI lowering. They will be removed once -aarch64-new-sme-abi=true is the186 // default.187 MachineBasicBlock *EmitInitTPIDR2Object(MachineInstr &MI,188 MachineBasicBlock *BB) const;189 MachineBasicBlock *EmitAllocateZABuffer(MachineInstr &MI,190 MachineBasicBlock *BB) const;191 MachineBasicBlock *EmitAllocateSMESaveBuffer(MachineInstr &MI,192 MachineBasicBlock *BB) const;193 MachineBasicBlock *EmitGetSMESaveSize(MachineInstr &MI,194 MachineBasicBlock *BB) const;195 MachineBasicBlock *EmitEntryPStateSM(MachineInstr &MI,196 MachineBasicBlock *BB) const;197 198 /// Replace (0, vreg) discriminator components with the operands of blend199 /// or with (immediate, NoRegister) when possible.200 void fixupPtrauthDiscriminator(MachineInstr &MI, MachineBasicBlock *BB,201 MachineOperand &IntDiscOp,202 MachineOperand &AddrDiscOp,203 const TargetRegisterClass *AddrDiscRC) const;204 205 MachineBasicBlock *206 EmitInstrWithCustomInserter(MachineInstr &MI,207 MachineBasicBlock *MBB) const override;208 209 bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,210 MachineFunction &MF,211 unsigned Intrinsic) const override;212 213 bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT,214 std::optional<unsigned> ByteOffset) const override;215 216 bool shouldRemoveRedundantExtend(SDValue Op) const override;217 218 bool isTruncateFree(Type *Ty1, Type *Ty2) const override;219 bool isTruncateFree(EVT VT1, EVT VT2) const override;220 221 bool isProfitableToHoist(Instruction *I) const override;222 223 bool isZExtFree(Type *Ty1, Type *Ty2) const override;224 bool isZExtFree(EVT VT1, EVT VT2) const override;225 bool isZExtFree(SDValue Val, EVT VT2) const override;226 227 bool optimizeExtendOrTruncateConversion(228 Instruction *I, Loop *L, const TargetTransformInfo &TTI) const override;229 230 bool hasPairedLoad(EVT LoadedType, Align &RequiredAlignment) const override;231 232 unsigned getMaxSupportedInterleaveFactor() const override { return 4; }233 234 bool lowerInterleavedLoad(Instruction *Load, Value *Mask,235 ArrayRef<ShuffleVectorInst *> Shuffles,236 ArrayRef<unsigned> Indices, unsigned Factor,237 const APInt &GapMask) const override;238 bool lowerInterleavedStore(Instruction *Store, Value *Mask,239 ShuffleVectorInst *SVI, unsigned Factor,240 const APInt &GapMask) const override;241 242 bool lowerDeinterleaveIntrinsicToLoad(Instruction *Load, Value *Mask,243 IntrinsicInst *DI) const override;244 245 bool lowerInterleaveIntrinsicToStore(246 Instruction *Store, Value *Mask,247 ArrayRef<Value *> InterleaveValues) const override;248 249 bool isLegalAddImmediate(int64_t) const override;250 bool isLegalAddScalableImmediate(int64_t) const override;251 bool isLegalICmpImmediate(int64_t) const override;252 253 bool isMulAddWithConstProfitable(SDValue AddNode,254 SDValue ConstNode) const override;255 256 bool shouldConsiderGEPOffsetSplit() const override;257 258 EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op,259 const AttributeList &FuncAttributes) const override;260 261 LLT getOptimalMemOpLLT(const MemOp &Op,262 const AttributeList &FuncAttributes) const override;263 264 /// Return true if the addressing mode represented by AM is legal for this265 /// target, for a load/store of the specified type.266 bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,267 unsigned AS,268 Instruction *I = nullptr) const override;269 270 int64_t getPreferredLargeGEPBaseOffset(int64_t MinOffset,271 int64_t MaxOffset) const override;272 273 /// Return true if an FMA operation is faster than a pair of fmul and fadd274 /// instructions. fmuladd intrinsics will be expanded to FMAs when this method275 /// returns true, otherwise fmuladd is expanded to fmul + fadd.276 bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,277 EVT VT) const override;278 bool isFMAFasterThanFMulAndFAdd(const Function &F, Type *Ty) const override;279 280 bool generateFMAsInMachineCombiner(EVT VT,281 CodeGenOptLevel OptLevel) const override;282 283 /// Return true if the target has native support for284 /// the specified value type and it is 'desirable' to use the type for the285 /// given node type.286 bool isTypeDesirableForOp(unsigned Opc, EVT VT) const override;287 288 const MCPhysReg *getScratchRegisters(CallingConv::ID CC) const override;289 ArrayRef<MCPhysReg> getRoundingControlRegisters() const override;290 291 /// Returns false if N is a bit extraction pattern of (X >> C) & Mask.292 bool isDesirableToCommuteWithShift(const SDNode *N,293 CombineLevel Level) const override;294 295 bool isDesirableToPullExtFromShl(const MachineInstr &MI) const override {296 return false;297 }298 299 /// Returns false if N is a bit extraction pattern of (X >> C) & Mask.300 bool isDesirableToCommuteXorWithShift(const SDNode *N) const override;301 302 /// Return true if it is profitable to fold a pair of shifts into a mask.303 bool shouldFoldConstantShiftPairToMask(const SDNode *N) const override;304 305 /// Return true if it is profitable to fold a pair of shifts into a mask.306 bool shouldFoldMaskToVariableShiftPair(SDValue Y) const override {307 EVT VT = Y.getValueType();308 309 if (VT.isVector())310 return false;311 312 return VT.getScalarSizeInBits() <= 64;313 }314 315 bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT,316 unsigned SelectOpcode, SDValue X,317 SDValue Y) const override;318 319 /// Returns true if it is beneficial to convert a load of a constant320 /// to just the constant itself.321 bool shouldConvertConstantLoadToIntImm(const APInt &Imm,322 Type *Ty) const override;323 324 /// Return true if EXTRACT_SUBVECTOR is cheap for this result type325 /// with this index.326 bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,327 unsigned Index) const override;328 329 bool shouldFormOverflowOp(unsigned Opcode, EVT VT,330 bool MathUsed) const override {331 // Using overflow ops for overflow checks only should beneficial on332 // AArch64.333 return TargetLowering::shouldFormOverflowOp(Opcode, VT, true);334 }335 336 // Return true if the target wants to optimize the mul overflow intrinsic337 // for the given \p VT.338 bool shouldOptimizeMulOverflowWithZeroHighBits(LLVMContext &Context,339 EVT VT) const override;340 341 Value *emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr,342 AtomicOrdering Ord) const override;343 Value *emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr,344 AtomicOrdering Ord) const override;345 346 void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const override;347 348 bool isOpSuitableForLDPSTP(const Instruction *I) const;349 bool isOpSuitableForLSE128(const Instruction *I) const;350 bool isOpSuitableForRCPC3(const Instruction *I) const;351 bool shouldInsertFencesForAtomic(const Instruction *I) const override;352 bool353 shouldInsertTrailingFenceForAtomicStore(const Instruction *I) const override;354 355 TargetLoweringBase::AtomicExpansionKind356 shouldExpandAtomicLoadInIR(LoadInst *LI) const override;357 TargetLoweringBase::AtomicExpansionKind358 shouldExpandAtomicStoreInIR(StoreInst *SI) const override;359 TargetLoweringBase::AtomicExpansionKind360 shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;361 362 TargetLoweringBase::AtomicExpansionKind363 shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override;364 365 bool useLoadStackGuardNode(const Module &M) const override;366 TargetLoweringBase::LegalizeTypeAction367 getPreferredVectorAction(MVT VT) const override;368 369 /// If the target has a standard location for the stack protector cookie,370 /// returns the address of that location. Otherwise, returns nullptr.371 Value *getIRStackGuard(IRBuilderBase &IRB) const override;372 373 void insertSSPDeclarations(Module &M) const override;374 375 /// If the target has a standard location for the unsafe stack pointer,376 /// returns the address of that location. Otherwise, returns nullptr.377 Value *getSafeStackPointerLocation(IRBuilderBase &IRB) const override;378 379 /// If a physical register, this returns the register that receives the380 /// exception address on entry to an EH pad.381 Register382 getExceptionPointerRegister(const Constant *PersonalityFn) const override;383 384 /// If a physical register, this returns the register that receives the385 /// exception typeid on entry to a landing pad.386 Register387 getExceptionSelectorRegister(const Constant *PersonalityFn) const override;388 389 bool isIntDivCheap(EVT VT, AttributeList Attr) const override;390 391 bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT,392 const MachineFunction &MF) const override;393 394 bool isCheapToSpeculateCttz(Type *) const override {395 return true;396 }397 398 bool isCheapToSpeculateCtlz(Type *) const override {399 return true;400 }401 402 bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override;403 404 bool hasAndNotCompare(SDValue V) const override {405 // We can use bics for any scalar.406 return V.getValueType().isScalarInteger();407 }408 409 bool hasAndNot(SDValue Y) const override {410 EVT VT = Y.getValueType();411 412 if (!VT.isVector())413 return hasAndNotCompare(Y);414 415 if (VT.isScalableVector())416 return true;417 418 return VT.getFixedSizeInBits() >= 64; // vector 'bic'419 }420 421 bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(422 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,423 unsigned OldShiftOpcode, unsigned NewShiftOpcode,424 SelectionDAG &DAG) const override;425 426 ShiftLegalizationStrategy427 preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N,428 unsigned ExpansionFactor) const override;429 430 bool shouldTransformSignedTruncationCheck(EVT XVT,431 unsigned KeptBits) const override {432 // For vectors, we don't have a preference..433 if (XVT.isVector())434 return false;435 436 auto VTIsOk = [](EVT VT) -> bool {437 return VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 ||438 VT == MVT::i64;439 };440 441 // We are ok with KeptBitsVT being byte/word/dword, what SXT supports.442 // XVT will be larger than KeptBitsVT.443 MVT KeptBitsVT = MVT::getIntegerVT(KeptBits);444 return VTIsOk(XVT) && VTIsOk(KeptBitsVT);445 }446 447 bool preferIncOfAddToSubOfNot(EVT VT) const override;448 449 bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override;450 451 bool preferSelectsOverBooleanArithmetic(EVT VT) const override;452 453 bool isComplexDeinterleavingSupported() const override;454 bool isComplexDeinterleavingOperationSupported(455 ComplexDeinterleavingOperation Operation, Type *Ty) const override;456 457 Value *createComplexDeinterleavingIR(458 IRBuilderBase &B, ComplexDeinterleavingOperation OperationType,459 ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB,460 Value *Accumulator = nullptr) const override;461 462 bool supportSplitCSR(MachineFunction *MF) const override {463 return MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS &&464 MF->getFunction().hasFnAttribute(Attribute::NoUnwind);465 }466 void initializeSplitCSR(MachineBasicBlock *Entry) const override;467 void insertCopiesSplitCSR(468 MachineBasicBlock *Entry,469 const SmallVectorImpl<MachineBasicBlock *> &Exits) const override;470 471 bool supportSwiftError() const override {472 return true;473 }474 475 bool supportPtrAuthBundles() const override { return true; }476 477 bool supportKCFIBundles() const override { return true; }478 479 MachineInstr *EmitKCFICheck(MachineBasicBlock &MBB,480 MachineBasicBlock::instr_iterator &MBBI,481 const TargetInstrInfo *TII) const override;482 483 /// Enable aggressive FMA fusion on targets that want it.484 bool enableAggressiveFMAFusion(EVT VT) const override;485 486 bool aggressivelyPreferBuildVectorSources(EVT VecVT) const override {487 return true;488 }489 490 /// Returns the size of the platform's va_list object.491 unsigned getVaListSizeInBits(const DataLayout &DL) const override;492 493 /// Returns true if \p VecTy is a legal interleaved access type. This494 /// function checks the vector element type and the overall width of the495 /// vector.496 bool isLegalInterleavedAccessType(VectorType *VecTy, const DataLayout &DL,497 bool &UseScalable) const;498 499 /// Returns the number of interleaved accesses that will be generated when500 /// lowering accesses of the given type.501 unsigned getNumInterleavedAccesses(VectorType *VecTy, const DataLayout &DL,502 bool UseScalable) const;503 504 MachineMemOperand::Flags getTargetMMOFlags(505 const Instruction &I) const override;506 507 bool functionArgumentNeedsConsecutiveRegisters(508 Type *Ty, CallingConv::ID CallConv, bool isVarArg,509 const DataLayout &DL) const override;510 511 /// Used for exception handling on Win64.512 bool needsFixedCatchObjects() const override;513 514 bool fallBackToDAGISel(const Instruction &Inst) const override;515 516 /// SVE code generation for fixed length vectors does not custom lower517 /// BUILD_VECTOR. This makes BUILD_VECTOR legalisation a source of stores to518 /// merge. However, merging them creates a BUILD_VECTOR that is just as519 /// illegal as the original, thus leading to an infinite legalisation loop.520 /// NOTE: Once BUILD_VECTOR is legal or can be custom lowered for all legal521 /// vector types this override can be removed.522 bool mergeStoresAfterLegalization(EVT VT) const override;523 524 // If the platform/function should have a redzone, return the size in bytes.525 unsigned getRedZoneSize(const Function &F) const {526 if (F.hasFnAttribute(Attribute::NoRedZone))527 return 0;528 return 128;529 }530 531 bool isAllActivePredicate(SelectionDAG &DAG, SDValue N) const;532 EVT getPromotedVTForPredicate(EVT VT) const;533 534 EVT getAsmOperandValueType(const DataLayout &DL, Type *Ty,535 bool AllowUnknown = false) const override;536 537 bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const override;538 539 bool shouldExpandCttzElements(EVT VT) const override;540 541 bool shouldExpandVectorMatch(EVT VT, unsigned SearchSize) const override;542 543 /// If a change in streaming mode is required on entry to/return from a544 /// function call it emits and returns the corresponding SMSTART or SMSTOP545 /// node. \p Condition should be one of the enum values from546 /// AArch64SME::ToggleCondition.547 SDValue changeStreamingMode(SelectionDAG &DAG, SDLoc DL, bool Enable,548 SDValue Chain, SDValue InGlue, unsigned Condition,549 bool InsertVectorLengthCheck = false) const;550 551 bool isVScaleKnownToBeAPowerOfTwo() const override { return true; }552 553 // Normally SVE is only used for byte size vectors that do not fit within a554 // NEON vector. This changes when OverrideNEON is true, allowing SVE to be555 // used for 64bit and 128bit vectors as well.556 bool useSVEForFixedLengthVectorVT(EVT VT, bool OverrideNEON = false) const;557 558 // Follow NEON ABI rules even when using SVE for fixed length vectors.559 MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC,560 EVT VT) const override;561 unsigned getNumRegistersForCallingConv(LLVMContext &Context,562 CallingConv::ID CC,563 EVT VT) const override;564 unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context,565 CallingConv::ID CC, EVT VT,566 EVT &IntermediateVT,567 unsigned &NumIntermediates,568 MVT &RegisterVT) const override;569 570 /// True if stack clash protection is enabled for this functions.571 bool hasInlineStackProbe(const MachineFunction &MF) const override;572 573 /// In AArch64, true if FEAT_CPA is present. Allows pointer arithmetic574 /// semantics to be preserved for instruction selection.575 bool shouldPreservePtrArith(const Function &F, EVT PtrVT) const override;576 577private:578 /// Keep a pointer to the AArch64Subtarget around so that we can579 /// make the right decision when generating code for different targets.580 const AArch64Subtarget *Subtarget;581 582 bool isExtFreeImpl(const Instruction *Ext) const override;583 584 void addTypeForNEON(MVT VT);585 void addTypeForFixedLengthSVE(MVT VT);586 void addDRType(MVT VT);587 void addQRType(MVT VT);588 589 bool shouldExpandBuildVectorWithShuffles(EVT, unsigned) const override;590 591 SDValue lowerEHPadEntry(SDValue Chain, SDLoc const &DL,592 SelectionDAG &DAG) const override;593 594 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,595 bool isVarArg,596 const SmallVectorImpl<ISD::InputArg> &Ins,597 const SDLoc &DL, SelectionDAG &DAG,598 SmallVectorImpl<SDValue> &InVals) const override;599 600 void AdjustInstrPostInstrSelection(MachineInstr &MI,601 SDNode *Node) const override;602 603 SDValue LowerCall(CallLoweringInfo & /*CLI*/,604 SmallVectorImpl<SDValue> &InVals) const override;605 606 SDValue LowerCallResult(SDValue Chain, SDValue InGlue,607 CallingConv::ID CallConv, bool isVarArg,608 const SmallVectorImpl<CCValAssign> &RVLocs,609 const SDLoc &DL, SelectionDAG &DAG,610 SmallVectorImpl<SDValue> &InVals, bool isThisReturn,611 SDValue ThisVal, bool RequiresSMChange) const;612 613 SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;614 SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;615 SDValue LowerStore128(SDValue Op, SelectionDAG &DAG) const;616 SDValue LowerABS(SDValue Op, SelectionDAG &DAG) const;617 SDValue LowerFMUL(SDValue Op, SelectionDAG &DAG) const;618 619 SDValue LowerMGATHER(SDValue Op, SelectionDAG &DAG) const;620 SDValue LowerMSCATTER(SDValue Op, SelectionDAG &DAG) const;621 622 SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG) const;623 624 SDValue LowerVECTOR_COMPRESS(SDValue Op, SelectionDAG &DAG) const;625 626 SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;627 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;628 SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;629 630 bool631 isEligibleForTailCallOptimization(const CallLoweringInfo &CLI) const;632 633 /// Finds the incoming stack arguments which overlap the given fixed stack634 /// object and incorporates their load into the current chain. This prevents635 /// an upcoming store from clobbering the stack argument before it's used.636 SDValue addTokenForArgument(SDValue Chain, SelectionDAG &DAG,637 MachineFrameInfo &MFI, int ClobberedFI) const;638 639 bool DoesCalleeRestoreStack(CallingConv::ID CallCC, bool TailCallOpt) const;640 641 void saveVarArgRegisters(CCState &CCInfo, SelectionDAG &DAG, const SDLoc &DL,642 SDValue &Chain) const;643 644 bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,645 bool isVarArg,646 const SmallVectorImpl<ISD::OutputArg> &Outs,647 LLVMContext &Context, const Type *RetTy) const override;648 649 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,650 const SmallVectorImpl<ISD::OutputArg> &Outs,651 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,652 SelectionDAG &DAG) const override;653 654 SDValue getTargetNode(GlobalAddressSDNode *N, EVT Ty, SelectionDAG &DAG,655 unsigned Flag) const;656 SDValue getTargetNode(JumpTableSDNode *N, EVT Ty, SelectionDAG &DAG,657 unsigned Flag) const;658 SDValue getTargetNode(ConstantPoolSDNode *N, EVT Ty, SelectionDAG &DAG,659 unsigned Flag) const;660 SDValue getTargetNode(BlockAddressSDNode *N, EVT Ty, SelectionDAG &DAG,661 unsigned Flag) const;662 SDValue getTargetNode(ExternalSymbolSDNode *N, EVT Ty, SelectionDAG &DAG,663 unsigned Flag) const;664 template <class NodeTy>665 SDValue getGOT(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;666 template <class NodeTy>667 SDValue getAddrLarge(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;668 template <class NodeTy>669 SDValue getAddr(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;670 template <class NodeTy>671 SDValue getAddrTiny(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;672 SDValue LowerADDROFRETURNADDR(SDValue Op, SelectionDAG &DAG) const;673 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;674 SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;675 SDValue LowerDarwinGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;676 SDValue LowerELFGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;677 SDValue LowerELFTLSLocalExec(const GlobalValue *GV, SDValue ThreadBase,678 const SDLoc &DL, SelectionDAG &DAG) const;679 SDValue LowerELFTLSDescCallSeq(SDValue SymAddr, const SDLoc &DL,680 SelectionDAG &DAG) const;681 SDValue LowerWindowsGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;682 SDValue LowerPtrAuthGlobalAddress(SDValue Op, SelectionDAG &DAG) const;683 SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;684 SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const;685 SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;686 SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;687 SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;688 SDValue LowerSELECT_CC(ISD::CondCode CC, SDValue LHS, SDValue RHS,689 SDValue TVal, SDValue FVal,690 iterator_range<SDNode::user_iterator> Users,691 SDNodeFlags Flags, const SDLoc &dl,692 SelectionDAG &DAG) const;693 SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;694 SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;695 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;696 SDValue LowerBR_JT(SDValue Op, SelectionDAG &DAG) const;697 SDValue LowerBRIND(SDValue Op, SelectionDAG &DAG) const;698 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;699 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;700 SDValue LowerAAPCS_VASTART(SDValue Op, SelectionDAG &DAG) const;701 SDValue LowerDarwin_VASTART(SDValue Op, SelectionDAG &DAG) const;702 SDValue LowerWin64_VASTART(SDValue Op, SelectionDAG &DAG) const;703 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;704 SDValue LowerVACOPY(SDValue Op, SelectionDAG &DAG) const;705 SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;706 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;707 SDValue LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const;708 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;709 SDValue LowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;710 SDValue LowerSET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;711 SDValue LowerGET_FPMODE(SDValue Op, SelectionDAG &DAG) const;712 SDValue LowerSET_FPMODE(SDValue Op, SelectionDAG &DAG) const;713 SDValue LowerRESET_FPMODE(SDValue Op, SelectionDAG &DAG) const;714 SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;715 SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;716 SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;717 SDValue LowerZERO_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const;718 SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;719 SDValue LowerSPLAT_VECTOR(SDValue Op, SelectionDAG &DAG) const;720 SDValue LowerDUPQLane(SDValue Op, SelectionDAG &DAG) const;721 SDValue LowerToPredicatedOp(SDValue Op, SelectionDAG &DAG,722 unsigned NewOp) const;723 SDValue LowerToScalableOp(SDValue Op, SelectionDAG &DAG) const;724 SDValue LowerVECTOR_SPLICE(SDValue Op, SelectionDAG &DAG) const;725 SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;726 SDValue LowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;727 SDValue LowerVECTOR_DEINTERLEAVE(SDValue Op, SelectionDAG &DAG) const;728 SDValue LowerVECTOR_INTERLEAVE(SDValue Op, SelectionDAG &DAG) const;729 SDValue LowerVECTOR_HISTOGRAM(SDValue Op, SelectionDAG &DAG) const;730 SDValue LowerPARTIAL_REDUCE_MLA(SDValue Op, SelectionDAG &DAG) const;731 SDValue LowerGET_ACTIVE_LANE_MASK(SDValue Op, SelectionDAG &DAG) const;732 SDValue LowerDIV(SDValue Op, SelectionDAG &DAG) const;733 SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) const;734 SDValue LowerVectorSRA_SRL_SHL(SDValue Op, SelectionDAG &DAG) const;735 SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) const;736 SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG) const;737 SDValue LowerCTPOP_PARITY(SDValue Op, SelectionDAG &DAG) const;738 SDValue LowerCTTZ(SDValue Op, SelectionDAG &DAG) const;739 SDValue LowerBitreverse(SDValue Op, SelectionDAG &DAG) const;740 SDValue LowerMinMax(SDValue Op, SelectionDAG &DAG) const;741 SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const;742 SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;743 SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;744 SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;745 SDValue LowerVectorFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const;746 SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;747 SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const;748 SDValue LowerVectorXRINT(SDValue Op, SelectionDAG &DAG) const;749 SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;750 SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;751 SDValue LowerVectorOR(SDValue Op, SelectionDAG &DAG) const;752 SDValue LowerXOR(SDValue Op, SelectionDAG &DAG) const;753 SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;754 SDValue LowerLOOP_DEPENDENCE_MASK(SDValue Op, SelectionDAG &DAG) const;755 SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) const;756 SDValue LowerVSCALE(SDValue Op, SelectionDAG &DAG) const;757 SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const;758 SDValue LowerVECREDUCE(SDValue Op, SelectionDAG &DAG) const;759 SDValue LowerVECREDUCE_MUL(SDValue Op, SelectionDAG &DAG) const;760 SDValue LowerATOMIC_LOAD_AND(SDValue Op, SelectionDAG &DAG) const;761 SDValue LowerWindowsDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;762 SDValue LowerInlineDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;763 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;764 SDValue LowerMSTORE(SDValue Op, SelectionDAG &DAG) const;765 766 SDValue LowerAVG(SDValue Op, SelectionDAG &DAG, unsigned NewOp) const;767 768 SDValue LowerFixedLengthVectorIntDivideToSVE(SDValue Op,769 SelectionDAG &DAG) const;770 SDValue LowerFixedLengthVectorIntExtendToSVE(SDValue Op,771 SelectionDAG &DAG) const;772 SDValue LowerFixedLengthVectorLoadToSVE(SDValue Op, SelectionDAG &DAG) const;773 SDValue LowerFixedLengthVectorMLoadToSVE(SDValue Op, SelectionDAG &DAG) const;774 SDValue LowerVECREDUCE_SEQ_FADD(SDValue ScalarOp, SelectionDAG &DAG) const;775 SDValue LowerPredReductionToSVE(SDValue ScalarOp, SelectionDAG &DAG) const;776 SDValue LowerReductionToSVE(unsigned Opcode, SDValue ScalarOp,777 SelectionDAG &DAG) const;778 SDValue LowerFixedLengthVectorSelectToSVE(SDValue Op, SelectionDAG &DAG) const;779 SDValue LowerFixedLengthVectorSetccToSVE(SDValue Op, SelectionDAG &DAG) const;780 SDValue LowerFixedLengthVectorStoreToSVE(SDValue Op, SelectionDAG &DAG) const;781 SDValue LowerFixedLengthVectorMStoreToSVE(SDValue Op,782 SelectionDAG &DAG) const;783 SDValue LowerFixedLengthVectorTruncateToSVE(SDValue Op,784 SelectionDAG &DAG) const;785 SDValue LowerFixedLengthExtractVectorElt(SDValue Op, SelectionDAG &DAG) const;786 SDValue LowerFixedLengthInsertVectorElt(SDValue Op, SelectionDAG &DAG) const;787 SDValue LowerFixedLengthBitcastToSVE(SDValue Op, SelectionDAG &DAG) const;788 SDValue LowerFixedLengthConcatVectorsToSVE(SDValue Op,789 SelectionDAG &DAG) const;790 SDValue LowerFixedLengthFPExtendToSVE(SDValue Op, SelectionDAG &DAG) const;791 SDValue LowerFixedLengthFPRoundToSVE(SDValue Op, SelectionDAG &DAG) const;792 SDValue LowerFixedLengthIntToFPToSVE(SDValue Op, SelectionDAG &DAG) const;793 SDValue LowerFixedLengthFPToIntToSVE(SDValue Op, SelectionDAG &DAG) const;794 SDValue LowerFixedLengthVECTOR_SHUFFLEToSVE(SDValue Op,795 SelectionDAG &DAG) const;796 SDValue LowerFixedLengthBuildVectorToSVE(SDValue Op, SelectionDAG &DAG) const;797 798 SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,799 SmallVectorImpl<SDNode *> &Created) const override;800 SDValue BuildSREMPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,801 SmallVectorImpl<SDNode *> &Created) const override;802 SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,803 int &ExtraSteps, bool &UseOneConst,804 bool Reciprocal) const override;805 SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,806 int &ExtraSteps) const override;807 SDValue getSqrtInputTest(SDValue Operand, SelectionDAG &DAG,808 const DenormalMode &Mode) const override;809 SDValue getSqrtResultForDenormInput(SDValue Operand,810 SelectionDAG &DAG) const override;811 unsigned combineRepeatedFPDivisors() const override;812 813 ConstraintType getConstraintType(StringRef Constraint) const override;814 Register getRegisterByName(const char* RegName, LLT VT,815 const MachineFunction &MF) const override;816 817 /// Examine constraint string and operand type and determine a weight value.818 /// The operand object must already have been set up with the operand type.819 ConstraintWeight820 getSingleConstraintMatchWeight(AsmOperandInfo &info,821 const char *constraint) const override;822 823 std::pair<unsigned, const TargetRegisterClass *>824 getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,825 StringRef Constraint, MVT VT) const override;826 827 const char *LowerXConstraint(EVT ConstraintVT) const override;828 829 void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint,830 std::vector<SDValue> &Ops,831 SelectionDAG &DAG) const override;832 833 InlineAsm::ConstraintCode834 getInlineAsmMemConstraint(StringRef ConstraintCode) const override {835 if (ConstraintCode == "Q")836 return InlineAsm::ConstraintCode::Q;837 // FIXME: clang has code for 'Ump', 'Utf', 'Usa', and 'Ush' but these are838 // followed by llvm_unreachable so we'll leave them unimplemented in839 // the backend for now.840 return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);841 }842 843 /// Handle Lowering flag assembly outputs.844 SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag,845 const SDLoc &DL,846 const AsmOperandInfo &Constraint,847 SelectionDAG &DAG) const override;848 849 bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const override;850 bool shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const override;851 bool isVectorLoadExtDesirable(SDValue ExtVal) const override;852 bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override;853 bool mayBeEmittedAsTailCall(const CallInst *CI) const override;854 bool getIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base,855 SDValue &Offset, SelectionDAG &DAG) const;856 bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset,857 ISD::MemIndexedMode &AM,858 SelectionDAG &DAG) const override;859 bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base,860 SDValue &Offset, ISD::MemIndexedMode &AM,861 SelectionDAG &DAG) const override;862 bool isIndexingLegal(MachineInstr &MI, Register Base, Register Offset,863 bool IsPre, MachineRegisterInfo &MRI) const override;864 865 void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results,866 SelectionDAG &DAG) const override;867 void ReplaceBITCASTResults(SDNode *N, SmallVectorImpl<SDValue> &Results,868 SelectionDAG &DAG) const;869 void ReplaceExtractSubVectorResults(SDNode *N,870 SmallVectorImpl<SDValue> &Results,871 SelectionDAG &DAG) const;872 void ReplaceGetActiveLaneMaskResults(SDNode *N,873 SmallVectorImpl<SDValue> &Results,874 SelectionDAG &DAG) const;875 876 bool shouldNormalizeToSelectSequence(LLVMContext &, EVT) const override;877 878 void finalizeLowering(MachineFunction &MF) const override;879 880 bool shouldLocalize(const MachineInstr &MI,881 const TargetTransformInfo *TTI) const override;882 883 bool SimplifyDemandedBitsForTargetNode(SDValue Op,884 const APInt &OriginalDemandedBits,885 const APInt &OriginalDemandedElts,886 KnownBits &Known,887 TargetLoweringOpt &TLO,888 unsigned Depth) const override;889 890 bool canCreateUndefOrPoisonForTargetNode(SDValue Op,891 const APInt &DemandedElts,892 const SelectionDAG &DAG,893 bool PoisonOnly, bool ConsiderFlags,894 unsigned Depth) const override;895 896 bool isTargetCanonicalConstantNode(SDValue Op) const override;897 898 // With the exception of data-predicate transitions, no instructions are899 // required to cast between legal scalable vector types. However:900 // 1. Packed and unpacked types have different bit lengths, meaning BITCAST901 // is not universally useable.902 // 2. Most unpacked integer types are not legal and thus integer extends903 // cannot be used to convert between unpacked and packed types.904 // These can make "bitcasting" a multiphase process. REINTERPRET_CAST is used905 // to transition between unpacked and packed types of the same element type,906 // with BITCAST used otherwise.907 // This function does not handle predicate bitcasts.908 SDValue getSVESafeBitCast(EVT VT, SDValue Op, SelectionDAG &DAG) const;909 910 // Returns the runtime value for PSTATE.SM by generating a call to911 // __arm_sme_state.912 SDValue getRuntimePStateSM(SelectionDAG &DAG, SDValue Chain, SDLoc DL,913 EVT VT) const;914 915 bool preferScalarizeSplat(SDNode *N) const override;916 917 unsigned getMinimumJumpTableEntries() const override;918 919 bool softPromoteHalfType() const override { return true; }920 921 bool shouldScalarizeBinop(SDValue VecOp) const override {922 return VecOp.getOpcode() == ISD::SETCC;923 }924 925 bool hasMultipleConditionRegisters(EVT VT) const override {926 return VT.isScalableVector();927 }928};929 930namespace AArch64 {931FastISel *createFastISel(FunctionLoweringInfo &funcInfo,932 const TargetLibraryInfo *libInfo);933} // end namespace AArch64934 935} // end namespace llvm936 937#endif938