545 lines · c
1//===- AArch64TargetTransformInfo.h - AArch64 specific TTI ------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8/// \file9/// This file a TargetTransformInfoImplBase conforming object specific to the10/// AArch64 target machine. It uses the target's detailed information to11/// provide more precise answers to certain TTI queries, while letting the12/// target independent and default TTI implementations handle the rest.13///14//===----------------------------------------------------------------------===//15 16#ifndef LLVM_LIB_TARGET_AARCH64_AARCH64TARGETTRANSFORMINFO_H17#define LLVM_LIB_TARGET_AARCH64_AARCH64TARGETTRANSFORMINFO_H18 19#include "AArch64.h"20#include "AArch64Subtarget.h"21#include "AArch64TargetMachine.h"22#include "llvm/Analysis/TargetTransformInfo.h"23#include "llvm/CodeGen/BasicTTIImpl.h"24#include "llvm/IR/Function.h"25#include "llvm/IR/Intrinsics.h"26#include "llvm/Support/InstructionCost.h"27#include <cstdint>28#include <optional>29 30namespace llvm {31 32class APInt;33class Instruction;34class IntrinsicInst;35class Loop;36class SCEV;37class ScalarEvolution;38class Type;39class Value;40class VectorType;41 42class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {43 using BaseT = BasicTTIImplBase<AArch64TTIImpl>;44 using TTI = TargetTransformInfo;45 46 friend BaseT;47 48 const AArch64Subtarget *ST;49 const AArch64TargetLowering *TLI;50 51 static const FeatureBitset InlineInverseFeatures;52 53 const AArch64Subtarget *getST() const { return ST; }54 const AArch64TargetLowering *getTLI() const { return TLI; }55 56 enum MemIntrinsicType {57 VECTOR_LDST_TWO_ELEMENTS,58 VECTOR_LDST_THREE_ELEMENTS,59 VECTOR_LDST_FOUR_ELEMENTS60 };61 62 /// Given a add/sub/mul operation, detect a widening addl/subl/mull pattern63 /// where both operands can be treated like extends. Returns the minimal type64 /// needed to compute the operation.65 Type *isBinExtWideningInstruction(unsigned Opcode, Type *DstTy,66 ArrayRef<const Value *> Args,67 Type *SrcOverrideTy = nullptr) const;68 /// Given a add/sub operation with a single extend operand, detect a69 /// widening addw/subw pattern.70 bool isSingleExtWideningInstruction(unsigned Opcode, Type *DstTy,71 ArrayRef<const Value *> Args,72 Type *SrcOverrideTy = nullptr) const;73 74 // A helper function called by 'getVectorInstrCost'.75 //76 // 'Val' and 'Index' are forwarded from 'getVectorInstrCost';77 // \param ScalarUserAndIdx encodes the information about extracts from a78 /// vector with 'Scalar' being the value being extracted,'User' being the user79 /// of the extract(nullptr if user is not known before vectorization) and80 /// 'Idx' being the extract lane.81 InstructionCost getVectorInstrCostHelper(82 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,83 const Instruction *I = nullptr, Value *Scalar = nullptr,84 ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx = {}) const;85 86public:87 explicit AArch64TTIImpl(const AArch64TargetMachine *TM, const Function &F)88 : BaseT(TM, F.getDataLayout()), ST(TM->getSubtargetImpl(F)),89 TLI(ST->getTargetLowering()) {}90 91 bool areInlineCompatible(const Function *Caller,92 const Function *Callee) const override;93 94 bool areTypesABICompatible(const Function *Caller, const Function *Callee,95 ArrayRef<Type *> Types) const override;96 97 unsigned getInlineCallPenalty(const Function *F, const CallBase &Call,98 unsigned DefaultCallPenalty) const override;99 100 APInt getFeatureMask(const Function &F) const override;101 102 bool isMultiversionedFunction(const Function &F) const override;103 104 /// \name Scalar TTI Implementations105 /// @{106 107 using BaseT::getIntImmCost;108 InstructionCost getIntImmCost(int64_t Val) const;109 InstructionCost getIntImmCost(const APInt &Imm, Type *Ty,110 TTI::TargetCostKind CostKind) const override;111 InstructionCost getIntImmCostInst(unsigned Opcode, unsigned Idx,112 const APInt &Imm, Type *Ty,113 TTI::TargetCostKind CostKind,114 Instruction *Inst = nullptr) const override;115 InstructionCost116 getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, const APInt &Imm,117 Type *Ty, TTI::TargetCostKind CostKind) const override;118 TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth) const override;119 120 /// @}121 122 /// \name Vector TTI Implementations123 /// @{124 125 bool enableInterleavedAccessVectorization() const override { return true; }126 127 bool enableMaskedInterleavedAccessVectorization() const override {128 return ST->hasSVE();129 }130 131 unsigned getNumberOfRegisters(unsigned ClassID) const override {132 bool Vector = (ClassID == 1);133 if (Vector) {134 if (ST->hasNEON())135 return 32;136 return 0;137 }138 return 31;139 }140 141 InstructionCost142 getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,143 TTI::TargetCostKind CostKind) const override;144 145 std::optional<Instruction *>146 instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const override;147 148 std::optional<Value *> simplifyDemandedVectorEltsIntrinsic(149 InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,150 APInt &UndefElts2, APInt &UndefElts3,151 std::function<void(Instruction *, unsigned, APInt, APInt &)>152 SimplifyAndSetOp) const override;153 154 TypeSize155 getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const override;156 157 unsigned getMinVectorRegisterBitWidth() const override {158 return ST->getMinVectorRegisterBitWidth();159 }160 161 std::optional<unsigned> getVScaleForTuning() const override {162 return ST->getVScaleForTuning();163 }164 165 bool isVScaleKnownToBeAPowerOfTwo() const override { return true; }166 167 bool shouldMaximizeVectorBandwidth(168 TargetTransformInfo::RegisterKind K) const override;169 170 /// Try to return an estimate cost factor that can be used as a multiplier171 /// when scalarizing an operation for a vector with ElementCount \p VF.172 /// For scalable vectors this currently takes the most pessimistic view based173 /// upon the maximum possible value for vscale.174 unsigned getMaxNumElements(ElementCount VF) const {175 if (!VF.isScalable())176 return VF.getFixedValue();177 178 return VF.getKnownMinValue() * ST->getVScaleForTuning();179 }180 181 unsigned getMaxInterleaveFactor(ElementCount VF) const override;182 183 bool prefersVectorizedAddressing() const override;184 185 /// Check whether Opcode1 has less throughput according to the scheduling186 /// model than Opcode2.187 bool hasKnownLowerThroughputFromSchedulingModel(unsigned Opcode1,188 unsigned Opcode2) const;189 190 InstructionCost191 getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,192 TTI::TargetCostKind CostKind) const override;193 194 InstructionCost195 getGatherScatterOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr,196 bool VariableMask, Align Alignment,197 TTI::TargetCostKind CostKind,198 const Instruction *I = nullptr) const override;199 200 bool isExtPartOfAvgExpr(const Instruction *ExtUser, Type *Dst,201 Type *Src) const;202 203 InstructionCost204 getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,205 TTI::CastContextHint CCH, TTI::TargetCostKind CostKind,206 const Instruction *I = nullptr) const override;207 208 InstructionCost209 getExtractWithExtendCost(unsigned Opcode, Type *Dst, VectorType *VecTy,210 unsigned Index,211 TTI::TargetCostKind CostKind) const override;212 213 InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,214 const Instruction *I = nullptr) const override;215 216 InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,217 TTI::TargetCostKind CostKind,218 unsigned Index, const Value *Op0,219 const Value *Op1) const override;220 221 /// \param ScalarUserAndIdx encodes the information about extracts from a222 /// vector with 'Scalar' being the value being extracted,'User' being the user223 /// of the extract(nullptr if user is not known before vectorization) and224 /// 'Idx' being the extract lane.225 InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,226 TTI::TargetCostKind CostKind,227 unsigned Index, Value *Scalar,228 ArrayRef<std::tuple<Value *, User *, int>>229 ScalarUserAndIdx) const override;230 231 InstructionCost getVectorInstrCost(const Instruction &I, Type *Val,232 TTI::TargetCostKind CostKind,233 unsigned Index) const override;234 235 InstructionCost236 getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Val,237 TTI::TargetCostKind CostKind,238 unsigned Index) const override;239 240 InstructionCost241 getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF,242 TTI::TargetCostKind CostKind) const override;243 244 InstructionCost245 getArithmeticReductionCostSVE(unsigned Opcode, VectorType *ValTy,246 TTI::TargetCostKind CostKind) const;247 248 InstructionCost getSpliceCost(VectorType *Tp, int Index,249 TTI::TargetCostKind CostKind) const;250 251 InstructionCost getArithmeticInstrCost(252 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,253 TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},254 TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},255 ArrayRef<const Value *> Args = {},256 const Instruction *CxtI = nullptr) const override;257 258 InstructionCost259 getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, const SCEV *Ptr,260 TTI::TargetCostKind CostKind) const override;261 262 InstructionCost getCmpSelInstrCost(263 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,264 TTI::TargetCostKind CostKind,265 TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},266 TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},267 const Instruction *I = nullptr) const override;268 269 TTI::MemCmpExpansionOptions270 enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const override;271 bool useNeonVector(const Type *Ty) const;272 273 InstructionCost getMemoryOpCost(274 unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,275 TTI::TargetCostKind CostKind,276 TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},277 const Instruction *I = nullptr) const override;278 279 InstructionCost280 getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const override;281 282 void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,283 TTI::UnrollingPreferences &UP,284 OptimizationRemarkEmitter *ORE) const override;285 286 void getPeelingPreferences(Loop *L, ScalarEvolution &SE,287 TTI::PeelingPreferences &PP) const override;288 289 Value *290 getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst, Type *ExpectedType,291 bool CanCreate = true) const override;292 293 bool getTgtMemIntrinsic(IntrinsicInst *Inst,294 MemIntrinsicInfo &Info) const override;295 296 bool isElementTypeLegalForScalableVector(Type *Ty) const override {297 if (Ty->isPointerTy())298 return true;299 300 if (Ty->isBFloatTy() && ST->hasBF16())301 return true;302 303 if (Ty->isHalfTy() || Ty->isFloatTy() || Ty->isDoubleTy())304 return true;305 306 if (Ty->isIntegerTy(1) || Ty->isIntegerTy(8) || Ty->isIntegerTy(16) ||307 Ty->isIntegerTy(32) || Ty->isIntegerTy(64))308 return true;309 310 return false;311 }312 313 bool isLegalMaskedLoadStore(Type *DataType, Align Alignment) const {314 if (!ST->isSVEorStreamingSVEAvailable())315 return false;316 317 // For fixed vectors, avoid scalarization if using SVE for them.318 if (isa<FixedVectorType>(DataType) && !ST->useSVEForFixedLengthVectors() &&319 DataType->getPrimitiveSizeInBits() != 128)320 return false; // Fall back to scalarization of masked operations.321 322 return isElementTypeLegalForScalableVector(DataType->getScalarType());323 }324 325 bool isLegalMaskedLoad(Type *DataType, Align Alignment,326 unsigned /*AddressSpace*/,327 TTI::MaskKind /*MaskKind*/) const override {328 return isLegalMaskedLoadStore(DataType, Alignment);329 }330 331 bool isLegalMaskedStore(Type *DataType, Align Alignment,332 unsigned /*AddressSpace*/,333 TTI::MaskKind /*MaskKind*/) const override {334 return isLegalMaskedLoadStore(DataType, Alignment);335 }336 337 bool isElementTypeLegalForCompressStore(Type *Ty) const {338 return Ty->isFloatTy() || Ty->isDoubleTy() || Ty->isIntegerTy(32) ||339 Ty->isIntegerTy(64);340 }341 342 bool isLegalMaskedCompressStore(Type *DataType,343 Align Alignment) const override {344 if (!ST->isSVEAvailable())345 return false;346 347 if (isa<FixedVectorType>(DataType) &&348 DataType->getPrimitiveSizeInBits() < 128)349 return false;350 351 return isElementTypeLegalForCompressStore(DataType->getScalarType());352 }353 354 bool isLegalMaskedGatherScatter(Type *DataType) const {355 if (!ST->isSVEAvailable())356 return false;357 358 // For fixed vectors, scalarize if not using SVE for them.359 auto *DataTypeFVTy = dyn_cast<FixedVectorType>(DataType);360 if (DataTypeFVTy && (!ST->useSVEForFixedLengthVectors() ||361 DataTypeFVTy->getNumElements() < 2))362 return false;363 364 return isElementTypeLegalForScalableVector(DataType->getScalarType());365 }366 367 bool isLegalMaskedGather(Type *DataType, Align Alignment) const override {368 return isLegalMaskedGatherScatter(DataType);369 }370 371 bool isLegalMaskedScatter(Type *DataType, Align Alignment) const override {372 return isLegalMaskedGatherScatter(DataType);373 }374 375 bool isLegalBroadcastLoad(Type *ElementTy,376 ElementCount NumElements) const override {377 // Return true if we can generate a `ld1r` splat load instruction.378 if (!ST->hasNEON() || NumElements.isScalable())379 return false;380 switch (unsigned ElementBits = ElementTy->getScalarSizeInBits()) {381 case 8:382 case 16:383 case 32:384 case 64: {385 // We accept bit-widths >= 64bits and elements {8,16,32,64} bits.386 unsigned VectorBits = NumElements.getFixedValue() * ElementBits;387 return VectorBits >= 64;388 }389 }390 return false;391 }392 393 bool isLegalNTStoreLoad(Type *DataType, Align Alignment) const {394 // NOTE: The logic below is mostly geared towards LV, which calls it with395 // vectors with 2 elements. We might want to improve that, if other396 // users show up.397 // Nontemporal vector loads/stores can be directly lowered to LDNP/STNP, if398 // the vector can be halved so that each half fits into a register. That's399 // the case if the element type fits into a register and the number of400 // elements is a power of 2 > 1.401 if (auto *DataTypeTy = dyn_cast<FixedVectorType>(DataType)) {402 unsigned NumElements = DataTypeTy->getNumElements();403 unsigned EltSize = DataTypeTy->getElementType()->getScalarSizeInBits();404 return NumElements > 1 && isPowerOf2_64(NumElements) && EltSize >= 8 &&405 EltSize <= 128 && isPowerOf2_64(EltSize);406 }407 return BaseT::isLegalNTStore(DataType, Alignment);408 }409 410 bool isLegalNTStore(Type *DataType, Align Alignment) const override {411 return isLegalNTStoreLoad(DataType, Alignment);412 }413 414 bool isLegalNTLoad(Type *DataType, Align Alignment) const override {415 // Only supports little-endian targets.416 if (ST->isLittleEndian())417 return isLegalNTStoreLoad(DataType, Alignment);418 return BaseT::isLegalNTLoad(DataType, Alignment);419 }420 421 InstructionCost getPartialReductionCost(422 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,423 ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,424 TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,425 TTI::TargetCostKind CostKind) const override;426 427 bool enableOrderedReductions() const override { return true; }428 429 InstructionCost getInterleavedMemoryOpCost(430 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,431 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,432 bool UseMaskForCond = false, bool UseMaskForGaps = false) const override;433 434 bool shouldConsiderAddressTypePromotion(435 const Instruction &I,436 bool &AllowPromotionWithoutCommonHeader) const override;437 438 bool shouldExpandReduction(const IntrinsicInst *II) const override {439 return false;440 }441 442 unsigned getGISelRematGlobalCost() const override { return 2; }443 444 unsigned getMinTripCountTailFoldingThreshold() const override {445 return ST->hasSVE() ? 5 : 0;446 }447 448 TailFoldingStyle449 getPreferredTailFoldingStyle(bool IVUpdateMayOverflow) const override {450 if (ST->hasSVE())451 return IVUpdateMayOverflow452 ? TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck453 : TailFoldingStyle::DataAndControlFlow;454 455 return TailFoldingStyle::DataWithoutLaneMask;456 }457 458 bool preferFixedOverScalableIfEqualCost(bool IsEpilogue) const override;459 460 unsigned getEpilogueVectorizationMinVF() const override;461 462 bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) const override;463 464 bool supportsScalableVectors() const override {465 return ST->isSVEorStreamingSVEAvailable();466 }467 468 bool enableScalableVectorization() const override;469 470 bool isLegalToVectorizeReduction(const RecurrenceDescriptor &RdxDesc,471 ElementCount VF) const override;472 473 bool preferPredicatedReductionSelect() const override { return ST->hasSVE(); }474 475 /// FP16 and BF16 operations are lowered to fptrunc(op(fpext, fpext) if the476 /// architecture features are not present.477 std::optional<InstructionCost> getFP16BF16PromoteCost(478 Type *Ty, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,479 TTI::OperandValueInfo Op2Info, bool IncludeTrunc, bool CanUseSVE,480 std::function<InstructionCost(Type *)> InstCost) const;481 482 InstructionCost483 getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,484 std::optional<FastMathFlags> FMF,485 TTI::TargetCostKind CostKind) const override;486 487 InstructionCost488 getExtendedReductionCost(unsigned Opcode, bool IsUnsigned, Type *ResTy,489 VectorType *ValTy, std::optional<FastMathFlags> FMF,490 TTI::TargetCostKind CostKind) const override;491 492 InstructionCost getMulAccReductionCost(493 bool IsUnsigned, unsigned RedOpcode, Type *ResTy, VectorType *Ty,494 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) const override;495 496 InstructionCost497 getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,498 ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,499 VectorType *SubTp, ArrayRef<const Value *> Args = {},500 const Instruction *CxtI = nullptr) const override;501 502 InstructionCost getScalarizationOverhead(503 VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,504 TTI::TargetCostKind CostKind, bool ForPoisonSrc = true,505 ArrayRef<Value *> VL = {}) const override;506 507 /// Return the cost of the scaling factor used in the addressing508 /// mode represented by AM for this target, for a load/store509 /// of the specified type.510 /// If the AM is supported, the return value must be >= 0.511 /// If the AM is not supported, it returns an invalid cost.512 InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,513 StackOffset BaseOffset, bool HasBaseReg,514 int64_t Scale,515 unsigned AddrSpace) const override;516 517 bool enableSelectOptimize() const override {518 return ST->enableSelectOptimize();519 }520 521 bool shouldTreatInstructionLikeSelect(const Instruction *I) const override;522 523 unsigned getStoreMinimumVF(unsigned VF, Type *ScalarMemTy,524 Type *ScalarValTy) const override {525 // We can vectorize store v4i8.526 if (ScalarMemTy->isIntegerTy(8) && isPowerOf2_32(VF) && VF >= 4)527 return 4;528 529 return BaseT::getStoreMinimumVF(VF, ScalarMemTy, ScalarValTy);530 }531 532 std::optional<unsigned> getMinPageSize() const override { return 4096; }533 534 bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1,535 const TargetTransformInfo::LSRCost &C2) const override;536 537 bool isProfitableToSinkOperands(Instruction *I,538 SmallVectorImpl<Use *> &Ops) const override;539 /// @}540};541 542} // end namespace llvm543 544#endif // LLVM_LIB_TARGET_AARCH64_AARCH64TARGETTRANSFORMINFO_H545