1544 lines · cpp
1//===- llvm/Analysis/TargetTransformInfo.cpp ------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/Analysis/TargetTransformInfo.h"10#include "llvm/ADT/SmallVector.h"11#include "llvm/Analysis/CFG.h"12#include "llvm/Analysis/LoopIterator.h"13#include "llvm/Analysis/TargetLibraryInfo.h"14#include "llvm/Analysis/TargetTransformInfoImpl.h"15#include "llvm/IR/CFG.h"16#include "llvm/IR/Dominators.h"17#include "llvm/IR/Instruction.h"18#include "llvm/IR/Instructions.h"19#include "llvm/IR/IntrinsicInst.h"20#include "llvm/IR/Module.h"21#include "llvm/IR/Operator.h"22#include "llvm/InitializePasses.h"23#include "llvm/Support/CommandLine.h"24#include <optional>25#include <utility>26 27using namespace llvm;28using namespace PatternMatch;29 30#define DEBUG_TYPE "tti"31 32static cl::opt<bool> EnableReduxCost("costmodel-reduxcost", cl::init(false),33 cl::Hidden,34 cl::desc("Recognize reduction patterns."));35 36static cl::opt<unsigned> CacheLineSize(37 "cache-line-size", cl::init(0), cl::Hidden,38 cl::desc("Use this to override the target cache line size when "39 "specified by the user."));40 41static cl::opt<unsigned> MinPageSize(42 "min-page-size", cl::init(0), cl::Hidden,43 cl::desc("Use this to override the target's minimum page size."));44 45static cl::opt<unsigned> PredictableBranchThreshold(46 "predictable-branch-threshold", cl::init(99), cl::Hidden,47 cl::desc(48 "Use this to override the target's predictable branch threshold (%)."));49 50namespace {51/// No-op implementation of the TTI interface using the utility base52/// classes.53///54/// This is used when no target specific information is available.55struct NoTTIImpl : TargetTransformInfoImplCRTPBase<NoTTIImpl> {56 explicit NoTTIImpl(const DataLayout &DL)57 : TargetTransformInfoImplCRTPBase<NoTTIImpl>(DL) {}58};59} // namespace60 61TargetTransformInfo::TargetTransformInfo(62 std::unique_ptr<const TargetTransformInfoImplBase> Impl)63 : TTIImpl(std::move(Impl)) {}64 65bool HardwareLoopInfo::canAnalyze(LoopInfo &LI) {66 // If the loop has irreducible control flow, it can not be converted to67 // Hardware loop.68 LoopBlocksRPO RPOT(L);69 RPOT.perform(&LI);70 if (containsIrreducibleCFG<const BasicBlock *>(RPOT, LI))71 return false;72 return true;73}74 75IntrinsicCostAttributes::IntrinsicCostAttributes(76 Intrinsic::ID Id, const CallBase &CI, InstructionCost ScalarizationCost,77 bool TypeBasedOnly, const TargetLibraryInfo *LibInfo)78 : II(dyn_cast<IntrinsicInst>(&CI)), RetTy(CI.getType()), IID(Id),79 ScalarizationCost(ScalarizationCost), LibInfo(LibInfo) {80 81 if (const auto *FPMO = dyn_cast<FPMathOperator>(&CI))82 FMF = FPMO->getFastMathFlags();83 84 if (!TypeBasedOnly)85 Arguments.insert(Arguments.begin(), CI.arg_begin(), CI.arg_end());86 FunctionType *FTy = CI.getCalledFunction()->getFunctionType();87 ParamTys.insert(ParamTys.begin(), FTy->param_begin(), FTy->param_end());88}89 90IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy,91 ArrayRef<Type *> Tys,92 FastMathFlags Flags,93 const IntrinsicInst *I,94 InstructionCost ScalarCost)95 : II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost) {96 ParamTys.insert(ParamTys.begin(), Tys.begin(), Tys.end());97}98 99IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *Ty,100 ArrayRef<const Value *> Args)101 : RetTy(Ty), IID(Id) {102 103 Arguments.insert(Arguments.begin(), Args.begin(), Args.end());104 ParamTys.reserve(Arguments.size());105 for (const Value *Argument : Arguments)106 ParamTys.push_back(Argument->getType());107}108 109IntrinsicCostAttributes::IntrinsicCostAttributes(110 Intrinsic::ID Id, Type *RTy, ArrayRef<const Value *> Args,111 ArrayRef<Type *> Tys, FastMathFlags Flags, const IntrinsicInst *I,112 InstructionCost ScalarCost, TargetLibraryInfo const *LibInfo)113 : II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost),114 LibInfo(LibInfo) {115 ParamTys.insert(ParamTys.begin(), Tys.begin(), Tys.end());116 Arguments.insert(Arguments.begin(), Args.begin(), Args.end());117}118 119HardwareLoopInfo::HardwareLoopInfo(Loop *L) : L(L) {120 // Match default options:121 // - hardware-loop-counter-bitwidth = 32122 // - hardware-loop-decrement = 1123 CountType = Type::getInt32Ty(L->getHeader()->getContext());124 LoopDecrement = ConstantInt::get(CountType, 1);125}126 127bool HardwareLoopInfo::isHardwareLoopCandidate(ScalarEvolution &SE,128 LoopInfo &LI, DominatorTree &DT,129 bool ForceNestedLoop,130 bool ForceHardwareLoopPHI) {131 SmallVector<BasicBlock *, 4> ExitingBlocks;132 L->getExitingBlocks(ExitingBlocks);133 134 for (BasicBlock *BB : ExitingBlocks) {135 // If we pass the updated counter back through a phi, we need to know136 // which latch the updated value will be coming from.137 if (!L->isLoopLatch(BB)) {138 if (ForceHardwareLoopPHI || CounterInReg)139 continue;140 }141 142 const SCEV *EC = SE.getExitCount(L, BB);143 if (isa<SCEVCouldNotCompute>(EC))144 continue;145 if (const SCEVConstant *ConstEC = dyn_cast<SCEVConstant>(EC)) {146 if (ConstEC->getValue()->isZero())147 continue;148 } else if (!SE.isLoopInvariant(EC, L))149 continue;150 151 if (SE.getTypeSizeInBits(EC->getType()) > CountType->getBitWidth())152 continue;153 154 // If this exiting block is contained in a nested loop, it is not eligible155 // for insertion of the branch-and-decrement since the inner loop would156 // end up messing up the value in the CTR.157 if (!IsNestingLegal && LI.getLoopFor(BB) != L && !ForceNestedLoop)158 continue;159 160 // We now have a loop-invariant count of loop iterations (which is not the161 // constant zero) for which we know that this loop will not exit via this162 // existing block.163 164 // We need to make sure that this block will run on every loop iteration.165 // For this to be true, we must dominate all blocks with backedges. Such166 // blocks are in-loop predecessors to the header block.167 bool NotAlways = false;168 for (BasicBlock *Pred : predecessors(L->getHeader())) {169 if (!L->contains(Pred))170 continue;171 172 if (!DT.dominates(BB, Pred)) {173 NotAlways = true;174 break;175 }176 }177 178 if (NotAlways)179 continue;180 181 // Make sure this blocks ends with a conditional branch.182 Instruction *TI = BB->getTerminator();183 if (!TI)184 continue;185 186 if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {187 if (!BI->isConditional())188 continue;189 190 ExitBranch = BI;191 } else192 continue;193 194 // Note that this block may not be the loop latch block, even if the loop195 // has a latch block.196 ExitBlock = BB;197 ExitCount = EC;198 break;199 }200 201 if (!ExitBlock)202 return false;203 return true;204}205 206TargetTransformInfo::TargetTransformInfo(const DataLayout &DL)207 : TTIImpl(std::make_unique<NoTTIImpl>(DL)) {}208 209TargetTransformInfo::~TargetTransformInfo() = default;210 211TargetTransformInfo::TargetTransformInfo(TargetTransformInfo &&Arg)212 : TTIImpl(std::move(Arg.TTIImpl)) {}213 214TargetTransformInfo &TargetTransformInfo::operator=(TargetTransformInfo &&RHS) {215 TTIImpl = std::move(RHS.TTIImpl);216 return *this;217}218 219unsigned TargetTransformInfo::getInliningThresholdMultiplier() const {220 return TTIImpl->getInliningThresholdMultiplier();221}222 223unsigned224TargetTransformInfo::getInliningCostBenefitAnalysisSavingsMultiplier() const {225 return TTIImpl->getInliningCostBenefitAnalysisSavingsMultiplier();226}227 228unsigned229TargetTransformInfo::getInliningCostBenefitAnalysisProfitableMultiplier()230 const {231 return TTIImpl->getInliningCostBenefitAnalysisProfitableMultiplier();232}233 234int TargetTransformInfo::getInliningLastCallToStaticBonus() const {235 return TTIImpl->getInliningLastCallToStaticBonus();236}237 238unsigned239TargetTransformInfo::adjustInliningThreshold(const CallBase *CB) const {240 return TTIImpl->adjustInliningThreshold(CB);241}242 243unsigned TargetTransformInfo::getCallerAllocaCost(const CallBase *CB,244 const AllocaInst *AI) const {245 return TTIImpl->getCallerAllocaCost(CB, AI);246}247 248int TargetTransformInfo::getInlinerVectorBonusPercent() const {249 return TTIImpl->getInlinerVectorBonusPercent();250}251 252InstructionCost TargetTransformInfo::getGEPCost(253 Type *PointeeType, const Value *Ptr, ArrayRef<const Value *> Operands,254 Type *AccessType, TTI::TargetCostKind CostKind) const {255 return TTIImpl->getGEPCost(PointeeType, Ptr, Operands, AccessType, CostKind);256}257 258InstructionCost TargetTransformInfo::getPointersChainCost(259 ArrayRef<const Value *> Ptrs, const Value *Base,260 const TTI::PointersChainInfo &Info, Type *AccessTy,261 TTI::TargetCostKind CostKind) const {262 assert((Base || !Info.isSameBase()) &&263 "If pointers have same base address it has to be provided.");264 return TTIImpl->getPointersChainCost(Ptrs, Base, Info, AccessTy, CostKind);265}266 267unsigned TargetTransformInfo::getEstimatedNumberOfCaseClusters(268 const SwitchInst &SI, unsigned &JTSize, ProfileSummaryInfo *PSI,269 BlockFrequencyInfo *BFI) const {270 return TTIImpl->getEstimatedNumberOfCaseClusters(SI, JTSize, PSI, BFI);271}272 273InstructionCost274TargetTransformInfo::getInstructionCost(const User *U,275 ArrayRef<const Value *> Operands,276 enum TargetCostKind CostKind) const {277 InstructionCost Cost = TTIImpl->getInstructionCost(U, Operands, CostKind);278 assert((CostKind == TTI::TCK_RecipThroughput || Cost >= 0) &&279 "TTI should not produce negative costs!");280 return Cost;281}282 283BranchProbability TargetTransformInfo::getPredictableBranchThreshold() const {284 return PredictableBranchThreshold.getNumOccurrences() > 0285 ? BranchProbability(PredictableBranchThreshold, 100)286 : TTIImpl->getPredictableBranchThreshold();287}288 289InstructionCost TargetTransformInfo::getBranchMispredictPenalty() const {290 return TTIImpl->getBranchMispredictPenalty();291}292 293bool TargetTransformInfo::hasBranchDivergence(const Function *F) const {294 return TTIImpl->hasBranchDivergence(F);295}296 297bool TargetTransformInfo::isSourceOfDivergence(const Value *V) const {298 if (const auto *Call = dyn_cast<CallBase>(V)) {299 if (Call->hasFnAttr(Attribute::NoDivergenceSource))300 return false;301 }302 return TTIImpl->isSourceOfDivergence(V);303}304 305bool llvm::TargetTransformInfo::isAlwaysUniform(const Value *V) const {306 return TTIImpl->isAlwaysUniform(V);307}308 309bool llvm::TargetTransformInfo::isValidAddrSpaceCast(unsigned FromAS,310 unsigned ToAS) const {311 return TTIImpl->isValidAddrSpaceCast(FromAS, ToAS);312}313 314bool llvm::TargetTransformInfo::addrspacesMayAlias(unsigned FromAS,315 unsigned ToAS) const {316 return TTIImpl->addrspacesMayAlias(FromAS, ToAS);317}318 319unsigned TargetTransformInfo::getFlatAddressSpace() const {320 return TTIImpl->getFlatAddressSpace();321}322 323bool TargetTransformInfo::collectFlatAddressOperands(324 SmallVectorImpl<int> &OpIndexes, Intrinsic::ID IID) const {325 return TTIImpl->collectFlatAddressOperands(OpIndexes, IID);326}327 328bool TargetTransformInfo::isNoopAddrSpaceCast(unsigned FromAS,329 unsigned ToAS) const {330 return TTIImpl->isNoopAddrSpaceCast(FromAS, ToAS);331}332 333bool TargetTransformInfo::canHaveNonUndefGlobalInitializerInAddressSpace(334 unsigned AS) const {335 return TTIImpl->canHaveNonUndefGlobalInitializerInAddressSpace(AS);336}337 338unsigned TargetTransformInfo::getAssumedAddrSpace(const Value *V) const {339 return TTIImpl->getAssumedAddrSpace(V);340}341 342bool TargetTransformInfo::isSingleThreaded() const {343 return TTIImpl->isSingleThreaded();344}345 346std::pair<const Value *, unsigned>347TargetTransformInfo::getPredicatedAddrSpace(const Value *V) const {348 return TTIImpl->getPredicatedAddrSpace(V);349}350 351Value *TargetTransformInfo::rewriteIntrinsicWithAddressSpace(352 IntrinsicInst *II, Value *OldV, Value *NewV) const {353 return TTIImpl->rewriteIntrinsicWithAddressSpace(II, OldV, NewV);354}355 356bool TargetTransformInfo::isLoweredToCall(const Function *F) const {357 return TTIImpl->isLoweredToCall(F);358}359 360bool TargetTransformInfo::isHardwareLoopProfitable(361 Loop *L, ScalarEvolution &SE, AssumptionCache &AC,362 TargetLibraryInfo *LibInfo, HardwareLoopInfo &HWLoopInfo) const {363 return TTIImpl->isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo);364}365 366unsigned TargetTransformInfo::getEpilogueVectorizationMinVF() const {367 return TTIImpl->getEpilogueVectorizationMinVF();368}369 370bool TargetTransformInfo::preferPredicateOverEpilogue(371 TailFoldingInfo *TFI) const {372 return TTIImpl->preferPredicateOverEpilogue(TFI);373}374 375TailFoldingStyle TargetTransformInfo::getPreferredTailFoldingStyle(376 bool IVUpdateMayOverflow) const {377 return TTIImpl->getPreferredTailFoldingStyle(IVUpdateMayOverflow);378}379 380std::optional<Instruction *>381TargetTransformInfo::instCombineIntrinsic(InstCombiner &IC,382 IntrinsicInst &II) const {383 return TTIImpl->instCombineIntrinsic(IC, II);384}385 386std::optional<Value *> TargetTransformInfo::simplifyDemandedUseBitsIntrinsic(387 InstCombiner &IC, IntrinsicInst &II, APInt DemandedMask, KnownBits &Known,388 bool &KnownBitsComputed) const {389 return TTIImpl->simplifyDemandedUseBitsIntrinsic(IC, II, DemandedMask, Known,390 KnownBitsComputed);391}392 393std::optional<Value *> TargetTransformInfo::simplifyDemandedVectorEltsIntrinsic(394 InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,395 APInt &UndefElts2, APInt &UndefElts3,396 std::function<void(Instruction *, unsigned, APInt, APInt &)>397 SimplifyAndSetOp) const {398 return TTIImpl->simplifyDemandedVectorEltsIntrinsic(399 IC, II, DemandedElts, UndefElts, UndefElts2, UndefElts3,400 SimplifyAndSetOp);401}402 403void TargetTransformInfo::getUnrollingPreferences(404 Loop *L, ScalarEvolution &SE, UnrollingPreferences &UP,405 OptimizationRemarkEmitter *ORE) const {406 return TTIImpl->getUnrollingPreferences(L, SE, UP, ORE);407}408 409void TargetTransformInfo::getPeelingPreferences(Loop *L, ScalarEvolution &SE,410 PeelingPreferences &PP) const {411 return TTIImpl->getPeelingPreferences(L, SE, PP);412}413 414bool TargetTransformInfo::isLegalAddImmediate(int64_t Imm) const {415 return TTIImpl->isLegalAddImmediate(Imm);416}417 418bool TargetTransformInfo::isLegalAddScalableImmediate(int64_t Imm) const {419 return TTIImpl->isLegalAddScalableImmediate(Imm);420}421 422bool TargetTransformInfo::isLegalICmpImmediate(int64_t Imm) const {423 return TTIImpl->isLegalICmpImmediate(Imm);424}425 426bool TargetTransformInfo::isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV,427 int64_t BaseOffset,428 bool HasBaseReg, int64_t Scale,429 unsigned AddrSpace,430 Instruction *I,431 int64_t ScalableOffset) const {432 return TTIImpl->isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg,433 Scale, AddrSpace, I, ScalableOffset);434}435 436bool TargetTransformInfo::isLSRCostLess(const LSRCost &C1,437 const LSRCost &C2) const {438 return TTIImpl->isLSRCostLess(C1, C2);439}440 441bool TargetTransformInfo::isNumRegsMajorCostOfLSR() const {442 return TTIImpl->isNumRegsMajorCostOfLSR();443}444 445bool TargetTransformInfo::shouldDropLSRSolutionIfLessProfitable() const {446 return TTIImpl->shouldDropLSRSolutionIfLessProfitable();447}448 449bool TargetTransformInfo::isProfitableLSRChainElement(Instruction *I) const {450 return TTIImpl->isProfitableLSRChainElement(I);451}452 453bool TargetTransformInfo::canMacroFuseCmp() const {454 return TTIImpl->canMacroFuseCmp();455}456 457bool TargetTransformInfo::canSaveCmp(Loop *L, BranchInst **BI,458 ScalarEvolution *SE, LoopInfo *LI,459 DominatorTree *DT, AssumptionCache *AC,460 TargetLibraryInfo *LibInfo) const {461 return TTIImpl->canSaveCmp(L, BI, SE, LI, DT, AC, LibInfo);462}463 464TTI::AddressingModeKind465TargetTransformInfo::getPreferredAddressingMode(const Loop *L,466 ScalarEvolution *SE) const {467 return TTIImpl->getPreferredAddressingMode(L, SE);468}469 470bool TargetTransformInfo::isLegalMaskedStore(Type *DataType, Align Alignment,471 unsigned AddressSpace,472 TTI::MaskKind MaskKind) const {473 return TTIImpl->isLegalMaskedStore(DataType, Alignment, AddressSpace,474 MaskKind);475}476 477bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType, Align Alignment,478 unsigned AddressSpace,479 TTI::MaskKind MaskKind) const {480 return TTIImpl->isLegalMaskedLoad(DataType, Alignment, AddressSpace,481 MaskKind);482}483 484bool TargetTransformInfo::isLegalNTStore(Type *DataType,485 Align Alignment) const {486 return TTIImpl->isLegalNTStore(DataType, Alignment);487}488 489bool TargetTransformInfo::isLegalNTLoad(Type *DataType, Align Alignment) const {490 return TTIImpl->isLegalNTLoad(DataType, Alignment);491}492 493bool TargetTransformInfo::isLegalBroadcastLoad(Type *ElementTy,494 ElementCount NumElements) const {495 return TTIImpl->isLegalBroadcastLoad(ElementTy, NumElements);496}497 498bool TargetTransformInfo::isLegalMaskedGather(Type *DataType,499 Align Alignment) const {500 return TTIImpl->isLegalMaskedGather(DataType, Alignment);501}502 503bool TargetTransformInfo::isLegalAltInstr(504 VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,505 const SmallBitVector &OpcodeMask) const {506 return TTIImpl->isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask);507}508 509bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType,510 Align Alignment) const {511 return TTIImpl->isLegalMaskedScatter(DataType, Alignment);512}513 514bool TargetTransformInfo::forceScalarizeMaskedGather(VectorType *DataType,515 Align Alignment) const {516 return TTIImpl->forceScalarizeMaskedGather(DataType, Alignment);517}518 519bool TargetTransformInfo::forceScalarizeMaskedScatter(VectorType *DataType,520 Align Alignment) const {521 return TTIImpl->forceScalarizeMaskedScatter(DataType, Alignment);522}523 524bool TargetTransformInfo::isLegalMaskedCompressStore(Type *DataType,525 Align Alignment) const {526 return TTIImpl->isLegalMaskedCompressStore(DataType, Alignment);527}528 529bool TargetTransformInfo::isLegalMaskedExpandLoad(Type *DataType,530 Align Alignment) const {531 return TTIImpl->isLegalMaskedExpandLoad(DataType, Alignment);532}533 534bool TargetTransformInfo::isLegalStridedLoadStore(Type *DataType,535 Align Alignment) const {536 return TTIImpl->isLegalStridedLoadStore(DataType, Alignment);537}538 539bool TargetTransformInfo::isLegalInterleavedAccessType(540 VectorType *VTy, unsigned Factor, Align Alignment,541 unsigned AddrSpace) const {542 return TTIImpl->isLegalInterleavedAccessType(VTy, Factor, Alignment,543 AddrSpace);544}545 546bool TargetTransformInfo::isLegalMaskedVectorHistogram(Type *AddrType,547 Type *DataType) const {548 return TTIImpl->isLegalMaskedVectorHistogram(AddrType, DataType);549}550 551bool TargetTransformInfo::enableOrderedReductions() const {552 return TTIImpl->enableOrderedReductions();553}554 555bool TargetTransformInfo::hasDivRemOp(Type *DataType, bool IsSigned) const {556 return TTIImpl->hasDivRemOp(DataType, IsSigned);557}558 559bool TargetTransformInfo::hasVolatileVariant(Instruction *I,560 unsigned AddrSpace) const {561 return TTIImpl->hasVolatileVariant(I, AddrSpace);562}563 564bool TargetTransformInfo::prefersVectorizedAddressing() const {565 return TTIImpl->prefersVectorizedAddressing();566}567 568InstructionCost TargetTransformInfo::getScalingFactorCost(569 Type *Ty, GlobalValue *BaseGV, StackOffset BaseOffset, bool HasBaseReg,570 int64_t Scale, unsigned AddrSpace) const {571 InstructionCost Cost = TTIImpl->getScalingFactorCost(572 Ty, BaseGV, BaseOffset, HasBaseReg, Scale, AddrSpace);573 assert(Cost >= 0 && "TTI should not produce negative costs!");574 return Cost;575}576 577bool TargetTransformInfo::LSRWithInstrQueries() const {578 return TTIImpl->LSRWithInstrQueries();579}580 581bool TargetTransformInfo::isTruncateFree(Type *Ty1, Type *Ty2) const {582 return TTIImpl->isTruncateFree(Ty1, Ty2);583}584 585bool TargetTransformInfo::isProfitableToHoist(Instruction *I) const {586 return TTIImpl->isProfitableToHoist(I);587}588 589bool TargetTransformInfo::useAA() const { return TTIImpl->useAA(); }590 591bool TargetTransformInfo::isTypeLegal(Type *Ty) const {592 return TTIImpl->isTypeLegal(Ty);593}594 595unsigned TargetTransformInfo::getRegUsageForType(Type *Ty) const {596 return TTIImpl->getRegUsageForType(Ty);597}598 599bool TargetTransformInfo::shouldBuildLookupTables() const {600 return TTIImpl->shouldBuildLookupTables();601}602 603bool TargetTransformInfo::shouldBuildLookupTablesForConstant(604 Constant *C) const {605 return TTIImpl->shouldBuildLookupTablesForConstant(C);606}607 608bool TargetTransformInfo::shouldBuildRelLookupTables() const {609 return TTIImpl->shouldBuildRelLookupTables();610}611 612bool TargetTransformInfo::useColdCCForColdCall(Function &F) const {613 return TTIImpl->useColdCCForColdCall(F);614}615 616bool TargetTransformInfo::useFastCCForInternalCall(Function &F) const {617 return TTIImpl->useFastCCForInternalCall(F);618}619 620bool TargetTransformInfo::isTargetIntrinsicTriviallyScalarizable(621 Intrinsic::ID ID) const {622 return TTIImpl->isTargetIntrinsicTriviallyScalarizable(ID);623}624 625bool TargetTransformInfo::isTargetIntrinsicWithScalarOpAtArg(626 Intrinsic::ID ID, unsigned ScalarOpdIdx) const {627 return TTIImpl->isTargetIntrinsicWithScalarOpAtArg(ID, ScalarOpdIdx);628}629 630bool TargetTransformInfo::isTargetIntrinsicWithOverloadTypeAtArg(631 Intrinsic::ID ID, int OpdIdx) const {632 return TTIImpl->isTargetIntrinsicWithOverloadTypeAtArg(ID, OpdIdx);633}634 635bool TargetTransformInfo::isTargetIntrinsicWithStructReturnOverloadAtField(636 Intrinsic::ID ID, int RetIdx) const {637 return TTIImpl->isTargetIntrinsicWithStructReturnOverloadAtField(ID, RetIdx);638}639 640InstructionCost TargetTransformInfo::getScalarizationOverhead(641 VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,642 TTI::TargetCostKind CostKind, bool ForPoisonSrc,643 ArrayRef<Value *> VL) const {644 return TTIImpl->getScalarizationOverhead(Ty, DemandedElts, Insert, Extract,645 CostKind, ForPoisonSrc, VL);646}647 648InstructionCost TargetTransformInfo::getOperandsScalarizationOverhead(649 ArrayRef<Type *> Tys, TTI::TargetCostKind CostKind) const {650 return TTIImpl->getOperandsScalarizationOverhead(Tys, CostKind);651}652 653bool TargetTransformInfo::supportsEfficientVectorElementLoadStore() const {654 return TTIImpl->supportsEfficientVectorElementLoadStore();655}656 657bool TargetTransformInfo::supportsTailCalls() const {658 return TTIImpl->supportsTailCalls();659}660 661bool TargetTransformInfo::supportsTailCallFor(const CallBase *CB) const {662 return TTIImpl->supportsTailCallFor(CB);663}664 665bool TargetTransformInfo::enableAggressiveInterleaving(666 bool LoopHasReductions) const {667 return TTIImpl->enableAggressiveInterleaving(LoopHasReductions);668}669 670TargetTransformInfo::MemCmpExpansionOptions671TargetTransformInfo::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {672 return TTIImpl->enableMemCmpExpansion(OptSize, IsZeroCmp);673}674 675bool TargetTransformInfo::enableSelectOptimize() const {676 return TTIImpl->enableSelectOptimize();677}678 679bool TargetTransformInfo::shouldTreatInstructionLikeSelect(680 const Instruction *I) const {681 return TTIImpl->shouldTreatInstructionLikeSelect(I);682}683 684bool TargetTransformInfo::enableInterleavedAccessVectorization() const {685 return TTIImpl->enableInterleavedAccessVectorization();686}687 688bool TargetTransformInfo::enableMaskedInterleavedAccessVectorization() const {689 return TTIImpl->enableMaskedInterleavedAccessVectorization();690}691 692bool TargetTransformInfo::isFPVectorizationPotentiallyUnsafe() const {693 return TTIImpl->isFPVectorizationPotentiallyUnsafe();694}695 696bool697TargetTransformInfo::allowsMisalignedMemoryAccesses(LLVMContext &Context,698 unsigned BitWidth,699 unsigned AddressSpace,700 Align Alignment,701 unsigned *Fast) const {702 return TTIImpl->allowsMisalignedMemoryAccesses(Context, BitWidth,703 AddressSpace, Alignment, Fast);704}705 706TargetTransformInfo::PopcntSupportKind707TargetTransformInfo::getPopcntSupport(unsigned IntTyWidthInBit) const {708 return TTIImpl->getPopcntSupport(IntTyWidthInBit);709}710 711bool TargetTransformInfo::haveFastSqrt(Type *Ty) const {712 return TTIImpl->haveFastSqrt(Ty);713}714 715bool TargetTransformInfo::isExpensiveToSpeculativelyExecute(716 const Instruction *I) const {717 return TTIImpl->isExpensiveToSpeculativelyExecute(I);718}719 720bool TargetTransformInfo::isFCmpOrdCheaperThanFCmpZero(Type *Ty) const {721 return TTIImpl->isFCmpOrdCheaperThanFCmpZero(Ty);722}723 724InstructionCost TargetTransformInfo::getFPOpCost(Type *Ty) const {725 InstructionCost Cost = TTIImpl->getFPOpCost(Ty);726 assert(Cost >= 0 && "TTI should not produce negative costs!");727 return Cost;728}729 730InstructionCost TargetTransformInfo::getIntImmCodeSizeCost(unsigned Opcode,731 unsigned Idx,732 const APInt &Imm,733 Type *Ty) const {734 InstructionCost Cost = TTIImpl->getIntImmCodeSizeCost(Opcode, Idx, Imm, Ty);735 assert(Cost >= 0 && "TTI should not produce negative costs!");736 return Cost;737}738 739InstructionCost740TargetTransformInfo::getIntImmCost(const APInt &Imm, Type *Ty,741 TTI::TargetCostKind CostKind) const {742 InstructionCost Cost = TTIImpl->getIntImmCost(Imm, Ty, CostKind);743 assert(Cost >= 0 && "TTI should not produce negative costs!");744 return Cost;745}746 747InstructionCost TargetTransformInfo::getIntImmCostInst(748 unsigned Opcode, unsigned Idx, const APInt &Imm, Type *Ty,749 TTI::TargetCostKind CostKind, Instruction *Inst) const {750 InstructionCost Cost =751 TTIImpl->getIntImmCostInst(Opcode, Idx, Imm, Ty, CostKind, Inst);752 assert(Cost >= 0 && "TTI should not produce negative costs!");753 return Cost;754}755 756InstructionCost757TargetTransformInfo::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,758 const APInt &Imm, Type *Ty,759 TTI::TargetCostKind CostKind) const {760 InstructionCost Cost =761 TTIImpl->getIntImmCostIntrin(IID, Idx, Imm, Ty, CostKind);762 assert(Cost >= 0 && "TTI should not produce negative costs!");763 return Cost;764}765 766bool TargetTransformInfo::preferToKeepConstantsAttached(767 const Instruction &Inst, const Function &Fn) const {768 return TTIImpl->preferToKeepConstantsAttached(Inst, Fn);769}770 771unsigned TargetTransformInfo::getNumberOfRegisters(unsigned ClassID) const {772 return TTIImpl->getNumberOfRegisters(ClassID);773}774 775bool TargetTransformInfo::hasConditionalLoadStoreForType(Type *Ty,776 bool IsStore) const {777 return TTIImpl->hasConditionalLoadStoreForType(Ty, IsStore);778}779 780unsigned TargetTransformInfo::getRegisterClassForType(bool Vector,781 Type *Ty) const {782 return TTIImpl->getRegisterClassForType(Vector, Ty);783}784 785const char *TargetTransformInfo::getRegisterClassName(unsigned ClassID) const {786 return TTIImpl->getRegisterClassName(ClassID);787}788 789TypeSize TargetTransformInfo::getRegisterBitWidth(790 TargetTransformInfo::RegisterKind K) const {791 return TTIImpl->getRegisterBitWidth(K);792}793 794unsigned TargetTransformInfo::getMinVectorRegisterBitWidth() const {795 return TTIImpl->getMinVectorRegisterBitWidth();796}797 798std::optional<unsigned> TargetTransformInfo::getMaxVScale() const {799 return TTIImpl->getMaxVScale();800}801 802std::optional<unsigned> TargetTransformInfo::getVScaleForTuning() const {803 return TTIImpl->getVScaleForTuning();804}805 806bool TargetTransformInfo::isVScaleKnownToBeAPowerOfTwo() const {807 return TTIImpl->isVScaleKnownToBeAPowerOfTwo();808}809 810bool TargetTransformInfo::shouldMaximizeVectorBandwidth(811 TargetTransformInfo::RegisterKind K) const {812 return TTIImpl->shouldMaximizeVectorBandwidth(K);813}814 815ElementCount TargetTransformInfo::getMinimumVF(unsigned ElemWidth,816 bool IsScalable) const {817 return TTIImpl->getMinimumVF(ElemWidth, IsScalable);818}819 820unsigned TargetTransformInfo::getMaximumVF(unsigned ElemWidth,821 unsigned Opcode) const {822 return TTIImpl->getMaximumVF(ElemWidth, Opcode);823}824 825unsigned TargetTransformInfo::getStoreMinimumVF(unsigned VF, Type *ScalarMemTy,826 Type *ScalarValTy) const {827 return TTIImpl->getStoreMinimumVF(VF, ScalarMemTy, ScalarValTy);828}829 830bool TargetTransformInfo::shouldConsiderAddressTypePromotion(831 const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const {832 return TTIImpl->shouldConsiderAddressTypePromotion(833 I, AllowPromotionWithoutCommonHeader);834}835 836unsigned TargetTransformInfo::getCacheLineSize() const {837 return CacheLineSize.getNumOccurrences() > 0 ? CacheLineSize838 : TTIImpl->getCacheLineSize();839}840 841std::optional<unsigned>842TargetTransformInfo::getCacheSize(CacheLevel Level) const {843 return TTIImpl->getCacheSize(Level);844}845 846std::optional<unsigned>847TargetTransformInfo::getCacheAssociativity(CacheLevel Level) const {848 return TTIImpl->getCacheAssociativity(Level);849}850 851std::optional<unsigned> TargetTransformInfo::getMinPageSize() const {852 return MinPageSize.getNumOccurrences() > 0 ? MinPageSize853 : TTIImpl->getMinPageSize();854}855 856unsigned TargetTransformInfo::getPrefetchDistance() const {857 return TTIImpl->getPrefetchDistance();858}859 860unsigned TargetTransformInfo::getMinPrefetchStride(861 unsigned NumMemAccesses, unsigned NumStridedMemAccesses,862 unsigned NumPrefetches, bool HasCall) const {863 return TTIImpl->getMinPrefetchStride(NumMemAccesses, NumStridedMemAccesses,864 NumPrefetches, HasCall);865}866 867unsigned TargetTransformInfo::getMaxPrefetchIterationsAhead() const {868 return TTIImpl->getMaxPrefetchIterationsAhead();869}870 871bool TargetTransformInfo::enableWritePrefetching() const {872 return TTIImpl->enableWritePrefetching();873}874 875bool TargetTransformInfo::shouldPrefetchAddressSpace(unsigned AS) const {876 return TTIImpl->shouldPrefetchAddressSpace(AS);877}878 879InstructionCost TargetTransformInfo::getPartialReductionCost(880 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,881 ElementCount VF, PartialReductionExtendKind OpAExtend,882 PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,883 TTI::TargetCostKind CostKind) const {884 return TTIImpl->getPartialReductionCost(Opcode, InputTypeA, InputTypeB,885 AccumType, VF, OpAExtend, OpBExtend,886 BinOp, CostKind);887}888 889unsigned TargetTransformInfo::getMaxInterleaveFactor(ElementCount VF) const {890 return TTIImpl->getMaxInterleaveFactor(VF);891}892 893TargetTransformInfo::OperandValueInfo894TargetTransformInfo::getOperandInfo(const Value *V) {895 OperandValueKind OpInfo = OK_AnyValue;896 OperandValueProperties OpProps = OP_None;897 898 // undef/poison don't materialize constants.899 if (isa<UndefValue>(V))900 return {OK_AnyValue, OP_None};901 902 if (isa<ConstantInt>(V) || isa<ConstantFP>(V)) {903 if (const auto *CI = dyn_cast<ConstantInt>(V)) {904 if (CI->getValue().isPowerOf2())905 OpProps = OP_PowerOf2;906 else if (CI->getValue().isNegatedPowerOf2())907 OpProps = OP_NegatedPowerOf2;908 }909 return {OK_UniformConstantValue, OpProps};910 }911 912 // A broadcast shuffle creates a uniform value.913 // TODO: Add support for non-zero index broadcasts.914 // TODO: Add support for different source vector width.915 if (const auto *ShuffleInst = dyn_cast<ShuffleVectorInst>(V))916 if (ShuffleInst->isZeroEltSplat())917 OpInfo = OK_UniformValue;918 919 const Value *Splat = getSplatValue(V);920 921 // Check for a splat of a constant or for a non uniform vector of constants922 // and check if the constant(s) are all powers of two.923 if (Splat) {924 // Check for a splat of a uniform value. This is not loop aware, so return925 // true only for the obviously uniform cases (argument, globalvalue)926 if (isa<Argument>(Splat) || isa<GlobalValue>(Splat)) {927 OpInfo = OK_UniformValue;928 } else if (isa<Constant>(Splat)) {929 OpInfo = OK_UniformConstantValue;930 if (auto *CI = dyn_cast<ConstantInt>(Splat)) {931 if (CI->getValue().isPowerOf2())932 OpProps = OP_PowerOf2;933 else if (CI->getValue().isNegatedPowerOf2())934 OpProps = OP_NegatedPowerOf2;935 }936 }937 } else if (const auto *CDS = dyn_cast<ConstantDataSequential>(V)) {938 OpInfo = OK_NonUniformConstantValue;939 bool AllPow2 = true, AllNegPow2 = true;940 for (uint64_t I = 0, E = CDS->getNumElements(); I != E; ++I) {941 if (auto *CI = dyn_cast<ConstantInt>(CDS->getElementAsConstant(I))) {942 AllPow2 &= CI->getValue().isPowerOf2();943 AllNegPow2 &= CI->getValue().isNegatedPowerOf2();944 if (AllPow2 || AllNegPow2)945 continue;946 }947 AllPow2 = AllNegPow2 = false;948 break;949 }950 OpProps = AllPow2 ? OP_PowerOf2 : OpProps;951 OpProps = AllNegPow2 ? OP_NegatedPowerOf2 : OpProps;952 } else if (isa<ConstantVector>(V) || isa<ConstantDataVector>(V)) {953 OpInfo = OK_NonUniformConstantValue;954 }955 956 return {OpInfo, OpProps};957}958 959InstructionCost TargetTransformInfo::getArithmeticInstrCost(960 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,961 OperandValueInfo Op1Info, OperandValueInfo Op2Info,962 ArrayRef<const Value *> Args, const Instruction *CxtI,963 const TargetLibraryInfo *TLibInfo) const {964 965 // Use call cost for frem intructions that have platform specific vector math966 // functions, as those will be replaced with calls later by SelectionDAG or967 // ReplaceWithVecLib pass.968 if (TLibInfo && Opcode == Instruction::FRem) {969 VectorType *VecTy = dyn_cast<VectorType>(Ty);970 LibFunc Func;971 if (VecTy &&972 TLibInfo->getLibFunc(Instruction::FRem, Ty->getScalarType(), Func) &&973 TLibInfo->isFunctionVectorizable(TLibInfo->getName(Func),974 VecTy->getElementCount()))975 return getCallInstrCost(nullptr, VecTy, {VecTy, VecTy}, CostKind);976 }977 978 InstructionCost Cost =979 TTIImpl->getArithmeticInstrCost(Opcode, Ty, CostKind,980 Op1Info, Op2Info,981 Args, CxtI);982 assert(Cost >= 0 && "TTI should not produce negative costs!");983 return Cost;984}985 986InstructionCost TargetTransformInfo::getAltInstrCost(987 VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,988 const SmallBitVector &OpcodeMask, TTI::TargetCostKind CostKind) const {989 InstructionCost Cost =990 TTIImpl->getAltInstrCost(VecTy, Opcode0, Opcode1, OpcodeMask, CostKind);991 assert(Cost >= 0 && "TTI should not produce negative costs!");992 return Cost;993}994 995InstructionCost TargetTransformInfo::getShuffleCost(996 ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, ArrayRef<int> Mask,997 TTI::TargetCostKind CostKind, int Index, VectorType *SubTp,998 ArrayRef<const Value *> Args, const Instruction *CxtI) const {999 assert((Mask.empty() || DstTy->isScalableTy() ||1000 Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&1001 "Expected the Mask to match the return size if given");1002 assert(SrcTy->getScalarType() == DstTy->getScalarType() &&1003 "Expected the same scalar types");1004 InstructionCost Cost = TTIImpl->getShuffleCost(1005 Kind, DstTy, SrcTy, Mask, CostKind, Index, SubTp, Args, CxtI);1006 assert(Cost >= 0 && "TTI should not produce negative costs!");1007 return Cost;1008}1009 1010TargetTransformInfo::PartialReductionExtendKind1011TargetTransformInfo::getPartialReductionExtendKind(Instruction *I) {1012 if (auto *Cast = dyn_cast<CastInst>(I))1013 return getPartialReductionExtendKind(Cast->getOpcode());1014 return PR_None;1015}1016 1017TargetTransformInfo::PartialReductionExtendKind1018TargetTransformInfo::getPartialReductionExtendKind(1019 Instruction::CastOps CastOpc) {1020 switch (CastOpc) {1021 case Instruction::CastOps::ZExt:1022 return PR_ZeroExtend;1023 case Instruction::CastOps::SExt:1024 return PR_SignExtend;1025 default:1026 return PR_None;1027 }1028 llvm_unreachable("Unhandled cast opcode");1029}1030 1031TTI::CastContextHint1032TargetTransformInfo::getCastContextHint(const Instruction *I) {1033 if (!I)1034 return CastContextHint::None;1035 1036 auto getLoadStoreKind = [](const Value *V, unsigned LdStOp, unsigned MaskedOp,1037 unsigned GatScatOp) {1038 const Instruction *I = dyn_cast<Instruction>(V);1039 if (!I)1040 return CastContextHint::None;1041 1042 if (I->getOpcode() == LdStOp)1043 return CastContextHint::Normal;1044 1045 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {1046 if (II->getIntrinsicID() == MaskedOp)1047 return TTI::CastContextHint::Masked;1048 if (II->getIntrinsicID() == GatScatOp)1049 return TTI::CastContextHint::GatherScatter;1050 }1051 1052 return TTI::CastContextHint::None;1053 };1054 1055 switch (I->getOpcode()) {1056 case Instruction::ZExt:1057 case Instruction::SExt:1058 case Instruction::FPExt:1059 return getLoadStoreKind(I->getOperand(0), Instruction::Load,1060 Intrinsic::masked_load, Intrinsic::masked_gather);1061 case Instruction::Trunc:1062 case Instruction::FPTrunc:1063 if (I->hasOneUse())1064 return getLoadStoreKind(*I->user_begin(), Instruction::Store,1065 Intrinsic::masked_store,1066 Intrinsic::masked_scatter);1067 break;1068 default:1069 return CastContextHint::None;1070 }1071 1072 return TTI::CastContextHint::None;1073}1074 1075InstructionCost TargetTransformInfo::getCastInstrCost(1076 unsigned Opcode, Type *Dst, Type *Src, CastContextHint CCH,1077 TTI::TargetCostKind CostKind, const Instruction *I) const {1078 assert((I == nullptr || I->getOpcode() == Opcode) &&1079 "Opcode should reflect passed instruction.");1080 InstructionCost Cost =1081 TTIImpl->getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);1082 assert(Cost >= 0 && "TTI should not produce negative costs!");1083 return Cost;1084}1085 1086InstructionCost TargetTransformInfo::getExtractWithExtendCost(1087 unsigned Opcode, Type *Dst, VectorType *VecTy, unsigned Index,1088 TTI::TargetCostKind CostKind) const {1089 InstructionCost Cost =1090 TTIImpl->getExtractWithExtendCost(Opcode, Dst, VecTy, Index, CostKind);1091 assert(Cost >= 0 && "TTI should not produce negative costs!");1092 return Cost;1093}1094 1095InstructionCost TargetTransformInfo::getCFInstrCost(1096 unsigned Opcode, TTI::TargetCostKind CostKind, const Instruction *I) const {1097 assert((I == nullptr || I->getOpcode() == Opcode) &&1098 "Opcode should reflect passed instruction.");1099 InstructionCost Cost = TTIImpl->getCFInstrCost(Opcode, CostKind, I);1100 assert(Cost >= 0 && "TTI should not produce negative costs!");1101 return Cost;1102}1103 1104InstructionCost TargetTransformInfo::getCmpSelInstrCost(1105 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,1106 TTI::TargetCostKind CostKind, OperandValueInfo Op1Info,1107 OperandValueInfo Op2Info, const Instruction *I) const {1108 assert((I == nullptr || I->getOpcode() == Opcode) &&1109 "Opcode should reflect passed instruction.");1110 InstructionCost Cost = TTIImpl->getCmpSelInstrCost(1111 Opcode, ValTy, CondTy, VecPred, CostKind, Op1Info, Op2Info, I);1112 assert(Cost >= 0 && "TTI should not produce negative costs!");1113 return Cost;1114}1115 1116InstructionCost TargetTransformInfo::getVectorInstrCost(1117 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,1118 const Value *Op0, const Value *Op1) const {1119 assert((Opcode == Instruction::InsertElement ||1120 Opcode == Instruction::ExtractElement) &&1121 "Expecting Opcode to be insertelement/extractelement.");1122 InstructionCost Cost =1123 TTIImpl->getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1);1124 assert(Cost >= 0 && "TTI should not produce negative costs!");1125 return Cost;1126}1127 1128InstructionCost TargetTransformInfo::getVectorInstrCost(1129 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,1130 Value *Scalar,1131 ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) const {1132 assert((Opcode == Instruction::InsertElement ||1133 Opcode == Instruction::ExtractElement) &&1134 "Expecting Opcode to be insertelement/extractelement.");1135 InstructionCost Cost = TTIImpl->getVectorInstrCost(1136 Opcode, Val, CostKind, Index, Scalar, ScalarUserAndIdx);1137 assert(Cost >= 0 && "TTI should not produce negative costs!");1138 return Cost;1139}1140 1141InstructionCost1142TargetTransformInfo::getVectorInstrCost(const Instruction &I, Type *Val,1143 TTI::TargetCostKind CostKind,1144 unsigned Index) const {1145 // FIXME: Assert that Opcode is either InsertElement or ExtractElement.1146 // This is mentioned in the interface description and respected by all1147 // callers, but never asserted upon.1148 InstructionCost Cost = TTIImpl->getVectorInstrCost(I, Val, CostKind, Index);1149 assert(Cost >= 0 && "TTI should not produce negative costs!");1150 return Cost;1151}1152 1153InstructionCost TargetTransformInfo::getIndexedVectorInstrCostFromEnd(1154 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind,1155 unsigned Index) const {1156 InstructionCost Cost =1157 TTIImpl->getIndexedVectorInstrCostFromEnd(Opcode, Val, CostKind, Index);1158 assert(Cost >= 0 && "TTI should not produce negative costs!");1159 return Cost;1160}1161 1162InstructionCost TargetTransformInfo::getInsertExtractValueCost(1163 unsigned Opcode, TTI::TargetCostKind CostKind) const {1164 assert((Opcode == Instruction::InsertValue ||1165 Opcode == Instruction::ExtractValue) &&1166 "Expecting Opcode to be insertvalue/extractvalue.");1167 InstructionCost Cost = TTIImpl->getInsertExtractValueCost(Opcode, CostKind);1168 assert(Cost >= 0 && "TTI should not produce negative costs!");1169 return Cost;1170}1171 1172InstructionCost TargetTransformInfo::getReplicationShuffleCost(1173 Type *EltTy, int ReplicationFactor, int VF, const APInt &DemandedDstElts,1174 TTI::TargetCostKind CostKind) const {1175 InstructionCost Cost = TTIImpl->getReplicationShuffleCost(1176 EltTy, ReplicationFactor, VF, DemandedDstElts, CostKind);1177 assert(Cost >= 0 && "TTI should not produce negative costs!");1178 return Cost;1179}1180 1181InstructionCost TargetTransformInfo::getMemoryOpCost(1182 unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,1183 TTI::TargetCostKind CostKind, TTI::OperandValueInfo OpInfo,1184 const Instruction *I) const {1185 assert((I == nullptr || I->getOpcode() == Opcode) &&1186 "Opcode should reflect passed instruction.");1187 InstructionCost Cost = TTIImpl->getMemoryOpCost(1188 Opcode, Src, Alignment, AddressSpace, CostKind, OpInfo, I);1189 assert(Cost >= 0 && "TTI should not produce negative costs!");1190 return Cost;1191}1192 1193InstructionCost TargetTransformInfo::getInterleavedMemoryOpCost(1194 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,1195 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,1196 bool UseMaskForCond, bool UseMaskForGaps) const {1197 InstructionCost Cost = TTIImpl->getInterleavedMemoryOpCost(1198 Opcode, VecTy, Factor, Indices, Alignment, AddressSpace, CostKind,1199 UseMaskForCond, UseMaskForGaps);1200 assert(Cost >= 0 && "TTI should not produce negative costs!");1201 return Cost;1202}1203 1204InstructionCost1205TargetTransformInfo::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,1206 TTI::TargetCostKind CostKind) const {1207 InstructionCost Cost = TTIImpl->getIntrinsicInstrCost(ICA, CostKind);1208 assert(Cost >= 0 && "TTI should not produce negative costs!");1209 return Cost;1210}1211 1212InstructionCost TargetTransformInfo::getMemIntrinsicInstrCost(1213 const MemIntrinsicCostAttributes &MICA,1214 TTI::TargetCostKind CostKind) const {1215 InstructionCost Cost = TTIImpl->getMemIntrinsicInstrCost(MICA, CostKind);1216 assert(Cost >= 0 && "TTI should not produce negative costs!");1217 return Cost;1218}1219 1220InstructionCost1221TargetTransformInfo::getCallInstrCost(Function *F, Type *RetTy,1222 ArrayRef<Type *> Tys,1223 TTI::TargetCostKind CostKind) const {1224 InstructionCost Cost = TTIImpl->getCallInstrCost(F, RetTy, Tys, CostKind);1225 assert(Cost >= 0 && "TTI should not produce negative costs!");1226 return Cost;1227}1228 1229unsigned TargetTransformInfo::getNumberOfParts(Type *Tp) const {1230 return TTIImpl->getNumberOfParts(Tp);1231}1232 1233InstructionCost TargetTransformInfo::getAddressComputationCost(1234 Type *PtrTy, ScalarEvolution *SE, const SCEV *Ptr,1235 TTI::TargetCostKind CostKind) const {1236 InstructionCost Cost =1237 TTIImpl->getAddressComputationCost(PtrTy, SE, Ptr, CostKind);1238 assert(Cost >= 0 && "TTI should not produce negative costs!");1239 return Cost;1240}1241 1242InstructionCost TargetTransformInfo::getMemcpyCost(const Instruction *I) const {1243 InstructionCost Cost = TTIImpl->getMemcpyCost(I);1244 assert(Cost >= 0 && "TTI should not produce negative costs!");1245 return Cost;1246}1247 1248uint64_t TargetTransformInfo::getMaxMemIntrinsicInlineSizeThreshold() const {1249 return TTIImpl->getMaxMemIntrinsicInlineSizeThreshold();1250}1251 1252InstructionCost TargetTransformInfo::getArithmeticReductionCost(1253 unsigned Opcode, VectorType *Ty, std::optional<FastMathFlags> FMF,1254 TTI::TargetCostKind CostKind) const {1255 InstructionCost Cost =1256 TTIImpl->getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);1257 assert(Cost >= 0 && "TTI should not produce negative costs!");1258 return Cost;1259}1260 1261InstructionCost TargetTransformInfo::getMinMaxReductionCost(1262 Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF,1263 TTI::TargetCostKind CostKind) const {1264 InstructionCost Cost =1265 TTIImpl->getMinMaxReductionCost(IID, Ty, FMF, CostKind);1266 assert(Cost >= 0 && "TTI should not produce negative costs!");1267 return Cost;1268}1269 1270InstructionCost TargetTransformInfo::getExtendedReductionCost(1271 unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *Ty,1272 std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) const {1273 return TTIImpl->getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty, FMF,1274 CostKind);1275}1276 1277InstructionCost TargetTransformInfo::getMulAccReductionCost(1278 bool IsUnsigned, unsigned RedOpcode, Type *ResTy, VectorType *Ty,1279 TTI::TargetCostKind CostKind) const {1280 return TTIImpl->getMulAccReductionCost(IsUnsigned, RedOpcode, ResTy, Ty,1281 CostKind);1282}1283 1284InstructionCost1285TargetTransformInfo::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {1286 return TTIImpl->getCostOfKeepingLiveOverCall(Tys);1287}1288 1289bool TargetTransformInfo::getTgtMemIntrinsic(IntrinsicInst *Inst,1290 MemIntrinsicInfo &Info) const {1291 return TTIImpl->getTgtMemIntrinsic(Inst, Info);1292}1293 1294unsigned TargetTransformInfo::getAtomicMemIntrinsicMaxElementSize() const {1295 return TTIImpl->getAtomicMemIntrinsicMaxElementSize();1296}1297 1298Value *TargetTransformInfo::getOrCreateResultFromMemIntrinsic(1299 IntrinsicInst *Inst, Type *ExpectedType, bool CanCreate) const {1300 return TTIImpl->getOrCreateResultFromMemIntrinsic(Inst, ExpectedType,1301 CanCreate);1302}1303 1304Type *TargetTransformInfo::getMemcpyLoopLoweringType(1305 LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,1306 unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,1307 std::optional<uint32_t> AtomicElementSize) const {1308 return TTIImpl->getMemcpyLoopLoweringType(Context, Length, SrcAddrSpace,1309 DestAddrSpace, SrcAlign, DestAlign,1310 AtomicElementSize);1311}1312 1313void TargetTransformInfo::getMemcpyLoopResidualLoweringType(1314 SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,1315 unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,1316 Align SrcAlign, Align DestAlign,1317 std::optional<uint32_t> AtomicCpySize) const {1318 TTIImpl->getMemcpyLoopResidualLoweringType(1319 OpsOut, Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, SrcAlign,1320 DestAlign, AtomicCpySize);1321}1322 1323bool TargetTransformInfo::areInlineCompatible(const Function *Caller,1324 const Function *Callee) const {1325 return TTIImpl->areInlineCompatible(Caller, Callee);1326}1327 1328unsigned1329TargetTransformInfo::getInlineCallPenalty(const Function *F,1330 const CallBase &Call,1331 unsigned DefaultCallPenalty) const {1332 return TTIImpl->getInlineCallPenalty(F, Call, DefaultCallPenalty);1333}1334 1335bool TargetTransformInfo::areTypesABICompatible(const Function *Caller,1336 const Function *Callee,1337 ArrayRef<Type *> Types) const {1338 return TTIImpl->areTypesABICompatible(Caller, Callee, Types);1339}1340 1341bool TargetTransformInfo::isIndexedLoadLegal(MemIndexedMode Mode,1342 Type *Ty) const {1343 return TTIImpl->isIndexedLoadLegal(Mode, Ty);1344}1345 1346bool TargetTransformInfo::isIndexedStoreLegal(MemIndexedMode Mode,1347 Type *Ty) const {1348 return TTIImpl->isIndexedStoreLegal(Mode, Ty);1349}1350 1351unsigned TargetTransformInfo::getLoadStoreVecRegBitWidth(unsigned AS) const {1352 return TTIImpl->getLoadStoreVecRegBitWidth(AS);1353}1354 1355bool TargetTransformInfo::isLegalToVectorizeLoad(LoadInst *LI) const {1356 return TTIImpl->isLegalToVectorizeLoad(LI);1357}1358 1359bool TargetTransformInfo::isLegalToVectorizeStore(StoreInst *SI) const {1360 return TTIImpl->isLegalToVectorizeStore(SI);1361}1362 1363bool TargetTransformInfo::isLegalToVectorizeLoadChain(1364 unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const {1365 return TTIImpl->isLegalToVectorizeLoadChain(ChainSizeInBytes, Alignment,1366 AddrSpace);1367}1368 1369bool TargetTransformInfo::isLegalToVectorizeStoreChain(1370 unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const {1371 return TTIImpl->isLegalToVectorizeStoreChain(ChainSizeInBytes, Alignment,1372 AddrSpace);1373}1374 1375bool TargetTransformInfo::isLegalToVectorizeReduction(1376 const RecurrenceDescriptor &RdxDesc, ElementCount VF) const {1377 return TTIImpl->isLegalToVectorizeReduction(RdxDesc, VF);1378}1379 1380bool TargetTransformInfo::isElementTypeLegalForScalableVector(Type *Ty) const {1381 return TTIImpl->isElementTypeLegalForScalableVector(Ty);1382}1383 1384unsigned TargetTransformInfo::getLoadVectorFactor(unsigned VF,1385 unsigned LoadSize,1386 unsigned ChainSizeInBytes,1387 VectorType *VecTy) const {1388 return TTIImpl->getLoadVectorFactor(VF, LoadSize, ChainSizeInBytes, VecTy);1389}1390 1391unsigned TargetTransformInfo::getStoreVectorFactor(unsigned VF,1392 unsigned StoreSize,1393 unsigned ChainSizeInBytes,1394 VectorType *VecTy) const {1395 return TTIImpl->getStoreVectorFactor(VF, StoreSize, ChainSizeInBytes, VecTy);1396}1397 1398bool TargetTransformInfo::preferFixedOverScalableIfEqualCost(1399 bool IsEpilogue) const {1400 return TTIImpl->preferFixedOverScalableIfEqualCost(IsEpilogue);1401}1402 1403bool TargetTransformInfo::preferInLoopReduction(RecurKind Kind,1404 Type *Ty) const {1405 return TTIImpl->preferInLoopReduction(Kind, Ty);1406}1407 1408bool TargetTransformInfo::preferAlternateOpcodeVectorization() const {1409 return TTIImpl->preferAlternateOpcodeVectorization();1410}1411 1412bool TargetTransformInfo::preferPredicatedReductionSelect() const {1413 return TTIImpl->preferPredicatedReductionSelect();1414}1415 1416bool TargetTransformInfo::preferEpilogueVectorization() const {1417 return TTIImpl->preferEpilogueVectorization();1418}1419 1420bool TargetTransformInfo::shouldConsiderVectorizationRegPressure() const {1421 return TTIImpl->shouldConsiderVectorizationRegPressure();1422}1423 1424TargetTransformInfo::VPLegalization1425TargetTransformInfo::getVPLegalizationStrategy(const VPIntrinsic &VPI) const {1426 return TTIImpl->getVPLegalizationStrategy(VPI);1427}1428 1429bool TargetTransformInfo::hasArmWideBranch(bool Thumb) const {1430 return TTIImpl->hasArmWideBranch(Thumb);1431}1432 1433APInt TargetTransformInfo::getFeatureMask(const Function &F) const {1434 return TTIImpl->getFeatureMask(F);1435}1436 1437bool TargetTransformInfo::isMultiversionedFunction(const Function &F) const {1438 return TTIImpl->isMultiversionedFunction(F);1439}1440 1441unsigned TargetTransformInfo::getMaxNumArgs() const {1442 return TTIImpl->getMaxNumArgs();1443}1444 1445bool TargetTransformInfo::shouldExpandReduction(const IntrinsicInst *II) const {1446 return TTIImpl->shouldExpandReduction(II);1447}1448 1449TargetTransformInfo::ReductionShuffle1450TargetTransformInfo::getPreferredExpandedReductionShuffle(1451 const IntrinsicInst *II) const {1452 return TTIImpl->getPreferredExpandedReductionShuffle(II);1453}1454 1455unsigned TargetTransformInfo::getGISelRematGlobalCost() const {1456 return TTIImpl->getGISelRematGlobalCost();1457}1458 1459unsigned TargetTransformInfo::getMinTripCountTailFoldingThreshold() const {1460 return TTIImpl->getMinTripCountTailFoldingThreshold();1461}1462 1463bool TargetTransformInfo::supportsScalableVectors() const {1464 return TTIImpl->supportsScalableVectors();1465}1466 1467bool TargetTransformInfo::enableScalableVectorization() const {1468 return TTIImpl->enableScalableVectorization();1469}1470 1471bool TargetTransformInfo::hasActiveVectorLength() const {1472 return TTIImpl->hasActiveVectorLength();1473}1474 1475bool TargetTransformInfo::isProfitableToSinkOperands(1476 Instruction *I, SmallVectorImpl<Use *> &OpsToSink) const {1477 return TTIImpl->isProfitableToSinkOperands(I, OpsToSink);1478}1479 1480bool TargetTransformInfo::isVectorShiftByScalarCheap(Type *Ty) const {1481 return TTIImpl->isVectorShiftByScalarCheap(Ty);1482}1483 1484unsigned1485TargetTransformInfo::getNumBytesToPadGlobalArray(unsigned Size,1486 Type *ArrayType) const {1487 return TTIImpl->getNumBytesToPadGlobalArray(Size, ArrayType);1488}1489 1490void TargetTransformInfo::collectKernelLaunchBounds(1491 const Function &F,1492 SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const {1493 return TTIImpl->collectKernelLaunchBounds(F, LB);1494}1495 1496bool TargetTransformInfo::allowVectorElementIndexingUsingGEP() const {1497 return TTIImpl->allowVectorElementIndexingUsingGEP();1498}1499 1500TargetTransformInfoImplBase::~TargetTransformInfoImplBase() = default;1501 1502TargetIRAnalysis::TargetIRAnalysis() : TTICallback(&getDefaultTTI) {}1503 1504TargetIRAnalysis::TargetIRAnalysis(1505 std::function<Result(const Function &)> TTICallback)1506 : TTICallback(std::move(TTICallback)) {}1507 1508TargetIRAnalysis::Result TargetIRAnalysis::run(const Function &F,1509 FunctionAnalysisManager &) {1510 assert(!F.isIntrinsic() && "Should not request TTI for intrinsics");1511 return TTICallback(F);1512}1513 1514AnalysisKey TargetIRAnalysis::Key;1515 1516TargetIRAnalysis::Result TargetIRAnalysis::getDefaultTTI(const Function &F) {1517 return Result(F.getDataLayout());1518}1519 1520// Register the basic pass.1521INITIALIZE_PASS(TargetTransformInfoWrapperPass, "tti",1522 "Target Transform Information", false, true)1523char TargetTransformInfoWrapperPass::ID = 0;1524 1525void TargetTransformInfoWrapperPass::anchor() {}1526 1527TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass()1528 : ImmutablePass(ID) {}1529 1530TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass(1531 TargetIRAnalysis TIRA)1532 : ImmutablePass(ID), TIRA(std::move(TIRA)) {}1533 1534TargetTransformInfo &TargetTransformInfoWrapperPass::getTTI(const Function &F) {1535 FunctionAnalysisManager DummyFAM;1536 TTI = TIRA.run(F, DummyFAM);1537 return *TTI;1538}1539 1540ImmutablePass *1541llvm::createTargetTransformInfoWrapperPass(TargetIRAnalysis TIRA) {1542 return new TargetTransformInfoWrapperPass(std::move(TIRA));1543}1544