1654 lines · cpp
1//===- AMDGPUTargetTransformInfo.cpp - AMDGPU specific TTI pass -----------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// \file10// This file implements a TargetTransformInfo analysis pass specific to the11// AMDGPU target machine. It uses the target's detailed information to provide12// more precise answers to certain TTI queries, while letting the target13// independent and default TTI implementations handle the rest.14//15//===----------------------------------------------------------------------===//16 17#include "AMDGPUTargetTransformInfo.h"18#include "AMDGPUTargetMachine.h"19#include "MCTargetDesc/AMDGPUMCTargetDesc.h"20#include "SIModeRegisterDefaults.h"21#include "llvm/Analysis/InlineCost.h"22#include "llvm/Analysis/LoopInfo.h"23#include "llvm/Analysis/ValueTracking.h"24#include "llvm/CodeGen/Analysis.h"25#include "llvm/IR/Function.h"26#include "llvm/IR/IRBuilder.h"27#include "llvm/IR/IntrinsicsAMDGPU.h"28#include "llvm/IR/PatternMatch.h"29#include "llvm/Support/KnownBits.h"30#include <optional>31 32using namespace llvm;33 34#define DEBUG_TYPE "AMDGPUtti"35 36static cl::opt<unsigned> UnrollThresholdPrivate(37 "amdgpu-unroll-threshold-private",38 cl::desc("Unroll threshold for AMDGPU if private memory used in a loop"),39 cl::init(2700), cl::Hidden);40 41static cl::opt<unsigned> UnrollThresholdLocal(42 "amdgpu-unroll-threshold-local",43 cl::desc("Unroll threshold for AMDGPU if local memory used in a loop"),44 cl::init(1000), cl::Hidden);45 46static cl::opt<unsigned> UnrollThresholdIf(47 "amdgpu-unroll-threshold-if",48 cl::desc("Unroll threshold increment for AMDGPU for each if statement inside loop"),49 cl::init(200), cl::Hidden);50 51static cl::opt<bool> UnrollRuntimeLocal(52 "amdgpu-unroll-runtime-local",53 cl::desc("Allow runtime unroll for AMDGPU if local memory used in a loop"),54 cl::init(true), cl::Hidden);55 56static cl::opt<unsigned> UnrollMaxBlockToAnalyze(57 "amdgpu-unroll-max-block-to-analyze",58 cl::desc("Inner loop block size threshold to analyze in unroll for AMDGPU"),59 cl::init(32), cl::Hidden);60 61static cl::opt<unsigned> ArgAllocaCost("amdgpu-inline-arg-alloca-cost",62 cl::Hidden, cl::init(4000),63 cl::desc("Cost of alloca argument"));64 65// If the amount of scratch memory to eliminate exceeds our ability to allocate66// it into registers we gain nothing by aggressively inlining functions for that67// heuristic.68static cl::opt<unsigned>69 ArgAllocaCutoff("amdgpu-inline-arg-alloca-cutoff", cl::Hidden,70 cl::init(256),71 cl::desc("Maximum alloca size to use for inline cost"));72 73// Inliner constraint to achieve reasonable compilation time.74static cl::opt<size_t> InlineMaxBB(75 "amdgpu-inline-max-bb", cl::Hidden, cl::init(1100),76 cl::desc("Maximum number of BBs allowed in a function after inlining"77 " (compile time constraint)"));78 79// This default unroll factor is based on microbenchmarks on gfx1030.80static cl::opt<unsigned> MemcpyLoopUnroll(81 "amdgpu-memcpy-loop-unroll",82 cl::desc("Unroll factor (affecting 4x32-bit operations) to use for memory "83 "operations when lowering memcpy as a loop"),84 cl::init(16), cl::Hidden);85 86static bool dependsOnLocalPhi(const Loop *L, const Value *Cond,87 unsigned Depth = 0) {88 const Instruction *I = dyn_cast<Instruction>(Cond);89 if (!I)90 return false;91 92 for (const Value *V : I->operand_values()) {93 if (!L->contains(I))94 continue;95 if (const PHINode *PHI = dyn_cast<PHINode>(V)) {96 if (llvm::none_of(L->getSubLoops(), [PHI](const Loop* SubLoop) {97 return SubLoop->contains(PHI); }))98 return true;99 } else if (Depth < 10 && dependsOnLocalPhi(L, V, Depth+1))100 return true;101 }102 return false;103}104 105AMDGPUTTIImpl::AMDGPUTTIImpl(const AMDGPUTargetMachine *TM, const Function &F)106 : BaseT(TM, F.getDataLayout()),107 TargetTriple(TM->getTargetTriple()),108 ST(static_cast<const GCNSubtarget *>(TM->getSubtargetImpl(F))),109 TLI(ST->getTargetLowering()) {}110 111void AMDGPUTTIImpl::getUnrollingPreferences(112 Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP,113 OptimizationRemarkEmitter *ORE) const {114 const Function &F = *L->getHeader()->getParent();115 UP.Threshold =116 F.getFnAttributeAsParsedInteger("amdgpu-unroll-threshold", 300);117 UP.MaxCount = std::numeric_limits<unsigned>::max();118 UP.Partial = true;119 120 // Conditional branch in a loop back edge needs 3 additional exec121 // manipulations in average.122 UP.BEInsns += 3;123 124 // We want to run unroll even for the loops which have been vectorized.125 UP.UnrollVectorizedLoop = true;126 127 // TODO: Do we want runtime unrolling?128 129 // Maximum alloca size than can fit registers. Reserve 16 registers.130 const unsigned MaxAlloca = (256 - 16) * 4;131 unsigned ThresholdPrivate = UnrollThresholdPrivate;132 unsigned ThresholdLocal = UnrollThresholdLocal;133 134 // If this loop has the amdgpu.loop.unroll.threshold metadata we will use the135 // provided threshold value as the default for Threshold136 if (MDNode *LoopUnrollThreshold =137 findOptionMDForLoop(L, "amdgpu.loop.unroll.threshold")) {138 if (LoopUnrollThreshold->getNumOperands() == 2) {139 ConstantInt *MetaThresholdValue = mdconst::extract_or_null<ConstantInt>(140 LoopUnrollThreshold->getOperand(1));141 if (MetaThresholdValue) {142 // We will also use the supplied value for PartialThreshold for now.143 // We may introduce additional metadata if it becomes necessary in the144 // future.145 UP.Threshold = MetaThresholdValue->getSExtValue();146 UP.PartialThreshold = UP.Threshold;147 ThresholdPrivate = std::min(ThresholdPrivate, UP.Threshold);148 ThresholdLocal = std::min(ThresholdLocal, UP.Threshold);149 }150 }151 }152 153 unsigned MaxBoost = std::max(ThresholdPrivate, ThresholdLocal);154 for (const BasicBlock *BB : L->getBlocks()) {155 const DataLayout &DL = BB->getDataLayout();156 unsigned LocalGEPsSeen = 0;157 158 if (llvm::any_of(L->getSubLoops(), [BB](const Loop* SubLoop) {159 return SubLoop->contains(BB); }))160 continue; // Block belongs to an inner loop.161 162 for (const Instruction &I : *BB) {163 // Unroll a loop which contains an "if" statement whose condition164 // defined by a PHI belonging to the loop. This may help to eliminate165 // if region and potentially even PHI itself, saving on both divergence166 // and registers used for the PHI.167 // Add a small bonus for each of such "if" statements.168 if (const BranchInst *Br = dyn_cast<BranchInst>(&I)) {169 if (UP.Threshold < MaxBoost && Br->isConditional()) {170 BasicBlock *Succ0 = Br->getSuccessor(0);171 BasicBlock *Succ1 = Br->getSuccessor(1);172 if ((L->contains(Succ0) && L->isLoopExiting(Succ0)) ||173 (L->contains(Succ1) && L->isLoopExiting(Succ1)))174 continue;175 if (dependsOnLocalPhi(L, Br->getCondition())) {176 UP.Threshold += UnrollThresholdIf;177 LLVM_DEBUG(dbgs() << "Set unroll threshold " << UP.Threshold178 << " for loop:\n"179 << *L << " due to " << *Br << '\n');180 if (UP.Threshold >= MaxBoost)181 return;182 }183 }184 continue;185 }186 187 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(&I);188 if (!GEP)189 continue;190 191 unsigned AS = GEP->getAddressSpace();192 unsigned Threshold = 0;193 if (AS == AMDGPUAS::PRIVATE_ADDRESS)194 Threshold = ThresholdPrivate;195 else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS)196 Threshold = ThresholdLocal;197 else198 continue;199 200 if (UP.Threshold >= Threshold)201 continue;202 203 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {204 const Value *Ptr = GEP->getPointerOperand();205 const AllocaInst *Alloca =206 dyn_cast<AllocaInst>(getUnderlyingObject(Ptr));207 if (!Alloca || !Alloca->isStaticAlloca())208 continue;209 Type *Ty = Alloca->getAllocatedType();210 unsigned AllocaSize = Ty->isSized() ? DL.getTypeAllocSize(Ty) : 0;211 if (AllocaSize > MaxAlloca)212 continue;213 } else if (AS == AMDGPUAS::LOCAL_ADDRESS ||214 AS == AMDGPUAS::REGION_ADDRESS) {215 LocalGEPsSeen++;216 // Inhibit unroll for local memory if we have seen addressing not to217 // a variable, most likely we will be unable to combine it.218 // Do not unroll too deep inner loops for local memory to give a chance219 // to unroll an outer loop for a more important reason.220 if (LocalGEPsSeen > 1 || L->getLoopDepth() > 2)221 continue;222 223 const Value *V = getUnderlyingObject(GEP->getPointerOperand());224 if (!isa<GlobalVariable>(V) && !isa<Argument>(V))225 continue;226 227 LLVM_DEBUG(dbgs() << "Allow unroll runtime for loop:\n"228 << *L << " due to LDS use.\n");229 UP.Runtime = UnrollRuntimeLocal;230 }231 232 // Check if GEP depends on a value defined by this loop itself.233 bool HasLoopDef = false;234 for (const Value *Op : GEP->operands()) {235 const Instruction *Inst = dyn_cast<Instruction>(Op);236 if (!Inst || L->isLoopInvariant(Op))237 continue;238 239 if (llvm::any_of(L->getSubLoops(), [Inst](const Loop* SubLoop) {240 return SubLoop->contains(Inst); }))241 continue;242 HasLoopDef = true;243 break;244 }245 if (!HasLoopDef)246 continue;247 248 // We want to do whatever we can to limit the number of alloca249 // instructions that make it through to the code generator. allocas250 // require us to use indirect addressing, which is slow and prone to251 // compiler bugs. If this loop does an address calculation on an252 // alloca ptr, then we want to use a higher than normal loop unroll253 // threshold. This will give SROA a better chance to eliminate these254 // allocas.255 //256 // We also want to have more unrolling for local memory to let ds257 // instructions with different offsets combine.258 //259 // Don't use the maximum allowed value here as it will make some260 // programs way too big.261 UP.Threshold = Threshold;262 LLVM_DEBUG(dbgs() << "Set unroll threshold " << Threshold263 << " for loop:\n"264 << *L << " due to " << *GEP << '\n');265 if (UP.Threshold >= MaxBoost)266 return;267 }268 269 // If we got a GEP in a small BB from inner loop then increase max trip270 // count to analyze for better estimation cost in unroll271 if (L->isInnermost() && BB->size() < UnrollMaxBlockToAnalyze)272 UP.MaxIterationsCountToAnalyze = 32;273 }274}275 276void AMDGPUTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,277 TTI::PeelingPreferences &PP) const {278 BaseT::getPeelingPreferences(L, SE, PP);279}280 281uint64_t AMDGPUTTIImpl::getMaxMemIntrinsicInlineSizeThreshold() const {282 return 1024;283}284 285const FeatureBitset GCNTTIImpl::InlineFeatureIgnoreList = {286 // Codegen control options which don't matter.287 AMDGPU::FeatureEnableLoadStoreOpt, AMDGPU::FeatureEnableSIScheduler,288 AMDGPU::FeatureEnableUnsafeDSOffsetFolding, AMDGPU::FeatureFlatForGlobal,289 AMDGPU::FeaturePromoteAlloca, AMDGPU::FeatureUnalignedScratchAccess,290 AMDGPU::FeatureUnalignedAccessMode,291 292 AMDGPU::FeatureAutoWaitcntBeforeBarrier,293 294 // Property of the kernel/environment which can't actually differ.295 AMDGPU::FeatureSGPRInitBug, AMDGPU::FeatureXNACK,296 AMDGPU::FeatureTrapHandler,297 298 // The default assumption needs to be ecc is enabled, but no directly299 // exposed operations depend on it, so it can be safely inlined.300 AMDGPU::FeatureSRAMECC,301 302 // Perf-tuning features303 AMDGPU::FeatureFastFMAF32, AMDGPU::HalfRate64Ops};304 305GCNTTIImpl::GCNTTIImpl(const AMDGPUTargetMachine *TM, const Function &F)306 : BaseT(TM, F.getDataLayout()),307 ST(static_cast<const GCNSubtarget *>(TM->getSubtargetImpl(F))),308 TLI(ST->getTargetLowering()), CommonTTI(TM, F),309 IsGraphics(AMDGPU::isGraphics(F.getCallingConv())) {310 SIModeRegisterDefaults Mode(F, *ST);311 HasFP32Denormals = Mode.FP32Denormals != DenormalMode::getPreserveSign();312 HasFP64FP16Denormals =313 Mode.FP64FP16Denormals != DenormalMode::getPreserveSign();314}315 316bool GCNTTIImpl::hasBranchDivergence(const Function *F) const {317 return !F || !ST->isSingleLaneExecution(*F);318}319 320unsigned GCNTTIImpl::getNumberOfRegisters(unsigned RCID) const {321 // NB: RCID is not an RCID. In fact it is 0 or 1 for scalar or vector322 // registers. See getRegisterClassForType for the implementation.323 // In this case vector registers are not vector in terms of324 // VGPRs, but those which can hold multiple values.325 326 // This is really the number of registers to fill when vectorizing /327 // interleaving loops, so we lie to avoid trying to use all registers.328 return 4;329}330 331TypeSize332GCNTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {333 switch (K) {334 case TargetTransformInfo::RGK_Scalar:335 return TypeSize::getFixed(32);336 case TargetTransformInfo::RGK_FixedWidthVector:337 return TypeSize::getFixed(ST->hasPackedFP32Ops() ? 64 : 32);338 case TargetTransformInfo::RGK_ScalableVector:339 return TypeSize::getScalable(0);340 }341 llvm_unreachable("Unsupported register kind");342}343 344unsigned GCNTTIImpl::getMinVectorRegisterBitWidth() const {345 return 32;346}347 348unsigned GCNTTIImpl::getMaximumVF(unsigned ElemWidth, unsigned Opcode) const {349 if (Opcode == Instruction::Load || Opcode == Instruction::Store)350 return 32 * 4 / ElemWidth;351 // For a given width return the max 0number of elements that can be combined352 // into a wider bit value:353 return (ElemWidth == 8 && ST->has16BitInsts()) ? 4354 : (ElemWidth == 16 && ST->has16BitInsts()) ? 2355 : (ElemWidth == 32 && ST->hasPackedFP32Ops()) ? 2356 : 1;357}358 359unsigned GCNTTIImpl::getLoadVectorFactor(unsigned VF, unsigned LoadSize,360 unsigned ChainSizeInBytes,361 VectorType *VecTy) const {362 unsigned VecRegBitWidth = VF * LoadSize;363 if (VecRegBitWidth > 128 && VecTy->getScalarSizeInBits() < 32)364 // TODO: Support element-size less than 32bit?365 return 128 / LoadSize;366 367 return VF;368}369 370unsigned GCNTTIImpl::getStoreVectorFactor(unsigned VF, unsigned StoreSize,371 unsigned ChainSizeInBytes,372 VectorType *VecTy) const {373 unsigned VecRegBitWidth = VF * StoreSize;374 if (VecRegBitWidth > 128)375 return 128 / StoreSize;376 377 return VF;378}379 380unsigned GCNTTIImpl::getLoadStoreVecRegBitWidth(unsigned AddrSpace) const {381 if (AddrSpace == AMDGPUAS::GLOBAL_ADDRESS ||382 AddrSpace == AMDGPUAS::CONSTANT_ADDRESS ||383 AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||384 AddrSpace == AMDGPUAS::BUFFER_FAT_POINTER ||385 AddrSpace == AMDGPUAS::BUFFER_RESOURCE ||386 AddrSpace == AMDGPUAS::BUFFER_STRIDED_POINTER) {387 return 512;388 }389 390 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS)391 return 8 * ST->getMaxPrivateElementSize();392 393 // Common to flat, global, local and region. Assume for unknown addrspace.394 return 128;395}396 397bool GCNTTIImpl::isLegalToVectorizeMemChain(unsigned ChainSizeInBytes,398 Align Alignment,399 unsigned AddrSpace) const {400 // We allow vectorization of flat stores, even though we may need to decompose401 // them later if they may access private memory. We don't have enough context402 // here, and legalization can handle it.403 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) {404 return (Alignment >= 4 || ST->hasUnalignedScratchAccessEnabled()) &&405 ChainSizeInBytes <= ST->getMaxPrivateElementSize();406 }407 return true;408}409 410bool GCNTTIImpl::isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes,411 Align Alignment,412 unsigned AddrSpace) const {413 return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);414}415 416bool GCNTTIImpl::isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,417 Align Alignment,418 unsigned AddrSpace) const {419 return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);420}421 422uint64_t GCNTTIImpl::getMaxMemIntrinsicInlineSizeThreshold() const {423 return 1024;424}425 426Type *GCNTTIImpl::getMemcpyLoopLoweringType(427 LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,428 unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,429 std::optional<uint32_t> AtomicElementSize) const {430 431 if (AtomicElementSize)432 return Type::getIntNTy(Context, *AtomicElementSize * 8);433 434 // 16-byte accesses achieve the highest copy throughput.435 // If the operation has a fixed known length that is large enough, it is436 // worthwhile to return an even wider type and let legalization lower it into437 // multiple accesses, effectively unrolling the memcpy loop.438 // We also rely on legalization to decompose into smaller accesses for439 // subtargets and address spaces where it is necessary.440 //441 // Don't unroll if Length is not a constant, since unrolling leads to worse442 // performance for length values that are smaller or slightly larger than the443 // total size of the type returned here. Mitigating that would require a more444 // complex lowering for variable-length memcpy and memmove.445 unsigned I32EltsInVector = 4;446 if (MemcpyLoopUnroll > 0 && isa<ConstantInt>(Length))447 return FixedVectorType::get(Type::getInt32Ty(Context),448 MemcpyLoopUnroll * I32EltsInVector);449 450 return FixedVectorType::get(Type::getInt32Ty(Context), I32EltsInVector);451}452 453void GCNTTIImpl::getMemcpyLoopResidualLoweringType(454 SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,455 unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,456 Align SrcAlign, Align DestAlign,457 std::optional<uint32_t> AtomicCpySize) const {458 459 if (AtomicCpySize)460 BaseT::getMemcpyLoopResidualLoweringType(461 OpsOut, Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, SrcAlign,462 DestAlign, AtomicCpySize);463 464 Type *I32x4Ty = FixedVectorType::get(Type::getInt32Ty(Context), 4);465 while (RemainingBytes >= 16) {466 OpsOut.push_back(I32x4Ty);467 RemainingBytes -= 16;468 }469 470 Type *I64Ty = Type::getInt64Ty(Context);471 while (RemainingBytes >= 8) {472 OpsOut.push_back(I64Ty);473 RemainingBytes -= 8;474 }475 476 Type *I32Ty = Type::getInt32Ty(Context);477 while (RemainingBytes >= 4) {478 OpsOut.push_back(I32Ty);479 RemainingBytes -= 4;480 }481 482 Type *I16Ty = Type::getInt16Ty(Context);483 while (RemainingBytes >= 2) {484 OpsOut.push_back(I16Ty);485 RemainingBytes -= 2;486 }487 488 Type *I8Ty = Type::getInt8Ty(Context);489 while (RemainingBytes) {490 OpsOut.push_back(I8Ty);491 --RemainingBytes;492 }493}494 495unsigned GCNTTIImpl::getMaxInterleaveFactor(ElementCount VF) const {496 // Disable unrolling if the loop is not vectorized.497 // TODO: Enable this again.498 if (VF.isScalar())499 return 1;500 501 return 8;502}503 504bool GCNTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,505 MemIntrinsicInfo &Info) const {506 switch (Inst->getIntrinsicID()) {507 case Intrinsic::amdgcn_ds_ordered_add:508 case Intrinsic::amdgcn_ds_ordered_swap: {509 auto *Ordering = dyn_cast<ConstantInt>(Inst->getArgOperand(2));510 auto *Volatile = dyn_cast<ConstantInt>(Inst->getArgOperand(4));511 if (!Ordering || !Volatile)512 return false; // Invalid.513 514 unsigned OrderingVal = Ordering->getZExtValue();515 if (OrderingVal > static_cast<unsigned>(AtomicOrdering::SequentiallyConsistent))516 return false;517 518 Info.PtrVal = Inst->getArgOperand(0);519 Info.Ordering = static_cast<AtomicOrdering>(OrderingVal);520 Info.ReadMem = true;521 Info.WriteMem = true;522 Info.IsVolatile = !Volatile->isZero();523 return true;524 }525 default:526 return false;527 }528}529 530InstructionCost GCNTTIImpl::getArithmeticInstrCost(531 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,532 TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,533 ArrayRef<const Value *> Args, const Instruction *CxtI) const {534 535 // Legalize the type.536 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);537 int ISD = TLI->InstructionOpcodeToISD(Opcode);538 539 // Because we don't have any legal vector operations, but the legal types, we540 // need to account for split vectors.541 unsigned NElts = LT.second.isVector() ?542 LT.second.getVectorNumElements() : 1;543 544 MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;545 546 switch (ISD) {547 case ISD::SHL:548 case ISD::SRL:549 case ISD::SRA:550 if (SLT == MVT::i64)551 return get64BitInstrCost(CostKind) * LT.first * NElts;552 553 if (ST->has16BitInsts() && SLT == MVT::i16)554 NElts = (NElts + 1) / 2;555 556 // i32557 return getFullRateInstrCost() * LT.first * NElts;558 case ISD::ADD:559 case ISD::SUB:560 case ISD::AND:561 case ISD::OR:562 case ISD::XOR:563 if (SLT == MVT::i64) {564 // and, or and xor are typically split into 2 VALU instructions.565 return 2 * getFullRateInstrCost() * LT.first * NElts;566 }567 568 if (ST->has16BitInsts() && SLT == MVT::i16)569 NElts = (NElts + 1) / 2;570 571 return LT.first * NElts * getFullRateInstrCost();572 case ISD::MUL: {573 const int QuarterRateCost = getQuarterRateInstrCost(CostKind);574 if (SLT == MVT::i64) {575 const int FullRateCost = getFullRateInstrCost();576 return (4 * QuarterRateCost + (2 * 2) * FullRateCost) * LT.first * NElts;577 }578 579 if (ST->has16BitInsts() && SLT == MVT::i16)580 NElts = (NElts + 1) / 2;581 582 // i32583 return QuarterRateCost * NElts * LT.first;584 }585 case ISD::FMUL:586 // Check possible fuse {fadd|fsub}(a,fmul(b,c)) and return zero cost for587 // fmul(b,c) supposing the fadd|fsub will get estimated cost for the whole588 // fused operation.589 if (CxtI && CxtI->hasOneUse())590 if (const auto *FAdd = dyn_cast<BinaryOperator>(*CxtI->user_begin())) {591 const int OPC = TLI->InstructionOpcodeToISD(FAdd->getOpcode());592 if (OPC == ISD::FADD || OPC == ISD::FSUB) {593 if (ST->hasMadMacF32Insts() && SLT == MVT::f32 && !HasFP32Denormals)594 return TargetTransformInfo::TCC_Free;595 if (ST->has16BitInsts() && SLT == MVT::f16 && !HasFP64FP16Denormals)596 return TargetTransformInfo::TCC_Free;597 598 // Estimate all types may be fused with contract/unsafe flags599 const TargetOptions &Options = TLI->getTargetMachine().Options;600 if (Options.AllowFPOpFusion == FPOpFusion::Fast ||601 (FAdd->hasAllowContract() && CxtI->hasAllowContract()))602 return TargetTransformInfo::TCC_Free;603 }604 }605 [[fallthrough]];606 case ISD::FADD:607 case ISD::FSUB:608 if (ST->hasPackedFP32Ops() && SLT == MVT::f32)609 NElts = (NElts + 1) / 2;610 if (ST->hasBF16PackedInsts() && SLT == MVT::bf16)611 NElts = (NElts + 1) / 2;612 if (SLT == MVT::f64)613 return LT.first * NElts * get64BitInstrCost(CostKind);614 615 if (ST->has16BitInsts() && SLT == MVT::f16)616 NElts = (NElts + 1) / 2;617 618 if (SLT == MVT::f32 || SLT == MVT::f16 || SLT == MVT::bf16)619 return LT.first * NElts * getFullRateInstrCost();620 break;621 case ISD::FDIV:622 case ISD::FREM:623 // FIXME: frem should be handled separately. The fdiv in it is most of it,624 // but the current lowering is also not entirely correct.625 if (SLT == MVT::f64) {626 int Cost = 7 * get64BitInstrCost(CostKind) +627 getQuarterRateInstrCost(CostKind) +628 3 * getHalfRateInstrCost(CostKind);629 // Add cost of workaround.630 if (!ST->hasUsableDivScaleConditionOutput())631 Cost += 3 * getFullRateInstrCost();632 633 return LT.first * Cost * NElts;634 }635 636 if (!Args.empty() && match(Args[0], PatternMatch::m_FPOne())) {637 // TODO: This is more complicated, unsafe flags etc.638 if ((SLT == MVT::f32 && !HasFP32Denormals) ||639 (SLT == MVT::f16 && ST->has16BitInsts())) {640 return LT.first * getQuarterRateInstrCost(CostKind) * NElts;641 }642 }643 644 if (SLT == MVT::f16 && ST->has16BitInsts()) {645 // 2 x v_cvt_f32_f16646 // f32 rcp647 // f32 fmul648 // v_cvt_f16_f32649 // f16 div_fixup650 int Cost =651 4 * getFullRateInstrCost() + 2 * getQuarterRateInstrCost(CostKind);652 return LT.first * Cost * NElts;653 }654 655 if (SLT == MVT::f32 && (CxtI && CxtI->hasApproxFunc())) {656 // Fast unsafe fdiv lowering:657 // f32 rcp658 // f32 fmul659 int Cost = getQuarterRateInstrCost(CostKind) + getFullRateInstrCost();660 return LT.first * Cost * NElts;661 }662 663 if (SLT == MVT::f32 || SLT == MVT::f16) {664 // 4 more v_cvt_* insts without f16 insts support665 int Cost = (SLT == MVT::f16 ? 14 : 10) * getFullRateInstrCost() +666 1 * getQuarterRateInstrCost(CostKind);667 668 if (!HasFP32Denormals) {669 // FP mode switches.670 Cost += 2 * getFullRateInstrCost();671 }672 673 return LT.first * NElts * Cost;674 }675 break;676 case ISD::FNEG:677 // Use the backend' estimation. If fneg is not free each element will cost678 // one additional instruction.679 return TLI->isFNegFree(SLT) ? 0 : NElts;680 default:681 break;682 }683 684 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,685 Args, CxtI);686}687 688// Return true if there's a potential benefit from using v2f16/v2i16689// instructions for an intrinsic, even if it requires nontrivial legalization.690static bool intrinsicHasPackedVectorBenefit(Intrinsic::ID ID) {691 switch (ID) {692 case Intrinsic::fma:693 case Intrinsic::fmuladd:694 case Intrinsic::copysign:695 case Intrinsic::minimumnum:696 case Intrinsic::maximumnum:697 case Intrinsic::canonicalize:698 // There's a small benefit to using vector ops in the legalized code.699 case Intrinsic::round:700 case Intrinsic::uadd_sat:701 case Intrinsic::usub_sat:702 case Intrinsic::sadd_sat:703 case Intrinsic::ssub_sat:704 case Intrinsic::abs:705 return true;706 default:707 return false;708 }709}710 711InstructionCost712GCNTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,713 TTI::TargetCostKind CostKind) const {714 switch (ICA.getID()) {715 case Intrinsic::fabs:716 // Free source modifier in the common case.717 return 0;718 case Intrinsic::amdgcn_workitem_id_x:719 case Intrinsic::amdgcn_workitem_id_y:720 case Intrinsic::amdgcn_workitem_id_z:721 // TODO: If hasPackedTID, or if the calling context is not an entry point722 // there may be a bit instruction.723 return 0;724 case Intrinsic::amdgcn_workgroup_id_x:725 case Intrinsic::amdgcn_workgroup_id_y:726 case Intrinsic::amdgcn_workgroup_id_z:727 case Intrinsic::amdgcn_lds_kernel_id:728 case Intrinsic::amdgcn_dispatch_ptr:729 case Intrinsic::amdgcn_dispatch_id:730 case Intrinsic::amdgcn_implicitarg_ptr:731 case Intrinsic::amdgcn_queue_ptr:732 // Read from an argument register.733 return 0;734 default:735 break;736 }737 738 if (!intrinsicHasPackedVectorBenefit(ICA.getID()))739 return BaseT::getIntrinsicInstrCost(ICA, CostKind);740 741 Type *RetTy = ICA.getReturnType();742 743 // Legalize the type.744 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(RetTy);745 746 unsigned NElts = LT.second.isVector() ?747 LT.second.getVectorNumElements() : 1;748 749 MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;750 751 if ((ST->hasVOP3PInsts() &&752 (SLT == MVT::f16 || SLT == MVT::i16 ||753 (SLT == MVT::bf16 && ST->hasBF16PackedInsts()))) ||754 (ST->hasPackedFP32Ops() && SLT == MVT::f32))755 NElts = (NElts + 1) / 2;756 757 // TODO: Get more refined intrinsic costs?758 unsigned InstRate = getQuarterRateInstrCost(CostKind);759 760 switch (ICA.getID()) {761 case Intrinsic::fma:762 case Intrinsic::fmuladd:763 if (SLT == MVT::f64) {764 InstRate = get64BitInstrCost(CostKind);765 break;766 }767 768 if ((SLT == MVT::f32 && ST->hasFastFMAF32()) || SLT == MVT::f16)769 InstRate = getFullRateInstrCost();770 else {771 InstRate = ST->hasFastFMAF32() ? getHalfRateInstrCost(CostKind)772 : getQuarterRateInstrCost(CostKind);773 }774 break;775 case Intrinsic::copysign:776 return NElts * getFullRateInstrCost();777 case Intrinsic::minimumnum:778 case Intrinsic::maximumnum: {779 // Instruction + 2 canonicalizes. For cases that need type promotion, we the780 // promotion takes the place of the canonicalize.781 unsigned NumOps = 3;782 if (const IntrinsicInst *II = ICA.getInst()) {783 // Directly legal with ieee=0784 // TODO: Not directly legal with strictfp785 if (fpenvIEEEMode(*II) == KnownIEEEMode::Off)786 NumOps = 1;787 }788 789 unsigned BaseRate =790 SLT == MVT::f64 ? get64BitInstrCost(CostKind) : getFullRateInstrCost();791 InstRate = BaseRate * NumOps;792 break;793 }794 case Intrinsic::canonicalize: {795 InstRate =796 SLT == MVT::f64 ? get64BitInstrCost(CostKind) : getFullRateInstrCost();797 break;798 }799 case Intrinsic::uadd_sat:800 case Intrinsic::usub_sat:801 case Intrinsic::sadd_sat:802 case Intrinsic::ssub_sat: {803 if (SLT == MVT::i16 || SLT == MVT::i32)804 InstRate = getFullRateInstrCost();805 806 static const auto ValidSatTys = {MVT::v2i16, MVT::v4i16};807 if (any_of(ValidSatTys, [<](MVT M) { return M == LT.second; }))808 NElts = 1;809 break;810 }811 case Intrinsic::abs:812 // Expansion takes 2 instructions for VALU813 if (SLT == MVT::i16 || SLT == MVT::i32)814 InstRate = 2 * getFullRateInstrCost();815 break;816 default:817 break;818 }819 820 return LT.first * NElts * InstRate;821}822 823InstructionCost GCNTTIImpl::getCFInstrCost(unsigned Opcode,824 TTI::TargetCostKind CostKind,825 const Instruction *I) const {826 assert((I == nullptr || I->getOpcode() == Opcode) &&827 "Opcode should reflect passed instruction.");828 const bool SCost =829 (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency);830 const int CBrCost = SCost ? 5 : 7;831 switch (Opcode) {832 case Instruction::Br: {833 // Branch instruction takes about 4 slots on gfx900.834 const auto *BI = dyn_cast_or_null<BranchInst>(I);835 if (BI && BI->isUnconditional())836 return SCost ? 1 : 4;837 // Suppose conditional branch takes additional 3 exec manipulations838 // instructions in average.839 return CBrCost;840 }841 case Instruction::Switch: {842 const auto *SI = dyn_cast_or_null<SwitchInst>(I);843 // Each case (including default) takes 1 cmp + 1 cbr instructions in844 // average.845 return (SI ? (SI->getNumCases() + 1) : 4) * (CBrCost + 1);846 }847 case Instruction::Ret:848 return SCost ? 1 : 10;849 }850 return BaseT::getCFInstrCost(Opcode, CostKind, I);851}852 853InstructionCost854GCNTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,855 std::optional<FastMathFlags> FMF,856 TTI::TargetCostKind CostKind) const {857 if (TTI::requiresOrderedReduction(FMF))858 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);859 860 EVT OrigTy = TLI->getValueType(DL, Ty);861 862 // Computes cost on targets that have packed math instructions(which support863 // 16-bit types only).864 if (!ST->hasVOP3PInsts() || OrigTy.getScalarSizeInBits() != 16)865 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);866 867 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);868 return LT.first * getFullRateInstrCost();869}870 871InstructionCost872GCNTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,873 FastMathFlags FMF,874 TTI::TargetCostKind CostKind) const {875 EVT OrigTy = TLI->getValueType(DL, Ty);876 877 // Computes cost on targets that have packed math instructions(which support878 // 16-bit types only).879 if (!ST->hasVOP3PInsts() || OrigTy.getScalarSizeInBits() != 16)880 return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);881 882 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);883 return LT.first * getHalfRateInstrCost(CostKind);884}885 886InstructionCost GCNTTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy,887 TTI::TargetCostKind CostKind,888 unsigned Index, const Value *Op0,889 const Value *Op1) const {890 switch (Opcode) {891 case Instruction::ExtractElement:892 case Instruction::InsertElement: {893 unsigned EltSize894 = DL.getTypeSizeInBits(cast<VectorType>(ValTy)->getElementType());895 if (EltSize < 32) {896 if (EltSize == 16 && Index == 0 && ST->has16BitInsts())897 return 0;898 return BaseT::getVectorInstrCost(Opcode, ValTy, CostKind, Index, Op0,899 Op1);900 }901 902 // Extracts are just reads of a subregister, so are free. Inserts are903 // considered free because we don't want to have any cost for scalarizing904 // operations, and we don't have to copy into a different register class.905 906 // Dynamic indexing isn't free and is best avoided.907 return Index == ~0u ? 2 : 0;908 }909 default:910 return BaseT::getVectorInstrCost(Opcode, ValTy, CostKind, Index, Op0, Op1);911 }912}913 914/// Analyze if the results of inline asm are divergent. If \p Indices is empty,915/// this is analyzing the collective result of all output registers. Otherwise,916/// this is only querying a specific result index if this returns multiple917/// registers in a struct.918bool GCNTTIImpl::isInlineAsmSourceOfDivergence(919 const CallInst *CI, ArrayRef<unsigned> Indices) const {920 // TODO: Handle complex extract indices921 if (Indices.size() > 1)922 return true;923 924 const DataLayout &DL = CI->getDataLayout();925 const SIRegisterInfo *TRI = ST->getRegisterInfo();926 TargetLowering::AsmOperandInfoVector TargetConstraints =927 TLI->ParseConstraints(DL, ST->getRegisterInfo(), *CI);928 929 const int TargetOutputIdx = Indices.empty() ? -1 : Indices[0];930 931 int OutputIdx = 0;932 for (auto &TC : TargetConstraints) {933 if (TC.Type != InlineAsm::isOutput)934 continue;935 936 // Skip outputs we don't care about.937 if (TargetOutputIdx != -1 && TargetOutputIdx != OutputIdx++)938 continue;939 940 TLI->ComputeConstraintToUse(TC, SDValue());941 942 const TargetRegisterClass *RC = TLI->getRegForInlineAsmConstraint(943 TRI, TC.ConstraintCode, TC.ConstraintVT).second;944 945 // For AGPR constraints null is returned on subtargets without AGPRs, so946 // assume divergent for null.947 if (!RC || !TRI->isSGPRClass(RC))948 return true;949 }950 951 return false;952}953 954bool GCNTTIImpl::isReadRegisterSourceOfDivergence(955 const IntrinsicInst *ReadReg) const {956 Metadata *MD =957 cast<MetadataAsValue>(ReadReg->getArgOperand(0))->getMetadata();958 StringRef RegName =959 cast<MDString>(cast<MDNode>(MD)->getOperand(0))->getString();960 961 // Special case registers that look like VCC.962 MVT VT = MVT::getVT(ReadReg->getType());963 if (VT == MVT::i1)964 return true;965 966 // Special case scalar registers that start with 'v'.967 if (RegName.starts_with("vcc") || RegName.empty())968 return false;969 970 // VGPR or AGPR is divergent. There aren't any specially named vector971 // registers.972 return RegName[0] == 'v' || RegName[0] == 'a';973}974 975/// \returns true if the result of the value could potentially be976/// different across workitems in a wavefront.977bool GCNTTIImpl::isSourceOfDivergence(const Value *V) const {978 if (const Argument *A = dyn_cast<Argument>(V))979 return !AMDGPU::isArgPassedInSGPR(A);980 981 // Loads from the private and flat address spaces are divergent, because982 // threads can execute the load instruction with the same inputs and get983 // different results.984 //985 // All other loads are not divergent, because if threads issue loads with the986 // same arguments, they will always get the same result.987 if (const LoadInst *Load = dyn_cast<LoadInst>(V))988 return Load->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS ||989 Load->getPointerAddressSpace() == AMDGPUAS::FLAT_ADDRESS;990 991 // Atomics are divergent because they are executed sequentially: when an992 // atomic operation refers to the same address in each thread, then each993 // thread after the first sees the value written by the previous thread as994 // original value.995 if (isa<AtomicRMWInst, AtomicCmpXchgInst>(V))996 return true;997 998 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(V)) {999 Intrinsic::ID IID = Intrinsic->getIntrinsicID();1000 switch (IID) {1001 case Intrinsic::read_register:1002 return isReadRegisterSourceOfDivergence(Intrinsic);1003 case Intrinsic::amdgcn_addrspacecast_nonnull: {1004 unsigned SrcAS =1005 Intrinsic->getOperand(0)->getType()->getPointerAddressSpace();1006 unsigned DstAS = Intrinsic->getType()->getPointerAddressSpace();1007 return SrcAS == AMDGPUAS::PRIVATE_ADDRESS &&1008 DstAS == AMDGPUAS::FLAT_ADDRESS &&1009 ST->hasGloballyAddressableScratch();1010 }1011 case Intrinsic::amdgcn_workitem_id_y:1012 case Intrinsic::amdgcn_workitem_id_z: {1013 const Function *F = Intrinsic->getFunction();1014 bool HasUniformYZ =1015 ST->hasWavefrontsEvenlySplittingXDim(*F, /*RequitezUniformYZ=*/true);1016 std::optional<unsigned> ThisDimSize = ST->getReqdWorkGroupSize(1017 *F, IID == Intrinsic::amdgcn_workitem_id_y ? 1 : 2);1018 return !HasUniformYZ && (!ThisDimSize || *ThisDimSize != 1);1019 }1020 default:1021 return AMDGPU::isIntrinsicSourceOfDivergence(IID);1022 }1023 }1024 1025 // Assume all function calls are a source of divergence.1026 if (const CallInst *CI = dyn_cast<CallInst>(V)) {1027 if (CI->isInlineAsm())1028 return isInlineAsmSourceOfDivergence(CI);1029 return true;1030 }1031 1032 // Assume all function calls are a source of divergence.1033 if (isa<InvokeInst>(V))1034 return true;1035 1036 // If the target supports globally addressable scratch, the mapping from1037 // scratch memory to the flat aperture changes therefore an address space cast1038 // is no longer uniform.1039 if (auto *CastI = dyn_cast<AddrSpaceCastInst>(V)) {1040 return CastI->getSrcAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS &&1041 CastI->getDestAddressSpace() == AMDGPUAS::FLAT_ADDRESS &&1042 ST->hasGloballyAddressableScratch();1043 }1044 1045 return false;1046}1047 1048bool GCNTTIImpl::isAlwaysUniform(const Value *V) const {1049 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(V))1050 return AMDGPU::isIntrinsicAlwaysUniform(Intrinsic->getIntrinsicID());1051 1052 if (const CallInst *CI = dyn_cast<CallInst>(V)) {1053 if (CI->isInlineAsm())1054 return !isInlineAsmSourceOfDivergence(CI);1055 return false;1056 }1057 1058 // In most cases TID / wavefrontsize is uniform.1059 //1060 // However, if a kernel has uneven dimesions we can have a value of1061 // workitem-id-x divided by the wavefrontsize non-uniform. For example1062 // dimensions (65, 2) will have workitems with address (64, 0) and (0, 1)1063 // packed into a same wave which gives 1 and 0 after the division by 641064 // respectively.1065 //1066 // The X dimension doesn't reset within a wave if either both the Y1067 // and Z dimensions are of length 1, or if the X dimension's required1068 // size is a power of 2. Note, however, if the X dimension's maximum1069 // size is a power of 2 < the wavefront size, division by the wavefront1070 // size is guaranteed to yield 0, so this is also a no-reset case.1071 bool XDimDoesntResetWithinWaves = false;1072 if (auto *I = dyn_cast<Instruction>(V)) {1073 const Function *F = I->getFunction();1074 XDimDoesntResetWithinWaves = ST->hasWavefrontsEvenlySplittingXDim(*F);1075 }1076 using namespace llvm::PatternMatch;1077 uint64_t C;1078 if (match(V, m_LShr(m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>(),1079 m_ConstantInt(C))) ||1080 match(V, m_AShr(m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>(),1081 m_ConstantInt(C)))) {1082 return C >= ST->getWavefrontSizeLog2() && XDimDoesntResetWithinWaves;1083 }1084 1085 Value *Mask;1086 if (match(V, m_c_And(m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>(),1087 m_Value(Mask)))) {1088 return computeKnownBits(Mask, DL).countMinTrailingZeros() >=1089 ST->getWavefrontSizeLog2() &&1090 XDimDoesntResetWithinWaves;1091 }1092 1093 const ExtractValueInst *ExtValue = dyn_cast<ExtractValueInst>(V);1094 if (!ExtValue)1095 return false;1096 1097 const CallInst *CI = dyn_cast<CallInst>(ExtValue->getOperand(0));1098 if (!CI)1099 return false;1100 1101 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(CI)) {1102 switch (Intrinsic->getIntrinsicID()) {1103 default:1104 return false;1105 case Intrinsic::amdgcn_if:1106 case Intrinsic::amdgcn_else: {1107 ArrayRef<unsigned> Indices = ExtValue->getIndices();1108 return Indices.size() == 1 && Indices[0] == 1;1109 }1110 }1111 }1112 1113 // If we have inline asm returning mixed SGPR and VGPR results, we inferred1114 // divergent for the overall struct return. We need to override it in the1115 // case we're extracting an SGPR component here.1116 if (CI->isInlineAsm())1117 return !isInlineAsmSourceOfDivergence(CI, ExtValue->getIndices());1118 1119 return false;1120}1121 1122bool GCNTTIImpl::collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes,1123 Intrinsic::ID IID) const {1124 switch (IID) {1125 case Intrinsic::amdgcn_is_shared:1126 case Intrinsic::amdgcn_is_private:1127 case Intrinsic::amdgcn_flat_atomic_fmax_num:1128 case Intrinsic::amdgcn_flat_atomic_fmin_num:1129 case Intrinsic::amdgcn_load_to_lds:1130 case Intrinsic::amdgcn_make_buffer_rsrc:1131 OpIndexes.push_back(0);1132 return true;1133 default:1134 return false;1135 }1136}1137 1138Value *GCNTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II,1139 Value *OldV,1140 Value *NewV) const {1141 auto IntrID = II->getIntrinsicID();1142 switch (IntrID) {1143 case Intrinsic::amdgcn_is_shared:1144 case Intrinsic::amdgcn_is_private: {1145 unsigned TrueAS = IntrID == Intrinsic::amdgcn_is_shared ?1146 AMDGPUAS::LOCAL_ADDRESS : AMDGPUAS::PRIVATE_ADDRESS;1147 unsigned NewAS = NewV->getType()->getPointerAddressSpace();1148 LLVMContext &Ctx = NewV->getType()->getContext();1149 ConstantInt *NewVal = (TrueAS == NewAS) ?1150 ConstantInt::getTrue(Ctx) : ConstantInt::getFalse(Ctx);1151 return NewVal;1152 }1153 case Intrinsic::ptrmask: {1154 unsigned OldAS = OldV->getType()->getPointerAddressSpace();1155 unsigned NewAS = NewV->getType()->getPointerAddressSpace();1156 Value *MaskOp = II->getArgOperand(1);1157 Type *MaskTy = MaskOp->getType();1158 1159 bool DoTruncate = false;1160 1161 const GCNTargetMachine &TM =1162 static_cast<const GCNTargetMachine &>(getTLI()->getTargetMachine());1163 if (!TM.isNoopAddrSpaceCast(OldAS, NewAS)) {1164 // All valid 64-bit to 32-bit casts work by chopping off the high1165 // bits. Any masking only clearing the low bits will also apply in the new1166 // address space.1167 if (DL.getPointerSizeInBits(OldAS) != 64 ||1168 DL.getPointerSizeInBits(NewAS) != 32)1169 return nullptr;1170 1171 // TODO: Do we need to thread more context in here?1172 KnownBits Known = computeKnownBits(MaskOp, DL, nullptr, II);1173 if (Known.countMinLeadingOnes() < 32)1174 return nullptr;1175 1176 DoTruncate = true;1177 }1178 1179 IRBuilder<> B(II);1180 if (DoTruncate) {1181 MaskTy = B.getInt32Ty();1182 MaskOp = B.CreateTrunc(MaskOp, MaskTy);1183 }1184 1185 return B.CreateIntrinsic(Intrinsic::ptrmask, {NewV->getType(), MaskTy},1186 {NewV, MaskOp});1187 }1188 case Intrinsic::amdgcn_flat_atomic_fmax_num:1189 case Intrinsic::amdgcn_flat_atomic_fmin_num: {1190 Type *DestTy = II->getType();1191 Type *SrcTy = NewV->getType();1192 unsigned NewAS = SrcTy->getPointerAddressSpace();1193 if (!AMDGPU::isExtendedGlobalAddrSpace(NewAS))1194 return nullptr;1195 Module *M = II->getModule();1196 Function *NewDecl = Intrinsic::getOrInsertDeclaration(1197 M, II->getIntrinsicID(), {DestTy, SrcTy, DestTy});1198 II->setArgOperand(0, NewV);1199 II->setCalledFunction(NewDecl);1200 return II;1201 }1202 case Intrinsic::amdgcn_load_to_lds: {1203 Type *SrcTy = NewV->getType();1204 Module *M = II->getModule();1205 Function *NewDecl =1206 Intrinsic::getOrInsertDeclaration(M, II->getIntrinsicID(), {SrcTy});1207 II->setArgOperand(0, NewV);1208 II->setCalledFunction(NewDecl);1209 return II;1210 }1211 case Intrinsic::amdgcn_make_buffer_rsrc: {1212 Type *SrcTy = NewV->getType();1213 Type *DstTy = II->getType();1214 Module *M = II->getModule();1215 Function *NewDecl = Intrinsic::getOrInsertDeclaration(1216 M, II->getIntrinsicID(), {DstTy, SrcTy});1217 II->setArgOperand(0, NewV);1218 II->setCalledFunction(NewDecl);1219 return II;1220 }1221 default:1222 return nullptr;1223 }1224}1225 1226InstructionCost GCNTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,1227 VectorType *DstTy, VectorType *SrcTy,1228 ArrayRef<int> Mask,1229 TTI::TargetCostKind CostKind,1230 int Index, VectorType *SubTp,1231 ArrayRef<const Value *> Args,1232 const Instruction *CxtI) const {1233 if (!isa<FixedVectorType>(SrcTy))1234 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, Mask, CostKind, Index,1235 SubTp);1236 1237 Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTp);1238 1239 unsigned ScalarSize = DL.getTypeSizeInBits(SrcTy->getElementType());1240 if (ST->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&1241 (ScalarSize == 16 || ScalarSize == 8)) {1242 // Larger vector widths may require additional instructions, but are1243 // typically cheaper than scalarized versions.1244 //1245 // We assume that shuffling at a register granularity can be done for free.1246 // This is not true for vectors fed into memory instructions, but it is1247 // effectively true for all other shuffling. The emphasis of the logic here1248 // is to assist generic transform in cleaning up / canonicalizing those1249 // shuffles.1250 1251 // With op_sel VOP3P instructions freely can access the low half or high1252 // half of a register, so any swizzle of two elements is free.1253 if (auto *SrcVecTy = dyn_cast<FixedVectorType>(SrcTy)) {1254 unsigned NumSrcElts = SrcVecTy->getNumElements();1255 if (ST->hasVOP3PInsts() && ScalarSize == 16 && NumSrcElts == 2 &&1256 (Kind == TTI::SK_Broadcast || Kind == TTI::SK_Reverse ||1257 Kind == TTI::SK_PermuteSingleSrc))1258 return 0;1259 }1260 1261 unsigned EltsPerReg = 32 / ScalarSize;1262 switch (Kind) {1263 case TTI::SK_Broadcast:1264 // A single v_perm_b32 can be re-used for all destination registers.1265 return 1;1266 case TTI::SK_Reverse:1267 // One instruction per register.1268 if (auto *DstVecTy = dyn_cast<FixedVectorType>(DstTy))1269 return divideCeil(DstVecTy->getNumElements(), EltsPerReg);1270 return InstructionCost::getInvalid();1271 case TTI::SK_ExtractSubvector:1272 if (Index % EltsPerReg == 0)1273 return 0; // Shuffling at register granularity1274 if (auto *DstVecTy = dyn_cast<FixedVectorType>(DstTy))1275 return divideCeil(DstVecTy->getNumElements(), EltsPerReg);1276 return InstructionCost::getInvalid();1277 case TTI::SK_InsertSubvector: {1278 auto *DstVecTy = dyn_cast<FixedVectorType>(DstTy);1279 if (!DstVecTy)1280 return InstructionCost::getInvalid();1281 unsigned NumDstElts = DstVecTy->getNumElements();1282 unsigned NumInsertElts = cast<FixedVectorType>(SubTp)->getNumElements();1283 unsigned EndIndex = Index + NumInsertElts;1284 unsigned BeginSubIdx = Index % EltsPerReg;1285 unsigned EndSubIdx = EndIndex % EltsPerReg;1286 unsigned Cost = 0;1287 1288 if (BeginSubIdx != 0) {1289 // Need to shift the inserted vector into place. The cost is the number1290 // of destination registers overlapped by the inserted vector.1291 Cost = divideCeil(EndIndex, EltsPerReg) - (Index / EltsPerReg);1292 }1293 1294 // If the last register overlap is partial, there may be three source1295 // registers feeding into it; that takes an extra instruction.1296 if (EndIndex < NumDstElts && BeginSubIdx < EndSubIdx)1297 Cost += 1;1298 1299 return Cost;1300 }1301 case TTI::SK_Splice: {1302 auto *DstVecTy = dyn_cast<FixedVectorType>(DstTy);1303 if (!DstVecTy)1304 return InstructionCost::getInvalid();1305 unsigned NumElts = DstVecTy->getNumElements();1306 assert(NumElts == cast<FixedVectorType>(SrcTy)->getNumElements());1307 // Determine the sub-region of the result vector that requires1308 // sub-register shuffles / mixing.1309 unsigned EltsFromLHS = NumElts - Index;1310 bool LHSIsAligned = (Index % EltsPerReg) == 0;1311 bool RHSIsAligned = (EltsFromLHS % EltsPerReg) == 0;1312 if (LHSIsAligned && RHSIsAligned)1313 return 0;1314 if (LHSIsAligned && !RHSIsAligned)1315 return divideCeil(NumElts, EltsPerReg) - (EltsFromLHS / EltsPerReg);1316 if (!LHSIsAligned && RHSIsAligned)1317 return divideCeil(EltsFromLHS, EltsPerReg);1318 return divideCeil(NumElts, EltsPerReg);1319 }1320 default:1321 break;1322 }1323 1324 if (!Mask.empty()) {1325 unsigned NumSrcElts = cast<FixedVectorType>(SrcTy)->getNumElements();1326 1327 // Generically estimate the cost by assuming that each destination1328 // register is derived from sources via v_perm_b32 instructions if it1329 // can't be copied as-is.1330 //1331 // For each destination register, derive the cost of obtaining it based1332 // on the number of source registers that feed into it.1333 unsigned Cost = 0;1334 for (unsigned DstIdx = 0; DstIdx < Mask.size(); DstIdx += EltsPerReg) {1335 SmallVector<int, 4> Regs;1336 bool Aligned = true;1337 for (unsigned I = 0; I < EltsPerReg && DstIdx + I < Mask.size(); ++I) {1338 int SrcIdx = Mask[DstIdx + I];1339 if (SrcIdx == -1)1340 continue;1341 int Reg;1342 if (SrcIdx < (int)NumSrcElts) {1343 Reg = SrcIdx / EltsPerReg;1344 if (SrcIdx % EltsPerReg != I)1345 Aligned = false;1346 } else {1347 Reg = NumSrcElts + (SrcIdx - NumSrcElts) / EltsPerReg;1348 if ((SrcIdx - NumSrcElts) % EltsPerReg != I)1349 Aligned = false;1350 }1351 if (!llvm::is_contained(Regs, Reg))1352 Regs.push_back(Reg);1353 }1354 if (Regs.size() >= 2)1355 Cost += Regs.size() - 1;1356 else if (!Aligned)1357 Cost += 1;1358 }1359 return Cost;1360 }1361 }1362 1363 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, Mask, CostKind, Index,1364 SubTp);1365}1366 1367/// Whether it is profitable to sink the operands of an1368/// Instruction I to the basic block of I.1369/// This helps using several modifiers (like abs and neg) more often.1370bool GCNTTIImpl::isProfitableToSinkOperands(Instruction *I,1371 SmallVectorImpl<Use *> &Ops) const {1372 using namespace PatternMatch;1373 1374 for (auto &Op : I->operands()) {1375 // Ensure we are not already sinking this operand.1376 if (any_of(Ops, [&](Use *U) { return U->get() == Op.get(); }))1377 continue;1378 1379 if (match(&Op, m_FAbs(m_Value())) || match(&Op, m_FNeg(m_Value())))1380 Ops.push_back(&Op);1381 }1382 1383 return !Ops.empty();1384}1385 1386bool GCNTTIImpl::areInlineCompatible(const Function *Caller,1387 const Function *Callee) const {1388 const TargetMachine &TM = getTLI()->getTargetMachine();1389 const GCNSubtarget *CallerST1390 = static_cast<const GCNSubtarget *>(TM.getSubtargetImpl(*Caller));1391 const GCNSubtarget *CalleeST1392 = static_cast<const GCNSubtarget *>(TM.getSubtargetImpl(*Callee));1393 1394 const FeatureBitset &CallerBits = CallerST->getFeatureBits();1395 const FeatureBitset &CalleeBits = CalleeST->getFeatureBits();1396 1397 FeatureBitset RealCallerBits = CallerBits & ~InlineFeatureIgnoreList;1398 FeatureBitset RealCalleeBits = CalleeBits & ~InlineFeatureIgnoreList;1399 if ((RealCallerBits & RealCalleeBits) != RealCalleeBits)1400 return false;1401 1402 // FIXME: dx10_clamp can just take the caller setting, but there seems to be1403 // no way to support merge for backend defined attributes.1404 SIModeRegisterDefaults CallerMode(*Caller, *CallerST);1405 SIModeRegisterDefaults CalleeMode(*Callee, *CalleeST);1406 if (!CallerMode.isInlineCompatible(CalleeMode))1407 return false;1408 1409 if (Callee->hasFnAttribute(Attribute::AlwaysInline) ||1410 Callee->hasFnAttribute(Attribute::InlineHint))1411 return true;1412 1413 // Hack to make compile times reasonable.1414 if (InlineMaxBB) {1415 // Single BB does not increase total BB amount.1416 if (Callee->size() == 1)1417 return true;1418 size_t BBSize = Caller->size() + Callee->size() - 1;1419 return BBSize <= InlineMaxBB;1420 }1421 1422 return true;1423}1424 1425static unsigned adjustInliningThresholdUsingCallee(const CallBase *CB,1426 const SITargetLowering *TLI,1427 const GCNTTIImpl *TTIImpl) {1428 const int NrOfSGPRUntilSpill = 26;1429 const int NrOfVGPRUntilSpill = 32;1430 1431 const DataLayout &DL = TTIImpl->getDataLayout();1432 1433 unsigned adjustThreshold = 0;1434 int SGPRsInUse = 0;1435 int VGPRsInUse = 0;1436 for (const Use &A : CB->args()) {1437 SmallVector<EVT, 4> ValueVTs;1438 ComputeValueVTs(*TLI, DL, A.get()->getType(), ValueVTs);1439 for (auto ArgVT : ValueVTs) {1440 unsigned CCRegNum = TLI->getNumRegistersForCallingConv(1441 CB->getContext(), CB->getCallingConv(), ArgVT);1442 if (AMDGPU::isArgPassedInSGPR(CB, CB->getArgOperandNo(&A)))1443 SGPRsInUse += CCRegNum;1444 else1445 VGPRsInUse += CCRegNum;1446 }1447 }1448 1449 // The cost of passing function arguments through the stack:1450 // 1 instruction to put a function argument on the stack in the caller.1451 // 1 instruction to take a function argument from the stack in callee.1452 // 1 instruction is explicitly take care of data dependencies in callee1453 // function.1454 InstructionCost ArgStackCost(1);1455 ArgStackCost += const_cast<GCNTTIImpl *>(TTIImpl)->getMemoryOpCost(1456 Instruction::Store, Type::getInt32Ty(CB->getContext()), Align(4),1457 AMDGPUAS::PRIVATE_ADDRESS, TTI::TCK_SizeAndLatency);1458 ArgStackCost += const_cast<GCNTTIImpl *>(TTIImpl)->getMemoryOpCost(1459 Instruction::Load, Type::getInt32Ty(CB->getContext()), Align(4),1460 AMDGPUAS::PRIVATE_ADDRESS, TTI::TCK_SizeAndLatency);1461 1462 // The penalty cost is computed relative to the cost of instructions and does1463 // not model any storage costs.1464 adjustThreshold += std::max(0, SGPRsInUse - NrOfSGPRUntilSpill) *1465 ArgStackCost.getValue() * InlineConstants::getInstrCost();1466 adjustThreshold += std::max(0, VGPRsInUse - NrOfVGPRUntilSpill) *1467 ArgStackCost.getValue() * InlineConstants::getInstrCost();1468 return adjustThreshold;1469}1470 1471static unsigned getCallArgsTotalAllocaSize(const CallBase *CB,1472 const DataLayout &DL) {1473 // If we have a pointer to a private array passed into a function1474 // it will not be optimized out, leaving scratch usage.1475 // This function calculates the total size in bytes of the memory that would1476 // end in scratch if the call was not inlined.1477 unsigned AllocaSize = 0;1478 SmallPtrSet<const AllocaInst *, 8> AIVisited;1479 for (Value *PtrArg : CB->args()) {1480 PointerType *Ty = dyn_cast<PointerType>(PtrArg->getType());1481 if (!Ty)1482 continue;1483 1484 unsigned AddrSpace = Ty->getAddressSpace();1485 if (AddrSpace != AMDGPUAS::FLAT_ADDRESS &&1486 AddrSpace != AMDGPUAS::PRIVATE_ADDRESS)1487 continue;1488 1489 const AllocaInst *AI = dyn_cast<AllocaInst>(getUnderlyingObject(PtrArg));1490 if (!AI || !AI->isStaticAlloca() || !AIVisited.insert(AI).second)1491 continue;1492 1493 AllocaSize += DL.getTypeAllocSize(AI->getAllocatedType());1494 }1495 return AllocaSize;1496}1497 1498int GCNTTIImpl::getInliningLastCallToStaticBonus() const {1499 return BaseT::getInliningLastCallToStaticBonus() *1500 getInliningThresholdMultiplier();1501}1502 1503unsigned GCNTTIImpl::adjustInliningThreshold(const CallBase *CB) const {1504 unsigned Threshold = adjustInliningThresholdUsingCallee(CB, TLI, this);1505 1506 // Private object passed as arguments may end up in scratch usage if the call1507 // is not inlined. Increase the inline threshold to promote inlining.1508 unsigned AllocaSize = getCallArgsTotalAllocaSize(CB, DL);1509 if (AllocaSize > 0)1510 Threshold += ArgAllocaCost;1511 return Threshold;1512}1513 1514unsigned GCNTTIImpl::getCallerAllocaCost(const CallBase *CB,1515 const AllocaInst *AI) const {1516 1517 // Below the cutoff, assume that the private memory objects would be1518 // optimized1519 auto AllocaSize = getCallArgsTotalAllocaSize(CB, DL);1520 if (AllocaSize <= ArgAllocaCutoff)1521 return 0;1522 1523 // Above the cutoff, we give a cost to each private memory object1524 // depending its size. If the array can be optimized by SROA this cost is not1525 // added to the total-cost in the inliner cost analysis.1526 //1527 // We choose the total cost of the alloca such that their sum cancels the1528 // bonus given in the threshold (ArgAllocaCost).1529 //1530 // Cost_Alloca_0 + ... + Cost_Alloca_N == ArgAllocaCost1531 //1532 // Awkwardly, the ArgAllocaCost bonus is multiplied by threshold-multiplier,1533 // the single-bb bonus and the vector-bonus.1534 //1535 // We compensate the first two multipliers, by repeating logic from the1536 // inliner-cost in here. The vector-bonus is 0 on AMDGPU.1537 static_assert(InlinerVectorBonusPercent == 0, "vector bonus assumed to be 0");1538 unsigned Threshold = ArgAllocaCost * getInliningThresholdMultiplier();1539 1540 bool SingleBB = none_of(*CB->getCalledFunction(), [](const BasicBlock &BB) {1541 return BB.getTerminator()->getNumSuccessors() > 1;1542 });1543 if (SingleBB) {1544 Threshold += Threshold / 2;1545 }1546 1547 auto ArgAllocaSize = DL.getTypeAllocSize(AI->getAllocatedType());1548 1549 // Attribute the bonus proportionally to the alloca size1550 unsigned AllocaThresholdBonus = (Threshold * ArgAllocaSize) / AllocaSize;1551 1552 return AllocaThresholdBonus;1553}1554 1555void GCNTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,1556 TTI::UnrollingPreferences &UP,1557 OptimizationRemarkEmitter *ORE) const {1558 CommonTTI.getUnrollingPreferences(L, SE, UP, ORE);1559}1560 1561void GCNTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,1562 TTI::PeelingPreferences &PP) const {1563 CommonTTI.getPeelingPreferences(L, SE, PP);1564}1565 1566int GCNTTIImpl::get64BitInstrCost(TTI::TargetCostKind CostKind) const {1567 return ST->hasFullRate64Ops()1568 ? getFullRateInstrCost()1569 : ST->hasHalfRate64Ops() ? getHalfRateInstrCost(CostKind)1570 : getQuarterRateInstrCost(CostKind);1571}1572 1573std::pair<InstructionCost, MVT>1574GCNTTIImpl::getTypeLegalizationCost(Type *Ty) const {1575 std::pair<InstructionCost, MVT> Cost = BaseT::getTypeLegalizationCost(Ty);1576 auto Size = DL.getTypeSizeInBits(Ty);1577 // Maximum load or store can handle 8 dwords for scalar and 4 for1578 // vector ALU. Let's assume anything above 8 dwords is expensive1579 // even if legal.1580 if (Size <= 256)1581 return Cost;1582 1583 Cost.first += (Size + 255) / 256;1584 return Cost;1585}1586 1587unsigned GCNTTIImpl::getPrefetchDistance() const {1588 return ST->hasPrefetch() ? 128 : 0;1589}1590 1591bool GCNTTIImpl::shouldPrefetchAddressSpace(unsigned AS) const {1592 return AMDGPU::isFlatGlobalAddrSpace(AS);1593}1594 1595void GCNTTIImpl::collectKernelLaunchBounds(1596 const Function &F,1597 SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const {1598 SmallVector<unsigned> MaxNumWorkgroups = ST->getMaxNumWorkGroups(F);1599 LB.push_back({"amdgpu-max-num-workgroups[0]", MaxNumWorkgroups[0]});1600 LB.push_back({"amdgpu-max-num-workgroups[1]", MaxNumWorkgroups[1]});1601 LB.push_back({"amdgpu-max-num-workgroups[2]", MaxNumWorkgroups[2]});1602 std::pair<unsigned, unsigned> FlatWorkGroupSize =1603 ST->getFlatWorkGroupSizes(F);1604 LB.push_back({"amdgpu-flat-work-group-size[0]", FlatWorkGroupSize.first});1605 LB.push_back({"amdgpu-flat-work-group-size[1]", FlatWorkGroupSize.second});1606 std::pair<unsigned, unsigned> WavesPerEU = ST->getWavesPerEU(F);1607 LB.push_back({"amdgpu-waves-per-eu[0]", WavesPerEU.first});1608 LB.push_back({"amdgpu-waves-per-eu[1]", WavesPerEU.second});1609}1610 1611GCNTTIImpl::KnownIEEEMode1612GCNTTIImpl::fpenvIEEEMode(const Instruction &I) const {1613 if (!ST->hasIEEEMode()) // Only mode on gfx121614 return KnownIEEEMode::On;1615 1616 const Function *F = I.getFunction();1617 if (!F)1618 return KnownIEEEMode::Unknown;1619 1620 Attribute IEEEAttr = F->getFnAttribute("amdgpu-ieee");1621 if (IEEEAttr.isValid())1622 return IEEEAttr.getValueAsBool() ? KnownIEEEMode::On : KnownIEEEMode::Off;1623 1624 return AMDGPU::isShader(F->getCallingConv()) ? KnownIEEEMode::Off1625 : KnownIEEEMode::On;1626}1627 1628InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,1629 Align Alignment,1630 unsigned AddressSpace,1631 TTI::TargetCostKind CostKind,1632 TTI::OperandValueInfo OpInfo,1633 const Instruction *I) const {1634 if (VectorType *VecTy = dyn_cast<VectorType>(Src)) {1635 if ((Opcode == Instruction::Load || Opcode == Instruction::Store) &&1636 VecTy->getElementType()->isIntegerTy(8)) {1637 return divideCeil(DL.getTypeSizeInBits(VecTy) - 1,1638 getLoadStoreVecRegBitWidth(AddressSpace));1639 }1640 }1641 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind,1642 OpInfo, I);1643}1644 1645unsigned GCNTTIImpl::getNumberOfParts(Type *Tp) const {1646 if (VectorType *VecTy = dyn_cast<VectorType>(Tp)) {1647 if (VecTy->getElementType()->isIntegerTy(8)) {1648 unsigned ElementCount = VecTy->getElementCount().getFixedValue();1649 return divideCeil(ElementCount - 1, 4);1650 }1651 }1652 return BaseT::getNumberOfParts(Tp);1653}1654