3444 lines · cpp
1//===-- RISCVTargetTransformInfo.cpp - RISC-V specific TTI ----------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "RISCVTargetTransformInfo.h"10#include "MCTargetDesc/RISCVMatInt.h"11#include "llvm/ADT/STLExtras.h"12#include "llvm/Analysis/TargetTransformInfo.h"13#include "llvm/CodeGen/BasicTTIImpl.h"14#include "llvm/CodeGen/CostTable.h"15#include "llvm/CodeGen/TargetLowering.h"16#include "llvm/CodeGen/ValueTypes.h"17#include "llvm/IR/Instructions.h"18#include "llvm/IR/IntrinsicsRISCV.h"19#include "llvm/IR/PatternMatch.h"20#include <cmath>21#include <optional>22using namespace llvm;23using namespace llvm::PatternMatch;24 25#define DEBUG_TYPE "riscvtti"26 27static cl::opt<unsigned> RVVRegisterWidthLMUL(28 "riscv-v-register-bit-width-lmul",29 cl::desc(30 "The LMUL to use for getRegisterBitWidth queries. Affects LMUL used "31 "by autovectorized code. Fractional LMULs are not supported."),32 cl::init(2), cl::Hidden);33 34static cl::opt<unsigned> SLPMaxVF(35 "riscv-v-slp-max-vf",36 cl::desc(37 "Overrides result used for getMaximumVF query which is used "38 "exclusively by SLP vectorizer."),39 cl::Hidden);40 41static cl::opt<unsigned>42 RVVMinTripCount("riscv-v-min-trip-count",43 cl::desc("Set the lower bound of a trip count to decide on "44 "vectorization while tail-folding."),45 cl::init(5), cl::Hidden);46 47InstructionCost48RISCVTTIImpl::getRISCVInstructionCost(ArrayRef<unsigned> OpCodes, MVT VT,49 TTI::TargetCostKind CostKind) const {50 // Check if the type is valid for all CostKind51 if (!VT.isVector())52 return InstructionCost::getInvalid();53 size_t NumInstr = OpCodes.size();54 if (CostKind == TTI::TCK_CodeSize)55 return NumInstr;56 InstructionCost LMULCost = TLI->getLMULCost(VT);57 if ((CostKind != TTI::TCK_RecipThroughput) && (CostKind != TTI::TCK_Latency))58 return LMULCost * NumInstr;59 InstructionCost Cost = 0;60 for (auto Op : OpCodes) {61 switch (Op) {62 case RISCV::VRGATHER_VI:63 Cost += TLI->getVRGatherVICost(VT);64 break;65 case RISCV::VRGATHER_VV:66 Cost += TLI->getVRGatherVVCost(VT);67 break;68 case RISCV::VSLIDEUP_VI:69 case RISCV::VSLIDEDOWN_VI:70 Cost += TLI->getVSlideVICost(VT);71 break;72 case RISCV::VSLIDEUP_VX:73 case RISCV::VSLIDEDOWN_VX:74 Cost += TLI->getVSlideVXCost(VT);75 break;76 case RISCV::VREDMAX_VS:77 case RISCV::VREDMIN_VS:78 case RISCV::VREDMAXU_VS:79 case RISCV::VREDMINU_VS:80 case RISCV::VREDSUM_VS:81 case RISCV::VREDAND_VS:82 case RISCV::VREDOR_VS:83 case RISCV::VREDXOR_VS:84 case RISCV::VFREDMAX_VS:85 case RISCV::VFREDMIN_VS:86 case RISCV::VFREDUSUM_VS: {87 unsigned VL = VT.getVectorMinNumElements();88 if (!VT.isFixedLengthVector())89 VL *= *getVScaleForTuning();90 Cost += Log2_32_Ceil(VL);91 break;92 }93 case RISCV::VFREDOSUM_VS: {94 unsigned VL = VT.getVectorMinNumElements();95 if (!VT.isFixedLengthVector())96 VL *= *getVScaleForTuning();97 Cost += VL;98 break;99 }100 case RISCV::VMV_X_S:101 case RISCV::VMV_S_X:102 case RISCV::VFMV_F_S:103 case RISCV::VFMV_S_F:104 case RISCV::VMOR_MM:105 case RISCV::VMXOR_MM:106 case RISCV::VMAND_MM:107 case RISCV::VMANDN_MM:108 case RISCV::VMNAND_MM:109 case RISCV::VCPOP_M:110 case RISCV::VFIRST_M:111 Cost += 1;112 break;113 default:114 Cost += LMULCost;115 }116 }117 return Cost;118}119 120static InstructionCost getIntImmCostImpl(const DataLayout &DL,121 const RISCVSubtarget *ST,122 const APInt &Imm, Type *Ty,123 TTI::TargetCostKind CostKind,124 bool FreeZeroes) {125 assert(Ty->isIntegerTy() &&126 "getIntImmCost can only estimate cost of materialising integers");127 128 // We have a Zero register, so 0 is always free.129 if (Imm == 0)130 return TTI::TCC_Free;131 132 // Otherwise, we check how many instructions it will take to materialise.133 return RISCVMatInt::getIntMatCost(Imm, DL.getTypeSizeInBits(Ty), *ST,134 /*CompressionCost=*/false, FreeZeroes);135}136 137InstructionCost138RISCVTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,139 TTI::TargetCostKind CostKind) const {140 return getIntImmCostImpl(getDataLayout(), getST(), Imm, Ty, CostKind, false);141}142 143// Look for patterns of shift followed by AND that can be turned into a pair of144// shifts. We won't need to materialize an immediate for the AND so these can145// be considered free.146static bool canUseShiftPair(Instruction *Inst, const APInt &Imm) {147 uint64_t Mask = Imm.getZExtValue();148 auto *BO = dyn_cast<BinaryOperator>(Inst->getOperand(0));149 if (!BO || !BO->hasOneUse())150 return false;151 152 if (BO->getOpcode() != Instruction::Shl)153 return false;154 155 if (!isa<ConstantInt>(BO->getOperand(1)))156 return false;157 158 unsigned ShAmt = cast<ConstantInt>(BO->getOperand(1))->getZExtValue();159 // (and (shl x, c2), c1) will be matched to (srli (slli x, c2+c3), c3) if c1160 // is a mask shifted by c2 bits with c3 leading zeros.161 if (isShiftedMask_64(Mask)) {162 unsigned Trailing = llvm::countr_zero(Mask);163 if (ShAmt == Trailing)164 return true;165 }166 167 return false;168}169 170// If this is i64 AND is part of (X & -(1 << C1) & 0xffffffff) == C2 << C1),171// DAGCombiner can convert this to (sraiw X, C1) == sext(C2) for RV64. On RV32,172// the type will be split so only the lower 32 bits need to be compared using173// (srai/srli X, C) == C2.174static bool canUseShiftCmp(Instruction *Inst, const APInt &Imm) {175 if (!Inst->hasOneUse())176 return false;177 178 // Look for equality comparison.179 auto *Cmp = dyn_cast<ICmpInst>(*Inst->user_begin());180 if (!Cmp || !Cmp->isEquality())181 return false;182 183 // Right hand side of comparison should be a constant.184 auto *C = dyn_cast<ConstantInt>(Cmp->getOperand(1));185 if (!C)186 return false;187 188 uint64_t Mask = Imm.getZExtValue();189 190 // Mask should be of the form -(1 << C) in the lower 32 bits.191 if (!isUInt<32>(Mask) || !isPowerOf2_32(-uint32_t(Mask)))192 return false;193 194 // Comparison constant should be a subset of Mask.195 uint64_t CmpC = C->getZExtValue();196 if ((CmpC & Mask) != CmpC)197 return false;198 199 // We'll need to sign extend the comparison constant and shift it right. Make200 // sure the new constant can use addi/xori+seqz/snez.201 unsigned ShiftBits = llvm::countr_zero(Mask);202 int64_t NewCmpC = SignExtend64<32>(CmpC) >> ShiftBits;203 return NewCmpC >= -2048 && NewCmpC <= 2048;204}205 206InstructionCost RISCVTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,207 const APInt &Imm, Type *Ty,208 TTI::TargetCostKind CostKind,209 Instruction *Inst) const {210 assert(Ty->isIntegerTy() &&211 "getIntImmCost can only estimate cost of materialising integers");212 213 // We have a Zero register, so 0 is always free.214 if (Imm == 0)215 return TTI::TCC_Free;216 217 // Some instructions in RISC-V can take a 12-bit immediate. Some of these are218 // commutative, in others the immediate comes from a specific argument index.219 bool Takes12BitImm = false;220 unsigned ImmArgIdx = ~0U;221 222 switch (Opcode) {223 case Instruction::GetElementPtr:224 // Never hoist any arguments to a GetElementPtr. CodeGenPrepare will225 // split up large offsets in GEP into better parts than ConstantHoisting226 // can.227 return TTI::TCC_Free;228 case Instruction::Store: {229 // Use the materialization cost regardless of if it's the address or the230 // value that is constant, except for if the store is misaligned and231 // misaligned accesses are not legal (experience shows constant hoisting232 // can sometimes be harmful in such cases).233 if (Idx == 1 || !Inst)234 return getIntImmCostImpl(getDataLayout(), getST(), Imm, Ty, CostKind,235 /*FreeZeroes=*/true);236 237 StoreInst *ST = cast<StoreInst>(Inst);238 if (!getTLI()->allowsMemoryAccessForAlignment(239 Ty->getContext(), DL, getTLI()->getValueType(DL, Ty),240 ST->getPointerAddressSpace(), ST->getAlign()))241 return TTI::TCC_Free;242 243 return getIntImmCostImpl(getDataLayout(), getST(), Imm, Ty, CostKind,244 /*FreeZeroes=*/true);245 }246 case Instruction::Load:247 // If the address is a constant, use the materialization cost.248 return getIntImmCost(Imm, Ty, CostKind);249 case Instruction::And:250 // zext.h251 if (Imm == UINT64_C(0xffff) && ST->hasStdExtZbb())252 return TTI::TCC_Free;253 // zext.w254 if (Imm == UINT64_C(0xffffffff) &&255 ((ST->hasStdExtZba() && ST->isRV64()) || ST->isRV32()))256 return TTI::TCC_Free;257 // bclri258 if (ST->hasStdExtZbs() && (~Imm).isPowerOf2())259 return TTI::TCC_Free;260 if (Inst && Idx == 1 && Imm.getBitWidth() <= ST->getXLen() &&261 canUseShiftPair(Inst, Imm))262 return TTI::TCC_Free;263 if (Inst && Idx == 1 && Imm.getBitWidth() == 64 &&264 canUseShiftCmp(Inst, Imm))265 return TTI::TCC_Free;266 Takes12BitImm = true;267 break;268 case Instruction::Add:269 Takes12BitImm = true;270 break;271 case Instruction::Or:272 case Instruction::Xor:273 // bseti/binvi274 if (ST->hasStdExtZbs() && Imm.isPowerOf2())275 return TTI::TCC_Free;276 Takes12BitImm = true;277 break;278 case Instruction::Mul:279 // Power of 2 is a shift. Negated power of 2 is a shift and a negate.280 if (Imm.isPowerOf2() || Imm.isNegatedPowerOf2())281 return TTI::TCC_Free;282 // One more or less than a power of 2 can use SLLI+ADD/SUB.283 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2())284 return TTI::TCC_Free;285 // FIXME: There is no MULI instruction.286 Takes12BitImm = true;287 break;288 case Instruction::Sub:289 case Instruction::Shl:290 case Instruction::LShr:291 case Instruction::AShr:292 Takes12BitImm = true;293 ImmArgIdx = 1;294 break;295 default:296 break;297 }298 299 if (Takes12BitImm) {300 // Check immediate is the correct argument...301 if (Instruction::isCommutative(Opcode) || Idx == ImmArgIdx) {302 // ... and fits into the 12-bit immediate.303 if (Imm.getSignificantBits() <= 64 &&304 getTLI()->isLegalAddImmediate(Imm.getSExtValue())) {305 return TTI::TCC_Free;306 }307 }308 309 // Otherwise, use the full materialisation cost.310 return getIntImmCost(Imm, Ty, CostKind);311 }312 313 // By default, prevent hoisting.314 return TTI::TCC_Free;315}316 317InstructionCost318RISCVTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,319 const APInt &Imm, Type *Ty,320 TTI::TargetCostKind CostKind) const {321 // Prevent hoisting in unknown cases.322 return TTI::TCC_Free;323}324 325bool RISCVTTIImpl::hasActiveVectorLength() const {326 return ST->hasVInstructions();327}328 329TargetTransformInfo::PopcntSupportKind330RISCVTTIImpl::getPopcntSupport(unsigned TyWidth) const {331 assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");332 return ST->hasCPOPLike() ? TTI::PSK_FastHardware : TTI::PSK_Software;333}334 335InstructionCost RISCVTTIImpl::getPartialReductionCost(336 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,337 ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,338 TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,339 TTI::TargetCostKind CostKind) const {340 341 // zve32x is broken for partial_reduce_umla, but let's make sure we342 // don't generate them.343 if (!ST->hasStdExtZvqdotq() || ST->getELen() < 64 ||344 Opcode != Instruction::Add || !BinOp || *BinOp != Instruction::Mul ||345 InputTypeA != InputTypeB || !InputTypeA->isIntegerTy(8) ||346 !AccumType->isIntegerTy(32) || !VF.isKnownMultipleOf(4))347 return InstructionCost::getInvalid();348 349 Type *Tp = VectorType::get(AccumType, VF.divideCoefficientBy(4));350 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Tp);351 // Note: Asuming all vqdot* variants are equal cost352 return LT.first *353 getRISCVInstructionCost(RISCV::VQDOT_VV, LT.second, CostKind);354}355 356bool RISCVTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {357 // Currently, the ExpandReductions pass can't expand scalable-vector358 // reductions, but we still request expansion as RVV doesn't support certain359 // reductions and the SelectionDAG can't legalize them either.360 switch (II->getIntrinsicID()) {361 default:362 return false;363 // These reductions have no equivalent in RVV364 case Intrinsic::vector_reduce_mul:365 case Intrinsic::vector_reduce_fmul:366 return true;367 }368}369 370std::optional<unsigned> RISCVTTIImpl::getMaxVScale() const {371 if (ST->hasVInstructions())372 return ST->getRealMaxVLen() / RISCV::RVVBitsPerBlock;373 return BaseT::getMaxVScale();374}375 376std::optional<unsigned> RISCVTTIImpl::getVScaleForTuning() const {377 if (ST->hasVInstructions())378 if (unsigned MinVLen = ST->getRealMinVLen();379 MinVLen >= RISCV::RVVBitsPerBlock)380 return MinVLen / RISCV::RVVBitsPerBlock;381 return BaseT::getVScaleForTuning();382}383 384TypeSize385RISCVTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {386 unsigned LMUL =387 llvm::bit_floor(std::clamp<unsigned>(RVVRegisterWidthLMUL, 1, 8));388 switch (K) {389 case TargetTransformInfo::RGK_Scalar:390 return TypeSize::getFixed(ST->getXLen());391 case TargetTransformInfo::RGK_FixedWidthVector:392 return TypeSize::getFixed(393 ST->useRVVForFixedLengthVectors() ? LMUL * ST->getRealMinVLen() : 0);394 case TargetTransformInfo::RGK_ScalableVector:395 return TypeSize::getScalable(396 (ST->hasVInstructions() &&397 ST->getRealMinVLen() >= RISCV::RVVBitsPerBlock)398 ? LMUL * RISCV::RVVBitsPerBlock399 : 0);400 }401 402 llvm_unreachable("Unsupported register kind");403}404 405InstructionCost406RISCVTTIImpl::getConstantPoolLoadCost(Type *Ty,407 TTI::TargetCostKind CostKind) const {408 // Add a cost of address generation + the cost of the load. The address409 // is expected to be a PC relative offset to a constant pool entry410 // using auipc/addi.411 return 2 + getMemoryOpCost(Instruction::Load, Ty, DL.getABITypeAlign(Ty),412 /*AddressSpace=*/0, CostKind);413}414 415static bool isRepeatedConcatMask(ArrayRef<int> Mask, int &SubVectorSize) {416 unsigned Size = Mask.size();417 if (!isPowerOf2_32(Size))418 return false;419 for (unsigned I = 0; I != Size; ++I) {420 if (static_cast<unsigned>(Mask[I]) == I)421 continue;422 if (Mask[I] != 0)423 return false;424 if (Size % I != 0)425 return false;426 for (unsigned J = I + 1; J != Size; ++J)427 // Check the pattern is repeated.428 if (static_cast<unsigned>(Mask[J]) != J % I)429 return false;430 SubVectorSize = I;431 return true;432 }433 // That means Mask is <0, 1, 2, 3>. This is not a concatenation.434 return false;435}436 437static VectorType *getVRGatherIndexType(MVT DataVT, const RISCVSubtarget &ST,438 LLVMContext &C) {439 assert((DataVT.getScalarSizeInBits() != 8 ||440 DataVT.getVectorNumElements() <= 256) && "unhandled case in lowering");441 MVT IndexVT = DataVT.changeTypeToInteger();442 if (IndexVT.getScalarType().bitsGT(ST.getXLenVT()))443 IndexVT = IndexVT.changeVectorElementType(MVT::i16);444 return cast<VectorType>(EVT(IndexVT).getTypeForEVT(C));445}446 447/// Attempt to approximate the cost of a shuffle which will require splitting448/// during legalization. Note that processShuffleMasks is not an exact proxy449/// for the algorithm used in LegalizeVectorTypes, but hopefully it's a450/// reasonably close upperbound.451static InstructionCost costShuffleViaSplitting(const RISCVTTIImpl &TTI,452 MVT LegalVT, VectorType *Tp,453 ArrayRef<int> Mask,454 TTI::TargetCostKind CostKind) {455 assert(LegalVT.isFixedLengthVector() && !Mask.empty() &&456 "Expected fixed vector type and non-empty mask");457 unsigned LegalNumElts = LegalVT.getVectorNumElements();458 // Number of destination vectors after legalization:459 unsigned NumOfDests = divideCeil(Mask.size(), LegalNumElts);460 // We are going to permute multiple sources and the result will be in461 // multiple destinations. Providing an accurate cost only for splits where462 // the element type remains the same.463 if (NumOfDests <= 1 ||464 LegalVT.getVectorElementType().getSizeInBits() !=465 Tp->getElementType()->getPrimitiveSizeInBits() ||466 LegalNumElts >= Tp->getElementCount().getFixedValue())467 return InstructionCost::getInvalid();468 469 unsigned VecTySize = TTI.getDataLayout().getTypeStoreSize(Tp);470 unsigned LegalVTSize = LegalVT.getStoreSize();471 // Number of source vectors after legalization:472 unsigned NumOfSrcs = divideCeil(VecTySize, LegalVTSize);473 474 auto *SingleOpTy = FixedVectorType::get(Tp->getElementType(), LegalNumElts);475 476 unsigned NormalizedVF = LegalNumElts * std::max(NumOfSrcs, NumOfDests);477 unsigned NumOfSrcRegs = NormalizedVF / LegalNumElts;478 unsigned NumOfDestRegs = NormalizedVF / LegalNumElts;479 SmallVector<int> NormalizedMask(NormalizedVF, PoisonMaskElem);480 assert(NormalizedVF >= Mask.size() &&481 "Normalized mask expected to be not shorter than original mask.");482 copy(Mask, NormalizedMask.begin());483 InstructionCost Cost = 0;484 SmallDenseSet<std::pair<ArrayRef<int>, unsigned>> ReusedSingleSrcShuffles;485 processShuffleMasks(486 NormalizedMask, NumOfSrcRegs, NumOfDestRegs, NumOfDestRegs, []() {},487 [&](ArrayRef<int> RegMask, unsigned SrcReg, unsigned DestReg) {488 if (ShuffleVectorInst::isIdentityMask(RegMask, RegMask.size()))489 return;490 if (!ReusedSingleSrcShuffles.insert(std::make_pair(RegMask, SrcReg))491 .second)492 return;493 Cost += TTI.getShuffleCost(494 TTI::SK_PermuteSingleSrc,495 FixedVectorType::get(SingleOpTy->getElementType(), RegMask.size()),496 SingleOpTy, RegMask, CostKind, 0, nullptr);497 },498 [&](ArrayRef<int> RegMask, unsigned Idx1, unsigned Idx2, bool NewReg) {499 Cost += TTI.getShuffleCost(500 TTI::SK_PermuteTwoSrc,501 FixedVectorType::get(SingleOpTy->getElementType(), RegMask.size()),502 SingleOpTy, RegMask, CostKind, 0, nullptr);503 });504 return Cost;505}506 507/// Try to perform better estimation of the permutation.508/// 1. Split the source/destination vectors into real registers.509/// 2. Do the mask analysis to identify which real registers are510/// permuted. If more than 1 source registers are used for the511/// destination register building, the cost for this destination register512/// is (Number_of_source_register - 1) * Cost_PermuteTwoSrc. If only one513/// source register is used, build mask and calculate the cost as a cost514/// of PermuteSingleSrc.515/// Also, for the single register permute we try to identify if the516/// destination register is just a copy of the source register or the517/// copy of the previous destination register (the cost is518/// TTI::TCC_Basic). If the source register is just reused, the cost for519/// this operation is 0.520static InstructionCost521costShuffleViaVRegSplitting(const RISCVTTIImpl &TTI, MVT LegalVT,522 std::optional<unsigned> VLen, VectorType *Tp,523 ArrayRef<int> Mask, TTI::TargetCostKind CostKind) {524 assert(LegalVT.isFixedLengthVector());525 if (!VLen || Mask.empty())526 return InstructionCost::getInvalid();527 MVT ElemVT = LegalVT.getVectorElementType();528 unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();529 LegalVT = TTI.getTypeLegalizationCost(530 FixedVectorType::get(Tp->getElementType(), ElemsPerVReg))531 .second;532 // Number of destination vectors after legalization:533 InstructionCost NumOfDests =534 divideCeil(Mask.size(), LegalVT.getVectorNumElements());535 if (NumOfDests <= 1 ||536 LegalVT.getVectorElementType().getSizeInBits() !=537 Tp->getElementType()->getPrimitiveSizeInBits() ||538 LegalVT.getVectorNumElements() >= Tp->getElementCount().getFixedValue())539 return InstructionCost::getInvalid();540 541 unsigned VecTySize = TTI.getDataLayout().getTypeStoreSize(Tp);542 unsigned LegalVTSize = LegalVT.getStoreSize();543 // Number of source vectors after legalization:544 unsigned NumOfSrcs = divideCeil(VecTySize, LegalVTSize);545 546 auto *SingleOpTy = FixedVectorType::get(Tp->getElementType(),547 LegalVT.getVectorNumElements());548 549 unsigned E = NumOfDests.getValue();550 unsigned NormalizedVF =551 LegalVT.getVectorNumElements() * std::max(NumOfSrcs, E);552 unsigned NumOfSrcRegs = NormalizedVF / LegalVT.getVectorNumElements();553 unsigned NumOfDestRegs = NormalizedVF / LegalVT.getVectorNumElements();554 SmallVector<int> NormalizedMask(NormalizedVF, PoisonMaskElem);555 assert(NormalizedVF >= Mask.size() &&556 "Normalized mask expected to be not shorter than original mask.");557 copy(Mask, NormalizedMask.begin());558 InstructionCost Cost = 0;559 int NumShuffles = 0;560 SmallDenseSet<std::pair<ArrayRef<int>, unsigned>> ReusedSingleSrcShuffles;561 processShuffleMasks(562 NormalizedMask, NumOfSrcRegs, NumOfDestRegs, NumOfDestRegs, []() {},563 [&](ArrayRef<int> RegMask, unsigned SrcReg, unsigned DestReg) {564 if (ShuffleVectorInst::isIdentityMask(RegMask, RegMask.size()))565 return;566 if (!ReusedSingleSrcShuffles.insert(std::make_pair(RegMask, SrcReg))567 .second)568 return;569 ++NumShuffles;570 Cost += TTI.getShuffleCost(TTI::SK_PermuteSingleSrc, SingleOpTy,571 SingleOpTy, RegMask, CostKind, 0, nullptr);572 },573 [&](ArrayRef<int> RegMask, unsigned Idx1, unsigned Idx2, bool NewReg) {574 Cost += TTI.getShuffleCost(TTI::SK_PermuteTwoSrc, SingleOpTy,575 SingleOpTy, RegMask, CostKind, 0, nullptr);576 NumShuffles += 2;577 });578 // Note: check that we do not emit too many shuffles here to prevent code579 // size explosion.580 // TODO: investigate, if it can be improved by extra analysis of the masks581 // to check if the code is more profitable.582 if ((NumOfDestRegs > 2 && NumShuffles <= static_cast<int>(NumOfDestRegs)) ||583 (NumOfDestRegs <= 2 && NumShuffles < 4))584 return Cost;585 return InstructionCost::getInvalid();586}587 588InstructionCost RISCVTTIImpl::getSlideCost(FixedVectorType *Tp,589 ArrayRef<int> Mask,590 TTI::TargetCostKind CostKind) const {591 // Avoid missing masks and length changing shuffles592 if (Mask.size() <= 2 || Mask.size() != Tp->getNumElements())593 return InstructionCost::getInvalid();594 595 int NumElts = Tp->getNumElements();596 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Tp);597 // Avoid scalarization cases598 if (!LT.second.isFixedLengthVector())599 return InstructionCost::getInvalid();600 601 // Requires moving elements between parts, which requires additional602 // unmodeled instructions.603 if (LT.first != 1)604 return InstructionCost::getInvalid();605 606 auto GetSlideOpcode = [&](int SlideAmt) {607 assert(SlideAmt != 0);608 bool IsVI = isUInt<5>(std::abs(SlideAmt));609 if (SlideAmt < 0)610 return IsVI ? RISCV::VSLIDEDOWN_VI : RISCV::VSLIDEDOWN_VX;611 return IsVI ? RISCV::VSLIDEUP_VI : RISCV::VSLIDEUP_VX;612 };613 614 std::array<std::pair<int, int>, 2> SrcInfo;615 if (!isMaskedSlidePair(Mask, NumElts, SrcInfo))616 return InstructionCost::getInvalid();617 618 if (SrcInfo[1].second == 0)619 std::swap(SrcInfo[0], SrcInfo[1]);620 621 InstructionCost FirstSlideCost = 0;622 if (SrcInfo[0].second != 0) {623 unsigned Opcode = GetSlideOpcode(SrcInfo[0].second);624 FirstSlideCost = getRISCVInstructionCost(Opcode, LT.second, CostKind);625 }626 627 if (SrcInfo[1].first == -1)628 return FirstSlideCost;629 630 InstructionCost SecondSlideCost = 0;631 if (SrcInfo[1].second != 0) {632 unsigned Opcode = GetSlideOpcode(SrcInfo[1].second);633 SecondSlideCost = getRISCVInstructionCost(Opcode, LT.second, CostKind);634 } else {635 SecondSlideCost =636 getRISCVInstructionCost(RISCV::VMERGE_VVM, LT.second, CostKind);637 }638 639 auto EC = Tp->getElementCount();640 VectorType *MaskTy =641 VectorType::get(IntegerType::getInt1Ty(Tp->getContext()), EC);642 InstructionCost MaskCost = getConstantPoolLoadCost(MaskTy, CostKind);643 return FirstSlideCost + SecondSlideCost + MaskCost;644}645 646InstructionCost647RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,648 VectorType *SrcTy, ArrayRef<int> Mask,649 TTI::TargetCostKind CostKind, int Index,650 VectorType *SubTp, ArrayRef<const Value *> Args,651 const Instruction *CxtI) const {652 assert((Mask.empty() || DstTy->isScalableTy() ||653 Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&654 "Expected the Mask to match the return size if given");655 assert(SrcTy->getScalarType() == DstTy->getScalarType() &&656 "Expected the same scalar types");657 658 Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTp);659 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(SrcTy);660 661 // First, handle cases where having a fixed length vector enables us to662 // give a more accurate cost than falling back to generic scalable codegen.663 // TODO: Each of these cases hints at a modeling gap around scalable vectors.664 if (auto *FVTp = dyn_cast<FixedVectorType>(SrcTy);665 FVTp && ST->hasVInstructions() && LT.second.isFixedLengthVector()) {666 InstructionCost VRegSplittingCost = costShuffleViaVRegSplitting(667 *this, LT.second, ST->getRealVLen(),668 Kind == TTI::SK_InsertSubvector ? DstTy : SrcTy, Mask, CostKind);669 if (VRegSplittingCost.isValid())670 return VRegSplittingCost;671 switch (Kind) {672 default:673 break;674 case TTI::SK_PermuteSingleSrc: {675 if (Mask.size() >= 2) {676 MVT EltTp = LT.second.getVectorElementType();677 // If the size of the element is < ELEN then shuffles of interleaves and678 // deinterleaves of 2 vectors can be lowered into the following679 // sequences680 if (EltTp.getScalarSizeInBits() < ST->getELen()) {681 // Example sequence:682 // vsetivli zero, 4, e8, mf4, ta, ma (ignored)683 // vwaddu.vv v10, v8, v9684 // li a0, -1 (ignored)685 // vwmaccu.vx v10, a0, v9686 if (ShuffleVectorInst::isInterleaveMask(Mask, 2, Mask.size()))687 return 2 * LT.first * TLI->getLMULCost(LT.second);688 689 if (Mask[0] == 0 || Mask[0] == 1) {690 auto DeinterleaveMask = createStrideMask(Mask[0], 2, Mask.size());691 // Example sequence:692 // vnsrl.wi v10, v8, 0693 if (equal(DeinterleaveMask, Mask))694 return LT.first * getRISCVInstructionCost(RISCV::VNSRL_WI,695 LT.second, CostKind);696 }697 }698 int SubVectorSize;699 if (LT.second.getScalarSizeInBits() != 1 &&700 isRepeatedConcatMask(Mask, SubVectorSize)) {701 InstructionCost Cost = 0;702 unsigned NumSlides = Log2_32(Mask.size() / SubVectorSize);703 // The cost of extraction from a subvector is 0 if the index is 0.704 for (unsigned I = 0; I != NumSlides; ++I) {705 unsigned InsertIndex = SubVectorSize * (1 << I);706 FixedVectorType *SubTp =707 FixedVectorType::get(SrcTy->getElementType(), InsertIndex);708 FixedVectorType *DestTp =709 FixedVectorType::getDoubleElementsVectorType(SubTp);710 std::pair<InstructionCost, MVT> DestLT =711 getTypeLegalizationCost(DestTp);712 // Add the cost of whole vector register move because the713 // destination vector register group for vslideup cannot overlap the714 // source.715 Cost += DestLT.first * TLI->getLMULCost(DestLT.second);716 Cost += getShuffleCost(TTI::SK_InsertSubvector, DestTp, DestTp, {},717 CostKind, InsertIndex, SubTp);718 }719 return Cost;720 }721 }722 723 if (InstructionCost SlideCost = getSlideCost(FVTp, Mask, CostKind);724 SlideCost.isValid())725 return SlideCost;726 727 // vrgather + cost of generating the mask constant.728 // We model this for an unknown mask with a single vrgather.729 if (LT.first == 1 && (LT.second.getScalarSizeInBits() != 8 ||730 LT.second.getVectorNumElements() <= 256)) {731 VectorType *IdxTy =732 getVRGatherIndexType(LT.second, *ST, SrcTy->getContext());733 InstructionCost IndexCost = getConstantPoolLoadCost(IdxTy, CostKind);734 return IndexCost +735 getRISCVInstructionCost(RISCV::VRGATHER_VV, LT.second, CostKind);736 }737 break;738 }739 case TTI::SK_Transpose:740 case TTI::SK_PermuteTwoSrc: {741 742 if (InstructionCost SlideCost = getSlideCost(FVTp, Mask, CostKind);743 SlideCost.isValid())744 return SlideCost;745 746 // 2 x (vrgather + cost of generating the mask constant) + cost of mask747 // register for the second vrgather. We model this for an unknown748 // (shuffle) mask.749 if (LT.first == 1 && (LT.second.getScalarSizeInBits() != 8 ||750 LT.second.getVectorNumElements() <= 256)) {751 auto &C = SrcTy->getContext();752 auto EC = SrcTy->getElementCount();753 VectorType *IdxTy = getVRGatherIndexType(LT.second, *ST, C);754 VectorType *MaskTy = VectorType::get(IntegerType::getInt1Ty(C), EC);755 InstructionCost IndexCost = getConstantPoolLoadCost(IdxTy, CostKind);756 InstructionCost MaskCost = getConstantPoolLoadCost(MaskTy, CostKind);757 return 2 * IndexCost +758 getRISCVInstructionCost({RISCV::VRGATHER_VV, RISCV::VRGATHER_VV},759 LT.second, CostKind) +760 MaskCost;761 }762 break;763 }764 }765 766 auto shouldSplit = [](TTI::ShuffleKind Kind) {767 switch (Kind) {768 default:769 return false;770 case TTI::SK_PermuteSingleSrc:771 case TTI::SK_Transpose:772 case TTI::SK_PermuteTwoSrc:773 return true;774 }775 };776 777 if (!Mask.empty() && LT.first.isValid() && LT.first != 1 &&778 shouldSplit(Kind)) {779 InstructionCost SplitCost =780 costShuffleViaSplitting(*this, LT.second, FVTp, Mask, CostKind);781 if (SplitCost.isValid())782 return SplitCost;783 }784 }785 786 // Handle scalable vectors (and fixed vectors legalized to scalable vectors).787 switch (Kind) {788 default:789 // Fallthrough to generic handling.790 // TODO: Most of these cases will return getInvalid in generic code, and791 // must be implemented here.792 break;793 case TTI::SK_ExtractSubvector:794 // Extract at zero is always a subregister extract795 if (Index == 0)796 return TTI::TCC_Free;797 798 // If we're extracting a subvector of at most m1 size at a sub-register799 // boundary - which unfortunately we need exact vlen to identify - this is800 // a subregister extract at worst and thus won't require a vslidedown.801 // TODO: Extend for aligned m2, m4 subvector extracts802 // TODO: Extend for misalgined (but contained) extracts803 // TODO: Extend for scalable subvector types804 if (std::pair<InstructionCost, MVT> SubLT = getTypeLegalizationCost(SubTp);805 SubLT.second.isValid() && SubLT.second.isFixedLengthVector()) {806 if (std::optional<unsigned> VLen = ST->getRealVLen();807 VLen && SubLT.second.getScalarSizeInBits() * Index % *VLen == 0 &&808 SubLT.second.getSizeInBits() <= *VLen)809 return TTI::TCC_Free;810 }811 812 // Example sequence:813 // vsetivli zero, 4, e8, mf2, tu, ma (ignored)814 // vslidedown.vi v8, v9, 2815 return LT.first *816 getRISCVInstructionCost(RISCV::VSLIDEDOWN_VI, LT.second, CostKind);817 case TTI::SK_InsertSubvector:818 // Example sequence:819 // vsetivli zero, 4, e8, mf2, tu, ma (ignored)820 // vslideup.vi v8, v9, 2821 LT = getTypeLegalizationCost(DstTy);822 return LT.first *823 getRISCVInstructionCost(RISCV::VSLIDEUP_VI, LT.second, CostKind);824 case TTI::SK_Select: {825 // Example sequence:826 // li a0, 90827 // vsetivli zero, 8, e8, mf2, ta, ma (ignored)828 // vmv.s.x v0, a0829 // vmerge.vvm v8, v9, v8, v0830 // We use 2 for the cost of the mask materialization as this is the true831 // cost for small masks and most shuffles are small. At worst, this cost832 // should be a very small constant for the constant pool load. As such,833 // we may bias towards large selects slightly more than truly warranted.834 return LT.first *835 (1 + getRISCVInstructionCost({RISCV::VMV_S_X, RISCV::VMERGE_VVM},836 LT.second, CostKind));837 }838 case TTI::SK_Broadcast: {839 bool HasScalar = (Args.size() > 0) && (Operator::getOpcode(Args[0]) ==840 Instruction::InsertElement);841 if (LT.second.getScalarSizeInBits() == 1) {842 if (HasScalar) {843 // Example sequence:844 // andi a0, a0, 1845 // vsetivli zero, 2, e8, mf8, ta, ma (ignored)846 // vmv.v.x v8, a0847 // vmsne.vi v0, v8, 0848 return LT.first *849 (1 + getRISCVInstructionCost({RISCV::VMV_V_X, RISCV::VMSNE_VI},850 LT.second, CostKind));851 }852 // Example sequence:853 // vsetivli zero, 2, e8, mf8, ta, mu (ignored)854 // vmv.v.i v8, 0855 // vmerge.vim v8, v8, 1, v0856 // vmv.x.s a0, v8857 // andi a0, a0, 1858 // vmv.v.x v8, a0859 // vmsne.vi v0, v8, 0860 861 return LT.first *862 (1 + getRISCVInstructionCost({RISCV::VMV_V_I, RISCV::VMERGE_VIM,863 RISCV::VMV_X_S, RISCV::VMV_V_X,864 RISCV::VMSNE_VI},865 LT.second, CostKind));866 }867 868 if (HasScalar) {869 // Example sequence:870 // vmv.v.x v8, a0871 return LT.first *872 getRISCVInstructionCost(RISCV::VMV_V_X, LT.second, CostKind);873 }874 875 // Example sequence:876 // vrgather.vi v9, v8, 0877 return LT.first *878 getRISCVInstructionCost(RISCV::VRGATHER_VI, LT.second, CostKind);879 }880 case TTI::SK_Splice: {881 // vslidedown+vslideup.882 // TODO: Multiplying by LT.first implies this legalizes into multiple copies883 // of similar code, but I think we expand through memory.884 unsigned Opcodes[2] = {RISCV::VSLIDEDOWN_VX, RISCV::VSLIDEUP_VX};885 if (Index >= 0 && Index < 32)886 Opcodes[0] = RISCV::VSLIDEDOWN_VI;887 else if (Index < 0 && Index > -32)888 Opcodes[1] = RISCV::VSLIDEUP_VI;889 return LT.first * getRISCVInstructionCost(Opcodes, LT.second, CostKind);890 }891 case TTI::SK_Reverse: {892 893 if (!LT.second.isVector())894 return InstructionCost::getInvalid();895 896 // TODO: Cases to improve here:897 // * Illegal vector types898 // * i64 on RV32899 if (SrcTy->getElementType()->isIntegerTy(1)) {900 VectorType *WideTy =901 VectorType::get(IntegerType::get(SrcTy->getContext(), 8),902 cast<VectorType>(SrcTy)->getElementCount());903 return getCastInstrCost(Instruction::ZExt, WideTy, SrcTy,904 TTI::CastContextHint::None, CostKind) +905 getShuffleCost(TTI::SK_Reverse, WideTy, WideTy, {}, CostKind, 0,906 nullptr) +907 getCastInstrCost(Instruction::Trunc, SrcTy, WideTy,908 TTI::CastContextHint::None, CostKind);909 }910 911 MVT ContainerVT = LT.second;912 if (LT.second.isFixedLengthVector())913 ContainerVT = TLI->getContainerForFixedLengthVector(LT.second);914 MVT M1VT = RISCVTargetLowering::getM1VT(ContainerVT);915 if (ContainerVT.bitsLE(M1VT)) {916 // Example sequence:917 // csrr a0, vlenb918 // srli a0, a0, 3919 // addi a0, a0, -1920 // vsetvli a1, zero, e8, mf8, ta, mu (ignored)921 // vid.v v9922 // vrsub.vx v10, v9, a0923 // vrgather.vv v9, v8, v10924 InstructionCost LenCost = 3;925 if (LT.second.isFixedLengthVector())926 // vrsub.vi has a 5 bit immediate field, otherwise an li suffices927 LenCost = isInt<5>(LT.second.getVectorNumElements() - 1) ? 0 : 1;928 unsigned Opcodes[] = {RISCV::VID_V, RISCV::VRSUB_VX, RISCV::VRGATHER_VV};929 if (LT.second.isFixedLengthVector() &&930 isInt<5>(LT.second.getVectorNumElements() - 1))931 Opcodes[1] = RISCV::VRSUB_VI;932 InstructionCost GatherCost =933 getRISCVInstructionCost(Opcodes, LT.second, CostKind);934 return LT.first * (LenCost + GatherCost);935 }936 937 // At high LMUL, we split into a series of M1 reverses (see938 // lowerVECTOR_REVERSE) and then do a single slide at the end to eliminate939 // the resulting gap at the bottom (for fixed vectors only). The important940 // bit is that the cost scales linearly, not quadratically with LMUL.941 unsigned M1Opcodes[] = {RISCV::VID_V, RISCV::VRSUB_VX};942 InstructionCost FixedCost =943 getRISCVInstructionCost(M1Opcodes, M1VT, CostKind) + 3;944 unsigned Ratio =945 ContainerVT.getVectorMinNumElements() / M1VT.getVectorMinNumElements();946 InstructionCost GatherCost =947 getRISCVInstructionCost({RISCV::VRGATHER_VV}, M1VT, CostKind) * Ratio;948 InstructionCost SlideCost = !LT.second.isFixedLengthVector() ? 0 :949 getRISCVInstructionCost({RISCV::VSLIDEDOWN_VX}, LT.second, CostKind);950 return FixedCost + LT.first * (GatherCost + SlideCost);951 }952 }953 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, Mask, CostKind, Index,954 SubTp);955}956 957static unsigned isM1OrSmaller(MVT VT) {958 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);959 return (LMUL == RISCVVType::VLMUL::LMUL_F8 ||960 LMUL == RISCVVType::VLMUL::LMUL_F4 ||961 LMUL == RISCVVType::VLMUL::LMUL_F2 ||962 LMUL == RISCVVType::VLMUL::LMUL_1);963}964 965InstructionCost RISCVTTIImpl::getScalarizationOverhead(966 VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,967 TTI::TargetCostKind CostKind, bool ForPoisonSrc,968 ArrayRef<Value *> VL) const {969 if (isa<ScalableVectorType>(Ty))970 return InstructionCost::getInvalid();971 972 // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)973 // For now, skip all fixed vector cost analysis when P extension is available974 // to avoid crashes in getMinRVVVectorSizeInBits()975 if (ST->enablePExtCodeGen() && isa<FixedVectorType>(Ty)) {976 return 1; // Treat as single instruction cost for now977 }978 979 // A build_vector (which is m1 sized or smaller) can be done in no980 // worse than one vslide1down.vx per element in the type. We could981 // in theory do an explode_vector in the inverse manner, but our982 // lowering today does not have a first class node for this pattern.983 InstructionCost Cost = BaseT::getScalarizationOverhead(984 Ty, DemandedElts, Insert, Extract, CostKind);985 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);986 if (Insert && !Extract && LT.first.isValid() && LT.second.isVector()) {987 if (Ty->getScalarSizeInBits() == 1) {988 auto *WideVecTy = cast<VectorType>(Ty->getWithNewBitWidth(8));989 // Note: Implicit scalar anyextend is assumed to be free since the i1990 // must be stored in a GPR.991 return getScalarizationOverhead(WideVecTy, DemandedElts, Insert, Extract,992 CostKind) +993 getCastInstrCost(Instruction::Trunc, Ty, WideVecTy,994 TTI::CastContextHint::None, CostKind, nullptr);995 }996 997 assert(LT.second.isFixedLengthVector());998 MVT ContainerVT = TLI->getContainerForFixedLengthVector(LT.second);999 if (isM1OrSmaller(ContainerVT)) {1000 InstructionCost BV =1001 cast<FixedVectorType>(Ty)->getNumElements() *1002 getRISCVInstructionCost(RISCV::VSLIDE1DOWN_VX, LT.second, CostKind);1003 if (BV < Cost)1004 Cost = BV;1005 }1006 }1007 return Cost;1008}1009 1010InstructionCost1011RISCVTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,1012 TTI::TargetCostKind CostKind) const {1013 Type *DataTy = MICA.getDataType();1014 Align Alignment = MICA.getAlignment();1015 switch (MICA.getID()) {1016 case Intrinsic::vp_load_ff: {1017 EVT DataTypeVT = TLI->getValueType(DL, DataTy);1018 if (!TLI->isLegalFirstFaultLoad(DataTypeVT, Alignment))1019 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);1020 1021 unsigned AS = MICA.getAddressSpace();1022 return getMemoryOpCost(Instruction::Load, DataTy, Alignment, AS, CostKind,1023 {TTI::OK_AnyValue, TTI::OP_None}, nullptr);1024 }1025 }1026 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);1027}1028 1029InstructionCost1030RISCVTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,1031 TTI::TargetCostKind CostKind) const {1032 unsigned Opcode = MICA.getID() == Intrinsic::masked_load ? Instruction::Load1033 : Instruction::Store;1034 Type *Src = MICA.getDataType();1035 Align Alignment = MICA.getAlignment();1036 unsigned AddressSpace = MICA.getAddressSpace();1037 1038 if (!isLegalMaskedLoadStore(Src, Alignment) ||1039 CostKind != TTI::TCK_RecipThroughput)1040 return BaseT::getMaskedMemoryOpCost(MICA, CostKind);1041 1042 return getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind);1043}1044 1045InstructionCost RISCVTTIImpl::getInterleavedMemoryOpCost(1046 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,1047 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,1048 bool UseMaskForCond, bool UseMaskForGaps) const {1049 1050 // The interleaved memory access pass will lower (de)interleave ops combined1051 // with an adjacent appropriate memory to vlseg/vsseg intrinsics. vlseg/vsseg1052 // only support masking per-iteration (i.e. condition), not per-segment (i.e.1053 // gap).1054 if (!UseMaskForGaps && Factor <= TLI->getMaxSupportedInterleaveFactor()) {1055 auto *VTy = cast<VectorType>(VecTy);1056 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(VTy);1057 // Need to make sure type has't been scalarized1058 if (LT.second.isVector()) {1059 auto *SubVecTy =1060 VectorType::get(VTy->getElementType(),1061 VTy->getElementCount().divideCoefficientBy(Factor));1062 if (VTy->getElementCount().isKnownMultipleOf(Factor) &&1063 TLI->isLegalInterleavedAccessType(SubVecTy, Factor, Alignment,1064 AddressSpace, DL)) {1065 1066 // Some processors optimize segment loads/stores as one wide memory op +1067 // Factor * LMUL shuffle ops.1068 if (ST->hasOptimizedSegmentLoadStore(Factor)) {1069 InstructionCost Cost =1070 getMemoryOpCost(Opcode, VTy, Alignment, AddressSpace, CostKind);1071 MVT SubVecVT = getTLI()->getValueType(DL, SubVecTy).getSimpleVT();1072 Cost += Factor * TLI->getLMULCost(SubVecVT);1073 return LT.first * Cost;1074 }1075 1076 // Otherwise, the cost is proportional to the number of elements (VL *1077 // Factor ops).1078 InstructionCost MemOpCost =1079 getMemoryOpCost(Opcode, VTy->getElementType(), Alignment, 0,1080 CostKind, {TTI::OK_AnyValue, TTI::OP_None});1081 unsigned NumLoads = getEstimatedVLFor(VTy);1082 return NumLoads * MemOpCost;1083 }1084 }1085 }1086 1087 // TODO: Return the cost of interleaved accesses for scalable vector when1088 // unable to convert to segment accesses instructions.1089 if (isa<ScalableVectorType>(VecTy))1090 return InstructionCost::getInvalid();1091 1092 auto *FVTy = cast<FixedVectorType>(VecTy);1093 InstructionCost MemCost =1094 getMemoryOpCost(Opcode, VecTy, Alignment, AddressSpace, CostKind);1095 unsigned VF = FVTy->getNumElements() / Factor;1096 1097 // An interleaved load will look like this for Factor=3:1098 // %wide.vec = load <12 x i32>, ptr %3, align 41099 // %strided.vec = shufflevector %wide.vec, poison, <4 x i32> <stride mask>1100 // %strided.vec1 = shufflevector %wide.vec, poison, <4 x i32> <stride mask>1101 // %strided.vec2 = shufflevector %wide.vec, poison, <4 x i32> <stride mask>1102 if (Opcode == Instruction::Load) {1103 InstructionCost Cost = MemCost;1104 for (unsigned Index : Indices) {1105 FixedVectorType *VecTy =1106 FixedVectorType::get(FVTy->getElementType(), VF * Factor);1107 auto Mask = createStrideMask(Index, Factor, VF);1108 Mask.resize(VF * Factor, -1);1109 InstructionCost ShuffleCost =1110 getShuffleCost(TTI::ShuffleKind::SK_PermuteSingleSrc, VecTy, VecTy,1111 Mask, CostKind, 0, nullptr, {});1112 Cost += ShuffleCost;1113 }1114 return Cost;1115 }1116 1117 // TODO: Model for NF > 21118 // We'll need to enhance getShuffleCost to model shuffles that are just1119 // inserts and extracts into subvectors, since they won't have the full cost1120 // of a vrgather.1121 // An interleaved store for 3 vectors of 4 lanes will look like1122 // %11 = shufflevector <4 x i32> %4, <4 x i32> %6, <8 x i32> <0...7>1123 // %12 = shufflevector <4 x i32> %9, <4 x i32> poison, <8 x i32> <0...3>1124 // %13 = shufflevector <8 x i32> %11, <8 x i32> %12, <12 x i32> <0...11>1125 // %interleaved.vec = shufflevector %13, poison, <12 x i32> <interleave mask>1126 // store <12 x i32> %interleaved.vec, ptr %10, align 41127 if (Factor != 2)1128 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,1129 Alignment, AddressSpace, CostKind,1130 UseMaskForCond, UseMaskForGaps);1131 1132 assert(Opcode == Instruction::Store && "Opcode must be a store");1133 // For an interleaving store of 2 vectors, we perform one large interleaving1134 // shuffle that goes into the wide store1135 auto Mask = createInterleaveMask(VF, Factor);1136 InstructionCost ShuffleCost =1137 getShuffleCost(TTI::ShuffleKind::SK_PermuteSingleSrc, FVTy, FVTy, Mask,1138 CostKind, 0, nullptr, {});1139 return MemCost + ShuffleCost;1140}1141 1142InstructionCost RISCVTTIImpl::getGatherScatterOpCost(1143 unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,1144 Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {1145 if (CostKind != TTI::TCK_RecipThroughput)1146 return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,1147 Alignment, CostKind, I);1148 1149 if ((Opcode == Instruction::Load &&1150 !isLegalMaskedGather(DataTy, Align(Alignment))) ||1151 (Opcode == Instruction::Store &&1152 !isLegalMaskedScatter(DataTy, Align(Alignment))))1153 return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,1154 Alignment, CostKind, I);1155 1156 // Cost is proportional to the number of memory operations implied. For1157 // scalable vectors, we use an estimate on that number since we don't1158 // know exactly what VL will be.1159 auto &VTy = *cast<VectorType>(DataTy);1160 InstructionCost MemOpCost =1161 getMemoryOpCost(Opcode, VTy.getElementType(), Alignment, 0, CostKind,1162 {TTI::OK_AnyValue, TTI::OP_None}, I);1163 unsigned NumLoads = getEstimatedVLFor(&VTy);1164 return NumLoads * MemOpCost;1165}1166 1167InstructionCost RISCVTTIImpl::getExpandCompressMemoryOpCost(1168 const MemIntrinsicCostAttributes &MICA,1169 TTI::TargetCostKind CostKind) const {1170 unsigned Opcode = MICA.getID() == Intrinsic::masked_expandload1171 ? Instruction::Load1172 : Instruction::Store;1173 Type *DataTy = MICA.getDataType();1174 bool VariableMask = MICA.getVariableMask();1175 Align Alignment = MICA.getAlignment();1176 bool IsLegal = (Opcode == Instruction::Store &&1177 isLegalMaskedCompressStore(DataTy, Alignment)) ||1178 (Opcode == Instruction::Load &&1179 isLegalMaskedExpandLoad(DataTy, Alignment));1180 if (!IsLegal || CostKind != TTI::TCK_RecipThroughput)1181 return BaseT::getExpandCompressMemoryOpCost(MICA, CostKind);1182 // Example compressstore sequence:1183 // vsetivli zero, 8, e32, m2, ta, ma (ignored)1184 // vcompress.vm v10, v8, v01185 // vcpop.m a1, v01186 // vsetvli zero, a1, e32, m2, ta, ma1187 // vse32.v v10, (a0)1188 // Example expandload sequence:1189 // vsetivli zero, 8, e8, mf2, ta, ma (ignored)1190 // vcpop.m a1, v01191 // vsetvli zero, a1, e32, m2, ta, ma1192 // vle32.v v10, (a0)1193 // vsetivli zero, 8, e32, m2, ta, ma1194 // viota.m v12, v01195 // vrgather.vv v8, v10, v12, v0.t1196 auto MemOpCost =1197 getMemoryOpCost(Opcode, DataTy, Alignment, /*AddressSpace*/ 0, CostKind);1198 auto LT = getTypeLegalizationCost(DataTy);1199 SmallVector<unsigned, 4> Opcodes{RISCV::VSETVLI};1200 if (VariableMask)1201 Opcodes.push_back(RISCV::VCPOP_M);1202 if (Opcode == Instruction::Store)1203 Opcodes.append({RISCV::VCOMPRESS_VM});1204 else1205 Opcodes.append({RISCV::VSETIVLI, RISCV::VIOTA_M, RISCV::VRGATHER_VV});1206 return MemOpCost +1207 LT.first * getRISCVInstructionCost(Opcodes, LT.second, CostKind);1208}1209 1210InstructionCost RISCVTTIImpl::getStridedMemoryOpCost(1211 unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,1212 Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {1213 if (((Opcode == Instruction::Load || Opcode == Instruction::Store) &&1214 !isLegalStridedLoadStore(DataTy, Alignment)) ||1215 (Opcode != Instruction::Load && Opcode != Instruction::Store))1216 return BaseT::getStridedMemoryOpCost(Opcode, DataTy, Ptr, VariableMask,1217 Alignment, CostKind, I);1218 1219 if (CostKind == TTI::TCK_CodeSize)1220 return TTI::TCC_Basic;1221 1222 // Cost is proportional to the number of memory operations implied. For1223 // scalable vectors, we use an estimate on that number since we don't1224 // know exactly what VL will be.1225 auto &VTy = *cast<VectorType>(DataTy);1226 InstructionCost MemOpCost =1227 getMemoryOpCost(Opcode, VTy.getElementType(), Alignment, 0, CostKind,1228 {TTI::OK_AnyValue, TTI::OP_None}, I);1229 unsigned NumLoads = getEstimatedVLFor(&VTy);1230 return NumLoads * MemOpCost;1231}1232 1233InstructionCost1234RISCVTTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {1235 // FIXME: This is a property of the default vector convention, not1236 // all possible calling conventions. Fixing that will require1237 // some TTI API and SLP rework.1238 InstructionCost Cost = 0;1239 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;1240 for (auto *Ty : Tys) {1241 if (!Ty->isVectorTy())1242 continue;1243 Align A = DL.getPrefTypeAlign(Ty);1244 Cost += getMemoryOpCost(Instruction::Store, Ty, A, 0, CostKind) +1245 getMemoryOpCost(Instruction::Load, Ty, A, 0, CostKind);1246 }1247 return Cost;1248}1249 1250// Currently, these represent both throughput and codesize costs1251// for the respective intrinsics. The costs in this table are simply1252// instruction counts with the following adjustments made:1253// * One vsetvli is considered free.1254static const CostTblEntry VectorIntrinsicCostTable[]{1255 {Intrinsic::floor, MVT::f32, 9},1256 {Intrinsic::floor, MVT::f64, 9},1257 {Intrinsic::ceil, MVT::f32, 9},1258 {Intrinsic::ceil, MVT::f64, 9},1259 {Intrinsic::trunc, MVT::f32, 7},1260 {Intrinsic::trunc, MVT::f64, 7},1261 {Intrinsic::round, MVT::f32, 9},1262 {Intrinsic::round, MVT::f64, 9},1263 {Intrinsic::roundeven, MVT::f32, 9},1264 {Intrinsic::roundeven, MVT::f64, 9},1265 {Intrinsic::rint, MVT::f32, 7},1266 {Intrinsic::rint, MVT::f64, 7},1267 {Intrinsic::nearbyint, MVT::f32, 9},1268 {Intrinsic::nearbyint, MVT::f64, 9},1269 {Intrinsic::bswap, MVT::i16, 3},1270 {Intrinsic::bswap, MVT::i32, 12},1271 {Intrinsic::bswap, MVT::i64, 31},1272 {Intrinsic::vp_bswap, MVT::i16, 3},1273 {Intrinsic::vp_bswap, MVT::i32, 12},1274 {Intrinsic::vp_bswap, MVT::i64, 31},1275 {Intrinsic::vp_fshl, MVT::i8, 7},1276 {Intrinsic::vp_fshl, MVT::i16, 7},1277 {Intrinsic::vp_fshl, MVT::i32, 7},1278 {Intrinsic::vp_fshl, MVT::i64, 7},1279 {Intrinsic::vp_fshr, MVT::i8, 7},1280 {Intrinsic::vp_fshr, MVT::i16, 7},1281 {Intrinsic::vp_fshr, MVT::i32, 7},1282 {Intrinsic::vp_fshr, MVT::i64, 7},1283 {Intrinsic::bitreverse, MVT::i8, 17},1284 {Intrinsic::bitreverse, MVT::i16, 24},1285 {Intrinsic::bitreverse, MVT::i32, 33},1286 {Intrinsic::bitreverse, MVT::i64, 52},1287 {Intrinsic::vp_bitreverse, MVT::i8, 17},1288 {Intrinsic::vp_bitreverse, MVT::i16, 24},1289 {Intrinsic::vp_bitreverse, MVT::i32, 33},1290 {Intrinsic::vp_bitreverse, MVT::i64, 52},1291 {Intrinsic::ctpop, MVT::i8, 12},1292 {Intrinsic::ctpop, MVT::i16, 19},1293 {Intrinsic::ctpop, MVT::i32, 20},1294 {Intrinsic::ctpop, MVT::i64, 21},1295 {Intrinsic::ctlz, MVT::i8, 19},1296 {Intrinsic::ctlz, MVT::i16, 28},1297 {Intrinsic::ctlz, MVT::i32, 31},1298 {Intrinsic::ctlz, MVT::i64, 35},1299 {Intrinsic::cttz, MVT::i8, 16},1300 {Intrinsic::cttz, MVT::i16, 23},1301 {Intrinsic::cttz, MVT::i32, 24},1302 {Intrinsic::cttz, MVT::i64, 25},1303 {Intrinsic::vp_ctpop, MVT::i8, 12},1304 {Intrinsic::vp_ctpop, MVT::i16, 19},1305 {Intrinsic::vp_ctpop, MVT::i32, 20},1306 {Intrinsic::vp_ctpop, MVT::i64, 21},1307 {Intrinsic::vp_ctlz, MVT::i8, 19},1308 {Intrinsic::vp_ctlz, MVT::i16, 28},1309 {Intrinsic::vp_ctlz, MVT::i32, 31},1310 {Intrinsic::vp_ctlz, MVT::i64, 35},1311 {Intrinsic::vp_cttz, MVT::i8, 16},1312 {Intrinsic::vp_cttz, MVT::i16, 23},1313 {Intrinsic::vp_cttz, MVT::i32, 24},1314 {Intrinsic::vp_cttz, MVT::i64, 25},1315};1316 1317InstructionCost1318RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,1319 TTI::TargetCostKind CostKind) const {1320 auto *RetTy = ICA.getReturnType();1321 switch (ICA.getID()) {1322 case Intrinsic::lrint:1323 case Intrinsic::llrint:1324 case Intrinsic::lround:1325 case Intrinsic::llround: {1326 auto LT = getTypeLegalizationCost(RetTy);1327 Type *SrcTy = ICA.getArgTypes().front();1328 auto SrcLT = getTypeLegalizationCost(SrcTy);1329 if (ST->hasVInstructions() && LT.second.isVector()) {1330 SmallVector<unsigned, 2> Ops;1331 unsigned SrcEltSz = DL.getTypeSizeInBits(SrcTy->getScalarType());1332 unsigned DstEltSz = DL.getTypeSizeInBits(RetTy->getScalarType());1333 if (LT.second.getVectorElementType() == MVT::bf16) {1334 if (!ST->hasVInstructionsBF16Minimal())1335 return InstructionCost::getInvalid();1336 if (DstEltSz == 32)1337 Ops = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFCVT_X_F_V};1338 else1339 Ops = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFWCVT_X_F_V};1340 } else if (LT.second.getVectorElementType() == MVT::f16 &&1341 !ST->hasVInstructionsF16()) {1342 if (!ST->hasVInstructionsF16Minimal())1343 return InstructionCost::getInvalid();1344 if (DstEltSz == 32)1345 Ops = {RISCV::VFWCVT_F_F_V, RISCV::VFCVT_X_F_V};1346 else1347 Ops = {RISCV::VFWCVT_F_F_V, RISCV::VFWCVT_X_F_V};1348 1349 } else if (SrcEltSz > DstEltSz) {1350 Ops = {RISCV::VFNCVT_X_F_W};1351 } else if (SrcEltSz < DstEltSz) {1352 Ops = {RISCV::VFWCVT_X_F_V};1353 } else {1354 Ops = {RISCV::VFCVT_X_F_V};1355 }1356 1357 // We need to use the source LMUL in the case of a narrowing op, and the1358 // destination LMUL otherwise.1359 if (SrcEltSz > DstEltSz)1360 return SrcLT.first *1361 getRISCVInstructionCost(Ops, SrcLT.second, CostKind);1362 return LT.first * getRISCVInstructionCost(Ops, LT.second, CostKind);1363 }1364 break;1365 }1366 case Intrinsic::ceil:1367 case Intrinsic::floor:1368 case Intrinsic::trunc:1369 case Intrinsic::rint:1370 case Intrinsic::round:1371 case Intrinsic::roundeven: {1372 // These all use the same code.1373 auto LT = getTypeLegalizationCost(RetTy);1374 if (!LT.second.isVector() && TLI->isOperationCustom(ISD::FCEIL, LT.second))1375 return LT.first * 8;1376 break;1377 }1378 case Intrinsic::umin:1379 case Intrinsic::umax:1380 case Intrinsic::smin:1381 case Intrinsic::smax: {1382 auto LT = getTypeLegalizationCost(RetTy);1383 if (LT.second.isScalarInteger() && ST->hasStdExtZbb())1384 return LT.first;1385 1386 if (ST->hasVInstructions() && LT.second.isVector()) {1387 unsigned Op;1388 switch (ICA.getID()) {1389 case Intrinsic::umin:1390 Op = RISCV::VMINU_VV;1391 break;1392 case Intrinsic::umax:1393 Op = RISCV::VMAXU_VV;1394 break;1395 case Intrinsic::smin:1396 Op = RISCV::VMIN_VV;1397 break;1398 case Intrinsic::smax:1399 Op = RISCV::VMAX_VV;1400 break;1401 }1402 return LT.first * getRISCVInstructionCost(Op, LT.second, CostKind);1403 }1404 break;1405 }1406 case Intrinsic::sadd_sat:1407 case Intrinsic::ssub_sat:1408 case Intrinsic::uadd_sat:1409 case Intrinsic::usub_sat: {1410 auto LT = getTypeLegalizationCost(RetTy);1411 if (ST->hasVInstructions() && LT.second.isVector()) {1412 unsigned Op;1413 switch (ICA.getID()) {1414 case Intrinsic::sadd_sat:1415 Op = RISCV::VSADD_VV;1416 break;1417 case Intrinsic::ssub_sat:1418 Op = RISCV::VSSUBU_VV;1419 break;1420 case Intrinsic::uadd_sat:1421 Op = RISCV::VSADDU_VV;1422 break;1423 case Intrinsic::usub_sat:1424 Op = RISCV::VSSUBU_VV;1425 break;1426 }1427 return LT.first * getRISCVInstructionCost(Op, LT.second, CostKind);1428 }1429 break;1430 }1431 case Intrinsic::fma:1432 case Intrinsic::fmuladd: {1433 // TODO: handle promotion with f16/bf16 with zvfhmin/zvfbfmin1434 auto LT = getTypeLegalizationCost(RetTy);1435 if (ST->hasVInstructions() && LT.second.isVector())1436 return LT.first *1437 getRISCVInstructionCost(RISCV::VFMADD_VV, LT.second, CostKind);1438 break;1439 }1440 case Intrinsic::fabs: {1441 auto LT = getTypeLegalizationCost(RetTy);1442 if (ST->hasVInstructions() && LT.second.isVector()) {1443 // lui a0, 81444 // addi a0, a0, -11445 // vsetvli a1, zero, e16, m1, ta, ma1446 // vand.vx v8, v8, a01447 // f16 with zvfhmin and bf16 with zvfhbmin1448 if (LT.second.getVectorElementType() == MVT::bf16 ||1449 (LT.second.getVectorElementType() == MVT::f16 &&1450 !ST->hasVInstructionsF16()))1451 return LT.first * getRISCVInstructionCost(RISCV::VAND_VX, LT.second,1452 CostKind) +1453 2;1454 else1455 return LT.first *1456 getRISCVInstructionCost(RISCV::VFSGNJX_VV, LT.second, CostKind);1457 }1458 break;1459 }1460 case Intrinsic::sqrt: {1461 auto LT = getTypeLegalizationCost(RetTy);1462 if (ST->hasVInstructions() && LT.second.isVector()) {1463 SmallVector<unsigned, 4> ConvOp;1464 SmallVector<unsigned, 2> FsqrtOp;1465 MVT ConvType = LT.second;1466 MVT FsqrtType = LT.second;1467 // f16 with zvfhmin and bf16 with zvfbfmin and the type of nxv32[b]f161468 // will be spilt.1469 if (LT.second.getVectorElementType() == MVT::bf16) {1470 if (LT.second == MVT::nxv32bf16) {1471 ConvOp = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFWCVTBF16_F_F_V,1472 RISCV::VFNCVTBF16_F_F_W, RISCV::VFNCVTBF16_F_F_W};1473 FsqrtOp = {RISCV::VFSQRT_V, RISCV::VFSQRT_V};1474 ConvType = MVT::nxv16f16;1475 FsqrtType = MVT::nxv16f32;1476 } else {1477 ConvOp = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFNCVTBF16_F_F_W};1478 FsqrtOp = {RISCV::VFSQRT_V};1479 FsqrtType = TLI->getTypeToPromoteTo(ISD::FSQRT, FsqrtType);1480 }1481 } else if (LT.second.getVectorElementType() == MVT::f16 &&1482 !ST->hasVInstructionsF16()) {1483 if (LT.second == MVT::nxv32f16) {1484 ConvOp = {RISCV::VFWCVT_F_F_V, RISCV::VFWCVT_F_F_V,1485 RISCV::VFNCVT_F_F_W, RISCV::VFNCVT_F_F_W};1486 FsqrtOp = {RISCV::VFSQRT_V, RISCV::VFSQRT_V};1487 ConvType = MVT::nxv16f16;1488 FsqrtType = MVT::nxv16f32;1489 } else {1490 ConvOp = {RISCV::VFWCVT_F_F_V, RISCV::VFNCVT_F_F_W};1491 FsqrtOp = {RISCV::VFSQRT_V};1492 FsqrtType = TLI->getTypeToPromoteTo(ISD::FSQRT, FsqrtType);1493 }1494 } else {1495 FsqrtOp = {RISCV::VFSQRT_V};1496 }1497 1498 return LT.first * (getRISCVInstructionCost(FsqrtOp, FsqrtType, CostKind) +1499 getRISCVInstructionCost(ConvOp, ConvType, CostKind));1500 }1501 break;1502 }1503 case Intrinsic::cttz:1504 case Intrinsic::ctlz:1505 case Intrinsic::ctpop: {1506 auto LT = getTypeLegalizationCost(RetTy);1507 if (ST->hasStdExtZvbb() && LT.second.isVector()) {1508 unsigned Op;1509 switch (ICA.getID()) {1510 case Intrinsic::cttz:1511 Op = RISCV::VCTZ_V;1512 break;1513 case Intrinsic::ctlz:1514 Op = RISCV::VCLZ_V;1515 break;1516 case Intrinsic::ctpop:1517 Op = RISCV::VCPOP_V;1518 break;1519 }1520 return LT.first * getRISCVInstructionCost(Op, LT.second, CostKind);1521 }1522 break;1523 }1524 case Intrinsic::abs: {1525 auto LT = getTypeLegalizationCost(RetTy);1526 if (ST->hasVInstructions() && LT.second.isVector()) {1527 // vrsub.vi v10, v8, 01528 // vmax.vv v8, v8, v101529 return LT.first *1530 getRISCVInstructionCost({RISCV::VRSUB_VI, RISCV::VMAX_VV},1531 LT.second, CostKind);1532 }1533 break;1534 }1535 case Intrinsic::get_active_lane_mask: {1536 if (ST->hasVInstructions()) {1537 Type *ExpRetTy = VectorType::get(1538 ICA.getArgTypes()[0], cast<VectorType>(RetTy)->getElementCount());1539 auto LT = getTypeLegalizationCost(ExpRetTy);1540 1541 // vid.v v8 // considered hoisted1542 // vsaddu.vx v8, v8, a01543 // vmsltu.vx v0, v8, a11544 return LT.first *1545 getRISCVInstructionCost({RISCV::VSADDU_VX, RISCV::VMSLTU_VX},1546 LT.second, CostKind);1547 }1548 break;1549 }1550 // TODO: add more intrinsic1551 case Intrinsic::stepvector: {1552 auto LT = getTypeLegalizationCost(RetTy);1553 // Legalisation of illegal types involves an `index' instruction plus1554 // (LT.first - 1) vector adds.1555 if (ST->hasVInstructions())1556 return getRISCVInstructionCost(RISCV::VID_V, LT.second, CostKind) +1557 (LT.first - 1) *1558 getRISCVInstructionCost(RISCV::VADD_VX, LT.second, CostKind);1559 return 1 + (LT.first - 1);1560 }1561 case Intrinsic::experimental_cttz_elts: {1562 Type *ArgTy = ICA.getArgTypes()[0];1563 EVT ArgType = TLI->getValueType(DL, ArgTy, true);1564 if (getTLI()->shouldExpandCttzElements(ArgType))1565 break;1566 InstructionCost Cost = getRISCVInstructionCost(1567 RISCV::VFIRST_M, getTypeLegalizationCost(ArgTy).second, CostKind);1568 1569 // If zero_is_poison is false, then we will generate additional1570 // cmp + select instructions to convert -1 to EVL.1571 Type *BoolTy = Type::getInt1Ty(RetTy->getContext());1572 if (ICA.getArgs().size() > 1 &&1573 cast<ConstantInt>(ICA.getArgs()[1])->isZero())1574 Cost += getCmpSelInstrCost(Instruction::ICmp, BoolTy, RetTy,1575 CmpInst::ICMP_SLT, CostKind) +1576 getCmpSelInstrCost(Instruction::Select, RetTy, BoolTy,1577 CmpInst::BAD_ICMP_PREDICATE, CostKind);1578 1579 return Cost;1580 }1581 case Intrinsic::experimental_vp_splat: {1582 auto LT = getTypeLegalizationCost(RetTy);1583 // TODO: Lower i1 experimental_vp_splat1584 if (!ST->hasVInstructions() || LT.second.getScalarType() == MVT::i1)1585 return InstructionCost::getInvalid();1586 return LT.first * getRISCVInstructionCost(LT.second.isFloatingPoint()1587 ? RISCV::VFMV_V_F1588 : RISCV::VMV_V_X,1589 LT.second, CostKind);1590 }1591 case Intrinsic::experimental_vp_splice: {1592 // To support type-based query from vectorizer, set the index to 0.1593 // Note that index only change the cost from vslide.vx to vslide.vi and in1594 // current implementations they have same costs.1595 return getShuffleCost(TTI::SK_Splice, cast<VectorType>(ICA.getReturnType()),1596 cast<VectorType>(ICA.getArgTypes()[0]), {}, CostKind,1597 0, cast<VectorType>(ICA.getReturnType()));1598 }1599 case Intrinsic::fptoui_sat:1600 case Intrinsic::fptosi_sat: {1601 InstructionCost Cost = 0;1602 bool IsSigned = ICA.getID() == Intrinsic::fptosi_sat;1603 Type *SrcTy = ICA.getArgTypes()[0];1604 1605 auto SrcLT = getTypeLegalizationCost(SrcTy);1606 auto DstLT = getTypeLegalizationCost(RetTy);1607 if (!SrcTy->isVectorTy())1608 break;1609 1610 if (!SrcLT.first.isValid() || !DstLT.first.isValid())1611 return InstructionCost::getInvalid();1612 1613 Cost +=1614 getCastInstrCost(IsSigned ? Instruction::FPToSI : Instruction::FPToUI,1615 RetTy, SrcTy, TTI::CastContextHint::None, CostKind);1616 1617 // Handle NaN.1618 // vmfne v0, v8, v8 # If v8[i] is NaN set v0[i] to 1.1619 // vmerge.vim v8, v8, 0, v0 # Convert NaN to 0.1620 Type *CondTy = RetTy->getWithNewBitWidth(1);1621 Cost += getCmpSelInstrCost(BinaryOperator::FCmp, SrcTy, CondTy,1622 CmpInst::FCMP_UNO, CostKind);1623 Cost += getCmpSelInstrCost(BinaryOperator::Select, RetTy, CondTy,1624 CmpInst::FCMP_UNO, CostKind);1625 return Cost;1626 }1627 }1628 1629 if (ST->hasVInstructions() && RetTy->isVectorTy()) {1630 if (auto LT = getTypeLegalizationCost(RetTy);1631 LT.second.isVector()) {1632 MVT EltTy = LT.second.getVectorElementType();1633 if (const auto *Entry = CostTableLookup(VectorIntrinsicCostTable,1634 ICA.getID(), EltTy))1635 return LT.first * Entry->Cost;1636 }1637 }1638 1639 return BaseT::getIntrinsicInstrCost(ICA, CostKind);1640}1641 1642InstructionCost1643RISCVTTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,1644 const SCEV *Ptr,1645 TTI::TargetCostKind CostKind) const {1646 // Address computations for vector indexed load/store likely require an offset1647 // and/or scaling.1648 if (ST->hasVInstructions() && PtrTy->isVectorTy())1649 return getArithmeticInstrCost(Instruction::Add, PtrTy, CostKind);1650 1651 return BaseT::getAddressComputationCost(PtrTy, SE, Ptr, CostKind);1652}1653 1654InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,1655 Type *Src,1656 TTI::CastContextHint CCH,1657 TTI::TargetCostKind CostKind,1658 const Instruction *I) const {1659 bool IsVectorType = isa<VectorType>(Dst) && isa<VectorType>(Src);1660 if (!IsVectorType)1661 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);1662 1663 // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)1664 // For now, skip all fixed vector cost analysis when P extension is available1665 // to avoid crashes in getMinRVVVectorSizeInBits()1666 if (ST->enablePExtCodeGen() &&1667 (isa<FixedVectorType>(Dst) || isa<FixedVectorType>(Src))) {1668 return 1; // Treat as single instruction cost for now1669 }1670 1671 // FIXME: Need to compute legalizing cost for illegal types. The current1672 // code handles only legal types and those which can be trivially1673 // promoted to legal.1674 if (!ST->hasVInstructions() || Src->getScalarSizeInBits() > ST->getELen() ||1675 Dst->getScalarSizeInBits() > ST->getELen())1676 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);1677 1678 int ISD = TLI->InstructionOpcodeToISD(Opcode);1679 assert(ISD && "Invalid opcode");1680 std::pair<InstructionCost, MVT> SrcLT = getTypeLegalizationCost(Src);1681 std::pair<InstructionCost, MVT> DstLT = getTypeLegalizationCost(Dst);1682 1683 // Handle i1 source and dest cases *before* calling logic in BasicTTI.1684 // The shared implementation doesn't model vector widening during legalization1685 // and instead assumes scalarization. In order to scalarize an <N x i1>1686 // vector, we need to extend/trunc to/from i8. If we don't special case1687 // this, we can get an infinite recursion cycle.1688 switch (ISD) {1689 default:1690 break;1691 case ISD::SIGN_EXTEND:1692 case ISD::ZERO_EXTEND:1693 if (Src->getScalarSizeInBits() == 1) {1694 // We do not use vsext/vzext to extend from mask vector.1695 // Instead we use the following instructions to extend from mask vector:1696 // vmv.v.i v8, 01697 // vmerge.vim v8, v8, -1, v0 (repeated per split)1698 return getRISCVInstructionCost(RISCV::VMV_V_I, DstLT.second, CostKind) +1699 DstLT.first * getRISCVInstructionCost(RISCV::VMERGE_VIM,1700 DstLT.second, CostKind) +1701 DstLT.first - 1;1702 }1703 break;1704 case ISD::TRUNCATE:1705 if (Dst->getScalarSizeInBits() == 1) {1706 // We do not use several vncvt to truncate to mask vector. So we could1707 // not use PowDiff to calculate it.1708 // Instead we use the following instructions to truncate to mask vector:1709 // vand.vi v8, v8, 11710 // vmsne.vi v0, v8, 01711 return SrcLT.first *1712 getRISCVInstructionCost({RISCV::VAND_VI, RISCV::VMSNE_VI},1713 SrcLT.second, CostKind) +1714 SrcLT.first - 1;1715 }1716 break;1717 };1718 1719 // Our actual lowering for the case where a wider legal type is available1720 // uses promotion to the wider type. This is reflected in the result of1721 // getTypeLegalizationCost, but BasicTTI assumes the widened cases are1722 // scalarized if the legalized Src and Dst are not equal sized.1723 const DataLayout &DL = this->getDataLayout();1724 if (!SrcLT.second.isVector() || !DstLT.second.isVector() ||1725 !SrcLT.first.isValid() || !DstLT.first.isValid() ||1726 !TypeSize::isKnownLE(DL.getTypeSizeInBits(Src),1727 SrcLT.second.getSizeInBits()) ||1728 !TypeSize::isKnownLE(DL.getTypeSizeInBits(Dst),1729 DstLT.second.getSizeInBits()) ||1730 SrcLT.first > 1 || DstLT.first > 1)1731 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);1732 1733 // The split cost is handled by the base getCastInstrCost1734 assert((SrcLT.first == 1) && (DstLT.first == 1) && "Illegal type");1735 1736 int PowDiff = (int)Log2_32(DstLT.second.getScalarSizeInBits()) -1737 (int)Log2_32(SrcLT.second.getScalarSizeInBits());1738 switch (ISD) {1739 case ISD::SIGN_EXTEND:1740 case ISD::ZERO_EXTEND: {1741 if ((PowDiff < 1) || (PowDiff > 3))1742 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);1743 unsigned SExtOp[] = {RISCV::VSEXT_VF2, RISCV::VSEXT_VF4, RISCV::VSEXT_VF8};1744 unsigned ZExtOp[] = {RISCV::VZEXT_VF2, RISCV::VZEXT_VF4, RISCV::VZEXT_VF8};1745 unsigned Op =1746 (ISD == ISD::SIGN_EXTEND) ? SExtOp[PowDiff - 1] : ZExtOp[PowDiff - 1];1747 return getRISCVInstructionCost(Op, DstLT.second, CostKind);1748 }1749 case ISD::TRUNCATE:1750 case ISD::FP_EXTEND:1751 case ISD::FP_ROUND: {1752 // Counts of narrow/widen instructions.1753 unsigned SrcEltSize = SrcLT.second.getScalarSizeInBits();1754 unsigned DstEltSize = DstLT.second.getScalarSizeInBits();1755 1756 unsigned Op = (ISD == ISD::TRUNCATE) ? RISCV::VNSRL_WI1757 : (ISD == ISD::FP_EXTEND) ? RISCV::VFWCVT_F_F_V1758 : RISCV::VFNCVT_F_F_W;1759 InstructionCost Cost = 0;1760 for (; SrcEltSize != DstEltSize;) {1761 MVT ElementMVT = (ISD == ISD::TRUNCATE)1762 ? MVT::getIntegerVT(DstEltSize)1763 : MVT::getFloatingPointVT(DstEltSize);1764 MVT DstMVT = DstLT.second.changeVectorElementType(ElementMVT);1765 DstEltSize =1766 (DstEltSize > SrcEltSize) ? DstEltSize >> 1 : DstEltSize << 1;1767 Cost += getRISCVInstructionCost(Op, DstMVT, CostKind);1768 }1769 return Cost;1770 }1771 case ISD::FP_TO_SINT:1772 case ISD::FP_TO_UINT: {1773 unsigned IsSigned = ISD == ISD::FP_TO_SINT;1774 unsigned FCVT = IsSigned ? RISCV::VFCVT_RTZ_X_F_V : RISCV::VFCVT_RTZ_XU_F_V;1775 unsigned FWCVT =1776 IsSigned ? RISCV::VFWCVT_RTZ_X_F_V : RISCV::VFWCVT_RTZ_XU_F_V;1777 unsigned FNCVT =1778 IsSigned ? RISCV::VFNCVT_RTZ_X_F_W : RISCV::VFNCVT_RTZ_XU_F_W;1779 unsigned SrcEltSize = Src->getScalarSizeInBits();1780 unsigned DstEltSize = Dst->getScalarSizeInBits();1781 InstructionCost Cost = 0;1782 if ((SrcEltSize == 16) &&1783 (!ST->hasVInstructionsF16() || ((DstEltSize / 2) > SrcEltSize))) {1784 // If the target only supports zvfhmin or it is fp16-to-i64 conversion1785 // pre-widening to f32 and then convert f32 to integer1786 VectorType *VecF32Ty =1787 VectorType::get(Type::getFloatTy(Dst->getContext()),1788 cast<VectorType>(Dst)->getElementCount());1789 std::pair<InstructionCost, MVT> VecF32LT =1790 getTypeLegalizationCost(VecF32Ty);1791 Cost +=1792 VecF32LT.first * getRISCVInstructionCost(RISCV::VFWCVT_F_F_V,1793 VecF32LT.second, CostKind);1794 Cost += getCastInstrCost(Opcode, Dst, VecF32Ty, CCH, CostKind, I);1795 return Cost;1796 }1797 if (DstEltSize == SrcEltSize)1798 Cost += getRISCVInstructionCost(FCVT, DstLT.second, CostKind);1799 else if (DstEltSize > SrcEltSize)1800 Cost += getRISCVInstructionCost(FWCVT, DstLT.second, CostKind);1801 else { // (SrcEltSize > DstEltSize)1802 // First do a narrowing conversion to an integer half the size, then1803 // truncate if needed.1804 MVT ElementVT = MVT::getIntegerVT(SrcEltSize / 2);1805 MVT VecVT = DstLT.second.changeVectorElementType(ElementVT);1806 Cost += getRISCVInstructionCost(FNCVT, VecVT, CostKind);1807 if ((SrcEltSize / 2) > DstEltSize) {1808 Type *VecTy = EVT(VecVT).getTypeForEVT(Dst->getContext());1809 Cost +=1810 getCastInstrCost(Instruction::Trunc, Dst, VecTy, CCH, CostKind, I);1811 }1812 }1813 return Cost;1814 }1815 case ISD::SINT_TO_FP:1816 case ISD::UINT_TO_FP: {1817 unsigned IsSigned = ISD == ISD::SINT_TO_FP;1818 unsigned FCVT = IsSigned ? RISCV::VFCVT_F_X_V : RISCV::VFCVT_F_XU_V;1819 unsigned FWCVT = IsSigned ? RISCV::VFWCVT_F_X_V : RISCV::VFWCVT_F_XU_V;1820 unsigned FNCVT = IsSigned ? RISCV::VFNCVT_F_X_W : RISCV::VFNCVT_F_XU_W;1821 unsigned SrcEltSize = Src->getScalarSizeInBits();1822 unsigned DstEltSize = Dst->getScalarSizeInBits();1823 1824 InstructionCost Cost = 0;1825 if ((DstEltSize == 16) &&1826 (!ST->hasVInstructionsF16() || ((SrcEltSize / 2) > DstEltSize))) {1827 // If the target only supports zvfhmin or it is i64-to-fp16 conversion1828 // it is converted to f32 and then converted to f161829 VectorType *VecF32Ty =1830 VectorType::get(Type::getFloatTy(Dst->getContext()),1831 cast<VectorType>(Dst)->getElementCount());1832 std::pair<InstructionCost, MVT> VecF32LT =1833 getTypeLegalizationCost(VecF32Ty);1834 Cost += getCastInstrCost(Opcode, VecF32Ty, Src, CCH, CostKind, I);1835 Cost += VecF32LT.first * getRISCVInstructionCost(RISCV::VFNCVT_F_F_W,1836 DstLT.second, CostKind);1837 return Cost;1838 }1839 1840 if (DstEltSize == SrcEltSize)1841 Cost += getRISCVInstructionCost(FCVT, DstLT.second, CostKind);1842 else if (DstEltSize > SrcEltSize) {1843 if ((DstEltSize / 2) > SrcEltSize) {1844 VectorType *VecTy =1845 VectorType::get(IntegerType::get(Dst->getContext(), DstEltSize / 2),1846 cast<VectorType>(Dst)->getElementCount());1847 unsigned Op = IsSigned ? Instruction::SExt : Instruction::ZExt;1848 Cost += getCastInstrCost(Op, VecTy, Src, CCH, CostKind, I);1849 }1850 Cost += getRISCVInstructionCost(FWCVT, DstLT.second, CostKind);1851 } else1852 Cost += getRISCVInstructionCost(FNCVT, DstLT.second, CostKind);1853 return Cost;1854 }1855 }1856 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);1857}1858 1859unsigned RISCVTTIImpl::getEstimatedVLFor(VectorType *Ty) const {1860 if (isa<ScalableVectorType>(Ty)) {1861 const unsigned EltSize = DL.getTypeSizeInBits(Ty->getElementType());1862 const unsigned MinSize = DL.getTypeSizeInBits(Ty).getKnownMinValue();1863 const unsigned VectorBits = *getVScaleForTuning() * RISCV::RVVBitsPerBlock;1864 return RISCVTargetLowering::computeVLMAX(VectorBits, EltSize, MinSize);1865 }1866 return cast<FixedVectorType>(Ty)->getNumElements();1867}1868 1869InstructionCost1870RISCVTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,1871 FastMathFlags FMF,1872 TTI::TargetCostKind CostKind) const {1873 if (isa<FixedVectorType>(Ty) && !ST->useRVVForFixedLengthVectors())1874 return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);1875 1876 // Skip if scalar size of Ty is bigger than ELEN.1877 if (Ty->getScalarSizeInBits() > ST->getELen())1878 return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);1879 1880 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);1881 if (Ty->getElementType()->isIntegerTy(1)) {1882 // SelectionDAGBuilder does following transforms:1883 // vector_reduce_{smin,umax}(<n x i1>) --> vector_reduce_or(<n x i1>)1884 // vector_reduce_{smax,umin}(<n x i1>) --> vector_reduce_and(<n x i1>)1885 if (IID == Intrinsic::umax || IID == Intrinsic::smin)1886 return getArithmeticReductionCost(Instruction::Or, Ty, FMF, CostKind);1887 else1888 return getArithmeticReductionCost(Instruction::And, Ty, FMF, CostKind);1889 }1890 1891 if (IID == Intrinsic::maximum || IID == Intrinsic::minimum) {1892 SmallVector<unsigned, 3> Opcodes;1893 InstructionCost ExtraCost = 0;1894 switch (IID) {1895 case Intrinsic::maximum:1896 if (FMF.noNaNs()) {1897 Opcodes = {RISCV::VFREDMAX_VS, RISCV::VFMV_F_S};1898 } else {1899 Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMAX_VS,1900 RISCV::VFMV_F_S};1901 // Cost of Canonical Nan + branch1902 // lui a0, 5232641903 // fmv.w.x fa0, a01904 Type *DstTy = Ty->getScalarType();1905 const unsigned EltTyBits = DstTy->getScalarSizeInBits();1906 Type *SrcTy = IntegerType::getIntNTy(DstTy->getContext(), EltTyBits);1907 ExtraCost = 1 +1908 getCastInstrCost(Instruction::UIToFP, DstTy, SrcTy,1909 TTI::CastContextHint::None, CostKind) +1910 getCFInstrCost(Instruction::Br, CostKind);1911 }1912 break;1913 1914 case Intrinsic::minimum:1915 if (FMF.noNaNs()) {1916 Opcodes = {RISCV::VFREDMIN_VS, RISCV::VFMV_F_S};1917 } else {1918 Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMIN_VS,1919 RISCV::VFMV_F_S};1920 // Cost of Canonical Nan + branch1921 // lui a0, 5232641922 // fmv.w.x fa0, a01923 Type *DstTy = Ty->getScalarType();1924 const unsigned EltTyBits = DL.getTypeSizeInBits(DstTy);1925 Type *SrcTy = IntegerType::getIntNTy(DstTy->getContext(), EltTyBits);1926 ExtraCost = 1 +1927 getCastInstrCost(Instruction::UIToFP, DstTy, SrcTy,1928 TTI::CastContextHint::None, CostKind) +1929 getCFInstrCost(Instruction::Br, CostKind);1930 }1931 break;1932 }1933 return ExtraCost + getRISCVInstructionCost(Opcodes, LT.second, CostKind);1934 }1935 1936 // IR Reduction is composed by one rvv reduction instruction and vmv1937 unsigned SplitOp;1938 SmallVector<unsigned, 3> Opcodes;1939 switch (IID) {1940 default:1941 llvm_unreachable("Unsupported intrinsic");1942 case Intrinsic::smax:1943 SplitOp = RISCV::VMAX_VV;1944 Opcodes = {RISCV::VREDMAX_VS, RISCV::VMV_X_S};1945 break;1946 case Intrinsic::smin:1947 SplitOp = RISCV::VMIN_VV;1948 Opcodes = {RISCV::VREDMIN_VS, RISCV::VMV_X_S};1949 break;1950 case Intrinsic::umax:1951 SplitOp = RISCV::VMAXU_VV;1952 Opcodes = {RISCV::VREDMAXU_VS, RISCV::VMV_X_S};1953 break;1954 case Intrinsic::umin:1955 SplitOp = RISCV::VMINU_VV;1956 Opcodes = {RISCV::VREDMINU_VS, RISCV::VMV_X_S};1957 break;1958 case Intrinsic::maxnum:1959 SplitOp = RISCV::VFMAX_VV;1960 Opcodes = {RISCV::VFREDMAX_VS, RISCV::VFMV_F_S};1961 break;1962 case Intrinsic::minnum:1963 SplitOp = RISCV::VFMIN_VV;1964 Opcodes = {RISCV::VFREDMIN_VS, RISCV::VFMV_F_S};1965 break;1966 }1967 // Add a cost for data larger than LMUL81968 InstructionCost SplitCost =1969 (LT.first > 1) ? (LT.first - 1) *1970 getRISCVInstructionCost(SplitOp, LT.second, CostKind)1971 : 0;1972 return SplitCost + getRISCVInstructionCost(Opcodes, LT.second, CostKind);1973}1974 1975InstructionCost1976RISCVTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,1977 std::optional<FastMathFlags> FMF,1978 TTI::TargetCostKind CostKind) const {1979 if (isa<FixedVectorType>(Ty) && !ST->useRVVForFixedLengthVectors())1980 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);1981 1982 // Skip if scalar size of Ty is bigger than ELEN.1983 if (Ty->getScalarSizeInBits() > ST->getELen())1984 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);1985 1986 int ISD = TLI->InstructionOpcodeToISD(Opcode);1987 assert(ISD && "Invalid opcode");1988 1989 if (ISD != ISD::ADD && ISD != ISD::OR && ISD != ISD::XOR && ISD != ISD::AND &&1990 ISD != ISD::FADD)1991 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);1992 1993 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);1994 Type *ElementTy = Ty->getElementType();1995 if (ElementTy->isIntegerTy(1)) {1996 // Example sequences:1997 // vfirst.m a0, v01998 // seqz a0, a01999 if (LT.second == MVT::v1i1)2000 return getRISCVInstructionCost(RISCV::VFIRST_M, LT.second, CostKind) +2001 getCmpSelInstrCost(Instruction::ICmp, ElementTy, ElementTy,2002 CmpInst::ICMP_EQ, CostKind);2003 2004 if (ISD == ISD::AND) {2005 // Example sequences:2006 // vmand.mm v8, v9, v8 ; needed every time type is split2007 // vmnot.m v8, v0 ; alias for vmnand2008 // vcpop.m a0, v82009 // seqz a0, a02010 2011 // See the discussion: https://github.com/llvm/llvm-project/pull/1191602012 // For LMUL <= 8, there is no splitting,2013 // the sequences are vmnot, vcpop and seqz.2014 // When LMUL > 8 and split = 1,2015 // the sequences are vmnand, vcpop and seqz.2016 // When LMUL > 8 and split > 1,2017 // the sequences are (LT.first-2) * vmand, vmnand, vcpop and seqz.2018 return ((LT.first > 2) ? (LT.first - 2) : 0) *2019 getRISCVInstructionCost(RISCV::VMAND_MM, LT.second, CostKind) +2020 getRISCVInstructionCost(RISCV::VMNAND_MM, LT.second, CostKind) +2021 getRISCVInstructionCost(RISCV::VCPOP_M, LT.second, CostKind) +2022 getCmpSelInstrCost(Instruction::ICmp, ElementTy, ElementTy,2023 CmpInst::ICMP_EQ, CostKind);2024 } else if (ISD == ISD::XOR || ISD == ISD::ADD) {2025 // Example sequences:2026 // vsetvli a0, zero, e8, mf8, ta, ma2027 // vmxor.mm v8, v0, v8 ; needed every time type is split2028 // vcpop.m a0, v82029 // andi a0, a0, 12030 return (LT.first - 1) *2031 getRISCVInstructionCost(RISCV::VMXOR_MM, LT.second, CostKind) +2032 getRISCVInstructionCost(RISCV::VCPOP_M, LT.second, CostKind) + 1;2033 } else {2034 assert(ISD == ISD::OR);2035 // Example sequences:2036 // vsetvli a0, zero, e8, mf8, ta, ma2037 // vmor.mm v8, v9, v8 ; needed every time type is split2038 // vcpop.m a0, v02039 // snez a0, a02040 return (LT.first - 1) *2041 getRISCVInstructionCost(RISCV::VMOR_MM, LT.second, CostKind) +2042 getRISCVInstructionCost(RISCV::VCPOP_M, LT.second, CostKind) +2043 getCmpSelInstrCost(Instruction::ICmp, ElementTy, ElementTy,2044 CmpInst::ICMP_NE, CostKind);2045 }2046 }2047 2048 // IR Reduction of or/and is composed by one vmv and one rvv reduction2049 // instruction, and others is composed by two vmv and one rvv reduction2050 // instruction2051 unsigned SplitOp;2052 SmallVector<unsigned, 3> Opcodes;2053 switch (ISD) {2054 case ISD::ADD:2055 SplitOp = RISCV::VADD_VV;2056 Opcodes = {RISCV::VMV_S_X, RISCV::VREDSUM_VS, RISCV::VMV_X_S};2057 break;2058 case ISD::OR:2059 SplitOp = RISCV::VOR_VV;2060 Opcodes = {RISCV::VREDOR_VS, RISCV::VMV_X_S};2061 break;2062 case ISD::XOR:2063 SplitOp = RISCV::VXOR_VV;2064 Opcodes = {RISCV::VMV_S_X, RISCV::VREDXOR_VS, RISCV::VMV_X_S};2065 break;2066 case ISD::AND:2067 SplitOp = RISCV::VAND_VV;2068 Opcodes = {RISCV::VREDAND_VS, RISCV::VMV_X_S};2069 break;2070 case ISD::FADD:2071 // We can't promote f16/bf16 fadd reductions.2072 if ((LT.second.getScalarType() == MVT::f16 && !ST->hasVInstructionsF16()) ||2073 LT.second.getScalarType() == MVT::bf16)2074 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);2075 if (TTI::requiresOrderedReduction(FMF)) {2076 Opcodes.push_back(RISCV::VFMV_S_F);2077 for (unsigned i = 0; i < LT.first.getValue(); i++)2078 Opcodes.push_back(RISCV::VFREDOSUM_VS);2079 Opcodes.push_back(RISCV::VFMV_F_S);2080 return getRISCVInstructionCost(Opcodes, LT.second, CostKind);2081 }2082 SplitOp = RISCV::VFADD_VV;2083 Opcodes = {RISCV::VFMV_S_F, RISCV::VFREDUSUM_VS, RISCV::VFMV_F_S};2084 break;2085 }2086 // Add a cost for data larger than LMUL82087 InstructionCost SplitCost =2088 (LT.first > 1) ? (LT.first - 1) *2089 getRISCVInstructionCost(SplitOp, LT.second, CostKind)2090 : 0;2091 return SplitCost + getRISCVInstructionCost(Opcodes, LT.second, CostKind);2092}2093 2094InstructionCost RISCVTTIImpl::getExtendedReductionCost(2095 unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *ValTy,2096 std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) const {2097 if (isa<FixedVectorType>(ValTy) && !ST->useRVVForFixedLengthVectors())2098 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, ValTy,2099 FMF, CostKind);2100 2101 // Skip if scalar size of ResTy is bigger than ELEN.2102 if (ResTy->getScalarSizeInBits() > ST->getELen())2103 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, ValTy,2104 FMF, CostKind);2105 2106 if (Opcode != Instruction::Add && Opcode != Instruction::FAdd)2107 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, ValTy,2108 FMF, CostKind);2109 2110 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy);2111 2112 if (IsUnsigned && Opcode == Instruction::Add &&2113 LT.second.isFixedLengthVector() && LT.second.getScalarType() == MVT::i1) {2114 // Represent vector_reduce_add(ZExt(<n x i1>)) as2115 // ZExtOrTrunc(ctpop(bitcast <n x i1> to in)).2116 return LT.first *2117 getRISCVInstructionCost(RISCV::VCPOP_M, LT.second, CostKind);2118 }2119 2120 if (ResTy->getScalarSizeInBits() != 2 * LT.second.getScalarSizeInBits())2121 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, ValTy,2122 FMF, CostKind);2123 2124 return (LT.first - 1) +2125 getArithmeticReductionCost(Opcode, ValTy, FMF, CostKind);2126}2127 2128InstructionCost2129RISCVTTIImpl::getStoreImmCost(Type *Ty, TTI::OperandValueInfo OpInfo,2130 TTI::TargetCostKind CostKind) const {2131 assert(OpInfo.isConstant() && "non constant operand?");2132 if (!isa<VectorType>(Ty))2133 // FIXME: We need to account for immediate materialization here, but doing2134 // a decent job requires more knowledge about the immediate than we2135 // currently have here.2136 return 0;2137 2138 if (OpInfo.isUniform())2139 // vmv.v.i, vmv.v.x, or vfmv.v.f2140 // We ignore the cost of the scalar constant materialization to be consistent2141 // with how we treat scalar constants themselves just above.2142 return 1;2143 2144 return getConstantPoolLoadCost(Ty, CostKind);2145}2146 2147InstructionCost RISCVTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,2148 Align Alignment,2149 unsigned AddressSpace,2150 TTI::TargetCostKind CostKind,2151 TTI::OperandValueInfo OpInfo,2152 const Instruction *I) const {2153 EVT VT = TLI->getValueType(DL, Src, true);2154 // Type legalization can't handle structs2155 if (VT == MVT::Other)2156 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,2157 CostKind, OpInfo, I);2158 2159 InstructionCost Cost = 0;2160 if (Opcode == Instruction::Store && OpInfo.isConstant())2161 Cost += getStoreImmCost(Src, OpInfo, CostKind);2162 2163 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Src);2164 2165 InstructionCost BaseCost = [&]() {2166 InstructionCost Cost = LT.first;2167 if (CostKind != TTI::TCK_RecipThroughput)2168 return Cost;2169 2170 // Our actual lowering for the case where a wider legal type is available2171 // uses the a VL predicated load on the wider type. This is reflected in2172 // the result of getTypeLegalizationCost, but BasicTTI assumes the2173 // widened cases are scalarized.2174 const DataLayout &DL = this->getDataLayout();2175 if (Src->isVectorTy() && LT.second.isVector() &&2176 TypeSize::isKnownLT(DL.getTypeStoreSizeInBits(Src),2177 LT.second.getSizeInBits()))2178 return Cost;2179 2180 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,2181 CostKind, OpInfo, I);2182 }();2183 2184 // Assume memory ops cost scale with the number of vector registers2185 // possible accessed by the instruction. Note that BasicTTI already2186 // handles the LT.first term for us.2187 if (ST->hasVInstructions() && LT.second.isVector() &&2188 CostKind != TTI::TCK_CodeSize)2189 BaseCost *= TLI->getLMULCost(LT.second);2190 return Cost + BaseCost;2191}2192 2193InstructionCost RISCVTTIImpl::getCmpSelInstrCost(2194 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,2195 TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,2196 TTI::OperandValueInfo Op2Info, const Instruction *I) const {2197 if (CostKind != TTI::TCK_RecipThroughput)2198 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,2199 Op1Info, Op2Info, I);2200 2201 if (isa<FixedVectorType>(ValTy) && !ST->useRVVForFixedLengthVectors())2202 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,2203 Op1Info, Op2Info, I);2204 2205 // Skip if scalar size of ValTy is bigger than ELEN.2206 if (ValTy->isVectorTy() && ValTy->getScalarSizeInBits() > ST->getELen())2207 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,2208 Op1Info, Op2Info, I);2209 2210 auto GetConstantMatCost =2211 [&](TTI::OperandValueInfo OpInfo) -> InstructionCost {2212 if (OpInfo.isUniform())2213 // We return 0 we currently ignore the cost of materializing scalar2214 // constants in GPRs.2215 return 0;2216 2217 return getConstantPoolLoadCost(ValTy, CostKind);2218 };2219 2220 InstructionCost ConstantMatCost;2221 if (Op1Info.isConstant())2222 ConstantMatCost += GetConstantMatCost(Op1Info);2223 if (Op2Info.isConstant())2224 ConstantMatCost += GetConstantMatCost(Op2Info);2225 2226 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy);2227 if (Opcode == Instruction::Select && ValTy->isVectorTy()) {2228 if (CondTy->isVectorTy()) {2229 if (ValTy->getScalarSizeInBits() == 1) {2230 // vmandn.mm v8, v8, v92231 // vmand.mm v9, v0, v92232 // vmor.mm v0, v9, v82233 return ConstantMatCost +2234 LT.first *2235 getRISCVInstructionCost(2236 {RISCV::VMANDN_MM, RISCV::VMAND_MM, RISCV::VMOR_MM},2237 LT.second, CostKind);2238 }2239 // vselect and max/min are supported natively.2240 return ConstantMatCost +2241 LT.first * getRISCVInstructionCost(RISCV::VMERGE_VVM, LT.second,2242 CostKind);2243 }2244 2245 if (ValTy->getScalarSizeInBits() == 1) {2246 // vmv.v.x v9, a02247 // vmsne.vi v9, v9, 02248 // vmandn.mm v8, v8, v92249 // vmand.mm v9, v0, v92250 // vmor.mm v0, v9, v82251 MVT InterimVT = LT.second.changeVectorElementType(MVT::i8);2252 return ConstantMatCost +2253 LT.first *2254 getRISCVInstructionCost({RISCV::VMV_V_X, RISCV::VMSNE_VI},2255 InterimVT, CostKind) +2256 LT.first * getRISCVInstructionCost(2257 {RISCV::VMANDN_MM, RISCV::VMAND_MM, RISCV::VMOR_MM},2258 LT.second, CostKind);2259 }2260 2261 // vmv.v.x v10, a02262 // vmsne.vi v0, v10, 02263 // vmerge.vvm v8, v9, v8, v02264 return ConstantMatCost +2265 LT.first * getRISCVInstructionCost(2266 {RISCV::VMV_V_X, RISCV::VMSNE_VI, RISCV::VMERGE_VVM},2267 LT.second, CostKind);2268 }2269 2270 if ((Opcode == Instruction::ICmp) && ValTy->isVectorTy() &&2271 CmpInst::isIntPredicate(VecPred)) {2272 // Use VMSLT_VV to represent VMSEQ, VMSNE, VMSLTU, VMSLEU, VMSLT, VMSLE2273 // provided they incur the same cost across all implementations2274 return ConstantMatCost + LT.first * getRISCVInstructionCost(RISCV::VMSLT_VV,2275 LT.second,2276 CostKind);2277 }2278 2279 if ((Opcode == Instruction::FCmp) && ValTy->isVectorTy() &&2280 CmpInst::isFPPredicate(VecPred)) {2281 2282 // Use VMXOR_MM and VMXNOR_MM to generate all true/false mask2283 if ((VecPred == CmpInst::FCMP_FALSE) || (VecPred == CmpInst::FCMP_TRUE))2284 return ConstantMatCost +2285 getRISCVInstructionCost(RISCV::VMXOR_MM, LT.second, CostKind);2286 2287 // If we do not support the input floating point vector type, use the base2288 // one which will calculate as:2289 // ScalarizeCost + Num * Cost for fixed vector,2290 // InvalidCost for scalable vector.2291 if ((ValTy->getScalarSizeInBits() == 16 && !ST->hasVInstructionsF16()) ||2292 (ValTy->getScalarSizeInBits() == 32 && !ST->hasVInstructionsF32()) ||2293 (ValTy->getScalarSizeInBits() == 64 && !ST->hasVInstructionsF64()))2294 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,2295 Op1Info, Op2Info, I);2296 2297 // Assuming vector fp compare and mask instructions are all the same cost2298 // until a need arises to differentiate them.2299 switch (VecPred) {2300 case CmpInst::FCMP_ONE: // vmflt.vv + vmflt.vv + vmor.mm2301 case CmpInst::FCMP_ORD: // vmfeq.vv + vmfeq.vv + vmand.mm2302 case CmpInst::FCMP_UNO: // vmfne.vv + vmfne.vv + vmor.mm2303 case CmpInst::FCMP_UEQ: // vmflt.vv + vmflt.vv + vmnor.mm2304 return ConstantMatCost +2305 LT.first * getRISCVInstructionCost(2306 {RISCV::VMFLT_VV, RISCV::VMFLT_VV, RISCV::VMOR_MM},2307 LT.second, CostKind);2308 2309 case CmpInst::FCMP_UGT: // vmfle.vv + vmnot.m2310 case CmpInst::FCMP_UGE: // vmflt.vv + vmnot.m2311 case CmpInst::FCMP_ULT: // vmfle.vv + vmnot.m2312 case CmpInst::FCMP_ULE: // vmflt.vv + vmnot.m2313 return ConstantMatCost +2314 LT.first *2315 getRISCVInstructionCost({RISCV::VMFLT_VV, RISCV::VMNAND_MM},2316 LT.second, CostKind);2317 2318 case CmpInst::FCMP_OEQ: // vmfeq.vv2319 case CmpInst::FCMP_OGT: // vmflt.vv2320 case CmpInst::FCMP_OGE: // vmfle.vv2321 case CmpInst::FCMP_OLT: // vmflt.vv2322 case CmpInst::FCMP_OLE: // vmfle.vv2323 case CmpInst::FCMP_UNE: // vmfne.vv2324 return ConstantMatCost +2325 LT.first *2326 getRISCVInstructionCost(RISCV::VMFLT_VV, LT.second, CostKind);2327 default:2328 break;2329 }2330 }2331 2332 // With ShortForwardBranchOpt or ConditionalMoveFusion, scalar icmp + select2333 // instructions will lower to SELECT_CC and lower to PseudoCCMOVGPR which will2334 // generate a conditional branch + mv. The cost of scalar (icmp + select) will2335 // be (0 + select instr cost).2336 if (ST->hasConditionalMoveFusion() && I && isa<ICmpInst>(I) &&2337 ValTy->isIntegerTy() && !I->user_empty()) {2338 if (all_of(I->users(), [&](const User *U) {2339 return match(U, m_Select(m_Specific(I), m_Value(), m_Value())) &&2340 U->getType()->isIntegerTy() &&2341 !isa<ConstantData>(U->getOperand(1)) &&2342 !isa<ConstantData>(U->getOperand(2));2343 }))2344 return 0;2345 }2346 2347 // TODO: Add cost for scalar type.2348 2349 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,2350 Op1Info, Op2Info, I);2351}2352 2353InstructionCost RISCVTTIImpl::getCFInstrCost(unsigned Opcode,2354 TTI::TargetCostKind CostKind,2355 const Instruction *I) const {2356 if (CostKind != TTI::TCK_RecipThroughput)2357 return Opcode == Instruction::PHI ? 0 : 1;2358 // Branches are assumed to be predicted.2359 return 0;2360}2361 2362InstructionCost RISCVTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,2363 TTI::TargetCostKind CostKind,2364 unsigned Index,2365 const Value *Op0,2366 const Value *Op1) const {2367 assert(Val->isVectorTy() && "This must be a vector type");2368 2369 // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)2370 // For now, skip all fixed vector cost analysis when P extension is available2371 // to avoid crashes in getMinRVVVectorSizeInBits()2372 if (ST->enablePExtCodeGen() && isa<FixedVectorType>(Val)) {2373 return 1; // Treat as single instruction cost for now2374 }2375 2376 if (Opcode != Instruction::ExtractElement &&2377 Opcode != Instruction::InsertElement)2378 return BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1);2379 2380 // Legalize the type.2381 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Val);2382 2383 // This type is legalized to a scalar type.2384 if (!LT.second.isVector()) {2385 auto *FixedVecTy = cast<FixedVectorType>(Val);2386 // If Index is a known constant, cost is zero.2387 if (Index != -1U)2388 return 0;2389 // Extract/InsertElement with non-constant index is very costly when2390 // scalarized; estimate cost of loads/stores sequence via the stack:2391 // ExtractElement cost: store vector to stack, load scalar;2392 // InsertElement cost: store vector to stack, store scalar, load vector.2393 Type *ElemTy = FixedVecTy->getElementType();2394 auto NumElems = FixedVecTy->getNumElements();2395 auto Align = DL.getPrefTypeAlign(ElemTy);2396 InstructionCost LoadCost =2397 getMemoryOpCost(Instruction::Load, ElemTy, Align, 0, CostKind);2398 InstructionCost StoreCost =2399 getMemoryOpCost(Instruction::Store, ElemTy, Align, 0, CostKind);2400 return Opcode == Instruction::ExtractElement2401 ? StoreCost * NumElems + LoadCost2402 : (StoreCost + LoadCost) * NumElems + StoreCost;2403 }2404 2405 // For unsupported scalable vector.2406 if (LT.second.isScalableVector() && !LT.first.isValid())2407 return LT.first;2408 2409 // Mask vector extract/insert is expanded via e8.2410 if (Val->getScalarSizeInBits() == 1) {2411 VectorType *WideTy =2412 VectorType::get(IntegerType::get(Val->getContext(), 8),2413 cast<VectorType>(Val)->getElementCount());2414 if (Opcode == Instruction::ExtractElement) {2415 InstructionCost ExtendCost2416 = getCastInstrCost(Instruction::ZExt, WideTy, Val,2417 TTI::CastContextHint::None, CostKind);2418 InstructionCost ExtractCost2419 = getVectorInstrCost(Opcode, WideTy, CostKind, Index, nullptr, nullptr);2420 return ExtendCost + ExtractCost;2421 }2422 InstructionCost ExtendCost2423 = getCastInstrCost(Instruction::ZExt, WideTy, Val,2424 TTI::CastContextHint::None, CostKind);2425 InstructionCost InsertCost2426 = getVectorInstrCost(Opcode, WideTy, CostKind, Index, nullptr, nullptr);2427 InstructionCost TruncCost2428 = getCastInstrCost(Instruction::Trunc, Val, WideTy,2429 TTI::CastContextHint::None, CostKind);2430 return ExtendCost + InsertCost + TruncCost;2431 }2432 2433 2434 // In RVV, we could use vslidedown + vmv.x.s to extract element from vector2435 // and vslideup + vmv.s.x to insert element to vector.2436 unsigned BaseCost = 1;2437 // When insertelement we should add the index with 1 as the input of vslideup.2438 unsigned SlideCost = Opcode == Instruction::InsertElement ? 2 : 1;2439 2440 if (Index != -1U) {2441 // The type may be split. For fixed-width vectors we can normalize the2442 // index to the new type.2443 if (LT.second.isFixedLengthVector()) {2444 unsigned Width = LT.second.getVectorNumElements();2445 Index = Index % Width;2446 }2447 2448 // If exact VLEN is known, we will insert/extract into the appropriate2449 // subvector with no additional subvector insert/extract cost.2450 if (auto VLEN = ST->getRealVLen()) {2451 unsigned EltSize = LT.second.getScalarSizeInBits();2452 unsigned M1Max = *VLEN / EltSize;2453 Index = Index % M1Max;2454 }2455 2456 if (Index == 0)2457 // We can extract/insert the first element without vslidedown/vslideup.2458 SlideCost = 0;2459 else if (ST->hasVendorXRivosVisni() && isUInt<5>(Index) &&2460 Val->getScalarType()->isIntegerTy())2461 SlideCost = 0; // With ri.vinsert/ri.vextract there is no slide needed2462 else if (Opcode == Instruction::InsertElement)2463 SlideCost = 1; // With a constant index, we do not need to use addi.2464 }2465 2466 // When the vector needs to split into multiple register groups and the index2467 // exceeds single vector register group, we need to insert/extract the element2468 // via stack.2469 if (LT.first > 1 &&2470 ((Index == -1U) || (Index >= LT.second.getVectorMinNumElements() &&2471 LT.second.isScalableVector()))) {2472 Type *ScalarType = Val->getScalarType();2473 Align VecAlign = DL.getPrefTypeAlign(Val);2474 Align SclAlign = DL.getPrefTypeAlign(ScalarType);2475 // Extra addi for unknown index.2476 InstructionCost IdxCost = Index == -1U ? 1 : 0;2477 2478 // Store all split vectors into stack and load the target element.2479 if (Opcode == Instruction::ExtractElement)2480 return getMemoryOpCost(Instruction::Store, Val, VecAlign, 0, CostKind) +2481 getMemoryOpCost(Instruction::Load, ScalarType, SclAlign, 0,2482 CostKind) +2483 IdxCost;2484 2485 // Store all split vectors into stack and store the target element and load2486 // vectors back.2487 return getMemoryOpCost(Instruction::Store, Val, VecAlign, 0, CostKind) +2488 getMemoryOpCost(Instruction::Load, Val, VecAlign, 0, CostKind) +2489 getMemoryOpCost(Instruction::Store, ScalarType, SclAlign, 0,2490 CostKind) +2491 IdxCost;2492 }2493 2494 // Extract i64 in the target that has XLEN=32 need more instruction.2495 if (Val->getScalarType()->isIntegerTy() &&2496 ST->getXLen() < Val->getScalarSizeInBits()) {2497 // For extractelement, we need the following instructions:2498 // vsetivli zero, 1, e64, m1, ta, mu (not count)2499 // vslidedown.vx v8, v8, a02500 // vmv.x.s a0, v82501 // li a1, 322502 // vsrl.vx v8, v8, a12503 // vmv.x.s a1, v82504 2505 // For insertelement, we need the following instructions:2506 // vsetivli zero, 2, e32, m4, ta, mu (not count)2507 // vmv.v.i v12, 02508 // vslide1up.vx v16, v12, a12509 // vslide1up.vx v12, v16, a02510 // addi a0, a2, 12511 // vsetvli zero, a0, e64, m4, tu, mu (not count)2512 // vslideup.vx v8, v12, a22513 2514 // TODO: should we count these special vsetvlis?2515 BaseCost = Opcode == Instruction::InsertElement ? 3 : 4;2516 }2517 return BaseCost + SlideCost;2518}2519 2520InstructionCost2521RISCVTTIImpl::getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Val,2522 TTI::TargetCostKind CostKind,2523 unsigned Index) const {2524 if (isa<FixedVectorType>(Val))2525 return BaseT::getIndexedVectorInstrCostFromEnd(Opcode, Val, CostKind,2526 Index);2527 2528 // TODO: This code replicates what LoopVectorize.cpp used to do when asking2529 // for the cost of extracting the last lane of a scalable vector. It probably2530 // needs a more accurate cost.2531 ElementCount EC = cast<VectorType>(Val)->getElementCount();2532 assert(Index < EC.getKnownMinValue() && "Unexpected reverse index");2533 return getVectorInstrCost(Opcode, Val, CostKind,2534 EC.getKnownMinValue() - 1 - Index, nullptr,2535 nullptr);2536}2537 2538InstructionCost RISCVTTIImpl::getArithmeticInstrCost(2539 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,2540 TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,2541 ArrayRef<const Value *> Args, const Instruction *CxtI) const {2542 2543 // TODO: Handle more cost kinds.2544 if (CostKind != TTI::TCK_RecipThroughput)2545 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,2546 Args, CxtI);2547 2548 if (isa<FixedVectorType>(Ty) && !ST->useRVVForFixedLengthVectors())2549 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,2550 Args, CxtI);2551 2552 // Skip if scalar size of Ty is bigger than ELEN.2553 if (isa<VectorType>(Ty) && Ty->getScalarSizeInBits() > ST->getELen())2554 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,2555 Args, CxtI);2556 2557 // Legalize the type.2558 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);2559 2560 // TODO: Handle scalar type.2561 if (!LT.second.isVector())2562 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,2563 Args, CxtI);2564 2565 // f16 with zvfhmin and bf16 will be promoted to f32.2566 // FIXME: nxv32[b]f16 will be custom lowered and split.2567 unsigned ISDOpcode = TLI->InstructionOpcodeToISD(Opcode);2568 InstructionCost CastCost = 0;2569 if ((LT.second.getVectorElementType() == MVT::f16 ||2570 LT.second.getVectorElementType() == MVT::bf16) &&2571 TLI->getOperationAction(ISDOpcode, LT.second) ==2572 TargetLoweringBase::LegalizeAction::Promote) {2573 MVT PromotedVT = TLI->getTypeToPromoteTo(ISDOpcode, LT.second);2574 Type *PromotedTy = EVT(PromotedVT).getTypeForEVT(Ty->getContext());2575 Type *LegalTy = EVT(LT.second).getTypeForEVT(Ty->getContext());2576 // Add cost of extending arguments2577 CastCost += LT.first * Args.size() *2578 getCastInstrCost(Instruction::FPExt, PromotedTy, LegalTy,2579 TTI::CastContextHint::None, CostKind);2580 // Add cost of truncating result2581 CastCost +=2582 LT.first * getCastInstrCost(Instruction::FPTrunc, LegalTy, PromotedTy,2583 TTI::CastContextHint::None, CostKind);2584 // Compute cost of op in promoted type2585 LT.second = PromotedVT;2586 }2587 2588 auto getConstantMatCost =2589 [&](unsigned Operand, TTI::OperandValueInfo OpInfo) -> InstructionCost {2590 if (OpInfo.isUniform() && canSplatOperand(Opcode, Operand))2591 // Two sub-cases:2592 // * Has a 5 bit immediate operand which can be splatted.2593 // * Has a larger immediate which must be materialized in scalar register2594 // We return 0 for both as we currently ignore the cost of materializing2595 // scalar constants in GPRs.2596 return 0;2597 2598 return getConstantPoolLoadCost(Ty, CostKind);2599 };2600 2601 // Add the cost of materializing any constant vectors required.2602 InstructionCost ConstantMatCost = 0;2603 if (Op1Info.isConstant())2604 ConstantMatCost += getConstantMatCost(0, Op1Info);2605 if (Op2Info.isConstant())2606 ConstantMatCost += getConstantMatCost(1, Op2Info);2607 2608 unsigned Op;2609 switch (ISDOpcode) {2610 case ISD::ADD:2611 case ISD::SUB:2612 Op = RISCV::VADD_VV;2613 break;2614 case ISD::SHL:2615 case ISD::SRL:2616 case ISD::SRA:2617 Op = RISCV::VSLL_VV;2618 break;2619 case ISD::AND:2620 case ISD::OR:2621 case ISD::XOR:2622 Op = (Ty->getScalarSizeInBits() == 1) ? RISCV::VMAND_MM : RISCV::VAND_VV;2623 break;2624 case ISD::MUL:2625 case ISD::MULHS:2626 case ISD::MULHU:2627 Op = RISCV::VMUL_VV;2628 break;2629 case ISD::SDIV:2630 case ISD::UDIV:2631 Op = RISCV::VDIV_VV;2632 break;2633 case ISD::SREM:2634 case ISD::UREM:2635 Op = RISCV::VREM_VV;2636 break;2637 case ISD::FADD:2638 case ISD::FSUB:2639 Op = RISCV::VFADD_VV;2640 break;2641 case ISD::FMUL:2642 Op = RISCV::VFMUL_VV;2643 break;2644 case ISD::FDIV:2645 Op = RISCV::VFDIV_VV;2646 break;2647 case ISD::FNEG:2648 Op = RISCV::VFSGNJN_VV;2649 break;2650 default:2651 // Assuming all other instructions have the same cost until a need arises to2652 // differentiate them.2653 return CastCost + ConstantMatCost +2654 BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,2655 Args, CxtI);2656 }2657 2658 InstructionCost InstrCost = getRISCVInstructionCost(Op, LT.second, CostKind);2659 // We use BasicTTIImpl to calculate scalar costs, which assumes floating point2660 // ops are twice as expensive as integer ops. Do the same for vectors so2661 // scalar floating point ops aren't cheaper than their vector equivalents.2662 if (Ty->isFPOrFPVectorTy())2663 InstrCost *= 2;2664 return CastCost + ConstantMatCost + LT.first * InstrCost;2665}2666 2667// TODO: Deduplicate from TargetTransformInfoImplCRTPBase.2668InstructionCost RISCVTTIImpl::getPointersChainCost(2669 ArrayRef<const Value *> Ptrs, const Value *Base,2670 const TTI::PointersChainInfo &Info, Type *AccessTy,2671 TTI::TargetCostKind CostKind) const {2672 InstructionCost Cost = TTI::TCC_Free;2673 // In the basic model we take into account GEP instructions only2674 // (although here can come alloca instruction, a value, constants and/or2675 // constant expressions, PHIs, bitcasts ... whatever allowed to be used as a2676 // pointer). Typically, if Base is a not a GEP-instruction and all the2677 // pointers are relative to the same base address, all the rest are2678 // either GEP instructions, PHIs, bitcasts or constants. When we have same2679 // base, we just calculate cost of each non-Base GEP as an ADD operation if2680 // any their index is a non-const.2681 // If no known dependencies between the pointers cost is calculated as a sum2682 // of costs of GEP instructions.2683 for (auto [I, V] : enumerate(Ptrs)) {2684 const auto *GEP = dyn_cast<GetElementPtrInst>(V);2685 if (!GEP)2686 continue;2687 if (Info.isSameBase() && V != Base) {2688 if (GEP->hasAllConstantIndices())2689 continue;2690 // If the chain is unit-stride and BaseReg + stride*i is a legal2691 // addressing mode, then presume the base GEP is sitting around in a2692 // register somewhere and check if we can fold the offset relative to2693 // it.2694 unsigned Stride = DL.getTypeStoreSize(AccessTy);2695 if (Info.isUnitStride() &&2696 isLegalAddressingMode(AccessTy,2697 /* BaseGV */ nullptr,2698 /* BaseOffset */ Stride * I,2699 /* HasBaseReg */ true,2700 /* Scale */ 0,2701 GEP->getType()->getPointerAddressSpace()))2702 continue;2703 Cost += getArithmeticInstrCost(Instruction::Add, GEP->getType(), CostKind,2704 {TTI::OK_AnyValue, TTI::OP_None},2705 {TTI::OK_AnyValue, TTI::OP_None}, {});2706 } else {2707 SmallVector<const Value *> Indices(GEP->indices());2708 Cost += getGEPCost(GEP->getSourceElementType(), GEP->getPointerOperand(),2709 Indices, AccessTy, CostKind);2710 }2711 }2712 return Cost;2713}2714 2715void RISCVTTIImpl::getUnrollingPreferences(2716 Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP,2717 OptimizationRemarkEmitter *ORE) const {2718 // TODO: More tuning on benchmarks and metrics with changes as needed2719 // would apply to all settings below to enable performance.2720 2721 2722 if (ST->enableDefaultUnroll())2723 return BasicTTIImplBase::getUnrollingPreferences(L, SE, UP, ORE);2724 2725 // Enable Upper bound unrolling universally, not dependent upon the conditions2726 // below.2727 UP.UpperBound = true;2728 2729 // Disable loop unrolling for Oz and Os.2730 UP.OptSizeThreshold = 0;2731 UP.PartialOptSizeThreshold = 0;2732 if (L->getHeader()->getParent()->hasOptSize())2733 return;2734 2735 SmallVector<BasicBlock *, 4> ExitingBlocks;2736 L->getExitingBlocks(ExitingBlocks);2737 LLVM_DEBUG(dbgs() << "Loop has:\n"2738 << "Blocks: " << L->getNumBlocks() << "\n"2739 << "Exit blocks: " << ExitingBlocks.size() << "\n");2740 2741 // Only allow another exit other than the latch. This acts as an early exit2742 // as it mirrors the profitability calculation of the runtime unroller.2743 if (ExitingBlocks.size() > 2)2744 return;2745 2746 // Limit the CFG of the loop body for targets with a branch predictor.2747 // Allowing 4 blocks permits if-then-else diamonds in the body.2748 if (L->getNumBlocks() > 4)2749 return;2750 2751 // Scan the loop: don't unroll loops with calls as this could prevent2752 // inlining. Don't unroll auto-vectorized loops either, though do allow2753 // unrolling of the scalar remainder.2754 bool IsVectorized = getBooleanLoopAttribute(L, "llvm.loop.isvectorized");2755 InstructionCost Cost = 0;2756 for (auto *BB : L->getBlocks()) {2757 for (auto &I : *BB) {2758 // Both auto-vectorized loops and the scalar remainder have the2759 // isvectorized attribute, so differentiate between them by the presence2760 // of vector instructions.2761 if (IsVectorized && I.getType()->isVectorTy())2762 return;2763 2764 if (isa<CallInst>(I) || isa<InvokeInst>(I)) {2765 if (const Function *F = cast<CallBase>(I).getCalledFunction()) {2766 if (!isLoweredToCall(F))2767 continue;2768 }2769 return;2770 }2771 2772 SmallVector<const Value *> Operands(I.operand_values());2773 Cost += getInstructionCost(&I, Operands,2774 TargetTransformInfo::TCK_SizeAndLatency);2775 }2776 }2777 2778 LLVM_DEBUG(dbgs() << "Cost of loop: " << Cost << "\n");2779 2780 UP.Partial = true;2781 UP.Runtime = true;2782 UP.UnrollRemainder = true;2783 UP.UnrollAndJam = true;2784 2785 // Force unrolling small loops can be very useful because of the branch2786 // taken cost of the backedge.2787 if (Cost < 12)2788 UP.Force = true;2789}2790 2791void RISCVTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,2792 TTI::PeelingPreferences &PP) const {2793 BaseT::getPeelingPreferences(L, SE, PP);2794}2795 2796bool RISCVTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,2797 MemIntrinsicInfo &Info) const {2798 const DataLayout &DL = getDataLayout();2799 Intrinsic::ID IID = Inst->getIntrinsicID();2800 LLVMContext &C = Inst->getContext();2801 bool HasMask = false;2802 2803 auto getSegNum = [](const IntrinsicInst *II, unsigned PtrOperandNo,2804 bool IsWrite) -> int64_t {2805 if (auto *TarExtTy =2806 dyn_cast<TargetExtType>(II->getArgOperand(0)->getType()))2807 return TarExtTy->getIntParameter(0);2808 2809 return 1;2810 };2811 2812 switch (IID) {2813 case Intrinsic::riscv_vle_mask:2814 case Intrinsic::riscv_vse_mask:2815 case Intrinsic::riscv_vlseg2_mask:2816 case Intrinsic::riscv_vlseg3_mask:2817 case Intrinsic::riscv_vlseg4_mask:2818 case Intrinsic::riscv_vlseg5_mask:2819 case Intrinsic::riscv_vlseg6_mask:2820 case Intrinsic::riscv_vlseg7_mask:2821 case Intrinsic::riscv_vlseg8_mask:2822 case Intrinsic::riscv_vsseg2_mask:2823 case Intrinsic::riscv_vsseg3_mask:2824 case Intrinsic::riscv_vsseg4_mask:2825 case Intrinsic::riscv_vsseg5_mask:2826 case Intrinsic::riscv_vsseg6_mask:2827 case Intrinsic::riscv_vsseg7_mask:2828 case Intrinsic::riscv_vsseg8_mask:2829 HasMask = true;2830 [[fallthrough]];2831 case Intrinsic::riscv_vle:2832 case Intrinsic::riscv_vse:2833 case Intrinsic::riscv_vlseg2:2834 case Intrinsic::riscv_vlseg3:2835 case Intrinsic::riscv_vlseg4:2836 case Intrinsic::riscv_vlseg5:2837 case Intrinsic::riscv_vlseg6:2838 case Intrinsic::riscv_vlseg7:2839 case Intrinsic::riscv_vlseg8:2840 case Intrinsic::riscv_vsseg2:2841 case Intrinsic::riscv_vsseg3:2842 case Intrinsic::riscv_vsseg4:2843 case Intrinsic::riscv_vsseg5:2844 case Intrinsic::riscv_vsseg6:2845 case Intrinsic::riscv_vsseg7:2846 case Intrinsic::riscv_vsseg8: {2847 // Intrinsic interface:2848 // riscv_vle(merge, ptr, vl)2849 // riscv_vle_mask(merge, ptr, mask, vl, policy)2850 // riscv_vse(val, ptr, vl)2851 // riscv_vse_mask(val, ptr, mask, vl, policy)2852 // riscv_vlseg#(merge, ptr, vl, sew)2853 // riscv_vlseg#_mask(merge, ptr, mask, vl, policy, sew)2854 // riscv_vsseg#(val, ptr, vl, sew)2855 // riscv_vsseg#_mask(val, ptr, mask, vl, sew)2856 bool IsWrite = Inst->getType()->isVoidTy();2857 Type *Ty = IsWrite ? Inst->getArgOperand(0)->getType() : Inst->getType();2858 // The results of segment loads are TargetExtType.2859 if (auto *TarExtTy = dyn_cast<TargetExtType>(Ty)) {2860 unsigned SEW =2861 1 << cast<ConstantInt>(Inst->getArgOperand(Inst->arg_size() - 1))2862 ->getZExtValue();2863 Ty = TarExtTy->getTypeParameter(0U);2864 Ty = ScalableVectorType::get(2865 IntegerType::get(C, SEW),2866 cast<ScalableVectorType>(Ty)->getMinNumElements() * 8 / SEW);2867 }2868 const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IID);2869 unsigned VLIndex = RVVIInfo->VLOperand;2870 unsigned PtrOperandNo = VLIndex - 1 - HasMask;2871 MaybeAlign Alignment =2872 Inst->getArgOperand(PtrOperandNo)->getPointerAlignment(DL);2873 Type *MaskType = Ty->getWithNewType(Type::getInt1Ty(C));2874 Value *Mask = ConstantInt::getTrue(MaskType);2875 if (HasMask)2876 Mask = Inst->getArgOperand(VLIndex - 1);2877 Value *EVL = Inst->getArgOperand(VLIndex);2878 unsigned SegNum = getSegNum(Inst, PtrOperandNo, IsWrite);2879 // RVV uses contiguous elements as a segment.2880 if (SegNum > 1) {2881 unsigned ElemSize = Ty->getScalarSizeInBits();2882 auto *SegTy = IntegerType::get(C, ElemSize * SegNum);2883 Ty = VectorType::get(SegTy, cast<VectorType>(Ty));2884 }2885 Info.InterestingOperands.emplace_back(Inst, PtrOperandNo, IsWrite, Ty,2886 Alignment, Mask, EVL);2887 return true;2888 }2889 case Intrinsic::riscv_vlse_mask:2890 case Intrinsic::riscv_vsse_mask:2891 case Intrinsic::riscv_vlsseg2_mask:2892 case Intrinsic::riscv_vlsseg3_mask:2893 case Intrinsic::riscv_vlsseg4_mask:2894 case Intrinsic::riscv_vlsseg5_mask:2895 case Intrinsic::riscv_vlsseg6_mask:2896 case Intrinsic::riscv_vlsseg7_mask:2897 case Intrinsic::riscv_vlsseg8_mask:2898 case Intrinsic::riscv_vssseg2_mask:2899 case Intrinsic::riscv_vssseg3_mask:2900 case Intrinsic::riscv_vssseg4_mask:2901 case Intrinsic::riscv_vssseg5_mask:2902 case Intrinsic::riscv_vssseg6_mask:2903 case Intrinsic::riscv_vssseg7_mask:2904 case Intrinsic::riscv_vssseg8_mask:2905 HasMask = true;2906 [[fallthrough]];2907 case Intrinsic::riscv_vlse:2908 case Intrinsic::riscv_vsse:2909 case Intrinsic::riscv_vlsseg2:2910 case Intrinsic::riscv_vlsseg3:2911 case Intrinsic::riscv_vlsseg4:2912 case Intrinsic::riscv_vlsseg5:2913 case Intrinsic::riscv_vlsseg6:2914 case Intrinsic::riscv_vlsseg7:2915 case Intrinsic::riscv_vlsseg8:2916 case Intrinsic::riscv_vssseg2:2917 case Intrinsic::riscv_vssseg3:2918 case Intrinsic::riscv_vssseg4:2919 case Intrinsic::riscv_vssseg5:2920 case Intrinsic::riscv_vssseg6:2921 case Intrinsic::riscv_vssseg7:2922 case Intrinsic::riscv_vssseg8: {2923 // Intrinsic interface:2924 // riscv_vlse(merge, ptr, stride, vl)2925 // riscv_vlse_mask(merge, ptr, stride, mask, vl, policy)2926 // riscv_vsse(val, ptr, stride, vl)2927 // riscv_vsse_mask(val, ptr, stride, mask, vl, policy)2928 // riscv_vlsseg#(merge, ptr, offset, vl, sew)2929 // riscv_vlsseg#_mask(merge, ptr, offset, mask, vl, policy, sew)2930 // riscv_vssseg#(val, ptr, offset, vl, sew)2931 // riscv_vssseg#_mask(val, ptr, offset, mask, vl, sew)2932 bool IsWrite = Inst->getType()->isVoidTy();2933 Type *Ty = IsWrite ? Inst->getArgOperand(0)->getType() : Inst->getType();2934 // The results of segment loads are TargetExtType.2935 if (auto *TarExtTy = dyn_cast<TargetExtType>(Ty)) {2936 unsigned SEW =2937 1 << cast<ConstantInt>(Inst->getArgOperand(Inst->arg_size() - 1))2938 ->getZExtValue();2939 Ty = TarExtTy->getTypeParameter(0U);2940 Ty = ScalableVectorType::get(2941 IntegerType::get(C, SEW),2942 cast<ScalableVectorType>(Ty)->getMinNumElements() * 8 / SEW);2943 }2944 const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IID);2945 unsigned VLIndex = RVVIInfo->VLOperand;2946 unsigned PtrOperandNo = VLIndex - 2 - HasMask;2947 MaybeAlign Alignment =2948 Inst->getArgOperand(PtrOperandNo)->getPointerAlignment(DL);2949 2950 Value *Stride = Inst->getArgOperand(PtrOperandNo + 1);2951 // Use the pointer alignment as the element alignment if the stride is a2952 // multiple of the pointer alignment. Otherwise, the element alignment2953 // should be the greatest common divisor of pointer alignment and stride.2954 // For simplicity, just consider unalignment for elements.2955 unsigned PointerAlign = Alignment.valueOrOne().value();2956 if (!isa<ConstantInt>(Stride) ||2957 cast<ConstantInt>(Stride)->getZExtValue() % PointerAlign != 0)2958 Alignment = Align(1);2959 2960 Type *MaskType = Ty->getWithNewType(Type::getInt1Ty(C));2961 Value *Mask = ConstantInt::getTrue(MaskType);2962 if (HasMask)2963 Mask = Inst->getArgOperand(VLIndex - 1);2964 Value *EVL = Inst->getArgOperand(VLIndex);2965 unsigned SegNum = getSegNum(Inst, PtrOperandNo, IsWrite);2966 // RVV uses contiguous elements as a segment.2967 if (SegNum > 1) {2968 unsigned ElemSize = Ty->getScalarSizeInBits();2969 auto *SegTy = IntegerType::get(C, ElemSize * SegNum);2970 Ty = VectorType::get(SegTy, cast<VectorType>(Ty));2971 }2972 Info.InterestingOperands.emplace_back(Inst, PtrOperandNo, IsWrite, Ty,2973 Alignment, Mask, EVL, Stride);2974 return true;2975 }2976 case Intrinsic::riscv_vloxei_mask:2977 case Intrinsic::riscv_vluxei_mask:2978 case Intrinsic::riscv_vsoxei_mask:2979 case Intrinsic::riscv_vsuxei_mask:2980 case Intrinsic::riscv_vloxseg2_mask:2981 case Intrinsic::riscv_vloxseg3_mask:2982 case Intrinsic::riscv_vloxseg4_mask:2983 case Intrinsic::riscv_vloxseg5_mask:2984 case Intrinsic::riscv_vloxseg6_mask:2985 case Intrinsic::riscv_vloxseg7_mask:2986 case Intrinsic::riscv_vloxseg8_mask:2987 case Intrinsic::riscv_vluxseg2_mask:2988 case Intrinsic::riscv_vluxseg3_mask:2989 case Intrinsic::riscv_vluxseg4_mask:2990 case Intrinsic::riscv_vluxseg5_mask:2991 case Intrinsic::riscv_vluxseg6_mask:2992 case Intrinsic::riscv_vluxseg7_mask:2993 case Intrinsic::riscv_vluxseg8_mask:2994 case Intrinsic::riscv_vsoxseg2_mask:2995 case Intrinsic::riscv_vsoxseg3_mask:2996 case Intrinsic::riscv_vsoxseg4_mask:2997 case Intrinsic::riscv_vsoxseg5_mask:2998 case Intrinsic::riscv_vsoxseg6_mask:2999 case Intrinsic::riscv_vsoxseg7_mask:3000 case Intrinsic::riscv_vsoxseg8_mask:3001 case Intrinsic::riscv_vsuxseg2_mask:3002 case Intrinsic::riscv_vsuxseg3_mask:3003 case Intrinsic::riscv_vsuxseg4_mask:3004 case Intrinsic::riscv_vsuxseg5_mask:3005 case Intrinsic::riscv_vsuxseg6_mask:3006 case Intrinsic::riscv_vsuxseg7_mask:3007 case Intrinsic::riscv_vsuxseg8_mask:3008 HasMask = true;3009 [[fallthrough]];3010 case Intrinsic::riscv_vloxei:3011 case Intrinsic::riscv_vluxei:3012 case Intrinsic::riscv_vsoxei:3013 case Intrinsic::riscv_vsuxei:3014 case Intrinsic::riscv_vloxseg2:3015 case Intrinsic::riscv_vloxseg3:3016 case Intrinsic::riscv_vloxseg4:3017 case Intrinsic::riscv_vloxseg5:3018 case Intrinsic::riscv_vloxseg6:3019 case Intrinsic::riscv_vloxseg7:3020 case Intrinsic::riscv_vloxseg8:3021 case Intrinsic::riscv_vluxseg2:3022 case Intrinsic::riscv_vluxseg3:3023 case Intrinsic::riscv_vluxseg4:3024 case Intrinsic::riscv_vluxseg5:3025 case Intrinsic::riscv_vluxseg6:3026 case Intrinsic::riscv_vluxseg7:3027 case Intrinsic::riscv_vluxseg8:3028 case Intrinsic::riscv_vsoxseg2:3029 case Intrinsic::riscv_vsoxseg3:3030 case Intrinsic::riscv_vsoxseg4:3031 case Intrinsic::riscv_vsoxseg5:3032 case Intrinsic::riscv_vsoxseg6:3033 case Intrinsic::riscv_vsoxseg7:3034 case Intrinsic::riscv_vsoxseg8:3035 case Intrinsic::riscv_vsuxseg2:3036 case Intrinsic::riscv_vsuxseg3:3037 case Intrinsic::riscv_vsuxseg4:3038 case Intrinsic::riscv_vsuxseg5:3039 case Intrinsic::riscv_vsuxseg6:3040 case Intrinsic::riscv_vsuxseg7:3041 case Intrinsic::riscv_vsuxseg8: {3042 // Intrinsic interface (only listed ordered version):3043 // riscv_vloxei(merge, ptr, index, vl)3044 // riscv_vloxei_mask(merge, ptr, index, mask, vl, policy)3045 // riscv_vsoxei(val, ptr, index, vl)3046 // riscv_vsoxei_mask(val, ptr, index, mask, vl, policy)3047 // riscv_vloxseg#(merge, ptr, index, vl, sew)3048 // riscv_vloxseg#_mask(merge, ptr, index, mask, vl, policy, sew)3049 // riscv_vsoxseg#(val, ptr, index, vl, sew)3050 // riscv_vsoxseg#_mask(val, ptr, index, mask, vl, sew)3051 bool IsWrite = Inst->getType()->isVoidTy();3052 Type *Ty = IsWrite ? Inst->getArgOperand(0)->getType() : Inst->getType();3053 // The results of segment loads are TargetExtType.3054 if (auto *TarExtTy = dyn_cast<TargetExtType>(Ty)) {3055 unsigned SEW =3056 1 << cast<ConstantInt>(Inst->getArgOperand(Inst->arg_size() - 1))3057 ->getZExtValue();3058 Ty = TarExtTy->getTypeParameter(0U);3059 Ty = ScalableVectorType::get(3060 IntegerType::get(C, SEW),3061 cast<ScalableVectorType>(Ty)->getMinNumElements() * 8 / SEW);3062 }3063 const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IID);3064 unsigned VLIndex = RVVIInfo->VLOperand;3065 unsigned PtrOperandNo = VLIndex - 2 - HasMask;3066 Value *Mask;3067 if (HasMask) {3068 Mask = Inst->getArgOperand(VLIndex - 1);3069 } else {3070 // Mask cannot be nullptr here: vector GEP produces <vscale x N x ptr>,3071 // and casting that to scalar i64 triggers a vector/scalar mismatch3072 // assertion in CreatePointerCast. Use an all-true mask so ASan lowers it3073 // via extractelement instead.3074 Type *MaskType = Ty->getWithNewType(Type::getInt1Ty(C));3075 Mask = ConstantInt::getTrue(MaskType);3076 }3077 Value *EVL = Inst->getArgOperand(VLIndex);3078 unsigned SegNum = getSegNum(Inst, PtrOperandNo, IsWrite);3079 // RVV uses contiguous elements as a segment.3080 if (SegNum > 1) {3081 unsigned ElemSize = Ty->getScalarSizeInBits();3082 auto *SegTy = IntegerType::get(C, ElemSize * SegNum);3083 Ty = VectorType::get(SegTy, cast<VectorType>(Ty));3084 }3085 Value *OffsetOp = Inst->getArgOperand(PtrOperandNo + 1);3086 Info.InterestingOperands.emplace_back(Inst, PtrOperandNo, IsWrite, Ty,3087 Align(1), Mask, EVL,3088 /* Stride */ nullptr, OffsetOp);3089 return true;3090 }3091 }3092 return false;3093}3094 3095unsigned RISCVTTIImpl::getRegUsageForType(Type *Ty) const {3096 if (Ty->isVectorTy()) {3097 // f16 with only zvfhmin and bf16 will be promoted to f323098 Type *EltTy = cast<VectorType>(Ty)->getElementType();3099 if ((EltTy->isHalfTy() && !ST->hasVInstructionsF16()) ||3100 EltTy->isBFloatTy())3101 Ty = VectorType::get(Type::getFloatTy(Ty->getContext()),3102 cast<VectorType>(Ty));3103 3104 TypeSize Size = DL.getTypeSizeInBits(Ty);3105 if (Size.isScalable() && ST->hasVInstructions())3106 return divideCeil(Size.getKnownMinValue(), RISCV::RVVBitsPerBlock);3107 3108 if (ST->useRVVForFixedLengthVectors())3109 return divideCeil(Size, ST->getRealMinVLen());3110 }3111 3112 return BaseT::getRegUsageForType(Ty);3113}3114 3115unsigned RISCVTTIImpl::getMaximumVF(unsigned ElemWidth, unsigned Opcode) const {3116 if (SLPMaxVF.getNumOccurrences())3117 return SLPMaxVF;3118 3119 // Return how many elements can fit in getRegisterBitwidth. This is the3120 // same routine as used in LoopVectorizer. We should probably be3121 // accounting for whether we actually have instructions with the right3122 // lane type, but we don't have enough information to do that without3123 // some additional plumbing which hasn't been justified yet.3124 TypeSize RegWidth =3125 getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector);3126 // If no vector registers, or absurd element widths, disable3127 // vectorization by returning 1.3128 return std::max<unsigned>(1U, RegWidth.getFixedValue() / ElemWidth);3129}3130 3131unsigned RISCVTTIImpl::getMinTripCountTailFoldingThreshold() const {3132 return RVVMinTripCount;3133}3134 3135bool RISCVTTIImpl::preferAlternateOpcodeVectorization() const {3136 return ST->enableUnalignedVectorMem();3137}3138 3139TTI::AddressingModeKind3140RISCVTTIImpl::getPreferredAddressingMode(const Loop *L,3141 ScalarEvolution *SE) const {3142 if (ST->hasVendorXCVmem() && !ST->is64Bit())3143 return TTI::AMK_PostIndexed;3144 3145 return BasicTTIImplBase::getPreferredAddressingMode(L, SE);3146}3147 3148bool RISCVTTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1,3149 const TargetTransformInfo::LSRCost &C2) const {3150 // RISC-V specific here are "instruction number 1st priority".3151 // If we need to emit adds inside the loop to add up base registers, then3152 // we need at least one extra temporary register.3153 unsigned C1NumRegs = C1.NumRegs + (C1.NumBaseAdds != 0);3154 unsigned C2NumRegs = C2.NumRegs + (C2.NumBaseAdds != 0);3155 return std::tie(C1.Insns, C1NumRegs, C1.AddRecCost,3156 C1.NumIVMuls, C1.NumBaseAdds,3157 C1.ScaleCost, C1.ImmCost, C1.SetupCost) <3158 std::tie(C2.Insns, C2NumRegs, C2.AddRecCost,3159 C2.NumIVMuls, C2.NumBaseAdds,3160 C2.ScaleCost, C2.ImmCost, C2.SetupCost);3161}3162 3163bool RISCVTTIImpl::isLegalMaskedExpandLoad(Type *DataTy,3164 Align Alignment) const {3165 auto *VTy = dyn_cast<VectorType>(DataTy);3166 if (!VTy || VTy->isScalableTy())3167 return false;3168 3169 if (!isLegalMaskedLoadStore(DataTy, Alignment))3170 return false;3171 3172 // FIXME: If it is an i8 vector and the element count exceeds 256, we should3173 // scalarize these types with LMUL >= maximum fixed-length LMUL.3174 if (VTy->getElementType()->isIntegerTy(8))3175 if (VTy->getElementCount().getFixedValue() > 256)3176 return VTy->getPrimitiveSizeInBits() / ST->getRealMinVLen() <3177 ST->getMaxLMULForFixedLengthVectors();3178 return true;3179}3180 3181bool RISCVTTIImpl::isLegalMaskedCompressStore(Type *DataTy,3182 Align Alignment) const {3183 auto *VTy = dyn_cast<VectorType>(DataTy);3184 if (!VTy || VTy->isScalableTy())3185 return false;3186 3187 if (!isLegalMaskedLoadStore(DataTy, Alignment))3188 return false;3189 return true;3190}3191 3192/// See if \p I should be considered for address type promotion. We check if \p3193/// I is a sext with right type and used in memory accesses. If it used in a3194/// "complex" getelementptr, we allow it to be promoted without finding other3195/// sext instructions that sign extended the same initial value. A getelementptr3196/// is considered as "complex" if it has more than 2 operands.3197bool RISCVTTIImpl::shouldConsiderAddressTypePromotion(3198 const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const {3199 bool Considerable = false;3200 AllowPromotionWithoutCommonHeader = false;3201 if (!isa<SExtInst>(&I))3202 return false;3203 Type *ConsideredSExtType =3204 Type::getInt64Ty(I.getParent()->getParent()->getContext());3205 if (I.getType() != ConsideredSExtType)3206 return false;3207 // See if the sext is the one with the right type and used in at least one3208 // GetElementPtrInst.3209 for (const User *U : I.users()) {3210 if (const GetElementPtrInst *GEPInst = dyn_cast<GetElementPtrInst>(U)) {3211 Considerable = true;3212 // A getelementptr is considered as "complex" if it has more than 23213 // operands. We will promote a SExt used in such complex GEP as we3214 // expect some computation to be merged if they are done on 64 bits.3215 if (GEPInst->getNumOperands() > 2) {3216 AllowPromotionWithoutCommonHeader = true;3217 break;3218 }3219 }3220 }3221 return Considerable;3222}3223 3224bool RISCVTTIImpl::canSplatOperand(unsigned Opcode, int Operand) const {3225 switch (Opcode) {3226 case Instruction::Add:3227 case Instruction::Sub:3228 case Instruction::Mul:3229 case Instruction::And:3230 case Instruction::Or:3231 case Instruction::Xor:3232 case Instruction::FAdd:3233 case Instruction::FSub:3234 case Instruction::FMul:3235 case Instruction::FDiv:3236 case Instruction::ICmp:3237 case Instruction::FCmp:3238 return true;3239 case Instruction::Shl:3240 case Instruction::LShr:3241 case Instruction::AShr:3242 case Instruction::UDiv:3243 case Instruction::SDiv:3244 case Instruction::URem:3245 case Instruction::SRem:3246 case Instruction::Select:3247 return Operand == 1;3248 default:3249 return false;3250 }3251}3252 3253bool RISCVTTIImpl::canSplatOperand(Instruction *I, int Operand) const {3254 if (!I->getType()->isVectorTy() || !ST->hasVInstructions())3255 return false;3256 3257 if (canSplatOperand(I->getOpcode(), Operand))3258 return true;3259 3260 auto *II = dyn_cast<IntrinsicInst>(I);3261 if (!II)3262 return false;3263 3264 switch (II->getIntrinsicID()) {3265 case Intrinsic::fma:3266 case Intrinsic::vp_fma:3267 case Intrinsic::fmuladd:3268 case Intrinsic::vp_fmuladd:3269 return Operand == 0 || Operand == 1;3270 case Intrinsic::vp_shl:3271 case Intrinsic::vp_lshr:3272 case Intrinsic::vp_ashr:3273 case Intrinsic::vp_udiv:3274 case Intrinsic::vp_sdiv:3275 case Intrinsic::vp_urem:3276 case Intrinsic::vp_srem:3277 case Intrinsic::ssub_sat:3278 case Intrinsic::vp_ssub_sat:3279 case Intrinsic::usub_sat:3280 case Intrinsic::vp_usub_sat:3281 case Intrinsic::vp_select:3282 return Operand == 1;3283 // These intrinsics are commutative.3284 case Intrinsic::vp_add:3285 case Intrinsic::vp_mul:3286 case Intrinsic::vp_and:3287 case Intrinsic::vp_or:3288 case Intrinsic::vp_xor:3289 case Intrinsic::vp_fadd:3290 case Intrinsic::vp_fmul:3291 case Intrinsic::vp_icmp:3292 case Intrinsic::vp_fcmp:3293 case Intrinsic::smin:3294 case Intrinsic::vp_smin:3295 case Intrinsic::umin:3296 case Intrinsic::vp_umin:3297 case Intrinsic::smax:3298 case Intrinsic::vp_smax:3299 case Intrinsic::umax:3300 case Intrinsic::vp_umax:3301 case Intrinsic::sadd_sat:3302 case Intrinsic::vp_sadd_sat:3303 case Intrinsic::uadd_sat:3304 case Intrinsic::vp_uadd_sat:3305 // These intrinsics have 'vr' versions.3306 case Intrinsic::vp_sub:3307 case Intrinsic::vp_fsub:3308 case Intrinsic::vp_fdiv:3309 return Operand == 0 || Operand == 1;3310 default:3311 return false;3312 }3313}3314 3315/// Check if sinking \p I's operands to I's basic block is profitable, because3316/// the operands can be folded into a target instruction, e.g.3317/// splats of scalars can fold into vector instructions.3318bool RISCVTTIImpl::isProfitableToSinkOperands(3319 Instruction *I, SmallVectorImpl<Use *> &Ops) const {3320 using namespace llvm::PatternMatch;3321 3322 if (I->isBitwiseLogicOp()) {3323 if (!I->getType()->isVectorTy()) {3324 if (ST->hasStdExtZbb() || ST->hasStdExtZbkb()) {3325 for (auto &Op : I->operands()) {3326 // (and/or/xor X, (not Y)) -> (andn/orn/xnor X, Y)3327 if (match(Op.get(), m_Not(m_Value()))) {3328 Ops.push_back(&Op);3329 return true;3330 }3331 }3332 }3333 } else if (I->getOpcode() == Instruction::And && ST->hasStdExtZvkb()) {3334 for (auto &Op : I->operands()) {3335 // (and X, (not Y)) -> (vandn.vv X, Y)3336 if (match(Op.get(), m_Not(m_Value()))) {3337 Ops.push_back(&Op);3338 return true;3339 }3340 // (and X, (splat (not Y))) -> (vandn.vx X, Y)3341 if (match(Op.get(), m_Shuffle(m_InsertElt(m_Value(), m_Not(m_Value()),3342 m_ZeroInt()),3343 m_Value(), m_ZeroMask()))) {3344 Use &InsertElt = cast<Instruction>(Op)->getOperandUse(0);3345 Use &Not = cast<Instruction>(InsertElt)->getOperandUse(1);3346 Ops.push_back(&Not);3347 Ops.push_back(&InsertElt);3348 Ops.push_back(&Op);3349 return true;3350 }3351 }3352 }3353 }3354 3355 if (!I->getType()->isVectorTy() || !ST->hasVInstructions())3356 return false;3357 3358 // Don't sink splat operands if the target prefers it. Some targets requires3359 // S2V transfer buffers and we can run out of them copying the same value3360 // repeatedly.3361 // FIXME: It could still be worth doing if it would improve vector register3362 // pressure and prevent a vector spill.3363 if (!ST->sinkSplatOperands())3364 return false;3365 3366 for (auto OpIdx : enumerate(I->operands())) {3367 if (!canSplatOperand(I, OpIdx.index()))3368 continue;3369 3370 Instruction *Op = dyn_cast<Instruction>(OpIdx.value().get());3371 // Make sure we are not already sinking this operand3372 if (!Op || any_of(Ops, [&](Use *U) { return U->get() == Op; }))3373 continue;3374 3375 // We are looking for a splat/vp.splat that can be sunk.3376 bool IsVPSplat = match(Op, m_Intrinsic<Intrinsic::experimental_vp_splat>(3377 m_Value(), m_Value(), m_Value()));3378 if (!IsVPSplat &&3379 !match(Op, m_Shuffle(m_InsertElt(m_Value(), m_Value(), m_ZeroInt()),3380 m_Value(), m_ZeroMask())))3381 continue;3382 3383 // Don't sink i1 splats.3384 if (cast<VectorType>(Op->getType())->getElementType()->isIntegerTy(1))3385 continue;3386 3387 // All uses of the shuffle should be sunk to avoid duplicating it across gpr3388 // and vector registers3389 for (Use &U : Op->uses()) {3390 Instruction *Insn = cast<Instruction>(U.getUser());3391 if (!canSplatOperand(Insn, U.getOperandNo()))3392 return false;3393 }3394 3395 // Sink any fpexts since they might be used in a widening fp pattern.3396 if (IsVPSplat) {3397 if (isa<FPExtInst>(Op->getOperand(0)))3398 Ops.push_back(&Op->getOperandUse(0));3399 } else {3400 Use *InsertEltUse = &Op->getOperandUse(0);3401 auto *InsertElt = cast<InsertElementInst>(InsertEltUse);3402 if (isa<FPExtInst>(InsertElt->getOperand(1)))3403 Ops.push_back(&InsertElt->getOperandUse(1));3404 Ops.push_back(InsertEltUse);3405 }3406 Ops.push_back(&OpIdx.value());3407 }3408 return true;3409}3410 3411RISCVTTIImpl::TTI::MemCmpExpansionOptions3412RISCVTTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {3413 TTI::MemCmpExpansionOptions Options;3414 // TODO: Enable expansion when unaligned access is not supported after we fix3415 // issues in ExpandMemcmp.3416 if (!ST->enableUnalignedScalarMem())3417 return Options;3418 3419 if (!ST->hasStdExtZbb() && !ST->hasStdExtZbkb() && !IsZeroCmp)3420 return Options;3421 3422 Options.AllowOverlappingLoads = true;3423 Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize);3424 Options.NumLoadsPerBlock = Options.MaxNumLoads;3425 if (ST->is64Bit()) {3426 Options.LoadSizes = {8, 4, 2, 1};3427 Options.AllowedTailExpansions = {3, 5, 6};3428 } else {3429 Options.LoadSizes = {4, 2, 1};3430 Options.AllowedTailExpansions = {3};3431 }3432 3433 if (IsZeroCmp && ST->hasVInstructions()) {3434 unsigned VLenB = ST->getRealMinVLen() / 8;3435 // The minimum size should be `XLen / 8 + 1`, and the maxinum size should be3436 // `VLenB * MaxLMUL` so that it fits in a single register group.3437 unsigned MinSize = ST->getXLen() / 8 + 1;3438 unsigned MaxSize = VLenB * ST->getMaxLMULForFixedLengthVectors();3439 for (unsigned Size = MinSize; Size <= MaxSize; Size++)3440 Options.LoadSizes.insert(Options.LoadSizes.begin(), Size);3441 }3442 return Options;3443}3444