890 lines · cpp
1//===-- Target.cpp ----------------------------------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "../Target.h"10#include "../ParallelSnippetGenerator.h"11#include "../SerialSnippetGenerator.h"12#include "../SnippetGenerator.h"13 14#include "MCTargetDesc/RISCVBaseInfo.h"15#include "MCTargetDesc/RISCVMCTargetDesc.h"16#include "MCTargetDesc/RISCVMatInt.h"17#include "RISCV.h"18#include "RISCVExegesisPasses.h"19#include "RISCVInstrInfo.h"20#include "RISCVRegisterInfo.h"21#include "llvm/Support/Regex.h"22#include "llvm/Support/raw_ostream.h"23 24// include computeAvailableFeatures and computeRequiredFeatures.25#define GET_AVAILABLE_OPCODE_CHECKER26#include "RISCVGenInstrInfo.inc"27 28#include "llvm/CodeGen/MachineInstrBuilder.h"29 30#include <vector>31 32namespace llvm {33namespace exegesis {34 35static cl::opt<bool>36 OnlyUsesVLMAXForVL("riscv-vlmax-for-vl",37 cl::desc("Only enumerate VLMAX for VL operand"),38 cl::init(false), cl::Hidden);39 40static cl::opt<bool>41 EnumerateRoundingModes("riscv-enumerate-rounding-modes",42 cl::desc("Enumerate different FRM and VXRM"),43 cl::init(true), cl::Hidden);44 45static cl::opt<std::string>46 FilterConfig("riscv-filter-config",47 cl::desc("Show only the configs matching this regex"),48 cl::init(""), cl::Hidden);49 50#include "RISCVGenExegesis.inc"51 52namespace {53 54template <class BaseT> class RISCVSnippetGenerator : public BaseT {55 static void printRoundingMode(raw_ostream &OS, unsigned Val, bool UsesVXRM) {56 if (UsesVXRM) {57 assert(RISCVVXRndMode::isValidRoundingMode(Val));58 OS << RISCVVXRndMode::roundingModeToString(59 static_cast<RISCVVXRndMode::RoundingMode>(Val));60 } else {61 assert(RISCVFPRndMode::isValidRoundingMode(Val));62 OS << RISCVFPRndMode::roundingModeToString(63 static_cast<RISCVFPRndMode::RoundingMode>(Val));64 }65 }66 67 static constexpr unsigned MinSEW = 8;68 // ELEN is basically SEW_max.69 unsigned ELEN = 64;70 71 // We can't know the real min/max VLEN w/o a Function, so we're72 // using the VLen from Zvl.73 unsigned ZvlVLen = 32;74 75 /// Mask for registers that are NOT standalone registers like X0 and V076 BitVector AggregateRegisters;77 78 // Returns true when opcode is available in any of the FBs.79 static bool80 isOpcodeAvailableIn(unsigned Opcode,81 ArrayRef<RISCV_MC::SubtargetFeatureBits> FBs) {82 FeatureBitset RequiredFeatures = RISCV_MC::computeRequiredFeatures(Opcode);83 for (uint8_t FB : FBs) {84 if (RequiredFeatures[FB])85 return true;86 }87 return false;88 }89 90 static bool isRVVFloatingPointOp(unsigned Opcode) {91 return isOpcodeAvailableIn(Opcode,92 {RISCV_MC::Feature_HasVInstructionsAnyFBit});93 }94 95 // Get the element group width of each vector cryptor extension.96 static unsigned getZvkEGWSize(unsigned Opcode, unsigned SEW) {97 using namespace RISCV_MC;98 if (isOpcodeAvailableIn(Opcode, {Feature_HasStdExtZvkgBit,99 Feature_HasStdExtZvknedBit,100 Feature_HasStdExtZvksedBit}))101 return 128U;102 if (isOpcodeAvailableIn(Opcode, {Feature_HasStdExtZvkshBit}))103 return 256U;104 if (isOpcodeAvailableIn(Opcode, {Feature_HasStdExtZvknhaOrZvknhbBit}))105 // In Zvknh[ab], when SEW=64 is used (i.e. Zvknhb), EGW is 256.106 // Otherwise it's 128.107 return SEW == 64 ? 256U : 128U;108 109 llvm_unreachable("Unsupported opcode");110 }111 112 // A handy utility to multiply or divide an integer by LMUL.113 template <typename T> static T multiplyLMul(T Val, RISCVVType::VLMUL VLMul) {114 auto [LMul, IsFractional] = RISCVVType::decodeVLMUL(VLMul);115 return IsFractional ? Val / LMul : Val * LMul;116 }117 118 /// Return the denominator of the fractional (i.e. the `x` in .vfx suffix) or119 /// nullopt if BaseOpcode is not a vector sext/zext.120 static std::optional<unsigned> isRVVSignZeroExtend(unsigned BaseOpcode) {121 switch (BaseOpcode) {122 case RISCV::VSEXT_VF2:123 case RISCV::VZEXT_VF2:124 return 2;125 case RISCV::VSEXT_VF4:126 case RISCV::VZEXT_VF4:127 return 4;128 case RISCV::VSEXT_VF8:129 case RISCV::VZEXT_VF8:130 return 8;131 default:132 return std::nullopt;133 }134 }135 136 void annotateWithVType(const CodeTemplate &CT, const Instruction &Instr,137 unsigned BaseOpcode,138 const BitVector &ForbiddenRegisters,139 std::vector<CodeTemplate> &Result) const;140 141public:142 RISCVSnippetGenerator(const LLVMState &State,143 const SnippetGenerator::Options &Opts)144 : BaseT(State, Opts),145 AggregateRegisters(State.getRegInfo().getNumRegs(), /*initVal=*/true) {146 // Initialize standalone registers mask.147 const MCRegisterInfo &RegInfo = State.getRegInfo();148 const unsigned StandaloneRegClasses[] = {149 RISCV::GPRRegClassID, RISCV::FPR16RegClassID, RISCV::VRRegClassID};150 151 for (unsigned RegClassID : StandaloneRegClasses)152 for (unsigned Reg : RegInfo.getRegClass(RegClassID))153 AggregateRegisters.reset(Reg);154 155 // Initialize ELEN and VLEN.156 // FIXME: We could have obtained these two constants from RISCVSubtarget157 // but in order to get that from TargetMachine, we need a Function.158 const MCSubtargetInfo &STI = State.getSubtargetInfo();159 ELEN = STI.hasFeature(RISCV::FeatureStdExtZve64x) ? 64 : 32;160 161 const unsigned ZvlFeatures[] = {162 RISCV::FeatureStdExtZvl32b, RISCV::FeatureStdExtZvl64b,163 RISCV::FeatureStdExtZvl128b, RISCV::FeatureStdExtZvl256b,164 RISCV::FeatureStdExtZvl512b, RISCV::FeatureStdExtZvl1024b,165 RISCV::FeatureStdExtZvl2048b, RISCV::FeatureStdExtZvl4096b,166 RISCV::FeatureStdExtZvl8192b, RISCV::FeatureStdExtZvl16384b,167 RISCV::FeatureStdExtZvl32768b, RISCV::FeatureStdExtZvl65536b};168 for (auto [Idx, Feature] : enumerate(ZvlFeatures)) {169 if (STI.hasFeature(Feature))170 ZvlVLen = std::max(ZvlVLen, 1u << (Idx + 5));171 }172 }173 174 Expected<std::vector<CodeTemplate>>175 generateCodeTemplates(InstructionTemplate Variant,176 const BitVector &ForbiddenRegisters) const override;177};178 179static bool isMaskedSibling(unsigned MaskedOp, unsigned UnmaskedOp) {180 const auto *RVVMasked = RISCV::getMaskedPseudoInfo(MaskedOp);181 return RVVMasked && RVVMasked->UnmaskedPseudo == UnmaskedOp;182}183 184// There are primarily two kinds of opcodes that are not eligible185// in a serial snippet:186// (1) Has a use operand that can not overlap with the def operand187// (i.e. early clobber).188// (2) The register file of the only use operand is different from189// that of the def operand. For instance, use operand is vector and190// the result is a scalar.191static bool isIneligibleOfSerialSnippets(unsigned BaseOpcode,192 const Instruction &I) {193 if (llvm::any_of(I.Operands,194 [](const Operand &Op) { return Op.isEarlyClobber(); }))195 return true;196 197 switch (BaseOpcode) {198 case RISCV::VCOMPRESS_VM:199 case RISCV::VCPOP_M:200 case RISCV::VCPOP_V:201 // The permutation instructions listed below cannot have destination202 // overlapping with the source.203 case RISCV::VRGATHEREI16_VV:204 case RISCV::VRGATHER_VI:205 case RISCV::VRGATHER_VV:206 case RISCV::VRGATHER_VX:207 case RISCV::VSLIDE1UP_VX:208 case RISCV::VSLIDEUP_VI:209 case RISCV::VSLIDEUP_VX:210 return true;211 default:212 return false;213 }214}215 216static bool isZvfhminZvfbfminOpcodes(unsigned BaseOpcode) {217 switch (BaseOpcode) {218 case RISCV::VFNCVT_F_F_W:219 case RISCV::VFWCVT_F_F_V:220 case RISCV::VFNCVTBF16_F_F_W:221 case RISCV::VFWCVTBF16_F_F_V:222 return true;223 default:224 return false;225 }226}227 228static bool isVectorReduction(unsigned BaseOpcode) {229 switch (BaseOpcode) {230 case RISCV::VREDAND_VS:231 case RISCV::VREDMAXU_VS:232 case RISCV::VREDMAX_VS:233 case RISCV::VREDMINU_VS:234 case RISCV::VREDMIN_VS:235 case RISCV::VREDOR_VS:236 case RISCV::VREDSUM_VS:237 case RISCV::VREDXOR_VS:238 case RISCV::VWREDSUMU_VS:239 case RISCV::VWREDSUM_VS:240 case RISCV::VFREDMAX_VS:241 case RISCV::VFREDMIN_VS:242 case RISCV::VFREDOSUM_VS:243 case RISCV::VFREDUSUM_VS:244 return true;245 default:246 return false;247 }248}249 250template <class BaseT>251void RISCVSnippetGenerator<BaseT>::annotateWithVType(252 const CodeTemplate &OrigCT, const Instruction &Instr, unsigned BaseOpcode,253 const BitVector &ForbiddenRegisters,254 std::vector<CodeTemplate> &Result) const {255 const MCSubtargetInfo &STI = SnippetGenerator::State.getSubtargetInfo();256 unsigned VPseudoOpcode = Instr.getOpcode();257 258 bool IsSerial = std::is_same_v<BaseT, SerialSnippetGenerator>;259 260 const MCInstrDesc &MIDesc = Instr.Description;261 const uint64_t TSFlags = MIDesc.TSFlags;262 263 RISCVVType::VLMUL VLMul = RISCVII::getLMul(TSFlags);264 265 const size_t StartingResultSize = Result.size();266 267 SmallPtrSet<const Operand *, 4> VTypeOperands;268 std::optional<AliasingConfigurations> SelfAliasing;269 // Exegesis see instructions with tied operands being inherently serial.270 // But for RVV instructions, those tied operands are passthru rather271 // than real read operands. So we manually put dependency between272 // destination (i.e. def) and any of the non-tied/SEW/policy/AVL/RM273 // operands.274 auto assignSerialRVVOperands = [&, this](InstructionTemplate &IT) {275 // Initialize SelfAliasing on first use.276 if (!SelfAliasing.has_value()) {277 BitVector ExcludeRegs = ForbiddenRegisters;278 ExcludeRegs |= AggregateRegisters;279 SelfAliasing = AliasingConfigurations(Instr, Instr, ExcludeRegs);280 bool EmptyUses = false;281 for (auto &ARO : SelfAliasing->Configurations) {282 auto &Uses = ARO.Uses;283 for (auto ROA = Uses.begin(); ROA != Uses.end();) {284 const Operand *Op = ROA->Op;285 // Exclude tied operand(s).286 if (Op->isTied()) {287 ROA = Uses.erase(ROA);288 continue;289 }290 291 // Special handling for reduction operations: for a given reduction292 // `vredop vd, vs2, vs1`, we don't want vd to be aliased with vs1293 // since we're only reading `vs1[0]` and many implementations294 // optimize for this case (e.g. chaining). Instead, we're forcing295 // it to create alias between vd and vs2.296 if (isVectorReduction(BaseOpcode) &&297 // vs1's operand index is always 3.298 Op->getIndex() == 3) {299 ROA = Uses.erase(ROA);300 continue;301 }302 303 // Exclude any special operands like SEW and VL -- we've already304 // assigned values to them.305 if (VTypeOperands.count(Op)) {306 ROA = Uses.erase(ROA);307 continue;308 }309 ++ROA;310 }311 312 // If any of the use operand candidate lists is empty, there is313 // no point to assign self aliasing registers.314 if (Uses.empty()) {315 EmptyUses = true;316 break;317 }318 }319 if (EmptyUses)320 SelfAliasing->Configurations.clear();321 }322 323 // This is a self aliasing instruction so defs and uses are from the same324 // instance, hence twice IT in the following call.325 if (!SelfAliasing->empty() && !SelfAliasing->hasImplicitAliasing())326 setRandomAliasing(*SelfAliasing, IT, IT);327 };328 329 // We are going to create a CodeTemplate (configuration) for each supported330 // SEW, policy, and VL.331 // FIXME: Account for EEW and EMUL.332 SmallVector<std::optional<unsigned>, 4> Log2SEWs;333 SmallVector<std::optional<unsigned>, 4> Policies;334 SmallVector<std::optional<int>, 3> AVLs;335 SmallVector<std::optional<unsigned>, 8> RoundingModes;336 337 bool HasSEWOp = RISCVII::hasSEWOp(TSFlags);338 bool HasPolicyOp = RISCVII::hasVecPolicyOp(TSFlags);339 bool HasVLOp = RISCVII::hasVLOp(TSFlags);340 bool HasRMOp = RISCVII::hasRoundModeOp(TSFlags);341 bool UsesVXRM = RISCVII::usesVXRM(TSFlags);342 343 if (HasSEWOp) {344 const Operand &SEWOp = Instr.Operands[RISCVII::getSEWOpNum(MIDesc)];345 VTypeOperands.insert(&SEWOp);346 347 if (SEWOp.Info->OperandType == RISCVOp::OPERAND_SEW_MASK) {348 // If it's a mask-producing instruction, the SEW operand is always zero.349 Log2SEWs.push_back(0);350 } else {351 SmallVector<unsigned, 4> SEWCandidates;352 353 // (RVV spec 3.4.2) For fractional LMUL, the supported SEW are between354 // [SEW_min, LMUL * ELEN].355 unsigned SEWUpperBound =356 VLMul >= RISCVVType::LMUL_F8 ? multiplyLMul(ELEN, VLMul) : ELEN;357 for (unsigned SEW = MinSEW; SEW <= SEWUpperBound; SEW <<= 1) {358 SEWCandidates.push_back(SEW);359 360 // Some scheduling classes already integrate SEW; only put361 // their corresponding SEW values at the SEW operands.362 // NOTE: It is imperative to put this condition in the front, otherwise363 // it is tricky and difficult to know if there is an integrated364 // SEW after other rules are applied to filter the candidates.365 const auto *RVVBase =366 RISCVVInversePseudosTable::getBaseInfo(BaseOpcode, VLMul, SEW);367 if (RVVBase && (RVVBase->Pseudo == VPseudoOpcode ||368 isMaskedSibling(VPseudoOpcode, RVVBase->Pseudo) ||369 isMaskedSibling(RVVBase->Pseudo, VPseudoOpcode))) {370 // There is an integrated SEW, remove all but the SEW pushed last.371 SEWCandidates.erase(SEWCandidates.begin(), SEWCandidates.end() - 1);372 break;373 }374 }375 376 // Filter out some candidates.377 for (auto SEW = SEWCandidates.begin(); SEW != SEWCandidates.end();) {378 // For floating point operations, only select SEW of the supported FLEN.379 if (isRVVFloatingPointOp(VPseudoOpcode)) {380 bool Supported = false;381 Supported |= isZvfhminZvfbfminOpcodes(BaseOpcode) && *SEW == 16;382 Supported |= STI.hasFeature(RISCV::FeatureStdExtZvfh) && *SEW == 16;383 Supported |= STI.hasFeature(RISCV::FeatureStdExtF) && *SEW == 32;384 Supported |= STI.hasFeature(RISCV::FeatureStdExtD) && *SEW == 64;385 if (!Supported) {386 SEW = SEWCandidates.erase(SEW);387 continue;388 }389 }390 391 // The EEW for source operand in VSEXT and VZEXT is a fraction392 // of the SEW, hence only SEWs that will lead to valid EEW are allowed.393 if (auto Frac = isRVVSignZeroExtend(BaseOpcode))394 if (*SEW / *Frac < MinSEW) {395 SEW = SEWCandidates.erase(SEW);396 continue;397 }398 399 // Most vector crypto 1.0 instructions only work on SEW=32.400 using namespace RISCV_MC;401 if (isOpcodeAvailableIn(BaseOpcode, {Feature_HasStdExtZvkgBit,402 Feature_HasStdExtZvknedBit,403 Feature_HasStdExtZvknhaOrZvknhbBit,404 Feature_HasStdExtZvksedBit,405 Feature_HasStdExtZvkshBit})) {406 if (*SEW != 32)407 // Zvknhb supports SEW=64 as well.408 if (*SEW != 64 || !STI.hasFeature(RISCV::FeatureStdExtZvknhb) ||409 !isOpcodeAvailableIn(BaseOpcode,410 {Feature_HasStdExtZvknhaOrZvknhbBit})) {411 SEW = SEWCandidates.erase(SEW);412 continue;413 }414 415 // We're also enforcing the requirement of `LMUL * VLEN >= EGW` here,416 // because some of the extensions have SEW-dependant EGW.417 unsigned EGW = getZvkEGWSize(BaseOpcode, *SEW);418 if (multiplyLMul(ZvlVLen, VLMul) < EGW) {419 SEW = SEWCandidates.erase(SEW);420 continue;421 }422 }423 424 ++SEW;425 }426 427 // We're not going to produce any result with zero SEW candidate.428 if (SEWCandidates.empty())429 return;430 431 for (unsigned SEW : SEWCandidates)432 Log2SEWs.push_back(Log2_32(SEW));433 }434 } else {435 Log2SEWs.push_back(std::nullopt);436 }437 438 if (HasPolicyOp) {439 VTypeOperands.insert(&Instr.Operands[RISCVII::getVecPolicyOpNum(MIDesc)]);440 441 Policies = {0, RISCVVType::TAIL_AGNOSTIC, RISCVVType::MASK_AGNOSTIC,442 (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC)};443 } else {444 Policies.push_back(std::nullopt);445 }446 447 if (HasVLOp) {448 VTypeOperands.insert(&Instr.Operands[RISCVII::getVLOpNum(MIDesc)]);449 450 if (OnlyUsesVLMAXForVL)451 AVLs.push_back(-1);452 else453 AVLs = {// 5-bit immediate value454 1,455 // VLMAX456 -1,457 // Non-X0 register458 0};459 } else {460 AVLs.push_back(std::nullopt);461 }462 463 if (HasRMOp) {464 VTypeOperands.insert(&Instr.Operands[RISCVII::getVLOpNum(MIDesc) - 1]);465 466 if (UsesVXRM) {467 // Use RNU as the default VXRM.468 RoundingModes = {RISCVVXRndMode::RNU};469 if (EnumerateRoundingModes)470 RoundingModes.append(471 {RISCVVXRndMode::RNE, RISCVVXRndMode::RDN, RISCVVXRndMode::ROD});472 } else {473 if (EnumerateRoundingModes)474 RoundingModes = {RISCVFPRndMode::RNE, RISCVFPRndMode::RTZ,475 RISCVFPRndMode::RDN, RISCVFPRndMode::RUP,476 RISCVFPRndMode::RMM};477 else478 // If we're not enumerating FRM, use DYN to instruct479 // RISCVInsertReadWriteCSRPass to insert nothing.480 RoundingModes = {RISCVFPRndMode::DYN};481 }482 } else {483 RoundingModes = {std::nullopt};484 }485 486 std::set<std::tuple<std::optional<unsigned>, std::optional<int>,487 std::optional<unsigned>, std::optional<unsigned>>>488 Combinations;489 for (auto AVL : AVLs) {490 for (auto Log2SEW : Log2SEWs)491 for (auto Policy : Policies) {492 for (auto RM : RoundingModes)493 Combinations.insert(std::make_tuple(RM, AVL, Log2SEW, Policy));494 }495 }496 497 std::string ConfigStr;498 SmallVector<std::pair<const Operand *, MCOperand>, 4> ValueAssignments;499 for (const auto &[RM, AVL, Log2SEW, Policy] : Combinations) {500 InstructionTemplate IT(&Instr);501 502 ListSeparator LS;503 ConfigStr = "vtype = {";504 raw_string_ostream SS(ConfigStr);505 506 ValueAssignments.clear();507 508 if (RM) {509 const Operand &Op = Instr.Operands[RISCVII::getVLOpNum(MIDesc) - 1];510 ValueAssignments.push_back({&Op, MCOperand::createImm(*RM)});511 printRoundingMode(SS << LS << (UsesVXRM ? "VXRM" : "FRM") << ": ", *RM,512 UsesVXRM);513 }514 515 if (AVL) {516 MCOperand OpVal;517 if (*AVL < 0) {518 // VLMAX519 OpVal = MCOperand::createImm(-1);520 SS << LS << "AVL: VLMAX";521 } else if (*AVL == 0) {522 // A register holding AVL.523 // TODO: Generate a random register.524 OpVal = MCOperand::createReg(RISCV::X5);525 OpVal.print(SS << LS << "AVL: ");526 } else {527 // A 5-bit immediate.528 // The actual value assignment is deferred to529 // RISCVExegesisTarget::randomizeTargetMCOperand.530 SS << LS << "AVL: simm5";531 }532 if (OpVal.isValid()) {533 const Operand &Op = Instr.Operands[RISCVII::getVLOpNum(MIDesc)];534 ValueAssignments.push_back({&Op, OpVal});535 }536 }537 538 if (Log2SEW) {539 const Operand &Op = Instr.Operands[RISCVII::getSEWOpNum(MIDesc)];540 ValueAssignments.push_back({&Op, MCOperand::createImm(*Log2SEW)});541 SS << LS << "SEW: e" << (*Log2SEW ? 1 << *Log2SEW : 8);542 }543 544 if (Policy) {545 const Operand &Op = Instr.Operands[RISCVII::getVecPolicyOpNum(MIDesc)];546 ValueAssignments.push_back({&Op, MCOperand::createImm(*Policy)});547 SS << LS548 << "Policy: " << (*Policy & RISCVVType::TAIL_AGNOSTIC ? "ta" : "tu")549 << "/" << (*Policy & RISCVVType::MASK_AGNOSTIC ? "ma" : "mu");550 }551 552 SS << "}";553 554 // Filter out some configurations, if needed.555 if (!FilterConfig.empty()) {556 if (!Regex(FilterConfig).match(ConfigStr))557 continue;558 }559 560 CodeTemplate CT = OrigCT.clone();561 CT.Config = std::move(ConfigStr);562 for (InstructionTemplate &IT : CT.Instructions) {563 if (IsSerial) {564 // Reset this template's value assignments and do it565 // ourselves.566 IT = InstructionTemplate(&Instr);567 assignSerialRVVOperands(IT);568 }569 570 for (const auto &[Op, OpVal] : ValueAssignments)571 IT.getValueFor(*Op) = OpVal;572 }573 Result.push_back(std::move(CT));574 if (Result.size() - StartingResultSize >=575 SnippetGenerator::Opts.MaxConfigsPerOpcode)576 return;577 }578}579 580template <class BaseT>581Expected<std::vector<CodeTemplate>>582RISCVSnippetGenerator<BaseT>::generateCodeTemplates(583 InstructionTemplate Variant, const BitVector &ForbiddenRegisters) const {584 const Instruction &Instr = Variant.getInstr();585 586 bool IsSerial = std::is_same_v<BaseT, SerialSnippetGenerator>;587 588 unsigned BaseOpcode = RISCV::getRVVMCOpcode(Instr.getOpcode());589 590 // Bail out ineligible opcodes before generating base code templates since591 // the latter is quite expensive.592 if (IsSerial && BaseOpcode && isIneligibleOfSerialSnippets(BaseOpcode, Instr))593 return std::vector<CodeTemplate>{};594 595 auto BaseCodeTemplates =596 BaseT::generateCodeTemplates(Variant, ForbiddenRegisters);597 if (!BaseCodeTemplates)598 return BaseCodeTemplates.takeError();599 600 if (!BaseOpcode)601 return BaseCodeTemplates;602 603 // Specialize for RVV pseudo.604 std::vector<CodeTemplate> ExpandedTemplates;605 for (const auto &BaseCT : *BaseCodeTemplates)606 annotateWithVType(BaseCT, Instr, BaseOpcode, ForbiddenRegisters,607 ExpandedTemplates);608 609 return ExpandedTemplates;610}611 612// Stores constant value to a general-purpose (integer) register.613static std::vector<MCInst> loadIntReg(const MCSubtargetInfo &STI,614 MCRegister Reg, const APInt &Value) {615 SmallVector<MCInst, 8> MCInstSeq;616 MCRegister DestReg = Reg;617 618 RISCVMatInt::generateMCInstSeq(Value.getSExtValue(), STI, DestReg, MCInstSeq);619 620 std::vector<MCInst> MatIntInstrs(MCInstSeq.begin(), MCInstSeq.end());621 return MatIntInstrs;622}623 624const MCPhysReg ScratchIntReg = RISCV::X30; // t5625 626// Stores constant bits to a floating-point register.627static std::vector<MCInst> loadFPRegBits(const MCSubtargetInfo &STI,628 MCRegister Reg, const APInt &Bits,629 unsigned FmvOpcode) {630 std::vector<MCInst> Instrs = loadIntReg(STI, ScratchIntReg, Bits);631 Instrs.push_back(MCInstBuilder(FmvOpcode).addReg(Reg).addReg(ScratchIntReg));632 return Instrs;633}634 635// main idea is:636// we support APInt only if (represented as double) it has zero fractional637// part: 1.0, 2.0, 3.0, etc... then we can do the trick: write int to tmp reg t5638// and then do FCVT this is only reliable thing in 32-bit mode, otherwise we639// need to use __floatsidf640static std::vector<MCInst> loadFP64RegBits32(const MCSubtargetInfo &STI,641 MCRegister Reg,642 const APInt &Bits) {643 double D = Bits.bitsToDouble();644 double IPart;645 double FPart = std::modf(D, &IPart);646 647 if (std::abs(FPart) > std::numeric_limits<double>::epsilon()) {648 errs() << "loadFP64RegBits32 is not implemented for doubles like " << D649 << ", please remove fractional part\n";650 return {};651 }652 653 std::vector<MCInst> Instrs = loadIntReg(STI, ScratchIntReg, Bits);654 Instrs.push_back(MCInstBuilder(RISCV::FCVT_D_W)655 .addReg(Reg)656 .addReg(ScratchIntReg)657 .addImm(RISCVFPRndMode::RNE));658 return Instrs;659}660 661class ExegesisRISCVTarget : public ExegesisTarget {662 // NOTE: Alternatively, we can use BitVector here, but the number of RVV MC663 // opcodes is just a small portion of the entire opcode space, so I thought it664 // would be a waste of space to use BitVector.665 mutable SmallSet<unsigned, 16> RVVMCOpcodesWithPseudos;666 667public:668 ExegesisRISCVTarget();669 670 bool matchesArch(Triple::ArchType Arch) const override;671 672 std::vector<MCInst> setRegTo(const MCSubtargetInfo &STI, MCRegister Reg,673 const APInt &Value) const override;674 675 const char *getIgnoredOpcodeReasonOrNull(const LLVMState &State,676 unsigned Opcode) const override {677 // We don't want to support RVV instructions that depend on VTYPE, because678 // those instructions by themselves don't carry any additional information679 // for us to setup the proper VTYPE environment via VSETVL instructions.680 // FIXME: Ideally, we should use RISCVVInversePseudosTable, but it requires681 // LMUL and SEW and I don't think enumerating those combinations is any682 // better than the ugly trick here that memorizes the corresponding MC683 // opcodes of the RVV pseudo we have processed previously. This works most684 // of the time because RVV pseudo opcodes are placed before any other RVV685 // opcodes. Of course this doesn't work if we're asked to benchmark only a686 // certain subset of opcodes.687 if (RVVMCOpcodesWithPseudos.count(Opcode))688 return "The MC opcode of RVV instructions are ignored";689 690 // We want to support all RVV pseudos.691 if (unsigned MCOpcode = RISCV::getRVVMCOpcode(Opcode)) {692 RVVMCOpcodesWithPseudos.insert(MCOpcode);693 return nullptr;694 }695 696 return ExegesisTarget::getIgnoredOpcodeReasonOrNull(State, Opcode);697 }698 699 MCRegister getDefaultLoopCounterRegister(const Triple &) const override;700 701 void decrementLoopCounterAndJump(MachineBasicBlock &MBB,702 MachineBasicBlock &TargetMBB,703 const MCInstrInfo &MII,704 MCRegister LoopRegister) const override;705 706 MCRegister getScratchMemoryRegister(const Triple &TT) const override;707 708 void fillMemoryOperands(InstructionTemplate &IT, MCRegister Reg,709 unsigned Offset) const override;710 711 ArrayRef<MCPhysReg> getUnavailableRegisters() const override;712 713 bool allowAsBackToBack(const Instruction &Instr) const override {714 return !Instr.Description.isPseudo();715 }716 717 Error randomizeTargetMCOperand(const Instruction &Instr, const Variable &Var,718 MCOperand &AssignedValue,719 const BitVector &ForbiddenRegs) const override;720 721 std::unique_ptr<SnippetGenerator> createSerialSnippetGenerator(722 const LLVMState &State,723 const SnippetGenerator::Options &Opts) const override {724 return std::make_unique<RISCVSnippetGenerator<SerialSnippetGenerator>>(725 State, Opts);726 }727 728 std::unique_ptr<SnippetGenerator> createParallelSnippetGenerator(729 const LLVMState &State,730 const SnippetGenerator::Options &Opts) const override {731 return std::make_unique<RISCVSnippetGenerator<ParallelSnippetGenerator>>(732 State, Opts);733 }734 735 std::vector<InstructionTemplate>736 generateInstructionVariants(const Instruction &Instr,737 unsigned MaxConfigsPerOpcode) const override;738 739 void addTargetSpecificPasses(PassManagerBase &PM) const override {740 // Turn AVL operand of physical registers into virtual registers.741 PM.add(exegesis::createRISCVPreprocessingPass());742 PM.add(createRISCVInsertVSETVLIPass());743 // Setting up the correct FRM.744 PM.add(createRISCVInsertReadWriteCSRPass());745 PM.add(createRISCVInsertWriteVXRMPass());746 // This will assign physical register to the result of VSETVLI instructions747 // that produce VLMAX.748 PM.add(exegesis::createRISCVPostprocessingPass());749 // PseudoRET will be expanded by RISCVAsmPrinter; we have to expand750 // PseudoMovImm with RISCVPostRAExpandPseudoPass though.751 PM.add(createRISCVPostRAExpandPseudoPass());752 }753};754 755ExegesisRISCVTarget::ExegesisRISCVTarget()756 : ExegesisTarget(RISCVCpuPfmCounters, RISCV_MC::isOpcodeAvailable) {}757 758bool ExegesisRISCVTarget::matchesArch(Triple::ArchType Arch) const {759 return Arch == Triple::riscv32 || Arch == Triple::riscv64;760}761 762std::vector<MCInst> ExegesisRISCVTarget::setRegTo(const MCSubtargetInfo &STI,763 MCRegister Reg,764 const APInt &Value) const {765 if (RISCV::GPRRegClass.contains(Reg))766 return loadIntReg(STI, Reg, Value);767 if (RISCV::FPR16RegClass.contains(Reg))768 return loadFPRegBits(STI, Reg, Value, RISCV::FMV_H_X);769 if (RISCV::FPR32RegClass.contains(Reg))770 return loadFPRegBits(STI, Reg, Value, RISCV::FMV_W_X);771 if (RISCV::FPR64RegClass.contains(Reg)) {772 if (STI.hasFeature(RISCV::Feature64Bit))773 return loadFPRegBits(STI, Reg, Value, RISCV::FMV_D_X);774 return loadFP64RegBits32(STI, Reg, Value);775 }776 // TODO: Emit proper code to initialize other kinds of registers.777 return {};778}779 780const MCPhysReg DefaultLoopCounterReg = RISCV::X31; // t6781const MCPhysReg ScratchMemoryReg = RISCV::X10; // a0782 783MCRegister784ExegesisRISCVTarget::getDefaultLoopCounterRegister(const Triple &) const {785 return DefaultLoopCounterReg;786}787 788void ExegesisRISCVTarget::decrementLoopCounterAndJump(789 MachineBasicBlock &MBB, MachineBasicBlock &TargetMBB,790 const MCInstrInfo &MII, MCRegister LoopRegister) const {791 BuildMI(&MBB, DebugLoc(), MII.get(RISCV::ADDI))792 .addDef(LoopRegister)793 .addUse(LoopRegister)794 .addImm(-1);795 BuildMI(&MBB, DebugLoc(), MII.get(RISCV::BNE))796 .addUse(LoopRegister)797 .addUse(RISCV::X0)798 .addMBB(&TargetMBB);799}800 801MCRegister802ExegesisRISCVTarget::getScratchMemoryRegister(const Triple &TT) const {803 return ScratchMemoryReg; // a0804}805 806void ExegesisRISCVTarget::fillMemoryOperands(InstructionTemplate &IT,807 MCRegister Reg,808 unsigned Offset) const {809 // TODO: for now we ignore Offset because have no way810 // to detect it in instruction.811 auto &I = IT.getInstr();812 813 auto MemOpIt =814 find_if(I.Operands, [](const Operand &Op) { return Op.isMemory(); });815 assert(MemOpIt != I.Operands.end() &&816 "Instruction must have memory operands");817 818 const Operand &MemOp = *MemOpIt;819 820 assert(MemOp.isReg() && "Memory operand expected to be register");821 822 IT.getValueFor(MemOp) = MCOperand::createReg(Reg);823}824 825const MCPhysReg UnavailableRegisters[4] = {RISCV::X0, DefaultLoopCounterReg,826 ScratchIntReg, ScratchMemoryReg};827 828ArrayRef<MCPhysReg> ExegesisRISCVTarget::getUnavailableRegisters() const {829 return UnavailableRegisters;830}831 832Error ExegesisRISCVTarget::randomizeTargetMCOperand(833 const Instruction &Instr, const Variable &Var, MCOperand &AssignedValue,834 const BitVector &ForbiddenRegs) const {835 uint8_t OperandType =836 Instr.getPrimaryOperand(Var).getExplicitOperandInfo().OperandType;837 838 switch (OperandType) {839 case RISCVOp::OPERAND_FRMARG:840 AssignedValue = MCOperand::createImm(RISCVFPRndMode::DYN);841 break;842 case RISCVOp::OPERAND_SIMM10_LSB0000_NONZERO:843 AssignedValue = MCOperand::createImm(0b1 << 4);844 break;845 case RISCVOp::OPERAND_SIMM6_NONZERO:846 case RISCVOp::OPERAND_UIMMLOG2XLEN_NONZERO:847 AssignedValue = MCOperand::createImm(1);848 break;849 case RISCVOp::OPERAND_SIMM5:850 // 5-bit signed immediate value.851 AssignedValue = MCOperand::createImm(randomIndex(31) - 16);852 break;853 case RISCVOp::OPERAND_AVL:854 case RISCVOp::OPERAND_UIMM5:855 // 5-bit unsigned immediate value.856 AssignedValue = MCOperand::createImm(randomIndex(31));857 break;858 default:859 if (OperandType >= RISCVOp::OPERAND_FIRST_RISCV_IMM &&860 OperandType <= RISCVOp::OPERAND_LAST_RISCV_IMM)861 AssignedValue = MCOperand::createImm(0);862 }863 return Error::success();864}865 866std::vector<InstructionTemplate>867ExegesisRISCVTarget::generateInstructionVariants(868 const Instruction &Instr, unsigned int MaxConfigsPerOpcode) const {869 InstructionTemplate IT{&Instr};870 for (const Operand &Op : Instr.Operands)871 if (Op.isMemory()) {872 IT.getValueFor(Op) = MCOperand::createReg(ScratchMemoryReg);873 }874 return {IT};875}876 877} // anonymous namespace878 879static ExegesisTarget *getTheRISCVExegesisTarget() {880 static ExegesisRISCVTarget Target;881 return &Target;882}883 884void InitializeRISCVExegesisTarget() {885 ExegesisTarget::registerTarget(getTheRISCVExegesisTarget());886}887 888} // namespace exegesis889} // namespace llvm890