1850 lines · cpp
1//===-- RISCVInstructionSelector.cpp -----------------------------*- C++ -*-==//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8/// \file9/// This file implements the targeting of the InstructionSelector class for10/// RISC-V.11/// \todo This should be generated by TableGen.12//===----------------------------------------------------------------------===//13 14#include "MCTargetDesc/RISCVMatInt.h"15#include "RISCVRegisterBankInfo.h"16#include "RISCVSubtarget.h"17#include "RISCVTargetMachine.h"18#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"19#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"20#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"21#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"22#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"23#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"24#include "llvm/CodeGen/MachineJumpTableInfo.h"25#include "llvm/IR/IntrinsicsRISCV.h"26#include "llvm/Support/Debug.h"27 28#define DEBUG_TYPE "riscv-isel"29 30using namespace llvm;31using namespace MIPatternMatch;32 33#define GET_GLOBALISEL_PREDICATE_BITSET34#include "RISCVGenGlobalISel.inc"35#undef GET_GLOBALISEL_PREDICATE_BITSET36 37namespace {38 39class RISCVInstructionSelector : public InstructionSelector {40public:41 RISCVInstructionSelector(const RISCVTargetMachine &TM,42 const RISCVSubtarget &STI,43 const RISCVRegisterBankInfo &RBI);44 45 bool select(MachineInstr &MI) override;46 47 void setupMF(MachineFunction &MF, GISelValueTracking *VT,48 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,49 BlockFrequencyInfo *BFI) override {50 InstructionSelector::setupMF(MF, VT, CoverageInfo, PSI, BFI);51 MRI = &MF.getRegInfo();52 }53 54 static const char *getName() { return DEBUG_TYPE; }55 56private:57 const TargetRegisterClass *58 getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB) const;59 60 static constexpr unsigned MaxRecursionDepth = 6;61 62 bool hasAllNBitUsers(const MachineInstr &MI, unsigned Bits,63 const unsigned Depth = 0) const;64 bool hasAllHUsers(const MachineInstr &MI) const {65 return hasAllNBitUsers(MI, 16);66 }67 bool hasAllWUsers(const MachineInstr &MI) const {68 return hasAllNBitUsers(MI, 32);69 }70 71 bool isRegInGprb(Register Reg) const;72 bool isRegInFprb(Register Reg) const;73 74 // tblgen-erated 'select' implementation, used as the initial selector for75 // the patterns that don't require complex C++.76 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;77 78 // A lowering phase that runs before any selection attempts.79 // Returns true if the instruction was modified.80 void preISelLower(MachineInstr &MI, MachineIRBuilder &MIB);81 82 bool replacePtrWithInt(MachineOperand &Op, MachineIRBuilder &MIB);83 84 // Custom selection methods85 bool selectCopy(MachineInstr &MI) const;86 bool selectImplicitDef(MachineInstr &MI, MachineIRBuilder &MIB) const;87 bool materializeImm(Register Reg, int64_t Imm, MachineIRBuilder &MIB) const;88 bool selectAddr(MachineInstr &MI, MachineIRBuilder &MIB, bool IsLocal = true,89 bool IsExternWeak = false) const;90 bool selectSelect(MachineInstr &MI, MachineIRBuilder &MIB) const;91 bool selectFPCompare(MachineInstr &MI, MachineIRBuilder &MIB) const;92 void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,93 MachineIRBuilder &MIB) const;94 bool selectUnmergeValues(MachineInstr &MI, MachineIRBuilder &MIB) const;95 void addVectorLoadStoreOperands(MachineInstr &I,96 SmallVectorImpl<SrcOp> &SrcOps,97 unsigned &CurOp, bool IsMasked,98 bool IsStridedOrIndexed,99 LLT *IndexVT = nullptr) const;100 bool selectIntrinsicWithSideEffects(MachineInstr &I,101 MachineIRBuilder &MIB) const;102 103 ComplexRendererFns selectShiftMask(MachineOperand &Root,104 unsigned ShiftWidth) const;105 ComplexRendererFns selectShiftMaskXLen(MachineOperand &Root) const {106 return selectShiftMask(Root, STI.getXLen());107 }108 ComplexRendererFns selectShiftMask32(MachineOperand &Root) const {109 return selectShiftMask(Root, 32);110 }111 ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;112 113 ComplexRendererFns selectSExtBits(MachineOperand &Root, unsigned Bits) const;114 template <unsigned Bits>115 ComplexRendererFns selectSExtBits(MachineOperand &Root) const {116 return selectSExtBits(Root, Bits);117 }118 119 ComplexRendererFns selectZExtBits(MachineOperand &Root, unsigned Bits) const;120 template <unsigned Bits>121 ComplexRendererFns selectZExtBits(MachineOperand &Root) const {122 return selectZExtBits(Root, Bits);123 }124 125 ComplexRendererFns selectSHXADDOp(MachineOperand &Root, unsigned ShAmt) const;126 template <unsigned ShAmt>127 ComplexRendererFns selectSHXADDOp(MachineOperand &Root) const {128 return selectSHXADDOp(Root, ShAmt);129 }130 131 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root,132 unsigned ShAmt) const;133 template <unsigned ShAmt>134 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root) const {135 return selectSHXADD_UWOp(Root, ShAmt);136 }137 138 ComplexRendererFns renderVLOp(MachineOperand &Root) const;139 140 // Custom renderers for tablegen141 void renderNegImm(MachineInstrBuilder &MIB, const MachineInstr &MI,142 int OpIdx) const;143 void renderImmSubFromXLen(MachineInstrBuilder &MIB, const MachineInstr &MI,144 int OpIdx) const;145 void renderImmSubFrom32(MachineInstrBuilder &MIB, const MachineInstr &MI,146 int OpIdx) const;147 void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI,148 int OpIdx) const;149 void renderFrameIndex(MachineInstrBuilder &MIB, const MachineInstr &MI,150 int OpIdx) const;151 152 void renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI,153 int OpIdx) const;154 void renderXLenSubTrailingOnes(MachineInstrBuilder &MIB,155 const MachineInstr &MI, int OpIdx) const;156 157 void renderAddiPairImmLarge(MachineInstrBuilder &MIB, const MachineInstr &MI,158 int OpIdx) const;159 void renderAddiPairImmSmall(MachineInstrBuilder &MIB, const MachineInstr &MI,160 int OpIdx) const;161 162 const RISCVSubtarget &STI;163 const RISCVInstrInfo &TII;164 const RISCVRegisterInfo &TRI;165 const RISCVRegisterBankInfo &RBI;166 const RISCVTargetMachine &TM;167 168 MachineRegisterInfo *MRI = nullptr;169 170 // FIXME: This is necessary because DAGISel uses "Subtarget->" and GlobalISel171 // uses "STI." in the code generated by TableGen. We need to unify the name of172 // Subtarget variable.173 const RISCVSubtarget *Subtarget = &STI;174 175#define GET_GLOBALISEL_PREDICATES_DECL176#include "RISCVGenGlobalISel.inc"177#undef GET_GLOBALISEL_PREDICATES_DECL178 179#define GET_GLOBALISEL_TEMPORARIES_DECL180#include "RISCVGenGlobalISel.inc"181#undef GET_GLOBALISEL_TEMPORARIES_DECL182};183 184} // end anonymous namespace185 186#define GET_GLOBALISEL_IMPL187#include "RISCVGenGlobalISel.inc"188#undef GET_GLOBALISEL_IMPL189 190RISCVInstructionSelector::RISCVInstructionSelector(191 const RISCVTargetMachine &TM, const RISCVSubtarget &STI,192 const RISCVRegisterBankInfo &RBI)193 : STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI),194 TM(TM),195 196#define GET_GLOBALISEL_PREDICATES_INIT197#include "RISCVGenGlobalISel.inc"198#undef GET_GLOBALISEL_PREDICATES_INIT199#define GET_GLOBALISEL_TEMPORARIES_INIT200#include "RISCVGenGlobalISel.inc"201#undef GET_GLOBALISEL_TEMPORARIES_INIT202{203}204 205// Mimics optimizations in ISel and RISCVOptWInst Pass206bool RISCVInstructionSelector::hasAllNBitUsers(const MachineInstr &MI,207 unsigned Bits,208 const unsigned Depth) const {209 210 assert((MI.getOpcode() == TargetOpcode::G_ADD ||211 MI.getOpcode() == TargetOpcode::G_SUB ||212 MI.getOpcode() == TargetOpcode::G_MUL ||213 MI.getOpcode() == TargetOpcode::G_SHL ||214 MI.getOpcode() == TargetOpcode::G_LSHR ||215 MI.getOpcode() == TargetOpcode::G_AND ||216 MI.getOpcode() == TargetOpcode::G_OR ||217 MI.getOpcode() == TargetOpcode::G_XOR ||218 MI.getOpcode() == TargetOpcode::G_SEXT_INREG || Depth != 0) &&219 "Unexpected opcode");220 221 if (Depth >= RISCVInstructionSelector::MaxRecursionDepth)222 return false;223 224 auto DestReg = MI.getOperand(0).getReg();225 for (auto &UserOp : MRI->use_nodbg_operands(DestReg)) {226 assert(UserOp.getParent() && "UserOp must have a parent");227 const MachineInstr &UserMI = *UserOp.getParent();228 unsigned OpIdx = UserOp.getOperandNo();229 230 switch (UserMI.getOpcode()) {231 default:232 return false;233 case RISCV::ADDW:234 case RISCV::ADDIW:235 case RISCV::SUBW:236 case RISCV::FCVT_D_W:237 case RISCV::FCVT_S_W:238 if (Bits >= 32)239 break;240 return false;241 case RISCV::SLL:242 case RISCV::SRA:243 case RISCV::SRL:244 // Shift amount operands only use log2(Xlen) bits.245 if (OpIdx == 2 && Bits >= Log2_32(Subtarget->getXLen()))246 break;247 return false;248 case RISCV::SLLI:249 // SLLI only uses the lower (XLen - ShAmt) bits.250 if (Bits >= Subtarget->getXLen() - UserMI.getOperand(2).getImm())251 break;252 return false;253 case RISCV::ANDI:254 if (Bits >= (unsigned)llvm::bit_width<uint64_t>(255 (uint64_t)UserMI.getOperand(2).getImm()))256 break;257 goto RecCheck;258 case RISCV::AND:259 case RISCV::OR:260 case RISCV::XOR:261 RecCheck:262 if (hasAllNBitUsers(UserMI, Bits, Depth + 1))263 break;264 return false;265 case RISCV::SRLI: {266 unsigned ShAmt = UserMI.getOperand(2).getImm();267 // If we are shifting right by less than Bits, and users don't demand any268 // bits that were shifted into [Bits-1:0], then we can consider this as an269 // N-Bit user.270 if (Bits > ShAmt && hasAllNBitUsers(UserMI, Bits - ShAmt, Depth + 1))271 break;272 return false;273 }274 }275 }276 277 return true;278}279 280InstructionSelector::ComplexRendererFns281RISCVInstructionSelector::selectShiftMask(MachineOperand &Root,282 unsigned ShiftWidth) const {283 if (!Root.isReg())284 return std::nullopt;285 286 using namespace llvm::MIPatternMatch;287 288 Register ShAmtReg = Root.getReg();289 // Peek through zext.290 Register ZExtSrcReg;291 if (mi_match(ShAmtReg, *MRI, m_GZExt(m_Reg(ZExtSrcReg))))292 ShAmtReg = ZExtSrcReg;293 294 APInt AndMask;295 Register AndSrcReg;296 // Try to combine the following pattern (applicable to other shift297 // instructions as well as 32-bit ones):298 //299 // %4:gprb(s64) = G_AND %3, %2300 // %5:gprb(s64) = G_LSHR %1, %4(s64)301 //302 // According to RISC-V's ISA manual, SLL, SRL, and SRA ignore other bits than303 // the lowest log2(XLEN) bits of register rs2. As for the above pattern, if304 // the lowest log2(XLEN) bits of register rd and rs2 of G_AND are the same,305 // then it can be eliminated. Given register rs1 or rs2 holding a constant306 // (the and mask), there are two cases G_AND can be erased:307 //308 // 1. the lowest log2(XLEN) bits of the and mask are all set309 // 2. the bits of the register being masked are already unset (zero set)310 if (mi_match(ShAmtReg, *MRI, m_GAnd(m_Reg(AndSrcReg), m_ICst(AndMask)))) {311 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);312 if (ShMask.isSubsetOf(AndMask)) {313 ShAmtReg = AndSrcReg;314 } else {315 // SimplifyDemandedBits may have optimized the mask so try restoring any316 // bits that are known zero.317 KnownBits Known = VT->getKnownBits(AndSrcReg);318 if (ShMask.isSubsetOf(AndMask | Known.Zero))319 ShAmtReg = AndSrcReg;320 }321 }322 323 APInt Imm;324 Register Reg;325 if (mi_match(ShAmtReg, *MRI, m_GAdd(m_Reg(Reg), m_ICst(Imm)))) {326 if (Imm != 0 && Imm.urem(ShiftWidth) == 0)327 // If we are shifting by X+N where N == 0 mod Size, then just shift by X328 // to avoid the ADD.329 ShAmtReg = Reg;330 } else if (mi_match(ShAmtReg, *MRI, m_GSub(m_ICst(Imm), m_Reg(Reg)))) {331 if (Imm != 0 && Imm.urem(ShiftWidth) == 0) {332 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X333 // to generate a NEG instead of a SUB of a constant.334 ShAmtReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);335 unsigned NegOpc = Subtarget->is64Bit() ? RISCV::SUBW : RISCV::SUB;336 return {{[=](MachineInstrBuilder &MIB) {337 MachineIRBuilder(*MIB.getInstr())338 .buildInstr(NegOpc, {ShAmtReg}, {Register(RISCV::X0), Reg});339 MIB.addReg(ShAmtReg);340 }}};341 }342 if (Imm.urem(ShiftWidth) == ShiftWidth - 1) {343 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X344 // to generate a NOT instead of a SUB of a constant.345 ShAmtReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);346 return {{[=](MachineInstrBuilder &MIB) {347 MachineIRBuilder(*MIB.getInstr())348 .buildInstr(RISCV::XORI, {ShAmtReg}, {Reg})349 .addImm(-1);350 MIB.addReg(ShAmtReg);351 }}};352 }353 }354 355 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(ShAmtReg); }}};356}357 358InstructionSelector::ComplexRendererFns359RISCVInstructionSelector::selectSExtBits(MachineOperand &Root,360 unsigned Bits) const {361 if (!Root.isReg())362 return std::nullopt;363 Register RootReg = Root.getReg();364 MachineInstr *RootDef = MRI->getVRegDef(RootReg);365 366 if (RootDef->getOpcode() == TargetOpcode::G_SEXT_INREG &&367 RootDef->getOperand(2).getImm() == Bits) {368 return {369 {[=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); }}};370 }371 372 unsigned Size = MRI->getType(RootReg).getScalarSizeInBits();373 if ((Size - VT->computeNumSignBits(RootReg)) < Bits)374 return {{[=](MachineInstrBuilder &MIB) { MIB.add(Root); }}};375 376 return std::nullopt;377}378 379InstructionSelector::ComplexRendererFns380RISCVInstructionSelector::selectZExtBits(MachineOperand &Root,381 unsigned Bits) const {382 if (!Root.isReg())383 return std::nullopt;384 Register RootReg = Root.getReg();385 386 Register RegX;387 uint64_t Mask = maskTrailingOnes<uint64_t>(Bits);388 if (mi_match(RootReg, *MRI, m_GAnd(m_Reg(RegX), m_SpecificICst(Mask)))) {389 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegX); }}};390 }391 392 if (mi_match(RootReg, *MRI, m_GZExt(m_Reg(RegX))) &&393 MRI->getType(RegX).getScalarSizeInBits() == Bits)394 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegX); }}};395 396 unsigned Size = MRI->getType(RootReg).getScalarSizeInBits();397 if (VT->maskedValueIsZero(RootReg, APInt::getBitsSetFrom(Size, Bits)))398 return {{[=](MachineInstrBuilder &MIB) { MIB.add(Root); }}};399 400 return std::nullopt;401}402 403InstructionSelector::ComplexRendererFns404RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,405 unsigned ShAmt) const {406 using namespace llvm::MIPatternMatch;407 408 if (!Root.isReg())409 return std::nullopt;410 Register RootReg = Root.getReg();411 412 const unsigned XLen = STI.getXLen();413 APInt Mask, C2;414 Register RegY;415 std::optional<bool> LeftShift;416 // (and (shl y, c2), mask)417 if (mi_match(RootReg, *MRI,418 m_GAnd(m_GShl(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))419 LeftShift = true;420 // (and (lshr y, c2), mask)421 else if (mi_match(RootReg, *MRI,422 m_GAnd(m_GLShr(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))423 LeftShift = false;424 425 if (LeftShift.has_value()) {426 if (*LeftShift)427 Mask &= maskTrailingZeros<uint64_t>(C2.getLimitedValue());428 else429 Mask &= maskTrailingOnes<uint64_t>(XLen - C2.getLimitedValue());430 431 if (Mask.isShiftedMask()) {432 unsigned Leading = XLen - Mask.getActiveBits();433 unsigned Trailing = Mask.countr_zero();434 // Given (and (shl y, c2), mask) in which mask has no leading zeros and435 // c3 trailing zeros. We can use an SRLI by c3 - c2 followed by a SHXADD.436 if (*LeftShift && Leading == 0 && C2.ult(Trailing) && Trailing == ShAmt) {437 Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);438 return {{[=](MachineInstrBuilder &MIB) {439 MachineIRBuilder(*MIB.getInstr())440 .buildInstr(RISCV::SRLI, {DstReg}, {RegY})441 .addImm(Trailing - C2.getLimitedValue());442 MIB.addReg(DstReg);443 }}};444 }445 446 // Given (and (lshr y, c2), mask) in which mask has c2 leading zeros and447 // c3 trailing zeros. We can use an SRLI by c2 + c3 followed by a SHXADD.448 if (!*LeftShift && Leading == C2 && Trailing == ShAmt) {449 Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);450 return {{[=](MachineInstrBuilder &MIB) {451 MachineIRBuilder(*MIB.getInstr())452 .buildInstr(RISCV::SRLI, {DstReg}, {RegY})453 .addImm(Leading + Trailing);454 MIB.addReg(DstReg);455 }}};456 }457 }458 }459 460 LeftShift.reset();461 462 // (shl (and y, mask), c2)463 if (mi_match(RootReg, *MRI,464 m_GShl(m_OneNonDBGUse(m_GAnd(m_Reg(RegY), m_ICst(Mask))),465 m_ICst(C2))))466 LeftShift = true;467 // (lshr (and y, mask), c2)468 else if (mi_match(RootReg, *MRI,469 m_GLShr(m_OneNonDBGUse(m_GAnd(m_Reg(RegY), m_ICst(Mask))),470 m_ICst(C2))))471 LeftShift = false;472 473 if (LeftShift.has_value() && Mask.isShiftedMask()) {474 unsigned Leading = XLen - Mask.getActiveBits();475 unsigned Trailing = Mask.countr_zero();476 477 // Given (shl (and y, mask), c2) in which mask has 32 leading zeros and478 // c3 trailing zeros. If c1 + c3 == ShAmt, we can emit SRLIW + SHXADD.479 bool Cond = *LeftShift && Leading == 32 && Trailing > 0 &&480 (Trailing + C2.getLimitedValue()) == ShAmt;481 if (!Cond)482 // Given (lshr (and y, mask), c2) in which mask has 32 leading zeros and483 // c3 trailing zeros. If c3 - c1 == ShAmt, we can emit SRLIW + SHXADD.484 Cond = !*LeftShift && Leading == 32 && C2.ult(Trailing) &&485 (Trailing - C2.getLimitedValue()) == ShAmt;486 487 if (Cond) {488 Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);489 return {{[=](MachineInstrBuilder &MIB) {490 MachineIRBuilder(*MIB.getInstr())491 .buildInstr(RISCV::SRLIW, {DstReg}, {RegY})492 .addImm(Trailing);493 MIB.addReg(DstReg);494 }}};495 }496 }497 498 return std::nullopt;499}500 501InstructionSelector::ComplexRendererFns502RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,503 unsigned ShAmt) const {504 using namespace llvm::MIPatternMatch;505 506 if (!Root.isReg())507 return std::nullopt;508 Register RootReg = Root.getReg();509 510 // Given (and (shl x, c2), mask) in which mask is a shifted mask with511 // 32 - ShAmt leading zeros and c2 trailing zeros. We can use SLLI by512 // c2 - ShAmt followed by SHXADD_UW with ShAmt for x amount.513 APInt Mask, C2;514 Register RegX;515 if (mi_match(516 RootReg, *MRI,517 m_OneNonDBGUse(m_GAnd(m_OneNonDBGUse(m_GShl(m_Reg(RegX), m_ICst(C2))),518 m_ICst(Mask))))) {519 Mask &= maskTrailingZeros<uint64_t>(C2.getLimitedValue());520 521 if (Mask.isShiftedMask()) {522 unsigned Leading = Mask.countl_zero();523 unsigned Trailing = Mask.countr_zero();524 if (Leading == 32 - ShAmt && C2 == Trailing && Trailing > ShAmt) {525 Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);526 return {{[=](MachineInstrBuilder &MIB) {527 MachineIRBuilder(*MIB.getInstr())528 .buildInstr(RISCV::SLLI, {DstReg}, {RegX})529 .addImm(C2.getLimitedValue() - ShAmt);530 MIB.addReg(DstReg);531 }}};532 }533 }534 }535 536 return std::nullopt;537}538 539InstructionSelector::ComplexRendererFns540RISCVInstructionSelector::renderVLOp(MachineOperand &Root) const {541 assert(Root.isReg() && "Expected operand to be a Register");542 MachineInstr *RootDef = MRI->getVRegDef(Root.getReg());543 544 if (RootDef->getOpcode() == TargetOpcode::G_CONSTANT) {545 auto C = RootDef->getOperand(1).getCImm();546 if (C->getValue().isAllOnes())547 // If the operand is a G_CONSTANT with value of all ones it is larger than548 // VLMAX. We convert it to an immediate with value VLMaxSentinel. This is549 // recognized specially by the vsetvli insertion pass.550 return {{[=](MachineInstrBuilder &MIB) {551 MIB.addImm(RISCV::VLMaxSentinel);552 }}};553 554 if (isUInt<5>(C->getZExtValue())) {555 uint64_t ZExtC = C->getZExtValue();556 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(ZExtC); }}};557 }558 }559 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); }}};560}561 562InstructionSelector::ComplexRendererFns563RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {564 if (!Root.isReg())565 return std::nullopt;566 567 MachineInstr *RootDef = MRI->getVRegDef(Root.getReg());568 if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {569 return {{570 [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },571 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },572 }};573 }574 575 if (isBaseWithConstantOffset(Root, *MRI)) {576 MachineOperand &LHS = RootDef->getOperand(1);577 MachineOperand &RHS = RootDef->getOperand(2);578 MachineInstr *LHSDef = MRI->getVRegDef(LHS.getReg());579 MachineInstr *RHSDef = MRI->getVRegDef(RHS.getReg());580 581 int64_t RHSC = RHSDef->getOperand(1).getCImm()->getSExtValue();582 if (isInt<12>(RHSC)) {583 if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)584 return {{585 [=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->getOperand(1)); },586 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },587 }};588 589 return {{[=](MachineInstrBuilder &MIB) { MIB.add(LHS); },590 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); }}};591 }592 }593 594 // TODO: Need to get the immediate from a G_PTR_ADD. Should this be done in595 // the combiner?596 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); },597 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }}};598}599 600/// Returns the RISCVCC::CondCode that corresponds to the CmpInst::Predicate CC.601/// CC Must be an ICMP Predicate.602static RISCVCC::CondCode getRISCVCCFromICmp(CmpInst::Predicate CC) {603 switch (CC) {604 default:605 llvm_unreachable("Expected ICMP CmpInst::Predicate.");606 case CmpInst::Predicate::ICMP_EQ:607 return RISCVCC::COND_EQ;608 case CmpInst::Predicate::ICMP_NE:609 return RISCVCC::COND_NE;610 case CmpInst::Predicate::ICMP_ULT:611 return RISCVCC::COND_LTU;612 case CmpInst::Predicate::ICMP_SLT:613 return RISCVCC::COND_LT;614 case CmpInst::Predicate::ICMP_UGE:615 return RISCVCC::COND_GEU;616 case CmpInst::Predicate::ICMP_SGE:617 return RISCVCC::COND_GE;618 }619}620 621static void getOperandsForBranch(Register CondReg, RISCVCC::CondCode &CC,622 Register &LHS, Register &RHS,623 MachineRegisterInfo &MRI) {624 // Try to fold an ICmp. If that fails, use a NE compare with X0.625 CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;626 if (!mi_match(CondReg, MRI, m_GICmp(m_Pred(Pred), m_Reg(LHS), m_Reg(RHS)))) {627 LHS = CondReg;628 RHS = RISCV::X0;629 CC = RISCVCC::COND_NE;630 return;631 }632 633 // We found an ICmp, do some canonicalization.634 635 // Adjust comparisons to use comparison with 0 if possible.636 if (auto Constant = getIConstantVRegSExtVal(RHS, MRI)) {637 switch (Pred) {638 case CmpInst::Predicate::ICMP_SGT:639 // Convert X > -1 to X >= 0640 if (*Constant == -1) {641 CC = RISCVCC::COND_GE;642 RHS = RISCV::X0;643 return;644 }645 break;646 case CmpInst::Predicate::ICMP_SLT:647 // Convert X < 1 to 0 >= X648 if (*Constant == 1) {649 CC = RISCVCC::COND_GE;650 RHS = LHS;651 LHS = RISCV::X0;652 return;653 }654 break;655 default:656 break;657 }658 }659 660 switch (Pred) {661 default:662 llvm_unreachable("Expected ICMP CmpInst::Predicate.");663 case CmpInst::Predicate::ICMP_EQ:664 case CmpInst::Predicate::ICMP_NE:665 case CmpInst::Predicate::ICMP_ULT:666 case CmpInst::Predicate::ICMP_SLT:667 case CmpInst::Predicate::ICMP_UGE:668 case CmpInst::Predicate::ICMP_SGE:669 // These CCs are supported directly by RISC-V branches.670 break;671 case CmpInst::Predicate::ICMP_SGT:672 case CmpInst::Predicate::ICMP_SLE:673 case CmpInst::Predicate::ICMP_UGT:674 case CmpInst::Predicate::ICMP_ULE:675 // These CCs are not supported directly by RISC-V branches, but changing the676 // direction of the CC and swapping LHS and RHS are.677 Pred = CmpInst::getSwappedPredicate(Pred);678 std::swap(LHS, RHS);679 break;680 }681 682 CC = getRISCVCCFromICmp(Pred);683}684 685/// Select the RISC-V Zalasr opcode for the G_LOAD or G_STORE operation686/// \p GenericOpc, appropriate for the GPR register bank and of memory access687/// size \p OpSize.688static unsigned selectZalasrLoadStoreOp(unsigned GenericOpc, unsigned OpSize) {689 const bool IsStore = GenericOpc == TargetOpcode::G_STORE;690 switch (OpSize) {691 default:692 llvm_unreachable("Unexpected memory size");693 case 8:694 return IsStore ? RISCV::SB_RL : RISCV::LB_AQ;695 case 16:696 return IsStore ? RISCV::SH_RL : RISCV::LH_AQ;697 case 32:698 return IsStore ? RISCV::SW_RL : RISCV::LW_AQ;699 case 64:700 return IsStore ? RISCV::SD_RL : RISCV::LD_AQ;701 }702}703 704/// Select the RISC-V regimm opcode for the G_LOAD or G_STORE operation705/// \p GenericOpc, appropriate for the GPR register bank and of memory access706/// size \p OpSize. \returns \p GenericOpc if the combination is unsupported.707static unsigned selectRegImmLoadStoreOp(unsigned GenericOpc, unsigned OpSize) {708 const bool IsStore = GenericOpc == TargetOpcode::G_STORE;709 switch (OpSize) {710 case 8:711 // Prefer unsigned due to no c.lb in Zcb.712 return IsStore ? RISCV::SB : RISCV::LBU;713 case 16:714 return IsStore ? RISCV::SH : RISCV::LH;715 case 32:716 return IsStore ? RISCV::SW : RISCV::LW;717 case 64:718 return IsStore ? RISCV::SD : RISCV::LD;719 }720 721 return GenericOpc;722}723 724void RISCVInstructionSelector::addVectorLoadStoreOperands(725 MachineInstr &I, SmallVectorImpl<SrcOp> &SrcOps, unsigned &CurOp,726 bool IsMasked, bool IsStridedOrIndexed, LLT *IndexVT) const {727 // Base Pointer728 auto PtrReg = I.getOperand(CurOp++).getReg();729 SrcOps.push_back(PtrReg);730 731 // Stride or Index732 if (IsStridedOrIndexed) {733 auto StrideReg = I.getOperand(CurOp++).getReg();734 SrcOps.push_back(StrideReg);735 if (IndexVT)736 *IndexVT = MRI->getType(StrideReg);737 }738 739 // Mask740 if (IsMasked) {741 auto MaskReg = I.getOperand(CurOp++).getReg();742 SrcOps.push_back(MaskReg);743 }744}745 746bool RISCVInstructionSelector::selectIntrinsicWithSideEffects(747 MachineInstr &I, MachineIRBuilder &MIB) const {748 // Find the intrinsic ID.749 unsigned IntrinID = cast<GIntrinsic>(I).getIntrinsicID();750 // Select the instruction.751 switch (IntrinID) {752 default:753 return false;754 case Intrinsic::riscv_vlm:755 case Intrinsic::riscv_vle:756 case Intrinsic::riscv_vle_mask:757 case Intrinsic::riscv_vlse:758 case Intrinsic::riscv_vlse_mask: {759 bool IsMasked = IntrinID == Intrinsic::riscv_vle_mask ||760 IntrinID == Intrinsic::riscv_vlse_mask;761 bool IsStrided = IntrinID == Intrinsic::riscv_vlse ||762 IntrinID == Intrinsic::riscv_vlse_mask;763 LLT VT = MRI->getType(I.getOperand(0).getReg());764 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());765 766 // Result vector767 const Register DstReg = I.getOperand(0).getReg();768 769 // Sources770 bool HasPassthruOperand = IntrinID != Intrinsic::riscv_vlm;771 unsigned CurOp = 2;772 SmallVector<SrcOp, 4> SrcOps; // Source registers.773 774 // Passthru775 if (HasPassthruOperand) {776 auto PassthruReg = I.getOperand(CurOp++).getReg();777 SrcOps.push_back(PassthruReg);778 } else {779 SrcOps.push_back(Register(RISCV::NoRegister));780 }781 782 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStrided);783 784 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(getMVTForLLT(VT));785 const RISCV::VLEPseudo *P =786 RISCV::getVLEPseudo(IsMasked, IsStrided, /*FF*/ false, Log2SEW,787 static_cast<unsigned>(LMUL));788 789 auto PseudoMI = MIB.buildInstr(P->Pseudo, {DstReg}, SrcOps);790 791 // Select VL792 auto VLOpFn = renderVLOp(I.getOperand(CurOp++));793 for (auto &RenderFn : *VLOpFn)794 RenderFn(PseudoMI);795 796 // SEW797 PseudoMI.addImm(Log2SEW);798 799 // Policy800 uint64_t Policy = RISCVVType::MASK_AGNOSTIC;801 if (IsMasked)802 Policy = I.getOperand(CurOp++).getImm();803 PseudoMI.addImm(Policy);804 805 // Memref806 PseudoMI.cloneMemRefs(I);807 808 I.eraseFromParent();809 return constrainSelectedInstRegOperands(*PseudoMI, TII, TRI, RBI);810 }811 case Intrinsic::riscv_vloxei:812 case Intrinsic::riscv_vloxei_mask:813 case Intrinsic::riscv_vluxei:814 case Intrinsic::riscv_vluxei_mask: {815 bool IsMasked = IntrinID == Intrinsic::riscv_vloxei_mask ||816 IntrinID == Intrinsic::riscv_vluxei_mask;817 bool IsOrdered = IntrinID == Intrinsic::riscv_vloxei ||818 IntrinID == Intrinsic::riscv_vloxei_mask;819 LLT VT = MRI->getType(I.getOperand(0).getReg());820 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());821 822 // Result vector823 const Register DstReg = I.getOperand(0).getReg();824 825 // Sources826 bool HasPassthruOperand = IntrinID != Intrinsic::riscv_vlm;827 unsigned CurOp = 2;828 SmallVector<SrcOp, 4> SrcOps; // Source registers.829 830 // Passthru831 if (HasPassthruOperand) {832 auto PassthruReg = I.getOperand(CurOp++).getReg();833 SrcOps.push_back(PassthruReg);834 } else {835 // Use NoRegister if there is no specified passthru.836 SrcOps.push_back(Register());837 }838 LLT IndexVT;839 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, true, &IndexVT);840 841 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(getMVTForLLT(VT));842 RISCVVType::VLMUL IndexLMUL =843 RISCVTargetLowering::getLMUL(getMVTForLLT(IndexVT));844 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());845 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {846 reportFatalUsageError("The V extension does not support EEW=64 for index "847 "values when XLEN=32");848 }849 const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(850 IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),851 static_cast<unsigned>(IndexLMUL));852 853 auto PseudoMI = MIB.buildInstr(P->Pseudo, {DstReg}, SrcOps);854 855 // Select VL856 auto VLOpFn = renderVLOp(I.getOperand(CurOp++));857 for (auto &RenderFn : *VLOpFn)858 RenderFn(PseudoMI);859 860 // SEW861 PseudoMI.addImm(Log2SEW);862 863 // Policy864 uint64_t Policy = RISCVVType::MASK_AGNOSTIC;865 if (IsMasked)866 Policy = I.getOperand(CurOp++).getImm();867 PseudoMI.addImm(Policy);868 869 // Memref870 PseudoMI.cloneMemRefs(I);871 872 I.eraseFromParent();873 return constrainSelectedInstRegOperands(*PseudoMI, TII, TRI, RBI);874 }875 case Intrinsic::riscv_vsm:876 case Intrinsic::riscv_vse:877 case Intrinsic::riscv_vse_mask:878 case Intrinsic::riscv_vsse:879 case Intrinsic::riscv_vsse_mask: {880 bool IsMasked = IntrinID == Intrinsic::riscv_vse_mask ||881 IntrinID == Intrinsic::riscv_vsse_mask;882 bool IsStrided = IntrinID == Intrinsic::riscv_vsse ||883 IntrinID == Intrinsic::riscv_vsse_mask;884 LLT VT = MRI->getType(I.getOperand(1).getReg());885 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());886 887 // Sources888 unsigned CurOp = 1;889 SmallVector<SrcOp, 4> SrcOps; // Source registers.890 891 // Store value892 auto PassthruReg = I.getOperand(CurOp++).getReg();893 SrcOps.push_back(PassthruReg);894 895 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStrided);896 897 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(getMVTForLLT(VT));898 const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(899 IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL));900 901 auto PseudoMI = MIB.buildInstr(P->Pseudo, {}, SrcOps);902 903 // Select VL904 auto VLOpFn = renderVLOp(I.getOperand(CurOp++));905 for (auto &RenderFn : *VLOpFn)906 RenderFn(PseudoMI);907 908 // SEW909 PseudoMI.addImm(Log2SEW);910 911 // Memref912 PseudoMI.cloneMemRefs(I);913 914 I.eraseFromParent();915 return constrainSelectedInstRegOperands(*PseudoMI, TII, TRI, RBI);916 }917 case Intrinsic::riscv_vsoxei:918 case Intrinsic::riscv_vsoxei_mask:919 case Intrinsic::riscv_vsuxei:920 case Intrinsic::riscv_vsuxei_mask: {921 bool IsMasked = IntrinID == Intrinsic::riscv_vsoxei_mask ||922 IntrinID == Intrinsic::riscv_vsuxei_mask;923 bool IsOrdered = IntrinID == Intrinsic::riscv_vsoxei ||924 IntrinID == Intrinsic::riscv_vsoxei_mask;925 LLT VT = MRI->getType(I.getOperand(1).getReg());926 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());927 928 // Sources929 unsigned CurOp = 1;930 SmallVector<SrcOp, 4> SrcOps; // Source registers.931 932 // Store value933 auto PassthruReg = I.getOperand(CurOp++).getReg();934 SrcOps.push_back(PassthruReg);935 936 LLT IndexVT;937 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, true, &IndexVT);938 939 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(getMVTForLLT(VT));940 RISCVVType::VLMUL IndexLMUL =941 RISCVTargetLowering::getLMUL(getMVTForLLT(IndexVT));942 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());943 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {944 reportFatalUsageError("The V extension does not support EEW=64 for index "945 "values when XLEN=32");946 }947 const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(948 IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),949 static_cast<unsigned>(IndexLMUL));950 951 auto PseudoMI = MIB.buildInstr(P->Pseudo, {}, SrcOps);952 953 // Select VL954 auto VLOpFn = renderVLOp(I.getOperand(CurOp++));955 for (auto &RenderFn : *VLOpFn)956 RenderFn(PseudoMI);957 958 // SEW959 PseudoMI.addImm(Log2SEW);960 961 // Memref962 PseudoMI.cloneMemRefs(I);963 964 I.eraseFromParent();965 return constrainSelectedInstRegOperands(*PseudoMI, TII, TRI, RBI);966 }967 }968}969 970bool RISCVInstructionSelector::select(MachineInstr &MI) {971 MachineIRBuilder MIB(MI);972 973 preISelLower(MI, MIB);974 const unsigned Opc = MI.getOpcode();975 976 if (!MI.isPreISelOpcode() || Opc == TargetOpcode::G_PHI) {977 if (Opc == TargetOpcode::PHI || Opc == TargetOpcode::G_PHI) {978 const Register DefReg = MI.getOperand(0).getReg();979 const LLT DefTy = MRI->getType(DefReg);980 981 const RegClassOrRegBank &RegClassOrBank =982 MRI->getRegClassOrRegBank(DefReg);983 984 const TargetRegisterClass *DefRC =985 dyn_cast<const TargetRegisterClass *>(RegClassOrBank);986 if (!DefRC) {987 if (!DefTy.isValid()) {988 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");989 return false;990 }991 992 const RegisterBank &RB = *cast<const RegisterBank *>(RegClassOrBank);993 DefRC = getRegClassForTypeOnBank(DefTy, RB);994 if (!DefRC) {995 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");996 return false;997 }998 }999 1000 MI.setDesc(TII.get(TargetOpcode::PHI));1001 return RBI.constrainGenericRegister(DefReg, *DefRC, *MRI);1002 }1003 1004 // Certain non-generic instructions also need some special handling.1005 if (MI.isCopy())1006 return selectCopy(MI);1007 1008 return true;1009 }1010 1011 if (selectImpl(MI, *CoverageInfo))1012 return true;1013 1014 switch (Opc) {1015 case TargetOpcode::G_ANYEXT:1016 case TargetOpcode::G_PTRTOINT:1017 case TargetOpcode::G_INTTOPTR:1018 case TargetOpcode::G_TRUNC:1019 case TargetOpcode::G_FREEZE:1020 return selectCopy(MI);1021 case TargetOpcode::G_CONSTANT: {1022 Register DstReg = MI.getOperand(0).getReg();1023 int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue();1024 1025 if (!materializeImm(DstReg, Imm, MIB))1026 return false;1027 1028 MI.eraseFromParent();1029 return true;1030 }1031 case TargetOpcode::G_ZEXT:1032 case TargetOpcode::G_SEXT: {1033 bool IsSigned = Opc != TargetOpcode::G_ZEXT;1034 Register DstReg = MI.getOperand(0).getReg();1035 Register SrcReg = MI.getOperand(1).getReg();1036 LLT SrcTy = MRI->getType(SrcReg);1037 unsigned SrcSize = SrcTy.getSizeInBits();1038 1039 if (SrcTy.isVector())1040 return false; // Should be handled by imported patterns.1041 1042 assert((*RBI.getRegBank(DstReg, *MRI, TRI)).getID() ==1043 RISCV::GPRBRegBankID &&1044 "Unexpected ext regbank");1045 1046 // Use addiw SrcReg, 0 (sext.w) for i32.1047 if (IsSigned && SrcSize == 32) {1048 MI.setDesc(TII.get(RISCV::ADDIW));1049 MI.addOperand(MachineOperand::CreateImm(0));1050 return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);1051 }1052 1053 // Use add.uw SrcReg, X0 (zext.w) for i32 with Zba.1054 if (!IsSigned && SrcSize == 32 && STI.hasStdExtZba()) {1055 MI.setDesc(TII.get(RISCV::ADD_UW));1056 MI.addOperand(MachineOperand::CreateReg(RISCV::X0, /*isDef=*/false));1057 return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);1058 }1059 1060 // Use sext.h/zext.h for i16 with Zbb.1061 if (SrcSize == 16 && STI.hasStdExtZbb()) {1062 MI.setDesc(TII.get(IsSigned ? RISCV::SEXT_H1063 : STI.isRV64() ? RISCV::ZEXT_H_RV641064 : RISCV::ZEXT_H_RV32));1065 return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);1066 }1067 1068 // Use pack(w) SrcReg, X0 for i16 zext with Zbkb.1069 if (!IsSigned && SrcSize == 16 && STI.hasStdExtZbkb()) {1070 MI.setDesc(TII.get(STI.is64Bit() ? RISCV::PACKW : RISCV::PACK));1071 MI.addOperand(MachineOperand::CreateReg(RISCV::X0, /*isDef=*/false));1072 return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);1073 }1074 1075 // Fall back to shift pair.1076 auto ShiftLeft =1077 MIB.buildInstr(RISCV::SLLI, {&RISCV::GPRRegClass}, {SrcReg})1078 .addImm(STI.getXLen() - SrcSize);1079 constrainSelectedInstRegOperands(*ShiftLeft, TII, TRI, RBI);1080 auto ShiftRight = MIB.buildInstr(IsSigned ? RISCV::SRAI : RISCV::SRLI,1081 {DstReg}, {ShiftLeft})1082 .addImm(STI.getXLen() - SrcSize);1083 constrainSelectedInstRegOperands(*ShiftRight, TII, TRI, RBI);1084 MI.eraseFromParent();1085 return true;1086 }1087 case TargetOpcode::G_FCONSTANT: {1088 // TODO: Use constant pool for complex constants.1089 Register DstReg = MI.getOperand(0).getReg();1090 const APFloat &FPimm = MI.getOperand(1).getFPImm()->getValueAPF();1091 unsigned Size = MRI->getType(DstReg).getSizeInBits();1092 if (Size == 16 || Size == 32 || (Size == 64 && Subtarget->is64Bit())) {1093 Register GPRReg;1094 if (FPimm.isPosZero()) {1095 GPRReg = RISCV::X0;1096 } else {1097 GPRReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);1098 APInt Imm = FPimm.bitcastToAPInt();1099 if (!materializeImm(GPRReg, Imm.getSExtValue(), MIB))1100 return false;1101 }1102 1103 unsigned Opcode = Size == 64 ? RISCV::FMV_D_X1104 : Size == 32 ? RISCV::FMV_W_X1105 : RISCV::FMV_H_X;1106 auto FMV = MIB.buildInstr(Opcode, {DstReg}, {GPRReg});1107 if (!FMV.constrainAllUses(TII, TRI, RBI))1108 return false;1109 } else {1110 // s64 on rv321111 assert(Size == 64 && !Subtarget->is64Bit() &&1112 "Unexpected size or subtarget");1113 1114 if (FPimm.isPosZero()) {1115 // Optimize +0.0 to use fcvt.d.w1116 MachineInstrBuilder FCVT =1117 MIB.buildInstr(RISCV::FCVT_D_W, {DstReg}, {Register(RISCV::X0)})1118 .addImm(RISCVFPRndMode::RNE);1119 if (!FCVT.constrainAllUses(TII, TRI, RBI))1120 return false;1121 1122 MI.eraseFromParent();1123 return true;1124 }1125 1126 // Split into two pieces and build through the stack.1127 Register GPRRegHigh = MRI->createVirtualRegister(&RISCV::GPRRegClass);1128 Register GPRRegLow = MRI->createVirtualRegister(&RISCV::GPRRegClass);1129 APInt Imm = FPimm.bitcastToAPInt();1130 if (!materializeImm(GPRRegHigh, Imm.extractBits(32, 32).getSExtValue(),1131 MIB))1132 return false;1133 if (!materializeImm(GPRRegLow, Imm.trunc(32).getSExtValue(), MIB))1134 return false;1135 MachineInstrBuilder PairF64 = MIB.buildInstr(1136 RISCV::BuildPairF64Pseudo, {DstReg}, {GPRRegLow, GPRRegHigh});1137 if (!PairF64.constrainAllUses(TII, TRI, RBI))1138 return false;1139 }1140 1141 MI.eraseFromParent();1142 return true;1143 }1144 case TargetOpcode::G_GLOBAL_VALUE: {1145 auto *GV = MI.getOperand(1).getGlobal();1146 if (GV->isThreadLocal()) {1147 // TODO: implement this case.1148 return false;1149 }1150 1151 return selectAddr(MI, MIB, GV->isDSOLocal(), GV->hasExternalWeakLinkage());1152 }1153 case TargetOpcode::G_JUMP_TABLE:1154 case TargetOpcode::G_CONSTANT_POOL:1155 return selectAddr(MI, MIB, MRI);1156 case TargetOpcode::G_BRCOND: {1157 Register LHS, RHS;1158 RISCVCC::CondCode CC;1159 getOperandsForBranch(MI.getOperand(0).getReg(), CC, LHS, RHS, *MRI);1160 1161 auto Bcc = MIB.buildInstr(RISCVCC::getBrCond(CC), {}, {LHS, RHS})1162 .addMBB(MI.getOperand(1).getMBB());1163 MI.eraseFromParent();1164 return constrainSelectedInstRegOperands(*Bcc, TII, TRI, RBI);1165 }1166 case TargetOpcode::G_BRINDIRECT:1167 MI.setDesc(TII.get(RISCV::PseudoBRIND));1168 MI.addOperand(MachineOperand::CreateImm(0));1169 return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);1170 case TargetOpcode::G_SELECT:1171 return selectSelect(MI, MIB);1172 case TargetOpcode::G_FCMP:1173 return selectFPCompare(MI, MIB);1174 case TargetOpcode::G_FENCE: {1175 AtomicOrdering FenceOrdering =1176 static_cast<AtomicOrdering>(MI.getOperand(0).getImm());1177 SyncScope::ID FenceSSID =1178 static_cast<SyncScope::ID>(MI.getOperand(1).getImm());1179 emitFence(FenceOrdering, FenceSSID, MIB);1180 MI.eraseFromParent();1181 return true;1182 }1183 case TargetOpcode::G_IMPLICIT_DEF:1184 return selectImplicitDef(MI, MIB);1185 case TargetOpcode::G_UNMERGE_VALUES:1186 return selectUnmergeValues(MI, MIB);1187 case TargetOpcode::G_LOAD:1188 case TargetOpcode::G_STORE: {1189 GLoadStore &LdSt = cast<GLoadStore>(MI);1190 const Register ValReg = LdSt.getReg(0);1191 const Register PtrReg = LdSt.getPointerReg();1192 LLT PtrTy = MRI->getType(PtrReg);1193 1194 const RegisterBank &RB = *RBI.getRegBank(ValReg, *MRI, TRI);1195 if (RB.getID() != RISCV::GPRBRegBankID)1196 return false;1197 1198#ifndef NDEBUG1199 const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, *MRI, TRI);1200 // Check that the pointer register is valid.1201 assert(PtrRB.getID() == RISCV::GPRBRegBankID &&1202 "Load/Store pointer operand isn't a GPR");1203 assert(PtrTy.isPointer() && "Load/Store pointer operand isn't a pointer");1204#endif1205 1206 // Can only handle AddressSpace 0.1207 if (PtrTy.getAddressSpace() != 0)1208 return false;1209 1210 unsigned MemSize = LdSt.getMemSizeInBits().getValue();1211 AtomicOrdering Order = LdSt.getMMO().getSuccessOrdering();1212 1213 if (isStrongerThanMonotonic(Order)) {1214 MI.setDesc(TII.get(selectZalasrLoadStoreOp(Opc, MemSize)));1215 return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);1216 }1217 1218 const unsigned NewOpc = selectRegImmLoadStoreOp(MI.getOpcode(), MemSize);1219 if (NewOpc == MI.getOpcode())1220 return false;1221 1222 // Check if we can fold anything into the addressing mode.1223 auto AddrModeFns = selectAddrRegImm(MI.getOperand(1));1224 if (!AddrModeFns)1225 return false;1226 1227 // Folded something. Create a new instruction and return it.1228 auto NewInst = MIB.buildInstr(NewOpc, {}, {}, MI.getFlags());1229 if (isa<GStore>(MI))1230 NewInst.addUse(ValReg);1231 else1232 NewInst.addDef(ValReg);1233 NewInst.cloneMemRefs(MI);1234 for (auto &Fn : *AddrModeFns)1235 Fn(NewInst);1236 MI.eraseFromParent();1237 1238 return constrainSelectedInstRegOperands(*NewInst, TII, TRI, RBI);1239 }1240 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:1241 return selectIntrinsicWithSideEffects(MI, MIB);1242 default:1243 return false;1244 }1245}1246 1247bool RISCVInstructionSelector::selectUnmergeValues(1248 MachineInstr &MI, MachineIRBuilder &MIB) const {1249 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);1250 1251 if (!Subtarget->hasStdExtZfa())1252 return false;1253 1254 // Split F64 Src into two s32 parts1255 if (MI.getNumOperands() != 3)1256 return false;1257 Register Src = MI.getOperand(2).getReg();1258 Register Lo = MI.getOperand(0).getReg();1259 Register Hi = MI.getOperand(1).getReg();1260 if (!isRegInFprb(Src) || !isRegInGprb(Lo) || !isRegInGprb(Hi))1261 return false;1262 1263 MachineInstr *ExtractLo = MIB.buildInstr(RISCV::FMV_X_W_FPR64, {Lo}, {Src});1264 if (!constrainSelectedInstRegOperands(*ExtractLo, TII, TRI, RBI))1265 return false;1266 1267 MachineInstr *ExtractHi = MIB.buildInstr(RISCV::FMVH_X_D, {Hi}, {Src});1268 if (!constrainSelectedInstRegOperands(*ExtractHi, TII, TRI, RBI))1269 return false;1270 1271 MI.eraseFromParent();1272 return true;1273}1274 1275bool RISCVInstructionSelector::replacePtrWithInt(MachineOperand &Op,1276 MachineIRBuilder &MIB) {1277 Register PtrReg = Op.getReg();1278 assert(MRI->getType(PtrReg).isPointer() && "Operand is not a pointer!");1279 1280 const LLT sXLen = LLT::scalar(STI.getXLen());1281 auto PtrToInt = MIB.buildPtrToInt(sXLen, PtrReg);1282 MRI->setRegBank(PtrToInt.getReg(0), RBI.getRegBank(RISCV::GPRBRegBankID));1283 Op.setReg(PtrToInt.getReg(0));1284 return select(*PtrToInt);1285}1286 1287void RISCVInstructionSelector::preISelLower(MachineInstr &MI,1288 MachineIRBuilder &MIB) {1289 switch (MI.getOpcode()) {1290 case TargetOpcode::G_PTR_ADD: {1291 Register DstReg = MI.getOperand(0).getReg();1292 const LLT sXLen = LLT::scalar(STI.getXLen());1293 1294 replacePtrWithInt(MI.getOperand(1), MIB);1295 MI.setDesc(TII.get(TargetOpcode::G_ADD));1296 MRI->setType(DstReg, sXLen);1297 break;1298 }1299 case TargetOpcode::G_PTRMASK: {1300 Register DstReg = MI.getOperand(0).getReg();1301 const LLT sXLen = LLT::scalar(STI.getXLen());1302 replacePtrWithInt(MI.getOperand(1), MIB);1303 MI.setDesc(TII.get(TargetOpcode::G_AND));1304 MRI->setType(DstReg, sXLen);1305 break;1306 }1307 }1308}1309 1310void RISCVInstructionSelector::renderNegImm(MachineInstrBuilder &MIB,1311 const MachineInstr &MI,1312 int OpIdx) const {1313 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&1314 "Expected G_CONSTANT");1315 int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();1316 MIB.addImm(-CstVal);1317}1318 1319void RISCVInstructionSelector::renderImmSubFromXLen(MachineInstrBuilder &MIB,1320 const MachineInstr &MI,1321 int OpIdx) const {1322 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&1323 "Expected G_CONSTANT");1324 uint64_t CstVal = MI.getOperand(1).getCImm()->getZExtValue();1325 MIB.addImm(STI.getXLen() - CstVal);1326}1327 1328void RISCVInstructionSelector::renderImmSubFrom32(MachineInstrBuilder &MIB,1329 const MachineInstr &MI,1330 int OpIdx) const {1331 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&1332 "Expected G_CONSTANT");1333 uint64_t CstVal = MI.getOperand(1).getCImm()->getZExtValue();1334 MIB.addImm(32 - CstVal);1335}1336 1337void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB,1338 const MachineInstr &MI,1339 int OpIdx) const {1340 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&1341 "Expected G_CONSTANT");1342 int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();1343 MIB.addImm(CstVal + 1);1344}1345 1346void RISCVInstructionSelector::renderFrameIndex(MachineInstrBuilder &MIB,1347 const MachineInstr &MI,1348 int OpIdx) const {1349 assert(MI.getOpcode() == TargetOpcode::G_FRAME_INDEX && OpIdx == -1 &&1350 "Expected G_FRAME_INDEX");1351 MIB.add(MI.getOperand(1));1352}1353 1354void RISCVInstructionSelector::renderTrailingZeros(MachineInstrBuilder &MIB,1355 const MachineInstr &MI,1356 int OpIdx) const {1357 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&1358 "Expected G_CONSTANT");1359 uint64_t C = MI.getOperand(1).getCImm()->getZExtValue();1360 MIB.addImm(llvm::countr_zero(C));1361}1362 1363void RISCVInstructionSelector::renderXLenSubTrailingOnes(1364 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {1365 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&1366 "Expected G_CONSTANT");1367 uint64_t C = MI.getOperand(1).getCImm()->getZExtValue();1368 MIB.addImm(Subtarget->getXLen() - llvm::countr_one(C));1369}1370 1371void RISCVInstructionSelector::renderAddiPairImmSmall(MachineInstrBuilder &MIB,1372 const MachineInstr &MI,1373 int OpIdx) const {1374 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&1375 "Expected G_CONSTANT");1376 int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue();1377 int64_t Adj = Imm < 0 ? -2048 : 2047;1378 MIB.addImm(Imm - Adj);1379}1380 1381void RISCVInstructionSelector::renderAddiPairImmLarge(MachineInstrBuilder &MIB,1382 const MachineInstr &MI,1383 int OpIdx) const {1384 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&1385 "Expected G_CONSTANT");1386 int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue() < 0 ? -2048 : 2047;1387 MIB.addImm(Imm);1388}1389 1390const TargetRegisterClass *RISCVInstructionSelector::getRegClassForTypeOnBank(1391 LLT Ty, const RegisterBank &RB) const {1392 if (RB.getID() == RISCV::GPRBRegBankID) {1393 if (Ty.getSizeInBits() <= 32 || (STI.is64Bit() && Ty.getSizeInBits() == 64))1394 return &RISCV::GPRRegClass;1395 }1396 1397 if (RB.getID() == RISCV::FPRBRegBankID) {1398 if (Ty.getSizeInBits() == 16)1399 return &RISCV::FPR16RegClass;1400 if (Ty.getSizeInBits() == 32)1401 return &RISCV::FPR32RegClass;1402 if (Ty.getSizeInBits() == 64)1403 return &RISCV::FPR64RegClass;1404 }1405 1406 if (RB.getID() == RISCV::VRBRegBankID) {1407 if (Ty.getSizeInBits().getKnownMinValue() <= 64)1408 return &RISCV::VRRegClass;1409 1410 if (Ty.getSizeInBits().getKnownMinValue() == 128)1411 return &RISCV::VRM2RegClass;1412 1413 if (Ty.getSizeInBits().getKnownMinValue() == 256)1414 return &RISCV::VRM4RegClass;1415 1416 if (Ty.getSizeInBits().getKnownMinValue() == 512)1417 return &RISCV::VRM8RegClass;1418 }1419 1420 return nullptr;1421}1422 1423bool RISCVInstructionSelector::isRegInGprb(Register Reg) const {1424 return RBI.getRegBank(Reg, *MRI, TRI)->getID() == RISCV::GPRBRegBankID;1425}1426 1427bool RISCVInstructionSelector::isRegInFprb(Register Reg) const {1428 return RBI.getRegBank(Reg, *MRI, TRI)->getID() == RISCV::FPRBRegBankID;1429}1430 1431bool RISCVInstructionSelector::selectCopy(MachineInstr &MI) const {1432 Register DstReg = MI.getOperand(0).getReg();1433 1434 if (DstReg.isPhysical())1435 return true;1436 1437 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(1438 MRI->getType(DstReg), *RBI.getRegBank(DstReg, *MRI, TRI));1439 assert(DstRC &&1440 "Register class not available for LLT, register bank combination");1441 1442 // No need to constrain SrcReg. It will get constrained when1443 // we hit another of its uses or its defs.1444 // Copies do not have constraints.1445 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) {1446 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())1447 << " operand\n");1448 return false;1449 }1450 1451 MI.setDesc(TII.get(RISCV::COPY));1452 return true;1453}1454 1455bool RISCVInstructionSelector::selectImplicitDef(MachineInstr &MI,1456 MachineIRBuilder &MIB) const {1457 assert(MI.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);1458 1459 const Register DstReg = MI.getOperand(0).getReg();1460 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(1461 MRI->getType(DstReg), *RBI.getRegBank(DstReg, *MRI, TRI));1462 1463 assert(DstRC &&1464 "Register class not available for LLT, register bank combination");1465 1466 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) {1467 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())1468 << " operand\n");1469 }1470 MI.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));1471 return true;1472}1473 1474bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,1475 MachineIRBuilder &MIB) const {1476 if (Imm == 0) {1477 MIB.buildCopy(DstReg, Register(RISCV::X0));1478 RBI.constrainGenericRegister(DstReg, RISCV::GPRRegClass, *MRI);1479 return true;1480 }1481 1482 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Imm, *Subtarget);1483 unsigned NumInsts = Seq.size();1484 Register SrcReg = RISCV::X0;1485 1486 for (unsigned i = 0; i < NumInsts; i++) {1487 Register TmpReg = i < NumInsts - 11488 ? MRI->createVirtualRegister(&RISCV::GPRRegClass)1489 : DstReg;1490 const RISCVMatInt::Inst &I = Seq[i];1491 MachineInstr *Result;1492 1493 switch (I.getOpndKind()) {1494 case RISCVMatInt::Imm:1495 // clang-format off1496 Result = MIB.buildInstr(I.getOpcode(), {TmpReg}, {})1497 .addImm(I.getImm());1498 // clang-format on1499 break;1500 case RISCVMatInt::RegX0:1501 Result = MIB.buildInstr(I.getOpcode(), {TmpReg},1502 {SrcReg, Register(RISCV::X0)});1503 break;1504 case RISCVMatInt::RegReg:1505 Result = MIB.buildInstr(I.getOpcode(), {TmpReg}, {SrcReg, SrcReg});1506 break;1507 case RISCVMatInt::RegImm:1508 Result =1509 MIB.buildInstr(I.getOpcode(), {TmpReg}, {SrcReg}).addImm(I.getImm());1510 break;1511 }1512 1513 if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))1514 return false;1515 1516 SrcReg = TmpReg;1517 }1518 1519 return true;1520}1521 1522bool RISCVInstructionSelector::selectAddr(MachineInstr &MI,1523 MachineIRBuilder &MIB, bool IsLocal,1524 bool IsExternWeak) const {1525 assert((MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||1526 MI.getOpcode() == TargetOpcode::G_JUMP_TABLE ||1527 MI.getOpcode() == TargetOpcode::G_CONSTANT_POOL) &&1528 "Unexpected opcode");1529 1530 const MachineOperand &DispMO = MI.getOperand(1);1531 1532 Register DefReg = MI.getOperand(0).getReg();1533 const LLT DefTy = MRI->getType(DefReg);1534 1535 // When HWASAN is used and tagging of global variables is enabled1536 // they should be accessed via the GOT, since the tagged address of a global1537 // is incompatible with existing code models. This also applies to non-pic1538 // mode.1539 if (TM.isPositionIndependent() || Subtarget->allowTaggedGlobals()) {1540 if (IsLocal && !Subtarget->allowTaggedGlobals()) {1541 // Use PC-relative addressing to access the symbol. This generates the1542 // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))1543 // %pcrel_lo(auipc)).1544 MI.setDesc(TII.get(RISCV::PseudoLLA));1545 return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);1546 }1547 1548 // Use PC-relative addressing to access the GOT for this symbol, then1549 // load the address from the GOT. This generates the pattern (PseudoLGA1550 // sym), which expands to (ld (addi (auipc %got_pcrel_hi(sym))1551 // %pcrel_lo(auipc))).1552 MachineFunction &MF = *MI.getParent()->getParent();1553 MachineMemOperand *MemOp = MF.getMachineMemOperand(1554 MachinePointerInfo::getGOT(MF),1555 MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |1556 MachineMemOperand::MOInvariant,1557 DefTy, Align(DefTy.getSizeInBits() / 8));1558 1559 auto Result = MIB.buildInstr(RISCV::PseudoLGA, {DefReg}, {})1560 .addDisp(DispMO, 0)1561 .addMemOperand(MemOp);1562 1563 if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))1564 return false;1565 1566 MI.eraseFromParent();1567 return true;1568 }1569 1570 switch (TM.getCodeModel()) {1571 default: {1572 reportGISelFailure(*MF, *TPC, *MORE, getName(),1573 "Unsupported code model for lowering", MI);1574 return false;1575 }1576 case CodeModel::Small: {1577 // Must lie within a single 2 GiB address range and must lie between1578 // absolute addresses -2 GiB and +2 GiB. This generates the pattern (addi1579 // (lui %hi(sym)) %lo(sym)).1580 Register AddrHiDest = MRI->createVirtualRegister(&RISCV::GPRRegClass);1581 MachineInstr *AddrHi = MIB.buildInstr(RISCV::LUI, {AddrHiDest}, {})1582 .addDisp(DispMO, 0, RISCVII::MO_HI);1583 1584 if (!constrainSelectedInstRegOperands(*AddrHi, TII, TRI, RBI))1585 return false;1586 1587 auto Result = MIB.buildInstr(RISCV::ADDI, {DefReg}, {AddrHiDest})1588 .addDisp(DispMO, 0, RISCVII::MO_LO);1589 1590 if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))1591 return false;1592 1593 MI.eraseFromParent();1594 return true;1595 }1596 case CodeModel::Medium:1597 // Emit LGA/LLA instead of the sequence it expands to because the pcrel_lo1598 // relocation needs to reference a label that points to the auipc1599 // instruction itself, not the global. This cannot be done inside the1600 // instruction selector.1601 if (IsExternWeak) {1602 // An extern weak symbol may be undefined, i.e. have value 0, which may1603 // not be within 2GiB of PC, so use GOT-indirect addressing to access the1604 // symbol. This generates the pattern (PseudoLGA sym), which expands to1605 // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).1606 MachineFunction &MF = *MI.getParent()->getParent();1607 MachineMemOperand *MemOp = MF.getMachineMemOperand(1608 MachinePointerInfo::getGOT(MF),1609 MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |1610 MachineMemOperand::MOInvariant,1611 DefTy, Align(DefTy.getSizeInBits() / 8));1612 1613 auto Result = MIB.buildInstr(RISCV::PseudoLGA, {DefReg}, {})1614 .addDisp(DispMO, 0)1615 .addMemOperand(MemOp);1616 1617 if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))1618 return false;1619 1620 MI.eraseFromParent();1621 return true;1622 }1623 1624 // Generate a sequence for accessing addresses within any 2GiB range1625 // within the address space. This generates the pattern (PseudoLLA sym),1626 // which expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).1627 MI.setDesc(TII.get(RISCV::PseudoLLA));1628 return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);1629 }1630 1631 return false;1632}1633 1634bool RISCVInstructionSelector::selectSelect(MachineInstr &MI,1635 MachineIRBuilder &MIB) const {1636 auto &SelectMI = cast<GSelect>(MI);1637 1638 Register LHS, RHS;1639 RISCVCC::CondCode CC;1640 getOperandsForBranch(SelectMI.getCondReg(), CC, LHS, RHS, *MRI);1641 1642 Register DstReg = SelectMI.getReg(0);1643 1644 unsigned Opc = RISCV::Select_GPR_Using_CC_GPR;1645 if (RBI.getRegBank(DstReg, *MRI, TRI)->getID() == RISCV::FPRBRegBankID) {1646 unsigned Size = MRI->getType(DstReg).getSizeInBits();1647 Opc = Size == 32 ? RISCV::Select_FPR32_Using_CC_GPR1648 : RISCV::Select_FPR64_Using_CC_GPR;1649 }1650 1651 MachineInstr *Result = MIB.buildInstr(Opc)1652 .addDef(DstReg)1653 .addReg(LHS)1654 .addReg(RHS)1655 .addImm(CC)1656 .addReg(SelectMI.getTrueReg())1657 .addReg(SelectMI.getFalseReg());1658 MI.eraseFromParent();1659 return constrainSelectedInstRegOperands(*Result, TII, TRI, RBI);1660}1661 1662// Convert an FCMP predicate to one of the supported F or D instructions.1663static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size) {1664 assert((Size == 16 || Size == 32 || Size == 64) && "Unsupported size");1665 switch (Pred) {1666 default:1667 llvm_unreachable("Unsupported predicate");1668 case CmpInst::FCMP_OLT:1669 return Size == 16 ? RISCV::FLT_H : Size == 32 ? RISCV::FLT_S : RISCV::FLT_D;1670 case CmpInst::FCMP_OLE:1671 return Size == 16 ? RISCV::FLE_H : Size == 32 ? RISCV::FLE_S : RISCV::FLE_D;1672 case CmpInst::FCMP_OEQ:1673 return Size == 16 ? RISCV::FEQ_H : Size == 32 ? RISCV::FEQ_S : RISCV::FEQ_D;1674 }1675}1676 1677// Try legalizing an FCMP by swapping or inverting the predicate to one that1678// is supported.1679static bool legalizeFCmpPredicate(Register &LHS, Register &RHS,1680 CmpInst::Predicate &Pred, bool &NeedInvert) {1681 auto isLegalFCmpPredicate = [](CmpInst::Predicate Pred) {1682 return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE ||1683 Pred == CmpInst::FCMP_OEQ;1684 };1685 1686 assert(!isLegalFCmpPredicate(Pred) && "Predicate already legal?");1687 1688 CmpInst::Predicate InvPred = CmpInst::getSwappedPredicate(Pred);1689 if (isLegalFCmpPredicate(InvPred)) {1690 Pred = InvPred;1691 std::swap(LHS, RHS);1692 return true;1693 }1694 1695 InvPred = CmpInst::getInversePredicate(Pred);1696 NeedInvert = true;1697 if (isLegalFCmpPredicate(InvPred)) {1698 Pred = InvPred;1699 return true;1700 }1701 InvPred = CmpInst::getSwappedPredicate(InvPred);1702 if (isLegalFCmpPredicate(InvPred)) {1703 Pred = InvPred;1704 std::swap(LHS, RHS);1705 return true;1706 }1707 1708 return false;1709}1710 1711// Emit a sequence of instructions to compare LHS and RHS using Pred. Return1712// the result in DstReg.1713// FIXME: Maybe we should expand this earlier.1714bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI,1715 MachineIRBuilder &MIB) const {1716 auto &CmpMI = cast<GFCmp>(MI);1717 CmpInst::Predicate Pred = CmpMI.getCond();1718 1719 Register DstReg = CmpMI.getReg(0);1720 Register LHS = CmpMI.getLHSReg();1721 Register RHS = CmpMI.getRHSReg();1722 1723 unsigned Size = MRI->getType(LHS).getSizeInBits();1724 assert((Size == 16 || Size == 32 || Size == 64) && "Unexpected size");1725 1726 Register TmpReg = DstReg;1727 1728 bool NeedInvert = false;1729 // First try swapping operands or inverting.1730 if (legalizeFCmpPredicate(LHS, RHS, Pred, NeedInvert)) {1731 if (NeedInvert)1732 TmpReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);1733 auto Cmp = MIB.buildInstr(getFCmpOpcode(Pred, Size), {TmpReg}, {LHS, RHS});1734 if (!Cmp.constrainAllUses(TII, TRI, RBI))1735 return false;1736 } else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UEQ) {1737 // fcmp one LHS, RHS => (OR (FLT LHS, RHS), (FLT RHS, LHS))1738 NeedInvert = Pred == CmpInst::FCMP_UEQ;1739 auto Cmp1 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OLT, Size),1740 {&RISCV::GPRRegClass}, {LHS, RHS});1741 if (!Cmp1.constrainAllUses(TII, TRI, RBI))1742 return false;1743 auto Cmp2 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OLT, Size),1744 {&RISCV::GPRRegClass}, {RHS, LHS});1745 if (!Cmp2.constrainAllUses(TII, TRI, RBI))1746 return false;1747 if (NeedInvert)1748 TmpReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);1749 auto Or =1750 MIB.buildInstr(RISCV::OR, {TmpReg}, {Cmp1.getReg(0), Cmp2.getReg(0)});1751 if (!Or.constrainAllUses(TII, TRI, RBI))1752 return false;1753 } else if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {1754 // fcmp ord LHS, RHS => (AND (FEQ LHS, LHS), (FEQ RHS, RHS))1755 // FIXME: If LHS and RHS are the same we can use a single FEQ.1756 NeedInvert = Pred == CmpInst::FCMP_UNO;1757 auto Cmp1 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OEQ, Size),1758 {&RISCV::GPRRegClass}, {LHS, LHS});1759 if (!Cmp1.constrainAllUses(TII, TRI, RBI))1760 return false;1761 auto Cmp2 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OEQ, Size),1762 {&RISCV::GPRRegClass}, {RHS, RHS});1763 if (!Cmp2.constrainAllUses(TII, TRI, RBI))1764 return false;1765 if (NeedInvert)1766 TmpReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);1767 auto And =1768 MIB.buildInstr(RISCV::AND, {TmpReg}, {Cmp1.getReg(0), Cmp2.getReg(0)});1769 if (!And.constrainAllUses(TII, TRI, RBI))1770 return false;1771 } else1772 llvm_unreachable("Unhandled predicate");1773 1774 // Emit an XORI to invert the result if needed.1775 if (NeedInvert) {1776 auto Xor = MIB.buildInstr(RISCV::XORI, {DstReg}, {TmpReg}).addImm(1);1777 if (!Xor.constrainAllUses(TII, TRI, RBI))1778 return false;1779 }1780 1781 MI.eraseFromParent();1782 return true;1783}1784 1785void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,1786 SyncScope::ID FenceSSID,1787 MachineIRBuilder &MIB) const {1788 if (STI.hasStdExtZtso()) {1789 // The only fence that needs an instruction is a sequentially-consistent1790 // cross-thread fence.1791 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&1792 FenceSSID == SyncScope::System) {1793 // fence rw, rw1794 MIB.buildInstr(RISCV::FENCE, {}, {})1795 .addImm(RISCVFenceField::R | RISCVFenceField::W)1796 .addImm(RISCVFenceField::R | RISCVFenceField::W);1797 return;1798 }1799 1800 // MEMBARRIER is a compiler barrier; it codegens to a no-op.1801 MIB.buildInstr(TargetOpcode::MEMBARRIER, {}, {});1802 return;1803 }1804 1805 // singlethread fences only synchronize with signal handlers on the same1806 // thread and thus only need to preserve instruction order, not actually1807 // enforce memory ordering.1808 if (FenceSSID == SyncScope::SingleThread) {1809 MIB.buildInstr(TargetOpcode::MEMBARRIER, {}, {});1810 return;1811 }1812 1813 // Refer to Table A.6 in the version 2.3 draft of the RISC-V Instruction Set1814 // Manual: Volume I.1815 unsigned Pred, Succ;1816 switch (FenceOrdering) {1817 default:1818 llvm_unreachable("Unexpected ordering");1819 case AtomicOrdering::AcquireRelease:1820 // fence acq_rel -> fence.tso1821 MIB.buildInstr(RISCV::FENCE_TSO, {}, {});1822 return;1823 case AtomicOrdering::Acquire:1824 // fence acquire -> fence r, rw1825 Pred = RISCVFenceField::R;1826 Succ = RISCVFenceField::R | RISCVFenceField::W;1827 break;1828 case AtomicOrdering::Release:1829 // fence release -> fence rw, w1830 Pred = RISCVFenceField::R | RISCVFenceField::W;1831 Succ = RISCVFenceField::W;1832 break;1833 case AtomicOrdering::SequentiallyConsistent:1834 // fence seq_cst -> fence rw, rw1835 Pred = RISCVFenceField::R | RISCVFenceField::W;1836 Succ = RISCVFenceField::R | RISCVFenceField::W;1837 break;1838 }1839 MIB.buildInstr(RISCV::FENCE, {}, {}).addImm(Pred).addImm(Succ);1840}1841 1842namespace llvm {1843InstructionSelector *1844createRISCVInstructionSelector(const RISCVTargetMachine &TM,1845 const RISCVSubtarget &Subtarget,1846 const RISCVRegisterBankInfo &RBI) {1847 return new RISCVInstructionSelector(TM, Subtarget, RBI);1848}1849} // end namespace llvm1850