2131 lines · cpp
1//===- llvm/CodeGen/GlobalISel/Utils.cpp -------------------------*- C++ -*-==//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8/// \file This file implements the utility functions used by the GlobalISel9/// pipeline.10//===----------------------------------------------------------------------===//11 12#include "llvm/CodeGen/GlobalISel/Utils.h"13#include "llvm/ADT/APFloat.h"14#include "llvm/ADT/APInt.h"15#include "llvm/Analysis/ValueTracking.h"16#include "llvm/CodeGen/CodeGenCommonISel.h"17#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"18#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"19#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"20#include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h"21#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"22#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"23#include "llvm/CodeGen/MachineInstr.h"24#include "llvm/CodeGen/MachineInstrBuilder.h"25#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"26#include "llvm/CodeGen/MachineRegisterInfo.h"27#include "llvm/CodeGen/MachineSizeOpts.h"28#include "llvm/CodeGen/RegisterBankInfo.h"29#include "llvm/CodeGen/StackProtector.h"30#include "llvm/CodeGen/TargetInstrInfo.h"31#include "llvm/CodeGen/TargetLowering.h"32#include "llvm/CodeGen/TargetOpcodes.h"33#include "llvm/CodeGen/TargetPassConfig.h"34#include "llvm/CodeGen/TargetRegisterInfo.h"35#include "llvm/IR/Constants.h"36#include "llvm/Target/TargetMachine.h"37#include "llvm/Transforms/Utils/SizeOpts.h"38#include <numeric>39#include <optional>40 41#define DEBUG_TYPE "globalisel-utils"42 43using namespace llvm;44using namespace MIPatternMatch;45 46Register llvm::constrainRegToClass(MachineRegisterInfo &MRI,47 const TargetInstrInfo &TII,48 const RegisterBankInfo &RBI, Register Reg,49 const TargetRegisterClass &RegClass) {50 if (!RBI.constrainGenericRegister(Reg, RegClass, MRI))51 return MRI.createVirtualRegister(&RegClass);52 53 return Reg;54}55 56Register llvm::constrainOperandRegClass(57 const MachineFunction &MF, const TargetRegisterInfo &TRI,58 MachineRegisterInfo &MRI, const TargetInstrInfo &TII,59 const RegisterBankInfo &RBI, MachineInstr &InsertPt,60 const TargetRegisterClass &RegClass, MachineOperand &RegMO) {61 Register Reg = RegMO.getReg();62 // Assume physical registers are properly constrained.63 assert(Reg.isVirtual() && "PhysReg not implemented");64 65 // Save the old register class to check whether66 // the change notifications will be required.67 // TODO: A better approach would be to pass68 // the observers to constrainRegToClass().69 auto *OldRegClass = MRI.getRegClassOrNull(Reg);70 Register ConstrainedReg = constrainRegToClass(MRI, TII, RBI, Reg, RegClass);71 // If we created a new virtual register because the class is not compatible72 // then create a copy between the new and the old register.73 if (ConstrainedReg != Reg) {74 MachineBasicBlock::iterator InsertIt(&InsertPt);75 MachineBasicBlock &MBB = *InsertPt.getParent();76 // FIXME: The copy needs to have the classes constrained for its operands.77 // Use operand's regbank to get the class for old register (Reg).78 if (RegMO.isUse()) {79 BuildMI(MBB, InsertIt, InsertPt.getDebugLoc(),80 TII.get(TargetOpcode::COPY), ConstrainedReg)81 .addReg(Reg);82 } else {83 assert(RegMO.isDef() && "Must be a definition");84 BuildMI(MBB, std::next(InsertIt), InsertPt.getDebugLoc(),85 TII.get(TargetOpcode::COPY), Reg)86 .addReg(ConstrainedReg);87 }88 if (GISelChangeObserver *Observer = MF.getObserver()) {89 Observer->changingInstr(*RegMO.getParent());90 }91 RegMO.setReg(ConstrainedReg);92 if (GISelChangeObserver *Observer = MF.getObserver()) {93 Observer->changedInstr(*RegMO.getParent());94 }95 } else if (OldRegClass != MRI.getRegClassOrNull(Reg)) {96 if (GISelChangeObserver *Observer = MF.getObserver()) {97 if (!RegMO.isDef()) {98 MachineInstr *RegDef = MRI.getVRegDef(Reg);99 Observer->changedInstr(*RegDef);100 }101 Observer->changingAllUsesOfReg(MRI, Reg);102 Observer->finishedChangingAllUsesOfReg();103 }104 }105 return ConstrainedReg;106}107 108Register llvm::constrainOperandRegClass(109 const MachineFunction &MF, const TargetRegisterInfo &TRI,110 MachineRegisterInfo &MRI, const TargetInstrInfo &TII,111 const RegisterBankInfo &RBI, MachineInstr &InsertPt, const MCInstrDesc &II,112 MachineOperand &RegMO, unsigned OpIdx) {113 Register Reg = RegMO.getReg();114 // Assume physical registers are properly constrained.115 assert(Reg.isVirtual() && "PhysReg not implemented");116 117 const TargetRegisterClass *OpRC = TII.getRegClass(II, OpIdx);118 // Some of the target independent instructions, like COPY, may not impose any119 // register class constraints on some of their operands: If it's a use, we can120 // skip constraining as the instruction defining the register would constrain121 // it.122 123 if (OpRC) {124 // Obtain the RC from incoming regbank if it is a proper sub-class. Operands125 // can have multiple regbanks for a superclass that combine different126 // register types (E.g., AMDGPU's VGPR and AGPR). The regbank ambiguity127 // resolved by targets during regbankselect should not be overridden.128 if (const auto *SubRC = TRI.getCommonSubClass(129 OpRC, TRI.getConstrainedRegClassForOperand(RegMO, MRI)))130 OpRC = SubRC;131 132 OpRC = TRI.getAllocatableClass(OpRC);133 }134 135 if (!OpRC) {136 assert((!isTargetSpecificOpcode(II.getOpcode()) || RegMO.isUse()) &&137 "Register class constraint is required unless either the "138 "instruction is target independent or the operand is a use");139 // FIXME: Just bailing out like this here could be not enough, unless we140 // expect the users of this function to do the right thing for PHIs and141 // COPY:142 // v1 = COPY v0143 // v2 = COPY v1144 // v1 here may end up not being constrained at all. Please notice that to145 // reproduce the issue we likely need a destination pattern of a selection146 // rule producing such extra copies, not just an input GMIR with them as147 // every existing target using selectImpl handles copies before calling it148 // and they never reach this function.149 return Reg;150 }151 return constrainOperandRegClass(MF, TRI, MRI, TII, RBI, InsertPt, *OpRC,152 RegMO);153}154 155bool llvm::constrainSelectedInstRegOperands(MachineInstr &I,156 const TargetInstrInfo &TII,157 const TargetRegisterInfo &TRI,158 const RegisterBankInfo &RBI) {159 assert(!isPreISelGenericOpcode(I.getOpcode()) &&160 "A selected instruction is expected");161 MachineBasicBlock &MBB = *I.getParent();162 MachineFunction &MF = *MBB.getParent();163 MachineRegisterInfo &MRI = MF.getRegInfo();164 165 for (unsigned OpI = 0, OpE = I.getNumExplicitOperands(); OpI != OpE; ++OpI) {166 MachineOperand &MO = I.getOperand(OpI);167 168 // There's nothing to be done on non-register operands.169 if (!MO.isReg())170 continue;171 172 LLVM_DEBUG(dbgs() << "Converting operand: " << MO << '\n');173 assert(MO.isReg() && "Unsupported non-reg operand");174 175 Register Reg = MO.getReg();176 // Physical registers don't need to be constrained.177 if (Reg.isPhysical())178 continue;179 180 // Register operands with a value of 0 (e.g. predicate operands) don't need181 // to be constrained.182 if (Reg == 0)183 continue;184 185 // If the operand is a vreg, we should constrain its regclass, and only186 // insert COPYs if that's impossible.187 // constrainOperandRegClass does that for us.188 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, I, I.getDesc(), MO, OpI);189 190 // Tie uses to defs as indicated in MCInstrDesc if this hasn't already been191 // done.192 if (MO.isUse()) {193 int DefIdx = I.getDesc().getOperandConstraint(OpI, MCOI::TIED_TO);194 if (DefIdx != -1 && !I.isRegTiedToUseOperand(DefIdx))195 I.tieOperands(DefIdx, OpI);196 }197 }198 return true;199}200 201bool llvm::canReplaceReg(Register DstReg, Register SrcReg,202 MachineRegisterInfo &MRI) {203 // Give up if either DstReg or SrcReg is a physical register.204 if (DstReg.isPhysical() || SrcReg.isPhysical())205 return false;206 // Give up if the types don't match.207 if (MRI.getType(DstReg) != MRI.getType(SrcReg))208 return false;209 // Replace if either DstReg has no constraints or the register210 // constraints match.211 const auto &DstRBC = MRI.getRegClassOrRegBank(DstReg);212 if (!DstRBC || DstRBC == MRI.getRegClassOrRegBank(SrcReg))213 return true;214 215 // Otherwise match if the Src is already a regclass that is covered by the Dst216 // RegBank.217 return isa<const RegisterBank *>(DstRBC) && MRI.getRegClassOrNull(SrcReg) &&218 cast<const RegisterBank *>(DstRBC)->covers(219 *MRI.getRegClassOrNull(SrcReg));220}221 222bool llvm::isTriviallyDead(const MachineInstr &MI,223 const MachineRegisterInfo &MRI) {224 // Instructions without side-effects are dead iff they only define dead regs.225 // This function is hot and this loop returns early in the common case,226 // so only perform additional checks before this if absolutely necessary.227 for (const auto &MO : MI.all_defs()) {228 Register Reg = MO.getReg();229 if (Reg.isPhysical() || !MRI.use_nodbg_empty(Reg))230 return false;231 }232 return MI.wouldBeTriviallyDead();233}234 235static void reportGISelDiagnostic(DiagnosticSeverity Severity,236 MachineFunction &MF,237 const TargetPassConfig &TPC,238 MachineOptimizationRemarkEmitter &MORE,239 MachineOptimizationRemarkMissed &R) {240 bool IsFatal = Severity == DS_Error &&241 TPC.isGlobalISelAbortEnabled();242 // Print the function name explicitly if we don't have a debug location (which243 // makes the diagnostic less useful) or if we're going to emit a raw error.244 if (!R.getLocation().isValid() || IsFatal)245 R << (" (in function: " + MF.getName() + ")").str();246 247 if (IsFatal)248 reportFatalUsageError(Twine(R.getMsg()));249 else250 MORE.emit(R);251}252 253void llvm::reportGISelWarning(MachineFunction &MF, const TargetPassConfig &TPC,254 MachineOptimizationRemarkEmitter &MORE,255 MachineOptimizationRemarkMissed &R) {256 reportGISelDiagnostic(DS_Warning, MF, TPC, MORE, R);257}258 259void llvm::reportGISelFailure(MachineFunction &MF, const TargetPassConfig &TPC,260 MachineOptimizationRemarkEmitter &MORE,261 MachineOptimizationRemarkMissed &R) {262 MF.getProperties().setFailedISel();263 reportGISelDiagnostic(DS_Error, MF, TPC, MORE, R);264}265 266void llvm::reportGISelFailure(MachineFunction &MF, const TargetPassConfig &TPC,267 MachineOptimizationRemarkEmitter &MORE,268 const char *PassName, StringRef Msg,269 const MachineInstr &MI) {270 MachineOptimizationRemarkMissed R(PassName, "GISelFailure: ",271 MI.getDebugLoc(), MI.getParent());272 R << Msg;273 // Printing MI is expensive; only do it if expensive remarks are enabled.274 if (TPC.isGlobalISelAbortEnabled() || MORE.allowExtraAnalysis(PassName))275 R << ": " << ore::MNV("Inst", MI);276 reportGISelFailure(MF, TPC, MORE, R);277}278 279unsigned llvm::getInverseGMinMaxOpcode(unsigned MinMaxOpc) {280 switch (MinMaxOpc) {281 case TargetOpcode::G_SMIN:282 return TargetOpcode::G_SMAX;283 case TargetOpcode::G_SMAX:284 return TargetOpcode::G_SMIN;285 case TargetOpcode::G_UMIN:286 return TargetOpcode::G_UMAX;287 case TargetOpcode::G_UMAX:288 return TargetOpcode::G_UMIN;289 default:290 llvm_unreachable("unrecognized opcode");291 }292}293 294std::optional<APInt> llvm::getIConstantVRegVal(Register VReg,295 const MachineRegisterInfo &MRI) {296 std::optional<ValueAndVReg> ValAndVReg = getIConstantVRegValWithLookThrough(297 VReg, MRI, /*LookThroughInstrs*/ false);298 assert((!ValAndVReg || ValAndVReg->VReg == VReg) &&299 "Value found while looking through instrs");300 if (!ValAndVReg)301 return std::nullopt;302 return ValAndVReg->Value;303}304 305const APInt &llvm::getIConstantFromReg(Register Reg,306 const MachineRegisterInfo &MRI) {307 MachineInstr *Const = MRI.getVRegDef(Reg);308 assert((Const && Const->getOpcode() == TargetOpcode::G_CONSTANT) &&309 "expected a G_CONSTANT on Reg");310 return Const->getOperand(1).getCImm()->getValue();311}312 313std::optional<int64_t>314llvm::getIConstantVRegSExtVal(Register VReg, const MachineRegisterInfo &MRI) {315 std::optional<APInt> Val = getIConstantVRegVal(VReg, MRI);316 if (Val && Val->getBitWidth() <= 64)317 return Val->getSExtValue();318 return std::nullopt;319}320 321namespace {322 323// This function is used in many places, and as such, it has some324// micro-optimizations to try and make it as fast as it can be.325//326// - We use template arguments to avoid an indirect call caused by passing a327// function_ref/std::function328// - GetAPCstValue does not return std::optional<APInt> as that's expensive.329// Instead it returns true/false and places the result in a pre-constructed330// APInt.331//332// Please change this function carefully and benchmark your changes.333template <bool (*IsConstantOpcode)(const MachineInstr *),334 bool (*GetAPCstValue)(const MachineInstr *MI, APInt &)>335std::optional<ValueAndVReg>336getConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI,337 bool LookThroughInstrs = true,338 bool LookThroughAnyExt = false) {339 SmallVector<std::pair<unsigned, unsigned>, 4> SeenOpcodes;340 MachineInstr *MI;341 342 while ((MI = MRI.getVRegDef(VReg)) && !IsConstantOpcode(MI) &&343 LookThroughInstrs) {344 switch (MI->getOpcode()) {345 case TargetOpcode::G_ANYEXT:346 if (!LookThroughAnyExt)347 return std::nullopt;348 [[fallthrough]];349 case TargetOpcode::G_TRUNC:350 case TargetOpcode::G_SEXT:351 case TargetOpcode::G_ZEXT:352 SeenOpcodes.push_back(std::make_pair(353 MI->getOpcode(),354 MRI.getType(MI->getOperand(0).getReg()).getSizeInBits()));355 VReg = MI->getOperand(1).getReg();356 break;357 case TargetOpcode::COPY:358 VReg = MI->getOperand(1).getReg();359 if (VReg.isPhysical())360 return std::nullopt;361 break;362 case TargetOpcode::G_INTTOPTR:363 VReg = MI->getOperand(1).getReg();364 break;365 default:366 return std::nullopt;367 }368 }369 if (!MI || !IsConstantOpcode(MI))370 return std::nullopt;371 372 APInt Val;373 if (!GetAPCstValue(MI, Val))374 return std::nullopt;375 for (auto &Pair : reverse(SeenOpcodes)) {376 switch (Pair.first) {377 case TargetOpcode::G_TRUNC:378 Val = Val.trunc(Pair.second);379 break;380 case TargetOpcode::G_ANYEXT:381 case TargetOpcode::G_SEXT:382 Val = Val.sext(Pair.second);383 break;384 case TargetOpcode::G_ZEXT:385 Val = Val.zext(Pair.second);386 break;387 }388 }389 390 return ValueAndVReg{std::move(Val), VReg};391}392 393bool isIConstant(const MachineInstr *MI) {394 if (!MI)395 return false;396 return MI->getOpcode() == TargetOpcode::G_CONSTANT;397}398 399bool isFConstant(const MachineInstr *MI) {400 if (!MI)401 return false;402 return MI->getOpcode() == TargetOpcode::G_FCONSTANT;403}404 405bool isAnyConstant(const MachineInstr *MI) {406 if (!MI)407 return false;408 unsigned Opc = MI->getOpcode();409 return Opc == TargetOpcode::G_CONSTANT || Opc == TargetOpcode::G_FCONSTANT;410}411 412bool getCImmAsAPInt(const MachineInstr *MI, APInt &Result) {413 const MachineOperand &CstVal = MI->getOperand(1);414 if (!CstVal.isCImm())415 return false;416 Result = CstVal.getCImm()->getValue();417 return true;418}419 420bool getCImmOrFPImmAsAPInt(const MachineInstr *MI, APInt &Result) {421 const MachineOperand &CstVal = MI->getOperand(1);422 if (CstVal.isCImm())423 Result = CstVal.getCImm()->getValue();424 else if (CstVal.isFPImm())425 Result = CstVal.getFPImm()->getValueAPF().bitcastToAPInt();426 else427 return false;428 return true;429}430 431} // end anonymous namespace432 433std::optional<ValueAndVReg> llvm::getIConstantVRegValWithLookThrough(434 Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs) {435 return getConstantVRegValWithLookThrough<isIConstant, getCImmAsAPInt>(436 VReg, MRI, LookThroughInstrs);437}438 439std::optional<ValueAndVReg> llvm::getAnyConstantVRegValWithLookThrough(440 Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs,441 bool LookThroughAnyExt) {442 return getConstantVRegValWithLookThrough<isAnyConstant,443 getCImmOrFPImmAsAPInt>(444 VReg, MRI, LookThroughInstrs, LookThroughAnyExt);445}446 447std::optional<FPValueAndVReg> llvm::getFConstantVRegValWithLookThrough(448 Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs) {449 auto Reg =450 getConstantVRegValWithLookThrough<isFConstant, getCImmOrFPImmAsAPInt>(451 VReg, MRI, LookThroughInstrs);452 if (!Reg)453 return std::nullopt;454 return FPValueAndVReg{getConstantFPVRegVal(Reg->VReg, MRI)->getValueAPF(),455 Reg->VReg};456}457 458const ConstantFP *459llvm::getConstantFPVRegVal(Register VReg, const MachineRegisterInfo &MRI) {460 MachineInstr *MI = MRI.getVRegDef(VReg);461 if (TargetOpcode::G_FCONSTANT != MI->getOpcode())462 return nullptr;463 return MI->getOperand(1).getFPImm();464}465 466std::optional<DefinitionAndSourceRegister>467llvm::getDefSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI) {468 Register DefSrcReg = Reg;469 // This assumes that the code is in SSA form, so there should only be one470 // definition.471 auto DefIt = MRI.def_begin(Reg);472 if (DefIt == MRI.def_end())473 return {};474 MachineOperand &DefOpnd = *DefIt;475 MachineInstr *DefMI = DefOpnd.getParent();476 auto DstTy = MRI.getType(DefOpnd.getReg());477 if (!DstTy.isValid())478 return std::nullopt;479 unsigned Opc = DefMI->getOpcode();480 while (Opc == TargetOpcode::COPY || isPreISelGenericOptimizationHint(Opc)) {481 Register SrcReg = DefMI->getOperand(1).getReg();482 auto SrcTy = MRI.getType(SrcReg);483 if (!SrcTy.isValid())484 break;485 DefMI = MRI.getVRegDef(SrcReg);486 DefSrcReg = SrcReg;487 Opc = DefMI->getOpcode();488 }489 return DefinitionAndSourceRegister{DefMI, DefSrcReg};490}491 492MachineInstr *llvm::getDefIgnoringCopies(Register Reg,493 const MachineRegisterInfo &MRI) {494 std::optional<DefinitionAndSourceRegister> DefSrcReg =495 getDefSrcRegIgnoringCopies(Reg, MRI);496 return DefSrcReg ? DefSrcReg->MI : nullptr;497}498 499Register llvm::getSrcRegIgnoringCopies(Register Reg,500 const MachineRegisterInfo &MRI) {501 std::optional<DefinitionAndSourceRegister> DefSrcReg =502 getDefSrcRegIgnoringCopies(Reg, MRI);503 return DefSrcReg ? DefSrcReg->Reg : Register();504}505 506void llvm::extractParts(Register Reg, LLT Ty, int NumParts,507 SmallVectorImpl<Register> &VRegs,508 MachineIRBuilder &MIRBuilder,509 MachineRegisterInfo &MRI) {510 for (int i = 0; i < NumParts; ++i)511 VRegs.push_back(MRI.createGenericVirtualRegister(Ty));512 MIRBuilder.buildUnmerge(VRegs, Reg);513}514 515bool llvm::extractParts(Register Reg, LLT RegTy, LLT MainTy, LLT &LeftoverTy,516 SmallVectorImpl<Register> &VRegs,517 SmallVectorImpl<Register> &LeftoverRegs,518 MachineIRBuilder &MIRBuilder,519 MachineRegisterInfo &MRI) {520 assert(!LeftoverTy.isValid() && "this is an out argument");521 522 unsigned RegSize = RegTy.getSizeInBits();523 unsigned MainSize = MainTy.getSizeInBits();524 unsigned NumParts = RegSize / MainSize;525 unsigned LeftoverSize = RegSize - NumParts * MainSize;526 527 // Use an unmerge when possible.528 if (LeftoverSize == 0) {529 for (unsigned I = 0; I < NumParts; ++I)530 VRegs.push_back(MRI.createGenericVirtualRegister(MainTy));531 MIRBuilder.buildUnmerge(VRegs, Reg);532 return true;533 }534 535 // Try to use unmerge for irregular vector split where possible536 // For example when splitting a <6 x i32> into <4 x i32> with <2 x i32>537 // leftover, it becomes:538 // <2 x i32> %2, <2 x i32>%3, <2 x i32> %4 = G_UNMERGE_VALUE <6 x i32> %1539 // <4 x i32> %5 = G_CONCAT_VECTOR <2 x i32> %2, <2 x i32> %3540 if (RegTy.isVector() && MainTy.isVector()) {541 unsigned RegNumElts = RegTy.getNumElements();542 unsigned MainNumElts = MainTy.getNumElements();543 unsigned LeftoverNumElts = RegNumElts % MainNumElts;544 // If can unmerge to LeftoverTy, do it545 if (MainNumElts % LeftoverNumElts == 0 &&546 RegNumElts % LeftoverNumElts == 0 &&547 RegTy.getScalarSizeInBits() == MainTy.getScalarSizeInBits() &&548 LeftoverNumElts > 1) {549 LeftoverTy = LLT::fixed_vector(LeftoverNumElts, RegTy.getElementType());550 551 // Unmerge the SrcReg to LeftoverTy vectors552 SmallVector<Register, 4> UnmergeValues;553 extractParts(Reg, LeftoverTy, RegNumElts / LeftoverNumElts, UnmergeValues,554 MIRBuilder, MRI);555 556 // Find how many LeftoverTy makes one MainTy557 unsigned LeftoverPerMain = MainNumElts / LeftoverNumElts;558 unsigned NumOfLeftoverVal =559 ((RegNumElts % MainNumElts) / LeftoverNumElts);560 561 // Create as many MainTy as possible using unmerged value562 SmallVector<Register, 4> MergeValues;563 for (unsigned I = 0; I < UnmergeValues.size() - NumOfLeftoverVal; I++) {564 MergeValues.push_back(UnmergeValues[I]);565 if (MergeValues.size() == LeftoverPerMain) {566 VRegs.push_back(567 MIRBuilder.buildMergeLikeInstr(MainTy, MergeValues).getReg(0));568 MergeValues.clear();569 }570 }571 // Populate LeftoverRegs with the leftovers572 for (unsigned I = UnmergeValues.size() - NumOfLeftoverVal;573 I < UnmergeValues.size(); I++) {574 LeftoverRegs.push_back(UnmergeValues[I]);575 }576 return true;577 }578 }579 // Perform irregular split. Leftover is last element of RegPieces.580 if (MainTy.isVector()) {581 SmallVector<Register, 8> RegPieces;582 extractVectorParts(Reg, MainTy.getNumElements(), RegPieces, MIRBuilder,583 MRI);584 for (unsigned i = 0; i < RegPieces.size() - 1; ++i)585 VRegs.push_back(RegPieces[i]);586 LeftoverRegs.push_back(RegPieces[RegPieces.size() - 1]);587 LeftoverTy = MRI.getType(LeftoverRegs[0]);588 return true;589 }590 591 LeftoverTy = LLT::scalar(LeftoverSize);592 // For irregular sizes, extract the individual parts.593 for (unsigned I = 0; I != NumParts; ++I) {594 Register NewReg = MRI.createGenericVirtualRegister(MainTy);595 VRegs.push_back(NewReg);596 MIRBuilder.buildExtract(NewReg, Reg, MainSize * I);597 }598 599 for (unsigned Offset = MainSize * NumParts; Offset < RegSize;600 Offset += LeftoverSize) {601 Register NewReg = MRI.createGenericVirtualRegister(LeftoverTy);602 LeftoverRegs.push_back(NewReg);603 MIRBuilder.buildExtract(NewReg, Reg, Offset);604 }605 606 return true;607}608 609void llvm::extractVectorParts(Register Reg, unsigned NumElts,610 SmallVectorImpl<Register> &VRegs,611 MachineIRBuilder &MIRBuilder,612 MachineRegisterInfo &MRI) {613 LLT RegTy = MRI.getType(Reg);614 assert(RegTy.isVector() && "Expected a vector type");615 616 LLT EltTy = RegTy.getElementType();617 LLT NarrowTy = (NumElts == 1) ? EltTy : LLT::fixed_vector(NumElts, EltTy);618 unsigned RegNumElts = RegTy.getNumElements();619 unsigned LeftoverNumElts = RegNumElts % NumElts;620 unsigned NumNarrowTyPieces = RegNumElts / NumElts;621 622 // Perfect split without leftover623 if (LeftoverNumElts == 0)624 return extractParts(Reg, NarrowTy, NumNarrowTyPieces, VRegs, MIRBuilder,625 MRI);626 627 // Irregular split. Provide direct access to all elements for artifact628 // combiner using unmerge to elements. Then build vectors with NumElts629 // elements. Remaining element(s) will be (used to build vector) Leftover.630 SmallVector<Register, 8> Elts;631 extractParts(Reg, EltTy, RegNumElts, Elts, MIRBuilder, MRI);632 633 unsigned Offset = 0;634 // Requested sub-vectors of NarrowTy.635 for (unsigned i = 0; i < NumNarrowTyPieces; ++i, Offset += NumElts) {636 ArrayRef<Register> Pieces(&Elts[Offset], NumElts);637 VRegs.push_back(MIRBuilder.buildMergeLikeInstr(NarrowTy, Pieces).getReg(0));638 }639 640 // Leftover element(s).641 if (LeftoverNumElts == 1) {642 VRegs.push_back(Elts[Offset]);643 } else {644 LLT LeftoverTy = LLT::fixed_vector(LeftoverNumElts, EltTy);645 ArrayRef<Register> Pieces(&Elts[Offset], LeftoverNumElts);646 VRegs.push_back(647 MIRBuilder.buildMergeLikeInstr(LeftoverTy, Pieces).getReg(0));648 }649}650 651MachineInstr *llvm::getOpcodeDef(unsigned Opcode, Register Reg,652 const MachineRegisterInfo &MRI) {653 MachineInstr *DefMI = getDefIgnoringCopies(Reg, MRI);654 return DefMI && DefMI->getOpcode() == Opcode ? DefMI : nullptr;655}656 657APFloat llvm::getAPFloatFromSize(double Val, unsigned Size) {658 if (Size == 32)659 return APFloat(float(Val));660 if (Size == 64)661 return APFloat(Val);662 if (Size != 16)663 llvm_unreachable("Unsupported FPConstant size");664 bool Ignored;665 APFloat APF(Val);666 APF.convert(APFloat::IEEEhalf(), APFloat::rmNearestTiesToEven, &Ignored);667 return APF;668}669 670std::optional<APInt> llvm::ConstantFoldBinOp(unsigned Opcode,671 const Register Op1,672 const Register Op2,673 const MachineRegisterInfo &MRI) {674 auto MaybeOp2Cst = getAnyConstantVRegValWithLookThrough(Op2, MRI, false);675 if (!MaybeOp2Cst)676 return std::nullopt;677 678 auto MaybeOp1Cst = getAnyConstantVRegValWithLookThrough(Op1, MRI, false);679 if (!MaybeOp1Cst)680 return std::nullopt;681 682 const APInt &C1 = MaybeOp1Cst->Value;683 const APInt &C2 = MaybeOp2Cst->Value;684 switch (Opcode) {685 default:686 break;687 case TargetOpcode::G_ADD:688 return C1 + C2;689 case TargetOpcode::G_PTR_ADD:690 // Types can be of different width here.691 // Result needs to be the same width as C1, so trunc or sext C2.692 return C1 + C2.sextOrTrunc(C1.getBitWidth());693 case TargetOpcode::G_AND:694 return C1 & C2;695 case TargetOpcode::G_ASHR:696 return C1.ashr(C2);697 case TargetOpcode::G_LSHR:698 return C1.lshr(C2);699 case TargetOpcode::G_MUL:700 return C1 * C2;701 case TargetOpcode::G_OR:702 return C1 | C2;703 case TargetOpcode::G_SHL:704 return C1 << C2;705 case TargetOpcode::G_SUB:706 return C1 - C2;707 case TargetOpcode::G_XOR:708 return C1 ^ C2;709 case TargetOpcode::G_UDIV:710 if (!C2.getBoolValue())711 break;712 return C1.udiv(C2);713 case TargetOpcode::G_SDIV:714 if (!C2.getBoolValue())715 break;716 return C1.sdiv(C2);717 case TargetOpcode::G_UREM:718 if (!C2.getBoolValue())719 break;720 return C1.urem(C2);721 case TargetOpcode::G_SREM:722 if (!C2.getBoolValue())723 break;724 return C1.srem(C2);725 case TargetOpcode::G_SMIN:726 return APIntOps::smin(C1, C2);727 case TargetOpcode::G_SMAX:728 return APIntOps::smax(C1, C2);729 case TargetOpcode::G_UMIN:730 return APIntOps::umin(C1, C2);731 case TargetOpcode::G_UMAX:732 return APIntOps::umax(C1, C2);733 }734 735 return std::nullopt;736}737 738std::optional<APFloat>739llvm::ConstantFoldFPBinOp(unsigned Opcode, const Register Op1,740 const Register Op2, const MachineRegisterInfo &MRI) {741 const ConstantFP *Op2Cst = getConstantFPVRegVal(Op2, MRI);742 if (!Op2Cst)743 return std::nullopt;744 745 const ConstantFP *Op1Cst = getConstantFPVRegVal(Op1, MRI);746 if (!Op1Cst)747 return std::nullopt;748 749 APFloat C1 = Op1Cst->getValueAPF();750 const APFloat &C2 = Op2Cst->getValueAPF();751 switch (Opcode) {752 case TargetOpcode::G_FADD:753 C1.add(C2, APFloat::rmNearestTiesToEven);754 return C1;755 case TargetOpcode::G_FSUB:756 C1.subtract(C2, APFloat::rmNearestTiesToEven);757 return C1;758 case TargetOpcode::G_FMUL:759 C1.multiply(C2, APFloat::rmNearestTiesToEven);760 return C1;761 case TargetOpcode::G_FDIV:762 C1.divide(C2, APFloat::rmNearestTiesToEven);763 return C1;764 case TargetOpcode::G_FREM:765 C1.mod(C2);766 return C1;767 case TargetOpcode::G_FCOPYSIGN:768 C1.copySign(C2);769 return C1;770 case TargetOpcode::G_FMINNUM:771 return minnum(C1, C2);772 case TargetOpcode::G_FMAXNUM:773 return maxnum(C1, C2);774 case TargetOpcode::G_FMINIMUM:775 return minimum(C1, C2);776 case TargetOpcode::G_FMAXIMUM:777 return maximum(C1, C2);778 case TargetOpcode::G_FMINNUM_IEEE:779 case TargetOpcode::G_FMAXNUM_IEEE:780 // FIXME: These operations were unfortunately named. fminnum/fmaxnum do not781 // follow the IEEE behavior for signaling nans and follow libm's fmin/fmax,782 // and currently there isn't a nice wrapper in APFloat for the version with783 // correct snan handling.784 break;785 default:786 break;787 }788 789 return std::nullopt;790}791 792SmallVector<APInt>793llvm::ConstantFoldVectorBinop(unsigned Opcode, const Register Op1,794 const Register Op2,795 const MachineRegisterInfo &MRI) {796 auto *SrcVec2 = getOpcodeDef<GBuildVector>(Op2, MRI);797 if (!SrcVec2)798 return SmallVector<APInt>();799 800 auto *SrcVec1 = getOpcodeDef<GBuildVector>(Op1, MRI);801 if (!SrcVec1)802 return SmallVector<APInt>();803 804 SmallVector<APInt> FoldedElements;805 for (unsigned Idx = 0, E = SrcVec1->getNumSources(); Idx < E; ++Idx) {806 auto MaybeCst = ConstantFoldBinOp(Opcode, SrcVec1->getSourceReg(Idx),807 SrcVec2->getSourceReg(Idx), MRI);808 if (!MaybeCst)809 return SmallVector<APInt>();810 FoldedElements.push_back(*MaybeCst);811 }812 return FoldedElements;813}814 815bool llvm::isKnownNeverNaN(Register Val, const MachineRegisterInfo &MRI,816 bool SNaN) {817 const MachineInstr *DefMI = MRI.getVRegDef(Val);818 if (!DefMI)819 return false;820 821 if (DefMI->getFlag(MachineInstr::FmNoNans))822 return true;823 824 // If the value is a constant, we can obviously see if it is a NaN or not.825 if (const ConstantFP *FPVal = getConstantFPVRegVal(Val, MRI)) {826 return !FPVal->getValueAPF().isNaN() ||827 (SNaN && !FPVal->getValueAPF().isSignaling());828 }829 830 if (DefMI->getOpcode() == TargetOpcode::G_BUILD_VECTOR) {831 for (const auto &Op : DefMI->uses())832 if (!isKnownNeverNaN(Op.getReg(), MRI, SNaN))833 return false;834 return true;835 }836 837 switch (DefMI->getOpcode()) {838 default:839 break;840 case TargetOpcode::G_FADD:841 case TargetOpcode::G_FSUB:842 case TargetOpcode::G_FMUL:843 case TargetOpcode::G_FDIV:844 case TargetOpcode::G_FREM:845 case TargetOpcode::G_FSIN:846 case TargetOpcode::G_FCOS:847 case TargetOpcode::G_FTAN:848 case TargetOpcode::G_FACOS:849 case TargetOpcode::G_FASIN:850 case TargetOpcode::G_FATAN:851 case TargetOpcode::G_FATAN2:852 case TargetOpcode::G_FCOSH:853 case TargetOpcode::G_FSINH:854 case TargetOpcode::G_FTANH:855 case TargetOpcode::G_FMA:856 case TargetOpcode::G_FMAD:857 if (SNaN)858 return true;859 860 // TODO: Need isKnownNeverInfinity861 return false;862 case TargetOpcode::G_FMINNUM_IEEE:863 case TargetOpcode::G_FMAXNUM_IEEE: {864 if (SNaN)865 return true;866 // This can return a NaN if either operand is an sNaN, or if both operands867 // are NaN.868 return (isKnownNeverNaN(DefMI->getOperand(1).getReg(), MRI) &&869 isKnownNeverSNaN(DefMI->getOperand(2).getReg(), MRI)) ||870 (isKnownNeverSNaN(DefMI->getOperand(1).getReg(), MRI) &&871 isKnownNeverNaN(DefMI->getOperand(2).getReg(), MRI));872 }873 case TargetOpcode::G_FMINNUM:874 case TargetOpcode::G_FMAXNUM: {875 // Only one needs to be known not-nan, since it will be returned if the876 // other ends up being one.877 return isKnownNeverNaN(DefMI->getOperand(1).getReg(), MRI, SNaN) ||878 isKnownNeverNaN(DefMI->getOperand(2).getReg(), MRI, SNaN);879 }880 }881 882 if (SNaN) {883 // FP operations quiet. For now, just handle the ones inserted during884 // legalization.885 switch (DefMI->getOpcode()) {886 case TargetOpcode::G_FPEXT:887 case TargetOpcode::G_FPTRUNC:888 case TargetOpcode::G_FCANONICALIZE:889 return true;890 default:891 return false;892 }893 }894 895 return false;896}897 898Align llvm::inferAlignFromPtrInfo(MachineFunction &MF,899 const MachinePointerInfo &MPO) {900 auto PSV = dyn_cast_if_present<const PseudoSourceValue *>(MPO.V);901 if (auto FSPV = dyn_cast_or_null<FixedStackPseudoSourceValue>(PSV)) {902 MachineFrameInfo &MFI = MF.getFrameInfo();903 return commonAlignment(MFI.getObjectAlign(FSPV->getFrameIndex()),904 MPO.Offset);905 }906 907 if (const Value *V = dyn_cast_if_present<const Value *>(MPO.V)) {908 const Module *M = MF.getFunction().getParent();909 return V->getPointerAlignment(M->getDataLayout());910 }911 912 return Align(1);913}914 915Register llvm::getFunctionLiveInPhysReg(MachineFunction &MF,916 const TargetInstrInfo &TII,917 MCRegister PhysReg,918 const TargetRegisterClass &RC,919 const DebugLoc &DL, LLT RegTy) {920 MachineBasicBlock &EntryMBB = MF.front();921 MachineRegisterInfo &MRI = MF.getRegInfo();922 Register LiveIn = MRI.getLiveInVirtReg(PhysReg);923 if (LiveIn) {924 MachineInstr *Def = MRI.getVRegDef(LiveIn);925 if (Def) {926 // FIXME: Should the verifier check this is in the entry block?927 assert(Def->getParent() == &EntryMBB && "live-in copy not in entry block");928 return LiveIn;929 }930 931 // It's possible the incoming argument register and copy was added during932 // lowering, but later deleted due to being/becoming dead. If this happens,933 // re-insert the copy.934 } else {935 // The live in register was not present, so add it.936 LiveIn = MF.addLiveIn(PhysReg, &RC);937 if (RegTy.isValid())938 MRI.setType(LiveIn, RegTy);939 }940 941 BuildMI(EntryMBB, EntryMBB.begin(), DL, TII.get(TargetOpcode::COPY), LiveIn)942 .addReg(PhysReg);943 if (!EntryMBB.isLiveIn(PhysReg))944 EntryMBB.addLiveIn(PhysReg);945 return LiveIn;946}947 948std::optional<APInt> llvm::ConstantFoldExtOp(unsigned Opcode,949 const Register Op1, uint64_t Imm,950 const MachineRegisterInfo &MRI) {951 auto MaybeOp1Cst = getIConstantVRegVal(Op1, MRI);952 if (MaybeOp1Cst) {953 switch (Opcode) {954 default:955 break;956 case TargetOpcode::G_SEXT_INREG: {957 LLT Ty = MRI.getType(Op1);958 return MaybeOp1Cst->trunc(Imm).sext(Ty.getScalarSizeInBits());959 }960 }961 }962 return std::nullopt;963}964 965std::optional<APInt> llvm::ConstantFoldCastOp(unsigned Opcode, LLT DstTy,966 const Register Op0,967 const MachineRegisterInfo &MRI) {968 std::optional<APInt> Val = getIConstantVRegVal(Op0, MRI);969 if (!Val)970 return Val;971 972 const unsigned DstSize = DstTy.getScalarSizeInBits();973 974 switch (Opcode) {975 case TargetOpcode::G_SEXT:976 return Val->sext(DstSize);977 case TargetOpcode::G_ZEXT:978 case TargetOpcode::G_ANYEXT:979 // TODO: DAG considers target preference when constant folding any_extend.980 return Val->zext(DstSize);981 default:982 break;983 }984 985 llvm_unreachable("unexpected cast opcode to constant fold");986}987 988std::optional<APFloat>989llvm::ConstantFoldIntToFloat(unsigned Opcode, LLT DstTy, Register Src,990 const MachineRegisterInfo &MRI) {991 assert(Opcode == TargetOpcode::G_SITOFP || Opcode == TargetOpcode::G_UITOFP);992 if (auto MaybeSrcVal = getIConstantVRegVal(Src, MRI)) {993 APFloat DstVal(getFltSemanticForLLT(DstTy));994 DstVal.convertFromAPInt(*MaybeSrcVal, Opcode == TargetOpcode::G_SITOFP,995 APFloat::rmNearestTiesToEven);996 return DstVal;997 }998 return std::nullopt;999}1000 1001std::optional<SmallVector<unsigned>>1002llvm::ConstantFoldCountZeros(Register Src, const MachineRegisterInfo &MRI,1003 std::function<unsigned(APInt)> CB) {1004 LLT Ty = MRI.getType(Src);1005 SmallVector<unsigned> FoldedCTLZs;1006 auto tryFoldScalar = [&](Register R) -> std::optional<unsigned> {1007 auto MaybeCst = getIConstantVRegVal(R, MRI);1008 if (!MaybeCst)1009 return std::nullopt;1010 return CB(*MaybeCst);1011 };1012 if (Ty.isVector()) {1013 // Try to constant fold each element.1014 auto *BV = getOpcodeDef<GBuildVector>(Src, MRI);1015 if (!BV)1016 return std::nullopt;1017 for (unsigned SrcIdx = 0; SrcIdx < BV->getNumSources(); ++SrcIdx) {1018 if (auto MaybeFold = tryFoldScalar(BV->getSourceReg(SrcIdx))) {1019 FoldedCTLZs.emplace_back(*MaybeFold);1020 continue;1021 }1022 return std::nullopt;1023 }1024 return FoldedCTLZs;1025 }1026 if (auto MaybeCst = tryFoldScalar(Src)) {1027 FoldedCTLZs.emplace_back(*MaybeCst);1028 return FoldedCTLZs;1029 }1030 return std::nullopt;1031}1032 1033std::optional<SmallVector<APInt>>1034llvm::ConstantFoldICmp(unsigned Pred, const Register Op1, const Register Op2,1035 unsigned DstScalarSizeInBits, unsigned ExtOp,1036 const MachineRegisterInfo &MRI) {1037 assert(ExtOp == TargetOpcode::G_SEXT || ExtOp == TargetOpcode::G_ZEXT ||1038 ExtOp == TargetOpcode::G_ANYEXT);1039 1040 const LLT Ty = MRI.getType(Op1);1041 1042 auto GetICmpResultCst = [&](bool IsTrue) {1043 if (IsTrue)1044 return ExtOp == TargetOpcode::G_SEXT1045 ? APInt::getAllOnes(DstScalarSizeInBits)1046 : APInt::getOneBitSet(DstScalarSizeInBits, 0);1047 return APInt::getZero(DstScalarSizeInBits);1048 };1049 1050 auto TryFoldScalar = [&](Register LHS, Register RHS) -> std::optional<APInt> {1051 auto RHSCst = getIConstantVRegVal(RHS, MRI);1052 if (!RHSCst)1053 return std::nullopt;1054 auto LHSCst = getIConstantVRegVal(LHS, MRI);1055 if (!LHSCst)1056 return std::nullopt;1057 1058 switch (Pred) {1059 case CmpInst::Predicate::ICMP_EQ:1060 return GetICmpResultCst(LHSCst->eq(*RHSCst));1061 case CmpInst::Predicate::ICMP_NE:1062 return GetICmpResultCst(LHSCst->ne(*RHSCst));1063 case CmpInst::Predicate::ICMP_UGT:1064 return GetICmpResultCst(LHSCst->ugt(*RHSCst));1065 case CmpInst::Predicate::ICMP_UGE:1066 return GetICmpResultCst(LHSCst->uge(*RHSCst));1067 case CmpInst::Predicate::ICMP_ULT:1068 return GetICmpResultCst(LHSCst->ult(*RHSCst));1069 case CmpInst::Predicate::ICMP_ULE:1070 return GetICmpResultCst(LHSCst->ule(*RHSCst));1071 case CmpInst::Predicate::ICMP_SGT:1072 return GetICmpResultCst(LHSCst->sgt(*RHSCst));1073 case CmpInst::Predicate::ICMP_SGE:1074 return GetICmpResultCst(LHSCst->sge(*RHSCst));1075 case CmpInst::Predicate::ICMP_SLT:1076 return GetICmpResultCst(LHSCst->slt(*RHSCst));1077 case CmpInst::Predicate::ICMP_SLE:1078 return GetICmpResultCst(LHSCst->sle(*RHSCst));1079 default:1080 return std::nullopt;1081 }1082 };1083 1084 SmallVector<APInt> FoldedICmps;1085 1086 if (Ty.isVector()) {1087 // Try to constant fold each element.1088 auto *BV1 = getOpcodeDef<GBuildVector>(Op1, MRI);1089 auto *BV2 = getOpcodeDef<GBuildVector>(Op2, MRI);1090 if (!BV1 || !BV2)1091 return std::nullopt;1092 assert(BV1->getNumSources() == BV2->getNumSources() && "Invalid vectors");1093 for (unsigned I = 0; I < BV1->getNumSources(); ++I) {1094 if (auto MaybeFold =1095 TryFoldScalar(BV1->getSourceReg(I), BV2->getSourceReg(I))) {1096 FoldedICmps.emplace_back(*MaybeFold);1097 continue;1098 }1099 return std::nullopt;1100 }1101 return FoldedICmps;1102 }1103 1104 if (auto MaybeCst = TryFoldScalar(Op1, Op2)) {1105 FoldedICmps.emplace_back(*MaybeCst);1106 return FoldedICmps;1107 }1108 1109 return std::nullopt;1110}1111 1112bool llvm::isKnownToBeAPowerOfTwo(Register Reg, const MachineRegisterInfo &MRI,1113 GISelValueTracking *VT) {1114 std::optional<DefinitionAndSourceRegister> DefSrcReg =1115 getDefSrcRegIgnoringCopies(Reg, MRI);1116 if (!DefSrcReg)1117 return false;1118 1119 const MachineInstr &MI = *DefSrcReg->MI;1120 const LLT Ty = MRI.getType(Reg);1121 1122 switch (MI.getOpcode()) {1123 case TargetOpcode::G_CONSTANT: {1124 unsigned BitWidth = Ty.getScalarSizeInBits();1125 const ConstantInt *CI = MI.getOperand(1).getCImm();1126 return CI->getValue().zextOrTrunc(BitWidth).isPowerOf2();1127 }1128 case TargetOpcode::G_SHL: {1129 // A left-shift of a constant one will have exactly one bit set because1130 // shifting the bit off the end is undefined.1131 1132 // TODO: Constant splat1133 if (auto ConstLHS = getIConstantVRegVal(MI.getOperand(1).getReg(), MRI)) {1134 if (*ConstLHS == 1)1135 return true;1136 }1137 1138 break;1139 }1140 case TargetOpcode::G_LSHR: {1141 if (auto ConstLHS = getIConstantVRegVal(MI.getOperand(1).getReg(), MRI)) {1142 if (ConstLHS->isSignMask())1143 return true;1144 }1145 1146 break;1147 }1148 case TargetOpcode::G_BUILD_VECTOR: {1149 // TODO: Probably should have a recursion depth guard since you could have1150 // bitcasted vector elements.1151 for (const MachineOperand &MO : llvm::drop_begin(MI.operands()))1152 if (!isKnownToBeAPowerOfTwo(MO.getReg(), MRI, VT))1153 return false;1154 1155 return true;1156 }1157 case TargetOpcode::G_BUILD_VECTOR_TRUNC: {1158 // Only handle constants since we would need to know if number of leading1159 // zeros is greater than the truncation amount.1160 const unsigned BitWidth = Ty.getScalarSizeInBits();1161 for (const MachineOperand &MO : llvm::drop_begin(MI.operands())) {1162 auto Const = getIConstantVRegVal(MO.getReg(), MRI);1163 if (!Const || !Const->zextOrTrunc(BitWidth).isPowerOf2())1164 return false;1165 }1166 1167 return true;1168 }1169 default:1170 break;1171 }1172 1173 if (!VT)1174 return false;1175 1176 // More could be done here, though the above checks are enough1177 // to handle some common cases.1178 1179 // Fall back to computeKnownBits to catch other known cases.1180 KnownBits Known = VT->getKnownBits(Reg);1181 return (Known.countMaxPopulation() == 1) && (Known.countMinPopulation() == 1);1182}1183 1184void llvm::getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU) {1185 AU.addPreserved<StackProtector>();1186}1187 1188LLT llvm::getLCMType(LLT OrigTy, LLT TargetTy) {1189 if (OrigTy.getSizeInBits() == TargetTy.getSizeInBits())1190 return OrigTy;1191 1192 if (OrigTy.isVector() && TargetTy.isVector()) {1193 LLT OrigElt = OrigTy.getElementType();1194 LLT TargetElt = TargetTy.getElementType();1195 1196 // TODO: The docstring for this function says the intention is to use this1197 // function to build MERGE/UNMERGE instructions. It won't be the case that1198 // we generate a MERGE/UNMERGE between fixed and scalable vector types. We1199 // could implement getLCMType between the two in the future if there was a1200 // need, but it is not worth it now as this function should not be used in1201 // that way.1202 assert(((OrigTy.isScalableVector() && !TargetTy.isFixedVector()) ||1203 (OrigTy.isFixedVector() && !TargetTy.isScalableVector())) &&1204 "getLCMType not implemented between fixed and scalable vectors.");1205 1206 if (OrigElt.getSizeInBits() == TargetElt.getSizeInBits()) {1207 int GCDMinElts = std::gcd(OrigTy.getElementCount().getKnownMinValue(),1208 TargetTy.getElementCount().getKnownMinValue());1209 // Prefer the original element type.1210 ElementCount Mul = OrigTy.getElementCount().multiplyCoefficientBy(1211 TargetTy.getElementCount().getKnownMinValue());1212 return LLT::vector(Mul.divideCoefficientBy(GCDMinElts),1213 OrigTy.getElementType());1214 }1215 unsigned LCM = std::lcm(OrigTy.getSizeInBits().getKnownMinValue(),1216 TargetTy.getSizeInBits().getKnownMinValue());1217 return LLT::vector(1218 ElementCount::get(LCM / OrigElt.getSizeInBits(), OrigTy.isScalable()),1219 OrigElt);1220 }1221 1222 // One type is scalar, one type is vector1223 if (OrigTy.isVector() || TargetTy.isVector()) {1224 LLT VecTy = OrigTy.isVector() ? OrigTy : TargetTy;1225 LLT ScalarTy = OrigTy.isVector() ? TargetTy : OrigTy;1226 LLT EltTy = VecTy.getElementType();1227 LLT OrigEltTy = OrigTy.isVector() ? OrigTy.getElementType() : OrigTy;1228 1229 // Prefer scalar type from OrigTy.1230 if (EltTy.getSizeInBits() == ScalarTy.getSizeInBits())1231 return LLT::vector(VecTy.getElementCount(), OrigEltTy);1232 1233 // Different size scalars. Create vector with the same total size.1234 // LCM will take fixed/scalable from VecTy.1235 unsigned LCM = std::lcm(EltTy.getSizeInBits().getFixedValue() *1236 VecTy.getElementCount().getKnownMinValue(),1237 ScalarTy.getSizeInBits().getFixedValue());1238 // Prefer type from OrigTy1239 return LLT::vector(ElementCount::get(LCM / OrigEltTy.getSizeInBits(),1240 VecTy.getElementCount().isScalable()),1241 OrigEltTy);1242 }1243 1244 // At this point, both types are scalars of different size1245 unsigned LCM = std::lcm(OrigTy.getSizeInBits().getFixedValue(),1246 TargetTy.getSizeInBits().getFixedValue());1247 // Preserve pointer types.1248 if (LCM == OrigTy.getSizeInBits())1249 return OrigTy;1250 if (LCM == TargetTy.getSizeInBits())1251 return TargetTy;1252 return LLT::scalar(LCM);1253}1254 1255LLT llvm::getCoverTy(LLT OrigTy, LLT TargetTy) {1256 1257 if ((OrigTy.isScalableVector() && TargetTy.isFixedVector()) ||1258 (OrigTy.isFixedVector() && TargetTy.isScalableVector()))1259 llvm_unreachable(1260 "getCoverTy not implemented between fixed and scalable vectors.");1261 1262 if (!OrigTy.isVector() || !TargetTy.isVector() || OrigTy == TargetTy ||1263 (OrigTy.getScalarSizeInBits() != TargetTy.getScalarSizeInBits()))1264 return getLCMType(OrigTy, TargetTy);1265 1266 unsigned OrigTyNumElts = OrigTy.getElementCount().getKnownMinValue();1267 unsigned TargetTyNumElts = TargetTy.getElementCount().getKnownMinValue();1268 if (OrigTyNumElts % TargetTyNumElts == 0)1269 return OrigTy;1270 1271 unsigned NumElts = alignTo(OrigTyNumElts, TargetTyNumElts);1272 return LLT::scalarOrVector(ElementCount::getFixed(NumElts),1273 OrigTy.getElementType());1274}1275 1276LLT llvm::getGCDType(LLT OrigTy, LLT TargetTy) {1277 if (OrigTy.getSizeInBits() == TargetTy.getSizeInBits())1278 return OrigTy;1279 1280 if (OrigTy.isVector() && TargetTy.isVector()) {1281 LLT OrigElt = OrigTy.getElementType();1282 1283 // TODO: The docstring for this function says the intention is to use this1284 // function to build MERGE/UNMERGE instructions. It won't be the case that1285 // we generate a MERGE/UNMERGE between fixed and scalable vector types. We1286 // could implement getGCDType between the two in the future if there was a1287 // need, but it is not worth it now as this function should not be used in1288 // that way.1289 assert(((OrigTy.isScalableVector() && !TargetTy.isFixedVector()) ||1290 (OrigTy.isFixedVector() && !TargetTy.isScalableVector())) &&1291 "getGCDType not implemented between fixed and scalable vectors.");1292 1293 unsigned GCD = std::gcd(OrigTy.getSizeInBits().getKnownMinValue(),1294 TargetTy.getSizeInBits().getKnownMinValue());1295 if (GCD == OrigElt.getSizeInBits())1296 return LLT::scalarOrVector(ElementCount::get(1, OrigTy.isScalable()),1297 OrigElt);1298 1299 // Cannot produce original element type, but both have vscale in common.1300 if (GCD < OrigElt.getSizeInBits())1301 return LLT::scalarOrVector(ElementCount::get(1, OrigTy.isScalable()),1302 GCD);1303 1304 return LLT::vector(1305 ElementCount::get(GCD / OrigElt.getSizeInBits().getFixedValue(),1306 OrigTy.isScalable()),1307 OrigElt);1308 }1309 1310 // If one type is vector and the element size matches the scalar size, then1311 // the gcd is the scalar type.1312 if (OrigTy.isVector() &&1313 OrigTy.getElementType().getSizeInBits() == TargetTy.getSizeInBits())1314 return OrigTy.getElementType();1315 if (TargetTy.isVector() &&1316 TargetTy.getElementType().getSizeInBits() == OrigTy.getSizeInBits())1317 return OrigTy;1318 1319 // At this point, both types are either scalars of different type or one is a1320 // vector and one is a scalar. If both types are scalars, the GCD type is the1321 // GCD between the two scalar sizes. If one is vector and one is scalar, then1322 // the GCD type is the GCD between the scalar and the vector element size.1323 LLT OrigScalar = OrigTy.getScalarType();1324 LLT TargetScalar = TargetTy.getScalarType();1325 unsigned GCD = std::gcd(OrigScalar.getSizeInBits().getFixedValue(),1326 TargetScalar.getSizeInBits().getFixedValue());1327 return LLT::scalar(GCD);1328}1329 1330std::optional<int> llvm::getSplatIndex(MachineInstr &MI) {1331 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR &&1332 "Only G_SHUFFLE_VECTOR can have a splat index!");1333 ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask();1334 auto FirstDefinedIdx = find_if(Mask, [](int Elt) { return Elt >= 0; });1335 1336 // If all elements are undefined, this shuffle can be considered a splat.1337 // Return 0 for better potential for callers to simplify.1338 if (FirstDefinedIdx == Mask.end())1339 return 0;1340 1341 // Make sure all remaining elements are either undef or the same1342 // as the first non-undef value.1343 int SplatValue = *FirstDefinedIdx;1344 if (any_of(make_range(std::next(FirstDefinedIdx), Mask.end()),1345 [&SplatValue](int Elt) { return Elt >= 0 && Elt != SplatValue; }))1346 return std::nullopt;1347 1348 return SplatValue;1349}1350 1351static bool isBuildVectorOp(unsigned Opcode) {1352 return Opcode == TargetOpcode::G_BUILD_VECTOR ||1353 Opcode == TargetOpcode::G_BUILD_VECTOR_TRUNC;1354}1355 1356namespace {1357 1358std::optional<ValueAndVReg> getAnyConstantSplat(Register VReg,1359 const MachineRegisterInfo &MRI,1360 bool AllowUndef) {1361 MachineInstr *MI = getDefIgnoringCopies(VReg, MRI);1362 if (!MI)1363 return std::nullopt;1364 1365 bool isConcatVectorsOp = MI->getOpcode() == TargetOpcode::G_CONCAT_VECTORS;1366 if (!isBuildVectorOp(MI->getOpcode()) && !isConcatVectorsOp)1367 return std::nullopt;1368 1369 std::optional<ValueAndVReg> SplatValAndReg;1370 for (MachineOperand &Op : MI->uses()) {1371 Register Element = Op.getReg();1372 // If we have a G_CONCAT_VECTOR, we recursively look into the1373 // vectors that we're concatenating to see if they're splats.1374 auto ElementValAndReg =1375 isConcatVectorsOp1376 ? getAnyConstantSplat(Element, MRI, AllowUndef)1377 : getAnyConstantVRegValWithLookThrough(Element, MRI, true, true);1378 1379 // If AllowUndef, treat undef as value that will result in a constant splat.1380 if (!ElementValAndReg) {1381 if (AllowUndef && isa<GImplicitDef>(MRI.getVRegDef(Element)))1382 continue;1383 return std::nullopt;1384 }1385 1386 // Record splat value1387 if (!SplatValAndReg)1388 SplatValAndReg = ElementValAndReg;1389 1390 // Different constant than the one already recorded, not a constant splat.1391 if (SplatValAndReg->Value != ElementValAndReg->Value)1392 return std::nullopt;1393 }1394 1395 return SplatValAndReg;1396}1397 1398} // end anonymous namespace1399 1400bool llvm::isBuildVectorConstantSplat(const Register Reg,1401 const MachineRegisterInfo &MRI,1402 int64_t SplatValue, bool AllowUndef) {1403 if (auto SplatValAndReg = getAnyConstantSplat(Reg, MRI, AllowUndef))1404 return SplatValAndReg->Value.getSExtValue() == SplatValue;1405 1406 return false;1407}1408 1409bool llvm::isBuildVectorConstantSplat(const Register Reg,1410 const MachineRegisterInfo &MRI,1411 const APInt &SplatValue,1412 bool AllowUndef) {1413 if (auto SplatValAndReg = getAnyConstantSplat(Reg, MRI, AllowUndef)) {1414 if (SplatValAndReg->Value.getBitWidth() < SplatValue.getBitWidth())1415 return APInt::isSameValue(1416 SplatValAndReg->Value.sext(SplatValue.getBitWidth()), SplatValue);1417 return APInt::isSameValue(1418 SplatValAndReg->Value,1419 SplatValue.sext(SplatValAndReg->Value.getBitWidth()));1420 }1421 1422 return false;1423}1424 1425bool llvm::isBuildVectorConstantSplat(const MachineInstr &MI,1426 const MachineRegisterInfo &MRI,1427 int64_t SplatValue, bool AllowUndef) {1428 return isBuildVectorConstantSplat(MI.getOperand(0).getReg(), MRI, SplatValue,1429 AllowUndef);1430}1431 1432bool llvm::isBuildVectorConstantSplat(const MachineInstr &MI,1433 const MachineRegisterInfo &MRI,1434 const APInt &SplatValue,1435 bool AllowUndef) {1436 return isBuildVectorConstantSplat(MI.getOperand(0).getReg(), MRI, SplatValue,1437 AllowUndef);1438}1439 1440std::optional<APInt>1441llvm::getIConstantSplatVal(const Register Reg, const MachineRegisterInfo &MRI) {1442 if (auto SplatValAndReg =1443 getAnyConstantSplat(Reg, MRI, /* AllowUndef */ false)) {1444 if (std::optional<ValueAndVReg> ValAndVReg =1445 getIConstantVRegValWithLookThrough(SplatValAndReg->VReg, MRI))1446 return ValAndVReg->Value;1447 }1448 1449 return std::nullopt;1450}1451 1452std::optional<APInt>1453llvm::getIConstantSplatVal(const MachineInstr &MI,1454 const MachineRegisterInfo &MRI) {1455 return getIConstantSplatVal(MI.getOperand(0).getReg(), MRI);1456}1457 1458std::optional<int64_t>1459llvm::getIConstantSplatSExtVal(const Register Reg,1460 const MachineRegisterInfo &MRI) {1461 if (auto SplatValAndReg =1462 getAnyConstantSplat(Reg, MRI, /* AllowUndef */ false))1463 return getIConstantVRegSExtVal(SplatValAndReg->VReg, MRI);1464 return std::nullopt;1465}1466 1467std::optional<int64_t>1468llvm::getIConstantSplatSExtVal(const MachineInstr &MI,1469 const MachineRegisterInfo &MRI) {1470 return getIConstantSplatSExtVal(MI.getOperand(0).getReg(), MRI);1471}1472 1473std::optional<FPValueAndVReg>1474llvm::getFConstantSplat(Register VReg, const MachineRegisterInfo &MRI,1475 bool AllowUndef) {1476 if (auto SplatValAndReg = getAnyConstantSplat(VReg, MRI, AllowUndef))1477 return getFConstantVRegValWithLookThrough(SplatValAndReg->VReg, MRI);1478 return std::nullopt;1479}1480 1481bool llvm::isBuildVectorAllZeros(const MachineInstr &MI,1482 const MachineRegisterInfo &MRI,1483 bool AllowUndef) {1484 return isBuildVectorConstantSplat(MI, MRI, 0, AllowUndef);1485}1486 1487bool llvm::isBuildVectorAllOnes(const MachineInstr &MI,1488 const MachineRegisterInfo &MRI,1489 bool AllowUndef) {1490 return isBuildVectorConstantSplat(MI, MRI, -1, AllowUndef);1491}1492 1493std::optional<RegOrConstant>1494llvm::getVectorSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI) {1495 unsigned Opc = MI.getOpcode();1496 if (!isBuildVectorOp(Opc))1497 return std::nullopt;1498 if (auto Splat = getIConstantSplatSExtVal(MI, MRI))1499 return RegOrConstant(*Splat);1500 auto Reg = MI.getOperand(1).getReg();1501 if (any_of(drop_begin(MI.operands(), 2),1502 [&Reg](const MachineOperand &Op) { return Op.getReg() != Reg; }))1503 return std::nullopt;1504 return RegOrConstant(Reg);1505}1506 1507static bool isConstantScalar(const MachineInstr &MI,1508 const MachineRegisterInfo &MRI,1509 bool AllowFP = true,1510 bool AllowOpaqueConstants = true) {1511 switch (MI.getOpcode()) {1512 case TargetOpcode::G_CONSTANT:1513 case TargetOpcode::G_IMPLICIT_DEF:1514 return true;1515 case TargetOpcode::G_FCONSTANT:1516 return AllowFP;1517 case TargetOpcode::G_GLOBAL_VALUE:1518 case TargetOpcode::G_FRAME_INDEX:1519 case TargetOpcode::G_BLOCK_ADDR:1520 case TargetOpcode::G_JUMP_TABLE:1521 return AllowOpaqueConstants;1522 default:1523 return false;1524 }1525}1526 1527bool llvm::isConstantOrConstantVector(MachineInstr &MI,1528 const MachineRegisterInfo &MRI) {1529 Register Def = MI.getOperand(0).getReg();1530 if (auto C = getIConstantVRegValWithLookThrough(Def, MRI))1531 return true;1532 GBuildVector *BV = dyn_cast<GBuildVector>(&MI);1533 if (!BV)1534 return false;1535 for (unsigned SrcIdx = 0; SrcIdx < BV->getNumSources(); ++SrcIdx) {1536 if (getIConstantVRegValWithLookThrough(BV->getSourceReg(SrcIdx), MRI) ||1537 getOpcodeDef<GImplicitDef>(BV->getSourceReg(SrcIdx), MRI))1538 continue;1539 return false;1540 }1541 return true;1542}1543 1544bool llvm::isConstantOrConstantVector(const MachineInstr &MI,1545 const MachineRegisterInfo &MRI,1546 bool AllowFP, bool AllowOpaqueConstants) {1547 if (isConstantScalar(MI, MRI, AllowFP, AllowOpaqueConstants))1548 return true;1549 1550 if (!isBuildVectorOp(MI.getOpcode()))1551 return false;1552 1553 const unsigned NumOps = MI.getNumOperands();1554 for (unsigned I = 1; I != NumOps; ++I) {1555 const MachineInstr *ElementDef = MRI.getVRegDef(MI.getOperand(I).getReg());1556 if (!isConstantScalar(*ElementDef, MRI, AllowFP, AllowOpaqueConstants))1557 return false;1558 }1559 1560 return true;1561}1562 1563std::optional<APInt>1564llvm::isConstantOrConstantSplatVector(MachineInstr &MI,1565 const MachineRegisterInfo &MRI) {1566 Register Def = MI.getOperand(0).getReg();1567 if (auto C = getIConstantVRegValWithLookThrough(Def, MRI))1568 return C->Value;1569 auto MaybeCst = getIConstantSplatSExtVal(MI, MRI);1570 if (!MaybeCst)1571 return std::nullopt;1572 const unsigned ScalarSize = MRI.getType(Def).getScalarSizeInBits();1573 return APInt(ScalarSize, *MaybeCst, true);1574}1575 1576std::optional<APFloat>1577llvm::isConstantOrConstantSplatVectorFP(MachineInstr &MI,1578 const MachineRegisterInfo &MRI) {1579 Register Def = MI.getOperand(0).getReg();1580 if (auto FpConst = getFConstantVRegValWithLookThrough(Def, MRI))1581 return FpConst->Value;1582 auto MaybeCstFP = getFConstantSplat(Def, MRI, /*allowUndef=*/false);1583 if (!MaybeCstFP)1584 return std::nullopt;1585 return MaybeCstFP->Value;1586}1587 1588bool llvm::isNullOrNullSplat(const MachineInstr &MI,1589 const MachineRegisterInfo &MRI, bool AllowUndefs) {1590 switch (MI.getOpcode()) {1591 case TargetOpcode::G_IMPLICIT_DEF:1592 return AllowUndefs;1593 case TargetOpcode::G_CONSTANT:1594 return MI.getOperand(1).getCImm()->isNullValue();1595 case TargetOpcode::G_FCONSTANT: {1596 const ConstantFP *FPImm = MI.getOperand(1).getFPImm();1597 return FPImm->isZero() && !FPImm->isNegative();1598 }1599 default:1600 if (!AllowUndefs) // TODO: isBuildVectorAllZeros assumes undef is OK already1601 return false;1602 return isBuildVectorAllZeros(MI, MRI);1603 }1604}1605 1606bool llvm::isAllOnesOrAllOnesSplat(const MachineInstr &MI,1607 const MachineRegisterInfo &MRI,1608 bool AllowUndefs) {1609 switch (MI.getOpcode()) {1610 case TargetOpcode::G_IMPLICIT_DEF:1611 return AllowUndefs;1612 case TargetOpcode::G_CONSTANT:1613 return MI.getOperand(1).getCImm()->isAllOnesValue();1614 default:1615 if (!AllowUndefs) // TODO: isBuildVectorAllOnes assumes undef is OK already1616 return false;1617 return isBuildVectorAllOnes(MI, MRI);1618 }1619}1620 1621bool llvm::matchUnaryPredicate(1622 const MachineRegisterInfo &MRI, Register Reg,1623 std::function<bool(const Constant *ConstVal)> Match, bool AllowUndefs) {1624 1625 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI);1626 if (AllowUndefs && Def->getOpcode() == TargetOpcode::G_IMPLICIT_DEF)1627 return Match(nullptr);1628 1629 // TODO: Also handle fconstant1630 if (Def->getOpcode() == TargetOpcode::G_CONSTANT)1631 return Match(Def->getOperand(1).getCImm());1632 1633 if (Def->getOpcode() != TargetOpcode::G_BUILD_VECTOR)1634 return false;1635 1636 for (unsigned I = 1, E = Def->getNumOperands(); I != E; ++I) {1637 Register SrcElt = Def->getOperand(I).getReg();1638 const MachineInstr *SrcDef = getDefIgnoringCopies(SrcElt, MRI);1639 if (AllowUndefs && SrcDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) {1640 if (!Match(nullptr))1641 return false;1642 continue;1643 }1644 1645 if (SrcDef->getOpcode() != TargetOpcode::G_CONSTANT ||1646 !Match(SrcDef->getOperand(1).getCImm()))1647 return false;1648 }1649 1650 return true;1651}1652 1653bool llvm::isConstTrueVal(const TargetLowering &TLI, int64_t Val, bool IsVector,1654 bool IsFP) {1655 switch (TLI.getBooleanContents(IsVector, IsFP)) {1656 case TargetLowering::UndefinedBooleanContent:1657 return Val & 0x1;1658 case TargetLowering::ZeroOrOneBooleanContent:1659 return Val == 1;1660 case TargetLowering::ZeroOrNegativeOneBooleanContent:1661 return Val == -1;1662 }1663 llvm_unreachable("Invalid boolean contents");1664}1665 1666bool llvm::isConstFalseVal(const TargetLowering &TLI, int64_t Val,1667 bool IsVector, bool IsFP) {1668 switch (TLI.getBooleanContents(IsVector, IsFP)) {1669 case TargetLowering::UndefinedBooleanContent:1670 return ~Val & 0x1;1671 case TargetLowering::ZeroOrOneBooleanContent:1672 case TargetLowering::ZeroOrNegativeOneBooleanContent:1673 return Val == 0;1674 }1675 llvm_unreachable("Invalid boolean contents");1676}1677 1678int64_t llvm::getICmpTrueVal(const TargetLowering &TLI, bool IsVector,1679 bool IsFP) {1680 switch (TLI.getBooleanContents(IsVector, IsFP)) {1681 case TargetLowering::UndefinedBooleanContent:1682 case TargetLowering::ZeroOrOneBooleanContent:1683 return 1;1684 case TargetLowering::ZeroOrNegativeOneBooleanContent:1685 return -1;1686 }1687 llvm_unreachable("Invalid boolean contents");1688}1689 1690void llvm::saveUsesAndErase(MachineInstr &MI, MachineRegisterInfo &MRI,1691 LostDebugLocObserver *LocObserver,1692 SmallInstListTy &DeadInstChain) {1693 for (MachineOperand &Op : MI.uses()) {1694 if (Op.isReg() && Op.getReg().isVirtual())1695 DeadInstChain.insert(MRI.getVRegDef(Op.getReg()));1696 }1697 LLVM_DEBUG(dbgs() << MI << "Is dead; erasing.\n");1698 DeadInstChain.remove(&MI);1699 MI.eraseFromParent();1700 if (LocObserver)1701 LocObserver->checkpoint(false);1702}1703 1704void llvm::eraseInstrs(ArrayRef<MachineInstr *> DeadInstrs,1705 MachineRegisterInfo &MRI,1706 LostDebugLocObserver *LocObserver) {1707 SmallInstListTy DeadInstChain;1708 for (MachineInstr *MI : DeadInstrs)1709 saveUsesAndErase(*MI, MRI, LocObserver, DeadInstChain);1710 1711 while (!DeadInstChain.empty()) {1712 MachineInstr *Inst = DeadInstChain.pop_back_val();1713 if (!isTriviallyDead(*Inst, MRI))1714 continue;1715 saveUsesAndErase(*Inst, MRI, LocObserver, DeadInstChain);1716 }1717}1718 1719void llvm::eraseInstr(MachineInstr &MI, MachineRegisterInfo &MRI,1720 LostDebugLocObserver *LocObserver) {1721 return eraseInstrs({&MI}, MRI, LocObserver);1722}1723 1724void llvm::salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI) {1725 for (auto &Def : MI.defs()) {1726 assert(Def.isReg() && "Must be a reg");1727 1728 SmallVector<MachineOperand *, 16> DbgUsers;1729 for (auto &MOUse : MRI.use_operands(Def.getReg())) {1730 MachineInstr *DbgValue = MOUse.getParent();1731 // Ignore partially formed DBG_VALUEs.1732 if (DbgValue->isNonListDebugValue() && DbgValue->getNumOperands() == 4) {1733 DbgUsers.push_back(&MOUse);1734 }1735 }1736 1737 if (!DbgUsers.empty()) {1738 salvageDebugInfoForDbgValue(MRI, MI, DbgUsers);1739 }1740 }1741}1742 1743bool llvm::isPreISelGenericFloatingPointOpcode(unsigned Opc) {1744 switch (Opc) {1745 case TargetOpcode::G_FABS:1746 case TargetOpcode::G_FADD:1747 case TargetOpcode::G_FCANONICALIZE:1748 case TargetOpcode::G_FCEIL:1749 case TargetOpcode::G_FCONSTANT:1750 case TargetOpcode::G_FCOPYSIGN:1751 case TargetOpcode::G_FCOS:1752 case TargetOpcode::G_FDIV:1753 case TargetOpcode::G_FEXP2:1754 case TargetOpcode::G_FEXP:1755 case TargetOpcode::G_FFLOOR:1756 case TargetOpcode::G_FLOG10:1757 case TargetOpcode::G_FLOG2:1758 case TargetOpcode::G_FLOG:1759 case TargetOpcode::G_FMA:1760 case TargetOpcode::G_FMAD:1761 case TargetOpcode::G_FMAXIMUM:1762 case TargetOpcode::G_FMAXIMUMNUM:1763 case TargetOpcode::G_FMAXNUM:1764 case TargetOpcode::G_FMAXNUM_IEEE:1765 case TargetOpcode::G_FMINIMUM:1766 case TargetOpcode::G_FMINIMUMNUM:1767 case TargetOpcode::G_FMINNUM:1768 case TargetOpcode::G_FMINNUM_IEEE:1769 case TargetOpcode::G_FMUL:1770 case TargetOpcode::G_FNEARBYINT:1771 case TargetOpcode::G_FNEG:1772 case TargetOpcode::G_FPEXT:1773 case TargetOpcode::G_FPOW:1774 case TargetOpcode::G_FPTRUNC:1775 case TargetOpcode::G_FREM:1776 case TargetOpcode::G_FRINT:1777 case TargetOpcode::G_FSIN:1778 case TargetOpcode::G_FTAN:1779 case TargetOpcode::G_FACOS:1780 case TargetOpcode::G_FASIN:1781 case TargetOpcode::G_FATAN:1782 case TargetOpcode::G_FATAN2:1783 case TargetOpcode::G_FCOSH:1784 case TargetOpcode::G_FSINH:1785 case TargetOpcode::G_FTANH:1786 case TargetOpcode::G_FSQRT:1787 case TargetOpcode::G_FSUB:1788 case TargetOpcode::G_INTRINSIC_ROUND:1789 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:1790 case TargetOpcode::G_INTRINSIC_TRUNC:1791 return true;1792 default:1793 return false;1794 }1795}1796 1797/// Shifts return poison if shiftwidth is larger than the bitwidth.1798static bool shiftAmountKnownInRange(Register ShiftAmount,1799 const MachineRegisterInfo &MRI) {1800 LLT Ty = MRI.getType(ShiftAmount);1801 1802 if (Ty.isScalableVector())1803 return false; // Can't tell, just return false to be safe1804 1805 if (Ty.isScalar()) {1806 std::optional<ValueAndVReg> Val =1807 getIConstantVRegValWithLookThrough(ShiftAmount, MRI);1808 if (!Val)1809 return false;1810 return Val->Value.ult(Ty.getScalarSizeInBits());1811 }1812 1813 GBuildVector *BV = getOpcodeDef<GBuildVector>(ShiftAmount, MRI);1814 if (!BV)1815 return false;1816 1817 unsigned Sources = BV->getNumSources();1818 for (unsigned I = 0; I < Sources; ++I) {1819 std::optional<ValueAndVReg> Val =1820 getIConstantVRegValWithLookThrough(BV->getSourceReg(I), MRI);1821 if (!Val)1822 return false;1823 if (!Val->Value.ult(Ty.getScalarSizeInBits()))1824 return false;1825 }1826 1827 return true;1828}1829 1830namespace {1831enum class UndefPoisonKind {1832 PoisonOnly = (1 << 0),1833 UndefOnly = (1 << 1),1834 UndefOrPoison = PoisonOnly | UndefOnly,1835};1836}1837 1838static bool includesPoison(UndefPoisonKind Kind) {1839 return (unsigned(Kind) & unsigned(UndefPoisonKind::PoisonOnly)) != 0;1840}1841 1842static bool includesUndef(UndefPoisonKind Kind) {1843 return (unsigned(Kind) & unsigned(UndefPoisonKind::UndefOnly)) != 0;1844}1845 1846static bool canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI,1847 bool ConsiderFlagsAndMetadata,1848 UndefPoisonKind Kind) {1849 MachineInstr *RegDef = MRI.getVRegDef(Reg);1850 1851 if (ConsiderFlagsAndMetadata && includesPoison(Kind))1852 if (auto *GMI = dyn_cast<GenericMachineInstr>(RegDef))1853 if (GMI->hasPoisonGeneratingFlags())1854 return true;1855 1856 // Check whether opcode is a poison/undef-generating operation.1857 switch (RegDef->getOpcode()) {1858 case TargetOpcode::G_BUILD_VECTOR:1859 case TargetOpcode::G_CONSTANT_FOLD_BARRIER:1860 return false;1861 case TargetOpcode::G_SHL:1862 case TargetOpcode::G_ASHR:1863 case TargetOpcode::G_LSHR:1864 return includesPoison(Kind) &&1865 !shiftAmountKnownInRange(RegDef->getOperand(2).getReg(), MRI);1866 case TargetOpcode::G_FPTOSI:1867 case TargetOpcode::G_FPTOUI:1868 // fptosi/ui yields poison if the resulting value does not fit in the1869 // destination type.1870 return true;1871 case TargetOpcode::G_CTLZ:1872 case TargetOpcode::G_CTTZ:1873 case TargetOpcode::G_ABS:1874 case TargetOpcode::G_CTPOP:1875 case TargetOpcode::G_BSWAP:1876 case TargetOpcode::G_BITREVERSE:1877 case TargetOpcode::G_FSHL:1878 case TargetOpcode::G_FSHR:1879 case TargetOpcode::G_SMAX:1880 case TargetOpcode::G_SMIN:1881 case TargetOpcode::G_SCMP:1882 case TargetOpcode::G_UMAX:1883 case TargetOpcode::G_UMIN:1884 case TargetOpcode::G_UCMP:1885 case TargetOpcode::G_PTRMASK:1886 case TargetOpcode::G_SADDO:1887 case TargetOpcode::G_SSUBO:1888 case TargetOpcode::G_UADDO:1889 case TargetOpcode::G_USUBO:1890 case TargetOpcode::G_SMULO:1891 case TargetOpcode::G_UMULO:1892 case TargetOpcode::G_SADDSAT:1893 case TargetOpcode::G_UADDSAT:1894 case TargetOpcode::G_SSUBSAT:1895 case TargetOpcode::G_USUBSAT:1896 case TargetOpcode::G_SBFX:1897 case TargetOpcode::G_UBFX:1898 return false;1899 case TargetOpcode::G_SSHLSAT:1900 case TargetOpcode::G_USHLSAT:1901 return includesPoison(Kind) &&1902 !shiftAmountKnownInRange(RegDef->getOperand(2).getReg(), MRI);1903 case TargetOpcode::G_INSERT_VECTOR_ELT: {1904 GInsertVectorElement *Insert = cast<GInsertVectorElement>(RegDef);1905 if (includesPoison(Kind)) {1906 std::optional<ValueAndVReg> Index =1907 getIConstantVRegValWithLookThrough(Insert->getIndexReg(), MRI);1908 if (!Index)1909 return true;1910 LLT VecTy = MRI.getType(Insert->getVectorReg());1911 return Index->Value.uge(VecTy.getElementCount().getKnownMinValue());1912 }1913 return false;1914 }1915 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {1916 GExtractVectorElement *Extract = cast<GExtractVectorElement>(RegDef);1917 if (includesPoison(Kind)) {1918 std::optional<ValueAndVReg> Index =1919 getIConstantVRegValWithLookThrough(Extract->getIndexReg(), MRI);1920 if (!Index)1921 return true;1922 LLT VecTy = MRI.getType(Extract->getVectorReg());1923 return Index->Value.uge(VecTy.getElementCount().getKnownMinValue());1924 }1925 return false;1926 }1927 case TargetOpcode::G_SHUFFLE_VECTOR: {1928 GShuffleVector *Shuffle = cast<GShuffleVector>(RegDef);1929 ArrayRef<int> Mask = Shuffle->getMask();1930 return includesPoison(Kind) && is_contained(Mask, -1);1931 }1932 case TargetOpcode::G_FNEG:1933 case TargetOpcode::G_PHI:1934 case TargetOpcode::G_SELECT:1935 case TargetOpcode::G_UREM:1936 case TargetOpcode::G_SREM:1937 case TargetOpcode::G_FREEZE:1938 case TargetOpcode::G_ICMP:1939 case TargetOpcode::G_FCMP:1940 case TargetOpcode::G_FADD:1941 case TargetOpcode::G_FSUB:1942 case TargetOpcode::G_FMUL:1943 case TargetOpcode::G_FDIV:1944 case TargetOpcode::G_FREM:1945 case TargetOpcode::G_PTR_ADD:1946 return false;1947 default:1948 return !isa<GCastOp>(RegDef) && !isa<GBinOp>(RegDef);1949 }1950}1951 1952static bool isGuaranteedNotToBeUndefOrPoison(Register Reg,1953 const MachineRegisterInfo &MRI,1954 unsigned Depth,1955 UndefPoisonKind Kind) {1956 if (Depth >= MaxAnalysisRecursionDepth)1957 return false;1958 1959 MachineInstr *RegDef = MRI.getVRegDef(Reg);1960 1961 switch (RegDef->getOpcode()) {1962 case TargetOpcode::G_FREEZE:1963 return true;1964 case TargetOpcode::G_IMPLICIT_DEF:1965 return !includesUndef(Kind);1966 case TargetOpcode::G_CONSTANT:1967 case TargetOpcode::G_FCONSTANT:1968 return true;1969 case TargetOpcode::G_BUILD_VECTOR: {1970 GBuildVector *BV = cast<GBuildVector>(RegDef);1971 unsigned NumSources = BV->getNumSources();1972 for (unsigned I = 0; I < NumSources; ++I)1973 if (!::isGuaranteedNotToBeUndefOrPoison(BV->getSourceReg(I), MRI,1974 Depth + 1, Kind))1975 return false;1976 return true;1977 }1978 case TargetOpcode::G_PHI: {1979 GPhi *Phi = cast<GPhi>(RegDef);1980 unsigned NumIncoming = Phi->getNumIncomingValues();1981 for (unsigned I = 0; I < NumIncoming; ++I)1982 if (!::isGuaranteedNotToBeUndefOrPoison(Phi->getIncomingValue(I), MRI,1983 Depth + 1, Kind))1984 return false;1985 return true;1986 }1987 default: {1988 auto MOCheck = [&](const MachineOperand &MO) {1989 if (!MO.isReg())1990 return true;1991 return ::isGuaranteedNotToBeUndefOrPoison(MO.getReg(), MRI, Depth + 1,1992 Kind);1993 };1994 return !::canCreateUndefOrPoison(Reg, MRI,1995 /*ConsiderFlagsAndMetadata=*/true, Kind) &&1996 all_of(RegDef->uses(), MOCheck);1997 }1998 }1999}2000 2001bool llvm::canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI,2002 bool ConsiderFlagsAndMetadata) {2003 return ::canCreateUndefOrPoison(Reg, MRI, ConsiderFlagsAndMetadata,2004 UndefPoisonKind::UndefOrPoison);2005}2006 2007bool canCreatePoison(Register Reg, const MachineRegisterInfo &MRI,2008 bool ConsiderFlagsAndMetadata = true) {2009 return ::canCreateUndefOrPoison(Reg, MRI, ConsiderFlagsAndMetadata,2010 UndefPoisonKind::PoisonOnly);2011}2012 2013bool llvm::isGuaranteedNotToBeUndefOrPoison(Register Reg,2014 const MachineRegisterInfo &MRI,2015 unsigned Depth) {2016 return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth,2017 UndefPoisonKind::UndefOrPoison);2018}2019 2020bool llvm::isGuaranteedNotToBePoison(Register Reg,2021 const MachineRegisterInfo &MRI,2022 unsigned Depth) {2023 return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth,2024 UndefPoisonKind::PoisonOnly);2025}2026 2027bool llvm::isGuaranteedNotToBeUndef(Register Reg,2028 const MachineRegisterInfo &MRI,2029 unsigned Depth) {2030 return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth,2031 UndefPoisonKind::UndefOnly);2032}2033 2034Type *llvm::getTypeForLLT(LLT Ty, LLVMContext &C) {2035 if (Ty.isVector())2036 return VectorType::get(IntegerType::get(C, Ty.getScalarSizeInBits()),2037 Ty.getElementCount());2038 return IntegerType::get(C, Ty.getSizeInBits());2039}2040 2041bool llvm::isAssertMI(const MachineInstr &MI) {2042 switch (MI.getOpcode()) {2043 default:2044 return false;2045 case TargetOpcode::G_ASSERT_ALIGN:2046 case TargetOpcode::G_ASSERT_SEXT:2047 case TargetOpcode::G_ASSERT_ZEXT:2048 return true;2049 }2050}2051 2052APInt llvm::GIConstant::getScalarValue() const {2053 assert(Kind == GIConstantKind::Scalar && "Expected scalar constant");2054 2055 return Value;2056}2057 2058std::optional<GIConstant>2059llvm::GIConstant::getConstant(Register Const, const MachineRegisterInfo &MRI) {2060 MachineInstr *Constant = getDefIgnoringCopies(Const, MRI);2061 2062 if (GSplatVector *Splat = dyn_cast<GSplatVector>(Constant)) {2063 std::optional<ValueAndVReg> MayBeConstant =2064 getIConstantVRegValWithLookThrough(Splat->getScalarReg(), MRI);2065 if (!MayBeConstant)2066 return std::nullopt;2067 return GIConstant(MayBeConstant->Value, GIConstantKind::ScalableVector);2068 }2069 2070 if (GBuildVector *Build = dyn_cast<GBuildVector>(Constant)) {2071 SmallVector<APInt> Values;2072 unsigned NumSources = Build->getNumSources();2073 for (unsigned I = 0; I < NumSources; ++I) {2074 Register SrcReg = Build->getSourceReg(I);2075 std::optional<ValueAndVReg> MayBeConstant =2076 getIConstantVRegValWithLookThrough(SrcReg, MRI);2077 if (!MayBeConstant)2078 return std::nullopt;2079 Values.push_back(MayBeConstant->Value);2080 }2081 return GIConstant(Values);2082 }2083 2084 std::optional<ValueAndVReg> MayBeConstant =2085 getIConstantVRegValWithLookThrough(Const, MRI);2086 if (!MayBeConstant)2087 return std::nullopt;2088 2089 return GIConstant(MayBeConstant->Value, GIConstantKind::Scalar);2090}2091 2092APFloat llvm::GFConstant::getScalarValue() const {2093 assert(Kind == GFConstantKind::Scalar && "Expected scalar constant");2094 2095 return Values[0];2096}2097 2098std::optional<GFConstant>2099llvm::GFConstant::getConstant(Register Const, const MachineRegisterInfo &MRI) {2100 MachineInstr *Constant = getDefIgnoringCopies(Const, MRI);2101 2102 if (GSplatVector *Splat = dyn_cast<GSplatVector>(Constant)) {2103 std::optional<FPValueAndVReg> MayBeConstant =2104 getFConstantVRegValWithLookThrough(Splat->getScalarReg(), MRI);2105 if (!MayBeConstant)2106 return std::nullopt;2107 return GFConstant(MayBeConstant->Value, GFConstantKind::ScalableVector);2108 }2109 2110 if (GBuildVector *Build = dyn_cast<GBuildVector>(Constant)) {2111 SmallVector<APFloat> Values;2112 unsigned NumSources = Build->getNumSources();2113 for (unsigned I = 0; I < NumSources; ++I) {2114 Register SrcReg = Build->getSourceReg(I);2115 std::optional<FPValueAndVReg> MayBeConstant =2116 getFConstantVRegValWithLookThrough(SrcReg, MRI);2117 if (!MayBeConstant)2118 return std::nullopt;2119 Values.push_back(MayBeConstant->Value);2120 }2121 return GFConstant(Values);2122 }2123 2124 std::optional<FPValueAndVReg> MayBeConstant =2125 getFConstantVRegValWithLookThrough(Const, MRI);2126 if (!MayBeConstant)2127 return std::nullopt;2128 2129 return GFConstant(MayBeConstant->Value, GFConstantKind::Scalar);2130}2131