346 lines · cpp
1//===- llvm/Target/TargetSchedule.cpp - Sched Machine Model ---------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements a wrapper around MCSchedModel that allows the interface10// to benefit from information currently only available in TargetInstrInfo.11//12//===----------------------------------------------------------------------===//13 14#include "llvm/CodeGen/TargetSchedule.h"15#include "llvm/CodeGen/MachineFunction.h"16#include "llvm/CodeGen/MachineInstr.h"17#include "llvm/CodeGen/MachineOperand.h"18#include "llvm/CodeGen/TargetInstrInfo.h"19#include "llvm/CodeGen/TargetSubtargetInfo.h"20#include "llvm/MC/MCInstrDesc.h"21#include "llvm/MC/MCInstrItineraries.h"22#include "llvm/MC/MCSchedule.h"23#include "llvm/Support/CommandLine.h"24#include "llvm/Support/ErrorHandling.h"25#include "llvm/Support/raw_ostream.h"26#include <algorithm>27#include <cassert>28 29using namespace llvm;30 31static cl::opt<bool> ForceEnableIntervals(32 "sched-model-force-enable-intervals", cl::Hidden, cl::init(false),33 cl::desc("Force the use of resource intervals in the schedule model"));34 35bool TargetSchedModel::hasInstrSchedModel() const {36 return EnableSchedModel && SchedModel.hasInstrSchedModel();37}38 39bool TargetSchedModel::hasInstrItineraries() const {40 return EnableSchedItins && !InstrItins.isEmpty();41}42 43void TargetSchedModel::init(const TargetSubtargetInfo *TSInfo,44 bool EnableSModel, bool EnableSItins) {45 STI = TSInfo;46 SchedModel = TSInfo->getSchedModel();47 TII = TSInfo->getInstrInfo();48 STI->initInstrItins(InstrItins);49 50 EnableSchedModel = EnableSModel;51 EnableSchedItins = EnableSItins;52 53 unsigned NumRes = SchedModel.getNumProcResourceKinds();54 ResourceFactors.resize(NumRes);55 ResourceLCM = SchedModel.IssueWidth;56 for (unsigned Idx = 0; Idx < NumRes; ++Idx) {57 unsigned NumUnits = SchedModel.getProcResource(Idx)->NumUnits;58 if (NumUnits > 0)59 ResourceLCM = std::lcm(ResourceLCM, NumUnits);60 }61 MicroOpFactor = ResourceLCM / SchedModel.IssueWidth;62 for (unsigned Idx = 0; Idx < NumRes; ++Idx) {63 unsigned NumUnits = SchedModel.getProcResource(Idx)->NumUnits;64 ResourceFactors[Idx] = NumUnits ? (ResourceLCM / NumUnits) : 0;65 }66}67 68/// Returns true only if instruction is specified as single issue.69bool TargetSchedModel::mustBeginGroup(const MachineInstr *MI,70 const MCSchedClassDesc *SC) const {71 if (hasInstrSchedModel()) {72 if (!SC)73 SC = resolveSchedClass(MI);74 if (SC->isValid())75 return SC->BeginGroup;76 }77 return false;78}79 80bool TargetSchedModel::mustEndGroup(const MachineInstr *MI,81 const MCSchedClassDesc *SC) const {82 if (hasInstrSchedModel()) {83 if (!SC)84 SC = resolveSchedClass(MI);85 if (SC->isValid())86 return SC->EndGroup;87 }88 return false;89}90 91unsigned TargetSchedModel::getNumMicroOps(const MachineInstr *MI,92 const MCSchedClassDesc *SC) const {93 if (hasInstrItineraries()) {94 int UOps = InstrItins.getNumMicroOps(MI->getDesc().getSchedClass());95 return (UOps >= 0) ? UOps : TII->getNumMicroOps(&InstrItins, *MI);96 }97 if (hasInstrSchedModel()) {98 if (!SC)99 SC = resolveSchedClass(MI);100 if (SC->isValid())101 return SC->NumMicroOps;102 }103 return MI->isTransient() ? 0 : 1;104}105 106// The machine model may explicitly specify an invalid latency, which107// effectively means infinite latency. Since users of the TargetSchedule API108// don't know how to handle this, we convert it to a very large latency that is109// easy to distinguish when debugging the DAG but won't induce overflow.110static unsigned capLatency(int Cycles) {111 return Cycles >= 0 ? Cycles : 1000;112}113 114/// Return the MCSchedClassDesc for this instruction. Some SchedClasses require115/// evaluation of predicates that depend on instruction operands or flags.116const MCSchedClassDesc *TargetSchedModel::117resolveSchedClass(const MachineInstr *MI) const {118 // Get the definition's scheduling class descriptor from this machine model.119 unsigned SchedClass = MI->getDesc().getSchedClass();120 const MCSchedClassDesc *SCDesc = SchedModel.getSchedClassDesc(SchedClass);121 if (!SCDesc->isValid())122 return SCDesc;123 124#ifndef NDEBUG125 unsigned NIter = 0;126#endif127 while (SCDesc->isVariant()) {128 assert(++NIter < 6 && "Variants are nested deeper than the magic number");129 130 SchedClass = STI->resolveSchedClass(SchedClass, MI, this);131 SCDesc = SchedModel.getSchedClassDesc(SchedClass);132 }133 return SCDesc;134}135 136/// Find the def index of this operand. This index maps to the machine model and137/// is independent of use operands. Def operands may be reordered with uses or138/// merged with uses without affecting the def index (e.g. before/after139/// regalloc). However, an instruction's def operands must never be reordered140/// with respect to each other.141static unsigned findDefIdx(const MachineInstr *MI, unsigned DefOperIdx) {142 unsigned DefIdx = 0;143 for (unsigned i = 0; i != DefOperIdx; ++i) {144 const MachineOperand &MO = MI->getOperand(i);145 if (MO.isReg() && MO.isDef())146 ++DefIdx;147 }148 return DefIdx;149}150 151/// Find the use index of this operand. This is independent of the instruction's152/// def operands.153///154/// Note that uses are not determined by the operand's isUse property, which155/// is simply the inverse of isDef. Here we consider any readsReg operand to be156/// a "use". The machine model allows an operand to be both a Def and Use.157static unsigned findUseIdx(const MachineInstr *MI, unsigned UseOperIdx) {158 unsigned UseIdx = 0;159 for (unsigned i = 0; i != UseOperIdx; ++i) {160 const MachineOperand &MO = MI->getOperand(i);161 if (MO.isReg() && MO.readsReg() && !MO.isDef())162 ++UseIdx;163 }164 return UseIdx;165}166 167// Top-level API for clients that know the operand indices. This doesn't need to168// return std::optional<unsigned>, as it always returns a valid latency.169unsigned TargetSchedModel::computeOperandLatency(170 const MachineInstr *DefMI, unsigned DefOperIdx,171 const MachineInstr *UseMI, unsigned UseOperIdx) const {172 173 const unsigned InstrLatency = computeInstrLatency(DefMI);174 const unsigned DefaultDefLatency = TII->defaultDefLatency(SchedModel, *DefMI);175 176 if (!hasInstrSchedModel() && !hasInstrItineraries())177 return DefaultDefLatency;178 179 if (hasInstrItineraries()) {180 std::optional<unsigned> OperLatency;181 if (UseMI) {182 OperLatency = TII->getOperandLatency(&InstrItins, *DefMI, DefOperIdx,183 *UseMI, UseOperIdx);184 }185 else {186 unsigned DefClass = DefMI->getDesc().getSchedClass();187 OperLatency = InstrItins.getOperandCycle(DefClass, DefOperIdx);188 }189 190 // Expected latency is the max of InstrLatency and DefaultDefLatency, if we191 // didn't find an operand latency.192 return OperLatency ? *OperLatency193 : std::max(InstrLatency, DefaultDefLatency);194 }195 196 // hasInstrSchedModel()197 const MCSchedClassDesc *SCDesc = resolveSchedClass(DefMI);198 unsigned DefIdx = findDefIdx(DefMI, DefOperIdx);199 if (DefIdx < SCDesc->NumWriteLatencyEntries) {200 // Lookup the definition's write latency in SubtargetInfo.201 const MCWriteLatencyEntry *WLEntry =202 STI->getWriteLatencyEntry(SCDesc, DefIdx);203 unsigned WriteID = WLEntry->WriteResourceID;204 unsigned Latency = capLatency(WLEntry->Cycles);205 if (!UseMI)206 return Latency;207 208 // Lookup the use's latency adjustment in SubtargetInfo.209 const MCSchedClassDesc *UseDesc = resolveSchedClass(UseMI);210 if (UseDesc->NumReadAdvanceEntries == 0)211 return Latency;212 unsigned UseIdx = findUseIdx(UseMI, UseOperIdx);213 int Advance = STI->getReadAdvanceCycles(UseDesc, UseIdx, WriteID);214 if (Advance > 0 && (unsigned)Advance > Latency) // unsigned wrap215 return 0;216 return Latency - Advance;217 }218 // If DefIdx does not exist in the model (e.g. implicit defs), then return219 // unit latency (defaultDefLatency may be too conservative).220#ifndef NDEBUG221 if (SCDesc->isValid() && !DefMI->getOperand(DefOperIdx).isImplicit() &&222 !DefMI->getDesc().operands()[DefOperIdx].isOptionalDef() &&223 SchedModel.isComplete()) {224 errs() << "DefIdx " << DefIdx << " exceeds machine model writes for "225 << *DefMI << " (Try with MCSchedModel.CompleteModel set to false)";226 llvm_unreachable("incomplete machine model");227 }228#endif229 // FIXME: Automatically giving all implicit defs defaultDefLatency is230 // undesirable. We should only do it for defs that are known to the MC231 // desc like flags. Truly implicit defs should get 1 cycle latency.232 return DefMI->isTransient() ? 0 : DefaultDefLatency;233}234 235unsigned236TargetSchedModel::computeInstrLatency(const MCSchedClassDesc &SCDesc) const {237 return capLatency(MCSchedModel::computeInstrLatency(*STI, SCDesc));238}239 240unsigned TargetSchedModel::computeInstrLatency(unsigned Opcode) const {241 assert(hasInstrSchedModel() && "Only call this function with a SchedModel");242 unsigned SCIdx = TII->get(Opcode).getSchedClass();243 return capLatency(SchedModel.computeInstrLatency(*STI, SCIdx));244}245 246unsigned TargetSchedModel::computeInstrLatency(const MCInst &Inst) const {247 if (hasInstrSchedModel())248 return capLatency(SchedModel.computeInstrLatency(*STI, *TII, Inst));249 return computeInstrLatency(Inst.getOpcode());250}251 252unsigned253TargetSchedModel::computeInstrLatency(const MachineInstr *MI,254 bool UseDefaultDefLatency) const {255 // For the itinerary model, fall back to the old subtarget hook.256 // Allow subtargets to compute Bundle latencies outside the machine model.257 if (hasInstrItineraries() || MI->isBundle() ||258 (!hasInstrSchedModel() && !UseDefaultDefLatency))259 return TII->getInstrLatency(&InstrItins, *MI);260 261 if (hasInstrSchedModel()) {262 const MCSchedClassDesc *SCDesc = resolveSchedClass(MI);263 if (SCDesc->isValid())264 return computeInstrLatency(*SCDesc);265 }266 return TII->defaultDefLatency(SchedModel, *MI);267}268 269unsigned TargetSchedModel::270computeOutputLatency(const MachineInstr *DefMI, unsigned DefOperIdx,271 const MachineInstr *DepMI) const {272 if (!SchedModel.isOutOfOrder())273 return 1;274 275 // Out-of-order processor can dispatch WAW dependencies in the same cycle.276 277 // Treat predication as a data dependency for out-of-order cpus. In-order278 // cpus do not need to treat predicated writes specially.279 //280 // TODO: The following hack exists because predication passes do not281 // correctly append imp-use operands, and readsReg() strangely returns false282 // for predicated defs.283 Register Reg = DefMI->getOperand(DefOperIdx).getReg();284 const MachineFunction &MF = *DefMI->getMF();285 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();286 if (!DepMI->readsRegister(Reg, TRI) && TII->isPredicated(*DepMI))287 return computeInstrLatency(DefMI);288 289 // If we have a per operand scheduling model, check if this def is writing290 // an unbuffered resource. If so, it treated like an in-order cpu.291 if (hasInstrSchedModel()) {292 const MCSchedClassDesc *SCDesc = resolveSchedClass(DefMI);293 if (SCDesc->isValid()) {294 for (const MCWriteProcResEntry *PRI = STI->getWriteProcResBegin(SCDesc),295 *PRE = STI->getWriteProcResEnd(SCDesc); PRI != PRE; ++PRI) {296 if (!SchedModel.getProcResource(PRI->ProcResourceIdx)->BufferSize)297 return 1;298 }299 }300 }301 return 0;302}303 304double305TargetSchedModel::computeReciprocalThroughput(const MachineInstr *MI) const {306 if (hasInstrItineraries()) {307 unsigned SchedClass = MI->getDesc().getSchedClass();308 return MCSchedModel::getReciprocalThroughput(SchedClass,309 *getInstrItineraries());310 }311 312 if (hasInstrSchedModel())313 return MCSchedModel::getReciprocalThroughput(*STI, *resolveSchedClass(MI));314 315 return 0.0;316}317 318double319TargetSchedModel::computeReciprocalThroughput(unsigned Opcode) const {320 unsigned SchedClass = TII->get(Opcode).getSchedClass();321 if (hasInstrItineraries())322 return MCSchedModel::getReciprocalThroughput(SchedClass,323 *getInstrItineraries());324 if (hasInstrSchedModel()) {325 const MCSchedClassDesc &SCDesc = *SchedModel.getSchedClassDesc(SchedClass);326 if (SCDesc.isValid() && !SCDesc.isVariant())327 return MCSchedModel::getReciprocalThroughput(*STI, SCDesc);328 }329 330 return 0.0;331}332 333double334TargetSchedModel::computeReciprocalThroughput(const MCInst &MI) const {335 if (hasInstrSchedModel())336 return SchedModel.getReciprocalThroughput(*STI, *TII, MI);337 return computeReciprocalThroughput(MI.getOpcode());338}339 340bool TargetSchedModel::enableIntervals() const {341 if (ForceEnableIntervals)342 return true;343 344 return SchedModel.EnableIntervals;345}346