167 lines · c
1//===-- GCNHazardRecognizers.h - GCN Hazard Recognizers ---------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file defines hazard recognizers for scheduling on GCN processors.10//11//===----------------------------------------------------------------------===//12 13#ifndef LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H14#define LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H15 16#include "llvm/ADT/BitVector.h"17#include "llvm/ADT/STLExtras.h"18#include "llvm/CodeGen/ScheduleHazardRecognizer.h"19#include "llvm/CodeGen/TargetSchedule.h"20#include <list>21 22namespace llvm {23 24class MachineFunction;25class MachineInstr;26class MachineOperand;27class MachineRegisterInfo;28class SIInstrInfo;29class SIRegisterInfo;30class GCNSubtarget;31 32class GCNHazardRecognizer final : public ScheduleHazardRecognizer {33public:34 typedef function_ref<bool(const MachineInstr &)> IsHazardFn;35 typedef function_ref<bool(const MachineInstr &, int WaitStates)> IsExpiredFn;36 typedef function_ref<unsigned int(const MachineInstr &)> GetNumWaitStatesFn;37 38private:39 // Distinguish if we are called from scheduler or hazard recognizer40 bool IsHazardRecognizerMode;41 42 // This variable stores the instruction that has been emitted this cycle. It43 // will be added to EmittedInstrs, when AdvanceCycle() or RecedeCycle() is44 // called.45 MachineInstr *CurrCycleInstr;46 std::list<MachineInstr*> EmittedInstrs;47 const MachineFunction &MF;48 const GCNSubtarget &ST;49 const SIInstrInfo &TII;50 const SIRegisterInfo &TRI;51 const TargetSchedModel &TSchedModel;52 bool RunLdsBranchVmemWARHazardFixup;53 54 /// RegUnits of uses in the current soft memory clause.55 BitVector ClauseUses;56 57 /// RegUnits of defs in the current soft memory clause.58 BitVector ClauseDefs;59 60 void resetClause() {61 ClauseUses.reset();62 ClauseDefs.reset();63 }64 65 void addClauseInst(const MachineInstr &MI);66 67 /// \returns the number of wait states before another MFMA instruction can be68 /// issued after \p MI.69 unsigned getMFMAPipelineWaitStates(const MachineInstr &MI) const;70 71 // Advance over a MachineInstr bundle. Look for hazards in the bundled72 // instructions.73 void processBundle();74 75 // Run on an individual instruction in hazard recognizer mode. This can be76 // used on a newly inserted instruction before returning from PreEmitNoops.77 void runOnInstruction(MachineInstr *MI);78 79 int getWaitStatesSince(IsHazardFn IsHazard, int Limit,80 GetNumWaitStatesFn GetNumWaitStates);81 int getWaitStatesSince(IsHazardFn IsHazard, int Limit);82 int getWaitStatesSinceDef(unsigned Reg, IsHazardFn IsHazardDef, int Limit);83 int getWaitStatesSinceSetReg(IsHazardFn IsHazard, int Limit);84 85 int checkSoftClauseHazards(MachineInstr *SMEM);86 int checkSMRDHazards(MachineInstr *SMRD);87 int checkVMEMHazards(MachineInstr* VMEM);88 int checkDPPHazards(MachineInstr *DPP);89 int checkDivFMasHazards(MachineInstr *DivFMas);90 int checkGetRegHazards(MachineInstr *GetRegInstr);91 int checkSetRegHazards(MachineInstr *SetRegInstr);92 int createsVALUHazard(const MachineInstr &MI);93 int checkVALUHazards(MachineInstr *VALU);94 int checkVALUHazardsHelper(const MachineOperand &Def, const MachineRegisterInfo &MRI);95 int checkRWLaneHazards(MachineInstr *RWLane);96 int checkRFEHazards(MachineInstr *RFE);97 int checkInlineAsmHazards(MachineInstr *IA);98 int checkReadM0Hazards(MachineInstr *SMovRel);99 int checkNSAtoVMEMHazard(MachineInstr *MI);100 int checkFPAtomicToDenormModeHazard(MachineInstr *MI);101 // Emit V_NOP instructions. \p WaitStatesNeeded is the number of V_NOPs we102 // need to insert, negative means not needed.103 bool emitVNops(MachineInstr *MI, int WaitStatesNeeded);104 void fixHazards(MachineInstr *MI);105 bool fixVcmpxPermlaneHazards(MachineInstr *MI);106 bool fixVMEMtoScalarWriteHazards(MachineInstr *MI);107 bool fixSMEMtoVectorWriteHazards(MachineInstr *MI);108 bool fixVcmpxExecWARHazard(MachineInstr *MI);109 bool fixLdsBranchVmemWARHazard(MachineInstr *MI);110 bool fixLdsDirectVALUHazard(MachineInstr *MI);111 bool fixLdsDirectVMEMHazard(MachineInstr *MI);112 bool fixVALUPartialForwardingHazard(MachineInstr *MI);113 bool fixVALUTransUseHazard(MachineInstr *MI);114 bool fixVALUTransCoexecutionHazards(MachineInstr *MI);115 bool fixWMMAHazards(MachineInstr *MI);116 int checkWMMACoexecutionHazards(MachineInstr *MI);117 bool fixShift64HighRegBug(MachineInstr *MI);118 bool fixVALUMaskWriteHazard(MachineInstr *MI);119 bool fixRequiredExportPriority(MachineInstr *MI);120 bool fixGetRegWaitIdle(MachineInstr *MI);121 bool fixDsAtomicAsyncBarrierArriveB64(MachineInstr *MI);122 bool fixScratchBaseForwardingHazard(MachineInstr *MI);123 bool fixSetRegMode(MachineInstr *MI);124 125 int checkMAIHazards(MachineInstr *MI);126 int checkMAIHazards908(MachineInstr *MI);127 int checkMAIHazards90A(MachineInstr *MI);128 /// Pad the latency between neighboring MFMA instructions with s_nops. The129 /// percentage of wait states to fill with s_nops is specified by the command130 /// line option '-amdgpu-mfma-padding-ratio'.131 ///132 /// For example, with '-amdgpu-mfma-padding-ratio=100':133 ///134 /// 2 pass MFMA instructions have a latency of 2 wait states. Therefore, a135 /// 'S_NOP 1' will be added between sequential MFMA instructions.136 ///137 /// V_MFMA_F32_4X4X1F32138 /// V_MFMA_F32_4X4X1F32139 ///-->140 /// V_MFMA_F32_4X4X1F32141 /// S_NOP 1142 /// V_MFMA_F32_4X4X1F32143 int checkMFMAPadding(MachineInstr *MI);144 int checkMAIVALUHazards(MachineInstr *MI);145 int checkMAILdStHazards(MachineInstr *MI);146 int checkPermlaneHazards(MachineInstr *MI);147 148public:149 GCNHazardRecognizer(const MachineFunction &MF);150 // We can only issue one instruction per cycle.151 bool atIssueLimit() const override { return true; }152 void EmitInstruction(SUnit *SU) override;153 void EmitInstruction(MachineInstr *MI) override;154 HazardType getHazardType(SUnit *SU, int Stalls) override;155 void EmitNoop() override;156 unsigned PreEmitNoops(MachineInstr *) override;157 unsigned PreEmitNoopsCommon(MachineInstr *);158 void AdvanceCycle() override;159 void RecedeCycle() override;160 bool ShouldPreferAnother(SUnit *SU) override;161 void Reset() override;162};163 164} // end namespace llvm165 166#endif //LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H167