323 lines · c
1//===-- AMDGPUISelDAGToDAG.h - A dag to dag inst selector for AMDGPU ----===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//==-----------------------------------------------------------------------===//8//9/// \file10/// Defines an instruction selector for the AMDGPU target.11//12//===----------------------------------------------------------------------===//13 14#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H15#define LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H16 17#include "AMDGPUSelectionDAGInfo.h"18#include "GCNSubtarget.h"19#include "SIMachineFunctionInfo.h"20#include "SIModeRegisterDefaults.h"21#include "llvm/Analysis/ValueTracking.h"22#include "llvm/CodeGen/SelectionDAGISel.h"23#include "llvm/Support/AMDGPUAddrSpace.h"24#include "llvm/Target/TargetMachine.h"25 26namespace llvm {27 28static inline bool getConstantValue(SDValue N, uint32_t &Out) {29 // This is only used for packed vectors, where using 0 for undef should30 // always be good.31 if (N.isUndef()) {32 Out = 0;33 return true;34 }35 36 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {37 Out = C->getAPIntValue().getSExtValue();38 return true;39 }40 41 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) {42 Out = C->getValueAPF().bitcastToAPInt().getSExtValue();43 return true;44 }45 46 return false;47}48 49/// AMDGPU specific code to select AMDGPU machine instructions for50/// SelectionDAG operations.51class AMDGPUDAGToDAGISel : public SelectionDAGISel {52 // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can53 // make the right decision when generating code for different targets.54 const GCNSubtarget *Subtarget;55 56 // Default FP mode for the current function.57 SIModeRegisterDefaults Mode;58 59 // Instructions that will be lowered with a final instruction that zeros the60 // high result bits.61 bool fp16SrcZerosHighBits(unsigned Opc) const;62 63public:64 AMDGPUDAGToDAGISel() = delete;65 66 explicit AMDGPUDAGToDAGISel(TargetMachine &TM, CodeGenOptLevel OptLevel);67 68 bool runOnMachineFunction(MachineFunction &MF) override;69 bool matchLoadD16FromBuildVector(SDNode *N) const;70 void PreprocessISelDAG() override;71 void Select(SDNode *N) override;72 void PostprocessISelDAG() override;73 74protected:75 void SelectBuildVector(SDNode *N, unsigned RegClassID);76 void SelectVectorShuffle(SDNode *N);77 78private:79 std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const;80 81 bool isInlineImmediate(const SDNode *N) const;82 83 bool isInlineImmediate(const APInt &Imm) const {84 return Subtarget->getInstrInfo()->isInlineConstant(Imm);85 }86 87 bool isInlineImmediate(const APFloat &Imm) const {88 return Subtarget->getInstrInfo()->isInlineConstant(Imm);89 }90 91 bool isVGPRImm(const SDNode *N) const;92 bool isUniformLoad(const SDNode *N) const;93 bool isUniformBr(const SDNode *N) const;94 95 // Returns true if ISD::AND SDNode `N`'s masking of the shift amount operand's96 // `ShAmtBits` bits is unneeded.97 bool isUnneededShiftMask(const SDNode *N, unsigned ShAmtBits) const;98 99 bool isBaseWithConstantOffset64(SDValue Addr, SDValue &LHS,100 SDValue &RHS) const;101 102 MachineSDNode *buildSMovImm64(SDLoc &DL, uint64_t Val, EVT VT) const;103 104 SDNode *packConstantV2I16(const SDNode *N, SelectionDAG &DAG) const;105 106 SDNode *glueCopyToOp(SDNode *N, SDValue NewChain, SDValue Glue) const;107 SDNode *glueCopyToM0(SDNode *N, SDValue Val) const;108 SDNode *glueCopyToM0LDSInit(SDNode *N) const;109 110 const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const;111 virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);112 virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);113 bool isDSOffsetLegal(SDValue Base, unsigned Offset) const;114 bool isDSOffset2Legal(SDValue Base, unsigned Offset0, unsigned Offset1,115 unsigned Size) const;116 117 bool isFlatScratchBaseLegal(SDValue Addr) const;118 bool isFlatScratchBaseLegalSV(SDValue Addr) const;119 bool isFlatScratchBaseLegalSVImm(SDValue Addr) const;120 bool isSOffsetLegalWithImmOffset(SDValue *SOffset, bool Imm32Only,121 bool IsBuffer, int64_t ImmOffset = 0) const;122 123 bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const;124 bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,125 SDValue &Offset1) const;126 bool SelectDS128Bit8ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,127 SDValue &Offset1) const;128 bool SelectDSReadWrite2(SDValue Ptr, SDValue &Base, SDValue &Offset0,129 SDValue &Offset1, unsigned Size) const;130 bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,131 SDValue &SOffset, SDValue &Offset, SDValue &Offen,132 SDValue &Idxen, SDValue &Addr64) const;133 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,134 SDValue &SOffset, SDValue &Offset) const;135 bool SelectMUBUFScratchOffen(SDNode *Parent, SDValue Addr, SDValue &RSrc,136 SDValue &VAddr, SDValue &SOffset,137 SDValue &ImmOffset) const;138 bool SelectMUBUFScratchOffset(SDNode *Parent, SDValue Addr, SDValue &SRsrc,139 SDValue &Soffset, SDValue &Offset) const;140 141 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,142 SDValue &Offset) const;143 bool SelectBUFSOffset(SDValue Addr, SDValue &SOffset) const;144 145 bool SelectFlatOffsetImpl(SDNode *N, SDValue Addr, SDValue &VAddr,146 SDValue &Offset, uint64_t FlatVariant) const;147 bool SelectFlatOffset(SDNode *N, SDValue Addr, SDValue &VAddr,148 SDValue &Offset) const;149 bool SelectGlobalOffset(SDNode *N, SDValue Addr, SDValue &VAddr,150 SDValue &Offset) const;151 bool SelectScratchOffset(SDNode *N, SDValue Addr, SDValue &VAddr,152 SDValue &Offset) const;153 bool SelectGlobalSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,154 SDValue &VOffset, SDValue &Offset, bool &ScaleOffset,155 bool NeedIOffset = true) const;156 bool SelectGlobalSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,157 SDValue &VOffset, SDValue &Offset,158 SDValue &CPol) const;159 bool SelectGlobalSAddrCPol(SDNode *N, SDValue Addr, SDValue &SAddr,160 SDValue &VOffset, SDValue &Offset,161 SDValue &CPol) const;162 bool SelectGlobalSAddrCPolM0(SDNode *N, SDValue Addr, SDValue &SAddr,163 SDValue &VOffset, SDValue &Offset,164 SDValue &CPol) const;165 bool SelectGlobalSAddrGLC(SDNode *N, SDValue Addr, SDValue &SAddr,166 SDValue &VOffset, SDValue &Offset,167 SDValue &CPol) const;168 bool SelectGlobalSAddrNoIOffset(SDNode *N, SDValue Addr, SDValue &SAddr,169 SDValue &VOffset, SDValue &CPol) const;170 bool SelectGlobalSAddrNoIOffsetM0(SDNode *N, SDValue Addr, SDValue &SAddr,171 SDValue &VOffset, SDValue &CPol) const;172 bool SelectScratchSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,173 SDValue &Offset) const;174 bool checkFlatScratchSVSSwizzleBug(SDValue VAddr, SDValue SAddr,175 uint64_t ImmOffset) const;176 bool SelectScratchSVAddr(SDNode *N, SDValue Addr, SDValue &VAddr,177 SDValue &SAddr, SDValue &Offset,178 SDValue &CPol) const;179 180 bool SelectSMRDOffset(SDNode *N, SDValue ByteOffsetNode, SDValue *SOffset,181 SDValue *Offset, bool Imm32Only = false,182 bool IsBuffer = false, bool HasSOffset = false,183 int64_t ImmOffset = 0,184 bool *ScaleOffset = nullptr) const;185 SDValue Expand32BitAddress(SDValue Addr) const;186 bool SelectSMRDBaseOffset(SDNode *N, SDValue Addr, SDValue &SBase,187 SDValue *SOffset, SDValue *Offset,188 bool Imm32Only = false, bool IsBuffer = false,189 bool HasSOffset = false, int64_t ImmOffset = 0,190 bool *ScaleOffset = nullptr) const;191 bool SelectSMRD(SDNode *N, SDValue Addr, SDValue &SBase, SDValue *SOffset,192 SDValue *Offset, bool Imm32Only = false,193 bool *ScaleOffset = nullptr) const;194 bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;195 bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;196 bool SelectScaleOffset(SDNode *N, SDValue &Offset, bool IsSigned) const;197 bool SelectSMRDSgpr(SDNode *N, SDValue Addr, SDValue &SBase, SDValue &SOffset,198 SDValue &CPol) const;199 bool SelectSMRDSgprImm(SDNode *N, SDValue Addr, SDValue &SBase,200 SDValue &SOffset, SDValue &Offset,201 SDValue &CPol) const;202 bool SelectSMRDBufferImm(SDValue N, SDValue &Offset) const;203 bool SelectSMRDBufferImm32(SDValue N, SDValue &Offset) const;204 bool SelectSMRDBufferSgprImm(SDValue N, SDValue &SOffset,205 SDValue &Offset) const;206 bool SelectSMRDPrefetchImm(SDValue Addr, SDValue &SBase,207 SDValue &Offset) const;208 bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;209 210 bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods,211 bool IsCanonicalizing = true,212 bool AllowAbs = true) const;213 bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;214 bool SelectVOP3ModsNonCanonicalizing(SDValue In, SDValue &Src,215 SDValue &SrcMods) const;216 bool SelectVOP3BMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;217 bool SelectVOP3NoMods(SDValue In, SDValue &Src) const;218 bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,219 SDValue &Clamp, SDValue &Omod) const;220 bool SelectVOP3BMods0(SDValue In, SDValue &Src, SDValue &SrcMods,221 SDValue &Clamp, SDValue &Omod) const;222 bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods,223 SDValue &Clamp, SDValue &Omod) const;224 225 bool SelectVINTERPModsImpl(SDValue In, SDValue &Src, SDValue &SrcMods,226 bool OpSel) const;227 bool SelectVINTERPMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;228 bool SelectVINTERPModsHi(SDValue In, SDValue &Src, SDValue &SrcMods) const;229 230 bool SelectVOP3OMods(SDValue In, SDValue &Src, SDValue &Clamp,231 SDValue &Omod) const;232 233 bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods,234 bool IsDOT = false) const;235 bool SelectVOP3PModsDOT(SDValue In, SDValue &Src, SDValue &SrcMods) const;236 237 bool SelectWMMAOpSelVOP3PMods(SDValue In, SDValue &Src) const;238 239 bool SelectWMMAModsF32NegAbs(SDValue In, SDValue &Src,240 SDValue &SrcMods) const;241 bool SelectWMMAModsF16Neg(SDValue In, SDValue &Src, SDValue &SrcMods) const;242 bool SelectWMMAModsF16NegAbs(SDValue In, SDValue &Src,243 SDValue &SrcMods) const;244 bool SelectWMMAVISrc(SDValue In, SDValue &Src) const;245 246 bool SelectSWMMACIndex8(SDValue In, SDValue &Src, SDValue &IndexKey) const;247 bool SelectSWMMACIndex16(SDValue In, SDValue &Src, SDValue &IndexKey) const;248 bool SelectSWMMACIndex32(SDValue In, SDValue &Src, SDValue &IndexKey) const;249 250 bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const;251 252 bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;253 bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, unsigned &Mods,254 MVT VT) const;255 bool SelectVOP3PMadMixModsExt(SDValue In, SDValue &Src,256 SDValue &SrcMods) const;257 bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;258 bool SelectVOP3PMadMixBF16ModsExt(SDValue In, SDValue &Src,259 SDValue &SrcMods) const;260 bool SelectVOP3PMadMixBF16Mods(SDValue In, SDValue &Src,261 SDValue &SrcMods) const;262 263 bool SelectBITOP3(SDValue In, SDValue &Src0, SDValue &Src1, SDValue &Src2,264 SDValue &Tbl) const;265 266 SDValue getHi16Elt(SDValue In) const;267 268 SDValue getMaterializedScalarImm32(int64_t Val, const SDLoc &DL) const;269 270 void SelectADD_SUB_I64(SDNode *N);271 void SelectAddcSubb(SDNode *N);272 void SelectUADDO_USUBO(SDNode *N);273 void SelectDIV_SCALE(SDNode *N);274 void SelectMAD_64_32(SDNode *N);275 void SelectMUL_LOHI(SDNode *N);276 void SelectFMA_W_CHAIN(SDNode *N);277 void SelectFMUL_W_CHAIN(SDNode *N);278 SDNode *getBFE32(bool IsSigned, const SDLoc &DL, SDValue Val, uint32_t Offset,279 uint32_t Width);280 void SelectS_BFEFromShifts(SDNode *N);281 void SelectS_BFE(SDNode *N);282 bool isCBranchSCC(const SDNode *N) const;283 void SelectBRCOND(SDNode *N);284 void SelectFMAD_FMA(SDNode *N);285 void SelectFP_EXTEND(SDNode *N);286 void SelectDSAppendConsume(SDNode *N, unsigned IntrID);287 void SelectDSBvhStackIntrinsic(SDNode *N, unsigned IntrID);288 void SelectDS_GWS(SDNode *N, unsigned IntrID);289 void SelectInterpP1F16(SDNode *N);290 void SelectINTRINSIC_W_CHAIN(SDNode *N);291 void SelectINTRINSIC_WO_CHAIN(SDNode *N);292 void SelectINTRINSIC_VOID(SDNode *N);293 void SelectWAVE_ADDRESS(SDNode *N);294 void SelectSTACKRESTORE(SDNode *N);295 296protected:297 // Include the pieces autogenerated from the target description.298#include "AMDGPUGenDAGISel.inc"299};300 301class AMDGPUISelDAGToDAGPass : public SelectionDAGISelPass {302public:303 AMDGPUISelDAGToDAGPass(TargetMachine &TM);304 305 PreservedAnalyses run(MachineFunction &MF,306 MachineFunctionAnalysisManager &MFAM);307};308 309class AMDGPUDAGToDAGISelLegacy : public SelectionDAGISelLegacy {310public:311 static char ID;312 313 AMDGPUDAGToDAGISelLegacy(TargetMachine &TM, CodeGenOptLevel OptLevel);314 315 bool runOnMachineFunction(MachineFunction &MF) override;316 void getAnalysisUsage(AnalysisUsage &AU) const override;317 StringRef getPassName() const override;318};319 320} // namespace llvm321 322#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H323