brintos / llvm-project-archived public Read only

1//===-- AMDGPUTargetMachine.cpp - TargetMachine for hw codegen targets-----===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9/// \file10/// This file contains both AMDGPU target machine and the CodeGen pass builder.11/// The AMDGPU target machine contains all of the hardware specific information12/// needed to emit code for SI+ GPUs in the legacy pass manager pipeline. The13/// CodeGen pass builder handles the pass pipeline for new pass manager.14//15//===----------------------------------------------------------------------===//16 17#include "AMDGPUTargetMachine.h"18#include "AMDGPU.h"19#include "AMDGPUAliasAnalysis.h"20#include "AMDGPUBarrierLatency.h"21#include "AMDGPUCtorDtorLowering.h"22#include "AMDGPUExportClustering.h"23#include "AMDGPUExportKernelRuntimeHandles.h"24#include "AMDGPUIGroupLP.h"25#include "AMDGPUISelDAGToDAG.h"26#include "AMDGPULowerVGPREncoding.h"27#include "AMDGPUMacroFusion.h"28#include "AMDGPUPerfHintAnalysis.h"29#include "AMDGPUPreloadKernArgProlog.h"30#include "AMDGPUPrepareAGPRAlloc.h"31#include "AMDGPURemoveIncompatibleFunctions.h"32#include "AMDGPUReserveWWMRegs.h"33#include "AMDGPUResourceUsageAnalysis.h"34#include "AMDGPUSplitModule.h"35#include "AMDGPUTargetObjectFile.h"36#include "AMDGPUTargetTransformInfo.h"37#include "AMDGPUUnifyDivergentExitNodes.h"38#include "AMDGPUWaitSGPRHazards.h"39#include "GCNDPPCombine.h"40#include "GCNIterativeScheduler.h"41#include "GCNNSAReassign.h"42#include "GCNPreRALongBranchReg.h"43#include "GCNPreRAOptimizations.h"44#include "GCNRewritePartialRegUses.h"45#include "GCNSchedStrategy.h"46#include "GCNVOPDUtils.h"47#include "R600.h"48#include "R600TargetMachine.h"49#include "SIFixSGPRCopies.h"50#include "SIFixVGPRCopies.h"51#include "SIFoldOperands.h"52#include "SIFormMemoryClauses.h"53#include "SILoadStoreOptimizer.h"54#include "SILowerControlFlow.h"55#include "SILowerSGPRSpills.h"56#include "SILowerWWMCopies.h"57#include "SIMachineFunctionInfo.h"58#include "SIMachineScheduler.h"59#include "SIOptimizeExecMasking.h"60#include "SIOptimizeExecMaskingPreRA.h"61#include "SIOptimizeVGPRLiveRange.h"62#include "SIPeepholeSDWA.h"63#include "SIPostRABundler.h"64#include "SIPreAllocateWWMRegs.h"65#include "SIShrinkInstructions.h"66#include "SIWholeQuadMode.h"67#include "TargetInfo/AMDGPUTargetInfo.h"68#include "Utils/AMDGPUBaseInfo.h"69#include "llvm/Analysis/CGSCCPassManager.h"70#include "llvm/Analysis/CallGraphSCCPass.h"71#include "llvm/Analysis/KernelInfo.h"72#include "llvm/Analysis/UniformityAnalysis.h"73#include "llvm/CodeGen/AtomicExpand.h"74#include "llvm/CodeGen/BranchRelaxation.h"75#include "llvm/CodeGen/DeadMachineInstructionElim.h"76#include "llvm/CodeGen/GlobalISel/CSEInfo.h"77#include "llvm/CodeGen/GlobalISel/IRTranslator.h"78#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"79#include "llvm/CodeGen/GlobalISel/Legalizer.h"80#include "llvm/CodeGen/GlobalISel/Localizer.h"81#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"82#include "llvm/CodeGen/MIRParser/MIParser.h"83#include "llvm/CodeGen/MachineCSE.h"84#include "llvm/CodeGen/MachineLICM.h"85#include "llvm/CodeGen/MachineScheduler.h"86#include "llvm/CodeGen/Passes.h"87#include "llvm/CodeGen/PostRAHazardRecognizer.h"88#include "llvm/CodeGen/RegAllocRegistry.h"89#include "llvm/CodeGen/TargetPassConfig.h"90#include "llvm/IR/IntrinsicsAMDGPU.h"91#include "llvm/IR/PassManager.h"92#include "llvm/IR/PatternMatch.h"93#include "llvm/InitializePasses.h"94#include "llvm/MC/TargetRegistry.h"95#include "llvm/Passes/CodeGenPassBuilder.h"96#include "llvm/Passes/PassBuilder.h"97#include "llvm/Support/Compiler.h"98#include "llvm/Support/FormatVariadic.h"99#include "llvm/Transforms/HipStdPar/HipStdPar.h"100#include "llvm/Transforms/IPO.h"101#include "llvm/Transforms/IPO/AlwaysInliner.h"102#include "llvm/Transforms/IPO/ExpandVariadics.h"103#include "llvm/Transforms/IPO/GlobalDCE.h"104#include "llvm/Transforms/IPO/Internalize.h"105#include "llvm/Transforms/Scalar.h"106#include "llvm/Transforms/Scalar/EarlyCSE.h"107#include "llvm/Transforms/Scalar/FlattenCFG.h"108#include "llvm/Transforms/Scalar/GVN.h"109#include "llvm/Transforms/Scalar/InferAddressSpaces.h"110#include "llvm/Transforms/Scalar/LICM.h"111#include "llvm/Transforms/Scalar/LoopDataPrefetch.h"112#include "llvm/Transforms/Scalar/LoopPassManager.h"113#include "llvm/Transforms/Scalar/NaryReassociate.h"114#include "llvm/Transforms/Scalar/SeparateConstOffsetFromGEP.h"115#include "llvm/Transforms/Scalar/Sink.h"116#include "llvm/Transforms/Scalar/StraightLineStrengthReduce.h"117#include "llvm/Transforms/Scalar/StructurizeCFG.h"118#include "llvm/Transforms/Utils.h"119#include "llvm/Transforms/Utils/FixIrreducible.h"120#include "llvm/Transforms/Utils/LCSSA.h"121#include "llvm/Transforms/Utils/LowerSwitch.h"122#include "llvm/Transforms/Utils/SimplifyLibCalls.h"123#include "llvm/Transforms/Utils/UnifyLoopExits.h"124#include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"125#include <optional>126 127using namespace llvm;128using namespace llvm::PatternMatch;129 130namespace {131//===----------------------------------------------------------------------===//132// AMDGPU CodeGen Pass Builder interface.133//===----------------------------------------------------------------------===//134 135class AMDGPUCodeGenPassBuilder136    : public CodeGenPassBuilder<AMDGPUCodeGenPassBuilder, GCNTargetMachine> {137  using Base = CodeGenPassBuilder<AMDGPUCodeGenPassBuilder, GCNTargetMachine>;138 139public:140  AMDGPUCodeGenPassBuilder(GCNTargetMachine &TM,141                           const CGPassBuilderOption &Opts,142                           PassInstrumentationCallbacks *PIC);143 144  void addIRPasses(AddIRPass &) const;145  void addCodeGenPrepare(AddIRPass &) const;146  void addPreISel(AddIRPass &addPass) const;147  void addILPOpts(AddMachinePass &) const;148  void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const;149  Error addInstSelector(AddMachinePass &) const;150  void addPreRewrite(AddMachinePass &) const;151  void addMachineSSAOptimization(AddMachinePass &) const;152  void addPostRegAlloc(AddMachinePass &) const;153  void addPreEmitPass(AddMachinePass &) const;154  void addPreEmitRegAlloc(AddMachinePass &) const;155  Error addRegAssignmentOptimized(AddMachinePass &) const;156  void addPreRegAlloc(AddMachinePass &) const;157  void addOptimizedRegAlloc(AddMachinePass &) const;158  void addPreSched2(AddMachinePass &) const;159 160  /// Check if a pass is enabled given \p Opt option. The option always161  /// overrides defaults if explicitly used. Otherwise its default will be used162  /// given that a pass shall work at an optimization \p Level minimum.163  bool isPassEnabled(const cl::opt<bool> &Opt,164                     CodeGenOptLevel Level = CodeGenOptLevel::Default) const;165  void addEarlyCSEOrGVNPass(AddIRPass &) const;166  void addStraightLineScalarOptimizationPasses(AddIRPass &) const;167};168 169class SGPRRegisterRegAlloc : public RegisterRegAllocBase<SGPRRegisterRegAlloc> {170public:171  SGPRRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)172    : RegisterRegAllocBase(N, D, C) {}173};174 175class VGPRRegisterRegAlloc : public RegisterRegAllocBase<VGPRRegisterRegAlloc> {176public:177  VGPRRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)178    : RegisterRegAllocBase(N, D, C) {}179};180 181class WWMRegisterRegAlloc : public RegisterRegAllocBase<WWMRegisterRegAlloc> {182public:183  WWMRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)184      : RegisterRegAllocBase(N, D, C) {}185};186 187static bool onlyAllocateSGPRs(const TargetRegisterInfo &TRI,188                              const MachineRegisterInfo &MRI,189                              const Register Reg) {190  const TargetRegisterClass *RC = MRI.getRegClass(Reg);191  return static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(RC);192}193 194static bool onlyAllocateVGPRs(const TargetRegisterInfo &TRI,195                              const MachineRegisterInfo &MRI,196                              const Register Reg) {197  const TargetRegisterClass *RC = MRI.getRegClass(Reg);198  return !static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(RC);199}200 201static bool onlyAllocateWWMRegs(const TargetRegisterInfo &TRI,202                                const MachineRegisterInfo &MRI,203                                const Register Reg) {204  const SIMachineFunctionInfo *MFI =205      MRI.getMF().getInfo<SIMachineFunctionInfo>();206  const TargetRegisterClass *RC = MRI.getRegClass(Reg);207  return !static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(RC) &&208         MFI->checkFlag(Reg, AMDGPU::VirtRegFlag::WWM_REG);209}210 211/// -{sgpr|wwm|vgpr}-regalloc=... command line option.212static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }213 214/// A dummy default pass factory indicates whether the register allocator is215/// overridden on the command line.216static llvm::once_flag InitializeDefaultSGPRRegisterAllocatorFlag;217static llvm::once_flag InitializeDefaultVGPRRegisterAllocatorFlag;218static llvm::once_flag InitializeDefaultWWMRegisterAllocatorFlag;219 220static SGPRRegisterRegAlloc221defaultSGPRRegAlloc("default",222                    "pick SGPR register allocator based on -O option",223                    useDefaultRegisterAllocator);224 225static cl::opt<SGPRRegisterRegAlloc::FunctionPassCtor, false,226               RegisterPassParser<SGPRRegisterRegAlloc>>227SGPRRegAlloc("sgpr-regalloc", cl::Hidden, cl::init(&useDefaultRegisterAllocator),228             cl::desc("Register allocator to use for SGPRs"));229 230static cl::opt<VGPRRegisterRegAlloc::FunctionPassCtor, false,231               RegisterPassParser<VGPRRegisterRegAlloc>>232VGPRRegAlloc("vgpr-regalloc", cl::Hidden, cl::init(&useDefaultRegisterAllocator),233             cl::desc("Register allocator to use for VGPRs"));234 235static cl::opt<WWMRegisterRegAlloc::FunctionPassCtor, false,236               RegisterPassParser<WWMRegisterRegAlloc>>237    WWMRegAlloc("wwm-regalloc", cl::Hidden,238                cl::init(&useDefaultRegisterAllocator),239                cl::desc("Register allocator to use for WWM registers"));240 241static void initializeDefaultSGPRRegisterAllocatorOnce() {242  RegisterRegAlloc::FunctionPassCtor Ctor = SGPRRegisterRegAlloc::getDefault();243 244  if (!Ctor) {245    Ctor = SGPRRegAlloc;246    SGPRRegisterRegAlloc::setDefault(SGPRRegAlloc);247  }248}249 250static void initializeDefaultVGPRRegisterAllocatorOnce() {251  RegisterRegAlloc::FunctionPassCtor Ctor = VGPRRegisterRegAlloc::getDefault();252 253  if (!Ctor) {254    Ctor = VGPRRegAlloc;255    VGPRRegisterRegAlloc::setDefault(VGPRRegAlloc);256  }257}258 259static void initializeDefaultWWMRegisterAllocatorOnce() {260  RegisterRegAlloc::FunctionPassCtor Ctor = WWMRegisterRegAlloc::getDefault();261 262  if (!Ctor) {263    Ctor = WWMRegAlloc;264    WWMRegisterRegAlloc::setDefault(WWMRegAlloc);265  }266}267 268static FunctionPass *createBasicSGPRRegisterAllocator() {269  return createBasicRegisterAllocator(onlyAllocateSGPRs);270}271 272static FunctionPass *createGreedySGPRRegisterAllocator() {273  return createGreedyRegisterAllocator(onlyAllocateSGPRs);274}275 276static FunctionPass *createFastSGPRRegisterAllocator() {277  return createFastRegisterAllocator(onlyAllocateSGPRs, false);278}279 280static FunctionPass *createBasicVGPRRegisterAllocator() {281  return createBasicRegisterAllocator(onlyAllocateVGPRs);282}283 284static FunctionPass *createGreedyVGPRRegisterAllocator() {285  return createGreedyRegisterAllocator(onlyAllocateVGPRs);286}287 288static FunctionPass *createFastVGPRRegisterAllocator() {289  return createFastRegisterAllocator(onlyAllocateVGPRs, true);290}291 292static FunctionPass *createBasicWWMRegisterAllocator() {293  return createBasicRegisterAllocator(onlyAllocateWWMRegs);294}295 296static FunctionPass *createGreedyWWMRegisterAllocator() {297  return createGreedyRegisterAllocator(onlyAllocateWWMRegs);298}299 300static FunctionPass *createFastWWMRegisterAllocator() {301  return createFastRegisterAllocator(onlyAllocateWWMRegs, false);302}303 304static SGPRRegisterRegAlloc basicRegAllocSGPR(305  "basic", "basic register allocator", createBasicSGPRRegisterAllocator);306static SGPRRegisterRegAlloc greedyRegAllocSGPR(307  "greedy", "greedy register allocator", createGreedySGPRRegisterAllocator);308 309static SGPRRegisterRegAlloc fastRegAllocSGPR(310  "fast", "fast register allocator", createFastSGPRRegisterAllocator);311 312 313static VGPRRegisterRegAlloc basicRegAllocVGPR(314  "basic", "basic register allocator", createBasicVGPRRegisterAllocator);315static VGPRRegisterRegAlloc greedyRegAllocVGPR(316  "greedy", "greedy register allocator", createGreedyVGPRRegisterAllocator);317 318static VGPRRegisterRegAlloc fastRegAllocVGPR(319  "fast", "fast register allocator", createFastVGPRRegisterAllocator);320static WWMRegisterRegAlloc basicRegAllocWWMReg("basic",321                                               "basic register allocator",322                                               createBasicWWMRegisterAllocator);323static WWMRegisterRegAlloc324    greedyRegAllocWWMReg("greedy", "greedy register allocator",325                         createGreedyWWMRegisterAllocator);326static WWMRegisterRegAlloc fastRegAllocWWMReg("fast", "fast register allocator",327                                              createFastWWMRegisterAllocator);328 329static bool isLTOPreLink(ThinOrFullLTOPhase Phase) {330  return Phase == ThinOrFullLTOPhase::FullLTOPreLink ||331         Phase == ThinOrFullLTOPhase::ThinLTOPreLink;332}333} // anonymous namespace334 335static cl::opt<bool>336EnableEarlyIfConversion("amdgpu-early-ifcvt", cl::Hidden,337                        cl::desc("Run early if-conversion"),338                        cl::init(false));339 340static cl::opt<bool>341OptExecMaskPreRA("amdgpu-opt-exec-mask-pre-ra", cl::Hidden,342            cl::desc("Run pre-RA exec mask optimizations"),343            cl::init(true));344 345static cl::opt<bool>346    LowerCtorDtor("amdgpu-lower-global-ctor-dtor",347                  cl::desc("Lower GPU ctor / dtors to globals on the device."),348                  cl::init(true), cl::Hidden);349 350// Option to disable vectorizer for tests.351static cl::opt<bool> EnableLoadStoreVectorizer(352  "amdgpu-load-store-vectorizer",353  cl::desc("Enable load store vectorizer"),354  cl::init(true),355  cl::Hidden);356 357// Option to control global loads scalarization358static cl::opt<bool> ScalarizeGlobal(359  "amdgpu-scalarize-global-loads",360  cl::desc("Enable global load scalarization"),361  cl::init(true),362  cl::Hidden);363 364// Option to run internalize pass.365static cl::opt<bool> InternalizeSymbols(366  "amdgpu-internalize-symbols",367  cl::desc("Enable elimination of non-kernel functions and unused globals"),368  cl::init(false),369  cl::Hidden);370 371// Option to inline all early.372static cl::opt<bool> EarlyInlineAll(373  "amdgpu-early-inline-all",374  cl::desc("Inline all functions early"),375  cl::init(false),376  cl::Hidden);377 378static cl::opt<bool> RemoveIncompatibleFunctions(379    "amdgpu-enable-remove-incompatible-functions", cl::Hidden,380    cl::desc("Enable removal of functions when they"381             "use features not supported by the target GPU"),382    cl::init(true));383 384static cl::opt<bool> EnableSDWAPeephole(385  "amdgpu-sdwa-peephole",386  cl::desc("Enable SDWA peepholer"),387  cl::init(true));388 389static cl::opt<bool> EnableDPPCombine(390  "amdgpu-dpp-combine",391  cl::desc("Enable DPP combiner"),392  cl::init(true));393 394// Enable address space based alias analysis395static cl::opt<bool> EnableAMDGPUAliasAnalysis("enable-amdgpu-aa", cl::Hidden,396  cl::desc("Enable AMDGPU Alias Analysis"),397  cl::init(true));398 399// Enable lib calls simplifications400static cl::opt<bool> EnableLibCallSimplify(401  "amdgpu-simplify-libcall",402  cl::desc("Enable amdgpu library simplifications"),403  cl::init(true),404  cl::Hidden);405 406static cl::opt<bool> EnableLowerKernelArguments(407  "amdgpu-ir-lower-kernel-arguments",408  cl::desc("Lower kernel argument loads in IR pass"),409  cl::init(true),410  cl::Hidden);411 412static cl::opt<bool> EnableRegReassign(413  "amdgpu-reassign-regs",414  cl::desc("Enable register reassign optimizations on gfx10+"),415  cl::init(true),416  cl::Hidden);417 418static cl::opt<bool> OptVGPRLiveRange(419    "amdgpu-opt-vgpr-liverange",420    cl::desc("Enable VGPR liverange optimizations for if-else structure"),421    cl::init(true), cl::Hidden);422 423static cl::opt<ScanOptions> AMDGPUAtomicOptimizerStrategy(424    "amdgpu-atomic-optimizer-strategy",425    cl::desc("Select DPP or Iterative strategy for scan"),426    cl::init(ScanOptions::Iterative),427    cl::values(428        clEnumValN(ScanOptions::DPP, "DPP", "Use DPP operations for scan"),429        clEnumValN(ScanOptions::Iterative, "Iterative",430                   "Use Iterative approach for scan"),431        clEnumValN(ScanOptions::None, "None", "Disable atomic optimizer")));432 433// Enable Mode register optimization434static cl::opt<bool> EnableSIModeRegisterPass(435  "amdgpu-mode-register",436  cl::desc("Enable mode register pass"),437  cl::init(true),438  cl::Hidden);439 440// Enable GFX11+ s_delay_alu insertion441static cl::opt<bool>442    EnableInsertDelayAlu("amdgpu-enable-delay-alu",443                         cl::desc("Enable s_delay_alu insertion"),444                         cl::init(true), cl::Hidden);445 446// Enable GFX11+ VOPD447static cl::opt<bool>448    EnableVOPD("amdgpu-enable-vopd",449               cl::desc("Enable VOPD, dual issue of VALU in wave32"),450               cl::init(true), cl::Hidden);451 452// Option is used in lit tests to prevent deadcoding of patterns inspected.453static cl::opt<bool>454EnableDCEInRA("amdgpu-dce-in-ra",455    cl::init(true), cl::Hidden,456    cl::desc("Enable machine DCE inside regalloc"));457 458static cl::opt<bool> EnableSetWavePriority("amdgpu-set-wave-priority",459                                           cl::desc("Adjust wave priority"),460                                           cl::init(false), cl::Hidden);461 462static cl::opt<bool> EnableScalarIRPasses(463  "amdgpu-scalar-ir-passes",464  cl::desc("Enable scalar IR passes"),465  cl::init(true),466  cl::Hidden);467 468static cl::opt<bool> EnableLowerExecSync(469    "amdgpu-enable-lower-exec-sync",470    cl::desc("Enable lowering of execution synchronization."), cl::init(true),471    cl::Hidden);472 473static cl::opt<bool>474    EnableSwLowerLDS("amdgpu-enable-sw-lower-lds",475                     cl::desc("Enable lowering of lds to global memory pass "476                              "and asan instrument resulting IR."),477                     cl::init(true), cl::Hidden);478 479static cl::opt<bool, true> EnableLowerModuleLDS(480    "amdgpu-enable-lower-module-lds", cl::desc("Enable lower module lds pass"),481    cl::location(AMDGPUTargetMachine::EnableLowerModuleLDS), cl::init(true),482    cl::Hidden);483 484static cl::opt<bool> EnablePreRAOptimizations(485    "amdgpu-enable-pre-ra-optimizations",486    cl::desc("Enable Pre-RA optimizations pass"), cl::init(true),487    cl::Hidden);488 489static cl::opt<bool> EnablePromoteKernelArguments(490    "amdgpu-enable-promote-kernel-arguments",491    cl::desc("Enable promotion of flat kernel pointer arguments to global"),492    cl::Hidden, cl::init(true));493 494static cl::opt<bool> EnableImageIntrinsicOptimizer(495    "amdgpu-enable-image-intrinsic-optimizer",496    cl::desc("Enable image intrinsic optimizer pass"), cl::init(true),497    cl::Hidden);498 499static cl::opt<bool>500    EnableLoopPrefetch("amdgpu-loop-prefetch",501                       cl::desc("Enable loop data prefetch on AMDGPU"),502                       cl::Hidden, cl::init(false));503 504static cl::opt<std::string>505    AMDGPUSchedStrategy("amdgpu-sched-strategy",506                        cl::desc("Select custom AMDGPU scheduling strategy."),507                        cl::Hidden, cl::init(""));508 509static cl::opt<bool> EnableRewritePartialRegUses(510    "amdgpu-enable-rewrite-partial-reg-uses",511    cl::desc("Enable rewrite partial reg uses pass"), cl::init(true),512    cl::Hidden);513 514static cl::opt<bool> EnableHipStdPar(515  "amdgpu-enable-hipstdpar",516  cl::desc("Enable HIP Standard Parallelism Offload support"), cl::init(false),517  cl::Hidden);518 519static cl::opt<bool>520    EnableAMDGPUAttributor("amdgpu-attributor-enable",521                           cl::desc("Enable AMDGPUAttributorPass"),522                           cl::init(true), cl::Hidden);523 524static cl::opt<bool> NewRegBankSelect(525    "new-reg-bank-select",526    cl::desc("Run amdgpu-regbankselect and amdgpu-regbanklegalize instead of "527             "regbankselect"),528    cl::init(false), cl::Hidden);529 530static cl::opt<bool> HasClosedWorldAssumption(531    "amdgpu-link-time-closed-world",532    cl::desc("Whether has closed-world assumption at link time"),533    cl::init(false), cl::Hidden);534 535static cl::opt<bool> EnableUniformIntrinsicCombine(536    "amdgpu-enable-uniform-intrinsic-combine",537    cl::desc("Enable/Disable the Uniform Intrinsic Combine Pass"),538    cl::init(true), cl::Hidden);539 540extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {541  // Register the target542  RegisterTargetMachine<R600TargetMachine> X(getTheR600Target());543  RegisterTargetMachine<GCNTargetMachine> Y(getTheGCNTarget());544 545  PassRegistry *PR = PassRegistry::getPassRegistry();546  initializeR600ClauseMergePassPass(*PR);547  initializeR600ControlFlowFinalizerPass(*PR);548  initializeR600PacketizerPass(*PR);549  initializeR600ExpandSpecialInstrsPassPass(*PR);550  initializeR600VectorRegMergerPass(*PR);551  initializeR600EmitClauseMarkersPass(*PR);552  initializeR600MachineCFGStructurizerPass(*PR);553  initializeGlobalISel(*PR);554  initializeAMDGPUAsmPrinterPass(*PR);555  initializeAMDGPUDAGToDAGISelLegacyPass(*PR);556  initializeAMDGPUPrepareAGPRAllocLegacyPass(*PR);557  initializeGCNDPPCombineLegacyPass(*PR);558  initializeSILowerI1CopiesLegacyPass(*PR);559  initializeAMDGPUGlobalISelDivergenceLoweringPass(*PR);560  initializeAMDGPURegBankSelectPass(*PR);561  initializeAMDGPURegBankLegalizePass(*PR);562  initializeSILowerWWMCopiesLegacyPass(*PR);563  initializeAMDGPUMarkLastScratchLoadLegacyPass(*PR);564  initializeSILowerSGPRSpillsLegacyPass(*PR);565  initializeSIFixSGPRCopiesLegacyPass(*PR);566  initializeSIFixVGPRCopiesLegacyPass(*PR);567  initializeSIFoldOperandsLegacyPass(*PR);568  initializeSIPeepholeSDWALegacyPass(*PR);569  initializeSIShrinkInstructionsLegacyPass(*PR);570  initializeSIOptimizeExecMaskingPreRALegacyPass(*PR);571  initializeSIOptimizeVGPRLiveRangeLegacyPass(*PR);572  initializeSILoadStoreOptimizerLegacyPass(*PR);573  initializeAMDGPUCtorDtorLoweringLegacyPass(*PR);574  initializeAMDGPUAlwaysInlinePass(*PR);575  initializeAMDGPULowerExecSyncLegacyPass(*PR);576  initializeAMDGPUSwLowerLDSLegacyPass(*PR);577  initializeAMDGPUAnnotateUniformValuesLegacyPass(*PR);578  initializeAMDGPUArgumentUsageInfoPass(*PR);579  initializeAMDGPUAtomicOptimizerPass(*PR);580  initializeAMDGPULowerKernelArgumentsPass(*PR);581  initializeAMDGPUPromoteKernelArgumentsPass(*PR);582  initializeAMDGPULowerKernelAttributesPass(*PR);583  initializeAMDGPUExportKernelRuntimeHandlesLegacyPass(*PR);584  initializeAMDGPUPostLegalizerCombinerPass(*PR);585  initializeAMDGPUPreLegalizerCombinerPass(*PR);586  initializeAMDGPURegBankCombinerPass(*PR);587  initializeAMDGPUPromoteAllocaPass(*PR);588  initializeAMDGPUCodeGenPreparePass(*PR);589  initializeAMDGPULateCodeGenPrepareLegacyPass(*PR);590  initializeAMDGPURemoveIncompatibleFunctionsLegacyPass(*PR);591  initializeAMDGPULowerModuleLDSLegacyPass(*PR);592  initializeAMDGPULowerBufferFatPointersPass(*PR);593  initializeAMDGPULowerIntrinsicsLegacyPass(*PR);594  initializeAMDGPUReserveWWMRegsLegacyPass(*PR);595  initializeAMDGPURewriteAGPRCopyMFMALegacyPass(*PR);596  initializeAMDGPURewriteOutArgumentsPass(*PR);597  initializeAMDGPURewriteUndefForPHILegacyPass(*PR);598  initializeSIAnnotateControlFlowLegacyPass(*PR);599  initializeAMDGPUInsertDelayAluLegacyPass(*PR);600  initializeAMDGPULowerVGPREncodingLegacyPass(*PR);601  initializeSIInsertHardClausesLegacyPass(*PR);602  initializeSIInsertWaitcntsLegacyPass(*PR);603  initializeSIModeRegisterLegacyPass(*PR);604  initializeSIWholeQuadModeLegacyPass(*PR);605  initializeSILowerControlFlowLegacyPass(*PR);606  initializeSIPreEmitPeepholeLegacyPass(*PR);607  initializeSILateBranchLoweringLegacyPass(*PR);608  initializeSIMemoryLegalizerLegacyPass(*PR);609  initializeSIOptimizeExecMaskingLegacyPass(*PR);610  initializeSIPreAllocateWWMRegsLegacyPass(*PR);611  initializeSIFormMemoryClausesLegacyPass(*PR);612  initializeSIPostRABundlerLegacyPass(*PR);613  initializeGCNCreateVOPDLegacyPass(*PR);614  initializeAMDGPUUnifyDivergentExitNodesPass(*PR);615  initializeAMDGPUAAWrapperPassPass(*PR);616  initializeAMDGPUExternalAAWrapperPass(*PR);617  initializeAMDGPUImageIntrinsicOptimizerPass(*PR);618  initializeAMDGPUPrintfRuntimeBindingPass(*PR);619  initializeAMDGPUResourceUsageAnalysisWrapperPassPass(*PR);620  initializeGCNNSAReassignLegacyPass(*PR);621  initializeGCNPreRAOptimizationsLegacyPass(*PR);622  initializeGCNPreRALongBranchRegLegacyPass(*PR);623  initializeGCNRewritePartialRegUsesLegacyPass(*PR);624  initializeGCNRegPressurePrinterPass(*PR);625  initializeAMDGPUPreloadKernArgPrologLegacyPass(*PR);626  initializeAMDGPUWaitSGPRHazardsLegacyPass(*PR);627  initializeAMDGPUPreloadKernelArgumentsLegacyPass(*PR);628  initializeAMDGPUUniformIntrinsicCombineLegacyPass(*PR);629}630 631static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {632  return std::make_unique<AMDGPUTargetObjectFile>();633}634 635static ScheduleDAGInstrs *createSIMachineScheduler(MachineSchedContext *C) {636  return new SIScheduleDAGMI(C);637}638 639static ScheduleDAGInstrs *640createGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {641  const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();642  ScheduleDAGMILive *DAG =643    new GCNScheduleDAGMILive(C, std::make_unique<GCNMaxOccupancySchedStrategy>(C));644  DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));645  if (ST.shouldClusterStores())646    DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));647  DAG->addMutation(createIGroupLPDAGMutation(AMDGPU::SchedulingPhase::Initial));648  DAG->addMutation(createAMDGPUMacroFusionDAGMutation());649  DAG->addMutation(createAMDGPUExportClusteringDAGMutation());650  DAG->addMutation(createAMDGPUBarrierLatencyDAGMutation(C->MF));651  return DAG;652}653 654static ScheduleDAGInstrs *655createGCNMaxILPMachineScheduler(MachineSchedContext *C) {656  ScheduleDAGMILive *DAG =657      new GCNScheduleDAGMILive(C, std::make_unique<GCNMaxILPSchedStrategy>(C));658  DAG->addMutation(createIGroupLPDAGMutation(AMDGPU::SchedulingPhase::Initial));659  return DAG;660}661 662static ScheduleDAGInstrs *663createGCNMaxMemoryClauseMachineScheduler(MachineSchedContext *C) {664  const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();665  ScheduleDAGMILive *DAG = new GCNScheduleDAGMILive(666      C, std::make_unique<GCNMaxMemoryClauseSchedStrategy>(C));667  DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));668  if (ST.shouldClusterStores())669    DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));670  DAG->addMutation(createAMDGPUExportClusteringDAGMutation());671  DAG->addMutation(createAMDGPUBarrierLatencyDAGMutation(C->MF));672  return DAG;673}674 675static ScheduleDAGInstrs *676createIterativeGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {677  const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();678  auto *DAG = new GCNIterativeScheduler(679      C, GCNIterativeScheduler::SCHEDULE_LEGACYMAXOCCUPANCY);680  DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));681  if (ST.shouldClusterStores())682    DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));683  DAG->addMutation(createIGroupLPDAGMutation(AMDGPU::SchedulingPhase::Initial));684  return DAG;685}686 687static ScheduleDAGInstrs *createMinRegScheduler(MachineSchedContext *C) {688  auto *DAG = new GCNIterativeScheduler(689      C, GCNIterativeScheduler::SCHEDULE_MINREGFORCED);690  DAG->addMutation(createIGroupLPDAGMutation(AMDGPU::SchedulingPhase::Initial));691  return DAG;692}693 694static ScheduleDAGInstrs *695createIterativeILPMachineScheduler(MachineSchedContext *C) {696  const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();697  auto *DAG = new GCNIterativeScheduler(C, GCNIterativeScheduler::SCHEDULE_ILP);698  DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));699  if (ST.shouldClusterStores())700    DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));701  DAG->addMutation(createAMDGPUMacroFusionDAGMutation());702  DAG->addMutation(createIGroupLPDAGMutation(AMDGPU::SchedulingPhase::Initial));703  return DAG;704}705 706static MachineSchedRegistry707SISchedRegistry("si", "Run SI's custom scheduler",708                createSIMachineScheduler);709 710static MachineSchedRegistry711GCNMaxOccupancySchedRegistry("gcn-max-occupancy",712                             "Run GCN scheduler to maximize occupancy",713                             createGCNMaxOccupancyMachineScheduler);714 715static MachineSchedRegistry716    GCNMaxILPSchedRegistry("gcn-max-ilp", "Run GCN scheduler to maximize ilp",717                           createGCNMaxILPMachineScheduler);718 719static MachineSchedRegistry GCNMaxMemoryClauseSchedRegistry(720    "gcn-max-memory-clause", "Run GCN scheduler to maximize memory clause",721    createGCNMaxMemoryClauseMachineScheduler);722 723static MachineSchedRegistry IterativeGCNMaxOccupancySchedRegistry(724    "gcn-iterative-max-occupancy-experimental",725    "Run GCN scheduler to maximize occupancy (experimental)",726    createIterativeGCNMaxOccupancyMachineScheduler);727 728static MachineSchedRegistry GCNMinRegSchedRegistry(729    "gcn-iterative-minreg",730    "Run GCN iterative scheduler for minimal register usage (experimental)",731    createMinRegScheduler);732 733static MachineSchedRegistry GCNILPSchedRegistry(734    "gcn-iterative-ilp",735    "Run GCN iterative scheduler for ILP scheduling (experimental)",736    createIterativeILPMachineScheduler);737 738LLVM_READNONE739static StringRef getGPUOrDefault(const Triple &TT, StringRef GPU) {740  if (!GPU.empty())741    return GPU;742 743  // Need to default to a target with flat support for HSA.744  if (TT.isAMDGCN())745    return TT.getOS() == Triple::AMDHSA ? "generic-hsa" : "generic";746 747  return "r600";748}749 750static Reloc::Model getEffectiveRelocModel() {751  // The AMDGPU toolchain only supports generating shared objects, so we752  // must always use PIC.753  return Reloc::PIC_;754}755 756AMDGPUTargetMachine::AMDGPUTargetMachine(const Target &T, const Triple &TT,757                                         StringRef CPU, StringRef FS,758                                         const TargetOptions &Options,759                                         std::optional<Reloc::Model> RM,760                                         std::optional<CodeModel::Model> CM,761                                         CodeGenOptLevel OptLevel)762    : CodeGenTargetMachineImpl(763          T, TT.computeDataLayout(), TT, getGPUOrDefault(TT, CPU), FS, Options,764          getEffectiveRelocModel(), getEffectiveCodeModel(CM, CodeModel::Small),765          OptLevel),766      TLOF(createTLOF(getTargetTriple())) {767  initAsmInfo();768  if (TT.isAMDGCN()) {769    if (getMCSubtargetInfo()->checkFeatures("+wavefrontsize64"))770      MRI.reset(llvm::createGCNMCRegisterInfo(AMDGPUDwarfFlavour::Wave64));771    else if (getMCSubtargetInfo()->checkFeatures("+wavefrontsize32"))772      MRI.reset(llvm::createGCNMCRegisterInfo(AMDGPUDwarfFlavour::Wave32));773  }774}775 776bool AMDGPUTargetMachine::EnableFunctionCalls = false;777bool AMDGPUTargetMachine::EnableLowerModuleLDS = true;778 779AMDGPUTargetMachine::~AMDGPUTargetMachine() = default;780 781StringRef AMDGPUTargetMachine::getGPUName(const Function &F) const {782  Attribute GPUAttr = F.getFnAttribute("target-cpu");783  return GPUAttr.isValid() ? GPUAttr.getValueAsString() : getTargetCPU();784}785 786StringRef AMDGPUTargetMachine::getFeatureString(const Function &F) const {787  Attribute FSAttr = F.getFnAttribute("target-features");788 789  return FSAttr.isValid() ? FSAttr.getValueAsString()790                          : getTargetFeatureString();791}792 793llvm::ScheduleDAGInstrs *794AMDGPUTargetMachine::createMachineScheduler(MachineSchedContext *C) const {795  const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();796  ScheduleDAGMILive *DAG = createSchedLive(C);797  DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));798  if (ST.shouldClusterStores())799    DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));800  return DAG;801}802 803/// Predicate for Internalize pass.804static bool mustPreserveGV(const GlobalValue &GV) {805  if (const Function *F = dyn_cast<Function>(&GV))806    return F->isDeclaration() || F->getName().starts_with("__asan_") ||807           F->getName().starts_with("__sanitizer_") ||808           AMDGPU::isEntryFunctionCC(F->getCallingConv());809 810  GV.removeDeadConstantUsers();811  return !GV.use_empty();812}813 814void AMDGPUTargetMachine::registerDefaultAliasAnalyses(AAManager &AAM) {815  AAM.registerFunctionAnalysis<AMDGPUAA>();816}817 818static Expected<ScanOptions>819parseAMDGPUAtomicOptimizerStrategy(StringRef Params) {820  if (Params.empty())821    return ScanOptions::Iterative;822  Params.consume_front("strategy=");823  auto Result = StringSwitch<std::optional<ScanOptions>>(Params)824                    .Case("dpp", ScanOptions::DPP)825                    .Cases({"iterative", ""}, ScanOptions::Iterative)826                    .Case("none", ScanOptions::None)827                    .Default(std::nullopt);828  if (Result)829    return *Result;830  return make_error<StringError>("invalid parameter", inconvertibleErrorCode());831}832 833Expected<AMDGPUAttributorOptions>834parseAMDGPUAttributorPassOptions(StringRef Params) {835  AMDGPUAttributorOptions Result;836  while (!Params.empty()) {837    StringRef ParamName;838    std::tie(ParamName, Params) = Params.split(';');839    if (ParamName == "closed-world") {840      Result.IsClosedWorld = true;841    } else {842      return make_error<StringError>(843          formatv("invalid AMDGPUAttributor pass parameter '{0}' ", ParamName)844              .str(),845          inconvertibleErrorCode());846    }847  }848  return Result;849}850 851void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {852 853#define GET_PASS_REGISTRY "AMDGPUPassRegistry.def"854#include "llvm/Passes/TargetPassRegistry.inc"855 856  PB.registerScalarOptimizerLateEPCallback(857      [](FunctionPassManager &FPM, OptimizationLevel Level) {858        if (Level == OptimizationLevel::O0)859          return;860 861        FPM.addPass(InferAddressSpacesPass());862      });863 864  PB.registerVectorizerEndEPCallback(865      [](FunctionPassManager &FPM, OptimizationLevel Level) {866        if (Level == OptimizationLevel::O0)867          return;868 869        FPM.addPass(InferAddressSpacesPass());870      });871 872  PB.registerPipelineEarlySimplificationEPCallback(873      [](ModulePassManager &PM, OptimizationLevel Level,874         ThinOrFullLTOPhase Phase) {875        if (!isLTOPreLink(Phase)) {876          // When we are not using -fgpu-rdc, we can run accelerator code877          // selection relatively early, but still after linking to prevent878          // eager removal of potentially reachable symbols.879          if (EnableHipStdPar) {880            PM.addPass(HipStdParMathFixupPass());881            PM.addPass(HipStdParAcceleratorCodeSelectionPass());882          }883          PM.addPass(AMDGPUPrintfRuntimeBindingPass());884        }885 886        if (Level == OptimizationLevel::O0)887          return;888 889        // We don't want to run internalization at per-module stage.890        if (InternalizeSymbols && !isLTOPreLink(Phase)) {891          PM.addPass(InternalizePass(mustPreserveGV));892          PM.addPass(GlobalDCEPass());893        }894 895        if (EarlyInlineAll && !EnableFunctionCalls)896          PM.addPass(AMDGPUAlwaysInlinePass());897      });898 899  PB.registerPeepholeEPCallback(900      [](FunctionPassManager &FPM, OptimizationLevel Level) {901        if (Level == OptimizationLevel::O0)902          return;903 904        FPM.addPass(AMDGPUUseNativeCallsPass());905        if (EnableLibCallSimplify)906          FPM.addPass(AMDGPUSimplifyLibCallsPass());907 908        if (EnableUniformIntrinsicCombine)909          FPM.addPass(AMDGPUUniformIntrinsicCombinePass());910      });911 912  PB.registerCGSCCOptimizerLateEPCallback(913      [this](CGSCCPassManager &PM, OptimizationLevel Level) {914        if (Level == OptimizationLevel::O0)915          return;916 917        FunctionPassManager FPM;918 919        // Add promote kernel arguments pass to the opt pipeline right before920        // infer address spaces which is needed to do actual address space921        // rewriting.922        if (Level.getSpeedupLevel() > OptimizationLevel::O1.getSpeedupLevel() &&923            EnablePromoteKernelArguments)924          FPM.addPass(AMDGPUPromoteKernelArgumentsPass());925 926        // Add infer address spaces pass to the opt pipeline after inlining927        // but before SROA to increase SROA opportunities.928        FPM.addPass(InferAddressSpacesPass());929 930        // This should run after inlining to have any chance of doing931        // anything, and before other cleanup optimizations.932        FPM.addPass(AMDGPULowerKernelAttributesPass());933 934        if (Level != OptimizationLevel::O0) {935          // Promote alloca to vector before SROA and loop unroll. If we936          // manage to eliminate allocas before unroll we may choose to unroll937          // less.938          FPM.addPass(AMDGPUPromoteAllocaToVectorPass(*this));939        }940 941        PM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));942      });943 944  // FIXME: Why is AMDGPUAttributor not in CGSCC?945  PB.registerOptimizerLastEPCallback([this](ModulePassManager &MPM,946                                            OptimizationLevel Level,947                                            ThinOrFullLTOPhase Phase) {948    if (Level != OptimizationLevel::O0) {949      if (!isLTOPreLink(Phase)) {950        if (EnableAMDGPUAttributor && getTargetTriple().isAMDGCN()) {951          AMDGPUAttributorOptions Opts;952          MPM.addPass(AMDGPUAttributorPass(*this, Opts, Phase));953        }954      }955    }956  });957 958  PB.registerFullLinkTimeOptimizationLastEPCallback(959      [this](ModulePassManager &PM, OptimizationLevel Level) {960        // When we are using -fgpu-rdc, we can only run accelerator code961        // selection after linking to prevent, otherwise we end up removing962        // potentially reachable symbols that were exported as external in other963        // modules.964        if (EnableHipStdPar) {965          PM.addPass(HipStdParMathFixupPass());966          PM.addPass(HipStdParAcceleratorCodeSelectionPass());967        }968        // We want to support the -lto-partitions=N option as "best effort".969        // For that, we need to lower LDS earlier in the pipeline before the970        // module is partitioned for codegen.971        if (EnableLowerExecSync)972          PM.addPass(AMDGPULowerExecSyncPass());973        if (EnableSwLowerLDS)974          PM.addPass(AMDGPUSwLowerLDSPass(*this));975        if (EnableLowerModuleLDS)976          PM.addPass(AMDGPULowerModuleLDSPass(*this));977        if (Level != OptimizationLevel::O0) {978          // We only want to run this with O2 or higher since inliner and SROA979          // don't run in O1.980          if (Level != OptimizationLevel::O1) {981            PM.addPass(982                createModuleToFunctionPassAdaptor(InferAddressSpacesPass()));983          }984          // Do we really need internalization in LTO?985          if (InternalizeSymbols) {986            PM.addPass(InternalizePass(mustPreserveGV));987            PM.addPass(GlobalDCEPass());988          }989          if (EnableAMDGPUAttributor && getTargetTriple().isAMDGCN()) {990            AMDGPUAttributorOptions Opt;991            if (HasClosedWorldAssumption)992              Opt.IsClosedWorld = true;993            PM.addPass(AMDGPUAttributorPass(994                *this, Opt, ThinOrFullLTOPhase::FullLTOPostLink));995          }996        }997        if (!NoKernelInfoEndLTO) {998          FunctionPassManager FPM;999          FPM.addPass(KernelInfoPrinter(this));1000          PM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));1001        }1002      });1003 1004  PB.registerRegClassFilterParsingCallback(1005      [](StringRef FilterName) -> RegAllocFilterFunc {1006        if (FilterName == "sgpr")1007          return onlyAllocateSGPRs;1008        if (FilterName == "vgpr")1009          return onlyAllocateVGPRs;1010        if (FilterName == "wwm")1011          return onlyAllocateWWMRegs;1012        return nullptr;1013      });1014}1015 1016int64_t AMDGPUTargetMachine::getNullPointerValue(unsigned AddrSpace) {1017  return (AddrSpace == AMDGPUAS::LOCAL_ADDRESS ||1018          AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ||1019          AddrSpace == AMDGPUAS::REGION_ADDRESS)1020             ? -11021             : 0;1022}1023 1024bool AMDGPUTargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,1025                                              unsigned DestAS) const {1026  return AMDGPU::isFlatGlobalAddrSpace(SrcAS) &&1027         AMDGPU::isFlatGlobalAddrSpace(DestAS);1028}1029 1030unsigned AMDGPUTargetMachine::getAssumedAddrSpace(const Value *V) const {1031  if (auto *Arg = dyn_cast<Argument>(V);1032      Arg &&1033      AMDGPU::isModuleEntryFunctionCC(Arg->getParent()->getCallingConv()) &&1034      !Arg->hasByRefAttr())1035    return AMDGPUAS::GLOBAL_ADDRESS;1036 1037  const auto *LD = dyn_cast<LoadInst>(V);1038  if (!LD) // TODO: Handle invariant load like constant.1039    return AMDGPUAS::UNKNOWN_ADDRESS_SPACE;1040 1041  // It must be a generic pointer loaded.1042  assert(V->getType()->getPointerAddressSpace() == AMDGPUAS::FLAT_ADDRESS);1043 1044  const auto *Ptr = LD->getPointerOperand();1045  if (Ptr->getType()->getPointerAddressSpace() != AMDGPUAS::CONSTANT_ADDRESS)1046    return AMDGPUAS::UNKNOWN_ADDRESS_SPACE;1047  // For a generic pointer loaded from the constant memory, it could be assumed1048  // as a global pointer since the constant memory is only populated on the1049  // host side. As implied by the offload programming model, only global1050  // pointers could be referenced on the host side.1051  return AMDGPUAS::GLOBAL_ADDRESS;1052}1053 1054std::pair<const Value *, unsigned>1055AMDGPUTargetMachine::getPredicatedAddrSpace(const Value *V) const {1056  if (auto *II = dyn_cast<IntrinsicInst>(V)) {1057    switch (II->getIntrinsicID()) {1058    case Intrinsic::amdgcn_is_shared:1059      return std::pair(II->getArgOperand(0), AMDGPUAS::LOCAL_ADDRESS);1060    case Intrinsic::amdgcn_is_private:1061      return std::pair(II->getArgOperand(0), AMDGPUAS::PRIVATE_ADDRESS);1062    default:1063      break;1064    }1065    return std::pair(nullptr, -1);1066  }1067  // Check the global pointer predication based on1068  // (!is_share(p) && !is_private(p)). Note that logic 'and' is commutative and1069  // the order of 'is_shared' and 'is_private' is not significant.1070  Value *Ptr;1071  if (match(1072          const_cast<Value *>(V),1073          m_c_And(m_Not(m_Intrinsic<Intrinsic::amdgcn_is_shared>(m_Value(Ptr))),1074                  m_Not(m_Intrinsic<Intrinsic::amdgcn_is_private>(1075                      m_Deferred(Ptr))))))1076    return std::pair(Ptr, AMDGPUAS::GLOBAL_ADDRESS);1077 1078  return std::pair(nullptr, -1);1079}1080 1081unsigned1082AMDGPUTargetMachine::getAddressSpaceForPseudoSourceKind(unsigned Kind) const {1083  switch (Kind) {1084  case PseudoSourceValue::Stack:1085  case PseudoSourceValue::FixedStack:1086    return AMDGPUAS::PRIVATE_ADDRESS;1087  case PseudoSourceValue::ConstantPool:1088  case PseudoSourceValue::GOT:1089  case PseudoSourceValue::JumpTable:1090  case PseudoSourceValue::GlobalValueCallEntry:1091  case PseudoSourceValue::ExternalSymbolCallEntry:1092    return AMDGPUAS::CONSTANT_ADDRESS;1093  }1094  return AMDGPUAS::FLAT_ADDRESS;1095}1096 1097bool AMDGPUTargetMachine::splitModule(1098    Module &M, unsigned NumParts,1099    function_ref<void(std::unique_ptr<Module> MPart)> ModuleCallback) {1100  // FIXME(?): Would be better to use an already existing Analysis/PassManager,1101  // but all current users of this API don't have one ready and would need to1102  // create one anyway. Let's hide the boilerplate for now to keep it simple.1103 1104  LoopAnalysisManager LAM;1105  FunctionAnalysisManager FAM;1106  CGSCCAnalysisManager CGAM;1107  ModuleAnalysisManager MAM;1108 1109  PassBuilder PB(this);1110  PB.registerModuleAnalyses(MAM);1111  PB.registerFunctionAnalyses(FAM);1112  PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);1113 1114  ModulePassManager MPM;1115  MPM.addPass(AMDGPUSplitModulePass(NumParts, ModuleCallback));1116  MPM.run(M, MAM);1117  return true;1118}1119 1120//===----------------------------------------------------------------------===//1121// GCN Target Machine (SI+)1122//===----------------------------------------------------------------------===//1123 1124GCNTargetMachine::GCNTargetMachine(const Target &T, const Triple &TT,1125                                   StringRef CPU, StringRef FS,1126                                   const TargetOptions &Options,1127                                   std::optional<Reloc::Model> RM,1128                                   std::optional<CodeModel::Model> CM,1129                                   CodeGenOptLevel OL, bool JIT)1130    : AMDGPUTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL) {}1131 1132const TargetSubtargetInfo *1133GCNTargetMachine::getSubtargetImpl(const Function &F) const {1134  StringRef GPU = getGPUName(F);1135  StringRef FS = getFeatureString(F);1136 1137  SmallString<128> SubtargetKey(GPU);1138  SubtargetKey.append(FS);1139 1140  auto &I = SubtargetMap[SubtargetKey];1141  if (!I) {1142    // This needs to be done before we create a new subtarget since any1143    // creation will depend on the TM and the code generation flags on the1144    // function that reside in TargetOptions.1145    resetTargetOptions(F);1146    I = std::make_unique<GCNSubtarget>(TargetTriple, GPU, FS, *this);1147  }1148 1149  I->setScalarizeGlobalBehavior(ScalarizeGlobal);1150 1151  return I.get();1152}1153 1154TargetTransformInfo1155GCNTargetMachine::getTargetTransformInfo(const Function &F) const {1156  return TargetTransformInfo(std::make_unique<GCNTTIImpl>(this, F));1157}1158 1159Error GCNTargetMachine::buildCodeGenPipeline(1160    ModulePassManager &MPM, raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut,1161    CodeGenFileType FileType, const CGPassBuilderOption &Opts,1162    PassInstrumentationCallbacks *PIC) {1163  AMDGPUCodeGenPassBuilder CGPB(*this, Opts, PIC);1164  return CGPB.buildPipeline(MPM, Out, DwoOut, FileType);1165}1166 1167ScheduleDAGInstrs *1168GCNTargetMachine::createMachineScheduler(MachineSchedContext *C) const {1169  const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();1170  if (ST.enableSIScheduler())1171    return createSIMachineScheduler(C);1172 1173  Attribute SchedStrategyAttr =1174      C->MF->getFunction().getFnAttribute("amdgpu-sched-strategy");1175  StringRef SchedStrategy = SchedStrategyAttr.isValid()1176                                ? SchedStrategyAttr.getValueAsString()1177                                : AMDGPUSchedStrategy;1178 1179  if (SchedStrategy == "max-ilp")1180    return createGCNMaxILPMachineScheduler(C);1181 1182  if (SchedStrategy == "max-memory-clause")1183    return createGCNMaxMemoryClauseMachineScheduler(C);1184 1185  if (SchedStrategy == "iterative-ilp")1186    return createIterativeILPMachineScheduler(C);1187 1188  if (SchedStrategy == "iterative-minreg")1189    return createMinRegScheduler(C);1190 1191  if (SchedStrategy == "iterative-maxocc")1192    return createIterativeGCNMaxOccupancyMachineScheduler(C);1193 1194  return createGCNMaxOccupancyMachineScheduler(C);1195}1196 1197ScheduleDAGInstrs *1198GCNTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {1199  ScheduleDAGMI *DAG =1200      new GCNPostScheduleDAGMILive(C, std::make_unique<PostGenericScheduler>(C),1201                                   /*RemoveKillFlags=*/true);1202  const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();1203  DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));1204  if (ST.shouldClusterStores())1205    DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));1206  DAG->addMutation(createIGroupLPDAGMutation(AMDGPU::SchedulingPhase::PostRA));1207  if ((EnableVOPD.getNumOccurrences() ||1208       getOptLevel() >= CodeGenOptLevel::Less) &&1209      EnableVOPD)1210    DAG->addMutation(createVOPDPairingMutation());1211  DAG->addMutation(createAMDGPUExportClusteringDAGMutation());1212  DAG->addMutation(createAMDGPUBarrierLatencyDAGMutation(C->MF));1213  return DAG;1214}1215//===----------------------------------------------------------------------===//1216// AMDGPU Legacy Pass Setup1217//===----------------------------------------------------------------------===//1218 1219std::unique_ptr<CSEConfigBase> llvm::AMDGPUPassConfig::getCSEConfig() const {1220  return getStandardCSEConfigForOpt(TM->getOptLevel());1221}1222 1223namespace {1224 1225class GCNPassConfig final : public AMDGPUPassConfig {1226public:1227  GCNPassConfig(TargetMachine &TM, PassManagerBase &PM)1228      : AMDGPUPassConfig(TM, PM) {1229    substitutePass(&PostRASchedulerID, &PostMachineSchedulerID);1230  }1231 1232  GCNTargetMachine &getGCNTargetMachine() const {1233    return getTM<GCNTargetMachine>();1234  }1235 1236  bool addPreISel() override;1237  void addMachineSSAOptimization() override;1238  bool addILPOpts() override;1239  bool addInstSelector() override;1240  bool addIRTranslator() override;1241  void addPreLegalizeMachineIR() override;1242  bool addLegalizeMachineIR() override;1243  void addPreRegBankSelect() override;1244  bool addRegBankSelect() override;1245  void addPreGlobalInstructionSelect() override;1246  bool addGlobalInstructionSelect() override;1247  void addPreRegAlloc() override;1248  void addFastRegAlloc() override;1249  void addOptimizedRegAlloc() override;1250 1251  FunctionPass *createSGPRAllocPass(bool Optimized);1252  FunctionPass *createVGPRAllocPass(bool Optimized);1253  FunctionPass *createWWMRegAllocPass(bool Optimized);1254  FunctionPass *createRegAllocPass(bool Optimized) override;1255 1256  bool addRegAssignAndRewriteFast() override;1257  bool addRegAssignAndRewriteOptimized() override;1258 1259  bool addPreRewrite() override;1260  void addPostRegAlloc() override;1261  void addPreSched2() override;1262  void addPreEmitPass() override;1263  void addPostBBSections() override;1264};1265 1266} // end anonymous namespace1267 1268AMDGPUPassConfig::AMDGPUPassConfig(TargetMachine &TM, PassManagerBase &PM)1269    : TargetPassConfig(TM, PM) {1270  // Exceptions and StackMaps are not supported, so these passes will never do1271  // anything.1272  disablePass(&StackMapLivenessID);1273  disablePass(&FuncletLayoutID);1274  // Garbage collection is not supported.1275  disablePass(&GCLoweringID);1276  disablePass(&ShadowStackGCLoweringID);1277}1278 1279void AMDGPUPassConfig::addEarlyCSEOrGVNPass() {1280  if (getOptLevel() == CodeGenOptLevel::Aggressive)1281    addPass(createGVNPass());1282  else1283    addPass(createEarlyCSEPass());1284}1285 1286void AMDGPUPassConfig::addStraightLineScalarOptimizationPasses() {1287  if (isPassEnabled(EnableLoopPrefetch, CodeGenOptLevel::Aggressive))1288    addPass(createLoopDataPrefetchPass());1289  addPass(createSeparateConstOffsetFromGEPPass());1290  // ReassociateGEPs exposes more opportunities for SLSR. See1291  // the example in reassociate-geps-and-slsr.ll.1292  addPass(createStraightLineStrengthReducePass());1293  // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN or1294  // EarlyCSE can reuse.1295  addEarlyCSEOrGVNPass();1296  // Run NaryReassociate after EarlyCSE/GVN to be more effective.1297  addPass(createNaryReassociatePass());1298  // NaryReassociate on GEPs creates redundant common expressions, so run1299  // EarlyCSE after it.1300  addPass(createEarlyCSEPass());1301}1302 1303void AMDGPUPassConfig::addIRPasses() {1304  const AMDGPUTargetMachine &TM = getAMDGPUTargetMachine();1305 1306  if (RemoveIncompatibleFunctions && TM.getTargetTriple().isAMDGCN())1307    addPass(createAMDGPURemoveIncompatibleFunctionsPass(&TM));1308 1309  // There is no reason to run these.1310  disablePass(&StackMapLivenessID);1311  disablePass(&FuncletLayoutID);1312  disablePass(&PatchableFunctionID);1313 1314  addPass(createAMDGPUPrintfRuntimeBinding());1315  if (LowerCtorDtor)1316    addPass(createAMDGPUCtorDtorLoweringLegacyPass());1317 1318  if (TM.getTargetTriple().isAMDGCN() &&1319      isPassEnabled(EnableImageIntrinsicOptimizer))1320    addPass(createAMDGPUImageIntrinsicOptimizerPass(&TM));1321 1322  if (EnableUniformIntrinsicCombine)1323    addPass(createAMDGPUUniformIntrinsicCombineLegacyPass());1324 1325  // This can be disabled by passing ::Disable here or on the command line1326  // with --expand-variadics-override=disable.1327  addPass(createExpandVariadicsPass(ExpandVariadicsMode::Lowering));1328 1329  // Function calls are not supported, so make sure we inline everything.1330  addPass(createAMDGPUAlwaysInlinePass());1331  addPass(createAlwaysInlinerLegacyPass());1332 1333  // Handle uses of OpenCL image2d_t, image3d_t and sampler_t arguments.1334  if (TM.getTargetTriple().getArch() == Triple::r600)1335    addPass(createR600OpenCLImageTypeLoweringPass());1336 1337  // Make enqueued block runtime handles externally visible.1338  addPass(createAMDGPUExportKernelRuntimeHandlesLegacyPass());1339 1340  // Lower special LDS accesses.1341  if (EnableLowerExecSync)1342    addPass(createAMDGPULowerExecSyncLegacyPass());1343 1344  // Lower LDS accesses to global memory pass if address sanitizer is enabled.1345  if (EnableSwLowerLDS)1346    addPass(createAMDGPUSwLowerLDSLegacyPass(&TM));1347 1348  // Runs before PromoteAlloca so the latter can account for function uses1349  if (EnableLowerModuleLDS) {1350    addPass(createAMDGPULowerModuleLDSLegacyPass(&TM));1351  }1352 1353  // Run atomic optimizer before Atomic Expand1354  if ((TM.getTargetTriple().isAMDGCN()) &&1355      (TM.getOptLevel() >= CodeGenOptLevel::Less) &&1356      (AMDGPUAtomicOptimizerStrategy != ScanOptions::None)) {1357    addPass(createAMDGPUAtomicOptimizerPass(AMDGPUAtomicOptimizerStrategy));1358  }1359 1360  addPass(createAtomicExpandLegacyPass());1361 1362  if (TM.getOptLevel() > CodeGenOptLevel::None) {1363    addPass(createAMDGPUPromoteAlloca());1364 1365    if (isPassEnabled(EnableScalarIRPasses))1366      addStraightLineScalarOptimizationPasses();1367 1368    if (EnableAMDGPUAliasAnalysis) {1369      addPass(createAMDGPUAAWrapperPass());1370      addPass(createExternalAAWrapperPass([](Pass &P, Function &,1371                                             AAResults &AAR) {1372        if (auto *WrapperPass = P.getAnalysisIfAvailable<AMDGPUAAWrapperPass>())1373          AAR.addAAResult(WrapperPass->getResult());1374        }));1375    }1376 1377    if (TM.getTargetTriple().isAMDGCN()) {1378      // TODO: May want to move later or split into an early and late one.1379      addPass(createAMDGPUCodeGenPreparePass());1380    }1381 1382    // Try to hoist loop invariant parts of divisions AMDGPUCodeGenPrepare may1383    // have expanded.1384    if (TM.getOptLevel() > CodeGenOptLevel::Less)1385      addPass(createLICMPass());1386  }1387 1388  TargetPassConfig::addIRPasses();1389 1390  // EarlyCSE is not always strong enough to clean up what LSR produces. For1391  // example, GVN can combine1392  //1393  //   %0 = add %a, %b1394  //   %1 = add %b, %a1395  //1396  // and1397  //1398  //   %0 = shl nsw %a, 21399  //   %1 = shl %a, 21400  //1401  // but EarlyCSE can do neither of them.1402  if (isPassEnabled(EnableScalarIRPasses))1403    addEarlyCSEOrGVNPass();1404}1405 1406void AMDGPUPassConfig::addCodeGenPrepare() {1407  if (TM->getTargetTriple().isAMDGCN() &&1408      TM->getOptLevel() > CodeGenOptLevel::None)1409    addPass(createAMDGPUPreloadKernelArgumentsLegacyPass(TM));1410 1411  if (TM->getTargetTriple().isAMDGCN() && EnableLowerKernelArguments)1412    addPass(createAMDGPULowerKernelArgumentsPass());1413 1414  TargetPassConfig::addCodeGenPrepare();1415 1416  if (isPassEnabled(EnableLoadStoreVectorizer))1417    addPass(createLoadStoreVectorizerPass());1418 1419  if (TM->getTargetTriple().isAMDGCN()) {1420    // This lowering has been placed after codegenprepare to take advantage of1421    // address mode matching (which is why it isn't put with the LDS lowerings).1422    // It could be placed anywhere before uniformity annotations (an analysis1423    // that it changes by splitting up fat pointers into their components)1424    // but has been put before switch lowering and CFG flattening so that those1425    // passes can run on the more optimized control flow this pass creates in1426    // many cases.1427    addPass(createAMDGPULowerBufferFatPointersPass());1428    addPass(createAMDGPULowerIntrinsicsLegacyPass());1429  }1430 1431  // LowerSwitch pass may introduce unreachable blocks that can1432  // cause unexpected behavior for subsequent passes. Placing it1433  // here seems better that these blocks would get cleaned up by1434  // UnreachableBlockElim inserted next in the pass flow.1435  addPass(createLowerSwitchPass());1436}1437 1438bool AMDGPUPassConfig::addPreISel() {1439  if (TM->getOptLevel() > CodeGenOptLevel::None)1440    addPass(createFlattenCFGPass());1441  return false;1442}1443 1444bool AMDGPUPassConfig::addInstSelector() {1445  addPass(createAMDGPUISelDag(getAMDGPUTargetMachine(), getOptLevel()));1446  return false;1447}1448 1449bool AMDGPUPassConfig::addGCPasses() {1450  // Do nothing. GC is not supported.1451  return false;1452}1453 1454//===----------------------------------------------------------------------===//1455// GCN Legacy Pass Setup1456//===----------------------------------------------------------------------===//1457 1458bool GCNPassConfig::addPreISel() {1459  AMDGPUPassConfig::addPreISel();1460 1461  if (TM->getOptLevel() > CodeGenOptLevel::None)1462    addPass(createSinkingPass());1463 1464  if (TM->getOptLevel() > CodeGenOptLevel::None)1465    addPass(createAMDGPULateCodeGenPrepareLegacyPass());1466 1467  // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit1468  // regions formed by them.1469  addPass(&AMDGPUUnifyDivergentExitNodesID);1470  addPass(createFixIrreduciblePass());1471  addPass(createUnifyLoopExitsPass());1472  addPass(createStructurizeCFGPass(false)); // true -> SkipUniformRegions1473 1474  addPass(createAMDGPUAnnotateUniformValuesLegacy());1475  addPass(createSIAnnotateControlFlowLegacyPass());1476  // TODO: Move this right after structurizeCFG to avoid extra divergence1477  // analysis. This depends on stopping SIAnnotateControlFlow from making1478  // control flow modifications.1479  addPass(createAMDGPURewriteUndefForPHILegacyPass());1480 1481  // SDAG requires LCSSA, GlobalISel does not. Disable LCSSA for -global-isel1482  // with -new-reg-bank-select and without any of the fallback options.1483  if (!getCGPassBuilderOption().EnableGlobalISelOption ||1484      !isGlobalISelAbortEnabled() || !NewRegBankSelect)1485    addPass(createLCSSAPass());1486 1487  if (TM->getOptLevel() > CodeGenOptLevel::Less)1488    addPass(&AMDGPUPerfHintAnalysisLegacyID);1489 1490  return false;1491}1492 1493void GCNPassConfig::addMachineSSAOptimization() {1494  TargetPassConfig::addMachineSSAOptimization();1495 1496  // We want to fold operands after PeepholeOptimizer has run (or as part of1497  // it), because it will eliminate extra copies making it easier to fold the1498  // real source operand. We want to eliminate dead instructions after, so that1499  // we see fewer uses of the copies. We then need to clean up the dead1500  // instructions leftover after the operands are folded as well.1501  //1502  // XXX - Can we get away without running DeadMachineInstructionElim again?1503  addPass(&SIFoldOperandsLegacyID);1504  if (EnableDPPCombine)1505    addPass(&GCNDPPCombineLegacyID);1506  addPass(&SILoadStoreOptimizerLegacyID);1507  if (isPassEnabled(EnableSDWAPeephole)) {1508    addPass(&SIPeepholeSDWALegacyID);1509    addPass(&EarlyMachineLICMID);1510    addPass(&MachineCSELegacyID);1511    addPass(&SIFoldOperandsLegacyID);1512  }1513  addPass(&DeadMachineInstructionElimID);1514  addPass(createSIShrinkInstructionsLegacyPass());1515}1516 1517bool GCNPassConfig::addILPOpts() {1518  if (EnableEarlyIfConversion)1519    addPass(&EarlyIfConverterLegacyID);1520 1521  TargetPassConfig::addILPOpts();1522  return false;1523}1524 1525bool GCNPassConfig::addInstSelector() {1526  AMDGPUPassConfig::addInstSelector();1527  addPass(&SIFixSGPRCopiesLegacyID);1528  addPass(createSILowerI1CopiesLegacyPass());1529  return false;1530}1531 1532bool GCNPassConfig::addIRTranslator() {1533  addPass(new IRTranslator(getOptLevel()));1534  return false;1535}1536 1537void GCNPassConfig::addPreLegalizeMachineIR() {1538  bool IsOptNone = getOptLevel() == CodeGenOptLevel::None;1539  addPass(createAMDGPUPreLegalizeCombiner(IsOptNone));1540  addPass(new Localizer());1541}1542 1543bool GCNPassConfig::addLegalizeMachineIR() {1544  addPass(new Legalizer());1545  return false;1546}1547 1548void GCNPassConfig::addPreRegBankSelect() {1549  bool IsOptNone = getOptLevel() == CodeGenOptLevel::None;1550  addPass(createAMDGPUPostLegalizeCombiner(IsOptNone));1551  addPass(createAMDGPUGlobalISelDivergenceLoweringPass());1552}1553 1554bool GCNPassConfig::addRegBankSelect() {1555  if (NewRegBankSelect) {1556    addPass(createAMDGPURegBankSelectPass());1557    addPass(createAMDGPURegBankLegalizePass());1558  } else {1559    addPass(new RegBankSelect());1560  }1561  return false;1562}1563 1564void GCNPassConfig::addPreGlobalInstructionSelect() {1565  bool IsOptNone = getOptLevel() == CodeGenOptLevel::None;1566  addPass(createAMDGPURegBankCombiner(IsOptNone));1567}1568 1569bool GCNPassConfig::addGlobalInstructionSelect() {1570  addPass(new InstructionSelect(getOptLevel()));1571  return false;1572}1573 1574void GCNPassConfig::addFastRegAlloc() {1575  // FIXME: We have to disable the verifier here because of PHIElimination +1576  // TwoAddressInstructions disabling it.1577 1578  // This must be run immediately after phi elimination and before1579  // TwoAddressInstructions, otherwise the processing of the tied operand of1580  // SI_ELSE will introduce a copy of the tied operand source after the else.1581  insertPass(&PHIEliminationID, &SILowerControlFlowLegacyID);1582 1583  insertPass(&TwoAddressInstructionPassID, &SIWholeQuadModeID);1584 1585  TargetPassConfig::addFastRegAlloc();1586}1587 1588void GCNPassConfig::addPreRegAlloc() {1589  if (getOptLevel() != CodeGenOptLevel::None)1590    addPass(&AMDGPUPrepareAGPRAllocLegacyID);1591}1592 1593void GCNPassConfig::addOptimizedRegAlloc() {1594  if (EnableDCEInRA)1595    insertPass(&DetectDeadLanesID, &DeadMachineInstructionElimID);1596 1597  // FIXME: when an instruction has a Killed operand, and the instruction is1598  // inside a bundle, seems only the BUNDLE instruction appears as the Kills of1599  // the register in LiveVariables, this would trigger a failure in verifier,1600  // we should fix it and enable the verifier.1601  if (OptVGPRLiveRange)1602    insertPass(&LiveVariablesID, &SIOptimizeVGPRLiveRangeLegacyID);1603 1604  // This must be run immediately after phi elimination and before1605  // TwoAddressInstructions, otherwise the processing of the tied operand of1606  // SI_ELSE will introduce a copy of the tied operand source after the else.1607  insertPass(&PHIEliminationID, &SILowerControlFlowLegacyID);1608 1609  if (EnableRewritePartialRegUses)1610    insertPass(&RenameIndependentSubregsID, &GCNRewritePartialRegUsesID);1611 1612  if (isPassEnabled(EnablePreRAOptimizations))1613    insertPass(&MachineSchedulerID, &GCNPreRAOptimizationsID);1614 1615  // Allow the scheduler to run before SIWholeQuadMode inserts exec manipulation1616  // instructions that cause scheduling barriers.1617  insertPass(&MachineSchedulerID, &SIWholeQuadModeID);1618 1619  if (OptExecMaskPreRA)1620    insertPass(&MachineSchedulerID, &SIOptimizeExecMaskingPreRAID);1621 1622  // This is not an essential optimization and it has a noticeable impact on1623  // compilation time, so we only enable it from O2.1624  if (TM->getOptLevel() > CodeGenOptLevel::Less)1625    insertPass(&MachineSchedulerID, &SIFormMemoryClausesID);1626 1627  TargetPassConfig::addOptimizedRegAlloc();1628}1629 1630bool GCNPassConfig::addPreRewrite() {1631  if (EnableRegReassign)1632    addPass(&GCNNSAReassignID);1633 1634  addPass(&AMDGPURewriteAGPRCopyMFMALegacyID);1635  return true;1636}1637 1638FunctionPass *GCNPassConfig::createSGPRAllocPass(bool Optimized) {1639  // Initialize the global default.1640  llvm::call_once(InitializeDefaultSGPRRegisterAllocatorFlag,1641                  initializeDefaultSGPRRegisterAllocatorOnce);1642 1643  RegisterRegAlloc::FunctionPassCtor Ctor = SGPRRegisterRegAlloc::getDefault();1644  if (Ctor != useDefaultRegisterAllocator)1645    return Ctor();1646 1647  if (Optimized)1648    return createGreedyRegisterAllocator(onlyAllocateSGPRs);1649 1650  return createFastRegisterAllocator(onlyAllocateSGPRs, false);1651}1652 1653FunctionPass *GCNPassConfig::createVGPRAllocPass(bool Optimized) {1654  // Initialize the global default.1655  llvm::call_once(InitializeDefaultVGPRRegisterAllocatorFlag,1656                  initializeDefaultVGPRRegisterAllocatorOnce);1657 1658  RegisterRegAlloc::FunctionPassCtor Ctor = VGPRRegisterRegAlloc::getDefault();1659  if (Ctor != useDefaultRegisterAllocator)1660    return Ctor();1661 1662  if (Optimized)1663    return createGreedyVGPRRegisterAllocator();1664 1665  return createFastVGPRRegisterAllocator();1666}1667 1668FunctionPass *GCNPassConfig::createWWMRegAllocPass(bool Optimized) {1669  // Initialize the global default.1670  llvm::call_once(InitializeDefaultWWMRegisterAllocatorFlag,1671                  initializeDefaultWWMRegisterAllocatorOnce);1672 1673  RegisterRegAlloc::FunctionPassCtor Ctor = WWMRegisterRegAlloc::getDefault();1674  if (Ctor != useDefaultRegisterAllocator)1675    return Ctor();1676 1677  if (Optimized)1678    return createGreedyWWMRegisterAllocator();1679 1680  return createFastWWMRegisterAllocator();1681}1682 1683FunctionPass *GCNPassConfig::createRegAllocPass(bool Optimized) {1684  llvm_unreachable("should not be used");1685}1686 1687static const char RegAllocOptNotSupportedMessage[] =1688    "-regalloc not supported with amdgcn. Use -sgpr-regalloc, -wwm-regalloc, "1689    "and -vgpr-regalloc";1690 1691bool GCNPassConfig::addRegAssignAndRewriteFast() {1692  if (!usingDefaultRegAlloc())1693    reportFatalUsageError(RegAllocOptNotSupportedMessage);1694 1695  addPass(&GCNPreRALongBranchRegID);1696 1697  addPass(createSGPRAllocPass(false));1698 1699  // Equivalent of PEI for SGPRs.1700  addPass(&SILowerSGPRSpillsLegacyID);1701 1702  // To Allocate wwm registers used in whole quad mode operations (for shaders).1703  addPass(&SIPreAllocateWWMRegsLegacyID);1704 1705  // For allocating other wwm register operands.1706  addPass(createWWMRegAllocPass(false));1707 1708  addPass(&SILowerWWMCopiesLegacyID);1709  addPass(&AMDGPUReserveWWMRegsLegacyID);1710 1711  // For allocating per-thread VGPRs.1712  addPass(createVGPRAllocPass(false));1713 1714  return true;1715}1716 1717bool GCNPassConfig::addRegAssignAndRewriteOptimized() {1718  if (!usingDefaultRegAlloc())1719    reportFatalUsageError(RegAllocOptNotSupportedMessage);1720 1721  addPass(&GCNPreRALongBranchRegID);1722 1723  addPass(createSGPRAllocPass(true));1724 1725  // Commit allocated register changes. This is mostly necessary because too1726  // many things rely on the use lists of the physical registers, such as the1727  // verifier. This is only necessary with allocators which use LiveIntervals,1728  // since FastRegAlloc does the replacements itself.1729  addPass(createVirtRegRewriter(false));1730 1731  // At this point, the sgpr-regalloc has been done and it is good to have the1732  // stack slot coloring to try to optimize the SGPR spill stack indices before1733  // attempting the custom SGPR spill lowering.1734  addPass(&StackSlotColoringID);1735 1736  // Equivalent of PEI for SGPRs.1737  addPass(&SILowerSGPRSpillsLegacyID);1738 1739  // To Allocate wwm registers used in whole quad mode operations (for shaders).1740  addPass(&SIPreAllocateWWMRegsLegacyID);1741 1742  // For allocating other whole wave mode registers.1743  addPass(createWWMRegAllocPass(true));1744  addPass(&SILowerWWMCopiesLegacyID);1745  addPass(createVirtRegRewriter(false));1746  addPass(&AMDGPUReserveWWMRegsLegacyID);1747 1748  // For allocating per-thread VGPRs.1749  addPass(createVGPRAllocPass(true));1750 1751  addPreRewrite();1752  addPass(&VirtRegRewriterID);1753 1754  addPass(&AMDGPUMarkLastScratchLoadID);1755 1756  return true;1757}1758 1759void GCNPassConfig::addPostRegAlloc() {1760  addPass(&SIFixVGPRCopiesID);1761  if (getOptLevel() > CodeGenOptLevel::None)1762    addPass(&SIOptimizeExecMaskingLegacyID);1763  TargetPassConfig::addPostRegAlloc();1764}1765 1766void GCNPassConfig::addPreSched2() {1767  if (TM->getOptLevel() > CodeGenOptLevel::None)1768    addPass(createSIShrinkInstructionsLegacyPass());1769  addPass(&SIPostRABundlerLegacyID);1770}1771 1772void GCNPassConfig::addPreEmitPass() {1773  if (isPassEnabled(EnableVOPD, CodeGenOptLevel::Less))1774    addPass(&GCNCreateVOPDID);1775  addPass(createSIMemoryLegalizerPass());1776  addPass(createSIInsertWaitcntsPass());1777 1778  addPass(createSIModeRegisterPass());1779 1780  if (getOptLevel() > CodeGenOptLevel::None)1781    addPass(&SIInsertHardClausesID);1782 1783  addPass(&SILateBranchLoweringPassID);1784  if (isPassEnabled(EnableSetWavePriority, CodeGenOptLevel::Less))1785    addPass(createAMDGPUSetWavePriorityPass());1786  if (getOptLevel() > CodeGenOptLevel::None)1787    addPass(&SIPreEmitPeepholeID);1788  // The hazard recognizer that runs as part of the post-ra scheduler does not1789  // guarantee to be able handle all hazards correctly. This is because if there1790  // are multiple scheduling regions in a basic block, the regions are scheduled1791  // bottom up, so when we begin to schedule a region we don't know what1792  // instructions were emitted directly before it.1793  //1794  // Here we add a stand-alone hazard recognizer pass which can handle all1795  // cases.1796  addPass(&PostRAHazardRecognizerID);1797 1798  addPass(&AMDGPUWaitSGPRHazardsLegacyID);1799 1800  addPass(&AMDGPULowerVGPREncodingLegacyID);1801 1802  if (isPassEnabled(EnableInsertDelayAlu, CodeGenOptLevel::Less))1803    addPass(&AMDGPUInsertDelayAluID);1804 1805  addPass(&BranchRelaxationPassID);1806}1807 1808void GCNPassConfig::addPostBBSections() {1809  // We run this later to avoid passes like livedebugvalues and BBSections1810  // having to deal with the apparent multi-entry functions we may generate.1811  addPass(createAMDGPUPreloadKernArgPrologLegacyPass());1812}1813 1814TargetPassConfig *GCNTargetMachine::createPassConfig(PassManagerBase &PM) {1815  return new GCNPassConfig(*this, PM);1816}1817 1818void GCNTargetMachine::registerMachineRegisterInfoCallback(1819    MachineFunction &MF) const {1820  SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();1821  MF.getRegInfo().addDelegate(MFI);1822}1823 1824MachineFunctionInfo *GCNTargetMachine::createMachineFunctionInfo(1825    BumpPtrAllocator &Allocator, const Function &F,1826    const TargetSubtargetInfo *STI) const {1827  return SIMachineFunctionInfo::create<SIMachineFunctionInfo>(1828      Allocator, F, static_cast<const GCNSubtarget *>(STI));1829}1830 1831yaml::MachineFunctionInfo *GCNTargetMachine::createDefaultFuncInfoYAML() const {1832  return new yaml::SIMachineFunctionInfo();1833}1834 1835yaml::MachineFunctionInfo *1836GCNTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {1837  const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();1838  return new yaml::SIMachineFunctionInfo(1839      *MFI, *MF.getSubtarget<GCNSubtarget>().getRegisterInfo(), MF);1840}1841 1842bool GCNTargetMachine::parseMachineFunctionInfo(1843    const yaml::MachineFunctionInfo &MFI_, PerFunctionMIParsingState &PFS,1844    SMDiagnostic &Error, SMRange &SourceRange) const {1845  const yaml::SIMachineFunctionInfo &YamlMFI =1846      static_cast<const yaml::SIMachineFunctionInfo &>(MFI_);1847  MachineFunction &MF = PFS.MF;1848  SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();1849  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();1850 1851  if (MFI->initializeBaseYamlFields(YamlMFI, MF, PFS, Error, SourceRange))1852    return true;1853 1854  if (MFI->Occupancy == 0) {1855    // Fixup the subtarget dependent default value.1856    MFI->Occupancy = ST.getOccupancyWithWorkGroupSizes(MF).second;1857  }1858 1859  auto parseRegister = [&](const yaml::StringValue &RegName, Register &RegVal) {1860    Register TempReg;1861    if (parseNamedRegisterReference(PFS, TempReg, RegName.Value, Error)) {1862      SourceRange = RegName.SourceRange;1863      return true;1864    }1865    RegVal = TempReg;1866 1867    return false;1868  };1869 1870  auto parseOptionalRegister = [&](const yaml::StringValue &RegName,1871                                   Register &RegVal) {1872    return !RegName.Value.empty() && parseRegister(RegName, RegVal);1873  };1874 1875  if (parseOptionalRegister(YamlMFI.VGPRForAGPRCopy, MFI->VGPRForAGPRCopy))1876    return true;1877 1878  if (parseOptionalRegister(YamlMFI.SGPRForEXECCopy, MFI->SGPRForEXECCopy))1879    return true;1880 1881  if (parseOptionalRegister(YamlMFI.LongBranchReservedReg,1882                            MFI->LongBranchReservedReg))1883    return true;1884 1885  auto diagnoseRegisterClass = [&](const yaml::StringValue &RegName) {1886    // Create a diagnostic for a the register string literal.1887    const MemoryBuffer &Buffer =1888        *PFS.SM->getMemoryBuffer(PFS.SM->getMainFileID());1889    Error = SMDiagnostic(*PFS.SM, SMLoc(), Buffer.getBufferIdentifier(), 1,1890                         RegName.Value.size(), SourceMgr::DK_Error,1891                         "incorrect register class for field", RegName.Value,1892                         {}, {});1893    SourceRange = RegName.SourceRange;1894    return true;1895  };1896 1897  if (parseRegister(YamlMFI.ScratchRSrcReg, MFI->ScratchRSrcReg) ||1898      parseRegister(YamlMFI.FrameOffsetReg, MFI->FrameOffsetReg) ||1899      parseRegister(YamlMFI.StackPtrOffsetReg, MFI->StackPtrOffsetReg))1900    return true;1901 1902  if (MFI->ScratchRSrcReg != AMDGPU::PRIVATE_RSRC_REG &&1903      !AMDGPU::SGPR_128RegClass.contains(MFI->ScratchRSrcReg)) {1904    return diagnoseRegisterClass(YamlMFI.ScratchRSrcReg);1905  }1906 1907  if (MFI->FrameOffsetReg != AMDGPU::FP_REG &&1908      !AMDGPU::SGPR_32RegClass.contains(MFI->FrameOffsetReg)) {1909    return diagnoseRegisterClass(YamlMFI.FrameOffsetReg);1910  }1911 1912  if (MFI->StackPtrOffsetReg != AMDGPU::SP_REG &&1913      !AMDGPU::SGPR_32RegClass.contains(MFI->StackPtrOffsetReg)) {1914    return diagnoseRegisterClass(YamlMFI.StackPtrOffsetReg);1915  }1916 1917  for (const auto &YamlReg : YamlMFI.WWMReservedRegs) {1918    Register ParsedReg;1919    if (parseRegister(YamlReg, ParsedReg))1920      return true;1921 1922    MFI->reserveWWMRegister(ParsedReg);1923  }1924 1925  for (const auto &[_, Info] : PFS.VRegInfosNamed) {1926    MFI->setFlag(Info->VReg, Info->Flags);1927  }1928  for (const auto &[_, Info] : PFS.VRegInfos) {1929    MFI->setFlag(Info->VReg, Info->Flags);1930  }1931 1932  for (const auto &YamlRegStr : YamlMFI.SpillPhysVGPRS) {1933    Register ParsedReg;1934    if (parseRegister(YamlRegStr, ParsedReg))1935      return true;1936    MFI->SpillPhysVGPRs.push_back(ParsedReg);1937  }1938 1939  auto parseAndCheckArgument = [&](const std::optional<yaml::SIArgument> &A,1940                                   const TargetRegisterClass &RC,1941                                   ArgDescriptor &Arg, unsigned UserSGPRs,1942                                   unsigned SystemSGPRs) {1943    // Skip parsing if it's not present.1944    if (!A)1945      return false;1946 1947    if (A->IsRegister) {1948      Register Reg;1949      if (parseNamedRegisterReference(PFS, Reg, A->RegisterName.Value, Error)) {1950        SourceRange = A->RegisterName.SourceRange;1951        return true;1952      }1953      if (!RC.contains(Reg))1954        return diagnoseRegisterClass(A->RegisterName);1955      Arg = ArgDescriptor::createRegister(Reg);1956    } else1957      Arg = ArgDescriptor::createStack(A->StackOffset);1958    // Check and apply the optional mask.1959    if (A->Mask)1960      Arg = ArgDescriptor::createArg(Arg, *A->Mask);1961 1962    MFI->NumUserSGPRs += UserSGPRs;1963    MFI->NumSystemSGPRs += SystemSGPRs;1964    return false;1965  };1966 1967  if (YamlMFI.ArgInfo &&1968      (parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentBuffer,1969                             AMDGPU::SGPR_128RegClass,1970                             MFI->ArgInfo.PrivateSegmentBuffer, 4, 0) ||1971       parseAndCheckArgument(YamlMFI.ArgInfo->DispatchPtr,1972                             AMDGPU::SReg_64RegClass, MFI->ArgInfo.DispatchPtr,1973                             2, 0) ||1974       parseAndCheckArgument(YamlMFI.ArgInfo->QueuePtr, AMDGPU::SReg_64RegClass,1975                             MFI->ArgInfo.QueuePtr, 2, 0) ||1976       parseAndCheckArgument(YamlMFI.ArgInfo->KernargSegmentPtr,1977                             AMDGPU::SReg_64RegClass,1978                             MFI->ArgInfo.KernargSegmentPtr, 2, 0) ||1979       parseAndCheckArgument(YamlMFI.ArgInfo->DispatchID,1980                             AMDGPU::SReg_64RegClass, MFI->ArgInfo.DispatchID,1981                             2, 0) ||1982       parseAndCheckArgument(YamlMFI.ArgInfo->FlatScratchInit,1983                             AMDGPU::SReg_64RegClass,1984                             MFI->ArgInfo.FlatScratchInit, 2, 0) ||1985       parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentSize,1986                             AMDGPU::SGPR_32RegClass,1987                             MFI->ArgInfo.PrivateSegmentSize, 0, 0) ||1988       parseAndCheckArgument(YamlMFI.ArgInfo->LDSKernelId,1989                             AMDGPU::SGPR_32RegClass,1990                             MFI->ArgInfo.LDSKernelId, 0, 1) ||1991       parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDX,1992                             AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDX,1993                             0, 1) ||1994       parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDY,1995                             AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDY,1996                             0, 1) ||1997       parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDZ,1998                             AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDZ,1999                             0, 1) ||2000       parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupInfo,2001                             AMDGPU::SGPR_32RegClass,2002                             MFI->ArgInfo.WorkGroupInfo, 0, 1) ||2003       parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentWaveByteOffset,2004                             AMDGPU::SGPR_32RegClass,2005                             MFI->ArgInfo.PrivateSegmentWaveByteOffset, 0, 1) ||2006       parseAndCheckArgument(YamlMFI.ArgInfo->ImplicitArgPtr,2007                             AMDGPU::SReg_64RegClass,2008                             MFI->ArgInfo.ImplicitArgPtr, 0, 0) ||2009       parseAndCheckArgument(YamlMFI.ArgInfo->ImplicitBufferPtr,2010                             AMDGPU::SReg_64RegClass,2011                             MFI->ArgInfo.ImplicitBufferPtr, 2, 0) ||2012       parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDX,2013                             AMDGPU::VGPR_32RegClass,2014                             MFI->ArgInfo.WorkItemIDX, 0, 0) ||2015       parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDY,2016                             AMDGPU::VGPR_32RegClass,2017                             MFI->ArgInfo.WorkItemIDY, 0, 0) ||2018       parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDZ,2019                             AMDGPU::VGPR_32RegClass,2020                             MFI->ArgInfo.WorkItemIDZ, 0, 0)))2021    return true;2022 2023  // Parse FirstKernArgPreloadReg separately, since it's a Register,2024  // not ArgDescriptor.2025  if (YamlMFI.ArgInfo && YamlMFI.ArgInfo->FirstKernArgPreloadReg) {2026    const yaml::SIArgument &A = *YamlMFI.ArgInfo->FirstKernArgPreloadReg;2027 2028    if (!A.IsRegister) {2029      // For stack arguments, we don't have RegisterName.SourceRange,2030      // but we should have some location info from the YAML parser2031      const MemoryBuffer &Buffer =2032          *PFS.SM->getMemoryBuffer(PFS.SM->getMainFileID());2033      // Create a minimal valid source range2034      SMLoc Loc = SMLoc::getFromPointer(Buffer.getBufferStart());2035      SMRange Range(Loc, Loc);2036 2037      Error = SMDiagnostic(2038          *PFS.SM, Loc, Buffer.getBufferIdentifier(), 1, 0, SourceMgr::DK_Error,2039          "firstKernArgPreloadReg must be a register, not a stack location", "",2040          {}, {});2041 2042      SourceRange = Range;2043      return true;2044    }2045 2046    Register Reg;2047    if (parseNamedRegisterReference(PFS, Reg, A.RegisterName.Value, Error)) {2048      SourceRange = A.RegisterName.SourceRange;2049      return true;2050    }2051 2052    if (!AMDGPU::SGPR_32RegClass.contains(Reg))2053      return diagnoseRegisterClass(A.RegisterName);2054 2055    MFI->ArgInfo.FirstKernArgPreloadReg = Reg;2056    MFI->NumUserSGPRs += YamlMFI.NumKernargPreloadSGPRs;2057  }2058 2059  if (ST.hasIEEEMode())2060    MFI->Mode.IEEE = YamlMFI.Mode.IEEE;2061  if (ST.hasDX10ClampMode())2062    MFI->Mode.DX10Clamp = YamlMFI.Mode.DX10Clamp;2063 2064  // FIXME: Move proper support for denormal-fp-math into base MachineFunction2065  MFI->Mode.FP32Denormals.Input = YamlMFI.Mode.FP32InputDenormals2066                                      ? DenormalMode::IEEE2067                                      : DenormalMode::PreserveSign;2068  MFI->Mode.FP32Denormals.Output = YamlMFI.Mode.FP32OutputDenormals2069                                       ? DenormalMode::IEEE2070                                       : DenormalMode::PreserveSign;2071 2072  MFI->Mode.FP64FP16Denormals.Input = YamlMFI.Mode.FP64FP16InputDenormals2073                                          ? DenormalMode::IEEE2074                                          : DenormalMode::PreserveSign;2075  MFI->Mode.FP64FP16Denormals.Output = YamlMFI.Mode.FP64FP16OutputDenormals2076                                           ? DenormalMode::IEEE2077                                           : DenormalMode::PreserveSign;2078 2079  if (YamlMFI.HasInitWholeWave)2080    MFI->setInitWholeWave();2081 2082  return false;2083}2084 2085//===----------------------------------------------------------------------===//2086// AMDGPU CodeGen Pass Builder interface.2087//===----------------------------------------------------------------------===//2088 2089AMDGPUCodeGenPassBuilder::AMDGPUCodeGenPassBuilder(2090    GCNTargetMachine &TM, const CGPassBuilderOption &Opts,2091    PassInstrumentationCallbacks *PIC)2092    : CodeGenPassBuilder(TM, Opts, PIC) {2093  Opt.MISchedPostRA = true;2094  Opt.RequiresCodeGenSCCOrder = true;2095  // Exceptions and StackMaps are not supported, so these passes will never do2096  // anything.2097  // Garbage collection is not supported.2098  disablePass<StackMapLivenessPass, FuncletLayoutPass,2099              ShadowStackGCLoweringPass>();2100}2101 2102void AMDGPUCodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const {2103  if (RemoveIncompatibleFunctions && TM.getTargetTriple().isAMDGCN())2104    addPass(AMDGPURemoveIncompatibleFunctionsPass(TM));2105 2106  addPass(AMDGPUPrintfRuntimeBindingPass());2107  if (LowerCtorDtor)2108    addPass(AMDGPUCtorDtorLoweringPass());2109 2110  if (isPassEnabled(EnableImageIntrinsicOptimizer))2111    addPass(AMDGPUImageIntrinsicOptimizerPass(TM));2112 2113  if (EnableUniformIntrinsicCombine)2114    addPass(AMDGPUUniformIntrinsicCombinePass());2115  // This can be disabled by passing ::Disable here or on the command line2116  // with --expand-variadics-override=disable.2117  addPass(ExpandVariadicsPass(ExpandVariadicsMode::Lowering));2118 2119  addPass(AMDGPUAlwaysInlinePass());2120  addPass(AlwaysInlinerPass());2121 2122  addPass(AMDGPUExportKernelRuntimeHandlesPass());2123 2124  if (EnableLowerExecSync)2125    addPass(AMDGPULowerExecSyncPass());2126 2127  if (EnableSwLowerLDS)2128    addPass(AMDGPUSwLowerLDSPass(TM));2129 2130  // Runs before PromoteAlloca so the latter can account for function uses2131  if (EnableLowerModuleLDS)2132    addPass(AMDGPULowerModuleLDSPass(TM));2133 2134  // Run atomic optimizer before Atomic Expand2135  if (TM.getOptLevel() >= CodeGenOptLevel::Less &&2136      (AMDGPUAtomicOptimizerStrategy != ScanOptions::None))2137    addPass(AMDGPUAtomicOptimizerPass(TM, AMDGPUAtomicOptimizerStrategy));2138 2139  addPass(AtomicExpandPass(TM));2140 2141  if (TM.getOptLevel() > CodeGenOptLevel::None) {2142    addPass(AMDGPUPromoteAllocaPass(TM));2143    if (isPassEnabled(EnableScalarIRPasses))2144      addStraightLineScalarOptimizationPasses(addPass);2145 2146    // TODO: Handle EnableAMDGPUAliasAnalysis2147 2148    // TODO: May want to move later or split into an early and late one.2149    addPass(AMDGPUCodeGenPreparePass(TM));2150 2151    // Try to hoist loop invariant parts of divisions AMDGPUCodeGenPrepare may2152    // have expanded.2153    if (TM.getOptLevel() > CodeGenOptLevel::Less) {2154      addPass(createFunctionToLoopPassAdaptor(LICMPass(LICMOptions()),2155                                              /*UseMemorySSA=*/true));2156    }2157  }2158 2159  Base::addIRPasses(addPass);2160 2161  // EarlyCSE is not always strong enough to clean up what LSR produces. For2162  // example, GVN can combine2163  //2164  //   %0 = add %a, %b2165  //   %1 = add %b, %a2166  //2167  // and2168  //2169  //   %0 = shl nsw %a, 22170  //   %1 = shl %a, 22171  //2172  // but EarlyCSE can do neither of them.2173  if (isPassEnabled(EnableScalarIRPasses))2174    addEarlyCSEOrGVNPass(addPass);2175}2176 2177void AMDGPUCodeGenPassBuilder::addCodeGenPrepare(AddIRPass &addPass) const {2178  if (TM.getOptLevel() > CodeGenOptLevel::None)2179    addPass(AMDGPUPreloadKernelArgumentsPass(TM));2180 2181  if (EnableLowerKernelArguments)2182    addPass(AMDGPULowerKernelArgumentsPass(TM));2183 2184  Base::addCodeGenPrepare(addPass);2185 2186  if (isPassEnabled(EnableLoadStoreVectorizer))2187    addPass(LoadStoreVectorizerPass());2188 2189  // This lowering has been placed after codegenprepare to take advantage of2190  // address mode matching (which is why it isn't put with the LDS lowerings).2191  // It could be placed anywhere before uniformity annotations (an analysis2192  // that it changes by splitting up fat pointers into their components)2193  // but has been put before switch lowering and CFG flattening so that those2194  // passes can run on the more optimized control flow this pass creates in2195  // many cases.2196  addPass(AMDGPULowerBufferFatPointersPass(TM));2197  addPass.requireCGSCCOrder();2198 2199  addPass(AMDGPULowerIntrinsicsPass(TM));2200 2201  // LowerSwitch pass may introduce unreachable blocks that can cause unexpected2202  // behavior for subsequent passes. Placing it here seems better that these2203  // blocks would get cleaned up by UnreachableBlockElim inserted next in the2204  // pass flow.2205  addPass(LowerSwitchPass());2206}2207 2208void AMDGPUCodeGenPassBuilder::addPreISel(AddIRPass &addPass) const {2209 2210  if (TM.getOptLevel() > CodeGenOptLevel::None) {2211    addPass(FlattenCFGPass());2212    addPass(SinkingPass());2213    addPass(AMDGPULateCodeGenPreparePass(TM));2214  }2215 2216  // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit2217  // regions formed by them.2218 2219  addPass(AMDGPUUnifyDivergentExitNodesPass());2220  addPass(FixIrreduciblePass());2221  addPass(UnifyLoopExitsPass());2222  addPass(StructurizeCFGPass(/*SkipUniformRegions=*/false));2223 2224  addPass(AMDGPUAnnotateUniformValuesPass());2225 2226  addPass(SIAnnotateControlFlowPass(TM));2227 2228  // TODO: Move this right after structurizeCFG to avoid extra divergence2229  // analysis. This depends on stopping SIAnnotateControlFlow from making2230  // control flow modifications.2231  addPass(AMDGPURewriteUndefForPHIPass());2232 2233  if (!getCGPassBuilderOption().EnableGlobalISelOption ||2234      !isGlobalISelAbortEnabled() || !NewRegBankSelect)2235    addPass(LCSSAPass());2236 2237  if (TM.getOptLevel() > CodeGenOptLevel::Less)2238    addPass(AMDGPUPerfHintAnalysisPass(TM));2239 2240  // FIXME: Why isn't this queried as required from AMDGPUISelDAGToDAG, and why2241  // isn't this in addInstSelector?2242  addPass(RequireAnalysisPass<UniformityInfoAnalysis, Function>(),2243          /*Force=*/true);2244}2245 2246void AMDGPUCodeGenPassBuilder::addILPOpts(AddMachinePass &addPass) const {2247  if (EnableEarlyIfConversion)2248    addPass(EarlyIfConverterPass());2249 2250  Base::addILPOpts(addPass);2251}2252 2253void AMDGPUCodeGenPassBuilder::addAsmPrinter(AddMachinePass &addPass,2254                                             CreateMCStreamer) const {2255  // TODO: Add AsmPrinter.2256}2257 2258Error AMDGPUCodeGenPassBuilder::addInstSelector(AddMachinePass &addPass) const {2259  addPass(AMDGPUISelDAGToDAGPass(TM));2260  addPass(SIFixSGPRCopiesPass());2261  addPass(SILowerI1CopiesPass());2262  return Error::success();2263}2264 2265void AMDGPUCodeGenPassBuilder::addPreRewrite(AddMachinePass &addPass) const {2266  if (EnableRegReassign) {2267    addPass(GCNNSAReassignPass());2268  }2269}2270 2271void AMDGPUCodeGenPassBuilder::addMachineSSAOptimization(2272    AddMachinePass &addPass) const {2273  Base::addMachineSSAOptimization(addPass);2274 2275  addPass(SIFoldOperandsPass());2276  if (EnableDPPCombine) {2277    addPass(GCNDPPCombinePass());2278  }2279  addPass(SILoadStoreOptimizerPass());2280  if (isPassEnabled(EnableSDWAPeephole)) {2281    addPass(SIPeepholeSDWAPass());2282    addPass(EarlyMachineLICMPass());2283    addPass(MachineCSEPass());2284    addPass(SIFoldOperandsPass());2285  }2286  addPass(DeadMachineInstructionElimPass());2287  addPass(SIShrinkInstructionsPass());2288}2289 2290void AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(2291    AddMachinePass &addPass) const {2292  if (EnableDCEInRA)2293    insertPass<DetectDeadLanesPass>(DeadMachineInstructionElimPass());2294 2295  // FIXME: when an instruction has a Killed operand, and the instruction is2296  // inside a bundle, seems only the BUNDLE instruction appears as the Kills of2297  // the register in LiveVariables, this would trigger a failure in verifier,2298  // we should fix it and enable the verifier.2299  if (OptVGPRLiveRange)2300    insertPass<RequireAnalysisPass<LiveVariablesAnalysis, MachineFunction>>(2301        SIOptimizeVGPRLiveRangePass());2302 2303  // This must be run immediately after phi elimination and before2304  // TwoAddressInstructions, otherwise the processing of the tied operand of2305  // SI_ELSE will introduce a copy of the tied operand source after the else.2306  insertPass<PHIEliminationPass>(SILowerControlFlowPass());2307 2308  if (EnableRewritePartialRegUses)2309    insertPass<RenameIndependentSubregsPass>(GCNRewritePartialRegUsesPass());2310 2311  if (isPassEnabled(EnablePreRAOptimizations))2312    insertPass<MachineSchedulerPass>(GCNPreRAOptimizationsPass());2313 2314  // Allow the scheduler to run before SIWholeQuadMode inserts exec manipulation2315  // instructions that cause scheduling barriers.2316  insertPass<MachineSchedulerPass>(SIWholeQuadModePass());2317 2318  if (OptExecMaskPreRA)2319    insertPass<MachineSchedulerPass>(SIOptimizeExecMaskingPreRAPass());2320 2321  // This is not an essential optimization and it has a noticeable impact on2322  // compilation time, so we only enable it from O2.2323  if (TM.getOptLevel() > CodeGenOptLevel::Less)2324    insertPass<MachineSchedulerPass>(SIFormMemoryClausesPass());2325 2326  Base::addOptimizedRegAlloc(addPass);2327}2328 2329void AMDGPUCodeGenPassBuilder::addPreRegAlloc(AddMachinePass &addPass) const {2330  if (getOptLevel() != CodeGenOptLevel::None)2331    addPass(AMDGPUPrepareAGPRAllocPass());2332}2333 2334Error AMDGPUCodeGenPassBuilder::addRegAssignmentOptimized(2335    AddMachinePass &addPass) const {2336  // TODO: Check --regalloc-npm option2337 2338  addPass(GCNPreRALongBranchRegPass());2339 2340  addPass(RAGreedyPass({onlyAllocateSGPRs, "sgpr"}));2341 2342  // Commit allocated register changes. This is mostly necessary because too2343  // many things rely on the use lists of the physical registers, such as the2344  // verifier. This is only necessary with allocators which use LiveIntervals,2345  // since FastRegAlloc does the replacements itself.2346  addPass(VirtRegRewriterPass(false));2347 2348  // At this point, the sgpr-regalloc has been done and it is good to have the2349  // stack slot coloring to try to optimize the SGPR spill stack indices before2350  // attempting the custom SGPR spill lowering.2351  addPass(StackSlotColoringPass());2352 2353  // Equivalent of PEI for SGPRs.2354  addPass(SILowerSGPRSpillsPass());2355 2356  // To Allocate wwm registers used in whole quad mode operations (for shaders).2357  addPass(SIPreAllocateWWMRegsPass());2358 2359  // For allocating other wwm register operands.2360  addPass(RAGreedyPass({onlyAllocateWWMRegs, "wwm"}));2361  addPass(SILowerWWMCopiesPass());2362  addPass(VirtRegRewriterPass(false));2363  addPass(AMDGPUReserveWWMRegsPass());2364 2365  // For allocating per-thread VGPRs.2366  addPass(RAGreedyPass({onlyAllocateVGPRs, "vgpr"}));2367 2368 2369  addPreRewrite(addPass);2370  addPass(VirtRegRewriterPass(true));2371 2372  addPass(AMDGPUMarkLastScratchLoadPass());2373  return Error::success();2374}2375 2376void AMDGPUCodeGenPassBuilder::addPostRegAlloc(AddMachinePass &addPass) const {2377  addPass(SIFixVGPRCopiesPass());2378  if (TM.getOptLevel() > CodeGenOptLevel::None)2379    addPass(SIOptimizeExecMaskingPass());2380  Base::addPostRegAlloc(addPass);2381}2382 2383void AMDGPUCodeGenPassBuilder::addPreSched2(AddMachinePass &addPass) const {2384  if (TM.getOptLevel() > CodeGenOptLevel::None)2385    addPass(SIShrinkInstructionsPass());2386  addPass(SIPostRABundlerPass());2387}2388 2389void AMDGPUCodeGenPassBuilder::addPreEmitPass(AddMachinePass &addPass) const {2390  if (isPassEnabled(EnableVOPD, CodeGenOptLevel::Less)) {2391    addPass(GCNCreateVOPDPass());2392  }2393 2394  addPass(SIMemoryLegalizerPass());2395  addPass(SIInsertWaitcntsPass());2396 2397  // TODO: addPass(SIModeRegisterPass());2398 2399  if (TM.getOptLevel() > CodeGenOptLevel::None) {2400    // TODO: addPass(SIInsertHardClausesPass());2401  }2402 2403  addPass(SILateBranchLoweringPass());2404 2405  if (isPassEnabled(EnableSetWavePriority, CodeGenOptLevel::Less))2406    addPass(AMDGPUSetWavePriorityPass());2407 2408  if (TM.getOptLevel() > CodeGenOptLevel::None)2409    addPass(SIPreEmitPeepholePass());2410 2411  // The hazard recognizer that runs as part of the post-ra scheduler does not2412  // guarantee to be able handle all hazards correctly. This is because if there2413  // are multiple scheduling regions in a basic block, the regions are scheduled2414  // bottom up, so when we begin to schedule a region we don't know what2415  // instructions were emitted directly before it.2416  //2417  // Here we add a stand-alone hazard recognizer pass which can handle all2418  // cases.2419  addPass(PostRAHazardRecognizerPass());2420  addPass(AMDGPUWaitSGPRHazardsPass());2421  addPass(AMDGPULowerVGPREncodingPass());2422 2423  if (isPassEnabled(EnableInsertDelayAlu, CodeGenOptLevel::Less)) {2424    addPass(AMDGPUInsertDelayAluPass());2425  }2426 2427  addPass(BranchRelaxationPass());2428}2429 2430bool AMDGPUCodeGenPassBuilder::isPassEnabled(const cl::opt<bool> &Opt,2431                                             CodeGenOptLevel Level) const {2432  if (Opt.getNumOccurrences())2433    return Opt;2434  if (TM.getOptLevel() < Level)2435    return false;2436  return Opt;2437}2438 2439void AMDGPUCodeGenPassBuilder::addEarlyCSEOrGVNPass(AddIRPass &addPass) const {2440  if (TM.getOptLevel() == CodeGenOptLevel::Aggressive)2441    addPass(GVNPass());2442  else2443    addPass(EarlyCSEPass());2444}2445 2446void AMDGPUCodeGenPassBuilder::addStraightLineScalarOptimizationPasses(2447    AddIRPass &addPass) const {2448  if (isPassEnabled(EnableLoopPrefetch, CodeGenOptLevel::Aggressive))2449    addPass(LoopDataPrefetchPass());2450 2451  addPass(SeparateConstOffsetFromGEPPass());2452 2453  // ReassociateGEPs exposes more opportunities for SLSR. See2454  // the example in reassociate-geps-and-slsr.ll.2455  addPass(StraightLineStrengthReducePass());2456 2457  // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN or2458  // EarlyCSE can reuse.2459  addEarlyCSEOrGVNPass(addPass);2460 2461  // Run NaryReassociate after EarlyCSE/GVN to be more effective.2462  addPass(NaryReassociatePass());2463 2464  // NaryReassociate on GEPs creates redundant common expressions, so run2465  // EarlyCSE after it.2466  addPass(EarlyCSEPass());2467}2468