155 lines · cpp
1//- NVPTXForwardParams.cpp - NVPTX Forward Device Params Removing Local Copy -//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// PTX supports 2 methods of accessing device function parameters:10//11// - "simple" case: If a parameters is only loaded, and all loads can address12// the parameter via a constant offset, then the parameter may be loaded via13// the ".param" address space. This case is not possible if the parameters14// is stored to or has it's address taken. This method is preferable when15// possible. Ex:16//17// ld.param.u32 %r1, [foo_param_1];18// ld.param.u32 %r2, [foo_param_1+4];19//20// - "move param" case: For more complex cases the address of the param may be21// placed in a register via a "mov" instruction. This "mov" also implicitly22// moves the param to the ".local" address space and allows for it to be23// written to. This essentially defers the responsibilty of the byval copy24// to the PTX calling convention.25//26// mov.b64 %rd1, foo_param_0;27// st.local.u32 [%rd1], 42;28// add.u64 %rd3, %rd1, %rd2;29// ld.local.u32 %r2, [%rd3];30//31// In NVPTXLowerArgs and SelectionDAG, we pessimistically assume that all32// parameters will use the "move param" case and the local address space. This33// pass is responsible for switching to the "simple" case when possible, as it34// is more efficient.35//36// We do this by simply traversing uses of the param "mov" instructions an37// trivially checking if they are all loads.38//39//===----------------------------------------------------------------------===//40 41#include "NVPTX.h"42#include "llvm/ADT/SmallVector.h"43#include "llvm/CodeGen/MachineFunctionPass.h"44#include "llvm/CodeGen/MachineInstr.h"45#include "llvm/CodeGen/MachineOperand.h"46#include "llvm/CodeGen/MachineRegisterInfo.h"47#include "llvm/CodeGen/TargetRegisterInfo.h"48#include "llvm/Support/ErrorHandling.h"49 50using namespace llvm;51 52static bool traverseMoveUse(MachineInstr &U, const MachineRegisterInfo &MRI,53 SmallVectorImpl<MachineInstr *> &RemoveList,54 SmallVectorImpl<MachineInstr *> &LoadInsts) {55 switch (U.getOpcode()) {56 case NVPTX::LD_i16:57 case NVPTX::LD_i32:58 case NVPTX::LD_i64:59 case NVPTX::LDV_i16_v2:60 case NVPTX::LDV_i16_v4:61 case NVPTX::LDV_i32_v2:62 case NVPTX::LDV_i32_v4:63 case NVPTX::LDV_i64_v2:64 case NVPTX::LDV_i64_v4: {65 LoadInsts.push_back(&U);66 return true;67 }68 case NVPTX::cvta_local:69 case NVPTX::cvta_local_64:70 case NVPTX::cvta_to_local:71 case NVPTX::cvta_to_local_64: {72 for (auto &U2 : MRI.use_instructions(U.operands_begin()->getReg()))73 if (!traverseMoveUse(U2, MRI, RemoveList, LoadInsts))74 return false;75 76 RemoveList.push_back(&U);77 return true;78 }79 default:80 return false;81 }82}83 84static bool eliminateMove(MachineInstr &Mov, const MachineRegisterInfo &MRI,85 SmallVectorImpl<MachineInstr *> &RemoveList) {86 SmallVector<MachineInstr *, 16> MaybeRemoveList;87 SmallVector<MachineInstr *, 16> LoadInsts;88 89 for (auto &U : MRI.use_instructions(Mov.operands_begin()->getReg()))90 if (!traverseMoveUse(U, MRI, MaybeRemoveList, LoadInsts))91 return false;92 93 RemoveList.append(MaybeRemoveList);94 RemoveList.push_back(&Mov);95 96 const MachineOperand *ParamSymbol = Mov.uses().begin();97 assert(ParamSymbol->isSymbol());98 99 constexpr unsigned LDInstBasePtrOpIdx = 6;100 constexpr unsigned LDInstAddrSpaceOpIdx = 2;101 for (auto *LI : LoadInsts) {102 (LI->uses().begin() + LDInstBasePtrOpIdx)103 ->ChangeToES(ParamSymbol->getSymbolName());104 (LI->uses().begin() + LDInstAddrSpaceOpIdx)105 ->ChangeToImmediate(NVPTX::AddressSpace::Param);106 }107 return true;108}109 110static bool forwardDeviceParams(MachineFunction &MF) {111 const auto &MRI = MF.getRegInfo();112 113 bool Changed = false;114 SmallVector<MachineInstr *, 16> RemoveList;115 for (auto &MI : make_early_inc_range(*MF.begin()))116 if (MI.getOpcode() == NVPTX::MOV32_PARAM ||117 MI.getOpcode() == NVPTX::MOV64_PARAM)118 Changed |= eliminateMove(MI, MRI, RemoveList);119 120 for (auto *MI : RemoveList)121 MI->eraseFromParent();122 123 return Changed;124}125 126/// ----------------------------------------------------------------------------127/// Pass (Manager) Boilerplate128/// ----------------------------------------------------------------------------129 130namespace {131struct NVPTXForwardParamsPass : public MachineFunctionPass {132 static char ID;133 NVPTXForwardParamsPass() : MachineFunctionPass(ID) {}134 135 bool runOnMachineFunction(MachineFunction &MF) override;136 137 void getAnalysisUsage(AnalysisUsage &AU) const override {138 MachineFunctionPass::getAnalysisUsage(AU);139 }140};141} // namespace142 143char NVPTXForwardParamsPass::ID = 0;144 145INITIALIZE_PASS(NVPTXForwardParamsPass, "nvptx-forward-params",146 "NVPTX Forward Params", false, false)147 148bool NVPTXForwardParamsPass::runOnMachineFunction(MachineFunction &MF) {149 return forwardDeviceParams(MF);150}151 152MachineFunctionPass *llvm::createNVPTXForwardParamsPass() {153 return new NVPTXForwardParamsPass();154}155