241 lines · cpp
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Lower LDS global variables with target extension type "amdgpu.named.barrier"10// that require specialized address assignment. It assigns a unique11// barrier identifier to each named-barrier LDS variable and encodes12// this identifier within the !absolute_symbol metadata of that global.13// This encoding ensures that subsequent LDS lowering passes can process these14// barriers correctly without conflicts.15//16//===----------------------------------------------------------------------===//17 18#include "AMDGPU.h"19#include "AMDGPUMemoryUtils.h"20#include "AMDGPUTargetMachine.h"21#include "llvm/ADT/DenseMap.h"22#include "llvm/Analysis/CallGraph.h"23#include "llvm/CodeGen/TargetPassConfig.h"24#include "llvm/IR/Constants.h"25#include "llvm/IR/Instructions.h"26#include "llvm/IR/ReplaceConstant.h"27#include "llvm/InitializePasses.h"28#include "llvm/Pass.h"29 30#include <algorithm>31 32#define DEBUG_TYPE "amdgpu-lower-exec-sync"33 34using namespace llvm;35using namespace AMDGPU;36 37namespace {38 39// If GV is also used directly by other kernels, create a new GV40// used only by this kernel and its function.41static GlobalVariable *uniquifyGVPerKernel(Module &M, GlobalVariable *GV,42 Function *KF) {43 bool NeedsReplacement = false;44 for (Use &U : GV->uses()) {45 if (auto *I = dyn_cast<Instruction>(U.getUser())) {46 Function *F = I->getFunction();47 if (isKernel(*F) && F != KF) {48 NeedsReplacement = true;49 break;50 }51 }52 }53 if (!NeedsReplacement)54 return GV;55 // Create a new GV used only by this kernel and its function56 GlobalVariable *NewGV = new GlobalVariable(57 M, GV->getValueType(), GV->isConstant(), GV->getLinkage(),58 GV->getInitializer(), GV->getName() + "." + KF->getName(), nullptr,59 GV->getThreadLocalMode(), GV->getType()->getAddressSpace());60 NewGV->copyAttributesFrom(GV);61 for (Use &U : make_early_inc_range(GV->uses())) {62 if (auto *I = dyn_cast<Instruction>(U.getUser())) {63 Function *F = I->getFunction();64 if (!isKernel(*F) || F == KF) {65 U.getUser()->replaceUsesOfWith(GV, NewGV);66 }67 }68 }69 return NewGV;70}71 72// Write the specified address into metadata where it can be retrieved by73// the assembler. Format is a half open range, [Address Address+1)74static void recordLDSAbsoluteAddress(Module *M, GlobalVariable *GV,75 uint32_t Address) {76 LLVMContext &Ctx = M->getContext();77 auto *IntTy = M->getDataLayout().getIntPtrType(Ctx, AMDGPUAS::LOCAL_ADDRESS);78 auto *MinC = ConstantAsMetadata::get(ConstantInt::get(IntTy, Address));79 auto *MaxC = ConstantAsMetadata::get(ConstantInt::get(IntTy, Address + 1));80 GV->setMetadata(LLVMContext::MD_absolute_symbol,81 MDNode::get(Ctx, {MinC, MaxC}));82}83 84template <typename T> SmallVector<T> sortByName(SmallVector<T> &&V) {85 sort(V, [](const auto *L, const auto *R) {86 return L->getName() < R->getName();87 });88 return {std::move(V)};89}90 91// Main utility function for special LDS variables lowering.92static bool lowerExecSyncGlobalVariables(93 Module &M, LDSUsesInfoTy &LDSUsesInfo,94 VariableFunctionMap &LDSToKernelsThatNeedToAccessItIndirectly) {95 bool Changed = false;96 const DataLayout &DL = M.getDataLayout();97 // The 1st round: give module-absolute assignments98 int NumAbsolutes = 0;99 SmallVector<GlobalVariable *> OrderedGVs;100 for (auto &K : LDSToKernelsThatNeedToAccessItIndirectly) {101 GlobalVariable *GV = K.first;102 if (!isNamedBarrier(*GV))103 continue;104 // give a module-absolute assignment if it is indirectly accessed by105 // multiple kernels. This is not precise, but we don't want to duplicate106 // a function when it is called by multiple kernels.107 if (LDSToKernelsThatNeedToAccessItIndirectly[GV].size() > 1) {108 OrderedGVs.push_back(GV);109 } else {110 // leave it to the 2nd round, which will give a kernel-relative111 // assignment if it is only indirectly accessed by one kernel112 LDSUsesInfo.direct_access[*K.second.begin()].insert(GV);113 }114 LDSToKernelsThatNeedToAccessItIndirectly.erase(GV);115 }116 OrderedGVs = sortByName(std::move(OrderedGVs));117 for (GlobalVariable *GV : OrderedGVs) {118 unsigned BarrierScope = AMDGPU::Barrier::BARRIER_SCOPE_WORKGROUP;119 unsigned BarId = NumAbsolutes + 1;120 unsigned BarCnt = DL.getTypeAllocSize(GV->getValueType()) / 16;121 NumAbsolutes += BarCnt;122 123 // 4 bits for alignment, 5 bits for the barrier num,124 // 3 bits for the barrier scope125 unsigned Offset = 0x802000u | BarrierScope << 9 | BarId << 4;126 recordLDSAbsoluteAddress(&M, GV, Offset);127 }128 OrderedGVs.clear();129 130 // The 2nd round: give a kernel-relative assignment for GV that131 // either only indirectly accessed by single kernel or only directly132 // accessed by multiple kernels.133 SmallVector<Function *> OrderedKernels;134 for (auto &K : LDSUsesInfo.direct_access) {135 Function *F = K.first;136 assert(isKernel(*F));137 OrderedKernels.push_back(F);138 }139 OrderedKernels = sortByName(std::move(OrderedKernels));140 141 DenseMap<Function *, uint32_t> Kernel2BarId;142 for (Function *F : OrderedKernels) {143 for (GlobalVariable *GV : LDSUsesInfo.direct_access[F]) {144 if (!isNamedBarrier(*GV))145 continue;146 147 LDSUsesInfo.direct_access[F].erase(GV);148 if (GV->isAbsoluteSymbolRef()) {149 // already assigned150 continue;151 }152 OrderedGVs.push_back(GV);153 }154 OrderedGVs = sortByName(std::move(OrderedGVs));155 for (GlobalVariable *GV : OrderedGVs) {156 // GV could also be used directly by other kernels. If so, we need to157 // create a new GV used only by this kernel and its function.158 auto NewGV = uniquifyGVPerKernel(M, GV, F);159 Changed |= (NewGV != GV);160 unsigned BarrierScope = AMDGPU::Barrier::BARRIER_SCOPE_WORKGROUP;161 unsigned BarId = Kernel2BarId[F];162 BarId += NumAbsolutes + 1;163 unsigned BarCnt = DL.getTypeAllocSize(GV->getValueType()) / 16;164 Kernel2BarId[F] += BarCnt;165 unsigned Offset = 0x802000u | BarrierScope << 9 | BarId << 4;166 recordLDSAbsoluteAddress(&M, NewGV, Offset);167 }168 OrderedGVs.clear();169 }170 // Also erase those special LDS variables from indirect_access.171 for (auto &K : LDSUsesInfo.indirect_access) {172 assert(isKernel(*K.first));173 for (GlobalVariable *GV : K.second) {174 if (isNamedBarrier(*GV))175 K.second.erase(GV);176 }177 }178 return Changed;179}180 181static bool runLowerExecSyncGlobals(Module &M) {182 CallGraph CG = CallGraph(M);183 bool Changed = false;184 Changed |= eliminateConstantExprUsesOfLDSFromAllInstructions(M);185 186 // For each kernel, what variables does it access directly or through187 // callees188 LDSUsesInfoTy LDSUsesInfo = getTransitiveUsesOfLDS(CG, M);189 190 // For each variable accessed through callees, which kernels access it191 VariableFunctionMap LDSToKernelsThatNeedToAccessItIndirectly;192 for (auto &K : LDSUsesInfo.indirect_access) {193 Function *F = K.first;194 assert(isKernel(*F));195 for (GlobalVariable *GV : K.second) {196 LDSToKernelsThatNeedToAccessItIndirectly[GV].insert(F);197 }198 }199 200 if (LDSUsesInfo.HasSpecialGVs) {201 // Special LDS variables need special address assignment202 Changed |= lowerExecSyncGlobalVariables(203 M, LDSUsesInfo, LDSToKernelsThatNeedToAccessItIndirectly);204 }205 return Changed;206}207 208class AMDGPULowerExecSyncLegacy : public ModulePass {209public:210 static char ID;211 AMDGPULowerExecSyncLegacy() : ModulePass(ID) {}212 bool runOnModule(Module &M) override;213};214 215} // namespace216 217char AMDGPULowerExecSyncLegacy::ID = 0;218char &llvm::AMDGPULowerExecSyncLegacyPassID = AMDGPULowerExecSyncLegacy::ID;219 220INITIALIZE_PASS_BEGIN(AMDGPULowerExecSyncLegacy, DEBUG_TYPE,221 "AMDGPU lowering of execution synchronization", false,222 false)223INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)224INITIALIZE_PASS_END(AMDGPULowerExecSyncLegacy, DEBUG_TYPE,225 "AMDGPU lowering of execution synchronization", false,226 false)227 228bool AMDGPULowerExecSyncLegacy::runOnModule(Module &M) {229 return runLowerExecSyncGlobals(M);230}231 232ModulePass *llvm::createAMDGPULowerExecSyncLegacyPass() {233 return new AMDGPULowerExecSyncLegacy();234}235 236PreservedAnalyses AMDGPULowerExecSyncPass::run(Module &M,237 ModuleAnalysisManager &AM) {238 return runLowerExecSyncGlobals(M) ? PreservedAnalyses::none()239 : PreservedAnalyses::all();240}241