445 lines · cpp
1//===-- AMDGPUMemoryUtils.cpp - -------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "AMDGPUMemoryUtils.h"10#include "AMDGPU.h"11#include "Utils/AMDGPUBaseInfo.h"12#include "llvm/ADT/SetOperations.h"13#include "llvm/Analysis/AliasAnalysis.h"14#include "llvm/Analysis/CallGraph.h"15#include "llvm/Analysis/MemorySSA.h"16#include "llvm/IR/DataLayout.h"17#include "llvm/IR/Instructions.h"18#include "llvm/IR/IntrinsicInst.h"19#include "llvm/IR/IntrinsicsAMDGPU.h"20#include "llvm/IR/ReplaceConstant.h"21 22#define DEBUG_TYPE "amdgpu-memory-utils"23 24using namespace llvm;25 26namespace llvm::AMDGPU {27 28Align getAlign(const DataLayout &DL, const GlobalVariable *GV) {29 return DL.getValueOrABITypeAlignment(GV->getPointerAlignment(DL),30 GV->getValueType());31}32 33// Returns the target extension type of a global variable,34// which can only be a TargetExtType, an array or single-element struct of it,35// or their nesting combination.36// TODO: allow struct of multiple TargetExtType elements of the same type.37// TODO: Disallow other uses of target("amdgcn.named.barrier") including:38// - Structs containing barriers in different scope/rank39// - Structs containing a mixture of barriers and other data.40// - Globals in other address spaces.41// - Allocas.42static TargetExtType *getTargetExtType(const GlobalVariable &GV) {43 Type *Ty = GV.getValueType();44 while (true) {45 if (auto *TTy = dyn_cast<TargetExtType>(Ty))46 return TTy;47 if (auto *STy = dyn_cast<StructType>(Ty)) {48 if (STy->getNumElements() != 1)49 return nullptr;50 Ty = STy->getElementType(0);51 continue;52 }53 if (auto *ATy = dyn_cast<ArrayType>(Ty)) {54 Ty = ATy->getElementType();55 continue;56 }57 return nullptr;58 }59}60 61TargetExtType *isNamedBarrier(const GlobalVariable &GV) {62 if (TargetExtType *Ty = getTargetExtType(GV))63 return Ty->getName() == "amdgcn.named.barrier" ? Ty : nullptr;64 return nullptr;65}66 67bool isDynamicLDS(const GlobalVariable &GV) {68 // external zero size addrspace(3) without initializer is dynlds.69 const Module *M = GV.getParent();70 const DataLayout &DL = M->getDataLayout();71 if (GV.getType()->getPointerAddressSpace() != AMDGPUAS::LOCAL_ADDRESS)72 return false;73 return DL.getTypeAllocSize(GV.getValueType()) == 0;74}75 76bool isLDSVariableToLower(const GlobalVariable &GV) {77 if (GV.getType()->getPointerAddressSpace() != AMDGPUAS::LOCAL_ADDRESS) {78 return false;79 }80 if (isDynamicLDS(GV)) {81 return true;82 }83 if (GV.isConstant()) {84 // A constant undef variable can't be written to, and any load is85 // undef, so it should be eliminated by the optimizer. It could be86 // dropped by the back end if not. This pass skips over it.87 return false;88 }89 if (GV.hasInitializer() && !isa<UndefValue>(GV.getInitializer())) {90 // Initializers are unimplemented for LDS address space.91 // Leave such variables in place for consistent error reporting.92 return false;93 }94 return true;95}96 97bool eliminateConstantExprUsesOfLDSFromAllInstructions(Module &M) {98 // Constants are uniqued within LLVM. A ConstantExpr referring to a LDS99 // global may have uses from multiple different functions as a result.100 // This pass specialises LDS variables with respect to the kernel that101 // allocates them.102 103 // This is semantically equivalent to (the unimplemented as slow):104 // for (auto &F : M.functions())105 // for (auto &BB : F)106 // for (auto &I : BB)107 // for (Use &Op : I.operands())108 // if (constantExprUsesLDS(Op))109 // replaceConstantExprInFunction(I, Op);110 111 SmallVector<Constant *> LDSGlobals;112 for (auto &GV : M.globals())113 if (AMDGPU::isLDSVariableToLower(GV))114 LDSGlobals.push_back(&GV);115 return convertUsersOfConstantsToInstructions(LDSGlobals);116}117 118void getUsesOfLDSByFunction(const CallGraph &CG, Module &M,119 FunctionVariableMap &kernels,120 FunctionVariableMap &Functions) {121 // Get uses from the current function, excluding uses by called Functions122 // Two output variables to avoid walking the globals list twice123 for (auto &GV : M.globals()) {124 if (!AMDGPU::isLDSVariableToLower(GV))125 continue;126 for (User *V : GV.users()) {127 if (auto *I = dyn_cast<Instruction>(V)) {128 Function *F = I->getFunction();129 if (isKernel(*F))130 kernels[F].insert(&GV);131 else132 Functions[F].insert(&GV);133 }134 }135 }136}137 138LDSUsesInfoTy getTransitiveUsesOfLDS(const CallGraph &CG, Module &M) {139 140 FunctionVariableMap DirectMapKernel;141 FunctionVariableMap DirectMapFunction;142 getUsesOfLDSByFunction(CG, M, DirectMapKernel, DirectMapFunction);143 144 // Collect functions whose address has escaped145 DenseSet<Function *> AddressTakenFuncs;146 for (Function &F : M.functions()) {147 if (!isKernel(F))148 if (F.hasAddressTaken(nullptr,149 /* IgnoreCallbackUses */ false,150 /* IgnoreAssumeLikeCalls */ false,151 /* IgnoreLLVMUsed */ true,152 /* IgnoreArcAttachedCall */ false)) {153 AddressTakenFuncs.insert(&F);154 }155 }156 157 // Collect variables that are used by functions whose address has escaped158 DenseSet<GlobalVariable *> VariablesReachableThroughFunctionPointer;159 for (Function *F : AddressTakenFuncs) {160 set_union(VariablesReachableThroughFunctionPointer, DirectMapFunction[F]);161 }162 163 auto FunctionMakesUnknownCall = [&](const Function *F) -> bool {164 assert(!F->isDeclaration());165 for (const CallGraphNode::CallRecord &R : *CG[F]) {166 if (!R.second->getFunction())167 return true;168 }169 return false;170 };171 172 // Work out which variables are reachable through function calls173 FunctionVariableMap TransitiveMapFunction = DirectMapFunction;174 175 // If the function makes any unknown call, assume the worst case that it can176 // access all variables accessed by functions whose address escaped177 for (Function &F : M.functions()) {178 if (!F.isDeclaration() && FunctionMakesUnknownCall(&F)) {179 if (!isKernel(F)) {180 set_union(TransitiveMapFunction[&F],181 VariablesReachableThroughFunctionPointer);182 }183 }184 }185 186 // Direct implementation of collecting all variables reachable from each187 // function188 for (Function &Func : M.functions()) {189 if (Func.isDeclaration() || isKernel(Func))190 continue;191 192 DenseSet<Function *> seen; // catches cycles193 SmallVector<Function *, 4> wip = {&Func};194 195 while (!wip.empty()) {196 Function *F = wip.pop_back_val();197 198 // Can accelerate this by referring to transitive map for functions that199 // have already been computed, with more care than this200 set_union(TransitiveMapFunction[&Func], DirectMapFunction[F]);201 202 for (const CallGraphNode::CallRecord &R : *CG[F]) {203 Function *Ith = R.second->getFunction();204 if (Ith) {205 if (!seen.contains(Ith)) {206 seen.insert(Ith);207 wip.push_back(Ith);208 }209 }210 }211 }212 }213 214 // Collect variables that are transitively used by functions whose address has215 // escaped216 for (Function *F : AddressTakenFuncs) {217 set_union(VariablesReachableThroughFunctionPointer,218 TransitiveMapFunction[F]);219 }220 221 // DirectMapKernel lists which variables are used by the kernel222 // find the variables which are used through a function call223 FunctionVariableMap IndirectMapKernel;224 225 for (Function &Func : M.functions()) {226 if (Func.isDeclaration() || !isKernel(Func))227 continue;228 229 for (const CallGraphNode::CallRecord &R : *CG[&Func]) {230 Function *Ith = R.second->getFunction();231 if (Ith) {232 set_union(IndirectMapKernel[&Func], TransitiveMapFunction[Ith]);233 }234 }235 236 // Check if the kernel encounters unknows calls, wheher directly or237 // indirectly.238 bool SeesUnknownCalls = [&]() {239 SmallVector<Function *> WorkList = {CG[&Func]->getFunction()};240 SmallPtrSet<Function *, 8> Visited;241 242 while (!WorkList.empty()) {243 Function *F = WorkList.pop_back_val();244 245 for (const CallGraphNode::CallRecord &CallRecord : *CG[F]) {246 if (!CallRecord.second)247 continue;248 249 Function *Callee = CallRecord.second->getFunction();250 if (!Callee)251 return true;252 253 if (Visited.insert(Callee).second)254 WorkList.push_back(Callee);255 }256 }257 return false;258 }();259 260 if (SeesUnknownCalls) {261 set_union(IndirectMapKernel[&Func],262 VariablesReachableThroughFunctionPointer);263 }264 }265 266 // Verify that we fall into one of 2 cases:267 // - All variables are either absolute268 // or direct mapped dynamic LDS that is not lowered.269 // this is a re-run of the pass270 // so we don't have anything to do.271 // - No variables are absolute.272 // Named-barriers which are absolute symbols are removed273 // from the maps.274 std::optional<bool> HasAbsoluteGVs;275 bool HasSpecialGVs = false;276 for (auto &Map : {DirectMapKernel, IndirectMapKernel}) {277 for (auto &[Fn, GVs] : Map) {278 for (auto *GV : GVs) {279 bool IsAbsolute = GV->isAbsoluteSymbolRef();280 bool IsDirectMapDynLDSGV =281 AMDGPU::isDynamicLDS(*GV) && DirectMapKernel.contains(Fn);282 if (IsDirectMapDynLDSGV)283 continue;284 if (isNamedBarrier(*GV)) {285 if (IsAbsolute) {286 DirectMapKernel[Fn].erase(GV);287 IndirectMapKernel[Fn].erase(GV);288 }289 HasSpecialGVs = true;290 continue;291 }292 if (HasAbsoluteGVs.has_value()) {293 if (*HasAbsoluteGVs != IsAbsolute) {294 reportFatalUsageError(295 "module cannot mix absolute and non-absolute LDS GVs");296 }297 } else298 HasAbsoluteGVs = IsAbsolute;299 }300 }301 }302 303 // If we only had absolute GVs, we have nothing to do, return an empty304 // result.305 if (HasAbsoluteGVs && *HasAbsoluteGVs)306 return {FunctionVariableMap(), FunctionVariableMap(), false};307 308 return {std::move(DirectMapKernel), std::move(IndirectMapKernel),309 HasSpecialGVs};310}311 312void removeFnAttrFromReachable(CallGraph &CG, Function *KernelRoot,313 ArrayRef<StringRef> FnAttrs) {314 for (StringRef Attr : FnAttrs)315 KernelRoot->removeFnAttr(Attr);316 317 SmallVector<Function *> WorkList = {CG[KernelRoot]->getFunction()};318 SmallPtrSet<Function *, 8> Visited;319 bool SeenUnknownCall = false;320 321 while (!WorkList.empty()) {322 Function *F = WorkList.pop_back_val();323 324 for (auto &CallRecord : *CG[F]) {325 if (!CallRecord.second)326 continue;327 328 Function *Callee = CallRecord.second->getFunction();329 if (!Callee) {330 if (!SeenUnknownCall) {331 SeenUnknownCall = true;332 333 // If we see any indirect calls, assume nothing about potential334 // targets.335 // TODO: This could be refined to possible LDS global users.336 for (auto &ExternalCallRecord : *CG.getExternalCallingNode()) {337 Function *PotentialCallee =338 ExternalCallRecord.second->getFunction();339 assert(PotentialCallee);340 if (!isKernel(*PotentialCallee)) {341 for (StringRef Attr : FnAttrs)342 PotentialCallee->removeFnAttr(Attr);343 }344 }345 }346 } else {347 for (StringRef Attr : FnAttrs)348 Callee->removeFnAttr(Attr);349 if (Visited.insert(Callee).second)350 WorkList.push_back(Callee);351 }352 }353 }354}355 356bool isReallyAClobber(const Value *Ptr, MemoryDef *Def, AAResults *AA) {357 Instruction *DefInst = Def->getMemoryInst();358 359 if (isa<FenceInst>(DefInst))360 return false;361 362 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(DefInst)) {363 switch (II->getIntrinsicID()) {364 case Intrinsic::amdgcn_s_barrier:365 case Intrinsic::amdgcn_s_cluster_barrier:366 case Intrinsic::amdgcn_s_barrier_signal:367 case Intrinsic::amdgcn_s_barrier_signal_var:368 case Intrinsic::amdgcn_s_barrier_signal_isfirst:369 case Intrinsic::amdgcn_s_barrier_init:370 case Intrinsic::amdgcn_s_barrier_join:371 case Intrinsic::amdgcn_s_barrier_wait:372 case Intrinsic::amdgcn_s_barrier_leave:373 case Intrinsic::amdgcn_s_get_barrier_state:374 case Intrinsic::amdgcn_wave_barrier:375 case Intrinsic::amdgcn_sched_barrier:376 case Intrinsic::amdgcn_sched_group_barrier:377 case Intrinsic::amdgcn_iglp_opt:378 return false;379 default:380 break;381 }382 }383 384 // Ignore atomics not aliasing with the original load, any atomic is a385 // universal MemoryDef from MSSA's point of view too, just like a fence.386 const auto checkNoAlias = [AA, Ptr](auto I) -> bool {387 return I && AA->isNoAlias(I->getPointerOperand(), Ptr);388 };389 390 if (checkNoAlias(dyn_cast<AtomicCmpXchgInst>(DefInst)) ||391 checkNoAlias(dyn_cast<AtomicRMWInst>(DefInst)))392 return false;393 394 return true;395}396 397bool isClobberedInFunction(const LoadInst *Load, MemorySSA *MSSA,398 AAResults *AA) {399 MemorySSAWalker *Walker = MSSA->getWalker();400 SmallVector<MemoryAccess *> WorkList{Walker->getClobberingMemoryAccess(Load)};401 SmallPtrSet<MemoryAccess *, 8> Visited;402 MemoryLocation Loc(MemoryLocation::get(Load));403 404 LLVM_DEBUG(dbgs() << "Checking clobbering of: " << *Load << '\n');405 406 // Start with a nearest dominating clobbering access, it will be either407 // live on entry (nothing to do, load is not clobbered), MemoryDef, or408 // MemoryPhi if several MemoryDefs can define this memory state. In that409 // case add all Defs to WorkList and continue going up and checking all410 // the definitions of this memory location until the root. When all the411 // defs are exhausted and came to the entry state we have no clobber.412 // Along the scan ignore barriers and fences which are considered clobbers413 // by the MemorySSA, but not really writing anything into the memory.414 while (!WorkList.empty()) {415 MemoryAccess *MA = WorkList.pop_back_val();416 if (!Visited.insert(MA).second)417 continue;418 419 if (MSSA->isLiveOnEntryDef(MA))420 continue;421 422 if (MemoryDef *Def = dyn_cast<MemoryDef>(MA)) {423 LLVM_DEBUG(dbgs() << " Def: " << *Def->getMemoryInst() << '\n');424 425 if (isReallyAClobber(Load->getPointerOperand(), Def, AA)) {426 LLVM_DEBUG(dbgs() << " -> load is clobbered\n");427 return true;428 }429 430 WorkList.push_back(431 Walker->getClobberingMemoryAccess(Def->getDefiningAccess(), Loc));432 continue;433 }434 435 const MemoryPhi *Phi = cast<MemoryPhi>(MA);436 for (const auto &Use : Phi->incoming_values())437 WorkList.push_back(cast<MemoryAccess>(&Use));438 }439 440 LLVM_DEBUG(dbgs() << " -> no clobber\n");441 return false;442}443 444} // end namespace llvm::AMDGPU445