381 lines · cpp
1//===- NVPTXUtilities.cpp - Utility Functions -----------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file contains miscellaneous utility functions10//11//===----------------------------------------------------------------------===//12 13#include "NVPTXUtilities.h"14#include "NVPTX.h"15#include "NVPTXTargetMachine.h"16#include "llvm/ADT/ArrayRef.h"17#include "llvm/ADT/SmallVector.h"18#include "llvm/ADT/StringRef.h"19#include "llvm/IR/Argument.h"20#include "llvm/IR/Constants.h"21#include "llvm/IR/Function.h"22#include "llvm/IR/GlobalVariable.h"23#include "llvm/IR/Module.h"24#include "llvm/Support/Alignment.h"25#include "llvm/Support/ModRef.h"26#include "llvm/Support/Mutex.h"27#include <cstdint>28#include <cstring>29#include <map>30#include <mutex>31#include <optional>32#include <string>33#include <vector>34 35namespace llvm {36 37namespace {38typedef std::map<std::string, std::vector<unsigned>> key_val_pair_t;39typedef std::map<const GlobalValue *, key_val_pair_t> global_val_annot_t;40 41struct AnnotationCache {42 sys::Mutex Lock;43 std::map<const Module *, global_val_annot_t> Cache;44};45 46AnnotationCache &getAnnotationCache() {47 static AnnotationCache AC;48 return AC;49}50} // anonymous namespace51 52void clearAnnotationCache(const Module *Mod) {53 auto &AC = getAnnotationCache();54 std::lock_guard<sys::Mutex> Guard(AC.Lock);55 AC.Cache.erase(Mod);56}57 58static void cacheAnnotationFromMD(const MDNode *MetadataNode,59 key_val_pair_t &retval) {60 auto &AC = getAnnotationCache();61 std::lock_guard<sys::Mutex> Guard(AC.Lock);62 assert(MetadataNode && "Invalid mdnode for annotation");63 assert((MetadataNode->getNumOperands() % 2) == 1 &&64 "Invalid number of operands");65 // start index = 1, to skip the global variable key66 // increment = 2, to skip the value for each property-value pairs67 for (unsigned i = 1, e = MetadataNode->getNumOperands(); i != e; i += 2) {68 // property69 const MDString *prop = dyn_cast<MDString>(MetadataNode->getOperand(i));70 assert(prop && "Annotation property not a string");71 std::string Key = prop->getString().str();72 73 // value74 if (ConstantInt *Val = mdconst::dyn_extract<ConstantInt>(75 MetadataNode->getOperand(i + 1))) {76 retval[Key].push_back(Val->getZExtValue());77 } else {78 llvm_unreachable("Value operand not a constant int");79 }80 }81}82 83static void cacheAnnotationFromMD(const Module *m, const GlobalValue *gv) {84 auto &AC = getAnnotationCache();85 std::lock_guard<sys::Mutex> Guard(AC.Lock);86 NamedMDNode *NMD = m->getNamedMetadata("nvvm.annotations");87 if (!NMD)88 return;89 key_val_pair_t tmp;90 for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) {91 const MDNode *elem = NMD->getOperand(i);92 93 GlobalValue *entity =94 mdconst::dyn_extract_or_null<GlobalValue>(elem->getOperand(0));95 // entity may be null due to DCE96 if (!entity)97 continue;98 if (entity != gv)99 continue;100 101 // accumulate annotations for entity in tmp102 cacheAnnotationFromMD(elem, tmp);103 }104 105 if (tmp.empty()) // no annotations for this gv106 return;107 108 AC.Cache[m][gv] = std::move(tmp);109}110 111static std::optional<unsigned> findOneNVVMAnnotation(const GlobalValue *gv,112 const std::string &prop) {113 auto &AC = getAnnotationCache();114 std::lock_guard<sys::Mutex> Guard(AC.Lock);115 const Module *m = gv->getParent();116 auto ACIt = AC.Cache.find(m);117 if (ACIt == AC.Cache.end())118 cacheAnnotationFromMD(m, gv);119 else if (ACIt->second.find(gv) == ACIt->second.end())120 cacheAnnotationFromMD(m, gv);121 // Look up AC.Cache[m][gv] again because cacheAnnotationFromMD may have122 // inserted the entry.123 auto &KVP = AC.Cache[m][gv];124 auto It = KVP.find(prop);125 if (It == KVP.end())126 return std::nullopt;127 return It->second[0];128}129 130static bool findAllNVVMAnnotation(const GlobalValue *gv,131 const std::string &prop,132 std::vector<unsigned> &retval) {133 auto &AC = getAnnotationCache();134 std::lock_guard<sys::Mutex> Guard(AC.Lock);135 const Module *m = gv->getParent();136 auto ACIt = AC.Cache.find(m);137 if (ACIt == AC.Cache.end())138 cacheAnnotationFromMD(m, gv);139 else if (ACIt->second.find(gv) == ACIt->second.end())140 cacheAnnotationFromMD(m, gv);141 // Look up AC.Cache[m][gv] again because cacheAnnotationFromMD may have142 // inserted the entry.143 auto &KVP = AC.Cache[m][gv];144 auto It = KVP.find(prop);145 if (It == KVP.end())146 return false;147 retval = It->second;148 return true;149}150 151static bool globalHasNVVMAnnotation(const Value &V, const std::string &Prop) {152 if (const auto *GV = dyn_cast<GlobalValue>(&V))153 if (const auto Annot = findOneNVVMAnnotation(GV, Prop)) {154 assert((*Annot == 1) && "Unexpected annotation on a symbol");155 return true;156 }157 158 return false;159}160 161static bool argHasNVVMAnnotation(const Value &Val,162 const std::string &Annotation) {163 if (const Argument *Arg = dyn_cast<Argument>(&Val)) {164 const Function *Func = Arg->getParent();165 std::vector<unsigned> Annot;166 if (findAllNVVMAnnotation(Func, Annotation, Annot)) {167 if (is_contained(Annot, Arg->getArgNo()))168 return true;169 }170 }171 return false;172}173 174static std::optional<unsigned> getFnAttrParsedInt(const Function &F,175 StringRef Attr) {176 return F.hasFnAttribute(Attr)177 ? std::optional(F.getFnAttributeAsParsedInteger(Attr))178 : std::nullopt;179}180 181static SmallVector<unsigned, 3> getFnAttrParsedVector(const Function &F,182 StringRef Attr) {183 SmallVector<unsigned, 3> V;184 auto &Ctx = F.getContext();185 186 if (F.hasFnAttribute(Attr)) {187 // We expect the attribute value to be of the form "x[,y[,z]]", where x, y,188 // and z are unsigned values.189 StringRef S = F.getFnAttribute(Attr).getValueAsString();190 for (unsigned I = 0; I < 3 && !S.empty(); I++) {191 auto [First, Rest] = S.split(",");192 unsigned IntVal;193 if (First.trim().getAsInteger(0, IntVal))194 Ctx.emitError("can't parse integer attribute " + First + " in " + Attr);195 196 V.push_back(IntVal);197 S = Rest;198 }199 }200 return V;201}202 203static std::optional<uint64_t> getVectorProduct(ArrayRef<unsigned> V) {204 if (V.empty())205 return std::nullopt;206 207 return std::accumulate(V.begin(), V.end(), 1, std::multiplies<uint64_t>{});208}209 210bool isParamGridConstant(const Argument &Arg) {211 assert(isKernelFunction(*Arg.getParent()) &&212 "only kernel arguments can be grid_constant");213 214 if (!Arg.hasByValAttr())215 return false;216 217 // Lowering an argument as a grid_constant violates the byval semantics (and218 // the C++ API) by reusing the same memory location for the argument across219 // multiple threads. If an argument doesn't read memory and its address is not220 // captured (its address is not compared with any value), then the tweak of221 // the C++ API and byval semantics is unobservable by the program and we can222 // lower the arg as a grid_constant.223 if (Arg.onlyReadsMemory()) {224 const auto CI = Arg.getAttributes().getCaptureInfo();225 if (!capturesAddress(CI) && !capturesFullProvenance(CI))226 return true;227 }228 229 // "grid_constant" counts argument indices starting from 1230 if (Arg.hasAttribute("nvvm.grid_constant"))231 return true;232 233 return false;234}235 236bool isTexture(const Value &V) { return globalHasNVVMAnnotation(V, "texture"); }237 238bool isSurface(const Value &V) { return globalHasNVVMAnnotation(V, "surface"); }239 240bool isSampler(const Value &V) {241 const char *AnnotationName = "sampler";242 243 return globalHasNVVMAnnotation(V, AnnotationName) ||244 argHasNVVMAnnotation(V, AnnotationName);245}246 247bool isImageReadOnly(const Value &V) {248 return argHasNVVMAnnotation(V, "rdoimage");249}250 251bool isImageWriteOnly(const Value &V) {252 return argHasNVVMAnnotation(V, "wroimage");253}254 255bool isImageReadWrite(const Value &V) {256 return argHasNVVMAnnotation(V, "rdwrimage");257}258 259bool isImage(const Value &V) {260 return isImageReadOnly(V) || isImageWriteOnly(V) || isImageReadWrite(V);261}262 263bool isManaged(const Value &V) { return globalHasNVVMAnnotation(V, "managed"); }264 265StringRef getTextureName(const Value &V) {266 assert(V.hasName() && "Found texture variable with no name");267 return V.getName();268}269 270StringRef getSurfaceName(const Value &V) {271 assert(V.hasName() && "Found surface variable with no name");272 return V.getName();273}274 275StringRef getSamplerName(const Value &V) {276 assert(V.hasName() && "Found sampler variable with no name");277 return V.getName();278}279 280SmallVector<unsigned, 3> getMaxNTID(const Function &F) {281 return getFnAttrParsedVector(F, "nvvm.maxntid");282}283 284SmallVector<unsigned, 3> getReqNTID(const Function &F) {285 return getFnAttrParsedVector(F, "nvvm.reqntid");286}287 288SmallVector<unsigned, 3> getClusterDim(const Function &F) {289 return getFnAttrParsedVector(F, "nvvm.cluster_dim");290}291 292std::optional<uint64_t> getOverallMaxNTID(const Function &F) {293 // Note: The semantics here are a bit strange. The PTX ISA states the294 // following (11.4.2. Performance-Tuning Directives: .maxntid):295 //296 // Note that this directive guarantees that the total number of threads does297 // not exceed the maximum, but does not guarantee that the limit in any298 // particular dimension is not exceeded.299 const auto MaxNTID = getMaxNTID(F);300 return getVectorProduct(MaxNTID);301}302 303std::optional<uint64_t> getOverallReqNTID(const Function &F) {304 // Note: The semantics here are a bit strange. See getMaxNTID.305 const auto ReqNTID = getReqNTID(F);306 return getVectorProduct(ReqNTID);307}308 309std::optional<uint64_t> getOverallClusterRank(const Function &F) {310 // maxclusterrank and cluster_dim are mutually exclusive.311 if (const auto ClusterRank = getMaxClusterRank(F))312 return ClusterRank;313 314 // Note: The semantics here are a bit strange. See getMaxNTID.315 const auto ClusterDim = getClusterDim(F);316 return getVectorProduct(ClusterDim);317}318 319std::optional<unsigned> getMaxClusterRank(const Function &F) {320 return getFnAttrParsedInt(F, "nvvm.maxclusterrank");321}322 323std::optional<unsigned> getMinCTASm(const Function &F) {324 return getFnAttrParsedInt(F, "nvvm.minctasm");325}326 327std::optional<unsigned> getMaxNReg(const Function &F) {328 return getFnAttrParsedInt(F, "nvvm.maxnreg");329}330 331bool hasBlocksAreClusters(const Function &F) {332 return F.hasFnAttribute("nvvm.blocksareclusters");333}334 335MaybeAlign getAlign(const CallInst &I, unsigned Index) {336 // First check the alignstack metadata337 if (MaybeAlign StackAlign =338 I.getAttributes().getAttributes(Index).getStackAlignment())339 return StackAlign;340 341 // If that is missing, check the legacy nvvm metadata342 if (MDNode *alignNode = I.getMetadata("callalign")) {343 for (int i = 0, n = alignNode->getNumOperands(); i < n; i++) {344 if (const ConstantInt *CI =345 mdconst::dyn_extract<ConstantInt>(alignNode->getOperand(i))) {346 unsigned V = CI->getZExtValue();347 if ((V >> 16) == Index)348 return Align(V & 0xFFFF);349 if ((V >> 16) > Index)350 return std::nullopt;351 }352 }353 }354 return std::nullopt;355}356 357Function *getMaybeBitcastedCallee(const CallBase *CB) {358 return dyn_cast<Function>(CB->getCalledOperand()->stripPointerCasts());359}360 361bool shouldEmitPTXNoReturn(const Value *V, const TargetMachine &TM) {362 const auto &ST =363 *static_cast<const NVPTXTargetMachine &>(TM).getSubtargetImpl();364 if (!ST.hasNoReturn())365 return false;366 367 assert((isa<Function>(V) || isa<CallInst>(V)) &&368 "Expect either a call instruction or a function");369 370 if (const CallInst *CallI = dyn_cast<CallInst>(V))371 return CallI->doesNotReturn() &&372 CallI->getFunctionType()->getReturnType()->isVoidTy();373 374 const Function *F = cast<Function>(V);375 return F->doesNotReturn() &&376 F->getFunctionType()->getReturnType()->isVoidTy() &&377 !isKernelFunction(*F);378}379 380} // namespace llvm381