430 lines · cpp
1//===- ACCImplicitDeclare.cpp ---------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This pass applies implicit `acc declare` actions to global variables10// referenced in OpenACC compute regions and routine functions.11//12// Overview:13// ---------14// Global references in an acc regions (for globals not marked with `acc15// declare` by the user) can be handled in one of two ways:16// - Mapped through data clauses17// - Implicitly marked as `acc declare` (this pass)18//19// Thus, the OpenACC specification focuses solely on implicit data mapping rules20// whose implementation is captured in `ACCImplicitData` pass.21//22// However, it is both advantageous and required for certain cases to23// use implicit `acc declare` instead:24// - Any functions that are implicitly marked as `acc routine` through25// `ACCImplicitRoutine` may reference globals. Since data mapping26// is only possible for compute regions, such globals can only be27// made available on device through `acc declare`.28// - Compiler can generate and use globals for cases needed in IR29// representation such as type descriptors or various names needed for30// runtime calls and error reporting - such cases often are introduced31// after a frontend semantic checking is done since it is related to32// implementation detail. Thus, such compiler generated globals would33// not have been visible for a user to mark with `acc declare`.34// - Constant globals such as filename strings or data initialization values35// are values that do not get mutated but are still needed for appropriate36// runtime execution. If a kernel is launched 1000 times, it is not a37// good idea to map such a global 1000 times. Therefore, such globals38// benefit from being marked with `acc declare`.39//40// This pass automatically41// marks global variables with the `acc.declare` attribute when they are42// referenced in OpenACC compute constructs or routine functions and meet43// the criteria noted above, ensuring44// they are properly handled for device execution.45//46// The pass performs two main optimizations:47//48// 1. Hoisting: For non-constant globals referenced in compute regions, the49// pass hoists the address-of operation out of the region when possible,50// allowing them to be implicitly mapped through normal data clause51// mechanisms rather than requiring declare marking.52//53// 2. Declaration: For globals that must be available on the device (constants,54// globals in routines, globals in recipe operations), the pass adds the55// `acc.declare` attribute with the copyin data clause.56//57// Requirements:58// -------------59// To use this pass in a pipeline, the following requirements must be met:60//61// 1. Operation Interface Implementation: Operations that compute addresses62// of global variables must implement the `acc::AddressOfGlobalOpInterface`63// and those that represent globals must implement the64// `acc::GlobalOpInterface`. Additionally, any operations that indirectly65// access globals must implement the `acc::IndirectGlobalAccessOpInterface`.66//67// 2. Analysis Registration (Optional): If custom behavior is needed for68// determining if a symbol use is valid within GPU regions, the dialect69// should pre-register the `acc::OpenACCSupport` analysis.70//71// Examples:72// ---------73//74// Example 1: Non-constant global in compute region (hoisted)75//76// Before:77// memref.global @g_scalar : memref<f32> = dense<0.0>78// func.func @test() {79// acc.serial {80// %addr = memref.get_global @g_scalar : memref<f32>81// %val = memref.load %addr[] : memref<f32>82// acc.yield83// }84// }85//86// After:87// memref.global @g_scalar : memref<f32> = dense<0.0>88// func.func @test() {89// %addr = memref.get_global @g_scalar : memref<f32>90// acc.serial {91// %val = memref.load %addr[] : memref<f32>92// acc.yield93// }94// }95//96// Example 2: Constant global in compute region (declared)97//98// Before:99// memref.global constant @g_const : memref<f32> = dense<1.0>100// func.func @test() {101// acc.serial {102// %addr = memref.get_global @g_const : memref<f32>103// %val = memref.load %addr[] : memref<f32>104// acc.yield105// }106// }107//108// After:109// memref.global constant @g_const : memref<f32> = dense<1.0>110// {acc.declare = #acc.declare<dataClause = acc_copyin>}111// func.func @test() {112// acc.serial {113// %addr = memref.get_global @g_const : memref<f32>114// %val = memref.load %addr[] : memref<f32>115// acc.yield116// }117// }118//119// Example 3: Global in acc routine (declared)120//121// Before:122// memref.global @g_data : memref<f32> = dense<0.0>123// acc.routine @routine_0 func(@device_func)124// func.func @device_func() attributes {acc.routine_info = ...} {125// %addr = memref.get_global @g_data : memref<f32>126// %val = memref.load %addr[] : memref<f32>127// }128//129// After:130// memref.global @g_data : memref<f32> = dense<0.0>131// {acc.declare = #acc.declare<dataClause = acc_copyin>}132// acc.routine @routine_0 func(@device_func)133// func.func @device_func() attributes {acc.routine_info = ...} {134// %addr = memref.get_global @g_data : memref<f32>135// %val = memref.load %addr[] : memref<f32>136// }137//138// Example 4: Global in private recipe (declared if recipe is used)139//140// Before:141// memref.global @g_init : memref<f32> = dense<0.0>142// acc.private.recipe @priv_recipe : memref<f32> init {143// ^bb0(%arg0: memref<f32>):144// %alloc = memref.alloc() : memref<f32>145// %global = memref.get_global @g_init : memref<f32>146// %val = memref.load %global[] : memref<f32>147// memref.store %val, %alloc[] : memref<f32>148// acc.yield %alloc : memref<f32>149// } destroy { ... }150// func.func @test() {151// %var = memref.alloc() : memref<f32>152// %priv = acc.private varPtr(%var : memref<f32>)153// recipe(@priv_recipe) -> memref<f32>154// acc.parallel private(%priv : memref<f32>) { ... }155// }156//157// After:158// memref.global @g_init : memref<f32> = dense<0.0>159// {acc.declare = #acc.declare<dataClause = acc_copyin>}160// acc.private.recipe @priv_recipe : memref<f32> init {161// ^bb0(%arg0: memref<f32>):162// %alloc = memref.alloc() : memref<f32>163// %global = memref.get_global @g_init : memref<f32>164// %val = memref.load %global[] : memref<f32>165// memref.store %val, %alloc[] : memref<f32>166// acc.yield %alloc : memref<f32>167// } destroy { ... }168// func.func @test() {169// %var = memref.alloc() : memref<f32>170// %priv = acc.private varPtr(%var : memref<f32>)171// recipe(@priv_recipe) -> memref<f32>172// acc.parallel private(%priv : memref<f32>) { ... }173// }174//175//===----------------------------------------------------------------------===//176 177#include "mlir/Dialect/OpenACC/Transforms/Passes.h"178 179#include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h"180#include "mlir/Dialect/OpenACC/OpenACC.h"181#include "mlir/IR/Builders.h"182#include "mlir/IR/BuiltinAttributes.h"183#include "mlir/IR/BuiltinOps.h"184#include "mlir/IR/Operation.h"185#include "mlir/IR/Value.h"186#include "mlir/Interfaces/FunctionInterfaces.h"187#include "llvm/ADT/SmallVector.h"188#include "llvm/ADT/TypeSwitch.h"189 190namespace mlir {191namespace acc {192#define GEN_PASS_DEF_ACCIMPLICITDECLARE193#include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc"194} // namespace acc195} // namespace mlir196 197#define DEBUG_TYPE "acc-implicit-declare"198 199using namespace mlir;200 201namespace {202 203using GlobalOpSetT = llvm::SmallSetVector<Operation *, 16>;204 205/// Checks whether a use of the requested `globalOp` should be considered206/// for hoisting out of acc region due to avoid `acc declare`ing something207/// that instead should be implicitly mapped.208static bool isGlobalUseCandidateForHoisting(Operation *globalOp,209 Operation *user,210 SymbolRefAttr symbol,211 acc::OpenACCSupport &accSupport) {212 // This symbol is valid in GPU region. This means semantics213 // would change if moved to host - therefore it is not a candidate.214 if (accSupport.isValidSymbolUse(user, symbol))215 return false;216 217 bool isConstant = false;218 bool isFunction = false;219 220 if (auto globalVarOp = dyn_cast<acc::GlobalVariableOpInterface>(globalOp))221 isConstant = globalVarOp.isConstant();222 223 if (isa<FunctionOpInterface>(globalOp))224 isFunction = true;225 226 // Constants should be kept in device code to ensure they are duplicated.227 // Function references should be kept in device code to ensure their device228 // addresses are computed. Everything else should be hoisted since we already229 // proved they are not valid symbols in GPU region.230 return !isConstant && !isFunction;231}232 233/// Checks whether it is valid to use acc.declare marking on the global.234bool isValidForAccDeclare(Operation *globalOp) {235 // For functions - we use acc.routine marking instead.236 return !isa<FunctionOpInterface>(globalOp);237}238 239/// Checks whether a recipe operation has meaningful use of its symbol that240/// justifies processing its regions for global references. Returns false if:241/// 1. The recipe has no symbol uses at all, or242/// 2. The only symbol use is the recipe's own symbol definition243template <typename RecipeOpT>244static bool hasRelevantRecipeUse(RecipeOpT &recipeOp, ModuleOp &mod) {245 std::optional<SymbolTable::UseRange> symbolUses = recipeOp.getSymbolUses(mod);246 247 // No recipe symbol uses.248 if (!symbolUses.has_value() || symbolUses->empty())249 return false;250 251 // If more than one use, assume it's used.252 auto begin = symbolUses->begin();253 auto end = symbolUses->end();254 if (begin != end && std::next(begin) != end)255 return true;256 257 // If single use, check if the use is the recipe itself.258 const SymbolTable::SymbolUse &use = *symbolUses->begin();259 return use.getUser() != recipeOp.getOperation();260}261 262// Hoists addr_of operations for non-constant globals out of OpenACC regions.263// This way - they are implicitly mapped instead of being considered for264// implicit declare.265template <typename AccConstructT>266static void hoistNonConstantDirectUses(AccConstructT accOp,267 acc::OpenACCSupport &accSupport) {268 accOp.walk([&](acc::AddressOfGlobalOpInterface addrOfOp) {269 SymbolRefAttr symRef = addrOfOp.getSymbol();270 if (symRef) {271 Operation *globalOp =272 SymbolTable::lookupNearestSymbolFrom(addrOfOp, symRef);273 if (isGlobalUseCandidateForHoisting(globalOp, addrOfOp, symRef,274 accSupport)) {275 addrOfOp->moveBefore(accOp);276 LLVM_DEBUG(277 llvm::dbgs() << "Hoisted:\n\t" << addrOfOp << "\n\tfrom:\n\t";278 accOp->print(llvm::dbgs(),279 OpPrintingFlags{}.skipRegions().enableDebugInfo());280 llvm::dbgs() << "\n");281 }282 }283 });284}285 286// Collects the globals referenced in a device region287static void collectGlobalsFromDeviceRegion(Region ®ion,288 GlobalOpSetT &globals,289 acc::OpenACCSupport &accSupport,290 SymbolTable &symTab) {291 region.walk([&](Operation *op) {292 // 1) Only consider relevant operations which use symbols293 auto addrOfOp = dyn_cast<acc::AddressOfGlobalOpInterface>(op);294 if (addrOfOp) {295 SymbolRefAttr symRef = addrOfOp.getSymbol();296 // 2) Found an operation which uses the symbol. Next determine if it297 // is a candidate for `acc declare`. Some of the criteria considered298 // is whether this symbol is not already a device one (either because299 // acc declare is already used or this is a CUF global).300 Operation *globalOp = nullptr;301 bool isCandidate = !accSupport.isValidSymbolUse(op, symRef, &globalOp);302 // 3) Add the candidate to the set of globals to be `acc declare`d.303 if (isCandidate && globalOp && isValidForAccDeclare(globalOp))304 globals.insert(globalOp);305 } else if (auto indirectAccessOp =306 dyn_cast<acc::IndirectGlobalAccessOpInterface>(op)) {307 // Process operations that indirectly access globals308 llvm::SmallVector<SymbolRefAttr> symbols;309 indirectAccessOp.getReferencedSymbols(symbols, &symTab);310 for (SymbolRefAttr symRef : symbols)311 if (Operation *globalOp = symTab.lookup(symRef.getLeafReference()))312 if (isValidForAccDeclare(globalOp))313 globals.insert(globalOp);314 }315 });316}317 318// Adds the declare attribute to the operation `op`.319static void addDeclareAttr(MLIRContext *context, Operation *op,320 acc::DataClause clause) {321 op->setAttr(acc::getDeclareAttrName(),322 acc::DeclareAttr::get(context,323 acc::DataClauseAttr::get(context, clause)));324}325 326// This pass applies implicit declare actions for globals referenced in327// OpenACC compute and routine regions.328class ACCImplicitDeclare329 : public acc::impl::ACCImplicitDeclareBase<ACCImplicitDeclare> {330public:331 using ACCImplicitDeclareBase<ACCImplicitDeclare>::ACCImplicitDeclareBase;332 333 void runOnOperation() override {334 ModuleOp mod = getOperation();335 MLIRContext *context = &getContext();336 acc::OpenACCSupport &accSupport = getAnalysis<acc::OpenACCSupport>();337 338 // 1) Start off by hoisting any AddressOf operations out of acc region339 // for any cases we do not want to `acc declare`. This is because we can340 // rely on implicit data mapping in majority of cases without uselessly341 // polluting the device globals.342 mod.walk([&](Operation *op) {343 TypeSwitch<Operation *, void>(op)344 .Case<ACC_COMPUTE_CONSTRUCT_OPS, acc::KernelEnvironmentOp>(345 [&](auto accOp) {346 hoistNonConstantDirectUses(accOp, accSupport);347 });348 });349 350 // 2) Collect global symbols which need to be `acc declare`d. Do it for351 // compute regions, acc routine, and existing globals with the declare352 // attribute.353 SymbolTable symTab(mod);354 GlobalOpSetT globalsToAccDeclare;355 mod.walk([&](Operation *op) {356 TypeSwitch<Operation *, void>(op)357 .Case<ACC_COMPUTE_CONSTRUCT_OPS, acc::KernelEnvironmentOp>(358 [&](auto accOp) {359 collectGlobalsFromDeviceRegion(360 accOp.getRegion(), globalsToAccDeclare, accSupport, symTab);361 })362 .Case<FunctionOpInterface>([&](auto func) {363 if (acc::isAccRoutineOp(func) && !func.isExternal())364 collectGlobalsFromDeviceRegion(func.getFunctionBody(),365 globalsToAccDeclare, accSupport,366 symTab);367 })368 .Case<acc::GlobalVariableOpInterface>([&](auto globalVarOp) {369 if (globalVarOp->getAttr(acc::getDeclareAttrName()))370 if (Region *initRegion = globalVarOp.getInitRegion())371 collectGlobalsFromDeviceRegion(*initRegion, globalsToAccDeclare,372 accSupport, symTab);373 })374 .Case<acc::PrivateRecipeOp>([&](auto privateRecipe) {375 if (hasRelevantRecipeUse(privateRecipe, mod)) {376 collectGlobalsFromDeviceRegion(privateRecipe.getInitRegion(),377 globalsToAccDeclare, accSupport,378 symTab);379 collectGlobalsFromDeviceRegion(privateRecipe.getDestroyRegion(),380 globalsToAccDeclare, accSupport,381 symTab);382 }383 })384 .Case<acc::FirstprivateRecipeOp>([&](auto firstprivateRecipe) {385 if (hasRelevantRecipeUse(firstprivateRecipe, mod)) {386 collectGlobalsFromDeviceRegion(firstprivateRecipe.getInitRegion(),387 globalsToAccDeclare, accSupport,388 symTab);389 collectGlobalsFromDeviceRegion(390 firstprivateRecipe.getDestroyRegion(), globalsToAccDeclare,391 accSupport, symTab);392 collectGlobalsFromDeviceRegion(firstprivateRecipe.getCopyRegion(),393 globalsToAccDeclare, accSupport,394 symTab);395 }396 })397 .Case<acc::ReductionRecipeOp>([&](auto reductionRecipe) {398 if (hasRelevantRecipeUse(reductionRecipe, mod)) {399 collectGlobalsFromDeviceRegion(reductionRecipe.getInitRegion(),400 globalsToAccDeclare, accSupport,401 symTab);402 collectGlobalsFromDeviceRegion(403 reductionRecipe.getCombinerRegion(), globalsToAccDeclare,404 accSupport, symTab);405 }406 });407 });408 409 // 3) Finally, generate the appropriate declare actions needed to ensure410 // this is considered for device global.411 for (Operation *globalOp : globalsToAccDeclare) {412 LLVM_DEBUG(413 llvm::dbgs() << "Global is being `acc declare copyin`d: ";414 globalOp->print(llvm::dbgs(),415 OpPrintingFlags{}.skipRegions().enableDebugInfo());416 llvm::dbgs() << "\n");417 418 // Mark it as declare copyin.419 addDeclareAttr(context, globalOp, acc::DataClause::acc_copyin);420 421 // TODO: May need to create the global constructor which does the mapping422 // action. It is not yet clear if this is needed yet (since the globals423 // might just end up in the GPU image without requiring mapping via424 // runtime).425 }426 }427};428 429} // namespace430