840 lines · cpp
1//===- ACCImplicitData.cpp ------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This pass implements the OpenACC specification for "Variables with10// Implicitly Determined Data Attributes" (OpenACC 3.4 spec, section 2.6.2).11//12// Overview:13// ---------14// The pass automatically generates data clause operations for variables used15// within OpenACC compute constructs (parallel, kernels, serial) that do not16// already have explicit data clauses. The semantics follow these rules:17//18// 1. If there is a default(none) clause visible, no implicit data actions19// apply.20//21// 2. An aggregate variable (arrays, derived types, etc.) will be treated as:22// - In a present clause when default(present) is visible.23// - In a copy clause otherwise.24//25// 3. A scalar variable will be treated as if it appears in:26// - A copy clause if the compute construct is a kernels construct.27// - A firstprivate clause otherwise (parallel, serial).28//29// Requirements:30// -------------31// To use this pass in a pipeline, the following requirements must be met:32//33// 1. Type Interface Implementation: Variables from the dialect being used34// must implement one or both of the following MLIR interfaces:35// `acc::MappableType` and/or `acc::PointerLikeType`36//37// These interfaces provide the necessary methods for the pass to:38// - Determine variable type categories (scalar vs. aggregate)39// - Generate appropriate bounds information40// - Generate privatization recipes41//42// 2. Operation Interface Implementation: Operations that access partial43// entities or create views should implement the following MLIR44// interfaces: `acc::PartialEntityAccess` and/or45// `mlir::ViewLikeOpInterface`46//47// These interfaces are used for proper data clause ordering, ensuring48// that base entities are mapped before derived entities (e.g., a49// struct is mapped before its fields, an array is mapped before50// subarray views).51//52// 3. Analysis Registration (Optional): If custom behavior is needed for53// variable name extraction or alias analysis, the dialect should54// pre-register the `acc::OpenACCSupport` and `mlir::AliasAnalysis` analyses.55//56// If not registered, default behavior will be used.57//58// Implementation Details:59// -----------------------60// The pass performs the following operations:61//62// 1. Finds candidate variables which are live-in to the compute region and63// are not already in a data clause or private clause.64//65// 2. Generates both data "entry" and "exit" clause operations that match66// the intended action depending on variable type:67// - copy -> acc.copyin (entry) + acc.copyout (exit)68// - present -> acc.present (entry) + acc.delete (exit)69// - firstprivate -> acc.firstprivate (entry only, no exit)70//71// 3. Ensures that default clause is taken into consideration by looking72// through current construct and parent constructs to find the "visible73// default clause".74//75// 4. Fixes up SSA value links so that uses in the acc region reference the76// result of the newly created data clause operations.77//78// 5. When generating implicit data clause operations, it also adds variable79// name information and marks them with the implicit flag.80//81// 6. Recipes are generated by calling the appropriate entrypoints in the82// MappableType and PointerLikeType interfaces.83//84// 7. AliasAnalysis is used to determine if a variable is already covered by85// an existing data clause (e.g., an interior pointer covered by its parent).86//87// Examples:88// ---------89//90// Example 1: Scalar in parallel construct (implicit firstprivate)91//92// Before:93// func.func @test() {94// %scalar = memref.alloca() {acc.var_name = "x"} : memref<f32>95// acc.parallel {96// %val = memref.load %scalar[] : memref<f32>97// acc.yield98// }99// }100//101// After:102// func.func @test() {103// %scalar = memref.alloca() {acc.var_name = "x"} : memref<f32>104// %firstpriv = acc.firstprivate varPtr(%scalar : memref<f32>)105// -> memref<f32> {implicit = true, name = "x"}106// acc.parallel firstprivate(@recipe -> %firstpriv : memref<f32>) {107// %val = memref.load %firstpriv[] : memref<f32>108// acc.yield109// }110// }111//112// Example 2: Scalar in kernels construct (implicit copy)113//114// Before:115// func.func @test() {116// %scalar = memref.alloca() {acc.var_name = "n"} : memref<i32>117// acc.kernels {118// %val = memref.load %scalar[] : memref<i32>119// acc.terminator120// }121// }122//123// After:124// func.func @test() {125// %scalar = memref.alloca() {acc.var_name = "n"} : memref<i32>126// %copyin = acc.copyin varPtr(%scalar : memref<i32>) -> memref<i32>127// {dataClause = #acc<data_clause acc_copy>,128// implicit = true, name = "n"}129// acc.kernels dataOperands(%copyin : memref<i32>) {130// %val = memref.load %copyin[] : memref<i32>131// acc.terminator132// }133// acc.copyout accPtr(%copyin : memref<i32>)134// to varPtr(%scalar : memref<i32>)135// {dataClause = #acc<data_clause acc_copy>,136// implicit = true, name = "n"}137// }138//139// Example 3: Array (aggregate) in parallel (implicit copy)140//141// Before:142// func.func @test() {143// %array = memref.alloca() {acc.var_name = "arr"} : memref<100xf32>144// acc.parallel {145// %c0 = arith.constant 0 : index146// %val = memref.load %array[%c0] : memref<100xf32>147// acc.yield148// }149// }150//151// After:152// func.func @test() {153// %array = memref.alloca() {acc.var_name = "arr"} : memref<100xf32>154// %copyin = acc.copyin varPtr(%array : memref<100xf32>)155// -> memref<100xf32>156// {dataClause = #acc<data_clause acc_copy>,157// implicit = true, name = "arr"}158// acc.parallel dataOperands(%copyin : memref<100xf32>) {159// %c0 = arith.constant 0 : index160// %val = memref.load %copyin[%c0] : memref<100xf32>161// acc.yield162// }163// acc.copyout accPtr(%copyin : memref<100xf32>)164// to varPtr(%array : memref<100xf32>)165// {dataClause = #acc<data_clause acc_copy>,166// implicit = true, name = "arr"}167// }168//169// Example 4: Array with default(present)170//171// Before:172// func.func @test() {173// %array = memref.alloca() {acc.var_name = "arr"} : memref<100xf32>174// acc.parallel {175// %c0 = arith.constant 0 : index176// %val = memref.load %array[%c0] : memref<100xf32>177// acc.yield178// } attributes {defaultAttr = #acc<defaultvalue present>}179// }180//181// After:182// func.func @test() {183// %array = memref.alloca() {acc.var_name = "arr"} : memref<100xf32>184// %present = acc.present varPtr(%array : memref<100xf32>)185// -> memref<100xf32>186// {implicit = true, name = "arr"}187// acc.parallel dataOperands(%present : memref<100xf32>)188// attributes {defaultAttr = #acc<defaultvalue present>} {189// %c0 = arith.constant 0 : index190// %val = memref.load %present[%c0] : memref<100xf32>191// acc.yield192// }193// acc.delete accPtr(%present : memref<100xf32>)194// {dataClause = #acc<data_clause acc_present>,195// implicit = true, name = "arr"}196// }197//198//===----------------------------------------------------------------------===//199 200#include "mlir/Dialect/OpenACC/Transforms/Passes.h"201 202#include "mlir/Analysis/AliasAnalysis.h"203#include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h"204#include "mlir/Dialect/OpenACC/OpenACC.h"205#include "mlir/Dialect/OpenACC/OpenACCUtils.h"206#include "mlir/IR/Builders.h"207#include "mlir/IR/BuiltinOps.h"208#include "mlir/IR/Dominance.h"209#include "mlir/IR/Operation.h"210#include "mlir/IR/Value.h"211#include "mlir/Interfaces/FunctionInterfaces.h"212#include "mlir/Interfaces/ViewLikeInterface.h"213#include "mlir/Transforms/RegionUtils.h"214#include "llvm/ADT/STLExtras.h"215#include "llvm/ADT/SmallVector.h"216#include "llvm/ADT/TypeSwitch.h"217#include "llvm/Support/ErrorHandling.h"218#include <type_traits>219 220namespace mlir {221namespace acc {222#define GEN_PASS_DEF_ACCIMPLICITDATA223#include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc"224} // namespace acc225} // namespace mlir226 227#define DEBUG_TYPE "acc-implicit-data"228 229using namespace mlir;230 231namespace {232 233class ACCImplicitData : public acc::impl::ACCImplicitDataBase<ACCImplicitData> {234public:235 using acc::impl::ACCImplicitDataBase<ACCImplicitData>::ACCImplicitDataBase;236 237 void runOnOperation() override;238 239private:240 /// Collects all data clauses that dominate the compute construct.241 /// Needed to determine if a variable is already covered by an existing data242 /// clause.243 SmallVector<Value> getDominatingDataClauses(Operation *computeConstructOp);244 245 /// Looks through the `dominatingDataClauses` to find the original data clause246 /// op for an alias. Returns nullptr if no original data clause op is found.247 template <typename OpT>248 Operation *getOriginalDataClauseOpForAlias(249 Value var, OpBuilder &builder, OpT computeConstructOp,250 const SmallVector<Value> &dominatingDataClauses);251 252 /// Generates the appropriate `acc.copyin`, `acc.present`,`acc.firstprivate`,253 /// etc. data clause op for a candidate variable.254 template <typename OpT>255 Operation *generateDataClauseOpForCandidate(256 Value var, ModuleOp &module, OpBuilder &builder, OpT computeConstructOp,257 const SmallVector<Value> &dominatingDataClauses,258 const std::optional<acc::ClauseDefaultValue> &defaultClause);259 260 /// Generates the implicit data ops for a compute construct.261 template <typename OpT>262 void generateImplicitDataOps(263 ModuleOp &module, OpT computeConstructOp,264 std::optional<acc::ClauseDefaultValue> &defaultClause);265 266 /// Generates a private recipe for a variable.267 acc::PrivateRecipeOp generatePrivateRecipe(ModuleOp &module, Value var,268 Location loc, OpBuilder &builder,269 acc::OpenACCSupport &accSupport);270 271 /// Generates a firstprivate recipe for a variable.272 acc::FirstprivateRecipeOp273 generateFirstprivateRecipe(ModuleOp &module, Value var, Location loc,274 OpBuilder &builder,275 acc::OpenACCSupport &accSupport);276 277 /// Generates recipes for a list of variables.278 void generateRecipes(ModuleOp &module, OpBuilder &builder,279 Operation *computeConstructOp,280 const SmallVector<Value> &newOperands);281};282 283/// Determines if a variable is a candidate for implicit data mapping.284/// Returns true if the variable is a candidate, false otherwise.285static bool isCandidateForImplicitData(Value val, Region &accRegion) {286 // Ensure the variable is an allowed type for data clause.287 if (!acc::isPointerLikeType(val.getType()) &&288 !acc::isMappableType(val.getType()))289 return false;290 291 // If this is already coming from a data clause, we do not need to generate292 // another.293 if (isa_and_nonnull<ACC_DATA_ENTRY_OPS>(val.getDefiningOp()))294 return false;295 296 // If this is only used by private clauses, it is not a real live-in.297 if (acc::isOnlyUsedByPrivateClauses(val, accRegion))298 return false;299 300 return true;301}302 303SmallVector<Value>304ACCImplicitData::getDominatingDataClauses(Operation *computeConstructOp) {305 llvm::SmallSetVector<Value, 8> dominatingDataClauses;306 307 llvm::TypeSwitch<Operation *>(computeConstructOp)308 .Case<acc::ParallelOp, acc::KernelsOp, acc::SerialOp>([&](auto op) {309 for (auto dataClause : op.getDataClauseOperands()) {310 dominatingDataClauses.insert(dataClause);311 }312 })313 .Default([](Operation *) {});314 315 // Collect the data clauses from enclosing data constructs.316 Operation *currParentOp = computeConstructOp->getParentOp();317 while (currParentOp) {318 if (isa<acc::DataOp>(currParentOp)) {319 for (auto dataClause :320 dyn_cast<acc::DataOp>(currParentOp).getDataClauseOperands()) {321 dominatingDataClauses.insert(dataClause);322 }323 }324 currParentOp = currParentOp->getParentOp();325 }326 327 // Find the enclosing function/subroutine328 auto funcOp = computeConstructOp->getParentOfType<FunctionOpInterface>();329 if (!funcOp)330 return dominatingDataClauses.takeVector();331 332 // Walk the function to find `acc.declare_enter`/`acc.declare_exit` pairs that333 // dominate and post-dominate the compute construct and add their data334 // clauses to the list.335 auto &domInfo = this->getAnalysis<DominanceInfo>();336 auto &postDomInfo = this->getAnalysis<PostDominanceInfo>();337 funcOp->walk([&](acc::DeclareEnterOp declareEnterOp) {338 if (domInfo.dominates(declareEnterOp.getOperation(), computeConstructOp)) {339 // Collect all `acc.declare_exit` ops for this token.340 SmallVector<acc::DeclareExitOp> exits;341 for (auto *user : declareEnterOp.getToken().getUsers())342 if (auto declareExit = dyn_cast<acc::DeclareExitOp>(user))343 exits.push_back(declareExit);344 345 // Only add clauses if every `acc.declare_exit` op post-dominates the346 // compute construct.347 if (!exits.empty() && llvm::all_of(exits, [&](acc::DeclareExitOp exitOp) {348 return postDomInfo.postDominates(exitOp, computeConstructOp);349 })) {350 for (auto dataClause : declareEnterOp.getDataClauseOperands())351 dominatingDataClauses.insert(dataClause);352 }353 }354 });355 356 return dominatingDataClauses.takeVector();357}358 359template <typename OpT>360Operation *ACCImplicitData::getOriginalDataClauseOpForAlias(361 Value var, OpBuilder &builder, OpT computeConstructOp,362 const SmallVector<Value> &dominatingDataClauses) {363 auto &aliasAnalysis = this->getAnalysis<AliasAnalysis>();364 for (auto dataClause : dominatingDataClauses) {365 if (auto *dataClauseOp = dataClause.getDefiningOp()) {366 // Only accept clauses that guarantee that the alias is present.367 if (isa<acc::CopyinOp, acc::CreateOp, acc::PresentOp, acc::NoCreateOp,368 acc::DevicePtrOp>(dataClauseOp))369 if (aliasAnalysis.alias(acc::getVar(dataClauseOp), var).isMust())370 return dataClauseOp;371 }372 }373 return nullptr;374}375 376// Generates bounds for variables that have unknown dimensions377static void fillInBoundsForUnknownDimensions(Operation *dataClauseOp,378 OpBuilder &builder) {379 380 if (!acc::getBounds(dataClauseOp).empty())381 // If bounds are already present, do not overwrite them.382 return;383 384 // For types that have unknown dimensions, attempt to generate bounds by385 // relying on MappableType being able to extract it from the IR.386 auto var = acc::getVar(dataClauseOp);387 auto type = var.getType();388 if (auto mappableTy = dyn_cast<acc::MappableType>(type)) {389 if (mappableTy.hasUnknownDimensions()) {390 TypeSwitch<Operation *>(dataClauseOp)391 .Case<ACC_DATA_ENTRY_OPS, ACC_DATA_EXIT_OPS>([&](auto dataClauseOp) {392 if (std::is_same_v<decltype(dataClauseOp), acc::DevicePtrOp>)393 return;394 OpBuilder::InsertionGuard guard(builder);395 builder.setInsertionPoint(dataClauseOp);396 auto bounds = mappableTy.generateAccBounds(var, builder);397 if (!bounds.empty())398 dataClauseOp.getBoundsMutable().assign(bounds);399 });400 }401 }402}403 404acc::PrivateRecipeOp405ACCImplicitData::generatePrivateRecipe(ModuleOp &module, Value var,406 Location loc, OpBuilder &builder,407 acc::OpenACCSupport &accSupport) {408 auto type = var.getType();409 std::string recipeName =410 accSupport.getRecipeName(acc::RecipeKind::private_recipe, type, var);411 412 // Check if recipe already exists413 auto existingRecipe = module.lookupSymbol<acc::PrivateRecipeOp>(recipeName);414 if (existingRecipe)415 return existingRecipe;416 417 // Set insertion point to module body in a scoped way418 OpBuilder::InsertionGuard guard(builder);419 builder.setInsertionPointToStart(module.getBody());420 421 auto recipe =422 acc::PrivateRecipeOp::createAndPopulate(builder, loc, recipeName, type);423 if (!recipe.has_value())424 return accSupport.emitNYI(loc, "implicit private"), nullptr;425 return recipe.value();426}427 428acc::FirstprivateRecipeOp429ACCImplicitData::generateFirstprivateRecipe(ModuleOp &module, Value var,430 Location loc, OpBuilder &builder,431 acc::OpenACCSupport &accSupport) {432 auto type = var.getType();433 std::string recipeName =434 accSupport.getRecipeName(acc::RecipeKind::firstprivate_recipe, type, var);435 436 // Check if recipe already exists437 auto existingRecipe =438 module.lookupSymbol<acc::FirstprivateRecipeOp>(recipeName);439 if (existingRecipe)440 return existingRecipe;441 442 // Set insertion point to module body in a scoped way443 OpBuilder::InsertionGuard guard(builder);444 builder.setInsertionPointToStart(module.getBody());445 446 auto recipe = acc::FirstprivateRecipeOp::createAndPopulate(builder, loc,447 recipeName, type);448 if (!recipe.has_value())449 return accSupport.emitNYI(loc, "implicit firstprivate"), nullptr;450 return recipe.value();451}452 453void ACCImplicitData::generateRecipes(ModuleOp &module, OpBuilder &builder,454 Operation *computeConstructOp,455 const SmallVector<Value> &newOperands) {456 auto &accSupport = this->getAnalysis<acc::OpenACCSupport>();457 for (auto var : newOperands) {458 auto loc{var.getLoc()};459 if (auto privateOp = dyn_cast<acc::PrivateOp>(var.getDefiningOp())) {460 auto recipe = generatePrivateRecipe(461 module, acc::getVar(var.getDefiningOp()), loc, builder, accSupport);462 if (recipe)463 privateOp.setRecipeAttr(464 SymbolRefAttr::get(module->getContext(), recipe.getSymName()));465 } else if (auto firstprivateOp =466 dyn_cast<acc::FirstprivateOp>(var.getDefiningOp())) {467 auto recipe = generateFirstprivateRecipe(468 module, acc::getVar(var.getDefiningOp()), loc, builder, accSupport);469 if (recipe)470 firstprivateOp.setRecipeAttr(SymbolRefAttr::get(471 module->getContext(), recipe.getSymName().str()));472 } else {473 accSupport.emitNYI(var.getLoc(), "implicit reduction");474 }475 }476}477 478// Generates the data entry data op clause so that it adheres to OpenACC479// rules as follows (line numbers and specification from OpenACC 3.4):480// 1388 An aggregate variable will be treated as if it appears either:481// 1389 - In a present clause if there is a default(present) clause visible at482// the compute construct.483// 1391 - In a copy clause otherwise.484// 1392 A scalar variable will be treated as if it appears either:485// 1393 - In a copy clause if the compute construct is a kernels construct.486// 1394 - In a firstprivate clause otherwise.487template <typename OpT>488Operation *ACCImplicitData::generateDataClauseOpForCandidate(489 Value var, ModuleOp &module, OpBuilder &builder, OpT computeConstructOp,490 const SmallVector<Value> &dominatingDataClauses,491 const std::optional<acc::ClauseDefaultValue> &defaultClause) {492 auto &accSupport = this->getAnalysis<acc::OpenACCSupport>();493 acc::VariableTypeCategory typeCategory =494 acc::VariableTypeCategory::uncategorized;495 if (auto mappableTy = dyn_cast<acc::MappableType>(var.getType())) {496 typeCategory = mappableTy.getTypeCategory(var);497 } else if (auto pointerLikeTy =498 dyn_cast<acc::PointerLikeType>(var.getType())) {499 typeCategory = pointerLikeTy.getPointeeTypeCategory(500 cast<TypedValue<acc::PointerLikeType>>(var),501 pointerLikeTy.getElementType());502 }503 504 bool isScalar =505 acc::bitEnumContainsAny(typeCategory, acc::VariableTypeCategory::scalar);506 bool isAnyAggregate = acc::bitEnumContainsAny(507 typeCategory, acc::VariableTypeCategory::aggregate);508 Location loc = computeConstructOp->getLoc();509 510 Operation *op = nullptr;511 op = getOriginalDataClauseOpForAlias(var, builder, computeConstructOp,512 dominatingDataClauses);513 if (op) {514 if (isa<acc::NoCreateOp>(op))515 return acc::NoCreateOp::create(builder, loc, var,516 /*structured=*/true, /*implicit=*/true,517 accSupport.getVariableName(var),518 acc::getBounds(op));519 520 if (isa<acc::DevicePtrOp>(op))521 return acc::DevicePtrOp::create(builder, loc, var,522 /*structured=*/true, /*implicit=*/true,523 accSupport.getVariableName(var),524 acc::getBounds(op));525 526 // The original data clause op is a PresentOp, CopyinOp, or CreateOp,527 // hence guaranteed to be present.528 return acc::PresentOp::create(builder, loc, var,529 /*structured=*/true, /*implicit=*/true,530 accSupport.getVariableName(var),531 acc::getBounds(op));532 } else if (isScalar) {533 if (enableImplicitReductionCopy &&534 acc::isOnlyUsedByReductionClauses(var,535 computeConstructOp->getRegion(0))) {536 auto copyinOp =537 acc::CopyinOp::create(builder, loc, var,538 /*structured=*/true, /*implicit=*/true,539 accSupport.getVariableName(var));540 copyinOp.setDataClause(acc::DataClause::acc_reduction);541 return copyinOp.getOperation();542 }543 if constexpr (std::is_same_v<OpT, acc::KernelsOp> ||544 std::is_same_v<OpT, acc::KernelEnvironmentOp>) {545 // Scalars are implicit copyin in kernels construct.546 // We also do the same for acc.kernel_environment because semantics547 // of user variable mappings should be applied while ACC construct exists548 // and at this point we should only be dealing with unmapped variables549 // that were made live-in by the compiler.550 // TODO: This may be revisited.551 auto copyinOp =552 acc::CopyinOp::create(builder, loc, var,553 /*structured=*/true, /*implicit=*/true,554 accSupport.getVariableName(var));555 copyinOp.setDataClause(acc::DataClause::acc_copy);556 return copyinOp.getOperation();557 } else {558 // Scalars are implicit firstprivate in parallel and serial construct.559 return acc::FirstprivateOp::create(builder, loc, var,560 /*structured=*/true, /*implicit=*/true,561 accSupport.getVariableName(var));562 }563 } else if (isAnyAggregate) {564 Operation *newDataOp = nullptr;565 566 // When default(present) is true, the implicit behavior is present.567 if (defaultClause.has_value() &&568 defaultClause.value() == acc::ClauseDefaultValue::Present) {569 newDataOp = acc::PresentOp::create(builder, loc, var,570 /*structured=*/true, /*implicit=*/true,571 accSupport.getVariableName(var));572 newDataOp->setAttr(acc::getFromDefaultClauseAttrName(),573 builder.getUnitAttr());574 } else {575 auto copyinOp =576 acc::CopyinOp::create(builder, loc, var,577 /*structured=*/true, /*implicit=*/true,578 accSupport.getVariableName(var));579 copyinOp.setDataClause(acc::DataClause::acc_copy);580 newDataOp = copyinOp.getOperation();581 }582 583 return newDataOp;584 } else {585 // This is not a fatal error - for example when the element type is586 // pointer type (aka we have a pointer of pointer), it is potentially a587 // deep copy scenario which is not being handled here.588 // Other types need to be canonicalized. Thus just log unhandled cases.589 LLVM_DEBUG(llvm::dbgs()590 << "Unhandled case for implicit data mapping " << var << "\n");591 }592 return nullptr;593}594 595// Ensures that result values from the acc data clause ops are used inside the596// acc region. ie:597// acc.kernels {598// use %val599// }600// =>601// %dev = acc.dataop %val602// acc.kernels {603// use %dev604// }605static void legalizeValuesInRegion(Region &accRegion,606 SmallVector<Value> &newPrivateOperands,607 SmallVector<Value> &newDataClauseOperands) {608 for (Value dataClause :609 llvm::concat<Value>(newDataClauseOperands, newPrivateOperands)) {610 Value var = acc::getVar(dataClause.getDefiningOp());611 replaceAllUsesInRegionWith(var, dataClause, accRegion);612 }613}614 615// Adds the private operands to the compute construct operation.616template <typename OpT>617static void addNewPrivateOperands(OpT &accOp,618 const SmallVector<Value> &privateOperands) {619 if (privateOperands.empty())620 return;621 622 for (auto priv : privateOperands) {623 if (isa<acc::PrivateOp>(priv.getDefiningOp())) {624 accOp.getPrivateOperandsMutable().append(priv);625 } else if (isa<acc::FirstprivateOp>(priv.getDefiningOp())) {626 accOp.getFirstprivateOperandsMutable().append(priv);627 } else {628 llvm_unreachable("unhandled reduction operand");629 }630 }631}632 633static Operation *findDataExitOp(Operation *dataEntryOp) {634 auto res = acc::getAccVar(dataEntryOp);635 for (auto *user : res.getUsers())636 if (isa<ACC_DATA_EXIT_OPS>(user))637 return user;638 return nullptr;639}640 641// Generates matching data exit operation as described in the acc dialect642// for how data clauses are decomposed:643// https://mlir.llvm.org/docs/Dialects/OpenACCDialect/#operation-categories644// Key ones used here:645// * acc {construct} copy -> acc.copyin (before region) + acc.copyout (after646// region)647// * acc {construct} present -> acc.present (before region) + acc.delete648// (after region)649static void650generateDataExitOperations(OpBuilder &builder, Operation *accOp,651 const SmallVector<Value> &newDataClauseOperands,652 const SmallVector<Value> &sortedDataClauseOperands) {653 builder.setInsertionPointAfter(accOp);654 Value lastDataClause = nullptr;655 for (auto dataEntry : llvm::reverse(sortedDataClauseOperands)) {656 if (llvm::find(newDataClauseOperands, dataEntry) ==657 newDataClauseOperands.end()) {658 // If this is not a new data clause operand, we should not generate an659 // exit operation for it.660 lastDataClause = dataEntry;661 continue;662 }663 if (lastDataClause)664 if (auto *dataExitOp = findDataExitOp(lastDataClause.getDefiningOp()))665 builder.setInsertionPointAfter(dataExitOp);666 Operation *dataEntryOp = dataEntry.getDefiningOp();667 if (isa<acc::CopyinOp>(dataEntryOp)) {668 auto copyoutOp = acc::CopyoutOp::create(669 builder, dataEntryOp->getLoc(), dataEntry, acc::getVar(dataEntryOp),670 /*structured=*/true, /*implicit=*/true,671 acc::getVarName(dataEntryOp).value(), acc::getBounds(dataEntryOp));672 copyoutOp.setDataClause(acc::DataClause::acc_copy);673 } else if (isa<acc::PresentOp, acc::NoCreateOp>(dataEntryOp)) {674 auto deleteOp = acc::DeleteOp::create(675 builder, dataEntryOp->getLoc(), dataEntry,676 /*structured=*/true, /*implicit=*/true,677 acc::getVarName(dataEntryOp).value(), acc::getBounds(dataEntryOp));678 deleteOp.setDataClause(acc::getDataClause(dataEntryOp).value());679 } else if (isa<acc::DevicePtrOp>(dataEntryOp)) {680 // Do nothing.681 } else {682 llvm_unreachable("unhandled data exit");683 }684 lastDataClause = dataEntry;685 }686}687 688/// Returns all base references of a value in order.689/// So for example, if we have a reference to a struct field like690/// s.f1.f2.f3, this will return <s, s.f1, s.f1.f2, s.f1.f2.f3>.691/// Any intermediate casts/view-like operations are included in the692/// chain as well.693static SmallVector<Value> getBaseRefsChain(Value val) {694 SmallVector<Value> baseRefs;695 baseRefs.push_back(val);696 while (true) {697 Value prevVal = val;698 699 val = acc::getBaseEntity(val);700 if (val != baseRefs.front())701 baseRefs.insert(baseRefs.begin(), val);702 703 // If this is a view-like operation, it is effectively another704 // view of the same entity so we should add it to the chain also.705 if (auto viewLikeOp = val.getDefiningOp<ViewLikeOpInterface>()) {706 val = viewLikeOp.getViewSource();707 baseRefs.insert(baseRefs.begin(), val);708 }709 710 // Continue loop if we made any progress711 if (val == prevVal)712 break;713 }714 715 return baseRefs;716}717 718static void insertInSortedOrder(SmallVector<Value> &sortedDataClauseOperands,719 Operation *newClause) {720 auto *insertPos =721 std::find_if(sortedDataClauseOperands.begin(),722 sortedDataClauseOperands.end(), [&](Value dataClauseVal) {723 // Get the base refs for the current clause we are looking724 // at.725 auto var = acc::getVar(dataClauseVal.getDefiningOp());726 auto baseRefs = getBaseRefsChain(var);727 728 // If the newClause is of a base ref of an existing clause,729 // we should insert it right before the current clause.730 // Thus return true to stop iteration when this is the731 // case.732 return std::find(baseRefs.begin(), baseRefs.end(),733 acc::getVar(newClause)) != baseRefs.end();734 });735 736 if (insertPos != sortedDataClauseOperands.end()) {737 newClause->moveBefore(insertPos->getDefiningOp());738 sortedDataClauseOperands.insert(insertPos, acc::getAccVar(newClause));739 } else {740 sortedDataClauseOperands.push_back(acc::getAccVar(newClause));741 }742}743 744template <typename OpT>745void ACCImplicitData::generateImplicitDataOps(746 ModuleOp &module, OpT computeConstructOp,747 std::optional<acc::ClauseDefaultValue> &defaultClause) {748 // Implicit data attributes are only applied if "[t]here is no default(none)749 // clause visible at the compute construct."750 if (defaultClause.has_value() &&751 defaultClause.value() == acc::ClauseDefaultValue::None)752 return;753 assert(!defaultClause.has_value() ||754 defaultClause.value() == acc::ClauseDefaultValue::Present);755 756 // 1) Collect live-in values.757 Region &accRegion = computeConstructOp->getRegion(0);758 SetVector<Value> liveInValues;759 getUsedValuesDefinedAbove(accRegion, liveInValues);760 761 // 2) Run the filtering to find relevant pointers that need copied.762 auto isCandidate{[&](Value val) -> bool {763 return isCandidateForImplicitData(val, accRegion);764 }};765 auto candidateVars(766 llvm::to_vector(llvm::make_filter_range(liveInValues, isCandidate)));767 if (candidateVars.empty())768 return;769 770 // 3) Generate data clauses for the variables.771 SmallVector<Value> newPrivateOperands;772 SmallVector<Value> newDataClauseOperands;773 OpBuilder builder(computeConstructOp);774 if (!candidateVars.empty()) {775 LLVM_DEBUG(llvm::dbgs() << "== Generating clauses for ==\n"776 << computeConstructOp << "\n");777 }778 auto dominatingDataClauses = getDominatingDataClauses(computeConstructOp);779 for (auto var : candidateVars) {780 auto newDataClauseOp = generateDataClauseOpForCandidate(781 var, module, builder, computeConstructOp, dominatingDataClauses,782 defaultClause);783 fillInBoundsForUnknownDimensions(newDataClauseOp, builder);784 LLVM_DEBUG(llvm::dbgs() << "Generated data clause for " << var << ":\n"785 << "\t" << *newDataClauseOp << "\n");786 if (isa_and_nonnull<acc::PrivateOp, acc::FirstprivateOp, acc::ReductionOp>(787 newDataClauseOp)) {788 newPrivateOperands.push_back(acc::getAccVar(newDataClauseOp));789 } else if (isa_and_nonnull<ACC_DATA_CLAUSE_OPS>(newDataClauseOp)) {790 newDataClauseOperands.push_back(acc::getAccVar(newDataClauseOp));791 dominatingDataClauses.push_back(acc::getAccVar(newDataClauseOp));792 }793 }794 795 // 4) Legalize values in region (aka the uses in the region are the result796 // of the data clause ops)797 legalizeValuesInRegion(accRegion, newPrivateOperands, newDataClauseOperands);798 799 // 5) Generate private recipes which are required for properly attaching800 // private operands.801 if constexpr (!std::is_same_v<OpT, acc::KernelsOp> &&802 !std::is_same_v<OpT, acc::KernelEnvironmentOp>)803 generateRecipes(module, builder, computeConstructOp, newPrivateOperands);804 805 // 6) Figure out insertion order for the new data clause operands.806 SmallVector<Value> sortedDataClauseOperands(807 computeConstructOp.getDataClauseOperands());808 for (auto newClause : newDataClauseOperands)809 insertInSortedOrder(sortedDataClauseOperands, newClause.getDefiningOp());810 811 // 7) Generate the data exit operations.812 generateDataExitOperations(builder, computeConstructOp, newDataClauseOperands,813 sortedDataClauseOperands);814 // 8) Add all of the new operands to the compute construct op.815 if constexpr (!std::is_same_v<OpT, acc::KernelsOp> &&816 !std::is_same_v<OpT, acc::KernelEnvironmentOp>)817 addNewPrivateOperands(computeConstructOp, newPrivateOperands);818 computeConstructOp.getDataClauseOperandsMutable().assign(819 sortedDataClauseOperands);820}821 822void ACCImplicitData::runOnOperation() {823 ModuleOp module = this->getOperation();824 module.walk([&](Operation *op) {825 if (isa<ACC_COMPUTE_CONSTRUCT_OPS, acc::KernelEnvironmentOp>(op)) {826 assert(op->getNumRegions() == 1 && "must have 1 region");827 828 auto defaultClause = acc::getDefaultAttr(op);829 llvm::TypeSwitch<Operation *, void>(op)830 .Case<ACC_COMPUTE_CONSTRUCT_OPS, acc::KernelEnvironmentOp>(831 [&](auto op) {832 generateImplicitDataOps(module, op, defaultClause);833 })834 .Default([&](Operation *) {});835 }836 });837}838 839} // namespace840