brintos / llvm-project-archived public Read only

1//===- OpenACC.cpp - OpenACC MLIR Operations ------------------------------===//2//3// Part of the MLIR Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7// =============================================================================8 9#include "mlir/Dialect/OpenACC/OpenACC.h"10#include "mlir/Dialect/Arith/IR/Arith.h"11#include "mlir/Dialect/LLVMIR/LLVMDialect.h"12#include "mlir/Dialect/LLVMIR/LLVMTypes.h"13#include "mlir/Dialect/MemRef/IR/MemRef.h"14#include "mlir/IR/Builders.h"15#include "mlir/IR/BuiltinAttributes.h"16#include "mlir/IR/BuiltinTypes.h"17#include "mlir/IR/DialectImplementation.h"18#include "mlir/IR/Matchers.h"19#include "mlir/IR/OpImplementation.h"20#include "mlir/IR/SymbolTable.h"21#include "mlir/Support/LLVM.h"22#include "mlir/Transforms/DialectConversion.h"23#include "llvm/ADT/SmallSet.h"24#include "llvm/ADT/TypeSwitch.h"25#include "llvm/Support/LogicalResult.h"26#include <variant>27 28using namespace mlir;29using namespace acc;30 31#include "mlir/Dialect/OpenACC/OpenACCOpsDialect.cpp.inc"32#include "mlir/Dialect/OpenACC/OpenACCOpsEnums.cpp.inc"33#include "mlir/Dialect/OpenACC/OpenACCOpsInterfaces.cpp.inc"34#include "mlir/Dialect/OpenACC/OpenACCTypeInterfaces.cpp.inc"35#include "mlir/Dialect/OpenACCMPCommon/Interfaces/OpenACCMPOpsInterfaces.cpp.inc"36 37namespace {38 39static bool isScalarLikeType(Type type) {40  return type.isIntOrIndexOrFloat() || isa<ComplexType>(type);41}42 43/// Helper function to attach the `VarName` attribute to an operation44/// if a variable name is provided.45static void attachVarNameAttr(Operation *op, OpBuilder &builder,46                              StringRef varName) {47  if (!varName.empty()) {48    auto varNameAttr = acc::VarNameAttr::get(builder.getContext(), varName);49    op->setAttr(acc::getVarNameAttrName(), varNameAttr);50  }51}52 53template <typename T>54struct MemRefPointerLikeModel55    : public PointerLikeType::ExternalModel<MemRefPointerLikeModel<T>, T> {56  Type getElementType(Type pointer) const {57    return cast<T>(pointer).getElementType();58  }59 60  mlir::acc::VariableTypeCategory61  getPointeeTypeCategory(Type pointer, TypedValue<PointerLikeType> varPtr,62                         Type varType) const {63    if (auto mappableTy = dyn_cast<MappableType>(varType)) {64      return mappableTy.getTypeCategory(varPtr);65    }66    auto memrefTy = cast<T>(pointer);67    if (!memrefTy.hasRank()) {68      // This memref is unranked - aka it could have any rank, including a69      // rank of 0 which could mean scalar. For now, return uncategorized.70      return mlir::acc::VariableTypeCategory::uncategorized;71    }72 73    if (memrefTy.getRank() == 0) {74      if (isScalarLikeType(memrefTy.getElementType())) {75        return mlir::acc::VariableTypeCategory::scalar;76      }77      // Zero-rank non-scalar - need further analysis to determine the type78      // category. For now, return uncategorized.79      return mlir::acc::VariableTypeCategory::uncategorized;80    }81 82    // It has a rank - must be an array.83    assert(memrefTy.getRank() > 0 && "rank expected to be positive");84    return mlir::acc::VariableTypeCategory::array;85  }86 87  mlir::Value genAllocate(Type pointer, OpBuilder &builder, Location loc,88                          StringRef varName, Type varType, Value originalVar,89                          bool &needsFree) const {90    auto memrefTy = cast<MemRefType>(pointer);91 92    // Check if this is a static memref (all dimensions are known) - if yes93    // then we can generate an alloca operation.94    if (memrefTy.hasStaticShape()) {95      needsFree = false; // alloca doesn't need deallocation96      auto allocaOp = memref::AllocaOp::create(builder, loc, memrefTy);97      attachVarNameAttr(allocaOp, builder, varName);98      return allocaOp.getResult();99    }100 101    // For dynamic memrefs, extract sizes from the original variable if102    // provided. Otherwise they cannot be handled.103    if (originalVar && originalVar.getType() == memrefTy &&104        memrefTy.hasRank()) {105      SmallVector<Value> dynamicSizes;106      for (int64_t i = 0; i < memrefTy.getRank(); ++i) {107        if (memrefTy.isDynamicDim(i)) {108          // Extract the size of dimension i from the original variable109          auto indexValue = arith::ConstantIndexOp::create(builder, loc, i);110          auto dimSize =111              memref::DimOp::create(builder, loc, originalVar, indexValue);112          dynamicSizes.push_back(dimSize);113        }114        // Note: We only add dynamic sizes to the dynamicSizes array115        // Static dimensions are handled automatically by AllocOp116      }117      needsFree = true; // alloc needs deallocation118      auto allocOp =119          memref::AllocOp::create(builder, loc, memrefTy, dynamicSizes);120      attachVarNameAttr(allocOp, builder, varName);121      return allocOp.getResult();122    }123 124    // TODO: Unranked not yet supported.125    return {};126  }127 128  bool genFree(Type pointer, OpBuilder &builder, Location loc,129               TypedValue<PointerLikeType> varToFree, Value allocRes,130               Type varType) const {131    if (auto memrefValue = dyn_cast<TypedValue<MemRefType>>(varToFree)) {132      // Use allocRes if provided to determine the allocation type133      Value valueToInspect = allocRes ? allocRes : memrefValue;134 135      // Walk through casts to find the original allocation136      Value currentValue = valueToInspect;137      Operation *originalAlloc = nullptr;138 139      // Follow the chain of operations to find the original allocation140      // even if a casted result is provided.141      while (currentValue) {142        if (auto *definingOp = currentValue.getDefiningOp()) {143          // Check if this is an allocation operation144          if (isa<memref::AllocOp, memref::AllocaOp>(definingOp)) {145            originalAlloc = definingOp;146            break;147          }148 149          // Check if this is a cast operation we can look through150          if (auto castOp = dyn_cast<memref::CastOp>(definingOp)) {151            currentValue = castOp.getSource();152            continue;153          }154 155          // Check for other cast-like operations156          if (auto reinterpretCastOp =157                  dyn_cast<memref::ReinterpretCastOp>(definingOp)) {158            currentValue = reinterpretCastOp.getSource();159            continue;160          }161 162          // If we can't look through this operation, stop163          break;164        }165        // This is a block argument or similar - can't trace further.166        break;167      }168 169      if (originalAlloc) {170        if (isa<memref::AllocaOp>(originalAlloc)) {171          // This is an alloca - no dealloc needed, but return true (success)172          return true;173        }174        if (isa<memref::AllocOp>(originalAlloc)) {175          // This is an alloc - generate dealloc on varToFree176          memref::DeallocOp::create(builder, loc, memrefValue);177          return true;178        }179      }180    }181 182    return false;183  }184 185  bool genCopy(Type pointer, OpBuilder &builder, Location loc,186               TypedValue<PointerLikeType> destination,187               TypedValue<PointerLikeType> source, Type varType) const {188    // Generate a copy operation between two memrefs189    auto destMemref = dyn_cast_if_present<TypedValue<MemRefType>>(destination);190    auto srcMemref = dyn_cast_if_present<TypedValue<MemRefType>>(source);191 192    // As per memref documentation, source and destination must have same193    // element type and shape in order to be compatible. We do not want to fail194    // with an IR verification error - thus check that before generating the195    // copy operation.196    if (destMemref && srcMemref &&197        destMemref.getType().getElementType() ==198            srcMemref.getType().getElementType() &&199        destMemref.getType().getShape() == srcMemref.getType().getShape()) {200      memref::CopyOp::create(builder, loc, srcMemref, destMemref);201      return true;202    }203 204    return false;205  }206};207 208struct LLVMPointerPointerLikeModel209    : public PointerLikeType::ExternalModel<LLVMPointerPointerLikeModel,210                                            LLVM::LLVMPointerType> {211  Type getElementType(Type pointer) const { return Type(); }212};213 214struct MemrefAddressOfGlobalModel215    : public AddressOfGlobalOpInterface::ExternalModel<216          MemrefAddressOfGlobalModel, memref::GetGlobalOp> {217  SymbolRefAttr getSymbol(Operation *op) const {218    auto getGlobalOp = cast<memref::GetGlobalOp>(op);219    return getGlobalOp.getNameAttr();220  }221};222 223struct MemrefGlobalVariableModel224    : public GlobalVariableOpInterface::ExternalModel<MemrefGlobalVariableModel,225                                                      memref::GlobalOp> {226  bool isConstant(Operation *op) const {227    auto globalOp = cast<memref::GlobalOp>(op);228    return globalOp.getConstant();229  }230 231  Region *getInitRegion(Operation *op) const {232    // GlobalOp uses attributes for initialization, not regions233    return nullptr;234  }235};236 237/// Helper function for any of the times we need to modify an ArrayAttr based on238/// a device type list.  Returns a new ArrayAttr with all of the239/// existingDeviceTypes, plus the effective new ones(or an added none if hte new240/// list is empty).241mlir::ArrayAttr addDeviceTypeAffectedOperandHelper(242    MLIRContext *context, mlir::ArrayAttr existingDeviceTypes,243    llvm::ArrayRef<acc::DeviceType> newDeviceTypes) {244  llvm::SmallVector<mlir::Attribute> deviceTypes;245  if (existingDeviceTypes)246    llvm::copy(existingDeviceTypes, std::back_inserter(deviceTypes));247 248  if (newDeviceTypes.empty())249    deviceTypes.push_back(250        acc::DeviceTypeAttr::get(context, acc::DeviceType::None));251 252  for (DeviceType dt : newDeviceTypes)253    deviceTypes.push_back(acc::DeviceTypeAttr::get(context, dt));254 255  return mlir::ArrayAttr::get(context, deviceTypes);256}257 258/// Helper function for any of the times we need to add operands that are259/// affected by a device type list. Returns a new ArrayAttr with all of the260/// existingDeviceTypes, plus the effective new ones (or an added none, if the261/// new list is empty). Additionally, adds the arguments to the argCollection262/// the correct number of times. This will also update a 'segments' array, even263/// if it won't be used.264mlir::ArrayAttr addDeviceTypeAffectedOperandHelper(265    MLIRContext *context, mlir::ArrayAttr existingDeviceTypes,266    llvm::ArrayRef<acc::DeviceType> newDeviceTypes, mlir::ValueRange arguments,267    mlir::MutableOperandRange argCollection,268    llvm::SmallVector<int32_t> &segments) {269  llvm::SmallVector<mlir::Attribute> deviceTypes;270  if (existingDeviceTypes)271    llvm::copy(existingDeviceTypes, std::back_inserter(deviceTypes));272 273  if (newDeviceTypes.empty()) {274    argCollection.append(arguments);275    segments.push_back(arguments.size());276    deviceTypes.push_back(277        acc::DeviceTypeAttr::get(context, acc::DeviceType::None));278  }279 280  for (DeviceType dt : newDeviceTypes) {281    argCollection.append(arguments);282    segments.push_back(arguments.size());283    deviceTypes.push_back(acc::DeviceTypeAttr::get(context, dt));284  }285 286  return mlir::ArrayAttr::get(context, deviceTypes);287}288 289/// Overload for when the 'segments' aren't needed.290mlir::ArrayAttr addDeviceTypeAffectedOperandHelper(291    MLIRContext *context, mlir::ArrayAttr existingDeviceTypes,292    llvm::ArrayRef<acc::DeviceType> newDeviceTypes, mlir::ValueRange arguments,293    mlir::MutableOperandRange argCollection) {294  llvm::SmallVector<int32_t> segments;295  return addDeviceTypeAffectedOperandHelper(context, existingDeviceTypes,296                                            newDeviceTypes, arguments,297                                            argCollection, segments);298}299} // namespace300 301//===----------------------------------------------------------------------===//302// OpenACC operations303//===----------------------------------------------------------------------===//304 305void OpenACCDialect::initialize() {306  addOperations<307#define GET_OP_LIST308#include "mlir/Dialect/OpenACC/OpenACCOps.cpp.inc"309      >();310  addAttributes<311#define GET_ATTRDEF_LIST312#include "mlir/Dialect/OpenACC/OpenACCOpsAttributes.cpp.inc"313      >();314  addTypes<315#define GET_TYPEDEF_LIST316#include "mlir/Dialect/OpenACC/OpenACCOpsTypes.cpp.inc"317      >();318 319  // By attaching interfaces here, we make the OpenACC dialect dependent on320  // the other dialects. This is probably better than having dialects like LLVM321  // and memref be dependent on OpenACC.322  MemRefType::attachInterface<MemRefPointerLikeModel<MemRefType>>(323      *getContext());324  UnrankedMemRefType::attachInterface<325      MemRefPointerLikeModel<UnrankedMemRefType>>(*getContext());326  LLVM::LLVMPointerType::attachInterface<LLVMPointerPointerLikeModel>(327      *getContext());328 329  // Attach operation interfaces330  memref::GetGlobalOp::attachInterface<MemrefAddressOfGlobalModel>(331      *getContext());332  memref::GlobalOp::attachInterface<MemrefGlobalVariableModel>(*getContext());333}334 335//===----------------------------------------------------------------------===//336// device_type support helpers337//===----------------------------------------------------------------------===//338 339static bool hasDeviceTypeValues(std::optional<mlir::ArrayAttr> arrayAttr) {340  return arrayAttr && *arrayAttr && arrayAttr->size() > 0;341}342 343static bool hasDeviceType(std::optional<mlir::ArrayAttr> arrayAttr,344                          mlir::acc::DeviceType deviceType) {345  if (!hasDeviceTypeValues(arrayAttr))346    return false;347 348  for (auto attr : *arrayAttr) {349    auto deviceTypeAttr = mlir::dyn_cast<mlir::acc::DeviceTypeAttr>(attr);350    if (deviceTypeAttr.getValue() == deviceType)351      return true;352  }353 354  return false;355}356 357static void printDeviceTypes(mlir::OpAsmPrinter &p,358                             std::optional<mlir::ArrayAttr> deviceTypes) {359  if (!hasDeviceTypeValues(deviceTypes))360    return;361 362  p << "[";363  llvm::interleaveComma(*deviceTypes, p,364                        [&](mlir::Attribute attr) { p << attr; });365  p << "]";366}367 368static std::optional<unsigned> findSegment(ArrayAttr segments,369                                           mlir::acc::DeviceType deviceType) {370  unsigned segmentIdx = 0;371  for (auto attr : segments) {372    auto deviceTypeAttr = mlir::dyn_cast<mlir::acc::DeviceTypeAttr>(attr);373    if (deviceTypeAttr.getValue() == deviceType)374      return std::make_optional(segmentIdx);375    ++segmentIdx;376  }377  return std::nullopt;378}379 380static mlir::Operation::operand_range381getValuesFromSegments(std::optional<mlir::ArrayAttr> arrayAttr,382                      mlir::Operation::operand_range range,383                      std::optional<llvm::ArrayRef<int32_t>> segments,384                      mlir::acc::DeviceType deviceType) {385  if (!arrayAttr)386    return range.take_front(0);387  if (auto pos = findSegment(*arrayAttr, deviceType)) {388    int32_t nbOperandsBefore = 0;389    for (unsigned i = 0; i < *pos; ++i)390      nbOperandsBefore += (*segments)[i];391    return range.drop_front(nbOperandsBefore).take_front((*segments)[*pos]);392  }393  return range.take_front(0);394}395 396static mlir::Value397getWaitDevnumValue(std::optional<mlir::ArrayAttr> deviceTypeAttr,398                   mlir::Operation::operand_range operands,399                   std::optional<llvm::ArrayRef<int32_t>> segments,400                   std::optional<mlir::ArrayAttr> hasWaitDevnum,401                   mlir::acc::DeviceType deviceType) {402  if (!hasDeviceTypeValues(deviceTypeAttr))403    return {};404  if (auto pos = findSegment(*deviceTypeAttr, deviceType))405    if (hasWaitDevnum->getValue()[*pos])406      return getValuesFromSegments(deviceTypeAttr, operands, segments,407                                   deviceType)408          .front();409  return {};410}411 412static mlir::Operation::operand_range413getWaitValuesWithoutDevnum(std::optional<mlir::ArrayAttr> deviceTypeAttr,414                           mlir::Operation::operand_range operands,415                           std::optional<llvm::ArrayRef<int32_t>> segments,416                           std::optional<mlir::ArrayAttr> hasWaitDevnum,417                           mlir::acc::DeviceType deviceType) {418  auto range =419      getValuesFromSegments(deviceTypeAttr, operands, segments, deviceType);420  if (range.empty())421    return range;422  if (auto pos = findSegment(*deviceTypeAttr, deviceType)) {423    if (hasWaitDevnum && *hasWaitDevnum) {424      auto boolAttr = mlir::dyn_cast<mlir::BoolAttr>((*hasWaitDevnum)[*pos]);425      if (boolAttr.getValue())426        return range.drop_front(1); // first value is devnum427    }428  }429  return range;430}431 432template <typename Op>433static LogicalResult checkWaitAndAsyncConflict(Op op) {434  for (uint32_t dtypeInt = 0; dtypeInt != acc::getMaxEnumValForDeviceType();435       ++dtypeInt) {436    auto dtype = static_cast<acc::DeviceType>(dtypeInt);437 438    // The asyncOnly attribute represent the async clause without value.439    // Therefore the attribute and operand cannot appear at the same time.440    if (hasDeviceType(op.getAsyncOperandsDeviceType(), dtype) &&441        op.hasAsyncOnly(dtype))442      return op.emitError(443          "asyncOnly attribute cannot appear with asyncOperand");444 445    // The wait attribute represent the wait clause without values. Therefore446    // the attribute and operands cannot appear at the same time.447    if (hasDeviceType(op.getWaitOperandsDeviceType(), dtype) &&448        op.hasWaitOnly(dtype))449      return op.emitError("wait attribute cannot appear with waitOperands");450  }451  return success();452}453 454template <typename Op>455static LogicalResult checkVarAndVarType(Op op) {456  if (!op.getVar())457    return op.emitError("must have var operand");458 459  // A variable must have a type that is either pointer-like or mappable.460  if (!mlir::isa<mlir::acc::PointerLikeType>(op.getVar().getType()) &&461      !mlir::isa<mlir::acc::MappableType>(op.getVar().getType()))462    return op.emitError("var must be mappable or pointer-like");463 464  // When it is a pointer-like type, the varType must capture the target type.465  if (mlir::isa<mlir::acc::PointerLikeType>(op.getVar().getType()) &&466      op.getVarType() == op.getVar().getType())467    return op.emitError("varType must capture the element type of var");468 469  return success();470}471 472template <typename Op>473static LogicalResult checkVarAndAccVar(Op op) {474  if (op.getVar().getType() != op.getAccVar().getType())475    return op.emitError("input and output types must match");476 477  return success();478}479 480template <typename Op>481static LogicalResult checkNoModifier(Op op) {482  if (op.getModifiers() != acc::DataClauseModifier::none)483    return op.emitError("no data clause modifiers are allowed");484  return success();485}486 487template <typename Op>488static LogicalResult489checkValidModifier(Op op, acc::DataClauseModifier validModifiers) {490  if (acc::bitEnumContainsAny(op.getModifiers(), ~validModifiers))491    return op.emitError(492        "invalid data clause modifiers: " +493        acc::stringifyDataClauseModifier(op.getModifiers() & ~validModifiers));494 495  return success();496}497 498template <typename OpT, typename RecipeOpT>499static LogicalResult checkRecipe(OpT op, llvm::StringRef operandName) {500  // Mappable types do not need a recipe because it is possible to generate one501  // from its API. Reject reductions though because no API is available for them502  // at this time.503  if (mlir::acc::isMappableType(op.getVar().getType()) &&504      !std::is_same_v<OpT, acc::ReductionOp>)505    return success();506 507  mlir::SymbolRefAttr operandRecipe = op.getRecipeAttr();508  if (!operandRecipe)509    return op->emitOpError() << "recipe expected for " << operandName;510 511  auto decl =512      SymbolTable::lookupNearestSymbolFrom<RecipeOpT>(op, operandRecipe);513  if (!decl)514    return op->emitOpError()515           << "expected symbol reference " << operandRecipe << " to point to a "516           << operandName << " declaration";517  return success();518}519 520static ParseResult parseVar(mlir::OpAsmParser &parser,521                            OpAsmParser::UnresolvedOperand &var) {522  // Either `var` or `varPtr` keyword is required.523  if (failed(parser.parseOptionalKeyword("varPtr"))) {524    if (failed(parser.parseKeyword("var")))525      return failure();526  }527  if (failed(parser.parseLParen()))528    return failure();529  if (failed(parser.parseOperand(var)))530    return failure();531 532  return success();533}534 535static void printVar(mlir::OpAsmPrinter &p, mlir::Operation *op,536                     mlir::Value var) {537  if (mlir::isa<mlir::acc::PointerLikeType>(var.getType()))538    p << "varPtr(";539  else540    p << "var(";541  p.printOperand(var);542}543 544static ParseResult parseAccVar(mlir::OpAsmParser &parser,545                               OpAsmParser::UnresolvedOperand &var,546                               mlir::Type &accVarType) {547  // Either `accVar` or `accPtr` keyword is required.548  if (failed(parser.parseOptionalKeyword("accPtr"))) {549    if (failed(parser.parseKeyword("accVar")))550      return failure();551  }552  if (failed(parser.parseLParen()))553    return failure();554  if (failed(parser.parseOperand(var)))555    return failure();556  if (failed(parser.parseColon()))557    return failure();558  if (failed(parser.parseType(accVarType)))559    return failure();560  if (failed(parser.parseRParen()))561    return failure();562 563  return success();564}565 566static void printAccVar(mlir::OpAsmPrinter &p, mlir::Operation *op,567                        mlir::Value accVar, mlir::Type accVarType) {568  if (mlir::isa<mlir::acc::PointerLikeType>(accVar.getType()))569    p << "accPtr(";570  else571    p << "accVar(";572  p.printOperand(accVar);573  p << " : ";574  p.printType(accVarType);575  p << ")";576}577 578static ParseResult parseVarPtrType(mlir::OpAsmParser &parser,579                                   mlir::Type &varPtrType,580                                   mlir::TypeAttr &varTypeAttr) {581  if (failed(parser.parseType(varPtrType)))582    return failure();583  if (failed(parser.parseRParen()))584    return failure();585 586  if (succeeded(parser.parseOptionalKeyword("varType"))) {587    if (failed(parser.parseLParen()))588      return failure();589    mlir::Type varType;590    if (failed(parser.parseType(varType)))591      return failure();592    varTypeAttr = mlir::TypeAttr::get(varType);593    if (failed(parser.parseRParen()))594      return failure();595  } else {596    // Set `varType` from the element type of the type of `varPtr`.597    if (mlir::isa<mlir::acc::PointerLikeType>(varPtrType))598      varTypeAttr = mlir::TypeAttr::get(599          mlir::cast<mlir::acc::PointerLikeType>(varPtrType).getElementType());600    else601      varTypeAttr = mlir::TypeAttr::get(varPtrType);602  }603 604  return success();605}606 607static void printVarPtrType(mlir::OpAsmPrinter &p, mlir::Operation *op,608                            mlir::Type varPtrType, mlir::TypeAttr varTypeAttr) {609  p.printType(varPtrType);610  p << ")";611 612  // Print the `varType` only if it differs from the element type of613  // `varPtr`'s type.614  mlir::Type varType = varTypeAttr.getValue();615  mlir::Type typeToCheckAgainst =616      mlir::isa<mlir::acc::PointerLikeType>(varPtrType)617          ? mlir::cast<mlir::acc::PointerLikeType>(varPtrType).getElementType()618          : varPtrType;619  if (typeToCheckAgainst != varType) {620    p << " varType(";621    p.printType(varType);622    p << ")";623  }624}625 626static ParseResult parseRecipeSym(mlir::OpAsmParser &parser,627                                  mlir::SymbolRefAttr &recipeAttr) {628  if (failed(parser.parseAttribute(recipeAttr)))629    return failure();630  return success();631}632 633static void printRecipeSym(mlir::OpAsmPrinter &p, mlir::Operation *op,634                           mlir::SymbolRefAttr recipeAttr) {635  p << recipeAttr;636}637 638//===----------------------------------------------------------------------===//639// DataBoundsOp640//===----------------------------------------------------------------------===//641LogicalResult acc::DataBoundsOp::verify() {642  auto extent = getExtent();643  auto upperbound = getUpperbound();644  if (!extent && !upperbound)645    return emitError("expected extent or upperbound.");646  return success();647}648 649//===----------------------------------------------------------------------===//650// PrivateOp651//===----------------------------------------------------------------------===//652LogicalResult acc::PrivateOp::verify() {653  if (getDataClause() != acc::DataClause::acc_private)654    return emitError(655        "data clause associated with private operation must match its intent");656  if (failed(checkVarAndVarType(*this)))657    return failure();658  if (failed(checkNoModifier(*this)))659    return failure();660  if (failed(661          checkRecipe<acc::PrivateOp, acc::PrivateRecipeOp>(*this, "private")))662    return failure();663  return success();664}665 666//===----------------------------------------------------------------------===//667// FirstprivateOp668//===----------------------------------------------------------------------===//669LogicalResult acc::FirstprivateOp::verify() {670  if (getDataClause() != acc::DataClause::acc_firstprivate)671    return emitError("data clause associated with firstprivate operation must "672                     "match its intent");673  if (failed(checkVarAndVarType(*this)))674    return failure();675  if (failed(checkNoModifier(*this)))676    return failure();677  if (failed(checkRecipe<acc::FirstprivateOp, acc::FirstprivateRecipeOp>(678          *this, "firstprivate")))679    return failure();680  return success();681}682 683//===----------------------------------------------------------------------===//684// FirstprivateMapInitialOp685//===----------------------------------------------------------------------===//686LogicalResult acc::FirstprivateMapInitialOp::verify() {687  if (getDataClause() != acc::DataClause::acc_firstprivate)688    return emitError("data clause associated with firstprivate operation must "689                     "match its intent");690  if (failed(checkVarAndVarType(*this)))691    return failure();692  if (failed(checkNoModifier(*this)))693    return failure();694  return success();695}696 697//===----------------------------------------------------------------------===//698// ReductionOp699//===----------------------------------------------------------------------===//700LogicalResult acc::ReductionOp::verify() {701  if (getDataClause() != acc::DataClause::acc_reduction)702    return emitError("data clause associated with reduction operation must "703                     "match its intent");704  if (failed(checkVarAndVarType(*this)))705    return failure();706  if (failed(checkNoModifier(*this)))707    return failure();708  if (failed(checkRecipe<acc::ReductionOp, acc::ReductionRecipeOp>(709          *this, "reduction")))710    return failure();711  return success();712}713 714//===----------------------------------------------------------------------===//715// DevicePtrOp716//===----------------------------------------------------------------------===//717LogicalResult acc::DevicePtrOp::verify() {718  if (getDataClause() != acc::DataClause::acc_deviceptr)719    return emitError("data clause associated with deviceptr operation must "720                     "match its intent");721  if (failed(checkVarAndVarType(*this)))722    return failure();723  if (failed(checkVarAndAccVar(*this)))724    return failure();725  if (failed(checkNoModifier(*this)))726    return failure();727  return success();728}729 730//===----------------------------------------------------------------------===//731// PresentOp732//===----------------------------------------------------------------------===//733LogicalResult acc::PresentOp::verify() {734  if (getDataClause() != acc::DataClause::acc_present)735    return emitError(736        "data clause associated with present operation must match its intent");737  if (failed(checkVarAndVarType(*this)))738    return failure();739  if (failed(checkVarAndAccVar(*this)))740    return failure();741  if (failed(checkNoModifier(*this)))742    return failure();743  return success();744}745 746//===----------------------------------------------------------------------===//747// CopyinOp748//===----------------------------------------------------------------------===//749LogicalResult acc::CopyinOp::verify() {750  // Test for all clauses this operation can be decomposed from:751  if (!getImplicit() && getDataClause() != acc::DataClause::acc_copyin &&752      getDataClause() != acc::DataClause::acc_copyin_readonly &&753      getDataClause() != acc::DataClause::acc_copy &&754      getDataClause() != acc::DataClause::acc_reduction)755    return emitError(756        "data clause associated with copyin operation must match its intent"757        " or specify original clause this operation was decomposed from");758  if (failed(checkVarAndVarType(*this)))759    return failure();760  if (failed(checkVarAndAccVar(*this)))761    return failure();762  if (failed(checkValidModifier(*this, acc::DataClauseModifier::readonly |763                                           acc::DataClauseModifier::always |764                                           acc::DataClauseModifier::capture)))765    return failure();766  return success();767}768 769bool acc::CopyinOp::isCopyinReadonly() {770  return getDataClause() == acc::DataClause::acc_copyin_readonly ||771         acc::bitEnumContainsAny(getModifiers(),772                                 acc::DataClauseModifier::readonly);773}774 775//===----------------------------------------------------------------------===//776// CreateOp777//===----------------------------------------------------------------------===//778LogicalResult acc::CreateOp::verify() {779  // Test for all clauses this operation can be decomposed from:780  if (getDataClause() != acc::DataClause::acc_create &&781      getDataClause() != acc::DataClause::acc_create_zero &&782      getDataClause() != acc::DataClause::acc_copyout &&783      getDataClause() != acc::DataClause::acc_copyout_zero)784    return emitError(785        "data clause associated with create operation must match its intent"786        " or specify original clause this operation was decomposed from");787  if (failed(checkVarAndVarType(*this)))788    return failure();789  if (failed(checkVarAndAccVar(*this)))790    return failure();791  // this op is the entry part of copyout, so it also needs to allow all792  // modifiers allowed on copyout.793  if (failed(checkValidModifier(*this, acc::DataClauseModifier::zero |794                                           acc::DataClauseModifier::always |795                                           acc::DataClauseModifier::capture)))796    return failure();797  return success();798}799 800bool acc::CreateOp::isCreateZero() {801  // The zero modifier is encoded in the data clause.802  return getDataClause() == acc::DataClause::acc_create_zero ||803         getDataClause() == acc::DataClause::acc_copyout_zero ||804         acc::bitEnumContainsAny(getModifiers(), acc::DataClauseModifier::zero);805}806 807//===----------------------------------------------------------------------===//808// NoCreateOp809//===----------------------------------------------------------------------===//810LogicalResult acc::NoCreateOp::verify() {811  if (getDataClause() != acc::DataClause::acc_no_create)812    return emitError("data clause associated with no_create operation must "813                     "match its intent");814  if (failed(checkVarAndVarType(*this)))815    return failure();816  if (failed(checkVarAndAccVar(*this)))817    return failure();818  if (failed(checkNoModifier(*this)))819    return failure();820  return success();821}822 823//===----------------------------------------------------------------------===//824// AttachOp825//===----------------------------------------------------------------------===//826LogicalResult acc::AttachOp::verify() {827  if (getDataClause() != acc::DataClause::acc_attach)828    return emitError(829        "data clause associated with attach operation must match its intent");830  if (failed(checkVarAndVarType(*this)))831    return failure();832  if (failed(checkVarAndAccVar(*this)))833    return failure();834  if (failed(checkNoModifier(*this)))835    return failure();836  return success();837}838 839//===----------------------------------------------------------------------===//840// DeclareDeviceResidentOp841//===----------------------------------------------------------------------===//842 843LogicalResult acc::DeclareDeviceResidentOp::verify() {844  if (getDataClause() != acc::DataClause::acc_declare_device_resident)845    return emitError("data clause associated with device_resident operation "846                     "must match its intent");847  if (failed(checkVarAndVarType(*this)))848    return failure();849  if (failed(checkVarAndAccVar(*this)))850    return failure();851  if (failed(checkNoModifier(*this)))852    return failure();853  return success();854}855 856//===----------------------------------------------------------------------===//857// DeclareLinkOp858//===----------------------------------------------------------------------===//859 860LogicalResult acc::DeclareLinkOp::verify() {861  if (getDataClause() != acc::DataClause::acc_declare_link)862    return emitError(863        "data clause associated with link operation must match its intent");864  if (failed(checkVarAndVarType(*this)))865    return failure();866  if (failed(checkVarAndAccVar(*this)))867    return failure();868  if (failed(checkNoModifier(*this)))869    return failure();870  return success();871}872 873//===----------------------------------------------------------------------===//874// CopyoutOp875//===----------------------------------------------------------------------===//876LogicalResult acc::CopyoutOp::verify() {877  // Test for all clauses this operation can be decomposed from:878  if (getDataClause() != acc::DataClause::acc_copyout &&879      getDataClause() != acc::DataClause::acc_copyout_zero &&880      getDataClause() != acc::DataClause::acc_copy &&881      getDataClause() != acc::DataClause::acc_reduction)882    return emitError(883        "data clause associated with copyout operation must match its intent"884        " or specify original clause this operation was decomposed from");885  if (!getVar() || !getAccVar())886    return emitError("must have both host and device pointers");887  if (failed(checkVarAndVarType(*this)))888    return failure();889  if (failed(checkVarAndAccVar(*this)))890    return failure();891  if (failed(checkValidModifier(*this, acc::DataClauseModifier::zero |892                                           acc::DataClauseModifier::always |893                                           acc::DataClauseModifier::capture)))894    return failure();895  return success();896}897 898bool acc::CopyoutOp::isCopyoutZero() {899  return getDataClause() == acc::DataClause::acc_copyout_zero ||900         acc::bitEnumContainsAny(getModifiers(), acc::DataClauseModifier::zero);901}902 903//===----------------------------------------------------------------------===//904// DeleteOp905//===----------------------------------------------------------------------===//906LogicalResult acc::DeleteOp::verify() {907  // Test for all clauses this operation can be decomposed from:908  if (getDataClause() != acc::DataClause::acc_delete &&909      getDataClause() != acc::DataClause::acc_create &&910      getDataClause() != acc::DataClause::acc_create_zero &&911      getDataClause() != acc::DataClause::acc_copyin &&912      getDataClause() != acc::DataClause::acc_copyin_readonly &&913      getDataClause() != acc::DataClause::acc_present &&914      getDataClause() != acc::DataClause::acc_no_create &&915      getDataClause() != acc::DataClause::acc_declare_device_resident &&916      getDataClause() != acc::DataClause::acc_declare_link)917    return emitError(918        "data clause associated with delete operation must match its intent"919        " or specify original clause this operation was decomposed from");920  if (!getAccVar())921    return emitError("must have device pointer");922  // This op is the exit part of copyin and create - thus allow all modifiers923  // allowed on either case.924  if (failed(checkValidModifier(*this, acc::DataClauseModifier::zero |925                                           acc::DataClauseModifier::readonly |926                                           acc::DataClauseModifier::always |927                                           acc::DataClauseModifier::capture)))928    return failure();929  return success();930}931 932//===----------------------------------------------------------------------===//933// DetachOp934//===----------------------------------------------------------------------===//935LogicalResult acc::DetachOp::verify() {936  // Test for all clauses this operation can be decomposed from:937  if (getDataClause() != acc::DataClause::acc_detach &&938      getDataClause() != acc::DataClause::acc_attach)939    return emitError(940        "data clause associated with detach operation must match its intent"941        " or specify original clause this operation was decomposed from");942  if (!getAccVar())943    return emitError("must have device pointer");944  if (failed(checkNoModifier(*this)))945    return failure();946  return success();947}948 949//===----------------------------------------------------------------------===//950// HostOp951//===----------------------------------------------------------------------===//952LogicalResult acc::UpdateHostOp::verify() {953  // Test for all clauses this operation can be decomposed from:954  if (getDataClause() != acc::DataClause::acc_update_host &&955      getDataClause() != acc::DataClause::acc_update_self)956    return emitError(957        "data clause associated with host operation must match its intent"958        " or specify original clause this operation was decomposed from");959  if (!getVar() || !getAccVar())960    return emitError("must have both host and device pointers");961  if (failed(checkVarAndVarType(*this)))962    return failure();963  if (failed(checkVarAndAccVar(*this)))964    return failure();965  if (failed(checkNoModifier(*this)))966    return failure();967  return success();968}969 970//===----------------------------------------------------------------------===//971// DeviceOp972//===----------------------------------------------------------------------===//973LogicalResult acc::UpdateDeviceOp::verify() {974  // Test for all clauses this operation can be decomposed from:975  if (getDataClause() != acc::DataClause::acc_update_device)976    return emitError(977        "data clause associated with device operation must match its intent"978        " or specify original clause this operation was decomposed from");979  if (failed(checkVarAndVarType(*this)))980    return failure();981  if (failed(checkVarAndAccVar(*this)))982    return failure();983  if (failed(checkNoModifier(*this)))984    return failure();985  return success();986}987 988//===----------------------------------------------------------------------===//989// UseDeviceOp990//===----------------------------------------------------------------------===//991LogicalResult acc::UseDeviceOp::verify() {992  // Test for all clauses this operation can be decomposed from:993  if (getDataClause() != acc::DataClause::acc_use_device)994    return emitError(995        "data clause associated with use_device operation must match its intent"996        " or specify original clause this operation was decomposed from");997  if (failed(checkVarAndVarType(*this)))998    return failure();999  if (failed(checkVarAndAccVar(*this)))1000    return failure();1001  if (failed(checkNoModifier(*this)))1002    return failure();1003  return success();1004}1005 1006//===----------------------------------------------------------------------===//1007// CacheOp1008//===----------------------------------------------------------------------===//1009LogicalResult acc::CacheOp::verify() {1010  // Test for all clauses this operation can be decomposed from:1011  if (getDataClause() != acc::DataClause::acc_cache &&1012      getDataClause() != acc::DataClause::acc_cache_readonly)1013    return emitError(1014        "data clause associated with cache operation must match its intent"1015        " or specify original clause this operation was decomposed from");1016  if (failed(checkVarAndVarType(*this)))1017    return failure();1018  if (failed(checkVarAndAccVar(*this)))1019    return failure();1020  if (failed(checkValidModifier(*this, acc::DataClauseModifier::readonly)))1021    return failure();1022  return success();1023}1024 1025bool acc::CacheOp::isCacheReadonly() {1026  return getDataClause() == acc::DataClause::acc_cache_readonly ||1027         acc::bitEnumContainsAny(getModifiers(),1028                                 acc::DataClauseModifier::readonly);1029}1030 1031template <typename StructureOp>1032static ParseResult parseRegions(OpAsmParser &parser, OperationState &state,1033                                unsigned nRegions = 1) {1034 1035  SmallVector<Region *, 2> regions;1036  for (unsigned i = 0; i < nRegions; ++i)1037    regions.push_back(state.addRegion());1038 1039  for (Region *region : regions)1040    if (parser.parseRegion(*region, /*arguments=*/{}, /*argTypes=*/{}))1041      return failure();1042 1043  return success();1044}1045 1046static bool isComputeOperation(Operation *op) {1047  return isa<ACC_COMPUTE_CONSTRUCT_AND_LOOP_OPS>(op);1048}1049 1050namespace {1051/// Pattern to remove operation without region that have constant false `ifCond`1052/// and remove the condition from the operation if the `ifCond` is a true1053/// constant.1054template <typename OpTy>1055struct RemoveConstantIfCondition : public OpRewritePattern<OpTy> {1056  using OpRewritePattern<OpTy>::OpRewritePattern;1057 1058  LogicalResult matchAndRewrite(OpTy op,1059                                PatternRewriter &rewriter) const override {1060    // Early return if there is no condition.1061    Value ifCond = op.getIfCond();1062    if (!ifCond)1063      return failure();1064 1065    IntegerAttr constAttr;1066    if (!matchPattern(ifCond, m_Constant(&constAttr)))1067      return failure();1068    if (constAttr.getInt())1069      rewriter.modifyOpInPlace(op, [&]() { op.getIfCondMutable().erase(0); });1070    else1071      rewriter.eraseOp(op);1072 1073    return success();1074  }1075};1076 1077/// Replaces the given op with the contents of the given single-block region,1078/// using the operands of the block terminator to replace operation results.1079static void replaceOpWithRegion(PatternRewriter &rewriter, Operation *op,1080                                Region &region, ValueRange blockArgs = {}) {1081  assert(region.hasOneBlock() && "expected single-block region");1082  Block *block = &region.front();1083  Operation *terminator = block->getTerminator();1084  ValueRange results = terminator->getOperands();1085  rewriter.inlineBlockBefore(block, op, blockArgs);1086  rewriter.replaceOp(op, results);1087  rewriter.eraseOp(terminator);1088}1089 1090/// Pattern to remove operation with region that have constant false `ifCond`1091/// and remove the condition from the operation if the `ifCond` is constant1092/// true.1093template <typename OpTy>1094struct RemoveConstantIfConditionWithRegion : public OpRewritePattern<OpTy> {1095  using OpRewritePattern<OpTy>::OpRewritePattern;1096 1097  LogicalResult matchAndRewrite(OpTy op,1098                                PatternRewriter &rewriter) const override {1099    // Early return if there is no condition.1100    Value ifCond = op.getIfCond();1101    if (!ifCond)1102      return failure();1103 1104    IntegerAttr constAttr;1105    if (!matchPattern(ifCond, m_Constant(&constAttr)))1106      return failure();1107    if (constAttr.getInt())1108      rewriter.modifyOpInPlace(op, [&]() { op.getIfCondMutable().erase(0); });1109    else1110      replaceOpWithRegion(rewriter, op, op.getRegion());1111 1112    return success();1113  }1114};1115 1116/// Remove empty acc.kernel_environment operations. If the operation has wait1117/// operands, create a acc.wait operation to preserve synchronization.1118struct RemoveEmptyKernelEnvironment1119    : public OpRewritePattern<acc::KernelEnvironmentOp> {1120  using OpRewritePattern<acc::KernelEnvironmentOp>::OpRewritePattern;1121 1122  LogicalResult matchAndRewrite(acc::KernelEnvironmentOp op,1123                                PatternRewriter &rewriter) const override {1124    assert(op->getNumRegions() == 1 && "expected op to have one region");1125 1126    Block &block = op.getRegion().front();1127    if (!block.empty())1128      return failure();1129 1130    // Conservatively disable canonicalization of empty acc.kernel_environment1131    // operations if the wait operands in the kernel_environment cannot be fully1132    // represented by acc.wait operation.1133 1134    // Disable canonicalization if device type is not the default1135    if (auto deviceTypeAttr = op.getWaitOperandsDeviceTypeAttr()) {1136      for (auto attr : deviceTypeAttr) {1137        if (auto dtAttr = mlir::dyn_cast<acc::DeviceTypeAttr>(attr)) {1138          if (dtAttr.getValue() != mlir::acc::DeviceType::None)1139            return failure();1140        }1141      }1142    }1143 1144    // Disable canonicalization if any wait segment has a devnum1145    if (auto hasDevnumAttr = op.getHasWaitDevnumAttr()) {1146      for (auto attr : hasDevnumAttr) {1147        if (auto boolAttr = mlir::dyn_cast<mlir::BoolAttr>(attr)) {1148          if (boolAttr.getValue())1149            return failure();1150        }1151      }1152    }1153 1154    // Disable canonicalization if there are multiple wait segments1155    if (auto segmentsAttr = op.getWaitOperandsSegmentsAttr()) {1156      if (segmentsAttr.size() > 1)1157        return failure();1158    }1159 1160    // Remove empty kernel environment.1161    // Preserve synchronization by creating acc.wait operation if needed.1162    if (!op.getWaitOperands().empty() || op.getWaitOnlyAttr())1163      rewriter.replaceOpWithNewOp<acc::WaitOp>(op, op.getWaitOperands(),1164                                               /*asyncOperand=*/Value(),1165                                               /*waitDevnum=*/Value(),1166                                               /*async=*/nullptr,1167                                               /*ifCond=*/Value());1168    else1169      rewriter.eraseOp(op);1170 1171    return success();1172  }1173};1174 1175//===----------------------------------------------------------------------===//1176// Recipe Region Helpers1177//===----------------------------------------------------------------------===//1178 1179/// Create and populate an init region for privatization recipes.1180/// Returns success if the region is populated, failure otherwise.1181/// Sets needsFree to indicate if the allocated memory requires deallocation.1182static LogicalResult createInitRegion(OpBuilder &builder, Location loc,1183                                      Region &initRegion, Type varType,1184                                      StringRef varName, ValueRange bounds,1185                                      bool &needsFree) {1186  // Create init block with arguments: original value + bounds1187  SmallVector<Type> argTypes{varType};1188  SmallVector<Location> argLocs{loc};1189  for (Value bound : bounds) {1190    argTypes.push_back(bound.getType());1191    argLocs.push_back(loc);1192  }1193 1194  Block *initBlock = builder.createBlock(&initRegion);1195  initBlock->addArguments(argTypes, argLocs);1196  builder.setInsertionPointToStart(initBlock);1197 1198  Value privatizedValue;1199 1200  // Get the block argument that represents the original variable1201  Value blockArgVar = initBlock->getArgument(0);1202 1203  // Generate init region body based on variable type1204  if (isa<MappableType>(varType)) {1205    auto mappableTy = cast<MappableType>(varType);1206    auto typedVar = cast<TypedValue<MappableType>>(blockArgVar);1207    privatizedValue = mappableTy.generatePrivateInit(1208        builder, loc, typedVar, varName, bounds, {}, needsFree);1209    if (!privatizedValue)1210      return failure();1211  } else {1212    assert(isa<PointerLikeType>(varType) && "Expected PointerLikeType");1213    auto pointerLikeTy = cast<PointerLikeType>(varType);1214    // Use PointerLikeType's allocation API with the block argument1215    privatizedValue = pointerLikeTy.genAllocate(builder, loc, varName, varType,1216                                                blockArgVar, needsFree);1217    if (!privatizedValue)1218      return failure();1219  }1220 1221  // Add yield operation to init block1222  acc::YieldOp::create(builder, loc, privatizedValue);1223 1224  return success();1225}1226 1227/// Create and populate a copy region for firstprivate recipes.1228/// Returns success if the region is populated, failure otherwise.1229/// TODO: Handle MappableType - it does not yet have a copy API.1230static LogicalResult createCopyRegion(OpBuilder &builder, Location loc,1231                                      Region &copyRegion, Type varType,1232                                      ValueRange bounds) {1233  // Create copy block with arguments: original value + privatized value +1234  // bounds1235  SmallVector<Type> copyArgTypes{varType, varType};1236  SmallVector<Location> copyArgLocs{loc, loc};1237  for (Value bound : bounds) {1238    copyArgTypes.push_back(bound.getType());1239    copyArgLocs.push_back(loc);1240  }1241 1242  Block *copyBlock = builder.createBlock(&copyRegion);1243  copyBlock->addArguments(copyArgTypes, copyArgLocs);1244  builder.setInsertionPointToStart(copyBlock);1245 1246  bool isMappable = isa<MappableType>(varType);1247  bool isPointerLike = isa<PointerLikeType>(varType);1248  // TODO: Handle MappableType - it does not yet have a copy API.1249  // Otherwise, for now just fallback to pointer-like behavior.1250  if (isMappable && !isPointerLike)1251    return failure();1252 1253  // Generate copy region body based on variable type1254  if (isPointerLike) {1255    auto pointerLikeTy = cast<PointerLikeType>(varType);1256    Value originalArg = copyBlock->getArgument(0);1257    Value privatizedArg = copyBlock->getArgument(1);1258 1259    // Generate copy operation using PointerLikeType interface1260    if (!pointerLikeTy.genCopy(1261            builder, loc, cast<TypedValue<PointerLikeType>>(privatizedArg),1262            cast<TypedValue<PointerLikeType>>(originalArg), varType))1263      return failure();1264  }1265 1266  // Add terminator to copy block1267  acc::TerminatorOp::create(builder, loc);1268 1269  return success();1270}1271 1272/// Create and populate a destroy region for privatization recipes.1273/// Returns success if the region is populated, failure otherwise.1274static LogicalResult createDestroyRegion(OpBuilder &builder, Location loc,1275                                         Region &destroyRegion, Type varType,1276                                         Value allocRes, ValueRange bounds) {1277  // Create destroy block with arguments: original value + privatized value +1278  // bounds1279  SmallVector<Type> destroyArgTypes{varType, varType};1280  SmallVector<Location> destroyArgLocs{loc, loc};1281  for (Value bound : bounds) {1282    destroyArgTypes.push_back(bound.getType());1283    destroyArgLocs.push_back(loc);1284  }1285 1286  Block *destroyBlock = builder.createBlock(&destroyRegion);1287  destroyBlock->addArguments(destroyArgTypes, destroyArgLocs);1288  builder.setInsertionPointToStart(destroyBlock);1289 1290  auto varToFree =1291      cast<TypedValue<PointerLikeType>>(destroyBlock->getArgument(1));1292  if (isa<MappableType>(varType)) {1293    auto mappableTy = cast<MappableType>(varType);1294    if (!mappableTy.generatePrivateDestroy(builder, loc, varToFree))1295      return failure();1296  } else {1297    assert(isa<PointerLikeType>(varType) && "Expected PointerLikeType");1298    auto pointerLikeTy = cast<PointerLikeType>(varType);1299    if (!pointerLikeTy.genFree(builder, loc, varToFree, allocRes, varType))1300      return failure();1301  }1302 1303  acc::TerminatorOp::create(builder, loc);1304  return success();1305}1306 1307} // namespace1308 1309//===----------------------------------------------------------------------===//1310// PrivateRecipeOp1311//===----------------------------------------------------------------------===//1312 1313static LogicalResult verifyInitLikeSingleArgRegion(1314    Operation *op, Region &region, StringRef regionType, StringRef regionName,1315    Type type, bool verifyYield, bool optional = false) {1316  if (optional && region.empty())1317    return success();1318 1319  if (region.empty())1320    return op->emitOpError() << "expects non-empty " << regionName << " region";1321  Block &firstBlock = region.front();1322  if (firstBlock.getNumArguments() < 1 ||1323      firstBlock.getArgument(0).getType() != type)1324    return op->emitOpError() << "expects " << regionName1325                             << " region first "1326                                "argument of the "1327                             << regionType << " type";1328 1329  if (verifyYield) {1330    for (YieldOp yieldOp : region.getOps<acc::YieldOp>()) {1331      if (yieldOp.getOperands().size() != 1 ||1332          yieldOp.getOperands().getTypes()[0] != type)1333        return op->emitOpError() << "expects " << regionName1334                                 << " region to "1335                                    "yield a value of the "1336                                 << regionType << " type";1337    }1338  }1339  return success();1340}1341 1342LogicalResult acc::PrivateRecipeOp::verifyRegions() {1343  if (failed(verifyInitLikeSingleArgRegion(*this, getInitRegion(),1344                                           "privatization", "init", getType(),1345                                           /*verifyYield=*/false)))1346    return failure();1347  if (failed(verifyInitLikeSingleArgRegion(1348          *this, getDestroyRegion(), "privatization", "destroy", getType(),1349          /*verifyYield=*/false, /*optional=*/true)))1350    return failure();1351  return success();1352}1353 1354std::optional<PrivateRecipeOp>1355PrivateRecipeOp::createAndPopulate(OpBuilder &builder, Location loc,1356                                   StringRef recipeName, Type varType,1357                                   StringRef varName, ValueRange bounds) {1358  // First, validate that we can handle this variable type1359  bool isMappable = isa<MappableType>(varType);1360  bool isPointerLike = isa<PointerLikeType>(varType);1361 1362  // Unsupported type1363  if (!isMappable && !isPointerLike)1364    return std::nullopt;1365 1366  OpBuilder::InsertionGuard guard(builder);1367 1368  // Create the recipe operation first so regions have proper parent context1369  auto recipe = PrivateRecipeOp::create(builder, loc, recipeName, varType);1370 1371  // Populate the init region1372  bool needsFree = false;1373  if (failed(createInitRegion(builder, loc, recipe.getInitRegion(), varType,1374                              varName, bounds, needsFree))) {1375    recipe.erase();1376    return std::nullopt;1377  }1378 1379  // Only create destroy region if the allocation needs deallocation1380  if (needsFree) {1381    // Extract the allocated value from the init block's yield operation1382    auto yieldOp =1383        cast<acc::YieldOp>(recipe.getInitRegion().front().getTerminator());1384    Value allocRes = yieldOp.getOperand(0);1385 1386    if (failed(createDestroyRegion(builder, loc, recipe.getDestroyRegion(),1387                                   varType, allocRes, bounds))) {1388      recipe.erase();1389      return std::nullopt;1390    }1391  }1392 1393  return recipe;1394}1395 1396//===----------------------------------------------------------------------===//1397// FirstprivateRecipeOp1398//===----------------------------------------------------------------------===//1399 1400LogicalResult acc::FirstprivateRecipeOp::verifyRegions() {1401  if (failed(verifyInitLikeSingleArgRegion(*this, getInitRegion(),1402                                           "privatization", "init", getType(),1403                                           /*verifyYield=*/false)))1404    return failure();1405 1406  if (getCopyRegion().empty())1407    return emitOpError() << "expects non-empty copy region";1408 1409  Block &firstBlock = getCopyRegion().front();1410  if (firstBlock.getNumArguments() < 2 ||1411      firstBlock.getArgument(0).getType() != getType())1412    return emitOpError() << "expects copy region with two arguments of the "1413                            "privatization type";1414 1415  if (getDestroyRegion().empty())1416    return success();1417 1418  if (failed(verifyInitLikeSingleArgRegion(*this, getDestroyRegion(),1419                                           "privatization", "destroy",1420                                           getType(), /*verifyYield=*/false)))1421    return failure();1422 1423  return success();1424}1425 1426std::optional<FirstprivateRecipeOp>1427FirstprivateRecipeOp::createAndPopulate(OpBuilder &builder, Location loc,1428                                        StringRef recipeName, Type varType,1429                                        StringRef varName, ValueRange bounds) {1430  // First, validate that we can handle this variable type1431  bool isMappable = isa<MappableType>(varType);1432  bool isPointerLike = isa<PointerLikeType>(varType);1433 1434  // Unsupported type1435  if (!isMappable && !isPointerLike)1436    return std::nullopt;1437 1438  OpBuilder::InsertionGuard guard(builder);1439 1440  // Create the recipe operation first so regions have proper parent context1441  auto recipe = FirstprivateRecipeOp::create(builder, loc, recipeName, varType);1442 1443  // Populate the init region1444  bool needsFree = false;1445  if (failed(createInitRegion(builder, loc, recipe.getInitRegion(), varType,1446                              varName, bounds, needsFree))) {1447    recipe.erase();1448    return std::nullopt;1449  }1450 1451  // Populate the copy region1452  if (failed(createCopyRegion(builder, loc, recipe.getCopyRegion(), varType,1453                              bounds))) {1454    recipe.erase();1455    return std::nullopt;1456  }1457 1458  // Only create destroy region if the allocation needs deallocation1459  if (needsFree) {1460    // Extract the allocated value from the init block's yield operation1461    auto yieldOp =1462        cast<acc::YieldOp>(recipe.getInitRegion().front().getTerminator());1463    Value allocRes = yieldOp.getOperand(0);1464 1465    if (failed(createDestroyRegion(builder, loc, recipe.getDestroyRegion(),1466                                   varType, allocRes, bounds))) {1467      recipe.erase();1468      return std::nullopt;1469    }1470  }1471 1472  return recipe;1473}1474 1475//===----------------------------------------------------------------------===//1476// ReductionRecipeOp1477//===----------------------------------------------------------------------===//1478 1479LogicalResult acc::ReductionRecipeOp::verifyRegions() {1480  if (failed(verifyInitLikeSingleArgRegion(*this, getInitRegion(), "reduction",1481                                           "init", getType(),1482                                           /*verifyYield=*/false)))1483    return failure();1484 1485  if (getCombinerRegion().empty())1486    return emitOpError() << "expects non-empty combiner region";1487 1488  Block &reductionBlock = getCombinerRegion().front();1489  if (reductionBlock.getNumArguments() < 2 ||1490      reductionBlock.getArgument(0).getType() != getType() ||1491      reductionBlock.getArgument(1).getType() != getType())1492    return emitOpError() << "expects combiner region with the first two "1493                         << "arguments of the reduction type";1494 1495  for (YieldOp yieldOp : getCombinerRegion().getOps<YieldOp>()) {1496    if (yieldOp.getOperands().size() != 1 ||1497        yieldOp.getOperands().getTypes()[0] != getType())1498      return emitOpError() << "expects combiner region to yield a value "1499                              "of the reduction type";1500  }1501 1502  return success();1503}1504 1505//===----------------------------------------------------------------------===//1506// ParallelOp1507//===----------------------------------------------------------------------===//1508 1509/// Check dataOperands for acc.parallel, acc.serial and acc.kernels.1510template <typename Op>1511static LogicalResult checkDataOperands(Op op,1512                                       const mlir::ValueRange &operands) {1513  for (mlir::Value operand : operands)1514    if (!mlir::isa<acc::AttachOp, acc::CopyinOp, acc::CopyoutOp, acc::CreateOp,1515                   acc::DeleteOp, acc::DetachOp, acc::DevicePtrOp,1516                   acc::GetDevicePtrOp, acc::NoCreateOp, acc::PresentOp>(1517            operand.getDefiningOp()))1518      return op.emitError(1519          "expect data entry/exit operation or acc.getdeviceptr "1520          "as defining op");1521  return success();1522}1523 1524template <typename OpT, typename RecipeOpT>1525static LogicalResult checkPrivateOperands(mlir::Operation *accConstructOp,1526                                          const mlir::ValueRange &operands,1527                                          llvm::StringRef operandName) {1528  llvm::DenseSet<Value> set;1529  for (mlir::Value operand : operands) {1530    if (!mlir::isa<OpT>(operand.getDefiningOp()))1531      return accConstructOp->emitOpError()1532             << "expected " << operandName << " as defining op";1533    if (!set.insert(operand).second)1534      return accConstructOp->emitOpError()1535             << operandName << " operand appears more than once";1536  }1537  return success();1538}1539 1540unsigned ParallelOp::getNumDataOperands() {1541  return getReductionOperands().size() + getPrivateOperands().size() +1542         getFirstprivateOperands().size() + getDataClauseOperands().size();1543}1544 1545Value ParallelOp::getDataOperand(unsigned i) {1546  unsigned numOptional = getAsyncOperands().size();1547  numOptional += getNumGangs().size();1548  numOptional += getNumWorkers().size();1549  numOptional += getVectorLength().size();1550  numOptional += getIfCond() ? 1 : 0;1551  numOptional += getSelfCond() ? 1 : 0;1552  return getOperand(getWaitOperands().size() + numOptional + i);1553}1554 1555template <typename Op>1556static LogicalResult verifyDeviceTypeCountMatch(Op op, OperandRange operands,1557                                                ArrayAttr deviceTypes,1558                                                llvm::StringRef keyword) {1559  if (!operands.empty() && deviceTypes.getValue().size() != operands.size())1560    return op.emitOpError() << keyword << " operands count must match "1561                            << keyword << " device_type count";1562  return success();1563}1564 1565template <typename Op>1566static LogicalResult verifyDeviceTypeAndSegmentCountMatch(1567    Op op, OperandRange operands, DenseI32ArrayAttr segments,1568    ArrayAttr deviceTypes, llvm::StringRef keyword, int32_t maxInSegment = 0) {1569  std::size_t numOperandsInSegments = 0;1570  std::size_t nbOfSegments = 0;1571 1572  if (segments) {1573    for (auto segCount : segments.asArrayRef()) {1574      if (maxInSegment != 0 && segCount > maxInSegment)1575        return op.emitOpError() << keyword << " expects a maximum of "1576                                << maxInSegment << " values per segment";1577      numOperandsInSegments += segCount;1578      ++nbOfSegments;1579    }1580  }1581 1582  if ((numOperandsInSegments != operands.size()) ||1583      (!deviceTypes && !operands.empty()))1584    return op.emitOpError()1585           << keyword << " operand count does not match count in segments";1586  if (deviceTypes && deviceTypes.getValue().size() != nbOfSegments)1587    return op.emitOpError()1588           << keyword << " segment count does not match device_type count";1589  return success();1590}1591 1592LogicalResult acc::ParallelOp::verify() {1593  if (failed(checkPrivateOperands<mlir::acc::PrivateOp,1594                                  mlir::acc::PrivateRecipeOp>(1595          *this, getPrivateOperands(), "private")))1596    return failure();1597  if (failed(checkPrivateOperands<mlir::acc::FirstprivateOp,1598                                  mlir::acc::FirstprivateRecipeOp>(1599          *this, getFirstprivateOperands(), "firstprivate")))1600    return failure();1601  if (failed(checkPrivateOperands<mlir::acc::ReductionOp,1602                                  mlir::acc::ReductionRecipeOp>(1603          *this, getReductionOperands(), "reduction")))1604    return failure();1605 1606  if (failed(verifyDeviceTypeAndSegmentCountMatch(1607          *this, getNumGangs(), getNumGangsSegmentsAttr(),1608          getNumGangsDeviceTypeAttr(), "num_gangs", 3)))1609    return failure();1610 1611  if (failed(verifyDeviceTypeAndSegmentCountMatch(1612          *this, getWaitOperands(), getWaitOperandsSegmentsAttr(),1613          getWaitOperandsDeviceTypeAttr(), "wait")))1614    return failure();1615 1616  if (failed(verifyDeviceTypeCountMatch(*this, getNumWorkers(),1617                                        getNumWorkersDeviceTypeAttr(),1618                                        "num_workers")))1619    return failure();1620 1621  if (failed(verifyDeviceTypeCountMatch(*this, getVectorLength(),1622                                        getVectorLengthDeviceTypeAttr(),1623                                        "vector_length")))1624    return failure();1625 1626  if (failed(verifyDeviceTypeCountMatch(*this, getAsyncOperands(),1627                                        getAsyncOperandsDeviceTypeAttr(),1628                                        "async")))1629    return failure();1630 1631  if (failed(checkWaitAndAsyncConflict<acc::ParallelOp>(*this)))1632    return failure();1633 1634  return checkDataOperands<acc::ParallelOp>(*this, getDataClauseOperands());1635}1636 1637static mlir::Value1638getValueInDeviceTypeSegment(std::optional<mlir::ArrayAttr> arrayAttr,1639                            mlir::Operation::operand_range range,1640                            mlir::acc::DeviceType deviceType) {1641  if (!arrayAttr)1642    return {};1643  if (auto pos = findSegment(*arrayAttr, deviceType))1644    return range[*pos];1645  return {};1646}1647 1648bool acc::ParallelOp::hasAsyncOnly() {1649  return hasAsyncOnly(mlir::acc::DeviceType::None);1650}1651 1652bool acc::ParallelOp::hasAsyncOnly(mlir::acc::DeviceType deviceType) {1653  return hasDeviceType(getAsyncOnly(), deviceType);1654}1655 1656mlir::Value acc::ParallelOp::getAsyncValue() {1657  return getAsyncValue(mlir::acc::DeviceType::None);1658}1659 1660mlir::Value acc::ParallelOp::getAsyncValue(mlir::acc::DeviceType deviceType) {1661  return getValueInDeviceTypeSegment(getAsyncOperandsDeviceType(),1662                                     getAsyncOperands(), deviceType);1663}1664 1665mlir::Value acc::ParallelOp::getNumWorkersValue() {1666  return getNumWorkersValue(mlir::acc::DeviceType::None);1667}1668 1669mlir::Value1670acc::ParallelOp::getNumWorkersValue(mlir::acc::DeviceType deviceType) {1671  return getValueInDeviceTypeSegment(getNumWorkersDeviceType(), getNumWorkers(),1672                                     deviceType);1673}1674 1675mlir::Value acc::ParallelOp::getVectorLengthValue() {1676  return getVectorLengthValue(mlir::acc::DeviceType::None);1677}1678 1679mlir::Value1680acc::ParallelOp::getVectorLengthValue(mlir::acc::DeviceType deviceType) {1681  return getValueInDeviceTypeSegment(getVectorLengthDeviceType(),1682                                     getVectorLength(), deviceType);1683}1684 1685mlir::Operation::operand_range ParallelOp::getNumGangsValues() {1686  return getNumGangsValues(mlir::acc::DeviceType::None);1687}1688 1689mlir::Operation::operand_range1690ParallelOp::getNumGangsValues(mlir::acc::DeviceType deviceType) {1691  return getValuesFromSegments(getNumGangsDeviceType(), getNumGangs(),1692                               getNumGangsSegments(), deviceType);1693}1694 1695bool acc::ParallelOp::hasWaitOnly() {1696  return hasWaitOnly(mlir::acc::DeviceType::None);1697}1698 1699bool acc::ParallelOp::hasWaitOnly(mlir::acc::DeviceType deviceType) {1700  return hasDeviceType(getWaitOnly(), deviceType);1701}1702 1703mlir::Operation::operand_range ParallelOp::getWaitValues() {1704  return getWaitValues(mlir::acc::DeviceType::None);1705}1706 1707mlir::Operation::operand_range1708ParallelOp::getWaitValues(mlir::acc::DeviceType deviceType) {1709  return getWaitValuesWithoutDevnum(1710      getWaitOperandsDeviceType(), getWaitOperands(), getWaitOperandsSegments(),1711      getHasWaitDevnum(), deviceType);1712}1713 1714mlir::Value ParallelOp::getWaitDevnum() {1715  return getWaitDevnum(mlir::acc::DeviceType::None);1716}1717 1718mlir::Value ParallelOp::getWaitDevnum(mlir::acc::DeviceType deviceType) {1719  return getWaitDevnumValue(getWaitOperandsDeviceType(), getWaitOperands(),1720                            getWaitOperandsSegments(), getHasWaitDevnum(),1721                            deviceType);1722}1723 1724void ParallelOp::build(mlir::OpBuilder &odsBuilder,1725                       mlir::OperationState &odsState,1726                       mlir::ValueRange numGangs, mlir::ValueRange numWorkers,1727                       mlir::ValueRange vectorLength,1728                       mlir::ValueRange asyncOperands,1729                       mlir::ValueRange waitOperands, mlir::Value ifCond,1730                       mlir::Value selfCond, mlir::ValueRange reductionOperands,1731                       mlir::ValueRange gangPrivateOperands,1732                       mlir::ValueRange gangFirstPrivateOperands,1733                       mlir::ValueRange dataClauseOperands) {1734  ParallelOp::build(1735      odsBuilder, odsState, asyncOperands, /*asyncOperandsDeviceType=*/nullptr,1736      /*asyncOnly=*/nullptr, waitOperands, /*waitOperandsSegments=*/nullptr,1737      /*waitOperandsDeviceType=*/nullptr, /*hasWaitDevnum=*/nullptr,1738      /*waitOnly=*/nullptr, numGangs, /*numGangsSegments=*/nullptr,1739      /*numGangsDeviceType=*/nullptr, numWorkers,1740      /*numWorkersDeviceType=*/nullptr, vectorLength,1741      /*vectorLengthDeviceType=*/nullptr, ifCond, selfCond,1742      /*selfAttr=*/nullptr, reductionOperands, gangPrivateOperands,1743      gangFirstPrivateOperands, dataClauseOperands,1744      /*defaultAttr=*/nullptr, /*combined=*/nullptr);1745}1746 1747void acc::ParallelOp::addNumWorkersOperand(1748    MLIRContext *context, mlir::Value newValue,1749    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {1750  setNumWorkersDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(1751      context, getNumWorkersDeviceTypeAttr(), effectiveDeviceTypes, newValue,1752      getNumWorkersMutable()));1753}1754void acc::ParallelOp::addVectorLengthOperand(1755    MLIRContext *context, mlir::Value newValue,1756    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {1757  setVectorLengthDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(1758      context, getVectorLengthDeviceTypeAttr(), effectiveDeviceTypes, newValue,1759      getVectorLengthMutable()));1760}1761 1762void acc::ParallelOp::addAsyncOnly(1763    MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {1764  setAsyncOnlyAttr(addDeviceTypeAffectedOperandHelper(1765      context, getAsyncOnlyAttr(), effectiveDeviceTypes));1766}1767 1768void acc::ParallelOp::addAsyncOperand(1769    MLIRContext *context, mlir::Value newValue,1770    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {1771  setAsyncOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(1772      context, getAsyncOperandsDeviceTypeAttr(), effectiveDeviceTypes, newValue,1773      getAsyncOperandsMutable()));1774}1775 1776void acc::ParallelOp::addNumGangsOperands(1777    MLIRContext *context, mlir::ValueRange newValues,1778    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {1779  llvm::SmallVector<int32_t> segments;1780  if (getNumGangsSegments())1781    llvm::copy(*getNumGangsSegments(), std::back_inserter(segments));1782 1783  setNumGangsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(1784      context, getNumGangsDeviceTypeAttr(), effectiveDeviceTypes, newValues,1785      getNumGangsMutable(), segments));1786 1787  setNumGangsSegments(segments);1788}1789void acc::ParallelOp::addWaitOnly(1790    MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {1791  setWaitOnlyAttr(addDeviceTypeAffectedOperandHelper(context, getWaitOnlyAttr(),1792                                                     effectiveDeviceTypes));1793}1794void acc::ParallelOp::addWaitOperands(1795    MLIRContext *context, bool hasDevnum, mlir::ValueRange newValues,1796    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {1797 1798  llvm::SmallVector<int32_t> segments;1799  if (getWaitOperandsSegments())1800    llvm::copy(*getWaitOperandsSegments(), std::back_inserter(segments));1801 1802  setWaitOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(1803      context, getWaitOperandsDeviceTypeAttr(), effectiveDeviceTypes, newValues,1804      getWaitOperandsMutable(), segments));1805  setWaitOperandsSegments(segments);1806 1807  llvm::SmallVector<mlir::Attribute> hasDevnums;1808  if (getHasWaitDevnumAttr())1809    llvm::copy(getHasWaitDevnumAttr(), std::back_inserter(hasDevnums));1810  hasDevnums.insert(1811      hasDevnums.end(),1812      std::max(effectiveDeviceTypes.size(), static_cast<size_t>(1)),1813      mlir::BoolAttr::get(context, hasDevnum));1814  setHasWaitDevnumAttr(mlir::ArrayAttr::get(context, hasDevnums));1815}1816 1817void acc::ParallelOp::addPrivatization(MLIRContext *context,1818                                       mlir::acc::PrivateOp op,1819                                       mlir::acc::PrivateRecipeOp recipe) {1820  op.setRecipeAttr(mlir::SymbolRefAttr::get(context, recipe.getSymName()));1821  getPrivateOperandsMutable().append(op.getResult());1822}1823 1824void acc::ParallelOp::addFirstPrivatization(1825    MLIRContext *context, mlir::acc::FirstprivateOp op,1826    mlir::acc::FirstprivateRecipeOp recipe) {1827  op.setRecipeAttr(mlir::SymbolRefAttr::get(context, recipe.getSymName()));1828  getFirstprivateOperandsMutable().append(op.getResult());1829}1830 1831void acc::ParallelOp::addReduction(MLIRContext *context,1832                                   mlir::acc::ReductionOp op,1833                                   mlir::acc::ReductionRecipeOp recipe) {1834  op.setRecipeAttr(mlir::SymbolRefAttr::get(context, recipe.getSymName()));1835  getReductionOperandsMutable().append(op.getResult());1836}1837 1838static ParseResult parseNumGangs(1839    mlir::OpAsmParser &parser,1840    llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &operands,1841    llvm::SmallVectorImpl<Type> &types, mlir::ArrayAttr &deviceTypes,1842    mlir::DenseI32ArrayAttr &segments) {1843  llvm::SmallVector<DeviceTypeAttr> attributes;1844  llvm::SmallVector<int32_t> seg;1845 1846  do {1847    if (failed(parser.parseLBrace()))1848      return failure();1849 1850    int32_t crtOperandsSize = operands.size();1851    if (failed(parser.parseCommaSeparatedList(1852            mlir::AsmParser::Delimiter::None, [&]() {1853              if (parser.parseOperand(operands.emplace_back()) ||1854                  parser.parseColonType(types.emplace_back()))1855                return failure();1856              return success();1857            })))1858      return failure();1859    seg.push_back(operands.size() - crtOperandsSize);1860 1861    if (failed(parser.parseRBrace()))1862      return failure();1863 1864    if (succeeded(parser.parseOptionalLSquare())) {1865      if (parser.parseAttribute(attributes.emplace_back()) ||1866          parser.parseRSquare())1867        return failure();1868    } else {1869      attributes.push_back(mlir::acc::DeviceTypeAttr::get(1870          parser.getContext(), mlir::acc::DeviceType::None));1871    }1872  } while (succeeded(parser.parseOptionalComma()));1873 1874  llvm::SmallVector<mlir::Attribute> arrayAttr(attributes.begin(),1875                                               attributes.end());1876  deviceTypes = ArrayAttr::get(parser.getContext(), arrayAttr);1877  segments = DenseI32ArrayAttr::get(parser.getContext(), seg);1878 1879  return success();1880}1881 1882static void printSingleDeviceType(mlir::OpAsmPrinter &p, mlir::Attribute attr) {1883  auto deviceTypeAttr = mlir::dyn_cast<mlir::acc::DeviceTypeAttr>(attr);1884  if (deviceTypeAttr.getValue() != mlir::acc::DeviceType::None)1885    p << " [" << attr << "]";1886}1887 1888static void printNumGangs(mlir::OpAsmPrinter &p, mlir::Operation *op,1889                          mlir::OperandRange operands, mlir::TypeRange types,1890                          std::optional<mlir::ArrayAttr> deviceTypes,1891                          std::optional<mlir::DenseI32ArrayAttr> segments) {1892  unsigned opIdx = 0;1893  llvm::interleaveComma(llvm::enumerate(*deviceTypes), p, [&](auto it) {1894    p << "{";1895    llvm::interleaveComma(1896        llvm::seq<int32_t>(0, (*segments)[it.index()]), p, [&](auto it) {1897          p << operands[opIdx] << " : " << operands[opIdx].getType();1898          ++opIdx;1899        });1900    p << "}";1901    printSingleDeviceType(p, it.value());1902  });1903}1904 1905static ParseResult parseDeviceTypeOperandsWithSegment(1906    mlir::OpAsmParser &parser,1907    llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &operands,1908    llvm::SmallVectorImpl<Type> &types, mlir::ArrayAttr &deviceTypes,1909    mlir::DenseI32ArrayAttr &segments) {1910  llvm::SmallVector<DeviceTypeAttr> attributes;1911  llvm::SmallVector<int32_t> seg;1912 1913  do {1914    if (failed(parser.parseLBrace()))1915      return failure();1916 1917    int32_t crtOperandsSize = operands.size();1918 1919    if (failed(parser.parseCommaSeparatedList(1920            mlir::AsmParser::Delimiter::None, [&]() {1921              if (parser.parseOperand(operands.emplace_back()) ||1922                  parser.parseColonType(types.emplace_back()))1923                return failure();1924              return success();1925            })))1926      return failure();1927 1928    seg.push_back(operands.size() - crtOperandsSize);1929 1930    if (failed(parser.parseRBrace()))1931      return failure();1932 1933    if (succeeded(parser.parseOptionalLSquare())) {1934      if (parser.parseAttribute(attributes.emplace_back()) ||1935          parser.parseRSquare())1936        return failure();1937    } else {1938      attributes.push_back(mlir::acc::DeviceTypeAttr::get(1939          parser.getContext(), mlir::acc::DeviceType::None));1940    }1941  } while (succeeded(parser.parseOptionalComma()));1942 1943  llvm::SmallVector<mlir::Attribute> arrayAttr(attributes.begin(),1944                                               attributes.end());1945  deviceTypes = ArrayAttr::get(parser.getContext(), arrayAttr);1946  segments = DenseI32ArrayAttr::get(parser.getContext(), seg);1947 1948  return success();1949}1950 1951static void printDeviceTypeOperandsWithSegment(1952    mlir::OpAsmPrinter &p, mlir::Operation *op, mlir::OperandRange operands,1953    mlir::TypeRange types, std::optional<mlir::ArrayAttr> deviceTypes,1954    std::optional<mlir::DenseI32ArrayAttr> segments) {1955  unsigned opIdx = 0;1956  llvm::interleaveComma(llvm::enumerate(*deviceTypes), p, [&](auto it) {1957    p << "{";1958    llvm::interleaveComma(1959        llvm::seq<int32_t>(0, (*segments)[it.index()]), p, [&](auto it) {1960          p << operands[opIdx] << " : " << operands[opIdx].getType();1961          ++opIdx;1962        });1963    p << "}";1964    printSingleDeviceType(p, it.value());1965  });1966}1967 1968static ParseResult parseWaitClause(1969    mlir::OpAsmParser &parser,1970    llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &operands,1971    llvm::SmallVectorImpl<Type> &types, mlir::ArrayAttr &deviceTypes,1972    mlir::DenseI32ArrayAttr &segments, mlir::ArrayAttr &hasDevNum,1973    mlir::ArrayAttr &keywordOnly) {1974  llvm::SmallVector<mlir::Attribute> deviceTypeAttrs, keywordAttrs, devnum;1975  llvm::SmallVector<int32_t> seg;1976 1977  bool needCommaBeforeOperands = false;1978 1979  // Keyword only1980  if (failed(parser.parseOptionalLParen())) {1981    keywordAttrs.push_back(mlir::acc::DeviceTypeAttr::get(1982        parser.getContext(), mlir::acc::DeviceType::None));1983    keywordOnly = ArrayAttr::get(parser.getContext(), keywordAttrs);1984    return success();1985  }1986 1987  // Parse keyword only attributes1988  if (succeeded(parser.parseOptionalLSquare())) {1989    if (failed(parser.parseCommaSeparatedList([&]() {1990          if (parser.parseAttribute(keywordAttrs.emplace_back()))1991            return failure();1992          return success();1993        })))1994      return failure();1995    if (parser.parseRSquare())1996      return failure();1997    needCommaBeforeOperands = true;1998  }1999 2000  if (needCommaBeforeOperands && failed(parser.parseComma()))2001    return failure();2002 2003  do {2004    if (failed(parser.parseLBrace()))2005      return failure();2006 2007    int32_t crtOperandsSize = operands.size();2008 2009    if (succeeded(parser.parseOptionalKeyword("devnum"))) {2010      if (failed(parser.parseColon()))2011        return failure();2012      devnum.push_back(BoolAttr::get(parser.getContext(), true));2013    } else {2014      devnum.push_back(BoolAttr::get(parser.getContext(), false));2015    }2016 2017    if (failed(parser.parseCommaSeparatedList(2018            mlir::AsmParser::Delimiter::None, [&]() {2019              if (parser.parseOperand(operands.emplace_back()) ||2020                  parser.parseColonType(types.emplace_back()))2021                return failure();2022              return success();2023            })))2024      return failure();2025 2026    seg.push_back(operands.size() - crtOperandsSize);2027 2028    if (failed(parser.parseRBrace()))2029      return failure();2030 2031    if (succeeded(parser.parseOptionalLSquare())) {2032      if (parser.parseAttribute(deviceTypeAttrs.emplace_back()) ||2033          parser.parseRSquare())2034        return failure();2035    } else {2036      deviceTypeAttrs.push_back(mlir::acc::DeviceTypeAttr::get(2037          parser.getContext(), mlir::acc::DeviceType::None));2038    }2039  } while (succeeded(parser.parseOptionalComma()));2040 2041  if (failed(parser.parseRParen()))2042    return failure();2043 2044  deviceTypes = ArrayAttr::get(parser.getContext(), deviceTypeAttrs);2045  keywordOnly = ArrayAttr::get(parser.getContext(), keywordAttrs);2046  segments = DenseI32ArrayAttr::get(parser.getContext(), seg);2047  hasDevNum = ArrayAttr::get(parser.getContext(), devnum);2048 2049  return success();2050}2051 2052static bool hasOnlyDeviceTypeNone(std::optional<mlir::ArrayAttr> attrs) {2053  if (!hasDeviceTypeValues(attrs))2054    return false;2055  if (attrs->size() != 1)2056    return false;2057  if (auto deviceTypeAttr =2058          mlir::dyn_cast<mlir::acc::DeviceTypeAttr>((*attrs)[0]))2059    return deviceTypeAttr.getValue() == mlir::acc::DeviceType::None;2060  return false;2061}2062 2063static void printWaitClause(mlir::OpAsmPrinter &p, mlir::Operation *op,2064                            mlir::OperandRange operands, mlir::TypeRange types,2065                            std::optional<mlir::ArrayAttr> deviceTypes,2066                            std::optional<mlir::DenseI32ArrayAttr> segments,2067                            std::optional<mlir::ArrayAttr> hasDevNum,2068                            std::optional<mlir::ArrayAttr> keywordOnly) {2069 2070  if (operands.begin() == operands.end() && hasOnlyDeviceTypeNone(keywordOnly))2071    return;2072 2073  p << "(";2074 2075  printDeviceTypes(p, keywordOnly);2076  if (hasDeviceTypeValues(keywordOnly) && hasDeviceTypeValues(deviceTypes))2077    p << ", ";2078 2079  if (hasDeviceTypeValues(deviceTypes)) {2080    unsigned opIdx = 0;2081    llvm::interleaveComma(llvm::enumerate(*deviceTypes), p, [&](auto it) {2082      p << "{";2083      auto boolAttr = mlir::dyn_cast<mlir::BoolAttr>((*hasDevNum)[it.index()]);2084      if (boolAttr && boolAttr.getValue())2085        p << "devnum: ";2086      llvm::interleaveComma(2087          llvm::seq<int32_t>(0, (*segments)[it.index()]), p, [&](auto it) {2088            p << operands[opIdx] << " : " << operands[opIdx].getType();2089            ++opIdx;2090          });2091      p << "}";2092      printSingleDeviceType(p, it.value());2093    });2094  }2095 2096  p << ")";2097}2098 2099static ParseResult parseDeviceTypeOperands(2100    mlir::OpAsmParser &parser,2101    llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &operands,2102    llvm::SmallVectorImpl<Type> &types, mlir::ArrayAttr &deviceTypes) {2103  llvm::SmallVector<DeviceTypeAttr> attributes;2104  if (failed(parser.parseCommaSeparatedList([&]() {2105        if (parser.parseOperand(operands.emplace_back()) ||2106            parser.parseColonType(types.emplace_back()))2107          return failure();2108        if (succeeded(parser.parseOptionalLSquare())) {2109          if (parser.parseAttribute(attributes.emplace_back()) ||2110              parser.parseRSquare())2111            return failure();2112        } else {2113          attributes.push_back(mlir::acc::DeviceTypeAttr::get(2114              parser.getContext(), mlir::acc::DeviceType::None));2115        }2116        return success();2117      })))2118    return failure();2119  llvm::SmallVector<mlir::Attribute> arrayAttr(attributes.begin(),2120                                               attributes.end());2121  deviceTypes = ArrayAttr::get(parser.getContext(), arrayAttr);2122  return success();2123}2124 2125static void2126printDeviceTypeOperands(mlir::OpAsmPrinter &p, mlir::Operation *op,2127                        mlir::OperandRange operands, mlir::TypeRange types,2128                        std::optional<mlir::ArrayAttr> deviceTypes) {2129  if (!hasDeviceTypeValues(deviceTypes))2130    return;2131  llvm::interleaveComma(llvm::zip(*deviceTypes, operands), p, [&](auto it) {2132    p << std::get<1>(it) << " : " << std::get<1>(it).getType();2133    printSingleDeviceType(p, std::get<0>(it));2134  });2135}2136 2137static ParseResult parseDeviceTypeOperandsWithKeywordOnly(2138    mlir::OpAsmParser &parser,2139    llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &operands,2140    llvm::SmallVectorImpl<Type> &types, mlir::ArrayAttr &deviceTypes,2141    mlir::ArrayAttr &keywordOnlyDeviceType) {2142 2143  llvm::SmallVector<mlir::Attribute> keywordOnlyDeviceTypeAttributes;2144  bool needCommaBeforeOperands = false;2145 2146  if (failed(parser.parseOptionalLParen())) {2147    // Keyword only2148    keywordOnlyDeviceTypeAttributes.push_back(mlir::acc::DeviceTypeAttr::get(2149        parser.getContext(), mlir::acc::DeviceType::None));2150    keywordOnlyDeviceType =2151        ArrayAttr::get(parser.getContext(), keywordOnlyDeviceTypeAttributes);2152    return success();2153  }2154 2155  // Parse keyword only attributes2156  if (succeeded(parser.parseOptionalLSquare())) {2157    // Parse keyword only attributes2158    if (failed(parser.parseCommaSeparatedList([&]() {2159          if (parser.parseAttribute(2160                  keywordOnlyDeviceTypeAttributes.emplace_back()))2161            return failure();2162          return success();2163        })))2164      return failure();2165    if (parser.parseRSquare())2166      return failure();2167    needCommaBeforeOperands = true;2168  }2169 2170  if (needCommaBeforeOperands && failed(parser.parseComma()))2171    return failure();2172 2173  llvm::SmallVector<DeviceTypeAttr> attributes;2174  if (failed(parser.parseCommaSeparatedList([&]() {2175        if (parser.parseOperand(operands.emplace_back()) ||2176            parser.parseColonType(types.emplace_back()))2177          return failure();2178        if (succeeded(parser.parseOptionalLSquare())) {2179          if (parser.parseAttribute(attributes.emplace_back()) ||2180              parser.parseRSquare())2181            return failure();2182        } else {2183          attributes.push_back(mlir::acc::DeviceTypeAttr::get(2184              parser.getContext(), mlir::acc::DeviceType::None));2185        }2186        return success();2187      })))2188    return failure();2189 2190  if (failed(parser.parseRParen()))2191    return failure();2192 2193  llvm::SmallVector<mlir::Attribute> arrayAttr(attributes.begin(),2194                                               attributes.end());2195  deviceTypes = ArrayAttr::get(parser.getContext(), arrayAttr);2196  return success();2197}2198 2199static void printDeviceTypeOperandsWithKeywordOnly(2200    mlir::OpAsmPrinter &p, mlir::Operation *op, mlir::OperandRange operands,2201    mlir::TypeRange types, std::optional<mlir::ArrayAttr> deviceTypes,2202    std::optional<mlir::ArrayAttr> keywordOnlyDeviceTypes) {2203 2204  if (operands.begin() == operands.end() &&2205      hasOnlyDeviceTypeNone(keywordOnlyDeviceTypes)) {2206    return;2207  }2208 2209  p << "(";2210  printDeviceTypes(p, keywordOnlyDeviceTypes);2211  if (hasDeviceTypeValues(keywordOnlyDeviceTypes) &&2212      hasDeviceTypeValues(deviceTypes))2213    p << ", ";2214  printDeviceTypeOperands(p, op, operands, types, deviceTypes);2215  p << ")";2216}2217 2218static ParseResult parseOperandWithKeywordOnly(2219    mlir::OpAsmParser &parser,2220    std::optional<OpAsmParser::UnresolvedOperand> &operand,2221    mlir::Type &operandType, mlir::UnitAttr &attr) {2222  // Keyword only2223  if (failed(parser.parseOptionalLParen())) {2224    attr = mlir::UnitAttr::get(parser.getContext());2225    return success();2226  }2227 2228  OpAsmParser::UnresolvedOperand op;2229  if (failed(parser.parseOperand(op)))2230    return failure();2231  operand = op;2232  if (failed(parser.parseColon()))2233    return failure();2234  if (failed(parser.parseType(operandType)))2235    return failure();2236  if (failed(parser.parseRParen()))2237    return failure();2238 2239  return success();2240}2241 2242static void printOperandWithKeywordOnly(mlir::OpAsmPrinter &p,2243                                        mlir::Operation *op,2244                                        std::optional<mlir::Value> operand,2245                                        mlir::Type operandType,2246                                        mlir::UnitAttr attr) {2247  if (attr)2248    return;2249 2250  p << "(";2251  p.printOperand(*operand);2252  p << " : ";2253  p.printType(operandType);2254  p << ")";2255}2256 2257static ParseResult parseOperandsWithKeywordOnly(2258    mlir::OpAsmParser &parser,2259    llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &operands,2260    llvm::SmallVectorImpl<Type> &types, mlir::UnitAttr &attr) {2261  // Keyword only2262  if (failed(parser.parseOptionalLParen())) {2263    attr = mlir::UnitAttr::get(parser.getContext());2264    return success();2265  }2266 2267  if (failed(parser.parseCommaSeparatedList([&]() {2268        if (parser.parseOperand(operands.emplace_back()))2269          return failure();2270        return success();2271      })))2272    return failure();2273  if (failed(parser.parseColon()))2274    return failure();2275  if (failed(parser.parseCommaSeparatedList([&]() {2276        if (parser.parseType(types.emplace_back()))2277          return failure();2278        return success();2279      })))2280    return failure();2281  if (failed(parser.parseRParen()))2282    return failure();2283 2284  return success();2285}2286 2287static void printOperandsWithKeywordOnly(mlir::OpAsmPrinter &p,2288                                         mlir::Operation *op,2289                                         mlir::OperandRange operands,2290                                         mlir::TypeRange types,2291                                         mlir::UnitAttr attr) {2292  if (attr)2293    return;2294 2295  p << "(";2296  llvm::interleaveComma(operands, p, [&](auto it) { p << it; });2297  p << " : ";2298  llvm::interleaveComma(types, p, [&](auto it) { p << it; });2299  p << ")";2300}2301 2302static ParseResult2303parseCombinedConstructsLoop(mlir::OpAsmParser &parser,2304                            mlir::acc::CombinedConstructsTypeAttr &attr) {2305  if (succeeded(parser.parseOptionalKeyword("kernels"))) {2306    attr = mlir::acc::CombinedConstructsTypeAttr::get(2307        parser.getContext(), mlir::acc::CombinedConstructsType::KernelsLoop);2308  } else if (succeeded(parser.parseOptionalKeyword("parallel"))) {2309    attr = mlir::acc::CombinedConstructsTypeAttr::get(2310        parser.getContext(), mlir::acc::CombinedConstructsType::ParallelLoop);2311  } else if (succeeded(parser.parseOptionalKeyword("serial"))) {2312    attr = mlir::acc::CombinedConstructsTypeAttr::get(2313        parser.getContext(), mlir::acc::CombinedConstructsType::SerialLoop);2314  } else {2315    parser.emitError(parser.getCurrentLocation(),2316                     "expected compute construct name");2317    return failure();2318  }2319  return success();2320}2321 2322static void2323printCombinedConstructsLoop(mlir::OpAsmPrinter &p, mlir::Operation *op,2324                            mlir::acc::CombinedConstructsTypeAttr attr) {2325  if (attr) {2326    switch (attr.getValue()) {2327    case mlir::acc::CombinedConstructsType::KernelsLoop:2328      p << "kernels";2329      break;2330    case mlir::acc::CombinedConstructsType::ParallelLoop:2331      p << "parallel";2332      break;2333    case mlir::acc::CombinedConstructsType::SerialLoop:2334      p << "serial";2335      break;2336    };2337  }2338}2339 2340//===----------------------------------------------------------------------===//2341// SerialOp2342//===----------------------------------------------------------------------===//2343 2344unsigned SerialOp::getNumDataOperands() {2345  return getReductionOperands().size() + getPrivateOperands().size() +2346         getFirstprivateOperands().size() + getDataClauseOperands().size();2347}2348 2349Value SerialOp::getDataOperand(unsigned i) {2350  unsigned numOptional = getAsyncOperands().size();2351  numOptional += getIfCond() ? 1 : 0;2352  numOptional += getSelfCond() ? 1 : 0;2353  return getOperand(getWaitOperands().size() + numOptional + i);2354}2355 2356bool acc::SerialOp::hasAsyncOnly() {2357  return hasAsyncOnly(mlir::acc::DeviceType::None);2358}2359 2360bool acc::SerialOp::hasAsyncOnly(mlir::acc::DeviceType deviceType) {2361  return hasDeviceType(getAsyncOnly(), deviceType);2362}2363 2364mlir::Value acc::SerialOp::getAsyncValue() {2365  return getAsyncValue(mlir::acc::DeviceType::None);2366}2367 2368mlir::Value acc::SerialOp::getAsyncValue(mlir::acc::DeviceType deviceType) {2369  return getValueInDeviceTypeSegment(getAsyncOperandsDeviceType(),2370                                     getAsyncOperands(), deviceType);2371}2372 2373bool acc::SerialOp::hasWaitOnly() {2374  return hasWaitOnly(mlir::acc::DeviceType::None);2375}2376 2377bool acc::SerialOp::hasWaitOnly(mlir::acc::DeviceType deviceType) {2378  return hasDeviceType(getWaitOnly(), deviceType);2379}2380 2381mlir::Operation::operand_range SerialOp::getWaitValues() {2382  return getWaitValues(mlir::acc::DeviceType::None);2383}2384 2385mlir::Operation::operand_range2386SerialOp::getWaitValues(mlir::acc::DeviceType deviceType) {2387  return getWaitValuesWithoutDevnum(2388      getWaitOperandsDeviceType(), getWaitOperands(), getWaitOperandsSegments(),2389      getHasWaitDevnum(), deviceType);2390}2391 2392mlir::Value SerialOp::getWaitDevnum() {2393  return getWaitDevnum(mlir::acc::DeviceType::None);2394}2395 2396mlir::Value SerialOp::getWaitDevnum(mlir::acc::DeviceType deviceType) {2397  return getWaitDevnumValue(getWaitOperandsDeviceType(), getWaitOperands(),2398                            getWaitOperandsSegments(), getHasWaitDevnum(),2399                            deviceType);2400}2401 2402LogicalResult acc::SerialOp::verify() {2403  if (failed(checkPrivateOperands<mlir::acc::PrivateOp,2404                                  mlir::acc::PrivateRecipeOp>(2405          *this, getPrivateOperands(), "private")))2406    return failure();2407  if (failed(checkPrivateOperands<mlir::acc::FirstprivateOp,2408                                  mlir::acc::FirstprivateRecipeOp>(2409          *this, getFirstprivateOperands(), "firstprivate")))2410    return failure();2411  if (failed(checkPrivateOperands<mlir::acc::ReductionOp,2412                                  mlir::acc::ReductionRecipeOp>(2413          *this, getReductionOperands(), "reduction")))2414    return failure();2415 2416  if (failed(verifyDeviceTypeAndSegmentCountMatch(2417          *this, getWaitOperands(), getWaitOperandsSegmentsAttr(),2418          getWaitOperandsDeviceTypeAttr(), "wait")))2419    return failure();2420 2421  if (failed(verifyDeviceTypeCountMatch(*this, getAsyncOperands(),2422                                        getAsyncOperandsDeviceTypeAttr(),2423                                        "async")))2424    return failure();2425 2426  if (failed(checkWaitAndAsyncConflict<acc::SerialOp>(*this)))2427    return failure();2428 2429  return checkDataOperands<acc::SerialOp>(*this, getDataClauseOperands());2430}2431 2432void acc::SerialOp::addAsyncOnly(2433    MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {2434  setAsyncOnlyAttr(addDeviceTypeAffectedOperandHelper(2435      context, getAsyncOnlyAttr(), effectiveDeviceTypes));2436}2437 2438void acc::SerialOp::addAsyncOperand(2439    MLIRContext *context, mlir::Value newValue,2440    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {2441  setAsyncOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(2442      context, getAsyncOperandsDeviceTypeAttr(), effectiveDeviceTypes, newValue,2443      getAsyncOperandsMutable()));2444}2445 2446void acc::SerialOp::addWaitOnly(2447    MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {2448  setWaitOnlyAttr(addDeviceTypeAffectedOperandHelper(context, getWaitOnlyAttr(),2449                                                     effectiveDeviceTypes));2450}2451void acc::SerialOp::addWaitOperands(2452    MLIRContext *context, bool hasDevnum, mlir::ValueRange newValues,2453    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {2454 2455  llvm::SmallVector<int32_t> segments;2456  if (getWaitOperandsSegments())2457    llvm::copy(*getWaitOperandsSegments(), std::back_inserter(segments));2458 2459  setWaitOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(2460      context, getWaitOperandsDeviceTypeAttr(), effectiveDeviceTypes, newValues,2461      getWaitOperandsMutable(), segments));2462  setWaitOperandsSegments(segments);2463 2464  llvm::SmallVector<mlir::Attribute> hasDevnums;2465  if (getHasWaitDevnumAttr())2466    llvm::copy(getHasWaitDevnumAttr(), std::back_inserter(hasDevnums));2467  hasDevnums.insert(2468      hasDevnums.end(),2469      std::max(effectiveDeviceTypes.size(), static_cast<size_t>(1)),2470      mlir::BoolAttr::get(context, hasDevnum));2471  setHasWaitDevnumAttr(mlir::ArrayAttr::get(context, hasDevnums));2472}2473 2474void acc::SerialOp::addPrivatization(MLIRContext *context,2475                                     mlir::acc::PrivateOp op,2476                                     mlir::acc::PrivateRecipeOp recipe) {2477  op.setRecipeAttr(mlir::SymbolRefAttr::get(context, recipe.getSymName()));2478  getPrivateOperandsMutable().append(op.getResult());2479}2480 2481void acc::SerialOp::addFirstPrivatization(2482    MLIRContext *context, mlir::acc::FirstprivateOp op,2483    mlir::acc::FirstprivateRecipeOp recipe) {2484  op.setRecipeAttr(mlir::SymbolRefAttr::get(context, recipe.getSymName()));2485  getFirstprivateOperandsMutable().append(op.getResult());2486}2487 2488void acc::SerialOp::addReduction(MLIRContext *context,2489                                 mlir::acc::ReductionOp op,2490                                 mlir::acc::ReductionRecipeOp recipe) {2491  op.setRecipeAttr(mlir::SymbolRefAttr::get(context, recipe.getSymName()));2492  getReductionOperandsMutable().append(op.getResult());2493}2494 2495//===----------------------------------------------------------------------===//2496// KernelsOp2497//===----------------------------------------------------------------------===//2498 2499unsigned KernelsOp::getNumDataOperands() {2500  return getDataClauseOperands().size();2501}2502 2503Value KernelsOp::getDataOperand(unsigned i) {2504  unsigned numOptional = getAsyncOperands().size();2505  numOptional += getWaitOperands().size();2506  numOptional += getNumGangs().size();2507  numOptional += getNumWorkers().size();2508  numOptional += getVectorLength().size();2509  numOptional += getIfCond() ? 1 : 0;2510  numOptional += getSelfCond() ? 1 : 0;2511  return getOperand(numOptional + i);2512}2513 2514bool acc::KernelsOp::hasAsyncOnly() {2515  return hasAsyncOnly(mlir::acc::DeviceType::None);2516}2517 2518bool acc::KernelsOp::hasAsyncOnly(mlir::acc::DeviceType deviceType) {2519  return hasDeviceType(getAsyncOnly(), deviceType);2520}2521 2522mlir::Value acc::KernelsOp::getAsyncValue() {2523  return getAsyncValue(mlir::acc::DeviceType::None);2524}2525 2526mlir::Value acc::KernelsOp::getAsyncValue(mlir::acc::DeviceType deviceType) {2527  return getValueInDeviceTypeSegment(getAsyncOperandsDeviceType(),2528                                     getAsyncOperands(), deviceType);2529}2530 2531mlir::Value acc::KernelsOp::getNumWorkersValue() {2532  return getNumWorkersValue(mlir::acc::DeviceType::None);2533}2534 2535mlir::Value2536acc::KernelsOp::getNumWorkersValue(mlir::acc::DeviceType deviceType) {2537  return getValueInDeviceTypeSegment(getNumWorkersDeviceType(), getNumWorkers(),2538                                     deviceType);2539}2540 2541mlir::Value acc::KernelsOp::getVectorLengthValue() {2542  return getVectorLengthValue(mlir::acc::DeviceType::None);2543}2544 2545mlir::Value2546acc::KernelsOp::getVectorLengthValue(mlir::acc::DeviceType deviceType) {2547  return getValueInDeviceTypeSegment(getVectorLengthDeviceType(),2548                                     getVectorLength(), deviceType);2549}2550 2551mlir::Operation::operand_range KernelsOp::getNumGangsValues() {2552  return getNumGangsValues(mlir::acc::DeviceType::None);2553}2554 2555mlir::Operation::operand_range2556KernelsOp::getNumGangsValues(mlir::acc::DeviceType deviceType) {2557  return getValuesFromSegments(getNumGangsDeviceType(), getNumGangs(),2558                               getNumGangsSegments(), deviceType);2559}2560 2561bool acc::KernelsOp::hasWaitOnly() {2562  return hasWaitOnly(mlir::acc::DeviceType::None);2563}2564 2565bool acc::KernelsOp::hasWaitOnly(mlir::acc::DeviceType deviceType) {2566  return hasDeviceType(getWaitOnly(), deviceType);2567}2568 2569mlir::Operation::operand_range KernelsOp::getWaitValues() {2570  return getWaitValues(mlir::acc::DeviceType::None);2571}2572 2573mlir::Operation::operand_range2574KernelsOp::getWaitValues(mlir::acc::DeviceType deviceType) {2575  return getWaitValuesWithoutDevnum(2576      getWaitOperandsDeviceType(), getWaitOperands(), getWaitOperandsSegments(),2577      getHasWaitDevnum(), deviceType);2578}2579 2580mlir::Value KernelsOp::getWaitDevnum() {2581  return getWaitDevnum(mlir::acc::DeviceType::None);2582}2583 2584mlir::Value KernelsOp::getWaitDevnum(mlir::acc::DeviceType deviceType) {2585  return getWaitDevnumValue(getWaitOperandsDeviceType(), getWaitOperands(),2586                            getWaitOperandsSegments(), getHasWaitDevnum(),2587                            deviceType);2588}2589 2590LogicalResult acc::KernelsOp::verify() {2591  if (failed(verifyDeviceTypeAndSegmentCountMatch(2592          *this, getNumGangs(), getNumGangsSegmentsAttr(),2593          getNumGangsDeviceTypeAttr(), "num_gangs", 3)))2594    return failure();2595 2596  if (failed(verifyDeviceTypeAndSegmentCountMatch(2597          *this, getWaitOperands(), getWaitOperandsSegmentsAttr(),2598          getWaitOperandsDeviceTypeAttr(), "wait")))2599    return failure();2600 2601  if (failed(verifyDeviceTypeCountMatch(*this, getNumWorkers(),2602                                        getNumWorkersDeviceTypeAttr(),2603                                        "num_workers")))2604    return failure();2605 2606  if (failed(verifyDeviceTypeCountMatch(*this, getVectorLength(),2607                                        getVectorLengthDeviceTypeAttr(),2608                                        "vector_length")))2609    return failure();2610 2611  if (failed(verifyDeviceTypeCountMatch(*this, getAsyncOperands(),2612                                        getAsyncOperandsDeviceTypeAttr(),2613                                        "async")))2614    return failure();2615 2616  if (failed(checkWaitAndAsyncConflict<acc::KernelsOp>(*this)))2617    return failure();2618 2619  return checkDataOperands<acc::KernelsOp>(*this, getDataClauseOperands());2620}2621 2622void acc::KernelsOp::addNumWorkersOperand(2623    MLIRContext *context, mlir::Value newValue,2624    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {2625  setNumWorkersDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(2626      context, getNumWorkersDeviceTypeAttr(), effectiveDeviceTypes, newValue,2627      getNumWorkersMutable()));2628}2629 2630void acc::KernelsOp::addVectorLengthOperand(2631    MLIRContext *context, mlir::Value newValue,2632    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {2633  setVectorLengthDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(2634      context, getVectorLengthDeviceTypeAttr(), effectiveDeviceTypes, newValue,2635      getVectorLengthMutable()));2636}2637void acc::KernelsOp::addAsyncOnly(2638    MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {2639  setAsyncOnlyAttr(addDeviceTypeAffectedOperandHelper(2640      context, getAsyncOnlyAttr(), effectiveDeviceTypes));2641}2642 2643void acc::KernelsOp::addAsyncOperand(2644    MLIRContext *context, mlir::Value newValue,2645    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {2646  setAsyncOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(2647      context, getAsyncOperandsDeviceTypeAttr(), effectiveDeviceTypes, newValue,2648      getAsyncOperandsMutable()));2649}2650 2651void acc::KernelsOp::addNumGangsOperands(2652    MLIRContext *context, mlir::ValueRange newValues,2653    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {2654  llvm::SmallVector<int32_t> segments;2655  if (getNumGangsSegmentsAttr())2656    llvm::copy(*getNumGangsSegments(), std::back_inserter(segments));2657 2658  setNumGangsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(2659      context, getNumGangsDeviceTypeAttr(), effectiveDeviceTypes, newValues,2660      getNumGangsMutable(), segments));2661 2662  setNumGangsSegments(segments);2663}2664 2665void acc::KernelsOp::addWaitOnly(2666    MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {2667  setWaitOnlyAttr(addDeviceTypeAffectedOperandHelper(context, getWaitOnlyAttr(),2668                                                     effectiveDeviceTypes));2669}2670void acc::KernelsOp::addWaitOperands(2671    MLIRContext *context, bool hasDevnum, mlir::ValueRange newValues,2672    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {2673 2674  llvm::SmallVector<int32_t> segments;2675  if (getWaitOperandsSegments())2676    llvm::copy(*getWaitOperandsSegments(), std::back_inserter(segments));2677 2678  setWaitOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(2679      context, getWaitOperandsDeviceTypeAttr(), effectiveDeviceTypes, newValues,2680      getWaitOperandsMutable(), segments));2681  setWaitOperandsSegments(segments);2682 2683  llvm::SmallVector<mlir::Attribute> hasDevnums;2684  if (getHasWaitDevnumAttr())2685    llvm::copy(getHasWaitDevnumAttr(), std::back_inserter(hasDevnums));2686  hasDevnums.insert(2687      hasDevnums.end(),2688      std::max(effectiveDeviceTypes.size(), static_cast<size_t>(1)),2689      mlir::BoolAttr::get(context, hasDevnum));2690  setHasWaitDevnumAttr(mlir::ArrayAttr::get(context, hasDevnums));2691}2692 2693//===----------------------------------------------------------------------===//2694// HostDataOp2695//===----------------------------------------------------------------------===//2696 2697LogicalResult acc::HostDataOp::verify() {2698  if (getDataClauseOperands().empty())2699    return emitError("at least one operand must appear on the host_data "2700                     "operation");2701 2702  for (mlir::Value operand : getDataClauseOperands())2703    if (!mlir::isa<acc::UseDeviceOp>(operand.getDefiningOp()))2704      return emitError("expect data entry operation as defining op");2705  return success();2706}2707 2708void acc::HostDataOp::getCanonicalizationPatterns(RewritePatternSet &results,2709                                                  MLIRContext *context) {2710  results.add<RemoveConstantIfConditionWithRegion<HostDataOp>>(context);2711}2712 2713//===----------------------------------------------------------------------===//2714// KernelEnvironmentOp2715//===----------------------------------------------------------------------===//2716 2717void acc::KernelEnvironmentOp::getCanonicalizationPatterns(2718    RewritePatternSet &results, MLIRContext *context) {2719  results.add<RemoveEmptyKernelEnvironment>(context);2720}2721 2722//===----------------------------------------------------------------------===//2723// LoopOp2724//===----------------------------------------------------------------------===//2725 2726static ParseResult parseGangValue(2727    OpAsmParser &parser, llvm::StringRef keyword,2728    llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &operands,2729    llvm::SmallVectorImpl<Type> &types,2730    llvm::SmallVector<GangArgTypeAttr> &attributes, GangArgTypeAttr gangArgType,2731    bool &needCommaBetweenValues, bool &newValue) {2732  if (succeeded(parser.parseOptionalKeyword(keyword))) {2733    if (parser.parseEqual())2734      return failure();2735    if (parser.parseOperand(operands.emplace_back()) ||2736        parser.parseColonType(types.emplace_back()))2737      return failure();2738    attributes.push_back(gangArgType);2739    needCommaBetweenValues = true;2740    newValue = true;2741  }2742  return success();2743}2744 2745static ParseResult parseGangClause(2746    OpAsmParser &parser,2747    llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &gangOperands,2748    llvm::SmallVectorImpl<Type> &gangOperandsType, mlir::ArrayAttr &gangArgType,2749    mlir::ArrayAttr &deviceType, mlir::DenseI32ArrayAttr &segments,2750    mlir::ArrayAttr &gangOnlyDeviceType) {2751  llvm::SmallVector<GangArgTypeAttr> gangArgTypeAttributes;2752  llvm::SmallVector<mlir::Attribute> deviceTypeAttributes;2753  llvm::SmallVector<mlir::Attribute> gangOnlyDeviceTypeAttributes;2754  llvm::SmallVector<int32_t> seg;2755  bool needCommaBetweenValues = false;2756  bool needCommaBeforeOperands = false;2757 2758  if (failed(parser.parseOptionalLParen())) {2759    // Gang only keyword2760    gangOnlyDeviceTypeAttributes.push_back(mlir::acc::DeviceTypeAttr::get(2761        parser.getContext(), mlir::acc::DeviceType::None));2762    gangOnlyDeviceType =2763        ArrayAttr::get(parser.getContext(), gangOnlyDeviceTypeAttributes);2764    return success();2765  }2766 2767  // Parse gang only attributes2768  if (succeeded(parser.parseOptionalLSquare())) {2769    // Parse gang only attributes2770    if (failed(parser.parseCommaSeparatedList([&]() {2771          if (parser.parseAttribute(2772                  gangOnlyDeviceTypeAttributes.emplace_back()))2773            return failure();2774          return success();2775        })))2776      return failure();2777    if (parser.parseRSquare())2778      return failure();2779    needCommaBeforeOperands = true;2780  }2781 2782  auto argNum = mlir::acc::GangArgTypeAttr::get(parser.getContext(),2783                                                mlir::acc::GangArgType::Num);2784  auto argDim = mlir::acc::GangArgTypeAttr::get(parser.getContext(),2785                                                mlir::acc::GangArgType::Dim);2786  auto argStatic = mlir::acc::GangArgTypeAttr::get(2787      parser.getContext(), mlir::acc::GangArgType::Static);2788 2789  do {2790    if (needCommaBeforeOperands) {2791      needCommaBeforeOperands = false;2792      continue;2793    }2794 2795    if (failed(parser.parseLBrace()))2796      return failure();2797 2798    int32_t crtOperandsSize = gangOperands.size();2799    while (true) {2800      bool newValue = false;2801      bool needValue = false;2802      if (needCommaBetweenValues) {2803        if (succeeded(parser.parseOptionalComma()))2804          needValue = true; // expect a new value after comma.2805        else2806          break;2807      }2808 2809      if (failed(parseGangValue(parser, LoopOp::getGangNumKeyword(),2810                                gangOperands, gangOperandsType,2811                                gangArgTypeAttributes, argNum,2812                                needCommaBetweenValues, newValue)))2813        return failure();2814      if (failed(parseGangValue(parser, LoopOp::getGangDimKeyword(),2815                                gangOperands, gangOperandsType,2816                                gangArgTypeAttributes, argDim,2817                                needCommaBetweenValues, newValue)))2818        return failure();2819      if (failed(parseGangValue(parser, LoopOp::getGangStaticKeyword(),2820                                gangOperands, gangOperandsType,2821                                gangArgTypeAttributes, argStatic,2822                                needCommaBetweenValues, newValue)))2823        return failure();2824 2825      if (!newValue && needValue) {2826        parser.emitError(parser.getCurrentLocation(),2827                         "new value expected after comma");2828        return failure();2829      }2830 2831      if (!newValue)2832        break;2833    }2834 2835    if (gangOperands.empty())2836      return parser.emitError(2837          parser.getCurrentLocation(),2838          "expect at least one of num, dim or static values");2839 2840    if (failed(parser.parseRBrace()))2841      return failure();2842 2843    if (succeeded(parser.parseOptionalLSquare())) {2844      if (parser.parseAttribute(deviceTypeAttributes.emplace_back()) ||2845          parser.parseRSquare())2846        return failure();2847    } else {2848      deviceTypeAttributes.push_back(mlir::acc::DeviceTypeAttr::get(2849          parser.getContext(), mlir::acc::DeviceType::None));2850    }2851 2852    seg.push_back(gangOperands.size() - crtOperandsSize);2853 2854  } while (succeeded(parser.parseOptionalComma()));2855 2856  if (failed(parser.parseRParen()))2857    return failure();2858 2859  llvm::SmallVector<mlir::Attribute> arrayAttr(gangArgTypeAttributes.begin(),2860                                               gangArgTypeAttributes.end());2861  gangArgType = ArrayAttr::get(parser.getContext(), arrayAttr);2862  deviceType = ArrayAttr::get(parser.getContext(), deviceTypeAttributes);2863 2864  llvm::SmallVector<mlir::Attribute> gangOnlyAttr(2865      gangOnlyDeviceTypeAttributes.begin(), gangOnlyDeviceTypeAttributes.end());2866  gangOnlyDeviceType = ArrayAttr::get(parser.getContext(), gangOnlyAttr);2867 2868  segments = DenseI32ArrayAttr::get(parser.getContext(), seg);2869  return success();2870}2871 2872void printGangClause(OpAsmPrinter &p, Operation *op,2873                     mlir::OperandRange operands, mlir::TypeRange types,2874                     std::optional<mlir::ArrayAttr> gangArgTypes,2875                     std::optional<mlir::ArrayAttr> deviceTypes,2876                     std::optional<mlir::DenseI32ArrayAttr> segments,2877                     std::optional<mlir::ArrayAttr> gangOnlyDeviceTypes) {2878 2879  if (operands.begin() == operands.end() &&2880      hasOnlyDeviceTypeNone(gangOnlyDeviceTypes)) {2881    return;2882  }2883 2884  p << "(";2885 2886  printDeviceTypes(p, gangOnlyDeviceTypes);2887 2888  if (hasDeviceTypeValues(gangOnlyDeviceTypes) &&2889      hasDeviceTypeValues(deviceTypes))2890    p << ", ";2891 2892  if (hasDeviceTypeValues(deviceTypes)) {2893    unsigned opIdx = 0;2894    llvm::interleaveComma(llvm::enumerate(*deviceTypes), p, [&](auto it) {2895      p << "{";2896      llvm::interleaveComma(2897          llvm::seq<int32_t>(0, (*segments)[it.index()]), p, [&](auto it) {2898            auto gangArgTypeAttr = mlir::dyn_cast<mlir::acc::GangArgTypeAttr>(2899                (*gangArgTypes)[opIdx]);2900            if (gangArgTypeAttr.getValue() == mlir::acc::GangArgType::Num)2901              p << LoopOp::getGangNumKeyword();2902            else if (gangArgTypeAttr.getValue() == mlir::acc::GangArgType::Dim)2903              p << LoopOp::getGangDimKeyword();2904            else if (gangArgTypeAttr.getValue() ==2905                     mlir::acc::GangArgType::Static)2906              p << LoopOp::getGangStaticKeyword();2907            p << "=" << operands[opIdx] << " : " << operands[opIdx].getType();2908            ++opIdx;2909          });2910      p << "}";2911      printSingleDeviceType(p, it.value());2912    });2913  }2914  p << ")";2915}2916 2917bool hasDuplicateDeviceTypes(2918    std::optional<mlir::ArrayAttr> segments,2919    llvm::SmallSet<mlir::acc::DeviceType, 3> &deviceTypes) {2920  if (!segments)2921    return false;2922  for (auto attr : *segments) {2923    auto deviceTypeAttr = mlir::dyn_cast<mlir::acc::DeviceTypeAttr>(attr);2924    if (!deviceTypes.insert(deviceTypeAttr.getValue()).second)2925      return true;2926  }2927  return false;2928}2929 2930/// Check for duplicates in the DeviceType array attribute.2931LogicalResult checkDeviceTypes(mlir::ArrayAttr deviceTypes) {2932  llvm::SmallSet<mlir::acc::DeviceType, 3> crtDeviceTypes;2933  if (!deviceTypes)2934    return success();2935  for (auto attr : deviceTypes) {2936    auto deviceTypeAttr =2937        mlir::dyn_cast_or_null<mlir::acc::DeviceTypeAttr>(attr);2938    if (!deviceTypeAttr)2939      return failure();2940    if (!crtDeviceTypes.insert(deviceTypeAttr.getValue()).second)2941      return failure();2942  }2943  return success();2944}2945 2946LogicalResult acc::LoopOp::verify() {2947  if (getUpperbound().size() != getStep().size())2948    return emitError() << "number of upperbounds expected to be the same as "2949                          "number of steps";2950 2951  if (getUpperbound().size() != getLowerbound().size())2952    return emitError() << "number of upperbounds expected to be the same as "2953                          "number of lowerbounds";2954 2955  if (!getUpperbound().empty() && getInclusiveUpperbound() &&2956      (getUpperbound().size() != getInclusiveUpperbound()->size()))2957    return emitError() << "inclusiveUpperbound size is expected to be the same"2958                       << " as upperbound size";2959 2960  // Check collapse2961  if (getCollapseAttr() && !getCollapseDeviceTypeAttr())2962    return emitOpError() << "collapse device_type attr must be define when"2963                         << " collapse attr is present";2964 2965  if (getCollapseAttr() && getCollapseDeviceTypeAttr() &&2966      getCollapseAttr().getValue().size() !=2967          getCollapseDeviceTypeAttr().getValue().size())2968    return emitOpError() << "collapse attribute count must match collapse"2969                         << " device_type count";2970  if (failed(checkDeviceTypes(getCollapseDeviceTypeAttr())))2971    return emitOpError()2972           << "duplicate device_type found in collapseDeviceType attribute";2973 2974  // Check gang2975  if (!getGangOperands().empty()) {2976    if (!getGangOperandsArgType())2977      return emitOpError() << "gangOperandsArgType attribute must be defined"2978                           << " when gang operands are present";2979 2980    if (getGangOperands().size() !=2981        getGangOperandsArgTypeAttr().getValue().size())2982      return emitOpError() << "gangOperandsArgType attribute count must match"2983                           << " gangOperands count";2984  }2985  if (getGangAttr() && failed(checkDeviceTypes(getGangAttr())))2986    return emitOpError() << "duplicate device_type found in gang attribute";2987 2988  if (failed(verifyDeviceTypeAndSegmentCountMatch(2989          *this, getGangOperands(), getGangOperandsSegmentsAttr(),2990          getGangOperandsDeviceTypeAttr(), "gang")))2991    return failure();2992 2993  // Check worker2994  if (failed(checkDeviceTypes(getWorkerAttr())))2995    return emitOpError() << "duplicate device_type found in worker attribute";2996  if (failed(checkDeviceTypes(getWorkerNumOperandsDeviceTypeAttr())))2997    return emitOpError() << "duplicate device_type found in "2998                            "workerNumOperandsDeviceType attribute";2999  if (failed(verifyDeviceTypeCountMatch(*this, getWorkerNumOperands(),3000                                        getWorkerNumOperandsDeviceTypeAttr(),3001                                        "worker")))3002    return failure();3003 3004  // Check vector3005  if (failed(checkDeviceTypes(getVectorAttr())))3006    return emitOpError() << "duplicate device_type found in vector attribute";3007  if (failed(checkDeviceTypes(getVectorOperandsDeviceTypeAttr())))3008    return emitOpError() << "duplicate device_type found in "3009                            "vectorOperandsDeviceType attribute";3010  if (failed(verifyDeviceTypeCountMatch(*this, getVectorOperands(),3011                                        getVectorOperandsDeviceTypeAttr(),3012                                        "vector")))3013    return failure();3014 3015  if (failed(verifyDeviceTypeAndSegmentCountMatch(3016          *this, getTileOperands(), getTileOperandsSegmentsAttr(),3017          getTileOperandsDeviceTypeAttr(), "tile")))3018    return failure();3019 3020  // auto, independent and seq attribute are mutually exclusive.3021  llvm::SmallSet<mlir::acc::DeviceType, 3> deviceTypes;3022  if (hasDuplicateDeviceTypes(getAuto_(), deviceTypes) ||3023      hasDuplicateDeviceTypes(getIndependent(), deviceTypes) ||3024      hasDuplicateDeviceTypes(getSeq(), deviceTypes)) {3025    return emitError() << "only one of auto, independent, seq can be present "3026                          "at the same time";3027  }3028 3029  // Check that at least one of auto, independent, or seq is present3030  // for the device-independent default clauses.3031  auto hasDeviceNone = [](mlir::acc::DeviceTypeAttr attr) -> bool {3032    return attr.getValue() == mlir::acc::DeviceType::None;3033  };3034  bool hasDefaultSeq =3035      getSeqAttr()3036          ? llvm::any_of(getSeqAttr().getAsRange<mlir::acc::DeviceTypeAttr>(),3037                         hasDeviceNone)3038          : false;3039  bool hasDefaultIndependent =3040      getIndependentAttr()3041          ? llvm::any_of(3042                getIndependentAttr().getAsRange<mlir::acc::DeviceTypeAttr>(),3043                hasDeviceNone)3044          : false;3045  bool hasDefaultAuto =3046      getAuto_Attr()3047          ? llvm::any_of(getAuto_Attr().getAsRange<mlir::acc::DeviceTypeAttr>(),3048                         hasDeviceNone)3049          : false;3050  if (!hasDefaultSeq && !hasDefaultIndependent && !hasDefaultAuto) {3051    return emitError()3052           << "at least one of auto, independent, seq must be present";3053  }3054 3055  // Gang, worker and vector are incompatible with seq.3056  if (getSeqAttr()) {3057    for (auto attr : getSeqAttr()) {3058      auto deviceTypeAttr = mlir::dyn_cast<mlir::acc::DeviceTypeAttr>(attr);3059      if (hasVector(deviceTypeAttr.getValue()) ||3060          getVectorValue(deviceTypeAttr.getValue()) ||3061          hasWorker(deviceTypeAttr.getValue()) ||3062          getWorkerValue(deviceTypeAttr.getValue()) ||3063          hasGang(deviceTypeAttr.getValue()) ||3064          getGangValue(mlir::acc::GangArgType::Num,3065                       deviceTypeAttr.getValue()) ||3066          getGangValue(mlir::acc::GangArgType::Dim,3067                       deviceTypeAttr.getValue()) ||3068          getGangValue(mlir::acc::GangArgType::Static,3069                       deviceTypeAttr.getValue()))3070        return emitError() << "gang, worker or vector cannot appear with seq";3071    }3072  }3073 3074  if (failed(checkPrivateOperands<mlir::acc::PrivateOp,3075                                  mlir::acc::PrivateRecipeOp>(3076          *this, getPrivateOperands(), "private")))3077    return failure();3078 3079  if (failed(checkPrivateOperands<mlir::acc::FirstprivateOp,3080                                  mlir::acc::FirstprivateRecipeOp>(3081          *this, getFirstprivateOperands(), "firstprivate")))3082    return failure();3083 3084  if (failed(checkPrivateOperands<mlir::acc::ReductionOp,3085                                  mlir::acc::ReductionRecipeOp>(3086          *this, getReductionOperands(), "reduction")))3087    return failure();3088 3089  if (getCombined().has_value() &&3090      (getCombined().value() != acc::CombinedConstructsType::ParallelLoop &&3091       getCombined().value() != acc::CombinedConstructsType::KernelsLoop &&3092       getCombined().value() != acc::CombinedConstructsType::SerialLoop)) {3093    return emitError("unexpected combined constructs attribute");3094  }3095 3096  // Check non-empty body().3097  if (getRegion().empty())3098    return emitError("expected non-empty body.");3099 3100  if (getUnstructured()) {3101    if (!isContainerLike())3102      return emitError(3103          "unstructured acc.loop must not have induction variables");3104  } else if (isContainerLike()) {3105    // When it is container-like - it is expected to hold a loop-like operation.3106    // Obtain the maximum collapse count - we use this to check that there3107    // are enough loops contained.3108    uint64_t collapseCount = getCollapseValue().value_or(1);3109    if (getCollapseAttr()) {3110      for (auto collapseEntry : getCollapseAttr()) {3111        auto intAttr = mlir::dyn_cast<IntegerAttr>(collapseEntry);3112        if (intAttr.getValue().getZExtValue() > collapseCount)3113          collapseCount = intAttr.getValue().getZExtValue();3114      }3115    }3116 3117    // We want to check that we find enough loop-like operations inside.3118    // PreOrder walk allows us to walk in a breadth-first manner at each nesting3119    // level.3120    mlir::Operation *expectedParent = this->getOperation();3121    bool foundSibling = false;3122    getRegion().walk<WalkOrder::PreOrder>([&](mlir::Operation *op) {3123      if (mlir::isa<mlir::LoopLikeOpInterface>(op)) {3124        // This effectively checks that we are not looking at a sibling loop.3125        if (op->getParentOfType<mlir::LoopLikeOpInterface>() !=3126            expectedParent) {3127          foundSibling = true;3128          return mlir::WalkResult::interrupt();3129        }3130 3131        collapseCount--;3132        expectedParent = op;3133      }3134      // We found enough contained loops.3135      if (collapseCount == 0)3136        return mlir::WalkResult::interrupt();3137      return mlir::WalkResult::advance();3138    });3139 3140    if (foundSibling)3141      return emitError("found sibling loops inside container-like acc.loop");3142    if (collapseCount != 0)3143      return emitError("failed to find enough loop-like operations inside "3144                       "container-like acc.loop");3145  }3146 3147  return success();3148}3149 3150unsigned LoopOp::getNumDataOperands() {3151  return getReductionOperands().size() + getPrivateOperands().size() +3152         getFirstprivateOperands().size();3153}3154 3155Value LoopOp::getDataOperand(unsigned i) {3156  unsigned numOptional =3157      getLowerbound().size() + getUpperbound().size() + getStep().size();3158  numOptional += getGangOperands().size();3159  numOptional += getVectorOperands().size();3160  numOptional += getWorkerNumOperands().size();3161  numOptional += getTileOperands().size();3162  numOptional += getCacheOperands().size();3163  return getOperand(numOptional + i);3164}3165 3166bool LoopOp::hasAuto() { return hasAuto(mlir::acc::DeviceType::None); }3167 3168bool LoopOp::hasAuto(mlir::acc::DeviceType deviceType) {3169  return hasDeviceType(getAuto_(), deviceType);3170}3171 3172bool LoopOp::hasIndependent() {3173  return hasIndependent(mlir::acc::DeviceType::None);3174}3175 3176bool LoopOp::hasIndependent(mlir::acc::DeviceType deviceType) {3177  return hasDeviceType(getIndependent(), deviceType);3178}3179 3180bool LoopOp::hasSeq() { return hasSeq(mlir::acc::DeviceType::None); }3181 3182bool LoopOp::hasSeq(mlir::acc::DeviceType deviceType) {3183  return hasDeviceType(getSeq(), deviceType);3184}3185 3186mlir::Value LoopOp::getVectorValue() {3187  return getVectorValue(mlir::acc::DeviceType::None);3188}3189 3190mlir::Value LoopOp::getVectorValue(mlir::acc::DeviceType deviceType) {3191  return getValueInDeviceTypeSegment(getVectorOperandsDeviceType(),3192                                     getVectorOperands(), deviceType);3193}3194 3195bool LoopOp::hasVector() { return hasVector(mlir::acc::DeviceType::None); }3196 3197bool LoopOp::hasVector(mlir::acc::DeviceType deviceType) {3198  return hasDeviceType(getVector(), deviceType);3199}3200 3201mlir::Value LoopOp::getWorkerValue() {3202  return getWorkerValue(mlir::acc::DeviceType::None);3203}3204 3205mlir::Value LoopOp::getWorkerValue(mlir::acc::DeviceType deviceType) {3206  return getValueInDeviceTypeSegment(getWorkerNumOperandsDeviceType(),3207                                     getWorkerNumOperands(), deviceType);3208}3209 3210bool LoopOp::hasWorker() { return hasWorker(mlir::acc::DeviceType::None); }3211 3212bool LoopOp::hasWorker(mlir::acc::DeviceType deviceType) {3213  return hasDeviceType(getWorker(), deviceType);3214}3215 3216mlir::Operation::operand_range LoopOp::getTileValues() {3217  return getTileValues(mlir::acc::DeviceType::None);3218}3219 3220mlir::Operation::operand_range3221LoopOp::getTileValues(mlir::acc::DeviceType deviceType) {3222  return getValuesFromSegments(getTileOperandsDeviceType(), getTileOperands(),3223                               getTileOperandsSegments(), deviceType);3224}3225 3226std::optional<int64_t> LoopOp::getCollapseValue() {3227  return getCollapseValue(mlir::acc::DeviceType::None);3228}3229 3230std::optional<int64_t>3231LoopOp::getCollapseValue(mlir::acc::DeviceType deviceType) {3232  if (!getCollapseAttr())3233    return std::nullopt;3234  if (auto pos = findSegment(getCollapseDeviceTypeAttr(), deviceType)) {3235    auto intAttr =3236        mlir::dyn_cast<IntegerAttr>(getCollapseAttr().getValue()[*pos]);3237    return intAttr.getValue().getZExtValue();3238  }3239  return std::nullopt;3240}3241 3242mlir::Value LoopOp::getGangValue(mlir::acc::GangArgType gangArgType) {3243  return getGangValue(gangArgType, mlir::acc::DeviceType::None);3244}3245 3246mlir::Value LoopOp::getGangValue(mlir::acc::GangArgType gangArgType,3247                                 mlir::acc::DeviceType deviceType) {3248  if (getGangOperands().empty())3249    return {};3250  if (auto pos = findSegment(*getGangOperandsDeviceType(), deviceType)) {3251    int32_t nbOperandsBefore = 0;3252    for (unsigned i = 0; i < *pos; ++i)3253      nbOperandsBefore += (*getGangOperandsSegments())[i];3254    mlir::Operation::operand_range values =3255        getGangOperands()3256            .drop_front(nbOperandsBefore)3257            .take_front((*getGangOperandsSegments())[*pos]);3258 3259    int32_t argTypeIdx = nbOperandsBefore;3260    for (auto value : values) {3261      auto gangArgTypeAttr = mlir::dyn_cast<mlir::acc::GangArgTypeAttr>(3262          (*getGangOperandsArgType())[argTypeIdx]);3263      if (gangArgTypeAttr.getValue() == gangArgType)3264        return value;3265      ++argTypeIdx;3266    }3267  }3268  return {};3269}3270 3271bool LoopOp::hasGang() { return hasGang(mlir::acc::DeviceType::None); }3272 3273bool LoopOp::hasGang(mlir::acc::DeviceType deviceType) {3274  return hasDeviceType(getGang(), deviceType);3275}3276 3277llvm::SmallVector<mlir::Region *> acc::LoopOp::getLoopRegions() {3278  return {&getRegion()};3279}3280 3281/// loop-control ::= `control` `(` ssa-id-and-type-list `)` `=`3282/// `(` ssa-id-and-type-list `)` `to` `(` ssa-id-and-type-list `)` `step`3283/// `(` ssa-id-and-type-list `)`3284/// region3285ParseResult3286parseLoopControl(OpAsmParser &parser, Region &region,3287                 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &lowerbound,3288                 SmallVectorImpl<Type> &lowerboundType,3289                 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &upperbound,3290                 SmallVectorImpl<Type> &upperboundType,3291                 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &step,3292                 SmallVectorImpl<Type> &stepType) {3293 3294  SmallVector<OpAsmParser::Argument> inductionVars;3295  if (succeeded(3296          parser.parseOptionalKeyword(acc::LoopOp::getControlKeyword()))) {3297    if (parser.parseLParen() ||3298        parser.parseArgumentList(inductionVars, OpAsmParser::Delimiter::None,3299                                 /*allowType=*/true) ||3300        parser.parseRParen() || parser.parseEqual() || parser.parseLParen() ||3301        parser.parseOperandList(lowerbound, inductionVars.size(),3302                                OpAsmParser::Delimiter::None) ||3303        parser.parseColonTypeList(lowerboundType) || parser.parseRParen() ||3304        parser.parseKeyword("to") || parser.parseLParen() ||3305        parser.parseOperandList(upperbound, inductionVars.size(),3306                                OpAsmParser::Delimiter::None) ||3307        parser.parseColonTypeList(upperboundType) || parser.parseRParen() ||3308        parser.parseKeyword("step") || parser.parseLParen() ||3309        parser.parseOperandList(step, inductionVars.size(),3310                                OpAsmParser::Delimiter::None) ||3311        parser.parseColonTypeList(stepType) || parser.parseRParen())3312      return failure();3313  }3314  return parser.parseRegion(region, inductionVars);3315}3316 3317void printLoopControl(OpAsmPrinter &p, Operation *op, Region &region,3318                      ValueRange lowerbound, TypeRange lowerboundType,3319                      ValueRange upperbound, TypeRange upperboundType,3320                      ValueRange steps, TypeRange stepType) {3321  ValueRange regionArgs = region.front().getArguments();3322  if (!regionArgs.empty()) {3323    p << acc::LoopOp::getControlKeyword() << "(";3324    llvm::interleaveComma(regionArgs, p,3325                          [&p](Value v) { p << v << " : " << v.getType(); });3326    p << ") = (" << lowerbound << " : " << lowerboundType << ") to ("3327      << upperbound << " : " << upperboundType << ") " << " step (" << steps3328      << " : " << stepType << ") ";3329  }3330  p.printRegion(region, /*printEntryBlockArgs=*/false);3331}3332 3333void acc::LoopOp::addSeq(MLIRContext *context,3334                         llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {3335  setSeqAttr(addDeviceTypeAffectedOperandHelper(context, getSeqAttr(),3336                                                effectiveDeviceTypes));3337}3338 3339void acc::LoopOp::addIndependent(3340    MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {3341  setIndependentAttr(addDeviceTypeAffectedOperandHelper(3342      context, getIndependentAttr(), effectiveDeviceTypes));3343}3344 3345void acc::LoopOp::addAuto(MLIRContext *context,3346                          llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {3347  setAuto_Attr(addDeviceTypeAffectedOperandHelper(context, getAuto_Attr(),3348                                                  effectiveDeviceTypes));3349}3350 3351void acc::LoopOp::setCollapseForDeviceTypes(3352    MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes,3353    llvm::APInt value) {3354  llvm::SmallVector<mlir::Attribute> newValues;3355  llvm::SmallVector<mlir::Attribute> newDeviceTypes;3356 3357  assert((getCollapseAttr() == nullptr) ==3358         (getCollapseDeviceTypeAttr() == nullptr));3359  assert(value.getBitWidth() == 64);3360 3361  if (getCollapseAttr()) {3362    for (const auto &existing :3363         llvm::zip_equal(getCollapseAttr(), getCollapseDeviceTypeAttr())) {3364      newValues.push_back(std::get<0>(existing));3365      newDeviceTypes.push_back(std::get<1>(existing));3366    }3367  }3368 3369  if (effectiveDeviceTypes.empty()) {3370    // If the effective device-types list is empty, this is before there are any3371    // being applied by device_type, so this should be added as a 'none'.3372    newValues.push_back(3373        mlir::IntegerAttr::get(mlir::IntegerType::get(context, 64), value));3374    newDeviceTypes.push_back(3375        acc::DeviceTypeAttr::get(context, DeviceType::None));3376  } else {3377    for (DeviceType dt : effectiveDeviceTypes) {3378      newValues.push_back(3379          mlir::IntegerAttr::get(mlir::IntegerType::get(context, 64), value));3380      newDeviceTypes.push_back(acc::DeviceTypeAttr::get(context, dt));3381    }3382  }3383 3384  setCollapseAttr(ArrayAttr::get(context, newValues));3385  setCollapseDeviceTypeAttr(ArrayAttr::get(context, newDeviceTypes));3386}3387 3388void acc::LoopOp::setTileForDeviceTypes(3389    MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes,3390    ValueRange values) {3391  llvm::SmallVector<int32_t> segments;3392  if (getTileOperandsSegments())3393    llvm::copy(*getTileOperandsSegments(), std::back_inserter(segments));3394 3395  setTileOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(3396      context, getTileOperandsDeviceTypeAttr(), effectiveDeviceTypes, values,3397      getTileOperandsMutable(), segments));3398 3399  setTileOperandsSegments(segments);3400}3401 3402void acc::LoopOp::addVectorOperand(3403    MLIRContext *context, mlir::Value newValue,3404    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {3405  setVectorOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(3406      context, getVectorOperandsDeviceTypeAttr(), effectiveDeviceTypes,3407      newValue, getVectorOperandsMutable()));3408}3409 3410void acc::LoopOp::addEmptyVector(3411    MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {3412  setVectorAttr(addDeviceTypeAffectedOperandHelper(context, getVectorAttr(),3413                                                   effectiveDeviceTypes));3414}3415 3416void acc::LoopOp::addWorkerNumOperand(3417    MLIRContext *context, mlir::Value newValue,3418    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {3419  setWorkerNumOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(3420      context, getWorkerNumOperandsDeviceTypeAttr(), effectiveDeviceTypes,3421      newValue, getWorkerNumOperandsMutable()));3422}3423 3424void acc::LoopOp::addEmptyWorker(3425    MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {3426  setWorkerAttr(addDeviceTypeAffectedOperandHelper(context, getWorkerAttr(),3427                                                   effectiveDeviceTypes));3428}3429 3430void acc::LoopOp::addEmptyGang(3431    MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {3432  setGangAttr(addDeviceTypeAffectedOperandHelper(context, getGangAttr(),3433                                                 effectiveDeviceTypes));3434}3435 3436bool acc::LoopOp::hasParallelismFlag(DeviceType dt) {3437  auto hasDevice = [=](DeviceTypeAttr attr) -> bool {3438    return attr.getValue() == dt;3439  };3440  auto testFromArr = [=](ArrayAttr arr) -> bool {3441    return llvm::any_of(arr.getAsRange<DeviceTypeAttr>(), hasDevice);3442  };3443 3444  if (ArrayAttr arr = getSeqAttr(); arr && testFromArr(arr))3445    return true;3446  if (ArrayAttr arr = getIndependentAttr(); arr && testFromArr(arr))3447    return true;3448  if (ArrayAttr arr = getAuto_Attr(); arr && testFromArr(arr))3449    return true;3450 3451  return false;3452}3453 3454bool acc::LoopOp::hasDefaultGangWorkerVector() {3455  return hasVector() || getVectorValue() || hasWorker() || getWorkerValue() ||3456         hasGang() || getGangValue(GangArgType::Num) ||3457         getGangValue(GangArgType::Dim) || getGangValue(GangArgType::Static);3458}3459 3460acc::LoopParMode3461acc::LoopOp::getDefaultOrDeviceTypeParallelism(DeviceType deviceType) {3462  if (hasSeq(deviceType))3463    return LoopParMode::loop_seq;3464  if (hasAuto(deviceType))3465    return LoopParMode::loop_auto;3466  if (hasIndependent(deviceType))3467    return LoopParMode::loop_independent;3468  if (hasSeq())3469    return LoopParMode::loop_seq;3470  if (hasAuto())3471    return LoopParMode::loop_auto;3472  assert(hasIndependent() &&3473         "loop must have default auto, seq, or independent");3474  return LoopParMode::loop_independent;3475}3476 3477void acc::LoopOp::addGangOperands(3478    MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes,3479    llvm::ArrayRef<GangArgType> argTypes, mlir::ValueRange values) {3480  llvm::SmallVector<int32_t> segments;3481  if (std::optional<ArrayRef<int32_t>> existingSegments =3482          getGangOperandsSegments())3483    llvm::copy(*existingSegments, std::back_inserter(segments));3484 3485  unsigned beforeCount = segments.size();3486 3487  setGangOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(3488      context, getGangOperandsDeviceTypeAttr(), effectiveDeviceTypes, values,3489      getGangOperandsMutable(), segments));3490 3491  setGangOperandsSegments(segments);3492 3493  // This is a bit of extra work to make sure we update the 'types' correctly by3494  // adding to the types collection the correct number of times. We could3495  // potentially add something similar to the3496  // addDeviceTypeAffectedOperandHelper, but it seems that would be pretty3497  // excessive for a one-off case.3498  unsigned numAdded = segments.size() - beforeCount;3499 3500  if (numAdded > 0) {3501    llvm::SmallVector<mlir::Attribute> gangTypes;3502    if (getGangOperandsArgTypeAttr())3503      llvm::copy(getGangOperandsArgTypeAttr(), std::back_inserter(gangTypes));3504 3505    for (auto i : llvm::index_range(0u, numAdded)) {3506      llvm::transform(argTypes, std::back_inserter(gangTypes),3507                      [=](mlir::acc::GangArgType gangTy) {3508                        return mlir::acc::GangArgTypeAttr::get(context, gangTy);3509                      });3510      (void)i;3511    }3512 3513    setGangOperandsArgTypeAttr(mlir::ArrayAttr::get(context, gangTypes));3514  }3515}3516 3517void acc::LoopOp::addPrivatization(MLIRContext *context,3518                                   mlir::acc::PrivateOp op,3519                                   mlir::acc::PrivateRecipeOp recipe) {3520  op.setRecipeAttr(mlir::SymbolRefAttr::get(context, recipe.getSymName()));3521  getPrivateOperandsMutable().append(op.getResult());3522}3523 3524void acc::LoopOp::addFirstPrivatization(3525    MLIRContext *context, mlir::acc::FirstprivateOp op,3526    mlir::acc::FirstprivateRecipeOp recipe) {3527  op.setRecipeAttr(mlir::SymbolRefAttr::get(context, recipe.getSymName()));3528  getFirstprivateOperandsMutable().append(op.getResult());3529}3530 3531void acc::LoopOp::addReduction(MLIRContext *context, mlir::acc::ReductionOp op,3532                               mlir::acc::ReductionRecipeOp recipe) {3533  op.setRecipeAttr(mlir::SymbolRefAttr::get(context, recipe.getSymName()));3534  getReductionOperandsMutable().append(op.getResult());3535}3536 3537//===----------------------------------------------------------------------===//3538// DataOp3539//===----------------------------------------------------------------------===//3540 3541LogicalResult acc::DataOp::verify() {3542  // 2.6.5. Data Construct restriction3543  // At least one copy, copyin, copyout, create, no_create, present, deviceptr,3544  // attach, or default clause must appear on a data construct.3545  if (getOperands().empty() && !getDefaultAttr())3546    return emitError("at least one operand or the default attribute "3547                     "must appear on the data operation");3548 3549  for (mlir::Value operand : getDataClauseOperands())3550    if (isa<BlockArgument>(operand) ||3551        !mlir::isa<acc::AttachOp, acc::CopyinOp, acc::CopyoutOp, acc::CreateOp,3552                   acc::DeleteOp, acc::DetachOp, acc::DevicePtrOp,3553                   acc::GetDevicePtrOp, acc::NoCreateOp, acc::PresentOp>(3554            operand.getDefiningOp()))3555      return emitError("expect data entry/exit operation or acc.getdeviceptr "3556                       "as defining op");3557 3558  if (failed(checkWaitAndAsyncConflict<acc::DataOp>(*this)))3559    return failure();3560 3561  return success();3562}3563 3564unsigned DataOp::getNumDataOperands() { return getDataClauseOperands().size(); }3565 3566Value DataOp::getDataOperand(unsigned i) {3567  unsigned numOptional = getIfCond() ? 1 : 0;3568  numOptional += getAsyncOperands().size() ? 1 : 0;3569  numOptional += getWaitOperands().size();3570  return getOperand(numOptional + i);3571}3572 3573bool acc::DataOp::hasAsyncOnly() {3574  return hasAsyncOnly(mlir::acc::DeviceType::None);3575}3576 3577bool acc::DataOp::hasAsyncOnly(mlir::acc::DeviceType deviceType) {3578  return hasDeviceType(getAsyncOnly(), deviceType);3579}3580 3581mlir::Value DataOp::getAsyncValue() {3582  return getAsyncValue(mlir::acc::DeviceType::None);3583}3584 3585mlir::Value DataOp::getAsyncValue(mlir::acc::DeviceType deviceType) {3586  return getValueInDeviceTypeSegment(getAsyncOperandsDeviceType(),3587                                     getAsyncOperands(), deviceType);3588}3589 3590bool DataOp::hasWaitOnly() { return hasWaitOnly(mlir::acc::DeviceType::None); }3591 3592bool DataOp::hasWaitOnly(mlir::acc::DeviceType deviceType) {3593  return hasDeviceType(getWaitOnly(), deviceType);3594}3595 3596mlir::Operation::operand_range DataOp::getWaitValues() {3597  return getWaitValues(mlir::acc::DeviceType::None);3598}3599 3600mlir::Operation::operand_range3601DataOp::getWaitValues(mlir::acc::DeviceType deviceType) {3602  return getWaitValuesWithoutDevnum(3603      getWaitOperandsDeviceType(), getWaitOperands(), getWaitOperandsSegments(),3604      getHasWaitDevnum(), deviceType);3605}3606 3607mlir::Value DataOp::getWaitDevnum() {3608  return getWaitDevnum(mlir::acc::DeviceType::None);3609}3610 3611mlir::Value DataOp::getWaitDevnum(mlir::acc::DeviceType deviceType) {3612  return getWaitDevnumValue(getWaitOperandsDeviceType(), getWaitOperands(),3613                            getWaitOperandsSegments(), getHasWaitDevnum(),3614                            deviceType);3615}3616 3617void acc::DataOp::addAsyncOnly(3618    MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {3619  setAsyncOnlyAttr(addDeviceTypeAffectedOperandHelper(3620      context, getAsyncOnlyAttr(), effectiveDeviceTypes));3621}3622 3623void acc::DataOp::addAsyncOperand(3624    MLIRContext *context, mlir::Value newValue,3625    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {3626  setAsyncOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(3627      context, getAsyncOperandsDeviceTypeAttr(), effectiveDeviceTypes, newValue,3628      getAsyncOperandsMutable()));3629}3630 3631void acc::DataOp::addWaitOnly(MLIRContext *context,3632                              llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {3633  setWaitOnlyAttr(addDeviceTypeAffectedOperandHelper(context, getWaitOnlyAttr(),3634                                                     effectiveDeviceTypes));3635}3636 3637void acc::DataOp::addWaitOperands(3638    MLIRContext *context, bool hasDevnum, mlir::ValueRange newValues,3639    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {3640 3641  llvm::SmallVector<int32_t> segments;3642  if (getWaitOperandsSegments())3643    llvm::copy(*getWaitOperandsSegments(), std::back_inserter(segments));3644 3645  setWaitOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(3646      context, getWaitOperandsDeviceTypeAttr(), effectiveDeviceTypes, newValues,3647      getWaitOperandsMutable(), segments));3648  setWaitOperandsSegments(segments);3649 3650  llvm::SmallVector<mlir::Attribute> hasDevnums;3651  if (getHasWaitDevnumAttr())3652    llvm::copy(getHasWaitDevnumAttr(), std::back_inserter(hasDevnums));3653  hasDevnums.insert(3654      hasDevnums.end(),3655      std::max(effectiveDeviceTypes.size(), static_cast<size_t>(1)),3656      mlir::BoolAttr::get(context, hasDevnum));3657  setHasWaitDevnumAttr(mlir::ArrayAttr::get(context, hasDevnums));3658}3659 3660//===----------------------------------------------------------------------===//3661// ExitDataOp3662//===----------------------------------------------------------------------===//3663 3664LogicalResult acc::ExitDataOp::verify() {3665  // 2.6.6. Data Exit Directive restriction3666  // At least one copyout, delete, or detach clause must appear on an exit data3667  // directive.3668  if (getDataClauseOperands().empty())3669    return emitError("at least one operand must be present in dataOperands on "3670                     "the exit data operation");3671 3672  // The async attribute represent the async clause without value. Therefore the3673  // attribute and operand cannot appear at the same time.3674  if (getAsyncOperand() && getAsync())3675    return emitError("async attribute cannot appear with asyncOperand");3676 3677  // The wait attribute represent the wait clause without values. Therefore the3678  // attribute and operands cannot appear at the same time.3679  if (!getWaitOperands().empty() && getWait())3680    return emitError("wait attribute cannot appear with waitOperands");3681 3682  if (getWaitDevnum() && getWaitOperands().empty())3683    return emitError("wait_devnum cannot appear without waitOperands");3684 3685  return success();3686}3687 3688unsigned ExitDataOp::getNumDataOperands() {3689  return getDataClauseOperands().size();3690}3691 3692Value ExitDataOp::getDataOperand(unsigned i) {3693  unsigned numOptional = getIfCond() ? 1 : 0;3694  numOptional += getAsyncOperand() ? 1 : 0;3695  numOptional += getWaitDevnum() ? 1 : 0;3696  return getOperand(getWaitOperands().size() + numOptional + i);3697}3698 3699void ExitDataOp::getCanonicalizationPatterns(RewritePatternSet &results,3700                                             MLIRContext *context) {3701  results.add<RemoveConstantIfCondition<ExitDataOp>>(context);3702}3703 3704void ExitDataOp::addAsyncOnly(MLIRContext *context,3705                              llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {3706  assert(effectiveDeviceTypes.empty());3707  assert(!getAsyncAttr());3708  assert(!getAsyncOperand());3709 3710  setAsyncAttr(mlir::UnitAttr::get(context));3711}3712 3713void ExitDataOp::addAsyncOperand(3714    MLIRContext *context, mlir::Value newValue,3715    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {3716  assert(effectiveDeviceTypes.empty());3717  assert(!getAsyncAttr());3718  assert(!getAsyncOperand());3719 3720  getAsyncOperandMutable().append(newValue);3721}3722 3723void ExitDataOp::addWaitOnly(MLIRContext *context,3724                             llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {3725  assert(effectiveDeviceTypes.empty());3726  assert(!getWaitAttr());3727  assert(getWaitOperands().empty());3728  assert(!getWaitDevnum());3729 3730  setWaitAttr(mlir::UnitAttr::get(context));3731}3732 3733void ExitDataOp::addWaitOperands(3734    MLIRContext *context, bool hasDevnum, mlir::ValueRange newValues,3735    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {3736  assert(effectiveDeviceTypes.empty());3737  assert(!getWaitAttr());3738  assert(getWaitOperands().empty());3739  assert(!getWaitDevnum());3740 3741  // if hasDevnum, the first value is the devnum. The 'rest' go into the3742  // operands list.3743  if (hasDevnum) {3744    getWaitDevnumMutable().append(newValues.front());3745    newValues = newValues.drop_front();3746  }3747 3748  getWaitOperandsMutable().append(newValues);3749}3750 3751//===----------------------------------------------------------------------===//3752// EnterDataOp3753//===----------------------------------------------------------------------===//3754 3755LogicalResult acc::EnterDataOp::verify() {3756  // 2.6.6. Data Enter Directive restriction3757  // At least one copyin, create, or attach clause must appear on an enter data3758  // directive.3759  if (getDataClauseOperands().empty())3760    return emitError("at least one operand must be present in dataOperands on "3761                     "the enter data operation");3762 3763  // The async attribute represent the async clause without value. Therefore the3764  // attribute and operand cannot appear at the same time.3765  if (getAsyncOperand() && getAsync())3766    return emitError("async attribute cannot appear with asyncOperand");3767 3768  // The wait attribute represent the wait clause without values. Therefore the3769  // attribute and operands cannot appear at the same time.3770  if (!getWaitOperands().empty() && getWait())3771    return emitError("wait attribute cannot appear with waitOperands");3772 3773  if (getWaitDevnum() && getWaitOperands().empty())3774    return emitError("wait_devnum cannot appear without waitOperands");3775 3776  for (mlir::Value operand : getDataClauseOperands())3777    if (!mlir::isa<acc::AttachOp, acc::CreateOp, acc::CopyinOp>(3778            operand.getDefiningOp()))3779      return emitError("expect data entry operation as defining op");3780 3781  return success();3782}3783 3784unsigned EnterDataOp::getNumDataOperands() {3785  return getDataClauseOperands().size();3786}3787 3788Value EnterDataOp::getDataOperand(unsigned i) {3789  unsigned numOptional = getIfCond() ? 1 : 0;3790  numOptional += getAsyncOperand() ? 1 : 0;3791  numOptional += getWaitDevnum() ? 1 : 0;3792  return getOperand(getWaitOperands().size() + numOptional + i);3793}3794 3795void EnterDataOp::getCanonicalizationPatterns(RewritePatternSet &results,3796                                              MLIRContext *context) {3797  results.add<RemoveConstantIfCondition<EnterDataOp>>(context);3798}3799 3800void EnterDataOp::addAsyncOnly(3801    MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {3802  assert(effectiveDeviceTypes.empty());3803  assert(!getAsyncAttr());3804  assert(!getAsyncOperand());3805 3806  setAsyncAttr(mlir::UnitAttr::get(context));3807}3808 3809void EnterDataOp::addAsyncOperand(3810    MLIRContext *context, mlir::Value newValue,3811    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {3812  assert(effectiveDeviceTypes.empty());3813  assert(!getAsyncAttr());3814  assert(!getAsyncOperand());3815 3816  getAsyncOperandMutable().append(newValue);3817}3818 3819void EnterDataOp::addWaitOnly(MLIRContext *context,3820                              llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {3821  assert(effectiveDeviceTypes.empty());3822  assert(!getWaitAttr());3823  assert(getWaitOperands().empty());3824  assert(!getWaitDevnum());3825 3826  setWaitAttr(mlir::UnitAttr::get(context));3827}3828 3829void EnterDataOp::addWaitOperands(3830    MLIRContext *context, bool hasDevnum, mlir::ValueRange newValues,3831    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {3832  assert(effectiveDeviceTypes.empty());3833  assert(!getWaitAttr());3834  assert(getWaitOperands().empty());3835  assert(!getWaitDevnum());3836 3837  // if hasDevnum, the first value is the devnum. The 'rest' go into the3838  // operands list.3839  if (hasDevnum) {3840    getWaitDevnumMutable().append(newValues.front());3841    newValues = newValues.drop_front();3842  }3843 3844  getWaitOperandsMutable().append(newValues);3845}3846 3847//===----------------------------------------------------------------------===//3848// AtomicReadOp3849//===----------------------------------------------------------------------===//3850 3851LogicalResult AtomicReadOp::verify() { return verifyCommon(); }3852 3853//===----------------------------------------------------------------------===//3854// AtomicWriteOp3855//===----------------------------------------------------------------------===//3856 3857LogicalResult AtomicWriteOp::verify() { return verifyCommon(); }3858 3859//===----------------------------------------------------------------------===//3860// AtomicUpdateOp3861//===----------------------------------------------------------------------===//3862 3863LogicalResult AtomicUpdateOp::canonicalize(AtomicUpdateOp op,3864                                           PatternRewriter &rewriter) {3865  if (op.isNoOp()) {3866    rewriter.eraseOp(op);3867    return success();3868  }3869 3870  if (Value writeVal = op.getWriteOpVal()) {3871    rewriter.replaceOpWithNewOp<AtomicWriteOp>(op, op.getX(), writeVal,3872                                               op.getIfCond());3873    return success();3874  }3875 3876  return failure();3877}3878 3879LogicalResult AtomicUpdateOp::verify() { return verifyCommon(); }3880 3881LogicalResult AtomicUpdateOp::verifyRegions() { return verifyRegionsCommon(); }3882 3883//===----------------------------------------------------------------------===//3884// AtomicCaptureOp3885//===----------------------------------------------------------------------===//3886 3887AtomicReadOp AtomicCaptureOp::getAtomicReadOp() {3888  if (auto op = dyn_cast<AtomicReadOp>(getFirstOp()))3889    return op;3890  return dyn_cast<AtomicReadOp>(getSecondOp());3891}3892 3893AtomicWriteOp AtomicCaptureOp::getAtomicWriteOp() {3894  if (auto op = dyn_cast<AtomicWriteOp>(getFirstOp()))3895    return op;3896  return dyn_cast<AtomicWriteOp>(getSecondOp());3897}3898 3899AtomicUpdateOp AtomicCaptureOp::getAtomicUpdateOp() {3900  if (auto op = dyn_cast<AtomicUpdateOp>(getFirstOp()))3901    return op;3902  return dyn_cast<AtomicUpdateOp>(getSecondOp());3903}3904 3905LogicalResult AtomicCaptureOp::verifyRegions() { return verifyRegionsCommon(); }3906 3907//===----------------------------------------------------------------------===//3908// DeclareEnterOp3909//===----------------------------------------------------------------------===//3910 3911template <typename Op>3912static LogicalResult3913checkDeclareOperands(Op &op, const mlir::ValueRange &operands,3914                     bool requireAtLeastOneOperand = true) {3915  if (operands.empty() && requireAtLeastOneOperand)3916    return emitError(3917        op->getLoc(),3918        "at least one operand must appear on the declare operation");3919 3920  for (mlir::Value operand : operands) {3921    if (isa<BlockArgument>(operand) ||3922        !mlir::isa<acc::CopyinOp, acc::CopyoutOp, acc::CreateOp,3923                   acc::DevicePtrOp, acc::GetDevicePtrOp, acc::PresentOp,3924                   acc::DeclareDeviceResidentOp, acc::DeclareLinkOp>(3925            operand.getDefiningOp()))3926      return op.emitError(3927          "expect valid declare data entry operation or acc.getdeviceptr "3928          "as defining op");3929 3930    mlir::Value var{getVar(operand.getDefiningOp())};3931    assert(var && "declare operands can only be data entry operations which "3932                  "must have var");3933    (void)var;3934    std::optional<mlir::acc::DataClause> dataClauseOptional{3935        getDataClause(operand.getDefiningOp())};3936    assert(dataClauseOptional.has_value() &&3937           "declare operands can only be data entry operations which must have "3938           "dataClause");3939    (void)dataClauseOptional;3940  }3941 3942  return success();3943}3944 3945LogicalResult acc::DeclareEnterOp::verify() {3946  return checkDeclareOperands(*this, this->getDataClauseOperands());3947}3948 3949//===----------------------------------------------------------------------===//3950// DeclareExitOp3951//===----------------------------------------------------------------------===//3952 3953LogicalResult acc::DeclareExitOp::verify() {3954  if (getToken())3955    return checkDeclareOperands(*this, this->getDataClauseOperands(),3956                                /*requireAtLeastOneOperand=*/false);3957  return checkDeclareOperands(*this, this->getDataClauseOperands());3958}3959 3960//===----------------------------------------------------------------------===//3961// DeclareOp3962//===----------------------------------------------------------------------===//3963 3964LogicalResult acc::DeclareOp::verify() {3965  return checkDeclareOperands(*this, this->getDataClauseOperands());3966}3967 3968//===----------------------------------------------------------------------===//3969// RoutineOp3970//===----------------------------------------------------------------------===//3971 3972static unsigned getParallelismForDeviceType(acc::RoutineOp op,3973                                            acc::DeviceType dtype) {3974  unsigned parallelism = 0;3975  parallelism += (op.hasGang(dtype) || op.getGangDimValue(dtype)) ? 1 : 0;3976  parallelism += op.hasWorker(dtype) ? 1 : 0;3977  parallelism += op.hasVector(dtype) ? 1 : 0;3978  parallelism += op.hasSeq(dtype) ? 1 : 0;3979  return parallelism;3980}3981 3982LogicalResult acc::RoutineOp::verify() {3983  unsigned baseParallelism =3984      getParallelismForDeviceType(*this, acc::DeviceType::None);3985 3986  if (baseParallelism > 1)3987    return emitError() << "only one of `gang`, `worker`, `vector`, `seq` can "3988                          "be present at the same time";3989 3990  for (uint32_t dtypeInt = 0; dtypeInt != acc::getMaxEnumValForDeviceType();3991       ++dtypeInt) {3992    auto dtype = static_cast<acc::DeviceType>(dtypeInt);3993    if (dtype == acc::DeviceType::None)3994      continue;3995    unsigned parallelism = getParallelismForDeviceType(*this, dtype);3996 3997    if (parallelism > 1 || (baseParallelism == 1 && parallelism == 1))3998      return emitError() << "only one of `gang`, `worker`, `vector`, `seq` can "3999                            "be present at the same time";4000  }4001 4002  return success();4003}4004 4005static ParseResult parseBindName(OpAsmParser &parser,4006                                 mlir::ArrayAttr &bindIdName,4007                                 mlir::ArrayAttr &bindStrName,4008                                 mlir::ArrayAttr &deviceIdTypes,4009                                 mlir::ArrayAttr &deviceStrTypes) {4010  llvm::SmallVector<mlir::Attribute> bindIdNameAttrs;4011  llvm::SmallVector<mlir::Attribute> bindStrNameAttrs;4012  llvm::SmallVector<mlir::Attribute> deviceIdTypeAttrs;4013  llvm::SmallVector<mlir::Attribute> deviceStrTypeAttrs;4014 4015  if (failed(parser.parseCommaSeparatedList([&]() {4016        mlir::Attribute newAttr;4017        bool isSymbolRefAttr;4018        auto parseResult = parser.parseAttribute(newAttr);4019        if (auto symbolRefAttr = dyn_cast<mlir::SymbolRefAttr>(newAttr)) {4020          bindIdNameAttrs.push_back(symbolRefAttr);4021          isSymbolRefAttr = true;4022        } else if (auto stringAttr = dyn_cast<mlir::StringAttr>(newAttr)) {4023          bindStrNameAttrs.push_back(stringAttr);4024          isSymbolRefAttr = false;4025        }4026        if (parseResult)4027          return failure();4028        if (failed(parser.parseOptionalLSquare())) {4029          if (isSymbolRefAttr) {4030            deviceIdTypeAttrs.push_back(mlir::acc::DeviceTypeAttr::get(4031                parser.getContext(), mlir::acc::DeviceType::None));4032          } else {4033            deviceStrTypeAttrs.push_back(mlir::acc::DeviceTypeAttr::get(4034                parser.getContext(), mlir::acc::DeviceType::None));4035          }4036        } else {4037          if (isSymbolRefAttr) {4038            if (parser.parseAttribute(deviceIdTypeAttrs.emplace_back()) ||4039                parser.parseRSquare())4040              return failure();4041          } else {4042            if (parser.parseAttribute(deviceStrTypeAttrs.emplace_back()) ||4043                parser.parseRSquare())4044              return failure();4045          }4046        }4047        return success();4048      })))4049    return failure();4050 4051  bindIdName = ArrayAttr::get(parser.getContext(), bindIdNameAttrs);4052  bindStrName = ArrayAttr::get(parser.getContext(), bindStrNameAttrs);4053  deviceIdTypes = ArrayAttr::get(parser.getContext(), deviceIdTypeAttrs);4054  deviceStrTypes = ArrayAttr::get(parser.getContext(), deviceStrTypeAttrs);4055 4056  return success();4057}4058 4059static void printBindName(mlir::OpAsmPrinter &p, mlir::Operation *op,4060                          std::optional<mlir::ArrayAttr> bindIdName,4061                          std::optional<mlir::ArrayAttr> bindStrName,4062                          std::optional<mlir::ArrayAttr> deviceIdTypes,4063                          std::optional<mlir::ArrayAttr> deviceStrTypes) {4064  // Create combined vectors for all bind names and device types4065  llvm::SmallVector<mlir::Attribute> allBindNames;4066  llvm::SmallVector<mlir::Attribute> allDeviceTypes;4067 4068  // Append bindIdName and deviceIdTypes4069  if (hasDeviceTypeValues(deviceIdTypes)) {4070    allBindNames.append(bindIdName->begin(), bindIdName->end());4071    allDeviceTypes.append(deviceIdTypes->begin(), deviceIdTypes->end());4072  }4073 4074  // Append bindStrName and deviceStrTypes4075  if (hasDeviceTypeValues(deviceStrTypes)) {4076    allBindNames.append(bindStrName->begin(), bindStrName->end());4077    allDeviceTypes.append(deviceStrTypes->begin(), deviceStrTypes->end());4078  }4079 4080  // Print the combined sequence4081  if (!allBindNames.empty())4082    llvm::interleaveComma(llvm::zip(allBindNames, allDeviceTypes), p,4083                          [&](const auto &pair) {4084                            p << std::get<0>(pair);4085                            printSingleDeviceType(p, std::get<1>(pair));4086                          });4087}4088 4089static ParseResult parseRoutineGangClause(OpAsmParser &parser,4090                                          mlir::ArrayAttr &gang,4091                                          mlir::ArrayAttr &gangDim,4092                                          mlir::ArrayAttr &gangDimDeviceTypes) {4093 4094  llvm::SmallVector<mlir::Attribute> gangAttrs, gangDimAttrs,4095      gangDimDeviceTypeAttrs;4096  bool needCommaBeforeOperands = false;4097 4098  // Gang keyword only4099  if (failed(parser.parseOptionalLParen())) {4100    gangAttrs.push_back(mlir::acc::DeviceTypeAttr::get(4101        parser.getContext(), mlir::acc::DeviceType::None));4102    gang = ArrayAttr::get(parser.getContext(), gangAttrs);4103    return success();4104  }4105 4106  // Parse keyword only attributes4107  if (succeeded(parser.parseOptionalLSquare())) {4108    if (failed(parser.parseCommaSeparatedList([&]() {4109          if (parser.parseAttribute(gangAttrs.emplace_back()))4110            return failure();4111          return success();4112        })))4113      return failure();4114    if (parser.parseRSquare())4115      return failure();4116    needCommaBeforeOperands = true;4117  }4118 4119  if (needCommaBeforeOperands && failed(parser.parseComma()))4120    return failure();4121 4122  if (failed(parser.parseCommaSeparatedList([&]() {4123        if (parser.parseKeyword(acc::RoutineOp::getGangDimKeyword()) ||4124            parser.parseColon() ||4125            parser.parseAttribute(gangDimAttrs.emplace_back()))4126          return failure();4127        if (succeeded(parser.parseOptionalLSquare())) {4128          if (parser.parseAttribute(gangDimDeviceTypeAttrs.emplace_back()) ||4129              parser.parseRSquare())4130            return failure();4131        } else {4132          gangDimDeviceTypeAttrs.push_back(mlir::acc::DeviceTypeAttr::get(4133              parser.getContext(), mlir::acc::DeviceType::None));4134        }4135        return success();4136      })))4137    return failure();4138 4139  if (failed(parser.parseRParen()))4140    return failure();4141 4142  gang = ArrayAttr::get(parser.getContext(), gangAttrs);4143  gangDim = ArrayAttr::get(parser.getContext(), gangDimAttrs);4144  gangDimDeviceTypes =4145      ArrayAttr::get(parser.getContext(), gangDimDeviceTypeAttrs);4146 4147  return success();4148}4149 4150void printRoutineGangClause(OpAsmPrinter &p, Operation *op,4151                            std::optional<mlir::ArrayAttr> gang,4152                            std::optional<mlir::ArrayAttr> gangDim,4153                            std::optional<mlir::ArrayAttr> gangDimDeviceTypes) {4154 4155  if (!hasDeviceTypeValues(gangDimDeviceTypes) && hasDeviceTypeValues(gang) &&4156      gang->size() == 1) {4157    auto deviceTypeAttr = mlir::dyn_cast<mlir::acc::DeviceTypeAttr>((*gang)[0]);4158    if (deviceTypeAttr.getValue() == mlir::acc::DeviceType::None)4159      return;4160  }4161 4162  p << "(";4163 4164  printDeviceTypes(p, gang);4165 4166  if (hasDeviceTypeValues(gang) && hasDeviceTypeValues(gangDimDeviceTypes))4167    p << ", ";4168 4169  if (hasDeviceTypeValues(gangDimDeviceTypes))4170    llvm::interleaveComma(llvm::zip(*gangDim, *gangDimDeviceTypes), p,4171                          [&](const auto &pair) {4172                            p << acc::RoutineOp::getGangDimKeyword() << ": ";4173                            p << std::get<0>(pair);4174                            printSingleDeviceType(p, std::get<1>(pair));4175                          });4176 4177  p << ")";4178}4179 4180static ParseResult parseDeviceTypeArrayAttr(OpAsmParser &parser,4181                                            mlir::ArrayAttr &deviceTypes) {4182  llvm::SmallVector<mlir::Attribute> attributes;4183  // Keyword only4184  if (failed(parser.parseOptionalLParen())) {4185    attributes.push_back(mlir::acc::DeviceTypeAttr::get(4186        parser.getContext(), mlir::acc::DeviceType::None));4187    deviceTypes = ArrayAttr::get(parser.getContext(), attributes);4188    return success();4189  }4190 4191  // Parse device type attributes4192  if (succeeded(parser.parseOptionalLSquare())) {4193    if (failed(parser.parseCommaSeparatedList([&]() {4194          if (parser.parseAttribute(attributes.emplace_back()))4195            return failure();4196          return success();4197        })))4198      return failure();4199    if (parser.parseRSquare() || parser.parseRParen())4200      return failure();4201  }4202  deviceTypes = ArrayAttr::get(parser.getContext(), attributes);4203  return success();4204}4205 4206static void4207printDeviceTypeArrayAttr(mlir::OpAsmPrinter &p, mlir::Operation *op,4208                         std::optional<mlir::ArrayAttr> deviceTypes) {4209 4210  if (hasDeviceTypeValues(deviceTypes) && deviceTypes->size() == 1) {4211    auto deviceTypeAttr =4212        mlir::dyn_cast<mlir::acc::DeviceTypeAttr>((*deviceTypes)[0]);4213    if (deviceTypeAttr.getValue() == mlir::acc::DeviceType::None)4214      return;4215  }4216 4217  if (!hasDeviceTypeValues(deviceTypes))4218    return;4219 4220  p << "([";4221  llvm::interleaveComma(*deviceTypes, p, [&](mlir::Attribute attr) {4222    auto dTypeAttr = mlir::dyn_cast<mlir::acc::DeviceTypeAttr>(attr);4223    p << dTypeAttr;4224  });4225  p << "])";4226}4227 4228bool RoutineOp::hasWorker() { return hasWorker(mlir::acc::DeviceType::None); }4229 4230bool RoutineOp::hasWorker(mlir::acc::DeviceType deviceType) {4231  return hasDeviceType(getWorker(), deviceType);4232}4233 4234bool RoutineOp::hasVector() { return hasVector(mlir::acc::DeviceType::None); }4235 4236bool RoutineOp::hasVector(mlir::acc::DeviceType deviceType) {4237  return hasDeviceType(getVector(), deviceType);4238}4239 4240bool RoutineOp::hasSeq() { return hasSeq(mlir::acc::DeviceType::None); }4241 4242bool RoutineOp::hasSeq(mlir::acc::DeviceType deviceType) {4243  return hasDeviceType(getSeq(), deviceType);4244}4245 4246std::optional<std::variant<mlir::SymbolRefAttr, mlir::StringAttr>>4247RoutineOp::getBindNameValue() {4248  return getBindNameValue(mlir::acc::DeviceType::None);4249}4250 4251std::optional<std::variant<mlir::SymbolRefAttr, mlir::StringAttr>>4252RoutineOp::getBindNameValue(mlir::acc::DeviceType deviceType) {4253  if (!hasDeviceTypeValues(getBindIdNameDeviceType()) &&4254      !hasDeviceTypeValues(getBindStrNameDeviceType())) {4255    return std::nullopt;4256  }4257 4258  if (auto pos = findSegment(*getBindIdNameDeviceType(), deviceType)) {4259    auto attr = (*getBindIdName())[*pos];4260    auto symbolRefAttr = dyn_cast<mlir::SymbolRefAttr>(attr);4261    assert(symbolRefAttr && "expected SymbolRef");4262    return symbolRefAttr;4263  }4264 4265  if (auto pos = findSegment(*getBindStrNameDeviceType(), deviceType)) {4266    auto attr = (*getBindStrName())[*pos];4267    auto stringAttr = dyn_cast<mlir::StringAttr>(attr);4268    assert(stringAttr && "expected String");4269    return stringAttr;4270  }4271 4272  return std::nullopt;4273}4274 4275bool RoutineOp::hasGang() { return hasGang(mlir::acc::DeviceType::None); }4276 4277bool RoutineOp::hasGang(mlir::acc::DeviceType deviceType) {4278  return hasDeviceType(getGang(), deviceType);4279}4280 4281std::optional<int64_t> RoutineOp::getGangDimValue() {4282  return getGangDimValue(mlir::acc::DeviceType::None);4283}4284 4285std::optional<int64_t>4286RoutineOp::getGangDimValue(mlir::acc::DeviceType deviceType) {4287  if (!hasDeviceTypeValues(getGangDimDeviceType()))4288    return std::nullopt;4289  if (auto pos = findSegment(*getGangDimDeviceType(), deviceType)) {4290    auto intAttr = mlir::dyn_cast<mlir::IntegerAttr>((*getGangDim())[*pos]);4291    return intAttr.getInt();4292  }4293  return std::nullopt;4294}4295 4296//===----------------------------------------------------------------------===//4297// InitOp4298//===----------------------------------------------------------------------===//4299 4300LogicalResult acc::InitOp::verify() {4301  Operation *currOp = *this;4302  while ((currOp = currOp->getParentOp()))4303    if (isComputeOperation(currOp))4304      return emitOpError("cannot be nested in a compute operation");4305  return success();4306}4307 4308void acc::InitOp::addDeviceType(MLIRContext *context,4309                                mlir::acc::DeviceType deviceType) {4310  llvm::SmallVector<mlir::Attribute> deviceTypes;4311  if (getDeviceTypesAttr())4312    llvm::copy(getDeviceTypesAttr(), std::back_inserter(deviceTypes));4313 4314  deviceTypes.push_back(acc::DeviceTypeAttr::get(context, deviceType));4315  setDeviceTypesAttr(mlir::ArrayAttr::get(context, deviceTypes));4316}4317 4318//===----------------------------------------------------------------------===//4319// ShutdownOp4320//===----------------------------------------------------------------------===//4321 4322LogicalResult acc::ShutdownOp::verify() {4323  Operation *currOp = *this;4324  while ((currOp = currOp->getParentOp()))4325    if (isComputeOperation(currOp))4326      return emitOpError("cannot be nested in a compute operation");4327  return success();4328}4329 4330void acc::ShutdownOp::addDeviceType(MLIRContext *context,4331                                    mlir::acc::DeviceType deviceType) {4332  llvm::SmallVector<mlir::Attribute> deviceTypes;4333  if (getDeviceTypesAttr())4334    llvm::copy(getDeviceTypesAttr(), std::back_inserter(deviceTypes));4335 4336  deviceTypes.push_back(acc::DeviceTypeAttr::get(context, deviceType));4337  setDeviceTypesAttr(mlir::ArrayAttr::get(context, deviceTypes));4338}4339 4340//===----------------------------------------------------------------------===//4341// SetOp4342//===----------------------------------------------------------------------===//4343 4344LogicalResult acc::SetOp::verify() {4345  Operation *currOp = *this;4346  while ((currOp = currOp->getParentOp()))4347    if (isComputeOperation(currOp))4348      return emitOpError("cannot be nested in a compute operation");4349  if (!getDeviceTypeAttr() && !getDefaultAsync() && !getDeviceNum())4350    return emitOpError("at least one default_async, device_num, or device_type "4351                       "operand must appear");4352  return success();4353}4354 4355//===----------------------------------------------------------------------===//4356// UpdateOp4357//===----------------------------------------------------------------------===//4358 4359LogicalResult acc::UpdateOp::verify() {4360  // At least one of host or device should have a value.4361  if (getDataClauseOperands().empty())4362    return emitError("at least one value must be present in dataOperands");4363 4364  if (failed(verifyDeviceTypeCountMatch(*this, getAsyncOperands(),4365                                        getAsyncOperandsDeviceTypeAttr(),4366                                        "async")))4367    return failure();4368 4369  if (failed(verifyDeviceTypeAndSegmentCountMatch(4370          *this, getWaitOperands(), getWaitOperandsSegmentsAttr(),4371          getWaitOperandsDeviceTypeAttr(), "wait")))4372    return failure();4373 4374  if (failed(checkWaitAndAsyncConflict<acc::UpdateOp>(*this)))4375    return failure();4376 4377  for (mlir::Value operand : getDataClauseOperands())4378    if (!mlir::isa<acc::UpdateDeviceOp, acc::UpdateHostOp, acc::GetDevicePtrOp>(4379            operand.getDefiningOp()))4380      return emitError("expect data entry/exit operation or acc.getdeviceptr "4381                       "as defining op");4382 4383  return success();4384}4385 4386unsigned UpdateOp::getNumDataOperands() {4387  return getDataClauseOperands().size();4388}4389 4390Value UpdateOp::getDataOperand(unsigned i) {4391  unsigned numOptional = getAsyncOperands().size();4392  numOptional += getIfCond() ? 1 : 0;4393  return getOperand(getWaitOperands().size() + numOptional + i);4394}4395 4396void UpdateOp::getCanonicalizationPatterns(RewritePatternSet &results,4397                                           MLIRContext *context) {4398  results.add<RemoveConstantIfCondition<UpdateOp>>(context);4399}4400 4401bool UpdateOp::hasAsyncOnly() {4402  return hasAsyncOnly(mlir::acc::DeviceType::None);4403}4404 4405bool UpdateOp::hasAsyncOnly(mlir::acc::DeviceType deviceType) {4406  return hasDeviceType(getAsyncOnly(), deviceType);4407}4408 4409mlir::Value UpdateOp::getAsyncValue() {4410  return getAsyncValue(mlir::acc::DeviceType::None);4411}4412 4413mlir::Value UpdateOp::getAsyncValue(mlir::acc::DeviceType deviceType) {4414  if (!hasDeviceTypeValues(getAsyncOperandsDeviceType()))4415    return {};4416 4417  if (auto pos = findSegment(*getAsyncOperandsDeviceType(), deviceType))4418    return getAsyncOperands()[*pos];4419 4420  return {};4421}4422 4423bool UpdateOp::hasWaitOnly() {4424  return hasWaitOnly(mlir::acc::DeviceType::None);4425}4426 4427bool UpdateOp::hasWaitOnly(mlir::acc::DeviceType deviceType) {4428  return hasDeviceType(getWaitOnly(), deviceType);4429}4430 4431mlir::Operation::operand_range UpdateOp::getWaitValues() {4432  return getWaitValues(mlir::acc::DeviceType::None);4433}4434 4435mlir::Operation::operand_range4436UpdateOp::getWaitValues(mlir::acc::DeviceType deviceType) {4437  return getWaitValuesWithoutDevnum(4438      getWaitOperandsDeviceType(), getWaitOperands(), getWaitOperandsSegments(),4439      getHasWaitDevnum(), deviceType);4440}4441 4442mlir::Value UpdateOp::getWaitDevnum() {4443  return getWaitDevnum(mlir::acc::DeviceType::None);4444}4445 4446mlir::Value UpdateOp::getWaitDevnum(mlir::acc::DeviceType deviceType) {4447  return getWaitDevnumValue(getWaitOperandsDeviceType(), getWaitOperands(),4448                            getWaitOperandsSegments(), getHasWaitDevnum(),4449                            deviceType);4450}4451 4452void UpdateOp::addAsyncOnly(MLIRContext *context,4453                            llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {4454  setAsyncOnlyAttr(addDeviceTypeAffectedOperandHelper(4455      context, getAsyncOnlyAttr(), effectiveDeviceTypes));4456}4457 4458void UpdateOp::addAsyncOperand(4459    MLIRContext *context, mlir::Value newValue,4460    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {4461  setAsyncOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(4462      context, getAsyncOperandsDeviceTypeAttr(), effectiveDeviceTypes, newValue,4463      getAsyncOperandsMutable()));4464}4465 4466void UpdateOp::addWaitOnly(MLIRContext *context,4467                           llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {4468  setWaitOnlyAttr(addDeviceTypeAffectedOperandHelper(context, getWaitOnlyAttr(),4469                                                     effectiveDeviceTypes));4470}4471 4472void UpdateOp::addWaitOperands(4473    MLIRContext *context, bool hasDevnum, mlir::ValueRange newValues,4474    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {4475 4476  llvm::SmallVector<int32_t> segments;4477  if (getWaitOperandsSegments())4478    llvm::copy(*getWaitOperandsSegments(), std::back_inserter(segments));4479 4480  setWaitOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(4481      context, getWaitOperandsDeviceTypeAttr(), effectiveDeviceTypes, newValues,4482      getWaitOperandsMutable(), segments));4483  setWaitOperandsSegments(segments);4484 4485  llvm::SmallVector<mlir::Attribute> hasDevnums;4486  if (getHasWaitDevnumAttr())4487    llvm::copy(getHasWaitDevnumAttr(), std::back_inserter(hasDevnums));4488  hasDevnums.insert(4489      hasDevnums.end(),4490      std::max(effectiveDeviceTypes.size(), static_cast<size_t>(1)),4491      mlir::BoolAttr::get(context, hasDevnum));4492  setHasWaitDevnumAttr(mlir::ArrayAttr::get(context, hasDevnums));4493}4494 4495//===----------------------------------------------------------------------===//4496// WaitOp4497//===----------------------------------------------------------------------===//4498 4499LogicalResult acc::WaitOp::verify() {4500  // The async attribute represent the async clause without value. Therefore the4501  // attribute and operand cannot appear at the same time.4502  if (getAsyncOperand() && getAsync())4503    return emitError("async attribute cannot appear with asyncOperand");4504 4505  if (getWaitDevnum() && getWaitOperands().empty())4506    return emitError("wait_devnum cannot appear without waitOperands");4507 4508  return success();4509}4510 4511#define GET_OP_CLASSES4512#include "mlir/Dialect/OpenACC/OpenACCOps.cpp.inc"4513 4514#define GET_ATTRDEF_CLASSES4515#include "mlir/Dialect/OpenACC/OpenACCOpsAttributes.cpp.inc"4516 4517#define GET_TYPEDEF_CLASSES4518#include "mlir/Dialect/OpenACC/OpenACCOpsTypes.cpp.inc"4519 4520//===----------------------------------------------------------------------===//4521// acc dialect utilities4522//===----------------------------------------------------------------------===//4523 4524mlir::TypedValue<mlir::acc::PointerLikeType>4525mlir::acc::getVarPtr(mlir::Operation *accDataClauseOp) {4526  auto varPtr{llvm::TypeSwitch<mlir::Operation *,4527                               mlir::TypedValue<mlir::acc::PointerLikeType>>(4528                  accDataClauseOp)4529                  .Case<ACC_DATA_ENTRY_OPS>(4530                      [&](auto entry) { return entry.getVarPtr(); })4531                  .Case<mlir::acc::CopyoutOp, mlir::acc::UpdateHostOp>(4532                      [&](auto exit) { return exit.getVarPtr(); })4533                  .Default([&](mlir::Operation *) {4534                    return mlir::TypedValue<mlir::acc::PointerLikeType>();4535                  })};4536  return varPtr;4537}4538 4539mlir::Value mlir::acc::getVar(mlir::Operation *accDataClauseOp) {4540  auto varPtr{4541      llvm::TypeSwitch<mlir::Operation *, mlir::Value>(accDataClauseOp)4542          .Case<ACC_DATA_ENTRY_OPS>([&](auto entry) { return entry.getVar(); })4543          .Default([&](mlir::Operation *) { return mlir::Value(); })};4544  return varPtr;4545}4546 4547mlir::Type mlir::acc::getVarType(mlir::Operation *accDataClauseOp) {4548  auto varType{llvm::TypeSwitch<mlir::Operation *, mlir::Type>(accDataClauseOp)4549                   .Case<ACC_DATA_ENTRY_OPS>(4550                       [&](auto entry) { return entry.getVarType(); })4551                   .Case<mlir::acc::CopyoutOp, mlir::acc::UpdateHostOp>(4552                       [&](auto exit) { return exit.getVarType(); })4553                   .Default([&](mlir::Operation *) { return mlir::Type(); })};4554  return varType;4555}4556 4557mlir::TypedValue<mlir::acc::PointerLikeType>4558mlir::acc::getAccPtr(mlir::Operation *accDataClauseOp) {4559  auto accPtr{llvm::TypeSwitch<mlir::Operation *,4560                               mlir::TypedValue<mlir::acc::PointerLikeType>>(4561                  accDataClauseOp)4562                  .Case<ACC_DATA_ENTRY_OPS, ACC_DATA_EXIT_OPS>(4563                      [&](auto dataClause) { return dataClause.getAccPtr(); })4564                  .Default([&](mlir::Operation *) {4565                    return mlir::TypedValue<mlir::acc::PointerLikeType>();4566                  })};4567  return accPtr;4568}4569 4570mlir::Value mlir::acc::getAccVar(mlir::Operation *accDataClauseOp) {4571  auto accPtr{llvm::TypeSwitch<mlir::Operation *, mlir::Value>(accDataClauseOp)4572                  .Case<ACC_DATA_ENTRY_OPS, ACC_DATA_EXIT_OPS>(4573                      [&](auto dataClause) { return dataClause.getAccVar(); })4574                  .Default([&](mlir::Operation *) { return mlir::Value(); })};4575  return accPtr;4576}4577 4578mlir::Value mlir::acc::getVarPtrPtr(mlir::Operation *accDataClauseOp) {4579  auto varPtrPtr{4580      llvm::TypeSwitch<mlir::Operation *, mlir::Value>(accDataClauseOp)4581          .Case<ACC_DATA_ENTRY_OPS>(4582              [&](auto dataClause) { return dataClause.getVarPtrPtr(); })4583          .Default([&](mlir::Operation *) { return mlir::Value(); })};4584  return varPtrPtr;4585}4586 4587mlir::SmallVector<mlir::Value>4588mlir::acc::getBounds(mlir::Operation *accDataClauseOp) {4589  mlir::SmallVector<mlir::Value> bounds{4590      llvm::TypeSwitch<mlir::Operation *, mlir::SmallVector<mlir::Value>>(4591          accDataClauseOp)4592          .Case<ACC_DATA_ENTRY_OPS, ACC_DATA_EXIT_OPS>([&](auto dataClause) {4593            return mlir::SmallVector<mlir::Value>(4594                dataClause.getBounds().begin(), dataClause.getBounds().end());4595          })4596          .Default([&](mlir::Operation *) {4597            return mlir::SmallVector<mlir::Value, 0>();4598          })};4599  return bounds;4600}4601 4602mlir::SmallVector<mlir::Value>4603mlir::acc::getAsyncOperands(mlir::Operation *accDataClauseOp) {4604  return llvm::TypeSwitch<mlir::Operation *, mlir::SmallVector<mlir::Value>>(4605             accDataClauseOp)4606      .Case<ACC_DATA_ENTRY_OPS, ACC_DATA_EXIT_OPS>([&](auto dataClause) {4607        return mlir::SmallVector<mlir::Value>(4608            dataClause.getAsyncOperands().begin(),4609            dataClause.getAsyncOperands().end());4610      })4611      .Default([&](mlir::Operation *) {4612        return mlir::SmallVector<mlir::Value, 0>();4613      });4614}4615 4616mlir::ArrayAttr4617mlir::acc::getAsyncOperandsDeviceType(mlir::Operation *accDataClauseOp) {4618  return llvm::TypeSwitch<mlir::Operation *, mlir::ArrayAttr>(accDataClauseOp)4619      .Case<ACC_DATA_ENTRY_OPS, ACC_DATA_EXIT_OPS>([&](auto dataClause) {4620        return dataClause.getAsyncOperandsDeviceTypeAttr();4621      })4622      .Default([&](mlir::Operation *) { return mlir::ArrayAttr{}; });4623}4624 4625mlir::ArrayAttr mlir::acc::getAsyncOnly(mlir::Operation *accDataClauseOp) {4626  return llvm::TypeSwitch<mlir::Operation *, mlir::ArrayAttr>(accDataClauseOp)4627      .Case<ACC_DATA_ENTRY_OPS, ACC_DATA_EXIT_OPS>(4628          [&](auto dataClause) { return dataClause.getAsyncOnlyAttr(); })4629      .Default([&](mlir::Operation *) { return mlir::ArrayAttr{}; });4630}4631 4632std::optional<llvm::StringRef> mlir::acc::getVarName(mlir::Operation *accOp) {4633  auto name{4634      llvm::TypeSwitch<mlir::Operation *, std::optional<llvm::StringRef>>(accOp)4635          .Case<ACC_DATA_ENTRY_OPS>([&](auto entry) { return entry.getName(); })4636          .Default([&](mlir::Operation *) -> std::optional<llvm::StringRef> {4637            return {};4638          })};4639  return name;4640}4641 4642std::optional<mlir::acc::DataClause>4643mlir::acc::getDataClause(mlir::Operation *accDataEntryOp) {4644  auto dataClause{4645      llvm::TypeSwitch<mlir::Operation *, std::optional<mlir::acc::DataClause>>(4646          accDataEntryOp)4647          .Case<ACC_DATA_ENTRY_OPS>(4648              [&](auto entry) { return entry.getDataClause(); })4649          .Default([&](mlir::Operation *) { return std::nullopt; })};4650  return dataClause;4651}4652 4653bool mlir::acc::getImplicitFlag(mlir::Operation *accDataEntryOp) {4654  auto implicit{llvm::TypeSwitch<mlir::Operation *, bool>(accDataEntryOp)4655                    .Case<ACC_DATA_ENTRY_OPS>(4656                        [&](auto entry) { return entry.getImplicit(); })4657                    .Default([&](mlir::Operation *) { return false; })};4658  return implicit;4659}4660 4661mlir::ValueRange mlir::acc::getDataOperands(mlir::Operation *accOp) {4662  auto dataOperands{4663      llvm::TypeSwitch<mlir::Operation *, mlir::ValueRange>(accOp)4664          .Case<ACC_COMPUTE_AND_DATA_CONSTRUCT_OPS>(4665              [&](auto entry) { return entry.getDataClauseOperands(); })4666          .Default([&](mlir::Operation *) { return mlir::ValueRange(); })};4667  return dataOperands;4668}4669 4670mlir::MutableOperandRange4671mlir::acc::getMutableDataOperands(mlir::Operation *accOp) {4672  auto dataOperands{4673      llvm::TypeSwitch<mlir::Operation *, mlir::MutableOperandRange>(accOp)4674          .Case<ACC_COMPUTE_AND_DATA_CONSTRUCT_OPS>(4675              [&](auto entry) { return entry.getDataClauseOperandsMutable(); })4676          .Default([&](mlir::Operation *) { return nullptr; })};4677  return dataOperands;4678}4679 4680mlir::SymbolRefAttr mlir::acc::getRecipe(mlir::Operation *accOp) {4681  auto recipe{4682      llvm::TypeSwitch<mlir::Operation *, mlir::SymbolRefAttr>(accOp)4683          .Case<ACC_DATA_ENTRY_OPS>(4684              [&](auto entry) { return entry.getRecipeAttr(); })4685          .Default([&](mlir::Operation *) { return mlir::SymbolRefAttr{}; })};4686  return recipe;4687}4688