brintos / llvm-project-archived public Read only

1//===- OpenMPDialect.cpp - MLIR Dialect for OpenMP implementation ---------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the OpenMP dialect and its operations.10//11//===----------------------------------------------------------------------===//12 13#include "mlir/Dialect/OpenMP/OpenMPDialect.h"14#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"15#include "mlir/Dialect/Func/IR/FuncOps.h"16#include "mlir/Dialect/LLVMIR/LLVMTypes.h"17#include "mlir/Dialect/OpenMP/OpenMPClauseOperands.h"18#include "mlir/IR/Attributes.h"19#include "mlir/IR/BuiltinAttributes.h"20#include "mlir/IR/DialectImplementation.h"21#include "mlir/IR/OpImplementation.h"22#include "mlir/IR/OperationSupport.h"23#include "mlir/IR/SymbolTable.h"24#include "mlir/Interfaces/FoldInterfaces.h"25 26#include "llvm/ADT/ArrayRef.h"27#include "llvm/ADT/PostOrderIterator.h"28#include "llvm/ADT/STLExtras.h"29#include "llvm/ADT/STLForwardCompat.h"30#include "llvm/ADT/SmallString.h"31#include "llvm/ADT/StringExtras.h"32#include "llvm/ADT/StringRef.h"33#include "llvm/ADT/TypeSwitch.h"34#include "llvm/ADT/bit.h"35#include "llvm/Support/InterleavedRange.h"36#include <cstddef>37#include <iterator>38#include <optional>39#include <variant>40 41#include "mlir/Dialect/OpenMP/OpenMPOpsDialect.cpp.inc"42#include "mlir/Dialect/OpenMP/OpenMPOpsEnums.cpp.inc"43#include "mlir/Dialect/OpenMP/OpenMPOpsInterfaces.cpp.inc"44#include "mlir/Dialect/OpenMP/OpenMPTypeInterfaces.cpp.inc"45 46using namespace mlir;47using namespace mlir::omp;48 49static ArrayAttr makeArrayAttr(MLIRContext *context,50                               llvm::ArrayRef<Attribute> attrs) {51  return attrs.empty() ? nullptr : ArrayAttr::get(context, attrs);52}53 54static DenseBoolArrayAttr55makeDenseBoolArrayAttr(MLIRContext *ctx, const ArrayRef<bool> boolArray) {56  return boolArray.empty() ? nullptr : DenseBoolArrayAttr::get(ctx, boolArray);57}58 59static DenseI64ArrayAttr60makeDenseI64ArrayAttr(MLIRContext *ctx, const ArrayRef<int64_t> intArray) {61  return intArray.empty() ? nullptr : DenseI64ArrayAttr::get(ctx, intArray);62}63 64namespace {65struct MemRefPointerLikeModel66    : public PointerLikeType::ExternalModel<MemRefPointerLikeModel,67                                            MemRefType> {68  Type getElementType(Type pointer) const {69    return llvm::cast<MemRefType>(pointer).getElementType();70  }71};72 73struct LLVMPointerPointerLikeModel74    : public PointerLikeType::ExternalModel<LLVMPointerPointerLikeModel,75                                            LLVM::LLVMPointerType> {76  Type getElementType(Type pointer) const { return Type(); }77};78} // namespace79 80/// Generate a name of a canonical loop nest of the format81/// `<prefix>(_r<idx>_s<idx>)*`. Hereby, `_r<idx>` identifies the region82/// argument index of an operation that has multiple regions, if the operation83/// has multiple regions.84/// `_s<idx>` identifies the position of an operation within a region, where85/// only operations that may potentially contain loops ("container operations"86/// i.e. have region arguments) are counted. Again, it is omitted if there is87/// only one such operation in a region. If there are canonical loops nested88/// inside each other, also may also use the format `_d<num>` where <num> is the89/// nesting depth of the loop.90///91/// The generated name is a best-effort to make canonical loop unique within an92/// SSA namespace. This also means that regions with IsolatedFromAbove property93/// do not consider any parents or siblings.94static std::string generateLoopNestingName(StringRef prefix,95                                           CanonicalLoopOp op) {96  struct Component {97    /// If true, this component describes a region operand of an operation (the98    /// operand's owner) If false, this component describes an operation located99    /// in a parent region100    bool isRegionArgOfOp;101    bool skip = false;102    bool isUnique = false;103 104    size_t idx;105    Operation *op;106    Region *parentRegion;107    size_t loopDepth;108 109    Operation *&getOwnerOp() {110      assert(isRegionArgOfOp && "Must describe a region operand");111      return op;112    }113    size_t &getArgIdx() {114      assert(isRegionArgOfOp && "Must describe a region operand");115      return idx;116    }117 118    Operation *&getContainerOp() {119      assert(!isRegionArgOfOp && "Must describe a operation of a region");120      return op;121    }122    size_t &getOpPos() {123      assert(!isRegionArgOfOp && "Must describe a operation of a region");124      return idx;125    }126    bool isLoopOp() const {127      assert(!isRegionArgOfOp && "Must describe a operation of a region");128      return isa<CanonicalLoopOp>(op);129    }130    Region *&getParentRegion() {131      assert(!isRegionArgOfOp && "Must describe a operation of a region");132      return parentRegion;133    }134    size_t &getLoopDepth() {135      assert(!isRegionArgOfOp && "Must describe a operation of a region");136      return loopDepth;137    }138 139    void skipIf(bool v = true) { skip = skip || v; }140  };141 142  // List of ancestors, from inner to outer.143  // Alternates between144  //  * region argument of an operation145  //  * operation within a region146  SmallVector<Component> components;147 148  // Gather a list of parent regions and operations, and the position within149  // their parent150  Operation *o = op.getOperation();151  while (o) {152    // Operation within a region153    Region *r = o->getParentRegion();154    if (!r)155      break;156 157    llvm::ReversePostOrderTraversal<Block *> traversal(&r->getBlocks().front());158    size_t idx = 0;159    bool found = false;160    size_t sequentialIdx = -1;161    bool isOnlyContainerOp = true;162    for (Block *b : traversal) {163      for (Operation &op : *b) {164        if (&op == o && !found) {165          sequentialIdx = idx;166          found = true;167        }168        if (op.getNumRegions()) {169          idx += 1;170          if (idx > 1)171            isOnlyContainerOp = false;172        }173        if (found && !isOnlyContainerOp)174          break;175      }176    }177 178    Component &containerOpInRegion = components.emplace_back();179    containerOpInRegion.isRegionArgOfOp = false;180    containerOpInRegion.isUnique = isOnlyContainerOp;181    containerOpInRegion.getContainerOp() = o;182    containerOpInRegion.getOpPos() = sequentialIdx;183    containerOpInRegion.getParentRegion() = r;184 185    Operation *parent = r->getParentOp();186 187    // Region argument of an operation188    Component &regionArgOfOperation = components.emplace_back();189    regionArgOfOperation.isRegionArgOfOp = true;190    regionArgOfOperation.isUnique = true;191    regionArgOfOperation.getArgIdx() = 0;192    regionArgOfOperation.getOwnerOp() = parent;193 194    // The IsolatedFromAbove trait of the parent operation implies that each195    // individual region argument has its own separate namespace, so no196    // ambiguity.197    if (!parent || parent->hasTrait<mlir::OpTrait::IsIsolatedFromAbove>())198      break;199 200    // Component only needed if operation has multiple region operands. Region201    // arguments may be optional, but we currently do not consider this.202    if (parent->getRegions().size() > 1) {203      auto getRegionIndex = [](Operation *o, Region *r) {204        for (auto [idx, region] : llvm::enumerate(o->getRegions())) {205          if (&region == r)206            return idx;207        }208        llvm_unreachable("Region not child of its parent operation");209      };210      regionArgOfOperation.isUnique = false;211      regionArgOfOperation.getArgIdx() = getRegionIndex(parent, r);212    }213 214    // next parent215    o = parent;216  }217 218  // Determine whether a region-argument component is not needed219  for (Component &c : components)220    c.skipIf(c.isRegionArgOfOp && c.isUnique);221 222  // Find runs of nested loops and determine each loop's depth in the loop nest223  size_t numSurroundingLoops = 0;224  for (Component &c : llvm::reverse(components)) {225    if (c.skip)226      continue;227 228    // non-skipped multi-argument operands interrupt the loop nest229    if (c.isRegionArgOfOp) {230      numSurroundingLoops = 0;231      continue;232    }233 234    // Multiple loops in a region means each of them is the outermost loop of a235    // new loop nest236    if (!c.isUnique)237      numSurroundingLoops = 0;238 239    c.getLoopDepth() = numSurroundingLoops;240 241    // Next loop is surrounded by one more loop242    if (isa<CanonicalLoopOp>(c.getContainerOp()))243      numSurroundingLoops += 1;244  }245 246  // In loop nests, skip all but the innermost loop that contains the depth247  // number248  bool isLoopNest = false;249  for (Component &c : components) {250    if (c.skip || c.isRegionArgOfOp)251      continue;252 253    if (!isLoopNest && c.getLoopDepth() >= 1) {254      // Innermost loop of a loop nest of at least two loops255      isLoopNest = true;256    } else if (isLoopNest) {257      // Non-innermost loop of a loop nest258      c.skipIf(c.isUnique);259 260      // If there is no surrounding loop left, this must have been the outermost261      // loop; leave loop-nest mode for the next iteration262      if (c.getLoopDepth() == 0)263        isLoopNest = false;264    }265  }266 267  // Skip non-loop unambiguous regions (but they should interrupt loop nests, so268  // we mark them as skipped only after computing loop nests)269  for (Component &c : components)270    c.skipIf(!c.isRegionArgOfOp && c.isUnique &&271             !isa<CanonicalLoopOp>(c.getContainerOp()));272 273  // Components can be skipped if they are already disambiguated by their parent274  // (or does not have a parent)275  bool newRegion = true;276  for (Component &c : llvm::reverse(components)) {277    c.skipIf(newRegion && c.isUnique);278 279    // non-skipped components disambiguate unique children280    if (!c.skip)281      newRegion = true;282 283    // ...except canonical loops that need a suffix for each nest284    if (!c.isRegionArgOfOp && c.getContainerOp())285      newRegion = false;286  }287 288  // Compile the nesting name string289  SmallString<64> Name{prefix};290  llvm::raw_svector_ostream NameOS(Name);291  for (auto &c : llvm::reverse(components)) {292    if (c.skip)293      continue;294 295    if (c.isRegionArgOfOp)296      NameOS << "_r" << c.getArgIdx();297    else if (c.getLoopDepth() >= 1)298      NameOS << "_d" << c.getLoopDepth();299    else300      NameOS << "_s" << c.getOpPos();301  }302 303  return NameOS.str().str();304}305 306void OpenMPDialect::initialize() {307  addOperations<308#define GET_OP_LIST309#include "mlir/Dialect/OpenMP/OpenMPOps.cpp.inc"310      >();311  addAttributes<312#define GET_ATTRDEF_LIST313#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"314      >();315  addTypes<316#define GET_TYPEDEF_LIST317#include "mlir/Dialect/OpenMP/OpenMPOpsTypes.cpp.inc"318      >();319 320  declarePromisedInterface<ConvertToLLVMPatternInterface, OpenMPDialect>();321 322  MemRefType::attachInterface<MemRefPointerLikeModel>(*getContext());323  LLVM::LLVMPointerType::attachInterface<LLVMPointerPointerLikeModel>(324      *getContext());325 326  // Attach default offload module interface to module op to access327  // offload functionality through328  mlir::ModuleOp::attachInterface<mlir::omp::OffloadModuleDefaultModel>(329      *getContext());330 331  // Attach default declare target interfaces to operations which can be marked332  // as declare target (Global Operations and Functions/Subroutines in dialects333  // that Fortran (or other languages that lower to MLIR) translates too334  mlir::LLVM::GlobalOp::attachInterface<335      mlir::omp::DeclareTargetDefaultModel<mlir::LLVM::GlobalOp>>(336      *getContext());337  mlir::LLVM::LLVMFuncOp::attachInterface<338      mlir::omp::DeclareTargetDefaultModel<mlir::LLVM::LLVMFuncOp>>(339      *getContext());340  mlir::func::FuncOp::attachInterface<341      mlir::omp::DeclareTargetDefaultModel<mlir::func::FuncOp>>(*getContext());342}343 344//===----------------------------------------------------------------------===//345// Parser and printer for Allocate Clause346//===----------------------------------------------------------------------===//347 348/// Parse an allocate clause with allocators and a list of operands with types.349///350/// allocate-operand-list :: = allocate-operand |351///                            allocator-operand `,` allocate-operand-list352/// allocate-operand :: = ssa-id-and-type -> ssa-id-and-type353/// ssa-id-and-type ::= ssa-id `:` type354static ParseResult parseAllocateAndAllocator(355    OpAsmParser &parser,356    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &allocateVars,357    SmallVectorImpl<Type> &allocateTypes,358    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &allocatorVars,359    SmallVectorImpl<Type> &allocatorTypes) {360 361  return parser.parseCommaSeparatedList([&]() {362    OpAsmParser::UnresolvedOperand operand;363    Type type;364    if (parser.parseOperand(operand) || parser.parseColonType(type))365      return failure();366    allocatorVars.push_back(operand);367    allocatorTypes.push_back(type);368    if (parser.parseArrow())369      return failure();370    if (parser.parseOperand(operand) || parser.parseColonType(type))371      return failure();372 373    allocateVars.push_back(operand);374    allocateTypes.push_back(type);375    return success();376  });377}378 379/// Print allocate clause380static void printAllocateAndAllocator(OpAsmPrinter &p, Operation *op,381                                      OperandRange allocateVars,382                                      TypeRange allocateTypes,383                                      OperandRange allocatorVars,384                                      TypeRange allocatorTypes) {385  for (unsigned i = 0; i < allocateVars.size(); ++i) {386    std::string separator = i == allocateVars.size() - 1 ? "" : ", ";387    p << allocatorVars[i] << " : " << allocatorTypes[i] << " -> ";388    p << allocateVars[i] << " : " << allocateTypes[i] << separator;389  }390}391 392//===----------------------------------------------------------------------===//393// Parser and printer for a clause attribute (StringEnumAttr)394//===----------------------------------------------------------------------===//395 396template <typename ClauseAttr>397static ParseResult parseClauseAttr(AsmParser &parser, ClauseAttr &attr) {398  using ClauseT = decltype(std::declval<ClauseAttr>().getValue());399  StringRef enumStr;400  SMLoc loc = parser.getCurrentLocation();401  if (parser.parseKeyword(&enumStr))402    return failure();403  if (std::optional<ClauseT> enumValue = symbolizeEnum<ClauseT>(enumStr)) {404    attr = ClauseAttr::get(parser.getContext(), *enumValue);405    return success();406  }407  return parser.emitError(loc, "invalid clause value: '") << enumStr << "'";408}409 410template <typename ClauseAttr>411static void printClauseAttr(OpAsmPrinter &p, Operation *op, ClauseAttr attr) {412  p << stringifyEnum(attr.getValue());413}414 415//===----------------------------------------------------------------------===//416// Parser and printer for Linear Clause417//===----------------------------------------------------------------------===//418 419/// linear ::= `linear` `(` linear-list `)`420/// linear-list := linear-val | linear-val linear-list421/// linear-val := ssa-id-and-type `=` ssa-id-and-type422static ParseResult parseLinearClause(423    OpAsmParser &parser,424    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &linearVars,425    SmallVectorImpl<Type> &linearTypes,426    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &linearStepVars) {427  return parser.parseCommaSeparatedList([&]() {428    OpAsmParser::UnresolvedOperand var;429    Type type;430    OpAsmParser::UnresolvedOperand stepVar;431    if (parser.parseOperand(var) || parser.parseEqual() ||432        parser.parseOperand(stepVar) || parser.parseColonType(type))433      return failure();434 435    linearVars.push_back(var);436    linearTypes.push_back(type);437    linearStepVars.push_back(stepVar);438    return success();439  });440}441 442/// Print Linear Clause443static void printLinearClause(OpAsmPrinter &p, Operation *op,444                              ValueRange linearVars, TypeRange linearTypes,445                              ValueRange linearStepVars) {446  size_t linearVarsSize = linearVars.size();447  for (unsigned i = 0; i < linearVarsSize; ++i) {448    std::string separator = i == linearVarsSize - 1 ? "" : ", ";449    p << linearVars[i];450    if (linearStepVars.size() > i)451      p << " = " << linearStepVars[i];452    p << " : " << linearVars[i].getType() << separator;453  }454}455 456//===----------------------------------------------------------------------===//457// Verifier for Nontemporal Clause458//===----------------------------------------------------------------------===//459 460static LogicalResult verifyNontemporalClause(Operation *op,461                                             OperandRange nontemporalVars) {462 463  // Check if each var is unique - OpenMP 5.0 -> 2.9.3.1 section464  DenseSet<Value> nontemporalItems;465  for (const auto &it : nontemporalVars)466    if (!nontemporalItems.insert(it).second)467      return op->emitOpError() << "nontemporal variable used more than once";468 469  return success();470}471 472//===----------------------------------------------------------------------===//473// Parser, verifier and printer for Aligned Clause474//===----------------------------------------------------------------------===//475static LogicalResult verifyAlignedClause(Operation *op,476                                         std::optional<ArrayAttr> alignments,477                                         OperandRange alignedVars) {478  // Check if number of alignment values equals to number of aligned variables479  if (!alignedVars.empty()) {480    if (!alignments || alignments->size() != alignedVars.size())481      return op->emitOpError()482             << "expected as many alignment values as aligned variables";483  } else {484    if (alignments)485      return op->emitOpError() << "unexpected alignment values attribute";486    return success();487  }488 489  // Check if each var is aligned only once - OpenMP 4.5 -> 2.8.1 section490  DenseSet<Value> alignedItems;491  for (auto it : alignedVars)492    if (!alignedItems.insert(it).second)493      return op->emitOpError() << "aligned variable used more than once";494 495  if (!alignments)496    return success();497 498  // Check if all alignment values are positive - OpenMP 4.5 -> 2.8.1 section499  for (unsigned i = 0; i < (*alignments).size(); ++i) {500    if (auto intAttr = llvm::dyn_cast<IntegerAttr>((*alignments)[i])) {501      if (intAttr.getValue().sle(0))502        return op->emitOpError() << "alignment should be greater than 0";503    } else {504      return op->emitOpError() << "expected integer alignment";505    }506  }507 508  return success();509}510 511/// aligned ::= `aligned` `(` aligned-list `)`512/// aligned-list := aligned-val | aligned-val aligned-list513/// aligned-val := ssa-id-and-type `->` alignment514static ParseResult515parseAlignedClause(OpAsmParser &parser,516                   SmallVectorImpl<OpAsmParser::UnresolvedOperand> &alignedVars,517                   SmallVectorImpl<Type> &alignedTypes,518                   ArrayAttr &alignmentsAttr) {519  SmallVector<Attribute> alignmentVec;520  if (failed(parser.parseCommaSeparatedList([&]() {521        if (parser.parseOperand(alignedVars.emplace_back()) ||522            parser.parseColonType(alignedTypes.emplace_back()) ||523            parser.parseArrow() ||524            parser.parseAttribute(alignmentVec.emplace_back())) {525          return failure();526        }527        return success();528      })))529    return failure();530  SmallVector<Attribute> alignments(alignmentVec.begin(), alignmentVec.end());531  alignmentsAttr = ArrayAttr::get(parser.getContext(), alignments);532  return success();533}534 535/// Print Aligned Clause536static void printAlignedClause(OpAsmPrinter &p, Operation *op,537                               ValueRange alignedVars, TypeRange alignedTypes,538                               std::optional<ArrayAttr> alignments) {539  for (unsigned i = 0; i < alignedVars.size(); ++i) {540    if (i != 0)541      p << ", ";542    p << alignedVars[i] << " : " << alignedVars[i].getType();543    p << " -> " << (*alignments)[i];544  }545}546 547//===----------------------------------------------------------------------===//548// Parser, printer and verifier for Schedule Clause549//===----------------------------------------------------------------------===//550 551static ParseResult552verifyScheduleModifiers(OpAsmParser &parser,553                        SmallVectorImpl<SmallString<12>> &modifiers) {554  if (modifiers.size() > 2)555    return parser.emitError(parser.getNameLoc()) << " unexpected modifier(s)";556  for (const auto &mod : modifiers) {557    // Translate the string. If it has no value, then it was not a valid558    // modifier!559    auto symbol = symbolizeScheduleModifier(mod);560    if (!symbol)561      return parser.emitError(parser.getNameLoc())562             << " unknown modifier type: " << mod;563  }564 565  // If we have one modifier that is "simd", then stick a "none" modiifer in566  // index 0.567  if (modifiers.size() == 1) {568    if (symbolizeScheduleModifier(modifiers[0]) == ScheduleModifier::simd) {569      modifiers.push_back(modifiers[0]);570      modifiers[0] = stringifyScheduleModifier(ScheduleModifier::none);571    }572  } else if (modifiers.size() == 2) {573    // If there are two modifier:574    // First modifier should not be simd, second one should be simd575    if (symbolizeScheduleModifier(modifiers[0]) == ScheduleModifier::simd ||576        symbolizeScheduleModifier(modifiers[1]) != ScheduleModifier::simd)577      return parser.emitError(parser.getNameLoc())578             << " incorrect modifier order";579  }580  return success();581}582 583/// schedule ::= `schedule` `(` sched-list `)`584/// sched-list ::= sched-val | sched-val sched-list |585///                sched-val `,` sched-modifier586/// sched-val ::= sched-with-chunk | sched-wo-chunk587/// sched-with-chunk ::= sched-with-chunk-types (`=` ssa-id-and-type)?588/// sched-with-chunk-types ::= `static` | `dynamic` | `guided`589/// sched-wo-chunk ::=  `auto` | `runtime`590/// sched-modifier ::=  sched-mod-val | sched-mod-val `,` sched-mod-val591/// sched-mod-val ::=  `monotonic` | `nonmonotonic` | `simd` | `none`592static ParseResult593parseScheduleClause(OpAsmParser &parser, ClauseScheduleKindAttr &scheduleAttr,594                    ScheduleModifierAttr &scheduleMod, UnitAttr &scheduleSimd,595                    std::optional<OpAsmParser::UnresolvedOperand> &chunkSize,596                    Type &chunkType) {597  StringRef keyword;598  if (parser.parseKeyword(&keyword))599    return failure();600  std::optional<mlir::omp::ClauseScheduleKind> schedule =601      symbolizeClauseScheduleKind(keyword);602  if (!schedule)603    return parser.emitError(parser.getNameLoc()) << " expected schedule kind";604 605  scheduleAttr = ClauseScheduleKindAttr::get(parser.getContext(), *schedule);606  switch (*schedule) {607  case ClauseScheduleKind::Static:608  case ClauseScheduleKind::Dynamic:609  case ClauseScheduleKind::Guided:610    if (succeeded(parser.parseOptionalEqual())) {611      chunkSize = OpAsmParser::UnresolvedOperand{};612      if (parser.parseOperand(*chunkSize) || parser.parseColonType(chunkType))613        return failure();614    } else {615      chunkSize = std::nullopt;616    }617    break;618  case ClauseScheduleKind::Auto:619  case ClauseScheduleKind::Runtime:620  case ClauseScheduleKind::Distribute:621    chunkSize = std::nullopt;622  }623 624  // If there is a comma, we have one or more modifiers..625  SmallVector<SmallString<12>> modifiers;626  while (succeeded(parser.parseOptionalComma())) {627    StringRef mod;628    if (parser.parseKeyword(&mod))629      return failure();630    modifiers.push_back(mod);631  }632 633  if (verifyScheduleModifiers(parser, modifiers))634    return failure();635 636  if (!modifiers.empty()) {637    SMLoc loc = parser.getCurrentLocation();638    if (std::optional<ScheduleModifier> mod =639            symbolizeScheduleModifier(modifiers[0])) {640      scheduleMod = ScheduleModifierAttr::get(parser.getContext(), *mod);641    } else {642      return parser.emitError(loc, "invalid schedule modifier");643    }644    // Only SIMD attribute is allowed here!645    if (modifiers.size() > 1) {646      assert(symbolizeScheduleModifier(modifiers[1]) == ScheduleModifier::simd);647      scheduleSimd = UnitAttr::get(parser.getBuilder().getContext());648    }649  }650 651  return success();652}653 654/// Print schedule clause655static void printScheduleClause(OpAsmPrinter &p, Operation *op,656                                ClauseScheduleKindAttr scheduleKind,657                                ScheduleModifierAttr scheduleMod,658                                UnitAttr scheduleSimd, Value scheduleChunk,659                                Type scheduleChunkType) {660  p << stringifyClauseScheduleKind(scheduleKind.getValue());661  if (scheduleChunk)662    p << " = " << scheduleChunk << " : " << scheduleChunk.getType();663  if (scheduleMod)664    p << ", " << stringifyScheduleModifier(scheduleMod.getValue());665  if (scheduleSimd)666    p << ", simd";667}668 669//===----------------------------------------------------------------------===//670// Parser and printer for Order Clause671//===----------------------------------------------------------------------===//672 673// order ::= `order` `(` [order-modifier ':'] concurrent `)`674// order-modifier ::= reproducible | unconstrained675static ParseResult parseOrderClause(OpAsmParser &parser,676                                    ClauseOrderKindAttr &order,677                                    OrderModifierAttr &orderMod) {678  StringRef enumStr;679  SMLoc loc = parser.getCurrentLocation();680  if (parser.parseKeyword(&enumStr))681    return failure();682  if (std::optional<OrderModifier> enumValue =683          symbolizeOrderModifier(enumStr)) {684    orderMod = OrderModifierAttr::get(parser.getContext(), *enumValue);685    if (parser.parseOptionalColon())686      return failure();687    loc = parser.getCurrentLocation();688    if (parser.parseKeyword(&enumStr))689      return failure();690  }691  if (std::optional<ClauseOrderKind> enumValue =692          symbolizeClauseOrderKind(enumStr)) {693    order = ClauseOrderKindAttr::get(parser.getContext(), *enumValue);694    return success();695  }696  return parser.emitError(loc, "invalid clause value: '") << enumStr << "'";697}698 699static void printOrderClause(OpAsmPrinter &p, Operation *op,700                             ClauseOrderKindAttr order,701                             OrderModifierAttr orderMod) {702  if (orderMod)703    p << stringifyOrderModifier(orderMod.getValue()) << ":";704  if (order)705    p << stringifyClauseOrderKind(order.getValue());706}707 708template <typename ClauseTypeAttr, typename ClauseType>709static ParseResult710parseGranularityClause(OpAsmParser &parser, ClauseTypeAttr &prescriptiveness,711                       std::optional<OpAsmParser::UnresolvedOperand> &operand,712                       Type &operandType,713                       std::optional<ClauseType> (*symbolizeClause)(StringRef),714                       StringRef clauseName) {715  StringRef enumStr;716  if (succeeded(parser.parseOptionalKeyword(&enumStr))) {717    if (std::optional<ClauseType> enumValue = symbolizeClause(enumStr)) {718      prescriptiveness = ClauseTypeAttr::get(parser.getContext(), *enumValue);719      if (parser.parseComma())720        return failure();721    } else {722      return parser.emitError(parser.getCurrentLocation())723             << "invalid " << clauseName << " modifier : '" << enumStr << "'";724      ;725    }726  }727 728  OpAsmParser::UnresolvedOperand var;729  if (succeeded(parser.parseOperand(var))) {730    operand = var;731  } else {732    return parser.emitError(parser.getCurrentLocation())733           << "expected " << clauseName << " operand";734  }735 736  if (operand.has_value()) {737    if (parser.parseColonType(operandType))738      return failure();739  }740 741  return success();742}743 744template <typename ClauseTypeAttr, typename ClauseType>745static void746printGranularityClause(OpAsmPrinter &p, Operation *op,747                       ClauseTypeAttr prescriptiveness, Value operand,748                       mlir::Type operandType,749                       StringRef (*stringifyClauseType)(ClauseType)) {750 751  if (prescriptiveness)752    p << stringifyClauseType(prescriptiveness.getValue()) << ", ";753 754  if (operand)755    p << operand << ": " << operandType;756}757 758//===----------------------------------------------------------------------===//759// Parser and printer for grainsize Clause760//===----------------------------------------------------------------------===//761 762// grainsize ::= `grainsize` `(` [strict ':'] grain-size `)`763static ParseResult764parseGrainsizeClause(OpAsmParser &parser, ClauseGrainsizeTypeAttr &grainsizeMod,765                     std::optional<OpAsmParser::UnresolvedOperand> &grainsize,766                     Type &grainsizeType) {767  return parseGranularityClause<ClauseGrainsizeTypeAttr, ClauseGrainsizeType>(768      parser, grainsizeMod, grainsize, grainsizeType,769      &symbolizeClauseGrainsizeType, "grainsize");770}771 772static void printGrainsizeClause(OpAsmPrinter &p, Operation *op,773                                 ClauseGrainsizeTypeAttr grainsizeMod,774                                 Value grainsize, mlir::Type grainsizeType) {775  printGranularityClause<ClauseGrainsizeTypeAttr, ClauseGrainsizeType>(776      p, op, grainsizeMod, grainsize, grainsizeType,777      &stringifyClauseGrainsizeType);778}779 780//===----------------------------------------------------------------------===//781// Parser and printer for num_tasks Clause782//===----------------------------------------------------------------------===//783 784// numtask ::= `num_tasks` `(` [strict ':'] num-tasks `)`785static ParseResult786parseNumTasksClause(OpAsmParser &parser, ClauseNumTasksTypeAttr &numTasksMod,787                    std::optional<OpAsmParser::UnresolvedOperand> &numTasks,788                    Type &numTasksType) {789  return parseGranularityClause<ClauseNumTasksTypeAttr, ClauseNumTasksType>(790      parser, numTasksMod, numTasks, numTasksType, &symbolizeClauseNumTasksType,791      "num_tasks");792}793 794static void printNumTasksClause(OpAsmPrinter &p, Operation *op,795                                ClauseNumTasksTypeAttr numTasksMod,796                                Value numTasks, mlir::Type numTasksType) {797  printGranularityClause<ClauseNumTasksTypeAttr, ClauseNumTasksType>(798      p, op, numTasksMod, numTasks, numTasksType, &stringifyClauseNumTasksType);799}800 801//===----------------------------------------------------------------------===//802// Parsers for operations including clauses that define entry block arguments.803//===----------------------------------------------------------------------===//804 805namespace {806struct MapParseArgs {807  SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;808  SmallVectorImpl<Type> &types;809  MapParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,810               SmallVectorImpl<Type> &types)811      : vars(vars), types(types) {}812};813struct PrivateParseArgs {814  llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;815  llvm::SmallVectorImpl<Type> &types;816  ArrayAttr &syms;817  UnitAttr &needsBarrier;818  DenseI64ArrayAttr *mapIndices;819  PrivateParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,820                   SmallVectorImpl<Type> &types, ArrayAttr &syms,821                   UnitAttr &needsBarrier,822                   DenseI64ArrayAttr *mapIndices = nullptr)823      : vars(vars), types(types), syms(syms), needsBarrier(needsBarrier),824        mapIndices(mapIndices) {}825};826 827struct ReductionParseArgs {828  SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;829  SmallVectorImpl<Type> &types;830  DenseBoolArrayAttr &byref;831  ArrayAttr &syms;832  ReductionModifierAttr *modifier;833  ReductionParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,834                     SmallVectorImpl<Type> &types, DenseBoolArrayAttr &byref,835                     ArrayAttr &syms, ReductionModifierAttr *mod = nullptr)836      : vars(vars), types(types), byref(byref), syms(syms), modifier(mod) {}837};838 839struct AllRegionParseArgs {840  std::optional<MapParseArgs> hasDeviceAddrArgs;841  std::optional<MapParseArgs> hostEvalArgs;842  std::optional<ReductionParseArgs> inReductionArgs;843  std::optional<MapParseArgs> mapArgs;844  std::optional<PrivateParseArgs> privateArgs;845  std::optional<ReductionParseArgs> reductionArgs;846  std::optional<ReductionParseArgs> taskReductionArgs;847  std::optional<MapParseArgs> useDeviceAddrArgs;848  std::optional<MapParseArgs> useDevicePtrArgs;849};850} // namespace851 852static inline constexpr StringRef getPrivateNeedsBarrierSpelling() {853  return "private_barrier";854}855 856static ParseResult parseClauseWithRegionArgs(857    OpAsmParser &parser,858    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &operands,859    SmallVectorImpl<Type> &types,860    SmallVectorImpl<OpAsmParser::Argument> &regionPrivateArgs,861    ArrayAttr *symbols = nullptr, DenseI64ArrayAttr *mapIndices = nullptr,862    DenseBoolArrayAttr *byref = nullptr,863    ReductionModifierAttr *modifier = nullptr,864    UnitAttr *needsBarrier = nullptr) {865  SmallVector<SymbolRefAttr> symbolVec;866  SmallVector<int64_t> mapIndicesVec;867  SmallVector<bool> isByRefVec;868  unsigned regionArgOffset = regionPrivateArgs.size();869 870  if (parser.parseLParen())871    return failure();872 873  if (modifier && succeeded(parser.parseOptionalKeyword("mod"))) {874    StringRef enumStr;875    if (parser.parseColon() || parser.parseKeyword(&enumStr) ||876        parser.parseComma())877      return failure();878    std::optional<ReductionModifier> enumValue =879        symbolizeReductionModifier(enumStr);880    if (!enumValue.has_value())881      return failure();882    *modifier = ReductionModifierAttr::get(parser.getContext(), *enumValue);883    if (!*modifier)884      return failure();885  }886 887  if (parser.parseCommaSeparatedList([&]() {888        if (byref)889          isByRefVec.push_back(890              parser.parseOptionalKeyword("byref").succeeded());891 892        if (symbols && parser.parseAttribute(symbolVec.emplace_back()))893          return failure();894 895        if (parser.parseOperand(operands.emplace_back()) ||896            parser.parseArrow() ||897            parser.parseArgument(regionPrivateArgs.emplace_back()))898          return failure();899 900        if (mapIndices) {901          if (parser.parseOptionalLSquare().succeeded()) {902            if (parser.parseKeyword("map_idx") || parser.parseEqual() ||903                parser.parseInteger(mapIndicesVec.emplace_back()) ||904                parser.parseRSquare())905              return failure();906          } else {907            mapIndicesVec.push_back(-1);908          }909        }910 911        return success();912      }))913    return failure();914 915  if (parser.parseColon())916    return failure();917 918  if (parser.parseCommaSeparatedList([&]() {919        if (parser.parseType(types.emplace_back()))920          return failure();921 922        return success();923      }))924    return failure();925 926  if (operands.size() != types.size())927    return failure();928 929  if (parser.parseRParen())930    return failure();931 932  if (needsBarrier) {933    if (parser.parseOptionalKeyword(getPrivateNeedsBarrierSpelling())934            .succeeded())935      *needsBarrier = mlir::UnitAttr::get(parser.getContext());936  }937 938  auto *argsBegin = regionPrivateArgs.begin();939  MutableArrayRef argsSubrange(argsBegin + regionArgOffset,940                               argsBegin + regionArgOffset + types.size());941  for (auto [prv, type] : llvm::zip_equal(argsSubrange, types)) {942    prv.type = type;943  }944 945  if (symbols) {946    SmallVector<Attribute> symbolAttrs(symbolVec.begin(), symbolVec.end());947    *symbols = ArrayAttr::get(parser.getContext(), symbolAttrs);948  }949 950  if (!mapIndicesVec.empty())951    *mapIndices =952        mlir::DenseI64ArrayAttr::get(parser.getContext(), mapIndicesVec);953 954  if (byref)955    *byref = makeDenseBoolArrayAttr(parser.getContext(), isByRefVec);956 957  return success();958}959 960static ParseResult parseBlockArgClause(961    OpAsmParser &parser,962    llvm::SmallVectorImpl<OpAsmParser::Argument> &entryBlockArgs,963    StringRef keyword, std::optional<MapParseArgs> mapArgs) {964  if (succeeded(parser.parseOptionalKeyword(keyword))) {965    if (!mapArgs)966      return failure();967 968    if (failed(parseClauseWithRegionArgs(parser, mapArgs->vars, mapArgs->types,969                                         entryBlockArgs)))970      return failure();971  }972  return success();973}974 975static ParseResult parseBlockArgClause(976    OpAsmParser &parser,977    llvm::SmallVectorImpl<OpAsmParser::Argument> &entryBlockArgs,978    StringRef keyword, std::optional<PrivateParseArgs> privateArgs) {979  if (succeeded(parser.parseOptionalKeyword(keyword))) {980    if (!privateArgs)981      return failure();982 983    if (failed(parseClauseWithRegionArgs(984            parser, privateArgs->vars, privateArgs->types, entryBlockArgs,985            &privateArgs->syms, privateArgs->mapIndices, /*byref=*/nullptr,986            /*modifier=*/nullptr, &privateArgs->needsBarrier)))987      return failure();988  }989  return success();990}991 992static ParseResult parseBlockArgClause(993    OpAsmParser &parser,994    llvm::SmallVectorImpl<OpAsmParser::Argument> &entryBlockArgs,995    StringRef keyword, std::optional<ReductionParseArgs> reductionArgs) {996  if (succeeded(parser.parseOptionalKeyword(keyword))) {997    if (!reductionArgs)998      return failure();999    if (failed(parseClauseWithRegionArgs(1000            parser, reductionArgs->vars, reductionArgs->types, entryBlockArgs,1001            &reductionArgs->syms, /*mapIndices=*/nullptr, &reductionArgs->byref,1002            reductionArgs->modifier)))1003      return failure();1004  }1005  return success();1006}1007 1008static ParseResult parseBlockArgRegion(OpAsmParser &parser, Region &region,1009                                       AllRegionParseArgs args) {1010  llvm::SmallVector<OpAsmParser::Argument> entryBlockArgs;1011 1012  if (failed(parseBlockArgClause(parser, entryBlockArgs, "has_device_addr",1013                                 args.hasDeviceAddrArgs)))1014    return parser.emitError(parser.getCurrentLocation())1015           << "invalid `has_device_addr` format";1016 1017  if (failed(parseBlockArgClause(parser, entryBlockArgs, "host_eval",1018                                 args.hostEvalArgs)))1019    return parser.emitError(parser.getCurrentLocation())1020           << "invalid `host_eval` format";1021 1022  if (failed(parseBlockArgClause(parser, entryBlockArgs, "in_reduction",1023                                 args.inReductionArgs)))1024    return parser.emitError(parser.getCurrentLocation())1025           << "invalid `in_reduction` format";1026 1027  if (failed(parseBlockArgClause(parser, entryBlockArgs, "map_entries",1028                                 args.mapArgs)))1029    return parser.emitError(parser.getCurrentLocation())1030           << "invalid `map_entries` format";1031 1032  if (failed(parseBlockArgClause(parser, entryBlockArgs, "private",1033                                 args.privateArgs)))1034    return parser.emitError(parser.getCurrentLocation())1035           << "invalid `private` format";1036 1037  if (failed(parseBlockArgClause(parser, entryBlockArgs, "reduction",1038                                 args.reductionArgs)))1039    return parser.emitError(parser.getCurrentLocation())1040           << "invalid `reduction` format";1041 1042  if (failed(parseBlockArgClause(parser, entryBlockArgs, "task_reduction",1043                                 args.taskReductionArgs)))1044    return parser.emitError(parser.getCurrentLocation())1045           << "invalid `task_reduction` format";1046 1047  if (failed(parseBlockArgClause(parser, entryBlockArgs, "use_device_addr",1048                                 args.useDeviceAddrArgs)))1049    return parser.emitError(parser.getCurrentLocation())1050           << "invalid `use_device_addr` format";1051 1052  if (failed(parseBlockArgClause(parser, entryBlockArgs, "use_device_ptr",1053                                 args.useDevicePtrArgs)))1054    return parser.emitError(parser.getCurrentLocation())1055           << "invalid `use_device_addr` format";1056 1057  return parser.parseRegion(region, entryBlockArgs);1058}1059 1060// These parseXyz functions correspond to the custom<Xyz> definitions1061// in the .td file(s).1062static ParseResult parseTargetOpRegion(1063    OpAsmParser &parser, Region &region,1064    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &hasDeviceAddrVars,1065    SmallVectorImpl<Type> &hasDeviceAddrTypes,1066    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &hostEvalVars,1067    SmallVectorImpl<Type> &hostEvalTypes,1068    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &inReductionVars,1069    SmallVectorImpl<Type> &inReductionTypes,1070    DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms,1071    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &mapVars,1072    SmallVectorImpl<Type> &mapTypes,1073    llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,1074    llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,1075    UnitAttr &privateNeedsBarrier, DenseI64ArrayAttr &privateMaps) {1076  AllRegionParseArgs args;1077  args.hasDeviceAddrArgs.emplace(hasDeviceAddrVars, hasDeviceAddrTypes);1078  args.hostEvalArgs.emplace(hostEvalVars, hostEvalTypes);1079  args.inReductionArgs.emplace(inReductionVars, inReductionTypes,1080                               inReductionByref, inReductionSyms);1081  args.mapArgs.emplace(mapVars, mapTypes);1082  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,1083                           privateNeedsBarrier, &privateMaps);1084  return parseBlockArgRegion(parser, region, args);1085}1086 1087static ParseResult parseInReductionPrivateRegion(1088    OpAsmParser &parser, Region &region,1089    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &inReductionVars,1090    SmallVectorImpl<Type> &inReductionTypes,1091    DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms,1092    llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,1093    llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,1094    UnitAttr &privateNeedsBarrier) {1095  AllRegionParseArgs args;1096  args.inReductionArgs.emplace(inReductionVars, inReductionTypes,1097                               inReductionByref, inReductionSyms);1098  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,1099                           privateNeedsBarrier);1100  return parseBlockArgRegion(parser, region, args);1101}1102 1103static ParseResult parseInReductionPrivateReductionRegion(1104    OpAsmParser &parser, Region &region,1105    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &inReductionVars,1106    SmallVectorImpl<Type> &inReductionTypes,1107    DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms,1108    llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,1109    llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,1110    UnitAttr &privateNeedsBarrier, ReductionModifierAttr &reductionMod,1111    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &reductionVars,1112    SmallVectorImpl<Type> &reductionTypes, DenseBoolArrayAttr &reductionByref,1113    ArrayAttr &reductionSyms) {1114  AllRegionParseArgs args;1115  args.inReductionArgs.emplace(inReductionVars, inReductionTypes,1116                               inReductionByref, inReductionSyms);1117  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,1118                           privateNeedsBarrier);1119  args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,1120                             reductionSyms, &reductionMod);1121  return parseBlockArgRegion(parser, region, args);1122}1123 1124static ParseResult parsePrivateRegion(1125    OpAsmParser &parser, Region &region,1126    llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,1127    llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,1128    UnitAttr &privateNeedsBarrier) {1129  AllRegionParseArgs args;1130  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,1131                           privateNeedsBarrier);1132  return parseBlockArgRegion(parser, region, args);1133}1134 1135static ParseResult parsePrivateReductionRegion(1136    OpAsmParser &parser, Region &region,1137    llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,1138    llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,1139    UnitAttr &privateNeedsBarrier, ReductionModifierAttr &reductionMod,1140    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &reductionVars,1141    SmallVectorImpl<Type> &reductionTypes, DenseBoolArrayAttr &reductionByref,1142    ArrayAttr &reductionSyms) {1143  AllRegionParseArgs args;1144  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,1145                           privateNeedsBarrier);1146  args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,1147                             reductionSyms, &reductionMod);1148  return parseBlockArgRegion(parser, region, args);1149}1150 1151static ParseResult parseTaskReductionRegion(1152    OpAsmParser &parser, Region &region,1153    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &taskReductionVars,1154    SmallVectorImpl<Type> &taskReductionTypes,1155    DenseBoolArrayAttr &taskReductionByref, ArrayAttr &taskReductionSyms) {1156  AllRegionParseArgs args;1157  args.taskReductionArgs.emplace(taskReductionVars, taskReductionTypes,1158                                 taskReductionByref, taskReductionSyms);1159  return parseBlockArgRegion(parser, region, args);1160}1161 1162static ParseResult parseUseDeviceAddrUseDevicePtrRegion(1163    OpAsmParser &parser, Region &region,1164    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &useDeviceAddrVars,1165    SmallVectorImpl<Type> &useDeviceAddrTypes,1166    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &useDevicePtrVars,1167    SmallVectorImpl<Type> &useDevicePtrTypes) {1168  AllRegionParseArgs args;1169  args.useDeviceAddrArgs.emplace(useDeviceAddrVars, useDeviceAddrTypes);1170  args.useDevicePtrArgs.emplace(useDevicePtrVars, useDevicePtrTypes);1171  return parseBlockArgRegion(parser, region, args);1172}1173 1174//===----------------------------------------------------------------------===//1175// Printers for operations including clauses that define entry block arguments.1176//===----------------------------------------------------------------------===//1177 1178namespace {1179struct MapPrintArgs {1180  ValueRange vars;1181  TypeRange types;1182  MapPrintArgs(ValueRange vars, TypeRange types) : vars(vars), types(types) {}1183};1184struct PrivatePrintArgs {1185  ValueRange vars;1186  TypeRange types;1187  ArrayAttr syms;1188  UnitAttr needsBarrier;1189  DenseI64ArrayAttr mapIndices;1190  PrivatePrintArgs(ValueRange vars, TypeRange types, ArrayAttr syms,1191                   UnitAttr needsBarrier, DenseI64ArrayAttr mapIndices)1192      : vars(vars), types(types), syms(syms), needsBarrier(needsBarrier),1193        mapIndices(mapIndices) {}1194};1195struct ReductionPrintArgs {1196  ValueRange vars;1197  TypeRange types;1198  DenseBoolArrayAttr byref;1199  ArrayAttr syms;1200  ReductionModifierAttr modifier;1201  ReductionPrintArgs(ValueRange vars, TypeRange types, DenseBoolArrayAttr byref,1202                     ArrayAttr syms, ReductionModifierAttr mod = nullptr)1203      : vars(vars), types(types), byref(byref), syms(syms), modifier(mod) {}1204};1205struct AllRegionPrintArgs {1206  std::optional<MapPrintArgs> hasDeviceAddrArgs;1207  std::optional<MapPrintArgs> hostEvalArgs;1208  std::optional<ReductionPrintArgs> inReductionArgs;1209  std::optional<MapPrintArgs> mapArgs;1210  std::optional<PrivatePrintArgs> privateArgs;1211  std::optional<ReductionPrintArgs> reductionArgs;1212  std::optional<ReductionPrintArgs> taskReductionArgs;1213  std::optional<MapPrintArgs> useDeviceAddrArgs;1214  std::optional<MapPrintArgs> useDevicePtrArgs;1215};1216} // namespace1217 1218static void printClauseWithRegionArgs(1219    OpAsmPrinter &p, MLIRContext *ctx, StringRef clauseName,1220    ValueRange argsSubrange, ValueRange operands, TypeRange types,1221    ArrayAttr symbols = nullptr, DenseI64ArrayAttr mapIndices = nullptr,1222    DenseBoolArrayAttr byref = nullptr,1223    ReductionModifierAttr modifier = nullptr, UnitAttr needsBarrier = nullptr) {1224  if (argsSubrange.empty())1225    return;1226 1227  p << clauseName << "(";1228 1229  if (modifier)1230    p << "mod: " << stringifyReductionModifier(modifier.getValue()) << ", ";1231 1232  if (!symbols) {1233    llvm::SmallVector<Attribute> values(operands.size(), nullptr);1234    symbols = ArrayAttr::get(ctx, values);1235  }1236 1237  if (!mapIndices) {1238    llvm::SmallVector<int64_t> values(operands.size(), -1);1239    mapIndices = DenseI64ArrayAttr::get(ctx, values);1240  }1241 1242  if (!byref) {1243    mlir::SmallVector<bool> values(operands.size(), false);1244    byref = DenseBoolArrayAttr::get(ctx, values);1245  }1246 1247  llvm::interleaveComma(llvm::zip_equal(operands, argsSubrange, symbols,1248                                        mapIndices.asArrayRef(),1249                                        byref.asArrayRef()),1250                        p, [&p](auto t) {1251                          auto [op, arg, sym, map, isByRef] = t;1252                          if (isByRef)1253                            p << "byref ";1254                          if (sym)1255                            p << sym << " ";1256 1257                          p << op << " -> " << arg;1258 1259                          if (map != -1)1260                            p << " [map_idx=" << map << "]";1261                        });1262  p << " : ";1263  llvm::interleaveComma(types, p);1264  p << ") ";1265 1266  if (needsBarrier)1267    p << getPrivateNeedsBarrierSpelling() << " ";1268}1269 1270static void printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx,1271                                StringRef clauseName, ValueRange argsSubrange,1272                                std::optional<MapPrintArgs> mapArgs) {1273  if (mapArgs)1274    printClauseWithRegionArgs(p, ctx, clauseName, argsSubrange, mapArgs->vars,1275                              mapArgs->types);1276}1277 1278static void printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx,1279                                StringRef clauseName, ValueRange argsSubrange,1280                                std::optional<PrivatePrintArgs> privateArgs) {1281  if (privateArgs)1282    printClauseWithRegionArgs(1283        p, ctx, clauseName, argsSubrange, privateArgs->vars, privateArgs->types,1284        privateArgs->syms, privateArgs->mapIndices, /*byref=*/nullptr,1285        /*modifier=*/nullptr, privateArgs->needsBarrier);1286}1287 1288static void1289printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx, StringRef clauseName,1290                    ValueRange argsSubrange,1291                    std::optional<ReductionPrintArgs> reductionArgs) {1292  if (reductionArgs)1293    printClauseWithRegionArgs(p, ctx, clauseName, argsSubrange,1294                              reductionArgs->vars, reductionArgs->types,1295                              reductionArgs->syms, /*mapIndices=*/nullptr,1296                              reductionArgs->byref, reductionArgs->modifier);1297}1298 1299static void printBlockArgRegion(OpAsmPrinter &p, Operation *op, Region &region,1300                                const AllRegionPrintArgs &args) {1301  auto iface = llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(op);1302  MLIRContext *ctx = op->getContext();1303 1304  printBlockArgClause(p, ctx, "has_device_addr",1305                      iface.getHasDeviceAddrBlockArgs(),1306                      args.hasDeviceAddrArgs);1307  printBlockArgClause(p, ctx, "host_eval", iface.getHostEvalBlockArgs(),1308                      args.hostEvalArgs);1309  printBlockArgClause(p, ctx, "in_reduction", iface.getInReductionBlockArgs(),1310                      args.inReductionArgs);1311  printBlockArgClause(p, ctx, "map_entries", iface.getMapBlockArgs(),1312                      args.mapArgs);1313  printBlockArgClause(p, ctx, "private", iface.getPrivateBlockArgs(),1314                      args.privateArgs);1315  printBlockArgClause(p, ctx, "reduction", iface.getReductionBlockArgs(),1316                      args.reductionArgs);1317  printBlockArgClause(p, ctx, "task_reduction",1318                      iface.getTaskReductionBlockArgs(),1319                      args.taskReductionArgs);1320  printBlockArgClause(p, ctx, "use_device_addr",1321                      iface.getUseDeviceAddrBlockArgs(),1322                      args.useDeviceAddrArgs);1323  printBlockArgClause(p, ctx, "use_device_ptr",1324                      iface.getUseDevicePtrBlockArgs(), args.useDevicePtrArgs);1325 1326  p.printRegion(region, /*printEntryBlockArgs=*/false);1327}1328 1329// These parseXyz functions correspond to the custom<Xyz> definitions1330// in the .td file(s).1331static void printTargetOpRegion(1332    OpAsmPrinter &p, Operation *op, Region &region,1333    ValueRange hasDeviceAddrVars, TypeRange hasDeviceAddrTypes,1334    ValueRange hostEvalVars, TypeRange hostEvalTypes,1335    ValueRange inReductionVars, TypeRange inReductionTypes,1336    DenseBoolArrayAttr inReductionByref, ArrayAttr inReductionSyms,1337    ValueRange mapVars, TypeRange mapTypes, ValueRange privateVars,1338    TypeRange privateTypes, ArrayAttr privateSyms, UnitAttr privateNeedsBarrier,1339    DenseI64ArrayAttr privateMaps) {1340  AllRegionPrintArgs args;1341  args.hasDeviceAddrArgs.emplace(hasDeviceAddrVars, hasDeviceAddrTypes);1342  args.hostEvalArgs.emplace(hostEvalVars, hostEvalTypes);1343  args.inReductionArgs.emplace(inReductionVars, inReductionTypes,1344                               inReductionByref, inReductionSyms);1345  args.mapArgs.emplace(mapVars, mapTypes);1346  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,1347                           privateNeedsBarrier, privateMaps);1348  printBlockArgRegion(p, op, region, args);1349}1350 1351static void printInReductionPrivateRegion(1352    OpAsmPrinter &p, Operation *op, Region &region, ValueRange inReductionVars,1353    TypeRange inReductionTypes, DenseBoolArrayAttr inReductionByref,1354    ArrayAttr inReductionSyms, ValueRange privateVars, TypeRange privateTypes,1355    ArrayAttr privateSyms, UnitAttr privateNeedsBarrier) {1356  AllRegionPrintArgs args;1357  args.inReductionArgs.emplace(inReductionVars, inReductionTypes,1358                               inReductionByref, inReductionSyms);1359  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,1360                           privateNeedsBarrier,1361                           /*mapIndices=*/nullptr);1362  printBlockArgRegion(p, op, region, args);1363}1364 1365static void printInReductionPrivateReductionRegion(1366    OpAsmPrinter &p, Operation *op, Region &region, ValueRange inReductionVars,1367    TypeRange inReductionTypes, DenseBoolArrayAttr inReductionByref,1368    ArrayAttr inReductionSyms, ValueRange privateVars, TypeRange privateTypes,1369    ArrayAttr privateSyms, UnitAttr privateNeedsBarrier,1370    ReductionModifierAttr reductionMod, ValueRange reductionVars,1371    TypeRange reductionTypes, DenseBoolArrayAttr reductionByref,1372    ArrayAttr reductionSyms) {1373  AllRegionPrintArgs args;1374  args.inReductionArgs.emplace(inReductionVars, inReductionTypes,1375                               inReductionByref, inReductionSyms);1376  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,1377                           privateNeedsBarrier,1378                           /*mapIndices=*/nullptr);1379  args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,1380                             reductionSyms, reductionMod);1381  printBlockArgRegion(p, op, region, args);1382}1383 1384static void printPrivateRegion(OpAsmPrinter &p, Operation *op, Region &region,1385                               ValueRange privateVars, TypeRange privateTypes,1386                               ArrayAttr privateSyms,1387                               UnitAttr privateNeedsBarrier) {1388  AllRegionPrintArgs args;1389  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,1390                           privateNeedsBarrier,1391                           /*mapIndices=*/nullptr);1392  printBlockArgRegion(p, op, region, args);1393}1394 1395static void printPrivateReductionRegion(1396    OpAsmPrinter &p, Operation *op, Region &region, ValueRange privateVars,1397    TypeRange privateTypes, ArrayAttr privateSyms, UnitAttr privateNeedsBarrier,1398    ReductionModifierAttr reductionMod, ValueRange reductionVars,1399    TypeRange reductionTypes, DenseBoolArrayAttr reductionByref,1400    ArrayAttr reductionSyms) {1401  AllRegionPrintArgs args;1402  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,1403                           privateNeedsBarrier,1404                           /*mapIndices=*/nullptr);1405  args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,1406                             reductionSyms, reductionMod);1407  printBlockArgRegion(p, op, region, args);1408}1409 1410static void printTaskReductionRegion(OpAsmPrinter &p, Operation *op,1411                                     Region &region,1412                                     ValueRange taskReductionVars,1413                                     TypeRange taskReductionTypes,1414                                     DenseBoolArrayAttr taskReductionByref,1415                                     ArrayAttr taskReductionSyms) {1416  AllRegionPrintArgs args;1417  args.taskReductionArgs.emplace(taskReductionVars, taskReductionTypes,1418                                 taskReductionByref, taskReductionSyms);1419  printBlockArgRegion(p, op, region, args);1420}1421 1422static void printUseDeviceAddrUseDevicePtrRegion(OpAsmPrinter &p, Operation *op,1423                                                 Region &region,1424                                                 ValueRange useDeviceAddrVars,1425                                                 TypeRange useDeviceAddrTypes,1426                                                 ValueRange useDevicePtrVars,1427                                                 TypeRange useDevicePtrTypes) {1428  AllRegionPrintArgs args;1429  args.useDeviceAddrArgs.emplace(useDeviceAddrVars, useDeviceAddrTypes);1430  args.useDevicePtrArgs.emplace(useDevicePtrVars, useDevicePtrTypes);1431  printBlockArgRegion(p, op, region, args);1432}1433 1434/// Verifies Reduction Clause1435static LogicalResult1436verifyReductionVarList(Operation *op, std::optional<ArrayAttr> reductionSyms,1437                       OperandRange reductionVars,1438                       std::optional<ArrayRef<bool>> reductionByref) {1439  if (!reductionVars.empty()) {1440    if (!reductionSyms || reductionSyms->size() != reductionVars.size())1441      return op->emitOpError()1442             << "expected as many reduction symbol references "1443                "as reduction variables";1444    if (reductionByref && reductionByref->size() != reductionVars.size())1445      return op->emitError() << "expected as many reduction variable by "1446                                "reference attributes as reduction variables";1447  } else {1448    if (reductionSyms)1449      return op->emitOpError() << "unexpected reduction symbol references";1450    return success();1451  }1452 1453  // TODO: The followings should be done in1454  // SymbolUserOpInterface::verifySymbolUses.1455  DenseSet<Value> accumulators;1456  for (auto args : llvm::zip(reductionVars, *reductionSyms)) {1457    Value accum = std::get<0>(args);1458 1459    if (!accumulators.insert(accum).second)1460      return op->emitOpError() << "accumulator variable used more than once";1461 1462    Type varType = accum.getType();1463    auto symbolRef = llvm::cast<SymbolRefAttr>(std::get<1>(args));1464    auto decl =1465        SymbolTable::lookupNearestSymbolFrom<DeclareReductionOp>(op, symbolRef);1466    if (!decl)1467      return op->emitOpError() << "expected symbol reference " << symbolRef1468                               << " to point to a reduction declaration";1469 1470    if (decl.getAccumulatorType() && decl.getAccumulatorType() != varType)1471      return op->emitOpError()1472             << "expected accumulator (" << varType1473             << ") to be the same type as reduction declaration ("1474             << decl.getAccumulatorType() << ")";1475  }1476 1477  return success();1478}1479 1480//===----------------------------------------------------------------------===//1481// Parser, printer and verifier for Copyprivate1482//===----------------------------------------------------------------------===//1483 1484/// copyprivate-entry-list ::= copyprivate-entry1485///                          | copyprivate-entry-list `,` copyprivate-entry1486/// copyprivate-entry ::= ssa-id `->` symbol-ref `:` type1487static ParseResult parseCopyprivate(1488    OpAsmParser &parser,1489    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &copyprivateVars,1490    SmallVectorImpl<Type> &copyprivateTypes, ArrayAttr &copyprivateSyms) {1491  SmallVector<SymbolRefAttr> symsVec;1492  if (failed(parser.parseCommaSeparatedList([&]() {1493        if (parser.parseOperand(copyprivateVars.emplace_back()) ||1494            parser.parseArrow() ||1495            parser.parseAttribute(symsVec.emplace_back()) ||1496            parser.parseColonType(copyprivateTypes.emplace_back()))1497          return failure();1498        return success();1499      })))1500    return failure();1501  SmallVector<Attribute> syms(symsVec.begin(), symsVec.end());1502  copyprivateSyms = ArrayAttr::get(parser.getContext(), syms);1503  return success();1504}1505 1506/// Print Copyprivate clause1507static void printCopyprivate(OpAsmPrinter &p, Operation *op,1508                             OperandRange copyprivateVars,1509                             TypeRange copyprivateTypes,1510                             std::optional<ArrayAttr> copyprivateSyms) {1511  if (!copyprivateSyms.has_value())1512    return;1513  llvm::interleaveComma(1514      llvm::zip(copyprivateVars, *copyprivateSyms, copyprivateTypes), p,1515      [&](const auto &args) {1516        p << std::get<0>(args) << " -> " << std::get<1>(args) << " : "1517          << std::get<2>(args);1518      });1519}1520 1521/// Verifies CopyPrivate Clause1522static LogicalResult1523verifyCopyprivateVarList(Operation *op, OperandRange copyprivateVars,1524                         std::optional<ArrayAttr> copyprivateSyms) {1525  size_t copyprivateSymsSize =1526      copyprivateSyms.has_value() ? copyprivateSyms->size() : 0;1527  if (copyprivateSymsSize != copyprivateVars.size())1528    return op->emitOpError() << "inconsistent number of copyprivate vars (= "1529                             << copyprivateVars.size()1530                             << ") and functions (= " << copyprivateSymsSize1531                             << "), both must be equal";1532  if (!copyprivateSyms.has_value())1533    return success();1534 1535  for (auto copyprivateVarAndSym :1536       llvm::zip(copyprivateVars, *copyprivateSyms)) {1537    auto symbolRef =1538        llvm::cast<SymbolRefAttr>(std::get<1>(copyprivateVarAndSym));1539    std::optional<std::variant<mlir::func::FuncOp, mlir::LLVM::LLVMFuncOp>>1540        funcOp;1541    if (mlir::func::FuncOp mlirFuncOp =1542            SymbolTable::lookupNearestSymbolFrom<mlir::func::FuncOp>(op,1543                                                                     symbolRef))1544      funcOp = mlirFuncOp;1545    else if (mlir::LLVM::LLVMFuncOp llvmFuncOp =1546                 SymbolTable::lookupNearestSymbolFrom<mlir::LLVM::LLVMFuncOp>(1547                     op, symbolRef))1548      funcOp = llvmFuncOp;1549 1550    auto getNumArguments = [&] {1551      return std::visit([](auto &f) { return f.getNumArguments(); }, *funcOp);1552    };1553 1554    auto getArgumentType = [&](unsigned i) {1555      return std::visit([i](auto &f) { return f.getArgumentTypes()[i]; },1556                        *funcOp);1557    };1558 1559    if (!funcOp)1560      return op->emitOpError() << "expected symbol reference " << symbolRef1561                               << " to point to a copy function";1562 1563    if (getNumArguments() != 2)1564      return op->emitOpError()1565             << "expected copy function " << symbolRef << " to have 2 operands";1566 1567    Type argTy = getArgumentType(0);1568    if (argTy != getArgumentType(1))1569      return op->emitOpError() << "expected copy function " << symbolRef1570                               << " arguments to have the same type";1571 1572    Type varType = std::get<0>(copyprivateVarAndSym).getType();1573    if (argTy != varType)1574      return op->emitOpError()1575             << "expected copy function arguments' type (" << argTy1576             << ") to be the same as copyprivate variable's type (" << varType1577             << ")";1578  }1579 1580  return success();1581}1582 1583//===----------------------------------------------------------------------===//1584// Parser, printer and verifier for DependVarList1585//===----------------------------------------------------------------------===//1586 1587/// depend-entry-list ::= depend-entry1588///                     | depend-entry-list `,` depend-entry1589/// depend-entry ::= depend-kind `->` ssa-id `:` type1590static ParseResult1591parseDependVarList(OpAsmParser &parser,1592                   SmallVectorImpl<OpAsmParser::UnresolvedOperand> &dependVars,1593                   SmallVectorImpl<Type> &dependTypes, ArrayAttr &dependKinds) {1594  SmallVector<ClauseTaskDependAttr> kindsVec;1595  if (failed(parser.parseCommaSeparatedList([&]() {1596        StringRef keyword;1597        if (parser.parseKeyword(&keyword) || parser.parseArrow() ||1598            parser.parseOperand(dependVars.emplace_back()) ||1599            parser.parseColonType(dependTypes.emplace_back()))1600          return failure();1601        if (std::optional<ClauseTaskDepend> keywordDepend =1602                (symbolizeClauseTaskDepend(keyword)))1603          kindsVec.emplace_back(1604              ClauseTaskDependAttr::get(parser.getContext(), *keywordDepend));1605        else1606          return failure();1607        return success();1608      })))1609    return failure();1610  SmallVector<Attribute> kinds(kindsVec.begin(), kindsVec.end());1611  dependKinds = ArrayAttr::get(parser.getContext(), kinds);1612  return success();1613}1614 1615/// Print Depend clause1616static void printDependVarList(OpAsmPrinter &p, Operation *op,1617                               OperandRange dependVars, TypeRange dependTypes,1618                               std::optional<ArrayAttr> dependKinds) {1619 1620  for (unsigned i = 0, e = dependKinds->size(); i < e; ++i) {1621    if (i != 0)1622      p << ", ";1623    p << stringifyClauseTaskDepend(1624             llvm::cast<mlir::omp::ClauseTaskDependAttr>((*dependKinds)[i])1625                 .getValue())1626      << " -> " << dependVars[i] << " : " << dependTypes[i];1627  }1628}1629 1630/// Verifies Depend clause1631static LogicalResult verifyDependVarList(Operation *op,1632                                         std::optional<ArrayAttr> dependKinds,1633                                         OperandRange dependVars) {1634  if (!dependVars.empty()) {1635    if (!dependKinds || dependKinds->size() != dependVars.size())1636      return op->emitOpError() << "expected as many depend values"1637                                  " as depend variables";1638  } else {1639    if (dependKinds && !dependKinds->empty())1640      return op->emitOpError() << "unexpected depend values";1641    return success();1642  }1643 1644  return success();1645}1646 1647//===----------------------------------------------------------------------===//1648// Parser, printer and verifier for Synchronization Hint (2.17.12)1649//===----------------------------------------------------------------------===//1650 1651/// Parses a Synchronization Hint clause. The value of hint is an integer1652/// which is a combination of different hints from `omp_sync_hint_t`.1653///1654/// hint-clause = `hint` `(` hint-value `)`1655static ParseResult parseSynchronizationHint(OpAsmParser &parser,1656                                            IntegerAttr &hintAttr) {1657  StringRef hintKeyword;1658  int64_t hint = 0;1659  if (succeeded(parser.parseOptionalKeyword("none"))) {1660    hintAttr = IntegerAttr::get(parser.getBuilder().getI64Type(), 0);1661    return success();1662  }1663  auto parseKeyword = [&]() -> ParseResult {1664    if (failed(parser.parseKeyword(&hintKeyword)))1665      return failure();1666    if (hintKeyword == "uncontended")1667      hint |= 1;1668    else if (hintKeyword == "contended")1669      hint |= 2;1670    else if (hintKeyword == "nonspeculative")1671      hint |= 4;1672    else if (hintKeyword == "speculative")1673      hint |= 8;1674    else1675      return parser.emitError(parser.getCurrentLocation())1676             << hintKeyword << " is not a valid hint";1677    return success();1678  };1679  if (parser.parseCommaSeparatedList(parseKeyword))1680    return failure();1681  hintAttr = IntegerAttr::get(parser.getBuilder().getI64Type(), hint);1682  return success();1683}1684 1685/// Prints a Synchronization Hint clause1686static void printSynchronizationHint(OpAsmPrinter &p, Operation *op,1687                                     IntegerAttr hintAttr) {1688  int64_t hint = hintAttr.getInt();1689 1690  if (hint == 0) {1691    p << "none";1692    return;1693  }1694 1695  // Helper function to get n-th bit from the right end of `value`1696  auto bitn = [](int value, int n) -> bool { return value & (1 << n); };1697 1698  bool uncontended = bitn(hint, 0);1699  bool contended = bitn(hint, 1);1700  bool nonspeculative = bitn(hint, 2);1701  bool speculative = bitn(hint, 3);1702 1703  SmallVector<StringRef> hints;1704  if (uncontended)1705    hints.push_back("uncontended");1706  if (contended)1707    hints.push_back("contended");1708  if (nonspeculative)1709    hints.push_back("nonspeculative");1710  if (speculative)1711    hints.push_back("speculative");1712 1713  llvm::interleaveComma(hints, p);1714}1715 1716/// Verifies a synchronization hint clause1717static LogicalResult verifySynchronizationHint(Operation *op, uint64_t hint) {1718 1719  // Helper function to get n-th bit from the right end of `value`1720  auto bitn = [](int value, int n) -> bool { return value & (1 << n); };1721 1722  bool uncontended = bitn(hint, 0);1723  bool contended = bitn(hint, 1);1724  bool nonspeculative = bitn(hint, 2);1725  bool speculative = bitn(hint, 3);1726 1727  if (uncontended && contended)1728    return op->emitOpError() << "the hints omp_sync_hint_uncontended and "1729                                "omp_sync_hint_contended cannot be combined";1730  if (nonspeculative && speculative)1731    return op->emitOpError() << "the hints omp_sync_hint_nonspeculative and "1732                                "omp_sync_hint_speculative cannot be combined.";1733  return success();1734}1735 1736//===----------------------------------------------------------------------===//1737// Parser, printer and verifier for Target1738//===----------------------------------------------------------------------===//1739 1740// Helper function to get bitwise AND of `value` and 'flag' then return it as a1741// boolean1742static bool mapTypeToBool(ClauseMapFlags value, ClauseMapFlags flag) {1743  return (value & flag) == flag;1744}1745 1746/// Parses a map_entries map type from a string format back into its numeric1747/// value.1748///1749/// map-clause = `map_clauses (  ( `(` `always, `? `implicit, `? `ompx_hold, `?1750/// `close, `? `present, `? ( `to` | `from` | `delete` `)` )+ `)` )1751static ParseResult parseMapClause(OpAsmParser &parser,1752                                  ClauseMapFlagsAttr &mapType) {1753  ClauseMapFlags mapTypeBits = ClauseMapFlags::none;1754  // This simply verifies the correct keyword is read in, the1755  // keyword itself is stored inside of the operation1756  auto parseTypeAndMod = [&]() -> ParseResult {1757    StringRef mapTypeMod;1758    if (parser.parseKeyword(&mapTypeMod))1759      return failure();1760 1761    if (mapTypeMod == "always")1762      mapTypeBits |= ClauseMapFlags::always;1763 1764    if (mapTypeMod == "implicit")1765      mapTypeBits |= ClauseMapFlags::implicit;1766 1767    if (mapTypeMod == "ompx_hold")1768      mapTypeBits |= ClauseMapFlags::ompx_hold;1769 1770    if (mapTypeMod == "close")1771      mapTypeBits |= ClauseMapFlags::close;1772 1773    if (mapTypeMod == "present")1774      mapTypeBits |= ClauseMapFlags::present;1775 1776    if (mapTypeMod == "to")1777      mapTypeBits |= ClauseMapFlags::to;1778 1779    if (mapTypeMod == "from")1780      mapTypeBits |= ClauseMapFlags::from;1781 1782    if (mapTypeMod == "tofrom")1783      mapTypeBits |= ClauseMapFlags::to | ClauseMapFlags::from;1784 1785    if (mapTypeMod == "delete")1786      mapTypeBits |= ClauseMapFlags::del;1787 1788    if (mapTypeMod == "storage")1789      mapTypeBits |= ClauseMapFlags::storage;1790 1791    if (mapTypeMod == "return_param")1792      mapTypeBits |= ClauseMapFlags::return_param;1793 1794    if (mapTypeMod == "private")1795      mapTypeBits |= ClauseMapFlags::priv;1796 1797    if (mapTypeMod == "literal")1798      mapTypeBits |= ClauseMapFlags::literal;1799 1800    if (mapTypeMod == "attach")1801      mapTypeBits |= ClauseMapFlags::attach;1802 1803    if (mapTypeMod == "attach_always")1804      mapTypeBits |= ClauseMapFlags::attach_always;1805 1806    if (mapTypeMod == "attach_none")1807      mapTypeBits |= ClauseMapFlags::attach_none;1808 1809    if (mapTypeMod == "attach_auto")1810      mapTypeBits |= ClauseMapFlags::attach_auto;1811 1812    if (mapTypeMod == "ref_ptr")1813      mapTypeBits |= ClauseMapFlags::ref_ptr;1814 1815    if (mapTypeMod == "ref_ptee")1816      mapTypeBits |= ClauseMapFlags::ref_ptee;1817 1818    if (mapTypeMod == "ref_ptr_ptee")1819      mapTypeBits |= ClauseMapFlags::ref_ptr_ptee;1820 1821    return success();1822  };1823 1824  if (parser.parseCommaSeparatedList(parseTypeAndMod))1825    return failure();1826 1827  mapType =1828      parser.getBuilder().getAttr<mlir::omp::ClauseMapFlagsAttr>(mapTypeBits);1829 1830  return success();1831}1832 1833/// Prints a map_entries map type from its numeric value out into its string1834/// format.1835static void printMapClause(OpAsmPrinter &p, Operation *op,1836                           ClauseMapFlagsAttr mapType) {1837  llvm::SmallVector<std::string, 4> mapTypeStrs;1838  ClauseMapFlags mapFlags = mapType.getValue();1839 1840  // handling of always, close, present placed at the beginning of the string1841  // to aid readability1842  if (mapTypeToBool(mapFlags, ClauseMapFlags::always))1843    mapTypeStrs.push_back("always");1844  if (mapTypeToBool(mapFlags, ClauseMapFlags::implicit))1845    mapTypeStrs.push_back("implicit");1846  if (mapTypeToBool(mapFlags, ClauseMapFlags::ompx_hold))1847    mapTypeStrs.push_back("ompx_hold");1848  if (mapTypeToBool(mapFlags, ClauseMapFlags::close))1849    mapTypeStrs.push_back("close");1850  if (mapTypeToBool(mapFlags, ClauseMapFlags::present))1851    mapTypeStrs.push_back("present");1852 1853  // special handling of to/from/tofrom/delete and release/alloc, release +1854  // alloc are the abscense of one of the other flags, whereas tofrom requires1855  // both the to and from flag to be set.1856  bool to = mapTypeToBool(mapFlags, ClauseMapFlags::to);1857  bool from = mapTypeToBool(mapFlags, ClauseMapFlags::from);1858 1859  if (to && from)1860    mapTypeStrs.push_back("tofrom");1861  else if (from)1862    mapTypeStrs.push_back("from");1863  else if (to)1864    mapTypeStrs.push_back("to");1865 1866  if (mapTypeToBool(mapFlags, ClauseMapFlags::del))1867    mapTypeStrs.push_back("delete");1868  if (mapTypeToBool(mapFlags, ClauseMapFlags::return_param))1869    mapTypeStrs.push_back("return_param");1870  if (mapTypeToBool(mapFlags, ClauseMapFlags::storage))1871    mapTypeStrs.push_back("storage");1872  if (mapTypeToBool(mapFlags, ClauseMapFlags::priv))1873    mapTypeStrs.push_back("private");1874  if (mapTypeToBool(mapFlags, ClauseMapFlags::literal))1875    mapTypeStrs.push_back("literal");1876  if (mapTypeToBool(mapFlags, ClauseMapFlags::attach))1877    mapTypeStrs.push_back("attach");1878  if (mapTypeToBool(mapFlags, ClauseMapFlags::attach_always))1879    mapTypeStrs.push_back("attach_always");1880  if (mapTypeToBool(mapFlags, ClauseMapFlags::attach_none))1881    mapTypeStrs.push_back("attach_none");1882  if (mapTypeToBool(mapFlags, ClauseMapFlags::attach_auto))1883    mapTypeStrs.push_back("attach_auto");1884  if (mapTypeToBool(mapFlags, ClauseMapFlags::ref_ptr))1885    mapTypeStrs.push_back("ref_ptr");1886  if (mapTypeToBool(mapFlags, ClauseMapFlags::ref_ptee))1887    mapTypeStrs.push_back("ref_ptee");1888  if (mapTypeToBool(mapFlags, ClauseMapFlags::ref_ptr_ptee))1889    mapTypeStrs.push_back("ref_ptr_ptee");1890  if (mapFlags == ClauseMapFlags::none)1891    mapTypeStrs.push_back("none");1892 1893  for (unsigned int i = 0; i < mapTypeStrs.size(); ++i) {1894    p << mapTypeStrs[i];1895    if (i + 1 < mapTypeStrs.size()) {1896      p << ", ";1897    }1898  }1899}1900 1901static ParseResult parseMembersIndex(OpAsmParser &parser,1902                                     ArrayAttr &membersIdx) {1903  SmallVector<Attribute> values, memberIdxs;1904 1905  auto parseIndices = [&]() -> ParseResult {1906    int64_t value;1907    if (parser.parseInteger(value))1908      return failure();1909    values.push_back(IntegerAttr::get(parser.getBuilder().getIntegerType(64),1910                                      APInt(64, value, /*isSigned=*/false)));1911    return success();1912  };1913 1914  do {1915    if (failed(parser.parseLSquare()))1916      return failure();1917 1918    if (parser.parseCommaSeparatedList(parseIndices))1919      return failure();1920 1921    if (failed(parser.parseRSquare()))1922      return failure();1923 1924    memberIdxs.push_back(ArrayAttr::get(parser.getContext(), values));1925    values.clear();1926  } while (succeeded(parser.parseOptionalComma()));1927 1928  if (!memberIdxs.empty())1929    membersIdx = ArrayAttr::get(parser.getContext(), memberIdxs);1930 1931  return success();1932}1933 1934static void printMembersIndex(OpAsmPrinter &p, MapInfoOp op,1935                              ArrayAttr membersIdx) {1936  if (!membersIdx)1937    return;1938 1939  llvm::interleaveComma(membersIdx, p, [&p](Attribute v) {1940    p << "[";1941    auto memberIdx = cast<ArrayAttr>(v);1942    llvm::interleaveComma(memberIdx.getValue(), p, [&p](Attribute v2) {1943      p << cast<IntegerAttr>(v2).getInt();1944    });1945    p << "]";1946  });1947}1948 1949static void printCaptureType(OpAsmPrinter &p, Operation *op,1950                             VariableCaptureKindAttr mapCaptureType) {1951  std::string typeCapStr;1952  llvm::raw_string_ostream typeCap(typeCapStr);1953  if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::ByRef)1954    typeCap << "ByRef";1955  if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::ByCopy)1956    typeCap << "ByCopy";1957  if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::VLAType)1958    typeCap << "VLAType";1959  if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::This)1960    typeCap << "This";1961  p << typeCapStr;1962}1963 1964static ParseResult parseCaptureType(OpAsmParser &parser,1965                                    VariableCaptureKindAttr &mapCaptureType) {1966  StringRef mapCaptureKey;1967  if (parser.parseKeyword(&mapCaptureKey))1968    return failure();1969 1970  if (mapCaptureKey == "This")1971    mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(1972        parser.getContext(), mlir::omp::VariableCaptureKind::This);1973  if (mapCaptureKey == "ByRef")1974    mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(1975        parser.getContext(), mlir::omp::VariableCaptureKind::ByRef);1976  if (mapCaptureKey == "ByCopy")1977    mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(1978        parser.getContext(), mlir::omp::VariableCaptureKind::ByCopy);1979  if (mapCaptureKey == "VLAType")1980    mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(1981        parser.getContext(), mlir::omp::VariableCaptureKind::VLAType);1982 1983  return success();1984}1985 1986static LogicalResult verifyMapClause(Operation *op, OperandRange mapVars) {1987  llvm::DenseSet<mlir::TypedValue<mlir::omp::PointerLikeType>> updateToVars;1988  llvm::DenseSet<mlir::TypedValue<mlir::omp::PointerLikeType>> updateFromVars;1989 1990  for (auto mapOp : mapVars) {1991    if (!mapOp.getDefiningOp())1992      return emitError(op->getLoc(), "missing map operation");1993 1994    if (auto mapInfoOp = mapOp.getDefiningOp<mlir::omp::MapInfoOp>()) {1995      mlir::omp::ClauseMapFlags mapTypeBits = mapInfoOp.getMapType();1996 1997      bool to = mapTypeToBool(mapTypeBits, ClauseMapFlags::to);1998      bool from = mapTypeToBool(mapTypeBits, ClauseMapFlags::from);1999      bool del = mapTypeToBool(mapTypeBits, ClauseMapFlags::del);2000 2001      bool always = mapTypeToBool(mapTypeBits, ClauseMapFlags::always);2002      bool close = mapTypeToBool(mapTypeBits, ClauseMapFlags::close);2003      bool implicit = mapTypeToBool(mapTypeBits, ClauseMapFlags::implicit);2004 2005      if ((isa<TargetDataOp>(op) || isa<TargetOp>(op)) && del)2006        return emitError(op->getLoc(),2007                         "to, from, tofrom and alloc map types are permitted");2008 2009      if (isa<TargetEnterDataOp>(op) && (from || del))2010        return emitError(op->getLoc(), "to and alloc map types are permitted");2011 2012      if (isa<TargetExitDataOp>(op) && to)2013        return emitError(op->getLoc(),2014                         "from, release and delete map types are permitted");2015 2016      if (isa<TargetUpdateOp>(op)) {2017        if (del) {2018          return emitError(op->getLoc(),2019                           "at least one of to or from map types must be "2020                           "specified, other map types are not permitted");2021        }2022 2023        if (!to && !from) {2024          return emitError(op->getLoc(),2025                           "at least one of to or from map types must be "2026                           "specified, other map types are not permitted");2027        }2028 2029        auto updateVar = mapInfoOp.getVarPtr();2030 2031        if ((to && from) || (to && updateFromVars.contains(updateVar)) ||2032            (from && updateToVars.contains(updateVar))) {2033          return emitError(2034              op->getLoc(),2035              "either to or from map types can be specified, not both");2036        }2037 2038        if (always || close || implicit) {2039          return emitError(2040              op->getLoc(),2041              "present, mapper and iterator map type modifiers are permitted");2042        }2043 2044        to ? updateToVars.insert(updateVar) : updateFromVars.insert(updateVar);2045      }2046    } else if (!isa<DeclareMapperInfoOp>(op)) {2047      return emitError(op->getLoc(),2048                       "map argument is not a map entry operation");2049    }2050  }2051 2052  return success();2053}2054 2055static LogicalResult verifyPrivateVarsMapping(TargetOp targetOp) {2056  std::optional<DenseI64ArrayAttr> privateMapIndices =2057      targetOp.getPrivateMapsAttr();2058 2059  // None of the private operands are mapped.2060  if (!privateMapIndices.has_value() || !privateMapIndices.value())2061    return success();2062 2063  OperandRange privateVars = targetOp.getPrivateVars();2064 2065  if (privateMapIndices.value().size() !=2066      static_cast<int64_t>(privateVars.size()))2067    return emitError(targetOp.getLoc(), "sizes of `private` operand range and "2068                                        "`private_maps` attribute mismatch");2069 2070  return success();2071}2072 2073//===----------------------------------------------------------------------===//2074// MapInfoOp2075//===----------------------------------------------------------------------===//2076 2077static LogicalResult verifyMapInfoDefinedArgs(Operation *op,2078                                              StringRef clauseName,2079                                              OperandRange vars) {2080  for (Value var : vars)2081    if (!llvm::isa_and_present<MapInfoOp>(var.getDefiningOp()))2082      return op->emitOpError()2083             << "'" << clauseName2084             << "' arguments must be defined by 'omp.map.info' ops";2085  return success();2086}2087 2088LogicalResult MapInfoOp::verify() {2089  if (getMapperId() &&2090      !SymbolTable::lookupNearestSymbolFrom<omp::DeclareMapperOp>(2091          *this, getMapperIdAttr())) {2092    return emitError("invalid mapper id");2093  }2094 2095  if (failed(verifyMapInfoDefinedArgs(*this, "members", getMembers())))2096    return failure();2097 2098  return success();2099}2100 2101//===----------------------------------------------------------------------===//2102// TargetDataOp2103//===----------------------------------------------------------------------===//2104 2105void TargetDataOp::build(OpBuilder &builder, OperationState &state,2106                         const TargetDataOperands &clauses) {2107  TargetDataOp::build(builder, state, clauses.device, clauses.ifExpr,2108                      clauses.mapVars, clauses.useDeviceAddrVars,2109                      clauses.useDevicePtrVars);2110}2111 2112LogicalResult TargetDataOp::verify() {2113  if (getMapVars().empty() && getUseDevicePtrVars().empty() &&2114      getUseDeviceAddrVars().empty()) {2115    return ::emitError(this->getLoc(),2116                       "At least one of map, use_device_ptr_vars, or "2117                       "use_device_addr_vars operand must be present");2118  }2119 2120  if (failed(verifyMapInfoDefinedArgs(*this, "use_device_ptr",2121                                      getUseDevicePtrVars())))2122    return failure();2123 2124  if (failed(verifyMapInfoDefinedArgs(*this, "use_device_addr",2125                                      getUseDeviceAddrVars())))2126    return failure();2127 2128  return verifyMapClause(*this, getMapVars());2129}2130 2131//===----------------------------------------------------------------------===//2132// TargetEnterDataOp2133//===----------------------------------------------------------------------===//2134 2135void TargetEnterDataOp::build(2136    OpBuilder &builder, OperationState &state,2137    const TargetEnterExitUpdateDataOperands &clauses) {2138  MLIRContext *ctx = builder.getContext();2139  TargetEnterDataOp::build(builder, state,2140                           makeArrayAttr(ctx, clauses.dependKinds),2141                           clauses.dependVars, clauses.device, clauses.ifExpr,2142                           clauses.mapVars, clauses.nowait);2143}2144 2145LogicalResult TargetEnterDataOp::verify() {2146  LogicalResult verifyDependVars =2147      verifyDependVarList(*this, getDependKinds(), getDependVars());2148  return failed(verifyDependVars) ? verifyDependVars2149                                  : verifyMapClause(*this, getMapVars());2150}2151 2152//===----------------------------------------------------------------------===//2153// TargetExitDataOp2154//===----------------------------------------------------------------------===//2155 2156void TargetExitDataOp::build(OpBuilder &builder, OperationState &state,2157                             const TargetEnterExitUpdateDataOperands &clauses) {2158  MLIRContext *ctx = builder.getContext();2159  TargetExitDataOp::build(builder, state,2160                          makeArrayAttr(ctx, clauses.dependKinds),2161                          clauses.dependVars, clauses.device, clauses.ifExpr,2162                          clauses.mapVars, clauses.nowait);2163}2164 2165LogicalResult TargetExitDataOp::verify() {2166  LogicalResult verifyDependVars =2167      verifyDependVarList(*this, getDependKinds(), getDependVars());2168  return failed(verifyDependVars) ? verifyDependVars2169                                  : verifyMapClause(*this, getMapVars());2170}2171 2172//===----------------------------------------------------------------------===//2173// TargetUpdateOp2174//===----------------------------------------------------------------------===//2175 2176void TargetUpdateOp::build(OpBuilder &builder, OperationState &state,2177                           const TargetEnterExitUpdateDataOperands &clauses) {2178  MLIRContext *ctx = builder.getContext();2179  TargetUpdateOp::build(builder, state, makeArrayAttr(ctx, clauses.dependKinds),2180                        clauses.dependVars, clauses.device, clauses.ifExpr,2181                        clauses.mapVars, clauses.nowait);2182}2183 2184LogicalResult TargetUpdateOp::verify() {2185  LogicalResult verifyDependVars =2186      verifyDependVarList(*this, getDependKinds(), getDependVars());2187  return failed(verifyDependVars) ? verifyDependVars2188                                  : verifyMapClause(*this, getMapVars());2189}2190 2191//===----------------------------------------------------------------------===//2192// TargetOp2193//===----------------------------------------------------------------------===//2194 2195void TargetOp::build(OpBuilder &builder, OperationState &state,2196                     const TargetOperands &clauses) {2197  MLIRContext *ctx = builder.getContext();2198  // TODO Store clauses in op: allocateVars, allocatorVars, inReductionVars,2199  // inReductionByref, inReductionSyms.2200  TargetOp::build(builder, state, /*allocate_vars=*/{}, /*allocator_vars=*/{},2201                  clauses.bare, makeArrayAttr(ctx, clauses.dependKinds),2202                  clauses.dependVars, clauses.device, clauses.hasDeviceAddrVars,2203                  clauses.hostEvalVars, clauses.ifExpr,2204                  /*in_reduction_vars=*/{}, /*in_reduction_byref=*/nullptr,2205                  /*in_reduction_syms=*/nullptr, clauses.isDevicePtrVars,2206                  clauses.mapVars, clauses.nowait, clauses.privateVars,2207                  makeArrayAttr(ctx, clauses.privateSyms),2208                  clauses.privateNeedsBarrier, clauses.threadLimit,2209                  /*private_maps=*/nullptr);2210}2211 2212LogicalResult TargetOp::verify() {2213  if (failed(verifyDependVarList(*this, getDependKinds(), getDependVars())))2214    return failure();2215 2216  if (failed(verifyMapInfoDefinedArgs(*this, "has_device_addr",2217                                      getHasDeviceAddrVars())))2218    return failure();2219 2220  if (failed(verifyMapClause(*this, getMapVars())))2221    return failure();2222 2223  return verifyPrivateVarsMapping(*this);2224}2225 2226LogicalResult TargetOp::verifyRegions() {2227  auto teamsOps = getOps<TeamsOp>();2228  if (std::distance(teamsOps.begin(), teamsOps.end()) > 1)2229    return emitError("target containing multiple 'omp.teams' nested ops");2230 2231  // Check that host_eval values are only used in legal ways.2232  Operation *capturedOp = getInnermostCapturedOmpOp();2233  TargetRegionFlags execFlags = getKernelExecFlags(capturedOp);2234  for (Value hostEvalArg :2235       cast<BlockArgOpenMPOpInterface>(getOperation()).getHostEvalBlockArgs()) {2236    for (Operation *user : hostEvalArg.getUsers()) {2237      if (auto teamsOp = dyn_cast<TeamsOp>(user)) {2238        if (llvm::is_contained({teamsOp.getNumTeamsLower(),2239                                teamsOp.getNumTeamsUpper(),2240                                teamsOp.getThreadLimit()},2241                               hostEvalArg))2242          continue;2243 2244        return emitOpError() << "host_eval argument only legal as 'num_teams' "2245                                "and 'thread_limit' in 'omp.teams'";2246      }2247      if (auto parallelOp = dyn_cast<ParallelOp>(user)) {2248        if (bitEnumContainsAny(execFlags, TargetRegionFlags::spmd) &&2249            parallelOp->isAncestor(capturedOp) &&2250            hostEvalArg == parallelOp.getNumThreads())2251          continue;2252 2253        return emitOpError()2254               << "host_eval argument only legal as 'num_threads' in "2255                  "'omp.parallel' when representing target SPMD";2256      }2257      if (auto loopNestOp = dyn_cast<LoopNestOp>(user)) {2258        if (bitEnumContainsAny(execFlags, TargetRegionFlags::trip_count) &&2259            loopNestOp.getOperation() == capturedOp &&2260            (llvm::is_contained(loopNestOp.getLoopLowerBounds(), hostEvalArg) ||2261             llvm::is_contained(loopNestOp.getLoopUpperBounds(), hostEvalArg) ||2262             llvm::is_contained(loopNestOp.getLoopSteps(), hostEvalArg)))2263          continue;2264 2265        return emitOpError() << "host_eval argument only legal as loop bounds "2266                                "and steps in 'omp.loop_nest' when trip count "2267                                "must be evaluated in the host";2268      }2269 2270      return emitOpError() << "host_eval argument illegal use in '"2271                           << user->getName() << "' operation";2272    }2273  }2274  return success();2275}2276 2277static Operation *2278findCapturedOmpOp(Operation *rootOp, bool checkSingleMandatoryExec,2279                  llvm::function_ref<bool(Operation *)> siblingAllowedFn) {2280  assert(rootOp && "expected valid operation");2281 2282  Dialect *ompDialect = rootOp->getDialect();2283  Operation *capturedOp = nullptr;2284  DominanceInfo domInfo;2285 2286  // Process in pre-order to check operations from outermost to innermost,2287  // ensuring we only enter the region of an operation if it meets the criteria2288  // for being captured. We stop the exploration of nested operations as soon as2289  // we process a region holding no operations to be captured.2290  rootOp->walk<WalkOrder::PreOrder>([&](Operation *op) {2291    if (op == rootOp)2292      return WalkResult::advance();2293 2294    // Ignore operations of other dialects or omp operations with no regions,2295    // because these will only be checked if they are siblings of an omp2296    // operation that can potentially be captured.2297    bool isOmpDialect = op->getDialect() == ompDialect;2298    bool hasRegions = op->getNumRegions() > 0;2299    if (!isOmpDialect || !hasRegions)2300      return WalkResult::skip();2301 2302    // This operation cannot be captured if it can be executed more than once2303    // (i.e. its block's successors can reach it) or if it's not guaranteed to2304    // be executed before all exits of the region (i.e. it doesn't dominate all2305    // blocks with no successors reachable from the entry block).2306    if (checkSingleMandatoryExec) {2307      Region *parentRegion = op->getParentRegion();2308      Block *parentBlock = op->getBlock();2309 2310      for (Block *successor : parentBlock->getSuccessors())2311        if (successor->isReachable(parentBlock))2312          return WalkResult::interrupt();2313 2314      for (Block &block : *parentRegion)2315        if (domInfo.isReachableFromEntry(&block) && block.hasNoSuccessors() &&2316            !domInfo.dominates(parentBlock, &block))2317          return WalkResult::interrupt();2318    }2319 2320    // Don't capture this op if it has a not-allowed sibling, and stop recursing2321    // into nested operations.2322    for (Operation &sibling : op->getParentRegion()->getOps())2323      if (&sibling != op && !siblingAllowedFn(&sibling))2324        return WalkResult::interrupt();2325 2326    // Don't continue capturing nested operations if we reach an omp.loop_nest.2327    // Otherwise, process the contents of this operation.2328    capturedOp = op;2329    return llvm::isa<LoopNestOp>(op) ? WalkResult::interrupt()2330                                     : WalkResult::advance();2331  });2332 2333  return capturedOp;2334}2335 2336Operation *TargetOp::getInnermostCapturedOmpOp() {2337  auto *ompDialect = getContext()->getLoadedDialect<omp::OpenMPDialect>();2338 2339  // Only allow OpenMP terminators and non-OpenMP ops that have known memory2340  // effects, but don't include a memory write effect.2341  return findCapturedOmpOp(2342      *this, /*checkSingleMandatoryExec=*/true, [&](Operation *sibling) {2343        if (!sibling)2344          return false;2345 2346        if (ompDialect == sibling->getDialect())2347          return sibling->hasTrait<OpTrait::IsTerminator>();2348 2349        if (auto memOp = dyn_cast<MemoryEffectOpInterface>(sibling)) {2350          SmallVector<SideEffects::EffectInstance<MemoryEffects::Effect>, 4>2351              effects;2352          memOp.getEffects(effects);2353          return !llvm::any_of(2354              effects, [&](MemoryEffects::EffectInstance &effect) {2355                return isa<MemoryEffects::Write>(effect.getEffect()) &&2356                       isa<SideEffects::AutomaticAllocationScopeResource>(2357                           effect.getResource());2358              });2359        }2360        return true;2361      });2362}2363 2364/// Check if we can promote SPMD kernel to No-Loop kernel.2365static bool canPromoteToNoLoop(Operation *capturedOp, TeamsOp teamsOp,2366                               WsloopOp *wsLoopOp) {2367  // num_teams clause can break no-loop teams/threads assumption.2368  if (teamsOp.getNumTeamsUpper())2369    return false;2370 2371  // Reduction kernels are slower in no-loop mode.2372  if (teamsOp.getNumReductionVars())2373    return false;2374  if (wsLoopOp->getNumReductionVars())2375    return false;2376 2377  // Check if the user allows the promotion of kernels to no-loop mode.2378  OffloadModuleInterface offloadMod =2379      capturedOp->getParentOfType<omp::OffloadModuleInterface>();2380  if (!offloadMod)2381    return false;2382  auto ompFlags = offloadMod.getFlags();2383  if (!ompFlags)2384    return false;2385  return ompFlags.getAssumeTeamsOversubscription() &&2386         ompFlags.getAssumeThreadsOversubscription();2387}2388 2389TargetRegionFlags TargetOp::getKernelExecFlags(Operation *capturedOp) {2390  // A non-null captured op is only valid if it resides inside of a TargetOp2391  // and is the result of calling getInnermostCapturedOmpOp() on it.2392  TargetOp targetOp =2393      capturedOp ? capturedOp->getParentOfType<TargetOp>() : nullptr;2394  assert((!capturedOp ||2395          (targetOp && targetOp.getInnermostCapturedOmpOp() == capturedOp)) &&2396         "unexpected captured op");2397 2398  // If it's not capturing a loop, it's a default target region.2399  if (!isa_and_present<LoopNestOp>(capturedOp))2400    return TargetRegionFlags::generic;2401 2402  // Get the innermost non-simd loop wrapper.2403  SmallVector<LoopWrapperInterface> loopWrappers;2404  cast<LoopNestOp>(capturedOp).gatherWrappers(loopWrappers);2405  assert(!loopWrappers.empty());2406 2407  LoopWrapperInterface *innermostWrapper = loopWrappers.begin();2408  if (isa<SimdOp>(innermostWrapper))2409    innermostWrapper = std::next(innermostWrapper);2410 2411  auto numWrappers = std::distance(innermostWrapper, loopWrappers.end());2412  if (numWrappers != 1 && numWrappers != 2)2413    return TargetRegionFlags::generic;2414 2415  // Detect target-teams-distribute-parallel-wsloop[-simd].2416  if (numWrappers == 2) {2417    WsloopOp *wsloopOp = dyn_cast<WsloopOp>(innermostWrapper);2418    if (!wsloopOp)2419      return TargetRegionFlags::generic;2420 2421    innermostWrapper = std::next(innermostWrapper);2422    if (!isa<DistributeOp>(innermostWrapper))2423      return TargetRegionFlags::generic;2424 2425    Operation *parallelOp = (*innermostWrapper)->getParentOp();2426    if (!isa_and_present<ParallelOp>(parallelOp))2427      return TargetRegionFlags::generic;2428 2429    TeamsOp teamsOp = dyn_cast<TeamsOp>(parallelOp->getParentOp());2430    if (!teamsOp)2431      return TargetRegionFlags::generic;2432 2433    if (teamsOp->getParentOp() == targetOp.getOperation()) {2434      TargetRegionFlags result =2435          TargetRegionFlags::spmd | TargetRegionFlags::trip_count;2436      if (canPromoteToNoLoop(capturedOp, teamsOp, wsloopOp))2437        result = result | TargetRegionFlags::no_loop;2438      return result;2439    }2440  }2441  // Detect target-teams-distribute[-simd] and target-teams-loop.2442  else if (isa<DistributeOp, LoopOp>(innermostWrapper)) {2443    Operation *teamsOp = (*innermostWrapper)->getParentOp();2444    if (!isa_and_present<TeamsOp>(teamsOp))2445      return TargetRegionFlags::generic;2446 2447    if (teamsOp->getParentOp() != targetOp.getOperation())2448      return TargetRegionFlags::generic;2449 2450    if (isa<LoopOp>(innermostWrapper))2451      return TargetRegionFlags::spmd | TargetRegionFlags::trip_count;2452 2453    // Find single immediately nested captured omp.parallel and add spmd flag2454    // (generic-spmd case).2455    //2456    // TODO: This shouldn't have to be done here, as it is too easy to break.2457    // The openmp-opt pass should be updated to be able to promote kernels like2458    // this from "Generic" to "Generic-SPMD". However, the use of the2459    // `kmpc_distribute_static_loop` family of functions produced by the2460    // OMPIRBuilder for these kernels prevents that from working.2461    Dialect *ompDialect = targetOp->getDialect();2462    Operation *nestedCapture = findCapturedOmpOp(2463        capturedOp, /*checkSingleMandatoryExec=*/false,2464        [&](Operation *sibling) {2465          return sibling && (ompDialect != sibling->getDialect() ||2466                             sibling->hasTrait<OpTrait::IsTerminator>());2467        });2468 2469    TargetRegionFlags result =2470        TargetRegionFlags::generic | TargetRegionFlags::trip_count;2471 2472    if (!nestedCapture)2473      return result;2474 2475    while (nestedCapture->getParentOp() != capturedOp)2476      nestedCapture = nestedCapture->getParentOp();2477 2478    return isa<ParallelOp>(nestedCapture) ? result | TargetRegionFlags::spmd2479                                          : result;2480  }2481  // Detect target-parallel-wsloop[-simd].2482  else if (isa<WsloopOp>(innermostWrapper)) {2483    Operation *parallelOp = (*innermostWrapper)->getParentOp();2484    if (!isa_and_present<ParallelOp>(parallelOp))2485      return TargetRegionFlags::generic;2486 2487    if (parallelOp->getParentOp() == targetOp.getOperation())2488      return TargetRegionFlags::spmd;2489  }2490 2491  return TargetRegionFlags::generic;2492}2493 2494//===----------------------------------------------------------------------===//2495// ParallelOp2496//===----------------------------------------------------------------------===//2497 2498void ParallelOp::build(OpBuilder &builder, OperationState &state,2499                       ArrayRef<NamedAttribute> attributes) {2500  ParallelOp::build(builder, state, /*allocate_vars=*/ValueRange(),2501                    /*allocator_vars=*/ValueRange(), /*if_expr=*/nullptr,2502                    /*num_threads=*/nullptr, /*private_vars=*/ValueRange(),2503                    /*private_syms=*/nullptr, /*private_needs_barrier=*/nullptr,2504                    /*proc_bind_kind=*/nullptr,2505                    /*reduction_mod =*/nullptr, /*reduction_vars=*/ValueRange(),2506                    /*reduction_byref=*/nullptr, /*reduction_syms=*/nullptr);2507  state.addAttributes(attributes);2508}2509 2510void ParallelOp::build(OpBuilder &builder, OperationState &state,2511                       const ParallelOperands &clauses) {2512  MLIRContext *ctx = builder.getContext();2513  ParallelOp::build(builder, state, clauses.allocateVars, clauses.allocatorVars,2514                    clauses.ifExpr, clauses.numThreads, clauses.privateVars,2515                    makeArrayAttr(ctx, clauses.privateSyms),2516                    clauses.privateNeedsBarrier, clauses.procBindKind,2517                    clauses.reductionMod, clauses.reductionVars,2518                    makeDenseBoolArrayAttr(ctx, clauses.reductionByref),2519                    makeArrayAttr(ctx, clauses.reductionSyms));2520}2521 2522template <typename OpType>2523static LogicalResult verifyPrivateVarList(OpType &op) {2524  auto privateVars = op.getPrivateVars();2525  auto privateSyms = op.getPrivateSymsAttr();2526 2527  if (privateVars.empty() && (privateSyms == nullptr || privateSyms.empty()))2528    return success();2529 2530  auto numPrivateVars = privateVars.size();2531  auto numPrivateSyms = (privateSyms == nullptr) ? 0 : privateSyms.size();2532 2533  if (numPrivateVars != numPrivateSyms)2534    return op.emitError() << "inconsistent number of private variables and "2535                             "privatizer op symbols, private vars: "2536                          << numPrivateVars2537                          << " vs. privatizer op symbols: " << numPrivateSyms;2538 2539  for (auto privateVarInfo : llvm::zip_equal(privateVars, privateSyms)) {2540    Type varType = std::get<0>(privateVarInfo).getType();2541    SymbolRefAttr privateSym = cast<SymbolRefAttr>(std::get<1>(privateVarInfo));2542    PrivateClauseOp privatizerOp =2543        SymbolTable::lookupNearestSymbolFrom<PrivateClauseOp>(op, privateSym);2544 2545    if (privatizerOp == nullptr)2546      return op.emitError() << "failed to lookup privatizer op with symbol: '"2547                            << privateSym << "'";2548 2549    Type privatizerType = privatizerOp.getArgType();2550 2551    if (privatizerType && (varType != privatizerType))2552      return op.emitError()2553             << "type mismatch between a "2554             << (privatizerOp.getDataSharingType() ==2555                         DataSharingClauseType::Private2556                     ? "private"2557                     : "firstprivate")2558             << " variable and its privatizer op, var type: " << varType2559             << " vs. privatizer op type: " << privatizerType;2560  }2561 2562  return success();2563}2564 2565LogicalResult ParallelOp::verify() {2566  if (getAllocateVars().size() != getAllocatorVars().size())2567    return emitError(2568        "expected equal sizes for allocate and allocator variables");2569 2570  if (failed(verifyPrivateVarList(*this)))2571    return failure();2572 2573  return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),2574                                getReductionByref());2575}2576 2577LogicalResult ParallelOp::verifyRegions() {2578  auto distChildOps = getOps<DistributeOp>();2579  int numDistChildOps = std::distance(distChildOps.begin(), distChildOps.end());2580  if (numDistChildOps > 1)2581    return emitError()2582           << "multiple 'omp.distribute' nested inside of 'omp.parallel'";2583 2584  if (numDistChildOps == 1) {2585    if (!isComposite())2586      return emitError()2587             << "'omp.composite' attribute missing from composite operation";2588 2589    auto *ompDialect = getContext()->getLoadedDialect<OpenMPDialect>();2590    Operation &distributeOp = **distChildOps.begin();2591    for (Operation &childOp : getOps()) {2592      if (&childOp == &distributeOp || ompDialect != childOp.getDialect())2593        continue;2594 2595      if (!childOp.hasTrait<OpTrait::IsTerminator>())2596        return emitError() << "unexpected OpenMP operation inside of composite "2597                              "'omp.parallel': "2598                           << childOp.getName();2599    }2600  } else if (isComposite()) {2601    return emitError()2602           << "'omp.composite' attribute present in non-composite operation";2603  }2604  return success();2605}2606 2607//===----------------------------------------------------------------------===//2608// TeamsOp2609//===----------------------------------------------------------------------===//2610 2611static bool opInGlobalImplicitParallelRegion(Operation *op) {2612  while ((op = op->getParentOp()))2613    if (isa<OpenMPDialect>(op->getDialect()))2614      return false;2615  return true;2616}2617 2618void TeamsOp::build(OpBuilder &builder, OperationState &state,2619                    const TeamsOperands &clauses) {2620  MLIRContext *ctx = builder.getContext();2621  // TODO Store clauses in op: privateVars, privateSyms, privateNeedsBarrier2622  TeamsOp::build(builder, state, clauses.allocateVars, clauses.allocatorVars,2623                 clauses.ifExpr, clauses.numTeamsLower, clauses.numTeamsUpper,2624                 /*private_vars=*/{}, /*private_syms=*/nullptr,2625                 /*private_needs_barrier=*/nullptr, clauses.reductionMod,2626                 clauses.reductionVars,2627                 makeDenseBoolArrayAttr(ctx, clauses.reductionByref),2628                 makeArrayAttr(ctx, clauses.reductionSyms),2629                 clauses.threadLimit);2630}2631 2632LogicalResult TeamsOp::verify() {2633  // Check parent region2634  // TODO If nested inside of a target region, also check that it does not2635  // contain any statements, declarations or directives other than this2636  // omp.teams construct. The issue is how to support the initialization of2637  // this operation's own arguments (allow SSA values across omp.target?).2638  Operation *op = getOperation();2639  if (!isa<TargetOp>(op->getParentOp()) &&2640      !opInGlobalImplicitParallelRegion(op))2641    return emitError("expected to be nested inside of omp.target or not nested "2642                     "in any OpenMP dialect operations");2643 2644  // Check for num_teams clause restrictions2645  if (auto numTeamsLowerBound = getNumTeamsLower()) {2646    auto numTeamsUpperBound = getNumTeamsUpper();2647    if (!numTeamsUpperBound)2648      return emitError("expected num_teams upper bound to be defined if the "2649                       "lower bound is defined");2650    if (numTeamsLowerBound.getType() != numTeamsUpperBound.getType())2651      return emitError(2652          "expected num_teams upper bound and lower bound to be the same type");2653  }2654 2655  // Check for allocate clause restrictions2656  if (getAllocateVars().size() != getAllocatorVars().size())2657    return emitError(2658        "expected equal sizes for allocate and allocator variables");2659 2660  return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),2661                                getReductionByref());2662}2663 2664//===----------------------------------------------------------------------===//2665// SectionOp2666//===----------------------------------------------------------------------===//2667 2668OperandRange SectionOp::getPrivateVars() {2669  return getParentOp().getPrivateVars();2670}2671 2672OperandRange SectionOp::getReductionVars() {2673  return getParentOp().getReductionVars();2674}2675 2676//===----------------------------------------------------------------------===//2677// SectionsOp2678//===----------------------------------------------------------------------===//2679 2680void SectionsOp::build(OpBuilder &builder, OperationState &state,2681                       const SectionsOperands &clauses) {2682  MLIRContext *ctx = builder.getContext();2683  // TODO Store clauses in op: privateVars, privateSyms, privateNeedsBarrier2684  SectionsOp::build(builder, state, clauses.allocateVars, clauses.allocatorVars,2685                    clauses.nowait, /*private_vars=*/{},2686                    /*private_syms=*/nullptr, /*private_needs_barrier=*/nullptr,2687                    clauses.reductionMod, clauses.reductionVars,2688                    makeDenseBoolArrayAttr(ctx, clauses.reductionByref),2689                    makeArrayAttr(ctx, clauses.reductionSyms));2690}2691 2692LogicalResult SectionsOp::verify() {2693  if (getAllocateVars().size() != getAllocatorVars().size())2694    return emitError(2695        "expected equal sizes for allocate and allocator variables");2696 2697  return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),2698                                getReductionByref());2699}2700 2701LogicalResult SectionsOp::verifyRegions() {2702  for (auto &inst : *getRegion().begin()) {2703    if (!(isa<SectionOp>(inst) || isa<TerminatorOp>(inst))) {2704      return emitOpError()2705             << "expected omp.section op or terminator op inside region";2706    }2707  }2708 2709  return success();2710}2711 2712//===----------------------------------------------------------------------===//2713// SingleOp2714//===----------------------------------------------------------------------===//2715 2716void SingleOp::build(OpBuilder &builder, OperationState &state,2717                     const SingleOperands &clauses) {2718  MLIRContext *ctx = builder.getContext();2719  // TODO Store clauses in op: privateVars, privateSyms, privateNeedsBarrier2720  SingleOp::build(builder, state, clauses.allocateVars, clauses.allocatorVars,2721                  clauses.copyprivateVars,2722                  makeArrayAttr(ctx, clauses.copyprivateSyms), clauses.nowait,2723                  /*private_vars=*/{}, /*private_syms=*/nullptr,2724                  /*private_needs_barrier=*/nullptr);2725}2726 2727LogicalResult SingleOp::verify() {2728  // Check for allocate clause restrictions2729  if (getAllocateVars().size() != getAllocatorVars().size())2730    return emitError(2731        "expected equal sizes for allocate and allocator variables");2732 2733  return verifyCopyprivateVarList(*this, getCopyprivateVars(),2734                                  getCopyprivateSyms());2735}2736 2737//===----------------------------------------------------------------------===//2738// WorkshareOp2739//===----------------------------------------------------------------------===//2740 2741void WorkshareOp::build(OpBuilder &builder, OperationState &state,2742                        const WorkshareOperands &clauses) {2743  WorkshareOp::build(builder, state, clauses.nowait);2744}2745 2746//===----------------------------------------------------------------------===//2747// WorkshareLoopWrapperOp2748//===----------------------------------------------------------------------===//2749 2750LogicalResult WorkshareLoopWrapperOp::verify() {2751  if (!(*this)->getParentOfType<WorkshareOp>())2752    return emitOpError() << "must be nested in an omp.workshare";2753  return success();2754}2755 2756LogicalResult WorkshareLoopWrapperOp::verifyRegions() {2757  if (isa_and_nonnull<LoopWrapperInterface>((*this)->getParentOp()) ||2758      getNestedWrapper())2759    return emitOpError() << "expected to be a standalone loop wrapper";2760 2761  return success();2762}2763 2764//===----------------------------------------------------------------------===//2765// LoopWrapperInterface2766//===----------------------------------------------------------------------===//2767 2768LogicalResult LoopWrapperInterface::verifyImpl() {2769  Operation *op = this->getOperation();2770  if (!op->hasTrait<OpTrait::NoTerminator>() ||2771      !op->hasTrait<OpTrait::SingleBlock>())2772    return emitOpError() << "loop wrapper must also have the `NoTerminator` "2773                            "and `SingleBlock` traits";2774 2775  if (op->getNumRegions() != 1)2776    return emitOpError() << "loop wrapper does not contain exactly one region";2777 2778  Region &region = op->getRegion(0);2779  if (range_size(region.getOps()) != 1)2780    return emitOpError()2781           << "loop wrapper does not contain exactly one nested op";2782 2783  Operation &firstOp = *region.op_begin();2784  if (!isa<LoopNestOp, LoopWrapperInterface>(firstOp))2785    return emitOpError() << "nested in loop wrapper is not another loop "2786                            "wrapper or `omp.loop_nest`";2787 2788  return success();2789}2790 2791//===----------------------------------------------------------------------===//2792// LoopOp2793//===----------------------------------------------------------------------===//2794 2795void LoopOp::build(OpBuilder &builder, OperationState &state,2796                   const LoopOperands &clauses) {2797  MLIRContext *ctx = builder.getContext();2798 2799  LoopOp::build(builder, state, clauses.bindKind, clauses.privateVars,2800                makeArrayAttr(ctx, clauses.privateSyms),2801                clauses.privateNeedsBarrier, clauses.order, clauses.orderMod,2802                clauses.reductionMod, clauses.reductionVars,2803                makeDenseBoolArrayAttr(ctx, clauses.reductionByref),2804                makeArrayAttr(ctx, clauses.reductionSyms));2805}2806 2807LogicalResult LoopOp::verify() {2808  return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),2809                                getReductionByref());2810}2811 2812LogicalResult LoopOp::verifyRegions() {2813  if (llvm::isa_and_nonnull<LoopWrapperInterface>((*this)->getParentOp()) ||2814      getNestedWrapper())2815    return emitOpError() << "expected to be a standalone loop wrapper";2816 2817  return success();2818}2819 2820//===----------------------------------------------------------------------===//2821// WsloopOp2822//===----------------------------------------------------------------------===//2823 2824void WsloopOp::build(OpBuilder &builder, OperationState &state,2825                     ArrayRef<NamedAttribute> attributes) {2826  build(builder, state, /*allocate_vars=*/{}, /*allocator_vars=*/{},2827        /*linear_vars=*/ValueRange(), /*linear_step_vars=*/ValueRange(),2828        /*nowait=*/false, /*order=*/nullptr, /*order_mod=*/nullptr,2829        /*ordered=*/nullptr, /*private_vars=*/{}, /*private_syms=*/nullptr,2830        /*private_needs_barrier=*/false,2831        /*reduction_mod=*/nullptr, /*reduction_vars=*/ValueRange(),2832        /*reduction_byref=*/nullptr,2833        /*reduction_syms=*/nullptr, /*schedule_kind=*/nullptr,2834        /*schedule_chunk=*/nullptr, /*schedule_mod=*/nullptr,2835        /*schedule_simd=*/false);2836  state.addAttributes(attributes);2837}2838 2839void WsloopOp::build(OpBuilder &builder, OperationState &state,2840                     const WsloopOperands &clauses) {2841  MLIRContext *ctx = builder.getContext();2842  // TODO: Store clauses in op: allocateVars, allocatorVars2843  WsloopOp::build(2844      builder, state,2845      /*allocate_vars=*/{}, /*allocator_vars=*/{}, clauses.linearVars,2846      clauses.linearStepVars, clauses.nowait, clauses.order, clauses.orderMod,2847      clauses.ordered, clauses.privateVars,2848      makeArrayAttr(ctx, clauses.privateSyms), clauses.privateNeedsBarrier,2849      clauses.reductionMod, clauses.reductionVars,2850      makeDenseBoolArrayAttr(ctx, clauses.reductionByref),2851      makeArrayAttr(ctx, clauses.reductionSyms), clauses.scheduleKind,2852      clauses.scheduleChunk, clauses.scheduleMod, clauses.scheduleSimd);2853}2854 2855LogicalResult WsloopOp::verify() {2856  return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),2857                                getReductionByref());2858}2859 2860LogicalResult WsloopOp::verifyRegions() {2861  bool isCompositeChildLeaf =2862      llvm::dyn_cast_if_present<LoopWrapperInterface>((*this)->getParentOp());2863 2864  if (LoopWrapperInterface nested = getNestedWrapper()) {2865    if (!isComposite())2866      return emitError()2867             << "'omp.composite' attribute missing from composite wrapper";2868 2869    // Check for the allowed leaf constructs that may appear in a composite2870    // construct directly after DO/FOR.2871    if (!isa<SimdOp>(nested))2872      return emitError() << "only supported nested wrapper is 'omp.simd'";2873 2874  } else if (isComposite() && !isCompositeChildLeaf) {2875    return emitError()2876           << "'omp.composite' attribute present in non-composite wrapper";2877  } else if (!isComposite() && isCompositeChildLeaf) {2878    return emitError()2879           << "'omp.composite' attribute missing from composite wrapper";2880  }2881 2882  return success();2883}2884 2885//===----------------------------------------------------------------------===//2886// Simd construct [2.9.3.1]2887//===----------------------------------------------------------------------===//2888 2889void SimdOp::build(OpBuilder &builder, OperationState &state,2890                   const SimdOperands &clauses) {2891  MLIRContext *ctx = builder.getContext();2892  // TODO Store clauses in op: linearVars, linearStepVars2893  SimdOp::build(builder, state, clauses.alignedVars,2894                makeArrayAttr(ctx, clauses.alignments), clauses.ifExpr,2895                /*linear_vars=*/{}, /*linear_step_vars=*/{},2896                clauses.nontemporalVars, clauses.order, clauses.orderMod,2897                clauses.privateVars, makeArrayAttr(ctx, clauses.privateSyms),2898                clauses.privateNeedsBarrier, clauses.reductionMod,2899                clauses.reductionVars,2900                makeDenseBoolArrayAttr(ctx, clauses.reductionByref),2901                makeArrayAttr(ctx, clauses.reductionSyms), clauses.safelen,2902                clauses.simdlen);2903}2904 2905LogicalResult SimdOp::verify() {2906  if (getSimdlen().has_value() && getSafelen().has_value() &&2907      getSimdlen().value() > getSafelen().value())2908    return emitOpError()2909           << "simdlen clause and safelen clause are both present, but the "2910              "simdlen value is not less than or equal to safelen value";2911 2912  if (verifyAlignedClause(*this, getAlignments(), getAlignedVars()).failed())2913    return failure();2914 2915  if (verifyNontemporalClause(*this, getNontemporalVars()).failed())2916    return failure();2917 2918  bool isCompositeChildLeaf =2919      llvm::dyn_cast_if_present<LoopWrapperInterface>((*this)->getParentOp());2920 2921  if (!isComposite() && isCompositeChildLeaf)2922    return emitError()2923           << "'omp.composite' attribute missing from composite wrapper";2924 2925  if (isComposite() && !isCompositeChildLeaf)2926    return emitError()2927           << "'omp.composite' attribute present in non-composite wrapper";2928 2929  // Firstprivate is not allowed for SIMD in the standard. Check that none of2930  // the private decls are for firstprivate.2931  std::optional<ArrayAttr> privateSyms = getPrivateSyms();2932  if (privateSyms) {2933    for (const Attribute &sym : *privateSyms) {2934      auto symRef = cast<SymbolRefAttr>(sym);2935      omp::PrivateClauseOp privatizer =2936          SymbolTable::lookupNearestSymbolFrom<omp::PrivateClauseOp>(2937              getOperation(), symRef);2938      if (!privatizer)2939        return emitError() << "Cannot find privatizer '" << symRef << "'";2940      if (privatizer.getDataSharingType() ==2941          DataSharingClauseType::FirstPrivate)2942        return emitError() << "FIRSTPRIVATE cannot be used with SIMD";2943    }2944  }2945 2946  return success();2947}2948 2949LogicalResult SimdOp::verifyRegions() {2950  if (getNestedWrapper())2951    return emitOpError() << "must wrap an 'omp.loop_nest' directly";2952 2953  return success();2954}2955 2956//===----------------------------------------------------------------------===//2957// Distribute construct [2.9.4.1]2958//===----------------------------------------------------------------------===//2959 2960void DistributeOp::build(OpBuilder &builder, OperationState &state,2961                         const DistributeOperands &clauses) {2962  DistributeOp::build(builder, state, clauses.allocateVars,2963                      clauses.allocatorVars, clauses.distScheduleStatic,2964                      clauses.distScheduleChunkSize, clauses.order,2965                      clauses.orderMod, clauses.privateVars,2966                      makeArrayAttr(builder.getContext(), clauses.privateSyms),2967                      clauses.privateNeedsBarrier);2968}2969 2970LogicalResult DistributeOp::verify() {2971  if (this->getDistScheduleChunkSize() && !this->getDistScheduleStatic())2972    return emitOpError() << "chunk size set without "2973                            "dist_schedule_static being present";2974 2975  if (getAllocateVars().size() != getAllocatorVars().size())2976    return emitError(2977        "expected equal sizes for allocate and allocator variables");2978 2979  return success();2980}2981 2982LogicalResult DistributeOp::verifyRegions() {2983  if (LoopWrapperInterface nested = getNestedWrapper()) {2984    if (!isComposite())2985      return emitError()2986             << "'omp.composite' attribute missing from composite wrapper";2987    // Check for the allowed leaf constructs that may appear in a composite2988    // construct directly after DISTRIBUTE.2989    if (isa<WsloopOp>(nested)) {2990      Operation *parentOp = (*this)->getParentOp();2991      if (!llvm::dyn_cast_if_present<ParallelOp>(parentOp) ||2992          !cast<ComposableOpInterface>(parentOp).isComposite()) {2993        return emitError() << "an 'omp.wsloop' nested wrapper is only allowed "2994                              "when a composite 'omp.parallel' is the direct "2995                              "parent";2996      }2997    } else if (!isa<SimdOp>(nested))2998      return emitError() << "only supported nested wrappers are 'omp.simd' and "2999                            "'omp.wsloop'";3000  } else if (isComposite()) {3001    return emitError()3002           << "'omp.composite' attribute present in non-composite wrapper";3003  }3004 3005  return success();3006}3007 3008//===----------------------------------------------------------------------===//3009// DeclareMapperOp / DeclareMapperInfoOp3010//===----------------------------------------------------------------------===//3011 3012LogicalResult DeclareMapperInfoOp::verify() {3013  return verifyMapClause(*this, getMapVars());3014}3015 3016LogicalResult DeclareMapperOp::verifyRegions() {3017  if (!llvm::isa_and_present<DeclareMapperInfoOp>(3018          getRegion().getBlocks().front().getTerminator()))3019    return emitOpError() << "expected terminator to be a DeclareMapperInfoOp";3020 3021  return success();3022}3023 3024//===----------------------------------------------------------------------===//3025// DeclareReductionOp3026//===----------------------------------------------------------------------===//3027 3028LogicalResult DeclareReductionOp::verifyRegions() {3029  if (!getAllocRegion().empty()) {3030    for (YieldOp yieldOp : getAllocRegion().getOps<YieldOp>()) {3031      if (yieldOp.getResults().size() != 1 ||3032          yieldOp.getResults().getTypes()[0] != getType())3033        return emitOpError() << "expects alloc region to yield a value "3034                                "of the reduction type";3035    }3036  }3037 3038  if (getInitializerRegion().empty())3039    return emitOpError() << "expects non-empty initializer region";3040  Block &initializerEntryBlock = getInitializerRegion().front();3041 3042  if (initializerEntryBlock.getNumArguments() == 1) {3043    if (!getAllocRegion().empty())3044      return emitOpError() << "expects two arguments to the initializer region "3045                              "when an allocation region is used";3046  } else if (initializerEntryBlock.getNumArguments() == 2) {3047    if (getAllocRegion().empty())3048      return emitOpError() << "expects one argument to the initializer region "3049                              "when no allocation region is used";3050  } else {3051    return emitOpError()3052           << "expects one or two arguments to the initializer region";3053  }3054 3055  for (mlir::Value arg : initializerEntryBlock.getArguments())3056    if (arg.getType() != getType())3057      return emitOpError() << "expects initializer region argument to match "3058                              "the reduction type";3059 3060  for (YieldOp yieldOp : getInitializerRegion().getOps<YieldOp>()) {3061    if (yieldOp.getResults().size() != 1 ||3062        yieldOp.getResults().getTypes()[0] != getType())3063      return emitOpError() << "expects initializer region to yield a value "3064                              "of the reduction type";3065  }3066 3067  if (getReductionRegion().empty())3068    return emitOpError() << "expects non-empty reduction region";3069  Block &reductionEntryBlock = getReductionRegion().front();3070  if (reductionEntryBlock.getNumArguments() != 2 ||3071      reductionEntryBlock.getArgumentTypes()[0] !=3072          reductionEntryBlock.getArgumentTypes()[1] ||3073      reductionEntryBlock.getArgumentTypes()[0] != getType())3074    return emitOpError() << "expects reduction region with two arguments of "3075                            "the reduction type";3076  for (YieldOp yieldOp : getReductionRegion().getOps<YieldOp>()) {3077    if (yieldOp.getResults().size() != 1 ||3078        yieldOp.getResults().getTypes()[0] != getType())3079      return emitOpError() << "expects reduction region to yield a value "3080                              "of the reduction type";3081  }3082 3083  if (!getAtomicReductionRegion().empty()) {3084    Block &atomicReductionEntryBlock = getAtomicReductionRegion().front();3085    if (atomicReductionEntryBlock.getNumArguments() != 2 ||3086        atomicReductionEntryBlock.getArgumentTypes()[0] !=3087            atomicReductionEntryBlock.getArgumentTypes()[1])3088      return emitOpError() << "expects atomic reduction region with two "3089                              "arguments of the same type";3090    auto ptrType = llvm::dyn_cast<PointerLikeType>(3091        atomicReductionEntryBlock.getArgumentTypes()[0]);3092    if (!ptrType ||3093        (ptrType.getElementType() && ptrType.getElementType() != getType()))3094      return emitOpError() << "expects atomic reduction region arguments to "3095                              "be accumulators containing the reduction type";3096  }3097 3098  if (getCleanupRegion().empty())3099    return success();3100  Block &cleanupEntryBlock = getCleanupRegion().front();3101  if (cleanupEntryBlock.getNumArguments() != 1 ||3102      cleanupEntryBlock.getArgument(0).getType() != getType())3103    return emitOpError() << "expects cleanup region with one argument "3104                            "of the reduction type";3105 3106  return success();3107}3108 3109//===----------------------------------------------------------------------===//3110// TaskOp3111//===----------------------------------------------------------------------===//3112 3113void TaskOp::build(OpBuilder &builder, OperationState &state,3114                   const TaskOperands &clauses) {3115  MLIRContext *ctx = builder.getContext();3116  TaskOp::build(builder, state, clauses.allocateVars, clauses.allocatorVars,3117                makeArrayAttr(ctx, clauses.dependKinds), clauses.dependVars,3118                clauses.final, clauses.ifExpr, clauses.inReductionVars,3119                makeDenseBoolArrayAttr(ctx, clauses.inReductionByref),3120                makeArrayAttr(ctx, clauses.inReductionSyms), clauses.mergeable,3121                clauses.priority, /*private_vars=*/clauses.privateVars,3122                /*private_syms=*/makeArrayAttr(ctx, clauses.privateSyms),3123                clauses.privateNeedsBarrier, clauses.untied,3124                clauses.eventHandle);3125}3126 3127LogicalResult TaskOp::verify() {3128  LogicalResult verifyDependVars =3129      verifyDependVarList(*this, getDependKinds(), getDependVars());3130  return failed(verifyDependVars)3131             ? verifyDependVars3132             : verifyReductionVarList(*this, getInReductionSyms(),3133                                      getInReductionVars(),3134                                      getInReductionByref());3135}3136 3137//===----------------------------------------------------------------------===//3138// TaskgroupOp3139//===----------------------------------------------------------------------===//3140 3141void TaskgroupOp::build(OpBuilder &builder, OperationState &state,3142                        const TaskgroupOperands &clauses) {3143  MLIRContext *ctx = builder.getContext();3144  TaskgroupOp::build(builder, state, clauses.allocateVars,3145                     clauses.allocatorVars, clauses.taskReductionVars,3146                     makeDenseBoolArrayAttr(ctx, clauses.taskReductionByref),3147                     makeArrayAttr(ctx, clauses.taskReductionSyms));3148}3149 3150LogicalResult TaskgroupOp::verify() {3151  return verifyReductionVarList(*this, getTaskReductionSyms(),3152                                getTaskReductionVars(),3153                                getTaskReductionByref());3154}3155 3156//===----------------------------------------------------------------------===//3157// TaskloopOp3158//===----------------------------------------------------------------------===//3159 3160void TaskloopOp::build(OpBuilder &builder, OperationState &state,3161                       const TaskloopOperands &clauses) {3162  MLIRContext *ctx = builder.getContext();3163  TaskloopOp::build(3164      builder, state, clauses.allocateVars, clauses.allocatorVars,3165      clauses.final, clauses.grainsizeMod, clauses.grainsize, clauses.ifExpr,3166      clauses.inReductionVars,3167      makeDenseBoolArrayAttr(ctx, clauses.inReductionByref),3168      makeArrayAttr(ctx, clauses.inReductionSyms), clauses.mergeable,3169      clauses.nogroup, clauses.numTasksMod, clauses.numTasks, clauses.priority,3170      /*private_vars=*/clauses.privateVars,3171      /*private_syms=*/makeArrayAttr(ctx, clauses.privateSyms),3172      clauses.privateNeedsBarrier, clauses.reductionMod, clauses.reductionVars,3173      makeDenseBoolArrayAttr(ctx, clauses.reductionByref),3174      makeArrayAttr(ctx, clauses.reductionSyms), clauses.untied);3175}3176 3177LogicalResult TaskloopOp::verify() {3178  if (getAllocateVars().size() != getAllocatorVars().size())3179    return emitError(3180        "expected equal sizes for allocate and allocator variables");3181  if (failed(verifyReductionVarList(*this, getReductionSyms(),3182                                    getReductionVars(), getReductionByref())) ||3183      failed(verifyReductionVarList(*this, getInReductionSyms(),3184                                    getInReductionVars(),3185                                    getInReductionByref())))3186    return failure();3187 3188  if (!getReductionVars().empty() && getNogroup())3189    return emitError("if a reduction clause is present on the taskloop "3190                     "directive, the nogroup clause must not be specified");3191  for (auto var : getReductionVars()) {3192    if (llvm::is_contained(getInReductionVars(), var))3193      return emitError("the same list item cannot appear in both a reduction "3194                       "and an in_reduction clause");3195  }3196 3197  if (getGrainsize() && getNumTasks()) {3198    return emitError(3199        "the grainsize clause and num_tasks clause are mutually exclusive and "3200        "may not appear on the same taskloop directive");3201  }3202 3203  return success();3204}3205 3206LogicalResult TaskloopOp::verifyRegions() {3207  if (LoopWrapperInterface nested = getNestedWrapper()) {3208    if (!isComposite())3209      return emitError()3210             << "'omp.composite' attribute missing from composite wrapper";3211 3212    // Check for the allowed leaf constructs that may appear in a composite3213    // construct directly after TASKLOOP.3214    if (!isa<SimdOp>(nested))3215      return emitError() << "only supported nested wrapper is 'omp.simd'";3216  } else if (isComposite()) {3217    return emitError()3218           << "'omp.composite' attribute present in non-composite wrapper";3219  }3220 3221  return success();3222}3223 3224//===----------------------------------------------------------------------===//3225// LoopNestOp3226//===----------------------------------------------------------------------===//3227 3228ParseResult LoopNestOp::parse(OpAsmParser &parser, OperationState &result) {3229  // Parse an opening `(` followed by induction variables followed by `)`3230  SmallVector<OpAsmParser::Argument> ivs;3231  SmallVector<OpAsmParser::UnresolvedOperand> lbs, ubs;3232  Type loopVarType;3233  if (parser.parseArgumentList(ivs, OpAsmParser::Delimiter::Paren) ||3234      parser.parseColonType(loopVarType) ||3235      // Parse loop bounds.3236      parser.parseEqual() ||3237      parser.parseOperandList(lbs, ivs.size(), OpAsmParser::Delimiter::Paren) ||3238      parser.parseKeyword("to") ||3239      parser.parseOperandList(ubs, ivs.size(), OpAsmParser::Delimiter::Paren))3240    return failure();3241 3242  for (auto &iv : ivs)3243    iv.type = loopVarType;3244 3245  auto *ctx = parser.getBuilder().getContext();3246  // Parse "inclusive" flag.3247  if (succeeded(parser.parseOptionalKeyword("inclusive")))3248    result.addAttribute("loop_inclusive", UnitAttr::get(ctx));3249 3250  // Parse step values.3251  SmallVector<OpAsmParser::UnresolvedOperand> steps;3252  if (parser.parseKeyword("step") ||3253      parser.parseOperandList(steps, ivs.size(), OpAsmParser::Delimiter::Paren))3254    return failure();3255 3256  // Parse collapse3257  int64_t value = 0;3258  if (!parser.parseOptionalKeyword("collapse") &&3259      (parser.parseLParen() || parser.parseInteger(value) ||3260       parser.parseRParen()))3261    return failure();3262  if (value > 1)3263    result.addAttribute(3264        "collapse_num_loops",3265        IntegerAttr::get(parser.getBuilder().getI64Type(), value));3266 3267  // Parse tiles3268  SmallVector<int64_t> tiles;3269  auto parseTiles = [&]() -> ParseResult {3270    int64_t tile;3271    if (parser.parseInteger(tile))3272      return failure();3273    tiles.push_back(tile);3274    return success();3275  };3276 3277  if (!parser.parseOptionalKeyword("tiles") &&3278      (parser.parseLParen() || parser.parseCommaSeparatedList(parseTiles) ||3279       parser.parseRParen()))3280    return failure();3281 3282  if (tiles.size() > 0)3283    result.addAttribute("tile_sizes", DenseI64ArrayAttr::get(ctx, tiles));3284 3285  // Parse the body.3286  Region *region = result.addRegion();3287  if (parser.parseRegion(*region, ivs))3288    return failure();3289 3290  // Resolve operands.3291  if (parser.resolveOperands(lbs, loopVarType, result.operands) ||3292      parser.resolveOperands(ubs, loopVarType, result.operands) ||3293      parser.resolveOperands(steps, loopVarType, result.operands))3294    return failure();3295 3296  // Parse the optional attribute list.3297  return parser.parseOptionalAttrDict(result.attributes);3298}3299 3300void LoopNestOp::print(OpAsmPrinter &p) {3301  Region &region = getRegion();3302  auto args = region.getArguments();3303  p << " (" << args << ") : " << args[0].getType() << " = ("3304    << getLoopLowerBounds() << ") to (" << getLoopUpperBounds() << ") ";3305  if (getLoopInclusive())3306    p << "inclusive ";3307  p << "step (" << getLoopSteps() << ") ";3308  if (int64_t numCollapse = getCollapseNumLoops())3309    if (numCollapse > 1)3310      p << "collapse(" << numCollapse << ") ";3311 3312  if (const auto tiles = getTileSizes())3313    p << "tiles(" << tiles.value() << ") ";3314 3315  p.printRegion(region, /*printEntryBlockArgs=*/false);3316}3317 3318void LoopNestOp::build(OpBuilder &builder, OperationState &state,3319                       const LoopNestOperands &clauses) {3320  MLIRContext *ctx = builder.getContext();3321  LoopNestOp::build(builder, state, clauses.collapseNumLoops,3322                    clauses.loopLowerBounds, clauses.loopUpperBounds,3323                    clauses.loopSteps, clauses.loopInclusive,3324                    makeDenseI64ArrayAttr(ctx, clauses.tileSizes));3325}3326 3327LogicalResult LoopNestOp::verify() {3328  if (getLoopLowerBounds().empty())3329    return emitOpError() << "must represent at least one loop";3330 3331  if (getLoopLowerBounds().size() != getIVs().size())3332    return emitOpError() << "number of range arguments and IVs do not match";3333 3334  for (auto [lb, iv] : llvm::zip_equal(getLoopLowerBounds(), getIVs())) {3335    if (lb.getType() != iv.getType())3336      return emitOpError()3337             << "range argument type does not match corresponding IV type";3338  }3339 3340  uint64_t numIVs = getIVs().size();3341 3342  if (const auto &numCollapse = getCollapseNumLoops())3343    if (numCollapse > numIVs)3344      return emitOpError()3345             << "collapse value is larger than the number of loops";3346 3347  if (const auto &tiles = getTileSizes())3348    if (tiles.value().size() > numIVs)3349      return emitOpError() << "too few canonical loops for tile dimensions";3350 3351  if (!llvm::dyn_cast_if_present<LoopWrapperInterface>((*this)->getParentOp()))3352    return emitOpError() << "expects parent op to be a loop wrapper";3353 3354  return success();3355}3356 3357void LoopNestOp::gatherWrappers(3358    SmallVectorImpl<LoopWrapperInterface> &wrappers) {3359  Operation *parent = (*this)->getParentOp();3360  while (auto wrapper =3361             llvm::dyn_cast_if_present<LoopWrapperInterface>(parent)) {3362    wrappers.push_back(wrapper);3363    parent = parent->getParentOp();3364  }3365}3366 3367//===----------------------------------------------------------------------===//3368// OpenMP canonical loop handling3369//===----------------------------------------------------------------------===//3370 3371std::tuple<NewCliOp, OpOperand *, OpOperand *>3372mlir::omp ::decodeCli(Value cli) {3373 3374  // Defining a CLI for a generated loop is optional; if there is none then3375  // there is no followup-tranformation3376  if (!cli)3377    return {{}, nullptr, nullptr};3378 3379  assert(cli.getType() == CanonicalLoopInfoType::get(cli.getContext()) &&3380         "Unexpected type of cli");3381 3382  NewCliOp create = cast<NewCliOp>(cli.getDefiningOp());3383  OpOperand *gen = nullptr;3384  OpOperand *cons = nullptr;3385  for (OpOperand &use : cli.getUses()) {3386    auto op = cast<LoopTransformationInterface>(use.getOwner());3387 3388    unsigned opnum = use.getOperandNumber();3389    if (op.isGeneratee(opnum)) {3390      assert(!gen && "Each CLI may have at most one def");3391      gen = &use;3392    } else if (op.isApplyee(opnum)) {3393      assert(!cons && "Each CLI may have at most one consumer");3394      cons = &use;3395    } else {3396      llvm_unreachable("Unexpected operand for a CLI");3397    }3398  }3399 3400  return {create, gen, cons};3401}3402 3403void NewCliOp::build(::mlir::OpBuilder &odsBuilder,3404                     ::mlir::OperationState &odsState) {3405  odsState.addTypes(CanonicalLoopInfoType::get(odsBuilder.getContext()));3406}3407 3408void NewCliOp::getAsmResultNames(OpAsmSetValueNameFn setNameFn) {3409  Value result = getResult();3410  auto [newCli, gen, cons] = decodeCli(result);3411 3412  // Structured binding `gen` cannot be captured in lambdas before C++203413  OpOperand *generator = gen;3414 3415  // Derive the CLI variable name from its generator:3416  //  * "canonloop" for omp.canonical_loop3417  //  * custom name for loop transformation generatees3418  //  * "cli" as fallback if no generator3419  //  * "_r<idx>" suffix for nested loops, where <idx> is the sequential order3420  //  at that level3421  //  * "_s<idx>" suffix for operations with multiple regions, where <idx> is3422  //  the index of that region3423  std::string cliName{"cli"};3424  if (gen) {3425    cliName =3426        TypeSwitch<Operation *, std::string>(gen->getOwner())3427            .Case([&](CanonicalLoopOp op) {3428              return generateLoopNestingName("canonloop", op);3429            })3430            .Case([&](UnrollHeuristicOp op) -> std::string {3431              llvm_unreachable("heuristic unrolling does not generate a loop");3432            })3433            .Case([&](TileOp op) -> std::string {3434              auto [generateesFirst, generateesCount] =3435                  op.getGenerateesODSOperandIndexAndLength();3436              unsigned firstGrid = generateesFirst;3437              unsigned firstIntratile = generateesFirst + generateesCount / 2;3438              unsigned end = generateesFirst + generateesCount;3439              unsigned opnum = generator->getOperandNumber();3440              // In the OpenMP apply and looprange clauses, indices are 1-based3441              if (firstGrid <= opnum && opnum < firstIntratile) {3442                unsigned gridnum = opnum - firstGrid + 1;3443                return ("grid" + Twine(gridnum)).str();3444              }3445              if (firstIntratile <= opnum && opnum < end) {3446                unsigned intratilenum = opnum - firstIntratile + 1;3447                return ("intratile" + Twine(intratilenum)).str();3448              }3449              llvm_unreachable("Unexpected generatee argument");3450            })3451            .DefaultUnreachable("TODO: Custom name for this operation");3452  }3453 3454  setNameFn(result, cliName);3455}3456 3457LogicalResult NewCliOp::verify() {3458  Value cli = getResult();3459 3460  assert(cli.getType() == CanonicalLoopInfoType::get(cli.getContext()) &&3461         "Unexpected type of cli");3462 3463  // Check that the CLI is used in at most generator and one consumer3464  OpOperand *gen = nullptr;3465  OpOperand *cons = nullptr;3466  for (mlir::OpOperand &use : cli.getUses()) {3467    auto op = cast<mlir::omp::LoopTransformationInterface>(use.getOwner());3468 3469    unsigned opnum = use.getOperandNumber();3470    if (op.isGeneratee(opnum)) {3471      if (gen) {3472        InFlightDiagnostic error =3473            emitOpError("CLI must have at most one generator");3474        error.attachNote(gen->getOwner()->getLoc())3475            .append("first generator here:");3476        error.attachNote(use.getOwner()->getLoc())3477            .append("second generator here:");3478        return error;3479      }3480 3481      gen = &use;3482    } else if (op.isApplyee(opnum)) {3483      if (cons) {3484        InFlightDiagnostic error =3485            emitOpError("CLI must have at most one consumer");3486        error.attachNote(cons->getOwner()->getLoc())3487            .append("first consumer here:")3488            .appendOp(*cons->getOwner(),3489                      OpPrintingFlags().printGenericOpForm());3490        error.attachNote(use.getOwner()->getLoc())3491            .append("second consumer here:")3492            .appendOp(*use.getOwner(), OpPrintingFlags().printGenericOpForm());3493        return error;3494      }3495 3496      cons = &use;3497    } else {3498      llvm_unreachable("Unexpected operand for a CLI");3499    }3500  }3501 3502  // If the CLI is source of a transformation, it must have a generator3503  if (cons && !gen) {3504    InFlightDiagnostic error = emitOpError("CLI has no generator");3505    error.attachNote(cons->getOwner()->getLoc())3506        .append("see consumer here: ")3507        .appendOp(*cons->getOwner(), OpPrintingFlags().printGenericOpForm());3508    return error;3509  }3510 3511  return success();3512}3513 3514void CanonicalLoopOp::build(OpBuilder &odsBuilder, OperationState &odsState,3515                            Value tripCount) {3516  odsState.addOperands(tripCount);3517  odsState.addOperands(Value());3518  (void)odsState.addRegion();3519}3520 3521void CanonicalLoopOp::build(OpBuilder &odsBuilder, OperationState &odsState,3522                            Value tripCount, ::mlir::Value cli) {3523  odsState.addOperands(tripCount);3524  odsState.addOperands(cli);3525  (void)odsState.addRegion();3526}3527 3528void CanonicalLoopOp::getAsmBlockNames(OpAsmSetBlockNameFn setNameFn) {3529  setNameFn(&getRegion().front(), "body_entry");3530}3531 3532void CanonicalLoopOp::getAsmBlockArgumentNames(Region &region,3533                                               OpAsmSetValueNameFn setNameFn) {3534  std::string ivName = generateLoopNestingName("iv", *this);3535  setNameFn(region.getArgument(0), ivName);3536}3537 3538void CanonicalLoopOp::print(OpAsmPrinter &p) {3539  if (getCli())3540    p << '(' << getCli() << ')';3541  p << ' ' << getInductionVar() << " : " << getInductionVar().getType()3542    << " in range(" << getTripCount() << ") ";3543 3544  p.printRegion(getRegion(), /*printEntryBlockArgs=*/false,3545                /*printBlockTerminators=*/true);3546 3547  p.printOptionalAttrDict((*this)->getAttrs());3548}3549 3550mlir::ParseResult CanonicalLoopOp::parse(::mlir::OpAsmParser &parser,3551                                         ::mlir::OperationState &result) {3552  CanonicalLoopInfoType cliType =3553      CanonicalLoopInfoType::get(parser.getContext());3554 3555  // Parse (optional) omp.cli identifier3556  OpAsmParser::UnresolvedOperand cli;3557  SmallVector<mlir::Value, 1> cliOperand;3558  if (!parser.parseOptionalLParen()) {3559    if (parser.parseOperand(cli) ||3560        parser.resolveOperand(cli, cliType, cliOperand) || parser.parseRParen())3561      return failure();3562  }3563 3564  // We derive the type of tripCount from inductionVariable. MLIR requires the3565  // type of tripCount to be known when calling resolveOperand so we have parse3566  // the type before processing the inductionVariable.3567  OpAsmParser::Argument inductionVariable;3568  OpAsmParser::UnresolvedOperand tripcount;3569  if (parser.parseArgument(inductionVariable, /*allowType*/ true) ||3570      parser.parseKeyword("in") || parser.parseKeyword("range") ||3571      parser.parseLParen() || parser.parseOperand(tripcount) ||3572      parser.parseRParen() ||3573      parser.resolveOperand(tripcount, inductionVariable.type, result.operands))3574    return failure();3575 3576  // Parse the loop body.3577  Region *region = result.addRegion();3578  if (parser.parseRegion(*region, {inductionVariable}))3579    return failure();3580 3581  // We parsed the cli operand forst, but because it is optional, it must be3582  // last in the operand list.3583  result.operands.append(cliOperand);3584 3585  // Parse the optional attribute list.3586  if (parser.parseOptionalAttrDict(result.attributes))3587    return failure();3588 3589  return mlir::success();3590}3591 3592LogicalResult CanonicalLoopOp::verify() {3593  // The region's entry must accept the induction variable3594  // It can also be empty if just created3595  if (!getRegion().empty()) {3596    Region &region = getRegion();3597    if (region.getNumArguments() != 1)3598      return emitOpError(3599          "Canonical loop region must have exactly one argument");3600 3601    if (getInductionVar().getType() != getTripCount().getType())3602      return emitOpError(3603          "Region argument must be the same type as the trip count");3604  }3605 3606  return success();3607}3608 3609Value CanonicalLoopOp::getInductionVar() { return getRegion().getArgument(0); }3610 3611std::pair<unsigned, unsigned>3612CanonicalLoopOp::getApplyeesODSOperandIndexAndLength() {3613  // No applyees3614  return {0, 0};3615}3616 3617std::pair<unsigned, unsigned>3618CanonicalLoopOp::getGenerateesODSOperandIndexAndLength() {3619  return getODSOperandIndexAndLength(odsIndex_cli);3620}3621 3622//===----------------------------------------------------------------------===//3623// UnrollHeuristicOp3624//===----------------------------------------------------------------------===//3625 3626void UnrollHeuristicOp::build(::mlir::OpBuilder &odsBuilder,3627                              ::mlir::OperationState &odsState,3628                              ::mlir::Value cli) {3629  odsState.addOperands(cli);3630}3631 3632void UnrollHeuristicOp::print(OpAsmPrinter &p) {3633  p << '(' << getApplyee() << ')';3634 3635  p.printOptionalAttrDict((*this)->getAttrs());3636}3637 3638mlir::ParseResult UnrollHeuristicOp::parse(::mlir::OpAsmParser &parser,3639                                           ::mlir::OperationState &result) {3640  auto cliType = CanonicalLoopInfoType::get(parser.getContext());3641 3642  if (parser.parseLParen())3643    return failure();3644 3645  OpAsmParser::UnresolvedOperand applyee;3646  if (parser.parseOperand(applyee) ||3647      parser.resolveOperand(applyee, cliType, result.operands))3648    return failure();3649 3650  if (parser.parseRParen())3651    return failure();3652 3653  // Optional output loop (full unrolling has none)3654  if (!parser.parseOptionalArrow()) {3655    if (parser.parseLParen() || parser.parseRParen())3656      return failure();3657  }3658 3659  // Parse the optional attribute list.3660  if (parser.parseOptionalAttrDict(result.attributes))3661    return failure();3662 3663  return mlir::success();3664}3665 3666std::pair<unsigned, unsigned>3667UnrollHeuristicOp ::getApplyeesODSOperandIndexAndLength() {3668  return getODSOperandIndexAndLength(odsIndex_applyee);3669}3670 3671std::pair<unsigned, unsigned>3672UnrollHeuristicOp::getGenerateesODSOperandIndexAndLength() {3673  return {0, 0};3674}3675 3676//===----------------------------------------------------------------------===//3677// TileOp3678//===----------------------------------------------------------------------===//3679 3680static void printLoopTransformClis(OpAsmPrinter &p, TileOp op,3681                                   OperandRange generatees,3682                                   OperandRange applyees) {3683  if (!generatees.empty())3684    p << '(' << llvm::interleaved(generatees) << ')';3685 3686  if (!applyees.empty())3687    p << " <- (" << llvm::interleaved(applyees) << ')';3688}3689 3690static ParseResult parseLoopTransformClis(3691    OpAsmParser &parser,3692    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &generateesOperands,3693    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &applyeesOperands) {3694  if (parser.parseOptionalLess()) {3695    // Syntax 1: generatees present3696 3697    if (parser.parseOperandList(generateesOperands,3698                                mlir::OpAsmParser::Delimiter::Paren))3699      return failure();3700 3701    if (parser.parseLess())3702      return failure();3703  } else {3704    // Syntax 2: generatees omitted3705  }3706 3707  // Parse `<-` (`<` has already been parsed)3708  if (parser.parseMinus())3709    return failure();3710 3711  if (parser.parseOperandList(applyeesOperands,3712                              mlir::OpAsmParser::Delimiter::Paren))3713    return failure();3714 3715  return success();3716}3717 3718LogicalResult TileOp::verify() {3719  if (getApplyees().empty())3720    return emitOpError() << "must apply to at least one loop";3721 3722  if (getSizes().size() != getApplyees().size())3723    return emitOpError() << "there must be one tile size for each applyee";3724 3725  if (!getGeneratees().empty() &&3726      2 * getSizes().size() != getGeneratees().size())3727    return emitOpError()3728           << "expecting two times the number of generatees than applyees";3729 3730  DenseSet<Value> parentIVs;3731 3732  Value parent = getApplyees().front();3733  for (auto &&applyee : llvm::drop_begin(getApplyees())) {3734    auto [parentCreate, parentGen, parentCons] = decodeCli(parent);3735    auto [create, gen, cons] = decodeCli(applyee);3736 3737    if (!parentGen)3738      return emitOpError() << "applyee CLI has no generator";3739 3740    auto parentLoop = dyn_cast_or_null<CanonicalLoopOp>(parentGen->getOwner());3741    if (!parentGen)3742      return emitOpError()3743             << "currently only supports omp.canonical_loop as applyee";3744 3745    parentIVs.insert(parentLoop.getInductionVar());3746 3747    if (!gen)3748      return emitOpError() << "applyee CLI has no generator";3749    auto loop = dyn_cast_or_null<CanonicalLoopOp>(gen->getOwner());3750    if (!loop)3751      return emitOpError()3752             << "currently only supports omp.canonical_loop as applyee";3753 3754    // Canonical loop must be perfectly nested, i.e. the body of the parent must3755    // only contain the omp.canonical_loop of the nested loops, and3756    // omp.terminator3757    bool isPerfectlyNested = [&]() {3758      auto &parentBody = parentLoop.getRegion();3759      if (!parentBody.hasOneBlock())3760        return false;3761      auto &parentBlock = parentBody.getBlocks().front();3762 3763      auto nestedLoopIt = parentBlock.begin();3764      if (nestedLoopIt == parentBlock.end() ||3765          (&*nestedLoopIt != loop.getOperation()))3766        return false;3767 3768      auto termIt = std::next(nestedLoopIt);3769      if (termIt == parentBlock.end() || !isa<TerminatorOp>(termIt))3770        return false;3771 3772      if (std::next(termIt) != parentBlock.end())3773        return false;3774 3775      return true;3776    }();3777    if (!isPerfectlyNested)3778      return emitOpError() << "tiled loop nest must be perfectly nested";3779 3780    if (parentIVs.contains(loop.getTripCount()))3781      return emitOpError() << "tiled loop nest must be rectangular";3782 3783    parent = applyee;3784  }3785 3786  // TODO: The tile sizes must be computed before the loop, but checking this3787  // requires dominance analysis. For instance:3788  //3789  //      %canonloop = omp.new_cli3790  //      omp.canonical_loop(%canonloop) %iv : i32 in range(%tc) {3791  //        // write to %x3792  //        omp.terminator3793  //      }3794  //      %ts = llvm.load %x3795  //      omp.tile <- (%canonloop) sizes(%ts : i32)3796 3797  return success();3798}3799 3800std::pair<unsigned, unsigned> TileOp ::getApplyeesODSOperandIndexAndLength() {3801  return getODSOperandIndexAndLength(odsIndex_applyees);3802}3803 3804std::pair<unsigned, unsigned> TileOp::getGenerateesODSOperandIndexAndLength() {3805  return getODSOperandIndexAndLength(odsIndex_generatees);3806}3807 3808//===----------------------------------------------------------------------===//3809// Critical construct (2.17.1)3810//===----------------------------------------------------------------------===//3811 3812void CriticalDeclareOp::build(OpBuilder &builder, OperationState &state,3813                              const CriticalDeclareOperands &clauses) {3814  CriticalDeclareOp::build(builder, state, clauses.symName, clauses.hint);3815}3816 3817LogicalResult CriticalDeclareOp::verify() {3818  return verifySynchronizationHint(*this, getHint());3819}3820 3821LogicalResult CriticalOp::verifySymbolUses(SymbolTableCollection &symbolTable) {3822  if (getNameAttr()) {3823    SymbolRefAttr symbolRef = getNameAttr();3824    auto decl = symbolTable.lookupNearestSymbolFrom<CriticalDeclareOp>(3825        *this, symbolRef);3826    if (!decl) {3827      return emitOpError() << "expected symbol reference " << symbolRef3828                           << " to point to a critical declaration";3829    }3830  }3831 3832  return success();3833}3834 3835//===----------------------------------------------------------------------===//3836// Ordered construct3837//===----------------------------------------------------------------------===//3838 3839static LogicalResult verifyOrderedParent(Operation &op) {3840  bool hasRegion = op.getNumRegions() > 0;3841  auto loopOp = op.getParentOfType<LoopNestOp>();3842  if (!loopOp) {3843    if (hasRegion)3844      return success();3845 3846    // TODO: Consider if this needs to be the case only for the standalone3847    // variant of the ordered construct.3848    return op.emitOpError() << "must be nested inside of a loop";3849  }3850 3851  Operation *wrapper = loopOp->getParentOp();3852  if (auto wsloopOp = dyn_cast<WsloopOp>(wrapper)) {3853    IntegerAttr orderedAttr = wsloopOp.getOrderedAttr();3854    if (!orderedAttr)3855      return op.emitOpError() << "the enclosing worksharing-loop region must "3856                                 "have an ordered clause";3857 3858    if (hasRegion && orderedAttr.getInt() != 0)3859      return op.emitOpError() << "the enclosing loop's ordered clause must not "3860                                 "have a parameter present";3861 3862    if (!hasRegion && orderedAttr.getInt() == 0)3863      return op.emitOpError() << "the enclosing loop's ordered clause must "3864                                 "have a parameter present";3865  } else if (!isa<SimdOp>(wrapper)) {3866    return op.emitOpError() << "must be nested inside of a worksharing, simd "3867                               "or worksharing simd loop";3868  }3869  return success();3870}3871 3872void OrderedOp::build(OpBuilder &builder, OperationState &state,3873                      const OrderedOperands &clauses) {3874  OrderedOp::build(builder, state, clauses.doacrossDependType,3875                   clauses.doacrossNumLoops, clauses.doacrossDependVars);3876}3877 3878LogicalResult OrderedOp::verify() {3879  if (failed(verifyOrderedParent(**this)))3880    return failure();3881 3882  auto wrapper = (*this)->getParentOfType<WsloopOp>();3883  if (!wrapper || *wrapper.getOrdered() != *getDoacrossNumLoops())3884    return emitOpError() << "number of variables in depend clause does not "3885                         << "match number of iteration variables in the "3886                         << "doacross loop";3887 3888  return success();3889}3890 3891void OrderedRegionOp::build(OpBuilder &builder, OperationState &state,3892                            const OrderedRegionOperands &clauses) {3893  OrderedRegionOp::build(builder, state, clauses.parLevelSimd);3894}3895 3896LogicalResult OrderedRegionOp::verify() { return verifyOrderedParent(**this); }3897 3898//===----------------------------------------------------------------------===//3899// TaskwaitOp3900//===----------------------------------------------------------------------===//3901 3902void TaskwaitOp::build(OpBuilder &builder, OperationState &state,3903                       const TaskwaitOperands &clauses) {3904  // TODO Store clauses in op: dependKinds, dependVars, nowait.3905  TaskwaitOp::build(builder, state, /*depend_kinds=*/nullptr,3906                    /*depend_vars=*/{}, /*nowait=*/nullptr);3907}3908 3909//===----------------------------------------------------------------------===//3910// Verifier for AtomicReadOp3911//===----------------------------------------------------------------------===//3912 3913LogicalResult AtomicReadOp::verify() {3914  if (verifyCommon().failed())3915    return mlir::failure();3916 3917  if (auto mo = getMemoryOrder()) {3918    if (*mo == ClauseMemoryOrderKind::Acq_rel ||3919        *mo == ClauseMemoryOrderKind::Release) {3920      return emitError(3921          "memory-order must not be acq_rel or release for atomic reads");3922    }3923  }3924  return verifySynchronizationHint(*this, getHint());3925}3926 3927//===----------------------------------------------------------------------===//3928// Verifier for AtomicWriteOp3929//===----------------------------------------------------------------------===//3930 3931LogicalResult AtomicWriteOp::verify() {3932  if (verifyCommon().failed())3933    return mlir::failure();3934 3935  if (auto mo = getMemoryOrder()) {3936    if (*mo == ClauseMemoryOrderKind::Acq_rel ||3937        *mo == ClauseMemoryOrderKind::Acquire) {3938      return emitError(3939          "memory-order must not be acq_rel or acquire for atomic writes");3940    }3941  }3942  return verifySynchronizationHint(*this, getHint());3943}3944 3945//===----------------------------------------------------------------------===//3946// Verifier for AtomicUpdateOp3947//===----------------------------------------------------------------------===//3948 3949LogicalResult AtomicUpdateOp::canonicalize(AtomicUpdateOp op,3950                                           PatternRewriter &rewriter) {3951  if (op.isNoOp()) {3952    rewriter.eraseOp(op);3953    return success();3954  }3955  if (Value writeVal = op.getWriteOpVal()) {3956    rewriter.replaceOpWithNewOp<AtomicWriteOp>(3957        op, op.getX(), writeVal, op.getHintAttr(), op.getMemoryOrderAttr());3958    return success();3959  }3960  return failure();3961}3962 3963LogicalResult AtomicUpdateOp::verify() {3964  if (verifyCommon().failed())3965    return mlir::failure();3966 3967  if (auto mo = getMemoryOrder()) {3968    if (*mo == ClauseMemoryOrderKind::Acq_rel ||3969        *mo == ClauseMemoryOrderKind::Acquire) {3970      return emitError(3971          "memory-order must not be acq_rel or acquire for atomic updates");3972    }3973  }3974 3975  return verifySynchronizationHint(*this, getHint());3976}3977 3978LogicalResult AtomicUpdateOp::verifyRegions() { return verifyRegionsCommon(); }3979 3980//===----------------------------------------------------------------------===//3981// Verifier for AtomicCaptureOp3982//===----------------------------------------------------------------------===//3983 3984AtomicReadOp AtomicCaptureOp::getAtomicReadOp() {3985  if (auto op = dyn_cast<AtomicReadOp>(getFirstOp()))3986    return op;3987  return dyn_cast<AtomicReadOp>(getSecondOp());3988}3989 3990AtomicWriteOp AtomicCaptureOp::getAtomicWriteOp() {3991  if (auto op = dyn_cast<AtomicWriteOp>(getFirstOp()))3992    return op;3993  return dyn_cast<AtomicWriteOp>(getSecondOp());3994}3995 3996AtomicUpdateOp AtomicCaptureOp::getAtomicUpdateOp() {3997  if (auto op = dyn_cast<AtomicUpdateOp>(getFirstOp()))3998    return op;3999  return dyn_cast<AtomicUpdateOp>(getSecondOp());4000}4001 4002LogicalResult AtomicCaptureOp::verify() {4003  return verifySynchronizationHint(*this, getHint());4004}4005 4006LogicalResult AtomicCaptureOp::verifyRegions() {4007  if (verifyRegionsCommon().failed())4008    return mlir::failure();4009 4010  if (getFirstOp()->getAttr("hint") || getSecondOp()->getAttr("hint"))4011    return emitOpError(4012        "operations inside capture region must not have hint clause");4013 4014  if (getFirstOp()->getAttr("memory_order") ||4015      getSecondOp()->getAttr("memory_order"))4016    return emitOpError(4017        "operations inside capture region must not have memory_order clause");4018  return success();4019}4020 4021//===----------------------------------------------------------------------===//4022// CancelOp4023//===----------------------------------------------------------------------===//4024 4025void CancelOp::build(OpBuilder &builder, OperationState &state,4026                     const CancelOperands &clauses) {4027  CancelOp::build(builder, state, clauses.cancelDirective, clauses.ifExpr);4028}4029 4030static Operation *getParentInSameDialect(Operation *thisOp) {4031  Operation *parent = thisOp->getParentOp();4032  while (parent) {4033    if (parent->getDialect() == thisOp->getDialect())4034      return parent;4035    parent = parent->getParentOp();4036  }4037  return nullptr;4038}4039 4040LogicalResult CancelOp::verify() {4041  ClauseCancellationConstructType cct = getCancelDirective();4042  // The next OpenMP operation in the chain of parents4043  Operation *structuralParent = getParentInSameDialect((*this).getOperation());4044  if (!structuralParent)4045    return emitOpError() << "Orphaned cancel construct";4046 4047  if ((cct == ClauseCancellationConstructType::Parallel) &&4048      !mlir::isa<ParallelOp>(structuralParent)) {4049    return emitOpError() << "cancel parallel must appear "4050                         << "inside a parallel region";4051  }4052  if (cct == ClauseCancellationConstructType::Loop) {4053    // structural parent will be omp.loop_nest, directly nested inside4054    // omp.wsloop4055    auto wsloopOp = mlir::dyn_cast<WsloopOp>(structuralParent->getParentOp());4056 4057    if (!wsloopOp) {4058      return emitOpError()4059             << "cancel loop must appear inside a worksharing-loop region";4060    }4061    if (wsloopOp.getNowaitAttr()) {4062      return emitError() << "A worksharing construct that is canceled "4063                         << "must not have a nowait clause";4064    }4065    if (wsloopOp.getOrderedAttr()) {4066      return emitError() << "A worksharing construct that is canceled "4067                         << "must not have an ordered clause";4068    }4069 4070  } else if (cct == ClauseCancellationConstructType::Sections) {4071    // structural parent will be an omp.section, directly nested inside4072    // omp.sections4073    auto sectionsOp =4074        mlir::dyn_cast<SectionsOp>(structuralParent->getParentOp());4075    if (!sectionsOp) {4076      return emitOpError() << "cancel sections must appear "4077                           << "inside a sections region";4078    }4079    if (sectionsOp.getNowait()) {4080      return emitError() << "A sections construct that is canceled "4081                         << "must not have a nowait clause";4082    }4083  }4084  if ((cct == ClauseCancellationConstructType::Taskgroup) &&4085      (!mlir::isa<omp::TaskOp>(structuralParent) &&4086       !mlir::isa<omp::TaskloopOp>(structuralParent->getParentOp()))) {4087    return emitOpError() << "cancel taskgroup must appear "4088                         << "inside a task region";4089  }4090  return success();4091}4092 4093//===----------------------------------------------------------------------===//4094// CancellationPointOp4095//===----------------------------------------------------------------------===//4096 4097void CancellationPointOp::build(OpBuilder &builder, OperationState &state,4098                                const CancellationPointOperands &clauses) {4099  CancellationPointOp::build(builder, state, clauses.cancelDirective);4100}4101 4102LogicalResult CancellationPointOp::verify() {4103  ClauseCancellationConstructType cct = getCancelDirective();4104  // The next OpenMP operation in the chain of parents4105  Operation *structuralParent = getParentInSameDialect((*this).getOperation());4106  if (!structuralParent)4107    return emitOpError() << "Orphaned cancellation point";4108 4109  if ((cct == ClauseCancellationConstructType::Parallel) &&4110      !mlir::isa<ParallelOp>(structuralParent)) {4111    return emitOpError() << "cancellation point parallel must appear "4112                         << "inside a parallel region";4113  }4114  // Strucutal parent here will be an omp.loop_nest. Get the parent of that to4115  // find the wsloop4116  if ((cct == ClauseCancellationConstructType::Loop) &&4117      !mlir::isa<WsloopOp>(structuralParent->getParentOp())) {4118    return emitOpError() << "cancellation point loop must appear "4119                         << "inside a worksharing-loop region";4120  }4121  if ((cct == ClauseCancellationConstructType::Sections) &&4122      !mlir::isa<omp::SectionOp>(structuralParent)) {4123    return emitOpError() << "cancellation point sections must appear "4124                         << "inside a sections region";4125  }4126  if ((cct == ClauseCancellationConstructType::Taskgroup) &&4127      !mlir::isa<omp::TaskOp>(structuralParent)) {4128    return emitOpError() << "cancellation point taskgroup must appear "4129                         << "inside a task region";4130  }4131  return success();4132}4133 4134//===----------------------------------------------------------------------===//4135// MapBoundsOp4136//===----------------------------------------------------------------------===//4137 4138LogicalResult MapBoundsOp::verify() {4139  auto extent = getExtent();4140  auto upperbound = getUpperBound();4141  if (!extent && !upperbound)4142    return emitError("expected extent or upperbound.");4143  return success();4144}4145 4146void PrivateClauseOp::build(OpBuilder &odsBuilder, OperationState &odsState,4147                            TypeRange /*result_types*/, StringAttr symName,4148                            TypeAttr type) {4149  PrivateClauseOp::build(4150      odsBuilder, odsState, symName, type,4151      DataSharingClauseTypeAttr::get(odsBuilder.getContext(),4152                                     DataSharingClauseType::Private));4153}4154 4155LogicalResult PrivateClauseOp::verifyRegions() {4156  Type argType = getArgType();4157  auto verifyTerminator = [&](Operation *terminator,4158                              bool yieldsValue) -> LogicalResult {4159    if (!terminator->getBlock()->getSuccessors().empty())4160      return success();4161 4162    if (!llvm::isa<YieldOp>(terminator))4163      return mlir::emitError(terminator->getLoc())4164             << "expected exit block terminator to be an `omp.yield` op.";4165 4166    YieldOp yieldOp = llvm::cast<YieldOp>(terminator);4167    TypeRange yieldedTypes = yieldOp.getResults().getTypes();4168 4169    if (!yieldsValue) {4170      if (yieldedTypes.empty())4171        return success();4172 4173      return mlir::emitError(terminator->getLoc())4174             << "Did not expect any values to be yielded.";4175    }4176 4177    if (yieldedTypes.size() == 1 && yieldedTypes.front() == argType)4178      return success();4179 4180    auto error = mlir::emitError(yieldOp.getLoc())4181                 << "Invalid yielded value. Expected type: " << argType4182                 << ", got: ";4183 4184    if (yieldedTypes.empty())4185      error << "None";4186    else4187      error << yieldedTypes;4188 4189    return error;4190  };4191 4192  auto verifyRegion = [&](Region &region, unsigned expectedNumArgs,4193                          StringRef regionName,4194                          bool yieldsValue) -> LogicalResult {4195    assert(!region.empty());4196 4197    if (region.getNumArguments() != expectedNumArgs)4198      return mlir::emitError(region.getLoc())4199             << "`" << regionName << "`: "4200             << "expected " << expectedNumArgs4201             << " region arguments, got: " << region.getNumArguments();4202 4203    for (Block &block : region) {4204      // MLIR will verify the absence of the terminator for us.4205      if (!block.mightHaveTerminator())4206        continue;4207 4208      if (failed(verifyTerminator(block.getTerminator(), yieldsValue)))4209        return failure();4210    }4211 4212    return success();4213  };4214 4215  // Ensure all of the region arguments have the same type4216  for (Region *region : getRegions())4217    for (Type ty : region->getArgumentTypes())4218      if (ty != argType)4219        return emitError() << "Region argument type mismatch: got " << ty4220                           << " expected " << argType << ".";4221 4222  mlir::Region &initRegion = getInitRegion();4223  if (!initRegion.empty() &&4224      failed(verifyRegion(getInitRegion(), /*expectedNumArgs=*/2, "init",4225                          /*yieldsValue=*/true)))4226    return failure();4227 4228  DataSharingClauseType dsType = getDataSharingType();4229 4230  if (dsType == DataSharingClauseType::Private && !getCopyRegion().empty())4231    return emitError("`private` clauses do not require a `copy` region.");4232 4233  if (dsType == DataSharingClauseType::FirstPrivate && getCopyRegion().empty())4234    return emitError(4235        "`firstprivate` clauses require at least a `copy` region.");4236 4237  if (dsType == DataSharingClauseType::FirstPrivate &&4238      failed(verifyRegion(getCopyRegion(), /*expectedNumArgs=*/2, "copy",4239                          /*yieldsValue=*/true)))4240    return failure();4241 4242  if (!getDeallocRegion().empty() &&4243      failed(verifyRegion(getDeallocRegion(), /*expectedNumArgs=*/1, "dealloc",4244                          /*yieldsValue=*/false)))4245    return failure();4246 4247  return success();4248}4249 4250//===----------------------------------------------------------------------===//4251// Spec 5.2: Masked construct (10.5)4252//===----------------------------------------------------------------------===//4253 4254void MaskedOp::build(OpBuilder &builder, OperationState &state,4255                     const MaskedOperands &clauses) {4256  MaskedOp::build(builder, state, clauses.filteredThreadId);4257}4258 4259//===----------------------------------------------------------------------===//4260// Spec 5.2: Scan construct (5.6)4261//===----------------------------------------------------------------------===//4262 4263void ScanOp::build(OpBuilder &builder, OperationState &state,4264                   const ScanOperands &clauses) {4265  ScanOp::build(builder, state, clauses.inclusiveVars, clauses.exclusiveVars);4266}4267 4268LogicalResult ScanOp::verify() {4269  if (hasExclusiveVars() == hasInclusiveVars())4270    return emitError(4271        "Exactly one of EXCLUSIVE or INCLUSIVE clause is expected");4272  if (WsloopOp parentWsLoopOp = (*this)->getParentOfType<WsloopOp>()) {4273    if (parentWsLoopOp.getReductionModAttr() &&4274        parentWsLoopOp.getReductionModAttr().getValue() ==4275            ReductionModifier::inscan)4276      return success();4277  }4278  if (SimdOp parentSimdOp = (*this)->getParentOfType<SimdOp>()) {4279    if (parentSimdOp.getReductionModAttr() &&4280        parentSimdOp.getReductionModAttr().getValue() ==4281            ReductionModifier::inscan)4282      return success();4283  }4284  return emitError("SCAN directive needs to be enclosed within a parent "4285                   "worksharing loop construct or SIMD construct with INSCAN "4286                   "reduction modifier");4287}4288 4289/// Verifies align clause in allocate directive4290 4291LogicalResult AllocateDirOp::verify() {4292  std::optional<uint64_t> align = this->getAlign();4293 4294  if (align.has_value()) {4295    if ((align.value() > 0) && !llvm::has_single_bit(align.value()))4296      return emitError() << "ALIGN value : " << align.value()4297                         << " must be power of 2";4298  }4299 4300  return success();4301}4302 4303//===----------------------------------------------------------------------===//4304// TargetAllocMemOp4305//===----------------------------------------------------------------------===//4306 4307mlir::Type omp::TargetAllocMemOp::getAllocatedType() {4308  return getInTypeAttr().getValue();4309}4310 4311/// operation ::= %res = (`omp.target_alloc_mem`) $device : devicetype,4312///                      $in_type ( `(` $typeparams `)` )? ( `,` $shape )?4313///                      attr-dict-without-keyword4314static mlir::ParseResult parseTargetAllocMemOp(mlir::OpAsmParser &parser,4315                                               mlir::OperationState &result) {4316  auto &builder = parser.getBuilder();4317  bool hasOperands = false;4318  std::int32_t typeparamsSize = 0;4319 4320  // Parse device number as a new operand4321  mlir::OpAsmParser::UnresolvedOperand deviceOperand;4322  mlir::Type deviceType;4323  if (parser.parseOperand(deviceOperand) || parser.parseColonType(deviceType))4324    return mlir::failure();4325  if (parser.resolveOperand(deviceOperand, deviceType, result.operands))4326    return mlir::failure();4327  if (parser.parseComma())4328    return mlir::failure();4329 4330  mlir::Type intype;4331  if (parser.parseType(intype))4332    return mlir::failure();4333  result.addAttribute("in_type", mlir::TypeAttr::get(intype));4334  llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> operands;4335  llvm::SmallVector<mlir::Type> typeVec;4336  if (!parser.parseOptionalLParen()) {4337    // parse the LEN params of the derived type. (<params> : <types>)4338    if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None) ||4339        parser.parseColonTypeList(typeVec) || parser.parseRParen())4340      return mlir::failure();4341    typeparamsSize = operands.size();4342    hasOperands = true;4343  }4344  std::int32_t shapeSize = 0;4345  if (!parser.parseOptionalComma()) {4346    // parse size to scale by, vector of n dimensions of type index4347    if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None))4348      return mlir::failure();4349    shapeSize = operands.size() - typeparamsSize;4350    auto idxTy = builder.getIndexType();4351    for (std::int32_t i = typeparamsSize, end = operands.size(); i != end; ++i)4352      typeVec.push_back(idxTy);4353    hasOperands = true;4354  }4355  if (hasOperands &&4356      parser.resolveOperands(operands, typeVec, parser.getNameLoc(),4357                             result.operands))4358    return mlir::failure();4359 4360  mlir::Type restype = builder.getIntegerType(64);4361  if (!restype) {4362    parser.emitError(parser.getNameLoc(), "invalid allocate type: ") << intype;4363    return mlir::failure();4364  }4365  llvm::SmallVector<std::int32_t> segmentSizes{1, typeparamsSize, shapeSize};4366  result.addAttribute("operandSegmentSizes",4367                      builder.getDenseI32ArrayAttr(segmentSizes));4368  if (parser.parseOptionalAttrDict(result.attributes) ||4369      parser.addTypeToList(restype, result.types))4370    return mlir::failure();4371  return mlir::success();4372}4373 4374mlir::ParseResult omp::TargetAllocMemOp::parse(mlir::OpAsmParser &parser,4375                                               mlir::OperationState &result) {4376  return parseTargetAllocMemOp(parser, result);4377}4378 4379void omp::TargetAllocMemOp::print(mlir::OpAsmPrinter &p) {4380  p << " ";4381  p.printOperand(getDevice());4382  p << " : ";4383  p << getDevice().getType();4384  p << ", ";4385  p << getInType();4386  if (!getTypeparams().empty()) {4387    p << '(' << getTypeparams() << " : " << getTypeparams().getTypes() << ')';4388  }4389  for (auto sh : getShape()) {4390    p << ", ";4391    p.printOperand(sh);4392  }4393  p.printOptionalAttrDict((*this)->getAttrs(),4394                          {"in_type", "operandSegmentSizes"});4395}4396 4397llvm::LogicalResult omp::TargetAllocMemOp::verify() {4398  mlir::Type outType = getType();4399  if (!mlir::dyn_cast<IntegerType>(outType))4400    return emitOpError("must be a integer type");4401  return mlir::success();4402}4403 4404//===----------------------------------------------------------------------===//4405// WorkdistributeOp4406//===----------------------------------------------------------------------===//4407 4408LogicalResult WorkdistributeOp::verify() {4409  // Check that region exists and is not empty4410  Region &region = getRegion();4411  if (region.empty())4412    return emitOpError("region cannot be empty");4413  // Verify single entry point.4414  Block &entryBlock = region.front();4415  if (entryBlock.empty())4416    return emitOpError("region must contain a structured block");4417  // Verify single exit point.4418  bool hasTerminator = false;4419  for (Block &block : region) {4420    if (isa<TerminatorOp>(block.back())) {4421      if (hasTerminator) {4422        return emitOpError("region must have exactly one terminator");4423      }4424      hasTerminator = true;4425    }4426  }4427  if (!hasTerminator) {4428    return emitOpError("region must be terminated with omp.terminator");4429  }4430  auto walkResult = region.walk([&](Operation *op) -> WalkResult {4431    // No implicit barrier at end4432    if (isa<BarrierOp>(op)) {4433      return emitOpError(4434          "explicit barriers are not allowed in workdistribute region");4435    }4436    // Check for invalid nested constructs4437    if (isa<ParallelOp>(op)) {4438      return emitOpError(4439          "nested parallel constructs not allowed in workdistribute");4440    }4441    if (isa<TeamsOp>(op)) {4442      return emitOpError(4443          "nested teams constructs not allowed in workdistribute");4444    }4445    return WalkResult::advance();4446  });4447  if (walkResult.wasInterrupted())4448    return failure();4449 4450  Operation *parentOp = (*this)->getParentOp();4451  if (!llvm::dyn_cast<TeamsOp>(parentOp))4452    return emitOpError("workdistribute must be nested under teams");4453  return success();4454}4455 4456#define GET_ATTRDEF_CLASSES4457#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"4458 4459#define GET_OP_CLASSES4460#include "mlir/Dialect/OpenMP/OpenMPOps.cpp.inc"4461 4462#define GET_TYPEDEF_CLASSES4463#include "mlir/Dialect/OpenMP/OpenMPOpsTypes.cpp.inc"4464