brintos / llvm-project-archived public Read only

1//===-- OpenMP.cpp -- Open MP directive lowering --------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/10//11//===----------------------------------------------------------------------===//12 13#include "flang/Lower/OpenMP.h"14 15#include "Atomic.h"16#include "ClauseProcessor.h"17#include "DataSharingProcessor.h"18#include "Decomposer.h"19#include "Utils.h"20#include "flang/Common/idioms.h"21#include "flang/Evaluate/type.h"22#include "flang/Lower/Bridge.h"23#include "flang/Lower/ConvertCall.h"24#include "flang/Lower/ConvertExpr.h"25#include "flang/Lower/ConvertExprToHLFIR.h"26#include "flang/Lower/ConvertVariable.h"27#include "flang/Lower/DirectivesCommon.h"28#include "flang/Lower/OpenMP/Clauses.h"29#include "flang/Lower/PFTBuilder.h"30#include "flang/Lower/StatementContext.h"31#include "flang/Lower/Support/ReductionProcessor.h"32#include "flang/Lower/SymbolMap.h"33#include "flang/Optimizer/Builder/BoxValue.h"34#include "flang/Optimizer/Builder/FIRBuilder.h"35#include "flang/Optimizer/Builder/Todo.h"36#include "flang/Optimizer/Dialect/FIRType.h"37#include "flang/Optimizer/HLFIR/HLFIROps.h"38#include "flang/Parser/characters.h"39#include "flang/Parser/openmp-utils.h"40#include "flang/Parser/parse-tree.h"41#include "flang/Parser/tools.h"42#include "flang/Semantics/openmp-directive-sets.h"43#include "flang/Semantics/openmp-utils.h"44#include "flang/Semantics/tools.h"45#include "flang/Support/Flags.h"46#include "flang/Support/OpenMP-utils.h"47#include "flang/Utils/OpenMP.h"48#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"49#include "mlir/Dialect/OpenMP/OpenMPDialect.h"50#include "mlir/Support/StateStack.h"51#include "mlir/Transforms/RegionUtils.h"52#include "llvm/ADT/STLExtras.h"53 54using namespace Fortran::lower::omp;55using namespace Fortran::common::openmp;56using namespace Fortran::utils::openmp;57 58//===----------------------------------------------------------------------===//59// Code generation helper functions60//===----------------------------------------------------------------------===//61 62static void genOMPDispatch(lower::AbstractConverter &converter,63                           lower::SymMap &symTable,64                           semantics::SemanticsContext &semaCtx,65                           lower::pft::Evaluation &eval, mlir::Location loc,66                           const ConstructQueue &queue,67                           ConstructQueue::const_iterator item);68 69static void processHostEvalClauses(lower::AbstractConverter &converter,70                                   semantics::SemanticsContext &semaCtx,71                                   lower::StatementContext &stmtCtx,72                                   lower::pft::Evaluation &eval,73                                   mlir::Location loc);74 75namespace {76/// Structure holding information that is needed to pass host-evaluated77/// information to later lowering stages.78class HostEvalInfo {79public:80  // Allow this function access to private members in order to initialize them.81  friend void ::processHostEvalClauses(lower::AbstractConverter &,82                                       semantics::SemanticsContext &,83                                       lower::StatementContext &,84                                       lower::pft::Evaluation &,85                                       mlir::Location);86 87  /// Fill \c vars with values stored in \c ops.88  ///89  /// The order in which values are stored matches the one expected by \see90  /// bindOperands().91  void collectValues(llvm::SmallVectorImpl<mlir::Value> &vars) const {92    vars.append(ops.loopLowerBounds);93    vars.append(ops.loopUpperBounds);94    vars.append(ops.loopSteps);95 96    if (ops.numTeamsLower)97      vars.push_back(ops.numTeamsLower);98 99    if (ops.numTeamsUpper)100      vars.push_back(ops.numTeamsUpper);101 102    if (ops.numThreads)103      vars.push_back(ops.numThreads);104 105    if (ops.threadLimit)106      vars.push_back(ops.threadLimit);107  }108 109  /// Update \c ops, replacing all values with the corresponding block argument110  /// in \c args.111  ///112  /// The order in which values are stored in \c args is the same as the one113  /// used by \see collectValues().114  void bindOperands(llvm::ArrayRef<mlir::BlockArgument> args) {115    assert(args.size() ==116               ops.loopLowerBounds.size() + ops.loopUpperBounds.size() +117                   ops.loopSteps.size() + (ops.numTeamsLower ? 1 : 0) +118                   (ops.numTeamsUpper ? 1 : 0) + (ops.numThreads ? 1 : 0) +119                   (ops.threadLimit ? 1 : 0) &&120           "invalid block argument list");121    int argIndex = 0;122    for (size_t i = 0; i < ops.loopLowerBounds.size(); ++i)123      ops.loopLowerBounds[i] = args[argIndex++];124 125    for (size_t i = 0; i < ops.loopUpperBounds.size(); ++i)126      ops.loopUpperBounds[i] = args[argIndex++];127 128    for (size_t i = 0; i < ops.loopSteps.size(); ++i)129      ops.loopSteps[i] = args[argIndex++];130 131    if (ops.numTeamsLower)132      ops.numTeamsLower = args[argIndex++];133 134    if (ops.numTeamsUpper)135      ops.numTeamsUpper = args[argIndex++];136 137    if (ops.numThreads)138      ops.numThreads = args[argIndex++];139 140    if (ops.threadLimit)141      ops.threadLimit = args[argIndex++];142  }143 144  /// Update \p clauseOps and \p ivOut with the corresponding host-evaluated145  /// values and Fortran symbols, respectively, if they have already been146  /// initialized but not yet applied.147  ///148  /// \returns whether an update was performed. If not, these clauses were not149  ///          evaluated in the host device.150  bool apply(mlir::omp::LoopNestOperands &clauseOps,151             llvm::SmallVectorImpl<const semantics::Symbol *> &ivOut) {152    if (iv.empty() || loopNestApplied) {153      loopNestApplied = true;154      return false;155    }156 157    loopNestApplied = true;158    clauseOps.loopLowerBounds = ops.loopLowerBounds;159    clauseOps.loopUpperBounds = ops.loopUpperBounds;160    clauseOps.loopSteps = ops.loopSteps;161    clauseOps.collapseNumLoops = ops.collapseNumLoops;162    ivOut.append(iv);163    return true;164  }165 166  /// Update \p clauseOps with the corresponding host-evaluated values if they167  /// have already been initialized but not yet applied.168  ///169  /// \returns whether an update was performed. If not, these clauses were not170  ///          evaluated in the host device.171  bool apply(mlir::omp::ParallelOperands &clauseOps) {172    if (!ops.numThreads || parallelApplied) {173      parallelApplied = true;174      return false;175    }176 177    parallelApplied = true;178    clauseOps.numThreads = ops.numThreads;179    return true;180  }181 182  /// Update \p clauseOps with the corresponding host-evaluated values if they183  /// have already been initialized.184  ///185  /// \returns whether an update was performed. If not, these clauses were not186  ///          evaluated in the host device.187  bool apply(mlir::omp::TeamsOperands &clauseOps) {188    if (!ops.numTeamsLower && !ops.numTeamsUpper && !ops.threadLimit)189      return false;190 191    clauseOps.numTeamsLower = ops.numTeamsLower;192    clauseOps.numTeamsUpper = ops.numTeamsUpper;193    clauseOps.threadLimit = ops.threadLimit;194    return true;195  }196 197private:198  mlir::omp::HostEvaluatedOperands ops;199  llvm::SmallVector<const semantics::Symbol *> iv;200  bool loopNestApplied = false, parallelApplied = false;201};202} // namespace203 204/// Stack of \see HostEvalInfo to represent the current nest of \c omp.target205/// operations being created.206///207/// The current implementation prevents nested 'target' regions from breaking208/// the handling of the outer region by keeping a stack of information209/// structures, but it will probably still require some further work to support210/// reverse offloading.211class HostEvalInfoStackFrame212    : public mlir::StateStackFrameBase<HostEvalInfoStackFrame> {213public:214  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(HostEvalInfoStackFrame)215 216  HostEvalInfo info;217};218 219static HostEvalInfo *220getHostEvalInfoStackTop(lower::AbstractConverter &converter) {221  HostEvalInfoStackFrame *frame =222      converter.getStateStack().getStackTop<HostEvalInfoStackFrame>();223  return frame ? &frame->info : nullptr;224}225 226/// Stack frame for storing the OpenMPSectionsConstruct currently being227/// processed so that it can be referred to when lowering the construct.228class SectionsConstructStackFrame229    : public mlir::StateStackFrameBase<SectionsConstructStackFrame> {230public:231  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(SectionsConstructStackFrame)232 233  explicit SectionsConstructStackFrame(234      const parser::OpenMPSectionsConstruct &sectionsConstruct)235      : sectionsConstruct{sectionsConstruct} {}236 237  const parser::OpenMPSectionsConstruct &sectionsConstruct;238};239 240static const parser::OpenMPSectionsConstruct *241getSectionsConstructStackTop(lower::AbstractConverter &converter) {242  SectionsConstructStackFrame *frame =243      converter.getStateStack().getStackTop<SectionsConstructStackFrame>();244  return frame ? &frame->sectionsConstruct : nullptr;245}246 247/// Bind symbols to their corresponding entry block arguments.248///249/// The binding will be performed inside of the current block, which does not250/// necessarily have to be part of the operation for which the binding is done.251/// However, block arguments must be accessible. This enables controlling the252/// insertion point of any new MLIR operations related to the binding of253/// arguments of a loop wrapper operation.254///255/// \param [in] converter - PFT to MLIR conversion interface.256/// \param [in]        op - owner operation of the block arguments to bind.257/// \param [in]      args - entry block arguments information for the given258///                         operation.259static void bindEntryBlockArgs(lower::AbstractConverter &converter,260                               mlir::omp::BlockArgOpenMPOpInterface op,261                               const EntryBlockArgs &args) {262  assert(op != nullptr && "invalid block argument-defining operation");263  assert(args.isValid() && "invalid args");264  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();265 266  auto bindSingleMapLike = [&converter](const semantics::Symbol &sym,267                                        const mlir::BlockArgument &arg) {268    fir::ExtendedValue extVal = converter.getSymbolExtendedValue(sym);269    auto refType = mlir::dyn_cast<fir::ReferenceType>(arg.getType());270    if (refType && fir::isa_builtin_cptr_type(refType.getElementType())) {271      converter.bindSymbol(sym, arg);272    } else {273      extVal.match(274          [&](const fir::BoxValue &v) {275            converter.bindSymbol(sym, fir::BoxValue(arg, v.getLBounds(),276                                                    v.getExplicitParameters(),277                                                    v.getExplicitExtents()));278          },279          [&](const fir::MutableBoxValue &v) {280            converter.bindSymbol(281                sym, fir::MutableBoxValue(arg, v.getLBounds(),282                                          v.getMutableProperties()));283          },284          [&](const fir::ArrayBoxValue &v) {285            converter.bindSymbol(sym, fir::ArrayBoxValue(arg, v.getExtents(),286                                                         v.getLBounds(),287                                                         v.getSourceBox()));288          },289          [&](const fir::CharArrayBoxValue &v) {290            converter.bindSymbol(sym, fir::CharArrayBoxValue(arg, v.getLen(),291                                                             v.getExtents(),292                                                             v.getLBounds()));293          },294          [&](const fir::CharBoxValue &v) {295            converter.bindSymbol(sym, fir::CharBoxValue(arg, v.getLen()));296          },297          [&](const fir::UnboxedValue &v) { converter.bindSymbol(sym, arg); },298          [&](const auto &) {299            TODO(converter.getCurrentLocation(),300                 "target map clause operand unsupported type");301          });302    }303  };304 305  auto bindMapLike =306      [&bindSingleMapLike](llvm::ArrayRef<const semantics::Symbol *> syms,307                           llvm::ArrayRef<mlir::BlockArgument> args) {308        // Structure component symbols don't have bindings, and can only be309        // explicitly mapped individually. If a member is captured implicitly310        // we map the entirety of the derived type when we find its symbol.311        llvm::SmallVector<const semantics::Symbol *> processedSyms;312        llvm::copy_if(syms, std::back_inserter(processedSyms),313                      [](auto *sym) { return !sym->owner().IsDerivedType(); });314 315        for (auto [sym, arg] : llvm::zip_equal(processedSyms, args))316          bindSingleMapLike(*sym, arg);317      };318 319  auto bindPrivateLike = [&converter, &firOpBuilder](320                             llvm::ArrayRef<const semantics::Symbol *> syms,321                             llvm::ArrayRef<mlir::Value> vars,322                             llvm::ArrayRef<mlir::BlockArgument> args) {323    llvm::SmallVector<const semantics::Symbol *> processedSyms;324    for (auto *sym : syms) {325      if (const auto *commonDet =326              sym->detailsIf<semantics::CommonBlockDetails>()) {327        llvm::transform(commonDet->objects(), std::back_inserter(processedSyms),328                        [&](const auto &mem) { return &*mem; });329      } else {330        processedSyms.push_back(sym);331      }332    }333 334    for (auto [sym, var, arg] : llvm::zip_equal(processedSyms, vars, args))335      converter.bindSymbol(336          *sym,337          hlfir::translateToExtendedValue(338              var.getLoc(), firOpBuilder, hlfir::Entity{arg},339              /*contiguousHint=*/340              evaluate::IsSimplyContiguous(*sym, converter.getFoldingContext()))341              .first);342  };343 344  // Process in clause name alphabetical order to match block arguments order.345  // Do not bind host_eval variables because they cannot be used inside of the346  // corresponding region, except for very specific cases handled separately.347  bindMapLike(args.hasDeviceAddr.syms, op.getHasDeviceAddrBlockArgs());348  bindPrivateLike(args.inReduction.syms, args.inReduction.vars,349                  op.getInReductionBlockArgs());350  bindMapLike(args.map.syms, op.getMapBlockArgs());351  bindPrivateLike(args.priv.syms, args.priv.vars, op.getPrivateBlockArgs());352  bindPrivateLike(args.reduction.syms, args.reduction.vars,353                  op.getReductionBlockArgs());354  bindPrivateLike(args.taskReduction.syms, args.taskReduction.vars,355                  op.getTaskReductionBlockArgs());356  bindMapLike(args.useDeviceAddr.syms, op.getUseDeviceAddrBlockArgs());357  bindMapLike(args.useDevicePtr.syms, op.getUseDevicePtrBlockArgs());358}359 360/// Get the list of base values that the specified map-like variables point to.361///362/// This function must be kept in sync with changes to the `createMapInfoOp`363/// utility function, since it must take into account the potential introduction364/// of levels of indirection (i.e. intermediate ops).365///366/// \param [in]     vars - list of values passed to map-like clauses, returned367///                        by an `omp.map.info` operation.368/// \param [out] baseOps - populated with the `var_ptr` values of the369///                        corresponding defining operations.370static void371extractMappedBaseValues(llvm::ArrayRef<mlir::Value> vars,372                        llvm::SmallVectorImpl<mlir::Value> &baseOps) {373  llvm::transform(vars, std::back_inserter(baseOps), [](mlir::Value map) {374    auto mapInfo = map.getDefiningOp<mlir::omp::MapInfoOp>();375    assert(mapInfo && "expected all map vars to be defined by omp.map.info");376 377    mlir::Value varPtr = mapInfo.getVarPtr();378    if (auto boxAddr = varPtr.getDefiningOp<fir::BoxAddrOp>())379      return boxAddr.getVal();380 381    return varPtr;382  });383}384 385/// Populate the global \see hostEvalInfo after processing clauses for the given386/// \p eval OpenMP target construct, or nested constructs, if these must be387/// evaluated outside of the target region per the spec.388///389/// In particular, this will ensure that in 'target teams' and equivalent nested390/// constructs, the \c thread_limit and \c num_teams clauses will be evaluated391/// in the host. Additionally, loop bounds, steps and the \c num_threads clause392/// will also be evaluated in the host if a target SPMD construct is detected393/// (i.e. 'target teams distribute parallel do [simd]' or equivalent nesting).394///395/// The result, stored as a global, is intended to be used to populate the \c396/// host_eval operands of the associated \c omp.target operation, and also to be397/// checked and used by later lowering steps to populate the corresponding398/// operands of the \c omp.teams, \c omp.parallel or \c omp.loop_nest399/// operations.400static void processHostEvalClauses(lower::AbstractConverter &converter,401                                   semantics::SemanticsContext &semaCtx,402                                   lower::StatementContext &stmtCtx,403                                   lower::pft::Evaluation &eval,404                                   mlir::Location loc) {405  // Obtain the list of clauses of the given OpenMP block or loop construct406  // evaluation. Other evaluations passed to this lambda keep `clauses`407  // unchanged.408  auto extractClauses = [&semaCtx](lower::pft::Evaluation &eval,409                                   List<Clause> &clauses) {410    const auto *ompEval = eval.getIf<parser::OpenMPConstruct>();411    if (!ompEval)412      return;413 414    const parser::OmpClauseList *beginClauseList = nullptr;415    const parser::OmpClauseList *endClauseList = nullptr;416    common::visit(417        [&](const auto &construct) {418          using Type = llvm::remove_cvref_t<decltype(construct)>;419          if constexpr (std::is_same_v<Type, parser::OmpBlockConstruct> ||420                        std::is_same_v<Type, parser::OpenMPLoopConstruct>) {421            beginClauseList = &construct.BeginDir().Clauses();422            if (auto &endSpec = construct.EndDir())423              endClauseList = &endSpec->Clauses();424          }425        },426        ompEval->u);427 428    assert(beginClauseList && "expected begin directive");429    clauses.append(makeClauses(*beginClauseList, semaCtx));430 431    if (endClauseList)432      clauses.append(makeClauses(*endClauseList, semaCtx));433  };434 435  // Return the directive that is immediately nested inside of the given436  // `parent` evaluation, if it is its only non-end-statement nested evaluation437  // and it represents an OpenMP construct.438  auto extractOnlyOmpNestedDir = [](lower::pft::Evaluation &parent)439      -> std::optional<llvm::omp::Directive> {440    if (!parent.hasNestedEvaluations())441      return std::nullopt;442 443    llvm::omp::Directive dir;444    auto &nested = parent.getFirstNestedEvaluation();445    if (const auto *ompEval = nested.getIf<parser::OpenMPConstruct>())446      dir = parser::omp::GetOmpDirectiveName(*ompEval).v;447    else448      return std::nullopt;449 450    for (auto &sibling : parent.getNestedEvaluations())451      if (&sibling != &nested && !sibling.isEndStmt())452        return std::nullopt;453 454    return dir;455  };456 457  // Process the given evaluation assuming it's part of a 'target' construct or458  // captured by one, and store results in the global `hostEvalInfo`.459  std::function<void(lower::pft::Evaluation &, const List<Clause> &)>460      processEval;461  processEval = [&](lower::pft::Evaluation &eval, const List<Clause> &clauses) {462    using namespace llvm::omp;463    ClauseProcessor cp(converter, semaCtx, clauses);464 465    // Call `processEval` recursively with the immediately nested evaluation and466    // its corresponding clauses if there is a single nested evaluation467    // representing an OpenMP directive that passes the given test.468    auto processSingleNestedIf = [&](llvm::function_ref<bool(Directive)> test) {469      std::optional<Directive> nestedDir = extractOnlyOmpNestedDir(eval);470      if (!nestedDir || !test(*nestedDir))471        return;472 473      lower::pft::Evaluation &nestedEval = eval.getFirstNestedEvaluation();474      List<lower::omp::Clause> nestedClauses;475      extractClauses(nestedEval, nestedClauses);476      processEval(nestedEval, nestedClauses);477    };478 479    const auto *ompEval = eval.getIf<parser::OpenMPConstruct>();480    if (!ompEval)481      return;482 483    HostEvalInfo *hostInfo = getHostEvalInfoStackTop(converter);484    assert(hostInfo && "expected HOST_EVAL info structure");485 486    switch (parser::omp::GetOmpDirectiveName(*ompEval).v) {487    case OMPD_teams_distribute_parallel_do:488    case OMPD_teams_distribute_parallel_do_simd:489      cp.processThreadLimit(stmtCtx, hostInfo->ops);490      [[fallthrough]];491    case OMPD_target_teams_distribute_parallel_do:492    case OMPD_target_teams_distribute_parallel_do_simd:493      cp.processNumTeams(stmtCtx, hostInfo->ops);494      [[fallthrough]];495    case OMPD_distribute_parallel_do:496    case OMPD_distribute_parallel_do_simd:497      cp.processNumThreads(stmtCtx, hostInfo->ops);498      [[fallthrough]];499    case OMPD_distribute:500    case OMPD_distribute_simd:501      cp.processCollapse(loc, eval, hostInfo->ops, hostInfo->ops, hostInfo->iv);502      break;503 504    case OMPD_teams:505      cp.processThreadLimit(stmtCtx, hostInfo->ops);506      [[fallthrough]];507    case OMPD_target_teams:508      cp.processNumTeams(stmtCtx, hostInfo->ops);509      processSingleNestedIf([](Directive nestedDir) {510        return topDistributeSet.test(nestedDir) || topLoopSet.test(nestedDir);511      });512      break;513 514    case OMPD_teams_distribute:515    case OMPD_teams_distribute_simd:516      cp.processThreadLimit(stmtCtx, hostInfo->ops);517      [[fallthrough]];518    case OMPD_target_teams_distribute:519    case OMPD_target_teams_distribute_simd:520      cp.processCollapse(loc, eval, hostInfo->ops, hostInfo->ops, hostInfo->iv);521      cp.processNumTeams(stmtCtx, hostInfo->ops);522      break;523 524    case OMPD_teams_loop:525      cp.processThreadLimit(stmtCtx, hostInfo->ops);526      [[fallthrough]];527    case OMPD_target_teams_loop:528      cp.processNumTeams(stmtCtx, hostInfo->ops);529      [[fallthrough]];530    case OMPD_loop:531      cp.processCollapse(loc, eval, hostInfo->ops, hostInfo->ops, hostInfo->iv);532      break;533 534    case OMPD_teams_workdistribute:535      cp.processThreadLimit(stmtCtx, hostInfo->ops);536      [[fallthrough]];537    case OMPD_target_teams_workdistribute:538      cp.processNumTeams(stmtCtx, hostInfo->ops);539      break;540 541    // Standalone 'target' case.542    case OMPD_target: {543      processSingleNestedIf(544          [](Directive nestedDir) { return topTeamsSet.test(nestedDir); });545      break;546    }547    default:548      break;549    }550  };551 552  const auto *ompEval = eval.getIf<parser::OpenMPConstruct>();553  assert(ompEval &&554         llvm::omp::allTargetSet.test(555             parser::omp::GetOmpDirectiveName(*ompEval).v) &&556         "expected TARGET construct evaluation");557  (void)ompEval;558 559  // Use the whole list of clauses passed to the construct here, rather than the560  // ones only applied to omp.target.561  List<lower::omp::Clause> clauses;562  extractClauses(eval, clauses);563  processEval(eval, clauses);564}565 566static lower::pft::Evaluation *567getCollapsedLoopEval(lower::pft::Evaluation &eval, int collapseValue) {568  // Return the Evaluation of the innermost collapsed loop, or the current one569  // if there was no COLLAPSE.570  if (collapseValue == 0)571    return &eval;572 573  lower::pft::Evaluation *curEval = &eval;574  for (int i = 0; i < collapseValue; i++)575    curEval = getNestedDoConstruct(*curEval);576  return curEval;577}578 579static void genNestedEvaluations(lower::AbstractConverter &converter,580                                 lower::pft::Evaluation &eval,581                                 int collapseValue = 0) {582  lower::pft::Evaluation *curEval = getCollapsedLoopEval(eval, collapseValue);583 584  for (lower::pft::Evaluation &e : curEval->getNestedEvaluations())585    converter.genEval(e);586}587 588static fir::GlobalOp globalInitialization(lower::AbstractConverter &converter,589                                          fir::FirOpBuilder &firOpBuilder,590                                          const semantics::Symbol &sym,591                                          const lower::pft::Variable &var,592                                          mlir::Location currentLocation) {593  std::string globalName = converter.mangleName(sym);594  mlir::StringAttr linkage = firOpBuilder.createInternalLinkage();595  return Fortran::lower::defineGlobal(converter, var, globalName, linkage);596}597 598// Get the extended value for \p val by extracting additional variable599// information from \p base.600static fir::ExtendedValue getExtendedValue(fir::ExtendedValue base,601                                           mlir::Value val) {602  return base.match(603      [&](const fir::MutableBoxValue &box) -> fir::ExtendedValue {604        return fir::MutableBoxValue(val, box.nonDeferredLenParams(), {});605      },606      [&](const auto &) -> fir::ExtendedValue {607        return fir::substBase(base, val);608      });609}610 611#ifndef NDEBUG612static bool isThreadPrivate(lower::SymbolRef sym) {613  if (const auto *details = sym->detailsIf<semantics::CommonBlockDetails>()) {614    for (const auto &obj : details->objects())615      if (!obj->test(semantics::Symbol::Flag::OmpThreadprivate))616        return false;617    return true;618  }619  return sym->test(semantics::Symbol::Flag::OmpThreadprivate);620}621#endif622 623static void threadPrivatizeVars(lower::AbstractConverter &converter,624                                lower::pft::Evaluation &eval) {625  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();626  mlir::Location currentLocation = converter.getCurrentLocation();627  mlir::OpBuilder::InsertionGuard guard(firOpBuilder);628  firOpBuilder.setInsertionPointToStart(firOpBuilder.getAllocaBlock());629 630  // If the symbol corresponds to the original ThreadprivateOp, use the symbol631  // value from that operation to create one ThreadprivateOp copy operation632  // inside the parallel region.633  // In some cases, however, the symbol will correspond to the original,634  // non-threadprivate variable. This can happen, for instance, with a common635  // block, declared in a separate module, used by a parent procedure and636  // privatized in its child procedure.637  auto genThreadprivateOp = [&](lower::SymbolRef sym) -> mlir::Value {638    assert(isThreadPrivate(sym));639    mlir::Value symValue = converter.getSymbolAddress(sym);640    mlir::Operation *op = symValue.getDefiningOp();641    if (auto declOp = mlir::dyn_cast<hlfir::DeclareOp>(op))642      op = declOp.getMemref().getDefiningOp();643    if (mlir::isa<mlir::omp::ThreadprivateOp>(op))644      symValue = mlir::dyn_cast<mlir::omp::ThreadprivateOp>(op).getSymAddr();645    return mlir::omp::ThreadprivateOp::create(firOpBuilder, currentLocation,646                                              symValue.getType(), symValue);647  };648 649  llvm::SetVector<const semantics::Symbol *> threadprivateSyms;650  converter.collectSymbolSet(eval, threadprivateSyms,651                             semantics::Symbol::Flag::OmpThreadprivate,652                             /*collectSymbols=*/true,653                             /*collectHostAssociatedSymbols=*/true);654  std::set<semantics::SourceName> threadprivateSymNames;655 656  // For a COMMON block, the ThreadprivateOp is generated for itself instead of657  // its members, so only bind the value of the new copied ThreadprivateOp658  // inside the parallel region to the common block symbol only once for659  // multiple members in one COMMON block.660  llvm::SetVector<const semantics::Symbol *> commonSyms;661  for (std::size_t i = 0; i < threadprivateSyms.size(); i++) {662    const semantics::Symbol *sym = threadprivateSyms[i];663    mlir::Value symThreadprivateValue;664    // The variable may be used more than once, and each reference has one665    // symbol with the same name. Only do once for references of one variable.666    if (threadprivateSymNames.find(sym->name()) != threadprivateSymNames.end())667      continue;668    threadprivateSymNames.insert(sym->name());669    if (const semantics::Symbol *common =670            semantics::FindCommonBlockContaining(sym->GetUltimate())) {671      mlir::Value commonThreadprivateValue;672      if (commonSyms.contains(common)) {673        commonThreadprivateValue = converter.getSymbolAddress(*common);674      } else {675        commonThreadprivateValue = genThreadprivateOp(*common);676        converter.bindSymbol(*common, commonThreadprivateValue);677        commonSyms.insert(common);678      }679      symThreadprivateValue = lower::genCommonBlockMember(680          converter, currentLocation, sym->GetUltimate(),681          commonThreadprivateValue, common->size());682    } else {683      symThreadprivateValue = genThreadprivateOp(*sym);684    }685 686    fir::ExtendedValue sexv = converter.getSymbolExtendedValue(*sym);687    fir::ExtendedValue symThreadprivateExv =688        getExtendedValue(sexv, symThreadprivateValue);689    converter.bindSymbol(*sym, symThreadprivateExv);690  }691}692 693static mlir::Operation *setLoopVar(lower::AbstractConverter &converter,694                                   mlir::Location loc, mlir::Value indexVal,695                                   const semantics::Symbol *sym) {696  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();697 698  mlir::OpBuilder::InsertPoint insPt = firOpBuilder.saveInsertionPoint();699  firOpBuilder.setInsertionPointToStart(firOpBuilder.getAllocaBlock());700  mlir::Type tempTy = converter.genType(*sym);701  firOpBuilder.restoreInsertionPoint(insPt);702 703  mlir::Value cvtVal = firOpBuilder.createConvert(loc, tempTy, indexVal);704  hlfir::Entity lhs{converter.getSymbolAddress(*sym)};705 706  lhs = hlfir::derefPointersAndAllocatables(loc, firOpBuilder, lhs);707 708  mlir::Operation *storeOp =709      hlfir::AssignOp::create(firOpBuilder, loc, cvtVal, lhs);710  return storeOp;711}712 713static mlir::Operation *714createAndSetPrivatizedLoopVar(lower::AbstractConverter &converter,715                              mlir::Location loc, mlir::Value indexVal,716                              const semantics::Symbol *sym) {717  assert(converter.isPresentShallowLookup(*sym) &&718         "Expected symbol to be in symbol table.");719  return setLoopVar(converter, loc, indexVal, sym);720}721 722// This helper function implements the functionality of "promoting" non-CPTR723// arguments of use_device_ptr to use_device_addr arguments (automagic724// conversion of use_device_ptr -> use_device_addr in these cases). The way we725// do so currently is through the shuffling of operands from the726// devicePtrOperands to deviceAddrOperands, as well as the types, locations and727// symbols.728//729// This effectively implements some deprecated OpenMP functionality that some730// legacy applications unfortunately depend on (deprecated in specification731// version 5.2):732//733// "If a list item in a use_device_ptr clause is not of type C_PTR, the behavior734//  is as if the list item appeared in a use_device_addr clause. Support for735//  such list items in a use_device_ptr clause is deprecated."736static void promoteNonCPtrUseDevicePtrArgsToUseDeviceAddr(737    llvm::SmallVectorImpl<mlir::Value> &useDeviceAddrVars,738    llvm::SmallVectorImpl<const semantics::Symbol *> &useDeviceAddrSyms,739    llvm::SmallVectorImpl<mlir::Value> &useDevicePtrVars,740    llvm::SmallVectorImpl<const semantics::Symbol *> &useDevicePtrSyms) {741  // Iterate over our use_device_ptr list and shift all non-cptr arguments into742  // use_device_addr.743  auto *varIt = useDevicePtrVars.begin();744  auto *symIt = useDevicePtrSyms.begin();745  while (varIt != useDevicePtrVars.end()) {746    if (fir::isa_builtin_cptr_type(fir::unwrapRefType(varIt->getType()))) {747      ++varIt;748      ++symIt;749      continue;750    }751 752    useDeviceAddrVars.push_back(*varIt);753    useDeviceAddrSyms.push_back(*symIt);754 755    varIt = useDevicePtrVars.erase(varIt);756    symIt = useDevicePtrSyms.erase(symIt);757  }758}759 760/// Extract the list of function and variable symbols affected by the given761/// 'declare target' directive and return the intended device type for them.762static void getDeclareTargetInfo(763    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,764    lower::pft::Evaluation &eval,765    const parser::OpenMPDeclareTargetConstruct &construct,766    mlir::omp::DeclareTargetOperands &clauseOps,767    llvm::SmallVectorImpl<DeclareTargetCaptureInfo> &symbolAndClause) {768 769  if (!construct.v.Arguments().v.empty()) {770    ObjectList objects{makeObjects(construct.v.Arguments(), semaCtx)};771    // Case: declare target(func, var1, var2)772    gatherFuncAndVarSyms(objects, mlir::omp::DeclareTargetCaptureClause::to,773                         symbolAndClause, /*automap=*/false);774  } else {775    List<Clause> clauses = makeClauses(construct.v.Clauses(), semaCtx);776    if (clauses.empty()) {777      Fortran::lower::pft::FunctionLikeUnit *owningProc =778          eval.getOwningProcedure();779      if (owningProc && (!owningProc->isMainProgram() ||780                         owningProc->getMainProgramSymbol())) {781        // Case: declare target, implicit capture of function782        symbolAndClause.emplace_back(mlir::omp::DeclareTargetCaptureClause::to,783                                     owningProc->getSubprogramSymbol());784      }785    }786 787    ClauseProcessor cp(converter, semaCtx, clauses);788    cp.processDeviceType(clauseOps);789    cp.processEnter(symbolAndClause);790    cp.processLink(symbolAndClause);791    cp.processTo(symbolAndClause);792 793    cp.processTODO<clause::Indirect>(converter.getCurrentLocation(),794                                     llvm::omp::Directive::OMPD_declare_target);795  }796}797 798static void collectDeferredDeclareTargets(799    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,800    lower::pft::Evaluation &eval,801    const parser::OpenMPDeclareTargetConstruct &declareTargetConstruct,802    llvm::SmallVectorImpl<lower::OMPDeferredDeclareTargetInfo>803        &deferredDeclareTarget) {804  mlir::omp::DeclareTargetOperands clauseOps;805  llvm::SmallVector<DeclareTargetCaptureInfo> symbolAndClause;806  getDeclareTargetInfo(converter, semaCtx, eval, declareTargetConstruct,807                       clauseOps, symbolAndClause);808  // Return the device type only if at least one of the targets for the809  // directive is a function or subroutine810  mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();811 812  for (const DeclareTargetCaptureInfo &symClause : symbolAndClause) {813    mlir::Operation *op =814        mod.lookupSymbol(converter.mangleName(symClause.symbol));815 816    if (!op) {817      deferredDeclareTarget.push_back({symClause.clause, clauseOps.deviceType,818                                       symClause.automap, symClause.symbol});819    }820  }821}822 823static std::optional<mlir::omp::DeclareTargetDeviceType>824getDeclareTargetFunctionDevice(825    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,826    lower::pft::Evaluation &eval,827    const parser::OpenMPDeclareTargetConstruct &declareTargetConstruct) {828  mlir::omp::DeclareTargetOperands clauseOps;829  llvm::SmallVector<DeclareTargetCaptureInfo> symbolAndClause;830  getDeclareTargetInfo(converter, semaCtx, eval, declareTargetConstruct,831                       clauseOps, symbolAndClause);832 833  // Return the device type only if at least one of the targets for the834  // directive is a function or subroutine835  mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();836  for (const DeclareTargetCaptureInfo &symClause : symbolAndClause) {837    mlir::Operation *op =838        mod.lookupSymbol(converter.mangleName(symClause.symbol));839 840    if (mlir::isa_and_nonnull<mlir::func::FuncOp>(op))841      return clauseOps.deviceType;842  }843 844  return std::nullopt;845}846 847/// Set up the entry block of the given `omp.loop_nest` operation, adding a848/// block argument for each loop induction variable and allocating and849/// initializing a private value to hold each of them.850///851/// This function can also bind the symbols of any variables that should match852/// block arguments on parent loop wrapper operations attached to the same853/// loop. This allows the introduction of any necessary `hlfir.declare`854/// operations inside of the entry block of the `omp.loop_nest` operation and855/// not directly under any of the wrappers, which would invalidate them.856///857/// \param [in]          op - the loop nest operation.858/// \param [in]   converter - PFT to MLIR conversion interface.859/// \param [in]         loc - location.860/// \param [in]        args - symbols of induction variables.861/// \param [in] wrapperArgs - list of parent loop wrappers and their associated862///                           entry block arguments.863static void genLoopVars(864    mlir::Operation *op, lower::AbstractConverter &converter,865    mlir::Location &loc, llvm::ArrayRef<const semantics::Symbol *> args,866    llvm::ArrayRef<867        std::pair<mlir::omp::BlockArgOpenMPOpInterface, const EntryBlockArgs &>>868        wrapperArgs = {}) {869  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();870  auto &region = op->getRegion(0);871 872  std::size_t loopVarTypeSize = 0;873  for (const semantics::Symbol *arg : args)874    loopVarTypeSize = std::max(loopVarTypeSize, arg->GetUltimate().size());875  mlir::Type loopVarType = getLoopVarType(converter, loopVarTypeSize);876  llvm::SmallVector<mlir::Type> tiv(args.size(), loopVarType);877  llvm::SmallVector<mlir::Location> locs(args.size(), loc);878  firOpBuilder.createBlock(&region, {}, tiv, locs);879 880  // Update nested wrapper operands if parent wrappers have mapped these values881  // to block arguments.882  //883  // Binding these values earlier would take care of this, but we cannot rely on884  // that approach because binding in between the creation of a wrapper and the885  // next one would result in 'hlfir.declare' operations being introduced inside886  // of a wrapper, which is illegal.887  mlir::IRMapping mapper;888  for (auto [argGeneratingOp, blockArgs] : wrapperArgs) {889    for (mlir::OpOperand &operand : argGeneratingOp->getOpOperands())890      operand.set(mapper.lookupOrDefault(operand.get()));891 892    for (const auto [arg, var] : llvm::zip_equal(893             argGeneratingOp->getRegion(0).getArguments(), blockArgs.getVars()))894      mapper.map(var, arg);895  }896 897  // Bind the entry block arguments of parent wrappers to the corresponding898  // symbols.899  for (auto [argGeneratingOp, blockArgs] : wrapperArgs)900    bindEntryBlockArgs(converter, argGeneratingOp, blockArgs);901 902  // The argument is not currently in memory, so make a temporary for the903  // argument, and store it there, then bind that location to the argument.904  mlir::Operation *storeOp = nullptr;905  for (auto [argIndex, argSymbol] : llvm::enumerate(args)) {906    mlir::Value indexVal = fir::getBase(region.front().getArgument(argIndex));907    storeOp =908        createAndSetPrivatizedLoopVar(converter, loc, indexVal, argSymbol);909  }910  firOpBuilder.setInsertionPointAfter(storeOp);911}912 913static clause::Defaultmap::ImplicitBehavior914getDefaultmapIfPresent(const DefaultMapsTy &defaultMaps, mlir::Type varType) {915  using DefMap = clause::Defaultmap;916 917  if (defaultMaps.empty())918    return DefMap::ImplicitBehavior::Default;919 920  if (llvm::is_contained(defaultMaps, DefMap::VariableCategory::All))921    return defaultMaps.at(DefMap::VariableCategory::All);922 923  // NOTE: Unsure if complex and/or vector falls into a scalar type924  // or aggregate, but the current default implicit behaviour is to925  // treat them as such (c_ptr has its own behaviour, so perhaps926  // being lumped in as a scalar isn't the right thing).927  if ((fir::isa_trivial(varType) || fir::isa_char(varType) ||928       fir::isa_builtin_cptr_type(varType)) &&929      llvm::is_contained(defaultMaps, DefMap::VariableCategory::Scalar))930    return defaultMaps.at(DefMap::VariableCategory::Scalar);931 932  if (fir::isPointerType(varType) &&933      llvm::is_contained(defaultMaps, DefMap::VariableCategory::Pointer))934    return defaultMaps.at(DefMap::VariableCategory::Pointer);935 936  if (fir::isAllocatableType(varType) &&937      llvm::is_contained(defaultMaps, DefMap::VariableCategory::Allocatable))938    return defaultMaps.at(DefMap::VariableCategory::Allocatable);939 940  if (fir::isa_aggregate(varType) &&941      llvm::is_contained(defaultMaps, DefMap::VariableCategory::Aggregate))942    return defaultMaps.at(DefMap::VariableCategory::Aggregate);943 944  return DefMap::ImplicitBehavior::Default;945}946 947static std::pair<mlir::omp::ClauseMapFlags, mlir::omp::VariableCaptureKind>948getImplicitMapTypeAndKind(fir::FirOpBuilder &firOpBuilder,949                          lower::AbstractConverter &converter,950                          const DefaultMapsTy &defaultMaps, mlir::Type varType,951                          mlir::Location loc, const semantics::Symbol &sym) {952  using DefMap = clause::Defaultmap;953  // Check if a value of type `type` can be passed to the kernel by value.954  // All kernel parameters are of pointer type, so if the value can be955  // represented inside of a pointer, then it can be passed by value.956  auto isLiteralType = [&](mlir::Type type) {957    const mlir::DataLayout &dl = firOpBuilder.getDataLayout();958    mlir::Type ptrTy =959        mlir::LLVM::LLVMPointerType::get(&converter.getMLIRContext());960    uint64_t ptrSize = dl.getTypeSize(ptrTy);961    uint64_t ptrAlign = dl.getTypePreferredAlignment(ptrTy);962 963    auto [size, align] = fir::getTypeSizeAndAlignmentOrCrash(964        loc, type, dl, converter.getKindMap());965    return size <= ptrSize && align <= ptrAlign;966  };967 968  mlir::omp::ClauseMapFlags mapFlag = mlir::omp::ClauseMapFlags::implicit;969 970  auto implicitBehaviour = getDefaultmapIfPresent(defaultMaps, varType);971  if (implicitBehaviour == DefMap::ImplicitBehavior::Default) {972    mlir::omp::VariableCaptureKind captureKind =973        mlir::omp::VariableCaptureKind::ByRef;974 975    // If a variable is specified in declare target link and if device976    // type is not specified as `nohost`, it needs to be mapped tofrom977    mlir::ModuleOp mod = firOpBuilder.getModule();978    mlir::Operation *op = mod.lookupSymbol(converter.mangleName(sym));979    auto declareTargetOp =980        llvm::dyn_cast_if_present<mlir::omp::DeclareTargetInterface>(op);981    if (declareTargetOp && declareTargetOp.isDeclareTarget()) {982      if (declareTargetOp.getDeclareTargetCaptureClause() ==983              mlir::omp::DeclareTargetCaptureClause::link &&984          declareTargetOp.getDeclareTargetDeviceType() !=985              mlir::omp::DeclareTargetDeviceType::nohost) {986        mapFlag |= mlir::omp::ClauseMapFlags::to;987        mapFlag |= mlir::omp::ClauseMapFlags::from;988      }989    } else if (fir::isa_trivial(varType) || fir::isa_char(varType)) {990      // Scalars behave as if they were "firstprivate".991      // TODO: Handle objects that are shared/lastprivate or were listed992      // in an in_reduction clause.993      if (isLiteralType(varType)) {994        captureKind = mlir::omp::VariableCaptureKind::ByCopy;995      } else {996        mapFlag |= mlir::omp::ClauseMapFlags::to;997      }998    } else if (!fir::isa_builtin_cptr_type(varType)) {999      mapFlag |= mlir::omp::ClauseMapFlags::to;1000      mapFlag |= mlir::omp::ClauseMapFlags::from;1001    }1002    return std::make_pair(mapFlag, captureKind);1003  }1004 1005  switch (implicitBehaviour) {1006  case DefMap::ImplicitBehavior::Alloc:1007    return std::make_pair(mlir::omp::ClauseMapFlags::storage,1008                          mlir::omp::VariableCaptureKind::ByRef);1009    break;1010  case DefMap::ImplicitBehavior::Firstprivate:1011    TODO(loc, "Firstprivate is currently unsupported defaultmap behaviour");1012    break;1013  case DefMap::ImplicitBehavior::From:1014    return std::make_pair(mapFlag |= mlir::omp::ClauseMapFlags::from,1015                          mlir::omp::VariableCaptureKind::ByRef);1016    break;1017  case DefMap::ImplicitBehavior::Present:1018    return std::make_pair(mapFlag |= mlir::omp::ClauseMapFlags::present,1019                          mlir::omp::VariableCaptureKind::ByRef);1020    break;1021  case DefMap::ImplicitBehavior::To:1022    return std::make_pair(mapFlag |= mlir::omp::ClauseMapFlags::to,1023                          (fir::isa_trivial(varType) || fir::isa_char(varType))1024                              ? mlir::omp::VariableCaptureKind::ByCopy1025                              : mlir::omp::VariableCaptureKind::ByRef);1026    break;1027  case DefMap::ImplicitBehavior::Tofrom:1028    return std::make_pair(mapFlag |= mlir::omp::ClauseMapFlags::from |1029                                     mlir::omp::ClauseMapFlags::to,1030                          mlir::omp::VariableCaptureKind::ByRef);1031    break;1032  case DefMap::ImplicitBehavior::Default:1033  case DefMap::ImplicitBehavior::None:1034    llvm_unreachable(1035        "Implicit None and Default behaviour should have been handled earlier");1036    break;1037  }1038 1039  return std::make_pair(mapFlag |= mlir::omp::ClauseMapFlags::from |1040                                   mlir::omp::ClauseMapFlags::to,1041                        mlir::omp::VariableCaptureKind::ByRef);1042}1043 1044static void1045markDeclareTarget(mlir::Operation *op, lower::AbstractConverter &converter,1046                  mlir::omp::DeclareTargetCaptureClause captureClause,1047                  mlir::omp::DeclareTargetDeviceType deviceType, bool automap) {1048  // TODO: Add support for program local variables with declare target applied1049  auto declareTargetOp = llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(op);1050  if (!declareTargetOp)1051    fir::emitFatalError(1052        converter.getCurrentLocation(),1053        "Attempt to apply declare target on unsupported operation");1054 1055  // The function or global already has a declare target applied to it, very1056  // likely through implicit capture (usage in another declare target1057  // function/subroutine). It should be marked as any if it has been assigned1058  // both host and nohost, else we skip, as there is no change1059  if (declareTargetOp.isDeclareTarget()) {1060    if (declareTargetOp.getDeclareTargetDeviceType() != deviceType)1061      declareTargetOp.setDeclareTarget(mlir::omp::DeclareTargetDeviceType::any,1062                                       captureClause, automap);1063    return;1064  }1065 1066  declareTargetOp.setDeclareTarget(deviceType, captureClause, automap);1067}1068 1069//===----------------------------------------------------------------------===//1070// Op body generation helper structures and functions1071//===----------------------------------------------------------------------===//1072 1073struct OpWithBodyGenInfo {1074  /// A type for a code-gen callback function. This takes as argument the op for1075  /// which the code is being generated and returns the arguments of the op's1076  /// region.1077  using GenOMPRegionEntryCBFn =1078      std::function<llvm::SmallVector<const semantics::Symbol *>(1079          mlir::Operation *)>;1080 1081  OpWithBodyGenInfo(lower::AbstractConverter &converter,1082                    lower::SymMap &symTable,1083                    semantics::SemanticsContext &semaCtx, mlir::Location loc,1084                    lower::pft::Evaluation &eval, llvm::omp::Directive dir)1085      : converter(converter), symTable(symTable), semaCtx(semaCtx), loc(loc),1086        eval(eval), dir(dir) {}1087 1088  OpWithBodyGenInfo &setClauses(const List<Clause> *value) {1089    clauses = value;1090    return *this;1091  }1092 1093  OpWithBodyGenInfo &setDataSharingProcessor(DataSharingProcessor *value) {1094    dsp = value;1095    return *this;1096  }1097 1098  OpWithBodyGenInfo &setEntryBlockArgs(const EntryBlockArgs *value) {1099    blockArgs = value;1100    return *this;1101  }1102 1103  OpWithBodyGenInfo &setGenRegionEntryCb(GenOMPRegionEntryCBFn value) {1104    genRegionEntryCB = value;1105    return *this;1106  }1107 1108  OpWithBodyGenInfo &setGenSkeletonOnly(bool value) {1109    genSkeletonOnly = value;1110    return *this;1111  }1112 1113  OpWithBodyGenInfo &setPrivatize(bool value) {1114    privatize = value;1115    return *this;1116  }1117 1118  /// [inout] converter to use for the clauses.1119  lower::AbstractConverter &converter;1120  /// [in] Symbol table1121  lower::SymMap &symTable;1122  /// [in] Semantics context1123  semantics::SemanticsContext &semaCtx;1124  /// [in] location in source code.1125  mlir::Location loc;1126  /// [in] current PFT node/evaluation.1127  lower::pft::Evaluation &eval;1128  /// [in] leaf directive for which to generate the op body.1129  llvm::omp::Directive dir;1130  /// [in] list of clauses to process.1131  const List<Clause> *clauses = nullptr;1132  /// [in] if provided, processes the construct's data-sharing attributes.1133  DataSharingProcessor *dsp = nullptr;1134  /// [in] if provided, it is used to create the op's region entry block. It is1135  /// overriden when a \see genRegionEntryCB is provided. This is only valid for1136  /// operations implementing the \see mlir::omp::BlockArgOpenMPOpInterface.1137  const EntryBlockArgs *blockArgs = nullptr;1138  /// [in] if provided, it overrides the default op's region entry block1139  /// creation.1140  GenOMPRegionEntryCBFn genRegionEntryCB = nullptr;1141  /// [in] if set to `true`, skip generating nested evaluations and dispatching1142  /// any further leaf constructs.1143  bool genSkeletonOnly = false;1144  /// [in] enables handling of privatized variable unless set to `false`.1145  bool privatize = true;1146};1147 1148/// Create the body (block) for an OpenMP Operation.1149///1150/// \param [in]   op  - the operation the body belongs to.1151/// \param [in] info  - options controlling code-gen for the construction.1152/// \param [in] queue - work queue with nested constructs.1153/// \param [in] item  - item in the queue to generate body for.1154static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info,1155                           const ConstructQueue &queue,1156                           ConstructQueue::const_iterator item) {1157  fir::FirOpBuilder &firOpBuilder = info.converter.getFirOpBuilder();1158 1159  auto insertMarker = [](fir::FirOpBuilder &builder) {1160    mlir::Value undef = fir::UndefOp::create(builder, builder.getUnknownLoc(),1161                                             builder.getIndexType());1162    return undef.getDefiningOp();1163  };1164 1165  // Create the entry block for the region and collect its arguments for use1166  // within the region. The entry block will be created as follows:1167  //   - By default, it will be empty and have no arguments.1168  //   - Operations implementing the omp::BlockArgOpenMPOpInterface can set the1169  //     `info.blockArgs` pointer so that block arguments will be those1170  //     corresponding to entry block argument-generating clauses. Binding of1171  //     Fortran symbols to the new MLIR values is done automatically.1172  //   - If the `info.genRegionEntryCB` callback is set, it takes precedence and1173  //     allows callers to manually create the entry block with its intended1174  //     list of arguments and to bind these arguments to their corresponding1175  //     Fortran symbols. This is used for e.g. loop induction variables.1176  auto regionArgs = [&]() -> llvm::SmallVector<const semantics::Symbol *> {1177    if (info.genRegionEntryCB)1178      return info.genRegionEntryCB(&op);1179 1180    if (info.blockArgs) {1181      genEntryBlock(firOpBuilder, *info.blockArgs, op.getRegion(0));1182      bindEntryBlockArgs(info.converter,1183                         llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(op),1184                         *info.blockArgs);1185      return llvm::to_vector(info.blockArgs->getSyms());1186    }1187 1188    firOpBuilder.createBlock(&op.getRegion(0));1189    return {};1190  }();1191 1192  // Mark the earliest insertion point.1193  mlir::Operation *marker = insertMarker(firOpBuilder);1194 1195  // If it is an unstructured region, create empty blocks for all evaluations.1196  if (lower::omp::isLastItemInQueue(item, queue) &&1197      info.eval.lowerAsUnstructured()) {1198    lower::createEmptyRegionBlocks<mlir::omp::TerminatorOp, mlir::omp::YieldOp>(1199        firOpBuilder, info.eval.getNestedEvaluations());1200  }1201 1202  // Start with privatization, so that the lowering of the nested1203  // code will use the right symbols.1204  bool isLoop = llvm::omp::getDirectiveAssociation(info.dir) ==1205                llvm::omp::Association::LoopNest;1206  bool privatize = info.clauses && info.privatize;1207 1208  firOpBuilder.setInsertionPoint(marker);1209  std::optional<DataSharingProcessor> tempDsp;1210  if (privatize && !info.dsp) {1211    tempDsp.emplace(info.converter, info.semaCtx, *info.clauses, info.eval,1212                    Fortran::lower::omp::isLastItemInQueue(item, queue),1213                    /*useDelayedPrivatization=*/false, info.symTable);1214    tempDsp->processStep1();1215  }1216 1217  if (info.dir == llvm::omp::Directive::OMPD_parallel) {1218    threadPrivatizeVars(info.converter, info.eval);1219    if (info.clauses) {1220      firOpBuilder.setInsertionPoint(marker);1221      ClauseProcessor(info.converter, info.semaCtx, *info.clauses)1222          .processCopyin();1223    }1224  }1225 1226  if (!info.genSkeletonOnly) {1227    if (ConstructQueue::const_iterator next = std::next(item);1228        next != queue.end()) {1229      genOMPDispatch(info.converter, info.symTable, info.semaCtx, info.eval,1230                     info.loc, queue, next);1231    } else {1232      // genFIR(Evaluation&) tries to patch up unterminated blocks, causing1233      // a lot of complications for our approach if the terminator generation1234      // is delayed past this point. Insert a temporary terminator here, then1235      // delete it.1236      firOpBuilder.setInsertionPointToEnd(&op.getRegion(0).back());1237      auto *temp = lower::genOpenMPTerminator(firOpBuilder, &op, info.loc);1238      firOpBuilder.setInsertionPointAfter(marker);1239      genNestedEvaluations(info.converter, info.eval);1240      temp->erase();1241    }1242  }1243 1244  // Get or create a unique exiting block from the given region, or1245  // return nullptr if there is no exiting block.1246  auto getUniqueExit = [&](mlir::Region &region) -> mlir::Block * {1247    // Find the blocks where the OMP terminator should go. In simple cases1248    // it is the single block in the operation's region. When the region1249    // is more complicated, especially with unstructured control flow, there1250    // may be multiple blocks, and some of them may have non-OMP terminators1251    // resulting from lowering of the code contained within the operation.1252    // All the remaining blocks are potential exit points from the op's region.1253    //1254    // Explicit control flow cannot exit any OpenMP region (other than via1255    // STOP), and that is enforced by semantic checks prior to lowering. STOP1256    // statements are lowered to a function call.1257 1258    // Collect unterminated blocks.1259    llvm::SmallVector<mlir::Block *> exits;1260    for (mlir::Block &b : region) {1261      if (b.empty() || !b.back().hasTrait<mlir::OpTrait::IsTerminator>())1262        exits.push_back(&b);1263    }1264 1265    if (exits.empty())1266      return nullptr;1267    // If there already is a unique exiting block, do not create another one.1268    // Additionally, some ops (e.g. omp.sections) require only 1 block in1269    // its region.1270    if (exits.size() == 1)1271      return exits[0];1272    mlir::Block *exit = firOpBuilder.createBlock(&region);1273    for (mlir::Block *b : exits) {1274      firOpBuilder.setInsertionPointToEnd(b);1275      mlir::cf::BranchOp::create(firOpBuilder, info.loc, exit);1276    }1277    return exit;1278  };1279 1280  if (auto *exitBlock = getUniqueExit(op.getRegion(0))) {1281    firOpBuilder.setInsertionPointToEnd(exitBlock);1282    auto *term = lower::genOpenMPTerminator(firOpBuilder, &op, info.loc);1283    // Only insert lastprivate code when there actually is an exit block.1284    // Such a block may not exist if the nested code produced an infinite1285    // loop (this may not make sense in production code, but a user could1286    // write that and we should handle it).1287    firOpBuilder.setInsertionPoint(term);1288    if (privatize) {1289      // DataSharingProcessor::processStep2() may create operations before/after1290      // the one passed as argument. We need to treat loop wrappers and their1291      // nested loop as a unit, so we need to pass the bottom level wrapper (if1292      // present). Otherwise, these operations will be inserted within a1293      // wrapper region.1294      mlir::Operation *privatizationBottomLevelOp = &op;1295      if (auto loopNest = llvm::dyn_cast<mlir::omp::LoopNestOp>(op)) {1296        llvm::SmallVector<mlir::omp::LoopWrapperInterface> wrappers;1297        loopNest.gatherWrappers(wrappers);1298        if (!wrappers.empty())1299          privatizationBottomLevelOp = &*wrappers.front();1300      }1301 1302      if (!info.dsp) {1303        assert(tempDsp.has_value());1304        tempDsp->processStep2(privatizationBottomLevelOp, isLoop);1305      } else {1306        if (isLoop && regionArgs.size() > 0) {1307          for (const auto &regionArg : regionArgs) {1308            info.dsp->pushLoopIV(info.converter.getSymbolAddress(*regionArg));1309          }1310        }1311        info.dsp->processStep2(privatizationBottomLevelOp, isLoop);1312      }1313    }1314  }1315 1316  firOpBuilder.setInsertionPointAfter(marker);1317  marker->erase();1318}1319 1320static void genBodyOfTargetDataOp(1321    lower::AbstractConverter &converter, lower::SymMap &symTable,1322    semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,1323    mlir::omp::TargetDataOp &dataOp, const EntryBlockArgs &args,1324    const mlir::Location &currentLocation, const ConstructQueue &queue,1325    ConstructQueue::const_iterator item) {1326  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();1327 1328  genEntryBlock(firOpBuilder, args, dataOp.getRegion());1329  bindEntryBlockArgs(converter, dataOp, args);1330 1331  // Insert dummy instruction to remember the insertion position. The1332  // marker will be deleted by clean up passes since there are no uses.1333  // Remembering the position for further insertion is important since1334  // there are hlfir.declares inserted above while setting block arguments1335  // and new code from the body should be inserted after that.1336  mlir::Value undefMarker = fir::UndefOp::create(firOpBuilder, dataOp.getLoc(),1337                                                 firOpBuilder.getIndexType());1338 1339  // Create blocks for unstructured regions. This has to be done since1340  // blocks are initially allocated with the function as the parent region.1341  if (eval.lowerAsUnstructured()) {1342    lower::createEmptyRegionBlocks<mlir::omp::TerminatorOp, mlir::omp::YieldOp>(1343        firOpBuilder, eval.getNestedEvaluations());1344  }1345 1346  mlir::omp::TerminatorOp::create(firOpBuilder, currentLocation);1347 1348  // Set the insertion point after the marker.1349  firOpBuilder.setInsertionPointAfter(undefMarker.getDefiningOp());1350 1351  if (ConstructQueue::const_iterator next = std::next(item);1352      next != queue.end()) {1353    genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,1354                   next);1355  } else {1356    genNestedEvaluations(converter, eval);1357  }1358}1359 1360// This generates intermediate common block member accesses within a region1361// and then rebinds the members symbol to the intermediate accessors we have1362// generated so that subsequent code generation will utilise these instead.1363//1364// When the scope changes, the bindings to the intermediate accessors should1365// be dropped in place of the original symbol bindings.1366//1367// This is for utilisation with TargetOp.1368static void genIntermediateCommonBlockAccessors(1369    Fortran::lower::AbstractConverter &converter,1370    const mlir::Location &currentLocation,1371    llvm::ArrayRef<mlir::BlockArgument> mapBlockArgs,1372    llvm::ArrayRef<const Fortran::semantics::Symbol *> mapSyms) {1373  // Iterate over the symbol list, which will be shorter than the list of1374  // arguments if new entry block arguments were introduced to implicitly map1375  // outside values used by the bounds cloned into the target region. In that1376  // case, the additional block arguments do not need processing here.1377  for (auto [mapSym, mapArg] : llvm::zip_first(mapSyms, mapBlockArgs)) {1378    auto *details = mapSym->detailsIf<Fortran::semantics::CommonBlockDetails>();1379    if (!details)1380      continue;1381 1382    for (auto obj : details->objects()) {1383      auto targetCBMemberBind = Fortran::lower::genCommonBlockMember(1384          converter, currentLocation, *obj, mapArg, mapSym->size());1385      fir::ExtendedValue sexv = converter.getSymbolExtendedValue(*obj);1386      fir::ExtendedValue targetCBExv =1387          getExtendedValue(sexv, targetCBMemberBind);1388      converter.bindSymbol(*obj, targetCBExv);1389    }1390  }1391}1392 1393// This functions creates a block for the body of the targetOp's region. It adds1394// all the symbols present in mapSymbols as block arguments to this block.1395static void genBodyOfTargetOp(1396    lower::AbstractConverter &converter, lower::SymMap &symTable,1397    semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,1398    mlir::omp::TargetOp &targetOp, const EntryBlockArgs &args,1399    const mlir::Location &currentLocation, const ConstructQueue &queue,1400    ConstructQueue::const_iterator item, DataSharingProcessor &dsp) {1401  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();1402  auto argIface = llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(*targetOp);1403 1404  mlir::Region &region = targetOp.getRegion();1405  genEntryBlock(firOpBuilder, args, region);1406  bindEntryBlockArgs(converter, targetOp, args);1407  if (HostEvalInfo *hostEvalInfo = getHostEvalInfoStackTop(converter))1408    hostEvalInfo->bindOperands(argIface.getHostEvalBlockArgs());1409 1410  // Check if cloning the bounds introduced any dependency on the outer region.1411  // If so, then either clone them as well if they are MemoryEffectFree, or else1412  // copy them to a new temporary and add them to the map and block_argument1413  // lists and replace their uses with the new temporary.1414  cloneOrMapRegionOutsiders(firOpBuilder, targetOp);1415 1416  // Insert dummy instruction to remember the insertion position. The1417  // marker will be deleted since there are not uses.1418  // In the HLFIR flow there are hlfir.declares inserted above while1419  // setting block arguments.1420  mlir::Value undefMarker = fir::UndefOp::create(1421      firOpBuilder, targetOp.getLoc(), firOpBuilder.getIndexType());1422 1423  // Create blocks for unstructured regions. This has to be done since1424  // blocks are initially allocated with the function as the parent region.1425  if (lower::omp::isLastItemInQueue(item, queue) &&1426      eval.lowerAsUnstructured()) {1427    lower::createEmptyRegionBlocks<mlir::omp::TerminatorOp, mlir::omp::YieldOp>(1428        firOpBuilder, eval.getNestedEvaluations());1429  }1430 1431  mlir::omp::TerminatorOp::create(firOpBuilder, currentLocation);1432 1433  // Create the insertion point after the marker.1434  firOpBuilder.setInsertionPointAfter(undefMarker.getDefiningOp());1435 1436  // If we map a common block using it's symbol e.g. map(tofrom: /common_block/)1437  // and accessing its members within the target region, there is a large1438  // chance we will end up with uses external to the region accessing the common1439  // resolve these, we do so by generating new common block member accesses1440  // within the region, binding them to the member symbol for the scope of the1441  // region so that subsequent code generation within the region will utilise1442  // our new member accesses we have created.1443  genIntermediateCommonBlockAccessors(1444      converter, currentLocation, argIface.getMapBlockArgs(), args.map.syms);1445 1446  if (ConstructQueue::const_iterator next = std::next(item);1447      next != queue.end()) {1448    genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,1449                   next);1450  } else {1451    genNestedEvaluations(converter, eval);1452  }1453 1454  dsp.processStep2(targetOp, /*isLoop=*/false);1455}1456 1457template <typename OpTy, typename... Args>1458static OpTy genOpWithBody(const OpWithBodyGenInfo &info,1459                          const ConstructQueue &queue,1460                          ConstructQueue::const_iterator item, Args &&...args) {1461  auto op = OpTy::create(info.converter.getFirOpBuilder(), info.loc,1462                         std::forward<Args>(args)...);1463  createBodyOfOp(*op, info, queue, item);1464  return op;1465}1466 1467template <typename OpTy, typename ClauseOpsTy>1468static OpTy genWrapperOp(lower::AbstractConverter &converter,1469                         mlir::Location loc, const ClauseOpsTy &clauseOps,1470                         const EntryBlockArgs &args) {1471  static_assert(1472      OpTy::template hasTrait<mlir::omp::LoopWrapperInterface::Trait>(),1473      "expected a loop wrapper");1474  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();1475 1476  // Create wrapper.1477  auto op = OpTy::create(firOpBuilder, loc, clauseOps);1478 1479  // Create entry block with arguments.1480  genEntryBlock(firOpBuilder, args, op.getRegion());1481 1482  return op;1483}1484 1485//===----------------------------------------------------------------------===//1486// Code generation functions for clauses1487//===----------------------------------------------------------------------===//1488 1489static void genCancelClauses(lower::AbstractConverter &converter,1490                             semantics::SemanticsContext &semaCtx,1491                             const List<Clause> &clauses, mlir::Location loc,1492                             mlir::omp::CancelOperands &clauseOps) {1493  ClauseProcessor cp(converter, semaCtx, clauses);1494  cp.processCancelDirectiveName(clauseOps);1495  cp.processIf(llvm::omp::Directive::OMPD_cancel, clauseOps);1496}1497 1498static void1499genCancellationPointClauses(lower::AbstractConverter &converter,1500                            semantics::SemanticsContext &semaCtx,1501                            const List<Clause> &clauses, mlir::Location loc,1502                            mlir::omp::CancellationPointOperands &clauseOps) {1503  ClauseProcessor cp(converter, semaCtx, clauses);1504  cp.processCancelDirectiveName(clauseOps);1505}1506 1507static void genCriticalDeclareClauses(1508    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,1509    const List<Clause> &clauses, mlir::Location loc,1510    mlir::omp::CriticalDeclareOperands &clauseOps, llvm::StringRef name) {1511  ClauseProcessor cp(converter, semaCtx, clauses);1512  cp.processHint(clauseOps);1513  clauseOps.symName =1514      mlir::StringAttr::get(converter.getFirOpBuilder().getContext(), name);1515}1516 1517static void genDistributeClauses(lower::AbstractConverter &converter,1518                                 semantics::SemanticsContext &semaCtx,1519                                 lower::StatementContext &stmtCtx,1520                                 const List<Clause> &clauses,1521                                 mlir::Location loc,1522                                 mlir::omp::DistributeOperands &clauseOps) {1523  ClauseProcessor cp(converter, semaCtx, clauses);1524  cp.processAllocate(clauseOps);1525  cp.processDistSchedule(stmtCtx, clauseOps);1526  cp.processOrder(clauseOps);1527}1528 1529static void genFlushClauses(lower::AbstractConverter &converter,1530                            semantics::SemanticsContext &semaCtx,1531                            const ObjectList &objects,1532                            const List<Clause> &clauses, mlir::Location loc,1533                            llvm::SmallVectorImpl<mlir::Value> &operandRange) {1534  if (!objects.empty())1535    genObjectList(objects, converter, operandRange);1536 1537  ClauseProcessor cp(converter, semaCtx, clauses);1538  cp.processTODO<clause::AcqRel, clause::Acquire, clause::Release,1539                 clause::SeqCst>(loc, llvm::omp::OMPD_flush);1540}1541 1542static void1543genLoopNestClauses(lower::AbstractConverter &converter,1544                   semantics::SemanticsContext &semaCtx,1545                   lower::pft::Evaluation &eval, const List<Clause> &clauses,1546                   mlir::Location loc, mlir::omp::LoopNestOperands &clauseOps,1547                   llvm::SmallVectorImpl<const semantics::Symbol *> &iv) {1548  ClauseProcessor cp(converter, semaCtx, clauses);1549 1550  HostEvalInfo *hostEvalInfo = getHostEvalInfoStackTop(converter);1551  if (!hostEvalInfo || !hostEvalInfo->apply(clauseOps, iv))1552    cp.processCollapse(loc, eval, clauseOps, clauseOps, iv);1553 1554  clauseOps.loopInclusive = converter.getFirOpBuilder().getUnitAttr();1555  cp.processTileSizes(eval, clauseOps);1556}1557 1558static void genLoopClauses(1559    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,1560    const List<Clause> &clauses, mlir::Location loc,1561    mlir::omp::LoopOperands &clauseOps,1562    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {1563  ClauseProcessor cp(converter, semaCtx, clauses);1564  cp.processBind(clauseOps);1565  cp.processOrder(clauseOps);1566  cp.processReduction(loc, clauseOps, reductionSyms);1567  cp.processTODO<clause::Lastprivate>(loc, llvm::omp::Directive::OMPD_loop);1568}1569 1570static void genMaskedClauses(lower::AbstractConverter &converter,1571                             semantics::SemanticsContext &semaCtx,1572                             lower::StatementContext &stmtCtx,1573                             const List<Clause> &clauses, mlir::Location loc,1574                             mlir::omp::MaskedOperands &clauseOps) {1575  ClauseProcessor cp(converter, semaCtx, clauses);1576  cp.processFilter(stmtCtx, clauseOps);1577}1578 1579static void1580genOrderedRegionClauses(lower::AbstractConverter &converter,1581                        semantics::SemanticsContext &semaCtx,1582                        const List<Clause> &clauses, mlir::Location loc,1583                        mlir::omp::OrderedRegionOperands &clauseOps) {1584  ClauseProcessor cp(converter, semaCtx, clauses);1585  cp.processTODO<clause::Simd>(loc, llvm::omp::Directive::OMPD_ordered);1586}1587 1588static void genParallelClauses(1589    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,1590    lower::StatementContext &stmtCtx, const List<Clause> &clauses,1591    mlir::Location loc, mlir::omp::ParallelOperands &clauseOps,1592    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {1593  ClauseProcessor cp(converter, semaCtx, clauses);1594  cp.processAllocate(clauseOps);1595  cp.processIf(llvm::omp::Directive::OMPD_parallel, clauseOps);1596 1597  HostEvalInfo *hostEvalInfo = getHostEvalInfoStackTop(converter);1598  if (!hostEvalInfo || !hostEvalInfo->apply(clauseOps))1599    cp.processNumThreads(stmtCtx, clauseOps);1600 1601  cp.processProcBind(clauseOps);1602  cp.processReduction(loc, clauseOps, reductionSyms);1603}1604 1605static void genScanClauses(lower::AbstractConverter &converter,1606                           semantics::SemanticsContext &semaCtx,1607                           const List<Clause> &clauses, mlir::Location loc,1608                           mlir::omp::ScanOperands &clauseOps) {1609  ClauseProcessor cp(converter, semaCtx, clauses);1610  cp.processInclusive(loc, clauseOps);1611  cp.processExclusive(loc, clauseOps);1612}1613 1614static void genSectionsClauses(1615    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,1616    const List<Clause> &clauses, mlir::Location loc,1617    mlir::omp::SectionsOperands &clauseOps,1618    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {1619  ClauseProcessor cp(converter, semaCtx, clauses);1620  cp.processAllocate(clauseOps);1621  cp.processNowait(clauseOps);1622  cp.processReduction(loc, clauseOps, reductionSyms);1623  // TODO Support delayed privatization.1624}1625 1626static void genSimdClauses(1627    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,1628    const List<Clause> &clauses, mlir::Location loc,1629    mlir::omp::SimdOperands &clauseOps,1630    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {1631  ClauseProcessor cp(converter, semaCtx, clauses);1632  cp.processAligned(clauseOps);1633  cp.processIf(llvm::omp::Directive::OMPD_simd, clauseOps);1634  cp.processNontemporal(clauseOps);1635  cp.processOrder(clauseOps);1636  cp.processReduction(loc, clauseOps, reductionSyms);1637  cp.processSafelen(clauseOps);1638  cp.processSimdlen(clauseOps);1639 1640  cp.processTODO<clause::Linear>(loc, llvm::omp::Directive::OMPD_simd);1641}1642 1643static void genSingleClauses(lower::AbstractConverter &converter,1644                             semantics::SemanticsContext &semaCtx,1645                             const List<Clause> &clauses, mlir::Location loc,1646                             mlir::omp::SingleOperands &clauseOps) {1647  ClauseProcessor cp(converter, semaCtx, clauses);1648  cp.processAllocate(clauseOps);1649  cp.processCopyprivate(loc, clauseOps);1650  cp.processNowait(clauseOps);1651  // TODO Support delayed privatization.1652}1653 1654static void genTargetClauses(1655    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,1656    lower::SymMap &symTable, lower::StatementContext &stmtCtx,1657    lower::pft::Evaluation &eval, const List<Clause> &clauses,1658    mlir::Location loc, mlir::omp::TargetOperands &clauseOps,1659    DefaultMapsTy &defaultMaps,1660    llvm::SmallVectorImpl<const semantics::Symbol *> &hasDeviceAddrSyms,1661    llvm::SmallVectorImpl<const semantics::Symbol *> &isDevicePtrSyms,1662    llvm::SmallVectorImpl<const semantics::Symbol *> &mapSyms) {1663  ClauseProcessor cp(converter, semaCtx, clauses);1664  cp.processBare(clauseOps);1665  cp.processDefaultMap(stmtCtx, defaultMaps);1666  cp.processDepend(symTable, stmtCtx, clauseOps);1667  cp.processDevice(stmtCtx, clauseOps);1668  cp.processHasDeviceAddr(stmtCtx, clauseOps, hasDeviceAddrSyms);1669  if (HostEvalInfo *hostEvalInfo = getHostEvalInfoStackTop(converter)) {1670    // Only process host_eval if compiling for the host device.1671    processHostEvalClauses(converter, semaCtx, stmtCtx, eval, loc);1672    hostEvalInfo->collectValues(clauseOps.hostEvalVars);1673  }1674  cp.processIf(llvm::omp::Directive::OMPD_target, clauseOps);1675  cp.processIsDevicePtr(clauseOps, isDevicePtrSyms);1676  cp.processMap(loc, stmtCtx, clauseOps, llvm::omp::Directive::OMPD_unknown,1677                &mapSyms);1678  cp.processNowait(clauseOps);1679  cp.processThreadLimit(stmtCtx, clauseOps);1680 1681  cp.processTODO<clause::Allocate, clause::InReduction, clause::UsesAllocators>(1682      loc, llvm::omp::Directive::OMPD_target);1683 1684  // `target private(..)` is only supported in delayed privatization mode.1685  if (!enableDelayedPrivatizationStaging)1686    cp.processTODO<clause::Firstprivate, clause::Private>(1687        loc, llvm::omp::Directive::OMPD_target);1688}1689 1690static void genTargetDataClauses(1691    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,1692    lower::StatementContext &stmtCtx, const List<Clause> &clauses,1693    mlir::Location loc, mlir::omp::TargetDataOperands &clauseOps,1694    llvm::SmallVectorImpl<const semantics::Symbol *> &useDeviceAddrSyms,1695    llvm::SmallVectorImpl<const semantics::Symbol *> &useDevicePtrSyms) {1696  ClauseProcessor cp(converter, semaCtx, clauses);1697  cp.processDevice(stmtCtx, clauseOps);1698  cp.processIf(llvm::omp::Directive::OMPD_target_data, clauseOps);1699  cp.processMap(loc, stmtCtx, clauseOps);1700  cp.processUseDeviceAddr(stmtCtx, clauseOps, useDeviceAddrSyms);1701  cp.processUseDevicePtr(stmtCtx, clauseOps, useDevicePtrSyms);1702 1703  // This function implements the deprecated functionality of use_device_ptr1704  // that allows users to provide non-CPTR arguments to it with the caveat1705  // that the compiler will treat them as use_device_addr. A lot of legacy1706  // code may still depend on this functionality, so we should support it1707  // in some manner. We do so currently by simply shifting non-cptr operands1708  // from the use_device_ptr lists into the use_device_addr lists.1709  // TODO: Perhaps create a user provideable compiler option that will1710  // re-introduce a hard-error rather than a warning in these cases.1711  promoteNonCPtrUseDevicePtrArgsToUseDeviceAddr(1712      clauseOps.useDeviceAddrVars, useDeviceAddrSyms,1713      clauseOps.useDevicePtrVars, useDevicePtrSyms);1714}1715 1716static void genTargetEnterExitUpdateDataClauses(1717    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,1718    lower::SymMap &symTable, lower::StatementContext &stmtCtx,1719    const List<Clause> &clauses, mlir::Location loc,1720    llvm::omp::Directive directive,1721    mlir::omp::TargetEnterExitUpdateDataOperands &clauseOps) {1722  ClauseProcessor cp(converter, semaCtx, clauses);1723  cp.processDepend(symTable, stmtCtx, clauseOps);1724  cp.processDevice(stmtCtx, clauseOps);1725  cp.processIf(directive, clauseOps);1726 1727  if (directive == llvm::omp::Directive::OMPD_target_update)1728    cp.processMotionClauses(stmtCtx, clauseOps);1729  else1730    cp.processMap(loc, stmtCtx, clauseOps, directive);1731 1732  cp.processNowait(clauseOps);1733}1734 1735static void genTaskClauses(1736    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,1737    lower::SymMap &symTable, lower::StatementContext &stmtCtx,1738    const List<Clause> &clauses, mlir::Location loc,1739    mlir::omp::TaskOperands &clauseOps,1740    llvm::SmallVectorImpl<const semantics::Symbol *> &inReductionSyms) {1741  ClauseProcessor cp(converter, semaCtx, clauses);1742  cp.processAllocate(clauseOps);1743  cp.processDepend(symTable, stmtCtx, clauseOps);1744  cp.processFinal(stmtCtx, clauseOps);1745  cp.processIf(llvm::omp::Directive::OMPD_task, clauseOps);1746  cp.processInReduction(loc, clauseOps, inReductionSyms);1747  cp.processMergeable(clauseOps);1748  cp.processPriority(stmtCtx, clauseOps);1749  cp.processUntied(clauseOps);1750  cp.processDetach(clauseOps);1751 1752  cp.processTODO<clause::Affinity>(loc, llvm::omp::Directive::OMPD_task);1753}1754 1755static void genTaskgroupClauses(1756    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,1757    const List<Clause> &clauses, mlir::Location loc,1758    mlir::omp::TaskgroupOperands &clauseOps,1759    llvm::SmallVectorImpl<const semantics::Symbol *> &taskReductionSyms) {1760  ClauseProcessor cp(converter, semaCtx, clauses);1761  cp.processAllocate(clauseOps);1762  cp.processTaskReduction(loc, clauseOps, taskReductionSyms);1763}1764 1765static void genTaskloopClauses(1766    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,1767    lower::StatementContext &stmtCtx, const List<Clause> &clauses,1768    mlir::Location loc, mlir::omp::TaskloopOperands &clauseOps,1769    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms,1770    llvm::SmallVectorImpl<const semantics::Symbol *> &inReductionSyms) {1771 1772  ClauseProcessor cp(converter, semaCtx, clauses);1773  cp.processAllocate(clauseOps);1774  cp.processFinal(stmtCtx, clauseOps);1775  cp.processGrainsize(stmtCtx, clauseOps);1776  cp.processIf(llvm::omp::Directive::OMPD_taskloop, clauseOps);1777  cp.processInReduction(loc, clauseOps, inReductionSyms);1778  cp.processMergeable(clauseOps);1779  cp.processNogroup(clauseOps);1780  cp.processNumTasks(stmtCtx, clauseOps);1781  cp.processPriority(stmtCtx, clauseOps);1782  cp.processReduction(loc, clauseOps, reductionSyms);1783  cp.processUntied(clauseOps);1784}1785 1786static void genTaskwaitClauses(lower::AbstractConverter &converter,1787                               semantics::SemanticsContext &semaCtx,1788                               const List<Clause> &clauses, mlir::Location loc,1789                               mlir::omp::TaskwaitOperands &clauseOps) {1790  ClauseProcessor cp(converter, semaCtx, clauses);1791  cp.processTODO<clause::Depend, clause::Nowait>(1792      loc, llvm::omp::Directive::OMPD_taskwait);1793}1794 1795static void genWorkshareClauses(lower::AbstractConverter &converter,1796                                semantics::SemanticsContext &semaCtx,1797                                lower::StatementContext &stmtCtx,1798                                const List<Clause> &clauses, mlir::Location loc,1799                                mlir::omp::WorkshareOperands &clauseOps) {1800  ClauseProcessor cp(converter, semaCtx, clauses);1801  cp.processNowait(clauseOps);1802}1803 1804static void genTeamsClauses(1805    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,1806    lower::StatementContext &stmtCtx, const List<Clause> &clauses,1807    mlir::Location loc, mlir::omp::TeamsOperands &clauseOps,1808    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {1809  ClauseProcessor cp(converter, semaCtx, clauses);1810  cp.processAllocate(clauseOps);1811  cp.processIf(llvm::omp::Directive::OMPD_teams, clauseOps);1812 1813  HostEvalInfo *hostEvalInfo = getHostEvalInfoStackTop(converter);1814  if (!hostEvalInfo || !hostEvalInfo->apply(clauseOps)) {1815    cp.processNumTeams(stmtCtx, clauseOps);1816    cp.processThreadLimit(stmtCtx, clauseOps);1817  }1818 1819  cp.processReduction(loc, clauseOps, reductionSyms);1820  // TODO Support delayed privatization.1821}1822 1823static void genWsloopClauses(1824    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,1825    lower::StatementContext &stmtCtx, const List<Clause> &clauses,1826    mlir::Location loc, mlir::omp::WsloopOperands &clauseOps,1827    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {1828  ClauseProcessor cp(converter, semaCtx, clauses);1829  cp.processNowait(clauseOps);1830  cp.processOrder(clauseOps);1831  cp.processOrdered(clauseOps);1832  cp.processReduction(loc, clauseOps, reductionSyms);1833  cp.processSchedule(stmtCtx, clauseOps);1834 1835  cp.processTODO<clause::Allocate, clause::Linear>(1836      loc, llvm::omp::Directive::OMPD_do);1837}1838 1839//===----------------------------------------------------------------------===//1840// Code generation functions for leaf constructs1841//===----------------------------------------------------------------------===//1842 1843static mlir::omp::BarrierOp1844genBarrierOp(lower::AbstractConverter &converter, lower::SymMap &symTable,1845             semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,1846             mlir::Location loc, const ConstructQueue &queue,1847             ConstructQueue::const_iterator item) {1848  return mlir::omp::BarrierOp::create(converter.getFirOpBuilder(), loc);1849}1850 1851static mlir::omp::CancelOp genCancelOp(lower::AbstractConverter &converter,1852                                       semantics::SemanticsContext &semaCtx,1853                                       lower::pft::Evaluation &eval,1854                                       mlir::Location loc,1855                                       const ConstructQueue &queue,1856                                       ConstructQueue::const_iterator item) {1857  mlir::omp::CancelOperands clauseOps;1858  genCancelClauses(converter, semaCtx, item->clauses, loc, clauseOps);1859 1860  return mlir::omp::CancelOp::create(converter.getFirOpBuilder(), loc,1861                                     clauseOps);1862}1863 1864static mlir::omp::CancellationPointOp genCancellationPointOp(1865    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,1866    lower::pft::Evaluation &eval, mlir::Location loc,1867    const ConstructQueue &queue, ConstructQueue::const_iterator item) {1868  mlir::omp::CancellationPointOperands clauseOps;1869  genCancellationPointClauses(converter, semaCtx, item->clauses, loc,1870                              clauseOps);1871 1872  return mlir::omp::CancellationPointOp::create(converter.getFirOpBuilder(),1873                                                loc, clauseOps);1874}1875 1876static mlir::omp::CriticalOp1877genCriticalOp(lower::AbstractConverter &converter, lower::SymMap &symTable,1878              semantics::SemanticsContext &semaCtx,1879              lower::pft::Evaluation &eval, mlir::Location loc,1880              const ConstructQueue &queue, ConstructQueue::const_iterator item,1881              const std::optional<parser::Name> &name) {1882  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();1883  mlir::FlatSymbolRefAttr nameAttr;1884 1885  if (name) {1886    std::string nameStr = name->ToString();1887    mlir::ModuleOp mod = firOpBuilder.getModule();1888    auto global = mod.lookupSymbol<mlir::omp::CriticalDeclareOp>(nameStr);1889    if (!global) {1890      mlir::omp::CriticalDeclareOperands clauseOps;1891      genCriticalDeclareClauses(converter, semaCtx, item->clauses, loc,1892                                clauseOps, nameStr);1893 1894      mlir::OpBuilder modBuilder(mod.getBodyRegion());1895      global = mlir::omp::CriticalDeclareOp::create(modBuilder, loc, clauseOps);1896    }1897    nameAttr = mlir::FlatSymbolRefAttr::get(firOpBuilder.getContext(),1898                                            global.getSymName());1899  }1900 1901  return genOpWithBody<mlir::omp::CriticalOp>(1902      OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,1903                        llvm::omp::Directive::OMPD_critical),1904      queue, item, nameAttr);1905}1906 1907static mlir::omp::FlushOp1908genFlushOp(lower::AbstractConverter &converter, lower::SymMap &symTable,1909           semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,1910           mlir::Location loc, const ObjectList &objects,1911           const ConstructQueue &queue, ConstructQueue::const_iterator item) {1912  llvm::SmallVector<mlir::Value> operandRange;1913  genFlushClauses(converter, semaCtx, objects, item->clauses, loc,1914                  operandRange);1915 1916  return mlir::omp::FlushOp::create(converter.getFirOpBuilder(),1917                                    converter.getCurrentLocation(),1918                                    operandRange);1919}1920 1921static mlir::omp::LoopNestOp genLoopNestOp(1922    lower::AbstractConverter &converter, lower::SymMap &symTable,1923    semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,1924    mlir::Location loc, const ConstructQueue &queue,1925    ConstructQueue::const_iterator item, mlir::omp::LoopNestOperands &clauseOps,1926    llvm::ArrayRef<const semantics::Symbol *> iv,1927    llvm::ArrayRef<1928        std::pair<mlir::omp::BlockArgOpenMPOpInterface, const EntryBlockArgs &>>1929        wrapperArgs,1930    llvm::omp::Directive directive, DataSharingProcessor &dsp) {1931  auto ivCallback = [&](mlir::Operation *op) {1932    genLoopVars(op, converter, loc, iv, wrapperArgs);1933    return llvm::SmallVector<const semantics::Symbol *>(iv);1934  };1935 1936  uint64_t nestValue = getCollapseValue(item->clauses);1937  nestValue = nestValue < iv.size() ? iv.size() : nestValue;1938  auto *nestedEval = getCollapsedLoopEval(eval, nestValue);1939  return genOpWithBody<mlir::omp::LoopNestOp>(1940      OpWithBodyGenInfo(converter, symTable, semaCtx, loc, *nestedEval,1941                        directive)1942          .setClauses(&item->clauses)1943          .setDataSharingProcessor(&dsp)1944          .setGenRegionEntryCb(ivCallback),1945      queue, item, clauseOps);1946}1947 1948static mlir::omp::LoopOp1949genLoopOp(lower::AbstractConverter &converter, lower::SymMap &symTable,1950          semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,1951          mlir::Location loc, const ConstructQueue &queue,1952          ConstructQueue::const_iterator item) {1953  mlir::omp::LoopOperands loopClauseOps;1954  llvm::SmallVector<const semantics::Symbol *> loopReductionSyms;1955  genLoopClauses(converter, semaCtx, item->clauses, loc, loopClauseOps,1956                 loopReductionSyms);1957 1958  DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,1959                           /*shouldCollectPreDeterminedSymbols=*/true,1960                           /*useDelayedPrivatization=*/true, symTable);1961  dsp.processStep1(&loopClauseOps);1962 1963  mlir::omp::LoopNestOperands loopNestClauseOps;1964  llvm::SmallVector<const semantics::Symbol *> iv;1965  genLoopNestClauses(converter, semaCtx, eval, item->clauses, loc,1966                     loopNestClauseOps, iv);1967 1968  EntryBlockArgs loopArgs;1969  loopArgs.priv.syms = dsp.getDelayedPrivSymbols();1970  loopArgs.priv.vars = loopClauseOps.privateVars;1971  loopArgs.reduction.syms = loopReductionSyms;1972  loopArgs.reduction.vars = loopClauseOps.reductionVars;1973 1974  auto loopOp =1975      genWrapperOp<mlir::omp::LoopOp>(converter, loc, loopClauseOps, loopArgs);1976  genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, item,1977                loopNestClauseOps, iv, {{loopOp, loopArgs}},1978                llvm::omp::Directive::OMPD_loop, dsp);1979  return loopOp;1980}1981 1982static void genCanonicalLoopNest(1983    lower::AbstractConverter &converter, lower::SymMap &symTable,1984    semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,1985    mlir::Location loc, const ConstructQueue &queue,1986    ConstructQueue::const_iterator item, size_t numLoops,1987    llvm::SmallVectorImpl<mlir::omp::CanonicalLoopOp> &loops) {1988  assert(loops.empty() && "Expecting empty list to fill");1989  assert(numLoops >= 1 && "Expecting at least one loop");1990 1991  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();1992 1993  mlir::omp::LoopRelatedClauseOps loopInfo;1994  llvm::SmallVector<const semantics::Symbol *, 3> ivs;1995  collectLoopRelatedInfo(converter, loc, eval, numLoops, loopInfo, ivs);1996  assert(ivs.size() == numLoops &&1997         "Expected to parse as many loop variables as there are loops");1998 1999  // Steps that follow:2000  // 1. Emit all of the loop's prologues (compute the tripcount)2001  // 2. Emit omp.canonical_loop nested inside each other (iteratively)2002  // 2.1. In the innermost omp.canonical_loop, emit the loop body prologue (in2003  // the body callback)2004  //2005  // Since emitting prologues and body code is split, remember prologue values2006  // for use when emitting the same loop's epilogues.2007  llvm::SmallVector<mlir::Value> tripcounts;2008  llvm::SmallVector<mlir::Value> clis;2009  llvm::SmallVector<lower::pft::Evaluation *> evals;2010  llvm::SmallVector<mlir::Type> loopVarTypes;2011  llvm::SmallVector<mlir::Value> loopStepVars;2012  llvm::SmallVector<mlir::Value> loopLBVars;2013  llvm::SmallVector<mlir::Value> blockArgs;2014 2015  // Step 1: Loop prologues2016  // Computing the trip count must happen before entering the outermost loop2017  lower::pft::Evaluation *innermostEval = &eval.getFirstNestedEvaluation();2018  for ([[maybe_unused]] auto iv : ivs) {2019    if (innermostEval->getIf<parser::DoConstruct>()->IsDoConcurrent()) {2020      // OpenMP specifies DO CONCURRENT only with the `!omp loop` construct.2021      // Will need to add special cases for this combination.2022      TODO(loc, "DO CONCURRENT as canonical loop not supported");2023    }2024 2025    auto &doLoopEval = innermostEval->getFirstNestedEvaluation();2026    evals.push_back(innermostEval);2027 2028    // Get the loop bounds (and increment)2029    // auto &doLoopEval = nestedEval.getFirstNestedEvaluation();2030    auto *doStmt = doLoopEval.getIf<parser::NonLabelDoStmt>();2031    assert(doStmt && "Expected do loop to be in the nested evaluation");2032    auto &loopControl = std::get<std::optional<parser::LoopControl>>(doStmt->t);2033    assert(loopControl.has_value());2034    auto *bounds = std::get_if<parser::LoopControl::Bounds>(&loopControl->u);2035    assert(bounds && "Expected bounds for canonical loop");2036    lower::StatementContext stmtCtx;2037    mlir::Value loopLBVar = fir::getBase(2038        converter.genExprValue(*semantics::GetExpr(bounds->lower), stmtCtx));2039    mlir::Value loopUBVar = fir::getBase(2040        converter.genExprValue(*semantics::GetExpr(bounds->upper), stmtCtx));2041    mlir::Value loopStepVar = [&]() {2042      if (bounds->step) {2043        return fir::getBase(2044            converter.genExprValue(*semantics::GetExpr(bounds->step), stmtCtx));2045      }2046 2047      // If `step` is not present, assume it is `1`.2048      auto intTy = firOpBuilder.getI32Type();2049      return firOpBuilder.createIntegerConstant(loc, intTy, 1);2050    }();2051 2052    // Get the integer kind for the loop variable and cast the loop bounds2053    size_t loopVarTypeSize = bounds->name.thing.symbol->GetUltimate().size();2054    mlir::Type loopVarType = getLoopVarType(converter, loopVarTypeSize);2055    loopVarTypes.push_back(loopVarType);2056    loopLBVar = firOpBuilder.createConvert(loc, loopVarType, loopLBVar);2057    loopUBVar = firOpBuilder.createConvert(loc, loopVarType, loopUBVar);2058    loopStepVar = firOpBuilder.createConvert(loc, loopVarType, loopStepVar);2059    loopLBVars.push_back(loopLBVar);2060    loopStepVars.push_back(loopStepVar);2061 2062    // Start lowering2063    mlir::Value zero = firOpBuilder.createIntegerConstant(loc, loopVarType, 0);2064    mlir::Value one = firOpBuilder.createIntegerConstant(loc, loopVarType, 1);2065    mlir::Value isDownwards = mlir::arith::CmpIOp::create(2066        firOpBuilder, loc, mlir::arith::CmpIPredicate::slt, loopStepVar, zero);2067 2068    // Ensure we are counting upwards. If not, negate step and swap lb and ub.2069    mlir::Value negStep =2070        mlir::arith::SubIOp::create(firOpBuilder, loc, zero, loopStepVar);2071    mlir::Value incr = mlir::arith::SelectOp::create(2072        firOpBuilder, loc, isDownwards, negStep, loopStepVar);2073    mlir::Value lb = mlir::arith::SelectOp::create(2074        firOpBuilder, loc, isDownwards, loopUBVar, loopLBVar);2075    mlir::Value ub = mlir::arith::SelectOp::create(2076        firOpBuilder, loc, isDownwards, loopLBVar, loopUBVar);2077 2078    // Compute the trip count assuming lb <= ub. This guarantees that the result2079    // is non-negative and we can use unsigned arithmetic.2080    mlir::Value span = mlir::arith::SubIOp::create(2081        firOpBuilder, loc, ub, lb, ::mlir::arith::IntegerOverflowFlags::nuw);2082    mlir::Value tcMinusOne =2083        mlir::arith::DivUIOp::create(firOpBuilder, loc, span, incr);2084    mlir::Value tcIfLooping =2085        mlir::arith::AddIOp::create(firOpBuilder, loc, tcMinusOne, one,2086                                    ::mlir::arith::IntegerOverflowFlags::nuw);2087 2088    // Fall back to 0 if lb > ub2089    mlir::Value isZeroTC = mlir::arith::CmpIOp::create(2090        firOpBuilder, loc, mlir::arith::CmpIPredicate::slt, ub, lb);2091    mlir::Value tripcount = mlir::arith::SelectOp::create(2092        firOpBuilder, loc, isZeroTC, zero, tcIfLooping);2093    tripcounts.push_back(tripcount);2094 2095    // Create the CLI handle.2096    auto newcli = mlir::omp::NewCliOp::create(firOpBuilder, loc);2097    mlir::Value cli = newcli.getResult();2098    clis.push_back(cli);2099 2100    innermostEval = &*std::next(innermostEval->getNestedEvaluations().begin());2101  }2102 2103  // Step 2: Create nested canoncial loops2104  for (auto i : llvm::seq<size_t>(numLoops)) {2105    bool isInnermost = (i == numLoops - 1);2106    mlir::Type loopVarType = loopVarTypes[i];2107    mlir::Value tripcount = tripcounts[i];2108    mlir::Value cli = clis[i];2109    auto &&eval = evals[i];2110 2111    auto ivCallback = [&, i, isInnermost](mlir::Operation *op)2112        -> llvm::SmallVector<const Fortran::semantics::Symbol *> {2113      mlir::Region &region = op->getRegion(0);2114 2115      // Create the op's region skeleton (BB taking the iv as argument)2116      firOpBuilder.createBlock(&region, {}, {loopVarType}, {loc});2117      blockArgs.push_back(region.front().getArgument(0));2118 2119      // Step 2.1: Emit body prologue code2120      // Compute the translation from logical iteration number to the value of2121      // the loop's iteration variable only in the innermost body. Currently,2122      // loop transformations do not allow any instruction between loops, but2123      // this will change with2124      if (isInnermost) {2125        assert(blockArgs.size() == numLoops &&2126               "Expecting all block args to have been collected by now");2127        for (auto j : llvm::seq<size_t>(numLoops)) {2128          mlir::Value natIterNum = fir::getBase(blockArgs[j]);2129          mlir::Value scaled = mlir::arith::MulIOp::create(2130              firOpBuilder, loc, natIterNum, loopStepVars[j]);2131          mlir::Value userVal = mlir::arith::AddIOp::create(2132              firOpBuilder, loc, loopLBVars[j], scaled);2133 2134          mlir::OpBuilder::InsertPoint insPt =2135              firOpBuilder.saveInsertionPoint();2136          firOpBuilder.setInsertionPointToStart(firOpBuilder.getAllocaBlock());2137          mlir::Type tempTy = converter.genType(*ivs[j]);2138          firOpBuilder.restoreInsertionPoint(insPt);2139 2140          // Write the loop value into loop variable2141          mlir::Value cvtVal = firOpBuilder.createConvert(loc, tempTy, userVal);2142          hlfir::Entity lhs{converter.getSymbolAddress(*ivs[j])};2143          lhs = hlfir::derefPointersAndAllocatables(loc, firOpBuilder, lhs);2144          mlir::Operation *storeOp =2145              hlfir::AssignOp::create(firOpBuilder, loc, cvtVal, lhs);2146          firOpBuilder.setInsertionPointAfter(storeOp);2147        }2148      }2149 2150      return {ivs[i]};2151    };2152 2153    // Create the omp.canonical_loop operation2154    auto opGenInfo = OpWithBodyGenInfo(converter, symTable, semaCtx, loc, *eval,2155                                       llvm::omp::Directive::OMPD_unknown)2156                         .setGenSkeletonOnly(!isInnermost)2157                         .setClauses(&item->clauses)2158                         .setPrivatize(false)2159                         .setGenRegionEntryCb(ivCallback);2160    auto canonLoop = genOpWithBody<mlir::omp::CanonicalLoopOp>(2161        std::move(opGenInfo), queue, item, tripcount, cli);2162    loops.push_back(canonLoop);2163 2164    // Insert next loop nested inside last loop2165    firOpBuilder.setInsertionPoint(2166        canonLoop.getRegion().back().getTerminator());2167  }2168 2169  firOpBuilder.setInsertionPointAfter(loops.front());2170}2171 2172static void genTileOp(Fortran::lower::AbstractConverter &converter,2173                      Fortran::lower::SymMap &symTable,2174                      lower::StatementContext &stmtCtx,2175                      Fortran::semantics::SemanticsContext &semaCtx,2176                      Fortran::lower::pft::Evaluation &eval, mlir::Location loc,2177                      const ConstructQueue &queue,2178                      ConstructQueue::const_iterator item) {2179  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();2180 2181  mlir::omp::SizesClauseOps sizesClause;2182  ClauseProcessor cp(converter, semaCtx, item->clauses);2183  cp.processSizes(stmtCtx, sizesClause);2184 2185  size_t numLoops = sizesClause.sizes.size();2186  llvm::SmallVector<mlir::omp::CanonicalLoopOp, 3> canonLoops;2187  canonLoops.reserve(numLoops);2188 2189  genCanonicalLoopNest(converter, symTable, semaCtx, eval, loc, queue, item,2190                       numLoops, canonLoops);2191  assert((canonLoops.size() == numLoops) &&2192         "Expecting the predetermined number of loops");2193 2194  llvm::SmallVector<mlir::Value, 3> applyees;2195  applyees.reserve(numLoops);2196  for (mlir::omp::CanonicalLoopOp l : canonLoops)2197    applyees.push_back(l.getCli());2198 2199  // Emit the associated loops and create a CLI for each affected loop2200  llvm::SmallVector<mlir::Value, 3> gridGeneratees;2201  llvm::SmallVector<mlir::Value, 3> intratileGeneratees;2202  gridGeneratees.reserve(numLoops);2203  intratileGeneratees.reserve(numLoops);2204  for ([[maybe_unused]] auto i : llvm::seq<int>(0, sizesClause.sizes.size())) {2205    auto gridCLI = mlir::omp::NewCliOp::create(firOpBuilder, loc);2206    gridGeneratees.push_back(gridCLI.getResult());2207    auto intratileCLI = mlir::omp::NewCliOp::create(firOpBuilder, loc);2208    intratileGeneratees.push_back(intratileCLI.getResult());2209  }2210 2211  llvm::SmallVector<mlir::Value, 6> generatees;2212  generatees.reserve(2 * numLoops);2213  generatees.append(gridGeneratees);2214  generatees.append(intratileGeneratees);2215 2216  mlir::omp::TileOp::create(firOpBuilder, loc, generatees, applyees,2217                            sizesClause.sizes);2218}2219 2220static void genUnrollOp(Fortran::lower::AbstractConverter &converter,2221                        Fortran::lower::SymMap &symTable,2222                        lower::StatementContext &stmtCtx,2223                        Fortran::semantics::SemanticsContext &semaCtx,2224                        Fortran::lower::pft::Evaluation &eval,2225                        mlir::Location loc, const ConstructQueue &queue,2226                        ConstructQueue::const_iterator item) {2227  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();2228 2229  // Clauses for unrolling not yet implemnted2230  ClauseProcessor cp(converter, semaCtx, item->clauses);2231  cp.processTODO<clause::Partial, clause::Full>(2232      loc, llvm::omp::Directive::OMPD_unroll);2233 2234  // Emit the associated loop2235  llvm::SmallVector<mlir::omp::CanonicalLoopOp, 1> canonLoops;2236  genCanonicalLoopNest(converter, symTable, semaCtx, eval, loc, queue, item, 1,2237                       canonLoops);2238 2239  llvm::SmallVector<mlir::Value, 1> applyees;2240  for (auto &&canonLoop : canonLoops)2241    applyees.push_back(canonLoop.getCli());2242 2243  // Apply unrolling to it2244  auto cli = llvm::getSingleElement(canonLoops).getCli();2245  mlir::omp::UnrollHeuristicOp::create(firOpBuilder, loc, cli);2246}2247 2248static mlir::omp::MaskedOp2249genMaskedOp(lower::AbstractConverter &converter, lower::SymMap &symTable,2250            lower::StatementContext &stmtCtx,2251            semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,2252            mlir::Location loc, const ConstructQueue &queue,2253            ConstructQueue::const_iterator item) {2254  mlir::omp::MaskedOperands clauseOps;2255  genMaskedClauses(converter, semaCtx, stmtCtx, item->clauses, loc, clauseOps);2256 2257  return genOpWithBody<mlir::omp::MaskedOp>(2258      OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,2259                        llvm::omp::Directive::OMPD_masked),2260      queue, item, clauseOps);2261}2262 2263static mlir::omp::MasterOp2264genMasterOp(lower::AbstractConverter &converter, lower::SymMap &symTable,2265            semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,2266            mlir::Location loc, const ConstructQueue &queue,2267            ConstructQueue::const_iterator item) {2268  return genOpWithBody<mlir::omp::MasterOp>(2269      OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,2270                        llvm::omp::Directive::OMPD_master),2271      queue, item);2272}2273 2274static mlir::omp::OrderedOp2275genOrderedOp(lower::AbstractConverter &converter, lower::SymMap &symTable,2276             semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,2277             mlir::Location loc, const ConstructQueue &queue,2278             ConstructQueue::const_iterator item) {2279  if (!semaCtx.langOptions().OpenMPSimd)2280    TODO(loc, "OMPD_ordered");2281  return nullptr;2282}2283 2284static mlir::omp::OrderedRegionOp2285genOrderedRegionOp(lower::AbstractConverter &converter, lower::SymMap &symTable,2286                   semantics::SemanticsContext &semaCtx,2287                   lower::pft::Evaluation &eval, mlir::Location loc,2288                   const ConstructQueue &queue,2289                   ConstructQueue::const_iterator item) {2290  mlir::omp::OrderedRegionOperands clauseOps;2291  genOrderedRegionClauses(converter, semaCtx, item->clauses, loc, clauseOps);2292 2293  return genOpWithBody<mlir::omp::OrderedRegionOp>(2294      OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,2295                        llvm::omp::Directive::OMPD_ordered),2296      queue, item, clauseOps);2297}2298 2299static mlir::omp::ParallelOp2300genParallelOp(lower::AbstractConverter &converter, lower::SymMap &symTable,2301              semantics::SemanticsContext &semaCtx,2302              lower::pft::Evaluation &eval, mlir::Location loc,2303              const ConstructQueue &queue, ConstructQueue::const_iterator item,2304              mlir::omp::ParallelOperands &clauseOps,2305              const EntryBlockArgs &args, DataSharingProcessor *dsp,2306              bool isComposite = false) {2307  assert((!enableDelayedPrivatization || dsp) &&2308         "expected valid DataSharingProcessor");2309 2310  OpWithBodyGenInfo genInfo =2311      OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,2312                        llvm::omp::Directive::OMPD_parallel)2313          .setClauses(&item->clauses)2314          .setEntryBlockArgs(&args)2315          .setGenSkeletonOnly(isComposite)2316          .setDataSharingProcessor(dsp);2317 2318  auto parallelOp =2319      genOpWithBody<mlir::omp::ParallelOp>(genInfo, queue, item, clauseOps);2320  parallelOp.setComposite(isComposite);2321  return parallelOp;2322}2323 2324static mlir::omp::ScanOp2325genScanOp(lower::AbstractConverter &converter, lower::SymMap &symTable,2326          semantics::SemanticsContext &semaCtx, mlir::Location loc,2327          const ConstructQueue &queue, ConstructQueue::const_iterator item) {2328  mlir::omp::ScanOperands clauseOps;2329  genScanClauses(converter, semaCtx, item->clauses, loc, clauseOps);2330  return mlir::omp::ScanOp::create(converter.getFirOpBuilder(),2331                                   converter.getCurrentLocation(), clauseOps);2332}2333 2334static mlir::omp::SectionsOp2335genSectionsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,2336              semantics::SemanticsContext &semaCtx,2337              lower::pft::Evaluation &eval, mlir::Location loc,2338              const ConstructQueue &queue,2339              ConstructQueue::const_iterator item) {2340  const parser::OpenMPSectionsConstruct *sectionsConstruct =2341      getSectionsConstructStackTop(converter);2342  assert(sectionsConstruct && "Missing additional parsing information");2343 2344  const auto &sectionBlocks =2345      std::get<std::list<parser::OpenMPConstruct>>(sectionsConstruct->t);2346  mlir::omp::SectionsOperands clauseOps;2347  llvm::SmallVector<const semantics::Symbol *> reductionSyms;2348  genSectionsClauses(converter, semaCtx, item->clauses, loc, clauseOps,2349                     reductionSyms);2350 2351  auto &builder = converter.getFirOpBuilder();2352 2353  // Insert privatizations before SECTIONS2354  lower::SymMapScope scope(symTable);2355  DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,2356                           lower::omp::isLastItemInQueue(item, queue),2357                           /*useDelayedPrivatization=*/false, symTable);2358  dsp.processStep1();2359 2360  List<Clause> nonDsaClauses;2361  List<const clause::Lastprivate *> lastprivates;2362 2363  for (const Clause &clause : item->clauses) {2364    if (clause.id == llvm::omp::Clause::OMPC_lastprivate) {2365      auto &lastp = std::get<clause::Lastprivate>(clause.u);2366      lastprivateModifierNotSupported(lastp, converter.getCurrentLocation());2367      lastprivates.push_back(&lastp);2368    } else {2369      switch (clause.id) {2370      case llvm::omp::Clause::OMPC_firstprivate:2371      case llvm::omp::Clause::OMPC_private:2372      case llvm::omp::Clause::OMPC_shared:2373        break;2374      default:2375        nonDsaClauses.push_back(clause);2376      }2377    }2378  }2379 2380  // SECTIONS construct.2381  auto sectionsOp = mlir::omp::SectionsOp::create(builder, loc, clauseOps);2382 2383  // Create entry block with reduction variables as arguments.2384  EntryBlockArgs args;2385  // TODO: Add private syms and vars.2386  args.reduction.syms = reductionSyms;2387  args.reduction.vars = clauseOps.reductionVars;2388 2389  genEntryBlock(builder, args, sectionsOp.getRegion());2390  mlir::Operation *terminator =2391      lower::genOpenMPTerminator(builder, sectionsOp, loc);2392 2393  // Generate nested SECTION constructs.2394  // This is done here rather than in genOMP([...], OpenMPSectionConstruct )2395  // because we need to run genReductionVars on each omp.section so that the2396  // reduction variable gets mapped to the private version2397  for (auto [construct, nestedEval] :2398       llvm::zip(sectionBlocks, eval.getNestedEvaluations())) {2399    const auto *sectionConstruct =2400        std::get_if<parser::OpenMPSectionConstruct>(&construct.u);2401    if (!sectionConstruct) {2402      assert(false &&2403             "unexpected construct nested inside of SECTIONS construct");2404      continue;2405    }2406 2407    ConstructQueue sectionQueue{buildConstructQueue(2408        converter.getFirOpBuilder().getModule(), semaCtx, nestedEval,2409        sectionConstruct->source, llvm::omp::Directive::OMPD_section, {})};2410 2411    builder.setInsertionPoint(terminator);2412    genOpWithBody<mlir::omp::SectionOp>(2413        OpWithBodyGenInfo(converter, symTable, semaCtx, loc, nestedEval,2414                          llvm::omp::Directive::OMPD_section)2415            .setClauses(&sectionQueue.begin()->clauses)2416            .setDataSharingProcessor(&dsp)2417            .setEntryBlockArgs(&args),2418        sectionQueue, sectionQueue.begin());2419  }2420 2421  if (!lastprivates.empty()) {2422    mlir::Region &sectionsBody = sectionsOp.getRegion();2423    assert(sectionsBody.hasOneBlock());2424    mlir::Block &body = sectionsBody.front();2425 2426    auto lastSectionOp = llvm::find_if(2427        llvm::reverse(body.getOperations()), [](const mlir::Operation &op) {2428          return llvm::isa<mlir::omp::SectionOp>(op);2429        });2430    assert(lastSectionOp != body.rend());2431 2432    for (const clause::Lastprivate *lastp : lastprivates) {2433      builder.setInsertionPoint(2434          lastSectionOp->getRegion(0).back().getTerminator());2435      mlir::OpBuilder::InsertPoint insp = builder.saveInsertionPoint();2436      const auto &objList = std::get<ObjectList>(lastp->t);2437      for (const Object &object : objList) {2438        semantics::Symbol *sym = object.sym();2439        if (const auto *common =2440                sym->detailsIf<semantics::CommonBlockDetails>()) {2441          for (const auto &obj : common->objects())2442            converter.copyHostAssociateVar(*obj, &insp, /*hostIsSource=*/false);2443        } else {2444          converter.copyHostAssociateVar(*sym, &insp, /*hostIsSource=*/false);2445        }2446      }2447    }2448  }2449 2450  // Perform DataSharingProcessor's step2 out of SECTIONS2451  builder.setInsertionPointAfter(sectionsOp.getOperation());2452  dsp.processStep2(sectionsOp, false);2453  // Emit implicit barrier to synchronize threads and avoid data2454  // races on post-update of lastprivate variables when `nowait`2455  // clause is present.2456  if (clauseOps.nowait && !lastprivates.empty())2457    mlir::omp::BarrierOp::create(builder, loc);2458 2459  return sectionsOp;2460}2461 2462static mlir::Operation *2463genScopeOp(lower::AbstractConverter &converter, lower::SymMap &symTable,2464           semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,2465           mlir::Location loc, const ConstructQueue &queue,2466           ConstructQueue::const_iterator item) {2467  if (!semaCtx.langOptions().OpenMPSimd)2468    TODO(loc, "Scope construct");2469  return nullptr;2470}2471 2472static mlir::omp::SingleOp2473genSingleOp(lower::AbstractConverter &converter, lower::SymMap &symTable,2474            semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,2475            mlir::Location loc, const ConstructQueue &queue,2476            ConstructQueue::const_iterator item) {2477  mlir::omp::SingleOperands clauseOps;2478  genSingleClauses(converter, semaCtx, item->clauses, loc, clauseOps);2479 2480  return genOpWithBody<mlir::omp::SingleOp>(2481      OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,2482                        llvm::omp::Directive::OMPD_single)2483          .setClauses(&item->clauses),2484      queue, item, clauseOps);2485}2486 2487static bool isDuplicateMappedSymbol(2488    const semantics::Symbol &sym,2489    const llvm::SetVector<const semantics::Symbol *> &privatizedSyms,2490    const llvm::SmallVectorImpl<const semantics::Symbol *> &hasDevSyms,2491    const llvm::SmallVectorImpl<const semantics::Symbol *> &mappedSyms) {2492  llvm::SmallVector<const semantics::Symbol *> concatSyms;2493  concatSyms.reserve(privatizedSyms.size() + hasDevSyms.size() +2494                     mappedSyms.size());2495  concatSyms.append(privatizedSyms.begin(), privatizedSyms.end());2496  concatSyms.append(hasDevSyms.begin(), hasDevSyms.end());2497  concatSyms.append(mappedSyms.begin(), mappedSyms.end());2498 2499  auto checkSymbol = [&](const semantics::Symbol &checkSym) {2500    return std::any_of(concatSyms.begin(), concatSyms.end(),2501                       [&](auto v) { return v->GetUltimate() == checkSym; });2502  };2503 2504  if (checkSymbol(sym))2505    return true;2506 2507  const auto *hostAssoc{sym.detailsIf<semantics::HostAssocDetails>()};2508  if (hostAssoc && checkSymbol(hostAssoc->symbol()))2509    return true;2510 2511  return checkSymbol(sym.GetUltimate());2512}2513 2514static mlir::omp::TargetOp2515genTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,2516            lower::StatementContext &stmtCtx,2517            semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,2518            mlir::Location loc, const ConstructQueue &queue,2519            ConstructQueue::const_iterator item) {2520  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();2521  bool isTargetDevice =2522      llvm::cast<mlir::omp::OffloadModuleInterface>(*converter.getModuleOp())2523          .getIsTargetDevice();2524 2525  // Introduce a new host_eval information structure for this target region.2526  if (!isTargetDevice)2527    converter.getStateStack().stackPush<HostEvalInfoStackFrame>();2528 2529  mlir::omp::TargetOperands clauseOps;2530  DefaultMapsTy defaultMaps;2531  llvm::SmallVector<const semantics::Symbol *> mapSyms, isDevicePtrSyms,2532      hasDeviceAddrSyms;2533  genTargetClauses(converter, semaCtx, symTable, stmtCtx, eval, item->clauses,2534                   loc, clauseOps, defaultMaps, hasDeviceAddrSyms,2535                   isDevicePtrSyms, mapSyms);2536 2537  DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,2538                           /*shouldCollectPreDeterminedSymbols=*/2539                           lower::omp::isLastItemInQueue(item, queue),2540                           /*useDelayedPrivatization=*/true, symTable,2541                           /*isTargetPrivitization=*/true);2542  dsp.processStep1(&clauseOps);2543 2544  // 5.8.1 Implicit Data-Mapping Attribute Rules2545  // The following code follows the implicit data-mapping rules to map all the2546  // symbols used inside the region that do not have explicit data-environment2547  // attribute clauses (neither data-sharing; e.g. `private`, nor `map`2548  // clauses).2549  auto captureImplicitMap = [&](const semantics::Symbol &sym) {2550    // Structure component symbols don't have bindings, and can only be2551    // explicitly mapped individually. If a member is captured implicitly2552    // we map the entirety of the derived type when we find its symbol.2553    if (sym.owner().IsDerivedType())2554      return;2555 2556    // if the symbol is part of an already mapped common block, do not make a2557    // map for it.2558    if (const Fortran::semantics::Symbol *common =2559            Fortran::semantics::FindCommonBlockContaining(sym.GetUltimate()))2560      if (llvm::is_contained(mapSyms, common))2561        return;2562 2563    // If we come across a symbol without a symbol address, we2564    // return as we cannot process it, this is intended as a2565    // catch all early exit for symbols that do not have a2566    // corresponding extended value. Such as subroutines,2567    // interfaces and named blocks.2568    if (!converter.getSymbolAddress(sym))2569      return;2570 2571    // Skip parameters/constants as they do not need to be mapped.2572    if (semantics::IsNamedConstant(sym))2573      return;2574 2575    if (!isDuplicateMappedSymbol(sym, dsp.getAllSymbolsToPrivatize(),2576                                 hasDeviceAddrSyms, mapSyms)) {2577      if (const auto *details =2578              sym.template detailsIf<semantics::HostAssocDetails>())2579        converter.copySymbolBinding(details->symbol(), sym);2580      std::stringstream name;2581      fir::ExtendedValue dataExv = converter.getSymbolExtendedValue(sym);2582      name << sym.name().ToString();2583 2584      fir::factory::AddrAndBoundsInfo info =2585          Fortran::lower::getDataOperandBaseAddr(2586              converter, firOpBuilder, sym.GetUltimate(),2587              converter.getCurrentLocation());2588      llvm::SmallVector<mlir::Value> bounds =2589          fir::factory::genImplicitBoundsOps<mlir::omp::MapBoundsOp,2590                                             mlir::omp::MapBoundsType>(2591              firOpBuilder, info, dataExv,2592              semantics::IsAssumedSizeArray(sym.GetUltimate()),2593              converter.getCurrentLocation());2594      mlir::Value baseOp = info.rawInput;2595      mlir::Type eleType = baseOp.getType();2596      if (auto refType = mlir::dyn_cast<fir::ReferenceType>(baseOp.getType()))2597        eleType = refType.getElementType();2598 2599      std::pair<mlir::omp::ClauseMapFlags, mlir::omp::VariableCaptureKind>2600          mapFlagAndKind = getImplicitMapTypeAndKind(2601              firOpBuilder, converter, defaultMaps, eleType, loc, sym);2602 2603      mlir::FlatSymbolRefAttr mapperId;2604      if (defaultMaps.empty()) {2605        // TODO: Honor user-provided defaultmap clauses (aggregates/pointers)2606        // instead of blanket-disabling implicit mapper generation whenever any2607        // explicit default map is present.2608        const semantics::DerivedTypeSpec *typeSpec =2609            sym.GetType() ? sym.GetType()->AsDerived() : nullptr;2610        if (typeSpec) {2611          std::string mapperIdName =2612              typeSpec->name().ToString() + llvm::omp::OmpDefaultMapperName;2613          if (auto *mapperSym =2614                  converter.getCurrentScope().FindSymbol(mapperIdName))2615            mapperIdName = converter.mangleName(2616                mapperIdName, mapperSym->GetUltimate().owner());2617          else2618            mapperIdName =2619                converter.mangleName(mapperIdName, *typeSpec->GetScope());2620 2621          if (!mapperIdName.empty()) {2622            bool allowImplicitMapper =2623                semantics::IsAllocatableOrObjectPointer(&sym);2624            bool hasDefaultMapper =2625                converter.getModuleOp().lookupSymbol(mapperIdName);2626            if (hasDefaultMapper || allowImplicitMapper) {2627              if (!hasDefaultMapper) {2628                if (auto recordType = mlir::dyn_cast_or_null<fir::RecordType>(2629                        converter.genType(*typeSpec)))2630                  mapperId = getOrGenImplicitDefaultDeclareMapper(2631                      converter, loc, recordType, mapperIdName);2632              } else {2633                mapperId = mlir::FlatSymbolRefAttr::get(2634                    &converter.getMLIRContext(), mapperIdName);2635              }2636            }2637          }2638        }2639      }2640 2641      mlir::Value mapOp = createMapInfoOp(2642          firOpBuilder, converter.getCurrentLocation(), baseOp,2643          /*varPtrPtr=*/mlir::Value{}, name.str(), bounds, /*members=*/{},2644          /*membersIndex=*/mlir::ArrayAttr{}, std::get<0>(mapFlagAndKind),2645          std::get<1>(mapFlagAndKind), baseOp.getType(),2646          /*partialMap=*/false, mapperId);2647 2648      clauseOps.mapVars.push_back(mapOp);2649      mapSyms.push_back(&sym);2650    }2651  };2652  lower::pft::visitAllSymbols(eval, captureImplicitMap);2653 2654  auto targetOp = mlir::omp::TargetOp::create(firOpBuilder, loc, clauseOps);2655 2656  llvm::SmallVector<mlir::Value> hasDeviceAddrBaseValues, mapBaseValues;2657  extractMappedBaseValues(clauseOps.hasDeviceAddrVars, hasDeviceAddrBaseValues);2658  extractMappedBaseValues(clauseOps.mapVars, mapBaseValues);2659 2660  EntryBlockArgs args;2661  args.hasDeviceAddr.syms = hasDeviceAddrSyms;2662  args.hasDeviceAddr.vars = hasDeviceAddrBaseValues;2663  args.hostEvalVars = clauseOps.hostEvalVars;2664  // TODO: Add in_reduction syms and vars.2665  args.map.syms = mapSyms;2666  args.map.vars = mapBaseValues;2667  args.priv.syms = dsp.getDelayedPrivSymbols();2668  args.priv.vars = clauseOps.privateVars;2669 2670  genBodyOfTargetOp(converter, symTable, semaCtx, eval, targetOp, args, loc,2671                    queue, item, dsp);2672 2673  // Remove the host_eval information structure created for this target region.2674  if (!isTargetDevice)2675    converter.getStateStack().stackPop();2676  return targetOp;2677}2678 2679static mlir::omp::TargetDataOp genTargetDataOp(2680    lower::AbstractConverter &converter, lower::SymMap &symTable,2681    lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,2682    lower::pft::Evaluation &eval, mlir::Location loc,2683    const ConstructQueue &queue, ConstructQueue::const_iterator item) {2684  mlir::omp::TargetDataOperands clauseOps;2685  llvm::SmallVector<const semantics::Symbol *> useDeviceAddrSyms,2686      useDevicePtrSyms;2687  genTargetDataClauses(converter, semaCtx, stmtCtx, item->clauses, loc,2688                       clauseOps, useDeviceAddrSyms, useDevicePtrSyms);2689 2690  auto targetDataOp = mlir::omp::TargetDataOp::create(2691      converter.getFirOpBuilder(), loc, clauseOps);2692 2693  llvm::SmallVector<mlir::Value> useDeviceAddrBaseValues,2694      useDevicePtrBaseValues;2695  extractMappedBaseValues(clauseOps.useDeviceAddrVars, useDeviceAddrBaseValues);2696  extractMappedBaseValues(clauseOps.useDevicePtrVars, useDevicePtrBaseValues);2697 2698  EntryBlockArgs args;2699  args.useDeviceAddr.syms = useDeviceAddrSyms;2700  args.useDeviceAddr.vars = useDeviceAddrBaseValues;2701  args.useDevicePtr.syms = useDevicePtrSyms;2702  args.useDevicePtr.vars = useDevicePtrBaseValues;2703 2704  genBodyOfTargetDataOp(converter, symTable, semaCtx, eval, targetDataOp, args,2705                        loc, queue, item);2706  return targetDataOp;2707}2708 2709template <typename OpTy>2710static OpTy genTargetEnterExitUpdateDataOp(2711    lower::AbstractConverter &converter, lower::SymMap &symTable,2712    lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,2713    mlir::Location loc, const ConstructQueue &queue,2714    ConstructQueue::const_iterator item) {2715  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();2716 2717  // GCC 9.3.0 emits a (probably) bogus warning about an unused variable.2718  [[maybe_unused]] llvm::omp::Directive directive;2719  if constexpr (std::is_same_v<OpTy, mlir::omp::TargetEnterDataOp>) {2720    directive = llvm::omp::Directive::OMPD_target_enter_data;2721  } else if constexpr (std::is_same_v<OpTy, mlir::omp::TargetExitDataOp>) {2722    directive = llvm::omp::Directive::OMPD_target_exit_data;2723  } else if constexpr (std::is_same_v<OpTy, mlir::omp::TargetUpdateOp>) {2724    directive = llvm::omp::Directive::OMPD_target_update;2725  } else {2726    llvm_unreachable("Unexpected TARGET DATA construct");2727  }2728 2729  mlir::omp::TargetEnterExitUpdateDataOperands clauseOps;2730  genTargetEnterExitUpdateDataClauses(converter, semaCtx, symTable, stmtCtx,2731                                      item->clauses, loc, directive, clauseOps);2732 2733  return OpTy::create(firOpBuilder, loc, clauseOps);2734}2735 2736static mlir::omp::TaskOp2737genTaskOp(lower::AbstractConverter &converter, lower::SymMap &symTable,2738          lower::StatementContext &stmtCtx,2739          semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,2740          mlir::Location loc, const ConstructQueue &queue,2741          ConstructQueue::const_iterator item) {2742  mlir::omp::TaskOperands clauseOps;2743  llvm::SmallVector<const semantics::Symbol *> inReductionSyms;2744  genTaskClauses(converter, semaCtx, symTable, stmtCtx, item->clauses, loc,2745                 clauseOps, inReductionSyms);2746 2747  if (!enableDelayedPrivatization)2748    return genOpWithBody<mlir::omp::TaskOp>(2749        OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,2750                          llvm::omp::Directive::OMPD_task)2751            .setClauses(&item->clauses),2752        queue, item, clauseOps);2753 2754  DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,2755                           lower::omp::isLastItemInQueue(item, queue),2756                           /*useDelayedPrivatization=*/true, symTable);2757  dsp.processStep1(&clauseOps);2758 2759  EntryBlockArgs taskArgs;2760  taskArgs.priv.syms = dsp.getDelayedPrivSymbols();2761  taskArgs.priv.vars = clauseOps.privateVars;2762  taskArgs.inReduction.syms = inReductionSyms;2763  taskArgs.inReduction.vars = clauseOps.inReductionVars;2764 2765  return genOpWithBody<mlir::omp::TaskOp>(2766      OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,2767                        llvm::omp::Directive::OMPD_task)2768          .setClauses(&item->clauses)2769          .setDataSharingProcessor(&dsp)2770          .setEntryBlockArgs(&taskArgs),2771      queue, item, clauseOps);2772}2773 2774static mlir::omp::TaskgroupOp2775genTaskgroupOp(lower::AbstractConverter &converter, lower::SymMap &symTable,2776               semantics::SemanticsContext &semaCtx,2777               lower::pft::Evaluation &eval, mlir::Location loc,2778               const ConstructQueue &queue,2779               ConstructQueue::const_iterator item) {2780  mlir::omp::TaskgroupOperands clauseOps;2781  llvm::SmallVector<const semantics::Symbol *> taskReductionSyms;2782  genTaskgroupClauses(converter, semaCtx, item->clauses, loc, clauseOps,2783                      taskReductionSyms);2784 2785  EntryBlockArgs taskgroupArgs;2786  taskgroupArgs.taskReduction.syms = taskReductionSyms;2787  taskgroupArgs.taskReduction.vars = clauseOps.taskReductionVars;2788 2789  return genOpWithBody<mlir::omp::TaskgroupOp>(2790      OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,2791                        llvm::omp::Directive::OMPD_taskgroup)2792          .setClauses(&item->clauses)2793          .setEntryBlockArgs(&taskgroupArgs),2794      queue, item, clauseOps);2795}2796 2797static mlir::omp::TaskwaitOp2798genTaskwaitOp(lower::AbstractConverter &converter, lower::SymMap &symTable,2799              semantics::SemanticsContext &semaCtx,2800              lower::pft::Evaluation &eval, mlir::Location loc,2801              const ConstructQueue &queue,2802              ConstructQueue::const_iterator item) {2803  mlir::omp::TaskwaitOperands clauseOps;2804  genTaskwaitClauses(converter, semaCtx, item->clauses, loc, clauseOps);2805  return mlir::omp::TaskwaitOp::create(converter.getFirOpBuilder(), loc,2806                                       clauseOps);2807}2808 2809static mlir::omp::TaskyieldOp2810genTaskyieldOp(lower::AbstractConverter &converter, lower::SymMap &symTable,2811               semantics::SemanticsContext &semaCtx,2812               lower::pft::Evaluation &eval, mlir::Location loc,2813               const ConstructQueue &queue,2814               ConstructQueue::const_iterator item) {2815  return mlir::omp::TaskyieldOp::create(converter.getFirOpBuilder(), loc);2816}2817 2818static mlir::omp::WorkshareOp genWorkshareOp(2819    lower::AbstractConverter &converter, lower::SymMap &symTable,2820    lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,2821    lower::pft::Evaluation &eval, mlir::Location loc,2822    const ConstructQueue &queue, ConstructQueue::const_iterator item) {2823  mlir::omp::WorkshareOperands clauseOps;2824  genWorkshareClauses(converter, semaCtx, stmtCtx, item->clauses, loc,2825                      clauseOps);2826 2827  return genOpWithBody<mlir::omp::WorkshareOp>(2828      OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,2829                        llvm::omp::Directive::OMPD_workshare)2830          .setClauses(&item->clauses),2831      queue, item, clauseOps);2832}2833 2834static mlir::omp::TeamsOp2835genTeamsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,2836           lower::StatementContext &stmtCtx,2837           semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,2838           mlir::Location loc, const ConstructQueue &queue,2839           ConstructQueue::const_iterator item) {2840  lower::SymMapScope scope(symTable);2841  mlir::omp::TeamsOperands clauseOps;2842  llvm::SmallVector<const semantics::Symbol *> reductionSyms;2843  genTeamsClauses(converter, semaCtx, stmtCtx, item->clauses, loc, clauseOps,2844                  reductionSyms);2845 2846  EntryBlockArgs args;2847  // TODO: Add private syms and vars.2848  args.reduction.syms = reductionSyms;2849  args.reduction.vars = clauseOps.reductionVars;2850  return genOpWithBody<mlir::omp::TeamsOp>(2851      OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,2852                        llvm::omp::Directive::OMPD_teams)2853          .setClauses(&item->clauses)2854          .setEntryBlockArgs(&args),2855      queue, item, clauseOps);2856}2857 2858static mlir::omp::WorkdistributeOp genWorkdistributeOp(2859    lower::AbstractConverter &converter, lower::SymMap &symTable,2860    semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,2861    mlir::Location loc, const ConstructQueue &queue,2862    ConstructQueue::const_iterator item) {2863  return genOpWithBody<mlir::omp::WorkdistributeOp>(2864      OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,2865                        llvm::omp::Directive::OMPD_workdistribute),2866      queue, item);2867}2868 2869//===----------------------------------------------------------------------===//2870// Code generation functions for the standalone version of constructs that can2871// also be a leaf of a composite construct2872//===----------------------------------------------------------------------===//2873 2874static mlir::omp::DistributeOp genStandaloneDistribute(2875    lower::AbstractConverter &converter, lower::SymMap &symTable,2876    lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,2877    lower::pft::Evaluation &eval, mlir::Location loc,2878    const ConstructQueue &queue, ConstructQueue::const_iterator item) {2879  mlir::omp::DistributeOperands distributeClauseOps;2880  genDistributeClauses(converter, semaCtx, stmtCtx, item->clauses, loc,2881                       distributeClauseOps);2882 2883  DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,2884                           /*shouldCollectPreDeterminedSymbols=*/true,2885                           enableDelayedPrivatization, symTable);2886  dsp.processStep1(&distributeClauseOps);2887 2888  mlir::omp::LoopNestOperands loopNestClauseOps;2889  llvm::SmallVector<const semantics::Symbol *> iv;2890  genLoopNestClauses(converter, semaCtx, eval, item->clauses, loc,2891                     loopNestClauseOps, iv);2892 2893  EntryBlockArgs distributeArgs;2894  distributeArgs.priv.syms = dsp.getDelayedPrivSymbols();2895  distributeArgs.priv.vars = distributeClauseOps.privateVars;2896  auto distributeOp = genWrapperOp<mlir::omp::DistributeOp>(2897      converter, loc, distributeClauseOps, distributeArgs);2898 2899  genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, item,2900                loopNestClauseOps, iv, {{distributeOp, distributeArgs}},2901                llvm::omp::Directive::OMPD_distribute, dsp);2902  return distributeOp;2903}2904 2905static mlir::omp::WsloopOp genStandaloneDo(2906    lower::AbstractConverter &converter, lower::SymMap &symTable,2907    lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,2908    lower::pft::Evaluation &eval, mlir::Location loc,2909    const ConstructQueue &queue, ConstructQueue::const_iterator item) {2910  mlir::omp::WsloopOperands wsloopClauseOps;2911  llvm::SmallVector<const semantics::Symbol *> wsloopReductionSyms;2912  genWsloopClauses(converter, semaCtx, stmtCtx, item->clauses, loc,2913                   wsloopClauseOps, wsloopReductionSyms);2914 2915  DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,2916                           /*shouldCollectPreDeterminedSymbols=*/true,2917                           enableDelayedPrivatization, symTable);2918  dsp.processStep1(&wsloopClauseOps);2919 2920  mlir::omp::LoopNestOperands loopNestClauseOps;2921  llvm::SmallVector<const semantics::Symbol *> iv;2922  genLoopNestClauses(converter, semaCtx, eval, item->clauses, loc,2923                     loopNestClauseOps, iv);2924 2925  EntryBlockArgs wsloopArgs;2926  wsloopArgs.priv.syms = dsp.getDelayedPrivSymbols();2927  wsloopArgs.priv.vars = wsloopClauseOps.privateVars;2928  wsloopArgs.reduction.syms = wsloopReductionSyms;2929  wsloopArgs.reduction.vars = wsloopClauseOps.reductionVars;2930  auto wsloopOp = genWrapperOp<mlir::omp::WsloopOp>(2931      converter, loc, wsloopClauseOps, wsloopArgs);2932 2933  genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, item,2934                loopNestClauseOps, iv, {{wsloopOp, wsloopArgs}},2935                llvm::omp::Directive::OMPD_do, dsp);2936  return wsloopOp;2937}2938 2939static mlir::omp::ParallelOp genStandaloneParallel(2940    lower::AbstractConverter &converter, lower::SymMap &symTable,2941    lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,2942    lower::pft::Evaluation &eval, mlir::Location loc,2943    const ConstructQueue &queue, ConstructQueue::const_iterator item) {2944  lower::SymMapScope scope(symTable);2945  mlir::omp::ParallelOperands parallelClauseOps;2946  llvm::SmallVector<const semantics::Symbol *> parallelReductionSyms;2947  genParallelClauses(converter, semaCtx, stmtCtx, item->clauses, loc,2948                     parallelClauseOps, parallelReductionSyms);2949 2950  std::optional<DataSharingProcessor> dsp;2951  if (enableDelayedPrivatization) {2952    dsp.emplace(converter, semaCtx, item->clauses, eval,2953                lower::omp::isLastItemInQueue(item, queue),2954                /*useDelayedPrivatization=*/true, symTable);2955    dsp->processStep1(&parallelClauseOps);2956  }2957 2958  EntryBlockArgs parallelArgs;2959  if (dsp)2960    parallelArgs.priv.syms = dsp->getDelayedPrivSymbols();2961  parallelArgs.priv.vars = parallelClauseOps.privateVars;2962  parallelArgs.reduction.syms = parallelReductionSyms;2963  parallelArgs.reduction.vars = parallelClauseOps.reductionVars;2964  return genParallelOp(converter, symTable, semaCtx, eval, loc, queue, item,2965                       parallelClauseOps, parallelArgs,2966                       enableDelayedPrivatization ? &dsp.value() : nullptr);2967}2968 2969static mlir::omp::SimdOp2970genStandaloneSimd(lower::AbstractConverter &converter, lower::SymMap &symTable,2971                  semantics::SemanticsContext &semaCtx,2972                  lower::pft::Evaluation &eval, mlir::Location loc,2973                  const ConstructQueue &queue,2974                  ConstructQueue::const_iterator item) {2975  mlir::omp::SimdOperands simdClauseOps;2976  llvm::SmallVector<const semantics::Symbol *> simdReductionSyms;2977  genSimdClauses(converter, semaCtx, item->clauses, loc, simdClauseOps,2978                 simdReductionSyms);2979 2980  DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,2981                           /*shouldCollectPreDeterminedSymbols=*/true,2982                           enableDelayedPrivatization, symTable);2983  dsp.processStep1(&simdClauseOps);2984 2985  mlir::omp::LoopNestOperands loopNestClauseOps;2986  llvm::SmallVector<const semantics::Symbol *> iv;2987  genLoopNestClauses(converter, semaCtx, eval, item->clauses, loc,2988                     loopNestClauseOps, iv);2989 2990  EntryBlockArgs simdArgs;2991  simdArgs.priv.syms = dsp.getDelayedPrivSymbols();2992  simdArgs.priv.vars = simdClauseOps.privateVars;2993  simdArgs.reduction.syms = simdReductionSyms;2994  simdArgs.reduction.vars = simdClauseOps.reductionVars;2995  auto simdOp =2996      genWrapperOp<mlir::omp::SimdOp>(converter, loc, simdClauseOps, simdArgs);2997 2998  genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, item,2999                loopNestClauseOps, iv, {{simdOp, simdArgs}},3000                llvm::omp::Directive::OMPD_simd, dsp);3001  return simdOp;3002}3003 3004static mlir::omp::TaskloopOp genStandaloneTaskloop(3005    lower::AbstractConverter &converter, lower::SymMap &symTable,3006    lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,3007    lower::pft::Evaluation &eval, mlir::Location loc,3008    const ConstructQueue &queue, ConstructQueue::const_iterator item) {3009  mlir::omp::TaskloopOperands taskloopClauseOps;3010  llvm::SmallVector<const semantics::Symbol *> reductionSyms;3011  llvm::SmallVector<const semantics::Symbol *> inReductionSyms;3012 3013  genTaskloopClauses(converter, semaCtx, stmtCtx, item->clauses, loc,3014                     taskloopClauseOps, reductionSyms, inReductionSyms);3015  DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,3016                           /*shouldCollectPreDeterminedSymbols=*/true,3017                           enableDelayedPrivatization, symTable);3018  dsp.processStep1(&taskloopClauseOps);3019 3020  mlir::omp::LoopNestOperands loopNestClauseOps;3021  llvm::SmallVector<const semantics::Symbol *> iv;3022  genLoopNestClauses(converter, semaCtx, eval, item->clauses, loc,3023                     loopNestClauseOps, iv);3024 3025  EntryBlockArgs taskloopArgs;3026  taskloopArgs.priv.syms = dsp.getDelayedPrivSymbols();3027  taskloopArgs.priv.vars = taskloopClauseOps.privateVars;3028  taskloopArgs.reduction.syms = reductionSyms;3029  taskloopArgs.reduction.vars = taskloopClauseOps.reductionVars;3030  taskloopArgs.inReduction.syms = inReductionSyms;3031  taskloopArgs.inReduction.vars = taskloopClauseOps.inReductionVars;3032 3033  auto taskLoopOp = genWrapperOp<mlir::omp::TaskloopOp>(3034      converter, loc, taskloopClauseOps, taskloopArgs);3035 3036  genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, item,3037                loopNestClauseOps, iv, {{taskLoopOp, taskloopArgs}},3038                llvm::omp::Directive::OMPD_taskloop, dsp);3039  return taskLoopOp;3040}3041 3042//===----------------------------------------------------------------------===//3043// Code generation functions for composite constructs3044//===----------------------------------------------------------------------===//3045 3046static mlir::omp::DistributeOp genCompositeDistributeParallelDo(3047    lower::AbstractConverter &converter, lower::SymMap &symTable,3048    lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,3049    lower::pft::Evaluation &eval, mlir::Location loc,3050    const ConstructQueue &queue, ConstructQueue::const_iterator item) {3051  assert(std::distance(item, queue.end()) == 3 && "Invalid leaf constructs");3052  ConstructQueue::const_iterator distributeItem = item;3053  ConstructQueue::const_iterator parallelItem = std::next(distributeItem);3054  ConstructQueue::const_iterator doItem = std::next(parallelItem);3055 3056  // Create parent omp.parallel first.3057  mlir::omp::ParallelOperands parallelClauseOps;3058  llvm::SmallVector<const semantics::Symbol *> parallelReductionSyms;3059  genParallelClauses(converter, semaCtx, stmtCtx, parallelItem->clauses, loc,3060                     parallelClauseOps, parallelReductionSyms);3061 3062  DataSharingProcessor dsp(converter, semaCtx, doItem->clauses, eval,3063                           /*shouldCollectPreDeterminedSymbols=*/true,3064                           /*useDelayedPrivatization=*/true, symTable);3065  dsp.processStep1(&parallelClauseOps);3066 3067  EntryBlockArgs parallelArgs;3068  parallelArgs.priv.syms = dsp.getDelayedPrivSymbols();3069  parallelArgs.priv.vars = parallelClauseOps.privateVars;3070  parallelArgs.reduction.syms = parallelReductionSyms;3071  parallelArgs.reduction.vars = parallelClauseOps.reductionVars;3072  genParallelOp(converter, symTable, semaCtx, eval, loc, queue, parallelItem,3073                parallelClauseOps, parallelArgs, &dsp, /*isComposite=*/true);3074 3075  // Clause processing.3076  mlir::omp::DistributeOperands distributeClauseOps;3077  genDistributeClauses(converter, semaCtx, stmtCtx, distributeItem->clauses,3078                       loc, distributeClauseOps);3079 3080  mlir::omp::WsloopOperands wsloopClauseOps;3081  llvm::SmallVector<const semantics::Symbol *> wsloopReductionSyms;3082  genWsloopClauses(converter, semaCtx, stmtCtx, doItem->clauses, loc,3083                   wsloopClauseOps, wsloopReductionSyms);3084 3085  mlir::omp::LoopNestOperands loopNestClauseOps;3086  llvm::SmallVector<const semantics::Symbol *> iv;3087  genLoopNestClauses(converter, semaCtx, eval, doItem->clauses, loc,3088                     loopNestClauseOps, iv);3089 3090  // Operation creation.3091  EntryBlockArgs distributeArgs;3092  // TODO: Add private syms and vars.3093  auto distributeOp = genWrapperOp<mlir::omp::DistributeOp>(3094      converter, loc, distributeClauseOps, distributeArgs);3095  distributeOp.setComposite(/*val=*/true);3096 3097  EntryBlockArgs wsloopArgs;3098  // TODO: Add private syms and vars.3099  wsloopArgs.reduction.syms = wsloopReductionSyms;3100  wsloopArgs.reduction.vars = wsloopClauseOps.reductionVars;3101  auto wsloopOp = genWrapperOp<mlir::omp::WsloopOp>(3102      converter, loc, wsloopClauseOps, wsloopArgs);3103  wsloopOp.setComposite(/*val=*/true);3104 3105  genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, doItem,3106                loopNestClauseOps, iv,3107                {{distributeOp, distributeArgs}, {wsloopOp, wsloopArgs}},3108                llvm::omp::Directive::OMPD_distribute_parallel_do, dsp);3109  return distributeOp;3110}3111 3112static mlir::omp::DistributeOp genCompositeDistributeParallelDoSimd(3113    lower::AbstractConverter &converter, lower::SymMap &symTable,3114    lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,3115    lower::pft::Evaluation &eval, mlir::Location loc,3116    const ConstructQueue &queue, ConstructQueue::const_iterator item) {3117  assert(std::distance(item, queue.end()) == 4 && "Invalid leaf constructs");3118  ConstructQueue::const_iterator distributeItem = item;3119  ConstructQueue::const_iterator parallelItem = std::next(distributeItem);3120  ConstructQueue::const_iterator doItem = std::next(parallelItem);3121  ConstructQueue::const_iterator simdItem = std::next(doItem);3122 3123  // Create parent omp.parallel first.3124  mlir::omp::ParallelOperands parallelClauseOps;3125  llvm::SmallVector<const semantics::Symbol *> parallelReductionSyms;3126  genParallelClauses(converter, semaCtx, stmtCtx, parallelItem->clauses, loc,3127                     parallelClauseOps, parallelReductionSyms);3128 3129  DataSharingProcessor parallelItemDSP(3130      converter, semaCtx, parallelItem->clauses, eval,3131      /*shouldCollectPreDeterminedSymbols=*/false,3132      /*useDelayedPrivatization=*/true, symTable);3133  parallelItemDSP.processStep1(&parallelClauseOps);3134 3135  EntryBlockArgs parallelArgs;3136  parallelArgs.priv.syms = parallelItemDSP.getDelayedPrivSymbols();3137  parallelArgs.priv.vars = parallelClauseOps.privateVars;3138  parallelArgs.reduction.syms = parallelReductionSyms;3139  parallelArgs.reduction.vars = parallelClauseOps.reductionVars;3140  genParallelOp(converter, symTable, semaCtx, eval, loc, queue, parallelItem,3141                parallelClauseOps, parallelArgs, &parallelItemDSP,3142                /*isComposite=*/true);3143 3144  // Clause processing.3145  mlir::omp::DistributeOperands distributeClauseOps;3146  genDistributeClauses(converter, semaCtx, stmtCtx, distributeItem->clauses,3147                       loc, distributeClauseOps);3148 3149  mlir::omp::WsloopOperands wsloopClauseOps;3150  llvm::SmallVector<const semantics::Symbol *> wsloopReductionSyms;3151  genWsloopClauses(converter, semaCtx, stmtCtx, doItem->clauses, loc,3152                   wsloopClauseOps, wsloopReductionSyms);3153 3154  mlir::omp::SimdOperands simdClauseOps;3155  llvm::SmallVector<const semantics::Symbol *> simdReductionSyms;3156  genSimdClauses(converter, semaCtx, simdItem->clauses, loc, simdClauseOps,3157                 simdReductionSyms);3158 3159  DataSharingProcessor simdItemDSP(converter, semaCtx, simdItem->clauses, eval,3160                                   /*shouldCollectPreDeterminedSymbols=*/true,3161                                   /*useDelayedPrivatization=*/true, symTable);3162  simdItemDSP.processStep1(&simdClauseOps);3163 3164  mlir::omp::LoopNestOperands loopNestClauseOps;3165  llvm::SmallVector<const semantics::Symbol *> iv;3166  genLoopNestClauses(converter, semaCtx, eval, simdItem->clauses, loc,3167                     loopNestClauseOps, iv);3168 3169  // Operation creation.3170  EntryBlockArgs distributeArgs;3171  // TODO: Add private syms and vars.3172  auto distributeOp = genWrapperOp<mlir::omp::DistributeOp>(3173      converter, loc, distributeClauseOps, distributeArgs);3174  distributeOp.setComposite(/*val=*/true);3175 3176  EntryBlockArgs wsloopArgs;3177  // TODO: Add private syms and vars.3178  wsloopArgs.reduction.syms = wsloopReductionSyms;3179  wsloopArgs.reduction.vars = wsloopClauseOps.reductionVars;3180  auto wsloopOp = genWrapperOp<mlir::omp::WsloopOp>(3181      converter, loc, wsloopClauseOps, wsloopArgs);3182  wsloopOp.setComposite(/*val=*/true);3183 3184  EntryBlockArgs simdArgs;3185  simdArgs.priv.syms = simdItemDSP.getDelayedPrivSymbols();3186  simdArgs.priv.vars = simdClauseOps.privateVars;3187  simdArgs.reduction.syms = simdReductionSyms;3188  simdArgs.reduction.vars = simdClauseOps.reductionVars;3189  auto simdOp =3190      genWrapperOp<mlir::omp::SimdOp>(converter, loc, simdClauseOps, simdArgs);3191  simdOp.setComposite(/*val=*/true);3192 3193  genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, simdItem,3194                loopNestClauseOps, iv,3195                {{distributeOp, distributeArgs},3196                 {wsloopOp, wsloopArgs},3197                 {simdOp, simdArgs}},3198                llvm::omp::Directive::OMPD_distribute_parallel_do_simd,3199                simdItemDSP);3200  return distributeOp;3201}3202 3203static mlir::omp::DistributeOp genCompositeDistributeSimd(3204    lower::AbstractConverter &converter, lower::SymMap &symTable,3205    lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,3206    lower::pft::Evaluation &eval, mlir::Location loc,3207    const ConstructQueue &queue, ConstructQueue::const_iterator item) {3208  assert(std::distance(item, queue.end()) == 2 && "Invalid leaf constructs");3209  ConstructQueue::const_iterator distributeItem = item;3210  ConstructQueue::const_iterator simdItem = std::next(distributeItem);3211 3212  // Clause processing.3213  mlir::omp::DistributeOperands distributeClauseOps;3214  genDistributeClauses(converter, semaCtx, stmtCtx, distributeItem->clauses,3215                       loc, distributeClauseOps);3216 3217  mlir::omp::SimdOperands simdClauseOps;3218  llvm::SmallVector<const semantics::Symbol *> simdReductionSyms;3219  genSimdClauses(converter, semaCtx, simdItem->clauses, loc, simdClauseOps,3220                 simdReductionSyms);3221 3222  DataSharingProcessor distributeItemDSP(3223      converter, semaCtx, distributeItem->clauses, eval,3224      /*shouldCollectPreDeterminedSymbols=*/false,3225      /*useDelayedPrivatization=*/true, symTable);3226  distributeItemDSP.processStep1(&distributeClauseOps);3227 3228  DataSharingProcessor simdItemDSP(converter, semaCtx, simdItem->clauses, eval,3229                                   /*shouldCollectPreDeterminedSymbols=*/true,3230                                   /*useDelayedPrivatization=*/true, symTable);3231  simdItemDSP.processStep1(&simdClauseOps);3232 3233  // Pass the innermost leaf construct's clauses because that's where COLLAPSE3234  // is placed by construct decomposition.3235  mlir::omp::LoopNestOperands loopNestClauseOps;3236  llvm::SmallVector<const semantics::Symbol *> iv;3237  genLoopNestClauses(converter, semaCtx, eval, simdItem->clauses, loc,3238                     loopNestClauseOps, iv);3239 3240  // Operation creation.3241  EntryBlockArgs distributeArgs;3242  distributeArgs.priv.syms = distributeItemDSP.getDelayedPrivSymbols();3243  distributeArgs.priv.vars = distributeClauseOps.privateVars;3244  auto distributeOp = genWrapperOp<mlir::omp::DistributeOp>(3245      converter, loc, distributeClauseOps, distributeArgs);3246  distributeOp.setComposite(/*val=*/true);3247 3248  EntryBlockArgs simdArgs;3249  simdArgs.priv.syms = simdItemDSP.getDelayedPrivSymbols();3250  simdArgs.priv.vars = simdClauseOps.privateVars;3251  simdArgs.reduction.syms = simdReductionSyms;3252  simdArgs.reduction.vars = simdClauseOps.reductionVars;3253  auto simdOp =3254      genWrapperOp<mlir::omp::SimdOp>(converter, loc, simdClauseOps, simdArgs);3255  simdOp.setComposite(/*val=*/true);3256 3257  genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, simdItem,3258                loopNestClauseOps, iv,3259                {{distributeOp, distributeArgs}, {simdOp, simdArgs}},3260                llvm::omp::Directive::OMPD_distribute_simd, simdItemDSP);3261  return distributeOp;3262}3263 3264static mlir::omp::WsloopOp genCompositeDoSimd(3265    lower::AbstractConverter &converter, lower::SymMap &symTable,3266    lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,3267    lower::pft::Evaluation &eval, mlir::Location loc,3268    const ConstructQueue &queue, ConstructQueue::const_iterator item) {3269  assert(std::distance(item, queue.end()) == 2 && "Invalid leaf constructs");3270  ConstructQueue::const_iterator doItem = item;3271  ConstructQueue::const_iterator simdItem = std::next(doItem);3272 3273  // Clause processing.3274  mlir::omp::WsloopOperands wsloopClauseOps;3275  llvm::SmallVector<const semantics::Symbol *> wsloopReductionSyms;3276  genWsloopClauses(converter, semaCtx, stmtCtx, doItem->clauses, loc,3277                   wsloopClauseOps, wsloopReductionSyms);3278 3279  mlir::omp::SimdOperands simdClauseOps;3280  llvm::SmallVector<const semantics::Symbol *> simdReductionSyms;3281  genSimdClauses(converter, semaCtx, simdItem->clauses, loc, simdClauseOps,3282                 simdReductionSyms);3283 3284  DataSharingProcessor wsloopItemDSP(3285      converter, semaCtx, doItem->clauses, eval,3286      /*shouldCollectPreDeterminedSymbols=*/false,3287      /*useDelayedPrivatization=*/true, symTable);3288  wsloopItemDSP.processStep1(&wsloopClauseOps);3289 3290  DataSharingProcessor simdItemDSP(converter, semaCtx, simdItem->clauses, eval,3291                                   /*shouldCollectPreDeterminedSymbols=*/true,3292                                   /*useDelayedPrivatization=*/true, symTable);3293  simdItemDSP.processStep1(&simdClauseOps, simdItem->id);3294 3295  // Pass the innermost leaf construct's clauses because that's where COLLAPSE3296  // is placed by construct decomposition.3297  mlir::omp::LoopNestOperands loopNestClauseOps;3298  llvm::SmallVector<const semantics::Symbol *> iv;3299  genLoopNestClauses(converter, semaCtx, eval, simdItem->clauses, loc,3300                     loopNestClauseOps, iv);3301 3302  // Operation creation.3303  EntryBlockArgs wsloopArgs;3304  wsloopArgs.priv.syms = wsloopItemDSP.getDelayedPrivSymbols();3305  wsloopArgs.priv.vars = wsloopClauseOps.privateVars;3306  wsloopArgs.reduction.syms = wsloopReductionSyms;3307  wsloopArgs.reduction.vars = wsloopClauseOps.reductionVars;3308  auto wsloopOp = genWrapperOp<mlir::omp::WsloopOp>(3309      converter, loc, wsloopClauseOps, wsloopArgs);3310  wsloopOp.setComposite(/*val=*/true);3311 3312  EntryBlockArgs simdArgs;3313  simdArgs.priv.syms = simdItemDSP.getDelayedPrivSymbols();3314  simdArgs.priv.vars = simdClauseOps.privateVars;3315  simdArgs.reduction.syms = simdReductionSyms;3316  simdArgs.reduction.vars = simdClauseOps.reductionVars;3317  auto simdOp =3318      genWrapperOp<mlir::omp::SimdOp>(converter, loc, simdClauseOps, simdArgs);3319  simdOp.setComposite(/*val=*/true);3320 3321  genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, simdItem,3322                loopNestClauseOps, iv,3323                {{wsloopOp, wsloopArgs}, {simdOp, simdArgs}},3324                llvm::omp::Directive::OMPD_do_simd, simdItemDSP);3325  return wsloopOp;3326}3327 3328static mlir::omp::TaskloopOp genCompositeTaskloopSimd(3329    lower::AbstractConverter &converter, lower::SymMap &symTable,3330    lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,3331    lower::pft::Evaluation &eval, mlir::Location loc,3332    const ConstructQueue &queue, ConstructQueue::const_iterator item) {3333  assert(std::distance(item, queue.end()) == 2 && "Invalid leaf constructs");3334  if (!semaCtx.langOptions().OpenMPSimd)3335    TODO(loc, "Composite TASKLOOP SIMD");3336  return nullptr;3337}3338 3339//===----------------------------------------------------------------------===//3340// Dispatch3341//===----------------------------------------------------------------------===//3342 3343static bool genOMPCompositeDispatch(3344    lower::AbstractConverter &converter, lower::SymMap &symTable,3345    lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,3346    lower::pft::Evaluation &eval, mlir::Location loc,3347    const ConstructQueue &queue, ConstructQueue::const_iterator item,3348    mlir::Operation *&newOp) {3349  using llvm::omp::Directive;3350  using lower::omp::matchLeafSequence;3351 3352  // TODO: Privatization for composite constructs is currently only done based3353  // on the clauses for their last leaf construct, which may not always be3354  // correct. Consider per-leaf privatization of composite constructs once3355  // delayed privatization is supported by all participating ops.3356  if (matchLeafSequence(item, queue, Directive::OMPD_distribute_parallel_do))3357    newOp = genCompositeDistributeParallelDo(converter, symTable, stmtCtx,3358                                             semaCtx, eval, loc, queue, item);3359  else if (matchLeafSequence(item, queue,3360                             Directive::OMPD_distribute_parallel_do_simd))3361    newOp = genCompositeDistributeParallelDoSimd(3362        converter, symTable, stmtCtx, semaCtx, eval, loc, queue, item);3363  else if (matchLeafSequence(item, queue, Directive::OMPD_distribute_simd))3364    newOp = genCompositeDistributeSimd(converter, symTable, stmtCtx, semaCtx,3365                                       eval, loc, queue, item);3366  else if (matchLeafSequence(item, queue, Directive::OMPD_do_simd))3367    newOp = genCompositeDoSimd(converter, symTable, stmtCtx, semaCtx, eval, loc,3368                               queue, item);3369  else if (matchLeafSequence(item, queue, Directive::OMPD_taskloop_simd))3370    newOp = genCompositeTaskloopSimd(converter, symTable, stmtCtx, semaCtx,3371                                     eval, loc, queue, item);3372  else3373    return false;3374 3375  return true;3376}3377 3378static void genOMPDispatch(lower::AbstractConverter &converter,3379                           lower::SymMap &symTable,3380                           semantics::SemanticsContext &semaCtx,3381                           lower::pft::Evaluation &eval, mlir::Location loc,3382                           const ConstructQueue &queue,3383                           ConstructQueue::const_iterator item) {3384  assert(item != queue.end());3385 3386  lower::StatementContext stmtCtx;3387  mlir::Operation *newOp = nullptr;3388 3389  // Generate cleanup code for the stmtCtx after newOp3390  auto finalizeStmtCtx = [&]() {3391    if (newOp) {3392      fir::FirOpBuilder &builder = converter.getFirOpBuilder();3393      fir::FirOpBuilder::InsertionGuard guard(builder);3394      builder.setInsertionPointAfter(newOp);3395      stmtCtx.finalizeAndPop();3396    }3397  };3398 3399  bool loopLeaf = llvm::omp::getDirectiveAssociation(item->id) ==3400                  llvm::omp::Association::LoopNest;3401  if (loopLeaf) {3402    symTable.pushScope();3403    if (genOMPCompositeDispatch(converter, symTable, stmtCtx, semaCtx, eval,3404                                loc, queue, item, newOp)) {3405      symTable.popScope();3406      finalizeStmtCtx();3407      return;3408    }3409  }3410 3411  llvm::omp::Directive dir = item->id;3412  switch (dir) {3413  case llvm::omp::Directive::OMPD_barrier:3414    newOp = genBarrierOp(converter, symTable, semaCtx, eval, loc, queue, item);3415    break;3416  case llvm::omp::Directive::OMPD_distribute:3417    newOp = genStandaloneDistribute(converter, symTable, stmtCtx, semaCtx, eval,3418                                    loc, queue, item);3419    break;3420  case llvm::omp::Directive::OMPD_do:3421    newOp = genStandaloneDo(converter, symTable, stmtCtx, semaCtx, eval, loc,3422                            queue, item);3423    break;3424  case llvm::omp::Directive::OMPD_loop:3425    newOp = genLoopOp(converter, symTable, semaCtx, eval, loc, queue, item);3426    break;3427  case llvm::omp::Directive::OMPD_masked:3428    newOp = genMaskedOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue,3429                        item);3430    break;3431  case llvm::omp::Directive::OMPD_master:3432    newOp = genMasterOp(converter, symTable, semaCtx, eval, loc, queue, item);3433    break;3434  case llvm::omp::Directive::OMPD_ordered:3435    // Block-associated "ordered" construct.3436    newOp = genOrderedRegionOp(converter, symTable, semaCtx, eval, loc, queue,3437                               item);3438    break;3439  case llvm::omp::Directive::OMPD_parallel:3440    newOp = genStandaloneParallel(converter, symTable, stmtCtx, semaCtx, eval,3441                                  loc, queue, item);3442    break;3443  case llvm::omp::Directive::OMPD_scan:3444    newOp = genScanOp(converter, symTable, semaCtx, loc, queue, item);3445    break;3446  case llvm::omp::Directive::OMPD_section:3447    llvm_unreachable("genOMPDispatch: OMPD_section");3448    // Lowered in the enclosing genSectionsOp.3449    break;3450  case llvm::omp::Directive::OMPD_sections:3451    genSectionsOp(converter, symTable, semaCtx, eval, loc, queue, item);3452    break;3453  case llvm::omp::Directive::OMPD_simd:3454    newOp =3455        genStandaloneSimd(converter, symTable, semaCtx, eval, loc, queue, item);3456    break;3457  case llvm::omp::Directive::OMPD_scope:3458    newOp = genScopeOp(converter, symTable, semaCtx, eval, loc, queue, item);3459    break;3460  case llvm::omp::Directive::OMPD_single:3461    newOp = genSingleOp(converter, symTable, semaCtx, eval, loc, queue, item);3462    break;3463  case llvm::omp::Directive::OMPD_target:3464    newOp = genTargetOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue,3465                        item);3466    break;3467  case llvm::omp::Directive::OMPD_target_data:3468    newOp = genTargetDataOp(converter, symTable, stmtCtx, semaCtx, eval, loc,3469                            queue, item);3470    break;3471  case llvm::omp::Directive::OMPD_target_enter_data:3472    newOp = genTargetEnterExitUpdateDataOp<mlir::omp::TargetEnterDataOp>(3473        converter, symTable, stmtCtx, semaCtx, loc, queue, item);3474    break;3475  case llvm::omp::Directive::OMPD_target_exit_data:3476    newOp = genTargetEnterExitUpdateDataOp<mlir::omp::TargetExitDataOp>(3477        converter, symTable, stmtCtx, semaCtx, loc, queue, item);3478    break;3479  case llvm::omp::Directive::OMPD_target_update:3480    newOp = genTargetEnterExitUpdateDataOp<mlir::omp::TargetUpdateOp>(3481        converter, symTable, stmtCtx, semaCtx, loc, queue, item);3482    break;3483  case llvm::omp::Directive::OMPD_task:3484    newOp = genTaskOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue,3485                      item);3486    break;3487  case llvm::omp::Directive::OMPD_taskgroup:3488    newOp =3489        genTaskgroupOp(converter, symTable, semaCtx, eval, loc, queue, item);3490    break;3491  case llvm::omp::Directive::OMPD_taskloop:3492    newOp = genStandaloneTaskloop(converter, symTable, stmtCtx, semaCtx, eval,3493                                  loc, queue, item);3494    break;3495  case llvm::omp::Directive::OMPD_taskwait:3496    newOp = genTaskwaitOp(converter, symTable, semaCtx, eval, loc, queue, item);3497    break;3498  case llvm::omp::Directive::OMPD_taskyield:3499    newOp =3500        genTaskyieldOp(converter, symTable, semaCtx, eval, loc, queue, item);3501    break;3502  case llvm::omp::Directive::OMPD_teams:3503    newOp = genTeamsOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue,3504                       item);3505    break;3506  case llvm::omp::Directive::OMPD_tile:3507    genTileOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue, item);3508    break;3509  case llvm::omp::Directive::OMPD_fuse: {3510    unsigned version = semaCtx.langOptions().OpenMPVersion;3511    if (!semaCtx.langOptions().OpenMPSimd)3512      TODO(loc, "Unhandled loop directive (" +3513                    llvm::omp::getOpenMPDirectiveName(dir, version) + ")");3514    break;3515  }3516  case llvm::omp::Directive::OMPD_unroll:3517    genUnrollOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue, item);3518    break;3519  case llvm::omp::Directive::OMPD_workdistribute:3520    newOp = genWorkdistributeOp(converter, symTable, semaCtx, eval, loc, queue,3521                                item);3522    break;3523  case llvm::omp::Directive::OMPD_workshare:3524    newOp = genWorkshareOp(converter, symTable, stmtCtx, semaCtx, eval, loc,3525                           queue, item);3526    break;3527  default:3528    // Combined and composite constructs should have been split into a sequence3529    // of leaf constructs when building the construct queue.3530    assert(!llvm::omp::isLeafConstruct(dir) &&3531           "Unexpected compound construct.");3532    break;3533  }3534 3535  finalizeStmtCtx();3536  if (loopLeaf)3537    symTable.popScope();3538}3539 3540//===----------------------------------------------------------------------===//3541// OpenMPDeclarativeConstruct visitors3542//===----------------------------------------------------------------------===//3543static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,3544                   semantics::SemanticsContext &semaCtx,3545                   lower::pft::Evaluation &eval,3546                   const parser::OpenMPUtilityConstruct &);3547 3548static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,3549                   semantics::SemanticsContext &semaCtx,3550                   lower::pft::Evaluation &eval,3551                   const parser::OmpAllocateDirective &allocate) {3552  if (!semaCtx.langOptions().OpenMPSimd)3553    TODO(converter.getCurrentLocation(), "OmpAllocateDirective");3554}3555 3556static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,3557                   semantics::SemanticsContext &semaCtx,3558                   lower::pft::Evaluation &eval,3559                   const parser::OpenMPDeclarativeAssumes &assumesConstruct) {3560  if (!semaCtx.langOptions().OpenMPSimd)3561    TODO(converter.getCurrentLocation(), "OpenMP ASSUMES declaration");3562}3563 3564static void3565genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,3566       semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,3567       const parser::OmpDeclareVariantDirective &declareVariantDirective) {3568  if (!semaCtx.langOptions().OpenMPSimd)3569    TODO(converter.getCurrentLocation(), "OmpDeclareVariantDirective");3570}3571 3572static ReductionProcessor::GenCombinerCBTy3573processReductionCombiner(lower::AbstractConverter &converter,3574                         lower::SymMap &symTable,3575                         semantics::SemanticsContext &semaCtx,3576                         const parser::OmpReductionSpecifier &specifier) {3577  ReductionProcessor::GenCombinerCBTy genCombinerCB;3578  const auto &combinerExpression =3579      std::get<std::optional<parser::OmpCombinerExpression>>(specifier.t)3580          .value();3581  const parser::OmpStylizedInstance &combinerInstance =3582      combinerExpression.v.front();3583  const parser::OmpStylizedInstance::Instance &instance =3584      std::get<parser::OmpStylizedInstance::Instance>(combinerInstance.t);3585 3586  std::optional<semantics::SomeExpr> evalExprOpt;3587  if (const auto *as = std::get_if<parser::AssignmentStmt>(&instance.u)) {3588    auto &expr = std::get<parser::Expr>(as->t);3589    evalExprOpt = makeExpr(expr, semaCtx);3590  } else if (const auto *call = std::get_if<parser::CallStmt>(&instance.u)) {3591    if (call->typedCall) {3592      const auto &procRef = *call->typedCall;3593      evalExprOpt = semantics::SomeExpr{procRef};3594    } else {3595      TODO(converter.getCurrentLocation(),3596           "CallStmt without typedCall is not yet supported");3597    }3598  } else {3599    TODO(converter.getCurrentLocation(), "Unsupported combiner instance type");3600  }3601 3602  assert(evalExprOpt.has_value() && "evalExpr must be initialized");3603  semantics::SomeExpr evalExpr = *evalExprOpt;3604 3605  genCombinerCB = [&, evalExpr](fir::FirOpBuilder &builder, mlir::Location loc,3606                                mlir::Type type, mlir::Value lhs,3607                                mlir::Value rhs, bool isByRef) {3608    lower::SymMapScope scope(symTable);3609    const std::list<parser::OmpStylizedDeclaration> &declList =3610        std::get<std::list<parser::OmpStylizedDeclaration>>(combinerInstance.t);3611    mlir::Value ompOutVar;3612    for (const parser::OmpStylizedDeclaration &decl : declList) {3613      auto &name = std::get<parser::ObjectName>(decl.var.t);3614      mlir::Value addr = lhs;3615      mlir::Type type = lhs.getType();3616      bool isRhs = name.ToString() == std::string("omp_in");3617      if (isRhs) {3618        addr = rhs;3619        type = rhs.getType();3620      }3621 3622      assert(name.symbol && "Reduction object name does not have a symbol");3623      if (!fir::conformsWithPassByRef(type)) {3624        addr = builder.createTemporary(loc, type);3625        fir::StoreOp::create(builder, loc, isRhs ? rhs : lhs, addr);3626      }3627      fir::FortranVariableFlagsEnum extraFlags = {};3628      fir::FortranVariableFlagsAttr attributes =3629          Fortran::lower::translateSymbolAttributes(builder.getContext(),3630                                                    *name.symbol, extraFlags);3631      auto declareOp =3632          hlfir::DeclareOp::create(builder, loc, addr, name.ToString(), nullptr,3633                                   {}, nullptr, nullptr, 0, attributes);3634      if (name.ToString() == "omp_out")3635        ompOutVar = declareOp.getResult(0);3636      symTable.addVariableDefinition(*name.symbol, declareOp);3637    }3638 3639    lower::StatementContext stmtCtx;3640    mlir::Value result = common::visit(3641        common::visitors{3642            [&](const evaluate::ProcedureRef &procRef) -> mlir::Value {3643              convertCallToHLFIR(loc, converter, procRef, std::nullopt,3644                                 symTable, stmtCtx);3645              auto outVal = fir::LoadOp::create(builder, loc, ompOutVar);3646              return outVal;3647            },3648            [&](const auto &expr) -> mlir::Value {3649              mlir::Value exprResult = fir::getBase(convertExprToValue(3650                  loc, converter, evalExpr, symTable, stmtCtx));3651              // Optional load may be generated if we get a reference to the3652              // reduction type.3653              if (auto refType =3654                      llvm::dyn_cast<fir::ReferenceType>(exprResult.getType()))3655                if (lhs.getType() == refType.getElementType())3656                  exprResult = fir::LoadOp::create(builder, loc, exprResult);3657              return exprResult;3658            }},3659        evalExpr.u);3660    stmtCtx.finalizeAndPop();3661    if (isByRef) {3662      fir::StoreOp::create(builder, loc, result, lhs);3663      mlir::omp::YieldOp::create(builder, loc, lhs);3664    } else {3665      mlir::omp::YieldOp::create(builder, loc, result);3666    }3667  };3668  return genCombinerCB;3669}3670 3671// Checks that the reduction type is either a trivial type or a derived type of3672// trivial types.3673static bool isSimpleReductionType(mlir::Type reductionType) {3674  if (fir::isa_trivial(reductionType))3675    return true;3676  if (auto recordTy = mlir::dyn_cast<fir::RecordType>(reductionType)) {3677    for (auto [_, fieldType] : recordTy.getTypeList()) {3678      if (!fir::isa_trivial(fieldType))3679        return false;3680    }3681  }3682  return true;3683}3684 3685// Getting the type from a symbol compared to a DeclSpec is simpler since we do3686// not need to consider derived vs intrinsic types. Semantics is guaranteed to3687// generate these symbols.3688static mlir::Type3689getReductionType(lower::AbstractConverter &converter,3690                 const parser::OmpReductionSpecifier &specifier) {3691  const auto &combinerExpression =3692      std::get<std::optional<parser::OmpCombinerExpression>>(specifier.t)3693          .value();3694  const parser::OmpStylizedInstance &combinerInstance =3695      combinerExpression.v.front();3696  const std::list<parser::OmpStylizedDeclaration> &declList =3697      std::get<std::list<parser::OmpStylizedDeclaration>>(combinerInstance.t);3698  const parser::OmpStylizedDeclaration &decl = declList.front();3699  const auto &name = std::get<parser::ObjectName>(decl.var.t);3700  const auto &symbol = semantics::SymbolRef(*name.symbol);3701  mlir::Type reductionType = converter.genType(symbol);3702 3703  if (!isSimpleReductionType(reductionType))3704    TODO(converter.getCurrentLocation(),3705         "declare reduction currently only supports trival types or derived "3706         "types containing trivial types");3707  return reductionType;3708}3709 3710static void genOMP(3711    lower::AbstractConverter &converter, lower::SymMap &symTable,3712    semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,3713    const parser::OpenMPDeclareReductionConstruct &declareReductionConstruct) {3714  if (semaCtx.langOptions().OpenMPSimd)3715    return;3716 3717  const parser::OmpArgumentList &args{declareReductionConstruct.v.Arguments()};3718  const parser::OmpArgument &arg{args.v.front()};3719  const auto &specifier = std::get<parser::OmpReductionSpecifier>(arg.u);3720 3721  if (std::get<parser::OmpTypeNameList>(specifier.t).v.size() > 1)3722    TODO(converter.getCurrentLocation(),3723         "multiple types in declare reduction is not yet supported");3724 3725  mlir::Type reductionType = getReductionType(converter, specifier);3726  ReductionProcessor::GenCombinerCBTy genCombinerCB =3727      processReductionCombiner(converter, symTable, semaCtx, specifier);3728  const parser::OmpClauseList &initializer =3729      declareReductionConstruct.v.Clauses();3730  if (initializer.v.size() > 0) {3731    List<Clause> clauses = makeClauses(initializer, semaCtx);3732    ReductionProcessor::GenInitValueCBTy genInitValueCB;3733    ClauseProcessor cp(converter, semaCtx, clauses);3734    const parser::OmpClause::Initializer &iclause{3735        std::get<parser::OmpClause::Initializer>(initializer.v.front().u)};3736    cp.processInitializer(symTable, iclause, genInitValueCB);3737    const auto &identifier =3738        std::get<parser::OmpReductionIdentifier>(specifier.t);3739    const auto &designator =3740        std::get<parser::ProcedureDesignator>(identifier.u);3741    const auto &reductionName = std::get<parser::Name>(designator.u);3742    bool isByRef = ReductionProcessor::doReductionByRef(reductionType);3743    ReductionProcessor::createDeclareReductionHelper<3744        mlir::omp::DeclareReductionOp>(3745        converter, reductionName.ToString(), reductionType,3746        converter.getCurrentLocation(), isByRef, genCombinerCB, genInitValueCB);3747  } else {3748    TODO(converter.getCurrentLocation(),3749         "declare reduction without an initializer clause is not yet "3750         "supported");3751  }3752}3753 3754static void3755genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,3756       semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,3757       const parser::OpenMPDeclareSimdConstruct &declareSimdConstruct) {3758  if (!semaCtx.langOptions().OpenMPSimd)3759    TODO(converter.getCurrentLocation(), "OpenMPDeclareSimdConstruct");3760}3761 3762static void genOpenMPDeclareMapperImpl(3763    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,3764    const parser::OpenMPDeclareMapperConstruct &construct,3765    const semantics::Symbol *mapperSymOpt = nullptr) {3766  mlir::Location loc = converter.genLocation(construct.source);3767  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();3768  const parser::OmpArgumentList &args = construct.v.Arguments();3769  assert(args.v.size() == 1 && "Expecting single argument");3770  lower::StatementContext stmtCtx;3771  const auto *spec = std::get_if<parser::OmpMapperSpecifier>(&args.v.front().u);3772  assert(spec && "Expecting mapper specifier");3773  const auto &mapperName{std::get<std::string>(spec->t)};3774  const auto &varType{std::get<parser::TypeSpec>(spec->t)};3775  const auto &varName{std::get<parser::Name>(spec->t)};3776  assert(varType.declTypeSpec->category() ==3777             semantics::DeclTypeSpec::Category::TypeDerived &&3778         "Expected derived type");3779 3780  std::string mapperNameStr = mapperName;3781  if (mapperSymOpt && mapperNameStr != "default") {3782    mapperNameStr = converter.mangleName(mapperNameStr, mapperSymOpt->owner());3783  } else if (auto *sym =3784                 converter.getCurrentScope().FindSymbol(mapperNameStr)) {3785    mapperNameStr = converter.mangleName(mapperNameStr, sym->owner());3786  }3787 3788  // If the mapper op already exists (e.g., created by regular lowering or by3789  // materialization of imported mappers), do not recreate it.3790  if (converter.getModuleOp().lookupSymbol(mapperNameStr))3791    return;3792 3793  // Save current insertion point before moving to the module scope to create3794  // the DeclareMapperOp3795  mlir::OpBuilder::InsertionGuard guard(firOpBuilder);3796 3797  firOpBuilder.setInsertionPointToStart(converter.getModuleOp().getBody());3798  auto mlirType = converter.genType(varType.declTypeSpec->derivedTypeSpec());3799  auto declMapperOp = mlir::omp::DeclareMapperOp::create(3800      firOpBuilder, loc, mapperNameStr, mlirType);3801  auto &region = declMapperOp.getRegion();3802  firOpBuilder.createBlock(&region);3803  auto varVal = region.addArgument(firOpBuilder.getRefType(mlirType), loc);3804  converter.bindSymbol(*varName.symbol, varVal);3805 3806  // Populate the declareMapper region with the map information.3807  mlir::omp::DeclareMapperInfoOperands clauseOps;3808  List<Clause> clauses = makeClauses(construct.v.Clauses(), semaCtx);3809  ClauseProcessor cp(converter, semaCtx, clauses);3810  cp.processMap(loc, stmtCtx, clauseOps);3811  mlir::omp::DeclareMapperInfoOp::create(firOpBuilder, loc, clauseOps.mapVars);3812}3813 3814static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,3815                   semantics::SemanticsContext &semaCtx,3816                   lower::pft::Evaluation &eval,3817                   const parser::OpenMPDeclareMapperConstruct &construct) {3818  genOpenMPDeclareMapperImpl(converter, semaCtx, construct);3819}3820 3821static void3822genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,3823       semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,3824       const parser::OpenMPDeclareTargetConstruct &declareTargetConstruct) {3825  mlir::omp::DeclareTargetOperands clauseOps;3826  llvm::SmallVector<DeclareTargetCaptureInfo> symbolAndClause;3827  mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();3828  getDeclareTargetInfo(converter, semaCtx, eval, declareTargetConstruct,3829                       clauseOps, symbolAndClause);3830 3831  for (const DeclareTargetCaptureInfo &symClause : symbolAndClause) {3832    mlir::Operation *op =3833        mod.lookupSymbol(converter.mangleName(symClause.symbol));3834 3835    // Some symbols are deferred until later in the module, these are handled3836    // upon finalization of the module for OpenMP inside of Bridge, so we simply3837    // skip for now.3838    if (!op)3839      continue;3840 3841    markDeclareTarget(op, converter, symClause.clause, clauseOps.deviceType,3842                      symClause.automap);3843  }3844}3845 3846static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,3847                   semantics::SemanticsContext &semaCtx,3848                   lower::pft::Evaluation &eval,3849                   const parser::OpenMPGroupprivate &directive) {3850  TODO(converter.getCurrentLocation(), "GROUPPRIVATE");3851}3852 3853static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,3854                   semantics::SemanticsContext &semaCtx,3855                   lower::pft::Evaluation &eval,3856                   const parser::OpenMPRequiresConstruct &requiresConstruct) {3857  // Requires directives are gathered and processed in semantics and3858  // then combined in the lowering bridge before triggering codegen3859  // just once. Hence, there is no need to lower each individual3860  // occurrence here.3861}3862 3863static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,3864                   semantics::SemanticsContext &semaCtx,3865                   lower::pft::Evaluation &eval,3866                   const parser::OpenMPThreadprivate &threadprivate) {3867  // The directive is lowered when instantiating the variable to3868  // support the case of threadprivate variable declared in module.3869}3870 3871static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,3872                   semantics::SemanticsContext &semaCtx,3873                   lower::pft::Evaluation &eval,3874                   const parser::OmpMetadirectiveDirective &meta) {3875  TODO(converter.getCurrentLocation(), "METADIRECTIVE");3876}3877 3878static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,3879                   semantics::SemanticsContext &semaCtx,3880                   lower::pft::Evaluation &eval,3881                   const parser::OpenMPDeclarativeConstruct &ompDeclConstruct) {3882  Fortran::common::visit(3883      [&](auto &&s) { return genOMP(converter, symTable, semaCtx, eval, s); },3884      ompDeclConstruct.u);3885}3886 3887//===----------------------------------------------------------------------===//3888// OpenMPStandaloneConstruct visitors3889//===----------------------------------------------------------------------===//3890 3891static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,3892                   semantics::SemanticsContext &semaCtx,3893                   lower::pft::Evaluation &eval,3894                   const parser::OpenMPSimpleStandaloneConstruct &construct) {3895  const auto &directive = std::get<parser::OmpDirectiveName>(construct.v.t);3896  List<Clause> clauses = makeClauses(construct.v.Clauses(), semaCtx);3897  mlir::Location currentLocation = converter.genLocation(directive.source);3898 3899  ConstructQueue queue{3900      buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,3901                          eval, directive.source, directive.v, clauses)};3902  if (directive.v == llvm::omp::Directive::OMPD_ordered) {3903    // Standalone "ordered" directive.3904    genOrderedOp(converter, symTable, semaCtx, eval, currentLocation, queue,3905                 queue.begin());3906  } else {3907    // Dispatch handles the "block-associated" variant of "ordered".3908    genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,3909                   queue.begin());3910  }3911}3912 3913static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,3914                   semantics::SemanticsContext &semaCtx,3915                   lower::pft::Evaluation &eval,3916                   const parser::OpenMPFlushConstruct &construct) {3917  const auto &argumentList = construct.v.Arguments();3918  const auto &clauseList = construct.v.Clauses();3919  ObjectList objects = makeObjects(argumentList, semaCtx);3920  List<Clause> clauses =3921      makeList(clauseList.v, [&](auto &&s) { return makeClause(s, semaCtx); });3922  mlir::Location currentLocation = converter.genLocation(construct.source);3923 3924  ConstructQueue queue{buildConstructQueue(3925      converter.getFirOpBuilder().getModule(), semaCtx, eval, construct.source,3926      llvm::omp::Directive::OMPD_flush, clauses)};3927  genFlushOp(converter, symTable, semaCtx, eval, currentLocation, objects,3928             queue, queue.begin());3929}3930 3931static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,3932                   semantics::SemanticsContext &semaCtx,3933                   lower::pft::Evaluation &eval,3934                   const parser::OpenMPCancelConstruct &cancelConstruct) {3935  List<Clause> clauses = makeList(cancelConstruct.v.Clauses().v, [&](auto &&s) {3936    return makeClause(s, semaCtx);3937  });3938  mlir::Location loc = converter.genLocation(cancelConstruct.source);3939 3940  ConstructQueue queue{buildConstructQueue(3941      converter.getFirOpBuilder().getModule(), semaCtx, eval,3942      cancelConstruct.source, llvm::omp::Directive::OMPD_cancel, clauses)};3943  genCancelOp(converter, semaCtx, eval, loc, queue, queue.begin());3944}3945 3946static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,3947                   semantics::SemanticsContext &semaCtx,3948                   lower::pft::Evaluation &eval,3949                   const parser::OpenMPCancellationPointConstruct3950                       &cancellationPointConstruct) {3951  List<Clause> clauses =3952      makeList(cancellationPointConstruct.v.Clauses().v,3953               [&](auto &&s) { return makeClause(s, semaCtx); });3954  mlir::Location loc = converter.genLocation(cancellationPointConstruct.source);3955 3956  ConstructQueue queue{3957      buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,3958                          eval, cancellationPointConstruct.source,3959                          llvm::omp::Directive::OMPD_cancel, clauses)};3960  genCancellationPointOp(converter, semaCtx, eval, loc, queue, queue.begin());3961}3962 3963static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,3964                   semantics::SemanticsContext &semaCtx,3965                   lower::pft::Evaluation &eval,3966                   const parser::OpenMPDepobjConstruct &construct) {3967  // These values will be ignored until the construct itself is implemented,3968  // but run them anyway for the sake of testing (via a Todo test).3969  ObjectList objects = makeObjects(construct.v.Arguments(), semaCtx);3970  assert(objects.size() == 1);3971  List<Clause> clauses = makeClauses(construct.v.Clauses(), semaCtx);3972  assert(clauses.size() == 1);3973  (void)objects;3974  (void)clauses;3975 3976  if (!semaCtx.langOptions().OpenMPSimd)3977    TODO(converter.getCurrentLocation(), "OpenMPDepobjConstruct");3978}3979 3980static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,3981                   semantics::SemanticsContext &semaCtx,3982                   lower::pft::Evaluation &eval,3983                   const parser::OpenMPInteropConstruct &interopConstruct) {3984  if (!semaCtx.langOptions().OpenMPSimd)3985    TODO(converter.getCurrentLocation(), "OpenMPInteropConstruct");3986}3987 3988static void3989genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,3990       semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,3991       const parser::OpenMPStandaloneConstruct &standaloneConstruct) {3992  Fortran::common::visit(3993      [&](auto &&s) { return genOMP(converter, symTable, semaCtx, eval, s); },3994      standaloneConstruct.u);3995}3996 3997static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,3998                   semantics::SemanticsContext &semaCtx,3999                   lower::pft::Evaluation &eval,4000                   const parser::OpenMPAllocatorsConstruct &allocsConstruct) {4001  if (!semaCtx.langOptions().OpenMPSimd)4002    TODO(converter.getCurrentLocation(), "OpenMPAllocatorsConstruct");4003}4004 4005//===----------------------------------------------------------------------===//4006// OpenMPConstruct visitors4007//===----------------------------------------------------------------------===//4008 4009static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,4010                   semantics::SemanticsContext &semaCtx,4011                   lower::pft::Evaluation &eval,4012                   const parser::OpenMPAtomicConstruct &construct) {4013  lowerAtomic(converter, symTable, semaCtx, eval, construct);4014}4015 4016static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,4017                   semantics::SemanticsContext &semaCtx,4018                   lower::pft::Evaluation &eval,4019                   const parser::OmpBlockConstruct &blockConstruct) {4020  const parser::OmpDirectiveSpecification &beginSpec =4021      blockConstruct.BeginDir();4022  List<Clause> clauses = makeClauses(beginSpec.Clauses(), semaCtx);4023  if (auto &endSpec = blockConstruct.EndDir())4024    clauses.append(makeClauses(endSpec->Clauses(), semaCtx));4025 4026  llvm::omp::Directive directive = beginSpec.DirId();4027  assert(llvm::omp::blockConstructSet.test(directive) &&4028         "Expected block construct");4029  mlir::Location currentLocation = converter.genLocation(beginSpec.source);4030 4031  for (const Clause &clause : clauses) {4032    mlir::Location clauseLocation = converter.genLocation(clause.source);4033    if (!std::holds_alternative<clause::Affinity>(clause.u) &&4034        !std::holds_alternative<clause::Allocate>(clause.u) &&4035        !std::holds_alternative<clause::Copyin>(clause.u) &&4036        !std::holds_alternative<clause::Copyprivate>(clause.u) &&4037        !std::holds_alternative<clause::Default>(clause.u) &&4038        !std::holds_alternative<clause::Defaultmap>(clause.u) &&4039        !std::holds_alternative<clause::Depend>(clause.u) &&4040        !std::holds_alternative<clause::Filter>(clause.u) &&4041        !std::holds_alternative<clause::Final>(clause.u) &&4042        !std::holds_alternative<clause::Firstprivate>(clause.u) &&4043        !std::holds_alternative<clause::HasDeviceAddr>(clause.u) &&4044        !std::holds_alternative<clause::If>(clause.u) &&4045        !std::holds_alternative<clause::IsDevicePtr>(clause.u) &&4046        !std::holds_alternative<clause::Map>(clause.u) &&4047        !std::holds_alternative<clause::Nowait>(clause.u) &&4048        !std::holds_alternative<clause::NumTeams>(clause.u) &&4049        !std::holds_alternative<clause::NumThreads>(clause.u) &&4050        !std::holds_alternative<clause::OmpxBare>(clause.u) &&4051        !std::holds_alternative<clause::Priority>(clause.u) &&4052        !std::holds_alternative<clause::Private>(clause.u) &&4053        !std::holds_alternative<clause::ProcBind>(clause.u) &&4054        !std::holds_alternative<clause::Reduction>(clause.u) &&4055        !std::holds_alternative<clause::Shared>(clause.u) &&4056        !std::holds_alternative<clause::Simd>(clause.u) &&4057        !std::holds_alternative<clause::ThreadLimit>(clause.u) &&4058        !std::holds_alternative<clause::Threads>(clause.u) &&4059        !std::holds_alternative<clause::UseDeviceAddr>(clause.u) &&4060        !std::holds_alternative<clause::UseDevicePtr>(clause.u) &&4061        !std::holds_alternative<clause::InReduction>(clause.u) &&4062        !std::holds_alternative<clause::Mergeable>(clause.u) &&4063        !std::holds_alternative<clause::Untied>(clause.u) &&4064        !std::holds_alternative<clause::TaskReduction>(clause.u) &&4065        !std::holds_alternative<clause::Detach>(clause.u)) {4066      std::string name =4067          parser::ToUpperCaseLetters(llvm::omp::getOpenMPClauseName(clause.id));4068      if (!semaCtx.langOptions().OpenMPSimd)4069        TODO(clauseLocation, name + " clause is not implemented yet");4070    }4071  }4072 4073  ConstructQueue queue{4074      buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,4075                          eval, beginSpec.source, directive, clauses)};4076  genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,4077                 queue.begin());4078}4079 4080static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,4081                   semantics::SemanticsContext &semaCtx,4082                   lower::pft::Evaluation &eval,4083                   const parser::OpenMPAssumeConstruct &assumeConstruct) {4084  mlir::Location clauseLocation = converter.genLocation(assumeConstruct.source);4085  if (!semaCtx.langOptions().OpenMPSimd)4086    TODO(clauseLocation, "OpenMP ASSUME construct");4087}4088 4089static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,4090                   semantics::SemanticsContext &semaCtx,4091                   lower::pft::Evaluation &eval,4092                   const parser::OpenMPCriticalConstruct &criticalConstruct) {4093  const parser::OmpDirectiveSpecification &beginSpec =4094      criticalConstruct.BeginDir();4095  List<Clause> clauses = makeClauses(beginSpec.Clauses(), semaCtx);4096 4097  ConstructQueue queue{buildConstructQueue(4098      converter.getFirOpBuilder().getModule(), semaCtx, eval, beginSpec.source,4099      llvm::omp::Directive::OMPD_critical, clauses)};4100 4101  std::optional<parser::Name> critName;4102  const parser::OmpArgumentList &args = beginSpec.Arguments();4103  if (!args.v.empty()) {4104    // All of these things should be guaranteed to exist after semantic checks.4105    auto *object = parser::Unwrap<parser::OmpObject>(args.v.front());4106    assert(object && "Expecting object as argument");4107    auto *designator = semantics::omp::GetDesignatorFromObj(*object);4108    assert(designator && "Expecting desginator in argument");4109    auto *name = parser::GetDesignatorNameIfDataRef(*designator);4110    assert(name && "Expecting dataref in designator");4111    critName = *name;4112  }4113  mlir::Location currentLocation = converter.getCurrentLocation();4114  genCriticalOp(converter, symTable, semaCtx, eval, currentLocation, queue,4115                queue.begin(), critName);4116}4117 4118static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,4119                   semantics::SemanticsContext &semaCtx,4120                   lower::pft::Evaluation &eval,4121                   const parser::OpenMPUtilityConstruct &) {4122  if (!semaCtx.langOptions().OpenMPSimd)4123    TODO(converter.getCurrentLocation(), "OpenMPUtilityConstruct");4124}4125 4126static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,4127                   semantics::SemanticsContext &semaCtx,4128                   lower::pft::Evaluation &eval,4129                   const parser::OpenMPDispatchConstruct &) {4130  if (!semaCtx.langOptions().OpenMPSimd)4131    TODO(converter.getCurrentLocation(), "OpenMPDispatchConstruct");4132}4133 4134static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,4135                   semantics::SemanticsContext &semaCtx,4136                   lower::pft::Evaluation &eval,4137                   const parser::OpenMPLoopConstruct &loopConstruct) {4138  const parser::OmpDirectiveSpecification &beginSpec = loopConstruct.BeginDir();4139  List<Clause> clauses = makeClauses(beginSpec.Clauses(), semaCtx);4140  if (auto &endSpec = loopConstruct.EndDir())4141    clauses.append(makeClauses(endSpec->Clauses(), semaCtx));4142 4143  mlir::Location currentLocation = converter.genLocation(beginSpec.source);4144 4145  for (auto &construct : std::get<parser::Block>(loopConstruct.t)) {4146    if (const parser::OpenMPLoopConstruct *ompNestedLoopCons =4147            parser::omp::GetOmpLoop(construct)) {4148      llvm::omp::Directive nestedDirective =4149          parser::omp::GetOmpDirectiveName(*ompNestedLoopCons).v;4150      switch (nestedDirective) {4151      case llvm::omp::Directive::OMPD_tile:4152        // Skip OMPD_tile since the tile sizes will be retrieved when4153        // generating the omp.loop_nest op.4154        break;4155      default: {4156        unsigned version = semaCtx.langOptions().OpenMPVersion;4157        TODO(currentLocation,4158             "Applying a loop-associated on the loop generated by the " +4159                 llvm::omp::getOpenMPDirectiveName(nestedDirective, version) +4160                 " construct");4161      }4162      }4163    }4164  }4165 4166  const parser::OmpDirectiveName &beginName = beginSpec.DirName();4167  ConstructQueue queue{4168      buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,4169                          eval, beginName.source, beginName.v, clauses)};4170  genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,4171                 queue.begin());4172}4173 4174static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,4175                   semantics::SemanticsContext &semaCtx,4176                   lower::pft::Evaluation &eval,4177                   const parser::OpenMPSectionConstruct &sectionConstruct) {4178  // Do nothing here. SECTION is lowered inside of the lowering for Sections4179}4180 4181static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,4182                   semantics::SemanticsContext &semaCtx,4183                   lower::pft::Evaluation &eval,4184                   const parser::OpenMPSectionsConstruct &construct) {4185  const parser::OmpDirectiveSpecification &beginSpec{construct.BeginDir()};4186  List<Clause> clauses = makeClauses(beginSpec.Clauses(), semaCtx);4187  const auto &endSpec{construct.EndDir()};4188  assert(endSpec &&4189         "Missing end section directive should have been handled in semantics");4190  clauses.append(makeClauses(endSpec->Clauses(), semaCtx));4191  mlir::Location currentLocation = converter.getCurrentLocation();4192 4193  const parser::OmpDirectiveName &beginName{beginSpec.DirName()};4194  ConstructQueue queue{4195      buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,4196                          eval, beginName.source, beginName.v, clauses)};4197 4198  mlir::SaveStateStack<SectionsConstructStackFrame> saveStateStack{4199      converter.getStateStack(), construct};4200  genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,4201                 queue.begin());4202}4203 4204static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,4205                   semantics::SemanticsContext &semaCtx,4206                   lower::pft::Evaluation &eval,4207                   const parser::OpenMPConstruct &ompConstruct) {4208  Fortran::common::visit(4209      [&](auto &&s) { return genOMP(converter, symTable, semaCtx, eval, s); },4210      ompConstruct.u);4211}4212 4213//===----------------------------------------------------------------------===//4214// Public functions4215//===----------------------------------------------------------------------===//4216 4217mlir::Operation *Fortran::lower::genOpenMPTerminator(fir::FirOpBuilder &builder,4218                                                     mlir::Operation *op,4219                                                     mlir::Location loc) {4220  if (mlir::isa<mlir::omp::AtomicUpdateOp, mlir::omp::DeclareReductionOp,4221                mlir::omp::LoopNestOp>(op))4222    return mlir::omp::YieldOp::create(builder, loc);4223  return mlir::omp::TerminatorOp::create(builder, loc);4224}4225 4226void Fortran::lower::genOpenMPConstruct(lower::AbstractConverter &converter,4227                                        lower::SymMap &symTable,4228                                        semantics::SemanticsContext &semaCtx,4229                                        lower::pft::Evaluation &eval,4230                                        const parser::OpenMPConstruct &omp) {4231  lower::SymMapScope scope(symTable);4232  genOMP(converter, symTable, semaCtx, eval, omp);4233}4234 4235void Fortran::lower::genOpenMPDeclarativeConstruct(4236    lower::AbstractConverter &converter, lower::SymMap &symTable,4237    semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,4238    const parser::OpenMPDeclarativeConstruct &omp) {4239  genOMP(converter, symTable, semaCtx, eval, omp);4240  genNestedEvaluations(converter, eval);4241}4242 4243void Fortran::lower::genOpenMPSymbolProperties(4244    lower::AbstractConverter &converter, const lower::pft::Variable &var) {4245  assert(var.hasSymbol() && "Expecting Symbol");4246  const semantics::Symbol &sym = var.getSymbol();4247 4248  if (sym.test(semantics::Symbol::Flag::OmpThreadprivate))4249    lower::genThreadprivateOp(converter, var);4250 4251  if (sym.test(semantics::Symbol::Flag::OmpDeclareTarget))4252    lower::genDeclareTargetIntGlobal(converter, var);4253}4254 4255void Fortran::lower::genThreadprivateOp(lower::AbstractConverter &converter,4256                                        const lower::pft::Variable &var) {4257  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();4258  mlir::Location currentLocation = converter.getCurrentLocation();4259 4260  const semantics::Symbol &sym = var.getSymbol();4261  mlir::Value symThreadprivateValue;4262  if (const semantics::Symbol *common =4263          semantics::FindCommonBlockContaining(sym.GetUltimate())) {4264    mlir::Value commonValue = converter.getSymbolAddress(*common);4265    if (mlir::isa<mlir::omp::ThreadprivateOp>(commonValue.getDefiningOp())) {4266      // Generate ThreadprivateOp for a common block instead of its members and4267      // only do it once for a common block.4268      return;4269    }4270    // Generate ThreadprivateOp and rebind the common block.4271    mlir::Value commonThreadprivateValue = mlir::omp::ThreadprivateOp::create(4272        firOpBuilder, currentLocation, commonValue.getType(), commonValue);4273    converter.bindSymbol(*common, commonThreadprivateValue);4274    // Generate the threadprivate value for the common block member.4275    symThreadprivateValue =4276        genCommonBlockMember(converter, currentLocation, sym,4277                             commonThreadprivateValue, common->size());4278  } else if (!var.isGlobal()) {4279    // Non-global variable which can be in threadprivate directive must be one4280    // variable in main program, and it has implicit SAVE attribute. Take it as4281    // with SAVE attribute, so to create GlobalOp for it to simplify the4282    // translation to LLVM IR.4283    // Avoids performing multiple globalInitializations.4284    fir::GlobalOp global;4285    auto module = converter.getModuleOp();4286    std::string globalName = converter.mangleName(sym);4287    if (module.lookupSymbol<fir::GlobalOp>(globalName))4288      global = module.lookupSymbol<fir::GlobalOp>(globalName);4289    else4290      global = globalInitialization(converter, firOpBuilder, sym, var,4291                                    currentLocation);4292 4293    mlir::Value symValue = fir::AddrOfOp::create(4294        firOpBuilder, currentLocation, global.resultType(), global.getSymbol());4295    symThreadprivateValue = mlir::omp::ThreadprivateOp::create(4296        firOpBuilder, currentLocation, symValue.getType(), symValue);4297  } else {4298    mlir::Value symValue = converter.getSymbolAddress(sym);4299 4300    // The symbol may be use-associated multiple times, and nothing needs to be4301    // done after the original symbol is mapped to the threadprivatized value4302    // for the first time. Use the threadprivatized value directly.4303    mlir::Operation *op;4304    if (auto declOp = symValue.getDefiningOp<hlfir::DeclareOp>())4305      op = declOp.getMemref().getDefiningOp();4306    else4307      op = symValue.getDefiningOp();4308    if (mlir::isa<mlir::omp::ThreadprivateOp>(op))4309      return;4310 4311    symThreadprivateValue = mlir::omp::ThreadprivateOp::create(4312        firOpBuilder, currentLocation, symValue.getType(), symValue);4313  }4314 4315  fir::ExtendedValue sexv = converter.getSymbolExtendedValue(sym);4316  fir::ExtendedValue symThreadprivateExv =4317      getExtendedValue(sexv, symThreadprivateValue);4318  converter.bindSymbol(sym, symThreadprivateExv);4319}4320 4321// This function replicates threadprivate's behaviour of generating4322// an internal fir.GlobalOp for non-global variables in the main program4323// that have the implicit SAVE attribute, to simplifiy LLVM-IR and MLIR4324// generation.4325void Fortran::lower::genDeclareTargetIntGlobal(4326    lower::AbstractConverter &converter, const lower::pft::Variable &var) {4327  if (!var.isGlobal()) {4328    // A non-global variable which can be in a declare target directive must4329    // be a variable in the main program, and it has the implicit SAVE4330    // attribute. We create a GlobalOp for it to simplify the translation to4331    // LLVM IR.4332    globalInitialization(converter, converter.getFirOpBuilder(),4333                         var.getSymbol(), var, converter.getCurrentLocation());4334  }4335}4336 4337bool Fortran::lower::isOpenMPTargetConstruct(4338    const parser::OpenMPConstruct &omp) {4339  llvm::omp::Directive dir = llvm::omp::Directive::OMPD_unknown;4340  if (const auto *block = std::get_if<parser::OmpBlockConstruct>(&omp.u)) {4341    dir = block->BeginDir().DirId();4342  } else if (const auto *loop =4343                 std::get_if<parser::OpenMPLoopConstruct>(&omp.u)) {4344    dir = loop->BeginDir().DirId();4345  }4346  return llvm::omp::allTargetSet.test(dir);4347}4348 4349void Fortran::lower::gatherOpenMPDeferredDeclareTargets(4350    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,4351    lower::pft::Evaluation &eval,4352    const parser::OpenMPDeclarativeConstruct &ompDecl,4353    llvm::SmallVectorImpl<OMPDeferredDeclareTargetInfo>4354        &deferredDeclareTarget) {4355  Fortran::common::visit(4356      common::visitors{4357          [&](const parser::OpenMPDeclareTargetConstruct &ompReq) {4358            collectDeferredDeclareTargets(converter, semaCtx, eval, ompReq,4359                                          deferredDeclareTarget);4360          },4361          [&](const auto &) {},4362      },4363      ompDecl.u);4364}4365 4366bool Fortran::lower::isOpenMPDeviceDeclareTarget(4367    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,4368    lower::pft::Evaluation &eval,4369    const parser::OpenMPDeclarativeConstruct &ompDecl) {4370  return Fortran::common::visit(4371      common::visitors{4372          [&](const parser::OpenMPDeclareTargetConstruct &ompReq) {4373            mlir::omp::DeclareTargetDeviceType targetType =4374                getDeclareTargetFunctionDevice(converter, semaCtx, eval, ompReq)4375                    .value_or(mlir::omp::DeclareTargetDeviceType::host);4376            return targetType != mlir::omp::DeclareTargetDeviceType::host;4377          },4378          [&](const auto &) { return false; },4379      },4380      ompDecl.u);4381}4382 4383// In certain cases such as subroutine or function interfaces which declare4384// but do not define or directly call the subroutine or function in the same4385// module, their lowering is delayed until after the declare target construct4386// itself is processed, so there symbol is not within the table.4387//4388// This function will also return true if we encounter any device declare4389// target cases, to satisfy checking if we require the requires attributes4390// on the module.4391bool Fortran::lower::markOpenMPDeferredDeclareTargetFunctions(4392    mlir::Operation *mod,4393    llvm::SmallVectorImpl<OMPDeferredDeclareTargetInfo> &deferredDeclareTargets,4394    AbstractConverter &converter) {4395  bool deviceCodeFound = false;4396  auto modOp = llvm::cast<mlir::ModuleOp>(mod);4397  for (auto declTar : deferredDeclareTargets) {4398    mlir::Operation *op = modOp.lookupSymbol(converter.mangleName(declTar.sym));4399 4400    // Due to interfaces being optionally emitted on usage in a module,4401    // not finding an operation at this point cannot be a hard error, we4402    // simply ignore it for now.4403    // TODO: Add semantic checks for detecting cases where an erronous4404    // (undefined) symbol has been supplied to a declare target clause4405    if (!op)4406      continue;4407 4408    auto devType = declTar.declareTargetDeviceType;4409    if (!deviceCodeFound && devType != mlir::omp::DeclareTargetDeviceType::host)4410      deviceCodeFound = true;4411 4412    markDeclareTarget(op, converter, declTar.declareTargetCaptureClause,4413                      devType, declTar.automap);4414  }4415 4416  return deviceCodeFound;4417}4418 4419void Fortran::lower::genOpenMPRequires(mlir::Operation *mod,4420                                       const semantics::Symbol *symbol) {4421  using MlirRequires = mlir::omp::ClauseRequires;4422 4423  if (auto offloadMod =4424          llvm::dyn_cast<mlir::omp::OffloadModuleInterface>(mod)) {4425    semantics::WithOmpDeclarative::RequiresClauses reqs;4426    if (symbol) {4427      common::visit(4428          [&](const auto &details) {4429            if constexpr (std::is_base_of_v<semantics::WithOmpDeclarative,4430                                            std::decay_t<decltype(details)>>) {4431              if (details.has_ompRequires())4432                reqs = *details.ompRequires();4433            }4434          },4435          symbol->details());4436    }4437 4438    // Use pre-populated omp.requires module attribute if it was set, so that4439    // the "-fopenmp-force-usm" compiler option is honored.4440    MlirRequires mlirFlags = offloadMod.getRequires();4441    if (reqs.test(llvm::omp::Clause::OMPC_dynamic_allocators))4442      mlirFlags = mlirFlags | MlirRequires::dynamic_allocators;4443    if (reqs.test(llvm::omp::Clause::OMPC_reverse_offload))4444      mlirFlags = mlirFlags | MlirRequires::reverse_offload;4445    if (reqs.test(llvm::omp::Clause::OMPC_unified_address))4446      mlirFlags = mlirFlags | MlirRequires::unified_address;4447    if (reqs.test(llvm::omp::Clause::OMPC_unified_shared_memory))4448      mlirFlags = mlirFlags | MlirRequires::unified_shared_memory;4449 4450    offloadMod.setRequires(mlirFlags);4451  }4452}4453 4454// Walk scopes and materialize omp.declare_mapper ops for mapper declarations4455// found in imported modules. If \p scope is null, start from the global scope.4456void Fortran::lower::materializeOpenMPDeclareMappers(4457    Fortran::lower::AbstractConverter &converter,4458    semantics::SemanticsContext &semaCtx, const semantics::Scope *scope) {4459  const semantics::Scope &root = scope ? *scope : semaCtx.globalScope();4460 4461  // Recurse into child scopes first (modules, submodules, etc.).4462  for (const semantics::Scope &child : root.children())4463    materializeOpenMPDeclareMappers(converter, semaCtx, &child);4464 4465  // Only consider module scopes to avoid duplicating local constructs.4466  if (!root.IsModule())4467    return;4468 4469  // Only materialize for modules coming from mod files to avoid duplicates.4470  if (!root.symbol() || !root.symbol()->test(semantics::Symbol::Flag::ModFile))4471    return;4472 4473  // Scan symbols in this module scope for MapperDetails.4474  for (auto &it : root) {4475    const semantics::Symbol &sym = *it.second;4476    if (auto *md = sym.detailsIf<semantics::MapperDetails>()) {4477      for (const auto *decl : md->GetDeclList()) {4478        if (const auto *mapperDecl =4479                std::get_if<parser::OpenMPDeclareMapperConstruct>(&decl->u)) {4480          genOpenMPDeclareMapperImpl(converter, semaCtx, *mapperDecl, &sym);4481        }4482      }4483    }4484  }4485}4486