brintos / llvm-project-archived public Read only

1//===- OpenMPToLLVMIRTranslation.cpp - Translate OpenMP dialect to LLVM IR-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements a translation between the MLIR OpenMP dialect and LLVM10// IR.11//12//===----------------------------------------------------------------------===//13#include "mlir/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.h"14#include "mlir/Analysis/TopologicalSortUtils.h"15#include "mlir/Dialect/LLVMIR/LLVMDialect.h"16#include "mlir/Dialect/LLVMIR/LLVMTypes.h"17#include "mlir/Dialect/OpenMP/OpenMPDialect.h"18#include "mlir/Dialect/OpenMP/OpenMPInterfaces.h"19#include "mlir/IR/Operation.h"20#include "mlir/Support/LLVM.h"21#include "mlir/Target/LLVMIR/Dialect/OpenMPCommon.h"22#include "mlir/Target/LLVMIR/ModuleTranslation.h"23 24#include "llvm/ADT/ArrayRef.h"25#include "llvm/ADT/SmallVector.h"26#include "llvm/ADT/TypeSwitch.h"27#include "llvm/Frontend/OpenMP/OMPConstants.h"28#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"29#include "llvm/IR/Constants.h"30#include "llvm/IR/DebugInfoMetadata.h"31#include "llvm/IR/DerivedTypes.h"32#include "llvm/IR/IRBuilder.h"33#include "llvm/IR/MDBuilder.h"34#include "llvm/IR/ReplaceConstant.h"35#include "llvm/Support/FileSystem.h"36#include "llvm/Support/VirtualFileSystem.h"37#include "llvm/TargetParser/Triple.h"38#include "llvm/Transforms/Utils/ModuleUtils.h"39 40#include <cstdint>41#include <iterator>42#include <numeric>43#include <optional>44#include <utility>45 46using namespace mlir;47 48namespace {49static llvm::omp::ScheduleKind50convertToScheduleKind(std::optional<omp::ClauseScheduleKind> schedKind) {51  if (!schedKind.has_value())52    return llvm::omp::OMP_SCHEDULE_Default;53  switch (schedKind.value()) {54  case omp::ClauseScheduleKind::Static:55    return llvm::omp::OMP_SCHEDULE_Static;56  case omp::ClauseScheduleKind::Dynamic:57    return llvm::omp::OMP_SCHEDULE_Dynamic;58  case omp::ClauseScheduleKind::Guided:59    return llvm::omp::OMP_SCHEDULE_Guided;60  case omp::ClauseScheduleKind::Auto:61    return llvm::omp::OMP_SCHEDULE_Auto;62  case omp::ClauseScheduleKind::Runtime:63    return llvm::omp::OMP_SCHEDULE_Runtime;64  case omp::ClauseScheduleKind::Distribute:65    return llvm::omp::OMP_SCHEDULE_Distribute;66  }67  llvm_unreachable("unhandled schedule clause argument");68}69 70/// ModuleTranslation stack frame for OpenMP operations. This keeps track of the71/// insertion points for allocas.72class OpenMPAllocaStackFrame73    : public StateStackFrameBase<OpenMPAllocaStackFrame> {74public:75  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OpenMPAllocaStackFrame)76 77  explicit OpenMPAllocaStackFrame(llvm::OpenMPIRBuilder::InsertPointTy allocaIP)78      : allocaInsertPoint(allocaIP) {}79  llvm::OpenMPIRBuilder::InsertPointTy allocaInsertPoint;80};81 82/// Stack frame to hold a \see llvm::CanonicalLoopInfo representing the83/// collapsed canonical loop information corresponding to an \c omp.loop_nest84/// operation.85class OpenMPLoopInfoStackFrame86    : public StateStackFrameBase<OpenMPLoopInfoStackFrame> {87public:88  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OpenMPLoopInfoStackFrame)89  llvm::CanonicalLoopInfo *loopInfo = nullptr;90};91 92/// Custom error class to signal translation errors that don't need reporting,93/// since encountering them will have already triggered relevant error messages.94///95/// Its purpose is to serve as the glue between MLIR failures represented as96/// \see LogicalResult instances and \see llvm::Error instances used to97/// propagate errors through the \see llvm::OpenMPIRBuilder. Generally, when an98/// error of the first type is raised, a message is emitted directly (the \see99/// LogicalResult itself does not hold any information). If we need to forward100/// this error condition as an \see llvm::Error while avoiding triggering some101/// redundant error reporting later on, we need a custom \see llvm::ErrorInfo102/// class to just signal this situation has happened.103///104/// For example, this class should be used to trigger errors from within105/// callbacks passed to the \see OpenMPIRBuilder when they were triggered by the106/// translation of their own regions. This unclutters the error log from107/// redundant messages.108class PreviouslyReportedError109    : public llvm::ErrorInfo<PreviouslyReportedError> {110public:111  void log(raw_ostream &) const override {112    // Do not log anything.113  }114 115  std::error_code convertToErrorCode() const override {116    llvm_unreachable(117        "PreviouslyReportedError doesn't support ECError conversion");118  }119 120  // Used by ErrorInfo::classID.121  static char ID;122};123 124char PreviouslyReportedError::ID = 0;125 126/*127 * Custom class for processing linear clause for omp.wsloop128 * and omp.simd. Linear clause translation requires setup,129 * initialization, update, and finalization at varying130 * basic blocks in the IR. This class helps maintain131 * internal state to allow consistent translation in132 * each of these stages.133 */134 135class LinearClauseProcessor {136 137private:138  SmallVector<llvm::Value *> linearPreconditionVars;139  SmallVector<llvm::Value *> linearLoopBodyTemps;140  SmallVector<llvm::AllocaInst *> linearOrigVars;141  SmallVector<llvm::Value *> linearOrigVal;142  SmallVector<llvm::Value *> linearSteps;143  llvm::BasicBlock *linearFinalizationBB;144  llvm::BasicBlock *linearExitBB;145  llvm::BasicBlock *linearLastIterExitBB;146 147public:148  // Allocate space for linear variabes149  void createLinearVar(llvm::IRBuilderBase &builder,150                       LLVM::ModuleTranslation &moduleTranslation,151                       mlir::Value &linearVar) {152    if (llvm::AllocaInst *linearVarAlloca = dyn_cast<llvm::AllocaInst>(153            moduleTranslation.lookupValue(linearVar))) {154      linearPreconditionVars.push_back(builder.CreateAlloca(155          linearVarAlloca->getAllocatedType(), nullptr, ".linear_var"));156      llvm::Value *linearLoopBodyTemp = builder.CreateAlloca(157          linearVarAlloca->getAllocatedType(), nullptr, ".linear_result");158      linearOrigVal.push_back(moduleTranslation.lookupValue(linearVar));159      linearLoopBodyTemps.push_back(linearLoopBodyTemp);160      linearOrigVars.push_back(linearVarAlloca);161    }162  }163 164  // Initialize linear step165  inline void initLinearStep(LLVM::ModuleTranslation &moduleTranslation,166                             mlir::Value &linearStep) {167    linearSteps.push_back(moduleTranslation.lookupValue(linearStep));168  }169 170  // Emit IR for initialization of linear variables171  llvm::OpenMPIRBuilder::InsertPointOrErrorTy172  initLinearVar(llvm::IRBuilderBase &builder,173                LLVM::ModuleTranslation &moduleTranslation,174                llvm::BasicBlock *loopPreHeader) {175    builder.SetInsertPoint(loopPreHeader->getTerminator());176    for (size_t index = 0; index < linearOrigVars.size(); index++) {177      llvm::LoadInst *linearVarLoad = builder.CreateLoad(178          linearOrigVars[index]->getAllocatedType(), linearOrigVars[index]);179      builder.CreateStore(linearVarLoad, linearPreconditionVars[index]);180    }181    llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =182        moduleTranslation.getOpenMPBuilder()->createBarrier(183            builder.saveIP(), llvm::omp::OMPD_barrier);184    return afterBarrierIP;185  }186 187  // Emit IR for updating Linear variables188  void updateLinearVar(llvm::IRBuilderBase &builder, llvm::BasicBlock *loopBody,189                       llvm::Value *loopInductionVar) {190    builder.SetInsertPoint(loopBody->getTerminator());191    for (size_t index = 0; index < linearPreconditionVars.size(); index++) {192      // Emit increments for linear vars193      llvm::LoadInst *linearVarStart =194          builder.CreateLoad(linearOrigVars[index]->getAllocatedType(),195 196                             linearPreconditionVars[index]);197      auto mulInst = builder.CreateMul(loopInductionVar, linearSteps[index]);198      auto addInst = builder.CreateAdd(linearVarStart, mulInst);199      builder.CreateStore(addInst, linearLoopBodyTemps[index]);200    }201  }202 203  // Linear variable finalization is conditional on the last logical iteration.204  // Create BB splits to manage the same.205  void outlineLinearFinalizationBB(llvm::IRBuilderBase &builder,206                                   llvm::BasicBlock *loopExit) {207    linearFinalizationBB = loopExit->splitBasicBlock(208        loopExit->getTerminator(), "omp_loop.linear_finalization");209    linearExitBB = linearFinalizationBB->splitBasicBlock(210        linearFinalizationBB->getTerminator(), "omp_loop.linear_exit");211    linearLastIterExitBB = linearFinalizationBB->splitBasicBlock(212        linearFinalizationBB->getTerminator(), "omp_loop.linear_lastiter_exit");213  }214 215  // Finalize the linear vars216  llvm::OpenMPIRBuilder::InsertPointOrErrorTy217  finalizeLinearVar(llvm::IRBuilderBase &builder,218                    LLVM::ModuleTranslation &moduleTranslation,219                    llvm::Value *lastIter) {220    // Emit condition to check whether last logical iteration is being executed221    builder.SetInsertPoint(linearFinalizationBB->getTerminator());222    llvm::Value *loopLastIterLoad = builder.CreateLoad(223        llvm::Type::getInt32Ty(builder.getContext()), lastIter);224    llvm::Value *isLast =225        builder.CreateCmp(llvm::CmpInst::ICMP_NE, loopLastIterLoad,226                          llvm::ConstantInt::get(227                              llvm::Type::getInt32Ty(builder.getContext()), 0));228    // Store the linear variable values to original variables.229    builder.SetInsertPoint(linearLastIterExitBB->getTerminator());230    for (size_t index = 0; index < linearOrigVars.size(); index++) {231      llvm::LoadInst *linearVarTemp =232          builder.CreateLoad(linearOrigVars[index]->getAllocatedType(),233                             linearLoopBodyTemps[index]);234      builder.CreateStore(linearVarTemp, linearOrigVars[index]);235    }236 237    // Create conditional branch such that the linear variable238    // values are stored to original variables only at the239    // last logical iteration240    builder.SetInsertPoint(linearFinalizationBB->getTerminator());241    builder.CreateCondBr(isLast, linearLastIterExitBB, linearExitBB);242    linearFinalizationBB->getTerminator()->eraseFromParent();243    // Emit barrier244    builder.SetInsertPoint(linearExitBB->getTerminator());245    return moduleTranslation.getOpenMPBuilder()->createBarrier(246        builder.saveIP(), llvm::omp::OMPD_barrier);247  }248 249  // Rewrite all uses of the original variable in `BBName`250  //  with the linear variable in-place251  void rewriteInPlace(llvm::IRBuilderBase &builder, const std::string &BBName,252                      size_t varIndex) {253    llvm::SmallVector<llvm::User *> users;254    for (llvm::User *user : linearOrigVal[varIndex]->users())255      users.push_back(user);256    for (auto *user : users) {257      if (auto *userInst = dyn_cast<llvm::Instruction>(user)) {258        if (userInst->getParent()->getName().str() == BBName)259          user->replaceUsesOfWith(linearOrigVal[varIndex],260                                  linearLoopBodyTemps[varIndex]);261      }262    }263  }264};265 266} // namespace267 268/// Looks up from the operation from and returns the PrivateClauseOp with269/// name symbolName270static omp::PrivateClauseOp findPrivatizer(Operation *from,271                                           SymbolRefAttr symbolName) {272  omp::PrivateClauseOp privatizer =273      SymbolTable::lookupNearestSymbolFrom<omp::PrivateClauseOp>(from,274                                                                 symbolName);275  assert(privatizer && "privatizer not found in the symbol table");276  return privatizer;277}278 279/// Check whether translation to LLVM IR for the given operation is currently280/// supported. If not, descriptive diagnostics will be emitted to let users know281/// this is a not-yet-implemented feature.282///283/// \returns success if no unimplemented features are needed to translate the284///          given operation.285static LogicalResult checkImplementationStatus(Operation &op) {286  auto todo = [&op](StringRef clauseName) {287    return op.emitError() << "not yet implemented: Unhandled clause "288                          << clauseName << " in " << op.getName()289                          << " operation";290  };291 292  auto checkAllocate = [&todo](auto op, LogicalResult &result) {293    if (!op.getAllocateVars().empty() || !op.getAllocatorVars().empty())294      result = todo("allocate");295  };296  auto checkBare = [&todo](auto op, LogicalResult &result) {297    if (op.getBare())298      result = todo("ompx_bare");299  };300  auto checkCancelDirective = [&todo](auto op, LogicalResult &result) {301    omp::ClauseCancellationConstructType cancelledDirective =302        op.getCancelDirective();303    // Cancelling a taskloop is not yet supported because we don't yet have LLVM304    // IR conversion for taskloop305    if (cancelledDirective == omp::ClauseCancellationConstructType::Taskgroup) {306      Operation *parent = op->getParentOp();307      while (parent) {308        if (parent->getDialect() == op->getDialect())309          break;310        parent = parent->getParentOp();311      }312      if (isa_and_nonnull<omp::TaskloopOp>(parent))313        result = todo("cancel directive inside of taskloop");314    }315  };316  auto checkDepend = [&todo](auto op, LogicalResult &result) {317    if (!op.getDependVars().empty() || op.getDependKinds())318      result = todo("depend");319  };320  auto checkDevice = [&todo](auto op, LogicalResult &result) {321    if (op.getDevice())322      result = todo("device");323  };324  auto checkHint = [](auto op, LogicalResult &) {325    if (op.getHint())326      op.emitWarning("hint clause discarded");327  };328  auto checkInReduction = [&todo](auto op, LogicalResult &result) {329    if (!op.getInReductionVars().empty() || op.getInReductionByref() ||330        op.getInReductionSyms())331      result = todo("in_reduction");332  };333  auto checkIsDevicePtr = [&todo](auto op, LogicalResult &result) {334    if (!op.getIsDevicePtrVars().empty())335      result = todo("is_device_ptr");336  };337  auto checkLinear = [&todo](auto op, LogicalResult &result) {338    if (!op.getLinearVars().empty() || !op.getLinearStepVars().empty())339      result = todo("linear");340  };341  auto checkNowait = [&todo](auto op, LogicalResult &result) {342    if (op.getNowait())343      result = todo("nowait");344  };345  auto checkOrder = [&todo](auto op, LogicalResult &result) {346    if (op.getOrder() || op.getOrderMod())347      result = todo("order");348  };349  auto checkParLevelSimd = [&todo](auto op, LogicalResult &result) {350    if (op.getParLevelSimd())351      result = todo("parallelization-level");352  };353  auto checkPriority = [&todo](auto op, LogicalResult &result) {354    if (op.getPriority())355      result = todo("priority");356  };357  auto checkPrivate = [&todo](auto op, LogicalResult &result) {358    if (!op.getPrivateVars().empty() || op.getPrivateSyms())359      result = todo("privatization");360  };361  auto checkReduction = [&todo](auto op, LogicalResult &result) {362    if (isa<omp::TeamsOp>(op))363      if (!op.getReductionVars().empty() || op.getReductionByref() ||364          op.getReductionSyms())365        result = todo("reduction");366    if (op.getReductionMod() &&367        op.getReductionMod().value() != omp::ReductionModifier::defaultmod)368      result = todo("reduction with modifier");369  };370  auto checkTaskReduction = [&todo](auto op, LogicalResult &result) {371    if (!op.getTaskReductionVars().empty() || op.getTaskReductionByref() ||372        op.getTaskReductionSyms())373      result = todo("task_reduction");374  };375  auto checkUntied = [&todo](auto op, LogicalResult &result) {376    if (op.getUntied())377      result = todo("untied");378  };379 380  LogicalResult result = success();381  llvm::TypeSwitch<Operation &>(op)382      .Case([&](omp::CancelOp op) { checkCancelDirective(op, result); })383      .Case([&](omp::CancellationPointOp op) {384        checkCancelDirective(op, result);385      })386      .Case([&](omp::DistributeOp op) {387        checkAllocate(op, result);388        checkOrder(op, result);389      })390      .Case([&](omp::OrderedRegionOp op) { checkParLevelSimd(op, result); })391      .Case([&](omp::SectionsOp op) {392        checkAllocate(op, result);393        checkPrivate(op, result);394        checkReduction(op, result);395      })396      .Case([&](omp::SingleOp op) {397        checkAllocate(op, result);398        checkPrivate(op, result);399      })400      .Case([&](omp::TeamsOp op) {401        checkAllocate(op, result);402        checkPrivate(op, result);403      })404      .Case([&](omp::TaskOp op) {405        checkAllocate(op, result);406        checkInReduction(op, result);407      })408      .Case([&](omp::TaskgroupOp op) {409        checkAllocate(op, result);410        checkTaskReduction(op, result);411      })412      .Case([&](omp::TaskwaitOp op) {413        checkDepend(op, result);414        checkNowait(op, result);415      })416      .Case([&](omp::TaskloopOp op) {417        // TODO: Add other clauses check418        checkUntied(op, result);419        checkPriority(op, result);420      })421      .Case([&](omp::WsloopOp op) {422        checkAllocate(op, result);423        checkLinear(op, result);424        checkOrder(op, result);425        checkReduction(op, result);426      })427      .Case([&](omp::ParallelOp op) {428        checkAllocate(op, result);429        checkReduction(op, result);430      })431      .Case([&](omp::SimdOp op) {432        checkLinear(op, result);433        checkReduction(op, result);434      })435      .Case<omp::AtomicReadOp, omp::AtomicWriteOp, omp::AtomicUpdateOp,436            omp::AtomicCaptureOp>([&](auto op) { checkHint(op, result); })437      .Case<omp::TargetEnterDataOp, omp::TargetExitDataOp, omp::TargetUpdateOp>(438          [&](auto op) { checkDepend(op, result); })439      .Case([&](omp::TargetOp op) {440        checkAllocate(op, result);441        checkBare(op, result);442        checkDevice(op, result);443        checkInReduction(op, result);444        checkIsDevicePtr(op, result);445      })446      .Default([](Operation &) {447        // Assume all clauses for an operation can be translated unless they are448        // checked above.449      });450  return result;451}452 453static LogicalResult handleError(llvm::Error error, Operation &op) {454  LogicalResult result = success();455  if (error) {456    llvm::handleAllErrors(457        std::move(error),458        [&](const PreviouslyReportedError &) { result = failure(); },459        [&](const llvm::ErrorInfoBase &err) {460          result = op.emitError(err.message());461        });462  }463  return result;464}465 466template <typename T>467static LogicalResult handleError(llvm::Expected<T> &result, Operation &op) {468  if (!result)469    return handleError(result.takeError(), op);470 471  return success();472}473 474/// Find the insertion point for allocas given the current insertion point for475/// normal operations in the builder.476static llvm::OpenMPIRBuilder::InsertPointTy477findAllocaInsertPoint(llvm::IRBuilderBase &builder,478                      LLVM::ModuleTranslation &moduleTranslation) {479  // If there is an alloca insertion point on stack, i.e. we are in a nested480  // operation and a specific point was provided by some surrounding operation,481  // use it.482  llvm::OpenMPIRBuilder::InsertPointTy allocaInsertPoint;483  WalkResult walkResult = moduleTranslation.stackWalk<OpenMPAllocaStackFrame>(484      [&](OpenMPAllocaStackFrame &frame) {485        allocaInsertPoint = frame.allocaInsertPoint;486        return WalkResult::interrupt();487      });488  // In cases with multiple levels of outlining, the tree walk might find an489  // alloca insertion point that is inside the original function while the490  // builder insertion point is inside the outlined function. We need to make491  // sure that we do not use it in those cases.492  if (walkResult.wasInterrupted() &&493      allocaInsertPoint.getBlock()->getParent() ==494          builder.GetInsertBlock()->getParent())495    return allocaInsertPoint;496 497  // Otherwise, insert to the entry block of the surrounding function.498  // If the current IRBuilder InsertPoint is the function's entry, it cannot499  // also be used for alloca insertion which would result in insertion order500  // confusion. Create a new BasicBlock for the Builder and use the entry block501  // for the allocs.502  // TODO: Create a dedicated alloca BasicBlock at function creation such that503  // we do not need to move the current InertPoint here.504  if (builder.GetInsertBlock() ==505      &builder.GetInsertBlock()->getParent()->getEntryBlock()) {506    assert(builder.GetInsertPoint() == builder.GetInsertBlock()->end() &&507           "Assuming end of basic block");508    llvm::BasicBlock *entryBB = llvm::BasicBlock::Create(509        builder.getContext(), "entry", builder.GetInsertBlock()->getParent(),510        builder.GetInsertBlock()->getNextNode());511    builder.CreateBr(entryBB);512    builder.SetInsertPoint(entryBB);513  }514 515  llvm::BasicBlock &funcEntryBlock =516      builder.GetInsertBlock()->getParent()->getEntryBlock();517  return llvm::OpenMPIRBuilder::InsertPointTy(518      &funcEntryBlock, funcEntryBlock.getFirstInsertionPt());519}520 521/// Find the loop information structure for the loop nest being translated. It522/// will return a `null` value unless called from the translation function for523/// a loop wrapper operation after successfully translating its body.524static llvm::CanonicalLoopInfo *525findCurrentLoopInfo(LLVM::ModuleTranslation &moduleTranslation) {526  llvm::CanonicalLoopInfo *loopInfo = nullptr;527  moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(528      [&](OpenMPLoopInfoStackFrame &frame) {529        loopInfo = frame.loopInfo;530        return WalkResult::interrupt();531      });532  return loopInfo;533}534 535/// Converts the given region that appears within an OpenMP dialect operation to536/// LLVM IR, creating a branch from the `sourceBlock` to the entry block of the537/// region, and a branch from any block with an successor-less OpenMP terminator538/// to `continuationBlock`. Populates `continuationBlockPHIs` with the PHI nodes539/// of the continuation block if provided.540static llvm::Expected<llvm::BasicBlock *> convertOmpOpRegions(541    Region &region, StringRef blockName, llvm::IRBuilderBase &builder,542    LLVM::ModuleTranslation &moduleTranslation,543    SmallVectorImpl<llvm::PHINode *> *continuationBlockPHIs = nullptr) {544  bool isLoopWrapper = isa<omp::LoopWrapperInterface>(region.getParentOp());545 546  llvm::BasicBlock *continuationBlock =547      splitBB(builder, true, "omp.region.cont");548  llvm::BasicBlock *sourceBlock = builder.GetInsertBlock();549 550  llvm::LLVMContext &llvmContext = builder.getContext();551  for (Block &bb : region) {552    llvm::BasicBlock *llvmBB = llvm::BasicBlock::Create(553        llvmContext, blockName, builder.GetInsertBlock()->getParent(),554        builder.GetInsertBlock()->getNextNode());555    moduleTranslation.mapBlock(&bb, llvmBB);556  }557 558  llvm::Instruction *sourceTerminator = sourceBlock->getTerminator();559 560  // Terminators (namely YieldOp) may be forwarding values to the region that561  // need to be available in the continuation block. Collect the types of these562  // operands in preparation of creating PHI nodes. This is skipped for loop563  // wrapper operations, for which we know in advance they have no terminators.564  SmallVector<llvm::Type *> continuationBlockPHITypes;565  unsigned numYields = 0;566 567  if (!isLoopWrapper) {568    bool operandsProcessed = false;569    for (Block &bb : region.getBlocks()) {570      if (omp::YieldOp yield = dyn_cast<omp::YieldOp>(bb.getTerminator())) {571        if (!operandsProcessed) {572          for (unsigned i = 0, e = yield->getNumOperands(); i < e; ++i) {573            continuationBlockPHITypes.push_back(574                moduleTranslation.convertType(yield->getOperand(i).getType()));575          }576          operandsProcessed = true;577        } else {578          assert(continuationBlockPHITypes.size() == yield->getNumOperands() &&579                 "mismatching number of values yielded from the region");580          for (unsigned i = 0, e = yield->getNumOperands(); i < e; ++i) {581            llvm::Type *operandType =582                moduleTranslation.convertType(yield->getOperand(i).getType());583            (void)operandType;584            assert(continuationBlockPHITypes[i] == operandType &&585                   "values of mismatching types yielded from the region");586          }587        }588        numYields++;589      }590    }591  }592 593  // Insert PHI nodes in the continuation block for any values forwarded by the594  // terminators in this region.595  if (!continuationBlockPHITypes.empty())596    assert(597        continuationBlockPHIs &&598        "expected continuation block PHIs if converted regions yield values");599  if (continuationBlockPHIs) {600    llvm::IRBuilderBase::InsertPointGuard guard(builder);601    continuationBlockPHIs->reserve(continuationBlockPHITypes.size());602    builder.SetInsertPoint(continuationBlock, continuationBlock->begin());603    for (llvm::Type *ty : continuationBlockPHITypes)604      continuationBlockPHIs->push_back(builder.CreatePHI(ty, numYields));605  }606 607  // Convert blocks one by one in topological order to ensure608  // defs are converted before uses.609  SetVector<Block *> blocks = getBlocksSortedByDominance(region);610  for (Block *bb : blocks) {611    llvm::BasicBlock *llvmBB = moduleTranslation.lookupBlock(bb);612    // Retarget the branch of the entry block to the entry block of the613    // converted region (regions are single-entry).614    if (bb->isEntryBlock()) {615      assert(sourceTerminator->getNumSuccessors() == 1 &&616             "provided entry block has multiple successors");617      assert(sourceTerminator->getSuccessor(0) == continuationBlock &&618             "ContinuationBlock is not the successor of the entry block");619      sourceTerminator->setSuccessor(0, llvmBB);620    }621 622    llvm::IRBuilderBase::InsertPointGuard guard(builder);623    if (failed(624            moduleTranslation.convertBlock(*bb, bb->isEntryBlock(), builder)))625      return llvm::make_error<PreviouslyReportedError>();626 627    // Create a direct branch here for loop wrappers to prevent their lack of a628    // terminator from causing a crash below.629    if (isLoopWrapper) {630      builder.CreateBr(continuationBlock);631      continue;632    }633 634    // Special handling for `omp.yield` and `omp.terminator` (we may have more635    // than one): they return the control to the parent OpenMP dialect operation636    // so replace them with the branch to the continuation block. We handle this637    // here to avoid relying inter-function communication through the638    // ModuleTranslation class to set up the correct insertion point. This is639    // also consistent with MLIR's idiom of handling special region terminators640    // in the same code that handles the region-owning operation.641    Operation *terminator = bb->getTerminator();642    if (isa<omp::TerminatorOp, omp::YieldOp>(terminator)) {643      builder.CreateBr(continuationBlock);644 645      for (unsigned i = 0, e = terminator->getNumOperands(); i < e; ++i)646        (*continuationBlockPHIs)[i]->addIncoming(647            moduleTranslation.lookupValue(terminator->getOperand(i)), llvmBB);648    }649  }650  // After all blocks have been traversed and values mapped, connect the PHI651  // nodes to the results of preceding blocks.652  LLVM::detail::connectPHINodes(region, moduleTranslation);653 654  // Remove the blocks and values defined in this region from the mapping since655  // they are not visible outside of this region. This allows the same region to656  // be converted several times, that is cloned, without clashes, and slightly657  // speeds up the lookups.658  moduleTranslation.forgetMapping(region);659 660  return continuationBlock;661}662 663/// Convert ProcBindKind from MLIR-generated enum to LLVM enum.664static llvm::omp::ProcBindKind getProcBindKind(omp::ClauseProcBindKind kind) {665  switch (kind) {666  case omp::ClauseProcBindKind::Close:667    return llvm::omp::ProcBindKind::OMP_PROC_BIND_close;668  case omp::ClauseProcBindKind::Master:669    return llvm::omp::ProcBindKind::OMP_PROC_BIND_master;670  case omp::ClauseProcBindKind::Primary:671    return llvm::omp::ProcBindKind::OMP_PROC_BIND_primary;672  case omp::ClauseProcBindKind::Spread:673    return llvm::omp::ProcBindKind::OMP_PROC_BIND_spread;674  }675  llvm_unreachable("Unknown ClauseProcBindKind kind");676}677 678/// Maps block arguments from \p blockArgIface (which are MLIR values) to the679/// corresponding LLVM values of \p the interface's operands. This is useful680/// when an OpenMP region with entry block arguments is converted to LLVM. In681/// this case the block arguments are (part of) of the OpenMP region's entry682/// arguments and the operands are (part of) of the operands to the OpenMP op683/// containing the region.684static void forwardArgs(LLVM::ModuleTranslation &moduleTranslation,685                        omp::BlockArgOpenMPOpInterface blockArgIface) {686  llvm::SmallVector<std::pair<Value, BlockArgument>> blockArgsPairs;687  blockArgIface.getBlockArgsPairs(blockArgsPairs);688  for (auto [var, arg] : blockArgsPairs)689    moduleTranslation.mapValue(arg, moduleTranslation.lookupValue(var));690}691 692/// Converts an OpenMP 'masked' operation into LLVM IR using OpenMPIRBuilder.693static LogicalResult694convertOmpMasked(Operation &opInst, llvm::IRBuilderBase &builder,695                 LLVM::ModuleTranslation &moduleTranslation) {696  auto maskedOp = cast<omp::MaskedOp>(opInst);697  using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;698 699  if (failed(checkImplementationStatus(opInst)))700    return failure();701 702  auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {703    // MaskedOp has only one region associated with it.704    auto &region = maskedOp.getRegion();705    builder.restoreIP(codeGenIP);706    return convertOmpOpRegions(region, "omp.masked.region", builder,707                               moduleTranslation)708        .takeError();709  };710 711  // TODO: Perform finalization actions for variables. This has to be712  // called for variables which have destructors/finalizers.713  auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); };714 715  llvm::Value *filterVal = nullptr;716  if (auto filterVar = maskedOp.getFilteredThreadId()) {717    filterVal = moduleTranslation.lookupValue(filterVar);718  } else {719    llvm::LLVMContext &llvmContext = builder.getContext();720    filterVal =721        llvm::ConstantInt::get(llvm::Type::getInt32Ty(llvmContext), /*V=*/0);722  }723  assert(filterVal != nullptr);724  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);725  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =726      moduleTranslation.getOpenMPBuilder()->createMasked(ompLoc, bodyGenCB,727                                                         finiCB, filterVal);728 729  if (failed(handleError(afterIP, opInst)))730    return failure();731 732  builder.restoreIP(*afterIP);733  return success();734}735 736/// Converts an OpenMP 'master' operation into LLVM IR using OpenMPIRBuilder.737static LogicalResult738convertOmpMaster(Operation &opInst, llvm::IRBuilderBase &builder,739                 LLVM::ModuleTranslation &moduleTranslation) {740  using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;741  auto masterOp = cast<omp::MasterOp>(opInst);742 743  if (failed(checkImplementationStatus(opInst)))744    return failure();745 746  auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {747    // MasterOp has only one region associated with it.748    auto &region = masterOp.getRegion();749    builder.restoreIP(codeGenIP);750    return convertOmpOpRegions(region, "omp.master.region", builder,751                               moduleTranslation)752        .takeError();753  };754 755  // TODO: Perform finalization actions for variables. This has to be756  // called for variables which have destructors/finalizers.757  auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); };758 759  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);760  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =761      moduleTranslation.getOpenMPBuilder()->createMaster(ompLoc, bodyGenCB,762                                                         finiCB);763 764  if (failed(handleError(afterIP, opInst)))765    return failure();766 767  builder.restoreIP(*afterIP);768  return success();769}770 771/// Converts an OpenMP 'critical' operation into LLVM IR using OpenMPIRBuilder.772static LogicalResult773convertOmpCritical(Operation &opInst, llvm::IRBuilderBase &builder,774                   LLVM::ModuleTranslation &moduleTranslation) {775  using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;776  auto criticalOp = cast<omp::CriticalOp>(opInst);777 778  if (failed(checkImplementationStatus(opInst)))779    return failure();780 781  auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {782    // CriticalOp has only one region associated with it.783    auto &region = cast<omp::CriticalOp>(opInst).getRegion();784    builder.restoreIP(codeGenIP);785    return convertOmpOpRegions(region, "omp.critical.region", builder,786                               moduleTranslation)787        .takeError();788  };789 790  // TODO: Perform finalization actions for variables. This has to be791  // called for variables which have destructors/finalizers.792  auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); };793 794  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);795  llvm::LLVMContext &llvmContext = moduleTranslation.getLLVMContext();796  llvm::Constant *hint = nullptr;797 798  // If it has a name, it probably has a hint too.799  if (criticalOp.getNameAttr()) {800    // The verifiers in OpenMP Dialect guarentee that all the pointers are801    // non-null802    auto symbolRef = cast<SymbolRefAttr>(criticalOp.getNameAttr());803    auto criticalDeclareOp =804        SymbolTable::lookupNearestSymbolFrom<omp::CriticalDeclareOp>(criticalOp,805                                                                     symbolRef);806    hint =807        llvm::ConstantInt::get(llvm::Type::getInt32Ty(llvmContext),808                               static_cast<int>(criticalDeclareOp.getHint()));809  }810  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =811      moduleTranslation.getOpenMPBuilder()->createCritical(812          ompLoc, bodyGenCB, finiCB, criticalOp.getName().value_or(""), hint);813 814  if (failed(handleError(afterIP, opInst)))815    return failure();816 817  builder.restoreIP(*afterIP);818  return success();819}820 821/// A util to collect info needed to convert delayed privatizers from MLIR to822/// LLVM.823struct PrivateVarsInfo {824  template <typename OP>825  PrivateVarsInfo(OP op)826      : blockArgs(827            cast<omp::BlockArgOpenMPOpInterface>(*op).getPrivateBlockArgs()) {828    mlirVars.reserve(blockArgs.size());829    llvmVars.reserve(blockArgs.size());830    collectPrivatizationDecls<OP>(op);831 832    for (mlir::Value privateVar : op.getPrivateVars())833      mlirVars.push_back(privateVar);834  }835 836  MutableArrayRef<BlockArgument> blockArgs;837  SmallVector<mlir::Value> mlirVars;838  SmallVector<llvm::Value *> llvmVars;839  SmallVector<omp::PrivateClauseOp> privatizers;840 841private:842  /// Populates `privatizations` with privatization declarations used for the843  /// given op.844  template <class OP>845  void collectPrivatizationDecls(OP op) {846    std::optional<ArrayAttr> attr = op.getPrivateSyms();847    if (!attr)848      return;849 850    privatizers.reserve(privatizers.size() + attr->size());851    for (auto symbolRef : attr->getAsRange<SymbolRefAttr>()) {852      privatizers.push_back(findPrivatizer(op, symbolRef));853    }854  }855};856 857/// Populates `reductions` with reduction declarations used in the given op.858template <typename T>859static void860collectReductionDecls(T op,861                      SmallVectorImpl<omp::DeclareReductionOp> &reductions) {862  std::optional<ArrayAttr> attr = op.getReductionSyms();863  if (!attr)864    return;865 866  reductions.reserve(reductions.size() + op.getNumReductionVars());867  for (auto symbolRef : attr->getAsRange<SymbolRefAttr>()) {868    reductions.push_back(869        SymbolTable::lookupNearestSymbolFrom<omp::DeclareReductionOp>(870            op, symbolRef));871  }872}873 874/// Translates the blocks contained in the given region and appends them to at875/// the current insertion point of `builder`. The operations of the entry block876/// are appended to the current insertion block. If set, `continuationBlockArgs`877/// is populated with translated values that correspond to the values878/// omp.yield'ed from the region.879static LogicalResult inlineConvertOmpRegions(880    Region &region, StringRef blockName, llvm::IRBuilderBase &builder,881    LLVM::ModuleTranslation &moduleTranslation,882    SmallVectorImpl<llvm::Value *> *continuationBlockArgs = nullptr) {883  if (region.empty())884    return success();885 886  // Special case for single-block regions that don't create additional blocks:887  // insert operations without creating additional blocks.888  if (region.hasOneBlock()) {889    llvm::Instruction *potentialTerminator =890        builder.GetInsertBlock()->empty() ? nullptr891                                          : &builder.GetInsertBlock()->back();892 893    if (potentialTerminator && potentialTerminator->isTerminator())894      potentialTerminator->removeFromParent();895    moduleTranslation.mapBlock(&region.front(), builder.GetInsertBlock());896 897    if (failed(moduleTranslation.convertBlock(898            region.front(), /*ignoreArguments=*/true, builder)))899      return failure();900 901    // The continuation arguments are simply the translated terminator operands.902    if (continuationBlockArgs)903      llvm::append_range(904          *continuationBlockArgs,905          moduleTranslation.lookupValues(region.front().back().getOperands()));906 907    // Drop the mapping that is no longer necessary so that the same region can908    // be processed multiple times.909    moduleTranslation.forgetMapping(region);910 911    if (potentialTerminator && potentialTerminator->isTerminator()) {912      llvm::BasicBlock *block = builder.GetInsertBlock();913      if (block->empty()) {914        // this can happen for really simple reduction init regions e.g.915        // %0 = llvm.mlir.constant(0 : i32) : i32916        // omp.yield(%0 : i32)917        // because the llvm.mlir.constant (MLIR op) isn't converted into any918        // llvm op919        potentialTerminator->insertInto(block, block->begin());920      } else {921        potentialTerminator->insertAfter(&block->back());922      }923    }924 925    return success();926  }927 928  SmallVector<llvm::PHINode *> phis;929  llvm::Expected<llvm::BasicBlock *> continuationBlock =930      convertOmpOpRegions(region, blockName, builder, moduleTranslation, &phis);931 932  if (failed(handleError(continuationBlock, *region.getParentOp())))933    return failure();934 935  if (continuationBlockArgs)936    llvm::append_range(*continuationBlockArgs, phis);937  builder.SetInsertPoint(*continuationBlock,938                         (*continuationBlock)->getFirstInsertionPt());939  return success();940}941 942namespace {943/// Owning equivalents of OpenMPIRBuilder::(Atomic)ReductionGen that are used to944/// store lambdas with capture.945using OwningReductionGen =946    std::function<llvm::OpenMPIRBuilder::InsertPointOrErrorTy(947        llvm::OpenMPIRBuilder::InsertPointTy, llvm::Value *, llvm::Value *,948        llvm::Value *&)>;949using OwningAtomicReductionGen =950    std::function<llvm::OpenMPIRBuilder::InsertPointOrErrorTy(951        llvm::OpenMPIRBuilder::InsertPointTy, llvm::Type *, llvm::Value *,952        llvm::Value *)>;953using OwningDataPtrPtrReductionGen =954    std::function<llvm::OpenMPIRBuilder::InsertPointOrErrorTy(955        llvm::OpenMPIRBuilder::InsertPointTy, llvm::Value *, llvm::Value *&)>;956} // namespace957 958/// Create an OpenMPIRBuilder-compatible reduction generator for the given959/// reduction declaration. The generator uses `builder` but ignores its960/// insertion point.961static OwningReductionGen962makeReductionGen(omp::DeclareReductionOp decl, llvm::IRBuilderBase &builder,963                 LLVM::ModuleTranslation &moduleTranslation) {964  // The lambda is mutable because we need access to non-const methods of decl965  // (which aren't actually mutating it), and we must capture decl by-value to966  // avoid the dangling reference after the parent function returns.967  OwningReductionGen gen =968      [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint,969                llvm::Value *lhs, llvm::Value *rhs,970                llvm::Value *&result) mutable971      -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {972    moduleTranslation.mapValue(decl.getReductionLhsArg(), lhs);973    moduleTranslation.mapValue(decl.getReductionRhsArg(), rhs);974    builder.restoreIP(insertPoint);975    SmallVector<llvm::Value *> phis;976    if (failed(inlineConvertOmpRegions(decl.getReductionRegion(),977                                       "omp.reduction.nonatomic.body", builder,978                                       moduleTranslation, &phis)))979      return llvm::createStringError(980          "failed to inline `combiner` region of `omp.declare_reduction`");981    result = llvm::getSingleElement(phis);982    return builder.saveIP();983  };984  return gen;985}986 987/// Create an OpenMPIRBuilder-compatible atomic reduction generator for the988/// given reduction declaration. The generator uses `builder` but ignores its989/// insertion point. Returns null if there is no atomic region available in the990/// reduction declaration.991static OwningAtomicReductionGen992makeAtomicReductionGen(omp::DeclareReductionOp decl,993                       llvm::IRBuilderBase &builder,994                       LLVM::ModuleTranslation &moduleTranslation) {995  if (decl.getAtomicReductionRegion().empty())996    return OwningAtomicReductionGen();997 998  // The lambda is mutable because we need access to non-const methods of decl999  // (which aren't actually mutating it), and we must capture decl by-value to1000  // avoid the dangling reference after the parent function returns.1001  OwningAtomicReductionGen atomicGen =1002      [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint, llvm::Type *,1003                llvm::Value *lhs, llvm::Value *rhs) mutable1004      -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {1005    moduleTranslation.mapValue(decl.getAtomicReductionLhsArg(), lhs);1006    moduleTranslation.mapValue(decl.getAtomicReductionRhsArg(), rhs);1007    builder.restoreIP(insertPoint);1008    SmallVector<llvm::Value *> phis;1009    if (failed(inlineConvertOmpRegions(decl.getAtomicReductionRegion(),1010                                       "omp.reduction.atomic.body", builder,1011                                       moduleTranslation, &phis)))1012      return llvm::createStringError(1013          "failed to inline `atomic` region of `omp.declare_reduction`");1014    assert(phis.empty());1015    return builder.saveIP();1016  };1017  return atomicGen;1018}1019 1020/// Create an OpenMPIRBuilder-compatible `data_ptr_ptr` reduction generator for1021/// the given reduction declaration. The generator uses `builder` but ignores1022/// its insertion point. Returns null if there is no `data_ptr_ptr` region1023/// available in the reduction declaration.1024static OwningDataPtrPtrReductionGen1025makeRefDataPtrGen(omp::DeclareReductionOp decl, llvm::IRBuilderBase &builder,1026                  LLVM::ModuleTranslation &moduleTranslation, bool isByRef) {1027  if (!isByRef)1028    return OwningDataPtrPtrReductionGen();1029 1030  OwningDataPtrPtrReductionGen refDataPtrGen =1031      [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint,1032                llvm::Value *byRefVal, llvm::Value *&result) mutable1033      -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {1034    moduleTranslation.mapValue(decl.getDataPtrPtrRegionArg(), byRefVal);1035    builder.restoreIP(insertPoint);1036    SmallVector<llvm::Value *> phis;1037    if (failed(inlineConvertOmpRegions(decl.getDataPtrPtrRegion(),1038                                       "omp.data_ptr_ptr.body", builder,1039                                       moduleTranslation, &phis)))1040      return llvm::createStringError(1041          "failed to inline `data_ptr_ptr` region of `omp.declare_reduction`");1042    result = llvm::getSingleElement(phis);1043    return builder.saveIP();1044  };1045 1046  return refDataPtrGen;1047}1048 1049/// Converts an OpenMP 'ordered' operation into LLVM IR using OpenMPIRBuilder.1050static LogicalResult1051convertOmpOrdered(Operation &opInst, llvm::IRBuilderBase &builder,1052                  LLVM::ModuleTranslation &moduleTranslation) {1053  auto orderedOp = cast<omp::OrderedOp>(opInst);1054 1055  if (failed(checkImplementationStatus(opInst)))1056    return failure();1057 1058  omp::ClauseDepend dependType = *orderedOp.getDoacrossDependType();1059  bool isDependSource = dependType == omp::ClauseDepend::dependsource;1060  unsigned numLoops = *orderedOp.getDoacrossNumLoops();1061  SmallVector<llvm::Value *> vecValues =1062      moduleTranslation.lookupValues(orderedOp.getDoacrossDependVars());1063 1064  size_t indexVecValues = 0;1065  while (indexVecValues < vecValues.size()) {1066    SmallVector<llvm::Value *> storeValues;1067    storeValues.reserve(numLoops);1068    for (unsigned i = 0; i < numLoops; i++) {1069      storeValues.push_back(vecValues[indexVecValues]);1070      indexVecValues++;1071    }1072    llvm::OpenMPIRBuilder::InsertPointTy allocaIP =1073        findAllocaInsertPoint(builder, moduleTranslation);1074    llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);1075    builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createOrderedDepend(1076        ompLoc, allocaIP, numLoops, storeValues, ".cnt.addr", isDependSource));1077  }1078  return success();1079}1080 1081/// Converts an OpenMP 'ordered_region' operation into LLVM IR using1082/// OpenMPIRBuilder.1083static LogicalResult1084convertOmpOrderedRegion(Operation &opInst, llvm::IRBuilderBase &builder,1085                        LLVM::ModuleTranslation &moduleTranslation) {1086  using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;1087  auto orderedRegionOp = cast<omp::OrderedRegionOp>(opInst);1088 1089  if (failed(checkImplementationStatus(opInst)))1090    return failure();1091 1092  auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {1093    // OrderedOp has only one region associated with it.1094    auto &region = cast<omp::OrderedRegionOp>(opInst).getRegion();1095    builder.restoreIP(codeGenIP);1096    return convertOmpOpRegions(region, "omp.ordered.region", builder,1097                               moduleTranslation)1098        .takeError();1099  };1100 1101  // TODO: Perform finalization actions for variables. This has to be1102  // called for variables which have destructors/finalizers.1103  auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); };1104 1105  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);1106  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =1107      moduleTranslation.getOpenMPBuilder()->createOrderedThreadsSimd(1108          ompLoc, bodyGenCB, finiCB, !orderedRegionOp.getParLevelSimd());1109 1110  if (failed(handleError(afterIP, opInst)))1111    return failure();1112 1113  builder.restoreIP(*afterIP);1114  return success();1115}1116 1117namespace {1118/// Contains the arguments for an LLVM store operation1119struct DeferredStore {1120  DeferredStore(llvm::Value *value, llvm::Value *address)1121      : value(value), address(address) {}1122 1123  llvm::Value *value;1124  llvm::Value *address;1125};1126} // namespace1127 1128/// Allocate space for privatized reduction variables.1129/// `deferredStores` contains information to create store operations which needs1130/// to be inserted after all allocas1131template <typename T>1132static LogicalResult1133allocReductionVars(T loop, ArrayRef<BlockArgument> reductionArgs,1134                   llvm::IRBuilderBase &builder,1135                   LLVM::ModuleTranslation &moduleTranslation,1136                   const llvm::OpenMPIRBuilder::InsertPointTy &allocaIP,1137                   SmallVectorImpl<omp::DeclareReductionOp> &reductionDecls,1138                   SmallVectorImpl<llvm::Value *> &privateReductionVariables,1139                   DenseMap<Value, llvm::Value *> &reductionVariableMap,1140                   SmallVectorImpl<DeferredStore> &deferredStores,1141                   llvm::ArrayRef<bool> isByRefs) {1142  llvm::IRBuilderBase::InsertPointGuard guard(builder);1143  builder.SetInsertPoint(allocaIP.getBlock()->getTerminator());1144 1145  // delay creating stores until after all allocas1146  deferredStores.reserve(loop.getNumReductionVars());1147 1148  for (std::size_t i = 0; i < loop.getNumReductionVars(); ++i) {1149    Region &allocRegion = reductionDecls[i].getAllocRegion();1150    if (isByRefs[i]) {1151      if (allocRegion.empty())1152        continue;1153 1154      SmallVector<llvm::Value *, 1> phis;1155      if (failed(inlineConvertOmpRegions(allocRegion, "omp.reduction.alloc",1156                                         builder, moduleTranslation, &phis)))1157        return loop.emitError(1158            "failed to inline `alloc` region of `omp.declare_reduction`");1159 1160      assert(phis.size() == 1 && "expected one allocation to be yielded");1161      builder.SetInsertPoint(allocaIP.getBlock()->getTerminator());1162 1163      // Allocate reduction variable (which is a pointer to the real reduction1164      // variable allocated in the inlined region)1165      llvm::Value *var = builder.CreateAlloca(1166          moduleTranslation.convertType(reductionDecls[i].getType()));1167 1168      llvm::Type *ptrTy = builder.getPtrTy();1169      llvm::Value *castVar =1170          builder.CreatePointerBitCastOrAddrSpaceCast(var, ptrTy);1171      llvm::Value *castPhi =1172          builder.CreatePointerBitCastOrAddrSpaceCast(phis[0], ptrTy);1173 1174      deferredStores.emplace_back(castPhi, castVar);1175 1176      privateReductionVariables[i] = castVar;1177      moduleTranslation.mapValue(reductionArgs[i], castPhi);1178      reductionVariableMap.try_emplace(loop.getReductionVars()[i], castPhi);1179    } else {1180      assert(allocRegion.empty() &&1181             "allocaction is implicit for by-val reduction");1182      llvm::Value *var = builder.CreateAlloca(1183          moduleTranslation.convertType(reductionDecls[i].getType()));1184 1185      llvm::Type *ptrTy = builder.getPtrTy();1186      llvm::Value *castVar =1187          builder.CreatePointerBitCastOrAddrSpaceCast(var, ptrTy);1188 1189      moduleTranslation.mapValue(reductionArgs[i], castVar);1190      privateReductionVariables[i] = castVar;1191      reductionVariableMap.try_emplace(loop.getReductionVars()[i], castVar);1192    }1193  }1194 1195  return success();1196}1197 1198/// Map input arguments to reduction initialization region1199template <typename T>1200static void1201mapInitializationArgs(T loop, LLVM::ModuleTranslation &moduleTranslation,1202                      llvm::IRBuilderBase &builder,1203                      SmallVectorImpl<omp::DeclareReductionOp> &reductionDecls,1204                      DenseMap<Value, llvm::Value *> &reductionVariableMap,1205                      unsigned i) {1206  // map input argument to the initialization region1207  mlir::omp::DeclareReductionOp &reduction = reductionDecls[i];1208  Region &initializerRegion = reduction.getInitializerRegion();1209  Block &entry = initializerRegion.front();1210 1211  mlir::Value mlirSource = loop.getReductionVars()[i];1212  llvm::Value *llvmSource = moduleTranslation.lookupValue(mlirSource);1213  llvm::Value *origVal = llvmSource;1214  // If a non-pointer value is expected, load the value from the source pointer.1215  if (!isa<LLVM::LLVMPointerType>(1216          reduction.getInitializerMoldArg().getType()) &&1217      isa<LLVM::LLVMPointerType>(mlirSource.getType())) {1218    origVal =1219        builder.CreateLoad(moduleTranslation.convertType(1220                               reduction.getInitializerMoldArg().getType()),1221                           llvmSource, "omp_orig");1222  }1223  moduleTranslation.mapValue(reduction.getInitializerMoldArg(), origVal);1224 1225  if (entry.getNumArguments() > 1) {1226    llvm::Value *allocation =1227        reductionVariableMap.lookup(loop.getReductionVars()[i]);1228    moduleTranslation.mapValue(reduction.getInitializerAllocArg(), allocation);1229  }1230}1231 1232static void1233setInsertPointForPossiblyEmptyBlock(llvm::IRBuilderBase &builder,1234                                    llvm::BasicBlock *block = nullptr) {1235  if (block == nullptr)1236    block = builder.GetInsertBlock();1237 1238  if (block->empty() || block->getTerminator() == nullptr)1239    builder.SetInsertPoint(block);1240  else1241    builder.SetInsertPoint(block->getTerminator());1242}1243 1244/// Inline reductions' `init` regions. This functions assumes that the1245/// `builder`'s insertion point is where the user wants the `init` regions to be1246/// inlined; i.e. it does not try to find a proper insertion location for the1247/// `init` regions. It also leaves the `builder's insertions point in a state1248/// where the user can continue the code-gen directly afterwards.1249template <typename OP>1250static LogicalResult1251initReductionVars(OP op, ArrayRef<BlockArgument> reductionArgs,1252                  llvm::IRBuilderBase &builder,1253                  LLVM::ModuleTranslation &moduleTranslation,1254                  llvm::BasicBlock *latestAllocaBlock,1255                  SmallVectorImpl<omp::DeclareReductionOp> &reductionDecls,1256                  SmallVectorImpl<llvm::Value *> &privateReductionVariables,1257                  DenseMap<Value, llvm::Value *> &reductionVariableMap,1258                  llvm::ArrayRef<bool> isByRef,1259                  SmallVectorImpl<DeferredStore> &deferredStores) {1260  if (op.getNumReductionVars() == 0)1261    return success();1262 1263  llvm::BasicBlock *initBlock = splitBB(builder, true, "omp.reduction.init");1264  auto allocaIP = llvm::IRBuilderBase::InsertPoint(1265      latestAllocaBlock, latestAllocaBlock->getTerminator()->getIterator());1266  builder.restoreIP(allocaIP);1267  SmallVector<llvm::Value *> byRefVars(op.getNumReductionVars());1268 1269  for (unsigned i = 0; i < op.getNumReductionVars(); ++i) {1270    if (isByRef[i]) {1271      if (!reductionDecls[i].getAllocRegion().empty())1272        continue;1273 1274      // TODO: remove after all users of by-ref are updated to use the alloc1275      // region: Allocate reduction variable (which is a pointer to the real1276      // reduciton variable allocated in the inlined region)1277      byRefVars[i] = builder.CreateAlloca(1278          moduleTranslation.convertType(reductionDecls[i].getType()));1279    }1280  }1281 1282  setInsertPointForPossiblyEmptyBlock(builder, initBlock);1283 1284  // store result of the alloc region to the allocated pointer to the real1285  // reduction variable1286  for (auto [data, addr] : deferredStores)1287    builder.CreateStore(data, addr);1288 1289  // Before the loop, store the initial values of reductions into reduction1290  // variables. Although this could be done after allocas, we don't want to mess1291  // up with the alloca insertion point.1292  for (unsigned i = 0; i < op.getNumReductionVars(); ++i) {1293    SmallVector<llvm::Value *, 1> phis;1294 1295    // map block argument to initializer region1296    mapInitializationArgs(op, moduleTranslation, builder, reductionDecls,1297                          reductionVariableMap, i);1298 1299    // TODO In some cases (specially on the GPU), the init regions may1300    // contains stack alloctaions. If the region is inlined in a loop, this is1301    // problematic. Instead of just inlining the region, handle allocations by1302    // hoisting fixed length allocations to the function entry and using1303    // stacksave and restore for variable length ones.1304    if (failed(inlineConvertOmpRegions(reductionDecls[i].getInitializerRegion(),1305                                       "omp.reduction.neutral", builder,1306                                       moduleTranslation, &phis)))1307      return failure();1308 1309    assert(phis.size() == 1 && "expected one value to be yielded from the "1310                               "reduction neutral element declaration region");1311 1312    setInsertPointForPossiblyEmptyBlock(builder);1313 1314    if (isByRef[i]) {1315      if (!reductionDecls[i].getAllocRegion().empty())1316        // done in allocReductionVars1317        continue;1318 1319      // TODO: this path can be removed once all users of by-ref are updated to1320      // use an alloc region1321 1322      // Store the result of the inlined region to the allocated reduction var1323      // ptr1324      builder.CreateStore(phis[0], byRefVars[i]);1325 1326      privateReductionVariables[i] = byRefVars[i];1327      moduleTranslation.mapValue(reductionArgs[i], phis[0]);1328      reductionVariableMap.try_emplace(op.getReductionVars()[i], phis[0]);1329    } else {1330      // for by-ref case the store is inside of the reduction region1331      builder.CreateStore(phis[0], privateReductionVariables[i]);1332      // the rest was handled in allocByValReductionVars1333    }1334 1335    // forget the mapping for the initializer region because we might need a1336    // different mapping if this reduction declaration is re-used for a1337    // different variable1338    moduleTranslation.forgetMapping(reductionDecls[i].getInitializerRegion());1339  }1340 1341  return success();1342}1343 1344/// Collect reduction info1345template <typename T>1346static void collectReductionInfo(1347    T loop, llvm::IRBuilderBase &builder,1348    LLVM::ModuleTranslation &moduleTranslation,1349    SmallVectorImpl<omp::DeclareReductionOp> &reductionDecls,1350    SmallVectorImpl<OwningReductionGen> &owningReductionGens,1351    SmallVectorImpl<OwningAtomicReductionGen> &owningAtomicReductionGens,1352    SmallVector<OwningDataPtrPtrReductionGen> &owningDataPtrPtrReductionGens,1353    const ArrayRef<llvm::Value *> privateReductionVariables,1354    SmallVectorImpl<llvm::OpenMPIRBuilder::ReductionInfo> &reductionInfos,1355    ArrayRef<bool> isByRef) {1356  unsigned numReductions = loop.getNumReductionVars();1357 1358  for (unsigned i = 0; i < numReductions; ++i) {1359    owningReductionGens.push_back(1360        makeReductionGen(reductionDecls[i], builder, moduleTranslation));1361    owningAtomicReductionGens.push_back(1362        makeAtomicReductionGen(reductionDecls[i], builder, moduleTranslation));1363    owningDataPtrPtrReductionGens.push_back(makeRefDataPtrGen(1364        reductionDecls[i], builder, moduleTranslation, isByRef[i]));1365  }1366 1367  // Collect the reduction information.1368  reductionInfos.reserve(numReductions);1369  for (unsigned i = 0; i < numReductions; ++i) {1370    llvm::OpenMPIRBuilder::ReductionGenAtomicCBTy atomicGen = nullptr;1371    if (owningAtomicReductionGens[i])1372      atomicGen = owningAtomicReductionGens[i];1373    llvm::Value *variable =1374        moduleTranslation.lookupValue(loop.getReductionVars()[i]);1375    mlir::Type allocatedType;1376    reductionDecls[i].getAllocRegion().walk([&](mlir::Operation *op) {1377      if (auto alloca = mlir::dyn_cast<LLVM::AllocaOp>(op)) {1378        allocatedType = alloca.getElemType();1379        return mlir::WalkResult::interrupt();1380      }1381 1382      return mlir::WalkResult::advance();1383    });1384 1385    reductionInfos.push_back(1386        {moduleTranslation.convertType(reductionDecls[i].getType()), variable,1387         privateReductionVariables[i],1388         /*EvaluationKind=*/llvm::OpenMPIRBuilder::EvalKind::Scalar,1389         owningReductionGens[i],1390         /*ReductionGenClang=*/nullptr, atomicGen,1391         owningDataPtrPtrReductionGens[i],1392         allocatedType ? moduleTranslation.convertType(allocatedType) : nullptr,1393         reductionDecls[i].getByrefElementType()1394             ? moduleTranslation.convertType(1395                   *reductionDecls[i].getByrefElementType())1396             : nullptr});1397  }1398}1399 1400/// handling of DeclareReductionOp's cleanup region1401static LogicalResult1402inlineOmpRegionCleanup(llvm::SmallVectorImpl<Region *> &cleanupRegions,1403                       llvm::ArrayRef<llvm::Value *> privateVariables,1404                       LLVM::ModuleTranslation &moduleTranslation,1405                       llvm::IRBuilderBase &builder, StringRef regionName,1406                       bool shouldLoadCleanupRegionArg = true) {1407  for (auto [i, cleanupRegion] : llvm::enumerate(cleanupRegions)) {1408    if (cleanupRegion->empty())1409      continue;1410 1411    // map the argument to the cleanup region1412    Block &entry = cleanupRegion->front();1413 1414    llvm::Instruction *potentialTerminator =1415        builder.GetInsertBlock()->empty() ? nullptr1416                                          : &builder.GetInsertBlock()->back();1417    if (potentialTerminator && potentialTerminator->isTerminator())1418      builder.SetInsertPoint(potentialTerminator);1419    llvm::Value *privateVarValue =1420        shouldLoadCleanupRegionArg1421            ? builder.CreateLoad(1422                  moduleTranslation.convertType(entry.getArgument(0).getType()),1423                  privateVariables[i])1424            : privateVariables[i];1425 1426    moduleTranslation.mapValue(entry.getArgument(0), privateVarValue);1427 1428    if (failed(inlineConvertOmpRegions(*cleanupRegion, regionName, builder,1429                                       moduleTranslation)))1430      return failure();1431 1432    // clear block argument mapping in case it needs to be re-created with a1433    // different source for another use of the same reduction decl1434    moduleTranslation.forgetMapping(*cleanupRegion);1435  }1436  return success();1437}1438 1439// TODO: not used by ParallelOp1440template <class OP>1441static LogicalResult createReductionsAndCleanup(1442    OP op, llvm::IRBuilderBase &builder,1443    LLVM::ModuleTranslation &moduleTranslation,1444    llvm::OpenMPIRBuilder::InsertPointTy &allocaIP,1445    SmallVectorImpl<omp::DeclareReductionOp> &reductionDecls,1446    ArrayRef<llvm::Value *> privateReductionVariables, ArrayRef<bool> isByRef,1447    bool isNowait = false, bool isTeamsReduction = false) {1448  // Process the reductions if required.1449  if (op.getNumReductionVars() == 0)1450    return success();1451 1452  SmallVector<OwningReductionGen> owningReductionGens;1453  SmallVector<OwningAtomicReductionGen> owningAtomicReductionGens;1454  SmallVector<OwningDataPtrPtrReductionGen> owningReductionGenRefDataPtrGens;1455  SmallVector<llvm::OpenMPIRBuilder::ReductionInfo, 2> reductionInfos;1456 1457  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();1458 1459  // Create the reduction generators. We need to own them here because1460  // ReductionInfo only accepts references to the generators.1461  collectReductionInfo(op, builder, moduleTranslation, reductionDecls,1462                       owningReductionGens, owningAtomicReductionGens,1463                       owningReductionGenRefDataPtrGens,1464                       privateReductionVariables, reductionInfos, isByRef);1465 1466  // The call to createReductions below expects the block to have a1467  // terminator. Create an unreachable instruction to serve as terminator1468  // and remove it later.1469  llvm::UnreachableInst *tempTerminator = builder.CreateUnreachable();1470  builder.SetInsertPoint(tempTerminator);1471  llvm::OpenMPIRBuilder::InsertPointOrErrorTy contInsertPoint =1472      ompBuilder->createReductions(builder.saveIP(), allocaIP, reductionInfos,1473                                   isByRef, isNowait, isTeamsReduction);1474 1475  if (failed(handleError(contInsertPoint, *op)))1476    return failure();1477 1478  if (!contInsertPoint->getBlock())1479    return op->emitOpError() << "failed to convert reductions";1480 1481  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =1482      ompBuilder->createBarrier(*contInsertPoint, llvm::omp::OMPD_for);1483 1484  if (failed(handleError(afterIP, *op)))1485    return failure();1486 1487  tempTerminator->eraseFromParent();1488  builder.restoreIP(*afterIP);1489 1490  // after the construct, deallocate private reduction variables1491  SmallVector<Region *> reductionRegions;1492  llvm::transform(reductionDecls, std::back_inserter(reductionRegions),1493                  [](omp::DeclareReductionOp reductionDecl) {1494                    return &reductionDecl.getCleanupRegion();1495                  });1496  return inlineOmpRegionCleanup(reductionRegions, privateReductionVariables,1497                                moduleTranslation, builder,1498                                "omp.reduction.cleanup");1499  return success();1500}1501 1502static ArrayRef<bool> getIsByRef(std::optional<ArrayRef<bool>> attr) {1503  if (!attr)1504    return {};1505  return *attr;1506}1507 1508// TODO: not used by omp.parallel1509template <typename OP>1510static LogicalResult allocAndInitializeReductionVars(1511    OP op, ArrayRef<BlockArgument> reductionArgs, llvm::IRBuilderBase &builder,1512    LLVM::ModuleTranslation &moduleTranslation,1513    llvm::OpenMPIRBuilder::InsertPointTy &allocaIP,1514    SmallVectorImpl<omp::DeclareReductionOp> &reductionDecls,1515    SmallVectorImpl<llvm::Value *> &privateReductionVariables,1516    DenseMap<Value, llvm::Value *> &reductionVariableMap,1517    llvm::ArrayRef<bool> isByRef) {1518  if (op.getNumReductionVars() == 0)1519    return success();1520 1521  SmallVector<DeferredStore> deferredStores;1522 1523  if (failed(allocReductionVars(op, reductionArgs, builder, moduleTranslation,1524                                allocaIP, reductionDecls,1525                                privateReductionVariables, reductionVariableMap,1526                                deferredStores, isByRef)))1527    return failure();1528 1529  return initReductionVars(op, reductionArgs, builder, moduleTranslation,1530                           allocaIP.getBlock(), reductionDecls,1531                           privateReductionVariables, reductionVariableMap,1532                           isByRef, deferredStores);1533}1534 1535/// Return the llvm::Value * corresponding to the `privateVar` that1536/// is being privatized. It isn't always as simple as looking up1537/// moduleTranslation with privateVar. For instance, in case of1538/// an allocatable, the descriptor for the allocatable is privatized.1539/// This descriptor is mapped using an MapInfoOp. So, this function1540/// will return a pointer to the llvm::Value corresponding to the1541/// block argument for the mapped descriptor.1542static llvm::Value *1543findAssociatedValue(Value privateVar, llvm::IRBuilderBase &builder,1544                    LLVM::ModuleTranslation &moduleTranslation,1545                    llvm::DenseMap<Value, Value> *mappedPrivateVars = nullptr) {1546  if (mappedPrivateVars == nullptr || !mappedPrivateVars->contains(privateVar))1547    return moduleTranslation.lookupValue(privateVar);1548 1549  Value blockArg = (*mappedPrivateVars)[privateVar];1550  Type privVarType = privateVar.getType();1551  Type blockArgType = blockArg.getType();1552  assert(isa<LLVM::LLVMPointerType>(blockArgType) &&1553         "A block argument corresponding to a mapped var should have "1554         "!llvm.ptr type");1555 1556  if (privVarType == blockArgType)1557    return moduleTranslation.lookupValue(blockArg);1558 1559  // This typically happens when the privatized type is lowered from1560  // boxchar<KIND> and gets lowered to !llvm.struct<(ptr, i64)>. That is the1561  // struct/pair is passed by value. But, mapped values are passed only as1562  // pointers, so before we privatize, we must load the pointer.1563  if (!isa<LLVM::LLVMPointerType>(privVarType))1564    return builder.CreateLoad(moduleTranslation.convertType(privVarType),1565                              moduleTranslation.lookupValue(blockArg));1566 1567  return moduleTranslation.lookupValue(privateVar);1568}1569 1570/// Initialize a single (first)private variable. You probably want to use1571/// allocateAndInitPrivateVars instead of this.1572/// This returns the private variable which has been initialized. This1573/// variable should be mapped before constructing the body of the Op.1574static llvm::Expected<llvm::Value *> initPrivateVar(1575    llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation,1576    omp::PrivateClauseOp &privDecl, Value mlirPrivVar, BlockArgument &blockArg,1577    llvm::Value *llvmPrivateVar, llvm::BasicBlock *privInitBlock,1578    llvm::DenseMap<Value, Value> *mappedPrivateVars = nullptr) {1579  Region &initRegion = privDecl.getInitRegion();1580  if (initRegion.empty())1581    return llvmPrivateVar;1582 1583  // map initialization region block arguments1584  llvm::Value *nonPrivateVar = findAssociatedValue(1585      mlirPrivVar, builder, moduleTranslation, mappedPrivateVars);1586  assert(nonPrivateVar);1587  moduleTranslation.mapValue(privDecl.getInitMoldArg(), nonPrivateVar);1588  moduleTranslation.mapValue(privDecl.getInitPrivateArg(), llvmPrivateVar);1589 1590  // in-place convert the private initialization region1591  SmallVector<llvm::Value *, 1> phis;1592  if (failed(inlineConvertOmpRegions(initRegion, "omp.private.init", builder,1593                                     moduleTranslation, &phis)))1594    return llvm::createStringError(1595        "failed to inline `init` region of `omp.private`");1596 1597  assert(phis.size() == 1 && "expected one allocation to be yielded");1598 1599  // clear init region block argument mapping in case it needs to be1600  // re-created with a different source for another use of the same1601  // reduction decl1602  moduleTranslation.forgetMapping(initRegion);1603 1604  // Prefer the value yielded from the init region to the allocated private1605  // variable in case the region is operating on arguments by-value (e.g.1606  // Fortran character boxes).1607  return phis[0];1608}1609 1610static llvm::Error1611initPrivateVars(llvm::IRBuilderBase &builder,1612                LLVM::ModuleTranslation &moduleTranslation,1613                PrivateVarsInfo &privateVarsInfo,1614                llvm::DenseMap<Value, Value> *mappedPrivateVars = nullptr) {1615  if (privateVarsInfo.blockArgs.empty())1616    return llvm::Error::success();1617 1618  llvm::BasicBlock *privInitBlock = splitBB(builder, true, "omp.private.init");1619  setInsertPointForPossiblyEmptyBlock(builder, privInitBlock);1620 1621  for (auto [idx, zip] : llvm::enumerate(llvm::zip_equal(1622           privateVarsInfo.privatizers, privateVarsInfo.mlirVars,1623           privateVarsInfo.blockArgs, privateVarsInfo.llvmVars))) {1624    auto [privDecl, mlirPrivVar, blockArg, llvmPrivateVar] = zip;1625    llvm::Expected<llvm::Value *> privVarOrErr = initPrivateVar(1626        builder, moduleTranslation, privDecl, mlirPrivVar, blockArg,1627        llvmPrivateVar, privInitBlock, mappedPrivateVars);1628 1629    if (!privVarOrErr)1630      return privVarOrErr.takeError();1631 1632    llvmPrivateVar = privVarOrErr.get();1633    moduleTranslation.mapValue(blockArg, llvmPrivateVar);1634 1635    setInsertPointForPossiblyEmptyBlock(builder);1636  }1637 1638  return llvm::Error::success();1639}1640 1641/// Allocate and initialize delayed private variables. Returns the basic block1642/// which comes after all of these allocations. llvm::Value * for each of these1643/// private variables are populated in llvmPrivateVars.1644static llvm::Expected<llvm::BasicBlock *>1645allocatePrivateVars(llvm::IRBuilderBase &builder,1646                    LLVM::ModuleTranslation &moduleTranslation,1647                    PrivateVarsInfo &privateVarsInfo,1648                    const llvm::OpenMPIRBuilder::InsertPointTy &allocaIP,1649                    llvm::DenseMap<Value, Value> *mappedPrivateVars = nullptr) {1650  // Allocate private vars1651  llvm::Instruction *allocaTerminator = allocaIP.getBlock()->getTerminator();1652  splitBB(llvm::OpenMPIRBuilder::InsertPointTy(allocaIP.getBlock(),1653                                               allocaTerminator->getIterator()),1654          true, allocaTerminator->getStableDebugLoc(),1655          "omp.region.after_alloca");1656 1657  llvm::IRBuilderBase::InsertPointGuard guard(builder);1658  // Update the allocaTerminator since the alloca block was split above.1659  allocaTerminator = allocaIP.getBlock()->getTerminator();1660  builder.SetInsertPoint(allocaTerminator);1661  // The new terminator is an uncondition branch created by the splitBB above.1662  assert(allocaTerminator->getNumSuccessors() == 1 &&1663         "This is an unconditional branch created by splitBB");1664 1665  llvm::DataLayout dataLayout = builder.GetInsertBlock()->getDataLayout();1666  llvm::BasicBlock *afterAllocas = allocaTerminator->getSuccessor(0);1667 1668  unsigned int allocaAS =1669      moduleTranslation.getLLVMModule()->getDataLayout().getAllocaAddrSpace();1670  unsigned int defaultAS = moduleTranslation.getLLVMModule()1671                               ->getDataLayout()1672                               .getProgramAddressSpace();1673 1674  for (auto [privDecl, mlirPrivVar, blockArg] :1675       llvm::zip_equal(privateVarsInfo.privatizers, privateVarsInfo.mlirVars,1676                       privateVarsInfo.blockArgs)) {1677    llvm::Type *llvmAllocType =1678        moduleTranslation.convertType(privDecl.getType());1679    builder.SetInsertPoint(allocaIP.getBlock()->getTerminator());1680    llvm::Value *llvmPrivateVar = builder.CreateAlloca(1681        llvmAllocType, /*ArraySize=*/nullptr, "omp.private.alloc");1682    if (allocaAS != defaultAS)1683      llvmPrivateVar = builder.CreateAddrSpaceCast(llvmPrivateVar,1684                                                   builder.getPtrTy(defaultAS));1685 1686    privateVarsInfo.llvmVars.push_back(llvmPrivateVar);1687  }1688 1689  return afterAllocas;1690}1691 1692static LogicalResult copyFirstPrivateVars(1693    mlir::Operation *op, llvm::IRBuilderBase &builder,1694    LLVM::ModuleTranslation &moduleTranslation,1695    SmallVectorImpl<mlir::Value> &mlirPrivateVars,1696    ArrayRef<llvm::Value *> llvmPrivateVars,1697    SmallVectorImpl<omp::PrivateClauseOp> &privateDecls, bool insertBarrier,1698    llvm::DenseMap<Value, Value> *mappedPrivateVars = nullptr) {1699  // Apply copy region for firstprivate.1700  bool needsFirstprivate =1701      llvm::any_of(privateDecls, [](omp::PrivateClauseOp &privOp) {1702        return privOp.getDataSharingType() ==1703               omp::DataSharingClauseType::FirstPrivate;1704      });1705 1706  if (!needsFirstprivate)1707    return success();1708 1709  llvm::BasicBlock *copyBlock =1710      splitBB(builder, /*CreateBranch=*/true, "omp.private.copy");1711  setInsertPointForPossiblyEmptyBlock(builder, copyBlock);1712 1713  for (auto [decl, mlirVar, llvmVar] :1714       llvm::zip_equal(privateDecls, mlirPrivateVars, llvmPrivateVars)) {1715    if (decl.getDataSharingType() != omp::DataSharingClauseType::FirstPrivate)1716      continue;1717 1718    // copyRegion implements `lhs = rhs`1719    Region &copyRegion = decl.getCopyRegion();1720 1721    // map copyRegion rhs arg1722    llvm::Value *nonPrivateVar = findAssociatedValue(1723        mlirVar, builder, moduleTranslation, mappedPrivateVars);1724    assert(nonPrivateVar);1725    moduleTranslation.mapValue(decl.getCopyMoldArg(), nonPrivateVar);1726 1727    // map copyRegion lhs arg1728    moduleTranslation.mapValue(decl.getCopyPrivateArg(), llvmVar);1729 1730    // in-place convert copy region1731    if (failed(inlineConvertOmpRegions(copyRegion, "omp.private.copy", builder,1732                                       moduleTranslation)))1733      return decl.emitError("failed to inline `copy` region of `omp.private`");1734 1735    setInsertPointForPossiblyEmptyBlock(builder);1736 1737    // ignore unused value yielded from copy region1738 1739    // clear copy region block argument mapping in case it needs to be1740    // re-created with different sources for reuse of the same reduction1741    // decl1742    moduleTranslation.forgetMapping(copyRegion);1743  }1744 1745  if (insertBarrier) {1746    llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();1747    llvm::OpenMPIRBuilder::InsertPointOrErrorTy res =1748        ompBuilder->createBarrier(builder.saveIP(), llvm::omp::OMPD_barrier);1749    if (failed(handleError(res, *op)))1750      return failure();1751  }1752 1753  return success();1754}1755 1756static LogicalResult1757cleanupPrivateVars(llvm::IRBuilderBase &builder,1758                   LLVM::ModuleTranslation &moduleTranslation, Location loc,1759                   SmallVectorImpl<llvm::Value *> &llvmPrivateVars,1760                   SmallVectorImpl<omp::PrivateClauseOp> &privateDecls) {1761  // private variable deallocation1762  SmallVector<Region *> privateCleanupRegions;1763  llvm::transform(privateDecls, std::back_inserter(privateCleanupRegions),1764                  [](omp::PrivateClauseOp privatizer) {1765                    return &privatizer.getDeallocRegion();1766                  });1767 1768  if (failed(inlineOmpRegionCleanup(1769          privateCleanupRegions, llvmPrivateVars, moduleTranslation, builder,1770          "omp.private.dealloc", /*shouldLoadCleanupRegionArg=*/false)))1771    return mlir::emitError(loc, "failed to inline `dealloc` region of an "1772                                "`omp.private` op in");1773 1774  return success();1775}1776 1777/// Returns true if the construct contains omp.cancel or omp.cancellation_point1778static bool constructIsCancellable(Operation *op) {1779  // omp.cancel and omp.cancellation_point must be "closely nested" so they will1780  // be visible and not inside of function calls. This is enforced by the1781  // verifier.1782  return op1783      ->walk([](Operation *child) {1784        if (mlir::isa<omp::CancelOp, omp::CancellationPointOp>(child))1785          return WalkResult::interrupt();1786        return WalkResult::advance();1787      })1788      .wasInterrupted();1789}1790 1791static LogicalResult1792convertOmpSections(Operation &opInst, llvm::IRBuilderBase &builder,1793                   LLVM::ModuleTranslation &moduleTranslation) {1794  using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;1795  using StorableBodyGenCallbackTy =1796      llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;1797 1798  auto sectionsOp = cast<omp::SectionsOp>(opInst);1799 1800  if (failed(checkImplementationStatus(opInst)))1801    return failure();1802 1803  llvm::ArrayRef<bool> isByRef = getIsByRef(sectionsOp.getReductionByref());1804  assert(isByRef.size() == sectionsOp.getNumReductionVars());1805 1806  SmallVector<omp::DeclareReductionOp> reductionDecls;1807  collectReductionDecls(sectionsOp, reductionDecls);1808  llvm::OpenMPIRBuilder::InsertPointTy allocaIP =1809      findAllocaInsertPoint(builder, moduleTranslation);1810 1811  SmallVector<llvm::Value *> privateReductionVariables(1812      sectionsOp.getNumReductionVars());1813  DenseMap<Value, llvm::Value *> reductionVariableMap;1814 1815  MutableArrayRef<BlockArgument> reductionArgs =1816      cast<omp::BlockArgOpenMPOpInterface>(opInst).getReductionBlockArgs();1817 1818  if (failed(allocAndInitializeReductionVars(1819          sectionsOp, reductionArgs, builder, moduleTranslation, allocaIP,1820          reductionDecls, privateReductionVariables, reductionVariableMap,1821          isByRef)))1822    return failure();1823 1824  SmallVector<StorableBodyGenCallbackTy> sectionCBs;1825 1826  for (Operation &op : *sectionsOp.getRegion().begin()) {1827    auto sectionOp = dyn_cast<omp::SectionOp>(op);1828    if (!sectionOp) // omp.terminator1829      continue;1830 1831    Region &region = sectionOp.getRegion();1832    auto sectionCB = [&sectionsOp, &region, &builder, &moduleTranslation](1833                         InsertPointTy allocaIP, InsertPointTy codeGenIP) {1834      builder.restoreIP(codeGenIP);1835 1836      // map the omp.section reduction block argument to the omp.sections block1837      // arguments1838      // TODO: this assumes that the only block arguments are reduction1839      // variables1840      assert(region.getNumArguments() ==1841             sectionsOp.getRegion().getNumArguments());1842      for (auto [sectionsArg, sectionArg] : llvm::zip_equal(1843               sectionsOp.getRegion().getArguments(), region.getArguments())) {1844        llvm::Value *llvmVal = moduleTranslation.lookupValue(sectionsArg);1845        assert(llvmVal);1846        moduleTranslation.mapValue(sectionArg, llvmVal);1847      }1848 1849      return convertOmpOpRegions(region, "omp.section.region", builder,1850                                 moduleTranslation)1851          .takeError();1852    };1853    sectionCBs.push_back(sectionCB);1854  }1855 1856  // No sections within omp.sections operation - skip generation. This situation1857  // is only possible if there is only a terminator operation inside the1858  // sections operation1859  if (sectionCBs.empty())1860    return success();1861 1862  assert(isa<omp::SectionOp>(*sectionsOp.getRegion().op_begin()));1863 1864  // TODO: Perform appropriate actions according to the data-sharing1865  // attribute (shared, private, firstprivate, ...) of variables.1866  // Currently defaults to shared.1867  auto privCB = [&](InsertPointTy, InsertPointTy codeGenIP, llvm::Value &,1868                    llvm::Value &vPtr, llvm::Value *&replacementValue)1869      -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {1870    replacementValue = &vPtr;1871    return codeGenIP;1872  };1873 1874  // TODO: Perform finalization actions for variables. This has to be1875  // called for variables which have destructors/finalizers.1876  auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); };1877 1878  allocaIP = findAllocaInsertPoint(builder, moduleTranslation);1879  bool isCancellable = constructIsCancellable(sectionsOp);1880  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);1881  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =1882      moduleTranslation.getOpenMPBuilder()->createSections(1883          ompLoc, allocaIP, sectionCBs, privCB, finiCB, isCancellable,1884          sectionsOp.getNowait());1885 1886  if (failed(handleError(afterIP, opInst)))1887    return failure();1888 1889  builder.restoreIP(*afterIP);1890 1891  // Process the reductions if required.1892  return createReductionsAndCleanup(1893      sectionsOp, builder, moduleTranslation, allocaIP, reductionDecls,1894      privateReductionVariables, isByRef, sectionsOp.getNowait());1895}1896 1897/// Converts an OpenMP single construct into LLVM IR using OpenMPIRBuilder.1898static LogicalResult1899convertOmpSingle(omp::SingleOp &singleOp, llvm::IRBuilderBase &builder,1900                 LLVM::ModuleTranslation &moduleTranslation) {1901  using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;1902  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);1903 1904  if (failed(checkImplementationStatus(*singleOp)))1905    return failure();1906 1907  auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {1908    builder.restoreIP(codegenIP);1909    return convertOmpOpRegions(singleOp.getRegion(), "omp.single.region",1910                               builder, moduleTranslation)1911        .takeError();1912  };1913  auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); };1914 1915  // Handle copyprivate1916  Operation::operand_range cpVars = singleOp.getCopyprivateVars();1917  std::optional<ArrayAttr> cpFuncs = singleOp.getCopyprivateSyms();1918  llvm::SmallVector<llvm::Value *> llvmCPVars;1919  llvm::SmallVector<llvm::Function *> llvmCPFuncs;1920  for (size_t i = 0, e = cpVars.size(); i < e; ++i) {1921    llvmCPVars.push_back(moduleTranslation.lookupValue(cpVars[i]));1922    auto llvmFuncOp = SymbolTable::lookupNearestSymbolFrom<LLVM::LLVMFuncOp>(1923        singleOp, cast<SymbolRefAttr>((*cpFuncs)[i]));1924    llvmCPFuncs.push_back(1925        moduleTranslation.lookupFunction(llvmFuncOp.getName()));1926  }1927 1928  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =1929      moduleTranslation.getOpenMPBuilder()->createSingle(1930          ompLoc, bodyCB, finiCB, singleOp.getNowait(), llvmCPVars,1931          llvmCPFuncs);1932 1933  if (failed(handleError(afterIP, *singleOp)))1934    return failure();1935 1936  builder.restoreIP(*afterIP);1937  return success();1938}1939 1940static bool teamsReductionContainedInDistribute(omp::TeamsOp teamsOp) {1941  auto iface =1942      llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(teamsOp.getOperation());1943  // Check that all uses of the reduction block arg has the same distribute op1944  // parent.1945  llvm::SmallVector<mlir::Operation *> debugUses;1946  Operation *distOp = nullptr;1947  for (auto ra : iface.getReductionBlockArgs())1948    for (auto &use : ra.getUses()) {1949      auto *useOp = use.getOwner();1950      // Ignore debug uses.1951      if (mlir::isa<LLVM::DbgDeclareOp, LLVM::DbgValueOp>(useOp)) {1952        debugUses.push_back(useOp);1953        continue;1954      }1955 1956      auto currentDistOp = useOp->getParentOfType<omp::DistributeOp>();1957      // Use is not inside a distribute op - return false1958      if (!currentDistOp)1959        return false;1960      // Multiple distribute operations - return false1961      Operation *currentOp = currentDistOp.getOperation();1962      if (distOp && (distOp != currentOp))1963        return false;1964 1965      distOp = currentOp;1966    }1967 1968  // If we are going to use distribute reduction then remove any debug uses of1969  // the reduction parameters in teamsOp. Otherwise they will be left without1970  // any mapped value in moduleTranslation and will eventually error out.1971  for (auto use : debugUses)1972    use->erase();1973  return true;1974}1975 1976// Convert an OpenMP Teams construct to LLVM IR using OpenMPIRBuilder1977static LogicalResult1978convertOmpTeams(omp::TeamsOp op, llvm::IRBuilderBase &builder,1979                LLVM::ModuleTranslation &moduleTranslation) {1980  using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;1981  if (failed(checkImplementationStatus(*op)))1982    return failure();1983 1984  DenseMap<Value, llvm::Value *> reductionVariableMap;1985  unsigned numReductionVars = op.getNumReductionVars();1986  SmallVector<omp::DeclareReductionOp> reductionDecls;1987  SmallVector<llvm::Value *> privateReductionVariables(numReductionVars);1988  llvm::ArrayRef<bool> isByRef;1989  llvm::OpenMPIRBuilder::InsertPointTy allocaIP =1990      findAllocaInsertPoint(builder, moduleTranslation);1991 1992  // Only do teams reduction if there is no distribute op that captures the1993  // reduction instead.1994  bool doTeamsReduction = !teamsReductionContainedInDistribute(op);1995  if (doTeamsReduction) {1996    isByRef = getIsByRef(op.getReductionByref());1997 1998    assert(isByRef.size() == op.getNumReductionVars());1999 2000    MutableArrayRef<BlockArgument> reductionArgs =2001        llvm::cast<omp::BlockArgOpenMPOpInterface>(*op).getReductionBlockArgs();2002 2003    collectReductionDecls(op, reductionDecls);2004 2005    if (failed(allocAndInitializeReductionVars(2006            op, reductionArgs, builder, moduleTranslation, allocaIP,2007            reductionDecls, privateReductionVariables, reductionVariableMap,2008            isByRef)))2009      return failure();2010  }2011 2012  auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {2013    LLVM::ModuleTranslation::SaveStack<OpenMPAllocaStackFrame> frame(2014        moduleTranslation, allocaIP);2015    builder.restoreIP(codegenIP);2016    return convertOmpOpRegions(op.getRegion(), "omp.teams.region", builder,2017                               moduleTranslation)2018        .takeError();2019  };2020 2021  llvm::Value *numTeamsLower = nullptr;2022  if (Value numTeamsLowerVar = op.getNumTeamsLower())2023    numTeamsLower = moduleTranslation.lookupValue(numTeamsLowerVar);2024 2025  llvm::Value *numTeamsUpper = nullptr;2026  if (Value numTeamsUpperVar = op.getNumTeamsUpper())2027    numTeamsUpper = moduleTranslation.lookupValue(numTeamsUpperVar);2028 2029  llvm::Value *threadLimit = nullptr;2030  if (Value threadLimitVar = op.getThreadLimit())2031    threadLimit = moduleTranslation.lookupValue(threadLimitVar);2032 2033  llvm::Value *ifExpr = nullptr;2034  if (Value ifVar = op.getIfExpr())2035    ifExpr = moduleTranslation.lookupValue(ifVar);2036 2037  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);2038  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =2039      moduleTranslation.getOpenMPBuilder()->createTeams(2040          ompLoc, bodyCB, numTeamsLower, numTeamsUpper, threadLimit, ifExpr);2041 2042  if (failed(handleError(afterIP, *op)))2043    return failure();2044 2045  builder.restoreIP(*afterIP);2046  if (doTeamsReduction) {2047    // Process the reductions if required.2048    return createReductionsAndCleanup(2049        op, builder, moduleTranslation, allocaIP, reductionDecls,2050        privateReductionVariables, isByRef,2051        /*isNoWait*/ false, /*isTeamsReduction*/ true);2052  }2053  return success();2054}2055 2056static void2057buildDependData(std::optional<ArrayAttr> dependKinds, OperandRange dependVars,2058                LLVM::ModuleTranslation &moduleTranslation,2059                SmallVectorImpl<llvm::OpenMPIRBuilder::DependData> &dds) {2060  if (dependVars.empty())2061    return;2062  for (auto dep : llvm::zip(dependVars, dependKinds->getValue())) {2063    llvm::omp::RTLDependenceKindTy type;2064    switch (2065        cast<mlir::omp::ClauseTaskDependAttr>(std::get<1>(dep)).getValue()) {2066    case mlir::omp::ClauseTaskDepend::taskdependin:2067      type = llvm::omp::RTLDependenceKindTy::DepIn;2068      break;2069    // The OpenMP runtime requires that the codegen for 'depend' clause for2070    // 'out' dependency kind must be the same as codegen for 'depend' clause2071    // with 'inout' dependency.2072    case mlir::omp::ClauseTaskDepend::taskdependout:2073    case mlir::omp::ClauseTaskDepend::taskdependinout:2074      type = llvm::omp::RTLDependenceKindTy::DepInOut;2075      break;2076    case mlir::omp::ClauseTaskDepend::taskdependmutexinoutset:2077      type = llvm::omp::RTLDependenceKindTy::DepMutexInOutSet;2078      break;2079    case mlir::omp::ClauseTaskDepend::taskdependinoutset:2080      type = llvm::omp::RTLDependenceKindTy::DepInOutSet;2081      break;2082    };2083    llvm::Value *depVal = moduleTranslation.lookupValue(std::get<0>(dep));2084    llvm::OpenMPIRBuilder::DependData dd(type, depVal->getType(), depVal);2085    dds.emplace_back(dd);2086  }2087}2088 2089/// Shared implementation of a callback which adds a termiator for the new block2090/// created for the branch taken when an openmp construct is cancelled. The2091/// terminator is saved in \p cancelTerminators. This callback is invoked only2092/// if there is cancellation inside of the taskgroup body.2093/// The terminator will need to be fixed to branch to the correct block to2094/// cleanup the construct.2095static void2096pushCancelFinalizationCB(SmallVectorImpl<llvm::BranchInst *> &cancelTerminators,2097                         llvm::IRBuilderBase &llvmBuilder,2098                         llvm::OpenMPIRBuilder &ompBuilder, mlir::Operation *op,2099                         llvm::omp::Directive cancelDirective) {2100  auto finiCB = [&](llvm::OpenMPIRBuilder::InsertPointTy ip) -> llvm::Error {2101    llvm::IRBuilderBase::InsertPointGuard guard(llvmBuilder);2102 2103    // ip is currently in the block branched to if cancellation occured.2104    // We need to create a branch to terminate that block.2105    llvmBuilder.restoreIP(ip);2106 2107    // We must still clean up the construct after cancelling it, so we need to2108    // branch to the block that finalizes the taskgroup.2109    // That block has not been created yet so use this block as a dummy for now2110    // and fix this after creating the operation.2111    cancelTerminators.push_back(llvmBuilder.CreateBr(ip.getBlock()));2112    return llvm::Error::success();2113  };2114  // We have to add the cleanup to the OpenMPIRBuilder before the body gets2115  // created in case the body contains omp.cancel (which will then expect to be2116  // able to find this cleanup callback).2117  ompBuilder.pushFinalizationCB(2118      {finiCB, cancelDirective, constructIsCancellable(op)});2119}2120 2121/// If we cancelled the construct, we should branch to the finalization block of2122/// that construct. OMPIRBuilder structures the CFG such that the cleanup block2123/// is immediately before the continuation block. Now this finalization has2124/// been created we can fix the branch.2125static void2126popCancelFinalizationCB(const ArrayRef<llvm::BranchInst *> cancelTerminators,2127                        llvm::OpenMPIRBuilder &ompBuilder,2128                        const llvm::OpenMPIRBuilder::InsertPointTy &afterIP) {2129  ompBuilder.popFinalizationCB();2130  llvm::BasicBlock *constructFini = afterIP.getBlock()->getSinglePredecessor();2131  for (llvm::BranchInst *cancelBranch : cancelTerminators) {2132    assert(cancelBranch->getNumSuccessors() == 1 &&2133           "cancel branch should have one target");2134    cancelBranch->setSuccessor(0, constructFini);2135  }2136}2137 2138namespace {2139/// TaskContextStructManager takes care of creating and freeing a structure2140/// containing information needed by the task body to execute.2141class TaskContextStructManager {2142public:2143  TaskContextStructManager(llvm::IRBuilderBase &builder,2144                           LLVM::ModuleTranslation &moduleTranslation,2145                           MutableArrayRef<omp::PrivateClauseOp> privateDecls)2146      : builder{builder}, moduleTranslation{moduleTranslation},2147        privateDecls{privateDecls} {}2148 2149  /// Creates a heap allocated struct containing space for each private2150  /// variable. Invariant: privateVarTypes, privateDecls, and the elements of2151  /// the structure should all have the same order (although privateDecls which2152  /// do not read from the mold argument are skipped).2153  void generateTaskContextStruct();2154 2155  /// Create GEPs to access each member of the structure representing a private2156  /// variable, adding them to llvmPrivateVars. Null values are added where2157  /// private decls were skipped so that the ordering continues to match the2158  /// private decls.2159  void createGEPsToPrivateVars();2160 2161  /// De-allocate the task context structure.2162  void freeStructPtr();2163 2164  MutableArrayRef<llvm::Value *> getLLVMPrivateVarGEPs() {2165    return llvmPrivateVarGEPs;2166  }2167 2168  llvm::Value *getStructPtr() { return structPtr; }2169 2170private:2171  llvm::IRBuilderBase &builder;2172  LLVM::ModuleTranslation &moduleTranslation;2173  MutableArrayRef<omp::PrivateClauseOp> privateDecls;2174 2175  /// The type of each member of the structure, in order.2176  SmallVector<llvm::Type *> privateVarTypes;2177 2178  /// LLVM values for each private variable, or null if that private variable is2179  /// not included in the task context structure2180  SmallVector<llvm::Value *> llvmPrivateVarGEPs;2181 2182  /// A pointer to the structure containing context for this task.2183  llvm::Value *structPtr = nullptr;2184  /// The type of the structure2185  llvm::Type *structTy = nullptr;2186};2187} // namespace2188 2189void TaskContextStructManager::generateTaskContextStruct() {2190  if (privateDecls.empty())2191    return;2192  privateVarTypes.reserve(privateDecls.size());2193 2194  for (omp::PrivateClauseOp &privOp : privateDecls) {2195    // Skip private variables which can safely be allocated and initialised2196    // inside of the task2197    if (!privOp.readsFromMold())2198      continue;2199    Type mlirType = privOp.getType();2200    privateVarTypes.push_back(moduleTranslation.convertType(mlirType));2201  }2202 2203  structTy = llvm::StructType::get(moduleTranslation.getLLVMContext(),2204                                   privateVarTypes);2205 2206  llvm::DataLayout dataLayout =2207      builder.GetInsertBlock()->getModule()->getDataLayout();2208  llvm::Type *intPtrTy = builder.getIntPtrTy(dataLayout);2209  llvm::Constant *allocSize = llvm::ConstantExpr::getSizeOf(structTy);2210 2211  // Heap allocate the structure2212  structPtr = builder.CreateMalloc(intPtrTy, structTy, allocSize,2213                                   /*ArraySize=*/nullptr, /*MallocF=*/nullptr,2214                                   "omp.task.context_ptr");2215}2216 2217void TaskContextStructManager::createGEPsToPrivateVars() {2218  if (!structPtr) {2219    assert(privateVarTypes.empty());2220    return;2221  }2222 2223  // Create GEPs for each struct member2224  llvmPrivateVarGEPs.clear();2225  llvmPrivateVarGEPs.reserve(privateDecls.size());2226  llvm::Value *zero = builder.getInt32(0);2227  unsigned i = 0;2228  for (auto privDecl : privateDecls) {2229    if (!privDecl.readsFromMold()) {2230      // Handle this inside of the task so we don't pass unnessecary vars in2231      llvmPrivateVarGEPs.push_back(nullptr);2232      continue;2233    }2234    llvm::Value *iVal = builder.getInt32(i);2235    llvm::Value *gep = builder.CreateGEP(structTy, structPtr, {zero, iVal});2236    llvmPrivateVarGEPs.push_back(gep);2237    i += 1;2238  }2239}2240 2241void TaskContextStructManager::freeStructPtr() {2242  if (!structPtr)2243    return;2244 2245  llvm::IRBuilderBase::InsertPointGuard guard{builder};2246  // Ensure we don't put the call to free() after the terminator2247  builder.SetInsertPoint(builder.GetInsertBlock()->getTerminator());2248  builder.CreateFree(structPtr);2249}2250 2251/// Converts an OpenMP task construct into LLVM IR using OpenMPIRBuilder.2252static LogicalResult2253convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder,2254                 LLVM::ModuleTranslation &moduleTranslation) {2255  using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;2256  if (failed(checkImplementationStatus(*taskOp)))2257    return failure();2258 2259  PrivateVarsInfo privateVarsInfo(taskOp);2260  TaskContextStructManager taskStructMgr{builder, moduleTranslation,2261                                         privateVarsInfo.privatizers};2262 2263  // Allocate and copy private variables before creating the task. This avoids2264  // accessing invalid memory if (after this scope ends) the private variables2265  // are initialized from host variables or if the variables are copied into2266  // from host variables (firstprivate). The insertion point is just before2267  // where the code for creating and scheduling the task will go. That puts this2268  // code outside of the outlined task region, which is what we want because2269  // this way the initialization and copy regions are executed immediately while2270  // the host variable data are still live.2271 2272  llvm::OpenMPIRBuilder::InsertPointTy allocaIP =2273      findAllocaInsertPoint(builder, moduleTranslation);2274 2275  // Not using splitBB() because that requires the current block to have a2276  // terminator.2277  assert(builder.GetInsertPoint() == builder.GetInsertBlock()->end());2278  llvm::BasicBlock *taskStartBlock = llvm::BasicBlock::Create(2279      builder.getContext(), "omp.task.start",2280      /*Parent=*/builder.GetInsertBlock()->getParent());2281  llvm::Instruction *branchToTaskStartBlock = builder.CreateBr(taskStartBlock);2282  builder.SetInsertPoint(branchToTaskStartBlock);2283 2284  // Now do this again to make the initialization and copy blocks2285  llvm::BasicBlock *copyBlock =2286      splitBB(builder, /*CreateBranch=*/true, "omp.private.copy");2287  llvm::BasicBlock *initBlock =2288      splitBB(builder, /*CreateBranch=*/true, "omp.private.init");2289 2290  // Now the control flow graph should look like2291  // starter_block:2292  //   <---- where we started when convertOmpTaskOp was called2293  //   br %omp.private.init2294  // omp.private.init:2295  //   br %omp.private.copy2296  // omp.private.copy:2297  //   br %omp.task.start2298  // omp.task.start:2299  //   <---- where we want the insertion point to be when we call createTask()2300 2301  // Save the alloca insertion point on ModuleTranslation stack for use in2302  // nested regions.2303  LLVM::ModuleTranslation::SaveStack<OpenMPAllocaStackFrame> frame(2304      moduleTranslation, allocaIP);2305 2306  // Allocate and initialize private variables2307  builder.SetInsertPoint(initBlock->getTerminator());2308 2309  // Create task variable structure2310  taskStructMgr.generateTaskContextStruct();2311  // GEPs so that we can initialize the variables. Don't use these GEPs inside2312  // of the body otherwise it will be the GEP not the struct which is fowarded2313  // to the outlined function. GEPs forwarded in this way are passed in a2314  // stack-allocated (by OpenMPIRBuilder) structure which is not safe for tasks2315  // which may not be executed until after the current stack frame goes out of2316  // scope.2317  taskStructMgr.createGEPsToPrivateVars();2318 2319  for (auto [privDecl, mlirPrivVar, blockArg, llvmPrivateVarAlloc] :2320       llvm::zip_equal(privateVarsInfo.privatizers, privateVarsInfo.mlirVars,2321                       privateVarsInfo.blockArgs,2322                       taskStructMgr.getLLVMPrivateVarGEPs())) {2323    // To be handled inside the task.2324    if (!privDecl.readsFromMold())2325      continue;2326    assert(llvmPrivateVarAlloc &&2327           "reads from mold so shouldn't have been skipped");2328 2329    llvm::Expected<llvm::Value *> privateVarOrErr =2330        initPrivateVar(builder, moduleTranslation, privDecl, mlirPrivVar,2331                       blockArg, llvmPrivateVarAlloc, initBlock);2332    if (!privateVarOrErr)2333      return handleError(privateVarOrErr, *taskOp.getOperation());2334 2335    setInsertPointForPossiblyEmptyBlock(builder);2336 2337    // TODO: this is a bit of a hack for Fortran character boxes.2338    // Character boxes are passed by value into the init region and then the2339    // initialized character box is yielded by value. Here we need to store the2340    // yielded value into the private allocation, and load the private2341    // allocation to match the type expected by region block arguments.2342    if ((privateVarOrErr.get() != llvmPrivateVarAlloc) &&2343        !mlir::isa<LLVM::LLVMPointerType>(blockArg.getType())) {2344      builder.CreateStore(privateVarOrErr.get(), llvmPrivateVarAlloc);2345      // Load it so we have the value pointed to by the GEP2346      llvmPrivateVarAlloc = builder.CreateLoad(privateVarOrErr.get()->getType(),2347                                               llvmPrivateVarAlloc);2348    }2349    assert(llvmPrivateVarAlloc->getType() ==2350           moduleTranslation.convertType(blockArg.getType()));2351 2352    // Mapping blockArg -> llvmPrivateVarAlloc is done inside the body callback2353    // so that OpenMPIRBuilder doesn't try to pass each GEP address through a2354    // stack allocated structure.2355  }2356 2357  // firstprivate copy region2358  setInsertPointForPossiblyEmptyBlock(builder, copyBlock);2359  if (failed(copyFirstPrivateVars(2360          taskOp, builder, moduleTranslation, privateVarsInfo.mlirVars,2361          taskStructMgr.getLLVMPrivateVarGEPs(), privateVarsInfo.privatizers,2362          taskOp.getPrivateNeedsBarrier())))2363    return llvm::failure();2364 2365  // Set up for call to createTask()2366  builder.SetInsertPoint(taskStartBlock);2367 2368  auto bodyCB = [&](InsertPointTy allocaIP,2369                    InsertPointTy codegenIP) -> llvm::Error {2370    // Save the alloca insertion point on ModuleTranslation stack for use in2371    // nested regions.2372    LLVM::ModuleTranslation::SaveStack<OpenMPAllocaStackFrame> frame(2373        moduleTranslation, allocaIP);2374 2375    // translate the body of the task:2376    builder.restoreIP(codegenIP);2377 2378    llvm::BasicBlock *privInitBlock = nullptr;2379    privateVarsInfo.llvmVars.resize(privateVarsInfo.blockArgs.size());2380    for (auto [i, zip] : llvm::enumerate(llvm::zip_equal(2381             privateVarsInfo.blockArgs, privateVarsInfo.privatizers,2382             privateVarsInfo.mlirVars))) {2383      auto [blockArg, privDecl, mlirPrivVar] = zip;2384      // This is handled before the task executes2385      if (privDecl.readsFromMold())2386        continue;2387 2388      llvm::IRBuilderBase::InsertPointGuard guard(builder);2389      llvm::Type *llvmAllocType =2390          moduleTranslation.convertType(privDecl.getType());2391      builder.SetInsertPoint(allocaIP.getBlock()->getTerminator());2392      llvm::Value *llvmPrivateVar = builder.CreateAlloca(2393          llvmAllocType, /*ArraySize=*/nullptr, "omp.private.alloc");2394 2395      llvm::Expected<llvm::Value *> privateVarOrError =2396          initPrivateVar(builder, moduleTranslation, privDecl, mlirPrivVar,2397                         blockArg, llvmPrivateVar, privInitBlock);2398      if (!privateVarOrError)2399        return privateVarOrError.takeError();2400      moduleTranslation.mapValue(blockArg, privateVarOrError.get());2401      privateVarsInfo.llvmVars[i] = privateVarOrError.get();2402    }2403 2404    taskStructMgr.createGEPsToPrivateVars();2405    for (auto [i, llvmPrivVar] :2406         llvm::enumerate(taskStructMgr.getLLVMPrivateVarGEPs())) {2407      if (!llvmPrivVar) {2408        assert(privateVarsInfo.llvmVars[i] &&2409               "This is added in the loop above");2410        continue;2411      }2412      privateVarsInfo.llvmVars[i] = llvmPrivVar;2413    }2414 2415    // Find and map the addresses of each variable within the task context2416    // structure2417    for (auto [blockArg, llvmPrivateVar, privateDecl] :2418         llvm::zip_equal(privateVarsInfo.blockArgs, privateVarsInfo.llvmVars,2419                         privateVarsInfo.privatizers)) {2420      // This was handled above.2421      if (!privateDecl.readsFromMold())2422        continue;2423      // Fix broken pass-by-value case for Fortran character boxes2424      if (!mlir::isa<LLVM::LLVMPointerType>(blockArg.getType())) {2425        llvmPrivateVar = builder.CreateLoad(2426            moduleTranslation.convertType(blockArg.getType()), llvmPrivateVar);2427      }2428      assert(llvmPrivateVar->getType() ==2429             moduleTranslation.convertType(blockArg.getType()));2430      moduleTranslation.mapValue(blockArg, llvmPrivateVar);2431    }2432 2433    auto continuationBlockOrError = convertOmpOpRegions(2434        taskOp.getRegion(), "omp.task.region", builder, moduleTranslation);2435    if (failed(handleError(continuationBlockOrError, *taskOp)))2436      return llvm::make_error<PreviouslyReportedError>();2437 2438    builder.SetInsertPoint(continuationBlockOrError.get()->getTerminator());2439 2440    if (failed(cleanupPrivateVars(builder, moduleTranslation, taskOp.getLoc(),2441                                  privateVarsInfo.llvmVars,2442                                  privateVarsInfo.privatizers)))2443      return llvm::make_error<PreviouslyReportedError>();2444 2445    // Free heap allocated task context structure at the end of the task.2446    taskStructMgr.freeStructPtr();2447 2448    return llvm::Error::success();2449  };2450 2451  llvm::OpenMPIRBuilder &ompBuilder = *moduleTranslation.getOpenMPBuilder();2452  SmallVector<llvm::BranchInst *> cancelTerminators;2453  // The directive to match here is OMPD_taskgroup because it is the taskgroup2454  // which is canceled. This is handled here because it is the task's cleanup2455  // block which should be branched to.2456  pushCancelFinalizationCB(cancelTerminators, builder, ompBuilder, taskOp,2457                           llvm::omp::Directive::OMPD_taskgroup);2458 2459  SmallVector<llvm::OpenMPIRBuilder::DependData> dds;2460  buildDependData(taskOp.getDependKinds(), taskOp.getDependVars(),2461                  moduleTranslation, dds);2462 2463  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);2464  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =2465      moduleTranslation.getOpenMPBuilder()->createTask(2466          ompLoc, allocaIP, bodyCB, !taskOp.getUntied(),2467          moduleTranslation.lookupValue(taskOp.getFinal()),2468          moduleTranslation.lookupValue(taskOp.getIfExpr()), dds,2469          taskOp.getMergeable(),2470          moduleTranslation.lookupValue(taskOp.getEventHandle()),2471          moduleTranslation.lookupValue(taskOp.getPriority()));2472 2473  if (failed(handleError(afterIP, *taskOp)))2474    return failure();2475 2476  // Set the correct branch target for task cancellation2477  popCancelFinalizationCB(cancelTerminators, ompBuilder, afterIP.get());2478 2479  builder.restoreIP(*afterIP);2480  return success();2481}2482 2483/// Converts an OpenMP taskgroup construct into LLVM IR using OpenMPIRBuilder.2484static LogicalResult2485convertOmpTaskgroupOp(omp::TaskgroupOp tgOp, llvm::IRBuilderBase &builder,2486                      LLVM::ModuleTranslation &moduleTranslation) {2487  using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;2488  if (failed(checkImplementationStatus(*tgOp)))2489    return failure();2490 2491  auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {2492    builder.restoreIP(codegenIP);2493    return convertOmpOpRegions(tgOp.getRegion(), "omp.taskgroup.region",2494                               builder, moduleTranslation)2495        .takeError();2496  };2497 2498  InsertPointTy allocaIP = findAllocaInsertPoint(builder, moduleTranslation);2499  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);2500  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =2501      moduleTranslation.getOpenMPBuilder()->createTaskgroup(ompLoc, allocaIP,2502                                                            bodyCB);2503 2504  if (failed(handleError(afterIP, *tgOp)))2505    return failure();2506 2507  builder.restoreIP(*afterIP);2508  return success();2509}2510 2511static LogicalResult2512convertOmpTaskwaitOp(omp::TaskwaitOp twOp, llvm::IRBuilderBase &builder,2513                     LLVM::ModuleTranslation &moduleTranslation) {2514  if (failed(checkImplementationStatus(*twOp)))2515    return failure();2516 2517  moduleTranslation.getOpenMPBuilder()->createTaskwait(builder.saveIP());2518  return success();2519}2520 2521/// Converts an OpenMP workshare loop into LLVM IR using OpenMPIRBuilder.2522static LogicalResult2523convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,2524                 LLVM::ModuleTranslation &moduleTranslation) {2525  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();2526  auto wsloopOp = cast<omp::WsloopOp>(opInst);2527  if (failed(checkImplementationStatus(opInst)))2528    return failure();2529 2530  auto loopOp = cast<omp::LoopNestOp>(wsloopOp.getWrappedLoop());2531  llvm::ArrayRef<bool> isByRef = getIsByRef(wsloopOp.getReductionByref());2532  assert(isByRef.size() == wsloopOp.getNumReductionVars());2533 2534  // Static is the default.2535  auto schedule =2536      wsloopOp.getScheduleKind().value_or(omp::ClauseScheduleKind::Static);2537 2538  // Find the loop configuration.2539  llvm::Value *step = moduleTranslation.lookupValue(loopOp.getLoopSteps()[0]);2540  llvm::Type *ivType = step->getType();2541  llvm::Value *chunk = nullptr;2542  if (wsloopOp.getScheduleChunk()) {2543    llvm::Value *chunkVar =2544        moduleTranslation.lookupValue(wsloopOp.getScheduleChunk());2545    chunk = builder.CreateSExtOrTrunc(chunkVar, ivType);2546  }2547 2548  omp::DistributeOp distributeOp = nullptr;2549  llvm::Value *distScheduleChunk = nullptr;2550  bool hasDistSchedule = false;2551  if (llvm::isa_and_present<omp::DistributeOp>(opInst.getParentOp())) {2552    distributeOp = cast<omp::DistributeOp>(opInst.getParentOp());2553    hasDistSchedule = distributeOp.getDistScheduleStatic();2554    if (distributeOp.getDistScheduleChunkSize()) {2555      llvm::Value *chunkVar = moduleTranslation.lookupValue(2556          distributeOp.getDistScheduleChunkSize());2557      distScheduleChunk = builder.CreateSExtOrTrunc(chunkVar, ivType);2558    }2559  }2560 2561  PrivateVarsInfo privateVarsInfo(wsloopOp);2562 2563  SmallVector<omp::DeclareReductionOp> reductionDecls;2564  collectReductionDecls(wsloopOp, reductionDecls);2565  llvm::OpenMPIRBuilder::InsertPointTy allocaIP =2566      findAllocaInsertPoint(builder, moduleTranslation);2567 2568  SmallVector<llvm::Value *> privateReductionVariables(2569      wsloopOp.getNumReductionVars());2570 2571  llvm::Expected<llvm::BasicBlock *> afterAllocas = allocatePrivateVars(2572      builder, moduleTranslation, privateVarsInfo, allocaIP);2573  if (handleError(afterAllocas, opInst).failed())2574    return failure();2575 2576  DenseMap<Value, llvm::Value *> reductionVariableMap;2577 2578  MutableArrayRef<BlockArgument> reductionArgs =2579      cast<omp::BlockArgOpenMPOpInterface>(opInst).getReductionBlockArgs();2580 2581  SmallVector<DeferredStore> deferredStores;2582 2583  if (failed(allocReductionVars(wsloopOp, reductionArgs, builder,2584                                moduleTranslation, allocaIP, reductionDecls,2585                                privateReductionVariables, reductionVariableMap,2586                                deferredStores, isByRef)))2587    return failure();2588 2589  if (handleError(initPrivateVars(builder, moduleTranslation, privateVarsInfo),2590                  opInst)2591          .failed())2592    return failure();2593 2594  if (failed(copyFirstPrivateVars(2595          wsloopOp, builder, moduleTranslation, privateVarsInfo.mlirVars,2596          privateVarsInfo.llvmVars, privateVarsInfo.privatizers,2597          wsloopOp.getPrivateNeedsBarrier())))2598    return failure();2599 2600  assert(afterAllocas.get()->getSinglePredecessor());2601  if (failed(initReductionVars(wsloopOp, reductionArgs, builder,2602                               moduleTranslation,2603                               afterAllocas.get()->getSinglePredecessor(),2604                               reductionDecls, privateReductionVariables,2605                               reductionVariableMap, isByRef, deferredStores)))2606    return failure();2607 2608  // TODO: Handle doacross loops when the ordered clause has a parameter.2609  bool isOrdered = wsloopOp.getOrdered().has_value();2610  std::optional<omp::ScheduleModifier> scheduleMod = wsloopOp.getScheduleMod();2611  bool isSimd = wsloopOp.getScheduleSimd();2612  bool loopNeedsBarrier = !wsloopOp.getNowait();2613 2614  // The only legal way for the direct parent to be omp.distribute is that this2615  // represents 'distribute parallel do'. Otherwise, this is a regular2616  // worksharing loop.2617  llvm::omp::WorksharingLoopType workshareLoopType =2618      llvm::isa_and_present<omp::DistributeOp>(opInst.getParentOp())2619          ? llvm::omp::WorksharingLoopType::DistributeForStaticLoop2620          : llvm::omp::WorksharingLoopType::ForStaticLoop;2621 2622  SmallVector<llvm::BranchInst *> cancelTerminators;2623  pushCancelFinalizationCB(cancelTerminators, builder, *ompBuilder, wsloopOp,2624                           llvm::omp::Directive::OMPD_for);2625 2626  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);2627 2628  // Initialize linear variables and linear step2629  LinearClauseProcessor linearClauseProcessor;2630  if (!wsloopOp.getLinearVars().empty()) {2631    for (mlir::Value linearVar : wsloopOp.getLinearVars())2632      linearClauseProcessor.createLinearVar(builder, moduleTranslation,2633                                            linearVar);2634    for (mlir::Value linearStep : wsloopOp.getLinearStepVars())2635      linearClauseProcessor.initLinearStep(moduleTranslation, linearStep);2636  }2637 2638  llvm::Expected<llvm::BasicBlock *> regionBlock = convertOmpOpRegions(2639      wsloopOp.getRegion(), "omp.wsloop.region", builder, moduleTranslation);2640 2641  if (failed(handleError(regionBlock, opInst)))2642    return failure();2643 2644  llvm::CanonicalLoopInfo *loopInfo = findCurrentLoopInfo(moduleTranslation);2645 2646  // Emit Initialization and Update IR for linear variables2647  if (!wsloopOp.getLinearVars().empty()) {2648    llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =2649        linearClauseProcessor.initLinearVar(builder, moduleTranslation,2650                                            loopInfo->getPreheader());2651    if (failed(handleError(afterBarrierIP, *loopOp)))2652      return failure();2653    builder.restoreIP(*afterBarrierIP);2654    linearClauseProcessor.updateLinearVar(builder, loopInfo->getBody(),2655                                          loopInfo->getIndVar());2656    linearClauseProcessor.outlineLinearFinalizationBB(builder,2657                                                      loopInfo->getExit());2658  }2659 2660  builder.SetInsertPoint(*regionBlock, (*regionBlock)->begin());2661 2662  // Check if we can generate no-loop kernel2663  bool noLoopMode = false;2664  omp::TargetOp targetOp = wsloopOp->getParentOfType<mlir::omp::TargetOp>();2665  if (targetOp) {2666    Operation *targetCapturedOp = targetOp.getInnermostCapturedOmpOp();2667    // We need this check because, without it, noLoopMode would be set to true2668    // for every omp.wsloop nested inside a no-loop SPMD target region, even if2669    // that loop is not the top-level SPMD one.2670    if (loopOp == targetCapturedOp) {2671      omp::TargetRegionFlags kernelFlags =2672          targetOp.getKernelExecFlags(targetCapturedOp);2673      if (omp::bitEnumContainsAll(kernelFlags,2674                                  omp::TargetRegionFlags::spmd |2675                                      omp::TargetRegionFlags::no_loop) &&2676          !omp::bitEnumContainsAny(kernelFlags,2677                                   omp::TargetRegionFlags::generic))2678        noLoopMode = true;2679    }2680  }2681 2682  llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =2683      ompBuilder->applyWorkshareLoop(2684          ompLoc.DL, loopInfo, allocaIP, loopNeedsBarrier,2685          convertToScheduleKind(schedule), chunk, isSimd,2686          scheduleMod == omp::ScheduleModifier::monotonic,2687          scheduleMod == omp::ScheduleModifier::nonmonotonic, isOrdered,2688          workshareLoopType, noLoopMode, hasDistSchedule, distScheduleChunk);2689 2690  if (failed(handleError(wsloopIP, opInst)))2691    return failure();2692 2693  // Emit finalization and in-place rewrites for linear vars.2694  if (!wsloopOp.getLinearVars().empty()) {2695    llvm::OpenMPIRBuilder::InsertPointTy oldIP = builder.saveIP();2696    assert(loopInfo->getLastIter() &&2697           "`lastiter` in CanonicalLoopInfo is nullptr");2698    llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =2699        linearClauseProcessor.finalizeLinearVar(builder, moduleTranslation,2700                                                loopInfo->getLastIter());2701    if (failed(handleError(afterBarrierIP, *loopOp)))2702      return failure();2703    for (size_t index = 0; index < wsloopOp.getLinearVars().size(); index++)2704      linearClauseProcessor.rewriteInPlace(builder, "omp.loop_nest.region",2705                                           index);2706    builder.restoreIP(oldIP);2707  }2708 2709  // Set the correct branch target for task cancellation2710  popCancelFinalizationCB(cancelTerminators, *ompBuilder, wsloopIP.get());2711 2712  // Process the reductions if required.2713  if (failed(createReductionsAndCleanup(2714          wsloopOp, builder, moduleTranslation, allocaIP, reductionDecls,2715          privateReductionVariables, isByRef, wsloopOp.getNowait(),2716          /*isTeamsReduction=*/false)))2717    return failure();2718 2719  return cleanupPrivateVars(builder, moduleTranslation, wsloopOp.getLoc(),2720                            privateVarsInfo.llvmVars,2721                            privateVarsInfo.privatizers);2722}2723 2724/// Converts the OpenMP parallel operation to LLVM IR.2725static LogicalResult2726convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,2727                   LLVM::ModuleTranslation &moduleTranslation) {2728  using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;2729  ArrayRef<bool> isByRef = getIsByRef(opInst.getReductionByref());2730  assert(isByRef.size() == opInst.getNumReductionVars());2731  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();2732  bool isCancellable = constructIsCancellable(opInst);2733 2734  if (failed(checkImplementationStatus(*opInst)))2735    return failure();2736 2737  PrivateVarsInfo privateVarsInfo(opInst);2738 2739  // Collect reduction declarations2740  SmallVector<omp::DeclareReductionOp> reductionDecls;2741  collectReductionDecls(opInst, reductionDecls);2742  SmallVector<llvm::Value *> privateReductionVariables(2743      opInst.getNumReductionVars());2744  SmallVector<DeferredStore> deferredStores;2745 2746  auto bodyGenCB = [&](InsertPointTy allocaIP,2747                       InsertPointTy codeGenIP) -> llvm::Error {2748    llvm::Expected<llvm::BasicBlock *> afterAllocas = allocatePrivateVars(2749        builder, moduleTranslation, privateVarsInfo, allocaIP);2750    if (handleError(afterAllocas, *opInst).failed())2751      return llvm::make_error<PreviouslyReportedError>();2752 2753    // Allocate reduction vars2754    DenseMap<Value, llvm::Value *> reductionVariableMap;2755 2756    MutableArrayRef<BlockArgument> reductionArgs =2757        cast<omp::BlockArgOpenMPOpInterface>(*opInst).getReductionBlockArgs();2758 2759    allocaIP =2760        InsertPointTy(allocaIP.getBlock(),2761                      allocaIP.getBlock()->getTerminator()->getIterator());2762 2763    if (failed(allocReductionVars(2764            opInst, reductionArgs, builder, moduleTranslation, allocaIP,2765            reductionDecls, privateReductionVariables, reductionVariableMap,2766            deferredStores, isByRef)))2767      return llvm::make_error<PreviouslyReportedError>();2768 2769    assert(afterAllocas.get()->getSinglePredecessor());2770    builder.restoreIP(codeGenIP);2771 2772    if (handleError(2773            initPrivateVars(builder, moduleTranslation, privateVarsInfo),2774            *opInst)2775            .failed())2776      return llvm::make_error<PreviouslyReportedError>();2777 2778    if (failed(copyFirstPrivateVars(2779            opInst, builder, moduleTranslation, privateVarsInfo.mlirVars,2780            privateVarsInfo.llvmVars, privateVarsInfo.privatizers,2781            opInst.getPrivateNeedsBarrier())))2782      return llvm::make_error<PreviouslyReportedError>();2783 2784    if (failed(2785            initReductionVars(opInst, reductionArgs, builder, moduleTranslation,2786                              afterAllocas.get()->getSinglePredecessor(),2787                              reductionDecls, privateReductionVariables,2788                              reductionVariableMap, isByRef, deferredStores)))2789      return llvm::make_error<PreviouslyReportedError>();2790 2791    // Save the alloca insertion point on ModuleTranslation stack for use in2792    // nested regions.2793    LLVM::ModuleTranslation::SaveStack<OpenMPAllocaStackFrame> frame(2794        moduleTranslation, allocaIP);2795 2796    // ParallelOp has only one region associated with it.2797    llvm::Expected<llvm::BasicBlock *> regionBlock = convertOmpOpRegions(2798        opInst.getRegion(), "omp.par.region", builder, moduleTranslation);2799    if (!regionBlock)2800      return regionBlock.takeError();2801 2802    // Process the reductions if required.2803    if (opInst.getNumReductionVars() > 0) {2804      // Collect reduction info2805      SmallVector<OwningReductionGen> owningReductionGens;2806      SmallVector<OwningAtomicReductionGen> owningAtomicReductionGens;2807      SmallVector<OwningDataPtrPtrReductionGen>2808          owningReductionGenRefDataPtrGens;2809      SmallVector<llvm::OpenMPIRBuilder::ReductionInfo, 2> reductionInfos;2810      collectReductionInfo(opInst, builder, moduleTranslation, reductionDecls,2811                           owningReductionGens, owningAtomicReductionGens,2812                           owningReductionGenRefDataPtrGens,2813                           privateReductionVariables, reductionInfos, isByRef);2814 2815      // Move to region cont block2816      builder.SetInsertPoint((*regionBlock)->getTerminator());2817 2818      // Generate reductions from info2819      llvm::UnreachableInst *tempTerminator = builder.CreateUnreachable();2820      builder.SetInsertPoint(tempTerminator);2821 2822      llvm::OpenMPIRBuilder::InsertPointOrErrorTy contInsertPoint =2823          ompBuilder->createReductions(2824              builder.saveIP(), allocaIP, reductionInfos, isByRef,2825              /*IsNoWait=*/false, /*IsTeamsReduction=*/false);2826      if (!contInsertPoint)2827        return contInsertPoint.takeError();2828 2829      if (!contInsertPoint->getBlock())2830        return llvm::make_error<PreviouslyReportedError>();2831 2832      tempTerminator->eraseFromParent();2833      builder.restoreIP(*contInsertPoint);2834    }2835 2836    return llvm::Error::success();2837  };2838 2839  auto privCB = [](InsertPointTy allocaIP, InsertPointTy codeGenIP,2840                   llvm::Value &, llvm::Value &val, llvm::Value *&replVal) {2841    // tell OpenMPIRBuilder not to do anything. We handled Privatisation in2842    // bodyGenCB.2843    replVal = &val;2844    return codeGenIP;2845  };2846 2847  // TODO: Perform finalization actions for variables. This has to be2848  // called for variables which have destructors/finalizers.2849  auto finiCB = [&](InsertPointTy codeGenIP) -> llvm::Error {2850    InsertPointTy oldIP = builder.saveIP();2851    builder.restoreIP(codeGenIP);2852 2853    // if the reduction has a cleanup region, inline it here to finalize the2854    // reduction variables2855    SmallVector<Region *> reductionCleanupRegions;2856    llvm::transform(reductionDecls, std::back_inserter(reductionCleanupRegions),2857                    [](omp::DeclareReductionOp reductionDecl) {2858                      return &reductionDecl.getCleanupRegion();2859                    });2860    if (failed(inlineOmpRegionCleanup(2861            reductionCleanupRegions, privateReductionVariables,2862            moduleTranslation, builder, "omp.reduction.cleanup")))2863      return llvm::createStringError(2864          "failed to inline `cleanup` region of `omp.declare_reduction`");2865 2866    if (failed(cleanupPrivateVars(builder, moduleTranslation, opInst.getLoc(),2867                                  privateVarsInfo.llvmVars,2868                                  privateVarsInfo.privatizers)))2869      return llvm::make_error<PreviouslyReportedError>();2870 2871    // If we could be performing cancellation, add the cancellation barrier on2872    // the way out of the outlined region.2873    if (isCancellable) {2874      auto IPOrErr = ompBuilder->createBarrier(2875          llvm::OpenMPIRBuilder::LocationDescription(builder),2876          llvm::omp::Directive::OMPD_unknown,2877          /* ForceSimpleCall */ false,2878          /* CheckCancelFlag */ false);2879      if (!IPOrErr)2880        return IPOrErr.takeError();2881    }2882 2883    builder.restoreIP(oldIP);2884    return llvm::Error::success();2885  };2886 2887  llvm::Value *ifCond = nullptr;2888  if (auto ifVar = opInst.getIfExpr())2889    ifCond = moduleTranslation.lookupValue(ifVar);2890  llvm::Value *numThreads = nullptr;2891  if (auto numThreadsVar = opInst.getNumThreads())2892    numThreads = moduleTranslation.lookupValue(numThreadsVar);2893  auto pbKind = llvm::omp::OMP_PROC_BIND_default;2894  if (auto bind = opInst.getProcBindKind())2895    pbKind = getProcBindKind(*bind);2896 2897  llvm::OpenMPIRBuilder::InsertPointTy allocaIP =2898      findAllocaInsertPoint(builder, moduleTranslation);2899  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);2900 2901  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =2902      ompBuilder->createParallel(ompLoc, allocaIP, bodyGenCB, privCB, finiCB,2903                                 ifCond, numThreads, pbKind, isCancellable);2904 2905  if (failed(handleError(afterIP, *opInst)))2906    return failure();2907 2908  builder.restoreIP(*afterIP);2909  return success();2910}2911 2912/// Convert Order attribute to llvm::omp::OrderKind.2913static llvm::omp::OrderKind2914convertOrderKind(std::optional<omp::ClauseOrderKind> o) {2915  if (!o)2916    return llvm::omp::OrderKind::OMP_ORDER_unknown;2917  switch (*o) {2918  case omp::ClauseOrderKind::Concurrent:2919    return llvm::omp::OrderKind::OMP_ORDER_concurrent;2920  }2921  llvm_unreachable("Unknown ClauseOrderKind kind");2922}2923 2924/// Converts an OpenMP simd loop into LLVM IR using OpenMPIRBuilder.2925static LogicalResult2926convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,2927               LLVM::ModuleTranslation &moduleTranslation) {2928  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();2929  auto simdOp = cast<omp::SimdOp>(opInst);2930 2931  if (failed(checkImplementationStatus(opInst)))2932    return failure();2933 2934  PrivateVarsInfo privateVarsInfo(simdOp);2935 2936  MutableArrayRef<BlockArgument> reductionArgs =2937      cast<omp::BlockArgOpenMPOpInterface>(opInst).getReductionBlockArgs();2938  DenseMap<Value, llvm::Value *> reductionVariableMap;2939  SmallVector<llvm::Value *> privateReductionVariables(2940      simdOp.getNumReductionVars());2941  SmallVector<DeferredStore> deferredStores;2942  SmallVector<omp::DeclareReductionOp> reductionDecls;2943  collectReductionDecls(simdOp, reductionDecls);2944  llvm::ArrayRef<bool> isByRef = getIsByRef(simdOp.getReductionByref());2945  assert(isByRef.size() == simdOp.getNumReductionVars());2946 2947  llvm::OpenMPIRBuilder::InsertPointTy allocaIP =2948      findAllocaInsertPoint(builder, moduleTranslation);2949 2950  llvm::Expected<llvm::BasicBlock *> afterAllocas = allocatePrivateVars(2951      builder, moduleTranslation, privateVarsInfo, allocaIP);2952  if (handleError(afterAllocas, opInst).failed())2953    return failure();2954 2955  if (failed(allocReductionVars(simdOp, reductionArgs, builder,2956                                moduleTranslation, allocaIP, reductionDecls,2957                                privateReductionVariables, reductionVariableMap,2958                                deferredStores, isByRef)))2959    return failure();2960 2961  if (handleError(initPrivateVars(builder, moduleTranslation, privateVarsInfo),2962                  opInst)2963          .failed())2964    return failure();2965 2966  // No call to copyFirstPrivateVars because FIRSTPRIVATE is not allowed for2967  // SIMD.2968 2969  assert(afterAllocas.get()->getSinglePredecessor());2970  if (failed(initReductionVars(simdOp, reductionArgs, builder,2971                               moduleTranslation,2972                               afterAllocas.get()->getSinglePredecessor(),2973                               reductionDecls, privateReductionVariables,2974                               reductionVariableMap, isByRef, deferredStores)))2975    return failure();2976 2977  llvm::ConstantInt *simdlen = nullptr;2978  if (std::optional<uint64_t> simdlenVar = simdOp.getSimdlen())2979    simdlen = builder.getInt64(simdlenVar.value());2980 2981  llvm::ConstantInt *safelen = nullptr;2982  if (std::optional<uint64_t> safelenVar = simdOp.getSafelen())2983    safelen = builder.getInt64(safelenVar.value());2984 2985  llvm::MapVector<llvm::Value *, llvm::Value *> alignedVars;2986  llvm::omp::OrderKind order = convertOrderKind(simdOp.getOrder());2987 2988  llvm::BasicBlock *sourceBlock = builder.GetInsertBlock();2989  std::optional<ArrayAttr> alignmentValues = simdOp.getAlignments();2990  mlir::OperandRange operands = simdOp.getAlignedVars();2991  for (size_t i = 0; i < operands.size(); ++i) {2992    llvm::Value *alignment = nullptr;2993    llvm::Value *llvmVal = moduleTranslation.lookupValue(operands[i]);2994    llvm::Type *ty = llvmVal->getType();2995 2996    auto intAttr = cast<IntegerAttr>((*alignmentValues)[i]);2997    alignment = builder.getInt64(intAttr.getInt());2998    assert(ty->isPointerTy() && "Invalid type for aligned variable");2999    assert(alignment && "Invalid alignment value");3000 3001    // Check if the alignment value is not a power of 2. If so, skip emitting3002    // alignment.3003    if (!intAttr.getValue().isPowerOf2())3004      continue;3005 3006    auto curInsert = builder.saveIP();3007    builder.SetInsertPoint(sourceBlock);3008    llvmVal = builder.CreateLoad(ty, llvmVal);3009    builder.restoreIP(curInsert);3010    alignedVars[llvmVal] = alignment;3011  }3012 3013  llvm::Expected<llvm::BasicBlock *> regionBlock = convertOmpOpRegions(3014      simdOp.getRegion(), "omp.simd.region", builder, moduleTranslation);3015 3016  if (failed(handleError(regionBlock, opInst)))3017    return failure();3018 3019  builder.SetInsertPoint(*regionBlock, (*regionBlock)->begin());3020  llvm::CanonicalLoopInfo *loopInfo = findCurrentLoopInfo(moduleTranslation);3021  ompBuilder->applySimd(loopInfo, alignedVars,3022                        simdOp.getIfExpr()3023                            ? moduleTranslation.lookupValue(simdOp.getIfExpr())3024                            : nullptr,3025                        order, simdlen, safelen);3026 3027  // We now need to reduce the per-simd-lane reduction variable into the3028  // original variable. This works a bit differently to other reductions (e.g.3029  // wsloop) because we don't need to call into the OpenMP runtime to handle3030  // threads: everything happened in this one thread.3031  for (auto [i, tuple] : llvm::enumerate(3032           llvm::zip(reductionDecls, isByRef, simdOp.getReductionVars(),3033                     privateReductionVariables))) {3034    auto [decl, byRef, reductionVar, privateReductionVar] = tuple;3035 3036    OwningReductionGen gen = makeReductionGen(decl, builder, moduleTranslation);3037    llvm::Value *originalVariable = moduleTranslation.lookupValue(reductionVar);3038    llvm::Type *reductionType = moduleTranslation.convertType(decl.getType());3039 3040    // We have one less load for by-ref case because that load is now inside of3041    // the reduction region.3042    llvm::Value *redValue = originalVariable;3043    if (!byRef)3044      redValue =3045          builder.CreateLoad(reductionType, redValue, "red.value." + Twine(i));3046    llvm::Value *privateRedValue = builder.CreateLoad(3047        reductionType, privateReductionVar, "red.private.value." + Twine(i));3048    llvm::Value *reduced;3049 3050    auto res = gen(builder.saveIP(), redValue, privateRedValue, reduced);3051    if (failed(handleError(res, opInst)))3052      return failure();3053    builder.restoreIP(res.get());3054 3055    // For by-ref case, the store is inside of the reduction region.3056    if (!byRef)3057      builder.CreateStore(reduced, originalVariable);3058  }3059 3060  // After the construct, deallocate private reduction variables.3061  SmallVector<Region *> reductionRegions;3062  llvm::transform(reductionDecls, std::back_inserter(reductionRegions),3063                  [](omp::DeclareReductionOp reductionDecl) {3064                    return &reductionDecl.getCleanupRegion();3065                  });3066  if (failed(inlineOmpRegionCleanup(reductionRegions, privateReductionVariables,3067                                    moduleTranslation, builder,3068                                    "omp.reduction.cleanup")))3069    return failure();3070 3071  return cleanupPrivateVars(builder, moduleTranslation, simdOp.getLoc(),3072                            privateVarsInfo.llvmVars,3073                            privateVarsInfo.privatizers);3074}3075 3076/// Converts an OpenMP loop nest into LLVM IR using OpenMPIRBuilder.3077static LogicalResult3078convertOmpLoopNest(Operation &opInst, llvm::IRBuilderBase &builder,3079                   LLVM::ModuleTranslation &moduleTranslation) {3080  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();3081  auto loopOp = cast<omp::LoopNestOp>(opInst);3082 3083  // Set up the source location value for OpenMP runtime.3084  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);3085 3086  // Generator of the canonical loop body.3087  SmallVector<llvm::CanonicalLoopInfo *> loopInfos;3088  SmallVector<llvm::OpenMPIRBuilder::InsertPointTy> bodyInsertPoints;3089  auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip,3090                     llvm::Value *iv) -> llvm::Error {3091    // Make sure further conversions know about the induction variable.3092    moduleTranslation.mapValue(3093        loopOp.getRegion().front().getArgument(loopInfos.size()), iv);3094 3095    // Capture the body insertion point for use in nested loops. BodyIP of the3096    // CanonicalLoopInfo always points to the beginning of the entry block of3097    // the body.3098    bodyInsertPoints.push_back(ip);3099 3100    if (loopInfos.size() != loopOp.getNumLoops() - 1)3101      return llvm::Error::success();3102 3103    // Convert the body of the loop.3104    builder.restoreIP(ip);3105    llvm::Expected<llvm::BasicBlock *> regionBlock = convertOmpOpRegions(3106        loopOp.getRegion(), "omp.loop_nest.region", builder, moduleTranslation);3107    if (!regionBlock)3108      return regionBlock.takeError();3109 3110    builder.SetInsertPoint(*regionBlock, (*regionBlock)->begin());3111    return llvm::Error::success();3112  };3113 3114  // Delegate actual loop construction to the OpenMP IRBuilder.3115  // TODO: this currently assumes omp.loop_nest is semantically similar to SCF3116  // loop, i.e. it has a positive step, uses signed integer semantics.3117  // Reconsider this code when the nested loop operation clearly supports more3118  // cases.3119  for (unsigned i = 0, e = loopOp.getNumLoops(); i < e; ++i) {3120    llvm::Value *lowerBound =3121        moduleTranslation.lookupValue(loopOp.getLoopLowerBounds()[i]);3122    llvm::Value *upperBound =3123        moduleTranslation.lookupValue(loopOp.getLoopUpperBounds()[i]);3124    llvm::Value *step = moduleTranslation.lookupValue(loopOp.getLoopSteps()[i]);3125 3126    // Make sure loop trip count are emitted in the preheader of the outermost3127    // loop at the latest so that they are all available for the new collapsed3128    // loop will be created below.3129    llvm::OpenMPIRBuilder::LocationDescription loc = ompLoc;3130    llvm::OpenMPIRBuilder::InsertPointTy computeIP = ompLoc.IP;3131    if (i != 0) {3132      loc = llvm::OpenMPIRBuilder::LocationDescription(bodyInsertPoints.back(),3133                                                       ompLoc.DL);3134      computeIP = loopInfos.front()->getPreheaderIP();3135    }3136 3137    llvm::Expected<llvm::CanonicalLoopInfo *> loopResult =3138        ompBuilder->createCanonicalLoop(3139            loc, bodyGen, lowerBound, upperBound, step,3140            /*IsSigned=*/true, loopOp.getLoopInclusive(), computeIP);3141 3142    if (failed(handleError(loopResult, *loopOp)))3143      return failure();3144 3145    loopInfos.push_back(*loopResult);3146  }3147 3148  llvm::OpenMPIRBuilder::InsertPointTy afterIP =3149      loopInfos.front()->getAfterIP();3150 3151  // Do tiling.3152  if (const auto &tiles = loopOp.getTileSizes()) {3153    llvm::Type *ivType = loopInfos.front()->getIndVarType();3154    SmallVector<llvm::Value *> tileSizes;3155 3156    for (auto tile : tiles.value()) {3157      llvm::Value *tileVal = llvm::ConstantInt::get(ivType, tile);3158      tileSizes.push_back(tileVal);3159    }3160 3161    std::vector<llvm::CanonicalLoopInfo *> newLoops =3162        ompBuilder->tileLoops(ompLoc.DL, loopInfos, tileSizes);3163 3164    // Update afterIP to get the correct insertion point after3165    // tiling.3166    llvm::BasicBlock *afterBB = newLoops.front()->getAfter();3167    llvm::BasicBlock *afterAfterBB = afterBB->getSingleSuccessor();3168    afterIP = {afterAfterBB, afterAfterBB->begin()};3169 3170    // Update the loop infos.3171    loopInfos.clear();3172    for (const auto &newLoop : newLoops)3173      loopInfos.push_back(newLoop);3174  } // Tiling done.3175 3176  // Do collapse.3177  const auto &numCollapse = loopOp.getCollapseNumLoops();3178  SmallVector<llvm::CanonicalLoopInfo *> collapseLoopInfos(3179      loopInfos.begin(), loopInfos.begin() + (numCollapse));3180 3181  auto newTopLoopInfo =3182      ompBuilder->collapseLoops(ompLoc.DL, collapseLoopInfos, {});3183 3184  assert(newTopLoopInfo && "New top loop information is missing");3185  moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(3186      [&](OpenMPLoopInfoStackFrame &frame) {3187        frame.loopInfo = newTopLoopInfo;3188        return WalkResult::interrupt();3189      });3190 3191  // Continue building IR after the loop. Note that the LoopInfo returned by3192  // `collapseLoops` points inside the outermost loop and is intended for3193  // potential further loop transformations. Use the insertion point stored3194  // before collapsing loops instead.3195  builder.restoreIP(afterIP);3196  return success();3197}3198 3199/// Convert an omp.canonical_loop to LLVM-IR3200static LogicalResult3201convertOmpCanonicalLoopOp(omp::CanonicalLoopOp op, llvm::IRBuilderBase &builder,3202                          LLVM::ModuleTranslation &moduleTranslation) {3203  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();3204 3205  llvm::OpenMPIRBuilder::LocationDescription loopLoc(builder);3206  Value loopIV = op.getInductionVar();3207  Value loopTC = op.getTripCount();3208 3209  llvm::Value *llvmTC = moduleTranslation.lookupValue(loopTC);3210 3211  llvm::Expected<llvm::CanonicalLoopInfo *> llvmOrError =3212      ompBuilder->createCanonicalLoop(3213          loopLoc,3214          [&](llvm::OpenMPIRBuilder::InsertPointTy ip, llvm::Value *llvmIV) {3215            // Register the mapping of MLIR induction variable to LLVM-IR3216            // induction variable3217            moduleTranslation.mapValue(loopIV, llvmIV);3218 3219            builder.restoreIP(ip);3220            llvm::Expected<llvm::BasicBlock *> bodyGenStatus =3221                convertOmpOpRegions(op.getRegion(), "omp.loop.region", builder,3222                                    moduleTranslation);3223 3224            return bodyGenStatus.takeError();3225          },3226          llvmTC, "omp.loop");3227  if (!llvmOrError)3228    return op.emitError(llvm::toString(llvmOrError.takeError()));3229 3230  llvm::CanonicalLoopInfo *llvmCLI = *llvmOrError;3231  llvm::IRBuilderBase::InsertPoint afterIP = llvmCLI->getAfterIP();3232  builder.restoreIP(afterIP);3233 3234  // Register the mapping of MLIR loop to LLVM-IR OpenMPIRBuilder loop3235  if (Value cli = op.getCli())3236    moduleTranslation.mapOmpLoop(cli, llvmCLI);3237 3238  return success();3239}3240 3241/// Apply a `#pragma omp unroll` / "!$omp unroll" transformation using the3242/// OpenMPIRBuilder.3243static LogicalResult3244applyUnrollHeuristic(omp::UnrollHeuristicOp op, llvm::IRBuilderBase &builder,3245                     LLVM::ModuleTranslation &moduleTranslation) {3246  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();3247 3248  Value applyee = op.getApplyee();3249  assert(applyee && "Loop to apply unrolling on required");3250 3251  llvm::CanonicalLoopInfo *consBuilderCLI =3252      moduleTranslation.lookupOMPLoop(applyee);3253  llvm::OpenMPIRBuilder::LocationDescription loc(builder);3254  ompBuilder->unrollLoopHeuristic(loc.DL, consBuilderCLI);3255 3256  moduleTranslation.invalidateOmpLoop(applyee);3257  return success();3258}3259 3260/// Apply a `#pragma omp tile` / `!$omp tile` transformation using the3261/// OpenMPIRBuilder.3262static LogicalResult applyTile(omp::TileOp op, llvm::IRBuilderBase &builder,3263                               LLVM::ModuleTranslation &moduleTranslation) {3264  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();3265  llvm::OpenMPIRBuilder::LocationDescription loc(builder);3266 3267  SmallVector<llvm::CanonicalLoopInfo *> translatedLoops;3268  SmallVector<llvm::Value *> translatedSizes;3269 3270  for (Value size : op.getSizes()) {3271    llvm::Value *translatedSize = moduleTranslation.lookupValue(size);3272    assert(translatedSize &&3273           "sizes clause arguments must already be translated");3274    translatedSizes.push_back(translatedSize);3275  }3276 3277  for (Value applyee : op.getApplyees()) {3278    llvm::CanonicalLoopInfo *consBuilderCLI =3279        moduleTranslation.lookupOMPLoop(applyee);3280    assert(applyee && "Canonical loop must already been translated");3281    translatedLoops.push_back(consBuilderCLI);3282  }3283 3284  auto generatedLoops =3285      ompBuilder->tileLoops(loc.DL, translatedLoops, translatedSizes);3286  if (!op.getGeneratees().empty()) {3287    for (auto [mlirLoop, genLoop] :3288         zip_equal(op.getGeneratees(), generatedLoops))3289      moduleTranslation.mapOmpLoop(mlirLoop, genLoop);3290  }3291 3292  // CLIs can only be consumed once3293  for (Value applyee : op.getApplyees())3294    moduleTranslation.invalidateOmpLoop(applyee);3295 3296  return success();3297}3298 3299/// Convert an Atomic Ordering attribute to llvm::AtomicOrdering.3300static llvm::AtomicOrdering3301convertAtomicOrdering(std::optional<omp::ClauseMemoryOrderKind> ao) {3302  if (!ao)3303    return llvm::AtomicOrdering::Monotonic; // Default Memory Ordering3304 3305  switch (*ao) {3306  case omp::ClauseMemoryOrderKind::Seq_cst:3307    return llvm::AtomicOrdering::SequentiallyConsistent;3308  case omp::ClauseMemoryOrderKind::Acq_rel:3309    return llvm::AtomicOrdering::AcquireRelease;3310  case omp::ClauseMemoryOrderKind::Acquire:3311    return llvm::AtomicOrdering::Acquire;3312  case omp::ClauseMemoryOrderKind::Release:3313    return llvm::AtomicOrdering::Release;3314  case omp::ClauseMemoryOrderKind::Relaxed:3315    return llvm::AtomicOrdering::Monotonic;3316  }3317  llvm_unreachable("Unknown ClauseMemoryOrderKind kind");3318}3319 3320/// Convert omp.atomic.read operation to LLVM IR.3321static LogicalResult3322convertOmpAtomicRead(Operation &opInst, llvm::IRBuilderBase &builder,3323                     LLVM::ModuleTranslation &moduleTranslation) {3324  auto readOp = cast<omp::AtomicReadOp>(opInst);3325  if (failed(checkImplementationStatus(opInst)))3326    return failure();3327 3328  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();3329  llvm::OpenMPIRBuilder::InsertPointTy allocaIP =3330      findAllocaInsertPoint(builder, moduleTranslation);3331 3332  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);3333 3334  llvm::AtomicOrdering AO = convertAtomicOrdering(readOp.getMemoryOrder());3335  llvm::Value *x = moduleTranslation.lookupValue(readOp.getX());3336  llvm::Value *v = moduleTranslation.lookupValue(readOp.getV());3337 3338  llvm::Type *elementType =3339      moduleTranslation.convertType(readOp.getElementType());3340 3341  llvm::OpenMPIRBuilder::AtomicOpValue V = {v, elementType, false, false};3342  llvm::OpenMPIRBuilder::AtomicOpValue X = {x, elementType, false, false};3343  builder.restoreIP(ompBuilder->createAtomicRead(ompLoc, X, V, AO, allocaIP));3344  return success();3345}3346 3347/// Converts an omp.atomic.write operation to LLVM IR.3348static LogicalResult3349convertOmpAtomicWrite(Operation &opInst, llvm::IRBuilderBase &builder,3350                      LLVM::ModuleTranslation &moduleTranslation) {3351  auto writeOp = cast<omp::AtomicWriteOp>(opInst);3352  if (failed(checkImplementationStatus(opInst)))3353    return failure();3354 3355  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();3356  llvm::OpenMPIRBuilder::InsertPointTy allocaIP =3357      findAllocaInsertPoint(builder, moduleTranslation);3358 3359  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);3360  llvm::AtomicOrdering ao = convertAtomicOrdering(writeOp.getMemoryOrder());3361  llvm::Value *expr = moduleTranslation.lookupValue(writeOp.getExpr());3362  llvm::Value *dest = moduleTranslation.lookupValue(writeOp.getX());3363  llvm::Type *ty = moduleTranslation.convertType(writeOp.getExpr().getType());3364  llvm::OpenMPIRBuilder::AtomicOpValue x = {dest, ty, /*isSigned=*/false,3365                                            /*isVolatile=*/false};3366  builder.restoreIP(3367      ompBuilder->createAtomicWrite(ompLoc, x, expr, ao, allocaIP));3368  return success();3369}3370 3371/// Converts an LLVM dialect binary operation to the corresponding enum value3372/// for `atomicrmw` supported binary operation.3373static llvm::AtomicRMWInst::BinOp convertBinOpToAtomic(Operation &op) {3374  return llvm::TypeSwitch<Operation *, llvm::AtomicRMWInst::BinOp>(&op)3375      .Case([&](LLVM::AddOp) { return llvm::AtomicRMWInst::BinOp::Add; })3376      .Case([&](LLVM::SubOp) { return llvm::AtomicRMWInst::BinOp::Sub; })3377      .Case([&](LLVM::AndOp) { return llvm::AtomicRMWInst::BinOp::And; })3378      .Case([&](LLVM::OrOp) { return llvm::AtomicRMWInst::BinOp::Or; })3379      .Case([&](LLVM::XOrOp) { return llvm::AtomicRMWInst::BinOp::Xor; })3380      .Case([&](LLVM::UMaxOp) { return llvm::AtomicRMWInst::BinOp::UMax; })3381      .Case([&](LLVM::UMinOp) { return llvm::AtomicRMWInst::BinOp::UMin; })3382      .Case([&](LLVM::FAddOp) { return llvm::AtomicRMWInst::BinOp::FAdd; })3383      .Case([&](LLVM::FSubOp) { return llvm::AtomicRMWInst::BinOp::FSub; })3384      .Default(llvm::AtomicRMWInst::BinOp::BAD_BINOP);3385}3386 3387static void extractAtomicControlFlags(omp::AtomicUpdateOp atomicUpdateOp,3388                                      bool &isIgnoreDenormalMode,3389                                      bool &isFineGrainedMemory,3390                                      bool &isRemoteMemory) {3391  isIgnoreDenormalMode = false;3392  isFineGrainedMemory = false;3393  isRemoteMemory = false;3394  if (atomicUpdateOp &&3395      atomicUpdateOp->hasAttr(atomicUpdateOp.getAtomicControlAttrName())) {3396    mlir::omp::AtomicControlAttr atomicControlAttr =3397        atomicUpdateOp.getAtomicControlAttr();3398    isIgnoreDenormalMode = atomicControlAttr.getIgnoreDenormalMode();3399    isFineGrainedMemory = atomicControlAttr.getFineGrainedMemory();3400    isRemoteMemory = atomicControlAttr.getRemoteMemory();3401  }3402}3403 3404/// Converts an OpenMP atomic update operation using OpenMPIRBuilder.3405static LogicalResult3406convertOmpAtomicUpdate(omp::AtomicUpdateOp &opInst,3407                       llvm::IRBuilderBase &builder,3408                       LLVM::ModuleTranslation &moduleTranslation) {3409  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();3410  if (failed(checkImplementationStatus(*opInst)))3411    return failure();3412 3413  // Convert values and types.3414  auto &innerOpList = opInst.getRegion().front().getOperations();3415  bool isXBinopExpr{false};3416  llvm::AtomicRMWInst::BinOp binop;3417  mlir::Value mlirExpr;3418  llvm::Value *llvmExpr = nullptr;3419  llvm::Value *llvmX = nullptr;3420  llvm::Type *llvmXElementType = nullptr;3421  if (innerOpList.size() == 2) {3422    // The two operations here are the update and the terminator.3423    // Since we can identify the update operation, there is a possibility3424    // that we can generate the atomicrmw instruction.3425    mlir::Operation &innerOp = *opInst.getRegion().front().begin();3426    if (!llvm::is_contained(innerOp.getOperands(),3427                            opInst.getRegion().getArgument(0))) {3428      return opInst.emitError("no atomic update operation with region argument"3429                              " as operand found inside atomic.update region");3430    }3431    binop = convertBinOpToAtomic(innerOp);3432    isXBinopExpr = innerOp.getOperand(0) == opInst.getRegion().getArgument(0);3433    mlirExpr = (isXBinopExpr ? innerOp.getOperand(1) : innerOp.getOperand(0));3434    llvmExpr = moduleTranslation.lookupValue(mlirExpr);3435  } else {3436    // Since the update region includes more than one operation3437    // we will resort to generating a cmpxchg loop.3438    binop = llvm::AtomicRMWInst::BinOp::BAD_BINOP;3439  }3440  llvmX = moduleTranslation.lookupValue(opInst.getX());3441  llvmXElementType = moduleTranslation.convertType(3442      opInst.getRegion().getArgument(0).getType());3443  llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicX = {llvmX, llvmXElementType,3444                                                      /*isSigned=*/false,3445                                                      /*isVolatile=*/false};3446 3447  llvm::AtomicOrdering atomicOrdering =3448      convertAtomicOrdering(opInst.getMemoryOrder());3449 3450  // Generate update code.3451  auto updateFn =3452      [&opInst, &moduleTranslation](3453          llvm::Value *atomicx,3454          llvm::IRBuilder<> &builder) -> llvm::Expected<llvm::Value *> {3455    Block &bb = *opInst.getRegion().begin();3456    moduleTranslation.mapValue(*opInst.getRegion().args_begin(), atomicx);3457    moduleTranslation.mapBlock(&bb, builder.GetInsertBlock());3458    if (failed(moduleTranslation.convertBlock(bb, true, builder)))3459      return llvm::make_error<PreviouslyReportedError>();3460 3461    omp::YieldOp yieldop = dyn_cast<omp::YieldOp>(bb.getTerminator());3462    assert(yieldop && yieldop.getResults().size() == 1 &&3463           "terminator must be omp.yield op and it must have exactly one "3464           "argument");3465    return moduleTranslation.lookupValue(yieldop.getResults()[0]);3466  };3467 3468  bool isIgnoreDenormalMode;3469  bool isFineGrainedMemory;3470  bool isRemoteMemory;3471  extractAtomicControlFlags(opInst, isIgnoreDenormalMode, isFineGrainedMemory,3472                            isRemoteMemory);3473  // Handle ambiguous alloca, if any.3474  auto allocaIP = findAllocaInsertPoint(builder, moduleTranslation);3475  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);3476  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =3477      ompBuilder->createAtomicUpdate(ompLoc, allocaIP, llvmAtomicX, llvmExpr,3478                                     atomicOrdering, binop, updateFn,3479                                     isXBinopExpr, isIgnoreDenormalMode,3480                                     isFineGrainedMemory, isRemoteMemory);3481 3482  if (failed(handleError(afterIP, *opInst)))3483    return failure();3484 3485  builder.restoreIP(*afterIP);3486  return success();3487}3488 3489static LogicalResult3490convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp,3491                        llvm::IRBuilderBase &builder,3492                        LLVM::ModuleTranslation &moduleTranslation) {3493  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();3494  if (failed(checkImplementationStatus(*atomicCaptureOp)))3495    return failure();3496 3497  mlir::Value mlirExpr;3498  bool isXBinopExpr = false, isPostfixUpdate = false;3499  llvm::AtomicRMWInst::BinOp binop = llvm::AtomicRMWInst::BinOp::BAD_BINOP;3500 3501  omp::AtomicUpdateOp atomicUpdateOp = atomicCaptureOp.getAtomicUpdateOp();3502  omp::AtomicWriteOp atomicWriteOp = atomicCaptureOp.getAtomicWriteOp();3503 3504  assert((atomicUpdateOp || atomicWriteOp) &&3505         "internal op must be an atomic.update or atomic.write op");3506 3507  if (atomicWriteOp) {3508    isPostfixUpdate = true;3509    mlirExpr = atomicWriteOp.getExpr();3510  } else {3511    isPostfixUpdate = atomicCaptureOp.getSecondOp() ==3512                      atomicCaptureOp.getAtomicUpdateOp().getOperation();3513    auto &innerOpList = atomicUpdateOp.getRegion().front().getOperations();3514    // Find the binary update operation that uses the region argument3515    // and get the expression to update3516    if (innerOpList.size() == 2) {3517      mlir::Operation &innerOp = *atomicUpdateOp.getRegion().front().begin();3518      if (!llvm::is_contained(innerOp.getOperands(),3519                              atomicUpdateOp.getRegion().getArgument(0))) {3520        return atomicUpdateOp.emitError(3521            "no atomic update operation with region argument"3522            " as operand found inside atomic.update region");3523      }3524      binop = convertBinOpToAtomic(innerOp);3525      isXBinopExpr =3526          innerOp.getOperand(0) == atomicUpdateOp.getRegion().getArgument(0);3527      mlirExpr = (isXBinopExpr ? innerOp.getOperand(1) : innerOp.getOperand(0));3528    } else {3529      binop = llvm::AtomicRMWInst::BinOp::BAD_BINOP;3530    }3531  }3532 3533  llvm::Value *llvmExpr = moduleTranslation.lookupValue(mlirExpr);3534  llvm::Value *llvmX =3535      moduleTranslation.lookupValue(atomicCaptureOp.getAtomicReadOp().getX());3536  llvm::Value *llvmV =3537      moduleTranslation.lookupValue(atomicCaptureOp.getAtomicReadOp().getV());3538  llvm::Type *llvmXElementType = moduleTranslation.convertType(3539      atomicCaptureOp.getAtomicReadOp().getElementType());3540  llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicX = {llvmX, llvmXElementType,3541                                                      /*isSigned=*/false,3542                                                      /*isVolatile=*/false};3543  llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicV = {llvmV, llvmXElementType,3544                                                      /*isSigned=*/false,3545                                                      /*isVolatile=*/false};3546 3547  llvm::AtomicOrdering atomicOrdering =3548      convertAtomicOrdering(atomicCaptureOp.getMemoryOrder());3549 3550  auto updateFn =3551      [&](llvm::Value *atomicx,3552          llvm::IRBuilder<> &builder) -> llvm::Expected<llvm::Value *> {3553    if (atomicWriteOp)3554      return moduleTranslation.lookupValue(atomicWriteOp.getExpr());3555    Block &bb = *atomicUpdateOp.getRegion().begin();3556    moduleTranslation.mapValue(*atomicUpdateOp.getRegion().args_begin(),3557                               atomicx);3558    moduleTranslation.mapBlock(&bb, builder.GetInsertBlock());3559    if (failed(moduleTranslation.convertBlock(bb, true, builder)))3560      return llvm::make_error<PreviouslyReportedError>();3561 3562    omp::YieldOp yieldop = dyn_cast<omp::YieldOp>(bb.getTerminator());3563    assert(yieldop && yieldop.getResults().size() == 1 &&3564           "terminator must be omp.yield op and it must have exactly one "3565           "argument");3566    return moduleTranslation.lookupValue(yieldop.getResults()[0]);3567  };3568 3569  bool isIgnoreDenormalMode;3570  bool isFineGrainedMemory;3571  bool isRemoteMemory;3572  extractAtomicControlFlags(atomicUpdateOp, isIgnoreDenormalMode,3573                            isFineGrainedMemory, isRemoteMemory);3574  // Handle ambiguous alloca, if any.3575  auto allocaIP = findAllocaInsertPoint(builder, moduleTranslation);3576  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);3577  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =3578      ompBuilder->createAtomicCapture(3579          ompLoc, allocaIP, llvmAtomicX, llvmAtomicV, llvmExpr, atomicOrdering,3580          binop, updateFn, atomicUpdateOp, isPostfixUpdate, isXBinopExpr,3581          isIgnoreDenormalMode, isFineGrainedMemory, isRemoteMemory);3582 3583  if (failed(handleError(afterIP, *atomicCaptureOp)))3584    return failure();3585 3586  builder.restoreIP(*afterIP);3587  return success();3588}3589 3590static llvm::omp::Directive convertCancellationConstructType(3591    omp::ClauseCancellationConstructType directive) {3592  switch (directive) {3593  case omp::ClauseCancellationConstructType::Loop:3594    return llvm::omp::Directive::OMPD_for;3595  case omp::ClauseCancellationConstructType::Parallel:3596    return llvm::omp::Directive::OMPD_parallel;3597  case omp::ClauseCancellationConstructType::Sections:3598    return llvm::omp::Directive::OMPD_sections;3599  case omp::ClauseCancellationConstructType::Taskgroup:3600    return llvm::omp::Directive::OMPD_taskgroup;3601  }3602  llvm_unreachable("Unhandled cancellation construct type");3603}3604 3605static LogicalResult3606convertOmpCancel(omp::CancelOp op, llvm::IRBuilderBase &builder,3607                 LLVM::ModuleTranslation &moduleTranslation) {3608  if (failed(checkImplementationStatus(*op.getOperation())))3609    return failure();3610 3611  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);3612  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();3613 3614  llvm::Value *ifCond = nullptr;3615  if (Value ifVar = op.getIfExpr())3616    ifCond = moduleTranslation.lookupValue(ifVar);3617 3618  llvm::omp::Directive cancelledDirective =3619      convertCancellationConstructType(op.getCancelDirective());3620 3621  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =3622      ompBuilder->createCancel(ompLoc, ifCond, cancelledDirective);3623 3624  if (failed(handleError(afterIP, *op.getOperation())))3625    return failure();3626 3627  builder.restoreIP(afterIP.get());3628 3629  return success();3630}3631 3632static LogicalResult3633convertOmpCancellationPoint(omp::CancellationPointOp op,3634                            llvm::IRBuilderBase &builder,3635                            LLVM::ModuleTranslation &moduleTranslation) {3636  if (failed(checkImplementationStatus(*op.getOperation())))3637    return failure();3638 3639  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);3640  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();3641 3642  llvm::omp::Directive cancelledDirective =3643      convertCancellationConstructType(op.getCancelDirective());3644 3645  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =3646      ompBuilder->createCancellationPoint(ompLoc, cancelledDirective);3647 3648  if (failed(handleError(afterIP, *op.getOperation())))3649    return failure();3650 3651  builder.restoreIP(afterIP.get());3652 3653  return success();3654}3655 3656/// Converts an OpenMP Threadprivate operation into LLVM IR using3657/// OpenMPIRBuilder.3658static LogicalResult3659convertOmpThreadprivate(Operation &opInst, llvm::IRBuilderBase &builder,3660                        LLVM::ModuleTranslation &moduleTranslation) {3661  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);3662  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();3663  auto threadprivateOp = cast<omp::ThreadprivateOp>(opInst);3664 3665  if (failed(checkImplementationStatus(opInst)))3666    return failure();3667 3668  Value symAddr = threadprivateOp.getSymAddr();3669  auto *symOp = symAddr.getDefiningOp();3670 3671  if (auto asCast = dyn_cast<LLVM::AddrSpaceCastOp>(symOp))3672    symOp = asCast.getOperand().getDefiningOp();3673 3674  if (!isa<LLVM::AddressOfOp>(symOp))3675    return opInst.emitError("Addressing symbol not found");3676  LLVM::AddressOfOp addressOfOp = dyn_cast<LLVM::AddressOfOp>(symOp);3677 3678  LLVM::GlobalOp global =3679      addressOfOp.getGlobal(moduleTranslation.symbolTable());3680  llvm::GlobalValue *globalValue = moduleTranslation.lookupGlobal(global);3681 3682  if (!ompBuilder->Config.isTargetDevice()) {3683    llvm::Type *type = globalValue->getValueType();3684    llvm::TypeSize typeSize =3685        builder.GetInsertBlock()->getModule()->getDataLayout().getTypeStoreSize(3686            type);3687    llvm::ConstantInt *size = builder.getInt64(typeSize.getFixedValue());3688    llvm::Value *callInst = ompBuilder->createCachedThreadPrivate(3689        ompLoc, globalValue, size, global.getSymName() + ".cache");3690    moduleTranslation.mapValue(opInst.getResult(0), callInst);3691  } else {3692    moduleTranslation.mapValue(opInst.getResult(0), globalValue);3693  }3694 3695  return success();3696}3697 3698static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind3699convertToDeviceClauseKind(mlir::omp::DeclareTargetDeviceType deviceClause) {3700  switch (deviceClause) {3701  case mlir::omp::DeclareTargetDeviceType::host:3702    return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost;3703    break;3704  case mlir::omp::DeclareTargetDeviceType::nohost:3705    return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost;3706    break;3707  case mlir::omp::DeclareTargetDeviceType::any:3708    return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny;3709    break;3710  }3711  llvm_unreachable("unhandled device clause");3712}3713 3714static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind3715convertToCaptureClauseKind(3716    mlir::omp::DeclareTargetCaptureClause captureClause) {3717  switch (captureClause) {3718  case mlir::omp::DeclareTargetCaptureClause::to:3719    return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;3720  case mlir::omp::DeclareTargetCaptureClause::link:3721    return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;3722  case mlir::omp::DeclareTargetCaptureClause::enter:3723    return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter;3724  case mlir::omp::DeclareTargetCaptureClause::none:3725    return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;3726  }3727  llvm_unreachable("unhandled capture clause");3728}3729 3730static Operation *getGlobalOpFromValue(Value value) {3731  Operation *op = value.getDefiningOp();3732  if (auto addrCast = dyn_cast_if_present<LLVM::AddrSpaceCastOp>(op))3733    op = addrCast->getOperand(0).getDefiningOp();3734  if (auto addressOfOp = dyn_cast_if_present<LLVM::AddressOfOp>(op)) {3735    auto modOp = addressOfOp->getParentOfType<mlir::ModuleOp>();3736    return modOp.lookupSymbol(addressOfOp.getGlobalName());3737  }3738  return nullptr;3739}3740 3741static llvm::SmallString<64>3742getDeclareTargetRefPtrSuffix(LLVM::GlobalOp globalOp,3743                             llvm::OpenMPIRBuilder &ompBuilder) {3744  llvm::SmallString<64> suffix;3745  llvm::raw_svector_ostream os(suffix);3746  if (globalOp.getVisibility() == mlir::SymbolTable::Visibility::Private) {3747    auto loc = globalOp->getLoc()->findInstanceOf<FileLineColLoc>();3748    auto fileInfoCallBack = [&loc]() {3749      return std::pair<std::string, uint64_t>(3750          llvm::StringRef(loc.getFilename()), loc.getLine());3751    };3752 3753    auto vfs = llvm::vfs::getRealFileSystem();3754    os << llvm::format(3755        "_%x",3756        ompBuilder.getTargetEntryUniqueInfo(fileInfoCallBack, *vfs).FileID);3757  }3758  os << "_decl_tgt_ref_ptr";3759 3760  return suffix;3761}3762 3763static bool isDeclareTargetLink(Value value) {3764  if (auto declareTargetGlobal =3765          dyn_cast_if_present<omp::DeclareTargetInterface>(3766              getGlobalOpFromValue(value)))3767    if (declareTargetGlobal.getDeclareTargetCaptureClause() ==3768        omp::DeclareTargetCaptureClause::link)3769      return true;3770  return false;3771}3772 3773static bool isDeclareTargetTo(Value value) {3774  if (auto declareTargetGlobal =3775          dyn_cast_if_present<omp::DeclareTargetInterface>(3776              getGlobalOpFromValue(value)))3777    if (declareTargetGlobal.getDeclareTargetCaptureClause() ==3778            omp::DeclareTargetCaptureClause::to ||3779        declareTargetGlobal.getDeclareTargetCaptureClause() ==3780            omp::DeclareTargetCaptureClause::enter)3781      return true;3782  return false;3783}3784 3785// Returns the reference pointer generated by the lowering of the declare3786// target operation in cases where the link clause is used or the to clause is3787// used in USM mode.3788static llvm::Value *3789getRefPtrIfDeclareTarget(Value value,3790                         LLVM::ModuleTranslation &moduleTranslation) {3791  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();3792  if (auto gOp =3793          dyn_cast_or_null<LLVM::GlobalOp>(getGlobalOpFromValue(value))) {3794    if (auto declareTargetGlobal =3795            dyn_cast<omp::DeclareTargetInterface>(gOp.getOperation())) {3796      // In this case, we must utilise the reference pointer generated by3797      // the declare target operation, similar to Clang3798      if ((declareTargetGlobal.getDeclareTargetCaptureClause() ==3799           omp::DeclareTargetCaptureClause::link) ||3800          (declareTargetGlobal.getDeclareTargetCaptureClause() ==3801               omp::DeclareTargetCaptureClause::to &&3802           ompBuilder->Config.hasRequiresUnifiedSharedMemory())) {3803        llvm::SmallString<64> suffix =3804            getDeclareTargetRefPtrSuffix(gOp, *ompBuilder);3805 3806        if (gOp.getSymName().contains(suffix))3807          return moduleTranslation.getLLVMModule()->getNamedValue(3808              gOp.getSymName());3809 3810        return moduleTranslation.getLLVMModule()->getNamedValue(3811            (gOp.getSymName().str() + suffix.str()).str());3812      }3813    }3814  }3815  return nullptr;3816}3817 3818namespace {3819// Append customMappers information to existing MapInfosTy3820struct MapInfosTy : llvm::OpenMPIRBuilder::MapInfosTy {3821  SmallVector<Operation *, 4> Mappers;3822 3823  /// Append arrays in \a CurInfo.3824  void append(MapInfosTy &curInfo) {3825    Mappers.append(curInfo.Mappers.begin(), curInfo.Mappers.end());3826    llvm::OpenMPIRBuilder::MapInfosTy::append(curInfo);3827  }3828};3829// A small helper structure to contain data gathered3830// for map lowering and coalese it into one area and3831// avoiding extra computations such as searches in the3832// llvm module for lowered mapped variables or checking3833// if something is declare target (and retrieving the3834// value) more than neccessary.3835struct MapInfoData : MapInfosTy {3836  llvm::SmallVector<bool, 4> IsDeclareTarget;3837  llvm::SmallVector<bool, 4> IsAMember;3838  // Identify if mapping was added by mapClause or use_device clauses.3839  llvm::SmallVector<bool, 4> IsAMapping;3840  llvm::SmallVector<mlir::Operation *, 4> MapClause;3841  llvm::SmallVector<llvm::Value *, 4> OriginalValue;3842  // Stripped off array/pointer to get the underlying3843  // element type3844  llvm::SmallVector<llvm::Type *, 4> BaseType;3845 3846  /// Append arrays in \a CurInfo.3847  void append(MapInfoData &CurInfo) {3848    IsDeclareTarget.append(CurInfo.IsDeclareTarget.begin(),3849                           CurInfo.IsDeclareTarget.end());3850    MapClause.append(CurInfo.MapClause.begin(), CurInfo.MapClause.end());3851    OriginalValue.append(CurInfo.OriginalValue.begin(),3852                         CurInfo.OriginalValue.end());3853    BaseType.append(CurInfo.BaseType.begin(), CurInfo.BaseType.end());3854    MapInfosTy::append(CurInfo);3855  }3856};3857 3858enum class TargetDirectiveEnumTy : uint32_t {3859  None = 0,3860  Target = 1,3861  TargetData = 2,3862  TargetEnterData = 3,3863  TargetExitData = 4,3864  TargetUpdate = 53865};3866 3867static TargetDirectiveEnumTy getTargetDirectiveEnumTyFromOp(Operation *op) {3868  return llvm::TypeSwitch<Operation *, TargetDirectiveEnumTy>(op)3869      .Case([](omp::TargetDataOp) { return TargetDirectiveEnumTy::TargetData; })3870      .Case([](omp::TargetEnterDataOp) {3871        return TargetDirectiveEnumTy::TargetEnterData;3872      })3873      .Case([&](omp::TargetExitDataOp) {3874        return TargetDirectiveEnumTy::TargetExitData;3875      })3876      .Case([&](omp::TargetUpdateOp) {3877        return TargetDirectiveEnumTy::TargetUpdate;3878      })3879      .Case([&](omp::TargetOp) { return TargetDirectiveEnumTy::Target; })3880      .Default([&](Operation *op) { return TargetDirectiveEnumTy::None; });3881}3882 3883} // namespace3884 3885static uint64_t getArrayElementSizeInBits(LLVM::LLVMArrayType arrTy,3886                                          DataLayout &dl) {3887  if (auto nestedArrTy = llvm::dyn_cast_if_present<LLVM::LLVMArrayType>(3888          arrTy.getElementType()))3889    return getArrayElementSizeInBits(nestedArrTy, dl);3890  return dl.getTypeSizeInBits(arrTy.getElementType());3891}3892 3893// This function calculates the size to be offloaded for a specified type, given3894// its associated map clause (which can contain bounds information which affects3895// the total size), this size is calculated based on the underlying element type3896// e.g. given a 1-D array of ints, we will calculate the size from the integer3897// type * number of elements in the array. This size can be used in other3898// calculations but is ultimately used as an argument to the OpenMP runtimes3899// kernel argument structure which is generated through the combinedInfo data3900// structures.3901// This function is somewhat equivalent to Clang's getExprTypeSize inside of3902// CGOpenMPRuntime.cpp.3903static llvm::Value *getSizeInBytes(DataLayout &dl, const mlir::Type &type,3904                                   Operation *clauseOp,3905                                   llvm::Value *basePointer,3906                                   llvm::Type *baseType,3907                                   llvm::IRBuilderBase &builder,3908                                   LLVM::ModuleTranslation &moduleTranslation) {3909  if (auto memberClause =3910          mlir::dyn_cast_if_present<mlir::omp::MapInfoOp>(clauseOp)) {3911    // This calculates the size to transfer based on bounds and the underlying3912    // element type, provided bounds have been specified (Fortran3913    // pointers/allocatables/target and arrays that have sections specified fall3914    // into this as well)3915    if (!memberClause.getBounds().empty()) {3916      llvm::Value *elementCount = builder.getInt64(1);3917      for (auto bounds : memberClause.getBounds()) {3918        if (auto boundOp = mlir::dyn_cast_if_present<mlir::omp::MapBoundsOp>(3919                bounds.getDefiningOp())) {3920          // The below calculation for the size to be mapped calculated from the3921          // map.info's bounds is: (elemCount * [UB - LB] + 1), later we3922          // multiply by the underlying element types byte size to get the full3923          // size to be offloaded based on the bounds3924          elementCount = builder.CreateMul(3925              elementCount,3926              builder.CreateAdd(3927                  builder.CreateSub(3928                      moduleTranslation.lookupValue(boundOp.getUpperBound()),3929                      moduleTranslation.lookupValue(boundOp.getLowerBound())),3930                  builder.getInt64(1)));3931        }3932      }3933 3934      // utilising getTypeSizeInBits instead of getTypeSize as getTypeSize gives3935      // the size in inconsistent byte or bit format.3936      uint64_t underlyingTypeSzInBits = dl.getTypeSizeInBits(type);3937      if (auto arrTy = llvm::dyn_cast_if_present<LLVM::LLVMArrayType>(type))3938        underlyingTypeSzInBits = getArrayElementSizeInBits(arrTy, dl);3939 3940      // The size in bytes x number of elements, the sizeInBytes stored is3941      // the underyling types size, e.g. if ptr<i32>, it'll be the i32's3942      // size, so we do some on the fly runtime math to get the size in3943      // bytes from the extent (ub - lb) * sizeInBytes. NOTE: This may need3944      // some adjustment for members with more complex types.3945      return builder.CreateMul(elementCount,3946                               builder.getInt64(underlyingTypeSzInBits / 8));3947    }3948  }3949 3950  return builder.getInt64(dl.getTypeSizeInBits(type) / 8);3951}3952 3953// Convert the MLIR map flag set to the runtime map flag set for embedding3954// in LLVM-IR. This is important as the two bit-flag lists do not correspond3955// 1-to-1 as there's flags the runtime doesn't care about and vice versa.3956// Certain flags are discarded here such as RefPtee and co.3957static llvm::omp::OpenMPOffloadMappingFlags3958convertClauseMapFlags(omp::ClauseMapFlags mlirFlags) {3959  auto mapTypeToBool = [&mlirFlags](omp::ClauseMapFlags flag) {3960    return (mlirFlags & flag) == flag;3961  };3962 3963  llvm::omp::OpenMPOffloadMappingFlags mapType =3964      llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE;3965 3966  if (mapTypeToBool(omp::ClauseMapFlags::to))3967    mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO;3968 3969  if (mapTypeToBool(omp::ClauseMapFlags::from))3970    mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM;3971 3972  if (mapTypeToBool(omp::ClauseMapFlags::always))3973    mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;3974 3975  if (mapTypeToBool(omp::ClauseMapFlags::del))3976    mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_DELETE;3977 3978  if (mapTypeToBool(omp::ClauseMapFlags::return_param))3979    mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;3980 3981  if (mapTypeToBool(omp::ClauseMapFlags::priv))3982    mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PRIVATE;3983 3984  if (mapTypeToBool(omp::ClauseMapFlags::literal))3985    mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL;3986 3987  if (mapTypeToBool(omp::ClauseMapFlags::implicit))3988    mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;3989 3990  if (mapTypeToBool(omp::ClauseMapFlags::close))3991    mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_CLOSE;3992 3993  if (mapTypeToBool(omp::ClauseMapFlags::present))3994    mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;3995 3996  if (mapTypeToBool(omp::ClauseMapFlags::ompx_hold))3997    mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;3998 3999  if (mapTypeToBool(omp::ClauseMapFlags::attach))4000    mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ATTACH;4001 4002  return mapType;4003}4004 4005static void collectMapDataFromMapOperands(4006    MapInfoData &mapData, SmallVectorImpl<Value> &mapVars,4007    LLVM::ModuleTranslation &moduleTranslation, DataLayout &dl,4008    llvm::IRBuilderBase &builder, ArrayRef<Value> useDevPtrOperands = {},4009    ArrayRef<Value> useDevAddrOperands = {},4010    ArrayRef<Value> hasDevAddrOperands = {}) {4011  auto checkIsAMember = [](const auto &mapVars, auto mapOp) {4012    // Check if this is a member mapping and correctly assign that it is, if4013    // it is a member of a larger object.4014    // TODO: Need better handling of members, and distinguishing of members4015    // that are implicitly allocated on device vs explicitly passed in as4016    // arguments.4017    // TODO: May require some further additions to support nested record4018    // types, i.e. member maps that can have member maps.4019    for (Value mapValue : mapVars) {4020      auto map = cast<omp::MapInfoOp>(mapValue.getDefiningOp());4021      for (auto member : map.getMembers())4022        if (member == mapOp)4023          return true;4024    }4025    return false;4026  };4027 4028  // Process MapOperands4029  for (Value mapValue : mapVars) {4030    auto mapOp = cast<omp::MapInfoOp>(mapValue.getDefiningOp());4031    Value offloadPtr =4032        mapOp.getVarPtrPtr() ? mapOp.getVarPtrPtr() : mapOp.getVarPtr();4033    mapData.OriginalValue.push_back(moduleTranslation.lookupValue(offloadPtr));4034    mapData.Pointers.push_back(mapData.OriginalValue.back());4035 4036    if (llvm::Value *refPtr =4037            getRefPtrIfDeclareTarget(offloadPtr, moduleTranslation)) {4038      mapData.IsDeclareTarget.push_back(true);4039      mapData.BasePointers.push_back(refPtr);4040    } else if (isDeclareTargetTo(offloadPtr)) {4041      mapData.IsDeclareTarget.push_back(true);4042      mapData.BasePointers.push_back(mapData.OriginalValue.back());4043    } else { // regular mapped variable4044      mapData.IsDeclareTarget.push_back(false);4045      mapData.BasePointers.push_back(mapData.OriginalValue.back());4046    }4047 4048    mapData.BaseType.push_back(4049        moduleTranslation.convertType(mapOp.getVarType()));4050    mapData.Sizes.push_back(4051        getSizeInBytes(dl, mapOp.getVarType(), mapOp, mapData.Pointers.back(),4052                       mapData.BaseType.back(), builder, moduleTranslation));4053    mapData.MapClause.push_back(mapOp.getOperation());4054    mapData.Types.push_back(convertClauseMapFlags(mapOp.getMapType()));4055    mapData.Names.push_back(LLVM::createMappingInformation(4056        mapOp.getLoc(), *moduleTranslation.getOpenMPBuilder()));4057    mapData.DevicePointers.push_back(llvm::OpenMPIRBuilder::DeviceInfoTy::None);4058    if (mapOp.getMapperId())4059      mapData.Mappers.push_back(4060          SymbolTable::lookupNearestSymbolFrom<omp::DeclareMapperOp>(4061              mapOp, mapOp.getMapperIdAttr()));4062    else4063      mapData.Mappers.push_back(nullptr);4064    mapData.IsAMapping.push_back(true);4065    mapData.IsAMember.push_back(checkIsAMember(mapVars, mapOp));4066  }4067 4068  auto findMapInfo = [&mapData](llvm::Value *val,4069                                llvm::OpenMPIRBuilder::DeviceInfoTy devInfoTy) {4070    unsigned index = 0;4071    bool found = false;4072    for (llvm::Value *basePtr : mapData.OriginalValue) {4073      if (basePtr == val && mapData.IsAMapping[index]) {4074        found = true;4075        mapData.Types[index] |=4076            llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;4077        mapData.DevicePointers[index] = devInfoTy;4078      }4079      index++;4080    }4081    return found;4082  };4083 4084  // Process useDevPtr(Addr)Operands4085  auto addDevInfos = [&](const llvm::ArrayRef<Value> &useDevOperands,4086                         llvm::OpenMPIRBuilder::DeviceInfoTy devInfoTy) {4087    for (Value mapValue : useDevOperands) {4088      auto mapOp = cast<omp::MapInfoOp>(mapValue.getDefiningOp());4089      Value offloadPtr =4090          mapOp.getVarPtrPtr() ? mapOp.getVarPtrPtr() : mapOp.getVarPtr();4091      llvm::Value *origValue = moduleTranslation.lookupValue(offloadPtr);4092 4093      // Check if map info is already present for this entry.4094      if (!findMapInfo(origValue, devInfoTy)) {4095        mapData.OriginalValue.push_back(origValue);4096        mapData.Pointers.push_back(mapData.OriginalValue.back());4097        mapData.IsDeclareTarget.push_back(false);4098        mapData.BasePointers.push_back(mapData.OriginalValue.back());4099        mapData.BaseType.push_back(4100            moduleTranslation.convertType(mapOp.getVarType()));4101        mapData.Sizes.push_back(builder.getInt64(0));4102        mapData.MapClause.push_back(mapOp.getOperation());4103        mapData.Types.push_back(4104            llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM);4105        mapData.Names.push_back(LLVM::createMappingInformation(4106            mapOp.getLoc(), *moduleTranslation.getOpenMPBuilder()));4107        mapData.DevicePointers.push_back(devInfoTy);4108        mapData.Mappers.push_back(nullptr);4109        mapData.IsAMapping.push_back(false);4110        mapData.IsAMember.push_back(checkIsAMember(useDevOperands, mapOp));4111      }4112    }4113  };4114 4115  addDevInfos(useDevAddrOperands, llvm::OpenMPIRBuilder::DeviceInfoTy::Address);4116  addDevInfos(useDevPtrOperands, llvm::OpenMPIRBuilder::DeviceInfoTy::Pointer);4117 4118  for (Value mapValue : hasDevAddrOperands) {4119    auto mapOp = cast<omp::MapInfoOp>(mapValue.getDefiningOp());4120    Value offloadPtr =4121        mapOp.getVarPtrPtr() ? mapOp.getVarPtrPtr() : mapOp.getVarPtr();4122    llvm::Value *origValue = moduleTranslation.lookupValue(offloadPtr);4123    auto mapType = convertClauseMapFlags(mapOp.getMapType());4124    auto mapTypeAlways = llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;4125 4126    mapData.OriginalValue.push_back(origValue);4127    mapData.BasePointers.push_back(origValue);4128    mapData.Pointers.push_back(origValue);4129    mapData.IsDeclareTarget.push_back(false);4130    mapData.BaseType.push_back(4131        moduleTranslation.convertType(mapOp.getVarType()));4132    mapData.Sizes.push_back(4133        builder.getInt64(dl.getTypeSize(mapOp.getVarType())));4134    mapData.MapClause.push_back(mapOp.getOperation());4135    if (llvm::to_underlying(mapType & mapTypeAlways)) {4136      // Descriptors are mapped with the ALWAYS flag, since they can get4137      // rematerialized, so the address of the decriptor for a given object4138      // may change from one place to another.4139      mapData.Types.push_back(mapType);4140      // Technically it's possible for a non-descriptor mapping to have4141      // both has-device-addr and ALWAYS, so lookup the mapper in case it4142      // exists.4143      if (mapOp.getMapperId()) {4144        mapData.Mappers.push_back(4145            SymbolTable::lookupNearestSymbolFrom<omp::DeclareMapperOp>(4146                mapOp, mapOp.getMapperIdAttr()));4147      } else {4148        mapData.Mappers.push_back(nullptr);4149      }4150    } else {4151      mapData.Types.push_back(4152          llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);4153      mapData.Mappers.push_back(nullptr);4154    }4155    mapData.Names.push_back(LLVM::createMappingInformation(4156        mapOp.getLoc(), *moduleTranslation.getOpenMPBuilder()));4157    mapData.DevicePointers.push_back(4158        llvm::OpenMPIRBuilder::DeviceInfoTy::Address);4159    mapData.IsAMapping.push_back(false);4160    mapData.IsAMember.push_back(checkIsAMember(hasDevAddrOperands, mapOp));4161  }4162}4163 4164static int getMapDataMemberIdx(MapInfoData &mapData, omp::MapInfoOp memberOp) {4165  auto *res = llvm::find(mapData.MapClause, memberOp);4166  assert(res != mapData.MapClause.end() &&4167         "MapInfoOp for member not found in MapData, cannot return index");4168  return std::distance(mapData.MapClause.begin(), res);4169}4170 4171static void sortMapIndices(llvm::SmallVectorImpl<size_t> &indices,4172                           omp::MapInfoOp mapInfo) {4173  ArrayAttr indexAttr = mapInfo.getMembersIndexAttr();4174  llvm::SmallVector<size_t> occludedChildren;4175  llvm::sort(4176      indices.begin(), indices.end(), [&](const size_t a, const size_t b) {4177        // Bail early if we are asked to look at the same index. If we do not4178        // bail early, we can end up mistakenly adding indices to4179        // occludedChildren. This can occur with some types of libc++ hardening.4180        if (a == b)4181          return false;4182 4183        auto memberIndicesA = cast<ArrayAttr>(indexAttr[a]);4184        auto memberIndicesB = cast<ArrayAttr>(indexAttr[b]);4185 4186        for (auto it : llvm::zip(memberIndicesA, memberIndicesB)) {4187          int64_t aIndex = mlir::cast<IntegerAttr>(std::get<0>(it)).getInt();4188          int64_t bIndex = mlir::cast<IntegerAttr>(std::get<1>(it)).getInt();4189 4190          if (aIndex == bIndex)4191            continue;4192 4193          if (aIndex < bIndex)4194            return true;4195 4196          if (aIndex > bIndex)4197            return false;4198        }4199 4200        // Iterated up until the end of the smallest member and4201        // they were found to be equal up to that point, so select4202        // the member with the lowest index count, so the "parent"4203        bool memberAParent = memberIndicesA.size() < memberIndicesB.size();4204        if (memberAParent)4205          occludedChildren.push_back(b);4206        else4207          occludedChildren.push_back(a);4208        return memberAParent;4209      });4210 4211  // We remove children from the index list that are overshadowed by4212  // a parent, this prevents us retrieving these as the first or last4213  // element when the parent is the correct element in these cases.4214  for (auto v : occludedChildren)4215    indices.erase(std::remove(indices.begin(), indices.end(), v),4216                  indices.end());4217}4218 4219static omp::MapInfoOp getFirstOrLastMappedMemberPtr(omp::MapInfoOp mapInfo,4220                                                    bool first) {4221  ArrayAttr indexAttr = mapInfo.getMembersIndexAttr();4222  // Only 1 member has been mapped, we can return it.4223  if (indexAttr.size() == 1)4224    return cast<omp::MapInfoOp>(mapInfo.getMembers()[0].getDefiningOp());4225  llvm::SmallVector<size_t> indices(indexAttr.size());4226  std::iota(indices.begin(), indices.end(), 0);4227  sortMapIndices(indices, mapInfo);4228  return llvm::cast<omp::MapInfoOp>(4229      mapInfo.getMembers()[first ? indices.front() : indices.back()]4230          .getDefiningOp());4231}4232 4233/// This function calculates the array/pointer offset for map data provided4234/// with bounds operations, e.g. when provided something like the following:4235///4236/// Fortran4237///     map(tofrom: array(2:5, 3:2))4238///4239/// We must calculate the initial pointer offset to pass across, this function4240/// performs this using bounds.4241///4242/// TODO/WARNING: This only supports Fortran's column major indexing currently4243/// as is noted in the note below and comments in the function, we must extend4244/// this function when we add a C++ frontend.4245/// NOTE: which while specified in row-major order it currently needs to be4246/// flipped for Fortran's column order array allocation and access (as4247/// opposed to C++'s row-major, hence the backwards processing where order is4248/// important). This is likely important to keep in mind for the future when4249/// we incorporate a C++ frontend, both frontends will need to agree on the4250/// ordering of generated bounds operations (one may have to flip them) to4251/// make the below lowering frontend agnostic. The offload size4252/// calcualtion may also have to be adjusted for C++.4253static std::vector<llvm::Value *>4254calculateBoundsOffset(LLVM::ModuleTranslation &moduleTranslation,4255                      llvm::IRBuilderBase &builder, bool isArrayTy,4256                      OperandRange bounds) {4257  std::vector<llvm::Value *> idx;4258  // There's no bounds to calculate an offset from, we can safely4259  // ignore and return no indices.4260  if (bounds.empty())4261    return idx;4262 4263  // If we have an array type, then we have its type so can treat it as a4264  // normal GEP instruction where the bounds operations are simply indexes4265  // into the array. We currently do reverse order of the bounds, which4266  // I believe leans more towards Fortran's column-major in memory.4267  if (isArrayTy) {4268    idx.push_back(builder.getInt64(0));4269    for (int i = bounds.size() - 1; i >= 0; --i) {4270      if (auto boundOp = dyn_cast_if_present<omp::MapBoundsOp>(4271              bounds[i].getDefiningOp())) {4272        idx.push_back(moduleTranslation.lookupValue(boundOp.getLowerBound()));4273      }4274    }4275  } else {4276    // If we do not have an array type, but we have bounds, then we're dealing4277    // with a pointer that's being treated like an array and we have the4278    // underlying type e.g. an i32, or f64 etc, e.g. a fortran descriptor base4279    // address (pointer pointing to the actual data) so we must caclulate the4280    // offset using a single index which the following loop attempts to4281    // compute using the standard column-major algorithm e.g for a 3D array:4282    //4283    // ((((c_idx * b_len) + b_idx) * a_len) + a_idx)4284    //4285    // It is of note that it's doing column-major rather than row-major at the4286    // moment, but having a way for the frontend to indicate which major format4287    // to use or standardizing/canonicalizing the order of the bounds to compute4288    // the offset may be useful in the future when there's other frontends with4289    // different formats.4290    std::vector<llvm::Value *> dimensionIndexSizeOffset;4291    for (int i = bounds.size() - 1; i >= 0; --i) {4292      if (auto boundOp = dyn_cast_if_present<omp::MapBoundsOp>(4293              bounds[i].getDefiningOp())) {4294        if (i == ((int)bounds.size() - 1))4295          idx.emplace_back(4296              moduleTranslation.lookupValue(boundOp.getLowerBound()));4297        else4298          idx.back() = builder.CreateAdd(4299              builder.CreateMul(idx.back(), moduleTranslation.lookupValue(4300                                                boundOp.getExtent())),4301              moduleTranslation.lookupValue(boundOp.getLowerBound()));4302      }4303    }4304  }4305 4306  return idx;4307}4308 4309static void getAsIntegers(ArrayAttr values, llvm::SmallVector<int64_t> &ints) {4310  llvm::transform(values, std::back_inserter(ints), [](Attribute value) {4311    return cast<IntegerAttr>(value).getInt();4312  });4313}4314 4315// Gathers members that are overlapping in the parent, excluding members that4316// themselves overlap, keeping the top-most (closest to parents level) map.4317static void4318getOverlappedMembers(llvm::SmallVectorImpl<size_t> &overlapMapDataIdxs,4319                     omp::MapInfoOp parentOp) {4320  // No members mapped, no overlaps.4321  if (parentOp.getMembers().empty())4322    return;4323 4324  // Single member, we can insert and return early.4325  if (parentOp.getMembers().size() == 1) {4326    overlapMapDataIdxs.push_back(0);4327    return;4328  }4329 4330  // 1) collect list of top-level overlapping members from MemberOp4331  llvm::SmallVector<std::pair<int, ArrayAttr>> memberByIndex;4332  ArrayAttr indexAttr = parentOp.getMembersIndexAttr();4333  for (auto [memIndex, indicesAttr] : llvm::enumerate(indexAttr))4334    memberByIndex.push_back(4335        std::make_pair(memIndex, cast<ArrayAttr>(indicesAttr)));4336 4337  // Sort the smallest first (higher up the parent -> member chain), so that4338  // when we remove members, we remove as much as we can in the initial4339  // iterations, shortening the number of passes required.4340  llvm::sort(memberByIndex.begin(), memberByIndex.end(),4341             [&](auto a, auto b) { return a.second.size() < b.second.size(); });4342 4343  // Remove elements from the vector if there is a parent element that4344  // supersedes it. i.e. if member [0] is mapped, we can remove members [0,1],4345  // [0,2].. etc.4346  llvm::SmallVector<std::pair<int, ArrayAttr>> skipList;4347  for (auto v : memberByIndex) {4348    llvm::SmallVector<int64_t> vArr(v.second.size());4349    getAsIntegers(v.second, vArr);4350    skipList.push_back(4351        *std::find_if(memberByIndex.begin(), memberByIndex.end(), [&](auto x) {4352          if (v == x)4353            return false;4354          llvm::SmallVector<int64_t> xArr(x.second.size());4355          getAsIntegers(x.second, xArr);4356          return std::equal(vArr.begin(), vArr.end(), xArr.begin()) &&4357                 xArr.size() >= vArr.size();4358        }));4359  }4360 4361  // Collect the indices, as we need the base pointer etc. from the MapData4362  // structure which is primarily accessible via index at the moment.4363  for (auto v : memberByIndex)4364    if (find(skipList.begin(), skipList.end(), v) == skipList.end())4365      overlapMapDataIdxs.push_back(v.first);4366}4367 4368// The intent is to verify if the mapped data being passed is a4369// pointer -> pointee that requires special handling in certain cases,4370// e.g. applying the OMP_MAP_PTR_AND_OBJ map type.4371//4372// There may be a better way to verify this, but unfortunately with4373// opaque pointers we lose the ability to easily check if something is4374// a pointer whilst maintaining access to the underlying type.4375static bool checkIfPointerMap(omp::MapInfoOp mapOp) {4376  // If we have a varPtrPtr field assigned then the underlying type is a pointer4377  if (mapOp.getVarPtrPtr())4378    return true;4379 4380  // If the map data is declare target with a link clause, then it's represented4381  // as a pointer when we lower it to LLVM-IR even if at the MLIR level it has4382  // no relation to pointers.4383  if (isDeclareTargetLink(mapOp.getVarPtr()))4384    return true;4385 4386  return false;4387}4388 4389// This creates two insertions into the MapInfosTy data structure for the4390// "parent" of a set of members, (usually a container e.g.4391// class/structure/derived type) when subsequent members have also been4392// explicitly mapped on the same map clause. Certain types, such as Fortran4393// descriptors are mapped like this as well, however, the members are4394// implicit as far as a user is concerned, but we must explicitly map them4395// internally.4396//4397// This function also returns the memberOfFlag for this particular parent,4398// which is utilised in subsequent member mappings (by modifying there map type4399// with it) to indicate that a member is part of this parent and should be4400// treated by the runtime as such. Important to achieve the correct mapping.4401//4402// This function borrows a lot from Clang's emitCombinedEntry function4403// inside of CGOpenMPRuntime.cpp4404static llvm::omp::OpenMPOffloadMappingFlags mapParentWithMembers(4405    LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder,4406    llvm::OpenMPIRBuilder &ompBuilder, DataLayout &dl, MapInfosTy &combinedInfo,4407    MapInfoData &mapData, uint64_t mapDataIndex,4408    TargetDirectiveEnumTy targetDirective) {4409  assert(!ompBuilder.Config.isTargetDevice() &&4410         "function only supported for host device codegen");4411 4412  // Map the first segment of the parent. If a user-defined mapper is attached,4413  // include the parent's to/from-style bits (and common modifiers) in this4414  // base entry so the mapper receives correct copy semantics via its 'type'4415  // parameter. Also keep TARGET_PARAM when required for kernel arguments.4416  llvm::omp::OpenMPOffloadMappingFlags baseFlag =4417      (targetDirective == TargetDirectiveEnumTy::Target &&4418       !mapData.IsDeclareTarget[mapDataIndex])4419          ? llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM4420          : llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE;4421 4422  // Detect if this mapping uses a user-defined mapper.4423  bool hasUserMapper = mapData.Mappers[mapDataIndex] != nullptr;4424  if (hasUserMapper) {4425    using mapFlags = llvm::omp::OpenMPOffloadMappingFlags;4426    // Preserve relevant map-type bits from the parent clause. These include4427    // the copy direction (TO/FROM), as well as commonly used modifiers that4428    // should be visible to the mapper for correct behaviour.4429    mapFlags parentFlags = mapData.Types[mapDataIndex];4430    mapFlags preserve = mapFlags::OMP_MAP_TO | mapFlags::OMP_MAP_FROM |4431                        mapFlags::OMP_MAP_ALWAYS | mapFlags::OMP_MAP_CLOSE |4432                        mapFlags::OMP_MAP_PRESENT | mapFlags::OMP_MAP_OMPX_HOLD;4433    baseFlag |= (parentFlags & preserve);4434  }4435 4436  combinedInfo.Types.emplace_back(baseFlag);4437  combinedInfo.DevicePointers.emplace_back(4438      mapData.DevicePointers[mapDataIndex]);4439  combinedInfo.Mappers.emplace_back(mapData.Mappers[mapDataIndex]);4440  combinedInfo.Names.emplace_back(LLVM::createMappingInformation(4441      mapData.MapClause[mapDataIndex]->getLoc(), ompBuilder));4442  combinedInfo.BasePointers.emplace_back(mapData.BasePointers[mapDataIndex]);4443 4444  // Calculate size of the parent object being mapped based on the4445  // addresses at runtime, highAddr - lowAddr = size. This of course4446  // doesn't factor in allocated data like pointers, hence the further4447  // processing of members specified by users, or in the case of4448  // Fortran pointers and allocatables, the mapping of the pointed to4449  // data by the descriptor (which itself, is a structure containing4450  // runtime information on the dynamically allocated data).4451  auto parentClause =4452      llvm::cast<omp::MapInfoOp>(mapData.MapClause[mapDataIndex]);4453  llvm::Value *lowAddr, *highAddr;4454  if (!parentClause.getPartialMap()) {4455    lowAddr = builder.CreatePointerCast(mapData.Pointers[mapDataIndex],4456                                        builder.getPtrTy());4457    highAddr = builder.CreatePointerCast(4458        builder.CreateConstGEP1_32(mapData.BaseType[mapDataIndex],4459                                   mapData.Pointers[mapDataIndex], 1),4460        builder.getPtrTy());4461    combinedInfo.Pointers.emplace_back(mapData.Pointers[mapDataIndex]);4462  } else {4463    auto mapOp = dyn_cast<omp::MapInfoOp>(mapData.MapClause[mapDataIndex]);4464    int firstMemberIdx = getMapDataMemberIdx(4465        mapData, getFirstOrLastMappedMemberPtr(mapOp, true));4466    lowAddr = builder.CreatePointerCast(mapData.Pointers[firstMemberIdx],4467                                        builder.getPtrTy());4468    int lastMemberIdx = getMapDataMemberIdx(4469        mapData, getFirstOrLastMappedMemberPtr(mapOp, false));4470    highAddr = builder.CreatePointerCast(4471        builder.CreateGEP(mapData.BaseType[lastMemberIdx],4472                          mapData.Pointers[lastMemberIdx], builder.getInt64(1)),4473        builder.getPtrTy());4474    combinedInfo.Pointers.emplace_back(mapData.Pointers[firstMemberIdx]);4475  }4476 4477  llvm::Value *size = builder.CreateIntCast(4478      builder.CreatePtrDiff(builder.getInt8Ty(), highAddr, lowAddr),4479      builder.getInt64Ty(),4480      /*isSigned=*/false);4481  combinedInfo.Sizes.push_back(size);4482 4483  llvm::omp::OpenMPOffloadMappingFlags memberOfFlag =4484      ompBuilder.getMemberOfFlag(combinedInfo.BasePointers.size() - 1);4485 4486  // This creates the initial MEMBER_OF mapping that consists of4487  // the parent/top level container (same as above effectively, except4488  // with a fixed initial compile time size and separate maptype which4489  // indicates the true mape type (tofrom etc.). This parent mapping is4490  // only relevant if the structure in its totality is being mapped,4491  // otherwise the above suffices.4492  if (!parentClause.getPartialMap()) {4493    // TODO: This will need to be expanded to include the whole host of logic4494    // for the map flags that Clang currently supports (e.g. it should do some4495    // further case specific flag modifications). For the moment, it handles4496    // what we support as expected.4497    llvm::omp::OpenMPOffloadMappingFlags mapFlag = mapData.Types[mapDataIndex];4498    bool hasMapClose = (llvm::omp::OpenMPOffloadMappingFlags(mapFlag) &4499                        llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_CLOSE) ==4500                       llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_CLOSE;4501    ompBuilder.setCorrectMemberOfFlag(mapFlag, memberOfFlag);4502 4503    if (targetDirective == TargetDirectiveEnumTy::TargetUpdate || hasMapClose) {4504      combinedInfo.Types.emplace_back(mapFlag);4505      combinedInfo.DevicePointers.emplace_back(4506          mapData.DevicePointers[mapDataIndex]);4507      combinedInfo.Names.emplace_back(LLVM::createMappingInformation(4508          mapData.MapClause[mapDataIndex]->getLoc(), ompBuilder));4509      combinedInfo.BasePointers.emplace_back(4510          mapData.BasePointers[mapDataIndex]);4511      combinedInfo.Pointers.emplace_back(mapData.Pointers[mapDataIndex]);4512      combinedInfo.Sizes.emplace_back(mapData.Sizes[mapDataIndex]);4513      combinedInfo.Mappers.emplace_back(mapData.Mappers[mapDataIndex]);4514    } else {4515      llvm::SmallVector<size_t> overlapIdxs;4516      // Find all of the members that "overlap", i.e. occlude other members that4517      // were mapped alongside the parent, e.g. member [0], occludes [0,1] and4518      // [0,2], but not [1,0].4519      getOverlappedMembers(overlapIdxs, parentClause);4520      // We need to make sure the overlapped members are sorted in order of4521      // lowest address to highest address.4522      sortMapIndices(overlapIdxs, parentClause);4523 4524      lowAddr = builder.CreatePointerCast(mapData.Pointers[mapDataIndex],4525                                          builder.getPtrTy());4526      highAddr = builder.CreatePointerCast(4527          builder.CreateConstGEP1_32(mapData.BaseType[mapDataIndex],4528                                     mapData.Pointers[mapDataIndex], 1),4529          builder.getPtrTy());4530 4531      // TODO: We may want to skip arrays/array sections in this as Clang does.4532      // It appears to be an optimisation rather than a necessity though,4533      // but this requires further investigation. However, we would have to make4534      // sure to not exclude maps with bounds that ARE pointers, as these are4535      // processed as separate components, i.e. pointer + data.4536      for (auto v : overlapIdxs) {4537        auto mapDataOverlapIdx = getMapDataMemberIdx(4538            mapData,4539            cast<omp::MapInfoOp>(parentClause.getMembers()[v].getDefiningOp()));4540        combinedInfo.Types.emplace_back(mapFlag);4541        combinedInfo.DevicePointers.emplace_back(4542            mapData.DevicePointers[mapDataOverlapIdx]);4543        combinedInfo.Names.emplace_back(LLVM::createMappingInformation(4544            mapData.MapClause[mapDataIndex]->getLoc(), ompBuilder));4545        combinedInfo.BasePointers.emplace_back(4546            mapData.BasePointers[mapDataIndex]);4547        combinedInfo.Mappers.emplace_back(mapData.Mappers[mapDataIndex]);4548        combinedInfo.Pointers.emplace_back(lowAddr);4549        combinedInfo.Sizes.emplace_back(builder.CreateIntCast(4550            builder.CreatePtrDiff(builder.getInt8Ty(),4551                                  mapData.OriginalValue[mapDataOverlapIdx],4552                                  lowAddr),4553            builder.getInt64Ty(), /*isSigned=*/true));4554        lowAddr = builder.CreateConstGEP1_32(4555            checkIfPointerMap(llvm::cast<omp::MapInfoOp>(4556                mapData.MapClause[mapDataOverlapIdx]))4557                ? builder.getPtrTy()4558                : mapData.BaseType[mapDataOverlapIdx],4559            mapData.BasePointers[mapDataOverlapIdx], 1);4560      }4561 4562      combinedInfo.Types.emplace_back(mapFlag);4563      combinedInfo.DevicePointers.emplace_back(4564          mapData.DevicePointers[mapDataIndex]);4565      combinedInfo.Names.emplace_back(LLVM::createMappingInformation(4566          mapData.MapClause[mapDataIndex]->getLoc(), ompBuilder));4567      combinedInfo.BasePointers.emplace_back(4568          mapData.BasePointers[mapDataIndex]);4569      combinedInfo.Mappers.emplace_back(mapData.Mappers[mapDataIndex]);4570      combinedInfo.Pointers.emplace_back(lowAddr);4571      combinedInfo.Sizes.emplace_back(builder.CreateIntCast(4572          builder.CreatePtrDiff(builder.getInt8Ty(), highAddr, lowAddr),4573          builder.getInt64Ty(), true));4574    }4575  }4576  return memberOfFlag;4577}4578 4579// This function is intended to add explicit mappings of members4580static void processMapMembersWithParent(4581    LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder,4582    llvm::OpenMPIRBuilder &ompBuilder, DataLayout &dl, MapInfosTy &combinedInfo,4583    MapInfoData &mapData, uint64_t mapDataIndex,4584    llvm::omp::OpenMPOffloadMappingFlags memberOfFlag,4585    TargetDirectiveEnumTy targetDirective) {4586  assert(!ompBuilder.Config.isTargetDevice() &&4587         "function only supported for host device codegen");4588 4589  auto parentClause =4590      llvm::cast<omp::MapInfoOp>(mapData.MapClause[mapDataIndex]);4591 4592  for (auto mappedMembers : parentClause.getMembers()) {4593    auto memberClause =4594        llvm::cast<omp::MapInfoOp>(mappedMembers.getDefiningOp());4595    int memberDataIdx = getMapDataMemberIdx(mapData, memberClause);4596 4597    assert(memberDataIdx >= 0 && "could not find mapped member of structure");4598 4599    // If we're currently mapping a pointer to a block of data, we must4600    // initially map the pointer, and then attatch/bind the data with a4601    // subsequent map to the pointer. This segment of code generates the4602    // pointer mapping, which can in certain cases be optimised out as Clang4603    // currently does in its lowering. However, for the moment we do not do so,4604    // in part as we currently have substantially less information on the data4605    // being mapped at this stage.4606    if (checkIfPointerMap(memberClause)) {4607      auto mapFlag = convertClauseMapFlags(memberClause.getMapType());4608      mapFlag &= ~llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;4609      mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;4610      ompBuilder.setCorrectMemberOfFlag(mapFlag, memberOfFlag);4611      combinedInfo.Types.emplace_back(mapFlag);4612      combinedInfo.DevicePointers.emplace_back(4613          llvm::OpenMPIRBuilder::DeviceInfoTy::None);4614      combinedInfo.Mappers.emplace_back(nullptr);4615      combinedInfo.Names.emplace_back(4616          LLVM::createMappingInformation(memberClause.getLoc(), ompBuilder));4617      combinedInfo.BasePointers.emplace_back(4618          mapData.BasePointers[mapDataIndex]);4619      combinedInfo.Pointers.emplace_back(mapData.BasePointers[memberDataIdx]);4620      combinedInfo.Sizes.emplace_back(builder.getInt64(4621          moduleTranslation.getLLVMModule()->getDataLayout().getPointerSize()));4622    }4623 4624    // Same MemberOfFlag to indicate its link with parent and other members4625    // of.4626    auto mapFlag = convertClauseMapFlags(memberClause.getMapType());4627    mapFlag &= ~llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;4628    mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;4629    ompBuilder.setCorrectMemberOfFlag(mapFlag, memberOfFlag);4630    bool isDeclTargetTo = isDeclareTargetTo(parentClause.getVarPtr()4631                                                ? parentClause.getVarPtr()4632                                                : parentClause.getVarPtrPtr());4633    if (checkIfPointerMap(memberClause) &&4634        (!isDeclTargetTo ||4635         (targetDirective != TargetDirectiveEnumTy::TargetUpdate &&4636          targetDirective != TargetDirectiveEnumTy::TargetData))) {4637      mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;4638    }4639 4640    combinedInfo.Types.emplace_back(mapFlag);4641    combinedInfo.DevicePointers.emplace_back(4642        mapData.DevicePointers[memberDataIdx]);4643    combinedInfo.Mappers.emplace_back(mapData.Mappers[memberDataIdx]);4644    combinedInfo.Names.emplace_back(4645        LLVM::createMappingInformation(memberClause.getLoc(), ompBuilder));4646    uint64_t basePointerIndex =4647        checkIfPointerMap(memberClause) ? memberDataIdx : mapDataIndex;4648    combinedInfo.BasePointers.emplace_back(4649        mapData.BasePointers[basePointerIndex]);4650    combinedInfo.Pointers.emplace_back(mapData.Pointers[memberDataIdx]);4651 4652    llvm::Value *size = mapData.Sizes[memberDataIdx];4653    if (checkIfPointerMap(memberClause)) {4654      size = builder.CreateSelect(4655          builder.CreateIsNull(mapData.Pointers[memberDataIdx]),4656          builder.getInt64(0), size);4657    }4658 4659    combinedInfo.Sizes.emplace_back(size);4660  }4661}4662 4663static void processIndividualMap(MapInfoData &mapData, size_t mapDataIdx,4664                                 MapInfosTy &combinedInfo,4665                                 TargetDirectiveEnumTy targetDirective,4666                                 int mapDataParentIdx = -1) {4667  // Declare Target Mappings are excluded from being marked as4668  // OMP_MAP_TARGET_PARAM as they are not passed as parameters, they're4669  // marked with OMP_MAP_PTR_AND_OBJ instead.4670  auto mapFlag = mapData.Types[mapDataIdx];4671  auto mapInfoOp = llvm::cast<omp::MapInfoOp>(mapData.MapClause[mapDataIdx]);4672 4673  bool isPtrTy = checkIfPointerMap(mapInfoOp);4674  if (isPtrTy)4675    mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;4676 4677  if (targetDirective == TargetDirectiveEnumTy::Target &&4678      !mapData.IsDeclareTarget[mapDataIdx])4679    mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;4680 4681  if (mapInfoOp.getMapCaptureType() == omp::VariableCaptureKind::ByCopy &&4682      !isPtrTy)4683    mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL;4684 4685  // if we're provided a mapDataParentIdx, then the data being mapped is4686  // part of a larger object (in a parent <-> member mapping) and in this4687  // case our BasePointer should be the parent.4688  if (mapDataParentIdx >= 0)4689    combinedInfo.BasePointers.emplace_back(4690        mapData.BasePointers[mapDataParentIdx]);4691  else4692    combinedInfo.BasePointers.emplace_back(mapData.BasePointers[mapDataIdx]);4693 4694  combinedInfo.Pointers.emplace_back(mapData.Pointers[mapDataIdx]);4695  combinedInfo.DevicePointers.emplace_back(mapData.DevicePointers[mapDataIdx]);4696  combinedInfo.Mappers.emplace_back(mapData.Mappers[mapDataIdx]);4697  combinedInfo.Names.emplace_back(mapData.Names[mapDataIdx]);4698  combinedInfo.Types.emplace_back(mapFlag);4699  combinedInfo.Sizes.emplace_back(mapData.Sizes[mapDataIdx]);4700}4701 4702static void processMapWithMembersOf(LLVM::ModuleTranslation &moduleTranslation,4703                                    llvm::IRBuilderBase &builder,4704                                    llvm::OpenMPIRBuilder &ompBuilder,4705                                    DataLayout &dl, MapInfosTy &combinedInfo,4706                                    MapInfoData &mapData, uint64_t mapDataIndex,4707                                    TargetDirectiveEnumTy targetDirective) {4708  assert(!ompBuilder.Config.isTargetDevice() &&4709         "function only supported for host device codegen");4710 4711  auto parentClause =4712      llvm::cast<omp::MapInfoOp>(mapData.MapClause[mapDataIndex]);4713 4714  // If we have a partial map (no parent referenced in the map clauses of the4715  // directive, only members) and only a single member, we do not need to bind4716  // the map of the member to the parent, we can pass the member separately.4717  if (parentClause.getMembers().size() == 1 && parentClause.getPartialMap()) {4718    auto memberClause = llvm::cast<omp::MapInfoOp>(4719        parentClause.getMembers()[0].getDefiningOp());4720    int memberDataIdx = getMapDataMemberIdx(mapData, memberClause);4721    // Note: Clang treats arrays with explicit bounds that fall into this4722    // category as a parent with map case, however, it seems this isn't a4723    // requirement, and processing them as an individual map is fine. So,4724    // we will handle them as individual maps for the moment, as it's4725    // difficult for us to check this as we always require bounds to be4726    // specified currently and it's also marginally more optimal (single4727    // map rather than two). The difference may come from the fact that4728    // Clang maps array without bounds as pointers (which we do not4729    // currently do), whereas we treat them as arrays in all cases4730    // currently.4731    processIndividualMap(mapData, memberDataIdx, combinedInfo, targetDirective,4732                         mapDataIndex);4733    return;4734  }4735 4736  llvm::omp::OpenMPOffloadMappingFlags memberOfParentFlag =4737      mapParentWithMembers(moduleTranslation, builder, ompBuilder, dl,4738                           combinedInfo, mapData, mapDataIndex,4739                           targetDirective);4740  processMapMembersWithParent(moduleTranslation, builder, ompBuilder, dl,4741                              combinedInfo, mapData, mapDataIndex,4742                              memberOfParentFlag, targetDirective);4743}4744 4745// This is a variation on Clang's GenerateOpenMPCapturedVars, which4746// generates different operation (e.g. load/store) combinations for4747// arguments to the kernel, based on map capture kinds which are then4748// utilised in the combinedInfo in place of the original Map value.4749static void4750createAlteredByCaptureMap(MapInfoData &mapData,4751                          LLVM::ModuleTranslation &moduleTranslation,4752                          llvm::IRBuilderBase &builder) {4753  assert(!moduleTranslation.getOpenMPBuilder()->Config.isTargetDevice() &&4754         "function only supported for host device codegen");4755  for (size_t i = 0; i < mapData.MapClause.size(); ++i) {4756    // if it's declare target, skip it, it's handled separately.4757    if (!mapData.IsDeclareTarget[i]) {4758      auto mapOp = cast<omp::MapInfoOp>(mapData.MapClause[i]);4759      omp::VariableCaptureKind captureKind = mapOp.getMapCaptureType();4760      bool isPtrTy = checkIfPointerMap(mapOp);4761 4762      // Currently handles array sectioning lowerbound case, but more4763      // logic may be required in the future. Clang invokes EmitLValue,4764      // which has specialised logic for special Clang types such as user4765      // defines, so it is possible we will have to extend this for4766      // structures or other complex types. As the general idea is that this4767      // function mimics some of the logic from Clang that we require for4768      // kernel argument passing from host -> device.4769      switch (captureKind) {4770      case omp::VariableCaptureKind::ByRef: {4771        llvm::Value *newV = mapData.Pointers[i];4772        std::vector<llvm::Value *> offsetIdx = calculateBoundsOffset(4773            moduleTranslation, builder, mapData.BaseType[i]->isArrayTy(),4774            mapOp.getBounds());4775        if (isPtrTy)4776          newV = builder.CreateLoad(builder.getPtrTy(), newV);4777 4778        if (!offsetIdx.empty())4779          newV = builder.CreateInBoundsGEP(mapData.BaseType[i], newV, offsetIdx,4780                                           "array_offset");4781        mapData.Pointers[i] = newV;4782      } break;4783      case omp::VariableCaptureKind::ByCopy: {4784        llvm::Type *type = mapData.BaseType[i];4785        llvm::Value *newV;4786        if (mapData.Pointers[i]->getType()->isPointerTy())4787          newV = builder.CreateLoad(type, mapData.Pointers[i]);4788        else4789          newV = mapData.Pointers[i];4790 4791        if (!isPtrTy) {4792          auto curInsert = builder.saveIP();4793          llvm::DebugLoc DbgLoc = builder.getCurrentDebugLocation();4794          builder.restoreIP(findAllocaInsertPoint(builder, moduleTranslation));4795          auto *memTempAlloc =4796              builder.CreateAlloca(builder.getPtrTy(), nullptr, ".casted");4797          builder.SetCurrentDebugLocation(DbgLoc);4798          builder.restoreIP(curInsert);4799 4800          builder.CreateStore(newV, memTempAlloc);4801          newV = builder.CreateLoad(builder.getPtrTy(), memTempAlloc);4802        }4803 4804        mapData.Pointers[i] = newV;4805        mapData.BasePointers[i] = newV;4806      } break;4807      case omp::VariableCaptureKind::This:4808      case omp::VariableCaptureKind::VLAType:4809        mapData.MapClause[i]->emitOpError("Unhandled capture kind");4810        break;4811      }4812    }4813  }4814}4815 4816// Generate all map related information and fill the combinedInfo.4817static void genMapInfos(llvm::IRBuilderBase &builder,4818                        LLVM::ModuleTranslation &moduleTranslation,4819                        DataLayout &dl, MapInfosTy &combinedInfo,4820                        MapInfoData &mapData,4821                        TargetDirectiveEnumTy targetDirective) {4822  assert(!moduleTranslation.getOpenMPBuilder()->Config.isTargetDevice() &&4823         "function only supported for host device codegen");4824  // We wish to modify some of the methods in which arguments are4825  // passed based on their capture type by the target region, this can4826  // involve generating new loads and stores, which changes the4827  // MLIR value to LLVM value mapping, however, we only wish to do this4828  // locally for the current function/target and also avoid altering4829  // ModuleTranslation, so we remap the base pointer or pointer stored4830  // in the map infos corresponding MapInfoData, which is later accessed4831  // by genMapInfos and createTarget to help generate the kernel and4832  // kernel arg structure. It primarily becomes relevant in cases like4833  // bycopy, or byref range'd arrays. In the default case, we simply4834  // pass thee pointer byref as both basePointer and pointer.4835  createAlteredByCaptureMap(mapData, moduleTranslation, builder);4836 4837  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();4838 4839  // We operate under the assumption that all vectors that are4840  // required in MapInfoData are of equal lengths (either filled with4841  // default constructed data or appropiate information) so we can4842  // utilise the size from any component of MapInfoData, if we can't4843  // something is missing from the initial MapInfoData construction.4844  for (size_t i = 0; i < mapData.MapClause.size(); ++i) {4845    // NOTE/TODO: We currently do not support arbitrary depth record4846    // type mapping.4847    if (mapData.IsAMember[i])4848      continue;4849 4850    auto mapInfoOp = dyn_cast<omp::MapInfoOp>(mapData.MapClause[i]);4851    if (!mapInfoOp.getMembers().empty()) {4852      processMapWithMembersOf(moduleTranslation, builder, *ompBuilder, dl,4853                              combinedInfo, mapData, i, targetDirective);4854      continue;4855    }4856 4857    processIndividualMap(mapData, i, combinedInfo, targetDirective);4858  }4859}4860 4861static llvm::Expected<llvm::Function *>4862emitUserDefinedMapper(Operation *declMapperOp, llvm::IRBuilderBase &builder,4863                      LLVM::ModuleTranslation &moduleTranslation,4864                      llvm::StringRef mapperFuncName,4865                      TargetDirectiveEnumTy targetDirective);4866 4867static llvm::Expected<llvm::Function *>4868getOrCreateUserDefinedMapperFunc(Operation *op, llvm::IRBuilderBase &builder,4869                                 LLVM::ModuleTranslation &moduleTranslation,4870                                 TargetDirectiveEnumTy targetDirective) {4871  assert(!moduleTranslation.getOpenMPBuilder()->Config.isTargetDevice() &&4872         "function only supported for host device codegen");4873  auto declMapperOp = cast<omp::DeclareMapperOp>(op);4874  std::string mapperFuncName =4875      moduleTranslation.getOpenMPBuilder()->createPlatformSpecificName(4876          {"omp_mapper", declMapperOp.getSymName()});4877 4878  if (auto *lookupFunc = moduleTranslation.lookupFunction(mapperFuncName))4879    return lookupFunc;4880 4881  return emitUserDefinedMapper(declMapperOp, builder, moduleTranslation,4882                               mapperFuncName, targetDirective);4883}4884 4885static llvm::Expected<llvm::Function *>4886emitUserDefinedMapper(Operation *op, llvm::IRBuilderBase &builder,4887                      LLVM::ModuleTranslation &moduleTranslation,4888                      llvm::StringRef mapperFuncName,4889                      TargetDirectiveEnumTy targetDirective) {4890  assert(!moduleTranslation.getOpenMPBuilder()->Config.isTargetDevice() &&4891         "function only supported for host device codegen");4892  auto declMapperOp = cast<omp::DeclareMapperOp>(op);4893  auto declMapperInfoOp = declMapperOp.getDeclareMapperInfo();4894  DataLayout dl = DataLayout(declMapperOp->getParentOfType<ModuleOp>());4895  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();4896  llvm::Type *varType = moduleTranslation.convertType(declMapperOp.getType());4897  SmallVector<Value> mapVars = declMapperInfoOp.getMapVars();4898 4899  using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;4900 4901  // Fill up the arrays with all the mapped variables.4902  MapInfosTy combinedInfo;4903  auto genMapInfoCB =4904      [&](InsertPointTy codeGenIP, llvm::Value *ptrPHI,4905          llvm::Value *unused2) -> llvm::OpenMPIRBuilder::MapInfosOrErrorTy {4906    builder.restoreIP(codeGenIP);4907    moduleTranslation.mapValue(declMapperOp.getSymVal(), ptrPHI);4908    moduleTranslation.mapBlock(&declMapperOp.getRegion().front(),4909                               builder.GetInsertBlock());4910    if (failed(moduleTranslation.convertBlock(declMapperOp.getRegion().front(),4911                                              /*ignoreArguments=*/true,4912                                              builder)))4913      return llvm::make_error<PreviouslyReportedError>();4914    MapInfoData mapData;4915    collectMapDataFromMapOperands(mapData, mapVars, moduleTranslation, dl,4916                                  builder);4917    genMapInfos(builder, moduleTranslation, dl, combinedInfo, mapData,4918                targetDirective);4919 4920    // Drop the mapping that is no longer necessary so that the same region4921    // can be processed multiple times.4922    moduleTranslation.forgetMapping(declMapperOp.getRegion());4923    return combinedInfo;4924  };4925 4926  auto customMapperCB = [&](unsigned i) -> llvm::Expected<llvm::Function *> {4927    if (!combinedInfo.Mappers[i])4928      return nullptr;4929    return getOrCreateUserDefinedMapperFunc(combinedInfo.Mappers[i], builder,4930                                            moduleTranslation, targetDirective);4931  };4932 4933  llvm::Expected<llvm::Function *> newFn = ompBuilder->emitUserDefinedMapper(4934      genMapInfoCB, varType, mapperFuncName, customMapperCB);4935  if (!newFn)4936    return newFn.takeError();4937  moduleTranslation.mapFunction(mapperFuncName, *newFn);4938  return *newFn;4939}4940 4941static LogicalResult4942convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,4943                     LLVM::ModuleTranslation &moduleTranslation) {4944  llvm::Value *ifCond = nullptr;4945  int64_t deviceID = llvm::omp::OMP_DEVICEID_UNDEF;4946  SmallVector<Value> mapVars;4947  SmallVector<Value> useDevicePtrVars;4948  SmallVector<Value> useDeviceAddrVars;4949  llvm::omp::RuntimeFunction RTLFn;4950  DataLayout DL = DataLayout(op->getParentOfType<ModuleOp>());4951  TargetDirectiveEnumTy targetDirective = getTargetDirectiveEnumTyFromOp(op);4952 4953  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();4954  llvm::OpenMPIRBuilder::TargetDataInfo info(4955      /*RequiresDevicePointerInfo=*/true,4956      /*SeparateBeginEndCalls=*/true);4957  bool isTargetDevice = ompBuilder->Config.isTargetDevice();4958  bool isOffloadEntry =4959      isTargetDevice || !ompBuilder->Config.TargetTriples.empty();4960 4961  LogicalResult result =4962      llvm::TypeSwitch<Operation *, LogicalResult>(op)4963          .Case([&](omp::TargetDataOp dataOp) {4964            if (failed(checkImplementationStatus(*dataOp)))4965              return failure();4966 4967            if (auto ifVar = dataOp.getIfExpr())4968              ifCond = moduleTranslation.lookupValue(ifVar);4969 4970            if (auto devId = dataOp.getDevice())4971              if (auto constOp = devId.getDefiningOp<LLVM::ConstantOp>())4972                if (auto intAttr = dyn_cast<IntegerAttr>(constOp.getValue()))4973                  deviceID = intAttr.getInt();4974 4975            mapVars = dataOp.getMapVars();4976            useDevicePtrVars = dataOp.getUseDevicePtrVars();4977            useDeviceAddrVars = dataOp.getUseDeviceAddrVars();4978            return success();4979          })4980          .Case([&](omp::TargetEnterDataOp enterDataOp) -> LogicalResult {4981            if (failed(checkImplementationStatus(*enterDataOp)))4982              return failure();4983 4984            if (auto ifVar = enterDataOp.getIfExpr())4985              ifCond = moduleTranslation.lookupValue(ifVar);4986 4987            if (auto devId = enterDataOp.getDevice())4988              if (auto constOp = devId.getDefiningOp<LLVM::ConstantOp>())4989                if (auto intAttr = dyn_cast<IntegerAttr>(constOp.getValue()))4990                  deviceID = intAttr.getInt();4991            RTLFn =4992                enterDataOp.getNowait()4993                    ? llvm::omp::OMPRTL___tgt_target_data_begin_nowait_mapper4994                    : llvm::omp::OMPRTL___tgt_target_data_begin_mapper;4995            mapVars = enterDataOp.getMapVars();4996            info.HasNoWait = enterDataOp.getNowait();4997            return success();4998          })4999          .Case([&](omp::TargetExitDataOp exitDataOp) -> LogicalResult {5000            if (failed(checkImplementationStatus(*exitDataOp)))5001              return failure();5002 5003            if (auto ifVar = exitDataOp.getIfExpr())5004              ifCond = moduleTranslation.lookupValue(ifVar);5005 5006            if (auto devId = exitDataOp.getDevice())5007              if (auto constOp = devId.getDefiningOp<LLVM::ConstantOp>())5008                if (auto intAttr = dyn_cast<IntegerAttr>(constOp.getValue()))5009                  deviceID = intAttr.getInt();5010 5011            RTLFn = exitDataOp.getNowait()5012                        ? llvm::omp::OMPRTL___tgt_target_data_end_nowait_mapper5013                        : llvm::omp::OMPRTL___tgt_target_data_end_mapper;5014            mapVars = exitDataOp.getMapVars();5015            info.HasNoWait = exitDataOp.getNowait();5016            return success();5017          })5018          .Case([&](omp::TargetUpdateOp updateDataOp) -> LogicalResult {5019            if (failed(checkImplementationStatus(*updateDataOp)))5020              return failure();5021 5022            if (auto ifVar = updateDataOp.getIfExpr())5023              ifCond = moduleTranslation.lookupValue(ifVar);5024 5025            if (auto devId = updateDataOp.getDevice())5026              if (auto constOp = devId.getDefiningOp<LLVM::ConstantOp>())5027                if (auto intAttr = dyn_cast<IntegerAttr>(constOp.getValue()))5028                  deviceID = intAttr.getInt();5029 5030            RTLFn =5031                updateDataOp.getNowait()5032                    ? llvm::omp::OMPRTL___tgt_target_data_update_nowait_mapper5033                    : llvm::omp::OMPRTL___tgt_target_data_update_mapper;5034            mapVars = updateDataOp.getMapVars();5035            info.HasNoWait = updateDataOp.getNowait();5036            return success();5037          })5038          .DefaultUnreachable("unexpected operation");5039 5040  if (failed(result))5041    return failure();5042  // Pretend we have IF(false) if we're not doing offload.5043  if (!isOffloadEntry)5044    ifCond = builder.getFalse();5045 5046  using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;5047  MapInfoData mapData;5048  collectMapDataFromMapOperands(mapData, mapVars, moduleTranslation, DL,5049                                builder, useDevicePtrVars, useDeviceAddrVars);5050 5051  // Fill up the arrays with all the mapped variables.5052  MapInfosTy combinedInfo;5053  auto genMapInfoCB = [&](InsertPointTy codeGenIP) -> MapInfosTy & {5054    builder.restoreIP(codeGenIP);5055    genMapInfos(builder, moduleTranslation, DL, combinedInfo, mapData,5056                targetDirective);5057    return combinedInfo;5058  };5059 5060  // Define a lambda to apply mappings between use_device_addr and5061  // use_device_ptr base pointers, and their associated block arguments.5062  auto mapUseDevice =5063      [&moduleTranslation](5064          llvm::OpenMPIRBuilder::DeviceInfoTy type,5065          llvm::ArrayRef<BlockArgument> blockArgs,5066          llvm::SmallVectorImpl<Value> &useDeviceVars, MapInfoData &mapInfoData,5067          llvm::function_ref<llvm::Value *(llvm::Value *)> mapper = nullptr) {5068        for (auto [arg, useDevVar] :5069             llvm::zip_equal(blockArgs, useDeviceVars)) {5070 5071          auto getMapBasePtr = [](omp::MapInfoOp mapInfoOp) {5072            return mapInfoOp.getVarPtrPtr() ? mapInfoOp.getVarPtrPtr()5073                                            : mapInfoOp.getVarPtr();5074          };5075 5076          auto useDevMap = cast<omp::MapInfoOp>(useDevVar.getDefiningOp());5077          for (auto [mapClause, devicePointer, basePointer] : llvm::zip_equal(5078                   mapInfoData.MapClause, mapInfoData.DevicePointers,5079                   mapInfoData.BasePointers)) {5080            auto mapOp = cast<omp::MapInfoOp>(mapClause);5081            if (getMapBasePtr(mapOp) != getMapBasePtr(useDevMap) ||5082                devicePointer != type)5083              continue;5084 5085            if (llvm::Value *devPtrInfoMap =5086                    mapper ? mapper(basePointer) : basePointer) {5087              moduleTranslation.mapValue(arg, devPtrInfoMap);5088              break;5089            }5090          }5091        }5092      };5093 5094  using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;5095  auto bodyGenCB = [&](InsertPointTy codeGenIP, BodyGenTy bodyGenType)5096      -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {5097    // We must always restoreIP regardless of doing anything the caller5098    // does not restore it, leading to incorrect (no) branch generation.5099    builder.restoreIP(codeGenIP);5100    assert(isa<omp::TargetDataOp>(op) &&5101           "BodyGen requested for non TargetDataOp");5102    auto blockArgIface = cast<omp::BlockArgOpenMPOpInterface>(op);5103    Region &region = cast<omp::TargetDataOp>(op).getRegion();5104    switch (bodyGenType) {5105    case BodyGenTy::Priv:5106      // Check if any device ptr/addr info is available5107      if (!info.DevicePtrInfoMap.empty()) {5108        mapUseDevice(llvm::OpenMPIRBuilder::DeviceInfoTy::Address,5109                     blockArgIface.getUseDeviceAddrBlockArgs(),5110                     useDeviceAddrVars, mapData,5111                     [&](llvm::Value *basePointer) -> llvm::Value * {5112                       if (!info.DevicePtrInfoMap[basePointer].second)5113                         return nullptr;5114                       return builder.CreateLoad(5115                           builder.getPtrTy(),5116                           info.DevicePtrInfoMap[basePointer].second);5117                     });5118        mapUseDevice(llvm::OpenMPIRBuilder::DeviceInfoTy::Pointer,5119                     blockArgIface.getUseDevicePtrBlockArgs(), useDevicePtrVars,5120                     mapData, [&](llvm::Value *basePointer) {5121                       return info.DevicePtrInfoMap[basePointer].second;5122                     });5123 5124        if (failed(inlineConvertOmpRegions(region, "omp.data.region", builder,5125                                           moduleTranslation)))5126          return llvm::make_error<PreviouslyReportedError>();5127      }5128      break;5129    case BodyGenTy::DupNoPriv:5130      if (info.DevicePtrInfoMap.empty()) {5131        // For host device we still need to do the mapping for codegen,5132        // otherwise it may try to lookup a missing value.5133        if (!ompBuilder->Config.IsTargetDevice.value_or(false)) {5134          mapUseDevice(llvm::OpenMPIRBuilder::DeviceInfoTy::Address,5135                       blockArgIface.getUseDeviceAddrBlockArgs(),5136                       useDeviceAddrVars, mapData);5137          mapUseDevice(llvm::OpenMPIRBuilder::DeviceInfoTy::Pointer,5138                       blockArgIface.getUseDevicePtrBlockArgs(),5139                       useDevicePtrVars, mapData);5140        }5141      }5142      break;5143    case BodyGenTy::NoPriv:5144      // If device info is available then region has already been generated5145      if (info.DevicePtrInfoMap.empty()) {5146        // For device pass, if use_device_ptr(addr) mappings were present,5147        // we need to link them here before codegen.5148        if (ompBuilder->Config.IsTargetDevice.value_or(false)) {5149          mapUseDevice(llvm::OpenMPIRBuilder::DeviceInfoTy::Address,5150                       blockArgIface.getUseDeviceAddrBlockArgs(),5151                       useDeviceAddrVars, mapData);5152          mapUseDevice(llvm::OpenMPIRBuilder::DeviceInfoTy::Pointer,5153                       blockArgIface.getUseDevicePtrBlockArgs(),5154                       useDevicePtrVars, mapData);5155        }5156 5157        if (failed(inlineConvertOmpRegions(region, "omp.data.region", builder,5158                                           moduleTranslation)))5159          return llvm::make_error<PreviouslyReportedError>();5160      }5161      break;5162    }5163    return builder.saveIP();5164  };5165 5166  auto customMapperCB =5167      [&](unsigned int i) -> llvm::Expected<llvm::Function *> {5168    if (!combinedInfo.Mappers[i])5169      return nullptr;5170    info.HasMapper = true;5171    return getOrCreateUserDefinedMapperFunc(combinedInfo.Mappers[i], builder,5172                                            moduleTranslation, targetDirective);5173  };5174 5175  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);5176  llvm::OpenMPIRBuilder::InsertPointTy allocaIP =5177      findAllocaInsertPoint(builder, moduleTranslation);5178  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = [&]() {5179    if (isa<omp::TargetDataOp>(op))5180      return ompBuilder->createTargetData(ompLoc, allocaIP, builder.saveIP(),5181                                          builder.getInt64(deviceID), ifCond,5182                                          info, genMapInfoCB, customMapperCB,5183                                          /*MapperFunc=*/nullptr, bodyGenCB,5184                                          /*DeviceAddrCB=*/nullptr);5185    return ompBuilder->createTargetData(5186        ompLoc, allocaIP, builder.saveIP(), builder.getInt64(deviceID), ifCond,5187        info, genMapInfoCB, customMapperCB, &RTLFn);5188  }();5189 5190  if (failed(handleError(afterIP, *op)))5191    return failure();5192 5193  builder.restoreIP(*afterIP);5194  return success();5195}5196 5197static LogicalResult5198convertOmpDistribute(Operation &opInst, llvm::IRBuilderBase &builder,5199                     LLVM::ModuleTranslation &moduleTranslation) {5200  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();5201  auto distributeOp = cast<omp::DistributeOp>(opInst);5202  if (failed(checkImplementationStatus(opInst)))5203    return failure();5204 5205  /// Process teams op reduction in distribute if the reduction is contained in5206  /// the distribute op.5207  omp::TeamsOp teamsOp = opInst.getParentOfType<omp::TeamsOp>();5208  bool doDistributeReduction =5209      teamsOp ? teamsReductionContainedInDistribute(teamsOp) : false;5210 5211  DenseMap<Value, llvm::Value *> reductionVariableMap;5212  unsigned numReductionVars = teamsOp ? teamsOp.getNumReductionVars() : 0;5213  SmallVector<omp::DeclareReductionOp> reductionDecls;5214  SmallVector<llvm::Value *> privateReductionVariables(numReductionVars);5215  llvm::ArrayRef<bool> isByRef;5216 5217  if (doDistributeReduction) {5218    isByRef = getIsByRef(teamsOp.getReductionByref());5219    assert(isByRef.size() == teamsOp.getNumReductionVars());5220 5221    collectReductionDecls(teamsOp, reductionDecls);5222    llvm::OpenMPIRBuilder::InsertPointTy allocaIP =5223        findAllocaInsertPoint(builder, moduleTranslation);5224 5225    MutableArrayRef<BlockArgument> reductionArgs =5226        llvm::cast<omp::BlockArgOpenMPOpInterface>(*teamsOp)5227            .getReductionBlockArgs();5228 5229    if (failed(allocAndInitializeReductionVars(5230            teamsOp, reductionArgs, builder, moduleTranslation, allocaIP,5231            reductionDecls, privateReductionVariables, reductionVariableMap,5232            isByRef)))5233      return failure();5234  }5235 5236  using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;5237  auto bodyGenCB = [&](InsertPointTy allocaIP,5238                       InsertPointTy codeGenIP) -> llvm::Error {5239    // Save the alloca insertion point on ModuleTranslation stack for use in5240    // nested regions.5241    LLVM::ModuleTranslation::SaveStack<OpenMPAllocaStackFrame> frame(5242        moduleTranslation, allocaIP);5243 5244    // DistributeOp has only one region associated with it.5245    builder.restoreIP(codeGenIP);5246    PrivateVarsInfo privVarsInfo(distributeOp);5247 5248    llvm::Expected<llvm::BasicBlock *> afterAllocas =5249        allocatePrivateVars(builder, moduleTranslation, privVarsInfo, allocaIP);5250    if (handleError(afterAllocas, opInst).failed())5251      return llvm::make_error<PreviouslyReportedError>();5252 5253    if (handleError(initPrivateVars(builder, moduleTranslation, privVarsInfo),5254                    opInst)5255            .failed())5256      return llvm::make_error<PreviouslyReportedError>();5257 5258    if (failed(copyFirstPrivateVars(5259            distributeOp, builder, moduleTranslation, privVarsInfo.mlirVars,5260            privVarsInfo.llvmVars, privVarsInfo.privatizers,5261            distributeOp.getPrivateNeedsBarrier())))5262      return llvm::make_error<PreviouslyReportedError>();5263 5264    llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();5265    llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);5266    llvm::Expected<llvm::BasicBlock *> regionBlock =5267        convertOmpOpRegions(distributeOp.getRegion(), "omp.distribute.region",5268                            builder, moduleTranslation);5269    if (!regionBlock)5270      return regionBlock.takeError();5271    builder.SetInsertPoint(*regionBlock, (*regionBlock)->begin());5272 5273    // Skip applying a workshare loop below when translating 'distribute5274    // parallel do' (it's been already handled by this point while translating5275    // the nested omp.wsloop).5276    if (!isa_and_present<omp::WsloopOp>(distributeOp.getNestedWrapper())) {5277      // TODO: Add support for clauses which are valid for DISTRIBUTE5278      // constructs. Static schedule is the default.5279      bool hasDistSchedule = distributeOp.getDistScheduleStatic();5280      auto schedule = hasDistSchedule ? omp::ClauseScheduleKind::Distribute5281                                      : omp::ClauseScheduleKind::Static;5282      // dist_schedule clauses are ordered - otherise this should be false5283      bool isOrdered = hasDistSchedule;5284      std::optional<omp::ScheduleModifier> scheduleMod;5285      bool isSimd = false;5286      llvm::omp::WorksharingLoopType workshareLoopType =5287          llvm::omp::WorksharingLoopType::DistributeStaticLoop;5288      bool loopNeedsBarrier = false;5289      llvm::Value *chunk = moduleTranslation.lookupValue(5290          distributeOp.getDistScheduleChunkSize());5291      llvm::CanonicalLoopInfo *loopInfo =5292          findCurrentLoopInfo(moduleTranslation);5293      llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =5294          ompBuilder->applyWorkshareLoop(5295              ompLoc.DL, loopInfo, allocaIP, loopNeedsBarrier,5296              convertToScheduleKind(schedule), chunk, isSimd,5297              scheduleMod == omp::ScheduleModifier::monotonic,5298              scheduleMod == omp::ScheduleModifier::nonmonotonic, isOrdered,5299              workshareLoopType, false, hasDistSchedule, chunk);5300 5301      if (!wsloopIP)5302        return wsloopIP.takeError();5303    }5304    if (failed(cleanupPrivateVars(builder, moduleTranslation,5305                                  distributeOp.getLoc(), privVarsInfo.llvmVars,5306                                  privVarsInfo.privatizers)))5307      return llvm::make_error<PreviouslyReportedError>();5308 5309    return llvm::Error::success();5310  };5311 5312  llvm::OpenMPIRBuilder::InsertPointTy allocaIP =5313      findAllocaInsertPoint(builder, moduleTranslation);5314  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);5315  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =5316      ompBuilder->createDistribute(ompLoc, allocaIP, bodyGenCB);5317 5318  if (failed(handleError(afterIP, opInst)))5319    return failure();5320 5321  builder.restoreIP(*afterIP);5322 5323  if (doDistributeReduction) {5324    // Process the reductions if required.5325    return createReductionsAndCleanup(5326        teamsOp, builder, moduleTranslation, allocaIP, reductionDecls,5327        privateReductionVariables, isByRef,5328        /*isNoWait*/ false, /*isTeamsReduction*/ true);5329  }5330  return success();5331}5332 5333/// Lowers the FlagsAttr which is applied to the module on the device5334/// pass when offloading, this attribute contains OpenMP RTL globals that can5335/// be passed as flags to the frontend, otherwise they are set to default5336static LogicalResult5337convertFlagsAttr(Operation *op, mlir::omp::FlagsAttr attribute,5338                 LLVM::ModuleTranslation &moduleTranslation) {5339  if (!cast<mlir::ModuleOp>(op))5340    return failure();5341 5342  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();5343 5344  ompBuilder->M.addModuleFlag(llvm::Module::Max, "openmp-device",5345                              attribute.getOpenmpDeviceVersion());5346 5347  if (attribute.getNoGpuLib())5348    return success();5349 5350  ompBuilder->createGlobalFlag(5351      attribute.getDebugKind() /*LangOpts().OpenMPTargetDebug*/,5352      "__omp_rtl_debug_kind");5353  ompBuilder->createGlobalFlag(5354      attribute5355          .getAssumeTeamsOversubscription() /*LangOpts().OpenMPTeamSubscription*/5356      ,5357      "__omp_rtl_assume_teams_oversubscription");5358  ompBuilder->createGlobalFlag(5359      attribute5360          .getAssumeThreadsOversubscription() /*LangOpts().OpenMPThreadSubscription*/5361      ,5362      "__omp_rtl_assume_threads_oversubscription");5363  ompBuilder->createGlobalFlag(5364      attribute.getAssumeNoThreadState() /*LangOpts().OpenMPNoThreadState*/,5365      "__omp_rtl_assume_no_thread_state");5366  ompBuilder->createGlobalFlag(5367      attribute5368          .getAssumeNoNestedParallelism() /*LangOpts().OpenMPNoNestedParallelism*/5369      ,5370      "__omp_rtl_assume_no_nested_parallelism");5371  return success();5372}5373 5374static void getTargetEntryUniqueInfo(llvm::TargetRegionEntryInfo &targetInfo,5375                                     omp::TargetOp targetOp,5376                                     llvm::StringRef parentName = "") {5377  auto fileLoc = targetOp.getLoc()->findInstanceOf<FileLineColLoc>();5378 5379  assert(fileLoc && "No file found from location");5380  StringRef fileName = fileLoc.getFilename().getValue();5381 5382  llvm::sys::fs::UniqueID id;5383  uint64_t line = fileLoc.getLine();5384  if (auto ec = llvm::sys::fs::getUniqueID(fileName, id)) {5385    size_t fileHash = llvm::hash_value(fileName.str());5386    size_t deviceId = 0xdeadf17e;5387    targetInfo =5388        llvm::TargetRegionEntryInfo(parentName, deviceId, fileHash, line);5389  } else {5390    targetInfo = llvm::TargetRegionEntryInfo(parentName, id.getDevice(),5391                                             id.getFile(), line);5392  }5393}5394 5395static void5396handleDeclareTargetMapVar(MapInfoData &mapData,5397                          LLVM::ModuleTranslation &moduleTranslation,5398                          llvm::IRBuilderBase &builder, llvm::Function *func) {5399  assert(moduleTranslation.getOpenMPBuilder()->Config.isTargetDevice() &&5400         "function only supported for target device codegen");5401  llvm::IRBuilderBase::InsertPointGuard guard(builder);5402  for (size_t i = 0; i < mapData.MapClause.size(); ++i) {5403    // In the case of declare target mapped variables, the basePointer is5404    // the reference pointer generated by the convertDeclareTargetAttr5405    // method. Whereas the kernelValue is the original variable, so for5406    // the device we must replace all uses of this original global variable5407    // (stored in kernelValue) with the reference pointer (stored in5408    // basePointer for declare target mapped variables), as for device the5409    // data is mapped into this reference pointer and should be loaded5410    // from it, the original variable is discarded. On host both exist and5411    // metadata is generated (elsewhere in the convertDeclareTargetAttr)5412    // function to link the two variables in the runtime and then both the5413    // reference pointer and the pointer are assigned in the kernel argument5414    // structure for the host.5415    if (mapData.IsDeclareTarget[i]) {5416      // If the original map value is a constant, then we have to make sure all5417      // of it's uses within the current kernel/function that we are going to5418      // rewrite are converted to instructions, as we will be altering the old5419      // use (OriginalValue) from a constant to an instruction, which will be5420      // illegal and ICE the compiler if the user is a constant expression of5421      // some kind e.g. a constant GEP.5422      if (auto *constant = dyn_cast<llvm::Constant>(mapData.OriginalValue[i]))5423        convertUsersOfConstantsToInstructions(constant, func, false);5424 5425      // The users iterator will get invalidated if we modify an element,5426      // so we populate this vector of uses to alter each user on an5427      // individual basis to emit its own load (rather than one load for5428      // all).5429      llvm::SmallVector<llvm::User *> userVec;5430      for (llvm::User *user : mapData.OriginalValue[i]->users())5431        userVec.push_back(user);5432 5433      for (llvm::User *user : userVec) {5434        if (auto *insn = dyn_cast<llvm::Instruction>(user)) {5435          if (insn->getFunction() == func) {5436            auto mapOp = cast<omp::MapInfoOp>(mapData.MapClause[i]);5437            llvm::Value *substitute = mapData.BasePointers[i];5438            if (isDeclareTargetLink(mapOp.getVarPtrPtr() ? mapOp.getVarPtrPtr()5439                                                         : mapOp.getVarPtr())) {5440              builder.SetCurrentDebugLocation(insn->getDebugLoc());5441              substitute = builder.CreateLoad(5442                  mapData.BasePointers[i]->getType(), mapData.BasePointers[i]);5443              cast<llvm::LoadInst>(substitute)->moveBefore(insn->getIterator());5444            }5445            user->replaceUsesOfWith(mapData.OriginalValue[i], substitute);5446          }5447        }5448      }5449    }5450  }5451}5452 5453// The createDeviceArgumentAccessor function generates5454// instructions for retrieving (acessing) kernel5455// arguments inside of the device kernel for use by5456// the kernel. This enables different semantics such as5457// the creation of temporary copies of data allowing5458// semantics like read-only/no host write back kernel5459// arguments.5460//5461// This currently implements a very light version of Clang's5462// EmitParmDecl's handling of direct argument handling as well5463// as a portion of the argument access generation based on5464// capture types found at the end of emitOutlinedFunctionPrologue5465// in Clang. The indirect path handling of EmitParmDecl's may be5466// required for future work, but a direct 1-to-1 copy doesn't seem5467// possible as the logic is rather scattered throughout Clang's5468// lowering and perhaps we wish to deviate slightly.5469//5470// \param mapData - A container containing vectors of information5471// corresponding to the input argument, which should have a5472// corresponding entry in the MapInfoData containers5473// OrigialValue's.5474// \param arg - This is the generated kernel function argument that5475// corresponds to the passed in input argument. We generated different5476// accesses of this Argument, based on capture type and other Input5477// related information.5478// \param input - This is the host side value that will be passed to5479// the kernel i.e. the kernel input, we rewrite all uses of this within5480// the kernel (as we generate the kernel body based on the target's region5481// which maintians references to the original input) to the retVal argument5482// apon exit of this function inside of the OMPIRBuilder. This interlinks5483// the kernel argument to future uses of it in the function providing5484// appropriate "glue" instructions inbetween.5485// \param retVal - This is the value that all uses of input inside of the5486// kernel will be re-written to, the goal of this function is to generate5487// an appropriate location for the kernel argument to be accessed from,5488// e.g. ByRef will result in a temporary allocation location and then5489// a store of the kernel argument into this allocated memory which5490// will then be loaded from, ByCopy will use the allocated memory5491// directly.5492static llvm::IRBuilderBase::InsertPoint5493createDeviceArgumentAccessor(MapInfoData &mapData, llvm::Argument &arg,5494                             llvm::Value *input, llvm::Value *&retVal,5495                             llvm::IRBuilderBase &builder,5496                             llvm::OpenMPIRBuilder &ompBuilder,5497                             LLVM::ModuleTranslation &moduleTranslation,5498                             llvm::IRBuilderBase::InsertPoint allocaIP,5499                             llvm::IRBuilderBase::InsertPoint codeGenIP) {5500  assert(ompBuilder.Config.isTargetDevice() &&5501         "function only supported for target device codegen");5502  builder.restoreIP(allocaIP);5503 5504  omp::VariableCaptureKind capture = omp::VariableCaptureKind::ByRef;5505  LLVM::TypeToLLVMIRTranslator typeToLLVMIRTranslator(5506      ompBuilder.M.getContext());5507  unsigned alignmentValue = 0;5508  // Find the associated MapInfoData entry for the current input5509  for (size_t i = 0; i < mapData.MapClause.size(); ++i)5510    if (mapData.OriginalValue[i] == input) {5511      auto mapOp = cast<omp::MapInfoOp>(mapData.MapClause[i]);5512      capture = mapOp.getMapCaptureType();5513      // Get information of alignment of mapped object5514      alignmentValue = typeToLLVMIRTranslator.getPreferredAlignment(5515          mapOp.getVarType(), ompBuilder.M.getDataLayout());5516      break;5517    }5518 5519  unsigned int allocaAS = ompBuilder.M.getDataLayout().getAllocaAddrSpace();5520  unsigned int defaultAS =5521      ompBuilder.M.getDataLayout().getProgramAddressSpace();5522 5523  // Create the alloca for the argument the current point.5524  llvm::Value *v = builder.CreateAlloca(arg.getType(), allocaAS);5525 5526  if (allocaAS != defaultAS && arg.getType()->isPointerTy())5527    v = builder.CreateAddrSpaceCast(v, builder.getPtrTy(defaultAS));5528 5529  builder.CreateStore(&arg, v);5530 5531  builder.restoreIP(codeGenIP);5532 5533  switch (capture) {5534  case omp::VariableCaptureKind::ByCopy: {5535    retVal = v;5536    break;5537  }5538  case omp::VariableCaptureKind::ByRef: {5539    llvm::LoadInst *loadInst = builder.CreateAlignedLoad(5540        v->getType(), v,5541        ompBuilder.M.getDataLayout().getPrefTypeAlign(v->getType()));5542    // CreateAlignedLoad function creates similar LLVM IR:5543    // %res = load ptr, ptr %input, align 85544    // This LLVM IR does not contain information about alignment5545    // of the loaded value. We need to add !align metadata to unblock5546    // optimizer. The existence of the !align metadata on the instruction5547    // tells the optimizer that the value loaded is known to be aligned to5548    // a boundary specified by the integer value in the metadata node.5549    // Example:5550    // %res = load ptr, ptr %input, align 8, !align !align_md_node5551    //                                 ^         ^5552    //                                 |         |5553    //            alignment of %input address    |5554    //                                           |5555    //                                     alignment of %res object5556    if (v->getType()->isPointerTy() && alignmentValue) {5557      llvm::MDBuilder MDB(builder.getContext());5558      loadInst->setMetadata(5559          llvm::LLVMContext::MD_align,5560          llvm::MDNode::get(builder.getContext(),5561                            MDB.createConstant(llvm::ConstantInt::get(5562                                llvm::Type::getInt64Ty(builder.getContext()),5563                                alignmentValue))));5564    }5565    retVal = loadInst;5566 5567    break;5568  }5569  case omp::VariableCaptureKind::This:5570  case omp::VariableCaptureKind::VLAType:5571    // TODO: Consider returning error to use standard reporting for5572    // unimplemented features.5573    assert(false && "Currently unsupported capture kind");5574    break;5575  }5576 5577  return builder.saveIP();5578}5579 5580/// Follow uses of `host_eval`-defined block arguments of the given `omp.target`5581/// operation and populate output variables with their corresponding host value5582/// (i.e. operand evaluated outside of the target region), based on their uses5583/// inside of the target region.5584///5585/// Loop bounds and steps are only optionally populated, if output vectors are5586/// provided.5587static void5588extractHostEvalClauses(omp::TargetOp targetOp, Value &numThreads,5589                       Value &numTeamsLower, Value &numTeamsUpper,5590                       Value &threadLimit,5591                       llvm::SmallVectorImpl<Value> *lowerBounds = nullptr,5592                       llvm::SmallVectorImpl<Value> *upperBounds = nullptr,5593                       llvm::SmallVectorImpl<Value> *steps = nullptr) {5594  auto blockArgIface = llvm::cast<omp::BlockArgOpenMPOpInterface>(*targetOp);5595  for (auto item : llvm::zip_equal(targetOp.getHostEvalVars(),5596                                   blockArgIface.getHostEvalBlockArgs())) {5597    Value hostEvalVar = std::get<0>(item), blockArg = std::get<1>(item);5598 5599    for (Operation *user : blockArg.getUsers()) {5600      llvm::TypeSwitch<Operation *>(user)5601          .Case([&](omp::TeamsOp teamsOp) {5602            if (teamsOp.getNumTeamsLower() == blockArg)5603              numTeamsLower = hostEvalVar;5604            else if (teamsOp.getNumTeamsUpper() == blockArg)5605              numTeamsUpper = hostEvalVar;5606            else if (teamsOp.getThreadLimit() == blockArg)5607              threadLimit = hostEvalVar;5608            else5609              llvm_unreachable("unsupported host_eval use");5610          })5611          .Case([&](omp::ParallelOp parallelOp) {5612            if (parallelOp.getNumThreads() == blockArg)5613              numThreads = hostEvalVar;5614            else5615              llvm_unreachable("unsupported host_eval use");5616          })5617          .Case([&](omp::LoopNestOp loopOp) {5618            auto processBounds =5619                [&](OperandRange opBounds,5620                    llvm::SmallVectorImpl<Value> *outBounds) -> bool {5621              bool found = false;5622              for (auto [i, lb] : llvm::enumerate(opBounds)) {5623                if (lb == blockArg) {5624                  found = true;5625                  if (outBounds)5626                    (*outBounds)[i] = hostEvalVar;5627                }5628              }5629              return found;5630            };5631            bool found =5632                processBounds(loopOp.getLoopLowerBounds(), lowerBounds);5633            found = processBounds(loopOp.getLoopUpperBounds(), upperBounds) ||5634                    found;5635            found = processBounds(loopOp.getLoopSteps(), steps) || found;5636            (void)found;5637            assert(found && "unsupported host_eval use");5638          })5639          .DefaultUnreachable("unsupported host_eval use");5640    }5641  }5642}5643 5644/// If \p op is of the given type parameter, return it casted to that type.5645/// Otherwise, if its immediate parent operation (or some other higher-level5646/// parent, if \p immediateParent is false) is of that type, return that parent5647/// casted to the given type.5648///5649/// If \p op is \c null or neither it or its parent(s) are of the specified5650/// type, return a \c null operation.5651template <typename OpTy>5652static OpTy castOrGetParentOfType(Operation *op, bool immediateParent = false) {5653  if (!op)5654    return OpTy();5655 5656  if (OpTy casted = dyn_cast<OpTy>(op))5657    return casted;5658 5659  if (immediateParent)5660    return dyn_cast_if_present<OpTy>(op->getParentOp());5661 5662  return op->getParentOfType<OpTy>();5663}5664 5665/// If the given \p value is defined by an \c llvm.mlir.constant operation and5666/// it is of an integer type, return its value.5667static std::optional<int64_t> extractConstInteger(Value value) {5668  if (!value)5669    return std::nullopt;5670 5671  if (auto constOp = value.getDefiningOp<LLVM::ConstantOp>())5672    if (auto constAttr = dyn_cast<IntegerAttr>(constOp.getValue()))5673      return constAttr.getInt();5674 5675  return std::nullopt;5676}5677 5678static uint64_t getTypeByteSize(mlir::Type type, const DataLayout &dl) {5679  uint64_t sizeInBits = dl.getTypeSizeInBits(type);5680  uint64_t sizeInBytes = sizeInBits / 8;5681  return sizeInBytes;5682}5683 5684template <typename OpTy>5685static uint64_t getReductionDataSize(OpTy &op) {5686  if (op.getNumReductionVars() > 0) {5687    SmallVector<omp::DeclareReductionOp> reductions;5688    collectReductionDecls(op, reductions);5689 5690    llvm::SmallVector<mlir::Type> members;5691    members.reserve(reductions.size());5692    for (omp::DeclareReductionOp &red : reductions)5693      members.push_back(red.getType());5694    Operation *opp = op.getOperation();5695    auto structType = mlir::LLVM::LLVMStructType::getLiteral(5696        opp->getContext(), members, /*isPacked=*/false);5697    DataLayout dl = DataLayout(opp->getParentOfType<ModuleOp>());5698    return getTypeByteSize(structType, dl);5699  }5700  return 0;5701}5702 5703/// Populate default `MinTeams`, `MaxTeams` and `MaxThreads` to their default5704/// values as stated by the corresponding clauses, if constant.5705///5706/// These default values must be set before the creation of the outlined LLVM5707/// function for the target region, so that they can be used to initialize the5708/// corresponding global `ConfigurationEnvironmentTy` structure.5709static void5710initTargetDefaultAttrs(omp::TargetOp targetOp, Operation *capturedOp,5711                       llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &attrs,5712                       bool isTargetDevice, bool isGPU) {5713  // TODO: Handle constant 'if' clauses.5714 5715  Value numThreads, numTeamsLower, numTeamsUpper, threadLimit;5716  if (!isTargetDevice) {5717    extractHostEvalClauses(targetOp, numThreads, numTeamsLower, numTeamsUpper,5718                           threadLimit);5719  } else {5720    // In the target device, values for these clauses are not passed as5721    // host_eval, but instead evaluated prior to entry to the region. This5722    // ensures values are mapped and available inside of the target region.5723    if (auto teamsOp = castOrGetParentOfType<omp::TeamsOp>(capturedOp)) {5724      numTeamsLower = teamsOp.getNumTeamsLower();5725      numTeamsUpper = teamsOp.getNumTeamsUpper();5726      threadLimit = teamsOp.getThreadLimit();5727    }5728 5729    if (auto parallelOp = castOrGetParentOfType<omp::ParallelOp>(capturedOp))5730      numThreads = parallelOp.getNumThreads();5731  }5732 5733  // Handle clauses impacting the number of teams.5734 5735  int32_t minTeamsVal = 1, maxTeamsVal = -1;5736  if (castOrGetParentOfType<omp::TeamsOp>(capturedOp)) {5737    // TODO: Use `hostNumTeamsLower` to initialize `minTeamsVal`. For now,5738    // match clang and set min and max to the same value.5739    if (numTeamsUpper) {5740      if (auto val = extractConstInteger(numTeamsUpper))5741        minTeamsVal = maxTeamsVal = *val;5742    } else {5743      minTeamsVal = maxTeamsVal = 0;5744    }5745  } else if (castOrGetParentOfType<omp::ParallelOp>(capturedOp,5746                                                    /*immediateParent=*/true) ||5747             castOrGetParentOfType<omp::SimdOp>(capturedOp,5748                                                /*immediateParent=*/true)) {5749    minTeamsVal = maxTeamsVal = 1;5750  } else {5751    minTeamsVal = maxTeamsVal = -1;5752  }5753 5754  // Handle clauses impacting the number of threads.5755 5756  auto setMaxValueFromClause = [](Value clauseValue, int32_t &result) {5757    if (!clauseValue)5758      return;5759 5760    if (auto val = extractConstInteger(clauseValue))5761      result = *val;5762 5763    // Found an applicable clause, so it's not undefined. Mark as unknown5764    // because it's not constant.5765    if (result < 0)5766      result = 0;5767  };5768 5769  // Extract 'thread_limit' clause from 'target' and 'teams' directives.5770  int32_t targetThreadLimitVal = -1, teamsThreadLimitVal = -1;5771  setMaxValueFromClause(targetOp.getThreadLimit(), targetThreadLimitVal);5772  setMaxValueFromClause(threadLimit, teamsThreadLimitVal);5773 5774  // Extract 'max_threads' clause from 'parallel' or set to 1 if it's SIMD.5775  int32_t maxThreadsVal = -1;5776  if (castOrGetParentOfType<omp::ParallelOp>(capturedOp))5777    setMaxValueFromClause(numThreads, maxThreadsVal);5778  else if (castOrGetParentOfType<omp::SimdOp>(capturedOp,5779                                              /*immediateParent=*/true))5780    maxThreadsVal = 1;5781 5782  // For max values, < 0 means unset, == 0 means set but unknown. Select the5783  // minimum value between 'max_threads' and 'thread_limit' clauses that were5784  // set.5785  int32_t combinedMaxThreadsVal = targetThreadLimitVal;5786  if (combinedMaxThreadsVal < 0 ||5787      (teamsThreadLimitVal >= 0 && teamsThreadLimitVal < combinedMaxThreadsVal))5788    combinedMaxThreadsVal = teamsThreadLimitVal;5789 5790  if (combinedMaxThreadsVal < 0 ||5791      (maxThreadsVal >= 0 && maxThreadsVal < combinedMaxThreadsVal))5792    combinedMaxThreadsVal = maxThreadsVal;5793 5794  int32_t reductionDataSize = 0;5795  if (isGPU && capturedOp) {5796    if (auto teamsOp = castOrGetParentOfType<omp::TeamsOp>(capturedOp))5797      reductionDataSize = getReductionDataSize(teamsOp);5798  }5799 5800  // Update kernel bounds structure for the `OpenMPIRBuilder` to use.5801  omp::TargetRegionFlags kernelFlags = targetOp.getKernelExecFlags(capturedOp);5802  assert(5803      omp::bitEnumContainsAny(kernelFlags, omp::TargetRegionFlags::generic |5804                                               omp::TargetRegionFlags::spmd) &&5805      "invalid kernel flags");5806  attrs.ExecFlags =5807      omp::bitEnumContainsAny(kernelFlags, omp::TargetRegionFlags::generic)5808          ? omp::bitEnumContainsAny(kernelFlags, omp::TargetRegionFlags::spmd)5809                ? llvm::omp::OMP_TGT_EXEC_MODE_GENERIC_SPMD5810                : llvm::omp::OMP_TGT_EXEC_MODE_GENERIC5811          : llvm::omp::OMP_TGT_EXEC_MODE_SPMD;5812  if (omp::bitEnumContainsAll(kernelFlags,5813                              omp::TargetRegionFlags::spmd |5814                                  omp::TargetRegionFlags::no_loop) &&5815      !omp::bitEnumContainsAny(kernelFlags, omp::TargetRegionFlags::generic))5816    attrs.ExecFlags = llvm::omp::OMP_TGT_EXEC_MODE_SPMD_NO_LOOP;5817 5818  attrs.MinTeams = minTeamsVal;5819  attrs.MaxTeams.front() = maxTeamsVal;5820  attrs.MinThreads = 1;5821  attrs.MaxThreads.front() = combinedMaxThreadsVal;5822  attrs.ReductionDataSize = reductionDataSize;5823  // TODO: Allow modified buffer length similar to5824  // fopenmp-cuda-teams-reduction-recs-num flag in clang.5825  if (attrs.ReductionDataSize != 0)5826    attrs.ReductionBufferLength = 1024;5827}5828 5829/// Gather LLVM runtime values for all clauses evaluated in the host that are5830/// passed to the kernel invocation.5831///5832/// This function must be called only when compiling for the host. Also, it will5833/// only provide correct results if it's called after the body of \c targetOp5834/// has been fully generated.5835static void5836initTargetRuntimeAttrs(llvm::IRBuilderBase &builder,5837                       LLVM::ModuleTranslation &moduleTranslation,5838                       omp::TargetOp targetOp, Operation *capturedOp,5839                       llvm::OpenMPIRBuilder::TargetKernelRuntimeAttrs &attrs) {5840  omp::LoopNestOp loopOp = castOrGetParentOfType<omp::LoopNestOp>(capturedOp);5841  unsigned numLoops = loopOp ? loopOp.getNumLoops() : 0;5842 5843  Value numThreads, numTeamsLower, numTeamsUpper, teamsThreadLimit;5844  llvm::SmallVector<Value> lowerBounds(numLoops), upperBounds(numLoops),5845      steps(numLoops);5846  extractHostEvalClauses(targetOp, numThreads, numTeamsLower, numTeamsUpper,5847                         teamsThreadLimit, &lowerBounds, &upperBounds, &steps);5848 5849  // TODO: Handle constant 'if' clauses.5850  if (Value targetThreadLimit = targetOp.getThreadLimit())5851    attrs.TargetThreadLimit.front() =5852        moduleTranslation.lookupValue(targetThreadLimit);5853 5854  if (numTeamsLower)5855    attrs.MinTeams = moduleTranslation.lookupValue(numTeamsLower);5856 5857  if (numTeamsUpper)5858    attrs.MaxTeams.front() = moduleTranslation.lookupValue(numTeamsUpper);5859 5860  if (teamsThreadLimit)5861    attrs.TeamsThreadLimit.front() =5862        moduleTranslation.lookupValue(teamsThreadLimit);5863 5864  if (numThreads)5865    attrs.MaxThreads = moduleTranslation.lookupValue(numThreads);5866 5867  if (omp::bitEnumContainsAny(targetOp.getKernelExecFlags(capturedOp),5868                              omp::TargetRegionFlags::trip_count)) {5869    llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();5870    attrs.LoopTripCount = nullptr;5871 5872    // To calculate the trip count, we multiply together the trip counts of5873    // every collapsed canonical loop. We don't need to create the loop nests5874    // here, since we're only interested in the trip count.5875    for (auto [loopLower, loopUpper, loopStep] :5876         llvm::zip_equal(lowerBounds, upperBounds, steps)) {5877      llvm::Value *lowerBound = moduleTranslation.lookupValue(loopLower);5878      llvm::Value *upperBound = moduleTranslation.lookupValue(loopUpper);5879      llvm::Value *step = moduleTranslation.lookupValue(loopStep);5880 5881      llvm::OpenMPIRBuilder::LocationDescription loc(builder);5882      llvm::Value *tripCount = ompBuilder->calculateCanonicalLoopTripCount(5883          loc, lowerBound, upperBound, step, /*IsSigned=*/true,5884          loopOp.getLoopInclusive());5885 5886      if (!attrs.LoopTripCount) {5887        attrs.LoopTripCount = tripCount;5888        continue;5889      }5890 5891      // TODO: Enable UndefinedSanitizer to diagnose an overflow here.5892      attrs.LoopTripCount = builder.CreateMul(attrs.LoopTripCount, tripCount,5893                                              {}, /*HasNUW=*/true);5894    }5895  }5896}5897 5898static LogicalResult5899convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,5900                 LLVM::ModuleTranslation &moduleTranslation) {5901  auto targetOp = cast<omp::TargetOp>(opInst);5902  // The current debug location already has the DISubprogram for the outlined5903  // function that will be created for the target op. We save it here so that5904  // we can set it on the outlined function.5905  llvm::DebugLoc outlinedFnLoc = builder.getCurrentDebugLocation();5906  if (failed(checkImplementationStatus(opInst)))5907    return failure();5908 5909  // During the handling of target op, we will generate instructions in the5910  // parent function like call to the oulined function or branch to a new5911  // BasicBlock. We set the debug location here to parent function so that those5912  // get the correct debug locations. For outlined functions, the normal MLIR op5913  // conversion will automatically pick the correct location.5914  llvm::BasicBlock *parentBB = builder.GetInsertBlock();5915  assert(parentBB && "No insert block is set for the builder");5916  llvm::Function *parentLLVMFn = parentBB->getParent();5917  assert(parentLLVMFn && "Parent Function must be valid");5918  if (llvm::DISubprogram *SP = parentLLVMFn->getSubprogram())5919    builder.SetCurrentDebugLocation(llvm::DILocation::get(5920        parentLLVMFn->getContext(), outlinedFnLoc.getLine(),5921        outlinedFnLoc.getCol(), SP, outlinedFnLoc.getInlinedAt()));5922 5923  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();5924  bool isTargetDevice = ompBuilder->Config.isTargetDevice();5925  bool isGPU = ompBuilder->Config.isGPU();5926 5927  auto parentFn = opInst.getParentOfType<LLVM::LLVMFuncOp>();5928  auto argIface = cast<omp::BlockArgOpenMPOpInterface>(opInst);5929  auto &targetRegion = targetOp.getRegion();5930  // Holds the private vars that have been mapped along with the block5931  // argument that corresponds to the MapInfoOp corresponding to the private5932  // var in question. So, for instance:5933  //5934  // %10 = omp.map.info var_ptr(%6#0 : !fir.ref<!fir.box<!fir.heap<i32>>>, ..)5935  // omp.target map_entries(%10 -> %arg0) private(@box.privatizer %6#0-> %arg1)5936  //5937  // Then, %10 has been created so that the descriptor can be used by the5938  // privatizer @box.privatizer on the device side. Here we'd record {%6#0,5939  // %arg0} in the mappedPrivateVars map.5940  llvm::DenseMap<Value, Value> mappedPrivateVars;5941  DataLayout dl = DataLayout(opInst.getParentOfType<ModuleOp>());5942  SmallVector<Value> mapVars = targetOp.getMapVars();5943  SmallVector<Value> hdaVars = targetOp.getHasDeviceAddrVars();5944  ArrayRef<BlockArgument> mapBlockArgs = argIface.getMapBlockArgs();5945  ArrayRef<BlockArgument> hdaBlockArgs = argIface.getHasDeviceAddrBlockArgs();5946  llvm::Function *llvmOutlinedFn = nullptr;5947  TargetDirectiveEnumTy targetDirective =5948      getTargetDirectiveEnumTyFromOp(&opInst);5949 5950  // TODO: It can also be false if a compile-time constant `false` IF clause is5951  // specified.5952  bool isOffloadEntry =5953      isTargetDevice || !ompBuilder->Config.TargetTriples.empty();5954 5955  // For some private variables, the MapsForPrivatizedVariablesPass5956  // creates MapInfoOp instances. Go through the private variables and5957  // the mapped variables so that during codegeneration we are able5958  // to quickly look up the corresponding map variable, if any for each5959  // private variable.5960  if (!targetOp.getPrivateVars().empty() && !targetOp.getMapVars().empty()) {5961    OperandRange privateVars = targetOp.getPrivateVars();5962    std::optional<ArrayAttr> privateSyms = targetOp.getPrivateSyms();5963    std::optional<DenseI64ArrayAttr> privateMapIndices =5964        targetOp.getPrivateMapsAttr();5965 5966    for (auto [privVarIdx, privVarSymPair] :5967         llvm::enumerate(llvm::zip_equal(privateVars, *privateSyms))) {5968      auto privVar = std::get<0>(privVarSymPair);5969      auto privSym = std::get<1>(privVarSymPair);5970 5971      SymbolRefAttr privatizerName = llvm::cast<SymbolRefAttr>(privSym);5972      omp::PrivateClauseOp privatizer =5973          findPrivatizer(targetOp, privatizerName);5974 5975      if (!privatizer.needsMap())5976        continue;5977 5978      mlir::Value mappedValue =5979          targetOp.getMappedValueForPrivateVar(privVarIdx);5980      assert(mappedValue && "Expected to find mapped value for a privatized "5981                            "variable that needs mapping");5982 5983      // The MapInfoOp defining the map var isn't really needed later.5984      // So, we don't store it in any datastructure. Instead, we just5985      // do some sanity checks on it right now.5986      auto mapInfoOp = mappedValue.getDefiningOp<omp::MapInfoOp>();5987      [[maybe_unused]] Type varType = mapInfoOp.getVarType();5988 5989      // Check #1: Check that the type of the private variable matches5990      // the type of the variable being mapped.5991      if (!isa<LLVM::LLVMPointerType>(privVar.getType()))5992        assert(5993            varType == privVar.getType() &&5994            "Type of private var doesn't match the type of the mapped value");5995 5996      // Ok, only 1 sanity check for now.5997      // Record the block argument corresponding to this mapvar.5998      mappedPrivateVars.insert(5999          {privVar,6000           targetRegion.getArgument(argIface.getMapBlockArgsStart() +6001                                    (*privateMapIndices)[privVarIdx])});6002    }6003  }6004 6005  using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;6006  auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP)6007      -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {6008    llvm::IRBuilderBase::InsertPointGuard guard(builder);6009    builder.SetCurrentDebugLocation(llvm::DebugLoc());6010    // Forward target-cpu and target-features function attributes from the6011    // original function to the new outlined function.6012    llvm::Function *llvmParentFn =6013        moduleTranslation.lookupFunction(parentFn.getName());6014    llvmOutlinedFn = codeGenIP.getBlock()->getParent();6015    assert(llvmParentFn && llvmOutlinedFn &&6016           "Both parent and outlined functions must exist at this point");6017 6018    if (outlinedFnLoc && llvmParentFn->getSubprogram())6019      llvmOutlinedFn->setSubprogram(outlinedFnLoc->getScope()->getSubprogram());6020 6021    if (auto attr = llvmParentFn->getFnAttribute("target-cpu");6022        attr.isStringAttribute())6023      llvmOutlinedFn->addFnAttr(attr);6024 6025    if (auto attr = llvmParentFn->getFnAttribute("target-features");6026        attr.isStringAttribute())6027      llvmOutlinedFn->addFnAttr(attr);6028 6029    for (auto [arg, mapOp] : llvm::zip_equal(mapBlockArgs, mapVars)) {6030      auto mapInfoOp = cast<omp::MapInfoOp>(mapOp.getDefiningOp());6031      llvm::Value *mapOpValue =6032          moduleTranslation.lookupValue(mapInfoOp.getVarPtr());6033      moduleTranslation.mapValue(arg, mapOpValue);6034    }6035    for (auto [arg, mapOp] : llvm::zip_equal(hdaBlockArgs, hdaVars)) {6036      auto mapInfoOp = cast<omp::MapInfoOp>(mapOp.getDefiningOp());6037      llvm::Value *mapOpValue =6038          moduleTranslation.lookupValue(mapInfoOp.getVarPtr());6039      moduleTranslation.mapValue(arg, mapOpValue);6040    }6041 6042    // Do privatization after moduleTranslation has already recorded6043    // mapped values.6044    PrivateVarsInfo privateVarsInfo(targetOp);6045 6046    llvm::Expected<llvm::BasicBlock *> afterAllocas =6047        allocatePrivateVars(builder, moduleTranslation, privateVarsInfo,6048                            allocaIP, &mappedPrivateVars);6049 6050    if (failed(handleError(afterAllocas, *targetOp)))6051      return llvm::make_error<PreviouslyReportedError>();6052 6053    builder.restoreIP(codeGenIP);6054    if (handleError(initPrivateVars(builder, moduleTranslation, privateVarsInfo,6055                                    &mappedPrivateVars),6056                    *targetOp)6057            .failed())6058      return llvm::make_error<PreviouslyReportedError>();6059 6060    if (failed(copyFirstPrivateVars(6061            targetOp, builder, moduleTranslation, privateVarsInfo.mlirVars,6062            privateVarsInfo.llvmVars, privateVarsInfo.privatizers,6063            targetOp.getPrivateNeedsBarrier(), &mappedPrivateVars)))6064      return llvm::make_error<PreviouslyReportedError>();6065 6066    SmallVector<Region *> privateCleanupRegions;6067    llvm::transform(privateVarsInfo.privatizers,6068                    std::back_inserter(privateCleanupRegions),6069                    [](omp::PrivateClauseOp privatizer) {6070                      return &privatizer.getDeallocRegion();6071                    });6072 6073    llvm::Expected<llvm::BasicBlock *> exitBlock = convertOmpOpRegions(6074        targetRegion, "omp.target", builder, moduleTranslation);6075 6076    if (!exitBlock)6077      return exitBlock.takeError();6078 6079    builder.SetInsertPoint(*exitBlock);6080    if (!privateCleanupRegions.empty()) {6081      if (failed(inlineOmpRegionCleanup(6082              privateCleanupRegions, privateVarsInfo.llvmVars,6083              moduleTranslation, builder, "omp.targetop.private.cleanup",6084              /*shouldLoadCleanupRegionArg=*/false))) {6085        return llvm::createStringError(6086            "failed to inline `dealloc` region of `omp.private` "6087            "op in the target region");6088      }6089      return builder.saveIP();6090    }6091 6092    return InsertPointTy(exitBlock.get(), exitBlock.get()->end());6093  };6094 6095  StringRef parentName = parentFn.getName();6096 6097  llvm::TargetRegionEntryInfo entryInfo;6098 6099  getTargetEntryUniqueInfo(entryInfo, targetOp, parentName);6100 6101  MapInfoData mapData;6102  collectMapDataFromMapOperands(mapData, mapVars, moduleTranslation, dl,6103                                builder, /*useDevPtrOperands=*/{},6104                                /*useDevAddrOperands=*/{}, hdaVars);6105 6106  MapInfosTy combinedInfos;6107  auto genMapInfoCB =6108      [&](llvm::OpenMPIRBuilder::InsertPointTy codeGenIP) -> MapInfosTy & {6109    builder.restoreIP(codeGenIP);6110    genMapInfos(builder, moduleTranslation, dl, combinedInfos, mapData,6111                targetDirective);6112    return combinedInfos;6113  };6114 6115  auto argAccessorCB = [&](llvm::Argument &arg, llvm::Value *input,6116                           llvm::Value *&retVal, InsertPointTy allocaIP,6117                           InsertPointTy codeGenIP)6118      -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {6119    llvm::IRBuilderBase::InsertPointGuard guard(builder);6120    builder.SetCurrentDebugLocation(llvm::DebugLoc());6121    // We just return the unaltered argument for the host function6122    // for now, some alterations may be required in the future to6123    // keep host fallback functions working identically to the device6124    // version (e.g. pass ByCopy values should be treated as such on6125    // host and device, currently not always the case)6126    if (!isTargetDevice) {6127      retVal = cast<llvm::Value>(&arg);6128      return codeGenIP;6129    }6130 6131    return createDeviceArgumentAccessor(mapData, arg, input, retVal, builder,6132                                        *ompBuilder, moduleTranslation,6133                                        allocaIP, codeGenIP);6134  };6135 6136  llvm::OpenMPIRBuilder::TargetKernelRuntimeAttrs runtimeAttrs;6137  llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs defaultAttrs;6138  Operation *targetCapturedOp = targetOp.getInnermostCapturedOmpOp();6139  initTargetDefaultAttrs(targetOp, targetCapturedOp, defaultAttrs,6140                         isTargetDevice, isGPU);6141 6142  // Collect host-evaluated values needed to properly launch the kernel from the6143  // host.6144  if (!isTargetDevice)6145    initTargetRuntimeAttrs(builder, moduleTranslation, targetOp,6146                           targetCapturedOp, runtimeAttrs);6147 6148  // Pass host-evaluated values as parameters to the kernel / host fallback,6149  // except if they are constants. In any case, map the MLIR block argument to6150  // the corresponding LLVM values.6151  llvm::SmallVector<llvm::Value *, 4> kernelInput;6152  SmallVector<Value> hostEvalVars = targetOp.getHostEvalVars();6153  ArrayRef<BlockArgument> hostEvalBlockArgs = argIface.getHostEvalBlockArgs();6154  for (auto [arg, var] : llvm::zip_equal(hostEvalBlockArgs, hostEvalVars)) {6155    llvm::Value *value = moduleTranslation.lookupValue(var);6156    moduleTranslation.mapValue(arg, value);6157 6158    if (!llvm::isa<llvm::Constant>(value))6159      kernelInput.push_back(value);6160  }6161 6162  for (size_t i = 0, e = mapData.OriginalValue.size(); i != e; ++i) {6163    // declare target arguments are not passed to kernels as arguments6164    // TODO: We currently do not handle cases where a member is explicitly6165    // passed in as an argument, this will likley need to be handled in6166    // the near future, rather than using IsAMember, it may be better to6167    // test if the relevant BlockArg is used within the target region and6168    // then use that as a basis for exclusion in the kernel inputs.6169    if (!mapData.IsDeclareTarget[i] && !mapData.IsAMember[i])6170      kernelInput.push_back(mapData.OriginalValue[i]);6171  }6172 6173  SmallVector<llvm::OpenMPIRBuilder::DependData> dds;6174  buildDependData(targetOp.getDependKinds(), targetOp.getDependVars(),6175                  moduleTranslation, dds);6176 6177  llvm::OpenMPIRBuilder::InsertPointTy allocaIP =6178      findAllocaInsertPoint(builder, moduleTranslation);6179  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);6180 6181  llvm::OpenMPIRBuilder::TargetDataInfo info(6182      /*RequiresDevicePointerInfo=*/false,6183      /*SeparateBeginEndCalls=*/true);6184 6185  auto customMapperCB =6186      [&](unsigned int i) -> llvm::Expected<llvm::Function *> {6187    if (!combinedInfos.Mappers[i])6188      return nullptr;6189    info.HasMapper = true;6190    return getOrCreateUserDefinedMapperFunc(combinedInfos.Mappers[i], builder,6191                                            moduleTranslation, targetDirective);6192  };6193 6194  llvm::Value *ifCond = nullptr;6195  if (Value targetIfCond = targetOp.getIfExpr())6196    ifCond = moduleTranslation.lookupValue(targetIfCond);6197 6198  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =6199      moduleTranslation.getOpenMPBuilder()->createTarget(6200          ompLoc, isOffloadEntry, allocaIP, builder.saveIP(), info, entryInfo,6201          defaultAttrs, runtimeAttrs, ifCond, kernelInput, genMapInfoCB, bodyCB,6202          argAccessorCB, customMapperCB, dds, targetOp.getNowait());6203 6204  if (failed(handleError(afterIP, opInst)))6205    return failure();6206 6207  builder.restoreIP(*afterIP);6208 6209  // Remap access operations to declare target reference pointers for the6210  // device, essentially generating extra loadop's as necessary6211  if (moduleTranslation.getOpenMPBuilder()->Config.isTargetDevice())6212    handleDeclareTargetMapVar(mapData, moduleTranslation, builder,6213                              llvmOutlinedFn);6214 6215  return success();6216}6217 6218static LogicalResult6219convertDeclareTargetAttr(Operation *op, mlir::omp::DeclareTargetAttr attribute,6220                         LLVM::ModuleTranslation &moduleTranslation) {6221  // Amend omp.declare_target by deleting the IR of the outlined functions6222  // created for target regions. They cannot be filtered out from MLIR earlier6223  // because the omp.target operation inside must be translated to LLVM, but6224  // the wrapper functions themselves must not remain at the end of the6225  // process. We know that functions where omp.declare_target does not match6226  // omp.is_target_device at this stage can only be wrapper functions because6227  // those that aren't are removed earlier as an MLIR transformation pass.6228  if (FunctionOpInterface funcOp = dyn_cast<FunctionOpInterface>(op)) {6229    if (auto offloadMod = dyn_cast<omp::OffloadModuleInterface>(6230            op->getParentOfType<ModuleOp>().getOperation())) {6231      if (!offloadMod.getIsTargetDevice())6232        return success();6233 6234      omp::DeclareTargetDeviceType declareType =6235          attribute.getDeviceType().getValue();6236 6237      if (declareType == omp::DeclareTargetDeviceType::host) {6238        llvm::Function *llvmFunc =6239            moduleTranslation.lookupFunction(funcOp.getName());6240        llvmFunc->dropAllReferences();6241        llvmFunc->eraseFromParent();6242      }6243    }6244    return success();6245  }6246 6247  if (LLVM::GlobalOp gOp = dyn_cast<LLVM::GlobalOp>(op)) {6248    llvm::Module *llvmModule = moduleTranslation.getLLVMModule();6249    if (auto *gVal = llvmModule->getNamedValue(gOp.getSymName())) {6250      llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();6251      bool isDeclaration = gOp.isDeclaration();6252      bool isExternallyVisible =6253          gOp.getVisibility() != mlir::SymbolTable::Visibility::Private;6254      auto loc = op->getLoc()->findInstanceOf<FileLineColLoc>();6255      llvm::StringRef mangledName = gOp.getSymName();6256      auto captureClause =6257          convertToCaptureClauseKind(attribute.getCaptureClause().getValue());6258      auto deviceClause =6259          convertToDeviceClauseKind(attribute.getDeviceType().getValue());6260      // unused for MLIR at the moment, required in Clang for book6261      // keeping6262      std::vector<llvm::GlobalVariable *> generatedRefs;6263 6264      std::vector<llvm::Triple> targetTriple;6265      auto targetTripleAttr = dyn_cast_or_null<mlir::StringAttr>(6266          op->getParentOfType<mlir::ModuleOp>()->getAttr(6267              LLVM::LLVMDialect::getTargetTripleAttrName()));6268      if (targetTripleAttr)6269        targetTriple.emplace_back(targetTripleAttr.data());6270 6271      auto fileInfoCallBack = [&loc]() {6272        std::string filename = "";6273        std::uint64_t lineNo = 0;6274 6275        if (loc) {6276          filename = loc.getFilename().str();6277          lineNo = loc.getLine();6278        }6279 6280        return std::pair<std::string, std::uint64_t>(llvm::StringRef(filename),6281                                                     lineNo);6282      };6283 6284      auto vfs = llvm::vfs::getRealFileSystem();6285 6286      ompBuilder->registerTargetGlobalVariable(6287          captureClause, deviceClause, isDeclaration, isExternallyVisible,6288          ompBuilder->getTargetEntryUniqueInfo(fileInfoCallBack, *vfs),6289          mangledName, generatedRefs, /*OpenMPSimd*/ false, targetTriple,6290          /*GlobalInitializer*/ nullptr, /*VariableLinkage*/ nullptr,6291          gVal->getType(), gVal);6292 6293      if (ompBuilder->Config.isTargetDevice() &&6294          (attribute.getCaptureClause().getValue() !=6295               mlir::omp::DeclareTargetCaptureClause::to ||6296           ompBuilder->Config.hasRequiresUnifiedSharedMemory())) {6297        ompBuilder->getAddrOfDeclareTargetVar(6298            captureClause, deviceClause, isDeclaration, isExternallyVisible,6299            ompBuilder->getTargetEntryUniqueInfo(fileInfoCallBack, *vfs),6300            mangledName, generatedRefs, /*OpenMPSimd*/ false, targetTriple,6301            gVal->getType(), /*GlobalInitializer*/ nullptr,6302            /*VariableLinkage*/ nullptr);6303      }6304    }6305  }6306 6307  return success();6308}6309 6310// Returns true if the operation is inside a TargetOp or6311// is part of a declare target function.6312static bool isTargetDeviceOp(Operation *op) {6313  // Assumes no reverse offloading6314  if (op->getParentOfType<omp::TargetOp>())6315    return true;6316 6317  // Certain operations return results, and whether utilised in host or6318  // target there is a chance an LLVM Dialect operation depends on it6319  // by taking it in as an operand, so we must always lower these in6320  // some manner or result in an ICE (whether they end up in a no-op6321  // or otherwise).6322  if (mlir::isa<omp::ThreadprivateOp>(op))6323    return true;6324 6325  if (mlir::isa<omp::TargetAllocMemOp>(op) ||6326      mlir::isa<omp::TargetFreeMemOp>(op))6327    return true;6328 6329  if (auto parentFn = op->getParentOfType<LLVM::LLVMFuncOp>())6330    if (auto declareTargetIface =6331            llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(6332                parentFn.getOperation()))6333      if (declareTargetIface.isDeclareTarget() &&6334          declareTargetIface.getDeclareTargetDeviceType() !=6335              mlir::omp::DeclareTargetDeviceType::host)6336        return true;6337 6338  return false;6339}6340 6341static llvm::Function *getOmpTargetAlloc(llvm::IRBuilderBase &builder,6342                                         llvm::Module *llvmModule) {6343  llvm::Type *i64Ty = builder.getInt64Ty();6344  llvm::Type *i32Ty = builder.getInt32Ty();6345  llvm::Type *returnType = builder.getPtrTy(0);6346  llvm::FunctionType *fnType =6347      llvm::FunctionType::get(returnType, {i64Ty, i32Ty}, false);6348  llvm::Function *func = cast<llvm::Function>(6349      llvmModule->getOrInsertFunction("omp_target_alloc", fnType).getCallee());6350  return func;6351}6352 6353static LogicalResult6354convertTargetAllocMemOp(Operation &opInst, llvm::IRBuilderBase &builder,6355                        LLVM::ModuleTranslation &moduleTranslation) {6356  auto allocMemOp = cast<omp::TargetAllocMemOp>(opInst);6357  if (!allocMemOp)6358    return failure();6359 6360  // Get "omp_target_alloc" function6361  llvm::Module *llvmModule = moduleTranslation.getLLVMModule();6362  llvm::Function *ompTargetAllocFunc = getOmpTargetAlloc(builder, llvmModule);6363  // Get the corresponding device value in llvm6364  mlir::Value deviceNum = allocMemOp.getDevice();6365  llvm::Value *llvmDeviceNum = moduleTranslation.lookupValue(deviceNum);6366  // Get the allocation size.6367  llvm::DataLayout dataLayout = llvmModule->getDataLayout();6368  mlir::Type heapTy = allocMemOp.getAllocatedType();6369  llvm::Type *llvmHeapTy = moduleTranslation.convertType(heapTy);6370  llvm::TypeSize typeSize = dataLayout.getTypeStoreSize(llvmHeapTy);6371  llvm::Value *allocSize = builder.getInt64(typeSize.getFixedValue());6372  for (auto typeParam : allocMemOp.getTypeparams())6373    allocSize =6374        builder.CreateMul(allocSize, moduleTranslation.lookupValue(typeParam));6375  // Create call to "omp_target_alloc" with the args as translated llvm values.6376  llvm::CallInst *call =6377      builder.CreateCall(ompTargetAllocFunc, {allocSize, llvmDeviceNum});6378  llvm::Value *resultI64 = builder.CreatePtrToInt(call, builder.getInt64Ty());6379 6380  // Map the result6381  moduleTranslation.mapValue(allocMemOp.getResult(), resultI64);6382  return success();6383}6384 6385static llvm::Function *getOmpTargetFree(llvm::IRBuilderBase &builder,6386                                        llvm::Module *llvmModule) {6387  llvm::Type *ptrTy = builder.getPtrTy(0);6388  llvm::Type *i32Ty = builder.getInt32Ty();6389  llvm::Type *voidTy = builder.getVoidTy();6390  llvm::FunctionType *fnType =6391      llvm::FunctionType::get(voidTy, {ptrTy, i32Ty}, false);6392  llvm::Function *func = dyn_cast<llvm::Function>(6393      llvmModule->getOrInsertFunction("omp_target_free", fnType).getCallee());6394  return func;6395}6396 6397static LogicalResult6398convertTargetFreeMemOp(Operation &opInst, llvm::IRBuilderBase &builder,6399                       LLVM::ModuleTranslation &moduleTranslation) {6400  auto freeMemOp = cast<omp::TargetFreeMemOp>(opInst);6401  if (!freeMemOp)6402    return failure();6403 6404  // Get "omp_target_free" function6405  llvm::Module *llvmModule = moduleTranslation.getLLVMModule();6406  llvm::Function *ompTragetFreeFunc = getOmpTargetFree(builder, llvmModule);6407  // Get the corresponding device value in llvm6408  mlir::Value deviceNum = freeMemOp.getDevice();6409  llvm::Value *llvmDeviceNum = moduleTranslation.lookupValue(deviceNum);6410  // Get the corresponding heapref value in llvm6411  mlir::Value heapref = freeMemOp.getHeapref();6412  llvm::Value *llvmHeapref = moduleTranslation.lookupValue(heapref);6413  // Convert heapref int to ptr and call "omp_target_free"6414  llvm::Value *intToPtr =6415      builder.CreateIntToPtr(llvmHeapref, builder.getPtrTy(0));6416  builder.CreateCall(ompTragetFreeFunc, {intToPtr, llvmDeviceNum});6417  return success();6418}6419 6420/// Given an OpenMP MLIR operation, create the corresponding LLVM IR (including6421/// OpenMP runtime calls).6422static LogicalResult6423convertHostOrTargetOperation(Operation *op, llvm::IRBuilderBase &builder,6424                             LLVM::ModuleTranslation &moduleTranslation) {6425  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();6426 6427  // For each loop, introduce one stack frame to hold loop information. Ensure6428  // this is only done for the outermost loop wrapper to prevent introducing6429  // multiple stack frames for a single loop. Initially set to null, the loop6430  // information structure is initialized during translation of the nested6431  // omp.loop_nest operation, making it available to translation of all loop6432  // wrappers after their body has been successfully translated.6433  bool isOutermostLoopWrapper =6434      isa_and_present<omp::LoopWrapperInterface>(op) &&6435      !dyn_cast_if_present<omp::LoopWrapperInterface>(op->getParentOp());6436 6437  if (isOutermostLoopWrapper)6438    moduleTranslation.stackPush<OpenMPLoopInfoStackFrame>();6439 6440  auto result =6441      llvm::TypeSwitch<Operation *, LogicalResult>(op)6442          .Case([&](omp::BarrierOp op) -> LogicalResult {6443            if (failed(checkImplementationStatus(*op)))6444              return failure();6445 6446            llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =6447                ompBuilder->createBarrier(builder.saveIP(),6448                                          llvm::omp::OMPD_barrier);6449            LogicalResult res = handleError(afterIP, *op);6450            if (res.succeeded()) {6451              // If the barrier generated a cancellation check, the insertion6452              // point might now need to be changed to a new continuation block6453              builder.restoreIP(*afterIP);6454            }6455            return res;6456          })6457          .Case([&](omp::TaskyieldOp op) {6458            if (failed(checkImplementationStatus(*op)))6459              return failure();6460 6461            ompBuilder->createTaskyield(builder.saveIP());6462            return success();6463          })6464          .Case([&](omp::FlushOp op) {6465            if (failed(checkImplementationStatus(*op)))6466              return failure();6467 6468            // No support in Openmp runtime function (__kmpc_flush) to accept6469            // the argument list.6470            // OpenMP standard states the following:6471            //  "An implementation may implement a flush with a list by ignoring6472            //   the list, and treating it the same as a flush without a list."6473            //6474            // The argument list is discarded so that, flush with a list is6475            // treated same as a flush without a list.6476            ompBuilder->createFlush(builder.saveIP());6477            return success();6478          })6479          .Case([&](omp::ParallelOp op) {6480            return convertOmpParallel(op, builder, moduleTranslation);6481          })6482          .Case([&](omp::MaskedOp) {6483            return convertOmpMasked(*op, builder, moduleTranslation);6484          })6485          .Case([&](omp::MasterOp) {6486            return convertOmpMaster(*op, builder, moduleTranslation);6487          })6488          .Case([&](omp::CriticalOp) {6489            return convertOmpCritical(*op, builder, moduleTranslation);6490          })6491          .Case([&](omp::OrderedRegionOp) {6492            return convertOmpOrderedRegion(*op, builder, moduleTranslation);6493          })6494          .Case([&](omp::OrderedOp) {6495            return convertOmpOrdered(*op, builder, moduleTranslation);6496          })6497          .Case([&](omp::WsloopOp) {6498            return convertOmpWsloop(*op, builder, moduleTranslation);6499          })6500          .Case([&](omp::SimdOp) {6501            return convertOmpSimd(*op, builder, moduleTranslation);6502          })6503          .Case([&](omp::AtomicReadOp) {6504            return convertOmpAtomicRead(*op, builder, moduleTranslation);6505          })6506          .Case([&](omp::AtomicWriteOp) {6507            return convertOmpAtomicWrite(*op, builder, moduleTranslation);6508          })6509          .Case([&](omp::AtomicUpdateOp op) {6510            return convertOmpAtomicUpdate(op, builder, moduleTranslation);6511          })6512          .Case([&](omp::AtomicCaptureOp op) {6513            return convertOmpAtomicCapture(op, builder, moduleTranslation);6514          })6515          .Case([&](omp::CancelOp op) {6516            return convertOmpCancel(op, builder, moduleTranslation);6517          })6518          .Case([&](omp::CancellationPointOp op) {6519            return convertOmpCancellationPoint(op, builder, moduleTranslation);6520          })6521          .Case([&](omp::SectionsOp) {6522            return convertOmpSections(*op, builder, moduleTranslation);6523          })6524          .Case([&](omp::SingleOp op) {6525            return convertOmpSingle(op, builder, moduleTranslation);6526          })6527          .Case([&](omp::TeamsOp op) {6528            return convertOmpTeams(op, builder, moduleTranslation);6529          })6530          .Case([&](omp::TaskOp op) {6531            return convertOmpTaskOp(op, builder, moduleTranslation);6532          })6533          .Case([&](omp::TaskgroupOp op) {6534            return convertOmpTaskgroupOp(op, builder, moduleTranslation);6535          })6536          .Case([&](omp::TaskwaitOp op) {6537            return convertOmpTaskwaitOp(op, builder, moduleTranslation);6538          })6539          .Case<omp::YieldOp, omp::TerminatorOp, omp::DeclareMapperOp,6540                omp::DeclareMapperInfoOp, omp::DeclareReductionOp,6541                omp::CriticalDeclareOp>([](auto op) {6542            // `yield` and `terminator` can be just omitted. The block structure6543            // was created in the region that handles their parent operation.6544            // `declare_reduction` will be used by reductions and is not6545            // converted directly, skip it.6546            // `declare_mapper` and `declare_mapper.info` are handled whenever6547            // they are referred to through a `map` clause.6548            // `critical.declare` is only used to declare names of critical6549            // sections which will be used by `critical` ops and hence can be6550            // ignored for lowering. The OpenMP IRBuilder will create unique6551            // name for critical section names.6552            return success();6553          })6554          .Case([&](omp::ThreadprivateOp) {6555            return convertOmpThreadprivate(*op, builder, moduleTranslation);6556          })6557          .Case<omp::TargetDataOp, omp::TargetEnterDataOp,6558                omp::TargetExitDataOp, omp::TargetUpdateOp>([&](auto op) {6559            return convertOmpTargetData(op, builder, moduleTranslation);6560          })6561          .Case([&](omp::TargetOp) {6562            return convertOmpTarget(*op, builder, moduleTranslation);6563          })6564          .Case([&](omp::DistributeOp) {6565            return convertOmpDistribute(*op, builder, moduleTranslation);6566          })6567          .Case([&](omp::LoopNestOp) {6568            return convertOmpLoopNest(*op, builder, moduleTranslation);6569          })6570          .Case<omp::MapInfoOp, omp::MapBoundsOp, omp::PrivateClauseOp>(6571              [&](auto op) {6572                // No-op, should be handled by relevant owning operations e.g.6573                // TargetOp, TargetEnterDataOp, TargetExitDataOp, TargetDataOp6574                // etc. and then discarded6575                return success();6576              })6577          .Case([&](omp::NewCliOp op) {6578            // Meta-operation: Doesn't do anything by itself, but used to6579            // identify a loop.6580            return success();6581          })6582          .Case([&](omp::CanonicalLoopOp op) {6583            return convertOmpCanonicalLoopOp(op, builder, moduleTranslation);6584          })6585          .Case([&](omp::UnrollHeuristicOp op) {6586            // FIXME: Handling omp.unroll_heuristic as an executable requires6587            // that the generator (e.g. omp.canonical_loop) has been seen first.6588            // For construct that require all codegen to occur inside a callback6589            // (e.g. OpenMPIRBilder::createParallel), all codegen of that6590            // contained region including their transformations must occur at6591            // the omp.canonical_loop.6592            return applyUnrollHeuristic(op, builder, moduleTranslation);6593          })6594          .Case([&](omp::TileOp op) {6595            return applyTile(op, builder, moduleTranslation);6596          })6597          .Case([&](omp::TargetAllocMemOp) {6598            return convertTargetAllocMemOp(*op, builder, moduleTranslation);6599          })6600          .Case([&](omp::TargetFreeMemOp) {6601            return convertTargetFreeMemOp(*op, builder, moduleTranslation);6602          })6603          .Default([&](Operation *inst) {6604            return inst->emitError()6605                   << "not yet implemented: " << inst->getName();6606          });6607 6608  if (isOutermostLoopWrapper)6609    moduleTranslation.stackPop();6610 6611  return result;6612}6613 6614static LogicalResult6615convertTargetDeviceOp(Operation *op, llvm::IRBuilderBase &builder,6616                      LLVM::ModuleTranslation &moduleTranslation) {6617  return convertHostOrTargetOperation(op, builder, moduleTranslation);6618}6619 6620static LogicalResult6621convertTargetOpsInNest(Operation *op, llvm::IRBuilderBase &builder,6622                       LLVM::ModuleTranslation &moduleTranslation) {6623  if (isa<omp::TargetOp>(op))6624    return convertOmpTarget(*op, builder, moduleTranslation);6625  if (isa<omp::TargetDataOp>(op))6626    return convertOmpTargetData(op, builder, moduleTranslation);6627  bool interrupted =6628      op->walk<WalkOrder::PreOrder>([&](Operation *oper) {6629          if (isa<omp::TargetOp>(oper)) {6630            if (failed(convertOmpTarget(*oper, builder, moduleTranslation)))6631              return WalkResult::interrupt();6632            return WalkResult::skip();6633          }6634          if (isa<omp::TargetDataOp>(oper)) {6635            if (failed(convertOmpTargetData(oper, builder, moduleTranslation)))6636              return WalkResult::interrupt();6637            return WalkResult::skip();6638          }6639 6640          // Non-target ops might nest target-related ops, therefore, we6641          // translate them as non-OpenMP scopes. Translating them is needed by6642          // nested target-related ops since they might need LLVM values defined6643          // in their parent non-target ops.6644          if (isa<omp::OpenMPDialect>(oper->getDialect()) &&6645              oper->getParentOfType<LLVM::LLVMFuncOp>() &&6646              !oper->getRegions().empty()) {6647            if (auto blockArgsIface =6648                    dyn_cast<omp::BlockArgOpenMPOpInterface>(oper))6649              forwardArgs(moduleTranslation, blockArgsIface);6650            else {6651              // Here we map entry block arguments of6652              // non-BlockArgOpenMPOpInterface ops if they can be encountered6653              // inside of a function and they define any of these arguments.6654              if (isa<mlir::omp::AtomicUpdateOp>(oper))6655                for (auto [operand, arg] :6656                     llvm::zip_equal(oper->getOperands(),6657                                     oper->getRegion(0).getArguments())) {6658                  moduleTranslation.mapValue(6659                      arg, builder.CreateLoad(6660                               moduleTranslation.convertType(arg.getType()),6661                               moduleTranslation.lookupValue(operand)));6662                }6663            }6664 6665            if (auto loopNest = dyn_cast<omp::LoopNestOp>(oper)) {6666              assert(builder.GetInsertBlock() &&6667                     "No insert block is set for the builder");6668              for (auto iv : loopNest.getIVs()) {6669                // Map iv to an undefined value just to keep the IR validity.6670                moduleTranslation.mapValue(6671                    iv, llvm::PoisonValue::get(6672                            moduleTranslation.convertType(iv.getType())));6673              }6674            }6675 6676            for (Region &region : oper->getRegions()) {6677              // Regions are fake in the sense that they are not a truthful6678              // translation of the OpenMP construct being converted (e.g. no6679              // OpenMP runtime calls will be generated). We just need this to6680              // prepare the kernel invocation args.6681              SmallVector<llvm::PHINode *> phis;6682              auto result = convertOmpOpRegions(6683                  region, oper->getName().getStringRef().str() + ".fake.region",6684                  builder, moduleTranslation, &phis);6685              if (failed(handleError(result, *oper)))6686                return WalkResult::interrupt();6687 6688              builder.SetInsertPoint(result.get(), result.get()->end());6689            }6690 6691            return WalkResult::skip();6692          }6693 6694          return WalkResult::advance();6695        }).wasInterrupted();6696  return failure(interrupted);6697}6698 6699namespace {6700 6701/// Implementation of the dialect interface that converts operations belonging6702/// to the OpenMP dialect to LLVM IR.6703class OpenMPDialectLLVMIRTranslationInterface6704    : public LLVMTranslationDialectInterface {6705public:6706  using LLVMTranslationDialectInterface::LLVMTranslationDialectInterface;6707 6708  /// Translates the given operation to LLVM IR using the provided IR builder6709  /// and saving the state in `moduleTranslation`.6710  LogicalResult6711  convertOperation(Operation *op, llvm::IRBuilderBase &builder,6712                   LLVM::ModuleTranslation &moduleTranslation) const final;6713 6714  /// Given an OpenMP MLIR attribute, create the corresponding LLVM-IR,6715  /// runtime calls, or operation amendments6716  LogicalResult6717  amendOperation(Operation *op, ArrayRef<llvm::Instruction *> instructions,6718                 NamedAttribute attribute,6719                 LLVM::ModuleTranslation &moduleTranslation) const final;6720};6721 6722} // namespace6723 6724LogicalResult OpenMPDialectLLVMIRTranslationInterface::amendOperation(6725    Operation *op, ArrayRef<llvm::Instruction *> instructions,6726    NamedAttribute attribute,6727    LLVM::ModuleTranslation &moduleTranslation) const {6728  return llvm::StringSwitch<llvm::function_ref<LogicalResult(Attribute)>>(6729             attribute.getName())6730      .Case("omp.is_target_device",6731            [&](Attribute attr) {6732              if (auto deviceAttr = dyn_cast<BoolAttr>(attr)) {6733                llvm::OpenMPIRBuilderConfig &config =6734                    moduleTranslation.getOpenMPBuilder()->Config;6735                config.setIsTargetDevice(deviceAttr.getValue());6736                return success();6737              }6738              return failure();6739            })6740      .Case("omp.is_gpu",6741            [&](Attribute attr) {6742              if (auto gpuAttr = dyn_cast<BoolAttr>(attr)) {6743                llvm::OpenMPIRBuilderConfig &config =6744                    moduleTranslation.getOpenMPBuilder()->Config;6745                config.setIsGPU(gpuAttr.getValue());6746                return success();6747              }6748              return failure();6749            })6750      .Case("omp.host_ir_filepath",6751            [&](Attribute attr) {6752              if (auto filepathAttr = dyn_cast<StringAttr>(attr)) {6753                llvm::OpenMPIRBuilder *ompBuilder =6754                    moduleTranslation.getOpenMPBuilder();6755                auto VFS = llvm::vfs::getRealFileSystem();6756                ompBuilder->loadOffloadInfoMetadata(*VFS,6757                                                    filepathAttr.getValue());6758                return success();6759              }6760              return failure();6761            })6762      .Case("omp.flags",6763            [&](Attribute attr) {6764              if (auto rtlAttr = dyn_cast<omp::FlagsAttr>(attr))6765                return convertFlagsAttr(op, rtlAttr, moduleTranslation);6766              return failure();6767            })6768      .Case("omp.version",6769            [&](Attribute attr) {6770              if (auto versionAttr = dyn_cast<omp::VersionAttr>(attr)) {6771                llvm::OpenMPIRBuilder *ompBuilder =6772                    moduleTranslation.getOpenMPBuilder();6773                ompBuilder->M.addModuleFlag(llvm::Module::Max, "openmp",6774                                            versionAttr.getVersion());6775                return success();6776              }6777              return failure();6778            })6779      .Case("omp.declare_target",6780            [&](Attribute attr) {6781              if (auto declareTargetAttr =6782                      dyn_cast<omp::DeclareTargetAttr>(attr))6783                return convertDeclareTargetAttr(op, declareTargetAttr,6784                                                moduleTranslation);6785              return failure();6786            })6787      .Case("omp.requires",6788            [&](Attribute attr) {6789              if (auto requiresAttr = dyn_cast<omp::ClauseRequiresAttr>(attr)) {6790                using Requires = omp::ClauseRequires;6791                Requires flags = requiresAttr.getValue();6792                llvm::OpenMPIRBuilderConfig &config =6793                    moduleTranslation.getOpenMPBuilder()->Config;6794                config.setHasRequiresReverseOffload(6795                    bitEnumContainsAll(flags, Requires::reverse_offload));6796                config.setHasRequiresUnifiedAddress(6797                    bitEnumContainsAll(flags, Requires::unified_address));6798                config.setHasRequiresUnifiedSharedMemory(6799                    bitEnumContainsAll(flags, Requires::unified_shared_memory));6800                config.setHasRequiresDynamicAllocators(6801                    bitEnumContainsAll(flags, Requires::dynamic_allocators));6802                return success();6803              }6804              return failure();6805            })6806      .Case("omp.target_triples",6807            [&](Attribute attr) {6808              if (auto triplesAttr = dyn_cast<ArrayAttr>(attr)) {6809                llvm::OpenMPIRBuilderConfig &config =6810                    moduleTranslation.getOpenMPBuilder()->Config;6811                config.TargetTriples.clear();6812                config.TargetTriples.reserve(triplesAttr.size());6813                for (Attribute tripleAttr : triplesAttr) {6814                  if (auto tripleStrAttr = dyn_cast<StringAttr>(tripleAttr))6815                    config.TargetTriples.emplace_back(tripleStrAttr.getValue());6816                  else6817                    return failure();6818                }6819                return success();6820              }6821              return failure();6822            })6823      .Default([](Attribute) {6824        // Fall through for omp attributes that do not require lowering.6825        return success();6826      })(attribute.getValue());6827 6828  return failure();6829}6830 6831/// Given an OpenMP MLIR operation, create the corresponding LLVM IR6832/// (including OpenMP runtime calls).6833LogicalResult OpenMPDialectLLVMIRTranslationInterface::convertOperation(6834    Operation *op, llvm::IRBuilderBase &builder,6835    LLVM::ModuleTranslation &moduleTranslation) const {6836 6837  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();6838  if (ompBuilder->Config.isTargetDevice()) {6839    if (isTargetDeviceOp(op)) {6840      return convertTargetDeviceOp(op, builder, moduleTranslation);6841    }6842    return convertTargetOpsInNest(op, builder, moduleTranslation);6843  }6844  return convertHostOrTargetOperation(op, builder, moduleTranslation);6845}6846 6847void mlir::registerOpenMPDialectTranslation(DialectRegistry &registry) {6848  registry.insert<omp::OpenMPDialect>();6849  registry.addExtension(+[](MLIRContext *ctx, omp::OpenMPDialect *dialect) {6850    dialect->addInterfaces<OpenMPDialectLLVMIRTranslationInterface>();6851  });6852}6853 6854void mlir::registerOpenMPDialectTranslation(MLIRContext &context) {6855  DialectRegistry registry;6856  registerOpenMPDialectTranslation(registry);6857  context.appendDialectRegistry(registry);6858}6859