brintos / llvm-project-archived public Read only

1//===- OpenMPIRBuilder.cpp - Builder for LLVM-IR for OpenMP directives ----===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8/// \file9///10/// This file implements the OpenMPIRBuilder class, which is used as a11/// convenient way to create LLVM instructions for OpenMP directives.12///13//===----------------------------------------------------------------------===//14 15#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"16#include "llvm/ADT/SmallBitVector.h"17#include "llvm/ADT/SmallSet.h"18#include "llvm/ADT/StringExtras.h"19#include "llvm/ADT/StringRef.h"20#include "llvm/Analysis/AssumptionCache.h"21#include "llvm/Analysis/CodeMetrics.h"22#include "llvm/Analysis/LoopInfo.h"23#include "llvm/Analysis/OptimizationRemarkEmitter.h"24#include "llvm/Analysis/ScalarEvolution.h"25#include "llvm/Analysis/TargetLibraryInfo.h"26#include "llvm/Bitcode/BitcodeReader.h"27#include "llvm/Frontend/Offloading/Utility.h"28#include "llvm/Frontend/OpenMP/OMPGridValues.h"29#include "llvm/IR/Attributes.h"30#include "llvm/IR/BasicBlock.h"31#include "llvm/IR/CFG.h"32#include "llvm/IR/CallingConv.h"33#include "llvm/IR/Constant.h"34#include "llvm/IR/Constants.h"35#include "llvm/IR/DIBuilder.h"36#include "llvm/IR/DebugInfoMetadata.h"37#include "llvm/IR/DerivedTypes.h"38#include "llvm/IR/Function.h"39#include "llvm/IR/GlobalVariable.h"40#include "llvm/IR/IRBuilder.h"41#include "llvm/IR/InstIterator.h"42#include "llvm/IR/IntrinsicInst.h"43#include "llvm/IR/LLVMContext.h"44#include "llvm/IR/MDBuilder.h"45#include "llvm/IR/Metadata.h"46#include "llvm/IR/PassInstrumentation.h"47#include "llvm/IR/PassManager.h"48#include "llvm/IR/ReplaceConstant.h"49#include "llvm/IR/Value.h"50#include "llvm/MC/TargetRegistry.h"51#include "llvm/Support/CommandLine.h"52#include "llvm/Support/ErrorHandling.h"53#include "llvm/Support/FileSystem.h"54#include "llvm/Support/VirtualFileSystem.h"55#include "llvm/Target/TargetMachine.h"56#include "llvm/Target/TargetOptions.h"57#include "llvm/Transforms/Utils/BasicBlockUtils.h"58#include "llvm/Transforms/Utils/Cloning.h"59#include "llvm/Transforms/Utils/CodeExtractor.h"60#include "llvm/Transforms/Utils/LoopPeel.h"61#include "llvm/Transforms/Utils/UnrollLoop.h"62 63#include <cstdint>64#include <optional>65 66#define DEBUG_TYPE "openmp-ir-builder"67 68using namespace llvm;69using namespace omp;70 71static cl::opt<bool>72    OptimisticAttributes("openmp-ir-builder-optimistic-attributes", cl::Hidden,73                         cl::desc("Use optimistic attributes describing "74                                  "'as-if' properties of runtime calls."),75                         cl::init(false));76 77static cl::opt<double> UnrollThresholdFactor(78    "openmp-ir-builder-unroll-threshold-factor", cl::Hidden,79    cl::desc("Factor for the unroll threshold to account for code "80             "simplifications still taking place"),81    cl::init(1.5));82 83#ifndef NDEBUG84/// Return whether IP1 and IP2 are ambiguous, i.e. that inserting instructions85/// at position IP1 may change the meaning of IP2 or vice-versa. This is because86/// an InsertPoint stores the instruction before something is inserted. For87/// instance, if both point to the same instruction, two IRBuilders alternating88/// creating instruction will cause the instructions to be interleaved.89static bool isConflictIP(IRBuilder<>::InsertPoint IP1,90                         IRBuilder<>::InsertPoint IP2) {91  if (!IP1.isSet() || !IP2.isSet())92    return false;93  return IP1.getBlock() == IP2.getBlock() && IP1.getPoint() == IP2.getPoint();94}95 96static bool isValidWorkshareLoopScheduleType(OMPScheduleType SchedType) {97  // Valid ordered/unordered and base algorithm combinations.98  switch (SchedType & ~OMPScheduleType::MonotonicityMask) {99  case OMPScheduleType::UnorderedStaticChunked:100  case OMPScheduleType::UnorderedStatic:101  case OMPScheduleType::UnorderedDynamicChunked:102  case OMPScheduleType::UnorderedGuidedChunked:103  case OMPScheduleType::UnorderedRuntime:104  case OMPScheduleType::UnorderedAuto:105  case OMPScheduleType::UnorderedTrapezoidal:106  case OMPScheduleType::UnorderedGreedy:107  case OMPScheduleType::UnorderedBalanced:108  case OMPScheduleType::UnorderedGuidedIterativeChunked:109  case OMPScheduleType::UnorderedGuidedAnalyticalChunked:110  case OMPScheduleType::UnorderedSteal:111  case OMPScheduleType::UnorderedStaticBalancedChunked:112  case OMPScheduleType::UnorderedGuidedSimd:113  case OMPScheduleType::UnorderedRuntimeSimd:114  case OMPScheduleType::OrderedStaticChunked:115  case OMPScheduleType::OrderedStatic:116  case OMPScheduleType::OrderedDynamicChunked:117  case OMPScheduleType::OrderedGuidedChunked:118  case OMPScheduleType::OrderedRuntime:119  case OMPScheduleType::OrderedAuto:120  case OMPScheduleType::OrderdTrapezoidal:121  case OMPScheduleType::NomergeUnorderedStaticChunked:122  case OMPScheduleType::NomergeUnorderedStatic:123  case OMPScheduleType::NomergeUnorderedDynamicChunked:124  case OMPScheduleType::NomergeUnorderedGuidedChunked:125  case OMPScheduleType::NomergeUnorderedRuntime:126  case OMPScheduleType::NomergeUnorderedAuto:127  case OMPScheduleType::NomergeUnorderedTrapezoidal:128  case OMPScheduleType::NomergeUnorderedGreedy:129  case OMPScheduleType::NomergeUnorderedBalanced:130  case OMPScheduleType::NomergeUnorderedGuidedIterativeChunked:131  case OMPScheduleType::NomergeUnorderedGuidedAnalyticalChunked:132  case OMPScheduleType::NomergeUnorderedSteal:133  case OMPScheduleType::NomergeOrderedStaticChunked:134  case OMPScheduleType::NomergeOrderedStatic:135  case OMPScheduleType::NomergeOrderedDynamicChunked:136  case OMPScheduleType::NomergeOrderedGuidedChunked:137  case OMPScheduleType::NomergeOrderedRuntime:138  case OMPScheduleType::NomergeOrderedAuto:139  case OMPScheduleType::NomergeOrderedTrapezoidal:140  case OMPScheduleType::OrderedDistributeChunked:141  case OMPScheduleType::OrderedDistribute:142    break;143  default:144    return false;145  }146 147  // Must not set both monotonicity modifiers at the same time.148  OMPScheduleType MonotonicityFlags =149      SchedType & OMPScheduleType::MonotonicityMask;150  if (MonotonicityFlags == OMPScheduleType::MonotonicityMask)151    return false;152 153  return true;154}155#endif156 157/// This is wrapper over IRBuilderBase::restoreIP that also restores the current158/// debug location to the last instruction in the specified basic block if the159/// insert point points to the end of the block.160static void restoreIPandDebugLoc(llvm::IRBuilderBase &Builder,161                                 llvm::IRBuilderBase::InsertPoint IP) {162  Builder.restoreIP(IP);163  llvm::BasicBlock *BB = Builder.GetInsertBlock();164  llvm::BasicBlock::iterator I = Builder.GetInsertPoint();165  if (!BB->empty() && I == BB->end())166    Builder.SetCurrentDebugLocation(BB->back().getStableDebugLoc());167}168 169static const omp::GV &getGridValue(const Triple &T, Function *Kernel) {170  if (T.isAMDGPU()) {171    StringRef Features =172        Kernel->getFnAttribute("target-features").getValueAsString();173    if (Features.count("+wavefrontsize64"))174      return omp::getAMDGPUGridValues<64>();175    return omp::getAMDGPUGridValues<32>();176  }177  if (T.isNVPTX())178    return omp::NVPTXGridValues;179  if (T.isSPIRV())180    return omp::SPIRVGridValues;181  llvm_unreachable("No grid value available for this architecture!");182}183 184/// Determine which scheduling algorithm to use, determined from schedule clause185/// arguments.186static OMPScheduleType187getOpenMPBaseScheduleType(llvm::omp::ScheduleKind ClauseKind, bool HasChunks,188                          bool HasSimdModifier, bool HasDistScheduleChunks) {189  // Currently, the default schedule it static.190  switch (ClauseKind) {191  case OMP_SCHEDULE_Default:192  case OMP_SCHEDULE_Static:193    return HasChunks ? OMPScheduleType::BaseStaticChunked194                     : OMPScheduleType::BaseStatic;195  case OMP_SCHEDULE_Dynamic:196    return OMPScheduleType::BaseDynamicChunked;197  case OMP_SCHEDULE_Guided:198    return HasSimdModifier ? OMPScheduleType::BaseGuidedSimd199                           : OMPScheduleType::BaseGuidedChunked;200  case OMP_SCHEDULE_Auto:201    return llvm::omp::OMPScheduleType::BaseAuto;202  case OMP_SCHEDULE_Runtime:203    return HasSimdModifier ? OMPScheduleType::BaseRuntimeSimd204                           : OMPScheduleType::BaseRuntime;205  case OMP_SCHEDULE_Distribute:206    return HasDistScheduleChunks ? OMPScheduleType::BaseDistributeChunked207                                 : OMPScheduleType::BaseDistribute;208  }209  llvm_unreachable("unhandled schedule clause argument");210}211 212/// Adds ordering modifier flags to schedule type.213static OMPScheduleType214getOpenMPOrderingScheduleType(OMPScheduleType BaseScheduleType,215                              bool HasOrderedClause) {216  assert((BaseScheduleType & OMPScheduleType::ModifierMask) ==217             OMPScheduleType::None &&218         "Must not have ordering nor monotonicity flags already set");219 220  OMPScheduleType OrderingModifier = HasOrderedClause221                                         ? OMPScheduleType::ModifierOrdered222                                         : OMPScheduleType::ModifierUnordered;223  OMPScheduleType OrderingScheduleType = BaseScheduleType | OrderingModifier;224 225  // Unsupported combinations226  if (OrderingScheduleType ==227      (OMPScheduleType::BaseGuidedSimd | OMPScheduleType::ModifierOrdered))228    return OMPScheduleType::OrderedGuidedChunked;229  else if (OrderingScheduleType == (OMPScheduleType::BaseRuntimeSimd |230                                    OMPScheduleType::ModifierOrdered))231    return OMPScheduleType::OrderedRuntime;232 233  return OrderingScheduleType;234}235 236/// Adds monotonicity modifier flags to schedule type.237static OMPScheduleType238getOpenMPMonotonicityScheduleType(OMPScheduleType ScheduleType,239                                  bool HasSimdModifier, bool HasMonotonic,240                                  bool HasNonmonotonic, bool HasOrderedClause) {241  assert((ScheduleType & OMPScheduleType::MonotonicityMask) ==242             OMPScheduleType::None &&243         "Must not have monotonicity flags already set");244  assert((!HasMonotonic || !HasNonmonotonic) &&245         "Monotonic and Nonmonotonic are contradicting each other");246 247  if (HasMonotonic) {248    return ScheduleType | OMPScheduleType::ModifierMonotonic;249  } else if (HasNonmonotonic) {250    return ScheduleType | OMPScheduleType::ModifierNonmonotonic;251  } else {252    // OpenMP 5.1, 2.11.4 Worksharing-Loop Construct, Description.253    // If the static schedule kind is specified or if the ordered clause is254    // specified, and if the nonmonotonic modifier is not specified, the255    // effect is as if the monotonic modifier is specified. Otherwise, unless256    // the monotonic modifier is specified, the effect is as if the257    // nonmonotonic modifier is specified.258    OMPScheduleType BaseScheduleType =259        ScheduleType & ~OMPScheduleType::ModifierMask;260    if ((BaseScheduleType == OMPScheduleType::BaseStatic) ||261        (BaseScheduleType == OMPScheduleType::BaseStaticChunked) ||262        HasOrderedClause) {263      // The monotonic is used by default in openmp runtime library, so no need264      // to set it.265      return ScheduleType;266    } else {267      return ScheduleType | OMPScheduleType::ModifierNonmonotonic;268    }269  }270}271 272/// Determine the schedule type using schedule and ordering clause arguments.273static OMPScheduleType274computeOpenMPScheduleType(ScheduleKind ClauseKind, bool HasChunks,275                          bool HasSimdModifier, bool HasMonotonicModifier,276                          bool HasNonmonotonicModifier, bool HasOrderedClause,277                          bool HasDistScheduleChunks) {278  OMPScheduleType BaseSchedule = getOpenMPBaseScheduleType(279      ClauseKind, HasChunks, HasSimdModifier, HasDistScheduleChunks);280  OMPScheduleType OrderedSchedule =281      getOpenMPOrderingScheduleType(BaseSchedule, HasOrderedClause);282  OMPScheduleType Result = getOpenMPMonotonicityScheduleType(283      OrderedSchedule, HasSimdModifier, HasMonotonicModifier,284      HasNonmonotonicModifier, HasOrderedClause);285 286  assert(isValidWorkshareLoopScheduleType(Result));287  return Result;288}289 290/// Make \p Source branch to \p Target.291///292/// Handles two situations:293/// * \p Source already has an unconditional branch.294/// * \p Source is a degenerate block (no terminator because the BB is295///             the current head of the IR construction).296static void redirectTo(BasicBlock *Source, BasicBlock *Target, DebugLoc DL) {297  if (Instruction *Term = Source->getTerminator()) {298    auto *Br = cast<BranchInst>(Term);299    assert(!Br->isConditional() &&300           "BB's terminator must be an unconditional branch (or degenerate)");301    BasicBlock *Succ = Br->getSuccessor(0);302    Succ->removePredecessor(Source, /*KeepOneInputPHIs=*/true);303    Br->setSuccessor(0, Target);304    return;305  }306 307  auto *NewBr = BranchInst::Create(Target, Source);308  NewBr->setDebugLoc(DL);309}310 311void llvm::spliceBB(IRBuilderBase::InsertPoint IP, BasicBlock *New,312                    bool CreateBranch, DebugLoc DL) {313  assert(New->getFirstInsertionPt() == New->begin() &&314         "Target BB must not have PHI nodes");315 316  // Move instructions to new block.317  BasicBlock *Old = IP.getBlock();318  // If the `Old` block is empty then there are no instructions to move. But in319  // the new debug scheme, it could have trailing debug records which will be320  // moved to `New` in `spliceDebugInfoEmptyBlock`. We dont want that for 2321  // reasons:322  // 1. If `New` is also empty, `BasicBlock::splice` crashes.323  // 2. Even if `New` is not empty, the rationale to move those records to `New`324  // (in `spliceDebugInfoEmptyBlock`) does not apply here. That function325  // assumes that `Old` is optimized out and is going away. This is not the case326  // here. The `Old` block is still being used e.g. a branch instruction is327  // added to it later in this function.328  // So we call `BasicBlock::splice` only when `Old` is not empty.329  if (!Old->empty())330    New->splice(New->begin(), Old, IP.getPoint(), Old->end());331 332  if (CreateBranch) {333    auto *NewBr = BranchInst::Create(New, Old);334    NewBr->setDebugLoc(DL);335  }336}337 338void llvm::spliceBB(IRBuilder<> &Builder, BasicBlock *New, bool CreateBranch) {339  DebugLoc DebugLoc = Builder.getCurrentDebugLocation();340  BasicBlock *Old = Builder.GetInsertBlock();341 342  spliceBB(Builder.saveIP(), New, CreateBranch, DebugLoc);343  if (CreateBranch)344    Builder.SetInsertPoint(Old->getTerminator());345  else346    Builder.SetInsertPoint(Old);347 348  // SetInsertPoint also updates the Builder's debug location, but we want to349  // keep the one the Builder was configured to use.350  Builder.SetCurrentDebugLocation(DebugLoc);351}352 353BasicBlock *llvm::splitBB(IRBuilderBase::InsertPoint IP, bool CreateBranch,354                          DebugLoc DL, llvm::Twine Name) {355  BasicBlock *Old = IP.getBlock();356  BasicBlock *New = BasicBlock::Create(357      Old->getContext(), Name.isTriviallyEmpty() ? Old->getName() : Name,358      Old->getParent(), Old->getNextNode());359  spliceBB(IP, New, CreateBranch, DL);360  New->replaceSuccessorsPhiUsesWith(Old, New);361  return New;362}363 364BasicBlock *llvm::splitBB(IRBuilderBase &Builder, bool CreateBranch,365                          llvm::Twine Name) {366  DebugLoc DebugLoc = Builder.getCurrentDebugLocation();367  BasicBlock *New = splitBB(Builder.saveIP(), CreateBranch, DebugLoc, Name);368  if (CreateBranch)369    Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());370  else371    Builder.SetInsertPoint(Builder.GetInsertBlock());372  // SetInsertPoint also updates the Builder's debug location, but we want to373  // keep the one the Builder was configured to use.374  Builder.SetCurrentDebugLocation(DebugLoc);375  return New;376}377 378BasicBlock *llvm::splitBB(IRBuilder<> &Builder, bool CreateBranch,379                          llvm::Twine Name) {380  DebugLoc DebugLoc = Builder.getCurrentDebugLocation();381  BasicBlock *New = splitBB(Builder.saveIP(), CreateBranch, DebugLoc, Name);382  if (CreateBranch)383    Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());384  else385    Builder.SetInsertPoint(Builder.GetInsertBlock());386  // SetInsertPoint also updates the Builder's debug location, but we want to387  // keep the one the Builder was configured to use.388  Builder.SetCurrentDebugLocation(DebugLoc);389  return New;390}391 392BasicBlock *llvm::splitBBWithSuffix(IRBuilderBase &Builder, bool CreateBranch,393                                    llvm::Twine Suffix) {394  BasicBlock *Old = Builder.GetInsertBlock();395  return splitBB(Builder, CreateBranch, Old->getName() + Suffix);396}397 398// This function creates a fake integer value and a fake use for the integer399// value. It returns the fake value created. This is useful in modeling the400// extra arguments to the outlined functions.401Value *createFakeIntVal(IRBuilderBase &Builder,402                        OpenMPIRBuilder::InsertPointTy OuterAllocaIP,403                        llvm::SmallVectorImpl<Instruction *> &ToBeDeleted,404                        OpenMPIRBuilder::InsertPointTy InnerAllocaIP,405                        const Twine &Name = "", bool AsPtr = true) {406  Builder.restoreIP(OuterAllocaIP);407  Instruction *FakeVal;408  AllocaInst *FakeValAddr =409      Builder.CreateAlloca(Builder.getInt32Ty(), nullptr, Name + ".addr");410  ToBeDeleted.push_back(FakeValAddr);411 412  if (AsPtr) {413    FakeVal = FakeValAddr;414  } else {415    FakeVal =416        Builder.CreateLoad(Builder.getInt32Ty(), FakeValAddr, Name + ".val");417    ToBeDeleted.push_back(FakeVal);418  }419 420  // Generate a fake use of this value421  Builder.restoreIP(InnerAllocaIP);422  Instruction *UseFakeVal;423  if (AsPtr) {424    UseFakeVal =425        Builder.CreateLoad(Builder.getInt32Ty(), FakeVal, Name + ".use");426  } else {427    UseFakeVal =428        cast<BinaryOperator>(Builder.CreateAdd(FakeVal, Builder.getInt32(10)));429  }430  ToBeDeleted.push_back(UseFakeVal);431  return FakeVal;432}433 434//===----------------------------------------------------------------------===//435// OpenMPIRBuilderConfig436//===----------------------------------------------------------------------===//437 438namespace {439LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();440/// Values for bit flags for marking which requires clauses have been used.441enum OpenMPOffloadingRequiresDirFlags {442  /// flag undefined.443  OMP_REQ_UNDEFINED = 0x000,444  /// no requires directive present.445  OMP_REQ_NONE = 0x001,446  /// reverse_offload clause.447  OMP_REQ_REVERSE_OFFLOAD = 0x002,448  /// unified_address clause.449  OMP_REQ_UNIFIED_ADDRESS = 0x004,450  /// unified_shared_memory clause.451  OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,452  /// dynamic_allocators clause.453  OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,454  LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_REQ_DYNAMIC_ALLOCATORS)455};456 457} // anonymous namespace458 459OpenMPIRBuilderConfig::OpenMPIRBuilderConfig()460    : RequiresFlags(OMP_REQ_UNDEFINED) {}461 462OpenMPIRBuilderConfig::OpenMPIRBuilderConfig(463    bool IsTargetDevice, bool IsGPU, bool OpenMPOffloadMandatory,464    bool HasRequiresReverseOffload, bool HasRequiresUnifiedAddress,465    bool HasRequiresUnifiedSharedMemory, bool HasRequiresDynamicAllocators)466    : IsTargetDevice(IsTargetDevice), IsGPU(IsGPU),467      OpenMPOffloadMandatory(OpenMPOffloadMandatory),468      RequiresFlags(OMP_REQ_UNDEFINED) {469  if (HasRequiresReverseOffload)470    RequiresFlags |= OMP_REQ_REVERSE_OFFLOAD;471  if (HasRequiresUnifiedAddress)472    RequiresFlags |= OMP_REQ_UNIFIED_ADDRESS;473  if (HasRequiresUnifiedSharedMemory)474    RequiresFlags |= OMP_REQ_UNIFIED_SHARED_MEMORY;475  if (HasRequiresDynamicAllocators)476    RequiresFlags |= OMP_REQ_DYNAMIC_ALLOCATORS;477}478 479bool OpenMPIRBuilderConfig::hasRequiresReverseOffload() const {480  return RequiresFlags & OMP_REQ_REVERSE_OFFLOAD;481}482 483bool OpenMPIRBuilderConfig::hasRequiresUnifiedAddress() const {484  return RequiresFlags & OMP_REQ_UNIFIED_ADDRESS;485}486 487bool OpenMPIRBuilderConfig::hasRequiresUnifiedSharedMemory() const {488  return RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY;489}490 491bool OpenMPIRBuilderConfig::hasRequiresDynamicAllocators() const {492  return RequiresFlags & OMP_REQ_DYNAMIC_ALLOCATORS;493}494 495int64_t OpenMPIRBuilderConfig::getRequiresFlags() const {496  return hasRequiresFlags() ? RequiresFlags497                            : static_cast<int64_t>(OMP_REQ_NONE);498}499 500void OpenMPIRBuilderConfig::setHasRequiresReverseOffload(bool Value) {501  if (Value)502    RequiresFlags |= OMP_REQ_REVERSE_OFFLOAD;503  else504    RequiresFlags &= ~OMP_REQ_REVERSE_OFFLOAD;505}506 507void OpenMPIRBuilderConfig::setHasRequiresUnifiedAddress(bool Value) {508  if (Value)509    RequiresFlags |= OMP_REQ_UNIFIED_ADDRESS;510  else511    RequiresFlags &= ~OMP_REQ_UNIFIED_ADDRESS;512}513 514void OpenMPIRBuilderConfig::setHasRequiresUnifiedSharedMemory(bool Value) {515  if (Value)516    RequiresFlags |= OMP_REQ_UNIFIED_SHARED_MEMORY;517  else518    RequiresFlags &= ~OMP_REQ_UNIFIED_SHARED_MEMORY;519}520 521void OpenMPIRBuilderConfig::setHasRequiresDynamicAllocators(bool Value) {522  if (Value)523    RequiresFlags |= OMP_REQ_DYNAMIC_ALLOCATORS;524  else525    RequiresFlags &= ~OMP_REQ_DYNAMIC_ALLOCATORS;526}527 528//===----------------------------------------------------------------------===//529// OpenMPIRBuilder530//===----------------------------------------------------------------------===//531 532void OpenMPIRBuilder::getKernelArgsVector(TargetKernelArgs &KernelArgs,533                                          IRBuilderBase &Builder,534                                          SmallVector<Value *> &ArgsVector) {535  Value *Version = Builder.getInt32(OMP_KERNEL_ARG_VERSION);536  Value *PointerNum = Builder.getInt32(KernelArgs.NumTargetItems);537  auto Int32Ty = Type::getInt32Ty(Builder.getContext());538  constexpr size_t MaxDim = 3;539  Value *ZeroArray = Constant::getNullValue(ArrayType::get(Int32Ty, MaxDim));540 541  Value *HasNoWaitFlag = Builder.getInt64(KernelArgs.HasNoWait);542 543  Value *DynCGroupMemFallbackFlag =544      Builder.getInt64(static_cast<uint64_t>(KernelArgs.DynCGroupMemFallback));545  DynCGroupMemFallbackFlag = Builder.CreateShl(DynCGroupMemFallbackFlag, 2);546  Value *Flags = Builder.CreateOr(HasNoWaitFlag, DynCGroupMemFallbackFlag);547 548  assert(!KernelArgs.NumTeams.empty() && !KernelArgs.NumThreads.empty());549 550  Value *NumTeams3D =551      Builder.CreateInsertValue(ZeroArray, KernelArgs.NumTeams[0], {0});552  Value *NumThreads3D =553      Builder.CreateInsertValue(ZeroArray, KernelArgs.NumThreads[0], {0});554  for (unsigned I :555       seq<unsigned>(1, std::min(KernelArgs.NumTeams.size(), MaxDim)))556    NumTeams3D =557        Builder.CreateInsertValue(NumTeams3D, KernelArgs.NumTeams[I], {I});558  for (unsigned I :559       seq<unsigned>(1, std::min(KernelArgs.NumThreads.size(), MaxDim)))560    NumThreads3D =561        Builder.CreateInsertValue(NumThreads3D, KernelArgs.NumThreads[I], {I});562 563  ArgsVector = {Version,564                PointerNum,565                KernelArgs.RTArgs.BasePointersArray,566                KernelArgs.RTArgs.PointersArray,567                KernelArgs.RTArgs.SizesArray,568                KernelArgs.RTArgs.MapTypesArray,569                KernelArgs.RTArgs.MapNamesArray,570                KernelArgs.RTArgs.MappersArray,571                KernelArgs.NumIterations,572                Flags,573                NumTeams3D,574                NumThreads3D,575                KernelArgs.DynCGroupMem};576}577 578void OpenMPIRBuilder::addAttributes(omp::RuntimeFunction FnID, Function &Fn) {579  LLVMContext &Ctx = Fn.getContext();580 581  // Get the function's current attributes.582  auto Attrs = Fn.getAttributes();583  auto FnAttrs = Attrs.getFnAttrs();584  auto RetAttrs = Attrs.getRetAttrs();585  SmallVector<AttributeSet, 4> ArgAttrs;586  for (size_t ArgNo = 0; ArgNo < Fn.arg_size(); ++ArgNo)587    ArgAttrs.emplace_back(Attrs.getParamAttrs(ArgNo));588 589  // Add AS to FnAS while taking special care with integer extensions.590  auto addAttrSet = [&](AttributeSet &FnAS, const AttributeSet &AS,591                        bool Param = true) -> void {592    bool HasSignExt = AS.hasAttribute(Attribute::SExt);593    bool HasZeroExt = AS.hasAttribute(Attribute::ZExt);594    if (HasSignExt || HasZeroExt) {595      assert(AS.getNumAttributes() == 1 &&596             "Currently not handling extension attr combined with others.");597      if (Param) {598        if (auto AK = TargetLibraryInfo::getExtAttrForI32Param(T, HasSignExt))599          FnAS = FnAS.addAttribute(Ctx, AK);600      } else if (auto AK =601                     TargetLibraryInfo::getExtAttrForI32Return(T, HasSignExt))602        FnAS = FnAS.addAttribute(Ctx, AK);603    } else {604      FnAS = FnAS.addAttributes(Ctx, AS);605    }606  };607 608#define OMP_ATTRS_SET(VarName, AttrSet) AttributeSet VarName = AttrSet;609#include "llvm/Frontend/OpenMP/OMPKinds.def"610 611  // Add attributes to the function declaration.612  switch (FnID) {613#define OMP_RTL_ATTRS(Enum, FnAttrSet, RetAttrSet, ArgAttrSets)                \614  case Enum:                                                                   \615    FnAttrs = FnAttrs.addAttributes(Ctx, FnAttrSet);                           \616    addAttrSet(RetAttrs, RetAttrSet, /*Param*/ false);                         \617    for (size_t ArgNo = 0; ArgNo < ArgAttrSets.size(); ++ArgNo)                \618      addAttrSet(ArgAttrs[ArgNo], ArgAttrSets[ArgNo]);                         \619    Fn.setAttributes(AttributeList::get(Ctx, FnAttrs, RetAttrs, ArgAttrs));    \620    break;621#include "llvm/Frontend/OpenMP/OMPKinds.def"622  default:623    // Attributes are optional.624    break;625  }626}627 628FunctionCallee629OpenMPIRBuilder::getOrCreateRuntimeFunction(Module &M, RuntimeFunction FnID) {630  FunctionType *FnTy = nullptr;631  Function *Fn = nullptr;632 633  // Try to find the declation in the module first.634  switch (FnID) {635#define OMP_RTL(Enum, Str, IsVarArg, ReturnType, ...)                          \636  case Enum:                                                                   \637    FnTy = FunctionType::get(ReturnType, ArrayRef<Type *>{__VA_ARGS__},        \638                             IsVarArg);                                        \639    Fn = M.getFunction(Str);                                                   \640    break;641#include "llvm/Frontend/OpenMP/OMPKinds.def"642  }643 644  if (!Fn) {645    // Create a new declaration if we need one.646    switch (FnID) {647#define OMP_RTL(Enum, Str, ...)                                                \648  case Enum:                                                                   \649    Fn = Function::Create(FnTy, GlobalValue::ExternalLinkage, Str, M);         \650    break;651#include "llvm/Frontend/OpenMP/OMPKinds.def"652    }653    Fn->setCallingConv(Config.getRuntimeCC());654    // Add information if the runtime function takes a callback function655    if (FnID == OMPRTL___kmpc_fork_call || FnID == OMPRTL___kmpc_fork_teams) {656      if (!Fn->hasMetadata(LLVMContext::MD_callback)) {657        LLVMContext &Ctx = Fn->getContext();658        MDBuilder MDB(Ctx);659        // Annotate the callback behavior of the runtime function:660        //  - The callback callee is argument number 2 (microtask).661        //  - The first two arguments of the callback callee are unknown (-1).662        //  - All variadic arguments to the runtime function are passed to the663        //    callback callee.664        Fn->addMetadata(665            LLVMContext::MD_callback,666            *MDNode::get(Ctx, {MDB.createCallbackEncoding(667                                  2, {-1, -1}, /* VarArgsArePassed */ true)}));668      }669    }670 671    LLVM_DEBUG(dbgs() << "Created OpenMP runtime function " << Fn->getName()672                      << " with type " << *Fn->getFunctionType() << "\n");673    addAttributes(FnID, *Fn);674 675  } else {676    LLVM_DEBUG(dbgs() << "Found OpenMP runtime function " << Fn->getName()677                      << " with type " << *Fn->getFunctionType() << "\n");678  }679 680  assert(Fn && "Failed to create OpenMP runtime function");681 682  return {FnTy, Fn};683}684 685Expected<BasicBlock *>686OpenMPIRBuilder::FinalizationInfo::getFiniBB(IRBuilderBase &Builder) {687  if (!FiniBB) {688    Function *ParentFunc = Builder.GetInsertBlock()->getParent();689    IRBuilderBase::InsertPointGuard Guard(Builder);690    FiniBB = BasicBlock::Create(Builder.getContext(), ".fini", ParentFunc);691    Builder.SetInsertPoint(FiniBB);692    // FiniCB adds the branch to the exit stub.693    if (Error Err = FiniCB(Builder.saveIP()))694      return Err;695  }696  return FiniBB;697}698 699Error OpenMPIRBuilder::FinalizationInfo::mergeFiniBB(IRBuilderBase &Builder,700                                                     BasicBlock *OtherFiniBB) {701  // Simple case: FiniBB does not exist yet: re-use OtherFiniBB.702  if (!FiniBB) {703    FiniBB = OtherFiniBB;704 705    Builder.SetInsertPoint(FiniBB->getFirstNonPHIIt());706    if (Error Err = FiniCB(Builder.saveIP()))707      return Err;708 709    return Error::success();710  }711 712  // Move instructions from FiniBB to the start of OtherFiniBB.713  auto EndIt = FiniBB->end();714  if (FiniBB->size() >= 1)715    if (auto Prev = std::prev(EndIt); Prev->isTerminator())716      EndIt = Prev;717  OtherFiniBB->splice(OtherFiniBB->getFirstNonPHIIt(), FiniBB, FiniBB->begin(),718                      EndIt);719 720  FiniBB->replaceAllUsesWith(OtherFiniBB);721  FiniBB->eraseFromParent();722  FiniBB = OtherFiniBB;723  return Error::success();724}725 726Function *OpenMPIRBuilder::getOrCreateRuntimeFunctionPtr(RuntimeFunction FnID) {727  FunctionCallee RTLFn = getOrCreateRuntimeFunction(M, FnID);728  auto *Fn = dyn_cast<llvm::Function>(RTLFn.getCallee());729  assert(Fn && "Failed to create OpenMP runtime function pointer");730  return Fn;731}732 733CallInst *OpenMPIRBuilder::createRuntimeFunctionCall(FunctionCallee Callee,734                                                     ArrayRef<Value *> Args,735                                                     StringRef Name) {736  CallInst *Call = Builder.CreateCall(Callee, Args, Name);737  Call->setCallingConv(Config.getRuntimeCC());738  return Call;739}740 741void OpenMPIRBuilder::initialize() { initializeTypes(M); }742 743static void raiseUserConstantDataAllocasToEntryBlock(IRBuilderBase &Builder,744                                                     Function *Function) {745  BasicBlock &EntryBlock = Function->getEntryBlock();746  BasicBlock::iterator MoveLocInst = EntryBlock.getFirstNonPHIIt();747 748  // Loop over blocks looking for constant allocas, skipping the entry block749  // as any allocas there are already in the desired location.750  for (auto Block = std::next(Function->begin(), 1); Block != Function->end();751       Block++) {752    for (auto Inst = Block->getReverseIterator()->begin();753         Inst != Block->getReverseIterator()->end();) {754      if (auto *AllocaInst = dyn_cast_if_present<llvm::AllocaInst>(Inst)) {755        Inst++;756        if (!isa<ConstantData>(AllocaInst->getArraySize()))757          continue;758        AllocaInst->moveBeforePreserving(MoveLocInst);759      } else {760        Inst++;761      }762    }763  }764}765 766void OpenMPIRBuilder::finalize(Function *Fn) {767  SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;768  SmallVector<BasicBlock *, 32> Blocks;769  SmallVector<OutlineInfo, 16> DeferredOutlines;770  for (OutlineInfo &OI : OutlineInfos) {771    // Skip functions that have not finalized yet; may happen with nested772    // function generation.773    if (Fn && OI.getFunction() != Fn) {774      DeferredOutlines.push_back(OI);775      continue;776    }777 778    ParallelRegionBlockSet.clear();779    Blocks.clear();780    OI.collectBlocks(ParallelRegionBlockSet, Blocks);781 782    Function *OuterFn = OI.getFunction();783    CodeExtractorAnalysisCache CEAC(*OuterFn);784    // If we generate code for the target device, we need to allocate785    // struct for aggregate params in the device default alloca address space.786    // OpenMP runtime requires that the params of the extracted functions are787    // passed as zero address space pointers. This flag ensures that788    // CodeExtractor generates correct code for extracted functions789    // which are used by OpenMP runtime.790    bool ArgsInZeroAddressSpace = Config.isTargetDevice();791    CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,792                            /* AggregateArgs */ true,793                            /* BlockFrequencyInfo */ nullptr,794                            /* BranchProbabilityInfo */ nullptr,795                            /* AssumptionCache */ nullptr,796                            /* AllowVarArgs */ true,797                            /* AllowAlloca */ true,798                            /* AllocaBlock*/ OI.OuterAllocaBB,799                            /* Suffix */ ".omp_par", ArgsInZeroAddressSpace);800 801    LLVM_DEBUG(dbgs() << "Before     outlining: " << *OuterFn << "\n");802    LLVM_DEBUG(dbgs() << "Entry " << OI.EntryBB->getName()803                      << " Exit: " << OI.ExitBB->getName() << "\n");804    assert(Extractor.isEligible() &&805           "Expected OpenMP outlining to be possible!");806 807    for (auto *V : OI.ExcludeArgsFromAggregate)808      Extractor.excludeArgFromAggregate(V);809 810    Function *OutlinedFn = Extractor.extractCodeRegion(CEAC);811 812    // Forward target-cpu, target-features attributes to the outlined function.813    auto TargetCpuAttr = OuterFn->getFnAttribute("target-cpu");814    if (TargetCpuAttr.isStringAttribute())815      OutlinedFn->addFnAttr(TargetCpuAttr);816 817    auto TargetFeaturesAttr = OuterFn->getFnAttribute("target-features");818    if (TargetFeaturesAttr.isStringAttribute())819      OutlinedFn->addFnAttr(TargetFeaturesAttr);820 821    LLVM_DEBUG(dbgs() << "After      outlining: " << *OuterFn << "\n");822    LLVM_DEBUG(dbgs() << "   Outlined function: " << *OutlinedFn << "\n");823    assert(OutlinedFn->getReturnType()->isVoidTy() &&824           "OpenMP outlined functions should not return a value!");825 826    // For compability with the clang CG we move the outlined function after the827    // one with the parallel region.828    OutlinedFn->removeFromParent();829    M.getFunctionList().insertAfter(OuterFn->getIterator(), OutlinedFn);830 831    // Remove the artificial entry introduced by the extractor right away, we832    // made our own entry block after all.833    {834      BasicBlock &ArtificialEntry = OutlinedFn->getEntryBlock();835      assert(ArtificialEntry.getUniqueSuccessor() == OI.EntryBB);836      assert(OI.EntryBB->getUniquePredecessor() == &ArtificialEntry);837      // Move instructions from the to-be-deleted ArtificialEntry to the entry838      // basic block of the parallel region. CodeExtractor generates839      // instructions to unwrap the aggregate argument and may sink840      // allocas/bitcasts for values that are solely used in the outlined region841      // and do not escape.842      assert(!ArtificialEntry.empty() &&843             "Expected instructions to add in the outlined region entry");844      for (BasicBlock::reverse_iterator It = ArtificialEntry.rbegin(),845                                        End = ArtificialEntry.rend();846           It != End;) {847        Instruction &I = *It;848        It++;849 850        if (I.isTerminator()) {851          // Absorb any debug value that terminator may have852          if (OI.EntryBB->getTerminator())853            OI.EntryBB->getTerminator()->adoptDbgRecords(854                &ArtificialEntry, I.getIterator(), false);855          continue;856        }857 858        I.moveBeforePreserving(*OI.EntryBB, OI.EntryBB->getFirstInsertionPt());859      }860 861      OI.EntryBB->moveBefore(&ArtificialEntry);862      ArtificialEntry.eraseFromParent();863    }864    assert(&OutlinedFn->getEntryBlock() == OI.EntryBB);865    assert(OutlinedFn && OutlinedFn->hasNUses(1));866 867    // Run a user callback, e.g. to add attributes.868    if (OI.PostOutlineCB)869      OI.PostOutlineCB(*OutlinedFn);870  }871 872  // Remove work items that have been completed.873  OutlineInfos = std::move(DeferredOutlines);874 875  // The createTarget functions embeds user written code into876  // the target region which may inject allocas which need to877  // be moved to the entry block of our target or risk malformed878  // optimisations by later passes, this is only relevant for879  // the device pass which appears to be a little more delicate880  // when it comes to optimisations (however, we do not block on881  // that here, it's up to the inserter to the list to do so).882  // This notbaly has to occur after the OutlinedInfo candidates883  // have been extracted so we have an end product that will not884  // be implicitly adversely affected by any raises unless885  // intentionally appended to the list.886  // NOTE: This only does so for ConstantData, it could be extended887  // to ConstantExpr's with further effort, however, they should888  // largely be folded when they get here. Extending it to runtime889  // defined/read+writeable allocation sizes would be non-trivial890  // (need to factor in movement of any stores to variables the891  // allocation size depends on, as well as the usual loads,892  // otherwise it'll yield the wrong result after movement) and893  // likely be more suitable as an LLVM optimisation pass.894  for (Function *F : ConstantAllocaRaiseCandidates)895    raiseUserConstantDataAllocasToEntryBlock(Builder, F);896 897  EmitMetadataErrorReportFunctionTy &&ErrorReportFn =898      [](EmitMetadataErrorKind Kind,899         const TargetRegionEntryInfo &EntryInfo) -> void {900    errs() << "Error of kind: " << Kind901           << " when emitting offload entries and metadata during "902              "OMPIRBuilder finalization \n";903  };904 905  if (!OffloadInfoManager.empty())906    createOffloadEntriesAndInfoMetadata(ErrorReportFn);907 908  if (Config.EmitLLVMUsedMetaInfo.value_or(false)) {909    std::vector<WeakTrackingVH> LLVMCompilerUsed = {910        M.getGlobalVariable("__openmp_nvptx_data_transfer_temporary_storage")};911    emitUsed("llvm.compiler.used", LLVMCompilerUsed);912  }913 914  IsFinalized = true;915}916 917bool OpenMPIRBuilder::isFinalized() { return IsFinalized; }918 919OpenMPIRBuilder::~OpenMPIRBuilder() {920  assert(OutlineInfos.empty() && "There must be no outstanding outlinings");921}922 923GlobalValue *OpenMPIRBuilder::createGlobalFlag(unsigned Value, StringRef Name) {924  IntegerType *I32Ty = Type::getInt32Ty(M.getContext());925  auto *GV =926      new GlobalVariable(M, I32Ty,927                         /* isConstant = */ true, GlobalValue::WeakODRLinkage,928                         ConstantInt::get(I32Ty, Value), Name);929  GV->setVisibility(GlobalValue::HiddenVisibility);930 931  return GV;932}933 934void OpenMPIRBuilder::emitUsed(StringRef Name, ArrayRef<WeakTrackingVH> List) {935  if (List.empty())936    return;937 938  // Convert List to what ConstantArray needs.939  SmallVector<Constant *, 8> UsedArray;940  UsedArray.resize(List.size());941  for (unsigned I = 0, E = List.size(); I != E; ++I)942    UsedArray[I] = ConstantExpr::getPointerBitCastOrAddrSpaceCast(943        cast<Constant>(&*List[I]), Builder.getPtrTy());944 945  if (UsedArray.empty())946    return;947  ArrayType *ATy = ArrayType::get(Builder.getPtrTy(), UsedArray.size());948 949  auto *GV = new GlobalVariable(M, ATy, false, GlobalValue::AppendingLinkage,950                                ConstantArray::get(ATy, UsedArray), Name);951 952  GV->setSection("llvm.metadata");953}954 955GlobalVariable *956OpenMPIRBuilder::emitKernelExecutionMode(StringRef KernelName,957                                         OMPTgtExecModeFlags Mode) {958  auto *Int8Ty = Builder.getInt8Ty();959  auto *GVMode = new GlobalVariable(960      M, Int8Ty, /*isConstant=*/true, GlobalValue::WeakAnyLinkage,961      ConstantInt::get(Int8Ty, Mode), Twine(KernelName, "_exec_mode"));962  GVMode->setVisibility(GlobalVariable::ProtectedVisibility);963  return GVMode;964}965 966Constant *OpenMPIRBuilder::getOrCreateIdent(Constant *SrcLocStr,967                                            uint32_t SrcLocStrSize,968                                            IdentFlag LocFlags,969                                            unsigned Reserve2Flags) {970  // Enable "C-mode".971  LocFlags |= OMP_IDENT_FLAG_KMPC;972 973  Constant *&Ident =974      IdentMap[{SrcLocStr, uint64_t(LocFlags) << 31 | Reserve2Flags}];975  if (!Ident) {976    Constant *I32Null = ConstantInt::getNullValue(Int32);977    Constant *IdentData[] = {I32Null,978                             ConstantInt::get(Int32, uint32_t(LocFlags)),979                             ConstantInt::get(Int32, Reserve2Flags),980                             ConstantInt::get(Int32, SrcLocStrSize), SrcLocStr};981 982    size_t SrcLocStrArgIdx = 4;983    if (OpenMPIRBuilder::Ident->getElementType(SrcLocStrArgIdx)984            ->getPointerAddressSpace() !=985        IdentData[SrcLocStrArgIdx]->getType()->getPointerAddressSpace())986      IdentData[SrcLocStrArgIdx] = ConstantExpr::getAddrSpaceCast(987          SrcLocStr, OpenMPIRBuilder::Ident->getElementType(SrcLocStrArgIdx));988    Constant *Initializer =989        ConstantStruct::get(OpenMPIRBuilder::Ident, IdentData);990 991    // Look for existing encoding of the location + flags, not needed but992    // minimizes the difference to the existing solution while we transition.993    for (GlobalVariable &GV : M.globals())994      if (GV.getValueType() == OpenMPIRBuilder::Ident && GV.hasInitializer())995        if (GV.getInitializer() == Initializer)996          Ident = &GV;997 998    if (!Ident) {999      auto *GV = new GlobalVariable(1000          M, OpenMPIRBuilder::Ident,1001          /* isConstant = */ true, GlobalValue::PrivateLinkage, Initializer, "",1002          nullptr, GlobalValue::NotThreadLocal,1003          M.getDataLayout().getDefaultGlobalsAddressSpace());1004      GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);1005      GV->setAlignment(Align(8));1006      Ident = GV;1007    }1008  }1009 1010  return ConstantExpr::getPointerBitCastOrAddrSpaceCast(Ident, IdentPtr);1011}1012 1013Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef LocStr,1014                                                uint32_t &SrcLocStrSize) {1015  SrcLocStrSize = LocStr.size();1016  Constant *&SrcLocStr = SrcLocStrMap[LocStr];1017  if (!SrcLocStr) {1018    Constant *Initializer =1019        ConstantDataArray::getString(M.getContext(), LocStr);1020 1021    // Look for existing encoding of the location, not needed but minimizes the1022    // difference to the existing solution while we transition.1023    for (GlobalVariable &GV : M.globals())1024      if (GV.isConstant() && GV.hasInitializer() &&1025          GV.getInitializer() == Initializer)1026        return SrcLocStr = ConstantExpr::getPointerCast(&GV, Int8Ptr);1027 1028    SrcLocStr = Builder.CreateGlobalString(1029        LocStr, /*Name=*/"", M.getDataLayout().getDefaultGlobalsAddressSpace(),1030        &M);1031  }1032  return SrcLocStr;1033}1034 1035Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef FunctionName,1036                                                StringRef FileName,1037                                                unsigned Line, unsigned Column,1038                                                uint32_t &SrcLocStrSize) {1039  SmallString<128> Buffer;1040  Buffer.push_back(';');1041  Buffer.append(FileName);1042  Buffer.push_back(';');1043  Buffer.append(FunctionName);1044  Buffer.push_back(';');1045  Buffer.append(std::to_string(Line));1046  Buffer.push_back(';');1047  Buffer.append(std::to_string(Column));1048  Buffer.push_back(';');1049  Buffer.push_back(';');1050  return getOrCreateSrcLocStr(Buffer.str(), SrcLocStrSize);1051}1052 1053Constant *1054OpenMPIRBuilder::getOrCreateDefaultSrcLocStr(uint32_t &SrcLocStrSize) {1055  StringRef UnknownLoc = ";unknown;unknown;0;0;;";1056  return getOrCreateSrcLocStr(UnknownLoc, SrcLocStrSize);1057}1058 1059Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(DebugLoc DL,1060                                                uint32_t &SrcLocStrSize,1061                                                Function *F) {1062  DILocation *DIL = DL.get();1063  if (!DIL)1064    return getOrCreateDefaultSrcLocStr(SrcLocStrSize);1065  StringRef FileName = M.getName();1066  if (DIFile *DIF = DIL->getFile())1067    if (std::optional<StringRef> Source = DIF->getSource())1068      FileName = *Source;1069  StringRef Function = DIL->getScope()->getSubprogram()->getName();1070  if (Function.empty() && F)1071    Function = F->getName();1072  return getOrCreateSrcLocStr(Function, FileName, DIL->getLine(),1073                              DIL->getColumn(), SrcLocStrSize);1074}1075 1076Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(const LocationDescription &Loc,1077                                                uint32_t &SrcLocStrSize) {1078  return getOrCreateSrcLocStr(Loc.DL, SrcLocStrSize,1079                              Loc.IP.getBlock()->getParent());1080}1081 1082Value *OpenMPIRBuilder::getOrCreateThreadID(Value *Ident) {1083  return createRuntimeFunctionCall(1084      getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num), Ident,1085      "omp_global_thread_num");1086}1087 1088OpenMPIRBuilder::InsertPointOrErrorTy1089OpenMPIRBuilder::createBarrier(const LocationDescription &Loc, Directive Kind,1090                               bool ForceSimpleCall, bool CheckCancelFlag) {1091  if (!updateToLocation(Loc))1092    return Loc.IP;1093 1094  // Build call __kmpc_cancel_barrier(loc, thread_id) or1095  //            __kmpc_barrier(loc, thread_id);1096 1097  IdentFlag BarrierLocFlags;1098  switch (Kind) {1099  case OMPD_for:1100    BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_FOR;1101    break;1102  case OMPD_sections:1103    BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SECTIONS;1104    break;1105  case OMPD_single:1106    BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SINGLE;1107    break;1108  case OMPD_barrier:1109    BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_EXPL;1110    break;1111  default:1112    BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL;1113    break;1114  }1115 1116  uint32_t SrcLocStrSize;1117  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);1118  Value *Args[] = {1119      getOrCreateIdent(SrcLocStr, SrcLocStrSize, BarrierLocFlags),1120      getOrCreateThreadID(getOrCreateIdent(SrcLocStr, SrcLocStrSize))};1121 1122  // If we are in a cancellable parallel region, barriers are cancellation1123  // points.1124  // TODO: Check why we would force simple calls or to ignore the cancel flag.1125  bool UseCancelBarrier =1126      !ForceSimpleCall && isLastFinalizationInfoCancellable(OMPD_parallel);1127 1128  Value *Result = createRuntimeFunctionCall(1129      getOrCreateRuntimeFunctionPtr(UseCancelBarrier1130                                        ? OMPRTL___kmpc_cancel_barrier1131                                        : OMPRTL___kmpc_barrier),1132      Args);1133 1134  if (UseCancelBarrier && CheckCancelFlag)1135    if (Error Err = emitCancelationCheckImpl(Result, OMPD_parallel))1136      return Err;1137 1138  return Builder.saveIP();1139}1140 1141OpenMPIRBuilder::InsertPointOrErrorTy1142OpenMPIRBuilder::createCancel(const LocationDescription &Loc,1143                              Value *IfCondition,1144                              omp::Directive CanceledDirective) {1145  if (!updateToLocation(Loc))1146    return Loc.IP;1147 1148  // LLVM utilities like blocks with terminators.1149  auto *UI = Builder.CreateUnreachable();1150 1151  Instruction *ThenTI = UI, *ElseTI = nullptr;1152  if (IfCondition) {1153    SplitBlockAndInsertIfThenElse(IfCondition, UI, &ThenTI, &ElseTI);1154 1155    // Even if the if condition evaluates to false, this should count as a1156    // cancellation point1157    Builder.SetInsertPoint(ElseTI);1158    auto ElseIP = Builder.saveIP();1159 1160    InsertPointOrErrorTy IPOrErr = createCancellationPoint(1161        LocationDescription{ElseIP, Loc.DL}, CanceledDirective);1162    if (!IPOrErr)1163      return IPOrErr;1164  }1165 1166  Builder.SetInsertPoint(ThenTI);1167 1168  Value *CancelKind = nullptr;1169  switch (CanceledDirective) {1170#define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value)                       \1171  case DirectiveEnum:                                                          \1172    CancelKind = Builder.getInt32(Value);                                      \1173    break;1174#include "llvm/Frontend/OpenMP/OMPKinds.def"1175  default:1176    llvm_unreachable("Unknown cancel kind!");1177  }1178 1179  uint32_t SrcLocStrSize;1180  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);1181  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);1182  Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind};1183  Value *Result = createRuntimeFunctionCall(1184      getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_cancel), Args);1185 1186  // The actual cancel logic is shared with others, e.g., cancel_barriers.1187  if (Error Err = emitCancelationCheckImpl(Result, CanceledDirective))1188    return Err;1189 1190  // Update the insertion point and remove the terminator we introduced.1191  Builder.SetInsertPoint(UI->getParent());1192  UI->eraseFromParent();1193 1194  return Builder.saveIP();1195}1196 1197OpenMPIRBuilder::InsertPointOrErrorTy1198OpenMPIRBuilder::createCancellationPoint(const LocationDescription &Loc,1199                                         omp::Directive CanceledDirective) {1200  if (!updateToLocation(Loc))1201    return Loc.IP;1202 1203  // LLVM utilities like blocks with terminators.1204  auto *UI = Builder.CreateUnreachable();1205  Builder.SetInsertPoint(UI);1206 1207  Value *CancelKind = nullptr;1208  switch (CanceledDirective) {1209#define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value)                       \1210  case DirectiveEnum:                                                          \1211    CancelKind = Builder.getInt32(Value);                                      \1212    break;1213#include "llvm/Frontend/OpenMP/OMPKinds.def"1214  default:1215    llvm_unreachable("Unknown cancel kind!");1216  }1217 1218  uint32_t SrcLocStrSize;1219  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);1220  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);1221  Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind};1222  Value *Result = createRuntimeFunctionCall(1223      getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_cancellationpoint), Args);1224 1225  // The actual cancel logic is shared with others, e.g., cancel_barriers.1226  if (Error Err = emitCancelationCheckImpl(Result, CanceledDirective))1227    return Err;1228 1229  // Update the insertion point and remove the terminator we introduced.1230  Builder.SetInsertPoint(UI->getParent());1231  UI->eraseFromParent();1232 1233  return Builder.saveIP();1234}1235 1236OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetKernel(1237    const LocationDescription &Loc, InsertPointTy AllocaIP, Value *&Return,1238    Value *Ident, Value *DeviceID, Value *NumTeams, Value *NumThreads,1239    Value *HostPtr, ArrayRef<Value *> KernelArgs) {1240  if (!updateToLocation(Loc))1241    return Loc.IP;1242 1243  Builder.restoreIP(AllocaIP);1244  auto *KernelArgsPtr =1245      Builder.CreateAlloca(OpenMPIRBuilder::KernelArgs, nullptr, "kernel_args");1246  updateToLocation(Loc);1247 1248  for (unsigned I = 0, Size = KernelArgs.size(); I != Size; ++I) {1249    llvm::Value *Arg =1250        Builder.CreateStructGEP(OpenMPIRBuilder::KernelArgs, KernelArgsPtr, I);1251    Builder.CreateAlignedStore(1252        KernelArgs[I], Arg,1253        M.getDataLayout().getPrefTypeAlign(KernelArgs[I]->getType()));1254  }1255 1256  SmallVector<Value *> OffloadingArgs{Ident,      DeviceID, NumTeams,1257                                      NumThreads, HostPtr,  KernelArgsPtr};1258 1259  Return = createRuntimeFunctionCall(1260      getOrCreateRuntimeFunction(M, OMPRTL___tgt_target_kernel),1261      OffloadingArgs);1262 1263  return Builder.saveIP();1264}1265 1266OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitKernelLaunch(1267    const LocationDescription &Loc, Value *OutlinedFnID,1268    EmitFallbackCallbackTy EmitTargetCallFallbackCB, TargetKernelArgs &Args,1269    Value *DeviceID, Value *RTLoc, InsertPointTy AllocaIP) {1270 1271  if (!updateToLocation(Loc))1272    return Loc.IP;1273 1274  // On top of the arrays that were filled up, the target offloading call1275  // takes as arguments the device id as well as the host pointer. The host1276  // pointer is used by the runtime library to identify the current target1277  // region, so it only has to be unique and not necessarily point to1278  // anything. It could be the pointer to the outlined function that1279  // implements the target region, but we aren't using that so that the1280  // compiler doesn't need to keep that, and could therefore inline the host1281  // function if proven worthwhile during optimization.1282 1283  // From this point on, we need to have an ID of the target region defined.1284  assert(OutlinedFnID && "Invalid outlined function ID!");1285  (void)OutlinedFnID;1286 1287  // Return value of the runtime offloading call.1288  Value *Return = nullptr;1289 1290  // Arguments for the target kernel.1291  SmallVector<Value *> ArgsVector;1292  getKernelArgsVector(Args, Builder, ArgsVector);1293 1294  // The target region is an outlined function launched by the runtime1295  // via calls to __tgt_target_kernel().1296  //1297  // Note that on the host and CPU targets, the runtime implementation of1298  // these calls simply call the outlined function without forking threads.1299  // The outlined functions themselves have runtime calls to1300  // __kmpc_fork_teams() and __kmpc_fork() for this purpose, codegen'd by1301  // the compiler in emitTeamsCall() and emitParallelCall().1302  //1303  // In contrast, on the NVPTX target, the implementation of1304  // __tgt_target_teams() launches a GPU kernel with the requested number1305  // of teams and threads so no additional calls to the runtime are required.1306  // Check the error code and execute the host version if required.1307  Builder.restoreIP(emitTargetKernel(1308      Builder, AllocaIP, Return, RTLoc, DeviceID, Args.NumTeams.front(),1309      Args.NumThreads.front(), OutlinedFnID, ArgsVector));1310 1311  BasicBlock *OffloadFailedBlock =1312      BasicBlock::Create(Builder.getContext(), "omp_offload.failed");1313  BasicBlock *OffloadContBlock =1314      BasicBlock::Create(Builder.getContext(), "omp_offload.cont");1315  Value *Failed = Builder.CreateIsNotNull(Return);1316  Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock);1317 1318  auto CurFn = Builder.GetInsertBlock()->getParent();1319  emitBlock(OffloadFailedBlock, CurFn);1320  InsertPointOrErrorTy AfterIP = EmitTargetCallFallbackCB(Builder.saveIP());1321  if (!AfterIP)1322    return AfterIP.takeError();1323  Builder.restoreIP(*AfterIP);1324  emitBranch(OffloadContBlock);1325  emitBlock(OffloadContBlock, CurFn, /*IsFinished=*/true);1326  return Builder.saveIP();1327}1328 1329Error OpenMPIRBuilder::emitCancelationCheckImpl(1330    Value *CancelFlag, omp::Directive CanceledDirective) {1331  assert(isLastFinalizationInfoCancellable(CanceledDirective) &&1332         "Unexpected cancellation!");1333 1334  // For a cancel barrier we create two new blocks.1335  BasicBlock *BB = Builder.GetInsertBlock();1336  BasicBlock *NonCancellationBlock;1337  if (Builder.GetInsertPoint() == BB->end()) {1338    // TODO: This branch will not be needed once we moved to the1339    // OpenMPIRBuilder codegen completely.1340    NonCancellationBlock = BasicBlock::Create(1341        BB->getContext(), BB->getName() + ".cont", BB->getParent());1342  } else {1343    NonCancellationBlock = SplitBlock(BB, &*Builder.GetInsertPoint());1344    BB->getTerminator()->eraseFromParent();1345    Builder.SetInsertPoint(BB);1346  }1347  BasicBlock *CancellationBlock = BasicBlock::Create(1348      BB->getContext(), BB->getName() + ".cncl", BB->getParent());1349 1350  // Jump to them based on the return value.1351  Value *Cmp = Builder.CreateIsNull(CancelFlag);1352  Builder.CreateCondBr(Cmp, NonCancellationBlock, CancellationBlock,1353                       /* TODO weight */ nullptr, nullptr);1354 1355  // From the cancellation block we finalize all variables and go to the1356  // post finalization block that is known to the FiniCB callback.1357  auto &FI = FinalizationStack.back();1358  Expected<BasicBlock *> FiniBBOrErr = FI.getFiniBB(Builder);1359  if (!FiniBBOrErr)1360    return FiniBBOrErr.takeError();1361  Builder.SetInsertPoint(CancellationBlock);1362  Builder.CreateBr(*FiniBBOrErr);1363 1364  // The continuation block is where code generation continues.1365  Builder.SetInsertPoint(NonCancellationBlock, NonCancellationBlock->begin());1366  return Error::success();1367}1368 1369// Callback used to create OpenMP runtime calls to support1370// omp parallel clause for the device.1371// We need to use this callback to replace call to the OutlinedFn in OuterFn1372// by the call to the OpenMP DeviceRTL runtime function (kmpc_parallel_51)1373static void targetParallelCallback(1374    OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn, Function *OuterFn,1375    BasicBlock *OuterAllocaBB, Value *Ident, Value *IfCondition,1376    Value *NumThreads, Instruction *PrivTID, AllocaInst *PrivTIDAddr,1377    Value *ThreadID, const SmallVector<Instruction *, 4> &ToBeDeleted) {1378  // Add some known attributes.1379  IRBuilder<> &Builder = OMPIRBuilder->Builder;1380  OutlinedFn.addParamAttr(0, Attribute::NoAlias);1381  OutlinedFn.addParamAttr(1, Attribute::NoAlias);1382  OutlinedFn.addParamAttr(0, Attribute::NoUndef);1383  OutlinedFn.addParamAttr(1, Attribute::NoUndef);1384  OutlinedFn.addFnAttr(Attribute::NoUnwind);1385 1386  assert(OutlinedFn.arg_size() >= 2 &&1387         "Expected at least tid and bounded tid as arguments");1388  unsigned NumCapturedVars = OutlinedFn.arg_size() - /* tid & bounded tid */ 2;1389 1390  CallInst *CI = cast<CallInst>(OutlinedFn.user_back());1391  assert(CI && "Expected call instruction to outlined function");1392  CI->getParent()->setName("omp_parallel");1393 1394  Builder.SetInsertPoint(CI);1395  Type *PtrTy = OMPIRBuilder->VoidPtr;1396  Value *NullPtrValue = Constant::getNullValue(PtrTy);1397 1398  // Add alloca for kernel args1399  OpenMPIRBuilder ::InsertPointTy CurrentIP = Builder.saveIP();1400  Builder.SetInsertPoint(OuterAllocaBB, OuterAllocaBB->getFirstInsertionPt());1401  AllocaInst *ArgsAlloca =1402      Builder.CreateAlloca(ArrayType::get(PtrTy, NumCapturedVars));1403  Value *Args = ArgsAlloca;1404  // Add address space cast if array for storing arguments is not allocated1405  // in address space 01406  if (ArgsAlloca->getAddressSpace())1407    Args = Builder.CreatePointerCast(ArgsAlloca, PtrTy);1408  Builder.restoreIP(CurrentIP);1409 1410  // Store captured vars which are used by kmpc_parallel_511411  for (unsigned Idx = 0; Idx < NumCapturedVars; Idx++) {1412    Value *V = *(CI->arg_begin() + 2 + Idx);1413    Value *StoreAddress = Builder.CreateConstInBoundsGEP2_64(1414        ArrayType::get(PtrTy, NumCapturedVars), Args, 0, Idx);1415    Builder.CreateStore(V, StoreAddress);1416  }1417 1418  Value *Cond =1419      IfCondition ? Builder.CreateSExtOrTrunc(IfCondition, OMPIRBuilder->Int32)1420                  : Builder.getInt32(1);1421 1422  // Build kmpc_parallel_51 call1423  Value *Parallel51CallArgs[] = {1424      /* identifier*/ Ident,1425      /* global thread num*/ ThreadID,1426      /* if expression */ Cond,1427      /* number of threads */ NumThreads ? NumThreads : Builder.getInt32(-1),1428      /* Proc bind */ Builder.getInt32(-1),1429      /* outlined function */ &OutlinedFn,1430      /* wrapper function */ NullPtrValue,1431      /* arguments of the outlined funciton*/ Args,1432      /* number of arguments */ Builder.getInt64(NumCapturedVars)};1433 1434  FunctionCallee RTLFn =1435      OMPIRBuilder->getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_parallel_51);1436 1437  OMPIRBuilder->createRuntimeFunctionCall(RTLFn, Parallel51CallArgs);1438 1439  LLVM_DEBUG(dbgs() << "With kmpc_parallel_51 placed: "1440                    << *Builder.GetInsertBlock()->getParent() << "\n");1441 1442  // Initialize the local TID stack location with the argument value.1443  Builder.SetInsertPoint(PrivTID);1444  Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin();1445  Builder.CreateStore(Builder.CreateLoad(OMPIRBuilder->Int32, OutlinedAI),1446                      PrivTIDAddr);1447 1448  // Remove redundant call to the outlined function.1449  CI->eraseFromParent();1450 1451  for (Instruction *I : ToBeDeleted) {1452    I->eraseFromParent();1453  }1454}1455 1456// Callback used to create OpenMP runtime calls to support1457// omp parallel clause for the host.1458// We need to use this callback to replace call to the OutlinedFn in OuterFn1459// by the call to the OpenMP host runtime function ( __kmpc_fork_call[_if])1460static void1461hostParallelCallback(OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn,1462                     Function *OuterFn, Value *Ident, Value *IfCondition,1463                     Instruction *PrivTID, AllocaInst *PrivTIDAddr,1464                     const SmallVector<Instruction *, 4> &ToBeDeleted) {1465  IRBuilder<> &Builder = OMPIRBuilder->Builder;1466  FunctionCallee RTLFn;1467  if (IfCondition) {1468    RTLFn =1469        OMPIRBuilder->getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_call_if);1470  } else {1471    RTLFn =1472        OMPIRBuilder->getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_call);1473  }1474  if (auto *F = dyn_cast<Function>(RTLFn.getCallee())) {1475    if (!F->hasMetadata(LLVMContext::MD_callback)) {1476      LLVMContext &Ctx = F->getContext();1477      MDBuilder MDB(Ctx);1478      // Annotate the callback behavior of the __kmpc_fork_call:1479      //  - The callback callee is argument number 2 (microtask).1480      //  - The first two arguments of the callback callee are unknown (-1).1481      //  - All variadic arguments to the __kmpc_fork_call are passed to the1482      //    callback callee.1483      F->addMetadata(LLVMContext::MD_callback,1484                     *MDNode::get(Ctx, {MDB.createCallbackEncoding(1485                                           2, {-1, -1},1486                                           /* VarArgsArePassed */ true)}));1487    }1488  }1489  // Add some known attributes.1490  OutlinedFn.addParamAttr(0, Attribute::NoAlias);1491  OutlinedFn.addParamAttr(1, Attribute::NoAlias);1492  OutlinedFn.addFnAttr(Attribute::NoUnwind);1493 1494  assert(OutlinedFn.arg_size() >= 2 &&1495         "Expected at least tid and bounded tid as arguments");1496  unsigned NumCapturedVars = OutlinedFn.arg_size() - /* tid & bounded tid */ 2;1497 1498  CallInst *CI = cast<CallInst>(OutlinedFn.user_back());1499  CI->getParent()->setName("omp_parallel");1500  Builder.SetInsertPoint(CI);1501 1502  // Build call __kmpc_fork_call[_if](Ident, n, microtask, var1, .., varn);1503  Value *ForkCallArgs[] = {Ident, Builder.getInt32(NumCapturedVars),1504                           &OutlinedFn};1505 1506  SmallVector<Value *, 16> RealArgs;1507  RealArgs.append(std::begin(ForkCallArgs), std::end(ForkCallArgs));1508  if (IfCondition) {1509    Value *Cond = Builder.CreateSExtOrTrunc(IfCondition, OMPIRBuilder->Int32);1510    RealArgs.push_back(Cond);1511  }1512  RealArgs.append(CI->arg_begin() + /* tid & bound tid */ 2, CI->arg_end());1513 1514  // __kmpc_fork_call_if always expects a void ptr as the last argument1515  // If there are no arguments, pass a null pointer.1516  auto PtrTy = OMPIRBuilder->VoidPtr;1517  if (IfCondition && NumCapturedVars == 0) {1518    Value *NullPtrValue = Constant::getNullValue(PtrTy);1519    RealArgs.push_back(NullPtrValue);1520  }1521 1522  OMPIRBuilder->createRuntimeFunctionCall(RTLFn, RealArgs);1523 1524  LLVM_DEBUG(dbgs() << "With fork_call placed: "1525                    << *Builder.GetInsertBlock()->getParent() << "\n");1526 1527  // Initialize the local TID stack location with the argument value.1528  Builder.SetInsertPoint(PrivTID);1529  Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin();1530  Builder.CreateStore(Builder.CreateLoad(OMPIRBuilder->Int32, OutlinedAI),1531                      PrivTIDAddr);1532 1533  // Remove redundant call to the outlined function.1534  CI->eraseFromParent();1535 1536  for (Instruction *I : ToBeDeleted) {1537    I->eraseFromParent();1538  }1539}1540 1541OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createParallel(1542    const LocationDescription &Loc, InsertPointTy OuterAllocaIP,1543    BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB,1544    FinalizeCallbackTy FiniCB, Value *IfCondition, Value *NumThreads,1545    omp::ProcBindKind ProcBind, bool IsCancellable) {1546  assert(!isConflictIP(Loc.IP, OuterAllocaIP) && "IPs must not be ambiguous");1547 1548  if (!updateToLocation(Loc))1549    return Loc.IP;1550 1551  uint32_t SrcLocStrSize;1552  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);1553  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);1554  Value *ThreadID = getOrCreateThreadID(Ident);1555  // If we generate code for the target device, we need to allocate1556  // struct for aggregate params in the device default alloca address space.1557  // OpenMP runtime requires that the params of the extracted functions are1558  // passed as zero address space pointers. This flag ensures that extracted1559  // function arguments are declared in zero address space1560  bool ArgsInZeroAddressSpace = Config.isTargetDevice();1561 1562  // Build call __kmpc_push_num_threads(&Ident, global_tid, num_threads)1563  // only if we compile for host side.1564  if (NumThreads && !Config.isTargetDevice()) {1565    Value *Args[] = {1566        Ident, ThreadID,1567        Builder.CreateIntCast(NumThreads, Int32, /*isSigned*/ false)};1568    createRuntimeFunctionCall(1569        getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_num_threads), Args);1570  }1571 1572  if (ProcBind != OMP_PROC_BIND_default) {1573    // Build call __kmpc_push_proc_bind(&Ident, global_tid, proc_bind)1574    Value *Args[] = {1575        Ident, ThreadID,1576        ConstantInt::get(Int32, unsigned(ProcBind), /*isSigned=*/true)};1577    createRuntimeFunctionCall(1578        getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_proc_bind), Args);1579  }1580 1581  BasicBlock *InsertBB = Builder.GetInsertBlock();1582  Function *OuterFn = InsertBB->getParent();1583 1584  // Save the outer alloca block because the insertion iterator may get1585  // invalidated and we still need this later.1586  BasicBlock *OuterAllocaBlock = OuterAllocaIP.getBlock();1587 1588  // Vector to remember instructions we used only during the modeling but which1589  // we want to delete at the end.1590  SmallVector<Instruction *, 4> ToBeDeleted;1591 1592  // Change the location to the outer alloca insertion point to create and1593  // initialize the allocas we pass into the parallel region.1594  InsertPointTy NewOuter(OuterAllocaBlock, OuterAllocaBlock->begin());1595  Builder.restoreIP(NewOuter);1596  AllocaInst *TIDAddrAlloca = Builder.CreateAlloca(Int32, nullptr, "tid.addr");1597  AllocaInst *ZeroAddrAlloca =1598      Builder.CreateAlloca(Int32, nullptr, "zero.addr");1599  Instruction *TIDAddr = TIDAddrAlloca;1600  Instruction *ZeroAddr = ZeroAddrAlloca;1601  if (ArgsInZeroAddressSpace && M.getDataLayout().getAllocaAddrSpace() != 0) {1602    // Add additional casts to enforce pointers in zero address space1603    TIDAddr = new AddrSpaceCastInst(1604        TIDAddrAlloca, PointerType ::get(M.getContext(), 0), "tid.addr.ascast");1605    TIDAddr->insertAfter(TIDAddrAlloca->getIterator());1606    ToBeDeleted.push_back(TIDAddr);1607    ZeroAddr = new AddrSpaceCastInst(ZeroAddrAlloca,1608                                     PointerType ::get(M.getContext(), 0),1609                                     "zero.addr.ascast");1610    ZeroAddr->insertAfter(ZeroAddrAlloca->getIterator());1611    ToBeDeleted.push_back(ZeroAddr);1612  }1613 1614  // We only need TIDAddr and ZeroAddr for modeling purposes to get the1615  // associated arguments in the outlined function, so we delete them later.1616  ToBeDeleted.push_back(TIDAddrAlloca);1617  ToBeDeleted.push_back(ZeroAddrAlloca);1618 1619  // Create an artificial insertion point that will also ensure the blocks we1620  // are about to split are not degenerated.1621  auto *UI = new UnreachableInst(Builder.getContext(), InsertBB);1622 1623  BasicBlock *EntryBB = UI->getParent();1624  BasicBlock *PRegEntryBB = EntryBB->splitBasicBlock(UI, "omp.par.entry");1625  BasicBlock *PRegBodyBB = PRegEntryBB->splitBasicBlock(UI, "omp.par.region");1626  BasicBlock *PRegPreFiniBB =1627      PRegBodyBB->splitBasicBlock(UI, "omp.par.pre_finalize");1628  BasicBlock *PRegExitBB = PRegPreFiniBB->splitBasicBlock(UI, "omp.par.exit");1629 1630  auto FiniCBWrapper = [&](InsertPointTy IP) {1631    // Hide "open-ended" blocks from the given FiniCB by setting the right jump1632    // target to the region exit block.1633    if (IP.getBlock()->end() == IP.getPoint()) {1634      IRBuilder<>::InsertPointGuard IPG(Builder);1635      Builder.restoreIP(IP);1636      Instruction *I = Builder.CreateBr(PRegExitBB);1637      IP = InsertPointTy(I->getParent(), I->getIterator());1638    }1639    assert(IP.getBlock()->getTerminator()->getNumSuccessors() == 1 &&1640           IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB &&1641           "Unexpected insertion point for finalization call!");1642    return FiniCB(IP);1643  };1644 1645  FinalizationStack.push_back({FiniCBWrapper, OMPD_parallel, IsCancellable});1646 1647  // Generate the privatization allocas in the block that will become the entry1648  // of the outlined function.1649  Builder.SetInsertPoint(PRegEntryBB->getTerminator());1650  InsertPointTy InnerAllocaIP = Builder.saveIP();1651 1652  AllocaInst *PrivTIDAddr =1653      Builder.CreateAlloca(Int32, nullptr, "tid.addr.local");1654  Instruction *PrivTID = Builder.CreateLoad(Int32, PrivTIDAddr, "tid");1655 1656  // Add some fake uses for OpenMP provided arguments.1657  ToBeDeleted.push_back(Builder.CreateLoad(Int32, TIDAddr, "tid.addr.use"));1658  Instruction *ZeroAddrUse =1659      Builder.CreateLoad(Int32, ZeroAddr, "zero.addr.use");1660  ToBeDeleted.push_back(ZeroAddrUse);1661 1662  // EntryBB1663  //   |1664  //   V1665  // PRegionEntryBB         <- Privatization allocas are placed here.1666  //   |1667  //   V1668  // PRegionBodyBB          <- BodeGen is invoked here.1669  //   |1670  //   V1671  // PRegPreFiniBB          <- The block we will start finalization from.1672  //   |1673  //   V1674  // PRegionExitBB          <- A common exit to simplify block collection.1675  //1676 1677  LLVM_DEBUG(dbgs() << "Before body codegen: " << *OuterFn << "\n");1678 1679  // Let the caller create the body.1680  assert(BodyGenCB && "Expected body generation callback!");1681  InsertPointTy CodeGenIP(PRegBodyBB, PRegBodyBB->begin());1682  if (Error Err = BodyGenCB(InnerAllocaIP, CodeGenIP))1683    return Err;1684 1685  LLVM_DEBUG(dbgs() << "After  body codegen: " << *OuterFn << "\n");1686 1687  OutlineInfo OI;1688  if (Config.isTargetDevice()) {1689    // Generate OpenMP target specific runtime call1690    OI.PostOutlineCB = [=, ToBeDeletedVec =1691                               std::move(ToBeDeleted)](Function &OutlinedFn) {1692      targetParallelCallback(this, OutlinedFn, OuterFn, OuterAllocaBlock, Ident,1693                             IfCondition, NumThreads, PrivTID, PrivTIDAddr,1694                             ThreadID, ToBeDeletedVec);1695    };1696  } else {1697    // Generate OpenMP host runtime call1698    OI.PostOutlineCB = [=, ToBeDeletedVec =1699                               std::move(ToBeDeleted)](Function &OutlinedFn) {1700      hostParallelCallback(this, OutlinedFn, OuterFn, Ident, IfCondition,1701                           PrivTID, PrivTIDAddr, ToBeDeletedVec);1702    };1703  }1704 1705  OI.OuterAllocaBB = OuterAllocaBlock;1706  OI.EntryBB = PRegEntryBB;1707  OI.ExitBB = PRegExitBB;1708 1709  SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;1710  SmallVector<BasicBlock *, 32> Blocks;1711  OI.collectBlocks(ParallelRegionBlockSet, Blocks);1712 1713  CodeExtractorAnalysisCache CEAC(*OuterFn);1714  CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,1715                          /* AggregateArgs */ false,1716                          /* BlockFrequencyInfo */ nullptr,1717                          /* BranchProbabilityInfo */ nullptr,1718                          /* AssumptionCache */ nullptr,1719                          /* AllowVarArgs */ true,1720                          /* AllowAlloca */ true,1721                          /* AllocationBlock */ OuterAllocaBlock,1722                          /* Suffix */ ".omp_par", ArgsInZeroAddressSpace);1723 1724  // Find inputs to, outputs from the code region.1725  BasicBlock *CommonExit = nullptr;1726  SetVector<Value *> Inputs, Outputs, SinkingCands, HoistingCands;1727  Extractor.findAllocas(CEAC, SinkingCands, HoistingCands, CommonExit);1728 1729  Extractor.findInputsOutputs(Inputs, Outputs, SinkingCands,1730                              /*CollectGlobalInputs=*/true);1731 1732  Inputs.remove_if([&](Value *I) {1733    if (auto *GV = dyn_cast_if_present<GlobalVariable>(I))1734      return GV->getValueType() == OpenMPIRBuilder::Ident;1735 1736    return false;1737  });1738 1739  LLVM_DEBUG(dbgs() << "Before privatization: " << *OuterFn << "\n");1740 1741  FunctionCallee TIDRTLFn =1742      getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num);1743 1744  auto PrivHelper = [&](Value &V) -> Error {1745    if (&V == TIDAddr || &V == ZeroAddr) {1746      OI.ExcludeArgsFromAggregate.push_back(&V);1747      return Error::success();1748    }1749 1750    SetVector<Use *> Uses;1751    for (Use &U : V.uses())1752      if (auto *UserI = dyn_cast<Instruction>(U.getUser()))1753        if (ParallelRegionBlockSet.count(UserI->getParent()))1754          Uses.insert(&U);1755 1756    // __kmpc_fork_call expects extra arguments as pointers. If the input1757    // already has a pointer type, everything is fine. Otherwise, store the1758    // value onto stack and load it back inside the to-be-outlined region. This1759    // will ensure only the pointer will be passed to the function.1760    // FIXME: if there are more than 15 trailing arguments, they must be1761    // additionally packed in a struct.1762    Value *Inner = &V;1763    if (!V.getType()->isPointerTy()) {1764      IRBuilder<>::InsertPointGuard Guard(Builder);1765      LLVM_DEBUG(llvm::dbgs() << "Forwarding input as pointer: " << V << "\n");1766 1767      Builder.restoreIP(OuterAllocaIP);1768      Value *Ptr =1769          Builder.CreateAlloca(V.getType(), nullptr, V.getName() + ".reloaded");1770 1771      // Store to stack at end of the block that currently branches to the entry1772      // block of the to-be-outlined region.1773      Builder.SetInsertPoint(InsertBB,1774                             InsertBB->getTerminator()->getIterator());1775      Builder.CreateStore(&V, Ptr);1776 1777      // Load back next to allocations in the to-be-outlined region.1778      Builder.restoreIP(InnerAllocaIP);1779      Inner = Builder.CreateLoad(V.getType(), Ptr);1780    }1781 1782    Value *ReplacementValue = nullptr;1783    CallInst *CI = dyn_cast<CallInst>(&V);1784    if (CI && CI->getCalledFunction() == TIDRTLFn.getCallee()) {1785      ReplacementValue = PrivTID;1786    } else {1787      InsertPointOrErrorTy AfterIP =1788          PrivCB(InnerAllocaIP, Builder.saveIP(), V, *Inner, ReplacementValue);1789      if (!AfterIP)1790        return AfterIP.takeError();1791      Builder.restoreIP(*AfterIP);1792      InnerAllocaIP = {1793          InnerAllocaIP.getBlock(),1794          InnerAllocaIP.getBlock()->getTerminator()->getIterator()};1795 1796      assert(ReplacementValue &&1797             "Expected copy/create callback to set replacement value!");1798      if (ReplacementValue == &V)1799        return Error::success();1800    }1801 1802    for (Use *UPtr : Uses)1803      UPtr->set(ReplacementValue);1804 1805    return Error::success();1806  };1807 1808  // Reset the inner alloca insertion as it will be used for loading the values1809  // wrapped into pointers before passing them into the to-be-outlined region.1810  // Configure it to insert immediately after the fake use of zero address so1811  // that they are available in the generated body and so that the1812  // OpenMP-related values (thread ID and zero address pointers) remain leading1813  // in the argument list.1814  InnerAllocaIP = IRBuilder<>::InsertPoint(1815      ZeroAddrUse->getParent(), ZeroAddrUse->getNextNode()->getIterator());1816 1817  // Reset the outer alloca insertion point to the entry of the relevant block1818  // in case it was invalidated.1819  OuterAllocaIP = IRBuilder<>::InsertPoint(1820      OuterAllocaBlock, OuterAllocaBlock->getFirstInsertionPt());1821 1822  for (Value *Input : Inputs) {1823    LLVM_DEBUG(dbgs() << "Captured input: " << *Input << "\n");1824    if (Error Err = PrivHelper(*Input))1825      return Err;1826  }1827  LLVM_DEBUG({1828    for (Value *Output : Outputs)1829      LLVM_DEBUG(dbgs() << "Captured output: " << *Output << "\n");1830  });1831  assert(Outputs.empty() &&1832         "OpenMP outlining should not produce live-out values!");1833 1834  LLVM_DEBUG(dbgs() << "After  privatization: " << *OuterFn << "\n");1835  LLVM_DEBUG({1836    for (auto *BB : Blocks)1837      dbgs() << " PBR: " << BB->getName() << "\n";1838  });1839 1840  // Adjust the finalization stack, verify the adjustment, and call the1841  // finalize function a last time to finalize values between the pre-fini1842  // block and the exit block if we left the parallel "the normal way".1843  auto FiniInfo = FinalizationStack.pop_back_val();1844  (void)FiniInfo;1845  assert(FiniInfo.DK == OMPD_parallel &&1846         "Unexpected finalization stack state!");1847 1848  Instruction *PRegPreFiniTI = PRegPreFiniBB->getTerminator();1849 1850  InsertPointTy PreFiniIP(PRegPreFiniBB, PRegPreFiniTI->getIterator());1851  Expected<BasicBlock *> FiniBBOrErr = FiniInfo.getFiniBB(Builder);1852  if (!FiniBBOrErr)1853    return FiniBBOrErr.takeError();1854  {1855    IRBuilderBase::InsertPointGuard Guard(Builder);1856    Builder.restoreIP(PreFiniIP);1857    Builder.CreateBr(*FiniBBOrErr);1858    // There's currently a branch to omp.par.exit. Delete it. We will get there1859    // via the fini block1860    if (Instruction *Term = Builder.GetInsertBlock()->getTerminator())1861      Term->eraseFromParent();1862  }1863 1864  // Register the outlined info.1865  addOutlineInfo(std::move(OI));1866 1867  InsertPointTy AfterIP(UI->getParent(), UI->getParent()->end());1868  UI->eraseFromParent();1869 1870  return AfterIP;1871}1872 1873void OpenMPIRBuilder::emitFlush(const LocationDescription &Loc) {1874  // Build call void __kmpc_flush(ident_t *loc)1875  uint32_t SrcLocStrSize;1876  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);1877  Value *Args[] = {getOrCreateIdent(SrcLocStr, SrcLocStrSize)};1878 1879  createRuntimeFunctionCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_flush),1880                            Args);1881}1882 1883void OpenMPIRBuilder::createFlush(const LocationDescription &Loc) {1884  if (!updateToLocation(Loc))1885    return;1886  emitFlush(Loc);1887}1888 1889void OpenMPIRBuilder::emitTaskwaitImpl(const LocationDescription &Loc) {1890  // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int321891  // global_tid);1892  uint32_t SrcLocStrSize;1893  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);1894  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);1895  Value *Args[] = {Ident, getOrCreateThreadID(Ident)};1896 1897  // Ignore return result until untied tasks are supported.1898  createRuntimeFunctionCall(1899      getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskwait), Args);1900}1901 1902void OpenMPIRBuilder::createTaskwait(const LocationDescription &Loc) {1903  if (!updateToLocation(Loc))1904    return;1905  emitTaskwaitImpl(Loc);1906}1907 1908void OpenMPIRBuilder::emitTaskyieldImpl(const LocationDescription &Loc) {1909  // Build call __kmpc_omp_taskyield(loc, thread_id, 0);1910  uint32_t SrcLocStrSize;1911  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);1912  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);1913  Constant *I32Null = ConstantInt::getNullValue(Int32);1914  Value *Args[] = {Ident, getOrCreateThreadID(Ident), I32Null};1915 1916  createRuntimeFunctionCall(1917      getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskyield), Args);1918}1919 1920void OpenMPIRBuilder::createTaskyield(const LocationDescription &Loc) {1921  if (!updateToLocation(Loc))1922    return;1923  emitTaskyieldImpl(Loc);1924}1925 1926// Processes the dependencies in Dependencies and does the following1927// - Allocates space on the stack of an array of DependInfo objects1928// - Populates each DependInfo object with relevant information of1929//   the corresponding dependence.1930// - All code is inserted in the entry block of the current function.1931static Value *emitTaskDependencies(1932    OpenMPIRBuilder &OMPBuilder,1933    const SmallVectorImpl<OpenMPIRBuilder::DependData> &Dependencies) {1934  // Early return if we have no dependencies to process1935  if (Dependencies.empty())1936    return nullptr;1937 1938  // Given a vector of DependData objects, in this function we create an1939  // array on the stack that holds kmp_dep_info objects corresponding1940  // to each dependency. This is then passed to the OpenMP runtime.1941  // For example, if there are 'n' dependencies then the following psedo1942  // code is generated. Assume the first dependence is on a variable 'a'1943  //1944  // \code{c}1945  // DepArray = alloc(n x sizeof(kmp_depend_info);1946  // idx = 0;1947  // DepArray[idx].base_addr = ptrtoint(&a);1948  // DepArray[idx].len = 8;1949  // DepArray[idx].flags = Dep.DepKind; /*(See OMPContants.h for DepKind)*/1950  // ++idx;1951  // DepArray[idx].base_addr = ...;1952  // \endcode1953 1954  IRBuilderBase &Builder = OMPBuilder.Builder;1955  Type *DependInfo = OMPBuilder.DependInfo;1956  Module &M = OMPBuilder.M;1957 1958  Value *DepArray = nullptr;1959  OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();1960  Builder.SetInsertPoint(1961      OldIP.getBlock()->getParent()->getEntryBlock().getTerminator());1962 1963  Type *DepArrayTy = ArrayType::get(DependInfo, Dependencies.size());1964  DepArray = Builder.CreateAlloca(DepArrayTy, nullptr, ".dep.arr.addr");1965 1966  Builder.restoreIP(OldIP);1967 1968  for (const auto &[DepIdx, Dep] : enumerate(Dependencies)) {1969    Value *Base =1970        Builder.CreateConstInBoundsGEP2_64(DepArrayTy, DepArray, 0, DepIdx);1971    // Store the pointer to the variable1972    Value *Addr = Builder.CreateStructGEP(1973        DependInfo, Base,1974        static_cast<unsigned int>(RTLDependInfoFields::BaseAddr));1975    Value *DepValPtr = Builder.CreatePtrToInt(Dep.DepVal, Builder.getInt64Ty());1976    Builder.CreateStore(DepValPtr, Addr);1977    // Store the size of the variable1978    Value *Size = Builder.CreateStructGEP(1979        DependInfo, Base, static_cast<unsigned int>(RTLDependInfoFields::Len));1980    Builder.CreateStore(1981        Builder.getInt64(M.getDataLayout().getTypeStoreSize(Dep.DepValueType)),1982        Size);1983    // Store the dependency kind1984    Value *Flags = Builder.CreateStructGEP(1985        DependInfo, Base,1986        static_cast<unsigned int>(RTLDependInfoFields::Flags));1987    Builder.CreateStore(1988        ConstantInt::get(Builder.getInt8Ty(),1989                         static_cast<unsigned int>(Dep.DepKind)),1990        Flags);1991  }1992  return DepArray;1993}1994 1995OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTask(1996    const LocationDescription &Loc, InsertPointTy AllocaIP,1997    BodyGenCallbackTy BodyGenCB, bool Tied, Value *Final, Value *IfCondition,1998    SmallVector<DependData> Dependencies, bool Mergeable, Value *EventHandle,1999    Value *Priority) {2000 2001  if (!updateToLocation(Loc))2002    return InsertPointTy();2003 2004  uint32_t SrcLocStrSize;2005  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);2006  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);2007  // The current basic block is split into four basic blocks. After outlining,2008  // they will be mapped as follows:2009  // ```2010  // def current_fn() {2011  //   current_basic_block:2012  //     br label %task.exit2013  //   task.exit:2014  //     ; instructions after task2015  // }2016  // def outlined_fn() {2017  //   task.alloca:2018  //     br label %task.body2019  //   task.body:2020  //     ret void2021  // }2022  // ```2023  BasicBlock *TaskExitBB = splitBB(Builder, /*CreateBranch=*/true, "task.exit");2024  BasicBlock *TaskBodyBB = splitBB(Builder, /*CreateBranch=*/true, "task.body");2025  BasicBlock *TaskAllocaBB =2026      splitBB(Builder, /*CreateBranch=*/true, "task.alloca");2027 2028  InsertPointTy TaskAllocaIP =2029      InsertPointTy(TaskAllocaBB, TaskAllocaBB->begin());2030  InsertPointTy TaskBodyIP = InsertPointTy(TaskBodyBB, TaskBodyBB->begin());2031  if (Error Err = BodyGenCB(TaskAllocaIP, TaskBodyIP))2032    return Err;2033 2034  OutlineInfo OI;2035  OI.EntryBB = TaskAllocaBB;2036  OI.OuterAllocaBB = AllocaIP.getBlock();2037  OI.ExitBB = TaskExitBB;2038 2039  // Add the thread ID argument.2040  SmallVector<Instruction *, 4> ToBeDeleted;2041  OI.ExcludeArgsFromAggregate.push_back(createFakeIntVal(2042      Builder, AllocaIP, ToBeDeleted, TaskAllocaIP, "global.tid", false));2043 2044  OI.PostOutlineCB = [this, Ident, Tied, Final, IfCondition, Dependencies,2045                      Mergeable, Priority, EventHandle, TaskAllocaBB,2046                      ToBeDeleted](Function &OutlinedFn) mutable {2047    // Replace the Stale CI by appropriate RTL function call.2048    assert(OutlinedFn.hasOneUse() &&2049           "there must be a single user for the outlined function");2050    CallInst *StaleCI = cast<CallInst>(OutlinedFn.user_back());2051 2052    // HasShareds is true if any variables are captured in the outlined region,2053    // false otherwise.2054    bool HasShareds = StaleCI->arg_size() > 1;2055    Builder.SetInsertPoint(StaleCI);2056 2057    // Gather the arguments for emitting the runtime call for2058    // @__kmpc_omp_task_alloc2059    Function *TaskAllocFn =2060        getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc);2061 2062    // Arguments - `loc_ref` (Ident) and `gtid` (ThreadID)2063    // call.2064    Value *ThreadID = getOrCreateThreadID(Ident);2065 2066    // Argument - `flags`2067    // Task is tied iff (Flags & 1) == 1.2068    // Task is untied iff (Flags & 1) == 0.2069    // Task is final iff (Flags & 2) == 2.2070    // Task is not final iff (Flags & 2) == 0.2071    // Task is mergeable iff (Flags & 4) == 4.2072    // Task is not mergeable iff (Flags & 4) == 0.2073    // Task is priority iff (Flags & 32) == 32.2074    // Task is not priority iff (Flags & 32) == 0.2075    // TODO: Handle the other flags.2076    Value *Flags = Builder.getInt32(Tied);2077    if (Final) {2078      Value *FinalFlag =2079          Builder.CreateSelect(Final, Builder.getInt32(2), Builder.getInt32(0));2080      Flags = Builder.CreateOr(FinalFlag, Flags);2081    }2082 2083    if (Mergeable)2084      Flags = Builder.CreateOr(Builder.getInt32(4), Flags);2085    if (Priority)2086      Flags = Builder.CreateOr(Builder.getInt32(32), Flags);2087 2088    // Argument - `sizeof_kmp_task_t` (TaskSize)2089    // Tasksize refers to the size in bytes of kmp_task_t data structure2090    // including private vars accessed in task.2091    // TODO: add kmp_task_t_with_privates (privates)2092    Value *TaskSize = Builder.getInt64(2093        divideCeil(M.getDataLayout().getTypeSizeInBits(Task), 8));2094 2095    // Argument - `sizeof_shareds` (SharedsSize)2096    // SharedsSize refers to the shareds array size in the kmp_task_t data2097    // structure.2098    Value *SharedsSize = Builder.getInt64(0);2099    if (HasShareds) {2100      AllocaInst *ArgStructAlloca =2101          dyn_cast<AllocaInst>(StaleCI->getArgOperand(1));2102      assert(ArgStructAlloca &&2103             "Unable to find the alloca instruction corresponding to arguments "2104             "for extracted function");2105      StructType *ArgStructType =2106          dyn_cast<StructType>(ArgStructAlloca->getAllocatedType());2107      assert(ArgStructType && "Unable to find struct type corresponding to "2108                              "arguments for extracted function");2109      SharedsSize =2110          Builder.getInt64(M.getDataLayout().getTypeStoreSize(ArgStructType));2111    }2112    // Emit the @__kmpc_omp_task_alloc runtime call2113    // The runtime call returns a pointer to an area where the task captured2114    // variables must be copied before the task is run (TaskData)2115    CallInst *TaskData = createRuntimeFunctionCall(2116        TaskAllocFn, {/*loc_ref=*/Ident, /*gtid=*/ThreadID, /*flags=*/Flags,2117                      /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,2118                      /*task_func=*/&OutlinedFn});2119 2120    // Emit detach clause initialization.2121    // evt = (typeof(evt))__kmpc_task_allow_completion_event(loc, tid,2122    // task_descriptor);2123    if (EventHandle) {2124      Function *TaskDetachFn = getOrCreateRuntimeFunctionPtr(2125          OMPRTL___kmpc_task_allow_completion_event);2126      llvm::Value *EventVal =2127          createRuntimeFunctionCall(TaskDetachFn, {Ident, ThreadID, TaskData});2128      llvm::Value *EventHandleAddr =2129          Builder.CreatePointerBitCastOrAddrSpaceCast(EventHandle,2130                                                      Builder.getPtrTy(0));2131      EventVal = Builder.CreatePtrToInt(EventVal, Builder.getInt64Ty());2132      Builder.CreateStore(EventVal, EventHandleAddr);2133    }2134    // Copy the arguments for outlined function2135    if (HasShareds) {2136      Value *Shareds = StaleCI->getArgOperand(1);2137      Align Alignment = TaskData->getPointerAlignment(M.getDataLayout());2138      Value *TaskShareds = Builder.CreateLoad(VoidPtr, TaskData);2139      Builder.CreateMemCpy(TaskShareds, Alignment, Shareds, Alignment,2140                           SharedsSize);2141    }2142 2143    if (Priority) {2144      //2145      // The return type of "__kmpc_omp_task_alloc" is "kmp_task_t *",2146      // we populate the priority information into the "kmp_task_t" here2147      //2148      // The struct "kmp_task_t" definition is available in kmp.h2149      // kmp_task_t = { shareds, routine, part_id, data1, data2 }2150      // data2 is used for priority2151      //2152      Type *Int32Ty = Builder.getInt32Ty();2153      Constant *Zero = ConstantInt::get(Int32Ty, 0);2154      // kmp_task_t* => { ptr }2155      Type *TaskPtr = StructType::get(VoidPtr);2156      Value *TaskGEP =2157          Builder.CreateInBoundsGEP(TaskPtr, TaskData, {Zero, Zero});2158      // kmp_task_t => { ptr, ptr, i32, ptr, ptr }2159      Type *TaskStructType = StructType::get(2160          VoidPtr, VoidPtr, Builder.getInt32Ty(), VoidPtr, VoidPtr);2161      Value *PriorityData = Builder.CreateInBoundsGEP(2162          TaskStructType, TaskGEP, {Zero, ConstantInt::get(Int32Ty, 4)});2163      // kmp_cmplrdata_t => { ptr, ptr }2164      Type *CmplrStructType = StructType::get(VoidPtr, VoidPtr);2165      Value *CmplrData = Builder.CreateInBoundsGEP(CmplrStructType,2166                                                   PriorityData, {Zero, Zero});2167      Builder.CreateStore(Priority, CmplrData);2168    }2169 2170    Value *DepArray = emitTaskDependencies(*this, Dependencies);2171 2172    // In the presence of the `if` clause, the following IR is generated:2173    //    ...2174    //    %data = call @__kmpc_omp_task_alloc(...)2175    //    br i1 %if_condition, label %then, label %else2176    //  then:2177    //    call @__kmpc_omp_task(...)2178    //    br label %exit2179    //  else:2180    //    ;; Wait for resolution of dependencies, if any, before2181    //    ;; beginning the task2182    //    call @__kmpc_omp_wait_deps(...)2183    //    call @__kmpc_omp_task_begin_if0(...)2184    //    call @outlined_fn(...)2185    //    call @__kmpc_omp_task_complete_if0(...)2186    //    br label %exit2187    //  exit:2188    //    ...2189    if (IfCondition) {2190      // `SplitBlockAndInsertIfThenElse` requires the block to have a2191      // terminator.2192      splitBB(Builder, /*CreateBranch=*/true, "if.end");2193      Instruction *IfTerminator =2194          Builder.GetInsertPoint()->getParent()->getTerminator();2195      Instruction *ThenTI = IfTerminator, *ElseTI = nullptr;2196      Builder.SetInsertPoint(IfTerminator);2197      SplitBlockAndInsertIfThenElse(IfCondition, IfTerminator, &ThenTI,2198                                    &ElseTI);2199      Builder.SetInsertPoint(ElseTI);2200 2201      if (Dependencies.size()) {2202        Function *TaskWaitFn =2203            getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_wait_deps);2204        createRuntimeFunctionCall(2205            TaskWaitFn,2206            {Ident, ThreadID, Builder.getInt32(Dependencies.size()), DepArray,2207             ConstantInt::get(Builder.getInt32Ty(), 0),2208             ConstantPointerNull::get(PointerType::getUnqual(M.getContext()))});2209      }2210      Function *TaskBeginFn =2211          getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_begin_if0);2212      Function *TaskCompleteFn =2213          getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_complete_if0);2214      createRuntimeFunctionCall(TaskBeginFn, {Ident, ThreadID, TaskData});2215      CallInst *CI = nullptr;2216      if (HasShareds)2217        CI = createRuntimeFunctionCall(&OutlinedFn, {ThreadID, TaskData});2218      else2219        CI = createRuntimeFunctionCall(&OutlinedFn, {ThreadID});2220      CI->setDebugLoc(StaleCI->getDebugLoc());2221      createRuntimeFunctionCall(TaskCompleteFn, {Ident, ThreadID, TaskData});2222      Builder.SetInsertPoint(ThenTI);2223    }2224 2225    if (Dependencies.size()) {2226      Function *TaskFn =2227          getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_with_deps);2228      createRuntimeFunctionCall(2229          TaskFn,2230          {Ident, ThreadID, TaskData, Builder.getInt32(Dependencies.size()),2231           DepArray, ConstantInt::get(Builder.getInt32Ty(), 0),2232           ConstantPointerNull::get(PointerType::getUnqual(M.getContext()))});2233 2234    } else {2235      // Emit the @__kmpc_omp_task runtime call to spawn the task2236      Function *TaskFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task);2237      createRuntimeFunctionCall(TaskFn, {Ident, ThreadID, TaskData});2238    }2239 2240    StaleCI->eraseFromParent();2241 2242    Builder.SetInsertPoint(TaskAllocaBB, TaskAllocaBB->begin());2243    if (HasShareds) {2244      LoadInst *Shareds = Builder.CreateLoad(VoidPtr, OutlinedFn.getArg(1));2245      OutlinedFn.getArg(1)->replaceUsesWithIf(2246          Shareds, [Shareds](Use &U) { return U.getUser() != Shareds; });2247    }2248 2249    for (Instruction *I : llvm::reverse(ToBeDeleted))2250      I->eraseFromParent();2251  };2252 2253  addOutlineInfo(std::move(OI));2254  Builder.SetInsertPoint(TaskExitBB, TaskExitBB->begin());2255 2256  return Builder.saveIP();2257}2258 2259OpenMPIRBuilder::InsertPointOrErrorTy2260OpenMPIRBuilder::createTaskgroup(const LocationDescription &Loc,2261                                 InsertPointTy AllocaIP,2262                                 BodyGenCallbackTy BodyGenCB) {2263  if (!updateToLocation(Loc))2264    return InsertPointTy();2265 2266  uint32_t SrcLocStrSize;2267  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);2268  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);2269  Value *ThreadID = getOrCreateThreadID(Ident);2270 2271  // Emit the @__kmpc_taskgroup runtime call to start the taskgroup2272  Function *TaskgroupFn =2273      getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_taskgroup);2274  createRuntimeFunctionCall(TaskgroupFn, {Ident, ThreadID});2275 2276  BasicBlock *TaskgroupExitBB = splitBB(Builder, true, "taskgroup.exit");2277  if (Error Err = BodyGenCB(AllocaIP, Builder.saveIP()))2278    return Err;2279 2280  Builder.SetInsertPoint(TaskgroupExitBB);2281  // Emit the @__kmpc_end_taskgroup runtime call to end the taskgroup2282  Function *EndTaskgroupFn =2283      getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_taskgroup);2284  createRuntimeFunctionCall(EndTaskgroupFn, {Ident, ThreadID});2285 2286  return Builder.saveIP();2287}2288 2289OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createSections(2290    const LocationDescription &Loc, InsertPointTy AllocaIP,2291    ArrayRef<StorableBodyGenCallbackTy> SectionCBs, PrivatizeCallbackTy PrivCB,2292    FinalizeCallbackTy FiniCB, bool IsCancellable, bool IsNowait) {2293  assert(!isConflictIP(AllocaIP, Loc.IP) && "Dedicated IP allocas required");2294 2295  if (!updateToLocation(Loc))2296    return Loc.IP;2297 2298  FinalizationStack.push_back({FiniCB, OMPD_sections, IsCancellable});2299 2300  // Each section is emitted as a switch case2301  // Each finalization callback is handled from clang.EmitOMPSectionDirective()2302  // -> OMP.createSection() which generates the IR for each section2303  // Iterate through all sections and emit a switch construct:2304  // switch (IV) {2305  //   case 0:2306  //     <SectionStmt[0]>;2307  //     break;2308  // ...2309  //   case <NumSection> - 1:2310  //     <SectionStmt[<NumSection> - 1]>;2311  //     break;2312  // }2313  // ...2314  // section_loop.after:2315  // <FiniCB>;2316  auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, Value *IndVar) -> Error {2317    Builder.restoreIP(CodeGenIP);2318    BasicBlock *Continue =2319        splitBBWithSuffix(Builder, /*CreateBranch=*/false, ".sections.after");2320    Function *CurFn = Continue->getParent();2321    SwitchInst *SwitchStmt = Builder.CreateSwitch(IndVar, Continue);2322 2323    unsigned CaseNumber = 0;2324    for (auto SectionCB : SectionCBs) {2325      BasicBlock *CaseBB = BasicBlock::Create(2326          M.getContext(), "omp_section_loop.body.case", CurFn, Continue);2327      SwitchStmt->addCase(Builder.getInt32(CaseNumber), CaseBB);2328      Builder.SetInsertPoint(CaseBB);2329      BranchInst *CaseEndBr = Builder.CreateBr(Continue);2330      if (Error Err = SectionCB(InsertPointTy(), {CaseEndBr->getParent(),2331                                                  CaseEndBr->getIterator()}))2332        return Err;2333      CaseNumber++;2334    }2335    // remove the existing terminator from body BB since there can be no2336    // terminators after switch/case2337    return Error::success();2338  };2339  // Loop body ends here2340  // LowerBound, UpperBound, and STride for createCanonicalLoop2341  Type *I32Ty = Type::getInt32Ty(M.getContext());2342  Value *LB = ConstantInt::get(I32Ty, 0);2343  Value *UB = ConstantInt::get(I32Ty, SectionCBs.size());2344  Value *ST = ConstantInt::get(I32Ty, 1);2345  Expected<CanonicalLoopInfo *> LoopInfo = createCanonicalLoop(2346      Loc, LoopBodyGenCB, LB, UB, ST, true, false, AllocaIP, "section_loop");2347  if (!LoopInfo)2348    return LoopInfo.takeError();2349 2350  InsertPointOrErrorTy WsloopIP =2351      applyStaticWorkshareLoop(Loc.DL, *LoopInfo, AllocaIP,2352                               WorksharingLoopType::ForStaticLoop, !IsNowait);2353  if (!WsloopIP)2354    return WsloopIP.takeError();2355  InsertPointTy AfterIP = *WsloopIP;2356 2357  BasicBlock *LoopFini = AfterIP.getBlock()->getSinglePredecessor();2358  assert(LoopFini && "Bad structure of static workshare loop finalization");2359 2360  // Apply the finalization callback in LoopAfterBB2361  auto FiniInfo = FinalizationStack.pop_back_val();2362  assert(FiniInfo.DK == OMPD_sections &&2363         "Unexpected finalization stack state!");2364  if (Error Err = FiniInfo.mergeFiniBB(Builder, LoopFini))2365    return Err;2366 2367  return AfterIP;2368}2369 2370OpenMPIRBuilder::InsertPointOrErrorTy2371OpenMPIRBuilder::createSection(const LocationDescription &Loc,2372                               BodyGenCallbackTy BodyGenCB,2373                               FinalizeCallbackTy FiniCB) {2374  if (!updateToLocation(Loc))2375    return Loc.IP;2376 2377  auto FiniCBWrapper = [&](InsertPointTy IP) {2378    if (IP.getBlock()->end() != IP.getPoint())2379      return FiniCB(IP);2380    // This must be done otherwise any nested constructs using FinalizeOMPRegion2381    // will fail because that function requires the Finalization Basic Block to2382    // have a terminator, which is already removed by EmitOMPRegionBody.2383    // IP is currently at cancelation block.2384    // We need to backtrack to the condition block to fetch2385    // the exit block and create a branch from cancelation2386    // to exit block.2387    IRBuilder<>::InsertPointGuard IPG(Builder);2388    Builder.restoreIP(IP);2389    auto *CaseBB = Loc.IP.getBlock();2390    auto *CondBB = CaseBB->getSinglePredecessor()->getSinglePredecessor();2391    auto *ExitBB = CondBB->getTerminator()->getSuccessor(1);2392    Instruction *I = Builder.CreateBr(ExitBB);2393    IP = InsertPointTy(I->getParent(), I->getIterator());2394    return FiniCB(IP);2395  };2396 2397  Directive OMPD = Directive::OMPD_sections;2398  // Since we are using Finalization Callback here, HasFinalize2399  // and IsCancellable have to be true2400  return EmitOMPInlinedRegion(OMPD, nullptr, nullptr, BodyGenCB, FiniCBWrapper,2401                              /*Conditional*/ false, /*hasFinalize*/ true,2402                              /*IsCancellable*/ true);2403}2404 2405static OpenMPIRBuilder::InsertPointTy getInsertPointAfterInstr(Instruction *I) {2406  BasicBlock::iterator IT(I);2407  IT++;2408  return OpenMPIRBuilder::InsertPointTy(I->getParent(), IT);2409}2410 2411Value *OpenMPIRBuilder::getGPUThreadID() {2412  return createRuntimeFunctionCall(2413      getOrCreateRuntimeFunction(M,2414                                 OMPRTL___kmpc_get_hardware_thread_id_in_block),2415      {});2416}2417 2418Value *OpenMPIRBuilder::getGPUWarpSize() {2419  return createRuntimeFunctionCall(2420      getOrCreateRuntimeFunction(M, OMPRTL___kmpc_get_warp_size), {});2421}2422 2423Value *OpenMPIRBuilder::getNVPTXWarpID() {2424  unsigned LaneIDBits = Log2_32(Config.getGridValue().GV_Warp_Size);2425  return Builder.CreateAShr(getGPUThreadID(), LaneIDBits, "nvptx_warp_id");2426}2427 2428Value *OpenMPIRBuilder::getNVPTXLaneID() {2429  unsigned LaneIDBits = Log2_32(Config.getGridValue().GV_Warp_Size);2430  assert(LaneIDBits < 32 && "Invalid LaneIDBits size in NVPTX device.");2431  unsigned LaneIDMask = ~0u >> (32u - LaneIDBits);2432  return Builder.CreateAnd(getGPUThreadID(), Builder.getInt32(LaneIDMask),2433                           "nvptx_lane_id");2434}2435 2436Value *OpenMPIRBuilder::castValueToType(InsertPointTy AllocaIP, Value *From,2437                                        Type *ToType) {2438  Type *FromType = From->getType();2439  uint64_t FromSize = M.getDataLayout().getTypeStoreSize(FromType);2440  uint64_t ToSize = M.getDataLayout().getTypeStoreSize(ToType);2441  assert(FromSize > 0 && "From size must be greater than zero");2442  assert(ToSize > 0 && "To size must be greater than zero");2443  if (FromType == ToType)2444    return From;2445  if (FromSize == ToSize)2446    return Builder.CreateBitCast(From, ToType);2447  if (ToType->isIntegerTy() && FromType->isIntegerTy())2448    return Builder.CreateIntCast(From, ToType, /*isSigned*/ true);2449  InsertPointTy SaveIP = Builder.saveIP();2450  Builder.restoreIP(AllocaIP);2451  Value *CastItem = Builder.CreateAlloca(ToType);2452  Builder.restoreIP(SaveIP);2453 2454  Value *ValCastItem = Builder.CreatePointerBitCastOrAddrSpaceCast(2455      CastItem, Builder.getPtrTy(0));2456  Builder.CreateStore(From, ValCastItem);2457  return Builder.CreateLoad(ToType, CastItem);2458}2459 2460Value *OpenMPIRBuilder::createRuntimeShuffleFunction(InsertPointTy AllocaIP,2461                                                     Value *Element,2462                                                     Type *ElementType,2463                                                     Value *Offset) {2464  uint64_t Size = M.getDataLayout().getTypeStoreSize(ElementType);2465  assert(Size <= 8 && "Unsupported bitwidth in shuffle instruction");2466 2467  // Cast all types to 32- or 64-bit values before calling shuffle routines.2468  Type *CastTy = Builder.getIntNTy(Size <= 4 ? 32 : 64);2469  Value *ElemCast = castValueToType(AllocaIP, Element, CastTy);2470  Value *WarpSize =2471      Builder.CreateIntCast(getGPUWarpSize(), Builder.getInt16Ty(), true);2472  Function *ShuffleFunc = getOrCreateRuntimeFunctionPtr(2473      Size <= 4 ? RuntimeFunction::OMPRTL___kmpc_shuffle_int322474                : RuntimeFunction::OMPRTL___kmpc_shuffle_int64);2475  Value *WarpSizeCast =2476      Builder.CreateIntCast(WarpSize, Builder.getInt16Ty(), /*isSigned=*/true);2477  Value *ShuffleCall =2478      createRuntimeFunctionCall(ShuffleFunc, {ElemCast, Offset, WarpSizeCast});2479  return castValueToType(AllocaIP, ShuffleCall, CastTy);2480}2481 2482void OpenMPIRBuilder::shuffleAndStore(InsertPointTy AllocaIP, Value *SrcAddr,2483                                      Value *DstAddr, Type *ElemType,2484                                      Value *Offset, Type *ReductionArrayTy,2485                                      bool IsByRefElem) {2486  uint64_t Size = M.getDataLayout().getTypeStoreSize(ElemType);2487  // Create the loop over the big sized data.2488  // ptr = (void*)Elem;2489  // ptrEnd = (void*) Elem + 1;2490  // Step = 8;2491  // while (ptr + Step < ptrEnd)2492  //   shuffle((int64_t)*ptr);2493  // Step = 4;2494  // while (ptr + Step < ptrEnd)2495  //   shuffle((int32_t)*ptr);2496  // ...2497  Type *IndexTy = Builder.getIndexTy(2498      M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());2499  Value *ElemPtr = DstAddr;2500  Value *Ptr = SrcAddr;2501  for (unsigned IntSize = 8; IntSize >= 1; IntSize /= 2) {2502    if (Size < IntSize)2503      continue;2504    Type *IntType = Builder.getIntNTy(IntSize * 8);2505    Ptr = Builder.CreatePointerBitCastOrAddrSpaceCast(2506        Ptr, Builder.getPtrTy(0), Ptr->getName() + ".ascast");2507    Value *SrcAddrGEP =2508        Builder.CreateGEP(ElemType, SrcAddr, {ConstantInt::get(IndexTy, 1)});2509    ElemPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(2510        ElemPtr, Builder.getPtrTy(0), ElemPtr->getName() + ".ascast");2511 2512    Function *CurFunc = Builder.GetInsertBlock()->getParent();2513    if ((Size / IntSize) > 1) {2514      Value *PtrEnd = Builder.CreatePointerBitCastOrAddrSpaceCast(2515          SrcAddrGEP, Builder.getPtrTy());2516      BasicBlock *PreCondBB =2517          BasicBlock::Create(M.getContext(), ".shuffle.pre_cond");2518      BasicBlock *ThenBB = BasicBlock::Create(M.getContext(), ".shuffle.then");2519      BasicBlock *ExitBB = BasicBlock::Create(M.getContext(), ".shuffle.exit");2520      BasicBlock *CurrentBB = Builder.GetInsertBlock();2521      emitBlock(PreCondBB, CurFunc);2522      PHINode *PhiSrc =2523          Builder.CreatePHI(Ptr->getType(), /*NumReservedValues=*/2);2524      PhiSrc->addIncoming(Ptr, CurrentBB);2525      PHINode *PhiDest =2526          Builder.CreatePHI(ElemPtr->getType(), /*NumReservedValues=*/2);2527      PhiDest->addIncoming(ElemPtr, CurrentBB);2528      Ptr = PhiSrc;2529      ElemPtr = PhiDest;2530      Value *PtrDiff = Builder.CreatePtrDiff(2531          Builder.getInt8Ty(), PtrEnd,2532          Builder.CreatePointerBitCastOrAddrSpaceCast(Ptr, Builder.getPtrTy()));2533      Builder.CreateCondBr(2534          Builder.CreateICmpSGT(PtrDiff, Builder.getInt64(IntSize - 1)), ThenBB,2535          ExitBB);2536      emitBlock(ThenBB, CurFunc);2537      Value *Res = createRuntimeShuffleFunction(2538          AllocaIP,2539          Builder.CreateAlignedLoad(2540              IntType, Ptr, M.getDataLayout().getPrefTypeAlign(ElemType)),2541          IntType, Offset);2542      Builder.CreateAlignedStore(Res, ElemPtr,2543                                 M.getDataLayout().getPrefTypeAlign(ElemType));2544      Value *LocalPtr =2545          Builder.CreateGEP(IntType, Ptr, {ConstantInt::get(IndexTy, 1)});2546      Value *LocalElemPtr =2547          Builder.CreateGEP(IntType, ElemPtr, {ConstantInt::get(IndexTy, 1)});2548      PhiSrc->addIncoming(LocalPtr, ThenBB);2549      PhiDest->addIncoming(LocalElemPtr, ThenBB);2550      emitBranch(PreCondBB);2551      emitBlock(ExitBB, CurFunc);2552    } else {2553      Value *Res = createRuntimeShuffleFunction(2554          AllocaIP, Builder.CreateLoad(IntType, Ptr), IntType, Offset);2555      if (ElemType->isIntegerTy() && ElemType->getScalarSizeInBits() <2556                                         Res->getType()->getScalarSizeInBits())2557        Res = Builder.CreateTrunc(Res, ElemType);2558      Builder.CreateStore(Res, ElemPtr);2559      Ptr = Builder.CreateGEP(IntType, Ptr, {ConstantInt::get(IndexTy, 1)});2560      ElemPtr =2561          Builder.CreateGEP(IntType, ElemPtr, {ConstantInt::get(IndexTy, 1)});2562    }2563    Size = Size % IntSize;2564  }2565}2566 2567Error OpenMPIRBuilder::emitReductionListCopy(2568    InsertPointTy AllocaIP, CopyAction Action, Type *ReductionArrayTy,2569    ArrayRef<ReductionInfo> ReductionInfos, Value *SrcBase, Value *DestBase,2570    ArrayRef<bool> IsByRef, CopyOptionsTy CopyOptions) {2571  Type *IndexTy = Builder.getIndexTy(2572      M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());2573  Value *RemoteLaneOffset = CopyOptions.RemoteLaneOffset;2574 2575  // Iterates, element-by-element, through the source Reduce list and2576  // make a copy.2577  for (auto En : enumerate(ReductionInfos)) {2578    const ReductionInfo &RI = En.value();2579    Value *SrcElementAddr = nullptr;2580    AllocaInst *DestAlloca = nullptr;2581    Value *DestElementAddr = nullptr;2582    Value *DestElementPtrAddr = nullptr;2583    // Should we shuffle in an element from a remote lane?2584    bool ShuffleInElement = false;2585    // Set to true to update the pointer in the dest Reduce list to a2586    // newly created element.2587    bool UpdateDestListPtr = false;2588 2589    // Step 1.1: Get the address for the src element in the Reduce list.2590    Value *SrcElementPtrAddr = Builder.CreateInBoundsGEP(2591        ReductionArrayTy, SrcBase,2592        {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});2593    SrcElementAddr = Builder.CreateLoad(Builder.getPtrTy(), SrcElementPtrAddr);2594 2595    // Step 1.2: Create a temporary to store the element in the destination2596    // Reduce list.2597    DestElementPtrAddr = Builder.CreateInBoundsGEP(2598        ReductionArrayTy, DestBase,2599        {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});2600    bool IsByRefElem = (!IsByRef.empty() && IsByRef[En.index()]);2601    switch (Action) {2602    case CopyAction::RemoteLaneToThread: {2603      InsertPointTy CurIP = Builder.saveIP();2604      Builder.restoreIP(AllocaIP);2605 2606      Type *DestAllocaType =2607          IsByRefElem ? RI.ByRefAllocatedType : RI.ElementType;2608      DestAlloca = Builder.CreateAlloca(DestAllocaType, nullptr,2609                                        ".omp.reduction.element");2610      DestAlloca->setAlignment(2611          M.getDataLayout().getPrefTypeAlign(DestAllocaType));2612      DestElementAddr = DestAlloca;2613      DestElementAddr =2614          Builder.CreateAddrSpaceCast(DestElementAddr, Builder.getPtrTy(),2615                                      DestElementAddr->getName() + ".ascast");2616      Builder.restoreIP(CurIP);2617      ShuffleInElement = true;2618      UpdateDestListPtr = true;2619      break;2620    }2621    case CopyAction::ThreadCopy: {2622      DestElementAddr =2623          Builder.CreateLoad(Builder.getPtrTy(), DestElementPtrAddr);2624      break;2625    }2626    }2627 2628    // Now that all active lanes have read the element in the2629    // Reduce list, shuffle over the value from the remote lane.2630    if (ShuffleInElement) {2631      Type *ShuffleType = RI.ElementType;2632      Value *ShuffleSrcAddr = SrcElementAddr;2633      Value *ShuffleDestAddr = DestElementAddr;2634      AllocaInst *LocalStorage = nullptr;2635 2636      if (IsByRefElem) {2637        assert(RI.ByRefElementType && "Expected by-ref element type to be set");2638        assert(RI.ByRefAllocatedType &&2639               "Expected by-ref allocated type to be set");2640        // For by-ref reductions, we need to copy from the remote lane the2641        // actual value of the partial reduction computed by that remote lane;2642        // rather than, for example, a pointer to that data or, even worse, a2643        // pointer to the descriptor of the by-ref reduction element.2644        ShuffleType = RI.ByRefElementType;2645 2646        InsertPointOrErrorTy GenResult =2647            RI.DataPtrPtrGen(Builder.saveIP(), ShuffleSrcAddr, ShuffleSrcAddr);2648 2649        if (!GenResult)2650          return GenResult.takeError();2651 2652        ShuffleSrcAddr = Builder.CreateLoad(Builder.getPtrTy(), ShuffleSrcAddr);2653 2654        {2655          InsertPointTy OldIP = Builder.saveIP();2656          Builder.restoreIP(AllocaIP);2657 2658          LocalStorage = Builder.CreateAlloca(ShuffleType);2659          Builder.restoreIP(OldIP);2660          ShuffleDestAddr = LocalStorage;2661        }2662      }2663 2664      shuffleAndStore(AllocaIP, ShuffleSrcAddr, ShuffleDestAddr, ShuffleType,2665                      RemoteLaneOffset, ReductionArrayTy, IsByRefElem);2666 2667      if (IsByRefElem) {2668        Value *GEP;2669        InsertPointOrErrorTy GenResult =2670            RI.DataPtrPtrGen(Builder.saveIP(),2671                             Builder.CreatePointerBitCastOrAddrSpaceCast(2672                                 DestAlloca, Builder.getPtrTy(), ".ascast"),2673                             GEP);2674 2675        if (!GenResult)2676          return GenResult.takeError();2677 2678        Builder.CreateStore(Builder.CreatePointerBitCastOrAddrSpaceCast(2679                                LocalStorage, Builder.getPtrTy(), ".ascast"),2680                            GEP);2681      }2682    } else {2683      switch (RI.EvaluationKind) {2684      case EvalKind::Scalar: {2685        Value *Elem = Builder.CreateLoad(RI.ElementType, SrcElementAddr);2686        // Store the source element value to the dest element address.2687        Builder.CreateStore(Elem, DestElementAddr);2688        break;2689      }2690      case EvalKind::Complex: {2691        Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(2692            RI.ElementType, SrcElementAddr, 0, 0, ".realp");2693        Value *SrcReal = Builder.CreateLoad(2694            RI.ElementType->getStructElementType(0), SrcRealPtr, ".real");2695        Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(2696            RI.ElementType, SrcElementAddr, 0, 1, ".imagp");2697        Value *SrcImg = Builder.CreateLoad(2698            RI.ElementType->getStructElementType(1), SrcImgPtr, ".imag");2699 2700        Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(2701            RI.ElementType, DestElementAddr, 0, 0, ".realp");2702        Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(2703            RI.ElementType, DestElementAddr, 0, 1, ".imagp");2704        Builder.CreateStore(SrcReal, DestRealPtr);2705        Builder.CreateStore(SrcImg, DestImgPtr);2706        break;2707      }2708      case EvalKind::Aggregate: {2709        Value *SizeVal = Builder.getInt64(2710            M.getDataLayout().getTypeStoreSize(RI.ElementType));2711        Builder.CreateMemCpy(2712            DestElementAddr, M.getDataLayout().getPrefTypeAlign(RI.ElementType),2713            SrcElementAddr, M.getDataLayout().getPrefTypeAlign(RI.ElementType),2714            SizeVal, false);2715        break;2716      }2717      };2718    }2719 2720    // Step 3.1: Modify reference in dest Reduce list as needed.2721    // Modifying the reference in Reduce list to point to the newly2722    // created element.  The element is live in the current function2723    // scope and that of functions it invokes (i.e., reduce_function).2724    // RemoteReduceData[i] = (void*)&RemoteElem2725    if (UpdateDestListPtr) {2726      Value *CastDestAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(2727          DestElementAddr, Builder.getPtrTy(),2728          DestElementAddr->getName() + ".ascast");2729      Builder.CreateStore(CastDestAddr, DestElementPtrAddr);2730    }2731  }2732 2733  return Error::success();2734}2735 2736Expected<Function *> OpenMPIRBuilder::emitInterWarpCopyFunction(2737    const LocationDescription &Loc, ArrayRef<ReductionInfo> ReductionInfos,2738    AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {2739  InsertPointTy SavedIP = Builder.saveIP();2740  LLVMContext &Ctx = M.getContext();2741  FunctionType *FuncTy = FunctionType::get(2742      Builder.getVoidTy(), {Builder.getPtrTy(), Builder.getInt32Ty()},2743      /* IsVarArg */ false);2744  Function *WcFunc =2745      Function::Create(FuncTy, GlobalVariable::InternalLinkage,2746                       "_omp_reduction_inter_warp_copy_func", &M);2747  WcFunc->setAttributes(FuncAttrs);2748  WcFunc->addParamAttr(0, Attribute::NoUndef);2749  WcFunc->addParamAttr(1, Attribute::NoUndef);2750  BasicBlock *EntryBB = BasicBlock::Create(M.getContext(), "entry", WcFunc);2751  Builder.SetInsertPoint(EntryBB);2752 2753  // ReduceList: thread local Reduce list.2754  // At the stage of the computation when this function is called, partially2755  // aggregated values reside in the first lane of every active warp.2756  Argument *ReduceListArg = WcFunc->getArg(0);2757  // NumWarps: number of warps active in the parallel region.  This could2758  // be smaller than 32 (max warps in a CTA) for partial block reduction.2759  Argument *NumWarpsArg = WcFunc->getArg(1);2760 2761  // This array is used as a medium to transfer, one reduce element at a time,2762  // the data from the first lane of every warp to lanes in the first warp2763  // in order to perform the final step of a reduction in a parallel region2764  // (reduction across warps).  The array is placed in NVPTX __shared__ memory2765  // for reduced latency, as well as to have a distinct copy for concurrently2766  // executing target regions.  The array is declared with common linkage so2767  // as to be shared across compilation units.2768  StringRef TransferMediumName =2769      "__openmp_nvptx_data_transfer_temporary_storage";2770  GlobalVariable *TransferMedium = M.getGlobalVariable(TransferMediumName);2771  unsigned WarpSize = Config.getGridValue().GV_Warp_Size;2772  ArrayType *ArrayTy = ArrayType::get(Builder.getInt32Ty(), WarpSize);2773  if (!TransferMedium) {2774    TransferMedium = new GlobalVariable(2775        M, ArrayTy, /*isConstant=*/false, GlobalVariable::WeakAnyLinkage,2776        UndefValue::get(ArrayTy), TransferMediumName,2777        /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,2778        /*AddressSpace=*/3);2779  }2780 2781  // Get the CUDA thread id of the current OpenMP thread on the GPU.2782  Value *GPUThreadID = getGPUThreadID();2783  // nvptx_lane_id = nvptx_id % warpsize2784  Value *LaneID = getNVPTXLaneID();2785  // nvptx_warp_id = nvptx_id / warpsize2786  Value *WarpID = getNVPTXWarpID();2787 2788  InsertPointTy AllocaIP =2789      InsertPointTy(Builder.GetInsertBlock(),2790                    Builder.GetInsertBlock()->getFirstInsertionPt());2791  Type *Arg0Type = ReduceListArg->getType();2792  Type *Arg1Type = NumWarpsArg->getType();2793  Builder.restoreIP(AllocaIP);2794  AllocaInst *ReduceListAlloca = Builder.CreateAlloca(2795      Arg0Type, nullptr, ReduceListArg->getName() + ".addr");2796  AllocaInst *NumWarpsAlloca =2797      Builder.CreateAlloca(Arg1Type, nullptr, NumWarpsArg->getName() + ".addr");2798  Value *ReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(2799      ReduceListAlloca, Arg0Type, ReduceListAlloca->getName() + ".ascast");2800  Value *NumWarpsAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(2801      NumWarpsAlloca, Builder.getPtrTy(0),2802      NumWarpsAlloca->getName() + ".ascast");2803  Builder.CreateStore(ReduceListArg, ReduceListAddrCast);2804  Builder.CreateStore(NumWarpsArg, NumWarpsAddrCast);2805  AllocaIP = getInsertPointAfterInstr(NumWarpsAlloca);2806  InsertPointTy CodeGenIP =2807      getInsertPointAfterInstr(&Builder.GetInsertBlock()->back());2808  Builder.restoreIP(CodeGenIP);2809 2810  Value *ReduceList =2811      Builder.CreateLoad(Builder.getPtrTy(), ReduceListAddrCast);2812 2813  for (auto En : enumerate(ReductionInfos)) {2814    //2815    // Warp master copies reduce element to transfer medium in __shared__2816    // memory.2817    //2818    const ReductionInfo &RI = En.value();2819    bool IsByRefElem = !IsByRef.empty() && IsByRef[En.index()];2820    unsigned RealTySize = M.getDataLayout().getTypeAllocSize(2821        IsByRefElem ? RI.ByRefElementType : RI.ElementType);2822    for (unsigned TySize = 4; TySize > 0 && RealTySize > 0; TySize /= 2) {2823      Type *CType = Builder.getIntNTy(TySize * 8);2824 2825      unsigned NumIters = RealTySize / TySize;2826      if (NumIters == 0)2827        continue;2828      Value *Cnt = nullptr;2829      Value *CntAddr = nullptr;2830      BasicBlock *PrecondBB = nullptr;2831      BasicBlock *ExitBB = nullptr;2832      if (NumIters > 1) {2833        CodeGenIP = Builder.saveIP();2834        Builder.restoreIP(AllocaIP);2835        CntAddr =2836            Builder.CreateAlloca(Builder.getInt32Ty(), nullptr, ".cnt.addr");2837 2838        CntAddr = Builder.CreateAddrSpaceCast(CntAddr, Builder.getPtrTy(),2839                                              CntAddr->getName() + ".ascast");2840        Builder.restoreIP(CodeGenIP);2841        Builder.CreateStore(Constant::getNullValue(Builder.getInt32Ty()),2842                            CntAddr,2843                            /*Volatile=*/false);2844        PrecondBB = BasicBlock::Create(Ctx, "precond");2845        ExitBB = BasicBlock::Create(Ctx, "exit");2846        BasicBlock *BodyBB = BasicBlock::Create(Ctx, "body");2847        emitBlock(PrecondBB, Builder.GetInsertBlock()->getParent());2848        Cnt = Builder.CreateLoad(Builder.getInt32Ty(), CntAddr,2849                                 /*Volatile=*/false);2850        Value *Cmp = Builder.CreateICmpULT(2851            Cnt, ConstantInt::get(Builder.getInt32Ty(), NumIters));2852        Builder.CreateCondBr(Cmp, BodyBB, ExitBB);2853        emitBlock(BodyBB, Builder.GetInsertBlock()->getParent());2854      }2855 2856      // kmpc_barrier.2857      InsertPointOrErrorTy BarrierIP1 =2858          createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),2859                        omp::Directive::OMPD_unknown,2860                        /* ForceSimpleCall */ false,2861                        /* CheckCancelFlag */ true);2862      if (!BarrierIP1)2863        return BarrierIP1.takeError();2864      BasicBlock *ThenBB = BasicBlock::Create(Ctx, "then");2865      BasicBlock *ElseBB = BasicBlock::Create(Ctx, "else");2866      BasicBlock *MergeBB = BasicBlock::Create(Ctx, "ifcont");2867 2868      // if (lane_id  == 0)2869      Value *IsWarpMaster = Builder.CreateIsNull(LaneID, "warp_master");2870      Builder.CreateCondBr(IsWarpMaster, ThenBB, ElseBB);2871      emitBlock(ThenBB, Builder.GetInsertBlock()->getParent());2872 2873      // Reduce element = LocalReduceList[i]2874      auto *RedListArrayTy =2875          ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());2876      Type *IndexTy = Builder.getIndexTy(2877          M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());2878      Value *ElemPtrPtr =2879          Builder.CreateInBoundsGEP(RedListArrayTy, ReduceList,2880                                    {ConstantInt::get(IndexTy, 0),2881                                     ConstantInt::get(IndexTy, En.index())});2882      // elemptr = ((CopyType*)(elemptrptr)) + I2883      Value *ElemPtr = Builder.CreateLoad(Builder.getPtrTy(), ElemPtrPtr);2884 2885      if (IsByRefElem) {2886        InsertPointOrErrorTy GenRes =2887            RI.DataPtrPtrGen(Builder.saveIP(), ElemPtr, ElemPtr);2888 2889        if (!GenRes)2890          return GenRes.takeError();2891 2892        ElemPtr = Builder.CreateLoad(Builder.getPtrTy(), ElemPtr);2893      }2894 2895      if (NumIters > 1)2896        ElemPtr = Builder.CreateGEP(Builder.getInt32Ty(), ElemPtr, Cnt);2897 2898      // Get pointer to location in transfer medium.2899      // MediumPtr = &medium[warp_id]2900      Value *MediumPtr = Builder.CreateInBoundsGEP(2901          ArrayTy, TransferMedium, {Builder.getInt64(0), WarpID});2902      // elem = *elemptr2903      //*MediumPtr = elem2904      Value *Elem = Builder.CreateLoad(CType, ElemPtr);2905      // Store the source element value to the dest element address.2906      Builder.CreateStore(Elem, MediumPtr,2907                          /*IsVolatile*/ true);2908      Builder.CreateBr(MergeBB);2909 2910      // else2911      emitBlock(ElseBB, Builder.GetInsertBlock()->getParent());2912      Builder.CreateBr(MergeBB);2913 2914      // endif2915      emitBlock(MergeBB, Builder.GetInsertBlock()->getParent());2916      InsertPointOrErrorTy BarrierIP2 =2917          createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),2918                        omp::Directive::OMPD_unknown,2919                        /* ForceSimpleCall */ false,2920                        /* CheckCancelFlag */ true);2921      if (!BarrierIP2)2922        return BarrierIP2.takeError();2923 2924      // Warp 0 copies reduce element from transfer medium2925      BasicBlock *W0ThenBB = BasicBlock::Create(Ctx, "then");2926      BasicBlock *W0ElseBB = BasicBlock::Create(Ctx, "else");2927      BasicBlock *W0MergeBB = BasicBlock::Create(Ctx, "ifcont");2928 2929      Value *NumWarpsVal =2930          Builder.CreateLoad(Builder.getInt32Ty(), NumWarpsAddrCast);2931      // Up to 32 threads in warp 0 are active.2932      Value *IsActiveThread =2933          Builder.CreateICmpULT(GPUThreadID, NumWarpsVal, "is_active_thread");2934      Builder.CreateCondBr(IsActiveThread, W0ThenBB, W0ElseBB);2935 2936      emitBlock(W0ThenBB, Builder.GetInsertBlock()->getParent());2937 2938      // SecMediumPtr = &medium[tid]2939      // SrcMediumVal = *SrcMediumPtr2940      Value *SrcMediumPtrVal = Builder.CreateInBoundsGEP(2941          ArrayTy, TransferMedium, {Builder.getInt64(0), GPUThreadID});2942      // TargetElemPtr = (CopyType*)(SrcDataAddr[i]) + I2943      Value *TargetElemPtrPtr =2944          Builder.CreateInBoundsGEP(RedListArrayTy, ReduceList,2945                                    {ConstantInt::get(IndexTy, 0),2946                                     ConstantInt::get(IndexTy, En.index())});2947      Value *TargetElemPtrVal =2948          Builder.CreateLoad(Builder.getPtrTy(), TargetElemPtrPtr);2949      Value *TargetElemPtr = TargetElemPtrVal;2950 2951      if (IsByRefElem) {2952        InsertPointOrErrorTy GenRes =2953            RI.DataPtrPtrGen(Builder.saveIP(), TargetElemPtr, TargetElemPtr);2954 2955        if (!GenRes)2956          return GenRes.takeError();2957 2958        TargetElemPtr = Builder.CreateLoad(Builder.getPtrTy(), TargetElemPtr);2959      }2960 2961      if (NumIters > 1)2962        TargetElemPtr =2963            Builder.CreateGEP(Builder.getInt32Ty(), TargetElemPtr, Cnt);2964 2965      // *TargetElemPtr = SrcMediumVal;2966      Value *SrcMediumValue =2967          Builder.CreateLoad(CType, SrcMediumPtrVal, /*IsVolatile*/ true);2968      Builder.CreateStore(SrcMediumValue, TargetElemPtr);2969      Builder.CreateBr(W0MergeBB);2970 2971      emitBlock(W0ElseBB, Builder.GetInsertBlock()->getParent());2972      Builder.CreateBr(W0MergeBB);2973 2974      emitBlock(W0MergeBB, Builder.GetInsertBlock()->getParent());2975 2976      if (NumIters > 1) {2977        Cnt = Builder.CreateNSWAdd(2978            Cnt, ConstantInt::get(Builder.getInt32Ty(), /*V=*/1));2979        Builder.CreateStore(Cnt, CntAddr, /*Volatile=*/false);2980 2981        auto *CurFn = Builder.GetInsertBlock()->getParent();2982        emitBranch(PrecondBB);2983        emitBlock(ExitBB, CurFn);2984      }2985      RealTySize %= TySize;2986    }2987  }2988 2989  Builder.CreateRetVoid();2990  Builder.restoreIP(SavedIP);2991 2992  return WcFunc;2993}2994 2995Expected<Function *> OpenMPIRBuilder::emitShuffleAndReduceFunction(2996    ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,2997    AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {2998  LLVMContext &Ctx = M.getContext();2999  FunctionType *FuncTy =3000      FunctionType::get(Builder.getVoidTy(),3001                        {Builder.getPtrTy(), Builder.getInt16Ty(),3002                         Builder.getInt16Ty(), Builder.getInt16Ty()},3003                        /* IsVarArg */ false);3004  Function *SarFunc =3005      Function::Create(FuncTy, GlobalVariable::InternalLinkage,3006                       "_omp_reduction_shuffle_and_reduce_func", &M);3007  SarFunc->setAttributes(FuncAttrs);3008  SarFunc->addParamAttr(0, Attribute::NoUndef);3009  SarFunc->addParamAttr(1, Attribute::NoUndef);3010  SarFunc->addParamAttr(2, Attribute::NoUndef);3011  SarFunc->addParamAttr(3, Attribute::NoUndef);3012  SarFunc->addParamAttr(1, Attribute::SExt);3013  SarFunc->addParamAttr(2, Attribute::SExt);3014  SarFunc->addParamAttr(3, Attribute::SExt);3015  BasicBlock *EntryBB = BasicBlock::Create(M.getContext(), "entry", SarFunc);3016  Builder.SetInsertPoint(EntryBB);3017 3018  // Thread local Reduce list used to host the values of data to be reduced.3019  Argument *ReduceListArg = SarFunc->getArg(0);3020  // Current lane id; could be logical.3021  Argument *LaneIDArg = SarFunc->getArg(1);3022  // Offset of the remote source lane relative to the current lane.3023  Argument *RemoteLaneOffsetArg = SarFunc->getArg(2);3024  // Algorithm version.  This is expected to be known at compile time.3025  Argument *AlgoVerArg = SarFunc->getArg(3);3026 3027  Type *ReduceListArgType = ReduceListArg->getType();3028  Type *LaneIDArgType = LaneIDArg->getType();3029  Type *LaneIDArgPtrType = Builder.getPtrTy(0);3030  Value *ReduceListAlloca = Builder.CreateAlloca(3031      ReduceListArgType, nullptr, ReduceListArg->getName() + ".addr");3032  Value *LaneIdAlloca = Builder.CreateAlloca(LaneIDArgType, nullptr,3033                                             LaneIDArg->getName() + ".addr");3034  Value *RemoteLaneOffsetAlloca = Builder.CreateAlloca(3035      LaneIDArgType, nullptr, RemoteLaneOffsetArg->getName() + ".addr");3036  Value *AlgoVerAlloca = Builder.CreateAlloca(LaneIDArgType, nullptr,3037                                              AlgoVerArg->getName() + ".addr");3038  ArrayType *RedListArrayTy =3039      ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());3040 3041  // Create a local thread-private variable to host the Reduce list3042  // from a remote lane.3043  Instruction *RemoteReductionListAlloca = Builder.CreateAlloca(3044      RedListArrayTy, nullptr, ".omp.reduction.remote_reduce_list");3045 3046  Value *ReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(3047      ReduceListAlloca, ReduceListArgType,3048      ReduceListAlloca->getName() + ".ascast");3049  Value *LaneIdAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(3050      LaneIdAlloca, LaneIDArgPtrType, LaneIdAlloca->getName() + ".ascast");3051  Value *RemoteLaneOffsetAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(3052      RemoteLaneOffsetAlloca, LaneIDArgPtrType,3053      RemoteLaneOffsetAlloca->getName() + ".ascast");3054  Value *AlgoVerAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(3055      AlgoVerAlloca, LaneIDArgPtrType, AlgoVerAlloca->getName() + ".ascast");3056  Value *RemoteListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(3057      RemoteReductionListAlloca, Builder.getPtrTy(),3058      RemoteReductionListAlloca->getName() + ".ascast");3059 3060  Builder.CreateStore(ReduceListArg, ReduceListAddrCast);3061  Builder.CreateStore(LaneIDArg, LaneIdAddrCast);3062  Builder.CreateStore(RemoteLaneOffsetArg, RemoteLaneOffsetAddrCast);3063  Builder.CreateStore(AlgoVerArg, AlgoVerAddrCast);3064 3065  Value *ReduceList = Builder.CreateLoad(ReduceListArgType, ReduceListAddrCast);3066  Value *LaneId = Builder.CreateLoad(LaneIDArgType, LaneIdAddrCast);3067  Value *RemoteLaneOffset =3068      Builder.CreateLoad(LaneIDArgType, RemoteLaneOffsetAddrCast);3069  Value *AlgoVer = Builder.CreateLoad(LaneIDArgType, AlgoVerAddrCast);3070 3071  InsertPointTy AllocaIP = getInsertPointAfterInstr(RemoteReductionListAlloca);3072 3073  // This loop iterates through the list of reduce elements and copies,3074  // element by element, from a remote lane in the warp to RemoteReduceList,3075  // hosted on the thread's stack.3076  Error EmitRedLsCpRes = emitReductionListCopy(3077      AllocaIP, CopyAction::RemoteLaneToThread, RedListArrayTy, ReductionInfos,3078      ReduceList, RemoteListAddrCast, IsByRef,3079      {RemoteLaneOffset, nullptr, nullptr});3080 3081  if (EmitRedLsCpRes)3082    return EmitRedLsCpRes;3083 3084  // The actions to be performed on the Remote Reduce list is dependent3085  // on the algorithm version.3086  //3087  //  if (AlgoVer==0) || (AlgoVer==1 && (LaneId < Offset)) || (AlgoVer==2 &&3088  //  LaneId % 2 == 0 && Offset > 0):3089  //    do the reduction value aggregation3090  //3091  //  The thread local variable Reduce list is mutated in place to host the3092  //  reduced data, which is the aggregated value produced from local and3093  //  remote lanes.3094  //3095  //  Note that AlgoVer is expected to be a constant integer known at compile3096  //  time.3097  //  When AlgoVer==0, the first conjunction evaluates to true, making3098  //    the entire predicate true during compile time.3099  //  When AlgoVer==1, the second conjunction has only the second part to be3100  //    evaluated during runtime.  Other conjunctions evaluates to false3101  //    during compile time.3102  //  When AlgoVer==2, the third conjunction has only the second part to be3103  //    evaluated during runtime.  Other conjunctions evaluates to false3104  //    during compile time.3105  Value *CondAlgo0 = Builder.CreateIsNull(AlgoVer);3106  Value *Algo1 = Builder.CreateICmpEQ(AlgoVer, Builder.getInt16(1));3107  Value *LaneComp = Builder.CreateICmpULT(LaneId, RemoteLaneOffset);3108  Value *CondAlgo1 = Builder.CreateAnd(Algo1, LaneComp);3109  Value *Algo2 = Builder.CreateICmpEQ(AlgoVer, Builder.getInt16(2));3110  Value *LaneIdAnd1 = Builder.CreateAnd(LaneId, Builder.getInt16(1));3111  Value *LaneIdComp = Builder.CreateIsNull(LaneIdAnd1);3112  Value *Algo2AndLaneIdComp = Builder.CreateAnd(Algo2, LaneIdComp);3113  Value *RemoteOffsetComp =3114      Builder.CreateICmpSGT(RemoteLaneOffset, Builder.getInt16(0));3115  Value *CondAlgo2 = Builder.CreateAnd(Algo2AndLaneIdComp, RemoteOffsetComp);3116  Value *CA0OrCA1 = Builder.CreateOr(CondAlgo0, CondAlgo1);3117  Value *CondReduce = Builder.CreateOr(CA0OrCA1, CondAlgo2);3118 3119  BasicBlock *ThenBB = BasicBlock::Create(Ctx, "then");3120  BasicBlock *ElseBB = BasicBlock::Create(Ctx, "else");3121  BasicBlock *MergeBB = BasicBlock::Create(Ctx, "ifcont");3122 3123  Builder.CreateCondBr(CondReduce, ThenBB, ElseBB);3124  emitBlock(ThenBB, Builder.GetInsertBlock()->getParent());3125  Value *LocalReduceListPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(3126      ReduceList, Builder.getPtrTy());3127  Value *RemoteReduceListPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(3128      RemoteListAddrCast, Builder.getPtrTy());3129  createRuntimeFunctionCall(ReduceFn, {LocalReduceListPtr, RemoteReduceListPtr})3130      ->addFnAttr(Attribute::NoUnwind);3131  Builder.CreateBr(MergeBB);3132 3133  emitBlock(ElseBB, Builder.GetInsertBlock()->getParent());3134  Builder.CreateBr(MergeBB);3135 3136  emitBlock(MergeBB, Builder.GetInsertBlock()->getParent());3137 3138  // if (AlgoVer==1 && (LaneId >= Offset)) copy Remote Reduce list to local3139  // Reduce list.3140  Algo1 = Builder.CreateICmpEQ(AlgoVer, Builder.getInt16(1));3141  Value *LaneIdGtOffset = Builder.CreateICmpUGE(LaneId, RemoteLaneOffset);3142  Value *CondCopy = Builder.CreateAnd(Algo1, LaneIdGtOffset);3143 3144  BasicBlock *CpyThenBB = BasicBlock::Create(Ctx, "then");3145  BasicBlock *CpyElseBB = BasicBlock::Create(Ctx, "else");3146  BasicBlock *CpyMergeBB = BasicBlock::Create(Ctx, "ifcont");3147  Builder.CreateCondBr(CondCopy, CpyThenBB, CpyElseBB);3148 3149  emitBlock(CpyThenBB, Builder.GetInsertBlock()->getParent());3150 3151  EmitRedLsCpRes = emitReductionListCopy(3152      AllocaIP, CopyAction::ThreadCopy, RedListArrayTy, ReductionInfos,3153      RemoteListAddrCast, ReduceList, IsByRef);3154 3155  if (EmitRedLsCpRes)3156    return EmitRedLsCpRes;3157 3158  Builder.CreateBr(CpyMergeBB);3159 3160  emitBlock(CpyElseBB, Builder.GetInsertBlock()->getParent());3161  Builder.CreateBr(CpyMergeBB);3162 3163  emitBlock(CpyMergeBB, Builder.GetInsertBlock()->getParent());3164 3165  Builder.CreateRetVoid();3166 3167  return SarFunc;3168}3169 3170Function *OpenMPIRBuilder::emitListToGlobalCopyFunction(3171    ArrayRef<ReductionInfo> ReductionInfos, Type *ReductionsBufferTy,3172    AttributeList FuncAttrs) {3173  OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();3174  LLVMContext &Ctx = M.getContext();3175  FunctionType *FuncTy = FunctionType::get(3176      Builder.getVoidTy(),3177      {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},3178      /* IsVarArg */ false);3179  Function *LtGCFunc =3180      Function::Create(FuncTy, GlobalVariable::InternalLinkage,3181                       "_omp_reduction_list_to_global_copy_func", &M);3182  LtGCFunc->setAttributes(FuncAttrs);3183  LtGCFunc->addParamAttr(0, Attribute::NoUndef);3184  LtGCFunc->addParamAttr(1, Attribute::NoUndef);3185  LtGCFunc->addParamAttr(2, Attribute::NoUndef);3186 3187  BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "entry", LtGCFunc);3188  Builder.SetInsertPoint(EntryBlock);3189 3190  // Buffer: global reduction buffer.3191  Argument *BufferArg = LtGCFunc->getArg(0);3192  // Idx: index of the buffer.3193  Argument *IdxArg = LtGCFunc->getArg(1);3194  // ReduceList: thread local Reduce list.3195  Argument *ReduceListArg = LtGCFunc->getArg(2);3196 3197  Value *BufferArgAlloca = Builder.CreateAlloca(Builder.getPtrTy(), nullptr,3198                                                BufferArg->getName() + ".addr");3199  Value *IdxArgAlloca = Builder.CreateAlloca(Builder.getInt32Ty(), nullptr,3200                                             IdxArg->getName() + ".addr");3201  Value *ReduceListArgAlloca = Builder.CreateAlloca(3202      Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr");3203  Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(3204      BufferArgAlloca, Builder.getPtrTy(),3205      BufferArgAlloca->getName() + ".ascast");3206  Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(3207      IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast");3208  Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(3209      ReduceListArgAlloca, Builder.getPtrTy(),3210      ReduceListArgAlloca->getName() + ".ascast");3211 3212  Builder.CreateStore(BufferArg, BufferArgAddrCast);3213  Builder.CreateStore(IdxArg, IdxArgAddrCast);3214  Builder.CreateStore(ReduceListArg, ReduceListArgAddrCast);3215 3216  Value *LocalReduceList =3217      Builder.CreateLoad(Builder.getPtrTy(), ReduceListArgAddrCast);3218  Value *BufferArgVal =3219      Builder.CreateLoad(Builder.getPtrTy(), BufferArgAddrCast);3220  Value *Idxs[] = {Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast)};3221  Type *IndexTy = Builder.getIndexTy(3222      M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());3223  for (auto En : enumerate(ReductionInfos)) {3224    const ReductionInfo &RI = En.value();3225    auto *RedListArrayTy =3226        ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());3227    // Reduce element = LocalReduceList[i]3228    Value *ElemPtrPtr = Builder.CreateInBoundsGEP(3229        RedListArrayTy, LocalReduceList,3230        {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});3231    // elemptr = ((CopyType*)(elemptrptr)) + I3232    Value *ElemPtr = Builder.CreateLoad(Builder.getPtrTy(), ElemPtrPtr);3233 3234    // Global = Buffer.VD[Idx];3235    Value *BufferVD =3236        Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferArgVal, Idxs);3237    Value *GlobVal = Builder.CreateConstInBoundsGEP2_32(3238        ReductionsBufferTy, BufferVD, 0, En.index());3239 3240    switch (RI.EvaluationKind) {3241    case EvalKind::Scalar: {3242      Value *TargetElement = Builder.CreateLoad(RI.ElementType, ElemPtr);3243      Builder.CreateStore(TargetElement, GlobVal);3244      break;3245    }3246    case EvalKind::Complex: {3247      Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(3248          RI.ElementType, ElemPtr, 0, 0, ".realp");3249      Value *SrcReal = Builder.CreateLoad(3250          RI.ElementType->getStructElementType(0), SrcRealPtr, ".real");3251      Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(3252          RI.ElementType, ElemPtr, 0, 1, ".imagp");3253      Value *SrcImg = Builder.CreateLoad(3254          RI.ElementType->getStructElementType(1), SrcImgPtr, ".imag");3255 3256      Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(3257          RI.ElementType, GlobVal, 0, 0, ".realp");3258      Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(3259          RI.ElementType, GlobVal, 0, 1, ".imagp");3260      Builder.CreateStore(SrcReal, DestRealPtr);3261      Builder.CreateStore(SrcImg, DestImgPtr);3262      break;3263    }3264    case EvalKind::Aggregate: {3265      Value *SizeVal =3266          Builder.getInt64(M.getDataLayout().getTypeStoreSize(RI.ElementType));3267      Builder.CreateMemCpy(3268          GlobVal, M.getDataLayout().getPrefTypeAlign(RI.ElementType), ElemPtr,3269          M.getDataLayout().getPrefTypeAlign(RI.ElementType), SizeVal, false);3270      break;3271    }3272    }3273  }3274 3275  Builder.CreateRetVoid();3276  Builder.restoreIP(OldIP);3277  return LtGCFunc;3278}3279 3280Function *OpenMPIRBuilder::emitListToGlobalReduceFunction(3281    ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,3282    Type *ReductionsBufferTy, AttributeList FuncAttrs) {3283  OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();3284  LLVMContext &Ctx = M.getContext();3285  FunctionType *FuncTy = FunctionType::get(3286      Builder.getVoidTy(),3287      {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},3288      /* IsVarArg */ false);3289  Function *LtGRFunc =3290      Function::Create(FuncTy, GlobalVariable::InternalLinkage,3291                       "_omp_reduction_list_to_global_reduce_func", &M);3292  LtGRFunc->setAttributes(FuncAttrs);3293  LtGRFunc->addParamAttr(0, Attribute::NoUndef);3294  LtGRFunc->addParamAttr(1, Attribute::NoUndef);3295  LtGRFunc->addParamAttr(2, Attribute::NoUndef);3296 3297  BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "entry", LtGRFunc);3298  Builder.SetInsertPoint(EntryBlock);3299 3300  // Buffer: global reduction buffer.3301  Argument *BufferArg = LtGRFunc->getArg(0);3302  // Idx: index of the buffer.3303  Argument *IdxArg = LtGRFunc->getArg(1);3304  // ReduceList: thread local Reduce list.3305  Argument *ReduceListArg = LtGRFunc->getArg(2);3306 3307  Value *BufferArgAlloca = Builder.CreateAlloca(Builder.getPtrTy(), nullptr,3308                                                BufferArg->getName() + ".addr");3309  Value *IdxArgAlloca = Builder.CreateAlloca(Builder.getInt32Ty(), nullptr,3310                                             IdxArg->getName() + ".addr");3311  Value *ReduceListArgAlloca = Builder.CreateAlloca(3312      Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr");3313  auto *RedListArrayTy =3314      ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());3315 3316  // 1. Build a list of reduction variables.3317  // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};3318  Value *LocalReduceList =3319      Builder.CreateAlloca(RedListArrayTy, nullptr, ".omp.reduction.red_list");3320 3321  Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(3322      BufferArgAlloca, Builder.getPtrTy(),3323      BufferArgAlloca->getName() + ".ascast");3324  Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(3325      IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast");3326  Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(3327      ReduceListArgAlloca, Builder.getPtrTy(),3328      ReduceListArgAlloca->getName() + ".ascast");3329  Value *LocalReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(3330      LocalReduceList, Builder.getPtrTy(),3331      LocalReduceList->getName() + ".ascast");3332 3333  Builder.CreateStore(BufferArg, BufferArgAddrCast);3334  Builder.CreateStore(IdxArg, IdxArgAddrCast);3335  Builder.CreateStore(ReduceListArg, ReduceListArgAddrCast);3336 3337  Value *BufferVal = Builder.CreateLoad(Builder.getPtrTy(), BufferArgAddrCast);3338  Value *Idxs[] = {Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast)};3339  Type *IndexTy = Builder.getIndexTy(3340      M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());3341  for (auto En : enumerate(ReductionInfos)) {3342    Value *TargetElementPtrPtr = Builder.CreateInBoundsGEP(3343        RedListArrayTy, LocalReduceListAddrCast,3344        {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});3345    Value *BufferVD =3346        Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferVal, Idxs);3347    // Global = Buffer.VD[Idx];3348    Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(3349        ReductionsBufferTy, BufferVD, 0, En.index());3350    Builder.CreateStore(GlobValPtr, TargetElementPtrPtr);3351  }3352 3353  // Call reduce_function(GlobalReduceList, ReduceList)3354  Value *ReduceList =3355      Builder.CreateLoad(Builder.getPtrTy(), ReduceListArgAddrCast);3356  createRuntimeFunctionCall(ReduceFn, {LocalReduceListAddrCast, ReduceList})3357      ->addFnAttr(Attribute::NoUnwind);3358  Builder.CreateRetVoid();3359  Builder.restoreIP(OldIP);3360  return LtGRFunc;3361}3362 3363Function *OpenMPIRBuilder::emitGlobalToListCopyFunction(3364    ArrayRef<ReductionInfo> ReductionInfos, Type *ReductionsBufferTy,3365    AttributeList FuncAttrs) {3366  OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();3367  LLVMContext &Ctx = M.getContext();3368  FunctionType *FuncTy = FunctionType::get(3369      Builder.getVoidTy(),3370      {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},3371      /* IsVarArg */ false);3372  Function *LtGCFunc =3373      Function::Create(FuncTy, GlobalVariable::InternalLinkage,3374                       "_omp_reduction_global_to_list_copy_func", &M);3375  LtGCFunc->setAttributes(FuncAttrs);3376  LtGCFunc->addParamAttr(0, Attribute::NoUndef);3377  LtGCFunc->addParamAttr(1, Attribute::NoUndef);3378  LtGCFunc->addParamAttr(2, Attribute::NoUndef);3379 3380  BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "entry", LtGCFunc);3381  Builder.SetInsertPoint(EntryBlock);3382 3383  // Buffer: global reduction buffer.3384  Argument *BufferArg = LtGCFunc->getArg(0);3385  // Idx: index of the buffer.3386  Argument *IdxArg = LtGCFunc->getArg(1);3387  // ReduceList: thread local Reduce list.3388  Argument *ReduceListArg = LtGCFunc->getArg(2);3389 3390  Value *BufferArgAlloca = Builder.CreateAlloca(Builder.getPtrTy(), nullptr,3391                                                BufferArg->getName() + ".addr");3392  Value *IdxArgAlloca = Builder.CreateAlloca(Builder.getInt32Ty(), nullptr,3393                                             IdxArg->getName() + ".addr");3394  Value *ReduceListArgAlloca = Builder.CreateAlloca(3395      Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr");3396  Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(3397      BufferArgAlloca, Builder.getPtrTy(),3398      BufferArgAlloca->getName() + ".ascast");3399  Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(3400      IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast");3401  Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(3402      ReduceListArgAlloca, Builder.getPtrTy(),3403      ReduceListArgAlloca->getName() + ".ascast");3404  Builder.CreateStore(BufferArg, BufferArgAddrCast);3405  Builder.CreateStore(IdxArg, IdxArgAddrCast);3406  Builder.CreateStore(ReduceListArg, ReduceListArgAddrCast);3407 3408  Value *LocalReduceList =3409      Builder.CreateLoad(Builder.getPtrTy(), ReduceListArgAddrCast);3410  Value *BufferVal = Builder.CreateLoad(Builder.getPtrTy(), BufferArgAddrCast);3411  Value *Idxs[] = {Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast)};3412  Type *IndexTy = Builder.getIndexTy(3413      M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());3414  for (auto En : enumerate(ReductionInfos)) {3415    const OpenMPIRBuilder::ReductionInfo &RI = En.value();3416    auto *RedListArrayTy =3417        ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());3418    // Reduce element = LocalReduceList[i]3419    Value *ElemPtrPtr = Builder.CreateInBoundsGEP(3420        RedListArrayTy, LocalReduceList,3421        {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});3422    // elemptr = ((CopyType*)(elemptrptr)) + I3423    Value *ElemPtr = Builder.CreateLoad(Builder.getPtrTy(), ElemPtrPtr);3424    // Global = Buffer.VD[Idx];3425    Value *BufferVD =3426        Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferVal, Idxs);3427    Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(3428        ReductionsBufferTy, BufferVD, 0, En.index());3429 3430    switch (RI.EvaluationKind) {3431    case EvalKind::Scalar: {3432      Value *TargetElement = Builder.CreateLoad(RI.ElementType, GlobValPtr);3433      Builder.CreateStore(TargetElement, ElemPtr);3434      break;3435    }3436    case EvalKind::Complex: {3437      Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(3438          RI.ElementType, GlobValPtr, 0, 0, ".realp");3439      Value *SrcReal = Builder.CreateLoad(3440          RI.ElementType->getStructElementType(0), SrcRealPtr, ".real");3441      Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(3442          RI.ElementType, GlobValPtr, 0, 1, ".imagp");3443      Value *SrcImg = Builder.CreateLoad(3444          RI.ElementType->getStructElementType(1), SrcImgPtr, ".imag");3445 3446      Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(3447          RI.ElementType, ElemPtr, 0, 0, ".realp");3448      Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(3449          RI.ElementType, ElemPtr, 0, 1, ".imagp");3450      Builder.CreateStore(SrcReal, DestRealPtr);3451      Builder.CreateStore(SrcImg, DestImgPtr);3452      break;3453    }3454    case EvalKind::Aggregate: {3455      Value *SizeVal =3456          Builder.getInt64(M.getDataLayout().getTypeStoreSize(RI.ElementType));3457      Builder.CreateMemCpy(3458          ElemPtr, M.getDataLayout().getPrefTypeAlign(RI.ElementType),3459          GlobValPtr, M.getDataLayout().getPrefTypeAlign(RI.ElementType),3460          SizeVal, false);3461      break;3462    }3463    }3464  }3465 3466  Builder.CreateRetVoid();3467  Builder.restoreIP(OldIP);3468  return LtGCFunc;3469}3470 3471Function *OpenMPIRBuilder::emitGlobalToListReduceFunction(3472    ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,3473    Type *ReductionsBufferTy, AttributeList FuncAttrs) {3474  OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();3475  LLVMContext &Ctx = M.getContext();3476  auto *FuncTy = FunctionType::get(3477      Builder.getVoidTy(),3478      {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},3479      /* IsVarArg */ false);3480  Function *LtGRFunc =3481      Function::Create(FuncTy, GlobalVariable::InternalLinkage,3482                       "_omp_reduction_global_to_list_reduce_func", &M);3483  LtGRFunc->setAttributes(FuncAttrs);3484  LtGRFunc->addParamAttr(0, Attribute::NoUndef);3485  LtGRFunc->addParamAttr(1, Attribute::NoUndef);3486  LtGRFunc->addParamAttr(2, Attribute::NoUndef);3487 3488  BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "entry", LtGRFunc);3489  Builder.SetInsertPoint(EntryBlock);3490 3491  // Buffer: global reduction buffer.3492  Argument *BufferArg = LtGRFunc->getArg(0);3493  // Idx: index of the buffer.3494  Argument *IdxArg = LtGRFunc->getArg(1);3495  // ReduceList: thread local Reduce list.3496  Argument *ReduceListArg = LtGRFunc->getArg(2);3497 3498  Value *BufferArgAlloca = Builder.CreateAlloca(Builder.getPtrTy(), nullptr,3499                                                BufferArg->getName() + ".addr");3500  Value *IdxArgAlloca = Builder.CreateAlloca(Builder.getInt32Ty(), nullptr,3501                                             IdxArg->getName() + ".addr");3502  Value *ReduceListArgAlloca = Builder.CreateAlloca(3503      Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr");3504  ArrayType *RedListArrayTy =3505      ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());3506 3507  // 1. Build a list of reduction variables.3508  // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};3509  Value *LocalReduceList =3510      Builder.CreateAlloca(RedListArrayTy, nullptr, ".omp.reduction.red_list");3511 3512  Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(3513      BufferArgAlloca, Builder.getPtrTy(),3514      BufferArgAlloca->getName() + ".ascast");3515  Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(3516      IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast");3517  Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(3518      ReduceListArgAlloca, Builder.getPtrTy(),3519      ReduceListArgAlloca->getName() + ".ascast");3520  Value *ReductionList = Builder.CreatePointerBitCastOrAddrSpaceCast(3521      LocalReduceList, Builder.getPtrTy(),3522      LocalReduceList->getName() + ".ascast");3523 3524  Builder.CreateStore(BufferArg, BufferArgAddrCast);3525  Builder.CreateStore(IdxArg, IdxArgAddrCast);3526  Builder.CreateStore(ReduceListArg, ReduceListArgAddrCast);3527 3528  Value *BufferVal = Builder.CreateLoad(Builder.getPtrTy(), BufferArgAddrCast);3529  Value *Idxs[] = {Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast)};3530  Type *IndexTy = Builder.getIndexTy(3531      M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());3532  for (auto En : enumerate(ReductionInfos)) {3533    Value *TargetElementPtrPtr = Builder.CreateInBoundsGEP(3534        RedListArrayTy, ReductionList,3535        {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});3536    // Global = Buffer.VD[Idx];3537    Value *BufferVD =3538        Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferVal, Idxs);3539    Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(3540        ReductionsBufferTy, BufferVD, 0, En.index());3541    Builder.CreateStore(GlobValPtr, TargetElementPtrPtr);3542  }3543 3544  // Call reduce_function(ReduceList, GlobalReduceList)3545  Value *ReduceList =3546      Builder.CreateLoad(Builder.getPtrTy(), ReduceListArgAddrCast);3547  createRuntimeFunctionCall(ReduceFn, {ReduceList, ReductionList})3548      ->addFnAttr(Attribute::NoUnwind);3549  Builder.CreateRetVoid();3550  Builder.restoreIP(OldIP);3551  return LtGRFunc;3552}3553 3554std::string OpenMPIRBuilder::getReductionFuncName(StringRef Name) const {3555  std::string Suffix =3556      createPlatformSpecificName({"omp", "reduction", "reduction_func"});3557  return (Name + Suffix).str();3558}3559 3560Expected<Function *> OpenMPIRBuilder::createReductionFunction(3561    StringRef ReducerName, ArrayRef<ReductionInfo> ReductionInfos,3562    ArrayRef<bool> IsByRef, ReductionGenCBKind ReductionGenCBKind,3563    AttributeList FuncAttrs) {3564  auto *FuncTy = FunctionType::get(Builder.getVoidTy(),3565                                   {Builder.getPtrTy(), Builder.getPtrTy()},3566                                   /* IsVarArg */ false);3567  std::string Name = getReductionFuncName(ReducerName);3568  Function *ReductionFunc =3569      Function::Create(FuncTy, GlobalVariable::InternalLinkage, Name, &M);3570  ReductionFunc->setAttributes(FuncAttrs);3571  ReductionFunc->addParamAttr(0, Attribute::NoUndef);3572  ReductionFunc->addParamAttr(1, Attribute::NoUndef);3573  BasicBlock *EntryBB =3574      BasicBlock::Create(M.getContext(), "entry", ReductionFunc);3575  Builder.SetInsertPoint(EntryBB);3576 3577  // Need to alloca memory here and deal with the pointers before getting3578  // LHS/RHS pointers out3579  Value *LHSArrayPtr = nullptr;3580  Value *RHSArrayPtr = nullptr;3581  Argument *Arg0 = ReductionFunc->getArg(0);3582  Argument *Arg1 = ReductionFunc->getArg(1);3583  Type *Arg0Type = Arg0->getType();3584  Type *Arg1Type = Arg1->getType();3585 3586  Value *LHSAlloca =3587      Builder.CreateAlloca(Arg0Type, nullptr, Arg0->getName() + ".addr");3588  Value *RHSAlloca =3589      Builder.CreateAlloca(Arg1Type, nullptr, Arg1->getName() + ".addr");3590  Value *LHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(3591      LHSAlloca, Arg0Type, LHSAlloca->getName() + ".ascast");3592  Value *RHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(3593      RHSAlloca, Arg1Type, RHSAlloca->getName() + ".ascast");3594  Builder.CreateStore(Arg0, LHSAddrCast);3595  Builder.CreateStore(Arg1, RHSAddrCast);3596  LHSArrayPtr = Builder.CreateLoad(Arg0Type, LHSAddrCast);3597  RHSArrayPtr = Builder.CreateLoad(Arg1Type, RHSAddrCast);3598 3599  Type *RedArrayTy = ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());3600  Type *IndexTy = Builder.getIndexTy(3601      M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());3602  SmallVector<Value *> LHSPtrs, RHSPtrs;3603  for (auto En : enumerate(ReductionInfos)) {3604    const ReductionInfo &RI = En.value();3605    Value *RHSI8PtrPtr = Builder.CreateInBoundsGEP(3606        RedArrayTy, RHSArrayPtr,3607        {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});3608    Value *RHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), RHSI8PtrPtr);3609    Value *RHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(3610        RHSI8Ptr, RI.PrivateVariable->getType(),3611        RHSI8Ptr->getName() + ".ascast");3612 3613    Value *LHSI8PtrPtr = Builder.CreateInBoundsGEP(3614        RedArrayTy, LHSArrayPtr,3615        {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});3616    Value *LHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), LHSI8PtrPtr);3617    Value *LHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(3618        LHSI8Ptr, RI.Variable->getType(), LHSI8Ptr->getName() + ".ascast");3619 3620    if (ReductionGenCBKind == ReductionGenCBKind::Clang) {3621      LHSPtrs.emplace_back(LHSPtr);3622      RHSPtrs.emplace_back(RHSPtr);3623    } else {3624      Value *LHS = LHSPtr;3625      Value *RHS = RHSPtr;3626 3627      if (!IsByRef.empty() && !IsByRef[En.index()]) {3628        LHS = Builder.CreateLoad(RI.ElementType, LHSPtr);3629        RHS = Builder.CreateLoad(RI.ElementType, RHSPtr);3630      }3631 3632      Value *Reduced;3633      InsertPointOrErrorTy AfterIP =3634          RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced);3635      if (!AfterIP)3636        return AfterIP.takeError();3637      if (!Builder.GetInsertBlock())3638        return ReductionFunc;3639 3640      Builder.restoreIP(*AfterIP);3641 3642      if (!IsByRef.empty() && !IsByRef[En.index()])3643        Builder.CreateStore(Reduced, LHSPtr);3644    }3645  }3646 3647  if (ReductionGenCBKind == ReductionGenCBKind::Clang)3648    for (auto En : enumerate(ReductionInfos)) {3649      unsigned Index = En.index();3650      const ReductionInfo &RI = En.value();3651      Value *LHSFixupPtr, *RHSFixupPtr;3652      Builder.restoreIP(RI.ReductionGenClang(3653          Builder.saveIP(), Index, &LHSFixupPtr, &RHSFixupPtr, ReductionFunc));3654 3655      // Fix the CallBack code genereated to use the correct Values for the LHS3656      // and RHS3657      LHSFixupPtr->replaceUsesWithIf(3658          LHSPtrs[Index], [ReductionFunc](const Use &U) {3659            return cast<Instruction>(U.getUser())->getParent()->getParent() ==3660                   ReductionFunc;3661          });3662      RHSFixupPtr->replaceUsesWithIf(3663          RHSPtrs[Index], [ReductionFunc](const Use &U) {3664            return cast<Instruction>(U.getUser())->getParent()->getParent() ==3665                   ReductionFunc;3666          });3667    }3668 3669  Builder.CreateRetVoid();3670  return ReductionFunc;3671}3672 3673static void3674checkReductionInfos(ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,3675                    bool IsGPU) {3676  for (const OpenMPIRBuilder::ReductionInfo &RI : ReductionInfos) {3677    (void)RI;3678    assert(RI.Variable && "expected non-null variable");3679    assert(RI.PrivateVariable && "expected non-null private variable");3680    assert((RI.ReductionGen || RI.ReductionGenClang) &&3681           "expected non-null reduction generator callback");3682    if (!IsGPU) {3683      assert(3684          RI.Variable->getType() == RI.PrivateVariable->getType() &&3685          "expected variables and their private equivalents to have the same "3686          "type");3687    }3688    assert(RI.Variable->getType()->isPointerTy() &&3689           "expected variables to be pointers");3690  }3691}3692 3693OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createReductionsGPU(3694    const LocationDescription &Loc, InsertPointTy AllocaIP,3695    InsertPointTy CodeGenIP, ArrayRef<ReductionInfo> ReductionInfos,3696    ArrayRef<bool> IsByRef, bool IsNoWait, bool IsTeamsReduction,3697    ReductionGenCBKind ReductionGenCBKind, std::optional<omp::GV> GridValue,3698    unsigned ReductionBufNum, Value *SrcLocInfo) {3699  if (!updateToLocation(Loc))3700    return InsertPointTy();3701  Builder.restoreIP(CodeGenIP);3702  checkReductionInfos(ReductionInfos, /*IsGPU*/ true);3703  LLVMContext &Ctx = M.getContext();3704 3705  // Source location for the ident struct3706  if (!SrcLocInfo) {3707    uint32_t SrcLocStrSize;3708    Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);3709    SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);3710  }3711 3712  if (ReductionInfos.size() == 0)3713    return Builder.saveIP();3714 3715  BasicBlock *ContinuationBlock = nullptr;3716  if (ReductionGenCBKind != ReductionGenCBKind::Clang) {3717    // Copied code from createReductions3718    BasicBlock *InsertBlock = Loc.IP.getBlock();3719    ContinuationBlock =3720        InsertBlock->splitBasicBlock(Loc.IP.getPoint(), "reduce.finalize");3721    InsertBlock->getTerminator()->eraseFromParent();3722    Builder.SetInsertPoint(InsertBlock, InsertBlock->end());3723  }3724 3725  Function *CurFunc = Builder.GetInsertBlock()->getParent();3726  AttributeList FuncAttrs;3727  AttrBuilder AttrBldr(Ctx);3728  for (auto Attr : CurFunc->getAttributes().getFnAttrs())3729    AttrBldr.addAttribute(Attr);3730  AttrBldr.removeAttribute(Attribute::OptimizeNone);3731  FuncAttrs = FuncAttrs.addFnAttributes(Ctx, AttrBldr);3732 3733  CodeGenIP = Builder.saveIP();3734  Expected<Function *> ReductionResult = createReductionFunction(3735      Builder.GetInsertBlock()->getParent()->getName(), ReductionInfos, IsByRef,3736      ReductionGenCBKind, FuncAttrs);3737  if (!ReductionResult)3738    return ReductionResult.takeError();3739  Function *ReductionFunc = *ReductionResult;3740  Builder.restoreIP(CodeGenIP);3741 3742  // Set the grid value in the config needed for lowering later on3743  if (GridValue.has_value())3744    Config.setGridValue(GridValue.value());3745  else3746    Config.setGridValue(getGridValue(T, ReductionFunc));3747 3748  // Build res = __kmpc_reduce{_nowait}(<gtid>, <n>, sizeof(RedList),3749  // RedList, shuffle_reduce_func, interwarp_copy_func);3750  // or3751  // Build res = __kmpc_reduce_teams_nowait_simple(<loc>, <gtid>, <lck>);3752  Value *Res;3753 3754  // 1. Build a list of reduction variables.3755  // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};3756  auto Size = ReductionInfos.size();3757  Type *PtrTy = PointerType::get(Ctx, Config.getDefaultTargetAS());3758  Type *FuncPtrTy =3759      Builder.getPtrTy(M.getDataLayout().getProgramAddressSpace());3760  Type *RedArrayTy = ArrayType::get(PtrTy, Size);3761  CodeGenIP = Builder.saveIP();3762  Builder.restoreIP(AllocaIP);3763  Value *ReductionListAlloca =3764      Builder.CreateAlloca(RedArrayTy, nullptr, ".omp.reduction.red_list");3765  Value *ReductionList = Builder.CreatePointerBitCastOrAddrSpaceCast(3766      ReductionListAlloca, PtrTy, ReductionListAlloca->getName() + ".ascast");3767  Builder.restoreIP(CodeGenIP);3768  Type *IndexTy = Builder.getIndexTy(3769      M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());3770  for (auto En : enumerate(ReductionInfos)) {3771    const ReductionInfo &RI = En.value();3772    Value *ElemPtr = Builder.CreateInBoundsGEP(3773        RedArrayTy, ReductionList,3774        {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});3775 3776    Value *PrivateVar = RI.PrivateVariable;3777    bool IsByRefElem = !IsByRef.empty() && IsByRef[En.index()];3778    if (IsByRefElem)3779      PrivateVar = Builder.CreateLoad(RI.ElementType, PrivateVar);3780 3781    Value *CastElem =3782        Builder.CreatePointerBitCastOrAddrSpaceCast(PrivateVar, PtrTy);3783    Builder.CreateStore(CastElem, ElemPtr);3784  }3785  CodeGenIP = Builder.saveIP();3786  Expected<Function *> SarFunc = emitShuffleAndReduceFunction(3787      ReductionInfos, ReductionFunc, FuncAttrs, IsByRef);3788 3789  if (!SarFunc)3790    return SarFunc.takeError();3791 3792  Expected<Function *> CopyResult =3793      emitInterWarpCopyFunction(Loc, ReductionInfos, FuncAttrs, IsByRef);3794  if (!CopyResult)3795    return CopyResult.takeError();3796  Function *WcFunc = *CopyResult;3797  Builder.restoreIP(CodeGenIP);3798 3799  Value *RL = Builder.CreatePointerBitCastOrAddrSpaceCast(ReductionList, PtrTy);3800 3801  unsigned MaxDataSize = 0;3802  SmallVector<Type *> ReductionTypeArgs;3803  for (auto En : enumerate(ReductionInfos)) {3804    auto Size = M.getDataLayout().getTypeStoreSize(En.value().ElementType);3805    if (Size > MaxDataSize)3806      MaxDataSize = Size;3807    ReductionTypeArgs.emplace_back(En.value().ElementType);3808  }3809  Value *ReductionDataSize =3810      Builder.getInt64(MaxDataSize * ReductionInfos.size());3811  if (!IsTeamsReduction) {3812    Value *SarFuncCast =3813        Builder.CreatePointerBitCastOrAddrSpaceCast(*SarFunc, FuncPtrTy);3814    Value *WcFuncCast =3815        Builder.CreatePointerBitCastOrAddrSpaceCast(WcFunc, FuncPtrTy);3816    Value *Args[] = {SrcLocInfo, ReductionDataSize, RL, SarFuncCast,3817                     WcFuncCast};3818    Function *Pv2Ptr = getOrCreateRuntimeFunctionPtr(3819        RuntimeFunction::OMPRTL___kmpc_nvptx_parallel_reduce_nowait_v2);3820    Res = createRuntimeFunctionCall(Pv2Ptr, Args);3821  } else {3822    CodeGenIP = Builder.saveIP();3823    StructType *ReductionsBufferTy = StructType::create(3824        Ctx, ReductionTypeArgs, "struct._globalized_locals_ty");3825    Function *RedFixedBuferFn = getOrCreateRuntimeFunctionPtr(3826        RuntimeFunction::OMPRTL___kmpc_reduction_get_fixed_buffer);3827    Function *LtGCFunc = emitListToGlobalCopyFunction(3828        ReductionInfos, ReductionsBufferTy, FuncAttrs);3829    Function *LtGRFunc = emitListToGlobalReduceFunction(3830        ReductionInfos, ReductionFunc, ReductionsBufferTy, FuncAttrs);3831    Function *GtLCFunc = emitGlobalToListCopyFunction(3832        ReductionInfos, ReductionsBufferTy, FuncAttrs);3833    Function *GtLRFunc = emitGlobalToListReduceFunction(3834        ReductionInfos, ReductionFunc, ReductionsBufferTy, FuncAttrs);3835    Builder.restoreIP(CodeGenIP);3836 3837    Value *KernelTeamsReductionPtr = createRuntimeFunctionCall(3838        RedFixedBuferFn, {}, "_openmp_teams_reductions_buffer_$_$ptr");3839 3840    Value *Args3[] = {SrcLocInfo,3841                      KernelTeamsReductionPtr,3842                      Builder.getInt32(ReductionBufNum),3843                      ReductionDataSize,3844                      RL,3845                      *SarFunc,3846                      WcFunc,3847                      LtGCFunc,3848                      LtGRFunc,3849                      GtLCFunc,3850                      GtLRFunc};3851 3852    Function *TeamsReduceFn = getOrCreateRuntimeFunctionPtr(3853        RuntimeFunction::OMPRTL___kmpc_nvptx_teams_reduce_nowait_v2);3854    Res = createRuntimeFunctionCall(TeamsReduceFn, Args3);3855  }3856 3857  // 5. Build if (res == 1)3858  BasicBlock *ExitBB = BasicBlock::Create(Ctx, ".omp.reduction.done");3859  BasicBlock *ThenBB = BasicBlock::Create(Ctx, ".omp.reduction.then");3860  Value *Cond = Builder.CreateICmpEQ(Res, Builder.getInt32(1));3861  Builder.CreateCondBr(Cond, ThenBB, ExitBB);3862 3863  // 6. Build then branch: where we have reduced values in the master3864  //    thread in each team.3865  //    __kmpc_end_reduce{_nowait}(<gtid>);3866  //    break;3867  emitBlock(ThenBB, CurFunc);3868 3869  // Add emission of __kmpc_end_reduce{_nowait}(<gtid>);3870  for (auto En : enumerate(ReductionInfos)) {3871    const ReductionInfo &RI = En.value();3872    Type *ValueType = RI.ElementType;3873    Value *RedValue = RI.Variable;3874    Value *RHS =3875        Builder.CreatePointerBitCastOrAddrSpaceCast(RI.PrivateVariable, PtrTy);3876 3877    if (ReductionGenCBKind == ReductionGenCBKind::Clang) {3878      Value *LHSPtr, *RHSPtr;3879      Builder.restoreIP(RI.ReductionGenClang(Builder.saveIP(), En.index(),3880                                             &LHSPtr, &RHSPtr, CurFunc));3881 3882      // Fix the CallBack code genereated to use the correct Values for the LHS3883      // and RHS3884      LHSPtr->replaceUsesWithIf(RedValue, [ReductionFunc](const Use &U) {3885        return cast<Instruction>(U.getUser())->getParent()->getParent() ==3886               ReductionFunc;3887      });3888      RHSPtr->replaceUsesWithIf(RHS, [ReductionFunc](const Use &U) {3889        return cast<Instruction>(U.getUser())->getParent()->getParent() ==3890               ReductionFunc;3891      });3892    } else {3893      if (IsByRef.empty() || !IsByRef[En.index()]) {3894        RedValue = Builder.CreateLoad(ValueType, RI.Variable,3895                                      "red.value." + Twine(En.index()));3896      }3897      Value *PrivateRedValue = Builder.CreateLoad(3898          ValueType, RHS, "red.private.value" + Twine(En.index()));3899      Value *Reduced;3900      InsertPointOrErrorTy AfterIP =3901          RI.ReductionGen(Builder.saveIP(), RedValue, PrivateRedValue, Reduced);3902      if (!AfterIP)3903        return AfterIP.takeError();3904      Builder.restoreIP(*AfterIP);3905 3906      if (!IsByRef.empty() && !IsByRef[En.index()])3907        Builder.CreateStore(Reduced, RI.Variable);3908    }3909  }3910  emitBlock(ExitBB, CurFunc);3911  if (ContinuationBlock) {3912    Builder.CreateBr(ContinuationBlock);3913    Builder.SetInsertPoint(ContinuationBlock);3914  }3915  Config.setEmitLLVMUsed();3916 3917  return Builder.saveIP();3918}3919 3920static Function *getFreshReductionFunc(Module &M) {3921  Type *VoidTy = Type::getVoidTy(M.getContext());3922  Type *Int8PtrTy = PointerType::getUnqual(M.getContext());3923  auto *FuncTy =3924      FunctionType::get(VoidTy, {Int8PtrTy, Int8PtrTy}, /* IsVarArg */ false);3925  return Function::Create(FuncTy, GlobalVariable::InternalLinkage,3926                          ".omp.reduction.func", &M);3927}3928 3929static Error populateReductionFunction(3930    Function *ReductionFunc,3931    ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,3932    IRBuilder<> &Builder, ArrayRef<bool> IsByRef, bool IsGPU) {3933  Module *Module = ReductionFunc->getParent();3934  BasicBlock *ReductionFuncBlock =3935      BasicBlock::Create(Module->getContext(), "", ReductionFunc);3936  Builder.SetInsertPoint(ReductionFuncBlock);3937  Value *LHSArrayPtr = nullptr;3938  Value *RHSArrayPtr = nullptr;3939  if (IsGPU) {3940    // Need to alloca memory here and deal with the pointers before getting3941    // LHS/RHS pointers out3942    //3943    Argument *Arg0 = ReductionFunc->getArg(0);3944    Argument *Arg1 = ReductionFunc->getArg(1);3945    Type *Arg0Type = Arg0->getType();3946    Type *Arg1Type = Arg1->getType();3947 3948    Value *LHSAlloca =3949        Builder.CreateAlloca(Arg0Type, nullptr, Arg0->getName() + ".addr");3950    Value *RHSAlloca =3951        Builder.CreateAlloca(Arg1Type, nullptr, Arg1->getName() + ".addr");3952    Value *LHSAddrCast =3953        Builder.CreatePointerBitCastOrAddrSpaceCast(LHSAlloca, Arg0Type);3954    Value *RHSAddrCast =3955        Builder.CreatePointerBitCastOrAddrSpaceCast(RHSAlloca, Arg1Type);3956    Builder.CreateStore(Arg0, LHSAddrCast);3957    Builder.CreateStore(Arg1, RHSAddrCast);3958    LHSArrayPtr = Builder.CreateLoad(Arg0Type, LHSAddrCast);3959    RHSArrayPtr = Builder.CreateLoad(Arg1Type, RHSAddrCast);3960  } else {3961    LHSArrayPtr = ReductionFunc->getArg(0);3962    RHSArrayPtr = ReductionFunc->getArg(1);3963  }3964 3965  unsigned NumReductions = ReductionInfos.size();3966  Type *RedArrayTy = ArrayType::get(Builder.getPtrTy(), NumReductions);3967 3968  for (auto En : enumerate(ReductionInfos)) {3969    const OpenMPIRBuilder::ReductionInfo &RI = En.value();3970    Value *LHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(3971        RedArrayTy, LHSArrayPtr, 0, En.index());3972    Value *LHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), LHSI8PtrPtr);3973    Value *LHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(3974        LHSI8Ptr, RI.Variable->getType());3975    Value *LHS = Builder.CreateLoad(RI.ElementType, LHSPtr);3976    Value *RHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(3977        RedArrayTy, RHSArrayPtr, 0, En.index());3978    Value *RHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), RHSI8PtrPtr);3979    Value *RHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(3980        RHSI8Ptr, RI.PrivateVariable->getType());3981    Value *RHS = Builder.CreateLoad(RI.ElementType, RHSPtr);3982    Value *Reduced;3983    OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =3984        RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced);3985    if (!AfterIP)3986      return AfterIP.takeError();3987 3988    Builder.restoreIP(*AfterIP);3989    // TODO: Consider flagging an error.3990    if (!Builder.GetInsertBlock())3991      return Error::success();3992 3993    // store is inside of the reduction region when using by-ref3994    if (!IsByRef[En.index()])3995      Builder.CreateStore(Reduced, LHSPtr);3996  }3997  Builder.CreateRetVoid();3998  return Error::success();3999}4000 4001OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createReductions(4002    const LocationDescription &Loc, InsertPointTy AllocaIP,4003    ArrayRef<ReductionInfo> ReductionInfos, ArrayRef<bool> IsByRef,4004    bool IsNoWait, bool IsTeamsReduction) {4005  assert(ReductionInfos.size() == IsByRef.size());4006  if (Config.isGPU())4007    return createReductionsGPU(Loc, AllocaIP, Builder.saveIP(), ReductionInfos,4008                               IsByRef, IsNoWait, IsTeamsReduction);4009 4010  checkReductionInfos(ReductionInfos, /*IsGPU*/ false);4011 4012  if (!updateToLocation(Loc))4013    return InsertPointTy();4014 4015  if (ReductionInfos.size() == 0)4016    return Builder.saveIP();4017 4018  BasicBlock *InsertBlock = Loc.IP.getBlock();4019  BasicBlock *ContinuationBlock =4020      InsertBlock->splitBasicBlock(Loc.IP.getPoint(), "reduce.finalize");4021  InsertBlock->getTerminator()->eraseFromParent();4022 4023  // Create and populate array of type-erased pointers to private reduction4024  // values.4025  unsigned NumReductions = ReductionInfos.size();4026  Type *RedArrayTy = ArrayType::get(Builder.getPtrTy(), NumReductions);4027  Builder.SetInsertPoint(AllocaIP.getBlock()->getTerminator());4028  Value *RedArray = Builder.CreateAlloca(RedArrayTy, nullptr, "red.array");4029 4030  Builder.SetInsertPoint(InsertBlock, InsertBlock->end());4031 4032  for (auto En : enumerate(ReductionInfos)) {4033    unsigned Index = En.index();4034    const ReductionInfo &RI = En.value();4035    Value *RedArrayElemPtr = Builder.CreateConstInBoundsGEP2_64(4036        RedArrayTy, RedArray, 0, Index, "red.array.elem." + Twine(Index));4037    Builder.CreateStore(RI.PrivateVariable, RedArrayElemPtr);4038  }4039 4040  // Emit a call to the runtime function that orchestrates the reduction.4041  // Declare the reduction function in the process.4042  Type *IndexTy = Builder.getIndexTy(4043      M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());4044  Function *Func = Builder.GetInsertBlock()->getParent();4045  Module *Module = Func->getParent();4046  uint32_t SrcLocStrSize;4047  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);4048  bool CanGenerateAtomic = all_of(ReductionInfos, [](const ReductionInfo &RI) {4049    return RI.AtomicReductionGen;4050  });4051  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize,4052                                  CanGenerateAtomic4053                                      ? IdentFlag::OMP_IDENT_FLAG_ATOMIC_REDUCE4054                                      : IdentFlag(0));4055  Value *ThreadId = getOrCreateThreadID(Ident);4056  Constant *NumVariables = Builder.getInt32(NumReductions);4057  const DataLayout &DL = Module->getDataLayout();4058  unsigned RedArrayByteSize = DL.getTypeStoreSize(RedArrayTy);4059  Constant *RedArraySize = ConstantInt::get(IndexTy, RedArrayByteSize);4060  Function *ReductionFunc = getFreshReductionFunc(*Module);4061  Value *Lock = getOMPCriticalRegionLock(".reduction");4062  Function *ReduceFunc = getOrCreateRuntimeFunctionPtr(4063      IsNoWait ? RuntimeFunction::OMPRTL___kmpc_reduce_nowait4064               : RuntimeFunction::OMPRTL___kmpc_reduce);4065  CallInst *ReduceCall =4066      createRuntimeFunctionCall(ReduceFunc,4067                                {Ident, ThreadId, NumVariables, RedArraySize,4068                                 RedArray, ReductionFunc, Lock},4069                                "reduce");4070 4071  // Create final reduction entry blocks for the atomic and non-atomic case.4072  // Emit IR that dispatches control flow to one of the blocks based on the4073  // reduction supporting the atomic mode.4074  BasicBlock *NonAtomicRedBlock =4075      BasicBlock::Create(Module->getContext(), "reduce.switch.nonatomic", Func);4076  BasicBlock *AtomicRedBlock =4077      BasicBlock::Create(Module->getContext(), "reduce.switch.atomic", Func);4078  SwitchInst *Switch =4079      Builder.CreateSwitch(ReduceCall, ContinuationBlock, /* NumCases */ 2);4080  Switch->addCase(Builder.getInt32(1), NonAtomicRedBlock);4081  Switch->addCase(Builder.getInt32(2), AtomicRedBlock);4082 4083  // Populate the non-atomic reduction using the elementwise reduction function.4084  // This loads the elements from the global and private variables and reduces4085  // them before storing back the result to the global variable.4086  Builder.SetInsertPoint(NonAtomicRedBlock);4087  for (auto En : enumerate(ReductionInfos)) {4088    const ReductionInfo &RI = En.value();4089    Type *ValueType = RI.ElementType;4090    // We have one less load for by-ref case because that load is now inside of4091    // the reduction region4092    Value *RedValue = RI.Variable;4093    if (!IsByRef[En.index()]) {4094      RedValue = Builder.CreateLoad(ValueType, RI.Variable,4095                                    "red.value." + Twine(En.index()));4096    }4097    Value *PrivateRedValue =4098        Builder.CreateLoad(ValueType, RI.PrivateVariable,4099                           "red.private.value." + Twine(En.index()));4100    Value *Reduced;4101    InsertPointOrErrorTy AfterIP =4102        RI.ReductionGen(Builder.saveIP(), RedValue, PrivateRedValue, Reduced);4103    if (!AfterIP)4104      return AfterIP.takeError();4105    Builder.restoreIP(*AfterIP);4106 4107    if (!Builder.GetInsertBlock())4108      return InsertPointTy();4109    // for by-ref case, the load is inside of the reduction region4110    if (!IsByRef[En.index()])4111      Builder.CreateStore(Reduced, RI.Variable);4112  }4113  Function *EndReduceFunc = getOrCreateRuntimeFunctionPtr(4114      IsNoWait ? RuntimeFunction::OMPRTL___kmpc_end_reduce_nowait4115               : RuntimeFunction::OMPRTL___kmpc_end_reduce);4116  createRuntimeFunctionCall(EndReduceFunc, {Ident, ThreadId, Lock});4117  Builder.CreateBr(ContinuationBlock);4118 4119  // Populate the atomic reduction using the atomic elementwise reduction4120  // function. There are no loads/stores here because they will be happening4121  // inside the atomic elementwise reduction.4122  Builder.SetInsertPoint(AtomicRedBlock);4123  if (CanGenerateAtomic && llvm::none_of(IsByRef, [](bool P) { return P; })) {4124    for (const ReductionInfo &RI : ReductionInfos) {4125      InsertPointOrErrorTy AfterIP = RI.AtomicReductionGen(4126          Builder.saveIP(), RI.ElementType, RI.Variable, RI.PrivateVariable);4127      if (!AfterIP)4128        return AfterIP.takeError();4129      Builder.restoreIP(*AfterIP);4130      if (!Builder.GetInsertBlock())4131        return InsertPointTy();4132    }4133    Builder.CreateBr(ContinuationBlock);4134  } else {4135    Builder.CreateUnreachable();4136  }4137 4138  // Populate the outlined reduction function using the elementwise reduction4139  // function. Partial values are extracted from the type-erased array of4140  // pointers to private variables.4141  Error Err = populateReductionFunction(ReductionFunc, ReductionInfos, Builder,4142                                        IsByRef, /*isGPU=*/false);4143  if (Err)4144    return Err;4145 4146  if (!Builder.GetInsertBlock())4147    return InsertPointTy();4148 4149  Builder.SetInsertPoint(ContinuationBlock);4150  return Builder.saveIP();4151}4152 4153OpenMPIRBuilder::InsertPointOrErrorTy4154OpenMPIRBuilder::createMaster(const LocationDescription &Loc,4155                              BodyGenCallbackTy BodyGenCB,4156                              FinalizeCallbackTy FiniCB) {4157  if (!updateToLocation(Loc))4158    return Loc.IP;4159 4160  Directive OMPD = Directive::OMPD_master;4161  uint32_t SrcLocStrSize;4162  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);4163  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);4164  Value *ThreadId = getOrCreateThreadID(Ident);4165  Value *Args[] = {Ident, ThreadId};4166 4167  Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_master);4168  Instruction *EntryCall = createRuntimeFunctionCall(EntryRTLFn, Args);4169 4170  Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_master);4171  Instruction *ExitCall = createRuntimeFunctionCall(ExitRTLFn, Args);4172 4173  return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,4174                              /*Conditional*/ true, /*hasFinalize*/ true);4175}4176 4177OpenMPIRBuilder::InsertPointOrErrorTy4178OpenMPIRBuilder::createMasked(const LocationDescription &Loc,4179                              BodyGenCallbackTy BodyGenCB,4180                              FinalizeCallbackTy FiniCB, Value *Filter) {4181  if (!updateToLocation(Loc))4182    return Loc.IP;4183 4184  Directive OMPD = Directive::OMPD_masked;4185  uint32_t SrcLocStrSize;4186  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);4187  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);4188  Value *ThreadId = getOrCreateThreadID(Ident);4189  Value *Args[] = {Ident, ThreadId, Filter};4190  Value *ArgsEnd[] = {Ident, ThreadId};4191 4192  Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_masked);4193  Instruction *EntryCall = createRuntimeFunctionCall(EntryRTLFn, Args);4194 4195  Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_masked);4196  Instruction *ExitCall = createRuntimeFunctionCall(ExitRTLFn, ArgsEnd);4197 4198  return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,4199                              /*Conditional*/ true, /*hasFinalize*/ true);4200}4201 4202static llvm::CallInst *emitNoUnwindRuntimeCall(IRBuilder<> &Builder,4203                                               llvm::FunctionCallee Callee,4204                                               ArrayRef<llvm::Value *> Args,4205                                               const llvm::Twine &Name) {4206  llvm::CallInst *Call = Builder.CreateCall(4207      Callee, Args, SmallVector<llvm::OperandBundleDef, 1>(), Name);4208  Call->setDoesNotThrow();4209  return Call;4210}4211 4212// Expects input basic block is dominated by BeforeScanBB.4213// Once Scan directive is encountered, the code after scan directive should be4214// dominated by AfterScanBB. Scan directive splits the code sequence to4215// scan and input phase. Based on whether inclusive or exclusive4216// clause is used in the scan directive and whether input loop or scan loop4217// is lowered, it adds jumps to input and scan phase. First Scan loop is the4218// input loop and second is the scan loop. The code generated handles only4219// inclusive scans now.4220OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createScan(4221    const LocationDescription &Loc, InsertPointTy AllocaIP,4222    ArrayRef<llvm::Value *> ScanVars, ArrayRef<llvm::Type *> ScanVarsType,4223    bool IsInclusive, ScanInfo *ScanRedInfo) {4224  if (ScanRedInfo->OMPFirstScanLoop) {4225    llvm::Error Err = emitScanBasedDirectiveDeclsIR(AllocaIP, ScanVars,4226                                                    ScanVarsType, ScanRedInfo);4227    if (Err)4228      return Err;4229  }4230  if (!updateToLocation(Loc))4231    return Loc.IP;4232 4233  llvm::Value *IV = ScanRedInfo->IV;4234 4235  if (ScanRedInfo->OMPFirstScanLoop) {4236    // Emit buffer[i] = red; at the end of the input phase.4237    for (size_t i = 0; i < ScanVars.size(); i++) {4238      Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]];4239      Value *Buff = Builder.CreateLoad(Builder.getPtrTy(), BuffPtr);4240      Type *DestTy = ScanVarsType[i];4241      Value *Val = Builder.CreateInBoundsGEP(DestTy, Buff, IV, "arrayOffset");4242      Value *Src = Builder.CreateLoad(DestTy, ScanVars[i]);4243 4244      Builder.CreateStore(Src, Val);4245    }4246  }4247  Builder.CreateBr(ScanRedInfo->OMPScanLoopExit);4248  emitBlock(ScanRedInfo->OMPScanDispatch,4249            Builder.GetInsertBlock()->getParent());4250 4251  if (!ScanRedInfo->OMPFirstScanLoop) {4252    IV = ScanRedInfo->IV;4253    // Emit red = buffer[i]; at the entrance to the scan phase.4254    // TODO: if exclusive scan, the red = buffer[i-1] needs to be updated.4255    for (size_t i = 0; i < ScanVars.size(); i++) {4256      Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]];4257      Value *Buff = Builder.CreateLoad(Builder.getPtrTy(), BuffPtr);4258      Type *DestTy = ScanVarsType[i];4259      Value *SrcPtr =4260          Builder.CreateInBoundsGEP(DestTy, Buff, IV, "arrayOffset");4261      Value *Src = Builder.CreateLoad(DestTy, SrcPtr);4262      Builder.CreateStore(Src, ScanVars[i]);4263    }4264  }4265 4266  // TODO: Update it to CreateBr and remove dead blocks4267  llvm::Value *CmpI = Builder.getInt1(true);4268  if (ScanRedInfo->OMPFirstScanLoop == IsInclusive) {4269    Builder.CreateCondBr(CmpI, ScanRedInfo->OMPBeforeScanBlock,4270                         ScanRedInfo->OMPAfterScanBlock);4271  } else {4272    Builder.CreateCondBr(CmpI, ScanRedInfo->OMPAfterScanBlock,4273                         ScanRedInfo->OMPBeforeScanBlock);4274  }4275  emitBlock(ScanRedInfo->OMPAfterScanBlock,4276            Builder.GetInsertBlock()->getParent());4277  Builder.SetInsertPoint(ScanRedInfo->OMPAfterScanBlock);4278  return Builder.saveIP();4279}4280 4281Error OpenMPIRBuilder::emitScanBasedDirectiveDeclsIR(4282    InsertPointTy AllocaIP, ArrayRef<Value *> ScanVars,4283    ArrayRef<Type *> ScanVarsType, ScanInfo *ScanRedInfo) {4284 4285  Builder.restoreIP(AllocaIP);4286  // Create the shared pointer at alloca IP.4287  for (size_t i = 0; i < ScanVars.size(); i++) {4288    llvm::Value *BuffPtr =4289        Builder.CreateAlloca(Builder.getPtrTy(), nullptr, "vla");4290    (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]] = BuffPtr;4291  }4292 4293  // Allocate temporary buffer by master thread4294  auto BodyGenCB = [&](InsertPointTy AllocaIP,4295                       InsertPointTy CodeGenIP) -> Error {4296    Builder.restoreIP(CodeGenIP);4297    Value *AllocSpan =4298        Builder.CreateAdd(ScanRedInfo->Span, Builder.getInt32(1));4299    for (size_t i = 0; i < ScanVars.size(); i++) {4300      Type *IntPtrTy = Builder.getInt32Ty();4301      Constant *Allocsize = ConstantExpr::getSizeOf(ScanVarsType[i]);4302      Allocsize = ConstantExpr::getTruncOrBitCast(Allocsize, IntPtrTy);4303      Value *Buff = Builder.CreateMalloc(IntPtrTy, ScanVarsType[i], Allocsize,4304                                         AllocSpan, nullptr, "arr");4305      Builder.CreateStore(Buff, (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]]);4306    }4307    return Error::success();4308  };4309  // TODO: Perform finalization actions for variables. This has to be4310  // called for variables which have destructors/finalizers.4311  auto FiniCB = [&](InsertPointTy CodeGenIP) { return llvm::Error::success(); };4312 4313  Builder.SetInsertPoint(ScanRedInfo->OMPScanInit->getTerminator());4314  llvm::Value *FilterVal = Builder.getInt32(0);4315  llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =4316      createMasked(Builder.saveIP(), BodyGenCB, FiniCB, FilterVal);4317 4318  if (!AfterIP)4319    return AfterIP.takeError();4320  Builder.restoreIP(*AfterIP);4321  BasicBlock *InputBB = Builder.GetInsertBlock();4322  if (InputBB->getTerminator())4323    Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());4324  AfterIP = createBarrier(Builder.saveIP(), llvm::omp::OMPD_barrier);4325  if (!AfterIP)4326    return AfterIP.takeError();4327  Builder.restoreIP(*AfterIP);4328 4329  return Error::success();4330}4331 4332Error OpenMPIRBuilder::emitScanBasedDirectiveFinalsIR(4333    ArrayRef<ReductionInfo> ReductionInfos, ScanInfo *ScanRedInfo) {4334  auto BodyGenCB = [&](InsertPointTy AllocaIP,4335                       InsertPointTy CodeGenIP) -> Error {4336    Builder.restoreIP(CodeGenIP);4337    for (ReductionInfo RedInfo : ReductionInfos) {4338      Value *PrivateVar = RedInfo.PrivateVariable;4339      Value *OrigVar = RedInfo.Variable;4340      Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[PrivateVar];4341      Value *Buff = Builder.CreateLoad(Builder.getPtrTy(), BuffPtr);4342 4343      Type *SrcTy = RedInfo.ElementType;4344      Value *Val = Builder.CreateInBoundsGEP(SrcTy, Buff, ScanRedInfo->Span,4345                                             "arrayOffset");4346      Value *Src = Builder.CreateLoad(SrcTy, Val);4347 4348      Builder.CreateStore(Src, OrigVar);4349      Builder.CreateFree(Buff);4350    }4351    return Error::success();4352  };4353  // TODO: Perform finalization actions for variables. This has to be4354  // called for variables which have destructors/finalizers.4355  auto FiniCB = [&](InsertPointTy CodeGenIP) { return llvm::Error::success(); };4356 4357  if (ScanRedInfo->OMPScanFinish->getTerminator())4358    Builder.SetInsertPoint(ScanRedInfo->OMPScanFinish->getTerminator());4359  else4360    Builder.SetInsertPoint(ScanRedInfo->OMPScanFinish);4361 4362  llvm::Value *FilterVal = Builder.getInt32(0);4363  llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =4364      createMasked(Builder.saveIP(), BodyGenCB, FiniCB, FilterVal);4365 4366  if (!AfterIP)4367    return AfterIP.takeError();4368  Builder.restoreIP(*AfterIP);4369  BasicBlock *InputBB = Builder.GetInsertBlock();4370  if (InputBB->getTerminator())4371    Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());4372  AfterIP = createBarrier(Builder.saveIP(), llvm::omp::OMPD_barrier);4373  if (!AfterIP)4374    return AfterIP.takeError();4375  Builder.restoreIP(*AfterIP);4376  return Error::success();4377}4378 4379OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(4380    const LocationDescription &Loc,4381    ArrayRef<llvm::OpenMPIRBuilder::ReductionInfo> ReductionInfos,4382    ScanInfo *ScanRedInfo) {4383 4384  if (!updateToLocation(Loc))4385    return Loc.IP;4386  auto BodyGenCB = [&](InsertPointTy AllocaIP,4387                       InsertPointTy CodeGenIP) -> Error {4388    Builder.restoreIP(CodeGenIP);4389    Function *CurFn = Builder.GetInsertBlock()->getParent();4390    // for (int k = 0; k <= ceil(log2(n)); ++k)4391    llvm::BasicBlock *LoopBB =4392        BasicBlock::Create(CurFn->getContext(), "omp.outer.log.scan.body");4393    llvm::BasicBlock *ExitBB =4394        splitBB(Builder, false, "omp.outer.log.scan.exit");4395    llvm::Function *F = llvm::Intrinsic::getOrInsertDeclaration(4396        Builder.GetInsertBlock()->getModule(),4397        (llvm::Intrinsic::ID)llvm::Intrinsic::log2, Builder.getDoubleTy());4398    llvm::BasicBlock *InputBB = Builder.GetInsertBlock();4399    llvm::Value *Arg =4400        Builder.CreateUIToFP(ScanRedInfo->Span, Builder.getDoubleTy());4401    llvm::Value *LogVal = emitNoUnwindRuntimeCall(Builder, F, Arg, "");4402    F = llvm::Intrinsic::getOrInsertDeclaration(4403        Builder.GetInsertBlock()->getModule(),4404        (llvm::Intrinsic::ID)llvm::Intrinsic::ceil, Builder.getDoubleTy());4405    LogVal = emitNoUnwindRuntimeCall(Builder, F, LogVal, "");4406    LogVal = Builder.CreateFPToUI(LogVal, Builder.getInt32Ty());4407    llvm::Value *NMin1 = Builder.CreateNUWSub(4408        ScanRedInfo->Span,4409        llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 1));4410    Builder.SetInsertPoint(InputBB);4411    Builder.CreateBr(LoopBB);4412    emitBlock(LoopBB, CurFn);4413    Builder.SetInsertPoint(LoopBB);4414 4415    PHINode *Counter = Builder.CreatePHI(Builder.getInt32Ty(), 2);4416    // size pow2k = 1;4417    PHINode *Pow2K = Builder.CreatePHI(Builder.getInt32Ty(), 2);4418    Counter->addIncoming(llvm::ConstantInt::get(Builder.getInt32Ty(), 0),4419                         InputBB);4420    Pow2K->addIncoming(llvm::ConstantInt::get(Builder.getInt32Ty(), 1),4421                       InputBB);4422    // for (size i = n - 1; i >= 2 ^ k; --i)4423    //   tmp[i] op= tmp[i-pow2k];4424    llvm::BasicBlock *InnerLoopBB =4425        BasicBlock::Create(CurFn->getContext(), "omp.inner.log.scan.body");4426    llvm::BasicBlock *InnerExitBB =4427        BasicBlock::Create(CurFn->getContext(), "omp.inner.log.scan.exit");4428    llvm::Value *CmpI = Builder.CreateICmpUGE(NMin1, Pow2K);4429    Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);4430    emitBlock(InnerLoopBB, CurFn);4431    Builder.SetInsertPoint(InnerLoopBB);4432    PHINode *IVal = Builder.CreatePHI(Builder.getInt32Ty(), 2);4433    IVal->addIncoming(NMin1, LoopBB);4434    for (ReductionInfo RedInfo : ReductionInfos) {4435      Value *ReductionVal = RedInfo.PrivateVariable;4436      Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ReductionVal];4437      Value *Buff = Builder.CreateLoad(Builder.getPtrTy(), BuffPtr);4438      Type *DestTy = RedInfo.ElementType;4439      Value *IV = Builder.CreateAdd(IVal, Builder.getInt32(1));4440      Value *LHSPtr =4441          Builder.CreateInBoundsGEP(DestTy, Buff, IV, "arrayOffset");4442      Value *OffsetIval = Builder.CreateNUWSub(IV, Pow2K);4443      Value *RHSPtr =4444          Builder.CreateInBoundsGEP(DestTy, Buff, OffsetIval, "arrayOffset");4445      Value *LHS = Builder.CreateLoad(DestTy, LHSPtr);4446      Value *RHS = Builder.CreateLoad(DestTy, RHSPtr);4447      llvm::Value *Result;4448      InsertPointOrErrorTy AfterIP =4449          RedInfo.ReductionGen(Builder.saveIP(), LHS, RHS, Result);4450      if (!AfterIP)4451        return AfterIP.takeError();4452      Builder.CreateStore(Result, LHSPtr);4453    }4454    llvm::Value *NextIVal = Builder.CreateNUWSub(4455        IVal, llvm::ConstantInt::get(Builder.getInt32Ty(), 1));4456    IVal->addIncoming(NextIVal, Builder.GetInsertBlock());4457    CmpI = Builder.CreateICmpUGE(NextIVal, Pow2K);4458    Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);4459    emitBlock(InnerExitBB, CurFn);4460    llvm::Value *Next = Builder.CreateNUWAdd(4461        Counter, llvm::ConstantInt::get(Counter->getType(), 1));4462    Counter->addIncoming(Next, Builder.GetInsertBlock());4463    // pow2k <<= 1;4464    llvm::Value *NextPow2K = Builder.CreateShl(Pow2K, 1, "", /*HasNUW=*/true);4465    Pow2K->addIncoming(NextPow2K, Builder.GetInsertBlock());4466    llvm::Value *Cmp = Builder.CreateICmpNE(Next, LogVal);4467    Builder.CreateCondBr(Cmp, LoopBB, ExitBB);4468    Builder.SetInsertPoint(ExitBB->getFirstInsertionPt());4469    return Error::success();4470  };4471 4472  // TODO: Perform finalization actions for variables. This has to be4473  // called for variables which have destructors/finalizers.4474  auto FiniCB = [&](InsertPointTy CodeGenIP) { return llvm::Error::success(); };4475 4476  llvm::Value *FilterVal = Builder.getInt32(0);4477  llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =4478      createMasked(Builder.saveIP(), BodyGenCB, FiniCB, FilterVal);4479 4480  if (!AfterIP)4481    return AfterIP.takeError();4482  Builder.restoreIP(*AfterIP);4483  AfterIP = createBarrier(Builder.saveIP(), llvm::omp::OMPD_barrier);4484 4485  if (!AfterIP)4486    return AfterIP.takeError();4487  Builder.restoreIP(*AfterIP);4488  Error Err = emitScanBasedDirectiveFinalsIR(ReductionInfos, ScanRedInfo);4489  if (Err)4490    return Err;4491 4492  return AfterIP;4493}4494 4495Error OpenMPIRBuilder::emitScanBasedDirectiveIR(4496    llvm::function_ref<Error()> InputLoopGen,4497    llvm::function_ref<Error(LocationDescription Loc)> ScanLoopGen,4498    ScanInfo *ScanRedInfo) {4499 4500  {4501    // Emit loop with input phase:4502    // for (i: 0..<num_iters>) {4503    //   <input phase>;4504    //   buffer[i] = red;4505    // }4506    ScanRedInfo->OMPFirstScanLoop = true;4507    Error Err = InputLoopGen();4508    if (Err)4509      return Err;4510  }4511  {4512    // Emit loop with scan phase:4513    // for (i: 0..<num_iters>) {4514    //   red = buffer[i];4515    //   <scan phase>;4516    // }4517    ScanRedInfo->OMPFirstScanLoop = false;4518    Error Err = ScanLoopGen(Builder.saveIP());4519    if (Err)4520      return Err;4521  }4522  return Error::success();4523}4524 4525void OpenMPIRBuilder::createScanBBs(ScanInfo *ScanRedInfo) {4526  Function *Fun = Builder.GetInsertBlock()->getParent();4527  ScanRedInfo->OMPScanDispatch =4528      BasicBlock::Create(Fun->getContext(), "omp.inscan.dispatch");4529  ScanRedInfo->OMPAfterScanBlock =4530      BasicBlock::Create(Fun->getContext(), "omp.after.scan.bb");4531  ScanRedInfo->OMPBeforeScanBlock =4532      BasicBlock::Create(Fun->getContext(), "omp.before.scan.bb");4533  ScanRedInfo->OMPScanLoopExit =4534      BasicBlock::Create(Fun->getContext(), "omp.scan.loop.exit");4535}4536CanonicalLoopInfo *OpenMPIRBuilder::createLoopSkeleton(4537    DebugLoc DL, Value *TripCount, Function *F, BasicBlock *PreInsertBefore,4538    BasicBlock *PostInsertBefore, const Twine &Name) {4539  Module *M = F->getParent();4540  LLVMContext &Ctx = M->getContext();4541  Type *IndVarTy = TripCount->getType();4542 4543  // Create the basic block structure.4544  BasicBlock *Preheader =4545      BasicBlock::Create(Ctx, "omp_" + Name + ".preheader", F, PreInsertBefore);4546  BasicBlock *Header =4547      BasicBlock::Create(Ctx, "omp_" + Name + ".header", F, PreInsertBefore);4548  BasicBlock *Cond =4549      BasicBlock::Create(Ctx, "omp_" + Name + ".cond", F, PreInsertBefore);4550  BasicBlock *Body =4551      BasicBlock::Create(Ctx, "omp_" + Name + ".body", F, PreInsertBefore);4552  BasicBlock *Latch =4553      BasicBlock::Create(Ctx, "omp_" + Name + ".inc", F, PostInsertBefore);4554  BasicBlock *Exit =4555      BasicBlock::Create(Ctx, "omp_" + Name + ".exit", F, PostInsertBefore);4556  BasicBlock *After =4557      BasicBlock::Create(Ctx, "omp_" + Name + ".after", F, PostInsertBefore);4558 4559  // Use specified DebugLoc for new instructions.4560  Builder.SetCurrentDebugLocation(DL);4561 4562  Builder.SetInsertPoint(Preheader);4563  Builder.CreateBr(Header);4564 4565  Builder.SetInsertPoint(Header);4566  PHINode *IndVarPHI = Builder.CreatePHI(IndVarTy, 2, "omp_" + Name + ".iv");4567  IndVarPHI->addIncoming(ConstantInt::get(IndVarTy, 0), Preheader);4568  Builder.CreateBr(Cond);4569 4570  Builder.SetInsertPoint(Cond);4571  Value *Cmp =4572      Builder.CreateICmpULT(IndVarPHI, TripCount, "omp_" + Name + ".cmp");4573  Builder.CreateCondBr(Cmp, Body, Exit);4574 4575  Builder.SetInsertPoint(Body);4576  Builder.CreateBr(Latch);4577 4578  Builder.SetInsertPoint(Latch);4579  Value *Next = Builder.CreateAdd(IndVarPHI, ConstantInt::get(IndVarTy, 1),4580                                  "omp_" + Name + ".next", /*HasNUW=*/true);4581  Builder.CreateBr(Header);4582  IndVarPHI->addIncoming(Next, Latch);4583 4584  Builder.SetInsertPoint(Exit);4585  Builder.CreateBr(After);4586 4587  // Remember and return the canonical control flow.4588  LoopInfos.emplace_front();4589  CanonicalLoopInfo *CL = &LoopInfos.front();4590 4591  CL->Header = Header;4592  CL->Cond = Cond;4593  CL->Latch = Latch;4594  CL->Exit = Exit;4595 4596#ifndef NDEBUG4597  CL->assertOK();4598#endif4599  return CL;4600}4601 4602Expected<CanonicalLoopInfo *>4603OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc,4604                                     LoopBodyGenCallbackTy BodyGenCB,4605                                     Value *TripCount, const Twine &Name) {4606  BasicBlock *BB = Loc.IP.getBlock();4607  BasicBlock *NextBB = BB->getNextNode();4608 4609  CanonicalLoopInfo *CL = createLoopSkeleton(Loc.DL, TripCount, BB->getParent(),4610                                             NextBB, NextBB, Name);4611  BasicBlock *After = CL->getAfter();4612 4613  // If location is not set, don't connect the loop.4614  if (updateToLocation(Loc)) {4615    // Split the loop at the insertion point: Branch to the preheader and move4616    // every following instruction to after the loop (the After BB). Also, the4617    // new successor is the loop's after block.4618    spliceBB(Builder, After, /*CreateBranch=*/false);4619    Builder.CreateBr(CL->getPreheader());4620  }4621 4622  // Emit the body content. We do it after connecting the loop to the CFG to4623  // avoid that the callback encounters degenerate BBs.4624  if (Error Err = BodyGenCB(CL->getBodyIP(), CL->getIndVar()))4625    return Err;4626 4627#ifndef NDEBUG4628  CL->assertOK();4629#endif4630  return CL;4631}4632 4633Expected<ScanInfo *> OpenMPIRBuilder::scanInfoInitialize() {4634  ScanInfos.emplace_front();4635  ScanInfo *Result = &ScanInfos.front();4636  return Result;4637}4638 4639Expected<SmallVector<llvm::CanonicalLoopInfo *>>4640OpenMPIRBuilder::createCanonicalScanLoops(4641    const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,4642    Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,4643    InsertPointTy ComputeIP, const Twine &Name, ScanInfo *ScanRedInfo) {4644  LocationDescription ComputeLoc =4645      ComputeIP.isSet() ? LocationDescription(ComputeIP, Loc.DL) : Loc;4646  updateToLocation(ComputeLoc);4647 4648  SmallVector<CanonicalLoopInfo *> Result;4649 4650  Value *TripCount = calculateCanonicalLoopTripCount(4651      ComputeLoc, Start, Stop, Step, IsSigned, InclusiveStop, Name);4652  ScanRedInfo->Span = TripCount;4653  ScanRedInfo->OMPScanInit = splitBB(Builder, true, "scan.init");4654  Builder.SetInsertPoint(ScanRedInfo->OMPScanInit);4655 4656  auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {4657    Builder.restoreIP(CodeGenIP);4658    ScanRedInfo->IV = IV;4659    createScanBBs(ScanRedInfo);4660    BasicBlock *InputBlock = Builder.GetInsertBlock();4661    Instruction *Terminator = InputBlock->getTerminator();4662    assert(Terminator->getNumSuccessors() == 1);4663    BasicBlock *ContinueBlock = Terminator->getSuccessor(0);4664    Terminator->setSuccessor(0, ScanRedInfo->OMPScanDispatch);4665    emitBlock(ScanRedInfo->OMPBeforeScanBlock,4666              Builder.GetInsertBlock()->getParent());4667    Builder.CreateBr(ScanRedInfo->OMPScanLoopExit);4668    emitBlock(ScanRedInfo->OMPScanLoopExit,4669              Builder.GetInsertBlock()->getParent());4670    Builder.CreateBr(ContinueBlock);4671    Builder.SetInsertPoint(4672        ScanRedInfo->OMPBeforeScanBlock->getFirstInsertionPt());4673    return BodyGenCB(Builder.saveIP(), IV);4674  };4675 4676  const auto &&InputLoopGen = [&]() -> Error {4677    Expected<CanonicalLoopInfo *> LoopInfo = createCanonicalLoop(4678        Builder.saveIP(), BodyGen, Start, Stop, Step, IsSigned, InclusiveStop,4679        ComputeIP, Name, true, ScanRedInfo);4680    if (!LoopInfo)4681      return LoopInfo.takeError();4682    Result.push_back(*LoopInfo);4683    Builder.restoreIP((*LoopInfo)->getAfterIP());4684    return Error::success();4685  };4686  const auto &&ScanLoopGen = [&](LocationDescription Loc) -> Error {4687    Expected<CanonicalLoopInfo *> LoopInfo =4688        createCanonicalLoop(Loc, BodyGen, Start, Stop, Step, IsSigned,4689                            InclusiveStop, ComputeIP, Name, true, ScanRedInfo);4690    if (!LoopInfo)4691      return LoopInfo.takeError();4692    Result.push_back(*LoopInfo);4693    Builder.restoreIP((*LoopInfo)->getAfterIP());4694    ScanRedInfo->OMPScanFinish = Builder.GetInsertBlock();4695    return Error::success();4696  };4697  Error Err = emitScanBasedDirectiveIR(InputLoopGen, ScanLoopGen, ScanRedInfo);4698  if (Err)4699    return Err;4700  return Result;4701}4702 4703Value *OpenMPIRBuilder::calculateCanonicalLoopTripCount(4704    const LocationDescription &Loc, Value *Start, Value *Stop, Value *Step,4705    bool IsSigned, bool InclusiveStop, const Twine &Name) {4706 4707  // Consider the following difficulties (assuming 8-bit signed integers):4708  //  * Adding \p Step to the loop counter which passes \p Stop may overflow:4709  //      DO I = 1, 100, 504710  ///  * A \p Step of INT_MIN cannot not be normalized to a positive direction:4711  //      DO I = 100, 0, -1284712 4713  // Start, Stop and Step must be of the same integer type.4714  auto *IndVarTy = cast<IntegerType>(Start->getType());4715  assert(IndVarTy == Stop->getType() && "Stop type mismatch");4716  assert(IndVarTy == Step->getType() && "Step type mismatch");4717 4718  updateToLocation(Loc);4719 4720  ConstantInt *Zero = ConstantInt::get(IndVarTy, 0);4721  ConstantInt *One = ConstantInt::get(IndVarTy, 1);4722 4723  // Like Step, but always positive.4724  Value *Incr = Step;4725 4726  // Distance between Start and Stop; always positive.4727  Value *Span;4728 4729  // Condition whether there are no iterations are executed at all, e.g. because4730  // UB < LB.4731  Value *ZeroCmp;4732 4733  if (IsSigned) {4734    // Ensure that increment is positive. If not, negate and invert LB and UB.4735    Value *IsNeg = Builder.CreateICmpSLT(Step, Zero);4736    Incr = Builder.CreateSelect(IsNeg, Builder.CreateNeg(Step), Step);4737    Value *LB = Builder.CreateSelect(IsNeg, Stop, Start);4738    Value *UB = Builder.CreateSelect(IsNeg, Start, Stop);4739    Span = Builder.CreateSub(UB, LB, "", false, true);4740    ZeroCmp = Builder.CreateICmp(4741        InclusiveStop ? CmpInst::ICMP_SLT : CmpInst::ICMP_SLE, UB, LB);4742  } else {4743    Span = Builder.CreateSub(Stop, Start, "", true);4744    ZeroCmp = Builder.CreateICmp(4745        InclusiveStop ? CmpInst::ICMP_ULT : CmpInst::ICMP_ULE, Stop, Start);4746  }4747 4748  Value *CountIfLooping;4749  if (InclusiveStop) {4750    CountIfLooping = Builder.CreateAdd(Builder.CreateUDiv(Span, Incr), One);4751  } else {4752    // Avoid incrementing past stop since it could overflow.4753    Value *CountIfTwo = Builder.CreateAdd(4754        Builder.CreateUDiv(Builder.CreateSub(Span, One), Incr), One);4755    Value *OneCmp = Builder.CreateICmp(CmpInst::ICMP_ULE, Span, Incr);4756    CountIfLooping = Builder.CreateSelect(OneCmp, One, CountIfTwo);4757  }4758 4759  return Builder.CreateSelect(ZeroCmp, Zero, CountIfLooping,4760                              "omp_" + Name + ".tripcount");4761}4762 4763Expected<CanonicalLoopInfo *> OpenMPIRBuilder::createCanonicalLoop(4764    const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,4765    Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,4766    InsertPointTy ComputeIP, const Twine &Name, bool InScan,4767    ScanInfo *ScanRedInfo) {4768  LocationDescription ComputeLoc =4769      ComputeIP.isSet() ? LocationDescription(ComputeIP, Loc.DL) : Loc;4770 4771  Value *TripCount = calculateCanonicalLoopTripCount(4772      ComputeLoc, Start, Stop, Step, IsSigned, InclusiveStop, Name);4773 4774  auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {4775    Builder.restoreIP(CodeGenIP);4776    Value *Span = Builder.CreateMul(IV, Step);4777    Value *IndVar = Builder.CreateAdd(Span, Start);4778    if (InScan)4779      ScanRedInfo->IV = IndVar;4780    return BodyGenCB(Builder.saveIP(), IndVar);4781  };4782  LocationDescription LoopLoc =4783      ComputeIP.isSet()4784          ? Loc4785          : LocationDescription(Builder.saveIP(),4786                                Builder.getCurrentDebugLocation());4787  return createCanonicalLoop(LoopLoc, BodyGen, TripCount, Name);4788}4789 4790// Returns an LLVM function to call for initializing loop bounds using OpenMP4791// static scheduling for composite `distribute parallel for` depending on4792// `type`. Only i32 and i64 are supported by the runtime. Always interpret4793// integers as unsigned similarly to CanonicalLoopInfo.4794static FunctionCallee4795getKmpcDistForStaticInitForType(Type *Ty, Module &M,4796                                OpenMPIRBuilder &OMPBuilder) {4797  unsigned Bitwidth = Ty->getIntegerBitWidth();4798  if (Bitwidth == 32)4799    return OMPBuilder.getOrCreateRuntimeFunction(4800        M, omp::RuntimeFunction::OMPRTL___kmpc_dist_for_static_init_4u);4801  if (Bitwidth == 64)4802    return OMPBuilder.getOrCreateRuntimeFunction(4803        M, omp::RuntimeFunction::OMPRTL___kmpc_dist_for_static_init_8u);4804  llvm_unreachable("unknown OpenMP loop iterator bitwidth");4805}4806 4807// Returns an LLVM function to call for initializing loop bounds using OpenMP4808// static scheduling depending on `type`. Only i32 and i64 are supported by the4809// runtime. Always interpret integers as unsigned similarly to4810// CanonicalLoopInfo.4811static FunctionCallee getKmpcForStaticInitForType(Type *Ty, Module &M,4812                                                  OpenMPIRBuilder &OMPBuilder) {4813  unsigned Bitwidth = Ty->getIntegerBitWidth();4814  if (Bitwidth == 32)4815    return OMPBuilder.getOrCreateRuntimeFunction(4816        M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u);4817  if (Bitwidth == 64)4818    return OMPBuilder.getOrCreateRuntimeFunction(4819        M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u);4820  llvm_unreachable("unknown OpenMP loop iterator bitwidth");4821}4822 4823OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::applyStaticWorkshareLoop(4824    DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,4825    WorksharingLoopType LoopType, bool NeedsBarrier, bool HasDistSchedule,4826    OMPScheduleType DistScheduleSchedType) {4827  assert(CLI->isValid() && "Requires a valid canonical loop");4828  assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) &&4829         "Require dedicated allocate IP");4830 4831  // Set up the source location value for OpenMP runtime.4832  Builder.restoreIP(CLI->getPreheaderIP());4833  Builder.SetCurrentDebugLocation(DL);4834 4835  uint32_t SrcLocStrSize;4836  Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);4837  Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize);4838 4839  // Declare useful OpenMP runtime functions.4840  Value *IV = CLI->getIndVar();4841  Type *IVTy = IV->getType();4842  FunctionCallee StaticInit =4843      LoopType == WorksharingLoopType::DistributeForStaticLoop4844          ? getKmpcDistForStaticInitForType(IVTy, M, *this)4845          : getKmpcForStaticInitForType(IVTy, M, *this);4846  FunctionCallee StaticFini =4847      getOrCreateRuntimeFunction(M, omp::OMPRTL___kmpc_for_static_fini);4848 4849  // Allocate space for computed loop bounds as expected by the "init" function.4850  Builder.SetInsertPoint(AllocaIP.getBlock()->getFirstNonPHIOrDbgOrAlloca());4851 4852  Type *I32Type = Type::getInt32Ty(M.getContext());4853  Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter");4854  Value *PLowerBound = Builder.CreateAlloca(IVTy, nullptr, "p.lowerbound");4855  Value *PUpperBound = Builder.CreateAlloca(IVTy, nullptr, "p.upperbound");4856  Value *PStride = Builder.CreateAlloca(IVTy, nullptr, "p.stride");4857  CLI->setLastIter(PLastIter);4858 4859  // At the end of the preheader, prepare for calling the "init" function by4860  // storing the current loop bounds into the allocated space. A canonical loop4861  // always iterates from 0 to trip-count with step 1. Note that "init" expects4862  // and produces an inclusive upper bound.4863  Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());4864  Constant *Zero = ConstantInt::get(IVTy, 0);4865  Constant *One = ConstantInt::get(IVTy, 1);4866  Builder.CreateStore(Zero, PLowerBound);4867  Value *UpperBound = Builder.CreateSub(CLI->getTripCount(), One);4868  Builder.CreateStore(UpperBound, PUpperBound);4869  Builder.CreateStore(One, PStride);4870 4871  Value *ThreadNum = getOrCreateThreadID(SrcLoc);4872 4873  OMPScheduleType SchedType =4874      (LoopType == WorksharingLoopType::DistributeStaticLoop)4875          ? OMPScheduleType::OrderedDistribute4876          : OMPScheduleType::UnorderedStatic;4877  Constant *SchedulingType =4878      ConstantInt::get(I32Type, static_cast<int>(SchedType));4879 4880  // Call the "init" function and update the trip count of the loop with the4881  // value it produced.4882  auto BuildInitCall = [LoopType, SrcLoc, ThreadNum, PLastIter, PLowerBound,4883                        PUpperBound, IVTy, PStride, One, Zero, StaticInit,4884                        this](Value *SchedulingType, auto &Builder) {4885    SmallVector<Value *, 10> Args({SrcLoc, ThreadNum, SchedulingType, PLastIter,4886                                   PLowerBound, PUpperBound});4887    if (LoopType == WorksharingLoopType::DistributeForStaticLoop) {4888      Value *PDistUpperBound =4889          Builder.CreateAlloca(IVTy, nullptr, "p.distupperbound");4890      Args.push_back(PDistUpperBound);4891    }4892    Args.append({PStride, One, Zero});4893    createRuntimeFunctionCall(StaticInit, Args);4894  };4895  BuildInitCall(SchedulingType, Builder);4896  if (HasDistSchedule &&4897      LoopType != WorksharingLoopType::DistributeStaticLoop) {4898    Constant *DistScheduleSchedType = ConstantInt::get(4899        I32Type, static_cast<int>(omp::OMPScheduleType::OrderedDistribute));4900    // We want to emit a second init function call for the dist_schedule clause4901    // to the Distribute construct. This should only be done however if a4902    // Workshare Loop is nested within a Distribute Construct4903    BuildInitCall(DistScheduleSchedType, Builder);4904  }4905  Value *LowerBound = Builder.CreateLoad(IVTy, PLowerBound);4906  Value *InclusiveUpperBound = Builder.CreateLoad(IVTy, PUpperBound);4907  Value *TripCountMinusOne = Builder.CreateSub(InclusiveUpperBound, LowerBound);4908  Value *TripCount = Builder.CreateAdd(TripCountMinusOne, One);4909  CLI->setTripCount(TripCount);4910 4911  // Update all uses of the induction variable except the one in the condition4912  // block that compares it with the actual upper bound, and the increment in4913  // the latch block.4914 4915  CLI->mapIndVar([&](Instruction *OldIV) -> Value * {4916    Builder.SetInsertPoint(CLI->getBody(),4917                           CLI->getBody()->getFirstInsertionPt());4918    Builder.SetCurrentDebugLocation(DL);4919    return Builder.CreateAdd(OldIV, LowerBound);4920  });4921 4922  // In the "exit" block, call the "fini" function.4923  Builder.SetInsertPoint(CLI->getExit(),4924                         CLI->getExit()->getTerminator()->getIterator());4925  createRuntimeFunctionCall(StaticFini, {SrcLoc, ThreadNum});4926 4927  // Add the barrier if requested.4928  if (NeedsBarrier) {4929    InsertPointOrErrorTy BarrierIP =4930        createBarrier(LocationDescription(Builder.saveIP(), DL),4931                      omp::Directive::OMPD_for, /* ForceSimpleCall */ false,4932                      /* CheckCancelFlag */ false);4933    if (!BarrierIP)4934      return BarrierIP.takeError();4935  }4936 4937  InsertPointTy AfterIP = CLI->getAfterIP();4938  CLI->invalidate();4939 4940  return AfterIP;4941}4942 4943static void addAccessGroupMetadata(BasicBlock *Block, MDNode *AccessGroup,4944                                   LoopInfo &LI);4945static void addLoopMetadata(CanonicalLoopInfo *Loop,4946                            ArrayRef<Metadata *> Properties);4947 4948static void applyParallelAccessesMetadata(CanonicalLoopInfo *CLI,4949                                          LLVMContext &Ctx, Loop *Loop,4950                                          LoopInfo &LoopInfo,4951                                          SmallVector<Metadata *> &LoopMDList) {4952  SmallSet<BasicBlock *, 8> Reachable;4953 4954  // Get the basic blocks from the loop in which memref instructions4955  // can be found.4956  // TODO: Generalize getting all blocks inside a CanonicalizeLoopInfo,4957  // preferably without running any passes.4958  for (BasicBlock *Block : Loop->getBlocks()) {4959    if (Block == CLI->getCond() || Block == CLI->getHeader())4960      continue;4961    Reachable.insert(Block);4962  }4963 4964  // Add access group metadata to memory-access instructions.4965  MDNode *AccessGroup = MDNode::getDistinct(Ctx, {});4966  for (BasicBlock *BB : Reachable)4967    addAccessGroupMetadata(BB, AccessGroup, LoopInfo);4968  // TODO:  If the loop has existing parallel access metadata, have4969  // to combine two lists.4970  LoopMDList.push_back(MDNode::get(4971      Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccessGroup}));4972}4973 4974OpenMPIRBuilder::InsertPointOrErrorTy4975OpenMPIRBuilder::applyStaticChunkedWorkshareLoop(4976    DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,4977    bool NeedsBarrier, Value *ChunkSize, OMPScheduleType SchedType,4978    Value *DistScheduleChunkSize, OMPScheduleType DistScheduleSchedType) {4979  assert(CLI->isValid() && "Requires a valid canonical loop");4980  assert(ChunkSize || DistScheduleChunkSize && "Chunk size is required");4981 4982  LLVMContext &Ctx = CLI->getFunction()->getContext();4983  Value *IV = CLI->getIndVar();4984  Value *OrigTripCount = CLI->getTripCount();4985  Type *IVTy = IV->getType();4986  assert(IVTy->getIntegerBitWidth() <= 64 &&4987         "Max supported tripcount bitwidth is 64 bits");4988  Type *InternalIVTy = IVTy->getIntegerBitWidth() <= 32 ? Type::getInt32Ty(Ctx)4989                                                        : Type::getInt64Ty(Ctx);4990  Type *I32Type = Type::getInt32Ty(M.getContext());4991  Constant *Zero = ConstantInt::get(InternalIVTy, 0);4992  Constant *One = ConstantInt::get(InternalIVTy, 1);4993 4994  Function *F = CLI->getFunction();4995  FunctionAnalysisManager FAM;4996  FAM.registerPass([]() { return DominatorTreeAnalysis(); });4997  FAM.registerPass([]() { return PassInstrumentationAnalysis(); });4998  LoopAnalysis LIA;4999  LoopInfo &&LI = LIA.run(*F, FAM);5000  Loop *L = LI.getLoopFor(CLI->getHeader());5001  SmallVector<Metadata *> LoopMDList;5002  if (ChunkSize || DistScheduleChunkSize)5003    applyParallelAccessesMetadata(CLI, Ctx, L, LI, LoopMDList);5004  addLoopMetadata(CLI, LoopMDList);5005 5006  // Declare useful OpenMP runtime functions.5007  FunctionCallee StaticInit =5008      getKmpcForStaticInitForType(InternalIVTy, M, *this);5009  FunctionCallee StaticFini =5010      getOrCreateRuntimeFunction(M, omp::OMPRTL___kmpc_for_static_fini);5011 5012  // Allocate space for computed loop bounds as expected by the "init" function.5013  Builder.restoreIP(AllocaIP);5014  Builder.SetCurrentDebugLocation(DL);5015  Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter");5016  Value *PLowerBound =5017      Builder.CreateAlloca(InternalIVTy, nullptr, "p.lowerbound");5018  Value *PUpperBound =5019      Builder.CreateAlloca(InternalIVTy, nullptr, "p.upperbound");5020  Value *PStride = Builder.CreateAlloca(InternalIVTy, nullptr, "p.stride");5021  CLI->setLastIter(PLastIter);5022 5023  // Set up the source location value for the OpenMP runtime.5024  Builder.restoreIP(CLI->getPreheaderIP());5025  Builder.SetCurrentDebugLocation(DL);5026 5027  // TODO: Detect overflow in ubsan or max-out with current tripcount.5028  Value *CastedChunkSize = Builder.CreateZExtOrTrunc(5029      ChunkSize ? ChunkSize : Zero, InternalIVTy, "chunksize");5030  Value *CastedDistScheduleChunkSize = Builder.CreateZExtOrTrunc(5031      DistScheduleChunkSize ? DistScheduleChunkSize : Zero, InternalIVTy,5032      "distschedulechunksize");5033  Value *CastedTripCount =5034      Builder.CreateZExt(OrigTripCount, InternalIVTy, "tripcount");5035 5036  Constant *SchedulingType =5037      ConstantInt::get(I32Type, static_cast<int>(SchedType));5038  Constant *DistSchedulingType =5039      ConstantInt::get(I32Type, static_cast<int>(DistScheduleSchedType));5040  Builder.CreateStore(Zero, PLowerBound);5041  Value *OrigUpperBound = Builder.CreateSub(CastedTripCount, One);5042  Builder.CreateStore(OrigUpperBound, PUpperBound);5043  Builder.CreateStore(One, PStride);5044 5045  // Call the "init" function and update the trip count of the loop with the5046  // value it produced.5047  uint32_t SrcLocStrSize;5048  Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);5049  Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize);5050  Value *ThreadNum = getOrCreateThreadID(SrcLoc);5051  auto BuildInitCall = [StaticInit, SrcLoc, ThreadNum, PLastIter, PLowerBound,5052                        PUpperBound, PStride, One,5053                        this](Value *SchedulingType, Value *ChunkSize,5054                              auto &Builder) {5055    createRuntimeFunctionCall(5056        StaticInit, {/*loc=*/SrcLoc, /*global_tid=*/ThreadNum,5057                     /*schedtype=*/SchedulingType, /*plastiter=*/PLastIter,5058                     /*plower=*/PLowerBound, /*pupper=*/PUpperBound,5059                     /*pstride=*/PStride, /*incr=*/One,5060                     /*chunk=*/ChunkSize});5061  };5062  BuildInitCall(SchedulingType, CastedChunkSize, Builder);5063  if (DistScheduleSchedType != OMPScheduleType::None &&5064      SchedType != OMPScheduleType::OrderedDistributeChunked &&5065      SchedType != OMPScheduleType::OrderedDistribute) {5066    // We want to emit a second init function call for the dist_schedule clause5067    // to the Distribute construct. This should only be done however if a5068    // Workshare Loop is nested within a Distribute Construct5069    BuildInitCall(DistSchedulingType, CastedDistScheduleChunkSize, Builder);5070  }5071 5072  // Load values written by the "init" function.5073  Value *FirstChunkStart =5074      Builder.CreateLoad(InternalIVTy, PLowerBound, "omp_firstchunk.lb");5075  Value *FirstChunkStop =5076      Builder.CreateLoad(InternalIVTy, PUpperBound, "omp_firstchunk.ub");5077  Value *FirstChunkEnd = Builder.CreateAdd(FirstChunkStop, One);5078  Value *ChunkRange =5079      Builder.CreateSub(FirstChunkEnd, FirstChunkStart, "omp_chunk.range");5080  Value *NextChunkStride =5081      Builder.CreateLoad(InternalIVTy, PStride, "omp_dispatch.stride");5082 5083  // Create outer "dispatch" loop for enumerating the chunks.5084  BasicBlock *DispatchEnter = splitBB(Builder, true);5085  Value *DispatchCounter;5086 5087  // It is safe to assume this didn't return an error because the callback5088  // passed into createCanonicalLoop is the only possible error source, and it5089  // always returns success.5090  CanonicalLoopInfo *DispatchCLI = cantFail(createCanonicalLoop(5091      {Builder.saveIP(), DL},5092      [&](InsertPointTy BodyIP, Value *Counter) {5093        DispatchCounter = Counter;5094        return Error::success();5095      },5096      FirstChunkStart, CastedTripCount, NextChunkStride,5097      /*IsSigned=*/false, /*InclusiveStop=*/false, /*ComputeIP=*/{},5098      "dispatch"));5099 5100  // Remember the BasicBlocks of the dispatch loop we need, then invalidate to5101  // not have to preserve the canonical invariant.5102  BasicBlock *DispatchBody = DispatchCLI->getBody();5103  BasicBlock *DispatchLatch = DispatchCLI->getLatch();5104  BasicBlock *DispatchExit = DispatchCLI->getExit();5105  BasicBlock *DispatchAfter = DispatchCLI->getAfter();5106  DispatchCLI->invalidate();5107 5108  // Rewire the original loop to become the chunk loop inside the dispatch loop.5109  redirectTo(DispatchAfter, CLI->getAfter(), DL);5110  redirectTo(CLI->getExit(), DispatchLatch, DL);5111  redirectTo(DispatchBody, DispatchEnter, DL);5112 5113  // Prepare the prolog of the chunk loop.5114  Builder.restoreIP(CLI->getPreheaderIP());5115  Builder.SetCurrentDebugLocation(DL);5116 5117  // Compute the number of iterations of the chunk loop.5118  Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());5119  Value *ChunkEnd = Builder.CreateAdd(DispatchCounter, ChunkRange);5120  Value *IsLastChunk =5121      Builder.CreateICmpUGE(ChunkEnd, CastedTripCount, "omp_chunk.is_last");5122  Value *CountUntilOrigTripCount =5123      Builder.CreateSub(CastedTripCount, DispatchCounter);5124  Value *ChunkTripCount = Builder.CreateSelect(5125      IsLastChunk, CountUntilOrigTripCount, ChunkRange, "omp_chunk.tripcount");5126  Value *BackcastedChunkTC =5127      Builder.CreateTrunc(ChunkTripCount, IVTy, "omp_chunk.tripcount.trunc");5128  CLI->setTripCount(BackcastedChunkTC);5129 5130  // Update all uses of the induction variable except the one in the condition5131  // block that compares it with the actual upper bound, and the increment in5132  // the latch block.5133  Value *BackcastedDispatchCounter =5134      Builder.CreateTrunc(DispatchCounter, IVTy, "omp_dispatch.iv.trunc");5135  CLI->mapIndVar([&](Instruction *) -> Value * {5136    Builder.restoreIP(CLI->getBodyIP());5137    return Builder.CreateAdd(IV, BackcastedDispatchCounter);5138  });5139 5140  // In the "exit" block, call the "fini" function.5141  Builder.SetInsertPoint(DispatchExit, DispatchExit->getFirstInsertionPt());5142  createRuntimeFunctionCall(StaticFini, {SrcLoc, ThreadNum});5143 5144  // Add the barrier if requested.5145  if (NeedsBarrier) {5146    InsertPointOrErrorTy AfterIP =5147        createBarrier(LocationDescription(Builder.saveIP(), DL), OMPD_for,5148                      /*ForceSimpleCall=*/false, /*CheckCancelFlag=*/false);5149    if (!AfterIP)5150      return AfterIP.takeError();5151  }5152 5153#ifndef NDEBUG5154  // Even though we currently do not support applying additional methods to it,5155  // the chunk loop should remain a canonical loop.5156  CLI->assertOK();5157#endif5158 5159  return InsertPointTy(DispatchAfter, DispatchAfter->getFirstInsertionPt());5160}5161 5162// Returns an LLVM function to call for executing an OpenMP static worksharing5163// for loop depending on `type`. Only i32 and i64 are supported by the runtime.5164// Always interpret integers as unsigned similarly to CanonicalLoopInfo.5165static FunctionCallee5166getKmpcForStaticLoopForType(Type *Ty, OpenMPIRBuilder *OMPBuilder,5167                            WorksharingLoopType LoopType) {5168  unsigned Bitwidth = Ty->getIntegerBitWidth();5169  Module &M = OMPBuilder->M;5170  switch (LoopType) {5171  case WorksharingLoopType::ForStaticLoop:5172    if (Bitwidth == 32)5173      return OMPBuilder->getOrCreateRuntimeFunction(5174          M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_loop_4u);5175    if (Bitwidth == 64)5176      return OMPBuilder->getOrCreateRuntimeFunction(5177          M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_loop_8u);5178    break;5179  case WorksharingLoopType::DistributeStaticLoop:5180    if (Bitwidth == 32)5181      return OMPBuilder->getOrCreateRuntimeFunction(5182          M, omp::RuntimeFunction::OMPRTL___kmpc_distribute_static_loop_4u);5183    if (Bitwidth == 64)5184      return OMPBuilder->getOrCreateRuntimeFunction(5185          M, omp::RuntimeFunction::OMPRTL___kmpc_distribute_static_loop_8u);5186    break;5187  case WorksharingLoopType::DistributeForStaticLoop:5188    if (Bitwidth == 32)5189      return OMPBuilder->getOrCreateRuntimeFunction(5190          M, omp::RuntimeFunction::OMPRTL___kmpc_distribute_for_static_loop_4u);5191    if (Bitwidth == 64)5192      return OMPBuilder->getOrCreateRuntimeFunction(5193          M, omp::RuntimeFunction::OMPRTL___kmpc_distribute_for_static_loop_8u);5194    break;5195  }5196  if (Bitwidth != 32 && Bitwidth != 64) {5197    llvm_unreachable("Unknown OpenMP loop iterator bitwidth");5198  }5199  llvm_unreachable("Unknown type of OpenMP worksharing loop");5200}5201 5202// Inserts a call to proper OpenMP Device RTL function which handles5203// loop worksharing.5204static void createTargetLoopWorkshareCall(OpenMPIRBuilder *OMPBuilder,5205                                          WorksharingLoopType LoopType,5206                                          BasicBlock *InsertBlock, Value *Ident,5207                                          Value *LoopBodyArg, Value *TripCount,5208                                          Function &LoopBodyFn, bool NoLoop) {5209  Type *TripCountTy = TripCount->getType();5210  Module &M = OMPBuilder->M;5211  IRBuilder<> &Builder = OMPBuilder->Builder;5212  FunctionCallee RTLFn =5213      getKmpcForStaticLoopForType(TripCountTy, OMPBuilder, LoopType);5214  SmallVector<Value *, 8> RealArgs;5215  RealArgs.push_back(Ident);5216  RealArgs.push_back(&LoopBodyFn);5217  RealArgs.push_back(LoopBodyArg);5218  RealArgs.push_back(TripCount);5219  if (LoopType == WorksharingLoopType::DistributeStaticLoop) {5220    RealArgs.push_back(ConstantInt::get(TripCountTy, 0));5221    RealArgs.push_back(ConstantInt::get(Builder.getInt8Ty(), 0));5222    Builder.restoreIP({InsertBlock, std::prev(InsertBlock->end())});5223    OMPBuilder->createRuntimeFunctionCall(RTLFn, RealArgs);5224    return;5225  }5226  FunctionCallee RTLNumThreads = OMPBuilder->getOrCreateRuntimeFunction(5227      M, omp::RuntimeFunction::OMPRTL_omp_get_num_threads);5228  Builder.restoreIP({InsertBlock, std::prev(InsertBlock->end())});5229  Value *NumThreads = OMPBuilder->createRuntimeFunctionCall(RTLNumThreads, {});5230 5231  RealArgs.push_back(5232      Builder.CreateZExtOrTrunc(NumThreads, TripCountTy, "num.threads.cast"));5233  RealArgs.push_back(ConstantInt::get(TripCountTy, 0));5234  if (LoopType == WorksharingLoopType::DistributeForStaticLoop) {5235    RealArgs.push_back(ConstantInt::get(TripCountTy, 0));5236    RealArgs.push_back(ConstantInt::get(Builder.getInt8Ty(), NoLoop));5237  } else {5238    RealArgs.push_back(ConstantInt::get(Builder.getInt8Ty(), 0));5239  }5240 5241  OMPBuilder->createRuntimeFunctionCall(RTLFn, RealArgs);5242}5243 5244static void workshareLoopTargetCallback(5245    OpenMPIRBuilder *OMPIRBuilder, CanonicalLoopInfo *CLI, Value *Ident,5246    Function &OutlinedFn, const SmallVector<Instruction *, 4> &ToBeDeleted,5247    WorksharingLoopType LoopType, bool NoLoop) {5248  IRBuilder<> &Builder = OMPIRBuilder->Builder;5249  BasicBlock *Preheader = CLI->getPreheader();5250  Value *TripCount = CLI->getTripCount();5251 5252  // After loop body outling, the loop body contains only set up5253  // of loop body argument structure and the call to the outlined5254  // loop body function. Firstly, we need to move setup of loop body args5255  // into loop preheader.5256  Preheader->splice(std::prev(Preheader->end()), CLI->getBody(),5257                    CLI->getBody()->begin(), std::prev(CLI->getBody()->end()));5258 5259  // The next step is to remove the whole loop. We do not it need anymore.5260  // That's why make an unconditional branch from loop preheader to loop5261  // exit block5262  Builder.restoreIP({Preheader, Preheader->end()});5263  Builder.SetCurrentDebugLocation(Preheader->getTerminator()->getDebugLoc());5264  Preheader->getTerminator()->eraseFromParent();5265  Builder.CreateBr(CLI->getExit());5266 5267  // Delete dead loop blocks5268  OpenMPIRBuilder::OutlineInfo CleanUpInfo;5269  SmallPtrSet<BasicBlock *, 32> RegionBlockSet;5270  SmallVector<BasicBlock *, 32> BlocksToBeRemoved;5271  CleanUpInfo.EntryBB = CLI->getHeader();5272  CleanUpInfo.ExitBB = CLI->getExit();5273  CleanUpInfo.collectBlocks(RegionBlockSet, BlocksToBeRemoved);5274  DeleteDeadBlocks(BlocksToBeRemoved);5275 5276  // Find the instruction which corresponds to loop body argument structure5277  // and remove the call to loop body function instruction.5278  Value *LoopBodyArg;5279  User *OutlinedFnUser = OutlinedFn.getUniqueUndroppableUser();5280  assert(OutlinedFnUser &&5281         "Expected unique undroppable user of outlined function");5282  CallInst *OutlinedFnCallInstruction = dyn_cast<CallInst>(OutlinedFnUser);5283  assert(OutlinedFnCallInstruction && "Expected outlined function call");5284  assert((OutlinedFnCallInstruction->getParent() == Preheader) &&5285         "Expected outlined function call to be located in loop preheader");5286  // Check in case no argument structure has been passed.5287  if (OutlinedFnCallInstruction->arg_size() > 1)5288    LoopBodyArg = OutlinedFnCallInstruction->getArgOperand(1);5289  else5290    LoopBodyArg = Constant::getNullValue(Builder.getPtrTy());5291  OutlinedFnCallInstruction->eraseFromParent();5292 5293  createTargetLoopWorkshareCall(OMPIRBuilder, LoopType, Preheader, Ident,5294                                LoopBodyArg, TripCount, OutlinedFn, NoLoop);5295 5296  for (auto &ToBeDeletedItem : ToBeDeleted)5297    ToBeDeletedItem->eraseFromParent();5298  CLI->invalidate();5299}5300 5301OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyWorkshareLoopTarget(5302    DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,5303    WorksharingLoopType LoopType, bool NoLoop) {5304  uint32_t SrcLocStrSize;5305  Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);5306  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);5307 5308  OutlineInfo OI;5309  OI.OuterAllocaBB = CLI->getPreheader();5310  Function *OuterFn = CLI->getPreheader()->getParent();5311 5312  // Instructions which need to be deleted at the end of code generation5313  SmallVector<Instruction *, 4> ToBeDeleted;5314 5315  OI.OuterAllocaBB = AllocaIP.getBlock();5316 5317  // Mark the body loop as region which needs to be extracted5318  OI.EntryBB = CLI->getBody();5319  OI.ExitBB = CLI->getLatch()->splitBasicBlock(CLI->getLatch()->begin(),5320                                               "omp.prelatch", true);5321 5322  // Prepare loop body for extraction5323  Builder.restoreIP({CLI->getPreheader(), CLI->getPreheader()->begin()});5324 5325  // Insert new loop counter variable which will be used only in loop5326  // body.5327  AllocaInst *NewLoopCnt = Builder.CreateAlloca(CLI->getIndVarType(), 0, "");5328  Instruction *NewLoopCntLoad =5329      Builder.CreateLoad(CLI->getIndVarType(), NewLoopCnt);5330  // New loop counter instructions are redundant in the loop preheader when5331  // code generation for workshare loop is finshed. That's why mark them as5332  // ready for deletion.5333  ToBeDeleted.push_back(NewLoopCntLoad);5334  ToBeDeleted.push_back(NewLoopCnt);5335 5336  // Analyse loop body region. Find all input variables which are used inside5337  // loop body region.5338  SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;5339  SmallVector<BasicBlock *, 32> Blocks;5340  OI.collectBlocks(ParallelRegionBlockSet, Blocks);5341 5342  CodeExtractorAnalysisCache CEAC(*OuterFn);5343  CodeExtractor Extractor(Blocks,5344                          /* DominatorTree */ nullptr,5345                          /* AggregateArgs */ true,5346                          /* BlockFrequencyInfo */ nullptr,5347                          /* BranchProbabilityInfo */ nullptr,5348                          /* AssumptionCache */ nullptr,5349                          /* AllowVarArgs */ true,5350                          /* AllowAlloca */ true,5351                          /* AllocationBlock */ CLI->getPreheader(),5352                          /* Suffix */ ".omp_wsloop",5353                          /* AggrArgsIn0AddrSpace */ true);5354 5355  BasicBlock *CommonExit = nullptr;5356  SetVector<Value *> SinkingCands, HoistingCands;5357 5358  // Find allocas outside the loop body region which are used inside loop5359  // body5360  Extractor.findAllocas(CEAC, SinkingCands, HoistingCands, CommonExit);5361 5362  // We need to model loop body region as the function f(cnt, loop_arg).5363  // That's why we replace loop induction variable by the new counter5364  // which will be one of loop body function argument5365  SmallVector<User *> Users(CLI->getIndVar()->user_begin(),5366                            CLI->getIndVar()->user_end());5367  for (auto Use : Users) {5368    if (Instruction *Inst = dyn_cast<Instruction>(Use)) {5369      if (ParallelRegionBlockSet.count(Inst->getParent())) {5370        Inst->replaceUsesOfWith(CLI->getIndVar(), NewLoopCntLoad);5371      }5372    }5373  }5374  // Make sure that loop counter variable is not merged into loop body5375  // function argument structure and it is passed as separate variable5376  OI.ExcludeArgsFromAggregate.push_back(NewLoopCntLoad);5377 5378  // PostOutline CB is invoked when loop body function is outlined and5379  // loop body is replaced by call to outlined function. We need to add5380  // call to OpenMP device rtl inside loop preheader. OpenMP device rtl5381  // function will handle loop control logic.5382  //5383  OI.PostOutlineCB = [=, ToBeDeletedVec =5384                             std::move(ToBeDeleted)](Function &OutlinedFn) {5385    workshareLoopTargetCallback(this, CLI, Ident, OutlinedFn, ToBeDeletedVec,5386                                LoopType, NoLoop);5387  };5388  addOutlineInfo(std::move(OI));5389  return CLI->getAfterIP();5390}5391 5392OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::applyWorkshareLoop(5393    DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,5394    bool NeedsBarrier, omp::ScheduleKind SchedKind, Value *ChunkSize,5395    bool HasSimdModifier, bool HasMonotonicModifier,5396    bool HasNonmonotonicModifier, bool HasOrderedClause,5397    WorksharingLoopType LoopType, bool NoLoop, bool HasDistSchedule,5398    Value *DistScheduleChunkSize) {5399  if (Config.isTargetDevice())5400    return applyWorkshareLoopTarget(DL, CLI, AllocaIP, LoopType, NoLoop);5401  OMPScheduleType EffectiveScheduleType = computeOpenMPScheduleType(5402      SchedKind, ChunkSize, HasSimdModifier, HasMonotonicModifier,5403      HasNonmonotonicModifier, HasOrderedClause, DistScheduleChunkSize);5404 5405  bool IsOrdered = (EffectiveScheduleType & OMPScheduleType::ModifierOrdered) ==5406                   OMPScheduleType::ModifierOrdered;5407  OMPScheduleType DistScheduleSchedType = OMPScheduleType::None;5408  if (HasDistSchedule) {5409    DistScheduleSchedType = DistScheduleChunkSize5410                                ? OMPScheduleType::OrderedDistributeChunked5411                                : OMPScheduleType::OrderedDistribute;5412  }5413  switch (EffectiveScheduleType & ~OMPScheduleType::ModifierMask) {5414  case OMPScheduleType::BaseStatic:5415  case OMPScheduleType::BaseDistribute:5416    assert(!ChunkSize || !DistScheduleChunkSize &&5417                             "No chunk size with static-chunked schedule");5418    if (IsOrdered && !HasDistSchedule)5419      return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, EffectiveScheduleType,5420                                       NeedsBarrier, ChunkSize);5421    // FIXME: Monotonicity ignored?5422    if (DistScheduleChunkSize)5423      return applyStaticChunkedWorkshareLoop(5424          DL, CLI, AllocaIP, NeedsBarrier, ChunkSize, EffectiveScheduleType,5425          DistScheduleChunkSize, DistScheduleSchedType);5426    return applyStaticWorkshareLoop(DL, CLI, AllocaIP, LoopType, NeedsBarrier,5427                                    HasDistSchedule);5428 5429  case OMPScheduleType::BaseStaticChunked:5430  case OMPScheduleType::BaseDistributeChunked:5431    if (IsOrdered && !HasDistSchedule)5432      return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, EffectiveScheduleType,5433                                       NeedsBarrier, ChunkSize);5434    // FIXME: Monotonicity ignored?5435    return applyStaticChunkedWorkshareLoop(5436        DL, CLI, AllocaIP, NeedsBarrier, ChunkSize, EffectiveScheduleType,5437        DistScheduleChunkSize, DistScheduleSchedType);5438 5439  case OMPScheduleType::BaseRuntime:5440  case OMPScheduleType::BaseAuto:5441  case OMPScheduleType::BaseGreedy:5442  case OMPScheduleType::BaseBalanced:5443  case OMPScheduleType::BaseSteal:5444  case OMPScheduleType::BaseGuidedSimd:5445  case OMPScheduleType::BaseRuntimeSimd:5446    assert(!ChunkSize &&5447           "schedule type does not support user-defined chunk sizes");5448    [[fallthrough]];5449  case OMPScheduleType::BaseDynamicChunked:5450  case OMPScheduleType::BaseGuidedChunked:5451  case OMPScheduleType::BaseGuidedIterativeChunked:5452  case OMPScheduleType::BaseGuidedAnalyticalChunked:5453  case OMPScheduleType::BaseStaticBalancedChunked:5454    return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, EffectiveScheduleType,5455                                     NeedsBarrier, ChunkSize);5456 5457  default:5458    llvm_unreachable("Unknown/unimplemented schedule kind");5459  }5460}5461 5462/// Returns an LLVM function to call for initializing loop bounds using OpenMP5463/// dynamic scheduling depending on `type`. Only i32 and i64 are supported by5464/// the runtime. Always interpret integers as unsigned similarly to5465/// CanonicalLoopInfo.5466static FunctionCallee5467getKmpcForDynamicInitForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {5468  unsigned Bitwidth = Ty->getIntegerBitWidth();5469  if (Bitwidth == 32)5470    return OMPBuilder.getOrCreateRuntimeFunction(5471        M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_4u);5472  if (Bitwidth == 64)5473    return OMPBuilder.getOrCreateRuntimeFunction(5474        M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_8u);5475  llvm_unreachable("unknown OpenMP loop iterator bitwidth");5476}5477 5478/// Returns an LLVM function to call for updating the next loop using OpenMP5479/// dynamic scheduling depending on `type`. Only i32 and i64 are supported by5480/// the runtime. Always interpret integers as unsigned similarly to5481/// CanonicalLoopInfo.5482static FunctionCallee5483getKmpcForDynamicNextForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {5484  unsigned Bitwidth = Ty->getIntegerBitWidth();5485  if (Bitwidth == 32)5486    return OMPBuilder.getOrCreateRuntimeFunction(5487        M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_4u);5488  if (Bitwidth == 64)5489    return OMPBuilder.getOrCreateRuntimeFunction(5490        M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_8u);5491  llvm_unreachable("unknown OpenMP loop iterator bitwidth");5492}5493 5494/// Returns an LLVM function to call for finalizing the dynamic loop using5495/// depending on `type`. Only i32 and i64 are supported by the runtime. Always5496/// interpret integers as unsigned similarly to CanonicalLoopInfo.5497static FunctionCallee5498getKmpcForDynamicFiniForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {5499  unsigned Bitwidth = Ty->getIntegerBitWidth();5500  if (Bitwidth == 32)5501    return OMPBuilder.getOrCreateRuntimeFunction(5502        M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_4u);5503  if (Bitwidth == 64)5504    return OMPBuilder.getOrCreateRuntimeFunction(5505        M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_8u);5506  llvm_unreachable("unknown OpenMP loop iterator bitwidth");5507}5508 5509OpenMPIRBuilder::InsertPointOrErrorTy5510OpenMPIRBuilder::applyDynamicWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI,5511                                           InsertPointTy AllocaIP,5512                                           OMPScheduleType SchedType,5513                                           bool NeedsBarrier, Value *Chunk) {5514  assert(CLI->isValid() && "Requires a valid canonical loop");5515  assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) &&5516         "Require dedicated allocate IP");5517  assert(isValidWorkshareLoopScheduleType(SchedType) &&5518         "Require valid schedule type");5519 5520  bool Ordered = (SchedType & OMPScheduleType::ModifierOrdered) ==5521                 OMPScheduleType::ModifierOrdered;5522 5523  // Set up the source location value for OpenMP runtime.5524  Builder.SetCurrentDebugLocation(DL);5525 5526  uint32_t SrcLocStrSize;5527  Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);5528  Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize);5529 5530  // Declare useful OpenMP runtime functions.5531  Value *IV = CLI->getIndVar();5532  Type *IVTy = IV->getType();5533  FunctionCallee DynamicInit = getKmpcForDynamicInitForType(IVTy, M, *this);5534  FunctionCallee DynamicNext = getKmpcForDynamicNextForType(IVTy, M, *this);5535 5536  // Allocate space for computed loop bounds as expected by the "init" function.5537  Builder.SetInsertPoint(AllocaIP.getBlock()->getFirstNonPHIOrDbgOrAlloca());5538  Type *I32Type = Type::getInt32Ty(M.getContext());5539  Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter");5540  Value *PLowerBound = Builder.CreateAlloca(IVTy, nullptr, "p.lowerbound");5541  Value *PUpperBound = Builder.CreateAlloca(IVTy, nullptr, "p.upperbound");5542  Value *PStride = Builder.CreateAlloca(IVTy, nullptr, "p.stride");5543  CLI->setLastIter(PLastIter);5544 5545  // At the end of the preheader, prepare for calling the "init" function by5546  // storing the current loop bounds into the allocated space. A canonical loop5547  // always iterates from 0 to trip-count with step 1. Note that "init" expects5548  // and produces an inclusive upper bound.5549  BasicBlock *PreHeader = CLI->getPreheader();5550  Builder.SetInsertPoint(PreHeader->getTerminator());5551  Constant *One = ConstantInt::get(IVTy, 1);5552  Builder.CreateStore(One, PLowerBound);5553  Value *UpperBound = CLI->getTripCount();5554  Builder.CreateStore(UpperBound, PUpperBound);5555  Builder.CreateStore(One, PStride);5556 5557  BasicBlock *Header = CLI->getHeader();5558  BasicBlock *Exit = CLI->getExit();5559  BasicBlock *Cond = CLI->getCond();5560  BasicBlock *Latch = CLI->getLatch();5561  InsertPointTy AfterIP = CLI->getAfterIP();5562 5563  // The CLI will be "broken" in the code below, as the loop is no longer5564  // a valid canonical loop.5565 5566  if (!Chunk)5567    Chunk = One;5568 5569  Value *ThreadNum = getOrCreateThreadID(SrcLoc);5570 5571  Constant *SchedulingType =5572      ConstantInt::get(I32Type, static_cast<int>(SchedType));5573 5574  // Call the "init" function.5575  createRuntimeFunctionCall(DynamicInit, {SrcLoc, ThreadNum, SchedulingType,5576                                          /* LowerBound */ One, UpperBound,5577                                          /* step */ One, Chunk});5578 5579  // An outer loop around the existing one.5580  BasicBlock *OuterCond = BasicBlock::Create(5581      PreHeader->getContext(), Twine(PreHeader->getName()) + ".outer.cond",5582      PreHeader->getParent());5583  // This needs to be 32-bit always, so can't use the IVTy Zero above.5584  Builder.SetInsertPoint(OuterCond, OuterCond->getFirstInsertionPt());5585  Value *Res = createRuntimeFunctionCall(5586      DynamicNext,5587      {SrcLoc, ThreadNum, PLastIter, PLowerBound, PUpperBound, PStride});5588  Constant *Zero32 = ConstantInt::get(I32Type, 0);5589  Value *MoreWork = Builder.CreateCmp(CmpInst::ICMP_NE, Res, Zero32);5590  Value *LowerBound =5591      Builder.CreateSub(Builder.CreateLoad(IVTy, PLowerBound), One, "lb");5592  Builder.CreateCondBr(MoreWork, Header, Exit);5593 5594  // Change PHI-node in loop header to use outer cond rather than preheader,5595  // and set IV to the LowerBound.5596  Instruction *Phi = &Header->front();5597  auto *PI = cast<PHINode>(Phi);5598  PI->setIncomingBlock(0, OuterCond);5599  PI->setIncomingValue(0, LowerBound);5600 5601  // Then set the pre-header to jump to the OuterCond5602  Instruction *Term = PreHeader->getTerminator();5603  auto *Br = cast<BranchInst>(Term);5604  Br->setSuccessor(0, OuterCond);5605 5606  // Modify the inner condition:5607  // * Use the UpperBound returned from the DynamicNext call.5608  // * jump to the loop outer loop when done with one of the inner loops.5609  Builder.SetInsertPoint(Cond, Cond->getFirstInsertionPt());5610  UpperBound = Builder.CreateLoad(IVTy, PUpperBound, "ub");5611  Instruction *Comp = &*Builder.GetInsertPoint();5612  auto *CI = cast<CmpInst>(Comp);5613  CI->setOperand(1, UpperBound);5614  // Redirect the inner exit to branch to outer condition.5615  Instruction *Branch = &Cond->back();5616  auto *BI = cast<BranchInst>(Branch);5617  assert(BI->getSuccessor(1) == Exit);5618  BI->setSuccessor(1, OuterCond);5619 5620  // Call the "fini" function if "ordered" is present in wsloop directive.5621  if (Ordered) {5622    Builder.SetInsertPoint(&Latch->back());5623    FunctionCallee DynamicFini = getKmpcForDynamicFiniForType(IVTy, M, *this);5624    createRuntimeFunctionCall(DynamicFini, {SrcLoc, ThreadNum});5625  }5626 5627  // Add the barrier if requested.5628  if (NeedsBarrier) {5629    Builder.SetInsertPoint(&Exit->back());5630    InsertPointOrErrorTy BarrierIP =5631        createBarrier(LocationDescription(Builder.saveIP(), DL),5632                      omp::Directive::OMPD_for, /* ForceSimpleCall */ false,5633                      /* CheckCancelFlag */ false);5634    if (!BarrierIP)5635      return BarrierIP.takeError();5636  }5637 5638  CLI->invalidate();5639  return AfterIP;5640}5641 5642/// Redirect all edges that branch to \p OldTarget to \p NewTarget. That is,5643/// after this \p OldTarget will be orphaned.5644static void redirectAllPredecessorsTo(BasicBlock *OldTarget,5645                                      BasicBlock *NewTarget, DebugLoc DL) {5646  for (BasicBlock *Pred : make_early_inc_range(predecessors(OldTarget)))5647    redirectTo(Pred, NewTarget, DL);5648}5649 5650/// Determine which blocks in \p BBs are reachable from outside and remove the5651/// ones that are not reachable from the function.5652static void removeUnusedBlocksFromParent(ArrayRef<BasicBlock *> BBs) {5653  SmallPtrSet<BasicBlock *, 6> BBsToErase(llvm::from_range, BBs);5654  auto HasRemainingUses = [&BBsToErase](BasicBlock *BB) {5655    for (Use &U : BB->uses()) {5656      auto *UseInst = dyn_cast<Instruction>(U.getUser());5657      if (!UseInst)5658        continue;5659      if (BBsToErase.count(UseInst->getParent()))5660        continue;5661      return true;5662    }5663    return false;5664  };5665 5666  while (BBsToErase.remove_if(HasRemainingUses)) {5667    // Try again if anything was removed.5668  }5669 5670  SmallVector<BasicBlock *, 7> BBVec(BBsToErase.begin(), BBsToErase.end());5671  DeleteDeadBlocks(BBVec);5672}5673 5674CanonicalLoopInfo *5675OpenMPIRBuilder::collapseLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,5676                               InsertPointTy ComputeIP) {5677  assert(Loops.size() >= 1 && "At least one loop required");5678  size_t NumLoops = Loops.size();5679 5680  // Nothing to do if there is already just one loop.5681  if (NumLoops == 1)5682    return Loops.front();5683 5684  CanonicalLoopInfo *Outermost = Loops.front();5685  CanonicalLoopInfo *Innermost = Loops.back();5686  BasicBlock *OrigPreheader = Outermost->getPreheader();5687  BasicBlock *OrigAfter = Outermost->getAfter();5688  Function *F = OrigPreheader->getParent();5689 5690  // Loop control blocks that may become orphaned later.5691  SmallVector<BasicBlock *, 12> OldControlBBs;5692  OldControlBBs.reserve(6 * Loops.size());5693  for (CanonicalLoopInfo *Loop : Loops)5694    Loop->collectControlBlocks(OldControlBBs);5695 5696  // Setup the IRBuilder for inserting the trip count computation.5697  Builder.SetCurrentDebugLocation(DL);5698  if (ComputeIP.isSet())5699    Builder.restoreIP(ComputeIP);5700  else5701    Builder.restoreIP(Outermost->getPreheaderIP());5702 5703  // Derive the collapsed' loop trip count.5704  // TODO: Find common/largest indvar type.5705  Value *CollapsedTripCount = nullptr;5706  for (CanonicalLoopInfo *L : Loops) {5707    assert(L->isValid() &&5708           "All loops to collapse must be valid canonical loops");5709    Value *OrigTripCount = L->getTripCount();5710    if (!CollapsedTripCount) {5711      CollapsedTripCount = OrigTripCount;5712      continue;5713    }5714 5715    // TODO: Enable UndefinedSanitizer to diagnose an overflow here.5716    CollapsedTripCount =5717        Builder.CreateNUWMul(CollapsedTripCount, OrigTripCount);5718  }5719 5720  // Create the collapsed loop control flow.5721  CanonicalLoopInfo *Result =5722      createLoopSkeleton(DL, CollapsedTripCount, F,5723                         OrigPreheader->getNextNode(), OrigAfter, "collapsed");5724 5725  // Build the collapsed loop body code.5726  // Start with deriving the input loop induction variables from the collapsed5727  // one, using a divmod scheme. To preserve the original loops' order, the5728  // innermost loop use the least significant bits.5729  Builder.restoreIP(Result->getBodyIP());5730 5731  Value *Leftover = Result->getIndVar();5732  SmallVector<Value *> NewIndVars;5733  NewIndVars.resize(NumLoops);5734  for (int i = NumLoops - 1; i >= 1; --i) {5735    Value *OrigTripCount = Loops[i]->getTripCount();5736 5737    Value *NewIndVar = Builder.CreateURem(Leftover, OrigTripCount);5738    NewIndVars[i] = NewIndVar;5739 5740    Leftover = Builder.CreateUDiv(Leftover, OrigTripCount);5741  }5742  // Outermost loop gets all the remaining bits.5743  NewIndVars[0] = Leftover;5744 5745  // Construct the loop body control flow.5746  // We progressively construct the branch structure following in direction of5747  // the control flow, from the leading in-between code, the loop nest body, the5748  // trailing in-between code, and rejoining the collapsed loop's latch.5749  // ContinueBlock and ContinuePred keep track of the source(s) of next edge. If5750  // the ContinueBlock is set, continue with that block. If ContinuePred, use5751  // its predecessors as sources.5752  BasicBlock *ContinueBlock = Result->getBody();5753  BasicBlock *ContinuePred = nullptr;5754  auto ContinueWith = [&ContinueBlock, &ContinuePred, DL](BasicBlock *Dest,5755                                                          BasicBlock *NextSrc) {5756    if (ContinueBlock)5757      redirectTo(ContinueBlock, Dest, DL);5758    else5759      redirectAllPredecessorsTo(ContinuePred, Dest, DL);5760 5761    ContinueBlock = nullptr;5762    ContinuePred = NextSrc;5763  };5764 5765  // The code before the nested loop of each level.5766  // Because we are sinking it into the nest, it will be executed more often5767  // that the original loop. More sophisticated schemes could keep track of what5768  // the in-between code is and instantiate it only once per thread.5769  for (size_t i = 0; i < NumLoops - 1; ++i)5770    ContinueWith(Loops[i]->getBody(), Loops[i + 1]->getHeader());5771 5772  // Connect the loop nest body.5773  ContinueWith(Innermost->getBody(), Innermost->getLatch());5774 5775  // The code after the nested loop at each level.5776  for (size_t i = NumLoops - 1; i > 0; --i)5777    ContinueWith(Loops[i]->getAfter(), Loops[i - 1]->getLatch());5778 5779  // Connect the finished loop to the collapsed loop latch.5780  ContinueWith(Result->getLatch(), nullptr);5781 5782  // Replace the input loops with the new collapsed loop.5783  redirectTo(Outermost->getPreheader(), Result->getPreheader(), DL);5784  redirectTo(Result->getAfter(), Outermost->getAfter(), DL);5785 5786  // Replace the input loop indvars with the derived ones.5787  for (size_t i = 0; i < NumLoops; ++i)5788    Loops[i]->getIndVar()->replaceAllUsesWith(NewIndVars[i]);5789 5790  // Remove unused parts of the input loops.5791  removeUnusedBlocksFromParent(OldControlBBs);5792 5793  for (CanonicalLoopInfo *L : Loops)5794    L->invalidate();5795 5796#ifndef NDEBUG5797  Result->assertOK();5798#endif5799  return Result;5800}5801 5802std::vector<CanonicalLoopInfo *>5803OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,5804                           ArrayRef<Value *> TileSizes) {5805  assert(TileSizes.size() == Loops.size() &&5806         "Must pass as many tile sizes as there are loops");5807  int NumLoops = Loops.size();5808  assert(NumLoops >= 1 && "At least one loop to tile required");5809 5810  CanonicalLoopInfo *OutermostLoop = Loops.front();5811  CanonicalLoopInfo *InnermostLoop = Loops.back();5812  Function *F = OutermostLoop->getBody()->getParent();5813  BasicBlock *InnerEnter = InnermostLoop->getBody();5814  BasicBlock *InnerLatch = InnermostLoop->getLatch();5815 5816  // Loop control blocks that may become orphaned later.5817  SmallVector<BasicBlock *, 12> OldControlBBs;5818  OldControlBBs.reserve(6 * Loops.size());5819  for (CanonicalLoopInfo *Loop : Loops)5820    Loop->collectControlBlocks(OldControlBBs);5821 5822  // Collect original trip counts and induction variable to be accessible by5823  // index. Also, the structure of the original loops is not preserved during5824  // the construction of the tiled loops, so do it before we scavenge the BBs of5825  // any original CanonicalLoopInfo.5826  SmallVector<Value *, 4> OrigTripCounts, OrigIndVars;5827  for (CanonicalLoopInfo *L : Loops) {5828    assert(L->isValid() && "All input loops must be valid canonical loops");5829    OrigTripCounts.push_back(L->getTripCount());5830    OrigIndVars.push_back(L->getIndVar());5831  }5832 5833  // Collect the code between loop headers. These may contain SSA definitions5834  // that are used in the loop nest body. To be usable with in the innermost5835  // body, these BasicBlocks will be sunk into the loop nest body. That is,5836  // these instructions may be executed more often than before the tiling.5837  // TODO: It would be sufficient to only sink them into body of the5838  // corresponding tile loop.5839  SmallVector<std::pair<BasicBlock *, BasicBlock *>, 4> InbetweenCode;5840  for (int i = 0; i < NumLoops - 1; ++i) {5841    CanonicalLoopInfo *Surrounding = Loops[i];5842    CanonicalLoopInfo *Nested = Loops[i + 1];5843 5844    BasicBlock *EnterBB = Surrounding->getBody();5845    BasicBlock *ExitBB = Nested->getHeader();5846    InbetweenCode.emplace_back(EnterBB, ExitBB);5847  }5848 5849  // Compute the trip counts of the floor loops.5850  Builder.SetCurrentDebugLocation(DL);5851  Builder.restoreIP(OutermostLoop->getPreheaderIP());5852  SmallVector<Value *, 4> FloorCompleteCount, FloorCount, FloorRems;5853  for (int i = 0; i < NumLoops; ++i) {5854    Value *TileSize = TileSizes[i];5855    Value *OrigTripCount = OrigTripCounts[i];5856    Type *IVType = OrigTripCount->getType();5857 5858    Value *FloorCompleteTripCount = Builder.CreateUDiv(OrigTripCount, TileSize);5859    Value *FloorTripRem = Builder.CreateURem(OrigTripCount, TileSize);5860 5861    // 0 if tripcount divides the tilesize, 1 otherwise.5862    // 1 means we need an additional iteration for a partial tile.5863    //5864    // Unfortunately we cannot just use the roundup-formula5865    //   (tripcount + tilesize - 1)/tilesize5866    // because the summation might overflow. We do not want introduce undefined5867    // behavior when the untiled loop nest did not.5868    Value *FloorTripOverflow =5869        Builder.CreateICmpNE(FloorTripRem, ConstantInt::get(IVType, 0));5870 5871    FloorTripOverflow = Builder.CreateZExt(FloorTripOverflow, IVType);5872    Value *FloorTripCount =5873        Builder.CreateAdd(FloorCompleteTripCount, FloorTripOverflow,5874                          "omp_floor" + Twine(i) + ".tripcount", true);5875 5876    // Remember some values for later use.5877    FloorCompleteCount.push_back(FloorCompleteTripCount);5878    FloorCount.push_back(FloorTripCount);5879    FloorRems.push_back(FloorTripRem);5880  }5881 5882  // Generate the new loop nest, from the outermost to the innermost.5883  std::vector<CanonicalLoopInfo *> Result;5884  Result.reserve(NumLoops * 2);5885 5886  // The basic block of the surrounding loop that enters the nest generated5887  // loop.5888  BasicBlock *Enter = OutermostLoop->getPreheader();5889 5890  // The basic block of the surrounding loop where the inner code should5891  // continue.5892  BasicBlock *Continue = OutermostLoop->getAfter();5893 5894  // Where the next loop basic block should be inserted.5895  BasicBlock *OutroInsertBefore = InnermostLoop->getExit();5896 5897  auto EmbeddNewLoop =5898      [this, DL, F, InnerEnter, &Enter, &Continue, &OutroInsertBefore](5899          Value *TripCount, const Twine &Name) -> CanonicalLoopInfo * {5900    CanonicalLoopInfo *EmbeddedLoop = createLoopSkeleton(5901        DL, TripCount, F, InnerEnter, OutroInsertBefore, Name);5902    redirectTo(Enter, EmbeddedLoop->getPreheader(), DL);5903    redirectTo(EmbeddedLoop->getAfter(), Continue, DL);5904 5905    // Setup the position where the next embedded loop connects to this loop.5906    Enter = EmbeddedLoop->getBody();5907    Continue = EmbeddedLoop->getLatch();5908    OutroInsertBefore = EmbeddedLoop->getLatch();5909    return EmbeddedLoop;5910  };5911 5912  auto EmbeddNewLoops = [&Result, &EmbeddNewLoop](ArrayRef<Value *> TripCounts,5913                                                  const Twine &NameBase) {5914    for (auto P : enumerate(TripCounts)) {5915      CanonicalLoopInfo *EmbeddedLoop =5916          EmbeddNewLoop(P.value(), NameBase + Twine(P.index()));5917      Result.push_back(EmbeddedLoop);5918    }5919  };5920 5921  EmbeddNewLoops(FloorCount, "floor");5922 5923  // Within the innermost floor loop, emit the code that computes the tile5924  // sizes.5925  Builder.SetInsertPoint(Enter->getTerminator());5926  SmallVector<Value *, 4> TileCounts;5927  for (int i = 0; i < NumLoops; ++i) {5928    CanonicalLoopInfo *FloorLoop = Result[i];5929    Value *TileSize = TileSizes[i];5930 5931    Value *FloorIsEpilogue =5932        Builder.CreateICmpEQ(FloorLoop->getIndVar(), FloorCompleteCount[i]);5933    Value *TileTripCount =5934        Builder.CreateSelect(FloorIsEpilogue, FloorRems[i], TileSize);5935 5936    TileCounts.push_back(TileTripCount);5937  }5938 5939  // Create the tile loops.5940  EmbeddNewLoops(TileCounts, "tile");5941 5942  // Insert the inbetween code into the body.5943  BasicBlock *BodyEnter = Enter;5944  BasicBlock *BodyEntered = nullptr;5945  for (std::pair<BasicBlock *, BasicBlock *> P : InbetweenCode) {5946    BasicBlock *EnterBB = P.first;5947    BasicBlock *ExitBB = P.second;5948 5949    if (BodyEnter)5950      redirectTo(BodyEnter, EnterBB, DL);5951    else5952      redirectAllPredecessorsTo(BodyEntered, EnterBB, DL);5953 5954    BodyEnter = nullptr;5955    BodyEntered = ExitBB;5956  }5957 5958  // Append the original loop nest body into the generated loop nest body.5959  if (BodyEnter)5960    redirectTo(BodyEnter, InnerEnter, DL);5961  else5962    redirectAllPredecessorsTo(BodyEntered, InnerEnter, DL);5963  redirectAllPredecessorsTo(InnerLatch, Continue, DL);5964 5965  // Replace the original induction variable with an induction variable computed5966  // from the tile and floor induction variables.5967  Builder.restoreIP(Result.back()->getBodyIP());5968  for (int i = 0; i < NumLoops; ++i) {5969    CanonicalLoopInfo *FloorLoop = Result[i];5970    CanonicalLoopInfo *TileLoop = Result[NumLoops + i];5971    Value *OrigIndVar = OrigIndVars[i];5972    Value *Size = TileSizes[i];5973 5974    Value *Scale =5975        Builder.CreateMul(Size, FloorLoop->getIndVar(), {}, /*HasNUW=*/true);5976    Value *Shift =5977        Builder.CreateAdd(Scale, TileLoop->getIndVar(), {}, /*HasNUW=*/true);5978    OrigIndVar->replaceAllUsesWith(Shift);5979  }5980 5981  // Remove unused parts of the original loops.5982  removeUnusedBlocksFromParent(OldControlBBs);5983 5984  for (CanonicalLoopInfo *L : Loops)5985    L->invalidate();5986 5987#ifndef NDEBUG5988  for (CanonicalLoopInfo *GenL : Result)5989    GenL->assertOK();5990#endif5991  return Result;5992}5993 5994/// Attach metadata \p Properties to the basic block described by \p BB. If the5995/// basic block already has metadata, the basic block properties are appended.5996static void addBasicBlockMetadata(BasicBlock *BB,5997                                  ArrayRef<Metadata *> Properties) {5998  // Nothing to do if no property to attach.5999  if (Properties.empty())6000    return;6001 6002  LLVMContext &Ctx = BB->getContext();6003  SmallVector<Metadata *> NewProperties;6004  NewProperties.push_back(nullptr);6005 6006  // If the basic block already has metadata, prepend it to the new metadata.6007  MDNode *Existing = BB->getTerminator()->getMetadata(LLVMContext::MD_loop);6008  if (Existing)6009    append_range(NewProperties, drop_begin(Existing->operands(), 1));6010 6011  append_range(NewProperties, Properties);6012  MDNode *BasicBlockID = MDNode::getDistinct(Ctx, NewProperties);6013  BasicBlockID->replaceOperandWith(0, BasicBlockID);6014 6015  BB->getTerminator()->setMetadata(LLVMContext::MD_loop, BasicBlockID);6016}6017 6018/// Attach loop metadata \p Properties to the loop described by \p Loop. If the6019/// loop already has metadata, the loop properties are appended.6020static void addLoopMetadata(CanonicalLoopInfo *Loop,6021                            ArrayRef<Metadata *> Properties) {6022  assert(Loop->isValid() && "Expecting a valid CanonicalLoopInfo");6023 6024  // Attach metadata to the loop's latch6025  BasicBlock *Latch = Loop->getLatch();6026  assert(Latch && "A valid CanonicalLoopInfo must have a unique latch");6027  addBasicBlockMetadata(Latch, Properties);6028}6029 6030/// Attach llvm.access.group metadata to the memref instructions of \p Block6031static void addAccessGroupMetadata(BasicBlock *Block, MDNode *AccessGroup,6032                                   LoopInfo &LI) {6033  for (Instruction &I : *Block) {6034    if (I.mayReadOrWriteMemory()) {6035      // TODO: This instruction may already have access group from6036      // other pragmas e.g. #pragma clang loop vectorize.  Append6037      // so that the existing metadata is not overwritten.6038      I.setMetadata(LLVMContext::MD_access_group, AccessGroup);6039    }6040  }6041}6042 6043void OpenMPIRBuilder::unrollLoopFull(DebugLoc, CanonicalLoopInfo *Loop) {6044  LLVMContext &Ctx = Builder.getContext();6045  addLoopMetadata(6046      Loop, {MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.enable")),6047             MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full"))});6048}6049 6050void OpenMPIRBuilder::unrollLoopHeuristic(DebugLoc, CanonicalLoopInfo *Loop) {6051  LLVMContext &Ctx = Builder.getContext();6052  addLoopMetadata(6053      Loop, {6054                MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.enable")),6055            });6056}6057 6058void OpenMPIRBuilder::createIfVersion(CanonicalLoopInfo *CanonicalLoop,6059                                      Value *IfCond, ValueToValueMapTy &VMap,6060                                      LoopAnalysis &LIA, LoopInfo &LI, Loop *L,6061                                      const Twine &NamePrefix) {6062  Function *F = CanonicalLoop->getFunction();6063 6064  // We can't do6065  // if (cond) {6066  //   simd_loop;6067  // } else {6068  //   non_simd_loop;6069  // }6070  // because then the CanonicalLoopInfo would only point to one of the loops:6071  // leading to other constructs operating on the same loop to malfunction.6072  // Instead generate6073  // while (...) {6074  //   if (cond) {6075  //     simd_body;6076  //   } else {6077  //     not_simd_body;6078  //   }6079  // }6080  // At least for simple loops, LLVM seems able to hoist the if out of the loop6081  // body at -O36082 6083  // Define where if branch should be inserted6084  auto SplitBeforeIt = CanonicalLoop->getBody()->getFirstNonPHIIt();6085 6086  // Create additional blocks for the if statement6087  BasicBlock *Cond = SplitBeforeIt->getParent();6088  llvm::LLVMContext &C = Cond->getContext();6089  llvm::BasicBlock *ThenBlock = llvm::BasicBlock::Create(6090      C, NamePrefix + ".if.then", Cond->getParent(), Cond->getNextNode());6091  llvm::BasicBlock *ElseBlock = llvm::BasicBlock::Create(6092      C, NamePrefix + ".if.else", Cond->getParent(), CanonicalLoop->getExit());6093 6094  // Create if condition branch.6095  Builder.SetInsertPoint(SplitBeforeIt);6096  Instruction *BrInstr =6097      Builder.CreateCondBr(IfCond, ThenBlock, /*ifFalse*/ ElseBlock);6098  InsertPointTy IP{BrInstr->getParent(), ++BrInstr->getIterator()};6099  // Then block contains branch to omp loop body which needs to be vectorized6100  spliceBB(IP, ThenBlock, false, Builder.getCurrentDebugLocation());6101  ThenBlock->replaceSuccessorsPhiUsesWith(Cond, ThenBlock);6102 6103  Builder.SetInsertPoint(ElseBlock);6104 6105  // Clone loop for the else branch6106  SmallVector<BasicBlock *, 8> NewBlocks;6107 6108  SmallVector<BasicBlock *, 8> ExistingBlocks;6109  ExistingBlocks.reserve(L->getNumBlocks() + 1);6110  ExistingBlocks.push_back(ThenBlock);6111  ExistingBlocks.append(L->block_begin(), L->block_end());6112  // Cond is the block that has the if clause condition6113  // LoopCond is omp_loop.cond6114  // LoopHeader is omp_loop.header6115  BasicBlock *LoopCond = Cond->getUniquePredecessor();6116  BasicBlock *LoopHeader = LoopCond->getUniquePredecessor();6117  assert(LoopCond && LoopHeader && "Invalid loop structure");6118  for (BasicBlock *Block : ExistingBlocks) {6119    if (Block == L->getLoopPreheader() || Block == L->getLoopLatch() ||6120        Block == LoopHeader || Block == LoopCond || Block == Cond) {6121      continue;6122    }6123    BasicBlock *NewBB = CloneBasicBlock(Block, VMap, "", F);6124 6125    // fix name not to be omp.if.then6126    if (Block == ThenBlock)6127      NewBB->setName(NamePrefix + ".if.else");6128 6129    NewBB->moveBefore(CanonicalLoop->getExit());6130    VMap[Block] = NewBB;6131    NewBlocks.push_back(NewBB);6132  }6133  remapInstructionsInBlocks(NewBlocks, VMap);6134  Builder.CreateBr(NewBlocks.front());6135 6136  // The loop latch must have only one predecessor. Currently it is branched to6137  // from both the 'then' and 'else' branches.6138  L->getLoopLatch()->splitBasicBlock(6139      L->getLoopLatch()->begin(), NamePrefix + ".pre_latch", /*Before=*/true);6140 6141  // Ensure that the then block is added to the loop so we add the attributes in6142  // the next step6143  L->addBasicBlockToLoop(ThenBlock, LI);6144}6145 6146unsigned6147OpenMPIRBuilder::getOpenMPDefaultSimdAlign(const Triple &TargetTriple,6148                                           const StringMap<bool> &Features) {6149  if (TargetTriple.isX86()) {6150    if (Features.lookup("avx512f"))6151      return 512;6152    else if (Features.lookup("avx"))6153      return 256;6154    return 128;6155  }6156  if (TargetTriple.isPPC())6157    return 128;6158  if (TargetTriple.isWasm())6159    return 128;6160  return 0;6161}6162 6163void OpenMPIRBuilder::applySimd(CanonicalLoopInfo *CanonicalLoop,6164                                MapVector<Value *, Value *> AlignedVars,6165                                Value *IfCond, OrderKind Order,6166                                ConstantInt *Simdlen, ConstantInt *Safelen) {6167  LLVMContext &Ctx = Builder.getContext();6168 6169  Function *F = CanonicalLoop->getFunction();6170 6171  // TODO: We should not rely on pass manager. Currently we use pass manager6172  // only for getting llvm::Loop which corresponds to given CanonicalLoopInfo6173  // object. We should have a method  which returns all blocks between6174  // CanonicalLoopInfo::getHeader() and CanonicalLoopInfo::getAfter()6175  FunctionAnalysisManager FAM;6176  FAM.registerPass([]() { return DominatorTreeAnalysis(); });6177  FAM.registerPass([]() { return LoopAnalysis(); });6178  FAM.registerPass([]() { return PassInstrumentationAnalysis(); });6179 6180  LoopAnalysis LIA;6181  LoopInfo &&LI = LIA.run(*F, FAM);6182 6183  Loop *L = LI.getLoopFor(CanonicalLoop->getHeader());6184  if (AlignedVars.size()) {6185    InsertPointTy IP = Builder.saveIP();6186    for (auto &AlignedItem : AlignedVars) {6187      Value *AlignedPtr = AlignedItem.first;6188      Value *Alignment = AlignedItem.second;6189      Instruction *loadInst = dyn_cast<Instruction>(AlignedPtr);6190      Builder.SetInsertPoint(loadInst->getNextNode());6191      Builder.CreateAlignmentAssumption(F->getDataLayout(), AlignedPtr,6192                                        Alignment);6193    }6194    Builder.restoreIP(IP);6195  }6196 6197  if (IfCond) {6198    ValueToValueMapTy VMap;6199    createIfVersion(CanonicalLoop, IfCond, VMap, LIA, LI, L, "simd");6200  }6201 6202  SmallPtrSet<BasicBlock *, 8> Reachable;6203 6204  // Get the basic blocks from the loop in which memref instructions6205  // can be found.6206  // TODO: Generalize getting all blocks inside a CanonicalizeLoopInfo,6207  // preferably without running any passes.6208  for (BasicBlock *Block : L->getBlocks()) {6209    if (Block == CanonicalLoop->getCond() ||6210        Block == CanonicalLoop->getHeader())6211      continue;6212    Reachable.insert(Block);6213  }6214 6215  SmallVector<Metadata *> LoopMDList;6216 6217  // In presence of finite 'safelen', it may be unsafe to mark all6218  // the memory instructions parallel, because loop-carried6219  // dependences of 'safelen' iterations are possible.6220  // If clause order(concurrent) is specified then the memory instructions6221  // are marked parallel even if 'safelen' is finite.6222  if ((Safelen == nullptr) || (Order == OrderKind::OMP_ORDER_concurrent))6223    applyParallelAccessesMetadata(CanonicalLoop, Ctx, L, LI, LoopMDList);6224 6225  // FIXME: the IF clause shares a loop backedge for the SIMD and non-SIMD6226  // versions so we can't add the loop attributes in that case.6227  if (IfCond) {6228    // we can still add llvm.loop.parallel_access6229    addLoopMetadata(CanonicalLoop, LoopMDList);6230    return;6231  }6232 6233  // Use the above access group metadata to create loop level6234  // metadata, which should be distinct for each loop.6235  ConstantAsMetadata *BoolConst =6236      ConstantAsMetadata::get(ConstantInt::getTrue(Type::getInt1Ty(Ctx)));6237  LoopMDList.push_back(MDNode::get(6238      Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"), BoolConst}));6239 6240  if (Simdlen || Safelen) {6241    // If both simdlen and safelen clauses are specified, the value of the6242    // simdlen parameter must be less than or equal to the value of the safelen6243    // parameter. Therefore, use safelen only in the absence of simdlen.6244    ConstantInt *VectorizeWidth = Simdlen == nullptr ? Safelen : Simdlen;6245    LoopMDList.push_back(6246        MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.width"),6247                          ConstantAsMetadata::get(VectorizeWidth)}));6248  }6249 6250  addLoopMetadata(CanonicalLoop, LoopMDList);6251}6252 6253/// Create the TargetMachine object to query the backend for optimization6254/// preferences.6255///6256/// Ideally, this would be passed from the front-end to the OpenMPBuilder, but6257/// e.g. Clang does not pass it to its CodeGen layer and creates it only when6258/// needed for the LLVM pass pipline. We use some default options to avoid6259/// having to pass too many settings from the frontend that probably do not6260/// matter.6261///6262/// Currently, TargetMachine is only used sometimes by the unrollLoopPartial6263/// method. If we are going to use TargetMachine for more purposes, especially6264/// those that are sensitive to TargetOptions, RelocModel and CodeModel, it6265/// might become be worth requiring front-ends to pass on their TargetMachine,6266/// or at least cache it between methods. Note that while fontends such as Clang6267/// have just a single main TargetMachine per translation unit, "target-cpu" and6268/// "target-features" that determine the TargetMachine are per-function and can6269/// be overrided using __attribute__((target("OPTIONS"))).6270static std::unique_ptr<TargetMachine>6271createTargetMachine(Function *F, CodeGenOptLevel OptLevel) {6272  Module *M = F->getParent();6273 6274  StringRef CPU = F->getFnAttribute("target-cpu").getValueAsString();6275  StringRef Features = F->getFnAttribute("target-features").getValueAsString();6276  const llvm::Triple &Triple = M->getTargetTriple();6277 6278  std::string Error;6279  const llvm::Target *TheTarget = TargetRegistry::lookupTarget(Triple, Error);6280  if (!TheTarget)6281    return {};6282 6283  llvm::TargetOptions Options;6284  return std::unique_ptr<TargetMachine>(TheTarget->createTargetMachine(6285      Triple, CPU, Features, Options, /*RelocModel=*/std::nullopt,6286      /*CodeModel=*/std::nullopt, OptLevel));6287}6288 6289/// Heuristically determine the best-performant unroll factor for \p CLI. This6290/// depends on the target processor. We are re-using the same heuristics as the6291/// LoopUnrollPass.6292static int32_t computeHeuristicUnrollFactor(CanonicalLoopInfo *CLI) {6293  Function *F = CLI->getFunction();6294 6295  // Assume the user requests the most aggressive unrolling, even if the rest of6296  // the code is optimized using a lower setting.6297  CodeGenOptLevel OptLevel = CodeGenOptLevel::Aggressive;6298  std::unique_ptr<TargetMachine> TM = createTargetMachine(F, OptLevel);6299 6300  FunctionAnalysisManager FAM;6301  FAM.registerPass([]() { return TargetLibraryAnalysis(); });6302  FAM.registerPass([]() { return AssumptionAnalysis(); });6303  FAM.registerPass([]() { return DominatorTreeAnalysis(); });6304  FAM.registerPass([]() { return LoopAnalysis(); });6305  FAM.registerPass([]() { return ScalarEvolutionAnalysis(); });6306  FAM.registerPass([]() { return PassInstrumentationAnalysis(); });6307  TargetIRAnalysis TIRA;6308  if (TM)6309    TIRA = TargetIRAnalysis(6310        [&](const Function &F) { return TM->getTargetTransformInfo(F); });6311  FAM.registerPass([&]() { return TIRA; });6312 6313  TargetIRAnalysis::Result &&TTI = TIRA.run(*F, FAM);6314  ScalarEvolutionAnalysis SEA;6315  ScalarEvolution &&SE = SEA.run(*F, FAM);6316  DominatorTreeAnalysis DTA;6317  DominatorTree &&DT = DTA.run(*F, FAM);6318  LoopAnalysis LIA;6319  LoopInfo &&LI = LIA.run(*F, FAM);6320  AssumptionAnalysis ACT;6321  AssumptionCache &&AC = ACT.run(*F, FAM);6322  OptimizationRemarkEmitter ORE{F};6323 6324  Loop *L = LI.getLoopFor(CLI->getHeader());6325  assert(L && "Expecting CanonicalLoopInfo to be recognized as a loop");6326 6327  TargetTransformInfo::UnrollingPreferences UP = gatherUnrollingPreferences(6328      L, SE, TTI,6329      /*BlockFrequencyInfo=*/nullptr,6330      /*ProfileSummaryInfo=*/nullptr, ORE, static_cast<int>(OptLevel),6331      /*UserThreshold=*/std::nullopt,6332      /*UserCount=*/std::nullopt,6333      /*UserAllowPartial=*/true,6334      /*UserAllowRuntime=*/true,6335      /*UserUpperBound=*/std::nullopt,6336      /*UserFullUnrollMaxCount=*/std::nullopt);6337 6338  UP.Force = true;6339 6340  // Account for additional optimizations taking place before the LoopUnrollPass6341  // would unroll the loop.6342  UP.Threshold *= UnrollThresholdFactor;6343  UP.PartialThreshold *= UnrollThresholdFactor;6344 6345  // Use normal unroll factors even if the rest of the code is optimized for6346  // size.6347  UP.OptSizeThreshold = UP.Threshold;6348  UP.PartialOptSizeThreshold = UP.PartialThreshold;6349 6350  LLVM_DEBUG(dbgs() << "Unroll heuristic thresholds:\n"6351                    << "  Threshold=" << UP.Threshold << "\n"6352                    << "  PartialThreshold=" << UP.PartialThreshold << "\n"6353                    << "  OptSizeThreshold=" << UP.OptSizeThreshold << "\n"6354                    << "  PartialOptSizeThreshold="6355                    << UP.PartialOptSizeThreshold << "\n");6356 6357  // Disable peeling.6358  TargetTransformInfo::PeelingPreferences PP =6359      gatherPeelingPreferences(L, SE, TTI,6360                               /*UserAllowPeeling=*/false,6361                               /*UserAllowProfileBasedPeeling=*/false,6362                               /*UnrollingSpecficValues=*/false);6363 6364  SmallPtrSet<const Value *, 32> EphValues;6365  CodeMetrics::collectEphemeralValues(L, &AC, EphValues);6366 6367  // Assume that reads and writes to stack variables can be eliminated by6368  // Mem2Reg, SROA or LICM. That is, don't count them towards the loop body's6369  // size.6370  for (BasicBlock *BB : L->blocks()) {6371    for (Instruction &I : *BB) {6372      Value *Ptr;6373      if (auto *Load = dyn_cast<LoadInst>(&I)) {6374        Ptr = Load->getPointerOperand();6375      } else if (auto *Store = dyn_cast<StoreInst>(&I)) {6376        Ptr = Store->getPointerOperand();6377      } else6378        continue;6379 6380      Ptr = Ptr->stripPointerCasts();6381 6382      if (auto *Alloca = dyn_cast<AllocaInst>(Ptr)) {6383        if (Alloca->getParent() == &F->getEntryBlock())6384          EphValues.insert(&I);6385      }6386    }6387  }6388 6389  UnrollCostEstimator UCE(L, TTI, EphValues, UP.BEInsns);6390 6391  // Loop is not unrollable if the loop contains certain instructions.6392  if (!UCE.canUnroll()) {6393    LLVM_DEBUG(dbgs() << "Loop not considered unrollable\n");6394    return 1;6395  }6396 6397  LLVM_DEBUG(dbgs() << "Estimated loop size is " << UCE.getRolledLoopSize()6398                    << "\n");6399 6400  // TODO: Determine trip count of \p CLI if constant, computeUnrollCount might6401  // be able to use it.6402  int TripCount = 0;6403  int MaxTripCount = 0;6404  bool MaxOrZero = false;6405  unsigned TripMultiple = 0;6406 6407  bool UseUpperBound = false;6408  computeUnrollCount(L, TTI, DT, &LI, &AC, SE, EphValues, &ORE, TripCount,6409                     MaxTripCount, MaxOrZero, TripMultiple, UCE, UP, PP,6410                     UseUpperBound);6411  unsigned Factor = UP.Count;6412  LLVM_DEBUG(dbgs() << "Suggesting unroll factor of " << Factor << "\n");6413 6414  // This function returns 1 to signal to not unroll a loop.6415  if (Factor == 0)6416    return 1;6417  return Factor;6418}6419 6420void OpenMPIRBuilder::unrollLoopPartial(DebugLoc DL, CanonicalLoopInfo *Loop,6421                                        int32_t Factor,6422                                        CanonicalLoopInfo **UnrolledCLI) {6423  assert(Factor >= 0 && "Unroll factor must not be negative");6424 6425  Function *F = Loop->getFunction();6426  LLVMContext &Ctx = F->getContext();6427 6428  // If the unrolled loop is not used for another loop-associated directive, it6429  // is sufficient to add metadata for the LoopUnrollPass.6430  if (!UnrolledCLI) {6431    SmallVector<Metadata *, 2> LoopMetadata;6432    LoopMetadata.push_back(6433        MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.enable")));6434 6435    if (Factor >= 1) {6436      ConstantAsMetadata *FactorConst = ConstantAsMetadata::get(6437          ConstantInt::get(Type::getInt32Ty(Ctx), APInt(32, Factor)));6438      LoopMetadata.push_back(MDNode::get(6439          Ctx, {MDString::get(Ctx, "llvm.loop.unroll.count"), FactorConst}));6440    }6441 6442    addLoopMetadata(Loop, LoopMetadata);6443    return;6444  }6445 6446  // Heuristically determine the unroll factor.6447  if (Factor == 0)6448    Factor = computeHeuristicUnrollFactor(Loop);6449 6450  // No change required with unroll factor 1.6451  if (Factor == 1) {6452    *UnrolledCLI = Loop;6453    return;6454  }6455 6456  assert(Factor >= 2 &&6457         "unrolling only makes sense with a factor of 2 or larger");6458 6459  Type *IndVarTy = Loop->getIndVarType();6460 6461  // Apply partial unrolling by tiling the loop by the unroll-factor, then fully6462  // unroll the inner loop.6463  Value *FactorVal =6464      ConstantInt::get(IndVarTy, APInt(IndVarTy->getIntegerBitWidth(), Factor,6465                                       /*isSigned=*/false));6466  std::vector<CanonicalLoopInfo *> LoopNest =6467      tileLoops(DL, {Loop}, {FactorVal});6468  assert(LoopNest.size() == 2 && "Expect 2 loops after tiling");6469  *UnrolledCLI = LoopNest[0];6470  CanonicalLoopInfo *InnerLoop = LoopNest[1];6471 6472  // LoopUnrollPass can only fully unroll loops with constant trip count.6473  // Unroll by the unroll factor with a fallback epilog for the remainder6474  // iterations if necessary.6475  ConstantAsMetadata *FactorConst = ConstantAsMetadata::get(6476      ConstantInt::get(Type::getInt32Ty(Ctx), APInt(32, Factor)));6477  addLoopMetadata(6478      InnerLoop,6479      {MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.enable")),6480       MDNode::get(6481           Ctx, {MDString::get(Ctx, "llvm.loop.unroll.count"), FactorConst})});6482 6483#ifndef NDEBUG6484  (*UnrolledCLI)->assertOK();6485#endif6486}6487 6488OpenMPIRBuilder::InsertPointTy6489OpenMPIRBuilder::createCopyPrivate(const LocationDescription &Loc,6490                                   llvm::Value *BufSize, llvm::Value *CpyBuf,6491                                   llvm::Value *CpyFn, llvm::Value *DidIt) {6492  if (!updateToLocation(Loc))6493    return Loc.IP;6494 6495  uint32_t SrcLocStrSize;6496  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);6497  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);6498  Value *ThreadId = getOrCreateThreadID(Ident);6499 6500  llvm::Value *DidItLD = Builder.CreateLoad(Builder.getInt32Ty(), DidIt);6501 6502  Value *Args[] = {Ident, ThreadId, BufSize, CpyBuf, CpyFn, DidItLD};6503 6504  Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_copyprivate);6505  createRuntimeFunctionCall(Fn, Args);6506 6507  return Builder.saveIP();6508}6509 6510OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createSingle(6511    const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,6512    FinalizeCallbackTy FiniCB, bool IsNowait, ArrayRef<llvm::Value *> CPVars,6513    ArrayRef<llvm::Function *> CPFuncs) {6514 6515  if (!updateToLocation(Loc))6516    return Loc.IP;6517 6518  // If needed allocate and initialize `DidIt` with 0.6519  // DidIt: flag variable: 1=single thread; 0=not single thread.6520  llvm::Value *DidIt = nullptr;6521  if (!CPVars.empty()) {6522    DidIt = Builder.CreateAlloca(llvm::Type::getInt32Ty(Builder.getContext()));6523    Builder.CreateStore(Builder.getInt32(0), DidIt);6524  }6525 6526  Directive OMPD = Directive::OMPD_single;6527  uint32_t SrcLocStrSize;6528  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);6529  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);6530  Value *ThreadId = getOrCreateThreadID(Ident);6531  Value *Args[] = {Ident, ThreadId};6532 6533  Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_single);6534  Instruction *EntryCall = createRuntimeFunctionCall(EntryRTLFn, Args);6535 6536  Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_single);6537  Instruction *ExitCall = createRuntimeFunctionCall(ExitRTLFn, Args);6538 6539  auto FiniCBWrapper = [&](InsertPointTy IP) -> Error {6540    if (Error Err = FiniCB(IP))6541      return Err;6542 6543    // The thread that executes the single region must set `DidIt` to 1.6544    // This is used by __kmpc_copyprivate, to know if the caller is the6545    // single thread or not.6546    if (DidIt)6547      Builder.CreateStore(Builder.getInt32(1), DidIt);6548 6549    return Error::success();6550  };6551 6552  // generates the following:6553  // if (__kmpc_single()) {6554  //		.... single region ...6555  // 		__kmpc_end_single6556  // }6557  // __kmpc_copyprivate6558  // __kmpc_barrier6559 6560  InsertPointOrErrorTy AfterIP =6561      EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCBWrapper,6562                           /*Conditional*/ true,6563                           /*hasFinalize*/ true);6564  if (!AfterIP)6565    return AfterIP.takeError();6566 6567  if (DidIt) {6568    for (size_t I = 0, E = CPVars.size(); I < E; ++I)6569      // NOTE BufSize is currently unused, so just pass 0.6570      createCopyPrivate(LocationDescription(Builder.saveIP(), Loc.DL),6571                        /*BufSize=*/ConstantInt::get(Int64, 0), CPVars[I],6572                        CPFuncs[I], DidIt);6573    // NOTE __kmpc_copyprivate already inserts a barrier6574  } else if (!IsNowait) {6575    InsertPointOrErrorTy AfterIP =6576        createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),6577                      omp::Directive::OMPD_unknown, /* ForceSimpleCall */ false,6578                      /* CheckCancelFlag */ false);6579    if (!AfterIP)6580      return AfterIP.takeError();6581  }6582  return Builder.saveIP();6583}6584 6585OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createCritical(6586    const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,6587    FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst) {6588 6589  if (!updateToLocation(Loc))6590    return Loc.IP;6591 6592  Directive OMPD = Directive::OMPD_critical;6593  uint32_t SrcLocStrSize;6594  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);6595  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);6596  Value *ThreadId = getOrCreateThreadID(Ident);6597  Value *LockVar = getOMPCriticalRegionLock(CriticalName);6598  Value *Args[] = {Ident, ThreadId, LockVar};6599 6600  SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args), std::end(Args));6601  Function *RTFn = nullptr;6602  if (HintInst) {6603    // Add Hint to entry Args and create call6604    EnterArgs.push_back(HintInst);6605    RTFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_critical_with_hint);6606  } else {6607    RTFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_critical);6608  }6609  Instruction *EntryCall = createRuntimeFunctionCall(RTFn, EnterArgs);6610 6611  Function *ExitRTLFn =6612      getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_critical);6613  Instruction *ExitCall = createRuntimeFunctionCall(ExitRTLFn, Args);6614 6615  return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,6616                              /*Conditional*/ false, /*hasFinalize*/ true);6617}6618 6619OpenMPIRBuilder::InsertPointTy6620OpenMPIRBuilder::createOrderedDepend(const LocationDescription &Loc,6621                                     InsertPointTy AllocaIP, unsigned NumLoops,6622                                     ArrayRef<llvm::Value *> StoreValues,6623                                     const Twine &Name, bool IsDependSource) {6624  assert(6625      llvm::all_of(StoreValues,6626                   [](Value *SV) { return SV->getType()->isIntegerTy(64); }) &&6627      "OpenMP runtime requires depend vec with i64 type");6628 6629  if (!updateToLocation(Loc))6630    return Loc.IP;6631 6632  // Allocate space for vector and generate alloc instruction.6633  auto *ArrI64Ty = ArrayType::get(Int64, NumLoops);6634  Builder.restoreIP(AllocaIP);6635  AllocaInst *ArgsBase = Builder.CreateAlloca(ArrI64Ty, nullptr, Name);6636  ArgsBase->setAlignment(Align(8));6637  updateToLocation(Loc);6638 6639  // Store the index value with offset in depend vector.6640  for (unsigned I = 0; I < NumLoops; ++I) {6641    Value *DependAddrGEPIter = Builder.CreateInBoundsGEP(6642        ArrI64Ty, ArgsBase, {Builder.getInt64(0), Builder.getInt64(I)});6643    StoreInst *STInst = Builder.CreateStore(StoreValues[I], DependAddrGEPIter);6644    STInst->setAlignment(Align(8));6645  }6646 6647  Value *DependBaseAddrGEP = Builder.CreateInBoundsGEP(6648      ArrI64Ty, ArgsBase, {Builder.getInt64(0), Builder.getInt64(0)});6649 6650  uint32_t SrcLocStrSize;6651  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);6652  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);6653  Value *ThreadId = getOrCreateThreadID(Ident);6654  Value *Args[] = {Ident, ThreadId, DependBaseAddrGEP};6655 6656  Function *RTLFn = nullptr;6657  if (IsDependSource)6658    RTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_doacross_post);6659  else6660    RTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_doacross_wait);6661  createRuntimeFunctionCall(RTLFn, Args);6662 6663  return Builder.saveIP();6664}6665 6666OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createOrderedThreadsSimd(6667    const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,6668    FinalizeCallbackTy FiniCB, bool IsThreads) {6669  if (!updateToLocation(Loc))6670    return Loc.IP;6671 6672  Directive OMPD = Directive::OMPD_ordered;6673  Instruction *EntryCall = nullptr;6674  Instruction *ExitCall = nullptr;6675 6676  if (IsThreads) {6677    uint32_t SrcLocStrSize;6678    Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);6679    Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);6680    Value *ThreadId = getOrCreateThreadID(Ident);6681    Value *Args[] = {Ident, ThreadId};6682 6683    Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_ordered);6684    EntryCall = createRuntimeFunctionCall(EntryRTLFn, Args);6685 6686    Function *ExitRTLFn =6687        getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_ordered);6688    ExitCall = createRuntimeFunctionCall(ExitRTLFn, Args);6689  }6690 6691  return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,6692                              /*Conditional*/ false, /*hasFinalize*/ true);6693}6694 6695OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::EmitOMPInlinedRegion(6696    Directive OMPD, Instruction *EntryCall, Instruction *ExitCall,6697    BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool Conditional,6698    bool HasFinalize, bool IsCancellable) {6699 6700  if (HasFinalize)6701    FinalizationStack.push_back({FiniCB, OMPD, IsCancellable});6702 6703  // Create inlined region's entry and body blocks, in preparation6704  // for conditional creation6705  BasicBlock *EntryBB = Builder.GetInsertBlock();6706  Instruction *SplitPos = EntryBB->getTerminator();6707  if (!isa_and_nonnull<BranchInst>(SplitPos))6708    SplitPos = new UnreachableInst(Builder.getContext(), EntryBB);6709  BasicBlock *ExitBB = EntryBB->splitBasicBlock(SplitPos, "omp_region.end");6710  BasicBlock *FiniBB =6711      EntryBB->splitBasicBlock(EntryBB->getTerminator(), "omp_region.finalize");6712 6713  Builder.SetInsertPoint(EntryBB->getTerminator());6714  emitCommonDirectiveEntry(OMPD, EntryCall, ExitBB, Conditional);6715 6716  // generate body6717  if (Error Err = BodyGenCB(/* AllocaIP */ InsertPointTy(),6718                            /* CodeGenIP */ Builder.saveIP()))6719    return Err;6720 6721  // emit exit call and do any needed finalization.6722  auto FinIP = InsertPointTy(FiniBB, FiniBB->getFirstInsertionPt());6723  assert(FiniBB->getTerminator()->getNumSuccessors() == 1 &&6724         FiniBB->getTerminator()->getSuccessor(0) == ExitBB &&6725         "Unexpected control flow graph state!!");6726  InsertPointOrErrorTy AfterIP =6727      emitCommonDirectiveExit(OMPD, FinIP, ExitCall, HasFinalize);6728  if (!AfterIP)6729    return AfterIP.takeError();6730 6731  // If we are skipping the region of a non conditional, remove the exit6732  // block, and clear the builder's insertion point.6733  assert(SplitPos->getParent() == ExitBB &&6734         "Unexpected Insertion point location!");6735  auto merged = MergeBlockIntoPredecessor(ExitBB);6736  BasicBlock *ExitPredBB = SplitPos->getParent();6737  auto InsertBB = merged ? ExitPredBB : ExitBB;6738  if (!isa_and_nonnull<BranchInst>(SplitPos))6739    SplitPos->eraseFromParent();6740  Builder.SetInsertPoint(InsertBB);6741 6742  return Builder.saveIP();6743}6744 6745OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveEntry(6746    Directive OMPD, Value *EntryCall, BasicBlock *ExitBB, bool Conditional) {6747  // if nothing to do, Return current insertion point.6748  if (!Conditional || !EntryCall)6749    return Builder.saveIP();6750 6751  BasicBlock *EntryBB = Builder.GetInsertBlock();6752  Value *CallBool = Builder.CreateIsNotNull(EntryCall);6753  auto *ThenBB = BasicBlock::Create(M.getContext(), "omp_region.body");6754  auto *UI = new UnreachableInst(Builder.getContext(), ThenBB);6755 6756  // Emit thenBB and set the Builder's insertion point there for6757  // body generation next. Place the block after the current block.6758  Function *CurFn = EntryBB->getParent();6759  CurFn->insert(std::next(EntryBB->getIterator()), ThenBB);6760 6761  // Move Entry branch to end of ThenBB, and replace with conditional6762  // branch (If-stmt)6763  Instruction *EntryBBTI = EntryBB->getTerminator();6764  Builder.CreateCondBr(CallBool, ThenBB, ExitBB);6765  EntryBBTI->removeFromParent();6766  Builder.SetInsertPoint(UI);6767  Builder.Insert(EntryBBTI);6768  UI->eraseFromParent();6769  Builder.SetInsertPoint(ThenBB->getTerminator());6770 6771  // return an insertion point to ExitBB.6772  return IRBuilder<>::InsertPoint(ExitBB, ExitBB->getFirstInsertionPt());6773}6774 6775OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitCommonDirectiveExit(6776    omp::Directive OMPD, InsertPointTy FinIP, Instruction *ExitCall,6777    bool HasFinalize) {6778 6779  Builder.restoreIP(FinIP);6780 6781  // If there is finalization to do, emit it before the exit call6782  if (HasFinalize) {6783    assert(!FinalizationStack.empty() &&6784           "Unexpected finalization stack state!");6785 6786    FinalizationInfo Fi = FinalizationStack.pop_back_val();6787    assert(Fi.DK == OMPD && "Unexpected Directive for Finalization call!");6788 6789    if (Error Err = Fi.mergeFiniBB(Builder, FinIP.getBlock()))6790      return std::move(Err);6791 6792    // Exit condition: insertion point is before the terminator of the new Fini6793    // block6794    Builder.SetInsertPoint(FinIP.getBlock()->getTerminator());6795  }6796 6797  if (!ExitCall)6798    return Builder.saveIP();6799 6800  // place the Exitcall as last instruction before Finalization block terminator6801  ExitCall->removeFromParent();6802  Builder.Insert(ExitCall);6803 6804  return IRBuilder<>::InsertPoint(ExitCall->getParent(),6805                                  ExitCall->getIterator());6806}6807 6808OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createCopyinClauseBlocks(6809    InsertPointTy IP, Value *MasterAddr, Value *PrivateAddr,6810    llvm::IntegerType *IntPtrTy, bool BranchtoEnd) {6811  if (!IP.isSet())6812    return IP;6813 6814  IRBuilder<>::InsertPointGuard IPG(Builder);6815 6816  // creates the following CFG structure6817  //	   OMP_Entry : (MasterAddr != PrivateAddr)?6818  //       F     T6819  //       |      \6820  //       |     copin.not.master6821  //       |      /6822  //       v     /6823  //   copyin.not.master.end6824  //		     |6825  //         v6826  //   OMP.Entry.Next6827 6828  BasicBlock *OMP_Entry = IP.getBlock();6829  Function *CurFn = OMP_Entry->getParent();6830  BasicBlock *CopyBegin =6831      BasicBlock::Create(M.getContext(), "copyin.not.master", CurFn);6832  BasicBlock *CopyEnd = nullptr;6833 6834  // If entry block is terminated, split to preserve the branch to following6835  // basic block (i.e. OMP.Entry.Next), otherwise, leave everything as is.6836  if (isa_and_nonnull<BranchInst>(OMP_Entry->getTerminator())) {6837    CopyEnd = OMP_Entry->splitBasicBlock(OMP_Entry->getTerminator(),6838                                         "copyin.not.master.end");6839    OMP_Entry->getTerminator()->eraseFromParent();6840  } else {6841    CopyEnd =6842        BasicBlock::Create(M.getContext(), "copyin.not.master.end", CurFn);6843  }6844 6845  Builder.SetInsertPoint(OMP_Entry);6846  Value *MasterPtr = Builder.CreatePtrToInt(MasterAddr, IntPtrTy);6847  Value *PrivatePtr = Builder.CreatePtrToInt(PrivateAddr, IntPtrTy);6848  Value *cmp = Builder.CreateICmpNE(MasterPtr, PrivatePtr);6849  Builder.CreateCondBr(cmp, CopyBegin, CopyEnd);6850 6851  Builder.SetInsertPoint(CopyBegin);6852  if (BranchtoEnd)6853    Builder.SetInsertPoint(Builder.CreateBr(CopyEnd));6854 6855  return Builder.saveIP();6856}6857 6858CallInst *OpenMPIRBuilder::createOMPAlloc(const LocationDescription &Loc,6859                                          Value *Size, Value *Allocator,6860                                          std::string Name) {6861  IRBuilder<>::InsertPointGuard IPG(Builder);6862  updateToLocation(Loc);6863 6864  uint32_t SrcLocStrSize;6865  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);6866  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);6867  Value *ThreadId = getOrCreateThreadID(Ident);6868  Value *Args[] = {ThreadId, Size, Allocator};6869 6870  Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_alloc);6871 6872  return createRuntimeFunctionCall(Fn, Args, Name);6873}6874 6875CallInst *OpenMPIRBuilder::createOMPFree(const LocationDescription &Loc,6876                                         Value *Addr, Value *Allocator,6877                                         std::string Name) {6878  IRBuilder<>::InsertPointGuard IPG(Builder);6879  updateToLocation(Loc);6880 6881  uint32_t SrcLocStrSize;6882  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);6883  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);6884  Value *ThreadId = getOrCreateThreadID(Ident);6885  Value *Args[] = {ThreadId, Addr, Allocator};6886  Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_free);6887  return createRuntimeFunctionCall(Fn, Args, Name);6888}6889 6890CallInst *OpenMPIRBuilder::createOMPInteropInit(6891    const LocationDescription &Loc, Value *InteropVar,6892    omp::OMPInteropType InteropType, Value *Device, Value *NumDependences,6893    Value *DependenceAddress, bool HaveNowaitClause) {6894  IRBuilder<>::InsertPointGuard IPG(Builder);6895  updateToLocation(Loc);6896 6897  uint32_t SrcLocStrSize;6898  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);6899  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);6900  Value *ThreadId = getOrCreateThreadID(Ident);6901  if (Device == nullptr)6902    Device = Constant::getAllOnesValue(Int32);6903  Constant *InteropTypeVal = ConstantInt::get(Int32, (int)InteropType);6904  if (NumDependences == nullptr) {6905    NumDependences = ConstantInt::get(Int32, 0);6906    PointerType *PointerTypeVar = PointerType::getUnqual(M.getContext());6907    DependenceAddress = ConstantPointerNull::get(PointerTypeVar);6908  }6909  Value *HaveNowaitClauseVal = ConstantInt::get(Int32, HaveNowaitClause);6910  Value *Args[] = {6911      Ident,  ThreadId,       InteropVar,        InteropTypeVal,6912      Device, NumDependences, DependenceAddress, HaveNowaitClauseVal};6913 6914  Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___tgt_interop_init);6915 6916  return createRuntimeFunctionCall(Fn, Args);6917}6918 6919CallInst *OpenMPIRBuilder::createOMPInteropDestroy(6920    const LocationDescription &Loc, Value *InteropVar, Value *Device,6921    Value *NumDependences, Value *DependenceAddress, bool HaveNowaitClause) {6922  IRBuilder<>::InsertPointGuard IPG(Builder);6923  updateToLocation(Loc);6924 6925  uint32_t SrcLocStrSize;6926  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);6927  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);6928  Value *ThreadId = getOrCreateThreadID(Ident);6929  if (Device == nullptr)6930    Device = Constant::getAllOnesValue(Int32);6931  if (NumDependences == nullptr) {6932    NumDependences = ConstantInt::get(Int32, 0);6933    PointerType *PointerTypeVar = PointerType::getUnqual(M.getContext());6934    DependenceAddress = ConstantPointerNull::get(PointerTypeVar);6935  }6936  Value *HaveNowaitClauseVal = ConstantInt::get(Int32, HaveNowaitClause);6937  Value *Args[] = {6938      Ident,          ThreadId,          InteropVar,         Device,6939      NumDependences, DependenceAddress, HaveNowaitClauseVal};6940 6941  Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___tgt_interop_destroy);6942 6943  return createRuntimeFunctionCall(Fn, Args);6944}6945 6946CallInst *OpenMPIRBuilder::createOMPInteropUse(const LocationDescription &Loc,6947                                               Value *InteropVar, Value *Device,6948                                               Value *NumDependences,6949                                               Value *DependenceAddress,6950                                               bool HaveNowaitClause) {6951  IRBuilder<>::InsertPointGuard IPG(Builder);6952  updateToLocation(Loc);6953  uint32_t SrcLocStrSize;6954  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);6955  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);6956  Value *ThreadId = getOrCreateThreadID(Ident);6957  if (Device == nullptr)6958    Device = Constant::getAllOnesValue(Int32);6959  if (NumDependences == nullptr) {6960    NumDependences = ConstantInt::get(Int32, 0);6961    PointerType *PointerTypeVar = PointerType::getUnqual(M.getContext());6962    DependenceAddress = ConstantPointerNull::get(PointerTypeVar);6963  }6964  Value *HaveNowaitClauseVal = ConstantInt::get(Int32, HaveNowaitClause);6965  Value *Args[] = {6966      Ident,          ThreadId,          InteropVar,         Device,6967      NumDependences, DependenceAddress, HaveNowaitClauseVal};6968 6969  Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___tgt_interop_use);6970 6971  return createRuntimeFunctionCall(Fn, Args);6972}6973 6974CallInst *OpenMPIRBuilder::createCachedThreadPrivate(6975    const LocationDescription &Loc, llvm::Value *Pointer,6976    llvm::ConstantInt *Size, const llvm::Twine &Name) {6977  IRBuilder<>::InsertPointGuard IPG(Builder);6978  updateToLocation(Loc);6979 6980  uint32_t SrcLocStrSize;6981  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);6982  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);6983  Value *ThreadId = getOrCreateThreadID(Ident);6984  Constant *ThreadPrivateCache =6985      getOrCreateInternalVariable(Int8PtrPtr, Name.str());6986  llvm::Value *Args[] = {Ident, ThreadId, Pointer, Size, ThreadPrivateCache};6987 6988  Function *Fn =6989      getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_threadprivate_cached);6990 6991  return createRuntimeFunctionCall(Fn, Args);6992}6993 6994OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetInit(6995    const LocationDescription &Loc,6996    const llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs) {6997  assert(!Attrs.MaxThreads.empty() && !Attrs.MaxTeams.empty() &&6998         "expected num_threads and num_teams to be specified");6999 7000  if (!updateToLocation(Loc))7001    return Loc.IP;7002 7003  uint32_t SrcLocStrSize;7004  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);7005  Constant *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);7006  Constant *IsSPMDVal = ConstantInt::getSigned(Int8, Attrs.ExecFlags);7007  Constant *UseGenericStateMachineVal = ConstantInt::getSigned(7008      Int8, Attrs.ExecFlags != omp::OMP_TGT_EXEC_MODE_SPMD);7009  Constant *MayUseNestedParallelismVal = ConstantInt::getSigned(Int8, true);7010  Constant *DebugIndentionLevelVal = ConstantInt::getSigned(Int16, 0);7011 7012  Function *DebugKernelWrapper = Builder.GetInsertBlock()->getParent();7013  Function *Kernel = DebugKernelWrapper;7014 7015  // We need to strip the debug prefix to get the correct kernel name.7016  StringRef KernelName = Kernel->getName();7017  const std::string DebugPrefix = "_debug__";7018  if (KernelName.ends_with(DebugPrefix)) {7019    KernelName = KernelName.drop_back(DebugPrefix.length());7020    Kernel = M.getFunction(KernelName);7021    assert(Kernel && "Expected the real kernel to exist");7022  }7023 7024  // Manifest the launch configuration in the metadata matching the kernel7025  // environment.7026  if (Attrs.MinTeams > 1 || Attrs.MaxTeams.front() > 0)7027    writeTeamsForKernel(T, *Kernel, Attrs.MinTeams, Attrs.MaxTeams.front());7028 7029  // If MaxThreads not set, select the maximum between the default workgroup7030  // size and the MinThreads value.7031  int32_t MaxThreadsVal = Attrs.MaxThreads.front();7032  if (MaxThreadsVal < 0)7033    MaxThreadsVal = std::max(7034        int32_t(getGridValue(T, Kernel).GV_Default_WG_Size), Attrs.MinThreads);7035 7036  if (MaxThreadsVal > 0)7037    writeThreadBoundsForKernel(T, *Kernel, Attrs.MinThreads, MaxThreadsVal);7038 7039  Constant *MinThreads = ConstantInt::getSigned(Int32, Attrs.MinThreads);7040  Constant *MaxThreads = ConstantInt::getSigned(Int32, MaxThreadsVal);7041  Constant *MinTeams = ConstantInt::getSigned(Int32, Attrs.MinTeams);7042  Constant *MaxTeams = ConstantInt::getSigned(Int32, Attrs.MaxTeams.front());7043  Constant *ReductionDataSize =7044      ConstantInt::getSigned(Int32, Attrs.ReductionDataSize);7045  Constant *ReductionBufferLength =7046      ConstantInt::getSigned(Int32, Attrs.ReductionBufferLength);7047 7048  Function *Fn = getOrCreateRuntimeFunctionPtr(7049      omp::RuntimeFunction::OMPRTL___kmpc_target_init);7050  const DataLayout &DL = Fn->getDataLayout();7051 7052  Twine DynamicEnvironmentName = KernelName + "_dynamic_environment";7053  Constant *DynamicEnvironmentInitializer =7054      ConstantStruct::get(DynamicEnvironment, {DebugIndentionLevelVal});7055  GlobalVariable *DynamicEnvironmentGV = new GlobalVariable(7056      M, DynamicEnvironment, /*IsConstant=*/false, GlobalValue::WeakODRLinkage,7057      DynamicEnvironmentInitializer, DynamicEnvironmentName,7058      /*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal,7059      DL.getDefaultGlobalsAddressSpace());7060  DynamicEnvironmentGV->setVisibility(GlobalValue::ProtectedVisibility);7061 7062  Constant *DynamicEnvironment =7063      DynamicEnvironmentGV->getType() == DynamicEnvironmentPtr7064          ? DynamicEnvironmentGV7065          : ConstantExpr::getAddrSpaceCast(DynamicEnvironmentGV,7066                                           DynamicEnvironmentPtr);7067 7068  Constant *ConfigurationEnvironmentInitializer = ConstantStruct::get(7069      ConfigurationEnvironment, {7070                                    UseGenericStateMachineVal,7071                                    MayUseNestedParallelismVal,7072                                    IsSPMDVal,7073                                    MinThreads,7074                                    MaxThreads,7075                                    MinTeams,7076                                    MaxTeams,7077                                    ReductionDataSize,7078                                    ReductionBufferLength,7079                                });7080  Constant *KernelEnvironmentInitializer = ConstantStruct::get(7081      KernelEnvironment, {7082                             ConfigurationEnvironmentInitializer,7083                             Ident,7084                             DynamicEnvironment,7085                         });7086  std::string KernelEnvironmentName =7087      (KernelName + "_kernel_environment").str();7088  GlobalVariable *KernelEnvironmentGV = new GlobalVariable(7089      M, KernelEnvironment, /*IsConstant=*/true, GlobalValue::WeakODRLinkage,7090      KernelEnvironmentInitializer, KernelEnvironmentName,7091      /*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal,7092      DL.getDefaultGlobalsAddressSpace());7093  KernelEnvironmentGV->setVisibility(GlobalValue::ProtectedVisibility);7094 7095  Constant *KernelEnvironment =7096      KernelEnvironmentGV->getType() == KernelEnvironmentPtr7097          ? KernelEnvironmentGV7098          : ConstantExpr::getAddrSpaceCast(KernelEnvironmentGV,7099                                           KernelEnvironmentPtr);7100  Value *KernelLaunchEnvironment = DebugKernelWrapper->getArg(0);7101  Type *KernelLaunchEnvParamTy = Fn->getFunctionType()->getParamType(1);7102  KernelLaunchEnvironment =7103      KernelLaunchEnvironment->getType() == KernelLaunchEnvParamTy7104          ? KernelLaunchEnvironment7105          : Builder.CreateAddrSpaceCast(KernelLaunchEnvironment,7106                                        KernelLaunchEnvParamTy);7107  CallInst *ThreadKind = createRuntimeFunctionCall(7108      Fn, {KernelEnvironment, KernelLaunchEnvironment});7109 7110  Value *ExecUserCode = Builder.CreateICmpEQ(7111      ThreadKind, Constant::getAllOnesValue(ThreadKind->getType()),7112      "exec_user_code");7113 7114  // ThreadKind = __kmpc_target_init(...)7115  // if (ThreadKind == -1)7116  //   user_code7117  // else7118  //   return;7119 7120  auto *UI = Builder.CreateUnreachable();7121  BasicBlock *CheckBB = UI->getParent();7122  BasicBlock *UserCodeEntryBB = CheckBB->splitBasicBlock(UI, "user_code.entry");7123 7124  BasicBlock *WorkerExitBB = BasicBlock::Create(7125      CheckBB->getContext(), "worker.exit", CheckBB->getParent());7126  Builder.SetInsertPoint(WorkerExitBB);7127  Builder.CreateRetVoid();7128 7129  auto *CheckBBTI = CheckBB->getTerminator();7130  Builder.SetInsertPoint(CheckBBTI);7131  Builder.CreateCondBr(ExecUserCode, UI->getParent(), WorkerExitBB);7132 7133  CheckBBTI->eraseFromParent();7134  UI->eraseFromParent();7135 7136  // Continue in the "user_code" block, see diagram above and in7137  // openmp/libomptarget/deviceRTLs/common/include/target.h .7138  return InsertPointTy(UserCodeEntryBB, UserCodeEntryBB->getFirstInsertionPt());7139}7140 7141void OpenMPIRBuilder::createTargetDeinit(const LocationDescription &Loc,7142                                         int32_t TeamsReductionDataSize,7143                                         int32_t TeamsReductionBufferLength) {7144  if (!updateToLocation(Loc))7145    return;7146 7147  Function *Fn = getOrCreateRuntimeFunctionPtr(7148      omp::RuntimeFunction::OMPRTL___kmpc_target_deinit);7149 7150  createRuntimeFunctionCall(Fn, {});7151 7152  if (!TeamsReductionBufferLength || !TeamsReductionDataSize)7153    return;7154 7155  Function *Kernel = Builder.GetInsertBlock()->getParent();7156  // We need to strip the debug prefix to get the correct kernel name.7157  StringRef KernelName = Kernel->getName();7158  const std::string DebugPrefix = "_debug__";7159  if (KernelName.ends_with(DebugPrefix))7160    KernelName = KernelName.drop_back(DebugPrefix.length());7161  auto *KernelEnvironmentGV =7162      M.getNamedGlobal((KernelName + "_kernel_environment").str());7163  assert(KernelEnvironmentGV && "Expected kernel environment global\n");7164  auto *KernelEnvironmentInitializer = KernelEnvironmentGV->getInitializer();7165  auto *NewInitializer = ConstantFoldInsertValueInstruction(7166      KernelEnvironmentInitializer,7167      ConstantInt::get(Int32, TeamsReductionDataSize), {0, 7});7168  NewInitializer = ConstantFoldInsertValueInstruction(7169      NewInitializer, ConstantInt::get(Int32, TeamsReductionBufferLength),7170      {0, 8});7171  KernelEnvironmentGV->setInitializer(NewInitializer);7172}7173 7174static void updateNVPTXAttr(Function &Kernel, StringRef Name, int32_t Value,7175                            bool Min) {7176  if (Kernel.hasFnAttribute(Name)) {7177    int32_t OldLimit = Kernel.getFnAttributeAsParsedInteger(Name);7178    Value = Min ? std::min(OldLimit, Value) : std::max(OldLimit, Value);7179  }7180  Kernel.addFnAttr(Name, llvm::utostr(Value));7181}7182 7183std::pair<int32_t, int32_t>7184OpenMPIRBuilder::readThreadBoundsForKernel(const Triple &T, Function &Kernel) {7185  int32_t ThreadLimit =7186      Kernel.getFnAttributeAsParsedInteger("omp_target_thread_limit");7187 7188  if (T.isAMDGPU()) {7189    const auto &Attr = Kernel.getFnAttribute("amdgpu-flat-work-group-size");7190    if (!Attr.isValid() || !Attr.isStringAttribute())7191      return {0, ThreadLimit};7192    auto [LBStr, UBStr] = Attr.getValueAsString().split(',');7193    int32_t LB, UB;7194    if (!llvm::to_integer(UBStr, UB, 10))7195      return {0, ThreadLimit};7196    UB = ThreadLimit ? std::min(ThreadLimit, UB) : UB;7197    if (!llvm::to_integer(LBStr, LB, 10))7198      return {0, UB};7199    return {LB, UB};7200  }7201 7202  if (Kernel.hasFnAttribute("nvvm.maxntid")) {7203    int32_t UB = Kernel.getFnAttributeAsParsedInteger("nvvm.maxntid");7204    return {0, ThreadLimit ? std::min(ThreadLimit, UB) : UB};7205  }7206  return {0, ThreadLimit};7207}7208 7209void OpenMPIRBuilder::writeThreadBoundsForKernel(const Triple &T,7210                                                 Function &Kernel, int32_t LB,7211                                                 int32_t UB) {7212  Kernel.addFnAttr("omp_target_thread_limit", std::to_string(UB));7213 7214  if (T.isAMDGPU()) {7215    Kernel.addFnAttr("amdgpu-flat-work-group-size",7216                     llvm::utostr(LB) + "," + llvm::utostr(UB));7217    return;7218  }7219 7220  updateNVPTXAttr(Kernel, "nvvm.maxntid", UB, true);7221}7222 7223std::pair<int32_t, int32_t>7224OpenMPIRBuilder::readTeamBoundsForKernel(const Triple &, Function &Kernel) {7225  // TODO: Read from backend annotations if available.7226  return {0, Kernel.getFnAttributeAsParsedInteger("omp_target_num_teams")};7227}7228 7229void OpenMPIRBuilder::writeTeamsForKernel(const Triple &T, Function &Kernel,7230                                          int32_t LB, int32_t UB) {7231  if (T.isNVPTX())7232    if (UB > 0)7233      Kernel.addFnAttr("nvvm.maxclusterrank", llvm::utostr(UB));7234  if (T.isAMDGPU())7235    Kernel.addFnAttr("amdgpu-max-num-workgroups", llvm::utostr(LB) + ",1,1");7236 7237  Kernel.addFnAttr("omp_target_num_teams", std::to_string(LB));7238}7239 7240void OpenMPIRBuilder::setOutlinedTargetRegionFunctionAttributes(7241    Function *OutlinedFn) {7242  if (Config.isTargetDevice()) {7243    OutlinedFn->setLinkage(GlobalValue::WeakODRLinkage);7244    // TODO: Determine if DSO local can be set to true.7245    OutlinedFn->setDSOLocal(false);7246    OutlinedFn->setVisibility(GlobalValue::ProtectedVisibility);7247    if (T.isAMDGCN())7248      OutlinedFn->setCallingConv(CallingConv::AMDGPU_KERNEL);7249    else if (T.isNVPTX())7250      OutlinedFn->setCallingConv(CallingConv::PTX_Kernel);7251    else if (T.isSPIRV())7252      OutlinedFn->setCallingConv(CallingConv::SPIR_KERNEL);7253  }7254}7255 7256Constant *OpenMPIRBuilder::createOutlinedFunctionID(Function *OutlinedFn,7257                                                    StringRef EntryFnIDName) {7258  if (Config.isTargetDevice()) {7259    assert(OutlinedFn && "The outlined function must exist if embedded");7260    return OutlinedFn;7261  }7262 7263  return new GlobalVariable(7264      M, Builder.getInt8Ty(), /*isConstant=*/true, GlobalValue::WeakAnyLinkage,7265      Constant::getNullValue(Builder.getInt8Ty()), EntryFnIDName);7266}7267 7268Constant *OpenMPIRBuilder::createTargetRegionEntryAddr(Function *OutlinedFn,7269                                                       StringRef EntryFnName) {7270  if (OutlinedFn)7271    return OutlinedFn;7272 7273  assert(!M.getGlobalVariable(EntryFnName, true) &&7274         "Named kernel already exists?");7275  return new GlobalVariable(7276      M, Builder.getInt8Ty(), /*isConstant=*/true, GlobalValue::InternalLinkage,7277      Constant::getNullValue(Builder.getInt8Ty()), EntryFnName);7278}7279 7280Error OpenMPIRBuilder::emitTargetRegionFunction(7281    TargetRegionEntryInfo &EntryInfo,7282    FunctionGenCallback &GenerateFunctionCallback, bool IsOffloadEntry,7283    Function *&OutlinedFn, Constant *&OutlinedFnID) {7284 7285  SmallString<64> EntryFnName;7286  OffloadInfoManager.getTargetRegionEntryFnName(EntryFnName, EntryInfo);7287 7288  if (Config.isTargetDevice() || !Config.openMPOffloadMandatory()) {7289    Expected<Function *> CBResult = GenerateFunctionCallback(EntryFnName);7290    if (!CBResult)7291      return CBResult.takeError();7292    OutlinedFn = *CBResult;7293  } else {7294    OutlinedFn = nullptr;7295  }7296 7297  // If this target outline function is not an offload entry, we don't need to7298  // register it. This may be in the case of a false if clause, or if there are7299  // no OpenMP targets.7300  if (!IsOffloadEntry)7301    return Error::success();7302 7303  std::string EntryFnIDName =7304      Config.isTargetDevice()7305          ? std::string(EntryFnName)7306          : createPlatformSpecificName({EntryFnName, "region_id"});7307 7308  OutlinedFnID = registerTargetRegionFunction(EntryInfo, OutlinedFn,7309                                              EntryFnName, EntryFnIDName);7310  return Error::success();7311}7312 7313Constant *OpenMPIRBuilder::registerTargetRegionFunction(7314    TargetRegionEntryInfo &EntryInfo, Function *OutlinedFn,7315    StringRef EntryFnName, StringRef EntryFnIDName) {7316  if (OutlinedFn)7317    setOutlinedTargetRegionFunctionAttributes(OutlinedFn);7318  auto OutlinedFnID = createOutlinedFunctionID(OutlinedFn, EntryFnIDName);7319  auto EntryAddr = createTargetRegionEntryAddr(OutlinedFn, EntryFnName);7320  OffloadInfoManager.registerTargetRegionEntryInfo(7321      EntryInfo, EntryAddr, OutlinedFnID,7322      OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion);7323  return OutlinedFnID;7324}7325 7326OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTargetData(7327    const LocationDescription &Loc, InsertPointTy AllocaIP,7328    InsertPointTy CodeGenIP, Value *DeviceID, Value *IfCond,7329    TargetDataInfo &Info, GenMapInfoCallbackTy GenMapInfoCB,7330    CustomMapperCallbackTy CustomMapperCB, omp::RuntimeFunction *MapperFunc,7331    function_ref<InsertPointOrErrorTy(InsertPointTy CodeGenIP,7332                                      BodyGenTy BodyGenType)>7333        BodyGenCB,7334    function_ref<void(unsigned int, Value *)> DeviceAddrCB, Value *SrcLocInfo) {7335  if (!updateToLocation(Loc))7336    return InsertPointTy();7337 7338  Builder.restoreIP(CodeGenIP);7339  // Disable TargetData CodeGen on Device pass.7340  if (Config.IsTargetDevice.value_or(false)) {7341    if (BodyGenCB) {7342      InsertPointOrErrorTy AfterIP =7343          BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv);7344      if (!AfterIP)7345        return AfterIP.takeError();7346      Builder.restoreIP(*AfterIP);7347    }7348    return Builder.saveIP();7349  }7350 7351  bool IsStandAlone = !BodyGenCB;7352  MapInfosTy *MapInfo;7353  // Generate the code for the opening of the data environment. Capture all the7354  // arguments of the runtime call by reference because they are used in the7355  // closing of the region.7356  auto BeginThenGen = [&](InsertPointTy AllocaIP,7357                          InsertPointTy CodeGenIP) -> Error {7358    MapInfo = &GenMapInfoCB(Builder.saveIP());7359    if (Error Err = emitOffloadingArrays(7360            AllocaIP, Builder.saveIP(), *MapInfo, Info, CustomMapperCB,7361            /*IsNonContiguous=*/true, DeviceAddrCB))7362      return Err;7363 7364    TargetDataRTArgs RTArgs;7365    emitOffloadingArraysArgument(Builder, RTArgs, Info);7366 7367    // Emit the number of elements in the offloading arrays.7368    Value *PointerNum = Builder.getInt32(Info.NumberOfPtrs);7369 7370    // Source location for the ident struct7371    if (!SrcLocInfo) {7372      uint32_t SrcLocStrSize;7373      Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);7374      SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);7375    }7376 7377    SmallVector<llvm::Value *, 13> OffloadingArgs = {7378        SrcLocInfo,           DeviceID,7379        PointerNum,           RTArgs.BasePointersArray,7380        RTArgs.PointersArray, RTArgs.SizesArray,7381        RTArgs.MapTypesArray, RTArgs.MapNamesArray,7382        RTArgs.MappersArray};7383 7384    if (IsStandAlone) {7385      assert(MapperFunc && "MapperFunc missing for standalone target data");7386 7387      auto TaskBodyCB = [&](Value *, Value *,7388                            IRBuilderBase::InsertPoint) -> Error {7389        if (Info.HasNoWait) {7390          OffloadingArgs.append({llvm::Constant::getNullValue(Int32),7391                                 llvm::Constant::getNullValue(VoidPtr),7392                                 llvm::Constant::getNullValue(Int32),7393                                 llvm::Constant::getNullValue(VoidPtr)});7394        }7395 7396        createRuntimeFunctionCall(getOrCreateRuntimeFunctionPtr(*MapperFunc),7397                                  OffloadingArgs);7398 7399        if (Info.HasNoWait) {7400          BasicBlock *OffloadContBlock =7401              BasicBlock::Create(Builder.getContext(), "omp_offload.cont");7402          Function *CurFn = Builder.GetInsertBlock()->getParent();7403          emitBlock(OffloadContBlock, CurFn, /*IsFinished=*/true);7404          Builder.restoreIP(Builder.saveIP());7405        }7406        return Error::success();7407      };7408 7409      bool RequiresOuterTargetTask = Info.HasNoWait;7410      if (!RequiresOuterTargetTask)7411        cantFail(TaskBodyCB(/*DeviceID=*/nullptr, /*RTLoc=*/nullptr,7412                            /*TargetTaskAllocaIP=*/{}));7413      else7414        cantFail(emitTargetTask(TaskBodyCB, DeviceID, SrcLocInfo, AllocaIP,7415                                /*Dependencies=*/{}, RTArgs, Info.HasNoWait));7416    } else {7417      Function *BeginMapperFunc = getOrCreateRuntimeFunctionPtr(7418          omp::OMPRTL___tgt_target_data_begin_mapper);7419 7420      createRuntimeFunctionCall(BeginMapperFunc, OffloadingArgs);7421 7422      for (auto DeviceMap : Info.DevicePtrInfoMap) {7423        if (isa<AllocaInst>(DeviceMap.second.second)) {7424          auto *LI =7425              Builder.CreateLoad(Builder.getPtrTy(), DeviceMap.second.first);7426          Builder.CreateStore(LI, DeviceMap.second.second);7427        }7428      }7429 7430      // If device pointer privatization is required, emit the body of the7431      // region here. It will have to be duplicated: with and without7432      // privatization.7433      InsertPointOrErrorTy AfterIP =7434          BodyGenCB(Builder.saveIP(), BodyGenTy::Priv);7435      if (!AfterIP)7436        return AfterIP.takeError();7437      Builder.restoreIP(*AfterIP);7438    }7439    return Error::success();7440  };7441 7442  // If we need device pointer privatization, we need to emit the body of the7443  // region with no privatization in the 'else' branch of the conditional.7444  // Otherwise, we don't have to do anything.7445  auto BeginElseGen = [&](InsertPointTy AllocaIP,7446                          InsertPointTy CodeGenIP) -> Error {7447    InsertPointOrErrorTy AfterIP =7448        BodyGenCB(Builder.saveIP(), BodyGenTy::DupNoPriv);7449    if (!AfterIP)7450      return AfterIP.takeError();7451    Builder.restoreIP(*AfterIP);7452    return Error::success();7453  };7454 7455  // Generate code for the closing of the data region.7456  auto EndThenGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {7457    TargetDataRTArgs RTArgs;7458    Info.EmitDebug = !MapInfo->Names.empty();7459    emitOffloadingArraysArgument(Builder, RTArgs, Info, /*ForEndCall=*/true);7460 7461    // Emit the number of elements in the offloading arrays.7462    Value *PointerNum = Builder.getInt32(Info.NumberOfPtrs);7463 7464    // Source location for the ident struct7465    if (!SrcLocInfo) {7466      uint32_t SrcLocStrSize;7467      Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);7468      SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);7469    }7470 7471    Value *OffloadingArgs[] = {SrcLocInfo,           DeviceID,7472                               PointerNum,           RTArgs.BasePointersArray,7473                               RTArgs.PointersArray, RTArgs.SizesArray,7474                               RTArgs.MapTypesArray, RTArgs.MapNamesArray,7475                               RTArgs.MappersArray};7476    Function *EndMapperFunc =7477        getOrCreateRuntimeFunctionPtr(omp::OMPRTL___tgt_target_data_end_mapper);7478 7479    createRuntimeFunctionCall(EndMapperFunc, OffloadingArgs);7480    return Error::success();7481  };7482 7483  // We don't have to do anything to close the region if the if clause evaluates7484  // to false.7485  auto EndElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {7486    return Error::success();7487  };7488 7489  Error Err = [&]() -> Error {7490    if (BodyGenCB) {7491      Error Err = [&]() {7492        if (IfCond)7493          return emitIfClause(IfCond, BeginThenGen, BeginElseGen, AllocaIP);7494        return BeginThenGen(AllocaIP, Builder.saveIP());7495      }();7496 7497      if (Err)7498        return Err;7499 7500      // If we don't require privatization of device pointers, we emit the body7501      // in between the runtime calls. This avoids duplicating the body code.7502      InsertPointOrErrorTy AfterIP =7503          BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv);7504      if (!AfterIP)7505        return AfterIP.takeError();7506      restoreIPandDebugLoc(Builder, *AfterIP);7507 7508      if (IfCond)7509        return emitIfClause(IfCond, EndThenGen, EndElseGen, AllocaIP);7510      return EndThenGen(AllocaIP, Builder.saveIP());7511    }7512    if (IfCond)7513      return emitIfClause(IfCond, BeginThenGen, EndElseGen, AllocaIP);7514    return BeginThenGen(AllocaIP, Builder.saveIP());7515  }();7516 7517  if (Err)7518    return Err;7519 7520  return Builder.saveIP();7521}7522 7523FunctionCallee7524OpenMPIRBuilder::createForStaticInitFunction(unsigned IVSize, bool IVSigned,7525                                             bool IsGPUDistribute) {7526  assert((IVSize == 32 || IVSize == 64) &&7527         "IV size is not compatible with the omp runtime");7528  RuntimeFunction Name;7529  if (IsGPUDistribute)7530    Name = IVSize == 327531               ? (IVSigned ? omp::OMPRTL___kmpc_distribute_static_init_47532                           : omp::OMPRTL___kmpc_distribute_static_init_4u)7533               : (IVSigned ? omp::OMPRTL___kmpc_distribute_static_init_87534                           : omp::OMPRTL___kmpc_distribute_static_init_8u);7535  else7536    Name = IVSize == 32 ? (IVSigned ? omp::OMPRTL___kmpc_for_static_init_47537                                    : omp::OMPRTL___kmpc_for_static_init_4u)7538                        : (IVSigned ? omp::OMPRTL___kmpc_for_static_init_87539                                    : omp::OMPRTL___kmpc_for_static_init_8u);7540 7541  return getOrCreateRuntimeFunction(M, Name);7542}7543 7544FunctionCallee OpenMPIRBuilder::createDispatchInitFunction(unsigned IVSize,7545                                                           bool IVSigned) {7546  assert((IVSize == 32 || IVSize == 64) &&7547         "IV size is not compatible with the omp runtime");7548  RuntimeFunction Name = IVSize == 327549                             ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_init_47550                                         : omp::OMPRTL___kmpc_dispatch_init_4u)7551                             : (IVSigned ? omp::OMPRTL___kmpc_dispatch_init_87552                                         : omp::OMPRTL___kmpc_dispatch_init_8u);7553 7554  return getOrCreateRuntimeFunction(M, Name);7555}7556 7557FunctionCallee OpenMPIRBuilder::createDispatchNextFunction(unsigned IVSize,7558                                                           bool IVSigned) {7559  assert((IVSize == 32 || IVSize == 64) &&7560         "IV size is not compatible with the omp runtime");7561  RuntimeFunction Name = IVSize == 327562                             ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_next_47563                                         : omp::OMPRTL___kmpc_dispatch_next_4u)7564                             : (IVSigned ? omp::OMPRTL___kmpc_dispatch_next_87565                                         : omp::OMPRTL___kmpc_dispatch_next_8u);7566 7567  return getOrCreateRuntimeFunction(M, Name);7568}7569 7570FunctionCallee OpenMPIRBuilder::createDispatchFiniFunction(unsigned IVSize,7571                                                           bool IVSigned) {7572  assert((IVSize == 32 || IVSize == 64) &&7573         "IV size is not compatible with the omp runtime");7574  RuntimeFunction Name = IVSize == 327575                             ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_fini_47576                                         : omp::OMPRTL___kmpc_dispatch_fini_4u)7577                             : (IVSigned ? omp::OMPRTL___kmpc_dispatch_fini_87578                                         : omp::OMPRTL___kmpc_dispatch_fini_8u);7579 7580  return getOrCreateRuntimeFunction(M, Name);7581}7582 7583FunctionCallee OpenMPIRBuilder::createDispatchDeinitFunction() {7584  return getOrCreateRuntimeFunction(M, omp::OMPRTL___kmpc_dispatch_deinit);7585}7586 7587static void FixupDebugInfoForOutlinedFunction(7588    OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, Function *Func,7589    DenseMap<Value *, std::tuple<Value *, unsigned>> &ValueReplacementMap) {7590 7591  DISubprogram *NewSP = Func->getSubprogram();7592  if (!NewSP)7593    return;7594 7595  SmallDenseMap<DILocalVariable *, DILocalVariable *> RemappedVariables;7596 7597  auto GetUpdatedDIVariable = [&](DILocalVariable *OldVar, unsigned arg) {7598    DILocalVariable *&NewVar = RemappedVariables[OldVar];7599    // Only use cached variable if the arg number matches. This is important7600    // so that DIVariable created for privatized variables are not discarded.7601    if (NewVar && (arg == NewVar->getArg()))7602      return NewVar;7603 7604    NewVar = llvm::DILocalVariable::get(7605        Builder.getContext(), OldVar->getScope(), OldVar->getName(),7606        OldVar->getFile(), OldVar->getLine(), OldVar->getType(), arg,7607        OldVar->getFlags(), OldVar->getAlignInBits(), OldVar->getAnnotations());7608    return NewVar;7609  };7610 7611  auto UpdateDebugRecord = [&](auto *DR) {7612    DILocalVariable *OldVar = DR->getVariable();7613    unsigned ArgNo = 0;7614    for (auto Loc : DR->location_ops()) {7615      auto Iter = ValueReplacementMap.find(Loc);7616      if (Iter != ValueReplacementMap.end()) {7617        DR->replaceVariableLocationOp(Loc, std::get<0>(Iter->second));7618        ArgNo = std::get<1>(Iter->second) + 1;7619      }7620    }7621    if (ArgNo != 0)7622      DR->setVariable(GetUpdatedDIVariable(OldVar, ArgNo));7623  };7624 7625  // The location and scope of variable intrinsics and records still point to7626  // the parent function of the target region. Update them.7627  for (Instruction &I : instructions(Func)) {7628    assert(!isa<llvm::DbgVariableIntrinsic>(&I) &&7629           "Unexpected debug intrinsic");7630    for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))7631      UpdateDebugRecord(&DVR);7632  }7633  // An extra argument is passed to the device. Create the debug data for it.7634  if (OMPBuilder.Config.isTargetDevice()) {7635    DICompileUnit *CU = NewSP->getUnit();7636    Module *M = Func->getParent();7637    DIBuilder DB(*M, true, CU);7638    DIType *VoidPtrTy =7639        DB.createQualifiedType(dwarf::DW_TAG_pointer_type, nullptr);7640    DILocalVariable *Var = DB.createParameterVariable(7641        NewSP, "dyn_ptr", /*ArgNo*/ 1, NewSP->getFile(), /*LineNo=*/0,7642        VoidPtrTy, /*AlwaysPreserve=*/false, DINode::DIFlags::FlagArtificial);7643    auto Loc = DILocation::get(Func->getContext(), 0, 0, NewSP, 0);7644    DB.insertDeclare(&(*Func->arg_begin()), Var, DB.createExpression(), Loc,7645                     &(*Func->begin()));7646  }7647}7648 7649static Value *removeASCastIfPresent(Value *V) {7650  if (Operator::getOpcode(V) == Instruction::AddrSpaceCast)7651    return cast<Operator>(V)->getOperand(0);7652  return V;7653}7654 7655static Expected<Function *> createOutlinedFunction(7656    OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,7657    const OpenMPIRBuilder::TargetKernelDefaultAttrs &DefaultAttrs,7658    StringRef FuncName, SmallVectorImpl<Value *> &Inputs,7659    OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc,7660    OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy &ArgAccessorFuncCB) {7661  SmallVector<Type *> ParameterTypes;7662  if (OMPBuilder.Config.isTargetDevice()) {7663    // Add the "implicit" runtime argument we use to provide launch specific7664    // information for target devices.7665    auto *Int8PtrTy = PointerType::getUnqual(Builder.getContext());7666    ParameterTypes.push_back(Int8PtrTy);7667 7668    // All parameters to target devices are passed as pointers7669    // or i64. This assumes 64-bit address spaces/pointers.7670    for (auto &Arg : Inputs)7671      ParameterTypes.push_back(Arg->getType()->isPointerTy()7672                                   ? Arg->getType()7673                                   : Type::getInt64Ty(Builder.getContext()));7674  } else {7675    for (auto &Arg : Inputs)7676      ParameterTypes.push_back(Arg->getType());7677  }7678 7679  auto BB = Builder.GetInsertBlock();7680  auto M = BB->getModule();7681  auto FuncType = FunctionType::get(Builder.getVoidTy(), ParameterTypes,7682                                    /*isVarArg*/ false);7683  auto Func =7684      Function::Create(FuncType, GlobalValue::InternalLinkage, FuncName, M);7685 7686  // Forward target-cpu and target-features function attributes from the7687  // original function to the new outlined function.7688  Function *ParentFn = Builder.GetInsertBlock()->getParent();7689 7690  auto TargetCpuAttr = ParentFn->getFnAttribute("target-cpu");7691  if (TargetCpuAttr.isStringAttribute())7692    Func->addFnAttr(TargetCpuAttr);7693 7694  auto TargetFeaturesAttr = ParentFn->getFnAttribute("target-features");7695  if (TargetFeaturesAttr.isStringAttribute())7696    Func->addFnAttr(TargetFeaturesAttr);7697 7698  if (OMPBuilder.Config.isTargetDevice()) {7699    Value *ExecMode =7700        OMPBuilder.emitKernelExecutionMode(FuncName, DefaultAttrs.ExecFlags);7701    OMPBuilder.emitUsed("llvm.compiler.used", {ExecMode});7702  }7703 7704  // Save insert point.7705  IRBuilder<>::InsertPointGuard IPG(Builder);7706  // We will generate the entries in the outlined function but the debug7707  // location may still be pointing to the parent function. Reset it now.7708  Builder.SetCurrentDebugLocation(llvm::DebugLoc());7709 7710  // Generate the region into the function.7711  BasicBlock *EntryBB = BasicBlock::Create(Builder.getContext(), "entry", Func);7712  Builder.SetInsertPoint(EntryBB);7713 7714  // Insert target init call in the device compilation pass.7715  if (OMPBuilder.Config.isTargetDevice())7716    Builder.restoreIP(OMPBuilder.createTargetInit(Builder, DefaultAttrs));7717 7718  BasicBlock *UserCodeEntryBB = Builder.GetInsertBlock();7719 7720  // As we embed the user code in the middle of our target region after we7721  // generate entry code, we must move what allocas we can into the entry7722  // block to avoid possible breaking optimisations for device7723  if (OMPBuilder.Config.isTargetDevice())7724    OMPBuilder.ConstantAllocaRaiseCandidates.emplace_back(Func);7725 7726  // Insert target deinit call in the device compilation pass.7727  BasicBlock *OutlinedBodyBB =7728      splitBB(Builder, /*CreateBranch=*/true, "outlined.body");7729  llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = CBFunc(7730      Builder.saveIP(),7731      OpenMPIRBuilder::InsertPointTy(OutlinedBodyBB, OutlinedBodyBB->begin()));7732  if (!AfterIP)7733    return AfterIP.takeError();7734  Builder.restoreIP(*AfterIP);7735  if (OMPBuilder.Config.isTargetDevice())7736    OMPBuilder.createTargetDeinit(Builder);7737 7738  // Insert return instruction.7739  Builder.CreateRetVoid();7740 7741  // New Alloca IP at entry point of created device function.7742  Builder.SetInsertPoint(EntryBB->getFirstNonPHIIt());7743  auto AllocaIP = Builder.saveIP();7744 7745  Builder.SetInsertPoint(UserCodeEntryBB->getFirstNonPHIOrDbg());7746 7747  // Skip the artificial dyn_ptr on the device.7748  const auto &ArgRange =7749      OMPBuilder.Config.isTargetDevice()7750          ? make_range(Func->arg_begin() + 1, Func->arg_end())7751          : Func->args();7752 7753  DenseMap<Value *, std::tuple<Value *, unsigned>> ValueReplacementMap;7754 7755  auto ReplaceValue = [](Value *Input, Value *InputCopy, Function *Func) {7756    // Things like GEP's can come in the form of Constants. Constants and7757    // ConstantExpr's do not have access to the knowledge of what they're7758    // contained in, so we must dig a little to find an instruction so we7759    // can tell if they're used inside of the function we're outlining. We7760    // also replace the original constant expression with a new instruction7761    // equivalent; an instruction as it allows easy modification in the7762    // following loop, as we can now know the constant (instruction) is7763    // owned by our target function and replaceUsesOfWith can now be invoked7764    // on it (cannot do this with constants it seems). A brand new one also7765    // allows us to be cautious as it is perhaps possible the old expression7766    // was used inside of the function but exists and is used externally7767    // (unlikely by the nature of a Constant, but still).7768    // NOTE: We cannot remove dead constants that have been rewritten to7769    // instructions at this stage, we run the risk of breaking later lowering7770    // by doing so as we could still be in the process of lowering the module7771    // from MLIR to LLVM-IR and the MLIR lowering may still require the original7772    // constants we have created rewritten versions of.7773    if (auto *Const = dyn_cast<Constant>(Input))7774      convertUsersOfConstantsToInstructions(Const, Func, false);7775 7776    // Collect users before iterating over them to avoid invalidating the7777    // iteration in case a user uses Input more than once (e.g. a call7778    // instruction).7779    SetVector<User *> Users(Input->users().begin(), Input->users().end());7780    // Collect all the instructions7781    for (User *User : make_early_inc_range(Users))7782      if (auto *Instr = dyn_cast<Instruction>(User))7783        if (Instr->getFunction() == Func)7784          Instr->replaceUsesOfWith(Input, InputCopy);7785  };7786 7787  SmallVector<std::pair<Value *, Value *>> DeferredReplacement;7788 7789  // Rewrite uses of input valus to parameters.7790  for (auto InArg : zip(Inputs, ArgRange)) {7791    Value *Input = std::get<0>(InArg);7792    Argument &Arg = std::get<1>(InArg);7793    Value *InputCopy = nullptr;7794 7795    llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =7796        ArgAccessorFuncCB(Arg, Input, InputCopy, AllocaIP, Builder.saveIP());7797    if (!AfterIP)7798      return AfterIP.takeError();7799    Builder.restoreIP(*AfterIP);7800    ValueReplacementMap[Input] = std::make_tuple(InputCopy, Arg.getArgNo());7801 7802    // In certain cases a Global may be set up for replacement, however, this7803    // Global may be used in multiple arguments to the kernel, just segmented7804    // apart, for example, if we have a global array, that is sectioned into7805    // multiple mappings (technically not legal in OpenMP, but there is a case7806    // in Fortran for Common Blocks where this is neccesary), we will end up7807    // with GEP's into this array inside the kernel, that refer to the Global7808    // but are technically seperate arguments to the kernel for all intents and7809    // purposes. If we have mapped a segment that requires a GEP into the 0-th7810    // index, it will fold into an referal to the Global, if we then encounter7811    // this folded GEP during replacement all of the references to the7812    // Global in the kernel will be replaced with the argument we have generated7813    // that corresponds to it, including any other GEP's that refer to the7814    // Global that may be other arguments. This will invalidate all of the other7815    // preceding mapped arguments that refer to the same global that may be7816    // seperate segments. To prevent this, we defer global processing until all7817    // other processing has been performed.7818    if (llvm::isa<llvm::GlobalValue, llvm::GlobalObject, llvm::GlobalVariable>(7819            removeASCastIfPresent(Input))) {7820      DeferredReplacement.push_back(std::make_pair(Input, InputCopy));7821      continue;7822    }7823 7824    if (isa<ConstantData>(Input))7825      continue;7826 7827    ReplaceValue(Input, InputCopy, Func);7828  }7829 7830  // Replace all of our deferred Input values, currently just Globals.7831  for (auto Deferred : DeferredReplacement)7832    ReplaceValue(std::get<0>(Deferred), std::get<1>(Deferred), Func);7833 7834  FixupDebugInfoForOutlinedFunction(OMPBuilder, Builder, Func,7835                                    ValueReplacementMap);7836  return Func;7837}7838/// Given a task descriptor, TaskWithPrivates, return the pointer to the block7839/// of pointers containing shared data between the parent task and the created7840/// task.7841static LoadInst *loadSharedDataFromTaskDescriptor(OpenMPIRBuilder &OMPIRBuilder,7842                                                  IRBuilderBase &Builder,7843                                                  Value *TaskWithPrivates,7844                                                  Type *TaskWithPrivatesTy) {7845 7846  Type *TaskTy = OMPIRBuilder.Task;7847  LLVMContext &Ctx = Builder.getContext();7848  Value *TaskT =7849      Builder.CreateStructGEP(TaskWithPrivatesTy, TaskWithPrivates, 0);7850  Value *Shareds = TaskT;7851  // TaskWithPrivatesTy can be one of the following7852  // 1. %struct.task_with_privates = type { %struct.kmp_task_ompbuilder_t,7853  //                                        %struct.privates }7854  // 2. %struct.kmp_task_ompbuilder_t ;; This is simply TaskTy7855  //7856  // In the former case, that is when  TaskWithPrivatesTy != TaskTy,7857  // its first member has to be the task descriptor. TaskTy is the type of the7858  // task descriptor. TaskT is the pointer to the task descriptor. Loading the7859  // first member of TaskT, gives us the pointer to shared data.7860  if (TaskWithPrivatesTy != TaskTy)7861    Shareds = Builder.CreateStructGEP(TaskTy, TaskT, 0);7862  return Builder.CreateLoad(PointerType::getUnqual(Ctx), Shareds);7863}7864/// Create an entry point for a target task with the following.7865/// It'll have the following signature7866/// void @.omp_target_task_proxy_func(i32 %thread.id, ptr %task)7867/// This function is called from emitTargetTask once the7868/// code to launch the target kernel has been outlined already.7869/// NumOffloadingArrays is the number of offloading arrays that we need to copy7870/// into the task structure so that the deferred target task can access this7871/// data even after the stack frame of the generating task has been rolled7872/// back. Offloading arrays contain base pointers, pointers, sizes etc7873/// of the data that the target kernel will access. These in effect are the7874/// non-empty arrays of pointers held by OpenMPIRBuilder::TargetDataRTArgs.7875static Function *emitTargetTaskProxyFunction(7876    OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, CallInst *StaleCI,7877    StructType *PrivatesTy, StructType *TaskWithPrivatesTy,7878    const size_t NumOffloadingArrays, const int SharedArgsOperandNo) {7879 7880  // If NumOffloadingArrays is non-zero, PrivatesTy better not be nullptr.7881  // This is because PrivatesTy is the type of the structure in which7882  // we pass the offloading arrays to the deferred target task.7883  assert((!NumOffloadingArrays || PrivatesTy) &&7884         "PrivatesTy cannot be nullptr when there are offloadingArrays"7885         "to privatize");7886 7887  Module &M = OMPBuilder.M;7888  // KernelLaunchFunction is the target launch function, i.e.7889  // the function that sets up kernel arguments and calls7890  // __tgt_target_kernel to launch the kernel on the device.7891  //7892  Function *KernelLaunchFunction = StaleCI->getCalledFunction();7893 7894  // StaleCI is the CallInst which is the call to the outlined7895  // target kernel launch function. If there are local live-in values7896  // that the outlined function uses then these are aggregated into a structure7897  // which is passed as the second argument. If there are no local live-in7898  // values or if all values used by the outlined kernel are global variables,7899  // then there's only one argument, the threadID. So, StaleCI can be7900  //7901  // %structArg = alloca { ptr, ptr }, align 87902  // %gep_ = getelementptr { ptr, ptr }, ptr %structArg, i32 0, i32 07903  // store ptr %20, ptr %gep_, align 87904  // %gep_8 = getelementptr { ptr, ptr }, ptr %structArg, i32 0, i32 17905  // store ptr %21, ptr %gep_8, align 87906  // call void @_QQmain..omp_par.1(i32 %global.tid.val6, ptr %structArg)7907  //7908  // OR7909  //7910  // call void @_QQmain..omp_par.1(i32 %global.tid.val6)7911  OpenMPIRBuilder::InsertPointTy IP(StaleCI->getParent(),7912                                    StaleCI->getIterator());7913 7914  LLVMContext &Ctx = StaleCI->getParent()->getContext();7915 7916  Type *ThreadIDTy = Type::getInt32Ty(Ctx);7917  Type *TaskPtrTy = OMPBuilder.TaskPtr;7918  [[maybe_unused]] Type *TaskTy = OMPBuilder.Task;7919 7920  auto ProxyFnTy =7921      FunctionType::get(Builder.getVoidTy(), {ThreadIDTy, TaskPtrTy},7922                        /* isVarArg */ false);7923  auto ProxyFn = Function::Create(ProxyFnTy, GlobalValue::InternalLinkage,7924                                  ".omp_target_task_proxy_func",7925                                  Builder.GetInsertBlock()->getModule());7926  Value *ThreadId = ProxyFn->getArg(0);7927  Value *TaskWithPrivates = ProxyFn->getArg(1);7928  ThreadId->setName("thread.id");7929  TaskWithPrivates->setName("task");7930 7931  bool HasShareds = SharedArgsOperandNo > 0;7932  bool HasOffloadingArrays = NumOffloadingArrays > 0;7933  BasicBlock *EntryBB =7934      BasicBlock::Create(Builder.getContext(), "entry", ProxyFn);7935  Builder.SetInsertPoint(EntryBB);7936 7937  SmallVector<Value *> KernelLaunchArgs;7938  KernelLaunchArgs.reserve(StaleCI->arg_size());7939  KernelLaunchArgs.push_back(ThreadId);7940 7941  if (HasOffloadingArrays) {7942    assert(TaskTy != TaskWithPrivatesTy &&7943           "If there are offloading arrays to pass to the target"7944           "TaskTy cannot be the same as TaskWithPrivatesTy");7945    (void)TaskTy;7946    Value *Privates =7947        Builder.CreateStructGEP(TaskWithPrivatesTy, TaskWithPrivates, 1);7948    for (unsigned int i = 0; i < NumOffloadingArrays; ++i)7949      KernelLaunchArgs.push_back(7950          Builder.CreateStructGEP(PrivatesTy, Privates, i));7951  }7952 7953  if (HasShareds) {7954    auto *ArgStructAlloca =7955        dyn_cast<AllocaInst>(StaleCI->getArgOperand(SharedArgsOperandNo));7956    assert(ArgStructAlloca &&7957           "Unable to find the alloca instruction corresponding to arguments "7958           "for extracted function");7959    auto *ArgStructType = cast<StructType>(ArgStructAlloca->getAllocatedType());7960 7961    AllocaInst *NewArgStructAlloca =7962        Builder.CreateAlloca(ArgStructType, nullptr, "structArg");7963 7964    Value *SharedsSize =7965        Builder.getInt64(M.getDataLayout().getTypeStoreSize(ArgStructType));7966 7967    LoadInst *LoadShared = loadSharedDataFromTaskDescriptor(7968        OMPBuilder, Builder, TaskWithPrivates, TaskWithPrivatesTy);7969 7970    Builder.CreateMemCpy(7971        NewArgStructAlloca, NewArgStructAlloca->getAlign(), LoadShared,7972        LoadShared->getPointerAlignment(M.getDataLayout()), SharedsSize);7973    KernelLaunchArgs.push_back(NewArgStructAlloca);7974  }7975  OMPBuilder.createRuntimeFunctionCall(KernelLaunchFunction, KernelLaunchArgs);7976  Builder.CreateRetVoid();7977  return ProxyFn;7978}7979static Type *getOffloadingArrayType(Value *V) {7980 7981  if (auto *GEP = dyn_cast<GetElementPtrInst>(V))7982    return GEP->getSourceElementType();7983  if (auto *Alloca = dyn_cast<AllocaInst>(V))7984    return Alloca->getAllocatedType();7985 7986  llvm_unreachable("Unhandled Instruction type");7987  return nullptr;7988}7989// This function returns a struct that has at most two members.7990// The first member is always %struct.kmp_task_ompbuilder_t, that is the task7991// descriptor. The second member, if needed, is a struct containing arrays7992// that need to be passed to the offloaded target kernel. For example,7993// if .offload_baseptrs, .offload_ptrs and .offload_sizes have to be passed to7994// the target kernel and their types are [3 x ptr], [3 x ptr] and [3 x i64]7995// respectively, then the types created  by this function are7996//7997// %struct.privates = type { [3 x ptr], [3 x ptr], [3 x i64] }7998// %struct.task_with_privates = type { %struct.kmp_task_ompbuilder_t,7999//                                     %struct.privates }8000// %struct.task_with_privates is returned by this function.8001// If there aren't any offloading arrays to pass to the target kernel,8002// %struct.kmp_task_ompbuilder_t is returned.8003static StructType *8004createTaskWithPrivatesTy(OpenMPIRBuilder &OMPIRBuilder,8005                         ArrayRef<Value *> OffloadingArraysToPrivatize) {8006 8007  if (OffloadingArraysToPrivatize.empty())8008    return OMPIRBuilder.Task;8009 8010  SmallVector<Type *, 4> StructFieldTypes;8011  for (Value *V : OffloadingArraysToPrivatize) {8012    assert(V->getType()->isPointerTy() &&8013           "Expected pointer to array to privatize. Got a non-pointer value "8014           "instead");8015    Type *ArrayTy = getOffloadingArrayType(V);8016    assert(ArrayTy && "ArrayType cannot be nullptr");8017    StructFieldTypes.push_back(ArrayTy);8018  }8019  StructType *PrivatesStructTy =8020      StructType::create(StructFieldTypes, "struct.privates");8021  return StructType::create({OMPIRBuilder.Task, PrivatesStructTy},8022                            "struct.task_with_privates");8023}8024static Error emitTargetOutlinedFunction(8025    OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, bool IsOffloadEntry,8026    TargetRegionEntryInfo &EntryInfo,8027    const OpenMPIRBuilder::TargetKernelDefaultAttrs &DefaultAttrs,8028    Function *&OutlinedFn, Constant *&OutlinedFnID,8029    SmallVectorImpl<Value *> &Inputs,8030    OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc,8031    OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy &ArgAccessorFuncCB) {8032 8033  OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =8034      [&](StringRef EntryFnName) {8035        return createOutlinedFunction(OMPBuilder, Builder, DefaultAttrs,8036                                      EntryFnName, Inputs, CBFunc,8037                                      ArgAccessorFuncCB);8038      };8039 8040  return OMPBuilder.emitTargetRegionFunction(8041      EntryInfo, GenerateOutlinedFunction, IsOffloadEntry, OutlinedFn,8042      OutlinedFnID);8043}8044 8045OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitTargetTask(8046    TargetTaskBodyCallbackTy TaskBodyCB, Value *DeviceID, Value *RTLoc,8047    OpenMPIRBuilder::InsertPointTy AllocaIP,8048    const SmallVector<llvm::OpenMPIRBuilder::DependData> &Dependencies,8049    const TargetDataRTArgs &RTArgs, bool HasNoWait) {8050 8051  // The following explains the code-gen scenario for the `target` directive. A8052  // similar scneario is followed for other device-related directives (e.g.8053  // `target enter data`) but in similar fashion since we only need to emit task8054  // that encapsulates the proper runtime call.8055  //8056  // When we arrive at this function, the target region itself has been8057  // outlined into the function OutlinedFn.8058  // So at ths point, for8059  // --------------------------------------------------------------8060  //   void user_code_that_offloads(...) {8061  //     omp target depend(..) map(from:a) map(to:b) private(i)8062  //     do i = 1, 108063  //        a(i) = b(i) + n8064  //   }8065  //8066  // --------------------------------------------------------------8067  //8068  // we have8069  //8070  // --------------------------------------------------------------8071  //8072  //   void user_code_that_offloads(...) {8073  //     %.offload_baseptrs = alloca [2 x ptr], align 88074  //     %.offload_ptrs = alloca [2 x ptr], align 88075  //     %.offload_mappers = alloca [2 x ptr], align 88076  //     ;; target region has been outlined and now we need to8077  //     ;; offload to it via a target task.8078  //   }8079  //   void outlined_device_function(ptr a, ptr b, ptr n) {8080  //     n = *n_ptr;8081  //     do i = 1, 108082  //       a(i) = b(i) +  n8083  //   }8084  //8085  // We have to now do the following8086  // (i)   Make an offloading call to outlined_device_function using the OpenMP8087  //       RTL. See 'kernel_launch_function' in the pseudo code below. This is8088  //       emitted by emitKernelLaunch8089  // (ii)  Create a task entry point function that calls kernel_launch_function8090  //       and is the entry point for the target task. See8091  //       '@.omp_target_task_proxy_func in the pseudocode below.8092  // (iii) Create a task with the task entry point created in (ii)8093  //8094  // That is we create the following8095  //   struct task_with_privates {8096  //      struct kmp_task_ompbuilder_t task_struct;8097  //      struct privates {8098  //         [2 x ptr] ; baseptrs8099  //         [2 x ptr] ; ptrs8100  //         [2 x i64] ; sizes8101  //      }8102  //   }8103  //   void user_code_that_offloads(...) {8104  //     %.offload_baseptrs = alloca [2 x ptr], align 88105  //     %.offload_ptrs = alloca [2 x ptr], align 88106  //     %.offload_sizes = alloca [2 x i64], align 88107  //8108  //     %structArg = alloca { ptr, ptr, ptr }, align 88109  //     %strucArg[0] = a8110  //     %strucArg[1] = b8111  //     %strucArg[2] = &n8112  //8113  //     target_task_with_privates = @__kmpc_omp_target_task_alloc(...,8114  //                                               sizeof(kmp_task_ompbuilder_t),8115  //                                               sizeof(structArg),8116  //                                               @.omp_target_task_proxy_func,8117  //                                               ...)8118  //     memcpy(target_task_with_privates->task_struct->shareds, %structArg,8119  //            sizeof(structArg))8120  //     memcpy(target_task_with_privates->privates->baseptrs,8121  //            offload_baseptrs, sizeof(offload_baseptrs)8122  //     memcpy(target_task_with_privates->privates->ptrs,8123  //            offload_ptrs, sizeof(offload_ptrs)8124  //     memcpy(target_task_with_privates->privates->sizes,8125  //            offload_sizes, sizeof(offload_sizes)8126  //     dependencies_array = ...8127  //     ;; if nowait not present8128  //     call @__kmpc_omp_wait_deps(..., dependencies_array)8129  //     call @__kmpc_omp_task_begin_if0(...)8130  //     call @ @.omp_target_task_proxy_func(i32 thread_id, ptr8131  //     %target_task_with_privates)8132  //     call @__kmpc_omp_task_complete_if0(...)8133  //   }8134  //8135  //   define internal void @.omp_target_task_proxy_func(i32 %thread.id,8136  //                                                     ptr %task) {8137  //       %structArg = alloca {ptr, ptr, ptr}8138  //       %task_ptr = getelementptr(%task, 0, 0)8139  //       %shared_data = load (getelementptr %task_ptr, 0, 0)8140  //       mempcy(%structArg, %shared_data, sizeof(%structArg))8141  //8142  //       %offloading_arrays = getelementptr(%task, 0, 1)8143  //       %offload_baseptrs = getelementptr(%offloading_arrays, 0, 0)8144  //       %offload_ptrs = getelementptr(%offloading_arrays, 0, 1)8145  //       %offload_sizes = getelementptr(%offloading_arrays, 0, 2)8146  //       kernel_launch_function(%thread.id, %offload_baseptrs, %offload_ptrs,8147  //                              %offload_sizes, %structArg)8148  //   }8149  //8150  //   We need the proxy function because the signature of the task entry point8151  //   expected by kmpc_omp_task is always the same and will be different from8152  //   that of the kernel_launch function.8153  //8154  //   kernel_launch_function is generated by emitKernelLaunch and has the8155  //   always_inline attribute. For this example, it'll look like so:8156  //   void kernel_launch_function(%thread_id, %offload_baseptrs, %offload_ptrs,8157  //                               %offload_sizes,  %structArg) alwaysinline {8158  //       %kernel_args = alloca %struct.__tgt_kernel_arguments, align 88159  //       ; load aggregated data from %structArg8160  //       ; setup kernel_args using offload_baseptrs, offload_ptrs and8161  //       ; offload_sizes8162  //       call i32 @__tgt_target_kernel(...,8163  //                                     outlined_device_function,8164  //                                     ptr %kernel_args)8165  //   }8166  //   void outlined_device_function(ptr a, ptr b, ptr n) {8167  //     n = *n_ptr;8168  //     do i = 1, 108169  //       a(i) = b(i) +  n8170  //   }8171  //8172  BasicBlock *TargetTaskBodyBB =8173      splitBB(Builder, /*CreateBranch=*/true, "target.task.body");8174  BasicBlock *TargetTaskAllocaBB =8175      splitBB(Builder, /*CreateBranch=*/true, "target.task.alloca");8176 8177  InsertPointTy TargetTaskAllocaIP(TargetTaskAllocaBB,8178                                   TargetTaskAllocaBB->begin());8179  InsertPointTy TargetTaskBodyIP(TargetTaskBodyBB, TargetTaskBodyBB->begin());8180 8181  OutlineInfo OI;8182  OI.EntryBB = TargetTaskAllocaBB;8183  OI.OuterAllocaBB = AllocaIP.getBlock();8184 8185  // Add the thread ID argument.8186  SmallVector<Instruction *, 4> ToBeDeleted;8187  OI.ExcludeArgsFromAggregate.push_back(createFakeIntVal(8188      Builder, AllocaIP, ToBeDeleted, TargetTaskAllocaIP, "global.tid", false));8189 8190  // Generate the task body which will subsequently be outlined.8191  Builder.restoreIP(TargetTaskBodyIP);8192  if (Error Err = TaskBodyCB(DeviceID, RTLoc, TargetTaskAllocaIP))8193    return Err;8194 8195  // The outliner (CodeExtractor) extract a sequence or vector of blocks that8196  // it is given. These blocks are enumerated by8197  // OpenMPIRBuilder::OutlineInfo::collectBlocks which expects the OI.ExitBlock8198  // to be outside the region. In other words, OI.ExitBlock is expected to be8199  // the start of the region after the outlining. We used to set OI.ExitBlock8200  // to the InsertBlock after TaskBodyCB is done. This is fine in most cases8201  // except when the task body is a single basic block. In that case,8202  // OI.ExitBlock is set to the single task body block and will get left out of8203  // the outlining process. So, simply create a new empty block to which we8204  // uncoditionally branch from where TaskBodyCB left off8205  OI.ExitBB = BasicBlock::Create(Builder.getContext(), "target.task.cont");8206  emitBlock(OI.ExitBB, Builder.GetInsertBlock()->getParent(),8207            /*IsFinished=*/true);8208 8209  SmallVector<Value *, 2> OffloadingArraysToPrivatize;8210  bool NeedsTargetTask = HasNoWait && DeviceID;8211  if (NeedsTargetTask) {8212    for (auto *V :8213         {RTArgs.BasePointersArray, RTArgs.PointersArray, RTArgs.MappersArray,8214          RTArgs.MapNamesArray, RTArgs.MapTypesArray, RTArgs.MapTypesArrayEnd,8215          RTArgs.SizesArray}) {8216      if (V && !isa<ConstantPointerNull, GlobalVariable>(V)) {8217        OffloadingArraysToPrivatize.push_back(V);8218        OI.ExcludeArgsFromAggregate.push_back(V);8219      }8220    }8221  }8222  OI.PostOutlineCB = [this, ToBeDeleted, Dependencies, NeedsTargetTask,8223                      DeviceID, OffloadingArraysToPrivatize](8224                         Function &OutlinedFn) mutable {8225    assert(OutlinedFn.hasOneUse() &&8226           "there must be a single user for the outlined function");8227 8228    CallInst *StaleCI = cast<CallInst>(OutlinedFn.user_back());8229 8230    // The first argument of StaleCI is always the thread id.8231    // The next few arguments are the pointers to offloading arrays8232    // if any. (see OffloadingArraysToPrivatize)8233    // Finally, all other local values that are live-in into the outlined region8234    // end up in a structure whose pointer is passed as the last argument. This8235    // piece of data is passed in the "shared" field of the task structure. So,8236    // we know we have to pass shareds to the task if the number of arguments is8237    // greater than OffloadingArraysToPrivatize.size() + 1 The 1 is for the8238    // thread id. Further, for safety, we assert that the number of arguments of8239    // StaleCI is exactly OffloadingArraysToPrivatize.size() + 28240    const unsigned int NumStaleCIArgs = StaleCI->arg_size();8241    bool HasShareds = NumStaleCIArgs > OffloadingArraysToPrivatize.size() + 1;8242    assert((!HasShareds ||8243            NumStaleCIArgs == (OffloadingArraysToPrivatize.size() + 2)) &&8244           "Wrong number of arguments for StaleCI when shareds are present");8245    int SharedArgOperandNo =8246        HasShareds ? OffloadingArraysToPrivatize.size() + 1 : 0;8247 8248    StructType *TaskWithPrivatesTy =8249        createTaskWithPrivatesTy(*this, OffloadingArraysToPrivatize);8250    StructType *PrivatesTy = nullptr;8251 8252    if (!OffloadingArraysToPrivatize.empty())8253      PrivatesTy =8254          static_cast<StructType *>(TaskWithPrivatesTy->getElementType(1));8255 8256    Function *ProxyFn = emitTargetTaskProxyFunction(8257        *this, Builder, StaleCI, PrivatesTy, TaskWithPrivatesTy,8258        OffloadingArraysToPrivatize.size(), SharedArgOperandNo);8259 8260    LLVM_DEBUG(dbgs() << "Proxy task entry function created: " << *ProxyFn8261                      << "\n");8262 8263    Builder.SetInsertPoint(StaleCI);8264 8265    // Gather the arguments for emitting the runtime call.8266    uint32_t SrcLocStrSize;8267    Constant *SrcLocStr =8268        getOrCreateSrcLocStr(LocationDescription(Builder), SrcLocStrSize);8269    Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);8270 8271    // @__kmpc_omp_task_alloc or @__kmpc_omp_target_task_alloc8272    //8273    // If `HasNoWait == true`, we call  @__kmpc_omp_target_task_alloc to provide8274    // the DeviceID to the deferred task and also since8275    // @__kmpc_omp_target_task_alloc creates an untied/async task.8276    Function *TaskAllocFn =8277        !NeedsTargetTask8278            ? getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)8279            : getOrCreateRuntimeFunctionPtr(8280                  OMPRTL___kmpc_omp_target_task_alloc);8281 8282    // Arguments - `loc_ref` (Ident) and `gtid` (ThreadID)8283    // call.8284    Value *ThreadID = getOrCreateThreadID(Ident);8285 8286    // Argument - `sizeof_kmp_task_t` (TaskSize)8287    // Tasksize refers to the size in bytes of kmp_task_t data structure8288    // plus any other data to be passed to the target task, if any, which8289    // is packed into a struct. kmp_task_t and the struct so created are8290    // packed into a wrapper struct whose type is TaskWithPrivatesTy.8291    Value *TaskSize = Builder.getInt64(8292        M.getDataLayout().getTypeStoreSize(TaskWithPrivatesTy));8293 8294    // Argument - `sizeof_shareds` (SharedsSize)8295    // SharedsSize refers to the shareds array size in the kmp_task_t data8296    // structure.8297    Value *SharedsSize = Builder.getInt64(0);8298    if (HasShareds) {8299      auto *ArgStructAlloca =8300          dyn_cast<AllocaInst>(StaleCI->getArgOperand(SharedArgOperandNo));8301      assert(ArgStructAlloca &&8302             "Unable to find the alloca instruction corresponding to arguments "8303             "for extracted function");8304      auto *ArgStructType =8305          dyn_cast<StructType>(ArgStructAlloca->getAllocatedType());8306      assert(ArgStructType && "Unable to find struct type corresponding to "8307                              "arguments for extracted function");8308      SharedsSize =8309          Builder.getInt64(M.getDataLayout().getTypeStoreSize(ArgStructType));8310    }8311 8312    // Argument - `flags`8313    // Task is tied iff (Flags & 1) == 1.8314    // Task is untied iff (Flags & 1) == 0.8315    // Task is final iff (Flags & 2) == 2.8316    // Task is not final iff (Flags & 2) == 0.8317    // A target task is not final and is untied.8318    Value *Flags = Builder.getInt32(0);8319 8320    // Emit the @__kmpc_omp_task_alloc runtime call8321    // The runtime call returns a pointer to an area where the task captured8322    // variables must be copied before the task is run (TaskData)8323    CallInst *TaskData = nullptr;8324 8325    SmallVector<llvm::Value *> TaskAllocArgs = {8326        /*loc_ref=*/Ident,        /*gtid=*/ThreadID,8327        /*flags=*/Flags,8328        /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,8329        /*task_func=*/ProxyFn};8330 8331    if (NeedsTargetTask) {8332      assert(DeviceID && "Expected non-empty device ID.");8333      TaskAllocArgs.push_back(DeviceID);8334    }8335 8336    TaskData = createRuntimeFunctionCall(TaskAllocFn, TaskAllocArgs);8337 8338    Align Alignment = TaskData->getPointerAlignment(M.getDataLayout());8339    if (HasShareds) {8340      Value *Shareds = StaleCI->getArgOperand(SharedArgOperandNo);8341      Value *TaskShareds = loadSharedDataFromTaskDescriptor(8342          *this, Builder, TaskData, TaskWithPrivatesTy);8343      Builder.CreateMemCpy(TaskShareds, Alignment, Shareds, Alignment,8344                           SharedsSize);8345    }8346    if (!OffloadingArraysToPrivatize.empty()) {8347      Value *Privates =8348          Builder.CreateStructGEP(TaskWithPrivatesTy, TaskData, 1);8349      for (unsigned int i = 0; i < OffloadingArraysToPrivatize.size(); ++i) {8350        Value *PtrToPrivatize = OffloadingArraysToPrivatize[i];8351        [[maybe_unused]] Type *ArrayType =8352            getOffloadingArrayType(PtrToPrivatize);8353        assert(ArrayType && "ArrayType cannot be nullptr");8354 8355        Type *ElementType = PrivatesTy->getElementType(i);8356        assert(ElementType == ArrayType &&8357               "ElementType should match ArrayType");8358        (void)ArrayType;8359 8360        Value *Dst = Builder.CreateStructGEP(PrivatesTy, Privates, i);8361        Builder.CreateMemCpy(8362            Dst, Alignment, PtrToPrivatize, Alignment,8363            Builder.getInt64(M.getDataLayout().getTypeStoreSize(ElementType)));8364      }8365    }8366 8367    Value *DepArray = emitTaskDependencies(*this, Dependencies);8368 8369    // ---------------------------------------------------------------8370    // V5.2 13.8 target construct8371    // If the nowait clause is present, execution of the target task8372    // may be deferred. If the nowait clause is not present, the target task is8373    // an included task.8374    // ---------------------------------------------------------------8375    // The above means that the lack of a nowait on the target construct8376    // translates to '#pragma omp task if(0)'8377    if (!NeedsTargetTask) {8378      if (DepArray) {8379        Function *TaskWaitFn =8380            getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_wait_deps);8381        createRuntimeFunctionCall(8382            TaskWaitFn,8383            {/*loc_ref=*/Ident, /*gtid=*/ThreadID,8384             /*ndeps=*/Builder.getInt32(Dependencies.size()),8385             /*dep_list=*/DepArray,8386             /*ndeps_noalias=*/ConstantInt::get(Builder.getInt32Ty(), 0),8387             /*noalias_dep_list=*/8388             ConstantPointerNull::get(PointerType::getUnqual(M.getContext()))});8389      }8390      // Included task.8391      Function *TaskBeginFn =8392          getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_begin_if0);8393      Function *TaskCompleteFn =8394          getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_complete_if0);8395      createRuntimeFunctionCall(TaskBeginFn, {Ident, ThreadID, TaskData});8396      CallInst *CI = createRuntimeFunctionCall(ProxyFn, {ThreadID, TaskData});8397      CI->setDebugLoc(StaleCI->getDebugLoc());8398      createRuntimeFunctionCall(TaskCompleteFn, {Ident, ThreadID, TaskData});8399    } else if (DepArray) {8400      // HasNoWait - meaning the task may be deferred. Call8401      // __kmpc_omp_task_with_deps if there are dependencies,8402      // else call __kmpc_omp_task8403      Function *TaskFn =8404          getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_with_deps);8405      createRuntimeFunctionCall(8406          TaskFn,8407          {Ident, ThreadID, TaskData, Builder.getInt32(Dependencies.size()),8408           DepArray, ConstantInt::get(Builder.getInt32Ty(), 0),8409           ConstantPointerNull::get(PointerType::getUnqual(M.getContext()))});8410    } else {8411      // Emit the @__kmpc_omp_task runtime call to spawn the task8412      Function *TaskFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task);8413      createRuntimeFunctionCall(TaskFn, {Ident, ThreadID, TaskData});8414    }8415 8416    StaleCI->eraseFromParent();8417    for (Instruction *I : llvm::reverse(ToBeDeleted))8418      I->eraseFromParent();8419  };8420  addOutlineInfo(std::move(OI));8421 8422  LLVM_DEBUG(dbgs() << "Insert block after emitKernelLaunch = \n"8423                    << *(Builder.GetInsertBlock()) << "\n");8424  LLVM_DEBUG(dbgs() << "Module after emitKernelLaunch = \n"8425                    << *(Builder.GetInsertBlock()->getParent()->getParent())8426                    << "\n");8427  return Builder.saveIP();8428}8429 8430Error OpenMPIRBuilder::emitOffloadingArraysAndArgs(8431    InsertPointTy AllocaIP, InsertPointTy CodeGenIP, TargetDataInfo &Info,8432    TargetDataRTArgs &RTArgs, MapInfosTy &CombinedInfo,8433    CustomMapperCallbackTy CustomMapperCB, bool IsNonContiguous,8434    bool ForEndCall, function_ref<void(unsigned int, Value *)> DeviceAddrCB) {8435  if (Error Err =8436          emitOffloadingArrays(AllocaIP, CodeGenIP, CombinedInfo, Info,8437                               CustomMapperCB, IsNonContiguous, DeviceAddrCB))8438    return Err;8439  emitOffloadingArraysArgument(Builder, RTArgs, Info, ForEndCall);8440  return Error::success();8441}8442 8443static void emitTargetCall(8444    OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,8445    OpenMPIRBuilder::InsertPointTy AllocaIP,8446    OpenMPIRBuilder::TargetDataInfo &Info,8447    const OpenMPIRBuilder::TargetKernelDefaultAttrs &DefaultAttrs,8448    const OpenMPIRBuilder::TargetKernelRuntimeAttrs &RuntimeAttrs,8449    Value *IfCond, Function *OutlinedFn, Constant *OutlinedFnID,8450    SmallVectorImpl<Value *> &Args,8451    OpenMPIRBuilder::GenMapInfoCallbackTy GenMapInfoCB,8452    OpenMPIRBuilder::CustomMapperCallbackTy CustomMapperCB,8453    const SmallVector<llvm::OpenMPIRBuilder::DependData> &Dependencies,8454    bool HasNoWait, Value *DynCGroupMem,8455    OMPDynGroupprivateFallbackType DynCGroupMemFallback) {8456  // Generate a function call to the host fallback implementation of the target8457  // region. This is called by the host when no offload entry was generated for8458  // the target region and when the offloading call fails at runtime.8459  auto &&EmitTargetCallFallbackCB = [&](OpenMPIRBuilder::InsertPointTy IP)8460      -> OpenMPIRBuilder::InsertPointOrErrorTy {8461    Builder.restoreIP(IP);8462    OMPBuilder.createRuntimeFunctionCall(OutlinedFn, Args);8463    return Builder.saveIP();8464  };8465 8466  bool HasDependencies = Dependencies.size() > 0;8467  bool RequiresOuterTargetTask = HasNoWait || HasDependencies;8468 8469  OpenMPIRBuilder::TargetKernelArgs KArgs;8470 8471  auto TaskBodyCB =8472      [&](Value *DeviceID, Value *RTLoc,8473          IRBuilderBase::InsertPoint TargetTaskAllocaIP) -> Error {8474    // Assume no error was returned because EmitTargetCallFallbackCB doesn't8475    // produce any.8476    llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail([&]() {8477      // emitKernelLaunch makes the necessary runtime call to offload the8478      // kernel. We then outline all that code into a separate function8479      // ('kernel_launch_function' in the pseudo code above). This function is8480      // then called by the target task proxy function (see8481      // '@.omp_target_task_proxy_func' in the pseudo code above)8482      // "@.omp_target_task_proxy_func' is generated by8483      // emitTargetTaskProxyFunction.8484      if (OutlinedFnID && DeviceID)8485        return OMPBuilder.emitKernelLaunch(Builder, OutlinedFnID,8486                                           EmitTargetCallFallbackCB, KArgs,8487                                           DeviceID, RTLoc, TargetTaskAllocaIP);8488 8489      // We only need to do the outlining if `DeviceID` is set to avoid calling8490      // `emitKernelLaunch` if we want to code-gen for the host; e.g. if we are8491      // generating the `else` branch of an `if` clause.8492      //8493      // When OutlinedFnID is set to nullptr, then it's not an offloading call.8494      // In this case, we execute the host implementation directly.8495      return EmitTargetCallFallbackCB(OMPBuilder.Builder.saveIP());8496    }());8497 8498    OMPBuilder.Builder.restoreIP(AfterIP);8499    return Error::success();8500  };8501 8502  auto &&EmitTargetCallElse =8503      [&](OpenMPIRBuilder::InsertPointTy AllocaIP,8504          OpenMPIRBuilder::InsertPointTy CodeGenIP) -> Error {8505    // Assume no error was returned because EmitTargetCallFallbackCB doesn't8506    // produce any.8507    OpenMPIRBuilder::InsertPointTy AfterIP = cantFail([&]() {8508      if (RequiresOuterTargetTask) {8509        // Arguments that are intended to be directly forwarded to an8510        // emitKernelLaunch call are pased as nullptr, since8511        // OutlinedFnID=nullptr results in that call not being done.8512        OpenMPIRBuilder::TargetDataRTArgs EmptyRTArgs;8513        return OMPBuilder.emitTargetTask(TaskBodyCB, /*DeviceID=*/nullptr,8514                                         /*RTLoc=*/nullptr, AllocaIP,8515                                         Dependencies, EmptyRTArgs, HasNoWait);8516      }8517      return EmitTargetCallFallbackCB(Builder.saveIP());8518    }());8519 8520    Builder.restoreIP(AfterIP);8521    return Error::success();8522  };8523 8524  auto &&EmitTargetCallThen =8525      [&](OpenMPIRBuilder::InsertPointTy AllocaIP,8526          OpenMPIRBuilder::InsertPointTy CodeGenIP) -> Error {8527    Info.HasNoWait = HasNoWait;8528    OpenMPIRBuilder::MapInfosTy &MapInfo = GenMapInfoCB(Builder.saveIP());8529    OpenMPIRBuilder::TargetDataRTArgs RTArgs;8530    if (Error Err = OMPBuilder.emitOffloadingArraysAndArgs(8531            AllocaIP, Builder.saveIP(), Info, RTArgs, MapInfo, CustomMapperCB,8532            /*IsNonContiguous=*/true,8533            /*ForEndCall=*/false))8534      return Err;8535 8536    SmallVector<Value *, 3> NumTeamsC;8537    for (auto [DefaultVal, RuntimeVal] :8538         zip_equal(DefaultAttrs.MaxTeams, RuntimeAttrs.MaxTeams))8539      NumTeamsC.push_back(RuntimeVal ? RuntimeVal8540                                     : Builder.getInt32(DefaultVal));8541 8542    // Calculate number of threads: 0 if no clauses specified, otherwise it is8543    // the minimum between optional THREAD_LIMIT and NUM_THREADS clauses.8544    auto InitMaxThreadsClause = [&Builder](Value *Clause) {8545      if (Clause)8546        Clause = Builder.CreateIntCast(Clause, Builder.getInt32Ty(),8547                                       /*isSigned=*/false);8548      return Clause;8549    };8550    auto CombineMaxThreadsClauses = [&Builder](Value *Clause, Value *&Result) {8551      if (Clause)8552        Result =8553            Result ? Builder.CreateSelect(Builder.CreateICmpULT(Result, Clause),8554                                          Result, Clause)8555                   : Clause;8556    };8557 8558    // If a multi-dimensional THREAD_LIMIT is set, it is the OMPX_BARE case, so8559    // the NUM_THREADS clause is overriden by THREAD_LIMIT.8560    SmallVector<Value *, 3> NumThreadsC;8561    Value *MaxThreadsClause =8562        RuntimeAttrs.TeamsThreadLimit.size() == 18563            ? InitMaxThreadsClause(RuntimeAttrs.MaxThreads)8564            : nullptr;8565 8566    for (auto [TeamsVal, TargetVal] : zip_equal(8567             RuntimeAttrs.TeamsThreadLimit, RuntimeAttrs.TargetThreadLimit)) {8568      Value *TeamsThreadLimitClause = InitMaxThreadsClause(TeamsVal);8569      Value *NumThreads = InitMaxThreadsClause(TargetVal);8570 8571      CombineMaxThreadsClauses(TeamsThreadLimitClause, NumThreads);8572      CombineMaxThreadsClauses(MaxThreadsClause, NumThreads);8573 8574      NumThreadsC.push_back(NumThreads ? NumThreads : Builder.getInt32(0));8575    }8576 8577    unsigned NumTargetItems = Info.NumberOfPtrs;8578    // TODO: Use correct device ID8579    Value *DeviceID = Builder.getInt64(OMP_DEVICEID_UNDEF);8580    uint32_t SrcLocStrSize;8581    Constant *SrcLocStr = OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);8582    Value *RTLoc = OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize,8583                                               llvm::omp::IdentFlag(0), 0);8584 8585    Value *TripCount = RuntimeAttrs.LoopTripCount8586                           ? Builder.CreateIntCast(RuntimeAttrs.LoopTripCount,8587                                                   Builder.getInt64Ty(),8588                                                   /*isSigned=*/false)8589                           : Builder.getInt64(0);8590 8591    // Request zero groupprivate bytes by default.8592    if (!DynCGroupMem)8593      DynCGroupMem = Builder.getInt32(0);8594 8595    KArgs = OpenMPIRBuilder::TargetKernelArgs(8596        NumTargetItems, RTArgs, TripCount, NumTeamsC, NumThreadsC, DynCGroupMem,8597        HasNoWait, DynCGroupMemFallback);8598 8599    // Assume no error was returned because TaskBodyCB and8600    // EmitTargetCallFallbackCB don't produce any.8601    OpenMPIRBuilder::InsertPointTy AfterIP = cantFail([&]() {8602      // The presence of certain clauses on the target directive require the8603      // explicit generation of the target task.8604      if (RequiresOuterTargetTask)8605        return OMPBuilder.emitTargetTask(TaskBodyCB, DeviceID, RTLoc, AllocaIP,8606                                         Dependencies, KArgs.RTArgs,8607                                         Info.HasNoWait);8608 8609      return OMPBuilder.emitKernelLaunch(Builder, OutlinedFnID,8610                                         EmitTargetCallFallbackCB, KArgs,8611                                         DeviceID, RTLoc, AllocaIP);8612    }());8613 8614    Builder.restoreIP(AfterIP);8615    return Error::success();8616  };8617 8618  // If we don't have an ID for the target region, it means an offload entry8619  // wasn't created. In this case we just run the host fallback directly and8620  // ignore any potential 'if' clauses.8621  if (!OutlinedFnID) {8622    cantFail(EmitTargetCallElse(AllocaIP, Builder.saveIP()));8623    return;8624  }8625 8626  // If there's no 'if' clause, only generate the kernel launch code path.8627  if (!IfCond) {8628    cantFail(EmitTargetCallThen(AllocaIP, Builder.saveIP()));8629    return;8630  }8631 8632  cantFail(OMPBuilder.emitIfClause(IfCond, EmitTargetCallThen,8633                                   EmitTargetCallElse, AllocaIP));8634}8635 8636OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTarget(8637    const LocationDescription &Loc, bool IsOffloadEntry, InsertPointTy AllocaIP,8638    InsertPointTy CodeGenIP, TargetDataInfo &Info,8639    TargetRegionEntryInfo &EntryInfo,8640    const TargetKernelDefaultAttrs &DefaultAttrs,8641    const TargetKernelRuntimeAttrs &RuntimeAttrs, Value *IfCond,8642    SmallVectorImpl<Value *> &Inputs, GenMapInfoCallbackTy GenMapInfoCB,8643    OpenMPIRBuilder::TargetBodyGenCallbackTy CBFunc,8644    OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy ArgAccessorFuncCB,8645    CustomMapperCallbackTy CustomMapperCB,8646    const SmallVector<DependData> &Dependencies, bool HasNowait,8647    Value *DynCGroupMem, OMPDynGroupprivateFallbackType DynCGroupMemFallback) {8648 8649  if (!updateToLocation(Loc))8650    return InsertPointTy();8651 8652  Builder.restoreIP(CodeGenIP);8653 8654  Function *OutlinedFn;8655  Constant *OutlinedFnID = nullptr;8656  // The target region is outlined into its own function. The LLVM IR for8657  // the target region itself is generated using the callbacks CBFunc8658  // and ArgAccessorFuncCB8659  if (Error Err = emitTargetOutlinedFunction(8660          *this, Builder, IsOffloadEntry, EntryInfo, DefaultAttrs, OutlinedFn,8661          OutlinedFnID, Inputs, CBFunc, ArgAccessorFuncCB))8662    return Err;8663 8664  // If we are not on the target device, then we need to generate code8665  // to make a remote call (offload) to the previously outlined function8666  // that represents the target region. Do that now.8667  if (!Config.isTargetDevice())8668    emitTargetCall(*this, Builder, AllocaIP, Info, DefaultAttrs, RuntimeAttrs,8669                   IfCond, OutlinedFn, OutlinedFnID, Inputs, GenMapInfoCB,8670                   CustomMapperCB, Dependencies, HasNowait, DynCGroupMem,8671                   DynCGroupMemFallback);8672  return Builder.saveIP();8673}8674 8675std::string OpenMPIRBuilder::getNameWithSeparators(ArrayRef<StringRef> Parts,8676                                                   StringRef FirstSeparator,8677                                                   StringRef Separator) {8678  SmallString<128> Buffer;8679  llvm::raw_svector_ostream OS(Buffer);8680  StringRef Sep = FirstSeparator;8681  for (StringRef Part : Parts) {8682    OS << Sep << Part;8683    Sep = Separator;8684  }8685  return OS.str().str();8686}8687 8688std::string8689OpenMPIRBuilder::createPlatformSpecificName(ArrayRef<StringRef> Parts) const {8690  return OpenMPIRBuilder::getNameWithSeparators(Parts, Config.firstSeparator(),8691                                                Config.separator());8692}8693 8694GlobalVariable *OpenMPIRBuilder::getOrCreateInternalVariable(8695    Type *Ty, const StringRef &Name, std::optional<unsigned> AddressSpace) {8696  auto &Elem = *InternalVars.try_emplace(Name, nullptr).first;8697  if (Elem.second) {8698    assert(Elem.second->getValueType() == Ty &&8699           "OMP internal variable has different type than requested");8700  } else {8701    // TODO: investigate the appropriate linkage type used for the global8702    // variable for possibly changing that to internal or private, or maybe8703    // create different versions of the function for different OMP internal8704    // variables.8705    const DataLayout &DL = M.getDataLayout();8706    // TODO: Investigate why AMDGPU expects AS 0 for globals even though the8707    // default global AS is 1.8708    // See double-target-call-with-declare-target.f90 and8709    // declare-target-vars-in-target-region.f90 libomptarget8710    // tests.8711    unsigned AddressSpaceVal = AddressSpace ? *AddressSpace8712                               : M.getTargetTriple().isAMDGPU()8713                                   ? 08714                                   : DL.getDefaultGlobalsAddressSpace();8715    auto Linkage = this->M.getTargetTriple().getArch() == Triple::wasm328716                       ? GlobalValue::InternalLinkage8717                       : GlobalValue::CommonLinkage;8718    auto *GV = new GlobalVariable(M, Ty, /*IsConstant=*/false, Linkage,8719                                  Constant::getNullValue(Ty), Elem.first(),8720                                  /*InsertBefore=*/nullptr,8721                                  GlobalValue::NotThreadLocal, AddressSpaceVal);8722    const llvm::Align TypeAlign = DL.getABITypeAlign(Ty);8723    const llvm::Align PtrAlign = DL.getPointerABIAlignment(AddressSpaceVal);8724    GV->setAlignment(std::max(TypeAlign, PtrAlign));8725    Elem.second = GV;8726  }8727 8728  return Elem.second;8729}8730 8731Value *OpenMPIRBuilder::getOMPCriticalRegionLock(StringRef CriticalName) {8732  std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();8733  std::string Name = getNameWithSeparators({Prefix, "var"}, ".", ".");8734  return getOrCreateInternalVariable(KmpCriticalNameTy, Name);8735}8736 8737Value *OpenMPIRBuilder::getSizeInBytes(Value *BasePtr) {8738  LLVMContext &Ctx = Builder.getContext();8739  Value *Null =8740      Constant::getNullValue(PointerType::getUnqual(BasePtr->getContext()));8741  Value *SizeGep =8742      Builder.CreateGEP(BasePtr->getType(), Null, Builder.getInt32(1));8743  Value *SizePtrToInt = Builder.CreatePtrToInt(SizeGep, Type::getInt64Ty(Ctx));8744  return SizePtrToInt;8745}8746 8747GlobalVariable *8748OpenMPIRBuilder::createOffloadMaptypes(SmallVectorImpl<uint64_t> &Mappings,8749                                       std::string VarName) {8750  llvm::Constant *MaptypesArrayInit =8751      llvm::ConstantDataArray::get(M.getContext(), Mappings);8752  auto *MaptypesArrayGlobal = new llvm::GlobalVariable(8753      M, MaptypesArrayInit->getType(),8754      /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, MaptypesArrayInit,8755      VarName);8756  MaptypesArrayGlobal->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);8757  return MaptypesArrayGlobal;8758}8759 8760void OpenMPIRBuilder::createMapperAllocas(const LocationDescription &Loc,8761                                          InsertPointTy AllocaIP,8762                                          unsigned NumOperands,8763                                          struct MapperAllocas &MapperAllocas) {8764  if (!updateToLocation(Loc))8765    return;8766 8767  auto *ArrI8PtrTy = ArrayType::get(Int8Ptr, NumOperands);8768  auto *ArrI64Ty = ArrayType::get(Int64, NumOperands);8769  Builder.restoreIP(AllocaIP);8770  AllocaInst *ArgsBase = Builder.CreateAlloca(8771      ArrI8PtrTy, /* ArraySize = */ nullptr, ".offload_baseptrs");8772  AllocaInst *Args = Builder.CreateAlloca(ArrI8PtrTy, /* ArraySize = */ nullptr,8773                                          ".offload_ptrs");8774  AllocaInst *ArgSizes = Builder.CreateAlloca(8775      ArrI64Ty, /* ArraySize = */ nullptr, ".offload_sizes");8776  updateToLocation(Loc);8777  MapperAllocas.ArgsBase = ArgsBase;8778  MapperAllocas.Args = Args;8779  MapperAllocas.ArgSizes = ArgSizes;8780}8781 8782void OpenMPIRBuilder::emitMapperCall(const LocationDescription &Loc,8783                                     Function *MapperFunc, Value *SrcLocInfo,8784                                     Value *MaptypesArg, Value *MapnamesArg,8785                                     struct MapperAllocas &MapperAllocas,8786                                     int64_t DeviceID, unsigned NumOperands) {8787  if (!updateToLocation(Loc))8788    return;8789 8790  auto *ArrI8PtrTy = ArrayType::get(Int8Ptr, NumOperands);8791  auto *ArrI64Ty = ArrayType::get(Int64, NumOperands);8792  Value *ArgsBaseGEP =8793      Builder.CreateInBoundsGEP(ArrI8PtrTy, MapperAllocas.ArgsBase,8794                                {Builder.getInt32(0), Builder.getInt32(0)});8795  Value *ArgsGEP =8796      Builder.CreateInBoundsGEP(ArrI8PtrTy, MapperAllocas.Args,8797                                {Builder.getInt32(0), Builder.getInt32(0)});8798  Value *ArgSizesGEP =8799      Builder.CreateInBoundsGEP(ArrI64Ty, MapperAllocas.ArgSizes,8800                                {Builder.getInt32(0), Builder.getInt32(0)});8801  Value *NullPtr =8802      Constant::getNullValue(PointerType::getUnqual(Int8Ptr->getContext()));8803  createRuntimeFunctionCall(MapperFunc, {SrcLocInfo, Builder.getInt64(DeviceID),8804                                         Builder.getInt32(NumOperands),8805                                         ArgsBaseGEP, ArgsGEP, ArgSizesGEP,8806                                         MaptypesArg, MapnamesArg, NullPtr});8807}8808 8809void OpenMPIRBuilder::emitOffloadingArraysArgument(IRBuilderBase &Builder,8810                                                   TargetDataRTArgs &RTArgs,8811                                                   TargetDataInfo &Info,8812                                                   bool ForEndCall) {8813  assert((!ForEndCall || Info.separateBeginEndCalls()) &&8814         "expected region end call to runtime only when end call is separate");8815  auto UnqualPtrTy = PointerType::getUnqual(M.getContext());8816  auto VoidPtrTy = UnqualPtrTy;8817  auto VoidPtrPtrTy = UnqualPtrTy;8818  auto Int64Ty = Type::getInt64Ty(M.getContext());8819  auto Int64PtrTy = UnqualPtrTy;8820 8821  if (!Info.NumberOfPtrs) {8822    RTArgs.BasePointersArray = ConstantPointerNull::get(VoidPtrPtrTy);8823    RTArgs.PointersArray = ConstantPointerNull::get(VoidPtrPtrTy);8824    RTArgs.SizesArray = ConstantPointerNull::get(Int64PtrTy);8825    RTArgs.MapTypesArray = ConstantPointerNull::get(Int64PtrTy);8826    RTArgs.MapNamesArray = ConstantPointerNull::get(VoidPtrPtrTy);8827    RTArgs.MappersArray = ConstantPointerNull::get(VoidPtrPtrTy);8828    return;8829  }8830 8831  RTArgs.BasePointersArray = Builder.CreateConstInBoundsGEP2_32(8832      ArrayType::get(VoidPtrTy, Info.NumberOfPtrs),8833      Info.RTArgs.BasePointersArray,8834      /*Idx0=*/0, /*Idx1=*/0);8835  RTArgs.PointersArray = Builder.CreateConstInBoundsGEP2_32(8836      ArrayType::get(VoidPtrTy, Info.NumberOfPtrs), Info.RTArgs.PointersArray,8837      /*Idx0=*/0,8838      /*Idx1=*/0);8839  RTArgs.SizesArray = Builder.CreateConstInBoundsGEP2_32(8840      ArrayType::get(Int64Ty, Info.NumberOfPtrs), Info.RTArgs.SizesArray,8841      /*Idx0=*/0, /*Idx1=*/0);8842  RTArgs.MapTypesArray = Builder.CreateConstInBoundsGEP2_32(8843      ArrayType::get(Int64Ty, Info.NumberOfPtrs),8844      ForEndCall && Info.RTArgs.MapTypesArrayEnd ? Info.RTArgs.MapTypesArrayEnd8845                                                 : Info.RTArgs.MapTypesArray,8846      /*Idx0=*/0,8847      /*Idx1=*/0);8848 8849  // Only emit the mapper information arrays if debug information is8850  // requested.8851  if (!Info.EmitDebug)8852    RTArgs.MapNamesArray = ConstantPointerNull::get(VoidPtrPtrTy);8853  else8854    RTArgs.MapNamesArray = Builder.CreateConstInBoundsGEP2_32(8855        ArrayType::get(VoidPtrTy, Info.NumberOfPtrs), Info.RTArgs.MapNamesArray,8856        /*Idx0=*/0,8857        /*Idx1=*/0);8858  // If there is no user-defined mapper, set the mapper array to nullptr to8859  // avoid an unnecessary data privatization8860  if (!Info.HasMapper)8861    RTArgs.MappersArray = ConstantPointerNull::get(VoidPtrPtrTy);8862  else8863    RTArgs.MappersArray =8864        Builder.CreatePointerCast(Info.RTArgs.MappersArray, VoidPtrPtrTy);8865}8866 8867void OpenMPIRBuilder::emitNonContiguousDescriptor(InsertPointTy AllocaIP,8868                                                  InsertPointTy CodeGenIP,8869                                                  MapInfosTy &CombinedInfo,8870                                                  TargetDataInfo &Info) {8871  MapInfosTy::StructNonContiguousInfo &NonContigInfo =8872      CombinedInfo.NonContigInfo;8873 8874  // Build an array of struct descriptor_dim and then assign it to8875  // offload_args.8876  //8877  // struct descriptor_dim {8878  //  uint64_t offset;8879  //  uint64_t count;8880  //  uint64_t stride8881  // };8882  Type *Int64Ty = Builder.getInt64Ty();8883  StructType *DimTy = StructType::create(8884      M.getContext(), ArrayRef<Type *>({Int64Ty, Int64Ty, Int64Ty}),8885      "struct.descriptor_dim");8886 8887  enum { OffsetFD = 0, CountFD, StrideFD };8888  // We need two index variable here since the size of "Dims" is the same as8889  // the size of Components, however, the size of offset, count, and stride is8890  // equal to the size of base declaration that is non-contiguous.8891  for (unsigned I = 0, L = 0, E = NonContigInfo.Dims.size(); I < E; ++I) {8892    // Skip emitting ir if dimension size is 1 since it cannot be8893    // non-contiguous.8894    if (NonContigInfo.Dims[I] == 1)8895      continue;8896    Builder.restoreIP(AllocaIP);8897    ArrayType *ArrayTy = ArrayType::get(DimTy, NonContigInfo.Dims[I]);8898    AllocaInst *DimsAddr =8899        Builder.CreateAlloca(ArrayTy, /* ArraySize = */ nullptr, "dims");8900    Builder.restoreIP(CodeGenIP);8901    for (unsigned II = 0, EE = NonContigInfo.Dims[I]; II < EE; ++II) {8902      unsigned RevIdx = EE - II - 1;8903      Value *DimsLVal = Builder.CreateInBoundsGEP(8904          DimsAddr->getAllocatedType(), DimsAddr,8905          {Builder.getInt64(0), Builder.getInt64(II)});8906      // Offset8907      Value *OffsetLVal = Builder.CreateStructGEP(DimTy, DimsLVal, OffsetFD);8908      Builder.CreateAlignedStore(8909          NonContigInfo.Offsets[L][RevIdx], OffsetLVal,8910          M.getDataLayout().getPrefTypeAlign(OffsetLVal->getType()));8911      // Count8912      Value *CountLVal = Builder.CreateStructGEP(DimTy, DimsLVal, CountFD);8913      Builder.CreateAlignedStore(8914          NonContigInfo.Counts[L][RevIdx], CountLVal,8915          M.getDataLayout().getPrefTypeAlign(CountLVal->getType()));8916      // Stride8917      Value *StrideLVal = Builder.CreateStructGEP(DimTy, DimsLVal, StrideFD);8918      Builder.CreateAlignedStore(8919          NonContigInfo.Strides[L][RevIdx], StrideLVal,8920          M.getDataLayout().getPrefTypeAlign(CountLVal->getType()));8921    }8922    // args[I] = &dims8923    Builder.restoreIP(CodeGenIP);8924    Value *DAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(8925        DimsAddr, Builder.getPtrTy());8926    Value *P = Builder.CreateConstInBoundsGEP2_32(8927        ArrayType::get(Builder.getPtrTy(), Info.NumberOfPtrs),8928        Info.RTArgs.PointersArray, 0, I);8929    Builder.CreateAlignedStore(8930        DAddr, P, M.getDataLayout().getPrefTypeAlign(Builder.getPtrTy()));8931    ++L;8932  }8933}8934 8935void OpenMPIRBuilder::emitUDMapperArrayInitOrDel(8936    Function *MapperFn, Value *MapperHandle, Value *Base, Value *Begin,8937    Value *Size, Value *MapType, Value *MapName, TypeSize ElementSize,8938    BasicBlock *ExitBB, bool IsInit) {8939  StringRef Prefix = IsInit ? ".init" : ".del";8940 8941  // Evaluate if this is an array section.8942  BasicBlock *BodyBB = BasicBlock::Create(8943      M.getContext(), createPlatformSpecificName({"omp.array", Prefix}));8944  Value *IsArray =8945      Builder.CreateICmpSGT(Size, Builder.getInt64(1), "omp.arrayinit.isarray");8946  Value *DeleteBit = Builder.CreateAnd(8947      MapType,8948      Builder.getInt64(8949          static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(8950              OpenMPOffloadMappingFlags::OMP_MAP_DELETE)));8951  Value *DeleteCond;8952  Value *Cond;8953  if (IsInit) {8954    // base != begin?8955    Value *BaseIsBegin = Builder.CreateICmpNE(Base, Begin);8956    // IsPtrAndObj?8957    Value *PtrAndObjBit = Builder.CreateAnd(8958        MapType,8959        Builder.getInt64(8960            static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(8961                OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ)));8962    PtrAndObjBit = Builder.CreateIsNotNull(PtrAndObjBit);8963    BaseIsBegin = Builder.CreateAnd(BaseIsBegin, PtrAndObjBit);8964    Cond = Builder.CreateOr(IsArray, BaseIsBegin);8965    DeleteCond = Builder.CreateIsNull(8966        DeleteBit,8967        createPlatformSpecificName({"omp.array", Prefix, ".delete"}));8968  } else {8969    Cond = IsArray;8970    DeleteCond = Builder.CreateIsNotNull(8971        DeleteBit,8972        createPlatformSpecificName({"omp.array", Prefix, ".delete"}));8973  }8974  Cond = Builder.CreateAnd(Cond, DeleteCond);8975  Builder.CreateCondBr(Cond, BodyBB, ExitBB);8976 8977  emitBlock(BodyBB, MapperFn);8978  // Get the array size by multiplying element size and element number (i.e., \p8979  // Size).8980  Value *ArraySize = Builder.CreateNUWMul(Size, Builder.getInt64(ElementSize));8981  // Remove OMP_MAP_TO and OMP_MAP_FROM from the map type, so that it achieves8982  // memory allocation/deletion purpose only.8983  Value *MapTypeArg = Builder.CreateAnd(8984      MapType,8985      Builder.getInt64(8986          ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(8987              OpenMPOffloadMappingFlags::OMP_MAP_TO |8988              OpenMPOffloadMappingFlags::OMP_MAP_FROM)));8989  MapTypeArg = Builder.CreateOr(8990      MapTypeArg,8991      Builder.getInt64(8992          static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(8993              OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT)));8994 8995  // Call the runtime API __tgt_push_mapper_component to fill up the runtime8996  // data structure.8997  Value *OffloadingArgs[] = {MapperHandle, Base,       Begin,8998                             ArraySize,    MapTypeArg, MapName};8999  createRuntimeFunctionCall(9000      getOrCreateRuntimeFunction(M, OMPRTL___tgt_push_mapper_component),9001      OffloadingArgs);9002}9003 9004Expected<Function *> OpenMPIRBuilder::emitUserDefinedMapper(9005    function_ref<MapInfosOrErrorTy(InsertPointTy CodeGenIP, llvm::Value *PtrPHI,9006                                   llvm::Value *BeginArg)>9007        GenMapInfoCB,9008    Type *ElemTy, StringRef FuncName, CustomMapperCallbackTy CustomMapperCB) {9009  SmallVector<Type *> Params;9010  Params.emplace_back(Builder.getPtrTy());9011  Params.emplace_back(Builder.getPtrTy());9012  Params.emplace_back(Builder.getPtrTy());9013  Params.emplace_back(Builder.getInt64Ty());9014  Params.emplace_back(Builder.getInt64Ty());9015  Params.emplace_back(Builder.getPtrTy());9016 9017  auto *FnTy =9018      FunctionType::get(Builder.getVoidTy(), Params, /* IsVarArg */ false);9019 9020  SmallString<64> TyStr;9021  raw_svector_ostream Out(TyStr);9022  Function *MapperFn =9023      Function::Create(FnTy, GlobalValue::InternalLinkage, FuncName, M);9024  MapperFn->addFnAttr(Attribute::NoInline);9025  MapperFn->addFnAttr(Attribute::NoUnwind);9026  MapperFn->addParamAttr(0, Attribute::NoUndef);9027  MapperFn->addParamAttr(1, Attribute::NoUndef);9028  MapperFn->addParamAttr(2, Attribute::NoUndef);9029  MapperFn->addParamAttr(3, Attribute::NoUndef);9030  MapperFn->addParamAttr(4, Attribute::NoUndef);9031  MapperFn->addParamAttr(5, Attribute::NoUndef);9032 9033  // Start the mapper function code generation.9034  BasicBlock *EntryBB = BasicBlock::Create(M.getContext(), "entry", MapperFn);9035  auto SavedIP = Builder.saveIP();9036  Builder.SetInsertPoint(EntryBB);9037 9038  Value *MapperHandle = MapperFn->getArg(0);9039  Value *BaseIn = MapperFn->getArg(1);9040  Value *BeginIn = MapperFn->getArg(2);9041  Value *Size = MapperFn->getArg(3);9042  Value *MapType = MapperFn->getArg(4);9043  Value *MapName = MapperFn->getArg(5);9044 9045  // Compute the starting and end addresses of array elements.9046  // Prepare common arguments for array initiation and deletion.9047  // Convert the size in bytes into the number of array elements.9048  TypeSize ElementSize = M.getDataLayout().getTypeStoreSize(ElemTy);9049  Size = Builder.CreateExactUDiv(Size, Builder.getInt64(ElementSize));9050  Value *PtrBegin = BeginIn;9051  Value *PtrEnd = Builder.CreateGEP(ElemTy, PtrBegin, Size);9052 9053  // Emit array initiation if this is an array section and \p MapType indicates9054  // that memory allocation is required.9055  BasicBlock *HeadBB = BasicBlock::Create(M.getContext(), "omp.arraymap.head");9056  emitUDMapperArrayInitOrDel(MapperFn, MapperHandle, BaseIn, BeginIn, Size,9057                             MapType, MapName, ElementSize, HeadBB,9058                             /*IsInit=*/true);9059 9060  // Emit a for loop to iterate through SizeArg of elements and map all of them.9061 9062  // Emit the loop header block.9063  emitBlock(HeadBB, MapperFn);9064  BasicBlock *BodyBB = BasicBlock::Create(M.getContext(), "omp.arraymap.body");9065  BasicBlock *DoneBB = BasicBlock::Create(M.getContext(), "omp.done");9066  // Evaluate whether the initial condition is satisfied.9067  Value *IsEmpty =9068      Builder.CreateICmpEQ(PtrBegin, PtrEnd, "omp.arraymap.isempty");9069  Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);9070 9071  // Emit the loop body block.9072  emitBlock(BodyBB, MapperFn);9073  BasicBlock *LastBB = BodyBB;9074  PHINode *PtrPHI =9075      Builder.CreatePHI(PtrBegin->getType(), 2, "omp.arraymap.ptrcurrent");9076  PtrPHI->addIncoming(PtrBegin, HeadBB);9077 9078  // Get map clause information. Fill up the arrays with all mapped variables.9079  MapInfosOrErrorTy Info = GenMapInfoCB(Builder.saveIP(), PtrPHI, BeginIn);9080  if (!Info)9081    return Info.takeError();9082 9083  // Call the runtime API __tgt_mapper_num_components to get the number of9084  // pre-existing components.9085  Value *OffloadingArgs[] = {MapperHandle};9086  Value *PreviousSize = createRuntimeFunctionCall(9087      getOrCreateRuntimeFunction(M, OMPRTL___tgt_mapper_num_components),9088      OffloadingArgs);9089  Value *ShiftedPreviousSize =9090      Builder.CreateShl(PreviousSize, Builder.getInt64(getFlagMemberOffset()));9091 9092  // Fill up the runtime mapper handle for all components.9093  for (unsigned I = 0; I < Info->BasePointers.size(); ++I) {9094    Value *CurBaseArg = Info->BasePointers[I];9095    Value *CurBeginArg = Info->Pointers[I];9096    Value *CurSizeArg = Info->Sizes[I];9097    Value *CurNameArg = Info->Names.size()9098                            ? Info->Names[I]9099                            : Constant::getNullValue(Builder.getPtrTy());9100 9101    // Extract the MEMBER_OF field from the map type.9102    Value *OriMapType = Builder.getInt64(9103        static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(9104            Info->Types[I]));9105    Value *MemberMapType =9106        Builder.CreateNUWAdd(OriMapType, ShiftedPreviousSize);9107 9108    // Combine the map type inherited from user-defined mapper with that9109    // specified in the program. According to the OMP_MAP_TO and OMP_MAP_FROM9110    // bits of the \a MapType, which is the input argument of the mapper9111    // function, the following code will set the OMP_MAP_TO and OMP_MAP_FROM9112    // bits of MemberMapType.9113    // [OpenMP 5.0], 1.2.6. map-type decay.9114    //        | alloc |  to   | from  | tofrom | release | delete9115    // ----------------------------------------------------------9116    // alloc  | alloc | alloc | alloc | alloc  | release | delete9117    // to     | alloc |  to   | alloc |   to   | release | delete9118    // from   | alloc | alloc | from  |  from  | release | delete9119    // tofrom | alloc |  to   | from  | tofrom | release | delete9120    Value *LeftToFrom = Builder.CreateAnd(9121        MapType,9122        Builder.getInt64(9123            static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(9124                OpenMPOffloadMappingFlags::OMP_MAP_TO |9125                OpenMPOffloadMappingFlags::OMP_MAP_FROM)));9126    BasicBlock *AllocBB = BasicBlock::Create(M.getContext(), "omp.type.alloc");9127    BasicBlock *AllocElseBB =9128        BasicBlock::Create(M.getContext(), "omp.type.alloc.else");9129    BasicBlock *ToBB = BasicBlock::Create(M.getContext(), "omp.type.to");9130    BasicBlock *ToElseBB =9131        BasicBlock::Create(M.getContext(), "omp.type.to.else");9132    BasicBlock *FromBB = BasicBlock::Create(M.getContext(), "omp.type.from");9133    BasicBlock *EndBB = BasicBlock::Create(M.getContext(), "omp.type.end");9134    Value *IsAlloc = Builder.CreateIsNull(LeftToFrom);9135    Builder.CreateCondBr(IsAlloc, AllocBB, AllocElseBB);9136    // In case of alloc, clear OMP_MAP_TO and OMP_MAP_FROM.9137    emitBlock(AllocBB, MapperFn);9138    Value *AllocMapType = Builder.CreateAnd(9139        MemberMapType,9140        Builder.getInt64(9141            ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(9142                OpenMPOffloadMappingFlags::OMP_MAP_TO |9143                OpenMPOffloadMappingFlags::OMP_MAP_FROM)));9144    Builder.CreateBr(EndBB);9145    emitBlock(AllocElseBB, MapperFn);9146    Value *IsTo = Builder.CreateICmpEQ(9147        LeftToFrom,9148        Builder.getInt64(9149            static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(9150                OpenMPOffloadMappingFlags::OMP_MAP_TO)));9151    Builder.CreateCondBr(IsTo, ToBB, ToElseBB);9152    // In case of to, clear OMP_MAP_FROM.9153    emitBlock(ToBB, MapperFn);9154    Value *ToMapType = Builder.CreateAnd(9155        MemberMapType,9156        Builder.getInt64(9157            ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(9158                OpenMPOffloadMappingFlags::OMP_MAP_FROM)));9159    Builder.CreateBr(EndBB);9160    emitBlock(ToElseBB, MapperFn);9161    Value *IsFrom = Builder.CreateICmpEQ(9162        LeftToFrom,9163        Builder.getInt64(9164            static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(9165                OpenMPOffloadMappingFlags::OMP_MAP_FROM)));9166    Builder.CreateCondBr(IsFrom, FromBB, EndBB);9167    // In case of from, clear OMP_MAP_TO.9168    emitBlock(FromBB, MapperFn);9169    Value *FromMapType = Builder.CreateAnd(9170        MemberMapType,9171        Builder.getInt64(9172            ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(9173                OpenMPOffloadMappingFlags::OMP_MAP_TO)));9174    // In case of tofrom, do nothing.9175    emitBlock(EndBB, MapperFn);9176    LastBB = EndBB;9177    PHINode *CurMapType =9178        Builder.CreatePHI(Builder.getInt64Ty(), 4, "omp.maptype");9179    CurMapType->addIncoming(AllocMapType, AllocBB);9180    CurMapType->addIncoming(ToMapType, ToBB);9181    CurMapType->addIncoming(FromMapType, FromBB);9182    CurMapType->addIncoming(MemberMapType, ToElseBB);9183 9184    Value *OffloadingArgs[] = {MapperHandle, CurBaseArg, CurBeginArg,9185                               CurSizeArg,   CurMapType, CurNameArg};9186 9187    auto ChildMapperFn = CustomMapperCB(I);9188    if (!ChildMapperFn)9189      return ChildMapperFn.takeError();9190    if (*ChildMapperFn) {9191      // Call the corresponding mapper function.9192      createRuntimeFunctionCall(*ChildMapperFn, OffloadingArgs)9193          ->setDoesNotThrow();9194    } else {9195      // Call the runtime API __tgt_push_mapper_component to fill up the runtime9196      // data structure.9197      createRuntimeFunctionCall(9198          getOrCreateRuntimeFunction(M, OMPRTL___tgt_push_mapper_component),9199          OffloadingArgs);9200    }9201  }9202 9203  // Update the pointer to point to the next element that needs to be mapped,9204  // and check whether we have mapped all elements.9205  Value *PtrNext = Builder.CreateConstGEP1_32(ElemTy, PtrPHI, /*Idx0=*/1,9206                                              "omp.arraymap.next");9207  PtrPHI->addIncoming(PtrNext, LastBB);9208  Value *IsDone = Builder.CreateICmpEQ(PtrNext, PtrEnd, "omp.arraymap.isdone");9209  BasicBlock *ExitBB = BasicBlock::Create(M.getContext(), "omp.arraymap.exit");9210  Builder.CreateCondBr(IsDone, ExitBB, BodyBB);9211 9212  emitBlock(ExitBB, MapperFn);9213  // Emit array deletion if this is an array section and \p MapType indicates9214  // that deletion is required.9215  emitUDMapperArrayInitOrDel(MapperFn, MapperHandle, BaseIn, BeginIn, Size,9216                             MapType, MapName, ElementSize, DoneBB,9217                             /*IsInit=*/false);9218 9219  // Emit the function exit block.9220  emitBlock(DoneBB, MapperFn, /*IsFinished=*/true);9221 9222  Builder.CreateRetVoid();9223  Builder.restoreIP(SavedIP);9224  return MapperFn;9225}9226 9227Error OpenMPIRBuilder::emitOffloadingArrays(9228    InsertPointTy AllocaIP, InsertPointTy CodeGenIP, MapInfosTy &CombinedInfo,9229    TargetDataInfo &Info, CustomMapperCallbackTy CustomMapperCB,9230    bool IsNonContiguous,9231    function_ref<void(unsigned int, Value *)> DeviceAddrCB) {9232 9233  // Reset the array information.9234  Info.clearArrayInfo();9235  Info.NumberOfPtrs = CombinedInfo.BasePointers.size();9236 9237  if (Info.NumberOfPtrs == 0)9238    return Error::success();9239 9240  Builder.restoreIP(AllocaIP);9241  // Detect if we have any capture size requiring runtime evaluation of the9242  // size so that a constant array could be eventually used.9243  ArrayType *PointerArrayType =9244      ArrayType::get(Builder.getPtrTy(), Info.NumberOfPtrs);9245 9246  Info.RTArgs.BasePointersArray = Builder.CreateAlloca(9247      PointerArrayType, /* ArraySize = */ nullptr, ".offload_baseptrs");9248 9249  Info.RTArgs.PointersArray = Builder.CreateAlloca(9250      PointerArrayType, /* ArraySize = */ nullptr, ".offload_ptrs");9251  AllocaInst *MappersArray = Builder.CreateAlloca(9252      PointerArrayType, /* ArraySize = */ nullptr, ".offload_mappers");9253  Info.RTArgs.MappersArray = MappersArray;9254 9255  // If we don't have any VLA types or other types that require runtime9256  // evaluation, we can use a constant array for the map sizes, otherwise we9257  // need to fill up the arrays as we do for the pointers.9258  Type *Int64Ty = Builder.getInt64Ty();9259  SmallVector<Constant *> ConstSizes(CombinedInfo.Sizes.size(),9260                                     ConstantInt::get(Int64Ty, 0));9261  SmallBitVector RuntimeSizes(CombinedInfo.Sizes.size());9262  for (unsigned I = 0, E = CombinedInfo.Sizes.size(); I < E; ++I) {9263    if (auto *CI = dyn_cast<Constant>(CombinedInfo.Sizes[I])) {9264      if (!isa<ConstantExpr>(CI) && !isa<GlobalValue>(CI)) {9265        if (IsNonContiguous &&9266            static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(9267                CombinedInfo.Types[I] &9268                OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG))9269          ConstSizes[I] =9270              ConstantInt::get(Int64Ty, CombinedInfo.NonContigInfo.Dims[I]);9271        else9272          ConstSizes[I] = CI;9273        continue;9274      }9275    }9276    RuntimeSizes.set(I);9277  }9278 9279  if (RuntimeSizes.all()) {9280    ArrayType *SizeArrayType = ArrayType::get(Int64Ty, Info.NumberOfPtrs);9281    Info.RTArgs.SizesArray = Builder.CreateAlloca(9282        SizeArrayType, /* ArraySize = */ nullptr, ".offload_sizes");9283    restoreIPandDebugLoc(Builder, CodeGenIP);9284  } else {9285    auto *SizesArrayInit = ConstantArray::get(9286        ArrayType::get(Int64Ty, ConstSizes.size()), ConstSizes);9287    std::string Name = createPlatformSpecificName({"offload_sizes"});9288    auto *SizesArrayGbl =9289        new GlobalVariable(M, SizesArrayInit->getType(), /*isConstant=*/true,9290                           GlobalValue::PrivateLinkage, SizesArrayInit, Name);9291    SizesArrayGbl->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);9292 9293    if (!RuntimeSizes.any()) {9294      Info.RTArgs.SizesArray = SizesArrayGbl;9295    } else {9296      unsigned IndexSize = M.getDataLayout().getIndexSizeInBits(0);9297      Align OffloadSizeAlign = M.getDataLayout().getABIIntegerTypeAlignment(64);9298      ArrayType *SizeArrayType = ArrayType::get(Int64Ty, Info.NumberOfPtrs);9299      AllocaInst *Buffer = Builder.CreateAlloca(9300          SizeArrayType, /* ArraySize = */ nullptr, ".offload_sizes");9301      Buffer->setAlignment(OffloadSizeAlign);9302      restoreIPandDebugLoc(Builder, CodeGenIP);9303      Builder.CreateMemCpy(9304          Buffer, M.getDataLayout().getPrefTypeAlign(Buffer->getType()),9305          SizesArrayGbl, OffloadSizeAlign,9306          Builder.getIntN(9307              IndexSize,9308              Buffer->getAllocationSize(M.getDataLayout())->getFixedValue()));9309 9310      Info.RTArgs.SizesArray = Buffer;9311    }9312    restoreIPandDebugLoc(Builder, CodeGenIP);9313  }9314 9315  // The map types are always constant so we don't need to generate code to9316  // fill arrays. Instead, we create an array constant.9317  SmallVector<uint64_t, 4> Mapping;9318  for (auto mapFlag : CombinedInfo.Types)9319    Mapping.push_back(9320        static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(9321            mapFlag));9322  std::string MaptypesName = createPlatformSpecificName({"offload_maptypes"});9323  auto *MapTypesArrayGbl = createOffloadMaptypes(Mapping, MaptypesName);9324  Info.RTArgs.MapTypesArray = MapTypesArrayGbl;9325 9326  // The information types are only built if provided.9327  if (!CombinedInfo.Names.empty()) {9328    auto *MapNamesArrayGbl = createOffloadMapnames(9329        CombinedInfo.Names, createPlatformSpecificName({"offload_mapnames"}));9330    Info.RTArgs.MapNamesArray = MapNamesArrayGbl;9331    Info.EmitDebug = true;9332  } else {9333    Info.RTArgs.MapNamesArray =9334        Constant::getNullValue(PointerType::getUnqual(Builder.getContext()));9335    Info.EmitDebug = false;9336  }9337 9338  // If there's a present map type modifier, it must not be applied to the end9339  // of a region, so generate a separate map type array in that case.9340  if (Info.separateBeginEndCalls()) {9341    bool EndMapTypesDiffer = false;9342    for (uint64_t &Type : Mapping) {9343      if (Type & static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(9344                     OpenMPOffloadMappingFlags::OMP_MAP_PRESENT)) {9345        Type &= ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(9346            OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);9347        EndMapTypesDiffer = true;9348      }9349    }9350    if (EndMapTypesDiffer) {9351      MapTypesArrayGbl = createOffloadMaptypes(Mapping, MaptypesName);9352      Info.RTArgs.MapTypesArrayEnd = MapTypesArrayGbl;9353    }9354  }9355 9356  PointerType *PtrTy = Builder.getPtrTy();9357  for (unsigned I = 0; I < Info.NumberOfPtrs; ++I) {9358    Value *BPVal = CombinedInfo.BasePointers[I];9359    Value *BP = Builder.CreateConstInBoundsGEP2_32(9360        ArrayType::get(PtrTy, Info.NumberOfPtrs), Info.RTArgs.BasePointersArray,9361        0, I);9362    Builder.CreateAlignedStore(BPVal, BP,9363                               M.getDataLayout().getPrefTypeAlign(PtrTy));9364 9365    if (Info.requiresDevicePointerInfo()) {9366      if (CombinedInfo.DevicePointers[I] == DeviceInfoTy::Pointer) {9367        CodeGenIP = Builder.saveIP();9368        Builder.restoreIP(AllocaIP);9369        Info.DevicePtrInfoMap[BPVal] = {BP, Builder.CreateAlloca(PtrTy)};9370        Builder.restoreIP(CodeGenIP);9371        if (DeviceAddrCB)9372          DeviceAddrCB(I, Info.DevicePtrInfoMap[BPVal].second);9373      } else if (CombinedInfo.DevicePointers[I] == DeviceInfoTy::Address) {9374        Info.DevicePtrInfoMap[BPVal] = {BP, BP};9375        if (DeviceAddrCB)9376          DeviceAddrCB(I, BP);9377      }9378    }9379 9380    Value *PVal = CombinedInfo.Pointers[I];9381    Value *P = Builder.CreateConstInBoundsGEP2_32(9382        ArrayType::get(PtrTy, Info.NumberOfPtrs), Info.RTArgs.PointersArray, 0,9383        I);9384    // TODO: Check alignment correct.9385    Builder.CreateAlignedStore(PVal, P,9386                               M.getDataLayout().getPrefTypeAlign(PtrTy));9387 9388    if (RuntimeSizes.test(I)) {9389      Value *S = Builder.CreateConstInBoundsGEP2_32(9390          ArrayType::get(Int64Ty, Info.NumberOfPtrs), Info.RTArgs.SizesArray,9391          /*Idx0=*/0,9392          /*Idx1=*/I);9393      Builder.CreateAlignedStore(Builder.CreateIntCast(CombinedInfo.Sizes[I],9394                                                       Int64Ty,9395                                                       /*isSigned=*/true),9396                                 S, M.getDataLayout().getPrefTypeAlign(PtrTy));9397    }9398    // Fill up the mapper array.9399    unsigned IndexSize = M.getDataLayout().getIndexSizeInBits(0);9400    Value *MFunc = ConstantPointerNull::get(PtrTy);9401 9402    auto CustomMFunc = CustomMapperCB(I);9403    if (!CustomMFunc)9404      return CustomMFunc.takeError();9405    if (*CustomMFunc)9406      MFunc = Builder.CreatePointerCast(*CustomMFunc, PtrTy);9407 9408    Value *MAddr = Builder.CreateInBoundsGEP(9409        MappersArray->getAllocatedType(), MappersArray,9410        {Builder.getIntN(IndexSize, 0), Builder.getIntN(IndexSize, I)});9411    Builder.CreateAlignedStore(9412        MFunc, MAddr, M.getDataLayout().getPrefTypeAlign(MAddr->getType()));9413  }9414 9415  if (!IsNonContiguous || CombinedInfo.NonContigInfo.Offsets.empty() ||9416      Info.NumberOfPtrs == 0)9417    return Error::success();9418  emitNonContiguousDescriptor(AllocaIP, CodeGenIP, CombinedInfo, Info);9419  return Error::success();9420}9421 9422void OpenMPIRBuilder::emitBranch(BasicBlock *Target) {9423  BasicBlock *CurBB = Builder.GetInsertBlock();9424 9425  if (!CurBB || CurBB->getTerminator()) {9426    // If there is no insert point or the previous block is already9427    // terminated, don't touch it.9428  } else {9429    // Otherwise, create a fall-through branch.9430    Builder.CreateBr(Target);9431  }9432 9433  Builder.ClearInsertionPoint();9434}9435 9436void OpenMPIRBuilder::emitBlock(BasicBlock *BB, Function *CurFn,9437                                bool IsFinished) {9438  BasicBlock *CurBB = Builder.GetInsertBlock();9439 9440  // Fall out of the current block (if necessary).9441  emitBranch(BB);9442 9443  if (IsFinished && BB->use_empty()) {9444    BB->eraseFromParent();9445    return;9446  }9447 9448  // Place the block after the current block, if possible, or else at9449  // the end of the function.9450  if (CurBB && CurBB->getParent())9451    CurFn->insert(std::next(CurBB->getIterator()), BB);9452  else9453    CurFn->insert(CurFn->end(), BB);9454  Builder.SetInsertPoint(BB);9455}9456 9457Error OpenMPIRBuilder::emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen,9458                                    BodyGenCallbackTy ElseGen,9459                                    InsertPointTy AllocaIP) {9460  // If the condition constant folds and can be elided, try to avoid emitting9461  // the condition and the dead arm of the if/else.9462  if (auto *CI = dyn_cast<ConstantInt>(Cond)) {9463    auto CondConstant = CI->getSExtValue();9464    if (CondConstant)9465      return ThenGen(AllocaIP, Builder.saveIP());9466 9467    return ElseGen(AllocaIP, Builder.saveIP());9468  }9469 9470  Function *CurFn = Builder.GetInsertBlock()->getParent();9471 9472  // Otherwise, the condition did not fold, or we couldn't elide it.  Just9473  // emit the conditional branch.9474  BasicBlock *ThenBlock = BasicBlock::Create(M.getContext(), "omp_if.then");9475  BasicBlock *ElseBlock = BasicBlock::Create(M.getContext(), "omp_if.else");9476  BasicBlock *ContBlock = BasicBlock::Create(M.getContext(), "omp_if.end");9477  Builder.CreateCondBr(Cond, ThenBlock, ElseBlock);9478  // Emit the 'then' code.9479  emitBlock(ThenBlock, CurFn);9480  if (Error Err = ThenGen(AllocaIP, Builder.saveIP()))9481    return Err;9482  emitBranch(ContBlock);9483  // Emit the 'else' code if present.9484  // There is no need to emit line number for unconditional branch.9485  emitBlock(ElseBlock, CurFn);9486  if (Error Err = ElseGen(AllocaIP, Builder.saveIP()))9487    return Err;9488  // There is no need to emit line number for unconditional branch.9489  emitBranch(ContBlock);9490  // Emit the continuation block for code after the if.9491  emitBlock(ContBlock, CurFn, /*IsFinished=*/true);9492  return Error::success();9493}9494 9495bool OpenMPIRBuilder::checkAndEmitFlushAfterAtomic(9496    const LocationDescription &Loc, llvm::AtomicOrdering AO, AtomicKind AK) {9497  assert(!(AO == AtomicOrdering::NotAtomic ||9498           AO == llvm::AtomicOrdering::Unordered) &&9499         "Unexpected Atomic Ordering.");9500 9501  bool Flush = false;9502  llvm::AtomicOrdering FlushAO = AtomicOrdering::Monotonic;9503 9504  switch (AK) {9505  case Read:9506    if (AO == AtomicOrdering::Acquire || AO == AtomicOrdering::AcquireRelease ||9507        AO == AtomicOrdering::SequentiallyConsistent) {9508      FlushAO = AtomicOrdering::Acquire;9509      Flush = true;9510    }9511    break;9512  case Write:9513  case Compare:9514  case Update:9515    if (AO == AtomicOrdering::Release || AO == AtomicOrdering::AcquireRelease ||9516        AO == AtomicOrdering::SequentiallyConsistent) {9517      FlushAO = AtomicOrdering::Release;9518      Flush = true;9519    }9520    break;9521  case Capture:9522    switch (AO) {9523    case AtomicOrdering::Acquire:9524      FlushAO = AtomicOrdering::Acquire;9525      Flush = true;9526      break;9527    case AtomicOrdering::Release:9528      FlushAO = AtomicOrdering::Release;9529      Flush = true;9530      break;9531    case AtomicOrdering::AcquireRelease:9532    case AtomicOrdering::SequentiallyConsistent:9533      FlushAO = AtomicOrdering::AcquireRelease;9534      Flush = true;9535      break;9536    default:9537      // do nothing - leave silently.9538      break;9539    }9540  }9541 9542  if (Flush) {9543    // Currently Flush RT call still doesn't take memory_ordering, so for when9544    // that happens, this tries to do the resolution of which atomic ordering9545    // to use with but issue the flush call9546    // TODO: pass `FlushAO` after memory ordering support is added9547    (void)FlushAO;9548    emitFlush(Loc);9549  }9550 9551  // for AO == AtomicOrdering::Monotonic and  all other case combinations9552  // do nothing9553  return Flush;9554}9555 9556OpenMPIRBuilder::InsertPointTy9557OpenMPIRBuilder::createAtomicRead(const LocationDescription &Loc,9558                                  AtomicOpValue &X, AtomicOpValue &V,9559                                  AtomicOrdering AO, InsertPointTy AllocaIP) {9560  if (!updateToLocation(Loc))9561    return Loc.IP;9562 9563  assert(X.Var->getType()->isPointerTy() &&9564         "OMP Atomic expects a pointer to target memory");9565  Type *XElemTy = X.ElemTy;9566  assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||9567          XElemTy->isPointerTy() || XElemTy->isStructTy()) &&9568         "OMP atomic read expected a scalar type");9569 9570  Value *XRead = nullptr;9571 9572  if (XElemTy->isIntegerTy()) {9573    LoadInst *XLD =9574        Builder.CreateLoad(XElemTy, X.Var, X.IsVolatile, "omp.atomic.read");9575    XLD->setAtomic(AO);9576    XRead = cast<Value>(XLD);9577  } else if (XElemTy->isStructTy()) {9578    // FIXME: Add checks to ensure __atomic_load is emitted iff the9579    // target does not support `atomicrmw` of the size of the struct9580    LoadInst *OldVal = Builder.CreateLoad(XElemTy, X.Var, "omp.atomic.read");9581    OldVal->setAtomic(AO);9582    const DataLayout &DL = OldVal->getModule()->getDataLayout();9583    unsigned LoadSize = DL.getTypeStoreSize(XElemTy);9584    OpenMPIRBuilder::AtomicInfo atomicInfo(9585        &Builder, XElemTy, LoadSize * 8, LoadSize * 8, OldVal->getAlign(),9586        OldVal->getAlign(), true /* UseLibcall */, AllocaIP, X.Var);9587    auto AtomicLoadRes = atomicInfo.EmitAtomicLoadLibcall(AO);9588    XRead = AtomicLoadRes.first;9589    OldVal->eraseFromParent();9590  } else {9591    // We need to perform atomic op as integer9592    IntegerType *IntCastTy =9593        IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits());9594    LoadInst *XLoad =9595        Builder.CreateLoad(IntCastTy, X.Var, X.IsVolatile, "omp.atomic.load");9596    XLoad->setAtomic(AO);9597    if (XElemTy->isFloatingPointTy()) {9598      XRead = Builder.CreateBitCast(XLoad, XElemTy, "atomic.flt.cast");9599    } else {9600      XRead = Builder.CreateIntToPtr(XLoad, XElemTy, "atomic.ptr.cast");9601    }9602  }9603  checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Read);9604  Builder.CreateStore(XRead, V.Var, V.IsVolatile);9605  return Builder.saveIP();9606}9607 9608OpenMPIRBuilder::InsertPointTy9609OpenMPIRBuilder::createAtomicWrite(const LocationDescription &Loc,9610                                   AtomicOpValue &X, Value *Expr,9611                                   AtomicOrdering AO, InsertPointTy AllocaIP) {9612  if (!updateToLocation(Loc))9613    return Loc.IP;9614 9615  assert(X.Var->getType()->isPointerTy() &&9616         "OMP Atomic expects a pointer to target memory");9617  Type *XElemTy = X.ElemTy;9618  assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||9619          XElemTy->isPointerTy() || XElemTy->isStructTy()) &&9620         "OMP atomic write expected a scalar type");9621 9622  if (XElemTy->isIntegerTy()) {9623    StoreInst *XSt = Builder.CreateStore(Expr, X.Var, X.IsVolatile);9624    XSt->setAtomic(AO);9625  } else if (XElemTy->isStructTy()) {9626    LoadInst *OldVal = Builder.CreateLoad(XElemTy, X.Var, "omp.atomic.read");9627    const DataLayout &DL = OldVal->getModule()->getDataLayout();9628    unsigned LoadSize = DL.getTypeStoreSize(XElemTy);9629    OpenMPIRBuilder::AtomicInfo atomicInfo(9630        &Builder, XElemTy, LoadSize * 8, LoadSize * 8, OldVal->getAlign(),9631        OldVal->getAlign(), true /* UseLibcall */, AllocaIP, X.Var);9632    atomicInfo.EmitAtomicStoreLibcall(AO, Expr);9633    OldVal->eraseFromParent();9634  } else {9635    // We need to bitcast and perform atomic op as integers9636    IntegerType *IntCastTy =9637        IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits());9638    Value *ExprCast =9639        Builder.CreateBitCast(Expr, IntCastTy, "atomic.src.int.cast");9640    StoreInst *XSt = Builder.CreateStore(ExprCast, X.Var, X.IsVolatile);9641    XSt->setAtomic(AO);9642  }9643 9644  checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Write);9645  return Builder.saveIP();9646}9647 9648OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicUpdate(9649    const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,9650    Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,9651    AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr,9652    bool IsIgnoreDenormalMode, bool IsFineGrainedMemory, bool IsRemoteMemory) {9653  assert(!isConflictIP(Loc.IP, AllocaIP) && "IPs must not be ambiguous");9654  if (!updateToLocation(Loc))9655    return Loc.IP;9656 9657  LLVM_DEBUG({9658    Type *XTy = X.Var->getType();9659    assert(XTy->isPointerTy() &&9660           "OMP Atomic expects a pointer to target memory");9661    Type *XElemTy = X.ElemTy;9662    assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||9663            XElemTy->isPointerTy()) &&9664           "OMP atomic update expected a scalar type");9665    assert((RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) &&9666           (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) &&9667           "OpenMP atomic does not support LT or GT operations");9668  });9669 9670  Expected<std::pair<Value *, Value *>> AtomicResult = emitAtomicUpdate(9671      AllocaIP, X.Var, X.ElemTy, Expr, AO, RMWOp, UpdateOp, X.IsVolatile,9672      IsXBinopExpr, IsIgnoreDenormalMode, IsFineGrainedMemory, IsRemoteMemory);9673  if (!AtomicResult)9674    return AtomicResult.takeError();9675  checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Update);9676  return Builder.saveIP();9677}9678 9679// FIXME: Duplicating AtomicExpand9680Value *OpenMPIRBuilder::emitRMWOpAsInstruction(Value *Src1, Value *Src2,9681                                               AtomicRMWInst::BinOp RMWOp) {9682  switch (RMWOp) {9683  case AtomicRMWInst::Add:9684    return Builder.CreateAdd(Src1, Src2);9685  case AtomicRMWInst::Sub:9686    return Builder.CreateSub(Src1, Src2);9687  case AtomicRMWInst::And:9688    return Builder.CreateAnd(Src1, Src2);9689  case AtomicRMWInst::Nand:9690    return Builder.CreateNeg(Builder.CreateAnd(Src1, Src2));9691  case AtomicRMWInst::Or:9692    return Builder.CreateOr(Src1, Src2);9693  case AtomicRMWInst::Xor:9694    return Builder.CreateXor(Src1, Src2);9695  case AtomicRMWInst::Xchg:9696  case AtomicRMWInst::FAdd:9697  case AtomicRMWInst::FSub:9698  case AtomicRMWInst::BAD_BINOP:9699  case AtomicRMWInst::Max:9700  case AtomicRMWInst::Min:9701  case AtomicRMWInst::UMax:9702  case AtomicRMWInst::UMin:9703  case AtomicRMWInst::FMax:9704  case AtomicRMWInst::FMin:9705  case AtomicRMWInst::FMaximum:9706  case AtomicRMWInst::FMinimum:9707  case AtomicRMWInst::UIncWrap:9708  case AtomicRMWInst::UDecWrap:9709  case AtomicRMWInst::USubCond:9710  case AtomicRMWInst::USubSat:9711    llvm_unreachable("Unsupported atomic update operation");9712  }9713  llvm_unreachable("Unsupported atomic update operation");9714}9715 9716Expected<std::pair<Value *, Value *>> OpenMPIRBuilder::emitAtomicUpdate(9717    InsertPointTy AllocaIP, Value *X, Type *XElemTy, Value *Expr,9718    AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,9719    AtomicUpdateCallbackTy &UpdateOp, bool VolatileX, bool IsXBinopExpr,9720    bool IsIgnoreDenormalMode, bool IsFineGrainedMemory, bool IsRemoteMemory) {9721  // TODO: handle the case where XElemTy is not byte-sized or not a power of 29722  // or a complex datatype.9723  bool emitRMWOp = false;9724  switch (RMWOp) {9725  case AtomicRMWInst::Add:9726  case AtomicRMWInst::And:9727  case AtomicRMWInst::Nand:9728  case AtomicRMWInst::Or:9729  case AtomicRMWInst::Xor:9730  case AtomicRMWInst::Xchg:9731    emitRMWOp = XElemTy;9732    break;9733  case AtomicRMWInst::Sub:9734    emitRMWOp = (IsXBinopExpr && XElemTy);9735    break;9736  default:9737    emitRMWOp = false;9738  }9739  emitRMWOp &= XElemTy->isIntegerTy();9740 9741  std::pair<Value *, Value *> Res;9742  if (emitRMWOp) {9743    AtomicRMWInst *RMWInst =9744        Builder.CreateAtomicRMW(RMWOp, X, Expr, llvm::MaybeAlign(), AO);9745    if (T.isAMDGPU()) {9746      if (IsIgnoreDenormalMode)9747        RMWInst->setMetadata("amdgpu.ignore.denormal.mode",9748                             llvm::MDNode::get(Builder.getContext(), {}));9749      if (!IsFineGrainedMemory)9750        RMWInst->setMetadata("amdgpu.no.fine.grained.memory",9751                             llvm::MDNode::get(Builder.getContext(), {}));9752      if (!IsRemoteMemory)9753        RMWInst->setMetadata("amdgpu.no.remote.memory",9754                             llvm::MDNode::get(Builder.getContext(), {}));9755    }9756    Res.first = RMWInst;9757    // not needed except in case of postfix captures. Generate anyway for9758    // consistency with the else part. Will be removed with any DCE pass.9759    // AtomicRMWInst::Xchg does not have a coressponding instruction.9760    if (RMWOp == AtomicRMWInst::Xchg)9761      Res.second = Res.first;9762    else9763      Res.second = emitRMWOpAsInstruction(Res.first, Expr, RMWOp);9764  } else if (RMWOp == llvm::AtomicRMWInst::BinOp::BAD_BINOP &&9765             XElemTy->isStructTy()) {9766    LoadInst *OldVal =9767        Builder.CreateLoad(XElemTy, X, X->getName() + ".atomic.load");9768    OldVal->setAtomic(AO);9769    const DataLayout &LoadDL = OldVal->getModule()->getDataLayout();9770    unsigned LoadSize =9771        LoadDL.getTypeStoreSize(OldVal->getPointerOperand()->getType());9772 9773    OpenMPIRBuilder::AtomicInfo atomicInfo(9774        &Builder, XElemTy, LoadSize * 8, LoadSize * 8, OldVal->getAlign(),9775        OldVal->getAlign(), true /* UseLibcall */, AllocaIP, X);9776    auto AtomicLoadRes = atomicInfo.EmitAtomicLoadLibcall(AO);9777    BasicBlock *CurBB = Builder.GetInsertBlock();9778    Instruction *CurBBTI = CurBB->getTerminator();9779    CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();9780    BasicBlock *ExitBB =9781        CurBB->splitBasicBlock(CurBBTI, X->getName() + ".atomic.exit");9782    BasicBlock *ContBB = CurBB->splitBasicBlock(CurBB->getTerminator(),9783                                                X->getName() + ".atomic.cont");9784    ContBB->getTerminator()->eraseFromParent();9785    Builder.restoreIP(AllocaIP);9786    AllocaInst *NewAtomicAddr = Builder.CreateAlloca(XElemTy);9787    NewAtomicAddr->setName(X->getName() + "x.new.val");9788    Builder.SetInsertPoint(ContBB);9789    llvm::PHINode *PHI = Builder.CreatePHI(OldVal->getType(), 2);9790    PHI->addIncoming(AtomicLoadRes.first, CurBB);9791    Value *OldExprVal = PHI;9792    Expected<Value *> CBResult = UpdateOp(OldExprVal, Builder);9793    if (!CBResult)9794      return CBResult.takeError();9795    Value *Upd = *CBResult;9796    Builder.CreateStore(Upd, NewAtomicAddr);9797    AtomicOrdering Failure =9798        llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);9799    auto Result = atomicInfo.EmitAtomicCompareExchangeLibcall(9800        AtomicLoadRes.second, NewAtomicAddr, AO, Failure);9801    LoadInst *PHILoad = Builder.CreateLoad(XElemTy, Result.first);9802    PHI->addIncoming(PHILoad, Builder.GetInsertBlock());9803    Builder.CreateCondBr(Result.second, ExitBB, ContBB);9804    OldVal->eraseFromParent();9805    Res.first = OldExprVal;9806    Res.second = Upd;9807 9808    if (UnreachableInst *ExitTI =9809            dyn_cast<UnreachableInst>(ExitBB->getTerminator())) {9810      CurBBTI->eraseFromParent();9811      Builder.SetInsertPoint(ExitBB);9812    } else {9813      Builder.SetInsertPoint(ExitTI);9814    }9815  } else {9816    IntegerType *IntCastTy =9817        IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits());9818    LoadInst *OldVal =9819        Builder.CreateLoad(IntCastTy, X, X->getName() + ".atomic.load");9820    OldVal->setAtomic(AO);9821    // CurBB9822    // |     /---\9823    // ContBB    |9824    // |     \---/9825    // ExitBB9826    BasicBlock *CurBB = Builder.GetInsertBlock();9827    Instruction *CurBBTI = CurBB->getTerminator();9828    CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();9829    BasicBlock *ExitBB =9830        CurBB->splitBasicBlock(CurBBTI, X->getName() + ".atomic.exit");9831    BasicBlock *ContBB = CurBB->splitBasicBlock(CurBB->getTerminator(),9832                                                X->getName() + ".atomic.cont");9833    ContBB->getTerminator()->eraseFromParent();9834    Builder.restoreIP(AllocaIP);9835    AllocaInst *NewAtomicAddr = Builder.CreateAlloca(XElemTy);9836    NewAtomicAddr->setName(X->getName() + "x.new.val");9837    Builder.SetInsertPoint(ContBB);9838    llvm::PHINode *PHI = Builder.CreatePHI(OldVal->getType(), 2);9839    PHI->addIncoming(OldVal, CurBB);9840    bool IsIntTy = XElemTy->isIntegerTy();9841    Value *OldExprVal = PHI;9842    if (!IsIntTy) {9843      if (XElemTy->isFloatingPointTy()) {9844        OldExprVal = Builder.CreateBitCast(PHI, XElemTy,9845                                           X->getName() + ".atomic.fltCast");9846      } else {9847        OldExprVal = Builder.CreateIntToPtr(PHI, XElemTy,9848                                            X->getName() + ".atomic.ptrCast");9849      }9850    }9851 9852    Expected<Value *> CBResult = UpdateOp(OldExprVal, Builder);9853    if (!CBResult)9854      return CBResult.takeError();9855    Value *Upd = *CBResult;9856    Builder.CreateStore(Upd, NewAtomicAddr);9857    LoadInst *DesiredVal = Builder.CreateLoad(IntCastTy, NewAtomicAddr);9858    AtomicOrdering Failure =9859        llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);9860    AtomicCmpXchgInst *Result = Builder.CreateAtomicCmpXchg(9861        X, PHI, DesiredVal, llvm::MaybeAlign(), AO, Failure);9862    Result->setVolatile(VolatileX);9863    Value *PreviousVal = Builder.CreateExtractValue(Result, /*Idxs=*/0);9864    Value *SuccessFailureVal = Builder.CreateExtractValue(Result, /*Idxs=*/1);9865    PHI->addIncoming(PreviousVal, Builder.GetInsertBlock());9866    Builder.CreateCondBr(SuccessFailureVal, ExitBB, ContBB);9867 9868    Res.first = OldExprVal;9869    Res.second = Upd;9870 9871    // set Insertion point in exit block9872    if (UnreachableInst *ExitTI =9873            dyn_cast<UnreachableInst>(ExitBB->getTerminator())) {9874      CurBBTI->eraseFromParent();9875      Builder.SetInsertPoint(ExitBB);9876    } else {9877      Builder.SetInsertPoint(ExitTI);9878    }9879  }9880 9881  return Res;9882}9883 9884OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicCapture(9885    const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,9886    AtomicOpValue &V, Value *Expr, AtomicOrdering AO,9887    AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp,9888    bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr,9889    bool IsIgnoreDenormalMode, bool IsFineGrainedMemory, bool IsRemoteMemory) {9890  if (!updateToLocation(Loc))9891    return Loc.IP;9892 9893  LLVM_DEBUG({9894    Type *XTy = X.Var->getType();9895    assert(XTy->isPointerTy() &&9896           "OMP Atomic expects a pointer to target memory");9897    Type *XElemTy = X.ElemTy;9898    assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||9899            XElemTy->isPointerTy()) &&9900           "OMP atomic capture expected a scalar type");9901    assert((RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) &&9902           "OpenMP atomic does not support LT or GT operations");9903  });9904 9905  // If UpdateExpr is 'x' updated with some `expr` not based on 'x',9906  // 'x' is simply atomically rewritten with 'expr'.9907  AtomicRMWInst::BinOp AtomicOp = (UpdateExpr ? RMWOp : AtomicRMWInst::Xchg);9908  Expected<std::pair<Value *, Value *>> AtomicResult = emitAtomicUpdate(9909      AllocaIP, X.Var, X.ElemTy, Expr, AO, AtomicOp, UpdateOp, X.IsVolatile,9910      IsXBinopExpr, IsIgnoreDenormalMode, IsFineGrainedMemory, IsRemoteMemory);9911  if (!AtomicResult)9912    return AtomicResult.takeError();9913  Value *CapturedVal =9914      (IsPostfixUpdate ? AtomicResult->first : AtomicResult->second);9915  Builder.CreateStore(CapturedVal, V.Var, V.IsVolatile);9916 9917  checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Capture);9918  return Builder.saveIP();9919}9920 9921OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(9922    const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V,9923    AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO,9924    omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate,9925    bool IsFailOnly) {9926 9927  AtomicOrdering Failure = AtomicCmpXchgInst::getStrongestFailureOrdering(AO);9928  return createAtomicCompare(Loc, X, V, R, E, D, AO, Op, IsXBinopExpr,9929                             IsPostfixUpdate, IsFailOnly, Failure);9930}9931 9932OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(9933    const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V,9934    AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO,9935    omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate,9936    bool IsFailOnly, AtomicOrdering Failure) {9937 9938  if (!updateToLocation(Loc))9939    return Loc.IP;9940 9941  assert(X.Var->getType()->isPointerTy() &&9942         "OMP atomic expects a pointer to target memory");9943  // compare capture9944  if (V.Var) {9945    assert(V.Var->getType()->isPointerTy() && "v.var must be of pointer type");9946    assert(V.ElemTy == X.ElemTy && "x and v must be of same type");9947  }9948 9949  bool IsInteger = E->getType()->isIntegerTy();9950 9951  if (Op == OMPAtomicCompareOp::EQ) {9952    AtomicCmpXchgInst *Result = nullptr;9953    if (!IsInteger) {9954      IntegerType *IntCastTy =9955          IntegerType::get(M.getContext(), X.ElemTy->getScalarSizeInBits());9956      Value *EBCast = Builder.CreateBitCast(E, IntCastTy);9957      Value *DBCast = Builder.CreateBitCast(D, IntCastTy);9958      Result = Builder.CreateAtomicCmpXchg(X.Var, EBCast, DBCast, MaybeAlign(),9959                                           AO, Failure);9960    } else {9961      Result =9962          Builder.CreateAtomicCmpXchg(X.Var, E, D, MaybeAlign(), AO, Failure);9963    }9964 9965    if (V.Var) {9966      Value *OldValue = Builder.CreateExtractValue(Result, /*Idxs=*/0);9967      if (!IsInteger)9968        OldValue = Builder.CreateBitCast(OldValue, X.ElemTy);9969      assert(OldValue->getType() == V.ElemTy &&9970             "OldValue and V must be of same type");9971      if (IsPostfixUpdate) {9972        Builder.CreateStore(OldValue, V.Var, V.IsVolatile);9973      } else {9974        Value *SuccessOrFail = Builder.CreateExtractValue(Result, /*Idxs=*/1);9975        if (IsFailOnly) {9976          // CurBB----9977          //   |     |9978          //   v     |9979          // ContBB  |9980          //   |     |9981          //   v     |9982          // ExitBB <-9983          //9984          // where ContBB only contains the store of old value to 'v'.9985          BasicBlock *CurBB = Builder.GetInsertBlock();9986          Instruction *CurBBTI = CurBB->getTerminator();9987          CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();9988          BasicBlock *ExitBB = CurBB->splitBasicBlock(9989              CurBBTI, X.Var->getName() + ".atomic.exit");9990          BasicBlock *ContBB = CurBB->splitBasicBlock(9991              CurBB->getTerminator(), X.Var->getName() + ".atomic.cont");9992          ContBB->getTerminator()->eraseFromParent();9993          CurBB->getTerminator()->eraseFromParent();9994 9995          Builder.CreateCondBr(SuccessOrFail, ExitBB, ContBB);9996 9997          Builder.SetInsertPoint(ContBB);9998          Builder.CreateStore(OldValue, V.Var);9999          Builder.CreateBr(ExitBB);10000 10001          if (UnreachableInst *ExitTI =10002                  dyn_cast<UnreachableInst>(ExitBB->getTerminator())) {10003            CurBBTI->eraseFromParent();10004            Builder.SetInsertPoint(ExitBB);10005          } else {10006            Builder.SetInsertPoint(ExitTI);10007          }10008        } else {10009          Value *CapturedValue =10010              Builder.CreateSelect(SuccessOrFail, E, OldValue);10011          Builder.CreateStore(CapturedValue, V.Var, V.IsVolatile);10012        }10013      }10014    }10015    // The comparison result has to be stored.10016    if (R.Var) {10017      assert(R.Var->getType()->isPointerTy() &&10018             "r.var must be of pointer type");10019      assert(R.ElemTy->isIntegerTy() && "r must be of integral type");10020 10021      Value *SuccessFailureVal = Builder.CreateExtractValue(Result, /*Idxs=*/1);10022      Value *ResultCast = R.IsSigned10023                              ? Builder.CreateSExt(SuccessFailureVal, R.ElemTy)10024                              : Builder.CreateZExt(SuccessFailureVal, R.ElemTy);10025      Builder.CreateStore(ResultCast, R.Var, R.IsVolatile);10026    }10027  } else {10028    assert((Op == OMPAtomicCompareOp::MAX || Op == OMPAtomicCompareOp::MIN) &&10029           "Op should be either max or min at this point");10030    assert(!IsFailOnly && "IsFailOnly is only valid when the comparison is ==");10031 10032    // Reverse the ordop as the OpenMP forms are different from LLVM forms.10033    // Let's take max as example.10034    // OpenMP form:10035    // x = x > expr ? expr : x;10036    // LLVM form:10037    // *ptr = *ptr > val ? *ptr : val;10038    // We need to transform to LLVM form.10039    // x = x <= expr ? x : expr;10040    AtomicRMWInst::BinOp NewOp;10041    if (IsXBinopExpr) {10042      if (IsInteger) {10043        if (X.IsSigned)10044          NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Min10045                                                : AtomicRMWInst::Max;10046        else10047          NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMin10048                                                : AtomicRMWInst::UMax;10049      } else {10050        NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::FMin10051                                              : AtomicRMWInst::FMax;10052      }10053    } else {10054      if (IsInteger) {10055        if (X.IsSigned)10056          NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Max10057                                                : AtomicRMWInst::Min;10058        else10059          NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMax10060                                                : AtomicRMWInst::UMin;10061      } else {10062        NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::FMax10063                                              : AtomicRMWInst::FMin;10064      }10065    }10066 10067    AtomicRMWInst *OldValue =10068        Builder.CreateAtomicRMW(NewOp, X.Var, E, MaybeAlign(), AO);10069    if (V.Var) {10070      Value *CapturedValue = nullptr;10071      if (IsPostfixUpdate) {10072        CapturedValue = OldValue;10073      } else {10074        CmpInst::Predicate Pred;10075        switch (NewOp) {10076        case AtomicRMWInst::Max:10077          Pred = CmpInst::ICMP_SGT;10078          break;10079        case AtomicRMWInst::UMax:10080          Pred = CmpInst::ICMP_UGT;10081          break;10082        case AtomicRMWInst::FMax:10083          Pred = CmpInst::FCMP_OGT;10084          break;10085        case AtomicRMWInst::Min:10086          Pred = CmpInst::ICMP_SLT;10087          break;10088        case AtomicRMWInst::UMin:10089          Pred = CmpInst::ICMP_ULT;10090          break;10091        case AtomicRMWInst::FMin:10092          Pred = CmpInst::FCMP_OLT;10093          break;10094        default:10095          llvm_unreachable("unexpected comparison op");10096        }10097        Value *NonAtomicCmp = Builder.CreateCmp(Pred, OldValue, E);10098        CapturedValue = Builder.CreateSelect(NonAtomicCmp, E, OldValue);10099      }10100      Builder.CreateStore(CapturedValue, V.Var, V.IsVolatile);10101    }10102  }10103 10104  checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Compare);10105 10106  return Builder.saveIP();10107}10108 10109OpenMPIRBuilder::InsertPointOrErrorTy10110OpenMPIRBuilder::createTeams(const LocationDescription &Loc,10111                             BodyGenCallbackTy BodyGenCB, Value *NumTeamsLower,10112                             Value *NumTeamsUpper, Value *ThreadLimit,10113                             Value *IfExpr) {10114  if (!updateToLocation(Loc))10115    return InsertPointTy();10116 10117  uint32_t SrcLocStrSize;10118  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);10119  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);10120  Function *CurrentFunction = Builder.GetInsertBlock()->getParent();10121 10122  // Outer allocation basicblock is the entry block of the current function.10123  BasicBlock &OuterAllocaBB = CurrentFunction->getEntryBlock();10124  if (&OuterAllocaBB == Builder.GetInsertBlock()) {10125    BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "teams.entry");10126    Builder.SetInsertPoint(BodyBB, BodyBB->begin());10127  }10128 10129  // The current basic block is split into four basic blocks. After outlining,10130  // they will be mapped as follows:10131  // ```10132  // def current_fn() {10133  //   current_basic_block:10134  //     br label %teams.exit10135  //   teams.exit:10136  //     ; instructions after teams10137  // }10138  //10139  // def outlined_fn() {10140  //   teams.alloca:10141  //     br label %teams.body10142  //   teams.body:10143  //     ; instructions within teams body10144  // }10145  // ```10146  BasicBlock *ExitBB = splitBB(Builder, /*CreateBranch=*/true, "teams.exit");10147  BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "teams.body");10148  BasicBlock *AllocaBB =10149      splitBB(Builder, /*CreateBranch=*/true, "teams.alloca");10150 10151  bool SubClausesPresent =10152      (NumTeamsLower || NumTeamsUpper || ThreadLimit || IfExpr);10153  // Push num_teams10154  if (!Config.isTargetDevice() && SubClausesPresent) {10155    assert((NumTeamsLower == nullptr || NumTeamsUpper != nullptr) &&10156           "if lowerbound is non-null, then upperbound must also be non-null "10157           "for bounds on num_teams");10158 10159    if (NumTeamsUpper == nullptr)10160      NumTeamsUpper = Builder.getInt32(0);10161 10162    if (NumTeamsLower == nullptr)10163      NumTeamsLower = NumTeamsUpper;10164 10165    if (IfExpr) {10166      assert(IfExpr->getType()->isIntegerTy() &&10167             "argument to if clause must be an integer value");10168 10169      // upper = ifexpr ? upper : 110170      if (IfExpr->getType() != Int1)10171        IfExpr = Builder.CreateICmpNE(IfExpr,10172                                      ConstantInt::get(IfExpr->getType(), 0));10173      NumTeamsUpper = Builder.CreateSelect(10174          IfExpr, NumTeamsUpper, Builder.getInt32(1), "numTeamsUpper");10175 10176      // lower = ifexpr ? lower : 110177      NumTeamsLower = Builder.CreateSelect(10178          IfExpr, NumTeamsLower, Builder.getInt32(1), "numTeamsLower");10179    }10180 10181    if (ThreadLimit == nullptr)10182      ThreadLimit = Builder.getInt32(0);10183 10184    Value *ThreadNum = getOrCreateThreadID(Ident);10185    createRuntimeFunctionCall(10186        getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_num_teams_51),10187        {Ident, ThreadNum, NumTeamsLower, NumTeamsUpper, ThreadLimit});10188  }10189  // Generate the body of teams.10190  InsertPointTy AllocaIP(AllocaBB, AllocaBB->begin());10191  InsertPointTy CodeGenIP(BodyBB, BodyBB->begin());10192  if (Error Err = BodyGenCB(AllocaIP, CodeGenIP))10193    return Err;10194 10195  OutlineInfo OI;10196  OI.EntryBB = AllocaBB;10197  OI.ExitBB = ExitBB;10198  OI.OuterAllocaBB = &OuterAllocaBB;10199 10200  // Insert fake values for global tid and bound tid.10201  SmallVector<Instruction *, 8> ToBeDeleted;10202  InsertPointTy OuterAllocaIP(&OuterAllocaBB, OuterAllocaBB.begin());10203  OI.ExcludeArgsFromAggregate.push_back(createFakeIntVal(10204      Builder, OuterAllocaIP, ToBeDeleted, AllocaIP, "gid", true));10205  OI.ExcludeArgsFromAggregate.push_back(createFakeIntVal(10206      Builder, OuterAllocaIP, ToBeDeleted, AllocaIP, "tid", true));10207 10208  auto HostPostOutlineCB = [this, Ident,10209                            ToBeDeleted](Function &OutlinedFn) mutable {10210    // The stale call instruction will be replaced with a new call instruction10211    // for runtime call with the outlined function.10212 10213    assert(OutlinedFn.hasOneUse() &&10214           "there must be a single user for the outlined function");10215    CallInst *StaleCI = cast<CallInst>(OutlinedFn.user_back());10216    ToBeDeleted.push_back(StaleCI);10217 10218    assert((OutlinedFn.arg_size() == 2 || OutlinedFn.arg_size() == 3) &&10219           "Outlined function must have two or three arguments only");10220 10221    bool HasShared = OutlinedFn.arg_size() == 3;10222 10223    OutlinedFn.getArg(0)->setName("global.tid.ptr");10224    OutlinedFn.getArg(1)->setName("bound.tid.ptr");10225    if (HasShared)10226      OutlinedFn.getArg(2)->setName("data");10227 10228    // Call to the runtime function for teams in the current function.10229    assert(StaleCI && "Error while outlining - no CallInst user found for the "10230                      "outlined function.");10231    Builder.SetInsertPoint(StaleCI);10232    SmallVector<Value *> Args = {10233        Ident, Builder.getInt32(StaleCI->arg_size() - 2), &OutlinedFn};10234    if (HasShared)10235      Args.push_back(StaleCI->getArgOperand(2));10236    createRuntimeFunctionCall(10237        getOrCreateRuntimeFunctionPtr(10238            omp::RuntimeFunction::OMPRTL___kmpc_fork_teams),10239        Args);10240 10241    for (Instruction *I : llvm::reverse(ToBeDeleted))10242      I->eraseFromParent();10243  };10244 10245  if (!Config.isTargetDevice())10246    OI.PostOutlineCB = HostPostOutlineCB;10247 10248  addOutlineInfo(std::move(OI));10249 10250  Builder.SetInsertPoint(ExitBB, ExitBB->begin());10251 10252  return Builder.saveIP();10253}10254 10255OpenMPIRBuilder::InsertPointOrErrorTy10256OpenMPIRBuilder::createDistribute(const LocationDescription &Loc,10257                                  InsertPointTy OuterAllocaIP,10258                                  BodyGenCallbackTy BodyGenCB) {10259  if (!updateToLocation(Loc))10260    return InsertPointTy();10261 10262  BasicBlock *OuterAllocaBB = OuterAllocaIP.getBlock();10263 10264  if (OuterAllocaBB == Builder.GetInsertBlock()) {10265    BasicBlock *BodyBB =10266        splitBB(Builder, /*CreateBranch=*/true, "distribute.entry");10267    Builder.SetInsertPoint(BodyBB, BodyBB->begin());10268  }10269  BasicBlock *ExitBB =10270      splitBB(Builder, /*CreateBranch=*/true, "distribute.exit");10271  BasicBlock *BodyBB =10272      splitBB(Builder, /*CreateBranch=*/true, "distribute.body");10273  BasicBlock *AllocaBB =10274      splitBB(Builder, /*CreateBranch=*/true, "distribute.alloca");10275 10276  // Generate the body of distribute clause10277  InsertPointTy AllocaIP(AllocaBB, AllocaBB->begin());10278  InsertPointTy CodeGenIP(BodyBB, BodyBB->begin());10279  if (Error Err = BodyGenCB(AllocaIP, CodeGenIP))10280    return Err;10281 10282  // When using target we use different runtime functions which require a10283  // callback.10284  if (Config.isTargetDevice()) {10285    OutlineInfo OI;10286    OI.OuterAllocaBB = OuterAllocaIP.getBlock();10287    OI.EntryBB = AllocaBB;10288    OI.ExitBB = ExitBB;10289 10290    addOutlineInfo(std::move(OI));10291  }10292  Builder.SetInsertPoint(ExitBB, ExitBB->begin());10293 10294  return Builder.saveIP();10295}10296 10297GlobalVariable *10298OpenMPIRBuilder::createOffloadMapnames(SmallVectorImpl<llvm::Constant *> &Names,10299                                       std::string VarName) {10300  llvm::Constant *MapNamesArrayInit = llvm::ConstantArray::get(10301      llvm::ArrayType::get(llvm::PointerType::getUnqual(M.getContext()),10302                           Names.size()),10303      Names);10304  auto *MapNamesArrayGlobal = new llvm::GlobalVariable(10305      M, MapNamesArrayInit->getType(),10306      /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, MapNamesArrayInit,10307      VarName);10308  return MapNamesArrayGlobal;10309}10310 10311// Create all simple and struct types exposed by the runtime and remember10312// the llvm::PointerTypes of them for easy access later.10313void OpenMPIRBuilder::initializeTypes(Module &M) {10314  LLVMContext &Ctx = M.getContext();10315  StructType *T;10316  unsigned DefaultTargetAS = Config.getDefaultTargetAS();10317  unsigned ProgramAS = M.getDataLayout().getProgramAddressSpace();10318#define OMP_TYPE(VarName, InitValue) VarName = InitValue;10319#define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize)                             \10320  VarName##Ty = ArrayType::get(ElemTy, ArraySize);                             \10321  VarName##PtrTy = PointerType::get(Ctx, DefaultTargetAS);10322#define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...)                  \10323  VarName = FunctionType::get(ReturnType, {__VA_ARGS__}, IsVarArg);            \10324  VarName##Ptr = PointerType::get(Ctx, ProgramAS);10325#define OMP_STRUCT_TYPE(VarName, StructName, Packed, ...)                      \10326  T = StructType::getTypeByName(Ctx, StructName);                              \10327  if (!T)                                                                      \10328    T = StructType::create(Ctx, {__VA_ARGS__}, StructName, Packed);            \10329  VarName = T;                                                                 \10330  VarName##Ptr = PointerType::get(Ctx, DefaultTargetAS);10331#include "llvm/Frontend/OpenMP/OMPKinds.def"10332}10333 10334void OpenMPIRBuilder::OutlineInfo::collectBlocks(10335    SmallPtrSetImpl<BasicBlock *> &BlockSet,10336    SmallVectorImpl<BasicBlock *> &BlockVector) {10337  SmallVector<BasicBlock *, 32> Worklist;10338  BlockSet.insert(EntryBB);10339  BlockSet.insert(ExitBB);10340 10341  Worklist.push_back(EntryBB);10342  while (!Worklist.empty()) {10343    BasicBlock *BB = Worklist.pop_back_val();10344    BlockVector.push_back(BB);10345    for (BasicBlock *SuccBB : successors(BB))10346      if (BlockSet.insert(SuccBB).second)10347        Worklist.push_back(SuccBB);10348  }10349}10350 10351void OpenMPIRBuilder::createOffloadEntry(Constant *ID, Constant *Addr,10352                                         uint64_t Size, int32_t Flags,10353                                         GlobalValue::LinkageTypes,10354                                         StringRef Name) {10355  if (!Config.isGPU()) {10356    llvm::offloading::emitOffloadingEntry(10357        M, object::OffloadKind::OFK_OpenMP, ID,10358        Name.empty() ? Addr->getName() : Name, Size, Flags, /*Data=*/0);10359    return;10360  }10361  // TODO: Add support for global variables on the device after declare target10362  // support.10363  Function *Fn = dyn_cast<Function>(Addr);10364  if (!Fn)10365    return;10366 10367  // Add a function attribute for the kernel.10368  Fn->addFnAttr("kernel");10369  if (T.isAMDGCN())10370    Fn->addFnAttr("uniform-work-group-size", "true");10371  Fn->addFnAttr(Attribute::MustProgress);10372}10373 10374// We only generate metadata for function that contain target regions.10375void OpenMPIRBuilder::createOffloadEntriesAndInfoMetadata(10376    EmitMetadataErrorReportFunctionTy &ErrorFn) {10377 10378  // If there are no entries, we don't need to do anything.10379  if (OffloadInfoManager.empty())10380    return;10381 10382  LLVMContext &C = M.getContext();10383  SmallVector<std::pair<const OffloadEntriesInfoManager::OffloadEntryInfo *,10384                        TargetRegionEntryInfo>,10385              16>10386      OrderedEntries(OffloadInfoManager.size());10387 10388  // Auxiliary methods to create metadata values and strings.10389  auto &&GetMDInt = [this](unsigned V) {10390    return ConstantAsMetadata::get(ConstantInt::get(Builder.getInt32Ty(), V));10391  };10392 10393  auto &&GetMDString = [&C](StringRef V) { return MDString::get(C, V); };10394 10395  // Create the offloading info metadata node.10396  NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info");10397  auto &&TargetRegionMetadataEmitter =10398      [&C, MD, &OrderedEntries, &GetMDInt, &GetMDString](10399          const TargetRegionEntryInfo &EntryInfo,10400          const OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion &E) {10401        // Generate metadata for target regions. Each entry of this metadata10402        // contains:10403        // - Entry 0 -> Kind of this type of metadata (0).10404        // - Entry 1 -> Device ID of the file where the entry was identified.10405        // - Entry 2 -> File ID of the file where the entry was identified.10406        // - Entry 3 -> Mangled name of the function where the entry was10407        // identified.10408        // - Entry 4 -> Line in the file where the entry was identified.10409        // - Entry 5 -> Count of regions at this DeviceID/FilesID/Line.10410        // - Entry 6 -> Order the entry was created.10411        // The first element of the metadata node is the kind.10412        Metadata *Ops[] = {10413            GetMDInt(E.getKind()),      GetMDInt(EntryInfo.DeviceID),10414            GetMDInt(EntryInfo.FileID), GetMDString(EntryInfo.ParentName),10415            GetMDInt(EntryInfo.Line),   GetMDInt(EntryInfo.Count),10416            GetMDInt(E.getOrder())};10417 10418        // Save this entry in the right position of the ordered entries array.10419        OrderedEntries[E.getOrder()] = std::make_pair(&E, EntryInfo);10420 10421        // Add metadata to the named metadata node.10422        MD->addOperand(MDNode::get(C, Ops));10423      };10424 10425  OffloadInfoManager.actOnTargetRegionEntriesInfo(TargetRegionMetadataEmitter);10426 10427  // Create function that emits metadata for each device global variable entry;10428  auto &&DeviceGlobalVarMetadataEmitter =10429      [&C, &OrderedEntries, &GetMDInt, &GetMDString, MD](10430          StringRef MangledName,10431          const OffloadEntriesInfoManager::OffloadEntryInfoDeviceGlobalVar &E) {10432        // Generate metadata for global variables. Each entry of this metadata10433        // contains:10434        // - Entry 0 -> Kind of this type of metadata (1).10435        // - Entry 1 -> Mangled name of the variable.10436        // - Entry 2 -> Declare target kind.10437        // - Entry 3 -> Order the entry was created.10438        // The first element of the metadata node is the kind.10439        Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDString(MangledName),10440                           GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};10441 10442        // Save this entry in the right position of the ordered entries array.10443        TargetRegionEntryInfo varInfo(MangledName, 0, 0, 0);10444        OrderedEntries[E.getOrder()] = std::make_pair(&E, varInfo);10445 10446        // Add metadata to the named metadata node.10447        MD->addOperand(MDNode::get(C, Ops));10448      };10449 10450  OffloadInfoManager.actOnDeviceGlobalVarEntriesInfo(10451      DeviceGlobalVarMetadataEmitter);10452 10453  for (const auto &E : OrderedEntries) {10454    assert(E.first && "All ordered entries must exist!");10455    if (const auto *CE =10456            dyn_cast<OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion>(10457                E.first)) {10458      if (!CE->getID() || !CE->getAddress()) {10459        // Do not blame the entry if the parent funtion is not emitted.10460        TargetRegionEntryInfo EntryInfo = E.second;10461        StringRef FnName = EntryInfo.ParentName;10462        if (!M.getNamedValue(FnName))10463          continue;10464        ErrorFn(EMIT_MD_TARGET_REGION_ERROR, EntryInfo);10465        continue;10466      }10467      createOffloadEntry(CE->getID(), CE->getAddress(),10468                         /*Size=*/0, CE->getFlags(),10469                         GlobalValue::WeakAnyLinkage);10470    } else if (const auto *CE = dyn_cast<10471                   OffloadEntriesInfoManager::OffloadEntryInfoDeviceGlobalVar>(10472                   E.first)) {10473      OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind Flags =10474          static_cast<OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind>(10475              CE->getFlags());10476      switch (Flags) {10477      case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter:10478      case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo:10479        if (Config.isTargetDevice() && Config.hasRequiresUnifiedSharedMemory())10480          continue;10481        if (!CE->getAddress()) {10482          ErrorFn(EMIT_MD_DECLARE_TARGET_ERROR, E.second);10483          continue;10484        }10485        // The vaiable has no definition - no need to add the entry.10486        if (CE->getVarSize() == 0)10487          continue;10488        break;10489      case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink:10490        assert(((Config.isTargetDevice() && !CE->getAddress()) ||10491                (!Config.isTargetDevice() && CE->getAddress())) &&10492               "Declaret target link address is set.");10493        if (Config.isTargetDevice())10494          continue;10495        if (!CE->getAddress()) {10496          ErrorFn(EMIT_MD_GLOBAL_VAR_LINK_ERROR, TargetRegionEntryInfo());10497          continue;10498        }10499        break;10500      default:10501        break;10502      }10503 10504      // Hidden or internal symbols on the device are not externally visible.10505      // We should not attempt to register them by creating an offloading10506      // entry. Indirect variables are handled separately on the device.10507      if (auto *GV = dyn_cast<GlobalValue>(CE->getAddress()))10508        if ((GV->hasLocalLinkage() || GV->hasHiddenVisibility()) &&10509            Flags != OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect)10510          continue;10511 10512      // Indirect globals need to use a special name that doesn't match the name10513      // of the associated host global.10514      if (Flags == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect)10515        createOffloadEntry(CE->getAddress(), CE->getAddress(), CE->getVarSize(),10516                           Flags, CE->getLinkage(), CE->getVarName());10517      else10518        createOffloadEntry(CE->getAddress(), CE->getAddress(), CE->getVarSize(),10519                           Flags, CE->getLinkage());10520 10521    } else {10522      llvm_unreachable("Unsupported entry kind.");10523    }10524  }10525 10526  // Emit requires directive globals to a special entry so the runtime can10527  // register them when the device image is loaded.10528  // TODO: This reduces the offloading entries to a 32-bit integer. Offloading10529  //       entries should be redesigned to better suit this use-case.10530  if (Config.hasRequiresFlags() && !Config.isTargetDevice())10531    offloading::emitOffloadingEntry(10532        M, object::OffloadKind::OFK_OpenMP,10533        Constant::getNullValue(PointerType::getUnqual(M.getContext())),10534        ".requires", /*Size=*/0,10535        OffloadEntriesInfoManager::OMPTargetGlobalRegisterRequires,10536        Config.getRequiresFlags());10537}10538 10539void TargetRegionEntryInfo::getTargetRegionEntryFnName(10540    SmallVectorImpl<char> &Name, StringRef ParentName, unsigned DeviceID,10541    unsigned FileID, unsigned Line, unsigned Count) {10542  raw_svector_ostream OS(Name);10543  OS << KernelNamePrefix << llvm::format("%x", DeviceID)10544     << llvm::format("_%x_", FileID) << ParentName << "_l" << Line;10545  if (Count)10546    OS << "_" << Count;10547}10548 10549void OffloadEntriesInfoManager::getTargetRegionEntryFnName(10550    SmallVectorImpl<char> &Name, const TargetRegionEntryInfo &EntryInfo) {10551  unsigned NewCount = getTargetRegionEntryInfoCount(EntryInfo);10552  TargetRegionEntryInfo::getTargetRegionEntryFnName(10553      Name, EntryInfo.ParentName, EntryInfo.DeviceID, EntryInfo.FileID,10554      EntryInfo.Line, NewCount);10555}10556 10557TargetRegionEntryInfo10558OpenMPIRBuilder::getTargetEntryUniqueInfo(FileIdentifierInfoCallbackTy CallBack,10559                                          vfs::FileSystem &VFS,10560                                          StringRef ParentName) {10561  sys::fs::UniqueID ID(0xdeadf17e, 0);10562  auto FileIDInfo = CallBack();10563  uint64_t FileID = 0;10564  if (ErrorOr<vfs::Status> Status = VFS.status(std::get<0>(FileIDInfo))) {10565    ID = Status->getUniqueID();10566    FileID = Status->getUniqueID().getFile();10567  } else {10568    // If the inode ID could not be determined, create a hash value10569    // the current file name and use that as an ID.10570    FileID = hash_value(std::get<0>(FileIDInfo));10571  }10572 10573  return TargetRegionEntryInfo(ParentName, ID.getDevice(), FileID,10574                               std::get<1>(FileIDInfo));10575}10576 10577unsigned OpenMPIRBuilder::getFlagMemberOffset() {10578  unsigned Offset = 0;10579  for (uint64_t Remain =10580           static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(10581               omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);10582       !(Remain & 1); Remain = Remain >> 1)10583    Offset++;10584  return Offset;10585}10586 10587omp::OpenMPOffloadMappingFlags10588OpenMPIRBuilder::getMemberOfFlag(unsigned Position) {10589  // Rotate by getFlagMemberOffset() bits.10590  return static_cast<omp::OpenMPOffloadMappingFlags>(((uint64_t)Position + 1)10591                                                     << getFlagMemberOffset());10592}10593 10594void OpenMPIRBuilder::setCorrectMemberOfFlag(10595    omp::OpenMPOffloadMappingFlags &Flags,10596    omp::OpenMPOffloadMappingFlags MemberOfFlag) {10597  // If the entry is PTR_AND_OBJ but has not been marked with the special10598  // placeholder value 0xFFFF in the MEMBER_OF field, then it should not be10599  // marked as MEMBER_OF.10600  if (static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(10601          Flags & omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ) &&10602      static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(10603          (Flags & omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=10604          omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF))10605    return;10606 10607  // Reset the placeholder value to prepare the flag for the assignment of the10608  // proper MEMBER_OF value.10609  Flags &= ~omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;10610  Flags |= MemberOfFlag;10611}10612 10613Constant *OpenMPIRBuilder::getAddrOfDeclareTargetVar(10614    OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind CaptureClause,10615    OffloadEntriesInfoManager::OMPTargetDeviceClauseKind DeviceClause,10616    bool IsDeclaration, bool IsExternallyVisible,10617    TargetRegionEntryInfo EntryInfo, StringRef MangledName,10618    std::vector<GlobalVariable *> &GeneratedRefs, bool OpenMPSIMD,10619    std::vector<Triple> TargetTriple, Type *LlvmPtrTy,10620    std::function<Constant *()> GlobalInitializer,10621    std::function<GlobalValue::LinkageTypes()> VariableLinkage) {10622  // TODO: convert this to utilise the IRBuilder Config rather than10623  // a passed down argument.10624  if (OpenMPSIMD)10625    return nullptr;10626 10627  if (CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink ||10628      ((CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo ||10629        CaptureClause ==10630            OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter) &&10631       Config.hasRequiresUnifiedSharedMemory())) {10632    SmallString<64> PtrName;10633    {10634      raw_svector_ostream OS(PtrName);10635      OS << MangledName;10636      if (!IsExternallyVisible)10637        OS << format("_%x", EntryInfo.FileID);10638      OS << "_decl_tgt_ref_ptr";10639    }10640 10641    Value *Ptr = M.getNamedValue(PtrName);10642 10643    if (!Ptr) {10644      GlobalValue *GlobalValue = M.getNamedValue(MangledName);10645      Ptr = getOrCreateInternalVariable(LlvmPtrTy, PtrName);10646 10647      auto *GV = cast<GlobalVariable>(Ptr);10648      GV->setLinkage(GlobalValue::WeakAnyLinkage);10649 10650      if (!Config.isTargetDevice()) {10651        if (GlobalInitializer)10652          GV->setInitializer(GlobalInitializer());10653        else10654          GV->setInitializer(GlobalValue);10655      }10656 10657      registerTargetGlobalVariable(10658          CaptureClause, DeviceClause, IsDeclaration, IsExternallyVisible,10659          EntryInfo, MangledName, GeneratedRefs, OpenMPSIMD, TargetTriple,10660          GlobalInitializer, VariableLinkage, LlvmPtrTy, cast<Constant>(Ptr));10661    }10662 10663    return cast<Constant>(Ptr);10664  }10665 10666  return nullptr;10667}10668 10669void OpenMPIRBuilder::registerTargetGlobalVariable(10670    OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind CaptureClause,10671    OffloadEntriesInfoManager::OMPTargetDeviceClauseKind DeviceClause,10672    bool IsDeclaration, bool IsExternallyVisible,10673    TargetRegionEntryInfo EntryInfo, StringRef MangledName,10674    std::vector<GlobalVariable *> &GeneratedRefs, bool OpenMPSIMD,10675    std::vector<Triple> TargetTriple,10676    std::function<Constant *()> GlobalInitializer,10677    std::function<GlobalValue::LinkageTypes()> VariableLinkage, Type *LlvmPtrTy,10678    Constant *Addr) {10679  if (DeviceClause != OffloadEntriesInfoManager::OMPTargetDeviceClauseAny ||10680      (TargetTriple.empty() && !Config.isTargetDevice()))10681    return;10682 10683  OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind Flags;10684  StringRef VarName;10685  int64_t VarSize;10686  GlobalValue::LinkageTypes Linkage;10687 10688  if ((CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo ||10689       CaptureClause ==10690           OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter) &&10691      !Config.hasRequiresUnifiedSharedMemory()) {10692    Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;10693    VarName = MangledName;10694    GlobalValue *LlvmVal = M.getNamedValue(VarName);10695 10696    if (!IsDeclaration)10697      VarSize = divideCeil(10698          M.getDataLayout().getTypeSizeInBits(LlvmVal->getValueType()), 8);10699    else10700      VarSize = 0;10701    Linkage = (VariableLinkage) ? VariableLinkage() : LlvmVal->getLinkage();10702 10703    // This is a workaround carried over from Clang which prevents undesired10704    // optimisation of internal variables.10705    if (Config.isTargetDevice() &&10706        (!IsExternallyVisible || Linkage == GlobalValue::LinkOnceODRLinkage)) {10707      // Do not create a "ref-variable" if the original is not also available10708      // on the host.10709      if (!OffloadInfoManager.hasDeviceGlobalVarEntryInfo(VarName))10710        return;10711 10712      std::string RefName = createPlatformSpecificName({VarName, "ref"});10713 10714      if (!M.getNamedValue(RefName)) {10715        Constant *AddrRef =10716            getOrCreateInternalVariable(Addr->getType(), RefName);10717        auto *GvAddrRef = cast<GlobalVariable>(AddrRef);10718        GvAddrRef->setConstant(true);10719        GvAddrRef->setLinkage(GlobalValue::InternalLinkage);10720        GvAddrRef->setInitializer(Addr);10721        GeneratedRefs.push_back(GvAddrRef);10722      }10723    }10724  } else {10725    if (CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink)10726      Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;10727    else10728      Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;10729 10730    if (Config.isTargetDevice()) {10731      VarName = (Addr) ? Addr->getName() : "";10732      Addr = nullptr;10733    } else {10734      Addr = getAddrOfDeclareTargetVar(10735          CaptureClause, DeviceClause, IsDeclaration, IsExternallyVisible,10736          EntryInfo, MangledName, GeneratedRefs, OpenMPSIMD, TargetTriple,10737          LlvmPtrTy, GlobalInitializer, VariableLinkage);10738      VarName = (Addr) ? Addr->getName() : "";10739    }10740    VarSize = M.getDataLayout().getPointerSize();10741    Linkage = GlobalValue::WeakAnyLinkage;10742  }10743 10744  OffloadInfoManager.registerDeviceGlobalVarEntryInfo(VarName, Addr, VarSize,10745                                                      Flags, Linkage);10746}10747 10748/// Loads all the offload entries information from the host IR10749/// metadata.10750void OpenMPIRBuilder::loadOffloadInfoMetadata(Module &M) {10751  // If we are in target mode, load the metadata from the host IR. This code has10752  // to match the metadata creation in createOffloadEntriesAndInfoMetadata().10753 10754  NamedMDNode *MD = M.getNamedMetadata(ompOffloadInfoName);10755  if (!MD)10756    return;10757 10758  for (MDNode *MN : MD->operands()) {10759    auto &&GetMDInt = [MN](unsigned Idx) {10760      auto *V = cast<ConstantAsMetadata>(MN->getOperand(Idx));10761      return cast<ConstantInt>(V->getValue())->getZExtValue();10762    };10763 10764    auto &&GetMDString = [MN](unsigned Idx) {10765      auto *V = cast<MDString>(MN->getOperand(Idx));10766      return V->getString();10767    };10768 10769    switch (GetMDInt(0)) {10770    default:10771      llvm_unreachable("Unexpected metadata!");10772      break;10773    case OffloadEntriesInfoManager::OffloadEntryInfo::10774        OffloadingEntryInfoTargetRegion: {10775      TargetRegionEntryInfo EntryInfo(/*ParentName=*/GetMDString(3),10776                                      /*DeviceID=*/GetMDInt(1),10777                                      /*FileID=*/GetMDInt(2),10778                                      /*Line=*/GetMDInt(4),10779                                      /*Count=*/GetMDInt(5));10780      OffloadInfoManager.initializeTargetRegionEntryInfo(EntryInfo,10781                                                         /*Order=*/GetMDInt(6));10782      break;10783    }10784    case OffloadEntriesInfoManager::OffloadEntryInfo::10785        OffloadingEntryInfoDeviceGlobalVar:10786      OffloadInfoManager.initializeDeviceGlobalVarEntryInfo(10787          /*MangledName=*/GetMDString(1),10788          static_cast<OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind>(10789              /*Flags=*/GetMDInt(2)),10790          /*Order=*/GetMDInt(3));10791      break;10792    }10793  }10794}10795 10796void OpenMPIRBuilder::loadOffloadInfoMetadata(vfs::FileSystem &VFS,10797                                              StringRef HostFilePath) {10798  if (HostFilePath.empty())10799    return;10800 10801  auto Buf = VFS.getBufferForFile(HostFilePath);10802  if (std::error_code Err = Buf.getError()) {10803    report_fatal_error(("error opening host file from host file path inside of "10804                        "OpenMPIRBuilder: " +10805                        Err.message())10806                           .c_str());10807  }10808 10809  LLVMContext Ctx;10810  auto M = expectedToErrorOrAndEmitErrors(10811      Ctx, parseBitcodeFile(Buf.get()->getMemBufferRef(), Ctx));10812  if (std::error_code Err = M.getError()) {10813    report_fatal_error(10814        ("error parsing host file inside of OpenMPIRBuilder: " + Err.message())10815            .c_str());10816  }10817 10818  loadOffloadInfoMetadata(*M.get());10819}10820 10821//===----------------------------------------------------------------------===//10822// OffloadEntriesInfoManager10823//===----------------------------------------------------------------------===//10824 10825bool OffloadEntriesInfoManager::empty() const {10826  return OffloadEntriesTargetRegion.empty() &&10827         OffloadEntriesDeviceGlobalVar.empty();10828}10829 10830unsigned OffloadEntriesInfoManager::getTargetRegionEntryInfoCount(10831    const TargetRegionEntryInfo &EntryInfo) const {10832  auto It = OffloadEntriesTargetRegionCount.find(10833      getTargetRegionEntryCountKey(EntryInfo));10834  if (It == OffloadEntriesTargetRegionCount.end())10835    return 0;10836  return It->second;10837}10838 10839void OffloadEntriesInfoManager::incrementTargetRegionEntryInfoCount(10840    const TargetRegionEntryInfo &EntryInfo) {10841  OffloadEntriesTargetRegionCount[getTargetRegionEntryCountKey(EntryInfo)] =10842      EntryInfo.Count + 1;10843}10844 10845/// Initialize target region entry.10846void OffloadEntriesInfoManager::initializeTargetRegionEntryInfo(10847    const TargetRegionEntryInfo &EntryInfo, unsigned Order) {10848  OffloadEntriesTargetRegion[EntryInfo] =10849      OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,10850                                   OMPTargetRegionEntryTargetRegion);10851  ++OffloadingEntriesNum;10852}10853 10854void OffloadEntriesInfoManager::registerTargetRegionEntryInfo(10855    TargetRegionEntryInfo EntryInfo, Constant *Addr, Constant *ID,10856    OMPTargetRegionEntryKind Flags) {10857  assert(EntryInfo.Count == 0 && "expected default EntryInfo");10858 10859  // Update the EntryInfo with the next available count for this location.10860  EntryInfo.Count = getTargetRegionEntryInfoCount(EntryInfo);10861 10862  // If we are emitting code for a target, the entry is already initialized,10863  // only has to be registered.10864  if (OMPBuilder->Config.isTargetDevice()) {10865    // This could happen if the device compilation is invoked standalone.10866    if (!hasTargetRegionEntryInfo(EntryInfo)) {10867      return;10868    }10869    auto &Entry = OffloadEntriesTargetRegion[EntryInfo];10870    Entry.setAddress(Addr);10871    Entry.setID(ID);10872    Entry.setFlags(Flags);10873  } else {10874    if (Flags == OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion &&10875        hasTargetRegionEntryInfo(EntryInfo, /*IgnoreAddressId*/ true))10876      return;10877    assert(!hasTargetRegionEntryInfo(EntryInfo) &&10878           "Target region entry already registered!");10879    OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);10880    OffloadEntriesTargetRegion[EntryInfo] = Entry;10881    ++OffloadingEntriesNum;10882  }10883  incrementTargetRegionEntryInfoCount(EntryInfo);10884}10885 10886bool OffloadEntriesInfoManager::hasTargetRegionEntryInfo(10887    TargetRegionEntryInfo EntryInfo, bool IgnoreAddressId) const {10888 10889  // Update the EntryInfo with the next available count for this location.10890  EntryInfo.Count = getTargetRegionEntryInfoCount(EntryInfo);10891 10892  auto It = OffloadEntriesTargetRegion.find(EntryInfo);10893  if (It == OffloadEntriesTargetRegion.end()) {10894    return false;10895  }10896  // Fail if this entry is already registered.10897  if (!IgnoreAddressId && (It->second.getAddress() || It->second.getID()))10898    return false;10899  return true;10900}10901 10902void OffloadEntriesInfoManager::actOnTargetRegionEntriesInfo(10903    const OffloadTargetRegionEntryInfoActTy &Action) {10904  // Scan all target region entries and perform the provided action.10905  for (const auto &It : OffloadEntriesTargetRegion) {10906    Action(It.first, It.second);10907  }10908}10909 10910void OffloadEntriesInfoManager::initializeDeviceGlobalVarEntryInfo(10911    StringRef Name, OMPTargetGlobalVarEntryKind Flags, unsigned Order) {10912  OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags);10913  ++OffloadingEntriesNum;10914}10915 10916void OffloadEntriesInfoManager::registerDeviceGlobalVarEntryInfo(10917    StringRef VarName, Constant *Addr, int64_t VarSize,10918    OMPTargetGlobalVarEntryKind Flags, GlobalValue::LinkageTypes Linkage) {10919  if (OMPBuilder->Config.isTargetDevice()) {10920    // This could happen if the device compilation is invoked standalone.10921    if (!hasDeviceGlobalVarEntryInfo(VarName))10922      return;10923    auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];10924    if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {10925      if (Entry.getVarSize() == 0) {10926        Entry.setVarSize(VarSize);10927        Entry.setLinkage(Linkage);10928      }10929      return;10930    }10931    Entry.setVarSize(VarSize);10932    Entry.setLinkage(Linkage);10933    Entry.setAddress(Addr);10934  } else {10935    if (hasDeviceGlobalVarEntryInfo(VarName)) {10936      auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];10937      assert(Entry.isValid() && Entry.getFlags() == Flags &&10938             "Entry not initialized!");10939      if (Entry.getVarSize() == 0) {10940        Entry.setVarSize(VarSize);10941        Entry.setLinkage(Linkage);10942      }10943      return;10944    }10945    if (Flags == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect)10946      OffloadEntriesDeviceGlobalVar.try_emplace(VarName, OffloadingEntriesNum,10947                                                Addr, VarSize, Flags, Linkage,10948                                                VarName.str());10949    else10950      OffloadEntriesDeviceGlobalVar.try_emplace(10951          VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage, "");10952    ++OffloadingEntriesNum;10953  }10954}10955 10956void OffloadEntriesInfoManager::actOnDeviceGlobalVarEntriesInfo(10957    const OffloadDeviceGlobalVarEntryInfoActTy &Action) {10958  // Scan all target region entries and perform the provided action.10959  for (const auto &E : OffloadEntriesDeviceGlobalVar)10960    Action(E.getKey(), E.getValue());10961}10962 10963//===----------------------------------------------------------------------===//10964// CanonicalLoopInfo10965//===----------------------------------------------------------------------===//10966 10967void CanonicalLoopInfo::collectControlBlocks(10968    SmallVectorImpl<BasicBlock *> &BBs) {10969  // We only count those BBs as control block for which we do not need to10970  // reverse the CFG, i.e. not the loop body which can contain arbitrary control10971  // flow. For consistency, this also means we do not add the Body block, which10972  // is just the entry to the body code.10973  BBs.reserve(BBs.size() + 6);10974  BBs.append({getPreheader(), Header, Cond, Latch, Exit, getAfter()});10975}10976 10977BasicBlock *CanonicalLoopInfo::getPreheader() const {10978  assert(isValid() && "Requires a valid canonical loop");10979  for (BasicBlock *Pred : predecessors(Header)) {10980    if (Pred != Latch)10981      return Pred;10982  }10983  llvm_unreachable("Missing preheader");10984}10985 10986void CanonicalLoopInfo::setTripCount(Value *TripCount) {10987  assert(isValid() && "Requires a valid canonical loop");10988 10989  Instruction *CmpI = &getCond()->front();10990  assert(isa<CmpInst>(CmpI) && "First inst must compare IV with TripCount");10991  CmpI->setOperand(1, TripCount);10992 10993#ifndef NDEBUG10994  assertOK();10995#endif10996}10997 10998void CanonicalLoopInfo::mapIndVar(10999    llvm::function_ref<Value *(Instruction *)> Updater) {11000  assert(isValid() && "Requires a valid canonical loop");11001 11002  Instruction *OldIV = getIndVar();11003 11004  // Record all uses excluding those introduced by the updater. Uses by the11005  // CanonicalLoopInfo itself to keep track of the number of iterations are11006  // excluded.11007  SmallVector<Use *> ReplacableUses;11008  for (Use &U : OldIV->uses()) {11009    auto *User = dyn_cast<Instruction>(U.getUser());11010    if (!User)11011      continue;11012    if (User->getParent() == getCond())11013      continue;11014    if (User->getParent() == getLatch())11015      continue;11016    ReplacableUses.push_back(&U);11017  }11018 11019  // Run the updater that may introduce new uses11020  Value *NewIV = Updater(OldIV);11021 11022  // Replace the old uses with the value returned by the updater.11023  for (Use *U : ReplacableUses)11024    U->set(NewIV);11025 11026#ifndef NDEBUG11027  assertOK();11028#endif11029}11030 11031void CanonicalLoopInfo::assertOK() const {11032#ifndef NDEBUG11033  // No constraints if this object currently does not describe a loop.11034  if (!isValid())11035    return;11036 11037  BasicBlock *Preheader = getPreheader();11038  BasicBlock *Body = getBody();11039  BasicBlock *After = getAfter();11040 11041  // Verify standard control-flow we use for OpenMP loops.11042  assert(Preheader);11043  assert(isa<BranchInst>(Preheader->getTerminator()) &&11044         "Preheader must terminate with unconditional branch");11045  assert(Preheader->getSingleSuccessor() == Header &&11046         "Preheader must jump to header");11047 11048  assert(Header);11049  assert(isa<BranchInst>(Header->getTerminator()) &&11050         "Header must terminate with unconditional branch");11051  assert(Header->getSingleSuccessor() == Cond &&11052         "Header must jump to exiting block");11053 11054  assert(Cond);11055  assert(Cond->getSinglePredecessor() == Header &&11056         "Exiting block only reachable from header");11057 11058  assert(isa<BranchInst>(Cond->getTerminator()) &&11059         "Exiting block must terminate with conditional branch");11060  assert(size(successors(Cond)) == 2 &&11061         "Exiting block must have two successors");11062  assert(cast<BranchInst>(Cond->getTerminator())->getSuccessor(0) == Body &&11063         "Exiting block's first successor jump to the body");11064  assert(cast<BranchInst>(Cond->getTerminator())->getSuccessor(1) == Exit &&11065         "Exiting block's second successor must exit the loop");11066 11067  assert(Body);11068  assert(Body->getSinglePredecessor() == Cond &&11069         "Body only reachable from exiting block");11070  assert(!isa<PHINode>(Body->front()));11071 11072  assert(Latch);11073  assert(isa<BranchInst>(Latch->getTerminator()) &&11074         "Latch must terminate with unconditional branch");11075  assert(Latch->getSingleSuccessor() == Header && "Latch must jump to header");11076  // TODO: To support simple redirecting of the end of the body code that has11077  // multiple; introduce another auxiliary basic block like preheader and after.11078  assert(Latch->getSinglePredecessor() != nullptr);11079  assert(!isa<PHINode>(Latch->front()));11080 11081  assert(Exit);11082  assert(isa<BranchInst>(Exit->getTerminator()) &&11083         "Exit block must terminate with unconditional branch");11084  assert(Exit->getSingleSuccessor() == After &&11085         "Exit block must jump to after block");11086 11087  assert(After);11088  assert(After->getSinglePredecessor() == Exit &&11089         "After block only reachable from exit block");11090  assert(After->empty() || !isa<PHINode>(After->front()));11091 11092  Instruction *IndVar = getIndVar();11093  assert(IndVar && "Canonical induction variable not found?");11094  assert(isa<IntegerType>(IndVar->getType()) &&11095         "Induction variable must be an integer");11096  assert(cast<PHINode>(IndVar)->getParent() == Header &&11097         "Induction variable must be a PHI in the loop header");11098  assert(cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader);11099  assert(11100      cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero());11101  assert(cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch);11102 11103  auto *NextIndVar = cast<PHINode>(IndVar)->getIncomingValue(1);11104  assert(cast<Instruction>(NextIndVar)->getParent() == Latch);11105  assert(cast<BinaryOperator>(NextIndVar)->getOpcode() == BinaryOperator::Add);11106  assert(cast<BinaryOperator>(NextIndVar)->getOperand(0) == IndVar);11107  assert(cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1))11108             ->isOne());11109 11110  Value *TripCount = getTripCount();11111  assert(TripCount && "Loop trip count not found?");11112  assert(IndVar->getType() == TripCount->getType() &&11113         "Trip count and induction variable must have the same type");11114 11115  auto *CmpI = cast<CmpInst>(&Cond->front());11116  assert(CmpI->getPredicate() == CmpInst::ICMP_ULT &&11117         "Exit condition must be a signed less-than comparison");11118  assert(CmpI->getOperand(0) == IndVar &&11119         "Exit condition must compare the induction variable");11120  assert(CmpI->getOperand(1) == TripCount &&11121         "Exit condition must compare with the trip count");11122#endif11123}11124 11125void CanonicalLoopInfo::invalidate() {11126  Header = nullptr;11127  Cond = nullptr;11128  Latch = nullptr;11129  Exit = nullptr;11130}11131