brintos / llvm-project-archived public Read only

1//===- llvm/unittest/IR/OpenMPIRBuilderTest.cpp - OpenMPIRBuilder tests ---===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/Frontend/OpenMP/OMPConstants.h"10#include "llvm/Frontend/OpenMP/OMPDeviceConstants.h"11#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"12#include "llvm/IR/BasicBlock.h"13#include "llvm/IR/DIBuilder.h"14#include "llvm/IR/DebugInfoMetadata.h"15#include "llvm/IR/Function.h"16#include "llvm/IR/InstIterator.h"17#include "llvm/IR/Instructions.h"18#include "llvm/IR/LLVMContext.h"19#include "llvm/IR/Module.h"20#include "llvm/IR/Verifier.h"21#include "llvm/Passes/PassBuilder.h"22#include "llvm/Support/Casting.h"23#include "llvm/Testing/Support/Error.h"24#include "llvm/Transforms/Utils/BasicBlockUtils.h"25#include "gmock/gmock.h"26#include "gtest/gtest.h"27#include <cstdlib>28#include <optional>29 30using namespace llvm;31using namespace omp;32 33// Helper that intends to be functionally equivalent to `VarType VarName = Init`34// for an `Init` that returns an `Expected<VarType>` value. It produces an error35// message and returns if `Init` didn't produce a valid result.36#define ASSERT_EXPECTED_INIT(VarType, VarName, Init)                           \37  auto __Expected##VarName = Init;                                             \38  ASSERT_THAT_EXPECTED(__Expected##VarName, Succeeded());                      \39  VarType VarName = *__Expected##VarName40 41// Similar to ASSERT_EXPECTED_INIT, but returns a given expression in case of42// error after printing the error message.43#define ASSERT_EXPECTED_INIT_RETURN(VarType, VarName, Init, Return)            \44  auto __Expected##VarName = Init;                                             \45  EXPECT_THAT_EXPECTED(__Expected##VarName, Succeeded());                      \46  if (!__Expected##VarName)                                                    \47    return Return;                                                             \48  VarType VarName = *__Expected##VarName49 50// Wrapper lambdas to allow using EXPECT*() macros inside of error-returning51// callbacks.52#define FINICB_WRAPPER(cb)                                                     \53  [&cb](InsertPointTy IP) -> Error {                                           \54    cb(IP);                                                                    \55    return Error::success();                                                   \56  }57 58#define BODYGENCB_WRAPPER(cb)                                                  \59  [&cb](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) -> Error {            \60    cb(AllocaIP, CodeGenIP);                                                   \61    return Error::success();                                                   \62  }63 64#define LOOP_BODYGENCB_WRAPPER(cb)                                             \65  [&cb](InsertPointTy CodeGenIP, Value *LC) -> Error {                         \66    cb(CodeGenIP, LC);                                                         \67    return Error::success();                                                   \68  }69 70namespace {71 72/// Create an instruction that uses the values in \p Values. We use "printf"73/// just because it is often used for this purpose in test code, but it is never74/// executed here.75static CallInst *createPrintfCall(IRBuilder<> &Builder, StringRef FormatStr,76                                  ArrayRef<Value *> Values) {77  Module *M = Builder.GetInsertBlock()->getParent()->getParent();78 79  GlobalVariable *GV = Builder.CreateGlobalString(FormatStr, "", 0, M);80  Constant *Zero = ConstantInt::get(Type::getInt32Ty(M->getContext()), 0);81  Constant *Indices[] = {Zero, Zero};82  Constant *FormatStrConst =83      ConstantExpr::getInBoundsGetElementPtr(GV->getValueType(), GV, Indices);84 85  Function *PrintfDecl = M->getFunction("printf");86  if (!PrintfDecl) {87    GlobalValue::LinkageTypes Linkage = Function::ExternalLinkage;88    FunctionType *Ty = FunctionType::get(Builder.getInt32Ty(), true);89    PrintfDecl = Function::Create(Ty, Linkage, "printf", M);90  }91 92  SmallVector<Value *, 4> Args;93  Args.push_back(FormatStrConst);94  Args.append(Values.begin(), Values.end());95  return Builder.CreateCall(PrintfDecl, Args);96}97 98/// Verify that blocks in \p RefOrder are corresponds to the depth-first visit99/// order the control flow of \p F.100///101/// This is an easy way to verify the branching structure of the CFG without102/// checking every branch instruction individually. For the CFG of a103/// CanonicalLoopInfo, the Cond BB's terminating branch's first edge is entering104/// the body, i.e. the DFS order corresponds to the execution order with one105/// loop iteration.106static testing::AssertionResult107verifyDFSOrder(Function *F, ArrayRef<BasicBlock *> RefOrder) {108  ArrayRef<BasicBlock *>::iterator It = RefOrder.begin();109  ArrayRef<BasicBlock *>::iterator E = RefOrder.end();110 111  df_iterator_default_set<BasicBlock *, 16> Visited;112  auto DFS = llvm::depth_first_ext(&F->getEntryBlock(), Visited);113 114  BasicBlock *Prev = nullptr;115  for (BasicBlock *BB : DFS) {116    if (It != E && BB == *It) {117      Prev = *It;118      ++It;119    }120  }121 122  if (It == E)123    return testing::AssertionSuccess();124  if (!Prev)125    return testing::AssertionFailure()126           << "Did not find " << (*It)->getName() << " in control flow";127  return testing::AssertionFailure()128         << "Expected " << Prev->getName() << " before " << (*It)->getName()129         << " in control flow";130}131 132/// Verify that blocks in \p RefOrder are in the same relative order in the133/// linked lists of blocks in \p F. The linked list may contain additional134/// blocks in-between.135///136/// While the order in the linked list is not relevant for semantics, keeping137/// the order roughly in execution order makes its printout easier to read.138static testing::AssertionResult139verifyListOrder(Function *F, ArrayRef<BasicBlock *> RefOrder) {140  ArrayRef<BasicBlock *>::iterator It = RefOrder.begin();141  ArrayRef<BasicBlock *>::iterator E = RefOrder.end();142 143  BasicBlock *Prev = nullptr;144  for (BasicBlock &BB : *F) {145    if (It != E && &BB == *It) {146      Prev = *It;147      ++It;148    }149  }150 151  if (It == E)152    return testing::AssertionSuccess();153  if (!Prev)154    return testing::AssertionFailure() << "Did not find " << (*It)->getName()155                                       << " in function " << F->getName();156  return testing::AssertionFailure()157         << "Expected " << Prev->getName() << " before " << (*It)->getName()158         << " in function " << F->getName();159}160 161/// Populate Calls with call instructions calling the function with the given162/// FnID from the given function F.163static void findCalls(Function *F, omp::RuntimeFunction FnID,164                      OpenMPIRBuilder &OMPBuilder,165                      SmallVectorImpl<CallInst *> &Calls) {166  Function *Fn = OMPBuilder.getOrCreateRuntimeFunctionPtr(FnID);167  for (BasicBlock &BB : *F) {168    for (Instruction &I : BB) {169      auto *Call = dyn_cast<CallInst>(&I);170      if (Call && Call->getCalledFunction() == Fn)171        Calls.push_back(Call);172    }173  }174}175 176/// Assuming \p F contains only one call to the function with the given \p FnID,177/// return that call.178static CallInst *findSingleCall(Function *F, omp::RuntimeFunction FnID,179                                OpenMPIRBuilder &OMPBuilder) {180  SmallVector<CallInst *, 1> Calls;181  findCalls(F, FnID, OMPBuilder, Calls);182  EXPECT_EQ(1u, Calls.size());183  if (Calls.size() != 1)184    return nullptr;185  return Calls.front();186}187 188static omp::ScheduleKind getSchedKind(omp::OMPScheduleType SchedType) {189  switch (SchedType & ~omp::OMPScheduleType::ModifierMask) {190  case omp::OMPScheduleType::BaseDynamicChunked:191    return omp::OMP_SCHEDULE_Dynamic;192  case omp::OMPScheduleType::BaseGuidedChunked:193    return omp::OMP_SCHEDULE_Guided;194  case omp::OMPScheduleType::BaseAuto:195    return omp::OMP_SCHEDULE_Auto;196  case omp::OMPScheduleType::BaseRuntime:197    return omp::OMP_SCHEDULE_Runtime;198  default:199    llvm_unreachable("unknown type for this test");200  }201}202 203class OpenMPIRBuilderTest : public testing::Test {204protected:205  void SetUp() override {206    M.reset(new Module("MyModule", Ctx));207    FunctionType *FTy =208        FunctionType::get(Type::getVoidTy(Ctx), {Type::getInt32Ty(Ctx)},209                          /*isVarArg=*/false);210    F = Function::Create(FTy, Function::ExternalLinkage, "", M.get());211    BB = BasicBlock::Create(Ctx, "", F);212 213    DIBuilder DIB(*M);214    auto File = DIB.createFile("test.dbg", "/src", std::nullopt,215                               std::optional<StringRef>("/src/test.dbg"));216    auto CU = DIB.createCompileUnit(DISourceLanguageName(dwarf::DW_LANG_C),217                                    File, "llvm-C", true, "", 0);218    auto Type = DIB.createSubroutineType(DIB.getOrCreateTypeArray({}));219    auto SP = DIB.createFunction(220        CU, "foo", "", File, 1, Type, 1, DINode::FlagZero,221        DISubprogram::SPFlagDefinition | DISubprogram::SPFlagOptimized);222    F->setSubprogram(SP);223    auto Scope = DIB.createLexicalBlockFile(SP, File, 0);224    DIB.finalize();225    DL = DILocation::get(Ctx, 3, 7, Scope);226  }227 228  void TearDown() override {229    BB = nullptr;230    M.reset();231  }232 233  /// Create a function with a simple loop that calls printf using the logical234  /// loop counter for use with tests that need a CanonicalLoopInfo object.235  CanonicalLoopInfo *buildSingleLoopFunction(DebugLoc DL,236                                             OpenMPIRBuilder &OMPBuilder,237                                             int UseIVBits,238                                             CallInst **Call = nullptr,239                                             BasicBlock **BodyCode = nullptr) {240    OMPBuilder.initialize();241    F->setName("func");242 243    IRBuilder<> Builder(BB);244    OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});245    Value *TripCount = F->getArg(0);246 247    Type *IVType = Type::getIntNTy(Builder.getContext(), UseIVBits);248    Value *CastedTripCount =249        Builder.CreateZExtOrTrunc(TripCount, IVType, "tripcount");250 251    auto LoopBodyGenCB = [&](OpenMPIRBuilder::InsertPointTy CodeGenIP,252                             llvm::Value *LC) {253      Builder.restoreIP(CodeGenIP);254      if (BodyCode)255        *BodyCode = Builder.GetInsertBlock();256 257      // Add something that consumes the induction variable to the body.258      CallInst *CallInst = createPrintfCall(Builder, "%d\\n", {LC});259      if (Call)260        *Call = CallInst;261 262      return Error::success();263    };264 265    ASSERT_EXPECTED_INIT_RETURN(266        CanonicalLoopInfo *, Loop,267        OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, CastedTripCount),268        nullptr);269 270    // Finalize the function.271    Builder.restoreIP(Loop->getAfterIP());272    Builder.CreateRetVoid();273 274    return Loop;275  }276 277  LLVMContext Ctx;278  std::unique_ptr<Module> M;279  Function *F;280  BasicBlock *BB;281  DebugLoc DL;282};283 284class OpenMPIRBuilderTestWithParams285    : public OpenMPIRBuilderTest,286      public ::testing::WithParamInterface<omp::OMPScheduleType> {};287 288class OpenMPIRBuilderTestWithIVBits289    : public OpenMPIRBuilderTest,290      public ::testing::WithParamInterface<int> {};291 292// Returns the value stored in the given allocation. Returns null if the given293// value is not a result of an InstTy instruction, if no value is stored or if294// there is more than one store.295template <typename InstTy> static Value *findStoredValue(Value *AllocaValue) {296  Instruction *Inst = dyn_cast<InstTy>(AllocaValue);297  if (!Inst)298    return nullptr;299  StoreInst *Store = nullptr;300  for (Use &U : Inst->uses()) {301    if (auto *CandidateStore = dyn_cast<StoreInst>(U.getUser())) {302      EXPECT_EQ(Store, nullptr);303      Store = CandidateStore;304    }305  }306  if (!Store)307    return nullptr;308  return Store->getValueOperand();309}310 311// Returns the value stored in the aggregate argument of an outlined function,312// or nullptr if it is not found.313static Value *findStoredValueInAggregateAt(LLVMContext &Ctx, Value *Aggregate,314                                           unsigned Idx) {315  GetElementPtrInst *GEPAtIdx = nullptr;316  // Find GEP instruction at that index.317  for (User *Usr : Aggregate->users()) {318    GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Usr);319    if (!GEP)320      continue;321 322    if (GEP->getOperand(2) != ConstantInt::get(Type::getInt32Ty(Ctx), Idx))323      continue;324 325    EXPECT_EQ(GEPAtIdx, nullptr);326    GEPAtIdx = GEP;327  }328 329  EXPECT_NE(GEPAtIdx, nullptr);330  EXPECT_TRUE(GEPAtIdx->hasOneUse());331 332  // Find the value stored to the aggregate.333  StoreInst *StoreToAgg = dyn_cast<StoreInst>(*GEPAtIdx->user_begin());334  Value *StoredAggValue = StoreToAgg->getValueOperand();335 336  Value *StoredValue = nullptr;337 338  // Find the value stored to the value stored in the aggregate.339  for (User *Usr : StoredAggValue->users()) {340    StoreInst *Store = dyn_cast<StoreInst>(Usr);341    if (!Store)342      continue;343 344    if (Store->getPointerOperand() != StoredAggValue)345      continue;346 347    EXPECT_EQ(StoredValue, nullptr);348    StoredValue = Store->getValueOperand();349  }350 351  return StoredValue;352}353 354// Returns the aggregate that the value is originating from.355static Value *findAggregateFromValue(Value *V) {356  // Expects a load instruction that loads from the aggregate.357  LoadInst *Load = dyn_cast<LoadInst>(V);358  EXPECT_NE(Load, nullptr);359  // Find the GEP instruction used in the load instruction.360  GetElementPtrInst *GEP =361      dyn_cast<GetElementPtrInst>(Load->getPointerOperand());362  EXPECT_NE(GEP, nullptr);363  // Find the aggregate used in the GEP instruction.364  Value *Aggregate = GEP->getPointerOperand();365 366  return Aggregate;367}368 369TEST_F(OpenMPIRBuilderTest, CreateBarrier) {370  OpenMPIRBuilder OMPBuilder(*M);371  OMPBuilder.initialize();372 373  IRBuilder<> Builder(BB);374 375  ASSERT_THAT_EXPECTED(376      OMPBuilder.createBarrier({IRBuilder<>::InsertPoint()}, OMPD_for),377      Succeeded());378  EXPECT_TRUE(M->global_empty());379  EXPECT_EQ(M->size(), 1U);380  EXPECT_EQ(F->size(), 1U);381  EXPECT_EQ(BB->size(), 0U);382 383  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});384  ASSERT_THAT_EXPECTED(OMPBuilder.createBarrier(Loc, OMPD_for), Succeeded());385  EXPECT_FALSE(M->global_empty());386  EXPECT_EQ(M->size(), 3U);387  EXPECT_EQ(F->size(), 1U);388  EXPECT_EQ(BB->size(), 2U);389 390  CallInst *GTID = dyn_cast<CallInst>(&BB->front());391  EXPECT_NE(GTID, nullptr);392  EXPECT_EQ(GTID->arg_size(), 1U);393  EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");394  EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());395  EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());396 397  CallInst *Barrier = dyn_cast<CallInst>(GTID->getNextNode());398  EXPECT_NE(Barrier, nullptr);399  EXPECT_EQ(Barrier->arg_size(), 2U);400  EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_barrier");401  EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());402  EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());403 404  EXPECT_EQ(cast<CallInst>(Barrier)->getArgOperand(1), GTID);405 406  Builder.CreateUnreachable();407  EXPECT_FALSE(verifyModule(*M, &errs()));408}409 410TEST_F(OpenMPIRBuilderTest, CreateCancel) {411  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;412  OpenMPIRBuilder OMPBuilder(*M);413  OMPBuilder.initialize();414 415  BasicBlock *CBB = BasicBlock::Create(Ctx, "", F);416  new UnreachableInst(Ctx, CBB);417  auto FiniCB = [&](InsertPointTy IP) {418    ASSERT_NE(IP.getBlock(), nullptr);419    ASSERT_EQ(IP.getBlock()->end(), IP.getPoint());420    BranchInst::Create(CBB, IP.getBlock());421  };422  OMPBuilder.pushFinalizationCB({FINICB_WRAPPER(FiniCB), OMPD_parallel, true});423 424  IRBuilder<> Builder(BB);425 426  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});427  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, NewIP,428                       OMPBuilder.createCancel(Loc, nullptr, OMPD_parallel));429  Builder.restoreIP(NewIP);430  EXPECT_FALSE(M->global_empty());431  EXPECT_EQ(M->size(), 3U);432  EXPECT_EQ(F->size(), 5U);433  EXPECT_EQ(BB->size(), 4U);434 435  CallInst *GTID = dyn_cast<CallInst>(&BB->front());436  EXPECT_NE(GTID, nullptr);437  EXPECT_EQ(GTID->arg_size(), 1U);438  EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");439  EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());440  EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());441 442  CallInst *Cancel = dyn_cast<CallInst>(GTID->getNextNode());443  EXPECT_NE(Cancel, nullptr);444  EXPECT_EQ(Cancel->arg_size(), 3U);445  EXPECT_EQ(Cancel->getCalledFunction()->getName(), "__kmpc_cancel");446  EXPECT_FALSE(Cancel->getCalledFunction()->doesNotAccessMemory());447  EXPECT_FALSE(Cancel->getCalledFunction()->doesNotFreeMemory());448  EXPECT_TRUE(Cancel->hasOneUse());449  Instruction *CancelBBTI = Cancel->getParent()->getTerminator();450  EXPECT_EQ(CancelBBTI->getNumSuccessors(), 2U);451  EXPECT_EQ(CancelBBTI->getSuccessor(0), NewIP.getBlock());452  EXPECT_EQ(CancelBBTI->getSuccessor(1)->size(), 1U);453  EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(),454            1U);455  // cancel branch instruction (1) -> .cncl -> .fini -> CBB456  EXPECT_EQ(CancelBBTI->getSuccessor(1)457                ->getTerminator()458                ->getSuccessor(0)459                ->getTerminator()460                ->getSuccessor(0),461            CBB);462 463  EXPECT_EQ(cast<CallInst>(Cancel)->getArgOperand(1), GTID);464 465  OMPBuilder.popFinalizationCB();466 467  Builder.CreateUnreachable();468  EXPECT_FALSE(verifyModule(*M, &errs()));469}470 471TEST_F(OpenMPIRBuilderTest, CreateCancelIfCond) {472  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;473  OpenMPIRBuilder OMPBuilder(*M);474  OMPBuilder.initialize();475 476  BasicBlock *CBB = BasicBlock::Create(Ctx, "", F);477  new UnreachableInst(Ctx, CBB);478  auto FiniCB = [&](InsertPointTy IP) {479    ASSERT_NE(IP.getBlock(), nullptr);480    ASSERT_EQ(IP.getBlock()->end(), IP.getPoint());481    BranchInst::Create(CBB, IP.getBlock());482  };483  OMPBuilder.pushFinalizationCB({FINICB_WRAPPER(FiniCB), OMPD_parallel, true});484 485  IRBuilder<> Builder(BB);486 487  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});488  ASSERT_EXPECTED_INIT(489      OpenMPIRBuilder::InsertPointTy, NewIP,490      OMPBuilder.createCancel(Loc, Builder.getTrue(), OMPD_parallel));491  Builder.restoreIP(NewIP);492  EXPECT_FALSE(M->global_empty());493  EXPECT_EQ(M->size(), 4U);494  EXPECT_EQ(F->size(), 10U);495  EXPECT_EQ(BB->size(), 1U);496  ASSERT_TRUE(isa<BranchInst>(BB->getTerminator()));497  ASSERT_EQ(BB->getTerminator()->getNumSuccessors(), 2U);498  BB = BB->getTerminator()->getSuccessor(0);499  EXPECT_EQ(BB->size(), 4U);500 501  CallInst *GTID = dyn_cast<CallInst>(&BB->front());502  EXPECT_NE(GTID, nullptr);503  EXPECT_EQ(GTID->arg_size(), 1U);504  EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");505  EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());506  EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());507 508  CallInst *Cancel = dyn_cast<CallInst>(GTID->getNextNode());509  EXPECT_NE(Cancel, nullptr);510  EXPECT_EQ(Cancel->arg_size(), 3U);511  EXPECT_EQ(Cancel->getCalledFunction()->getName(), "__kmpc_cancel");512  EXPECT_FALSE(Cancel->getCalledFunction()->doesNotAccessMemory());513  EXPECT_FALSE(Cancel->getCalledFunction()->doesNotFreeMemory());514  EXPECT_TRUE(Cancel->hasOneUse());515  Instruction *CancelBBTI = Cancel->getParent()->getTerminator();516  EXPECT_EQ(CancelBBTI->getNumSuccessors(), 2U);517  EXPECT_EQ(CancelBBTI->getSuccessor(0)->size(), 1U);518  EXPECT_EQ(CancelBBTI->getSuccessor(0)->getUniqueSuccessor(),519            NewIP.getBlock());520  EXPECT_EQ(CancelBBTI->getSuccessor(1)->size(), 1U);521  EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(),522            1U);523  EXPECT_EQ(CancelBBTI->getSuccessor(1)524                ->getTerminator()525                ->getSuccessor(0)526                ->getTerminator()527                ->getSuccessor(0),528            CBB);529 530  EXPECT_EQ(cast<CallInst>(Cancel)->getArgOperand(1), GTID);531 532  OMPBuilder.popFinalizationCB();533 534  Builder.CreateUnreachable();535  EXPECT_FALSE(verifyModule(*M, &errs()));536}537 538TEST_F(OpenMPIRBuilderTest, CreateCancelBarrier) {539  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;540  OpenMPIRBuilder OMPBuilder(*M);541  OMPBuilder.initialize();542 543  BasicBlock *CBB = BasicBlock::Create(Ctx, "", F);544  new UnreachableInst(Ctx, CBB);545  auto FiniCB = [&](InsertPointTy IP) {546    ASSERT_NE(IP.getBlock(), nullptr);547    ASSERT_EQ(IP.getBlock()->end(), IP.getPoint());548    BranchInst::Create(CBB, IP.getBlock());549  };550  OMPBuilder.pushFinalizationCB({FINICB_WRAPPER(FiniCB), OMPD_parallel, true});551 552  IRBuilder<> Builder(BB);553 554  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});555  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, NewIP,556                       OMPBuilder.createBarrier(Loc, OMPD_for));557  Builder.restoreIP(NewIP);558  EXPECT_FALSE(M->global_empty());559  EXPECT_EQ(M->size(), 3U);560  EXPECT_EQ(F->size(), 5U);561  EXPECT_EQ(BB->size(), 4U);562 563  CallInst *GTID = dyn_cast<CallInst>(&BB->front());564  EXPECT_NE(GTID, nullptr);565  EXPECT_EQ(GTID->arg_size(), 1U);566  EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");567  EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());568  EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());569 570  CallInst *Barrier = dyn_cast<CallInst>(GTID->getNextNode());571  EXPECT_NE(Barrier, nullptr);572  EXPECT_EQ(Barrier->arg_size(), 2U);573  EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_cancel_barrier");574  EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());575  EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());576  EXPECT_TRUE(Barrier->hasOneUse());577  Instruction *BarrierBBTI = Barrier->getParent()->getTerminator();578  EXPECT_EQ(BarrierBBTI->getNumSuccessors(), 2U);579  EXPECT_EQ(BarrierBBTI->getSuccessor(0), NewIP.getBlock());580  EXPECT_EQ(BarrierBBTI->getSuccessor(1)->size(), 1U);581  EXPECT_EQ(BarrierBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(),582            1U);583  EXPECT_EQ(BarrierBBTI->getSuccessor(1)584                ->getTerminator()585                ->getSuccessor(0)586                ->getTerminator()587                ->getSuccessor(0),588            CBB);589 590  EXPECT_EQ(cast<CallInst>(Barrier)->getArgOperand(1), GTID);591 592  OMPBuilder.popFinalizationCB();593 594  Builder.CreateUnreachable();595  EXPECT_FALSE(verifyModule(*M, &errs()));596}597 598TEST_F(OpenMPIRBuilderTest, DbgLoc) {599  OpenMPIRBuilder OMPBuilder(*M);600  OMPBuilder.initialize();601  F->setName("func");602 603  IRBuilder<> Builder(BB);604 605  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});606  ASSERT_THAT_EXPECTED(OMPBuilder.createBarrier(Loc, OMPD_for), Succeeded());607  CallInst *GTID = dyn_cast<CallInst>(&BB->front());608  CallInst *Barrier = dyn_cast<CallInst>(GTID->getNextNode());609  EXPECT_EQ(GTID->getDebugLoc(), DL);610  EXPECT_EQ(Barrier->getDebugLoc(), DL);611  EXPECT_TRUE(isa<GlobalVariable>(Barrier->getOperand(0)));612  if (!isa<GlobalVariable>(Barrier->getOperand(0)))613    return;614  GlobalVariable *Ident = cast<GlobalVariable>(Barrier->getOperand(0));615  EXPECT_TRUE(Ident->hasInitializer());616  if (!Ident->hasInitializer())617    return;618  Constant *Initializer = Ident->getInitializer();619  EXPECT_TRUE(620      isa<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts()));621  GlobalVariable *SrcStrGlob =622      cast<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts());623  if (!SrcStrGlob)624    return;625  EXPECT_TRUE(isa<ConstantDataArray>(SrcStrGlob->getInitializer()));626  ConstantDataArray *SrcSrc =627      dyn_cast<ConstantDataArray>(SrcStrGlob->getInitializer());628  if (!SrcSrc)629    return;630  EXPECT_EQ(SrcSrc->getAsCString(), ";/src/test.dbg;foo;3;7;;");631}632 633TEST_F(OpenMPIRBuilderTest, ParallelSimpleGPU) {634  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;635  std::string oldDLStr = M->getDataLayoutStr();636  M->setDataLayout(637      "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:"638      "256:256:32-p8:128:128:128:48-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-"639      "v192:"640      "256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8");641  OpenMPIRBuilder OMPBuilder(*M);642  OMPBuilder.Config.IsTargetDevice = true;643  OMPBuilder.initialize();644  F->setName("func");645  IRBuilder<> Builder(BB);646  BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);647  Builder.CreateBr(EnterBB);648  Builder.SetInsertPoint(EnterBB);649  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});650 651  AllocaInst *PrivAI = nullptr;652 653  unsigned NumBodiesGenerated = 0;654  unsigned NumPrivatizedVars = 0;655  unsigned NumFinalizationPoints = 0;656 657  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {658    ++NumBodiesGenerated;659 660    Builder.restoreIP(AllocaIP);661    PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());662    Builder.CreateStore(F->arg_begin(), PrivAI);663 664    Builder.restoreIP(CodeGenIP);665    Value *PrivLoad =666        Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");667    Value *Cmp = Builder.CreateICmpNE(F->arg_begin(), PrivLoad);668    Instruction *ThenTerm, *ElseTerm;669    SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(),670                                  &ThenTerm, &ElseTerm);671    return Error::success();672  };673 674  auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,675                    Value &Orig, Value &Inner,676                    Value *&ReplacementValue) -> InsertPointTy {677    ++NumPrivatizedVars;678 679    if (!isa<AllocaInst>(Orig)) {680      EXPECT_EQ(&Orig, F->arg_begin());681      ReplacementValue = &Inner;682      return CodeGenIP;683    }684 685    // Since the original value is an allocation, it has a pointer type and686    // therefore no additional wrapping should happen.687    EXPECT_EQ(&Orig, &Inner);688 689    // Trivial copy (=firstprivate).690    Builder.restoreIP(AllocaIP);691    Type *VTy = ReplacementValue->getType();692    Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");693    ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");694    Builder.restoreIP(CodeGenIP);695    Builder.CreateStore(V, ReplacementValue);696    return CodeGenIP;697  };698 699  auto FiniCB = [&](InsertPointTy CodeGenIP) {700    ++NumFinalizationPoints;701    return Error::success();702  };703 704  IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),705                                    F->getEntryBlock().getFirstInsertionPt());706  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,707                       OMPBuilder.createParallel(708                           Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, nullptr,709                           nullptr, OMP_PROC_BIND_default, false));710 711  EXPECT_EQ(NumBodiesGenerated, 1U);712  EXPECT_EQ(NumPrivatizedVars, 1U);713  EXPECT_EQ(NumFinalizationPoints, 1U);714 715  Builder.restoreIP(AfterIP);716  Builder.CreateRetVoid();717 718  OMPBuilder.finalize();719  Function *OutlinedFn = PrivAI->getFunction();720  EXPECT_FALSE(verifyModule(*M, &errs()));721  EXPECT_NE(OutlinedFn, F);722  EXPECT_TRUE(OutlinedFn->hasFnAttribute(Attribute::NoUnwind));723  EXPECT_TRUE(OutlinedFn->hasParamAttribute(0, Attribute::NoAlias));724  EXPECT_TRUE(OutlinedFn->hasParamAttribute(1, Attribute::NoAlias));725 726  EXPECT_TRUE(OutlinedFn->hasInternalLinkage());727  EXPECT_EQ(OutlinedFn->arg_size(), 3U);728  // Make sure that arguments are pointers in 0 address address space729  EXPECT_EQ(OutlinedFn->getArg(0)->getType(),730            PointerType::get(M->getContext(), 0));731  EXPECT_EQ(OutlinedFn->getArg(1)->getType(),732            PointerType::get(M->getContext(), 0));733  EXPECT_EQ(OutlinedFn->getArg(2)->getType(),734            PointerType::get(M->getContext(), 0));735  EXPECT_EQ(&OutlinedFn->getEntryBlock(), PrivAI->getParent());736  EXPECT_TRUE(OutlinedFn->hasOneUse());737  User *Usr = OutlinedFn->user_back();738  ASSERT_TRUE(isa<CallInst>(Usr));739  CallInst *Parallel51CI = dyn_cast<CallInst>(Usr);740  ASSERT_NE(Parallel51CI, nullptr);741 742  EXPECT_EQ(Parallel51CI->getCalledFunction()->getName(), "__kmpc_parallel_51");743  EXPECT_EQ(Parallel51CI->arg_size(), 9U);744  EXPECT_EQ(Parallel51CI->getArgOperand(5), OutlinedFn);745  EXPECT_TRUE(746      isa<GlobalVariable>(Parallel51CI->getArgOperand(0)->stripPointerCasts()));747  EXPECT_EQ(Parallel51CI, Usr);748  M->setDataLayout(oldDLStr);749}750 751TEST_F(OpenMPIRBuilderTest, ParallelSimple) {752  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;753  OpenMPIRBuilder OMPBuilder(*M);754  OMPBuilder.Config.IsTargetDevice = false;755  OMPBuilder.initialize();756  F->setName("func");757  IRBuilder<> Builder(BB);758 759  BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);760  Builder.CreateBr(EnterBB);761  Builder.SetInsertPoint(EnterBB);762  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});763 764  AllocaInst *PrivAI = nullptr;765 766  unsigned NumBodiesGenerated = 0;767  unsigned NumPrivatizedVars = 0;768  unsigned NumFinalizationPoints = 0;769 770  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {771    ++NumBodiesGenerated;772 773    Builder.restoreIP(AllocaIP);774    PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());775    Builder.CreateStore(F->arg_begin(), PrivAI);776 777    Builder.restoreIP(CodeGenIP);778    Value *PrivLoad =779        Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");780    Value *Cmp = Builder.CreateICmpNE(F->arg_begin(), PrivLoad);781    Instruction *ThenTerm, *ElseTerm;782    SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(),783                                  &ThenTerm, &ElseTerm);784    return Error::success();785  };786 787  auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,788                    Value &Orig, Value &Inner,789                    Value *&ReplacementValue) -> InsertPointTy {790    ++NumPrivatizedVars;791 792    if (!isa<AllocaInst>(Orig)) {793      EXPECT_EQ(&Orig, F->arg_begin());794      ReplacementValue = &Inner;795      return CodeGenIP;796    }797 798    // Since the original value is an allocation, it has a pointer type and799    // therefore no additional wrapping should happen.800    EXPECT_EQ(&Orig, &Inner);801 802    // Trivial copy (=firstprivate).803    Builder.restoreIP(AllocaIP);804    Type *VTy = ReplacementValue->getType();805    Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");806    ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");807    Builder.restoreIP(CodeGenIP);808    Builder.CreateStore(V, ReplacementValue);809    return CodeGenIP;810  };811 812  auto FiniCB = [&](InsertPointTy CodeGenIP) {813    ++NumFinalizationPoints;814    return Error::success();815  };816 817  IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),818                                    F->getEntryBlock().getFirstInsertionPt());819  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,820                       OMPBuilder.createParallel(821                           Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, nullptr,822                           nullptr, OMP_PROC_BIND_default, false));823  EXPECT_EQ(NumBodiesGenerated, 1U);824  EXPECT_EQ(NumPrivatizedVars, 1U);825  EXPECT_EQ(NumFinalizationPoints, 1U);826 827  Builder.restoreIP(AfterIP);828  Builder.CreateRetVoid();829 830  OMPBuilder.finalize();831 832  EXPECT_NE(PrivAI, nullptr);833  Function *OutlinedFn = PrivAI->getFunction();834  EXPECT_NE(F, OutlinedFn);835  EXPECT_FALSE(verifyModule(*M, &errs()));836  EXPECT_TRUE(OutlinedFn->hasFnAttribute(Attribute::NoUnwind));837  EXPECT_TRUE(OutlinedFn->hasParamAttribute(0, Attribute::NoAlias));838  EXPECT_TRUE(OutlinedFn->hasParamAttribute(1, Attribute::NoAlias));839 840  EXPECT_TRUE(OutlinedFn->hasInternalLinkage());841  EXPECT_EQ(OutlinedFn->arg_size(), 3U);842 843  EXPECT_EQ(&OutlinedFn->getEntryBlock(), PrivAI->getParent());844  EXPECT_TRUE(OutlinedFn->hasOneUse());845  User *Usr = OutlinedFn->user_back();846  ASSERT_TRUE(isa<CallInst>(Usr));847  CallInst *ForkCI = dyn_cast<CallInst>(Usr);848  ASSERT_NE(ForkCI, nullptr);849 850  EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call");851  EXPECT_EQ(ForkCI->arg_size(), 4U);852  EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));853  EXPECT_EQ(ForkCI->getArgOperand(1),854            ConstantInt::get(Type::getInt32Ty(Ctx), 1U));855  EXPECT_EQ(ForkCI, Usr);856  Value *StoredValue =857      findStoredValueInAggregateAt(Ctx, ForkCI->getArgOperand(3), 0);858  EXPECT_EQ(StoredValue, F->arg_begin());859}860 861TEST_F(OpenMPIRBuilderTest, ParallelNested) {862  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;863  OpenMPIRBuilder OMPBuilder(*M);864  OMPBuilder.Config.IsTargetDevice = false;865  OMPBuilder.initialize();866  F->setName("func");867  IRBuilder<> Builder(BB);868 869  BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);870  Builder.CreateBr(EnterBB);871  Builder.SetInsertPoint(EnterBB);872  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});873 874  unsigned NumInnerBodiesGenerated = 0;875  unsigned NumOuterBodiesGenerated = 0;876  unsigned NumFinalizationPoints = 0;877 878  auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {879    ++NumInnerBodiesGenerated;880    return Error::success();881  };882 883  auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,884                    Value &Orig, Value &Inner,885                    Value *&ReplacementValue) -> InsertPointTy {886    // Trivial copy (=firstprivate).887    Builder.restoreIP(AllocaIP);888    Type *VTy = ReplacementValue->getType();889    Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");890    ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");891    Builder.restoreIP(CodeGenIP);892    Builder.CreateStore(V, ReplacementValue);893    return CodeGenIP;894  };895 896  auto FiniCB = [&](InsertPointTy CodeGenIP) {897    ++NumFinalizationPoints;898    return Error::success();899  };900 901  auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {902    ++NumOuterBodiesGenerated;903    Builder.restoreIP(CodeGenIP);904    BasicBlock *CGBB = CodeGenIP.getBlock();905    BasicBlock *NewBB = SplitBlock(CGBB, &*CodeGenIP.getPoint());906    CGBB->getTerminator()->eraseFromParent();907 908    ASSERT_EXPECTED_INIT(909        OpenMPIRBuilder::InsertPointTy, AfterIP,910        OMPBuilder.createParallel(InsertPointTy(CGBB, CGBB->end()), AllocaIP,911                                  InnerBodyGenCB, PrivCB, FiniCB, nullptr,912                                  nullptr, OMP_PROC_BIND_default, false));913 914    Builder.restoreIP(AfterIP);915    Builder.CreateBr(NewBB);916  };917 918  IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),919                                    F->getEntryBlock().getFirstInsertionPt());920  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,921                       OMPBuilder.createParallel(922                           Loc, AllocaIP, BODYGENCB_WRAPPER(OuterBodyGenCB),923                           PrivCB, FiniCB, nullptr, nullptr,924                           OMP_PROC_BIND_default, false));925 926  EXPECT_EQ(NumInnerBodiesGenerated, 1U);927  EXPECT_EQ(NumOuterBodiesGenerated, 1U);928  EXPECT_EQ(NumFinalizationPoints, 2U);929 930  Builder.restoreIP(AfterIP);931  Builder.CreateRetVoid();932 933  OMPBuilder.finalize();934 935  EXPECT_EQ(M->size(), 5U);936  for (Function &OutlinedFn : *M) {937    if (F == &OutlinedFn || OutlinedFn.isDeclaration())938      continue;939    EXPECT_FALSE(verifyModule(*M, &errs()));940    EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoUnwind));941    EXPECT_TRUE(OutlinedFn.hasParamAttribute(0, Attribute::NoAlias));942    EXPECT_TRUE(OutlinedFn.hasParamAttribute(1, Attribute::NoAlias));943 944    EXPECT_TRUE(OutlinedFn.hasInternalLinkage());945    EXPECT_EQ(OutlinedFn.arg_size(), 2U);946 947    EXPECT_TRUE(OutlinedFn.hasOneUse());948    User *Usr = OutlinedFn.user_back();949    ASSERT_TRUE(isa<CallInst>(Usr));950    CallInst *ForkCI = dyn_cast<CallInst>(Usr);951    ASSERT_NE(ForkCI, nullptr);952 953    EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call");954    EXPECT_EQ(ForkCI->arg_size(), 3U);955    EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));956    EXPECT_EQ(ForkCI->getArgOperand(1),957              ConstantInt::get(Type::getInt32Ty(Ctx), 0U));958    EXPECT_EQ(ForkCI, Usr);959  }960}961 962TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) {963  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;964  OpenMPIRBuilder OMPBuilder(*M);965  OMPBuilder.Config.IsTargetDevice = false;966  OMPBuilder.initialize();967  F->setName("func");968  IRBuilder<> Builder(BB);969 970  BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);971  Builder.CreateBr(EnterBB);972  Builder.SetInsertPoint(EnterBB);973  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});974 975  unsigned NumInnerBodiesGenerated = 0;976  unsigned NumOuterBodiesGenerated = 0;977  unsigned NumFinalizationPoints = 0;978 979  auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {980    ++NumInnerBodiesGenerated;981    return Error::success();982  };983 984  auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,985                    Value &Orig, Value &Inner,986                    Value *&ReplacementValue) -> InsertPointTy {987    // Trivial copy (=firstprivate).988    Builder.restoreIP(AllocaIP);989    Type *VTy = ReplacementValue->getType();990    Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");991    ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");992    Builder.restoreIP(CodeGenIP);993    Builder.CreateStore(V, ReplacementValue);994    return CodeGenIP;995  };996 997  auto FiniCB = [&](InsertPointTy CodeGenIP) {998    ++NumFinalizationPoints;999    return Error::success();1000  };1001 1002  auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {1003    ++NumOuterBodiesGenerated;1004    Builder.restoreIP(CodeGenIP);1005    BasicBlock *CGBB = CodeGenIP.getBlock();1006    BasicBlock *NewBB1 = SplitBlock(CGBB, &*CodeGenIP.getPoint());1007    BasicBlock *NewBB2 = SplitBlock(NewBB1, &*NewBB1->getFirstInsertionPt());1008    CGBB->getTerminator()->eraseFromParent();1009    ;1010    NewBB1->getTerminator()->eraseFromParent();1011    ;1012 1013    ASSERT_EXPECTED_INIT(1014        OpenMPIRBuilder::InsertPointTy, AfterIP1,1015        OMPBuilder.createParallel(InsertPointTy(CGBB, CGBB->end()), AllocaIP,1016                                  InnerBodyGenCB, PrivCB, FiniCB, nullptr,1017                                  nullptr, OMP_PROC_BIND_default, false));1018 1019    Builder.restoreIP(AfterIP1);1020    Builder.CreateBr(NewBB1);1021 1022    ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP2,1023                         OMPBuilder.createParallel(1024                             InsertPointTy(NewBB1, NewBB1->end()), AllocaIP,1025                             InnerBodyGenCB, PrivCB, FiniCB, nullptr, nullptr,1026                             OMP_PROC_BIND_default, false));1027 1028    Builder.restoreIP(AfterIP2);1029    Builder.CreateBr(NewBB2);1030  };1031 1032  IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),1033                                    F->getEntryBlock().getFirstInsertionPt());1034  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,1035                       OMPBuilder.createParallel(1036                           Loc, AllocaIP, BODYGENCB_WRAPPER(OuterBodyGenCB),1037                           PrivCB, FiniCB, nullptr, nullptr,1038                           OMP_PROC_BIND_default, false));1039 1040  EXPECT_EQ(NumInnerBodiesGenerated, 2U);1041  EXPECT_EQ(NumOuterBodiesGenerated, 1U);1042  EXPECT_EQ(NumFinalizationPoints, 3U);1043 1044  Builder.restoreIP(AfterIP);1045  Builder.CreateRetVoid();1046 1047  OMPBuilder.finalize();1048 1049  EXPECT_EQ(M->size(), 6U);1050  for (Function &OutlinedFn : *M) {1051    if (F == &OutlinedFn || OutlinedFn.isDeclaration())1052      continue;1053    EXPECT_FALSE(verifyModule(*M, &errs()));1054    EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoUnwind));1055    EXPECT_TRUE(OutlinedFn.hasParamAttribute(0, Attribute::NoAlias));1056    EXPECT_TRUE(OutlinedFn.hasParamAttribute(1, Attribute::NoAlias));1057 1058    EXPECT_TRUE(OutlinedFn.hasInternalLinkage());1059    EXPECT_EQ(OutlinedFn.arg_size(), 2U);1060 1061    unsigned NumAllocas = 0;1062    for (Instruction &I : instructions(OutlinedFn))1063      NumAllocas += isa<AllocaInst>(I);1064    EXPECT_EQ(NumAllocas, 1U);1065 1066    EXPECT_TRUE(OutlinedFn.hasOneUse());1067    User *Usr = OutlinedFn.user_back();1068    ASSERT_TRUE(isa<CallInst>(Usr));1069    CallInst *ForkCI = dyn_cast<CallInst>(Usr);1070    ASSERT_NE(ForkCI, nullptr);1071 1072    EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call");1073    EXPECT_EQ(ForkCI->arg_size(), 3U);1074    EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));1075    EXPECT_EQ(ForkCI->getArgOperand(1),1076              ConstantInt::get(Type::getInt32Ty(Ctx), 0U));1077    EXPECT_EQ(ForkCI, Usr);1078  }1079}1080 1081TEST_F(OpenMPIRBuilderTest, ParallelIfCond) {1082  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;1083  OpenMPIRBuilder OMPBuilder(*M);1084  OMPBuilder.Config.IsTargetDevice = false;1085  OMPBuilder.initialize();1086  F->setName("func");1087  IRBuilder<> Builder(BB);1088 1089  BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);1090  Builder.CreateBr(EnterBB);1091  Builder.SetInsertPoint(EnterBB);1092  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});1093 1094  AllocaInst *PrivAI = nullptr;1095 1096  unsigned NumBodiesGenerated = 0;1097  unsigned NumPrivatizedVars = 0;1098  unsigned NumFinalizationPoints = 0;1099 1100  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {1101    ++NumBodiesGenerated;1102 1103    Builder.restoreIP(AllocaIP);1104    PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());1105    Builder.CreateStore(F->arg_begin(), PrivAI);1106 1107    Builder.restoreIP(CodeGenIP);1108    Value *PrivLoad =1109        Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");1110    Value *Cmp = Builder.CreateICmpNE(F->arg_begin(), PrivLoad);1111    Instruction *ThenTerm, *ElseTerm;1112    SplitBlockAndInsertIfThenElse(Cmp, &*Builder.GetInsertPoint(), &ThenTerm,1113                                  &ElseTerm);1114    return Error::success();1115  };1116 1117  auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,1118                    Value &Orig, Value &Inner,1119                    Value *&ReplacementValue) -> InsertPointTy {1120    ++NumPrivatizedVars;1121 1122    if (!isa<AllocaInst>(Orig)) {1123      EXPECT_EQ(&Orig, F->arg_begin());1124      ReplacementValue = &Inner;1125      return CodeGenIP;1126    }1127 1128    // Since the original value is an allocation, it has a pointer type and1129    // therefore no additional wrapping should happen.1130    EXPECT_EQ(&Orig, &Inner);1131 1132    // Trivial copy (=firstprivate).1133    Builder.restoreIP(AllocaIP);1134    Type *VTy = ReplacementValue->getType();1135    Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");1136    ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");1137    Builder.restoreIP(CodeGenIP);1138    Builder.CreateStore(V, ReplacementValue);1139    return CodeGenIP;1140  };1141 1142  auto FiniCB = [&](InsertPointTy CodeGenIP) {1143    ++NumFinalizationPoints;1144    // No destructors.1145    return Error::success();1146  };1147 1148  IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),1149                                    F->getEntryBlock().getFirstInsertionPt());1150  ASSERT_EXPECTED_INIT(1151      OpenMPIRBuilder::InsertPointTy, AfterIP,1152      OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,1153                                Builder.CreateIsNotNull(F->arg_begin()),1154                                nullptr, OMP_PROC_BIND_default, false));1155 1156  EXPECT_EQ(NumBodiesGenerated, 1U);1157  EXPECT_EQ(NumPrivatizedVars, 1U);1158  EXPECT_EQ(NumFinalizationPoints, 1U);1159 1160  Builder.restoreIP(AfterIP);1161  Builder.CreateRetVoid();1162  OMPBuilder.finalize();1163 1164  EXPECT_NE(PrivAI, nullptr);1165  Function *OutlinedFn = PrivAI->getFunction();1166  EXPECT_NE(F, OutlinedFn);1167  EXPECT_FALSE(verifyModule(*M, &errs()));1168 1169  EXPECT_TRUE(OutlinedFn->hasInternalLinkage());1170  EXPECT_EQ(OutlinedFn->arg_size(), 3U);1171 1172  EXPECT_EQ(&OutlinedFn->getEntryBlock(), PrivAI->getParent());1173  ASSERT_TRUE(OutlinedFn->hasOneUse());1174 1175  CallInst *ForkCI = nullptr;1176  for (User *Usr : OutlinedFn->users()) {1177    ASSERT_TRUE(isa<CallInst>(Usr));1178    ForkCI = cast<CallInst>(Usr);1179  }1180 1181  EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call_if");1182  EXPECT_EQ(ForkCI->arg_size(), 5U);1183  EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));1184  EXPECT_EQ(ForkCI->getArgOperand(1),1185            ConstantInt::get(Type::getInt32Ty(Ctx), 1));1186  EXPECT_EQ(ForkCI->getArgOperand(3)->getType(), Type::getInt32Ty(Ctx));1187}1188 1189TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) {1190  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;1191  OpenMPIRBuilder OMPBuilder(*M);1192  OMPBuilder.Config.IsTargetDevice = false;1193  OMPBuilder.initialize();1194  F->setName("func");1195  IRBuilder<> Builder(BB);1196 1197  BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);1198  Builder.CreateBr(EnterBB);1199  Builder.SetInsertPoint(EnterBB);1200  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});1201 1202  unsigned NumBodiesGenerated = 0;1203  unsigned NumPrivatizedVars = 0;1204  unsigned NumFinalizationPoints = 0;1205 1206  CallInst *CheckedBarrier = nullptr;1207  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {1208    ++NumBodiesGenerated;1209 1210    Builder.restoreIP(CodeGenIP);1211 1212    // Create three barriers, two cancel barriers but only one checked.1213    Function *CBFn, *BFn;1214 1215    ASSERT_EXPECTED_INIT(1216        OpenMPIRBuilder::InsertPointTy, BarrierIP1,1217        OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel));1218    Builder.restoreIP(BarrierIP1);1219 1220    CBFn = M->getFunction("__kmpc_cancel_barrier");1221    BFn = M->getFunction("__kmpc_barrier");1222    ASSERT_NE(CBFn, nullptr);1223    ASSERT_EQ(BFn, nullptr);1224    ASSERT_TRUE(CBFn->hasOneUse());1225    ASSERT_TRUE(isa<CallInst>(CBFn->user_back()));1226    ASSERT_TRUE(CBFn->user_back()->hasOneUse());1227    CheckedBarrier = cast<CallInst>(CBFn->user_back());1228 1229    ASSERT_EXPECTED_INIT(1230        OpenMPIRBuilder::InsertPointTy, BarrierIP2,1231        OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel, true));1232    Builder.restoreIP(BarrierIP2);1233    CBFn = M->getFunction("__kmpc_cancel_barrier");1234    BFn = M->getFunction("__kmpc_barrier");1235    ASSERT_NE(CBFn, nullptr);1236    ASSERT_NE(BFn, nullptr);1237    ASSERT_TRUE(CBFn->hasOneUse());1238    ASSERT_TRUE(BFn->hasOneUse());1239    ASSERT_TRUE(isa<CallInst>(BFn->user_back()));1240    ASSERT_TRUE(BFn->user_back()->use_empty());1241 1242    ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, BarrierIP3,1243                         OMPBuilder.createBarrier(Builder.saveIP(),1244                                                  OMPD_parallel, false, false));1245    Builder.restoreIP(BarrierIP3);1246    ASSERT_TRUE(CBFn->hasNUses(2));1247    ASSERT_TRUE(BFn->hasOneUse());1248    ASSERT_TRUE(CBFn->user_back() != CheckedBarrier);1249    ASSERT_TRUE(isa<CallInst>(CBFn->user_back()));1250    ASSERT_TRUE(CBFn->user_back()->use_empty());1251  };1252 1253  auto PrivCB = [&](InsertPointTy, InsertPointTy, Value &V, Value &,1254                    Value *&) -> InsertPointTy {1255    ++NumPrivatizedVars;1256    llvm_unreachable("No privatization callback call expected!");1257  };1258 1259  FunctionType *FakeDestructorTy =1260      FunctionType::get(Type::getVoidTy(Ctx), {Type::getInt32Ty(Ctx)},1261                        /*isVarArg=*/false);1262  auto *FakeDestructor = Function::Create(1263      FakeDestructorTy, Function::ExternalLinkage, "fakeDestructor", M.get());1264 1265  auto FiniCB = [&](InsertPointTy IP) {1266    ++NumFinalizationPoints;1267    Builder.restoreIP(IP);1268    Builder.CreateCall(FakeDestructor,1269                       {Builder.getInt32(NumFinalizationPoints)});1270    return Error::success();1271  };1272 1273  IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),1274                                    F->getEntryBlock().getFirstInsertionPt());1275  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,1276                       OMPBuilder.createParallel(1277                           Loc, AllocaIP, BODYGENCB_WRAPPER(BodyGenCB), PrivCB,1278                           FiniCB, Builder.CreateIsNotNull(F->arg_begin()),1279                           nullptr, OMP_PROC_BIND_default, true));1280 1281  EXPECT_EQ(NumBodiesGenerated, 1U);1282  EXPECT_EQ(NumPrivatizedVars, 0U);1283  EXPECT_EQ(NumFinalizationPoints, 1U);1284  EXPECT_TRUE(FakeDestructor->hasNUses(1));1285 1286  Builder.restoreIP(AfterIP);1287  Builder.CreateRetVoid();1288  OMPBuilder.finalize();1289 1290  EXPECT_FALSE(verifyModule(*M, &errs()));1291 1292  BasicBlock *ExitBB = nullptr;1293  for (const User *Usr : FakeDestructor->users()) {1294    const CallInst *CI = dyn_cast<CallInst>(Usr);1295    ASSERT_EQ(CI->getCalledFunction(), FakeDestructor);1296    ASSERT_TRUE(isa<BranchInst>(CI->getNextNode()));1297    ASSERT_EQ(CI->getNextNode()->getNumSuccessors(), 1U);1298    if (ExitBB)1299      ASSERT_EQ(CI->getNextNode()->getSuccessor(0), ExitBB);1300    else1301      ExitBB = CI->getNextNode()->getSuccessor(0);1302    ASSERT_EQ(ExitBB->size(), 1U);1303    if (!isa<ReturnInst>(ExitBB->front())) {1304      ASSERT_TRUE(isa<BranchInst>(ExitBB->front()));1305      ASSERT_EQ(cast<BranchInst>(ExitBB->front()).getNumSuccessors(), 1U);1306      ASSERT_TRUE(isa<ReturnInst>(1307          cast<BranchInst>(ExitBB->front()).getSuccessor(0)->front()));1308    }1309  }1310}1311 1312TEST_F(OpenMPIRBuilderTest, ParallelForwardAsPointers) {1313  OpenMPIRBuilder OMPBuilder(*M);1314  OMPBuilder.Config.IsTargetDevice = false;1315  OMPBuilder.initialize();1316  F->setName("func");1317  IRBuilder<> Builder(BB);1318  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});1319  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;1320 1321  Type *I32Ty = Type::getInt32Ty(M->getContext());1322  Type *PtrTy = PointerType::get(M->getContext(), 0);1323  Type *StructTy = StructType::get(I32Ty, PtrTy);1324  Type *VoidTy = Type::getVoidTy(M->getContext());1325  FunctionCallee RetI32Func = M->getOrInsertFunction("ret_i32", I32Ty);1326  FunctionCallee TakeI32Func =1327      M->getOrInsertFunction("take_i32", VoidTy, I32Ty);1328  FunctionCallee RetI32PtrFunc = M->getOrInsertFunction("ret_i32ptr", PtrTy);1329  FunctionCallee TakeI32PtrFunc =1330      M->getOrInsertFunction("take_i32ptr", VoidTy, PtrTy);1331  FunctionCallee RetStructFunc = M->getOrInsertFunction("ret_struct", StructTy);1332  FunctionCallee TakeStructFunc =1333      M->getOrInsertFunction("take_struct", VoidTy, StructTy);1334  FunctionCallee RetStructPtrFunc =1335      M->getOrInsertFunction("ret_structptr", PtrTy);1336  FunctionCallee TakeStructPtrFunc =1337      M->getOrInsertFunction("take_structPtr", VoidTy, PtrTy);1338  Value *I32Val = Builder.CreateCall(RetI32Func);1339  Value *I32PtrVal = Builder.CreateCall(RetI32PtrFunc);1340  Value *StructVal = Builder.CreateCall(RetStructFunc);1341  Value *StructPtrVal = Builder.CreateCall(RetStructPtrFunc);1342 1343  Instruction *Internal;1344  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {1345    IRBuilder<>::InsertPointGuard Guard(Builder);1346    Builder.restoreIP(CodeGenIP);1347    Internal = Builder.CreateCall(TakeI32Func, I32Val);1348    Builder.CreateCall(TakeI32PtrFunc, I32PtrVal);1349    Builder.CreateCall(TakeStructFunc, StructVal);1350    Builder.CreateCall(TakeStructPtrFunc, StructPtrVal);1351    return Error::success();1352  };1353  auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &,1354                    Value &Inner, Value *&ReplacementValue) {1355    ReplacementValue = &Inner;1356    return CodeGenIP;1357  };1358  auto FiniCB = [](InsertPointTy) { return Error::success(); };1359 1360  IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),1361                                    F->getEntryBlock().getFirstInsertionPt());1362  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,1363                       OMPBuilder.createParallel(1364                           Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, nullptr,1365                           nullptr, OMP_PROC_BIND_default, false));1366  Builder.restoreIP(AfterIP);1367  Builder.CreateRetVoid();1368 1369  OMPBuilder.finalize();1370 1371  EXPECT_FALSE(verifyModule(*M, &errs()));1372  Function *OutlinedFn = Internal->getFunction();1373 1374  Type *Arg2Type = OutlinedFn->getArg(2)->getType();1375  EXPECT_TRUE(Arg2Type->isPointerTy());1376}1377 1378TEST_F(OpenMPIRBuilderTest, CanonicalLoopSimple) {1379  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;1380  OpenMPIRBuilder OMPBuilder(*M);1381  OMPBuilder.initialize();1382  IRBuilder<> Builder(BB);1383  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});1384  Value *TripCount = F->getArg(0);1385 1386  unsigned NumBodiesGenerated = 0;1387  auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {1388    NumBodiesGenerated += 1;1389 1390    Builder.restoreIP(CodeGenIP);1391 1392    Value *Cmp = Builder.CreateICmpEQ(LC, TripCount);1393    Instruction *ThenTerm, *ElseTerm;1394    SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(),1395                                  &ThenTerm, &ElseTerm);1396    return Error::success();1397  };1398 1399  ASSERT_EXPECTED_INIT(1400      CanonicalLoopInfo *, Loop,1401      OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, TripCount));1402 1403  Builder.restoreIP(Loop->getAfterIP());1404  ReturnInst *RetInst = Builder.CreateRetVoid();1405  OMPBuilder.finalize();1406 1407  Loop->assertOK();1408  EXPECT_FALSE(verifyModule(*M, &errs()));1409 1410  EXPECT_EQ(NumBodiesGenerated, 1U);1411 1412  // Verify control flow structure (in addition to Loop->assertOK()).1413  EXPECT_EQ(Loop->getPreheader()->getSinglePredecessor(), &F->getEntryBlock());1414  EXPECT_EQ(Loop->getAfter(), Builder.GetInsertBlock());1415 1416  Instruction *IndVar = Loop->getIndVar();1417  EXPECT_TRUE(isa<PHINode>(IndVar));1418  EXPECT_EQ(IndVar->getType(), TripCount->getType());1419  EXPECT_EQ(IndVar->getParent(), Loop->getHeader());1420 1421  EXPECT_EQ(Loop->getTripCount(), TripCount);1422 1423  BasicBlock *Body = Loop->getBody();1424  Instruction *CmpInst = &Body->front();1425  EXPECT_TRUE(isa<ICmpInst>(CmpInst));1426  EXPECT_EQ(CmpInst->getOperand(0), IndVar);1427 1428  BasicBlock *LatchPred = Loop->getLatch()->getSinglePredecessor();1429  EXPECT_TRUE(llvm::all_of(successors(Body), [=](BasicBlock *SuccBB) {1430    return SuccBB->getSingleSuccessor() == LatchPred;1431  }));1432 1433  EXPECT_EQ(&Loop->getAfter()->front(), RetInst);1434}1435 1436TEST_F(OpenMPIRBuilderTest, CanonicalLoopTripCount) {1437  OpenMPIRBuilder OMPBuilder(*M);1438  OMPBuilder.initialize();1439  IRBuilder<> Builder(BB);1440 1441  // Check the trip count is computed correctly. We generate the canonical loop1442  // but rely on the IRBuilder's constant folder to compute the final result1443  // since all inputs are constant. To verify overflow situations, limit the1444  // trip count / loop counter widths to 16 bits.1445  auto EvalTripCount = [&](int64_t Start, int64_t Stop, int64_t Step,1446                           bool IsSigned, bool InclusiveStop) -> int64_t {1447    OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});1448    Type *LCTy = Type::getInt16Ty(Ctx);1449    Value *StartVal = ConstantInt::get(LCTy, Start);1450    Value *StopVal = ConstantInt::get(LCTy, Stop);1451    Value *StepVal = ConstantInt::get(LCTy, Step);1452    Value *TripCount = OMPBuilder.calculateCanonicalLoopTripCount(1453        Loc, StartVal, StopVal, StepVal, IsSigned, InclusiveStop);1454    return cast<ConstantInt>(TripCount)->getValue().getZExtValue();1455  };1456 1457  EXPECT_EQ(EvalTripCount(0, 0, 1, false, false), 0);1458  EXPECT_EQ(EvalTripCount(0, 1, 2, false, false), 1);1459  EXPECT_EQ(EvalTripCount(0, 42, 1, false, false), 42);1460  EXPECT_EQ(EvalTripCount(0, 42, 2, false, false), 21);1461  EXPECT_EQ(EvalTripCount(21, 42, 1, false, false), 21);1462  EXPECT_EQ(EvalTripCount(0, 5, 5, false, false), 1);1463  EXPECT_EQ(EvalTripCount(0, 9, 5, false, false), 2);1464  EXPECT_EQ(EvalTripCount(0, 11, 5, false, false), 3);1465  EXPECT_EQ(EvalTripCount(0, 0xFFFF, 1, false, false), 0xFFFF);1466  EXPECT_EQ(EvalTripCount(0xFFFF, 0, 1, false, false), 0);1467  EXPECT_EQ(EvalTripCount(0xFFFE, 0xFFFF, 1, false, false), 1);1468  EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0x100, false, false), 0x100);1469  EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0xFFFF, false, false), 1);1470 1471  EXPECT_EQ(EvalTripCount(0, 6, 5, false, false), 2);1472  EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0xFFFE, false, false), 2);1473  EXPECT_EQ(EvalTripCount(0, 0, 1, false, true), 1);1474  EXPECT_EQ(EvalTripCount(0, 0, 0xFFFF, false, true), 1);1475  EXPECT_EQ(EvalTripCount(0, 0xFFFE, 1, false, true), 0xFFFF);1476  EXPECT_EQ(EvalTripCount(0, 0xFFFE, 2, false, true), 0x8000);1477 1478  EXPECT_EQ(EvalTripCount(0, 0, -1, true, false), 0);1479  EXPECT_EQ(EvalTripCount(0, 1, -1, true, true), 0);1480  EXPECT_EQ(EvalTripCount(20, 5, -5, true, false), 3);1481  EXPECT_EQ(EvalTripCount(20, 5, -5, true, true), 4);1482  EXPECT_EQ(EvalTripCount(-4, -2, 2, true, false), 1);1483  EXPECT_EQ(EvalTripCount(-4, -3, 2, true, false), 1);1484  EXPECT_EQ(EvalTripCount(-4, -2, 2, true, true), 2);1485 1486  EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 1, true, false), 0x8000);1487  EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 1, true, true), 0x8001);1488  EXPECT_EQ(EvalTripCount(INT16_MIN, 0x7FFF, 1, true, false), 0xFFFF);1489  EXPECT_EQ(EvalTripCount(INT16_MIN + 1, 0x7FFF, 1, true, true), 0xFFFF);1490  EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 0x7FFF, true, false), 2);1491  EXPECT_EQ(EvalTripCount(0x7FFF, 0, -1, true, false), 0x7FFF);1492  EXPECT_EQ(EvalTripCount(0, INT16_MIN, -1, true, false), 0x8000);1493  EXPECT_EQ(EvalTripCount(0, INT16_MIN, -16, true, false), 0x800);1494  EXPECT_EQ(EvalTripCount(0x7FFF, INT16_MIN, -1, true, false), 0xFFFF);1495  EXPECT_EQ(EvalTripCount(0x7FFF, 1, INT16_MIN, true, false), 1);1496  EXPECT_EQ(EvalTripCount(0x7FFF, -1, INT16_MIN, true, true), 2);1497 1498  // Finalize the function and verify it.1499  Builder.CreateRetVoid();1500  OMPBuilder.finalize();1501  EXPECT_FALSE(verifyModule(*M, &errs()));1502}1503 1504TEST_F(OpenMPIRBuilderTest, CollapseNestedLoops) {1505  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;1506  OpenMPIRBuilder OMPBuilder(*M);1507  OMPBuilder.initialize();1508  F->setName("func");1509 1510  IRBuilder<> Builder(BB);1511 1512  Type *LCTy = F->getArg(0)->getType();1513  Constant *One = ConstantInt::get(LCTy, 1);1514  Constant *Two = ConstantInt::get(LCTy, 2);1515  Value *OuterTripCount =1516      Builder.CreateAdd(F->getArg(0), Two, "tripcount.outer");1517  Value *InnerTripCount =1518      Builder.CreateAdd(F->getArg(0), One, "tripcount.inner");1519 1520  // Fix an insertion point for ComputeIP.1521  BasicBlock *LoopNextEnter =1522      BasicBlock::Create(M->getContext(), "loopnest.enter", F,1523                         Builder.GetInsertBlock()->getNextNode());1524  BranchInst *EnterBr = Builder.CreateBr(LoopNextEnter);1525  InsertPointTy ComputeIP{EnterBr->getParent(), EnterBr->getIterator()};1526 1527  Builder.SetInsertPoint(LoopNextEnter);1528  OpenMPIRBuilder::LocationDescription OuterLoc(Builder.saveIP(), DL);1529 1530  CanonicalLoopInfo *InnerLoop = nullptr;1531  CallInst *InbetweenLead = nullptr;1532  CallInst *InbetweenTrail = nullptr;1533  CallInst *Call = nullptr;1534  auto OuterLoopBodyGenCB = [&](InsertPointTy OuterCodeGenIP, Value *OuterLC) {1535    Builder.restoreIP(OuterCodeGenIP);1536    InbetweenLead =1537        createPrintfCall(Builder, "In-between lead i=%d\\n", {OuterLC});1538 1539    auto InnerLoopBodyGenCB = [&](InsertPointTy InnerCodeGenIP,1540                                  Value *InnerLC) {1541      Builder.restoreIP(InnerCodeGenIP);1542      Call = createPrintfCall(Builder, "body i=%d j=%d\\n", {OuterLC, InnerLC});1543      return Error::success();1544    };1545    ASSERT_EXPECTED_INIT(1546        CanonicalLoopInfo *, InnerLoopResult,1547        OMPBuilder.createCanonicalLoop(Builder.saveIP(), InnerLoopBodyGenCB,1548                                       InnerTripCount, "inner"));1549    InnerLoop = InnerLoopResult;1550 1551    Builder.restoreIP(InnerLoop->getAfterIP());1552    InbetweenTrail =1553        createPrintfCall(Builder, "In-between trail i=%d\\n", {OuterLC});1554  };1555  ASSERT_EXPECTED_INIT(CanonicalLoopInfo *, OuterLoop,1556                       OMPBuilder.createCanonicalLoop(1557                           OuterLoc, LOOP_BODYGENCB_WRAPPER(OuterLoopBodyGenCB),1558                           OuterTripCount, "outer"));1559 1560  // Finish the function.1561  Builder.restoreIP(OuterLoop->getAfterIP());1562  Builder.CreateRetVoid();1563 1564  CanonicalLoopInfo *Collapsed =1565      OMPBuilder.collapseLoops(DL, {OuterLoop, InnerLoop}, ComputeIP);1566 1567  OMPBuilder.finalize();1568  EXPECT_FALSE(verifyModule(*M, &errs()));1569 1570  // Verify control flow and BB order.1571  BasicBlock *RefOrder[] = {1572      Collapsed->getPreheader(),   Collapsed->getHeader(),1573      Collapsed->getCond(),        Collapsed->getBody(),1574      InbetweenLead->getParent(),  Call->getParent(),1575      InbetweenTrail->getParent(), Collapsed->getLatch(),1576      Collapsed->getExit(),        Collapsed->getAfter(),1577  };1578  EXPECT_TRUE(verifyDFSOrder(F, RefOrder));1579  EXPECT_TRUE(verifyListOrder(F, RefOrder));1580 1581  // Verify the total trip count.1582  auto *TripCount = cast<MulOperator>(Collapsed->getTripCount());1583  EXPECT_EQ(TripCount->getOperand(0), OuterTripCount);1584  EXPECT_EQ(TripCount->getOperand(1), InnerTripCount);1585 1586  // Verify the changed indvar.1587  auto *OuterIV = cast<BinaryOperator>(Call->getOperand(1));1588  EXPECT_EQ(OuterIV->getOpcode(), Instruction::UDiv);1589  EXPECT_EQ(OuterIV->getParent(), Collapsed->getBody());1590  EXPECT_EQ(OuterIV->getOperand(1), InnerTripCount);1591  EXPECT_EQ(OuterIV->getOperand(0), Collapsed->getIndVar());1592 1593  auto *InnerIV = cast<BinaryOperator>(Call->getOperand(2));1594  EXPECT_EQ(InnerIV->getOpcode(), Instruction::URem);1595  EXPECT_EQ(InnerIV->getParent(), Collapsed->getBody());1596  EXPECT_EQ(InnerIV->getOperand(0), Collapsed->getIndVar());1597  EXPECT_EQ(InnerIV->getOperand(1), InnerTripCount);1598 1599  EXPECT_EQ(InbetweenLead->getOperand(1), OuterIV);1600  EXPECT_EQ(InbetweenTrail->getOperand(1), OuterIV);1601}1602 1603TEST_F(OpenMPIRBuilderTest, TileSingleLoop) {1604  OpenMPIRBuilder OMPBuilder(*M);1605  CallInst *Call;1606  BasicBlock *BodyCode;1607  CanonicalLoopInfo *Loop =1608      buildSingleLoopFunction(DL, OMPBuilder, 32, &Call, &BodyCode);1609  ASSERT_NE(Loop, nullptr);1610 1611  Instruction *OrigIndVar = Loop->getIndVar();1612  EXPECT_EQ(Call->getOperand(1), OrigIndVar);1613 1614  // Tile the loop.1615  Constant *TileSize = ConstantInt::get(Loop->getIndVarType(), APInt(32, 7));1616  std::vector<CanonicalLoopInfo *> GenLoops =1617      OMPBuilder.tileLoops(DL, {Loop}, {TileSize});1618 1619  OMPBuilder.finalize();1620  EXPECT_FALSE(verifyModule(*M, &errs()));1621 1622  EXPECT_EQ(GenLoops.size(), 2u);1623  CanonicalLoopInfo *Floor = GenLoops[0];1624  CanonicalLoopInfo *Tile = GenLoops[1];1625 1626  BasicBlock *RefOrder[] = {1627      Floor->getPreheader(), Floor->getHeader(),   Floor->getCond(),1628      Floor->getBody(),      Tile->getPreheader(), Tile->getHeader(),1629      Tile->getCond(),       Tile->getBody(),      BodyCode,1630      Tile->getLatch(),      Tile->getExit(),      Tile->getAfter(),1631      Floor->getLatch(),     Floor->getExit(),     Floor->getAfter(),1632  };1633  EXPECT_TRUE(verifyDFSOrder(F, RefOrder));1634  EXPECT_TRUE(verifyListOrder(F, RefOrder));1635 1636  // Check the induction variable.1637  EXPECT_EQ(Call->getParent(), BodyCode);1638  auto *Shift = cast<AddOperator>(Call->getOperand(1));1639  EXPECT_EQ(cast<Instruction>(Shift)->getParent(), Tile->getBody());1640  EXPECT_EQ(Shift->getOperand(1), Tile->getIndVar());1641  auto *Scale = cast<MulOperator>(Shift->getOperand(0));1642  EXPECT_EQ(cast<Instruction>(Scale)->getParent(), Tile->getBody());1643  EXPECT_EQ(Scale->getOperand(0), TileSize);1644  EXPECT_EQ(Scale->getOperand(1), Floor->getIndVar());1645}1646 1647TEST_F(OpenMPIRBuilderTest, TileNestedLoops) {1648  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;1649  OpenMPIRBuilder OMPBuilder(*M);1650  OMPBuilder.initialize();1651  F->setName("func");1652 1653  IRBuilder<> Builder(BB);1654  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});1655  Value *TripCount = F->getArg(0);1656  Type *LCTy = TripCount->getType();1657 1658  BasicBlock *BodyCode = nullptr;1659  CanonicalLoopInfo *InnerLoop = nullptr;1660  auto OuterLoopBodyGenCB = [&](InsertPointTy OuterCodeGenIP,1661                                llvm::Value *OuterLC) {1662    auto InnerLoopBodyGenCB = [&](InsertPointTy InnerCodeGenIP,1663                                  llvm::Value *InnerLC) {1664      Builder.restoreIP(InnerCodeGenIP);1665      BodyCode = Builder.GetInsertBlock();1666 1667      // Add something that consumes the induction variables to the body.1668      createPrintfCall(Builder, "i=%d j=%d\\n", {OuterLC, InnerLC});1669      return Error::success();1670    };1671    ASSERT_EXPECTED_INIT(CanonicalLoopInfo *, InnerLoopResult,1672                         OMPBuilder.createCanonicalLoop(OuterCodeGenIP,1673                                                        InnerLoopBodyGenCB,1674                                                        TripCount, "inner"));1675    InnerLoop = InnerLoopResult;1676  };1677  ASSERT_EXPECTED_INIT(1678      CanonicalLoopInfo *, OuterLoop,1679      OMPBuilder.createCanonicalLoop(1680          Loc, LOOP_BODYGENCB_WRAPPER(OuterLoopBodyGenCB), TripCount, "outer"));1681 1682  // Finalize the function.1683  Builder.restoreIP(OuterLoop->getAfterIP());1684  Builder.CreateRetVoid();1685 1686  // Tile to loop nest.1687  Constant *OuterTileSize = ConstantInt::get(LCTy, APInt(32, 11));1688  Constant *InnerTileSize = ConstantInt::get(LCTy, APInt(32, 7));1689  std::vector<CanonicalLoopInfo *> GenLoops = OMPBuilder.tileLoops(1690      DL, {OuterLoop, InnerLoop}, {OuterTileSize, InnerTileSize});1691 1692  OMPBuilder.finalize();1693  EXPECT_FALSE(verifyModule(*M, &errs()));1694 1695  EXPECT_EQ(GenLoops.size(), 4u);1696  CanonicalLoopInfo *Floor1 = GenLoops[0];1697  CanonicalLoopInfo *Floor2 = GenLoops[1];1698  CanonicalLoopInfo *Tile1 = GenLoops[2];1699  CanonicalLoopInfo *Tile2 = GenLoops[3];1700 1701  BasicBlock *RefOrder[] = {1702      Floor1->getPreheader(),1703      Floor1->getHeader(),1704      Floor1->getCond(),1705      Floor1->getBody(),1706      Floor2->getPreheader(),1707      Floor2->getHeader(),1708      Floor2->getCond(),1709      Floor2->getBody(),1710      Tile1->getPreheader(),1711      Tile1->getHeader(),1712      Tile1->getCond(),1713      Tile1->getBody(),1714      Tile2->getPreheader(),1715      Tile2->getHeader(),1716      Tile2->getCond(),1717      Tile2->getBody(),1718      BodyCode,1719      Tile2->getLatch(),1720      Tile2->getExit(),1721      Tile2->getAfter(),1722      Tile1->getLatch(),1723      Tile1->getExit(),1724      Tile1->getAfter(),1725      Floor2->getLatch(),1726      Floor2->getExit(),1727      Floor2->getAfter(),1728      Floor1->getLatch(),1729      Floor1->getExit(),1730      Floor1->getAfter(),1731  };1732  EXPECT_TRUE(verifyDFSOrder(F, RefOrder));1733  EXPECT_TRUE(verifyListOrder(F, RefOrder));1734}1735 1736TEST_F(OpenMPIRBuilderTest, TileNestedLoopsWithBounds) {1737  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;1738  OpenMPIRBuilder OMPBuilder(*M);1739  OMPBuilder.initialize();1740  F->setName("func");1741 1742  IRBuilder<> Builder(BB);1743  Value *TripCount = F->getArg(0);1744  Type *LCTy = TripCount->getType();1745 1746  Value *OuterStartVal = ConstantInt::get(LCTy, 2);1747  Value *OuterStopVal = TripCount;1748  Value *OuterStep = ConstantInt::get(LCTy, 5);1749  Value *InnerStartVal = ConstantInt::get(LCTy, 13);1750  Value *InnerStopVal = TripCount;1751  Value *InnerStep = ConstantInt::get(LCTy, 3);1752 1753  // Fix an insertion point for ComputeIP.1754  BasicBlock *LoopNextEnter =1755      BasicBlock::Create(M->getContext(), "loopnest.enter", F,1756                         Builder.GetInsertBlock()->getNextNode());1757  BranchInst *EnterBr = Builder.CreateBr(LoopNextEnter);1758  InsertPointTy ComputeIP{EnterBr->getParent(), EnterBr->getIterator()};1759 1760  InsertPointTy LoopIP{LoopNextEnter, LoopNextEnter->begin()};1761  OpenMPIRBuilder::LocationDescription Loc({LoopIP, DL});1762 1763  BasicBlock *BodyCode = nullptr;1764  CanonicalLoopInfo *InnerLoop = nullptr;1765  CallInst *Call = nullptr;1766  auto OuterLoopBodyGenCB = [&](InsertPointTy OuterCodeGenIP,1767                                llvm::Value *OuterLC) {1768    auto InnerLoopBodyGenCB = [&](InsertPointTy InnerCodeGenIP,1769                                  llvm::Value *InnerLC) {1770      Builder.restoreIP(InnerCodeGenIP);1771      BodyCode = Builder.GetInsertBlock();1772 1773      // Add something that consumes the induction variable to the body.1774      Call = createPrintfCall(Builder, "i=%d j=%d\\n", {OuterLC, InnerLC});1775      return Error::success();1776    };1777    ASSERT_EXPECTED_INIT(1778        CanonicalLoopInfo *, InnerLoopResult,1779        OMPBuilder.createCanonicalLoop(OuterCodeGenIP, InnerLoopBodyGenCB,1780                                       InnerStartVal, InnerStopVal, InnerStep,1781                                       false, false, ComputeIP, "inner"));1782    InnerLoop = InnerLoopResult;1783  };1784  ASSERT_EXPECTED_INIT(CanonicalLoopInfo *, OuterLoop,1785                       OMPBuilder.createCanonicalLoop(1786                           Loc, LOOP_BODYGENCB_WRAPPER(OuterLoopBodyGenCB),1787                           OuterStartVal, OuterStopVal, OuterStep, false, false,1788                           ComputeIP, "outer"));1789 1790  // Finalize the function1791  Builder.restoreIP(OuterLoop->getAfterIP());1792  Builder.CreateRetVoid();1793 1794  // Tile the loop nest.1795  Constant *TileSize0 = ConstantInt::get(LCTy, APInt(32, 11));1796  Constant *TileSize1 = ConstantInt::get(LCTy, APInt(32, 7));1797  std::vector<CanonicalLoopInfo *> GenLoops =1798      OMPBuilder.tileLoops(DL, {OuterLoop, InnerLoop}, {TileSize0, TileSize1});1799 1800  OMPBuilder.finalize();1801  EXPECT_FALSE(verifyModule(*M, &errs()));1802 1803  EXPECT_EQ(GenLoops.size(), 4u);1804  CanonicalLoopInfo *Floor0 = GenLoops[0];1805  CanonicalLoopInfo *Floor1 = GenLoops[1];1806  CanonicalLoopInfo *Tile0 = GenLoops[2];1807  CanonicalLoopInfo *Tile1 = GenLoops[3];1808 1809  BasicBlock *RefOrder[] = {1810      Floor0->getPreheader(),1811      Floor0->getHeader(),1812      Floor0->getCond(),1813      Floor0->getBody(),1814      Floor1->getPreheader(),1815      Floor1->getHeader(),1816      Floor1->getCond(),1817      Floor1->getBody(),1818      Tile0->getPreheader(),1819      Tile0->getHeader(),1820      Tile0->getCond(),1821      Tile0->getBody(),1822      Tile1->getPreheader(),1823      Tile1->getHeader(),1824      Tile1->getCond(),1825      Tile1->getBody(),1826      BodyCode,1827      Tile1->getLatch(),1828      Tile1->getExit(),1829      Tile1->getAfter(),1830      Tile0->getLatch(),1831      Tile0->getExit(),1832      Tile0->getAfter(),1833      Floor1->getLatch(),1834      Floor1->getExit(),1835      Floor1->getAfter(),1836      Floor0->getLatch(),1837      Floor0->getExit(),1838      Floor0->getAfter(),1839  };1840  EXPECT_TRUE(verifyDFSOrder(F, RefOrder));1841  EXPECT_TRUE(verifyListOrder(F, RefOrder));1842 1843  EXPECT_EQ(Call->getParent(), BodyCode);1844 1845  auto *RangeShift0 = cast<AddOperator>(Call->getOperand(1));1846  EXPECT_EQ(RangeShift0->getOperand(1), OuterStartVal);1847  auto *RangeScale0 = cast<MulOperator>(RangeShift0->getOperand(0));1848  EXPECT_EQ(RangeScale0->getOperand(1), OuterStep);1849  auto *TileShift0 = cast<AddOperator>(RangeScale0->getOperand(0));1850  EXPECT_EQ(cast<Instruction>(TileShift0)->getParent(), Tile1->getBody());1851  EXPECT_EQ(TileShift0->getOperand(1), Tile0->getIndVar());1852  auto *TileScale0 = cast<MulOperator>(TileShift0->getOperand(0));1853  EXPECT_EQ(cast<Instruction>(TileScale0)->getParent(), Tile1->getBody());1854  EXPECT_EQ(TileScale0->getOperand(0), TileSize0);1855  EXPECT_EQ(TileScale0->getOperand(1), Floor0->getIndVar());1856 1857  auto *RangeShift1 = cast<AddOperator>(Call->getOperand(2));1858  EXPECT_EQ(cast<Instruction>(RangeShift1)->getParent(), BodyCode);1859  EXPECT_EQ(RangeShift1->getOperand(1), InnerStartVal);1860  auto *RangeScale1 = cast<MulOperator>(RangeShift1->getOperand(0));1861  EXPECT_EQ(cast<Instruction>(RangeScale1)->getParent(), BodyCode);1862  EXPECT_EQ(RangeScale1->getOperand(1), InnerStep);1863  auto *TileShift1 = cast<AddOperator>(RangeScale1->getOperand(0));1864  EXPECT_EQ(cast<Instruction>(TileShift1)->getParent(), Tile1->getBody());1865  EXPECT_EQ(TileShift1->getOperand(1), Tile1->getIndVar());1866  auto *TileScale1 = cast<MulOperator>(TileShift1->getOperand(0));1867  EXPECT_EQ(cast<Instruction>(TileScale1)->getParent(), Tile1->getBody());1868  EXPECT_EQ(TileScale1->getOperand(0), TileSize1);1869  EXPECT_EQ(TileScale1->getOperand(1), Floor1->getIndVar());1870}1871 1872TEST_F(OpenMPIRBuilderTest, TileSingleLoopCounts) {1873  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;1874  OpenMPIRBuilder OMPBuilder(*M);1875  OMPBuilder.initialize();1876  IRBuilder<> Builder(BB);1877 1878  // Create a loop, tile it, and extract its trip count. All input values are1879  // constant and IRBuilder evaluates all-constant arithmetic inplace, such that1880  // the floor trip count itself will be a ConstantInt. Unfortunately we cannot1881  // do the same for the tile loop.1882  auto GetFloorCount = [&](int64_t Start, int64_t Stop, int64_t Step,1883                           bool IsSigned, bool InclusiveStop,1884                           int64_t TileSize) -> uint64_t {1885    OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);1886    Type *LCTy = Type::getInt16Ty(Ctx);1887    Value *StartVal = ConstantInt::get(LCTy, Start);1888    Value *StopVal = ConstantInt::get(LCTy, Stop);1889    Value *StepVal = ConstantInt::get(LCTy, Step);1890 1891    // Generate a loop.1892    auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {1893      return Error::success();1894    };1895    ASSERT_EXPECTED_INIT_RETURN(1896        CanonicalLoopInfo *, Loop,1897        OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, StartVal, StopVal,1898                                       StepVal, IsSigned, InclusiveStop),1899        (unsigned)-1);1900    InsertPointTy AfterIP = Loop->getAfterIP();1901 1902    // Tile the loop.1903    Value *TileSizeVal = ConstantInt::get(LCTy, TileSize);1904    std::vector<CanonicalLoopInfo *> GenLoops =1905        OMPBuilder.tileLoops(Loc.DL, {Loop}, {TileSizeVal});1906 1907    // Set the insertion pointer to after loop, where the next loop will be1908    // emitted.1909    Builder.restoreIP(AfterIP);1910 1911    // Extract the trip count.1912    CanonicalLoopInfo *FloorLoop = GenLoops[0];1913    Value *FloorTripCount = FloorLoop->getTripCount();1914    return cast<ConstantInt>(FloorTripCount)->getValue().getZExtValue();1915  };1916 1917  // Empty iteration domain.1918  EXPECT_EQ(GetFloorCount(0, 0, 1, false, false, 7), 0u);1919  EXPECT_EQ(GetFloorCount(0, -1, 1, false, true, 7), 0u);1920  EXPECT_EQ(GetFloorCount(-1, -1, -1, true, false, 7), 0u);1921  EXPECT_EQ(GetFloorCount(-1, 0, -1, true, true, 7), 0u);1922  EXPECT_EQ(GetFloorCount(-1, -1, 3, true, false, 7), 0u);1923 1924  // Only complete tiles.1925  EXPECT_EQ(GetFloorCount(0, 14, 1, false, false, 7), 2u);1926  EXPECT_EQ(GetFloorCount(0, 14, 1, false, false, 7), 2u);1927  EXPECT_EQ(GetFloorCount(1, 15, 1, false, false, 7), 2u);1928  EXPECT_EQ(GetFloorCount(0, -14, -1, true, false, 7), 2u);1929  EXPECT_EQ(GetFloorCount(-1, -14, -1, true, true, 7), 2u);1930  EXPECT_EQ(GetFloorCount(0, 3 * 7 * 2, 3, false, false, 7), 2u);1931 1932  // Only a partial tile.1933  EXPECT_EQ(GetFloorCount(0, 1, 1, false, false, 7), 1u);1934  EXPECT_EQ(GetFloorCount(0, 6, 1, false, false, 7), 1u);1935  EXPECT_EQ(GetFloorCount(-1, 1, 3, true, false, 7), 1u);1936  EXPECT_EQ(GetFloorCount(-1, -2, -1, true, false, 7), 1u);1937  EXPECT_EQ(GetFloorCount(0, 2, 3, false, false, 7), 1u);1938 1939  // Complete and partial tiles.1940  EXPECT_EQ(GetFloorCount(0, 13, 1, false, false, 7), 2u);1941  EXPECT_EQ(GetFloorCount(0, 15, 1, false, false, 7), 3u);1942  EXPECT_EQ(GetFloorCount(-1, -14, -1, true, false, 7), 2u);1943  EXPECT_EQ(GetFloorCount(0, 3 * 7 * 5 - 1, 3, false, false, 7), 5u);1944  EXPECT_EQ(GetFloorCount(-1, -3 * 7 * 5, -3, true, false, 7), 5u);1945 1946  // Close to 16-bit integer range.1947  EXPECT_EQ(GetFloorCount(0, 0xFFFF, 1, false, false, 1), 0xFFFFu);1948  EXPECT_EQ(GetFloorCount(0, 0xFFFF, 1, false, false, 7), 0xFFFFu / 7 + 1);1949  EXPECT_EQ(GetFloorCount(0, 0xFFFE, 1, false, true, 7), 0xFFFFu / 7 + 1);1950  EXPECT_EQ(GetFloorCount(-0x8000, 0x7FFF, 1, true, false, 7), 0xFFFFu / 7 + 1);1951  EXPECT_EQ(GetFloorCount(-0x7FFF, 0x7FFF, 1, true, true, 7), 0xFFFFu / 7 + 1);1952  EXPECT_EQ(GetFloorCount(0, 0xFFFE, 1, false, false, 0xFFFF), 1u);1953  EXPECT_EQ(GetFloorCount(-0x8000, 0x7FFF, 1, true, false, 0xFFFF), 1u);1954 1955  // Finalize the function.1956  Builder.CreateRetVoid();1957  OMPBuilder.finalize();1958 1959  EXPECT_FALSE(verifyModule(*M, &errs()));1960}1961 1962TEST_F(OpenMPIRBuilderTest, ApplySimd) {1963  OpenMPIRBuilder OMPBuilder(*M);1964  MapVector<Value *, Value *> AlignedVars;1965  CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);1966  ASSERT_NE(CLI, nullptr);1967 1968  // Simd-ize the loop.1969  OMPBuilder.applySimd(CLI, AlignedVars, /* IfCond */ nullptr,1970                       OrderKind::OMP_ORDER_unknown,1971                       /* Simdlen */ nullptr,1972                       /* Safelen */ nullptr);1973 1974  OMPBuilder.finalize();1975  EXPECT_FALSE(verifyModule(*M, &errs()));1976 1977  PassBuilder PB;1978  FunctionAnalysisManager FAM;1979  PB.registerFunctionAnalyses(FAM);1980  LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);1981 1982  const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();1983  EXPECT_EQ(TopLvl.size(), 1u);1984 1985  Loop *L = TopLvl.front();1986  EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));1987  EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));1988 1989  // Check for llvm.access.group metadata attached to the printf1990  // function in the loop body.1991  BasicBlock *LoopBody = CLI->getBody();1992  EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) {1993    return I.getMetadata("llvm.access.group") != nullptr;1994  }));1995}1996 1997TEST_F(OpenMPIRBuilderTest, ApplySimdCustomAligned) {1998  OpenMPIRBuilder OMPBuilder(*M);1999  IRBuilder<> Builder(BB);2000  const int AlignmentValue = 32;2001  llvm::BasicBlock *sourceBlock = Builder.GetInsertBlock();2002  AllocaInst *Alloc1 =2003      Builder.CreateAlloca(Builder.getPtrTy(), Builder.getInt64(1));2004  LoadInst *Load1 = Builder.CreateLoad(Alloc1->getAllocatedType(), Alloc1);2005  MapVector<Value *, Value *> AlignedVars;2006  AlignedVars.insert({Load1, Builder.getInt64(AlignmentValue)});2007 2008  CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);2009  ASSERT_NE(CLI, nullptr);2010 2011  // Simd-ize the loop.2012  OMPBuilder.applySimd(CLI, AlignedVars, /* IfCond */ nullptr,2013                       OrderKind::OMP_ORDER_unknown,2014                       /* Simdlen */ nullptr,2015                       /* Safelen */ nullptr);2016 2017  OMPBuilder.finalize();2018  EXPECT_FALSE(verifyModule(*M, &errs()));2019 2020  PassBuilder PB;2021  FunctionAnalysisManager FAM;2022  PB.registerFunctionAnalyses(FAM);2023  LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);2024 2025  const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();2026  EXPECT_EQ(TopLvl.size(), 1u);2027 2028  Loop *L = TopLvl.front();2029  EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));2030  EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));2031 2032  // Check for llvm.access.group metadata attached to the printf2033  // function in the loop body.2034  BasicBlock *LoopBody = CLI->getBody();2035  EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) {2036    return I.getMetadata("llvm.access.group") != nullptr;2037  }));2038 2039  // Check if number of assumption instructions is equal to number of aligned2040  // variables2041  size_t NumAssummptionCallsInPreheader =2042      count_if(*sourceBlock, [](Instruction &I) { return isa<AssumeInst>(I); });2043  EXPECT_EQ(NumAssummptionCallsInPreheader, AlignedVars.size());2044 2045  // Check if variables are correctly aligned2046  for (Instruction &Instr : *sourceBlock) {2047    if (!isa<AssumeInst>(Instr))2048      continue;2049    AssumeInst *AssumeInstruction = cast<AssumeInst>(&Instr);2050    if (AssumeInstruction->getNumTotalBundleOperands()) {2051      auto Bundle = AssumeInstruction->getOperandBundleAt(0);2052      if (Bundle.getTagName() == "align") {2053        EXPECT_TRUE(isa<ConstantInt>(Bundle.Inputs[1]));2054        auto ConstIntVal = dyn_cast<ConstantInt>(Bundle.Inputs[1]);2055        EXPECT_EQ(ConstIntVal->getSExtValue(), AlignmentValue);2056      }2057    }2058  }2059}2060TEST_F(OpenMPIRBuilderTest, ApplySimdlen) {2061  OpenMPIRBuilder OMPBuilder(*M);2062  MapVector<Value *, Value *> AlignedVars;2063  CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);2064  ASSERT_NE(CLI, nullptr);2065 2066  // Simd-ize the loop.2067  OMPBuilder.applySimd(CLI, AlignedVars,2068                       /* IfCond */ nullptr, OrderKind::OMP_ORDER_unknown,2069                       ConstantInt::get(Type::getInt32Ty(Ctx), 3),2070                       /* Safelen */ nullptr);2071 2072  OMPBuilder.finalize();2073  EXPECT_FALSE(verifyModule(*M, &errs()));2074 2075  PassBuilder PB;2076  FunctionAnalysisManager FAM;2077  PB.registerFunctionAnalyses(FAM);2078  LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);2079 2080  const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();2081  EXPECT_EQ(TopLvl.size(), 1u);2082 2083  Loop *L = TopLvl.front();2084  EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));2085  EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));2086  EXPECT_EQ(getIntLoopAttribute(L, "llvm.loop.vectorize.width"), 3);2087 2088  // Check for llvm.access.group metadata attached to the printf2089  // function in the loop body.2090  BasicBlock *LoopBody = CLI->getBody();2091  EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) {2092    return I.getMetadata("llvm.access.group") != nullptr;2093  }));2094}2095 2096TEST_F(OpenMPIRBuilderTest, ApplySafelenOrderConcurrent) {2097  OpenMPIRBuilder OMPBuilder(*M);2098  MapVector<Value *, Value *> AlignedVars;2099 2100  CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);2101  ASSERT_NE(CLI, nullptr);2102 2103  // Simd-ize the loop.2104  OMPBuilder.applySimd(2105      CLI, AlignedVars, /* IfCond */ nullptr, OrderKind::OMP_ORDER_concurrent,2106      /* Simdlen */ nullptr, ConstantInt::get(Type::getInt32Ty(Ctx), 3));2107 2108  OMPBuilder.finalize();2109  EXPECT_FALSE(verifyModule(*M, &errs()));2110 2111  PassBuilder PB;2112  FunctionAnalysisManager FAM;2113  PB.registerFunctionAnalyses(FAM);2114  LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);2115 2116  const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();2117  EXPECT_EQ(TopLvl.size(), 1u);2118 2119  Loop *L = TopLvl.front();2120  // Parallel metadata shoudl be attached because of presence of2121  // the order(concurrent) OpenMP clause2122  EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));2123  EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));2124  EXPECT_EQ(getIntLoopAttribute(L, "llvm.loop.vectorize.width"), 3);2125 2126  // Check for llvm.access.group metadata attached to the printf2127  // function in the loop body.2128  BasicBlock *LoopBody = CLI->getBody();2129  EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) {2130    return I.getMetadata("llvm.access.group") != nullptr;2131  }));2132}2133 2134TEST_F(OpenMPIRBuilderTest, ApplySafelen) {2135  OpenMPIRBuilder OMPBuilder(*M);2136  MapVector<Value *, Value *> AlignedVars;2137 2138  CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);2139  ASSERT_NE(CLI, nullptr);2140 2141  OMPBuilder.applySimd(2142      CLI, AlignedVars, /* IfCond */ nullptr, OrderKind::OMP_ORDER_unknown,2143      /* Simdlen */ nullptr, ConstantInt::get(Type::getInt32Ty(Ctx), 3));2144 2145  OMPBuilder.finalize();2146  EXPECT_FALSE(verifyModule(*M, &errs()));2147 2148  PassBuilder PB;2149  FunctionAnalysisManager FAM;2150  PB.registerFunctionAnalyses(FAM);2151  LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);2152 2153  const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();2154  EXPECT_EQ(TopLvl.size(), 1u);2155 2156  Loop *L = TopLvl.front();2157  EXPECT_FALSE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));2158  EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));2159  EXPECT_EQ(getIntLoopAttribute(L, "llvm.loop.vectorize.width"), 3);2160 2161  // Check for llvm.access.group metadata attached to the printf2162  // function in the loop body.2163  BasicBlock *LoopBody = CLI->getBody();2164  EXPECT_FALSE(any_of(*LoopBody, [](Instruction &I) {2165    return I.getMetadata("llvm.access.group") != nullptr;2166  }));2167}2168 2169TEST_F(OpenMPIRBuilderTest, ApplySimdlenSafelen) {2170  OpenMPIRBuilder OMPBuilder(*M);2171  MapVector<Value *, Value *> AlignedVars;2172 2173  CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);2174  ASSERT_NE(CLI, nullptr);2175 2176  OMPBuilder.applySimd(CLI, AlignedVars, /* IfCond */ nullptr,2177                       OrderKind::OMP_ORDER_unknown,2178                       ConstantInt::get(Type::getInt32Ty(Ctx), 2),2179                       ConstantInt::get(Type::getInt32Ty(Ctx), 3));2180 2181  OMPBuilder.finalize();2182  EXPECT_FALSE(verifyModule(*M, &errs()));2183 2184  PassBuilder PB;2185  FunctionAnalysisManager FAM;2186  PB.registerFunctionAnalyses(FAM);2187  LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);2188 2189  const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();2190  EXPECT_EQ(TopLvl.size(), 1u);2191 2192  Loop *L = TopLvl.front();2193  EXPECT_FALSE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));2194  EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));2195  EXPECT_EQ(getIntLoopAttribute(L, "llvm.loop.vectorize.width"), 2);2196 2197  // Check for llvm.access.group metadata attached to the printf2198  // function in the loop body.2199  BasicBlock *LoopBody = CLI->getBody();2200  EXPECT_FALSE(any_of(*LoopBody, [](Instruction &I) {2201    return I.getMetadata("llvm.access.group") != nullptr;2202  }));2203}2204 2205TEST_F(OpenMPIRBuilderTest, ApplySimdIf) {2206  OpenMPIRBuilder OMPBuilder(*M);2207  IRBuilder<> Builder(BB);2208  MapVector<Value *, Value *> AlignedVars;2209  AllocaInst *Alloc1 = Builder.CreateAlloca(Builder.getInt32Ty());2210  AllocaInst *Alloc2 = Builder.CreateAlloca(Builder.getInt32Ty());2211 2212  // Generation of if condition2213  Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), Alloc1);2214  Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 1U), Alloc2);2215  LoadInst *Load1 = Builder.CreateLoad(Alloc1->getAllocatedType(), Alloc1);2216  LoadInst *Load2 = Builder.CreateLoad(Alloc2->getAllocatedType(), Alloc2);2217 2218  Value *IfCmp = Builder.CreateICmpNE(Load1, Load2);2219 2220  CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);2221  ASSERT_NE(CLI, nullptr);2222 2223  // Simd-ize the loop with if condition2224  OMPBuilder.applySimd(CLI, AlignedVars, IfCmp, OrderKind::OMP_ORDER_unknown,2225                       ConstantInt::get(Type::getInt32Ty(Ctx), 3),2226                       /* Safelen */ nullptr);2227 2228  OMPBuilder.finalize();2229  EXPECT_FALSE(verifyModule(*M, &errs()));2230 2231  PassBuilder PB;2232  FunctionAnalysisManager FAM;2233  PB.registerFunctionAnalyses(FAM);2234  LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);2235 2236  // Check if there is one loop containing branches with and without2237  // vectorization2238  const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();2239  EXPECT_EQ(TopLvl.size(), 1u);2240 2241  Loop *L = TopLvl[0];2242  EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));2243  // These attributes cannot not be set because the loop is shared between simd2244  // and non-simd versions2245  EXPECT_FALSE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));2246  EXPECT_EQ(getIntLoopAttribute(L, "llvm.loop.vectorize.width"), 0);2247 2248  // Check for if condition2249  BasicBlock *LoopBody = CLI->getBody();2250  BranchInst *IfCond = cast<BranchInst>(LoopBody->getTerminator());2251  EXPECT_EQ(IfCond->getCondition(), IfCmp);2252  BasicBlock *TrueBranch = IfCond->getSuccessor(0);2253  BasicBlock *FalseBranch = IfCond->getSuccessor(1)->getUniqueSuccessor();2254 2255  // Check for llvm.access.group metadata attached to the printf2256  // function in the true body.2257  EXPECT_TRUE(any_of(*TrueBranch, [](Instruction &I) {2258    return I.getMetadata("llvm.access.group") != nullptr;2259  }));2260 2261  // Check for llvm.access.group metadata attached to the printf2262  // function in the false body.2263  EXPECT_FALSE(any_of(*FalseBranch, [](Instruction &I) {2264    return I.getMetadata("llvm.access.group") != nullptr;2265  }));2266}2267 2268TEST_F(OpenMPIRBuilderTest, UnrollLoopFull) {2269  OpenMPIRBuilder OMPBuilder(*M);2270 2271  CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);2272  ASSERT_NE(CLI, nullptr);2273 2274  // Unroll the loop.2275  OMPBuilder.unrollLoopFull(DL, CLI);2276 2277  OMPBuilder.finalize();2278  EXPECT_FALSE(verifyModule(*M, &errs()));2279 2280  PassBuilder PB;2281  FunctionAnalysisManager FAM;2282  PB.registerFunctionAnalyses(FAM);2283  LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);2284 2285  const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();2286  EXPECT_EQ(TopLvl.size(), 1u);2287 2288  Loop *L = TopLvl.front();2289  EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.unroll.enable"));2290  EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.unroll.full"));2291}2292 2293TEST_F(OpenMPIRBuilderTest, UnrollLoopPartial) {2294  OpenMPIRBuilder OMPBuilder(*M);2295  CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);2296  ASSERT_NE(CLI, nullptr);2297 2298  // Unroll the loop.2299  CanonicalLoopInfo *UnrolledLoop = nullptr;2300  OMPBuilder.unrollLoopPartial(DL, CLI, 5, &UnrolledLoop);2301  ASSERT_NE(UnrolledLoop, nullptr);2302 2303  OMPBuilder.finalize();2304  EXPECT_FALSE(verifyModule(*M, &errs()));2305  UnrolledLoop->assertOK();2306 2307  PassBuilder PB;2308  FunctionAnalysisManager FAM;2309  PB.registerFunctionAnalyses(FAM);2310  LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);2311 2312  const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();2313  EXPECT_EQ(TopLvl.size(), 1u);2314  Loop *Outer = TopLvl.front();2315  EXPECT_EQ(Outer->getHeader(), UnrolledLoop->getHeader());2316  EXPECT_EQ(Outer->getLoopLatch(), UnrolledLoop->getLatch());2317  EXPECT_EQ(Outer->getExitingBlock(), UnrolledLoop->getCond());2318  EXPECT_EQ(Outer->getExitBlock(), UnrolledLoop->getExit());2319 2320  EXPECT_EQ(Outer->getSubLoops().size(), 1u);2321  Loop *Inner = Outer->getSubLoops().front();2322 2323  EXPECT_TRUE(getBooleanLoopAttribute(Inner, "llvm.loop.unroll.enable"));2324  EXPECT_EQ(getIntLoopAttribute(Inner, "llvm.loop.unroll.count"), 5);2325}2326 2327TEST_F(OpenMPIRBuilderTest, UnrollLoopHeuristic) {2328  OpenMPIRBuilder OMPBuilder(*M);2329 2330  CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);2331  ASSERT_NE(CLI, nullptr);2332 2333  // Unroll the loop.2334  OMPBuilder.unrollLoopHeuristic(DL, CLI);2335 2336  OMPBuilder.finalize();2337  EXPECT_FALSE(verifyModule(*M, &errs()));2338 2339  PassBuilder PB;2340  FunctionAnalysisManager FAM;2341  PB.registerFunctionAnalyses(FAM);2342  LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);2343 2344  const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();2345  EXPECT_EQ(TopLvl.size(), 1u);2346 2347  Loop *L = TopLvl.front();2348  EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.unroll.enable"));2349}2350 2351TEST_F(OpenMPIRBuilderTest, StaticWorkshareLoopTarget) {2352  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;2353  M->setDataLayout(2354      "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:"2355      "256:256:32-p8:128:128:128:48-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-"2356      "v192:"2357      "256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8");2358  OpenMPIRBuilder OMPBuilder(*M);2359  OMPBuilder.Config.IsTargetDevice = true;2360  OMPBuilder.Config.setIsGPU(false);2361  OMPBuilder.initialize();2362  IRBuilder<> Builder(BB);2363  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});2364  InsertPointTy AllocaIP = Builder.saveIP();2365 2366  Type *LCTy = Type::getInt32Ty(Ctx);2367  Value *StartVal = ConstantInt::get(LCTy, 10);2368  Value *StopVal = ConstantInt::get(LCTy, 52);2369  Value *StepVal = ConstantInt::get(LCTy, 2);2370  auto LoopBodyGen = [&](InsertPointTy, Value *) { return Error::success(); };2371 2372  ASSERT_EXPECTED_INIT(CanonicalLoopInfo *, CLI,2373                       OMPBuilder.createCanonicalLoop(Loc, LoopBodyGen,2374                                                      StartVal, StopVal,2375                                                      StepVal, false, false));2376  BasicBlock *Preheader = CLI->getPreheader();2377  Value *TripCount = CLI->getTripCount();2378 2379  Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());2380 2381  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,2382                       OMPBuilder.applyWorkshareLoop(2383                           DL, CLI, AllocaIP, true, OMP_SCHEDULE_Static,2384                           nullptr, false, false, false, false,2385                           WorksharingLoopType::ForStaticLoop));2386  Builder.restoreIP(AfterIP);2387  Builder.CreateRetVoid();2388 2389  OMPBuilder.finalize();2390  EXPECT_FALSE(verifyModule(*M, &errs()));2391 2392  CallInst *WorkshareLoopRuntimeCall = nullptr;2393  int WorkshareLoopRuntimeCallCnt = 0;2394  for (auto Inst = Preheader->begin(); Inst != Preheader->end(); ++Inst) {2395    CallInst *Call = dyn_cast<CallInst>(Inst);2396    if (!Call)2397      continue;2398    if (!Call->getCalledFunction())2399      continue;2400 2401    if (Call->getCalledFunction()->getName() == "__kmpc_for_static_loop_4u") {2402      WorkshareLoopRuntimeCall = Call;2403      WorkshareLoopRuntimeCallCnt++;2404    }2405  }2406  EXPECT_NE(WorkshareLoopRuntimeCall, nullptr);2407  // Verify that there is only one call to workshare loop function2408  EXPECT_EQ(WorkshareLoopRuntimeCallCnt, 1);2409  // Check that pointer to loop body function is passed as second argument2410  Value *LoopBodyFuncArg = WorkshareLoopRuntimeCall->getArgOperand(1);2411  EXPECT_EQ(Builder.getPtrTy(), LoopBodyFuncArg->getType());2412  Function *ArgFunction = dyn_cast<Function>(LoopBodyFuncArg);2413  EXPECT_NE(ArgFunction, nullptr);2414  EXPECT_EQ(ArgFunction->arg_size(), 1u);2415  EXPECT_EQ(ArgFunction->getArg(0)->getType(), TripCount->getType());2416  // Check that no variables except for loop counter are used in loop body2417  EXPECT_EQ(Constant::getNullValue(Builder.getPtrTy()),2418            WorkshareLoopRuntimeCall->getArgOperand(2));2419  // Check loop trip count argument2420  EXPECT_EQ(TripCount, WorkshareLoopRuntimeCall->getArgOperand(3));2421}2422 2423TEST_F(OpenMPIRBuilderTest, StaticWorkShareLoop) {2424  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;2425  OpenMPIRBuilder OMPBuilder(*M);2426  OMPBuilder.Config.IsTargetDevice = false;2427  OMPBuilder.initialize();2428  IRBuilder<> Builder(BB);2429  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});2430 2431  Type *LCTy = Type::getInt32Ty(Ctx);2432  Value *StartVal = ConstantInt::get(LCTy, 10);2433  Value *StopVal = ConstantInt::get(LCTy, 52);2434  Value *StepVal = ConstantInt::get(LCTy, 2);2435  auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {2436    return Error::success();2437  };2438  ASSERT_EXPECTED_INIT(CanonicalLoopInfo *, CLI,2439                       OMPBuilder.createCanonicalLoop(2440                           Loc, LoopBodyGen, StartVal, StopVal, StepVal,2441                           /*IsSigned=*/false, /*InclusiveStop=*/false));2442  BasicBlock *Preheader = CLI->getPreheader();2443  BasicBlock *Body = CLI->getBody();2444  Value *IV = CLI->getIndVar();2445  BasicBlock *ExitBlock = CLI->getExit();2446 2447  Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());2448  InsertPointTy AllocaIP = Builder.saveIP();2449 2450  ASSERT_THAT_EXPECTED(OMPBuilder.applyWorkshareLoop(DL, CLI, AllocaIP,2451                                                     /*NeedsBarrier=*/true,2452                                                     OMP_SCHEDULE_Static),2453                       Succeeded());2454 2455  BasicBlock *Cond = Body->getSinglePredecessor();2456  Instruction *Cmp = &*Cond->begin();2457  Value *TripCount = Cmp->getOperand(1);2458 2459  auto AllocaIter = BB->begin();2460  ASSERT_GE(std::distance(BB->begin(), BB->end()), 4);2461  AllocaInst *PLastIter = dyn_cast<AllocaInst>(&*(AllocaIter++));2462  AllocaInst *PLowerBound = dyn_cast<AllocaInst>(&*(AllocaIter++));2463  AllocaInst *PUpperBound = dyn_cast<AllocaInst>(&*(AllocaIter++));2464  AllocaInst *PStride = dyn_cast<AllocaInst>(&*(AllocaIter++));2465  EXPECT_NE(PLastIter, nullptr);2466  EXPECT_NE(PLowerBound, nullptr);2467  EXPECT_NE(PUpperBound, nullptr);2468  EXPECT_NE(PStride, nullptr);2469 2470  auto PreheaderIter = Preheader->begin();2471  ASSERT_GE(std::distance(Preheader->begin(), Preheader->end()), 7);2472  StoreInst *LowerBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++));2473  StoreInst *UpperBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++));2474  StoreInst *StrideStore = dyn_cast<StoreInst>(&*(PreheaderIter++));2475  ASSERT_NE(LowerBoundStore, nullptr);2476  ASSERT_NE(UpperBoundStore, nullptr);2477  ASSERT_NE(StrideStore, nullptr);2478 2479  auto *OrigLowerBound =2480      dyn_cast<ConstantInt>(LowerBoundStore->getValueOperand());2481  auto *OrigUpperBound =2482      dyn_cast<ConstantInt>(UpperBoundStore->getValueOperand());2483  auto *OrigStride = dyn_cast<ConstantInt>(StrideStore->getValueOperand());2484  ASSERT_NE(OrigLowerBound, nullptr);2485  ASSERT_NE(OrigUpperBound, nullptr);2486  ASSERT_NE(OrigStride, nullptr);2487  EXPECT_EQ(OrigLowerBound->getValue(), 0);2488  EXPECT_EQ(OrigUpperBound->getValue(), 20);2489  EXPECT_EQ(OrigStride->getValue(), 1);2490 2491  // Check that the loop IV is updated to account for the lower bound returned2492  // by the OpenMP runtime call.2493  BinaryOperator *Add = dyn_cast<BinaryOperator>(&Body->front());2494  EXPECT_EQ(Add->getOperand(0), IV);2495  auto *LoadedLowerBound = dyn_cast<LoadInst>(Add->getOperand(1));2496  ASSERT_NE(LoadedLowerBound, nullptr);2497  EXPECT_EQ(LoadedLowerBound->getPointerOperand(), PLowerBound);2498 2499  // Check that the trip count is updated to account for the lower and upper2500  // bounds return by the OpenMP runtime call.2501  auto *AddOne = dyn_cast<Instruction>(TripCount);2502  ASSERT_NE(AddOne, nullptr);2503  ASSERT_TRUE(AddOne->isBinaryOp());2504  auto *One = dyn_cast<ConstantInt>(AddOne->getOperand(1));2505  ASSERT_NE(One, nullptr);2506  EXPECT_EQ(One->getValue(), 1);2507  auto *Difference = dyn_cast<Instruction>(AddOne->getOperand(0));2508  ASSERT_NE(Difference, nullptr);2509  ASSERT_TRUE(Difference->isBinaryOp());2510  EXPECT_EQ(Difference->getOperand(1), LoadedLowerBound);2511  auto *LoadedUpperBound = dyn_cast<LoadInst>(Difference->getOperand(0));2512  ASSERT_NE(LoadedUpperBound, nullptr);2513  EXPECT_EQ(LoadedUpperBound->getPointerOperand(), PUpperBound);2514 2515  // The original loop iterator should only be used in the condition, in the2516  // increment and in the statement that adds the lower bound to it.2517  EXPECT_EQ(std::distance(IV->use_begin(), IV->use_end()), 3);2518 2519  // The exit block should contain the "fini" call and the barrier call,2520  // plus the call to obtain the thread ID.2521  size_t NumCallsInExitBlock =2522      count_if(*ExitBlock, [](Instruction &I) { return isa<CallInst>(I); });2523  EXPECT_EQ(NumCallsInExitBlock, 3u);2524}2525 2526TEST_P(OpenMPIRBuilderTestWithIVBits, StaticChunkedWorkshareLoop) {2527  unsigned IVBits = GetParam();2528 2529  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;2530  OpenMPIRBuilder OMPBuilder(*M);2531  OMPBuilder.Config.IsTargetDevice = false;2532 2533  BasicBlock *Body;2534  CallInst *Call;2535  CanonicalLoopInfo *CLI =2536      buildSingleLoopFunction(DL, OMPBuilder, IVBits, &Call, &Body);2537  ASSERT_NE(CLI, nullptr);2538 2539  Instruction *OrigIndVar = CLI->getIndVar();2540  EXPECT_EQ(Call->getOperand(1), OrigIndVar);2541 2542  Type *LCTy = Type::getInt32Ty(Ctx);2543  Value *ChunkSize = ConstantInt::get(LCTy, 5);2544  InsertPointTy AllocaIP{&F->getEntryBlock(),2545                         F->getEntryBlock().getFirstInsertionPt()};2546  ASSERT_THAT_EXPECTED(OMPBuilder.applyWorkshareLoop(DL, CLI, AllocaIP,2547                                                     /*NeedsBarrier=*/true,2548                                                     OMP_SCHEDULE_Static,2549                                                     ChunkSize),2550                       Succeeded());2551 2552  OMPBuilder.finalize();2553  EXPECT_FALSE(verifyModule(*M, &errs()));2554 2555  BasicBlock *Entry = &F->getEntryBlock();2556  BasicBlock *Preheader = Entry->getSingleSuccessor();2557 2558  BasicBlock *DispatchPreheader = Preheader->getSingleSuccessor();2559  BasicBlock *DispatchHeader = DispatchPreheader->getSingleSuccessor();2560  BasicBlock *DispatchCond = DispatchHeader->getSingleSuccessor();2561  BasicBlock *DispatchBody = succ_begin(DispatchCond)[0];2562  BasicBlock *DispatchExit = succ_begin(DispatchCond)[1];2563  BasicBlock *DispatchAfter = DispatchExit->getSingleSuccessor();2564  BasicBlock *Return = DispatchAfter->getSingleSuccessor();2565 2566  BasicBlock *ChunkPreheader = DispatchBody->getSingleSuccessor();2567  BasicBlock *ChunkHeader = ChunkPreheader->getSingleSuccessor();2568  BasicBlock *ChunkCond = ChunkHeader->getSingleSuccessor();2569  BasicBlock *ChunkBody = succ_begin(ChunkCond)[0];2570  BasicBlock *ChunkExit = succ_begin(ChunkCond)[1];2571  BasicBlock *ChunkInc = ChunkBody->getSingleSuccessor();2572  BasicBlock *ChunkAfter = ChunkExit->getSingleSuccessor();2573 2574  BasicBlock *DispatchInc = ChunkAfter;2575 2576  EXPECT_EQ(ChunkBody, Body);2577  EXPECT_EQ(ChunkInc->getSingleSuccessor(), ChunkHeader);2578  EXPECT_EQ(DispatchInc->getSingleSuccessor(), DispatchHeader);2579 2580  EXPECT_TRUE(isa<ReturnInst>(Return->front()));2581 2582  Value *NewIV = Call->getOperand(1);2583  EXPECT_EQ(NewIV->getType()->getScalarSizeInBits(), IVBits);2584 2585  CallInst *InitCall = findSingleCall(2586      F,2587      (IVBits > 32) ? omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u2588                    : omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u,2589      OMPBuilder);2590  EXPECT_EQ(InitCall->getParent(), Preheader);2591  EXPECT_EQ(cast<ConstantInt>(InitCall->getArgOperand(2))->getSExtValue(), 33);2592  EXPECT_EQ(cast<ConstantInt>(InitCall->getArgOperand(7))->getSExtValue(), 1);2593  EXPECT_EQ(cast<ConstantInt>(InitCall->getArgOperand(8))->getSExtValue(), 5);2594 2595  CallInst *FiniCall = findSingleCall(2596      F, omp::RuntimeFunction::OMPRTL___kmpc_for_static_fini, OMPBuilder);2597  EXPECT_EQ(FiniCall->getParent(), DispatchExit);2598 2599  CallInst *BarrierCall = findSingleCall(2600      F, omp::RuntimeFunction::OMPRTL___kmpc_barrier, OMPBuilder);2601  EXPECT_EQ(BarrierCall->getParent(), DispatchExit);2602}2603 2604INSTANTIATE_TEST_SUITE_P(IVBits, OpenMPIRBuilderTestWithIVBits,2605                         ::testing::Values(8, 16, 32, 64));2606 2607TEST_P(OpenMPIRBuilderTestWithParams, DynamicWorkShareLoop) {2608  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;2609  OpenMPIRBuilder OMPBuilder(*M);2610  OMPBuilder.Config.IsTargetDevice = false;2611  OMPBuilder.initialize();2612  IRBuilder<> Builder(BB);2613  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});2614 2615  omp::OMPScheduleType SchedType = GetParam();2616  uint32_t ChunkSize = 1;2617  switch (SchedType & ~OMPScheduleType::ModifierMask) {2618  case omp::OMPScheduleType::BaseDynamicChunked:2619  case omp::OMPScheduleType::BaseGuidedChunked:2620    ChunkSize = 7;2621    break;2622  case omp::OMPScheduleType::BaseAuto:2623  case omp::OMPScheduleType::BaseRuntime:2624    ChunkSize = 1;2625    break;2626  default:2627    assert(0 && "unknown type for this test");2628    break;2629  }2630 2631  Type *LCTy = Type::getInt32Ty(Ctx);2632  Value *StartVal = ConstantInt::get(LCTy, 10);2633  Value *StopVal = ConstantInt::get(LCTy, 52);2634  Value *StepVal = ConstantInt::get(LCTy, 2);2635  Value *ChunkVal =2636      (ChunkSize == 1) ? nullptr : ConstantInt::get(LCTy, ChunkSize);2637  auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {2638    return Error::success();2639  };2640 2641  ASSERT_EXPECTED_INIT(CanonicalLoopInfo *, CLI,2642                       OMPBuilder.createCanonicalLoop(2643                           Loc, LoopBodyGen, StartVal, StopVal, StepVal,2644                           /*IsSigned=*/false, /*InclusiveStop=*/false));2645 2646  Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());2647  InsertPointTy AllocaIP = Builder.saveIP();2648 2649  // Collect all the info from CLI, as it isn't usable after the call to2650  // createDynamicWorkshareLoop.2651  InsertPointTy AfterIP = CLI->getAfterIP();2652  BasicBlock *Preheader = CLI->getPreheader();2653  BasicBlock *ExitBlock = CLI->getExit();2654  BasicBlock *LatchBlock = CLI->getLatch();2655  Value *IV = CLI->getIndVar();2656 2657  ASSERT_EXPECTED_INIT(2658      OpenMPIRBuilder::InsertPointTy, EndIP,2659      OMPBuilder.applyWorkshareLoop(2660          DL, CLI, AllocaIP, /*NeedsBarrier=*/true, getSchedKind(SchedType),2661          ChunkVal, /*Simd=*/false,2662          (SchedType & omp::OMPScheduleType::ModifierMonotonic) ==2663              omp::OMPScheduleType::ModifierMonotonic,2664          (SchedType & omp::OMPScheduleType::ModifierNonmonotonic) ==2665              omp::OMPScheduleType::ModifierNonmonotonic,2666          /*Ordered=*/false));2667 2668  // The returned value should be the "after" point.2669  ASSERT_EQ(EndIP.getBlock(), AfterIP.getBlock());2670  ASSERT_EQ(EndIP.getPoint(), AfterIP.getPoint());2671 2672  auto AllocaIter = BB->begin();2673  ASSERT_GE(std::distance(BB->begin(), BB->end()), 4);2674  AllocaInst *PLastIter = dyn_cast<AllocaInst>(&*(AllocaIter++));2675  AllocaInst *PLowerBound = dyn_cast<AllocaInst>(&*(AllocaIter++));2676  AllocaInst *PUpperBound = dyn_cast<AllocaInst>(&*(AllocaIter++));2677  AllocaInst *PStride = dyn_cast<AllocaInst>(&*(AllocaIter++));2678  EXPECT_NE(PLastIter, nullptr);2679  EXPECT_NE(PLowerBound, nullptr);2680  EXPECT_NE(PUpperBound, nullptr);2681  EXPECT_NE(PStride, nullptr);2682 2683  auto PreheaderIter = Preheader->begin();2684  ASSERT_GE(std::distance(Preheader->begin(), Preheader->end()), 6);2685  StoreInst *LowerBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++));2686  StoreInst *UpperBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++));2687  StoreInst *StrideStore = dyn_cast<StoreInst>(&*(PreheaderIter++));2688  ASSERT_NE(LowerBoundStore, nullptr);2689  ASSERT_NE(UpperBoundStore, nullptr);2690  ASSERT_NE(StrideStore, nullptr);2691 2692  CallInst *ThreadIdCall = dyn_cast<CallInst>(&*(PreheaderIter++));2693  ASSERT_NE(ThreadIdCall, nullptr);2694  EXPECT_EQ(ThreadIdCall->getCalledFunction()->getName(),2695            "__kmpc_global_thread_num");2696 2697  CallInst *InitCall = dyn_cast<CallInst>(&*PreheaderIter);2698 2699  ASSERT_NE(InitCall, nullptr);2700  EXPECT_EQ(InitCall->getCalledFunction()->getName(),2701            "__kmpc_dispatch_init_4u");2702  EXPECT_EQ(InitCall->arg_size(), 7U);2703  EXPECT_EQ(InitCall->getArgOperand(6), ConstantInt::get(LCTy, ChunkSize));2704  ConstantInt *SchedVal = cast<ConstantInt>(InitCall->getArgOperand(2));2705  if ((SchedType & OMPScheduleType::MonotonicityMask) ==2706      OMPScheduleType::None) {2707    // Implementation is allowed to add default nonmonotonicity flag2708    EXPECT_EQ(2709        static_cast<OMPScheduleType>(SchedVal->getValue().getZExtValue()) |2710            OMPScheduleType::ModifierNonmonotonic,2711        SchedType | OMPScheduleType::ModifierNonmonotonic);2712  } else {2713    EXPECT_EQ(static_cast<OMPScheduleType>(SchedVal->getValue().getZExtValue()),2714              SchedType);2715  }2716 2717  ConstantInt *OrigLowerBound =2718      dyn_cast<ConstantInt>(LowerBoundStore->getValueOperand());2719  ConstantInt *OrigUpperBound =2720      dyn_cast<ConstantInt>(UpperBoundStore->getValueOperand());2721  ConstantInt *OrigStride =2722      dyn_cast<ConstantInt>(StrideStore->getValueOperand());2723  ASSERT_NE(OrigLowerBound, nullptr);2724  ASSERT_NE(OrigUpperBound, nullptr);2725  ASSERT_NE(OrigStride, nullptr);2726  EXPECT_EQ(OrigLowerBound->getValue(), 1);2727  EXPECT_EQ(OrigUpperBound->getValue(), 21);2728  EXPECT_EQ(OrigStride->getValue(), 1);2729 2730  CallInst *FiniCall =2731      dyn_cast<CallInst>(&*(LatchBlock->getTerminator()->getPrevNode()));2732  EXPECT_EQ(FiniCall, nullptr);2733 2734  // The original loop iterator should only be used in the condition, in the2735  // increment and in the statement that adds the lower bound to it.2736  EXPECT_EQ(std::distance(IV->use_begin(), IV->use_end()), 3);2737 2738  // The exit block should contain the barrier call, plus the call to obtain2739  // the thread ID.2740  size_t NumCallsInExitBlock =2741      count_if(*ExitBlock, [](Instruction &I) { return isa<CallInst>(I); });2742  EXPECT_EQ(NumCallsInExitBlock, 2u);2743 2744  // Add a termination to our block and check that it is internally consistent.2745  Builder.restoreIP(EndIP);2746  Builder.CreateRetVoid();2747  OMPBuilder.finalize();2748  EXPECT_FALSE(verifyModule(*M, &errs()));2749}2750 2751INSTANTIATE_TEST_SUITE_P(2752    OpenMPWSLoopSchedulingTypes, OpenMPIRBuilderTestWithParams,2753    ::testing::Values(omp::OMPScheduleType::UnorderedDynamicChunked,2754                      omp::OMPScheduleType::UnorderedGuidedChunked,2755                      omp::OMPScheduleType::UnorderedAuto,2756                      omp::OMPScheduleType::UnorderedRuntime,2757                      omp::OMPScheduleType::UnorderedDynamicChunked |2758                          omp::OMPScheduleType::ModifierMonotonic,2759                      omp::OMPScheduleType::UnorderedDynamicChunked |2760                          omp::OMPScheduleType::ModifierNonmonotonic,2761                      omp::OMPScheduleType::UnorderedGuidedChunked |2762                          omp::OMPScheduleType::ModifierMonotonic,2763                      omp::OMPScheduleType::UnorderedGuidedChunked |2764                          omp::OMPScheduleType::ModifierNonmonotonic,2765                      omp::OMPScheduleType::UnorderedAuto |2766                          omp::OMPScheduleType::ModifierMonotonic,2767                      omp::OMPScheduleType::UnorderedRuntime |2768                          omp::OMPScheduleType::ModifierMonotonic));2769 2770TEST_F(OpenMPIRBuilderTest, DynamicWorkShareLoopOrdered) {2771  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;2772  OpenMPIRBuilder OMPBuilder(*M);2773  OMPBuilder.Config.IsTargetDevice = false;2774  OMPBuilder.initialize();2775  IRBuilder<> Builder(BB);2776  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});2777 2778  uint32_t ChunkSize = 1;2779  Type *LCTy = Type::getInt32Ty(Ctx);2780  Value *StartVal = ConstantInt::get(LCTy, 10);2781  Value *StopVal = ConstantInt::get(LCTy, 52);2782  Value *StepVal = ConstantInt::get(LCTy, 2);2783  Value *ChunkVal = ConstantInt::get(LCTy, ChunkSize);2784  auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {2785    return llvm::Error::success();2786  };2787 2788  ASSERT_EXPECTED_INIT(CanonicalLoopInfo *, CLI,2789                       OMPBuilder.createCanonicalLoop(2790                           Loc, LoopBodyGen, StartVal, StopVal, StepVal,2791                           /*IsSigned=*/false, /*InclusiveStop=*/false));2792 2793  Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());2794  InsertPointTy AllocaIP = Builder.saveIP();2795 2796  // Collect all the info from CLI, as it isn't usable after the call to2797  // createDynamicWorkshareLoop.2798  BasicBlock *Preheader = CLI->getPreheader();2799  BasicBlock *ExitBlock = CLI->getExit();2800  BasicBlock *LatchBlock = CLI->getLatch();2801  Value *IV = CLI->getIndVar();2802 2803  ASSERT_EXPECTED_INIT(2804      OpenMPIRBuilder::InsertPointTy, EndIP,2805      OMPBuilder.applyWorkshareLoop(DL, CLI, AllocaIP, /*NeedsBarrier=*/true,2806                                    OMP_SCHEDULE_Static, ChunkVal,2807                                    /*HasSimdModifier=*/false,2808                                    /*HasMonotonicModifier=*/false,2809                                    /*HasNonmonotonicModifier=*/false,2810                                    /*HasOrderedClause=*/true));2811 2812  // Add a termination to our block and check that it is internally consistent.2813  Builder.restoreIP(EndIP);2814  Builder.CreateRetVoid();2815  OMPBuilder.finalize();2816  EXPECT_FALSE(verifyModule(*M, &errs()));2817 2818  CallInst *InitCall = nullptr;2819  for (Instruction &EI : *Preheader) {2820    Instruction *Cur = &EI;2821    if (isa<CallInst>(Cur)) {2822      InitCall = cast<CallInst>(Cur);2823      if (InitCall->getCalledFunction()->getName() == "__kmpc_dispatch_init_4u")2824        break;2825      InitCall = nullptr;2826    }2827  }2828  EXPECT_NE(InitCall, nullptr);2829  EXPECT_EQ(InitCall->arg_size(), 7U);2830  ConstantInt *SchedVal = cast<ConstantInt>(InitCall->getArgOperand(2));2831  EXPECT_EQ(SchedVal->getValue(),2832            static_cast<uint64_t>(OMPScheduleType::OrderedStaticChunked));2833 2834  CallInst *FiniCall =2835      dyn_cast<CallInst>(&*(LatchBlock->getTerminator()->getPrevNode()));2836  ASSERT_NE(FiniCall, nullptr);2837  EXPECT_EQ(FiniCall->getCalledFunction()->getName(),2838            "__kmpc_dispatch_fini_4u");2839  EXPECT_EQ(FiniCall->arg_size(), 2U);2840  EXPECT_EQ(InitCall->getArgOperand(0), FiniCall->getArgOperand(0));2841  EXPECT_EQ(InitCall->getArgOperand(1), FiniCall->getArgOperand(1));2842 2843  // The original loop iterator should only be used in the condition, in the2844  // increment and in the statement that adds the lower bound to it.2845  EXPECT_EQ(std::distance(IV->use_begin(), IV->use_end()), 3);2846 2847  // The exit block should contain the barrier call, plus the call to obtain2848  // the thread ID.2849  size_t NumCallsInExitBlock =2850      count_if(*ExitBlock, [](Instruction &I) { return isa<CallInst>(I); });2851  EXPECT_EQ(NumCallsInExitBlock, 2u);2852}2853 2854TEST_F(OpenMPIRBuilderTest, MasterDirective) {2855  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;2856  OpenMPIRBuilder OMPBuilder(*M);2857  OMPBuilder.initialize();2858  F->setName("func");2859  IRBuilder<> Builder(BB);2860 2861  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});2862 2863  AllocaInst *PrivAI = nullptr;2864 2865  BasicBlock *EntryBB = nullptr;2866  BasicBlock *ThenBB = nullptr;2867 2868  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {2869    if (AllocaIP.isSet())2870      Builder.restoreIP(AllocaIP);2871    else2872      Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));2873    PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());2874    Builder.CreateStore(F->arg_begin(), PrivAI);2875 2876    llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();2877    llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();2878    EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);2879 2880    Builder.restoreIP(CodeGenIP);2881 2882    // collect some info for checks later2883    ThenBB = Builder.GetInsertBlock();2884    EntryBB = ThenBB->getUniquePredecessor();2885 2886    // simple instructions for body2887    Value *PrivLoad =2888        Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");2889    Builder.CreateICmpNE(F->arg_begin(), PrivLoad);2890  };2891 2892  auto FiniCB = [&](InsertPointTy IP) {2893    BasicBlock *IPBB = IP.getBlock();2894    EXPECT_NE(IPBB->end(), IP.getPoint());2895  };2896 2897  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,2898                       OMPBuilder.createMaster(Builder,2899                                               BODYGENCB_WRAPPER(BodyGenCB),2900                                               FINICB_WRAPPER(FiniCB)));2901  Builder.restoreIP(AfterIP);2902  Value *EntryBBTI = EntryBB->getTerminator();2903  EXPECT_NE(EntryBBTI, nullptr);2904  EXPECT_TRUE(isa<BranchInst>(EntryBBTI));2905  BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator());2906  EXPECT_TRUE(EntryBr->isConditional());2907  EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);2908  BasicBlock *FinalizeBB = ThenBB->getUniqueSuccessor();2909  BasicBlock *ExitBB = FinalizeBB->getUniqueSuccessor();2910  EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);2911 2912  CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition());2913  EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));2914 2915  CallInst *MasterEntryCI = cast<CallInst>(CondInst->getOperand(0));2916  EXPECT_EQ(MasterEntryCI->arg_size(), 2U);2917  EXPECT_EQ(MasterEntryCI->getCalledFunction()->getName(), "__kmpc_master");2918  EXPECT_TRUE(isa<GlobalVariable>(MasterEntryCI->getArgOperand(0)));2919 2920  CallInst *MasterEndCI = nullptr;2921  for (auto &FI : *FinalizeBB) {2922    Instruction *cur = &FI;2923    if (isa<CallInst>(cur)) {2924      MasterEndCI = cast<CallInst>(cur);2925      if (MasterEndCI->getCalledFunction()->getName() == "__kmpc_end_master")2926        break;2927      MasterEndCI = nullptr;2928    }2929  }2930  EXPECT_NE(MasterEndCI, nullptr);2931  EXPECT_EQ(MasterEndCI->arg_size(), 2U);2932  EXPECT_TRUE(isa<GlobalVariable>(MasterEndCI->getArgOperand(0)));2933  EXPECT_EQ(MasterEndCI->getArgOperand(1), MasterEntryCI->getArgOperand(1));2934}2935 2936TEST_F(OpenMPIRBuilderTest, MaskedDirective) {2937  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;2938  OpenMPIRBuilder OMPBuilder(*M);2939  OMPBuilder.initialize();2940  F->setName("func");2941  IRBuilder<> Builder(BB);2942 2943  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});2944 2945  AllocaInst *PrivAI = nullptr;2946 2947  BasicBlock *EntryBB = nullptr;2948  BasicBlock *ThenBB = nullptr;2949 2950  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {2951    if (AllocaIP.isSet())2952      Builder.restoreIP(AllocaIP);2953    else2954      Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));2955    PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());2956    Builder.CreateStore(F->arg_begin(), PrivAI);2957 2958    llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();2959    llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();2960    EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);2961 2962    Builder.restoreIP(CodeGenIP);2963 2964    // collect some info for checks later2965    ThenBB = Builder.GetInsertBlock();2966    EntryBB = ThenBB->getUniquePredecessor();2967 2968    // simple instructions for body2969    Value *PrivLoad =2970        Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");2971    Builder.CreateICmpNE(F->arg_begin(), PrivLoad);2972  };2973 2974  auto FiniCB = [&](InsertPointTy IP) {2975    BasicBlock *IPBB = IP.getBlock();2976    EXPECT_NE(IPBB->end(), IP.getPoint());2977  };2978 2979  Constant *Filter = ConstantInt::get(Type::getInt32Ty(M->getContext()), 0);2980  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,2981                       OMPBuilder.createMasked(Builder,2982                                               BODYGENCB_WRAPPER(BodyGenCB),2983                                               FINICB_WRAPPER(FiniCB), Filter));2984  Builder.restoreIP(AfterIP);2985  Value *EntryBBTI = EntryBB->getTerminator();2986  EXPECT_NE(EntryBBTI, nullptr);2987  EXPECT_TRUE(isa<BranchInst>(EntryBBTI));2988  BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator());2989  EXPECT_TRUE(EntryBr->isConditional());2990  EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);2991  BasicBlock *FinalizeBB = ThenBB->getUniqueSuccessor();2992  BasicBlock *ExitBB = FinalizeBB->getUniqueSuccessor();2993  EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);2994 2995  CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition());2996  EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));2997 2998  CallInst *MaskedEntryCI = cast<CallInst>(CondInst->getOperand(0));2999  EXPECT_EQ(MaskedEntryCI->arg_size(), 3U);3000  EXPECT_EQ(MaskedEntryCI->getCalledFunction()->getName(), "__kmpc_masked");3001  EXPECT_TRUE(isa<GlobalVariable>(MaskedEntryCI->getArgOperand(0)));3002 3003  CallInst *MaskedEndCI = nullptr;3004  for (auto &FI : *FinalizeBB) {3005    Instruction *cur = &FI;3006    if (isa<CallInst>(cur)) {3007      MaskedEndCI = cast<CallInst>(cur);3008      if (MaskedEndCI->getCalledFunction()->getName() == "__kmpc_end_masked")3009        break;3010      MaskedEndCI = nullptr;3011    }3012  }3013  EXPECT_NE(MaskedEndCI, nullptr);3014  EXPECT_EQ(MaskedEndCI->arg_size(), 2U);3015  EXPECT_TRUE(isa<GlobalVariable>(MaskedEndCI->getArgOperand(0)));3016  EXPECT_EQ(MaskedEndCI->getArgOperand(1), MaskedEntryCI->getArgOperand(1));3017}3018 3019TEST_F(OpenMPIRBuilderTest, CriticalDirective) {3020  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;3021  OpenMPIRBuilder OMPBuilder(*M);3022  OMPBuilder.initialize();3023  F->setName("func");3024  IRBuilder<> Builder(BB);3025 3026  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});3027 3028  AllocaInst *PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());3029 3030  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {3031    // actual start for bodyCB3032    llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();3033    llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();3034    EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);3035 3036    // body begin3037    Builder.restoreIP(CodeGenIP);3038    Builder.CreateStore(F->arg_begin(), PrivAI);3039    Value *PrivLoad =3040        Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");3041    Builder.CreateICmpNE(F->arg_begin(), PrivLoad);3042  };3043 3044  auto FiniCB = [&](InsertPointTy IP) {3045    BasicBlock *IPBB = IP.getBlock();3046    EXPECT_NE(IPBB->end(), IP.getPoint());3047  };3048  BasicBlock *EntryBB = Builder.GetInsertBlock();3049 3050  ASSERT_EXPECTED_INIT(3051      OpenMPIRBuilder::InsertPointTy, AfterIP,3052      OMPBuilder.createCritical(Builder, BODYGENCB_WRAPPER(BodyGenCB),3053                                FINICB_WRAPPER(FiniCB), "testCRT", nullptr));3054  Builder.restoreIP(AfterIP);3055 3056  BasicBlock *FinalizeBB = EntryBB->getUniqueSuccessor();3057  EXPECT_NE(FinalizeBB, nullptr);3058 3059  CallInst *CriticalEntryCI = nullptr;3060  for (auto &EI : *EntryBB) {3061    Instruction *cur = &EI;3062    if (isa<CallInst>(cur)) {3063      CriticalEntryCI = cast<CallInst>(cur);3064      if (CriticalEntryCI->getCalledFunction()->getName() == "__kmpc_critical")3065        break;3066      CriticalEntryCI = nullptr;3067    }3068  }3069  EXPECT_NE(CriticalEntryCI, nullptr);3070  EXPECT_EQ(CriticalEntryCI->arg_size(), 3U);3071  EXPECT_EQ(CriticalEntryCI->getCalledFunction()->getName(), "__kmpc_critical");3072  EXPECT_TRUE(isa<GlobalVariable>(CriticalEntryCI->getArgOperand(0)));3073 3074  CallInst *CriticalEndCI = nullptr;3075  for (auto &FI : *FinalizeBB) {3076    Instruction *cur = &FI;3077    if (isa<CallInst>(cur)) {3078      CriticalEndCI = cast<CallInst>(cur);3079      if (CriticalEndCI->getCalledFunction()->getName() ==3080          "__kmpc_end_critical")3081        break;3082      CriticalEndCI = nullptr;3083    }3084  }3085  EXPECT_NE(CriticalEndCI, nullptr);3086  EXPECT_EQ(CriticalEndCI->arg_size(), 3U);3087  EXPECT_TRUE(isa<GlobalVariable>(CriticalEndCI->getArgOperand(0)));3088  EXPECT_EQ(CriticalEndCI->getArgOperand(1), CriticalEntryCI->getArgOperand(1));3089  PointerType *CriticalNamePtrTy = PointerType::getUnqual(Ctx);3090  EXPECT_EQ(CriticalEndCI->getArgOperand(2), CriticalEntryCI->getArgOperand(2));3091  GlobalVariable *GV =3092      dyn_cast<GlobalVariable>(CriticalEndCI->getArgOperand(2));3093  ASSERT_NE(GV, nullptr);3094  EXPECT_EQ(GV->getType(), CriticalNamePtrTy);3095  const DataLayout &DL = M->getDataLayout();3096  const llvm::Align TypeAlign = DL.getABITypeAlign(CriticalNamePtrTy);3097  const llvm::Align PtrAlign = DL.getPointerABIAlignment(GV->getAddressSpace());3098  if (const llvm::MaybeAlign Alignment = GV->getAlign())3099    EXPECT_EQ(*Alignment, std::max(TypeAlign, PtrAlign));3100}3101 3102TEST_F(OpenMPIRBuilderTest, OrderedDirectiveDependSource) {3103  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;3104  OpenMPIRBuilder OMPBuilder(*M);3105  OMPBuilder.initialize();3106  F->setName("func");3107  IRBuilder<> Builder(BB);3108  LLVMContext &Ctx = M->getContext();3109 3110  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});3111 3112  InsertPointTy AllocaIP(&F->getEntryBlock(),3113                         F->getEntryBlock().getFirstInsertionPt());3114 3115  unsigned NumLoops = 2;3116  SmallVector<Value *, 2> StoreValues;3117  Type *LCTy = Type::getInt64Ty(Ctx);3118  StoreValues.emplace_back(ConstantInt::get(LCTy, 1));3119  StoreValues.emplace_back(ConstantInt::get(LCTy, 2));3120 3121  // Test for "#omp ordered depend(source)"3122  Builder.restoreIP(OMPBuilder.createOrderedDepend(Builder, AllocaIP, NumLoops,3123                                                   StoreValues, ".cnt.addr",3124                                                   /*IsDependSource=*/true));3125 3126  Builder.CreateRetVoid();3127  OMPBuilder.finalize();3128  EXPECT_FALSE(verifyModule(*M, &errs()));3129 3130  AllocaInst *AllocInst = dyn_cast<AllocaInst>(&BB->front());3131  ASSERT_NE(AllocInst, nullptr);3132  ArrayType *ArrType = dyn_cast<ArrayType>(AllocInst->getAllocatedType());3133  EXPECT_EQ(ArrType->getNumElements(), NumLoops);3134  EXPECT_TRUE(3135      AllocInst->getAllocatedType()->getArrayElementType()->isIntegerTy(64));3136 3137  Instruction *IterInst = dyn_cast<Instruction>(AllocInst);3138  for (unsigned Iter = 0; Iter < NumLoops; Iter++) {3139    GetElementPtrInst *DependAddrGEPIter =3140        dyn_cast<GetElementPtrInst>(IterInst->getNextNode());3141    ASSERT_NE(DependAddrGEPIter, nullptr);3142    EXPECT_EQ(DependAddrGEPIter->getPointerOperand(), AllocInst);3143    EXPECT_EQ(DependAddrGEPIter->getNumIndices(), (unsigned)2);3144    auto *FirstIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(1));3145    auto *SecondIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(2));3146    ASSERT_NE(FirstIdx, nullptr);3147    ASSERT_NE(SecondIdx, nullptr);3148    EXPECT_EQ(FirstIdx->getValue(), 0);3149    EXPECT_EQ(SecondIdx->getValue(), Iter);3150    StoreInst *StoreValue =3151        dyn_cast<StoreInst>(DependAddrGEPIter->getNextNode());3152    ASSERT_NE(StoreValue, nullptr);3153    EXPECT_EQ(StoreValue->getValueOperand(), StoreValues[Iter]);3154    EXPECT_EQ(StoreValue->getPointerOperand(), DependAddrGEPIter);3155    EXPECT_EQ(StoreValue->getAlign(), Align(8));3156    IterInst = dyn_cast<Instruction>(StoreValue);3157  }3158 3159  GetElementPtrInst *DependBaseAddrGEP =3160      dyn_cast<GetElementPtrInst>(IterInst->getNextNode());3161  ASSERT_NE(DependBaseAddrGEP, nullptr);3162  EXPECT_EQ(DependBaseAddrGEP->getPointerOperand(), AllocInst);3163  EXPECT_EQ(DependBaseAddrGEP->getNumIndices(), (unsigned)2);3164  auto *FirstIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(1));3165  auto *SecondIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(2));3166  ASSERT_NE(FirstIdx, nullptr);3167  ASSERT_NE(SecondIdx, nullptr);3168  EXPECT_EQ(FirstIdx->getValue(), 0);3169  EXPECT_EQ(SecondIdx->getValue(), 0);3170 3171  CallInst *GTID = dyn_cast<CallInst>(DependBaseAddrGEP->getNextNode());3172  ASSERT_NE(GTID, nullptr);3173  EXPECT_EQ(GTID->arg_size(), 1U);3174  EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");3175  EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());3176  EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());3177 3178  CallInst *Depend = dyn_cast<CallInst>(GTID->getNextNode());3179  ASSERT_NE(Depend, nullptr);3180  EXPECT_EQ(Depend->arg_size(), 3U);3181  EXPECT_EQ(Depend->getCalledFunction()->getName(), "__kmpc_doacross_post");3182  EXPECT_TRUE(isa<GlobalVariable>(Depend->getArgOperand(0)));3183  EXPECT_EQ(Depend->getArgOperand(1), GTID);3184  EXPECT_EQ(Depend->getArgOperand(2), DependBaseAddrGEP);3185}3186 3187TEST_F(OpenMPIRBuilderTest, OrderedDirectiveDependSink) {3188  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;3189  OpenMPIRBuilder OMPBuilder(*M);3190  OMPBuilder.initialize();3191  F->setName("func");3192  IRBuilder<> Builder(BB);3193  LLVMContext &Ctx = M->getContext();3194 3195  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});3196 3197  InsertPointTy AllocaIP(&F->getEntryBlock(),3198                         F->getEntryBlock().getFirstInsertionPt());3199 3200  unsigned NumLoops = 2;3201  SmallVector<Value *, 2> StoreValues;3202  Type *LCTy = Type::getInt64Ty(Ctx);3203  StoreValues.emplace_back(ConstantInt::get(LCTy, 1));3204  StoreValues.emplace_back(ConstantInt::get(LCTy, 2));3205 3206  // Test for "#omp ordered depend(sink: vec)"3207  Builder.restoreIP(OMPBuilder.createOrderedDepend(Builder, AllocaIP, NumLoops,3208                                                   StoreValues, ".cnt.addr",3209                                                   /*IsDependSource=*/false));3210 3211  Builder.CreateRetVoid();3212  OMPBuilder.finalize();3213  EXPECT_FALSE(verifyModule(*M, &errs()));3214 3215  AllocaInst *AllocInst = dyn_cast<AllocaInst>(&BB->front());3216  ASSERT_NE(AllocInst, nullptr);3217  ArrayType *ArrType = dyn_cast<ArrayType>(AllocInst->getAllocatedType());3218  EXPECT_EQ(ArrType->getNumElements(), NumLoops);3219  EXPECT_TRUE(3220      AllocInst->getAllocatedType()->getArrayElementType()->isIntegerTy(64));3221 3222  Instruction *IterInst = dyn_cast<Instruction>(AllocInst);3223  for (unsigned Iter = 0; Iter < NumLoops; Iter++) {3224    GetElementPtrInst *DependAddrGEPIter =3225        dyn_cast<GetElementPtrInst>(IterInst->getNextNode());3226    ASSERT_NE(DependAddrGEPIter, nullptr);3227    EXPECT_EQ(DependAddrGEPIter->getPointerOperand(), AllocInst);3228    EXPECT_EQ(DependAddrGEPIter->getNumIndices(), (unsigned)2);3229    auto *FirstIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(1));3230    auto *SecondIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(2));3231    ASSERT_NE(FirstIdx, nullptr);3232    ASSERT_NE(SecondIdx, nullptr);3233    EXPECT_EQ(FirstIdx->getValue(), 0);3234    EXPECT_EQ(SecondIdx->getValue(), Iter);3235    StoreInst *StoreValue =3236        dyn_cast<StoreInst>(DependAddrGEPIter->getNextNode());3237    ASSERT_NE(StoreValue, nullptr);3238    EXPECT_EQ(StoreValue->getValueOperand(), StoreValues[Iter]);3239    EXPECT_EQ(StoreValue->getPointerOperand(), DependAddrGEPIter);3240    EXPECT_EQ(StoreValue->getAlign(), Align(8));3241    IterInst = dyn_cast<Instruction>(StoreValue);3242  }3243 3244  GetElementPtrInst *DependBaseAddrGEP =3245      dyn_cast<GetElementPtrInst>(IterInst->getNextNode());3246  ASSERT_NE(DependBaseAddrGEP, nullptr);3247  EXPECT_EQ(DependBaseAddrGEP->getPointerOperand(), AllocInst);3248  EXPECT_EQ(DependBaseAddrGEP->getNumIndices(), (unsigned)2);3249  auto *FirstIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(1));3250  auto *SecondIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(2));3251  ASSERT_NE(FirstIdx, nullptr);3252  ASSERT_NE(SecondIdx, nullptr);3253  EXPECT_EQ(FirstIdx->getValue(), 0);3254  EXPECT_EQ(SecondIdx->getValue(), 0);3255 3256  CallInst *GTID = dyn_cast<CallInst>(DependBaseAddrGEP->getNextNode());3257  ASSERT_NE(GTID, nullptr);3258  EXPECT_EQ(GTID->arg_size(), 1U);3259  EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");3260  EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());3261  EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());3262 3263  CallInst *Depend = dyn_cast<CallInst>(GTID->getNextNode());3264  ASSERT_NE(Depend, nullptr);3265  EXPECT_EQ(Depend->arg_size(), 3U);3266  EXPECT_EQ(Depend->getCalledFunction()->getName(), "__kmpc_doacross_wait");3267  EXPECT_TRUE(isa<GlobalVariable>(Depend->getArgOperand(0)));3268  EXPECT_EQ(Depend->getArgOperand(1), GTID);3269  EXPECT_EQ(Depend->getArgOperand(2), DependBaseAddrGEP);3270}3271 3272TEST_F(OpenMPIRBuilderTest, OrderedDirectiveThreads) {3273  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;3274  OpenMPIRBuilder OMPBuilder(*M);3275  OMPBuilder.initialize();3276  F->setName("func");3277  IRBuilder<> Builder(BB);3278 3279  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});3280 3281  AllocaInst *PrivAI =3282      Builder.CreateAlloca(F->arg_begin()->getType(), nullptr, "priv.inst");3283 3284  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {3285    llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();3286    llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();3287    EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);3288 3289    Builder.restoreIP(CodeGenIP);3290    Builder.CreateStore(F->arg_begin(), PrivAI);3291    Value *PrivLoad =3292        Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");3293    Builder.CreateICmpNE(F->arg_begin(), PrivLoad);3294  };3295 3296  auto FiniCB = [&](InsertPointTy IP) {3297    BasicBlock *IPBB = IP.getBlock();3298    EXPECT_NE(IPBB->end(), IP.getPoint());3299  };3300 3301  // Test for "#omp ordered [threads]"3302  BasicBlock *EntryBB = Builder.GetInsertBlock();3303  ASSERT_EXPECTED_INIT(3304      OpenMPIRBuilder::InsertPointTy, AfterIP,3305      OMPBuilder.createOrderedThreadsSimd(Builder, BODYGENCB_WRAPPER(BodyGenCB),3306                                          FINICB_WRAPPER(FiniCB), true));3307  Builder.restoreIP(AfterIP);3308 3309  BasicBlock *FinalizeBB = EntryBB->getUniqueSuccessor();3310  EXPECT_NE(FinalizeBB, nullptr);3311 3312  Builder.CreateRetVoid();3313  OMPBuilder.finalize();3314  EXPECT_FALSE(verifyModule(*M, &errs()));3315 3316  EXPECT_NE(EntryBB->getTerminator(), nullptr);3317 3318  CallInst *OrderedEntryCI = nullptr;3319  for (auto &EI : *EntryBB) {3320    Instruction *Cur = &EI;3321    if (isa<CallInst>(Cur)) {3322      OrderedEntryCI = cast<CallInst>(Cur);3323      if (OrderedEntryCI->getCalledFunction()->getName() == "__kmpc_ordered")3324        break;3325      OrderedEntryCI = nullptr;3326    }3327  }3328  EXPECT_NE(OrderedEntryCI, nullptr);3329  EXPECT_EQ(OrderedEntryCI->arg_size(), 2U);3330  EXPECT_EQ(OrderedEntryCI->getCalledFunction()->getName(), "__kmpc_ordered");3331  EXPECT_TRUE(isa<GlobalVariable>(OrderedEntryCI->getArgOperand(0)));3332 3333  CallInst *OrderedEndCI = nullptr;3334  for (auto &FI : *FinalizeBB) {3335    Instruction *Cur = &FI;3336    if (isa<CallInst>(Cur)) {3337      OrderedEndCI = cast<CallInst>(Cur);3338      if (OrderedEndCI->getCalledFunction()->getName() == "__kmpc_end_ordered")3339        break;3340      OrderedEndCI = nullptr;3341    }3342  }3343  EXPECT_NE(OrderedEndCI, nullptr);3344  EXPECT_EQ(OrderedEndCI->arg_size(), 2U);3345  EXPECT_TRUE(isa<GlobalVariable>(OrderedEndCI->getArgOperand(0)));3346  EXPECT_EQ(OrderedEndCI->getArgOperand(1), OrderedEntryCI->getArgOperand(1));3347}3348 3349TEST_F(OpenMPIRBuilderTest, OrderedDirectiveSimd) {3350  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;3351  OpenMPIRBuilder OMPBuilder(*M);3352  OMPBuilder.initialize();3353  F->setName("func");3354  IRBuilder<> Builder(BB);3355 3356  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});3357 3358  AllocaInst *PrivAI =3359      Builder.CreateAlloca(F->arg_begin()->getType(), nullptr, "priv.inst");3360 3361  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {3362    llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();3363    llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();3364    EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);3365 3366    Builder.restoreIP(CodeGenIP);3367    Builder.CreateStore(F->arg_begin(), PrivAI);3368    Value *PrivLoad =3369        Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");3370    Builder.CreateICmpNE(F->arg_begin(), PrivLoad);3371  };3372 3373  auto FiniCB = [&](InsertPointTy IP) {3374    BasicBlock *IPBB = IP.getBlock();3375    EXPECT_NE(IPBB->end(), IP.getPoint());3376  };3377 3378  // Test for "#omp ordered simd"3379  BasicBlock *EntryBB = Builder.GetInsertBlock();3380  ASSERT_EXPECTED_INIT(3381      OpenMPIRBuilder::InsertPointTy, AfterIP,3382      OMPBuilder.createOrderedThreadsSimd(Builder, BODYGENCB_WRAPPER(BodyGenCB),3383                                          FINICB_WRAPPER(FiniCB), false));3384  Builder.restoreIP(AfterIP);3385 3386  Builder.CreateRetVoid();3387  OMPBuilder.finalize();3388  EXPECT_FALSE(verifyModule(*M, &errs()));3389 3390  EXPECT_NE(EntryBB->getTerminator(), nullptr);3391 3392  CallInst *OrderedEntryCI = nullptr;3393  for (auto &EI : *EntryBB) {3394    Instruction *Cur = &EI;3395    if (isa<CallInst>(Cur)) {3396      OrderedEntryCI = cast<CallInst>(Cur);3397      if (OrderedEntryCI->getCalledFunction()->getName() == "__kmpc_ordered")3398        break;3399      OrderedEntryCI = nullptr;3400    }3401  }3402  EXPECT_EQ(OrderedEntryCI, nullptr);3403 3404  CallInst *OrderedEndCI = nullptr;3405  for (auto &FI : *EntryBB) {3406    Instruction *Cur = &FI;3407    if (isa<CallInst>(Cur)) {3408      OrderedEndCI = cast<CallInst>(Cur);3409      if (OrderedEndCI->getCalledFunction()->getName() == "__kmpc_end_ordered")3410        break;3411      OrderedEndCI = nullptr;3412    }3413  }3414  EXPECT_EQ(OrderedEndCI, nullptr);3415}3416 3417TEST_F(OpenMPIRBuilderTest, CopyinBlocks) {3418  OpenMPIRBuilder OMPBuilder(*M);3419  OMPBuilder.initialize();3420  F->setName("func");3421  IRBuilder<> Builder(BB);3422 3423  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});3424 3425  IntegerType *Int32 = Type::getInt32Ty(M->getContext());3426  AllocaInst *MasterAddress = Builder.CreateAlloca(Builder.getPtrTy());3427  AllocaInst *PrivAddress = Builder.CreateAlloca(Builder.getPtrTy());3428 3429  BasicBlock *EntryBB = BB;3430 3431  OMPBuilder.createCopyinClauseBlocks(Builder.saveIP(), MasterAddress,3432                                      PrivAddress, Int32, /*BranchtoEnd*/ true);3433 3434  BranchInst *EntryBr = dyn_cast_or_null<BranchInst>(EntryBB->getTerminator());3435 3436  EXPECT_NE(EntryBr, nullptr);3437  EXPECT_TRUE(EntryBr->isConditional());3438 3439  BasicBlock *NotMasterBB = EntryBr->getSuccessor(0);3440  BasicBlock *CopyinEnd = EntryBr->getSuccessor(1);3441  CmpInst *CMP = dyn_cast_or_null<CmpInst>(EntryBr->getCondition());3442 3443  EXPECT_NE(CMP, nullptr);3444  EXPECT_NE(NotMasterBB, nullptr);3445  EXPECT_NE(CopyinEnd, nullptr);3446 3447  BranchInst *NotMasterBr =3448      dyn_cast_or_null<BranchInst>(NotMasterBB->getTerminator());3449  EXPECT_NE(NotMasterBr, nullptr);3450  EXPECT_FALSE(NotMasterBr->isConditional());3451  EXPECT_EQ(CopyinEnd, NotMasterBr->getSuccessor(0));3452}3453 3454TEST_F(OpenMPIRBuilderTest, SingleDirective) {3455  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;3456  OpenMPIRBuilder OMPBuilder(*M);3457  OMPBuilder.initialize();3458  F->setName("func");3459  IRBuilder<> Builder(BB);3460 3461  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});3462 3463  AllocaInst *PrivAI = nullptr;3464 3465  BasicBlock *EntryBB = nullptr;3466  BasicBlock *ThenBB = nullptr;3467 3468  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {3469    if (AllocaIP.isSet())3470      Builder.restoreIP(AllocaIP);3471    else3472      Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));3473    PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());3474    Builder.CreateStore(F->arg_begin(), PrivAI);3475 3476    llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();3477    llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();3478    EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);3479 3480    Builder.restoreIP(CodeGenIP);3481 3482    // collect some info for checks later3483    ThenBB = Builder.GetInsertBlock();3484    EntryBB = ThenBB->getUniquePredecessor();3485 3486    // simple instructions for body3487    Value *PrivLoad =3488        Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");3489    Builder.CreateICmpNE(F->arg_begin(), PrivLoad);3490  };3491 3492  auto FiniCB = [&](InsertPointTy IP) {3493    BasicBlock *IPBB = IP.getBlock();3494    EXPECT_NE(IPBB->end(), IP.getPoint());3495  };3496 3497  ASSERT_EXPECTED_INIT(3498      OpenMPIRBuilder::InsertPointTy, AfterIP,3499      OMPBuilder.createSingle(Builder, BODYGENCB_WRAPPER(BodyGenCB),3500                              FINICB_WRAPPER(FiniCB), /*IsNowait*/ false));3501  Builder.restoreIP(AfterIP);3502  Value *EntryBBTI = EntryBB->getTerminator();3503  EXPECT_NE(EntryBBTI, nullptr);3504  EXPECT_TRUE(isa<BranchInst>(EntryBBTI));3505  BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator());3506  EXPECT_TRUE(EntryBr->isConditional());3507  EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);3508  BasicBlock *FinalizeBB = ThenBB->getUniqueSuccessor();3509  BasicBlock *ExitBB = FinalizeBB->getUniqueSuccessor();3510  EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);3511 3512  CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition());3513  EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));3514 3515  CallInst *SingleEntryCI = cast<CallInst>(CondInst->getOperand(0));3516  EXPECT_EQ(SingleEntryCI->arg_size(), 2U);3517  EXPECT_EQ(SingleEntryCI->getCalledFunction()->getName(), "__kmpc_single");3518  EXPECT_TRUE(isa<GlobalVariable>(SingleEntryCI->getArgOperand(0)));3519 3520  CallInst *SingleEndCI = nullptr;3521  for (auto &FI : *FinalizeBB) {3522    Instruction *cur = &FI;3523    if (isa<CallInst>(cur)) {3524      SingleEndCI = cast<CallInst>(cur);3525      if (SingleEndCI->getCalledFunction()->getName() == "__kmpc_end_single")3526        break;3527      SingleEndCI = nullptr;3528    }3529  }3530  EXPECT_NE(SingleEndCI, nullptr);3531  EXPECT_EQ(SingleEndCI->arg_size(), 2U);3532  EXPECT_TRUE(isa<GlobalVariable>(SingleEndCI->getArgOperand(0)));3533  EXPECT_EQ(SingleEndCI->getArgOperand(1), SingleEntryCI->getArgOperand(1));3534 3535  bool FoundBarrier = false;3536  for (auto &FI : *ExitBB) {3537    Instruction *cur = &FI;3538    if (auto CI = dyn_cast<CallInst>(cur)) {3539      if (CI->getCalledFunction()->getName() == "__kmpc_barrier") {3540        FoundBarrier = true;3541        break;3542      }3543    }3544  }3545  EXPECT_TRUE(FoundBarrier);3546}3547 3548TEST_F(OpenMPIRBuilderTest, SingleDirectiveNowait) {3549  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;3550  OpenMPIRBuilder OMPBuilder(*M);3551  OMPBuilder.initialize();3552  F->setName("func");3553  IRBuilder<> Builder(BB);3554 3555  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});3556 3557  AllocaInst *PrivAI = nullptr;3558 3559  BasicBlock *EntryBB = nullptr;3560  BasicBlock *ThenBB = nullptr;3561 3562  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {3563    if (AllocaIP.isSet())3564      Builder.restoreIP(AllocaIP);3565    else3566      Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));3567    PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());3568    Builder.CreateStore(F->arg_begin(), PrivAI);3569 3570    llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();3571    llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();3572    EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);3573 3574    Builder.restoreIP(CodeGenIP);3575 3576    // collect some info for checks later3577    ThenBB = Builder.GetInsertBlock();3578    EntryBB = ThenBB->getUniquePredecessor();3579 3580    // simple instructions for body3581    Value *PrivLoad =3582        Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");3583    Builder.CreateICmpNE(F->arg_begin(), PrivLoad);3584  };3585 3586  auto FiniCB = [&](InsertPointTy IP) {3587    BasicBlock *IPBB = IP.getBlock();3588    EXPECT_NE(IPBB->end(), IP.getPoint());3589  };3590 3591  ASSERT_EXPECTED_INIT(3592      OpenMPIRBuilder::InsertPointTy, AfterIP,3593      OMPBuilder.createSingle(Builder, BODYGENCB_WRAPPER(BodyGenCB),3594                              FINICB_WRAPPER(FiniCB), /*IsNowait*/ true));3595  Builder.restoreIP(AfterIP);3596  Value *EntryBBTI = EntryBB->getTerminator();3597  EXPECT_NE(EntryBBTI, nullptr);3598  EXPECT_TRUE(isa<BranchInst>(EntryBBTI));3599  BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator());3600  EXPECT_TRUE(EntryBr->isConditional());3601  EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);3602  BasicBlock *FinalizeBB = ThenBB->getUniqueSuccessor();3603  BasicBlock *ExitBB = FinalizeBB->getUniqueSuccessor();3604  EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);3605 3606  CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition());3607  EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));3608 3609  CallInst *SingleEntryCI = cast<CallInst>(CondInst->getOperand(0));3610  EXPECT_EQ(SingleEntryCI->arg_size(), 2U);3611  EXPECT_EQ(SingleEntryCI->getCalledFunction()->getName(), "__kmpc_single");3612  EXPECT_TRUE(isa<GlobalVariable>(SingleEntryCI->getArgOperand(0)));3613 3614  CallInst *SingleEndCI = nullptr;3615  for (auto &FI : *FinalizeBB) {3616    Instruction *cur = &FI;3617    if (isa<CallInst>(cur)) {3618      SingleEndCI = cast<CallInst>(cur);3619      if (SingleEndCI->getCalledFunction()->getName() == "__kmpc_end_single")3620        break;3621      SingleEndCI = nullptr;3622    }3623  }3624  EXPECT_NE(SingleEndCI, nullptr);3625  EXPECT_EQ(SingleEndCI->arg_size(), 2U);3626  EXPECT_TRUE(isa<GlobalVariable>(SingleEndCI->getArgOperand(0)));3627  EXPECT_EQ(SingleEndCI->getArgOperand(1), SingleEntryCI->getArgOperand(1));3628 3629  CallInst *ExitBarrier = nullptr;3630  for (auto &FI : *ExitBB) {3631    Instruction *cur = &FI;3632    if (auto CI = dyn_cast<CallInst>(cur)) {3633      if (CI->getCalledFunction()->getName() == "__kmpc_barrier") {3634        ExitBarrier = CI;3635        break;3636      }3637    }3638  }3639  EXPECT_EQ(ExitBarrier, nullptr);3640}3641 3642// Helper class to check each instruction of a BB.3643class BBInstIter {3644  BasicBlock *BB;3645  BasicBlock::iterator BBI;3646 3647public:3648  BBInstIter(BasicBlock *BB) : BB(BB), BBI(BB->begin()) {}3649 3650  bool hasNext() const { return BBI != BB->end(); }3651 3652  template <typename InstTy> InstTy *next() {3653    if (!hasNext())3654      return nullptr;3655    Instruction *Cur = &*BBI++;3656    if (!isa<InstTy>(Cur))3657      return nullptr;3658    return cast<InstTy>(Cur);3659  }3660};3661 3662TEST_F(OpenMPIRBuilderTest, SingleDirectiveCopyPrivate) {3663  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;3664  OpenMPIRBuilder OMPBuilder(*M);3665  OMPBuilder.initialize();3666  F->setName("func");3667  IRBuilder<> Builder(BB);3668 3669  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});3670 3671  AllocaInst *PrivAI = nullptr;3672 3673  BasicBlock *EntryBB = nullptr;3674  BasicBlock *ThenBB = nullptr;3675 3676  Value *CPVar = Builder.CreateAlloca(F->arg_begin()->getType());3677  Builder.CreateStore(F->arg_begin(), CPVar);3678 3679  FunctionType *CopyFuncTy = FunctionType::get(3680      Builder.getVoidTy(), {Builder.getPtrTy(), Builder.getPtrTy()}, false);3681  Function *CopyFunc =3682      Function::Create(CopyFuncTy, Function::PrivateLinkage, "copy_var", *M);3683 3684  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {3685    if (AllocaIP.isSet())3686      Builder.restoreIP(AllocaIP);3687    else3688      Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));3689    PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());3690    Builder.CreateStore(F->arg_begin(), PrivAI);3691 3692    llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();3693    llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();3694    EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);3695 3696    Builder.restoreIP(CodeGenIP);3697 3698    // collect some info for checks later3699    ThenBB = Builder.GetInsertBlock();3700    EntryBB = ThenBB->getUniquePredecessor();3701 3702    // simple instructions for body3703    Value *PrivLoad =3704        Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");3705    Builder.CreateICmpNE(F->arg_begin(), PrivLoad);3706  };3707 3708  auto FiniCB = [&](InsertPointTy IP) {3709    BasicBlock *IPBB = IP.getBlock();3710    // IP must be before the unconditional branch to ExitBB3711    EXPECT_NE(IPBB->end(), IP.getPoint());3712  };3713 3714  ASSERT_EXPECTED_INIT(3715      OpenMPIRBuilder::InsertPointTy, AfterIP,3716      OMPBuilder.createSingle(Builder, BODYGENCB_WRAPPER(BodyGenCB),3717                              FINICB_WRAPPER(FiniCB),3718                              /*IsNowait*/ false, {CPVar}, {CopyFunc}));3719  Builder.restoreIP(AfterIP);3720  Value *EntryBBTI = EntryBB->getTerminator();3721  EXPECT_NE(EntryBBTI, nullptr);3722  EXPECT_TRUE(isa<BranchInst>(EntryBBTI));3723  BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator());3724  EXPECT_TRUE(EntryBr->isConditional());3725  EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);3726  BasicBlock *FinalizeBB = ThenBB->getUniqueSuccessor();3727  BasicBlock *ExitBB = FinalizeBB->getUniqueSuccessor();3728  EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);3729 3730  CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition());3731  EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));3732 3733  CallInst *SingleEntryCI = cast<CallInst>(CondInst->getOperand(0));3734  EXPECT_EQ(SingleEntryCI->arg_size(), 2U);3735  EXPECT_EQ(SingleEntryCI->getCalledFunction()->getName(), "__kmpc_single");3736  EXPECT_TRUE(isa<GlobalVariable>(SingleEntryCI->getArgOperand(0)));3737 3738  // check ThenBB3739  BBInstIter ThenBBI(ThenBB);3740  // load PrivAI3741  auto *PrivLI = ThenBBI.next<LoadInst>();3742  EXPECT_NE(PrivLI, nullptr);3743  EXPECT_EQ(PrivLI->getPointerOperand(), PrivAI);3744  // icmp3745  EXPECT_TRUE(ThenBBI.next<ICmpInst>());3746 3747  // check FinalizeBB3748  BBInstIter FinalizeBBI(FinalizeBB);3749  // store 1, DidIt3750  auto *DidItSI = FinalizeBBI.next<StoreInst>();3751  EXPECT_NE(DidItSI, nullptr);3752  EXPECT_EQ(DidItSI->getValueOperand(),3753            ConstantInt::get(Type::getInt32Ty(Ctx), 1));3754  Value *DidIt = DidItSI->getPointerOperand();3755  // call __kmpc_end_single3756  auto *SingleEndCI = FinalizeBBI.next<CallInst>();3757  EXPECT_NE(SingleEndCI, nullptr);3758  EXPECT_EQ(SingleEndCI->getCalledFunction()->getName(), "__kmpc_end_single");3759  EXPECT_EQ(SingleEndCI->arg_size(), 2U);3760  EXPECT_TRUE(isa<GlobalVariable>(SingleEndCI->getArgOperand(0)));3761  EXPECT_EQ(SingleEndCI->getArgOperand(1), SingleEntryCI->getArgOperand(1));3762  // br ExitBB3763  auto *ExitBBBI = FinalizeBBI.next<BranchInst>();3764  EXPECT_NE(ExitBBBI, nullptr);3765  EXPECT_TRUE(ExitBBBI->isUnconditional());3766  EXPECT_EQ(ExitBBBI->getOperand(0), ExitBB);3767  EXPECT_FALSE(FinalizeBBI.hasNext());3768 3769  // check ExitBB3770  BBInstIter ExitBBI(ExitBB);3771  // call __kmpc_global_thread_num3772  auto *ThreadNumCI = ExitBBI.next<CallInst>();3773  EXPECT_NE(ThreadNumCI, nullptr);3774  EXPECT_EQ(ThreadNumCI->getCalledFunction()->getName(),3775            "__kmpc_global_thread_num");3776  // load DidIt3777  auto *DidItLI = ExitBBI.next<LoadInst>();3778  EXPECT_NE(DidItLI, nullptr);3779  EXPECT_EQ(DidItLI->getPointerOperand(), DidIt);3780  // call __kmpc_copyprivate3781  auto *CopyPrivateCI = ExitBBI.next<CallInst>();3782  EXPECT_NE(CopyPrivateCI, nullptr);3783  EXPECT_EQ(CopyPrivateCI->arg_size(), 6U);3784  EXPECT_TRUE(isa<AllocaInst>(CopyPrivateCI->getArgOperand(3)));3785  EXPECT_EQ(CopyPrivateCI->getArgOperand(3), CPVar);3786  EXPECT_TRUE(isa<Function>(CopyPrivateCI->getArgOperand(4)));3787  EXPECT_EQ(CopyPrivateCI->getArgOperand(4), CopyFunc);3788  EXPECT_TRUE(isa<LoadInst>(CopyPrivateCI->getArgOperand(5)));3789  DidItLI = cast<LoadInst>(CopyPrivateCI->getArgOperand(5));3790  EXPECT_EQ(DidItLI->getOperand(0), DidIt);3791  EXPECT_FALSE(ExitBBI.hasNext());3792}3793 3794TEST_F(OpenMPIRBuilderTest, OMPAtomicReadFlt) {3795  OpenMPIRBuilder OMPBuilder(*M);3796  OMPBuilder.initialize();3797  F->setName("func");3798  IRBuilder<> Builder(BB);3799 3800  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});3801  BasicBlock *EntryBB = BB;3802  OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,3803                                          EntryBB->getFirstInsertionPt());3804 3805  Type *Float32 = Type::getFloatTy(M->getContext());3806  AllocaInst *XVal = Builder.CreateAlloca(Float32);3807  XVal->setName("AtomicVar");3808  AllocaInst *VVal = Builder.CreateAlloca(Float32);3809  VVal->setName("AtomicRead");3810  AtomicOrdering AO = AtomicOrdering::Monotonic;3811  OpenMPIRBuilder::AtomicOpValue X = {XVal, Float32, false, false};3812  OpenMPIRBuilder::AtomicOpValue V = {VVal, Float32, false, false};3813 3814  Builder.restoreIP(OMPBuilder.createAtomicRead(Loc, X, V, AO, AllocaIP));3815 3816  IntegerType *IntCastTy =3817      IntegerType::get(M->getContext(), Float32->getScalarSizeInBits());3818 3819  LoadInst *AtomicLoad = cast<LoadInst>(VVal->getNextNode());3820  EXPECT_TRUE(AtomicLoad->isAtomic());3821  EXPECT_EQ(AtomicLoad->getPointerOperand(), XVal);3822 3823  BitCastInst *CastToFlt = cast<BitCastInst>(AtomicLoad->getNextNode());3824  EXPECT_EQ(CastToFlt->getSrcTy(), IntCastTy);3825  EXPECT_EQ(CastToFlt->getDestTy(), Float32);3826  EXPECT_EQ(CastToFlt->getOperand(0), AtomicLoad);3827 3828  StoreInst *StoreofAtomic = cast<StoreInst>(CastToFlt->getNextNode());3829  EXPECT_EQ(StoreofAtomic->getValueOperand(), CastToFlt);3830  EXPECT_EQ(StoreofAtomic->getPointerOperand(), VVal);3831 3832  Builder.CreateRetVoid();3833  OMPBuilder.finalize();3834  EXPECT_FALSE(verifyModule(*M, &errs()));3835}3836 3837TEST_F(OpenMPIRBuilderTest, OMPAtomicReadInt) {3838  OpenMPIRBuilder OMPBuilder(*M);3839  OMPBuilder.initialize();3840  F->setName("func");3841  IRBuilder<> Builder(BB);3842 3843  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});3844  BasicBlock *EntryBB = BB;3845  OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,3846                                          EntryBB->getFirstInsertionPt());3847 3848  IntegerType *Int32 = Type::getInt32Ty(M->getContext());3849  AllocaInst *XVal = Builder.CreateAlloca(Int32);3850  XVal->setName("AtomicVar");3851  AllocaInst *VVal = Builder.CreateAlloca(Int32);3852  VVal->setName("AtomicRead");3853  AtomicOrdering AO = AtomicOrdering::Monotonic;3854  OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false};3855  OpenMPIRBuilder::AtomicOpValue V = {VVal, Int32, false, false};3856 3857  Builder.restoreIP(OMPBuilder.createAtomicRead(Loc, X, V, AO, AllocaIP));3858 3859  LoadInst *AtomicLoad = nullptr;3860  StoreInst *StoreofAtomic = nullptr;3861 3862  for (Instruction &Cur : *EntryBB) {3863    if (isa<LoadInst>(Cur)) {3864      AtomicLoad = cast<LoadInst>(&Cur);3865      if (AtomicLoad->getPointerOperand() == XVal)3866        continue;3867      AtomicLoad = nullptr;3868    } else if (isa<StoreInst>(Cur)) {3869      StoreofAtomic = cast<StoreInst>(&Cur);3870      if (StoreofAtomic->getPointerOperand() == VVal)3871        continue;3872      StoreofAtomic = nullptr;3873    }3874  }3875 3876  EXPECT_NE(AtomicLoad, nullptr);3877  EXPECT_TRUE(AtomicLoad->isAtomic());3878 3879  EXPECT_NE(StoreofAtomic, nullptr);3880  EXPECT_EQ(StoreofAtomic->getValueOperand(), AtomicLoad);3881 3882  Builder.CreateRetVoid();3883  OMPBuilder.finalize();3884 3885  EXPECT_FALSE(verifyModule(*M, &errs()));3886}3887 3888TEST_F(OpenMPIRBuilderTest, OMPAtomicWriteFlt) {3889  OpenMPIRBuilder OMPBuilder(*M);3890  OMPBuilder.initialize();3891  F->setName("func");3892  IRBuilder<> Builder(BB);3893 3894  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});3895  BasicBlock *EntryBB = BB;3896  OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,3897                                          EntryBB->getFirstInsertionPt());3898 3899  LLVMContext &Ctx = M->getContext();3900  Type *Float32 = Type::getFloatTy(Ctx);3901  AllocaInst *XVal = Builder.CreateAlloca(Float32);3902  XVal->setName("AtomicVar");3903  OpenMPIRBuilder::AtomicOpValue X = {XVal, Float32, false, false};3904  AtomicOrdering AO = AtomicOrdering::Monotonic;3905  Constant *ValToWrite = ConstantFP::get(Float32, 1.0);3906 3907  Builder.restoreIP(3908      OMPBuilder.createAtomicWrite(Loc, X, ValToWrite, AO, AllocaIP));3909 3910  IntegerType *IntCastTy =3911      IntegerType::get(M->getContext(), Float32->getScalarSizeInBits());3912 3913  Value *ExprCast = Builder.CreateBitCast(ValToWrite, IntCastTy);3914 3915  StoreInst *StoreofAtomic = cast<StoreInst>(XVal->getNextNode());3916  EXPECT_EQ(StoreofAtomic->getValueOperand(), ExprCast);3917  EXPECT_EQ(StoreofAtomic->getPointerOperand(), XVal);3918  EXPECT_TRUE(StoreofAtomic->isAtomic());3919 3920  Builder.CreateRetVoid();3921  OMPBuilder.finalize();3922  EXPECT_FALSE(verifyModule(*M, &errs()));3923}3924 3925TEST_F(OpenMPIRBuilderTest, OMPAtomicWriteInt) {3926  OpenMPIRBuilder OMPBuilder(*M);3927  OMPBuilder.initialize();3928  F->setName("func");3929  IRBuilder<> Builder(BB);3930 3931  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});3932 3933  LLVMContext &Ctx = M->getContext();3934  IntegerType *Int32 = Type::getInt32Ty(Ctx);3935  AllocaInst *XVal = Builder.CreateAlloca(Int32);3936  XVal->setName("AtomicVar");3937  OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false};3938  AtomicOrdering AO = AtomicOrdering::Monotonic;3939  ConstantInt *ValToWrite = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);3940 3941  BasicBlock *EntryBB = BB;3942  OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,3943                                          EntryBB->getFirstInsertionPt());3944 3945  Builder.restoreIP(3946      OMPBuilder.createAtomicWrite(Loc, X, ValToWrite, AO, AllocaIP));3947 3948  StoreInst *StoreofAtomic = nullptr;3949 3950  for (Instruction &Cur : *EntryBB) {3951    if (isa<StoreInst>(Cur)) {3952      StoreofAtomic = cast<StoreInst>(&Cur);3953      if (StoreofAtomic->getPointerOperand() == XVal)3954        continue;3955      StoreofAtomic = nullptr;3956    }3957  }3958 3959  EXPECT_NE(StoreofAtomic, nullptr);3960  EXPECT_TRUE(StoreofAtomic->isAtomic());3961  EXPECT_EQ(StoreofAtomic->getValueOperand(), ValToWrite);3962 3963  Builder.CreateRetVoid();3964  OMPBuilder.finalize();3965  EXPECT_FALSE(verifyModule(*M, &errs()));3966}3967 3968TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdate) {3969  OpenMPIRBuilder OMPBuilder(*M);3970  OMPBuilder.initialize();3971  F->setName("func");3972  IRBuilder<> Builder(BB);3973 3974  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});3975 3976  IntegerType *Int32 = Type::getInt32Ty(M->getContext());3977  AllocaInst *XVal = Builder.CreateAlloca(Int32);3978  XVal->setName("AtomicVar");3979  Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), XVal);3980  OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false};3981  AtomicOrdering AO = AtomicOrdering::Monotonic;3982  ConstantInt *ConstVal = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);3983  Value *Expr = nullptr;3984  AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::Sub;3985  bool IsXLHSInRHSPart = false;3986 3987  BasicBlock *EntryBB = BB;3988  OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,3989                                          EntryBB->getFirstInsertionPt());3990  Value *Sub = nullptr;3991 3992  auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) {3993    Sub = IRB.CreateSub(ConstVal, Atomic);3994    return Sub;3995  };3996  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,3997                       OMPBuilder.createAtomicUpdate(Builder, AllocaIP, X, Expr,3998                                                     AO, RMWOp, UpdateOp,3999                                                     IsXLHSInRHSPart));4000  Builder.restoreIP(AfterIP);4001  BasicBlock *ContBB = EntryBB->getSingleSuccessor();4002  BranchInst *ContTI = dyn_cast<BranchInst>(ContBB->getTerminator());4003  EXPECT_NE(ContTI, nullptr);4004  BasicBlock *EndBB = ContTI->getSuccessor(0);4005  EXPECT_TRUE(ContTI->isConditional());4006  EXPECT_EQ(ContTI->getSuccessor(1), ContBB);4007  EXPECT_NE(EndBB, nullptr);4008 4009  PHINode *Phi = dyn_cast<PHINode>(&ContBB->front());4010  EXPECT_NE(Phi, nullptr);4011  EXPECT_EQ(Phi->getNumIncomingValues(), 2U);4012  EXPECT_EQ(Phi->getIncomingBlock(0), EntryBB);4013  EXPECT_EQ(Phi->getIncomingBlock(1), ContBB);4014 4015  EXPECT_TRUE(Sub->hasOneUse());4016  StoreInst *St = dyn_cast<StoreInst>(Sub->user_back());4017  AllocaInst *UpdateTemp = dyn_cast<AllocaInst>(St->getPointerOperand());4018 4019  ExtractValueInst *ExVI1 =4020      dyn_cast<ExtractValueInst>(Phi->getIncomingValueForBlock(ContBB));4021  EXPECT_NE(ExVI1, nullptr);4022  AtomicCmpXchgInst *CmpExchg =4023      dyn_cast<AtomicCmpXchgInst>(ExVI1->getAggregateOperand());4024  EXPECT_NE(CmpExchg, nullptr);4025  EXPECT_EQ(CmpExchg->getPointerOperand(), XVal);4026  EXPECT_EQ(CmpExchg->getCompareOperand(), Phi);4027  EXPECT_EQ(CmpExchg->getSuccessOrdering(), AtomicOrdering::Monotonic);4028 4029  LoadInst *Ld = dyn_cast<LoadInst>(CmpExchg->getNewValOperand());4030  EXPECT_NE(Ld, nullptr);4031  EXPECT_EQ(UpdateTemp, Ld->getPointerOperand());4032 4033  Builder.CreateRetVoid();4034  OMPBuilder.finalize();4035  EXPECT_FALSE(verifyModule(*M, &errs()));4036}4037 4038TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdateFloat) {4039  OpenMPIRBuilder OMPBuilder(*M);4040  OMPBuilder.initialize();4041  F->setName("func");4042  IRBuilder<> Builder(BB);4043 4044  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});4045 4046  Type *FloatTy = Type::getFloatTy(M->getContext());4047  AllocaInst *XVal = Builder.CreateAlloca(FloatTy);4048  XVal->setName("AtomicVar");4049  Builder.CreateStore(ConstantFP::get(Type::getFloatTy(Ctx), 0.0), XVal);4050  OpenMPIRBuilder::AtomicOpValue X = {XVal, FloatTy, false, false};4051  AtomicOrdering AO = AtomicOrdering::Monotonic;4052  Constant *ConstVal = ConstantFP::get(Type::getFloatTy(Ctx), 1.0);4053  Value *Expr = nullptr;4054  AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::FSub;4055  bool IsXLHSInRHSPart = false;4056 4057  BasicBlock *EntryBB = BB;4058  OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,4059                                          EntryBB->getFirstInsertionPt());4060  Value *Sub = nullptr;4061 4062  auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) {4063    Sub = IRB.CreateFSub(ConstVal, Atomic);4064    return Sub;4065  };4066  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,4067                       OMPBuilder.createAtomicUpdate(Builder, AllocaIP, X, Expr,4068                                                     AO, RMWOp, UpdateOp,4069                                                     IsXLHSInRHSPart));4070  Builder.restoreIP(AfterIP);4071  BasicBlock *ContBB = EntryBB->getSingleSuccessor();4072  BranchInst *ContTI = dyn_cast<BranchInst>(ContBB->getTerminator());4073  EXPECT_NE(ContTI, nullptr);4074  BasicBlock *EndBB = ContTI->getSuccessor(0);4075  EXPECT_TRUE(ContTI->isConditional());4076  EXPECT_EQ(ContTI->getSuccessor(1), ContBB);4077  EXPECT_NE(EndBB, nullptr);4078 4079  PHINode *Phi = dyn_cast<PHINode>(&ContBB->front());4080  EXPECT_NE(Phi, nullptr);4081  EXPECT_EQ(Phi->getNumIncomingValues(), 2U);4082  EXPECT_EQ(Phi->getIncomingBlock(0), EntryBB);4083  EXPECT_EQ(Phi->getIncomingBlock(1), ContBB);4084 4085  EXPECT_TRUE(Sub->hasOneUse());4086  StoreInst *St = dyn_cast<StoreInst>(Sub->user_back());4087  AllocaInst *UpdateTemp = dyn_cast<AllocaInst>(St->getPointerOperand());4088 4089  ExtractValueInst *ExVI1 =4090      dyn_cast<ExtractValueInst>(Phi->getIncomingValueForBlock(ContBB));4091  EXPECT_NE(ExVI1, nullptr);4092  AtomicCmpXchgInst *CmpExchg =4093      dyn_cast<AtomicCmpXchgInst>(ExVI1->getAggregateOperand());4094  EXPECT_NE(CmpExchg, nullptr);4095  EXPECT_EQ(CmpExchg->getPointerOperand(), XVal);4096  EXPECT_EQ(CmpExchg->getCompareOperand(), Phi);4097  EXPECT_EQ(CmpExchg->getSuccessOrdering(), AtomicOrdering::Monotonic);4098 4099  LoadInst *Ld = dyn_cast<LoadInst>(CmpExchg->getNewValOperand());4100  EXPECT_NE(Ld, nullptr);4101  EXPECT_EQ(UpdateTemp, Ld->getPointerOperand());4102  Builder.CreateRetVoid();4103  OMPBuilder.finalize();4104  EXPECT_FALSE(verifyModule(*M, &errs()));4105}4106 4107TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdateIntr) {4108  OpenMPIRBuilder OMPBuilder(*M);4109  OMPBuilder.initialize();4110  F->setName("func");4111  IRBuilder<> Builder(BB);4112 4113  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});4114 4115  Type *IntTy = Type::getInt32Ty(M->getContext());4116  AllocaInst *XVal = Builder.CreateAlloca(IntTy);4117  XVal->setName("AtomicVar");4118  Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0), XVal);4119  OpenMPIRBuilder::AtomicOpValue X = {XVal, IntTy, false, false};4120  AtomicOrdering AO = AtomicOrdering::Monotonic;4121  Constant *ConstVal = ConstantInt::get(Type::getInt32Ty(Ctx), 1);4122  Value *Expr = ConstantInt::get(Type::getInt32Ty(Ctx), 1);4123  AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::UMax;4124  bool IsXLHSInRHSPart = false;4125 4126  BasicBlock *EntryBB = BB;4127  OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,4128                                          EntryBB->getFirstInsertionPt());4129  Value *Sub = nullptr;4130 4131  auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) {4132    Sub = IRB.CreateSub(ConstVal, Atomic);4133    return Sub;4134  };4135  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,4136                       OMPBuilder.createAtomicUpdate(Builder, AllocaIP, X, Expr,4137                                                     AO, RMWOp, UpdateOp,4138                                                     IsXLHSInRHSPart));4139  Builder.restoreIP(AfterIP);4140  BasicBlock *ContBB = EntryBB->getSingleSuccessor();4141  BranchInst *ContTI = dyn_cast<BranchInst>(ContBB->getTerminator());4142  EXPECT_NE(ContTI, nullptr);4143  BasicBlock *EndBB = ContTI->getSuccessor(0);4144  EXPECT_TRUE(ContTI->isConditional());4145  EXPECT_EQ(ContTI->getSuccessor(1), ContBB);4146  EXPECT_NE(EndBB, nullptr);4147 4148  PHINode *Phi = dyn_cast<PHINode>(&ContBB->front());4149  EXPECT_NE(Phi, nullptr);4150  EXPECT_EQ(Phi->getNumIncomingValues(), 2U);4151  EXPECT_EQ(Phi->getIncomingBlock(0), EntryBB);4152  EXPECT_EQ(Phi->getIncomingBlock(1), ContBB);4153 4154  EXPECT_TRUE(Sub->hasOneUse());4155  StoreInst *St = dyn_cast<StoreInst>(Sub->user_back());4156  AllocaInst *UpdateTemp = dyn_cast<AllocaInst>(St->getPointerOperand());4157 4158  ExtractValueInst *ExVI1 =4159      dyn_cast<ExtractValueInst>(Phi->getIncomingValueForBlock(ContBB));4160  EXPECT_NE(ExVI1, nullptr);4161  AtomicCmpXchgInst *CmpExchg =4162      dyn_cast<AtomicCmpXchgInst>(ExVI1->getAggregateOperand());4163  EXPECT_NE(CmpExchg, nullptr);4164  EXPECT_EQ(CmpExchg->getPointerOperand(), XVal);4165  EXPECT_EQ(CmpExchg->getCompareOperand(), Phi);4166  EXPECT_EQ(CmpExchg->getSuccessOrdering(), AtomicOrdering::Monotonic);4167 4168  LoadInst *Ld = dyn_cast<LoadInst>(CmpExchg->getNewValOperand());4169  EXPECT_NE(Ld, nullptr);4170  EXPECT_EQ(UpdateTemp, Ld->getPointerOperand());4171 4172  Builder.CreateRetVoid();4173  OMPBuilder.finalize();4174  EXPECT_FALSE(verifyModule(*M, &errs()));4175}4176 4177TEST_F(OpenMPIRBuilderTest, OMPAtomicCapture) {4178  OpenMPIRBuilder OMPBuilder(*M);4179  OMPBuilder.initialize();4180  F->setName("func");4181  IRBuilder<> Builder(BB);4182 4183  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});4184 4185  LLVMContext &Ctx = M->getContext();4186  IntegerType *Int32 = Type::getInt32Ty(Ctx);4187  AllocaInst *XVal = Builder.CreateAlloca(Int32);4188  XVal->setName("AtomicVar");4189  AllocaInst *VVal = Builder.CreateAlloca(Int32);4190  VVal->setName("AtomicCapTar");4191  StoreInst *Init =4192      Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), XVal);4193 4194  OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false};4195  OpenMPIRBuilder::AtomicOpValue V = {VVal, Int32, false, false};4196  AtomicOrdering AO = AtomicOrdering::Monotonic;4197  ConstantInt *Expr = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);4198  AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::Add;4199  bool IsXLHSInRHSPart = true;4200  bool IsPostfixUpdate = true;4201  bool UpdateExpr = true;4202 4203  BasicBlock *EntryBB = BB;4204  OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,4205                                          EntryBB->getFirstInsertionPt());4206 4207  // integer update - not used4208  auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) { return nullptr; };4209 4210  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,4211                       OMPBuilder.createAtomicCapture(4212                           Builder, AllocaIP, X, V, Expr, AO, RMWOp, UpdateOp,4213                           UpdateExpr, IsPostfixUpdate, IsXLHSInRHSPart));4214  Builder.restoreIP(AfterIP);4215  EXPECT_EQ(EntryBB->getParent()->size(), 1U);4216  AtomicRMWInst *ARWM = dyn_cast<AtomicRMWInst>(Init->getNextNode());4217  EXPECT_NE(ARWM, nullptr);4218  EXPECT_EQ(ARWM->getPointerOperand(), XVal);4219  EXPECT_EQ(ARWM->getOperation(), RMWOp);4220  StoreInst *St = dyn_cast<StoreInst>(ARWM->user_back());4221  EXPECT_NE(St, nullptr);4222  EXPECT_EQ(St->getPointerOperand(), VVal);4223 4224  Builder.CreateRetVoid();4225  OMPBuilder.finalize();4226  EXPECT_FALSE(verifyModule(*M, &errs()));4227}4228 4229TEST_F(OpenMPIRBuilderTest, OMPAtomicCompare) {4230  OpenMPIRBuilder OMPBuilder(*M);4231  OMPBuilder.initialize();4232  F->setName("func");4233  IRBuilder<> Builder(BB);4234 4235  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});4236 4237  LLVMContext &Ctx = M->getContext();4238  IntegerType *Int32 = Type::getInt32Ty(Ctx);4239  AllocaInst *XVal = Builder.CreateAlloca(Int32);4240  XVal->setName("x");4241  StoreInst *Init =4242      Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), XVal);4243 4244  OpenMPIRBuilder::AtomicOpValue XSigned = {XVal, Int32, true, false};4245  OpenMPIRBuilder::AtomicOpValue XUnsigned = {XVal, Int32, false, false};4246  // V and R are not used in atomic compare4247  OpenMPIRBuilder::AtomicOpValue V = {nullptr, nullptr, false, false};4248  OpenMPIRBuilder::AtomicOpValue R = {nullptr, nullptr, false, false};4249  AtomicOrdering AO = AtomicOrdering::Monotonic;4250  ConstantInt *Expr = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);4251  ConstantInt *D = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);4252  OMPAtomicCompareOp OpMax = OMPAtomicCompareOp::MAX;4253  OMPAtomicCompareOp OpEQ = OMPAtomicCompareOp::EQ;4254 4255  Builder.restoreIP(OMPBuilder.createAtomicCompare(4256      Builder, XSigned, V, R, Expr, nullptr, AO, OpMax, true, false, false));4257  Builder.restoreIP(OMPBuilder.createAtomicCompare(4258      Builder, XUnsigned, V, R, Expr, nullptr, AO, OpMax, false, false, false));4259  Builder.restoreIP(OMPBuilder.createAtomicCompare(4260      Builder, XSigned, V, R, Expr, D, AO, OpEQ, true, false, false));4261 4262  BasicBlock *EntryBB = BB;4263  EXPECT_EQ(EntryBB->getParent()->size(), 1U);4264  EXPECT_EQ(EntryBB->size(), 5U);4265 4266  AtomicRMWInst *ARWM1 = dyn_cast<AtomicRMWInst>(Init->getNextNode());4267  EXPECT_NE(ARWM1, nullptr);4268  EXPECT_EQ(ARWM1->getPointerOperand(), XVal);4269  EXPECT_EQ(ARWM1->getValOperand(), Expr);4270  EXPECT_EQ(ARWM1->getOperation(), AtomicRMWInst::Min);4271 4272  AtomicRMWInst *ARWM2 = dyn_cast<AtomicRMWInst>(ARWM1->getNextNode());4273  EXPECT_NE(ARWM2, nullptr);4274  EXPECT_EQ(ARWM2->getPointerOperand(), XVal);4275  EXPECT_EQ(ARWM2->getValOperand(), Expr);4276  EXPECT_EQ(ARWM2->getOperation(), AtomicRMWInst::UMax);4277 4278  AtomicCmpXchgInst *AXCHG = dyn_cast<AtomicCmpXchgInst>(ARWM2->getNextNode());4279  EXPECT_NE(AXCHG, nullptr);4280  EXPECT_EQ(AXCHG->getPointerOperand(), XVal);4281  EXPECT_EQ(AXCHG->getCompareOperand(), Expr);4282  EXPECT_EQ(AXCHG->getNewValOperand(), D);4283 4284  Builder.CreateRetVoid();4285  OMPBuilder.finalize();4286  EXPECT_FALSE(verifyModule(*M, &errs()));4287}4288 4289TEST_F(OpenMPIRBuilderTest, OMPAtomicCompareCapture) {4290  OpenMPIRBuilder OMPBuilder(*M);4291  OMPBuilder.initialize();4292  F->setName("func");4293  IRBuilder<> Builder(BB);4294 4295  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});4296 4297  LLVMContext &Ctx = M->getContext();4298  IntegerType *Int32 = Type::getInt32Ty(Ctx);4299  AllocaInst *XVal = Builder.CreateAlloca(Int32);4300  XVal->setName("x");4301  AllocaInst *VVal = Builder.CreateAlloca(Int32);4302  VVal->setName("v");4303  AllocaInst *RVal = Builder.CreateAlloca(Int32);4304  RVal->setName("r");4305 4306  StoreInst *Init =4307      Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), XVal);4308 4309  OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, true, false};4310  OpenMPIRBuilder::AtomicOpValue V = {VVal, Int32, false, false};4311  OpenMPIRBuilder::AtomicOpValue NoV = {nullptr, nullptr, false, false};4312  OpenMPIRBuilder::AtomicOpValue R = {RVal, Int32, false, false};4313  OpenMPIRBuilder::AtomicOpValue NoR = {nullptr, nullptr, false, false};4314 4315  AtomicOrdering AO = AtomicOrdering::Monotonic;4316  ConstantInt *Expr = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);4317  ConstantInt *D = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);4318  OMPAtomicCompareOp OpMax = OMPAtomicCompareOp::MAX;4319  OMPAtomicCompareOp OpEQ = OMPAtomicCompareOp::EQ;4320 4321  // { cond-update-stmt v = x; }4322  Builder.restoreIP(OMPBuilder.createAtomicCompare(4323      Builder, X, V, NoR, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,4324      /* IsPostfixUpdate */ false,4325      /* IsFailOnly */ false));4326  // { v = x; cond-update-stmt }4327  Builder.restoreIP(OMPBuilder.createAtomicCompare(4328      Builder, X, V, NoR, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,4329      /* IsPostfixUpdate */ true,4330      /* IsFailOnly */ false));4331  // if(x == e) { x = d; } else { v = x; }4332  Builder.restoreIP(OMPBuilder.createAtomicCompare(4333      Builder, X, V, NoR, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,4334      /* IsPostfixUpdate */ false,4335      /* IsFailOnly */ true));4336  // { r = x == e; if(r) { x = d; } }4337  Builder.restoreIP(OMPBuilder.createAtomicCompare(4338      Builder, X, NoV, R, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,4339      /* IsPostfixUpdate */ false,4340      /* IsFailOnly */ false));4341  // { r = x == e; if(r) { x = d; } else { v = x; } }4342  Builder.restoreIP(OMPBuilder.createAtomicCompare(4343      Builder, X, V, R, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,4344      /* IsPostfixUpdate */ false,4345      /* IsFailOnly */ true));4346 4347  // { v = x; cond-update-stmt }4348  Builder.restoreIP(OMPBuilder.createAtomicCompare(4349      Builder, X, V, NoR, Expr, nullptr, AO, OpMax, /* IsXBinopExpr */ true,4350      /* IsPostfixUpdate */ true,4351      /* IsFailOnly */ false));4352  // { cond-update-stmt v = x; }4353  Builder.restoreIP(OMPBuilder.createAtomicCompare(4354      Builder, X, V, NoR, Expr, nullptr, AO, OpMax, /* IsXBinopExpr */ false,4355      /* IsPostfixUpdate */ false,4356      /* IsFailOnly */ false));4357 4358  BasicBlock *EntryBB = BB;4359  EXPECT_EQ(EntryBB->getParent()->size(), 5U);4360  BasicBlock *Cont1 = dyn_cast<BasicBlock>(EntryBB->getNextNode());4361  EXPECT_NE(Cont1, nullptr);4362  BasicBlock *Exit1 = dyn_cast<BasicBlock>(Cont1->getNextNode());4363  EXPECT_NE(Exit1, nullptr);4364  BasicBlock *Cont2 = dyn_cast<BasicBlock>(Exit1->getNextNode());4365  EXPECT_NE(Cont2, nullptr);4366  BasicBlock *Exit2 = dyn_cast<BasicBlock>(Cont2->getNextNode());4367  EXPECT_NE(Exit2, nullptr);4368 4369  AtomicCmpXchgInst *CmpXchg1 =4370      dyn_cast<AtomicCmpXchgInst>(Init->getNextNode());4371  EXPECT_NE(CmpXchg1, nullptr);4372  EXPECT_EQ(CmpXchg1->getPointerOperand(), XVal);4373  EXPECT_EQ(CmpXchg1->getCompareOperand(), Expr);4374  EXPECT_EQ(CmpXchg1->getNewValOperand(), D);4375  ExtractValueInst *ExtVal1 =4376      dyn_cast<ExtractValueInst>(CmpXchg1->getNextNode());4377  EXPECT_NE(ExtVal1, nullptr);4378  EXPECT_EQ(ExtVal1->getAggregateOperand(), CmpXchg1);4379  EXPECT_EQ(ExtVal1->getIndices(), ArrayRef<unsigned int>(0U));4380  ExtractValueInst *ExtVal2 =4381      dyn_cast<ExtractValueInst>(ExtVal1->getNextNode());4382  EXPECT_NE(ExtVal2, nullptr);4383  EXPECT_EQ(ExtVal2->getAggregateOperand(), CmpXchg1);4384  EXPECT_EQ(ExtVal2->getIndices(), ArrayRef<unsigned int>(1U));4385  SelectInst *Sel1 = dyn_cast<SelectInst>(ExtVal2->getNextNode());4386  EXPECT_NE(Sel1, nullptr);4387  EXPECT_EQ(Sel1->getCondition(), ExtVal2);4388  EXPECT_EQ(Sel1->getTrueValue(), Expr);4389  EXPECT_EQ(Sel1->getFalseValue(), ExtVal1);4390  StoreInst *Store1 = dyn_cast<StoreInst>(Sel1->getNextNode());4391  EXPECT_NE(Store1, nullptr);4392  EXPECT_EQ(Store1->getPointerOperand(), VVal);4393  EXPECT_EQ(Store1->getValueOperand(), Sel1);4394 4395  AtomicCmpXchgInst *CmpXchg2 =4396      dyn_cast<AtomicCmpXchgInst>(Store1->getNextNode());4397  EXPECT_NE(CmpXchg2, nullptr);4398  EXPECT_EQ(CmpXchg2->getPointerOperand(), XVal);4399  EXPECT_EQ(CmpXchg2->getCompareOperand(), Expr);4400  EXPECT_EQ(CmpXchg2->getNewValOperand(), D);4401  ExtractValueInst *ExtVal3 =4402      dyn_cast<ExtractValueInst>(CmpXchg2->getNextNode());4403  EXPECT_NE(ExtVal3, nullptr);4404  EXPECT_EQ(ExtVal3->getAggregateOperand(), CmpXchg2);4405  EXPECT_EQ(ExtVal3->getIndices(), ArrayRef<unsigned int>(0U));4406  StoreInst *Store2 = dyn_cast<StoreInst>(ExtVal3->getNextNode());4407  EXPECT_NE(Store2, nullptr);4408  EXPECT_EQ(Store2->getPointerOperand(), VVal);4409  EXPECT_EQ(Store2->getValueOperand(), ExtVal3);4410 4411  AtomicCmpXchgInst *CmpXchg3 =4412      dyn_cast<AtomicCmpXchgInst>(Store2->getNextNode());4413  EXPECT_NE(CmpXchg3, nullptr);4414  EXPECT_EQ(CmpXchg3->getPointerOperand(), XVal);4415  EXPECT_EQ(CmpXchg3->getCompareOperand(), Expr);4416  EXPECT_EQ(CmpXchg3->getNewValOperand(), D);4417  ExtractValueInst *ExtVal4 =4418      dyn_cast<ExtractValueInst>(CmpXchg3->getNextNode());4419  EXPECT_NE(ExtVal4, nullptr);4420  EXPECT_EQ(ExtVal4->getAggregateOperand(), CmpXchg3);4421  EXPECT_EQ(ExtVal4->getIndices(), ArrayRef<unsigned int>(0U));4422  ExtractValueInst *ExtVal5 =4423      dyn_cast<ExtractValueInst>(ExtVal4->getNextNode());4424  EXPECT_NE(ExtVal5, nullptr);4425  EXPECT_EQ(ExtVal5->getAggregateOperand(), CmpXchg3);4426  EXPECT_EQ(ExtVal5->getIndices(), ArrayRef<unsigned int>(1U));4427  BranchInst *Br1 = dyn_cast<BranchInst>(ExtVal5->getNextNode());4428  EXPECT_NE(Br1, nullptr);4429  EXPECT_EQ(Br1->isConditional(), true);4430  EXPECT_EQ(Br1->getCondition(), ExtVal5);4431  EXPECT_EQ(Br1->getSuccessor(0), Exit1);4432  EXPECT_EQ(Br1->getSuccessor(1), Cont1);4433 4434  StoreInst *Store3 = dyn_cast<StoreInst>(&Cont1->front());4435  EXPECT_NE(Store3, nullptr);4436  EXPECT_EQ(Store3->getPointerOperand(), VVal);4437  EXPECT_EQ(Store3->getValueOperand(), ExtVal4);4438  BranchInst *Br2 = dyn_cast<BranchInst>(Store3->getNextNode());4439  EXPECT_NE(Br2, nullptr);4440  EXPECT_EQ(Br2->isUnconditional(), true);4441  EXPECT_EQ(Br2->getSuccessor(0), Exit1);4442 4443  AtomicCmpXchgInst *CmpXchg4 = dyn_cast<AtomicCmpXchgInst>(&Exit1->front());4444  EXPECT_NE(CmpXchg4, nullptr);4445  EXPECT_EQ(CmpXchg4->getPointerOperand(), XVal);4446  EXPECT_EQ(CmpXchg4->getCompareOperand(), Expr);4447  EXPECT_EQ(CmpXchg4->getNewValOperand(), D);4448  ExtractValueInst *ExtVal6 =4449      dyn_cast<ExtractValueInst>(CmpXchg4->getNextNode());4450  EXPECT_NE(ExtVal6, nullptr);4451  EXPECT_EQ(ExtVal6->getAggregateOperand(), CmpXchg4);4452  EXPECT_EQ(ExtVal6->getIndices(), ArrayRef<unsigned int>(1U));4453  ZExtInst *ZExt1 = dyn_cast<ZExtInst>(ExtVal6->getNextNode());4454  EXPECT_NE(ZExt1, nullptr);4455  EXPECT_EQ(ZExt1->getDestTy(), Int32);4456  StoreInst *Store4 = dyn_cast<StoreInst>(ZExt1->getNextNode());4457  EXPECT_NE(Store4, nullptr);4458  EXPECT_EQ(Store4->getPointerOperand(), RVal);4459  EXPECT_EQ(Store4->getValueOperand(), ZExt1);4460 4461  AtomicCmpXchgInst *CmpXchg5 =4462      dyn_cast<AtomicCmpXchgInst>(Store4->getNextNode());4463  EXPECT_NE(CmpXchg5, nullptr);4464  EXPECT_EQ(CmpXchg5->getPointerOperand(), XVal);4465  EXPECT_EQ(CmpXchg5->getCompareOperand(), Expr);4466  EXPECT_EQ(CmpXchg5->getNewValOperand(), D);4467  ExtractValueInst *ExtVal7 =4468      dyn_cast<ExtractValueInst>(CmpXchg5->getNextNode());4469  EXPECT_NE(ExtVal7, nullptr);4470  EXPECT_EQ(ExtVal7->getAggregateOperand(), CmpXchg5);4471  EXPECT_EQ(ExtVal7->getIndices(), ArrayRef<unsigned int>(0U));4472  ExtractValueInst *ExtVal8 =4473      dyn_cast<ExtractValueInst>(ExtVal7->getNextNode());4474  EXPECT_NE(ExtVal8, nullptr);4475  EXPECT_EQ(ExtVal8->getAggregateOperand(), CmpXchg5);4476  EXPECT_EQ(ExtVal8->getIndices(), ArrayRef<unsigned int>(1U));4477  BranchInst *Br3 = dyn_cast<BranchInst>(ExtVal8->getNextNode());4478  EXPECT_NE(Br3, nullptr);4479  EXPECT_EQ(Br3->isConditional(), true);4480  EXPECT_EQ(Br3->getCondition(), ExtVal8);4481  EXPECT_EQ(Br3->getSuccessor(0), Exit2);4482  EXPECT_EQ(Br3->getSuccessor(1), Cont2);4483 4484  StoreInst *Store5 = dyn_cast<StoreInst>(&Cont2->front());4485  EXPECT_NE(Store5, nullptr);4486  EXPECT_EQ(Store5->getPointerOperand(), VVal);4487  EXPECT_EQ(Store5->getValueOperand(), ExtVal7);4488  BranchInst *Br4 = dyn_cast<BranchInst>(Store5->getNextNode());4489  EXPECT_NE(Br4, nullptr);4490  EXPECT_EQ(Br4->isUnconditional(), true);4491  EXPECT_EQ(Br4->getSuccessor(0), Exit2);4492 4493  ExtractValueInst *ExtVal9 = dyn_cast<ExtractValueInst>(&Exit2->front());4494  EXPECT_NE(ExtVal9, nullptr);4495  EXPECT_EQ(ExtVal9->getAggregateOperand(), CmpXchg5);4496  EXPECT_EQ(ExtVal9->getIndices(), ArrayRef<unsigned int>(1U));4497  ZExtInst *ZExt2 = dyn_cast<ZExtInst>(ExtVal9->getNextNode());4498  EXPECT_NE(ZExt2, nullptr);4499  EXPECT_EQ(ZExt2->getDestTy(), Int32);4500  StoreInst *Store6 = dyn_cast<StoreInst>(ZExt2->getNextNode());4501  EXPECT_NE(Store6, nullptr);4502  EXPECT_EQ(Store6->getPointerOperand(), RVal);4503  EXPECT_EQ(Store6->getValueOperand(), ZExt2);4504 4505  AtomicRMWInst *ARWM1 = dyn_cast<AtomicRMWInst>(Store6->getNextNode());4506  EXPECT_NE(ARWM1, nullptr);4507  EXPECT_EQ(ARWM1->getPointerOperand(), XVal);4508  EXPECT_EQ(ARWM1->getValOperand(), Expr);4509  EXPECT_EQ(ARWM1->getOperation(), AtomicRMWInst::Min);4510  StoreInst *Store7 = dyn_cast<StoreInst>(ARWM1->getNextNode());4511  EXPECT_NE(Store7, nullptr);4512  EXPECT_EQ(Store7->getPointerOperand(), VVal);4513  EXPECT_EQ(Store7->getValueOperand(), ARWM1);4514 4515  AtomicRMWInst *ARWM2 = dyn_cast<AtomicRMWInst>(Store7->getNextNode());4516  EXPECT_NE(ARWM2, nullptr);4517  EXPECT_EQ(ARWM2->getPointerOperand(), XVal);4518  EXPECT_EQ(ARWM2->getValOperand(), Expr);4519  EXPECT_EQ(ARWM2->getOperation(), AtomicRMWInst::Max);4520  CmpInst *Cmp1 = dyn_cast<CmpInst>(ARWM2->getNextNode());4521  EXPECT_NE(Cmp1, nullptr);4522  EXPECT_EQ(Cmp1->getPredicate(), CmpInst::ICMP_SGT);4523  EXPECT_EQ(Cmp1->getOperand(0), ARWM2);4524  EXPECT_EQ(Cmp1->getOperand(1), Expr);4525  SelectInst *Sel2 = dyn_cast<SelectInst>(Cmp1->getNextNode());4526  EXPECT_NE(Sel2, nullptr);4527  EXPECT_EQ(Sel2->getCondition(), Cmp1);4528  EXPECT_EQ(Sel2->getTrueValue(), Expr);4529  EXPECT_EQ(Sel2->getFalseValue(), ARWM2);4530  StoreInst *Store8 = dyn_cast<StoreInst>(Sel2->getNextNode());4531  EXPECT_NE(Store8, nullptr);4532  EXPECT_EQ(Store8->getPointerOperand(), VVal);4533  EXPECT_EQ(Store8->getValueOperand(), Sel2);4534 4535  Builder.CreateRetVoid();4536  OMPBuilder.finalize();4537  EXPECT_FALSE(verifyModule(*M, &errs()));4538}4539 4540TEST_F(OpenMPIRBuilderTest, OMPAtomicRWStructType) {4541  // Test for issue #165184: atomic read/write on struct types should use4542  // element type size, not pointer size.4543  OpenMPIRBuilder OMPBuilder(*M);4544  OMPBuilder.initialize();4545  F->setName("func");4546  IRBuilder<> Builder(BB);4547 4548  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});4549  BasicBlock *EntryBB = BB;4550  OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,4551                                          EntryBB->getFirstInsertionPt());4552 4553  LLVMContext &Ctx = M->getContext();4554 4555  // Create a struct type {double, double} to simulate complex(8) - 16 bytes4556  StructType *Complex8Ty = StructType::create(4557      Ctx, {Type::getDoubleTy(Ctx), Type::getDoubleTy(Ctx)}, "complex");4558 4559  AllocaInst *XVal = Builder.CreateAlloca(Complex8Ty);4560  XVal->setName("AtomicVar");4561  OpenMPIRBuilder::AtomicOpValue X = {XVal, Complex8Ty, false, false};4562  AtomicOrdering AO = AtomicOrdering::SequentiallyConsistent;4563 4564  // Create value to write: {1.0, 1.0}4565  Constant *Real = ConstantFP::get(Type::getDoubleTy(Ctx), 1.0);4566  Constant *Imag = ConstantFP::get(Type::getDoubleTy(Ctx), 1.0);4567  Constant *ValToWrite = ConstantStruct::get(Complex8Ty, {Real, Imag});4568 4569  // Test atomic write4570  Builder.restoreIP(4571      OMPBuilder.createAtomicWrite(Loc, X, ValToWrite, AO, AllocaIP));4572 4573  // Test atomic read4574  AllocaInst *VVal = Builder.CreateAlloca(Complex8Ty);4575  VVal->setName("ReadDest");4576  OpenMPIRBuilder::AtomicOpValue V = {VVal, Complex8Ty, false, false};4577 4578  Builder.restoreIP(OMPBuilder.createAtomicRead(Loc, X, V, AO, AllocaIP));4579 4580  Builder.CreateRetVoid();4581  OMPBuilder.finalize();4582  EXPECT_FALSE(verifyModule(*M, &errs()));4583 4584  // Verify that __atomic_store and __atomic_load are called with size 164585  bool FoundAtomicStore = false;4586  bool FoundAtomicLoad = false;4587 4588  for (Function &Fn : *M) {4589    if (Fn.getName().starts_with("__atomic_store")) {4590      // Check that first call to __atomic_store has size argument = 164591      for (User *U : Fn.users()) {4592        if (auto *CB = dyn_cast<CallBase>(U)) {4593          if (auto *SizeArg = dyn_cast<ConstantInt>(CB->getArgOperand(0))) {4594            EXPECT_EQ(SizeArg->getZExtValue(), 16U);4595            FoundAtomicStore = true;4596            break;4597          }4598        }4599      }4600    }4601    if (Fn.getName().starts_with("__atomic_load")) {4602      // Check that first call to __atomic_load has size argument = 164603      for (User *U : Fn.users()) {4604        if (auto *CB = dyn_cast<CallBase>(U)) {4605          if (auto *SizeArg = dyn_cast<ConstantInt>(CB->getArgOperand(0))) {4606            EXPECT_EQ(SizeArg->getZExtValue(), 16U);4607            FoundAtomicLoad = true;4608            break;4609          }4610        }4611      }4612    }4613  }4614 4615  EXPECT_TRUE(FoundAtomicStore) << "Did not find __atomic_store call";4616  EXPECT_TRUE(FoundAtomicLoad) << "Did not find __atomic_load call";4617}4618 4619TEST_F(OpenMPIRBuilderTest, CreateTeams) {4620  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;4621  OpenMPIRBuilder OMPBuilder(*M);4622  OMPBuilder.Config.IsTargetDevice = false;4623  OMPBuilder.initialize();4624  F->setName("func");4625  IRBuilder<> Builder(BB);4626 4627  AllocaInst *ValPtr32 = Builder.CreateAlloca(Builder.getInt32Ty());4628  AllocaInst *ValPtr128 = Builder.CreateAlloca(Builder.getInt128Ty());4629  Value *Val128 = Builder.CreateLoad(Builder.getInt128Ty(), ValPtr128, "load");4630 4631  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {4632    Builder.restoreIP(AllocaIP);4633    AllocaInst *Local128 = Builder.CreateAlloca(Builder.getInt128Ty(), nullptr,4634                                                "bodygen.alloca128");4635 4636    Builder.restoreIP(CodeGenIP);4637    // Loading and storing captured pointer and values4638    Builder.CreateStore(Val128, Local128);4639    Value *Val32 = Builder.CreateLoad(ValPtr32->getAllocatedType(), ValPtr32,4640                                      "bodygen.load32");4641 4642    LoadInst *PrivLoad128 = Builder.CreateLoad(4643        Local128->getAllocatedType(), Local128, "bodygen.local.load128");4644    Value *Cmp = Builder.CreateICmpNE(4645        Val32, Builder.CreateTrunc(PrivLoad128, Val32->getType()));4646    Instruction *ThenTerm, *ElseTerm;4647    SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(),4648                                  &ThenTerm, &ElseTerm);4649    return Error::success();4650  };4651 4652  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});4653  ASSERT_EXPECTED_INIT(4654      OpenMPIRBuilder::InsertPointTy, AfterIP,4655      OMPBuilder.createTeams(Builder, BodyGenCB, /*NumTeamsLower=*/nullptr,4656                             /*NumTeamsUpper=*/nullptr,4657                             /*ThreadLimit=*/nullptr, /*IfExpr=*/nullptr));4658  Builder.restoreIP(AfterIP);4659 4660  OMPBuilder.finalize();4661  Builder.CreateRetVoid();4662 4663  EXPECT_FALSE(verifyModule(*M, &errs()));4664 4665  CallInst *TeamsForkCall = dyn_cast<CallInst>(4666      OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams)4667          ->user_back());4668 4669  // Verify the Ident argument4670  GlobalVariable *Ident = cast<GlobalVariable>(TeamsForkCall->getArgOperand(0));4671  ASSERT_NE(Ident, nullptr);4672  EXPECT_TRUE(Ident->hasInitializer());4673  Constant *Initializer = Ident->getInitializer();4674  GlobalVariable *SrcStrGlob =4675      cast<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts());4676  ASSERT_NE(SrcStrGlob, nullptr);4677  ConstantDataArray *SrcSrc =4678      dyn_cast<ConstantDataArray>(SrcStrGlob->getInitializer());4679  ASSERT_NE(SrcSrc, nullptr);4680 4681  // Verify the outlined function signature.4682  Function *OutlinedFn =4683      dyn_cast<Function>(TeamsForkCall->getArgOperand(2)->stripPointerCasts());4684  ASSERT_NE(OutlinedFn, nullptr);4685  EXPECT_FALSE(OutlinedFn->isDeclaration());4686  EXPECT_TRUE(OutlinedFn->arg_size() >= 3);4687  EXPECT_EQ(OutlinedFn->getArg(0)->getType(), Builder.getPtrTy()); // global_tid4688  EXPECT_EQ(OutlinedFn->getArg(1)->getType(), Builder.getPtrTy()); // bound_tid4689  EXPECT_EQ(OutlinedFn->getArg(2)->getType(),4690            Builder.getPtrTy()); // captured args4691 4692  // Check for TruncInst and ICmpInst in the outlined function.4693  EXPECT_TRUE(any_of(instructions(OutlinedFn),4694                     [](Instruction &inst) { return isa<TruncInst>(&inst); }));4695  EXPECT_TRUE(any_of(instructions(OutlinedFn),4696                     [](Instruction &inst) { return isa<ICmpInst>(&inst); }));4697}4698 4699TEST_F(OpenMPIRBuilderTest, CreateTeamsWithThreadLimit) {4700  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;4701  OpenMPIRBuilder OMPBuilder(*M);4702  OMPBuilder.Config.IsTargetDevice = false;4703  OMPBuilder.initialize();4704  F->setName("func");4705  IRBuilder<> &Builder = OMPBuilder.Builder;4706  Builder.SetInsertPoint(BB);4707 4708  Function *FakeFunction =4709      Function::Create(FunctionType::get(Builder.getVoidTy(), false),4710                       GlobalValue::ExternalLinkage, "fakeFunction", M.get());4711 4712  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {4713    Builder.restoreIP(CodeGenIP);4714    Builder.CreateCall(FakeFunction, {});4715    return Error::success();4716  };4717 4718  // `F` has an argument - an integer, so we use that as the thread limit.4719  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,4720                       OMPBuilder.createTeams(4721                           /*=*/Builder, BodyGenCB, /*NumTeamsLower=*/nullptr,4722                           /*NumTeamsUpper=*/nullptr,4723                           /*ThreadLimit=*/F->arg_begin(),4724                           /*IfExpr=*/nullptr));4725  Builder.restoreIP(AfterIP);4726 4727  Builder.CreateRetVoid();4728  OMPBuilder.finalize();4729 4730  ASSERT_FALSE(verifyModule(*M));4731 4732  CallInst *PushNumTeamsCallInst =4733      findSingleCall(F, OMPRTL___kmpc_push_num_teams_51, OMPBuilder);4734  ASSERT_NE(PushNumTeamsCallInst, nullptr);4735 4736  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(2), Builder.getInt32(0));4737  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(3), Builder.getInt32(0));4738  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(4), &*F->arg_begin());4739 4740  // Verifying that the next instruction to execute is kmpc_fork_teams4741  BranchInst *BrInst =4742      dyn_cast<BranchInst>(PushNumTeamsCallInst->getNextNode());4743  ASSERT_NE(BrInst, nullptr);4744  ASSERT_EQ(BrInst->getNumSuccessors(), 1U);4745  BasicBlock::iterator NextInstruction =4746      BrInst->getSuccessor(0)->getFirstNonPHIOrDbgOrLifetime();4747  CallInst *ForkTeamsCI = nullptr;4748  if (NextInstruction != BrInst->getSuccessor(0)->end())4749    ForkTeamsCI = dyn_cast_if_present<CallInst>(NextInstruction);4750  ASSERT_NE(ForkTeamsCI, nullptr);4751  EXPECT_EQ(ForkTeamsCI->getCalledFunction(),4752            OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams));4753}4754 4755TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsUpper) {4756  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;4757  OpenMPIRBuilder OMPBuilder(*M);4758  OMPBuilder.Config.IsTargetDevice = false;4759  OMPBuilder.initialize();4760  F->setName("func");4761  IRBuilder<> &Builder = OMPBuilder.Builder;4762  Builder.SetInsertPoint(BB);4763 4764  Function *FakeFunction =4765      Function::Create(FunctionType::get(Builder.getVoidTy(), false),4766                       GlobalValue::ExternalLinkage, "fakeFunction", M.get());4767 4768  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {4769    Builder.restoreIP(CodeGenIP);4770    Builder.CreateCall(FakeFunction, {});4771    return Error::success();4772  };4773 4774  // `F` already has an integer argument, so we use that as upper bound to4775  // `num_teams`4776  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,4777                       OMPBuilder.createTeams(Builder, BodyGenCB,4778                                              /*NumTeamsLower=*/nullptr,4779                                              /*NumTeamsUpper=*/F->arg_begin(),4780                                              /*ThreadLimit=*/nullptr,4781                                              /*IfExpr=*/nullptr));4782  Builder.restoreIP(AfterIP);4783 4784  Builder.CreateRetVoid();4785  OMPBuilder.finalize();4786 4787  ASSERT_FALSE(verifyModule(*M));4788 4789  CallInst *PushNumTeamsCallInst =4790      findSingleCall(F, OMPRTL___kmpc_push_num_teams_51, OMPBuilder);4791  ASSERT_NE(PushNumTeamsCallInst, nullptr);4792 4793  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(2), &*F->arg_begin());4794  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(3), &*F->arg_begin());4795  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(4), Builder.getInt32(0));4796 4797  // Verifying that the next instruction to execute is kmpc_fork_teams4798  BranchInst *BrInst =4799      dyn_cast<BranchInst>(PushNumTeamsCallInst->getNextNode());4800  ASSERT_NE(BrInst, nullptr);4801  ASSERT_EQ(BrInst->getNumSuccessors(), 1U);4802  BasicBlock::iterator NextInstruction =4803      BrInst->getSuccessor(0)->getFirstNonPHIOrDbgOrLifetime();4804  CallInst *ForkTeamsCI = nullptr;4805  if (NextInstruction != BrInst->getSuccessor(0)->end())4806    ForkTeamsCI = dyn_cast_if_present<CallInst>(NextInstruction);4807  ASSERT_NE(ForkTeamsCI, nullptr);4808  EXPECT_EQ(ForkTeamsCI->getCalledFunction(),4809            OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams));4810}4811 4812TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsBoth) {4813  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;4814  OpenMPIRBuilder OMPBuilder(*M);4815  OMPBuilder.Config.IsTargetDevice = false;4816  OMPBuilder.initialize();4817  F->setName("func");4818  IRBuilder<> &Builder = OMPBuilder.Builder;4819  Builder.SetInsertPoint(BB);4820 4821  Function *FakeFunction =4822      Function::Create(FunctionType::get(Builder.getVoidTy(), false),4823                       GlobalValue::ExternalLinkage, "fakeFunction", M.get());4824 4825  Value *NumTeamsLower =4826      Builder.CreateAdd(F->arg_begin(), Builder.getInt32(5), "numTeamsLower");4827  Value *NumTeamsUpper =4828      Builder.CreateAdd(F->arg_begin(), Builder.getInt32(10), "numTeamsUpper");4829 4830  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {4831    Builder.restoreIP(CodeGenIP);4832    Builder.CreateCall(FakeFunction, {});4833    return Error::success();4834  };4835 4836  // `F` already has an integer argument, so we use that as upper bound to4837  // `num_teams`4838  ASSERT_EXPECTED_INIT(4839      OpenMPIRBuilder::InsertPointTy, AfterIP,4840      OMPBuilder.createTeams(Builder, BodyGenCB, NumTeamsLower, NumTeamsUpper,4841                             /*ThreadLimit=*/nullptr, /*IfExpr=*/nullptr));4842  Builder.restoreIP(AfterIP);4843 4844  Builder.CreateRetVoid();4845  OMPBuilder.finalize();4846 4847  ASSERT_FALSE(verifyModule(*M));4848 4849  CallInst *PushNumTeamsCallInst =4850      findSingleCall(F, OMPRTL___kmpc_push_num_teams_51, OMPBuilder);4851  ASSERT_NE(PushNumTeamsCallInst, nullptr);4852 4853  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(2), NumTeamsLower);4854  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(3), NumTeamsUpper);4855  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(4), Builder.getInt32(0));4856 4857  // Verifying that the next instruction to execute is kmpc_fork_teams4858  BranchInst *BrInst =4859      dyn_cast<BranchInst>(PushNumTeamsCallInst->getNextNode());4860  ASSERT_NE(BrInst, nullptr);4861  ASSERT_EQ(BrInst->getNumSuccessors(), 1U);4862  BasicBlock::iterator NextInstruction =4863      BrInst->getSuccessor(0)->getFirstNonPHIOrDbgOrLifetime();4864  CallInst *ForkTeamsCI = nullptr;4865  if (NextInstruction != BrInst->getSuccessor(0)->end())4866    ForkTeamsCI = dyn_cast_if_present<CallInst>(NextInstruction);4867  ASSERT_NE(ForkTeamsCI, nullptr);4868  EXPECT_EQ(ForkTeamsCI->getCalledFunction(),4869            OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams));4870}4871 4872TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsAndThreadLimit) {4873  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;4874  OpenMPIRBuilder OMPBuilder(*M);4875  OMPBuilder.Config.IsTargetDevice = false;4876  OMPBuilder.initialize();4877  F->setName("func");4878  IRBuilder<> &Builder = OMPBuilder.Builder;4879  Builder.SetInsertPoint(BB);4880 4881  BasicBlock *CodegenBB = splitBB(Builder, true);4882  Builder.SetInsertPoint(CodegenBB);4883 4884  // Generate values for `num_teams` and `thread_limit` using the first argument4885  // of the testing function.4886  Value *NumTeamsLower =4887      Builder.CreateAdd(F->arg_begin(), Builder.getInt32(5), "numTeamsLower");4888  Value *NumTeamsUpper =4889      Builder.CreateAdd(F->arg_begin(), Builder.getInt32(10), "numTeamsUpper");4890  Value *ThreadLimit =4891      Builder.CreateAdd(F->arg_begin(), Builder.getInt32(20), "threadLimit");4892 4893  Function *FakeFunction =4894      Function::Create(FunctionType::get(Builder.getVoidTy(), false),4895                       GlobalValue::ExternalLinkage, "fakeFunction", M.get());4896 4897  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {4898    Builder.restoreIP(CodeGenIP);4899    Builder.CreateCall(FakeFunction, {});4900    return Error::success();4901  };4902 4903  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});4904  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,4905                       OMPBuilder.createTeams(Builder, BodyGenCB, NumTeamsLower,4906                                              NumTeamsUpper, ThreadLimit,4907                                              nullptr));4908  Builder.restoreIP(AfterIP);4909 4910  Builder.CreateRetVoid();4911  OMPBuilder.finalize();4912 4913  ASSERT_FALSE(verifyModule(*M));4914 4915  CallInst *PushNumTeamsCallInst =4916      findSingleCall(F, OMPRTL___kmpc_push_num_teams_51, OMPBuilder);4917  ASSERT_NE(PushNumTeamsCallInst, nullptr);4918 4919  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(2), NumTeamsLower);4920  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(3), NumTeamsUpper);4921  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(4), ThreadLimit);4922 4923  // Verifying that the next instruction to execute is kmpc_fork_teams4924  BranchInst *BrInst =4925      dyn_cast<BranchInst>(PushNumTeamsCallInst->getNextNode());4926  ASSERT_NE(BrInst, nullptr);4927  ASSERT_EQ(BrInst->getNumSuccessors(), 1U);4928  BasicBlock::iterator NextInstruction =4929      BrInst->getSuccessor(0)->getFirstNonPHIOrDbgOrLifetime();4930  CallInst *ForkTeamsCI = nullptr;4931  if (NextInstruction != BrInst->getSuccessor(0)->end())4932    ForkTeamsCI = dyn_cast_if_present<CallInst>(NextInstruction);4933  ASSERT_NE(ForkTeamsCI, nullptr);4934  EXPECT_EQ(ForkTeamsCI->getCalledFunction(),4935            OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams));4936}4937 4938TEST_F(OpenMPIRBuilderTest, CreateTeamsWithIfCondition) {4939  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;4940  OpenMPIRBuilder OMPBuilder(*M);4941  OMPBuilder.Config.IsTargetDevice = false;4942  OMPBuilder.initialize();4943  F->setName("func");4944  IRBuilder<> &Builder = OMPBuilder.Builder;4945  Builder.SetInsertPoint(BB);4946 4947  Value *IfExpr = Builder.CreateLoad(Builder.getInt1Ty(),4948                                     Builder.CreateAlloca(Builder.getInt1Ty()));4949 4950  Function *FakeFunction =4951      Function::Create(FunctionType::get(Builder.getVoidTy(), false),4952                       GlobalValue::ExternalLinkage, "fakeFunction", M.get());4953 4954  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {4955    Builder.restoreIP(CodeGenIP);4956    Builder.CreateCall(FakeFunction, {});4957    return Error::success();4958  };4959 4960  // `F` already has an integer argument, so we use that as upper bound to4961  // `num_teams`4962  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,4963                       OMPBuilder.createTeams(Builder, BodyGenCB,4964                                              /*NumTeamsLower=*/nullptr,4965                                              /*NumTeamsUpper=*/nullptr,4966                                              /*ThreadLimit=*/nullptr, IfExpr));4967  Builder.restoreIP(AfterIP);4968 4969  Builder.CreateRetVoid();4970  OMPBuilder.finalize();4971 4972  ASSERT_FALSE(verifyModule(*M));4973 4974  CallInst *PushNumTeamsCallInst =4975      findSingleCall(F, OMPRTL___kmpc_push_num_teams_51, OMPBuilder);4976  ASSERT_NE(PushNumTeamsCallInst, nullptr);4977  Value *NumTeamsLower = PushNumTeamsCallInst->getArgOperand(2);4978  Value *NumTeamsUpper = PushNumTeamsCallInst->getArgOperand(3);4979  Value *ThreadLimit = PushNumTeamsCallInst->getArgOperand(4);4980 4981  // Check the lower_bound4982  ASSERT_NE(NumTeamsLower, nullptr);4983  SelectInst *NumTeamsLowerSelectInst = dyn_cast<SelectInst>(NumTeamsLower);4984  ASSERT_NE(NumTeamsLowerSelectInst, nullptr);4985  EXPECT_EQ(NumTeamsLowerSelectInst->getCondition(), IfExpr);4986  EXPECT_EQ(NumTeamsLowerSelectInst->getTrueValue(), Builder.getInt32(0));4987  EXPECT_EQ(NumTeamsLowerSelectInst->getFalseValue(), Builder.getInt32(1));4988 4989  // Check the upper_bound4990  ASSERT_NE(NumTeamsUpper, nullptr);4991  SelectInst *NumTeamsUpperSelectInst = dyn_cast<SelectInst>(NumTeamsUpper);4992  ASSERT_NE(NumTeamsUpperSelectInst, nullptr);4993  EXPECT_EQ(NumTeamsUpperSelectInst->getCondition(), IfExpr);4994  EXPECT_EQ(NumTeamsUpperSelectInst->getTrueValue(), Builder.getInt32(0));4995  EXPECT_EQ(NumTeamsUpperSelectInst->getFalseValue(), Builder.getInt32(1));4996 4997  // Check thread_limit4998  EXPECT_EQ(ThreadLimit, Builder.getInt32(0));4999}5000 5001TEST_F(OpenMPIRBuilderTest, CreateTeamsWithIfConditionAndNumTeams) {5002  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;5003  OpenMPIRBuilder OMPBuilder(*M);5004  OMPBuilder.Config.IsTargetDevice = false;5005  OMPBuilder.initialize();5006  F->setName("func");5007  IRBuilder<> &Builder = OMPBuilder.Builder;5008  Builder.SetInsertPoint(BB);5009 5010  Value *IfExpr = Builder.CreateLoad(5011      Builder.getInt32Ty(), Builder.CreateAlloca(Builder.getInt32Ty()));5012  Value *NumTeamsLower = Builder.CreateAdd(F->arg_begin(), Builder.getInt32(5));5013  Value *NumTeamsUpper =5014      Builder.CreateAdd(F->arg_begin(), Builder.getInt32(10));5015  Value *ThreadLimit = Builder.CreateAdd(F->arg_begin(), Builder.getInt32(20));5016 5017  Function *FakeFunction =5018      Function::Create(FunctionType::get(Builder.getVoidTy(), false),5019                       GlobalValue::ExternalLinkage, "fakeFunction", M.get());5020 5021  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {5022    Builder.restoreIP(CodeGenIP);5023    Builder.CreateCall(FakeFunction, {});5024    return Error::success();5025  };5026 5027  // `F` already has an integer argument, so we use that as upper bound to5028  // `num_teams`5029  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,5030                       OMPBuilder.createTeams(Builder, BodyGenCB, NumTeamsLower,5031                                              NumTeamsUpper, ThreadLimit,5032                                              IfExpr));5033  Builder.restoreIP(AfterIP);5034 5035  Builder.CreateRetVoid();5036  OMPBuilder.finalize();5037 5038  ASSERT_FALSE(verifyModule(*M));5039 5040  CallInst *PushNumTeamsCallInst =5041      findSingleCall(F, OMPRTL___kmpc_push_num_teams_51, OMPBuilder);5042  ASSERT_NE(PushNumTeamsCallInst, nullptr);5043  Value *NumTeamsLowerArg = PushNumTeamsCallInst->getArgOperand(2);5044  Value *NumTeamsUpperArg = PushNumTeamsCallInst->getArgOperand(3);5045  Value *ThreadLimitArg = PushNumTeamsCallInst->getArgOperand(4);5046 5047  // Get the boolean conversion of if expression5048  ASSERT_TRUE(IfExpr->hasOneUse());5049  User *IfExprInst = IfExpr->user_back();5050  ICmpInst *IfExprCmpInst = dyn_cast<ICmpInst>(IfExprInst);5051  ASSERT_NE(IfExprCmpInst, nullptr);5052  EXPECT_EQ(IfExprCmpInst->getPredicate(), ICmpInst::Predicate::ICMP_NE);5053  EXPECT_EQ(IfExprCmpInst->getOperand(0), IfExpr);5054  EXPECT_EQ(IfExprCmpInst->getOperand(1), Builder.getInt32(0));5055 5056  // Check the lower_bound5057  ASSERT_NE(NumTeamsLowerArg, nullptr);5058  SelectInst *NumTeamsLowerSelectInst = dyn_cast<SelectInst>(NumTeamsLowerArg);5059  ASSERT_NE(NumTeamsLowerSelectInst, nullptr);5060  EXPECT_EQ(NumTeamsLowerSelectInst->getCondition(), IfExprCmpInst);5061  EXPECT_EQ(NumTeamsLowerSelectInst->getTrueValue(), NumTeamsLower);5062  EXPECT_EQ(NumTeamsLowerSelectInst->getFalseValue(), Builder.getInt32(1));5063 5064  // Check the upper_bound5065  ASSERT_NE(NumTeamsUpperArg, nullptr);5066  SelectInst *NumTeamsUpperSelectInst = dyn_cast<SelectInst>(NumTeamsUpperArg);5067  ASSERT_NE(NumTeamsUpperSelectInst, nullptr);5068  EXPECT_EQ(NumTeamsUpperSelectInst->getCondition(), IfExprCmpInst);5069  EXPECT_EQ(NumTeamsUpperSelectInst->getTrueValue(), NumTeamsUpper);5070  EXPECT_EQ(NumTeamsUpperSelectInst->getFalseValue(), Builder.getInt32(1));5071 5072  // Check thread_limit5073  EXPECT_EQ(ThreadLimitArg, ThreadLimit);5074}5075 5076/// Returns the single instruction of InstTy type in BB that uses the value V.5077/// If there is more than one such instruction, returns null.5078template <typename InstTy>5079static InstTy *findSingleUserInBlock(Value *V, BasicBlock *BB) {5080  InstTy *Result = nullptr;5081  for (User *U : V->users()) {5082    auto *Inst = dyn_cast<InstTy>(U);5083    if (!Inst || Inst->getParent() != BB)5084      continue;5085    if (Result) {5086      if (auto *SI = dyn_cast<StoreInst>(Inst)) {5087        if (V == SI->getValueOperand())5088          continue;5089      } else {5090        return nullptr;5091      }5092    }5093    Result = Inst;5094  }5095  return Result;5096}5097 5098/// Returns true if BB contains a simple binary reduction that loads a value5099/// from Accum, performs some binary operation with it, and stores it back to5100/// Accum.5101static bool isSimpleBinaryReduction(Value *Accum, BasicBlock *BB,5102                                    Instruction::BinaryOps *OpCode = nullptr) {5103  StoreInst *Store = findSingleUserInBlock<StoreInst>(Accum, BB);5104  if (!Store)5105    return false;5106  auto *Stored = dyn_cast<BinaryOperator>(Store->getOperand(0));5107  if (!Stored)5108    return false;5109  if (OpCode && *OpCode != Stored->getOpcode())5110    return false;5111  auto *Load = dyn_cast<LoadInst>(Stored->getOperand(0));5112  return Load && Load->getOperand(0) == Accum;5113}5114 5115/// Returns true if BB contains a binary reduction that reduces V using a binary5116/// operator into an accumulator that is a function argument.5117static bool isValueReducedToFuncArg(Value *V, BasicBlock *BB) {5118  auto *ReductionOp = findSingleUserInBlock<BinaryOperator>(V, BB);5119  if (!ReductionOp)5120    return false;5121 5122  auto *GlobalLoad = dyn_cast<LoadInst>(ReductionOp->getOperand(0));5123  if (!GlobalLoad)5124    return false;5125 5126  auto *Store = findSingleUserInBlock<StoreInst>(ReductionOp, BB);5127  if (!Store)5128    return false;5129 5130  return Store->getPointerOperand() == GlobalLoad->getPointerOperand() &&5131         isa<Argument>(findAggregateFromValue(GlobalLoad->getPointerOperand()));5132}5133 5134/// Finds among users of Ptr a pair of GEP instructions with indices [0, 0] and5135/// [0, 1], respectively, and assigns results of these instructions to Zero and5136/// One. Returns true on success, false on failure or if such instructions are5137/// not unique among the users of Ptr.5138static bool findGEPZeroOne(Value *Ptr, Value *&Zero, Value *&One) {5139  Zero = nullptr;5140  One = nullptr;5141  for (User *U : Ptr->users()) {5142    if (auto *GEP = dyn_cast<GetElementPtrInst>(U)) {5143      if (GEP->getNumIndices() != 2)5144        continue;5145      auto *FirstIdx = dyn_cast<ConstantInt>(GEP->getOperand(1));5146      auto *SecondIdx = dyn_cast<ConstantInt>(GEP->getOperand(2));5147      EXPECT_NE(FirstIdx, nullptr);5148      EXPECT_NE(SecondIdx, nullptr);5149 5150      EXPECT_TRUE(FirstIdx->isZero());5151      if (SecondIdx->isZero()) {5152        if (Zero)5153          return false;5154        Zero = GEP;5155      } else if (SecondIdx->isOne()) {5156        if (One)5157          return false;5158        One = GEP;5159      } else {5160        return false;5161      }5162    }5163  }5164  return Zero != nullptr && One != nullptr;5165}5166 5167static OpenMPIRBuilder::InsertPointTy5168sumReduction(OpenMPIRBuilder::InsertPointTy IP, Value *LHS, Value *RHS,5169             Value *&Result) {5170  IRBuilder<> Builder(IP.getBlock(), IP.getPoint());5171  Result = Builder.CreateFAdd(LHS, RHS, "red.add");5172  return Builder.saveIP();5173}5174 5175static OpenMPIRBuilder::InsertPointTy5176sumAtomicReduction(OpenMPIRBuilder::InsertPointTy IP, Type *Ty, Value *LHS,5177                   Value *RHS) {5178  IRBuilder<> Builder(IP.getBlock(), IP.getPoint());5179  Value *Partial = Builder.CreateLoad(Ty, RHS, "red.partial");5180  Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, LHS, Partial, std::nullopt,5181                          AtomicOrdering::Monotonic);5182  return Builder.saveIP();5183}5184 5185static OpenMPIRBuilder::InsertPointTy5186xorReduction(OpenMPIRBuilder::InsertPointTy IP, Value *LHS, Value *RHS,5187             Value *&Result) {5188  IRBuilder<> Builder(IP.getBlock(), IP.getPoint());5189  Result = Builder.CreateXor(LHS, RHS, "red.xor");5190  return Builder.saveIP();5191}5192 5193static OpenMPIRBuilder::InsertPointTy5194xorAtomicReduction(OpenMPIRBuilder::InsertPointTy IP, Type *Ty, Value *LHS,5195                   Value *RHS) {5196  IRBuilder<> Builder(IP.getBlock(), IP.getPoint());5197  Value *Partial = Builder.CreateLoad(Ty, RHS, "red.partial");5198  Builder.CreateAtomicRMW(AtomicRMWInst::Xor, LHS, Partial, std::nullopt,5199                          AtomicOrdering::Monotonic);5200  return Builder.saveIP();5201}5202 5203TEST_F(OpenMPIRBuilderTest, CreateReductions) {5204  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;5205  OpenMPIRBuilder OMPBuilder(*M);5206  OMPBuilder.Config.IsTargetDevice = false;5207  OMPBuilder.initialize();5208  F->setName("func");5209  IRBuilder<> Builder(BB);5210 5211  BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);5212  Builder.CreateBr(EnterBB);5213  Builder.SetInsertPoint(EnterBB);5214  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});5215 5216  // Create variables to be reduced.5217  InsertPointTy OuterAllocaIP(&F->getEntryBlock(),5218                              F->getEntryBlock().getFirstInsertionPt());5219  Type *SumType = Builder.getFloatTy();5220  Type *XorType = Builder.getInt32Ty();5221  Value *SumReduced;5222  Value *XorReduced;5223  {5224    IRBuilderBase::InsertPointGuard Guard(Builder);5225    Builder.restoreIP(OuterAllocaIP);5226    SumReduced = Builder.CreateAlloca(SumType);5227    XorReduced = Builder.CreateAlloca(XorType);5228  }5229 5230  // Store initial values of reductions into global variables.5231  Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0), SumReduced);5232  Builder.CreateStore(Builder.getInt32(1), XorReduced);5233 5234  // The loop body computes two reductions:5235  //   sum of (float) thread-id;5236  //   xor of thread-id;5237  // and store the result in global variables.5238  InsertPointTy BodyIP, BodyAllocaIP;5239  auto BodyGenCB = [&](InsertPointTy InnerAllocaIP, InsertPointTy CodeGenIP) {5240    IRBuilderBase::InsertPointGuard Guard(Builder);5241    Builder.restoreIP(CodeGenIP);5242 5243    uint32_t StrSize;5244    Constant *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(Loc, StrSize);5245    Value *Ident = OMPBuilder.getOrCreateIdent(SrcLocStr, StrSize);5246    Value *TID = OMPBuilder.getOrCreateThreadID(Ident);5247    Value *SumLocal =5248        Builder.CreateUIToFP(TID, Builder.getFloatTy(), "sum.local");5249    Value *SumPartial = Builder.CreateLoad(SumType, SumReduced, "sum.partial");5250    Value *XorPartial = Builder.CreateLoad(XorType, XorReduced, "xor.partial");5251    Value *Sum = Builder.CreateFAdd(SumPartial, SumLocal, "sum");5252    Value *Xor = Builder.CreateXor(XorPartial, TID, "xor");5253    Builder.CreateStore(Sum, SumReduced);5254    Builder.CreateStore(Xor, XorReduced);5255 5256    BodyIP = Builder.saveIP();5257    BodyAllocaIP = InnerAllocaIP;5258    return Error::success();5259  };5260 5261  // Privatization for reduction creates local copies of reduction variables and5262  // initializes them to reduction-neutral values.5263  Value *SumPrivatized;5264  Value *XorPrivatized;5265  auto PrivCB = [&](InsertPointTy InnerAllocaIP, InsertPointTy CodeGenIP,5266                    Value &Original, Value &Inner, Value *&ReplVal) {5267    IRBuilderBase::InsertPointGuard Guard(Builder);5268    Builder.restoreIP(InnerAllocaIP);5269    if (&Original == SumReduced) {5270      SumPrivatized = Builder.CreateAlloca(Builder.getFloatTy());5271      ReplVal = SumPrivatized;5272    } else if (&Original == XorReduced) {5273      XorPrivatized = Builder.CreateAlloca(Builder.getInt32Ty());5274      ReplVal = XorPrivatized;5275    } else {5276      ReplVal = &Inner;5277      return CodeGenIP;5278    }5279 5280    Builder.restoreIP(CodeGenIP);5281    if (&Original == SumReduced)5282      Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0),5283                          SumPrivatized);5284    else if (&Original == XorReduced)5285      Builder.CreateStore(Builder.getInt32(0), XorPrivatized);5286 5287    return Builder.saveIP();5288  };5289 5290  // Do nothing in finalization.5291  auto FiniCB = [&](InsertPointTy CodeGenIP) { return Error::success(); };5292 5293  ASSERT_EXPECTED_INIT(5294      OpenMPIRBuilder::InsertPointTy, AfterIP,5295      OMPBuilder.createParallel(Loc, OuterAllocaIP, BodyGenCB, PrivCB, FiniCB,5296                                /* IfCondition */ nullptr,5297                                /* NumThreads */ nullptr, OMP_PROC_BIND_default,5298                                /* IsCancellable */ false));5299  Builder.restoreIP(AfterIP);5300 5301  OpenMPIRBuilder::ReductionInfo ReductionInfos[] = {5302      {SumType, SumReduced, SumPrivatized,5303       /*EvaluationKind=*/OpenMPIRBuilder::EvalKind::Scalar, sumReduction,5304       /*ReductionGenClang=*/nullptr, sumAtomicReduction,5305       /*DataPtrPtrGen=*/nullptr},5306      {XorType, XorReduced, XorPrivatized,5307       /*EvaluationKind=*/OpenMPIRBuilder::EvalKind::Scalar, xorReduction,5308       /*ReductionGenClang=*/nullptr, xorAtomicReduction,5309       /*DataPtrPtrGen=*/nullptr}};5310  OMPBuilder.Config.setIsGPU(false);5311 5312  bool ReduceVariableByRef[] = {false, false};5313  ASSERT_THAT_EXPECTED(OMPBuilder.createReductions(BodyIP, BodyAllocaIP,5314                                                   ReductionInfos,5315                                                   ReduceVariableByRef),5316                       Succeeded());5317 5318  Builder.restoreIP(AfterIP);5319  Builder.CreateRetVoid();5320 5321  OMPBuilder.finalize(F);5322 5323  // The IR must be valid.5324  EXPECT_FALSE(verifyModule(*M));5325 5326  // Outlining must have happened.5327  SmallVector<CallInst *> ForkCalls;5328  findCalls(F, omp::RuntimeFunction::OMPRTL___kmpc_fork_call, OMPBuilder,5329            ForkCalls);5330  ASSERT_EQ(ForkCalls.size(), 1u);5331  Value *CalleeVal = ForkCalls[0]->getOperand(2);5332  Function *Outlined = dyn_cast<Function>(CalleeVal);5333  EXPECT_NE(Outlined, nullptr);5334 5335  // Check that the lock variable was created with the expected name.5336  GlobalVariable *LockVar =5337      M->getGlobalVariable(".gomp_critical_user_.reduction.var");5338  EXPECT_NE(LockVar, nullptr);5339 5340  // Find the allocation of a local array that will be used to call the runtime5341  // reduciton function.5342  BasicBlock &AllocBlock = Outlined->getEntryBlock();5343  Value *LocalArray = nullptr;5344  for (Instruction &I : AllocBlock) {5345    if (AllocaInst *Alloc = dyn_cast<AllocaInst>(&I)) {5346      if (!Alloc->getAllocatedType()->isArrayTy() ||5347          !Alloc->getAllocatedType()->getArrayElementType()->isPointerTy())5348        continue;5349      LocalArray = Alloc;5350      break;5351    }5352  }5353  ASSERT_NE(LocalArray, nullptr);5354 5355  // Find the call to the runtime reduction function.5356  BasicBlock *BB = AllocBlock.getUniqueSuccessor();5357  Value *LocalArrayPtr = nullptr;5358  Value *ReductionFnVal = nullptr;5359  Value *SwitchArg = nullptr;5360  for (Instruction &I : *BB) {5361    if (CallInst *Call = dyn_cast<CallInst>(&I)) {5362      if (Call->getCalledFunction() !=5363          OMPBuilder.getOrCreateRuntimeFunctionPtr(5364              RuntimeFunction::OMPRTL___kmpc_reduce))5365        continue;5366      LocalArrayPtr = Call->getOperand(4);5367      ReductionFnVal = Call->getOperand(5);5368      SwitchArg = Call;5369      break;5370    }5371  }5372 5373  // Check that the local array is passed to the function.5374  ASSERT_NE(LocalArrayPtr, nullptr);5375  EXPECT_EQ(LocalArrayPtr, LocalArray);5376 5377  // Find the GEP instructions preceding stores to the local array.5378  Value *FirstArrayElemPtr = nullptr;5379  Value *SecondArrayElemPtr = nullptr;5380  EXPECT_TRUE(LocalArray->hasNUses(3));5381  ASSERT_TRUE(5382      findGEPZeroOne(LocalArray, FirstArrayElemPtr, SecondArrayElemPtr));5383 5384  // Check that the values stored into the local array are privatized reduction5385  // variables.5386  auto *FirstPrivatized = dyn_cast_or_null<AllocaInst>(5387      findStoredValue<GetElementPtrInst>(FirstArrayElemPtr));5388  auto *SecondPrivatized = dyn_cast_or_null<AllocaInst>(5389      findStoredValue<GetElementPtrInst>(SecondArrayElemPtr));5390  ASSERT_NE(FirstPrivatized, nullptr);5391  ASSERT_NE(SecondPrivatized, nullptr);5392  ASSERT_TRUE(isa<Instruction>(FirstArrayElemPtr));5393  EXPECT_TRUE(isSimpleBinaryReduction(5394      FirstPrivatized, cast<Instruction>(FirstArrayElemPtr)->getParent()));5395  EXPECT_TRUE(isSimpleBinaryReduction(5396      SecondPrivatized, cast<Instruction>(FirstArrayElemPtr)->getParent()));5397 5398  // Check that the result of the runtime reduction call is used for further5399  // dispatch.5400  ASSERT_TRUE(SwitchArg->hasOneUse());5401  SwitchInst *Switch = dyn_cast<SwitchInst>(*SwitchArg->user_begin());5402  ASSERT_NE(Switch, nullptr);5403  EXPECT_EQ(Switch->getNumSuccessors(), 3u);5404  BasicBlock *NonAtomicBB = Switch->case_begin()->getCaseSuccessor();5405  BasicBlock *AtomicBB = std::next(Switch->case_begin())->getCaseSuccessor();5406 5407  // Non-atomic block contains reductions to the global reduction variable,5408  // which is passed into the outlined function as an argument.5409  Value *FirstLoad =5410      findSingleUserInBlock<LoadInst>(FirstPrivatized, NonAtomicBB);5411  Value *SecondLoad =5412      findSingleUserInBlock<LoadInst>(SecondPrivatized, NonAtomicBB);5413  EXPECT_TRUE(isValueReducedToFuncArg(FirstLoad, NonAtomicBB));5414  EXPECT_TRUE(isValueReducedToFuncArg(SecondLoad, NonAtomicBB));5415 5416  // Atomic block also constains reductions to the global reduction variable.5417  FirstLoad = findSingleUserInBlock<LoadInst>(FirstPrivatized, AtomicBB);5418  SecondLoad = findSingleUserInBlock<LoadInst>(SecondPrivatized, AtomicBB);5419  auto *FirstAtomic = findSingleUserInBlock<AtomicRMWInst>(FirstLoad, AtomicBB);5420  auto *SecondAtomic =5421      findSingleUserInBlock<AtomicRMWInst>(SecondLoad, AtomicBB);5422  ASSERT_NE(FirstAtomic, nullptr);5423  Value *AtomicStorePointer = FirstAtomic->getPointerOperand();5424  EXPECT_TRUE(isa<Argument>(findAggregateFromValue(AtomicStorePointer)));5425  ASSERT_NE(SecondAtomic, nullptr);5426  AtomicStorePointer = SecondAtomic->getPointerOperand();5427  EXPECT_TRUE(isa<Argument>(findAggregateFromValue(AtomicStorePointer)));5428 5429  // Check that the separate reduction function also performs (non-atomic)5430  // reductions after extracting reduction variables from its arguments.5431  Function *ReductionFn = cast<Function>(ReductionFnVal);5432  BasicBlock *FnReductionBB = &ReductionFn->getEntryBlock();5433  Value *FirstLHSPtr;5434  Value *SecondLHSPtr;5435  ASSERT_TRUE(5436      findGEPZeroOne(ReductionFn->getArg(0), FirstLHSPtr, SecondLHSPtr));5437  Value *Opaque = findSingleUserInBlock<LoadInst>(FirstLHSPtr, FnReductionBB);5438  ASSERT_NE(Opaque, nullptr);5439  EXPECT_TRUE(isSimpleBinaryReduction(Opaque, FnReductionBB));5440  Opaque = findSingleUserInBlock<LoadInst>(SecondLHSPtr, FnReductionBB);5441  ASSERT_NE(Opaque, nullptr);5442  EXPECT_TRUE(isSimpleBinaryReduction(Opaque, FnReductionBB));5443 5444  Value *FirstRHS;5445  Value *SecondRHS;5446  EXPECT_TRUE(findGEPZeroOne(ReductionFn->getArg(1), FirstRHS, SecondRHS));5447}5448 5449static void createScan(llvm::Value *scanVar, llvm::Type *scanType,5450                       OpenMPIRBuilder &OMPBuilder, IRBuilder<> &Builder,5451                       OpenMPIRBuilder::LocationDescription Loc,5452                       OpenMPIRBuilder::InsertPointTy &allocaIP,5453                       ScanInfo *&ScanRedInfo) {5454  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;5455  ASSERT_EXPECTED_INIT(InsertPointTy, retIp,5456                       OMPBuilder.createScan(Loc, allocaIP, {scanVar},5457                                             {scanType}, true, ScanRedInfo));5458  Builder.restoreIP(retIp);5459}5460/*5461 Following is the pseudocode of the code generated by the test case5462 <declare pointer to buffer> ptr5463  size num_iters = 1005464  // temp buffer allocation5465  omp masked {5466    buff = malloc(num_iters*scanvarstype)5467    *ptr = buff5468  }5469 barrier;5470  // input phase loop5471  for (i: 0..<num_iters>) {5472    <input phase>;5473    buffer = *ptr;5474    buffer[i] = red;5475  }5476  // scan reduction5477  omp masked5478  {5479    for (int k = 0; k != ceil(log2(num_iters)); ++k) {5480      i=pow(2,k)5481      for (size cnt = last_iter; cnt >= i; --cnt) {5482        buffer = *ptr;5483        buffer[cnt] op= buffer[cnt-i];5484      }5485    }5486  }5487 barrier;5488 // scan phase loop5489  for (0..<num_iters>) {5490    buffer = *ptr;5491    red = buffer[i] ;5492    <scan phase>;5493  }5494  // temp buffer deletion5495  omp masked {5496    free(*ptr)5497  }5498  barrier;5499*/5500TEST_F(OpenMPIRBuilderTest, ScanReduction) {5501  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;5502  OpenMPIRBuilder OMPBuilder(*M);5503  OMPBuilder.initialize();5504  IRBuilder<> Builder(BB);5505  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});5506  Value *TripCount = F->getArg(0);5507  Type *LCTy = TripCount->getType();5508  Value *StartVal = ConstantInt::get(LCTy, 1);5509  Value *StopVal = ConstantInt::get(LCTy, 100);5510  Value *Step = ConstantInt::get(LCTy, 1);5511  auto AllocaIP = Builder.saveIP();5512 5513  llvm::Value *ScanVar = Builder.CreateAlloca(Builder.getFloatTy());5514  llvm::Value *OrigVar = Builder.CreateAlloca(Builder.getFloatTy());5515  unsigned NumBodiesGenerated = 0;5516  ScanInfo *ScanRedInfo;5517  ASSERT_EXPECTED_INIT(ScanInfo *, ScanInformation,5518                       OMPBuilder.scanInfoInitialize());5519  ScanRedInfo = ScanInformation;5520  auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {5521    NumBodiesGenerated += 1;5522    Builder.restoreIP(CodeGenIP);5523    createScan(ScanVar, Builder.getFloatTy(), OMPBuilder, Builder, Loc,5524               AllocaIP, ScanRedInfo);5525    return Error::success();5526  };5527  llvm::SmallVector<CanonicalLoopInfo *> loops;5528  ASSERT_EXPECTED_INIT(llvm::SmallVector<CanonicalLoopInfo *>, loopvec,5529                       OMPBuilder.createCanonicalScanLoops(5530                           Loc, LoopBodyGenCB, StartVal, StopVal, Step, false,5531                           false, Builder.saveIP(), "scan", ScanRedInfo));5532  loops = loopvec;5533  CanonicalLoopInfo *InputLoop = loops.front();5534  CanonicalLoopInfo *ScanLoop = loops.back();5535  Builder.restoreIP(ScanLoop->getAfterIP());5536  InputLoop->assertOK();5537  ScanLoop->assertOK();5538 5539  EXPECT_EQ(ScanLoop->getAfter(), Builder.GetInsertBlock());5540  EXPECT_EQ(NumBodiesGenerated, 2U);5541  SmallVector<OpenMPIRBuilder::ReductionInfo, 2> ReductionInfos = {5542      {Builder.getFloatTy(), OrigVar, ScanVar,5543       /*EvaluationKind=*/OpenMPIRBuilder::EvalKind::Scalar, sumReduction,5544       /*ReductionGenClang=*/nullptr, sumAtomicReduction,5545       /*DataPtrPtrGen=*/nullptr}};5546  OpenMPIRBuilder::LocationDescription RedLoc({InputLoop->getAfterIP(), DL});5547  llvm::BasicBlock *Cont = splitBB(Builder, false, "omp.scan.loop.cont");5548  ASSERT_EXPECTED_INIT(5549      InsertPointTy, retIp,5550      OMPBuilder.emitScanReduction(RedLoc, ReductionInfos, ScanRedInfo));5551  Builder.restoreIP(retIp);5552  Builder.CreateBr(Cont);5553  Builder.SetInsertPoint(Cont);5554  unsigned NumMallocs = 0;5555  unsigned NumFrees = 0;5556  unsigned NumMasked = 0;5557  unsigned NumEndMasked = 0;5558  unsigned NumLog = 0;5559  unsigned NumCeil = 0;5560  for (Instruction &I : instructions(F)) {5561    if (!isa<CallInst>(I))5562      continue;5563    CallInst *Call = dyn_cast<CallInst>(&I);5564    StringRef Name = Call->getCalledFunction()->getName();5565    if (Name.equals_insensitive("malloc")) {5566      NumMallocs += 1;5567    } else if (Name.equals_insensitive("free")) {5568      NumFrees += 1;5569    } else if (Name.equals_insensitive("__kmpc_masked")) {5570      NumMasked += 1;5571    } else if (Name.equals_insensitive("__kmpc_end_masked")) {5572      NumEndMasked += 1;5573    } else if (Name.equals_insensitive("llvm.log2.f64")) {5574      NumLog += 1;5575    } else if (Name.equals_insensitive("llvm.ceil.f64")) {5576      NumCeil += 1;5577    }5578  }5579  EXPECT_EQ(NumBodiesGenerated, 2U);5580  EXPECT_EQ(NumMasked, 3U);5581  EXPECT_EQ(NumEndMasked, 3U);5582  EXPECT_EQ(NumMallocs, 1U);5583  EXPECT_EQ(NumFrees, 1U);5584  EXPECT_EQ(NumLog, 1U);5585  EXPECT_EQ(NumCeil, 1U);5586}5587 5588TEST_F(OpenMPIRBuilderTest, CreateTwoReductions) {5589  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;5590  OpenMPIRBuilder OMPBuilder(*M);5591  OMPBuilder.Config.IsTargetDevice = false;5592  OMPBuilder.initialize();5593  F->setName("func");5594  IRBuilder<> Builder(BB);5595 5596  BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);5597  Builder.CreateBr(EnterBB);5598  Builder.SetInsertPoint(EnterBB);5599  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});5600 5601  // Create variables to be reduced.5602  InsertPointTy OuterAllocaIP(&F->getEntryBlock(),5603                              F->getEntryBlock().getFirstInsertionPt());5604  Type *SumType = Builder.getFloatTy();5605  Type *XorType = Builder.getInt32Ty();5606  Value *SumReduced;5607  Value *XorReduced;5608  {5609    IRBuilderBase::InsertPointGuard Guard(Builder);5610    Builder.restoreIP(OuterAllocaIP);5611    SumReduced = Builder.CreateAlloca(SumType);5612    XorReduced = Builder.CreateAlloca(XorType);5613  }5614 5615  // Store initial values of reductions into global variables.5616  Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0), SumReduced);5617  Builder.CreateStore(Builder.getInt32(1), XorReduced);5618 5619  InsertPointTy FirstBodyIP, FirstBodyAllocaIP;5620  auto FirstBodyGenCB = [&](InsertPointTy InnerAllocaIP,5621                            InsertPointTy CodeGenIP) {5622    IRBuilderBase::InsertPointGuard Guard(Builder);5623    Builder.restoreIP(CodeGenIP);5624 5625    uint32_t StrSize;5626    Constant *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(Loc, StrSize);5627    Value *Ident = OMPBuilder.getOrCreateIdent(SrcLocStr, StrSize);5628    Value *TID = OMPBuilder.getOrCreateThreadID(Ident);5629    Value *SumLocal =5630        Builder.CreateUIToFP(TID, Builder.getFloatTy(), "sum.local");5631    Value *SumPartial = Builder.CreateLoad(SumType, SumReduced, "sum.partial");5632    Value *Sum = Builder.CreateFAdd(SumPartial, SumLocal, "sum");5633    Builder.CreateStore(Sum, SumReduced);5634 5635    FirstBodyIP = Builder.saveIP();5636    FirstBodyAllocaIP = InnerAllocaIP;5637    return Error::success();5638  };5639 5640  InsertPointTy SecondBodyIP, SecondBodyAllocaIP;5641  auto SecondBodyGenCB = [&](InsertPointTy InnerAllocaIP,5642                             InsertPointTy CodeGenIP) {5643    IRBuilderBase::InsertPointGuard Guard(Builder);5644    Builder.restoreIP(CodeGenIP);5645 5646    uint32_t StrSize;5647    Constant *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(Loc, StrSize);5648    Value *Ident = OMPBuilder.getOrCreateIdent(SrcLocStr, StrSize);5649    Value *TID = OMPBuilder.getOrCreateThreadID(Ident);5650    Value *XorPartial = Builder.CreateLoad(XorType, XorReduced, "xor.partial");5651    Value *Xor = Builder.CreateXor(XorPartial, TID, "xor");5652    Builder.CreateStore(Xor, XorReduced);5653 5654    SecondBodyIP = Builder.saveIP();5655    SecondBodyAllocaIP = InnerAllocaIP;5656    return Error::success();5657  };5658 5659  // Privatization for reduction creates local copies of reduction variables and5660  // initializes them to reduction-neutral values. The same privatization5661  // callback is used for both loops, with dispatch based on the value being5662  // privatized.5663  Value *SumPrivatized;5664  Value *XorPrivatized;5665  auto PrivCB = [&](InsertPointTy InnerAllocaIP, InsertPointTy CodeGenIP,5666                    Value &Original, Value &Inner, Value *&ReplVal) {5667    IRBuilderBase::InsertPointGuard Guard(Builder);5668    Builder.restoreIP(InnerAllocaIP);5669    if (&Original == SumReduced) {5670      SumPrivatized = Builder.CreateAlloca(Builder.getFloatTy());5671      ReplVal = SumPrivatized;5672    } else if (&Original == XorReduced) {5673      XorPrivatized = Builder.CreateAlloca(Builder.getInt32Ty());5674      ReplVal = XorPrivatized;5675    } else {5676      ReplVal = &Inner;5677      return CodeGenIP;5678    }5679 5680    Builder.restoreIP(CodeGenIP);5681    if (&Original == SumReduced)5682      Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0),5683                          SumPrivatized);5684    else if (&Original == XorReduced)5685      Builder.CreateStore(Builder.getInt32(0), XorPrivatized);5686 5687    return Builder.saveIP();5688  };5689 5690  // Do nothing in finalization.5691  auto FiniCB = [&](InsertPointTy CodeGenIP) { return Error::success(); };5692 5693  ASSERT_EXPECTED_INIT(5694      OpenMPIRBuilder::InsertPointTy, AfterIP1,5695      OMPBuilder.createParallel(Loc, OuterAllocaIP, FirstBodyGenCB, PrivCB,5696                                FiniCB, /* IfCondition */ nullptr,5697                                /* NumThreads */ nullptr, OMP_PROC_BIND_default,5698                                /* IsCancellable */ false));5699  Builder.restoreIP(AfterIP1);5700  ASSERT_EXPECTED_INIT(5701      OpenMPIRBuilder::InsertPointTy, AfterIP2,5702      OMPBuilder.createParallel({Builder.saveIP(), DL}, OuterAllocaIP,5703                                SecondBodyGenCB, PrivCB, FiniCB,5704                                /* IfCondition */ nullptr,5705                                /* NumThreads */ nullptr, OMP_PROC_BIND_default,5706                                /* IsCancellable */ false));5707  Builder.restoreIP(AfterIP2);5708 5709  OMPBuilder.Config.setIsGPU(false);5710  bool ReduceVariableByRef[] = {false};5711 5712  ASSERT_THAT_EXPECTED(5713      OMPBuilder.createReductions(5714          FirstBodyIP, FirstBodyAllocaIP,5715          {{SumType, SumReduced, SumPrivatized,5716            /*EvaluationKind=*/OpenMPIRBuilder::EvalKind::Scalar, sumReduction,5717            /*ReductionGenClang=*/nullptr, sumAtomicReduction,5718            /*DataPtrPtrGen=*/nullptr}},5719          ReduceVariableByRef),5720      Succeeded());5721  ASSERT_THAT_EXPECTED(5722      OMPBuilder.createReductions(5723          SecondBodyIP, SecondBodyAllocaIP,5724          {{XorType, XorReduced, XorPrivatized,5725            /*EvaluationKind=*/OpenMPIRBuilder::EvalKind::Scalar, xorReduction,5726            /*ReductionGenClang=*/nullptr, xorAtomicReduction,5727            /*DataPtrPtrGen=*/nullptr}},5728          ReduceVariableByRef),5729      Succeeded());5730 5731  Builder.restoreIP(AfterIP2);5732  Builder.CreateRetVoid();5733 5734  OMPBuilder.finalize(F);5735 5736  // The IR must be valid.5737  EXPECT_FALSE(verifyModule(*M));5738 5739  // Two different outlined functions must have been created.5740  SmallVector<CallInst *> ForkCalls;5741  findCalls(F, omp::RuntimeFunction::OMPRTL___kmpc_fork_call, OMPBuilder,5742            ForkCalls);5743  ASSERT_EQ(ForkCalls.size(), 2u);5744  Value *CalleeVal = ForkCalls[0]->getOperand(2);5745  Function *FirstCallee = cast<Function>(CalleeVal);5746  CalleeVal = ForkCalls[1]->getOperand(2);5747  Function *SecondCallee = cast<Function>(CalleeVal);5748  EXPECT_NE(FirstCallee, SecondCallee);5749 5750  // Two different reduction functions must have been created.5751  SmallVector<CallInst *> ReduceCalls;5752  findCalls(FirstCallee, omp::RuntimeFunction::OMPRTL___kmpc_reduce, OMPBuilder,5753            ReduceCalls);5754  ASSERT_EQ(ReduceCalls.size(), 1u);5755  auto *AddReduction = cast<Function>(ReduceCalls[0]->getOperand(5));5756  ReduceCalls.clear();5757  findCalls(SecondCallee, omp::RuntimeFunction::OMPRTL___kmpc_reduce,5758            OMPBuilder, ReduceCalls);5759  auto *XorReduction = cast<Function>(ReduceCalls[0]->getOperand(5));5760  EXPECT_NE(AddReduction, XorReduction);5761 5762  // Each reduction function does its own kind of reduction.5763  BasicBlock *FnReductionBB = &AddReduction->getEntryBlock();5764  Value *FirstLHSPtr = findSingleUserInBlock<GetElementPtrInst>(5765      AddReduction->getArg(0), FnReductionBB);5766  ASSERT_NE(FirstLHSPtr, nullptr);5767  Value *Opaque = findSingleUserInBlock<LoadInst>(FirstLHSPtr, FnReductionBB);5768  ASSERT_NE(Opaque, nullptr);5769  Instruction::BinaryOps Opcode = Instruction::FAdd;5770  EXPECT_TRUE(isSimpleBinaryReduction(Opaque, FnReductionBB, &Opcode));5771 5772  FnReductionBB = &XorReduction->getEntryBlock();5773  Value *SecondLHSPtr = findSingleUserInBlock<GetElementPtrInst>(5774      XorReduction->getArg(0), FnReductionBB);5775  ASSERT_NE(FirstLHSPtr, nullptr);5776  Opaque = findSingleUserInBlock<LoadInst>(SecondLHSPtr, FnReductionBB);5777  ASSERT_NE(Opaque, nullptr);5778  Opcode = Instruction::Xor;5779  EXPECT_TRUE(isSimpleBinaryReduction(Opaque, FnReductionBB, &Opcode));5780}5781 5782TEST_F(OpenMPIRBuilderTest, CreateSectionsSimple) {5783  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;5784  using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;5785  OpenMPIRBuilder OMPBuilder(*M);5786  OMPBuilder.initialize();5787  F->setName("func");5788  IRBuilder<> Builder(BB);5789 5790  BasicBlock *EnterBB = BasicBlock::Create(Ctx, "sections.enter", F);5791  Builder.CreateBr(EnterBB);5792  Builder.SetInsertPoint(EnterBB);5793  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});5794 5795  llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;5796 5797  auto FiniCB = [&](InsertPointTy IP) { return Error::success(); };5798  auto SectionCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {5799    return Error::success();5800  };5801  SectionCBVector.push_back(SectionCB);5802 5803  auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,5804                   llvm::Value &, llvm::Value &Val,5805                   llvm::Value *&ReplVal) { return CodeGenIP; };5806  IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),5807                                    F->getEntryBlock().getFirstInsertionPt());5808  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,5809                       OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector,5810                                                 PrivCB, FiniCB, false, false));5811  Builder.restoreIP(AfterIP);5812  Builder.CreateRetVoid(); // Required at the end of the function5813  EXPECT_NE(F->getEntryBlock().getTerminator(), nullptr);5814  EXPECT_FALSE(verifyModule(*M, &errs()));5815}5816 5817TEST_F(OpenMPIRBuilderTest, CreateSections) {5818  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;5819  using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;5820  OpenMPIRBuilder OMPBuilder(*M);5821  OMPBuilder.initialize();5822  F->setName("func");5823  IRBuilder<> Builder(BB);5824 5825  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});5826  llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;5827  llvm::SmallVector<BasicBlock *, 4> CaseBBs;5828 5829  BasicBlock *SwitchBB = nullptr;5830  AllocaInst *PrivAI = nullptr;5831  SwitchInst *Switch = nullptr;5832 5833  unsigned NumBodiesGenerated = 0;5834  unsigned NumFiniCBCalls = 0;5835  PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());5836 5837  auto FiniCB = [&](InsertPointTy IP) {5838    ++NumFiniCBCalls;5839    BasicBlock *IPBB = IP.getBlock();5840    EXPECT_NE(IPBB->end(), IP.getPoint());5841  };5842 5843  auto SectionCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {5844    ++NumBodiesGenerated;5845    CaseBBs.push_back(CodeGenIP.getBlock());5846    SwitchBB = CodeGenIP.getBlock()->getSinglePredecessor();5847    Builder.restoreIP(CodeGenIP);5848    Builder.CreateStore(F->arg_begin(), PrivAI);5849    Value *PrivLoad =5850        Builder.CreateLoad(F->arg_begin()->getType(), PrivAI, "local.alloca");5851    Builder.CreateICmpNE(F->arg_begin(), PrivLoad);5852    return Error::success();5853  };5854  auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,5855                   llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {5856    // TODO: Privatization not implemented yet5857    return CodeGenIP;5858  };5859 5860  SectionCBVector.push_back(SectionCB);5861  SectionCBVector.push_back(SectionCB);5862 5863  IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),5864                                    F->getEntryBlock().getFirstInsertionPt());5865  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,5866                       OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector,5867                                                 PrivCB, FINICB_WRAPPER(FiniCB),5868                                                 false, false));5869  Builder.restoreIP(AfterIP);5870  Builder.CreateRetVoid(); // Required at the end of the function5871 5872  // Switch BB's predecessor is loop condition BB, whose successor at index 1 is5873  // loop's exit BB5874  BasicBlock *ForExitBB =5875      SwitchBB->getSinglePredecessor()->getTerminator()->getSuccessor(1);5876  EXPECT_NE(ForExitBB, nullptr);5877 5878  EXPECT_NE(PrivAI, nullptr);5879  Function *OutlinedFn = PrivAI->getFunction();5880  EXPECT_EQ(F, OutlinedFn);5881  EXPECT_FALSE(verifyModule(*M, &errs()));5882  EXPECT_EQ(OutlinedFn->arg_size(), 1U);5883 5884  BasicBlock *LoopPreheaderBB =5885      OutlinedFn->getEntryBlock().getSingleSuccessor();5886  // loop variables are 5 - lower bound, upper bound, stride, islastiter, and5887  // iterator/counter5888  bool FoundForInit = false;5889  for (Instruction &Inst : *LoopPreheaderBB) {5890    if (isa<CallInst>(Inst)) {5891      if (cast<CallInst>(&Inst)->getCalledFunction()->getName() ==5892          "__kmpc_for_static_init_4u") {5893        FoundForInit = true;5894      }5895    }5896  }5897  EXPECT_EQ(FoundForInit, true);5898 5899  bool FoundForExit = false;5900  bool FoundBarrier = false;5901  for (Instruction &Inst : *ForExitBB) {5902    if (isa<CallInst>(Inst)) {5903      if (cast<CallInst>(&Inst)->getCalledFunction()->getName() ==5904          "__kmpc_for_static_fini") {5905        FoundForExit = true;5906      }5907      if (cast<CallInst>(&Inst)->getCalledFunction()->getName() ==5908          "__kmpc_barrier") {5909        FoundBarrier = true;5910      }5911      if (FoundForExit && FoundBarrier)5912        break;5913    }5914  }5915  EXPECT_EQ(FoundForExit, true);5916  EXPECT_EQ(FoundBarrier, true);5917 5918  EXPECT_NE(SwitchBB, nullptr);5919  EXPECT_NE(SwitchBB->getTerminator(), nullptr);5920  EXPECT_EQ(isa<SwitchInst>(SwitchBB->getTerminator()), true);5921  Switch = cast<SwitchInst>(SwitchBB->getTerminator());5922  EXPECT_EQ(Switch->getNumCases(), 2U);5923 5924  EXPECT_EQ(CaseBBs.size(), 2U);5925  for (auto *&CaseBB : CaseBBs) {5926    EXPECT_EQ(CaseBB->getParent(), OutlinedFn);5927  }5928 5929  ASSERT_EQ(NumBodiesGenerated, 2U);5930  ASSERT_EQ(NumFiniCBCalls, 1U);5931  EXPECT_FALSE(verifyModule(*M, &errs()));5932}5933 5934TEST_F(OpenMPIRBuilderTest, CreateSectionsNoWait) {5935  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;5936  using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;5937  OpenMPIRBuilder OMPBuilder(*M);5938  OMPBuilder.initialize();5939  F->setName("func");5940  IRBuilder<> Builder(BB);5941 5942  BasicBlock *EnterBB = BasicBlock::Create(Ctx, "sections.enter", F);5943  Builder.CreateBr(EnterBB);5944  Builder.SetInsertPoint(EnterBB);5945  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});5946 5947  IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),5948                                    F->getEntryBlock().getFirstInsertionPt());5949  llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;5950  auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,5951                   llvm::Value &, llvm::Value &Val,5952                   llvm::Value *&ReplVal) { return CodeGenIP; };5953  auto FiniCB = [&](InsertPointTy IP) { return Error::success(); };5954 5955  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,5956                       OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector,5957                                                 PrivCB, FiniCB, false, true));5958  Builder.restoreIP(AfterIP);5959  Builder.CreateRetVoid(); // Required at the end of the function5960  for (auto &Inst : instructions(*F)) {5961    EXPECT_FALSE(isa<CallInst>(Inst) &&5962                 cast<CallInst>(&Inst)->getCalledFunction()->getName() ==5963                     "__kmpc_barrier" &&5964                 "call to function __kmpc_barrier found with nowait");5965  }5966}5967 5968TEST_F(OpenMPIRBuilderTest, CreateOffloadMaptypes) {5969  OpenMPIRBuilder OMPBuilder(*M);5970  OMPBuilder.initialize();5971 5972  IRBuilder<> Builder(BB);5973 5974  SmallVector<uint64_t> Mappings = {0, 1};5975  GlobalVariable *OffloadMaptypesGlobal =5976      OMPBuilder.createOffloadMaptypes(Mappings, "offload_maptypes");5977  EXPECT_FALSE(M->global_empty());5978  EXPECT_EQ(OffloadMaptypesGlobal->getName(), "offload_maptypes");5979  EXPECT_TRUE(OffloadMaptypesGlobal->isConstant());5980  EXPECT_TRUE(OffloadMaptypesGlobal->hasGlobalUnnamedAddr());5981  EXPECT_TRUE(OffloadMaptypesGlobal->hasPrivateLinkage());5982  EXPECT_TRUE(OffloadMaptypesGlobal->hasInitializer());5983  Constant *Initializer = OffloadMaptypesGlobal->getInitializer();5984  EXPECT_TRUE(isa<ConstantDataArray>(Initializer));5985  ConstantDataArray *MappingInit = dyn_cast<ConstantDataArray>(Initializer);5986  EXPECT_EQ(MappingInit->getNumElements(), Mappings.size());5987  EXPECT_TRUE(MappingInit->getType()->getElementType()->isIntegerTy(64));5988  Constant *CA = ConstantDataArray::get(Builder.getContext(), Mappings);5989  EXPECT_EQ(MappingInit, CA);5990}5991 5992TEST_F(OpenMPIRBuilderTest, CreateOffloadMapnames) {5993  OpenMPIRBuilder OMPBuilder(*M);5994  OMPBuilder.initialize();5995 5996  IRBuilder<> Builder(BB);5997 5998  uint32_t StrSize;5999  Constant *Cst1 =6000      OMPBuilder.getOrCreateSrcLocStr("array1", "file1", 2, 5, StrSize);6001  Constant *Cst2 =6002      OMPBuilder.getOrCreateSrcLocStr("array2", "file1", 3, 5, StrSize);6003  SmallVector<llvm::Constant *> Names = {Cst1, Cst2};6004 6005  GlobalVariable *OffloadMaptypesGlobal =6006      OMPBuilder.createOffloadMapnames(Names, "offload_mapnames");6007  EXPECT_FALSE(M->global_empty());6008  EXPECT_EQ(OffloadMaptypesGlobal->getName(), "offload_mapnames");6009  EXPECT_TRUE(OffloadMaptypesGlobal->isConstant());6010  EXPECT_FALSE(OffloadMaptypesGlobal->hasGlobalUnnamedAddr());6011  EXPECT_TRUE(OffloadMaptypesGlobal->hasPrivateLinkage());6012  EXPECT_TRUE(OffloadMaptypesGlobal->hasInitializer());6013  Constant *Initializer = OffloadMaptypesGlobal->getInitializer();6014  EXPECT_TRUE(isa<Constant>(Initializer->getOperand(0)->stripPointerCasts()));6015  EXPECT_TRUE(isa<Constant>(Initializer->getOperand(1)->stripPointerCasts()));6016 6017  GlobalVariable *Name1Gbl =6018      cast<GlobalVariable>(Initializer->getOperand(0)->stripPointerCasts());6019  EXPECT_TRUE(isa<ConstantDataArray>(Name1Gbl->getInitializer()));6020  ConstantDataArray *Name1GblCA =6021      dyn_cast<ConstantDataArray>(Name1Gbl->getInitializer());6022  EXPECT_EQ(Name1GblCA->getAsCString(), ";file1;array1;2;5;;");6023 6024  GlobalVariable *Name2Gbl =6025      cast<GlobalVariable>(Initializer->getOperand(1)->stripPointerCasts());6026  EXPECT_TRUE(isa<ConstantDataArray>(Name2Gbl->getInitializer()));6027  ConstantDataArray *Name2GblCA =6028      dyn_cast<ConstantDataArray>(Name2Gbl->getInitializer());6029  EXPECT_EQ(Name2GblCA->getAsCString(), ";file1;array2;3;5;;");6030 6031  EXPECT_TRUE(Initializer->getType()->getArrayElementType()->isPointerTy());6032  EXPECT_EQ(Initializer->getType()->getArrayNumElements(), Names.size());6033}6034 6035TEST_F(OpenMPIRBuilderTest, CreateMapperAllocas) {6036  OpenMPIRBuilder OMPBuilder(*M);6037  OMPBuilder.initialize();6038  F->setName("func");6039  IRBuilder<> Builder(BB);6040 6041  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});6042 6043  unsigned TotalNbOperand = 2;6044 6045  OpenMPIRBuilder::MapperAllocas MapperAllocas;6046  IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),6047                                    F->getEntryBlock().getFirstInsertionPt());6048  OMPBuilder.createMapperAllocas(Loc, AllocaIP, TotalNbOperand, MapperAllocas);6049  EXPECT_NE(MapperAllocas.ArgsBase, nullptr);6050  EXPECT_NE(MapperAllocas.Args, nullptr);6051  EXPECT_NE(MapperAllocas.ArgSizes, nullptr);6052  EXPECT_TRUE(MapperAllocas.ArgsBase->getAllocatedType()->isArrayTy());6053  ArrayType *ArrType =6054      dyn_cast<ArrayType>(MapperAllocas.ArgsBase->getAllocatedType());6055  EXPECT_EQ(ArrType->getNumElements(), TotalNbOperand);6056  EXPECT_TRUE(MapperAllocas.ArgsBase->getAllocatedType()6057                  ->getArrayElementType()6058                  ->isPointerTy());6059 6060  EXPECT_TRUE(MapperAllocas.Args->getAllocatedType()->isArrayTy());6061  ArrType = dyn_cast<ArrayType>(MapperAllocas.Args->getAllocatedType());6062  EXPECT_EQ(ArrType->getNumElements(), TotalNbOperand);6063  EXPECT_TRUE(MapperAllocas.Args->getAllocatedType()6064                  ->getArrayElementType()6065                  ->isPointerTy());6066 6067  EXPECT_TRUE(MapperAllocas.ArgSizes->getAllocatedType()->isArrayTy());6068  ArrType = dyn_cast<ArrayType>(MapperAllocas.ArgSizes->getAllocatedType());6069  EXPECT_EQ(ArrType->getNumElements(), TotalNbOperand);6070  EXPECT_TRUE(MapperAllocas.ArgSizes->getAllocatedType()6071                  ->getArrayElementType()6072                  ->isIntegerTy(64));6073}6074 6075TEST_F(OpenMPIRBuilderTest, EmitMapperCall) {6076  OpenMPIRBuilder OMPBuilder(*M);6077  OMPBuilder.initialize();6078  F->setName("func");6079  IRBuilder<> Builder(BB);6080  LLVMContext &Ctx = M->getContext();6081 6082  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});6083 6084  unsigned TotalNbOperand = 2;6085 6086  OpenMPIRBuilder::MapperAllocas MapperAllocas;6087  IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),6088                                    F->getEntryBlock().getFirstInsertionPt());6089  OMPBuilder.createMapperAllocas(Loc, AllocaIP, TotalNbOperand, MapperAllocas);6090 6091  auto *BeginMapperFunc = OMPBuilder.getOrCreateRuntimeFunctionPtr(6092      omp::OMPRTL___tgt_target_data_begin_mapper);6093 6094  SmallVector<uint64_t> Flags = {0, 2};6095 6096  uint32_t StrSize;6097  Constant *SrcLocCst =6098      OMPBuilder.getOrCreateSrcLocStr("", "file1", 2, 5, StrSize);6099  Value *SrcLocInfo = OMPBuilder.getOrCreateIdent(SrcLocCst, StrSize);6100 6101  Constant *Cst1 =6102      OMPBuilder.getOrCreateSrcLocStr("array1", "file1", 2, 5, StrSize);6103  Constant *Cst2 =6104      OMPBuilder.getOrCreateSrcLocStr("array2", "file1", 3, 5, StrSize);6105  SmallVector<llvm::Constant *> Names = {Cst1, Cst2};6106 6107  GlobalVariable *Maptypes =6108      OMPBuilder.createOffloadMaptypes(Flags, ".offload_maptypes");6109  Value *MaptypesArg = Builder.CreateConstInBoundsGEP2_32(6110      ArrayType::get(Type::getInt64Ty(Ctx), TotalNbOperand), Maptypes,6111      /*Idx0=*/0, /*Idx1=*/0);6112 6113  GlobalVariable *Mapnames =6114      OMPBuilder.createOffloadMapnames(Names, ".offload_mapnames");6115  Value *MapnamesArg = Builder.CreateConstInBoundsGEP2_32(6116      ArrayType::get(PointerType::getUnqual(Ctx), TotalNbOperand), Mapnames,6117      /*Idx0=*/0, /*Idx1=*/0);6118 6119  OMPBuilder.emitMapperCall(Builder.saveIP(), BeginMapperFunc, SrcLocInfo,6120                            MaptypesArg, MapnamesArg, MapperAllocas, -1,6121                            TotalNbOperand);6122 6123  CallInst *MapperCall = dyn_cast<CallInst>(&BB->back());6124  EXPECT_NE(MapperCall, nullptr);6125  EXPECT_EQ(MapperCall->arg_size(), 9U);6126  EXPECT_EQ(MapperCall->getCalledFunction()->getName(),6127            "__tgt_target_data_begin_mapper");6128  EXPECT_EQ(MapperCall->getOperand(0), SrcLocInfo);6129  EXPECT_TRUE(MapperCall->getOperand(1)->getType()->isIntegerTy(64));6130  EXPECT_TRUE(MapperCall->getOperand(2)->getType()->isIntegerTy(32));6131 6132  EXPECT_EQ(MapperCall->getOperand(6), MaptypesArg);6133  EXPECT_EQ(MapperCall->getOperand(7), MapnamesArg);6134  EXPECT_TRUE(MapperCall->getOperand(8)->getType()->isPointerTy());6135}6136 6137TEST_F(OpenMPIRBuilderTest, TargetEnterData) {6138  OpenMPIRBuilder OMPBuilder(*M);6139  OMPBuilder.initialize();6140  F->setName("func");6141  IRBuilder<> Builder(BB);6142  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});6143 6144  int64_t DeviceID = 2;6145 6146  AllocaInst *Val1 =6147      Builder.CreateAlloca(Builder.getInt32Ty(), Builder.getInt64(1));6148  ASSERT_NE(Val1, nullptr);6149 6150  IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),6151                                    F->getEntryBlock().getFirstInsertionPt());6152 6153  llvm::OpenMPIRBuilder::MapInfosTy CombinedInfo;6154  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;6155  auto GenMapInfoCB =6156      [&](InsertPointTy codeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {6157    // Get map clause information.6158    Builder.restoreIP(codeGenIP);6159 6160    CombinedInfo.BasePointers.emplace_back(Val1);6161    CombinedInfo.Pointers.emplace_back(Val1);6162    CombinedInfo.DevicePointers.emplace_back(6163        llvm::OpenMPIRBuilder::DeviceInfoTy::None);6164    CombinedInfo.Sizes.emplace_back(Builder.getInt64(4));6165    CombinedInfo.Types.emplace_back(llvm::omp::OpenMPOffloadMappingFlags(1));6166    uint32_t temp;6167    CombinedInfo.Names.emplace_back(6168        OMPBuilder.getOrCreateSrcLocStr("unknown", temp));6169    return CombinedInfo;6170  };6171 6172  auto CustomMapperCB = [&](unsigned int I) { return nullptr; };6173  llvm::OpenMPIRBuilder::TargetDataInfo Info(6174      /*RequiresDevicePointerInfo=*/false,6175      /*SeparateBeginEndCalls=*/true);6176 6177  OMPBuilder.Config.setIsGPU(true);6178 6179  llvm::omp::RuntimeFunction RTLFunc = OMPRTL___tgt_target_data_begin_mapper;6180  ASSERT_EXPECTED_INIT(6181      OpenMPIRBuilder::InsertPointTy, AfterIP,6182      OMPBuilder.createTargetData(6183          Loc, AllocaIP, Builder.saveIP(), Builder.getInt64(DeviceID),6184          /* IfCond= */ nullptr, Info, GenMapInfoCB, CustomMapperCB, &RTLFunc));6185  Builder.restoreIP(AfterIP);6186 6187  CallInst *TargetDataCall = dyn_cast<CallInst>(&BB->back());6188  EXPECT_NE(TargetDataCall, nullptr);6189  EXPECT_EQ(TargetDataCall->arg_size(), 9U);6190  EXPECT_EQ(TargetDataCall->getCalledFunction()->getName(),6191            "__tgt_target_data_begin_mapper");6192  EXPECT_TRUE(TargetDataCall->getOperand(1)->getType()->isIntegerTy(64));6193  EXPECT_TRUE(TargetDataCall->getOperand(2)->getType()->isIntegerTy(32));6194  EXPECT_TRUE(TargetDataCall->getOperand(8)->getType()->isPointerTy());6195 6196  Builder.CreateRetVoid();6197  EXPECT_FALSE(verifyModule(*M, &errs()));6198}6199 6200TEST_F(OpenMPIRBuilderTest, TargetExitData) {6201  OpenMPIRBuilder OMPBuilder(*M);6202  OMPBuilder.initialize();6203  F->setName("func");6204  IRBuilder<> Builder(BB);6205  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});6206 6207  int64_t DeviceID = 2;6208 6209  AllocaInst *Val1 =6210      Builder.CreateAlloca(Builder.getInt32Ty(), Builder.getInt64(1));6211  ASSERT_NE(Val1, nullptr);6212 6213  IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),6214                                    F->getEntryBlock().getFirstInsertionPt());6215 6216  llvm::OpenMPIRBuilder::MapInfosTy CombinedInfo;6217  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;6218  auto GenMapInfoCB =6219      [&](InsertPointTy codeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {6220    // Get map clause information.6221    Builder.restoreIP(codeGenIP);6222 6223    CombinedInfo.BasePointers.emplace_back(Val1);6224    CombinedInfo.Pointers.emplace_back(Val1);6225    CombinedInfo.DevicePointers.emplace_back(6226        llvm::OpenMPIRBuilder::DeviceInfoTy::None);6227    CombinedInfo.Sizes.emplace_back(Builder.getInt64(4));6228    CombinedInfo.Types.emplace_back(llvm::omp::OpenMPOffloadMappingFlags(2));6229    uint32_t temp;6230    CombinedInfo.Names.emplace_back(6231        OMPBuilder.getOrCreateSrcLocStr("unknown", temp));6232    return CombinedInfo;6233  };6234 6235  auto CustomMapperCB = [&](unsigned int I) { return nullptr; };6236  llvm::OpenMPIRBuilder::TargetDataInfo Info(6237      /*RequiresDevicePointerInfo=*/false,6238      /*SeparateBeginEndCalls=*/true);6239 6240  OMPBuilder.Config.setIsGPU(true);6241 6242  llvm::omp::RuntimeFunction RTLFunc = OMPRTL___tgt_target_data_end_mapper;6243  ASSERT_EXPECTED_INIT(6244      OpenMPIRBuilder::InsertPointTy, AfterIP,6245      OMPBuilder.createTargetData(6246          Loc, AllocaIP, Builder.saveIP(), Builder.getInt64(DeviceID),6247          /* IfCond= */ nullptr, Info, GenMapInfoCB, CustomMapperCB, &RTLFunc));6248  Builder.restoreIP(AfterIP);6249 6250  CallInst *TargetDataCall = dyn_cast<CallInst>(&BB->back());6251  EXPECT_NE(TargetDataCall, nullptr);6252  EXPECT_EQ(TargetDataCall->arg_size(), 9U);6253  EXPECT_EQ(TargetDataCall->getCalledFunction()->getName(),6254            "__tgt_target_data_end_mapper");6255  EXPECT_TRUE(TargetDataCall->getOperand(1)->getType()->isIntegerTy(64));6256  EXPECT_TRUE(TargetDataCall->getOperand(2)->getType()->isIntegerTy(32));6257  EXPECT_TRUE(TargetDataCall->getOperand(8)->getType()->isPointerTy());6258 6259  Builder.CreateRetVoid();6260  EXPECT_FALSE(verifyModule(*M, &errs()));6261}6262 6263TEST_F(OpenMPIRBuilderTest, TargetDataRegion) {6264  OpenMPIRBuilder OMPBuilder(*M);6265  OMPBuilder.initialize();6266  F->setName("func");6267  IRBuilder<> Builder(BB);6268  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});6269 6270  int64_t DeviceID = 2;6271 6272  AllocaInst *Val1 =6273      Builder.CreateAlloca(Builder.getInt32Ty(), Builder.getInt64(1));6274  ASSERT_NE(Val1, nullptr);6275 6276  AllocaInst *Val2 = Builder.CreateAlloca(Builder.getPtrTy());6277  ASSERT_NE(Val2, nullptr);6278 6279  AllocaInst *Val3 = Builder.CreateAlloca(Builder.getPtrTy());6280  ASSERT_NE(Val3, nullptr);6281 6282  IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),6283                                    F->getEntryBlock().getFirstInsertionPt());6284 6285  using DeviceInfoTy = llvm::OpenMPIRBuilder::DeviceInfoTy;6286  llvm::OpenMPIRBuilder::MapInfosTy CombinedInfo;6287  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;6288  auto GenMapInfoCB =6289      [&](InsertPointTy codeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {6290    // Get map clause information.6291    Builder.restoreIP(codeGenIP);6292    uint32_t temp;6293 6294    CombinedInfo.BasePointers.emplace_back(Val1);6295    CombinedInfo.Pointers.emplace_back(Val1);6296    CombinedInfo.DevicePointers.emplace_back(DeviceInfoTy::None);6297    CombinedInfo.Sizes.emplace_back(Builder.getInt64(4));6298    CombinedInfo.Types.emplace_back(llvm::omp::OpenMPOffloadMappingFlags(3));6299    CombinedInfo.Names.emplace_back(6300        OMPBuilder.getOrCreateSrcLocStr("unknown", temp));6301 6302    CombinedInfo.BasePointers.emplace_back(Val2);6303    CombinedInfo.Pointers.emplace_back(Val2);6304    CombinedInfo.DevicePointers.emplace_back(DeviceInfoTy::Pointer);6305    CombinedInfo.Sizes.emplace_back(Builder.getInt64(8));6306    CombinedInfo.Types.emplace_back(llvm::omp::OpenMPOffloadMappingFlags(67));6307    CombinedInfo.Names.emplace_back(6308        OMPBuilder.getOrCreateSrcLocStr("unknown", temp));6309 6310    CombinedInfo.BasePointers.emplace_back(Val3);6311    CombinedInfo.Pointers.emplace_back(Val3);6312    CombinedInfo.DevicePointers.emplace_back(DeviceInfoTy::Address);6313    CombinedInfo.Sizes.emplace_back(Builder.getInt64(8));6314    CombinedInfo.Types.emplace_back(llvm::omp::OpenMPOffloadMappingFlags(67));6315    CombinedInfo.Names.emplace_back(6316        OMPBuilder.getOrCreateSrcLocStr("unknown", temp));6317    return CombinedInfo;6318  };6319 6320  auto CustomMapperCB = [&](unsigned int I) { return nullptr; };6321  llvm::OpenMPIRBuilder::TargetDataInfo Info(6322      /*RequiresDevicePointerInfo=*/true,6323      /*SeparateBeginEndCalls=*/true);6324 6325  OMPBuilder.Config.setIsGPU(true);6326 6327  using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;6328  auto BodyCB = [&](InsertPointTy CodeGenIP, BodyGenTy BodyGenType) {6329    if (BodyGenType == BodyGenTy::Priv) {6330      EXPECT_EQ(Info.DevicePtrInfoMap.size(), 2u);6331      Builder.restoreIP(CodeGenIP);6332      CallInst *TargetDataCall =6333          dyn_cast<CallInst>(BB->back().getPrevNode()->getPrevNode());6334      EXPECT_NE(TargetDataCall, nullptr);6335      EXPECT_EQ(TargetDataCall->arg_size(), 9U);6336      EXPECT_EQ(TargetDataCall->getCalledFunction()->getName(),6337                "__tgt_target_data_begin_mapper");6338      EXPECT_TRUE(TargetDataCall->getOperand(1)->getType()->isIntegerTy(64));6339      EXPECT_TRUE(TargetDataCall->getOperand(2)->getType()->isIntegerTy(32));6340      EXPECT_TRUE(TargetDataCall->getOperand(8)->getType()->isPointerTy());6341 6342      LoadInst *LI = dyn_cast<LoadInst>(BB->back().getPrevNode());6343      EXPECT_NE(LI, nullptr);6344      StoreInst *SI = dyn_cast<StoreInst>(&BB->back());6345      EXPECT_NE(SI, nullptr);6346      EXPECT_EQ(SI->getValueOperand(), LI);6347      EXPECT_EQ(SI->getPointerOperand(), Info.DevicePtrInfoMap[Val2].second);6348      EXPECT_TRUE(isa<AllocaInst>(Info.DevicePtrInfoMap[Val2].second));6349      EXPECT_TRUE(isa<GetElementPtrInst>(Info.DevicePtrInfoMap[Val3].second));6350      Builder.CreateStore(Builder.getInt32(99), Val1);6351    }6352    return Builder.saveIP();6353  };6354 6355  ASSERT_EXPECTED_INIT(6356      OpenMPIRBuilder::InsertPointTy, TargetDataIP1,6357      OMPBuilder.createTargetData(Loc, AllocaIP, Builder.saveIP(),6358                                  Builder.getInt64(DeviceID),6359                                  /* IfCond= */ nullptr, Info, GenMapInfoCB,6360                                  CustomMapperCB, nullptr, BodyCB));6361  Builder.restoreIP(TargetDataIP1);6362 6363  CallInst *TargetDataCall = dyn_cast<CallInst>(&BB->back());6364  EXPECT_NE(TargetDataCall, nullptr);6365  EXPECT_EQ(TargetDataCall->arg_size(), 9U);6366  EXPECT_EQ(TargetDataCall->getCalledFunction()->getName(),6367            "__tgt_target_data_end_mapper");6368  EXPECT_TRUE(TargetDataCall->getOperand(1)->getType()->isIntegerTy(64));6369  EXPECT_TRUE(TargetDataCall->getOperand(2)->getType()->isIntegerTy(32));6370  EXPECT_TRUE(TargetDataCall->getOperand(8)->getType()->isPointerTy());6371 6372  // Check that BodyGenCB is still made when IsTargetDevice is set to true.6373  OMPBuilder.Config.setIsTargetDevice(true);6374  bool CheckDevicePassBodyGen = false;6375  auto BodyTargetCB = [&](InsertPointTy CodeGenIP, BodyGenTy BodyGenType) {6376    CheckDevicePassBodyGen = true;6377    Builder.restoreIP(CodeGenIP);6378    CallInst *TargetDataCall =6379        dyn_cast<CallInst>(BB->back().getPrevNode()->getPrevNode());6380    // Make sure no begin_mapper call is present for device pass.6381    EXPECT_EQ(TargetDataCall, nullptr);6382    return Builder.saveIP();6383  };6384  ASSERT_EXPECTED_INIT(6385      OpenMPIRBuilder::InsertPointTy, TargetDataIP2,6386      OMPBuilder.createTargetData(Loc, AllocaIP, Builder.saveIP(),6387                                  Builder.getInt64(DeviceID),6388                                  /* IfCond= */ nullptr, Info, GenMapInfoCB,6389                                  CustomMapperCB, nullptr, BodyTargetCB));6390  Builder.restoreIP(TargetDataIP2);6391  EXPECT_TRUE(CheckDevicePassBodyGen);6392 6393  Builder.CreateRetVoid();6394  EXPECT_FALSE(verifyModule(*M, &errs()));6395}6396 6397namespace {6398// Some basic handling of argument mapping for the moment6399void CreateDefaultMapInfos(llvm::OpenMPIRBuilder &OmpBuilder,6400                           llvm::SmallVectorImpl<llvm::Value *> &Args,6401                           llvm::OpenMPIRBuilder::MapInfosTy &CombinedInfo) {6402  for (auto Arg : Args) {6403    CombinedInfo.BasePointers.emplace_back(Arg);6404    CombinedInfo.Pointers.emplace_back(Arg);6405    uint32_t SrcLocStrSize;6406    CombinedInfo.Names.emplace_back(OmpBuilder.getOrCreateSrcLocStr(6407        "Unknown loc - stub implementation", SrcLocStrSize));6408    CombinedInfo.Types.emplace_back(llvm::omp::OpenMPOffloadMappingFlags(6409        llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO |6410        llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM |6411        llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM));6412    CombinedInfo.Sizes.emplace_back(OmpBuilder.Builder.getInt64(6413        OmpBuilder.M.getDataLayout().getTypeAllocSize(Arg->getType())));6414  }6415}6416} // namespace6417 6418TEST_F(OpenMPIRBuilderTest, TargetRegion) {6419  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;6420  OpenMPIRBuilder OMPBuilder(*M);6421  OMPBuilder.initialize();6422  OpenMPIRBuilderConfig Config(false, false, false, false, false, false, false);6423  OMPBuilder.setConfig(Config);6424  F->setName("func");6425  F->addFnAttr("target-cpu", "x86-64");6426  F->addFnAttr("target-features", "+mmx,+sse");6427  IRBuilder<> Builder(BB);6428  auto *Int32Ty = Builder.getInt32Ty();6429  Builder.SetCurrentDebugLocation(DL);6430 6431  AllocaInst *APtr = Builder.CreateAlloca(Int32Ty, nullptr, "a_ptr");6432  AllocaInst *BPtr = Builder.CreateAlloca(Int32Ty, nullptr, "b_ptr");6433  AllocaInst *CPtr = Builder.CreateAlloca(Int32Ty, nullptr, "c_ptr");6434 6435  Builder.CreateStore(Builder.getInt32(10), APtr);6436  Builder.CreateStore(Builder.getInt32(20), BPtr);6437  auto BodyGenCB = [&](InsertPointTy AllocaIP,6438                       InsertPointTy CodeGenIP) -> InsertPointTy {6439    IRBuilderBase::InsertPointGuard guard(Builder);6440    Builder.SetCurrentDebugLocation(llvm::DebugLoc());6441    Builder.restoreIP(CodeGenIP);6442    LoadInst *AVal = Builder.CreateLoad(Int32Ty, APtr);6443    LoadInst *BVal = Builder.CreateLoad(Int32Ty, BPtr);6444    Value *Sum = Builder.CreateAdd(AVal, BVal);6445    Builder.CreateStore(Sum, CPtr);6446    return Builder.saveIP();6447  };6448 6449  llvm::SmallVector<llvm::Value *> Inputs;6450  Inputs.push_back(APtr);6451  Inputs.push_back(BPtr);6452  Inputs.push_back(CPtr);6453 6454  auto SimpleArgAccessorCB =6455      [&](llvm::Argument &Arg, llvm::Value *Input, llvm::Value *&RetVal,6456          llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,6457          llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP) {6458        IRBuilderBase::InsertPointGuard guard(Builder);6459        Builder.SetCurrentDebugLocation(llvm::DebugLoc());6460        if (!OMPBuilder.Config.isTargetDevice()) {6461          RetVal = cast<llvm::Value>(&Arg);6462          return CodeGenIP;6463        }6464 6465        Builder.restoreIP(AllocaIP);6466 6467        llvm::Value *Addr = Builder.CreateAlloca(6468            Arg.getType()->isPointerTy()6469                ? Arg.getType()6470                : Type::getInt64Ty(Builder.getContext()),6471            OMPBuilder.M.getDataLayout().getAllocaAddrSpace());6472        llvm::Value *AddrAscast =6473            Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Input->getType());6474        Builder.CreateStore(&Arg, AddrAscast);6475 6476        Builder.restoreIP(CodeGenIP);6477 6478        RetVal = Builder.CreateLoad(Arg.getType(), AddrAscast);6479 6480        return Builder.saveIP();6481      };6482 6483  llvm::OpenMPIRBuilder::MapInfosTy CombinedInfos;6484  auto GenMapInfoCB = [&](llvm::OpenMPIRBuilder::InsertPointTy codeGenIP)6485      -> llvm::OpenMPIRBuilder::MapInfosTy & {6486    CreateDefaultMapInfos(OMPBuilder, Inputs, CombinedInfos);6487    return CombinedInfos;6488  };6489 6490  auto CustomMapperCB = [&](unsigned int I) { return nullptr; };6491  llvm::OpenMPIRBuilder::TargetDataInfo Info(6492      /*RequiresDevicePointerInfo=*/false,6493      /*SeparateBeginEndCalls=*/true);6494 6495  TargetRegionEntryInfo EntryInfo("func", 42, 4711, 17);6496  OpenMPIRBuilder::LocationDescription OmpLoc({Builder.saveIP(), DL});6497  OpenMPIRBuilder::TargetKernelRuntimeAttrs RuntimeAttrs;6498  OpenMPIRBuilder::TargetKernelDefaultAttrs DefaultAttrs = {6499      /*ExecFlags=*/omp::OMPTgtExecModeFlags::OMP_TGT_EXEC_MODE_GENERIC,6500      /*MaxTeams=*/{10}, /*MinTeams=*/0, /*MaxThreads=*/{0}, /*MinThreads=*/0};6501  RuntimeAttrs.TargetThreadLimit[0] = Builder.getInt32(20);6502  RuntimeAttrs.TeamsThreadLimit[0] = Builder.getInt32(30);6503  RuntimeAttrs.MaxThreads = Builder.getInt32(40);6504 6505  ASSERT_EXPECTED_INIT(6506      OpenMPIRBuilder::InsertPointTy, AfterIP,6507      OMPBuilder.createTarget(OmpLoc, /*IsOffloadEntry=*/true, Builder.saveIP(),6508                              Builder.saveIP(), Info, EntryInfo, DefaultAttrs,6509                              RuntimeAttrs, /*IfCond=*/nullptr, Inputs,6510                              GenMapInfoCB, BodyGenCB, SimpleArgAccessorCB,6511                              CustomMapperCB, {}, false));6512  EXPECT_EQ(DL, Builder.getCurrentDebugLocation());6513  Builder.restoreIP(AfterIP);6514 6515  OMPBuilder.finalize();6516  Builder.CreateRetVoid();6517 6518  // Check the kernel launch sequence6519  auto Iter = F->getEntryBlock().rbegin();6520  EXPECT_TRUE(isa<BranchInst>(&*(Iter)));6521  BranchInst *Branch = dyn_cast<BranchInst>(&*(Iter));6522  EXPECT_TRUE(isa<CmpInst>(&*(++Iter)));6523  EXPECT_TRUE(isa<CallInst>(&*(++Iter)));6524  CallInst *Call = dyn_cast<CallInst>(&*(Iter));6525 6526  // Check that the kernel launch function is called6527  Function *KernelLaunchFunc = Call->getCalledFunction();6528  EXPECT_NE(KernelLaunchFunc, nullptr);6529  StringRef FunctionName = KernelLaunchFunc->getName();6530  EXPECT_TRUE(FunctionName.starts_with("__tgt_target_kernel"));6531 6532  // Check num_teams and num_threads in call arguments6533  EXPECT_TRUE(Call->arg_size() >= 4);6534  Value *NumTeamsArg = Call->getArgOperand(2);6535  EXPECT_TRUE(isa<ConstantInt>(NumTeamsArg));6536  EXPECT_EQ(10U, cast<ConstantInt>(NumTeamsArg)->getZExtValue());6537  Value *NumThreadsArg = Call->getArgOperand(3);6538  EXPECT_TRUE(isa<ConstantInt>(NumThreadsArg));6539  EXPECT_EQ(20U, cast<ConstantInt>(NumThreadsArg)->getZExtValue());6540 6541  // Check num_teams and num_threads kernel arguments (use number 5 starting6542  // from the end and counting the call to __tgt_target_kernel as the first use)6543  Value *KernelArgs = Call->getArgOperand(Call->arg_size() - 1);6544  EXPECT_TRUE(KernelArgs->hasNUsesOrMore(4));6545  Value *NumTeamsGetElemPtr = *std::next(KernelArgs->user_begin(), 3);6546  EXPECT_TRUE(isa<GetElementPtrInst>(NumTeamsGetElemPtr));6547  Value *NumTeamsStore = NumTeamsGetElemPtr->getUniqueUndroppableUser();6548  EXPECT_TRUE(isa<StoreInst>(NumTeamsStore));6549  Value *NumTeamsStoreArg = cast<StoreInst>(NumTeamsStore)->getValueOperand();6550  EXPECT_TRUE(isa<ConstantDataSequential>(NumTeamsStoreArg));6551  auto *NumTeamsStoreValue = cast<ConstantDataSequential>(NumTeamsStoreArg);6552  EXPECT_EQ(3U, NumTeamsStoreValue->getNumElements());6553  EXPECT_EQ(10U, NumTeamsStoreValue->getElementAsInteger(0));6554  EXPECT_EQ(0U, NumTeamsStoreValue->getElementAsInteger(1));6555  EXPECT_EQ(0U, NumTeamsStoreValue->getElementAsInteger(2));6556  Value *NumThreadsGetElemPtr = *std::next(KernelArgs->user_begin(), 2);6557  EXPECT_TRUE(isa<GetElementPtrInst>(NumThreadsGetElemPtr));6558  Value *NumThreadsStore = NumThreadsGetElemPtr->getUniqueUndroppableUser();6559  EXPECT_TRUE(isa<StoreInst>(NumThreadsStore));6560  Value *NumThreadsStoreArg =6561      cast<StoreInst>(NumThreadsStore)->getValueOperand();6562  EXPECT_TRUE(isa<ConstantDataSequential>(NumThreadsStoreArg));6563  auto *NumThreadsStoreValue = cast<ConstantDataSequential>(NumThreadsStoreArg);6564  EXPECT_EQ(3U, NumThreadsStoreValue->getNumElements());6565  EXPECT_EQ(20U, NumThreadsStoreValue->getElementAsInteger(0));6566  EXPECT_EQ(0U, NumThreadsStoreValue->getElementAsInteger(1));6567  EXPECT_EQ(0U, NumThreadsStoreValue->getElementAsInteger(2));6568 6569  // Check the fallback call6570  BasicBlock *FallbackBlock = Branch->getSuccessor(0);6571  Iter = FallbackBlock->rbegin();6572  CallInst *FCall = dyn_cast<CallInst>(&*(++Iter));6573  // 'F' has a dummy DISubprogram which causes OutlinedFunc to also6574  // have a DISubprogram. In this case, the call to OutlinedFunc needs6575  // to have a debug loc, otherwise verifier will complain.6576  FCall->setDebugLoc(DL);6577  EXPECT_NE(FCall, nullptr);6578 6579  // Check that the correct aguments are passed in6580  for (auto ArgInput : zip(FCall->args(), Inputs)) {6581    EXPECT_EQ(std::get<0>(ArgInput), std::get<1>(ArgInput));6582  }6583 6584  // Check that the outlined function exists with the expected prefix6585  Function *OutlinedFunc = FCall->getCalledFunction();6586  EXPECT_NE(OutlinedFunc, nullptr);6587  StringRef FunctionName2 = OutlinedFunc->getName();6588  EXPECT_TRUE(FunctionName2.starts_with("__omp_offloading"));6589 6590  // Check that target-cpu and target-features were propagated to the outlined6591  // function6592  EXPECT_EQ(OutlinedFunc->getFnAttribute("target-cpu"),6593            F->getFnAttribute("target-cpu"));6594  EXPECT_EQ(OutlinedFunc->getFnAttribute("target-features"),6595            F->getFnAttribute("target-features"));6596 6597  EXPECT_FALSE(verifyModule(*M, &errs()));6598}6599 6600TEST_F(OpenMPIRBuilderTest, TargetRegionDevice) {6601  OpenMPIRBuilder OMPBuilder(*M);6602  OMPBuilder.setConfig(6603      OpenMPIRBuilderConfig(true, false, false, false, false, false, false));6604  OMPBuilder.initialize();6605 6606  F->setName("func");6607  F->addFnAttr("target-cpu", "gfx90a");6608  F->addFnAttr("target-features", "+gfx9-insts,+wavefrontsize64");6609  IRBuilder<> Builder(BB);6610  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});6611  Builder.SetCurrentDebugLocation(DL);6612 6613  LoadInst *Value = nullptr;6614  StoreInst *TargetStore = nullptr;6615  llvm::SmallVector<llvm::Value *, 2> CapturedArgs = {6616      Constant::getNullValue(PointerType::get(Ctx, 0)),6617      Constant::getNullValue(PointerType::get(Ctx, 0))};6618 6619  auto SimpleArgAccessorCB =6620      [&](llvm::Argument &Arg, llvm::Value *Input, llvm::Value *&RetVal,6621          llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,6622          llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP) {6623        IRBuilderBase::InsertPointGuard guard(Builder);6624        Builder.SetCurrentDebugLocation(llvm::DebugLoc());6625        if (!OMPBuilder.Config.isTargetDevice()) {6626          RetVal = cast<llvm::Value>(&Arg);6627          return CodeGenIP;6628        }6629 6630        Builder.restoreIP(AllocaIP);6631 6632        llvm::Value *Addr = Builder.CreateAlloca(6633            Arg.getType()->isPointerTy()6634                ? Arg.getType()6635                : Type::getInt64Ty(Builder.getContext()),6636            OMPBuilder.M.getDataLayout().getAllocaAddrSpace());6637        llvm::Value *AddrAscast =6638            Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Input->getType());6639        Builder.CreateStore(&Arg, AddrAscast);6640 6641        Builder.restoreIP(CodeGenIP);6642 6643        RetVal = Builder.CreateLoad(Arg.getType(), AddrAscast);6644 6645        return Builder.saveIP();6646      };6647 6648  llvm::OpenMPIRBuilder::MapInfosTy CombinedInfos;6649  auto GenMapInfoCB = [&](llvm::OpenMPIRBuilder::InsertPointTy codeGenIP)6650      -> llvm::OpenMPIRBuilder::MapInfosTy & {6651    CreateDefaultMapInfos(OMPBuilder, CapturedArgs, CombinedInfos);6652    return CombinedInfos;6653  };6654 6655  auto CustomMapperCB = [&](unsigned int I) { return nullptr; };6656  auto BodyGenCB = [&](OpenMPIRBuilder::InsertPointTy AllocaIP,6657                       OpenMPIRBuilder::InsertPointTy CodeGenIP)6658      -> OpenMPIRBuilder::InsertPointTy {6659    IRBuilderBase::InsertPointGuard guard(Builder);6660    Builder.SetCurrentDebugLocation(llvm::DebugLoc());6661    Builder.restoreIP(CodeGenIP);6662    Value = Builder.CreateLoad(Type::getInt32Ty(Ctx), CapturedArgs[0]);6663    TargetStore = Builder.CreateStore(Value, CapturedArgs[1]);6664    return Builder.saveIP();6665  };6666 6667  IRBuilder<>::InsertPoint EntryIP(&F->getEntryBlock(),6668                                   F->getEntryBlock().getFirstInsertionPt());6669  TargetRegionEntryInfo EntryInfo("parent", /*DeviceID=*/1, /*FileID=*/2,6670                                  /*Line=*/3, /*Count=*/0);6671  OpenMPIRBuilder::TargetKernelRuntimeAttrs RuntimeAttrs;6672  OpenMPIRBuilder::TargetKernelDefaultAttrs DefaultAttrs = {6673      /*ExecFlags=*/omp::OMPTgtExecModeFlags::OMP_TGT_EXEC_MODE_GENERIC,6674      /*MaxTeams=*/{-1}, /*MinTeams=*/0, /*MaxThreads=*/{0}, /*MinThreads=*/0};6675  llvm::OpenMPIRBuilder::TargetDataInfo Info(6676      /*RequiresDevicePointerInfo=*/false,6677      /*SeparateBeginEndCalls=*/true);6678 6679  ASSERT_EXPECTED_INIT(6680      OpenMPIRBuilder::InsertPointTy, AfterIP,6681      OMPBuilder.createTarget(Loc, /*IsOffloadEntry=*/true, EntryIP, EntryIP,6682                              Info, EntryInfo, DefaultAttrs, RuntimeAttrs,6683                              /*IfCond=*/nullptr, CapturedArgs, GenMapInfoCB,6684                              BodyGenCB, SimpleArgAccessorCB, CustomMapperCB,6685                              {}, false));6686  EXPECT_EQ(DL, Builder.getCurrentDebugLocation());6687  Builder.restoreIP(AfterIP);6688 6689  Builder.CreateRetVoid();6690  OMPBuilder.finalize();6691 6692  // Check outlined function6693  EXPECT_FALSE(verifyModule(*M, &errs()));6694  EXPECT_NE(TargetStore, nullptr);6695  Function *OutlinedFn = TargetStore->getFunction();6696  EXPECT_NE(F, OutlinedFn);6697 6698  // Check that target-cpu and target-features were propagated to the outlined6699  // function6700  EXPECT_EQ(OutlinedFn->getFnAttribute("target-cpu"),6701            F->getFnAttribute("target-cpu"));6702  EXPECT_EQ(OutlinedFn->getFnAttribute("target-features"),6703            F->getFnAttribute("target-features"));6704 6705  EXPECT_TRUE(OutlinedFn->hasWeakODRLinkage());6706  // Account for the "implicit" first argument.6707  EXPECT_EQ(OutlinedFn->getName(), "__omp_offloading_1_2_parent_l3");6708  EXPECT_EQ(OutlinedFn->arg_size(), 3U);6709  EXPECT_TRUE(OutlinedFn->getArg(1)->getType()->isPointerTy());6710  EXPECT_TRUE(OutlinedFn->getArg(2)->getType()->isPointerTy());6711 6712  // Check entry block6713  auto &EntryBlock = OutlinedFn->getEntryBlock();6714  Instruction *Alloca1 = &*EntryBlock.getFirstNonPHIIt();6715  EXPECT_NE(Alloca1, nullptr);6716 6717  EXPECT_TRUE(isa<AllocaInst>(Alloca1));6718  auto *Store1 = Alloca1->getNextNode();6719  EXPECT_TRUE(isa<StoreInst>(Store1));6720  auto *Alloca2 = Store1->getNextNode();6721  EXPECT_TRUE(isa<AllocaInst>(Alloca2));6722  auto *Store2 = Alloca2->getNextNode();6723  EXPECT_TRUE(isa<StoreInst>(Store2));6724 6725  auto *InitCall = dyn_cast<CallInst>(Store2->getNextNode());6726  EXPECT_NE(InitCall, nullptr);6727  EXPECT_EQ(InitCall->getCalledFunction()->getName(), "__kmpc_target_init");6728  EXPECT_EQ(InitCall->arg_size(), 2U);6729  EXPECT_TRUE(isa<GlobalVariable>(InitCall->getArgOperand(0)));6730  auto *KernelEnvGV = cast<GlobalVariable>(InitCall->getArgOperand(0));6731  EXPECT_TRUE(isa<ConstantStruct>(KernelEnvGV->getInitializer()));6732  auto *KernelEnvC = cast<ConstantStruct>(KernelEnvGV->getInitializer());6733  EXPECT_TRUE(isa<ConstantStruct>(KernelEnvC->getAggregateElement(0U)));6734  auto ConfigC = cast<ConstantStruct>(KernelEnvC->getAggregateElement(0U));6735  EXPECT_EQ(ConfigC->getAggregateElement(0U),6736            ConstantInt::get(Type::getInt8Ty(Ctx), true));6737  EXPECT_EQ(ConfigC->getAggregateElement(1U),6738            ConstantInt::get(Type::getInt8Ty(Ctx), true));6739  EXPECT_EQ(ConfigC->getAggregateElement(2U),6740            ConstantInt::get(Type::getInt8Ty(Ctx), OMP_TGT_EXEC_MODE_GENERIC));6741 6742  auto *EntryBlockBranch = EntryBlock.getTerminator();6743  EXPECT_NE(EntryBlockBranch, nullptr);6744  EXPECT_EQ(EntryBlockBranch->getNumSuccessors(), 2U);6745 6746  // Check user code block6747  auto *UserCodeBlock = EntryBlockBranch->getSuccessor(0);6748  EXPECT_EQ(UserCodeBlock->getName(), "user_code.entry");6749  Instruction *Load1 = &*UserCodeBlock->getFirstNonPHIIt();6750  EXPECT_TRUE(isa<LoadInst>(Load1));6751  auto *Load2 = Load1->getNextNode();6752  EXPECT_TRUE(isa<LoadInst>(Load2));6753 6754  auto *OutlinedBlockBr = Load2->getNextNode();6755  EXPECT_TRUE(isa<BranchInst>(OutlinedBlockBr));6756 6757  auto *OutlinedBlock = OutlinedBlockBr->getSuccessor(0);6758  EXPECT_EQ(OutlinedBlock->getName(), "outlined.body");6759 6760  Instruction *Value1 = &*OutlinedBlock->getFirstNonPHIIt();6761  EXPECT_EQ(Value1, Value);6762  EXPECT_EQ(Value1->getNextNode(), TargetStore);6763  auto *Deinit = TargetStore->getNextNode();6764  EXPECT_NE(Deinit, nullptr);6765 6766  auto *DeinitCall = dyn_cast<CallInst>(Deinit);6767  EXPECT_NE(DeinitCall, nullptr);6768  EXPECT_EQ(DeinitCall->getCalledFunction()->getName(), "__kmpc_target_deinit");6769  EXPECT_EQ(DeinitCall->arg_size(), 0U);6770 6771  EXPECT_TRUE(isa<ReturnInst>(DeinitCall->getNextNode()));6772 6773  // Check exit block6774  auto *ExitBlock = EntryBlockBranch->getSuccessor(1);6775  EXPECT_EQ(ExitBlock->getName(), "worker.exit");6776  EXPECT_TRUE(isa<ReturnInst>(ExitBlock->getFirstNonPHIIt()));6777 6778  // Check global exec_mode.6779  GlobalVariable *Used = M->getGlobalVariable("llvm.compiler.used");6780  EXPECT_NE(Used, nullptr);6781  Constant *UsedInit = Used->getInitializer();6782  EXPECT_NE(UsedInit, nullptr);6783  EXPECT_TRUE(isa<ConstantArray>(UsedInit));6784  auto *UsedInitData = cast<ConstantArray>(UsedInit);6785  EXPECT_EQ(1U, UsedInitData->getNumOperands());6786  Constant *ExecMode = UsedInitData->getOperand(0);6787  EXPECT_TRUE(isa<GlobalVariable>(ExecMode));6788  Constant *ExecModeValue = cast<GlobalVariable>(ExecMode)->getInitializer();6789  EXPECT_NE(ExecModeValue, nullptr);6790  EXPECT_TRUE(isa<ConstantInt>(ExecModeValue));6791  EXPECT_EQ(OMP_TGT_EXEC_MODE_GENERIC,6792            cast<ConstantInt>(ExecModeValue)->getZExtValue());6793}6794 6795TEST_F(OpenMPIRBuilderTest, TargetRegionSPMD) {6796  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;6797  OpenMPIRBuilder OMPBuilder(*M);6798  OMPBuilder.initialize();6799  OpenMPIRBuilderConfig Config(/*IsTargetDevice=*/false, /*IsGPU=*/false,6800                               /*OpenMPOffloadMandatory=*/false,6801                               /*HasRequiresReverseOffload=*/false,6802                               /*HasRequiresUnifiedAddress=*/false,6803                               /*HasRequiresUnifiedSharedMemory=*/false,6804                               /*HasRequiresDynamicAllocators=*/false);6805  OMPBuilder.setConfig(Config);6806  F->setName("func");6807  IRBuilder<> Builder(BB);6808 6809  auto CustomMapperCB = [&](unsigned int I) { return nullptr; };6810  auto BodyGenCB = [&](InsertPointTy,6811                       InsertPointTy CodeGenIP) -> InsertPointTy {6812    Builder.restoreIP(CodeGenIP);6813    return Builder.saveIP();6814  };6815 6816  auto SimpleArgAccessorCB = [&](Argument &, Value *, Value *&,6817                                 OpenMPIRBuilder::InsertPointTy,6818                                 OpenMPIRBuilder::InsertPointTy CodeGenIP) {6819    Builder.restoreIP(CodeGenIP);6820    return Builder.saveIP();6821  };6822 6823  SmallVector<Value *> Inputs;6824  OpenMPIRBuilder::MapInfosTy CombinedInfos;6825  auto GenMapInfoCB =6826      [&](OpenMPIRBuilder::InsertPointTy) -> OpenMPIRBuilder::MapInfosTy & {6827    return CombinedInfos;6828  };6829 6830  TargetRegionEntryInfo EntryInfo("func", 42, 4711, 17);6831  OpenMPIRBuilder::LocationDescription OmpLoc({Builder.saveIP(), DL});6832  OpenMPIRBuilder::TargetKernelRuntimeAttrs RuntimeAttrs;6833  OpenMPIRBuilder::TargetKernelDefaultAttrs DefaultAttrs = {6834      /*ExecFlags=*/omp::OMPTgtExecModeFlags::OMP_TGT_EXEC_MODE_SPMD,6835      /*MaxTeams=*/{-1}, /*MinTeams=*/0, /*MaxThreads=*/{0}, /*MinThreads=*/0};6836  RuntimeAttrs.LoopTripCount = Builder.getInt64(1000);6837  llvm::OpenMPIRBuilder::TargetDataInfo Info(6838      /*RequiresDevicePointerInfo=*/false,6839      /*SeparateBeginEndCalls=*/true);6840 6841  ASSERT_EXPECTED_INIT(6842      OpenMPIRBuilder::InsertPointTy, AfterIP,6843      OMPBuilder.createTarget(OmpLoc, /*IsOffloadEntry=*/true, Builder.saveIP(),6844                              Builder.saveIP(), Info, EntryInfo, DefaultAttrs,6845                              RuntimeAttrs, /*IfCond=*/nullptr, Inputs,6846                              GenMapInfoCB, BodyGenCB, SimpleArgAccessorCB,6847                              CustomMapperCB, {}));6848  Builder.restoreIP(AfterIP);6849 6850  OMPBuilder.finalize();6851  Builder.CreateRetVoid();6852 6853  // Check the kernel launch sequence6854  auto Iter = F->getEntryBlock().rbegin();6855  EXPECT_TRUE(isa<BranchInst>(&*(Iter)));6856  BranchInst *Branch = dyn_cast<BranchInst>(&*(Iter));6857  EXPECT_TRUE(isa<CmpInst>(&*(++Iter)));6858  EXPECT_TRUE(isa<CallInst>(&*(++Iter)));6859  CallInst *Call = dyn_cast<CallInst>(&*(Iter));6860 6861  // Check that the kernel launch function is called6862  Function *KernelLaunchFunc = Call->getCalledFunction();6863  EXPECT_NE(KernelLaunchFunc, nullptr);6864  StringRef FunctionName = KernelLaunchFunc->getName();6865  EXPECT_TRUE(FunctionName.starts_with("__tgt_target_kernel"));6866 6867  // Check the trip count kernel argument (use number 5 starting from the end6868  // and counting the call to __tgt_target_kernel as the first use)6869  Value *KernelArgs = Call->getArgOperand(Call->arg_size() - 1);6870  EXPECT_TRUE(KernelArgs->hasNUsesOrMore(6));6871  Value *TripCountGetElemPtr = *std::next(KernelArgs->user_begin(), 5);6872  EXPECT_TRUE(isa<GetElementPtrInst>(TripCountGetElemPtr));6873  Value *TripCountStore = TripCountGetElemPtr->getUniqueUndroppableUser();6874  EXPECT_TRUE(isa<StoreInst>(TripCountStore));6875  Value *TripCountStoreArg = cast<StoreInst>(TripCountStore)->getValueOperand();6876  EXPECT_TRUE(isa<ConstantInt>(TripCountStoreArg));6877  EXPECT_EQ(1000U, cast<ConstantInt>(TripCountStoreArg)->getZExtValue());6878 6879  // Check the fallback call6880  BasicBlock *FallbackBlock = Branch->getSuccessor(0);6881  Iter = FallbackBlock->rbegin();6882  CallInst *FCall = dyn_cast<CallInst>(&*(++Iter));6883  // 'F' has a dummy DISubprogram which causes OutlinedFunc to also6884  // have a DISubprogram. In this case, the call to OutlinedFunc needs6885  // to have a debug loc, otherwise verifier will complain.6886  FCall->setDebugLoc(DL);6887  EXPECT_NE(FCall, nullptr);6888 6889  // Check that the outlined function exists with the expected prefix6890  Function *OutlinedFunc = FCall->getCalledFunction();6891  EXPECT_NE(OutlinedFunc, nullptr);6892  StringRef FunctionName2 = OutlinedFunc->getName();6893  EXPECT_TRUE(FunctionName2.starts_with("__omp_offloading"));6894 6895  EXPECT_FALSE(verifyModule(*M, &errs()));6896}6897 6898TEST_F(OpenMPIRBuilderTest, TargetRegionDeviceSPMD) {6899  OpenMPIRBuilder OMPBuilder(*M);6900  OMPBuilder.setConfig(6901      OpenMPIRBuilderConfig(/*IsTargetDevice=*/true, /*IsGPU=*/false,6902                            /*OpenMPOffloadMandatory=*/false,6903                            /*HasRequiresReverseOffload=*/false,6904                            /*HasRequiresUnifiedAddress=*/false,6905                            /*HasRequiresUnifiedSharedMemory=*/false,6906                            /*HasRequiresDynamicAllocators=*/false));6907  OMPBuilder.initialize();6908  F->setName("func");6909  IRBuilder<> Builder(BB);6910  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});6911 6912  Function *OutlinedFn = nullptr;6913  SmallVector<Value *> CapturedArgs;6914 6915  auto SimpleArgAccessorCB = [&](Argument &, Value *, Value *&,6916                                 OpenMPIRBuilder::InsertPointTy,6917                                 OpenMPIRBuilder::InsertPointTy CodeGenIP) {6918    Builder.restoreIP(CodeGenIP);6919    return Builder.saveIP();6920  };6921 6922  OpenMPIRBuilder::MapInfosTy CombinedInfos;6923  auto GenMapInfoCB =6924      [&](OpenMPIRBuilder::InsertPointTy) -> OpenMPIRBuilder::MapInfosTy & {6925    return CombinedInfos;6926  };6927 6928  auto CustomMapperCB = [&](unsigned int I) { return nullptr; };6929  auto BodyGenCB = [&](OpenMPIRBuilder::InsertPointTy,6930                       OpenMPIRBuilder::InsertPointTy CodeGenIP)6931      -> OpenMPIRBuilder::InsertPointTy {6932    Builder.restoreIP(CodeGenIP);6933    OutlinedFn = CodeGenIP.getBlock()->getParent();6934    return Builder.saveIP();6935  };6936 6937  IRBuilder<>::InsertPoint EntryIP(&F->getEntryBlock(),6938                                   F->getEntryBlock().getFirstInsertionPt());6939  TargetRegionEntryInfo EntryInfo("parent", /*DeviceID=*/1, /*FileID=*/2,6940                                  /*Line=*/3, /*Count=*/0);6941  OpenMPIRBuilder::TargetKernelRuntimeAttrs RuntimeAttrs;6942  OpenMPIRBuilder::TargetKernelDefaultAttrs DefaultAttrs = {6943      /*ExecFlags=*/omp::OMPTgtExecModeFlags::OMP_TGT_EXEC_MODE_SPMD,6944      /*MaxTeams=*/{-1}, /*MinTeams=*/0, /*MaxThreads=*/{0}, /*MinThreads=*/0};6945  llvm::OpenMPIRBuilder::TargetDataInfo Info(6946      /*RequiresDevicePointerInfo=*/false,6947      /*SeparateBeginEndCalls=*/true);6948 6949  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,6950                       OMPBuilder.createTarget(6951                           Loc, /*IsOffloadEntry=*/true, EntryIP, EntryIP, Info,6952                           EntryInfo, DefaultAttrs, RuntimeAttrs,6953                           /*IfCond=*/nullptr, CapturedArgs, GenMapInfoCB,6954                           BodyGenCB, SimpleArgAccessorCB, CustomMapperCB, {}));6955  Builder.restoreIP(AfterIP);6956 6957  Builder.CreateRetVoid();6958  OMPBuilder.finalize();6959 6960  // Check outlined function6961  EXPECT_FALSE(verifyModule(*M, &errs()));6962  EXPECT_NE(OutlinedFn, nullptr);6963  EXPECT_NE(F, OutlinedFn);6964 6965  // Check that target-cpu and target-features were propagated to the outlined6966  // function6967  EXPECT_EQ(OutlinedFn->getFnAttribute("target-cpu"),6968            F->getFnAttribute("target-cpu"));6969  EXPECT_EQ(OutlinedFn->getFnAttribute("target-features"),6970            F->getFnAttribute("target-features"));6971 6972  EXPECT_TRUE(OutlinedFn->hasWeakODRLinkage());6973  // Account for the "implicit" first argument.6974  EXPECT_EQ(OutlinedFn->getName(), "__omp_offloading_1_2_parent_l3");6975  EXPECT_EQ(OutlinedFn->arg_size(), 1U);6976 6977  // Check global exec_mode.6978  GlobalVariable *Used = M->getGlobalVariable("llvm.compiler.used");6979  EXPECT_NE(Used, nullptr);6980  Constant *UsedInit = Used->getInitializer();6981  EXPECT_NE(UsedInit, nullptr);6982  EXPECT_TRUE(isa<ConstantArray>(UsedInit));6983  auto *UsedInitData = cast<ConstantArray>(UsedInit);6984  EXPECT_EQ(1U, UsedInitData->getNumOperands());6985  Constant *ExecMode = UsedInitData->getOperand(0);6986  EXPECT_TRUE(isa<GlobalVariable>(ExecMode));6987  Constant *ExecModeValue = cast<GlobalVariable>(ExecMode)->getInitializer();6988  EXPECT_NE(ExecModeValue, nullptr);6989  EXPECT_TRUE(isa<ConstantInt>(ExecModeValue));6990  EXPECT_EQ(OMP_TGT_EXEC_MODE_SPMD,6991            cast<ConstantInt>(ExecModeValue)->getZExtValue());6992}6993 6994TEST_F(OpenMPIRBuilderTest, ConstantAllocaRaise) {6995  OpenMPIRBuilder OMPBuilder(*M);6996  OMPBuilder.setConfig(6997      OpenMPIRBuilderConfig(true, false, false, false, false, false, false));6998  OMPBuilder.initialize();6999 7000  F->setName("func");7001  IRBuilder<> Builder(BB);7002  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});7003  Builder.SetCurrentDebugLocation(DL);7004 7005  LoadInst *Value = nullptr;7006  StoreInst *TargetStore = nullptr;7007  llvm::SmallVector<llvm::Value *, 1> CapturedArgs = {7008      Constant::getNullValue(PointerType::get(Ctx, 0))};7009 7010  auto SimpleArgAccessorCB =7011      [&](llvm::Argument &Arg, llvm::Value *Input, llvm::Value *&RetVal,7012          llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,7013          llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP) {7014        IRBuilderBase::InsertPointGuard guard(Builder);7015        Builder.SetCurrentDebugLocation(llvm::DebugLoc());7016        if (!OMPBuilder.Config.isTargetDevice()) {7017          RetVal = cast<llvm::Value>(&Arg);7018          return CodeGenIP;7019        }7020 7021        Builder.restoreIP(AllocaIP);7022 7023        llvm::Value *Addr = Builder.CreateAlloca(7024            Arg.getType()->isPointerTy()7025                ? Arg.getType()7026                : Type::getInt64Ty(Builder.getContext()),7027            OMPBuilder.M.getDataLayout().getAllocaAddrSpace());7028        llvm::Value *AddrAscast =7029            Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Input->getType());7030        Builder.CreateStore(&Arg, AddrAscast);7031 7032        Builder.restoreIP(CodeGenIP);7033 7034        RetVal = Builder.CreateLoad(Arg.getType(), AddrAscast);7035 7036        return Builder.saveIP();7037      };7038 7039  llvm::OpenMPIRBuilder::MapInfosTy CombinedInfos;7040  auto GenMapInfoCB = [&](llvm::OpenMPIRBuilder::InsertPointTy codeGenIP)7041      -> llvm::OpenMPIRBuilder::MapInfosTy & {7042    CreateDefaultMapInfos(OMPBuilder, CapturedArgs, CombinedInfos);7043    return CombinedInfos;7044  };7045 7046  llvm::Value *RaiseAlloca = nullptr;7047 7048  auto CustomMapperCB = [&](unsigned int I) { return nullptr; };7049  auto BodyGenCB = [&](OpenMPIRBuilder::InsertPointTy AllocaIP,7050                       OpenMPIRBuilder::InsertPointTy CodeGenIP)7051      -> OpenMPIRBuilder::InsertPointTy {7052    IRBuilderBase::InsertPointGuard guard(Builder);7053    Builder.SetCurrentDebugLocation(llvm::DebugLoc());7054    Builder.restoreIP(CodeGenIP);7055    RaiseAlloca = Builder.CreateAlloca(Builder.getInt32Ty());7056    Value = Builder.CreateLoad(Type::getInt32Ty(Ctx), CapturedArgs[0]);7057    TargetStore = Builder.CreateStore(Value, RaiseAlloca);7058    return Builder.saveIP();7059  };7060 7061  IRBuilder<>::InsertPoint EntryIP(&F->getEntryBlock(),7062                                   F->getEntryBlock().getFirstInsertionPt());7063  TargetRegionEntryInfo EntryInfo("parent", /*DeviceID=*/1, /*FileID=*/2,7064                                  /*Line=*/3, /*Count=*/0);7065  OpenMPIRBuilder::TargetKernelRuntimeAttrs RuntimeAttrs;7066  OpenMPIRBuilder::TargetKernelDefaultAttrs DefaultAttrs = {7067      /*ExecFlags=*/omp::OMPTgtExecModeFlags::OMP_TGT_EXEC_MODE_GENERIC,7068      /*MaxTeams=*/{-1}, /*MinTeams=*/0, /*MaxThreads=*/{0}, /*MinThreads=*/0};7069  llvm::OpenMPIRBuilder::TargetDataInfo Info(7070      /*RequiresDevicePointerInfo=*/false,7071      /*SeparateBeginEndCalls=*/true);7072 7073  ASSERT_EXPECTED_INIT(7074      OpenMPIRBuilder::InsertPointTy, AfterIP,7075      OMPBuilder.createTarget(Loc, /*IsOffloadEntry=*/true, EntryIP, EntryIP,7076                              Info, EntryInfo, DefaultAttrs, RuntimeAttrs,7077                              /*IfCond=*/nullptr, CapturedArgs, GenMapInfoCB,7078                              BodyGenCB, SimpleArgAccessorCB, CustomMapperCB,7079                              {}, false));7080  EXPECT_EQ(DL, Builder.getCurrentDebugLocation());7081  Builder.restoreIP(AfterIP);7082 7083  Builder.CreateRetVoid();7084  OMPBuilder.finalize();7085 7086  // Check outlined function7087  EXPECT_FALSE(verifyModule(*M, &errs()));7088  EXPECT_NE(TargetStore, nullptr);7089  Function *OutlinedFn = TargetStore->getFunction();7090  EXPECT_NE(F, OutlinedFn);7091 7092  EXPECT_TRUE(OutlinedFn->hasWeakODRLinkage());7093  // Account for the "implicit" first argument.7094  EXPECT_EQ(OutlinedFn->getName(), "__omp_offloading_1_2_parent_l3");7095  EXPECT_EQ(OutlinedFn->arg_size(), 2U);7096  EXPECT_TRUE(OutlinedFn->getArg(1)->getType()->isPointerTy());7097 7098  // Check entry block, to see if we have raised our alloca7099  // from the body to the entry block.7100  auto &EntryBlock = OutlinedFn->getEntryBlock();7101 7102  // Check that we have moved our alloca created in the7103  // BodyGenCB function, to the top of the function.7104  Instruction *Alloca1 = &*EntryBlock.getFirstNonPHIIt();7105  EXPECT_NE(Alloca1, nullptr);7106  EXPECT_TRUE(isa<AllocaInst>(Alloca1));7107  EXPECT_EQ(Alloca1, RaiseAlloca);7108 7109  // Verify we have not altered the rest of the function7110  // inappropriately with our alloca movement.7111  auto *Alloca2 = Alloca1->getNextNode();7112  EXPECT_TRUE(isa<AllocaInst>(Alloca2));7113  auto *Store2 = Alloca2->getNextNode();7114  EXPECT_TRUE(isa<StoreInst>(Store2));7115 7116  auto *InitCall = dyn_cast<CallInst>(Store2->getNextNode());7117  EXPECT_NE(InitCall, nullptr);7118  EXPECT_EQ(InitCall->getCalledFunction()->getName(), "__kmpc_target_init");7119  EXPECT_EQ(InitCall->arg_size(), 2U);7120  EXPECT_TRUE(isa<GlobalVariable>(InitCall->getArgOperand(0)));7121  auto *KernelEnvGV = cast<GlobalVariable>(InitCall->getArgOperand(0));7122  EXPECT_TRUE(isa<ConstantStruct>(KernelEnvGV->getInitializer()));7123  auto *KernelEnvC = cast<ConstantStruct>(KernelEnvGV->getInitializer());7124  EXPECT_TRUE(isa<ConstantStruct>(KernelEnvC->getAggregateElement(0U)));7125  auto *ConfigC = cast<ConstantStruct>(KernelEnvC->getAggregateElement(0U));7126  EXPECT_EQ(ConfigC->getAggregateElement(0U),7127            ConstantInt::get(Type::getInt8Ty(Ctx), true));7128  EXPECT_EQ(ConfigC->getAggregateElement(1U),7129            ConstantInt::get(Type::getInt8Ty(Ctx), true));7130  EXPECT_EQ(ConfigC->getAggregateElement(2U),7131            ConstantInt::get(Type::getInt8Ty(Ctx), OMP_TGT_EXEC_MODE_GENERIC));7132 7133  auto *EntryBlockBranch = EntryBlock.getTerminator();7134  EXPECT_NE(EntryBlockBranch, nullptr);7135  EXPECT_EQ(EntryBlockBranch->getNumSuccessors(), 2U);7136 7137  // Check user code block7138  auto *UserCodeBlock = EntryBlockBranch->getSuccessor(0);7139  EXPECT_EQ(UserCodeBlock->getName(), "user_code.entry");7140  BasicBlock::iterator Load1 = UserCodeBlock->getFirstNonPHIIt();7141  EXPECT_TRUE(isa<LoadInst>(Load1));7142 7143  auto *OutlinedBlockBr = Load1->getNextNode();7144  EXPECT_TRUE(isa<BranchInst>(OutlinedBlockBr));7145 7146  auto *OutlinedBlock = OutlinedBlockBr->getSuccessor(0);7147  EXPECT_EQ(OutlinedBlock->getName(), "outlined.body");7148 7149  Instruction *Load2 = &*OutlinedBlock->getFirstNonPHIIt();7150  EXPECT_TRUE(isa<LoadInst>(Load2));7151  EXPECT_EQ(Load2, Value);7152  EXPECT_EQ(Load2->getNextNode(), TargetStore);7153  auto *Deinit = TargetStore->getNextNode();7154  EXPECT_NE(Deinit, nullptr);7155 7156  auto *DeinitCall = dyn_cast<CallInst>(Deinit);7157  EXPECT_NE(DeinitCall, nullptr);7158  EXPECT_EQ(DeinitCall->getCalledFunction()->getName(), "__kmpc_target_deinit");7159  EXPECT_EQ(DeinitCall->arg_size(), 0U);7160 7161  EXPECT_TRUE(isa<ReturnInst>(DeinitCall->getNextNode()));7162 7163  // Check exit block7164  auto *ExitBlock = EntryBlockBranch->getSuccessor(1);7165  EXPECT_EQ(ExitBlock->getName(), "worker.exit");7166  EXPECT_TRUE(isa<ReturnInst>(ExitBlock->getFirstNonPHIIt()));7167}7168 7169TEST_F(OpenMPIRBuilderTest, CreateTask) {7170  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;7171  OpenMPIRBuilder OMPBuilder(*M);7172  OMPBuilder.Config.IsTargetDevice = false;7173  OMPBuilder.initialize();7174  F->setName("func");7175  IRBuilder<> Builder(BB);7176 7177  AllocaInst *ValPtr32 = Builder.CreateAlloca(Builder.getInt32Ty());7178  AllocaInst *ValPtr128 = Builder.CreateAlloca(Builder.getInt128Ty());7179  Value *Val128 =7180      Builder.CreateLoad(Builder.getInt128Ty(), ValPtr128, "bodygen.load");7181 7182  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {7183    Builder.restoreIP(AllocaIP);7184    AllocaInst *Local128 = Builder.CreateAlloca(Builder.getInt128Ty(), nullptr,7185                                                "bodygen.alloca128");7186 7187    Builder.restoreIP(CodeGenIP);7188    // Loading and storing captured pointer and values7189    Builder.CreateStore(Val128, Local128);7190    Value *Val32 = Builder.CreateLoad(ValPtr32->getAllocatedType(), ValPtr32,7191                                      "bodygen.load32");7192 7193    LoadInst *PrivLoad128 = Builder.CreateLoad(7194        Local128->getAllocatedType(), Local128, "bodygen.local.load128");7195    Value *Cmp = Builder.CreateICmpNE(7196        Val32, Builder.CreateTrunc(PrivLoad128, Val32->getType()));7197    Instruction *ThenTerm, *ElseTerm;7198    SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(),7199                                  &ThenTerm, &ElseTerm);7200    return Error::success();7201  };7202 7203  BasicBlock *AllocaBB = Builder.GetInsertBlock();7204  BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");7205  OpenMPIRBuilder::LocationDescription Loc(7206      InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);7207  ASSERT_EXPECTED_INIT(7208      OpenMPIRBuilder::InsertPointTy, AfterIP,7209      OMPBuilder.createTask(7210          Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),7211          BodyGenCB));7212  Builder.restoreIP(AfterIP);7213  OMPBuilder.finalize();7214  Builder.CreateRetVoid();7215 7216  EXPECT_FALSE(verifyModule(*M, &errs()));7217 7218  CallInst *TaskAllocCall = dyn_cast<CallInst>(7219      OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)7220          ->user_back());7221 7222  // Verify the Ident argument7223  GlobalVariable *Ident = cast<GlobalVariable>(TaskAllocCall->getArgOperand(0));7224  ASSERT_NE(Ident, nullptr);7225  EXPECT_TRUE(Ident->hasInitializer());7226  Constant *Initializer = Ident->getInitializer();7227  GlobalVariable *SrcStrGlob =7228      cast<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts());7229  ASSERT_NE(SrcStrGlob, nullptr);7230  ConstantDataArray *SrcSrc =7231      dyn_cast<ConstantDataArray>(SrcStrGlob->getInitializer());7232  ASSERT_NE(SrcSrc, nullptr);7233 7234  // Verify the num_threads argument.7235  CallInst *GTID = dyn_cast<CallInst>(TaskAllocCall->getArgOperand(1));7236  ASSERT_NE(GTID, nullptr);7237  EXPECT_EQ(GTID->arg_size(), 1U);7238  EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");7239 7240  // Verify the flags7241  // TODO: Check for others flags. Currently testing only for tiedness.7242  ConstantInt *Flags = dyn_cast<ConstantInt>(TaskAllocCall->getArgOperand(2));7243  ASSERT_NE(Flags, nullptr);7244  EXPECT_EQ(Flags->getSExtValue(), 1);7245 7246  // Verify the data size7247  ConstantInt *DataSize =7248      dyn_cast<ConstantInt>(TaskAllocCall->getArgOperand(3));7249  ASSERT_NE(DataSize, nullptr);7250  EXPECT_EQ(DataSize->getSExtValue(), 40);7251 7252  ConstantInt *SharedsSize =7253      dyn_cast<ConstantInt>(TaskAllocCall->getOperand(4));7254  EXPECT_EQ(SharedsSize->getSExtValue(),7255            24); // 64-bit pointer + 128-bit integer7256 7257  // Verify Wrapper function7258  Function *OutlinedFn =7259      dyn_cast<Function>(TaskAllocCall->getArgOperand(5)->stripPointerCasts());7260  ASSERT_NE(OutlinedFn, nullptr);7261 7262  LoadInst *SharedsLoad = dyn_cast<LoadInst>(OutlinedFn->begin()->begin());7263  ASSERT_NE(SharedsLoad, nullptr);7264  EXPECT_EQ(SharedsLoad->getPointerOperand(), OutlinedFn->getArg(1));7265 7266  EXPECT_FALSE(OutlinedFn->isDeclaration());7267  EXPECT_EQ(OutlinedFn->getArg(0)->getType(), Builder.getInt32Ty());7268 7269  // Verify that the data argument is used only once, and that too in the load7270  // instruction that is then used for accessing shared data.7271  Value *DataPtr = OutlinedFn->getArg(1);7272  EXPECT_TRUE(DataPtr->hasOneUse());7273  EXPECT_TRUE(isa<LoadInst>(DataPtr->uses().begin()->getUser()));7274  Value *Data = DataPtr->uses().begin()->getUser();7275  EXPECT_TRUE(all_of(Data->uses(), [](Use &U) {7276    return isa<GetElementPtrInst>(U.getUser());7277  }));7278 7279  // Verify the presence of `trunc` and `icmp` instructions in Outlined function7280  EXPECT_TRUE(any_of(instructions(OutlinedFn),7281                     [](Instruction &inst) { return isa<TruncInst>(&inst); }));7282  EXPECT_TRUE(any_of(instructions(OutlinedFn),7283                     [](Instruction &inst) { return isa<ICmpInst>(&inst); }));7284 7285  // Verify the execution of the task7286  CallInst *TaskCall = dyn_cast<CallInst>(7287      OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task)7288          ->user_back());7289  ASSERT_NE(TaskCall, nullptr);7290  EXPECT_EQ(TaskCall->getArgOperand(0), Ident);7291  EXPECT_EQ(TaskCall->getArgOperand(1), GTID);7292  EXPECT_EQ(TaskCall->getArgOperand(2), TaskAllocCall);7293 7294  // Verify that the argument data has been copied7295  for (User *in : TaskAllocCall->users()) {7296    if (MemCpyInst *memCpyInst = dyn_cast<MemCpyInst>(in)) {7297      EXPECT_EQ(memCpyInst->getDest(), TaskAllocCall);7298    }7299  }7300}7301 7302TEST_F(OpenMPIRBuilderTest, CreateTaskNoArgs) {7303  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;7304  OpenMPIRBuilder OMPBuilder(*M);7305  OMPBuilder.Config.IsTargetDevice = false;7306  OMPBuilder.initialize();7307  F->setName("func");7308  IRBuilder<> Builder(BB);7309 7310  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {7311    return Error::success();7312  };7313 7314  BasicBlock *AllocaBB = Builder.GetInsertBlock();7315  BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");7316  OpenMPIRBuilder::LocationDescription Loc(7317      InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);7318  ASSERT_EXPECTED_INIT(7319      OpenMPIRBuilder::InsertPointTy, AfterIP,7320      OMPBuilder.createTask(7321          Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),7322          BodyGenCB));7323  Builder.restoreIP(AfterIP);7324  OMPBuilder.finalize();7325  Builder.CreateRetVoid();7326 7327  EXPECT_FALSE(verifyModule(*M, &errs()));7328 7329  // Check that the outlined function has only one argument.7330  CallInst *TaskAllocCall = dyn_cast<CallInst>(7331      OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)7332          ->user_back());7333  Function *OutlinedFn = dyn_cast<Function>(TaskAllocCall->getArgOperand(5));7334  ASSERT_NE(OutlinedFn, nullptr);7335  ASSERT_EQ(OutlinedFn->arg_size(), 1U);7336}7337 7338TEST_F(OpenMPIRBuilderTest, CreateTaskUntied) {7339  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;7340  OpenMPIRBuilder OMPBuilder(*M);7341  OMPBuilder.Config.IsTargetDevice = false;7342  OMPBuilder.initialize();7343  F->setName("func");7344  IRBuilder<> Builder(BB);7345  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {7346    return Error::success();7347  };7348  BasicBlock *AllocaBB = Builder.GetInsertBlock();7349  BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");7350  OpenMPIRBuilder::LocationDescription Loc(7351      InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);7352  ASSERT_EXPECTED_INIT(7353      OpenMPIRBuilder::InsertPointTy, AfterIP,7354      OMPBuilder.createTask(7355          Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),7356          BodyGenCB,7357          /*Tied=*/false));7358  Builder.restoreIP(AfterIP);7359  OMPBuilder.finalize();7360  Builder.CreateRetVoid();7361 7362  // Check for the `Tied` argument7363  CallInst *TaskAllocCall = dyn_cast<CallInst>(7364      OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)7365          ->user_back());7366  ASSERT_NE(TaskAllocCall, nullptr);7367  ConstantInt *Flags = dyn_cast<ConstantInt>(TaskAllocCall->getArgOperand(2));7368  ASSERT_NE(Flags, nullptr);7369  EXPECT_EQ(Flags->getZExtValue() & 1U, 0U);7370 7371  EXPECT_FALSE(verifyModule(*M, &errs()));7372}7373 7374TEST_F(OpenMPIRBuilderTest, CreateTaskDepend) {7375  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;7376  OpenMPIRBuilder OMPBuilder(*M);7377  OMPBuilder.Config.IsTargetDevice = false;7378  OMPBuilder.initialize();7379  F->setName("func");7380  IRBuilder<> Builder(BB);7381  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {7382    return Error::success();7383  };7384  BasicBlock *AllocaBB = Builder.GetInsertBlock();7385  BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");7386  OpenMPIRBuilder::LocationDescription Loc(7387      InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);7388  AllocaInst *InDep = Builder.CreateAlloca(Type::getInt32Ty(M->getContext()));7389  SmallVector<OpenMPIRBuilder::DependData> DDS;7390  {7391    OpenMPIRBuilder::DependData DDIn(RTLDependenceKindTy::DepIn,7392                                     Type::getInt32Ty(M->getContext()), InDep);7393    DDS.push_back(DDIn);7394  }7395  ASSERT_EXPECTED_INIT(7396      OpenMPIRBuilder::InsertPointTy, AfterIP,7397      OMPBuilder.createTask(7398          Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),7399          BodyGenCB,7400          /*Tied=*/false, /*Final*/ nullptr, /*IfCondition*/ nullptr, DDS));7401  Builder.restoreIP(AfterIP);7402  OMPBuilder.finalize();7403  Builder.CreateRetVoid();7404 7405  // Check for the `NumDeps` argument7406  CallInst *TaskAllocCall = dyn_cast<CallInst>(7407      OMPBuilder7408          .getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_with_deps)7409          ->user_back());7410  ASSERT_NE(TaskAllocCall, nullptr);7411  ConstantInt *NumDeps = dyn_cast<ConstantInt>(TaskAllocCall->getArgOperand(3));7412  ASSERT_NE(NumDeps, nullptr);7413  EXPECT_EQ(NumDeps->getZExtValue(), 1U);7414 7415  // Check for the `DepInfo` array argument7416  AllocaInst *DepArray = dyn_cast<AllocaInst>(TaskAllocCall->getOperand(4));7417  ASSERT_NE(DepArray, nullptr);7418  Value::user_iterator DepArrayI = DepArray->user_begin();7419  ++DepArrayI;7420  Value::user_iterator DepInfoI = DepArrayI->user_begin();7421  // Check for the `DependKind` flag in the `DepInfo` array7422  Value *Flag = findStoredValue<GetElementPtrInst>(*DepInfoI);7423  ASSERT_NE(Flag, nullptr);7424  ConstantInt *FlagInt = dyn_cast<ConstantInt>(Flag);7425  ASSERT_NE(FlagInt, nullptr);7426  EXPECT_EQ(FlagInt->getZExtValue(),7427            static_cast<unsigned int>(RTLDependenceKindTy::DepIn));7428  ++DepInfoI;7429  // Check for the size in the `DepInfo` array7430  Value *Size = findStoredValue<GetElementPtrInst>(*DepInfoI);7431  ASSERT_NE(Size, nullptr);7432  ConstantInt *SizeInt = dyn_cast<ConstantInt>(Size);7433  ASSERT_NE(SizeInt, nullptr);7434  EXPECT_EQ(SizeInt->getZExtValue(), 4U);7435  ++DepInfoI;7436  // Check for the variable address in the `DepInfo` array7437  Value *AddrStored = findStoredValue<GetElementPtrInst>(*DepInfoI);7438  ASSERT_NE(AddrStored, nullptr);7439  PtrToIntInst *AddrInt = dyn_cast<PtrToIntInst>(AddrStored);7440  ASSERT_NE(AddrInt, nullptr);7441  Value *Addr = AddrInt->getPointerOperand();7442  EXPECT_EQ(Addr, InDep);7443 7444  ConstantInt *NumDepsNoAlias =7445      dyn_cast<ConstantInt>(TaskAllocCall->getArgOperand(5));7446  ASSERT_NE(NumDepsNoAlias, nullptr);7447  EXPECT_EQ(NumDepsNoAlias->getZExtValue(), 0U);7448  EXPECT_EQ(TaskAllocCall->getOperand(6),7449            ConstantPointerNull::get(PointerType::getUnqual(M->getContext())));7450 7451  EXPECT_FALSE(verifyModule(*M, &errs()));7452}7453 7454TEST_F(OpenMPIRBuilderTest, CreateTaskFinal) {7455  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;7456  OpenMPIRBuilder OMPBuilder(*M);7457  OMPBuilder.Config.IsTargetDevice = false;7458  OMPBuilder.initialize();7459  F->setName("func");7460  IRBuilder<> Builder(BB);7461  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {7462    return Error::success();7463  };7464  BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");7465  IRBuilderBase::InsertPoint AllocaIP = Builder.saveIP();7466  Builder.SetInsertPoint(BodyBB);7467  Value *Final = Builder.CreateICmp(7468      CmpInst::Predicate::ICMP_EQ, F->getArg(0),7469      ConstantInt::get(Type::getInt32Ty(M->getContext()), 0U));7470  OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);7471  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,7472                       OMPBuilder.createTask(Loc, AllocaIP, BodyGenCB,7473                                             /*Tied=*/false, Final));7474  Builder.restoreIP(AfterIP);7475  OMPBuilder.finalize();7476  Builder.CreateRetVoid();7477 7478  // Check for the `Tied` argument7479  CallInst *TaskAllocCall = dyn_cast<CallInst>(7480      OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)7481          ->user_back());7482  ASSERT_NE(TaskAllocCall, nullptr);7483  BinaryOperator *OrInst =7484      dyn_cast<BinaryOperator>(TaskAllocCall->getArgOperand(2));7485  ASSERT_NE(OrInst, nullptr);7486  EXPECT_EQ(OrInst->getOpcode(), BinaryOperator::BinaryOps::Or);7487 7488  // One of the arguments to `or` instruction is the tied flag, which is equal7489  // to zero.7490  EXPECT_TRUE(any_of(OrInst->operands(), [](Value *op) {7491    if (ConstantInt *TiedValue = dyn_cast<ConstantInt>(op))7492      return TiedValue->getSExtValue() == 0;7493    return false;7494  }));7495 7496  // One of the arguments to `or` instruction is the final condition.7497  EXPECT_TRUE(any_of(OrInst->operands(), [Final](Value *op) {7498    if (SelectInst *Select = dyn_cast<SelectInst>(op)) {7499      ConstantInt *TrueValue = dyn_cast<ConstantInt>(Select->getTrueValue());7500      ConstantInt *FalseValue = dyn_cast<ConstantInt>(Select->getFalseValue());7501      if (!TrueValue || !FalseValue)7502        return false;7503      return Select->getCondition() == Final &&7504             TrueValue->getSExtValue() == 2 && FalseValue->getSExtValue() == 0;7505    }7506    return false;7507  }));7508 7509  EXPECT_FALSE(verifyModule(*M, &errs()));7510}7511 7512TEST_F(OpenMPIRBuilderTest, CreateTaskIfCondition) {7513  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;7514  OpenMPIRBuilder OMPBuilder(*M);7515  OMPBuilder.Config.IsTargetDevice = false;7516  OMPBuilder.initialize();7517  F->setName("func");7518  IRBuilder<> Builder(BB);7519  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {7520    return Error::success();7521  };7522  BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");7523  IRBuilderBase::InsertPoint AllocaIP = Builder.saveIP();7524  Builder.SetInsertPoint(BodyBB);7525  Value *IfCondition = Builder.CreateICmp(7526      CmpInst::Predicate::ICMP_EQ, F->getArg(0),7527      ConstantInt::get(Type::getInt32Ty(M->getContext()), 0U));7528  OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);7529  ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, AfterIP,7530                       OMPBuilder.createTask(Loc, AllocaIP, BodyGenCB,7531                                             /*Tied=*/false, /*Final=*/nullptr,7532                                             IfCondition));7533  Builder.restoreIP(AfterIP);7534  OMPBuilder.finalize();7535  Builder.CreateRetVoid();7536 7537  EXPECT_FALSE(verifyModule(*M, &errs()));7538 7539  CallInst *TaskAllocCall = dyn_cast<CallInst>(7540      OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)7541          ->user_back());7542  ASSERT_NE(TaskAllocCall, nullptr);7543 7544  // Check the branching is based on the if condition argument.7545  BranchInst *IfConditionBranchInst =7546      dyn_cast<BranchInst>(TaskAllocCall->getParent()->getTerminator());7547  ASSERT_NE(IfConditionBranchInst, nullptr);7548  ASSERT_TRUE(IfConditionBranchInst->isConditional());7549  EXPECT_EQ(IfConditionBranchInst->getCondition(), IfCondition);7550 7551  // Check that the `__kmpc_omp_task` executes only in the then branch.7552  CallInst *TaskCall = dyn_cast<CallInst>(7553      OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task)7554          ->user_back());7555  ASSERT_NE(TaskCall, nullptr);7556  EXPECT_EQ(TaskCall->getParent(), IfConditionBranchInst->getSuccessor(0));7557 7558  // Check that the OpenMP Runtime Functions specific to `if` clause execute7559  // only in the else branch. Also check that the function call is between the7560  // `__kmpc_omp_task_begin_if0` and `__kmpc_omp_task_complete_if0` calls.7561  CallInst *TaskBeginIfCall = dyn_cast<CallInst>(7562      OMPBuilder7563          .getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_begin_if0)7564          ->user_back());7565  CallInst *TaskCompleteCall = dyn_cast<CallInst>(7566      OMPBuilder7567          .getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_complete_if0)7568          ->user_back());7569  ASSERT_NE(TaskBeginIfCall, nullptr);7570  ASSERT_NE(TaskCompleteCall, nullptr);7571  Function *OulinedFn =7572      dyn_cast<Function>(TaskAllocCall->getArgOperand(5)->stripPointerCasts());7573  ASSERT_NE(OulinedFn, nullptr);7574  CallInst *OulinedFnCall = dyn_cast<CallInst>(OulinedFn->user_back());7575  ASSERT_NE(OulinedFnCall, nullptr);7576  EXPECT_EQ(TaskBeginIfCall->getParent(),7577            IfConditionBranchInst->getSuccessor(1));7578 7579  EXPECT_EQ(TaskBeginIfCall->getNextNode(), OulinedFnCall);7580  EXPECT_EQ(OulinedFnCall->getNextNode(), TaskCompleteCall);7581}7582 7583TEST_F(OpenMPIRBuilderTest, CreateTaskgroup) {7584  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;7585  OpenMPIRBuilder OMPBuilder(*M);7586  OMPBuilder.initialize();7587  F->setName("func");7588  IRBuilder<> Builder(BB);7589 7590  AllocaInst *ValPtr32 = Builder.CreateAlloca(Builder.getInt32Ty());7591  AllocaInst *ValPtr128 = Builder.CreateAlloca(Builder.getInt128Ty());7592  Value *Val128 =7593      Builder.CreateLoad(Builder.getInt128Ty(), ValPtr128, "bodygen.load");7594  Instruction *ThenTerm, *ElseTerm;7595 7596  Value *InternalStoreInst, *InternalLoad32, *InternalLoad128, *InternalIfCmp;7597 7598  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {7599    Builder.restoreIP(AllocaIP);7600    AllocaInst *Local128 = Builder.CreateAlloca(Builder.getInt128Ty(), nullptr,7601                                                "bodygen.alloca128");7602 7603    Builder.restoreIP(CodeGenIP);7604    // Loading and storing captured pointer and values7605    InternalStoreInst = Builder.CreateStore(Val128, Local128);7606    InternalLoad32 = Builder.CreateLoad(ValPtr32->getAllocatedType(), ValPtr32,7607                                        "bodygen.load32");7608 7609    InternalLoad128 = Builder.CreateLoad(Local128->getAllocatedType(), Local128,7610                                         "bodygen.local.load128");7611    InternalIfCmp = Builder.CreateICmpNE(7612        InternalLoad32,7613        Builder.CreateTrunc(InternalLoad128, InternalLoad32->getType()));7614    SplitBlockAndInsertIfThenElse(InternalIfCmp,7615                                  CodeGenIP.getBlock()->getTerminator(),7616                                  &ThenTerm, &ElseTerm);7617    return Error::success();7618  };7619 7620  BasicBlock *AllocaBB = Builder.GetInsertBlock();7621  BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");7622  OpenMPIRBuilder::LocationDescription Loc(7623      InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);7624  ASSERT_EXPECTED_INIT(7625      OpenMPIRBuilder::InsertPointTy, AfterIP,7626      OMPBuilder.createTaskgroup(7627          Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),7628          BodyGenCB));7629  Builder.restoreIP(AfterIP);7630  OMPBuilder.finalize();7631  Builder.CreateRetVoid();7632 7633  EXPECT_FALSE(verifyModule(*M, &errs()));7634 7635  CallInst *TaskgroupCall = dyn_cast<CallInst>(7636      OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_taskgroup)7637          ->user_back());7638  ASSERT_NE(TaskgroupCall, nullptr);7639  CallInst *EndTaskgroupCall = dyn_cast<CallInst>(7640      OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_taskgroup)7641          ->user_back());7642  ASSERT_NE(EndTaskgroupCall, nullptr);7643 7644  // Verify the Ident argument7645  GlobalVariable *Ident = cast<GlobalVariable>(TaskgroupCall->getArgOperand(0));7646  ASSERT_NE(Ident, nullptr);7647  EXPECT_TRUE(Ident->hasInitializer());7648  Constant *Initializer = Ident->getInitializer();7649  GlobalVariable *SrcStrGlob =7650      cast<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts());7651  ASSERT_NE(SrcStrGlob, nullptr);7652  ConstantDataArray *SrcSrc =7653      dyn_cast<ConstantDataArray>(SrcStrGlob->getInitializer());7654  ASSERT_NE(SrcSrc, nullptr);7655 7656  // Verify the num_threads argument.7657  CallInst *GTID = dyn_cast<CallInst>(TaskgroupCall->getArgOperand(1));7658  ASSERT_NE(GTID, nullptr);7659  EXPECT_EQ(GTID->arg_size(), 1U);7660  EXPECT_EQ(GTID->getCalledFunction(), OMPBuilder.getOrCreateRuntimeFunctionPtr(7661                                           OMPRTL___kmpc_global_thread_num));7662 7663  // Checking the general structure of the IR generated is same as expected.7664  Instruction *GeneratedStoreInst = TaskgroupCall->getNextNode();7665  EXPECT_EQ(GeneratedStoreInst, InternalStoreInst);7666  Instruction *GeneratedLoad32 = GeneratedStoreInst->getNextNode();7667  EXPECT_EQ(GeneratedLoad32, InternalLoad32);7668  Instruction *GeneratedLoad128 = GeneratedLoad32->getNextNode();7669  EXPECT_EQ(GeneratedLoad128, InternalLoad128);7670 7671  // Checking the ordering because of the if statements and that7672  // `__kmp_end_taskgroup` call is after the if branching.7673  BasicBlock *RefOrder[] = {TaskgroupCall->getParent(), ThenTerm->getParent(),7674                            ThenTerm->getSuccessor(0),7675                            EndTaskgroupCall->getParent(),7676                            ElseTerm->getParent()};7677  verifyDFSOrder(F, RefOrder);7678}7679 7680TEST_F(OpenMPIRBuilderTest, CreateTaskgroupWithTasks) {7681  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;7682  OpenMPIRBuilder OMPBuilder(*M);7683  OMPBuilder.Config.IsTargetDevice = false;7684  OMPBuilder.initialize();7685  F->setName("func");7686  IRBuilder<> Builder(BB);7687 7688  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {7689    Builder.restoreIP(AllocaIP);7690    AllocaInst *Alloca32 =7691        Builder.CreateAlloca(Builder.getInt32Ty(), nullptr, "bodygen.alloca32");7692    AllocaInst *Alloca64 =7693        Builder.CreateAlloca(Builder.getInt64Ty(), nullptr, "bodygen.alloca64");7694    Builder.restoreIP(CodeGenIP);7695    auto TaskBodyGenCB1 = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {7696      Builder.restoreIP(CodeGenIP);7697      LoadInst *LoadValue =7698          Builder.CreateLoad(Alloca64->getAllocatedType(), Alloca64);7699      Value *AddInst = Builder.CreateAdd(LoadValue, Builder.getInt64(64));7700      Builder.CreateStore(AddInst, Alloca64);7701      return Error::success();7702    };7703    OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);7704    ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, TaskIP1,7705                         OMPBuilder.createTask(Loc, AllocaIP, TaskBodyGenCB1));7706    Builder.restoreIP(TaskIP1);7707 7708    auto TaskBodyGenCB2 = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {7709      Builder.restoreIP(CodeGenIP);7710      LoadInst *LoadValue =7711          Builder.CreateLoad(Alloca32->getAllocatedType(), Alloca32);7712      Value *AddInst = Builder.CreateAdd(LoadValue, Builder.getInt32(32));7713      Builder.CreateStore(AddInst, Alloca32);7714      return Error::success();7715    };7716    OpenMPIRBuilder::LocationDescription Loc2(Builder.saveIP(), DL);7717    ASSERT_EXPECTED_INIT(OpenMPIRBuilder::InsertPointTy, TaskIP2,7718                         OMPBuilder.createTask(Loc2, AllocaIP, TaskBodyGenCB2));7719    Builder.restoreIP(TaskIP2);7720  };7721 7722  BasicBlock *AllocaBB = Builder.GetInsertBlock();7723  BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");7724  OpenMPIRBuilder::LocationDescription Loc(7725      InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);7726  ASSERT_EXPECTED_INIT(7727      OpenMPIRBuilder::InsertPointTy, AfterIP,7728      OMPBuilder.createTaskgroup(7729          Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),7730          BODYGENCB_WRAPPER(BodyGenCB)));7731  Builder.restoreIP(AfterIP);7732  OMPBuilder.finalize();7733  Builder.CreateRetVoid();7734 7735  EXPECT_FALSE(verifyModule(*M, &errs()));7736 7737  CallInst *TaskgroupCall = dyn_cast<CallInst>(7738      OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_taskgroup)7739          ->user_back());7740  ASSERT_NE(TaskgroupCall, nullptr);7741  CallInst *EndTaskgroupCall = dyn_cast<CallInst>(7742      OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_taskgroup)7743          ->user_back());7744  ASSERT_NE(EndTaskgroupCall, nullptr);7745 7746  Function *TaskAllocFn =7747      OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc);7748  ASSERT_TRUE(TaskAllocFn->hasNUses(2));7749 7750  CallInst *FirstTaskAllocCall =7751      dyn_cast_or_null<CallInst>(*TaskAllocFn->users().begin());7752  CallInst *SecondTaskAllocCall =7753      dyn_cast_or_null<CallInst>(*TaskAllocFn->users().begin()++);7754  ASSERT_NE(FirstTaskAllocCall, nullptr);7755  ASSERT_NE(SecondTaskAllocCall, nullptr);7756 7757  // Verify that the tasks have been generated in order and inside taskgroup7758  // construct.7759  BasicBlock *RefOrder[] = {7760      TaskgroupCall->getParent(), FirstTaskAllocCall->getParent(),7761      SecondTaskAllocCall->getParent(), EndTaskgroupCall->getParent()};7762  verifyDFSOrder(F, RefOrder);7763}7764 7765TEST_F(OpenMPIRBuilderTest, EmitOffloadingArraysArguments) {7766  OpenMPIRBuilder OMPBuilder(*M);7767  OMPBuilder.initialize();7768 7769  IRBuilder<> Builder(BB);7770 7771  OpenMPIRBuilder::TargetDataRTArgs RTArgs;7772  OpenMPIRBuilder::TargetDataInfo Info(true, false);7773 7774  auto VoidPtrPtrTy = PointerType::getUnqual(Builder.getContext());7775  auto Int64PtrTy = PointerType::getUnqual(Builder.getContext());7776 7777  Info.RTArgs.BasePointersArray = ConstantPointerNull::get(Builder.getPtrTy(0));7778  Info.RTArgs.PointersArray = ConstantPointerNull::get(Builder.getPtrTy(0));7779  Info.RTArgs.SizesArray = ConstantPointerNull::get(Builder.getPtrTy(0));7780  Info.RTArgs.MapTypesArray = ConstantPointerNull::get(Builder.getPtrTy(0));7781  Info.RTArgs.MapNamesArray = ConstantPointerNull::get(Builder.getPtrTy(0));7782  Info.RTArgs.MappersArray = ConstantPointerNull::get(Builder.getPtrTy(0));7783  Info.NumberOfPtrs = 4;7784  Info.EmitDebug = false;7785  OMPBuilder.emitOffloadingArraysArgument(Builder, RTArgs, Info, false);7786 7787  EXPECT_NE(RTArgs.BasePointersArray, nullptr);7788  EXPECT_NE(RTArgs.PointersArray, nullptr);7789  EXPECT_NE(RTArgs.SizesArray, nullptr);7790  EXPECT_NE(RTArgs.MapTypesArray, nullptr);7791  EXPECT_NE(RTArgs.MappersArray, nullptr);7792  EXPECT_NE(RTArgs.MapNamesArray, nullptr);7793  EXPECT_EQ(RTArgs.MapTypesArrayEnd, nullptr);7794 7795  EXPECT_EQ(RTArgs.BasePointersArray->getType(), VoidPtrPtrTy);7796  EXPECT_EQ(RTArgs.PointersArray->getType(), VoidPtrPtrTy);7797  EXPECT_EQ(RTArgs.SizesArray->getType(), Int64PtrTy);7798  EXPECT_EQ(RTArgs.MapTypesArray->getType(), Int64PtrTy);7799  EXPECT_EQ(RTArgs.MappersArray->getType(), VoidPtrPtrTy);7800  EXPECT_EQ(RTArgs.MapNamesArray->getType(), VoidPtrPtrTy);7801}7802 7803TEST_F(OpenMPIRBuilderTest, OffloadEntriesInfoManager) {7804  OpenMPIRBuilder OMPBuilder(*M);7805  OMPBuilder.setConfig(7806      OpenMPIRBuilderConfig(true, false, false, false, false, false, false));7807  OffloadEntriesInfoManager &InfoManager = OMPBuilder.OffloadInfoManager;7808  TargetRegionEntryInfo EntryInfo("parent", 1, 2, 4, 0);7809  InfoManager.initializeTargetRegionEntryInfo(EntryInfo, 0);7810  EXPECT_TRUE(InfoManager.hasTargetRegionEntryInfo(EntryInfo));7811  InfoManager.initializeDeviceGlobalVarEntryInfo(7812      "gvar", OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo, 0);7813  InfoManager.registerTargetRegionEntryInfo(7814      EntryInfo, nullptr, nullptr,7815      OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion);7816  InfoManager.registerDeviceGlobalVarEntryInfo(7817      "gvar", 0x0, 8, OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo,7818      GlobalValue::WeakAnyLinkage);7819  EXPECT_TRUE(InfoManager.hasDeviceGlobalVarEntryInfo("gvar"));7820}7821 7822// Tests both registerTargetGlobalVariable and getAddrOfDeclareTargetVar as they7823// call each other (recursively in some cases). The test case test these7824// functions by utilising them for host code generation for declare target7825// global variables7826TEST_F(OpenMPIRBuilderTest, registerTargetGlobalVariable) {7827  OpenMPIRBuilder OMPBuilder(*M);7828  OMPBuilder.initialize();7829  OpenMPIRBuilderConfig Config(false, false, false, false, false, false, false);7830  OMPBuilder.setConfig(Config);7831 7832  std::vector<llvm::Triple> TargetTriple;7833  TargetTriple.emplace_back("amdgcn-amd-amdhsa");7834 7835  TargetRegionEntryInfo EntryInfo("", 42, 4711, 17);7836  std::vector<GlobalVariable *> RefsGathered;7837 7838  std::vector<Constant *> Globals;7839  auto *IntTy = Type::getInt32Ty(Ctx);7840  for (int I = 0; I < 2; ++I) {7841    Globals.push_back(M->getOrInsertGlobal(7842        "test_data_int_" + std::to_string(I), IntTy, [&]() -> GlobalVariable * {7843          return new GlobalVariable(7844              *M, IntTy, false, GlobalValue::LinkageTypes::WeakAnyLinkage,7845              ConstantInt::get(IntTy, I), "test_data_int_" + std::to_string(I));7846        }));7847  }7848 7849  OMPBuilder.registerTargetGlobalVariable(7850      OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo,7851      OffloadEntriesInfoManager::OMPTargetDeviceClauseAny, false, true,7852      EntryInfo, Globals[0]->getName(), RefsGathered, false, TargetTriple,7853      nullptr, nullptr, Globals[0]->getType(), Globals[0]);7854 7855  OMPBuilder.registerTargetGlobalVariable(7856      OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink,7857      OffloadEntriesInfoManager::OMPTargetDeviceClauseAny, false, true,7858      EntryInfo, Globals[1]->getName(), RefsGathered, false, TargetTriple,7859      nullptr, nullptr, Globals[1]->getType(), Globals[1]);7860 7861  llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy &&ErrorReportfn =7862      [](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind,7863         const llvm::TargetRegionEntryInfo &EntryInfo) -> void {7864    // If this is invoked, then we want to emit an error, even if it is not7865    // neccesarily the most readable, as something has went wrong. The7866    // test-suite unfortunately eats up all error output7867    ASSERT_EQ(Kind, Kind);7868  };7869 7870  OMPBuilder.createOffloadEntriesAndInfoMetadata(ErrorReportfn);7871 7872  // Clauses for data_int_0 with To + Any clauses for the host7873  std::vector<GlobalVariable *> OffloadEntries;7874  OffloadEntries.push_back(M->getNamedGlobal(".offloading.entry_name"));7875  OffloadEntries.push_back(7876      M->getNamedGlobal(".offloading.entry.test_data_int_0"));7877 7878  // Clauses for data_int_1 with Link + Any clauses for the host7879  OffloadEntries.push_back(7880      M->getNamedGlobal("test_data_int_1_decl_tgt_ref_ptr"));7881  OffloadEntries.push_back(M->getNamedGlobal(".offloading.entry_name.1"));7882  OffloadEntries.push_back(7883      M->getNamedGlobal(".offloading.entry.test_data_int_1_decl_tgt_ref_ptr"));7884 7885  for (unsigned I = 0; I < OffloadEntries.size(); ++I)7886    EXPECT_NE(OffloadEntries[I], nullptr);7887 7888  // Metadata generated for the host offload module7889  NamedMDNode *OffloadMetadata = M->getNamedMetadata("omp_offload.info");7890  ASSERT_THAT(OffloadMetadata, testing::NotNull());7891  StringRef Nodes[2] = {7892      cast<MDString>(OffloadMetadata->getOperand(0)->getOperand(1))7893          ->getString(),7894      cast<MDString>(OffloadMetadata->getOperand(1)->getOperand(1))7895          ->getString()};7896  EXPECT_THAT(7897      Nodes, testing::UnorderedElementsAre("test_data_int_0",7898                                           "test_data_int_1_decl_tgt_ref_ptr"));7899}7900 7901TEST_F(OpenMPIRBuilderTest, createGPUOffloadEntry) {7902  OpenMPIRBuilder OMPBuilder(*M);7903  OMPBuilder.initialize();7904  OpenMPIRBuilderConfig Config(/* IsTargetDevice = */ true,7905                               /* IsGPU = */ true,7906                               /* OpenMPOffloadMandatory = */ false,7907                               /* HasRequiresReverseOffload = */ false,7908                               /* HasRequiresUnifiedAddress = */ false,7909                               /* HasRequiresUnifiedSharedMemory = */ false,7910                               /* HasRequiresDynamicAllocators = */ false);7911  OMPBuilder.setConfig(Config);7912 7913  FunctionCallee FnTypeAndCallee =7914      M->getOrInsertFunction("test_kernel", Type::getVoidTy(Ctx));7915 7916  auto *Fn = cast<Function>(FnTypeAndCallee.getCallee());7917  OMPBuilder.createOffloadEntry(/* ID = */ nullptr, Fn,7918                                /* Size = */ 0,7919                                /* Flags = */ 0, GlobalValue::WeakAnyLinkage);7920 7921  // Check kernel attributes7922  EXPECT_TRUE(Fn->hasFnAttribute("kernel"));7923  EXPECT_TRUE(Fn->hasFnAttribute(Attribute::MustProgress));7924}7925 7926TEST_F(OpenMPIRBuilderTest, splitBB) {7927  OpenMPIRBuilder OMPBuilder(*M);7928  OMPBuilder.Config.IsTargetDevice = false;7929  OMPBuilder.initialize();7930  F->setName("func");7931  IRBuilder<> Builder(BB);7932 7933  Builder.SetCurrentDebugLocation(DL);7934  AllocaInst *alloc = Builder.CreateAlloca(Builder.getInt32Ty());7935  EXPECT_TRUE(DL == alloc->getStableDebugLoc());7936  BasicBlock *AllocaBB = Builder.GetInsertBlock();7937  splitBB(Builder, /*CreateBranch=*/true, "test");7938  if (AllocaBB->getTerminator())7939    EXPECT_TRUE(DL == AllocaBB->getTerminator()->getStableDebugLoc());7940}7941 7942TEST_F(OpenMPIRBuilderTest, spliceBBWithEmptyBB) {7943  OpenMPIRBuilder OMPBuilder(*M);7944  OMPBuilder.Config.IsTargetDevice = false;7945  OMPBuilder.initialize();7946  F->setName("func");7947  IRBuilder<> Builder(BB);7948 7949  // Test calling spliceBB with an empty Block (but having trailing debug7950  // records).7951  DIBuilder DIB(*M);7952  DISubprogram *SP = F->getSubprogram();7953  DIType *VoidPtrTy =7954      DIB.createQualifiedType(dwarf::DW_TAG_pointer_type, nullptr);7955  DILocalVariable *Var = DIB.createParameterVariable(7956      SP, "test", /*ArgNo*/ 1, SP->getFile(), /*LineNo=*/0, VoidPtrTy);7957  DIB.insertDeclare(F->getArg(0), Var, DIB.createExpression(), DL,7958                    Builder.GetInsertPoint());7959  BasicBlock *New = BasicBlock::Create(Ctx, "", F);7960  spliceBB(Builder.saveIP(), New, true, DL);7961  Instruction *Terminator = BB->getTerminator();7962  EXPECT_NE(Terminator, nullptr);7963  EXPECT_FALSE(Terminator->getDbgRecordRange().empty());7964}7965 7966} // namespace7967