1276 lines · cpp
1/*2 * ompt-tsan.cpp -- Archer runtime library, TSan annotations for Archer3 */4 5//===----------------------------------------------------------------------===//6//7// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.8// See https://llvm.org/LICENSE.txt for details.9// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception10//11//===----------------------------------------------------------------------===//12 13#ifndef __STDC_FORMAT_MACROS14#define __STDC_FORMAT_MACROS15#endif16 17#include <algorithm>18#include <atomic>19#include <cassert>20#include <cstdlib>21#include <cstring>22#include <dlfcn.h>23#include <inttypes.h>24#include <iostream>25#include <list>26#include <mutex>27#include <sstream>28#include <string>29#include <sys/resource.h>30#include <unistd.h>31#include <unordered_map>32#include <vector>33 34#include "omp-tools.h"35 36// Define attribute that indicates that the fall through from the previous37// case label is intentional and should not be diagnosed by a compiler38// Code from libcxx/include/__config39// Use a function like macro to imply that it must be followed by a semicolon40#if __cplusplus > 201402L && __has_cpp_attribute(fallthrough)41#define KMP_FALLTHROUGH() [[fallthrough]]42// icc cannot properly tell this attribute is absent so force off43#elif defined(__INTEL_COMPILER)44#define KMP_FALLTHROUGH() ((void)0)45#elif __has_cpp_attribute(clang::fallthrough)46#define KMP_FALLTHROUGH() [[clang::fallthrough]]47#elif __has_attribute(fallthrough) || __GNUC__ >= 748#define KMP_FALLTHROUGH() __attribute__((__fallthrough__))49#else50#define KMP_FALLTHROUGH() ((void)0)51#endif52 53static int hasReductionCallback;54 55namespace {56class ArcherFlags {57public:58#if (LLVM_VERSION) >= 4059 int flush_shadow{0};60#endif61 int print_max_rss{0};62 int verbose{0};63 int enabled{1};64 int report_data_leak{0};65 int ignore_serial{0};66 std::atomic<int> all_memory{0};67 68 ArcherFlags(const char *env) {69 if (env) {70 std::vector<std::string> tokens;71 std::string token;72 std::string str(env);73 std::istringstream iss(str);74 int tmp_int;75 while (std::getline(iss, token, ' '))76 tokens.push_back(token);77 78 for (std::vector<std::string>::iterator it = tokens.begin();79 it != tokens.end(); ++it) {80#if (LLVM_VERSION) >= 4081 if (sscanf(it->c_str(), "flush_shadow=%d", &flush_shadow))82 continue;83#endif84 if (sscanf(it->c_str(), "print_max_rss=%d", &print_max_rss))85 continue;86 if (sscanf(it->c_str(), "verbose=%d", &verbose))87 continue;88 if (sscanf(it->c_str(), "report_data_leak=%d", &report_data_leak))89 continue;90 if (sscanf(it->c_str(), "enable=%d", &enabled))91 continue;92 if (sscanf(it->c_str(), "ignore_serial=%d", &ignore_serial))93 continue;94 if (sscanf(it->c_str(), "all_memory=%d", &tmp_int)) {95 all_memory = tmp_int;96 continue;97 }98 std::cerr << "Illegal values for ARCHER_OPTIONS variable: " << token99 << std::endl;100 }101 }102 }103};104 105class TsanFlags {106public:107 int ignore_noninstrumented_modules;108 109 TsanFlags(const char *env) : ignore_noninstrumented_modules(0) {110 if (env) {111 std::vector<std::string> tokens;112 std::string str(env);113 auto end = str.end();114 auto it = str.begin();115 auto is_sep = [](char c) {116 return c == ' ' || c == ',' || c == ':' || c == '\n' || c == '\t' ||117 c == '\r';118 };119 while (it != end) {120 auto next_it = std::find_if(it, end, is_sep);121 tokens.emplace_back(it, next_it);122 it = next_it;123 if (it != end) {124 ++it;125 }126 }127 128 for (const auto &token : tokens) {129 // we are interested in ignore_noninstrumented_modules to print a130 // warning131 if (sscanf(token.c_str(), "ignore_noninstrumented_modules=%d",132 &ignore_noninstrumented_modules))133 continue;134 }135 }136 }137};138} // namespace139 140#if (LLVM_VERSION) >= 40141extern "C" {142int __attribute__((weak)) __archer_get_omp_status();143void __attribute__((weak)) __tsan_flush_memory() {}144}145#endif146static ArcherFlags *archer_flags;147 148#ifndef TsanHappensBefore149 150template <typename... Args> static void __ompt_tsan_func(Args...) {}151 152#define DECLARE_TSAN_FUNCTION(name, ...) \153 static void (*name)(__VA_ARGS__) = __ompt_tsan_func<__VA_ARGS__>;154 155// Thread Sanitizer is a tool that finds races in code.156// See http://code.google.com/p/data-race-test/wiki/DynamicAnnotations .157// tsan detects these exact functions by name.158extern "C" {159DECLARE_TSAN_FUNCTION(AnnotateHappensAfter, const char *, int,160 const volatile void *)161DECLARE_TSAN_FUNCTION(AnnotateHappensBefore, const char *, int,162 const volatile void *)163DECLARE_TSAN_FUNCTION(AnnotateIgnoreWritesBegin, const char *, int)164DECLARE_TSAN_FUNCTION(AnnotateIgnoreWritesEnd, const char *, int)165DECLARE_TSAN_FUNCTION(AnnotateNewMemory, const char *, int,166 const volatile void *, size_t)167DECLARE_TSAN_FUNCTION(__tsan_func_entry, const void *)168DECLARE_TSAN_FUNCTION(__tsan_func_exit)169}170 171// This marker is used to define a happens-before arc. The race detector will172// infer an arc from the begin to the end when they share the same pointer173// argument.174#define TsanHappensBefore(cv) AnnotateHappensBefore(__FILE__, __LINE__, cv)175 176// This marker defines the destination of a happens-before arc.177#define TsanHappensAfter(cv) AnnotateHappensAfter(__FILE__, __LINE__, cv)178 179// Ignore any races on writes between here and the next TsanIgnoreWritesEnd.180#define TsanIgnoreWritesBegin() AnnotateIgnoreWritesBegin(__FILE__, __LINE__)181 182// Resume checking for racy writes.183#define TsanIgnoreWritesEnd() AnnotateIgnoreWritesEnd(__FILE__, __LINE__)184 185// We don't really delete the clock for now186#define TsanDeleteClock(cv)187 188// newMemory189#define TsanNewMemory(addr, size) \190 AnnotateNewMemory(__FILE__, __LINE__, addr, size)191#define TsanFreeMemory(addr, size) \192 AnnotateNewMemory(__FILE__, __LINE__, addr, size)193#endif194 195// Function entry/exit196#define TsanFuncEntry(pc) __tsan_func_entry(pc)197#define TsanFuncExit() __tsan_func_exit()198 199/// Required OMPT inquiry functions.200static ompt_get_parallel_info_t ompt_get_parallel_info;201static ompt_get_thread_data_t ompt_get_thread_data;202 203typedef char ompt_tsan_clockid;204 205static uint64_t my_next_id() {206 static uint64_t ID = 0;207 uint64_t ret = __sync_fetch_and_add(&ID, 1);208 return ret;209}210 211static int pagesize{0};212 213// Data structure to provide a threadsafe pool of reusable objects.214// DataPool<Type of objects>215namespace {216template <typename T> struct DataPool final {217 static __thread DataPool<T> *ThreadDataPool;218 std::mutex DPMutex{};219 220 // store unused objects221 std::vector<T *> DataPointer{};222 std::vector<T *> RemoteDataPointer{};223 224 // store all allocated memory to finally release225 std::list<void *> memory;226 227 // count remotely returned data (RemoteDataPointer.size())228 std::atomic<int> remote{0};229 230 // totally allocated data objects in pool231 int total{0};232#ifdef DEBUG_DATA233 int remoteReturn{0};234 int localReturn{0};235 236 int getRemote() { return remoteReturn + remote; }237 int getLocal() { return localReturn; }238#endif239 int getTotal() { return total; }240 int getMissing() {241 return total - DataPointer.size() - RemoteDataPointer.size();242 }243 244 // fill the pool by allocating a page of memory245 void newDatas() {246 if (remote > 0) {247 const std::lock_guard<std::mutex> lock(DPMutex);248 // DataPointer is empty, so just swap the vectors249 DataPointer.swap(RemoteDataPointer);250 remote = 0;251 return;252 }253 // calculate size of an object including padding to cacheline size254 size_t elemSize = sizeof(T);255 size_t paddedSize = (((elemSize - 1) / 64) + 1) * 64;256 // number of padded elements to allocate257 int ndatas = pagesize / paddedSize;258 char *datas = (char *)malloc(ndatas * paddedSize);259 memory.push_back(datas);260 for (int i = 0; i < ndatas; i++) {261 DataPointer.push_back(new (datas + i * paddedSize) T(this));262 }263 total += ndatas;264 }265 266 // get data from the pool267 T *getData() {268 T *ret;269 if (DataPointer.empty())270 newDatas();271 ret = DataPointer.back();272 DataPointer.pop_back();273 return ret;274 }275 276 // accesses to the thread-local datapool don't need locks277 void returnOwnData(T *data) {278 DataPointer.emplace_back(data);279#ifdef DEBUG_DATA280 localReturn++;281#endif282 }283 284 // returning to a remote datapool using lock285 void returnData(T *data) {286 const std::lock_guard<std::mutex> lock(DPMutex);287 RemoteDataPointer.emplace_back(data);288 remote++;289#ifdef DEBUG_DATA290 remoteReturn++;291#endif292 }293 294 ~DataPool() {295 // we assume all memory is returned when the thread finished / destructor is296 // called297 if (archer_flags->report_data_leak && getMissing() != 0) {298 printf("ERROR: While freeing DataPool (%s) we are missing %i data "299 "objects.\n",300 __PRETTY_FUNCTION__, getMissing());301 exit(-3);302 }303 for (auto i : DataPointer)304 if (i)305 i->~T();306 for (auto i : RemoteDataPointer)307 if (i)308 i->~T();309 for (auto i : memory)310 if (i)311 free(i);312 }313};314 315template <typename T> struct DataPoolEntry {316 DataPool<T> *owner;317 318 static T *New() { return DataPool<T>::ThreadDataPool->getData(); }319 320 void Delete() {321 static_cast<T *>(this)->Reset();322 if (owner == DataPool<T>::ThreadDataPool)323 owner->returnOwnData(static_cast<T *>(this));324 else325 owner->returnData(static_cast<T *>(this));326 }327 328 DataPoolEntry(DataPool<T> *dp) : owner(dp) {}329};330 331struct DependencyData;332typedef DataPool<DependencyData> DependencyDataPool;333template <>334__thread DependencyDataPool *DependencyDataPool::ThreadDataPool = nullptr;335 336/// Data structure to store additional information for task dependency.337struct DependencyData final : DataPoolEntry<DependencyData> {338 ompt_tsan_clockid in;339 ompt_tsan_clockid out;340 ompt_tsan_clockid inoutset;341 void *GetInPtr() { return ∈ }342 void *GetOutPtr() { return &out; }343 void *GetInoutsetPtr() { return &inoutset; }344 345 void Reset() {}346 347 static DependencyData *New() { return DataPoolEntry<DependencyData>::New(); }348 349 DependencyData(DataPool<DependencyData> *dp)350 : DataPoolEntry<DependencyData>(dp) {}351};352 353struct TaskDependency {354 void *inPtr;355 void *outPtr;356 void *inoutsetPtr;357 ompt_dependence_type_t type;358 TaskDependency(DependencyData *depData, ompt_dependence_type_t type)359 : inPtr(depData->GetInPtr()), outPtr(depData->GetOutPtr()),360 inoutsetPtr(depData->GetInoutsetPtr()), type(type) {}361 void AnnotateBegin() {362 if (type == ompt_dependence_type_out ||363 type == ompt_dependence_type_inout ||364 type == ompt_dependence_type_mutexinoutset) {365 TsanHappensAfter(inPtr);366 TsanHappensAfter(outPtr);367 TsanHappensAfter(inoutsetPtr);368 } else if (type == ompt_dependence_type_in) {369 TsanHappensAfter(outPtr);370 TsanHappensAfter(inoutsetPtr);371 } else if (type == ompt_dependence_type_inoutset) {372 TsanHappensAfter(inPtr);373 TsanHappensAfter(outPtr);374 }375 }376 void AnnotateEnd() {377 if (type == ompt_dependence_type_out ||378 type == ompt_dependence_type_inout ||379 type == ompt_dependence_type_mutexinoutset) {380 TsanHappensBefore(outPtr);381 } else if (type == ompt_dependence_type_in) {382 TsanHappensBefore(inPtr);383 } else if (type == ompt_dependence_type_inoutset) {384 TsanHappensBefore(inoutsetPtr);385 }386 }387};388 389struct ParallelData;390typedef DataPool<ParallelData> ParallelDataPool;391template <>392__thread ParallelDataPool *ParallelDataPool::ThreadDataPool = nullptr;393 394/// Data structure to store additional information for parallel regions.395struct ParallelData final : DataPoolEntry<ParallelData> {396 397 // Parallel fork is just another barrier, use Barrier[1]398 399 /// Two addresses for relationships with barriers.400 ompt_tsan_clockid Barrier[2];401 402 const void *codePtr;403 404 void *GetParallelPtr() { return &(Barrier[1]); }405 406 void *GetBarrierPtr(unsigned Index) { return &(Barrier[Index]); }407 408 ParallelData *Init(const void *codeptr) {409 codePtr = codeptr;410 return this;411 }412 413 void Reset() {}414 415 static ParallelData *New(const void *codeptr) {416 return DataPoolEntry<ParallelData>::New()->Init(codeptr);417 }418 419 ParallelData(DataPool<ParallelData> *dp) : DataPoolEntry<ParallelData>(dp) {}420};421 422static inline ParallelData *ToParallelData(ompt_data_t *parallel_data) {423 return reinterpret_cast<ParallelData *>(parallel_data->ptr);424}425 426struct Taskgroup;427typedef DataPool<Taskgroup> TaskgroupPool;428template <> __thread TaskgroupPool *TaskgroupPool::ThreadDataPool = nullptr;429 430/// Data structure to support stacking of taskgroups and allow synchronization.431struct Taskgroup final : DataPoolEntry<Taskgroup> {432 /// Its address is used for relationships of the taskgroup's task set.433 ompt_tsan_clockid Ptr;434 435 /// Reference to the parent taskgroup.436 Taskgroup *Parent;437 438 void *GetPtr() { return &Ptr; }439 440 Taskgroup *Init(Taskgroup *parent) {441 Parent = parent;442 return this;443 }444 445 void Reset() {}446 447 static Taskgroup *New(Taskgroup *Parent) {448 return DataPoolEntry<Taskgroup>::New()->Init(Parent);449 }450 451 Taskgroup(DataPool<Taskgroup> *dp) : DataPoolEntry<Taskgroup>(dp) {}452};453 454enum ArcherTaskFlag { ArcherTaskFulfilled = 0x00010000 };455 456struct TaskData;457typedef DataPool<TaskData> TaskDataPool;458template <> __thread TaskDataPool *TaskDataPool::ThreadDataPool = nullptr;459 460/// Data structure to store additional information for tasks.461struct TaskData final : DataPoolEntry<TaskData> {462 /// Its address is used for relationships of this task.463 ompt_tsan_clockid Task{0};464 465 /// Child tasks use its address to declare a relationship to a taskwait in466 /// this task.467 ompt_tsan_clockid Taskwait{0};468 469 /// Child tasks use its address to model omp_all_memory dependencies470 ompt_tsan_clockid AllMemory[2]{0};471 472 /// Index of which barrier to use next.473 char BarrierIndex{0};474 475 /// Whether this task is currently executing a barrier.476 bool InBarrier{false};477 478 /// Whether this task is an included task.479 int TaskType{0};480 481 /// count execution phase482 int execution{0};483 484 /// Count how often this structure has been put into child tasks + 1.485 std::atomic_int RefCount{1};486 487 /// Reference to the parent that created this task.488 TaskData *Parent{nullptr};489 490 /// Reference to the team of this task.491 ParallelData *Team{nullptr};492 493 /// Reference to the current taskgroup that this task either belongs to or494 /// that it just created.495 Taskgroup *TaskGroup{nullptr};496 497 /// Dependency information for this task.498 TaskDependency *Dependencies{nullptr};499 500 /// Number of dependency entries.501 unsigned DependencyCount{0};502 503 // The dependency-map stores DependencyData objects representing504 // the dependency variables used on the sibling tasks created from505 // this task506 // We expect a rare need for the dependency-map, so alloc on demand507 std::unordered_map<void *, DependencyData *> *DependencyMap{nullptr};508 509#ifdef DEBUG510 int freed{0};511#endif512 513 bool isIncluded() { return TaskType & ompt_task_undeferred; }514 bool isUntied() { return TaskType & ompt_task_untied; }515 bool isFinal() { return TaskType & ompt_task_final; }516 bool isMergable() { return TaskType & ompt_task_mergeable; }517 bool isMerged() { return TaskType & ompt_task_merged; }518 519 bool isExplicit() { return TaskType & ompt_task_explicit; }520 bool isImplicit() { return TaskType & ompt_task_implicit; }521 bool isInitial() { return TaskType & ompt_task_initial; }522 bool isTarget() { return TaskType & ompt_task_target; }523 524 bool isFulfilled() { return TaskType & ArcherTaskFulfilled; }525 void setFulfilled() { TaskType |= ArcherTaskFulfilled; }526 527 void setAllMemoryDep() { AllMemory[0] = 1; }528 bool hasAllMemoryDep() { return AllMemory[0]; }529 530 void *GetTaskPtr() { return &Task; }531 532 void *GetTaskwaitPtr() { return &Taskwait; }533 534 void *GetLastAllMemoryPtr() { return AllMemory; }535 void *GetNextAllMemoryPtr() { return AllMemory + 1; }536 537 TaskData *Init(TaskData *parent, int taskType) {538 TaskType = taskType;539 Parent = parent;540 Team = Parent->Team;541 BarrierIndex = Parent->BarrierIndex;542 if (Parent != nullptr) {543 Parent->RefCount++;544 // Copy over pointer to taskgroup. This task may set up its own stack545 // but for now belongs to its parent's taskgroup.546 TaskGroup = Parent->TaskGroup;547 }548 return this;549 }550 551 TaskData *Init(ParallelData *team, int taskType) {552 TaskType = taskType;553 execution = 1;554 Team = team;555 return this;556 }557 558 void Reset() {559 InBarrier = false;560 TaskType = 0;561 execution = 0;562 BarrierIndex = 0;563 RefCount = 1;564 Parent = nullptr;565 Team = nullptr;566 TaskGroup = nullptr;567 if (DependencyMap) {568 for (auto i : *DependencyMap)569 i.second->Delete();570 delete DependencyMap;571 }572 DependencyMap = nullptr;573 if (Dependencies)574 free(Dependencies);575 Dependencies = nullptr;576 DependencyCount = 0;577#ifdef DEBUG578 freed = 0;579#endif580 }581 582 static TaskData *New(TaskData *parent, int taskType) {583 return DataPoolEntry<TaskData>::New()->Init(parent, taskType);584 }585 586 static TaskData *New(ParallelData *team, int taskType) {587 return DataPoolEntry<TaskData>::New()->Init(team, taskType);588 }589 590 TaskData(DataPool<TaskData> *dp) : DataPoolEntry<TaskData>(dp) {}591};592} // namespace593 594static inline TaskData *ToTaskData(ompt_data_t *task_data) {595 if (task_data)596 return reinterpret_cast<TaskData *>(task_data->ptr);597 return nullptr;598}599 600/// Store a mutex for each wait_id to resolve race condition with callbacks.601static std::unordered_map<ompt_wait_id_t, std::mutex> Locks;602static std::mutex LocksMutex;603 604static void ompt_tsan_thread_begin(ompt_thread_t thread_type,605 ompt_data_t *thread_data) {606 ParallelDataPool::ThreadDataPool = new ParallelDataPool;607 TsanNewMemory(ParallelDataPool::ThreadDataPool,608 sizeof(ParallelDataPool::ThreadDataPool));609 TaskgroupPool::ThreadDataPool = new TaskgroupPool;610 TsanNewMemory(TaskgroupPool::ThreadDataPool,611 sizeof(TaskgroupPool::ThreadDataPool));612 TaskDataPool::ThreadDataPool = new TaskDataPool;613 TsanNewMemory(TaskDataPool::ThreadDataPool,614 sizeof(TaskDataPool::ThreadDataPool));615 DependencyDataPool::ThreadDataPool = new DependencyDataPool;616 TsanNewMemory(DependencyDataPool::ThreadDataPool,617 sizeof(DependencyDataPool::ThreadDataPool));618 thread_data->value = my_next_id();619}620 621static void ompt_tsan_thread_end(ompt_data_t *thread_data) {622 TsanIgnoreWritesBegin();623 delete ParallelDataPool::ThreadDataPool;624 delete TaskgroupPool::ThreadDataPool;625 delete TaskDataPool::ThreadDataPool;626 delete DependencyDataPool::ThreadDataPool;627 TsanIgnoreWritesEnd();628}629 630/// OMPT event callbacks for handling parallel regions.631 632static void ompt_tsan_parallel_begin(ompt_data_t *parent_task_data,633 const ompt_frame_t *parent_task_frame,634 ompt_data_t *parallel_data,635 uint32_t requested_team_size, int flag,636 const void *codeptr_ra) {637 ParallelData *Data = ParallelData::New(codeptr_ra);638 parallel_data->ptr = Data;639 640 TsanHappensBefore(Data->GetParallelPtr());641 if (archer_flags->ignore_serial && ToTaskData(parent_task_data)->isInitial())642 TsanIgnoreWritesEnd();643}644 645static void ompt_tsan_parallel_end(ompt_data_t *parallel_data,646 ompt_data_t *task_data, int flag,647 const void *codeptr_ra) {648 if (archer_flags->ignore_serial && ToTaskData(task_data)->isInitial())649 TsanIgnoreWritesBegin();650 ParallelData *Data = ToParallelData(parallel_data);651 TsanHappensAfter(Data->GetBarrierPtr(0));652 TsanHappensAfter(Data->GetBarrierPtr(1));653 654 Data->Delete();655 656#if (LLVM_VERSION >= 40)657 if (&__archer_get_omp_status) {658 if (__archer_get_omp_status() == 0 && archer_flags->flush_shadow)659 __tsan_flush_memory();660 }661#endif662}663 664static void ompt_tsan_implicit_task(ompt_scope_endpoint_t endpoint,665 ompt_data_t *parallel_data,666 ompt_data_t *task_data,667 unsigned int team_size,668 unsigned int thread_num, int type) {669 switch (endpoint) {670 case ompt_scope_begin:671 if (type & ompt_task_initial) {672 parallel_data->ptr = ParallelData::New(nullptr);673 }674 task_data->ptr = TaskData::New(ToParallelData(parallel_data), type);675 TsanHappensAfter(ToParallelData(parallel_data)->GetParallelPtr());676 TsanFuncEntry(ToParallelData(parallel_data)->codePtr);677 break;678 case ompt_scope_end: {679 TaskData *Data = ToTaskData(task_data);680#ifdef DEBUG681 assert(Data->freed == 0 && "Implicit task end should only be called once!");682 Data->freed = 1;683#endif684 assert(Data->RefCount == 1 &&685 "All tasks should have finished at the implicit barrier!");686 if (type & ompt_task_initial) {687 Data->Team->Delete();688 }689 Data->Delete();690 TsanFuncExit();691 break;692 }693 case ompt_scope_beginend:694 // Should not occur according to OpenMP 5.1695 // Tested in OMPT tests696 break;697 }698}699 700static void ompt_tsan_sync_region(ompt_sync_region_t kind,701 ompt_scope_endpoint_t endpoint,702 ompt_data_t *parallel_data,703 ompt_data_t *task_data,704 const void *codeptr_ra) {705 TaskData *Data = ToTaskData(task_data);706 switch (endpoint) {707 case ompt_scope_begin:708 case ompt_scope_beginend:709 TsanFuncEntry(codeptr_ra);710 switch (kind) {711 case ompt_sync_region_barrier_implementation:712 case ompt_sync_region_barrier_implicit:713 case ompt_sync_region_barrier_explicit:714 case ompt_sync_region_barrier_implicit_parallel:715 case ompt_sync_region_barrier_implicit_workshare:716 case ompt_sync_region_barrier_teams:717 case ompt_sync_region_barrier: {718 char BarrierIndex = Data->BarrierIndex;719 TsanHappensBefore(Data->Team->GetBarrierPtr(BarrierIndex));720 721 if (hasReductionCallback < ompt_set_always) {722 // We ignore writes inside the barrier. These would either occur during723 // 1. reductions performed by the runtime which are guaranteed to be724 // race-free.725 // 2. execution of another task.726 // For the latter case we will re-enable tracking in task_switch.727 Data->InBarrier = true;728 TsanIgnoreWritesBegin();729 }730 731 break;732 }733 734 case ompt_sync_region_taskwait:735 break;736 737 case ompt_sync_region_taskgroup:738 Data->TaskGroup = Taskgroup::New(Data->TaskGroup);739 break;740 741 case ompt_sync_region_reduction:742 // should never be reached743 break;744 }745 if (endpoint == ompt_scope_begin)746 break;747 KMP_FALLTHROUGH();748 case ompt_scope_end:749 TsanFuncExit();750 switch (kind) {751 case ompt_sync_region_barrier_implementation:752 case ompt_sync_region_barrier_implicit:753 case ompt_sync_region_barrier_explicit:754 case ompt_sync_region_barrier_implicit_parallel:755 case ompt_sync_region_barrier_implicit_workshare:756 case ompt_sync_region_barrier_teams:757 case ompt_sync_region_barrier: {758 if (hasReductionCallback < ompt_set_always) {759 // We want to track writes after the barrier again.760 Data->InBarrier = false;761 TsanIgnoreWritesEnd();762 }763 764 char BarrierIndex = Data->BarrierIndex;765 // Barrier will end after it has been entered by all threads.766 if (parallel_data)767 TsanHappensAfter(Data->Team->GetBarrierPtr(BarrierIndex));768 769 // It is not guaranteed that all threads have exited this barrier before770 // we enter the next one. So we will use a different address.771 // We are however guaranteed that this current barrier is finished772 // by the time we exit the next one. So we can then reuse the first773 // address.774 Data->BarrierIndex = (BarrierIndex + 1) % 2;775 break;776 }777 778 case ompt_sync_region_taskwait: {779 if (Data->execution > 1)780 TsanHappensAfter(Data->GetTaskwaitPtr());781 break;782 }783 784 case ompt_sync_region_taskgroup: {785 assert(Data->TaskGroup != nullptr &&786 "Should have at least one taskgroup!");787 788 TsanHappensAfter(Data->TaskGroup->GetPtr());789 790 // Delete this allocated taskgroup, all descendent task are finished by791 // now.792 Taskgroup *Parent = Data->TaskGroup->Parent;793 Data->TaskGroup->Delete();794 Data->TaskGroup = Parent;795 break;796 }797 798 case ompt_sync_region_reduction:799 // Should not occur according to OpenMP 5.1800 // Tested in OMPT tests801 break;802 }803 break;804 }805}806 807static void ompt_tsan_reduction(ompt_sync_region_t kind,808 ompt_scope_endpoint_t endpoint,809 ompt_data_t *parallel_data,810 ompt_data_t *task_data,811 const void *codeptr_ra) {812 switch (endpoint) {813 case ompt_scope_begin:814 switch (kind) {815 case ompt_sync_region_reduction:816 TsanIgnoreWritesBegin();817 break;818 default:819 break;820 }821 break;822 case ompt_scope_end:823 switch (kind) {824 case ompt_sync_region_reduction:825 TsanIgnoreWritesEnd();826 break;827 default:828 break;829 }830 break;831 case ompt_scope_beginend:832 // Should not occur according to OpenMP 5.1833 // Tested in OMPT tests834 // Would have no implications for DR detection835 break;836 }837}838 839/// OMPT event callbacks for handling tasks.840 841static void ompt_tsan_task_create(842 ompt_data_t *parent_task_data, /* id of parent task */843 const ompt_frame_t *parent_frame, /* frame data for parent task */844 ompt_data_t *new_task_data, /* id of created task */845 int type, int has_dependences,846 const void *codeptr_ra) /* pointer to outlined function */847{848 TaskData *Data;849 assert(new_task_data->ptr == NULL &&850 "Task data should be initialized to NULL");851 if (type & ompt_task_initial) {852 ompt_data_t *parallel_data;853 int team_size = 1;854 ompt_get_parallel_info(0, ¶llel_data, &team_size);855 ParallelData *PData = ParallelData::New(nullptr);856 parallel_data->ptr = PData;857 858 Data = TaskData::New(PData, type);859 new_task_data->ptr = Data;860 } else if (type & ompt_task_undeferred) {861 Data = TaskData::New(ToTaskData(parent_task_data), type);862 new_task_data->ptr = Data;863 } else if (type & ompt_task_explicit || type & ompt_task_target) {864 Data = TaskData::New(ToTaskData(parent_task_data), type);865 new_task_data->ptr = Data;866 867 // Use the newly created address. We cannot use a single address from the868 // parent because that would declare wrong relationships with other869 // sibling tasks that may be created before this task is started!870 TsanHappensBefore(Data->GetTaskPtr());871 ToTaskData(parent_task_data)->execution++;872 }873}874 875static void freeTask(TaskData *task) {876 while (task != nullptr && --task->RefCount == 0) {877 TaskData *Parent = task->Parent;878 task->Delete();879 task = Parent;880 }881}882 883// LastAllMemoryPtr marks the beginning of an all_memory epoch884// NextAllMemoryPtr marks the end of an all_memory epoch885// All tasks with depend begin execution after LastAllMemoryPtr886// and end before NextAllMemoryPtr887static void releaseDependencies(TaskData *task) {888 if (archer_flags->all_memory) {889 if (task->hasAllMemoryDep()) {890 TsanHappensBefore(task->Parent->GetLastAllMemoryPtr());891 TsanHappensBefore(task->Parent->GetNextAllMemoryPtr());892 } else if (task->DependencyCount)893 TsanHappensBefore(task->Parent->GetNextAllMemoryPtr());894 }895 for (unsigned i = 0; i < task->DependencyCount; i++) {896 task->Dependencies[i].AnnotateEnd();897 }898}899 900static void acquireDependencies(TaskData *task) {901 if (archer_flags->all_memory) {902 if (task->hasAllMemoryDep())903 TsanHappensAfter(task->Parent->GetNextAllMemoryPtr());904 else if (task->DependencyCount)905 TsanHappensAfter(task->Parent->GetLastAllMemoryPtr());906 }907 for (unsigned i = 0; i < task->DependencyCount; i++) {908 task->Dependencies[i].AnnotateBegin();909 }910}911 912static void completeTask(TaskData *FromTask) {913 if (!FromTask)914 return;915 // Task-end happens after a possible omp_fulfill_event call916 if (FromTask->isFulfilled())917 TsanHappensAfter(FromTask->GetTaskPtr());918 // Included tasks are executed sequentially, no need to track919 // synchronization920 if (!FromTask->isIncluded()) {921 // Task will finish before a barrier in the surrounding parallel region922 // ...923 ParallelData *PData = FromTask->Team;924 TsanHappensBefore(PData->GetBarrierPtr(FromTask->BarrierIndex));925 926 // ... and before an eventual taskwait by the parent thread.927 TsanHappensBefore(FromTask->Parent->GetTaskwaitPtr());928 929 if (FromTask->TaskGroup != nullptr) {930 // This task is part of a taskgroup, so it will finish before the931 // corresponding taskgroup_end.932 TsanHappensBefore(FromTask->TaskGroup->GetPtr());933 }934 }935 // release dependencies936 releaseDependencies(FromTask);937}938 939static void suspendTask(TaskData *FromTask) {940 if (!FromTask)941 return;942 // Task may be resumed at a later point in time.943 TsanHappensBefore(FromTask->GetTaskPtr());944}945 946static void switchTasks(TaskData *FromTask, TaskData *ToTask) {947 // Legacy handling for missing reduction callback948 if (hasReductionCallback < ompt_set_always) {949 if (FromTask && FromTask->InBarrier) {950 // We want to ignore writes in the runtime code during barriers,951 // but not when executing tasks with user code!952 TsanIgnoreWritesEnd();953 }954 if (ToTask && ToTask->InBarrier) {955 // We want to ignore writes in the runtime code during barriers,956 // but not when executing tasks with user code!957 TsanIgnoreWritesBegin();958 }959 }960 //// Not yet used961 // if (FromTask)962 // FromTask->deactivate();963 // if (ToTask)964 // ToTask->activate();965}966 967static void endTask(TaskData *FromTask) {968 if (!FromTask)969 return;970}971 972static void startTask(TaskData *ToTask) {973 if (!ToTask)974 return;975 // Handle dependencies on first execution of the task976 if (ToTask->execution == 0) {977 ToTask->execution++;978 acquireDependencies(ToTask);979 }980 // 1. Task will begin execution after it has been created.981 // 2. Task will resume after it has been switched away.982 TsanHappensAfter(ToTask->GetTaskPtr());983}984 985static void ompt_tsan_task_schedule(ompt_data_t *first_task_data,986 ompt_task_status_t prior_task_status,987 ompt_data_t *second_task_data) {988 989 //990 // The necessary action depends on prior_task_status:991 //992 // ompt_task_early_fulfill = 5,993 // -> ignored994 //995 // ompt_task_late_fulfill = 6,996 // -> first completed, first freed, second ignored997 //998 // ompt_task_complete = 1,999 // ompt_task_cancel = 3,1000 // -> first completed, first freed, second starts1001 //1002 // ompt_taskwait_complete = 8,1003 // -> first starts, first completes, first freed, second ignored1004 //1005 // ompt_task_detach = 4,1006 // ompt_task_yield = 2,1007 // ompt_task_switch = 71008 // -> first suspended, second starts1009 //1010 1011 TaskData *FromTask = ToTaskData(first_task_data);1012 TaskData *ToTask = ToTaskData(second_task_data);1013 1014 switch (prior_task_status) {1015 case ompt_task_early_fulfill:1016 TsanHappensBefore(FromTask->GetTaskPtr());1017 FromTask->setFulfilled();1018 return;1019 case ompt_task_late_fulfill:1020 TsanHappensAfter(FromTask->GetTaskPtr());1021 completeTask(FromTask);1022 freeTask(FromTask);1023 return;1024 case ompt_taskwait_complete:1025 acquireDependencies(FromTask);1026 freeTask(FromTask);1027 return;1028 case ompt_task_complete:1029 completeTask(FromTask);1030 endTask(FromTask);1031 switchTasks(FromTask, ToTask);1032 freeTask(FromTask);1033 return;1034 case ompt_task_cancel:1035 completeTask(FromTask);1036 endTask(FromTask);1037 switchTasks(FromTask, ToTask);1038 freeTask(FromTask);1039 startTask(ToTask);1040 return;1041 case ompt_task_detach:1042 endTask(FromTask);1043 suspendTask(FromTask);1044 switchTasks(FromTask, ToTask);1045 startTask(ToTask);1046 return;1047 case ompt_task_yield:1048 suspendTask(FromTask);1049 switchTasks(FromTask, ToTask);1050 startTask(ToTask);1051 return;1052 case ompt_task_switch:1053 suspendTask(FromTask);1054 switchTasks(FromTask, ToTask);1055 startTask(ToTask);1056 return;1057 }1058}1059 1060static void ompt_tsan_dependences(ompt_data_t *task_data,1061 const ompt_dependence_t *deps, int ndeps) {1062 if (ndeps > 0) {1063 // Copy the data to use it in task_switch and task_end.1064 TaskData *Data = ToTaskData(task_data);1065 if (!Data->Parent) {1066 // Return since doacross dependences are not supported yet.1067 return;1068 }1069 if (!Data->Parent->DependencyMap)1070 Data->Parent->DependencyMap =1071 new std::unordered_map<void *, DependencyData *>();1072 Data->Dependencies =1073 (TaskDependency *)malloc(sizeof(TaskDependency) * ndeps);1074 Data->DependencyCount = ndeps;1075 for (int i = 0, d = 0; i < ndeps; i++, d++) {1076 if (deps[i].dependence_type == ompt_dependence_type_out_all_memory ||1077 deps[i].dependence_type == ompt_dependence_type_inout_all_memory) {1078 Data->setAllMemoryDep();1079 Data->DependencyCount--;1080 if (!archer_flags->all_memory) {1081 printf("The application uses omp_all_memory, but Archer was\n"1082 "started to not consider omp_all_memory. This can lead\n"1083 "to false data race alerts.\n"1084 "Include all_memory=1 in ARCHER_OPTIONS to consider\n"1085 "omp_all_memory from the beginning.\n");1086 archer_flags->all_memory = 1;1087 }1088 d--;1089 continue;1090 }1091 auto ret = Data->Parent->DependencyMap->insert(1092 std::make_pair(deps[i].variable.ptr, nullptr));1093 if (ret.second) {1094 ret.first->second = DependencyData::New();1095 }1096 new ((void *)(Data->Dependencies + d))1097 TaskDependency(ret.first->second, deps[i].dependence_type);1098 }1099 1100 // This callback is executed before this task is first started.1101 TsanHappensBefore(Data->GetTaskPtr());1102 }1103}1104 1105/// OMPT event callbacks for handling locking.1106static void ompt_tsan_mutex_acquired(ompt_mutex_t kind, ompt_wait_id_t wait_id,1107 const void *codeptr_ra) {1108 1109 // Acquire our own lock to make sure that1110 // 1. the previous release has finished.1111 // 2. the next acquire doesn't start before we have finished our release.1112 LocksMutex.lock();1113 std::mutex &Lock = Locks[wait_id];1114 LocksMutex.unlock();1115 1116 Lock.lock();1117 TsanHappensAfter(&Lock);1118}1119 1120static void ompt_tsan_mutex_released(ompt_mutex_t kind, ompt_wait_id_t wait_id,1121 const void *codeptr_ra) {1122 LocksMutex.lock();1123 std::mutex &Lock = Locks[wait_id];1124 LocksMutex.unlock();1125 TsanHappensBefore(&Lock);1126 1127 Lock.unlock();1128}1129 1130// callback , signature , variable to store result , required support level1131#define SET_OPTIONAL_CALLBACK_T(event, type, result, level) \1132 do { \1133 ompt_callback_##type##_t tsan_##event = &ompt_tsan_##event; \1134 result = ompt_set_callback(ompt_callback_##event, \1135 (ompt_callback_t)tsan_##event); \1136 if (result < level) \1137 printf("Registered callback '" #event "' is not supported at " #level \1138 " (%i)\n", \1139 result); \1140 } while (0)1141 1142#define SET_CALLBACK_T(event, type) \1143 do { \1144 int res; \1145 SET_OPTIONAL_CALLBACK_T(event, type, res, ompt_set_always); \1146 } while (0)1147 1148#define SET_CALLBACK(event) SET_CALLBACK_T(event, event)1149 1150#define findTsanFunction(f, fSig) \1151 do { \1152 void *fp = dlsym(RTLD_DEFAULT, #f); \1153 if (fp) \1154 f = fSig fp; \1155 else \1156 printf("Unable to find TSan function " #f ".\n"); \1157 } while (0)1158 1159#define findTsanFunctionSilent(f, fSig) f = fSig dlsym(RTLD_DEFAULT, #f)1160 1161static int ompt_tsan_initialize(ompt_function_lookup_t lookup, int device_num,1162 ompt_data_t *tool_data) {1163 const char *options = getenv("TSAN_OPTIONS");1164 TsanFlags tsan_flags(options);1165 1166 ompt_set_callback_t ompt_set_callback =1167 (ompt_set_callback_t)lookup("ompt_set_callback");1168 if (ompt_set_callback == NULL) {1169 std::cerr << "Could not set callback, exiting..." << std::endl;1170 std::exit(1);1171 }1172 ompt_get_parallel_info =1173 (ompt_get_parallel_info_t)lookup("ompt_get_parallel_info");1174 ompt_get_thread_data = (ompt_get_thread_data_t)lookup("ompt_get_thread_data");1175 1176 if (ompt_get_parallel_info == NULL) {1177 fprintf(stderr, "Could not get inquiry function 'ompt_get_parallel_info', "1178 "exiting...\n");1179 exit(1);1180 }1181 1182 findTsanFunction(AnnotateHappensAfter,1183 (void (*)(const char *, int, const volatile void *)));1184 findTsanFunction(AnnotateHappensBefore,1185 (void (*)(const char *, int, const volatile void *)));1186 findTsanFunction(AnnotateIgnoreWritesBegin, (void (*)(const char *, int)));1187 findTsanFunction(AnnotateIgnoreWritesEnd, (void (*)(const char *, int)));1188 findTsanFunction(1189 AnnotateNewMemory,1190 (void (*)(const char *, int, const volatile void *, size_t)));1191 findTsanFunction(__tsan_func_entry, (void (*)(const void *)));1192 findTsanFunction(__tsan_func_exit, (void (*)(void)));1193 1194 SET_CALLBACK(thread_begin);1195 SET_CALLBACK(thread_end);1196 SET_CALLBACK(parallel_begin);1197 SET_CALLBACK(implicit_task);1198 SET_CALLBACK(sync_region);1199 SET_CALLBACK(parallel_end);1200 1201 SET_CALLBACK(task_create);1202 SET_CALLBACK(task_schedule);1203 SET_CALLBACK(dependences);1204 1205 SET_CALLBACK_T(mutex_acquired, mutex);1206 SET_CALLBACK_T(mutex_released, mutex);1207 SET_OPTIONAL_CALLBACK_T(reduction, sync_region, hasReductionCallback,1208 ompt_set_never);1209 1210 if (!tsan_flags.ignore_noninstrumented_modules)1211 fprintf(stderr,1212 "Warning: please export "1213 "TSAN_OPTIONS='ignore_noninstrumented_modules=1' "1214 "to avoid false positive reports from the OpenMP runtime!\n");1215 if (archer_flags->ignore_serial)1216 TsanIgnoreWritesBegin();1217 1218 return 1; // success1219}1220 1221static void ompt_tsan_finalize(ompt_data_t *tool_data) {1222 if (archer_flags->ignore_serial)1223 TsanIgnoreWritesEnd();1224 if (archer_flags->print_max_rss) {1225 struct rusage end;1226 getrusage(RUSAGE_SELF, &end);1227 printf("MAX RSS[KiB] during execution: %ld\n", end.ru_maxrss);1228 }1229 1230 if (archer_flags)1231 delete archer_flags;1232}1233 1234extern "C" ompt_start_tool_result_t *1235ompt_start_tool(unsigned int omp_version, const char *runtime_version) {1236 const char *options = getenv("ARCHER_OPTIONS");1237 archer_flags = new ArcherFlags(options);1238 if (!archer_flags->enabled) {1239 if (archer_flags->verbose)1240 std::cout << "Archer disabled, stopping operation" << std::endl;1241 delete archer_flags;1242 return NULL;1243 }1244 1245 pagesize = getpagesize();1246 1247 static ompt_start_tool_result_t ompt_start_tool_result = {1248 &ompt_tsan_initialize, &ompt_tsan_finalize, {0}};1249 1250 // The OMPT start-up code uses dlopen with RTLD_LAZY. Therefore, we cannot1251 // rely on dlopen to fail if TSan is missing, but would get a runtime error1252 // for the first TSan call. We use __tsan_init to detect whether1253 // an implementation of the Annotation interface is available in the1254 // execution or disable the tool (by returning NULL).1255 1256 void (*__tsan_init)(void) = nullptr;1257 1258 findTsanFunctionSilent(__tsan_init, (void (*)(void)));1259 if (!__tsan_init) // if we are not running on TSAN, give a different1260 // tool the chance to be loaded1261 {1262 if (archer_flags->verbose)1263 std::cout << "Archer detected OpenMP application without TSan; "1264 "stopping operation"1265 << std::endl;1266 delete archer_flags;1267 return NULL;1268 }1269 1270 if (archer_flags->verbose)1271 std::cout << "Archer detected OpenMP application with TSan, supplying "1272 "OpenMP synchronization semantics"1273 << std::endl;1274 return &ompt_start_tool_result;1275}1276