532 lines · cpp
1//===-- OpenMP/OMPT/Callback.cpp - OpenMP Tooling Callback implementation -===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Implementation of OMPT callback interfaces for target independent layer10//11//===----------------------------------------------------------------------===//12 13#ifdef OMPT_SUPPORT14 15#include <cstdlib>16#include <cstring>17#include <memory>18 19#include "Shared/Debug.h"20 21#include "OpenMP/OMPT/Callback.h"22#include "OpenMP/OMPT/Connector.h"23#include "OpenMP/OMPT/Interface.h"24 25#include "llvm/Support/DynamicLibrary.h"26 27#undef DEBUG_PREFIX28#define DEBUG_PREFIX "OMPT"29 30// Define OMPT callback functions (bound to actual callbacks later on)31#define defineOmptCallback(Name, Type, Code) \32 Name##_t llvm::omp::target::ompt::Name##_fn = nullptr;33FOREACH_OMPT_NOEMI_EVENT(defineOmptCallback)34FOREACH_OMPT_EMI_EVENT(defineOmptCallback)35#undef defineOmptCallback36 37using namespace llvm::omp::target::ompt;38 39/// Forward declaration40class LibomptargetRtlFinalizer;41 42/// Object that will maintain the RTL finalizer from the plugin43LibomptargetRtlFinalizer *LibraryFinalizer = nullptr;44 45thread_local Interface llvm::omp::target::ompt::RegionInterface;46 47thread_local void *llvm::omp::target::ompt::ReturnAddress = nullptr;48 49bool llvm::omp::target::ompt::Initialized = false;50 51ompt_get_callback_t llvm::omp::target::ompt::lookupCallbackByCode = nullptr;52ompt_function_lookup_t llvm::omp::target::ompt::lookupCallbackByName = nullptr;53ompt_get_target_task_data_t ompt_get_target_task_data_fn = nullptr;54ompt_get_task_data_t ompt_get_task_data_fn = nullptr;55 56/// Unique correlation id57static std::atomic<uint64_t> IdCounter(1);58 59/// Used to create a new correlation id60static uint64_t createId() { return IdCounter.fetch_add(1); }61 62/// Create a new correlation id and update the operations id63static uint64_t createOpId() {64 uint64_t NewId = createId();65 RegionInterface.setHostOpId(NewId);66 return NewId;67}68 69/// Create a new correlation id and update the target region id70static uint64_t createRegionId() {71 uint64_t NewId = createId();72 RegionInterface.setTargetDataValue(NewId);73 return NewId;74}75 76void Interface::beginTargetDataAlloc(int64_t DeviceId, void *HstPtrBegin,77 void **TgtPtrBegin, size_t Size,78 void *Code) {79 beginTargetDataOperation();80 if (ompt_callback_target_data_op_emi_fn) {81 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI82 // callback83 ompt_callback_target_data_op_emi_fn(84 ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId,85 ompt_target_data_alloc, HstPtrBegin,86 /*SrcDeviceNum=*/omp_get_initial_device(), *TgtPtrBegin,87 /*TgtDeviceNum=*/DeviceId, Size, Code);88 } else if (ompt_callback_target_data_op_fn) {89 // HostOpId is set by the runtime90 HostOpId = createOpId();91 // Invoke the tool supplied data op callback92 ompt_callback_target_data_op_fn(93 TargetData.value, HostOpId, ompt_target_data_alloc, HstPtrBegin,94 /*SrcDeviceNum=*/omp_get_initial_device(), *TgtPtrBegin,95 /*TgtDeviceNum=*/DeviceId, Size, Code);96 }97}98 99void Interface::endTargetDataAlloc(int64_t DeviceId, void *HstPtrBegin,100 void **TgtPtrBegin, size_t Size,101 void *Code) {102 // Only EMI callback handles end scope103 if (ompt_callback_target_data_op_emi_fn) {104 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI105 // callback106 ompt_callback_target_data_op_emi_fn(107 ompt_scope_end, TargetTaskData, &TargetData, &HostOpId,108 ompt_target_data_alloc, HstPtrBegin,109 /*SrcDeviceNum=*/omp_get_initial_device(), *TgtPtrBegin,110 /*TgtDeviceNum=*/DeviceId, Size, Code);111 }112 endTargetDataOperation();113}114 115void Interface::beginTargetDataSubmit(int64_t SrcDeviceId, void *SrcPtrBegin,116 int64_t DstDeviceId, void *DstPtrBegin,117 size_t Size, void *Code) {118 beginTargetDataOperation();119 if (ompt_callback_target_data_op_emi_fn) {120 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI121 // callback122 ompt_callback_target_data_op_emi_fn(123 ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId,124 ompt_target_data_transfer_to_device, SrcPtrBegin, SrcDeviceId,125 DstPtrBegin, DstDeviceId, Size, Code);126 } else if (ompt_callback_target_data_op_fn) {127 // HostOpId is set by the runtime128 HostOpId = createOpId();129 // Invoke the tool supplied data op callback130 ompt_callback_target_data_op_fn(131 TargetData.value, HostOpId, ompt_target_data_transfer_to_device,132 SrcPtrBegin, SrcDeviceId, DstPtrBegin, DstDeviceId, Size, Code);133 }134}135 136void Interface::endTargetDataSubmit(int64_t SrcDeviceId, void *SrcPtrBegin,137 int64_t DstDeviceId, void *DstPtrBegin,138 size_t Size, void *Code) {139 // Only EMI callback handles end scope140 if (ompt_callback_target_data_op_emi_fn) {141 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI142 // callback143 ompt_callback_target_data_op_emi_fn(144 ompt_scope_end, TargetTaskData, &TargetData, &HostOpId,145 ompt_target_data_transfer_to_device, SrcPtrBegin, SrcDeviceId,146 DstPtrBegin, DstDeviceId, Size, Code);147 }148 endTargetDataOperation();149}150 151void Interface::beginTargetDataDelete(int64_t DeviceId, void *TgtPtrBegin,152 void *Code) {153 beginTargetDataOperation();154 if (ompt_callback_target_data_op_emi_fn) {155 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI156 // callback157 ompt_callback_target_data_op_emi_fn(158 ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId,159 ompt_target_data_delete, TgtPtrBegin, DeviceId,160 /*TgtPtrBegin=*/nullptr, /*TgtDeviceNum=*/-1, /*Bytes=*/0, Code);161 } else if (ompt_callback_target_data_op_fn) {162 // HostOpId is set by the runtime163 HostOpId = createOpId();164 // Invoke the tool supplied data op callback165 ompt_callback_target_data_op_fn(TargetData.value, HostOpId,166 ompt_target_data_delete, TgtPtrBegin,167 DeviceId, /*TgtPtrBegin=*/nullptr,168 /*TgtDeviceNum=*/-1, /*Bytes=*/0, Code);169 }170}171 172void Interface::endTargetDataDelete(int64_t DeviceId, void *TgtPtrBegin,173 void *Code) {174 // Only EMI callback handles end scope175 if (ompt_callback_target_data_op_emi_fn) {176 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI177 // callback178 ompt_callback_target_data_op_emi_fn(179 ompt_scope_end, TargetTaskData, &TargetData, &HostOpId,180 ompt_target_data_delete, TgtPtrBegin, DeviceId,181 /*TgtPtrBegin=*/nullptr, /*TgtDeviceNum=*/-1, /*Bytes=*/0, Code);182 }183 endTargetDataOperation();184}185 186void Interface::beginTargetDataRetrieve(int64_t SrcDeviceId, void *SrcPtrBegin,187 int64_t DstDeviceId, void *DstPtrBegin,188 size_t Size, void *Code) {189 beginTargetDataOperation();190 if (ompt_callback_target_data_op_emi_fn) {191 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI192 // callback193 ompt_callback_target_data_op_emi_fn(194 ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId,195 ompt_target_data_transfer_from_device, SrcPtrBegin, SrcDeviceId,196 DstPtrBegin, DstDeviceId, Size, Code);197 } else if (ompt_callback_target_data_op_fn) {198 // HostOpId is set by the runtime199 HostOpId = createOpId();200 // Invoke the tool supplied data op callback201 ompt_callback_target_data_op_fn(202 TargetData.value, HostOpId, ompt_target_data_transfer_from_device,203 SrcPtrBegin, SrcDeviceId, DstPtrBegin, DstDeviceId, Size, Code);204 }205}206 207void Interface::endTargetDataRetrieve(int64_t SrcDeviceId, void *SrcPtrBegin,208 int64_t DstDeviceId, void *DstPtrBegin,209 size_t Size, void *Code) {210 // Only EMI callback handles end scope211 if (ompt_callback_target_data_op_emi_fn) {212 // HostOpId will be set by the tool. Invoke the tool supplied data op EMI213 // callback214 ompt_callback_target_data_op_emi_fn(215 ompt_scope_end, TargetTaskData, &TargetData, &HostOpId,216 ompt_target_data_transfer_from_device, SrcPtrBegin, SrcDeviceId,217 DstPtrBegin, DstDeviceId, Size, Code);218 }219 endTargetDataOperation();220}221 222void Interface::beginTargetSubmit(unsigned int NumTeams) {223 if (ompt_callback_target_submit_emi_fn) {224 // HostOpId is set by the tool. Invoke the tool supplied target submit EMI225 // callback226 ompt_callback_target_submit_emi_fn(ompt_scope_begin, &TargetData, &HostOpId,227 NumTeams);228 } else if (ompt_callback_target_submit_fn) {229 // HostOpId is set by the runtime230 HostOpId = createOpId();231 ompt_callback_target_submit_fn(TargetData.value, HostOpId, NumTeams);232 }233}234 235void Interface::endTargetSubmit(unsigned int NumTeams) {236 // Only EMI callback handles end scope237 if (ompt_callback_target_submit_emi_fn) {238 // HostOpId is set by the tool. Invoke the tool supplied target submit EMI239 // callback240 ompt_callback_target_submit_emi_fn(ompt_scope_end, &TargetData, &HostOpId,241 NumTeams);242 }243}244 245void Interface::beginTargetDataEnter(int64_t DeviceId, void *Code) {246 beginTargetRegion();247 if (ompt_callback_target_emi_fn) {248 // Invoke the tool supplied target EMI callback249 ompt_callback_target_emi_fn(ompt_target_enter_data, ompt_scope_begin,250 DeviceId, TaskData, TargetTaskData, &TargetData,251 Code);252 } else if (ompt_callback_target_fn) {253 // Invoke the tool supplied target callback254 ompt_callback_target_fn(ompt_target_enter_data, ompt_scope_begin, DeviceId,255 TaskData, TargetData.value, Code);256 }257}258 259void Interface::endTargetDataEnter(int64_t DeviceId, void *Code) {260 if (ompt_callback_target_emi_fn) {261 // Invoke the tool supplied target EMI callback262 ompt_callback_target_emi_fn(ompt_target_enter_data, ompt_scope_end,263 DeviceId, TaskData, TargetTaskData, &TargetData,264 Code);265 } else if (ompt_callback_target_fn) {266 // Invoke the tool supplied target callback267 ompt_callback_target_fn(ompt_target_enter_data, ompt_scope_end, DeviceId,268 TaskData, TargetData.value, Code);269 }270 endTargetRegion();271}272 273void Interface::beginTargetDataExit(int64_t DeviceId, void *Code) {274 beginTargetRegion();275 if (ompt_callback_target_emi_fn) {276 // Invoke the tool supplied target EMI callback277 ompt_callback_target_emi_fn(ompt_target_exit_data, ompt_scope_begin,278 DeviceId, TaskData, TargetTaskData, &TargetData,279 Code);280 } else if (ompt_callback_target_fn) {281 TargetData.value = createRegionId();282 // Invoke the tool supplied target callback283 ompt_callback_target_fn(ompt_target_exit_data, ompt_scope_begin, DeviceId,284 TaskData, TargetData.value, Code);285 }286}287 288void Interface::endTargetDataExit(int64_t DeviceId, void *Code) {289 if (ompt_callback_target_emi_fn) {290 // Invoke the tool supplied target EMI callback291 ompt_callback_target_emi_fn(ompt_target_exit_data, ompt_scope_end, DeviceId,292 TaskData, TargetTaskData, &TargetData, Code);293 } else if (ompt_callback_target_fn) {294 // Invoke the tool supplied target callback295 ompt_callback_target_fn(ompt_target_exit_data, ompt_scope_end, DeviceId,296 TaskData, TargetData.value, Code);297 }298 endTargetRegion();299}300 301void Interface::beginTargetUpdate(int64_t DeviceId, void *Code) {302 beginTargetRegion();303 if (ompt_callback_target_emi_fn) {304 // Invoke the tool supplied target EMI callback305 ompt_callback_target_emi_fn(ompt_target_update, ompt_scope_begin, DeviceId,306 TaskData, TargetTaskData, &TargetData, Code);307 } else if (ompt_callback_target_fn) {308 TargetData.value = createRegionId();309 // Invoke the tool supplied target callback310 ompt_callback_target_fn(ompt_target_update, ompt_scope_begin, DeviceId,311 TaskData, TargetData.value, Code);312 }313}314 315void Interface::endTargetUpdate(int64_t DeviceId, void *Code) {316 if (ompt_callback_target_emi_fn) {317 // Invoke the tool supplied target EMI callback318 ompt_callback_target_emi_fn(ompt_target_update, ompt_scope_end, DeviceId,319 TaskData, TargetTaskData, &TargetData, Code);320 } else if (ompt_callback_target_fn) {321 // Invoke the tool supplied target callback322 ompt_callback_target_fn(ompt_target_update, ompt_scope_end, DeviceId,323 TaskData, TargetData.value, Code);324 }325 endTargetRegion();326}327 328void Interface::beginTargetAssociatePointer(int64_t DeviceId, void *HstPtrBegin,329 void *TgtPtrBegin, size_t Size,330 void *Code) {331 beginTargetDataOperation();332 if (ompt_callback_target_data_op_emi_fn) {333 ompt_callback_target_data_op_emi_fn(334 ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId,335 ompt_target_data_associate, HstPtrBegin, omp_get_initial_device(),336 TgtPtrBegin, DeviceId, Size, Code);337 } else if (ompt_callback_target_data_op_fn) {338 HostOpId = createOpId();339 ompt_callback_target_data_op_fn(340 TargetData.value, HostOpId, ompt_target_data_associate, HstPtrBegin,341 omp_get_initial_device(), TgtPtrBegin, DeviceId, Size, Code);342 }343}344 345void Interface::endTargetAssociatePointer(int64_t DeviceId, void *HstPtrBegin,346 void *TgtPtrBegin, size_t Size,347 void *Code) {348 if (ompt_callback_target_data_op_emi_fn) {349 ompt_callback_target_data_op_emi_fn(350 ompt_scope_end, TargetTaskData, &TargetData, &HostOpId,351 ompt_target_data_associate, HstPtrBegin, omp_get_initial_device(),352 TgtPtrBegin, DeviceId, Size, Code);353 }354}355 356void Interface::beginTargetDisassociatePointer(int64_t DeviceId,357 void *HstPtrBegin,358 void *TgtPtrBegin, size_t Size,359 void *Code) {360 beginTargetDataOperation();361 if (ompt_callback_target_data_op_emi_fn) {362 ompt_callback_target_data_op_emi_fn(363 ompt_scope_begin, TargetTaskData, &TargetData, &HostOpId,364 ompt_target_data_disassociate, HstPtrBegin, omp_get_initial_device(),365 TgtPtrBegin, DeviceId, Size, Code);366 } else if (ompt_callback_target_data_op_fn) {367 HostOpId = createOpId();368 ompt_callback_target_data_op_fn(369 TargetData.value, HostOpId, ompt_target_data_disassociate, HstPtrBegin,370 omp_get_initial_device(), TgtPtrBegin, DeviceId, Size, Code);371 }372}373void Interface::endTargetDisassociatePointer(int64_t DeviceId,374 void *HstPtrBegin,375 void *TgtPtrBegin, size_t Size,376 void *Code) {377 if (ompt_callback_target_data_op_emi_fn) {378 ompt_callback_target_data_op_emi_fn(379 ompt_scope_end, TargetTaskData, &TargetData, &HostOpId,380 ompt_target_data_disassociate, HstPtrBegin, omp_get_initial_device(),381 TgtPtrBegin, DeviceId, Size, Code);382 }383}384 385void Interface::beginTarget(int64_t DeviceId, void *Code) {386 beginTargetRegion();387 if (ompt_callback_target_emi_fn) {388 // Invoke the tool supplied target EMI callback389 ompt_callback_target_emi_fn(ompt_target, ompt_scope_begin, DeviceId,390 TaskData, TargetTaskData, &TargetData, Code);391 } else if (ompt_callback_target_fn) {392 TargetData.value = createRegionId();393 // Invoke the tool supplied target callback394 ompt_callback_target_fn(ompt_target, ompt_scope_begin, DeviceId, TaskData,395 TargetData.value, Code);396 }397}398 399void Interface::endTarget(int64_t DeviceId, void *Code) {400 if (ompt_callback_target_emi_fn) {401 // Invoke the tool supplied target EMI callback402 ompt_callback_target_emi_fn(ompt_target, ompt_scope_end, DeviceId, TaskData,403 TargetTaskData, &TargetData, Code);404 } else if (ompt_callback_target_fn) {405 // Invoke the tool supplied target callback406 ompt_callback_target_fn(ompt_target, ompt_scope_end, DeviceId, TaskData,407 TargetData.value, Code);408 }409 endTargetRegion();410}411 412void Interface::beginTargetDataOperation() {413 DP("in ompt_target_region_begin (TargetRegionId = %lu)\n", TargetData.value);414}415 416void Interface::endTargetDataOperation() {417 DP("in ompt_target_region_end (TargetRegionId = %lu)\n", TargetData.value);418}419 420void Interface::beginTargetRegion() {421 // Set up task state422 assert(ompt_get_task_data_fn && "Calling a null task data function");423 TaskData = ompt_get_task_data_fn();424 // Set up target task state425 assert(ompt_get_target_task_data_fn &&426 "Calling a null target task data function");427 TargetTaskData = ompt_get_target_task_data_fn();428 // Target state will be set later429 TargetData = ompt_data_none;430}431 432void Interface::endTargetRegion() {433 TaskData = 0;434 TargetTaskData = 0;435 TargetData = ompt_data_none;436}437 438/// Used to maintain the finalization functions that are received439/// from the plugins during connect.440/// Note: Currently, there are no plugin-specific finalizations, so each plugin441/// will call the same (empty) function.442class LibomptargetRtlFinalizer {443public:444 LibomptargetRtlFinalizer() {}445 446 void registerRtl(ompt_finalize_t FinalizationFunction) {447 if (FinalizationFunction) {448 RtlFinalizationFunctions.emplace_back(FinalizationFunction);449 }450 }451 452 void finalize() {453 for (auto FinalizationFunction : RtlFinalizationFunctions)454 FinalizationFunction(/*tool_data=*/nullptr);455 RtlFinalizationFunctions.clear();456 }457 458private:459 llvm::SmallVector<ompt_finalize_t> RtlFinalizationFunctions;460};461 462int llvm::omp::target::ompt::initializeLibrary(ompt_function_lookup_t lookup,463 int initial_device_num,464 ompt_data_t *tool_data) {465 DP("Executing initializeLibrary\n");466#define bindOmptFunctionName(OmptFunction, DestinationFunction) \467 if (lookup) \468 DestinationFunction = (OmptFunction##_t)lookup(#OmptFunction); \469 DP("initializeLibrary bound %s=%p\n", #DestinationFunction, \470 ((void *)(uint64_t)DestinationFunction));471 472 bindOmptFunctionName(ompt_get_callback, lookupCallbackByCode);473 bindOmptFunctionName(ompt_get_task_data, ompt_get_task_data_fn);474 bindOmptFunctionName(ompt_get_target_task_data, ompt_get_target_task_data_fn);475#undef bindOmptFunctionName476 477 // Store pointer of 'ompt_libomp_target_fn_lookup' for use by libomptarget478 lookupCallbackByName = lookup;479 480 assert(lookupCallbackByCode && "lookupCallbackByCode should be non-null");481 assert(lookupCallbackByName && "lookupCallbackByName should be non-null");482 assert(ompt_get_task_data_fn && "ompt_get_task_data_fn should be non-null");483 assert(ompt_get_target_task_data_fn &&484 "ompt_get_target_task_data_fn should be non-null");485 assert(LibraryFinalizer == nullptr &&486 "LibraryFinalizer should not be initialized yet");487 488 LibraryFinalizer = new LibomptargetRtlFinalizer();489 490 Initialized = true;491 492 return 0;493}494 495void llvm::omp::target::ompt::finalizeLibrary(ompt_data_t *data) {496 DP("Executing finalizeLibrary\n");497 // Before disabling OMPT, call the (plugin) finalizations that were registered498 // with this library499 LibraryFinalizer->finalize();500 delete LibraryFinalizer;501 Initialized = false;502}503 504void llvm::omp::target::ompt::connectLibrary() {505 DP("Entering connectLibrary\n");506 // Connect with libomp507 static OmptLibraryConnectorTy LibompConnector("libomp");508 static ompt_start_tool_result_t OmptResult;509 510 // Initialize OmptResult with the init and fini functions that will be511 // called by the connector512 OmptResult.initialize = ompt::initializeLibrary;513 OmptResult.finalize = ompt::finalizeLibrary;514 OmptResult.tool_data.value = 0;515 516 // Now call connect that causes the above init/fini functions to be called517 LibompConnector.connect(&OmptResult);518 519#define bindOmptCallback(Name, Type, Code) \520 if (lookupCallbackByCode) \521 lookupCallbackByCode( \522 (ompt_callbacks_t)(Code), \523 (ompt_callback_t *)&(llvm::omp::target::ompt::Name##_fn));524 FOREACH_OMPT_NOEMI_EVENT(bindOmptCallback)525 FOREACH_OMPT_EMI_EVENT(bindOmptCallback)526#undef bindOmptCallback527 528 DP("Exiting connectLibrary\n");529}530 531#endif // OMPT_SUPPORT532