580 lines · c
1/*===- InstrProfilingUtil.c - Support library for PGO instrumentation -----===*\2|*3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4|* See https://llvm.org/LICENSE.txt for license information.5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6|*7\*===----------------------------------------------------------------------===*/8 9#ifdef _WIN3210#include <direct.h>11#include <process.h>12#include <windows.h>13#include "WindowsMMap.h"14#else15#if defined(__linux__)16// For fdopen(), fileno(), getpagesize(), madvise()17#define _DEFAULT_SOURCE18#endif19 20#include <errno.h>21#include <fcntl.h>22#include <sys/file.h>23#include <sys/mman.h>24#include <sys/stat.h>25#include <sys/types.h>26#include <unistd.h>27#endif28 29#ifdef _AIX30#include <sys/statfs.h>31// <sys/vmount.h> depends on `uint` to be a typedef from <sys/types.h> to32// `uint_t`; however, <sys/types.h> does not always declare `uint`. We provide33// the typedef prior to including <sys/vmount.h> to work around this issue.34typedef uint_t uint;35#include <sys/vmount.h>36#endif37 38#ifdef COMPILER_RT_HAS_UNAME39#include <sys/utsname.h>40#endif41 42#include <stdlib.h>43#include <string.h>44 45#if defined(__linux__)46#include <signal.h>47#include <sys/prctl.h>48#endif49 50#if defined(__Fuchsia__)51#include <zircon/process.h>52#include <zircon/syscalls.h>53#endif54 55#if defined(__FreeBSD__)56#include <signal.h>57#include <sys/procctl.h>58#endif59 60#include "InstrProfiling.h"61#include "InstrProfilingUtil.h"62 63COMPILER_RT_VISIBILITY unsigned lprofDirMode = 0755;64 65COMPILER_RT_VISIBILITY66void __llvm_profile_recursive_mkdir(char *path) {67 int i;68 int start = 1;69 70#if defined(__ANDROID__) && defined(__ANDROID_API__) && \71 defined(__ANDROID_API_FUTURE__) && \72 __ANDROID_API__ == __ANDROID_API_FUTURE__73 // Avoid spammy selinux denial messages in Android by not attempting to74 // create directories in GCOV_PREFIX. These denials occur when creating (or75 // even attempting to stat()) top-level directories like "/data".76 //77 // Do so by ignoring ${GCOV_PREFIX} when invoking mkdir().78 const char *gcov_prefix = getenv("GCOV_PREFIX");79 if (gcov_prefix != NULL) {80 const int gcov_prefix_len = strlen(gcov_prefix);81 if (strncmp(path, gcov_prefix, gcov_prefix_len) == 0)82 start = gcov_prefix_len;83 }84#endif85 86 for (i = start; path[i] != '\0'; ++i) {87 char save = path[i];88 if (!IS_DIR_SEPARATOR(path[i]))89 continue;90 path[i] = '\0';91#ifdef _WIN3292 _mkdir(path);93#else94 /* Some of these will fail, ignore it. */95 mkdir(path, __llvm_profile_get_dir_mode());96#endif97 path[i] = save;98 }99}100 101COMPILER_RT_VISIBILITY102void __llvm_profile_set_dir_mode(unsigned Mode) { lprofDirMode = Mode; }103 104COMPILER_RT_VISIBILITY105unsigned __llvm_profile_get_dir_mode(void) { return lprofDirMode; }106 107#if COMPILER_RT_HAS_ATOMICS != 1108COMPILER_RT_VISIBILITY109uint32_t lprofBoolCmpXchg(void **Ptr, void *OldV, void *NewV) {110 void *R = *Ptr;111 if (R == OldV) {112 *Ptr = NewV;113 return 1;114 }115 return 0;116}117COMPILER_RT_VISIBILITY118void *lprofPtrFetchAdd(void **Mem, long ByteIncr) {119 void *Old = *Mem;120 *((char **)Mem) += ByteIncr;121 return Old;122}123 124#endif125 126#ifdef _WIN32127COMPILER_RT_VISIBILITY int lprofGetHostName(char *Name, int Len) {128 WCHAR Buffer[COMPILER_RT_MAX_HOSTLEN];129 DWORD BufferSize = sizeof(Buffer);130 BOOL Result =131 GetComputerNameExW(ComputerNameDnsFullyQualified, Buffer, &BufferSize);132 if (!Result)133 return -1;134 if (WideCharToMultiByte(CP_UTF8, 0, Buffer, -1, Name, Len, NULL, NULL) == 0)135 return -1;136 return 0;137}138#elif defined(COMPILER_RT_HAS_UNAME)139COMPILER_RT_VISIBILITY int lprofGetHostName(char *Name, int Len) {140 struct utsname N;141 int R = uname(&N);142 if (R >= 0) {143 strncpy(Name, N.nodename, Len);144 return 0;145 }146 return R;147}148#endif149 150COMPILER_RT_VISIBILITY int lprofLockFd(int fd) {151#ifdef COMPILER_RT_HAS_FCNTL_LCK152 struct flock s_flock;153 154 s_flock.l_whence = SEEK_SET;155 s_flock.l_start = 0;156 s_flock.l_len = 0; /* Until EOF. */157 s_flock.l_pid = getpid();158 s_flock.l_type = F_WRLCK;159 160 while (fcntl(fd, F_SETLKW, &s_flock) == -1) {161 if (errno != EINTR) {162 if (errno == ENOLCK) {163 return -1;164 }165 break;166 }167 }168 return 0;169#elif defined(COMPILER_RT_HAS_FLOCK) || defined(_WIN32)170 // Windows doesn't have flock but WindowsMMap.h provides a shim171 flock(fd, LOCK_EX);172 return 0;173#else174 return 0;175#endif176}177 178COMPILER_RT_VISIBILITY int lprofUnlockFd(int fd) {179#ifdef COMPILER_RT_HAS_FCNTL_LCK180 struct flock s_flock;181 182 s_flock.l_whence = SEEK_SET;183 s_flock.l_start = 0;184 s_flock.l_len = 0; /* Until EOF. */185 s_flock.l_pid = getpid();186 s_flock.l_type = F_UNLCK;187 188 while (fcntl(fd, F_SETLKW, &s_flock) == -1) {189 if (errno != EINTR) {190 if (errno == ENOLCK) {191 return -1;192 }193 break;194 }195 }196 return 0;197#elif defined(COMPILER_RT_HAS_FLOCK) || defined(_WIN32)198 // Windows doesn't have flock but WindowsMMap.h provides a shim199 flock(fd, LOCK_UN);200 return 0;201#else202 return 0;203#endif204}205 206COMPILER_RT_VISIBILITY int lprofLockFileHandle(FILE *F) {207 int fd;208#if defined(_WIN32)209 fd = _fileno(F);210#else211 fd = fileno(F);212#endif213 return lprofLockFd(fd);214}215 216COMPILER_RT_VISIBILITY int lprofUnlockFileHandle(FILE *F) {217 int fd;218#if defined(_WIN32)219 fd = _fileno(F);220#else221 fd = fileno(F);222#endif223 return lprofUnlockFd(fd);224}225 226COMPILER_RT_VISIBILITY FILE *lprofOpenFileEx(const char *ProfileName) {227 FILE *f;228 int fd;229#ifdef COMPILER_RT_HAS_FCNTL_LCK230 fd = open(ProfileName, O_RDWR | O_CREAT, 0666);231 if (fd < 0)232 return NULL;233 234 if (lprofLockFd(fd) != 0)235 PROF_WARN("Data may be corrupted during profile merging : %s\n",236 "Fail to obtain file lock due to system limit.");237 238 f = fdopen(fd, "r+b");239#elif defined(_WIN32)240 // FIXME: Use the wide variants to handle Unicode filenames.241 HANDLE h = CreateFileA(ProfileName, GENERIC_READ | GENERIC_WRITE,242 FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_ALWAYS,243 FILE_ATTRIBUTE_NORMAL, 0);244 if (h == INVALID_HANDLE_VALUE)245 return NULL;246 247 fd = _open_osfhandle((intptr_t)h, 0);248 if (fd == -1) {249 CloseHandle(h);250 return NULL;251 }252 253 if (lprofLockFd(fd) != 0)254 PROF_WARN("Data may be corrupted during profile merging : %s\n",255 "Fail to obtain file lock due to system limit.");256 257 f = _fdopen(fd, "r+b");258 if (f == 0) {259 CloseHandle(h);260 return NULL;261 }262#else263 /* Worst case no locking applied. */264 PROF_WARN("Concurrent file access is not supported : %s\n",265 "lack file locking");266 fd = open(ProfileName, O_RDWR | O_CREAT, 0666);267 if (fd < 0)268 return NULL;269 f = fdopen(fd, "r+b");270#endif271 272 return f;273}274 275#if defined(_AIX)276// Return 1 (true) if the file descriptor Fd represents a file that is on a277// local filesystem, otherwise return 0.278static int isLocalFilesystem(int Fd) {279 struct statfs Vfs;280 if (fstatfs(Fd, &Vfs) != 0) {281 PROF_ERR("%s: fstatfs(%d) failed: %s\n", __func__, Fd, strerror(errno));282 return 0;283 }284 285 int Ret;286 size_t BufSize = 2048u;287 char *Buf;288 int Tries = 3;289 while (Tries--) {290 Buf = malloc(BufSize);291 // mntctl returns -1 if `Buf` is `NULL`.292 Ret = mntctl(MCTL_QUERY, BufSize, Buf);293 if (Ret != 0)294 break;295 BufSize = *(unsigned int *)Buf;296 free(Buf);297 }298 299 if (Ret != -1) {300 // Look for the correct vmount entry.301 char *CurObjPtr = Buf;302 while (Ret--) {303 struct vmount *Vp = (struct vmount *)CurObjPtr;304 _Static_assert(sizeof(Vfs.f_fsid) == sizeof(Vp->vmt_fsid),305 "fsid length mismatch");306 if (memcmp(&Vfs.f_fsid, &Vp->vmt_fsid, sizeof Vfs.f_fsid) == 0) {307 int Answer = (Vp->vmt_flags & MNT_REMOTE) == 0;308 free(Buf);309 return Answer;310 }311 CurObjPtr += Vp->vmt_length;312 }313 }314 315 free(Buf);316 // There was an error in mntctl or vmount entry not found; "remote" is the317 // conservative answer.318 return 0;319}320#endif321 322static int isMmapSafe(int Fd) {323 if (getenv("LLVM_PROFILE_NO_MMAP")) // For testing purposes.324 return 0;325#ifdef _AIX326 return isLocalFilesystem(Fd);327#else328 return 1;329#endif330}331 332COMPILER_RT_VISIBILITY void lprofGetFileContentBuffer(FILE *F, uint64_t Length,333 ManagedMemory *Buf) {334 Buf->Status = MS_INVALID;335 if (isMmapSafe(fileno(F))) {336 Buf->Addr =337 mmap(NULL, Length, PROT_READ, MAP_SHARED | MAP_FILE, fileno(F), 0);338 if (Buf->Addr == MAP_FAILED)339 PROF_ERR("%s: mmap failed: %s\n", __func__, strerror(errno))340 else341 Buf->Status = MS_MMAP;342 return;343 }344 345 if (getenv("LLVM_PROFILE_VERBOSE"))346 PROF_NOTE("%s\n", "could not use mmap; using fread instead");347 348 void *Buffer = malloc(Length);349 if (!Buffer) {350 PROF_ERR("%s: malloc failed: %s\n", __func__, strerror(errno));351 return;352 }353 if (ftell(F) != 0) {354 PROF_ERR("%s: expecting ftell to return zero\n", __func__);355 free(Buffer);356 return;357 }358 359 // Read the entire file into memory.360 size_t BytesRead = fread(Buffer, 1, Length, F);361 if (BytesRead != (size_t)Length) {362 PROF_ERR("%s: fread failed%s\n", __func__,363 feof(F) ? ": end of file reached" : "");364 free(Buffer);365 return;366 }367 368 // Reading was successful, record the result in the Buf parameter.369 Buf->Addr = Buffer;370 Buf->Status = MS_MALLOC;371}372 373COMPILER_RT_VISIBILITY374void lprofReleaseBuffer(ManagedMemory *Buf, size_t Length) {375 switch (Buf->Status) {376 case MS_MALLOC:377 free(Buf->Addr);378 break;379 case MS_MMAP:380 (void)munmap(Buf->Addr, Length);381 break;382 default:383 PROF_ERR("%s: Buffer has invalid state: %d\n", __func__, Buf->Status);384 break;385 }386 Buf->Addr = NULL;387 Buf->Status = MS_INVALID;388}389 390COMPILER_RT_VISIBILITY const char *lprofGetPathPrefix(int *PrefixStrip,391 size_t *PrefixLen) {392 const char *Prefix = getenv("GCOV_PREFIX");393 const char *PrefixStripStr = getenv("GCOV_PREFIX_STRIP");394 395 *PrefixLen = 0;396 *PrefixStrip = 0;397 if (Prefix == NULL || Prefix[0] == '\0')398 return NULL;399 400 if (PrefixStripStr) {401 *PrefixStrip = atoi(PrefixStripStr);402 403 /* Negative GCOV_PREFIX_STRIP values are ignored */404 if (*PrefixStrip < 0)405 *PrefixStrip = 0;406 } else {407 *PrefixStrip = 0;408 }409 *PrefixLen = strlen(Prefix);410 411 return Prefix;412}413 414COMPILER_RT_VISIBILITY void415lprofApplyPathPrefix(char *Dest, const char *PathStr, const char *Prefix,416 size_t PrefixLen, int PrefixStrip) {417 418 const char *Ptr;419 int Level;420 const char *StrippedPathStr = PathStr;421 422 for (Level = 0, Ptr = PathStr + 1; Level < PrefixStrip; ++Ptr) {423 if (*Ptr == '\0')424 break;425 426 if (!IS_DIR_SEPARATOR(*Ptr))427 continue;428 429 StrippedPathStr = Ptr;430 ++Level;431 }432 433 memcpy(Dest, Prefix, PrefixLen);434 435 if (!IS_DIR_SEPARATOR(Prefix[PrefixLen - 1]))436 Dest[PrefixLen++] = DIR_SEPARATOR;437 438 memcpy(Dest + PrefixLen, StrippedPathStr, strlen(StrippedPathStr) + 1);439}440 441COMPILER_RT_VISIBILITY const char *442lprofFindFirstDirSeparator(const char *Path) {443 const char *Sep = strchr(Path, DIR_SEPARATOR);444#if defined(DIR_SEPARATOR_2)445 const char *Sep2 = strchr(Path, DIR_SEPARATOR_2);446 if (Sep2 && (!Sep || Sep2 < Sep))447 Sep = Sep2;448#endif449 return Sep;450}451 452COMPILER_RT_VISIBILITY const char *lprofFindLastDirSeparator(const char *Path) {453 const char *Sep = strrchr(Path, DIR_SEPARATOR);454#if defined(DIR_SEPARATOR_2)455 const char *Sep2 = strrchr(Path, DIR_SEPARATOR_2);456 if (Sep2 && (!Sep || Sep2 > Sep))457 Sep = Sep2;458#endif459 return Sep;460}461 462COMPILER_RT_VISIBILITY int lprofSuspendSigKill(void) {463#if defined(__linux__)464 int PDeachSig = 0;465 /* Temporarily suspend getting SIGKILL upon exit of the parent process. */466 if (prctl(PR_GET_PDEATHSIG, &PDeachSig) == 0 && PDeachSig == SIGKILL)467 prctl(PR_SET_PDEATHSIG, 0);468 return (PDeachSig == SIGKILL);469#elif defined(__FreeBSD__)470 int PDeachSig = 0, PDisableSig = 0;471 if (procctl(P_PID, 0, PROC_PDEATHSIG_STATUS, &PDeachSig) == 0 &&472 PDeachSig == SIGKILL)473 procctl(P_PID, 0, PROC_PDEATHSIG_CTL, &PDisableSig);474 return (PDeachSig == SIGKILL);475#else476 return 0;477#endif478}479 480COMPILER_RT_VISIBILITY void lprofRestoreSigKill(void) {481#if defined(__linux__)482 prctl(PR_SET_PDEATHSIG, SIGKILL);483#elif defined(__FreeBSD__)484 int PEnableSig = SIGKILL;485 procctl(P_PID, 0, PROC_PDEATHSIG_CTL, &PEnableSig);486#endif487}488 489COMPILER_RT_VISIBILITY int lprofReleaseMemoryPagesToOS(uintptr_t Begin,490 uintptr_t End) {491#if defined(__ve__) || defined(__wasi__)492 // VE and WASI doesn't support madvise.493 return 0;494#else495 size_t PageSize = getpagesize();496 uintptr_t BeginAligned = lprofRoundUpTo((uintptr_t)Begin, PageSize);497 uintptr_t EndAligned = lprofRoundDownTo((uintptr_t)End, PageSize);498 if (BeginAligned < EndAligned) {499#if defined(__Fuchsia__)500 return _zx_vmar_op_range(_zx_vmar_root_self(), ZX_VMAR_OP_DECOMMIT,501 (zx_vaddr_t)BeginAligned,502 EndAligned - BeginAligned, NULL, 0);503#else504 return madvise((void *)BeginAligned, EndAligned - BeginAligned,505 MADV_DONTNEED);506#endif507 }508 return 0;509#endif510}511 512#ifdef _AIX513typedef struct fn_node {514 AtExit_Fn_ptr func;515 struct fn_node *next;516} fn_node;517typedef struct {518 fn_node *top;519} fn_stack;520 521static void fn_stack_push(fn_stack *, AtExit_Fn_ptr);522static AtExit_Fn_ptr fn_stack_pop(fn_stack *);523/* return 1 if stack is empty, 0 otherwise */524static int fn_stack_is_empty(fn_stack *);525 526static fn_stack AtExit_stack = {0};527#define ATEXIT_STACK (&AtExit_stack)528 529/* On AIX, atexit() functions registered by a shared library do not get called530 * when the library is dlclose'd, causing a crash when they are eventually531 * called at main program exit. However, a destructor does get called. So we532 * collect all atexit functions registered by profile-rt and at program533 * termination time (normal exit, shared library unload, or dlclose) we walk534 * the list and execute any function that is still sitting in the atexit system535 * queue.536 */537__attribute__((__destructor__)) static void cleanup() {538 while (!fn_stack_is_empty(ATEXIT_STACK)) {539 AtExit_Fn_ptr func = fn_stack_pop(ATEXIT_STACK);540 if (func && unatexit(func) == 0)541 func();542 }543}544 545static void fn_stack_push(fn_stack *st, AtExit_Fn_ptr func) {546 fn_node *old_top, *n = (fn_node *)malloc(sizeof(fn_node));547 n->func = func;548 549 while (1) {550 old_top = st->top;551 n->next = old_top;552 if (COMPILER_RT_BOOL_CMPXCHG(&st->top, old_top, n))553 return;554 }555}556static AtExit_Fn_ptr fn_stack_pop(fn_stack *st) {557 fn_node *old_top, *new_top;558 while (1) {559 old_top = st->top;560 if (old_top == 0)561 return 0;562 new_top = old_top->next;563 if (COMPILER_RT_BOOL_CMPXCHG(&st->top, old_top, new_top)) {564 AtExit_Fn_ptr func = old_top->func;565 free(old_top);566 return func;567 }568 }569}570 571static int fn_stack_is_empty(fn_stack *st) { return st->top == 0; }572#endif573 574COMPILER_RT_VISIBILITY int lprofAtExit(AtExit_Fn_ptr func) {575#ifdef _AIX576 fn_stack_push(ATEXIT_STACK, func);577#endif578 return atexit(func);579}580