1096 lines · cpp
1//===-- sanitizer_linux_libcdep.cpp ---------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file is shared between AddressSanitizer and ThreadSanitizer10// run-time libraries and implements linux-specific functions from11// sanitizer_libc.h.12//===----------------------------------------------------------------------===//13 14#include "sanitizer_platform.h"15 16#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \17 SANITIZER_SOLARIS || SANITIZER_HAIKU18 19# include "sanitizer_allocator_internal.h"20# include "sanitizer_atomic.h"21# include "sanitizer_common.h"22# include "sanitizer_file.h"23# include "sanitizer_flags.h"24# include "sanitizer_getauxval.h"25# include "sanitizer_glibc_version.h"26# include "sanitizer_linux.h"27# include "sanitizer_placement_new.h"28# include "sanitizer_procmaps.h"29# include "sanitizer_solaris.h"30 31# if SANITIZER_HAIKU32# define _GNU_SOURCE33# define _DEFAULT_SOURCE34# endif35 36# if SANITIZER_NETBSD37# // for __lwp_gettcb_fast() / __lwp_getprivate_fast()38# define _RTLD_SOURCE39# include <machine/mcontext.h>40# undef _RTLD_SOURCE41# include <sys/param.h>42# if __NetBSD_Version__ >= 109900120043# include <machine/lwp_private.h>44# endif45# endif46 47# include <dlfcn.h> // for dlsym()48# include <link.h>49# include <pthread.h>50# include <signal.h>51# include <sys/mman.h>52# include <sys/resource.h>53# include <syslog.h>54 55# if SANITIZER_GLIBC56# include <gnu/libc-version.h>57# endif58 59# if !defined(ElfW)60# define ElfW(type) Elf_##type61# endif62 63# if SANITIZER_FREEBSD64# include <pthread_np.h>65# include <sys/auxv.h>66# include <sys/sysctl.h>67# define pthread_getattr_np pthread_attr_get_np68// The MAP_NORESERVE define has been removed in FreeBSD 11.x, and even before69// that, it was never implemented. So just define it to zero.70# undef MAP_NORESERVE71# define MAP_NORESERVE 072extern const Elf_Auxinfo *__elf_aux_vector __attribute__((weak));73extern "C" int __sys_sigaction(int signum, const struct sigaction *act,74 struct sigaction *oldact);75# endif76 77# if SANITIZER_NETBSD78# include <lwp.h>79# include <sys/sysctl.h>80# include <sys/tls.h>81# endif82 83# if SANITIZER_SOLARIS84# include <stddef.h>85# include <stdlib.h>86# include <thread.h>87# endif88 89# if SANITIZER_HAIKU90# include <kernel/OS.h>91# include <sys/link_elf.h>92# endif93 94# if !SANITIZER_ANDROID95# include <elf.h>96# include <unistd.h>97# endif98 99namespace __sanitizer {100 101SANITIZER_WEAK_ATTRIBUTE int real_sigaction(int signum, const void *act,102 void *oldact);103 104int internal_sigaction(int signum, const void *act, void *oldact) {105# if SANITIZER_FREEBSD106 // On FreeBSD, call the sigaction syscall directly (part of libsys in FreeBSD107 // 15) since the libc version goes via a global interposing table. Due to108 // library initialization order the table can be relocated after the call to109 // InitializeDeadlySignals() which then crashes when dereferencing the110 // uninitialized pointer in libc.111 return __sys_sigaction(signum, (const struct sigaction *)act,112 (struct sigaction *)oldact);113# else114# if !SANITIZER_GO115 if (&real_sigaction)116 return real_sigaction(signum, act, oldact);117# endif118 return sigaction(signum, (const struct sigaction *)act,119 (struct sigaction *)oldact);120# endif121}122 123void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,124 uptr *stack_bottom) {125 CHECK(stack_top);126 CHECK(stack_bottom);127 if (at_initialization) {128 // This is the main thread. Libpthread may not be initialized yet.129 struct rlimit rl;130 CHECK_EQ(getrlimit(RLIMIT_STACK, &rl), 0);131 132 // Find the mapping that contains a stack variable.133 MemoryMappingLayout proc_maps(/*cache_enabled*/ true);134 if (proc_maps.Error()) {135 *stack_top = *stack_bottom = 0;136 return;137 }138 MemoryMappedSegment segment;139 uptr prev_end = 0;140 while (proc_maps.Next(&segment)) {141 if ((uptr)&rl < segment.end)142 break;143 prev_end = segment.end;144 }145 CHECK((uptr)&rl >= segment.start && (uptr)&rl < segment.end);146 147 // Get stacksize from rlimit, but clip it so that it does not overlap148 // with other mappings.149 uptr stacksize = rl.rlim_cur;150 if (stacksize > segment.end - prev_end)151 stacksize = segment.end - prev_end;152 // When running with unlimited stack size, we still want to set some limit.153 // The unlimited stack size is caused by 'ulimit -s unlimited'.154 // Also, for some reason, GNU make spawns subprocesses with unlimited stack.155 if (stacksize > kMaxThreadStackSize)156 stacksize = kMaxThreadStackSize;157 *stack_top = segment.end;158 *stack_bottom = segment.end - stacksize;159 160 uptr maxAddr = GetMaxUserVirtualAddress();161 // Edge case: the stack mapping on some systems may be off-by-one e.g.,162 // fffffffdf000-1000000000000 rw-p 00000000 00:00 0 [stack]163 // instead of:164 // fffffffdf000- ffffffffffff165 // The out-of-range stack_top can result in an invalid shadow address166 // calculation, since those usually assume the parameters are in range.167 if (*stack_top == maxAddr + 1)168 *stack_top = maxAddr;169 else170 CHECK_LE(*stack_top, maxAddr);171 172 return;173 }174 uptr stacksize = 0;175 void *stackaddr = nullptr;176# if SANITIZER_SOLARIS177 stack_t ss;178 CHECK_EQ(thr_stksegment(&ss), 0);179 stacksize = ss.ss_size;180 stackaddr = (char *)ss.ss_sp - stacksize;181# else // !SANITIZER_SOLARIS182 pthread_attr_t attr;183 pthread_attr_init(&attr);184 CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0);185 internal_pthread_attr_getstack(&attr, &stackaddr, &stacksize);186 pthread_attr_destroy(&attr);187# endif // SANITIZER_SOLARIS188 189 *stack_top = (uptr)stackaddr + stacksize;190 *stack_bottom = (uptr)stackaddr;191}192 193# if !SANITIZER_GO194bool SetEnv(const char *name, const char *value) {195 void *f = dlsym(RTLD_NEXT, "setenv");196 if (!f)197 return false;198 typedef int (*setenv_ft)(const char *name, const char *value, int overwrite);199 setenv_ft setenv_f;200 CHECK_EQ(sizeof(setenv_f), sizeof(f));201 internal_memcpy(&setenv_f, &f, sizeof(f));202 return setenv_f(name, value, 1) == 0;203}204# endif205 206// True if we can use dlpi_tls_data. glibc before 2.25 may leave NULL (BZ207// #19826) so dlpi_tls_data cannot be used.208//209// musl before 1.2.3 and FreeBSD as of 12.2 incorrectly set dlpi_tls_data to210// the TLS initialization image211// https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=254774212__attribute__((unused)) static int g_use_dlpi_tls_data;213 214# if SANITIZER_GLIBC && !SANITIZER_GO215static void GetGLibcVersion(int *major, int *minor, int *patch) {216 const char *p = gnu_get_libc_version();217 *major = internal_simple_strtoll(p, &p, 10);218 // Caller does not expect anything else.219 CHECK_EQ(*major, 2);220 *minor = (*p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;221 *patch = (*p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;222}223 224static uptr ThreadDescriptorSizeFallback() {225# if defined(__x86_64__) || defined(__i386__) || defined(__arm__) || \226 SANITIZER_RISCV64227 int major;228 int minor;229 int patch;230 GetGLibcVersion(&major, &minor, &patch);231# endif232 233# if defined(__x86_64__) || defined(__i386__) || defined(__arm__)234 /* sizeof(struct pthread) values from various glibc versions. */235 if (SANITIZER_X32)236 return 1728; // Assume only one particular version for x32.237 // For ARM sizeof(struct pthread) changed in Glibc 2.23.238 if (SANITIZER_ARM)239 return minor <= 22 ? 1120 : 1216;240 if (minor <= 3)241 return FIRST_32_SECOND_64(1104, 1696);242 if (minor == 4)243 return FIRST_32_SECOND_64(1120, 1728);244 if (minor == 5)245 return FIRST_32_SECOND_64(1136, 1728);246 if (minor <= 9)247 return FIRST_32_SECOND_64(1136, 1712);248 if (minor == 10)249 return FIRST_32_SECOND_64(1168, 1776);250 if (minor == 11 || (minor == 12 && patch == 1))251 return FIRST_32_SECOND_64(1168, 2288);252 if (minor <= 14)253 return FIRST_32_SECOND_64(1168, 2304);254 if (minor < 32) // Unknown version255 return FIRST_32_SECOND_64(1216, 2304);256 // minor == 32257 return FIRST_32_SECOND_64(1344, 2496);258# endif259 260# if SANITIZER_RISCV64261 // TODO: consider adding an optional runtime check for an unknown (untested)262 // glibc version263 if (minor <= 28) // WARNING: the highest tested version is 2.29264 return 1772; // no guarantees for this one265 if (minor <= 31)266 return 1772; // tested against glibc 2.29, 2.31267 return 1936; // tested against glibc 2.32268# endif269 270# if defined(__s390__) || defined(__sparc__)271 // The size of a prefix of TCB including pthread::{specific_1stblock,specific}272 // suffices. Just return offsetof(struct pthread, specific_used), which hasn't273 // changed since 2007-05. Technically this applies to i386/x86_64 as well but274 // we call _dl_get_tls_static_info and need the precise size of struct275 // pthread.276 return FIRST_32_SECOND_64(524, 1552);277# endif278 279# if defined(__mips__)280 // TODO(sagarthakur): add more values as per different glibc versions.281 return FIRST_32_SECOND_64(1152, 1776);282# endif283 284# if SANITIZER_LOONGARCH64285 return 1856; // from glibc 2.36286# endif287 288# if defined(__aarch64__)289 // The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22.290 return 1776;291# endif292 293# if defined(__powerpc64__)294 return 1776; // from glibc.ppc64le 2.20-8.fc21295# endif296}297# endif // SANITIZER_GLIBC && !SANITIZER_GO298 299# if SANITIZER_FREEBSD && !SANITIZER_GO300// FIXME: Implementation is very GLIBC specific, but it's used by FreeBSD.301static uptr ThreadDescriptorSizeFallback() {302# if defined(__s390__) || defined(__sparc__)303 // The size of a prefix of TCB including pthread::{specific_1stblock,specific}304 // suffices. Just return offsetof(struct pthread, specific_used), which hasn't305 // changed since 2007-05. Technically this applies to i386/x86_64 as well but306 // we call _dl_get_tls_static_info and need the precise size of struct307 // pthread.308 return FIRST_32_SECOND_64(524, 1552);309# endif310 311# if defined(__mips__)312 // TODO(sagarthakur): add more values as per different glibc versions.313 return FIRST_32_SECOND_64(1152, 1776);314# endif315 316# if SANITIZER_LOONGARCH64317 return 1856; // from glibc 2.36318# endif319 320# if defined(__aarch64__)321 // The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22.322 return 1776;323# endif324 325# if defined(__powerpc64__)326 return 1776; // from glibc.ppc64le 2.20-8.fc21327# endif328 329 return 0;330}331# endif // SANITIZER_FREEBSD && !SANITIZER_GO332 333# if (SANITIZER_FREEBSD || SANITIZER_GLIBC) && !SANITIZER_GO334// On glibc x86_64, ThreadDescriptorSize() needs to be precise due to the usage335// of g_tls_size. On other targets, ThreadDescriptorSize() is only used by lsan336// to get the pointer to thread-specific data keys in the thread control block.337// sizeof(struct pthread) from glibc.338static uptr thread_descriptor_size;339 340uptr ThreadDescriptorSize() { return thread_descriptor_size; }341 342# if SANITIZER_GLIBC343__attribute__((unused)) static size_t g_tls_size;344# endif345 346void InitTlsSize() {347# if SANITIZER_GLIBC348 int major, minor, patch;349 GetGLibcVersion(&major, &minor, &patch);350 g_use_dlpi_tls_data = major == 2 && minor >= 25;351 352 if (major == 2 && minor >= 34) {353 // _thread_db_sizeof_pthread is a GLIBC_PRIVATE symbol that is exported in354 // glibc 2.34 and later.355 if (unsigned *psizeof = static_cast<unsigned *>(356 dlsym(RTLD_DEFAULT, "_thread_db_sizeof_pthread"))) {357 thread_descriptor_size = *psizeof;358 }359 }360 361# if defined(__aarch64__) || defined(__x86_64__) || \362 defined(__powerpc64__) || defined(__loongarch__)363 auto *get_tls_static_info = (void (*)(size_t *, size_t *))dlsym(364 RTLD_DEFAULT, "_dl_get_tls_static_info");365 size_t tls_align;366 // Can be null if static link.367 if (get_tls_static_info)368 get_tls_static_info(&g_tls_size, &tls_align);369# endif370 371# endif // SANITIZER_GLIBC372 373 if (!thread_descriptor_size)374 thread_descriptor_size = ThreadDescriptorSizeFallback();375}376 377# if defined(__mips__) || defined(__powerpc64__) || SANITIZER_RISCV64 || \378 SANITIZER_LOONGARCH64379// TlsPreTcbSize includes size of struct pthread_descr and size of tcb380// head structure. It lies before the static tls blocks.381static uptr TlsPreTcbSize() {382# if defined(__mips__)383 const uptr kTcbHead = 16; // sizeof (tcbhead_t)384# elif defined(__powerpc64__)385 const uptr kTcbHead = 88; // sizeof (tcbhead_t)386# elif SANITIZER_RISCV64387 const uptr kTcbHead = 16; // sizeof (tcbhead_t)388# elif SANITIZER_LOONGARCH64389 const uptr kTcbHead = 16; // sizeof (tcbhead_t)390# endif391 const uptr kTlsAlign = 16;392 const uptr kTlsPreTcbSize =393 RoundUpTo(ThreadDescriptorSize() + kTcbHead, kTlsAlign);394 return kTlsPreTcbSize;395}396# endif397# else // (SANITIZER_FREEBSD || SANITIZER_GLIBC) && !SANITIZER_GO398void InitTlsSize() {}399uptr ThreadDescriptorSize() { return 0; }400# endif // (SANITIZER_FREEBSD || SANITIZER_GLIBC) && !SANITIZER_GO401 402# if (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_SOLARIS) && \403 !SANITIZER_ANDROID && !SANITIZER_GO404namespace {405struct TlsBlock {406 uptr begin, end, align;407 size_t tls_modid;408 bool operator<(const TlsBlock &rhs) const { return begin < rhs.begin; }409};410} // namespace411 412# ifdef __s390__413extern "C" uptr __tls_get_offset(void *arg);414 415static uptr TlsGetOffset(uptr ti_module, uptr ti_offset) {416 // The __tls_get_offset ABI requires %r12 to point to GOT and %r2 to be an417 // offset of a struct tls_index inside GOT. We don't possess either of the418 // two, so violate the letter of the "ELF Handling For Thread-Local419 // Storage" document and assume that the implementation just dereferences420 // %r2 + %r12.421 uptr tls_index[2] = {ti_module, ti_offset};422 register uptr r2 asm("2") = 0;423 register void *r12 asm("12") = tls_index;424 asm("basr %%r14, %[__tls_get_offset]"425 : "+r"(r2)426 : [__tls_get_offset] "r"(__tls_get_offset), "r"(r12)427 : "memory", "cc", "0", "1", "3", "4", "5", "14");428 return r2;429}430# else431extern "C" void *__tls_get_addr(size_t *);432# endif433 434static size_t main_tls_modid;435 436static int CollectStaticTlsBlocks(struct dl_phdr_info *info, size_t size,437 void *data) {438 size_t tls_modid;439# if SANITIZER_SOLARIS440 // dlpi_tls_modid is only available since Solaris 11.4 SRU 10. Use441 // dlinfo(RTLD_DI_LINKMAP) instead which works on all of Solaris 11.3,442 // 11.4, and Illumos. The tlsmodid of the executable was changed to 1 in443 // 11.4 to match other implementations.444 if (size >= offsetof(dl_phdr_info_test, dlpi_tls_modid))445 main_tls_modid = 1;446 else447 main_tls_modid = 0;448 g_use_dlpi_tls_data = 0;449 Rt_map *map;450 dlinfo(RTLD_SELF, RTLD_DI_LINKMAP, &map);451 tls_modid = map->rt_tlsmodid;452# else453 main_tls_modid = 1;454 tls_modid = info->dlpi_tls_modid;455# endif456 457 if (tls_modid < main_tls_modid)458 return 0;459 uptr begin;460# if !SANITIZER_SOLARIS461 begin = (uptr)info->dlpi_tls_data;462# endif463 if (!g_use_dlpi_tls_data) {464 // Call __tls_get_addr as a fallback. This forces TLS allocation on glibc465 // and FreeBSD.466# ifdef __s390__467 begin = (uptr)__builtin_thread_pointer() + TlsGetOffset(tls_modid, 0);468# else469 size_t mod_and_off[2] = {tls_modid, 0};470 begin = (uptr)__tls_get_addr(mod_and_off);471# endif472 }473 for (unsigned i = 0; i != info->dlpi_phnum; ++i)474 if (info->dlpi_phdr[i].p_type == PT_TLS) {475 static_cast<InternalMmapVector<TlsBlock> *>(data)->push_back(476 TlsBlock{begin, begin + info->dlpi_phdr[i].p_memsz,477 info->dlpi_phdr[i].p_align, tls_modid});478 break;479 }480 return 0;481}482 483__attribute__((unused)) static void GetStaticTlsBoundary(uptr *addr, uptr *size,484 uptr *align) {485 InternalMmapVector<TlsBlock> ranges;486 dl_iterate_phdr(CollectStaticTlsBlocks, &ranges);487 uptr len = ranges.size();488 Sort(ranges.begin(), len);489 // Find the range with tls_modid == main_tls_modid. For glibc, because490 // libc.so uses PT_TLS, this module is guaranteed to exist and is one of491 // the initially loaded modules.492 uptr one = 0;493 while (one != len && ranges[one].tls_modid != main_tls_modid) ++one;494 if (one == len) {495 // This may happen with musl if no module uses PT_TLS.496 *addr = 0;497 *size = 0;498 *align = 1;499 return;500 }501 // Find the maximum consecutive ranges. We consider two modules consecutive if502 // the gap is smaller than the alignment of the latter range. The dynamic503 // loader places static TLS blocks this way not to waste space.504 uptr l = one;505 *align = ranges[l].align;506 while (l != 0 && ranges[l].begin < ranges[l - 1].end + ranges[l].align)507 *align = Max(*align, ranges[--l].align);508 uptr r = one + 1;509 while (r != len && ranges[r].begin < ranges[r - 1].end + ranges[r].align)510 *align = Max(*align, ranges[r++].align);511 *addr = ranges[l].begin;512 *size = ranges[r - 1].end - ranges[l].begin;513}514# endif // (x86_64 || i386 || mips || ...) && (SANITIZER_FREEBSD ||515 // SANITIZER_LINUX) && !SANITIZER_ANDROID && !SANITIZER_GO516 517# if SANITIZER_NETBSD518static struct tls_tcb *ThreadSelfTlsTcb() {519 struct tls_tcb *tcb = nullptr;520# ifdef __HAVE___LWP_GETTCB_FAST521 tcb = (struct tls_tcb *)__lwp_gettcb_fast();522# elif defined(__HAVE___LWP_GETPRIVATE_FAST)523 tcb = (struct tls_tcb *)__lwp_getprivate_fast();524# endif525 return tcb;526}527 528uptr ThreadSelf() { return (uptr)ThreadSelfTlsTcb()->tcb_pthread; }529 530int GetSizeFromHdr(struct dl_phdr_info *info, size_t size, void *data) {531 const Elf_Phdr *hdr = info->dlpi_phdr;532 const Elf_Phdr *last_hdr = hdr + info->dlpi_phnum;533 534 for (; hdr != last_hdr; ++hdr) {535 if (hdr->p_type == PT_TLS && info->dlpi_tls_modid == 1) {536 *(uptr *)data = hdr->p_memsz;537 break;538 }539 }540 return 0;541}542# endif // SANITIZER_NETBSD543 544# if SANITIZER_ANDROID545// Bionic provides this API since S.546extern "C" SANITIZER_WEAK_ATTRIBUTE void __libc_get_static_tls_bounds(void **,547 void **);548# endif549 550# if !SANITIZER_GO551static void GetTls(uptr *addr, uptr *size) {552# if SANITIZER_ANDROID553 if (&__libc_get_static_tls_bounds) {554 void *start_addr;555 void *end_addr;556 __libc_get_static_tls_bounds(&start_addr, &end_addr);557 *addr = reinterpret_cast<uptr>(start_addr);558 *size =559 reinterpret_cast<uptr>(end_addr) - reinterpret_cast<uptr>(start_addr);560 } else {561 *addr = 0;562 *size = 0;563 }564# elif SANITIZER_GLIBC && defined(__x86_64__)565 // For aarch64 and x86-64, use an O(1) approach which requires relatively566 // precise ThreadDescriptorSize. g_tls_size was initialized in InitTlsSize.567# if SANITIZER_X32568 asm("mov %%fs:8,%0" : "=r"(*addr));569# else570 asm("mov %%fs:16,%0" : "=r"(*addr));571# endif572 *size = g_tls_size;573 *addr -= *size;574 *addr += ThreadDescriptorSize();575# elif SANITIZER_GLIBC && defined(__aarch64__)576 *addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -577 ThreadDescriptorSize();578 *size = g_tls_size + ThreadDescriptorSize();579# elif SANITIZER_GLIBC && defined(__loongarch__)580# ifdef __clang__581 *addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -582 ThreadDescriptorSize();583# else584 asm("or %0,$tp,$zero" : "=r"(*addr));585 *addr -= ThreadDescriptorSize();586# endif587 *size = g_tls_size + ThreadDescriptorSize();588# elif SANITIZER_GLIBC && defined(__powerpc64__)589 // Workaround for glibc<2.25(?). 2.27 is known to not need this.590 uptr tp;591 asm("addi %0,13,-0x7000" : "=r"(tp));592 const uptr pre_tcb_size = TlsPreTcbSize();593 *addr = tp - pre_tcb_size;594 *size = g_tls_size + pre_tcb_size;595# elif SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_SOLARIS596 uptr align;597 GetStaticTlsBoundary(addr, size, &align);598# if defined(__x86_64__) || defined(__i386__) || defined(__s390__) || \599 defined(__sparc__)600 if (SANITIZER_GLIBC) {601# if defined(__x86_64__) || defined(__i386__)602 align = Max<uptr>(align, 64);603# else604 align = Max<uptr>(align, 16);605# endif606 }607 const uptr tp = RoundUpTo(*addr + *size, align);608 609 // lsan requires the range to additionally cover the static TLS surplus610 // (elf/dl-tls.c defines 1664). Otherwise there may be false positives for611 // allocations only referenced by tls in dynamically loaded modules.612 if (SANITIZER_GLIBC)613 *size += 1644;614 else if (SANITIZER_FREEBSD)615 *size += 128; // RTLD_STATIC_TLS_EXTRA616 617 // Extend the range to include the thread control block. On glibc, lsan needs618 // the range to include pthread::{specific_1stblock,specific} so that619 // allocations only referenced by pthread_setspecific can be scanned. This may620 // underestimate by at most TLS_TCB_ALIGN-1 bytes but it should be fine621 // because the number of bytes after pthread::specific is larger.622 *addr = tp - RoundUpTo(*size, align);623 *size = tp - *addr + ThreadDescriptorSize();624# else625# if SANITIZER_GLIBC626 *size += 1664;627# elif SANITIZER_FREEBSD628 *size += 128; // RTLD_STATIC_TLS_EXTRA629# if defined(__mips__) || defined(__powerpc64__) || SANITIZER_RISCV64630 const uptr pre_tcb_size = TlsPreTcbSize();631 *addr -= pre_tcb_size;632 *size += pre_tcb_size;633# else634 // arm and aarch64 reserve two words at TP, so this underestimates the range.635 // However, this is sufficient for the purpose of finding the pointers to636 // thread-specific data keys.637 const uptr tcb_size = ThreadDescriptorSize();638 *addr -= tcb_size;639 *size += tcb_size;640# endif641# endif642# endif643# elif SANITIZER_NETBSD644 struct tls_tcb *const tcb = ThreadSelfTlsTcb();645 *addr = 0;646 *size = 0;647 if (tcb != 0) {648 // Find size (p_memsz) of dlpi_tls_modid 1 (TLS block of the main program).649 // ld.elf_so hardcodes the index 1.650 dl_iterate_phdr(GetSizeFromHdr, size);651 652 if (*size != 0) {653 // The block has been found and tcb_dtv[1] contains the base address654 *addr = (uptr)tcb->tcb_dtv[1];655 }656 }657# elif SANITIZER_HAIKU658# else659# error "Unknown OS"660# endif661}662# endif663 664# if !SANITIZER_GO665uptr GetTlsSize() {666# if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \667 SANITIZER_SOLARIS668 uptr addr, size;669 GetTls(&addr, &size);670 return size;671# else672 return 0;673# endif674}675# endif676 677void GetThreadStackAndTls(bool main, uptr *stk_begin, uptr *stk_end,678 uptr *tls_begin, uptr *tls_end) {679# if SANITIZER_GO680 // Stub implementation for Go.681 *stk_begin = 0;682 *stk_end = 0;683 *tls_begin = 0;684 *tls_end = 0;685# else686 uptr tls_addr = 0;687 uptr tls_size = 0;688 GetTls(&tls_addr, &tls_size);689 *tls_begin = tls_addr;690 *tls_end = tls_addr + tls_size;691 692 uptr stack_top, stack_bottom;693 GetThreadStackTopAndBottom(main, &stack_top, &stack_bottom);694 *stk_begin = stack_bottom;695 *stk_end = stack_top;696 697 if (!main) {698 // If stack and tls intersect, make them non-intersecting.699 if (*tls_begin > *stk_begin && *tls_begin < *stk_end) {700 if (*stk_end < *tls_end)701 *tls_end = *stk_end;702 *stk_end = *tls_begin;703 }704 }705# endif706}707 708# if !SANITIZER_FREEBSD709typedef ElfW(Phdr) Elf_Phdr;710# endif711 712struct DlIteratePhdrData {713 InternalMmapVectorNoCtor<LoadedModule> *modules;714 bool first;715};716 717static int AddModuleSegments(const char *module_name, dl_phdr_info *info,718 InternalMmapVectorNoCtor<LoadedModule> *modules) {719 if (module_name[0] == '\0')720 return 0;721 LoadedModule cur_module;722 cur_module.set(module_name, info->dlpi_addr);723 for (int i = 0; i < (int)info->dlpi_phnum; i++) {724 const Elf_Phdr *phdr = &info->dlpi_phdr[i];725 if (phdr->p_type == PT_LOAD) {726 uptr cur_beg = info->dlpi_addr + phdr->p_vaddr;727 uptr cur_end = cur_beg + phdr->p_memsz;728# if SANITIZER_HAIKU729 bool executable = phdr->p_flags & PF_EXECUTE;730 bool writable = phdr->p_flags & PF_WRITE;731# else732 bool executable = phdr->p_flags & PF_X;733 bool writable = phdr->p_flags & PF_W;734# endif735 cur_module.addAddressRange(cur_beg, cur_end, executable, writable);736 } else if (phdr->p_type == PT_NOTE) {737# ifdef NT_GNU_BUILD_ID738 uptr off = 0;739 while (off + sizeof(ElfW(Nhdr)) < phdr->p_memsz) {740 auto *nhdr = reinterpret_cast<const ElfW(Nhdr) *>(info->dlpi_addr +741 phdr->p_vaddr + off);742 constexpr auto kGnuNamesz = 4; // "GNU" with NUL-byte.743 static_assert(kGnuNamesz % 4 == 0, "kGnuNameSize is aligned to 4.");744 if (nhdr->n_type == NT_GNU_BUILD_ID && nhdr->n_namesz == kGnuNamesz) {745 if (off + sizeof(ElfW(Nhdr)) + nhdr->n_namesz + nhdr->n_descsz >746 phdr->p_memsz) {747 // Something is very wrong, bail out instead of reading potentially748 // arbitrary memory.749 break;750 }751 const char *name =752 reinterpret_cast<const char *>(nhdr) + sizeof(*nhdr);753 if (internal_memcmp(name, "GNU", 3) == 0) {754 const char *value = reinterpret_cast<const char *>(nhdr) +755 sizeof(*nhdr) + kGnuNamesz;756 cur_module.setUuid(value, nhdr->n_descsz);757 break;758 }759 }760 off += sizeof(*nhdr) + RoundUpTo(nhdr->n_namesz, 4) +761 RoundUpTo(nhdr->n_descsz, 4);762 }763# endif764 }765 }766 modules->push_back(cur_module);767 return 0;768}769 770static int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *arg) {771 DlIteratePhdrData *data = (DlIteratePhdrData *)arg;772 if (data->first) {773 InternalMmapVector<char> module_name(kMaxPathLength);774 data->first = false;775 // First module is the binary itself.776 ReadBinaryNameCached(module_name.data(), module_name.size());777 return AddModuleSegments(module_name.data(), info, data->modules);778 }779 780 if (info->dlpi_name)781 return AddModuleSegments(info->dlpi_name, info, data->modules);782 783 return 0;784}785 786void ListOfModules::init() {787 clearOrInit();788 DlIteratePhdrData data = {&modules_, true};789 dl_iterate_phdr(dl_iterate_phdr_cb, &data);790}791 792void ListOfModules::fallbackInit() { clear(); }793 794// getrusage does not give us the current RSS, only the max RSS.795// Still, this is better than nothing if /proc/self/statm is not available796// for some reason, e.g. due to a sandbox.797static uptr GetRSSFromGetrusage() {798 struct rusage usage;799 if (getrusage(RUSAGE_SELF, &usage)) // Failed, probably due to a sandbox.800 return 0;801 return usage.ru_maxrss << 10; // ru_maxrss is in Kb.802}803 804uptr GetRSS() {805 if (!common_flags()->can_use_proc_maps_statm)806 return GetRSSFromGetrusage();807 fd_t fd = OpenFile("/proc/self/statm", RdOnly);808 if (fd == kInvalidFd)809 return GetRSSFromGetrusage();810 char buf[64];811 uptr len = internal_read(fd, buf, sizeof(buf) - 1);812 internal_close(fd);813 if ((sptr)len <= 0)814 return 0;815 buf[len] = 0;816 // The format of the file is:817 // 1084 89 69 11 0 79 0818 // We need the second number which is RSS in pages.819 char *pos = buf;820 // Skip the first number.821 while (*pos >= '0' && *pos <= '9') pos++;822 // Skip whitespaces.823 while (!(*pos >= '0' && *pos <= '9') && *pos != 0) pos++;824 // Read the number.825 uptr rss = 0;826 while (*pos >= '0' && *pos <= '9') rss = rss * 10 + *pos++ - '0';827 return rss * GetPageSizeCached();828}829 830// sysconf(_SC_NPROCESSORS_{CONF,ONLN}) cannot be used on most platforms as831// they allocate memory.832u32 GetNumberOfCPUs() {833# if SANITIZER_FREEBSD || SANITIZER_NETBSD834 u32 ncpu;835 int req[2];836 uptr len = sizeof(ncpu);837 req[0] = CTL_HW;838# ifdef HW_NCPUONLINE839 req[1] = HW_NCPUONLINE;840# else841 req[1] = HW_NCPU;842# endif843 CHECK_EQ(internal_sysctl(req, 2, &ncpu, &len, NULL, 0), 0);844 return ncpu;845# elif SANITIZER_HAIKU846 system_info info;847 get_system_info(&info);848 return info.cpu_count;849# elif SANITIZER_SOLARIS850 return sysconf(_SC_NPROCESSORS_ONLN);851# else852 cpu_set_t CPUs;853 CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0);854 return CPU_COUNT(&CPUs);855# endif856}857 858# if SANITIZER_LINUX859 860# if SANITIZER_ANDROID861static atomic_uint8_t android_log_initialized;862 863void AndroidLogInit() {864 openlog(GetProcessName(), 0, LOG_USER);865 atomic_store(&android_log_initialized, 1, memory_order_release);866}867 868static bool ShouldLogAfterPrintf() {869 return atomic_load(&android_log_initialized, memory_order_acquire);870}871 872extern "C" SANITIZER_WEAK_ATTRIBUTE int async_safe_write_log(int pri,873 const char *tag,874 const char *msg);875extern "C" SANITIZER_WEAK_ATTRIBUTE int __android_log_write(int prio,876 const char *tag,877 const char *msg);878 879// ANDROID_LOG_INFO is 4, but can't be resolved at runtime.880# define SANITIZER_ANDROID_LOG_INFO 4881 882// async_safe_write_log is a new public version of __libc_write_log that is883// used behind syslog. It is preferable to syslog as it will not do any dynamic884// memory allocation or formatting.885// If the function is not available, syslog is preferred for L+ (it was broken886// pre-L) as __android_log_write triggers a racey behavior with the strncpy887// interceptor. Fallback to __android_log_write pre-L.888void WriteOneLineToSyslog(const char *s) {889 if (&async_safe_write_log) {890 async_safe_write_log(SANITIZER_ANDROID_LOG_INFO, GetProcessName(), s);891 } else {892 syslog(LOG_INFO, "%s", s);893 }894}895 896extern "C" SANITIZER_WEAK_ATTRIBUTE void android_set_abort_message(897 const char *);898 899void SetAbortMessage(const char *str) {900 if (&android_set_abort_message)901 android_set_abort_message(str);902}903# else904void AndroidLogInit() {}905 906static bool ShouldLogAfterPrintf() { return true; }907 908void WriteOneLineToSyslog(const char *s) { syslog(LOG_INFO, "%s", s); }909 910void SetAbortMessage(const char *str) {}911# endif // SANITIZER_ANDROID912 913void LogMessageOnPrintf(const char *str) {914 if (common_flags()->log_to_syslog && ShouldLogAfterPrintf())915 WriteToSyslog(str);916}917 918# endif // SANITIZER_LINUX919 920# if SANITIZER_GLIBC && !SANITIZER_GO921// glibc crashes when using clock_gettime from a preinit_array function as the922// vDSO function pointers haven't been initialized yet. __progname is923// initialized after the vDSO function pointers, so if it exists, is not null924// and is not empty, we can use clock_gettime.925extern "C" SANITIZER_WEAK_ATTRIBUTE char *__progname;926inline bool CanUseVDSO() { return &__progname && __progname && *__progname; }927 928// MonotonicNanoTime is a timing function that can leverage the vDSO by calling929// clock_gettime. real_clock_gettime only exists if clock_gettime is930// intercepted, so define it weakly and use it if available.931extern "C" SANITIZER_WEAK_ATTRIBUTE int real_clock_gettime(u32 clk_id,932 void *tp);933u64 MonotonicNanoTime() {934 timespec ts;935 if (CanUseVDSO()) {936 if (&real_clock_gettime)937 real_clock_gettime(CLOCK_MONOTONIC, &ts);938 else939 clock_gettime(CLOCK_MONOTONIC, &ts);940 } else {941 internal_clock_gettime(CLOCK_MONOTONIC, &ts);942 }943 return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;944}945# else946// Non-glibc & Go always use the regular function.947u64 MonotonicNanoTime() {948 timespec ts;949 clock_gettime(CLOCK_MONOTONIC, &ts);950 return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;951}952# endif // SANITIZER_GLIBC && !SANITIZER_GO953 954void ReExec() {955 const char *pathname = "/proc/self/exe";956 957# if SANITIZER_FREEBSD958 for (const auto *aux = __elf_aux_vector; aux->a_type != AT_NULL; aux++) {959 if (aux->a_type == AT_EXECPATH) {960 pathname = static_cast<const char *>(aux->a_un.a_ptr);961 break;962 }963 }964# elif SANITIZER_NETBSD965 static const int name[] = {966 CTL_KERN,967 KERN_PROC_ARGS,968 -1,969 KERN_PROC_PATHNAME,970 };971 char path[400];972 uptr len;973 974 len = sizeof(path);975 if (internal_sysctl(name, ARRAY_SIZE(name), path, &len, NULL, 0) != -1)976 pathname = path;977# elif SANITIZER_SOLARIS978 pathname = getexecname();979 CHECK_NE(pathname, NULL);980# elif SANITIZER_USE_GETAUXVAL981 // Calling execve with /proc/self/exe sets that as $EXEC_ORIGIN. Binaries that982 // rely on that will fail to load shared libraries. Query AT_EXECFN instead.983 pathname = reinterpret_cast<const char *>(getauxval(AT_EXECFN));984# endif985 986 uptr rv = internal_execve(pathname, GetArgv(), GetEnviron());987 int rverrno;988 CHECK_EQ(internal_iserror(rv, &rverrno), true);989 Printf("execve failed, errno %d\n", rverrno);990 Die();991}992 993void UnmapFromTo(uptr from, uptr to) {994 if (to == from)995 return;996 CHECK(to >= from);997 uptr res = internal_munmap(reinterpret_cast<void *>(from), to - from);998 if (UNLIKELY(internal_iserror(res))) {999 Report("ERROR: %s failed to unmap 0x%zx (%zd) bytes at address %p\n",1000 SanitizerToolName, to - from, to - from, (void *)from);1001 CHECK("unable to unmap" && 0);1002 }1003}1004 1005uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,1006 uptr min_shadow_base_alignment, UNUSED uptr &high_mem_end,1007 uptr granularity) {1008 const uptr alignment =1009 Max<uptr>(granularity << shadow_scale, 1ULL << min_shadow_base_alignment);1010 const uptr left_padding =1011 Max<uptr>(granularity, 1ULL << min_shadow_base_alignment);1012 1013 const uptr shadow_size = RoundUpTo(shadow_size_bytes, granularity);1014 const uptr map_size = shadow_size + left_padding + alignment;1015 1016 const uptr map_start = (uptr)MmapNoAccess(map_size);1017 CHECK_NE(map_start, ~(uptr)0);1018 1019 const uptr shadow_start = RoundUpTo(map_start + left_padding, alignment);1020 1021 UnmapFromTo(map_start, shadow_start - left_padding);1022 UnmapFromTo(shadow_start + shadow_size, map_start + map_size);1023 1024 return shadow_start;1025}1026 1027static uptr MmapSharedNoReserve(uptr addr, uptr size) {1028 return internal_mmap(1029 reinterpret_cast<void *>(addr), size, PROT_READ | PROT_WRITE,1030 MAP_FIXED | MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);1031}1032 1033static uptr MremapCreateAlias(uptr base_addr, uptr alias_addr,1034 uptr alias_size) {1035# if SANITIZER_LINUX1036 return internal_mremap(reinterpret_cast<void *>(base_addr), 0, alias_size,1037 MREMAP_MAYMOVE | MREMAP_FIXED,1038 reinterpret_cast<void *>(alias_addr));1039# else1040 CHECK(false && "mremap is not supported outside of Linux");1041 return 0;1042# endif1043}1044 1045static void CreateAliases(uptr start_addr, uptr alias_size, uptr num_aliases) {1046 uptr total_size = alias_size * num_aliases;1047 uptr mapped = MmapSharedNoReserve(start_addr, total_size);1048 CHECK_EQ(mapped, start_addr);1049 1050 for (uptr i = 1; i < num_aliases; ++i) {1051 uptr alias_addr = start_addr + i * alias_size;1052 CHECK_EQ(MremapCreateAlias(start_addr, alias_addr, alias_size), alias_addr);1053 }1054}1055 1056uptr MapDynamicShadowAndAliases(uptr shadow_size, uptr alias_size,1057 uptr num_aliases, uptr ring_buffer_size) {1058 CHECK_EQ(alias_size & (alias_size - 1), 0);1059 CHECK_EQ(num_aliases & (num_aliases - 1), 0);1060 CHECK_EQ(ring_buffer_size & (ring_buffer_size - 1), 0);1061 1062 const uptr granularity = GetMmapGranularity();1063 shadow_size = RoundUpTo(shadow_size, granularity);1064 CHECK_EQ(shadow_size & (shadow_size - 1), 0);1065 1066 const uptr alias_region_size = alias_size * num_aliases;1067 const uptr alignment =1068 2 * Max(Max(shadow_size, alias_region_size), ring_buffer_size);1069 const uptr left_padding = ring_buffer_size;1070 1071 const uptr right_size = alignment;1072 const uptr map_size = left_padding + 2 * alignment;1073 1074 const uptr map_start = reinterpret_cast<uptr>(MmapNoAccess(map_size));1075 CHECK_NE(map_start, static_cast<uptr>(-1));1076 const uptr right_start = RoundUpTo(map_start + left_padding, alignment);1077 1078 UnmapFromTo(map_start, right_start - left_padding);1079 UnmapFromTo(right_start + right_size, map_start + map_size);1080 1081 CreateAliases(right_start + right_size / 2, alias_size, num_aliases);1082 1083 return right_start;1084}1085 1086void InitializePlatformCommonFlags(CommonFlags *cf) {1087# if SANITIZER_ANDROID1088 if (&__libc_get_static_tls_bounds == nullptr)1089 cf->detect_leaks = false;1090# endif1091}1092 1093} // namespace __sanitizer1094 1095#endif1096