725 lines · c
1//===------------------------ malloc.c ------------------*- C -*-=============//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This library provides a cross-platform lock free thread caching malloc
10// implementation in C11.
11//
12//
13// This file provides overrides for the standard library malloc entry points for
14// C and new/delete operators for C++ It also provides automatic
15// initialization/finalization of process and threads
16//
17//===----------------------------------------------------------------------===//
18
19#if defined(__TINYC__)
20#include <sys/types.h>
21#endif
22
23#ifndef ARCH_64BIT
24#if defined(__LLP64__) || defined(__LP64__) || defined(_WIN64)
25#define ARCH_64BIT 1
26_Static_assert(sizeof(size_t) == 8, "Data type size mismatch");
27_Static_assert(sizeof(void *) == 8, "Data type size mismatch");
28#else
29#define ARCH_64BIT 0
30_Static_assert(sizeof(size_t) == 4, "Data type size mismatch");
31_Static_assert(sizeof(void *) == 4, "Data type size mismatch");
32#endif
33#endif
34
35#if (defined(__GNUC__) || defined(__clang__))
36#pragma GCC visibility push(default)
37#endif
38
39#define USE_IMPLEMENT 1
40#define USE_INTERPOSE 0
41#define USE_ALIAS 0
42
43#if defined(__APPLE__)
44#undef USE_INTERPOSE
45#define USE_INTERPOSE 1
46
47typedef struct interpose_t {
48 void *new_func;
49 void *orig_func;
50} interpose_t;
51
52#define MAC_INTERPOSE_PAIR(newf, oldf) {(void *)newf, (void *)oldf}
53#define MAC_INTERPOSE_SINGLE(newf, oldf) \
54 __attribute__((used)) static const interpose_t macinterpose##newf##oldf \
55 __attribute__((section("__DATA, __interpose"))) = \
56 MAC_INTERPOSE_PAIR(newf, oldf)
57
58#endif
59
60#if !defined(_WIN32) && !defined(__APPLE__)
61#undef USE_IMPLEMENT
62#undef USE_ALIAS
63#define USE_IMPLEMENT 0
64#define USE_ALIAS 1
65#endif
66
67#ifdef _MSC_VER
68#pragma warning(disable : 4100)
69#undef malloc
70#undef free
71#undef calloc
72#define RPMALLOC_RESTRICT __declspec(restrict)
73#else
74#define RPMALLOC_RESTRICT
75#endif
76
77#if ENABLE_OVERRIDE
78
79typedef struct rp_nothrow_t {
80 int __dummy;
81} rp_nothrow_t;
82
83#if USE_IMPLEMENT
84
85extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL malloc(size_t size) {
86 return rpmalloc(size);
87}
88extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL calloc(size_t count,
89 size_t size) {
90 return rpcalloc(count, size);
91}
92extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL realloc(void *ptr,
93 size_t size) {
94 return rprealloc(ptr, size);
95}
96extern inline void *RPMALLOC_CDECL reallocf(void *ptr, size_t size) {
97 return rprealloc(ptr, size);
98}
99extern inline void *RPMALLOC_CDECL aligned_alloc(size_t alignment,
100 size_t size) {
101 return rpaligned_alloc(alignment, size);
102}
103extern inline void *RPMALLOC_CDECL memalign(size_t alignment, size_t size) {
104 return rpmemalign(alignment, size);
105}
106extern inline int RPMALLOC_CDECL posix_memalign(void **memptr, size_t alignment,
107 size_t size) {
108 return rpposix_memalign(memptr, alignment, size);
109}
110extern inline void RPMALLOC_CDECL free(void *ptr) { rpfree(ptr); }
111extern inline void RPMALLOC_CDECL cfree(void *ptr) { rpfree(ptr); }
112extern inline size_t RPMALLOC_CDECL malloc_usable_size(void *ptr) {
113 return rpmalloc_usable_size(ptr);
114}
115extern inline size_t RPMALLOC_CDECL malloc_size(void *ptr) {
116 return rpmalloc_usable_size(ptr);
117}
118
119#ifdef _WIN32
120extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL _malloc_base(size_t size) {
121 return rpmalloc(size);
122}
123extern inline void RPMALLOC_CDECL _free_base(void *ptr) { rpfree(ptr); }
124extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL _calloc_base(size_t count,
125 size_t size) {
126 return rpcalloc(count, size);
127}
128extern inline size_t RPMALLOC_CDECL _msize(void *ptr) {
129 return rpmalloc_usable_size(ptr);
130}
131extern inline size_t RPMALLOC_CDECL _msize_base(void *ptr) {
132 return rpmalloc_usable_size(ptr);
133}
134extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL
135_realloc_base(void *ptr, size_t size) {
136 return rprealloc(ptr, size);
137}
138#endif
139
140#ifdef _WIN32
141// For Windows, #include <rpnew.h> in one source file to get the C++ operator
142// overrides implemented in your module
143#else
144// Overload the C++ operators using the mangled names
145// (https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling) operators
146// delete and delete[]
147#define RPDEFVIS __attribute__((visibility("default")))
148extern void _ZdlPv(void *p);
149void RPDEFVIS _ZdlPv(void *p) { rpfree(p); }
150extern void _ZdaPv(void *p);
151void RPDEFVIS _ZdaPv(void *p) { rpfree(p); }
152#if ARCH_64BIT
153// 64-bit operators new and new[], normal and aligned
154extern void *_Znwm(uint64_t size);
155void *RPDEFVIS _Znwm(uint64_t size) { return rpmalloc(size); }
156extern void *_Znam(uint64_t size);
157void *RPDEFVIS _Znam(uint64_t size) { return rpmalloc(size); }
158extern void *_Znwmm(uint64_t size, uint64_t align);
159void *RPDEFVIS _Znwmm(uint64_t size, uint64_t align) {
160 return rpaligned_alloc(align, size);
161}
162extern void *_Znamm(uint64_t size, uint64_t align);
163void *RPDEFVIS _Znamm(uint64_t size, uint64_t align) {
164 return rpaligned_alloc(align, size);
165}
166extern void *_ZnwmSt11align_val_t(uint64_t size, uint64_t align);
167void *RPDEFVIS _ZnwmSt11align_val_t(uint64_t size, uint64_t align) {
168 return rpaligned_alloc(align, size);
169}
170extern void *_ZnamSt11align_val_t(uint64_t size, uint64_t align);
171void *RPDEFVIS _ZnamSt11align_val_t(uint64_t size, uint64_t align) {
172 return rpaligned_alloc(align, size);
173}
174extern void *_ZnwmRKSt9nothrow_t(uint64_t size, rp_nothrow_t t);
175void *RPDEFVIS _ZnwmRKSt9nothrow_t(uint64_t size, rp_nothrow_t t) {
176 (void)sizeof(t);
177 return rpmalloc(size);
178}
179extern void *_ZnamRKSt9nothrow_t(uint64_t size, rp_nothrow_t t);
180void *RPDEFVIS _ZnamRKSt9nothrow_t(uint64_t size, rp_nothrow_t t) {
181 (void)sizeof(t);
182 return rpmalloc(size);
183}
184extern void *_ZnwmSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
185 rp_nothrow_t t);
186void *RPDEFVIS _ZnwmSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
187 rp_nothrow_t t) {
188 (void)sizeof(t);
189 return rpaligned_alloc(align, size);
190}
191extern void *_ZnamSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
192 rp_nothrow_t t);
193void *RPDEFVIS _ZnamSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
194 rp_nothrow_t t) {
195 (void)sizeof(t);
196 return rpaligned_alloc(align, size);
197}
198// 64-bit operators sized delete and delete[], normal and aligned
199extern void _ZdlPvm(void *p, uint64_t size);
200void RPDEFVIS _ZdlPvm(void *p, uint64_t size) {
201 rpfree(p);
202 (void)sizeof(size);
203}
204extern void _ZdaPvm(void *p, uint64_t size);
205void RPDEFVIS _ZdaPvm(void *p, uint64_t size) {
206 rpfree(p);
207 (void)sizeof(size);
208}
209extern void _ZdlPvSt11align_val_t(void *p, uint64_t align);
210void RPDEFVIS _ZdlPvSt11align_val_t(void *p, uint64_t align) {
211 rpfree(p);
212 (void)sizeof(align);
213}
214extern void _ZdaPvSt11align_val_t(void *p, uint64_t align);
215void RPDEFVIS _ZdaPvSt11align_val_t(void *p, uint64_t align) {
216 rpfree(p);
217 (void)sizeof(align);
218}
219extern void _ZdlPvmSt11align_val_t(void *p, uint64_t size, uint64_t align);
220void RPDEFVIS _ZdlPvmSt11align_val_t(void *p, uint64_t size, uint64_t align) {
221 rpfree(p);
222 (void)sizeof(size);
223 (void)sizeof(align);
224}
225extern void _ZdaPvmSt11align_val_t(void *p, uint64_t size, uint64_t align);
226void RPDEFVIS _ZdaPvmSt11align_val_t(void *p, uint64_t size, uint64_t align) {
227 rpfree(p);
228 (void)sizeof(size);
229 (void)sizeof(align);
230}
231#else
232// 32-bit operators new and new[], normal and aligned
233extern void *_Znwj(uint32_t size);
234void *RPDEFVIS _Znwj(uint32_t size) { return rpmalloc(size); }
235extern void *_Znaj(uint32_t size);
236void *RPDEFVIS _Znaj(uint32_t size) { return rpmalloc(size); }
237extern void *_Znwjj(uint32_t size, uint32_t align);
238void *RPDEFVIS _Znwjj(uint32_t size, uint32_t align) {
239 return rpaligned_alloc(align, size);
240}
241extern void *_Znajj(uint32_t size, uint32_t align);
242void *RPDEFVIS _Znajj(uint32_t size, uint32_t align) {
243 return rpaligned_alloc(align, size);
244}
245extern void *_ZnwjSt11align_val_t(size_t size, size_t align);
246void *RPDEFVIS _ZnwjSt11align_val_t(size_t size, size_t align) {
247 return rpaligned_alloc(align, size);
248}
249extern void *_ZnajSt11align_val_t(size_t size, size_t align);
250void *RPDEFVIS _ZnajSt11align_val_t(size_t size, size_t align) {
251 return rpaligned_alloc(align, size);
252}
253extern void *_ZnwjRKSt9nothrow_t(size_t size, rp_nothrow_t t);
254void *RPDEFVIS _ZnwjRKSt9nothrow_t(size_t size, rp_nothrow_t t) {
255 (void)sizeof(t);
256 return rpmalloc(size);
257}
258extern void *_ZnajRKSt9nothrow_t(size_t size, rp_nothrow_t t);
259void *RPDEFVIS _ZnajRKSt9nothrow_t(size_t size, rp_nothrow_t t) {
260 (void)sizeof(t);
261 return rpmalloc(size);
262}
263extern void *_ZnwjSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
264 rp_nothrow_t t);
265void *RPDEFVIS _ZnwjSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
266 rp_nothrow_t t) {
267 (void)sizeof(t);
268 return rpaligned_alloc(align, size);
269}
270extern void *_ZnajSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
271 rp_nothrow_t t);
272void *RPDEFVIS _ZnajSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
273 rp_nothrow_t t) {
274 (void)sizeof(t);
275 return rpaligned_alloc(align, size);
276}
277// 32-bit operators sized delete and delete[], normal and aligned
278extern void _ZdlPvj(void *p, uint64_t size);
279void RPDEFVIS _ZdlPvj(void *p, uint64_t size) {
280 rpfree(p);
281 (void)sizeof(size);
282}
283extern void _ZdaPvj(void *p, uint64_t size);
284void RPDEFVIS _ZdaPvj(void *p, uint64_t size) {
285 rpfree(p);
286 (void)sizeof(size);
287}
288extern void _ZdlPvSt11align_val_t(void *p, uint32_t align);
289void RPDEFVIS _ZdlPvSt11align_val_t(void *p, uint64_t a) {
290 rpfree(p);
291 (void)sizeof(align);
292}
293extern void _ZdaPvSt11align_val_t(void *p, uint32_t align);
294void RPDEFVIS _ZdaPvSt11align_val_t(void *p, uint64_t a) {
295 rpfree(p);
296 (void)sizeof(align);
297}
298extern void _ZdlPvjSt11align_val_t(void *p, uint32_t size, uint32_t align);
299void RPDEFVIS _ZdlPvjSt11align_val_t(void *p, uint64_t size, uint64_t align) {
300 rpfree(p);
301 (void)sizeof(size);
302 (void)sizeof(a);
303}
304extern void _ZdaPvjSt11align_val_t(void *p, uint32_t size, uint32_t align);
305void RPDEFVIS _ZdaPvjSt11align_val_t(void *p, uint64_t size, uint64_t align) {
306 rpfree(p);
307 (void)sizeof(size);
308 (void)sizeof(a);
309}
310#endif
311#endif
312#endif
313
314#if USE_INTERPOSE || USE_ALIAS
315
316static void *rpmalloc_nothrow(size_t size, rp_nothrow_t t) {
317 (void)sizeof(t);
318 return rpmalloc(size);
319}
320static void *rpaligned_alloc_reverse(size_t size, size_t align) {
321 return rpaligned_alloc(align, size);
322}
323static void *rpaligned_alloc_reverse_nothrow(size_t size, size_t align,
324 rp_nothrow_t t) {
325 (void)sizeof(t);
326 return rpaligned_alloc(align, size);
327}
328static void rpfree_size(void *p, size_t size) {
329 (void)sizeof(size);
330 rpfree(p);
331}
332static void rpfree_aligned(void *p, size_t align) {
333 (void)sizeof(align);
334 rpfree(p);
335}
336static void rpfree_size_aligned(void *p, size_t size, size_t align) {
337 (void)sizeof(size);
338 (void)sizeof(align);
339 rpfree(p);
340}
341
342#endif
343
344#if USE_INTERPOSE
345
346__attribute__((used)) static const interpose_t macinterpose_malloc[]
347 __attribute__((section("__DATA, __interpose"))) = {
348 // new and new[]
349 MAC_INTERPOSE_PAIR(rpmalloc, _Znwm),
350 MAC_INTERPOSE_PAIR(rpmalloc, _Znam),
351 MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _Znwmm),
352 MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _Znamm),
353 MAC_INTERPOSE_PAIR(rpmalloc_nothrow, _ZnwmRKSt9nothrow_t),
354 MAC_INTERPOSE_PAIR(rpmalloc_nothrow, _ZnamRKSt9nothrow_t),
355 MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _ZnwmSt11align_val_t),
356 MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _ZnamSt11align_val_t),
357 MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse_nothrow,
358 _ZnwmSt11align_val_tRKSt9nothrow_t),
359 MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse_nothrow,
360 _ZnamSt11align_val_tRKSt9nothrow_t),
361 // delete and delete[]
362 MAC_INTERPOSE_PAIR(rpfree, _ZdlPv), MAC_INTERPOSE_PAIR(rpfree, _ZdaPv),
363 MAC_INTERPOSE_PAIR(rpfree_size, _ZdlPvm),
364 MAC_INTERPOSE_PAIR(rpfree_size, _ZdaPvm),
365 MAC_INTERPOSE_PAIR(rpfree_aligned, _ZdlPvSt11align_val_t),
366 MAC_INTERPOSE_PAIR(rpfree_aligned, _ZdaPvSt11align_val_t),
367 MAC_INTERPOSE_PAIR(rpfree_size_aligned, _ZdlPvmSt11align_val_t),
368 MAC_INTERPOSE_PAIR(rpfree_size_aligned, _ZdaPvmSt11align_val_t),
369 // libc entry points
370 MAC_INTERPOSE_PAIR(rpmalloc, malloc),
371 MAC_INTERPOSE_PAIR(rpmalloc, calloc),
372 MAC_INTERPOSE_PAIR(rprealloc, realloc),
373 MAC_INTERPOSE_PAIR(rprealloc, reallocf),
374#if defined(__MAC_10_15) && __MAC_OS_X_VERSION_MIN_REQUIRED >= __MAC_10_15
375 MAC_INTERPOSE_PAIR(rpaligned_alloc, aligned_alloc),
376#endif
377 MAC_INTERPOSE_PAIR(rpmemalign, memalign),
378 MAC_INTERPOSE_PAIR(rpposix_memalign, posix_memalign),
379 MAC_INTERPOSE_PAIR(rpfree, free), MAC_INTERPOSE_PAIR(rpfree, cfree),
380 MAC_INTERPOSE_PAIR(rpmalloc_usable_size, malloc_usable_size),
381 MAC_INTERPOSE_PAIR(rpmalloc_usable_size, malloc_size)};
382
383#endif
384
385#if USE_ALIAS
386
387#define RPALIAS(fn) __attribute__((alias(#fn), used, visibility("default")));
388
389// Alias the C++ operators using the mangled names
390// (https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling)
391
392// operators delete and delete[]
393void _ZdlPv(void *p) RPALIAS(rpfree) void _ZdaPv(void *p) RPALIAS(rpfree)
394
395#if ARCH_64BIT
396 // 64-bit operators new and new[], normal and aligned
397 void *_Znwm(uint64_t size) RPMALLOC_ATTRIB_MALLOC
398 RPMALLOC_ATTRIB_ALLOC_SIZE(1)
399 RPALIAS(rpmalloc) void *_Znam(uint64_t size) RPMALLOC_ATTRIB_MALLOC
400 RPMALLOC_ATTRIB_ALLOC_SIZE(1) RPALIAS(rpmalloc) void *_Znwmm(uint64_t size,
401 uint64_t align)
402 RPALIAS(rpaligned_alloc_reverse) void *_Znamm(uint64_t size,
403 uint64_t align)
404 RPALIAS(rpaligned_alloc_reverse) void *_ZnwmSt11align_val_t(
405 size_t size, size_t align)
406 RPALIAS(rpaligned_alloc_reverse) void *_ZnamSt11align_val_t(
407 size_t size, size_t align)
408 RPALIAS(rpaligned_alloc_reverse) void *_ZnwmRKSt9nothrow_t(
409 size_t size, rp_nothrow_t t)
410 RPALIAS(rpmalloc_nothrow) void *_ZnamRKSt9nothrow_t(
411 size_t size,
412 rp_nothrow_t t) RPALIAS(rpmalloc_nothrow) void
413 *_ZnwmSt11align_val_tRKSt9nothrow_t(size_t size,
414 size_t align,
415 rp_nothrow_t t)
416 RPALIAS(rpaligned_alloc_reverse_nothrow) void
417 *_ZnamSt11align_val_tRKSt9nothrow_t(
418 size_t size, size_t align,
419 rp_nothrow_t t)
420 RPALIAS(rpaligned_alloc_reverse_nothrow)
421 // 64-bit operators delete and delete[], sized and aligned
422 void _ZdlPvm(void *p, size_t n) RPALIAS(rpfree_size) void _ZdaPvm(void *p,
423 size_t n)
424 RPALIAS(rpfree_size) void _ZdlPvSt11align_val_t(void *p, size_t a)
425 RPALIAS(rpfree_aligned) void _ZdaPvSt11align_val_t(void *p,
426 size_t a)
427 RPALIAS(rpfree_aligned) void _ZdlPvmSt11align_val_t(void *p,
428 size_t n,
429 size_t a)
430 RPALIAS(rpfree_size_aligned) void _ZdaPvmSt11align_val_t(
431 void *p, size_t n, size_t a)
432 RPALIAS(rpfree_size_aligned)
433#else
434 // 32-bit operators new and new[], normal and aligned
435 void *_Znwj(uint32_t size) RPMALLOC_ATTRIB_MALLOC
436 RPMALLOC_ATTRIB_ALLOC_SIZE(1)
437 RPALIAS(rpmalloc) void *_Znaj(uint32_t size) RPMALLOC_ATTRIB_MALLOC
438 RPMALLOC_ATTRIB_ALLOC_SIZE(1) RPALIAS(rpmalloc) void *_Znwjj(uint32_t size,
439 uint32_t align)
440 RPALIAS(rpaligned_alloc_reverse) void *_Znajj(uint32_t size,
441 uint32_t align)
442 RPALIAS(rpaligned_alloc_reverse) void *_ZnwjSt11align_val_t(
443 size_t size, size_t align)
444 RPALIAS(rpaligned_alloc_reverse) void *_ZnajSt11align_val_t(
445 size_t size, size_t align)
446 RPALIAS(rpaligned_alloc_reverse) void *_ZnwjRKSt9nothrow_t(
447 size_t size, rp_nothrow_t t)
448 RPALIAS(rpmalloc_nothrow) void *_ZnajRKSt9nothrow_t(
449 size_t size,
450 rp_nothrow_t t) RPALIAS(rpmalloc_nothrow) void
451 *_ZnwjSt11align_val_tRKSt9nothrow_t(size_t size,
452 size_t align,
453 rp_nothrow_t t)
454 RPALIAS(rpaligned_alloc_reverse_nothrow) void
455 *_ZnajSt11align_val_tRKSt9nothrow_t(
456 size_t size, size_t align,
457 rp_nothrow_t t)
458 RPALIAS(rpaligned_alloc_reverse_nothrow)
459 // 32-bit operators delete and delete[], sized and aligned
460 void _ZdlPvj(void *p, size_t n) RPALIAS(rpfree_size) void _ZdaPvj(void *p,
461 size_t n)
462 RPALIAS(rpfree_size) void _ZdlPvSt11align_val_t(void *p, size_t a)
463 RPALIAS(rpfree_aligned) void _ZdaPvSt11align_val_t(void *p,
464 size_t a)
465 RPALIAS(rpfree_aligned) void _ZdlPvjSt11align_val_t(void *p,
466 size_t n,
467 size_t a)
468 RPALIAS(rpfree_size_aligned) void _ZdaPvjSt11align_val_t(
469 void *p, size_t n, size_t a)
470 RPALIAS(rpfree_size_aligned)
471#endif
472
473 void *malloc(size_t size) RPMALLOC_ATTRIB_MALLOC
474 RPMALLOC_ATTRIB_ALLOC_SIZE(1)
475 RPALIAS(rpmalloc) void *calloc(size_t count, size_t size)
476 RPALIAS(rpcalloc) void *realloc(void *ptr, size_t size)
477 RPALIAS(rprealloc) void *reallocf(void *ptr, size_t size)
478 RPMALLOC_ATTRIB_MALLOC
479 RPMALLOC_ATTRIB_ALLOC_SIZE(2)
480 RPALIAS(rprealloc) void *aligned_alloc(size_t alignment, size_t size)
481 RPALIAS(rpaligned_alloc) void *memalign(
482 size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC
483 RPMALLOC_ATTRIB_ALLOC_SIZE(2)
484 RPALIAS(rpmemalign) int posix_memalign(void **memptr, size_t alignment,
485 size_t size)
486 RPALIAS(rpposix_memalign) void free(void *ptr)
487 RPALIAS(rpfree) void cfree(void *ptr) RPALIAS(rpfree)
488#if defined(__ANDROID__) || defined(__FreeBSD__)
489 size_t
490 malloc_usable_size(const void *ptr) RPALIAS(rpmalloc_usable_size)
491#else
492 size_t
493 malloc_usable_size(void *ptr) RPALIAS(rpmalloc_usable_size)
494#endif
495 size_t malloc_size(void *ptr) RPALIAS(rpmalloc_usable_size)
496
497#endif
498
499 static inline size_t _rpmalloc_page_size(void) {
500 return _memory_page_size;
501}
502
503extern void *RPMALLOC_CDECL reallocarray(void *ptr, size_t count, size_t size);
504
505extern void *RPMALLOC_CDECL reallocarray(void *ptr, size_t count, size_t size) {
506 size_t total;
507#if ENABLE_VALIDATE_ARGS
508#ifdef _MSC_VER
509 int err = SizeTMult(count, size, &total);
510 if ((err != S_OK) || (total >= MAX_ALLOC_SIZE)) {
511 errno = EINVAL;
512 return 0;
513 }
514#else
515 int err = __builtin_umull_overflow(count, size, &total);
516 if (err || (total >= MAX_ALLOC_SIZE)) {
517 errno = EINVAL;
518 return 0;
519 }
520#endif
521#else
522 total = count * size;
523#endif
524 return realloc(ptr, total);
525}
526
527extern inline void *RPMALLOC_CDECL valloc(size_t size) {
528 get_thread_heap();
529 return rpaligned_alloc(_rpmalloc_page_size(), size);
530}
531
532extern inline void *RPMALLOC_CDECL pvalloc(size_t size) {
533 get_thread_heap();
534 const size_t page_size = _rpmalloc_page_size();
535 const size_t aligned_size = ((size + page_size - 1) / page_size) * page_size;
536#if ENABLE_VALIDATE_ARGS
537 if (aligned_size < size) {
538 errno = EINVAL;
539 return 0;
540 }
541#endif
542 return rpaligned_alloc(_rpmalloc_page_size(), aligned_size);
543}
544
545#endif // ENABLE_OVERRIDE
546
547#if ENABLE_PRELOAD
548
549#ifdef _WIN32
550
551#if defined(BUILD_DYNAMIC_LINK) && BUILD_DYNAMIC_LINK
552
553extern __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE instance,
554 DWORD reason, LPVOID reserved);
555
556extern __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE instance,
557 DWORD reason,
558 LPVOID reserved) {
559 (void)sizeof(reserved);
560 (void)sizeof(instance);
561 if (reason == DLL_PROCESS_ATTACH)
562 rpmalloc_initialize();
563 else if (reason == DLL_PROCESS_DETACH)
564 rpmalloc_finalize();
565 else if (reason == DLL_THREAD_ATTACH)
566 rpmalloc_thread_initialize();
567 else if (reason == DLL_THREAD_DETACH)
568 rpmalloc_thread_finalize(1);
569 return TRUE;
570}
571
572// end BUILD_DYNAMIC_LINK
573#else
574
575extern void _global_rpmalloc_init(void) {
576 rpmalloc_set_main_thread();
577 rpmalloc_initialize();
578}
579
580#if defined(__clang__) || defined(__GNUC__)
581
582static void __attribute__((constructor)) initializer(void) {
583 _global_rpmalloc_init();
584}
585
586#elif defined(_MSC_VER)
587
588static int _global_rpmalloc_xib(void) {
589 _global_rpmalloc_init();
590 return 0;
591}
592
593#pragma section(".CRT$XIB", read)
594__declspec(allocate(".CRT$XIB")) void (*_rpmalloc_module_init)(void) =
595 _global_rpmalloc_xib;
596#if defined(_M_IX86) || defined(__i386__)
597#pragma comment(linker, "/include:" \
598 "__rpmalloc_module_init")
599#else
600#pragma comment(linker, "/include:" \
601 "_rpmalloc_module_init")
602#endif
603
604#endif
605
606// end !BUILD_DYNAMIC_LINK
607#endif
608
609#else
610
611#include <pthread.h>
612#include <stdint.h>
613#include <stdlib.h>
614#include <unistd.h>
615
616extern void rpmalloc_set_main_thread(void);
617
618static pthread_key_t destructor_key;
619
620static void thread_destructor(void *);
621
622static void __attribute__((constructor)) initializer(void) {
623 rpmalloc_set_main_thread();
624 rpmalloc_initialize();
625 pthread_key_create(&destructor_key, thread_destructor);
626}
627
628static void __attribute__((destructor)) finalizer(void) { rpmalloc_finalize(); }
629
630typedef struct {
631 void *(*real_start)(void *);
632 void *real_arg;
633} thread_starter_arg;
634
635static void *thread_starter(void *argptr) {
636 thread_starter_arg *arg = argptr;
637 void *(*real_start)(void *) = arg->real_start;
638 void *real_arg = arg->real_arg;
639 rpmalloc_thread_initialize();
640 rpfree(argptr);
641 pthread_setspecific(destructor_key, (void *)1);
642 return (*real_start)(real_arg);
643}
644
645static void thread_destructor(void *value) {
646 (void)sizeof(value);
647 rpmalloc_thread_finalize(1);
648}
649
650#ifdef __APPLE__
651
652static int pthread_create_proxy(pthread_t *thread, const pthread_attr_t *attr,
653 void *(*start_routine)(void *), void *arg) {
654 rpmalloc_initialize();
655 thread_starter_arg *starter_arg = rpmalloc(sizeof(thread_starter_arg));
656 starter_arg->real_start = start_routine;
657 starter_arg->real_arg = arg;
658 return pthread_create(thread, attr, thread_starter, starter_arg);
659}
660
661MAC_INTERPOSE_SINGLE(pthread_create_proxy, pthread_create);
662
663#else
664
665#include <dlfcn.h>
666
667int pthread_create(pthread_t *thread, const pthread_attr_t *attr,
668 void *(*start_routine)(void *), void *arg) {
669#if defined(__linux__) || defined(__FreeBSD__) || defined(__OpenBSD__) || \
670 defined(__NetBSD__) || defined(__DragonFly__) || defined(__APPLE__) || \
671 defined(__HAIKU__)
672 char fname[] = "pthread_create";
673#else
674 char fname[] = "_pthread_create";
675#endif
676 void *real_pthread_create = dlsym(RTLD_NEXT, fname);
677 rpmalloc_thread_initialize();
678 thread_starter_arg *starter_arg = rpmalloc(sizeof(thread_starter_arg));
679 starter_arg->real_start = start_routine;
680 starter_arg->real_arg = arg;
681 return (*(int (*)(pthread_t *, const pthread_attr_t *, void *(*)(void *),
682 void *))real_pthread_create)(thread, attr, thread_starter,
683 starter_arg);
684}
685
686#endif
687
688#endif
689
690#endif
691
692#if ENABLE_OVERRIDE
693
694#if defined(__GLIBC__) && defined(__linux__)
695
696void *__libc_malloc(size_t size) RPMALLOC_ATTRIB_MALLOC
697 RPMALLOC_ATTRIB_ALLOC_SIZE(1)
698 RPALIAS(rpmalloc) void *__libc_calloc(size_t count, size_t size)
699 RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE2(1, 2)
700 RPALIAS(rpcalloc) void *__libc_realloc(void *p, size_t size)
701 RPMALLOC_ATTRIB_MALLOC
702 RPMALLOC_ATTRIB_ALLOC_SIZE(2) RPALIAS(rprealloc) void __libc_free(void *p)
703 RPALIAS(rpfree) void __libc_cfree(void *p)
704 RPALIAS(rpfree) void *__libc_memalign(size_t align, size_t size)
705 RPMALLOC_ATTRIB_MALLOC
706 RPMALLOC_ATTRIB_ALLOC_SIZE(2)
707 RPALIAS(rpmemalign) int __posix_memalign(void **p, size_t align,
708 size_t size)
709 RPALIAS(rpposix_memalign)
710
711 extern void *__libc_valloc(size_t size);
712extern void *__libc_pvalloc(size_t size);
713
714void *__libc_valloc(size_t size) { return valloc(size); }
715
716void *__libc_pvalloc(size_t size) { return pvalloc(size); }
717
718#endif
719
720#endif
721
722#if (defined(__GNUC__) || defined(__clang__))
723#pragma GCC visibility pop
724#endif
725