429 lines · c
1//===---------------------- rpmalloc.h ------------------*- C -*-=============//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This library provides a cross-platform lock free thread caching malloc10// implementation in C11.11//12//===----------------------------------------------------------------------===//13 14#pragma once15 16#include <stddef.h>17 18#ifdef __cplusplus19extern "C" {20#endif21 22#if defined(__clang__) || defined(__GNUC__)23#define RPMALLOC_EXPORT __attribute__((visibility("default")))24#define RPMALLOC_ALLOCATOR25#if (defined(__clang_major__) && (__clang_major__ < 4)) || \26 (defined(__GNUC__) && defined(ENABLE_PRELOAD) && ENABLE_PRELOAD)27#define RPMALLOC_ATTRIB_MALLOC28#define RPMALLOC_ATTRIB_ALLOC_SIZE(size)29#define RPMALLOC_ATTRIB_ALLOC_SIZE2(count, size)30#else31#define RPMALLOC_ATTRIB_MALLOC __attribute__((__malloc__))32#define RPMALLOC_ATTRIB_ALLOC_SIZE(size) __attribute__((alloc_size(size)))33#define RPMALLOC_ATTRIB_ALLOC_SIZE2(count, size) \34 __attribute__((alloc_size(count, size)))35#endif36#define RPMALLOC_CDECL37#elif defined(_MSC_VER)38#define RPMALLOC_EXPORT39#define RPMALLOC_ALLOCATOR __declspec(allocator) __declspec(restrict)40#define RPMALLOC_ATTRIB_MALLOC41#define RPMALLOC_ATTRIB_ALLOC_SIZE(size)42#define RPMALLOC_ATTRIB_ALLOC_SIZE2(count, size)43#define RPMALLOC_CDECL __cdecl44#else45#define RPMALLOC_EXPORT46#define RPMALLOC_ALLOCATOR47#define RPMALLOC_ATTRIB_MALLOC48#define RPMALLOC_ATTRIB_ALLOC_SIZE(size)49#define RPMALLOC_ATTRIB_ALLOC_SIZE2(count, size)50#define RPMALLOC_CDECL51#endif52 53//! Define RPMALLOC_CONFIGURABLE to enable configuring sizes. Will introduce54// a very small overhead due to some size calculations not being compile time55// constants56#ifndef RPMALLOC_CONFIGURABLE57#define RPMALLOC_CONFIGURABLE 058#endif59 60//! Define RPMALLOC_FIRST_CLASS_HEAPS to enable heap based API (rpmalloc_heap_*61//! functions).62// Will introduce a very small overhead to track fully allocated spans in heaps63#ifndef RPMALLOC_FIRST_CLASS_HEAPS64#define RPMALLOC_FIRST_CLASS_HEAPS 065#endif66 67//! Flag to rpaligned_realloc to not preserve content in reallocation68#define RPMALLOC_NO_PRESERVE 169//! Flag to rpaligned_realloc to fail and return null pointer if grow cannot be70//! done in-place,71// in which case the original pointer is still valid (just like a call to72// realloc which failes to allocate a new block).73#define RPMALLOC_GROW_OR_FAIL 274 75typedef struct rpmalloc_global_statistics_t {76 //! Current amount of virtual memory mapped, all of which might not have been77 //! committed (only if ENABLE_STATISTICS=1)78 size_t mapped;79 //! Peak amount of virtual memory mapped, all of which might not have been80 //! committed (only if ENABLE_STATISTICS=1)81 size_t mapped_peak;82 //! Current amount of memory in global caches for small and medium sizes83 //! (<32KiB)84 size_t cached;85 //! Current amount of memory allocated in huge allocations, i.e larger than86 //! LARGE_SIZE_LIMIT which is 2MiB by default (only if ENABLE_STATISTICS=1)87 size_t huge_alloc;88 //! Peak amount of memory allocated in huge allocations, i.e larger than89 //! LARGE_SIZE_LIMIT which is 2MiB by default (only if ENABLE_STATISTICS=1)90 size_t huge_alloc_peak;91 //! Total amount of memory mapped since initialization (only if92 //! ENABLE_STATISTICS=1)93 size_t mapped_total;94 //! Total amount of memory unmapped since initialization (only if95 //! ENABLE_STATISTICS=1)96 size_t unmapped_total;97} rpmalloc_global_statistics_t;98 99typedef struct rpmalloc_thread_statistics_t {100 //! Current number of bytes available in thread size class caches for small101 //! and medium sizes (<32KiB)102 size_t sizecache;103 //! Current number of bytes available in thread span caches for small and104 //! medium sizes (<32KiB)105 size_t spancache;106 //! Total number of bytes transitioned from thread cache to global cache (only107 //! if ENABLE_STATISTICS=1)108 size_t thread_to_global;109 //! Total number of bytes transitioned from global cache to thread cache (only110 //! if ENABLE_STATISTICS=1)111 size_t global_to_thread;112 //! Per span count statistics (only if ENABLE_STATISTICS=1)113 struct {114 //! Currently used number of spans115 size_t current;116 //! High water mark of spans used117 size_t peak;118 //! Number of spans transitioned to global cache119 size_t to_global;120 //! Number of spans transitioned from global cache121 size_t from_global;122 //! Number of spans transitioned to thread cache123 size_t to_cache;124 //! Number of spans transitioned from thread cache125 size_t from_cache;126 //! Number of spans transitioned to reserved state127 size_t to_reserved;128 //! Number of spans transitioned from reserved state129 size_t from_reserved;130 //! Number of raw memory map calls (not hitting the reserve spans but131 //! resulting in actual OS mmap calls)132 size_t map_calls;133 } span_use[64];134 //! Per size class statistics (only if ENABLE_STATISTICS=1)135 struct {136 //! Current number of allocations137 size_t alloc_current;138 //! Peak number of allocations139 size_t alloc_peak;140 //! Total number of allocations141 size_t alloc_total;142 //! Total number of frees143 size_t free_total;144 //! Number of spans transitioned to cache145 size_t spans_to_cache;146 //! Number of spans transitioned from cache147 size_t spans_from_cache;148 //! Number of spans transitioned from reserved state149 size_t spans_from_reserved;150 //! Number of raw memory map calls (not hitting the reserve spans but151 //! resulting in actual OS mmap calls)152 size_t map_calls;153 } size_use[128];154} rpmalloc_thread_statistics_t;155 156typedef struct rpmalloc_config_t {157 //! Map memory pages for the given number of bytes. The returned address MUST158 //! be159 // aligned to the rpmalloc span size, which will always be a power of two.160 // Optionally the function can store an alignment offset in the offset161 // variable in case it performs alignment and the returned pointer is offset162 // from the actual start of the memory region due to this alignment. The163 // alignment offset will be passed to the memory unmap function. The164 // alignment offset MUST NOT be larger than 65535 (storable in an uint16_t),165 // if it is you must use natural alignment to shift it into 16 bits. If you166 // set a memory_map function, you must also set a memory_unmap function or167 // else the default implementation will be used for both. This function must168 // be thread safe, it can be called by multiple threads simultaneously.169 void *(*memory_map)(size_t size, size_t *offset);170 //! Unmap the memory pages starting at address and spanning the given number171 //! of bytes.172 // If release is set to non-zero, the unmap is for an entire span range as173 // returned by a previous call to memory_map and that the entire range should174 // be released. The release argument holds the size of the entire span range.175 // If release is set to 0, the unmap is a partial decommit of a subset of the176 // mapped memory range. If you set a memory_unmap function, you must also set177 // a memory_map function or else the default implementation will be used for178 // both. This function must be thread safe, it can be called by multiple179 // threads simultaneously.180 void (*memory_unmap)(void *address, size_t size, size_t offset,181 size_t release);182 //! Called when an assert fails, if asserts are enabled. Will use the standard183 //! assert()184 // if this is not set.185 void (*error_callback)(const char *message);186 //! Called when a call to map memory pages fails (out of memory). If this187 //! callback is188 // not set or returns zero the library will return a null pointer in the189 // allocation call. If this callback returns non-zero the map call will be190 // retried. The argument passed is the number of bytes that was requested in191 // the map call. Only used if the default system memory map function is used192 // (memory_map callback is not set).193 int (*map_fail_callback)(size_t size);194 //! Size of memory pages. The page size MUST be a power of two. All memory195 //! mapping196 // requests to memory_map will be made with size set to a multiple of the197 // page size. Used if RPMALLOC_CONFIGURABLE is defined to 1, otherwise system198 // page size is used.199 size_t page_size;200 //! Size of a span of memory blocks. MUST be a power of two, and in201 //! [4096,262144]202 // range (unless 0 - set to 0 to use the default span size). Used if203 // RPMALLOC_CONFIGURABLE is defined to 1.204 size_t span_size;205 //! Number of spans to map at each request to map new virtual memory blocks.206 //! This can207 // be used to minimize the system call overhead at the cost of virtual memory208 // address space. The extra mapped pages will not be written until actually209 // used, so physical committed memory should not be affected in the default210 // implementation. Will be aligned to a multiple of spans that match memory211 // page size in case of huge pages.212 size_t span_map_count;213 //! Enable use of large/huge pages. If this flag is set to non-zero and page214 //! size is215 // zero, the allocator will try to enable huge pages and auto detect the216 // configuration. If this is set to non-zero and page_size is also non-zero,217 // the allocator will assume huge pages have been configured and enabled218 // prior to initializing the allocator. For Windows, see219 // https://docs.microsoft.com/en-us/windows/desktop/memory/large-page-support220 // For Linux, see https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt221 int enable_huge_pages;222 //! Respectively allocated pages and huge allocated pages names for systems223 // supporting it to be able to distinguish among anonymous regions.224 const char *page_name;225 const char *huge_page_name;226} rpmalloc_config_t;227 228//! Initialize allocator with default configuration229RPMALLOC_EXPORT int rpmalloc_initialize(void);230 231//! Initialize allocator with given configuration232RPMALLOC_EXPORT int rpmalloc_initialize_config(const rpmalloc_config_t *config);233 234//! Get allocator configuration235RPMALLOC_EXPORT const rpmalloc_config_t *rpmalloc_config(void);236 237//! Finalize allocator238RPMALLOC_EXPORT void rpmalloc_finalize(void);239 240//! Initialize allocator for calling thread241RPMALLOC_EXPORT void rpmalloc_thread_initialize(void);242 243//! Finalize allocator for calling thread244RPMALLOC_EXPORT void rpmalloc_thread_finalize(int release_caches);245 246//! Perform deferred deallocations pending for the calling thread heap247RPMALLOC_EXPORT void rpmalloc_thread_collect(void);248 249//! Query if allocator is initialized for calling thread250RPMALLOC_EXPORT int rpmalloc_is_thread_initialized(void);251 252//! Get per-thread statistics253RPMALLOC_EXPORT void254rpmalloc_thread_statistics(rpmalloc_thread_statistics_t *stats);255 256//! Get global statistics257RPMALLOC_EXPORT void258rpmalloc_global_statistics(rpmalloc_global_statistics_t *stats);259 260//! Dump all statistics in human readable format to file (should be a FILE*)261RPMALLOC_EXPORT void rpmalloc_dump_statistics(void *file);262 263//! Allocate a memory block of at least the given size264RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *265rpmalloc(size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(1);266 267//! Free the given memory block268RPMALLOC_EXPORT void rpfree(void *ptr);269 270//! Allocate a memory block of at least the given size and zero initialize it271RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *272rpcalloc(size_t num, size_t size) RPMALLOC_ATTRIB_MALLOC273 RPMALLOC_ATTRIB_ALLOC_SIZE2(1, 2);274 275//! Reallocate the given block to at least the given size276RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *277rprealloc(void *ptr, size_t size) RPMALLOC_ATTRIB_MALLOC278 RPMALLOC_ATTRIB_ALLOC_SIZE(2);279 280//! Reallocate the given block to at least the given size and alignment,281// with optional control flags (see RPMALLOC_NO_PRESERVE).282// Alignment must be a power of two and a multiple of sizeof(void*),283// and should ideally be less than memory page size. A caveat of rpmalloc284// internals is that this must also be strictly less than the span size285// (default 64KiB)286RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *287rpaligned_realloc(void *ptr, size_t alignment, size_t size, size_t oldsize,288 unsigned int flags) RPMALLOC_ATTRIB_MALLOC289 RPMALLOC_ATTRIB_ALLOC_SIZE(3);290 291//! Allocate a memory block of at least the given size and alignment.292// Alignment must be a power of two and a multiple of sizeof(void*),293// and should ideally be less than memory page size. A caveat of rpmalloc294// internals is that this must also be strictly less than the span size295// (default 64KiB)296RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *297rpaligned_alloc(size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC298 RPMALLOC_ATTRIB_ALLOC_SIZE(2);299 300//! Allocate a memory block of at least the given size and alignment, and zero301//! initialize it.302// Alignment must be a power of two and a multiple of sizeof(void*),303// and should ideally be less than memory page size. A caveat of rpmalloc304// internals is that this must also be strictly less than the span size305// (default 64KiB)306RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *307rpaligned_calloc(size_t alignment, size_t num,308 size_t size) RPMALLOC_ATTRIB_MALLOC309 RPMALLOC_ATTRIB_ALLOC_SIZE2(2, 3);310 311//! Allocate a memory block of at least the given size and alignment.312// Alignment must be a power of two and a multiple of sizeof(void*),313// and should ideally be less than memory page size. A caveat of rpmalloc314// internals is that this must also be strictly less than the span size315// (default 64KiB)316RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *317rpmemalign(size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC318 RPMALLOC_ATTRIB_ALLOC_SIZE(2);319 320//! Allocate a memory block of at least the given size and alignment.321// Alignment must be a power of two and a multiple of sizeof(void*),322// and should ideally be less than memory page size. A caveat of rpmalloc323// internals is that this must also be strictly less than the span size324// (default 64KiB)325RPMALLOC_EXPORT int rpposix_memalign(void **memptr, size_t alignment,326 size_t size);327 328//! Query the usable size of the given memory block (from given pointer to the329//! end of block)330RPMALLOC_EXPORT size_t rpmalloc_usable_size(void *ptr);331 332//! Dummy empty function for forcing linker symbol inclusion333RPMALLOC_EXPORT void rpmalloc_linker_reference(void);334 335#if RPMALLOC_FIRST_CLASS_HEAPS336 337//! Heap type338typedef struct heap_t rpmalloc_heap_t;339 340//! Acquire a new heap. Will reuse existing released heaps or allocate memory341//! for a new heap342// if none available. Heap API is implemented with the strict assumption that343// only one single thread will call heap functions for a given heap at any344// given time, no functions are thread safe.345RPMALLOC_EXPORT rpmalloc_heap_t *rpmalloc_heap_acquire(void);346 347//! Release a heap (does NOT free the memory allocated by the heap, use348//! rpmalloc_heap_free_all before destroying the heap).349// Releasing a heap will enable it to be reused by other threads. Safe to pass350// a null pointer.351RPMALLOC_EXPORT void rpmalloc_heap_release(rpmalloc_heap_t *heap);352 353//! Allocate a memory block of at least the given size using the given heap.354RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *355rpmalloc_heap_alloc(rpmalloc_heap_t *heap, size_t size) RPMALLOC_ATTRIB_MALLOC356 RPMALLOC_ATTRIB_ALLOC_SIZE(2);357 358//! Allocate a memory block of at least the given size using the given heap. The359//! returned360// block will have the requested alignment. Alignment must be a power of two361// and a multiple of sizeof(void*), and should ideally be less than memory page362// size. A caveat of rpmalloc internals is that this must also be strictly less363// than the span size (default 64KiB).364RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *365rpmalloc_heap_aligned_alloc(rpmalloc_heap_t *heap, size_t alignment,366 size_t size) RPMALLOC_ATTRIB_MALLOC367 RPMALLOC_ATTRIB_ALLOC_SIZE(3);368 369//! Allocate a memory block of at least the given size using the given heap and370//! zero initialize it.371RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *372rpmalloc_heap_calloc(rpmalloc_heap_t *heap, size_t num,373 size_t size) RPMALLOC_ATTRIB_MALLOC374 RPMALLOC_ATTRIB_ALLOC_SIZE2(2, 3);375 376//! Allocate a memory block of at least the given size using the given heap and377//! zero initialize it. The returned378// block will have the requested alignment. Alignment must either be zero, or a379// power of two and a multiple of sizeof(void*), and should ideally be less380// than memory page size. A caveat of rpmalloc internals is that this must also381// be strictly less than the span size (default 64KiB).382RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *383rpmalloc_heap_aligned_calloc(rpmalloc_heap_t *heap, size_t alignment,384 size_t num, size_t size) RPMALLOC_ATTRIB_MALLOC385 RPMALLOC_ATTRIB_ALLOC_SIZE2(2, 3);386 387//! Reallocate the given block to at least the given size. The memory block MUST388//! be allocated389// by the same heap given to this function.390RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *391rpmalloc_heap_realloc(rpmalloc_heap_t *heap, void *ptr, size_t size,392 unsigned int flags) RPMALLOC_ATTRIB_MALLOC393 RPMALLOC_ATTRIB_ALLOC_SIZE(3);394 395//! Reallocate the given block to at least the given size. The memory block MUST396//! be allocated397// by the same heap given to this function. The returned block will have the398// requested alignment. Alignment must be either zero, or a power of two and a399// multiple of sizeof(void*), and should ideally be less than memory page size.400// A caveat of rpmalloc internals is that this must also be strictly less than401// the span size (default 64KiB).402RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void *rpmalloc_heap_aligned_realloc(403 rpmalloc_heap_t *heap, void *ptr, size_t alignment, size_t size,404 unsigned int flags) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(4);405 406//! Free the given memory block from the given heap. The memory block MUST be407//! allocated408// by the same heap given to this function.409RPMALLOC_EXPORT void rpmalloc_heap_free(rpmalloc_heap_t *heap, void *ptr);410 411//! Free all memory allocated by the heap412RPMALLOC_EXPORT void rpmalloc_heap_free_all(rpmalloc_heap_t *heap);413 414//! Set the given heap as the current heap for the calling thread. A heap MUST415//! only be current heap416// for a single thread, a heap can never be shared between multiple threads.417// The previous current heap for the calling thread is released to be reused by418// other threads.419RPMALLOC_EXPORT void rpmalloc_heap_thread_set_current(rpmalloc_heap_t *heap);420 421//! Returns which heap the given pointer is allocated on422RPMALLOC_EXPORT rpmalloc_heap_t *rpmalloc_get_heap_for_ptr(void *ptr);423 424#endif425 426#ifdef __cplusplus427}428#endif429