LiVES
3.2.0
|
#include "rpmalloc.h"
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <errno.h>
#include <sys/mman.h>
#include <sched.h>
#include <stdatomic.h>
Go to the source code of this file.
Macros | |
#define | HEAP_ARRAY_SIZE 47 |
Build time configurable limits. More... | |
#define | ENABLE_THREAD_CACHE 1 |
Enable per-thread cache. More... | |
#define | ENABLE_GLOBAL_CACHE 1 |
Enable global cache shared between all threads, requires thread cache. More... | |
#define | ENABLE_VALIDATE_ARGS 0 |
Enable validation of args to public entry points. More... | |
#define | ENABLE_STATISTICS 0 |
Enable statistics collection. More... | |
#define | ENABLE_ASSERTS 0 |
Enable asserts. More... | |
#define | ENABLE_OVERRIDE 0 |
Override standard library malloc/free and new/delete entry points. More... | |
#define | ENABLE_PRELOAD 0 |
Support preloading. More... | |
#define | DISABLE_UNMAP 0 |
Disable unmapping memory pages. More... | |
#define | DEFAULT_SPAN_MAP_COUNT 64 |
Default number of spans to map in call to map more virtual memory (default values yield 4MiB here) More... | |
#define | ENABLE_UNLIMITED_CACHE 0 |
Unlimited thread and global cache. More... | |
#define | ENABLE_UNLIMITED_THREAD_CACHE ENABLE_UNLIMITED_CACHE |
Unlimited cache disables any thread cache limitations. More... | |
#define | THREAD_CACHE_MULTIPLIER 16 |
Multiplier for thread cache (cache limit will be span release count multiplied by this value) More... | |
#define | ENABLE_ADAPTIVE_THREAD_CACHE 0 |
Enable adaptive size of per-thread cache (still bounded by THREAD_CACHE_MULTIPLIER hard limit) More... | |
#define | ENABLE_UNLIMITED_GLOBAL_CACHE ENABLE_UNLIMITED_CACHE |
Unlimited cache disables any global cache limitations. More... | |
#define | GLOBAL_CACHE_MULTIPLIER (THREAD_CACHE_MULTIPLIER * 6) |
Multiplier for global cache (cache limit will be span release count multiplied by this value) More... | |
#define | PLATFORM_WINDOWS 0 |
#define | PLATFORM_POSIX 1 |
#define | FORCEINLINE inline __attribute__((__always_inline__)) |
Platform and arch specifics. More... | |
#define | MAP_UNINITIALIZED 0 |
#define | assert(x) do {} while(0) |
#define | EXPECTED(x) __builtin_expect((x), 1) |
#define | UNEXPECTED(x) __builtin_expect((x), 0) |
#define | _rpmalloc_stat_inc(counter) do {} while(0) |
Statistics related functions (evaluate to nothing when statistics not enabled) More... | |
#define | _rpmalloc_stat_dec(counter) do {} while(0) |
#define | _rpmalloc_stat_add(counter, value) do {} while(0) |
#define | _rpmalloc_stat_add64(counter, value) do {} while(0) |
#define | _rpmalloc_stat_add_peak(counter, value, peak) do {} while (0) |
#define | _rpmalloc_stat_sub(counter, value) do {} while(0) |
#define | _rpmalloc_stat_inc_alloc(heap, class_idx) do {} while(0) |
#define | _rpmalloc_stat_inc_free(heap, class_idx) do {} while(0) |
#define | SMALL_GRANULARITY 16 |
Preconfigured limits and sizes. More... | |
#define | SMALL_GRANULARITY_SHIFT 4 |
Small granularity shift count. More... | |
#define | SMALL_CLASS_COUNT 65 |
Number of small block size classes. More... | |
#define | SMALL_SIZE_LIMIT (SMALL_GRANULARITY * (SMALL_CLASS_COUNT - 1)) |
Maximum size of a small block. More... | |
#define | MEDIUM_GRANULARITY 512 |
Granularity of a medium allocation block. More... | |
#define | MEDIUM_GRANULARITY_SHIFT 9 |
Medium granularity shift count. More... | |
#define | MEDIUM_CLASS_COUNT 61 |
Number of medium block size classes. More... | |
#define | SIZE_CLASS_COUNT (SMALL_CLASS_COUNT + MEDIUM_CLASS_COUNT) |
Total number of small + medium size classes. More... | |
#define | LARGE_CLASS_COUNT 32 |
Number of large block size classes. More... | |
#define | MEDIUM_SIZE_LIMIT (SMALL_SIZE_LIMIT + (MEDIUM_GRANULARITY * MEDIUM_CLASS_COUNT)) |
Maximum size of a medium block. More... | |
#define | LARGE_SIZE_LIMIT ((LARGE_CLASS_COUNT * _memory_span_size) - SPAN_HEADER_SIZE) |
Maximum size of a large block. More... | |
#define | HEAP_ORPHAN_ABA_SIZE 512 |
ABA protection size in orhpan heap list (also becomes limit of smallest page size) More... | |
#define | SPAN_HEADER_SIZE 128 |
Size of a span header (must be a multiple of SMALL_GRANULARITY and a power of two) More... | |
#define | pointer_offset(ptr, ofs) (void*)((char*)(ptr) + (ptrdiff_t)(ofs)) |
#define | pointer_diff(first, second) (ptrdiff_t)((const char*)(first) - (const char*)(second)) |
#define | INVALID_POINTER ((void*)((uintptr_t)-1)) |
#define | SIZE_CLASS_LARGE SIZE_CLASS_COUNT |
#define | SIZE_CLASS_HUGE ((uint32_t)-1) |
#define | SPAN_FLAG_MASTER 1U |
Flag indicating span is the first (master) span of a split superspan. More... | |
#define | SPAN_FLAG_SUBSPAN 2U |
Flag indicating span is a secondary (sub) span of a split superspan. More... | |
#define | SPAN_FLAG_ALIGNED_BLOCKS 4U |
Flag indicating span has blocks with increased alignment. More... | |
#define | _memory_span_size (64 * 1024) |
Hardwired span size (64KiB) More... | |
#define | _memory_span_size_shift 16 |
#define | _memory_span_mask (~((uintptr_t)(_memory_span_size - 1))) |
#define | TLS_MODEL __attribute__((tls_model("initial-exec"))) |
Thread local heap and ID. More... | |
Typedefs | |
typedef volatile | _Atomic(int32_t) |
Atomic access abstraction (since MSVC does not do C11 yet) More... | |
typedef struct span_list_t | span_list_t |
Span list. More... | |
typedef struct span_active_t | span_active_t |
Span active data. More... | |
Functions | |
_Static_assert ((SMALL_GRANULARITY &(SMALL_GRANULARITY - 1))==0, "Small granularity must be power of two") | |
_Static_assert ((SPAN_HEADER_SIZE &(SPAN_HEADER_SIZE - 1))==0, "Span header size must be power of two") | |
_Static_assert (sizeof(span_t)<=SPAN_HEADER_SIZE, "span size mismatch") | |
_Static_assert (sizeof(size_class_t)==8, "Size class size mismatch") | |
int | rpmalloc_initialize (void) |
Initialize the allocator and setup global data. More... | |
int | rpmalloc_initialize_config (const rpmalloc_config_t *config) |
Initialize allocator with given configuration. More... | |
void | rpmalloc_finalize (void) |
Finalize the allocator. More... | |
void | rpmalloc_thread_initialize (void) |
Initialize thread, assign heap. More... | |
void | rpmalloc_thread_finalize (void) |
Finalize thread, orphan heap. More... | |
int | rpmalloc_is_thread_initialized (void) |
Query if allocator is initialized for calling thread. More... | |
const rpmalloc_config_t * | rpmalloc_config (void) |
Get allocator configuration. More... | |
RPMALLOC_ALLOCATOR void * | rpmalloc (size_t size) |
Allocate a memory block of at least the given size. More... | |
void | rpfree (void *ptr) |
Free the given memory block. More... | |
RPMALLOC_ALLOCATOR void * | rpcalloc (size_t num, size_t size) |
RPMALLOC_ALLOCATOR void * | rprealloc (void *ptr, size_t size) |
Reallocate the given block to at least the given size. More... | |
RPMALLOC_ALLOCATOR void * | rpaligned_realloc (void *ptr, size_t alignment, size_t size, size_t oldsize, unsigned int flags) |
Reallocate the given block to at least the given size and alignment,. More... | |
RPMALLOC_ALLOCATOR void * | rpaligned_alloc (size_t alignment, size_t size) |
Allocate a memory block of at least the given size and alignment. More... | |
RPMALLOC_ALLOCATOR void * | rpaligned_calloc (size_t alignment, size_t num, size_t size) |
RPMALLOC_ALLOCATOR void * | rpmemalign (size_t alignment, size_t size) |
Allocate a memory block of at least the given size and alignment. More... | |
int | rpposix_memalign (void **memptr, size_t alignment, size_t size) |
Allocate a memory block of at least the given size and alignment. More... | |
size_t | rpmalloc_usable_size (void *ptr) |
Query the usable size of the given memory block (from given pointer to the end of block) More... | |
void | rpmalloc_thread_collect (void) |
Perform deferred deallocations pending for the calling thread heap. More... | |
void | rpmalloc_thread_statistics (rpmalloc_thread_statistics_t *stats) |
Get per-thread statistics. More... | |
void | rpmalloc_global_statistics (rpmalloc_global_statistics_t *stats) |
Get global statistics. More... | |
void | rpmalloc_dump_statistics (void *file) |
Dump all statistics in human readable format to file (should be a FILE*) More... | |
#define _memory_span_mask (~((uintptr_t)(_memory_span_size - 1))) |
Definition at line 563 of file rpmalloc.c.
#define _memory_span_size (64 * 1024) |
Hardwired span size (64KiB)
Definition at line 561 of file rpmalloc.c.
#define _memory_span_size_shift 16 |
Definition at line 562 of file rpmalloc.c.
#define _rpmalloc_stat_add | ( | counter, | |
value | |||
) | do {} while(0) |
Definition at line 276 of file rpmalloc.c.
#define _rpmalloc_stat_add64 | ( | counter, | |
value | |||
) | do {} while(0) |
Definition at line 277 of file rpmalloc.c.
#define _rpmalloc_stat_add_peak | ( | counter, | |
value, | |||
peak | |||
) | do {} while (0) |
Definition at line 278 of file rpmalloc.c.
#define _rpmalloc_stat_dec | ( | counter | ) | do {} while(0) |
Definition at line 275 of file rpmalloc.c.
#define _rpmalloc_stat_inc | ( | counter | ) | do {} while(0) |
Statistics related functions (evaluate to nothing when statistics not enabled)
///
Definition at line 274 of file rpmalloc.c.
#define _rpmalloc_stat_inc_alloc | ( | heap, | |
class_idx | |||
) | do {} while(0) |
Definition at line 280 of file rpmalloc.c.
#define _rpmalloc_stat_inc_free | ( | heap, | |
class_idx | |||
) | do {} while(0) |
Definition at line 281 of file rpmalloc.c.
#define _rpmalloc_stat_sub | ( | counter, | |
value | |||
) | do {} while(0) |
Definition at line 279 of file rpmalloc.c.
#define assert | ( | x | ) | do {} while(0) |
Definition at line 186 of file rpmalloc.c.
#define DEFAULT_SPAN_MAP_COUNT 64 |
Default number of spans to map in call to map more virtual memory (default values yield 4MiB here)
Definition at line 58 of file rpmalloc.c.
#define DISABLE_UNMAP 0 |
Disable unmapping memory pages.
Definition at line 54 of file rpmalloc.c.
#define ENABLE_ADAPTIVE_THREAD_CACHE 0 |
Enable adaptive size of per-thread cache (still bounded by THREAD_CACHE_MULTIPLIER hard limit)
Definition at line 77 of file rpmalloc.c.
#define ENABLE_ASSERTS 0 |
Enable asserts.
Definition at line 42 of file rpmalloc.c.
#define ENABLE_GLOBAL_CACHE 1 |
Enable global cache shared between all threads, requires thread cache.
Definition at line 30 of file rpmalloc.c.
#define ENABLE_OVERRIDE 0 |
Override standard library malloc/free and new/delete entry points.
Definition at line 46 of file rpmalloc.c.
#define ENABLE_PRELOAD 0 |
Support preloading.
Definition at line 50 of file rpmalloc.c.
#define ENABLE_STATISTICS 0 |
Enable statistics collection.
Definition at line 38 of file rpmalloc.c.
#define ENABLE_THREAD_CACHE 1 |
Enable per-thread cache.
Definition at line 26 of file rpmalloc.c.
#define ENABLE_UNLIMITED_CACHE 0 |
Unlimited thread and global cache.
Definition at line 64 of file rpmalloc.c.
#define ENABLE_UNLIMITED_GLOBAL_CACHE ENABLE_UNLIMITED_CACHE |
Unlimited cache disables any global cache limitations.
Definition at line 89 of file rpmalloc.c.
#define ENABLE_UNLIMITED_THREAD_CACHE ENABLE_UNLIMITED_CACHE |
Unlimited cache disables any thread cache limitations.
Definition at line 68 of file rpmalloc.c.
#define ENABLE_VALIDATE_ARGS 0 |
Enable validation of args to public entry points.
Definition at line 34 of file rpmalloc.c.
#define EXPECTED | ( | x | ) | __builtin_expect((x), 1) |
Definition at line 245 of file rpmalloc.c.
#define FORCEINLINE inline __attribute__((__always_inline__)) |
Platform and arch specifics.
Definition at line 125 of file rpmalloc.c.
#define GLOBAL_CACHE_MULTIPLIER (THREAD_CACHE_MULTIPLIER * 6) |
Multiplier for global cache (cache limit will be span release count multiplied by this value)
Definition at line 93 of file rpmalloc.c.
#define HEAP_ARRAY_SIZE 47 |
#define HEAP_ORPHAN_ABA_SIZE 512 |
ABA protection size in orhpan heap list (also becomes limit of smallest page size)
Definition at line 311 of file rpmalloc.c.
#define INVALID_POINTER ((void*)((uintptr_t)-1)) |
Definition at line 327 of file rpmalloc.c.
#define LARGE_CLASS_COUNT 32 |
Number of large block size classes.
Definition at line 305 of file rpmalloc.c.
#define LARGE_SIZE_LIMIT ((LARGE_CLASS_COUNT * _memory_span_size) - SPAN_HEADER_SIZE) |
Maximum size of a large block.
Definition at line 309 of file rpmalloc.c.
#define MAP_UNINITIALIZED 0 |
Definition at line 173 of file rpmalloc.c.
#define MEDIUM_CLASS_COUNT 61 |
Number of medium block size classes.
Definition at line 301 of file rpmalloc.c.
#define MEDIUM_GRANULARITY 512 |
Granularity of a medium allocation block.
Definition at line 297 of file rpmalloc.c.
#define MEDIUM_GRANULARITY_SHIFT 9 |
Medium granularity shift count.
Definition at line 299 of file rpmalloc.c.
#define MEDIUM_SIZE_LIMIT (SMALL_SIZE_LIMIT + (MEDIUM_GRANULARITY * MEDIUM_CLASS_COUNT)) |
Maximum size of a medium block.
Definition at line 307 of file rpmalloc.c.
#define PLATFORM_POSIX 1 |
Definition at line 114 of file rpmalloc.c.
#define PLATFORM_WINDOWS 0 |
Definition at line 113 of file rpmalloc.c.
#define pointer_diff | ( | first, | |
second | |||
) | (ptrdiff_t)((const char*)(first) - (const char*)(second)) |
Definition at line 325 of file rpmalloc.c.
#define pointer_offset | ( | ptr, | |
ofs | |||
) | (void*)((char*)(ptr) + (ptrdiff_t)(ofs)) |
Definition at line 324 of file rpmalloc.c.
#define SIZE_CLASS_COUNT (SMALL_CLASS_COUNT + MEDIUM_CLASS_COUNT) |
Total number of small + medium size classes.
Definition at line 303 of file rpmalloc.c.
#define SIZE_CLASS_HUGE ((uint32_t)-1) |
Definition at line 330 of file rpmalloc.c.
#define SIZE_CLASS_LARGE SIZE_CLASS_COUNT |
Definition at line 329 of file rpmalloc.c.
#define SMALL_CLASS_COUNT 65 |
Number of small block size classes.
Definition at line 293 of file rpmalloc.c.
#define SMALL_GRANULARITY 16 |
Preconfigured limits and sizes.
Granularity of a small allocation block (must be power of two)
Definition at line 289 of file rpmalloc.c.
#define SMALL_GRANULARITY_SHIFT 4 |
Small granularity shift count.
Definition at line 291 of file rpmalloc.c.
#define SMALL_SIZE_LIMIT (SMALL_GRANULARITY * (SMALL_CLASS_COUNT - 1)) |
Maximum size of a small block.
Definition at line 295 of file rpmalloc.c.
#define SPAN_FLAG_ALIGNED_BLOCKS 4U |
Flag indicating span has blocks with increased alignment.
Definition at line 356 of file rpmalloc.c.
#define SPAN_FLAG_MASTER 1U |
Flag indicating span is the first (master) span of a split superspan.
Definition at line 352 of file rpmalloc.c.
#define SPAN_FLAG_SUBSPAN 2U |
Flag indicating span is a secondary (sub) span of a split superspan.
Definition at line 354 of file rpmalloc.c.
#define SPAN_HEADER_SIZE 128 |
Size of a span header (must be a multiple of SMALL_GRANULARITY and a power of two)
Definition at line 313 of file rpmalloc.c.
#define THREAD_CACHE_MULTIPLIER 16 |
Multiplier for thread cache (cache limit will be span release count multiplied by this value)
Definition at line 73 of file rpmalloc.c.
#define TLS_MODEL __attribute__((tls_model("initial-exec"))) |
#define UNEXPECTED | ( | x | ) | __builtin_expect((x), 0) |
Definition at line 246 of file rpmalloc.c.
typedef volatile _Atomic(int32_t) |
Atomic access abstraction (since MSVC does not do C11 yet)
///
Definition at line 226 of file rpmalloc.c.
typedef struct span_active_t span_active_t |
Span active data.
Definition at line 316 of file rpmalloc.c.
typedef struct span_list_t span_list_t |
Span list.
Definition at line 316 of file rpmalloc.c.
_Static_assert | ( | (SMALL_GRANULARITY &(SMALL_GRANULARITY - 1)) | = =0 , |
"Small granularity must be power of two" | |||
) |
_Static_assert | ( | (SPAN_HEADER_SIZE &(SPAN_HEADER_SIZE - 1)) | = =0 , |
"Span header size must be power of two" | |||
) |
_Static_assert | ( | sizeof(size_class_t) | = =8 , |
"Size class size mismatch" | |||
) |
_Static_assert | ( | sizeof(span_t)<= | SPAN_HEADER_SIZE, |
"span size mismatch" | |||
) |
RPMALLOC_ALLOCATOR void* rpaligned_alloc | ( | size_t | alignment, |
size_t | size | ||
) |
Allocate a memory block of at least the given size and alignment.
Definition at line 2669 of file rpmalloc.c.
Referenced by rpaligned_calloc(), rpmemalign(), and rpposix_memalign().
|
inline |
Definition at line 2675 of file rpmalloc.c.
Referenced by quick_calloc().
RPMALLOC_ALLOCATOR void* rpaligned_realloc | ( | void * | ptr, |
size_t | alignment, | ||
size_t | size, | ||
size_t | oldsize, | ||
unsigned int | flags | ||
) |
Reallocate the given block to at least the given size and alignment,.
Definition at line 2656 of file rpmalloc.c.
|
inline |
Definition at line 2617 of file rpmalloc.c.
|
inline |
Free the given memory block.
Definition at line 2612 of file rpmalloc.c.
Referenced by _ext_free(), and quick_free().
|
inline |
Allocate a memory block of at least the given size.
Definition at line 2600 of file rpmalloc.c.
Referenced by _ext_malloc().
const rpmalloc_config_t* rpmalloc_config | ( | void | ) |
Get allocator configuration.
Definition at line 2593 of file rpmalloc.c.
void rpmalloc_dump_statistics | ( | void * | file | ) |
Dump all statistics in human readable format to file (should be a FILE*)
Definition at line 2861 of file rpmalloc.c.
void rpmalloc_finalize | ( | void | ) |
void rpmalloc_global_statistics | ( | rpmalloc_global_statistics_t * | stats | ) |
Get global statistics.
Definition at line 2787 of file rpmalloc.c.
|
inline |
Initialize the allocator and setup global data.
Initialize allocator with default configuration.
Definition at line 2318 of file rpmalloc.c.
Referenced by init_memfuncs().
int rpmalloc_initialize_config | ( | const rpmalloc_config_t * | config | ) |
Initialize allocator with given configuration.
Definition at line 2327 of file rpmalloc.c.
int rpmalloc_is_thread_initialized | ( | void | ) |
Query if allocator is initialized for calling thread.
Definition at line 2588 of file rpmalloc.c.
|
inline |
Perform deferred deallocations pending for the calling thread heap.
Definition at line 2720 of file rpmalloc.c.
void rpmalloc_thread_finalize | ( | void | ) |
Finalize thread, orphan heap.
Finalize allocator for calling thread.
Definition at line 2577 of file rpmalloc.c.
Referenced by rpmalloc_finalize().
|
inline |
Initialize thread, assign heap.
Initialize allocator for calling thread.
Definition at line 2562 of file rpmalloc.c.
Referenced by init_thread_memfuncs().
void rpmalloc_thread_statistics | ( | rpmalloc_thread_statistics_t * | stats | ) |
Get per-thread statistics.
Definition at line 2724 of file rpmalloc.c.
|
inline |
Query the usable size of the given memory block (from given pointer to the end of block)
Definition at line 2715 of file rpmalloc.c.
|
inline |
Allocate a memory block of at least the given size and alignment.
Definition at line 2701 of file rpmalloc.c.
|
inline |
Allocate a memory block of at least the given size and alignment.
Definition at line 2706 of file rpmalloc.c.
|
inline |
Reallocate the given block to at least the given size.
Definition at line 2644 of file rpmalloc.c.
Referenced by _ext_realloc().