20 #ifndef HEAP_ARRAY_SIZE
21 #define HEAP_ARRAY_SIZE 47
24 #ifndef ENABLE_THREAD_CACHE
25 #define ENABLE_THREAD_CACHE 1
28 #ifndef ENABLE_GLOBAL_CACHE
29 #define ENABLE_GLOBAL_CACHE 1
32 #ifndef ENABLE_VALIDATE_ARGS
33 #define ENABLE_VALIDATE_ARGS 0
36 #ifndef ENABLE_STATISTICS
37 #define ENABLE_STATISTICS 0
40 #ifndef ENABLE_ASSERTS
41 #define ENABLE_ASSERTS 0
44 #ifndef ENABLE_OVERRIDE
45 #define ENABLE_OVERRIDE 0
48 #ifndef ENABLE_PRELOAD
49 #define ENABLE_PRELOAD 0
53 #define DISABLE_UNMAP 0
56 #ifndef DEFAULT_SPAN_MAP_COUNT
57 #define DEFAULT_SPAN_MAP_COUNT 64
61 #if ENABLE_THREAD_CACHE
62 #ifndef ENABLE_UNLIMITED_CACHE
63 #define ENABLE_UNLIMITED_CACHE 0
66 #ifndef ENABLE_UNLIMITED_THREAD_CACHE
67 #define ENABLE_UNLIMITED_THREAD_CACHE ENABLE_UNLIMITED_CACHE
70 #if !ENABLE_UNLIMITED_THREAD_CACHE
71 #ifndef THREAD_CACHE_MULTIPLIER
72 #define THREAD_CACHE_MULTIPLIER 16
75 #ifndef ENABLE_ADAPTIVE_THREAD_CACHE
76 #define ENABLE_ADAPTIVE_THREAD_CACHE 0
82 #if ENABLE_GLOBAL_CACHE && ENABLE_THREAD_CACHE
84 #undef ENABLE_UNLIMITED_GLOBAL_CACHE
85 #define ENABLE_UNLIMITED_GLOBAL_CACHE 1
87 #ifndef ENABLE_UNLIMITED_GLOBAL_CACHE
88 #define ENABLE_UNLIMITED_GLOBAL_CACHE ENABLE_UNLIMITED_CACHE
91 #if !ENABLE_UNLIMITED_GLOBAL_CACHE
92 #define GLOBAL_CACHE_MULTIPLIER (THREAD_CACHE_MULTIPLIER * 6)
96 # undef ENABLE_GLOBAL_CACHE
97 # define ENABLE_GLOBAL_CACHE 0
100 #if !ENABLE_THREAD_CACHE || ENABLE_UNLIMITED_THREAD_CACHE
101 # undef ENABLE_ADAPTIVE_THREAD_CACHE
102 # define ENABLE_ADAPTIVE_THREAD_CACHE 0
105 #if DISABLE_UNMAP && !ENABLE_GLOBAL_CACHE
106 # error Must use global cache if unmap is disabled
109 #if defined( _WIN32 ) || defined( __WIN32__ ) || defined( _WIN64 )
110 # define PLATFORM_WINDOWS 1
111 # define PLATFORM_POSIX 0
113 # define PLATFORM_WINDOWS 0
114 # define PLATFORM_POSIX 1
118 #if defined(_MSC_VER) && !defined(__clang__)
120 # define FORCEINLINE inline __forceinline
122 # define _Static_assert static_assert
125 # define FORCEINLINE inline __attribute__((__always_inline__))
129 # ifndef WIN32_LEAN_AND_MEAN
130 # define WIN32_LEAN_AND_MEAN
132 # include <Windows.h>
133 # if ENABLE_VALIDATE_ARGS
134 # include <Intsafe.h>
140 # if defined(__APPLE__)
141 # include <mach/mach_vm.h>
142 # include <mach/vm_statistics.h>
143 # include <pthread.h>
145 # if defined(__HAIKU__)
147 # include <pthread.h>
155 #if defined(_WIN32) && (!defined(BUILD_DYNAMIC_LINK) || !BUILD_DYNAMIC_LINK)
156 #include <fibersapi.h>
157 static DWORD fls_key;
159 _rpmalloc_thread_destructor(
void *value) {
166 # include <sys/mman.h>
169 # include <sys/sysctl.h>
170 # define MAP_HUGETLB MAP_ALIGNED_SUPER
172 # ifndef MAP_UNINITIALIZED
173 # define MAP_UNINITIALIZED 0
180 # if defined(_MSC_VER) && !defined(_DEBUG)
186 # define assert(x) do {} while(0)
188 #if ENABLE_STATISTICS
198 #if defined(_MSC_VER) && !defined(__clang__)
200 typedef volatile long atomic32_t;
201 typedef volatile long long atomic64_t;
202 typedef volatile void *atomicptr_t;
204 static FORCEINLINE int32_t atomic_load32(atomic32_t *src) {
return *src; }
205 static FORCEINLINE void atomic_store32(atomic32_t *dst, int32_t val) { *dst = val; }
206 static FORCEINLINE int32_t atomic_incr32(atomic32_t *val) {
return (int32_t)InterlockedIncrement(val); }
207 static FORCEINLINE int32_t atomic_decr32(atomic32_t *val) {
return (int32_t)InterlockedDecrement(val); }
208 #if ENABLE_STATISTICS || ENABLE_ADAPTIVE_THREAD_CACHE
209 static FORCEINLINE int64_t atomic_load64(atomic64_t *src) {
return *src; }
210 static FORCEINLINE int64_t atomic_add64(atomic64_t *val, int64_t add) {
return (int64_t)InterlockedExchangeAdd64(val, add) + add; }
212 static FORCEINLINE int32_t atomic_add32(atomic32_t *val, int32_t add) {
return (int32_t)InterlockedExchangeAdd(val, add) + add; }
213 static FORCEINLINE void *atomic_load_ptr(atomicptr_t *src) {
return (
void *)*src; }
214 static FORCEINLINE void atomic_store_ptr(atomicptr_t *dst,
void *val) { *dst = val; }
215 static FORCEINLINE void atomic_store_ptr_release(atomicptr_t *dst,
void *val) { *dst = val; }
216 static FORCEINLINE int atomic_cas_ptr(atomicptr_t *dst,
void *val,
void *ref) {
return (InterlockedCompareExchangePointer((
void *
volatile *)dst, val, ref) == ref) ? 1 : 0; }
217 static FORCEINLINE int atomic_cas_ptr_acquire(atomicptr_t *dst,
void *val,
void *ref) {
return atomic_cas_ptr(dst, val, ref); }
219 #define EXPECTED(x) (x)
220 #define UNEXPECTED(x) (x)
224 #include <stdatomic.h>
227 typedef volatile _Atomic(int64_t) atomic64_t;
228 typedef volatile _Atomic(
void *) atomicptr_t;
230 static FORCEINLINE int32_t atomic_load32(atomic32_t *src) {
return atomic_load_explicit(src, memory_order_relaxed); }
231 static FORCEINLINE void atomic_store32(atomic32_t *dst, int32_t val) { atomic_store_explicit(dst, val, memory_order_relaxed); }
232 static FORCEINLINE int32_t atomic_incr32(atomic32_t *val) {
return atomic_fetch_add_explicit(val, 1, memory_order_relaxed) + 1; }
233 static FORCEINLINE int32_t atomic_decr32(atomic32_t *val) {
return atomic_fetch_add_explicit(val, -1, memory_order_relaxed) - 1; }
234 #if ENABLE_STATISTICS || ENABLE_ADAPTIVE_THREAD_CACHE
235 static FORCEINLINE int64_t atomic_load64(atomic64_t *val) {
return atomic_load_explicit(val, memory_order_relaxed); }
236 static FORCEINLINE int64_t atomic_add64(atomic64_t *val, int64_t add) {
return atomic_fetch_add_explicit(val, add, memory_order_relaxed) + add; }
238 static FORCEINLINE int32_t atomic_add32(atomic32_t *val, int32_t add) {
return atomic_fetch_add_explicit(val, add, memory_order_relaxed) + add; }
239 static FORCEINLINE void *atomic_load_ptr(atomicptr_t *src) {
return atomic_load_explicit(src, memory_order_relaxed); }
240 static FORCEINLINE void atomic_store_ptr(atomicptr_t *dst,
void *val) { atomic_store_explicit(dst, val, memory_order_relaxed); }
241 static FORCEINLINE void atomic_store_ptr_release(atomicptr_t *dst,
void *val) { atomic_store_explicit(dst, val, memory_order_release); }
242 static FORCEINLINE int atomic_cas_ptr(atomicptr_t *dst,
void *val,
void *ref) {
return atomic_compare_exchange_weak_explicit(dst, &ref, val, memory_order_relaxed, memory_order_relaxed); }
243 static FORCEINLINE int atomic_cas_ptr_acquire(atomicptr_t *dst,
void *val,
void *ref) {
return atomic_compare_exchange_weak_explicit(dst, &ref, val, memory_order_acquire, memory_order_relaxed); }
245 #define EXPECTED(x) __builtin_expect((x), 1)
246 #define UNEXPECTED(x) __builtin_expect((x), 0)
256 #if ENABLE_STATISTICS
257 # define _rpmalloc_stat_inc(counter) atomic_incr32(counter)
258 # define _rpmalloc_stat_dec(counter) atomic_decr32(counter)
259 # define _rpmalloc_stat_add(counter, value) atomic_add32(counter, (int32_t)(value))
260 # define _rpmalloc_stat_add64(counter, value) atomic_add64(counter, (int64_t)(value))
261 # define _rpmalloc_stat_add_peak(counter, value, peak) do { int32_t _cur_count = atomic_add32(counter, (int32_t)(value)); if (_cur_count > (peak)) peak = _cur_count; } while (0)
262 # define _rpmalloc_stat_sub(counter, value) atomic_add32(counter, -(int32_t)(value))
263 # define _rpmalloc_stat_inc_alloc(heap, class_idx) do { \
264 int32_t alloc_current = atomic_incr32(&heap->size_class_use[class_idx].alloc_current); \
265 if (alloc_current > heap->size_class_use[class_idx].alloc_peak) \
266 heap->size_class_use[class_idx].alloc_peak = alloc_current; \
267 atomic_incr32(&heap->size_class_use[class_idx].alloc_total); \
269 # define _rpmalloc_stat_inc_free(heap, class_idx) do { \
270 atomic_decr32(&heap->size_class_use[class_idx].alloc_current); \
271 atomic_incr32(&heap->size_class_use[class_idx].free_total); \
274 # define _rpmalloc_stat_inc(counter) do {} while(0)
275 # define _rpmalloc_stat_dec(counter) do {} while(0)
276 # define _rpmalloc_stat_add(counter, value) do {} while(0)
277 # define _rpmalloc_stat_add64(counter, value) do {} while(0)
278 # define _rpmalloc_stat_add_peak(counter, value, peak) do {} while (0)
279 # define _rpmalloc_stat_sub(counter, value) do {} while(0)
280 # define _rpmalloc_stat_inc_alloc(heap, class_idx) do {} while(0)
281 # define _rpmalloc_stat_inc_free(heap, class_idx) do {} while(0)
289 #define SMALL_GRANULARITY 16
290 #define SMALL_GRANULARITY_SHIFT 4
292 #define SMALL_CLASS_COUNT 65
294 #define SMALL_SIZE_LIMIT (SMALL_GRANULARITY * (SMALL_CLASS_COUNT - 1))
296 #define MEDIUM_GRANULARITY 512
298 #define MEDIUM_GRANULARITY_SHIFT 9
300 #define MEDIUM_CLASS_COUNT 61
302 #define SIZE_CLASS_COUNT (SMALL_CLASS_COUNT + MEDIUM_CLASS_COUNT)
304 #define LARGE_CLASS_COUNT 32
306 #define MEDIUM_SIZE_LIMIT (SMALL_SIZE_LIMIT + (MEDIUM_GRANULARITY * MEDIUM_CLASS_COUNT))
308 #define LARGE_SIZE_LIMIT ((LARGE_CLASS_COUNT * _memory_span_size) - SPAN_HEADER_SIZE)
310 #define HEAP_ORPHAN_ABA_SIZE 512
312 #define SPAN_HEADER_SIZE 128
318 #if ENABLE_VALIDATE_ARGS
319 #undef MAX_ALLOC_SIZE
321 #define MAX_ALLOC_SIZE (((size_t)-1) - _memory_span_size)
324 #define pointer_offset(ptr, ofs) (void*)((char*)(ptr) + (ptrdiff_t)(ofs))
325 #define pointer_diff(first, second) (ptrdiff_t)((const char*)(first) - (const char*)(second))
327 #define INVALID_POINTER ((void*)((uintptr_t)-1))
329 #define SIZE_CLASS_LARGE SIZE_CLASS_COUNT
330 #define SIZE_CLASS_HUGE ((uint32_t)-1)
339 typedef struct heap_t heap_t;
341 typedef struct span_t span_t;
347 typedef struct size_class_t size_class_t;
349 typedef struct global_cache_t global_cache_t;
352 #define SPAN_FLAG_MASTER 1U
353 #define SPAN_FLAG_SUBSPAN 2U
355 #define SPAN_FLAG_ALIGNED_BLOCKS 4U
358 #if ENABLE_ADAPTIVE_THREAD_CACHE || ENABLE_STATISTICS
364 #if ENABLE_STATISTICS
365 atomic32_t spans_to_global;
368 atomic32_t spans_from_global;
370 atomic32_t spans_to_cache;
372 atomic32_t spans_from_cache;
374 atomic32_t spans_to_reserved;
376 atomic32_t spans_from_reserved;
378 atomic32_t spans_map_calls;
381 typedef struct span_use_t span_use_t;
384 #if ENABLE_STATISTICS
385 struct size_class_use_t {
387 atomic32_t alloc_current;
391 atomic32_t alloc_total;
393 atomic32_t free_total;
395 atomic32_t spans_current;
399 atomic32_t spans_to_cache;
401 atomic32_t spans_from_cache;
403 atomic32_t spans_from_reserved;
405 atomic32_t spans_map_calls;
407 typedef struct size_class_use_t size_class_use_t;
422 uint32_t block_count;
426 uint32_t free_list_limit;
430 atomicptr_t free_list_deferred;
440 uint32_t total_spans;
442 uint32_t offset_from_master;
444 atomic32_t remaining_spans;
446 uint32_t align_offset;
459 uintptr_t owner_thread;
465 #if RPMALLOC_FIRST_CLASS_HEAPS
470 #if ENABLE_THREAD_CACHE
474 atomicptr_t span_free_deferred;
476 #if ENABLE_ADAPTIVE_THREAD_CACHE || ENABLE_STATISTICS
480 #if RPMALLOC_FIRST_CLASS_HEAPS
481 span_t *large_huge_span;
484 size_t full_span_count;
487 span_t *span_reserve;
489 span_t *span_reserve_master;
491 size_t spans_reserved;
505 atomic32_t child_count;
506 #if ENABLE_STATISTICS
507 atomic64_t thread_to_global;
510 atomic64_t global_to_thread;
517 struct size_class_t {
521 uint16_t block_count;
527 struct global_cache_t {
543 static int _rpmalloc_initialized;
547 static size_t _memory_page_size;
549 static size_t _memory_page_size_shift;
551 static size_t _memory_map_granularity;
552 #if RPMALLOC_CONFIGURABLE
560 #define _memory_span_size (64 * 1024)
562 #define _memory_span_size_shift 16
563 #define _memory_span_mask (~((uintptr_t)(_memory_span_size - 1)))
565 static size_t _memory_span_map_count;
568 static size_t _memory_span_release_count;
570 static size_t _memory_span_release_count_large;
574 static size_t _memory_medium_size_limit;
576 static atomic32_t _memory_heap_id;
578 static int _memory_huge_pages;
579 #if ENABLE_GLOBAL_CACHE
586 static atomicptr_t _memory_orphan_heaps;
587 #if RPMALLOC_FIRST_CLASS_HEAPS
588 static atomicptr_t _memory_first_class_orphan_heaps;
591 static atomic32_t _memory_orphan_counter;
593 #if ENABLE_STATISTICS
594 static atomic32_t _memory_active_heaps;
597 static atomic32_t _mapped_pages;
599 static int32_t _mapped_pages_peak;
601 static atomic32_t _master_spans;
603 static atomic32_t _reserved_spans;
605 static atomic32_t _mapped_total;
607 static atomic32_t _unmapped_total;
609 static atomic32_t _mapped_pages_os;
611 static atomic32_t _huge_pages_current;
613 static int32_t _huge_pages_peak;
623 #if (defined(__APPLE__) || defined(__HAIKU__)) && ENABLE_PRELOAD
624 static pthread_key_t _memory_thread_heap;
627 # define _Thread_local __declspec(thread)
630 # define TLS_MODEL __attribute__((tls_model("initial-exec")))
631 # if !defined(__clang__) && defined(__GNUC__)
632 # define _Thread_local __thread
635 static _Thread_local heap_t *_memory_thread_heap
TLS_MODEL;
638 static inline heap_t *
639 get_thread_heap_raw(
void) {
640 #if (defined(__APPLE__) || defined(__HAIKU__)) && ENABLE_PRELOAD
641 return pthread_getspecific(_memory_thread_heap);
643 return _memory_thread_heap;
648 static inline heap_t *
649 get_thread_heap(
void) {
650 heap_t *heap = get_thread_heap_raw();
655 return get_thread_heap_raw();
662 static inline uintptr_t
663 get_thread_id(
void) {
665 return (uintptr_t)((
void *)NtCurrentTeb());
666 #elif defined(__GNUC__) || defined(__clang__)
668 # if defined(__i386__)
669 __asm__(
"movl %%gs:0, %0" :
"=r"(tid) : :);
670 # elif defined(__MACH__)
671 __asm__(
"movq %%gs:0, %0" :
"=r"(tid) : :);
672 # elif defined(__x86_64__)
673 __asm__(
"movq %%fs:0, %0" :
"=r"(tid) : :);
674 # elif defined(__arm__)
675 __asm__
volatile(
"mrc p15, 0, %0, c13, c0, 3" :
"=r"(tid));
676 # elif defined(__aarch64__)
677 __asm__
volatile(
"mrs %0, tpidr_el0" :
"=r"(tid));
679 tid = (uintptr_t)get_thread_heap_raw();
683 return (uintptr_t)get_thread_heap_raw();
689 set_thread_heap(heap_t *heap) {
690 #if (defined(__APPLE__) || defined(__HAIKU__)) && ENABLE_PRELOAD
691 pthread_setspecific(_memory_thread_heap, heap);
693 _memory_thread_heap = heap;
696 heap->owner_thread = get_thread_id();
710 _rpmalloc_mmap(
size_t size,
size_t *offset) {
711 assert(!(size % _memory_page_size));
712 assert(size >= _memory_page_size);
715 return _memory_config.
memory_map(size, offset);
724 _rpmalloc_unmap(
void *address,
size_t size,
size_t offset,
size_t release) {
725 assert(!release || (release >= size));
726 assert(!release || (release >= _memory_page_size));
728 assert(!(release % _memory_page_size));
732 _memory_config.
memory_unmap(address, size, offset, release);
737 _rpmalloc_mmap_os(
size_t size,
size_t *offset) {
740 assert(size >= _memory_page_size);
743 void *ptr = VirtualAlloc(0, size + padding, (_memory_huge_pages ? MEM_LARGE_PAGES : 0) | MEM_RESERVE | MEM_COMMIT,
746 assert(ptr &&
"Failed to map virtual memory block");
751 # if defined(__APPLE__)
752 int fd = (int)VM_MAKE_TAG(240U);
753 if (_memory_huge_pages)
754 fd |= VM_FLAGS_SUPERPAGE_SIZE_2MB;
755 void *ptr = mmap(0, size + padding, PROT_READ | PROT_WRITE, flags, fd, 0);
756 # elif defined(MAP_HUGETLB)
757 void *ptr = mmap(0, size + padding, PROT_READ | PROT_WRITE, (_memory_huge_pages ? MAP_HUGETLB : 0) | flags, -1, 0);
759 void *ptr = mmap(0, size + padding, PROT_READ | PROT_WRITE, flags, -1, 0);
761 if ((ptr == MAP_FAILED) || !ptr) {
762 assert(
"Failed to map virtual memory block" == 0);
766 _rpmalloc_stat_add(&_mapped_pages_os, (int32_t)((size + padding) >> _memory_page_size_shift));
770 assert(final_padding <= padding);
771 assert(!(final_padding % 8));
773 *offset = final_padding >> 3;
781 _rpmalloc_unmap_os(
void *address,
size_t size,
size_t offset,
size_t release) {
782 assert(release || (offset == 0));
783 assert(!release || (release >= _memory_page_size));
784 assert(size >= _memory_page_size);
785 if (release && offset) {
795 if (!VirtualFree(address, release ? 0 : size, release ? MEM_RELEASE : MEM_DECOMMIT)) {
796 assert(address &&
"Failed to unmap virtual memory block");
800 if (munmap(address, release)) {
801 assert(
"Failed to unmap virtual memory block" == 0);
804 #if defined(POSIX_MADV_FREE)
805 if (posix_madvise(address, size, POSIX_MADV_FREE))
807 if (posix_madvise(address, size, POSIX_MADV_DONTNEED)) {
808 assert(
"Failed to madvise virtual memory block as free" == 0);
824 #if ENABLE_THREAD_CACHE
827 _rpmalloc_span_unmap(span_t *span);
831 _rpmalloc_span_list_unmap_all(span_t *span) {
832 size_t list_size = span->list_size;
833 for (
size_t ispan = 0; ispan < list_size; ++ispan) {
834 span_t *next_span = span->next;
835 _rpmalloc_span_unmap(span);
843 _rpmalloc_span_list_push(span_t **head, span_t *span) {
846 span->list_size = (*head)->list_size + 1;
850 return span->list_size;
855 _rpmalloc_span_list_pop(span_t **head) {
856 span_t *span = *head;
857 span_t *next_span = 0;
858 if (span->list_size > 1) {
860 next_span = span->next;
862 next_span->list_size = span->list_size - 1;
870 _rpmalloc_span_list_split(span_t *span,
size_t limit) {
874 if (span->list_size > limit) {
875 uint32_t list_size = 1;
878 while (list_size < limit) {
885 next->list_size = span->list_size - list_size;
886 span->list_size = list_size;
896 _rpmalloc_span_double_link_list_add(span_t **head, span_t *span) {
899 (*head)->prev = span;
908 _rpmalloc_span_double_link_list_pop_head(span_t **head, span_t *span) {
916 _rpmalloc_span_double_link_list_remove(span_t **head, span_t *span) {
921 span_t *next_span = span->next;
922 span_t *prev_span = span->prev;
923 prev_span->next = next_span;
925 next_span->prev = prev_span;
938 _rpmalloc_heap_cache_insert(heap_t *heap, span_t *span);
941 _rpmalloc_heap_finalize(heap_t *heap);
944 _rpmalloc_heap_set_reserved_spans(heap_t *heap, span_t *master, span_t *reserve,
size_t reserve_span_count);
948 _rpmalloc_span_mark_as_subspan_unless_master(span_t *master, span_t *subspan,
size_t span_count) {
950 if (subspan != master) {
953 subspan->align_offset = 0;
955 subspan->span_count = (uint32_t)span_count;
960 _rpmalloc_span_map_from_reserve(heap_t *heap,
size_t span_count) {
962 span_t *span = heap->span_reserve;
964 heap->spans_reserved -= span_count;
966 _rpmalloc_span_mark_as_subspan_unless_master(heap->span_reserve_master, span, span_count);
975 _rpmalloc_span_align_count(
size_t span_count) {
976 size_t request_count = (span_count > _memory_span_map_count) ? span_count : _memory_span_map_count;
978 request_count += _memory_span_map_count - (request_count % _memory_span_map_count);
979 return request_count;
984 _rpmalloc_span_initialize(span_t *span,
size_t total_span_count,
size_t span_count,
size_t align_offset) {
985 span->total_spans = (uint32_t)total_span_count;
986 span->span_count = (uint32_t)span_count;
987 span->align_offset = (uint32_t)align_offset;
989 atomic_store32(&span->remaining_spans, (int32_t)total_span_count);
994 _rpmalloc_span_map_aligned_count(heap_t *heap,
size_t span_count) {
997 size_t aligned_span_count = _rpmalloc_span_align_count(span_count);
998 size_t align_offset = 0;
999 span_t *span = (span_t *)_rpmalloc_mmap(aligned_span_count *
_memory_span_size, &align_offset);
1002 _rpmalloc_span_initialize(span, aligned_span_count, span_count, align_offset);
1007 if (aligned_span_count > span_count) {
1009 size_t reserved_count = aligned_span_count - span_count;
1010 if (heap->spans_reserved) {
1011 _rpmalloc_span_mark_as_subspan_unless_master(heap->span_reserve_master, heap->span_reserve, heap->spans_reserved);
1012 _rpmalloc_heap_cache_insert(heap, heap->span_reserve);
1014 _rpmalloc_heap_set_reserved_spans(heap, span, reserved_spans, reserved_count);
1021 _rpmalloc_span_map(heap_t *heap,
size_t span_count) {
1022 if (span_count <= heap->spans_reserved)
1023 return _rpmalloc_span_map_from_reserve(heap, span_count);
1024 return _rpmalloc_span_map_aligned_count(heap, span_count);
1029 _rpmalloc_span_unmap(span_t *span) {
1034 span_t *master = is_master ? span : ((span_t *)
pointer_offset(span,
1039 size_t span_count = span->span_count;
1042 assert(span->align_offset == 0);
1053 if (atomic_add32(&master->remaining_spans, -(int32_t)span_count) <= 0) {
1056 size_t unmap_count = master->span_count;
1058 unmap_count = master->total_spans;
1067 _rpmalloc_span_release_to_cache(heap_t *heap, span_t *span) {
1068 assert(heap == span->heap);
1070 #if ENABLE_ADAPTIVE_THREAD_CACHE || ENABLE_STATISTICS
1071 atomic_decr32(&heap->span_use[0].current);
1076 _rpmalloc_heap_cache_insert(heap, span);
1082 free_list_partial_init(
void **list,
void **first_block,
void *page_start,
void *block_start,
1083 uint32_t block_count, uint32_t block_size) {
1085 *first_block = block_start;
1086 if (block_count > 1) {
1088 void *block_end =
pointer_offset(block_start, (
size_t)block_size * block_count);
1090 if (block_size < (_memory_page_size >> 1)) {
1092 if (page_end < block_end)
1093 block_end = page_end;
1098 while (next_block < block_end) {
1099 *((
void **)free_block) = next_block;
1100 free_block = next_block;
1104 *((
void **)free_block) = 0;
1113 _rpmalloc_span_initialize_new(heap_t *heap, span_t *span, uint32_t class_idx) {
1114 assert(span->span_count == 1);
1115 size_class_t *size_class = _memory_size_class + class_idx;
1116 span->size_class = class_idx;
1119 span->block_size = size_class->block_size;
1120 span->block_count = size_class->block_count;
1121 span->free_list = 0;
1122 span->list_size = 0;
1123 atomic_store_ptr_release(&span->free_list_deferred, 0);
1127 span->free_list_limit = free_list_partial_init(&heap->free_list[class_idx], &block,
1130 if (span->free_list_limit < span->block_count) {
1131 _rpmalloc_span_double_link_list_add(&heap->partial_span[class_idx], span);
1132 span->used_count = span->free_list_limit;
1134 #if RPMALLOC_FIRST_CLASS_HEAPS
1135 _rpmalloc_span_double_link_list_add(&heap->full_span[class_idx], span);
1137 ++heap->full_span_count;
1138 span->used_count = span->block_count;
1144 _rpmalloc_span_extract_free_list_deferred(span_t *span) {
1148 span->free_list = atomic_load_ptr(&span->free_list_deferred);
1150 !atomic_cas_ptr_acquire(&span->free_list_deferred,
INVALID_POINTER, span->free_list));
1151 span->used_count -= span->list_size;
1152 span->list_size = 0;
1153 atomic_store_ptr_release(&span->free_list_deferred, 0);
1157 _rpmalloc_span_is_fully_utilized(span_t *span) {
1158 assert(span->free_list_limit <= span->block_count);
1159 return !span->free_list && (span->free_list_limit >= span->block_count);
1163 _rpmalloc_span_finalize(heap_t *heap,
size_t iclass, span_t *span, span_t **list_head) {
1164 span_t *class_span = (span_t *)((uintptr_t)heap->free_list[iclass] &
_memory_span_mask);
1165 if (span == class_span) {
1167 void *block = span->free_list;
1168 void *last_block = 0;
1171 block = *((
void **)block);
1173 uint32_t free_count = 0;
1174 block = heap->free_list[iclass];
1177 block = *((
void **)block);
1180 *((
void **)last_block) = heap->free_list[iclass];
1182 span->free_list = heap->free_list[iclass];
1184 heap->free_list[iclass] = 0;
1185 span->used_count -= free_count;
1188 assert(span->list_size == span->used_count);
1189 if (span->list_size == span->used_count) {
1194 _rpmalloc_span_double_link_list_remove(list_head, span);
1195 _rpmalloc_span_unmap(span);
1208 #if ENABLE_GLOBAL_CACHE
1212 _rpmalloc_global_cache_insert(global_cache_t *cache, span_t *span,
size_t cache_limit) {
1213 assert((span->list_size == 1) || (span->next != 0));
1214 int32_t list_size = (int32_t)span->list_size;
1218 if (atomic_add32(&cache->size, list_size) > (int32_t)cache_limit) {
1219 #if !ENABLE_UNLIMITED_GLOBAL_CACHE
1220 _rpmalloc_span_list_unmap_all(span);
1221 atomic_add32(&cache->size, -list_size);
1225 void *current_cache, *new_cache;
1227 current_cache = atomic_load_ptr(&cache->cache);
1229 new_cache = (
void *)((uintptr_t)span | ((uintptr_t)atomic_incr32(&cache->counter) & ~
_memory_span_mask));
1230 }
while (!atomic_cas_ptr(&cache->cache, new_cache, current_cache));
1235 _rpmalloc_global_cache_extract(global_cache_t *cache) {
1238 void *global_span = atomic_load_ptr(&cache->cache);
1241 span_t *span = (span_t *)span_ptr;
1244 void *new_cache = (
void *)((uintptr_t)span->prev | ((uintptr_t)atomic_incr32(&cache->counter) & ~
_memory_span_mask));
1245 if (atomic_cas_ptr(&cache->cache, new_cache, global_span)) {
1246 atomic_add32(&cache->size, -(int32_t)span->list_size);
1256 _rpmalloc_global_cache_finalize(global_cache_t *cache) {
1257 void *current_cache = atomic_load_ptr(&cache->cache);
1261 atomic_add32(&cache->size, -(int32_t)span->list_size);
1262 _rpmalloc_span_list_unmap_all(span);
1265 assert(!atomic_load32(&cache->size));
1266 atomic_store_ptr(&cache->cache, 0);
1267 atomic_store32(&cache->size, 0);
1272 _rpmalloc_global_cache_insert_span_list(span_t *span) {
1273 size_t span_count = span->span_count;
1274 #if ENABLE_UNLIMITED_GLOBAL_CACHE
1275 _rpmalloc_global_cache_insert(&_memory_span_cache[span_count - 1], span, 0);
1278 _memory_span_release_count_large));
1279 _rpmalloc_global_cache_insert(&_memory_span_cache[span_count - 1], span, cache_limit);
1285 _rpmalloc_global_cache_extract_span_list(
size_t span_count) {
1286 span_t *span = _rpmalloc_global_cache_extract(&_memory_span_cache[span_count - 1]);
1287 assert(!span || (span->span_count == span_count));
1300 static void _rpmalloc_deallocate_huge(span_t *);
1304 _rpmalloc_heap_set_reserved_spans(heap_t *heap, span_t *master, span_t *reserve,
size_t reserve_span_count) {
1305 heap->span_reserve_master = master;
1306 heap->span_reserve = reserve;
1307 heap->spans_reserved = reserve_span_count;
1312 _rpmalloc_heap_cache_adopt_deferred(heap_t *heap, span_t **single_span) {
1313 span_t *span = (span_t *)atomic_load_ptr(&heap->span_free_deferred);
1316 while (!atomic_cas_ptr(&heap->span_free_deferred, 0, span))
1317 span = (span_t *)atomic_load_ptr(&heap->span_free_deferred);
1319 span_t *next_span = (span_t *)span->free_list;
1320 assert(span->heap == heap);
1322 assert(heap->full_span_count);
1323 --heap->full_span_count;
1324 #if RPMALLOC_FIRST_CLASS_HEAPS
1325 _rpmalloc_span_double_link_list_remove(&heap->full_span[span->size_class], span);
1327 if (single_span && !*single_span) {
1328 *single_span = span;
1332 _rpmalloc_heap_cache_insert(heap, span);
1336 _rpmalloc_deallocate_huge(span);
1339 assert(heap->full_span_count);
1340 --heap->full_span_count;
1341 #if RPMALLOC_FIRST_CLASS_HEAPS
1342 _rpmalloc_span_double_link_list_remove(&heap->large_huge_span, span);
1344 uint32_t idx = span->span_count - 1;
1345 if (!idx && single_span && !*single_span) {
1346 *single_span = span;
1349 _rpmalloc_heap_cache_insert(heap, span);
1358 _rpmalloc_heap_unmap(heap_t *heap) {
1359 if (!heap->master_heap) {
1360 if ((heap->finalize > 1) && !atomic_load32(&heap->child_count)) {
1361 size_t heap_size =
sizeof(heap_t);
1362 size_t block_size = _memory_page_size * ((heap_size + _memory_page_size - 1) >> _memory_page_size_shift);
1363 _rpmalloc_unmap(heap, block_size, heap->align_offset, block_size);
1366 if (atomic_decr32(&heap->master_heap->child_count) == 0) {
1367 _rpmalloc_heap_unmap(heap->master_heap);
1373 _rpmalloc_heap_global_finalize(heap_t *heap) {
1374 if (heap->finalize++ > 1) {
1379 _rpmalloc_heap_finalize(heap);
1381 #if ENABLE_THREAD_CACHE
1383 span_t *span = heap->span_cache[iclass];
1384 heap->span_cache[iclass] = 0;
1386 _rpmalloc_span_list_unmap_all(span);
1390 if (heap->full_span_count) {
1396 if (heap->free_list[iclass] || heap->partial_span[iclass]) {
1403 heap_t *list_heap = (heap_t *)atomic_load_ptr(&_memory_heaps[list_idx]);
1404 if (list_heap == heap) {
1405 atomic_store_ptr(&_memory_heaps[list_idx], heap->next_heap);
1407 while (list_heap->next_heap != heap)
1408 list_heap = list_heap->next_heap;
1409 list_heap->next_heap = heap->next_heap;
1412 _rpmalloc_heap_unmap(heap);
1417 _rpmalloc_heap_cache_insert(heap_t *heap, span_t *span) {
1419 _rpmalloc_span_unmap(span);
1420 _rpmalloc_heap_global_finalize(heap);
1423 #if ENABLE_THREAD_CACHE
1424 size_t span_count = span->span_count;
1425 size_t idx = span_count - 1;
1427 #if ENABLE_UNLIMITED_THREAD_CACHE
1428 _rpmalloc_span_list_push(&heap->span_cache[idx], span);
1430 const size_t release_count = (!idx ? _memory_span_release_count : _memory_span_release_count_large);
1431 size_t current_cache_size = _rpmalloc_span_list_push(&heap->span_cache[idx], span);
1432 if (current_cache_size <= release_count)
1435 if (current_cache_size <= hard_limit) {
1436 #if ENABLE_ADAPTIVE_THREAD_CACHE
1438 const size_t high_mark = heap->span_use[idx].high;
1439 const size_t min_limit = (high_mark >> 2) + release_count + 1;
1440 if (current_cache_size < min_limit)
1446 heap->span_cache[idx] = _rpmalloc_span_list_split(span, release_count);
1447 assert(span->list_size == release_count);
1448 #if ENABLE_GLOBAL_CACHE
1451 _rpmalloc_global_cache_insert_span_list(span);
1453 _rpmalloc_span_list_unmap_all(span);
1458 _rpmalloc_span_unmap(span);
1464 _rpmalloc_heap_thread_cache_extract(heap_t *heap,
size_t span_count) {
1466 size_t idx = span_count - 1;
1468 _rpmalloc_heap_cache_adopt_deferred(heap, &span);
1469 #if ENABLE_THREAD_CACHE
1470 if (!span && heap->span_cache[idx]) {
1472 span = _rpmalloc_span_list_pop(&heap->span_cache[idx]);
1479 _rpmalloc_heap_reserved_extract(heap_t *heap,
size_t span_count) {
1480 if (heap->spans_reserved >= span_count)
1481 return _rpmalloc_span_map(heap, span_count);
1487 _rpmalloc_heap_global_cache_extract(heap_t *heap,
size_t span_count) {
1488 #if ENABLE_GLOBAL_CACHE
1489 size_t idx = span_count - 1;
1490 heap->span_cache[idx] = _rpmalloc_global_cache_extract_span_list(span_count);
1491 if (heap->span_cache[idx]) {
1493 _rpmalloc_stat_add(&heap->span_use[idx].spans_from_global, heap->span_cache[idx]->list_size);
1494 return _rpmalloc_span_list_pop(&heap->span_cache[idx]);
1498 (void)
sizeof(span_count);
1504 _rpmalloc_heap_extract_new_span(heap_t *heap,
size_t span_count, uint32_t class_idx) {
1505 (void)
sizeof(class_idx);
1506 #if ENABLE_ADAPTIVE_THREAD_CACHE || ENABLE_STATISTICS
1507 uint32_t idx = (uint32_t)span_count - 1;
1508 uint32_t current_count = (uint32_t)atomic_incr32(&heap->span_use[idx].current);
1509 if (current_count > (uint32_t)atomic_load32(&heap->span_use[idx].high))
1510 atomic_store32(&heap->span_use[idx].high, (int32_t)current_count);
1511 _rpmalloc_stat_add_peak(&heap->size_class_use[class_idx].spans_current, 1, heap->size_class_use[class_idx].spans_peak);
1513 span_t *span = _rpmalloc_heap_thread_cache_extract(heap, span_count);
1518 span = _rpmalloc_heap_reserved_extract(heap, span_count);
1523 span = _rpmalloc_heap_global_cache_extract(heap, span_count);
1529 span = _rpmalloc_span_map(heap, span_count);
1535 _rpmalloc_heap_initialize(heap_t *heap) {
1536 memset(heap, 0,
sizeof(heap_t));
1539 heap->id = 1 + atomic_incr32(&_memory_heap_id);
1545 next_heap = (heap_t *)atomic_load_ptr(&_memory_heaps[list_idx]);
1546 heap->next_heap = next_heap;
1547 }
while (!atomic_cas_ptr(&_memory_heaps[list_idx], heap, next_heap));
1551 _rpmalloc_heap_orphan(heap_t *heap,
int first_class) {
1553 uintptr_t orphan_counter;
1555 heap->owner_thread = (uintptr_t) - 1;
1556 #if RPMALLOC_FIRST_CLASS_HEAPS
1557 atomicptr_t *heap_list = (first_class ? &_memory_first_class_orphan_heaps : &_memory_orphan_heaps);
1559 (void)
sizeof(first_class);
1560 atomicptr_t *heap_list = &_memory_orphan_heaps;
1563 last_heap = (heap_t *)atomic_load_ptr(heap_list);
1565 orphan_counter = (uintptr_t)atomic_incr32(&_memory_orphan_counter);
1566 raw_heap = (
void *)((uintptr_t)heap | (orphan_counter & (uintptr_t)(
HEAP_ORPHAN_ABA_SIZE - 1)));
1567 }
while (!atomic_cas_ptr(heap_list, raw_heap, last_heap));
1572 _rpmalloc_heap_allocate_new(
void) {
1574 size_t align_offset = 0;
1575 size_t heap_size =
sizeof(heap_t);
1576 size_t block_size = _memory_page_size * ((heap_size + _memory_page_size - 1) >> _memory_page_size_shift);
1577 heap_t *heap = (heap_t *)_rpmalloc_mmap(block_size, &align_offset);
1581 _rpmalloc_heap_initialize(heap);
1582 heap->align_offset = align_offset;
1586 size_t num_heaps = block_size / aligned_heap_size;
1587 atomic_store32(&heap->child_count, (int32_t)num_heaps - 1);
1588 heap_t *extra_heap = (heap_t *)
pointer_offset(heap, aligned_heap_size);
1589 while (num_heaps > 1) {
1590 _rpmalloc_heap_initialize(extra_heap);
1591 extra_heap->master_heap = heap;
1592 _rpmalloc_heap_orphan(extra_heap, 1);
1593 extra_heap = (heap_t *)
pointer_offset(extra_heap, aligned_heap_size);
1600 _rpmalloc_heap_extract_orphan(atomicptr_t *heap_list) {
1602 void *next_raw_heap;
1603 uintptr_t orphan_counter;
1607 raw_heap = atomic_load_ptr(heap_list);
1611 next_heap = heap->next_orphan;
1612 orphan_counter = (uintptr_t)atomic_incr32(&_memory_orphan_counter);
1613 next_raw_heap = (
void *)((uintptr_t)next_heap | (orphan_counter & (uintptr_t)(
HEAP_ORPHAN_ABA_SIZE - 1)));
1614 }
while (!atomic_cas_ptr(heap_list, next_raw_heap, raw_heap));
1620 _rpmalloc_heap_allocate(
int first_class) {
1622 if (first_class == 0)
1623 heap = _rpmalloc_heap_extract_orphan(&_memory_orphan_heaps);
1624 #if RPMALLOC_FIRST_CLASS_HEAPS
1626 heap = _rpmalloc_heap_extract_orphan(&_memory_first_class_orphan_heaps);
1629 heap = _rpmalloc_heap_allocate_new();
1634 _rpmalloc_heap_release(
void *heapptr,
int first_class) {
1635 heap_t *heap = (heap_t *)heapptr;
1639 _rpmalloc_heap_cache_adopt_deferred(heap, 0);
1640 #if ENABLE_THREAD_CACHE
1642 span_t *span = heap->span_cache[iclass];
1643 heap->span_cache[iclass] = 0;
1644 if (span && heap->finalize) {
1645 _rpmalloc_span_list_unmap_all(span);
1648 #if ENABLE_GLOBAL_CACHE
1650 assert(span->span_count == (iclass + 1));
1651 size_t release_count = (!iclass ? _memory_span_release_count : _memory_span_release_count_large);
1652 span_t *next = _rpmalloc_span_list_split(span, (uint32_t)release_count);
1655 _rpmalloc_global_cache_insert_span_list(span);
1660 _rpmalloc_span_list_unmap_all(span);
1666 _rpmalloc_heap_orphan(heap, first_class);
1668 if (get_thread_heap_raw() == heap)
1670 #if ENABLE_STATISTICS
1671 atomic_decr32(&_memory_active_heaps);
1672 assert(atomic_load32(&_memory_active_heaps) >= 0);
1677 _rpmalloc_heap_release_raw(
void *heapptr) {
1678 _rpmalloc_heap_release(heapptr, 0);
1682 _rpmalloc_heap_finalize(heap_t *heap) {
1683 if (heap->spans_reserved) {
1684 span_t *span = _rpmalloc_span_map(heap, heap->spans_reserved);
1685 _rpmalloc_span_unmap(span);
1686 heap->spans_reserved = 0;
1689 _rpmalloc_heap_cache_adopt_deferred(heap, 0);
1692 span_t *span = heap->partial_span[iclass];
1694 span_t *next = span->next;
1695 _rpmalloc_span_finalize(heap, iclass, span, &heap->partial_span[iclass]);
1699 if (heap->free_list[iclass]) {
1700 span_t *class_span = (span_t *)((uintptr_t)heap->free_list[iclass] &
_memory_span_mask);
1702 #if RPMALLOC_FIRST_CLASS_HEAPS
1703 list = &heap->full_span[iclass];
1705 --heap->full_span_count;
1706 if (!_rpmalloc_span_finalize(heap, iclass, class_span, list)) {
1708 _rpmalloc_span_double_link_list_remove(list, class_span);
1709 _rpmalloc_span_double_link_list_add(&heap->partial_span[iclass], class_span);
1714 #if ENABLE_THREAD_CACHE
1716 if (heap->span_cache[iclass]) {
1717 _rpmalloc_span_list_unmap_all(heap->span_cache[iclass]);
1718 heap->span_cache[iclass] = 0;
1722 assert(!atomic_load_ptr(&heap->span_free_deferred));
1734 free_list_pop(
void **list) {
1735 void *block = *list;
1736 *list = *((
void **)block);
1742 _rpmalloc_allocate_from_heap_fallback(heap_t *heap, uint32_t class_idx) {
1743 span_t *span = heap->partial_span[class_idx];
1745 assert(span->block_count == _memory_size_class[span->size_class].block_count);
1746 assert(!_rpmalloc_span_is_fully_utilized(span));
1748 if (span->free_list) {
1750 heap->free_list[class_idx] = span->free_list;
1751 span->free_list = 0;
1752 block = free_list_pop(&heap->free_list[class_idx]);
1756 span->free_list_limit += free_list_partial_init(&heap->free_list[class_idx], &block,
1757 (
void *)((uintptr_t)block_start & ~(_memory_page_size - 1)), block_start,
1758 span->block_count - span->free_list_limit, span->block_size);
1760 assert(span->free_list_limit <= span->block_count);
1761 span->used_count = span->free_list_limit;
1764 if (atomic_load_ptr(&span->free_list_deferred))
1765 _rpmalloc_span_extract_free_list_deferred(span);
1768 if (!_rpmalloc_span_is_fully_utilized(span))
1772 _rpmalloc_span_double_link_list_pop_head(&heap->partial_span[class_idx], span);
1773 #if RPMALLOC_FIRST_CLASS_HEAPS
1774 _rpmalloc_span_double_link_list_add(&heap->full_span[class_idx], span);
1776 ++heap->full_span_count;
1781 span = _rpmalloc_heap_extract_new_span(heap, 1, class_idx);
1784 return _rpmalloc_span_initialize_new(heap, span, class_idx);
1792 _rpmalloc_allocate_small(heap_t *heap,
size_t size) {
1797 if (
EXPECTED(heap->free_list[class_idx] != 0))
1798 return free_list_pop(&heap->free_list[class_idx]);
1799 return _rpmalloc_allocate_from_heap_fallback(heap, class_idx);
1804 _rpmalloc_allocate_medium(heap_t *heap,
size_t size) {
1808 const uint32_t class_idx = _memory_size_class[base_idx].class_idx;
1810 if (
EXPECTED(heap->free_list[class_idx] != 0))
1811 return free_list_pop(&heap->free_list[class_idx]);
1812 return _rpmalloc_allocate_from_heap_fallback(heap, class_idx);
1817 _rpmalloc_allocate_large(heap_t *heap,
size_t size) {
1828 span_t *span = _rpmalloc_heap_extract_new_span(heap, span_count,
SIZE_CLASS_LARGE);
1833 assert(span->span_count == span_count);
1837 #if RPMALLOC_FIRST_CLASS_HEAPS
1838 _rpmalloc_span_double_link_list_add(&heap->large_huge_span, span);
1840 ++heap->full_span_count;
1847 _rpmalloc_allocate_huge(heap_t *heap,
size_t size) {
1850 size_t num_pages = size >> _memory_page_size_shift;
1851 if (size & (_memory_page_size - 1))
1853 size_t align_offset = 0;
1854 span_t *span = (span_t *)_rpmalloc_mmap(num_pages * _memory_page_size, &align_offset);
1860 span->span_count = (uint32_t)num_pages;
1861 span->align_offset = (uint32_t)align_offset;
1865 #if RPMALLOC_FIRST_CLASS_HEAPS
1866 _rpmalloc_span_double_link_list_add(&heap->large_huge_span, span);
1868 ++heap->full_span_count;
1875 _rpmalloc_allocate(heap_t *heap,
size_t size) {
1877 return _rpmalloc_allocate_small(heap, size);
1878 else if (size <= _memory_medium_size_limit)
1879 return _rpmalloc_allocate_medium(heap, size);
1881 return _rpmalloc_allocate_large(heap, size);
1882 return _rpmalloc_allocate_huge(heap, size);
1886 _rpmalloc_aligned_allocate(heap_t *heap,
size_t alignment,
size_t size) {
1888 return _rpmalloc_allocate(heap, size);
1890 #if ENABLE_VALIDATE_ARGS
1891 if ((size + alignment) < size) {
1895 if (alignment & (alignment - 1)) {
1901 if ((alignment <=
SPAN_HEADER_SIZE) && (size < _memory_medium_size_limit)) {
1907 if (multiple_size <= (size + alignment))
1908 return _rpmalloc_allocate(heap, multiple_size);
1912 size_t align_mask = alignment - 1;
1913 if (alignment <= _memory_page_size) {
1914 ptr = _rpmalloc_allocate(heap, size + alignment);
1915 if ((uintptr_t)ptr & align_mask) {
1916 ptr = (
void *)(((uintptr_t)ptr & ~(uintptr_t)align_mask) + alignment);
1933 if (alignment & align_mask) {
1942 size_t extra_pages = alignment / _memory_page_size;
1945 size_t num_pages = 1 + (size / _memory_page_size);
1946 if (size & (_memory_page_size - 1))
1949 if (extra_pages > num_pages)
1950 num_pages = 1 + extra_pages;
1952 size_t original_pages = num_pages;
1954 if (limit_pages < (original_pages * 2))
1955 limit_pages = original_pages * 2;
1957 size_t mapped_size, align_offset;
1962 mapped_size = num_pages * _memory_page_size;
1964 span = (span_t *)_rpmalloc_mmap(mapped_size, &align_offset);
1971 if ((uintptr_t)ptr & align_mask)
1972 ptr = (
void *)(((uintptr_t)ptr & ~(uintptr_t)align_mask) + alignment);
1977 _rpmalloc_unmap(span, mapped_size, align_offset, mapped_size);
1979 if (num_pages > limit_pages) {
1988 span->span_count = (uint32_t)num_pages;
1989 span->align_offset = (uint32_t)align_offset;
1993 #if RPMALLOC_FIRST_CLASS_HEAPS
1994 _rpmalloc_span_double_link_list_add(&heap->large_huge_span, span);
1996 ++heap->full_span_count;
2010 _rpmalloc_deallocate_direct_small_or_medium(span_t *span,
void *block) {
2011 heap_t *heap = span->heap;
2012 assert(heap->owner_thread == get_thread_id() || heap->finalize);
2014 if (
UNEXPECTED(_rpmalloc_span_is_fully_utilized(span))) {
2015 span->used_count = span->block_count;
2016 #if RPMALLOC_FIRST_CLASS_HEAPS
2017 _rpmalloc_span_double_link_list_remove(&heap->full_span[span->size_class], span);
2019 _rpmalloc_span_double_link_list_add(&heap->partial_span[span->size_class], span);
2020 --heap->full_span_count;
2023 *((
void **)block) = span->free_list;
2024 span->free_list = block;
2025 if (
UNEXPECTED(span->used_count == span->list_size)) {
2026 _rpmalloc_span_double_link_list_remove(&heap->partial_span[span->size_class], span);
2027 _rpmalloc_span_release_to_cache(heap, span);
2032 _rpmalloc_deallocate_defer_free_span(heap_t *heap, span_t *span) {
2035 span->free_list = atomic_load_ptr(&heap->span_free_deferred);
2036 }
while (!atomic_cas_ptr(&heap->span_free_deferred, span, span->free_list));
2041 _rpmalloc_deallocate_defer_small_or_medium(span_t *span,
void *block) {
2048 free_list = atomic_load_ptr(&span->free_list_deferred);
2049 *((
void **)block) = free_list;
2051 uint32_t free_count = ++span->list_size;
2052 atomic_store_ptr_release(&span->free_list_deferred, block);
2053 if (free_count == span->block_count) {
2057 _rpmalloc_deallocate_defer_free_span(span->heap, span);
2062 _rpmalloc_deallocate_small_or_medium(span_t *span,
void *p) {
2067 uint32_t block_offset = (uint32_t)
pointer_diff(p, blocks_start);
2068 p =
pointer_offset(p, -(int32_t)(block_offset % span->block_size));
2071 if ((span->heap->owner_thread == get_thread_id()) || span->heap->finalize)
2072 _rpmalloc_deallocate_direct_small_or_medium(span, p);
2074 _rpmalloc_deallocate_defer_small_or_medium(span, p);
2079 _rpmalloc_deallocate_large(span_t *span) {
2084 int defer = (span->heap->owner_thread != get_thread_id()) && !span->heap->finalize;
2086 _rpmalloc_deallocate_defer_free_span(span->heap, span);
2089 assert(span->heap->full_span_count);
2090 --span->heap->full_span_count;
2091 #if RPMALLOC_FIRST_CLASS_HEAPS
2092 _rpmalloc_span_double_link_list_remove(&span->heap->large_huge_span, span);
2094 #if ENABLE_ADAPTIVE_THREAD_CACHE || ENABLE_STATISTICS
2096 size_t idx = span->span_count - 1;
2097 atomic_decr32(&span->heap->span_use[idx].current);
2099 heap_t *heap = get_thread_heap();
2102 if ((span->span_count > 1) && !heap->finalize && !heap->spans_reserved) {
2103 heap->span_reserve = span;
2104 heap->spans_reserved = span->span_count;
2106 heap->span_reserve_master = span;
2109 heap->span_reserve_master = master;
2111 assert(atomic_load32(&master->remaining_spans) >= (int32_t)span->span_count);
2116 _rpmalloc_heap_cache_insert(heap, span);
2122 _rpmalloc_deallocate_huge(span_t *span) {
2124 if ((span->heap->owner_thread != get_thread_id()) && !span->heap->finalize) {
2125 _rpmalloc_deallocate_defer_free_span(span->heap, span);
2128 assert(span->heap->full_span_count);
2129 --span->heap->full_span_count;
2130 #if RPMALLOC_FIRST_CLASS_HEAPS
2131 _rpmalloc_span_double_link_list_remove(&span->heap->large_huge_span, span);
2135 size_t num_pages = span->span_count;
2136 _rpmalloc_unmap(span, num_pages * _memory_page_size, span->align_offset, num_pages * _memory_page_size);
2142 _rpmalloc_deallocate(
void *p) {
2149 _rpmalloc_deallocate_small_or_medium(span, p);
2151 _rpmalloc_deallocate_large(span);
2153 _rpmalloc_deallocate_huge(span);
2164 _rpmalloc_usable_size(
void *p);
2168 _rpmalloc_reallocate(heap_t *heap,
void *p,
size_t size,
size_t oldsize,
unsigned int flags) {
2174 assert(span->span_count == 1);
2176 uint32_t block_offset = (uint32_t)
pointer_diff(p, blocks_start);
2177 uint32_t block_idx = block_offset / span->block_size;
2178 void *block =
pointer_offset(blocks_start, (
size_t)block_idx * span->block_size);
2180 oldsize = (size_t)((ptrdiff_t)span->block_size -
pointer_diff(p, block));
2181 if ((
size_t)span->block_size >= size) {
2184 memmove(block, p, oldsize);
2193 size_t current_spans = span->span_count;
2197 if ((current_spans >= num_spans) && (num_spans >= (current_spans / 2))) {
2200 memmove(block, p, oldsize);
2206 size_t num_pages = total_size >> _memory_page_size_shift;
2207 if (total_size & (_memory_page_size - 1))
2210 size_t current_pages = span->span_count;
2214 if ((current_pages >= num_pages) && (num_pages >= (current_pages / 2))) {
2217 memmove(block, p, oldsize);
2230 size_t lower_bound = oldsize + (oldsize >> 2) + (oldsize >> 3);
2231 size_t new_size = (size > lower_bound) ? size : ((size > oldsize) ? lower_bound : size);
2232 void *block = _rpmalloc_allocate(heap, new_size);
2235 memcpy(block, p, oldsize < new_size ? oldsize : new_size);
2236 _rpmalloc_deallocate(p);
2243 _rpmalloc_aligned_reallocate(heap_t *heap,
void *ptr,
size_t alignment,
size_t size,
size_t oldsize,
2244 unsigned int flags) {
2246 return _rpmalloc_reallocate(heap, ptr, size, oldsize, flags);
2249 size_t usablesize = _rpmalloc_usable_size(ptr);
2250 if ((usablesize >= size) && !((uintptr_t)ptr & (alignment - 1))) {
2251 if (no_alloc || (size >= (usablesize / 2)))
2255 void *block = (!no_alloc ? _rpmalloc_aligned_allocate(heap, alignment, size) : 0);
2259 oldsize = usablesize;
2260 memcpy(block, ptr, oldsize < size ? oldsize : size);
2262 _rpmalloc_deallocate(ptr);
2276 _rpmalloc_usable_size(
void *p) {
2282 fprintf(stderr,
"RPV %p, %p %p %d\n", span, blocks_start, p, span->block_size);
2283 return span->block_size - ((size_t)
pointer_diff(p, blocks_start) % span->block_size);
2287 size_t current_spans = span->span_count;
2291 size_t current_pages = span->span_count;
2292 return (current_pages * _memory_page_size) - (size_t)
pointer_diff(p, span);
2297 _rpmalloc_adjust_size_class(
size_t iclass) {
2298 size_t block_size = _memory_size_class[iclass].block_size;
2301 _memory_size_class[iclass].block_count = (uint16_t)block_count;
2302 _memory_size_class[iclass].class_idx = (uint16_t)iclass;
2305 size_t prevclass = iclass;
2306 while (prevclass > 0) {
2309 if (_memory_size_class[prevclass].block_count == _memory_size_class[iclass].block_count)
2310 memcpy(_memory_size_class + prevclass, _memory_size_class + iclass,
sizeof(_memory_size_class[iclass]));
2319 if (_rpmalloc_initialized) {
2328 if (_rpmalloc_initialized) {
2332 _rpmalloc_initialized = 1;
2340 _memory_config.
memory_map = _rpmalloc_mmap_os;
2344 #if RPMALLOC_CONFIGURABLE
2345 _memory_page_size = _memory_config.
page_size;
2347 _memory_page_size = 0;
2349 _memory_huge_pages = 0;
2350 _memory_map_granularity = _memory_page_size;
2351 if (!_memory_page_size) {
2352 #if PLATFORM_WINDOWS
2353 SYSTEM_INFO system_info;
2354 memset(&system_info, 0,
sizeof(system_info));
2355 GetSystemInfo(&system_info);
2356 _memory_page_size = system_info.dwPageSize;
2357 _memory_map_granularity = system_info.dwAllocationGranularity;
2359 _memory_page_size = (size_t)sysconf(_SC_PAGESIZE);
2360 _memory_map_granularity = _memory_page_size;
2362 #if defined(__linux__)
2363 size_t huge_page_size = 0;
2364 FILE *meminfo = fopen(
"/proc/meminfo",
"r");
2367 while (!huge_page_size && fgets(line,
sizeof(line) - 1, meminfo)) {
2368 line[
sizeof(line) - 1] = 0;
2369 if (strstr(line,
"Hugepagesize:"))
2370 huge_page_size = (
size_t)strtol(line + 13, 0, 10) * 1024;
2374 if (huge_page_size) {
2375 _memory_huge_pages = 1;
2376 _memory_page_size = huge_page_size;
2377 _memory_map_granularity = huge_page_size;
2379 #elif defined(__FreeBSD__)
2381 size_t sz =
sizeof(rc);
2383 if (sysctlbyname(
"vm.pmap.pg_ps_enabled", &rc, &sz, NULL, 0) == 0 && rc == 1) {
2384 _memory_huge_pages = 1;
2385 _memory_page_size = 2 * 1024 * 1024;
2386 _memory_map_granularity = _memory_page_size;
2388 #elif defined(__APPLE__)
2389 _memory_huge_pages = 1;
2390 _memory_page_size = 2 * 1024 * 1024;
2391 _memory_map_granularity = _memory_page_size;
2397 _memory_huge_pages = 1;
2400 #if PLATFORM_WINDOWS
2403 size_t large_page_minimum = GetLargePageMinimum();
2404 if (large_page_minimum)
2405 OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &token);
2408 if (LookupPrivilegeValue(0, SE_LOCK_MEMORY_NAME, &luid)) {
2409 TOKEN_PRIVILEGES token_privileges;
2410 memset(&token_privileges, 0,
sizeof(token_privileges));
2411 token_privileges.PrivilegeCount = 1;
2412 token_privileges.Privileges[0].Luid = luid;
2413 token_privileges.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
2414 if (AdjustTokenPrivileges(token,
FALSE, &token_privileges, 0, 0, 0)) {
2415 DWORD err = GetLastError();
2416 if (err == ERROR_SUCCESS) {
2417 _memory_huge_pages = 1;
2418 if (large_page_minimum > _memory_page_size)
2419 _memory_page_size = large_page_minimum;
2420 if (large_page_minimum > _memory_map_granularity)
2421 _memory_map_granularity = large_page_minimum;
2432 size_t max_page_size;
2433 #if UINTPTR_MAX > 0xFFFFFFFF
2434 max_page_size = 4096ULL * 1024ULL * 1024ULL;
2436 max_page_size = 4 * 1024 * 1024;
2438 if (_memory_page_size < min_span_size)
2439 _memory_page_size = min_span_size;
2440 if (_memory_page_size > max_page_size)
2441 _memory_page_size = max_page_size;
2442 _memory_page_size_shift = 0;
2443 size_t page_size_bit = _memory_page_size;
2444 while (page_size_bit != 1) {
2445 ++_memory_page_size_shift;
2446 page_size_bit >>= 1;
2448 _memory_page_size = ((size_t)1 << _memory_page_size_shift);
2450 #if RPMALLOC_CONFIGURABLE
2451 size_t span_size = _memory_config.
span_size;
2453 span_size = (64 * 1024);
2454 if (span_size > (256 * 1024))
2455 span_size = (256 * 1024);
2471 _memory_config.
page_size = _memory_page_size;
2476 _memory_span_release_count = (_memory_span_map_count > 4 ? ((_memory_span_map_count < 64) ? _memory_span_map_count : 64) : 4);
2477 _memory_span_release_count_large = (_memory_span_release_count > 8 ? (_memory_span_release_count / 4) : 2);
2479 #if (defined(__APPLE__) || defined(__HAIKU__)) && ENABLE_PRELOAD
2480 if (pthread_key_create(&_memory_thread_heap, _memory_heap_release_raw))
2483 #if defined(_WIN32) && (!defined(BUILD_DYNAMIC_LINK) || !BUILD_DYNAMIC_LINK)
2484 fls_key = FlsAlloc(&_rpmalloc_thread_destructor);
2490 _rpmalloc_adjust_size_class(iclass);
2493 _memory_size_class[iclass].block_size = (uint32_t)size;
2494 _rpmalloc_adjust_size_class(iclass);
2502 if (size > _memory_medium_size_limit)
2508 atomic_store_ptr(&_memory_orphan_heaps, 0);
2509 #if RPMALLOC_FIRST_CLASS_HEAPS
2510 atomic_store_ptr(&_memory_first_class_orphan_heaps, 0);
2512 for (
size_t ilist = 0, lsize = (
sizeof(_memory_heaps) /
sizeof(_memory_heaps[0])); ilist < lsize; ++ilist)
2513 atomic_store_ptr(&_memory_heaps[ilist], 0);
2528 heap_t *heap = (heap_t *)atomic_load_ptr(&_memory_heaps[list_idx]);
2530 heap_t *next_heap = heap->next_heap;
2532 _rpmalloc_heap_global_finalize(heap);
2537 #if ENABLE_GLOBAL_CACHE
2540 _rpmalloc_global_cache_finalize(&_memory_span_cache[iclass]);
2543 #if (defined(__APPLE__) || defined(__HAIKU__)) && ENABLE_PRELOAD
2544 pthread_key_delete(_memory_thread_heap);
2546 #if defined(_WIN32) && (!defined(BUILD_DYNAMIC_LINK) || !BUILD_DYNAMIC_LINK)
2550 #if ENABLE_STATISTICS
2552 assert(!atomic_load32(&_mapped_pages));
2553 assert(!atomic_load32(&_reserved_spans));
2554 assert(!atomic_load32(&_mapped_pages_os));
2557 _rpmalloc_initialized = 0;
2563 if (!get_thread_heap_raw()) {
2564 heap_t *heap = _rpmalloc_heap_allocate(0);
2567 set_thread_heap(heap);
2568 #if defined(_WIN32) && (!defined(BUILD_DYNAMIC_LINK) || !BUILD_DYNAMIC_LINK)
2569 FlsSetValue(fls_key, heap);
2578 heap_t *heap = get_thread_heap_raw();
2580 _rpmalloc_heap_release_raw(heap);
2582 #if defined(_WIN32) && (!defined(BUILD_DYNAMIC_LINK) || !BUILD_DYNAMIC_LINK)
2583 FlsSetValue(fls_key, 0);
2589 return (get_thread_heap_raw() != 0) ? 1 : 0;
2594 return &_memory_config;
2601 #if ENABLE_VALIDATE_ARGS
2602 if (size >= MAX_ALLOC_SIZE) {
2607 heap_t *heap = get_thread_heap();
2608 return _rpmalloc_allocate(heap, size);
2613 _rpmalloc_deallocate(ptr);
2619 #if ENABLE_VALIDATE_ARGS
2620 #if PLATFORM_WINDOWS
2621 int err = SizeTMult(num, size, &total);
2622 if ((err != S_OK) || (total >= MAX_ALLOC_SIZE)) {
2627 int err = __builtin_umull_overflow(num, size, &total);
2628 if (err || (total >= MAX_ALLOC_SIZE)) {
2636 heap_t *heap = get_thread_heap();
2637 void *block = _rpmalloc_allocate(heap, total);
2639 memset(block, 0, total);
2645 #if ENABLE_VALIDATE_ARGS
2646 if (size >= MAX_ALLOC_SIZE) {
2651 heap_t *heap = get_thread_heap();
2652 return _rpmalloc_reallocate(heap, ptr, size, 0, 0);
2657 unsigned int flags) {
2658 #if ENABLE_VALIDATE_ARGS
2659 if ((size + alignment < size) || (alignment > _memory_page_size)) {
2664 heap_t *heap = get_thread_heap();
2665 return _rpmalloc_aligned_reallocate(heap, ptr, alignment, size, oldsize, flags);
2670 heap_t *heap = get_thread_heap();
2671 return _rpmalloc_aligned_allocate(heap, alignment, size);
2677 #if ENABLE_VALIDATE_ARGS
2678 #if PLATFORM_WINDOWS
2679 int err = SizeTMult(num, size, &total);
2680 if ((err != S_OK) || (total >= MAX_ALLOC_SIZE)) {
2685 int err = __builtin_umull_overflow(num, size, &total);
2686 if (err || (total >= MAX_ALLOC_SIZE)) {
2696 memset(block, 0, total);
2711 return *memptr ? 0 : ENOMEM;
2714 extern inline size_t
2716 return (ptr ? _rpmalloc_usable_size(ptr) : 0);
2726 heap_t *heap = get_thread_heap_raw();
2731 size_class_t *size_class = _memory_size_class + iclass;
2732 span_t *span = heap->partial_span[iclass];
2734 size_t free_count = span->list_size;
2735 size_t block_count = size_class->block_count;
2736 if (span->free_list_limit < block_count)
2737 block_count = span->free_list_limit;
2738 free_count += (block_count - span->used_count);
2739 stats->
sizecache = free_count * size_class->block_size;
2744 #if ENABLE_THREAD_CACHE
2746 if (heap->span_cache[iclass])
2751 span_t *deferred = (span_t *)atomic_load_ptr(&heap->span_free_deferred);
2755 deferred = (span_t *)deferred->free_list;
2758 #if ENABLE_STATISTICS
2763 stats->
span_use[iclass].
current = (size_t)atomic_load32(&heap->span_use[iclass].current);
2764 stats->
span_use[iclass].
peak = (size_t)atomic_load32(&heap->span_use[iclass].high);
2765 stats->
span_use[iclass].
to_global = (size_t)atomic_load32(&heap->span_use[iclass].spans_to_global);
2766 stats->
span_use[iclass].
from_global = (size_t)atomic_load32(&heap->span_use[iclass].spans_from_global);
2767 stats->
span_use[iclass].
to_cache = (size_t)atomic_load32(&heap->span_use[iclass].spans_to_cache);
2768 stats->
span_use[iclass].
from_cache = (size_t)atomic_load32(&heap->span_use[iclass].spans_from_cache);
2769 stats->
span_use[iclass].
to_reserved = (size_t)atomic_load32(&heap->span_use[iclass].spans_to_reserved);
2770 stats->
span_use[iclass].
from_reserved = (size_t)atomic_load32(&heap->span_use[iclass].spans_from_reserved);
2771 stats->
span_use[iclass].
map_calls = (size_t)atomic_load32(&heap->span_use[iclass].spans_map_calls);
2774 stats->
size_use[iclass].
alloc_current = (size_t)atomic_load32(&heap->size_class_use[iclass].alloc_current);
2775 stats->
size_use[iclass].
alloc_peak = (size_t)heap->size_class_use[iclass].alloc_peak;
2776 stats->
size_use[iclass].
alloc_total = (
size_t)atomic_load32(&heap->size_class_use[iclass].alloc_total);
2777 stats->
size_use[iclass].
free_total = (size_t)atomic_load32(&heap->size_class_use[iclass].free_total);
2778 stats->
size_use[iclass].
spans_to_cache = (size_t)atomic_load32(&heap->size_class_use[iclass].spans_to_cache);
2781 stats->
size_use[iclass].
map_calls = (size_t)atomic_load32(&heap->size_class_use[iclass].spans_map_calls);
2789 #if ENABLE_STATISTICS
2790 stats->
mapped = (size_t)atomic_load32(&_mapped_pages) * _memory_page_size;
2791 stats->
mapped_peak = (size_t)_mapped_pages_peak * _memory_page_size;
2792 stats->
mapped_total = (size_t)atomic_load32(&_mapped_total) * _memory_page_size;
2793 stats->
unmapped_total = (size_t)atomic_load32(&_unmapped_total) * _memory_page_size;
2794 stats->
huge_alloc = (size_t)atomic_load32(&_huge_pages_current) * _memory_page_size;
2795 stats->
huge_alloc_peak = (size_t)_huge_pages_peak * _memory_page_size;
2797 #if ENABLE_GLOBAL_CACHE
2804 #if ENABLE_STATISTICS
2807 _memory_heap_dump_statistics(heap_t *heap,
void *file) {
2808 fprintf(file,
"Heap %d stats:\n", heap->id);
2810 "Class CurAlloc PeakAlloc TotAlloc TotFree BlkSize BlkCount SpansCur SpansPeak PeakAllocMiB ToCacheMiB FromCacheMiB FromReserveMiB MmapCalls\n");
2812 if (!atomic_load32(&heap->size_class_use[iclass].alloc_total))
2814 fprintf(file,
"%3u: %10u %10u %10u %10u %8u %8u %8d %9d %13zu %11zu %12zu %14zu %9u\n", (uint32_t)iclass,
2815 atomic_load32(&heap->size_class_use[iclass].alloc_current),
2816 heap->size_class_use[iclass].alloc_peak,
2817 atomic_load32(&heap->size_class_use[iclass].alloc_total),
2818 atomic_load32(&heap->size_class_use[iclass].free_total),
2819 _memory_size_class[iclass].block_size,
2820 _memory_size_class[iclass].block_count,
2821 atomic_load32(&heap->size_class_use[iclass].spans_current),
2822 heap->size_class_use[iclass].spans_peak,
2823 ((
size_t)heap->size_class_use[iclass].alloc_peak * (
size_t)_memory_size_class[iclass].block_size) / (
size_t)(1024 * 1024),
2824 ((
size_t)atomic_load32(&heap->size_class_use[iclass].spans_to_cache) *
_memory_span_size) / (
size_t)(1024 * 1024),
2825 ((
size_t)atomic_load32(&heap->size_class_use[iclass].spans_from_cache) *
_memory_span_size) / (
size_t)(1024 * 1024),
2826 ((
size_t)atomic_load32(&heap->size_class_use[iclass].spans_from_reserved) *
_memory_span_size) / (
size_t)(1024 * 1024),
2827 atomic_load32(&heap->size_class_use[iclass].spans_map_calls));
2830 "Spans Current Peak PeakMiB Cached ToCacheMiB FromCacheMiB ToReserveMiB FromReserveMiB ToGlobalMiB FromGlobalMiB MmapCalls\n");
2832 if (!atomic_load32(&heap->span_use[iclass].high) && !atomic_load32(&heap->span_use[iclass].spans_map_calls))
2834 fprintf(file,
"%4u: %8d %8u %8zu %7u %11zu %12zu %12zu %14zu %11zu %13zu %10u\n", (uint32_t)(iclass + 1),
2835 atomic_load32(&heap->span_use[iclass].current),
2836 atomic_load32(&heap->span_use[iclass].high),
2837 ((
size_t)atomic_load32(&heap->span_use[iclass].high) * (
size_t)
_memory_span_size * (iclass + 1)) / (
size_t)(1024 * 1024),
2839 heap->span_cache[iclass] ? heap->span_cache[iclass]->list_size : 0,
2840 ((
size_t)atomic_load32(&heap->span_use[iclass].spans_to_cache) * (iclass + 1) *
_memory_span_size) / (
size_t)(1024 * 1024),
2841 ((
size_t)atomic_load32(&heap->span_use[iclass].spans_from_cache) * (iclass + 1) *
_memory_span_size) / (
size_t)(1024 * 1024),
2845 ((
size_t)atomic_load32(&heap->span_use[iclass].spans_to_reserved) * (iclass + 1) *
_memory_span_size) / (
size_t)(1024 * 1024),
2846 ((
size_t)atomic_load32(&heap->span_use[iclass].spans_from_reserved) * (iclass + 1) *
_memory_span_size) / (
size_t)(1024 * 1024),
2847 ((
size_t)atomic_load32(&heap->span_use[iclass].spans_to_global) * (
size_t)
_memory_span_size * (iclass + 1)) / (
size_t)(
2849 ((
size_t)atomic_load32(&heap->span_use[iclass].spans_from_global) * (
size_t)
_memory_span_size * (iclass + 1)) / (
size_t)(
2851 atomic_load32(&heap->span_use[iclass].spans_map_calls));
2853 fprintf(file,
"ThreadToGlobalMiB GlobalToThreadMiB\n");
2854 fprintf(file,
"%17zu %17zu\n", (
size_t)atomic_load64(&heap->thread_to_global) / (
size_t)(1024 * 1024),
2855 (
size_t)atomic_load64(&heap->global_to_thread) / (
size_t)(1024 * 1024));
2862 #if ENABLE_STATISTICS
2864 assert(atomic_load32(&_memory_active_heaps) == 0);
2866 heap_t *heap = atomic_load_ptr(&_memory_heaps[list_idx]);
2869 for (
size_t iclass = 0; !need_dump && (iclass <
SIZE_CLASS_COUNT); ++iclass) {
2870 if (!atomic_load32(&heap->size_class_use[iclass].alloc_total)) {
2871 assert(!atomic_load32(&heap->size_class_use[iclass].free_total));
2872 assert(!atomic_load32(&heap->size_class_use[iclass].spans_map_calls));
2878 if (!atomic_load32(&heap->span_use[iclass].high) && !atomic_load32(&heap->span_use[iclass].spans_map_calls))
2883 _memory_heap_dump_statistics(heap, file);
2884 heap = heap->next_heap;
2887 fprintf(file,
"Global stats:\n");
2888 size_t huge_current = (size_t)atomic_load32(&_huge_pages_current) * _memory_page_size;
2889 size_t huge_peak = (size_t)_huge_pages_peak * _memory_page_size;
2890 fprintf(file,
"HugeCurrentMiB HugePeakMiB\n");
2891 fprintf(file,
"%14zu %11zu\n", huge_current / (
size_t)(1024 * 1024), huge_peak / (
size_t)(1024 * 1024));
2893 size_t mapped = (size_t)atomic_load32(&_mapped_pages) * _memory_page_size;
2894 size_t mapped_os = (size_t)atomic_load32(&_mapped_pages_os) * _memory_page_size;
2895 size_t mapped_peak = (size_t)_mapped_pages_peak * _memory_page_size;
2896 size_t mapped_total = (size_t)atomic_load32(&_mapped_total) * _memory_page_size;
2897 size_t unmapped_total = (size_t)atomic_load32(&_unmapped_total) * _memory_page_size;
2898 size_t reserved_total = (size_t)atomic_load32(&_reserved_spans) *
_memory_span_size;
2899 fprintf(file,
"MappedMiB MappedOSMiB MappedPeakMiB MappedTotalMiB UnmappedTotalMiB ReservedTotalMiB\n");
2900 fprintf(file,
"%9zu %11zu %13zu %14zu %16zu %16zu\n",
2901 mapped / (
size_t)(1024 * 1024),
2902 mapped_os / (
size_t)(1024 * 1024),
2903 mapped_peak / (
size_t)(1024 * 1024),
2904 mapped_total / (
size_t)(1024 * 1024),
2905 unmapped_total / (
size_t)(1024 * 1024),
2906 reserved_total / (
size_t)(1024 * 1024));
2908 fprintf(file,
"\n");
2914 #if RPMALLOC_FIRST_CLASS_HEAPS
2916 extern inline rpmalloc_heap_t *
2917 rpmalloc_heap_acquire(
void) {
2921 heap_t *heap = _rpmalloc_heap_allocate(1);
2927 rpmalloc_heap_release(rpmalloc_heap_t *heap) {
2929 _rpmalloc_heap_release(heap, 1);
2933 rpmalloc_heap_alloc(rpmalloc_heap_t *heap,
size_t size) {
2934 #if ENABLE_VALIDATE_ARGS
2935 if (size >= MAX_ALLOC_SIZE) {
2940 return _rpmalloc_allocate(heap, size);
2944 rpmalloc_heap_aligned_alloc(rpmalloc_heap_t *heap,
size_t alignment,
size_t size) {
2945 #if ENABLE_VALIDATE_ARGS
2946 if (size >= MAX_ALLOC_SIZE) {
2951 return _rpmalloc_aligned_allocate(heap, alignment, size);
2955 rpmalloc_heap_calloc(rpmalloc_heap_t *heap,
size_t num,
size_t size) {
2956 return rpmalloc_heap_aligned_calloc(heap, 0, num, size);
2960 rpmalloc_heap_aligned_calloc(rpmalloc_heap_t *heap,
size_t alignment,
size_t num,
size_t size) {
2962 #if ENABLE_VALIDATE_ARGS
2963 #if PLATFORM_WINDOWS
2964 int err = SizeTMult(num, size, &total);
2965 if ((err != S_OK) || (total >= MAX_ALLOC_SIZE)) {
2970 int err = __builtin_umull_overflow(num, size, &total);
2971 if (err || (total >= MAX_ALLOC_SIZE)) {
2979 void *block = _rpmalloc_aligned_allocate(heap, alignment, total);
2981 memset(block, 0, total);
2986 rpmalloc_heap_realloc(rpmalloc_heap_t *heap,
void *ptr,
size_t size,
unsigned int flags) {
2987 #if ENABLE_VALIDATE_ARGS
2988 if (size >= MAX_ALLOC_SIZE) {
2993 return _rpmalloc_reallocate(heap, ptr, size, 0, flags);
2997 rpmalloc_heap_aligned_realloc(rpmalloc_heap_t *heap,
void *ptr,
size_t alignment,
size_t size,
unsigned int flags) {
2998 #if ENABLE_VALIDATE_ARGS
2999 if ((size + alignment < size) || (alignment > _memory_page_size)) {
3004 return _rpmalloc_aligned_reallocate(heap, ptr, alignment, size, 0, flags);
3008 rpmalloc_heap_free(rpmalloc_heap_t *heap,
void *ptr) {
3010 _rpmalloc_deallocate(ptr);
3014 rpmalloc_heap_free_all(rpmalloc_heap_t *heap) {
3018 _rpmalloc_heap_cache_adopt_deferred(heap, 0);
3021 span = heap->partial_span[iclass];
3023 next_span = span->next;
3024 _rpmalloc_heap_cache_insert(heap, span);
3027 heap->partial_span[iclass] = 0;
3028 span = heap->full_span[iclass];
3030 next_span = span->next;
3031 _rpmalloc_heap_cache_insert(heap, span);
3035 memset(heap->free_list, 0,
sizeof(heap->free_list));
3036 memset(heap->partial_span, 0,
sizeof(heap->partial_span));
3037 memset(heap->full_span, 0,
sizeof(heap->full_span));
3039 span = heap->large_huge_span;
3041 next_span = span->next;
3043 _rpmalloc_deallocate_huge(span);
3045 _rpmalloc_heap_cache_insert(heap, span);
3048 heap->large_huge_span = 0;
3049 heap->full_span_count = 0;
3051 #if ENABLE_THREAD_CACHE
3053 span = heap->span_cache[iclass];
3054 #if ENABLE_GLOBAL_CACHE
3056 assert(span->span_count == (iclass + 1));
3057 size_t release_count = (!iclass ? _memory_span_release_count : _memory_span_release_count_large);
3058 next_span = _rpmalloc_span_list_split(span, (uint32_t)release_count);
3061 _rpmalloc_global_cache_insert_span_list(span);
3066 _rpmalloc_span_list_unmap_all(span);
3068 heap->span_cache[iclass] = 0;
3072 #if ENABLE_STATISTICS
3074 atomic_store32(&heap->size_class_use[iclass].alloc_current, 0);
3075 atomic_store32(&heap->size_class_use[iclass].spans_current, 0);
3078 atomic_store32(&heap->span_use[iclass].current, 0);
3084 rpmalloc_heap_thread_set_current(rpmalloc_heap_t *heap) {
3085 heap_t *prev_heap = get_thread_heap_raw();
3086 if (prev_heap != heap) {
3087 set_thread_heap(heap);
3089 rpmalloc_heap_release(prev_heap);
3095 #if ENABLE_PRELOAD || ENABLE_OVERRIDE