diff --git a/dep/PackageList.txt b/dep/PackageList.txt index 68b568164..cb7fcd144 100644 --- a/dep/PackageList.txt +++ b/dep/PackageList.txt @@ -18,7 +18,7 @@ G3D (a commercial-grade C++ 3D engine available as Open Source (BSD License) jemalloc (a general-purpose scalable concurrent malloc-implementation) https://github.com/jemalloc/jemalloc - Version: 5.3.0 + Version: 5.3.1 libreadline (command line editing library) https://cnswww.cns.cwru.edu/php/chet/readline/rltop.html diff --git a/dep/jemalloc/CMakeLists.txt b/dep/jemalloc/CMakeLists.txt index 21e5c5661..a5bb4a259 100644 --- a/dep/jemalloc/CMakeLists.txt +++ b/dep/jemalloc/CMakeLists.txt @@ -42,17 +42,18 @@ else() set(CPU_SPINWAIT "_mm_pause()") else() set(CPU_SPINWAIT "__asm__ volatile\(\"pause\"\)") + set(JEMALLOC_HAVE_RDTSCP 1) endif() endif() # git describe --long --abbrev=40 -set(JEMALLOC_VERSION "5.3.0-0-g54eaed1d8b56b1aa528be3bdd1877e59c56fa90c") +set(JEMALLOC_VERSION "5.3.1-12-gee4d7b7f9a5c1b48d462be2b65c16e4ead8e3396") set(JEMALLOC_VERSION_MAJOR 5) set(JEMALLOC_VERSION_MINOR 3) -set(JEMALLOC_VERSION_BUGFIX 0) -set(JEMALLOC_VERSION_NREV 0) -set(JEMALLOC_VERSION_GID "54eaed1d8b56b1aa528be3bdd1877e59c56fa90c") -set(JEMALLOC_VERSION_GID_IDENT 54eaed1d8b56b1aa528be3bdd1877e59c56fa90c) +set(JEMALLOC_VERSION_BUGFIX 1) +set(JEMALLOC_VERSION_NREV 12) +set(JEMALLOC_VERSION_GID "ee4d7b7f9a5c1b48d462be2b65c16e4ead8e3396") +set(JEMALLOC_VERSION_GID_IDENT ee4d7b7f9a5c1b48d462be2b65c16e4ead8e3396) set(JEMALLOC_MAPS_COALESCE 1) set(JEMALLOC_TLS 1) @@ -67,6 +68,9 @@ if(APPLE) set(JEMALLOC_ZONE 1) unset(JEMALLOC_TLS) check_symbol_exists(mach_absolute_time "mach/mach_time.h" JEMALLOC_HAVE_MACH_ABSOLUTE_TIME) + if(LG_SIZEOF_PTR STREQUAL "3") + set(JEMALLOC_RETAIN 1) + endif() elseif(CMAKE_SYSTEM_NAME STREQUAL "OpenBSD") set(JEMALLOC_DSS 1) unset(JEMALLOC_TLS) @@ -114,6 +118,7 @@ if(NOT MSVC) set(JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1) # don't bother detecting atomic support, all TC-supported compilers (excluding msvc) have gcc-like atomics set(JEMALLOC_HAVE_ATTR 1) set(JEMALLOC_TLS_MODEL "__attribute__\(\(tls_model\(\"initial-exec\"\)\)\)") + set(JEMALLOC_HAVE_ATTR_DEPRECATED 1) set(JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF 1) set(JEMALLOC_HAVE_ATTR_FORMAT_PRINTF 1) set(JEMALLOC_HAVE_ATTR_FORMAT_ARG 1) @@ -146,11 +151,15 @@ set(JEMALLOC_CONFIG_MALLOC_CONF "") set(JEMALLOC_C11_ATOMICS 1) set(JEMALLOC_ENABLE_CXX 1) set(JEMALLOC_FILL 1) +set(JEMALLOC_HAVE_CXX_EXCEPTIONS 1) set(JEMALLOC_PRIVATE_NAMESPACE "je_") set(JEMALLOC_STATS 1) +unset(JEMALLOC_CONFIG_ENV) +unset(JEMALLOC_CONFIG_FILE) unset(JEMALLOC_LOG) unset(JEMALLOC_OPT_SAFETY_CHECKS) unset(JEMALLOC_OPT_SIZE_CHECKS) +unset(JEMALLOC_PAGEID) unset(JEMALLOC_READLINKAT) unset(JEMALLOC_UAF_DETECTION) unset(JEMALLOC_UTRACE) @@ -161,6 +170,7 @@ set(CMAKE_REQUIRED_DEFINITIONS ${JEMALLOC_EXTRA_DEFINES}) check_symbol_exists(memalign "malloc.h" JEMALLOC_OVERRIDE_MEMALIGN) check_symbol_exists(valloc "stdlib.h" JEMALLOC_OVERRIDE_VALLOC) +check_symbol_exists(pvalloc "malloc.h" JEMALLOC_OVERRIDE_PVALLOC) check_symbol_exists(malloc_size "malloc/malloc.h" JEMALLOC_HAVE_MALLOC_SIZE) set(wrap_syms "") @@ -173,6 +183,14 @@ if(NOT JEMALLOC_PREFIX) if(JEMALLOC_OVERRIDE___LIBC_FREE) list(APPEND wrap_syms "__libc_free") endif() + check_function_exists(__libc_free_sized JEMALLOC_OVERRIDE___LIBC_FREE_SIZED) + if(JEMALLOC_OVERRIDE___LIBC_FREE_SIZED) + list(APPEND wrap_syms "__libc_free_sized") + endif() + check_function_exists(__libc_free_aligned_sized JEMALLOC_OVERRIDE___LIBC_FREE_ALIGNED_SIZED) + if(JEMALLOC_OVERRIDE___LIBC_FREE_ALIGNED_SIZED) + list(APPEND wrap_syms "__libc_free_aligned_sized") + endif() check_function_exists(__libc_malloc JEMALLOC_OVERRIDE___LIBC_MALLOC) if(JEMALLOC_OVERRIDE___LIBC_MALLOC) list(APPEND wrap_syms "__libc_malloc") @@ -189,6 +207,10 @@ if(NOT JEMALLOC_PREFIX) if(JEMALLOC_OVERRIDE___LIBC_VALLOC) list(APPEND wrap_syms "__libc_valloc") endif() + check_function_exists(__libc_pvalloc JEMALLOC_OVERRIDE___LIBC_PVALLOC) + if(JEMALLOC_OVERRIDE___LIBC_PVALLOC) + list(APPEND wrap_syms "__libc_pvalloc") + endif() check_function_exists(__posix_memalign JEMALLOC_OVERRIDE___POSIX_MEMALIGN) if(JEMALLOC_OVERRIDE___POSIX_MEMALIGN) list(APPEND wrap_syms "__posix_memalign") @@ -203,7 +225,8 @@ if(NOT WIN32) check_symbol_exists(pthread_atfork "pthread.h" JEMALLOC_HAVE_PTHREAD_ATFORK) check_symbol_exists(pthread_setname_np "pthread.h" JEMALLOC_HAVE_PTHREAD_SETNAME_NP) check_symbol_exists(pthread_getname_np "pthread.h" JEMALLOC_HAVE_PTHREAD_GETNAME_NP) - check_symbol_exists(pthread_get_name_np "pthread.h" JEMALLOC_HAVE_PTHREAD_GET_NAME_NP) + check_symbol_exists(pthread_set_name_np "pthread_np.h" JEMALLOC_HAVE_PTHREAD_SET_NAME_NP) + check_symbol_exists(pthread_get_name_np "pthread_np.h" JEMALLOC_HAVE_PTHREAD_GET_NAME_NP) check_c_source_compiles(" #include int main() @@ -214,9 +237,11 @@ if(NOT WIN32) pthread_mutexattr_destroy(&attr); } " JEMALLOC_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP) + check_symbol_exists(gettid "unistd.h" JEMALLOC_HAVE_GETTID) check_symbol_exists(CLOCK_MONOTONIC_COARSE "time.h" JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE) check_symbol_exists(CLOCK_MONOTONIC "time.h" JEMALLOC_HAVE_CLOCK_MONOTONIC) check_symbol_exists(CLOCK_REALTIME "time.h" JEMALLOC_HAVE_CLOCK_REALTIME) + check_symbol_exists(clock_gettime_nsec_np "time.h" JEMALLOC_HAVE_CLOCK_GETTIME_NSEC_NP) set(CMAKE_REQUIRED_FLAGS "-Werror") check_c_source_compiles(" @@ -229,6 +254,7 @@ if(NOT WIN32) check_symbol_exists(secure_getenv "stdlib.h" JEMALLOC_HAVE_SECURE_GETENV) check_symbol_exists(sched_getcpu "sched.h" JEMALLOC_HAVE_SCHED_GETCPU) check_symbol_exists(sched_setaffinity "sched.h" JEMALLOC_HAVE_SCHED_SETAFFINITY) + check_symbol_exists(pthread_setaffinity_np "pthread.h" JEMALLOC_HAVE_PTHREAD_SETAFFINITY_NP) check_symbol_exists(issetugid "unistd.h" JEMALLOC_HAVE_ISSETUGID) check_symbol_exists(_malloc_thread_cleanup "pthread.h" JEMALLOC_MALLOC_THREAD_CLEANUP) if(JEMALLOC_MALLOC_THREAD_CLEANUP) @@ -244,6 +270,7 @@ if(NOT WIN32) endif() check_symbol_exists(memcntl "sys/types.h;sys/mman.h" JEMALLOC_HAVE_MEMCNTL) + check_symbol_exists(prctl "sys/prctl.h" JEMALLOC_HAVE_PRCTL) check_symbol_exists(madvise "sys/mman.h" JEMALLOC_HAVE_MADVISE) if(JEMALLOC_HAVE_MADVISE) check_symbol_exists(MADV_FREE "sys/mman.h" JEMALLOC_PURGE_MADVISE_FREE) @@ -254,9 +281,11 @@ if(NOT WIN32) check_symbol_exists(MADV_DONTNEED "sys/mman.h" JEMALLOC_PURGE_MADVISE_DONTNEED) check_symbol_exists(MADV_DONTDUMP "sys/mman.h" JEMALLOC_MADVISE_DONTDUMP) check_symbol_exists(MADV_NOCORE "sys/mman.h" JEMALLOC_MADVISE_NOCORE) + check_symbol_exists(MADV_COLLAPSE "sys/mman.h" JEMALLOC_HAVE_MADVISE_COLLAPSE) if(TRINITY_SYSTEM_PROCESSOR MATCHES "x86|amd64") check_symbol_exists(MADV_HUGEPAGE "sys/mman.h" JEMALLOC_HAVE_MADVISE_HUGE) endif() + check_symbol_exists(SYS_process_madvise "sys/syscall.h" JEMALLOC_HAVE_PROCESS_MADVISE) else() check_symbol_exists(posix_madvise "sys/mman.h" JEMALLOC_HAVE_POSIX_MADVISE) if(JEMALLOC_HAVE_POSIX_MADVISE) @@ -283,6 +312,13 @@ if(NOT WIN32) } " JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE) unset(CMAKE_REQUIRED_FLAGS) + check_c_source_compiles(" + int main() + { + __int128 temp = 0; + return temp; + } + " JEMALLOC_HAVE_INT128) endif() unset(CMAKE_REQUIRED_DEFINITIONS) @@ -312,6 +348,7 @@ add_library(jemalloc STATIC ${CMAKE_CURRENT_SOURCE_DIR}/src/buf_writer.c ${CMAKE_CURRENT_SOURCE_DIR}/src/cache_bin.c ${CMAKE_CURRENT_SOURCE_DIR}/src/ckh.c + ${CMAKE_CURRENT_SOURCE_DIR}/src/conf.c ${CMAKE_CURRENT_SOURCE_DIR}/src/counter.c ${CMAKE_CURRENT_SOURCE_DIR}/src/ctl.c ${CMAKE_CURRENT_SOURCE_DIR}/src/decay.c @@ -331,7 +368,9 @@ add_library(jemalloc STATIC ${CMAKE_CURRENT_SOURCE_DIR}/src/san_bump.c ${CMAKE_CURRENT_SOURCE_DIR}/src/hook.c ${CMAKE_CURRENT_SOURCE_DIR}/src/hpa.c + ${CMAKE_CURRENT_SOURCE_DIR}/src/hpa_central.c ${CMAKE_CURRENT_SOURCE_DIR}/src/hpa_hooks.c + ${CMAKE_CURRENT_SOURCE_DIR}/src/hpa_utils.c ${CMAKE_CURRENT_SOURCE_DIR}/src/hpdata.c ${CMAKE_CURRENT_SOURCE_DIR}/src/inspect.c ${CMAKE_CURRENT_SOURCE_DIR}/src/large.c @@ -341,7 +380,6 @@ add_library(jemalloc STATIC ${CMAKE_CURRENT_SOURCE_DIR}/src/nstime.c ${CMAKE_CURRENT_SOURCE_DIR}/src/pa.c ${CMAKE_CURRENT_SOURCE_DIR}/src/pa_extra.c - ${CMAKE_CURRENT_SOURCE_DIR}/src/pai.c ${CMAKE_CURRENT_SOURCE_DIR}/src/pac.c ${CMAKE_CURRENT_SOURCE_DIR}/src/pages.c ${CMAKE_CURRENT_SOURCE_DIR}/src/peak_event.c @@ -349,6 +387,7 @@ add_library(jemalloc STATIC ${CMAKE_CURRENT_SOURCE_DIR}/src/prof_data.c ${CMAKE_CURRENT_SOURCE_DIR}/src/prof_log.c ${CMAKE_CURRENT_SOURCE_DIR}/src/prof_recent.c + ${CMAKE_CURRENT_SOURCE_DIR}/src/prof_stack_range.c ${CMAKE_CURRENT_SOURCE_DIR}/src/prof_stats.c ${CMAKE_CURRENT_SOURCE_DIR}/src/prof_sys.c ${CMAKE_CURRENT_SOURCE_DIR}/src/psset.c @@ -361,8 +400,10 @@ add_library(jemalloc STATIC ${CMAKE_CURRENT_SOURCE_DIR}/src/tcache.c ${CMAKE_CURRENT_SOURCE_DIR}/src/test_hooks.c ${CMAKE_CURRENT_SOURCE_DIR}/src/thread_event.c + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread_event_registry.c ${CMAKE_CURRENT_SOURCE_DIR}/src/ticker.c ${CMAKE_CURRENT_SOURCE_DIR}/src/tsd.c + ${CMAKE_CURRENT_SOURCE_DIR}/src/util.c ${CMAKE_CURRENT_SOURCE_DIR}/src/witness.c) if(APPLE) diff --git a/dep/jemalloc/include/jemalloc/internal/activity_callback.h b/dep/jemalloc/include/jemalloc/internal/activity_callback.h deleted file mode 100644 index 6c2e84e31..000000000 --- a/dep/jemalloc/include/jemalloc/internal/activity_callback.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_ACTIVITY_CALLBACK_H -#define JEMALLOC_INTERNAL_ACTIVITY_CALLBACK_H - -/* - * The callback to be executed "periodically", in response to some amount of - * allocator activity. - * - * This callback need not be computing any sort of peak (although that's the - * intended first use case), but we drive it from the peak counter, so it's - * keeps things tidy to keep it here. - * - * The calls to this thunk get driven by the peak_event module. - */ -#define ACTIVITY_CALLBACK_THUNK_INITIALIZER {NULL, NULL} -typedef void (*activity_callback_t)(void *uctx, uint64_t allocated, - uint64_t deallocated); -typedef struct activity_callback_thunk_s activity_callback_thunk_t; -struct activity_callback_thunk_s { - activity_callback_t callback; - void *uctx; -}; - -#endif /* JEMALLOC_INTERNAL_ACTIVITY_CALLBACK_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/arena_externs.h b/dep/jemalloc/include/jemalloc/internal/arena_externs.h index e6fceaafe..39794b3e3 100644 --- a/dep/jemalloc/include/jemalloc/internal/arena_externs.h +++ b/dep/jemalloc/include/jemalloc/internal/arena_externs.h @@ -1,8 +1,11 @@ #ifndef JEMALLOC_INTERNAL_ARENA_EXTERNS_H #define JEMALLOC_INTERNAL_ARENA_EXTERNS_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/arena_stats.h" #include "jemalloc/internal/bin.h" #include "jemalloc/internal/div.h" +#include "jemalloc/internal/emap.h" #include "jemalloc/internal/extent_dss.h" #include "jemalloc/internal/hook.h" #include "jemalloc/internal/pages.h" @@ -18,104 +21,105 @@ extern ssize_t opt_dirty_decay_ms; extern ssize_t opt_muzzy_decay_ms; extern percpu_arena_mode_t opt_percpu_arena; -extern const char *percpu_arena_mode_names[]; +extern const char *const percpu_arena_mode_names[]; extern div_info_t arena_binind_div_info[SC_NBINS]; -extern malloc_mutex_t arenas_lock; extern emap_t arena_emap_global; extern size_t opt_oversize_threshold; extern size_t oversize_threshold; +extern bool opt_huge_arena_pac_thp; +extern pac_thp_t huge_arena_pac_thp; + /* * arena_bin_offsets[binind] is the offset of the first bin shard for size class * binind. */ extern uint32_t arena_bin_offsets[SC_NBINS]; -void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, - unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, - ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy); +void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, + const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, + size_t *nactive, size_t *ndirty, size_t *nmuzzy); void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats, - bin_stats_data_t *bstats, arena_stats_large_t *lstats, - pac_estats_t *estats, hpa_shard_stats_t *hpastats, sec_stats_t *secstats); + bin_stats_data_t *bstats, arena_stats_large_t *lstats, pac_estats_t *estats, + hpa_shard_stats_t *hpastats); void arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena); -edata_t *arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, - size_t usize, size_t alignment, bool zero); -void arena_extent_dalloc_large_prep(tsdn_t *tsdn, arena_t *arena, - edata_t *edata); -void arena_extent_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, - edata_t *edata, size_t oldsize); -void arena_extent_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena, - edata_t *edata, size_t oldsize); -bool arena_decay_ms_set(tsdn_t *tsdn, arena_t *arena, extent_state_t state, - ssize_t decay_ms); +edata_t *arena_extent_alloc_large( + tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero); +void arena_extent_dalloc_large_prep( + tsdn_t *tsdn, arena_t *arena, edata_t *edata); +void arena_extent_ralloc_large_shrink( + tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t oldusize); +void arena_extent_ralloc_large_expand( + tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t oldusize); +bool arena_decay_ms_set( + tsdn_t *tsdn, arena_t *arena, extent_state_t state, ssize_t decay_ms); ssize_t arena_decay_ms_get(arena_t *arena, extent_state_t state); -void arena_decay(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, - bool all); -uint64_t arena_time_until_deferred(tsdn_t *tsdn, arena_t *arena); -void arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena); -void arena_reset(tsd_t *tsd, arena_t *arena); -void arena_destroy(tsd_t *tsd, arena_t *arena); -void arena_cache_bin_fill_small(tsdn_t *tsdn, arena_t *arena, - cache_bin_t *cache_bin, cache_bin_info_t *cache_bin_info, szind_t binind, - const unsigned nfill); +void arena_decay( + tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all); +uint64_t arena_time_until_deferred(tsdn_t *tsdn, arena_t *arena); +void arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena); +void arena_reset(tsd_t *tsd, arena_t *arena); +void arena_destroy(tsd_t *tsd, arena_t *arena); +cache_bin_sz_t arena_ptr_array_fill_small(tsdn_t *tsdn, arena_t *arena, + szind_t binind, cache_bin_ptr_array_t *arr, const cache_bin_sz_t nfill_min, + const cache_bin_sz_t nfill_max, cache_bin_stats_t merge_stats); -void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, - szind_t ind, bool zero); -void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, - size_t alignment, bool zero, tcache_t *tcache); -void arena_prof_promote(tsdn_t *tsdn, void *ptr, size_t usize); -void arena_dalloc_promoted(tsdn_t *tsdn, void *ptr, tcache_t *tcache, - bool slow_path); +void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, + bool zero, bool slab); +void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, + bool zero, bool slab, tcache_t *tcache); +void arena_prof_promote( + tsdn_t *tsdn, void *ptr, size_t usize, size_t bumped_usize); +void arena_dalloc_promoted( + tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path); void arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *slab); -void arena_dalloc_bin_locked_handle_newly_empty(tsdn_t *tsdn, arena_t *arena, - edata_t *slab, bin_t *bin); -void arena_dalloc_bin_locked_handle_newly_nonempty(tsdn_t *tsdn, arena_t *arena, - edata_t *slab, bin_t *bin); -void arena_dalloc_small(tsdn_t *tsdn, void *ptr); -bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, - size_t extra, bool zero, size_t *newsize); +void arena_dalloc_small(tsdn_t *tsdn, void *ptr); +void arena_ptr_array_flush(tsd_t *tsd, szind_t binind, + cache_bin_ptr_array_t *arr, unsigned nflush, bool small, + arena_t *stats_arena, cache_bin_stats_t merge_stats); +bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, + size_t extra, bool zero, size_t *newsize); void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, - size_t size, size_t alignment, bool zero, tcache_t *tcache, + size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache, hook_ralloc_args_t *hook_args); -dss_prec_t arena_dss_prec_get(arena_t *arena); -ehooks_t *arena_get_ehooks(arena_t *arena); -extent_hooks_t *arena_set_extent_hooks(tsd_t *tsd, arena_t *arena, - extent_hooks_t *extent_hooks); -bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); +dss_prec_t arena_dss_prec_get(arena_t *arena); +ehooks_t *arena_get_ehooks(arena_t *arena); +extent_hooks_t *arena_set_extent_hooks( + tsd_t *tsd, arena_t *arena, extent_hooks_t *extent_hooks); +bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); +void arena_name_get(arena_t *arena, char *name); +void arena_name_set(arena_t *arena, const char *name); ssize_t arena_dirty_decay_ms_default_get(void); -bool arena_dirty_decay_ms_default_set(ssize_t decay_ms); +bool arena_dirty_decay_ms_default_set(ssize_t decay_ms); ssize_t arena_muzzy_decay_ms_default_get(void); -bool arena_muzzy_decay_ms_default_set(ssize_t decay_ms); -bool arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena, - size_t *old_limit, size_t *new_limit); +bool arena_muzzy_decay_ms_default_set(ssize_t decay_ms); +bool arena_retain_grow_limit_get_set( + tsd_t *tsd, arena_t *arena, size_t *old_limit, size_t *new_limit); unsigned arena_nthreads_get(arena_t *arena, bool internal); -void arena_nthreads_inc(arena_t *arena, bool internal); -void arena_nthreads_dec(arena_t *arena, bool internal); +void arena_nthreads_inc(arena_t *arena, bool internal); +void arena_nthreads_dec(arena_t *arena, bool internal); arena_t *arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config); -bool arena_init_huge(void); -bool arena_is_huge(unsigned arena_ind); +bool arena_init_huge(tsdn_t *tsdn, arena_t *a0); arena_t *arena_choose_huge(tsd_t *tsd); -bin_t *arena_bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind, - unsigned *binshard); size_t arena_fill_small_fresh(tsdn_t *tsdn, arena_t *arena, szind_t binind, void **ptrs, size_t nfill, bool zero); -bool arena_boot(sc_data_t *sc_data, base_t *base, bool hpa); -void arena_prefork0(tsdn_t *tsdn, arena_t *arena); -void arena_prefork1(tsdn_t *tsdn, arena_t *arena); -void arena_prefork2(tsdn_t *tsdn, arena_t *arena); -void arena_prefork3(tsdn_t *tsdn, arena_t *arena); -void arena_prefork4(tsdn_t *tsdn, arena_t *arena); -void arena_prefork5(tsdn_t *tsdn, arena_t *arena); -void arena_prefork6(tsdn_t *tsdn, arena_t *arena); -void arena_prefork7(tsdn_t *tsdn, arena_t *arena); -void arena_prefork8(tsdn_t *tsdn, arena_t *arena); -void arena_postfork_parent(tsdn_t *tsdn, arena_t *arena); -void arena_postfork_child(tsdn_t *tsdn, arena_t *arena); +bool arena_boot(sc_data_t *sc_data, base_t *base, bool hpa); +void arena_prefork0(tsdn_t *tsdn, arena_t *arena); +void arena_prefork1(tsdn_t *tsdn, arena_t *arena); +void arena_prefork2(tsdn_t *tsdn, arena_t *arena); +void arena_prefork3(tsdn_t *tsdn, arena_t *arena); +void arena_prefork4(tsdn_t *tsdn, arena_t *arena); +void arena_prefork5(tsdn_t *tsdn, arena_t *arena); +void arena_prefork6(tsdn_t *tsdn, arena_t *arena); +void arena_prefork7(tsdn_t *tsdn, arena_t *arena); +void arena_prefork8(tsdn_t *tsdn, arena_t *arena); +void arena_postfork_parent(tsdn_t *tsdn, arena_t *arena); +void arena_postfork_child(tsdn_t *tsdn, arena_t *arena); #endif /* JEMALLOC_INTERNAL_ARENA_EXTERNS_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/arena_inlines_a.h b/dep/jemalloc/include/jemalloc/internal/arena_inlines_a.h index 8568358c7..214ce80b2 100644 --- a/dep/jemalloc/include/jemalloc/internal/arena_inlines_a.h +++ b/dep/jemalloc/include/jemalloc/internal/arena_inlines_a.h @@ -1,6 +1,9 @@ #ifndef JEMALLOC_INTERNAL_ARENA_INLINES_A_H #define JEMALLOC_INTERNAL_ARENA_INLINES_A_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/arena_structs.h" + static inline unsigned arena_ind_get(const arena_t *arena) { return arena->ind; diff --git a/dep/jemalloc/include/jemalloc/internal/arena_inlines_b.h b/dep/jemalloc/include/jemalloc/internal/arena_inlines_b.h index fa81537c4..bda256b99 100644 --- a/dep/jemalloc/include/jemalloc/internal/arena_inlines_b.h +++ b/dep/jemalloc/include/jemalloc/internal/arena_inlines_b.h @@ -1,20 +1,29 @@ #ifndef JEMALLOC_INTERNAL_ARENA_INLINES_B_H #define JEMALLOC_INTERNAL_ARENA_INLINES_B_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/arena_externs.h" +#include "jemalloc/internal/arena_structs.h" +#include "jemalloc/internal/bin_inlines.h" #include "jemalloc/internal/div.h" #include "jemalloc/internal/emap.h" +#include "jemalloc/internal/jemalloc_internal_inlines_b.h" #include "jemalloc/internal/jemalloc_internal_types.h" +#include "jemalloc/internal/large_externs.h" #include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/prof_externs.h" +#include "jemalloc/internal/prof_structs.h" #include "jemalloc/internal/rtree.h" #include "jemalloc/internal/safety_check.h" #include "jemalloc/internal/sc.h" #include "jemalloc/internal/sz.h" +#include "jemalloc/internal/tcache_inlines.h" #include "jemalloc/internal/ticker.h" static inline arena_t * arena_get_from_edata(edata_t *edata) { - return (arena_t *)atomic_load_p(&arenas[edata_arena_ind_get(edata)], - ATOMIC_RELAXED); + return (arena_t *)atomic_load_p( + &arenas[edata_arena_ind_get(edata)], ATOMIC_RELAXED); } JEMALLOC_ALWAYS_INLINE arena_t * @@ -28,14 +37,48 @@ arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) { * 1) is using auto arena selection (i.e. arena == NULL), and 2) the * thread is not assigned to a manual arena. */ - if (unlikely(size >= oversize_threshold)) { - arena_t *tsd_arena = tsd_arena_get(tsd); - if (tsd_arena == NULL || arena_is_auto(tsd_arena)) { - return arena_choose_huge(tsd); - } + arena_t *tsd_arena = tsd_arena_get(tsd); + if (tsd_arena == NULL) { + tsd_arena = arena_choose(tsd, NULL); } - return arena_choose(tsd, NULL); + size_t threshold = atomic_load_zu( + &tsd_arena->pa_shard.pac.oversize_threshold, ATOMIC_RELAXED); + if (unlikely(size >= threshold) && arena_is_auto(tsd_arena)) { + return arena_choose_huge(tsd); + } + + return tsd_arena; +} + +JEMALLOC_ALWAYS_INLINE bool +large_dalloc_safety_checks(edata_t *edata, const void *ptr, size_t input_size) { + if (!config_opt_safety_checks) { + return false; + } + + /* + * Eagerly detect double free and sized dealloc bugs for large sizes. + * The cost is low enough (as edata will be accessed anyway) to be + * enabled all the time. + */ + if (unlikely(edata == NULL + || edata_state_get(edata) != extent_state_active)) { + safety_check_fail( + "Invalid deallocation detected: " + "pages being freed (%p) not currently active, " + "possibly caused by double free bugs.", + ptr); + return true; + } + if (unlikely(input_size != edata_usize_get(edata) + || input_size > SC_LARGE_MAXCLASS)) { + safety_check_fail_sized_dealloc(/* current_dealloc */ true, ptr, + /* true_size */ edata_usize_get(edata), input_size); + return true; + } + + return false; } JEMALLOC_ALWAYS_INLINE void @@ -46,48 +89,56 @@ arena_prof_info_get(tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx, assert(prof_info != NULL); edata_t *edata = NULL; - bool is_slab; + bool is_slab; /* Static check. */ if (alloc_ctx == NULL) { - edata = emap_edata_lookup(tsd_tsdn(tsd), &arena_emap_global, - ptr); + edata = emap_edata_lookup( + tsd_tsdn(tsd), &arena_emap_global, ptr); is_slab = edata_slab_get(edata); } else if (unlikely(!(is_slab = alloc_ctx->slab))) { - edata = emap_edata_lookup(tsd_tsdn(tsd), &arena_emap_global, - ptr); + edata = emap_edata_lookup( + tsd_tsdn(tsd), &arena_emap_global, ptr); } if (unlikely(!is_slab)) { /* edata must have been initialized at this point. */ assert(edata != NULL); + size_t usize = (alloc_ctx == NULL) + ? edata_usize_get(edata) + : emap_alloc_ctx_usize_get(alloc_ctx); + if (reset_recent + && large_dalloc_safety_checks(edata, ptr, usize)) { + prof_info->alloc_tctx = PROF_TCTX_SENTINEL; + return; + } large_prof_info_get(tsd, edata, prof_info, reset_recent); } else { - prof_info->alloc_tctx = (prof_tctx_t *)(uintptr_t)1U; + prof_info->alloc_tctx = PROF_TCTX_SENTINEL; /* * No need to set other fields in prof_info; they will never be - * accessed if (uintptr_t)alloc_tctx == (uintptr_t)1U. + * accessed if alloc_tctx == PROF_TCTX_SENTINEL. */ } } JEMALLOC_ALWAYS_INLINE void -arena_prof_tctx_reset(tsd_t *tsd, const void *ptr, - emap_alloc_ctx_t *alloc_ctx) { +arena_prof_tctx_reset( + tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx) { cassert(config_prof); assert(ptr != NULL); /* Static check. */ if (alloc_ctx == NULL) { - edata_t *edata = emap_edata_lookup(tsd_tsdn(tsd), - &arena_emap_global, ptr); + edata_t *edata = emap_edata_lookup( + tsd_tsdn(tsd), &arena_emap_global, ptr); if (unlikely(!edata_slab_get(edata))) { large_prof_tctx_reset(edata); } } else { if (unlikely(!alloc_ctx->slab)) { - edata_t *edata = emap_edata_lookup(tsd_tsdn(tsd), - &arena_emap_global, ptr); + edata_t *edata = emap_edata_lookup( + tsd_tsdn(tsd), &arena_emap_global, ptr); large_prof_tctx_reset(edata); } } @@ -98,16 +149,16 @@ arena_prof_tctx_reset_sampled(tsd_t *tsd, const void *ptr) { cassert(config_prof); assert(ptr != NULL); - edata_t *edata = emap_edata_lookup(tsd_tsdn(tsd), &arena_emap_global, - ptr); + edata_t *edata = emap_edata_lookup( + tsd_tsdn(tsd), &arena_emap_global, ptr); assert(!edata_slab_get(edata)); large_prof_tctx_reset(edata); } JEMALLOC_ALWAYS_INLINE void -arena_prof_info_set(tsd_t *tsd, edata_t *edata, prof_tctx_t *tctx, - size_t size) { +arena_prof_info_set( + tsd_t *tsd, edata_t *edata, prof_tctx_t *tctx, size_t size) { cassert(config_prof); assert(!edata_slab_get(edata)); @@ -130,8 +181,9 @@ arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) { * use a single ticker for all of them. */ ticker_geom_t *decay_ticker = tsd_arena_decay_tickerp_get(tsd); - uint64_t *prng_state = tsd_prng_statep_get(tsd); - if (unlikely(ticker_geom_ticks(decay_ticker, prng_state, nticks))) { + uint64_t *prng_state = tsd_prng_statep_get(tsd); + if (unlikely(ticker_geom_ticks(decay_ticker, prng_state, nticks, + tsd_reentrancy_level_get(tsd) > 0))) { arena_decay(tsdn, arena, false, false); } } @@ -143,23 +195,24 @@ arena_decay_tick(tsdn_t *tsdn, arena_t *arena) { JEMALLOC_ALWAYS_INLINE void * arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero, - tcache_t *tcache, bool slow_path) { + bool slab, tcache_t *tcache, bool slow_path) { assert(!tsdn_null(tsdn) || tcache == NULL); if (likely(tcache != NULL)) { - if (likely(size <= SC_SMALL_MAXCLASS)) { - return tcache_alloc_small(tsdn_tsd(tsdn), arena, - tcache, size, ind, zero, slow_path); + if (likely(slab)) { + assert(sz_can_use_slab(size)); + return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache, + size, ind, zero, slow_path); + } else if (likely(ind < tcache_nbins_get(tcache->tcache_slow) + && !tcache_bin_disabled(ind, &tcache->bins[ind], + tcache->tcache_slow))) { + return tcache_alloc_large(tsdn_tsd(tsdn), arena, tcache, + size, ind, zero, slow_path); } - if (likely(size <= tcache_maxclass)) { - return tcache_alloc_large(tsdn_tsd(tsdn), arena, - tcache, size, ind, zero, slow_path); - } - /* (size > tcache_maxclass) case falls through. */ - assert(size > tcache_maxclass); + /* (size > tcache_max) case falls through. */ } - return arena_malloc_hard(tsdn, arena, size, ind, zero); + return arena_malloc_hard(tsdn, arena, size, ind, zero, slab); } JEMALLOC_ALWAYS_INLINE arena_t * @@ -176,7 +229,7 @@ arena_salloc(tsdn_t *tsdn, const void *ptr) { emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx); assert(alloc_ctx.szind != SC_NSIZES); - return sz_index2size(alloc_ctx.szind); + return emap_alloc_ctx_usize_get(&alloc_ctx); } JEMALLOC_ALWAYS_INLINE size_t @@ -191,8 +244,8 @@ arena_vsalloc(tsdn_t *tsdn, const void *ptr) { */ emap_full_alloc_ctx_t full_alloc_ctx; - bool missing = emap_full_alloc_ctx_try_lookup(tsdn, &arena_emap_global, - ptr, &full_alloc_ctx); + bool missing = emap_full_alloc_ctx_try_lookup( + tsdn, &arena_emap_global, ptr, &full_alloc_ctx); if (missing) { return 0; } @@ -207,46 +260,24 @@ arena_vsalloc(tsdn_t *tsdn, const void *ptr) { assert(full_alloc_ctx.szind != SC_NSIZES); - return sz_index2size(full_alloc_ctx.szind); -} - -JEMALLOC_ALWAYS_INLINE bool -large_dalloc_safety_checks(edata_t *edata, void *ptr, szind_t szind) { - if (!config_opt_safety_checks) { - return false; - } - - /* - * Eagerly detect double free and sized dealloc bugs for large sizes. - * The cost is low enough (as edata will be accessed anyway) to be - * enabled all the time. - */ - if (unlikely(edata == NULL || - edata_state_get(edata) != extent_state_active)) { - safety_check_fail("Invalid deallocation detected: " - "pages being freed (%p) not currently active, " - "possibly caused by double free bugs.", - (uintptr_t)edata_addr_get(edata)); - return true; - } - size_t input_size = sz_index2size(szind); - if (unlikely(input_size != edata_usize_get(edata))) { - safety_check_fail_sized_dealloc(/* current_dealloc */ true, ptr, - /* true_size */ edata_usize_get(edata), input_size); - return true; - } - - return false; + return edata_usize_get(full_alloc_ctx.edata); } static inline void -arena_dalloc_large_no_tcache(tsdn_t *tsdn, void *ptr, szind_t szind) { +arena_dalloc_large_no_tcache( + tsdn_t *tsdn, void *ptr, szind_t szind, size_t usize) { + /* + * szind is still needed in this function mainly becuase + * szind < SC_NBINS determines not only if this is a small alloc, + * but also if szind is valid (an inactive extent would have + * szind == SC_NSIZES). + */ if (config_prof && unlikely(szind < SC_NBINS)) { arena_dalloc_promoted(tsdn, ptr, NULL, true); } else { - edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, - ptr); - if (large_dalloc_safety_checks(edata, ptr, szind)) { + edata_t *edata = emap_edata_lookup( + tsdn, &arena_emap_global, ptr); + if (large_dalloc_safety_checks(edata, ptr, usize)) { /* See the comment in isfree. */ return; } @@ -262,42 +293,76 @@ arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) { emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx); if (config_debug) { - edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, - ptr); + edata_t *edata = emap_edata_lookup( + tsdn, &arena_emap_global, ptr); assert(alloc_ctx.szind == edata_szind_get(edata)); assert(alloc_ctx.szind < SC_NSIZES); assert(alloc_ctx.slab == edata_slab_get(edata)); + assert(emap_alloc_ctx_usize_get(&alloc_ctx) + == edata_usize_get(edata)); } if (likely(alloc_ctx.slab)) { /* Small allocation. */ arena_dalloc_small(tsdn, ptr); } else { - arena_dalloc_large_no_tcache(tsdn, ptr, alloc_ctx.szind); + arena_dalloc_large_no_tcache(tsdn, ptr, alloc_ctx.szind, + emap_alloc_ctx_usize_get(&alloc_ctx)); } } JEMALLOC_ALWAYS_INLINE void arena_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache, szind_t szind, - bool slow_path) { - if (szind < nhbins) { - if (config_prof && unlikely(szind < SC_NBINS)) { - arena_dalloc_promoted(tsdn, ptr, tcache, slow_path); - } else { - tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, szind, - slow_path); - } + size_t usize, bool slow_path) { + assert(!tsdn_null(tsdn) && tcache != NULL); + bool is_sample_promoted = config_prof && szind < SC_NBINS; + if (unlikely(is_sample_promoted)) { + arena_dalloc_promoted(tsdn, ptr, tcache, slow_path); } else { - edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, - ptr); - if (large_dalloc_safety_checks(edata, ptr, szind)) { - /* See the comment in isfree. */ - return; + if (szind < tcache_nbins_get(tcache->tcache_slow) + && !tcache_bin_disabled( + szind, &tcache->bins[szind], tcache->tcache_slow)) { + tcache_dalloc_large( + tsdn_tsd(tsdn), tcache, ptr, szind, slow_path); + } else { + edata_t *edata = emap_edata_lookup( + tsdn, &arena_emap_global, ptr); + if (large_dalloc_safety_checks(edata, ptr, usize)) { + /* See the comment in isfree. */ + return; + } + large_dalloc(tsdn, edata); } - large_dalloc(tsdn, edata); } } +JEMALLOC_ALWAYS_INLINE bool +arena_tcache_dalloc_small_safety_check(tsdn_t *tsdn, void *ptr) { + if (!config_debug) { + return false; + } + edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr); + szind_t binind = edata_szind_get(edata); + div_info_t div_info = arena_binind_div_info[binind]; + /* + * Calls the internal function bin_slab_regind_impl because the + * safety check does not require a lock. + */ + size_t regind = bin_slab_regind_impl(&div_info, binind, edata, ptr); + slab_data_t *slab_data = edata_slab_data_get(edata); + const bin_info_t *bin_info = &bin_infos[binind]; + assert(edata_nfree_get(edata) < bin_info->nregs); + if (unlikely(!bitmap_get( + slab_data->bitmap, &bin_info->bitmap_info, regind))) { + safety_check_fail( + "Invalid deallocation detected: the pointer being freed (%p) not " + "currently active, possibly caused by double free bugs.\n", + ptr); + return true; + } + return false; +} + JEMALLOC_ALWAYS_INLINE void arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) { @@ -313,26 +378,31 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, if (caller_alloc_ctx != NULL) { alloc_ctx = *caller_alloc_ctx; } else { - util_assume(!tsdn_null(tsdn)); - emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, - &alloc_ctx); + util_assume(tsdn != NULL); + emap_alloc_ctx_lookup( + tsdn, &arena_emap_global, ptr, &alloc_ctx); } if (config_debug) { - edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, - ptr); + edata_t *edata = emap_edata_lookup( + tsdn, &arena_emap_global, ptr); assert(alloc_ctx.szind == edata_szind_get(edata)); assert(alloc_ctx.szind < SC_NSIZES); assert(alloc_ctx.slab == edata_slab_get(edata)); + assert(emap_alloc_ctx_usize_get(&alloc_ctx) + == edata_usize_get(edata)); } if (likely(alloc_ctx.slab)) { /* Small allocation. */ - tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, - alloc_ctx.szind, slow_path); + if (arena_tcache_dalloc_small_safety_check(tsdn, ptr)) { + return; + } + tcache_dalloc_small( + tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path); } else { arena_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind, - slow_path); + emap_alloc_ctx_usize_get(&alloc_ctx), slow_path); } } @@ -347,21 +417,22 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) { * There is no risk of being confused by a promoted sampled * object, so base szind and slab on the given size. */ - alloc_ctx.szind = sz_size2index(size); - alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS); + szind_t szind = sz_size2index(size); + emap_alloc_ctx_init( + &alloc_ctx, szind, (szind < SC_NBINS), size); } if ((config_prof && opt_prof) || config_debug) { - emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, - &alloc_ctx); + emap_alloc_ctx_lookup( + tsdn, &arena_emap_global, ptr, &alloc_ctx); assert(alloc_ctx.szind == sz_size2index(size)); assert((config_prof && opt_prof) || alloc_ctx.slab == (alloc_ctx.szind < SC_NBINS)); if (config_debug) { - edata_t *edata = emap_edata_lookup(tsdn, - &arena_emap_global, ptr); + edata_t *edata = emap_edata_lookup( + tsdn, &arena_emap_global, ptr); assert(alloc_ctx.szind == edata_szind_get(edata)); assert(alloc_ctx.slab == edata_slab_get(edata)); } @@ -371,7 +442,8 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) { /* Small allocation. */ arena_dalloc_small(tsdn, ptr); } else { - arena_dalloc_large_no_tcache(tsdn, ptr, alloc_ctx.szind); + arena_dalloc_large_no_tcache(tsdn, ptr, alloc_ctx.szind, + emap_alloc_ctx_usize_get(&alloc_ctx)); } } @@ -391,9 +463,10 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, if (config_prof && opt_prof) { if (caller_alloc_ctx == NULL) { /* Uncommon case and should be a static check. */ - emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, - &alloc_ctx); + emap_alloc_ctx_lookup( + tsdn, &arena_emap_global, ptr, &alloc_ctx); assert(alloc_ctx.szind == sz_size2index(size)); + assert(emap_alloc_ctx_usize_get(&alloc_ctx) == size); } else { alloc_ctx = *caller_alloc_ctx; } @@ -407,30 +480,37 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, } if (config_debug) { - edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, - ptr); + edata_t *edata = emap_edata_lookup( + tsdn, &arena_emap_global, ptr); assert(alloc_ctx.szind == edata_szind_get(edata)); assert(alloc_ctx.slab == edata_slab_get(edata)); + emap_alloc_ctx_init( + &alloc_ctx, alloc_ctx.szind, alloc_ctx.slab, sz_s2u(size)); + assert(emap_alloc_ctx_usize_get(&alloc_ctx) + == edata_usize_get(edata)); } if (likely(alloc_ctx.slab)) { /* Small allocation. */ - tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, - alloc_ctx.szind, slow_path); + if (arena_tcache_dalloc_small_safety_check(tsdn, ptr)) { + return; + } + tcache_dalloc_small( + tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path); } else { arena_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind, - slow_path); + sz_s2u(size), slow_path); } } static inline void -arena_cache_oblivious_randomize(tsdn_t *tsdn, arena_t *arena, edata_t *edata, - size_t alignment) { +arena_cache_oblivious_randomize( + tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t alignment) { assert(edata_base_get(edata) == edata_addr_get(edata)); if (alignment < PAGE) { - unsigned lg_range = LG_PAGE - - lg_floor(CACHELINE_CEILING(alignment)); + unsigned lg_range = LG_PAGE + - lg_floor(CACHELINE_CEILING(alignment)); size_t r; if (!tsdn_null(tsdn)) { tsd_t *tsd = tsdn_tsd(tsdn); @@ -440,110 +520,18 @@ arena_cache_oblivious_randomize(tsdn_t *tsdn, arena_t *arena, edata_t *edata, uint64_t stack_value = (uint64_t)(uintptr_t)&r; r = (size_t)prng_lg_range_u64(&stack_value, lg_range); } - uintptr_t random_offset = ((uintptr_t)r) << (LG_PAGE - - lg_range); - edata->e_addr = (void *)((uintptr_t)edata->e_addr + - random_offset); - assert(ALIGNMENT_ADDR2BASE(edata->e_addr, alignment) == - edata->e_addr); - } -} - -/* - * The dalloc bin info contains just the information that the common paths need - * during tcache flushes. By force-inlining these paths, and using local copies - * of data (so that the compiler knows it's constant), we avoid a whole bunch of - * redundant loads and stores by leaving this information in registers. - */ -typedef struct arena_dalloc_bin_locked_info_s arena_dalloc_bin_locked_info_t; -struct arena_dalloc_bin_locked_info_s { - div_info_t div_info; - uint32_t nregs; - uint64_t ndalloc; -}; - -JEMALLOC_ALWAYS_INLINE size_t -arena_slab_regind(arena_dalloc_bin_locked_info_t *info, szind_t binind, - edata_t *slab, const void *ptr) { - size_t diff, regind; - - /* Freeing a pointer outside the slab can cause assertion failure. */ - assert((uintptr_t)ptr >= (uintptr_t)edata_addr_get(slab)); - assert((uintptr_t)ptr < (uintptr_t)edata_past_get(slab)); - /* Freeing an interior pointer can cause assertion failure. */ - assert(((uintptr_t)ptr - (uintptr_t)edata_addr_get(slab)) % - (uintptr_t)bin_infos[binind].reg_size == 0); - - diff = (size_t)((uintptr_t)ptr - (uintptr_t)edata_addr_get(slab)); - - /* Avoid doing division with a variable divisor. */ - regind = div_compute(&info->div_info, diff); - - assert(regind < bin_infos[binind].nregs); - - return regind; -} - -JEMALLOC_ALWAYS_INLINE void -arena_dalloc_bin_locked_begin(arena_dalloc_bin_locked_info_t *info, - szind_t binind) { - info->div_info = arena_binind_div_info[binind]; - info->nregs = bin_infos[binind].nregs; - info->ndalloc = 0; -} - -/* - * Does the deallocation work associated with freeing a single pointer (a - * "step") in between a arena_dalloc_bin_locked begin and end call. - * - * Returns true if arena_slab_dalloc must be called on slab. Doesn't do - * stats updates, which happen during finish (this lets running counts get left - * in a register). - */ -JEMALLOC_ALWAYS_INLINE bool -arena_dalloc_bin_locked_step(tsdn_t *tsdn, arena_t *arena, bin_t *bin, - arena_dalloc_bin_locked_info_t *info, szind_t binind, edata_t *slab, - void *ptr) { - const bin_info_t *bin_info = &bin_infos[binind]; - size_t regind = arena_slab_regind(info, binind, slab, ptr); - slab_data_t *slab_data = edata_slab_data_get(slab); - - assert(edata_nfree_get(slab) < bin_info->nregs); - /* Freeing an unallocated pointer can cause assertion failure. */ - assert(bitmap_get(slab_data->bitmap, &bin_info->bitmap_info, regind)); - - bitmap_unset(slab_data->bitmap, &bin_info->bitmap_info, regind); - edata_nfree_inc(slab); - - if (config_stats) { - info->ndalloc++; - } - - unsigned nfree = edata_nfree_get(slab); - if (nfree == bin_info->nregs) { - arena_dalloc_bin_locked_handle_newly_empty(tsdn, arena, slab, - bin); - return true; - } else if (nfree == 1 && slab != bin->slabcur) { - arena_dalloc_bin_locked_handle_newly_nonempty(tsdn, arena, slab, - bin); - } - return false; -} - -JEMALLOC_ALWAYS_INLINE void -arena_dalloc_bin_locked_finish(tsdn_t *tsdn, arena_t *arena, bin_t *bin, - arena_dalloc_bin_locked_info_t *info) { - if (config_stats) { - bin->stats.ndalloc += info->ndalloc; - assert(bin->stats.curregs >= (size_t)info->ndalloc); - bin->stats.curregs -= (size_t)info->ndalloc; + uintptr_t random_offset = ((uintptr_t)r) + << (LG_PAGE - lg_range); + edata->e_addr = (void *)((byte_t *)edata->e_addr + + random_offset); + assert(ALIGNMENT_ADDR2BASE(edata->e_addr, alignment) + == edata->e_addr); } } static inline bin_t * arena_get_bin(arena_t *arena, szind_t binind, unsigned binshard) { - bin_t *shard0 = (bin_t *)((uintptr_t)arena + arena_bin_offsets[binind]); + bin_t *shard0 = (bin_t *)((byte_t *)arena + arena_bin_offsets[binind]); return shard0 + binshard; } diff --git a/dep/jemalloc/include/jemalloc/internal/arena_stats.h b/dep/jemalloc/include/jemalloc/internal/arena_stats.h index 15f1d345f..01012f68d 100644 --- a/dep/jemalloc/include/jemalloc/internal/arena_stats.h +++ b/dep/jemalloc/include/jemalloc/internal/arena_stats.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_ARENA_STATS_H #define JEMALLOC_INTERNAL_ARENA_STATS_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/lockedint.h" #include "jemalloc/internal/mutex.h" @@ -13,28 +14,34 @@ JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS typedef struct arena_stats_large_s arena_stats_large_t; struct arena_stats_large_s { /* - * Total number of allocation/deallocation requests served directly by - * the arena. + * Total number of large allocation/deallocation requests served directly + * by the arena. */ - locked_u64_t nmalloc; - locked_u64_t ndalloc; + locked_u64_t nmalloc; + locked_u64_t ndalloc; + + /* + * Total large active bytes (allocated - deallocated) served directly + * by the arena. + */ + locked_u64_t active_bytes; /* * Number of allocation requests that correspond to this size class. * This includes requests served by tcache, though tcache only * periodically merges into this counter. */ - locked_u64_t nrequests; /* Partially derived. */ + locked_u64_t nrequests; /* Partially derived. */ /* * Number of tcache fills / flushes for large (similarly, periodically * merged). Note that there is no large tcache batch-fill currently * (i.e. only fill 1 at a time); however flush may be batched. */ - locked_u64_t nfills; /* Partially derived. */ - locked_u64_t nflushes; /* Partially derived. */ + locked_u64_t nfills; /* Partially derived. */ + locked_u64_t nflushes; /* Partially derived. */ /* Current number of allocations of this size class. */ - size_t curlextents; /* Derived. */ + size_t curlextents; /* Derived. */ }; /* @@ -50,38 +57,40 @@ struct arena_stats_s { * resident includes the base stats -- that's why it lives here and not * in pa_shard_stats_t. */ - size_t base; /* Derived. */ - size_t resident; /* Derived. */ - size_t metadata_thp; /* Derived. */ - size_t mapped; /* Derived. */ + size_t base; /* Derived. */ + size_t metadata_edata; /* Derived. */ + size_t metadata_rtree; /* Derived. */ + size_t resident; /* Derived. */ + size_t metadata_thp; /* Derived. */ + size_t mapped; /* Derived. */ - atomic_zu_t internal; + atomic_zu_t internal; - size_t allocated_large; /* Derived. */ - uint64_t nmalloc_large; /* Derived. */ - uint64_t ndalloc_large; /* Derived. */ - uint64_t nfills_large; /* Derived. */ - uint64_t nflushes_large; /* Derived. */ - uint64_t nrequests_large; /* Derived. */ + size_t allocated_large; /* Derived. */ + uint64_t nmalloc_large; /* Derived. */ + uint64_t ndalloc_large; /* Derived. */ + uint64_t nfills_large; /* Derived. */ + uint64_t nflushes_large; /* Derived. */ + uint64_t nrequests_large; /* Derived. */ /* * The stats logically owned by the pa_shard in the same arena. This * lives here only because it's convenient for the purposes of the ctl * module -- it only knows about the single arena_stats. */ - pa_shard_stats_t pa_shard_stats; + pa_shard_stats_t pa_shard_stats; /* Number of bytes cached in tcache associated with this arena. */ - size_t tcache_bytes; /* Derived. */ - size_t tcache_stashed_bytes; /* Derived. */ + size_t tcache_bytes; /* Derived. */ + size_t tcache_stashed_bytes; /* Derived. */ mutex_prof_data_t mutex_prof_data[mutex_prof_num_arena_mutexes]; /* One element for each large size class. */ - arena_stats_large_t lstats[SC_NSIZES - SC_NBINS]; + arena_stats_large_t lstats[SC_NSIZES - SC_NBINS]; /* Arena uptime. */ - nstime_t uptime; + nstime_t uptime; }; static inline bool @@ -92,7 +101,7 @@ arena_stats_init(tsdn_t *tsdn, arena_stats_t *arena_stats) { } } if (LOCKEDINT_MTX_INIT(arena_stats->mtx, "arena_stats", - WITNESS_RANK_ARENA_STATS, malloc_mutex_rank_exclusive)) { + WITNESS_RANK_ARENA_STATS, malloc_mutex_rank_exclusive)) { return true; } /* Memory is zeroed, so there is no need to clear stats. */ @@ -106,8 +115,8 @@ arena_stats_large_flush_nrequests_add(tsdn_t *tsdn, arena_stats_t *arena_stats, arena_stats_large_t *lstats = &arena_stats->lstats[szind - SC_NBINS]; locked_inc_u64(tsdn, LOCKEDINT_MTX(arena_stats->mtx), &lstats->nrequests, nrequests); - locked_inc_u64(tsdn, LOCKEDINT_MTX(arena_stats->mtx), - &lstats->nflushes, 1); + locked_inc_u64( + tsdn, LOCKEDINT_MTX(arena_stats->mtx), &lstats->nflushes, 1); LOCKEDINT_MTX_UNLOCK(tsdn, arena_stats->mtx); } diff --git a/dep/jemalloc/include/jemalloc/internal/arena_structs.h b/dep/jemalloc/include/jemalloc/internal/arena_structs.h index e2a5a4087..471f76920 100644 --- a/dep/jemalloc/include/jemalloc/internal/arena_structs.h +++ b/dep/jemalloc/include/jemalloc/internal/arena_structs.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_ARENA_STRUCTS_H #define JEMALLOC_INTERNAL_ARENA_STRUCTS_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/arena_stats.h" #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/bin.h" @@ -31,20 +32,20 @@ struct arena_s { * * Synchronization: atomic. */ - atomic_u_t nthreads[2]; + atomic_u_t nthreads[2]; /* Next bin shard for binding new threads. Synchronization: atomic. */ - atomic_u_t binshard_next; + atomic_u_t binshard_next; /* * When percpu_arena is enabled, to amortize the cost of reading / * updating the current CPU id, track the most recent thread accessing * this arena, and only read CPU if there is a mismatch. */ - tsdn_t *last_thd; + tsdn_t *last_thd; /* Synchronization: internal. */ - arena_stats_t stats; + arena_stats_t stats; /* * Lists of tcaches and cache_bin_array_descriptors for extant threads @@ -53,28 +54,28 @@ struct arena_s { * * Synchronization: tcache_ql_mtx. */ - ql_head(tcache_slow_t) tcache_ql; - ql_head(cache_bin_array_descriptor_t) cache_bin_array_descriptor_ql; - malloc_mutex_t tcache_ql_mtx; + ql_head(tcache_slow_t) tcache_ql; + ql_head(cache_bin_array_descriptor_t) cache_bin_array_descriptor_ql; + malloc_mutex_t tcache_ql_mtx; /* * Represents a dss_prec_t, but atomically. * * Synchronization: atomic. */ - atomic_u_t dss_prec; + atomic_u_t dss_prec; /* * Extant large allocations. * * Synchronization: large_mtx. */ - edata_list_active_t large; + edata_list_active_t large; /* Synchronizes all large allocation/update/deallocation. */ - malloc_mutex_t large_mtx; + malloc_mutex_t large_mtx; /* The page-level allocator shard this arena uses. */ - pa_shard_t pa_shard; + pa_shard_t pa_shard; /* * A cached copy of base->ind. This can get accessed on hot paths; @@ -87,15 +88,24 @@ struct arena_s { * * Synchronization: internal. */ - base_t *base; + base_t *base; /* Used to determine uptime. Read-only after initialization. */ - nstime_t create_time; + nstime_t create_time; + + /* The name of the arena. */ + char name[ARENA_NAME_LEN]; /* * The arena is allocated alongside its bins; really this is a * dynamically sized array determined by the binshard settings. + * Enforcing cacheline-alignment to minimize the number of cachelines + * touched on the hot paths. */ - bin_t bins[0]; + JEMALLOC_WARN_ON_USAGE( + "Do not use this field directly. " + "Use `arena_get_bin` instead.") + JEMALLOC_ALIGNED(CACHELINE) + bin_t all_bins[0]; }; #endif /* JEMALLOC_INTERNAL_ARENA_STRUCTS_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/arena_types.h b/dep/jemalloc/include/jemalloc/internal/arena_types.h index d0e129176..c586164f3 100644 --- a/dep/jemalloc/include/jemalloc/internal/arena_types.h +++ b/dep/jemalloc/include/jemalloc/internal/arena_types.h @@ -1,39 +1,41 @@ #ifndef JEMALLOC_INTERNAL_ARENA_TYPES_H #define JEMALLOC_INTERNAL_ARENA_TYPES_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/sc.h" /* Default decay times in milliseconds. */ -#define DIRTY_DECAY_MS_DEFAULT ZD(10 * 1000) -#define MUZZY_DECAY_MS_DEFAULT (0) +#define DIRTY_DECAY_MS_DEFAULT ZD(10 * 1000) +#define MUZZY_DECAY_MS_DEFAULT (0) /* Number of event ticks between time checks. */ -#define ARENA_DECAY_NTICKS_PER_UPDATE 1000 +#define ARENA_DECAY_NTICKS_PER_UPDATE 1000 +/* Maximum length of the arena name. */ +#define ARENA_NAME_LEN 32 -typedef struct arena_decay_s arena_decay_t; typedef struct arena_s arena_t; typedef enum { - percpu_arena_mode_names_base = 0, /* Used for options processing. */ + percpu_arena_mode_names_base = 0, /* Used for options processing. */ /* * *_uninit are used only during bootstrapping, and must correspond * to initialized variant plus percpu_arena_mode_enabled_base. */ - percpu_arena_uninit = 0, - per_phycpu_arena_uninit = 1, + percpu_arena_uninit = 0, + per_phycpu_arena_uninit = 1, /* All non-disabled modes must come after percpu_arena_disabled. */ - percpu_arena_disabled = 2, + percpu_arena_disabled = 2, - percpu_arena_mode_names_limit = 3, /* Used for options processing. */ + percpu_arena_mode_names_limit = 3, /* Used for options processing. */ percpu_arena_mode_enabled_base = 3, - percpu_arena = 3, - per_phycpu_arena = 4 /* Hyper threads share arena. */ + percpu_arena = 3, + per_phycpu_arena = 4 /* Hyper threads share arena. */ } percpu_arena_mode_t; -#define PERCPU_ARENA_ENABLED(m) ((m) >= percpu_arena_mode_enabled_base) -#define PERCPU_ARENA_DEFAULT percpu_arena_disabled +#define PERCPU_ARENA_ENABLED(m) ((m) >= percpu_arena_mode_enabled_base) +#define PERCPU_ARENA_DEFAULT percpu_arena_disabled /* * When allocation_size >= oversize_threshold, use the dedicated huge arena diff --git a/dep/jemalloc/include/jemalloc/internal/assert.h b/dep/jemalloc/include/jemalloc/internal/assert.h index be4d45b32..1b5da72f0 100644 --- a/dep/jemalloc/include/jemalloc/internal/assert.h +++ b/dep/jemalloc/include/jemalloc/internal/assert.h @@ -1,3 +1,4 @@ +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/util.h" @@ -6,51 +7,57 @@ * assertion failure. */ #ifndef assert -#define assert(e) do { \ - if (unlikely(config_debug && !(e))) { \ - malloc_printf( \ - ": %s:%d: Failed assertion: \"%s\"\n", \ - __FILE__, __LINE__, #e); \ - abort(); \ - } \ -} while (0) +# define assert(e) \ + do { \ + if (unlikely(config_debug && !(e))) { \ + malloc_printf( \ + ": %s:%d: Failed assertion: \"%s\"\n", \ + __FILE__, __LINE__, #e); \ + abort(); \ + } \ + } while (0) #endif #ifndef not_reached -#define not_reached() do { \ - if (config_debug) { \ - malloc_printf( \ - ": %s:%d: Unreachable code reached\n", \ - __FILE__, __LINE__); \ - abort(); \ - } \ - unreachable(); \ -} while (0) +# define not_reached() \ + do { \ + if (config_debug) { \ + malloc_printf( \ + ": %s:%d: Unreachable code reached\n", \ + __FILE__, __LINE__); \ + abort(); \ + } \ + unreachable(); \ + } while (0) #endif #ifndef not_implemented -#define not_implemented() do { \ - if (config_debug) { \ - malloc_printf(": %s:%d: Not implemented\n", \ - __FILE__, __LINE__); \ - abort(); \ - } \ -} while (0) +# define not_implemented() \ + do { \ + if (config_debug) { \ + malloc_printf( \ + ": %s:%d: Not implemented\n", \ + __FILE__, __LINE__); \ + abort(); \ + } \ + } while (0) #endif #ifndef assert_not_implemented -#define assert_not_implemented(e) do { \ - if (unlikely(config_debug && !(e))) { \ - not_implemented(); \ - } \ -} while (0) +# define assert_not_implemented(e) \ + do { \ + if (unlikely(config_debug && !(e))) { \ + not_implemented(); \ + } \ + } while (0) #endif /* Use to assert a particular configuration, e.g., cassert(config_debug). */ #ifndef cassert -#define cassert(c) do { \ - if (unlikely(!(c))) { \ - not_reached(); \ - } \ -} while (0) +# define cassert(c) \ + do { \ + if (unlikely(!(c))) { \ + not_reached(); \ + } \ + } while (0) #endif diff --git a/dep/jemalloc/include/jemalloc/internal/atomic.h b/dep/jemalloc/include/jemalloc/internal/atomic.h index c0f73122a..f80e56403 100644 --- a/dep/jemalloc/include/jemalloc/internal/atomic.h +++ b/dep/jemalloc/include/jemalloc/internal/atomic.h @@ -1,27 +1,29 @@ #ifndef JEMALLOC_INTERNAL_ATOMIC_H #define JEMALLOC_INTERNAL_ATOMIC_H -#define ATOMIC_INLINE JEMALLOC_ALWAYS_INLINE +#include "jemalloc/internal/jemalloc_preamble.h" #define JEMALLOC_U8_ATOMICS #if defined(JEMALLOC_GCC_ATOMIC_ATOMICS) -# include "jemalloc/internal/atomic_gcc_atomic.h" -# if !defined(JEMALLOC_GCC_U8_ATOMIC_ATOMICS) -# undef JEMALLOC_U8_ATOMICS -# endif +# include "jemalloc/internal/atomic_gcc_atomic.h" +# if !defined(JEMALLOC_GCC_U8_ATOMIC_ATOMICS) +# undef JEMALLOC_U8_ATOMICS +# endif #elif defined(JEMALLOC_GCC_SYNC_ATOMICS) -# include "jemalloc/internal/atomic_gcc_sync.h" -# if !defined(JEMALLOC_GCC_U8_SYNC_ATOMICS) -# undef JEMALLOC_U8_ATOMICS -# endif +# include "jemalloc/internal/atomic_gcc_sync.h" +# if !defined(JEMALLOC_GCC_U8_SYNC_ATOMICS) +# undef JEMALLOC_U8_ATOMICS +# endif #elif defined(_MSC_VER) -# include "jemalloc/internal/atomic_msvc.h" +# include "jemalloc/internal/atomic_msvc.h" #elif defined(JEMALLOC_C11_ATOMICS) -# include "jemalloc/internal/atomic_c11.h" +# include "jemalloc/internal/atomic_c11.h" #else -# error "Don't have atomics implemented on this platform." +# error "Don't have atomics implemented on this platform." #endif +#define ATOMIC_INLINE JEMALLOC_ALWAYS_INLINE + /* * This header gives more or less a backport of C11 atomics. The user can write * JEMALLOC_GENERATE_ATOMICS(type, short_type, lg_sizeof_type); to generate @@ -54,22 +56,19 @@ /* * Another convenience -- simple atomic helper functions. */ -#define JEMALLOC_GENERATE_EXPANDED_INT_ATOMICS(type, short_type, \ - lg_size) \ - JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, lg_size) \ - ATOMIC_INLINE void \ - atomic_load_add_store_##short_type(atomic_##short_type##_t *a, \ - type inc) { \ - type oldval = atomic_load_##short_type(a, ATOMIC_RELAXED); \ - type newval = oldval + inc; \ - atomic_store_##short_type(a, newval, ATOMIC_RELAXED); \ - } \ - ATOMIC_INLINE void \ - atomic_load_sub_store_##short_type(atomic_##short_type##_t *a, \ - type inc) { \ - type oldval = atomic_load_##short_type(a, ATOMIC_RELAXED); \ - type newval = oldval - inc; \ - atomic_store_##short_type(a, newval, ATOMIC_RELAXED); \ +#define JEMALLOC_GENERATE_EXPANDED_INT_ATOMICS(type, short_type, lg_size) \ + JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, lg_size) \ + ATOMIC_INLINE void atomic_load_add_store_##short_type( \ + atomic_##short_type##_t *a, type inc) { \ + type oldval = atomic_load_##short_type(a, ATOMIC_RELAXED); \ + type newval = oldval + inc; \ + atomic_store_##short_type(a, newval, ATOMIC_RELAXED); \ + } \ + ATOMIC_INLINE void atomic_load_sub_store_##short_type( \ + atomic_##short_type##_t *a, type inc) { \ + type oldval = atomic_load_##short_type(a, ATOMIC_RELAXED); \ + type newval = oldval - inc; \ + atomic_store_##short_type(a, newval, ATOMIC_RELAXED); \ } /* @@ -77,7 +76,7 @@ * fact. */ #if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3) -# define JEMALLOC_ATOMIC_U64 +# define JEMALLOC_ATOMIC_U64 #endif JEMALLOC_GENERATE_ATOMICS(void *, p, LG_SIZEOF_PTR) @@ -90,6 +89,8 @@ JEMALLOC_GENERATE_ATOMICS(bool, b, 0) JEMALLOC_GENERATE_EXPANDED_INT_ATOMICS(unsigned, u, LG_SIZEOF_INT) +JEMALLOC_GENERATE_EXPANDED_INT_ATOMICS(int, i, LG_SIZEOF_INT) + JEMALLOC_GENERATE_EXPANDED_INT_ATOMICS(size_t, zu, LG_SIZEOF_PTR) JEMALLOC_GENERATE_EXPANDED_INT_ATOMICS(ssize_t, zd, LG_SIZEOF_PTR) diff --git a/dep/jemalloc/include/jemalloc/internal/atomic_c11.h b/dep/jemalloc/include/jemalloc/internal/atomic_c11.h index a5f9313a6..1e86e2a04 100644 --- a/dep/jemalloc/include/jemalloc/internal/atomic_c11.h +++ b/dep/jemalloc/include/jemalloc/internal/atomic_c11.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_ATOMIC_C11_H #define JEMALLOC_INTERNAL_ATOMIC_C11_H +#include "jemalloc/internal/jemalloc_preamble.h" #include #define ATOMIC_INIT(...) ATOMIC_VAR_INIT(__VA_ARGS__) @@ -14,6 +15,7 @@ #define atomic_fence atomic_thread_fence +/* clang-format off */ #define JEMALLOC_GENERATE_ATOMICS(type, short_type, \ /* unused */ lg_size) \ typedef _Atomic(type) atomic_##short_type##_t; \ @@ -58,40 +60,35 @@ atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \ return atomic_compare_exchange_strong_explicit(a, expected, \ desired, success_mo, failure_mo); \ } +/* clang-format on */ /* * Integral types have some special operations available that non-integral ones * lack. */ -#define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, \ - /* unused */ lg_size) \ -JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size) \ - \ -ATOMIC_INLINE type \ -atomic_fetch_add_##short_type(atomic_##short_type##_t *a, \ - type val, atomic_memory_order_t mo) { \ - return atomic_fetch_add_explicit(a, val, mo); \ -} \ - \ -ATOMIC_INLINE type \ -atomic_fetch_sub_##short_type(atomic_##short_type##_t *a, \ - type val, atomic_memory_order_t mo) { \ - return atomic_fetch_sub_explicit(a, val, mo); \ -} \ -ATOMIC_INLINE type \ -atomic_fetch_and_##short_type(atomic_##short_type##_t *a, \ - type val, atomic_memory_order_t mo) { \ - return atomic_fetch_and_explicit(a, val, mo); \ -} \ -ATOMIC_INLINE type \ -atomic_fetch_or_##short_type(atomic_##short_type##_t *a, \ - type val, atomic_memory_order_t mo) { \ - return atomic_fetch_or_explicit(a, val, mo); \ -} \ -ATOMIC_INLINE type \ -atomic_fetch_xor_##short_type(atomic_##short_type##_t *a, \ - type val, atomic_memory_order_t mo) { \ - return atomic_fetch_xor_explicit(a, val, mo); \ -} +#define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, /* unused */ lg_size) \ + JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size) \ + \ + ATOMIC_INLINE type atomic_fetch_add_##short_type( \ + atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \ + return atomic_fetch_add_explicit(a, val, mo); \ + } \ + \ + ATOMIC_INLINE type atomic_fetch_sub_##short_type( \ + atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \ + return atomic_fetch_sub_explicit(a, val, mo); \ + } \ + ATOMIC_INLINE type atomic_fetch_and_##short_type( \ + atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \ + return atomic_fetch_and_explicit(a, val, mo); \ + } \ + ATOMIC_INLINE type atomic_fetch_or_##short_type( \ + atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \ + return atomic_fetch_or_explicit(a, val, mo); \ + } \ + ATOMIC_INLINE type atomic_fetch_xor_##short_type( \ + atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \ + return atomic_fetch_xor_explicit(a, val, mo); \ + } #endif /* JEMALLOC_INTERNAL_ATOMIC_C11_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/atomic_gcc_atomic.h b/dep/jemalloc/include/jemalloc/internal/atomic_gcc_atomic.h index 471515e82..a828a6b07 100644 --- a/dep/jemalloc/include/jemalloc/internal/atomic_gcc_atomic.h +++ b/dep/jemalloc/include/jemalloc/internal/atomic_gcc_atomic.h @@ -1,9 +1,13 @@ #ifndef JEMALLOC_INTERNAL_ATOMIC_GCC_ATOMIC_H #define JEMALLOC_INTERNAL_ATOMIC_GCC_ATOMIC_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/assert.h" -#define ATOMIC_INIT(...) {__VA_ARGS__} +#define ATOMIC_INLINE JEMALLOC_ALWAYS_INLINE + +#define ATOMIC_INIT(...) \ + { __VA_ARGS__ } typedef enum { atomic_memory_order_relaxed, @@ -36,94 +40,82 @@ atomic_fence(atomic_memory_order_t mo) { __atomic_thread_fence(atomic_enum_to_builtin(mo)); } -#define JEMALLOC_GENERATE_ATOMICS(type, short_type, \ - /* unused */ lg_size) \ -typedef struct { \ - type repr; \ -} atomic_##short_type##_t; \ - \ -ATOMIC_INLINE type \ -atomic_load_##short_type(const atomic_##short_type##_t *a, \ - atomic_memory_order_t mo) { \ - type result; \ - __atomic_load(&a->repr, &result, atomic_enum_to_builtin(mo)); \ - return result; \ -} \ - \ -ATOMIC_INLINE void \ -atomic_store_##short_type(atomic_##short_type##_t *a, type val, \ - atomic_memory_order_t mo) { \ - __atomic_store(&a->repr, &val, atomic_enum_to_builtin(mo)); \ -} \ - \ -ATOMIC_INLINE type \ -atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \ - atomic_memory_order_t mo) { \ - type result; \ - __atomic_exchange(&a->repr, &val, &result, \ - atomic_enum_to_builtin(mo)); \ - return result; \ -} \ - \ -ATOMIC_INLINE bool \ -atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \ - UNUSED type *expected, type desired, \ - atomic_memory_order_t success_mo, \ - atomic_memory_order_t failure_mo) { \ - return __atomic_compare_exchange(&a->repr, expected, &desired, \ - true, atomic_enum_to_builtin(success_mo), \ - atomic_enum_to_builtin(failure_mo)); \ -} \ - \ -ATOMIC_INLINE bool \ -atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \ - UNUSED type *expected, type desired, \ - atomic_memory_order_t success_mo, \ - atomic_memory_order_t failure_mo) { \ - return __atomic_compare_exchange(&a->repr, expected, &desired, \ - false, \ - atomic_enum_to_builtin(success_mo), \ - atomic_enum_to_builtin(failure_mo)); \ -} +#define JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size) \ + typedef struct { \ + type repr; \ + } atomic_##short_type##_t; \ + \ + ATOMIC_INLINE type atomic_load_##short_type( \ + const atomic_##short_type##_t *a, atomic_memory_order_t mo) { \ + type result; \ + __atomic_load(&a->repr, &result, atomic_enum_to_builtin(mo)); \ + return result; \ + } \ + \ + ATOMIC_INLINE void atomic_store_##short_type( \ + atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \ + __atomic_store(&a->repr, &val, atomic_enum_to_builtin(mo)); \ + } \ + \ + ATOMIC_INLINE type atomic_exchange_##short_type( \ + atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \ + type result; \ + __atomic_exchange( \ + &a->repr, &val, &result, atomic_enum_to_builtin(mo)); \ + return result; \ + } \ + \ + ATOMIC_INLINE bool atomic_compare_exchange_weak_##short_type( \ + atomic_##short_type##_t *a, UNUSED type *expected, type desired, \ + atomic_memory_order_t success_mo, \ + atomic_memory_order_t failure_mo) { \ + return __atomic_compare_exchange(&a->repr, expected, &desired, \ + true, atomic_enum_to_builtin(success_mo), \ + atomic_enum_to_builtin(failure_mo)); \ + } \ + \ + ATOMIC_INLINE bool atomic_compare_exchange_strong_##short_type( \ + atomic_##short_type##_t *a, UNUSED type *expected, type desired, \ + atomic_memory_order_t success_mo, \ + atomic_memory_order_t failure_mo) { \ + return __atomic_compare_exchange(&a->repr, expected, &desired, \ + false, atomic_enum_to_builtin(success_mo), \ + atomic_enum_to_builtin(failure_mo)); \ + } +#define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, /* unused */ lg_size) \ + JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size) \ + \ + ATOMIC_INLINE type atomic_fetch_add_##short_type( \ + atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \ + return __atomic_fetch_add( \ + &a->repr, val, atomic_enum_to_builtin(mo)); \ + } \ + \ + ATOMIC_INLINE type atomic_fetch_sub_##short_type( \ + atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \ + return __atomic_fetch_sub( \ + &a->repr, val, atomic_enum_to_builtin(mo)); \ + } \ + \ + ATOMIC_INLINE type atomic_fetch_and_##short_type( \ + atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \ + return __atomic_fetch_and( \ + &a->repr, val, atomic_enum_to_builtin(mo)); \ + } \ + \ + ATOMIC_INLINE type atomic_fetch_or_##short_type( \ + atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \ + return __atomic_fetch_or( \ + &a->repr, val, atomic_enum_to_builtin(mo)); \ + } \ + \ + ATOMIC_INLINE type atomic_fetch_xor_##short_type( \ + atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \ + return __atomic_fetch_xor( \ + &a->repr, val, atomic_enum_to_builtin(mo)); \ + } -#define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, \ - /* unused */ lg_size) \ -JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size) \ - \ -ATOMIC_INLINE type \ -atomic_fetch_add_##short_type(atomic_##short_type##_t *a, type val, \ - atomic_memory_order_t mo) { \ - return __atomic_fetch_add(&a->repr, val, \ - atomic_enum_to_builtin(mo)); \ -} \ - \ -ATOMIC_INLINE type \ -atomic_fetch_sub_##short_type(atomic_##short_type##_t *a, type val, \ - atomic_memory_order_t mo) { \ - return __atomic_fetch_sub(&a->repr, val, \ - atomic_enum_to_builtin(mo)); \ -} \ - \ -ATOMIC_INLINE type \ -atomic_fetch_and_##short_type(atomic_##short_type##_t *a, type val, \ - atomic_memory_order_t mo) { \ - return __atomic_fetch_and(&a->repr, val, \ - atomic_enum_to_builtin(mo)); \ -} \ - \ -ATOMIC_INLINE type \ -atomic_fetch_or_##short_type(atomic_##short_type##_t *a, type val, \ - atomic_memory_order_t mo) { \ - return __atomic_fetch_or(&a->repr, val, \ - atomic_enum_to_builtin(mo)); \ -} \ - \ -ATOMIC_INLINE type \ -atomic_fetch_xor_##short_type(atomic_##short_type##_t *a, type val, \ - atomic_memory_order_t mo) { \ - return __atomic_fetch_xor(&a->repr, val, \ - atomic_enum_to_builtin(mo)); \ -} +#undef ATOMIC_INLINE #endif /* JEMALLOC_INTERNAL_ATOMIC_GCC_ATOMIC_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h b/dep/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h index e02b7cbe3..9e2ff9c88 100644 --- a/dep/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h +++ b/dep/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h @@ -1,7 +1,12 @@ #ifndef JEMALLOC_INTERNAL_ATOMIC_GCC_SYNC_H #define JEMALLOC_INTERNAL_ATOMIC_GCC_SYNC_H -#define ATOMIC_INIT(...) {__VA_ARGS__} +#include "jemalloc/internal/jemalloc_preamble.h" + +#define ATOMIC_INLINE JEMALLOC_ALWAYS_INLINE + +#define ATOMIC_INIT(...) \ + { __VA_ARGS__ } typedef enum { atomic_memory_order_relaxed, @@ -25,13 +30,13 @@ atomic_fence(atomic_memory_order_t mo) { return; } asm volatile("" ::: "memory"); -# if defined(__i386__) || defined(__x86_64__) +#if defined(__i386__) || defined(__x86_64__) /* This is implicit on x86. */ -# elif defined(__ppc64__) +#elif defined(__ppc64__) asm volatile("lwsync"); -# elif defined(__ppc__) +#elif defined(__ppc__) asm volatile("sync"); -# elif defined(__sparc__) && defined(__arch64__) +#elif defined(__sparc__) && defined(__arch64__) if (mo == atomic_memory_order_acquire) { asm volatile("membar #LoadLoad | #LoadStore"); } else if (mo == atomic_memory_order_release) { @@ -39,9 +44,9 @@ atomic_fence(atomic_memory_order_t mo) { } else { asm volatile("membar #LoadLoad | #LoadStore | #StoreStore"); } -# else +#else __sync_synchronize(); -# endif +#endif asm volatile("" ::: "memory"); } @@ -64,25 +69,25 @@ atomic_fence(atomic_memory_order_t mo) { ATOMIC_INLINE void atomic_pre_sc_load_fence() { -# if defined(__i386__) || defined(__x86_64__) || \ - (defined(__sparc__) && defined(__arch64__)) +#if defined(__i386__) || defined(__x86_64__) \ + || (defined(__sparc__) && defined(__arch64__)) atomic_fence(atomic_memory_order_relaxed); -# else +#else atomic_fence(atomic_memory_order_seq_cst); -# endif +#endif } ATOMIC_INLINE void atomic_post_sc_store_fence() { -# if defined(__i386__) || defined(__x86_64__) || \ - (defined(__sparc__) && defined(__arch64__)) +#if defined(__i386__) || defined(__x86_64__) \ + || (defined(__sparc__) && defined(__arch64__)) atomic_fence(atomic_memory_order_seq_cst); -# else +#else atomic_fence(atomic_memory_order_relaxed); -# endif - +#endif } +/* clang-format off */ #define JEMALLOC_GENERATE_ATOMICS(type, short_type, \ /* unused */ lg_size) \ typedef struct { \ @@ -157,39 +162,36 @@ atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \ return false; \ } \ } +/* clang-format on */ -#define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, \ - /* unused */ lg_size) \ -JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size) \ - \ -ATOMIC_INLINE type \ -atomic_fetch_add_##short_type(atomic_##short_type##_t *a, type val, \ - atomic_memory_order_t mo) { \ - return __sync_fetch_and_add(&a->repr, val); \ -} \ - \ -ATOMIC_INLINE type \ -atomic_fetch_sub_##short_type(atomic_##short_type##_t *a, type val, \ - atomic_memory_order_t mo) { \ - return __sync_fetch_and_sub(&a->repr, val); \ -} \ - \ -ATOMIC_INLINE type \ -atomic_fetch_and_##short_type(atomic_##short_type##_t *a, type val, \ - atomic_memory_order_t mo) { \ - return __sync_fetch_and_and(&a->repr, val); \ -} \ - \ -ATOMIC_INLINE type \ -atomic_fetch_or_##short_type(atomic_##short_type##_t *a, type val, \ - atomic_memory_order_t mo) { \ - return __sync_fetch_and_or(&a->repr, val); \ -} \ - \ -ATOMIC_INLINE type \ -atomic_fetch_xor_##short_type(atomic_##short_type##_t *a, type val, \ - atomic_memory_order_t mo) { \ - return __sync_fetch_and_xor(&a->repr, val); \ -} +#define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, /* unused */ lg_size) \ + JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size) \ + \ + ATOMIC_INLINE type atomic_fetch_add_##short_type( \ + atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \ + return __sync_fetch_and_add(&a->repr, val); \ + } \ + \ + ATOMIC_INLINE type atomic_fetch_sub_##short_type( \ + atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \ + return __sync_fetch_and_sub(&a->repr, val); \ + } \ + \ + ATOMIC_INLINE type atomic_fetch_and_##short_type( \ + atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \ + return __sync_fetch_and_and(&a->repr, val); \ + } \ + \ + ATOMIC_INLINE type atomic_fetch_or_##short_type( \ + atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \ + return __sync_fetch_and_or(&a->repr, val); \ + } \ + \ + ATOMIC_INLINE type atomic_fetch_xor_##short_type( \ + atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \ + return __sync_fetch_and_xor(&a->repr, val); \ + } + +#undef ATOMIC_INLINE #endif /* JEMALLOC_INTERNAL_ATOMIC_GCC_SYNC_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/atomic_msvc.h b/dep/jemalloc/include/jemalloc/internal/atomic_msvc.h index 67057ce50..7accca635 100644 --- a/dep/jemalloc/include/jemalloc/internal/atomic_msvc.h +++ b/dep/jemalloc/include/jemalloc/internal/atomic_msvc.h @@ -1,7 +1,12 @@ #ifndef JEMALLOC_INTERNAL_ATOMIC_MSVC_H #define JEMALLOC_INTERNAL_ATOMIC_MSVC_H -#define ATOMIC_INIT(...) {__VA_ARGS__} +#include "jemalloc/internal/jemalloc_preamble.h" + +#define ATOMIC_INLINE JEMALLOC_ALWAYS_INLINE + +#define ATOMIC_INIT(...) \ + { __VA_ARGS__ } typedef enum { atomic_memory_order_relaxed, @@ -11,109 +16,106 @@ typedef enum { atomic_memory_order_seq_cst } atomic_memory_order_t; -typedef char atomic_repr_0_t; -typedef short atomic_repr_1_t; -typedef long atomic_repr_2_t; +typedef char atomic_repr_0_t; +typedef short atomic_repr_1_t; +typedef long atomic_repr_2_t; typedef __int64 atomic_repr_3_t; ATOMIC_INLINE void atomic_fence(atomic_memory_order_t mo) { _ReadWriteBarrier(); -# if defined(_M_ARM) || defined(_M_ARM64) +#if defined(_M_ARM) || defined(_M_ARM64) /* ARM needs a barrier for everything but relaxed. */ if (mo != atomic_memory_order_relaxed) { MemoryBarrier(); } -# elif defined(_M_IX86) || defined (_M_X64) +#elif defined(_M_IX86) || defined(_M_X64) /* x86 needs a barrier only for seq_cst. */ if (mo == atomic_memory_order_seq_cst) { MemoryBarrier(); } -# else -# error "Don't know how to create atomics for this platform for MSVC." -# endif +#else +# error "Don't know how to create atomics for this platform for MSVC." +#endif _ReadWriteBarrier(); } -#define ATOMIC_INTERLOCKED_REPR(lg_size) atomic_repr_ ## lg_size ## _t +#define ATOMIC_INTERLOCKED_REPR(lg_size) atomic_repr_##lg_size##_t #define ATOMIC_CONCAT(a, b) ATOMIC_RAW_CONCAT(a, b) -#define ATOMIC_RAW_CONCAT(a, b) a ## b +#define ATOMIC_RAW_CONCAT(a, b) a##b -#define ATOMIC_INTERLOCKED_NAME(base_name, lg_size) ATOMIC_CONCAT( \ - base_name, ATOMIC_INTERLOCKED_SUFFIX(lg_size)) +#define ATOMIC_INTERLOCKED_NAME(base_name, lg_size) \ + ATOMIC_CONCAT(base_name, ATOMIC_INTERLOCKED_SUFFIX(lg_size)) -#define ATOMIC_INTERLOCKED_SUFFIX(lg_size) \ - ATOMIC_CONCAT(ATOMIC_INTERLOCKED_SUFFIX_, lg_size) +#define ATOMIC_INTERLOCKED_SUFFIX(lg_size) \ + ATOMIC_CONCAT(ATOMIC_INTERLOCKED_SUFFIX_, lg_size) #define ATOMIC_INTERLOCKED_SUFFIX_0 8 #define ATOMIC_INTERLOCKED_SUFFIX_1 16 #define ATOMIC_INTERLOCKED_SUFFIX_2 #define ATOMIC_INTERLOCKED_SUFFIX_3 64 -#define JEMALLOC_GENERATE_ATOMICS(type, short_type, lg_size) \ -typedef struct { \ - ATOMIC_INTERLOCKED_REPR(lg_size) repr; \ -} atomic_##short_type##_t; \ - \ -ATOMIC_INLINE type \ -atomic_load_##short_type(const atomic_##short_type##_t *a, \ - atomic_memory_order_t mo) { \ - ATOMIC_INTERLOCKED_REPR(lg_size) ret = a->repr; \ - if (mo != atomic_memory_order_relaxed) { \ - atomic_fence(atomic_memory_order_acquire); \ - } \ - return (type) ret; \ -} \ - \ -ATOMIC_INLINE void \ -atomic_store_##short_type(atomic_##short_type##_t *a, \ - type val, atomic_memory_order_t mo) { \ - if (mo != atomic_memory_order_relaxed) { \ - atomic_fence(atomic_memory_order_release); \ - } \ - a->repr = (ATOMIC_INTERLOCKED_REPR(lg_size)) val; \ - if (mo == atomic_memory_order_seq_cst) { \ - atomic_fence(atomic_memory_order_seq_cst); \ - } \ -} \ - \ -ATOMIC_INLINE type \ -atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \ - atomic_memory_order_t mo) { \ - return (type)ATOMIC_INTERLOCKED_NAME(_InterlockedExchange, \ - lg_size)(&a->repr, (ATOMIC_INTERLOCKED_REPR(lg_size))val); \ -} \ - \ -ATOMIC_INLINE bool \ -atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \ - type *expected, type desired, atomic_memory_order_t success_mo, \ - atomic_memory_order_t failure_mo) { \ - ATOMIC_INTERLOCKED_REPR(lg_size) e = \ - (ATOMIC_INTERLOCKED_REPR(lg_size))*expected; \ - ATOMIC_INTERLOCKED_REPR(lg_size) d = \ - (ATOMIC_INTERLOCKED_REPR(lg_size))desired; \ - ATOMIC_INTERLOCKED_REPR(lg_size) old = \ - ATOMIC_INTERLOCKED_NAME(_InterlockedCompareExchange, \ - lg_size)(&a->repr, d, e); \ - if (old == e) { \ - return true; \ - } else { \ - *expected = (type)old; \ - return false; \ - } \ -} \ - \ -ATOMIC_INLINE bool \ -atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \ - type *expected, type desired, atomic_memory_order_t success_mo, \ - atomic_memory_order_t failure_mo) { \ - /* We implement the weak version with strong semantics. */ \ - return atomic_compare_exchange_weak_##short_type(a, expected, \ - desired, success_mo, failure_mo); \ -} - +#define JEMALLOC_GENERATE_ATOMICS(type, short_type, lg_size) \ + typedef struct { \ + ATOMIC_INTERLOCKED_REPR(lg_size) repr; \ + } atomic_##short_type##_t; \ + \ + ATOMIC_INLINE type atomic_load_##short_type( \ + const atomic_##short_type##_t *a, atomic_memory_order_t mo) { \ + ATOMIC_INTERLOCKED_REPR(lg_size) ret = a->repr; \ + if (mo != atomic_memory_order_relaxed) { \ + atomic_fence(atomic_memory_order_acquire); \ + } \ + return (type)ret; \ + } \ + \ + ATOMIC_INLINE void atomic_store_##short_type( \ + atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \ + if (mo != atomic_memory_order_relaxed) { \ + atomic_fence(atomic_memory_order_release); \ + } \ + a->repr = (ATOMIC_INTERLOCKED_REPR(lg_size))val; \ + if (mo == atomic_memory_order_seq_cst) { \ + atomic_fence(atomic_memory_order_seq_cst); \ + } \ + } \ + \ + ATOMIC_INLINE type atomic_exchange_##short_type( \ + atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \ + return (type)ATOMIC_INTERLOCKED_NAME(_InterlockedExchange, \ + lg_size)(&a->repr, (ATOMIC_INTERLOCKED_REPR(lg_size))val); \ + } \ + \ + ATOMIC_INLINE bool atomic_compare_exchange_weak_##short_type( \ + atomic_##short_type##_t *a, type *expected, type desired, \ + atomic_memory_order_t success_mo, \ + atomic_memory_order_t failure_mo) { \ + ATOMIC_INTERLOCKED_REPR(lg_size) \ + e = (ATOMIC_INTERLOCKED_REPR(lg_size)) * expected; \ + ATOMIC_INTERLOCKED_REPR(lg_size) \ + d = (ATOMIC_INTERLOCKED_REPR(lg_size))desired; \ + ATOMIC_INTERLOCKED_REPR(lg_size) \ + old = ATOMIC_INTERLOCKED_NAME( \ + _InterlockedCompareExchange, lg_size)(&a->repr, d, e); \ + if (old == e) { \ + return true; \ + } else { \ + *expected = (type)old; \ + return false; \ + } \ + } \ + \ + ATOMIC_INLINE bool atomic_compare_exchange_strong_##short_type( \ + atomic_##short_type##_t *a, type *expected, type desired, \ + atomic_memory_order_t success_mo, \ + atomic_memory_order_t failure_mo) { \ + /* We implement the weak version with strong semantics. */ \ + return atomic_compare_exchange_weak_##short_type( \ + a, expected, desired, success_mo, failure_mo); \ + } +/* clang-format off */ #define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, lg_size) \ JEMALLOC_GENERATE_ATOMICS(type, short_type, lg_size) \ \ @@ -154,5 +156,8 @@ atomic_fetch_xor_##short_type(atomic_##short_type##_t *a, \ return (type)ATOMIC_INTERLOCKED_NAME(_InterlockedXor, lg_size)( \ &a->repr, (ATOMIC_INTERLOCKED_REPR(lg_size))val); \ } +/* clang-format on */ + +#undef ATOMIC_INLINE #endif /* JEMALLOC_INTERNAL_ATOMIC_MSVC_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/background_thread_externs.h b/dep/jemalloc/include/jemalloc/internal/background_thread_externs.h index 6ae3c8d89..efc0aaa47 100644 --- a/dep/jemalloc/include/jemalloc/internal/background_thread_externs.h +++ b/dep/jemalloc/include/jemalloc/internal/background_thread_externs.h @@ -1,26 +1,31 @@ #ifndef JEMALLOC_INTERNAL_BACKGROUND_THREAD_EXTERNS_H #define JEMALLOC_INTERNAL_BACKGROUND_THREAD_EXTERNS_H -extern bool opt_background_thread; -extern size_t opt_max_background_threads; -extern malloc_mutex_t background_thread_lock; -extern atomic_b_t background_thread_enabled_state; -extern size_t n_background_threads; -extern size_t max_background_threads; +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/background_thread_structs.h" +#include "jemalloc/internal/base.h" +#include "jemalloc/internal/mutex.h" + +extern bool opt_background_thread; +extern size_t opt_max_background_threads; +extern malloc_mutex_t background_thread_lock; +extern atomic_b_t background_thread_enabled_state; +extern size_t n_background_threads; +extern size_t max_background_threads; extern background_thread_info_t *background_thread_info; bool background_thread_create(tsd_t *tsd, unsigned arena_ind); bool background_threads_enable(tsd_t *tsd); bool background_threads_disable(tsd_t *tsd); -bool background_thread_is_started(background_thread_info_t* info); -void background_thread_wakeup_early(background_thread_info_t *info, - nstime_t *remaining_sleep); +bool background_thread_is_started(background_thread_info_t *info); +void background_thread_wakeup_early( + background_thread_info_t *info, nstime_t *remaining_sleep); void background_thread_prefork0(tsdn_t *tsdn); void background_thread_prefork1(tsdn_t *tsdn); void background_thread_postfork_parent(tsdn_t *tsdn); void background_thread_postfork_child(tsdn_t *tsdn); -bool background_thread_stats_read(tsdn_t *tsdn, - background_thread_stats_t *stats); +bool background_thread_stats_read( + tsdn_t *tsdn, background_thread_stats_t *stats); void background_thread_ctl_init(tsdn_t *tsdn); #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER diff --git a/dep/jemalloc/include/jemalloc/internal/background_thread_inlines.h b/dep/jemalloc/include/jemalloc/internal/background_thread_inlines.h index 92c5febe7..e822a3f74 100644 --- a/dep/jemalloc/include/jemalloc/internal/background_thread_inlines.h +++ b/dep/jemalloc/include/jemalloc/internal/background_thread_inlines.h @@ -1,15 +1,25 @@ #ifndef JEMALLOC_INTERNAL_BACKGROUND_THREAD_INLINES_H #define JEMALLOC_INTERNAL_BACKGROUND_THREAD_INLINES_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/arena_inlines_a.h" +#include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/background_thread_externs.h" + JEMALLOC_ALWAYS_INLINE bool background_thread_enabled(void) { return atomic_load_b(&background_thread_enabled_state, ATOMIC_RELAXED); } +JEMALLOC_ALWAYS_INLINE void +background_thread_enabled_set_impl(bool state) { + atomic_store_b(&background_thread_enabled_state, state, ATOMIC_RELAXED); +} + JEMALLOC_ALWAYS_INLINE void background_thread_enabled_set(tsdn_t *tsdn, bool state) { malloc_mutex_assert_owner(tsdn, &background_thread_lock); - atomic_store_b(&background_thread_enabled_state, state, ATOMIC_RELAXED); + background_thread_enabled_set_impl(state); } JEMALLOC_ALWAYS_INLINE background_thread_info_t * @@ -26,14 +36,14 @@ background_thread_info_get(size_t ind) { JEMALLOC_ALWAYS_INLINE uint64_t background_thread_wakeup_time_get(background_thread_info_t *info) { uint64_t next_wakeup = nstime_ns(&info->next_wakeup); - assert(atomic_load_b(&info->indefinite_sleep, ATOMIC_ACQUIRE) == - (next_wakeup == BACKGROUND_THREAD_INDEFINITE_SLEEP)); + assert(atomic_load_b(&info->indefinite_sleep, ATOMIC_ACQUIRE) + == (next_wakeup == BACKGROUND_THREAD_INDEFINITE_SLEEP)); return next_wakeup; } JEMALLOC_ALWAYS_INLINE void -background_thread_wakeup_time_set(tsdn_t *tsdn, background_thread_info_t *info, - uint64_t wakeup_time) { +background_thread_wakeup_time_set( + tsdn_t *tsdn, background_thread_info_t *info, uint64_t wakeup_time) { malloc_mutex_assert_owner(tsdn, &info->mtx); atomic_store_b(&info->indefinite_sleep, wakeup_time == BACKGROUND_THREAD_INDEFINITE_SLEEP, ATOMIC_RELEASE); diff --git a/dep/jemalloc/include/jemalloc/internal/background_thread_structs.h b/dep/jemalloc/include/jemalloc/internal/background_thread_structs.h index 83a919846..d56673dab 100644 --- a/dep/jemalloc/include/jemalloc/internal/background_thread_structs.h +++ b/dep/jemalloc/include/jemalloc/internal/background_thread_structs.h @@ -1,10 +1,13 @@ #ifndef JEMALLOC_INTERNAL_BACKGROUND_THREAD_STRUCTS_H #define JEMALLOC_INTERNAL_BACKGROUND_THREAD_STRUCTS_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/mutex.h" + /* This file really combines "structs" and "types", but only transitionally. */ #if defined(JEMALLOC_BACKGROUND_THREAD) || defined(JEMALLOC_LAZY_LOCK) -# define JEMALLOC_PTHREAD_CREATE_WRAPPER +# define JEMALLOC_PTHREAD_CREATE_WRAPPER #endif #define BACKGROUND_THREAD_INDEFINITE_SLEEP UINT64_MAX @@ -32,33 +35,33 @@ typedef enum { struct background_thread_info_s { #ifdef JEMALLOC_BACKGROUND_THREAD /* Background thread is pthread specific. */ - pthread_t thread; - pthread_cond_t cond; + pthread_t thread; + pthread_cond_t cond; #endif - malloc_mutex_t mtx; - background_thread_state_t state; + malloc_mutex_t mtx; + background_thread_state_t state; /* When true, it means no wakeup scheduled. */ - atomic_b_t indefinite_sleep; + atomic_b_t indefinite_sleep; /* Next scheduled wakeup time (absolute time in ns). */ - nstime_t next_wakeup; + nstime_t next_wakeup; /* * Since the last background thread run, newly added number of pages * that need to be purged by the next wakeup. This is adjusted on * epoch advance, and is used to determine whether we should signal the * background thread to wake up earlier. */ - size_t npages_to_purge_new; + size_t npages_to_purge_new; /* Stats: total number of runs since started. */ - uint64_t tot_n_runs; + uint64_t tot_n_runs; /* Stats: total sleep time since started. */ - nstime_t tot_sleep_time; + nstime_t tot_sleep_time; }; typedef struct background_thread_info_s background_thread_info_t; struct background_thread_stats_s { - size_t num_threads; - uint64_t num_runs; - nstime_t run_interval; + size_t num_threads; + uint64_t num_runs; + nstime_t run_interval; mutex_prof_data_t max_counter_per_bg_thd; }; typedef struct background_thread_stats_s background_thread_stats_t; diff --git a/dep/jemalloc/include/jemalloc/internal/base.h b/dep/jemalloc/include/jemalloc/internal/base.h index 9b2c9fb10..f71a874c2 100644 --- a/dep/jemalloc/include/jemalloc/internal/base.h +++ b/dep/jemalloc/include/jemalloc/internal/base.h @@ -1,12 +1,19 @@ #ifndef JEMALLOC_INTERNAL_BASE_H #define JEMALLOC_INTERNAL_BASE_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/edata.h" #include "jemalloc/internal/ehooks.h" #include "jemalloc/internal/mutex.h" +/* + * Alignment when THP is not enabled. Set to constant 2M in case the HUGEPAGE + * value is unexpected high (which would cause VM over-reservation). + */ +#define BASE_BLOCK_MIN_ALIGN ((size_t)2 << 20) + enum metadata_thp_mode_e { - metadata_thp_disabled = 0, + metadata_thp_disabled = 0, /* * Lazily enable hugepage for metadata. To avoid high RSS caused by THP * + low usage arena (i.e. THP becomes a significant percentage), the @@ -15,16 +22,15 @@ enum metadata_thp_mode_e { * arena), "auto" behaves the same as "always", i.e. madvise hugepage * right away. */ - metadata_thp_auto = 1, - metadata_thp_always = 2, + metadata_thp_auto = 1, + metadata_thp_always = 2, metadata_thp_mode_limit = 3 }; typedef enum metadata_thp_mode_e metadata_thp_mode_t; #define METADATA_THP_DEFAULT metadata_thp_disabled extern metadata_thp_mode_t opt_metadata_thp; -extern const char *metadata_thp_mode_names[]; - +extern const char *const metadata_thp_mode_names[]; /* Embedded at the beginning of every block of base-managed virtual memory. */ typedef struct base_block_s base_block_t; @@ -72,8 +78,13 @@ struct base_s { /* Heap of extents that track unused trailing space within blocks. */ edata_heap_t avail[SC_NSIZES]; + /* Contains reusable base edata (used by tcache_stacks currently). */ + edata_avail_t edata_avail; + /* Stats, only maintained if config_stats. */ size_t allocated; + size_t edata_allocated; + size_t rtree_allocated; size_t resident; size_t mapped; /* Number of THP regions touched. */ @@ -91,20 +102,24 @@ metadata_thp_enabled(void) { } base_t *b0get(void); -base_t *base_new(tsdn_t *tsdn, unsigned ind, - const extent_hooks_t *extent_hooks, bool metadata_use_hooks); -void base_delete(tsdn_t *tsdn, base_t *base); -ehooks_t *base_ehooks_get(base_t *base); -ehooks_t *base_ehooks_get_for_metadata(base_t *base); -extent_hooks_t *base_extent_hooks_set(base_t *base, - extent_hooks_t *extent_hooks); -void *base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment); +base_t *base_new(tsdn_t *tsdn, unsigned ind, const extent_hooks_t *extent_hooks, + bool metadata_use_hooks); +void base_delete(tsdn_t *tsdn, base_t *base); +ehooks_t *base_ehooks_get(base_t *base); +ehooks_t *base_ehooks_get_for_metadata(base_t *base); +extent_hooks_t *base_extent_hooks_set( + base_t *base, extent_hooks_t *extent_hooks); +void *base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment); edata_t *base_alloc_edata(tsdn_t *tsdn, base_t *base); -void base_stats_get(tsdn_t *tsdn, base_t *base, size_t *allocated, - size_t *resident, size_t *mapped, size_t *n_thp); -void base_prefork(tsdn_t *tsdn, base_t *base); -void base_postfork_parent(tsdn_t *tsdn, base_t *base); -void base_postfork_child(tsdn_t *tsdn, base_t *base); -bool base_boot(tsdn_t *tsdn); +void *base_alloc_rtree(tsdn_t *tsdn, base_t *base, size_t size); +void *b0_alloc_tcache_stack(tsdn_t *tsdn, size_t size); +void b0_dalloc_tcache_stack(tsdn_t *tsdn, void *tcache_stack); +void base_stats_get(tsdn_t *tsdn, base_t *base, size_t *allocated, + size_t *edata_allocated, size_t *rtree_allocated, size_t *resident, + size_t *mapped, size_t *n_thp); +void base_prefork(tsdn_t *tsdn, base_t *base); +void base_postfork_parent(tsdn_t *tsdn, base_t *base); +void base_postfork_child(tsdn_t *tsdn, base_t *base); +bool base_boot(tsdn_t *tsdn); #endif /* JEMALLOC_INTERNAL_BASE_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/bin.h b/dep/jemalloc/include/jemalloc/internal/bin.h index 63f97395e..51d4c89ef 100644 --- a/dep/jemalloc/include/jemalloc/internal/bin.h +++ b/dep/jemalloc/include/jemalloc/internal/bin.h @@ -1,6 +1,8 @@ #ifndef JEMALLOC_INTERNAL_BIN_H #define JEMALLOC_INTERNAL_BIN_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/bin_info.h" #include "jemalloc/internal/bin_stats.h" #include "jemalloc/internal/bin_types.h" #include "jemalloc/internal/edata.h" @@ -14,13 +16,13 @@ typedef struct bin_s bin_t; struct bin_s { /* All operations on bin_t fields require lock ownership. */ - malloc_mutex_t lock; + malloc_mutex_t lock; /* * Bin statistics. These get touched every time the lock is acquired, * so put them close by in the hopes of getting some cache locality. */ - bin_stats_t stats; + bin_stats_t stats; /* * Current slab being used to service allocations of this bin's size @@ -28,17 +30,17 @@ struct bin_s { * slabcur is reassigned, the previous slab must be deallocated or * inserted into slabs_{nonfull,full}. */ - edata_t *slabcur; + edata_t *slabcur; /* * Heap of non-full slabs. This heap is used to assure that new * allocations come from the non-full slab that is oldest/lowest in * memory. */ - edata_heap_t slabs_nonfull; + edata_heap_t slabs_nonfull; /* List used to track full slabs. */ - edata_list_active_t slabs_full; + edata_list_active_t slabs_full; }; /* A set of sharded bins of the same size class. */ @@ -48,7 +50,7 @@ struct bins_s { bin_t *bin_shards; }; -void bin_shard_sizes_boot(unsigned bin_shards[SC_NBINS]); +void bin_shard_sizes_boot(unsigned bin_shard_sizes[SC_NBINS]); bool bin_update_shard_size(unsigned bin_shards[SC_NBINS], size_t start_size, size_t end_size, size_t nshards); @@ -60,6 +62,43 @@ void bin_prefork(tsdn_t *tsdn, bin_t *bin); void bin_postfork_parent(tsdn_t *tsdn, bin_t *bin); void bin_postfork_child(tsdn_t *tsdn, bin_t *bin); +/* Slab region allocation. */ +void *bin_slab_reg_alloc(edata_t *slab, const bin_info_t *bin_info); +void bin_slab_reg_alloc_batch( + edata_t *slab, const bin_info_t *bin_info, unsigned cnt, void **ptrs); + +/* Slab list management. */ +void bin_slabs_nonfull_insert(bin_t *bin, edata_t *slab); +void bin_slabs_nonfull_remove(bin_t *bin, edata_t *slab); +edata_t *bin_slabs_nonfull_tryget(bin_t *bin); +void bin_slabs_full_insert(bool is_auto, bin_t *bin, edata_t *slab); +void bin_slabs_full_remove(bool is_auto, bin_t *bin, edata_t *slab); + +/* Slab association / demotion. */ +void bin_dissociate_slab(bool is_auto, edata_t *slab, bin_t *bin); +void bin_lower_slab(tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin); + +/* Deallocation helpers (called under bin lock). */ +void bin_dalloc_slab_prepare(tsdn_t *tsdn, edata_t *slab, bin_t *bin); +void bin_dalloc_locked_handle_newly_empty( + tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin); +void bin_dalloc_locked_handle_newly_nonempty( + tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin); + +/* Slabcur refill and allocation. */ +void bin_refill_slabcur_with_fresh_slab(tsdn_t *tsdn, bin_t *bin, + szind_t binind, edata_t *fresh_slab); +void *bin_malloc_with_fresh_slab(tsdn_t *tsdn, bin_t *bin, + szind_t binind, edata_t *fresh_slab); +bool bin_refill_slabcur_no_fresh_slab(tsdn_t *tsdn, bool is_auto, + bin_t *bin); +void *bin_malloc_no_fresh_slab(tsdn_t *tsdn, bool is_auto, bin_t *bin, + szind_t binind); + +/* Bin selection. */ +bin_t *bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind, + unsigned *binshard_p); + /* Stats. */ static inline void bin_stats_merge(tsdn_t *tsdn, bin_stats_data_t *dst_bin_stats, bin_t *bin) { diff --git a/dep/jemalloc/include/jemalloc/internal/bin_info.h b/dep/jemalloc/include/jemalloc/internal/bin_info.h index 7fe65c866..8c563deeb 100644 --- a/dep/jemalloc/include/jemalloc/internal/bin_info.h +++ b/dep/jemalloc/include/jemalloc/internal/bin_info.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_BIN_INFO_H #define JEMALLOC_INTERNAL_BIN_INFO_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/bitmap.h" /* @@ -25,22 +26,22 @@ typedef struct bin_info_s bin_info_t; struct bin_info_s { /* Size of regions in a slab for this bin's size class. */ - size_t reg_size; + size_t reg_size; /* Total size of a slab for this bin's size class. */ - size_t slab_size; + size_t slab_size; /* Total number of regions in a slab for this bin's size class. */ - uint32_t nregs; + uint32_t nregs; /* Number of sharded bins in each arena for this size class. */ - uint32_t n_shards; + uint32_t n_shards; /* * Metadata used to manipulate bitmaps for slabs associated with this * bin. */ - bitmap_info_t bitmap_info; + bitmap_info_t bitmap_info; }; extern bin_info_t bin_infos[SC_NBINS]; diff --git a/dep/jemalloc/include/jemalloc/internal/bin_inlines.h b/dep/jemalloc/include/jemalloc/internal/bin_inlines.h new file mode 100644 index 000000000..f42911698 --- /dev/null +++ b/dep/jemalloc/include/jemalloc/internal/bin_inlines.h @@ -0,0 +1,112 @@ +#ifndef JEMALLOC_INTERNAL_BIN_INLINES_H +#define JEMALLOC_INTERNAL_BIN_INLINES_H + +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/bin.h" +#include "jemalloc/internal/bin_info.h" +#include "jemalloc/internal/bitmap.h" +#include "jemalloc/internal/div.h" +#include "jemalloc/internal/edata.h" +#include "jemalloc/internal/sc.h" + +/* + * The dalloc bin info contains just the information that the common paths need + * during tcache flushes. By force-inlining these paths, and using local copies + * of data (so that the compiler knows it's constant), we avoid a whole bunch of + * redundant loads and stores by leaving this information in registers. + */ +typedef struct bin_dalloc_locked_info_s bin_dalloc_locked_info_t; +struct bin_dalloc_locked_info_s { + div_info_t div_info; + uint32_t nregs; + uint64_t ndalloc; +}; + +/* Find the region index of a pointer within a slab. */ +JEMALLOC_ALWAYS_INLINE size_t +bin_slab_regind_impl( + div_info_t *div_info, szind_t binind, edata_t *slab, const void *ptr) { + size_t diff, regind; + + /* Freeing a pointer outside the slab can cause assertion failure. */ + assert((uintptr_t)ptr >= (uintptr_t)edata_addr_get(slab)); + assert((uintptr_t)ptr < (uintptr_t)edata_past_get(slab)); + /* Freeing an interior pointer can cause assertion failure. */ + assert(((uintptr_t)ptr - (uintptr_t)edata_addr_get(slab)) + % (uintptr_t)bin_infos[binind].reg_size + == 0); + + diff = (size_t)((uintptr_t)ptr - (uintptr_t)edata_addr_get(slab)); + + /* Avoid doing division with a variable divisor. */ + regind = div_compute(div_info, diff); + assert(regind < bin_infos[binind].nregs); + return regind; +} + +JEMALLOC_ALWAYS_INLINE size_t +bin_slab_regind(bin_dalloc_locked_info_t *info, szind_t binind, + edata_t *slab, const void *ptr) { + size_t regind = bin_slab_regind_impl( + &info->div_info, binind, slab, ptr); + return regind; +} + +JEMALLOC_ALWAYS_INLINE void +bin_dalloc_locked_begin( + bin_dalloc_locked_info_t *info, szind_t binind) { + info->div_info = arena_binind_div_info[binind]; + info->nregs = bin_infos[binind].nregs; + info->ndalloc = 0; +} + +/* + * Does the deallocation work associated with freeing a single pointer (a + * "step") in between a bin_dalloc_locked begin and end call. + * + * Returns true if arena_slab_dalloc must be called on slab. Doesn't do + * stats updates, which happen during finish (this lets running counts get left + * in a register). + */ +JEMALLOC_ALWAYS_INLINE bool +bin_dalloc_locked_step(tsdn_t *tsdn, bool is_auto, bin_t *bin, + bin_dalloc_locked_info_t *info, szind_t binind, edata_t *slab, + void *ptr) { + const bin_info_t *bin_info = &bin_infos[binind]; + size_t regind = bin_slab_regind(info, binind, slab, ptr); + slab_data_t *slab_data = edata_slab_data_get(slab); + + assert(edata_nfree_get(slab) < bin_info->nregs); + /* Freeing an unallocated pointer can cause assertion failure. */ + assert(bitmap_get(slab_data->bitmap, &bin_info->bitmap_info, regind)); + + bitmap_unset(slab_data->bitmap, &bin_info->bitmap_info, regind); + edata_nfree_inc(slab); + + if (config_stats) { + info->ndalloc++; + } + + unsigned nfree = edata_nfree_get(slab); + if (nfree == bin_info->nregs) { + bin_dalloc_locked_handle_newly_empty( + tsdn, is_auto, slab, bin); + return true; + } else if (nfree == 1 && slab != bin->slabcur) { + bin_dalloc_locked_handle_newly_nonempty( + tsdn, is_auto, slab, bin); + } + return false; +} + +JEMALLOC_ALWAYS_INLINE void +bin_dalloc_locked_finish(tsdn_t *tsdn, bin_t *bin, + bin_dalloc_locked_info_t *info) { + if (config_stats) { + bin->stats.ndalloc += info->ndalloc; + assert(bin->stats.curregs >= (size_t)info->ndalloc); + bin->stats.curregs -= (size_t)info->ndalloc; + } +} + +#endif /* JEMALLOC_INTERNAL_BIN_INLINES_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/bin_stats.h b/dep/jemalloc/include/jemalloc/internal/bin_stats.h index 0b99297c0..9900e0d18 100644 --- a/dep/jemalloc/include/jemalloc/internal/bin_stats.h +++ b/dep/jemalloc/include/jemalloc/internal/bin_stats.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_BIN_STATS_H #define JEMALLOC_INTERNAL_BIN_STATS_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/mutex_prof.h" typedef struct bin_stats_s bin_stats_t; @@ -11,47 +12,47 @@ struct bin_stats_s { * many times, resulting many increments to nrequests, but only one * each to nmalloc and ndalloc. */ - uint64_t nmalloc; - uint64_t ndalloc; + uint64_t nmalloc; + uint64_t ndalloc; /* * Number of allocation requests that correspond to the size of this * bin. This includes requests served by tcache, though tcache only * periodically merges into this counter. */ - uint64_t nrequests; + uint64_t nrequests; /* * Current number of regions of this size class, including regions * currently cached by tcache. */ - size_t curregs; + size_t curregs; /* Number of tcache fills from this bin. */ - uint64_t nfills; + uint64_t nfills; /* Number of tcache flushes to this bin. */ - uint64_t nflushes; + uint64_t nflushes; /* Total number of slabs created for this bin's size class. */ - uint64_t nslabs; + uint64_t nslabs; /* * Total number of slabs reused by extracting them from the slabs heap * for this bin's size class. */ - uint64_t reslabs; + uint64_t reslabs; /* Current number of slabs in this bin. */ - size_t curslabs; + size_t curslabs; /* Current size of nonfull slabs heap in this bin. */ - size_t nonfull_slabs; + size_t nonfull_slabs; }; typedef struct bin_stats_data_s bin_stats_data_t; struct bin_stats_data_s { - bin_stats_t stats_data; + bin_stats_t stats_data; mutex_prof_data_t mutex_data; }; #endif /* JEMALLOC_INTERNAL_BIN_STATS_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/bin_types.h b/dep/jemalloc/include/jemalloc/internal/bin_types.h index 945e8326c..b6bad37e5 100644 --- a/dep/jemalloc/include/jemalloc/internal/bin_types.h +++ b/dep/jemalloc/include/jemalloc/internal/bin_types.h @@ -1,13 +1,17 @@ #ifndef JEMALLOC_INTERNAL_BIN_TYPES_H #define JEMALLOC_INTERNAL_BIN_TYPES_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/sc.h" #define BIN_SHARDS_MAX (1 << EDATA_BITS_BINSHARD_WIDTH) #define N_BIN_SHARDS_DEFAULT 1 /* Used in TSD static initializer only. Real init in arena_bind(). */ -#define TSD_BINSHARDS_ZERO_INITIALIZER {{UINT8_MAX}} +#define TSD_BINSHARDS_ZERO_INITIALIZER \ + { \ + { UINT8_MAX } \ + } typedef struct tsd_binshards_s tsd_binshards_t; struct tsd_binshards_s { diff --git a/dep/jemalloc/include/jemalloc/internal/bit_util.h b/dep/jemalloc/include/jemalloc/internal/bit_util.h index bac59140f..cfd95c4e8 100644 --- a/dep/jemalloc/include/jemalloc/internal/bit_util.h +++ b/dep/jemalloc/include/jemalloc/internal/bit_util.h @@ -1,12 +1,13 @@ #ifndef JEMALLOC_INTERNAL_BIT_UTIL_H #define JEMALLOC_INTERNAL_BIT_UTIL_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/assert.h" /* Sanity check. */ -#if !defined(JEMALLOC_INTERNAL_FFSLL) || !defined(JEMALLOC_INTERNAL_FFSL) \ +#if !defined(JEMALLOC_INTERNAL_FFSLL) || !defined(JEMALLOC_INTERNAL_FFSL) \ || !defined(JEMALLOC_INTERNAL_FFS) -# error JEMALLOC_INTERNAL_FFS{,L,LL} should have been defined by configure +# error JEMALLOC_INTERNAL_FFS{,L,LL} should have been defined by configure #endif /* @@ -34,6 +35,7 @@ ffs_u(unsigned x) { return JEMALLOC_INTERNAL_FFS(x) - 1; } +/* clang-format off */ #define DO_FLS_SLOW(x, suffix) do { \ util_assume(x != 0); \ x |= (x >> 1); \ @@ -57,6 +59,7 @@ ffs_u(unsigned x) { } \ return ffs_##suffix(x) - 1; \ } while(0) +/* clang-format on */ static inline unsigned fls_llu_slow(unsigned long long x) { @@ -107,16 +110,19 @@ fls_u(unsigned x) { } #elif defined(_MSC_VER) -#if LG_SIZEOF_PTR == 3 -#define DO_BSR64(bit, x) _BitScanReverse64(&bit, x) -#else +# if LG_SIZEOF_PTR == 3 +# define DO_BSR64(bit, x) _BitScanReverse64(&bit, x) +# else /* * This never actually runs; we're just dodging a compiler error for the * never-taken branch where sizeof(void *) == 8. */ -#define DO_BSR64(bit, x) bit = 0; unreachable() -#endif +# define DO_BSR64(bit, x) \ + bit = 0; \ + unreachable() +# endif +/* clang-format off */ #define DO_FLS(x) do { \ if (x == 0) { \ return 8 * sizeof(x); \ @@ -143,6 +149,7 @@ fls_u(unsigned x) { } \ unreachable(); \ } while (0) +/* clang-format on */ static inline unsigned fls_llu(unsigned long long x) { @@ -159,8 +166,8 @@ fls_u(unsigned x) { DO_FLS(x); } -#undef DO_FLS -#undef DO_BSR64 +# undef DO_FLS +# undef DO_BSR64 #else static inline unsigned @@ -180,14 +187,15 @@ fls_u(unsigned x) { #endif #if LG_SIZEOF_LONG_LONG > 3 -# error "Haven't implemented popcount for 16-byte ints." +# error "Haven't implemented popcount for 16-byte ints." #endif +/* clang-format off */ #define DO_POPCOUNT(x, type) do { \ /* \ * Algorithm from an old AMD optimization reference manual. \ * We're putting a little bit more work than you might expect \ - * into the no-instrinsic case, since we only support the \ + * into the no-intrinsic case, since we only support the \ * GCC intrinsics spelling of popcount (for now). Detecting \ * whether or not the popcount builtin is actually useable in \ * MSVC is nontrivial. \ @@ -204,7 +212,7 @@ fls_u(unsigned x) { * That costs an extra mask relative to this, though. \ */ \ x = x - ((x >> 1) & (0x55U * bmul)); \ - /* Replace each 4 bits with their sideays sum. 0x3 = 0b0011. */\ + /* Replace each 4 bits with their sideways sum. 0x3 = 0b0011. */\ x = (x & (bmul * 0x33U)) + ((x >> 2) & (bmul * 0x33U)); \ /* \ * Replace each 8 bits with their sideways sum. Note that we \ @@ -226,6 +234,7 @@ fls_u(unsigned x) { x >>= ((sizeof(x) - 1) * 8); \ return (unsigned)x; \ } while(0) +/* clang-format on */ static inline unsigned popcount_u_slow(unsigned bitmap) { @@ -277,7 +286,7 @@ popcount_llu(unsigned long long bitmap) { */ static inline size_t -cfs_lu(unsigned long* bitmap) { +cfs_lu(unsigned long *bitmap) { util_assume(*bitmap != 0); size_t bit = ffs_lu(*bitmap); *bitmap ^= ZU(1) << bit; @@ -293,7 +302,7 @@ ffs_zu(size_t x) { #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG_LONG return ffs_llu(x); #else -#error No implementation for size_t ffs() +# error No implementation for size_t ffs() #endif } @@ -306,11 +315,10 @@ fls_zu(size_t x) { #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG_LONG return fls_llu(x); #else -#error No implementation for size_t fls() +# error No implementation for size_t fls() #endif } - static inline unsigned ffs_u64(uint64_t x) { #if LG_SIZEOF_LONG == 3 @@ -318,7 +326,7 @@ ffs_u64(uint64_t x) { #elif LG_SIZEOF_LONG_LONG == 3 return ffs_llu(x); #else -#error No implementation for 64-bit ffs() +# error No implementation for 64-bit ffs() #endif } @@ -329,7 +337,7 @@ fls_u64(uint64_t x) { #elif LG_SIZEOF_LONG_LONG == 3 return fls_llu(x); #else -#error No implementation for 64-bit fls() +# error No implementation for 64-bit fls() #endif } @@ -338,9 +346,8 @@ ffs_u32(uint32_t x) { #if LG_SIZEOF_INT == 2 return ffs_u(x); #else -#error No implementation for 32-bit ffs() +# error No implementation for 32-bit ffs() #endif - return ffs_u(x); } static inline unsigned @@ -348,9 +355,8 @@ fls_u32(uint32_t x) { #if LG_SIZEOF_INT == 2 return fls_u(x); #else -#error No implementation for 32-bit fls() +# error No implementation for 32-bit fls() #endif - return fls_u(x); } static inline uint64_t @@ -370,7 +376,7 @@ pow2_ceil_u64(uint64_t x) { static inline uint32_t pow2_ceil_u32(uint32_t x) { if (unlikely(x <= 1)) { - return x; + return x; } size_t msb_on_index = fls_u32(x - 1); /* As above. */ @@ -408,13 +414,16 @@ lg_ceil(size_t x) { #define LG_FLOOR_2(x) (x < (1ULL << 1) ? LG_FLOOR_1(x) : 1 + LG_FLOOR_1(x >> 1)) #define LG_FLOOR_4(x) (x < (1ULL << 2) ? LG_FLOOR_2(x) : 2 + LG_FLOOR_2(x >> 2)) #define LG_FLOOR_8(x) (x < (1ULL << 4) ? LG_FLOOR_4(x) : 4 + LG_FLOOR_4(x >> 4)) -#define LG_FLOOR_16(x) (x < (1ULL << 8) ? LG_FLOOR_8(x) : 8 + LG_FLOOR_8(x >> 8)) -#define LG_FLOOR_32(x) (x < (1ULL << 16) ? LG_FLOOR_16(x) : 16 + LG_FLOOR_16(x >> 16)) -#define LG_FLOOR_64(x) (x < (1ULL << 32) ? LG_FLOOR_32(x) : 32 + LG_FLOOR_32(x >> 32)) +#define LG_FLOOR_16(x) \ + (x < (1ULL << 8) ? LG_FLOOR_8(x) : 8 + LG_FLOOR_8(x >> 8)) +#define LG_FLOOR_32(x) \ + (x < (1ULL << 16) ? LG_FLOOR_16(x) : 16 + LG_FLOOR_16(x >> 16)) +#define LG_FLOOR_64(x) \ + (x < (1ULL << 32) ? LG_FLOOR_32(x) : 32 + LG_FLOOR_32(x >> 32)) #if LG_SIZEOF_PTR == 2 -# define LG_FLOOR(x) LG_FLOOR_32((x)) +# define LG_FLOOR(x) LG_FLOOR_32((x)) #else -# define LG_FLOOR(x) LG_FLOOR_64((x)) +# define LG_FLOOR(x) LG_FLOOR_64((x)) #endif #define LG_CEIL(x) (LG_FLOOR(x) + (((x) & ((x) - 1)) == 0 ? 0 : 1)) diff --git a/dep/jemalloc/include/jemalloc/internal/bitmap.h b/dep/jemalloc/include/jemalloc/internal/bitmap.h index dc19454d4..e0f596fba 100644 --- a/dep/jemalloc/include/jemalloc/internal/bitmap.h +++ b/dep/jemalloc/include/jemalloc/internal/bitmap.h @@ -1,26 +1,27 @@ #ifndef JEMALLOC_INTERNAL_BITMAP_H #define JEMALLOC_INTERNAL_BITMAP_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/bit_util.h" #include "jemalloc/internal/sc.h" typedef unsigned long bitmap_t; -#define LG_SIZEOF_BITMAP LG_SIZEOF_LONG +#define LG_SIZEOF_BITMAP LG_SIZEOF_LONG /* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */ #if SC_LG_SLAB_MAXREGS > LG_CEIL(SC_NSIZES) /* Maximum bitmap bit count is determined by maximum regions per slab. */ -# define LG_BITMAP_MAXBITS SC_LG_SLAB_MAXREGS +# define LG_BITMAP_MAXBITS SC_LG_SLAB_MAXREGS #else /* Maximum bitmap bit count is determined by number of extent size classes. */ -# define LG_BITMAP_MAXBITS LG_CEIL(SC_NSIZES) +# define LG_BITMAP_MAXBITS LG_CEIL(SC_NSIZES) #endif -#define BITMAP_MAXBITS (ZU(1) << LG_BITMAP_MAXBITS) +#define BITMAP_MAXBITS (ZU(1) << LG_BITMAP_MAXBITS) /* Number of bits per group. */ -#define LG_BITMAP_GROUP_NBITS (LG_SIZEOF_BITMAP + 3) -#define BITMAP_GROUP_NBITS (1U << LG_BITMAP_GROUP_NBITS) -#define BITMAP_GROUP_NBITS_MASK (BITMAP_GROUP_NBITS-1) +#define LG_BITMAP_GROUP_NBITS (LG_SIZEOF_BITMAP + 3) +#define BITMAP_GROUP_NBITS (1U << LG_BITMAP_GROUP_NBITS) +#define BITMAP_GROUP_NBITS_MASK (BITMAP_GROUP_NBITS - 1) /* * Do some analysis on how big the bitmap is before we use a tree. For a brute @@ -28,67 +29,64 @@ typedef unsigned long bitmap_t; * use a tree instead. */ #if LG_BITMAP_MAXBITS - LG_BITMAP_GROUP_NBITS > 3 -# define BITMAP_USE_TREE +# define BITMAP_USE_TREE #endif /* Number of groups required to store a given number of bits. */ -#define BITMAP_BITS2GROUPS(nbits) \ - (((nbits) + BITMAP_GROUP_NBITS_MASK) >> LG_BITMAP_GROUP_NBITS) +#define BITMAP_BITS2GROUPS(nbits) \ + (((nbits) + BITMAP_GROUP_NBITS_MASK) >> LG_BITMAP_GROUP_NBITS) /* * Number of groups required at a particular level for a given number of bits. */ -#define BITMAP_GROUPS_L0(nbits) \ - BITMAP_BITS2GROUPS(nbits) -#define BITMAP_GROUPS_L1(nbits) \ - BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(nbits)) -#define BITMAP_GROUPS_L2(nbits) \ - BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS((nbits)))) -#define BITMAP_GROUPS_L3(nbits) \ - BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS( \ - BITMAP_BITS2GROUPS((nbits))))) -#define BITMAP_GROUPS_L4(nbits) \ - BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS( \ - BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS((nbits)))))) +#define BITMAP_GROUPS_L0(nbits) BITMAP_BITS2GROUPS(nbits) +#define BITMAP_GROUPS_L1(nbits) BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(nbits)) +#define BITMAP_GROUPS_L2(nbits) \ + BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS((nbits)))) +#define BITMAP_GROUPS_L3(nbits) \ + BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS( \ + BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS((nbits))))) +#define BITMAP_GROUPS_L4(nbits) \ + BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS( \ + BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS((nbits)))))) /* * Assuming the number of levels, number of groups required for a given number * of bits. */ -#define BITMAP_GROUPS_1_LEVEL(nbits) \ - BITMAP_GROUPS_L0(nbits) -#define BITMAP_GROUPS_2_LEVEL(nbits) \ - (BITMAP_GROUPS_1_LEVEL(nbits) + BITMAP_GROUPS_L1(nbits)) -#define BITMAP_GROUPS_3_LEVEL(nbits) \ - (BITMAP_GROUPS_2_LEVEL(nbits) + BITMAP_GROUPS_L2(nbits)) -#define BITMAP_GROUPS_4_LEVEL(nbits) \ - (BITMAP_GROUPS_3_LEVEL(nbits) + BITMAP_GROUPS_L3(nbits)) -#define BITMAP_GROUPS_5_LEVEL(nbits) \ - (BITMAP_GROUPS_4_LEVEL(nbits) + BITMAP_GROUPS_L4(nbits)) +#define BITMAP_GROUPS_1_LEVEL(nbits) BITMAP_GROUPS_L0(nbits) +#define BITMAP_GROUPS_2_LEVEL(nbits) \ + (BITMAP_GROUPS_1_LEVEL(nbits) + BITMAP_GROUPS_L1(nbits)) +#define BITMAP_GROUPS_3_LEVEL(nbits) \ + (BITMAP_GROUPS_2_LEVEL(nbits) + BITMAP_GROUPS_L2(nbits)) +#define BITMAP_GROUPS_4_LEVEL(nbits) \ + (BITMAP_GROUPS_3_LEVEL(nbits) + BITMAP_GROUPS_L3(nbits)) +#define BITMAP_GROUPS_5_LEVEL(nbits) \ + (BITMAP_GROUPS_4_LEVEL(nbits) + BITMAP_GROUPS_L4(nbits)) /* * Maximum number of groups required to support LG_BITMAP_MAXBITS. */ #ifdef BITMAP_USE_TREE -#if LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS -# define BITMAP_GROUPS(nbits) BITMAP_GROUPS_1_LEVEL(nbits) -# define BITMAP_GROUPS_MAX BITMAP_GROUPS_1_LEVEL(BITMAP_MAXBITS) -#elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 2 -# define BITMAP_GROUPS(nbits) BITMAP_GROUPS_2_LEVEL(nbits) -# define BITMAP_GROUPS_MAX BITMAP_GROUPS_2_LEVEL(BITMAP_MAXBITS) -#elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 3 -# define BITMAP_GROUPS(nbits) BITMAP_GROUPS_3_LEVEL(nbits) -# define BITMAP_GROUPS_MAX BITMAP_GROUPS_3_LEVEL(BITMAP_MAXBITS) -#elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 4 -# define BITMAP_GROUPS(nbits) BITMAP_GROUPS_4_LEVEL(nbits) -# define BITMAP_GROUPS_MAX BITMAP_GROUPS_4_LEVEL(BITMAP_MAXBITS) -#elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 5 -# define BITMAP_GROUPS(nbits) BITMAP_GROUPS_5_LEVEL(nbits) -# define BITMAP_GROUPS_MAX BITMAP_GROUPS_5_LEVEL(BITMAP_MAXBITS) -#else -# error "Unsupported bitmap size" -#endif +# if LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS +# define BITMAP_GROUPS(nbits) BITMAP_GROUPS_1_LEVEL(nbits) +# define BITMAP_GROUPS_MAX BITMAP_GROUPS_1_LEVEL(BITMAP_MAXBITS) +# elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 2 +# define BITMAP_GROUPS(nbits) BITMAP_GROUPS_2_LEVEL(nbits) +# define BITMAP_GROUPS_MAX BITMAP_GROUPS_2_LEVEL(BITMAP_MAXBITS) +# elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 3 +# define BITMAP_GROUPS(nbits) BITMAP_GROUPS_3_LEVEL(nbits) +# define BITMAP_GROUPS_MAX BITMAP_GROUPS_3_LEVEL(BITMAP_MAXBITS) +# elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 4 +# define BITMAP_GROUPS(nbits) BITMAP_GROUPS_4_LEVEL(nbits) +# define BITMAP_GROUPS_MAX BITMAP_GROUPS_4_LEVEL(BITMAP_MAXBITS) +# elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 5 +# define BITMAP_GROUPS(nbits) BITMAP_GROUPS_5_LEVEL(nbits) +# define BITMAP_GROUPS_MAX BITMAP_GROUPS_5_LEVEL(BITMAP_MAXBITS) +# else +# error "Unsupported bitmap size" +# endif /* * Maximum number of levels possible. This could be statically computed based @@ -104,42 +102,53 @@ typedef unsigned long bitmap_t; * unused trailing entries in bitmap_info_t structures; the bitmaps themselves * are not impacted. */ -#define BITMAP_MAX_LEVELS 5 +# define BITMAP_MAX_LEVELS 5 -#define BITMAP_INFO_INITIALIZER(nbits) { \ - /* nbits. */ \ - nbits, \ - /* nlevels. */ \ - (BITMAP_GROUPS_L0(nbits) > BITMAP_GROUPS_L1(nbits)) + \ - (BITMAP_GROUPS_L1(nbits) > BITMAP_GROUPS_L2(nbits)) + \ - (BITMAP_GROUPS_L2(nbits) > BITMAP_GROUPS_L3(nbits)) + \ - (BITMAP_GROUPS_L3(nbits) > BITMAP_GROUPS_L4(nbits)) + 1, \ - /* levels. */ \ - { \ - {0}, \ - {BITMAP_GROUPS_L0(nbits)}, \ - {BITMAP_GROUPS_L1(nbits) + BITMAP_GROUPS_L0(nbits)}, \ - {BITMAP_GROUPS_L2(nbits) + BITMAP_GROUPS_L1(nbits) + \ - BITMAP_GROUPS_L0(nbits)}, \ - {BITMAP_GROUPS_L3(nbits) + BITMAP_GROUPS_L2(nbits) + \ - BITMAP_GROUPS_L1(nbits) + BITMAP_GROUPS_L0(nbits)}, \ - {BITMAP_GROUPS_L4(nbits) + BITMAP_GROUPS_L3(nbits) + \ - BITMAP_GROUPS_L2(nbits) + BITMAP_GROUPS_L1(nbits) \ - + BITMAP_GROUPS_L0(nbits)} \ - } \ -} +# define BITMAP_INFO_INITIALIZER(nbits) \ + { \ + /* nbits. */ \ + nbits, /* nlevels. */ \ + (BITMAP_GROUPS_L0(nbits) \ + > BITMAP_GROUPS_L1(nbits)) \ + + (BITMAP_GROUPS_L1(nbits) \ + > BITMAP_GROUPS_L2(nbits)) \ + + (BITMAP_GROUPS_L2(nbits) \ + > BITMAP_GROUPS_L3(nbits)) \ + + (BITMAP_GROUPS_L3(nbits) \ + > BITMAP_GROUPS_L4(nbits)) \ + + 1, /* levels. */ \ + { \ + {0}, {BITMAP_GROUPS_L0(nbits)}, \ + {BITMAP_GROUPS_L1(nbits) \ + + BITMAP_GROUPS_L0(nbits)}, \ + {BITMAP_GROUPS_L2(nbits) \ + + BITMAP_GROUPS_L1(nbits) \ + + BITMAP_GROUPS_L0(nbits)}, \ + {BITMAP_GROUPS_L3(nbits) \ + + BITMAP_GROUPS_L2(nbits) \ + + BITMAP_GROUPS_L1(nbits) \ + + BITMAP_GROUPS_L0(nbits)}, \ + { \ + BITMAP_GROUPS_L4(nbits) \ + + BITMAP_GROUPS_L3(nbits) \ + + BITMAP_GROUPS_L2(nbits) \ + + BITMAP_GROUPS_L1(nbits) \ + + BITMAP_GROUPS_L0(nbits) \ + } \ + } \ + } #else /* BITMAP_USE_TREE */ -#define BITMAP_GROUPS(nbits) BITMAP_BITS2GROUPS(nbits) -#define BITMAP_GROUPS_MAX BITMAP_BITS2GROUPS(BITMAP_MAXBITS) +# define BITMAP_GROUPS(nbits) BITMAP_BITS2GROUPS(nbits) +# define BITMAP_GROUPS_MAX BITMAP_BITS2GROUPS(BITMAP_MAXBITS) -#define BITMAP_INFO_INITIALIZER(nbits) { \ - /* nbits. */ \ - nbits, \ - /* ngroups. */ \ - BITMAP_BITS2GROUPS(nbits) \ -} +# define BITMAP_INFO_INITIALIZER(nbits) \ + { \ + /* nbits. */ \ + nbits, /* ngroups. */ \ + BITMAP_BITS2GROUPS(nbits) \ + } #endif /* BITMAP_USE_TREE */ @@ -160,21 +169,21 @@ typedef struct bitmap_info_s { * Only the first (nlevels+1) elements are used, and levels are ordered * bottom to top (e.g. the bottom level is stored in levels[0]). */ - bitmap_level_t levels[BITMAP_MAX_LEVELS+1]; -#else /* BITMAP_USE_TREE */ + bitmap_level_t levels[BITMAP_MAX_LEVELS + 1]; +#else /* BITMAP_USE_TREE */ /* Number of groups necessary for nbits. */ size_t ngroups; #endif /* BITMAP_USE_TREE */ } bitmap_info_t; -void bitmap_info_init(bitmap_info_t *binfo, size_t nbits); -void bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo, bool fill); +void bitmap_info_init(bitmap_info_t *binfo, size_t nbits); +void bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo, bool fill); size_t bitmap_size(const bitmap_info_t *binfo); static inline bool bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo) { #ifdef BITMAP_USE_TREE - size_t rgoff = binfo->levels[binfo->nlevels].group_offset - 1; + size_t rgoff = binfo->levels[binfo->nlevels].group_offset - 1; bitmap_t rg = bitmap[rgoff]; /* The bitmap is full iff the root group is 0. */ return (rg == 0); @@ -192,7 +201,7 @@ bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo) { static inline bool bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) { - size_t goff; + size_t goff; bitmap_t g; assert(bit < binfo->nbits); @@ -203,9 +212,9 @@ bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) { static inline void bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) { - size_t goff; + size_t goff; bitmap_t *gp; - bitmap_t g; + bitmap_t g; assert(bit < binfo->nbits); assert(!bitmap_get(bitmap, binfo, bit)); @@ -244,12 +253,13 @@ bitmap_ffu(const bitmap_t *bitmap, const bitmap_info_t *binfo, size_t min_bit) { #ifdef BITMAP_USE_TREE size_t bit = 0; for (unsigned level = binfo->nlevels; level--;) { - size_t lg_bits_per_group = (LG_BITMAP_GROUP_NBITS * (level + - 1)); - bitmap_t group = bitmap[binfo->levels[level].group_offset + (bit - >> lg_bits_per_group)]; - unsigned group_nmask = (unsigned)(((min_bit > bit) ? (min_bit - - bit) : 0) >> (lg_bits_per_group - LG_BITMAP_GROUP_NBITS)); + size_t lg_bits_per_group = (LG_BITMAP_GROUP_NBITS + * (level + 1)); + bitmap_t group = bitmap[binfo->levels[level].group_offset + + (bit >> lg_bits_per_group)]; + unsigned group_nmask = + (unsigned)(((min_bit > bit) ? (min_bit - bit) : 0) + >> (lg_bits_per_group - LG_BITMAP_GROUP_NBITS)); assert(group_nmask <= BITMAP_GROUP_NBITS); bitmap_t group_mask = ~((1LU << group_nmask) - 1); bitmap_t group_masked = group & group_mask; @@ -272,25 +282,28 @@ bitmap_ffu(const bitmap_t *bitmap, const bitmap_info_t *binfo, size_t min_bit) { } return bitmap_ffu(bitmap, binfo, sib_base); } - bit += ((size_t)ffs_lu(group_masked)) << - (lg_bits_per_group - LG_BITMAP_GROUP_NBITS); + bit += ((size_t)ffs_lu(group_masked)) + << (lg_bits_per_group - LG_BITMAP_GROUP_NBITS); } assert(bit >= min_bit); assert(bit < binfo->nbits); return bit; #else - size_t i = min_bit >> LG_BITMAP_GROUP_NBITS; - bitmap_t g = bitmap[i] & ~((1LU << (min_bit & BITMAP_GROUP_NBITS_MASK)) - - 1); + size_t i = min_bit >> LG_BITMAP_GROUP_NBITS; + bitmap_t g = bitmap[i] + & ~((1LU << (min_bit & BITMAP_GROUP_NBITS_MASK)) - 1); size_t bit; - do { + while (1) { if (g != 0) { bit = ffs_lu(g); return (i << LG_BITMAP_GROUP_NBITS) + bit; } i++; + if (i >= binfo->ngroups) { + break; + } g = bitmap[i]; - } while (i < binfo->ngroups); + } return binfo->nbits; #endif } @@ -298,7 +311,7 @@ bitmap_ffu(const bitmap_t *bitmap, const bitmap_info_t *binfo, size_t min_bit) { /* sfu: set first unset. */ static inline size_t bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo) { - size_t bit; + size_t bit; bitmap_t g; unsigned i; @@ -328,9 +341,9 @@ bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo) { static inline void bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) { - size_t goff; - bitmap_t *gp; - bitmap_t g; + size_t goff; + bitmap_t *gp; + bitmap_t g; UNUSED bool propagate; assert(bit < binfo->nbits); diff --git a/dep/jemalloc/include/jemalloc/internal/buf_writer.h b/dep/jemalloc/include/jemalloc/internal/buf_writer.h index 37aa6de5b..5ee9af4e9 100644 --- a/dep/jemalloc/include/jemalloc/internal/buf_writer.h +++ b/dep/jemalloc/include/jemalloc/internal/buf_writer.h @@ -1,6 +1,10 @@ #ifndef JEMALLOC_INTERNAL_BUF_WRITER_H #define JEMALLOC_INTERNAL_BUF_WRITER_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_types.h" +#include "jemalloc/internal/tsd_types.h" + /* * Note: when using the buffered writer, cbopaque is passed to write_cb only * when the buffer is flushed. It would make a difference if cbopaque points @@ -12,21 +16,21 @@ typedef struct { write_cb_t *write_cb; - void *cbopaque; - char *buf; - size_t buf_size; - size_t buf_end; - bool internal_buf; + void *cbopaque; + char *buf; + size_t buf_size; + size_t buf_end; + bool internal_buf; } buf_writer_t; -bool buf_writer_init(tsdn_t *tsdn, buf_writer_t *buf_writer, - write_cb_t *write_cb, void *cbopaque, char *buf, size_t buf_len); -void buf_writer_flush(buf_writer_t *buf_writer); +bool buf_writer_init(tsdn_t *tsdn, buf_writer_t *buf_writer, + write_cb_t *write_cb, void *cbopaque, char *buf, size_t buf_len); +void buf_writer_flush(buf_writer_t *buf_writer); write_cb_t buf_writer_cb; -void buf_writer_terminate(tsdn_t *tsdn, buf_writer_t *buf_writer); +void buf_writer_terminate(tsdn_t *tsdn, buf_writer_t *buf_writer); -typedef ssize_t (read_cb_t)(void *read_cbopaque, void *buf, size_t limit); -void buf_writer_pipe(buf_writer_t *buf_writer, read_cb_t *read_cb, - void *read_cbopaque); +typedef ssize_t(read_cb_t)(void *read_cbopaque, void *buf, size_t limit); +void buf_writer_pipe( + buf_writer_t *buf_writer, read_cb_t *read_cb, void *read_cbopaque); #endif /* JEMALLOC_INTERNAL_BUF_WRITER_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/cache_bin.h b/dep/jemalloc/include/jemalloc/internal/cache_bin.h index caf5be338..bea3a2fc0 100644 --- a/dep/jemalloc/include/jemalloc/internal/cache_bin.h +++ b/dep/jemalloc/include/jemalloc/internal/cache_bin.h @@ -1,7 +1,10 @@ #ifndef JEMALLOC_INTERNAL_CACHE_BIN_H #define JEMALLOC_INTERNAL_CACHE_BIN_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_externs.h" #include "jemalloc/internal/ql.h" +#include "jemalloc/internal/safety_check.h" #include "jemalloc/internal/sz.h" /* @@ -20,16 +23,20 @@ */ typedef uint16_t cache_bin_sz_t; +#define JUNK_ADDR ((uintptr_t)0x7a7a7a7a7a7a7a7aULL) /* * Leave a noticeable mark pattern on the cache bin stack boundaries, in case a * bug starts leaking those. Make it look like the junk pattern but be distinct * from it. */ -static const uintptr_t cache_bin_preceding_junk = - (uintptr_t)0x7a7a7a7a7a7a7a7aULL; -/* Note: a7 vs. 7a above -- this tells you which pointer leaked. */ -static const uintptr_t cache_bin_trailing_junk = - (uintptr_t)0xa7a7a7a7a7a7a7a7ULL; +static const uintptr_t cache_bin_preceding_junk = JUNK_ADDR; +/* Note: JUNK_ADDR vs. JUNK_ADDR + 1 -- this tells you which pointer leaked. */ +static const uintptr_t cache_bin_trailing_junk = JUNK_ADDR + 1; +/* + * A pointer used to initialize a fake stack_head for disabled small bins + * so that the enabled/disabled assessment does not rely on ncached_max. + */ +extern const uintptr_t disabled_bin; /* * That implies the following value, for the maximum number of items in any @@ -38,8 +45,8 @@ static const uintptr_t cache_bin_trailing_junk = * 1 << (sizeof(cache_bin_sz_t) * 8) * bytes spread across pointer sized objects to get the maximum. */ -#define CACHE_BIN_NCACHED_MAX (((size_t)1 << sizeof(cache_bin_sz_t) * 8) \ - / sizeof(void *) - 1) +#define CACHE_BIN_NCACHED_MAX \ + (((size_t)1 << sizeof(cache_bin_sz_t) * 8) / sizeof(void *) - 1) /* * This lives inside the cache_bin (for locality reasons), and is initialized @@ -101,7 +108,7 @@ struct cache_bin_s { * Since the stack grows down, this is a higher address than * low_bits_full. */ - uint16_t low_bits_low_water; + cache_bin_sz_t low_bits_low_water; /* * The low bits of the value that stack_head will take on when the array @@ -112,7 +119,7 @@ struct cache_bin_s { * Recall that since the stack grows down, this is the lowest available * address in the array for caching. Only adjusted when stashing items. */ - uint16_t low_bits_full; + cache_bin_sz_t low_bits_full; /* * The low bits of the value that stack_head will take on when the array @@ -121,7 +128,10 @@ struct cache_bin_s { * The stack grows down -- this is one past the highest address in the * array. Immutable after initialization. */ - uint16_t low_bits_empty; + cache_bin_sz_t low_bits_empty; + + /* The maximum number of cached items in the bin. */ + cache_bin_info_t bin_info; }; /* @@ -142,8 +152,8 @@ struct cache_bin_array_descriptor_s { }; static inline void -cache_bin_array_descriptor_init(cache_bin_array_descriptor_t *descriptor, - cache_bin_t *bins) { +cache_bin_array_descriptor_init( + cache_bin_array_descriptor_t *descriptor, cache_bin_t *bins) { ql_elm_new(descriptor, link); descriptor->bins = bins; } @@ -168,10 +178,41 @@ cache_bin_nonfast_aligned(const void *ptr) { return ((uintptr_t)ptr & san_cache_bin_nonfast_mask) == 0; } +static inline const void * +cache_bin_disabled_bin_stack(void) { + return &disabled_bin; +} + +/* + * If a cache bin was zero initialized (either because it lives in static or + * thread-local storage, or was memset to 0), this function indicates whether or + * not cache_bin_init was called on it. + */ +static inline bool +cache_bin_still_zero_initialized(cache_bin_t *bin) { + return bin->stack_head == NULL; +} + +static inline bool +cache_bin_disabled(cache_bin_t *bin) { + bool disabled = (bin->stack_head == cache_bin_disabled_bin_stack()); + if (disabled) { + assert((uintptr_t)(*bin->stack_head) == JUNK_ADDR); + } + return disabled; +} + +/* Gets ncached_max without asserting that the bin is enabled. */ +static inline cache_bin_sz_t +cache_bin_ncached_max_get_unsafe(cache_bin_t *bin) { + return bin->bin_info.ncached_max; +} + /* Returns ncached_max: Upper limit on ncached. */ static inline cache_bin_sz_t -cache_bin_info_ncached_max(cache_bin_info_t *info) { - return info->ncached_max; +cache_bin_ncached_max_get(cache_bin_t *bin) { + assert(!cache_bin_disabled(bin)); + return cache_bin_ncached_max_get_unsafe(bin); } /* @@ -181,7 +222,8 @@ cache_bin_info_ncached_max(cache_bin_info_t *info) { * with later. */ static inline void -cache_bin_assert_earlier(cache_bin_t *bin, uint16_t earlier, uint16_t later) { +cache_bin_assert_earlier( + cache_bin_t *bin, cache_bin_sz_t earlier, cache_bin_sz_t later) { if (earlier > later) { assert(bin->low_bits_full > bin->low_bits_empty); } @@ -193,28 +235,19 @@ cache_bin_assert_earlier(cache_bin_t *bin, uint16_t earlier, uint16_t later) { * Does difference calculations that handle wraparound correctly. Earlier must * be associated with the position earlier in memory. */ -static inline uint16_t -cache_bin_diff(cache_bin_t *bin, uint16_t earlier, uint16_t later, bool racy) { - /* - * When it's racy, bin->low_bits_full can be modified concurrently. It - * can cross the uint16_t max value and become less than - * bin->low_bits_empty at the time of the check. - */ - if (!racy) { - cache_bin_assert_earlier(bin, earlier, later); - } +static inline cache_bin_sz_t +cache_bin_diff(cache_bin_t *bin, cache_bin_sz_t earlier, cache_bin_sz_t later) { + cache_bin_assert_earlier(bin, earlier, later); return later - earlier; } /* * Number of items currently cached in the bin, without checking ncached_max. - * We require specifying whether or not the request is racy or not (i.e. whether - * or not concurrent modifications are possible). */ static inline cache_bin_sz_t -cache_bin_ncached_get_internal(cache_bin_t *bin, bool racy) { +cache_bin_ncached_get_internal(cache_bin_t *bin) { cache_bin_sz_t diff = cache_bin_diff(bin, - (uint16_t)(uintptr_t)bin->stack_head, bin->low_bits_empty, racy); + (cache_bin_sz_t)(uintptr_t)bin->stack_head, bin->low_bits_empty); cache_bin_sz_t n = diff / sizeof(void *); /* * We have undefined behavior here; if this function is called from the @@ -225,7 +258,7 @@ cache_bin_ncached_get_internal(cache_bin_t *bin, bool racy) { * fast paths. This should still be "safe" in the sense of generating * the correct assembly for the foreseeable future, though. */ - assert(n == 0 || *(bin->stack_head) != NULL || racy); + assert(n == 0 || *(bin->stack_head) != NULL); return n; } @@ -235,10 +268,9 @@ cache_bin_ncached_get_internal(cache_bin_t *bin, bool racy) { * possible. */ static inline cache_bin_sz_t -cache_bin_ncached_get_local(cache_bin_t *bin, cache_bin_info_t *info) { - cache_bin_sz_t n = cache_bin_ncached_get_internal(bin, - /* racy */ false); - assert(n <= cache_bin_info_ncached_max(info)); +cache_bin_ncached_get_local(cache_bin_t *bin) { + cache_bin_sz_t n = cache_bin_ncached_get_internal(bin); + assert(n <= cache_bin_ncached_max_get(bin)); return n; } @@ -253,10 +285,9 @@ cache_bin_ncached_get_local(cache_bin_t *bin, cache_bin_info_t *info) { static inline void ** cache_bin_empty_position_get(cache_bin_t *bin) { cache_bin_sz_t diff = cache_bin_diff(bin, - (uint16_t)(uintptr_t)bin->stack_head, bin->low_bits_empty, - /* racy */ false); - uintptr_t empty_bits = (uintptr_t)bin->stack_head + diff; - void **ret = (void **)empty_bits; + (cache_bin_sz_t)(uintptr_t)bin->stack_head, bin->low_bits_empty); + byte_t *empty_bits = (byte_t *)bin->stack_head + diff; + void **ret = (void **)empty_bits; assert(ret >= bin->stack_head); @@ -273,10 +304,10 @@ cache_bin_empty_position_get(cache_bin_t *bin) { * multithreaded environment. Currently concurrent access happens only during * arena statistics collection. */ -static inline uint16_t -cache_bin_low_bits_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) { - return (uint16_t)bin->low_bits_empty - - info->ncached_max * sizeof(void *); +static inline cache_bin_sz_t +cache_bin_low_bits_low_bound_get(cache_bin_t *bin) { + return (cache_bin_sz_t)bin->low_bits_empty + - cache_bin_ncached_max_get(bin) * sizeof(void *); } /* @@ -285,9 +316,9 @@ cache_bin_low_bits_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) { * A pointer to the position with the lowest address of the backing array. */ static inline void ** -cache_bin_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) { - cache_bin_sz_t ncached_max = cache_bin_info_ncached_max(info); - void **ret = cache_bin_empty_position_get(bin) - ncached_max; +cache_bin_low_bound_get(cache_bin_t *bin) { + cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(bin); + void **ret = cache_bin_empty_position_get(bin) - ncached_max; assert(ret <= bin->stack_head); return ret; @@ -298,8 +329,8 @@ cache_bin_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) { * batch fill a nonempty cache bin. */ static inline void -cache_bin_assert_empty(cache_bin_t *bin, cache_bin_info_t *info) { - assert(cache_bin_ncached_get_local(bin, info) == 0); +cache_bin_assert_empty(cache_bin_t *bin) { + assert(cache_bin_ncached_get_local(bin) == 0); assert(cache_bin_empty_position_get(bin) == bin->stack_head); } @@ -310,18 +341,19 @@ cache_bin_assert_empty(cache_bin_t *bin, cache_bin_info_t *info) { */ static inline cache_bin_sz_t cache_bin_low_water_get_internal(cache_bin_t *bin) { - return cache_bin_diff(bin, bin->low_bits_low_water, - bin->low_bits_empty, /* racy */ false) / sizeof(void *); + return cache_bin_diff(bin, bin->low_bits_low_water, bin->low_bits_empty) + / sizeof(void *); } /* Returns the numeric value of low water in [0, ncached]. */ static inline cache_bin_sz_t -cache_bin_low_water_get(cache_bin_t *bin, cache_bin_info_t *info) { +cache_bin_low_water_get(cache_bin_t *bin) { cache_bin_sz_t low_water = cache_bin_low_water_get_internal(bin); - assert(low_water <= cache_bin_info_ncached_max(info)); - assert(low_water <= cache_bin_ncached_get_local(bin, info)); + assert(low_water <= cache_bin_ncached_max_get(bin)); + assert(low_water <= cache_bin_ncached_get_local(bin)); - cache_bin_assert_earlier(bin, (uint16_t)(uintptr_t)bin->stack_head, + cache_bin_assert_earlier(bin, + (cache_bin_sz_t)(uintptr_t)bin->stack_head, bin->low_bits_low_water); return low_water; @@ -333,12 +365,14 @@ cache_bin_low_water_get(cache_bin_t *bin, cache_bin_info_t *info) { */ static inline void cache_bin_low_water_set(cache_bin_t *bin) { - bin->low_bits_low_water = (uint16_t)(uintptr_t)bin->stack_head; + assert(!cache_bin_disabled(bin)); + bin->low_bits_low_water = (cache_bin_sz_t)(uintptr_t)bin->stack_head; } static inline void cache_bin_low_water_adjust(cache_bin_t *bin) { - if (cache_bin_ncached_get_internal(bin, /* racy */ false) + assert(!cache_bin_disabled(bin)); + if (cache_bin_ncached_get_internal(bin) < cache_bin_low_water_get_internal(bin)) { cache_bin_low_water_set(bin); } @@ -358,9 +392,9 @@ cache_bin_alloc_impl(cache_bin_t *bin, bool *success, bool adjust_low_water) { * This may read from the empty position; however the loaded value won't * be used. It's safe because the stack has one more slot reserved. */ - void *ret = *bin->stack_head; - uint16_t low_bits = (uint16_t)(uintptr_t)bin->stack_head; - void **new_head = bin->stack_head + 1; + void *ret = *bin->stack_head; + cache_bin_sz_t low_bits = (cache_bin_sz_t)(uintptr_t)bin->stack_head; + void **new_head = bin->stack_head + 1; /* * Note that the low water mark is at most empty; if we pass this check, @@ -382,7 +416,7 @@ cache_bin_alloc_impl(cache_bin_t *bin, bool *success, bool adjust_low_water) { */ if (likely(low_bits != bin->low_bits_empty)) { bin->stack_head = new_head; - bin->low_bits_low_water = (uint16_t)(uintptr_t)new_head; + bin->low_bits_low_water = (cache_bin_sz_t)(uintptr_t)new_head; *success = true; return ret; } @@ -410,8 +444,7 @@ cache_bin_alloc(cache_bin_t *bin, bool *success) { JEMALLOC_ALWAYS_INLINE cache_bin_sz_t cache_bin_alloc_batch(cache_bin_t *bin, size_t num, void **out) { - cache_bin_sz_t n = cache_bin_ncached_get_internal(bin, - /* racy */ false); + cache_bin_sz_t n = cache_bin_ncached_get_internal(bin); if (n > num) { n = (cache_bin_sz_t)num; } @@ -424,7 +457,37 @@ cache_bin_alloc_batch(cache_bin_t *bin, size_t num, void **out) { JEMALLOC_ALWAYS_INLINE bool cache_bin_full(cache_bin_t *bin) { - return ((uint16_t)(uintptr_t)bin->stack_head == bin->low_bits_full); + return ( + (cache_bin_sz_t)(uintptr_t)bin->stack_head == bin->low_bits_full); +} + +/* + * Scans the allocated area of the cache_bin for the given pointer up to limit. + * Fires safety_check_fail if the ptr is found and returns true. + */ +JEMALLOC_ALWAYS_INLINE bool +cache_bin_dalloc_safety_checks(cache_bin_t *bin, void *ptr) { + if (!config_debug || opt_debug_double_free_max_scan == 0) { + return false; + } + + cache_bin_sz_t ncached = cache_bin_ncached_get_internal(bin); + unsigned max_scan = opt_debug_double_free_max_scan < ncached + ? opt_debug_double_free_max_scan + : ncached; + + void **cur = bin->stack_head; + void **limit = cur + max_scan; + for (; cur < limit; cur++) { + if (*cur == ptr) { + safety_check_fail( + "Invalid deallocation detected: double free of " + "pointer %p\n", + ptr); + return true; + } + } + return false; } /* @@ -436,10 +499,14 @@ cache_bin_dalloc_easy(cache_bin_t *bin, void *ptr) { return false; } + if (unlikely(cache_bin_dalloc_safety_checks(bin, ptr))) { + return true; + } + bin->stack_head--; *bin->stack_head = ptr; cache_bin_assert_earlier(bin, bin->low_bits_full, - (uint16_t)(uintptr_t)bin->stack_head); + (cache_bin_sz_t)(uintptr_t)bin->stack_head); return true; } @@ -452,11 +519,12 @@ cache_bin_stash(cache_bin_t *bin, void *ptr) { } /* Stash at the full position, in the [full, head) range. */ - uint16_t low_bits_head = (uint16_t)(uintptr_t)bin->stack_head; + cache_bin_sz_t low_bits_head = (cache_bin_sz_t)(uintptr_t) + bin->stack_head; /* Wraparound handled as well. */ - uint16_t diff = cache_bin_diff(bin, bin->low_bits_full, low_bits_head, - /* racy */ false); - *(void **)((uintptr_t)bin->stack_head - diff) = ptr; + cache_bin_sz_t diff = cache_bin_diff( + bin, bin->low_bits_full, low_bits_head); + *(void **)((byte_t *)bin->stack_head - diff) = ptr; assert(!cache_bin_full(bin)); bin->low_bits_full += sizeof(void *); @@ -465,67 +533,101 @@ cache_bin_stash(cache_bin_t *bin, void *ptr) { return true; } -/* - * Get the number of stashed pointers. - * - * When called from a thread not owning the TLS (i.e. racy = true), it's - * important to keep in mind that 'bin->stack_head' and 'bin->low_bits_full' can - * be modified concurrently and almost none assertions about their values can be - * made. - */ +/* Get the number of stashed pointers. */ JEMALLOC_ALWAYS_INLINE cache_bin_sz_t -cache_bin_nstashed_get_internal(cache_bin_t *bin, cache_bin_info_t *info, - bool racy) { - cache_bin_sz_t ncached_max = cache_bin_info_ncached_max(info); - uint16_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(bin, - info); +cache_bin_nstashed_get_internal(cache_bin_t *bin) { + cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(bin); + cache_bin_sz_t low_bits_low_bound = cache_bin_low_bits_low_bound_get( + bin); - cache_bin_sz_t n = cache_bin_diff(bin, low_bits_low_bound, - bin->low_bits_full, racy) / sizeof(void *); + cache_bin_sz_t n = cache_bin_diff( + bin, low_bits_low_bound, bin->low_bits_full) + / sizeof(void *); assert(n <= ncached_max); - - if (!racy) { + if (config_debug && n != 0) { /* Below are for assertions only. */ - void **low_bound = cache_bin_low_bound_get(bin, info); + void **low_bound = cache_bin_low_bound_get(bin); - assert((uint16_t)(uintptr_t)low_bound == low_bits_low_bound); + assert( + (cache_bin_sz_t)(uintptr_t)low_bound == low_bits_low_bound); void *stashed = *(low_bound + n - 1); - bool aligned = cache_bin_nonfast_aligned(stashed); + bool aligned = cache_bin_nonfast_aligned(stashed); #ifdef JEMALLOC_JET /* Allow arbitrary pointers to be stashed in tests. */ aligned = true; #endif - assert(n == 0 || (stashed != NULL && aligned)); + assert(stashed != NULL && aligned); } return n; } JEMALLOC_ALWAYS_INLINE cache_bin_sz_t -cache_bin_nstashed_get_local(cache_bin_t *bin, cache_bin_info_t *info) { - cache_bin_sz_t n = cache_bin_nstashed_get_internal(bin, info, - /* racy */ false); - assert(n <= cache_bin_info_ncached_max(info)); +cache_bin_nstashed_get_local(cache_bin_t *bin) { + cache_bin_sz_t n = cache_bin_nstashed_get_internal(bin); + assert(n <= cache_bin_ncached_max_get(bin)); return n; } /* * Obtain a racy view of the number of items currently in the cache bin, in the * presence of possible concurrent modifications. + * + * Note that this is the only racy function in this header. Any other functions + * are assumed to be non-racy. The "racy" term here means accessed from another + * thread (that is not the owner of the specific cache bin). This only happens + * when gathering stats (read-only). The only change because of the racy + * condition is that assertions based on mutable fields are omitted. + * + * It's important to keep in mind that 'bin->stack_head' and + * 'bin->low_bits_full' can be modified concurrently and almost no assertions + * about their values can be made. + * + * This function should not call other utility functions because the racy + * condition may cause unexpected / undefined behaviors in unverified utility + * functions. Currently, this function calls two utility functions + * cache_bin_ncached_max_get and cache_bin_low_bits_low_bound_get because + * they help access values that will not be concurrently modified. */ static inline void -cache_bin_nitems_get_remote(cache_bin_t *bin, cache_bin_info_t *info, - cache_bin_sz_t *ncached, cache_bin_sz_t *nstashed) { - cache_bin_sz_t n = cache_bin_ncached_get_internal(bin, /* racy */ true); - assert(n <= cache_bin_info_ncached_max(info)); +cache_bin_nitems_get_remote( + cache_bin_t *bin, cache_bin_sz_t *ncached, cache_bin_sz_t *nstashed) { + /* Racy version of cache_bin_ncached_get_internal. */ + cache_bin_sz_t diff = bin->low_bits_empty + - (cache_bin_sz_t)(uintptr_t)bin->stack_head; + cache_bin_sz_t n = diff / sizeof(void *); *ncached = n; - n = cache_bin_nstashed_get_internal(bin, info, /* racy */ true); - assert(n <= cache_bin_info_ncached_max(info)); + /* Racy version of cache_bin_nstashed_get_internal. */ + cache_bin_sz_t low_bits_low_bound = cache_bin_low_bits_low_bound_get( + bin); + n = (bin->low_bits_full - low_bits_low_bound) / sizeof(void *); *nstashed = n; - /* Note that cannot assert ncached + nstashed <= ncached_max (racy). */ + /* + * Note that cannot assert anything regarding ncached_max because + * it can be configured on the fly and is thus racy. + */ } +/* + * For small bins, used to calculate how many items to fill at a time. + * The final nfill is calculated by (ncached_max >> (base - offset)). + */ +typedef struct cache_bin_fill_ctl_s cache_bin_fill_ctl_t; +struct cache_bin_fill_ctl_s { + uint8_t base; + uint8_t offset; +}; + +/* + * Limit how many items can be flushed in a batch (Which is the upper bound + * for the nflush parameter in tcache_bin_flush_impl()). + * This is to avoid stack overflow when we do batch edata look up, which + * reserves a nflush * sizeof(emap_batch_lookup_result_t) stack variable. + */ +#define CACHE_BIN_NFLUSH_BATCH_MAX \ + ((VARIABLE_ARRAY_SIZE_MAX >> LG_SIZEOF_PTR) - 1) + /* * Filling and flushing are done in batch, on arrays of void *s. For filling, * the arrays go forward, and can be accessed with ordinary array arithmetic. @@ -546,7 +648,7 @@ cache_bin_nitems_get_remote(cache_bin_t *bin, cache_bin_info_t *info, typedef struct cache_bin_ptr_array_s cache_bin_ptr_array_t; struct cache_bin_ptr_array_s { cache_bin_sz_t n; - void **ptr; + void **ptr; }; /* @@ -558,18 +660,18 @@ struct cache_bin_ptr_array_s { * representations is easy (since they'll require an alloca in the calling * frame). */ -#define CACHE_BIN_PTR_ARRAY_DECLARE(name, nval) \ - cache_bin_ptr_array_t name; \ - name.n = (nval) +#define CACHE_BIN_PTR_ARRAY_DECLARE(name, nval) \ + cache_bin_ptr_array_t name; \ + name.n = (nval) /* * Start a fill. The bin must be empty, and This must be followed by a * finish_fill call before doing any alloc/dalloc operations on the bin. */ static inline void -cache_bin_init_ptr_array_for_fill(cache_bin_t *bin, cache_bin_info_t *info, - cache_bin_ptr_array_t *arr, cache_bin_sz_t nfill) { - cache_bin_assert_empty(bin, info); +cache_bin_init_ptr_array_for_fill( + cache_bin_t *bin, cache_bin_ptr_array_t *arr, cache_bin_sz_t nfill) { + cache_bin_assert_empty(bin); arr->ptr = cache_bin_empty_position_get(bin) - nfill; } @@ -579,15 +681,19 @@ cache_bin_init_ptr_array_for_fill(cache_bin_t *bin, cache_bin_info_t *info, * case of OOM. */ static inline void -cache_bin_finish_fill(cache_bin_t *bin, cache_bin_info_t *info, - cache_bin_ptr_array_t *arr, cache_bin_sz_t nfilled) { - cache_bin_assert_empty(bin, info); +cache_bin_finish_fill( + cache_bin_t *bin, cache_bin_ptr_array_t *arr, cache_bin_sz_t nfilled) { + cache_bin_assert_empty(bin); void **empty_position = cache_bin_empty_position_get(bin); if (nfilled < arr->n) { memmove(empty_position - nfilled, empty_position - arr->n, nfilled * sizeof(void *)); } bin->stack_head = empty_position - nfilled; + /* Reset the bin stats as it's merged during fill. */ + if (config_stats) { + bin->tstats.nrequests = 0; + } } /* @@ -595,55 +701,61 @@ cache_bin_finish_fill(cache_bin_t *bin, cache_bin_info_t *info, * everything we give them. */ static inline void -cache_bin_init_ptr_array_for_flush(cache_bin_t *bin, cache_bin_info_t *info, - cache_bin_ptr_array_t *arr, cache_bin_sz_t nflush) { +cache_bin_init_ptr_array_for_flush( + cache_bin_t *bin, cache_bin_ptr_array_t *arr, cache_bin_sz_t nflush) { arr->ptr = cache_bin_empty_position_get(bin) - nflush; - assert(cache_bin_ncached_get_local(bin, info) == 0 - || *arr->ptr != NULL); + assert(cache_bin_ncached_get_local(bin) == 0 || *arr->ptr != NULL); } static inline void -cache_bin_finish_flush(cache_bin_t *bin, cache_bin_info_t *info, - cache_bin_ptr_array_t *arr, cache_bin_sz_t nflushed) { - unsigned rem = cache_bin_ncached_get_local(bin, info) - nflushed; - memmove(bin->stack_head + nflushed, bin->stack_head, - rem * sizeof(void *)); - bin->stack_head = bin->stack_head + nflushed; +cache_bin_finish_flush( + cache_bin_t *bin, cache_bin_ptr_array_t *arr, cache_bin_sz_t nflushed) { + unsigned rem = cache_bin_ncached_get_local(bin) - nflushed; + memmove( + bin->stack_head + nflushed, bin->stack_head, rem * sizeof(void *)); + bin->stack_head += nflushed; cache_bin_low_water_adjust(bin); + /* Reset the bin stats as it's merged during flush. */ + if (config_stats) { + bin->tstats.nrequests = 0; + } } static inline void cache_bin_init_ptr_array_for_stashed(cache_bin_t *bin, szind_t binind, - cache_bin_info_t *info, cache_bin_ptr_array_t *arr, - cache_bin_sz_t nstashed) { + cache_bin_ptr_array_t *arr, cache_bin_sz_t nstashed) { assert(nstashed > 0); - assert(cache_bin_nstashed_get_local(bin, info) == nstashed); + assert(cache_bin_nstashed_get_local(bin) == nstashed); - void **low_bound = cache_bin_low_bound_get(bin, info); + void **low_bound = cache_bin_low_bound_get(bin); arr->ptr = low_bound; assert(*arr->ptr != NULL); } static inline void -cache_bin_finish_flush_stashed(cache_bin_t *bin, cache_bin_info_t *info) { - void **low_bound = cache_bin_low_bound_get(bin, info); +cache_bin_finish_flush_stashed(cache_bin_t *bin) { + void **low_bound = cache_bin_low_bound_get(bin); /* Reset the bin local full position. */ bin->low_bits_full = (uint16_t)(uintptr_t)low_bound; - assert(cache_bin_nstashed_get_local(bin, info) == 0); + assert(cache_bin_nstashed_get_local(bin) == 0); + /* Reset the bin stats as it's merged during flush. */ + if (config_stats) { + bin->tstats.nrequests = 0; + } } /* * Initialize a cache_bin_info to represent up to the given number of items in * the cache_bins it is associated with. */ -void cache_bin_info_init(cache_bin_info_t *bin_info, - cache_bin_sz_t ncached_max); +void cache_bin_info_init( + cache_bin_info_t *bin_info, cache_bin_sz_t ncached_max); /* * Given an array of initialized cache_bin_info_ts, determine how big an * allocation is required to initialize a full set of cache_bin_ts. */ -void cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos, +void cache_bin_info_compute_alloc(const cache_bin_info_t *infos, szind_t ninfos, size_t *size, size_t *alignment); /* @@ -653,18 +765,13 @@ void cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos, * cache_bin_postincrement. *alloc_cur will then point immediately past the end * of the allocation. */ -void cache_bin_preincrement(cache_bin_info_t *infos, szind_t ninfos, +void cache_bin_preincrement(const cache_bin_info_t *infos, szind_t ninfos, void *alloc, size_t *cur_offset); -void cache_bin_postincrement(cache_bin_info_t *infos, szind_t ninfos, - void *alloc, size_t *cur_offset); -void cache_bin_init(cache_bin_t *bin, cache_bin_info_t *info, void *alloc, +void cache_bin_postincrement(void *alloc, size_t *cur_offset); +void cache_bin_init(cache_bin_t *bin, const cache_bin_info_t *info, void *alloc, size_t *cur_offset); +void cache_bin_init_disabled(cache_bin_t *bin, cache_bin_sz_t ncached_max); -/* - * If a cache bin was zero initialized (either because it lives in static or - * thread-local storage, or was memset to 0), this function indicates whether or - * not cache_bin_init was called on it. - */ -bool cache_bin_still_zero_initialized(cache_bin_t *bin); +bool cache_bin_stack_use_thp(void); #endif /* JEMALLOC_INTERNAL_CACHE_BIN_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/ckh.h b/dep/jemalloc/include/jemalloc/internal/ckh.h index 7b3850bc1..01b27e8f7 100644 --- a/dep/jemalloc/include/jemalloc/internal/ckh.h +++ b/dep/jemalloc/include/jemalloc/internal/ckh.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_CKH_H #define JEMALLOC_INTERNAL_CKH_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/tsd.h" /* Cuckoo hashing implementation. Skip to the end for the interface. */ @@ -21,8 +22,8 @@ #define LG_CKH_BUCKET_CELLS (LG_CACHELINE - LG_SIZEOF_PTR - 1) /* Typedefs to allow easy function pointer passing. */ -typedef void ckh_hash_t (const void *, size_t[2]); -typedef bool ckh_keycomp_t (const void *, const void *); +typedef void ckh_hash_t(const void *, size_t[2]); +typedef bool ckh_keycomp_t(const void *, const void *); /* Hash table cell. */ typedef struct { @@ -55,7 +56,7 @@ typedef struct { unsigned lg_curbuckets; /* Hash and comparison functions. */ - ckh_hash_t *hash; + ckh_hash_t *hash; ckh_keycomp_t *keycomp; /* Hash table with 2^lg_curbuckets buckets. */ @@ -88,8 +89,8 @@ bool ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data); * the key and value, and doesn't do any lifetime management. */ bool ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data); -bool ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key, - void **data); +bool ckh_remove( + tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key, void **data); bool ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data); /* Some useful hash and comparison functions for strings and pointers. */ diff --git a/dep/jemalloc/include/jemalloc/internal/conf.h b/dep/jemalloc/include/jemalloc/internal/conf.h new file mode 100644 index 000000000..26983ee9f --- /dev/null +++ b/dep/jemalloc/include/jemalloc/internal/conf.h @@ -0,0 +1,23 @@ +#ifndef JEMALLOC_INTERNAL_CONF_H +#define JEMALLOC_INTERNAL_CONF_H + +#include "jemalloc/internal/sc.h" + +void malloc_conf_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], + char readlink_buf[PATH_MAX + 1]); +void malloc_abort_invalid_conf(void); + +#ifdef JEMALLOC_JET +extern bool had_conf_error; + +bool conf_next(char const **opts_p, char const **k_p, size_t *klen_p, + char const **v_p, size_t *vlen_p); +void conf_error( + const char *msg, const char *k, size_t klen, const char *v, size_t vlen); +bool conf_handle_bool(const char *v, size_t vlen, bool *result); +bool conf_handle_signed(const char *v, size_t vlen, intmax_t min, intmax_t max, + bool check_min, bool check_max, bool clip, intmax_t *result); +bool conf_handle_char_p(const char *v, size_t vlen, char *dest, size_t dest_sz); +#endif + +#endif /* JEMALLOC_INTERNAL_CONF_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/counter.h b/dep/jemalloc/include/jemalloc/internal/counter.h index 79abf0648..0f38d40c6 100644 --- a/dep/jemalloc/include/jemalloc/internal/counter.h +++ b/dep/jemalloc/include/jemalloc/internal/counter.h @@ -1,12 +1,14 @@ #ifndef JEMALLOC_INTERNAL_COUNTER_H #define JEMALLOC_INTERNAL_COUNTER_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/lockedint.h" #include "jemalloc/internal/mutex.h" typedef struct counter_accum_s { LOCKEDINT_MTX_DECLARE(mtx) locked_u64_t accumbytes; - uint64_t interval; + uint64_t interval; } counter_accum_t; JEMALLOC_ALWAYS_INLINE bool diff --git a/dep/jemalloc/include/jemalloc/internal/ctl.h b/dep/jemalloc/include/jemalloc/internal/ctl.h index 63d27f8aa..82035fe36 100644 --- a/dep/jemalloc/include/jemalloc/internal/ctl.h +++ b/dep/jemalloc/include/jemalloc/internal/ctl.h @@ -1,6 +1,10 @@ #ifndef JEMALLOC_INTERNAL_CTL_H #define JEMALLOC_INTERNAL_CTL_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/arena_stats.h" +#include "jemalloc/internal/background_thread_structs.h" +#include "jemalloc/internal/bin_stats.h" #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/mutex_prof.h" @@ -9,50 +13,52 @@ #include "jemalloc/internal/stats.h" /* Maximum ctl tree depth. */ -#define CTL_MAX_DEPTH 7 +#define CTL_MAX_DEPTH 7 +#define CTL_MULTI_SETTING_MAX_LEN 1000 typedef struct ctl_node_s { bool named; } ctl_node_t; typedef struct ctl_named_node_s { - ctl_node_t node; + ctl_node_t node; const char *name; /* If (nchildren == 0), this is a terminal node. */ - size_t nchildren; + size_t nchildren; const ctl_node_t *children; - int (*ctl)(tsd_t *, const size_t *, size_t, void *, size_t *, void *, - size_t); + int (*ctl)( + tsd_t *, const size_t *, size_t, void *, size_t *, void *, size_t); } ctl_named_node_t; typedef struct ctl_indexed_node_s { struct ctl_node_s node; - const ctl_named_node_t *(*index)(tsdn_t *, const size_t *, size_t, - size_t); + const ctl_named_node_t *(*index)( + tsdn_t *, const size_t *, size_t, size_t); } ctl_indexed_node_t; typedef struct ctl_arena_stats_s { arena_stats_t astats; /* Aggregate stats for small size classes, based on bin stats. */ - size_t allocated_small; + size_t allocated_small; uint64_t nmalloc_small; uint64_t ndalloc_small; uint64_t nrequests_small; uint64_t nfills_small; uint64_t nflushes_small; - bin_stats_data_t bstats[SC_NBINS]; + bin_stats_data_t bstats[SC_NBINS]; arena_stats_large_t lstats[SC_NSIZES - SC_NBINS]; - pac_estats_t estats[SC_NPSIZES]; - hpa_shard_stats_t hpastats; - sec_stats_t secstats; + pac_estats_t estats[SC_NPSIZES]; + hpa_shard_stats_t hpastats; } ctl_arena_stats_t; typedef struct ctl_stats_s { size_t allocated; size_t active; size_t metadata; + size_t metadata_edata; + size_t metadata_rtree; size_t metadata_thp; size_t resident; size_t mapped; @@ -65,17 +71,17 @@ typedef struct ctl_stats_s { typedef struct ctl_arena_s ctl_arena_t; struct ctl_arena_s { unsigned arena_ind; - bool initialized; + bool initialized; ql_elm(ctl_arena_t) destroyed_link; /* Basic stats, supported even if !config_stats. */ - unsigned nthreads; + unsigned nthreads; const char *dss; - ssize_t dirty_decay_ms; - ssize_t muzzy_decay_ms; - size_t pactive; - size_t pdirty; - size_t pmuzzy; + ssize_t dirty_decay_ms; + ssize_t muzzy_decay_ms; + size_t pactive; + size_t pdirty; + size_t pmuzzy; /* NULL if !config_stats. */ ctl_arena_stats_t *astats; @@ -100,60 +106,67 @@ int ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp, int ctl_nametomib(tsd_t *tsd, const char *name, size_t *mibp, size_t *miblenp); int ctl_bymib(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen); -int ctl_mibnametomib(tsd_t *tsd, size_t *mib, size_t miblen, const char *name, - size_t *miblenp); -int ctl_bymibname(tsd_t *tsd, size_t *mib, size_t miblen, const char *name, - size_t *miblenp, void *oldp, size_t *oldlenp, void *newp, size_t newlen); +int ctl_mibnametomib( + tsd_t *tsd, size_t *mib, size_t miblen, const char *name, size_t *miblenp); +int ctl_bymibname(tsd_t *tsd, size_t *mib, size_t miblen, const char *name, + size_t *miblenp, void *oldp, size_t *oldlenp, void *newp, size_t newlen); bool ctl_boot(void); void ctl_prefork(tsdn_t *tsdn); void ctl_postfork_parent(tsdn_t *tsdn); void ctl_postfork_child(tsdn_t *tsdn); void ctl_mtx_assert_held(tsdn_t *tsdn); -#define xmallctl(name, oldp, oldlenp, newp, newlen) do { \ - if (je_mallctl(name, oldp, oldlenp, newp, newlen) \ - != 0) { \ - malloc_printf( \ - ": Failure in xmallctl(\"%s\", ...)\n", \ - name); \ - abort(); \ - } \ -} while (0) +#define xmallctl(name, oldp, oldlenp, newp, newlen) \ + do { \ + if (je_mallctl(name, oldp, oldlenp, newp, newlen) != 0) { \ + malloc_printf( \ + ": Failure in xmallctl(\"%s\", ...)\n", \ + name); \ + abort(); \ + } \ + } while (0) -#define xmallctlnametomib(name, mibp, miblenp) do { \ - if (je_mallctlnametomib(name, mibp, miblenp) != 0) { \ - malloc_printf(": Failure in " \ - "xmallctlnametomib(\"%s\", ...)\n", name); \ - abort(); \ - } \ -} while (0) +#define xmallctlnametomib(name, mibp, miblenp) \ + do { \ + if (je_mallctlnametomib(name, mibp, miblenp) != 0) { \ + malloc_printf( \ + ": Failure in " \ + "xmallctlnametomib(\"%s\", ...)\n", \ + name); \ + abort(); \ + } \ + } while (0) -#define xmallctlbymib(mib, miblen, oldp, oldlenp, newp, newlen) do { \ - if (je_mallctlbymib(mib, miblen, oldp, oldlenp, newp, \ - newlen) != 0) { \ - malloc_write( \ - ": Failure in xmallctlbymib()\n"); \ - abort(); \ - } \ -} while (0) +#define xmallctlbymib(mib, miblen, oldp, oldlenp, newp, newlen) \ + do { \ + if (je_mallctlbymib(mib, miblen, oldp, oldlenp, newp, newlen) \ + != 0) { \ + malloc_write( \ + ": Failure in xmallctlbymib()\n"); \ + abort(); \ + } \ + } while (0) -#define xmallctlmibnametomib(mib, miblen, name, miblenp) do { \ - if (ctl_mibnametomib(tsd_fetch(), mib, miblen, name, miblenp) \ - != 0) { \ - malloc_write( \ - ": Failure in ctl_mibnametomib()\n"); \ - abort(); \ - } \ -} while (0) +#define xmallctlmibnametomib(mib, miblen, name, miblenp) \ + do { \ + if (ctl_mibnametomib(tsd_fetch(), mib, miblen, name, miblenp) \ + != 0) { \ + malloc_write( \ + ": Failure in ctl_mibnametomib()\n"); \ + abort(); \ + } \ + } while (0) -#define xmallctlbymibname(mib, miblen, name, miblenp, oldp, oldlenp, \ - newp, newlen) do { \ - if (ctl_bymibname(tsd_fetch(), mib, miblen, name, miblenp, \ - oldp, oldlenp, newp, newlen) != 0) { \ - malloc_write( \ - ": Failure in ctl_bymibname()\n"); \ - abort(); \ - } \ -} while (0) +#define xmallctlbymibname( \ + mib, miblen, name, miblenp, oldp, oldlenp, newp, newlen) \ + do { \ + if (ctl_bymibname(tsd_fetch(), mib, miblen, name, miblenp, \ + oldp, oldlenp, newp, newlen) \ + != 0) { \ + malloc_write( \ + ": Failure in ctl_bymibname()\n"); \ + abort(); \ + } \ + } while (0) #endif /* JEMALLOC_INTERNAL_CTL_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/decay.h b/dep/jemalloc/include/jemalloc/internal/decay.h index cf6a9d22c..e87736550 100644 --- a/dep/jemalloc/include/jemalloc/internal/decay.h +++ b/dep/jemalloc/include/jemalloc/internal/decay.h @@ -1,9 +1,11 @@ #ifndef JEMALLOC_INTERNAL_DECAY_H #define JEMALLOC_INTERNAL_DECAY_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/mutex.h" #include "jemalloc/internal/smoothstep.h" -#define DECAY_UNBOUNDED_TIME_TO_PURGE ((uint64_t)-1) +#define DECAY_UNBOUNDED_TIME_TO_PURGE ((uint64_t) - 1) /* * The decay_t computes the number of pages we should purge at any given time. @@ -166,12 +168,12 @@ void decay_reinit(decay_t *decay, nstime_t *cur_time, ssize_t decay_ms); /* * Compute how many of 'npages_new' pages we would need to purge in 'time'. */ -uint64_t decay_npages_purge_in(decay_t *decay, nstime_t *time, - size_t npages_new); +uint64_t decay_npages_purge_in( + decay_t *decay, nstime_t *time, size_t npages_new); /* Returns true if the epoch advanced and there are pages to purge. */ -bool decay_maybe_advance_epoch(decay_t *decay, nstime_t *new_time, - size_t current_npages); +bool decay_maybe_advance_epoch( + decay_t *decay, nstime_t *new_time, size_t current_npages); /* * Calculates wait time until a number of pages in the interval @@ -180,7 +182,7 @@ bool decay_maybe_advance_epoch(decay_t *decay, nstime_t *new_time, * Returns number of nanoseconds or DECAY_UNBOUNDED_TIME_TO_PURGE in case of * indefinite wait. */ -uint64_t decay_ns_until_purge(decay_t *decay, size_t npages_current, - uint64_t npages_threshold); +uint64_t decay_ns_until_purge( + decay_t *decay, size_t npages_current, uint64_t npages_threshold); #endif /* JEMALLOC_INTERNAL_DECAY_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/div.h b/dep/jemalloc/include/jemalloc/internal/div.h index aebae9398..56d5f463f 100644 --- a/dep/jemalloc/include/jemalloc/internal/div.h +++ b/dep/jemalloc/include/jemalloc/internal/div.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_DIV_H #define JEMALLOC_INTERNAL_DIV_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/assert.h" /* diff --git a/dep/jemalloc/include/jemalloc/internal/ecache.h b/dep/jemalloc/include/jemalloc/internal/ecache.h index 71cae3e34..605733b51 100644 --- a/dep/jemalloc/include/jemalloc/internal/ecache.h +++ b/dep/jemalloc/include/jemalloc/internal/ecache.h @@ -1,15 +1,16 @@ #ifndef JEMALLOC_INTERNAL_ECACHE_H #define JEMALLOC_INTERNAL_ECACHE_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/eset.h" -#include "jemalloc/internal/san.h" #include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/san.h" typedef struct ecache_s ecache_t; struct ecache_s { malloc_mutex_t mtx; - eset_t eset; - eset_t guarded_eset; + eset_t eset; + eset_t guarded_eset; /* All stored extents must be in the same state. */ extent_state_t state; /* The index of the ehooks the ecache is associated with. */ @@ -23,22 +24,22 @@ struct ecache_s { static inline size_t ecache_npages_get(ecache_t *ecache) { - return eset_npages_get(&ecache->eset) + - eset_npages_get(&ecache->guarded_eset); + return eset_npages_get(&ecache->eset) + + eset_npages_get(&ecache->guarded_eset); } /* Get the number of extents in the given page size index. */ static inline size_t ecache_nextents_get(ecache_t *ecache, pszind_t ind) { - return eset_nextents_get(&ecache->eset, ind) + - eset_nextents_get(&ecache->guarded_eset, ind); + return eset_nextents_get(&ecache->eset, ind) + + eset_nextents_get(&ecache->guarded_eset, ind); } /* Get the sum total bytes of the extents in the given page size index. */ static inline size_t ecache_nbytes_get(ecache_t *ecache, pszind_t ind) { - return eset_nbytes_get(&ecache->eset, ind) + - eset_nbytes_get(&ecache->guarded_eset, ind); + return eset_nbytes_get(&ecache->eset, ind) + + eset_nbytes_get(&ecache->guarded_eset, ind); } static inline unsigned diff --git a/dep/jemalloc/include/jemalloc/internal/edata.h b/dep/jemalloc/include/jemalloc/internal/edata.h index af039ea73..06b6c5454 100644 --- a/dep/jemalloc/include/jemalloc/internal/edata.h +++ b/dep/jemalloc/include/jemalloc/internal/edata.h @@ -1,12 +1,14 @@ #ifndef JEMALLOC_INTERNAL_EDATA_H #define JEMALLOC_INTERNAL_EDATA_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/bin_info.h" #include "jemalloc/internal/bit_util.h" #include "jemalloc/internal/hpdata.h" #include "jemalloc/internal/nstime.h" #include "jemalloc/internal/ph.h" +#include "jemalloc/internal/prof_types.h" #include "jemalloc/internal/ql.h" #include "jemalloc/internal/sc.h" #include "jemalloc/internal/slab_data.h" @@ -19,10 +21,18 @@ */ #define EDATA_ALIGNMENT 128 +/* + * Defines how many nodes visited when enumerating the heap to search for + * qualified extents. More nodes visited may result in better choices at + * the cost of longer search time. This size should not exceed 2^16 - 1 + * because we use uint16_t for accessing the queue needed for enumeration. + */ +#define ESET_ENUMERATE_MAX_NUM 32 + enum extent_state_e { - extent_state_active = 0, - extent_state_dirty = 1, - extent_state_muzzy = 2, + extent_state_active = 0, + extent_state_dirty = 1, + extent_state_muzzy = 2, extent_state_retained = 3, extent_state_transition = 4, /* States below are intermediate. */ extent_state_merging = 5, @@ -32,7 +42,7 @@ typedef enum extent_state_e extent_state_t; enum extent_head_state_e { EXTENT_NOT_HEAD, - EXTENT_IS_HEAD /* See comments in ehooks_default_merge_impl(). */ + EXTENT_IS_HEAD /* See comments in ehooks_default_merge_impl(). */ }; typedef enum extent_head_state_e extent_head_state_t; @@ -40,25 +50,22 @@ typedef enum extent_head_state_e extent_head_state_t; * Which implementation of the page allocator interface, (PAI, defined in * pai.h) owns the given extent? */ -enum extent_pai_e { - EXTENT_PAI_PAC = 0, - EXTENT_PAI_HPA = 1 -}; +enum extent_pai_e { EXTENT_PAI_PAC = 0, EXTENT_PAI_HPA = 1 }; typedef enum extent_pai_e extent_pai_t; struct e_prof_info_s { /* Time when this was allocated. */ - nstime_t e_prof_alloc_time; + nstime_t e_prof_alloc_time; /* Allocation request size. */ - size_t e_prof_alloc_size; + size_t e_prof_alloc_size; /* Points to a prof_tctx_t. */ - atomic_p_t e_prof_tctx; + atomic_p_t e_prof_tctx; /* * Points to a prof_recent_t for the allocation; NULL * means the recent allocation record no longer exists. * Protected by prof_recent_alloc_mtx. */ - atomic_p_t e_prof_recent_alloc; + atomic_p_t e_prof_recent_alloc; }; typedef struct e_prof_info_s e_prof_info_t; @@ -75,20 +82,20 @@ typedef struct e_prof_info_s e_prof_info_t; */ typedef struct edata_map_info_s edata_map_info_t; struct edata_map_info_s { - bool slab; + bool slab; szind_t szind; }; typedef struct edata_cmp_summary_s edata_cmp_summary_t; struct edata_cmp_summary_s { - uint64_t sn; + uint64_t sn; uintptr_t addr; }; /* Extent (span of pages). Use accessor functions for e_* fields. */ typedef struct edata_s edata_t; -ph_structs(edata_avail, edata_t); -ph_structs(edata_heap, edata_t); +ph_structs(edata_avail, edata_t, ESET_ENUMERATE_MAX_NUM); +ph_structs(edata_heap, edata_t, ESET_ENUMERATE_MAX_NUM); struct edata_s { /* * Bitfield containing several fields: @@ -139,55 +146,72 @@ struct edata_s { * * bin_shard: the shard of the bin from which this extent came. */ - uint64_t e_bits; -#define MASK(CURRENT_FIELD_WIDTH, CURRENT_FIELD_SHIFT) ((((((uint64_t)0x1U) << (CURRENT_FIELD_WIDTH)) - 1)) << (CURRENT_FIELD_SHIFT)) + uint64_t e_bits; +#define MASK(CURRENT_FIELD_WIDTH, CURRENT_FIELD_SHIFT) \ + ((((((uint64_t)0x1U) << (CURRENT_FIELD_WIDTH)) - 1)) \ + << (CURRENT_FIELD_SHIFT)) -#define EDATA_BITS_ARENA_WIDTH MALLOCX_ARENA_BITS -#define EDATA_BITS_ARENA_SHIFT 0 -#define EDATA_BITS_ARENA_MASK MASK(EDATA_BITS_ARENA_WIDTH, EDATA_BITS_ARENA_SHIFT) +#define EDATA_BITS_ARENA_WIDTH MALLOCX_ARENA_BITS +#define EDATA_BITS_ARENA_SHIFT 0 +#define EDATA_BITS_ARENA_MASK \ + MASK(EDATA_BITS_ARENA_WIDTH, EDATA_BITS_ARENA_SHIFT) -#define EDATA_BITS_SLAB_WIDTH 1 -#define EDATA_BITS_SLAB_SHIFT (EDATA_BITS_ARENA_WIDTH + EDATA_BITS_ARENA_SHIFT) -#define EDATA_BITS_SLAB_MASK MASK(EDATA_BITS_SLAB_WIDTH, EDATA_BITS_SLAB_SHIFT) +#define EDATA_BITS_SLAB_WIDTH 1 +#define EDATA_BITS_SLAB_SHIFT (EDATA_BITS_ARENA_WIDTH + EDATA_BITS_ARENA_SHIFT) +#define EDATA_BITS_SLAB_MASK MASK(EDATA_BITS_SLAB_WIDTH, EDATA_BITS_SLAB_SHIFT) -#define EDATA_BITS_COMMITTED_WIDTH 1 -#define EDATA_BITS_COMMITTED_SHIFT (EDATA_BITS_SLAB_WIDTH + EDATA_BITS_SLAB_SHIFT) -#define EDATA_BITS_COMMITTED_MASK MASK(EDATA_BITS_COMMITTED_WIDTH, EDATA_BITS_COMMITTED_SHIFT) +#define EDATA_BITS_COMMITTED_WIDTH 1 +#define EDATA_BITS_COMMITTED_SHIFT \ + (EDATA_BITS_SLAB_WIDTH + EDATA_BITS_SLAB_SHIFT) +#define EDATA_BITS_COMMITTED_MASK \ + MASK(EDATA_BITS_COMMITTED_WIDTH, EDATA_BITS_COMMITTED_SHIFT) -#define EDATA_BITS_PAI_WIDTH 1 -#define EDATA_BITS_PAI_SHIFT (EDATA_BITS_COMMITTED_WIDTH + EDATA_BITS_COMMITTED_SHIFT) -#define EDATA_BITS_PAI_MASK MASK(EDATA_BITS_PAI_WIDTH, EDATA_BITS_PAI_SHIFT) +#define EDATA_BITS_PAI_WIDTH 1 +#define EDATA_BITS_PAI_SHIFT \ + (EDATA_BITS_COMMITTED_WIDTH + EDATA_BITS_COMMITTED_SHIFT) +#define EDATA_BITS_PAI_MASK MASK(EDATA_BITS_PAI_WIDTH, EDATA_BITS_PAI_SHIFT) -#define EDATA_BITS_ZEROED_WIDTH 1 -#define EDATA_BITS_ZEROED_SHIFT (EDATA_BITS_PAI_WIDTH + EDATA_BITS_PAI_SHIFT) -#define EDATA_BITS_ZEROED_MASK MASK(EDATA_BITS_ZEROED_WIDTH, EDATA_BITS_ZEROED_SHIFT) +#define EDATA_BITS_ZEROED_WIDTH 1 +#define EDATA_BITS_ZEROED_SHIFT (EDATA_BITS_PAI_WIDTH + EDATA_BITS_PAI_SHIFT) +#define EDATA_BITS_ZEROED_MASK \ + MASK(EDATA_BITS_ZEROED_WIDTH, EDATA_BITS_ZEROED_SHIFT) -#define EDATA_BITS_GUARDED_WIDTH 1 -#define EDATA_BITS_GUARDED_SHIFT (EDATA_BITS_ZEROED_WIDTH + EDATA_BITS_ZEROED_SHIFT) -#define EDATA_BITS_GUARDED_MASK MASK(EDATA_BITS_GUARDED_WIDTH, EDATA_BITS_GUARDED_SHIFT) +#define EDATA_BITS_GUARDED_WIDTH 1 +#define EDATA_BITS_GUARDED_SHIFT \ + (EDATA_BITS_ZEROED_WIDTH + EDATA_BITS_ZEROED_SHIFT) +#define EDATA_BITS_GUARDED_MASK \ + MASK(EDATA_BITS_GUARDED_WIDTH, EDATA_BITS_GUARDED_SHIFT) -#define EDATA_BITS_STATE_WIDTH 3 -#define EDATA_BITS_STATE_SHIFT (EDATA_BITS_GUARDED_WIDTH + EDATA_BITS_GUARDED_SHIFT) -#define EDATA_BITS_STATE_MASK MASK(EDATA_BITS_STATE_WIDTH, EDATA_BITS_STATE_SHIFT) +#define EDATA_BITS_STATE_WIDTH 3 +#define EDATA_BITS_STATE_SHIFT \ + (EDATA_BITS_GUARDED_WIDTH + EDATA_BITS_GUARDED_SHIFT) +#define EDATA_BITS_STATE_MASK \ + MASK(EDATA_BITS_STATE_WIDTH, EDATA_BITS_STATE_SHIFT) -#define EDATA_BITS_SZIND_WIDTH LG_CEIL(SC_NSIZES) -#define EDATA_BITS_SZIND_SHIFT (EDATA_BITS_STATE_WIDTH + EDATA_BITS_STATE_SHIFT) -#define EDATA_BITS_SZIND_MASK MASK(EDATA_BITS_SZIND_WIDTH, EDATA_BITS_SZIND_SHIFT) +#define EDATA_BITS_SZIND_WIDTH LG_CEIL(SC_NSIZES) +#define EDATA_BITS_SZIND_SHIFT (EDATA_BITS_STATE_WIDTH + EDATA_BITS_STATE_SHIFT) +#define EDATA_BITS_SZIND_MASK \ + MASK(EDATA_BITS_SZIND_WIDTH, EDATA_BITS_SZIND_SHIFT) -#define EDATA_BITS_NFREE_WIDTH (SC_LG_SLAB_MAXREGS + 1) -#define EDATA_BITS_NFREE_SHIFT (EDATA_BITS_SZIND_WIDTH + EDATA_BITS_SZIND_SHIFT) -#define EDATA_BITS_NFREE_MASK MASK(EDATA_BITS_NFREE_WIDTH, EDATA_BITS_NFREE_SHIFT) +#define EDATA_BITS_NFREE_WIDTH (SC_LG_SLAB_MAXREGS + 1) +#define EDATA_BITS_NFREE_SHIFT (EDATA_BITS_SZIND_WIDTH + EDATA_BITS_SZIND_SHIFT) +#define EDATA_BITS_NFREE_MASK \ + MASK(EDATA_BITS_NFREE_WIDTH, EDATA_BITS_NFREE_SHIFT) -#define EDATA_BITS_BINSHARD_WIDTH 6 -#define EDATA_BITS_BINSHARD_SHIFT (EDATA_BITS_NFREE_WIDTH + EDATA_BITS_NFREE_SHIFT) -#define EDATA_BITS_BINSHARD_MASK MASK(EDATA_BITS_BINSHARD_WIDTH, EDATA_BITS_BINSHARD_SHIFT) +#define EDATA_BITS_BINSHARD_WIDTH 6 +#define EDATA_BITS_BINSHARD_SHIFT \ + (EDATA_BITS_NFREE_WIDTH + EDATA_BITS_NFREE_SHIFT) +#define EDATA_BITS_BINSHARD_MASK \ + MASK(EDATA_BITS_BINSHARD_WIDTH, EDATA_BITS_BINSHARD_SHIFT) #define EDATA_BITS_IS_HEAD_WIDTH 1 -#define EDATA_BITS_IS_HEAD_SHIFT (EDATA_BITS_BINSHARD_WIDTH + EDATA_BITS_BINSHARD_SHIFT) -#define EDATA_BITS_IS_HEAD_MASK MASK(EDATA_BITS_IS_HEAD_WIDTH, EDATA_BITS_IS_HEAD_SHIFT) +#define EDATA_BITS_IS_HEAD_SHIFT \ + (EDATA_BITS_BINSHARD_WIDTH + EDATA_BITS_BINSHARD_SHIFT) +#define EDATA_BITS_IS_HEAD_MASK \ + MASK(EDATA_BITS_IS_HEAD_WIDTH, EDATA_BITS_IS_HEAD_SHIFT) /* Pointer to the extent that this structure is responsible for. */ - void *e_addr; + void *e_addr; union { /* @@ -197,16 +221,16 @@ struct edata_s { * * ssssssss [...] ssssssss ssssnnnn nnnnnnnn */ - size_t e_size_esn; - #define EDATA_SIZE_MASK ((size_t)~(PAGE-1)) - #define EDATA_ESN_MASK ((size_t)PAGE-1) + size_t e_size_esn; +#define EDATA_SIZE_MASK ((size_t) ~(PAGE - 1)) +#define EDATA_ESN_MASK ((size_t)PAGE - 1) /* Base extent size, which may not be a multiple of PAGE. */ - size_t e_bsize; + size_t e_bsize; }; /* * If this edata is a user allocation from an HPA, it comes out of some - * pageslab (we don't yet support huegpage allocations that don't fit + * pageslab (we don't yet support hugepage allocations that don't fit * into pageslabs). This tracks it. */ hpdata_t *e_ps; @@ -222,7 +246,7 @@ struct edata_s { * List linkage used when the edata_t is active; either in * arena's large allocations or bin_t's slabs_full. */ - ql_elm(edata_t) ql_link_active; + ql_elm(edata_t) ql_link_active; /* * Pairing heap linkage. Used whenever the extent is inactive * (in the page allocators), or when it is active and in @@ -230,7 +254,7 @@ struct edata_s { * extent and sitting in an edata_cache. */ union { - edata_heap_link_t heap_link; + edata_heap_link_t heap_link; edata_avail_link_t avail_link; }; }; @@ -243,10 +267,10 @@ struct edata_s { */ ql_elm(edata_t) ql_link_inactive; /* Small region slab metadata. */ - slab_data_t e_slab_data; + slab_data_t e_slab_data; /* Profiling data, used for large objects. */ - e_prof_info_t e_prof_info; + e_prof_info_t e_prof_info; }; }; @@ -255,8 +279,8 @@ TYPED_LIST(edata_list_inactive, edata_t, ql_link_inactive) static inline unsigned edata_arena_ind_get(const edata_t *edata) { - unsigned arena_ind = (unsigned)((edata->e_bits & - EDATA_BITS_ARENA_MASK) >> EDATA_BITS_ARENA_SHIFT); + unsigned arena_ind = (unsigned)((edata->e_bits & EDATA_BITS_ARENA_MASK) + >> EDATA_BITS_ARENA_SHIFT); assert(arena_ind < MALLOCX_ARENA_LIMIT); return arena_ind; @@ -264,8 +288,8 @@ edata_arena_ind_get(const edata_t *edata) { static inline szind_t edata_szind_get_maybe_invalid(const edata_t *edata) { - szind_t szind = (szind_t)((edata->e_bits & EDATA_BITS_SZIND_MASK) >> - EDATA_BITS_SZIND_SHIFT); + szind_t szind = (szind_t)((edata->e_bits & EDATA_BITS_SZIND_MASK) + >> EDATA_BITS_SZIND_SHIFT); assert(szind <= SC_NSIZES); return szind; } @@ -279,13 +303,61 @@ edata_szind_get(const edata_t *edata) { static inline size_t edata_usize_get(const edata_t *edata) { - return sz_index2size(edata_szind_get(edata)); + assert(edata != NULL); + /* + * When sz_large_size_classes_disabled() is true, two cases: + * 1. if usize_from_ind is not smaller than SC_LARGE_MINCLASS, + * usize_from_size is accurate; + * 2. otherwise, usize_from_ind is accurate. + * + * When sz_large_size_classes_disabled() is not true, the two should be the + * same when usize_from_ind is not smaller than SC_LARGE_MINCLASS. + * + * Note sampled small allocs will be promoted. Their extent size is + * recorded in edata_size_get(edata), while their szind reflects the + * true usize. Thus, usize retrieved here is still accurate for + * sampled small allocs. + */ + szind_t szind = edata_szind_get(edata); +#ifdef JEMALLOC_JET + /* + * Double free is invalid and results in undefined behavior. However, + * for double free tests to end gracefully, return an invalid usize + * when szind shows the edata is not active, i.e., szind == SC_NSIZES. + */ + if (unlikely(szind == SC_NSIZES)) { + return SC_LARGE_MAXCLASS + 1; + } +#endif + + if (!sz_large_size_classes_disabled() || szind < SC_NBINS) { + size_t usize_from_ind = sz_index2size(szind); + if (!sz_large_size_classes_disabled() + && usize_from_ind >= SC_LARGE_MINCLASS) { + size_t size = (edata->e_size_esn & EDATA_SIZE_MASK); + assert(size > sz_large_pad); + size_t usize_from_size = size - sz_large_pad; + assert(usize_from_ind == usize_from_size); + } + return usize_from_ind; + } + + size_t size = (edata->e_size_esn & EDATA_SIZE_MASK); + assert(size > sz_large_pad); + size_t usize_from_size = size - sz_large_pad; + /* + * no matter large size classes disabled or not, usize retrieved from + * size is not accurate when smaller than SC_LARGE_MINCLASS. + */ + assert(usize_from_size >= SC_LARGE_MINCLASS); + return usize_from_size; } static inline unsigned edata_binshard_get(const edata_t *edata) { - unsigned binshard = (unsigned)((edata->e_bits & - EDATA_BITS_BINSHARD_MASK) >> EDATA_BITS_BINSHARD_SHIFT); + unsigned binshard = (unsigned)((edata->e_bits + & EDATA_BITS_BINSHARD_MASK) + >> EDATA_BITS_BINSHARD_SHIFT); assert(binshard < bin_infos[edata_szind_get(edata)].n_shards); return binshard; } @@ -297,58 +369,58 @@ edata_sn_get(const edata_t *edata) { static inline extent_state_t edata_state_get(const edata_t *edata) { - return (extent_state_t)((edata->e_bits & EDATA_BITS_STATE_MASK) >> - EDATA_BITS_STATE_SHIFT); + return (extent_state_t)((edata->e_bits & EDATA_BITS_STATE_MASK) + >> EDATA_BITS_STATE_SHIFT); } static inline bool edata_guarded_get(const edata_t *edata) { - return (bool)((edata->e_bits & EDATA_BITS_GUARDED_MASK) >> - EDATA_BITS_GUARDED_SHIFT); + return (bool)((edata->e_bits & EDATA_BITS_GUARDED_MASK) + >> EDATA_BITS_GUARDED_SHIFT); } static inline bool edata_zeroed_get(const edata_t *edata) { - return (bool)((edata->e_bits & EDATA_BITS_ZEROED_MASK) >> - EDATA_BITS_ZEROED_SHIFT); + return (bool)((edata->e_bits & EDATA_BITS_ZEROED_MASK) + >> EDATA_BITS_ZEROED_SHIFT); } static inline bool edata_committed_get(const edata_t *edata) { - return (bool)((edata->e_bits & EDATA_BITS_COMMITTED_MASK) >> - EDATA_BITS_COMMITTED_SHIFT); + return (bool)((edata->e_bits & EDATA_BITS_COMMITTED_MASK) + >> EDATA_BITS_COMMITTED_SHIFT); } static inline extent_pai_t edata_pai_get(const edata_t *edata) { - return (extent_pai_t)((edata->e_bits & EDATA_BITS_PAI_MASK) >> - EDATA_BITS_PAI_SHIFT); + return (extent_pai_t)((edata->e_bits & EDATA_BITS_PAI_MASK) + >> EDATA_BITS_PAI_SHIFT); } static inline bool edata_slab_get(const edata_t *edata) { - return (bool)((edata->e_bits & EDATA_BITS_SLAB_MASK) >> - EDATA_BITS_SLAB_SHIFT); + return (bool)((edata->e_bits & EDATA_BITS_SLAB_MASK) + >> EDATA_BITS_SLAB_SHIFT); } static inline unsigned edata_nfree_get(const edata_t *edata) { assert(edata_slab_get(edata)); - return (unsigned)((edata->e_bits & EDATA_BITS_NFREE_MASK) >> - EDATA_BITS_NFREE_SHIFT); + return (unsigned)((edata->e_bits & EDATA_BITS_NFREE_MASK) + >> EDATA_BITS_NFREE_SHIFT); } static inline void * edata_base_get(const edata_t *edata) { - assert(edata->e_addr == PAGE_ADDR2BASE(edata->e_addr) || - !edata_slab_get(edata)); + assert(edata->e_addr == PAGE_ADDR2BASE(edata->e_addr) + || !edata_slab_get(edata)); return PAGE_ADDR2BASE(edata->e_addr); } static inline void * edata_addr_get(const edata_t *edata) { - assert(edata->e_addr == PAGE_ADDR2BASE(edata->e_addr) || - !edata_slab_get(edata)); + assert(edata->e_addr == PAGE_ADDR2BASE(edata->e_addr) + || !edata_slab_get(edata)); return edata->e_addr; } @@ -375,19 +447,19 @@ edata_ps_get(const edata_t *edata) { static inline void * edata_before_get(const edata_t *edata) { - return (void *)((uintptr_t)edata_base_get(edata) - PAGE); + return (void *)((byte_t *)edata_base_get(edata) - PAGE); } static inline void * edata_last_get(const edata_t *edata) { - return (void *)((uintptr_t)edata_base_get(edata) + - edata_size_get(edata) - PAGE); + return (void *)((byte_t *)edata_base_get(edata) + edata_size_get(edata) + - PAGE); } static inline void * edata_past_get(const edata_t *edata) { - return (void *)((uintptr_t)edata_base_get(edata) + - edata_size_get(edata)); + return ( + void *)((byte_t *)edata_base_get(edata) + edata_size_get(edata)); } static inline slab_data_t * @@ -404,8 +476,8 @@ edata_slab_data_get_const(const edata_t *edata) { static inline prof_tctx_t * edata_prof_tctx_get(const edata_t *edata) { - return (prof_tctx_t *)atomic_load_p(&edata->e_prof_info.e_prof_tctx, - ATOMIC_ACQUIRE); + return (prof_tctx_t *)atomic_load_p( + &edata->e_prof_info.e_prof_tctx, ATOMIC_ACQUIRE); } static inline const nstime_t * @@ -426,16 +498,16 @@ edata_prof_recent_alloc_get_dont_call_directly(const edata_t *edata) { static inline void edata_arena_ind_set(edata_t *edata, unsigned arena_ind) { - edata->e_bits = (edata->e_bits & ~EDATA_BITS_ARENA_MASK) | - ((uint64_t)arena_ind << EDATA_BITS_ARENA_SHIFT); + edata->e_bits = (edata->e_bits & ~EDATA_BITS_ARENA_MASK) + | ((uint64_t)arena_ind << EDATA_BITS_ARENA_SHIFT); } static inline void edata_binshard_set(edata_t *edata, unsigned binshard) { /* The assertion assumes szind is set already. */ assert(binshard < bin_infos[edata_szind_get(edata)].n_shards); - edata->e_bits = (edata->e_bits & ~EDATA_BITS_BINSHARD_MASK) | - ((uint64_t)binshard << EDATA_BITS_BINSHARD_SHIFT); + edata->e_bits = (edata->e_bits & ~EDATA_BITS_BINSHARD_MASK) + | ((uint64_t)binshard << EDATA_BITS_BINSHARD_SHIFT); } static inline void @@ -451,8 +523,8 @@ edata_size_set(edata_t *edata, size_t size) { static inline void edata_esn_set(edata_t *edata, size_t esn) { - edata->e_size_esn = (edata->e_size_esn & ~EDATA_ESN_MASK) | (esn & - EDATA_ESN_MASK); + edata->e_size_esn = (edata->e_size_esn & ~EDATA_ESN_MASK) + | (esn & EDATA_ESN_MASK); } static inline void @@ -469,25 +541,26 @@ edata_ps_set(edata_t *edata, hpdata_t *ps) { static inline void edata_szind_set(edata_t *edata, szind_t szind) { assert(szind <= SC_NSIZES); /* SC_NSIZES means "invalid". */ - edata->e_bits = (edata->e_bits & ~EDATA_BITS_SZIND_MASK) | - ((uint64_t)szind << EDATA_BITS_SZIND_SHIFT); + edata->e_bits = (edata->e_bits & ~EDATA_BITS_SZIND_MASK) + | ((uint64_t)szind << EDATA_BITS_SZIND_SHIFT); } static inline void edata_nfree_set(edata_t *edata, unsigned nfree) { assert(edata_slab_get(edata)); - edata->e_bits = (edata->e_bits & ~EDATA_BITS_NFREE_MASK) | - ((uint64_t)nfree << EDATA_BITS_NFREE_SHIFT); + edata->e_bits = (edata->e_bits & ~EDATA_BITS_NFREE_MASK) + | ((uint64_t)nfree << EDATA_BITS_NFREE_SHIFT); } static inline void edata_nfree_binshard_set(edata_t *edata, unsigned nfree, unsigned binshard) { /* The assertion assumes szind is set already. */ assert(binshard < bin_infos[edata_szind_get(edata)].n_shards); - edata->e_bits = (edata->e_bits & - (~EDATA_BITS_NFREE_MASK & ~EDATA_BITS_BINSHARD_MASK)) | - ((uint64_t)binshard << EDATA_BITS_BINSHARD_SHIFT) | - ((uint64_t)nfree << EDATA_BITS_NFREE_SHIFT); + edata->e_bits = (edata->e_bits + & (~EDATA_BITS_NFREE_MASK + & ~EDATA_BITS_BINSHARD_MASK)) + | ((uint64_t)binshard << EDATA_BITS_BINSHARD_SHIFT) + | ((uint64_t)nfree << EDATA_BITS_NFREE_SHIFT); } static inline void @@ -515,38 +588,38 @@ edata_sn_set(edata_t *edata, uint64_t sn) { static inline void edata_state_set(edata_t *edata, extent_state_t state) { - edata->e_bits = (edata->e_bits & ~EDATA_BITS_STATE_MASK) | - ((uint64_t)state << EDATA_BITS_STATE_SHIFT); + edata->e_bits = (edata->e_bits & ~EDATA_BITS_STATE_MASK) + | ((uint64_t)state << EDATA_BITS_STATE_SHIFT); } static inline void edata_guarded_set(edata_t *edata, bool guarded) { - edata->e_bits = (edata->e_bits & ~EDATA_BITS_GUARDED_MASK) | - ((uint64_t)guarded << EDATA_BITS_GUARDED_SHIFT); + edata->e_bits = (edata->e_bits & ~EDATA_BITS_GUARDED_MASK) + | ((uint64_t)guarded << EDATA_BITS_GUARDED_SHIFT); } static inline void edata_zeroed_set(edata_t *edata, bool zeroed) { - edata->e_bits = (edata->e_bits & ~EDATA_BITS_ZEROED_MASK) | - ((uint64_t)zeroed << EDATA_BITS_ZEROED_SHIFT); + edata->e_bits = (edata->e_bits & ~EDATA_BITS_ZEROED_MASK) + | ((uint64_t)zeroed << EDATA_BITS_ZEROED_SHIFT); } static inline void edata_committed_set(edata_t *edata, bool committed) { - edata->e_bits = (edata->e_bits & ~EDATA_BITS_COMMITTED_MASK) | - ((uint64_t)committed << EDATA_BITS_COMMITTED_SHIFT); + edata->e_bits = (edata->e_bits & ~EDATA_BITS_COMMITTED_MASK) + | ((uint64_t)committed << EDATA_BITS_COMMITTED_SHIFT); } static inline void edata_pai_set(edata_t *edata, extent_pai_t pai) { - edata->e_bits = (edata->e_bits & ~EDATA_BITS_PAI_MASK) | - ((uint64_t)pai << EDATA_BITS_PAI_SHIFT); + edata->e_bits = (edata->e_bits & ~EDATA_BITS_PAI_MASK) + | ((uint64_t)pai << EDATA_BITS_PAI_SHIFT); } static inline void edata_slab_set(edata_t *edata, bool slab) { - edata->e_bits = (edata->e_bits & ~EDATA_BITS_SLAB_MASK) | - ((uint64_t)slab << EDATA_BITS_SLAB_SHIFT); + edata->e_bits = (edata->e_bits & ~EDATA_BITS_SLAB_MASK) + | ((uint64_t)slab << EDATA_BITS_SLAB_SHIFT); } static inline void @@ -565,22 +638,22 @@ edata_prof_alloc_size_set(edata_t *edata, size_t size) { } static inline void -edata_prof_recent_alloc_set_dont_call_directly(edata_t *edata, - prof_recent_t *recent_alloc) { +edata_prof_recent_alloc_set_dont_call_directly( + edata_t *edata, prof_recent_t *recent_alloc) { atomic_store_p(&edata->e_prof_info.e_prof_recent_alloc, recent_alloc, ATOMIC_RELAXED); } static inline bool edata_is_head_get(edata_t *edata) { - return (bool)((edata->e_bits & EDATA_BITS_IS_HEAD_MASK) >> - EDATA_BITS_IS_HEAD_SHIFT); + return (bool)((edata->e_bits & EDATA_BITS_IS_HEAD_MASK) + >> EDATA_BITS_IS_HEAD_SHIFT); } static inline void edata_is_head_set(edata_t *edata, bool is_head) { - edata->e_bits = (edata->e_bits & ~EDATA_BITS_IS_HEAD_MASK) | - ((uint64_t)is_head << EDATA_BITS_IS_HEAD_SHIFT); + edata->e_bits = (edata->e_bits & ~EDATA_BITS_IS_HEAD_MASK) + | ((uint64_t)is_head << EDATA_BITS_IS_HEAD_SHIFT); } static inline bool @@ -619,7 +692,8 @@ edata_init(edata_t *edata, unsigned arena_ind, void *addr, size_t size, } static inline void -edata_binit(edata_t *edata, void *addr, size_t bsize, uint64_t sn) { +edata_binit( + edata_t *edata, void *addr, size_t bsize, uint64_t sn, bool reused) { edata_arena_ind_set(edata, (1U << MALLOCX_ARENA_BITS) - 1); edata_addr_set(edata, addr); edata_bsize_set(edata, bsize); @@ -627,7 +701,8 @@ edata_binit(edata_t *edata, void *addr, size_t bsize, uint64_t sn) { edata_szind_set(edata, SC_NSIZES); edata_sn_set(edata, sn); edata_state_set(edata, extent_state_active); - edata_guarded_set(edata, false); + /* See comments in base_edata_is_reused. */ + edata_guarded_set(edata, reused); edata_zeroed_set(edata, true); edata_committed_set(edata, true); /* @@ -656,20 +731,47 @@ edata_ead_comp(const edata_t *a, const edata_t *b) { static inline edata_cmp_summary_t edata_cmp_summary_get(const edata_t *edata) { - return (edata_cmp_summary_t){edata_sn_get(edata), - (uintptr_t)edata_addr_get(edata)}; + edata_cmp_summary_t result; + result.sn = edata_sn_get(edata); + result.addr = (uintptr_t)edata_addr_get(edata); + return result; +} + +#ifdef JEMALLOC_HAVE_INT128 +JEMALLOC_ALWAYS_INLINE unsigned __int128 +edata_cmp_summary_encode(edata_cmp_summary_t src) { + return ((unsigned __int128)src.sn << 64) | src.addr; } static inline int edata_cmp_summary_comp(edata_cmp_summary_t a, edata_cmp_summary_t b) { - int ret; - ret = (a.sn > b.sn) - (a.sn < b.sn); - if (ret != 0) { - return ret; - } - ret = (a.addr > b.addr) - (a.addr < b.addr); - return ret; + unsigned __int128 a_encoded = edata_cmp_summary_encode(a); + unsigned __int128 b_encoded = edata_cmp_summary_encode(b); + if (a_encoded < b_encoded) + return -1; + if (a_encoded == b_encoded) + return 0; + return 1; } +#else +static inline int +edata_cmp_summary_comp(edata_cmp_summary_t a, edata_cmp_summary_t b) { + /* + * Logically, what we're doing here is comparing based on `.sn`, and + * falling back to comparing on `.addr` in the case that `a.sn == b.sn`. + * We accomplish this by multiplying the result of the `.sn` comparison + * by 2, so that so long as it is not 0, it will dominate the `.addr` + * comparison in determining the sign of the returned result value. + * The justification for doing things this way is that this is + * branchless - all of the branches that would be present in a + * straightforward implementation are common cases, and thus the branch + * prediction accuracy is not great. As a result, this implementation + * is measurably faster (by around 30%). + */ + return (2 * ((a.sn > b.sn) - (a.sn < b.sn))) + + ((a.addr > b.addr) - (a.addr < b.addr)); +} +#endif static inline int edata_snad_comp(const edata_t *a, const edata_t *b) { @@ -681,18 +783,13 @@ edata_snad_comp(const edata_t *a, const edata_t *b) { static inline int edata_esnead_comp(const edata_t *a, const edata_t *b) { - int ret; - - ret = edata_esn_comp(a, b); - if (ret != 0) { - return ret; - } - - ret = edata_ead_comp(a, b); - return ret; + /* + * Similar to `edata_cmp_summary_comp`, we've opted for a + * branchless implementation for the sake of performance. + */ + return (2 * edata_esn_comp(a, b)) + edata_ead_comp(a, b); } -ph_proto(, edata_avail, edata_t) -ph_proto(, edata_heap, edata_t) +ph_proto(, edata_avail, edata_t) ph_proto(, edata_heap, edata_t) #endif /* JEMALLOC_INTERNAL_EDATA_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/edata_cache.h b/dep/jemalloc/include/jemalloc/internal/edata_cache.h index 8b6c0ef79..d92d90cb4 100644 --- a/dep/jemalloc/include/jemalloc/internal/edata_cache.h +++ b/dep/jemalloc/include/jemalloc/internal/edata_cache.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_EDATA_CACHE_H #define JEMALLOC_INTERNAL_EDATA_CACHE_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/base.h" /* For tests only. */ @@ -14,13 +15,13 @@ typedef struct edata_cache_s edata_cache_t; struct edata_cache_s { - edata_avail_t avail; - atomic_zu_t count; + edata_avail_t avail; + atomic_zu_t count; malloc_mutex_t mtx; - base_t *base; + base_t *base; }; -bool edata_cache_init(edata_cache_t *edata_cache, base_t *base); +bool edata_cache_init(edata_cache_t *edata_cache, base_t *base); edata_t *edata_cache_get(tsdn_t *tsdn, edata_cache_t *edata_cache); void edata_cache_put(tsdn_t *tsdn, edata_cache_t *edata_cache, edata_t *edata); @@ -36,14 +37,14 @@ void edata_cache_postfork_child(tsdn_t *tsdn, edata_cache_t *edata_cache); typedef struct edata_cache_fast_s edata_cache_fast_t; struct edata_cache_fast_s { edata_list_inactive_t list; - edata_cache_t *fallback; - bool disabled; + edata_cache_t *fallback; + bool disabled; }; void edata_cache_fast_init(edata_cache_fast_t *ecs, edata_cache_t *fallback); edata_t *edata_cache_fast_get(tsdn_t *tsdn, edata_cache_fast_t *ecs); -void edata_cache_fast_put(tsdn_t *tsdn, edata_cache_fast_t *ecs, - edata_t *edata); +void edata_cache_fast_put( + tsdn_t *tsdn, edata_cache_fast_t *ecs, edata_t *edata); void edata_cache_fast_disable(tsdn_t *tsdn, edata_cache_fast_t *ecs); #endif /* JEMALLOC_INTERNAL_EDATA_CACHE_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/ehooks.h b/dep/jemalloc/include/jemalloc/internal/ehooks.h index 8d9513e25..c65e189af 100644 --- a/dep/jemalloc/include/jemalloc/internal/ehooks.h +++ b/dep/jemalloc/include/jemalloc/internal/ehooks.h @@ -1,8 +1,11 @@ #ifndef JEMALLOC_INTERNAL_EHOOKS_H #define JEMALLOC_INTERNAL_EHOOKS_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/extent_mmap.h" +#include "jemalloc/internal/tsd.h" +#include "jemalloc/internal/tsd_types.h" /* * This module is the internal interface to the extent hooks (both @@ -43,17 +46,17 @@ extern const extent_hooks_t ehooks_default_extent_hooks; */ void *ehooks_default_alloc_impl(tsdn_t *tsdn, void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit, unsigned arena_ind); -bool ehooks_default_dalloc_impl(void *addr, size_t size); -void ehooks_default_destroy_impl(void *addr, size_t size); -bool ehooks_default_commit_impl(void *addr, size_t offset, size_t length); -bool ehooks_default_decommit_impl(void *addr, size_t offset, size_t length); +bool ehooks_default_dalloc_impl(void *addr, size_t size); +void ehooks_default_destroy_impl(void *addr, size_t size); +bool ehooks_default_commit_impl(void *addr, size_t offset, size_t length); +bool ehooks_default_decommit_impl(void *addr, size_t offset, size_t length); #ifdef PAGES_CAN_PURGE_LAZY bool ehooks_default_purge_lazy_impl(void *addr, size_t offset, size_t length); #endif #ifdef PAGES_CAN_PURGE_FORCED bool ehooks_default_purge_forced_impl(void *addr, size_t offset, size_t length); #endif -bool ehooks_default_split_impl(); +bool ehooks_default_split_impl(void); /* * Merge is the only default extent hook we declare -- see the comment in * ehooks_merge. @@ -113,8 +116,8 @@ ehooks_get_extent_hooks_ptr(ehooks_t *ehooks) { static inline bool ehooks_are_default(ehooks_t *ehooks) { - return ehooks_get_extent_hooks_ptr(ehooks) == - &ehooks_default_extent_hooks; + return ehooks_get_extent_hooks_ptr(ehooks) + == &ehooks_default_extent_hooks; } /* @@ -186,16 +189,15 @@ ehooks_debug_zero_check(void *addr, size_t size) { } } - static inline void * ehooks_alloc(tsdn_t *tsdn, ehooks_t *ehooks, void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit) { - bool orig_zero = *zero; - void *ret; + bool orig_zero = *zero; + void *ret; extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); if (extent_hooks == &ehooks_default_extent_hooks) { - ret = ehooks_default_alloc_impl(tsdn, new_addr, size, - alignment, zero, commit, ehooks_ind_get(ehooks)); + ret = ehooks_default_alloc_impl(tsdn, new_addr, size, alignment, + zero, commit, ehooks_ind_get(ehooks)); } else { ehooks_pre_reentrancy(tsdn); ret = extent_hooks->alloc(extent_hooks, new_addr, size, @@ -211,8 +213,8 @@ ehooks_alloc(tsdn_t *tsdn, ehooks_t *ehooks, void *new_addr, size_t size, } static inline bool -ehooks_dalloc(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size, - bool committed) { +ehooks_dalloc( + tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size, bool committed) { extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); if (extent_hooks == &ehooks_default_extent_hooks) { return ehooks_default_dalloc_impl(addr, size); @@ -228,8 +230,8 @@ ehooks_dalloc(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size, } static inline void -ehooks_destroy(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size, - bool committed) { +ehooks_destroy( + tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size, bool committed) { extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); if (extent_hooks == &ehooks_default_extent_hooks) { ehooks_default_destroy_impl(addr, size); @@ -247,15 +249,15 @@ static inline bool ehooks_commit(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size, size_t offset, size_t length) { extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); - bool err; + bool err; if (extent_hooks == &ehooks_default_extent_hooks) { err = ehooks_default_commit_impl(addr, offset, length); } else if (extent_hooks->commit == NULL) { err = true; } else { ehooks_pre_reentrancy(tsdn); - err = extent_hooks->commit(extent_hooks, addr, size, - offset, length, ehooks_ind_get(ehooks)); + err = extent_hooks->commit(extent_hooks, addr, size, offset, + length, ehooks_ind_get(ehooks)); ehooks_post_reentrancy(tsdn); } if (!err) { @@ -381,7 +383,7 @@ ehooks_zero(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size) { static inline bool ehooks_guard(tsdn_t *tsdn, ehooks_t *ehooks, void *guard1, void *guard2) { - bool err; + bool err; extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); if (extent_hooks == &ehooks_default_extent_hooks) { @@ -396,7 +398,7 @@ ehooks_guard(tsdn_t *tsdn, ehooks_t *ehooks, void *guard1, void *guard2) { static inline bool ehooks_unguard(tsdn_t *tsdn, ehooks_t *ehooks, void *guard1, void *guard2) { - bool err; + bool err; extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); if (extent_hooks == &ehooks_default_extent_hooks) { diff --git a/dep/jemalloc/include/jemalloc/internal/emap.h b/dep/jemalloc/include/jemalloc/internal/emap.h index 847af3278..886923564 100644 --- a/dep/jemalloc/include/jemalloc/internal/emap.h +++ b/dep/jemalloc/include/jemalloc/internal/emap.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_EMAP_H #define JEMALLOC_INTERNAL_EMAP_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/base.h" #include "jemalloc/internal/rtree.h" @@ -9,9 +10,9 @@ * EMAP_DECLARE_RTREE_CTX; * in uses will avoid empty-statement warnings. */ -#define EMAP_DECLARE_RTREE_CTX \ - rtree_ctx_t rtree_ctx_fallback; \ - rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback) +#define EMAP_DECLARE_RTREE_CTX \ + rtree_ctx_t rtree_ctx_fallback; \ + rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback) typedef struct emap_s emap_t; struct emap_s { @@ -19,26 +20,27 @@ struct emap_s { }; /* Used to pass rtree lookup context down the path. */ -typedef struct emap_alloc_ctx_t emap_alloc_ctx_t; -struct emap_alloc_ctx_t { +typedef struct emap_alloc_ctx_s emap_alloc_ctx_t; +struct emap_alloc_ctx_s { + size_t usize; szind_t szind; - bool slab; + bool slab; }; typedef struct emap_full_alloc_ctx_s emap_full_alloc_ctx_t; struct emap_full_alloc_ctx_s { - szind_t szind; - bool slab; + szind_t szind; + bool slab; edata_t *edata; }; bool emap_init(emap_t *emap, base_t *base, bool zeroed); -void emap_remap(tsdn_t *tsdn, emap_t *emap, edata_t *edata, szind_t szind, - bool slab); +void emap_remap( + tsdn_t *tsdn, emap_t *emap, edata_t *edata, szind_t szind, bool slab); -void emap_update_edata_state(tsdn_t *tsdn, emap_t *emap, edata_t *edata, - extent_state_t state); +void emap_update_edata_state( + tsdn_t *tsdn, emap_t *emap, edata_t *edata, extent_state_t state); /* * The two acquire functions below allow accessing neighbor edatas, if it's safe @@ -60,16 +62,16 @@ edata_t *emap_try_acquire_edata_neighbor(tsdn_t *tsdn, emap_t *emap, bool forward); edata_t *emap_try_acquire_edata_neighbor_expand(tsdn_t *tsdn, emap_t *emap, edata_t *edata, extent_pai_t pai, extent_state_t expected_state); -void emap_release_edata(tsdn_t *tsdn, emap_t *emap, edata_t *edata, - extent_state_t new_state); +void emap_release_edata( + tsdn_t *tsdn, emap_t *emap, edata_t *edata, extent_state_t new_state); /* * Associate the given edata with its beginning and end address, setting the * szind and slab info appropriately. * Returns true on error (i.e. resource exhaustion). */ -bool emap_register_boundary(tsdn_t *tsdn, emap_t *emap, edata_t *edata, - szind_t szind, bool slab); +bool emap_register_boundary( + tsdn_t *tsdn, emap_t *emap, edata_t *edata, szind_t szind, bool slab); /* * Does the same thing, but with the interior of the range, for slab @@ -90,8 +92,8 @@ bool emap_register_boundary(tsdn_t *tsdn, emap_t *emap, edata_t *edata, * touched, so no allocation is necessary to fill the interior once the boundary * has been touched. */ -void emap_register_interior(tsdn_t *tsdn, emap_t *emap, edata_t *edata, - szind_t szind); +void emap_register_interior( + tsdn_t *tsdn, emap_t *emap, edata_t *edata, szind_t szind); void emap_deregister_boundary(tsdn_t *tsdn, emap_t *emap, edata_t *edata); void emap_deregister_interior(tsdn_t *tsdn, emap_t *emap, edata_t *edata); @@ -159,8 +161,8 @@ emap_edata_in_transition(tsdn_t *tsdn, emap_t *emap, edata_t *edata) { emap_assert_mapped(tsdn, emap, edata); EMAP_DECLARE_RTREE_CTX; - rtree_contents_t contents = rtree_read(tsdn, &emap->rtree, rtree_ctx, - (uintptr_t)edata_base_get(edata)); + rtree_contents_t contents = rtree_read( + tsdn, &emap->rtree, rtree_ctx, (uintptr_t)edata_base_get(edata)); return edata_state_in_transition(contents.metadata.state); } @@ -185,16 +187,16 @@ emap_edata_is_acquired(tsdn_t *tsdn, emap_t *emap, edata_t *edata) { */ EMAP_DECLARE_RTREE_CTX; rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, &emap->rtree, - rtree_ctx, (uintptr_t)edata_base_get(edata), /* dependent */ true, + rtree_ctx, (uintptr_t)edata_base_get(edata), /* dependent */ false, /* init_missing */ false); if (elm == NULL) { return true; } rtree_contents_t contents = rtree_leaf_elm_read(tsdn, &emap->rtree, elm, - /* dependent */ true); - if (contents.edata == NULL || - contents.metadata.state == extent_state_active || - edata_state_in_transition(contents.metadata.state)) { + /* dependent */ false); + if (contents.edata == NULL + || contents.metadata.state == extent_state_active + || edata_state_in_transition(contents.metadata.state)) { return true; } @@ -209,8 +211,8 @@ extent_assert_can_coalesce(const edata_t *inner, const edata_t *outer) { assert(edata_state_get(inner) == extent_state_active); assert(edata_state_get(outer) == extent_state_merging); assert(!edata_guarded_get(inner) && !edata_guarded_get(outer)); - assert(edata_base_get(inner) == edata_past_get(outer) || - edata_base_get(outer) == edata_past_get(inner)); + assert(edata_base_get(inner) == edata_past_get(outer) + || edata_base_get(outer) == edata_past_get(inner)); } JEMALLOC_ALWAYS_INLINE void @@ -229,16 +231,46 @@ emap_edata_lookup(tsdn_t *tsdn, emap_t *emap, const void *ptr) { return rtree_read(tsdn, &emap->rtree, rtree_ctx, (uintptr_t)ptr).edata; } +JEMALLOC_ALWAYS_INLINE void +emap_alloc_ctx_init( + emap_alloc_ctx_t *alloc_ctx, szind_t szind, bool slab, size_t usize) { + alloc_ctx->szind = szind; + alloc_ctx->slab = slab; + alloc_ctx->usize = usize; + assert( + sz_large_size_classes_disabled() || usize == sz_index2size(szind)); +} + +JEMALLOC_ALWAYS_INLINE size_t +emap_alloc_ctx_usize_get(emap_alloc_ctx_t *alloc_ctx) { + assert(alloc_ctx->szind < SC_NSIZES); + if (alloc_ctx->slab) { + assert(alloc_ctx->usize == sz_index2size(alloc_ctx->szind)); + return sz_index2size(alloc_ctx->szind); + } + assert(sz_large_size_classes_disabled() + || alloc_ctx->usize == sz_index2size(alloc_ctx->szind)); + assert(alloc_ctx->usize <= SC_LARGE_MAXCLASS); + return alloc_ctx->usize; +} + /* Fills in alloc_ctx with the info in the map. */ JEMALLOC_ALWAYS_INLINE void -emap_alloc_ctx_lookup(tsdn_t *tsdn, emap_t *emap, const void *ptr, - emap_alloc_ctx_t *alloc_ctx) { +emap_alloc_ctx_lookup( + tsdn_t *tsdn, emap_t *emap, const void *ptr, emap_alloc_ctx_t *alloc_ctx) { EMAP_DECLARE_RTREE_CTX; - rtree_metadata_t metadata = rtree_metadata_read(tsdn, &emap->rtree, - rtree_ctx, (uintptr_t)ptr); - alloc_ctx->szind = metadata.szind; - alloc_ctx->slab = metadata.slab; + rtree_contents_t contents = rtree_read( + tsdn, &emap->rtree, rtree_ctx, (uintptr_t)ptr); + /* + * If the alloc is invalid, do not calculate usize since edata + * could be corrupted. + */ + emap_alloc_ctx_init(alloc_ctx, contents.metadata.szind, + contents.metadata.slab, + (contents.metadata.szind == SC_NSIZES || contents.edata == NULL) + ? 0 + : edata_usize_get(contents.edata)); } /* The pointer must be mapped. */ @@ -247,8 +279,8 @@ emap_full_alloc_ctx_lookup(tsdn_t *tsdn, emap_t *emap, const void *ptr, emap_full_alloc_ctx_t *full_alloc_ctx) { EMAP_DECLARE_RTREE_CTX; - rtree_contents_t contents = rtree_read(tsdn, &emap->rtree, rtree_ctx, - (uintptr_t)ptr); + rtree_contents_t contents = rtree_read( + tsdn, &emap->rtree, rtree_ctx, (uintptr_t)ptr); full_alloc_ctx->edata = contents.edata; full_alloc_ctx->szind = contents.metadata.szind; full_alloc_ctx->slab = contents.metadata.slab; @@ -265,8 +297,8 @@ emap_full_alloc_ctx_try_lookup(tsdn_t *tsdn, emap_t *emap, const void *ptr, EMAP_DECLARE_RTREE_CTX; rtree_contents_t contents; - bool err = rtree_read_independent(tsdn, &emap->rtree, rtree_ctx, - (uintptr_t)ptr, &contents); + bool err = rtree_read_independent( + tsdn, &emap->rtree, rtree_ctx, (uintptr_t)ptr, &contents); if (err) { return true; } @@ -281,19 +313,26 @@ emap_full_alloc_ctx_try_lookup(tsdn_t *tsdn, emap_t *emap, const void *ptr, * fast path, e.g. when the metadata key is not cached. */ JEMALLOC_ALWAYS_INLINE bool -emap_alloc_ctx_try_lookup_fast(tsd_t *tsd, emap_t *emap, const void *ptr, - emap_alloc_ctx_t *alloc_ctx) { +emap_alloc_ctx_try_lookup_fast( + tsd_t *tsd, emap_t *emap, const void *ptr, emap_alloc_ctx_t *alloc_ctx) { /* Use the unsafe getter since this may gets called during exit. */ rtree_ctx_t *rtree_ctx = tsd_rtree_ctxp_get_unsafe(tsd); rtree_metadata_t metadata; - bool err = rtree_metadata_try_read_fast(tsd_tsdn(tsd), &emap->rtree, - rtree_ctx, (uintptr_t)ptr, &metadata); + bool err = rtree_metadata_try_read_fast( + tsd_tsdn(tsd), &emap->rtree, rtree_ctx, (uintptr_t)ptr, &metadata); if (err) { return true; } + /* + * Small allocs using the fastpath can always use index to get the + * usize. Therefore, do not set alloc_ctx->usize here. + */ alloc_ctx->szind = metadata.szind; alloc_ctx->slab = metadata.slab; + if (config_debug) { + alloc_ctx->usize = SC_LARGE_MAXCLASS + 1; + } return false; } @@ -308,11 +347,12 @@ typedef const void *(*emap_ptr_getter)(void *ctx, size_t ind); * This allows size-checking assertions, which we can only do while we're in the * process of edata lookups. */ -typedef void (*emap_metadata_visitor)(void *ctx, emap_full_alloc_ctx_t *alloc_ctx); +typedef void (*emap_metadata_visitor)( + void *ctx, emap_full_alloc_ctx_t *alloc_ctx); typedef union emap_batch_lookup_result_u emap_batch_lookup_result_t; union emap_batch_lookup_result_u { - edata_t *edata; + edata_t *edata; rtree_leaf_elm_t *rtree_leaf; }; @@ -338,8 +378,8 @@ emap_edata_lookup_batch(tsd_t *tsd, emap_t *emap, size_t nptrs, for (size_t i = 0; i < nptrs; i++) { rtree_leaf_elm_t *elm = result[i].rtree_leaf; - rtree_contents_t contents = rtree_leaf_elm_read(tsd_tsdn(tsd), - &emap->rtree, elm, /* dependent */ true); + rtree_contents_t contents = rtree_leaf_elm_read( + tsd_tsdn(tsd), &emap->rtree, elm, /* dependent */ true); result[i].edata = contents.edata; emap_full_alloc_ctx_t alloc_ctx; /* diff --git a/dep/jemalloc/include/jemalloc/internal/emitter.h b/dep/jemalloc/include/jemalloc/internal/emitter.h index 9482f68bc..a4073e6aa 100644 --- a/dep/jemalloc/include/jemalloc/internal/emitter.h +++ b/dep/jemalloc/include/jemalloc/internal/emitter.h @@ -1,6 +1,10 @@ #ifndef JEMALLOC_INTERNAL_EMITTER_H #define JEMALLOC_INTERNAL_EMITTER_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/assert.h" +#include "jemalloc/internal/jemalloc_internal_types.h" +#include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/ql.h" typedef enum emitter_output_e emitter_output_t; @@ -40,18 +44,18 @@ typedef struct emitter_col_s emitter_col_t; struct emitter_col_s { /* Filled in by the user. */ emitter_justify_t justify; - int width; - emitter_type_t type; + int width; + emitter_type_t type; union { - bool bool_val; - int int_val; - unsigned unsigned_val; - uint32_t uint32_val; - uint32_t uint32_t_val; - uint64_t uint64_val; - uint64_t uint64_t_val; - size_t size_val; - ssize_t ssize_val; + bool bool_val; + int int_val; + unsigned unsigned_val; + uint32_t uint32_val; + uint32_t uint32_t_val; + uint64_t uint64_val; + uint64_t uint64_t_val; + size_t size_val; + ssize_t ssize_val; const char *str_val; }; @@ -69,8 +73,8 @@ struct emitter_s { emitter_output_t output; /* The output information. */ write_cb_t *write_cb; - void *cbopaque; - int nesting_depth; + void *cbopaque; + int nesting_depth; /* True if we've already emitted a value at the given depth. */ bool item_at_depth; /* True if we emitted a key and will emit corresponding value next. */ @@ -79,8 +83,8 @@ struct emitter_s { static inline bool emitter_outputs_json(emitter_t *emitter) { - return emitter->output == emitter_output_json || - emitter->output == emitter_output_json_compact; + return emitter->output == emitter_output_json + || emitter->output == emitter_output_json_compact; } /* Internal convenience function. Write to the emitter the given string. */ @@ -94,26 +98,57 @@ emitter_printf(emitter_t *emitter, const char *format, ...) { va_end(ap); } -static inline const char * JEMALLOC_FORMAT_ARG(3) -emitter_gen_fmt(char *out_fmt, size_t out_size, const char *fmt_specifier, - emitter_justify_t justify, int width) { +static inline const char * +JEMALLOC_FORMAT_ARG(3) emitter_gen_fmt(char *out_fmt, size_t out_size, + const char *fmt_specifier, emitter_justify_t justify, int width) { size_t written; fmt_specifier++; if (justify == emitter_justify_none) { - written = malloc_snprintf(out_fmt, out_size, - "%%%s", fmt_specifier); + written = malloc_snprintf( + out_fmt, out_size, "%%%s", fmt_specifier); } else if (justify == emitter_justify_left) { - written = malloc_snprintf(out_fmt, out_size, - "%%-%d%s", width, fmt_specifier); + written = malloc_snprintf( + out_fmt, out_size, "%%-%d%s", width, fmt_specifier); } else { - written = malloc_snprintf(out_fmt, out_size, - "%%%d%s", width, fmt_specifier); + written = malloc_snprintf( + out_fmt, out_size, "%%%d%s", width, fmt_specifier); } /* Only happens in case of bad format string, which *we* choose. */ - assert(written < out_size); + assert(written < out_size); return out_fmt; } +static inline void +emitter_emit_str(emitter_t *emitter, emitter_justify_t justify, int width, + char *fmt, size_t fmt_size, const char *str) { +#define BUF_SIZE 256 + char buf[BUF_SIZE]; + size_t str_written = malloc_snprintf(buf, BUF_SIZE, "\"%s\"", str); + emitter_printf( + emitter, emitter_gen_fmt(fmt, fmt_size, "%s", justify, width), buf); + if (str_written < BUF_SIZE) { + return; + } + /* + * There is no support for long string justification at the moment as + * we output them partially with multiple malloc_snprintf calls and + * justufication will work correctly only withing one call. + * Fortunately this is not a big concern as we don't use justufication + * with long strings right now. + * + * We emitted leading quotation mark and trailing '\0', hence need to + * exclude extra characters from str shift. + */ + str += BUF_SIZE - 2; + do { + str_written = malloc_snprintf(buf, BUF_SIZE, "%s\"", str); + str += str_written >= BUF_SIZE ? BUF_SIZE - 1 : str_written; + emitter_printf(emitter, + emitter_gen_fmt(fmt, fmt_size, "%s", justify, width), buf); + } while (str_written >= BUF_SIZE); +#undef BUF_SIZE +} + /* * Internal. Emit the given value type in the relevant encoding (so that the * bool true gets mapped to json "true", but the string "true" gets mapped to @@ -124,8 +159,6 @@ emitter_gen_fmt(char *out_fmt, size_t out_size, const char *fmt_specifier, static inline void emitter_print_value(emitter_t *emitter, emitter_justify_t justify, int width, emitter_type_t value_type, const void *value) { - size_t str_written; -#define BUF_SIZE 256 #define FMT_SIZE 10 /* * We dynamically generate a format string to emit, to let us use the @@ -134,18 +167,17 @@ emitter_print_value(emitter_t *emitter, emitter_justify_t justify, int width, * cases. */ char fmt[FMT_SIZE]; - char buf[BUF_SIZE]; -#define EMIT_SIMPLE(type, format) \ - emitter_printf(emitter, \ - emitter_gen_fmt(fmt, FMT_SIZE, format, justify, width), \ +#define EMIT_SIMPLE(type, format) \ + emitter_printf(emitter, \ + emitter_gen_fmt(fmt, FMT_SIZE, format, justify, width), \ *(const type *)value); switch (value_type) { case emitter_type_bool: emitter_printf(emitter, emitter_gen_fmt(fmt, FMT_SIZE, "%s", justify, width), - *(const bool *)value ? "true" : "false"); + *(const bool *)value ? "true" : "false"); break; case emitter_type_int: EMIT_SIMPLE(int, "%d") @@ -163,15 +195,8 @@ emitter_print_value(emitter_t *emitter, emitter_justify_t justify, int width, EMIT_SIMPLE(size_t, "%zu") break; case emitter_type_string: - str_written = malloc_snprintf(buf, BUF_SIZE, "\"%s\"", + emitter_emit_str(emitter, justify, width, fmt, FMT_SIZE, *(const char *const *)value); - /* - * We control the strings we output; we shouldn't get anything - * anywhere near the fmt size. - */ - assert(str_written < BUF_SIZE); - emitter_printf(emitter, - emitter_gen_fmt(fmt, FMT_SIZE, "%s", justify, width), buf); break; case emitter_type_uint32: EMIT_SIMPLE(uint32_t, "%" FMTu32) @@ -185,11 +210,9 @@ emitter_print_value(emitter_t *emitter, emitter_justify_t justify, int width, default: unreachable(); } -#undef BUF_SIZE #undef FMT_SIZE } - /* Internal functions. In json mode, tracks nesting state. */ static inline void emitter_nest_inc(emitter_t *emitter) { @@ -205,7 +228,7 @@ emitter_nest_dec(emitter_t *emitter) { static inline void emitter_indent(emitter_t *emitter) { - int amount = emitter->nesting_depth; + int amount = emitter->nesting_depth; const char *indent_str; assert(emitter->output != emitter_output_json_compact); if (emitter->output == emitter_output_json) { @@ -267,12 +290,12 @@ emitter_json_key(emitter_t *emitter, const char *json_key) { } static inline void -emitter_json_value(emitter_t *emitter, emitter_type_t value_type, - const void *value) { +emitter_json_value( + emitter_t *emitter, emitter_type_t value_type, const void *value) { if (emitter_outputs_json(emitter)) { emitter_json_key_prefix(emitter); - emitter_print_value(emitter, emitter_justify_none, -1, - value_type, value); + emitter_print_value( + emitter, emitter_justify_none, -1, value_type, value); emitter->item_at_depth = true; } } @@ -343,7 +366,6 @@ emitter_json_object_end(emitter_t *emitter) { } } - /******************************************************************************/ /* Table public API. */ @@ -365,14 +387,13 @@ emitter_table_dict_end(emitter_t *emitter) { static inline void emitter_table_kv_note(emitter_t *emitter, const char *table_key, - emitter_type_t value_type, const void *value, - const char *table_note_key, emitter_type_t table_note_value_type, - const void *table_note_value) { + emitter_type_t value_type, const void *value, const char *table_note_key, + emitter_type_t table_note_value_type, const void *table_note_value) { if (emitter->output == emitter_output_table) { emitter_indent(emitter); emitter_printf(emitter, "%s: ", table_key); - emitter_print_value(emitter, emitter_justify_none, -1, - value_type, value); + emitter_print_value( + emitter, emitter_justify_none, -1, value_type, value); if (table_note_key != NULL) { emitter_printf(emitter, " (%s: ", table_note_key); emitter_print_value(emitter, emitter_justify_none, -1, @@ -391,7 +412,6 @@ emitter_table_kv(emitter_t *emitter, const char *table_key, emitter_type_bool, NULL); } - /* Write to the emitter the given string, but only in table mode. */ JEMALLOC_FORMAT_PRINTF(2, 3) static inline void @@ -399,7 +419,8 @@ emitter_table_printf(emitter_t *emitter, const char *format, ...) { if (emitter->output == emitter_output_table) { va_list ap; va_start(ap, format); - malloc_vcprintf(emitter->write_cb, emitter->cbopaque, format, ap); + malloc_vcprintf( + emitter->write_cb, emitter->cbopaque, format, ap); va_end(ap); } } @@ -410,7 +431,7 @@ emitter_table_row(emitter_t *emitter, emitter_row_t *row) { return; } emitter_col_t *col; - ql_foreach(col, &row->cols, link) { + ql_foreach (col, &row->cols, link) { emitter_print_value(emitter, col->justify, col->width, col->type, (const void *)&col->bool_val); } @@ -428,7 +449,6 @@ emitter_col_init(emitter_col_t *col, emitter_row_t *row) { ql_tail_insert(&row->cols, col, link); } - /******************************************************************************/ /* * Generalized public API. Emits using either JSON or table, according to @@ -440,9 +460,8 @@ emitter_col_init(emitter_col_t *col, emitter_row_t *row) { */ static inline void emitter_kv_note(emitter_t *emitter, const char *json_key, const char *table_key, - emitter_type_t value_type, const void *value, - const char *table_note_key, emitter_type_t table_note_value_type, - const void *table_note_value) { + emitter_type_t value_type, const void *value, const char *table_note_key, + emitter_type_t table_note_value_type, const void *table_note_value) { if (emitter_outputs_json(emitter)) { emitter_json_key(emitter, json_key); emitter_json_value(emitter, value_type, value); @@ -461,8 +480,8 @@ emitter_kv(emitter_t *emitter, const char *json_key, const char *table_key, } static inline void -emitter_dict_begin(emitter_t *emitter, const char *json_key, - const char *table_header) { +emitter_dict_begin( + emitter_t *emitter, const char *json_key, const char *table_header) { if (emitter_outputs_json(emitter)) { emitter_json_key(emitter, json_key); emitter_json_object_begin(emitter); @@ -502,8 +521,9 @@ emitter_end(emitter_t *emitter) { if (emitter_outputs_json(emitter)) { assert(emitter->nesting_depth == 1); emitter_nest_dec(emitter); - emitter_printf(emitter, "%s", emitter->output == - emitter_output_json_compact ? "}" : "\n}\n"); + emitter_printf(emitter, "%s", + emitter->output == emitter_output_json_compact ? "}" + : "\n}\n"); } } diff --git a/dep/jemalloc/include/jemalloc/internal/eset.h b/dep/jemalloc/include/jemalloc/internal/eset.h index 4f689b47d..9b7c4a89e 100644 --- a/dep/jemalloc/include/jemalloc/internal/eset.h +++ b/dep/jemalloc/include/jemalloc/internal/eset.h @@ -1,9 +1,10 @@ #ifndef JEMALLOC_INTERNAL_ESET_H #define JEMALLOC_INTERNAL_ESET_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/fb.h" #include "jemalloc/internal/edata.h" +#include "jemalloc/internal/fb.h" #include "jemalloc/internal/mutex.h" /* diff --git a/dep/jemalloc/include/jemalloc/internal/exp_grow.h b/dep/jemalloc/include/jemalloc/internal/exp_grow.h index 8566b8a4c..8206ba857 100644 --- a/dep/jemalloc/include/jemalloc/internal/exp_grow.h +++ b/dep/jemalloc/include/jemalloc/internal/exp_grow.h @@ -1,6 +1,8 @@ #ifndef JEMALLOC_INTERNAL_EXP_GROW_H #define JEMALLOC_INTERNAL_EXP_GROW_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/sz.h" typedef struct exp_grow_s exp_grow_t; struct exp_grow_s { /* @@ -25,8 +27,7 @@ exp_grow_size_prepare(exp_grow_t *exp_grow, size_t alloc_size_min, *r_alloc_size = sz_pind2sz(exp_grow->next + *r_skip); while (*r_alloc_size < alloc_size_min) { (*r_skip)++; - if (exp_grow->next + *r_skip >= - sz_psz2ind(SC_LARGE_MAXCLASS)) { + if (exp_grow->next + *r_skip >= sz_psz2ind(SC_LARGE_MAXCLASS)) { /* Outside legal range. */ return true; } @@ -42,7 +43,6 @@ exp_grow_size_commit(exp_grow_t *exp_grow, pszind_t skip) { } else { exp_grow->next = exp_grow->limit; } - } void exp_grow_init(exp_grow_t *exp_grow); diff --git a/dep/jemalloc/include/jemalloc/internal/extent.h b/dep/jemalloc/include/jemalloc/internal/extent.h index 1d51d4109..a9f81cb78 100644 --- a/dep/jemalloc/include/jemalloc/internal/extent.h +++ b/dep/jemalloc/include/jemalloc/internal/extent.h @@ -1,8 +1,10 @@ #ifndef JEMALLOC_INTERNAL_EXTENT_H #define JEMALLOC_INTERNAL_EXTENT_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/ecache.h" #include "jemalloc/internal/ehooks.h" +#include "jemalloc/internal/pac.h" #include "jemalloc/internal/ph.h" #include "jemalloc/internal/rtree.h" @@ -19,50 +21,58 @@ #define LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT 6 extern size_t opt_lg_extent_max_active_fit; +#define PROCESS_MADVISE_MAX_BATCH_DEFAULT 0 +extern size_t opt_process_madvise_max_batch; + +#ifdef JEMALLOC_HAVE_PROCESS_MADVISE +/* The iovec is on stack. Limit the max batch to avoid stack overflow. */ +# define PROCESS_MADVISE_MAX_BATCH_LIMIT \ + (VARIABLE_ARRAY_SIZE_MAX / sizeof(struct iovec)) +#else +# define PROCESS_MADVISE_MAX_BATCH_LIMIT 0 +#endif + edata_t *ecache_alloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, edata_t *expand_edata, size_t size, size_t alignment, bool zero, bool guarded); edata_t *ecache_alloc_grow(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, edata_t *expand_edata, size_t size, size_t alignment, bool zero, bool guarded); -void ecache_dalloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, - ecache_t *ecache, edata_t *edata); +void ecache_dalloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, + edata_t *edata); edata_t *ecache_evict(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, size_t npages_min); void extent_gdump_add(tsdn_t *tsdn, const edata_t *edata); void extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, edata_t *edata); -void extent_dalloc_gap(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, - edata_t *edata); +void extent_dalloc_gap( + tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata); edata_t *extent_alloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, void *new_addr, size_t size, size_t alignment, bool zero, bool *commit, bool growing_retained); -void extent_dalloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, - edata_t *edata); -void extent_destroy_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, - edata_t *edata); -bool extent_commit_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, - size_t offset, size_t length); -bool extent_decommit_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, - size_t offset, size_t length); +void extent_dalloc_wrapper( + tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata); +void extent_dalloc_wrapper_purged( + tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata); +void extent_destroy_wrapper( + tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata); bool extent_purge_lazy_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, size_t offset, size_t length); bool extent_purge_forced_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, size_t offset, size_t length); -edata_t *extent_split_wrapper(tsdn_t *tsdn, pac_t *pac, - ehooks_t *ehooks, edata_t *edata, size_t size_a, size_t size_b, - bool holding_core_locks); -bool extent_merge_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, - edata_t *a, edata_t *b); -bool extent_commit_zero(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, - bool commit, bool zero, bool growing_retained); +edata_t *extent_split_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, + edata_t *edata, size_t size_a, size_t size_b, bool holding_core_locks); +bool extent_merge_wrapper( + tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *a, edata_t *b); +bool extent_commit_zero(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, + bool commit, bool zero, bool growing_retained); size_t extent_sn_next(pac_t *pac); -bool extent_boot(void); +bool extent_boot(void); JEMALLOC_ALWAYS_INLINE bool -extent_neighbor_head_state_mergeable(bool edata_is_head, - bool neighbor_is_head, bool forward) { +extent_neighbor_head_state_mergeable( + bool edata_is_head, bool neighbor_is_head, bool forward) { /* * Head states checking: disallow merging if the higher addr extent is a * head extent. This helps preserve first-fit, and more importantly @@ -90,8 +100,8 @@ extent_can_acquire_neighbor(edata_t *edata, rtree_contents_t contents, } /* It's not safe to access *neighbor yet; must verify states first. */ bool neighbor_is_head = contents.metadata.is_head; - if (!extent_neighbor_head_state_mergeable(edata_is_head_get(edata), - neighbor_is_head, forward)) { + if (!extent_neighbor_head_state_mergeable( + edata_is_head_get(edata), neighbor_is_head, forward)) { return false; } extent_state_t neighbor_state = contents.metadata.state; @@ -100,8 +110,9 @@ extent_can_acquire_neighbor(edata_t *edata, rtree_contents_t contents, return false; } /* From this point, it's safe to access *neighbor. */ - if (!expanding && (edata_committed_get(edata) != - edata_committed_get(neighbor))) { + if (!expanding + && (edata_committed_get(edata) + != edata_committed_get(neighbor))) { /* * Some platforms (e.g. Windows) require an explicit * commit step (and writing to uncommitted memory is not @@ -121,11 +132,11 @@ extent_can_acquire_neighbor(edata_t *edata, rtree_contents_t contents, return false; } if (opt_retain) { - assert(edata_arena_ind_get(edata) == - edata_arena_ind_get(neighbor)); + assert(edata_arena_ind_get(edata) + == edata_arena_ind_get(neighbor)); } else { - if (edata_arena_ind_get(edata) != - edata_arena_ind_get(neighbor)) { + if (edata_arena_ind_get(edata) + != edata_arena_ind_get(neighbor)) { return false; } } diff --git a/dep/jemalloc/include/jemalloc/internal/extent_dss.h b/dep/jemalloc/include/jemalloc/internal/extent_dss.h index e8f02ce2a..4bb3f51d7 100644 --- a/dep/jemalloc/include/jemalloc/internal/extent_dss.h +++ b/dep/jemalloc/include/jemalloc/internal/extent_dss.h @@ -1,26 +1,30 @@ #ifndef JEMALLOC_INTERNAL_EXTENT_DSS_H #define JEMALLOC_INTERNAL_EXTENT_DSS_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/arena_types.h" +#include "jemalloc/internal/tsd_types.h" + typedef enum { - dss_prec_disabled = 0, - dss_prec_primary = 1, + dss_prec_disabled = 0, + dss_prec_primary = 1, dss_prec_secondary = 2, - dss_prec_limit = 3 + dss_prec_limit = 3 } dss_prec_t; #define DSS_PREC_DEFAULT dss_prec_secondary #define DSS_DEFAULT "secondary" -extern const char *dss_prec_names[]; +extern const char *const dss_prec_names[]; extern const char *opt_dss; dss_prec_t extent_dss_prec_get(void); -bool extent_dss_prec_set(dss_prec_t dss_prec); -void *extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, - size_t size, size_t alignment, bool *zero, bool *commit); -bool extent_in_dss(void *addr); -bool extent_dss_mergeable(void *addr_a, void *addr_b); -void extent_dss_boot(void); +bool extent_dss_prec_set(dss_prec_t dss_prec); +void *extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, + size_t size, size_t alignment, bool *zero, bool *commit); +bool extent_in_dss(void *addr); +bool extent_dss_mergeable(void *addr_a, void *addr_b); +void extent_dss_boot(void); #endif /* JEMALLOC_INTERNAL_EXTENT_DSS_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/extent_mmap.h b/dep/jemalloc/include/jemalloc/internal/extent_mmap.h index 55f17ee48..aa4698966 100644 --- a/dep/jemalloc/include/jemalloc/internal/extent_mmap.h +++ b/dep/jemalloc/include/jemalloc/internal/extent_mmap.h @@ -1,10 +1,12 @@ #ifndef JEMALLOC_INTERNAL_EXTENT_MMAP_EXTERNS_H #define JEMALLOC_INTERNAL_EXTENT_MMAP_EXTERNS_H +#include "jemalloc/internal/jemalloc_preamble.h" + extern bool opt_retain; -void *extent_alloc_mmap(void *new_addr, size_t size, size_t alignment, - bool *zero, bool *commit); +void *extent_alloc_mmap( + void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit); bool extent_dalloc_mmap(void *addr, size_t size); #endif /* JEMALLOC_INTERNAL_EXTENT_MMAP_EXTERNS_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/fb.h b/dep/jemalloc/include/jemalloc/internal/fb.h index 90c4091ff..bf76f3629 100644 --- a/dep/jemalloc/include/jemalloc/internal/fb.h +++ b/dep/jemalloc/include/jemalloc/internal/fb.h @@ -1,6 +1,10 @@ #ifndef JEMALLOC_INTERNAL_FB_H #define JEMALLOC_INTERNAL_FB_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/assert.h" +#include "jemalloc/internal/bit_util.h" + /* * The flat bitmap module. This has a larger API relative to the bitmap module * (supporting things like backwards searches, and searching for both set and @@ -11,8 +15,8 @@ typedef unsigned long fb_group_t; #define FB_GROUP_BITS (ZU(1) << (LG_SIZEOF_LONG + 3)) -#define FB_NGROUPS(nbits) ((nbits) / FB_GROUP_BITS \ - + ((nbits) % FB_GROUP_BITS == 0 ? 0 : 1)) +#define FB_NGROUPS(nbits) \ + ((nbits) / FB_GROUP_BITS + ((nbits) % FB_GROUP_BITS == 0 ? 0 : 1)) static inline void fb_init(fb_group_t *fb, size_t nbits) { @@ -71,7 +75,6 @@ fb_unset(fb_group_t *fb, size_t nbits, size_t bit) { fb[group_ind] &= ~((fb_group_t)1 << bit_ind); } - /* * Some implementation details. This visitation function lets us apply a group * visitor to each group in the bitmap (potentially modifying it). The mask @@ -90,7 +93,8 @@ fb_visit_impl(fb_group_t *fb, size_t nbits, fb_group_visitor_t visit, void *ctx, * to from bit 0. */ size_t first_group_cnt = (start_bit_ind + cnt > FB_GROUP_BITS - ? FB_GROUP_BITS - start_bit_ind : cnt); + ? FB_GROUP_BITS - start_bit_ind + : cnt); /* * We can basically split affected words into: * - The first group, where we touch only the high bits @@ -100,8 +104,8 @@ fb_visit_impl(fb_group_t *fb, size_t nbits, fb_group_visitor_t visit, void *ctx, * this can lead to bad codegen for those middle words. */ /* First group */ - fb_group_t mask = ((~(fb_group_t)0) - >> (FB_GROUP_BITS - first_group_cnt)) + fb_group_t mask = + ((~(fb_group_t)0) >> (FB_GROUP_BITS - first_group_cnt)) << start_bit_ind; visit(ctx, &fb[group_ind], mask); @@ -172,12 +176,12 @@ fb_ucount(fb_group_t *fb, size_t nbits, size_t start, size_t cnt) { * Returns the number of bits in the bitmap if no such bit exists. */ JEMALLOC_ALWAYS_INLINE ssize_t -fb_find_impl(fb_group_t *fb, size_t nbits, size_t start, bool val, - bool forward) { +fb_find_impl( + fb_group_t *fb, size_t nbits, size_t start, bool val, bool forward) { assert(start < nbits); - size_t ngroups = FB_NGROUPS(nbits); + size_t ngroups = FB_NGROUPS(nbits); ssize_t group_ind = start / FB_GROUP_BITS; - size_t bit_ind = start % FB_GROUP_BITS; + size_t bit_ind = start % FB_GROUP_BITS; fb_group_t maybe_invert = (val ? 0 : (fb_group_t)-1); @@ -261,8 +265,8 @@ fb_iter_range_impl(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin, return false; } /* Half open range; the set bits are [begin, end). */ - ssize_t next_range_end = fb_find_impl(fb, nbits, next_range_begin, !val, - forward); + ssize_t next_range_end = fb_find_impl( + fb, nbits, next_range_begin, !val, forward); if (forward) { *r_begin = next_range_begin; *r_len = next_range_end - next_range_begin; @@ -320,8 +324,9 @@ fb_range_longest_impl(fb_group_t *fb, size_t nbits, bool val) { size_t begin = 0; size_t longest_len = 0; size_t len = 0; - while (begin < nbits && fb_iter_range_impl(fb, nbits, begin, &begin, - &len, val, /* forward */ true)) { + while (begin < nbits + && fb_iter_range_impl( + fb, nbits, begin, &begin, &len, val, /* forward */ true)) { if (len > longest_len) { longest_len = len; } diff --git a/dep/jemalloc/include/jemalloc/internal/fxp.h b/dep/jemalloc/include/jemalloc/internal/fxp.h index 415a98289..8ca4f3c67 100644 --- a/dep/jemalloc/include/jemalloc/internal/fxp.h +++ b/dep/jemalloc/include/jemalloc/internal/fxp.h @@ -1,6 +1,9 @@ #ifndef JEMALLOC_INTERNAL_FXP_H #define JEMALLOC_INTERNAL_FXP_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/assert.h" + /* * A simple fixed-point math implementation, supporting only unsigned values * (with overflow being an error). @@ -86,7 +89,7 @@ fxp_round_down(fxp_t a) { static inline uint32_t fxp_round_nearest(fxp_t a) { - uint32_t fractional_part = (a & ((1U << 16) - 1)); + uint32_t fractional_part = (a & ((1U << 16) - 1)); uint32_t increment = (uint32_t)(fractional_part >= (1U << 15)); return (a >> 16) + increment; } diff --git a/dep/jemalloc/include/jemalloc/internal/hash.h b/dep/jemalloc/include/jemalloc/internal/hash.h index 7f945679e..73e2214eb 100644 --- a/dep/jemalloc/include/jemalloc/internal/hash.h +++ b/dep/jemalloc/include/jemalloc/internal/hash.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_HASH_H #define JEMALLOC_INTERNAL_HASH_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/assert.h" /* @@ -24,7 +25,7 @@ hash_rotl_64(uint64_t x, int8_t r) { static inline uint32_t hash_get_block_32(const uint32_t *p, int i) { /* Handle unaligned read. */ - if (unlikely((uintptr_t)p & (sizeof(uint32_t)-1)) != 0) { + if (unlikely((uintptr_t)p & (sizeof(uint32_t) - 1)) != 0) { uint32_t ret; memcpy(&ret, (uint8_t *)(p + i), sizeof(uint32_t)); @@ -37,7 +38,7 @@ hash_get_block_32(const uint32_t *p, int i) { static inline uint64_t hash_get_block_64(const uint64_t *p, int i) { /* Handle unaligned read. */ - if (unlikely((uintptr_t)p & (sizeof(uint64_t)-1)) != 0) { + if (unlikely((uintptr_t)p & (sizeof(uint64_t) - 1)) != 0) { uint64_t ret; memcpy(&ret, (uint8_t *)(p + i), sizeof(uint64_t)); @@ -71,8 +72,8 @@ hash_fmix_64(uint64_t k) { static inline uint32_t hash_x86_32(const void *key, int len, uint32_t seed) { - const uint8_t *data = (const uint8_t *) key; - const int nblocks = len / 4; + const uint8_t *data = (const uint8_t *)key; + const int nblocks = len / 4; uint32_t h1 = seed; @@ -81,8 +82,8 @@ hash_x86_32(const void *key, int len, uint32_t seed) { /* body */ { - const uint32_t *blocks = (const uint32_t *) (data + nblocks*4); - int i; + const uint32_t *blocks = (const uint32_t *)(data + nblocks * 4); + int i; for (i = -nblocks; i; i++) { uint32_t k1 = hash_get_block_32(blocks, i); @@ -93,21 +94,29 @@ hash_x86_32(const void *key, int len, uint32_t seed) { h1 ^= k1; h1 = hash_rotl_32(h1, 13); - h1 = h1*5 + 0xe6546b64; + h1 = h1 * 5 + 0xe6546b64; } } /* tail */ { - const uint8_t *tail = (const uint8_t *) (data + nblocks*4); + const uint8_t *tail = (const uint8_t *)(data + nblocks * 4); uint32_t k1 = 0; switch (len & 3) { - case 3: k1 ^= tail[2] << 16; JEMALLOC_FALLTHROUGH; - case 2: k1 ^= tail[1] << 8; JEMALLOC_FALLTHROUGH; - case 1: k1 ^= tail[0]; k1 *= c1; k1 = hash_rotl_32(k1, 15); - k1 *= c2; h1 ^= k1; + case 3: + k1 ^= tail[2] << 16; + JEMALLOC_FALLTHROUGH; + case 2: + k1 ^= tail[1] << 8; + JEMALLOC_FALLTHROUGH; + case 1: + k1 ^= tail[0]; + k1 *= c1; + k1 = hash_rotl_32(k1, 15); + k1 *= c2; + h1 ^= k1; } } @@ -120,10 +129,9 @@ hash_x86_32(const void *key, int len, uint32_t seed) { } static inline void -hash_x86_128(const void *key, const int len, uint32_t seed, - uint64_t r_out[2]) { - const uint8_t * data = (const uint8_t *) key; - const int nblocks = len / 16; +hash_x86_128(const void *key, const int len, uint32_t seed, uint64_t r_out[2]) { + const uint8_t *data = (const uint8_t *)key; + const int nblocks = len / 16; uint32_t h1 = seed; uint32_t h2 = seed; @@ -137,95 +145,161 @@ hash_x86_128(const void *key, const int len, uint32_t seed, /* body */ { - const uint32_t *blocks = (const uint32_t *) (data + nblocks*16); - int i; + const uint32_t *blocks = (const uint32_t *)(data + + nblocks * 16); + int i; for (i = -nblocks; i; i++) { - uint32_t k1 = hash_get_block_32(blocks, i*4 + 0); - uint32_t k2 = hash_get_block_32(blocks, i*4 + 1); - uint32_t k3 = hash_get_block_32(blocks, i*4 + 2); - uint32_t k4 = hash_get_block_32(blocks, i*4 + 3); + uint32_t k1 = hash_get_block_32(blocks, i * 4 + 0); + uint32_t k2 = hash_get_block_32(blocks, i * 4 + 1); + uint32_t k3 = hash_get_block_32(blocks, i * 4 + 2); + uint32_t k4 = hash_get_block_32(blocks, i * 4 + 3); - k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1; + k1 *= c1; + k1 = hash_rotl_32(k1, 15); + k1 *= c2; + h1 ^= k1; - h1 = hash_rotl_32(h1, 19); h1 += h2; - h1 = h1*5 + 0x561ccd1b; + h1 = hash_rotl_32(h1, 19); + h1 += h2; + h1 = h1 * 5 + 0x561ccd1b; - k2 *= c2; k2 = hash_rotl_32(k2, 16); k2 *= c3; h2 ^= k2; + k2 *= c2; + k2 = hash_rotl_32(k2, 16); + k2 *= c3; + h2 ^= k2; - h2 = hash_rotl_32(h2, 17); h2 += h3; - h2 = h2*5 + 0x0bcaa747; + h2 = hash_rotl_32(h2, 17); + h2 += h3; + h2 = h2 * 5 + 0x0bcaa747; - k3 *= c3; k3 = hash_rotl_32(k3, 17); k3 *= c4; h3 ^= k3; + k3 *= c3; + k3 = hash_rotl_32(k3, 17); + k3 *= c4; + h3 ^= k3; - h3 = hash_rotl_32(h3, 15); h3 += h4; - h3 = h3*5 + 0x96cd1c35; + h3 = hash_rotl_32(h3, 15); + h3 += h4; + h3 = h3 * 5 + 0x96cd1c35; - k4 *= c4; k4 = hash_rotl_32(k4, 18); k4 *= c1; h4 ^= k4; + k4 *= c4; + k4 = hash_rotl_32(k4, 18); + k4 *= c1; + h4 ^= k4; - h4 = hash_rotl_32(h4, 13); h4 += h1; - h4 = h4*5 + 0x32ac3b17; + h4 = hash_rotl_32(h4, 13); + h4 += h1; + h4 = h4 * 5 + 0x32ac3b17; } } /* tail */ { - const uint8_t *tail = (const uint8_t *) (data + nblocks*16); - uint32_t k1 = 0; - uint32_t k2 = 0; - uint32_t k3 = 0; - uint32_t k4 = 0; + const uint8_t *tail = (const uint8_t *)(data + nblocks * 16); + uint32_t k1 = 0; + uint32_t k2 = 0; + uint32_t k3 = 0; + uint32_t k4 = 0; switch (len & 15) { - case 15: k4 ^= tail[14] << 16; JEMALLOC_FALLTHROUGH; - case 14: k4 ^= tail[13] << 8; JEMALLOC_FALLTHROUGH; - case 13: k4 ^= tail[12] << 0; - k4 *= c4; k4 = hash_rotl_32(k4, 18); k4 *= c1; h4 ^= k4; + case 15: + k4 ^= tail[14] << 16; JEMALLOC_FALLTHROUGH; - case 12: k3 ^= (uint32_t) tail[11] << 24; JEMALLOC_FALLTHROUGH; - case 11: k3 ^= tail[10] << 16; JEMALLOC_FALLTHROUGH; - case 10: k3 ^= tail[ 9] << 8; JEMALLOC_FALLTHROUGH; - case 9: k3 ^= tail[ 8] << 0; - k3 *= c3; k3 = hash_rotl_32(k3, 17); k3 *= c4; h3 ^= k3; + case 14: + k4 ^= tail[13] << 8; JEMALLOC_FALLTHROUGH; - case 8: k2 ^= (uint32_t) tail[ 7] << 24; JEMALLOC_FALLTHROUGH; - case 7: k2 ^= tail[ 6] << 16; JEMALLOC_FALLTHROUGH; - case 6: k2 ^= tail[ 5] << 8; JEMALLOC_FALLTHROUGH; - case 5: k2 ^= tail[ 4] << 0; - k2 *= c2; k2 = hash_rotl_32(k2, 16); k2 *= c3; h2 ^= k2; + case 13: + k4 ^= tail[12] << 0; + k4 *= c4; + k4 = hash_rotl_32(k4, 18); + k4 *= c1; + h4 ^= k4; JEMALLOC_FALLTHROUGH; - case 4: k1 ^= (uint32_t) tail[ 3] << 24; JEMALLOC_FALLTHROUGH; - case 3: k1 ^= tail[ 2] << 16; JEMALLOC_FALLTHROUGH; - case 2: k1 ^= tail[ 1] << 8; JEMALLOC_FALLTHROUGH; - case 1: k1 ^= tail[ 0] << 0; - k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1; + case 12: + k3 ^= (uint32_t)tail[11] << 24; + JEMALLOC_FALLTHROUGH; + case 11: + k3 ^= tail[10] << 16; + JEMALLOC_FALLTHROUGH; + case 10: + k3 ^= tail[9] << 8; + JEMALLOC_FALLTHROUGH; + case 9: + k3 ^= tail[8] << 0; + k3 *= c3; + k3 = hash_rotl_32(k3, 17); + k3 *= c4; + h3 ^= k3; + JEMALLOC_FALLTHROUGH; + case 8: + k2 ^= (uint32_t)tail[7] << 24; + JEMALLOC_FALLTHROUGH; + case 7: + k2 ^= tail[6] << 16; + JEMALLOC_FALLTHROUGH; + case 6: + k2 ^= tail[5] << 8; + JEMALLOC_FALLTHROUGH; + case 5: + k2 ^= tail[4] << 0; + k2 *= c2; + k2 = hash_rotl_32(k2, 16); + k2 *= c3; + h2 ^= k2; + JEMALLOC_FALLTHROUGH; + case 4: + k1 ^= (uint32_t)tail[3] << 24; + JEMALLOC_FALLTHROUGH; + case 3: + k1 ^= tail[2] << 16; + JEMALLOC_FALLTHROUGH; + case 2: + k1 ^= tail[1] << 8; + JEMALLOC_FALLTHROUGH; + case 1: + k1 ^= tail[0] << 0; + k1 *= c1; + k1 = hash_rotl_32(k1, 15); + k1 *= c2; + h1 ^= k1; break; } } /* finalization */ - h1 ^= len; h2 ^= len; h3 ^= len; h4 ^= len; + h1 ^= len; + h2 ^= len; + h3 ^= len; + h4 ^= len; - h1 += h2; h1 += h3; h1 += h4; - h2 += h1; h3 += h1; h4 += h1; + h1 += h2; + h1 += h3; + h1 += h4; + h2 += h1; + h3 += h1; + h4 += h1; h1 = hash_fmix_32(h1); h2 = hash_fmix_32(h2); h3 = hash_fmix_32(h3); h4 = hash_fmix_32(h4); - h1 += h2; h1 += h3; h1 += h4; - h2 += h1; h3 += h1; h4 += h1; + h1 += h2; + h1 += h3; + h1 += h4; + h2 += h1; + h3 += h1; + h4 += h1; - r_out[0] = (((uint64_t) h2) << 32) | h1; - r_out[1] = (((uint64_t) h4) << 32) | h3; + r_out[0] = (((uint64_t)h2) << 32) | h1; + r_out[1] = (((uint64_t)h4) << 32) | h3; } static inline void -hash_x64_128(const void *key, const int len, const uint32_t seed, - uint64_t r_out[2]) { - const uint8_t *data = (const uint8_t *) key; - const int nblocks = len / 16; +hash_x64_128( + const void *key, const int len, const uint32_t seed, uint64_t r_out[2]) { + const uint8_t *data = (const uint8_t *)key; + const int nblocks = len / 16; uint64_t h1 = seed; uint64_t h2 = seed; @@ -235,56 +309,99 @@ hash_x64_128(const void *key, const int len, const uint32_t seed, /* body */ { - const uint64_t *blocks = (const uint64_t *) (data); - int i; + const uint64_t *blocks = (const uint64_t *)(data); + int i; for (i = 0; i < nblocks; i++) { - uint64_t k1 = hash_get_block_64(blocks, i*2 + 0); - uint64_t k2 = hash_get_block_64(blocks, i*2 + 1); + uint64_t k1 = hash_get_block_64(blocks, i * 2 + 0); + uint64_t k2 = hash_get_block_64(blocks, i * 2 + 1); - k1 *= c1; k1 = hash_rotl_64(k1, 31); k1 *= c2; h1 ^= k1; + k1 *= c1; + k1 = hash_rotl_64(k1, 31); + k1 *= c2; + h1 ^= k1; - h1 = hash_rotl_64(h1, 27); h1 += h2; - h1 = h1*5 + 0x52dce729; + h1 = hash_rotl_64(h1, 27); + h1 += h2; + h1 = h1 * 5 + 0x52dce729; - k2 *= c2; k2 = hash_rotl_64(k2, 33); k2 *= c1; h2 ^= k2; + k2 *= c2; + k2 = hash_rotl_64(k2, 33); + k2 *= c1; + h2 ^= k2; - h2 = hash_rotl_64(h2, 31); h2 += h1; - h2 = h2*5 + 0x38495ab5; + h2 = hash_rotl_64(h2, 31); + h2 += h1; + h2 = h2 * 5 + 0x38495ab5; } } /* tail */ { - const uint8_t *tail = (const uint8_t*)(data + nblocks*16); - uint64_t k1 = 0; - uint64_t k2 = 0; + const uint8_t *tail = (const uint8_t *)(data + nblocks * 16); + uint64_t k1 = 0; + uint64_t k2 = 0; switch (len & 15) { - case 15: k2 ^= ((uint64_t)(tail[14])) << 48; JEMALLOC_FALLTHROUGH; - case 14: k2 ^= ((uint64_t)(tail[13])) << 40; JEMALLOC_FALLTHROUGH; - case 13: k2 ^= ((uint64_t)(tail[12])) << 32; JEMALLOC_FALLTHROUGH; - case 12: k2 ^= ((uint64_t)(tail[11])) << 24; JEMALLOC_FALLTHROUGH; - case 11: k2 ^= ((uint64_t)(tail[10])) << 16; JEMALLOC_FALLTHROUGH; - case 10: k2 ^= ((uint64_t)(tail[ 9])) << 8; JEMALLOC_FALLTHROUGH; - case 9: k2 ^= ((uint64_t)(tail[ 8])) << 0; - k2 *= c2; k2 = hash_rotl_64(k2, 33); k2 *= c1; h2 ^= k2; + case 15: + k2 ^= ((uint64_t)(tail[14])) << 48; JEMALLOC_FALLTHROUGH; - case 8: k1 ^= ((uint64_t)(tail[ 7])) << 56; JEMALLOC_FALLTHROUGH; - case 7: k1 ^= ((uint64_t)(tail[ 6])) << 48; JEMALLOC_FALLTHROUGH; - case 6: k1 ^= ((uint64_t)(tail[ 5])) << 40; JEMALLOC_FALLTHROUGH; - case 5: k1 ^= ((uint64_t)(tail[ 4])) << 32; JEMALLOC_FALLTHROUGH; - case 4: k1 ^= ((uint64_t)(tail[ 3])) << 24; JEMALLOC_FALLTHROUGH; - case 3: k1 ^= ((uint64_t)(tail[ 2])) << 16; JEMALLOC_FALLTHROUGH; - case 2: k1 ^= ((uint64_t)(tail[ 1])) << 8; JEMALLOC_FALLTHROUGH; - case 1: k1 ^= ((uint64_t)(tail[ 0])) << 0; - k1 *= c1; k1 = hash_rotl_64(k1, 31); k1 *= c2; h1 ^= k1; + case 14: + k2 ^= ((uint64_t)(tail[13])) << 40; + JEMALLOC_FALLTHROUGH; + case 13: + k2 ^= ((uint64_t)(tail[12])) << 32; + JEMALLOC_FALLTHROUGH; + case 12: + k2 ^= ((uint64_t)(tail[11])) << 24; + JEMALLOC_FALLTHROUGH; + case 11: + k2 ^= ((uint64_t)(tail[10])) << 16; + JEMALLOC_FALLTHROUGH; + case 10: + k2 ^= ((uint64_t)(tail[9])) << 8; + JEMALLOC_FALLTHROUGH; + case 9: + k2 ^= ((uint64_t)(tail[8])) << 0; + k2 *= c2; + k2 = hash_rotl_64(k2, 33); + k2 *= c1; + h2 ^= k2; + JEMALLOC_FALLTHROUGH; + case 8: + k1 ^= ((uint64_t)(tail[7])) << 56; + JEMALLOC_FALLTHROUGH; + case 7: + k1 ^= ((uint64_t)(tail[6])) << 48; + JEMALLOC_FALLTHROUGH; + case 6: + k1 ^= ((uint64_t)(tail[5])) << 40; + JEMALLOC_FALLTHROUGH; + case 5: + k1 ^= ((uint64_t)(tail[4])) << 32; + JEMALLOC_FALLTHROUGH; + case 4: + k1 ^= ((uint64_t)(tail[3])) << 24; + JEMALLOC_FALLTHROUGH; + case 3: + k1 ^= ((uint64_t)(tail[2])) << 16; + JEMALLOC_FALLTHROUGH; + case 2: + k1 ^= ((uint64_t)(tail[1])) << 8; + JEMALLOC_FALLTHROUGH; + case 1: + k1 ^= ((uint64_t)(tail[0])) << 0; + k1 *= c1; + k1 = hash_rotl_64(k1, 31); + k1 *= c2; + h1 ^= k1; break; } } /* finalization */ - h1 ^= len; h2 ^= len; + h1 ^= len; + h2 ^= len; h1 += h2; h2 += h1; diff --git a/dep/jemalloc/include/jemalloc/internal/hook.h b/dep/jemalloc/include/jemalloc/internal/hook.h index ee246b1e0..bbbcb3203 100644 --- a/dep/jemalloc/include/jemalloc/internal/hook.h +++ b/dep/jemalloc/include/jemalloc/internal/hook.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_HOOK_H #define JEMALLOC_INTERNAL_HOOK_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/tsd.h" /* @@ -55,6 +56,7 @@ enum hook_alloc_e { hook_alloc_calloc, hook_alloc_memalign, hook_alloc_valloc, + hook_alloc_pvalloc, hook_alloc_mallocx, /* The reallocating functions have both alloc and dalloc variants */ @@ -81,7 +83,6 @@ enum hook_dalloc_e { }; typedef enum hook_dalloc_e hook_dalloc_t; - enum hook_expand_e { hook_expand_realloc, hook_expand_rallocx, @@ -89,23 +90,22 @@ enum hook_expand_e { }; typedef enum hook_expand_e hook_expand_t; -typedef void (*hook_alloc)( - void *extra, hook_alloc_t type, void *result, uintptr_t result_raw, - uintptr_t args_raw[3]); +typedef void (*hook_alloc)(void *extra, hook_alloc_t type, void *result, + uintptr_t result_raw, uintptr_t args_raw[3]); typedef void (*hook_dalloc)( void *extra, hook_dalloc_t type, void *address, uintptr_t args_raw[3]); -typedef void (*hook_expand)( - void *extra, hook_expand_t type, void *address, size_t old_usize, - size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]); +typedef void (*hook_expand)(void *extra, hook_expand_t type, void *address, + size_t old_usize, size_t new_usize, uintptr_t result_raw, + uintptr_t args_raw[4]); typedef struct hooks_s hooks_t; struct hooks_s { - hook_alloc alloc_hook; + hook_alloc alloc_hook; hook_dalloc dalloc_hook; hook_expand expand_hook; - void *extra; + void *extra; }; /* @@ -143,9 +143,9 @@ struct hook_ralloc_args_s { * Returns an opaque handle to be used when removing the hook. NULL means that * we couldn't install the hook. */ -bool hook_boot(); +bool hook_boot(void); -void *hook_install(tsdn_t *tsdn, hooks_t *hooks); +void *hook_install(tsdn_t *tsdn, hooks_t *to_install); /* Uninstalls the hook with the handle previously returned from hook_install. */ void hook_remove(tsdn_t *tsdn, void *opaque); @@ -154,8 +154,8 @@ void hook_remove(tsdn_t *tsdn, void *opaque); void hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw, uintptr_t args_raw[3]); -void hook_invoke_dalloc(hook_dalloc_t type, void *address, - uintptr_t args_raw[3]); +void hook_invoke_dalloc( + hook_dalloc_t type, void *address, uintptr_t args_raw[3]); void hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize, size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]); diff --git a/dep/jemalloc/include/jemalloc/internal/hpa.h b/dep/jemalloc/include/jemalloc/internal/hpa.h index f3562853e..6783eb7fe 100644 --- a/dep/jemalloc/include/jemalloc/internal/hpa.h +++ b/dep/jemalloc/include/jemalloc/internal/hpa.h @@ -1,42 +1,18 @@ #ifndef JEMALLOC_INTERNAL_HPA_H #define JEMALLOC_INTERNAL_HPA_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/base.h" +#include "jemalloc/internal/edata_cache.h" +#include "jemalloc/internal/emap.h" #include "jemalloc/internal/exp_grow.h" +#include "jemalloc/internal/hpa_central.h" #include "jemalloc/internal/hpa_hooks.h" #include "jemalloc/internal/hpa_opts.h" +#include "jemalloc/internal/mutex.h" #include "jemalloc/internal/pai.h" #include "jemalloc/internal/psset.h" - -typedef struct hpa_central_s hpa_central_t; -struct hpa_central_s { - /* - * The mutex guarding most of the operations on the central data - * structure. - */ - malloc_mutex_t mtx; - /* - * Guards expansion of eden. We separate this from the regular mutex so - * that cheaper operations can still continue while we're doing the OS - * call. - */ - malloc_mutex_t grow_mtx; - /* - * Either NULL (if empty), or some integer multiple of a - * hugepage-aligned number of hugepages. We carve them off one at a - * time to satisfy new pageslab requests. - * - * Guarded by grow_mtx. - */ - void *eden; - size_t eden_len; - /* Source for metadata. */ - base_t *base; - /* Number of grow operations done on this hpa_central_t. */ - uint64_t age_counter; - - /* The HPA hooks. */ - hpa_hooks_t hooks; -}; +#include "jemalloc/internal/sec.h" typedef struct hpa_shard_nonderived_stats_s hpa_shard_nonderived_stats_t; struct hpa_shard_nonderived_stats_s { @@ -61,6 +37,14 @@ struct hpa_shard_nonderived_stats_s { * Guarded by mtx. */ uint64_t nhugifies; + + /* + * The number of times we've tried to hugify a pageslab, but failed. + * + * Guarded by mtx. + */ + uint64_t nhugify_failures; + /* * The number of times we've dehugified a pageslab. * @@ -72,8 +56,9 @@ struct hpa_shard_nonderived_stats_s { /* Completely derived; only used by CTL. */ typedef struct hpa_shard_stats_s hpa_shard_stats_t; struct hpa_shard_stats_s { - psset_stats_t psset_stats; + psset_stats_t psset_stats; hpa_shard_nonderived_stats_t nonderived_stats; + sec_stats_t secstats; }; typedef struct hpa_shard_s hpa_shard_t; @@ -86,14 +71,17 @@ struct hpa_shard_s { /* The central allocator we get our hugepages from. */ hpa_central_t *central; + /* Protects most of this shard's state. */ malloc_mutex_t mtx; + /* * Guards the shard's access to the central allocator (preventing * multiple threads operating on this shard from accessing the central * allocator). */ malloc_mutex_t grow_mtx; + /* The base metadata allocator. */ base_t *base; @@ -104,6 +92,9 @@ struct hpa_shard_s { */ edata_cache_fast_t ecf; + /* Small extent cache (not guarded by mtx) */ + JEMALLOC_ALIGNED(CACHELINE) sec_t sec; + psset_t psset; /* @@ -141,22 +132,31 @@ struct hpa_shard_s { * Last time we performed purge on this shard. */ nstime_t last_purge; + + /* + * Last time when we attempted work (purging or hugifying). If deferral + * of the work is allowed (we have background thread), this is the time + * when background thread checked if purging or hugifying needs to be + * done. If deferral is not allowed, this is the time of (hpa_alloc or + * hpa_dalloc) activity in the shard. + */ + nstime_t last_time_work_attempted; }; +bool hpa_hugepage_size_exceeds_limit(void); /* - * Whether or not the HPA can be used given the current configuration. This is + * Whether or not the HPA can be used given the current configuration. This * is not necessarily a guarantee that it backs its allocations by hugepages, * just that it can function properly given the system it's running on. */ -bool hpa_supported(); -bool hpa_central_init(hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks); -bool hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap, - base_t *base, edata_cache_t *edata_cache, unsigned ind, - const hpa_shard_opts_t *opts); +bool hpa_supported(void); +bool hpa_shard_init(tsdn_t *tsdn, hpa_shard_t *shard, hpa_central_t *central, + emap_t *emap, base_t *base, edata_cache_t *edata_cache, unsigned ind, + const hpa_shard_opts_t *opts, const sec_opts_t *sec_opts); void hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src); -void hpa_shard_stats_merge(tsdn_t *tsdn, hpa_shard_t *shard, - hpa_shard_stats_t *dst); +void hpa_shard_stats_merge( + tsdn_t *tsdn, hpa_shard_t *shard, hpa_shard_stats_t *dst); /* * Notify the shard that we won't use it for allocations much longer. Due to @@ -165,15 +165,18 @@ void hpa_shard_stats_merge(tsdn_t *tsdn, hpa_shard_t *shard, */ void hpa_shard_disable(tsdn_t *tsdn, hpa_shard_t *shard); void hpa_shard_destroy(tsdn_t *tsdn, hpa_shard_t *shard); +/* Flush caches that shard may be using */ +void hpa_shard_flush(tsdn_t *tsdn, hpa_shard_t *shard); -void hpa_shard_set_deferral_allowed(tsdn_t *tsdn, hpa_shard_t *shard, - bool deferral_allowed); +void hpa_shard_set_deferral_allowed( + tsdn_t *tsdn, hpa_shard_t *shard, bool deferral_allowed); void hpa_shard_do_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard); /* * We share the fork ordering with the PA and arena prefork handling; that's why - * these are 3 and 4 rather than 0 and 1. + * these are 2, 3 and 4 rather than 0 and 1. */ +void hpa_shard_prefork2(tsdn_t *tsdn, hpa_shard_t *shard); void hpa_shard_prefork3(tsdn_t *tsdn, hpa_shard_t *shard); void hpa_shard_prefork4(tsdn_t *tsdn, hpa_shard_t *shard); void hpa_shard_postfork_parent(tsdn_t *tsdn, hpa_shard_t *shard); diff --git a/dep/jemalloc/include/jemalloc/internal/hpa_central.h b/dep/jemalloc/include/jemalloc/internal/hpa_central.h new file mode 100644 index 000000000..3e0ff7dae --- /dev/null +++ b/dep/jemalloc/include/jemalloc/internal/hpa_central.h @@ -0,0 +1,41 @@ +#ifndef JEMALLOC_INTERNAL_HPA_CENTRAL_H +#define JEMALLOC_INTERNAL_HPA_CENTRAL_H + +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/base.h" +#include "jemalloc/internal/hpa_hooks.h" +#include "jemalloc/internal/hpdata.h" +#include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/tsd_types.h" + +typedef struct hpa_central_s hpa_central_t; +struct hpa_central_s { + /* + * Guards expansion of eden. We separate this from the regular mutex so + * that cheaper operations can still continue while we're doing the OS + * call. + */ + malloc_mutex_t grow_mtx; + /* + * Either NULL (if empty), or some integer multiple of a + * hugepage-aligned number of hugepages. We carve them off one at a + * time to satisfy new pageslab requests. + * + * Guarded by grow_mtx. + */ + void *eden; + size_t eden_len; + /* Source for metadata. */ + base_t *base; + + /* The HPA hooks. */ + hpa_hooks_t hooks; +}; + +bool hpa_central_init( + hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks); + +hpdata_t *hpa_central_extract(tsdn_t *tsdn, hpa_central_t *central, size_t size, + uint64_t age, bool hugify_eager, bool *oom); + +#endif /* JEMALLOC_INTERNAL_HPA_CENTRAL_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/hpa_hooks.h b/dep/jemalloc/include/jemalloc/internal/hpa_hooks.h index 4ea221cb0..f50ff58f1 100644 --- a/dep/jemalloc/include/jemalloc/internal/hpa_hooks.h +++ b/dep/jemalloc/include/jemalloc/internal/hpa_hooks.h @@ -1,17 +1,21 @@ #ifndef JEMALLOC_INTERNAL_HPA_HOOKS_H #define JEMALLOC_INTERNAL_HPA_HOOKS_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/nstime.h" + typedef struct hpa_hooks_s hpa_hooks_t; struct hpa_hooks_s { void *(*map)(size_t size); void (*unmap)(void *ptr, size_t size); void (*purge)(void *ptr, size_t size); - void (*hugify)(void *ptr, size_t size); + bool (*hugify)(void *ptr, size_t size, bool sync); void (*dehugify)(void *ptr, size_t size); void (*curtime)(nstime_t *r_time, bool first_reading); uint64_t (*ms_since)(nstime_t *r_time); + bool (*vectorized_purge)(void *vec, size_t vlen, size_t nbytes); }; -extern hpa_hooks_t hpa_hooks_default; +extern const hpa_hooks_t hpa_hooks_default; #endif /* JEMALLOC_INTERNAL_HPA_HOOKS_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/hpa_opts.h b/dep/jemalloc/include/jemalloc/internal/hpa_opts.h index ee84fea13..6747c2db8 100644 --- a/dep/jemalloc/include/jemalloc/internal/hpa_opts.h +++ b/dep/jemalloc/include/jemalloc/internal/hpa_opts.h @@ -1,13 +1,66 @@ #ifndef JEMALLOC_INTERNAL_HPA_OPTS_H #define JEMALLOC_INTERNAL_HPA_OPTS_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/fxp.h" /* * This file is morally part of hpa.h, but is split out for header-ordering * reasons. + * + * All of these hpa_shard_opts below are experimental. We are exploring more + * efficient packing, hugifying, and purging approaches to make efficient + * trade-offs between CPU, memory, latency, and usability. This means all of + * them are at the risk of being deprecated and corresponding configurations + * should be updated once the final version settles. */ +/* + * This enum controls how jemalloc hugifies/dehugifies pages. Each style may be + * more suitable depending on deployment environments. + * + * hpa_hugify_style_none + * Using this means that jemalloc will not be hugifying or dehugifying pages, + * but will let the kernel make those decisions. This style only makes sense + * when deploying on systems where THP are enabled in 'always' mode. With this + * style, you most likely want to have no purging at all (dirty_mult=-1) or + * purge_threshold=HUGEPAGE bytes (2097152 for 2Mb page), although other + * thresholds may work well depending on kernel settings of your deployment + * targets. + * + * hpa_hugify_style_eager + * This style results in jemalloc giving hugepage advice, if needed, to + * anonymous memory immediately after it is mapped, so huge pages can be backing + * that memory at page-fault time. This is usually more efficient than doing + * it later, and it allows us to benefit from the hugepages from the start. + * Same options for purging as for the style 'none' are good starting choices: + * no purging, or purge_threshold=HUGEPAGE, some min_purge_delay_ms that allows + * for page not to be purged quickly, etc. This is a good choice if you can + * afford extra memory and your application gets performance increase from + * transparent hughepages. + * + * hpa_hugify_style_lazy + * This style is suitable when you purge more aggressively (you sacrifice CPU + * performance for less memory). When this style is chosen, jemalloc will + * hugify once hugification_threshold is reached, and dehugify before purging. + * If the kernel is configured to use direct compaction you may experience some + * allocation latency when using this style. The best is to measure what works + * better for your application needs, and in the target deployment environment. + * This is a good choice for apps that cannot afford a lot of memory regression, + * but would still like to benefit from backing certain memory regions with + * hugepages. + */ +enum hpa_hugify_style_e { + hpa_hugify_style_auto = 0, + hpa_hugify_style_none = 1, + hpa_hugify_style_eager = 2, + hpa_hugify_style_lazy = 3, + hpa_hugify_style_limit = hpa_hugify_style_lazy + 1 +}; +typedef enum hpa_hugify_style_e hpa_hugify_style_t; + +extern const char *const hpa_hugify_style_names[]; + typedef struct hpa_shard_opts_s hpa_shard_opts_t; struct hpa_shard_opts_s { /* @@ -44,12 +97,64 @@ struct hpa_shard_opts_s { */ uint64_t hugify_delay_ms; + /* + * Hugify pages synchronously (hugify will happen even if hugify_style + * is not hpa_hugify_style_lazy). + */ + bool hugify_sync; + /* * Minimum amount of time between purges. */ uint64_t min_purge_interval_ms; + + /* + * Maximum number of hugepages to purge on each purging attempt. + */ + ssize_t experimental_max_purge_nhp; + + /* + * Minimum number of inactive bytes needed for a non-empty page to be + * considered purgable. + * + * When the number of touched inactive bytes on non-empty hugepage is + * >= purge_threshold, the page is purgable. Empty pages are always + * purgable. Setting this to HUGEPAGE bytes would only purge empty + * pages if using hugify_style_eager and the purges would be exactly + * HUGEPAGE bytes. Depending on your kernel settings, this may result + * in better performance. + * + * Please note, when threshold is reached, we will purge all the dirty + * bytes, and not just up to the threshold. If this is PAGE bytes, then + * all the pages that have any dirty bytes are purgable. We treat + * purgability constraint for purge_threshold as stronger than + * dirty_mult, IOW, if no page meets purge_threshold, we will not purge + * even if we are above dirty_mult. + */ + size_t purge_threshold; + + /* + * Minimum number of ms that needs to elapse between HP page becoming + * eligible for purging and actually getting purged. + * + * Setting this to a larger number would give better chance of reusing + * that memory. Setting it to 0 means that page is eligible for purging + * as soon as it meets the purge_threshold. The clock resets when + * purgability of the page changes (page goes from being non-purgable to + * purgable). When using eager style you probably want to allow for + * some delay, to avoid purging the page too quickly and give it time to + * be used. + */ + uint64_t min_purge_delay_ms; + + /* + * Style of hugification/dehugification (see comment at + * hpa_hugify_style_t for options). + */ + hpa_hugify_style_t hugify_style; }; +/* clang-format off */ #define HPA_SHARD_OPTS_DEFAULT { \ /* slab_max_alloc */ \ 64 * 1024, \ @@ -67,8 +172,19 @@ struct hpa_shard_opts_s { false, \ /* hugify_delay_ms */ \ 10 * 1000, \ + /* hugify_sync */ \ + false, \ /* min_purge_interval_ms */ \ - 5 * 1000 \ + 5 * 1000, \ + /* experimental_max_purge_nhp */ \ + -1, \ + /* size_t purge_threshold */ \ + PAGE, \ + /* min_purge_delay_ms */ \ + 0, \ + /* hugify_style */ \ + hpa_hugify_style_lazy \ } +/* clang-format on */ #endif /* JEMALLOC_INTERNAL_HPA_OPTS_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/hpa_utils.h b/dep/jemalloc/include/jemalloc/internal/hpa_utils.h new file mode 100644 index 000000000..6b006cffc --- /dev/null +++ b/dep/jemalloc/include/jemalloc/internal/hpa_utils.h @@ -0,0 +1,161 @@ +#ifndef JEMALLOC_INTERNAL_HPA_UTILS_H +#define JEMALLOC_INTERNAL_HPA_UTILS_H + +#include "jemalloc/internal/hpa.h" +#include "jemalloc/internal/extent.h" + +#define HPA_MIN_VAR_VEC_SIZE 8 +/* + * This is used for jemalloc internal tuning and may change in the future based + * on production traffic. + * + * This value protects two things: + * 1. Stack size + * 2. Number of huge pages that are being purged in a batch as we do not + * allow allocations while making madvise syscall. + */ +#define HPA_PURGE_BATCH_MAX 16 + +#ifdef JEMALLOC_HAVE_PROCESS_MADVISE +typedef struct iovec hpa_io_vector_t; +#else +typedef struct { + void *iov_base; + size_t iov_len; +} hpa_io_vector_t; +#endif + +static inline size_t +hpa_process_madvise_max_iovec_len(void) { + assert( + opt_process_madvise_max_batch <= PROCESS_MADVISE_MAX_BATCH_LIMIT); + return opt_process_madvise_max_batch == 0 + ? HPA_MIN_VAR_VEC_SIZE + : opt_process_madvise_max_batch; +} + +/* Actually invoke hooks. If we fail vectorized, use single purges */ +static void +hpa_try_vectorized_purge( + hpa_hooks_t *hooks, hpa_io_vector_t *vec, size_t vlen, size_t nbytes) { + bool success = opt_process_madvise_max_batch > 0 + && !hooks->vectorized_purge(vec, vlen, nbytes); + if (!success) { + /* On failure, it is safe to purge again (potential perf + * penalty) If kernel can tell exactly which regions + * failed, we could avoid that penalty. + */ + for (size_t i = 0; i < vlen; ++i) { + hooks->purge(vec[i].iov_base, vec[i].iov_len); + } + } +} + +/* + * This structure accumulates the regions for process_madvise. It invokes the + * hook when batch limit is reached. + */ +typedef struct { + hpa_io_vector_t *vp; + size_t cur; + size_t total_bytes; + size_t capacity; +} hpa_range_accum_t; + +static inline void +hpa_range_accum_init(hpa_range_accum_t *ra, hpa_io_vector_t *v, size_t sz) { + ra->vp = v; + ra->capacity = sz; + ra->total_bytes = 0; + ra->cur = 0; +} + +static inline void +hpa_range_accum_flush(hpa_range_accum_t *ra, hpa_hooks_t *hooks) { + assert(ra->total_bytes > 0 && ra->cur > 0); + hpa_try_vectorized_purge(hooks, ra->vp, ra->cur, ra->total_bytes); + ra->cur = 0; + ra->total_bytes = 0; +} + +static inline void +hpa_range_accum_add( + hpa_range_accum_t *ra, void *addr, size_t sz, hpa_hooks_t *hooks) { + assert(ra->cur < ra->capacity); + + ra->vp[ra->cur].iov_base = addr; + ra->vp[ra->cur].iov_len = sz; + ra->total_bytes += sz; + ra->cur++; + + if (ra->cur == ra->capacity) { + hpa_range_accum_flush(ra, hooks); + } +} + +static inline void +hpa_range_accum_finish(hpa_range_accum_t *ra, hpa_hooks_t *hooks) { + if (ra->cur > 0) { + hpa_range_accum_flush(ra, hooks); + } +} + +/* + * For purging more than one page we use batch of these items + */ +typedef struct { + hpdata_purge_state_t state; + hpdata_t *hp; + bool dehugify; +} hpa_purge_item_t; + +typedef struct hpa_purge_batch_s hpa_purge_batch_t; +struct hpa_purge_batch_s { + hpa_purge_item_t *items; + size_t items_capacity; + /* Number of huge pages to purge in current batch */ + size_t item_cnt; + /* Number of ranges to purge in current batch */ + size_t nranges; + /* Total number of dirty pages in current batch*/ + size_t ndirty_in_batch; + + /* Max number of huge pages to purge */ + size_t max_hp; + /* + * Once we are above this watermark we should not add more pages + * to the same batch. This is because while we want to minimize + * number of madvise calls we also do not want to be preventing + * allocations from too many huge pages (which we have to do + * while they are being purged) + */ + size_t range_watermark; + + size_t npurged_hp_total; +}; + +static inline bool +hpa_batch_full(hpa_purge_batch_t *b) { + /* It's okay for ranges to go above */ + return b->npurged_hp_total == b->max_hp + || b->item_cnt == b->items_capacity + || b->nranges >= b->range_watermark; +} + +static inline void +hpa_batch_pass_start(hpa_purge_batch_t *b) { + b->item_cnt = 0; + b->nranges = 0; + b->ndirty_in_batch = 0; +} + +static inline bool +hpa_batch_empty(hpa_purge_batch_t *b) { + return b->item_cnt == 0; +} + +/* Purge pages in a batch using given hooks */ +void hpa_purge_batch( + hpa_hooks_t *hooks, hpa_purge_item_t *batch, size_t batch_sz); + +#endif /* JEMALLOC_INTERNAL_HPA_UTILS_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/hpdata.h b/dep/jemalloc/include/jemalloc/internal/hpdata.h index 1fb534db0..a9c507f09 100644 --- a/dep/jemalloc/include/jemalloc/internal/hpdata.h +++ b/dep/jemalloc/include/jemalloc/internal/hpdata.h @@ -1,7 +1,10 @@ #ifndef JEMALLOC_INTERNAL_HPDATA_H #define JEMALLOC_INTERNAL_HPDATA_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/fb.h" +#include "jemalloc/internal/nstime.h" +#include "jemalloc/internal/pages.h" #include "jemalloc/internal/ph.h" #include "jemalloc/internal/ql.h" #include "jemalloc/internal/typed_list.h" @@ -17,8 +20,14 @@ * an observable property of any given region of address space). It's just * hugepage-sized and hugepage-aligned; it's *potentially* huge. */ + +/* + * The max enumeration num should not exceed 2^16 - 1, see comments in edata.h + * for ESET_ENUMERATE_MAX_NUM for more details. + */ +#define PSSET_ENUMERATE_MAX_NUM 32 typedef struct hpdata_s hpdata_t; -ph_structs(hpdata_age_heap, hpdata_t); +ph_structs(hpdata_age_heap, hpdata_t, PSSET_ENUMERATE_MAX_NUM); struct hpdata_s { /* * We likewise follow the edata convention of mangling names and forcing @@ -64,7 +73,7 @@ struct hpdata_s { bool h_hugify_allowed; /* When we became a hugification candidate. */ nstime_t h_time_hugify_allowed; - bool h_in_psset_hugify_container; + bool h_in_psset_hugify_container; /* Whether or not a purge or hugify is currently happening. */ bool h_mid_purge; @@ -115,6 +124,12 @@ struct hpdata_s { /* The touched pages (using the same definition as above). */ fb_group_t touched_pages[FB_NGROUPS(HUGEPAGE_PAGES)]; + + /* Time when this extent (hpdata) becomes eligible for purging */ + nstime_t h_time_purge_allowed; + + /* True if the extent was huge and empty last time when it was purged */ + bool h_purged_when_empty_and_huge; }; TYPED_LIST(hpdata_empty_list, hpdata_t, ql_link_empty) @@ -177,8 +192,8 @@ hpdata_purge_allowed_get(const hpdata_t *hpdata) { static inline void hpdata_purge_allowed_set(hpdata_t *hpdata, bool purge_allowed) { - assert(purge_allowed == false || !hpdata->h_mid_purge); - hpdata->h_purge_allowed = purge_allowed; + assert(purge_allowed == false || !hpdata->h_mid_purge); + hpdata->h_purge_allowed = purge_allowed; } static inline bool @@ -241,7 +256,6 @@ hpdata_changing_state_get(const hpdata_t *hpdata) { return hpdata->h_mid_purge || hpdata->h_mid_hugify; } - static inline bool hpdata_updating_get(const hpdata_t *hpdata) { return hpdata->h_updating; @@ -276,17 +290,17 @@ hpdata_longest_free_range_set(hpdata_t *hpdata, size_t longest_free_range) { } static inline size_t -hpdata_nactive_get(hpdata_t *hpdata) { +hpdata_nactive_get(const hpdata_t *hpdata) { return hpdata->h_nactive; } static inline size_t -hpdata_ntouched_get(hpdata_t *hpdata) { +hpdata_ntouched_get(const hpdata_t *hpdata) { return hpdata->h_ntouched; } static inline size_t -hpdata_ndirty_get(hpdata_t *hpdata) { +hpdata_ndirty_get(const hpdata_t *hpdata) { return hpdata->h_ntouched - hpdata->h_nactive; } @@ -295,6 +309,26 @@ hpdata_nretained_get(hpdata_t *hpdata) { return HUGEPAGE_PAGES - hpdata->h_ntouched; } +static inline void +hpdata_time_purge_allowed_set(hpdata_t *hpdata, const nstime_t *v) { + nstime_copy(&hpdata->h_time_purge_allowed, v); +} + +static inline const nstime_t * +hpdata_time_purge_allowed_get(const hpdata_t *hpdata) { + return &hpdata->h_time_purge_allowed; +} + +static inline bool +hpdata_purged_when_empty_and_huge_get(const hpdata_t *hpdata) { + return hpdata->h_purged_when_empty_and_huge; +} + +static inline void +hpdata_purged_when_empty_and_huge_set(hpdata_t *hpdata, bool v) { + hpdata->h_purged_when_empty_and_huge = v; +} + static inline void hpdata_assert_empty(hpdata_t *hpdata) { assert(fb_empty(hpdata->active_pages, HUGEPAGE_PAGES)); @@ -308,58 +342,95 @@ hpdata_assert_empty(hpdata_t *hpdata) { */ static inline bool hpdata_consistent(hpdata_t *hpdata) { - if(fb_urange_longest(hpdata->active_pages, HUGEPAGE_PAGES) - != hpdata_longest_free_range_get(hpdata)) { - return false; + bool res = true; + + const size_t active_urange_longest = fb_urange_longest( + hpdata->active_pages, HUGEPAGE_PAGES); + const size_t longest_free_range = hpdata_longest_free_range_get(hpdata); + if (active_urange_longest != longest_free_range) { + malloc_printf( + ": active_fb_urange_longest=%zu != hpdata_longest_free_range=%zu\n", + active_urange_longest, longest_free_range); + res = false; } - if (fb_scount(hpdata->active_pages, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES) - != hpdata->h_nactive) { - return false; + + const size_t active_scount = fb_scount( + hpdata->active_pages, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES); + if (active_scount != hpdata->h_nactive) { + malloc_printf( + ": active_fb_scount=%zu != hpdata_nactive=%zu\n", + active_scount, hpdata->h_nactive); + res = false; } - if (fb_scount(hpdata->touched_pages, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES) - != hpdata->h_ntouched) { - return false; + + const size_t touched_scount = fb_scount( + hpdata->touched_pages, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES); + if (touched_scount != hpdata->h_ntouched) { + malloc_printf( + ": touched_fb_scount=%zu != hpdata_ntouched=%zu\n", + touched_scount, hpdata->h_ntouched); + res = false; } + if (hpdata->h_ntouched < hpdata->h_nactive) { - return false; + malloc_printf( + ": hpdata_ntouched=%zu < hpdata_nactive=%zu\n", + hpdata->h_ntouched, hpdata->h_nactive); + res = false; } - if (hpdata->h_huge && hpdata->h_ntouched != HUGEPAGE_PAGES) { - return false; + + if (hpdata->h_huge && (hpdata->h_ntouched != HUGEPAGE_PAGES)) { + malloc_printf( + ": hpdata_huge=%d && (hpdata_ntouched=%zu != hugepage_pages=%zu)\n", + hpdata->h_huge, hpdata->h_ntouched, HUGEPAGE_PAGES); + res = false; } - if (hpdata_changing_state_get(hpdata) - && ((hpdata->h_purge_allowed) || hpdata->h_hugify_allowed)) { - return false; + + const bool changing_state = hpdata_changing_state_get(hpdata); + if (changing_state + && (hpdata->h_purge_allowed || hpdata->h_hugify_allowed)) { + malloc_printf( + ": hpdata_changing_state=%d && (hpdata_purge_allowed=%d || hpdata_hugify_allowed=%d)\n", + changing_state, hpdata->h_purge_allowed, + hpdata->h_hugify_allowed); + res = false; } + if (hpdata_hugify_allowed_get(hpdata) != hpdata_in_psset_hugify_container_get(hpdata)) { - return false; + malloc_printf( + ": hpdata_hugify_allowed=%d != hpdata_in_psset_hugify_container=%d\n", + hpdata_hugify_allowed_get(hpdata), + hpdata_in_psset_hugify_container_get(hpdata)); + res = false; } - return true; + + return res; } -static inline void -hpdata_assert_consistent(hpdata_t *hpdata) { - assert(hpdata_consistent(hpdata)); -} +#define hpdata_assert_consistent(hpdata) \ + do { \ + assert(hpdata_consistent(hpdata)); \ + } while (0) static inline bool -hpdata_empty(hpdata_t *hpdata) { +hpdata_empty(const hpdata_t *hpdata) { return hpdata->h_nactive == 0; } static inline bool -hpdata_full(hpdata_t *hpdata) { +hpdata_full(const hpdata_t *hpdata) { return hpdata->h_nactive == HUGEPAGE_PAGES; } -void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age); +void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age, bool is_huge); /* * Given an hpdata which can serve an allocation request, pick and reserve an * offset within that allocation. */ void *hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz); -void hpdata_unreserve(hpdata_t *hpdata, void *begin, size_t sz); +void hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz); /* * The hpdata_purge_prepare_t allows grabbing the metadata required to purge @@ -368,10 +439,10 @@ void hpdata_unreserve(hpdata_t *hpdata, void *begin, size_t sz); */ typedef struct hpdata_purge_state_s hpdata_purge_state_t; struct hpdata_purge_state_s { - size_t npurged; - size_t ndirty_to_purge; + size_t npurged; + size_t ndirty_to_purge; fb_group_t to_purge[FB_NGROUPS(HUGEPAGE_PAGES)]; - size_t next_purge_search_begin; + size_t next_purge_search_begin; }; /* @@ -386,9 +457,11 @@ struct hpdata_purge_state_s { * until you're done, and then end. Allocating out of an hpdata undergoing * purging is not allowed. * - * Returns the number of dirty pages that will be purged. + * Returns the number of dirty pages that will be purged and sets nranges + * to number of ranges with dirty pages that will be purged. */ -size_t hpdata_purge_begin(hpdata_t *hpdata, hpdata_purge_state_t *purge_state); +size_t hpdata_purge_begin( + hpdata_t *hpdata, hpdata_purge_state_t *purge_state, size_t *nranges); /* * If there are more extents to purge, sets *r_purge_addr and *r_purge_size to diff --git a/dep/jemalloc/include/jemalloc/internal/inspect.h b/dep/jemalloc/include/jemalloc/internal/inspect.h index 65fef51df..e8ed44d32 100644 --- a/dep/jemalloc/include/jemalloc/internal/inspect.h +++ b/dep/jemalloc/include/jemalloc/internal/inspect.h @@ -1,6 +1,9 @@ #ifndef JEMALLOC_INTERNAL_INSPECT_H #define JEMALLOC_INTERNAL_INSPECT_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/tsd_types.h" + /* * This module contains the heap introspection capabilities. For now they are * exposed purely through mallctl APIs in the experimental namespace, but this @@ -23,7 +26,7 @@ typedef struct inspect_extent_util_stats_verbose_s inspect_extent_util_stats_verbose_t; struct inspect_extent_util_stats_verbose_s { - void *slabcur_addr; + void *slabcur_addr; size_t nfree; size_t nregs; size_t size; @@ -31,10 +34,10 @@ struct inspect_extent_util_stats_verbose_s { size_t bin_nregs; }; -void inspect_extent_util_stats_get(tsdn_t *tsdn, const void *ptr, - size_t *nfree, size_t *nregs, size_t *size); +void inspect_extent_util_stats_get( + tsdn_t *tsdn, const void *ptr, size_t *nfree, size_t *nregs, size_t *size); void inspect_extent_util_stats_verbose_get(tsdn_t *tsdn, const void *ptr, - size_t *nfree, size_t *nregs, size_t *size, - size_t *bin_nfree, size_t *bin_nregs, void **slabcur_addr); + size_t *nfree, size_t *nregs, size_t *size, size_t *bin_nfree, + size_t *bin_nregs, void **slabcur_addr); #endif /* JEMALLOC_INTERNAL_INSPECT_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h index 983027c86..2ca12c4ab 100644 --- a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h +++ b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h @@ -3,64 +3,65 @@ #include #ifdef _WIN32 -# include -# include "msvc_compat/windows_extra.h" -# include "msvc_compat/strings.h" -# ifdef _WIN64 -# if LG_VADDR <= 32 -# error Generate the headers using x64 vcargs -# endif -# else -# if LG_VADDR > 32 -# undef LG_VADDR -# define LG_VADDR 32 -# endif -# endif +# include +# include "msvc_compat/windows_extra.h" +# include "msvc_compat/strings.h" +# ifdef _WIN64 +# if LG_VADDR <= 32 +# error Generate the headers using x64 vcargs +# endif +# else +# if LG_VADDR > 32 +# undef LG_VADDR +# define LG_VADDR 32 +# endif +# endif #else -# include -# include -# if !defined(__pnacl__) && !defined(__native_client__) -# include -# if !defined(SYS_write) && defined(__NR_write) -# define SYS_write __NR_write -# endif -# if defined(SYS_open) && defined(__aarch64__) - /* Android headers may define SYS_open to __NR_open even though +# include +# include +# if !defined(__pnacl__) && !defined(__native_client__) +# include +# if !defined(SYS_write) && defined(__NR_write) +# define SYS_write __NR_write +# endif +# if defined(SYS_open) && defined(__aarch64__) +/* Android headers may define SYS_open to __NR_open even though * __NR_open may not exist on AArch64 (superseded by __NR_openat). */ -# undef SYS_open -# endif -# include -# endif -# include -# if defined(__FreeBSD__) || defined(__DragonFly__) -# include -# include -# if defined(__FreeBSD__) -# define cpu_set_t cpuset_t -# endif -# endif -# include -# ifdef JEMALLOC_OS_UNFAIR_LOCK -# include -# endif -# ifdef JEMALLOC_GLIBC_MALLOC_HOOK -# include -# endif -# include -# include -# include -# ifdef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME -# include -# endif +# undef SYS_open +# endif +# include +# endif +# include +# if defined(__FreeBSD__) || defined(__DragonFly__) \ + || defined(__OpenBSD__) +# include +# include +# if defined(__FreeBSD__) +# define cpu_set_t cpuset_t +# endif +# endif +# include +# ifdef JEMALLOC_OS_UNFAIR_LOCK +# include +# endif +# ifdef JEMALLOC_GLIBC_MALLOC_HOOK +# include +# endif +# include +# include +# include +# ifdef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME +# include +# endif #endif #include #include #ifndef SIZE_T_MAX -# define SIZE_T_MAX SIZE_MAX +# define SIZE_T_MAX SIZE_MAX #endif #ifndef SSIZE_MAX -# define SSIZE_MAX ((ssize_t)(SIZE_T_MAX >> 1)) +# define SSIZE_MAX ((ssize_t)(SIZE_T_MAX >> 1)) #endif #include #include @@ -69,30 +70,30 @@ #include #include #ifndef offsetof -# define offsetof(type, member) ((size_t)&(((type *)NULL)->member)) +# define offsetof(type, member) ((size_t) & (((type *)NULL)->member)) #endif #include #include #include #ifdef _MSC_VER -# include +# include typedef intptr_t ssize_t; -# define PATH_MAX 1024 -# define STDERR_FILENO 2 -# define __func__ __FUNCTION__ -# ifdef JEMALLOC_HAS_RESTRICT -# define restrict __restrict -# endif +# define PATH_MAX 1024 +# define STDERR_FILENO 2 +# define __func__ __FUNCTION__ +# ifdef JEMALLOC_HAS_RESTRICT +# define restrict __restrict +# endif /* Disable warnings about deprecated system functions. */ -# pragma warning(disable: 4996) -#if _MSC_VER < 1800 +# pragma warning(disable : 4996) +# if _MSC_VER < 1800 static int isblank(int c) { return (c == '\t' || c == ' '); } -#endif +# endif #else -# include +# include #endif #include @@ -102,7 +103,24 @@ isblank(int c) { * classes. */ #ifdef small -# undef small +# undef small #endif +/* + * Oftentimes we'd like to perform some kind of arithmetic to obtain + * a pointer from another pointer but with some offset or mask applied. + * Naively you would accomplish this by casting the source pointer to + * `uintptr_t`, performing all of the relevant arithmetic, and then casting + * the result to the desired pointer type. However, this has the unfortunate + * side-effect of concealing pointer provenance, hiding useful information for + * optimization from the compiler (see here for details: + * https://clang.llvm.org/extra/clang-tidy/checks/performance/no-int-to-ptr.html + * ) + * Instead what one should do is cast the source pointer to `char *` and perform + * the equivalent arithmetic (since `char` of course represents one byte). But + * because `char *` has the semantic meaning of "string", we define this typedef + * simply to make it clearer where we are performing such pointer arithmetic. + */ +typedef char byte_t; + #endif /* JEMALLOC_INTERNAL_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in index cb69334ec..e6a95790f 100644 --- a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -14,10 +14,13 @@ */ #cmakedefine JEMALLOC_OVERRIDE___LIBC_CALLOC #cmakedefine JEMALLOC_OVERRIDE___LIBC_FREE +#cmakedefine JEMALLOC_OVERRIDE___LIBC_FREE_SIZED +#cmakedefine JEMALLOC_OVERRIDE___LIBC_FREE_ALIGNED_SIZED #cmakedefine JEMALLOC_OVERRIDE___LIBC_MALLOC #cmakedefine JEMALLOC_OVERRIDE___LIBC_MEMALIGN #cmakedefine JEMALLOC_OVERRIDE___LIBC_REALLOC #cmakedefine JEMALLOC_OVERRIDE___LIBC_VALLOC +#cmakedefine JEMALLOC_OVERRIDE___LIBC_PVALLOC #cmakedefine JEMALLOC_OVERRIDE___POSIX_MEMALIGN /* @@ -88,6 +91,9 @@ /* Defined if pthread_getname_np(3) is available. */ #cmakedefine JEMALLOC_HAVE_PTHREAD_GETNAME_NP +/* Defined if pthread_set_name_np(3) is available. */ +#cmakedefine JEMALLOC_HAVE_PTHREAD_SET_NAME_NP + /* Defined if pthread_get_name_np(3) is available. */ #cmakedefine JEMALLOC_HAVE_PTHREAD_GET_NAME_NP @@ -111,6 +117,11 @@ */ #cmakedefine JEMALLOC_HAVE_CLOCK_REALTIME +/* + * Defined if clock_gettime_nsec_np(CLOCK_UPTIME_RAW) is available. + */ +#cmakedefine JEMALLOC_HAVE_CLOCK_GETTIME_NSEC_NP + /* * Defined if _malloc_thread_cleanup() exists. At least in the case of * FreeBSD, pthread_key_create() allocates, which if used during malloc @@ -161,6 +172,15 @@ /* Use gcc intrinsics for profile backtracing if defined. */ #cmakedefine JEMALLOC_PROF_GCC +/* Use frame pointer for profile backtracing if defined. Linux only. */ +#cmakedefine JEMALLOC_PROF_FRAME_POINTER + +/* JEMALLOC_PAGEID enabled page id */ +#cmakedefine JEMALLOC_PAGEID + +/* JEMALLOC_HAVE_PRCTL checks prctl */ +#cmakedefine JEMALLOC_HAVE_PRCTL + /* * JEMALLOC_DSS enables use of sbrk(2) to allocate extents from the data storage * segment (DSS). @@ -259,6 +279,12 @@ */ #cmakedefine JEMALLOC_READLINKAT +/* + * If defined, use getenv() (instead of secure_getenv() or + * alternatives) to access MALLOC_CONF. + */ +#cmakedefine JEMALLOC_FORCE_GETENV + /* * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings. */ @@ -282,6 +308,13 @@ */ #cmakedefine JEMALLOC_HAVE_MADVISE_HUGE +/* + * Defined if best-effort synchronous collapse of the native + * pages mapped by the memory range into transparent huge pages is supported + * via MADV_COLLAPSE arguments to madvise(2). + */ +#cmakedefine JEMALLOC_HAVE_MADVISE_COLLAPSE + /* * Methods for purging unused pages differ between operating systems. * @@ -312,9 +345,23 @@ */ #cmakedefine JEMALLOC_MADVISE_NOCORE +/* Defined if process_madvise(2) is available. */ +#cmakedefine JEMALLOC_HAVE_PROCESS_MADVISE + +#cmakedefine EXPERIMENTAL_SYS_PROCESS_MADVISE_NR @EXPERIMENTAL_SYS_PROCESS_MADVISE_NR@ + /* Defined if mprotect(2) is available. */ #cmakedefine JEMALLOC_HAVE_MPROTECT +/* Defined if sys/sdt.h is available and sdt tracing enabled */ +#cmakedefine JEMALLOC_EXPERIMENTAL_USDT_STAP + +/* + * Defined if sys/sdt.h is unavailable, sdt tracing enabled, and + * platform is supported + */ +#cmakedefine JEMALLOC_EXPERIMENTAL_USDT_CUSTOM + /* * Defined if transparent huge pages (THPs) are supported via the * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. @@ -378,12 +425,18 @@ /* Adaptive mutex support in pthreads. */ #cmakedefine JEMALLOC_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP +/* gettid() support */ +#cmakedefine JEMALLOC_HAVE_GETTID + /* GNU specific sched_getcpu support */ #cmakedefine JEMALLOC_HAVE_SCHED_GETCPU /* GNU specific sched_setaffinity support */ #cmakedefine JEMALLOC_HAVE_SCHED_SETAFFINITY +/* pthread_setaffinity_np support */ +#cmakedefine JEMALLOC_HAVE_PTHREAD_SETAFFINITY_NP + /* * If defined, all the features necessary for background threads are present. */ @@ -412,6 +465,9 @@ /* Is C++ support being built? */ #cmakedefine JEMALLOC_ENABLE_CXX +/* Are C++ exceptions enabled? */ +#cmakedefine JEMALLOC_HAVE_CXX_EXCEPTIONS + /* Performs additional size checks when defined. */ #cmakedefine JEMALLOC_OPT_SIZE_CHECKS @@ -424,4 +480,18 @@ /* If defined, realloc(ptr, 0) defaults to "free" instead of "alloc". */ #cmakedefine JEMALLOC_ZERO_REALLOC_DEFAULT_FREE +/* If defined, use volatile asm during benchmarks. */ +#cmakedefine JEMALLOC_HAVE_ASM_VOLATILE + +/* + * If defined, support the use of rdtscp to get the time stamp counter + * and the processor ID. + */ +#cmakedefine JEMALLOC_HAVE_RDTSCP + +/* If defined, use __int128 for optimization. */ +#cmakedefine JEMALLOC_HAVE_INT128 + +#include "jemalloc/internal/jemalloc_internal_overrides.h" + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h index fc834c673..9911c199f 100644 --- a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h +++ b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h @@ -1,39 +1,52 @@ #ifndef JEMALLOC_INTERNAL_EXTERNS_H #define JEMALLOC_INTERNAL_EXTERNS_H +#include "jemalloc/internal/arena_types.h" #include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/fxp.h" #include "jemalloc/internal/hpa_opts.h" +#include "jemalloc/internal/nstime.h" #include "jemalloc/internal/sec_opts.h" #include "jemalloc/internal/tsd_types.h" -#include "jemalloc/internal/nstime.h" /* TSD checks this to set thread local slow state accordingly. */ extern bool malloc_slow; /* Run-time options. */ -extern bool opt_abort; -extern bool opt_abort_conf; -extern bool opt_trust_madvise; -extern bool opt_confirm_conf; -extern bool opt_hpa; +extern bool opt_abort; +extern bool opt_abort_conf; +extern bool opt_trust_madvise; +extern bool opt_experimental_hpa_start_huge_if_thp_always; +extern bool opt_experimental_hpa_enforce_hugify; +extern bool opt_confirm_conf; +extern bool opt_hpa; extern hpa_shard_opts_t opt_hpa_opts; -extern sec_opts_t opt_hpa_sec_opts; +extern sec_opts_t opt_hpa_sec_opts; extern const char *opt_junk; -extern bool opt_junk_alloc; -extern bool opt_junk_free; -extern void (*junk_free_callback)(void *ptr, size_t size); -extern void (*junk_alloc_callback)(void *ptr, size_t size); -extern bool opt_utrace; -extern bool opt_xmalloc; -extern bool opt_experimental_infallible_new; -extern bool opt_zero; -extern unsigned opt_narenas; +extern bool opt_junk_alloc; +extern bool opt_junk_free; +extern void (*JET_MUTABLE junk_free_callback)(void *ptr, size_t size); +extern void (*JET_MUTABLE junk_alloc_callback)(void *ptr, size_t size); +extern void (*JET_MUTABLE invalid_conf_abort)(void); +extern bool opt_utrace; +extern bool opt_xmalloc; +extern bool opt_experimental_infallible_new; +extern bool opt_experimental_tcache_gc; +extern bool opt_zero; +extern unsigned opt_narenas; +extern fxp_t opt_narenas_ratio; extern zero_realloc_action_t opt_zero_realloc_action; -extern malloc_init_t malloc_init_state; -extern const char *zero_realloc_mode_names[]; -extern atomic_zu_t zero_realloc_count; -extern bool opt_cache_oblivious; +extern malloc_init_t malloc_init_state; +extern const char *const zero_realloc_mode_names[]; +extern atomic_zu_t zero_realloc_count; +extern bool opt_cache_oblivious; +extern unsigned opt_debug_double_free_max_scan; +extern size_t opt_calloc_madvise_threshold; +extern bool opt_disable_large_size_classes; + +extern const char *opt_malloc_conf_symlink; +extern const char *opt_malloc_conf_env_var; /* Escape free-fastpath when ptr & mask == 0 (for sanitization purpose). */ extern uintptr_t san_cache_bin_nonfast_mask; @@ -53,23 +66,26 @@ extern unsigned manual_arena_base; */ extern atomic_p_t arenas[]; -void *a0malloc(size_t size); -void a0dalloc(void *ptr); -void *bootstrap_malloc(size_t size); -void *bootstrap_calloc(size_t num, size_t size); -void bootstrap_free(void *ptr); -void arena_set(unsigned ind, arena_t *arena); +extern unsigned huge_arena_ind; + +void *a0malloc(size_t size); +void a0dalloc(void *ptr); +void *bootstrap_malloc(size_t size); +void *bootstrap_calloc(size_t num, size_t size); +void bootstrap_free(void *ptr); +void arena_set(unsigned ind, arena_t *arena); unsigned narenas_total_get(void); arena_t *arena_init(tsdn_t *tsdn, unsigned ind, const arena_config_t *config); arena_t *arena_choose_hard(tsd_t *tsd, bool internal); -void arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena); -void iarena_cleanup(tsd_t *tsd); -void arena_cleanup(tsd_t *tsd); -size_t batch_alloc(void **ptrs, size_t num, size_t size, int flags); -void jemalloc_prefork(void); -void jemalloc_postfork_parent(void); -void jemalloc_postfork_child(void); -void je_sdallocx_noflags(void *ptr, size_t size); -void *malloc_default(size_t size); +void arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena); +void iarena_cleanup(tsd_t *tsd); +void arena_cleanup(tsd_t *tsd); +size_t batch_alloc(void **ptrs, size_t num, size_t size, int flags); +void jemalloc_prefork(void); +void jemalloc_postfork_parent(void); +void jemalloc_postfork_child(void); +void sdallocx_default(void *ptr, size_t size, int flags); +void free_default(void *ptr); +void *malloc_default(size_t size); #endif /* JEMALLOC_INTERNAL_EXTERNS_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_a.h b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_a.h index 9e27cc301..8513effd5 100644 --- a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_a.h +++ b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_a.h @@ -1,10 +1,14 @@ #ifndef JEMALLOC_INTERNAL_INLINES_A_H #define JEMALLOC_INTERNAL_INLINES_A_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/arena_externs.h" +#include "jemalloc/internal/arena_types.h" #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/bit_util.h" #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/sc.h" +#include "jemalloc/internal/tcache_externs.h" #include "jemalloc/internal/ticker.h" JEMALLOC_ALWAYS_INLINE malloc_cpuid_t @@ -14,6 +18,15 @@ malloc_getcpu(void) { return GetCurrentProcessorNumber(); #elif defined(JEMALLOC_HAVE_SCHED_GETCPU) return (malloc_cpuid_t)sched_getcpu(); +#elif defined(JEMALLOC_HAVE_RDTSCP) + unsigned int ecx; + asm volatile("rdtscp" : "=c"(ecx)::"eax", "edx"); + return (malloc_cpuid_t)(ecx & 0xfff); +#elif defined(__aarch64__) && defined(__APPLE__) + /* Other oses most likely use tpidr_el0 instead */ + uintptr_t c; + asm volatile("mrs %x0, tpidrro_el0" : "=r"(c)::"memory"); + return (malloc_cpuid_t)(c & (1 << 3) - 1); #else not_reached(); return -1; @@ -29,8 +42,8 @@ percpu_arena_choose(void) { assert(cpuid >= 0); unsigned arena_ind; - if ((opt_percpu_arena == percpu_arena) || ((unsigned)cpuid < ncpus / - 2)) { + if ((opt_percpu_arena == percpu_arena) + || ((unsigned)cpuid < ncpus / 2)) { arena_ind = cpuid; } else { assert(opt_percpu_arena == per_phycpu_arena); diff --git a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_b.h b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_b.h index 152f8a039..dad37a9c1 100644 --- a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_b.h +++ b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_b.h @@ -1,7 +1,10 @@ #ifndef JEMALLOC_INTERNAL_INLINES_B_H #define JEMALLOC_INTERNAL_INLINES_B_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/arena_inlines_a.h" #include "jemalloc/internal/extent.h" +#include "jemalloc/internal/jemalloc_internal_inlines_a.h" static inline void percpu_arena_update(tsd_t *tsd, unsigned cpu) { @@ -20,13 +23,13 @@ percpu_arena_update(tsd_t *tsd, unsigned cpu) { tcache_t *tcache = tcache_get(tsd); if (tcache != NULL) { tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd); - tcache_arena_reassociate(tsd_tsdn(tsd), tcache_slow, - tcache, newarena); + assert(tcache_slow->arena != NULL); + tcache_arena_reassociate( + tsd_tsdn(tsd), tcache_slow, tcache, newarena); } } } - /* Choose an arena based on a per-thread value. */ static inline arena_t * arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) { @@ -47,18 +50,18 @@ arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) { assert(ret); if (tcache_available(tsd)) { tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd); - tcache_t *tcache = tsd_tcachep_get(tsd); + tcache_t *tcache = tsd_tcachep_get(tsd); if (tcache_slow->arena != NULL) { /* See comments in tsd_tcache_data_init().*/ - assert(tcache_slow->arena == - arena_get(tsd_tsdn(tsd), 0, false)); + assert(tcache_slow->arena + == arena_get(tsd_tsdn(tsd), 0, false)); if (tcache_slow->arena != ret) { tcache_arena_reassociate(tsd_tsdn(tsd), tcache_slow, tcache, ret); } } else { - tcache_arena_associate(tsd_tsdn(tsd), - tcache_slow, tcache, ret); + tcache_arena_associate( + tsd_tsdn(tsd), tcache_slow, tcache, ret); } } } @@ -68,10 +71,10 @@ arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) { * auto percpu arena range, (i.e. thread is assigned to a manually * managed arena), then percpu arena is skipped. */ - if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena) && - !internal && (arena_ind_get(ret) < - percpu_arena_ind_limit(opt_percpu_arena)) && (ret->last_thd != - tsd_tsdn(tsd))) { + if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena) + && !internal + && (arena_ind_get(ret) < percpu_arena_ind_limit(opt_percpu_arena)) + && (ret->last_thd != tsd_tsdn(tsd))) { unsigned ind = percpu_arena_choose(); if (arena_ind_get(ret) != ind) { percpu_arena_update(tsd, ind); diff --git a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h index b0868b7d6..2c61f8c4f 100644 --- a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h +++ b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h @@ -1,6 +1,10 @@ #ifndef JEMALLOC_INTERNAL_INLINES_C_H #define JEMALLOC_INTERNAL_INLINES_C_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/arena_externs.h" +#include "jemalloc/internal/arena_inlines_b.h" +#include "jemalloc/internal/emap.h" #include "jemalloc/internal/hook.h" #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/log.h" @@ -8,6 +12,15 @@ #include "jemalloc/internal/thread_event.h" #include "jemalloc/internal/witness.h" +/* + * These correspond to the macros in jemalloc/jemalloc_macros.h. Broadly, we + * should have one constant here per magic value there. Note however that the + * representations need not be related. + */ +#define TCACHE_IND_NONE ((unsigned)-1) +#define TCACHE_IND_AUTOMATIC ((unsigned)-2) +#define ARENA_IND_AUTOMATIC ((unsigned)-1) + /* * Translating the names of the 'i' functions: * Abbreviations used in the first part of the function name (before @@ -41,24 +54,35 @@ isalloc(tsdn_t *tsdn, const void *ptr) { } JEMALLOC_ALWAYS_INLINE void * -iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache, - bool is_internal, arena_t *arena, bool slow_path) { +iallocztm_explicit_slab(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, + bool slab, tcache_t *tcache, bool is_internal, arena_t *arena, + bool slow_path) { void *ret; + assert(!slab || sz_can_use_slab(size)); /* slab && large is illegal */ assert(!is_internal || tcache == NULL); assert(!is_internal || arena == NULL || arena_is_auto(arena)); if (!tsdn_null(tsdn) && tsd_reentrancy_level_get(tsdn_tsd(tsdn)) == 0) { - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); } - ret = arena_malloc(tsdn, arena, size, ind, zero, tcache, slow_path); + ret = arena_malloc( + tsdn, arena, size, ind, zero, slab, tcache, slow_path); if (config_stats && is_internal && likely(ret != NULL)) { arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret)); } return ret; } +JEMALLOC_ALWAYS_INLINE void * +iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache, + bool is_internal, arena_t *arena, bool slow_path) { + bool slab = sz_can_use_slab(size); + return iallocztm_explicit_slab( + tsdn, size, ind, zero, slab, tcache, is_internal, arena, slow_path); +} + JEMALLOC_ALWAYS_INLINE void * ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero, bool slow_path) { return iallocztm(tsd_tsdn(tsd), size, ind, zero, tcache_get(tsd), false, @@ -66,18 +90,19 @@ ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero, bool slow_path) { } JEMALLOC_ALWAYS_INLINE void * -ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, - tcache_t *tcache, bool is_internal, arena_t *arena) { +ipallocztm_explicit_slab(tsdn_t *tsdn, size_t usize, size_t alignment, + bool zero, bool slab, tcache_t *tcache, bool is_internal, arena_t *arena) { void *ret; + assert(!slab || sz_can_use_slab(usize)); /* slab && large is illegal */ assert(usize != 0); assert(usize == sz_sa2u(usize, alignment)); assert(!is_internal || tcache == NULL); assert(!is_internal || arena == NULL || arena_is_auto(arena)); - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); - ret = arena_palloc(tsdn, arena, usize, alignment, zero, tcache); + ret = arena_palloc(tsdn, arena, usize, alignment, zero, slab, tcache); assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret); if (config_stats && is_internal && likely(ret != NULL)) { arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret)); @@ -85,12 +110,26 @@ ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, return ret; } +JEMALLOC_ALWAYS_INLINE void * +ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, + tcache_t *tcache, bool is_internal, arena_t *arena) { + return ipallocztm_explicit_slab(tsdn, usize, alignment, zero, + sz_can_use_slab(usize), tcache, is_internal, arena); +} + JEMALLOC_ALWAYS_INLINE void * ipalloct(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena) { return ipallocztm(tsdn, usize, alignment, zero, tcache, false, arena); } +JEMALLOC_ALWAYS_INLINE void * +ipalloct_explicit_slab(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, + bool slab, tcache_t *tcache, arena_t *arena) { + return ipallocztm_explicit_slab( + tsdn, usize, alignment, zero, slab, tcache, false, arena); +} + JEMALLOC_ALWAYS_INLINE void * ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero) { return ipallocztm(tsd_tsdn(tsd), usize, alignment, zero, @@ -108,13 +147,13 @@ idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, assert(ptr != NULL); assert(!is_internal || tcache == NULL); assert(!is_internal || arena_is_auto(iaalloc(tsdn, ptr))); - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); if (config_stats && is_internal) { arena_internal_sub(iaalloc(tsdn, ptr), isalloc(tsdn, ptr)); } - if (!is_internal && !tsdn_null(tsdn) && - tsd_reentrancy_level_get(tsdn_tsd(tsdn)) != 0) { + if (!is_internal && !tsdn_null(tsdn) + && tsd_reentrancy_level_get(tsdn_tsd(tsdn)) != 0) { assert(tcache == NULL); } arena_dalloc(tsdn, ptr, tcache, alloc_ctx, slow_path); @@ -128,25 +167,26 @@ idalloc(tsd_t *tsd, void *ptr) { JEMALLOC_ALWAYS_INLINE void isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, emap_alloc_ctx_t *alloc_ctx, bool slow_path) { - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); arena_sdalloc(tsdn, ptr, size, tcache, alloc_ctx, slow_path); } JEMALLOC_ALWAYS_INLINE void * iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, - size_t alignment, bool zero, tcache_t *tcache, arena_t *arena, + size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena, hook_ralloc_args_t *hook_args) { - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); - void *p; + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); + void *p; size_t usize, copysize; usize = sz_sa2u(size, alignment); if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { return NULL; } - p = ipalloct(tsdn, usize, alignment, zero, tcache, arena); + p = ipalloct_explicit_slab( + tsdn, usize, alignment, zero, slab, tcache, arena); if (p == NULL) { return NULL; } @@ -156,11 +196,12 @@ iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, */ copysize = (size < oldsize) ? size : oldsize; memcpy(p, ptr, copysize); - hook_invoke_alloc(hook_args->is_realloc - ? hook_alloc_realloc : hook_alloc_rallocx, p, (uintptr_t)p, - hook_args->args); - hook_invoke_dalloc(hook_args->is_realloc - ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args); + hook_invoke_alloc( + hook_args->is_realloc ? hook_alloc_realloc : hook_alloc_rallocx, p, + (uintptr_t)p, hook_args->args); + hook_invoke_dalloc( + hook_args->is_realloc ? hook_dalloc_realloc : hook_dalloc_rallocx, + ptr, hook_args->args); isdalloct(tsdn, ptr, oldsize, tcache, NULL, true); return p; } @@ -173,33 +214,42 @@ iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, * passed-around anywhere. */ JEMALLOC_ALWAYS_INLINE void * -iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment, - bool zero, tcache_t *tcache, arena_t *arena, hook_ralloc_args_t *hook_args) -{ +iralloct_explicit_slab(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, + size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena, + hook_ralloc_args_t *hook_args) { assert(ptr != NULL); assert(size != 0); - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); - if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) - != 0) { + if (alignment != 0 + && ((uintptr_t)ptr & ((uintptr_t)alignment - 1)) != 0) { /* * Existing object alignment is inadequate; allocate new space * and copy. */ return iralloct_realign(tsdn, ptr, oldsize, size, alignment, - zero, tcache, arena, hook_args); + zero, slab, tcache, arena, hook_args); } return arena_ralloc(tsdn, arena, ptr, oldsize, size, alignment, zero, - tcache, hook_args); + slab, tcache, hook_args); +} + +JEMALLOC_ALWAYS_INLINE void * +iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment, + size_t usize, bool zero, tcache_t *tcache, arena_t *arena, + hook_ralloc_args_t *hook_args) { + bool slab = sz_can_use_slab(usize); + return iralloct_explicit_slab(tsdn, ptr, oldsize, size, alignment, zero, + slab, tcache, arena, hook_args); } JEMALLOC_ALWAYS_INLINE void * iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, - bool zero, hook_ralloc_args_t *hook_args) { - return iralloct(tsd_tsdn(tsd), ptr, oldsize, size, alignment, zero, - tcache_get(tsd), NULL, hook_args); + size_t usize, bool zero, hook_ralloc_args_t *hook_args) { + return iralloct(tsd_tsdn(tsd), ptr, oldsize, size, alignment, usize, + zero, tcache_get(tsd), NULL, hook_args); } JEMALLOC_ALWAYS_INLINE bool @@ -207,29 +257,27 @@ ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero, size_t *newsize) { assert(ptr != NULL); assert(size != 0); - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); - if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) - != 0) { + if (alignment != 0 + && ((uintptr_t)ptr & ((uintptr_t)alignment - 1)) != 0) { /* Existing object alignment is inadequate. */ *newsize = oldsize; return true; } - return arena_ralloc_no_move(tsdn, ptr, oldsize, size, extra, zero, - newsize); + return arena_ralloc_no_move( + tsdn, ptr, oldsize, size, extra, zero, newsize); } JEMALLOC_ALWAYS_INLINE void -fastpath_success_finish(tsd_t *tsd, uint64_t allocated_after, - cache_bin_t *bin, void *ret) { +fastpath_success_finish( + tsd_t *tsd, uint64_t allocated_after, cache_bin_t *bin, void *ret) { thread_allocated_set(tsd, allocated_after); if (config_stats) { bin->tstats.nrequests++; } - - LOG("core.malloc.exit", "result: %p", ret); } JEMALLOC_ALWAYS_INLINE bool @@ -256,7 +304,6 @@ malloc_initialized(void) { */ JEMALLOC_ALWAYS_INLINE void * imalloc_fastpath(size_t size, void *(fallback_alloc)(size_t)) { - LOG("core.malloc.entry", "size: %zu", size); if (tsd_get_allocates() && unlikely(!malloc_initialized())) { return fallback_alloc(size); } @@ -284,8 +331,8 @@ imalloc_fastpath(size_t size, void *(fallback_alloc)(size_t)) { sz_size2index_usize_fastpath(size, &ind, &usize); /* Fast path relies on size being a bin. */ assert(ind < SC_NBINS); - assert((SC_LOOKUP_MAXCLASS < SC_SMALL_MAXCLASS) && - (size <= SC_SMALL_MAXCLASS)); + assert((SC_LOOKUP_MAXCLASS < SC_SMALL_MAXCLASS) + && (size <= SC_SMALL_MAXCLASS)); uint64_t allocated, threshold; te_malloc_fastpath_ctx(tsd, &allocated, &threshold); @@ -314,7 +361,9 @@ imalloc_fastpath(size_t size, void *(fallback_alloc)(size_t)) { tcache_t *tcache = tsd_tcachep_get(tsd); assert(tcache == tcache_get(tsd)); cache_bin_t *bin = &tcache->bins[ind]; - bool tcache_success; + /* Suppress spurious warning from static analysis */ + assert(bin != NULL); + bool tcache_success; void *ret; /* @@ -337,4 +386,215 @@ imalloc_fastpath(size_t size, void *(fallback_alloc)(size_t)) { return fallback_alloc(size); } +JEMALLOC_ALWAYS_INLINE tcache_t * +tcache_get_from_ind(tsd_t *tsd, unsigned tcache_ind, bool slow, bool is_alloc) { + tcache_t *tcache; + if (tcache_ind == TCACHE_IND_AUTOMATIC) { + if (likely(!slow)) { + /* Getting tcache ptr unconditionally. */ + tcache = tsd_tcachep_get(tsd); + assert(tcache == tcache_get(tsd)); + } else if (is_alloc + || likely(tsd_reentrancy_level_get(tsd) == 0)) { + tcache = tcache_get(tsd); + } else { + tcache = NULL; + } + } else { + /* + * Should not specify tcache on deallocation path when being + * reentrant. + */ + assert(is_alloc || tsd_reentrancy_level_get(tsd) == 0 + || tsd_state_nocleanup(tsd)); + if (tcache_ind == TCACHE_IND_NONE) { + tcache = NULL; + } else { + tcache = tcaches_get(tsd, tcache_ind); + } + } + return tcache; +} + +JEMALLOC_ALWAYS_INLINE bool +maybe_check_alloc_ctx(tsd_t *tsd, void *ptr, emap_alloc_ctx_t *alloc_ctx) { + if (config_opt_size_checks) { + emap_alloc_ctx_t dbg_ctx; + emap_alloc_ctx_lookup( + tsd_tsdn(tsd), &arena_emap_global, ptr, &dbg_ctx); + if (alloc_ctx->szind != dbg_ctx.szind) { + safety_check_fail_sized_dealloc( + /* current_dealloc */ true, ptr, + /* true_size */ emap_alloc_ctx_usize_get(&dbg_ctx), + /* input_size */ + emap_alloc_ctx_usize_get(alloc_ctx)); + return true; + } + if (alloc_ctx->slab != dbg_ctx.slab) { + safety_check_fail( + "Internal heap corruption detected: " + "mismatch in slab bit"); + return true; + } + } + return false; +} + +JEMALLOC_ALWAYS_INLINE bool +prof_sample_aligned(const void *ptr) { + return ((uintptr_t)ptr & PROF_SAMPLE_ALIGNMENT_MASK) == 0; +} + +JEMALLOC_ALWAYS_INLINE bool +free_fastpath_nonfast_aligned(void *ptr, bool check_prof) { + /* + * free_fastpath do not handle two uncommon cases: 1) sampled profiled + * objects and 2) sampled junk & stash for use-after-free detection. + * Both have special alignments which are used to escape the fastpath. + * + * prof_sample is page-aligned, which covers the UAF check when both + * are enabled (the assertion below). Avoiding redundant checks since + * this is on the fastpath -- at most one runtime branch from this. + */ + if (config_debug && cache_bin_nonfast_aligned(ptr)) { + assert(prof_sample_aligned(ptr)); + } + + if (config_prof && check_prof) { + /* When prof is enabled, the prof_sample alignment is enough. */ + if (prof_sample_aligned(ptr)) { + return true; + } else { + return false; + } + } + + if (config_uaf_detection) { + if (cache_bin_nonfast_aligned(ptr)) { + return true; + } else { + return false; + } + } + + return false; +} + +/* Returns whether or not the free attempt was successful. */ +JEMALLOC_ALWAYS_INLINE +bool +free_fastpath(void *ptr, size_t size, bool size_hint) { + tsd_t *tsd = tsd_get(false); + /* The branch gets optimized away unless tsd_get_allocates(). */ + if (unlikely(tsd == NULL)) { + return false; + } + /* + * The tsd_fast() / initialized checks are folded into the branch + * testing (deallocated_after >= threshold) later in this function. + * The threshold will be set to 0 when !tsd_fast. + */ + assert(tsd_fast(tsd) + || *tsd_thread_deallocated_next_event_fastp_get_unsafe(tsd) == 0); + + emap_alloc_ctx_t alloc_ctx JEMALLOC_CC_SILENCE_INIT({0, 0, false}); + size_t usize; + if (!size_hint) { + bool err = emap_alloc_ctx_try_lookup_fast( + tsd, &arena_emap_global, ptr, &alloc_ctx); + + /* Note: profiled objects will have alloc_ctx.slab set */ + if (unlikely(err || !alloc_ctx.slab + || free_fastpath_nonfast_aligned(ptr, + /* check_prof */ false))) { + return false; + } + assert(alloc_ctx.szind != SC_NSIZES); + usize = sz_index2size(alloc_ctx.szind); + } else { + /* + * Check for both sizes that are too large, and for sampled / + * special aligned objects. The alignment check will also check + * for null ptr. + */ + if (unlikely(size > SC_LOOKUP_MAXCLASS + || free_fastpath_nonfast_aligned(ptr, + /* check_prof */ true))) { + return false; + } + sz_size2index_usize_fastpath(size, &alloc_ctx.szind, &usize); + /* Max lookup class must be small. */ + assert(alloc_ctx.szind < SC_NBINS); + /* This is a dead store, except when opt size checking is on. */ + alloc_ctx.slab = true; + } + /* + * Currently the fastpath only handles small sizes. The branch on + * SC_LOOKUP_MAXCLASS makes sure of it. This lets us avoid checking + * tcache szind upper limit (i.e. tcache_max) as well. + */ + assert(alloc_ctx.slab); + + uint64_t deallocated, threshold; + te_free_fastpath_ctx(tsd, &deallocated, &threshold); + + uint64_t deallocated_after = deallocated + usize; + /* + * Check for events and tsd non-nominal (fast_threshold will be set to + * 0) in a single branch. Note that this handles the uninitialized case + * as well (TSD init will be triggered on the non-fastpath). Therefore + * anything depends on a functional TSD (e.g. the alloc_ctx sanity check + * below) needs to be after this branch. + */ + if (unlikely(deallocated_after >= threshold)) { + return false; + } + assert(tsd_fast(tsd)); + bool fail = maybe_check_alloc_ctx(tsd, ptr, &alloc_ctx); + if (fail) { + /* See the comment in isfree. */ + return true; + } + + tcache_t *tcache = tcache_get_from_ind(tsd, TCACHE_IND_AUTOMATIC, + /* slow */ false, /* is_alloc */ false); + cache_bin_t *bin = &tcache->bins[alloc_ctx.szind]; + + /* + * If junking were enabled, this is where we would do it. It's not + * though, since we ensured above that we're on the fast path. Assert + * that to double-check. + */ + assert(!opt_junk_free); + + if (!cache_bin_dalloc_easy(bin, ptr)) { + return false; + } + + *tsd_thread_deallocatedp_get(tsd) = deallocated_after; + + return true; +} + +JEMALLOC_ALWAYS_INLINE void JEMALLOC_NOTHROW +je_sdallocx_noflags(void *ptr, size_t size) { + if (!free_fastpath(ptr, size, true)) { + sdallocx_default(ptr, size, 0); + } +} + +JEMALLOC_ALWAYS_INLINE void JEMALLOC_NOTHROW +je_sdallocx_impl(void *ptr, size_t size, int flags) { + if (flags != 0 || !free_fastpath(ptr, size, true)) { + sdallocx_default(ptr, size, flags); + } +} + +JEMALLOC_ALWAYS_INLINE void JEMALLOC_NOTHROW +je_free_impl(void *ptr) { + if (!free_fastpath(ptr, 0, false)) { + free_default(ptr); + } +} + #endif /* JEMALLOC_INTERNAL_INLINES_C_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h index e97b5f907..eb1ca1190 100644 --- a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h +++ b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h @@ -2,43 +2,46 @@ #define JEMALLOC_INTERNAL_MACROS_H #ifdef JEMALLOC_DEBUG -# define JEMALLOC_ALWAYS_INLINE static inline +# define JEMALLOC_ALWAYS_INLINE static inline #else -# ifdef _MSC_VER -# define JEMALLOC_ALWAYS_INLINE static __forceinline -# else -# define JEMALLOC_ALWAYS_INLINE JEMALLOC_ATTR(always_inline) static inline -# endif +# ifdef _MSC_VER +# define JEMALLOC_ALWAYS_INLINE static __forceinline +# else +# define JEMALLOC_ALWAYS_INLINE \ + JEMALLOC_ATTR(always_inline) static inline +# endif #endif #ifdef _MSC_VER -# define inline _inline +# define inline _inline #endif #define UNUSED JEMALLOC_ATTR(unused) -#define ZU(z) ((size_t)z) -#define ZD(z) ((ssize_t)z) -#define QU(q) ((uint64_t)q) -#define QD(q) ((int64_t)q) +#define ZU(z) ((size_t)z) +#define ZD(z) ((ssize_t)z) +#define QU(q) ((uint64_t)q) +#define QD(q) ((int64_t)q) -#define KZU(z) ZU(z##ULL) -#define KZD(z) ZD(z##LL) -#define KQU(q) QU(q##ULL) -#define KQD(q) QI(q##LL) +#define KZU(z) ZU(z##ULL) +#define KZD(z) ZD(z##LL) +#define KQU(q) QU(q##ULL) +#define KQD(q) QI(q##LL) #ifndef __DECONST -# define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var)) +# define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var)) #endif #if !defined(JEMALLOC_HAS_RESTRICT) || defined(__cplusplus) -# define restrict +# define restrict #endif /* Various function pointers are static and immutable except during testing. */ #ifdef JEMALLOC_JET -# define JET_MUTABLE +# define JET_MUTABLE +# define JET_EXTERN extern #else -# define JET_MUTABLE const +# define JET_MUTABLE const +# define JET_EXTERN static #endif #define JEMALLOC_VA_ARGS_HEAD(head, ...) head @@ -46,62 +49,94 @@ /* Diagnostic suppression macros */ #if defined(_MSC_VER) && !defined(__clang__) -# define JEMALLOC_DIAGNOSTIC_PUSH __pragma(warning(push)) -# define JEMALLOC_DIAGNOSTIC_POP __pragma(warning(pop)) -# define JEMALLOC_DIAGNOSTIC_IGNORE(W) __pragma(warning(disable:W)) -# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS -# define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS -# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN -# define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS +# define JEMALLOC_DIAGNOSTIC_PUSH __pragma(warning(push)) +# define JEMALLOC_DIAGNOSTIC_POP __pragma(warning(pop)) +# define JEMALLOC_DIAGNOSTIC_IGNORE(W) __pragma(warning(disable : W)) +# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS +# define JEMALLOC_DIAGNOSTIC_IGNORE_FRAME_ADDRESS +# define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS +# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN +# define JEMALLOC_DIAGNOSTIC_IGNORE_DEPRECATED +# define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS /* #pragma GCC diagnostic first appeared in gcc 4.6. */ -#elif (defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && \ - (__GNUC_MINOR__ > 5)))) || defined(__clang__) +#elif (defined(__GNUC__) \ + && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))) \ + || defined(__clang__) /* * The JEMALLOC_PRAGMA__ macro is an implementation detail of the GCC and Clang * diagnostic suppression macros and should not be used anywhere else. */ -# define JEMALLOC_PRAGMA__(X) _Pragma(#X) -# define JEMALLOC_DIAGNOSTIC_PUSH JEMALLOC_PRAGMA__(GCC diagnostic push) -# define JEMALLOC_DIAGNOSTIC_POP JEMALLOC_PRAGMA__(GCC diagnostic pop) -# define JEMALLOC_DIAGNOSTIC_IGNORE(W) \ - JEMALLOC_PRAGMA__(GCC diagnostic ignored W) +# define JEMALLOC_PRAGMA__(X) _Pragma(#X) +# define JEMALLOC_DIAGNOSTIC_PUSH JEMALLOC_PRAGMA__(GCC diagnostic push) +# define JEMALLOC_DIAGNOSTIC_POP JEMALLOC_PRAGMA__(GCC diagnostic pop) +# define JEMALLOC_DIAGNOSTIC_IGNORE(W) \ + JEMALLOC_PRAGMA__(GCC diagnostic ignored W) /* * The -Wmissing-field-initializers warning is buggy in GCC versions < 5.1 and * all clang versions up to version 7 (currently trunk, unreleased). This macro * suppresses the warning for the affected compiler versions only. */ -# if ((defined(__GNUC__) && !defined(__clang__)) && (__GNUC__ < 5)) || \ - defined(__clang__) -# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS \ - JEMALLOC_DIAGNOSTIC_IGNORE("-Wmissing-field-initializers") -# else -# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS -# endif +# if ((defined(__GNUC__) && !defined(__clang__)) && (__GNUC__ < 5)) \ + || defined(__clang__) +# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS \ + JEMALLOC_DIAGNOSTIC_IGNORE( \ + "-Wmissing-field-initializers") +# else +# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS +# endif -# define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS \ - JEMALLOC_DIAGNOSTIC_IGNORE("-Wtype-limits") -# define JEMALLOC_DIAGNOSTIC_IGNORE_UNUSED_PARAMETER \ - JEMALLOC_DIAGNOSTIC_IGNORE("-Wunused-parameter") -# if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 7) -# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN \ - JEMALLOC_DIAGNOSTIC_IGNORE("-Walloc-size-larger-than=") -# else -# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN -# endif -# define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS \ - JEMALLOC_DIAGNOSTIC_PUSH \ - JEMALLOC_DIAGNOSTIC_IGNORE_UNUSED_PARAMETER +# define JEMALLOC_DIAGNOSTIC_IGNORE_FRAME_ADDRESS \ + JEMALLOC_DIAGNOSTIC_IGNORE("-Wframe-address") +# define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS \ + JEMALLOC_DIAGNOSTIC_IGNORE("-Wtype-limits") +# define JEMALLOC_DIAGNOSTIC_IGNORE_UNUSED_PARAMETER \ + JEMALLOC_DIAGNOSTIC_IGNORE("-Wunused-parameter") +# if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 7) +# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN \ + JEMALLOC_DIAGNOSTIC_IGNORE("-Walloc-size-larger-than=") +# else +# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN +# endif +# ifdef JEMALLOC_HAVE_ATTR_DEPRECATED +# define JEMALLOC_DIAGNOSTIC_IGNORE_DEPRECATED \ + JEMALLOC_DIAGNOSTIC_IGNORE("-Wdeprecated-declarations") +# else +# define JEMALLOC_DIAGNOSTIC_IGNORE_DEPRECATED +# endif +# define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS \ + JEMALLOC_DIAGNOSTIC_PUSH \ + JEMALLOC_DIAGNOSTIC_IGNORE_UNUSED_PARAMETER #else -# define JEMALLOC_DIAGNOSTIC_PUSH -# define JEMALLOC_DIAGNOSTIC_POP -# define JEMALLOC_DIAGNOSTIC_IGNORE(W) -# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS -# define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS -# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN -# define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS +# define JEMALLOC_DIAGNOSTIC_PUSH +# define JEMALLOC_DIAGNOSTIC_POP +# define JEMALLOC_DIAGNOSTIC_IGNORE(W) +# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS +# define JEMALLOC_DIAGNOSTIC_IGNORE_FRAME_ADDRESS +# define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS +# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN +# define JEMALLOC_DIAGNOSTIC_IGNORE_DEPRECATED +# define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS #endif +#ifdef __clang_analyzer__ +# define JEMALLOC_CLANG_ANALYZER +#endif + +#ifdef JEMALLOC_CLANG_ANALYZER +# define JEMALLOC_CLANG_ANALYZER_SUPPRESS __attribute__((suppress)) +# define JEMALLOC_CLANG_ANALYZER_SILENCE_INIT(v) = v +#else +# define JEMALLOC_CLANG_ANALYZER_SUPPRESS +# define JEMALLOC_CLANG_ANALYZER_SILENCE_INIT(v) +#endif + +#define JEMALLOC_SUPPRESS_WARN_ON_USAGE(...) \ + JEMALLOC_DIAGNOSTIC_PUSH \ + JEMALLOC_DIAGNOSTIC_IGNORE_DEPRECATED \ + __VA_ARGS__ \ + JEMALLOC_DIAGNOSTIC_POP + /* * Disables spurious diagnostics for all headers. Since these headers are not * included by users directly, it does not affect their diagnostic settings. diff --git a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_overrides.h b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_overrides.h new file mode 100644 index 000000000..bf74a6121 --- /dev/null +++ b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_overrides.h @@ -0,0 +1,22 @@ +#ifndef JEMALLOC_INTERNAL_OVERRIDES_H +#define JEMALLOC_INTERNAL_OVERRIDES_H + +/* + * Under normal circumstances this header serves no purpose, as these settings + * can be customized via the corresponding autoconf options at configure-time. + * Overriding in this fashion is useful when the header files generated by + * autoconf are used as input for another build system. + */ + +#ifdef JEMALLOC_OVERRIDE_LG_PAGE +# undef LG_PAGE +# define LG_PAGE JEMALLOC_OVERRIDE_LG_PAGE +#endif + +#ifdef JEMALLOC_OVERRIDE_JEMALLOC_CONFIG_MALLOC_CONF +# undef JEMALLOC_CONFIG_MALLOC_CONF +# define JEMALLOC_CONFIG_MALLOC_CONF \ + JEMALLOC_OVERRIDE_JEMALLOC_CONFIG_MALLOC_CONF +#endif + +#endif /* JEMALLOC_INTERNAL_OVERRIDES_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_types.h b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_types.h index 62c2b59c7..0ade5461b 100644 --- a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_types.h +++ b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_types.h @@ -18,13 +18,13 @@ enum zero_realloc_action_e { typedef enum zero_realloc_action_e zero_realloc_action_t; /* Signature of write callback. */ -typedef void (write_cb_t)(void *, const char *); +typedef void(write_cb_t)(void *, const char *); enum malloc_init_e { - malloc_init_uninitialized = 3, - malloc_init_a0_initialized = 2, - malloc_init_recursible = 1, - malloc_init_initialized = 0 /* Common case --> jnz. */ + malloc_init_uninitialized = 3, + malloc_init_a0_initialized = 2, + malloc_init_recursible = 1, + malloc_init_initialized = 0 /* Common case --> jnz. */ }; typedef enum malloc_init_e malloc_init_t; @@ -39,48 +39,46 @@ typedef enum malloc_init_e malloc_init_t; * * aaaaaaaa aaaatttt tttttttt 0znnnnnn */ -#define MALLOCX_ARENA_BITS 12 -#define MALLOCX_TCACHE_BITS 12 -#define MALLOCX_LG_ALIGN_BITS 6 -#define MALLOCX_ARENA_SHIFT 20 -#define MALLOCX_TCACHE_SHIFT 8 -#define MALLOCX_ARENA_MASK \ - (((1 << MALLOCX_ARENA_BITS) - 1) << MALLOCX_ARENA_SHIFT) +#define MALLOCX_ARENA_BITS 12 +#define MALLOCX_TCACHE_BITS 12 +#define MALLOCX_LG_ALIGN_BITS 6 +#define MALLOCX_ARENA_SHIFT 20 +#define MALLOCX_TCACHE_SHIFT 8 +#define MALLOCX_ARENA_MASK \ + ((unsigned)(((1U << MALLOCX_ARENA_BITS) - 1) << MALLOCX_ARENA_SHIFT)) /* NB: Arena index bias decreases the maximum number of arenas by 1. */ -#define MALLOCX_ARENA_LIMIT ((1 << MALLOCX_ARENA_BITS) - 1) -#define MALLOCX_TCACHE_MASK \ - (((1 << MALLOCX_TCACHE_BITS) - 1) << MALLOCX_TCACHE_SHIFT) -#define MALLOCX_TCACHE_MAX ((1 << MALLOCX_TCACHE_BITS) - 3) -#define MALLOCX_LG_ALIGN_MASK ((1 << MALLOCX_LG_ALIGN_BITS) - 1) +#define MALLOCX_ARENA_LIMIT ((unsigned)((1U << MALLOCX_ARENA_BITS) - 1)) +#define MALLOCX_TCACHE_MASK \ + ((unsigned)(((1U << MALLOCX_TCACHE_BITS) - 1) << MALLOCX_TCACHE_SHIFT)) +#define MALLOCX_TCACHE_MAX ((unsigned)((1U << MALLOCX_TCACHE_BITS) - 3)) +#define MALLOCX_LG_ALIGN_MASK ((1 << MALLOCX_LG_ALIGN_BITS) - 1) /* Use MALLOCX_ALIGN_GET() if alignment may not be specified in flags. */ -#define MALLOCX_ALIGN_GET_SPECIFIED(flags) \ - (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)) -#define MALLOCX_ALIGN_GET(flags) \ - (MALLOCX_ALIGN_GET_SPECIFIED(flags) & (SIZE_T_MAX-1)) -#define MALLOCX_ZERO_GET(flags) \ - ((bool)(flags & MALLOCX_ZERO)) +#define MALLOCX_ALIGN_GET_SPECIFIED(flags) \ + (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)) +#define MALLOCX_ALIGN_GET(flags) \ + (MALLOCX_ALIGN_GET_SPECIFIED(flags) & (SIZE_T_MAX - 1)) +#define MALLOCX_ZERO_GET(flags) ((bool)(flags & MALLOCX_ZERO)) -#define MALLOCX_TCACHE_GET(flags) \ - (((unsigned)((flags & MALLOCX_TCACHE_MASK) >> MALLOCX_TCACHE_SHIFT)) - 2) -#define MALLOCX_ARENA_GET(flags) \ - (((unsigned)(((unsigned)flags) >> MALLOCX_ARENA_SHIFT)) - 1) +#define MALLOCX_TCACHE_GET(flags) \ + (((unsigned)((flags & MALLOCX_TCACHE_MASK) >> MALLOCX_TCACHE_SHIFT)) \ + - 2) +#define MALLOCX_ARENA_GET(flags) \ + (((unsigned)(((unsigned)flags) >> MALLOCX_ARENA_SHIFT)) - 1) /* Smallest size class to support. */ -#define TINY_MIN (1U << LG_TINY_MIN) +#define TINY_MIN (1U << LG_TINY_MIN) -#define LONG ((size_t)(1U << LG_SIZEOF_LONG)) -#define LONG_MASK (LONG - 1) +#define LONG ((size_t)(1U << LG_SIZEOF_LONG)) +#define LONG_MASK (LONG - 1) /* Return the smallest long multiple that is >= a. */ -#define LONG_CEILING(a) \ - (((a) + LONG_MASK) & ~LONG_MASK) +#define LONG_CEILING(a) (((a) + LONG_MASK) & ~LONG_MASK) -#define SIZEOF_PTR (1U << LG_SIZEOF_PTR) -#define PTR_MASK (SIZEOF_PTR - 1) +#define SIZEOF_PTR (1U << LG_SIZEOF_PTR) +#define PTR_MASK (SIZEOF_PTR - 1) /* Return the smallest (void *) multiple that is >= a. */ -#define PTR_CEILING(a) \ - (((a) + PTR_MASK) & ~PTR_MASK) +#define PTR_CEILING(a) (((a) + PTR_MASK) & ~PTR_MASK) /* * Maximum size of L1 cache line. This is used to avoid cache line aliasing. @@ -89,42 +87,62 @@ typedef enum malloc_init_e malloc_init_t; * CACHELINE cannot be based on LG_CACHELINE because __declspec(align()) can * only handle raw constants. */ -#define LG_CACHELINE 6 -#define CACHELINE 64 -#define CACHELINE_MASK (CACHELINE - 1) +#define LG_CACHELINE 6 +#define CACHELINE 64 +#define CACHELINE_MASK (CACHELINE - 1) /* Return the smallest cacheline multiple that is >= s. */ -#define CACHELINE_CEILING(s) \ - (((s) + CACHELINE_MASK) & ~CACHELINE_MASK) +#define CACHELINE_CEILING(s) (((s) + CACHELINE_MASK) & ~CACHELINE_MASK) /* Return the nearest aligned address at or below a. */ -#define ALIGNMENT_ADDR2BASE(a, alignment) \ - ((void *)((uintptr_t)(a) & ((~(alignment)) + 1))) +#define ALIGNMENT_ADDR2BASE(a, alignment) \ + ((void *)(((byte_t *)(a)) \ + - (((uintptr_t)(a)) - ((uintptr_t)(a) & ((~(alignment)) + 1))))) /* Return the offset between a and the nearest aligned address at or below a. */ -#define ALIGNMENT_ADDR2OFFSET(a, alignment) \ +#define ALIGNMENT_ADDR2OFFSET(a, alignment) \ ((size_t)((uintptr_t)(a) & (alignment - 1))) /* Return the smallest alignment multiple that is >= s. */ -#define ALIGNMENT_CEILING(s, alignment) \ +#define ALIGNMENT_CEILING(s, alignment) \ (((s) + (alignment - 1)) & ((~(alignment)) + 1)) +/* + * Return the nearest aligned address at or above a. + * + * While at first glance this would appear to be merely a more complicated + * way to perform the same computation as `ALIGNMENT_CEILING`, + * this has the important additional property of not concealing pointer + * provenance from the compiler. See the block-comment on the + * definition of `byte_t` for more details. + */ +#define ALIGNMENT_ADDR2CEILING(a, alignment) \ + ((void *)(((byte_t *)(a)) \ + + (((((uintptr_t)(a)) + (alignment - 1)) & ((~(alignment)) + 1)) \ + - ((uintptr_t)(a))))) + /* Declare a variable-length array. */ -#if __STDC_VERSION__ < 199901L -# ifdef _MSC_VER -# include -# define alloca _alloca -# else -# ifdef JEMALLOC_HAS_ALLOCA_H -# include -# else -# include -# endif -# endif -# define VARIABLE_ARRAY(type, name, count) \ - type *name = alloca(sizeof(type) * (count)) +#if __STDC_VERSION__ < 199901L || defined(__STDC_NO_VLA__) +# ifdef _MSC_VER +# include +# define alloca _alloca +# else +# ifdef JEMALLOC_HAS_ALLOCA_H +# include +# else +# include +# endif +# endif +# define VARIABLE_ARRAY_UNSAFE(type, name, count) \ + type *name = alloca(sizeof(type) * (count)) #else -# define VARIABLE_ARRAY(type, name, count) type name[(count)] +# define VARIABLE_ARRAY_UNSAFE(type, name, count) type name[(count)] #endif +#define VARIABLE_ARRAY_SIZE_MAX 2048 +#define VARIABLE_ARRAY(type, name, count) \ + assert(sizeof(type) * (count) <= VARIABLE_ARRAY_SIZE_MAX); \ + VARIABLE_ARRAY_UNSAFE(type, name, count) + +#define CALLOC_MADVISE_THRESHOLD_DEFAULT (((size_t)1) << 23) /* 8 MB */ #endif /* JEMALLOC_INTERNAL_TYPES_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/jemalloc_preamble.h b/dep/jemalloc/include/jemalloc/internal/jemalloc_preamble.h index 3be27ed5f..9c3382d05 100644 --- a/dep/jemalloc/include/jemalloc/internal/jemalloc_preamble.h +++ b/dep/jemalloc/include/jemalloc/internal/jemalloc_preamble.h @@ -57,6 +57,15 @@ # define JEMALLOC_MADV_FREE 8 #endif +/* + * Can be defined at compile time, in cases, when it is known + * madvise(..., MADV_COLLAPSE) feature is supported, but MADV_COLLAPSE + * constant is not defined. + */ +#ifdef JEMALLOC_DEFINE_MADVISE_COLLAPSE +# define JEMALLOC_MADV_COLLAPSE 25 +#endif + static const bool config_debug = #ifdef JEMALLOC_DEBUG true @@ -78,6 +87,13 @@ static const bool have_madvise_huge = false #endif ; +static const bool have_process_madvise = +#ifdef JEMALLOC_HAVE_PROCESS_MADVISE + true +#else + false +#endif + ; static const bool config_fill = #ifdef JEMALLOC_FILL true @@ -114,6 +130,13 @@ static const bool config_prof_libunwind = false #endif ; +static const bool config_prof_frameptr = +#ifdef JEMALLOC_PROF_FRAME_POINTER + true +#else + false +#endif + ; static const bool maps_coalesce = #ifdef JEMALLOC_MAPS_COALESCE true @@ -215,7 +238,7 @@ static const bool config_enable_cxx = #endif ; -#if defined(_WIN32) || defined(JEMALLOC_HAVE_SCHED_GETCPU) +#if defined(_WIN32) || defined(__APPLE__) || defined(JEMALLOC_HAVE_SCHED_GETCPU) /* Currently percpu_arena depends on sched_getcpu. */ #define JEMALLOC_PERCPU_ARENA #endif diff --git a/dep/jemalloc/include/jemalloc/internal/jemalloc_probe.h b/dep/jemalloc/include/jemalloc/internal/jemalloc_probe.h new file mode 100644 index 000000000..8ef3105d1 --- /dev/null +++ b/dep/jemalloc/include/jemalloc/internal/jemalloc_probe.h @@ -0,0 +1,49 @@ +#ifndef JEMALLOC_INTERNAL_JEMALLOC_PROBE_H +#define JEMALLOC_INTERNAL_JEMALLOC_PROBE_H + +#include + +#ifdef JEMALLOC_EXPERIMENTAL_USDT_STAP +#include +#elif defined(JEMALLOC_EXPERIMENTAL_USDT_CUSTOM) +#include +#elif defined(_MSC_VER) +#define JE_USDT(name, N, ...) /* Nothing */ +#else /* no USDT, just check the args */ + +#define JE_USDT(name, N, ...) _JE_USDT_CHECK_ARG##N(__VA_ARGS__) + +#define _JE_USDT_CHECK_ARG1(a) \ + do { \ + (void)(a); \ + } while (0) +#define _JE_USDT_CHECK_ARG2(a, b) \ + do { \ + (void)(a); \ + (void)(b); \ + } while (0) +#define _JE_USDT_CHECK_ARG3(a, b, c) \ + do { \ + (void)(a); \ + (void)(b); \ + (void)(c); \ + } while (0) +#define _JE_USDT_CHECK_ARG4(a, b, c, d) \ + do { \ + (void)(a); \ + (void)(b); \ + (void)(c); \ + (void)(d); \ + } while (0) +#define _JE_USDT_CHECK_ARG5(a, b, c, d, e) \ + do { \ + (void)(a); \ + (void)(b); \ + (void)(c); \ + (void)(d); \ + (void)(e); \ + } while (0) + +#endif /* JEMALLOC_EXPERIMENTAL_USDT_* */ + +#endif /* JEMALLOC_INTERNAL_JEMALLOC_PROBE_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/jemalloc_probe_custom.h b/dep/jemalloc/include/jemalloc/internal/jemalloc_probe_custom.h new file mode 100644 index 000000000..3c22749f1 --- /dev/null +++ b/dep/jemalloc/include/jemalloc/internal/jemalloc_probe_custom.h @@ -0,0 +1,148 @@ +#ifndef JEMALLOC_INTERNAL_JEMALLOC_PROBE_CUSTOM_H +#define JEMALLOC_INTERNAL_JEMALLOC_PROBE_CUSTOM_H + +/* clang-format off */ + +/* + * This section is based on sys/sdt.h and + * https://sourceware.org/systemtap/wiki/UserSpaceProbeImplementation + */ + +/* Emit NOP for the probe. */ +#if (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__) || \ + defined(__arm__)) && defined(__linux__) +#define JE_SDT_NOP nop +#else +#error "Architecture not supported" +#endif + +/* Assembly macros */ +#define JE_SDT_S(x) #x + +#define JE_SDT_ASM_1(x) JE_SDT_S(x) "\n" + +#define JE_SDT_ASM_2(x, y) \ + JE_SDT_S(x) "," JE_SDT_S(y) "\n" + +#define JE_SDT_ASM_3(x, y, z) \ + JE_SDT_S(x) "," JE_SDT_S(y) "," JE_SDT_S(z) "\n" + +#define JE_SDT_ASM_3(x, y, z) \ + JE_SDT_S(x) "," JE_SDT_S(y) "," JE_SDT_S(z) "\n" + +#define JE_SDT_ASM_4(x, y, z, p) \ + JE_SDT_S(x) "," JE_SDT_S(y) "," JE_SDT_S(z) "," JE_SDT_S(p) "\n" + +#define JE_SDT_ASM_5(x, y, z, p, q) \ + JE_SDT_S(x) "," JE_SDT_S(y) "," JE_SDT_S(z) "," JE_SDT_S(p) "," \ + JE_SDT_S(q) "\n" + +/* Arg size */ +#ifdef __LP64__ +#define JE_SDT_ASM_ADDR .8byte +#else +#define JE_SDT_ASM_ADDR .4byte +#endif + +#define JE_SDT_NOTE_NAME "stapsdt" +#define JE_SDT_NOTE_TYPE 3 + +#define JE_SDT_SEMAPHORE_NONE(provider, name) \ + JE_SDT_ASM_1(JE_SDT_ASM_ADDR 0) /* No Semaphore support */ +#define JE_SDT_SEMAPHORE_OPERAND(provider, name) \ + [__sdt_semaphore] "ip" (0) /* No Semaphore */ + +#define JE_SDT_ASM_STRING(x) JE_SDT_ASM_1(.asciz JE_SDT_S(x)) + +#define JE_SDT_NOTE(provider, name, arg_template) \ + JE_SDT_ASM_1(990: JE_SDT_NOP) \ + JE_SDT_ASM_3( .pushsection .note.stapsdt,"?","note") \ + JE_SDT_ASM_1( .balign 4) \ + JE_SDT_ASM_3( .4byte 992f-991f, 994f-993f, JE_SDT_NOTE_TYPE) \ + JE_SDT_ASM_1(991: .asciz JE_SDT_NOTE_NAME) \ + JE_SDT_ASM_1(992: .balign 4) \ + JE_SDT_ASM_1(993: JE_SDT_ASM_ADDR 990b) \ + JE_SDT_ASM_1( JE_SDT_ASM_ADDR _.stapsdt.base) \ + JE_SDT_SEMAPHORE_NONE(provider, name) \ + JE_SDT_ASM_STRING(provider) \ + JE_SDT_ASM_STRING(name) \ + JE_SDT_ASM_STRING(arg_template) \ + JE_SDT_ASM_1(994: .balign 4) \ + JE_SDT_ASM_1( .popsection) + +#define JE_SDT_BASE \ + JE_SDT_ASM_1( .ifndef _.stapsdt.base) \ + JE_SDT_ASM_5( .pushsection .stapsdt.base, "aG", "progbits", \ + .stapsdt.base,comdat) \ + JE_SDT_ASM_1( .weak _.stapsdt.base) \ + JE_SDT_ASM_1( .hidden _.stapsdt.base) \ + JE_SDT_ASM_1( _.stapsdt.base: .space 1) \ + JE_SDT_ASM_2( .size _.stapsdt.base, 1) \ + JE_SDT_ASM_1( .popsection) \ + JE_SDT_ASM_1( .endif) + + +/* + * Default constraint for probes arguments. + * See https://gcc.gnu.org/onlinedocs/gcc/Constraints.html + */ +#ifndef JE_SDT_ARG_CONSTRAINT +#define JE_SDT_ARG_CONSTRAINT "nor" +#endif + +#define JE_SDT_ARGARRAY(x) ((__builtin_classify_type(x) == 14) || \ + (__builtin_classify_type(x) == 5)) +#define JE_SDT_ARGSIZE(x) (JE_SDT_ARGARRAY(x) ? sizeof(void*) : sizeof(x)) + +/* + * Format of each probe argument as operand. Size tagged with JE_SDT_Sn, + * with "n" constraint. Value is tagged with JE_SDT_An with configured + * constraint. + */ +#define JE_SDT_ARG(n, x) \ + [JE_SDT_S##n] "n" ((size_t)JE_SDT_ARGSIZE(x)), \ + [JE_SDT_A##n] JE_SDT_ARG_CONSTRAINT(x) + +/* Templates to append arguments as operands. */ +#define JE_SDT_OPERANDS_0() [__sdt_dummy] "g" (0) +#define JE_SDT_OPERANDS_1(_1) JE_SDT_ARG(1, _1) +#define JE_SDT_OPERANDS_2(_1, _2) JE_SDT_OPERANDS_1(_1), JE_SDT_ARG(2, _2) +#define JE_SDT_OPERANDS_3(_1, _2, _3) JE_SDT_OPERANDS_2(_1, _2), JE_SDT_ARG(3, _3) +#define JE_SDT_OPERANDS_4(_1, _2, _3, _4) \ + JE_SDT_OPERANDS_3(_1, _2, _3), JE_SDT_ARG(4, _4) +#define JE_SDT_OPERANDS_5(_1, _2, _3, _4, _5) \ + JE_SDT_OPERANDS_4(_1, _2, _3, _4), JE_SDT_ARG(5, _5) +#define JE_SDT_OPERANDS_6(_1, _2, _3, _4, _5, _6) \ + JE_SDT_OPERANDS_5(_1, _2, _3, _4, _5), JE_SDT_ARG(6, _6) +#define JE_SDT_OPERANDS_7(_1, _2, _3, _4, _5, _6, _7) \ + JE_SDT_OPERANDS_6(_1, _2, _3, _4, _5, _6), JE_SDT_ARG(7, _7) + +/* Templates to reference the arguments from operands. */ +#define JE_SDT_ARGFMT(num) %n[JE_SDT_S##num]@%[JE_SDT_A##num] +#define JE_SDT_ARG_TEMPLATE_0 /* No args */ +#define JE_SDT_ARG_TEMPLATE_1 JE_SDT_ARGFMT(1) +#define JE_SDT_ARG_TEMPLATE_2 JE_SDT_ARG_TEMPLATE_1 JE_SDT_ARGFMT(2) +#define JE_SDT_ARG_TEMPLATE_3 JE_SDT_ARG_TEMPLATE_2 JE_SDT_ARGFMT(3) +#define JE_SDT_ARG_TEMPLATE_4 JE_SDT_ARG_TEMPLATE_3 JE_SDT_ARGFMT(4) +#define JE_SDT_ARG_TEMPLATE_5 JE_SDT_ARG_TEMPLATE_4 JE_SDT_ARGFMT(5) +#define JE_SDT_ARG_TEMPLATE_6 JE_SDT_ARG_TEMPLATE_5 JE_SDT_ARGFMT(6) +#define JE_SDT_ARG_TEMPLATE_7 JE_SDT_ARG_TEMPLATE_6 JE_SDT_ARGFMT(7) + +#define JE_SDT_PROBE( \ + provider, name, n, arglist) \ + do { \ + __asm__ __volatile__( \ + JE_SDT_NOTE(provider, name, \ + JE_SDT_ARG_TEMPLATE_##n) \ + :: JE_SDT_SEMAPHORE_OPERAND(provider, name), \ + JE_SDT_OPERANDS_##n arglist); \ + __asm__ __volatile__(JE_SDT_BASE); \ + } while (0) + +#define JE_USDT(name, N, ...) \ + JE_SDT_PROBE(jemalloc, name, N, (__VA_ARGS__)) + + +#endif /* JEMALLOC_INTERNAL_JEMALLOC_PROBE_CUSTOM_H */ + +/* clang-format on */ diff --git a/dep/jemalloc/include/jemalloc/internal/jemalloc_probe_stap.h b/dep/jemalloc/include/jemalloc/internal/jemalloc_probe_stap.h new file mode 100644 index 000000000..302b6cbb2 --- /dev/null +++ b/dep/jemalloc/include/jemalloc/internal/jemalloc_probe_stap.h @@ -0,0 +1,11 @@ +#ifndef JEMALLOC_INTERNAL_JEMALLOC_PROBE_STAP_H +#define JEMALLOC_INTERNAL_JEMALLOC_PROBE_STAP_H + +#include + +#define JE_USDT(name, N, ...) JE_USDT_PROBE_N(name, N, ##__VA_ARGS__) + +#define JE_USDT_PROBE_N(name, N, ...) \ + STAP_PROBE##N(jemalloc, name, ##__VA_ARGS__) + +#endif /* JEMALLOC_INTERNAL_JEMALLOC_PROBE_STAP_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/large_externs.h b/dep/jemalloc/include/jemalloc/internal/large_externs.h index 8e09122df..84c6c5d62 100644 --- a/dep/jemalloc/include/jemalloc/internal/large_externs.h +++ b/dep/jemalloc/include/jemalloc/internal/large_externs.h @@ -1,23 +1,24 @@ #ifndef JEMALLOC_INTERNAL_LARGE_EXTERNS_H #define JEMALLOC_INTERNAL_LARGE_EXTERNS_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/edata.h" #include "jemalloc/internal/hook.h" void *large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero); -void *large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, - bool zero); -bool large_ralloc_no_move(tsdn_t *tsdn, edata_t *edata, size_t usize_min, - size_t usize_max, bool zero); +void *large_palloc( + tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero); +bool large_ralloc_no_move(tsdn_t *tsdn, edata_t *edata, size_t usize_min, + size_t usize_max, bool zero); void *large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize, size_t alignment, bool zero, tcache_t *tcache, hook_ralloc_args_t *hook_args); -void large_dalloc_prep_locked(tsdn_t *tsdn, edata_t *edata); -void large_dalloc_finish(tsdn_t *tsdn, edata_t *edata); -void large_dalloc(tsdn_t *tsdn, edata_t *edata); -size_t large_salloc(tsdn_t *tsdn, const edata_t *edata); -void large_prof_info_get(tsd_t *tsd, edata_t *edata, prof_info_t *prof_info, - bool reset_recent); +void large_dalloc_prep_locked(tsdn_t *tsdn, edata_t *edata); +void large_dalloc_finish(tsdn_t *tsdn, edata_t *edata); +void large_dalloc(tsdn_t *tsdn, edata_t *edata); +void large_prof_info_get( + tsd_t *tsd, edata_t *edata, prof_info_t *prof_info, bool reset_recent); void large_prof_tctx_reset(edata_t *edata); void large_prof_info_set(edata_t *edata, prof_tctx_t *tctx, size_t size); diff --git a/dep/jemalloc/include/jemalloc/internal/lockedint.h b/dep/jemalloc/include/jemalloc/internal/lockedint.h index d020ebec1..46aba8ff3 100644 --- a/dep/jemalloc/include/jemalloc/internal/lockedint.h +++ b/dep/jemalloc/include/jemalloc/internal/lockedint.h @@ -1,6 +1,11 @@ #ifndef JEMALLOC_INTERNAL_LOCKEDINT_H #define JEMALLOC_INTERNAL_LOCKEDINT_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/tsd_types.h" + /* * In those architectures that support 64-bit atomics, we use atomic updates for * our 64-bit values. Otherwise, we use a plain uint64_t and synchronize @@ -25,33 +30,34 @@ struct locked_zu_s { }; #ifndef JEMALLOC_ATOMIC_U64 -# define LOCKEDINT_MTX_DECLARE(name) malloc_mutex_t name; -# define LOCKEDINT_MTX_INIT(mu, name, rank, rank_mode) \ - malloc_mutex_init(&(mu), name, rank, rank_mode) -# define LOCKEDINT_MTX(mtx) (&(mtx)) -# define LOCKEDINT_MTX_LOCK(tsdn, mu) malloc_mutex_lock(tsdn, &(mu)) -# define LOCKEDINT_MTX_UNLOCK(tsdn, mu) malloc_mutex_unlock(tsdn, &(mu)) -# define LOCKEDINT_MTX_PREFORK(tsdn, mu) malloc_mutex_prefork(tsdn, &(mu)) -# define LOCKEDINT_MTX_POSTFORK_PARENT(tsdn, mu) \ - malloc_mutex_postfork_parent(tsdn, &(mu)) -# define LOCKEDINT_MTX_POSTFORK_CHILD(tsdn, mu) \ - malloc_mutex_postfork_child(tsdn, &(mu)) +# define LOCKEDINT_MTX_DECLARE(name) malloc_mutex_t name; +# define LOCKEDINT_MTX_INIT(mu, name, rank, rank_mode) \ + malloc_mutex_init(&(mu), name, rank, rank_mode) +# define LOCKEDINT_MTX(mtx) (&(mtx)) +# define LOCKEDINT_MTX_LOCK(tsdn, mu) malloc_mutex_lock(tsdn, &(mu)) +# define LOCKEDINT_MTX_UNLOCK(tsdn, mu) malloc_mutex_unlock(tsdn, &(mu)) +# define LOCKEDINT_MTX_PREFORK(tsdn, mu) \ + malloc_mutex_prefork(tsdn, &(mu)) +# define LOCKEDINT_MTX_POSTFORK_PARENT(tsdn, mu) \ + malloc_mutex_postfork_parent(tsdn, &(mu)) +# define LOCKEDINT_MTX_POSTFORK_CHILD(tsdn, mu) \ + malloc_mutex_postfork_child(tsdn, &(mu)) #else -# define LOCKEDINT_MTX_DECLARE(name) -# define LOCKEDINT_MTX(mtx) NULL -# define LOCKEDINT_MTX_INIT(mu, name, rank, rank_mode) false -# define LOCKEDINT_MTX_LOCK(tsdn, mu) -# define LOCKEDINT_MTX_UNLOCK(tsdn, mu) -# define LOCKEDINT_MTX_PREFORK(tsdn, mu) -# define LOCKEDINT_MTX_POSTFORK_PARENT(tsdn, mu) -# define LOCKEDINT_MTX_POSTFORK_CHILD(tsdn, mu) +# define LOCKEDINT_MTX_DECLARE(name) +# define LOCKEDINT_MTX(mtx) NULL +# define LOCKEDINT_MTX_INIT(mu, name, rank, rank_mode) false +# define LOCKEDINT_MTX_LOCK(tsdn, mu) +# define LOCKEDINT_MTX_UNLOCK(tsdn, mu) +# define LOCKEDINT_MTX_PREFORK(tsdn, mu) +# define LOCKEDINT_MTX_POSTFORK_PARENT(tsdn, mu) +# define LOCKEDINT_MTX_POSTFORK_CHILD(tsdn, mu) #endif #ifdef JEMALLOC_ATOMIC_U64 -# define LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx) assert((mtx) == NULL) +# define LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx) assert((mtx) == NULL) #else -# define LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx) \ - malloc_mutex_assert_owner(tsdn, (mtx)) +# define LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx) \ + malloc_mutex_assert_owner(tsdn, (mtx)) #endif static inline uint64_t @@ -65,8 +71,7 @@ locked_read_u64(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_u64_t *p) { } static inline void -locked_inc_u64(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_u64_t *p, - uint64_t x) { +locked_inc_u64(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_u64_t *p, uint64_t x) { LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx); #ifdef JEMALLOC_ATOMIC_U64 atomic_fetch_add_u64(&p->val, x, ATOMIC_RELAXED); @@ -76,8 +81,7 @@ locked_inc_u64(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_u64_t *p, } static inline void -locked_dec_u64(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_u64_t *p, - uint64_t x) { +locked_dec_u64(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_u64_t *p, uint64_t x) { LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx); #ifdef JEMALLOC_ATOMIC_U64 uint64_t r = atomic_fetch_sub_u64(&p->val, x, ATOMIC_RELAXED); @@ -94,7 +98,7 @@ locked_inc_mod_u64(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_u64_t *p, const uint64_t x, const uint64_t modulus) { LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx); uint64_t before, after; - bool overflow; + bool overflow; #ifdef JEMALLOC_ATOMIC_U64 before = atomic_load_u64(&p->val, ATOMIC_RELAXED); do { @@ -104,8 +108,8 @@ locked_inc_mod_u64(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_u64_t *p, if (overflow) { after %= modulus; } - } while (!atomic_compare_exchange_weak_u64(&p->val, &before, after, - ATOMIC_RELAXED, ATOMIC_RELAXED)); + } while (!atomic_compare_exchange_weak_u64( + &p->val, &before, after, ATOMIC_RELAXED, ATOMIC_RELAXED)); #else before = p->val; after = before + x; @@ -162,8 +166,7 @@ locked_read_zu(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_zu_t *p) { } static inline void -locked_inc_zu(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_zu_t *p, - size_t x) { +locked_inc_zu(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_zu_t *p, size_t x) { LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx); #ifdef JEMALLOC_ATOMIC_U64 atomic_fetch_add_zu(&p->val, x, ATOMIC_RELAXED); @@ -174,8 +177,7 @@ locked_inc_zu(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_zu_t *p, } static inline void -locked_dec_zu(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_zu_t *p, - size_t x) { +locked_dec_zu(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_zu_t *p, size_t x) { LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx); #ifdef JEMALLOC_ATOMIC_U64 size_t r = atomic_fetch_sub_zu(&p->val, x, ATOMIC_RELAXED); diff --git a/dep/jemalloc/include/jemalloc/internal/log.h b/dep/jemalloc/include/jemalloc/internal/log.h index 642085863..f213bedad 100644 --- a/dep/jemalloc/include/jemalloc/internal/log.h +++ b/dep/jemalloc/include/jemalloc/internal/log.h @@ -1,14 +1,15 @@ #ifndef JEMALLOC_INTERNAL_LOG_H #define JEMALLOC_INTERNAL_LOG_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/mutex.h" #ifdef JEMALLOC_LOG -# define JEMALLOC_LOG_VAR_BUFSIZE 1000 +# define JEMALLOC_LOG_VAR_BUFSIZE 1000 #else -# define JEMALLOC_LOG_VAR_BUFSIZE 1 +# define JEMALLOC_LOG_VAR_BUFSIZE 1 #endif #define JEMALLOC_LOG_BUFSIZE 4096 @@ -26,16 +27,16 @@ * log("extent.a", "log msg for extent.a"); // 5 * log("extent.b", "log msg for extent.b"); // 6 * - * And your malloc_conf option is "log=arena.a|extent", then lines 2, 4, 5, and + * And your malloc_conf option is "log:arena.a|extent", then lines 2, 4, 5, and * 6 will print at runtime. You can enable logging from all log vars by - * writing "log=.". + * writing "log:.". * * None of this should be regarded as a stable API for right now. It's intended * as a debugging interface, to let us keep around some of our printf-debugging * statements. */ -extern char log_var_names[JEMALLOC_LOG_VAR_BUFSIZE]; +extern char log_var_names[JEMALLOC_LOG_VAR_BUFSIZE]; extern atomic_b_t log_init_done; typedef struct log_var_s log_var_t; @@ -44,7 +45,7 @@ struct log_var_s { * Lowest bit is "inited", second lowest is "enabled". Putting them in * a single word lets us avoid any fences on weak architectures. */ - atomic_u_t state; + atomic_u_t state; const char *name; }; @@ -52,7 +53,8 @@ struct log_var_s { #define LOG_INITIALIZED_NOT_ENABLED 1U #define LOG_ENABLED 2U -#define LOG_VAR_INIT(name_str) {ATOMIC_INIT(LOG_NOT_INITIALIZED), name_str} +#define LOG_VAR_INIT(name_str) \ + { ATOMIC_INIT(LOG_NOT_INITIALIZED), name_str } /* * Returns the value we should assume for state (which is not necessarily @@ -62,21 +64,21 @@ struct log_var_s { unsigned log_var_update_state(log_var_t *log_var); /* We factor out the metadata management to allow us to test more easily. */ -#define log_do_begin(log_var) \ -if (config_log) { \ - unsigned log_state = atomic_load_u(&(log_var).state, \ - ATOMIC_RELAXED); \ - if (unlikely(log_state == LOG_NOT_INITIALIZED)) { \ - log_state = log_var_update_state(&(log_var)); \ - assert(log_state != LOG_NOT_INITIALIZED); \ - } \ - if (log_state == LOG_ENABLED) { \ - { - /* User code executes here. */ -#define log_do_end(log_var) \ - } \ - } \ -} +#define log_do_begin(log_var) \ + if (config_log) { \ + unsigned log_state = atomic_load_u( \ + &(log_var).state, ATOMIC_RELAXED); \ + if (unlikely(log_state == LOG_NOT_INITIALIZED)) { \ + log_state = log_var_update_state(&(log_var)); \ + assert(log_state != LOG_NOT_INITIALIZED); \ + } \ + if (log_state == LOG_ENABLED) { \ + { +/* User code executes here. */ +#define log_do_end(log_var) \ + } \ + } \ + } /* * MSVC has some preprocessor bugs in its expansion of __VA_ARGS__ during @@ -87,29 +89,29 @@ if (config_log) { \ */ static inline void log_impl_varargs(const char *name, ...) { - char buf[JEMALLOC_LOG_BUFSIZE]; + char buf[JEMALLOC_LOG_BUFSIZE]; va_list ap; va_start(ap, name); const char *format = va_arg(ap, const char *); - size_t dst_offset = 0; + size_t dst_offset = 0; dst_offset += malloc_snprintf(buf, JEMALLOC_LOG_BUFSIZE, "%s: ", name); - dst_offset += malloc_vsnprintf(buf + dst_offset, - JEMALLOC_LOG_BUFSIZE - dst_offset, format, ap); - dst_offset += malloc_snprintf(buf + dst_offset, - JEMALLOC_LOG_BUFSIZE - dst_offset, "\n"); + dst_offset += malloc_vsnprintf( + buf + dst_offset, JEMALLOC_LOG_BUFSIZE - dst_offset, format, ap); + malloc_snprintf( + buf + dst_offset, JEMALLOC_LOG_BUFSIZE - dst_offset, "\n"); va_end(ap); malloc_write(buf); } /* Call as log("log.var.str", "format_string %d", arg_for_format_string); */ -#define LOG(log_var_str, ...) \ -do { \ - static log_var_t log_var = LOG_VAR_INIT(log_var_str); \ - log_do_begin(log_var) \ - log_impl_varargs((log_var).name, __VA_ARGS__); \ - log_do_end(log_var) \ -} while (0) +#define LOG(log_var_str, ...) \ + do { \ + static log_var_t log_var = LOG_VAR_INIT(log_var_str); \ + log_do_begin(log_var) \ + log_impl_varargs((log_var).name, __VA_ARGS__); \ + log_do_end(log_var) \ + } while (0) #endif /* JEMALLOC_INTERNAL_LOG_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/malloc_io.h b/dep/jemalloc/include/jemalloc/internal/malloc_io.h index a375bdae0..0f70c3c30 100644 --- a/dep/jemalloc/include/jemalloc/internal/malloc_io.h +++ b/dep/jemalloc/include/jemalloc/internal/malloc_io.h @@ -1,105 +1,101 @@ #ifndef JEMALLOC_INTERNAL_MALLOC_IO_H #define JEMALLOC_INTERNAL_MALLOC_IO_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/jemalloc_internal_types.h" #ifdef _WIN32 -# ifdef _WIN64 -# define FMT64_PREFIX "ll" -# define FMTPTR_PREFIX "ll" -# else -# define FMT64_PREFIX "ll" -# define FMTPTR_PREFIX "" -# endif -# define FMTd32 "d" -# define FMTu32 "u" -# define FMTx32 "x" -# define FMTd64 FMT64_PREFIX "d" -# define FMTu64 FMT64_PREFIX "u" -# define FMTx64 FMT64_PREFIX "x" -# define FMTdPTR FMTPTR_PREFIX "d" -# define FMTuPTR FMTPTR_PREFIX "u" -# define FMTxPTR FMTPTR_PREFIX "x" +# ifdef _WIN64 +# define FMT64_PREFIX "ll" +# define FMTPTR_PREFIX "ll" +# else +# define FMT64_PREFIX "ll" +# define FMTPTR_PREFIX "" +# endif +# define FMTd32 "d" +# define FMTu32 "u" +# define FMTx32 "x" +# define FMTd64 FMT64_PREFIX "d" +# define FMTu64 FMT64_PREFIX "u" +# define FMTx64 FMT64_PREFIX "x" +# define FMTdPTR FMTPTR_PREFIX "d" +# define FMTuPTR FMTPTR_PREFIX "u" +# define FMTxPTR FMTPTR_PREFIX "x" #else -# include -# define FMTd32 PRId32 -# define FMTu32 PRIu32 -# define FMTx32 PRIx32 -# define FMTd64 PRId64 -# define FMTu64 PRIu64 -# define FMTx64 PRIx64 -# define FMTdPTR PRIdPTR -# define FMTuPTR PRIuPTR -# define FMTxPTR PRIxPTR +# include +# define FMTd32 PRId32 +# define FMTu32 PRIu32 +# define FMTx32 PRIx32 +# define FMTd64 PRId64 +# define FMTu64 PRIu64 +# define FMTx64 PRIx64 +# define FMTdPTR PRIdPTR +# define FMTuPTR PRIuPTR +# define FMTxPTR PRIxPTR #endif /* Size of stack-allocated buffer passed to buferror(). */ -#define BUFERROR_BUF 64 +#define BUFERROR_BUF 64 /* * Size of stack-allocated buffer used by malloc_{,v,vc}printf(). This must be * large enough for all possible uses within jemalloc. */ -#define MALLOC_PRINTF_BUFSIZE 4096 +#define MALLOC_PRINTF_BUFSIZE 4096 write_cb_t wrtmessage; -int buferror(int err, char *buf, size_t buflen); -uintmax_t malloc_strtoumax(const char *restrict nptr, char **restrict endptr, - int base); +int buferror(int err, char *buf, size_t buflen); +uintmax_t malloc_strtoumax( + const char *restrict nptr, char **restrict endptr, int base); void malloc_write(const char *s); /* * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating * point math. */ -size_t malloc_vsnprintf(char *str, size_t size, const char *format, - va_list ap); +size_t malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap); size_t malloc_snprintf(char *str, size_t size, const char *format, ...) JEMALLOC_FORMAT_PRINTF(3, 4); /* * The caller can set write_cb to null to choose to print with the * je_malloc_message hook. */ -void malloc_vcprintf(write_cb_t *write_cb, void *cbopaque, const char *format, - va_list ap); +void malloc_vcprintf( + write_cb_t *write_cb, void *cbopaque, const char *format, va_list ap); void malloc_cprintf(write_cb_t *write_cb, void *cbopaque, const char *format, ...) JEMALLOC_FORMAT_PRINTF(3, 4); void malloc_printf(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2); -static inline ssize_t -malloc_write_fd(int fd, const void *buf, size_t count) { -#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_write) - /* - * Use syscall(2) rather than write(2) when possible in order to avoid - * the possibility of memory allocation within libc. This is necessary - * on FreeBSD; most operating systems do not have this problem though. - * - * syscall() returns long or int, depending on platform, so capture the - * result in the widest plausible type to avoid compiler warnings. - */ - long result = syscall(SYS_write, fd, buf, count); +ssize_t malloc_write_fd(int fd, const void *buf, size_t count); +ssize_t malloc_read_fd(int fd, void *buf, size_t count); + +static inline int +malloc_open(const char *path, int flags) { +#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_open) + return (int)syscall(SYS_open, path, flags); +#elif defined(JEMALLOC_USE_SYSCALL) && defined(SYS_openat) + return (int)syscall(SYS_openat, AT_FDCWD, path, flags); #else - ssize_t result = (ssize_t)write(fd, buf, -#ifdef _WIN32 - (unsigned int) + return open(path, flags); #endif - count); -#endif - return (ssize_t)result; } -static inline ssize_t -malloc_read_fd(int fd, void *buf, size_t count) { -#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_read) - long result = syscall(SYS_read, fd, buf, count); +static inline int +malloc_close(int fd) { +#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_close) + return (int)syscall(SYS_close, fd); #else - ssize_t result = read(fd, buf, -#ifdef _WIN32 - (unsigned int) + return close(fd); #endif - count); +} + +static inline off_t +malloc_lseek(int fd, off_t offset, int whence) { +#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_lseek) + return (off_t)syscall(SYS_lseek, fd, offset, whence); +#else + return lseek(fd, offset, whence); #endif - return (ssize_t)result; } #endif /* JEMALLOC_INTERNAL_MALLOC_IO_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/mpsc_queue.h b/dep/jemalloc/include/jemalloc/internal/mpsc_queue.h index 316ea9b16..86f4898fd 100644 --- a/dep/jemalloc/include/jemalloc/internal/mpsc_queue.h +++ b/dep/jemalloc/include/jemalloc/internal/mpsc_queue.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_MPSC_QUEUE_H #define JEMALLOC_INTERNAL_MPSC_QUEUE_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/atomic.h" /* @@ -25,6 +26,7 @@ * two-stack tricks reverses orders in the lock-free first stack). */ +/* clang-format off */ #define mpsc_queue(a_type) \ struct { \ atomic_p_t tail; \ @@ -130,5 +132,6 @@ a_prefix##pop_batch(a_queue_type *queue, a_list_type *dst) { \ } \ ql_concat(dst, &reversed, a_link); \ } +/* clang-format on */ #endif /* JEMALLOC_INTERNAL_MPSC_QUEUE_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/mutex.h b/dep/jemalloc/include/jemalloc/internal/mutex.h index 63a0b1b36..943c7928a 100644 --- a/dep/jemalloc/include/jemalloc/internal/mutex.h +++ b/dep/jemalloc/include/jemalloc/internal/mutex.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_MUTEX_H #define JEMALLOC_INTERNAL_MUTEX_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/mutex_prof.h" #include "jemalloc/internal/tsd.h" @@ -30,26 +31,29 @@ struct malloc_mutex_s { * avoid prefetching a modified cacheline (for the * unlocking thread). */ - mutex_prof_data_t prof_data; -#ifdef _WIN32 -# if _WIN32_WINNT >= 0x0600 - SRWLOCK lock; -# else - CRITICAL_SECTION lock; -# endif -#elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) - os_unfair_lock lock; -#elif (defined(JEMALLOC_MUTEX_INIT_CB)) - pthread_mutex_t lock; - malloc_mutex_t *postponed_next; -#else - pthread_mutex_t lock; -#endif + mutex_prof_data_t prof_data; /* * Hint flag to avoid exclusive cache line contention - * during spin waiting + * during spin waiting. Placed along with prof_data + * since it's always modified even with no contention. + * Modified by the lock owner only (after acquired, and + * before release), and may be read by other threads. */ - atomic_b_t locked; + atomic_b_t locked; +#ifdef _WIN32 +# if _WIN32_WINNT >= 0x0600 + SRWLOCK lock; +# else + CRITICAL_SECTION lock; +# endif +#elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) + os_unfair_lock lock; +#elif (defined(JEMALLOC_MUTEX_INIT_CB)) + pthread_mutex_t lock; + malloc_mutex_t *postponed_next; +#else + pthread_mutex_t lock; +#endif }; /* * We only touch witness when configured w/ debug. However we @@ -58,82 +62,118 @@ struct malloc_mutex_s { * memory cost. */ #if !defined(JEMALLOC_DEBUG) - witness_t witness; - malloc_mutex_lock_order_t lock_order; + witness_t witness; + malloc_mutex_lock_order_t lock_order; #endif }; #if defined(JEMALLOC_DEBUG) - witness_t witness; - malloc_mutex_lock_order_t lock_order; + witness_t witness; + malloc_mutex_lock_order_t lock_order; #endif }; #ifdef _WIN32 -# if _WIN32_WINNT >= 0x0600 -# define MALLOC_MUTEX_LOCK(m) AcquireSRWLockExclusive(&(m)->lock) -# define MALLOC_MUTEX_UNLOCK(m) ReleaseSRWLockExclusive(&(m)->lock) -# define MALLOC_MUTEX_TRYLOCK(m) (!TryAcquireSRWLockExclusive(&(m)->lock)) -# else -# define MALLOC_MUTEX_LOCK(m) EnterCriticalSection(&(m)->lock) -# define MALLOC_MUTEX_UNLOCK(m) LeaveCriticalSection(&(m)->lock) -# define MALLOC_MUTEX_TRYLOCK(m) (!TryEnterCriticalSection(&(m)->lock)) -# endif +# if _WIN32_WINNT >= 0x0600 +# define MALLOC_MUTEX_LOCK(m) AcquireSRWLockExclusive(&(m)->lock) +# define MALLOC_MUTEX_UNLOCK(m) \ + ReleaseSRWLockExclusive(&(m)->lock) +# define MALLOC_MUTEX_TRYLOCK(m) \ + (!TryAcquireSRWLockExclusive(&(m)->lock)) +# else +# define MALLOC_MUTEX_LOCK(m) EnterCriticalSection(&(m)->lock) +# define MALLOC_MUTEX_UNLOCK(m) LeaveCriticalSection(&(m)->lock) +# define MALLOC_MUTEX_TRYLOCK(m) \ + (!TryEnterCriticalSection(&(m)->lock)) +# endif #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) -# define MALLOC_MUTEX_LOCK(m) os_unfair_lock_lock(&(m)->lock) -# define MALLOC_MUTEX_UNLOCK(m) os_unfair_lock_unlock(&(m)->lock) -# define MALLOC_MUTEX_TRYLOCK(m) (!os_unfair_lock_trylock(&(m)->lock)) +# define MALLOC_MUTEX_LOCK(m) os_unfair_lock_lock(&(m)->lock) +# define MALLOC_MUTEX_UNLOCK(m) os_unfair_lock_unlock(&(m)->lock) +# define MALLOC_MUTEX_TRYLOCK(m) (!os_unfair_lock_trylock(&(m)->lock)) #else -# define MALLOC_MUTEX_LOCK(m) pthread_mutex_lock(&(m)->lock) -# define MALLOC_MUTEX_UNLOCK(m) pthread_mutex_unlock(&(m)->lock) -# define MALLOC_MUTEX_TRYLOCK(m) (pthread_mutex_trylock(&(m)->lock) != 0) +# define MALLOC_MUTEX_LOCK(m) pthread_mutex_lock(&(m)->lock) +# define MALLOC_MUTEX_UNLOCK(m) pthread_mutex_unlock(&(m)->lock) +# define MALLOC_MUTEX_TRYLOCK(m) (pthread_mutex_trylock(&(m)->lock) != 0) #endif -#define LOCK_PROF_DATA_INITIALIZER \ - {NSTIME_ZERO_INITIALIZER, NSTIME_ZERO_INITIALIZER, 0, 0, 0, \ - ATOMIC_INIT(0), 0, NULL, 0} +#define LOCK_PROF_DATA_INITIALIZER \ + { \ + NSTIME_ZERO_INITIALIZER, NSTIME_ZERO_INITIALIZER, 0, 0, 0, \ + ATOMIC_INIT(0), 0, NULL, 0 \ + } #ifdef _WIN32 -# define MALLOC_MUTEX_INITIALIZER +# define MALLOC_MUTEX_INITIALIZER #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) -# if defined(JEMALLOC_DEBUG) -# define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \ - WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} -# else -# define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \ - WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} -# endif +# if defined(JEMALLOC_DEBUG) +# define MALLOC_MUTEX_INITIALIZER \ + { \ + {{LOCK_PROF_DATA_INITIALIZER, \ + ATOMIC_INIT(false), OS_UNFAIR_LOCK_INIT}}, \ + WITNESS_INITIALIZER( \ + "mutex", WITNESS_RANK_OMIT), \ + 0 \ + } +# else +# define MALLOC_MUTEX_INITIALIZER \ + { \ + {{LOCK_PROF_DATA_INITIALIZER, \ + ATOMIC_INIT(false), OS_UNFAIR_LOCK_INIT}}, \ + WITNESS_INITIALIZER( \ + "mutex", WITNESS_RANK_OMIT) \ + } +# endif #elif (defined(JEMALLOC_MUTEX_INIT_CB)) -# if (defined(JEMALLOC_DEBUG)) -# define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}}, \ - WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} -# else -# define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}}, \ - WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} -# endif +# if (defined(JEMALLOC_DEBUG)) +# define MALLOC_MUTEX_INITIALIZER \ + { \ + {{LOCK_PROF_DATA_INITIALIZER, \ + ATOMIC_INIT(false), \ + PTHREAD_MUTEX_INITIALIZER, NULL}}, \ + WITNESS_INITIALIZER( \ + "mutex", WITNESS_RANK_OMIT), \ + 0 \ + } +# else +# define MALLOC_MUTEX_INITIALIZER \ + { \ + {{LOCK_PROF_DATA_INITIALIZER, \ + ATOMIC_INIT(false), \ + PTHREAD_MUTEX_INITIALIZER, NULL}}, \ + WITNESS_INITIALIZER( \ + "mutex", WITNESS_RANK_OMIT) \ + } +# endif #else -# define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT -# if defined(JEMALLOC_DEBUG) -# define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \ - WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} -# else -# define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \ - WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} -# endif +# define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT +# if defined(JEMALLOC_DEBUG) +# define MALLOC_MUTEX_INITIALIZER \ + { \ + {{LOCK_PROF_DATA_INITIALIZER, \ + ATOMIC_INIT(false), \ + PTHREAD_MUTEX_INITIALIZER}}, \ + WITNESS_INITIALIZER( \ + "mutex", WITNESS_RANK_OMIT), \ + 0 \ + } +# else +# define MALLOC_MUTEX_INITIALIZER \ + { \ + {{LOCK_PROF_DATA_INITIALIZER, \ + ATOMIC_INIT(false), \ + PTHREAD_MUTEX_INITIALIZER}}, \ + WITNESS_INITIALIZER( \ + "mutex", WITNESS_RANK_OMIT) \ + } +# endif #endif #ifdef JEMALLOC_LAZY_LOCK extern bool isthreaded; #else -# undef isthreaded /* Undo private_namespace.h definition. */ -# define isthreaded true +# undef isthreaded /* Undo private_namespace.h definition. */ +# define isthreaded true #endif bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, @@ -154,7 +194,12 @@ malloc_mutex_lock_final(malloc_mutex_t *mutex) { static inline bool malloc_mutex_trylock_final(malloc_mutex_t *mutex) { - return MALLOC_MUTEX_TRYLOCK(mutex); + bool failed = MALLOC_MUTEX_TRYLOCK(mutex); + if (!failed) { + atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); + } + + return failed; } static inline void @@ -169,15 +214,21 @@ mutex_owner_stats_update(tsdn_t *tsdn, malloc_mutex_t *mutex) { } } +static inline bool +malloc_mutex_is_locked(malloc_mutex_t *mutex) { + /* Used for sanity checking only. */ + return atomic_load_b(&mutex->locked, ATOMIC_RELAXED); +} + /* Trylock: return false if the lock is successfully acquired. */ static inline bool malloc_mutex_trylock(tsdn_t *tsdn, malloc_mutex_t *mutex) { witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); if (isthreaded) { if (malloc_mutex_trylock_final(mutex)) { - atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); return true; } + assert(malloc_mutex_is_locked(mutex)); mutex_owner_stats_update(tsdn, mutex); } witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); @@ -199,12 +250,12 @@ malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data) { if (sum->max_n_thds < data->max_n_thds) { sum->max_n_thds = data->max_n_thds; } - uint32_t cur_n_waiting_thds = atomic_load_u32(&sum->n_waiting_thds, - ATOMIC_RELAXED); - uint32_t new_n_waiting_thds = cur_n_waiting_thds + atomic_load_u32( - &data->n_waiting_thds, ATOMIC_RELAXED); - atomic_store_u32(&sum->n_waiting_thds, new_n_waiting_thds, - ATOMIC_RELAXED); + uint32_t cur_n_waiting_thds = atomic_load_u32( + &sum->n_waiting_thds, ATOMIC_RELAXED); + uint32_t new_n_waiting_thds = cur_n_waiting_thds + + atomic_load_u32(&data->n_waiting_thds, ATOMIC_RELAXED); + atomic_store_u32( + &sum->n_waiting_thds, new_n_waiting_thds, ATOMIC_RELAXED); sum->n_owner_switches += data->n_owner_switches; sum->n_lock_ops += data->n_lock_ops; } @@ -215,8 +266,8 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { if (isthreaded) { if (malloc_mutex_trylock_final(mutex)) { malloc_mutex_lock_slow(mutex); - atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); } + assert(malloc_mutex_is_locked(mutex)); mutex_owner_stats_update(tsdn, mutex); } witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); @@ -224,9 +275,10 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { static inline void malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) { - atomic_store_b(&mutex->locked, false, ATOMIC_RELAXED); witness_unlock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); if (isthreaded) { + assert(malloc_mutex_is_locked(mutex)); + atomic_store_b(&mutex->locked, false, ATOMIC_RELAXED); MALLOC_MUTEX_UNLOCK(mutex); } } @@ -234,6 +286,9 @@ malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) { static inline void malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) { witness_assert_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); + if (isthreaded) { + assert(malloc_mutex_is_locked(mutex)); + } } static inline void @@ -255,16 +310,16 @@ malloc_mutex_prof_copy(mutex_prof_data_t *dst, mutex_prof_data_t *source) { /* Copy the prof data from mutex for processing. */ static inline void -malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data, - malloc_mutex_t *mutex) { +malloc_mutex_prof_read( + tsdn_t *tsdn, mutex_prof_data_t *data, malloc_mutex_t *mutex) { /* Can only read holding the mutex. */ malloc_mutex_assert_owner(tsdn, mutex); malloc_mutex_prof_copy(data, &mutex->prof_data); } static inline void -malloc_mutex_prof_accum(tsdn_t *tsdn, mutex_prof_data_t *data, - malloc_mutex_t *mutex) { +malloc_mutex_prof_accum( + tsdn_t *tsdn, mutex_prof_data_t *data, malloc_mutex_t *mutex) { mutex_prof_data_t *source = &mutex->prof_data; /* Can only read holding the mutex. */ malloc_mutex_assert_owner(tsdn, mutex); @@ -286,8 +341,8 @@ malloc_mutex_prof_accum(tsdn_t *tsdn, mutex_prof_data_t *data, /* Compare the prof data and update to the maximum. */ static inline void -malloc_mutex_prof_max_update(tsdn_t *tsdn, mutex_prof_data_t *data, - malloc_mutex_t *mutex) { +malloc_mutex_prof_max_update( + tsdn_t *tsdn, mutex_prof_data_t *data, malloc_mutex_t *mutex) { mutex_prof_data_t *source = &mutex->prof_data; /* Can only read holding the mutex. */ malloc_mutex_assert_owner(tsdn, mutex); diff --git a/dep/jemalloc/include/jemalloc/internal/mutex_prof.h b/dep/jemalloc/include/jemalloc/internal/mutex_prof.h index 4a526a5ae..572200f35 100644 --- a/dep/jemalloc/include/jemalloc/internal/mutex_prof.h +++ b/dep/jemalloc/include/jemalloc/internal/mutex_prof.h @@ -1,80 +1,81 @@ #ifndef JEMALLOC_INTERNAL_MUTEX_PROF_H #define JEMALLOC_INTERNAL_MUTEX_PROF_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/nstime.h" #include "jemalloc/internal/tsd_types.h" -#define MUTEX_PROF_GLOBAL_MUTEXES \ - OP(background_thread) \ - OP(max_per_bg_thd) \ - OP(ctl) \ - OP(prof) \ - OP(prof_thds_data) \ - OP(prof_dump) \ - OP(prof_recent_alloc) \ - OP(prof_recent_dump) \ - OP(prof_stats) +#define MUTEX_PROF_GLOBAL_MUTEXES \ + OP(background_thread) \ + OP(max_per_bg_thd) \ + OP(ctl) \ + OP(prof) \ + OP(prof_thds_data) \ + OP(prof_dump) \ + OP(prof_recent_alloc) \ + OP(prof_recent_dump) \ + OP(prof_stats) typedef enum { #define OP(mtx) global_prof_mutex_##mtx, MUTEX_PROF_GLOBAL_MUTEXES #undef OP - mutex_prof_num_global_mutexes + mutex_prof_num_global_mutexes } mutex_prof_global_ind_t; -#define MUTEX_PROF_ARENA_MUTEXES \ - OP(large) \ - OP(extent_avail) \ - OP(extents_dirty) \ - OP(extents_muzzy) \ - OP(extents_retained) \ - OP(decay_dirty) \ - OP(decay_muzzy) \ - OP(base) \ - OP(tcache_list) \ - OP(hpa_shard) \ - OP(hpa_shard_grow) \ - OP(hpa_sec) +#define MUTEX_PROF_ARENA_MUTEXES \ + OP(large) \ + OP(extent_avail) \ + OP(extents_dirty) \ + OP(extents_muzzy) \ + OP(extents_retained) \ + OP(decay_dirty) \ + OP(decay_muzzy) \ + OP(base) \ + OP(tcache_list) \ + OP(hpa_shard) \ + OP(hpa_shard_grow) \ + OP(hpa_sec) typedef enum { #define OP(mtx) arena_prof_mutex_##mtx, MUTEX_PROF_ARENA_MUTEXES #undef OP - mutex_prof_num_arena_mutexes + mutex_prof_num_arena_mutexes } mutex_prof_arena_ind_t; /* * The forth parameter is a boolean value that is true for derived rate counters * and false for real ones. */ -#define MUTEX_PROF_UINT64_COUNTERS \ - OP(num_ops, uint64_t, "n_lock_ops", false, num_ops) \ - OP(num_ops_ps, uint64_t, "(#/sec)", true, num_ops) \ - OP(num_wait, uint64_t, "n_waiting", false, num_wait) \ - OP(num_wait_ps, uint64_t, "(#/sec)", true, num_wait) \ - OP(num_spin_acq, uint64_t, "n_spin_acq", false, num_spin_acq) \ - OP(num_spin_acq_ps, uint64_t, "(#/sec)", true, num_spin_acq) \ - OP(num_owner_switch, uint64_t, "n_owner_switch", false, num_owner_switch) \ - OP(num_owner_switch_ps, uint64_t, "(#/sec)", true, num_owner_switch) \ - OP(total_wait_time, uint64_t, "total_wait_ns", false, total_wait_time) \ - OP(total_wait_time_ps, uint64_t, "(#/sec)", true, total_wait_time) \ - OP(max_wait_time, uint64_t, "max_wait_ns", false, max_wait_time) +#define MUTEX_PROF_UINT64_COUNTERS \ + OP(num_ops, uint64_t, "n_lock_ops", false, num_ops) \ + OP(num_ops_ps, uint64_t, "(#/sec)", true, num_ops) \ + OP(num_wait, uint64_t, "n_waiting", false, num_wait) \ + OP(num_wait_ps, uint64_t, "(#/sec)", true, num_wait) \ + OP(num_spin_acq, uint64_t, "n_spin_acq", false, num_spin_acq) \ + OP(num_spin_acq_ps, uint64_t, "(#/sec)", true, num_spin_acq) \ + OP(num_owner_switch, uint64_t, "n_owner_switch", false, \ + num_owner_switch) \ + OP(num_owner_switch_ps, uint64_t, "(#/sec)", true, num_owner_switch) \ + OP(total_wait_time, uint64_t, "total_wait_ns", false, total_wait_time) \ + OP(total_wait_time_ps, uint64_t, "(#/sec)", true, total_wait_time) \ + OP(max_wait_time, uint64_t, "max_wait_ns", false, max_wait_time) -#define MUTEX_PROF_UINT32_COUNTERS \ - OP(max_num_thds, uint32_t, "max_n_thds", false, max_num_thds) +#define MUTEX_PROF_UINT32_COUNTERS \ + OP(max_num_thds, uint32_t, "max_n_thds", false, max_num_thds) -#define MUTEX_PROF_COUNTERS \ - MUTEX_PROF_UINT64_COUNTERS \ - MUTEX_PROF_UINT32_COUNTERS +#define MUTEX_PROF_COUNTERS \ + MUTEX_PROF_UINT64_COUNTERS \ + MUTEX_PROF_UINT32_COUNTERS #define OP(counter, type, human, derived, base_counter) mutex_counter_##counter, -#define COUNTER_ENUM(counter_list, t) \ - typedef enum { \ - counter_list \ - mutex_prof_num_##t##_counters \ - } mutex_prof_##t##_counter_ind_t; +#define COUNTER_ENUM(counter_list, t) \ + typedef enum { \ + counter_list mutex_prof_num_##t##_counters \ + } mutex_prof_##t##_counter_ind_t; COUNTER_ENUM(MUTEX_PROF_UINT64_COUNTERS, uint64_t) COUNTER_ENUM(MUTEX_PROF_UINT32_COUNTERS, uint32_t) @@ -88,17 +89,17 @@ typedef struct { * contention. We update them once we have the lock. */ /* Total time (in nano seconds) spent waiting on this mutex. */ - nstime_t tot_wait_time; + nstime_t tot_wait_time; /* Max time (in nano seconds) spent on a single lock operation. */ - nstime_t max_wait_time; + nstime_t max_wait_time; /* # of times have to wait for this mutex (after spinning). */ - uint64_t n_wait_times; + uint64_t n_wait_times; /* # of times acquired the mutex through local spinning. */ - uint64_t n_spin_acquired; + uint64_t n_spin_acquired; /* Max # of threads waiting for the mutex at the same time. */ - uint32_t max_n_thds; + uint32_t max_n_thds; /* Current # of threads waiting on the lock. Atomic synced. */ - atomic_u32_t n_waiting_thds; + atomic_u32_t n_waiting_thds; /* * Data touched on the fast path. These are modified right after we @@ -107,11 +108,11 @@ typedef struct { * cacheline. */ /* # of times the mutex holder is different than the previous one. */ - uint64_t n_owner_switches; + uint64_t n_owner_switches; /* Previous mutex holder, to facilitate n_owner_switches. */ - tsdn_t *prev_owner; + tsdn_t *prev_owner; /* # of lock() operations in total. */ - uint64_t n_lock_ops; + uint64_t n_lock_ops; } mutex_prof_data_t; #endif /* JEMALLOC_INTERNAL_MUTEX_PROF_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/nstime.h b/dep/jemalloc/include/jemalloc/internal/nstime.h index 486e5ccac..0848b9d0f 100644 --- a/dep/jemalloc/include/jemalloc/internal/nstime.h +++ b/dep/jemalloc/include/jemalloc/internal/nstime.h @@ -1,14 +1,19 @@ #ifndef JEMALLOC_INTERNAL_NSTIME_H #define JEMALLOC_INTERNAL_NSTIME_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/assert.h" + /* Maximum supported number of seconds (~584 years). */ #define NSTIME_SEC_MAX KQU(18446744072) #define NSTIME_MAGIC ((uint32_t)0xb8a9ce37) #ifdef JEMALLOC_DEBUG -# define NSTIME_ZERO_INITIALIZER {0, NSTIME_MAGIC} +# define NSTIME_ZERO_INITIALIZER \ + { 0, NSTIME_MAGIC } #else -# define NSTIME_ZERO_INITIALIZER {0} +# define NSTIME_ZERO_INITIALIZER \ + { 0 } #endif typedef struct { @@ -20,43 +25,43 @@ typedef struct { static const nstime_t nstime_zero = NSTIME_ZERO_INITIALIZER; -void nstime_init(nstime_t *time, uint64_t ns); -void nstime_init2(nstime_t *time, uint64_t sec, uint64_t nsec); +void nstime_init(nstime_t *time, uint64_t ns); +void nstime_init2(nstime_t *time, uint64_t sec, uint64_t nsec); uint64_t nstime_ns(const nstime_t *time); +uint64_t nstime_ms(const nstime_t *time); uint64_t nstime_sec(const nstime_t *time); -uint64_t nstime_msec(const nstime_t *time); uint64_t nstime_nsec(const nstime_t *time); -void nstime_copy(nstime_t *time, const nstime_t *source); -int nstime_compare(const nstime_t *a, const nstime_t *b); -void nstime_add(nstime_t *time, const nstime_t *addend); -void nstime_iadd(nstime_t *time, uint64_t addend); -void nstime_subtract(nstime_t *time, const nstime_t *subtrahend); -void nstime_isubtract(nstime_t *time, uint64_t subtrahend); -void nstime_imultiply(nstime_t *time, uint64_t multiplier); -void nstime_idivide(nstime_t *time, uint64_t divisor); +void nstime_copy(nstime_t *time, const nstime_t *source); +int nstime_compare(const nstime_t *a, const nstime_t *b); +void nstime_add(nstime_t *time, const nstime_t *addend); +void nstime_iadd(nstime_t *time, uint64_t addend); +void nstime_subtract(nstime_t *time, const nstime_t *subtrahend); +void nstime_isubtract(nstime_t *time, uint64_t subtrahend); +void nstime_imultiply(nstime_t *time, uint64_t multiplier); +void nstime_idivide(nstime_t *time, uint64_t divisor); uint64_t nstime_divide(const nstime_t *time, const nstime_t *divisor); +uint64_t nstime_ns_between(const nstime_t *earlier, const nstime_t *later); +uint64_t nstime_ms_between(const nstime_t *earlier, const nstime_t *later); uint64_t nstime_ns_since(const nstime_t *past); +uint64_t nstime_ms_since(const nstime_t *past); -typedef bool (nstime_monotonic_t)(void); +typedef bool(nstime_monotonic_t)(void); extern nstime_monotonic_t *JET_MUTABLE nstime_monotonic; -typedef void (nstime_update_t)(nstime_t *); +typedef void(nstime_update_t)(nstime_t *); extern nstime_update_t *JET_MUTABLE nstime_update; -typedef void (nstime_prof_update_t)(nstime_t *); +typedef void(nstime_prof_update_t)(nstime_t *); extern nstime_prof_update_t *JET_MUTABLE nstime_prof_update; void nstime_init_update(nstime_t *time); void nstime_prof_init_update(nstime_t *time); -enum prof_time_res_e { - prof_time_res_default = 0, - prof_time_res_high = 1 -}; +enum prof_time_res_e { prof_time_res_default = 0, prof_time_res_high = 1 }; typedef enum prof_time_res_e prof_time_res_t; -extern prof_time_res_t opt_prof_time_res; -extern const char *prof_time_res_mode_names[]; +extern prof_time_res_t opt_prof_time_res; +extern const char *const prof_time_res_mode_names[]; JEMALLOC_ALWAYS_INLINE void nstime_init_zero(nstime_t *time) { @@ -64,7 +69,7 @@ nstime_init_zero(nstime_t *time) { } JEMALLOC_ALWAYS_INLINE bool -nstime_equals_zero(nstime_t *time) { +nstime_equals_zero(const nstime_t *time) { int diff = nstime_compare(time, &nstime_zero); assert(diff >= 0); return diff == 0; diff --git a/dep/jemalloc/include/jemalloc/internal/pa.h b/dep/jemalloc/include/jemalloc/internal/pa.h index 4748a05b6..f3910ad83 100644 --- a/dep/jemalloc/include/jemalloc/internal/pa.h +++ b/dep/jemalloc/include/jemalloc/internal/pa.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_PA_H #define JEMALLOC_INTERNAL_PA_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/base.h" #include "jemalloc/internal/decay.h" #include "jemalloc/internal/ecache.h" @@ -95,12 +96,6 @@ struct pa_shard_s { /* Allocates from a PAC. */ pac_t pac; - /* - * We place a small extent cache in front of the HPA, since we intend - * these configurations to use many fewer arenas, and therefore have a - * higher risk of hot locks. - */ - sec_t hpa_sec; hpa_shard_t hpa_shard; /* The source of edata_t objects. */ @@ -108,7 +103,7 @@ struct pa_shard_s { unsigned ind; - malloc_mutex_t *stats_mtx; + malloc_mutex_t *stats_mtx; pa_shard_stats_t *stats; /* The emap this shard is tied to. */ @@ -120,8 +115,8 @@ struct pa_shard_s { static inline bool pa_shard_dont_decay_muzzy(pa_shard_t *shard) { - return ecache_npages_get(&shard->pac.ecache_muzzy) == 0 && - pac_decay_ms_get(&shard->pac, extent_state_muzzy) <= 0; + return ecache_npages_get(&shard->pac.ecache_muzzy) == 0 + && pac_decay_ms_get(&shard->pac, extent_state_muzzy) <= 0; } static inline ehooks_t * @@ -131,7 +126,7 @@ pa_shard_ehooks_get(pa_shard_t *shard) { /* Returns true on error. */ bool pa_central_init(pa_central_t *central, base_t *base, bool hpa, - hpa_hooks_t *hpa_hooks); + const hpa_hooks_t *hpa_hooks); /* Returns true on error. */ bool pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, pa_central_t *central, @@ -165,6 +160,9 @@ void pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard); */ void pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard); +/* Flush any caches used by shard */ +void pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard); + /* Gets an edata for the given allocation. */ edata_t *pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment, bool slab, szind_t szind, bool zero, bool guarded, @@ -185,10 +183,10 @@ bool pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size, * (We could make generated_dirty the return value of course, but this is more * consistent with the shrink pathway and our error codes here). */ -void pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, - bool *deferred_work_generated); -bool pa_decay_ms_set(tsdn_t *tsdn, pa_shard_t *shard, extent_state_t state, - ssize_t decay_ms, pac_purge_eagerness_t eagerness); +void pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, + bool *deferred_work_generated); +bool pa_decay_ms_set(tsdn_t *tsdn, pa_shard_t *shard, extent_state_t state, + ssize_t decay_ms, pac_purge_eagerness_t eagerness); ssize_t pa_decay_ms_get(pa_shard_t *shard, extent_state_t state); /* @@ -198,10 +196,10 @@ ssize_t pa_decay_ms_get(pa_shard_t *shard, extent_state_t state); * though, the arena, background thread, and PAC modules are tightly interwoven * in a way that's tricky to extricate, so we only do the HPA-specific parts. */ -void pa_shard_set_deferral_allowed(tsdn_t *tsdn, pa_shard_t *shard, - bool deferral_allowed); -void pa_shard_do_deferred_work(tsdn_t *tsdn, pa_shard_t *shard); -void pa_shard_try_deferred_work(tsdn_t *tsdn, pa_shard_t *shard); +void pa_shard_set_deferral_allowed( + tsdn_t *tsdn, pa_shard_t *shard, bool deferral_allowed); +void pa_shard_do_deferred_work(tsdn_t *tsdn, pa_shard_t *shard); +void pa_shard_try_deferred_work(tsdn_t *tsdn, pa_shard_t *shard); uint64_t pa_shard_time_until_deferred_work(tsdn_t *tsdn, pa_shard_t *shard); /******************************************************************************/ @@ -223,13 +221,16 @@ void pa_shard_prefork5(tsdn_t *tsdn, pa_shard_t *shard); void pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard); void pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard); -void pa_shard_basic_stats_merge(pa_shard_t *shard, size_t *nactive, - size_t *ndirty, size_t *nmuzzy); +size_t pa_shard_nactive(pa_shard_t *shard); +size_t pa_shard_ndirty(pa_shard_t *shard); +size_t pa_shard_nmuzzy(pa_shard_t *shard); + +void pa_shard_basic_stats_merge( + pa_shard_t *shard, size_t *nactive, size_t *ndirty, size_t *nmuzzy); void pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard, pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out, - hpa_shard_stats_t *hpa_stats_out, sec_stats_t *sec_stats_out, - size_t *resident); + hpa_shard_stats_t *hpa_stats_out, size_t *resident); /* * Reads the PA-owned mutex stats into the output stats array, at the diff --git a/dep/jemalloc/include/jemalloc/internal/pac.h b/dep/jemalloc/include/jemalloc/internal/pac.h index 01c4e6afa..a19c8b353 100644 --- a/dep/jemalloc/include/jemalloc/internal/pac.h +++ b/dep/jemalloc/include/jemalloc/internal/pac.h @@ -1,11 +1,15 @@ #ifndef JEMALLOC_INTERNAL_PAC_H #define JEMALLOC_INTERNAL_PAC_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/decay.h" +#include "jemalloc/internal/ecache.h" +#include "jemalloc/internal/edata_cache.h" #include "jemalloc/internal/exp_grow.h" +#include "jemalloc/internal/lockedint.h" #include "jemalloc/internal/pai.h" #include "san_bump.h" - /* * Page allocator classic; an implementation of the PAI interface that: * - Can be used for arenas with custom extent hooks. @@ -91,12 +95,12 @@ struct pac_s { ecache_t ecache_muzzy; ecache_t ecache_retained; - base_t *base; - emap_t *emap; + base_t *base; + emap_t *emap; edata_cache_t *edata_cache; /* The grow info for the retained ecache. */ - exp_grow_t exp_grow; + exp_grow_t exp_grow; malloc_mutex_t grow_mtx; /* Special allocator for guarded frequently reused extents. */ @@ -115,12 +119,37 @@ struct pac_s { decay_t decay_muzzy; /* muzzy --> retained */ malloc_mutex_t *stats_mtx; - pac_stats_t *stats; + pac_stats_t *stats; /* Extent serial number generator state. */ atomic_zu_t extent_sn_next; }; +typedef struct pac_thp_s pac_thp_t; +struct pac_thp_s { + /* + * opt_thp controls THP for user requested allocations. Settings + * "always", "never" and "default" are available if THP is supported + * by the OS and the default extent hooks are used: + * - "always" and "never" are covered by pages_set_thp_state() in + * ehooks_default_alloc_impl(). + * - "default" makes no change for all the other auto arenas except + * the huge arena. For the huge arena, we might also look at + * opt_metadata_thp to decide whether to use THP or not. + * This is a temporary remedy before HPA is fully supported. + */ + bool thp_madvise; + /* Below fields are protected by the lock. */ + malloc_mutex_t lock; + bool auto_thp_switched; + atomic_u_t n_thp_lazy; + /* + * List that tracks HUGEPAGE aligned regions that're lazily hugified + * in auto thp mode. + */ + edata_list_active_t thp_lazy_list; +}; + bool pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap, edata_cache_t *edata_cache, nstime_t *cur_time, size_t oversize_threshold, ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms, pac_stats_t *pac_stats, @@ -166,11 +195,11 @@ bool pac_maybe_decay_purge(tsdn_t *tsdn, pac_t *pac, decay_t *decay, * * Returns true on error (if the new limit is not valid). */ -bool pac_retain_grow_limit_get_set(tsdn_t *tsdn, pac_t *pac, size_t *old_limit, - size_t *new_limit); +bool pac_retain_grow_limit_get_set( + tsdn_t *tsdn, pac_t *pac, size_t *old_limit, size_t *new_limit); -bool pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state, - ssize_t decay_ms, pac_purge_eagerness_t eagerness); +bool pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state, + ssize_t decay_ms, pac_purge_eagerness_t eagerness); ssize_t pac_decay_ms_get(pac_t *pac, extent_state_t state); void pac_reset(tsdn_t *tsdn, pac_t *pac); diff --git a/dep/jemalloc/include/jemalloc/internal/pages.h b/dep/jemalloc/include/jemalloc/internal/pages.h index ad1f606a8..a4282c9bb 100644 --- a/dep/jemalloc/include/jemalloc/internal/pages.h +++ b/dep/jemalloc/include/jemalloc/internal/pages.h @@ -1,28 +1,39 @@ #ifndef JEMALLOC_INTERNAL_PAGES_EXTERNS_H #define JEMALLOC_INTERNAL_PAGES_EXTERNS_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_types.h" + +/* Actual operating system page size, detected during bootstrap, <= PAGE. */ +extern size_t os_page; + /* Page size. LG_PAGE is determined by the configure script. */ #ifdef PAGE_MASK -# undef PAGE_MASK +# undef PAGE_MASK #endif -#define PAGE ((size_t)(1U << LG_PAGE)) -#define PAGE_MASK ((size_t)(PAGE - 1)) +#define PAGE ((size_t)(1U << LG_PAGE)) +#define PAGE_MASK ((size_t)(PAGE - 1)) /* Return the page base address for the page containing address a. */ -#define PAGE_ADDR2BASE(a) \ - ((void *)((uintptr_t)(a) & ~PAGE_MASK)) +#define PAGE_ADDR2BASE(a) ALIGNMENT_ADDR2BASE(a, PAGE) /* Return the smallest pagesize multiple that is >= s. */ -#define PAGE_CEILING(s) \ - (((s) + PAGE_MASK) & ~PAGE_MASK) +#define PAGE_CEILING(s) (((s) + PAGE_MASK) & ~PAGE_MASK) /* Return the largest pagesize multiple that is <=s. */ -#define PAGE_FLOOR(s) \ - ((s) & ~PAGE_MASK) +#define PAGE_FLOOR(s) ((s) & ~PAGE_MASK) /* Huge page size. LG_HUGEPAGE is determined by the configure script. */ -#define HUGEPAGE ((size_t)(1U << LG_HUGEPAGE)) -#define HUGEPAGE_MASK ((size_t)(HUGEPAGE - 1)) +#define HUGEPAGE ((size_t)(1U << LG_HUGEPAGE)) +#define HUGEPAGE_MASK ((size_t)(HUGEPAGE - 1)) + +/* + * Used to validate that the hugepage size is not unexpectedly high. The huge + * page features (HPA, metadata_thp) are primarily designed with a 2M THP size + * in mind. Much larger sizes are not tested and likely to cause issues such as + * bad fragmentation or simply broken. + */ +#define HUGEPAGE_MAX_EXPECTED_SIZE ((size_t)(16U << 20)) #if LG_HUGEPAGE != 0 -# define HUGEPAGE_PAGES (HUGEPAGE / PAGE) +# define HUGEPAGE_PAGES (HUGEPAGE / PAGE) #else /* * It's convenient to define arrays (or bitmaps) of HUGEPAGE_PAGES lengths. If @@ -31,19 +42,17 @@ * that this value is at least 1. (We won't ever run in this degraded state; * hpa_supported() returns false in this case. */ -# define HUGEPAGE_PAGES 1 +# define HUGEPAGE_PAGES 1 #endif /* Return the huge page base address for the huge page containing address a. */ -#define HUGEPAGE_ADDR2BASE(a) \ - ((void *)((uintptr_t)(a) & ~HUGEPAGE_MASK)) +#define HUGEPAGE_ADDR2BASE(a) ALIGNMENT_ADDR2BASE(a, HUGEPAGE) /* Return the smallest pagesize multiple that is >= s. */ -#define HUGEPAGE_CEILING(s) \ - (((s) + HUGEPAGE_MASK) & ~HUGEPAGE_MASK) +#define HUGEPAGE_CEILING(s) (((s) + HUGEPAGE_MASK) & ~HUGEPAGE_MASK) /* PAGES_CAN_PURGE_LAZY is defined if lazy purging is supported. */ #if defined(_WIN32) || defined(JEMALLOC_PURGE_MADVISE_FREE) -# define PAGES_CAN_PURGE_LAZY +# define PAGES_CAN_PURGE_LAZY #endif /* * PAGES_CAN_PURGE_FORCED is defined if forced purging is supported. @@ -54,10 +63,11 @@ * next step after purging on Windows anyway, there's no point in adding such * complexity. */ -#if !defined(_WIN32) && ((defined(JEMALLOC_PURGE_MADVISE_DONTNEED) && \ - defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS)) || \ - defined(JEMALLOC_MAPS_COALESCE)) -# define PAGES_CAN_PURGE_FORCED +#if !defined(_WIN32) \ + && ((defined(JEMALLOC_PURGE_MADVISE_DONTNEED) \ + && defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS)) \ + || defined(JEMALLOC_MAPS_COALESCE)) +# define PAGES_CAN_PURGE_FORCED #endif static const bool pages_can_purge_lazy = @@ -76,7 +86,7 @@ static const bool pages_can_purge_forced = ; #if defined(JEMALLOC_HAVE_MADVISE_HUGE) || defined(JEMALLOC_HAVE_MEMCNTL) -# define PAGES_CAN_HUGIFY +# define PAGES_CAN_HUGIFY #endif static const bool pages_can_hugify = @@ -87,32 +97,46 @@ static const bool pages_can_hugify = #endif ; +/* + * thp_mode_t are values for opt.thp, while system_thp_mode_t is for kernel thp + * settings, i.e., init_system_thp_mode. + */ typedef enum { - thp_mode_default = 0, /* Do not change hugepage settings. */ - thp_mode_always = 1, /* Always set MADV_HUGEPAGE. */ - thp_mode_never = 2, /* Always set MADV_NOHUGEPAGE. */ + thp_mode_do_nothing = 0, /* Respect kernel thp settings. */ + thp_mode_always = 1, /* Always set MADV_HUGEPAGE. */ + thp_mode_never = 2, /* Always set MADV_NOHUGEPAGE. */ - thp_mode_names_limit = 3, /* Used for option processing. */ - thp_mode_not_supported = 3 /* No THP support detected. */ + thp_mode_names_limit = 3, /* Used for option processing. */ + thp_mode_not_supported = 3 /* No THP support detected. */ } thp_mode_t; -#define THP_MODE_DEFAULT thp_mode_default -extern thp_mode_t opt_thp; -extern thp_mode_t init_system_thp_mode; /* Initial system wide state. */ -extern const char *thp_mode_names[]; +typedef enum { + system_thp_mode_madvise = 0, /* Kernel THP mode: madvise */ + system_thp_mode_always = 1, /* Kernel THP mode: always */ + system_thp_mode_never = 2, /* Kernel THP mode: never */ + system_thp_mode_not_supported = 3 /* No THP support detected. */ +} system_thp_mode_t; + +#define THP_MODE_DEFAULT thp_mode_do_nothing +extern thp_mode_t opt_thp; +extern system_thp_mode_t init_system_thp_mode; /* Initial system wide state. */ +extern const char *const thp_mode_names[]; +extern const char *const system_thp_mode_names[]; void *pages_map(void *addr, size_t size, size_t alignment, bool *commit); -void pages_unmap(void *addr, size_t size); -bool pages_commit(void *addr, size_t size); -bool pages_decommit(void *addr, size_t size); -bool pages_purge_lazy(void *addr, size_t size); -bool pages_purge_forced(void *addr, size_t size); +void pages_unmap(void *addr, size_t size); +bool pages_commit(void *addr, size_t size); +bool pages_decommit(void *addr, size_t size); +bool pages_purge_lazy(void *addr, size_t size); +bool pages_purge_forced(void *addr, size_t size); +bool pages_purge_process_madvise(void *vec, size_t ven_len, size_t total_bytes); bool pages_huge(void *addr, size_t size); bool pages_nohuge(void *addr, size_t size); +bool pages_collapse(void *addr, size_t size); bool pages_dontdump(void *addr, size_t size); bool pages_dodump(void *addr, size_t size); bool pages_boot(void); -void pages_set_thp_state (void *ptr, size_t size); +void pages_set_thp_state(void *ptr, size_t size); void pages_mark_guards(void *head, void *tail); void pages_unmark_guards(void *head, void *tail); diff --git a/dep/jemalloc/include/jemalloc/internal/pai.h b/dep/jemalloc/include/jemalloc/internal/pai.h index d978cd7d2..9b4c257b2 100644 --- a/dep/jemalloc/include/jemalloc/internal/pai.h +++ b/dep/jemalloc/include/jemalloc/internal/pai.h @@ -1,6 +1,10 @@ #ifndef JEMALLOC_INTERNAL_PAI_H #define JEMALLOC_INTERNAL_PAI_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/edata.h" +#include "jemalloc/internal/tsd_types.h" + /* An interface for page allocation. */ typedef struct pai_s pai_t; @@ -9,15 +13,6 @@ struct pai_s { edata_t *(*alloc)(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero, bool guarded, bool frequent_reuse, bool *deferred_work_generated); - /* - * Returns the number of extents added to the list (which may be fewer - * than requested, in case of OOM). The list should already be - * initialized. The only alignment guarantee is page-alignment, and - * the results are not necessarily zeroed. - */ - size_t (*alloc_batch)(tsdn_t *tsdn, pai_t *self, size_t size, - size_t nallocs, edata_list_active_t *results, - bool *deferred_work_generated); bool (*expand)(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated); @@ -25,9 +20,6 @@ struct pai_s { size_t old_size, size_t new_size, bool *deferred_work_generated); void (*dalloc)(tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated); - /* This function empties out list as a side-effect of being called. */ - void (*dalloc_batch)(tsdn_t *tsdn, pai_t *self, - edata_list_active_t *list, bool *deferred_work_generated); uint64_t (*time_until_deferred_work)(tsdn_t *tsdn, pai_t *self); }; @@ -37,20 +29,12 @@ struct pai_s { */ static inline edata_t * -pai_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, - bool zero, bool guarded, bool frequent_reuse, - bool *deferred_work_generated) { +pai_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero, + bool guarded, bool frequent_reuse, bool *deferred_work_generated) { return self->alloc(tsdn, self, size, alignment, zero, guarded, frequent_reuse, deferred_work_generated); } -static inline size_t -pai_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs, - edata_list_active_t *results, bool *deferred_work_generated) { - return self->alloc_batch(tsdn, self, size, nallocs, results, - deferred_work_generated); -} - static inline bool pai_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated) { @@ -61,35 +45,19 @@ pai_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, static inline bool pai_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, size_t new_size, bool *deferred_work_generated) { - return self->shrink(tsdn, self, edata, old_size, new_size, - deferred_work_generated); + return self->shrink( + tsdn, self, edata, old_size, new_size, deferred_work_generated); } static inline void -pai_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata, - bool *deferred_work_generated) { +pai_dalloc( + tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) { self->dalloc(tsdn, self, edata, deferred_work_generated); } -static inline void -pai_dalloc_batch(tsdn_t *tsdn, pai_t *self, edata_list_active_t *list, - bool *deferred_work_generated) { - self->dalloc_batch(tsdn, self, list, deferred_work_generated); -} - static inline uint64_t pai_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) { return self->time_until_deferred_work(tsdn, self); } -/* - * An implementation of batch allocation that simply calls alloc once for - * each item in the list. - */ -size_t pai_alloc_batch_default(tsdn_t *tsdn, pai_t *self, size_t size, - size_t nallocs, edata_list_active_t *results, bool *deferred_work_generated); -/* Ditto, for dalloc. */ -void pai_dalloc_batch_default(tsdn_t *tsdn, pai_t *self, - edata_list_active_t *list, bool *deferred_work_generated); - #endif /* JEMALLOC_INTERNAL_PAI_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/peak.h b/dep/jemalloc/include/jemalloc/internal/peak.h index 59da3e41b..599f1a026 100644 --- a/dep/jemalloc/include/jemalloc/internal/peak.h +++ b/dep/jemalloc/include/jemalloc/internal/peak.h @@ -1,6 +1,8 @@ #ifndef JEMALLOC_INTERNAL_PEAK_H #define JEMALLOC_INTERNAL_PEAK_H +#include "jemalloc/internal/jemalloc_preamble.h" + typedef struct peak_s peak_t; struct peak_s { /* The highest recorded peak value, after adjustment (see below). */ @@ -12,7 +14,8 @@ struct peak_s { uint64_t adjustment; }; -#define PEAK_INITIALIZER {0, 0} +#define PEAK_INITIALIZER \ + { 0, 0 } static inline uint64_t peak_max(peak_t *peak) { diff --git a/dep/jemalloc/include/jemalloc/internal/peak_event.h b/dep/jemalloc/include/jemalloc/internal/peak_event.h index b808ce043..0d1f16271 100644 --- a/dep/jemalloc/include/jemalloc/internal/peak_event.h +++ b/dep/jemalloc/include/jemalloc/internal/peak_event.h @@ -1,6 +1,17 @@ #ifndef JEMALLOC_INTERNAL_PEAK_EVENT_H #define JEMALLOC_INTERNAL_PEAK_EVENT_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/tsd_types.h" + +/* + * Update every 64K by default. We're not exposing this as a configuration + * option for now; we don't want to bind ourselves too tightly to any particular + * performance requirements for small values, or guarantee that we'll even be + * able to provide fine-grained accuracy. + */ +#define PEAK_EVENT_WAIT (64 * 1024) + /* * While peak.h contains the simple helper struct that tracks state, this * contains the allocator tie-ins (and knows about tsd, the event module, etc.). @@ -9,16 +20,9 @@ /* Update the peak with current tsd state. */ void peak_event_update(tsd_t *tsd); /* Set current state to zero. */ -void peak_event_zero(tsd_t *tsd); +void peak_event_zero(tsd_t *tsd); uint64_t peak_event_max(tsd_t *tsd); -/* Manual hooks. */ -/* The activity-triggered hooks. */ -uint64_t peak_alloc_new_event_wait(tsd_t *tsd); -uint64_t peak_alloc_postponed_event_wait(tsd_t *tsd); -void peak_alloc_event_handler(tsd_t *tsd, uint64_t elapsed); -uint64_t peak_dalloc_new_event_wait(tsd_t *tsd); -uint64_t peak_dalloc_postponed_event_wait(tsd_t *tsd); -void peak_dalloc_event_handler(tsd_t *tsd, uint64_t elapsed); +extern te_base_cb_t peak_te_handler; #endif /* JEMALLOC_INTERNAL_PEAK_EVENT_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/ph.h b/dep/jemalloc/include/jemalloc/internal/ph.h index 5f091c5fb..803d2cbd0 100644 --- a/dep/jemalloc/include/jemalloc/internal/ph.h +++ b/dep/jemalloc/include/jemalloc/internal/ph.h @@ -1,6 +1,10 @@ #ifndef JEMALLOC_INTERNAL_PH_H #define JEMALLOC_INTERNAL_PH_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/assert.h" +#include "jemalloc/internal/bit_util.h" + /* * A Pairing Heap implementation. * @@ -71,9 +75,19 @@ struct ph_s { size_t auxcount; }; +typedef struct ph_enumerate_vars_s ph_enumerate_vars_t; +struct ph_enumerate_vars_s { + uint16_t front; + uint16_t rear; + uint16_t queue_size; + uint16_t visited_num; + uint16_t max_visit_num; + uint16_t max_queue_size; +}; + JEMALLOC_ALWAYS_INLINE phn_link_t * phn_link_get(void *phn, size_t offset) { - return (phn_link_t *)(((uintptr_t)phn) + offset); + return (phn_link_t *)(((char *)phn) + offset); } JEMALLOC_ALWAYS_INLINE void @@ -115,8 +129,7 @@ phn_prev_set(void *phn, void *prev, size_t offset) { } JEMALLOC_ALWAYS_INLINE void -phn_merge_ordered(void *phn0, void *phn1, size_t offset, - ph_cmp_t cmp) { +phn_merge_ordered(void *phn0, void *phn1, size_t offset, ph_cmp_t cmp) { void *phn0child; assert(phn0 != NULL); @@ -127,6 +140,7 @@ phn_merge_ordered(void *phn0, void *phn1, size_t offset, phn0child = phn_lchild_get(phn0, offset); phn_next_set(phn1, phn0child, offset); if (phn0child != NULL) { + /* NOLINTNEXTLINE(readability-suspicious-call-argument) */ phn_prev_set(phn0child, phn1, offset); } phn_lchild_set(phn0, phn1, offset); @@ -143,6 +157,7 @@ phn_merge(void *phn0, void *phn1, size_t offset, ph_cmp_t cmp) { phn_merge_ordered(phn0, phn1, offset, cmp); result = phn0; } else { + /* NOLINTNEXTLINE(readability-suspicious-call-argument) */ phn_merge_ordered(phn1, phn0, offset, cmp); result = phn1; } @@ -156,6 +171,10 @@ phn_merge_siblings(void *phn, size_t offset, ph_cmp_t cmp) { void *phn0 = phn; void *phn1 = phn_next_get(phn0, offset); + if (phn1 == NULL) { + return phn0; + } + /* * Multipass merge, wherein the first two elements of a FIFO * are repeatedly merged, and each result is appended to the @@ -164,59 +183,61 @@ phn_merge_siblings(void *phn, size_t offset, ph_cmp_t cmp) { * its tail, so we do a single pass over the sibling list to * populate the FIFO. */ - if (phn1 != NULL) { - void *phnrest = phn_next_get(phn1, offset); - if (phnrest != NULL) { - phn_prev_set(phnrest, NULL, offset); - } - phn_prev_set(phn0, NULL, offset); - phn_next_set(phn0, NULL, offset); - phn_prev_set(phn1, NULL, offset); - phn_next_set(phn1, NULL, offset); - phn0 = phn_merge(phn0, phn1, offset, cmp); - head = tail = phn0; - phn0 = phnrest; - while (phn0 != NULL) { - phn1 = phn_next_get(phn0, offset); - if (phn1 != NULL) { - phnrest = phn_next_get(phn1, offset); - if (phnrest != NULL) { - phn_prev_set(phnrest, NULL, offset); - } - phn_prev_set(phn0, NULL, offset); - phn_next_set(phn0, NULL, offset); - phn_prev_set(phn1, NULL, offset); - phn_next_set(phn1, NULL, offset); - phn0 = phn_merge(phn0, phn1, offset, cmp); - phn_next_set(tail, phn0, offset); - tail = phn0; - phn0 = phnrest; - } else { - phn_next_set(tail, phn0, offset); - tail = phn0; - phn0 = NULL; - } - } - phn0 = head; + void *phnrest = phn_next_get(phn1, offset); + if (phnrest != NULL) { + phn_prev_set(phnrest, NULL, offset); + } + phn_prev_set(phn0, NULL, offset); + phn_next_set(phn0, NULL, offset); + phn_prev_set(phn1, NULL, offset); + phn_next_set(phn1, NULL, offset); + phn0 = phn_merge(phn0, phn1, offset, cmp); + head = tail = phn0; + phn0 = phnrest; + while (phn0 != NULL) { phn1 = phn_next_get(phn0, offset); if (phn1 != NULL) { - while (true) { - head = phn_next_get(phn1, offset); - assert(phn_prev_get(phn0, offset) == NULL); - phn_next_set(phn0, NULL, offset); - assert(phn_prev_get(phn1, offset) == NULL); - phn_next_set(phn1, NULL, offset); - phn0 = phn_merge(phn0, phn1, offset, cmp); - if (head == NULL) { - break; - } - phn_next_set(tail, phn0, offset); - tail = phn0; - phn0 = head; - phn1 = phn_next_get(phn0, offset); + phnrest = phn_next_get(phn1, offset); + if (phnrest != NULL) { + phn_prev_set(phnrest, NULL, offset); } + phn_prev_set(phn0, NULL, offset); + phn_next_set(phn0, NULL, offset); + phn_prev_set(phn1, NULL, offset); + phn_next_set(phn1, NULL, offset); + phn0 = phn_merge(phn0, phn1, offset, cmp); + /* NOLINTNEXTLINE(readability-suspicious-call-argument) */ + phn_next_set(tail, phn0, offset); + tail = phn0; + phn0 = phnrest; + } else { + /* NOLINTNEXTLINE(readability-suspicious-call-argument) */ + phn_next_set(tail, phn0, offset); + tail = phn0; + phn0 = NULL; } } + phn0 = head; + phn1 = phn_next_get(phn0, offset); + if (phn1 != NULL) { + while (true) { + head = phn_next_get(phn1, offset); + assert(phn_prev_get(phn0, offset) == NULL); + phn_next_set(phn0, NULL, offset); + assert(phn_prev_get(phn1, offset) == NULL); + phn_next_set(phn1, NULL, offset); + phn0 = phn_merge(phn0, phn1, offset, cmp); + if (head == NULL) { + break; + } + /* NOLINTNEXTLINE(readability-suspicious-call-argument) */ + phn_next_set(tail, phn0, offset); + tail = phn0; + phn0 = head; + phn1 = phn_next_get(phn0, offset); + } + } + return phn0; } @@ -230,7 +251,7 @@ ph_merge_aux(ph_t *ph, size_t offset, ph_cmp_t cmp) { phn_prev_set(phn, NULL, offset); phn = phn_merge_siblings(phn, offset, cmp); assert(phn_next_get(phn, offset) == NULL); - ph->root = phn_merge(ph->root, phn, offset, cmp); + phn_merge_ordered(ph->root, phn, offset, cmp); } } @@ -298,6 +319,7 @@ ph_try_aux_merge_pair(ph_t *ph, size_t offset, ph_cmp_t cmp) { phn0 = phn_merge(phn0, phn1, offset, cmp); phn_next_set(phn0, next_phn1, offset); if (next_phn1 != NULL) { + /* NOLINTNEXTLINE(readability-suspicious-call-argument) */ phn_prev_set(next_phn1, phn0, offset); } phn_next_set(ph->root, phn0, offset); @@ -318,36 +340,36 @@ ph_insert(ph_t *ph, void *phn, size_t offset, ph_cmp_t cmp) { */ if (ph->root == NULL) { ph->root = phn; - } else { - /* - * As a special case, check to see if we can replace the root. - * This is practically common in some important cases, and lets - * us defer some insertions (hopefully, until the point where - * some of the items in the aux list have been removed, savings - * us from linking them at all). - */ - if (cmp(phn, ph->root) < 0) { - phn_lchild_set(phn, ph->root, offset); - phn_prev_set(ph->root, phn, offset); - ph->root = phn; - ph->auxcount = 0; - return; - } - ph->auxcount++; - phn_next_set(phn, phn_next_get(ph->root, offset), offset); - if (phn_next_get(ph->root, offset) != NULL) { - phn_prev_set(phn_next_get(ph->root, offset), phn, - offset); - } - phn_prev_set(phn, ph->root, offset); - phn_next_set(ph->root, phn, offset); + return; } - if (ph->auxcount > 1) { - unsigned nmerges = ffs_zu(ph->auxcount - 1); - bool done = false; - for (unsigned i = 0; i < nmerges && !done; i++) { - done = ph_try_aux_merge_pair(ph, offset, cmp); - } + + /* + * As a special case, check to see if we can replace the root. + * This is practically common in some important cases, and lets + * us defer some insertions (hopefully, until the point where + * some of the items in the aux list have been removed, savings + * us from linking them at all). + */ + if (cmp(phn, ph->root) < 0) { + phn_lchild_set(phn, ph->root, offset); + phn_prev_set(ph->root, phn, offset); + ph->root = phn; + ph->auxcount = 0; + return; + } + + phn_next_set(phn, phn_next_get(ph->root, offset), offset); + if (phn_next_get(ph->root, offset) != NULL) { + phn_prev_set(phn_next_get(ph->root, offset), phn, offset); + } + phn_prev_set(phn, ph->root, offset); + phn_next_set(ph->root, phn, offset); + + ph->auxcount++; + unsigned nmerges = ffs_zu(ph->auxcount); + bool done = false; + for (unsigned i = 0; i < nmerges && !done; i++) { + done = ph_try_aux_merge_pair(ph, offset, cmp); } } @@ -363,158 +385,210 @@ ph_remove_first(ph_t *ph, size_t offset, ph_cmp_t cmp) { ph->root = ph_merge_children(ph->root, offset, cmp); return ret; - } JEMALLOC_ALWAYS_INLINE void ph_remove(ph_t *ph, void *phn, size_t offset, ph_cmp_t cmp) { - void *replace; - void *parent; - if (ph->root == phn) { - /* - * We can delete from aux list without merging it, but we need - * to merge if we are dealing with the root node and it has - * children. - */ - if (phn_lchild_get(phn, offset) == NULL) { - ph->root = phn_next_get(phn, offset); - if (ph->root != NULL) { - phn_prev_set(ph->root, NULL, offset); - } - return; - } ph_merge_aux(ph, offset, cmp); - if (ph->root == phn) { - ph->root = ph_merge_children(ph->root, offset, cmp); - return; - } + ph->root = ph_merge_children(phn, offset, cmp); + return; } - /* Get parent (if phn is leftmost child) before mutating. */ - if ((parent = phn_prev_get(phn, offset)) != NULL) { - if (phn_lchild_get(parent, offset) != phn) { - parent = NULL; - } - } - /* Find a possible replacement node, and link to parent. */ - replace = ph_merge_children(phn, offset, cmp); - /* Set next/prev for sibling linked list. */ + void *prev = phn_prev_get(phn, offset); + void *next = phn_next_get(phn, offset); + + /* If we have children, then we integrate them back in the heap. */ + void *replace = ph_merge_children(phn, offset, cmp); if (replace != NULL) { - if (parent != NULL) { - phn_prev_set(replace, parent, offset); - phn_lchild_set(parent, replace, offset); - } else { - phn_prev_set(replace, phn_prev_get(phn, offset), - offset); - if (phn_prev_get(phn, offset) != NULL) { - phn_next_set(phn_prev_get(phn, offset), replace, - offset); - } - } - phn_next_set(replace, phn_next_get(phn, offset), offset); - if (phn_next_get(phn, offset) != NULL) { - phn_prev_set(phn_next_get(phn, offset), replace, - offset); + phn_next_set(replace, next, offset); + if (next != NULL) { + phn_prev_set(next, replace, offset); } + + next = replace; + } + + if (next != NULL) { + phn_prev_set(next, prev, offset); + } + + assert(prev != NULL); + if (phn_lchild_get(prev, offset) == phn) { + phn_lchild_set(prev, next, offset); } else { - if (parent != NULL) { - void *next = phn_next_get(phn, offset); - phn_lchild_set(parent, next, offset); - if (next != NULL) { - phn_prev_set(next, parent, offset); - } - } else { - assert(phn_prev_get(phn, offset) != NULL); - phn_next_set( - phn_prev_get(phn, offset), - phn_next_get(phn, offset), offset); - } - if (phn_next_get(phn, offset) != NULL) { - phn_prev_set( - phn_next_get(phn, offset), - phn_prev_get(phn, offset), offset); - } + phn_next_set(prev, next, offset); } } -#define ph_structs(a_prefix, a_type) \ -typedef struct { \ - phn_link_t link; \ -} a_prefix##_link_t; \ - \ -typedef struct { \ - ph_t ph; \ -} a_prefix##_t; +JEMALLOC_ALWAYS_INLINE void +ph_enumerate_vars_init(ph_enumerate_vars_t *vars, uint16_t max_visit_num, + uint16_t max_queue_size) { + vars->queue_size = 0; + vars->visited_num = 0; + vars->front = 0; + vars->rear = 0; + vars->max_visit_num = max_visit_num; + vars->max_queue_size = max_queue_size; + assert(vars->max_visit_num > 0); + /* + * max_queue_size must be able to support max_visit_num, which means + * the queue will not overflow before reaching max_visit_num. + */ + assert(vars->max_queue_size >= (vars->max_visit_num + 1) / 2); +} + +JEMALLOC_ALWAYS_INLINE void +ph_enumerate_queue_push( + void *phn, void **bfs_queue, ph_enumerate_vars_t *vars) { + assert(vars->queue_size < vars->max_queue_size); + bfs_queue[vars->rear] = phn; + vars->rear = (vars->rear + 1) % vars->max_queue_size; + (vars->queue_size)++; +} + +JEMALLOC_ALWAYS_INLINE void * +ph_enumerate_queue_pop(void **bfs_queue, ph_enumerate_vars_t *vars) { + assert(vars->queue_size > 0); + assert(vars->queue_size <= vars->max_queue_size); + void *ret = bfs_queue[vars->front]; + vars->front = (vars->front + 1) % vars->max_queue_size; + (vars->queue_size)--; + return ret; +} + +/* + * The two functions below offer a solution to enumerate the pairing heap. + * Whe enumerating, always call ph_enumerate_prepare first to prepare the queue + * needed for BFS. Next, call ph_enumerate_next to get the next element in + * the enumeration. When enumeration ends, ph_enumerate_next returns NULL and + * should not be called again. Enumeration ends when all elements in the heap + * has been enumerated or the number of visited elements exceed + * max_visit_num. + */ +JEMALLOC_ALWAYS_INLINE void +ph_enumerate_prepare(ph_t *ph, void **bfs_queue, ph_enumerate_vars_t *vars, + uint16_t max_visit_num, uint16_t max_queue_size) { + ph_enumerate_vars_init(vars, max_visit_num, max_queue_size); + ph_enumerate_queue_push(ph->root, bfs_queue, vars); +} + +JEMALLOC_ALWAYS_INLINE void * +ph_enumerate_next( + ph_t *ph, size_t offset, void **bfs_queue, ph_enumerate_vars_t *vars) { + if (vars->queue_size == 0) { + return NULL; + } + + (vars->visited_num)++; + if (vars->visited_num > vars->max_visit_num) { + return NULL; + } + + void *ret = ph_enumerate_queue_pop(bfs_queue, vars); + assert(ret != NULL); + void *left = phn_lchild_get(ret, offset); + void *right = phn_next_get(ret, offset); + if (left) { + ph_enumerate_queue_push(left, bfs_queue, vars); + } + if (right) { + ph_enumerate_queue_push(right, bfs_queue, vars); + } + return ret; +} + +#define ph_structs(a_prefix, a_type, a_max_queue_size) \ + typedef struct { \ + phn_link_t link; \ + } a_prefix##_link_t; \ + \ + typedef struct { \ + ph_t ph; \ + } a_prefix##_t; \ + \ + typedef struct { \ + void *bfs_queue[a_max_queue_size]; \ + ph_enumerate_vars_t vars; \ + } a_prefix##_enumerate_helper_t; /* * The ph_proto() macro generates function prototypes that correspond to the * functions generated by an equivalently parameterized call to ph_gen(). */ -#define ph_proto(a_attr, a_prefix, a_type) \ - \ -a_attr void a_prefix##_new(a_prefix##_t *ph); \ -a_attr bool a_prefix##_empty(a_prefix##_t *ph); \ -a_attr a_type *a_prefix##_first(a_prefix##_t *ph); \ -a_attr a_type *a_prefix##_any(a_prefix##_t *ph); \ -a_attr void a_prefix##_insert(a_prefix##_t *ph, a_type *phn); \ -a_attr a_type *a_prefix##_remove_first(a_prefix##_t *ph); \ -a_attr void a_prefix##_remove(a_prefix##_t *ph, a_type *phn); \ -a_attr a_type *a_prefix##_remove_any(a_prefix##_t *ph); +#define ph_proto(a_attr, a_prefix, a_type) \ + \ + a_attr void a_prefix##_new(a_prefix##_t *ph); \ + a_attr bool a_prefix##_empty(a_prefix##_t *ph); \ + a_attr a_type *a_prefix##_first(a_prefix##_t *ph); \ + a_attr a_type *a_prefix##_any(a_prefix##_t *ph); \ + a_attr void a_prefix##_insert(a_prefix##_t *ph, a_type *phn); \ + a_attr a_type *a_prefix##_remove_first(a_prefix##_t *ph); \ + a_attr void a_prefix##_remove(a_prefix##_t *ph, a_type *phn); \ + a_attr a_type *a_prefix##_remove_any(a_prefix##_t *ph); \ + a_attr void a_prefix##_enumerate_prepare(a_prefix##_t *ph, \ + a_prefix##_enumerate_helper_t *helper, uint16_t max_visit_num, \ + uint16_t max_queue_size); \ + a_attr a_type *a_prefix##_enumerate_next( \ + a_prefix##_t *ph, a_prefix##_enumerate_helper_t *helper); /* The ph_gen() macro generates a type-specific pairing heap implementation. */ -#define ph_gen(a_attr, a_prefix, a_type, a_field, a_cmp) \ -JEMALLOC_ALWAYS_INLINE int \ -a_prefix##_ph_cmp(void *a, void *b) { \ - return a_cmp((a_type *)a, (a_type *)b); \ -} \ - \ -a_attr void \ -a_prefix##_new(a_prefix##_t *ph) { \ - ph_new(&ph->ph); \ -} \ - \ -a_attr bool \ -a_prefix##_empty(a_prefix##_t *ph) { \ - return ph_empty(&ph->ph); \ -} \ - \ -a_attr a_type * \ -a_prefix##_first(a_prefix##_t *ph) { \ - return ph_first(&ph->ph, offsetof(a_type, a_field), \ - &a_prefix##_ph_cmp); \ -} \ - \ -a_attr a_type * \ -a_prefix##_any(a_prefix##_t *ph) { \ - return ph_any(&ph->ph, offsetof(a_type, a_field)); \ -} \ - \ -a_attr void \ -a_prefix##_insert(a_prefix##_t *ph, a_type *phn) { \ - ph_insert(&ph->ph, phn, offsetof(a_type, a_field), \ - a_prefix##_ph_cmp); \ -} \ - \ -a_attr a_type * \ -a_prefix##_remove_first(a_prefix##_t *ph) { \ - return ph_remove_first(&ph->ph, offsetof(a_type, a_field), \ - a_prefix##_ph_cmp); \ -} \ - \ -a_attr void \ -a_prefix##_remove(a_prefix##_t *ph, a_type *phn) { \ - ph_remove(&ph->ph, phn, offsetof(a_type, a_field), \ - a_prefix##_ph_cmp); \ -} \ - \ -a_attr a_type * \ -a_prefix##_remove_any(a_prefix##_t *ph) { \ - a_type *ret = a_prefix##_any(ph); \ - if (ret != NULL) { \ - a_prefix##_remove(ph, ret); \ - } \ - return ret; \ -} +#define ph_gen(a_attr, a_prefix, a_type, a_field, a_cmp) \ + JEMALLOC_ALWAYS_INLINE int a_prefix##_ph_cmp(void *a, void *b) { \ + return a_cmp((a_type *)a, (a_type *)b); \ + } \ + \ + a_attr void a_prefix##_new(a_prefix##_t *ph) { \ + ph_new(&ph->ph); \ + } \ + \ + a_attr bool a_prefix##_empty(a_prefix##_t *ph) { \ + return ph_empty(&ph->ph); \ + } \ + \ + a_attr a_type *a_prefix##_first(a_prefix##_t *ph) { \ + return ph_first( \ + &ph->ph, offsetof(a_type, a_field), &a_prefix##_ph_cmp); \ + } \ + \ + a_attr a_type *a_prefix##_any(a_prefix##_t *ph) { \ + return ph_any(&ph->ph, offsetof(a_type, a_field)); \ + } \ + \ + a_attr void a_prefix##_insert(a_prefix##_t *ph, a_type *phn) { \ + ph_insert(&ph->ph, phn, offsetof(a_type, a_field), \ + a_prefix##_ph_cmp); \ + } \ + \ + a_attr a_type *a_prefix##_remove_first(a_prefix##_t *ph) { \ + return ph_remove_first( \ + &ph->ph, offsetof(a_type, a_field), a_prefix##_ph_cmp); \ + } \ + \ + a_attr void a_prefix##_remove(a_prefix##_t *ph, a_type *phn) { \ + ph_remove(&ph->ph, phn, offsetof(a_type, a_field), \ + a_prefix##_ph_cmp); \ + } \ + \ + a_attr a_type *a_prefix##_remove_any(a_prefix##_t *ph) { \ + a_type *ret = a_prefix##_any(ph); \ + if (ret != NULL) { \ + a_prefix##_remove(ph, ret); \ + } \ + return ret; \ + } \ + \ + a_attr void a_prefix##_enumerate_prepare(a_prefix##_t *ph, \ + a_prefix##_enumerate_helper_t *helper, uint16_t max_visit_num, \ + uint16_t max_queue_size) { \ + ph_enumerate_prepare(&ph->ph, helper->bfs_queue, \ + &helper->vars, max_visit_num, max_queue_size); \ + } \ + \ + a_attr a_type *a_prefix##_enumerate_next( \ + a_prefix##_t *ph, a_prefix##_enumerate_helper_t *helper) { \ + return ph_enumerate_next(&ph->ph, offsetof(a_type, a_field), \ + helper->bfs_queue, &helper->vars); \ + } #endif /* JEMALLOC_INTERNAL_PH_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/private_namespace.h b/dep/jemalloc/include/jemalloc/internal/private_namespace.h index 2bc15848a..141818a5d 100644 --- a/dep/jemalloc/include/jemalloc/internal/private_namespace.h +++ b/dep/jemalloc/include/jemalloc/internal/private_namespace.h @@ -6,17 +6,16 @@ #define arena_migrate JEMALLOC_N(arena_migrate) #define arenas JEMALLOC_N(arenas) #define arena_set JEMALLOC_N(arena_set) -#define arenas_lock JEMALLOC_N(arenas_lock) #define batch_alloc JEMALLOC_N(batch_alloc) #define bootstrap_calloc JEMALLOC_N(bootstrap_calloc) #define bootstrap_free JEMALLOC_N(bootstrap_free) #define bootstrap_malloc JEMALLOC_N(bootstrap_malloc) #define free_default JEMALLOC_N(free_default) #define iarena_cleanup JEMALLOC_N(iarena_cleanup) +#define invalid_conf_abort JEMALLOC_N(invalid_conf_abort) #define jemalloc_postfork_child JEMALLOC_N(jemalloc_postfork_child) #define jemalloc_postfork_parent JEMALLOC_N(jemalloc_postfork_parent) #define jemalloc_prefork JEMALLOC_N(jemalloc_prefork) -#define je_sdallocx_noflags JEMALLOC_N(je_sdallocx_noflags) #define junk_alloc_callback JEMALLOC_N(junk_alloc_callback) #define junk_free_callback JEMALLOC_N(junk_free_callback) #define malloc_default JEMALLOC_N(malloc_default) @@ -29,14 +28,20 @@ #define opt_abort JEMALLOC_N(opt_abort) #define opt_abort_conf JEMALLOC_N(opt_abort_conf) #define opt_cache_oblivious JEMALLOC_N(opt_cache_oblivious) +#define opt_calloc_madvise_threshold JEMALLOC_N(opt_calloc_madvise_threshold) #define opt_confirm_conf JEMALLOC_N(opt_confirm_conf) +#define opt_debug_double_free_max_scan JEMALLOC_N(opt_debug_double_free_max_scan) +#define opt_disable_large_size_classes JEMALLOC_N(opt_disable_large_size_classes) #define opt_experimental_infallible_new JEMALLOC_N(opt_experimental_infallible_new) +#define opt_experimental_tcache_gc JEMALLOC_N(opt_experimental_tcache_gc) #define opt_hpa JEMALLOC_N(opt_hpa) #define opt_hpa_opts JEMALLOC_N(opt_hpa_opts) #define opt_hpa_sec_opts JEMALLOC_N(opt_hpa_sec_opts) #define opt_junk JEMALLOC_N(opt_junk) #define opt_junk_alloc JEMALLOC_N(opt_junk_alloc) #define opt_junk_free JEMALLOC_N(opt_junk_free) +#define opt_malloc_conf_env_var JEMALLOC_N(opt_malloc_conf_env_var) +#define opt_malloc_conf_symlink JEMALLOC_N(opt_malloc_conf_symlink) #define opt_narenas JEMALLOC_N(opt_narenas) #define opt_narenas_ratio JEMALLOC_N(opt_narenas_ratio) #define opt_trust_madvise JEMALLOC_N(opt_trust_madvise) @@ -48,15 +53,11 @@ #define zero_realloc_count JEMALLOC_N(zero_realloc_count) #define zero_realloc_mode_names JEMALLOC_N(zero_realloc_mode_names) #define arena_basic_stats_merge JEMALLOC_N(arena_basic_stats_merge) -#define arena_bin_choose JEMALLOC_N(arena_bin_choose) #define arena_binind_div_info JEMALLOC_N(arena_binind_div_info) #define arena_bin_offsets JEMALLOC_N(arena_bin_offsets) #define arena_boot JEMALLOC_N(arena_boot) -#define arena_cache_bin_fill_small JEMALLOC_N(arena_cache_bin_fill_small) #define arena_choose_huge JEMALLOC_N(arena_choose_huge) #define arena_config_default JEMALLOC_N(arena_config_default) -#define arena_dalloc_bin_locked_handle_newly_empty JEMALLOC_N(arena_dalloc_bin_locked_handle_newly_empty) -#define arena_dalloc_bin_locked_handle_newly_nonempty JEMALLOC_N(arena_dalloc_bin_locked_handle_newly_nonempty) #define arena_dalloc_promoted JEMALLOC_N(arena_dalloc_promoted) #define arena_dalloc_small JEMALLOC_N(arena_dalloc_small) #define arena_decay JEMALLOC_N(arena_decay) @@ -77,15 +78,15 @@ #define arena_get_ehooks JEMALLOC_N(arena_get_ehooks) #define arena_handle_deferred_work JEMALLOC_N(arena_handle_deferred_work) #define arena_init_huge JEMALLOC_N(arena_init_huge) -#define arena_is_huge JEMALLOC_N(arena_is_huge) #define arena_malloc_hard JEMALLOC_N(arena_malloc_hard) #define arena_muzzy_decay_ms_default_get JEMALLOC_N(arena_muzzy_decay_ms_default_get) #define arena_muzzy_decay_ms_default_set JEMALLOC_N(arena_muzzy_decay_ms_default_set) +#define arena_name_get JEMALLOC_N(arena_name_get) +#define arena_name_set JEMALLOC_N(arena_name_set) #define arena_new JEMALLOC_N(arena_new) #define arena_nthreads_dec JEMALLOC_N(arena_nthreads_dec) #define arena_nthreads_get JEMALLOC_N(arena_nthreads_get) #define arena_nthreads_inc JEMALLOC_N(arena_nthreads_inc) -#define arena_pa_central_global JEMALLOC_N(arena_pa_central_global) #define arena_palloc JEMALLOC_N(arena_palloc) #define arena_postfork_child JEMALLOC_N(arena_postfork_child) #define arena_postfork_parent JEMALLOC_N(arena_postfork_parent) @@ -99,6 +100,8 @@ #define arena_prefork7 JEMALLOC_N(arena_prefork7) #define arena_prefork8 JEMALLOC_N(arena_prefork8) #define arena_prof_promote JEMALLOC_N(arena_prof_promote) +#define arena_ptr_array_fill_small JEMALLOC_N(arena_ptr_array_fill_small) +#define arena_ptr_array_flush JEMALLOC_N(arena_ptr_array_flush) #define arena_ralloc JEMALLOC_N(arena_ralloc) #define arena_ralloc_no_move JEMALLOC_N(arena_ralloc_no_move) #define arena_reset JEMALLOC_N(arena_reset) @@ -106,7 +109,10 @@ #define arena_set_extent_hooks JEMALLOC_N(arena_set_extent_hooks) #define arena_slab_dalloc JEMALLOC_N(arena_slab_dalloc) #define arena_stats_merge JEMALLOC_N(arena_stats_merge) +#define huge_arena_ind JEMALLOC_N(huge_arena_ind) +#define huge_arena_pac_thp JEMALLOC_N(huge_arena_pac_thp) #define opt_dirty_decay_ms JEMALLOC_N(opt_dirty_decay_ms) +#define opt_huge_arena_pac_thp JEMALLOC_N(opt_huge_arena_pac_thp) #define opt_muzzy_decay_ms JEMALLOC_N(opt_muzzy_decay_ms) #define opt_oversize_threshold JEMALLOC_N(opt_oversize_threshold) #define opt_percpu_arena JEMALLOC_N(opt_percpu_arena) @@ -133,9 +139,12 @@ #define opt_background_thread JEMALLOC_N(opt_background_thread) #define opt_max_background_threads JEMALLOC_N(opt_max_background_threads) #define pthread_create_wrapper JEMALLOC_N(pthread_create_wrapper) +#define b0_alloc_tcache_stack JEMALLOC_N(b0_alloc_tcache_stack) +#define b0_dalloc_tcache_stack JEMALLOC_N(b0_dalloc_tcache_stack) #define b0get JEMALLOC_N(b0get) #define base_alloc JEMALLOC_N(base_alloc) #define base_alloc_edata JEMALLOC_N(base_alloc_edata) +#define base_alloc_rtree JEMALLOC_N(base_alloc_rtree) #define base_boot JEMALLOC_N(base_boot) #define base_delete JEMALLOC_N(base_delete) #define base_ehooks_get JEMALLOC_N(base_ehooks_get) @@ -148,11 +157,28 @@ #define base_stats_get JEMALLOC_N(base_stats_get) #define metadata_thp_mode_names JEMALLOC_N(metadata_thp_mode_names) #define opt_metadata_thp JEMALLOC_N(opt_metadata_thp) +#define bin_choose JEMALLOC_N(bin_choose) +#define bin_dalloc_locked_handle_newly_empty JEMALLOC_N(bin_dalloc_locked_handle_newly_empty) +#define bin_dalloc_locked_handle_newly_nonempty JEMALLOC_N(bin_dalloc_locked_handle_newly_nonempty) +#define bin_dalloc_slab_prepare JEMALLOC_N(bin_dalloc_slab_prepare) +#define bin_dissociate_slab JEMALLOC_N(bin_dissociate_slab) #define bin_init JEMALLOC_N(bin_init) +#define bin_lower_slab JEMALLOC_N(bin_lower_slab) +#define bin_malloc_no_fresh_slab JEMALLOC_N(bin_malloc_no_fresh_slab) +#define bin_malloc_with_fresh_slab JEMALLOC_N(bin_malloc_with_fresh_slab) #define bin_postfork_child JEMALLOC_N(bin_postfork_child) #define bin_postfork_parent JEMALLOC_N(bin_postfork_parent) #define bin_prefork JEMALLOC_N(bin_prefork) +#define bin_refill_slabcur_no_fresh_slab JEMALLOC_N(bin_refill_slabcur_no_fresh_slab) +#define bin_refill_slabcur_with_fresh_slab JEMALLOC_N(bin_refill_slabcur_with_fresh_slab) #define bin_shard_sizes_boot JEMALLOC_N(bin_shard_sizes_boot) +#define bin_slab_reg_alloc JEMALLOC_N(bin_slab_reg_alloc) +#define bin_slab_reg_alloc_batch JEMALLOC_N(bin_slab_reg_alloc_batch) +#define bin_slabs_full_insert JEMALLOC_N(bin_slabs_full_insert) +#define bin_slabs_full_remove JEMALLOC_N(bin_slabs_full_remove) +#define bin_slabs_nonfull_insert JEMALLOC_N(bin_slabs_nonfull_insert) +#define bin_slabs_nonfull_remove JEMALLOC_N(bin_slabs_nonfull_remove) +#define bin_slabs_nonfull_tryget JEMALLOC_N(bin_slabs_nonfull_tryget) #define bin_update_shard_size JEMALLOC_N(bin_update_shard_size) #define bin_info_boot JEMALLOC_N(bin_info_boot) #define bin_infos JEMALLOC_N(bin_infos) @@ -167,9 +193,11 @@ #define cache_bin_info_compute_alloc JEMALLOC_N(cache_bin_info_compute_alloc) #define cache_bin_info_init JEMALLOC_N(cache_bin_info_init) #define cache_bin_init JEMALLOC_N(cache_bin_init) +#define cache_bin_init_disabled JEMALLOC_N(cache_bin_init_disabled) #define cache_bin_postincrement JEMALLOC_N(cache_bin_postincrement) #define cache_bin_preincrement JEMALLOC_N(cache_bin_preincrement) -#define cache_bin_still_zero_initialized JEMALLOC_N(cache_bin_still_zero_initialized) +#define cache_bin_stack_use_thp JEMALLOC_N(cache_bin_stack_use_thp) +#define disabled_bin JEMALLOC_N(disabled_bin) #define ckh_count JEMALLOC_N(ckh_count) #define ckh_delete JEMALLOC_N(ckh_delete) #define ckh_insert JEMALLOC_N(ckh_insert) @@ -195,7 +223,6 @@ #define ctl_postfork_child JEMALLOC_N(ctl_postfork_child) #define ctl_postfork_parent JEMALLOC_N(ctl_postfork_parent) #define ctl_prefork JEMALLOC_N(ctl_prefork) -#define decay_deadline_init JEMALLOC_N(decay_deadline_init) #define decay_init JEMALLOC_N(decay_init) #define decay_maybe_advance_epoch JEMALLOC_N(decay_maybe_advance_epoch) #define decay_ms_valid JEMALLOC_N(decay_ms_valid) @@ -209,6 +236,8 @@ #define ecache_prefork JEMALLOC_N(ecache_prefork) #define edata_avail_any JEMALLOC_N(edata_avail_any) #define edata_avail_empty JEMALLOC_N(edata_avail_empty) +#define edata_avail_enumerate_next JEMALLOC_N(edata_avail_enumerate_next) +#define edata_avail_enumerate_prepare JEMALLOC_N(edata_avail_enumerate_prepare) #define edata_avail_first JEMALLOC_N(edata_avail_first) #define edata_avail_insert JEMALLOC_N(edata_avail_insert) #define edata_avail_new JEMALLOC_N(edata_avail_new) @@ -217,6 +246,8 @@ #define edata_avail_remove_first JEMALLOC_N(edata_avail_remove_first) #define edata_heap_any JEMALLOC_N(edata_heap_any) #define edata_heap_empty JEMALLOC_N(edata_heap_empty) +#define edata_heap_enumerate_next JEMALLOC_N(edata_heap_enumerate_next) +#define edata_heap_enumerate_prepare JEMALLOC_N(edata_heap_enumerate_prepare) #define edata_heap_first JEMALLOC_N(edata_heap_first) #define edata_heap_insert JEMALLOC_N(edata_heap_insert) #define edata_heap_new JEMALLOC_N(edata_heap_new) @@ -278,11 +309,10 @@ #define ecache_evict JEMALLOC_N(ecache_evict) #define extent_alloc_wrapper JEMALLOC_N(extent_alloc_wrapper) #define extent_boot JEMALLOC_N(extent_boot) -#define extent_commit_wrapper JEMALLOC_N(extent_commit_wrapper) #define extent_commit_zero JEMALLOC_N(extent_commit_zero) #define extent_dalloc_gap JEMALLOC_N(extent_dalloc_gap) #define extent_dalloc_wrapper JEMALLOC_N(extent_dalloc_wrapper) -#define extent_decommit_wrapper JEMALLOC_N(extent_decommit_wrapper) +#define extent_dalloc_wrapper_purged JEMALLOC_N(extent_dalloc_wrapper_purged) #define extent_destroy_wrapper JEMALLOC_N(extent_destroy_wrapper) #define extent_gdump_add JEMALLOC_N(extent_gdump_add) #define extent_merge_wrapper JEMALLOC_N(extent_merge_wrapper) @@ -292,6 +322,7 @@ #define extent_sn_next JEMALLOC_N(extent_sn_next) #define extent_split_wrapper JEMALLOC_N(extent_split_wrapper) #define opt_lg_extent_max_active_fit JEMALLOC_N(opt_lg_extent_max_active_fit) +#define opt_process_madvise_max_batch JEMALLOC_N(opt_process_madvise_max_batch) #define dss_prec_names JEMALLOC_N(dss_prec_names) #define extent_alloc_dss JEMALLOC_N(extent_alloc_dss) #define extent_dss_boot JEMALLOC_N(extent_dss_boot) @@ -322,23 +353,32 @@ #define hook_invoke_dalloc JEMALLOC_N(hook_invoke_dalloc) #define hook_invoke_expand JEMALLOC_N(hook_invoke_expand) #define hook_remove JEMALLOC_N(hook_remove) -#define hpa_central_extract JEMALLOC_N(hpa_central_extract) -#define hpa_central_init JEMALLOC_N(hpa_central_init) +#define hpa_hugepage_size_exceeds_limit JEMALLOC_N(hpa_hugepage_size_exceeds_limit) +#define hpa_hugify_style_names JEMALLOC_N(hpa_hugify_style_names) #define hpa_shard_destroy JEMALLOC_N(hpa_shard_destroy) #define hpa_shard_disable JEMALLOC_N(hpa_shard_disable) #define hpa_shard_do_deferred_work JEMALLOC_N(hpa_shard_do_deferred_work) +#define hpa_shard_flush JEMALLOC_N(hpa_shard_flush) #define hpa_shard_init JEMALLOC_N(hpa_shard_init) #define hpa_shard_postfork_child JEMALLOC_N(hpa_shard_postfork_child) #define hpa_shard_postfork_parent JEMALLOC_N(hpa_shard_postfork_parent) +#define hpa_shard_prefork2 JEMALLOC_N(hpa_shard_prefork2) #define hpa_shard_prefork3 JEMALLOC_N(hpa_shard_prefork3) #define hpa_shard_prefork4 JEMALLOC_N(hpa_shard_prefork4) #define hpa_shard_set_deferral_allowed JEMALLOC_N(hpa_shard_set_deferral_allowed) #define hpa_shard_stats_accum JEMALLOC_N(hpa_shard_stats_accum) #define hpa_shard_stats_merge JEMALLOC_N(hpa_shard_stats_merge) #define hpa_supported JEMALLOC_N(hpa_supported) +#define opt_experimental_hpa_enforce_hugify JEMALLOC_N(opt_experimental_hpa_enforce_hugify) +#define opt_experimental_hpa_start_huge_if_thp_always JEMALLOC_N(opt_experimental_hpa_start_huge_if_thp_always) +#define hpa_central_extract JEMALLOC_N(hpa_central_extract) +#define hpa_central_init JEMALLOC_N(hpa_central_init) #define hpa_hooks_default JEMALLOC_N(hpa_hooks_default) +#define hpa_purge_batch JEMALLOC_N(hpa_purge_batch) #define hpdata_age_heap_any JEMALLOC_N(hpdata_age_heap_any) #define hpdata_age_heap_empty JEMALLOC_N(hpdata_age_heap_empty) +#define hpdata_age_heap_enumerate_next JEMALLOC_N(hpdata_age_heap_enumerate_next) +#define hpdata_age_heap_enumerate_prepare JEMALLOC_N(hpdata_age_heap_enumerate_prepare) #define hpdata_age_heap_first JEMALLOC_N(hpdata_age_heap_first) #define hpdata_age_heap_insert JEMALLOC_N(hpdata_age_heap_insert) #define hpdata_age_heap_new JEMALLOC_N(hpdata_age_heap_new) @@ -365,19 +405,23 @@ #define large_prof_tctx_reset JEMALLOC_N(large_prof_tctx_reset) #define large_ralloc JEMALLOC_N(large_ralloc) #define large_ralloc_no_move JEMALLOC_N(large_ralloc_no_move) -#define large_salloc JEMALLOC_N(large_salloc) #define log_init_done JEMALLOC_N(log_init_done) #define log_var_names JEMALLOC_N(log_var_names) #define log_var_update_state JEMALLOC_N(log_var_update_state) #define buferror JEMALLOC_N(buferror) #define malloc_cprintf JEMALLOC_N(malloc_cprintf) #define malloc_printf JEMALLOC_N(malloc_printf) +#define malloc_read_fd JEMALLOC_N(malloc_read_fd) #define malloc_snprintf JEMALLOC_N(malloc_snprintf) #define malloc_strtoumax JEMALLOC_N(malloc_strtoumax) #define malloc_vcprintf JEMALLOC_N(malloc_vcprintf) #define malloc_vsnprintf JEMALLOC_N(malloc_vsnprintf) #define malloc_write JEMALLOC_N(malloc_write) +#define malloc_write_fd JEMALLOC_N(malloc_write_fd) #define wrtmessage JEMALLOC_N(wrtmessage) +#define had_conf_error JEMALLOC_N(had_conf_error) +#define malloc_abort_invalid_conf JEMALLOC_N(malloc_abort_invalid_conf) +#define malloc_conf_init JEMALLOC_N(malloc_conf_init) #define malloc_mutex_boot JEMALLOC_N(malloc_mutex_boot) #define malloc_mutex_init JEMALLOC_N(malloc_mutex_init) #define malloc_mutex_lock_slow JEMALLOC_N(malloc_mutex_lock_slow) @@ -398,8 +442,11 @@ #define nstime_init_update JEMALLOC_N(nstime_init_update) #define nstime_isubtract JEMALLOC_N(nstime_isubtract) #define nstime_monotonic JEMALLOC_N(nstime_monotonic) -#define nstime_msec JEMALLOC_N(nstime_msec) +#define nstime_ms JEMALLOC_N(nstime_ms) +#define nstime_ms_between JEMALLOC_N(nstime_ms_between) +#define nstime_ms_since JEMALLOC_N(nstime_ms_since) #define nstime_ns JEMALLOC_N(nstime_ns) +#define nstime_ns_between JEMALLOC_N(nstime_ns_between) #define nstime_nsec JEMALLOC_N(nstime_nsec) #define nstime_ns_since JEMALLOC_N(nstime_ns_since) #define nstime_prof_init_update JEMALLOC_N(nstime_prof_init_update) @@ -419,14 +466,17 @@ #define pa_shard_disable_hpa JEMALLOC_N(pa_shard_disable_hpa) #define pa_shard_do_deferred_work JEMALLOC_N(pa_shard_do_deferred_work) #define pa_shard_enable_hpa JEMALLOC_N(pa_shard_enable_hpa) +#define pa_shard_flush JEMALLOC_N(pa_shard_flush) #define pa_shard_init JEMALLOC_N(pa_shard_init) #define pa_shard_reset JEMALLOC_N(pa_shard_reset) -#define pa_shard_retain_grow_limit_get_set JEMALLOC_N(pa_shard_retain_grow_limit_get_set) #define pa_shard_set_deferral_allowed JEMALLOC_N(pa_shard_set_deferral_allowed) #define pa_shard_time_until_deferred_work JEMALLOC_N(pa_shard_time_until_deferred_work) #define pa_shrink JEMALLOC_N(pa_shrink) #define pa_shard_basic_stats_merge JEMALLOC_N(pa_shard_basic_stats_merge) #define pa_shard_mtx_stats_read JEMALLOC_N(pa_shard_mtx_stats_read) +#define pa_shard_nactive JEMALLOC_N(pa_shard_nactive) +#define pa_shard_ndirty JEMALLOC_N(pa_shard_ndirty) +#define pa_shard_nmuzzy JEMALLOC_N(pa_shard_nmuzzy) #define pa_shard_postfork_child JEMALLOC_N(pa_shard_postfork_child) #define pa_shard_postfork_parent JEMALLOC_N(pa_shard_postfork_parent) #define pa_shard_prefork0 JEMALLOC_N(pa_shard_prefork0) @@ -435,8 +485,6 @@ #define pa_shard_prefork4 JEMALLOC_N(pa_shard_prefork4) #define pa_shard_prefork5 JEMALLOC_N(pa_shard_prefork5) #define pa_shard_stats_merge JEMALLOC_N(pa_shard_stats_merge) -#define pai_alloc_batch_default JEMALLOC_N(pai_alloc_batch_default) -#define pai_dalloc_batch_default JEMALLOC_N(pai_dalloc_batch_default) #define pac_decay_all JEMALLOC_N(pac_decay_all) #define pac_decay_ms_get JEMALLOC_N(pac_decay_ms_get) #define pac_decay_ms_set JEMALLOC_N(pac_decay_ms_set) @@ -447,7 +495,9 @@ #define pac_retain_grow_limit_get_set JEMALLOC_N(pac_retain_grow_limit_get_set) #define init_system_thp_mode JEMALLOC_N(init_system_thp_mode) #define opt_thp JEMALLOC_N(opt_thp) +#define os_page JEMALLOC_N(os_page) #define pages_boot JEMALLOC_N(pages_boot) +#define pages_collapse JEMALLOC_N(pages_collapse) #define pages_commit JEMALLOC_N(pages_commit) #define pages_decommit JEMALLOC_N(pages_decommit) #define pages_dodump JEMALLOC_N(pages_dodump) @@ -458,29 +508,28 @@ #define pages_nohuge JEMALLOC_N(pages_nohuge) #define pages_purge_forced JEMALLOC_N(pages_purge_forced) #define pages_purge_lazy JEMALLOC_N(pages_purge_lazy) +#define pages_purge_process_madvise JEMALLOC_N(pages_purge_process_madvise) #define pages_set_thp_state JEMALLOC_N(pages_set_thp_state) #define pages_unmap JEMALLOC_N(pages_unmap) #define pages_unmark_guards JEMALLOC_N(pages_unmark_guards) +#define system_thp_mode_names JEMALLOC_N(system_thp_mode_names) #define thp_mode_names JEMALLOC_N(thp_mode_names) -#define peak_alloc_event_handler JEMALLOC_N(peak_alloc_event_handler) -#define peak_alloc_new_event_wait JEMALLOC_N(peak_alloc_new_event_wait) -#define peak_alloc_postponed_event_wait JEMALLOC_N(peak_alloc_postponed_event_wait) -#define peak_dalloc_event_handler JEMALLOC_N(peak_dalloc_event_handler) -#define peak_dalloc_new_event_wait JEMALLOC_N(peak_dalloc_new_event_wait) -#define peak_dalloc_postponed_event_wait JEMALLOC_N(peak_dalloc_postponed_event_wait) #define peak_event_max JEMALLOC_N(peak_event_max) #define peak_event_update JEMALLOC_N(peak_event_update) #define peak_event_zero JEMALLOC_N(peak_event_zero) +#define peak_te_handler JEMALLOC_N(peak_te_handler) #define lg_prof_sample JEMALLOC_N(lg_prof_sample) #define opt_lg_prof_interval JEMALLOC_N(opt_lg_prof_interval) #define opt_lg_prof_sample JEMALLOC_N(opt_lg_prof_sample) #define opt_prof JEMALLOC_N(opt_prof) #define opt_prof_accum JEMALLOC_N(opt_prof_accum) #define opt_prof_active JEMALLOC_N(opt_prof_active) +#define opt_prof_bt_max JEMALLOC_N(opt_prof_bt_max) #define opt_prof_final JEMALLOC_N(opt_prof_final) #define opt_prof_gdump JEMALLOC_N(opt_prof_gdump) #define opt_prof_leak JEMALLOC_N(opt_prof_leak) #define opt_prof_leak_error JEMALLOC_N(opt_prof_leak_error) +#define opt_prof_pid_namespace JEMALLOC_N(opt_prof_pid_namespace) #define opt_prof_prefix JEMALLOC_N(opt_prof_prefix) #define opt_prof_sys_thread_name JEMALLOC_N(opt_prof_sys_thread_name) #define opt_prof_thread_active_init JEMALLOC_N(opt_prof_thread_active_init) @@ -489,14 +538,12 @@ #define prof_active_set JEMALLOC_N(prof_active_set) #define prof_active_state JEMALLOC_N(prof_active_state) #define prof_alloc_rollback JEMALLOC_N(prof_alloc_rollback) -#define prof_backtrace_hook JEMALLOC_N(prof_backtrace_hook) #define prof_backtrace_hook_get JEMALLOC_N(prof_backtrace_hook_get) #define prof_backtrace_hook_set JEMALLOC_N(prof_backtrace_hook_set) #define prof_boot0 JEMALLOC_N(prof_boot0) #define prof_boot1 JEMALLOC_N(prof_boot1) #define prof_boot2 JEMALLOC_N(prof_boot2) #define prof_booted JEMALLOC_N(prof_booted) -#define prof_dump_hook JEMALLOC_N(prof_dump_hook) #define prof_dump_hook_get JEMALLOC_N(prof_dump_hook_get) #define prof_dump_hook_set JEMALLOC_N(prof_dump_hook_set) #define prof_free_sampled_object JEMALLOC_N(prof_free_sampled_object) @@ -513,8 +560,12 @@ #define prof_prefork0 JEMALLOC_N(prof_prefork0) #define prof_prefork1 JEMALLOC_N(prof_prefork1) #define prof_sample_event_handler JEMALLOC_N(prof_sample_event_handler) +#define prof_sample_free_hook_get JEMALLOC_N(prof_sample_free_hook_get) +#define prof_sample_free_hook_set JEMALLOC_N(prof_sample_free_hook_set) +#define prof_sample_hook_get JEMALLOC_N(prof_sample_hook_get) +#define prof_sample_hook_set JEMALLOC_N(prof_sample_hook_set) #define prof_sample_new_event_wait JEMALLOC_N(prof_sample_new_event_wait) -#define prof_sample_postponed_event_wait JEMALLOC_N(prof_sample_postponed_event_wait) +#define prof_sample_te_handler JEMALLOC_N(prof_sample_te_handler) #define prof_tctx_create JEMALLOC_N(prof_tctx_create) #define prof_tdata_cleanup JEMALLOC_N(prof_tdata_cleanup) #define prof_tdata_init JEMALLOC_N(prof_tdata_init) @@ -525,6 +576,7 @@ #define prof_thread_active_set JEMALLOC_N(prof_thread_active_set) #define prof_thread_name_get JEMALLOC_N(prof_thread_name_get) #define prof_thread_name_set JEMALLOC_N(prof_thread_name_set) +#define tsd_prof_sample_event_wait_get JEMALLOC_N(tsd_prof_sample_event_wait_get) #define bt2gctx_mtx JEMALLOC_N(bt2gctx_mtx) #define gctx_locks JEMALLOC_N(gctx_locks) #define prof_bt_count JEMALLOC_N(prof_bt_count) @@ -541,7 +593,6 @@ #define prof_tdata_count JEMALLOC_N(prof_tdata_count) #define prof_tdata_detach JEMALLOC_N(prof_tdata_detach) #define prof_tdata_init_impl JEMALLOC_N(prof_tdata_init_impl) -#define prof_thread_name_alloc JEMALLOC_N(prof_thread_name_alloc) #define prof_thread_name_set_impl JEMALLOC_N(prof_thread_name_set_impl) #define prof_unbiased_sz JEMALLOC_N(prof_unbiased_sz) #define prof_unbias_map_init JEMALLOC_N(prof_unbias_map_init) @@ -552,7 +603,6 @@ #define prof_log_alloc_count JEMALLOC_N(prof_log_alloc_count) #define prof_log_bt_count JEMALLOC_N(prof_log_bt_count) #define prof_log_dummy_set JEMALLOC_N(prof_log_dummy_set) -#define prof_logging_state JEMALLOC_N(prof_logging_state) #define prof_log_init JEMALLOC_N(prof_log_init) #define prof_log_is_logging JEMALLOC_N(prof_log_is_logging) #define prof_log_rep_check JEMALLOC_N(prof_log_rep_check) @@ -574,6 +624,7 @@ #define prof_recent_alloc_reset JEMALLOC_N(prof_recent_alloc_reset) #define prof_recent_dump_mtx JEMALLOC_N(prof_recent_dump_mtx) #define prof_recent_init JEMALLOC_N(prof_recent_init) +#define prof_thread_stack_range JEMALLOC_N(prof_thread_stack_range) #define opt_prof_stats JEMALLOC_N(opt_prof_stats) #define prof_stats_dec JEMALLOC_N(prof_stats_dec) #define prof_stats_get_accum JEMALLOC_N(prof_stats_get_accum) @@ -583,7 +634,6 @@ #define bt_init JEMALLOC_N(bt_init) #define prof_backtrace JEMALLOC_N(prof_backtrace) #define prof_base JEMALLOC_N(prof_base) -#define prof_do_mock JEMALLOC_N(prof_do_mock) #define prof_dump_filename_mtx JEMALLOC_N(prof_dump_filename_mtx) #define prof_dump_open_file JEMALLOC_N(prof_dump_open_file) #define prof_dump_open_maps JEMALLOC_N(prof_dump_open_maps) @@ -618,7 +668,9 @@ #define sc_boot JEMALLOC_N(sc_boot) #define sc_data_init JEMALLOC_N(sc_data_init) #define sc_data_update_slab_size JEMALLOC_N(sc_data_update_slab_size) -#define sec_disable JEMALLOC_N(sec_disable) +#define sec_alloc JEMALLOC_N(sec_alloc) +#define sec_dalloc JEMALLOC_N(sec_dalloc) +#define sec_fill JEMALLOC_N(sec_fill) #define sec_flush JEMALLOC_N(sec_flush) #define sec_init JEMALLOC_N(sec_init) #define sec_mutex_stats_read JEMALLOC_N(sec_mutex_stats_read) @@ -626,16 +678,16 @@ #define sec_postfork_parent JEMALLOC_N(sec_postfork_parent) #define sec_prefork2 JEMALLOC_N(sec_prefork2) #define sec_stats_merge JEMALLOC_N(sec_stats_merge) -#define arena_mutex_names JEMALLOC_N(arena_mutex_names) -#define global_mutex_names JEMALLOC_N(global_mutex_names) #define opt_stats_interval JEMALLOC_N(opt_stats_interval) #define opt_stats_interval_opts JEMALLOC_N(opt_stats_interval_opts) #define opt_stats_print JEMALLOC_N(opt_stats_print) #define opt_stats_print_opts JEMALLOC_N(opt_stats_print_opts) #define stats_boot JEMALLOC_N(stats_boot) +#define stats_interval_accum_batch JEMALLOC_N(stats_interval_accum_batch) #define stats_interval_event_handler JEMALLOC_N(stats_interval_event_handler) #define stats_interval_new_event_wait JEMALLOC_N(stats_interval_new_event_wait) #define stats_interval_postponed_event_wait JEMALLOC_N(stats_interval_postponed_event_wait) +#define stats_interval_te_handler JEMALLOC_N(stats_interval_te_handler) #define stats_postfork_child JEMALLOC_N(stats_postfork_child) #define stats_postfork_parent JEMALLOC_N(stats_postfork_parent) #define stats_prefork JEMALLOC_N(stats_prefork) @@ -647,7 +699,8 @@ #define sz_psz_quantize_ceil JEMALLOC_N(sz_psz_quantize_ceil) #define sz_psz_quantize_floor JEMALLOC_N(sz_psz_quantize_floor) #define sz_size2index_tab JEMALLOC_N(sz_size2index_tab) -#define nhbins JEMALLOC_N(nhbins) +#define global_do_not_change_tcache_maxclass JEMALLOC_N(global_do_not_change_tcache_maxclass) +#define global_do_not_change_tcache_nbins JEMALLOC_N(global_do_not_change_tcache_nbins) #define opt_lg_tcache_flush_large_div JEMALLOC_N(opt_lg_tcache_flush_large_div) #define opt_lg_tcache_flush_small_div JEMALLOC_N(opt_lg_tcache_flush_small_div) #define opt_lg_tcache_nslots_mul JEMALLOC_N(opt_lg_tcache_nslots_mul) @@ -665,18 +718,17 @@ #define tcache_bin_flush_large JEMALLOC_N(tcache_bin_flush_large) #define tcache_bin_flush_small JEMALLOC_N(tcache_bin_flush_small) #define tcache_bin_flush_stashed JEMALLOC_N(tcache_bin_flush_stashed) -#define tcache_bin_info JEMALLOC_N(tcache_bin_info) +#define tcache_bin_info_default_init JEMALLOC_N(tcache_bin_info_default_init) +#define tcache_bin_ncached_max_read JEMALLOC_N(tcache_bin_ncached_max_read) +#define tcache_bins_ncached_max_write JEMALLOC_N(tcache_bins_ncached_max_write) #define tcache_boot JEMALLOC_N(tcache_boot) #define tcache_cleanup JEMALLOC_N(tcache_cleanup) #define tcache_create_explicit JEMALLOC_N(tcache_create_explicit) +#define tcache_enabled_set JEMALLOC_N(tcache_enabled_set) #define tcache_flush JEMALLOC_N(tcache_flush) -#define tcache_gc_dalloc_event_handler JEMALLOC_N(tcache_gc_dalloc_event_handler) -#define tcache_gc_dalloc_new_event_wait JEMALLOC_N(tcache_gc_dalloc_new_event_wait) -#define tcache_gc_dalloc_postponed_event_wait JEMALLOC_N(tcache_gc_dalloc_postponed_event_wait) -#define tcache_gc_event_handler JEMALLOC_N(tcache_gc_event_handler) #define tcache_gc_new_event_wait JEMALLOC_N(tcache_gc_new_event_wait) #define tcache_gc_postponed_event_wait JEMALLOC_N(tcache_gc_postponed_event_wait) -#define tcache_maxclass JEMALLOC_N(tcache_maxclass) +#define tcache_gc_te_handler JEMALLOC_N(tcache_gc_te_handler) #define tcache_postfork_child JEMALLOC_N(tcache_postfork_child) #define tcache_postfork_parent JEMALLOC_N(tcache_postfork_parent) #define tcache_prefork JEMALLOC_N(tcache_prefork) @@ -685,15 +737,22 @@ #define tcaches_create JEMALLOC_N(tcaches_create) #define tcaches_destroy JEMALLOC_N(tcaches_destroy) #define tcaches_flush JEMALLOC_N(tcaches_flush) +#define tcache_stack_alloc JEMALLOC_N(tcache_stack_alloc) #define tcache_stats_merge JEMALLOC_N(tcache_stats_merge) -#define tsd_tcache_data_init JEMALLOC_N(tsd_tcache_data_init) +#define thread_tcache_max_set JEMALLOC_N(thread_tcache_max_set) #define tsd_tcache_enabled_data_init JEMALLOC_N(tsd_tcache_enabled_data_init) #define test_hooks_arena_new_hook JEMALLOC_N(test_hooks_arena_new_hook) #define test_hooks_libc_hook JEMALLOC_N(test_hooks_libc_hook) +#define te_adjust_thresholds_helper JEMALLOC_N(te_adjust_thresholds_helper) #define te_assert_invariants_debug JEMALLOC_N(te_assert_invariants_debug) #define te_event_trigger JEMALLOC_N(te_event_trigger) #define te_recompute_fast_threshold JEMALLOC_N(te_recompute_fast_threshold) #define tsd_te_init JEMALLOC_N(tsd_te_init) +#define experimental_thread_events_boot JEMALLOC_N(experimental_thread_events_boot) +#define te_alloc_handlers JEMALLOC_N(te_alloc_handlers) +#define te_dalloc_handlers JEMALLOC_N(te_dalloc_handlers) +#define te_register_user_handler JEMALLOC_N(te_register_user_handler) +#define te_user_event_enabled JEMALLOC_N(te_user_event_enabled) #define ticker_geom_table JEMALLOC_N(ticker_geom_table) #define malloc_tsd_boot0 JEMALLOC_N(malloc_tsd_boot0) #define malloc_tsd_boot1 JEMALLOC_N(malloc_tsd_boot1) @@ -712,6 +771,7 @@ #define tsd_state_set JEMALLOC_N(tsd_state_set) #define tsd_tls JEMALLOC_N(tsd_tls) #define tsd_tsd JEMALLOC_N(tsd_tsd) +#define multi_setting_parse_next JEMALLOC_N(multi_setting_parse_next) #define witness_depth_error JEMALLOC_N(witness_depth_error) #define witnesses_cleanup JEMALLOC_N(witnesses_cleanup) #define witness_init JEMALLOC_N(witness_init) diff --git a/dep/jemalloc/include/jemalloc/internal/prng.h b/dep/jemalloc/include/jemalloc/internal/prng.h index 14542aa12..04049519a 100644 --- a/dep/jemalloc/include/jemalloc/internal/prng.h +++ b/dep/jemalloc/include/jemalloc/internal/prng.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_PRNG_H #define JEMALLOC_INTERNAL_PRNG_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/bit_util.h" /* @@ -25,11 +26,11 @@ /******************************************************************************/ /* INTERNAL DEFINITIONS -- IGNORE */ /******************************************************************************/ -#define PRNG_A_32 UINT32_C(1103515241) -#define PRNG_C_32 UINT32_C(12347) +#define PRNG_A_32 UINT32_C(1103515241) +#define PRNG_C_32 UINT32_C(12347) -#define PRNG_A_64 UINT64_C(6364136223846793005) -#define PRNG_C_64 UINT64_C(1442695040888963407) +#define PRNG_A_64 UINT64_C(6364136223846793005) +#define PRNG_C_64 UINT64_C(1442695040888963407) JEMALLOC_ALWAYS_INLINE uint32_t prng_state_next_u32(uint32_t state) { @@ -48,7 +49,7 @@ prng_state_next_zu(size_t state) { #elif LG_SIZEOF_PTR == 3 return (state * PRNG_A_64) + PRNG_C_64; #else -#error Unsupported pointer size +# error Unsupported pointer size #endif } diff --git a/dep/jemalloc/include/jemalloc/internal/prof_data.h b/dep/jemalloc/include/jemalloc/internal/prof_data.h index 4c8e22c76..0af5835c6 100644 --- a/dep/jemalloc/include/jemalloc/internal/prof_data.h +++ b/dep/jemalloc/include/jemalloc/internal/prof_data.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_PROF_DATA_H #define JEMALLOC_INTERNAL_PROF_DATA_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/mutex.h" extern malloc_mutex_t bt2gctx_mtx; @@ -16,22 +17,21 @@ extern size_t prof_shifted_unbiased_cnt[PROF_SC_NSIZES]; void prof_bt_hash(const void *key, size_t r_hash[2]); bool prof_bt_keycomp(const void *k1, const void *k2); -bool prof_data_init(tsd_t *tsd); +bool prof_data_init(tsd_t *tsd); prof_tctx_t *prof_lookup(tsd_t *tsd, prof_bt_t *bt); -char *prof_thread_name_alloc(tsd_t *tsd, const char *thread_name); -int prof_thread_name_set_impl(tsd_t *tsd, const char *thread_name); -void prof_unbias_map_init(); +int prof_thread_name_set_impl(tsd_t *tsd, const char *thread_name); +void prof_unbias_map_init(void); void prof_dump_impl(tsd_t *tsd, write_cb_t *prof_dump_write, void *cbopaque, prof_tdata_t *tdata, bool leakcheck); -prof_tdata_t * prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, +prof_tdata_t *prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim, char *thread_name, bool active); -void prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata); -void prof_reset(tsd_t *tsd, size_t lg_sample); -void prof_tctx_try_destroy(tsd_t *tsd, prof_tctx_t *tctx); +void prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata); +void prof_reset(tsd_t *tsd, size_t lg_sample); +void prof_tctx_try_destroy(tsd_t *tsd, prof_tctx_t *tctx); /* Used in unit tests. */ size_t prof_tdata_count(void); size_t prof_bt_count(void); -void prof_cnt_all(prof_cnt_t *cnt_all); +void prof_cnt_all(prof_cnt_t *cnt_all); #endif /* JEMALLOC_INTERNAL_PROF_DATA_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/prof_externs.h b/dep/jemalloc/include/jemalloc/internal/prof_externs.h index bdff1349a..e07e69f55 100644 --- a/dep/jemalloc/include/jemalloc/internal/prof_externs.h +++ b/dep/jemalloc/include/jemalloc/internal/prof_externs.h @@ -1,28 +1,35 @@ #ifndef JEMALLOC_INTERNAL_PROF_EXTERNS_H #define JEMALLOC_INTERNAL_PROF_EXTERNS_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/base.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/prof_hook.h" +#include "jemalloc/internal/thread_event_registry.h" -extern bool opt_prof; -extern bool opt_prof_active; -extern bool opt_prof_thread_active_init; -extern size_t opt_lg_prof_sample; /* Mean bytes between samples. */ -extern ssize_t opt_lg_prof_interval; /* lg(prof_interval). */ -extern bool opt_prof_gdump; /* High-water memory dumping. */ -extern bool opt_prof_final; /* Final profile dumping. */ -extern bool opt_prof_leak; /* Dump leak summary at exit. */ -extern bool opt_prof_leak_error; /* Exit with error code if memory leaked */ -extern bool opt_prof_accum; /* Report cumulative bytes. */ -extern bool opt_prof_log; /* Turn logging on at boot. */ -extern char opt_prof_prefix[ - /* Minimize memory bloat for non-prof builds. */ +extern bool opt_prof; +extern bool opt_prof_active; +extern bool opt_prof_thread_active_init; +extern unsigned opt_prof_bt_max; +extern size_t opt_lg_prof_sample; /* Mean bytes between samples. */ +extern ssize_t opt_lg_prof_interval; /* lg(prof_interval). */ +extern bool opt_prof_gdump; /* High-water memory dumping. */ +extern bool opt_prof_final; /* Final profile dumping. */ +extern bool opt_prof_leak; /* Dump leak summary at exit. */ +extern bool opt_prof_leak_error; /* Exit with error code if memory leaked */ +extern bool opt_prof_accum; /* Report cumulative bytes. */ +extern bool opt_prof_log; /* Turn logging on at boot. */ +extern char opt_prof_prefix[ +/* Minimize memory bloat for non-prof builds. */ #ifdef JEMALLOC_PROF PATH_MAX + #endif 1]; extern bool opt_prof_unbias; +/* Include pid namespace in profile file names. */ +extern bool opt_prof_pid_namespace; + /* For recording recent allocations */ extern ssize_t opt_prof_recent_alloc_max; @@ -49,47 +56,110 @@ extern size_t lg_prof_sample; extern bool prof_booted; -void prof_backtrace_hook_set(prof_backtrace_hook_t hook); -prof_backtrace_hook_t prof_backtrace_hook_get(); +void prof_backtrace_hook_set(prof_backtrace_hook_t hook); +prof_backtrace_hook_t prof_backtrace_hook_get(void); -void prof_dump_hook_set(prof_dump_hook_t hook); -prof_dump_hook_t prof_dump_hook_get(); +void prof_dump_hook_set(prof_dump_hook_t hook); +prof_dump_hook_t prof_dump_hook_get(void); + +void prof_sample_hook_set(prof_sample_hook_t hook); +prof_sample_hook_t prof_sample_hook_get(void); + +void prof_sample_free_hook_set(prof_sample_free_hook_t hook); +prof_sample_free_hook_t prof_sample_free_hook_get(void); /* Functions only accessed in prof_inlines.h */ prof_tdata_t *prof_tdata_init(tsd_t *tsd); prof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata); void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx); -void prof_malloc_sample_object(tsd_t *tsd, const void *ptr, size_t size, - size_t usize, prof_tctx_t *tctx); -void prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_info_t *prof_info); +void prof_malloc_sample_object( + tsd_t *tsd, const void *ptr, size_t size, size_t usize, prof_tctx_t *tctx); +void prof_free_sampled_object( + tsd_t *tsd, const void *ptr, size_t usize, prof_info_t *prof_info); prof_tctx_t *prof_tctx_create(tsd_t *tsd); -void prof_idump(tsdn_t *tsdn); -bool prof_mdump(tsd_t *tsd, const char *filename); -void prof_gdump(tsdn_t *tsdn); +void prof_idump(tsdn_t *tsdn); +bool prof_mdump(tsd_t *tsd, const char *filename); +void prof_gdump(tsdn_t *tsdn); -void prof_tdata_cleanup(tsd_t *tsd); -bool prof_active_get(tsdn_t *tsdn); -bool prof_active_set(tsdn_t *tsdn, bool active); +void prof_tdata_cleanup(tsd_t *tsd); +bool prof_active_get(tsdn_t *tsdn); +bool prof_active_set(tsdn_t *tsdn, bool active); const char *prof_thread_name_get(tsd_t *tsd); -int prof_thread_name_set(tsd_t *tsd, const char *thread_name); -bool prof_thread_active_get(tsd_t *tsd); -bool prof_thread_active_set(tsd_t *tsd, bool active); -bool prof_thread_active_init_get(tsdn_t *tsdn); -bool prof_thread_active_init_set(tsdn_t *tsdn, bool active_init); -bool prof_gdump_get(tsdn_t *tsdn); -bool prof_gdump_set(tsdn_t *tsdn, bool active); -void prof_boot0(void); -void prof_boot1(void); -bool prof_boot2(tsd_t *tsd, base_t *base); -void prof_prefork0(tsdn_t *tsdn); -void prof_prefork1(tsdn_t *tsdn); -void prof_postfork_parent(tsdn_t *tsdn); -void prof_postfork_child(tsdn_t *tsdn); +int prof_thread_name_set(tsd_t *tsd, const char *thread_name); +bool prof_thread_active_get(tsd_t *tsd); +bool prof_thread_active_set(tsd_t *tsd, bool active); +bool prof_thread_active_init_get(tsdn_t *tsdn); +bool prof_thread_active_init_set(tsdn_t *tsdn, bool active_init); +bool prof_gdump_get(tsdn_t *tsdn); +bool prof_gdump_set(tsdn_t *tsdn, bool active); +void prof_boot0(void); +void prof_boot1(void); +bool prof_boot2(tsd_t *tsd, base_t *base); +void prof_prefork0(tsdn_t *tsdn); +void prof_prefork1(tsdn_t *tsdn); +void prof_postfork_parent(tsdn_t *tsdn); +void prof_postfork_child(tsdn_t *tsdn); -/* Only accessed by thread event. */ uint64_t prof_sample_new_event_wait(tsd_t *tsd); -uint64_t prof_sample_postponed_event_wait(tsd_t *tsd); -void prof_sample_event_handler(tsd_t *tsd, uint64_t elapsed); +uint64_t tsd_prof_sample_event_wait_get(tsd_t *tsd); + +/* + * The lookahead functionality facilitates events to be able to lookahead, i.e. + * without touching the event counters, to determine whether an event would be + * triggered. The event counters are not advanced until the end of the + * allocation / deallocation calls, so the lookahead can be useful if some + * preparation work for some event must be done early in the allocation / + * deallocation calls. + * + * Currently only the profiling sampling event needs the lookahead + * functionality, so we don't yet define general purpose lookahead functions. + * + * Surplus is a terminology referring to the amount of bytes beyond what's + * needed for triggering an event, which can be a useful quantity to have in + * general when lookahead is being called. + * + * This function returns true if allocation of usize would go above the next + * trigger for prof event, and false otherwise. + * If function returns true surplus will contain number of bytes beyond that + * trigger. + */ + +JEMALLOC_ALWAYS_INLINE bool +te_prof_sample_event_lookahead_surplus( + tsd_t *tsd, size_t usize, size_t *surplus) { + if (surplus != NULL) { + /* + * This is a dead store: the surplus will be overwritten before + * any read. The initialization suppresses compiler warnings. + * Meanwhile, using SIZE_MAX to initialize is good for + * debugging purpose, because a valid surplus value is strictly + * less than usize, which is at most SIZE_MAX. + */ + *surplus = SIZE_MAX; + } + if (unlikely(!tsd_nominal(tsd) || tsd_reentrancy_level_get(tsd) > 0)) { + return false; + } + /* The subtraction is intentionally susceptible to underflow. */ + uint64_t accumbytes = tsd_thread_allocated_get(tsd) + usize + - tsd_thread_allocated_last_event_get(tsd); + uint64_t sample_wait = tsd_prof_sample_event_wait_get(tsd); + if (accumbytes < sample_wait) { + return false; + } + assert(accumbytes - sample_wait < (uint64_t)usize); + if (surplus != NULL) { + *surplus = (size_t)(accumbytes - sample_wait); + } + return true; +} + +JEMALLOC_ALWAYS_INLINE bool +te_prof_sample_event_lookahead(tsd_t *tsd, size_t usize) { + return te_prof_sample_event_lookahead_surplus(tsd, usize, NULL); +} + +extern te_base_cb_t prof_sample_te_handler; #endif /* JEMALLOC_INTERNAL_PROF_EXTERNS_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/prof_hook.h b/dep/jemalloc/include/jemalloc/internal/prof_hook.h index 150d19d3d..d5a9b0ff8 100644 --- a/dep/jemalloc/include/jemalloc/internal/prof_hook.h +++ b/dep/jemalloc/include/jemalloc/internal/prof_hook.h @@ -1,6 +1,8 @@ #ifndef JEMALLOC_INTERNAL_PROF_HOOK_H #define JEMALLOC_INTERNAL_PROF_HOOK_H +#include "jemalloc/internal/jemalloc_preamble.h" + /* * The hooks types of which are declared in this file are experimental and * undocumented, thus the typedefs are located in an 'internal' header. @@ -18,4 +20,11 @@ typedef void (*prof_backtrace_hook_t)(void **, unsigned *, unsigned); */ typedef void (*prof_dump_hook_t)(const char *filename); +/* ptr, size, backtrace vector, backtrace vector length, usize */ +typedef void (*prof_sample_hook_t)(const void *ptr, size_t size, + void **backtrace, unsigned backtrace_length, size_t usize); + +/* ptr, size */ +typedef void (*prof_sample_free_hook_t)(const void *, size_t); + #endif /* JEMALLOC_INTERNAL_PROF_HOOK_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/prof_inlines.h b/dep/jemalloc/include/jemalloc/internal/prof_inlines.h index a8e7e7fb6..4a36bd7a2 100644 --- a/dep/jemalloc/include/jemalloc/internal/prof_inlines.h +++ b/dep/jemalloc/include/jemalloc/internal/prof_inlines.h @@ -1,12 +1,17 @@ #ifndef JEMALLOC_INTERNAL_PROF_INLINES_H #define JEMALLOC_INTERNAL_PROF_INLINES_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/arena_inlines_b.h" +#include "jemalloc/internal/jemalloc_internal_inlines_c.h" +#include "jemalloc/internal/prof_externs.h" +#include "jemalloc/internal/prof_structs.h" #include "jemalloc/internal/safety_check.h" #include "jemalloc/internal/sz.h" #include "jemalloc/internal/thread_event.h" JEMALLOC_ALWAYS_INLINE void -prof_active_assert() { +prof_active_assert(void) { cassert(config_prof); /* * If opt_prof is off, then prof_active must always be off, regardless @@ -37,6 +42,22 @@ prof_gdump_get_unlocked(void) { return prof_gdump_val; } +JEMALLOC_ALWAYS_INLINE void +prof_thread_name_assert(prof_tdata_t *tdata) { + if (!config_debug) { + return; + } + prof_active_assert(); + + bool terminated = false; + for (unsigned i = 0; i < PROF_THREAD_NAME_MAX_LEN; i++) { + if (tdata->thread_name[i] == '\0') { + terminated = true; + } + } + assert(terminated); +} + JEMALLOC_ALWAYS_INLINE prof_tdata_t * prof_tdata_get(tsd_t *tsd, bool create) { prof_tdata_t *tdata; @@ -58,6 +79,10 @@ prof_tdata_get(tsd_t *tsd, bool create) { assert(tdata == NULL || tdata->attached); } + if (tdata != NULL) { + prof_thread_name_assert(tdata); + } + return tdata; } @@ -81,6 +106,11 @@ prof_info_get_and_reset_recent(tsd_t *tsd, const void *ptr, arena_prof_info_get(tsd, ptr, alloc_ctx, prof_info, true); } +JEMALLOC_ALWAYS_INLINE bool +prof_tctx_is_valid(const prof_tctx_t *tctx) { + return tctx != NULL && tctx != PROF_TCTX_SENTINEL; +} + JEMALLOC_ALWAYS_INLINE void prof_tctx_reset(tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx) { cassert(config_prof); @@ -101,7 +131,7 @@ JEMALLOC_ALWAYS_INLINE void prof_info_set(tsd_t *tsd, edata_t *edata, prof_tctx_t *tctx, size_t size) { cassert(config_prof); assert(edata != NULL); - assert((uintptr_t)tctx > (uintptr_t)1U); + assert(prof_tctx_is_valid(tctx)); arena_prof_info_set(tsd, edata, tctx, size); } @@ -134,9 +164,9 @@ JEMALLOC_ALWAYS_INLINE prof_tctx_t * prof_alloc_prep(tsd_t *tsd, bool prof_active, bool sample_event) { prof_tctx_t *ret; - if (!prof_active || - likely(prof_sample_should_skip(tsd, sample_event))) { - ret = (prof_tctx_t *)(uintptr_t)1U; + if (!prof_active + || likely(prof_sample_should_skip(tsd, sample_event))) { + ret = PROF_TCTX_SENTINEL; } else { ret = prof_tctx_create(tsd); } @@ -151,7 +181,7 @@ prof_malloc(tsd_t *tsd, const void *ptr, size_t size, size_t usize, assert(ptr != NULL); assert(usize == isalloc(tsd_tsdn(tsd), ptr)); - if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) { + if (unlikely(prof_tctx_is_valid(tctx))) { prof_malloc_sample_object(tsd, ptr, size, usize, tctx); } else { prof_tctx_reset(tsd, ptr, alloc_ctx); @@ -165,7 +195,7 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t size, size_t usize, bool sampled, old_sampled, moved; cassert(config_prof); - assert(ptr != NULL || (uintptr_t)tctx <= (uintptr_t)1U); + assert(ptr != NULL || !prof_tctx_is_valid(tctx)); if (prof_active && ptr != NULL) { assert(usize == isalloc(tsd_tsdn(tsd), ptr)); @@ -178,12 +208,12 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t size, size_t usize, * sample threshold. */ prof_alloc_rollback(tsd, tctx); - tctx = (prof_tctx_t *)(uintptr_t)1U; + tctx = PROF_TCTX_SENTINEL; } } - sampled = ((uintptr_t)tctx > (uintptr_t)1U); - old_sampled = ((uintptr_t)old_prof_info->alloc_tctx > (uintptr_t)1U); + sampled = prof_tctx_is_valid(tctx); + old_sampled = prof_tctx_is_valid(old_prof_info->alloc_tctx); moved = (ptr != old_ptr); if (unlikely(sampled)) { @@ -201,7 +231,7 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t size, size_t usize, } else { prof_info_t prof_info; prof_info_get(tsd, ptr, NULL, &prof_info); - assert((uintptr_t)prof_info.alloc_tctx == (uintptr_t)1U); + assert(prof_info.alloc_tctx == PROF_TCTX_SENTINEL); } /* @@ -212,31 +242,29 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t size, size_t usize, * counters. */ if (unlikely(old_sampled)) { - prof_free_sampled_object(tsd, old_usize, old_prof_info); + prof_free_sampled_object( + tsd, old_ptr, old_usize, old_prof_info); } } JEMALLOC_ALWAYS_INLINE size_t -prof_sample_align(size_t orig_align) { +prof_sample_align(size_t usize, size_t orig_align) { /* - * Enforce page alignment, so that sampled allocations can be identified + * Enforce alignment, so that sampled allocations can be identified * w/o metadata lookup. */ assert(opt_prof); - return (opt_cache_oblivious && orig_align < PAGE) ? PAGE : - orig_align; -} - -JEMALLOC_ALWAYS_INLINE bool -prof_sample_aligned(const void *ptr) { - return ((uintptr_t)ptr & PAGE_MASK) == 0; + return (orig_align < PROF_SAMPLE_ALIGNMENT + && (sz_can_use_slab(usize) || opt_cache_oblivious)) + ? PROF_SAMPLE_ALIGNMENT + : orig_align; } JEMALLOC_ALWAYS_INLINE bool prof_sampled(tsd_t *tsd, const void *ptr) { prof_info_t prof_info; prof_info_get(tsd, ptr, NULL, &prof_info); - bool sampled = (uintptr_t)prof_info.alloc_tctx > (uintptr_t)1U; + bool sampled = prof_tctx_is_valid(prof_info.alloc_tctx); if (sampled) { assert(prof_sample_aligned(ptr)); } @@ -244,18 +272,32 @@ prof_sampled(tsd_t *tsd, const void *ptr) { } JEMALLOC_ALWAYS_INLINE void -prof_free(tsd_t *tsd, const void *ptr, size_t usize, - emap_alloc_ctx_t *alloc_ctx) { +prof_free( + tsd_t *tsd, const void *ptr, size_t usize, emap_alloc_ctx_t *alloc_ctx) { prof_info_t prof_info; prof_info_get_and_reset_recent(tsd, ptr, alloc_ctx, &prof_info); cassert(config_prof); assert(usize == isalloc(tsd_tsdn(tsd), ptr)); - if (unlikely((uintptr_t)prof_info.alloc_tctx > (uintptr_t)1U)) { + if (unlikely(prof_tctx_is_valid(prof_info.alloc_tctx))) { assert(prof_sample_aligned(ptr)); - prof_free_sampled_object(tsd, usize, &prof_info); + prof_free_sampled_object(tsd, ptr, usize, &prof_info); } } +JEMALLOC_ALWAYS_INLINE bool +prof_thread_name_empty(prof_tdata_t *tdata) { + prof_active_assert(); + + return (tdata->thread_name[0] == '\0'); +} + +JEMALLOC_ALWAYS_INLINE void +prof_thread_name_clear(prof_tdata_t *tdata) { + prof_active_assert(); + + tdata->thread_name[0] = '\0'; +} + #endif /* JEMALLOC_INTERNAL_PROF_INLINES_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/prof_log.h b/dep/jemalloc/include/jemalloc/internal/prof_log.h index ccb557dde..d9b97dc87 100644 --- a/dep/jemalloc/include/jemalloc/internal/prof_log.h +++ b/dep/jemalloc/include/jemalloc/internal/prof_log.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_PROF_LOG_H #define JEMALLOC_INTERNAL_PROF_LOG_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/mutex.h" extern malloc_mutex_t log_mtx; @@ -12,9 +13,9 @@ bool prof_log_init(tsd_t *tsdn); size_t prof_log_bt_count(void); size_t prof_log_alloc_count(void); size_t prof_log_thr_count(void); -bool prof_log_is_logging(void); -bool prof_log_rep_check(void); -void prof_log_dummy_set(bool new_value); +bool prof_log_is_logging(void); +bool prof_log_rep_check(void); +void prof_log_dummy_set(bool new_value); bool prof_log_start(tsdn_t *tsdn, const char *filename); bool prof_log_stop(tsdn_t *tsdn); diff --git a/dep/jemalloc/include/jemalloc/internal/prof_recent.h b/dep/jemalloc/include/jemalloc/internal/prof_recent.h index df4102362..33649e6da 100644 --- a/dep/jemalloc/include/jemalloc/internal/prof_recent.h +++ b/dep/jemalloc/include/jemalloc/internal/prof_recent.h @@ -1,13 +1,17 @@ #ifndef JEMALLOC_INTERNAL_PROF_RECENT_H #define JEMALLOC_INTERNAL_PROF_RECENT_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/edata.h" +#include "jemalloc/internal/mutex.h" + extern malloc_mutex_t prof_recent_alloc_mtx; extern malloc_mutex_t prof_recent_dump_mtx; bool prof_recent_alloc_prepare(tsd_t *tsd, prof_tctx_t *tctx); void prof_recent_alloc(tsd_t *tsd, edata_t *edata, size_t size, size_t usize); void prof_recent_alloc_reset(tsd_t *tsd, edata_t *edata); -bool prof_recent_init(); +bool prof_recent_init(void); void edata_prof_recent_alloc_init(edata_t *edata); /* Used in unit tests. */ @@ -16,7 +20,7 @@ extern prof_recent_list_t prof_recent_alloc_list; edata_t *prof_recent_alloc_edata_get_no_lock_test(const prof_recent_t *node); prof_recent_t *edata_prof_recent_alloc_get_no_lock_test(const edata_t *edata); -ssize_t prof_recent_alloc_max_ctl_read(); +ssize_t prof_recent_alloc_max_ctl_read(void); ssize_t prof_recent_alloc_max_ctl_write(tsd_t *tsd, ssize_t max); void prof_recent_alloc_dump(tsd_t *tsd, write_cb_t *write_cb, void *cbopaque); diff --git a/dep/jemalloc/include/jemalloc/internal/prof_stats.h b/dep/jemalloc/include/jemalloc/internal/prof_stats.h index 7954e82de..c4d269e54 100644 --- a/dep/jemalloc/include/jemalloc/internal/prof_stats.h +++ b/dep/jemalloc/include/jemalloc/internal/prof_stats.h @@ -1,6 +1,9 @@ #ifndef JEMALLOC_INTERNAL_PROF_STATS_H #define JEMALLOC_INTERNAL_PROF_STATS_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/mutex.h" + typedef struct prof_stats_s prof_stats_t; struct prof_stats_s { uint64_t req_sum; diff --git a/dep/jemalloc/include/jemalloc/internal/prof_structs.h b/dep/jemalloc/include/jemalloc/internal/prof_structs.h index dd22115f6..d38b15eac 100644 --- a/dep/jemalloc/include/jemalloc/internal/prof_structs.h +++ b/dep/jemalloc/include/jemalloc/internal/prof_structs.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_PROF_STRUCTS_H #define JEMALLOC_INTERNAL_PROF_STRUCTS_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/ckh.h" #include "jemalloc/internal/edata.h" #include "jemalloc/internal/mutex.h" @@ -9,29 +10,29 @@ struct prof_bt_s { /* Backtrace, stored as len program counters. */ - void **vec; - unsigned len; + void **vec; + unsigned len; }; #ifdef JEMALLOC_PROF_LIBGCC /* Data structure passed to libgcc _Unwind_Backtrace() callback functions. */ typedef struct { - void **vec; - unsigned *len; - unsigned max; + void **vec; + unsigned *len; + unsigned max; } prof_unwind_data_t; #endif struct prof_cnt_s { /* Profiling counters. */ - uint64_t curobjs; - uint64_t curobjs_shifted_unbiased; - uint64_t curbytes; - uint64_t curbytes_unbiased; - uint64_t accumobjs; - uint64_t accumobjs_shifted_unbiased; - uint64_t accumbytes; - uint64_t accumbytes_unbiased; + uint64_t curobjs; + uint64_t curobjs_shifted_unbiased; + uint64_t curbytes; + uint64_t curbytes_unbiased; + uint64_t accumobjs; + uint64_t accumobjs_shifted_unbiased; + uint64_t accumbytes; + uint64_t accumbytes_unbiased; }; typedef enum { @@ -43,26 +44,26 @@ typedef enum { struct prof_tctx_s { /* Thread data for thread that performed the allocation. */ - prof_tdata_t *tdata; + prof_tdata_t *tdata; /* * Copy of tdata->thr_{uid,discrim}, necessary because tdata may be * defunct during teardown. */ - uint64_t thr_uid; - uint64_t thr_discrim; + uint64_t thr_uid; + uint64_t thr_discrim; /* * Reference count of how many times this tctx object is referenced in * recent allocation / deallocation records, protected by tdata->lock. */ - uint64_t recent_count; + uint64_t recent_count; /* Profiling counters, protected by tdata->lock. */ - prof_cnt_t cnts; + prof_cnt_t cnts; /* Associated global context. */ - prof_gctx_t *gctx; + prof_gctx_t *gctx; /* * UID that distinguishes multiple tctx's created by the same thread, @@ -77,40 +78,40 @@ struct prof_tctx_s { * threshold can be hit again before the first consumer finishes * executing prof_tctx_destroy(). */ - uint64_t tctx_uid; + uint64_t tctx_uid; /* Linkage into gctx's tctxs. */ - rb_node(prof_tctx_t) tctx_link; + rb_node(prof_tctx_t) tctx_link; /* * True during prof_alloc_prep()..prof_malloc_sample_object(), prevents * sample vs destroy race. */ - bool prepared; + bool prepared; /* Current dump-related state, protected by gctx->lock. */ - prof_tctx_state_t state; + prof_tctx_state_t state; /* * Copy of cnts snapshotted during early dump phase, protected by * dump_mtx. */ - prof_cnt_t dump_cnts; + prof_cnt_t dump_cnts; }; typedef rb_tree(prof_tctx_t) prof_tctx_tree_t; struct prof_info_s { /* Time when the allocation was made. */ - nstime_t alloc_time; + nstime_t alloc_time; /* Points to the prof_tctx_t corresponding to the allocation. */ - prof_tctx_t *alloc_tctx; + prof_tctx_t *alloc_tctx; /* Allocation request size. */ - size_t alloc_size; + size_t alloc_size; }; struct prof_gctx_s { /* Protects nlimbo, cnt_summed, and tctxs. */ - malloc_mutex_t *lock; + malloc_mutex_t *lock; /* * Number of threads that currently cause this gctx to be in a state of @@ -122,54 +123,48 @@ struct prof_gctx_s { * nlimbo must be 1 (single destroyer) in order to safely destroy the * gctx. */ - unsigned nlimbo; + unsigned nlimbo; /* * Tree of profile counters, one for each thread that has allocated in * this context. */ - prof_tctx_tree_t tctxs; + prof_tctx_tree_t tctxs; /* Linkage for tree of contexts to be dumped. */ - rb_node(prof_gctx_t) dump_link; + rb_node(prof_gctx_t) dump_link; /* Temporary storage for summation during dump. */ - prof_cnt_t cnt_summed; + prof_cnt_t cnt_summed; /* Associated backtrace. */ - prof_bt_t bt; + prof_bt_t bt; /* Backtrace vector, variable size, referred to by bt. */ - void *vec[1]; + void *vec[1]; }; typedef rb_tree(prof_gctx_t) prof_gctx_tree_t; struct prof_tdata_s { - malloc_mutex_t *lock; + malloc_mutex_t *lock; /* Monotonically increasing unique thread identifier. */ - uint64_t thr_uid; + uint64_t thr_uid; /* * Monotonically increasing discriminator among tdata structures * associated with the same thr_uid. */ - uint64_t thr_discrim; + uint64_t thr_discrim; - /* Included in heap profile dumps if non-NULL. */ - char *thread_name; - - bool attached; - bool expired; - - rb_node(prof_tdata_t) tdata_link; + rb_node(prof_tdata_t) tdata_link; /* * Counter used to initialize prof_tctx_t's tctx_uid. No locking is * necessary when incrementing this field, because only one thread ever * does so. */ - uint64_t tctx_uid_next; + uint64_t tctx_uid_next; /* * Hash of (prof_bt_t *)-->(prof_tctx_t *). Each thread tracks @@ -177,12 +172,15 @@ struct prof_tdata_s { * associated with thread-specific prof_tctx_t objects. Other threads * may write to prof_tctx_t contents when freeing associated objects. */ - ckh_t bt2tctx; + ckh_t bt2tctx; + + /* Included in heap profile dumps if has content. */ + char thread_name[PROF_THREAD_NAME_MAX_LEN]; /* State used to avoid dumping while operating on prof internals. */ - bool enq; - bool enq_idump; - bool enq_gdump; + bool enq; + bool enq_idump; + bool enq_gdump; /* * Set to true during an early dump phase for tdata's which are @@ -190,19 +188,22 @@ struct prof_tdata_s { * to false so that they aren't accidentally included in later dump * phases. */ - bool dumping; + bool dumping; /* * True if profiling is active for this tdata's thread * (thread.prof.active mallctl). */ - bool active; + bool active; + + bool attached; + bool expired; /* Temporary storage for summation during dump. */ - prof_cnt_t cnt_summed; + prof_cnt_t cnt_summed; /* Backtrace vector, used for calls to prof_backtrace(). */ - void *vec[PROF_BT_MAX]; + void **vec; }; typedef rb_tree(prof_tdata_t) prof_tdata_tree_t; @@ -211,9 +212,9 @@ struct prof_recent_s { nstime_t dalloc_time; ql_elm(prof_recent_t) link; - size_t size; - size_t usize; - atomic_p_t alloc_edata; /* NULL means allocation has been freed. */ + size_t size; + size_t usize; + atomic_p_t alloc_edata; /* NULL means allocation has been freed. */ prof_tctx_t *alloc_tctx; prof_tctx_t *dalloc_tctx; }; diff --git a/dep/jemalloc/include/jemalloc/internal/prof_sys.h b/dep/jemalloc/include/jemalloc/internal/prof_sys.h index 3d25a4295..0745b991c 100644 --- a/dep/jemalloc/include/jemalloc/internal/prof_sys.h +++ b/dep/jemalloc/include/jemalloc/internal/prof_sys.h @@ -1,30 +1,35 @@ #ifndef JEMALLOC_INTERNAL_PROF_SYS_H #define JEMALLOC_INTERNAL_PROF_SYS_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/base.h" +#include "jemalloc/internal/mutex.h" + extern malloc_mutex_t prof_dump_filename_mtx; -extern base_t *prof_base; +extern base_t *prof_base; void bt_init(prof_bt_t *bt, void **vec); void prof_backtrace(tsd_t *tsd, prof_bt_t *bt); -void prof_hooks_init(); -void prof_unwind_init(); +void prof_hooks_init(void); +void prof_unwind_init(void); void prof_sys_thread_name_fetch(tsd_t *tsd); -int prof_getpid(void); +int prof_getpid(void); void prof_get_default_filename(tsdn_t *tsdn, char *filename, uint64_t ind); bool prof_prefix_set(tsdn_t *tsdn, const char *prefix); void prof_fdump_impl(tsd_t *tsd); void prof_idump_impl(tsd_t *tsd); bool prof_mdump_impl(tsd_t *tsd, const char *filename); void prof_gdump_impl(tsd_t *tsd); +int prof_thread_stack_range(uintptr_t fp, uintptr_t *low, uintptr_t *high); /* Used in unit tests. */ -typedef int (prof_sys_thread_name_read_t)(char *buf, size_t limit); +typedef int(prof_sys_thread_name_read_t)(char *buf, size_t limit); extern prof_sys_thread_name_read_t *JET_MUTABLE prof_sys_thread_name_read; -typedef int (prof_dump_open_file_t)(const char *, int); +typedef int(prof_dump_open_file_t)(const char *, int); extern prof_dump_open_file_t *JET_MUTABLE prof_dump_open_file; -typedef ssize_t (prof_dump_write_file_t)(int, const void *, size_t); +typedef ssize_t(prof_dump_write_file_t)(int, const void *, size_t); extern prof_dump_write_file_t *JET_MUTABLE prof_dump_write_file; -typedef int (prof_dump_open_maps_t)(); +typedef int(prof_dump_open_maps_t)(void); extern prof_dump_open_maps_t *JET_MUTABLE prof_dump_open_maps; #endif /* JEMALLOC_INTERNAL_PROF_SYS_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/prof_types.h b/dep/jemalloc/include/jemalloc/internal/prof_types.h index ba6286548..7468885ef 100644 --- a/dep/jemalloc/include/jemalloc/internal/prof_types.h +++ b/dep/jemalloc/include/jemalloc/internal/prof_types.h @@ -1,75 +1,94 @@ #ifndef JEMALLOC_INTERNAL_PROF_TYPES_H #define JEMALLOC_INTERNAL_PROF_TYPES_H -typedef struct prof_bt_s prof_bt_t; -typedef struct prof_cnt_s prof_cnt_t; -typedef struct prof_tctx_s prof_tctx_t; -typedef struct prof_info_s prof_info_t; -typedef struct prof_gctx_s prof_gctx_t; -typedef struct prof_tdata_s prof_tdata_t; +typedef struct prof_bt_s prof_bt_t; +typedef struct prof_cnt_s prof_cnt_t; +typedef struct prof_tctx_s prof_tctx_t; +typedef struct prof_info_s prof_info_t; +typedef struct prof_gctx_s prof_gctx_t; +typedef struct prof_tdata_s prof_tdata_t; typedef struct prof_recent_s prof_recent_t; /* Option defaults. */ #ifdef JEMALLOC_PROF -# define PROF_PREFIX_DEFAULT "jeprof" +# define PROF_PREFIX_DEFAULT "jeprof" #else -# define PROF_PREFIX_DEFAULT "" +# define PROF_PREFIX_DEFAULT "" #endif -#define LG_PROF_SAMPLE_DEFAULT 19 -#define LG_PROF_INTERVAL_DEFAULT -1 +#define LG_PROF_SAMPLE_DEFAULT 19 +#define LG_PROF_INTERVAL_DEFAULT -1 /* * Hard limit on stack backtrace depth. The version of prof_backtrace() that * is based on __builtin_return_address() necessarily has a hard-coded number * of backtrace frame handlers, and should be kept in sync with this setting. */ -#define PROF_BT_MAX 128 +#ifdef JEMALLOC_PROF_GCC +# define PROF_BT_MAX_LIMIT 256 +#else +# define PROF_BT_MAX_LIMIT UINT_MAX +#endif +#define PROF_BT_MAX_DEFAULT 128 /* Initial hash table size. */ -#define PROF_CKH_MINITEMS 64 +#define PROF_CKH_MINITEMS 64 /* Size of memory buffer to use when writing dump files. */ #ifndef JEMALLOC_PROF /* Minimize memory bloat for non-prof builds. */ -# define PROF_DUMP_BUFSIZE 1 +# define PROF_DUMP_BUFSIZE 1 #elif defined(JEMALLOC_DEBUG) /* Use a small buffer size in debug build, mainly to facilitate testing. */ -# define PROF_DUMP_BUFSIZE 16 +# define PROF_DUMP_BUFSIZE 16 #else -# define PROF_DUMP_BUFSIZE 65536 +# define PROF_DUMP_BUFSIZE 65536 #endif /* Size of size class related tables */ #ifdef JEMALLOC_PROF -# define PROF_SC_NSIZES SC_NSIZES +# define PROF_SC_NSIZES SC_NSIZES #else /* Minimize memory bloat for non-prof builds. */ -# define PROF_SC_NSIZES 1 +# define PROF_SC_NSIZES 1 #endif /* Size of stack-allocated buffer used by prof_printf(). */ -#define PROF_PRINTF_BUFSIZE 128 +#define PROF_PRINTF_BUFSIZE 128 /* * Number of mutexes shared among all gctx's. No space is allocated for these * unless profiling is enabled, so it's okay to over-provision. */ -#define PROF_NCTX_LOCKS 1024 +#define PROF_NCTX_LOCKS 1024 /* * Number of mutexes shared among all tdata's. No space is allocated for these * unless profiling is enabled, so it's okay to over-provision. */ -#define PROF_NTDATA_LOCKS 256 +#define PROF_NTDATA_LOCKS 256 /* Minimize memory bloat for non-prof builds. */ #ifdef JEMALLOC_PROF -#define PROF_DUMP_FILENAME_LEN (PATH_MAX + 1) +# define PROF_DUMP_FILENAME_LEN (PATH_MAX + 1) #else -#define PROF_DUMP_FILENAME_LEN 1 +# define PROF_DUMP_FILENAME_LEN 1 #endif /* Default number of recent allocations to record. */ #define PROF_RECENT_ALLOC_MAX_DEFAULT 0 +/* Thread name storage size limit. */ +#define PROF_THREAD_NAME_MAX_LEN 16 + +/* + * Minimum required alignment for sampled allocations. Over-aligning sampled + * allocations allows us to quickly identify them on the dalloc path without + * resorting to metadata lookup. + */ +#define PROF_SAMPLE_ALIGNMENT PAGE +#define PROF_SAMPLE_ALIGNMENT_MASK PAGE_MASK + +/* NOLINTNEXTLINE(performance-no-int-to-ptr) */ +#define PROF_TCTX_SENTINEL ((prof_tctx_t *)((uintptr_t)1U)) + #endif /* JEMALLOC_INTERNAL_PROF_TYPES_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/psset.h b/dep/jemalloc/include/jemalloc/internal/psset.h index e1d64970e..f096e414a 100644 --- a/dep/jemalloc/include/jemalloc/internal/psset.h +++ b/dep/jemalloc/include/jemalloc/internal/psset.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_PSSET_H #define JEMALLOC_INTERNAL_PSSET_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/hpdata.h" /* @@ -20,6 +21,12 @@ */ #define PSSET_NPSIZES 64 +/* + * We store non-hugefied and hugified pageslabs metadata separately. + * [0] corresponds to non-hugified and [1] to hugified pageslabs. + */ +#define PSSET_NHUGE 2 + /* * We keep two purge lists per page size class; one for hugified hpdatas (at * index 2*pszind), and one for the non-hugified hpdatas (at index 2*pszind + @@ -43,21 +50,36 @@ struct psset_bin_stats_s { typedef struct psset_stats_s psset_stats_t; struct psset_stats_s { + /* + * Merged stats for all pageslabs in psset. This lets us quickly + * answer queries for the number of dirty and active pages in the + * entire set. + */ + psset_bin_stats_t merged; + + /* + * Below are the same stats, but aggregated by different + * properties of pageslabs: huginess or fullness. + */ + + /* Non-huge and huge slabs. */ + psset_bin_stats_t slabs[PSSET_NHUGE]; + /* * The second index is huge stats; nonfull_slabs[pszind][0] contains * stats for the non-huge slabs in bucket pszind, while * nonfull_slabs[pszind][1] contains stats for the huge slabs. */ - psset_bin_stats_t nonfull_slabs[PSSET_NPSIZES][2]; + psset_bin_stats_t nonfull_slabs[PSSET_NPSIZES][PSSET_NHUGE]; /* * Full slabs don't live in any edata heap, but we still track their * stats. */ - psset_bin_stats_t full_slabs[2]; + psset_bin_stats_t full_slabs[PSSET_NHUGE]; /* Empty slabs are similar. */ - psset_bin_stats_t empty_slabs[2]; + psset_bin_stats_t empty_slabs[PSSET_NHUGE]; }; typedef struct psset_s psset_t; @@ -68,13 +90,7 @@ struct psset_s { */ hpdata_age_heap_t pageslabs[PSSET_NPSIZES]; /* Bitmap for which set bits correspond to non-empty heaps. */ - fb_group_t pageslab_bitmap[FB_NGROUPS(PSSET_NPSIZES)]; - /* - * The sum of all bin stats in stats. This lets us quickly answer - * queries for the number of dirty, active, and retained pages in the - * entire set. - */ - psset_bin_stats_t merged_stats; + fb_group_t pageslab_bitmap[FB_NGROUPS(PSSET_NPSIZES)]; psset_stats_t stats; /* * Slabs with no active allocations, but which are allowed to serve new @@ -105,8 +121,12 @@ void psset_update_end(psset_t *psset, hpdata_t *ps); /* Analogous to the eset_fit; pick a hpdata to serve the request. */ hpdata_t *psset_pick_alloc(psset_t *psset, size_t size); -/* Pick one to purge. */ -hpdata_t *psset_pick_purge(psset_t *psset); +/* + * Pick one to purge that is purgable before given time (inclusive). If now + * is NULL then time is not considered. + */ +hpdata_t *psset_pick_purge(psset_t *psset, const nstime_t *now); + /* Pick one to hugify. */ hpdata_t *psset_pick_hugify(psset_t *psset); @@ -115,17 +135,17 @@ void psset_remove(psset_t *psset, hpdata_t *ps); static inline size_t psset_npageslabs(psset_t *psset) { - return psset->merged_stats.npageslabs; + return psset->stats.merged.npageslabs; } static inline size_t psset_nactive(psset_t *psset) { - return psset->merged_stats.nactive; + return psset->stats.merged.nactive; } static inline size_t psset_ndirty(psset_t *psset) { - return psset->merged_stats.ndirty; + return psset->stats.merged.ndirty; } #endif /* JEMALLOC_INTERNAL_PSSET_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/public_namespace.h b/dep/jemalloc/include/jemalloc/internal/public_namespace.h index a0ee73d4c..3904dd23a 100644 --- a/dep/jemalloc/include/jemalloc/internal/public_namespace.h +++ b/dep/jemalloc/include/jemalloc/internal/public_namespace.h @@ -2,6 +2,8 @@ #define je_calloc JEMALLOC_N(calloc) #define je_dallocx JEMALLOC_N(dallocx) #define je_free JEMALLOC_N(free) +#define je_free_sized JEMALLOC_N(free_sized) +#define je_free_aligned_sized JEMALLOC_N(free_aligned_sized) #define je_mallctl JEMALLOC_N(mallctl) #define je_mallctlbymib JEMALLOC_N(mallctlbymib) #define je_mallctlnametomib JEMALLOC_N(mallctlnametomib) @@ -12,7 +14,7 @@ #define je_malloc_stats_print JEMALLOC_N(malloc_stats_print) #define je_malloc_usable_size JEMALLOC_N(malloc_usable_size) #define je_mallocx JEMALLOC_N(mallocx) -#define je_smallocx_54eaed1d8b56b1aa528be3bdd1877e59c56fa90c JEMALLOC_N(smallocx_54eaed1d8b56b1aa528be3bdd1877e59c56fa90c) +#define je_smallocx_ee4d7b7f9a5c1b48d462be2b65c16e4ead8e3396 JEMALLOC_N(smallocx_ee4d7b7f9a5c1b48d462be2b65c16e4ead8e3396) #define je_nallocx JEMALLOC_N(nallocx) #define je_posix_memalign JEMALLOC_N(posix_memalign) #define je_rallocx JEMALLOC_N(rallocx) @@ -22,4 +24,5 @@ #define je_xallocx JEMALLOC_N(xallocx) #define je_memalign JEMALLOC_N(memalign) #define je_valloc JEMALLOC_N(valloc) +#define je_pvalloc JEMALLOC_N(pvalloc) #define je_malloc_size JEMALLOC_N(malloc_size) diff --git a/dep/jemalloc/include/jemalloc/internal/public_unnamespace.h b/dep/jemalloc/include/jemalloc/internal/public_unnamespace.h index 7542c38a4..52b2ed032 100644 --- a/dep/jemalloc/include/jemalloc/internal/public_unnamespace.h +++ b/dep/jemalloc/include/jemalloc/internal/public_unnamespace.h @@ -2,6 +2,8 @@ #undef je_calloc #undef je_dallocx #undef je_free +#undef je_free_sized +#undef je_free_aligned_sized #undef je_mallctl #undef je_mallctlbymib #undef je_mallctlnametomib @@ -12,7 +14,7 @@ #undef je_malloc_stats_print #undef je_malloc_usable_size #undef je_mallocx -#undef je_smallocx_54eaed1d8b56b1aa528be3bdd1877e59c56fa90c +#undef je_smallocx_ee4d7b7f9a5c1b48d462be2b65c16e4ead8e3396 #undef je_nallocx #undef je_posix_memalign #undef je_rallocx @@ -22,4 +24,5 @@ #undef je_xallocx #undef je_memalign #undef je_valloc +#undef je_pvalloc #undef je_malloc_size diff --git a/dep/jemalloc/include/jemalloc/internal/ql.h b/dep/jemalloc/include/jemalloc/internal/ql.h index c7f52f862..9c1776a45 100644 --- a/dep/jemalloc/include/jemalloc/internal/ql.h +++ b/dep/jemalloc/include/jemalloc/internal/ql.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_QL_H #define JEMALLOC_INTERNAL_QL_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/qr.h" /* @@ -27,33 +28,36 @@ */ /* List definitions. */ -#define ql_head(a_type) \ -struct { \ - a_type *qlh_first; \ -} +#define ql_head(a_type) \ + struct { \ + a_type *qlh_first; \ + } /* Static initializer for an empty list. */ -#define ql_head_initializer(a_head) {NULL} +#define ql_head_initializer(a_head) \ + { NULL } /* The field definition. */ -#define ql_elm(a_type) qr(a_type) +#define ql_elm(a_type) qr(a_type) /* A pointer to the first element in the list, or NULL if the list is empty. */ #define ql_first(a_head) ((a_head)->qlh_first) /* Dynamically initializes a list. */ -#define ql_new(a_head) do { \ - ql_first(a_head) = NULL; \ -} while (0) +#define ql_new(a_head) \ + do { \ + ql_first(a_head) = NULL; \ + } while (0) /* * Sets dest to be the contents of src (overwriting any elements there), leaving * src empty. */ -#define ql_move(a_head_dest, a_head_src) do { \ - ql_first(a_head_dest) = ql_first(a_head_src); \ - ql_new(a_head_src); \ -} while (0) +#define ql_move(a_head_dest, a_head_src) \ + do { \ + ql_first(a_head_dest) = ql_first(a_head_src); \ + ql_new(a_head_src); \ + } while (0) /* True if the list is empty, otherwise false. */ #define ql_empty(a_head) (ql_first(a_head) == NULL) @@ -67,85 +71,91 @@ struct { \ /* * Obtains the last item in the list. */ -#define ql_last(a_head, a_field) \ +#define ql_last(a_head, a_field) \ (ql_empty(a_head) ? NULL : qr_prev(ql_first(a_head), a_field)) /* * Gets a pointer to the next/prev element in the list. Trying to advance past * the end or retreat before the beginning of the list returns NULL. */ -#define ql_next(a_head, a_elm, a_field) \ - ((ql_last(a_head, a_field) != (a_elm)) \ - ? qr_next((a_elm), a_field) : NULL) -#define ql_prev(a_head, a_elm, a_field) \ - ((ql_first(a_head) != (a_elm)) ? qr_prev((a_elm), a_field) \ - : NULL) +#define ql_next(a_head, a_elm, a_field) \ + ((ql_last(a_head, a_field) != (a_elm)) ? qr_next((a_elm), a_field) \ + : NULL) +#define ql_prev(a_head, a_elm, a_field) \ + ((ql_first(a_head) != (a_elm)) ? qr_prev((a_elm), a_field) : NULL) /* Inserts a_elm before a_qlelm in the list. */ -#define ql_before_insert(a_head, a_qlelm, a_elm, a_field) do { \ - qr_before_insert((a_qlelm), (a_elm), a_field); \ - if (ql_first(a_head) == (a_qlelm)) { \ - ql_first(a_head) = (a_elm); \ - } \ -} while (0) +#define ql_before_insert(a_head, a_qlelm, a_elm, a_field) \ + do { \ + qr_before_insert((a_qlelm), (a_elm), a_field); \ + if (ql_first(a_head) == (a_qlelm)) { \ + ql_first(a_head) = (a_elm); \ + } \ + } while (0) /* Inserts a_elm after a_qlelm in the list. */ -#define ql_after_insert(a_qlelm, a_elm, a_field) \ +#define ql_after_insert(a_qlelm, a_elm, a_field) \ qr_after_insert((a_qlelm), (a_elm), a_field) /* Inserts a_elm as the first item in the list. */ -#define ql_head_insert(a_head, a_elm, a_field) do { \ - if (!ql_empty(a_head)) { \ - qr_before_insert(ql_first(a_head), (a_elm), a_field); \ - } \ - ql_first(a_head) = (a_elm); \ -} while (0) +#define ql_head_insert(a_head, a_elm, a_field) \ + do { \ + if (!ql_empty(a_head)) { \ + qr_before_insert(ql_first(a_head), (a_elm), a_field); \ + } \ + ql_first(a_head) = (a_elm); \ + } while (0) /* Inserts a_elm as the last item in the list. */ -#define ql_tail_insert(a_head, a_elm, a_field) do { \ - if (!ql_empty(a_head)) { \ - qr_before_insert(ql_first(a_head), (a_elm), a_field); \ - } \ - ql_first(a_head) = qr_next((a_elm), a_field); \ -} while (0) +#define ql_tail_insert(a_head, a_elm, a_field) \ + do { \ + if (!ql_empty(a_head)) { \ + qr_before_insert(ql_first(a_head), (a_elm), a_field); \ + } \ + ql_first(a_head) = qr_next((a_elm), a_field); \ + } while (0) /* * Given lists a = [a_1, ..., a_n] and [b_1, ..., b_n], results in: * a = [a1, ..., a_n, b_1, ..., b_n] and b = []. */ -#define ql_concat(a_head_a, a_head_b, a_field) do { \ - if (ql_empty(a_head_a)) { \ - ql_move(a_head_a, a_head_b); \ - } else if (!ql_empty(a_head_b)) { \ - qr_meld(ql_first(a_head_a), ql_first(a_head_b), \ - a_field); \ - ql_new(a_head_b); \ - } \ -} while (0) +#define ql_concat(a_head_a, a_head_b, a_field) \ + do { \ + if (ql_empty(a_head_a)) { \ + ql_move(a_head_a, a_head_b); \ + } else if (!ql_empty(a_head_b)) { \ + qr_meld( \ + ql_first(a_head_a), ql_first(a_head_b), a_field); \ + ql_new(a_head_b); \ + } \ + } while (0) /* Removes a_elm from the list. */ -#define ql_remove(a_head, a_elm, a_field) do { \ - if (ql_first(a_head) == (a_elm)) { \ - ql_first(a_head) = qr_next(ql_first(a_head), a_field); \ - } \ - if (ql_first(a_head) != (a_elm)) { \ - qr_remove((a_elm), a_field); \ - } else { \ - ql_new(a_head); \ - } \ -} while (0) +#define ql_remove(a_head, a_elm, a_field) \ + do { \ + if (ql_first(a_head) == (a_elm)) { \ + ql_first(a_head) = qr_next(ql_first(a_head), a_field); \ + } \ + if (ql_first(a_head) != (a_elm)) { \ + qr_remove((a_elm), a_field); \ + } else { \ + ql_new(a_head); \ + } \ + } while (0) /* Removes the first item in the list. */ -#define ql_head_remove(a_head, a_type, a_field) do { \ - a_type *t = ql_first(a_head); \ - ql_remove((a_head), t, a_field); \ -} while (0) +#define ql_head_remove(a_head, a_type, a_field) \ + do { \ + a_type *t = ql_first(a_head); \ + ql_remove((a_head), t, a_field); \ + } while (0) /* Removes the last item in the list. */ -#define ql_tail_remove(a_head, a_type, a_field) do { \ - a_type *t = ql_last(a_head, a_field); \ - ql_remove((a_head), t, a_field); \ -} while (0) +#define ql_tail_remove(a_head, a_type, a_field) \ + do { \ + a_type *t = ql_last(a_head, a_field); \ + ql_remove((a_head), t, a_field); \ + } while (0) /* * Given a = [a_1, a_2, ..., a_n-1, a_n, a_n+1, ...], @@ -154,14 +164,15 @@ struct { \ * and replaces b's contents with: * b = [a_n, a_n+1, ...] */ -#define ql_split(a_head_a, a_elm, a_head_b, a_field) do { \ - if (ql_first(a_head_a) == (a_elm)) { \ - ql_move(a_head_b, a_head_a); \ - } else { \ - qr_split(ql_first(a_head_a), (a_elm), a_field); \ - ql_first(a_head_b) = (a_elm); \ - } \ -} while (0) +#define ql_split(a_head_a, a_elm, a_head_b, a_field) \ + do { \ + if (ql_first(a_head_a) == (a_elm)) { \ + ql_move(a_head_b, a_head_a); \ + } else { \ + qr_split(ql_first(a_head_a), (a_elm), a_field); \ + ql_first(a_head_b) = (a_elm); \ + } \ + } while (0) /* * An optimized version of: @@ -169,9 +180,10 @@ struct { \ * ql_remove((a_head), t, a_field); * ql_tail_insert((a_head), t, a_field); */ -#define ql_rotate(a_head, a_field) do { \ - ql_first(a_head) = qr_next(ql_first(a_head), a_field); \ -} while (0) +#define ql_rotate(a_head, a_field) \ + do { \ + ql_first(a_head) = qr_next(ql_first(a_head), a_field); \ + } while (0) /* * Helper macro to iterate over each element in a list in order, starting from @@ -188,10 +200,10 @@ struct { \ * } */ -#define ql_foreach(a_var, a_head, a_field) \ - qr_foreach((a_var), ql_first(a_head), a_field) +#define ql_foreach(a_var, a_head, a_field) \ + qr_foreach ((a_var), ql_first(a_head), a_field) -#define ql_reverse_foreach(a_var, a_head, a_field) \ +#define ql_reverse_foreach(a_var, a_head, a_field) \ qr_reverse_foreach((a_var), ql_first(a_head), a_field) #endif /* JEMALLOC_INTERNAL_QL_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/qr.h b/dep/jemalloc/include/jemalloc/internal/qr.h index ece4f5568..1bd61f384 100644 --- a/dep/jemalloc/include/jemalloc/internal/qr.h +++ b/dep/jemalloc/include/jemalloc/internal/qr.h @@ -17,21 +17,22 @@ */ /* Ring definitions. */ -#define qr(a_type) \ -struct { \ - a_type *qre_next; \ - a_type *qre_prev; \ -} +#define qr(a_type) \ + struct { \ + a_type *qre_next; \ + a_type *qre_prev; \ + } /* * Initialize a qr link. Every link must be initialized before being used, even * if that initialization is going to be immediately overwritten (say, by being * passed into an insertion macro). */ -#define qr_new(a_qr, a_field) do { \ - (a_qr)->a_field.qre_next = (a_qr); \ - (a_qr)->a_field.qre_prev = (a_qr); \ -} while (0) +#define qr_new(a_qr, a_field) \ + do { \ + (a_qr)->a_field.qre_next = (a_qr); \ + (a_qr)->a_field.qre_prev = (a_qr); \ + } while (0) /* * Go forwards or backwards in the ring. Note that (the ring being circular), this @@ -58,26 +59,27 @@ struct { \ * * a_qr_a can directly be a qr_next() macro, but a_qr_b cannot. */ -#define qr_meld(a_qr_a, a_qr_b, a_field) do { \ - (a_qr_b)->a_field.qre_prev->a_field.qre_next = \ - (a_qr_a)->a_field.qre_prev; \ - (a_qr_a)->a_field.qre_prev = (a_qr_b)->a_field.qre_prev; \ - (a_qr_b)->a_field.qre_prev = \ - (a_qr_b)->a_field.qre_prev->a_field.qre_next; \ - (a_qr_a)->a_field.qre_prev->a_field.qre_next = (a_qr_a); \ - (a_qr_b)->a_field.qre_prev->a_field.qre_next = (a_qr_b); \ -} while (0) +#define qr_meld(a_qr_a, a_qr_b, a_field) \ + do { \ + (a_qr_b)->a_field.qre_prev->a_field.qre_next = \ + (a_qr_a)->a_field.qre_prev; \ + (a_qr_a)->a_field.qre_prev = (a_qr_b)->a_field.qre_prev; \ + (a_qr_b)->a_field.qre_prev = \ + (a_qr_b)->a_field.qre_prev->a_field.qre_next; \ + (a_qr_a)->a_field.qre_prev->a_field.qre_next = (a_qr_a); \ + (a_qr_b)->a_field.qre_prev->a_field.qre_next = (a_qr_b); \ + } while (0) /* * Logically, this is just a meld. The intent, though, is that a_qrelm is a * single-element ring, so that "before" has a more obvious interpretation than * meld. */ -#define qr_before_insert(a_qrelm, a_qr, a_field) \ +#define qr_before_insert(a_qrelm, a_qr, a_field) \ qr_meld((a_qrelm), (a_qr), a_field) /* Ditto, but inserting after rather than before. */ -#define qr_after_insert(a_qrelm, a_qr, a_field) \ +#define qr_after_insert(a_qrelm, a_qr, a_field) \ qr_before_insert(qr_next(a_qrelm, a_field), (a_qr), a_field) /* @@ -98,14 +100,13 @@ struct { \ * qr_meld() and qr_split() are functionally equivalent, so there's no need to * have two copies of the code. */ -#define qr_split(a_qr_a, a_qr_b, a_field) \ - qr_meld((a_qr_a), (a_qr_b), a_field) +#define qr_split(a_qr_a, a_qr_b, a_field) qr_meld((a_qr_a), (a_qr_b), a_field) /* * Splits off a_qr from the rest of its ring, so that it becomes a * single-element ring. */ -#define qr_remove(a_qr, a_field) \ +#define qr_remove(a_qr, a_field) \ qr_split(qr_next(a_qr, a_field), (a_qr), a_field) /* @@ -121,20 +122,19 @@ struct { \ * return sum; * } */ -#define qr_foreach(var, a_qr, a_field) \ - for ((var) = (a_qr); \ - (var) != NULL; \ - (var) = (((var)->a_field.qre_next != (a_qr)) \ - ? (var)->a_field.qre_next : NULL)) +#define qr_foreach(var, a_qr, a_field) \ + for ((var) = (a_qr); (var) != NULL; \ + (var) = (((var)->a_field.qre_next != (a_qr)) \ + ? (var)->a_field.qre_next \ + : NULL)) /* * The same (and with the same usage) as qr_foreach, but in the opposite order, * ending with a_qr. */ -#define qr_reverse_foreach(var, a_qr, a_field) \ - for ((var) = ((a_qr) != NULL) ? qr_prev(a_qr, a_field) : NULL; \ - (var) != NULL; \ - (var) = (((var) != (a_qr)) \ - ? (var)->a_field.qre_prev : NULL)) +#define qr_reverse_foreach(var, a_qr, a_field) \ + for ((var) = ((a_qr) != NULL) ? qr_prev(a_qr, a_field) : NULL; \ + (var) != NULL; \ + (var) = (((var) != (a_qr)) ? (var)->a_field.qre_prev : NULL)) #endif /* JEMALLOC_INTERNAL_QR_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/quantum.h b/dep/jemalloc/include/jemalloc/internal/quantum.h index c22d753aa..2f7c0466a 100644 --- a/dep/jemalloc/include/jemalloc/internal/quantum.h +++ b/dep/jemalloc/include/jemalloc/internal/quantum.h @@ -6,82 +6,84 @@ * classes). */ #ifndef LG_QUANTUM -# if (defined(__i386__) || defined(_M_IX86)) -# define LG_QUANTUM 4 -# endif -# ifdef __ia64__ -# define LG_QUANTUM 4 -# endif -# ifdef __alpha__ -# define LG_QUANTUM 4 -# endif -# if (defined(__sparc64__) || defined(__sparcv9) || defined(__sparc_v9__)) -# define LG_QUANTUM 4 -# endif -# if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64)) -# define LG_QUANTUM 4 -# endif -# ifdef __arm__ -# define LG_QUANTUM 3 -# endif -# ifdef __aarch64__ -# define LG_QUANTUM 4 -# endif -# ifdef __hppa__ -# define LG_QUANTUM 4 -# endif -# ifdef __loongarch__ -# define LG_QUANTUM 4 -# endif -# ifdef __m68k__ -# define LG_QUANTUM 3 -# endif -# ifdef __mips__ -# if defined(__mips_n32) || defined(__mips_n64) -# define LG_QUANTUM 4 -# else -# define LG_QUANTUM 3 -# endif -# endif -# ifdef __nios2__ -# define LG_QUANTUM 3 -# endif -# ifdef __or1k__ -# define LG_QUANTUM 3 -# endif -# ifdef __powerpc__ -# define LG_QUANTUM 4 -# endif -# if defined(__riscv) || defined(__riscv__) -# define LG_QUANTUM 4 -# endif -# ifdef __s390__ -# define LG_QUANTUM 4 -# endif -# if (defined (__SH3E__) || defined(__SH4_SINGLE__) || defined(__SH4__) || \ - defined(__SH4_SINGLE_ONLY__)) -# define LG_QUANTUM 4 -# endif -# ifdef __tile__ -# define LG_QUANTUM 4 -# endif -# ifdef __le32__ -# define LG_QUANTUM 4 -# endif -# ifdef __arc__ -# define LG_QUANTUM 3 -# endif -# ifndef LG_QUANTUM -# error "Unknown minimum alignment for architecture; specify via " - "--with-lg-quantum" -# endif +# if (defined(__i386__) || defined(_M_IX86)) +# define LG_QUANTUM 4 +# endif +# ifdef __ia64__ +# define LG_QUANTUM 4 +# endif +# ifdef __alpha__ +# define LG_QUANTUM 4 +# endif +# if (defined(__sparc64__) || defined(__sparcv9) \ + || defined(__sparc_v9__)) +# define LG_QUANTUM 4 +# endif +# if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64)) +# define LG_QUANTUM 4 +# endif +# ifdef __arm__ +# define LG_QUANTUM 3 +# endif +# if defined(__aarch64__) || defined(_M_ARM64) +# define LG_QUANTUM 4 +# endif +# ifdef __hppa__ +# define LG_QUANTUM 4 +# endif +# ifdef __loongarch__ +# define LG_QUANTUM 4 +# endif +# ifdef __m68k__ +# define LG_QUANTUM 3 +# endif +# ifdef __mips__ +# if defined(__mips_n32) || defined(__mips_n64) +# define LG_QUANTUM 4 +# else +# define LG_QUANTUM 3 +# endif +# endif +# ifdef __nios2__ +# define LG_QUANTUM 3 +# endif +# ifdef __or1k__ +# define LG_QUANTUM 3 +# endif +# if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) \ + || defined(__ppc64__) +# define LG_QUANTUM 4 +# endif +# if defined(__riscv) || defined(__riscv__) +# define LG_QUANTUM 4 +# endif +# ifdef __s390__ +# define LG_QUANTUM 4 +# endif +# if (defined(__SH3E__) || defined(__SH4_SINGLE__) || defined(__SH4__) \ + || defined(__SH4_SINGLE_ONLY__)) +# define LG_QUANTUM 4 +# endif +# ifdef __tile__ +# define LG_QUANTUM 4 +# endif +# ifdef __le32__ +# define LG_QUANTUM 4 +# endif +# ifdef __arc__ +# define LG_QUANTUM 3 +# endif +# ifndef LG_QUANTUM +# error \ + "Unknown minimum alignment for architecture; specify via " +"--with-lg-quantum" +# endif #endif -#define QUANTUM ((size_t)(1U << LG_QUANTUM)) -#define QUANTUM_MASK (QUANTUM - 1) +#define QUANTUM ((size_t)(1U << LG_QUANTUM)) +#define QUANTUM_MASK (QUANTUM - 1) /* Return the smallest quantum multiple that is >= a. */ -#define QUANTUM_CEILING(a) \ - (((a) + QUANTUM_MASK) & ~QUANTUM_MASK) +#define QUANTUM_CEILING(a) (((a) + QUANTUM_MASK) & ~QUANTUM_MASK) #endif /* JEMALLOC_INTERNAL_QUANTUM_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/rb.h b/dep/jemalloc/include/jemalloc/internal/rb.h index a9a51cb68..58510e4db 100644 --- a/dep/jemalloc/include/jemalloc/internal/rb.h +++ b/dep/jemalloc/include/jemalloc/internal/rb.h @@ -1,6 +1,9 @@ #ifndef JEMALLOC_INTERNAL_RB_H #define JEMALLOC_INTERNAL_RB_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/safety_check.h" + /*- ******************************************************************************* * @@ -23,7 +26,7 @@ */ #ifndef __PGI -#define RB_COMPACT +# define RB_COMPACT #endif /* @@ -35,6 +38,7 @@ */ #define RB_MAX_DEPTH (sizeof(void *) << 4) +/* clang-format off */ #ifdef RB_COMPACT /* Node structure. */ #define rb_node(a_type) \ @@ -560,6 +564,20 @@ a_prefix##reverse_iter_filtered(a_rbt_type *rbtree, a_type *start, \ * the same as with the unfiltered version, with the added constraint that the * returned node must pass the filter. */ +JEMALLOC_ALWAYS_INLINE void +rb_remove_safety_checks(const void *nodep, const char *function_name) { + if (!config_opt_safety_checks) { + return; + } + if (unlikely(nodep == NULL)) { + safety_check_fail( + ": Invalid deallocation detected in %s: " + "attempting to remove node from tree but node was " + "not found. Possibly caused by double free bugs.", + function_name); + } +} + #define rb_gen(a_attr, a_prefix, a_rbt_type, a_type, a_field, a_cmp) \ rb_gen_impl(a_attr, a_prefix, a_rbt_type, a_type, a_field, a_cmp, \ rb_empty_summarize, false) @@ -852,6 +870,8 @@ a_prefix##remove(a_rbt_type *rbtree, a_type *node) { \ } \ } \ } \ + rb_remove_safety_checks(nodep, __func__); \ + assert(nodep != NULL); \ assert(nodep->node == node); \ pathp--; \ if (pathp->node != node) { \ @@ -1852,5 +1872,6 @@ a_prefix##reverse_iter_filtered(a_rbt_type *rbtree, a_type *start, \ return ret; \ } \ ) /* end rb_summarized_only */ +/* clang-format on */ #endif /* JEMALLOC_INTERNAL_RB_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/rtree.h b/dep/jemalloc/include/jemalloc/internal/rtree.h index a00adb298..07205958e 100644 --- a/dep/jemalloc/include/jemalloc/internal/rtree.h +++ b/dep/jemalloc/include/jemalloc/internal/rtree.h @@ -1,7 +1,10 @@ #ifndef JEMALLOC_INTERNAL_RTREE_H #define JEMALLOC_INTERNAL_RTREE_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/base.h" #include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/edata.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/rtree_tsd.h" #include "jemalloc/internal/sc.h" @@ -15,48 +18,49 @@ */ /* Number of high insignificant bits. */ -#define RTREE_NHIB ((1U << (LG_SIZEOF_PTR+3)) - LG_VADDR) +#define RTREE_NHIB ((1U << (LG_SIZEOF_PTR + 3)) - LG_VADDR) /* Number of low insigificant bits. */ #define RTREE_NLIB LG_PAGE /* Number of significant bits. */ #define RTREE_NSB (LG_VADDR - RTREE_NLIB) /* Number of levels in radix tree. */ #if RTREE_NSB <= 10 -# define RTREE_HEIGHT 1 +# define RTREE_HEIGHT 1 #elif RTREE_NSB <= 36 -# define RTREE_HEIGHT 2 +# define RTREE_HEIGHT 2 #elif RTREE_NSB <= 52 -# define RTREE_HEIGHT 3 +# define RTREE_HEIGHT 3 #else -# error Unsupported number of significant virtual address bits +# error Unsupported number of significant virtual address bits #endif /* Use compact leaf representation if virtual address encoding allows. */ #if RTREE_NHIB >= LG_CEIL(SC_NSIZES) -# define RTREE_LEAF_COMPACT +# define RTREE_LEAF_COMPACT #endif typedef struct rtree_node_elm_s rtree_node_elm_t; struct rtree_node_elm_s { - atomic_p_t child; /* (rtree_{node,leaf}_elm_t *) */ + atomic_p_t child; /* (rtree_{node,leaf}_elm_t *) */ }; typedef struct rtree_metadata_s rtree_metadata_t; struct rtree_metadata_s { - szind_t szind; - extent_state_t state; /* Mirrors edata->state. */ - bool is_head; /* Mirrors edata->is_head. */ - bool slab; + szind_t szind; + extent_state_t state; /* Mirrors edata->state. */ + bool is_head; /* Mirrors edata->is_head. */ + bool slab; }; typedef struct rtree_contents_s rtree_contents_t; struct rtree_contents_s { - edata_t *edata; + edata_t *edata; rtree_metadata_t metadata; }; #define RTREE_LEAF_STATE_WIDTH EDATA_BITS_STATE_WIDTH #define RTREE_LEAF_STATE_SHIFT 2 -#define RTREE_LEAF_STATE_MASK MASK(RTREE_LEAF_STATE_WIDTH, RTREE_LEAF_STATE_SHIFT) +#define RTREE_LEAF_STATE_MASK \ + MASK(RTREE_LEAF_STATE_WIDTH, RTREE_LEAF_STATE_SHIFT) struct rtree_leaf_elm_s { #ifdef RTREE_LEAF_COMPACT @@ -74,36 +78,36 @@ struct rtree_leaf_elm_s { * * 00000000 xxxxxxxx eeeeeeee [...] eeeeeeee e00ssshb */ - atomic_p_t le_bits; + atomic_p_t le_bits; #else - atomic_p_t le_edata; /* (edata_t *) */ + atomic_p_t le_edata; /* (edata_t *) */ /* * From high to low bits: szind (8 bits), state (4 bits), is_head, slab */ - atomic_u_t le_metadata; + atomic_u_t le_metadata; #endif }; typedef struct rtree_level_s rtree_level_t; struct rtree_level_s { /* Number of key bits distinguished by this level. */ - unsigned bits; + unsigned bits; /* * Cumulative number of key bits distinguished by traversing to * corresponding tree level. */ - unsigned cumbits; + unsigned cumbits; }; typedef struct rtree_s rtree_t; struct rtree_s { - base_t *base; - malloc_mutex_t init_lock; + base_t *base; + malloc_mutex_t init_lock; /* Number of elements based on rtree_levels[0].bits. */ #if RTREE_HEIGHT > 1 - rtree_node_elm_t root[1U << (RTREE_NSB/RTREE_HEIGHT)]; + rtree_node_elm_t root[1U << (RTREE_NSB / RTREE_HEIGHT)]; #else - rtree_leaf_elm_t root[1U << (RTREE_NSB/RTREE_HEIGHT)]; + rtree_leaf_elm_t root[1U << (RTREE_NSB / RTREE_HEIGHT)]; #endif }; @@ -115,17 +119,17 @@ struct rtree_s { */ static const rtree_level_t rtree_levels[] = { #if RTREE_HEIGHT == 1 - {RTREE_NSB, RTREE_NHIB + RTREE_NSB} + {RTREE_NSB, RTREE_NHIB + RTREE_NSB} #elif RTREE_HEIGHT == 2 - {RTREE_NSB/2, RTREE_NHIB + RTREE_NSB/2}, - {RTREE_NSB/2 + RTREE_NSB%2, RTREE_NHIB + RTREE_NSB} + {RTREE_NSB / 2, RTREE_NHIB + RTREE_NSB / 2}, + {RTREE_NSB / 2 + RTREE_NSB % 2, RTREE_NHIB + RTREE_NSB} #elif RTREE_HEIGHT == 3 - {RTREE_NSB/3, RTREE_NHIB + RTREE_NSB/3}, - {RTREE_NSB/3 + RTREE_NSB%3/2, - RTREE_NHIB + RTREE_NSB/3*2 + RTREE_NSB%3/2}, - {RTREE_NSB/3 + RTREE_NSB%3 - RTREE_NSB%3/2, RTREE_NHIB + RTREE_NSB} + {RTREE_NSB / 3, RTREE_NHIB + RTREE_NSB / 3}, + {RTREE_NSB / 3 + RTREE_NSB % 3 / 2, + RTREE_NHIB + RTREE_NSB / 3 * 2 + RTREE_NSB % 3 / 2}, + {RTREE_NSB / 3 + RTREE_NSB % 3 - RTREE_NSB % 3 / 2, RTREE_NHIB + RTREE_NSB} #else -# error Unsupported rtree height +# error Unsupported rtree height #endif }; @@ -136,9 +140,9 @@ rtree_leaf_elm_t *rtree_leaf_elm_lookup_hard(tsdn_t *tsdn, rtree_t *rtree, JEMALLOC_ALWAYS_INLINE unsigned rtree_leaf_maskbits(void) { - unsigned ptrbits = ZU(1) << (LG_SIZEOF_PTR+3); - unsigned cumbits = (rtree_levels[RTREE_HEIGHT-1].cumbits - - rtree_levels[RTREE_HEIGHT-1].bits); + unsigned ptrbits = ZU(1) << (LG_SIZEOF_PTR + 3); + unsigned cumbits = (rtree_levels[RTREE_HEIGHT - 1].cumbits + - rtree_levels[RTREE_HEIGHT - 1].bits); return ptrbits - cumbits; } @@ -150,16 +154,16 @@ rtree_leafkey(uintptr_t key) { JEMALLOC_ALWAYS_INLINE size_t rtree_cache_direct_map(uintptr_t key) { - return (size_t)((key >> rtree_leaf_maskbits()) & - (RTREE_CTX_NCACHE - 1)); + return ( + size_t)((key >> rtree_leaf_maskbits()) & (RTREE_CTX_NCACHE - 1)); } JEMALLOC_ALWAYS_INLINE uintptr_t rtree_subkey(uintptr_t key, unsigned level) { - unsigned ptrbits = ZU(1) << (LG_SIZEOF_PTR+3); - unsigned cumbits = rtree_levels[level].cumbits; - unsigned shiftbits = ptrbits - cumbits; - unsigned maskbits = rtree_levels[level].bits; + unsigned ptrbits = ZU(1) << (LG_SIZEOF_PTR + 3); + unsigned cumbits = rtree_levels[level].cumbits; + unsigned shiftbits = ptrbits - cumbits; + unsigned maskbits = rtree_levels[level].bits; uintptr_t mask = (ZU(1) << maskbits) - 1; return ((key >> shiftbits) & mask); } @@ -175,12 +179,12 @@ rtree_subkey(uintptr_t key, unsigned level) { * dependent on a previous rtree write, which means a stale read * could result if synchronization were omitted here. */ -# ifdef RTREE_LEAF_COMPACT +#ifdef RTREE_LEAF_COMPACT JEMALLOC_ALWAYS_INLINE uintptr_t -rtree_leaf_elm_bits_read(tsdn_t *tsdn, rtree_t *rtree, - rtree_leaf_elm_t *elm, bool dependent) { - return (uintptr_t)atomic_load_p(&elm->le_bits, dependent - ? ATOMIC_RELAXED : ATOMIC_ACQUIRE); +rtree_leaf_elm_bits_read( + tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, bool dependent) { + return (uintptr_t)atomic_load_p( + &elm->le_bits, dependent ? ATOMIC_RELAXED : ATOMIC_ACQUIRE); } JEMALLOC_ALWAYS_INLINE uintptr_t @@ -192,10 +196,10 @@ rtree_leaf_elm_bits_encode(rtree_contents_t contents) { uintptr_t szind_bits = (uintptr_t)contents.metadata.szind << LG_VADDR; uintptr_t slab_bits = (uintptr_t)contents.metadata.slab; uintptr_t is_head_bits = (uintptr_t)contents.metadata.is_head << 1; - uintptr_t state_bits = (uintptr_t)contents.metadata.state << - RTREE_LEAF_STATE_SHIFT; - uintptr_t metadata_bits = szind_bits | state_bits | is_head_bits | - slab_bits; + uintptr_t state_bits = (uintptr_t)contents.metadata.state + << RTREE_LEAF_STATE_SHIFT; + uintptr_t metadata_bits = szind_bits | state_bits | is_head_bits + | slab_bits; assert((edata_bits & metadata_bits) == 0); return edata_bits | metadata_bits; @@ -209,13 +213,13 @@ rtree_leaf_elm_bits_decode(uintptr_t bits) { contents.metadata.slab = (bool)(bits & 1); contents.metadata.is_head = (bool)(bits & (1 << 1)); - uintptr_t state_bits = (bits & RTREE_LEAF_STATE_MASK) >> - RTREE_LEAF_STATE_SHIFT; + uintptr_t state_bits = (bits & RTREE_LEAF_STATE_MASK) + >> RTREE_LEAF_STATE_SHIFT; assert(state_bits <= extent_state_max); contents.metadata.state = (extent_state_t)state_bits; uintptr_t low_bit_mask = ~((uintptr_t)EDATA_ALIGNMENT - 1); -# ifdef __aarch64__ +# ifdef __aarch64__ /* * aarch64 doesn't sign extend the highest virtual address bit to set * the higher ones. Instead, the high bits get zeroed. @@ -223,64 +227,72 @@ rtree_leaf_elm_bits_decode(uintptr_t bits) { uintptr_t high_bit_mask = ((uintptr_t)1 << LG_VADDR) - 1; /* Mask off metadata. */ uintptr_t mask = high_bit_mask & low_bit_mask; + /* NOLINTNEXTLINE(performance-no-int-to-ptr) */ contents.edata = (edata_t *)(bits & mask); -# else +# else /* Restore sign-extended high bits, mask metadata bits. */ + /* NOLINTNEXTLINE(performance-no-int-to-ptr) */ contents.edata = (edata_t *)((uintptr_t)((intptr_t)(bits << RTREE_NHIB) - >> RTREE_NHIB) & low_bit_mask); -# endif + >> RTREE_NHIB) + & low_bit_mask); +# endif assert((uintptr_t)contents.edata % (uintptr_t)EDATA_ALIGNMENT == 0); return contents; } -# endif /* RTREE_LEAF_COMPACT */ +#endif /* RTREE_LEAF_COMPACT */ JEMALLOC_ALWAYS_INLINE rtree_contents_t -rtree_leaf_elm_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, - bool dependent) { +rtree_leaf_elm_read( + tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, bool dependent) { #ifdef RTREE_LEAF_COMPACT uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent); rtree_contents_t contents = rtree_leaf_elm_bits_decode(bits); return contents; #else rtree_contents_t contents; - unsigned metadata_bits = atomic_load_u(&elm->le_metadata, dependent - ? ATOMIC_RELAXED : ATOMIC_ACQUIRE); + unsigned metadata_bits = atomic_load_u( + &elm->le_metadata, dependent ? ATOMIC_RELAXED : ATOMIC_ACQUIRE); contents.metadata.slab = (bool)(metadata_bits & 1); contents.metadata.is_head = (bool)(metadata_bits & (1 << 1)); - uintptr_t state_bits = (metadata_bits & RTREE_LEAF_STATE_MASK) >> - RTREE_LEAF_STATE_SHIFT; + uintptr_t state_bits = (metadata_bits & RTREE_LEAF_STATE_MASK) + >> RTREE_LEAF_STATE_SHIFT; assert(state_bits <= extent_state_max); contents.metadata.state = (extent_state_t)state_bits; - contents.metadata.szind = metadata_bits >> (RTREE_LEAF_STATE_SHIFT + - RTREE_LEAF_STATE_WIDTH); + contents.metadata.szind = metadata_bits + >> (RTREE_LEAF_STATE_SHIFT + RTREE_LEAF_STATE_WIDTH); - contents.edata = (edata_t *)atomic_load_p(&elm->le_edata, dependent - ? ATOMIC_RELAXED : ATOMIC_ACQUIRE); + contents.edata = (edata_t *)atomic_load_p( + &elm->le_edata, dependent ? ATOMIC_RELAXED : ATOMIC_ACQUIRE); return contents; #endif } JEMALLOC_ALWAYS_INLINE void -rtree_contents_encode(rtree_contents_t contents, void **bits, - unsigned *additional) { +rtree_contents_encode( + rtree_contents_t contents, void **bits, unsigned *additional) { #ifdef RTREE_LEAF_COMPACT + /* NOLINTNEXTLINE(performance-no-int-to-ptr) */ *bits = (void *)rtree_leaf_elm_bits_encode(contents); + /* Suppress spurious warning from static analysis */ + if (config_debug) { + *additional = 0; + } #else *additional = (unsigned)contents.metadata.slab | ((unsigned)contents.metadata.is_head << 1) | ((unsigned)contents.metadata.state << RTREE_LEAF_STATE_SHIFT) - | ((unsigned)contents.metadata.szind << (RTREE_LEAF_STATE_SHIFT + - RTREE_LEAF_STATE_WIDTH)); + | ((unsigned)contents.metadata.szind + << (RTREE_LEAF_STATE_SHIFT + RTREE_LEAF_STATE_WIDTH)); *bits = contents.edata; #endif } JEMALLOC_ALWAYS_INLINE void -rtree_leaf_elm_write_commit(tsdn_t *tsdn, rtree_t *rtree, - rtree_leaf_elm_t *elm, void *bits, unsigned additional) { +rtree_leaf_elm_write_commit(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, + void *bits, unsigned additional) { #ifdef RTREE_LEAF_COMPACT atomic_store_p(&elm->le_bits, bits, ATOMIC_RELEASE); #else @@ -294,12 +306,11 @@ rtree_leaf_elm_write_commit(tsdn_t *tsdn, rtree_t *rtree, } JEMALLOC_ALWAYS_INLINE void -rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, - rtree_leaf_elm_t *elm, rtree_contents_t contents) { +rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, + rtree_contents_t contents) { assert((uintptr_t)contents.edata % EDATA_ALIGNMENT == 0); - void *bits; + void *bits; unsigned additional; - rtree_contents_encode(contents, &bits, &additional); rtree_leaf_elm_write_commit(tsdn, rtree, elm, bits, additional); } @@ -314,8 +325,10 @@ rtree_leaf_elm_state_update(tsdn_t *tsdn, rtree_t *rtree, /* dependent */ true); bits &= ~RTREE_LEAF_STATE_MASK; bits |= state << RTREE_LEAF_STATE_SHIFT; + /* NOLINTNEXTLINE(performance-no-int-to-ptr) */ atomic_store_p(&elm1->le_bits, (void *)bits, ATOMIC_RELEASE); if (elm2 != NULL) { + /* NOLINTNEXTLINE(performance-no-int-to-ptr) */ atomic_store_p(&elm2->le_bits, (void *)bits, ATOMIC_RELEASE); } #else @@ -337,7 +350,7 @@ rtree_leaf_elm_state_update(tsdn_t *tsdn, rtree_t *rtree, JEMALLOC_ALWAYS_INLINE bool rtree_leaf_elm_lookup_fast(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key, rtree_leaf_elm_t **elm) { - size_t slot = rtree_cache_direct_map(key); + size_t slot = rtree_cache_direct_map(key); uintptr_t leafkey = rtree_leafkey(key); assert(leafkey != RTREE_LEAFKEY_INVALID); @@ -347,7 +360,7 @@ rtree_leaf_elm_lookup_fast(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, rtree_leaf_elm_t *leaf = rtree_ctx->cache[slot].leaf; assert(leaf != NULL); - uintptr_t subkey = rtree_subkey(key, RTREE_HEIGHT-1); + uintptr_t subkey = rtree_subkey(key, RTREE_HEIGHT - 1); *elm = &leaf[subkey]; return false; @@ -359,7 +372,7 @@ rtree_leaf_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, assert(key != 0); assert(!dependent || !init_missing); - size_t slot = rtree_cache_direct_map(key); + size_t slot = rtree_cache_direct_map(key); uintptr_t leafkey = rtree_leafkey(key); assert(leafkey != RTREE_LEAFKEY_INVALID); @@ -367,39 +380,41 @@ rtree_leaf_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, if (likely(rtree_ctx->cache[slot].leafkey == leafkey)) { rtree_leaf_elm_t *leaf = rtree_ctx->cache[slot].leaf; assert(leaf != NULL); - uintptr_t subkey = rtree_subkey(key, RTREE_HEIGHT-1); + uintptr_t subkey = rtree_subkey(key, RTREE_HEIGHT - 1); return &leaf[subkey]; } /* * Search the L2 LRU cache. On hit, swap the matching element into the * slot in L1 cache, and move the position in L2 up by 1. */ -#define RTREE_CACHE_CHECK_L2(i) do { \ - if (likely(rtree_ctx->l2_cache[i].leafkey == leafkey)) { \ - rtree_leaf_elm_t *leaf = rtree_ctx->l2_cache[i].leaf; \ - assert(leaf != NULL); \ - if (i > 0) { \ - /* Bubble up by one. */ \ - rtree_ctx->l2_cache[i].leafkey = \ - rtree_ctx->l2_cache[i - 1].leafkey; \ - rtree_ctx->l2_cache[i].leaf = \ - rtree_ctx->l2_cache[i - 1].leaf; \ - rtree_ctx->l2_cache[i - 1].leafkey = \ - rtree_ctx->cache[slot].leafkey; \ - rtree_ctx->l2_cache[i - 1].leaf = \ - rtree_ctx->cache[slot].leaf; \ - } else { \ - rtree_ctx->l2_cache[0].leafkey = \ - rtree_ctx->cache[slot].leafkey; \ - rtree_ctx->l2_cache[0].leaf = \ - rtree_ctx->cache[slot].leaf; \ - } \ - rtree_ctx->cache[slot].leafkey = leafkey; \ - rtree_ctx->cache[slot].leaf = leaf; \ - uintptr_t subkey = rtree_subkey(key, RTREE_HEIGHT-1); \ - return &leaf[subkey]; \ - } \ -} while (0) +#define RTREE_CACHE_CHECK_L2(i) \ + do { \ + if (likely(rtree_ctx->l2_cache[i].leafkey == leafkey)) { \ + rtree_leaf_elm_t *leaf = rtree_ctx->l2_cache[i].leaf; \ + assert(leaf != NULL); \ + if (i > 0) { \ + /* Bubble up by one. */ \ + rtree_ctx->l2_cache[i].leafkey = \ + rtree_ctx->l2_cache[i - 1].leafkey; \ + rtree_ctx->l2_cache[i].leaf = \ + rtree_ctx->l2_cache[i - 1].leaf; \ + rtree_ctx->l2_cache[i - 1].leafkey = \ + rtree_ctx->cache[slot].leafkey; \ + rtree_ctx->l2_cache[i - 1].leaf = \ + rtree_ctx->cache[slot].leaf; \ + } else { \ + rtree_ctx->l2_cache[0].leafkey = \ + rtree_ctx->cache[slot].leafkey; \ + rtree_ctx->l2_cache[0].leaf = \ + rtree_ctx->cache[slot].leaf; \ + } \ + rtree_ctx->cache[slot].leafkey = leafkey; \ + rtree_ctx->cache[slot].leaf = leaf; \ + uintptr_t subkey = rtree_subkey( \ + key, RTREE_HEIGHT - 1); \ + return &leaf[subkey]; \ + } \ + } while (0) /* Check the first cache entry. */ RTREE_CACHE_CHECK_L2(0); /* Search the remaining cache elements. */ @@ -408,8 +423,8 @@ rtree_leaf_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, } #undef RTREE_CACHE_CHECK_L2 - return rtree_leaf_elm_lookup_hard(tsdn, rtree, rtree_ctx, key, - dependent, init_missing); + return rtree_leaf_elm_lookup_hard( + tsdn, rtree, rtree_ctx, key, dependent, init_missing); } /* @@ -429,8 +444,8 @@ rtree_read_independent(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, } static inline rtree_contents_t -rtree_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, - uintptr_t key) { +rtree_read( + tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key) { rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx, key, /* dependent */ true, /* init_missing */ false); assert(elm != NULL); @@ -438,21 +453,22 @@ rtree_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, } static inline rtree_metadata_t -rtree_metadata_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, - uintptr_t key) { +rtree_metadata_read( + tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key) { rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx, key, /* dependent */ true, /* init_missing */ false); assert(elm != NULL); return rtree_leaf_elm_read(tsdn, rtree, elm, - /* dependent */ true).metadata; + /* dependent */ true) + .metadata; } /* * Returns true when the request cannot be fulfilled by fastpath. */ static inline bool -rtree_metadata_try_read_fast(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, - uintptr_t key, rtree_metadata_t *r_rtree_metadata) { +rtree_metadata_try_read_fast(tsdn_t *tsdn, rtree_t *rtree, + rtree_ctx_t *rtree_ctx, uintptr_t key, rtree_metadata_t *r_rtree_metadata) { rtree_leaf_elm_t *elm; /* * Should check the bool return value (lookup success or not) instead of @@ -465,7 +481,8 @@ rtree_metadata_try_read_fast(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ct } assert(elm != NULL); *r_rtree_metadata = rtree_leaf_elm_read(tsdn, rtree, elm, - /* dependent */ true).metadata; + /* dependent */ true) + .metadata; return false; } @@ -479,22 +496,27 @@ rtree_write_range_impl(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, * are dependent w/o init_missing, assuming the range spans across at * most 2 rtree leaf nodes (each covers 1 GiB of vaddr). */ - void *bits; + void *bits; unsigned additional; rtree_contents_encode(contents, &bits, &additional); rtree_leaf_elm_t *elm = NULL; /* Dead store. */ for (uintptr_t addr = base; addr <= end; addr += PAGE) { - if (addr == base || - (addr & ((ZU(1) << rtree_leaf_maskbits()) - 1)) == 0) { - elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx, addr, + if (addr == base + || (addr & ((ZU(1) << rtree_leaf_maskbits()) - 1)) == 0) { + elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx, + addr, /* dependent */ true, /* init_missing */ false); assert(elm != NULL); } - assert(elm == rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx, addr, - /* dependent */ true, /* init_missing */ false)); - assert(!clearing || rtree_leaf_elm_read(tsdn, rtree, elm, - /* dependent */ true).edata != NULL); + assert(elm + == rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx, addr, + /* dependent */ true, /* init_missing */ false)); + assert(!clearing + || rtree_leaf_elm_read(tsdn, rtree, elm, + /* dependent */ true) + .edata + != NULL); rtree_leaf_elm_write_commit(tsdn, rtree, elm, bits, additional); elm++; } @@ -522,13 +544,15 @@ rtree_write(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key, } static inline void -rtree_clear(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, - uintptr_t key) { +rtree_clear( + tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key) { rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx, key, /* dependent */ true, /* init_missing */ false); assert(elm != NULL); assert(rtree_leaf_elm_read(tsdn, rtree, elm, - /* dependent */ true).edata != NULL); + /* dependent */ true) + .edata + != NULL); rtree_contents_t contents; contents.edata = NULL; contents.metadata.szind = SC_NSIZES; diff --git a/dep/jemalloc/include/jemalloc/internal/rtree_tsd.h b/dep/jemalloc/include/jemalloc/internal/rtree_tsd.h index e45525c5e..4014dde08 100644 --- a/dep/jemalloc/include/jemalloc/internal/rtree_tsd.h +++ b/dep/jemalloc/include/jemalloc/internal/rtree_tsd.h @@ -1,6 +1,8 @@ #ifndef JEMALLOC_INTERNAL_RTREE_CTX_H #define JEMALLOC_INTERNAL_RTREE_CTX_H +#include "jemalloc/internal/jemalloc_preamble.h" + /* * Number of leafkey/leaf pairs to cache in L1 and L2 level respectively. Each * entry supports an entire leaf, so the cache hit rate is typically high even @@ -23,7 +25,8 @@ /* Needed for initialization only. */ #define RTREE_LEAFKEY_INVALID ((uintptr_t)1) -#define RTREE_CTX_CACHE_ELM_INVALID {RTREE_LEAFKEY_INVALID, NULL} +#define RTREE_CTX_CACHE_ELM_INVALID \ + { RTREE_LEAFKEY_INVALID, NULL } #define RTREE_CTX_INIT_ELM_1 RTREE_CTX_CACHE_ELM_INVALID #define RTREE_CTX_INIT_ELM_2 RTREE_CTX_INIT_ELM_1, RTREE_CTX_INIT_ELM_1 @@ -38,23 +41,27 @@ * Static initializer (to invalidate the cache entries) is required because the * free fastpath may access the rtree cache before a full tsd initialization. */ -#define RTREE_CTX_INITIALIZER {{RTREE_CTX_INIT_ELM_DATA(RTREE_CTX_NCACHE)}, \ - {RTREE_CTX_INIT_ELM_DATA(RTREE_CTX_NCACHE_L2)}} +#define RTREE_CTX_INITIALIZER \ + { \ + {RTREE_CTX_INIT_ELM_DATA(RTREE_CTX_NCACHE)}, { \ + RTREE_CTX_INIT_ELM_DATA(RTREE_CTX_NCACHE_L2) \ + } \ + } typedef struct rtree_leaf_elm_s rtree_leaf_elm_t; typedef struct rtree_ctx_cache_elm_s rtree_ctx_cache_elm_t; struct rtree_ctx_cache_elm_s { - uintptr_t leafkey; - rtree_leaf_elm_t *leaf; + uintptr_t leafkey; + rtree_leaf_elm_t *leaf; }; typedef struct rtree_ctx_s rtree_ctx_t; struct rtree_ctx_s { /* Direct mapped cache. */ - rtree_ctx_cache_elm_t cache[RTREE_CTX_NCACHE]; + rtree_ctx_cache_elm_t cache[RTREE_CTX_NCACHE]; /* L2 LRU cache. */ - rtree_ctx_cache_elm_t l2_cache[RTREE_CTX_NCACHE_L2]; + rtree_ctx_cache_elm_t l2_cache[RTREE_CTX_NCACHE_L2]; }; void rtree_ctx_data_init(rtree_ctx_t *ctx); diff --git a/dep/jemalloc/include/jemalloc/internal/safety_check.h b/dep/jemalloc/include/jemalloc/internal/safety_check.h index f1a74f174..2b4b2d0e7 100644 --- a/dep/jemalloc/include/jemalloc/internal/safety_check.h +++ b/dep/jemalloc/include/jemalloc/internal/safety_check.h @@ -1,8 +1,14 @@ #ifndef JEMALLOC_INTERNAL_SAFETY_CHECK_H #define JEMALLOC_INTERNAL_SAFETY_CHECK_H -void safety_check_fail_sized_dealloc(bool current_dealloc, const void *ptr, - size_t true_size, size_t input_size); +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/assert.h" +#include "jemalloc/internal/pages.h" + +#define SAFETY_CHECK_DOUBLE_FREE_MAX_SCAN_DEFAULT 32 + +void safety_check_fail_sized_dealloc( + bool current_dealloc, const void *ptr, size_t true_size, size_t input_size); void safety_check_fail(const char *format, ...); typedef void (*safety_check_abort_hook_t)(const char *message); @@ -10,22 +16,51 @@ typedef void (*safety_check_abort_hook_t)(const char *message); /* Can set to NULL for a default. */ void safety_check_set_abort(safety_check_abort_hook_t abort_fn); +#define REDZONE_SIZE ((size_t)32) +#define REDZONE_FILL_VALUE 0xBC + +/* + * Normally the redzone extends `REDZONE_SIZE` bytes beyond the end of + * the allocation. However, we don't let the redzone extend onto another + * OS page because this would impose additional overhead if that page was + * not already resident in memory. + */ +JEMALLOC_ALWAYS_INLINE const unsigned char * +compute_redzone_end(const void *_ptr, size_t usize, size_t bumped_usize) { + const unsigned char *ptr = (const unsigned char *)_ptr; + const unsigned char *redzone_end = usize + REDZONE_SIZE < bumped_usize + ? &ptr[usize + REDZONE_SIZE] + : &ptr[bumped_usize]; + const unsigned char *page_end = (const unsigned char *) + ALIGNMENT_ADDR2CEILING(&ptr[usize], os_page); + return redzone_end < page_end ? redzone_end : page_end; +} + JEMALLOC_ALWAYS_INLINE void safety_check_set_redzone(void *ptr, size_t usize, size_t bumped_usize) { - assert(usize < bumped_usize); - for (size_t i = usize; i < bumped_usize && i < usize + 32; ++i) { - *((unsigned char *)ptr + i) = 0xBC; + assert(usize <= bumped_usize); + const unsigned char *redzone_end = compute_redzone_end( + ptr, usize, bumped_usize); + for (unsigned char *curr = &((unsigned char *)ptr)[usize]; + curr < redzone_end; curr++) { + *curr = REDZONE_FILL_VALUE; } } JEMALLOC_ALWAYS_INLINE void -safety_check_verify_redzone(const void *ptr, size_t usize, size_t bumped_usize) -{ - for (size_t i = usize; i < bumped_usize && i < usize + 32; ++i) { - if (unlikely(*((unsigned char *)ptr + i) != 0xBC)) { +safety_check_verify_redzone( + const void *ptr, size_t usize, size_t bumped_usize) { + const unsigned char *redzone_end = compute_redzone_end( + ptr, usize, bumped_usize); + for (const unsigned char *curr = &((const unsigned char *)ptr)[usize]; + curr < redzone_end; curr++) { + if (unlikely(*curr != REDZONE_FILL_VALUE)) { safety_check_fail("Use after free error\n"); } } } +#undef REDZONE_SIZE +#undef REDZONE_FILL_VALUE + #endif /*JEMALLOC_INTERNAL_SAFETY_CHECK_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/san.h b/dep/jemalloc/include/jemalloc/internal/san.h index 8813d6bbe..5dcae3766 100644 --- a/dep/jemalloc/include/jemalloc/internal/san.h +++ b/dep/jemalloc/include/jemalloc/internal/san.h @@ -1,8 +1,11 @@ #ifndef JEMALLOC_INTERNAL_GUARD_H #define JEMALLOC_INTERNAL_GUARD_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/ehooks.h" #include "jemalloc/internal/emap.h" +#include "jemalloc/internal/jemalloc_internal_externs.h" +#include "jemalloc/internal/tsd.h" #define SAN_PAGE_GUARD PAGE #define SAN_PAGE_GUARDS_SIZE (SAN_PAGE_GUARD * 2) @@ -29,22 +32,22 @@ void san_unguard_pages(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, * Unguard the extent, but don't modify emap boundaries. Must be called on an * extent that has been erased from emap and shouldn't be placed back. */ -void san_unguard_pages_pre_destroy(tsdn_t *tsdn, ehooks_t *ehooks, - edata_t *edata, emap_t *emap); +void san_unguard_pages_pre_destroy( + tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, emap_t *emap); void san_check_stashed_ptrs(void **ptrs, size_t nstashed, size_t usize); void tsd_san_init(tsd_t *tsd); void san_init(ssize_t lg_san_uaf_align); static inline void -san_guard_pages_two_sided(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, - emap_t *emap, bool remap) { +san_guard_pages_two_sided( + tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, emap_t *emap, bool remap) { san_guard_pages(tsdn, ehooks, edata, emap, true, true, remap); } static inline void -san_unguard_pages_two_sided(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, - emap_t *emap) { +san_unguard_pages_two_sided( + tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, emap_t *emap) { san_unguard_pages(tsdn, ehooks, edata, emap, true, true); } @@ -80,14 +83,14 @@ san_guard_enabled(void) { } static inline bool -san_large_extent_decide_guard(tsdn_t *tsdn, ehooks_t *ehooks, size_t size, - size_t alignment) { - if (opt_san_guard_large == 0 || ehooks_guard_will_fail(ehooks) || - tsdn_null(tsdn)) { +san_large_extent_decide_guard( + tsdn_t *tsdn, ehooks_t *ehooks, size_t size, size_t alignment) { + if (opt_san_guard_large == 0 || ehooks_guard_will_fail(ehooks) + || tsdn_null(tsdn)) { return false; } - tsd_t *tsd = tsdn_tsd(tsdn); + tsd_t *tsd = tsdn_tsd(tsdn); uint64_t n = tsd_san_extents_until_guard_large_get(tsd); assert(n >= 1); if (n > 1) { @@ -98,10 +101,10 @@ san_large_extent_decide_guard(tsdn_t *tsdn, ehooks_t *ehooks, size_t size, *tsd_san_extents_until_guard_largep_get(tsd) = n - 1; } - if (n == 1 && (alignment <= PAGE) && - (san_two_side_guarded_sz(size) <= SC_LARGE_MAXCLASS)) { - *tsd_san_extents_until_guard_largep_get(tsd) = - opt_san_guard_large; + if (n == 1 && (alignment <= PAGE) + && (san_two_side_guarded_sz(size) <= SC_LARGE_MAXCLASS)) { + *tsd_san_extents_until_guard_largep_get( + tsd) = opt_san_guard_large; return true; } else { assert(tsd_san_extents_until_guard_large_get(tsd) >= 1); @@ -111,17 +114,17 @@ san_large_extent_decide_guard(tsdn_t *tsdn, ehooks_t *ehooks, size_t size, static inline bool san_slab_extent_decide_guard(tsdn_t *tsdn, ehooks_t *ehooks) { - if (opt_san_guard_small == 0 || ehooks_guard_will_fail(ehooks) || - tsdn_null(tsdn)) { + if (opt_san_guard_small == 0 || ehooks_guard_will_fail(ehooks) + || tsdn_null(tsdn)) { return false; } - tsd_t *tsd = tsdn_tsd(tsdn); + tsd_t *tsd = tsdn_tsd(tsdn); uint64_t n = tsd_san_extents_until_guard_small_get(tsd); assert(n >= 1); if (n == 1) { - *tsd_san_extents_until_guard_smallp_get(tsd) = - opt_san_guard_small; + *tsd_san_extents_until_guard_smallp_get( + tsd) = opt_san_guard_small; return true; } else { *tsd_san_extents_until_guard_smallp_get(tsd) = n - 1; @@ -131,13 +134,13 @@ san_slab_extent_decide_guard(tsdn_t *tsdn, ehooks_t *ehooks) { } static inline void -san_junk_ptr_locations(void *ptr, size_t usize, void **first, void **mid, - void **last) { +san_junk_ptr_locations( + void *ptr, size_t usize, void **first, void **mid, void **last) { size_t ptr_sz = sizeof(void *); *first = ptr; - *mid = (void *)((uintptr_t)ptr + ((usize >> 1) & ~(ptr_sz - 1))); + *mid = (void *)((byte_t *)ptr + ((usize >> 1) & ~(ptr_sz - 1))); assert(*first != *mid || usize == ptr_sz); assert((uintptr_t)*first <= (uintptr_t)*mid); @@ -148,7 +151,7 @@ san_junk_ptr_locations(void *ptr, size_t usize, void **first, void **mid, * default the tcache only goes up to the 32K size class, and is usually * tuned lower instead of higher, which makes it less of a concern. */ - *last = (void *)((uintptr_t)ptr + usize - sizeof(uaf_detect_junk)); + *last = (void *)((byte_t *)ptr + usize - sizeof(uaf_detect_junk)); assert(*first != *last || usize == ptr_sz); assert(*mid != *last || usize <= ptr_sz * 2); assert((uintptr_t)*mid <= (uintptr_t)*last); @@ -181,8 +184,8 @@ static inline bool san_uaf_detection_enabled(void) { bool ret = config_uaf_detection && (opt_lg_san_uaf_align != -1); if (config_uaf_detection && ret) { - assert(san_cache_bin_nonfast_mask == ((uintptr_t)1 << - opt_lg_san_uaf_align) - 1); + assert(san_cache_bin_nonfast_mask + == ((uintptr_t)1 << opt_lg_san_uaf_align) - 1); } return ret; diff --git a/dep/jemalloc/include/jemalloc/internal/san_bump.h b/dep/jemalloc/include/jemalloc/internal/san_bump.h index 8ec4a710d..9e42b69b1 100644 --- a/dep/jemalloc/include/jemalloc/internal/san_bump.h +++ b/dep/jemalloc/include/jemalloc/internal/san_bump.h @@ -1,16 +1,18 @@ #ifndef JEMALLOC_INTERNAL_SAN_BUMP_H #define JEMALLOC_INTERNAL_SAN_BUMP_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/edata.h" #include "jemalloc/internal/exp_grow.h" #include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/witness.h" #define SBA_RETAINED_ALLOC_SIZE ((size_t)4 << 20) extern bool opt_retain; typedef struct ehooks_s ehooks_t; -typedef struct pac_s pac_t; +typedef struct pac_s pac_t; typedef struct san_bump_alloc_s san_bump_alloc_t; struct san_bump_alloc_s { @@ -20,7 +22,7 @@ struct san_bump_alloc_s { }; static inline bool -san_bump_enabled() { +san_bump_enabled(void) { /* * We enable san_bump allocator only when it's possible to break up a * mapping and unmap a part of it (maps_coalesce). This is needed to @@ -34,7 +36,7 @@ san_bump_enabled() { } static inline bool -san_bump_alloc_init(san_bump_alloc_t* sba) { +san_bump_alloc_init(san_bump_alloc_t *sba) { bool err = malloc_mutex_init(&sba->mtx, "sanitizer_bump_allocator", WITNESS_RANK_SAN_BUMP_ALLOC, malloc_mutex_rank_exclusive); if (err) { @@ -45,8 +47,7 @@ san_bump_alloc_init(san_bump_alloc_t* sba) { return false; } -edata_t * -san_bump_alloc(tsdn_t *tsdn, san_bump_alloc_t* sba, pac_t *pac, ehooks_t *ehooks, - size_t size, bool zero); +edata_t *san_bump_alloc(tsdn_t *tsdn, san_bump_alloc_t *sba, pac_t *pac, + ehooks_t *ehooks, size_t size, bool zero); #endif /* JEMALLOC_INTERNAL_SAN_BUMP_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/sc.h b/dep/jemalloc/include/jemalloc/internal/sc.h index 9bab347be..17a8278a0 100644 --- a/dep/jemalloc/include/jemalloc/internal/sc.h +++ b/dep/jemalloc/include/jemalloc/internal/sc.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_SC_H #define JEMALLOC_INTERNAL_SC_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/jemalloc_internal_types.h" /* @@ -173,7 +174,7 @@ #if SC_LG_TINY_MIN == 0 /* The div module doesn't support division by 1, which this would require. */ -#error "Unsupported LG_TINY_MIN" +# error "Unsupported LG_TINY_MIN" #endif /* @@ -193,8 +194,8 @@ * We could probably save some space in arenas by capping this at LG_VADDR size. */ #define SC_LG_BASE_MAX (SC_PTR_BITS - 2) -#define SC_NREGULAR (SC_NGROUP * \ - (SC_LG_BASE_MAX - SC_LG_FIRST_REGULAR_BASE + 1) - 1) +#define SC_NREGULAR \ + (SC_NGROUP * (SC_LG_BASE_MAX - SC_LG_FIRST_REGULAR_BASE + 1) - 1) #define SC_NSIZES (SC_NTINY + SC_NPSEUDO + SC_NREGULAR) /* @@ -221,29 +222,29 @@ * * This gives us the quantity we seek. */ -#define SC_NPSIZES ( \ - SC_NGROUP \ - + (SC_LG_BASE_MAX - (LG_PAGE + SC_LG_NGROUP)) * SC_NGROUP \ - + SC_NGROUP - 1) +#define SC_NPSIZES \ + (SC_NGROUP + (SC_LG_BASE_MAX - (LG_PAGE + SC_LG_NGROUP)) * SC_NGROUP \ + + SC_NGROUP - 1) /* * We declare a size class is binnable if size < page size * group. Or, in other * words, lg(size) < lg(page size) + lg(group size). */ -#define SC_NBINS ( \ - /* Sub-regular size classes. */ \ - SC_NTINY + SC_NPSEUDO \ - /* Groups with lg_regular_min_base <= lg_base <= lg_base_max */ \ - + SC_NGROUP * (LG_PAGE + SC_LG_NGROUP - SC_LG_FIRST_REGULAR_BASE) \ - /* Last SC of the last group hits the bound exactly; exclude it. */ \ - - 1) +#define SC_NBINS \ + (/* Sub-regular size classes. */ \ + SC_NTINY \ + + SC_NPSEUDO /* Groups with lg_regular_min_base <= lg_base <= lg_base_max */ \ + + SC_NGROUP \ + * (LG_PAGE + SC_LG_NGROUP \ + - SC_LG_FIRST_REGULAR_BASE) /* Last SC of the last group hits the bound exactly; exclude it. */ \ + - 1) /* * The size2index_tab lookup table uses uint8_t to encode each bin index, so we * cannot support more than 256 small size classes. */ #if (SC_NBINS > 256) -# error "Too many small size classes" +# error "Too many small size classes" #endif /* The largest size class in the lookup table, and its binary log. */ @@ -255,12 +256,12 @@ #define SC_SMALL_MAX_DELTA (1 << (LG_PAGE - 1)) /* The largest size class allocated out of a slab. */ -#define SC_SMALL_MAXCLASS (SC_SMALL_MAX_BASE \ - + (SC_NGROUP - 1) * SC_SMALL_MAX_DELTA) +#define SC_SMALL_MAXCLASS \ + (SC_SMALL_MAX_BASE + (SC_NGROUP - 1) * SC_SMALL_MAX_DELTA) /* The fastpath assumes all lookup-able sizes are small. */ #if (SC_SMALL_MAXCLASS < SC_LOOKUP_MAXCLASS) -# error "Lookup table sizes must be small" +# error "Lookup table sizes must be small" #endif /* The smallest size class not allocated out of a slab. */ @@ -276,15 +277,33 @@ /* Maximum number of regions in one slab. */ #ifndef CONFIG_LG_SLAB_MAXREGS -# define SC_LG_SLAB_MAXREGS (LG_PAGE - SC_LG_TINY_MIN) +# define SC_LG_SLAB_MAXREGS (LG_PAGE - SC_LG_TINY_MIN) #else -# if CONFIG_LG_SLAB_MAXREGS < (LG_PAGE - SC_LG_TINY_MIN) -# error "Unsupported SC_LG_SLAB_MAXREGS" -# else -# define SC_LG_SLAB_MAXREGS CONFIG_LG_SLAB_MAXREGS -# endif +# if CONFIG_LG_SLAB_MAXREGS < (LG_PAGE - SC_LG_TINY_MIN) +# error "Unsupported SC_LG_SLAB_MAXREGS" +# else +# define SC_LG_SLAB_MAXREGS CONFIG_LG_SLAB_MAXREGS +# endif #endif +/* + * When large size classes are disabled, there is no concept of size classes + * for sizes > SC_SMALLMAXCLASS (or >= SC_LARGE_MINCLASS). This ensures that + * the overhead between the usable size and the user request size will not + * exceed PAGE. Between SC_LARGE_MINCLASS (SC_NGROUP * PAGE) and + * 2 * SC_NGROUP * PAGE, the size classes also happen to be aligned with PAGE. + * Since tcache relies on size classes to work and it greatly increases the + * perf of allocs & deallocs, we extend the existence of size class to + * 2 * SC_NGROUP * PAGE ONLY for the tcache module. This means for all other + * modules, there is no size class for sizes >= SC_LARGE_MINCLASS. Yet for + * tcache, the threshold is moved up to 2 * SC_NGROUP * PAGE, which is + * USIZE_GROW_SLOW_THRESHOLD defined below. With the default SC_NGROUP being + * 2, and PAGE being 4KB, the threshold for tcache (USIZE_GROW_SLOW_THRESHOLD) + * is 32KB. + */ +#define LG_USIZE_GROW_SLOW_THRESHOLD (SC_LG_NGROUP + LG_PAGE + 1) +#define USIZE_GROW_SLOW_THRESHOLD (1U << LG_USIZE_GROW_SLOW_THRESHOLD) + #define SC_SLAB_MAXREGS (1U << SC_LG_SLAB_MAXREGS) typedef struct sc_s sc_t; @@ -345,13 +364,13 @@ struct sc_data_s { }; size_t reg_size_compute(int lg_base, int lg_delta, int ndelta); -void sc_data_init(sc_data_t *data); +void sc_data_init(sc_data_t *data); /* * Updates slab sizes in [begin, end] to be pgs pages in length, if possible. * Otherwise, does its best to accommodate the request. */ -void sc_data_update_slab_size(sc_data_t *data, size_t begin, size_t end, - int pgs); +void sc_data_update_slab_size( + sc_data_t *data, size_t begin, size_t end, int pgs); void sc_boot(sc_data_t *data); #endif /* JEMALLOC_INTERNAL_SC_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/sec.h b/dep/jemalloc/include/jemalloc/internal/sec.h index fa863382d..cc458b9d5 100644 --- a/dep/jemalloc/include/jemalloc/internal/sec.h +++ b/dep/jemalloc/include/jemalloc/internal/sec.h @@ -1,8 +1,12 @@ #ifndef JEMALLOC_INTERNAL_SEC_H #define JEMALLOC_INTERNAL_SEC_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/base.h" #include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/mutex.h" #include "jemalloc/internal/pai.h" +#include "jemalloc/internal/sec_opts.h" /* * Small extent cache. @@ -13,91 +17,104 @@ * knowledge of the underlying PAI implementation). */ -/* - * For now, this is just one field; eventually, we'll probably want to get more - * fine-grained data out (like per-size class statistics). - */ +typedef struct sec_bin_stats_s sec_bin_stats_t; +struct sec_bin_stats_s { + /* Number of alloc requests that did not find extent in this bin */ + size_t nmisses; + /* Number of successful alloc requests. */ + size_t nhits; + /* Number of dallocs causing the flush */ + size_t ndalloc_flush; + /* Number of dallocs not causing the flush */ + size_t ndalloc_noflush; + /* Number of fills that hit max_bytes */ + size_t noverfills; +}; typedef struct sec_stats_s sec_stats_t; struct sec_stats_s { /* Sum of bytes_cur across all shards. */ size_t bytes; + + /* Totals of bin_stats. */ + sec_bin_stats_t total; }; +static inline void +sec_bin_stats_init(sec_bin_stats_t *stats) { + stats->ndalloc_flush = 0; + stats->nmisses = 0; + stats->nhits = 0; + stats->ndalloc_noflush = 0; + stats->noverfills = 0; +} + +static inline void +sec_bin_stats_accum(sec_bin_stats_t *dst, sec_bin_stats_t *src) { + dst->nmisses += src->nmisses; + dst->nhits += src->nhits; + dst->ndalloc_flush += src->ndalloc_flush; + dst->ndalloc_noflush += src->ndalloc_noflush; + dst->noverfills += src->noverfills; +} + static inline void sec_stats_accum(sec_stats_t *dst, sec_stats_t *src) { dst->bytes += src->bytes; + sec_bin_stats_accum(&dst->total, &src->total); } /* A collections of free extents, all of the same size. */ typedef struct sec_bin_s sec_bin_t; struct sec_bin_s { /* - * When we fail to fulfill an allocation, we do a batch-alloc on the - * underlying allocator to fill extra items, as well. We drop the SEC - * lock while doing so, to allow operations on other bins to succeed. - * That introduces the possibility of other threads also trying to - * allocate out of this bin, failing, and also going to the backing - * allocator. To avoid a thundering herd problem in which lots of - * threads do batch allocs and overfill this bin as a result, we only - * allow one batch allocation at a time for a bin. This bool tracks - * whether or not some thread is already batch allocating. - * - * Eventually, the right answer may be a smarter sharding policy for the - * bins (e.g. a mutex per bin, which would also be more scalable - * generally; the batch-allocating thread could hold it while - * batch-allocating). - */ - bool being_batch_filled; - - /* - * Number of bytes in this particular bin (as opposed to the - * sec_shard_t's bytes_cur. This isn't user visible or reported in - * stats; rather, it allows us to quickly determine the change in the - * centralized counter when flushing. - */ - size_t bytes_cur; - edata_list_active_t freelist; -}; - -typedef struct sec_shard_s sec_shard_t; -struct sec_shard_s { - /* - * We don't keep per-bin mutexes, even though that would allow more - * sharding; this allows global cache-eviction, which in turn allows for - * better balancing across free lists. + * Protects the data members of the bin. */ malloc_mutex_t mtx; + /* - * A SEC may need to be shut down (i.e. flushed of its contents and - * prevented from further caching). To avoid tricky synchronization - * issues, we just track enabled-status in each shard, guarded by a - * mutex. In practice, this is only ever checked during brief races, - * since the arena-level atomic boolean tracking HPA enabled-ness means - * that we won't go down these pathways very often after custom extent - * hooks are installed. + * Number of bytes in this particular bin. */ - bool enabled; - sec_bin_t *bins; - /* Number of bytes in all bins in the shard. */ - size_t bytes_cur; - /* The next pszind to flush in the flush-some pathways. */ - pszind_t to_flush_next; + size_t bytes_cur; + edata_list_active_t freelist; + sec_bin_stats_t stats; }; typedef struct sec_s sec_t; struct sec_s { - pai_t pai; - pai_t *fallback; - sec_opts_t opts; - sec_shard_t *shards; - pszind_t npsizes; + sec_bin_t *bins; + pszind_t npsizes; }; -bool sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, pai_t *fallback, - const sec_opts_t *opts); -void sec_flush(tsdn_t *tsdn, sec_t *sec); -void sec_disable(tsdn_t *tsdn, sec_t *sec); +static inline bool +sec_is_used(sec_t *sec) { + return sec->opts.nshards != 0; +} + +static inline bool +sec_size_supported(sec_t *sec, size_t size) { + return sec_is_used(sec) && size <= sec->opts.max_alloc; +} + +/* If sec does not have extent available, it will return NULL. */ +edata_t *sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size); +void sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size, + edata_list_active_t *result, size_t nallocs); + +/* + * Upon return dalloc_list may be empty if edata is consumed by sec or non-empty + * if there are extents that need to be flushed from cache. Please note, that + * if we need to flush, extent(s) returned in the list to be deallocated + * will almost certainly not contain the one being dalloc-ed (that one will be + * considered "hot" and preserved in the cache, while "colder" ones are + * returned). + */ +void sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list); + +bool sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts); + +/* Fills to_flush with extents that need to be deallocated */ +void sec_flush(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *to_flush); /* * Morally, these two stats methods probably ought to be a single one (and the @@ -106,8 +123,8 @@ void sec_disable(tsdn_t *tsdn, sec_t *sec); * split), which simplifies the stats management. */ void sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats); -void sec_mutex_stats_read(tsdn_t *tsdn, sec_t *sec, - mutex_prof_data_t *mutex_prof_data); +void sec_mutex_stats_read( + tsdn_t *tsdn, sec_t *sec, mutex_prof_data_t *mutex_prof_data); /* * We use the arena lock ordering; these are acquired in phase 2 of forking, but diff --git a/dep/jemalloc/include/jemalloc/internal/sec_opts.h b/dep/jemalloc/include/jemalloc/internal/sec_opts.h index a3ad72fbe..039d423c3 100644 --- a/dep/jemalloc/include/jemalloc/internal/sec_opts.h +++ b/dep/jemalloc/include/jemalloc/internal/sec_opts.h @@ -1,6 +1,8 @@ #ifndef JEMALLOC_INTERNAL_SEC_OPTS_H #define JEMALLOC_INTERNAL_SEC_OPTS_H +#include "jemalloc/internal/jemalloc_preamble.h" + /* * The configuration settings used by an sec_t. Morally, this is part of the * SEC interface, but we put it here for header-ordering reasons. @@ -10,50 +12,39 @@ typedef struct sec_opts_s sec_opts_t; struct sec_opts_s { /* * We don't necessarily always use all the shards; requests are - * distributed across shards [0, nshards - 1). + * distributed across shards [0, nshards - 1). Once thread picks a + * shard it will always use that one. If this value is set to 0 sec is + * not used. */ size_t nshards; /* * We'll automatically refuse to cache any objects in this sec if - * they're larger than max_alloc bytes, instead forwarding such objects - * directly to the fallback. + * they're larger than max_alloc bytes. */ size_t max_alloc; /* - * Exceeding this amount of cached extents in a shard causes us to start - * flushing bins in that shard until we fall below bytes_after_flush. + * Exceeding this amount of cached extents in a bin causes us to flush + * until we are 1/4 below max_bytes. */ size_t max_bytes; - /* - * The number of bytes (in all bins) we flush down to when we exceed - * bytes_cur. We want this to be less than bytes_cur, because - * otherwise we could get into situations where a shard undergoing - * net-deallocation keeps bytes_cur very near to max_bytes, so that - * most deallocations get immediately forwarded to the underlying PAI - * implementation, defeating the point of the SEC. - */ - size_t bytes_after_flush; /* * When we can't satisfy an allocation out of the SEC because there are - * no available ones cached, we allocate multiple of that size out of - * the fallback allocator. Eventually we might want to do something - * cleverer, but for now we just grab a fixed number. + * no available ones cached, allocator will allocate a batch with extra + * batch_fill_extra extents of the same size. */ size_t batch_fill_extra; }; -#define SEC_OPTS_DEFAULT { \ - /* nshards */ \ - 4, \ - /* max_alloc */ \ - (32 * 1024) < PAGE ? PAGE : (32 * 1024), \ - /* max_bytes */ \ - 256 * 1024, \ - /* bytes_after_flush */ \ - 128 * 1024, \ - /* batch_fill_extra */ \ - 0 \ -} +#define SEC_OPTS_NSHARDS_DEFAULT 2 +#define SEC_OPTS_BATCH_FILL_EXTRA_DEFAULT 3 +#define SEC_OPTS_MAX_ALLOC_DEFAULT ((32 * 1024) < PAGE ? PAGE : (32 * 1024)) +#define SEC_OPTS_MAX_BYTES_DEFAULT \ + ((256 * 1024) < (4 * SEC_OPTS_MAX_ALLOC_DEFAULT) \ + ? (4 * SEC_OPTS_MAX_ALLOC_DEFAULT) \ + : (256 * 1024)) +#define SEC_OPTS_DEFAULT \ + {SEC_OPTS_NSHARDS_DEFAULT, SEC_OPTS_MAX_ALLOC_DEFAULT, \ + SEC_OPTS_MAX_BYTES_DEFAULT, SEC_OPTS_BATCH_FILL_EXTRA_DEFAULT} #endif /* JEMALLOC_INTERNAL_SEC_OPTS_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/seq.h b/dep/jemalloc/include/jemalloc/internal/seq.h index ef2df4c6e..d2c0d1fc6 100644 --- a/dep/jemalloc/include/jemalloc/internal/seq.h +++ b/dep/jemalloc/include/jemalloc/internal/seq.h @@ -1,12 +1,14 @@ #ifndef JEMALLOC_INTERNAL_SEQ_H #define JEMALLOC_INTERNAL_SEQ_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/atomic.h" /* * A simple seqlock implementation. */ +/* clang-format off */ #define seq_define(type, short_type) \ typedef struct { \ atomic_zu_t seq; \ @@ -51,5 +53,6 @@ seq_try_load_##short_type(type *dst, seq_##short_type##_t *src) { \ memcpy(dst, buf, sizeof(type)); \ return true; \ } +/* clang-format on */ #endif /* JEMALLOC_INTERNAL_SEQ_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/slab_data.h b/dep/jemalloc/include/jemalloc/internal/slab_data.h index e821863d8..724c71e36 100644 --- a/dep/jemalloc/include/jemalloc/internal/slab_data.h +++ b/dep/jemalloc/include/jemalloc/internal/slab_data.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_SLAB_DATA_H #define JEMALLOC_INTERNAL_SLAB_DATA_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/bitmap.h" typedef struct slab_data_s slab_data_t; diff --git a/dep/jemalloc/include/jemalloc/internal/smoothstep.h b/dep/jemalloc/include/jemalloc/internal/smoothstep.h index 2e14430f5..135c49053 100644 --- a/dep/jemalloc/include/jemalloc/internal/smoothstep.h +++ b/dep/jemalloc/include/jemalloc/internal/smoothstep.h @@ -23,210 +23,210 @@ * smootheststep(x) = -20x + 70x - 84x + 35x */ -#define SMOOTHSTEP_VARIANT "smoother" -#define SMOOTHSTEP_NSTEPS 200 -#define SMOOTHSTEP_BFP 24 -#define SMOOTHSTEP \ - /* STEP(step, h, x, y) */ \ - STEP( 1, UINT64_C(0x0000000000000014), 0.005, 0.000001240643750) \ - STEP( 2, UINT64_C(0x00000000000000a5), 0.010, 0.000009850600000) \ - STEP( 3, UINT64_C(0x0000000000000229), 0.015, 0.000032995181250) \ - STEP( 4, UINT64_C(0x0000000000000516), 0.020, 0.000077619200000) \ - STEP( 5, UINT64_C(0x00000000000009dc), 0.025, 0.000150449218750) \ - STEP( 6, UINT64_C(0x00000000000010e8), 0.030, 0.000257995800000) \ - STEP( 7, UINT64_C(0x0000000000001aa4), 0.035, 0.000406555756250) \ - STEP( 8, UINT64_C(0x0000000000002777), 0.040, 0.000602214400000) \ - STEP( 9, UINT64_C(0x00000000000037c2), 0.045, 0.000850847793750) \ - STEP( 10, UINT64_C(0x0000000000004be6), 0.050, 0.001158125000000) \ - STEP( 11, UINT64_C(0x000000000000643c), 0.055, 0.001529510331250) \ - STEP( 12, UINT64_C(0x000000000000811f), 0.060, 0.001970265600000) \ - STEP( 13, UINT64_C(0x000000000000a2e2), 0.065, 0.002485452368750) \ - STEP( 14, UINT64_C(0x000000000000c9d8), 0.070, 0.003079934200000) \ - STEP( 15, UINT64_C(0x000000000000f64f), 0.075, 0.003758378906250) \ - STEP( 16, UINT64_C(0x0000000000012891), 0.080, 0.004525260800000) \ - STEP( 17, UINT64_C(0x00000000000160e7), 0.085, 0.005384862943750) \ - STEP( 18, UINT64_C(0x0000000000019f95), 0.090, 0.006341279400000) \ - STEP( 19, UINT64_C(0x000000000001e4dc), 0.095, 0.007398417481250) \ - STEP( 20, UINT64_C(0x00000000000230fc), 0.100, 0.008560000000000) \ - STEP( 21, UINT64_C(0x0000000000028430), 0.105, 0.009829567518750) \ - STEP( 22, UINT64_C(0x000000000002deb0), 0.110, 0.011210480600000) \ - STEP( 23, UINT64_C(0x00000000000340b1), 0.115, 0.012705922056250) \ - STEP( 24, UINT64_C(0x000000000003aa67), 0.120, 0.014318899200000) \ - STEP( 25, UINT64_C(0x0000000000041c00), 0.125, 0.016052246093750) \ - STEP( 26, UINT64_C(0x00000000000495a8), 0.130, 0.017908625800000) \ - STEP( 27, UINT64_C(0x000000000005178b), 0.135, 0.019890532631250) \ - STEP( 28, UINT64_C(0x000000000005a1cf), 0.140, 0.022000294400000) \ - STEP( 29, UINT64_C(0x0000000000063498), 0.145, 0.024240074668750) \ - STEP( 30, UINT64_C(0x000000000006d009), 0.150, 0.026611875000000) \ - STEP( 31, UINT64_C(0x000000000007743f), 0.155, 0.029117537206250) \ - STEP( 32, UINT64_C(0x0000000000082157), 0.160, 0.031758745600000) \ - STEP( 33, UINT64_C(0x000000000008d76b), 0.165, 0.034537029243750) \ - STEP( 34, UINT64_C(0x0000000000099691), 0.170, 0.037453764200000) \ - STEP( 35, UINT64_C(0x00000000000a5edf), 0.175, 0.040510175781250) \ - STEP( 36, UINT64_C(0x00000000000b3067), 0.180, 0.043707340800000) \ - STEP( 37, UINT64_C(0x00000000000c0b38), 0.185, 0.047046189818750) \ - STEP( 38, UINT64_C(0x00000000000cef5e), 0.190, 0.050527509400000) \ - STEP( 39, UINT64_C(0x00000000000ddce6), 0.195, 0.054151944356250) \ - STEP( 40, UINT64_C(0x00000000000ed3d8), 0.200, 0.057920000000000) \ - STEP( 41, UINT64_C(0x00000000000fd439), 0.205, 0.061832044393750) \ - STEP( 42, UINT64_C(0x000000000010de0e), 0.210, 0.065888310600000) \ - STEP( 43, UINT64_C(0x000000000011f158), 0.215, 0.070088898931250) \ - STEP( 44, UINT64_C(0x0000000000130e17), 0.220, 0.074433779200000) \ - STEP( 45, UINT64_C(0x0000000000143448), 0.225, 0.078922792968750) \ - STEP( 46, UINT64_C(0x00000000001563e7), 0.230, 0.083555655800000) \ - STEP( 47, UINT64_C(0x0000000000169cec), 0.235, 0.088331959506250) \ - STEP( 48, UINT64_C(0x000000000017df4f), 0.240, 0.093251174400000) \ - STEP( 49, UINT64_C(0x0000000000192b04), 0.245, 0.098312651543750) \ - STEP( 50, UINT64_C(0x00000000001a8000), 0.250, 0.103515625000000) \ - STEP( 51, UINT64_C(0x00000000001bde32), 0.255, 0.108859214081250) \ - STEP( 52, UINT64_C(0x00000000001d458b), 0.260, 0.114342425600000) \ - STEP( 53, UINT64_C(0x00000000001eb5f8), 0.265, 0.119964156118750) \ - STEP( 54, UINT64_C(0x0000000000202f65), 0.270, 0.125723194200000) \ - STEP( 55, UINT64_C(0x000000000021b1bb), 0.275, 0.131618222656250) \ - STEP( 56, UINT64_C(0x0000000000233ce3), 0.280, 0.137647820800000) \ - STEP( 57, UINT64_C(0x000000000024d0c3), 0.285, 0.143810466693750) \ - STEP( 58, UINT64_C(0x0000000000266d40), 0.290, 0.150104539400000) \ - STEP( 59, UINT64_C(0x000000000028123d), 0.295, 0.156528321231250) \ - STEP( 60, UINT64_C(0x000000000029bf9c), 0.300, 0.163080000000000) \ - STEP( 61, UINT64_C(0x00000000002b753d), 0.305, 0.169757671268750) \ - STEP( 62, UINT64_C(0x00000000002d32fe), 0.310, 0.176559340600000) \ - STEP( 63, UINT64_C(0x00000000002ef8bc), 0.315, 0.183482925806250) \ - STEP( 64, UINT64_C(0x000000000030c654), 0.320, 0.190526259200000) \ - STEP( 65, UINT64_C(0x0000000000329b9f), 0.325, 0.197687089843750) \ - STEP( 66, UINT64_C(0x0000000000347875), 0.330, 0.204963085800000) \ - STEP( 67, UINT64_C(0x0000000000365cb0), 0.335, 0.212351836381250) \ - STEP( 68, UINT64_C(0x0000000000384825), 0.340, 0.219850854400000) \ - STEP( 69, UINT64_C(0x00000000003a3aa8), 0.345, 0.227457578418750) \ - STEP( 70, UINT64_C(0x00000000003c340f), 0.350, 0.235169375000000) \ - STEP( 71, UINT64_C(0x00000000003e342b), 0.355, 0.242983540956250) \ - STEP( 72, UINT64_C(0x0000000000403ace), 0.360, 0.250897305600000) \ - STEP( 73, UINT64_C(0x00000000004247c8), 0.365, 0.258907832993750) \ - STEP( 74, UINT64_C(0x0000000000445ae9), 0.370, 0.267012224200000) \ - STEP( 75, UINT64_C(0x0000000000467400), 0.375, 0.275207519531250) \ - STEP( 76, UINT64_C(0x00000000004892d8), 0.380, 0.283490700800000) \ - STEP( 77, UINT64_C(0x00000000004ab740), 0.385, 0.291858693568750) \ - STEP( 78, UINT64_C(0x00000000004ce102), 0.390, 0.300308369400000) \ - STEP( 79, UINT64_C(0x00000000004f0fe9), 0.395, 0.308836548106250) \ - STEP( 80, UINT64_C(0x00000000005143bf), 0.400, 0.317440000000000) \ - STEP( 81, UINT64_C(0x0000000000537c4d), 0.405, 0.326115448143750) \ - STEP( 82, UINT64_C(0x000000000055b95b), 0.410, 0.334859570600000) \ - STEP( 83, UINT64_C(0x000000000057fab1), 0.415, 0.343669002681250) \ - STEP( 84, UINT64_C(0x00000000005a4015), 0.420, 0.352540339200000) \ - STEP( 85, UINT64_C(0x00000000005c894e), 0.425, 0.361470136718750) \ - STEP( 86, UINT64_C(0x00000000005ed622), 0.430, 0.370454915800000) \ - STEP( 87, UINT64_C(0x0000000000612655), 0.435, 0.379491163256250) \ - STEP( 88, UINT64_C(0x00000000006379ac), 0.440, 0.388575334400000) \ - STEP( 89, UINT64_C(0x000000000065cfeb), 0.445, 0.397703855293750) \ - STEP( 90, UINT64_C(0x00000000006828d6), 0.450, 0.406873125000000) \ - STEP( 91, UINT64_C(0x00000000006a842f), 0.455, 0.416079517831250) \ - STEP( 92, UINT64_C(0x00000000006ce1bb), 0.460, 0.425319385600000) \ - STEP( 93, UINT64_C(0x00000000006f413a), 0.465, 0.434589059868750) \ - STEP( 94, UINT64_C(0x000000000071a270), 0.470, 0.443884854200000) \ - STEP( 95, UINT64_C(0x000000000074051d), 0.475, 0.453203066406250) \ - STEP( 96, UINT64_C(0x0000000000766905), 0.480, 0.462539980800000) \ - STEP( 97, UINT64_C(0x000000000078cde7), 0.485, 0.471891870443750) \ - STEP( 98, UINT64_C(0x00000000007b3387), 0.490, 0.481254999400000) \ - STEP( 99, UINT64_C(0x00000000007d99a4), 0.495, 0.490625624981250) \ - STEP( 100, UINT64_C(0x0000000000800000), 0.500, 0.500000000000000) \ - STEP( 101, UINT64_C(0x000000000082665b), 0.505, 0.509374375018750) \ - STEP( 102, UINT64_C(0x000000000084cc78), 0.510, 0.518745000600000) \ - STEP( 103, UINT64_C(0x0000000000873218), 0.515, 0.528108129556250) \ - STEP( 104, UINT64_C(0x00000000008996fa), 0.520, 0.537460019200000) \ - STEP( 105, UINT64_C(0x00000000008bfae2), 0.525, 0.546796933593750) \ - STEP( 106, UINT64_C(0x00000000008e5d8f), 0.530, 0.556115145800000) \ - STEP( 107, UINT64_C(0x000000000090bec5), 0.535, 0.565410940131250) \ - STEP( 108, UINT64_C(0x0000000000931e44), 0.540, 0.574680614400000) \ - STEP( 109, UINT64_C(0x0000000000957bd0), 0.545, 0.583920482168750) \ - STEP( 110, UINT64_C(0x000000000097d729), 0.550, 0.593126875000000) \ - STEP( 111, UINT64_C(0x00000000009a3014), 0.555, 0.602296144706250) \ - STEP( 112, UINT64_C(0x00000000009c8653), 0.560, 0.611424665600000) \ - STEP( 113, UINT64_C(0x00000000009ed9aa), 0.565, 0.620508836743750) \ - STEP( 114, UINT64_C(0x0000000000a129dd), 0.570, 0.629545084200000) \ - STEP( 115, UINT64_C(0x0000000000a376b1), 0.575, 0.638529863281250) \ - STEP( 116, UINT64_C(0x0000000000a5bfea), 0.580, 0.647459660800000) \ - STEP( 117, UINT64_C(0x0000000000a8054e), 0.585, 0.656330997318750) \ - STEP( 118, UINT64_C(0x0000000000aa46a4), 0.590, 0.665140429400000) \ - STEP( 119, UINT64_C(0x0000000000ac83b2), 0.595, 0.673884551856250) \ - STEP( 120, UINT64_C(0x0000000000aebc40), 0.600, 0.682560000000000) \ - STEP( 121, UINT64_C(0x0000000000b0f016), 0.605, 0.691163451893750) \ - STEP( 122, UINT64_C(0x0000000000b31efd), 0.610, 0.699691630600000) \ - STEP( 123, UINT64_C(0x0000000000b548bf), 0.615, 0.708141306431250) \ - STEP( 124, UINT64_C(0x0000000000b76d27), 0.620, 0.716509299200000) \ - STEP( 125, UINT64_C(0x0000000000b98c00), 0.625, 0.724792480468750) \ - STEP( 126, UINT64_C(0x0000000000bba516), 0.630, 0.732987775800000) \ - STEP( 127, UINT64_C(0x0000000000bdb837), 0.635, 0.741092167006250) \ - STEP( 128, UINT64_C(0x0000000000bfc531), 0.640, 0.749102694400000) \ - STEP( 129, UINT64_C(0x0000000000c1cbd4), 0.645, 0.757016459043750) \ - STEP( 130, UINT64_C(0x0000000000c3cbf0), 0.650, 0.764830625000000) \ - STEP( 131, UINT64_C(0x0000000000c5c557), 0.655, 0.772542421581250) \ - STEP( 132, UINT64_C(0x0000000000c7b7da), 0.660, 0.780149145600000) \ - STEP( 133, UINT64_C(0x0000000000c9a34f), 0.665, 0.787648163618750) \ - STEP( 134, UINT64_C(0x0000000000cb878a), 0.670, 0.795036914200000) \ - STEP( 135, UINT64_C(0x0000000000cd6460), 0.675, 0.802312910156250) \ - STEP( 136, UINT64_C(0x0000000000cf39ab), 0.680, 0.809473740800000) \ - STEP( 137, UINT64_C(0x0000000000d10743), 0.685, 0.816517074193750) \ - STEP( 138, UINT64_C(0x0000000000d2cd01), 0.690, 0.823440659400000) \ - STEP( 139, UINT64_C(0x0000000000d48ac2), 0.695, 0.830242328731250) \ - STEP( 140, UINT64_C(0x0000000000d64063), 0.700, 0.836920000000000) \ - STEP( 141, UINT64_C(0x0000000000d7edc2), 0.705, 0.843471678768750) \ - STEP( 142, UINT64_C(0x0000000000d992bf), 0.710, 0.849895460600000) \ - STEP( 143, UINT64_C(0x0000000000db2f3c), 0.715, 0.856189533306250) \ - STEP( 144, UINT64_C(0x0000000000dcc31c), 0.720, 0.862352179200000) \ - STEP( 145, UINT64_C(0x0000000000de4e44), 0.725, 0.868381777343750) \ - STEP( 146, UINT64_C(0x0000000000dfd09a), 0.730, 0.874276805800000) \ - STEP( 147, UINT64_C(0x0000000000e14a07), 0.735, 0.880035843881250) \ - STEP( 148, UINT64_C(0x0000000000e2ba74), 0.740, 0.885657574400000) \ - STEP( 149, UINT64_C(0x0000000000e421cd), 0.745, 0.891140785918750) \ - STEP( 150, UINT64_C(0x0000000000e58000), 0.750, 0.896484375000000) \ - STEP( 151, UINT64_C(0x0000000000e6d4fb), 0.755, 0.901687348456250) \ - STEP( 152, UINT64_C(0x0000000000e820b0), 0.760, 0.906748825600000) \ - STEP( 153, UINT64_C(0x0000000000e96313), 0.765, 0.911668040493750) \ - STEP( 154, UINT64_C(0x0000000000ea9c18), 0.770, 0.916444344200000) \ - STEP( 155, UINT64_C(0x0000000000ebcbb7), 0.775, 0.921077207031250) \ - STEP( 156, UINT64_C(0x0000000000ecf1e8), 0.780, 0.925566220800000) \ - STEP( 157, UINT64_C(0x0000000000ee0ea7), 0.785, 0.929911101068750) \ - STEP( 158, UINT64_C(0x0000000000ef21f1), 0.790, 0.934111689400000) \ - STEP( 159, UINT64_C(0x0000000000f02bc6), 0.795, 0.938167955606250) \ - STEP( 160, UINT64_C(0x0000000000f12c27), 0.800, 0.942080000000000) \ - STEP( 161, UINT64_C(0x0000000000f22319), 0.805, 0.945848055643750) \ - STEP( 162, UINT64_C(0x0000000000f310a1), 0.810, 0.949472490600000) \ - STEP( 163, UINT64_C(0x0000000000f3f4c7), 0.815, 0.952953810181250) \ - STEP( 164, UINT64_C(0x0000000000f4cf98), 0.820, 0.956292659200000) \ - STEP( 165, UINT64_C(0x0000000000f5a120), 0.825, 0.959489824218750) \ - STEP( 166, UINT64_C(0x0000000000f6696e), 0.830, 0.962546235800000) \ - STEP( 167, UINT64_C(0x0000000000f72894), 0.835, 0.965462970756250) \ - STEP( 168, UINT64_C(0x0000000000f7dea8), 0.840, 0.968241254400000) \ - STEP( 169, UINT64_C(0x0000000000f88bc0), 0.845, 0.970882462793750) \ - STEP( 170, UINT64_C(0x0000000000f92ff6), 0.850, 0.973388125000000) \ - STEP( 171, UINT64_C(0x0000000000f9cb67), 0.855, 0.975759925331250) \ - STEP( 172, UINT64_C(0x0000000000fa5e30), 0.860, 0.977999705600000) \ - STEP( 173, UINT64_C(0x0000000000fae874), 0.865, 0.980109467368750) \ - STEP( 174, UINT64_C(0x0000000000fb6a57), 0.870, 0.982091374200000) \ - STEP( 175, UINT64_C(0x0000000000fbe400), 0.875, 0.983947753906250) \ - STEP( 176, UINT64_C(0x0000000000fc5598), 0.880, 0.985681100800000) \ - STEP( 177, UINT64_C(0x0000000000fcbf4e), 0.885, 0.987294077943750) \ - STEP( 178, UINT64_C(0x0000000000fd214f), 0.890, 0.988789519400000) \ - STEP( 179, UINT64_C(0x0000000000fd7bcf), 0.895, 0.990170432481250) \ - STEP( 180, UINT64_C(0x0000000000fdcf03), 0.900, 0.991440000000000) \ - STEP( 181, UINT64_C(0x0000000000fe1b23), 0.905, 0.992601582518750) \ - STEP( 182, UINT64_C(0x0000000000fe606a), 0.910, 0.993658720600000) \ - STEP( 183, UINT64_C(0x0000000000fe9f18), 0.915, 0.994615137056250) \ - STEP( 184, UINT64_C(0x0000000000fed76e), 0.920, 0.995474739200000) \ - STEP( 185, UINT64_C(0x0000000000ff09b0), 0.925, 0.996241621093750) \ - STEP( 186, UINT64_C(0x0000000000ff3627), 0.930, 0.996920065800000) \ - STEP( 187, UINT64_C(0x0000000000ff5d1d), 0.935, 0.997514547631250) \ - STEP( 188, UINT64_C(0x0000000000ff7ee0), 0.940, 0.998029734400000) \ - STEP( 189, UINT64_C(0x0000000000ff9bc3), 0.945, 0.998470489668750) \ - STEP( 190, UINT64_C(0x0000000000ffb419), 0.950, 0.998841875000000) \ - STEP( 191, UINT64_C(0x0000000000ffc83d), 0.955, 0.999149152206250) \ - STEP( 192, UINT64_C(0x0000000000ffd888), 0.960, 0.999397785600000) \ - STEP( 193, UINT64_C(0x0000000000ffe55b), 0.965, 0.999593444243750) \ - STEP( 194, UINT64_C(0x0000000000ffef17), 0.970, 0.999742004200000) \ - STEP( 195, UINT64_C(0x0000000000fff623), 0.975, 0.999849550781250) \ - STEP( 196, UINT64_C(0x0000000000fffae9), 0.980, 0.999922380800000) \ - STEP( 197, UINT64_C(0x0000000000fffdd6), 0.985, 0.999967004818750) \ - STEP( 198, UINT64_C(0x0000000000ffff5a), 0.990, 0.999990149400000) \ - STEP( 199, UINT64_C(0x0000000000ffffeb), 0.995, 0.999998759356250) \ - STEP( 200, UINT64_C(0x0000000001000000), 1.000, 1.000000000000000) \ +#define SMOOTHSTEP_VARIANT "smoother" +#define SMOOTHSTEP_NSTEPS 200 +#define SMOOTHSTEP_BFP 24 +#define SMOOTHSTEP \ + /* STEP(step, h, x, y) */ \ + STEP(1, UINT64_C(0x0000000000000014), 0.005, 0.000001240643750) \ + STEP(2, UINT64_C(0x00000000000000a5), 0.010, 0.000009850600000) \ + STEP(3, UINT64_C(0x0000000000000229), 0.015, 0.000032995181250) \ + STEP(4, UINT64_C(0x0000000000000516), 0.020, 0.000077619200000) \ + STEP(5, UINT64_C(0x00000000000009dc), 0.025, 0.000150449218750) \ + STEP(6, UINT64_C(0x00000000000010e8), 0.030, 0.000257995800000) \ + STEP(7, UINT64_C(0x0000000000001aa4), 0.035, 0.000406555756250) \ + STEP(8, UINT64_C(0x0000000000002777), 0.040, 0.000602214400000) \ + STEP(9, UINT64_C(0x00000000000037c2), 0.045, 0.000850847793750) \ + STEP(10, UINT64_C(0x0000000000004be6), 0.050, 0.001158125000000) \ + STEP(11, UINT64_C(0x000000000000643c), 0.055, 0.001529510331250) \ + STEP(12, UINT64_C(0x000000000000811f), 0.060, 0.001970265600000) \ + STEP(13, UINT64_C(0x000000000000a2e2), 0.065, 0.002485452368750) \ + STEP(14, UINT64_C(0x000000000000c9d8), 0.070, 0.003079934200000) \ + STEP(15, UINT64_C(0x000000000000f64f), 0.075, 0.003758378906250) \ + STEP(16, UINT64_C(0x0000000000012891), 0.080, 0.004525260800000) \ + STEP(17, UINT64_C(0x00000000000160e7), 0.085, 0.005384862943750) \ + STEP(18, UINT64_C(0x0000000000019f95), 0.090, 0.006341279400000) \ + STEP(19, UINT64_C(0x000000000001e4dc), 0.095, 0.007398417481250) \ + STEP(20, UINT64_C(0x00000000000230fc), 0.100, 0.008560000000000) \ + STEP(21, UINT64_C(0x0000000000028430), 0.105, 0.009829567518750) \ + STEP(22, UINT64_C(0x000000000002deb0), 0.110, 0.011210480600000) \ + STEP(23, UINT64_C(0x00000000000340b1), 0.115, 0.012705922056250) \ + STEP(24, UINT64_C(0x000000000003aa67), 0.120, 0.014318899200000) \ + STEP(25, UINT64_C(0x0000000000041c00), 0.125, 0.016052246093750) \ + STEP(26, UINT64_C(0x00000000000495a8), 0.130, 0.017908625800000) \ + STEP(27, UINT64_C(0x000000000005178b), 0.135, 0.019890532631250) \ + STEP(28, UINT64_C(0x000000000005a1cf), 0.140, 0.022000294400000) \ + STEP(29, UINT64_C(0x0000000000063498), 0.145, 0.024240074668750) \ + STEP(30, UINT64_C(0x000000000006d009), 0.150, 0.026611875000000) \ + STEP(31, UINT64_C(0x000000000007743f), 0.155, 0.029117537206250) \ + STEP(32, UINT64_C(0x0000000000082157), 0.160, 0.031758745600000) \ + STEP(33, UINT64_C(0x000000000008d76b), 0.165, 0.034537029243750) \ + STEP(34, UINT64_C(0x0000000000099691), 0.170, 0.037453764200000) \ + STEP(35, UINT64_C(0x00000000000a5edf), 0.175, 0.040510175781250) \ + STEP(36, UINT64_C(0x00000000000b3067), 0.180, 0.043707340800000) \ + STEP(37, UINT64_C(0x00000000000c0b38), 0.185, 0.047046189818750) \ + STEP(38, UINT64_C(0x00000000000cef5e), 0.190, 0.050527509400000) \ + STEP(39, UINT64_C(0x00000000000ddce6), 0.195, 0.054151944356250) \ + STEP(40, UINT64_C(0x00000000000ed3d8), 0.200, 0.057920000000000) \ + STEP(41, UINT64_C(0x00000000000fd439), 0.205, 0.061832044393750) \ + STEP(42, UINT64_C(0x000000000010de0e), 0.210, 0.065888310600000) \ + STEP(43, UINT64_C(0x000000000011f158), 0.215, 0.070088898931250) \ + STEP(44, UINT64_C(0x0000000000130e17), 0.220, 0.074433779200000) \ + STEP(45, UINT64_C(0x0000000000143448), 0.225, 0.078922792968750) \ + STEP(46, UINT64_C(0x00000000001563e7), 0.230, 0.083555655800000) \ + STEP(47, UINT64_C(0x0000000000169cec), 0.235, 0.088331959506250) \ + STEP(48, UINT64_C(0x000000000017df4f), 0.240, 0.093251174400000) \ + STEP(49, UINT64_C(0x0000000000192b04), 0.245, 0.098312651543750) \ + STEP(50, UINT64_C(0x00000000001a8000), 0.250, 0.103515625000000) \ + STEP(51, UINT64_C(0x00000000001bde32), 0.255, 0.108859214081250) \ + STEP(52, UINT64_C(0x00000000001d458b), 0.260, 0.114342425600000) \ + STEP(53, UINT64_C(0x00000000001eb5f8), 0.265, 0.119964156118750) \ + STEP(54, UINT64_C(0x0000000000202f65), 0.270, 0.125723194200000) \ + STEP(55, UINT64_C(0x000000000021b1bb), 0.275, 0.131618222656250) \ + STEP(56, UINT64_C(0x0000000000233ce3), 0.280, 0.137647820800000) \ + STEP(57, UINT64_C(0x000000000024d0c3), 0.285, 0.143810466693750) \ + STEP(58, UINT64_C(0x0000000000266d40), 0.290, 0.150104539400000) \ + STEP(59, UINT64_C(0x000000000028123d), 0.295, 0.156528321231250) \ + STEP(60, UINT64_C(0x000000000029bf9c), 0.300, 0.163080000000000) \ + STEP(61, UINT64_C(0x00000000002b753d), 0.305, 0.169757671268750) \ + STEP(62, UINT64_C(0x00000000002d32fe), 0.310, 0.176559340600000) \ + STEP(63, UINT64_C(0x00000000002ef8bc), 0.315, 0.183482925806250) \ + STEP(64, UINT64_C(0x000000000030c654), 0.320, 0.190526259200000) \ + STEP(65, UINT64_C(0x0000000000329b9f), 0.325, 0.197687089843750) \ + STEP(66, UINT64_C(0x0000000000347875), 0.330, 0.204963085800000) \ + STEP(67, UINT64_C(0x0000000000365cb0), 0.335, 0.212351836381250) \ + STEP(68, UINT64_C(0x0000000000384825), 0.340, 0.219850854400000) \ + STEP(69, UINT64_C(0x00000000003a3aa8), 0.345, 0.227457578418750) \ + STEP(70, UINT64_C(0x00000000003c340f), 0.350, 0.235169375000000) \ + STEP(71, UINT64_C(0x00000000003e342b), 0.355, 0.242983540956250) \ + STEP(72, UINT64_C(0x0000000000403ace), 0.360, 0.250897305600000) \ + STEP(73, UINT64_C(0x00000000004247c8), 0.365, 0.258907832993750) \ + STEP(74, UINT64_C(0x0000000000445ae9), 0.370, 0.267012224200000) \ + STEP(75, UINT64_C(0x0000000000467400), 0.375, 0.275207519531250) \ + STEP(76, UINT64_C(0x00000000004892d8), 0.380, 0.283490700800000) \ + STEP(77, UINT64_C(0x00000000004ab740), 0.385, 0.291858693568750) \ + STEP(78, UINT64_C(0x00000000004ce102), 0.390, 0.300308369400000) \ + STEP(79, UINT64_C(0x00000000004f0fe9), 0.395, 0.308836548106250) \ + STEP(80, UINT64_C(0x00000000005143bf), 0.400, 0.317440000000000) \ + STEP(81, UINT64_C(0x0000000000537c4d), 0.405, 0.326115448143750) \ + STEP(82, UINT64_C(0x000000000055b95b), 0.410, 0.334859570600000) \ + STEP(83, UINT64_C(0x000000000057fab1), 0.415, 0.343669002681250) \ + STEP(84, UINT64_C(0x00000000005a4015), 0.420, 0.352540339200000) \ + STEP(85, UINT64_C(0x00000000005c894e), 0.425, 0.361470136718750) \ + STEP(86, UINT64_C(0x00000000005ed622), 0.430, 0.370454915800000) \ + STEP(87, UINT64_C(0x0000000000612655), 0.435, 0.379491163256250) \ + STEP(88, UINT64_C(0x00000000006379ac), 0.440, 0.388575334400000) \ + STEP(89, UINT64_C(0x000000000065cfeb), 0.445, 0.397703855293750) \ + STEP(90, UINT64_C(0x00000000006828d6), 0.450, 0.406873125000000) \ + STEP(91, UINT64_C(0x00000000006a842f), 0.455, 0.416079517831250) \ + STEP(92, UINT64_C(0x00000000006ce1bb), 0.460, 0.425319385600000) \ + STEP(93, UINT64_C(0x00000000006f413a), 0.465, 0.434589059868750) \ + STEP(94, UINT64_C(0x000000000071a270), 0.470, 0.443884854200000) \ + STEP(95, UINT64_C(0x000000000074051d), 0.475, 0.453203066406250) \ + STEP(96, UINT64_C(0x0000000000766905), 0.480, 0.462539980800000) \ + STEP(97, UINT64_C(0x000000000078cde7), 0.485, 0.471891870443750) \ + STEP(98, UINT64_C(0x00000000007b3387), 0.490, 0.481254999400000) \ + STEP(99, UINT64_C(0x00000000007d99a4), 0.495, 0.490625624981250) \ + STEP(100, UINT64_C(0x0000000000800000), 0.500, 0.500000000000000) \ + STEP(101, UINT64_C(0x000000000082665b), 0.505, 0.509374375018750) \ + STEP(102, UINT64_C(0x000000000084cc78), 0.510, 0.518745000600000) \ + STEP(103, UINT64_C(0x0000000000873218), 0.515, 0.528108129556250) \ + STEP(104, UINT64_C(0x00000000008996fa), 0.520, 0.537460019200000) \ + STEP(105, UINT64_C(0x00000000008bfae2), 0.525, 0.546796933593750) \ + STEP(106, UINT64_C(0x00000000008e5d8f), 0.530, 0.556115145800000) \ + STEP(107, UINT64_C(0x000000000090bec5), 0.535, 0.565410940131250) \ + STEP(108, UINT64_C(0x0000000000931e44), 0.540, 0.574680614400000) \ + STEP(109, UINT64_C(0x0000000000957bd0), 0.545, 0.583920482168750) \ + STEP(110, UINT64_C(0x000000000097d729), 0.550, 0.593126875000000) \ + STEP(111, UINT64_C(0x00000000009a3014), 0.555, 0.602296144706250) \ + STEP(112, UINT64_C(0x00000000009c8653), 0.560, 0.611424665600000) \ + STEP(113, UINT64_C(0x00000000009ed9aa), 0.565, 0.620508836743750) \ + STEP(114, UINT64_C(0x0000000000a129dd), 0.570, 0.629545084200000) \ + STEP(115, UINT64_C(0x0000000000a376b1), 0.575, 0.638529863281250) \ + STEP(116, UINT64_C(0x0000000000a5bfea), 0.580, 0.647459660800000) \ + STEP(117, UINT64_C(0x0000000000a8054e), 0.585, 0.656330997318750) \ + STEP(118, UINT64_C(0x0000000000aa46a4), 0.590, 0.665140429400000) \ + STEP(119, UINT64_C(0x0000000000ac83b2), 0.595, 0.673884551856250) \ + STEP(120, UINT64_C(0x0000000000aebc40), 0.600, 0.682560000000000) \ + STEP(121, UINT64_C(0x0000000000b0f016), 0.605, 0.691163451893750) \ + STEP(122, UINT64_C(0x0000000000b31efd), 0.610, 0.699691630600000) \ + STEP(123, UINT64_C(0x0000000000b548bf), 0.615, 0.708141306431250) \ + STEP(124, UINT64_C(0x0000000000b76d27), 0.620, 0.716509299200000) \ + STEP(125, UINT64_C(0x0000000000b98c00), 0.625, 0.724792480468750) \ + STEP(126, UINT64_C(0x0000000000bba516), 0.630, 0.732987775800000) \ + STEP(127, UINT64_C(0x0000000000bdb837), 0.635, 0.741092167006250) \ + STEP(128, UINT64_C(0x0000000000bfc531), 0.640, 0.749102694400000) \ + STEP(129, UINT64_C(0x0000000000c1cbd4), 0.645, 0.757016459043750) \ + STEP(130, UINT64_C(0x0000000000c3cbf0), 0.650, 0.764830625000000) \ + STEP(131, UINT64_C(0x0000000000c5c557), 0.655, 0.772542421581250) \ + STEP(132, UINT64_C(0x0000000000c7b7da), 0.660, 0.780149145600000) \ + STEP(133, UINT64_C(0x0000000000c9a34f), 0.665, 0.787648163618750) \ + STEP(134, UINT64_C(0x0000000000cb878a), 0.670, 0.795036914200000) \ + STEP(135, UINT64_C(0x0000000000cd6460), 0.675, 0.802312910156250) \ + STEP(136, UINT64_C(0x0000000000cf39ab), 0.680, 0.809473740800000) \ + STEP(137, UINT64_C(0x0000000000d10743), 0.685, 0.816517074193750) \ + STEP(138, UINT64_C(0x0000000000d2cd01), 0.690, 0.823440659400000) \ + STEP(139, UINT64_C(0x0000000000d48ac2), 0.695, 0.830242328731250) \ + STEP(140, UINT64_C(0x0000000000d64063), 0.700, 0.836920000000000) \ + STEP(141, UINT64_C(0x0000000000d7edc2), 0.705, 0.843471678768750) \ + STEP(142, UINT64_C(0x0000000000d992bf), 0.710, 0.849895460600000) \ + STEP(143, UINT64_C(0x0000000000db2f3c), 0.715, 0.856189533306250) \ + STEP(144, UINT64_C(0x0000000000dcc31c), 0.720, 0.862352179200000) \ + STEP(145, UINT64_C(0x0000000000de4e44), 0.725, 0.868381777343750) \ + STEP(146, UINT64_C(0x0000000000dfd09a), 0.730, 0.874276805800000) \ + STEP(147, UINT64_C(0x0000000000e14a07), 0.735, 0.880035843881250) \ + STEP(148, UINT64_C(0x0000000000e2ba74), 0.740, 0.885657574400000) \ + STEP(149, UINT64_C(0x0000000000e421cd), 0.745, 0.891140785918750) \ + STEP(150, UINT64_C(0x0000000000e58000), 0.750, 0.896484375000000) \ + STEP(151, UINT64_C(0x0000000000e6d4fb), 0.755, 0.901687348456250) \ + STEP(152, UINT64_C(0x0000000000e820b0), 0.760, 0.906748825600000) \ + STEP(153, UINT64_C(0x0000000000e96313), 0.765, 0.911668040493750) \ + STEP(154, UINT64_C(0x0000000000ea9c18), 0.770, 0.916444344200000) \ + STEP(155, UINT64_C(0x0000000000ebcbb7), 0.775, 0.921077207031250) \ + STEP(156, UINT64_C(0x0000000000ecf1e8), 0.780, 0.925566220800000) \ + STEP(157, UINT64_C(0x0000000000ee0ea7), 0.785, 0.929911101068750) \ + STEP(158, UINT64_C(0x0000000000ef21f1), 0.790, 0.934111689400000) \ + STEP(159, UINT64_C(0x0000000000f02bc6), 0.795, 0.938167955606250) \ + STEP(160, UINT64_C(0x0000000000f12c27), 0.800, 0.942080000000000) \ + STEP(161, UINT64_C(0x0000000000f22319), 0.805, 0.945848055643750) \ + STEP(162, UINT64_C(0x0000000000f310a1), 0.810, 0.949472490600000) \ + STEP(163, UINT64_C(0x0000000000f3f4c7), 0.815, 0.952953810181250) \ + STEP(164, UINT64_C(0x0000000000f4cf98), 0.820, 0.956292659200000) \ + STEP(165, UINT64_C(0x0000000000f5a120), 0.825, 0.959489824218750) \ + STEP(166, UINT64_C(0x0000000000f6696e), 0.830, 0.962546235800000) \ + STEP(167, UINT64_C(0x0000000000f72894), 0.835, 0.965462970756250) \ + STEP(168, UINT64_C(0x0000000000f7dea8), 0.840, 0.968241254400000) \ + STEP(169, UINT64_C(0x0000000000f88bc0), 0.845, 0.970882462793750) \ + STEP(170, UINT64_C(0x0000000000f92ff6), 0.850, 0.973388125000000) \ + STEP(171, UINT64_C(0x0000000000f9cb67), 0.855, 0.975759925331250) \ + STEP(172, UINT64_C(0x0000000000fa5e30), 0.860, 0.977999705600000) \ + STEP(173, UINT64_C(0x0000000000fae874), 0.865, 0.980109467368750) \ + STEP(174, UINT64_C(0x0000000000fb6a57), 0.870, 0.982091374200000) \ + STEP(175, UINT64_C(0x0000000000fbe400), 0.875, 0.983947753906250) \ + STEP(176, UINT64_C(0x0000000000fc5598), 0.880, 0.985681100800000) \ + STEP(177, UINT64_C(0x0000000000fcbf4e), 0.885, 0.987294077943750) \ + STEP(178, UINT64_C(0x0000000000fd214f), 0.890, 0.988789519400000) \ + STEP(179, UINT64_C(0x0000000000fd7bcf), 0.895, 0.990170432481250) \ + STEP(180, UINT64_C(0x0000000000fdcf03), 0.900, 0.991440000000000) \ + STEP(181, UINT64_C(0x0000000000fe1b23), 0.905, 0.992601582518750) \ + STEP(182, UINT64_C(0x0000000000fe606a), 0.910, 0.993658720600000) \ + STEP(183, UINT64_C(0x0000000000fe9f18), 0.915, 0.994615137056250) \ + STEP(184, UINT64_C(0x0000000000fed76e), 0.920, 0.995474739200000) \ + STEP(185, UINT64_C(0x0000000000ff09b0), 0.925, 0.996241621093750) \ + STEP(186, UINT64_C(0x0000000000ff3627), 0.930, 0.996920065800000) \ + STEP(187, UINT64_C(0x0000000000ff5d1d), 0.935, 0.997514547631250) \ + STEP(188, UINT64_C(0x0000000000ff7ee0), 0.940, 0.998029734400000) \ + STEP(189, UINT64_C(0x0000000000ff9bc3), 0.945, 0.998470489668750) \ + STEP(190, UINT64_C(0x0000000000ffb419), 0.950, 0.998841875000000) \ + STEP(191, UINT64_C(0x0000000000ffc83d), 0.955, 0.999149152206250) \ + STEP(192, UINT64_C(0x0000000000ffd888), 0.960, 0.999397785600000) \ + STEP(193, UINT64_C(0x0000000000ffe55b), 0.965, 0.999593444243750) \ + STEP(194, UINT64_C(0x0000000000ffef17), 0.970, 0.999742004200000) \ + STEP(195, UINT64_C(0x0000000000fff623), 0.975, 0.999849550781250) \ + STEP(196, UINT64_C(0x0000000000fffae9), 0.980, 0.999922380800000) \ + STEP(197, UINT64_C(0x0000000000fffdd6), 0.985, 0.999967004818750) \ + STEP(198, UINT64_C(0x0000000000ffff5a), 0.990, 0.999990149400000) \ + STEP(199, UINT64_C(0x0000000000ffffeb), 0.995, 0.999998759356250) \ + STEP(200, UINT64_C(0x0000000001000000), 1.000, 1.000000000000000) #endif /* JEMALLOC_INTERNAL_SMOOTHSTEP_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/spin.h b/dep/jemalloc/include/jemalloc/internal/spin.h index 22804c687..4cd5e1db1 100644 --- a/dep/jemalloc/include/jemalloc/internal/spin.h +++ b/dep/jemalloc/include/jemalloc/internal/spin.h @@ -1,20 +1,23 @@ #ifndef JEMALLOC_INTERNAL_SPIN_H #define JEMALLOC_INTERNAL_SPIN_H -#define SPIN_INITIALIZER {0U} +#include "jemalloc/internal/jemalloc_preamble.h" + +#define SPIN_INITIALIZER \ + { 0U } typedef struct { unsigned iteration; } spin_t; static inline void -spin_cpu_spinwait() { -# if HAVE_CPU_SPINWAIT +spin_cpu_spinwait(void) { +#if HAVE_CPU_SPINWAIT CPU_SPINWAIT; -# else +#else volatile int x = 0; x = x; -# endif +#endif } static inline void diff --git a/dep/jemalloc/include/jemalloc/internal/stats.h b/dep/jemalloc/include/jemalloc/internal/stats.h index 727f7dcbd..1c7b23e05 100644 --- a/dep/jemalloc/include/jemalloc/internal/stats.h +++ b/dep/jemalloc/include/jemalloc/internal/stats.h @@ -1,33 +1,38 @@ #ifndef JEMALLOC_INTERNAL_STATS_H #define JEMALLOC_INTERNAL_STATS_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_types.h" +#include "jemalloc/internal/thread_event_registry.h" +#include "jemalloc/internal/tsd_types.h" + /* OPTION(opt, var_name, default, set_value_to) */ -#define STATS_PRINT_OPTIONS \ - OPTION('J', json, false, true) \ - OPTION('g', general, true, false) \ - OPTION('m', merged, config_stats, false) \ - OPTION('d', destroyed, config_stats, false) \ - OPTION('a', unmerged, config_stats, false) \ - OPTION('b', bins, true, false) \ - OPTION('l', large, true, false) \ - OPTION('x', mutex, true, false) \ - OPTION('e', extents, true, false) \ - OPTION('h', hpa, config_stats, false) +#define STATS_PRINT_OPTIONS \ + OPTION('J', json, false, true) \ + OPTION('g', general, true, false) \ + OPTION('m', merged, config_stats, false) \ + OPTION('d', destroyed, config_stats, false) \ + OPTION('a', unmerged, config_stats, false) \ + OPTION('b', bins, true, false) \ + OPTION('l', large, true, false) \ + OPTION('x', mutex, true, false) \ + OPTION('e', extents, true, false) \ + OPTION('h', hpa, config_stats, false) enum { #define OPTION(o, v, d, s) stats_print_option_num_##v, - STATS_PRINT_OPTIONS + STATS_PRINT_OPTIONS #undef OPTION - stats_print_tot_num_options + stats_print_tot_num_options }; /* Options for stats_print. */ extern bool opt_stats_print; -extern char opt_stats_print_opts[stats_print_tot_num_options+1]; +extern char opt_stats_print_opts[stats_print_tot_num_options + 1]; /* Utilities for stats_interval. */ extern int64_t opt_stats_interval; -extern char opt_stats_interval_opts[stats_print_tot_num_options+1]; +extern char opt_stats_interval_opts[stats_print_tot_num_options + 1]; #define STATS_INTERVAL_DEFAULT -1 /* @@ -39,9 +44,7 @@ extern char opt_stats_interval_opts[stats_print_tot_num_options+1]; #define STATS_INTERVAL_ACCUM_BATCH_MAX (4 << 20) /* Only accessed by thread event. */ -uint64_t stats_interval_new_event_wait(tsd_t *tsd); -uint64_t stats_interval_postponed_event_wait(tsd_t *tsd); -void stats_interval_event_handler(tsd_t *tsd, uint64_t elapsed); +extern te_base_cb_t stats_interval_te_handler; /* Implements je_malloc_stats_print. */ void stats_print(write_cb_t *write_cb, void *cbopaque, const char *opts); diff --git a/dep/jemalloc/include/jemalloc/internal/sz.h b/dep/jemalloc/include/jemalloc/internal/sz.h index 3c0fc1da3..d75a3034d 100644 --- a/dep/jemalloc/include/jemalloc/internal/sz.h +++ b/dep/jemalloc/include/jemalloc/internal/sz.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_SIZE_H #define JEMALLOC_INTERNAL_SIZE_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/bit_util.h" #include "jemalloc/internal/pages.h" #include "jemalloc/internal/sc.h" @@ -53,6 +54,11 @@ extern size_t sz_large_pad; extern void sz_boot(const sc_data_t *sc_data, bool cache_oblivious); +JEMALLOC_ALWAYS_INLINE bool +sz_large_size_classes_disabled(void) { + return opt_disable_large_size_classes; +} + JEMALLOC_ALWAYS_INLINE pszind_t sz_psz2ind(size_t psz) { assert(psz > 0); @@ -70,8 +76,9 @@ sz_psz2ind(size_t psz) { * SC_NGROUP. off_to_first_ps_rg begins from 1, instead of 0. e.g. * off_to_first_ps_rg is 1 when psz is (PAGE * SC_NGROUP + 1). */ - pszind_t off_to_first_ps_rg = (x < SC_LG_NGROUP + LG_PAGE) ? - 0 : x - (SC_LG_NGROUP + LG_PAGE); + pszind_t off_to_first_ps_rg = (x < SC_LG_NGROUP + LG_PAGE) + ? 0 + : x - (SC_LG_NGROUP + LG_PAGE); /* * Same as sc_s::lg_delta. @@ -79,8 +86,9 @@ sz_psz2ind(size_t psz) { * for each increase in offset, it's multiplied by two. * Therefore, lg_delta = LG_PAGE + (off_to_first_ps_rg - 1). */ - pszind_t lg_delta = (off_to_first_ps_rg == 0) ? - LG_PAGE : LG_PAGE + (off_to_first_ps_rg - 1); + pszind_t lg_delta = (off_to_first_ps_rg == 0) + ? LG_PAGE + : LG_PAGE + (off_to_first_ps_rg - 1); /* * Let's write psz in binary, e.g. 0011 for 0x3, 0111 for 0x7. @@ -112,13 +120,13 @@ sz_pind2sz_compute(pszind_t pind) { size_t grp = pind >> SC_LG_NGROUP; size_t mod = pind & ((ZU(1) << SC_LG_NGROUP) - 1); - size_t grp_size_mask = ~((!!grp)-1); - size_t grp_size = ((ZU(1) << (LG_PAGE + (SC_LG_NGROUP-1))) << grp) + size_t grp_size_mask = ~((!!grp) - 1); + size_t grp_size = ((ZU(1) << (LG_PAGE + (SC_LG_NGROUP - 1))) << grp) & grp_size_mask; size_t shift = (grp == 0) ? 1 : grp; - size_t lg_delta = shift + (LG_PAGE-1); - size_t mod_size = (mod+1) << lg_delta; + size_t lg_delta = shift + (LG_PAGE - 1); + size_t mod_size = (mod + 1) << lg_delta; size_t sz = grp_size + mod_size; return sz; @@ -142,17 +150,18 @@ sz_psz2u(size_t psz) { if (unlikely(psz > SC_LARGE_MAXCLASS)) { return SC_LARGE_MAXCLASS + PAGE; } - size_t x = lg_floor((psz<<1)-1); - size_t lg_delta = (x < SC_LG_NGROUP + LG_PAGE + 1) ? - LG_PAGE : x - SC_LG_NGROUP - 1; + size_t x = lg_floor((psz << 1) - 1); + size_t lg_delta = (x < SC_LG_NGROUP + LG_PAGE + 1) + ? LG_PAGE + : x - SC_LG_NGROUP - 1; size_t delta = ZU(1) << lg_delta; size_t delta_mask = delta - 1; size_t usize = (psz + delta_mask) & ~delta_mask; return usize; } -static inline szind_t -sz_size2index_compute(size_t size) { +JEMALLOC_ALWAYS_INLINE szind_t +sz_size2index_compute_inline(size_t size) { if (unlikely(size > SC_LARGE_MAXCLASS)) { return SC_NSIZES; } @@ -168,23 +177,30 @@ sz_size2index_compute(size_t size) { } #endif { - szind_t x = lg_floor((size<<1)-1); - szind_t shift = (x < SC_LG_NGROUP + LG_QUANTUM) ? 0 : - x - (SC_LG_NGROUP + LG_QUANTUM); + szind_t x = lg_floor((size << 1) - 1); + szind_t shift = (x < SC_LG_NGROUP + LG_QUANTUM) + ? 0 + : x - (SC_LG_NGROUP + LG_QUANTUM); szind_t grp = shift << SC_LG_NGROUP; szind_t lg_delta = (x < SC_LG_NGROUP + LG_QUANTUM + 1) - ? LG_QUANTUM : x - SC_LG_NGROUP - 1; + ? LG_QUANTUM + : x - SC_LG_NGROUP - 1; - size_t delta_inverse_mask = ZU(-1) << lg_delta; - szind_t mod = ((((size-1) & delta_inverse_mask) >> lg_delta)) & - ((ZU(1) << SC_LG_NGROUP) - 1); + size_t delta_inverse_mask = ZU(-1) << lg_delta; + szind_t mod = ((((size - 1) & delta_inverse_mask) >> lg_delta)) + & ((ZU(1) << SC_LG_NGROUP) - 1); szind_t index = SC_NTINY + grp + mod; return index; } } +static inline szind_t +sz_size2index_compute(size_t size) { + return sz_size2index_compute_inline(size); +} + JEMALLOC_ALWAYS_INLINE szind_t sz_size2index_lookup_impl(size_t size) { assert(size <= SC_LOOKUP_MAXCLASS); @@ -207,8 +223,8 @@ sz_size2index(size_t size) { return sz_size2index_compute(size); } -static inline size_t -sz_index2size_compute(szind_t index) { +JEMALLOC_ALWAYS_INLINE size_t +sz_index2size_compute_inline(szind_t index) { #if (SC_NTINY > 0) if (index < SC_NTINY) { return (ZU(1) << (SC_LG_TINY_MAXCLASS - SC_NTINY + 1 + index)); @@ -217,22 +233,27 @@ sz_index2size_compute(szind_t index) { { size_t reduced_index = index - SC_NTINY; size_t grp = reduced_index >> SC_LG_NGROUP; - size_t mod = reduced_index & ((ZU(1) << SC_LG_NGROUP) - - 1); + size_t mod = reduced_index & ((ZU(1) << SC_LG_NGROUP) - 1); - size_t grp_size_mask = ~((!!grp)-1); - size_t grp_size = ((ZU(1) << (LG_QUANTUM + - (SC_LG_NGROUP-1))) << grp) & grp_size_mask; + size_t grp_size_mask = ~((!!grp) - 1); + size_t grp_size = ((ZU(1) << (LG_QUANTUM + (SC_LG_NGROUP - 1))) + << grp) + & grp_size_mask; size_t shift = (grp == 0) ? 1 : grp; - size_t lg_delta = shift + (LG_QUANTUM-1); - size_t mod_size = (mod+1) << lg_delta; + size_t lg_delta = shift + (LG_QUANTUM - 1); + size_t mod_size = (mod + 1) << lg_delta; size_t usize = grp_size + mod_size; return usize; } } +static inline size_t +sz_index2size_compute(szind_t index) { + return sz_index2size_compute_inline(index); +} + JEMALLOC_ALWAYS_INLINE size_t sz_index2size_lookup_impl(szind_t index) { return sz_index2size_tab[index]; @@ -246,15 +267,61 @@ sz_index2size_lookup(szind_t index) { } JEMALLOC_ALWAYS_INLINE size_t -sz_index2size(szind_t index) { +sz_index2size_unsafe(szind_t index) { assert(index < SC_NSIZES); return sz_index2size_lookup(index); } +JEMALLOC_ALWAYS_INLINE size_t +sz_index2size(szind_t index) { + assert(!sz_large_size_classes_disabled() + || index <= sz_size2index(USIZE_GROW_SLOW_THRESHOLD)); + size_t size = sz_index2size_unsafe(index); + /* + * With large size classes disabled, the usize above + * SC_LARGE_MINCLASS should grow by PAGE. However, for sizes + * in [SC_LARGE_MINCLASS, USIZE_GROW_SLOW_THRESHOLD], the + * usize would not change because the size class gap in this + * range is just the same as PAGE. Although we use + * SC_LARGE_MINCLASS as the threshold in most places, we + * allow tcache and sec to cache up to + * USIZE_GROW_SLOW_THRESHOLD to minimize the side effect of + * not having size classes for larger sizes. Thus, we assert + * the size is no larger than USIZE_GROW_SLOW_THRESHOLD here + * instead of SC_LARGE_MINCLASS. + */ + assert(!sz_large_size_classes_disabled() + || size <= USIZE_GROW_SLOW_THRESHOLD); + return size; +} + JEMALLOC_ALWAYS_INLINE void sz_size2index_usize_fastpath(size_t size, szind_t *ind, size_t *usize) { - *ind = sz_size2index_lookup_impl(size); - *usize = sz_index2size_lookup_impl(*ind); + if (util_compile_time_const(size)) { + /* + * When inlined, the size may become known at compile + * time, which allows static computation through LTO. + */ + *ind = sz_size2index_compute_inline(size); + assert(*ind == sz_size2index_lookup_impl(size)); + *usize = sz_index2size_compute_inline(*ind); + assert(*usize == sz_index2size_lookup_impl(*ind)); + } else { + *ind = sz_size2index_lookup_impl(size); + *usize = sz_index2size_lookup_impl(*ind); + } +} + +JEMALLOC_ALWAYS_INLINE size_t +sz_s2u_compute_using_delta(size_t size) { + size_t x = lg_floor((size << 1) - 1); + size_t lg_delta = (x < SC_LG_NGROUP + LG_QUANTUM + 1) + ? LG_QUANTUM + : x - SC_LG_NGROUP - 1; + size_t delta = ZU(1) << lg_delta; + size_t delta_mask = delta - 1; + size_t usize = (size + delta_mask) & ~delta_mask; + return usize; } JEMALLOC_ALWAYS_INLINE size_t @@ -270,23 +337,28 @@ sz_s2u_compute(size_t size) { if (size <= (ZU(1) << SC_LG_TINY_MAXCLASS)) { size_t lg_tmin = SC_LG_TINY_MAXCLASS - SC_NTINY + 1; size_t lg_ceil = lg_floor(pow2_ceil_zu(size)); - return (lg_ceil < lg_tmin ? (ZU(1) << lg_tmin) : - (ZU(1) << lg_ceil)); + return (lg_ceil < lg_tmin ? (ZU(1) << lg_tmin) + : (ZU(1) << lg_ceil)); } #endif - { - size_t x = lg_floor((size<<1)-1); - size_t lg_delta = (x < SC_LG_NGROUP + LG_QUANTUM + 1) - ? LG_QUANTUM : x - SC_LG_NGROUP - 1; - size_t delta = ZU(1) << lg_delta; - size_t delta_mask = delta - 1; - size_t usize = (size + delta_mask) & ~delta_mask; + if (size <= SC_SMALL_MAXCLASS || !sz_large_size_classes_disabled()) { + return sz_s2u_compute_using_delta(size); + } else { + /* + * With sz_large_size_classes_disabled() == true, usize of a large + * allocation is calculated by ceiling size to the smallest + * multiple of PAGE to minimize the memory overhead, especially + * when using hugepages. + */ + size_t usize = PAGE_CEILING(size); + assert(usize - size < PAGE); return usize; } } JEMALLOC_ALWAYS_INLINE size_t sz_s2u_lookup(size_t size) { + assert(size < SC_LARGE_MINCLASS); size_t ret = sz_index2size_lookup(sz_size2index_lookup(size)); assert(ret == sz_s2u_compute(size)); @@ -365,6 +437,21 @@ sz_sa2u(size_t size, size_t alignment) { return usize; } +/* + * Under normal circumstances, whether or not to use a slab + * to satisfy an allocation depends solely on the allocation's + * effective size. However, this is *not* the case when an allocation + * is sampled for profiling, in which case you *must not* use a slab + * regardless of the effective size. Thus `sz_can_use_slab` is called + * on the common path, but there exist `*_explicit_slab` variants of + * several functions for handling the aforementioned case of + * sampled allocations. + */ +JEMALLOC_ALWAYS_INLINE bool +sz_can_use_slab(size_t size) { + return size <= SC_SMALL_MAXCLASS; +} + size_t sz_psz_quantize_floor(size_t size); size_t sz_psz_quantize_ceil(size_t size); diff --git a/dep/jemalloc/include/jemalloc/internal/tcache_externs.h b/dep/jemalloc/include/jemalloc/internal/tcache_externs.h index a2ab7101b..b7fdb5a46 100644 --- a/dep/jemalloc/include/jemalloc/internal/tcache_externs.h +++ b/dep/jemalloc/include/jemalloc/internal/tcache_externs.h @@ -1,28 +1,40 @@ #ifndef JEMALLOC_INTERNAL_TCACHE_EXTERNS_H #define JEMALLOC_INTERNAL_TCACHE_EXTERNS_H -extern bool opt_tcache; -extern size_t opt_tcache_max; -extern ssize_t opt_lg_tcache_nslots_mul; +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/base.h" +#include "jemalloc/internal/cache_bin.h" +#include "jemalloc/internal/sz.h" +#include "jemalloc/internal/tcache_types.h" +#include "jemalloc/internal/thread_event_registry.h" + +extern bool opt_tcache; +extern size_t opt_tcache_max; +extern ssize_t opt_lg_tcache_nslots_mul; extern unsigned opt_tcache_nslots_small_min; extern unsigned opt_tcache_nslots_small_max; extern unsigned opt_tcache_nslots_large; -extern ssize_t opt_lg_tcache_shift; -extern size_t opt_tcache_gc_incr_bytes; -extern size_t opt_tcache_gc_delay_bytes; +extern ssize_t opt_lg_tcache_shift; +extern size_t opt_tcache_gc_incr_bytes; +extern size_t opt_tcache_gc_delay_bytes; extern unsigned opt_lg_tcache_flush_small_div; extern unsigned opt_lg_tcache_flush_large_div; /* * Number of tcache bins. There are SC_NBINS small-object bins, plus 0 or more - * large-object bins. + * large-object bins. This is only used during threads initialization and + * changing it will not reflect on initialized threads as expected. Thus, + * it should not be changed on the fly. To change the number of tcache bins + * in use, refer to tcache_nbins of each tcache. */ -extern unsigned nhbins; +extern unsigned global_do_not_change_tcache_nbins; -/* Maximum cached size class. */ -extern size_t tcache_maxclass; - -extern cache_bin_info_t *tcache_bin_info; +/* + * Maximum cached size class. Same as above, this is only used during threads + * initialization and should not be changed. To change the maximum cached size + * class, refer to tcache_max of each tcache. + */ +extern size_t global_do_not_change_tcache_maxclass; /* * Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and @@ -32,44 +44,47 @@ extern cache_bin_info_t *tcache_bin_info; * touched. This allows the entire array to be allocated the first time an * explicit tcache is created without a disproportionate impact on memory usage. */ -extern tcaches_t *tcaches; +extern tcaches_t *tcaches; size_t tcache_salloc(tsdn_t *tsdn, const void *ptr); -void *tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, - cache_bin_t *tbin, szind_t binind, bool *tcache_success); +void *tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, + cache_bin_t *cache_bin, szind_t binind, bool *tcache_success); -void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin, - szind_t binind, unsigned rem); -void tcache_bin_flush_large(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin, - szind_t binind, unsigned rem); -void tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache, cache_bin_t *bin, - szind_t binind, bool is_small); -void tcache_arena_reassociate(tsdn_t *tsdn, tcache_slow_t *tcache_slow, - tcache_t *tcache, arena_t *arena); +void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, + cache_bin_t *cache_bin, szind_t binind, unsigned rem); +void tcache_bin_flush_large(tsd_t *tsd, tcache_t *tcache, + cache_bin_t *cache_bin, szind_t binind, unsigned rem); +void tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache, + cache_bin_t *cache_bin, szind_t binind, bool is_small); +bool tcache_bin_info_default_init( + const char *bin_settings_segment_cur, size_t len_left); +bool tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len); +bool tcache_bin_ncached_max_read( + tsd_t *tsd, size_t bin_size, cache_bin_sz_t *ncached_max); +void tcache_arena_reassociate( + tsdn_t *tsdn, tcache_slow_t *tcache_slow, tcache_t *tcache, arena_t *arena); tcache_t *tcache_create_explicit(tsd_t *tsd); -void tcache_cleanup(tsd_t *tsd); -void tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena); -bool tcaches_create(tsd_t *tsd, base_t *base, unsigned *r_ind); -void tcaches_flush(tsd_t *tsd, unsigned ind); -void tcaches_destroy(tsd_t *tsd, unsigned ind); -bool tcache_boot(tsdn_t *tsdn, base_t *base); -void tcache_arena_associate(tsdn_t *tsdn, tcache_slow_t *tcache_slow, - tcache_t *tcache, arena_t *arena); +bool thread_tcache_max_set(tsd_t *tsd, size_t tcache_max); +void tcache_cleanup(tsd_t *tsd); +void tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena); +bool tcaches_create(tsd_t *tsd, base_t *base, unsigned *r_ind); +void tcaches_flush(tsd_t *tsd, unsigned ind); +void tcaches_destroy(tsd_t *tsd, unsigned ind); +bool tcache_boot(tsdn_t *tsdn, base_t *base); +void tcache_arena_associate( + tsdn_t *tsdn, tcache_slow_t *tcache_slow, tcache_t *tcache, arena_t *arena); void tcache_prefork(tsdn_t *tsdn); void tcache_postfork_parent(tsdn_t *tsdn); void tcache_postfork_child(tsdn_t *tsdn); void tcache_flush(tsd_t *tsd); -bool tsd_tcache_data_init(tsd_t *tsd); bool tsd_tcache_enabled_data_init(tsd_t *tsd); +void tcache_enabled_set(tsd_t *tsd, bool enabled); + +extern void *(*JET_MUTABLE tcache_stack_alloc)(tsdn_t *tsdn, size_t size, + size_t alignment); void tcache_assert_initialized(tcache_t *tcache); -/* Only accessed by thread event. */ -uint64_t tcache_gc_new_event_wait(tsd_t *tsd); -uint64_t tcache_gc_postponed_event_wait(tsd_t *tsd); -void tcache_gc_event_handler(tsd_t *tsd, uint64_t elapsed); -uint64_t tcache_gc_dalloc_new_event_wait(tsd_t *tsd); -uint64_t tcache_gc_dalloc_postponed_event_wait(tsd_t *tsd); -void tcache_gc_dalloc_event_handler(tsd_t *tsd, uint64_t elapsed); +extern te_base_cb_t tcache_gc_te_handler; #endif /* JEMALLOC_INTERNAL_TCACHE_EXTERNS_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/tcache_inlines.h b/dep/jemalloc/include/jemalloc/internal/tcache_inlines.h index 2634f145d..5f8ed317f 100644 --- a/dep/jemalloc/include/jemalloc/internal/tcache_inlines.h +++ b/dep/jemalloc/include/jemalloc/internal/tcache_inlines.h @@ -1,11 +1,16 @@ #ifndef JEMALLOC_INTERNAL_TCACHE_INLINES_H #define JEMALLOC_INTERNAL_TCACHE_INLINES_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/arena_externs.h" #include "jemalloc/internal/bin.h" +#include "jemalloc/internal/jemalloc_internal_inlines_b.h" #include "jemalloc/internal/jemalloc_internal_types.h" +#include "jemalloc/internal/large_externs.h" #include "jemalloc/internal/san.h" #include "jemalloc/internal/sc.h" #include "jemalloc/internal/sz.h" +#include "jemalloc/internal/tcache_externs.h" #include "jemalloc/internal/util.h" static inline bool @@ -13,39 +18,79 @@ tcache_enabled_get(tsd_t *tsd) { return tsd_tcache_enabled_get(tsd); } -static inline void -tcache_enabled_set(tsd_t *tsd, bool enabled) { - bool was_enabled = tsd_tcache_enabled_get(tsd); +static inline unsigned +tcache_nbins_get(tcache_slow_t *tcache_slow) { + assert(tcache_slow != NULL); + unsigned nbins = tcache_slow->tcache_nbins; + assert(nbins <= TCACHE_NBINS_MAX); + return nbins; +} - if (!was_enabled && enabled) { - tsd_tcache_data_init(tsd); - } else if (was_enabled && !enabled) { - tcache_cleanup(tsd); +static inline size_t +tcache_max_get(tcache_slow_t *tcache_slow) { + assert(tcache_slow != NULL); + size_t tcache_max = sz_index2size(tcache_nbins_get(tcache_slow) - 1); + assert(tcache_max <= TCACHE_MAXCLASS_LIMIT); + return tcache_max; +} + +static inline void +tcache_max_set(tcache_slow_t *tcache_slow, size_t tcache_max) { + assert(tcache_slow != NULL); + assert(tcache_max <= TCACHE_MAXCLASS_LIMIT); + tcache_slow->tcache_nbins = sz_size2index(tcache_max) + 1; +} + +static inline void +tcache_bin_settings_backup( + tcache_t *tcache, cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) { + for (unsigned i = 0; i < TCACHE_NBINS_MAX; i++) { + cache_bin_info_init(&tcache_bin_info[i], + cache_bin_ncached_max_get_unsafe(&tcache->bins[i])); } - /* Commit the state last. Above calls check current state. */ - tsd_tcache_enabled_set(tsd, enabled); - tsd_slow_update(tsd); } JEMALLOC_ALWAYS_INLINE bool -tcache_small_bin_disabled(szind_t ind, cache_bin_t *bin) { - assert(ind < SC_NBINS); - bool ret = (cache_bin_info_ncached_max(&tcache_bin_info[ind]) == 0); - if (ret && bin != NULL) { - /* small size class but cache bin disabled. */ - assert(ind >= nhbins); - assert((uintptr_t)(*bin->stack_head) == - cache_bin_preceding_junk); +tcache_bin_disabled(szind_t ind, cache_bin_t *bin, tcache_slow_t *tcache_slow) { + assert(bin != NULL); + assert(ind < TCACHE_NBINS_MAX); + bool disabled = cache_bin_disabled(bin); + + /* + * If a bin's ind >= nbins or ncached_max == 0, it must be disabled. + * However, when ind < nbins, it could be either enabled + * (ncached_max > 0) or disabled (ncached_max == 0). Similarly, when + * ncached_max > 0, it could be either enabled (ind < nbins) or + * disabled (ind >= nbins). Thus, if a bin is disabled, it has either + * ind >= nbins or ncached_max == 0. If a bin is enabled, it has + * ind < nbins and ncached_max > 0. + */ + unsigned nbins = tcache_nbins_get(tcache_slow); + cache_bin_sz_t ncached_max = cache_bin_ncached_max_get_unsafe(bin); + if (ind >= nbins) { + assert(disabled); + } else { + assert(!disabled || ncached_max == 0); + } + if (ncached_max == 0) { + assert(disabled); + } else { + assert(!disabled || ind >= nbins); + } + if (disabled) { + assert(ind >= nbins || ncached_max == 0); + } else { + assert(ind < nbins && ncached_max > 0); } - return ret; + return disabled; } JEMALLOC_ALWAYS_INLINE void * -tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, - size_t size, szind_t binind, bool zero, bool slow_path) { +tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, + szind_t binind, bool zero, bool slow_path) { void *ret; - bool tcache_success; + bool tcache_success; assert(binind < SC_NBINS); cache_bin_t *bin = &tcache->bins[binind]; @@ -57,16 +102,17 @@ tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, if (unlikely(arena == NULL)) { return NULL; } - if (unlikely(tcache_small_bin_disabled(binind, bin))) { + if (unlikely(tcache_bin_disabled( + binind, bin, tcache->tcache_slow))) { /* stats and zero are handled directly by the arena. */ return arena_malloc_hard(tsd_tsdn(tsd), arena, size, - binind, zero); + binind, zero, /* slab */ true); } tcache_bin_flush_stashed(tsd, tcache, bin, binind, /* is_small */ true); - ret = tcache_alloc_small_hard(tsd_tsdn(tsd), arena, tcache, - bin, binind, &tcache_hard_success); + ret = tcache_alloc_small_hard(tsd_tsdn(tsd), arena, tcache, bin, + binind, &tcache_hard_success); if (tcache_hard_success == false) { return NULL; } @@ -88,10 +134,11 @@ JEMALLOC_ALWAYS_INLINE void * tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, szind_t binind, bool zero, bool slow_path) { void *ret; - bool tcache_success; + bool tcache_success; - assert(binind >= SC_NBINS && binind < nhbins); cache_bin_t *bin = &tcache->bins[binind]; + assert(binind >= SC_NBINS + && !tcache_bin_disabled(binind, bin, tcache->tcache_slow)); ret = cache_bin_alloc(bin, &tcache_success); assert(tcache_success == (ret != NULL)); if (unlikely(!tcache_success)) { @@ -113,21 +160,21 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, } else { if (unlikely(zero)) { size_t usize = sz_index2size(binind); - assert(usize <= tcache_maxclass); + assert(usize <= tcache_max_get(tcache->tcache_slow)); memset(ret, 0, usize); } + } - if (config_stats) { - bin->tstats.nrequests++; - } + if (config_stats) { + bin->tstats.nrequests++; } return ret; } JEMALLOC_ALWAYS_INLINE void -tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, - bool slow_path) { +tcache_dalloc_small( + tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, bool slow_path) { assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= SC_SMALL_MAXCLASS); cache_bin_t *bin = &tcache->bins[binind]; @@ -147,13 +194,13 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, } if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) { - if (unlikely(tcache_small_bin_disabled(binind, bin))) { + if (unlikely(tcache_bin_disabled( + binind, bin, tcache->tcache_slow))) { arena_dalloc_small(tsd_tsdn(tsd), ptr); return; } - cache_bin_sz_t max = cache_bin_info_ncached_max( - &tcache_bin_info[binind]); - unsigned remain = max >> opt_lg_tcache_flush_small_div; + cache_bin_sz_t max = cache_bin_ncached_max_get(bin); + unsigned remain = max >> opt_lg_tcache_flush_small_div; tcache_bin_flush_small(tsd, tcache, bin, binind, remain); bool ret = cache_bin_dalloc_easy(bin, ptr); assert(ret); @@ -161,17 +208,18 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, } JEMALLOC_ALWAYS_INLINE void -tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, - bool slow_path) { - +tcache_dalloc_large( + tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, bool slow_path) { + assert(tcache_salloc(tsd_tsdn(tsd), ptr) > SC_SMALL_MAXCLASS); assert(tcache_salloc(tsd_tsdn(tsd), ptr) - > SC_SMALL_MAXCLASS); - assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= tcache_maxclass); + <= tcache_max_get(tcache->tcache_slow)); + assert(!tcache_bin_disabled( + binind, &tcache->bins[binind], tcache->tcache_slow)); cache_bin_t *bin = &tcache->bins[binind]; if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) { - unsigned remain = cache_bin_info_ncached_max( - &tcache_bin_info[binind]) >> opt_lg_tcache_flush_large_div; + unsigned remain = cache_bin_ncached_max_get(bin) + >> opt_lg_tcache_flush_large_div; tcache_bin_flush_large(tsd, tcache, bin, binind, remain); bool ret = cache_bin_dalloc_easy(bin, ptr); assert(ret); diff --git a/dep/jemalloc/include/jemalloc/internal/tcache_structs.h b/dep/jemalloc/include/jemalloc/internal/tcache_structs.h index 176d73de9..2c000de31 100644 --- a/dep/jemalloc/include/jemalloc/internal/tcache_structs.h +++ b/dep/jemalloc/include/jemalloc/internal/tcache_structs.h @@ -1,11 +1,12 @@ #ifndef JEMALLOC_INTERNAL_TCACHE_STRUCTS_H #define JEMALLOC_INTERNAL_TCACHE_STRUCTS_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/cache_bin.h" #include "jemalloc/internal/ql.h" #include "jemalloc/internal/sc.h" +#include "jemalloc/internal/tcache_types.h" #include "jemalloc/internal/ticker.h" -#include "jemalloc/internal/tsd_types.h" /* * The tcache state is split into the slow and hot path data. Each has a @@ -29,39 +30,45 @@ struct tcache_slow_s { cache_bin_array_descriptor_t cache_bin_array_descriptor; /* The arena this tcache is associated with. */ - arena_t *arena; + arena_t *arena; + /* The number of bins activated in the tcache. */ + unsigned tcache_nbins; + /* Last time GC has been performed. */ + nstime_t last_gc_time; /* Next bin to GC. */ - szind_t next_gc_bin; - /* For small bins, fill (ncached_max >> lg_fill_div). */ - uint8_t lg_fill_div[SC_NBINS]; + szind_t next_gc_bin; + szind_t next_gc_bin_small; + szind_t next_gc_bin_large; + /* For small bins, help determine how many items to fill at a time. */ + cache_bin_fill_ctl_t bin_fill_ctl_do_not_access_directly[SC_NBINS]; /* For small bins, whether has been refilled since last GC. */ - bool bin_refilled[SC_NBINS]; + bool bin_refilled[SC_NBINS]; /* * For small bins, the number of items we can pretend to flush before * actually flushing. */ - uint8_t bin_flush_delay_items[SC_NBINS]; + uint8_t bin_flush_delay_items[SC_NBINS]; /* * The start of the allocation containing the dynamic allocation for * either the cache bins alone, or the cache bin memory as well as this * tcache_slow_t and its associated tcache_t. */ - void *dyn_alloc; + void *dyn_alloc; /* The associated bins. */ - tcache_t *tcache; + tcache_t *tcache; }; struct tcache_s { - tcache_slow_t *tcache_slow; - cache_bin_t bins[TCACHE_NBINS_MAX]; + tcache_slow_t *tcache_slow; + cache_bin_t bins[TCACHE_NBINS_MAX]; }; /* Linkage for list of available (previously used) explicit tcache IDs. */ struct tcaches_s { union { - tcache_t *tcache; - tcaches_t *next; + tcache_t *tcache; + tcaches_t *next; }; }; diff --git a/dep/jemalloc/include/jemalloc/internal/tcache_types.h b/dep/jemalloc/include/jemalloc/internal/tcache_types.h index 583677ea2..27d80d3cd 100644 --- a/dep/jemalloc/include/jemalloc/internal/tcache_types.h +++ b/dep/jemalloc/include/jemalloc/internal/tcache_types.h @@ -1,35 +1,37 @@ #ifndef JEMALLOC_INTERNAL_TCACHE_TYPES_H #define JEMALLOC_INTERNAL_TCACHE_TYPES_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/sc.h" typedef struct tcache_slow_s tcache_slow_t; -typedef struct tcache_s tcache_t; -typedef struct tcaches_s tcaches_t; - -/* - * tcache pointers close to NULL are used to encode state information that is - * used for two purposes: preventing thread caching on a per thread basis and - * cleaning up during thread shutdown. - */ -#define TCACHE_STATE_DISABLED ((tcache_t *)(uintptr_t)1) -#define TCACHE_STATE_REINCARNATED ((tcache_t *)(uintptr_t)2) -#define TCACHE_STATE_PURGATORY ((tcache_t *)(uintptr_t)3) -#define TCACHE_STATE_MAX TCACHE_STATE_PURGATORY +typedef struct tcache_s tcache_t; +typedef struct tcaches_s tcaches_t; /* Used in TSD static initializer only. Real init in tsd_tcache_data_init(). */ -#define TCACHE_ZERO_INITIALIZER {0} -#define TCACHE_SLOW_ZERO_INITIALIZER {0} +#define TCACHE_ZERO_INITIALIZER \ + { 0 } +#define TCACHE_SLOW_ZERO_INITIALIZER \ + { \ + { 0 } \ + } /* Used in TSD static initializer only. Will be initialized to opt_tcache. */ #define TCACHE_ENABLED_ZERO_INITIALIZER false /* Used for explicit tcache only. Means flushed but not destroyed. */ +/* NOLINTNEXTLINE(performance-no-int-to-ptr) */ #define TCACHES_ELM_NEED_REINIT ((tcache_t *)(uintptr_t)1) -#define TCACHE_LG_MAXCLASS_LIMIT 23 /* tcache_maxclass = 8M */ +#define TCACHE_LG_MAXCLASS_LIMIT LG_USIZE_GROW_SLOW_THRESHOLD #define TCACHE_MAXCLASS_LIMIT ((size_t)1 << TCACHE_LG_MAXCLASS_LIMIT) -#define TCACHE_NBINS_MAX (SC_NBINS + SC_NGROUP * \ - (TCACHE_LG_MAXCLASS_LIMIT - SC_LG_LARGE_MINCLASS) + 1) +#define TCACHE_NBINS_MAX \ + (SC_NBINS \ + + SC_NGROUP * (TCACHE_LG_MAXCLASS_LIMIT - SC_LG_LARGE_MINCLASS) \ + + 1) +#define TCACHE_GC_NEIGHBOR_LIMIT ((uintptr_t)1 << 21) /* 2M */ +#define TCACHE_GC_INTERVAL_NS ((uint64_t)10 * KQU(1000000)) /* 10ms */ +#define TCACHE_GC_SMALL_NBINS_MAX ((SC_NBINS > 8) ? (SC_NBINS >> 3) : 1) +#define TCACHE_GC_LARGE_NBINS_MAX 1 #endif /* JEMALLOC_INTERNAL_TCACHE_TYPES_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/test_hooks.h b/dep/jemalloc/include/jemalloc/internal/test_hooks.h index 3d530b5c5..35f3a2114 100644 --- a/dep/jemalloc/include/jemalloc/internal/test_hooks.h +++ b/dep/jemalloc/include/jemalloc/internal/test_hooks.h @@ -1,24 +1,28 @@ #ifndef JEMALLOC_INTERNAL_TEST_HOOKS_H #define JEMALLOC_INTERNAL_TEST_HOOKS_H -extern JEMALLOC_EXPORT void (*test_hooks_arena_new_hook)(); -extern JEMALLOC_EXPORT void (*test_hooks_libc_hook)(); +#include "jemalloc/internal/jemalloc_preamble.h" + +extern JEMALLOC_EXPORT void (*test_hooks_arena_new_hook)(void); +extern JEMALLOC_EXPORT void (*test_hooks_libc_hook)(void); #if defined(JEMALLOC_JET) || defined(JEMALLOC_UNIT_TEST) -# define JEMALLOC_TEST_HOOK(fn, hook) ((void)(hook != NULL && (hook(), 0)), fn) +# define JEMALLOC_TEST_HOOK(fn, hook) \ + ((void)(hook != NULL && (hook(), 0)), fn) -# define open JEMALLOC_TEST_HOOK(open, test_hooks_libc_hook) -# define read JEMALLOC_TEST_HOOK(read, test_hooks_libc_hook) -# define write JEMALLOC_TEST_HOOK(write, test_hooks_libc_hook) -# define readlink JEMALLOC_TEST_HOOK(readlink, test_hooks_libc_hook) -# define close JEMALLOC_TEST_HOOK(close, test_hooks_libc_hook) -# define creat JEMALLOC_TEST_HOOK(creat, test_hooks_libc_hook) -# define secure_getenv JEMALLOC_TEST_HOOK(secure_getenv, test_hooks_libc_hook) +# define open JEMALLOC_TEST_HOOK(open, test_hooks_libc_hook) +# define read JEMALLOC_TEST_HOOK(read, test_hooks_libc_hook) +# define write JEMALLOC_TEST_HOOK(write, test_hooks_libc_hook) +# define readlink JEMALLOC_TEST_HOOK(readlink, test_hooks_libc_hook) +# define close JEMALLOC_TEST_HOOK(close, test_hooks_libc_hook) +# define creat JEMALLOC_TEST_HOOK(creat, test_hooks_libc_hook) +# define secure_getenv \ + JEMALLOC_TEST_HOOK(secure_getenv, test_hooks_libc_hook) /* Note that this is undef'd and re-define'd in src/prof.c. */ -# define _Unwind_Backtrace JEMALLOC_TEST_HOOK(_Unwind_Backtrace, test_hooks_libc_hook) +# define _Unwind_Backtrace \ + JEMALLOC_TEST_HOOK(_Unwind_Backtrace, test_hooks_libc_hook) #else -# define JEMALLOC_TEST_HOOK(fn, hook) fn +# define JEMALLOC_TEST_HOOK(fn, hook) fn #endif - #endif /* JEMALLOC_INTERNAL_TEST_HOOKS_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/thread_event.h b/dep/jemalloc/include/jemalloc/internal/thread_event.h index 2f4e1b39c..e9e2b6cd8 100644 --- a/dep/jemalloc/include/jemalloc/internal/thread_event.h +++ b/dep/jemalloc/include/jemalloc/internal/thread_event.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_THREAD_EVENT_H #define JEMALLOC_INTERNAL_THREAD_EVENT_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/tsd.h" /* "te" is short for "thread_event" */ @@ -36,7 +37,7 @@ #define TE_INVALID_ELAPSED UINT64_MAX typedef struct te_ctx_s { - bool is_alloc; + bool is_alloc; uint64_t *current; uint64_t *last_event; uint64_t *next_event; @@ -47,36 +48,20 @@ void te_assert_invariants_debug(tsd_t *tsd); void te_event_trigger(tsd_t *tsd, te_ctx_t *ctx); void te_recompute_fast_threshold(tsd_t *tsd); void tsd_te_init(tsd_t *tsd); - -/* - * List of all events, in the following format: - * E(event, (condition), is_alloc_event) - */ -#define ITERATE_OVER_ALL_EVENTS \ - E(tcache_gc, (opt_tcache_gc_incr_bytes > 0), true) \ - E(prof_sample, (config_prof && opt_prof), true) \ - E(stats_interval, (opt_stats_interval >= 0), true) \ - E(tcache_gc_dalloc, (opt_tcache_gc_incr_bytes > 0), false) \ - E(peak_alloc, config_stats, true) \ - E(peak_dalloc, config_stats, false) - -#define E(event, condition_unused, is_alloc_event_unused) \ - C(event##_event_wait) +void te_adjust_thresholds_helper(tsd_t *tsd, te_ctx_t *ctx, uint64_t wait); /* List of all thread event counters. */ -#define ITERATE_OVER_ALL_COUNTERS \ - C(thread_allocated) \ - C(thread_allocated_last_event) \ - ITERATE_OVER_ALL_EVENTS \ - C(prof_sample_last_event) \ - C(stats_interval_last_event) +#define ITERATE_OVER_ALL_COUNTERS \ + C(thread_allocated) \ + C(thread_allocated_last_event) \ + C(prof_sample_last_event) \ + C(stats_interval_last_event) /* Getters directly wrap TSD getters. */ -#define C(counter) \ -JEMALLOC_ALWAYS_INLINE uint64_t \ -counter##_get(tsd_t *tsd) { \ - return tsd_##counter##_get(tsd); \ -} +#define C(counter) \ + JEMALLOC_ALWAYS_INLINE uint64_t counter##_get(tsd_t *tsd) { \ + return tsd_##counter##_get(tsd); \ + } ITERATE_OVER_ALL_COUNTERS #undef C @@ -88,21 +73,14 @@ ITERATE_OVER_ALL_COUNTERS * temporarily delay the event and let it be immediately triggered at the next * allocation call. */ -#define C(counter) \ -JEMALLOC_ALWAYS_INLINE void \ -counter##_set(tsd_t *tsd, uint64_t v) { \ - *tsd_##counter##p_get(tsd) = v; \ -} +#define C(counter) \ + JEMALLOC_ALWAYS_INLINE void counter##_set(tsd_t *tsd, uint64_t v) { \ + *tsd_##counter##p_get(tsd) = v; \ + } ITERATE_OVER_ALL_COUNTERS #undef C -/* - * For generating _event_wait getter / setter functions for each individual - * event. - */ -#undef E - /* * The malloc and free fastpath getters -- use the unsafe getters since tsd may * be non-nominal, in which case the fast_threshold will be set to 0. This @@ -219,57 +197,6 @@ te_ctx_get(tsd_t *tsd, te_ctx_t *ctx, bool is_alloc) { } } -/* - * The lookahead functionality facilitates events to be able to lookahead, i.e. - * without touching the event counters, to determine whether an event would be - * triggered. The event counters are not advanced until the end of the - * allocation / deallocation calls, so the lookahead can be useful if some - * preparation work for some event must be done early in the allocation / - * deallocation calls. - * - * Currently only the profiling sampling event needs the lookahead - * functionality, so we don't yet define general purpose lookahead functions. - * - * Surplus is a terminology referring to the amount of bytes beyond what's - * needed for triggering an event, which can be a useful quantity to have in - * general when lookahead is being called. - */ - -JEMALLOC_ALWAYS_INLINE bool -te_prof_sample_event_lookahead_surplus(tsd_t *tsd, size_t usize, - size_t *surplus) { - if (surplus != NULL) { - /* - * This is a dead store: the surplus will be overwritten before - * any read. The initialization suppresses compiler warnings. - * Meanwhile, using SIZE_MAX to initialize is good for - * debugging purpose, because a valid surplus value is strictly - * less than usize, which is at most SIZE_MAX. - */ - *surplus = SIZE_MAX; - } - if (unlikely(!tsd_nominal(tsd) || tsd_reentrancy_level_get(tsd) > 0)) { - return false; - } - /* The subtraction is intentionally susceptible to underflow. */ - uint64_t accumbytes = tsd_thread_allocated_get(tsd) + usize - - tsd_thread_allocated_last_event_get(tsd); - uint64_t sample_wait = tsd_prof_sample_event_wait_get(tsd); - if (accumbytes < sample_wait) { - return false; - } - assert(accumbytes - sample_wait < (uint64_t)usize); - if (surplus != NULL) { - *surplus = (size_t)(accumbytes - sample_wait); - } - return true; -} - -JEMALLOC_ALWAYS_INLINE bool -te_prof_sample_event_lookahead(tsd_t *tsd, size_t usize) { - return te_prof_sample_event_lookahead_surplus(tsd, usize, NULL); -} - JEMALLOC_ALWAYS_INLINE void te_event_advance(tsd_t *tsd, size_t usize, bool is_alloc) { te_assert_invariants(tsd); diff --git a/dep/jemalloc/include/jemalloc/internal/thread_event_registry.h b/dep/jemalloc/include/jemalloc/internal/thread_event_registry.h new file mode 100644 index 000000000..bfb140aaf --- /dev/null +++ b/dep/jemalloc/include/jemalloc/internal/thread_event_registry.h @@ -0,0 +1,107 @@ +#ifndef JEMALLOC_INTERNAL_THREAD_EVENT_REGISTRY_H +#define JEMALLOC_INTERNAL_THREAD_EVENT_REGISTRY_H + +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/tsd_types.h" + +#define TE_MAX_USER_EVENTS 4 + +/* "te" is short for "thread_event" */ +enum te_alloc_e { +#ifdef JEMALLOC_PROF + te_alloc_prof_sample, +#endif + te_alloc_stats_interval, + te_alloc_tcache_gc, +#ifdef JEMALLOC_STATS + te_alloc_peak, +#endif + te_alloc_user0, + te_alloc_user1, + te_alloc_user2, + te_alloc_user3, + te_alloc_last = te_alloc_user3, + te_alloc_count = te_alloc_last + 1 +}; +typedef enum te_alloc_e te_alloc_t; + +enum te_dalloc_e { + te_dalloc_tcache_gc, +#ifdef JEMALLOC_STATS + te_dalloc_peak, +#endif + te_dalloc_user0, + te_dalloc_user1, + te_dalloc_user2, + te_dalloc_user3, + te_dalloc_last = te_dalloc_user3, + te_dalloc_count = te_dalloc_last + 1 +}; +typedef enum te_dalloc_e te_dalloc_t; + +/* These will live in tsd */ +typedef struct te_data_s te_data_t; +struct te_data_s { + uint64_t alloc_wait[te_alloc_count]; + uint64_t dalloc_wait[te_dalloc_count]; +}; +#define TE_DATA_INITIALIZER \ + { \ + {0}, { \ + 0 \ + } \ + } + +/* + * Check if user event is installed, installed and enabled, or not + * installed. + * + */ +enum te_enabled_e { te_enabled_not_installed, te_enabled_yes, te_enabled_no }; +typedef enum te_enabled_e te_enabled_t; + +typedef struct te_base_cb_s te_base_cb_t; +struct te_base_cb_s { + te_enabled_t (*enabled)(void); + uint64_t (*new_event_wait)(tsd_t *tsd); + uint64_t (*postponed_event_wait)(tsd_t *tsd); + void (*event_handler)(tsd_t *tsd); +}; + +extern te_base_cb_t *te_alloc_handlers[te_alloc_count]; +extern te_base_cb_t *te_dalloc_handlers[te_dalloc_count]; + +bool experimental_thread_events_boot(void); + +/* + * User callback for thread events + * + * is_alloc - true if event is allocation, false if event is free + * tallocated - number of bytes allocated on current thread so far + * tdallocated - number of bytes allocated on current thread so far + */ +typedef void (*user_event_cb_t)( + bool is_alloc, uint64_t tallocated, uint64_t tdallocated); + +typedef struct user_hook_object_s user_hook_object_t; +struct user_hook_object_s { + user_event_cb_t callback; + uint64_t interval; + bool is_alloc_only; +}; + +/* + * register user callback + * + * return zero if event was registered + * + * if interval is zero or callback is NULL, or + * no more slots are available event will not be registered + * and non-zero value will be returned + * + */ +int te_register_user_handler(tsdn_t *tsdn, user_hook_object_t *te_uobj); + +te_enabled_t te_user_event_enabled(size_t ue_idx, bool is_alloc); + +#endif /* JEMALLOC_INTERNAL_THREAD_EVENT_REGISTRY_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/ticker.h b/dep/jemalloc/include/jemalloc/internal/ticker.h index 6b51ddec4..a1eec628e 100644 --- a/dep/jemalloc/include/jemalloc/internal/ticker.h +++ b/dep/jemalloc/include/jemalloc/internal/ticker.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_TICKER_H #define JEMALLOC_INTERNAL_TICKER_H +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/prng.h" #include "jemalloc/internal/util.h" @@ -52,28 +53,32 @@ ticker_read(const ticker_t *ticker) { * worth the hassle, but this is on the fast path of both malloc and free (via * tcache_event). */ -#if defined(__GNUC__) && !defined(__clang__) \ +#if defined(__GNUC__) && !defined(__clang__) \ && (defined(__x86_64__) || defined(__i386__)) JEMALLOC_NOINLINE #endif static bool -ticker_fixup(ticker_t *ticker) { +ticker_fixup(ticker_t *ticker, bool delay_trigger) { + if (delay_trigger) { + ticker->tick = 0; + return false; + } ticker->tick = ticker->nticks; return true; } static inline bool -ticker_ticks(ticker_t *ticker, int32_t nticks) { +ticker_ticks(ticker_t *ticker, int32_t nticks, bool delay_trigger) { ticker->tick -= nticks; if (unlikely(ticker->tick < 0)) { - return ticker_fixup(ticker); + return ticker_fixup(ticker, delay_trigger); } return false; } static inline bool -ticker_tick(ticker_t *ticker) { - return ticker_ticks(ticker, 1); +ticker_tick(ticker_t *ticker, bool delay_trigger) { + return ticker_ticks(ticker, 1, delay_trigger); } /* @@ -124,7 +129,8 @@ struct ticker_geom_s { * the behavior over long periods of time rather than the exact timing of the * initial ticks. */ -#define TICKER_GEOM_INIT(nticks) {nticks, nticks} +#define TICKER_GEOM_INIT(nticks) \ + { nticks, nticks } static inline void ticker_geom_init(ticker_geom_t *ticker, int32_t nticks) { @@ -145,31 +151,39 @@ ticker_geom_read(const ticker_geom_t *ticker) { } /* Same deal as above. */ -#if defined(__GNUC__) && !defined(__clang__) \ +#if defined(__GNUC__) && !defined(__clang__) \ && (defined(__x86_64__) || defined(__i386__)) JEMALLOC_NOINLINE #endif static bool -ticker_geom_fixup(ticker_geom_t *ticker, uint64_t *prng_state) { +ticker_geom_fixup( + ticker_geom_t *ticker, uint64_t *prng_state, bool delay_trigger) { + if (delay_trigger) { + ticker->tick = 0; + return false; + } + uint64_t idx = prng_lg_range_u64(prng_state, TICKER_GEOM_NBITS); - ticker->tick = (uint32_t)( - (uint64_t)ticker->nticks * (uint64_t)ticker_geom_table[idx] - / (uint64_t)TICKER_GEOM_MUL); + ticker->tick = (uint32_t)((uint64_t)ticker->nticks + * (uint64_t)ticker_geom_table[idx] / (uint64_t)TICKER_GEOM_MUL); + return true; } static inline bool -ticker_geom_ticks(ticker_geom_t *ticker, uint64_t *prng_state, int32_t nticks) { +ticker_geom_ticks(ticker_geom_t *ticker, uint64_t *prng_state, int32_t nticks, + bool delay_trigger) { ticker->tick -= nticks; if (unlikely(ticker->tick < 0)) { - return ticker_geom_fixup(ticker, prng_state); + return ticker_geom_fixup(ticker, prng_state, delay_trigger); } return false; } static inline bool -ticker_geom_tick(ticker_geom_t *ticker, uint64_t *prng_state) { - return ticker_geom_ticks(ticker, prng_state, 1); +ticker_geom_tick( + ticker_geom_t *ticker, uint64_t *prng_state, bool delay_trigger) { + return ticker_geom_ticks(ticker, prng_state, 1, delay_trigger); } #endif /* JEMALLOC_INTERNAL_TICKER_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/tsd.h b/dep/jemalloc/include/jemalloc/internal/tsd.h index 66d688227..84101c65a 100644 --- a/dep/jemalloc/include/jemalloc/internal/tsd.h +++ b/dep/jemalloc/include/jemalloc/internal/tsd.h @@ -1,313 +1,20 @@ #ifndef JEMALLOC_INTERNAL_TSD_H #define JEMALLOC_INTERNAL_TSD_H -#include "jemalloc/internal/activity_callback.h" -#include "jemalloc/internal/arena_types.h" -#include "jemalloc/internal/assert.h" -#include "jemalloc/internal/bin_types.h" -#include "jemalloc/internal/jemalloc_internal_externs.h" -#include "jemalloc/internal/peak.h" -#include "jemalloc/internal/prof_types.h" -#include "jemalloc/internal/ql.h" -#include "jemalloc/internal/rtree_tsd.h" -#include "jemalloc/internal/tcache_types.h" -#include "jemalloc/internal/tcache_structs.h" -#include "jemalloc/internal/util.h" -#include "jemalloc/internal/witness.h" - -/* - * Thread-Specific-Data layout - * - * At least some thread-local data gets touched on the fast-path of almost all - * malloc operations. But much of it is only necessary down slow-paths, or - * testing. We want to colocate the fast-path data so that it can live on the - * same cacheline if possible. So we define three tiers of hotness: - * TSD_DATA_FAST: Touched on the alloc/dalloc fast paths. - * TSD_DATA_SLOW: Touched down slow paths. "Slow" here is sort of general; - * there are "semi-slow" paths like "not a sized deallocation, but can still - * live in the tcache". We'll want to keep these closer to the fast-path - * data. - * TSD_DATA_SLOWER: Only touched in test or debug modes, or not touched at all. - * - * An additional concern is that the larger tcache bins won't be used (we have a - * bin per size class, but by default only cache relatively small objects). So - * the earlier bins are in the TSD_DATA_FAST tier, but the later ones are in the - * TSD_DATA_SLOWER tier. - * - * As a result of all this, we put the slow data first, then the fast data, then - * the slower data, while keeping the tcache as the last element of the fast - * data (so that the fast -> slower transition happens midway through the - * tcache). While we don't yet play alignment tricks to guarantee it, this - * increases our odds of getting some cache/page locality on fast paths. - */ - -#ifdef JEMALLOC_JET -typedef void (*test_callback_t)(int *); -# define MALLOC_TSD_TEST_DATA_INIT 0x72b65c10 -# define MALLOC_TEST_TSD \ - O(test_data, int, int) \ - O(test_callback, test_callback_t, int) -# define MALLOC_TEST_TSD_INITIALIZER , MALLOC_TSD_TEST_DATA_INIT, NULL -#else -# define MALLOC_TEST_TSD -# define MALLOC_TEST_TSD_INITIALIZER -#endif - -typedef ql_elm(tsd_t) tsd_link_t; - -/* O(name, type, nullable type) */ -#define TSD_DATA_SLOW \ - O(tcache_enabled, bool, bool) \ - O(reentrancy_level, int8_t, int8_t) \ - O(thread_allocated_last_event, uint64_t, uint64_t) \ - O(thread_allocated_next_event, uint64_t, uint64_t) \ - O(thread_deallocated_last_event, uint64_t, uint64_t) \ - O(thread_deallocated_next_event, uint64_t, uint64_t) \ - O(tcache_gc_event_wait, uint64_t, uint64_t) \ - O(tcache_gc_dalloc_event_wait, uint64_t, uint64_t) \ - O(prof_sample_event_wait, uint64_t, uint64_t) \ - O(prof_sample_last_event, uint64_t, uint64_t) \ - O(stats_interval_event_wait, uint64_t, uint64_t) \ - O(stats_interval_last_event, uint64_t, uint64_t) \ - O(peak_alloc_event_wait, uint64_t, uint64_t) \ - O(peak_dalloc_event_wait, uint64_t, uint64_t) \ - O(prof_tdata, prof_tdata_t *, prof_tdata_t *) \ - O(prng_state, uint64_t, uint64_t) \ - O(san_extents_until_guard_small, uint64_t, uint64_t) \ - O(san_extents_until_guard_large, uint64_t, uint64_t) \ - O(iarena, arena_t *, arena_t *) \ - O(arena, arena_t *, arena_t *) \ - O(arena_decay_ticker, ticker_geom_t, ticker_geom_t) \ - O(sec_shard, uint8_t, uint8_t) \ - O(binshards, tsd_binshards_t, tsd_binshards_t)\ - O(tsd_link, tsd_link_t, tsd_link_t) \ - O(in_hook, bool, bool) \ - O(peak, peak_t, peak_t) \ - O(activity_callback_thunk, activity_callback_thunk_t, \ - activity_callback_thunk_t) \ - O(tcache_slow, tcache_slow_t, tcache_slow_t) \ - O(rtree_ctx, rtree_ctx_t, rtree_ctx_t) - -#define TSD_DATA_SLOW_INITIALIZER \ - /* tcache_enabled */ TCACHE_ENABLED_ZERO_INITIALIZER, \ - /* reentrancy_level */ 0, \ - /* thread_allocated_last_event */ 0, \ - /* thread_allocated_next_event */ 0, \ - /* thread_deallocated_last_event */ 0, \ - /* thread_deallocated_next_event */ 0, \ - /* tcache_gc_event_wait */ 0, \ - /* tcache_gc_dalloc_event_wait */ 0, \ - /* prof_sample_event_wait */ 0, \ - /* prof_sample_last_event */ 0, \ - /* stats_interval_event_wait */ 0, \ - /* stats_interval_last_event */ 0, \ - /* peak_alloc_event_wait */ 0, \ - /* peak_dalloc_event_wait */ 0, \ - /* prof_tdata */ NULL, \ - /* prng_state */ 0, \ - /* san_extents_until_guard_small */ 0, \ - /* san_extents_until_guard_large */ 0, \ - /* iarena */ NULL, \ - /* arena */ NULL, \ - /* arena_decay_ticker */ \ - TICKER_GEOM_INIT(ARENA_DECAY_NTICKS_PER_UPDATE), \ - /* sec_shard */ (uint8_t)-1, \ - /* binshards */ TSD_BINSHARDS_ZERO_INITIALIZER, \ - /* tsd_link */ {NULL}, \ - /* in_hook */ false, \ - /* peak */ PEAK_INITIALIZER, \ - /* activity_callback_thunk */ \ - ACTIVITY_CALLBACK_THUNK_INITIALIZER, \ - /* tcache_slow */ TCACHE_SLOW_ZERO_INITIALIZER, \ - /* rtree_ctx */ RTREE_CTX_INITIALIZER, - -/* O(name, type, nullable type) */ -#define TSD_DATA_FAST \ - O(thread_allocated, uint64_t, uint64_t) \ - O(thread_allocated_next_event_fast, uint64_t, uint64_t) \ - O(thread_deallocated, uint64_t, uint64_t) \ - O(thread_deallocated_next_event_fast, uint64_t, uint64_t) \ - O(tcache, tcache_t, tcache_t) - -#define TSD_DATA_FAST_INITIALIZER \ - /* thread_allocated */ 0, \ - /* thread_allocated_next_event_fast */ 0, \ - /* thread_deallocated */ 0, \ - /* thread_deallocated_next_event_fast */ 0, \ - /* tcache */ TCACHE_ZERO_INITIALIZER, - -/* O(name, type, nullable type) */ -#define TSD_DATA_SLOWER \ - O(witness_tsd, witness_tsd_t, witness_tsdn_t) \ - MALLOC_TEST_TSD - -#define TSD_DATA_SLOWER_INITIALIZER \ - /* witness */ WITNESS_TSD_INITIALIZER \ - /* test data */ MALLOC_TEST_TSD_INITIALIZER - - -#define TSD_INITIALIZER { \ - TSD_DATA_SLOW_INITIALIZER \ - /* state */ ATOMIC_INIT(tsd_state_uninitialized), \ - TSD_DATA_FAST_INITIALIZER \ - TSD_DATA_SLOWER_INITIALIZER \ -} - -#if defined(JEMALLOC_MALLOC_THREAD_CLEANUP) || defined(_WIN32) -void _malloc_tsd_cleanup_register(bool (*f)(void)); -#endif - -void *malloc_tsd_malloc(size_t size); -void malloc_tsd_dalloc(void *wrapper); -tsd_t *malloc_tsd_boot0(void); -void malloc_tsd_boot1(void); -void tsd_cleanup(void *arg); -tsd_t *tsd_fetch_slow(tsd_t *tsd, bool internal); -void tsd_state_set(tsd_t *tsd, uint8_t new_state); -void tsd_slow_update(tsd_t *tsd); -void tsd_prefork(tsd_t *tsd); -void tsd_postfork_parent(tsd_t *tsd); -void tsd_postfork_child(tsd_t *tsd); - -/* - * Call ..._inc when your module wants to take all threads down the slow paths, - * and ..._dec when it no longer needs to. - */ -void tsd_global_slow_inc(tsdn_t *tsdn); -void tsd_global_slow_dec(tsdn_t *tsdn); -bool tsd_global_slow(); - -enum { - /* Common case --> jnz. */ - tsd_state_nominal = 0, - /* Initialized but on slow path. */ - tsd_state_nominal_slow = 1, - /* - * Some thread has changed global state in such a way that all nominal - * threads need to recompute their fast / slow status the next time they - * get a chance. - * - * Any thread can change another thread's status *to* recompute, but - * threads are the only ones who can change their status *from* - * recompute. - */ - tsd_state_nominal_recompute = 2, - /* - * The above nominal states should be lower values. We use - * tsd_nominal_max to separate nominal states from threads in the - * process of being born / dying. - */ - tsd_state_nominal_max = 2, - - /* - * A thread might free() during its death as its only allocator action; - * in such scenarios, we need tsd, but set up in such a way that no - * cleanup is necessary. - */ - tsd_state_minimal_initialized = 3, - /* States during which we know we're in thread death. */ - tsd_state_purgatory = 4, - tsd_state_reincarnated = 5, - /* - * What it says on the tin; tsd that hasn't been initialized. Note - * that even when the tsd struct lives in TLS, when need to keep track - * of stuff like whether or not our pthread destructors have been - * scheduled, so this really truly is different than the nominal state. - */ - tsd_state_uninitialized = 6 -}; - -/* - * Some TSD accesses can only be done in a nominal state. To enforce this, we - * wrap TSD member access in a function that asserts on TSD state, and mangle - * field names to prevent touching them accidentally. - */ -#define TSD_MANGLE(n) cant_access_tsd_items_directly_use_a_getter_or_setter_##n - -#ifdef JEMALLOC_U8_ATOMICS -# define tsd_state_t atomic_u8_t -# define tsd_atomic_load atomic_load_u8 -# define tsd_atomic_store atomic_store_u8 -# define tsd_atomic_exchange atomic_exchange_u8 -#else -# define tsd_state_t atomic_u32_t -# define tsd_atomic_load atomic_load_u32 -# define tsd_atomic_store atomic_store_u32 -# define tsd_atomic_exchange atomic_exchange_u32 -#endif - -/* The actual tsd. */ -struct tsd_s { - /* - * The contents should be treated as totally opaque outside the tsd - * module. Access any thread-local state through the getters and - * setters below. - */ - -#define O(n, t, nt) \ - t TSD_MANGLE(n); - - TSD_DATA_SLOW - /* - * We manually limit the state to just a single byte. Unless the 8-bit - * atomics are unavailable (which is rare). - */ - tsd_state_t state; - TSD_DATA_FAST - TSD_DATA_SLOWER -#undef O -}; - -JEMALLOC_ALWAYS_INLINE uint8_t -tsd_state_get(tsd_t *tsd) { - /* - * This should be atomic. Unfortunately, compilers right now can't tell - * that this can be done as a memory comparison, and forces a load into - * a register that hurts fast-path performance. - */ - /* return atomic_load_u8(&tsd->state, ATOMIC_RELAXED); */ - return *(uint8_t *)&tsd->state; -} - -/* - * Wrapper around tsd_t that makes it possible to avoid implicit conversion - * between tsd_t and tsdn_t, where tsdn_t is "nullable" and has to be - * explicitly converted to tsd_t, which is non-nullable. - */ -struct tsdn_s { - tsd_t tsd; -}; -#define TSDN_NULL ((tsdn_t *)0) -JEMALLOC_ALWAYS_INLINE tsdn_t * -tsd_tsdn(tsd_t *tsd) { - return (tsdn_t *)tsd; -} - -JEMALLOC_ALWAYS_INLINE bool -tsdn_null(const tsdn_t *tsdn) { - return tsdn == NULL; -} - -JEMALLOC_ALWAYS_INLINE tsd_t * -tsdn_tsd(tsdn_t *tsdn) { - assert(!tsdn_null(tsdn)); - - return &tsdn->tsd; -} - /* * We put the platform-specific data declarations and inlines into their own * header files to avoid cluttering this file. They define tsd_boot0, * tsd_boot1, tsd_boot, tsd_booted_get, tsd_get_allocates, tsd_get, and tsd_set. */ #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP -#include "jemalloc/internal/tsd_malloc_thread_cleanup.h" +# include "jemalloc/internal/jemalloc_preamble.h" +# include "jemalloc/internal/tsd_malloc_thread_cleanup.h" #elif (defined(JEMALLOC_TLS)) -#include "jemalloc/internal/tsd_tls.h" +# include "jemalloc/internal/tsd_tls.h" #elif (defined(_WIN32)) -#include "jemalloc/internal/tsd_win.h" +# include "jemalloc/internal/tsd_win.h" #else -#include "jemalloc/internal/tsd_generic.h" +# include "jemalloc/internal/tsd_generic.h" #endif /* @@ -315,16 +22,16 @@ tsdn_tsd(tsdn_t *tsdn) { * foo. This omits some safety checks, and so can be used during tsd * initialization and cleanup. */ -#define O(n, t, nt) \ -JEMALLOC_ALWAYS_INLINE t * \ -tsd_##n##p_get_unsafe(tsd_t *tsd) { \ - return &tsd->TSD_MANGLE(n); \ -} +#define O(n, t, nt) \ + JEMALLOC_ALWAYS_INLINE t *tsd_##n##p_get_unsafe(tsd_t *tsd) { \ + return &tsd->TSD_MANGLE(n); \ + } TSD_DATA_SLOW TSD_DATA_FAST TSD_DATA_SLOWER #undef O +/* clang-format off */ /* tsd_foop_get(tsd) returns a pointer to the thread-local instance of foo. */ #define O(n, t, nt) \ JEMALLOC_ALWAYS_INLINE t * \ @@ -341,6 +48,7 @@ tsd_##n##p_get(tsd_t *tsd) { \ state == tsd_state_minimal_initialized); \ return tsd_##n##p_get_unsafe(tsd); \ } +/* clang-format on */ TSD_DATA_SLOW TSD_DATA_FAST TSD_DATA_SLOWER @@ -350,39 +58,36 @@ TSD_DATA_SLOWER * tsdn_foop_get(tsdn) returns either the thread-local instance of foo (if tsdn * isn't NULL), or NULL (if tsdn is NULL), cast to the nullable pointer type. */ -#define O(n, t, nt) \ -JEMALLOC_ALWAYS_INLINE nt * \ -tsdn_##n##p_get(tsdn_t *tsdn) { \ - if (tsdn_null(tsdn)) { \ - return NULL; \ - } \ - tsd_t *tsd = tsdn_tsd(tsdn); \ - return (nt *)tsd_##n##p_get(tsd); \ -} +#define O(n, t, nt) \ + JEMALLOC_ALWAYS_INLINE nt *tsdn_##n##p_get(tsdn_t *tsdn) { \ + if (tsdn_null(tsdn)) { \ + return NULL; \ + } \ + tsd_t *tsd = tsdn_tsd(tsdn); \ + return (nt *)tsd_##n##p_get(tsd); \ + } TSD_DATA_SLOW TSD_DATA_FAST TSD_DATA_SLOWER #undef O /* tsd_foo_get(tsd) returns the value of the thread-local instance of foo. */ -#define O(n, t, nt) \ -JEMALLOC_ALWAYS_INLINE t \ -tsd_##n##_get(tsd_t *tsd) { \ - return *tsd_##n##p_get(tsd); \ -} +#define O(n, t, nt) \ + JEMALLOC_ALWAYS_INLINE t tsd_##n##_get(tsd_t *tsd) { \ + return *tsd_##n##p_get(tsd); \ + } TSD_DATA_SLOW TSD_DATA_FAST TSD_DATA_SLOWER #undef O /* tsd_foo_set(tsd, val) updates the thread-local instance of foo to be val. */ -#define O(n, t, nt) \ -JEMALLOC_ALWAYS_INLINE void \ -tsd_##n##_set(tsd_t *tsd, t val) { \ - assert(tsd_state_get(tsd) != tsd_state_reincarnated && \ - tsd_state_get(tsd) != tsd_state_minimal_initialized); \ - *tsd_##n##p_get(tsd) = val; \ -} +#define O(n, t, nt) \ + JEMALLOC_ALWAYS_INLINE void tsd_##n##_set(tsd_t *tsd, t val) { \ + assert(tsd_state_get(tsd) != tsd_state_reincarnated \ + && tsd_state_get(tsd) != tsd_state_minimal_initialized); \ + *tsd_##n##p_get(tsd) = val; \ + } TSD_DATA_SLOW TSD_DATA_FAST TSD_DATA_SLOWER @@ -395,8 +100,8 @@ tsd_assert_fast(tsd_t *tsd) { * counters; it's not in general possible to ensure that they won't * change asynchronously from underneath us. */ - assert(!malloc_slow && tsd_tcache_enabled_get(tsd) && - tsd_reentrancy_level_get(tsd) == 0); + assert(!malloc_slow && tsd_tcache_enabled_get(tsd) + && tsd_reentrancy_level_get(tsd) == 0); } JEMALLOC_ALWAYS_INLINE bool @@ -485,8 +190,8 @@ tsdn_rtree_ctx(tsdn_t *tsdn, rtree_ctx_t *fallback) { static inline bool tsd_state_nocleanup(tsd_t *tsd) { - return tsd_state_get(tsd) == tsd_state_reincarnated || - tsd_state_get(tsd) == tsd_state_minimal_initialized; + return tsd_state_get(tsd) == tsd_state_reincarnated + || tsd_state_get(tsd) == tsd_state_minimal_initialized; } /* diff --git a/dep/jemalloc/include/jemalloc/internal/tsd_generic.h b/dep/jemalloc/include/jemalloc/internal/tsd_generic.h index a718472f3..e049766f6 100644 --- a/dep/jemalloc/include/jemalloc/internal/tsd_generic.h +++ b/dep/jemalloc/include/jemalloc/internal/tsd_generic.h @@ -1,31 +1,35 @@ #ifdef JEMALLOC_INTERNAL_TSD_GENERIC_H -#error This file should be included only once, by tsd.h. +# error This file should be included only once, by tsd.h. #endif #define JEMALLOC_INTERNAL_TSD_GENERIC_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/ql.h" +#include "jemalloc/internal/tsd_internals.h" +#include "jemalloc/internal/tsd_types.h" + typedef struct tsd_init_block_s tsd_init_block_t; struct tsd_init_block_s { ql_elm(tsd_init_block_t) link; pthread_t thread; - void *data; + void *data; }; /* Defined in tsd.c, to allow the mutex headers to have tsd dependencies. */ typedef struct tsd_init_head_s tsd_init_head_t; typedef struct { - bool initialized; + bool initialized; tsd_t val; } tsd_wrapper_t; -void *tsd_init_check_recursion(tsd_init_head_t *head, - tsd_init_block_t *block); -void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block); +void *tsd_init_check_recursion(tsd_init_head_t *head, tsd_init_block_t *block); +void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block); -extern pthread_key_t tsd_tsd; +extern pthread_key_t tsd_tsd; extern tsd_init_head_t tsd_init_head; -extern tsd_wrapper_t tsd_boot_wrapper; -extern bool tsd_booted; +extern tsd_wrapper_t tsd_boot_wrapper; +extern bool tsd_booted; /* Initialization/cleanup. */ JEMALLOC_ALWAYS_INLINE void @@ -37,8 +41,8 @@ tsd_cleanup_wrapper(void *arg) { tsd_cleanup(&wrapper->val); if (wrapper->initialized) { /* Trigger another cleanup round. */ - if (pthread_setspecific(tsd_tsd, (void *)wrapper) != 0) - { + if (pthread_setspecific(tsd_tsd, (void *)wrapper) + != 0) { malloc_write(": Error setting TSD\n"); if (opt_abort) { abort(); @@ -73,23 +77,23 @@ tsd_wrapper_get(bool init) { if (init && unlikely(wrapper == NULL)) { tsd_init_block_t block; - wrapper = (tsd_wrapper_t *) - tsd_init_check_recursion(&tsd_init_head, &block); + wrapper = (tsd_wrapper_t *)tsd_init_check_recursion( + &tsd_init_head, &block); if (wrapper) { return wrapper; } - wrapper = (tsd_wrapper_t *) - malloc_tsd_malloc(sizeof(tsd_wrapper_t)); + wrapper = (tsd_wrapper_t *)malloc_tsd_malloc( + sizeof(tsd_wrapper_t)); block.data = (void *)wrapper; if (wrapper == NULL) { malloc_write(": Error allocating TSD\n"); abort(); } else { wrapper->initialized = false; - JEMALLOC_DIAGNOSTIC_PUSH - JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS + JEMALLOC_DIAGNOSTIC_PUSH + JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS tsd_t initializer = TSD_INITIALIZER; - JEMALLOC_DIAGNOSTIC_POP + JEMALLOC_DIAGNOSTIC_POP wrapper->val = initializer; } tsd_wrapper_set(wrapper); @@ -100,11 +104,11 @@ tsd_wrapper_get(bool init) { JEMALLOC_ALWAYS_INLINE bool tsd_boot0(void) { - tsd_wrapper_t *wrapper; + tsd_wrapper_t *wrapper; tsd_init_block_t block; - wrapper = (tsd_wrapper_t *) - tsd_init_check_recursion(&tsd_init_head, &block); + wrapper = (tsd_wrapper_t *)tsd_init_check_recursion( + &tsd_init_head, &block); if (wrapper) { return false; } @@ -129,10 +133,10 @@ tsd_boot1(void) { tsd_boot_wrapper.initialized = false; tsd_cleanup(&tsd_boot_wrapper.val); wrapper->initialized = false; - JEMALLOC_DIAGNOSTIC_PUSH - JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS + JEMALLOC_DIAGNOSTIC_PUSH + JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS tsd_t initializer = TSD_INITIALIZER; - JEMALLOC_DIAGNOSTIC_POP + JEMALLOC_DIAGNOSTIC_POP wrapper->val = initializer; tsd_wrapper_set(wrapper); } diff --git a/dep/jemalloc/include/jemalloc/internal/tsd_internals.h b/dep/jemalloc/include/jemalloc/internal/tsd_internals.h new file mode 100644 index 000000000..53b58d0c9 --- /dev/null +++ b/dep/jemalloc/include/jemalloc/internal/tsd_internals.h @@ -0,0 +1,280 @@ +#ifdef JEMALLOC_INTERNAL_TSD_INTERNALS_H +#error This file should be included only once, by one of tsd_malloc_thread_cleanup.h, tsd_tls.h, tsd_generic.h, or tsd_win.h +#endif +#define JEMALLOC_INTERNAL_TSD_INTERNALS_H + +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/arena_types.h" +#include "jemalloc/internal/assert.h" +#include "jemalloc/internal/bin_types.h" +#include "jemalloc/internal/jemalloc_internal_externs.h" +#include "jemalloc/internal/peak.h" +#include "jemalloc/internal/prof_types.h" +#include "jemalloc/internal/ql.h" +#include "jemalloc/internal/rtree_tsd.h" +#include "jemalloc/internal/tcache_structs.h" +#include "jemalloc/internal/tcache_types.h" +#include "jemalloc/internal/thread_event_registry.h" +#include "jemalloc/internal/tsd_types.h" +#include "jemalloc/internal/util.h" +#include "jemalloc/internal/witness.h" + +/* + * Thread-Specific-Data layout + * + * At least some thread-local data gets touched on the fast-path of almost all + * malloc operations. But much of it is only necessary down slow-paths, or + * testing. We want to colocate the fast-path data so that it can live on the + * same cacheline if possible. So we define three tiers of hotness: + * TSD_DATA_FAST: Touched on the alloc/dalloc fast paths. + * TSD_DATA_SLOW: Touched down slow paths. "Slow" here is sort of general; + * there are "semi-slow" paths like "not a sized deallocation, but can still + * live in the tcache". We'll want to keep these closer to the fast-path + * data. + * TSD_DATA_SLOWER: Only touched in test or debug modes, or not touched at all. + * + * An additional concern is that the larger tcache bins won't be used (we have a + * bin per size class, but by default only cache relatively small objects). So + * the earlier bins are in the TSD_DATA_FAST tier, but the later ones are in the + * TSD_DATA_SLOWER tier. + * + * As a result of all this, we put the slow data first, then the fast data, then + * the slower data, while keeping the tcache as the last element of the fast + * data (so that the fast -> slower transition happens midway through the + * tcache). While we don't yet play alignment tricks to guarantee it, this + * increases our odds of getting some cache/page locality on fast paths. + */ + +#ifdef JEMALLOC_JET +typedef void (*test_callback_t)(int *); +# define MALLOC_TSD_TEST_DATA_INIT 0x72b65c10 +# define MALLOC_TEST_TSD \ + O(test_data, int, int) \ + O(test_callback, test_callback_t, int) +# define MALLOC_TEST_TSD_INITIALIZER , MALLOC_TSD_TEST_DATA_INIT, NULL +#else +# define MALLOC_TEST_TSD +# define MALLOC_TEST_TSD_INITIALIZER +#endif + +typedef ql_elm(tsd_t) tsd_link_t; + +/* O(name, type, nullable type) */ +#define TSD_DATA_SLOW \ + O(tcache_enabled, bool, bool) \ + O(reentrancy_level, int8_t, int8_t) \ + O(min_init_state_nfetched, uint8_t, uint8_t) \ + O(thread_allocated_last_event, uint64_t, uint64_t) \ + O(thread_allocated_next_event, uint64_t, uint64_t) \ + O(thread_deallocated_last_event, uint64_t, uint64_t) \ + O(thread_deallocated_next_event, uint64_t, uint64_t) \ + O(te_data, te_data_t, te_data_t) \ + O(prof_sample_last_event, uint64_t, uint64_t) \ + O(stats_interval_last_event, uint64_t, uint64_t) \ + O(prof_tdata, prof_tdata_t *, prof_tdata_t *) \ + O(prng_state, uint64_t, uint64_t) \ + O(san_extents_until_guard_small, uint64_t, uint64_t) \ + O(san_extents_until_guard_large, uint64_t, uint64_t) \ + O(iarena, arena_t *, arena_t *) \ + O(arena, arena_t *, arena_t *) \ + O(arena_decay_ticker, ticker_geom_t, ticker_geom_t) \ + O(sec_shard, uint8_t, uint8_t) \ + O(binshards, tsd_binshards_t, tsd_binshards_t) \ + O(tsd_link, tsd_link_t, tsd_link_t) \ + O(in_hook, bool, bool) \ + O(peak, peak_t, peak_t) \ + O(tcache_slow, tcache_slow_t, tcache_slow_t) \ + O(rtree_ctx, rtree_ctx_t, rtree_ctx_t) + +#define TSD_DATA_SLOW_INITIALIZER \ + /* tcache_enabled */ TCACHE_ENABLED_ZERO_INITIALIZER, \ + /* reentrancy_level */ 0, /* min_init_state_nfetched */ 0, \ + /* thread_allocated_last_event */ 0, \ + /* thread_allocated_next_event */ 0, \ + /* thread_deallocated_last_event */ 0, \ + /* thread_deallocated_next_event */ 0, \ + /* te_data */ TE_DATA_INITIALIZER, /* prof_sample_last_event */ 0, \ + /* stats_interval_last_event */ 0, /* prof_tdata */ NULL, \ + /* prng_state */ 0, /* san_extents_until_guard_small */ 0, \ + /* san_extents_until_guard_large */ 0, /* iarena */ NULL, \ + /* arena */ NULL, /* arena_decay_ticker */ \ + TICKER_GEOM_INIT(ARENA_DECAY_NTICKS_PER_UPDATE), \ + /* sec_shard */ (uint8_t) - 1, \ + /* binshards */ TSD_BINSHARDS_ZERO_INITIALIZER, \ + /* tsd_link */ {NULL}, /* in_hook */ false, \ + /* peak */ PEAK_INITIALIZER, \ + /* tcache_slow */ TCACHE_SLOW_ZERO_INITIALIZER, \ + /* rtree_ctx */ RTREE_CTX_INITIALIZER, + +/* O(name, type, nullable type) */ +#define TSD_DATA_FAST \ + O(thread_allocated, uint64_t, uint64_t) \ + O(thread_allocated_next_event_fast, uint64_t, uint64_t) \ + O(thread_deallocated, uint64_t, uint64_t) \ + O(thread_deallocated_next_event_fast, uint64_t, uint64_t) \ + O(tcache, tcache_t, tcache_t) + +#define TSD_DATA_FAST_INITIALIZER \ + /* thread_allocated */ 0, /* thread_allocated_next_event_fast */ 0, \ + /* thread_deallocated */ 0, \ + /* thread_deallocated_next_event_fast */ 0, \ + /* tcache */ TCACHE_ZERO_INITIALIZER, + +/* O(name, type, nullable type) */ +#define TSD_DATA_SLOWER \ + O(witness_tsd, witness_tsd_t, witness_tsdn_t) \ + MALLOC_TEST_TSD + +#define TSD_DATA_SLOWER_INITIALIZER \ + /* witness */ WITNESS_TSD_INITIALIZER \ + /* test data */ MALLOC_TEST_TSD_INITIALIZER + +#define TSD_INITIALIZER \ + { \ + TSD_DATA_SLOW_INITIALIZER \ + /* state */ ATOMIC_INIT(tsd_state_uninitialized), \ + TSD_DATA_FAST_INITIALIZER TSD_DATA_SLOWER_INITIALIZER \ + } + +#if defined(JEMALLOC_MALLOC_THREAD_CLEANUP) || defined(_WIN32) +void _malloc_tsd_cleanup_register(bool (*f)(void)); +#endif + +void *malloc_tsd_malloc(size_t size); +void malloc_tsd_dalloc(void *wrapper); +tsd_t *malloc_tsd_boot0(void); +void malloc_tsd_boot1(void); +void tsd_cleanup(void *arg); +tsd_t *tsd_fetch_slow(tsd_t *tsd, bool minimal); +void tsd_state_set(tsd_t *tsd, uint8_t new_state); +void tsd_slow_update(tsd_t *tsd); +void tsd_prefork(tsd_t *tsd); +void tsd_postfork_parent(tsd_t *tsd); +void tsd_postfork_child(tsd_t *tsd); + +/* + * Call ..._inc when your module wants to take all threads down the slow paths, + * and ..._dec when it no longer needs to. + */ +void tsd_global_slow_inc(tsdn_t *tsdn); +void tsd_global_slow_dec(tsdn_t *tsdn); +bool tsd_global_slow(void); + +#define TSD_MIN_INIT_STATE_MAX_FETCHED (128) + +enum { + /* Common case --> jnz. */ + tsd_state_nominal = 0, + /* Initialized but on slow path. */ + tsd_state_nominal_slow = 1, + /* + * Some thread has changed global state in such a way that all nominal + * threads need to recompute their fast / slow status the next time they + * get a chance. + * + * Any thread can change another thread's status *to* recompute, but + * threads are the only ones who can change their status *from* + * recompute. + */ + tsd_state_nominal_recompute = 2, + /* + * The above nominal states should be lower values. We use + * tsd_nominal_max to separate nominal states from threads in the + * process of being born / dying. + */ + tsd_state_nominal_max = 2, + + /* + * A thread might free() during its death as its only allocator action; + * in such scenarios, we need tsd, but set up in such a way that no + * cleanup is necessary. + */ + tsd_state_minimal_initialized = 3, + /* States during which we know we're in thread death. */ + tsd_state_purgatory = 4, + tsd_state_reincarnated = 5, + /* + * What it says on the tin; tsd that hasn't been initialized. Note + * that even when the tsd struct lives in TLS, when need to keep track + * of stuff like whether or not our pthread destructors have been + * scheduled, so this really truly is different than the nominal state. + */ + tsd_state_uninitialized = 6 +}; + +/* + * Some TSD accesses can only be done in a nominal state. To enforce this, we + * wrap TSD member access in a function that asserts on TSD state, and mangle + * field names to prevent touching them accidentally. + */ +#define TSD_MANGLE(n) cant_access_tsd_items_directly_use_a_getter_or_setter_##n + +#ifdef JEMALLOC_U8_ATOMICS +# define tsd_state_t atomic_u8_t +# define tsd_atomic_load atomic_load_u8 +# define tsd_atomic_store atomic_store_u8 +# define tsd_atomic_exchange atomic_exchange_u8 +#else +# define tsd_state_t atomic_u32_t +# define tsd_atomic_load atomic_load_u32 +# define tsd_atomic_store atomic_store_u32 +# define tsd_atomic_exchange atomic_exchange_u32 +#endif + +/* The actual tsd. */ +struct tsd_s { + /* + * The contents should be treated as totally opaque outside the tsd + * module. Access any thread-local state through the getters and + * setters below. + */ + +#define O(n, t, nt) t TSD_MANGLE(n); + + TSD_DATA_SLOW + /* + * We manually limit the state to just a single byte. Unless the 8-bit + * atomics are unavailable (which is rare). + */ + tsd_state_t state; + TSD_DATA_FAST + TSD_DATA_SLOWER +#undef O +}; + +JEMALLOC_ALWAYS_INLINE uint8_t +tsd_state_get(tsd_t *tsd) { + /* + * This should be atomic. Unfortunately, compilers right now can't tell + * that this can be done as a memory comparison, and forces a load into + * a register that hurts fast-path performance. + */ + /* return atomic_load_u8(&tsd->state, ATOMIC_RELAXED); */ + return *(uint8_t *)&tsd->state; +} + +/* + * Wrapper around tsd_t that makes it possible to avoid implicit conversion + * between tsd_t and tsdn_t, where tsdn_t is "nullable" and has to be + * explicitly converted to tsd_t, which is non-nullable. + */ +struct tsdn_s { + tsd_t tsd; +}; +#define TSDN_NULL ((tsdn_t *)0) +JEMALLOC_ALWAYS_INLINE tsdn_t * +tsd_tsdn(tsd_t *tsd) { + return (tsdn_t *)tsd; +} + +JEMALLOC_ALWAYS_INLINE bool +tsdn_null(const tsdn_t *tsdn) { + return tsdn == NULL; +} + +JEMALLOC_ALWAYS_INLINE tsd_t * +tsdn_tsd(tsdn_t *tsdn) { + assert(!tsdn_null(tsdn)); + + return &tsdn->tsd; +} diff --git a/dep/jemalloc/include/jemalloc/internal/tsd_malloc_thread_cleanup.h b/dep/jemalloc/include/jemalloc/internal/tsd_malloc_thread_cleanup.h index d8f3ef13c..00756df1d 100644 --- a/dep/jemalloc/include/jemalloc/internal/tsd_malloc_thread_cleanup.h +++ b/dep/jemalloc/include/jemalloc/internal/tsd_malloc_thread_cleanup.h @@ -1,8 +1,12 @@ #ifdef JEMALLOC_INTERNAL_TSD_MALLOC_THREAD_CLEANUP_H -#error This file should be included only once, by tsd.h. +# error This file should be included only once, by tsd.h. #endif #define JEMALLOC_INTERNAL_TSD_MALLOC_THREAD_CLEANUP_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/tsd_internals.h" +#include "jemalloc/internal/tsd_types.h" + #define JEMALLOC_TSD_TYPE_ATTR(type) __thread type JEMALLOC_TLS_MODEL extern JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls; diff --git a/dep/jemalloc/include/jemalloc/internal/tsd_tls.h b/dep/jemalloc/include/jemalloc/internal/tsd_tls.h index 7d6c805be..6536eb540 100644 --- a/dep/jemalloc/include/jemalloc/internal/tsd_tls.h +++ b/dep/jemalloc/include/jemalloc/internal/tsd_tls.h @@ -1,13 +1,17 @@ #ifdef JEMALLOC_INTERNAL_TSD_TLS_H -#error This file should be included only once, by tsd.h. +# error This file should be included only once, by tsd.h. #endif #define JEMALLOC_INTERNAL_TSD_TLS_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/tsd_internals.h" +#include "jemalloc/internal/tsd_types.h" + #define JEMALLOC_TSD_TYPE_ATTR(type) __thread type JEMALLOC_TLS_MODEL extern JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls; extern pthread_key_t tsd_tsd; -extern bool tsd_booted; +extern bool tsd_booted; /* Initialization/cleanup. */ JEMALLOC_ALWAYS_INLINE bool diff --git a/dep/jemalloc/include/jemalloc/internal/tsd_types.h b/dep/jemalloc/include/jemalloc/internal/tsd_types.h index a6ae37da5..464795064 100644 --- a/dep/jemalloc/include/jemalloc/internal/tsd_types.h +++ b/dep/jemalloc/include/jemalloc/internal/tsd_types.h @@ -1,9 +1,11 @@ #ifndef JEMALLOC_INTERNAL_TSD_TYPES_H #define JEMALLOC_INTERNAL_TSD_TYPES_H -#define MALLOC_TSD_CLEANUPS_MAX 4 +#define MALLOC_TSD_CLEANUPS_MAX 4 -typedef struct tsd_s tsd_t; +#include "jemalloc/internal/jemalloc_preamble.h" + +typedef struct tsd_s tsd_t; typedef struct tsdn_s tsdn_t; typedef bool (*malloc_tsd_cleanup_t)(void); diff --git a/dep/jemalloc/include/jemalloc/internal/tsd_win.h b/dep/jemalloc/include/jemalloc/internal/tsd_win.h index a91dac88e..8b22bec18 100644 --- a/dep/jemalloc/include/jemalloc/internal/tsd_win.h +++ b/dep/jemalloc/include/jemalloc/internal/tsd_win.h @@ -1,21 +1,41 @@ #ifdef JEMALLOC_INTERNAL_TSD_WIN_H -#error This file should be included only once, by tsd.h. +# error This file should be included only once, by tsd.h. #endif #define JEMALLOC_INTERNAL_TSD_WIN_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/tsd_internals.h" +#include "jemalloc/internal/tsd_types.h" + +/* val should always be the first field of tsd_wrapper_t since accessing + val is the common path and having val as the first field makes it possible + that converting a pointer to tsd_wrapper_t to a pointer to val is no more + than a type cast. */ typedef struct { - bool initialized; tsd_t val; + bool initialized; } tsd_wrapper_t; -extern DWORD tsd_tsd; +#if defined(JEMALLOC_LEGACY_WINDOWS_SUPPORT) || !defined(_MSC_VER) + +extern DWORD tsd_tsd; extern tsd_wrapper_t tsd_boot_wrapper; -extern bool tsd_booted; +extern bool tsd_booted; +# if defined(_M_ARM64EC) +# define JEMALLOC_WIN32_TLSGETVALUE2 0 +# else +# define JEMALLOC_WIN32_TLSGETVALUE2 1 +# endif +# if JEMALLOC_WIN32_TLSGETVALUE2 +typedef LPVOID(WINAPI *TGV2)(DWORD dwTlsIndex); +extern TGV2 tls_get_value2; +extern HMODULE tgv2_mod; +# endif /* Initialization/cleanup. */ JEMALLOC_ALWAYS_INLINE bool tsd_cleanup_wrapper(void) { - DWORD error = GetLastError(); + DWORD error = GetLastError(); tsd_wrapper_t *wrapper = (tsd_wrapper_t *)TlsGetValue(tsd_tsd); SetLastError(error); @@ -45,13 +65,21 @@ tsd_wrapper_set(tsd_wrapper_t *wrapper) { JEMALLOC_ALWAYS_INLINE tsd_wrapper_t * tsd_wrapper_get(bool init) { - DWORD error = GetLastError(); - tsd_wrapper_t *wrapper = (tsd_wrapper_t *) TlsGetValue(tsd_tsd); - SetLastError(error); + tsd_wrapper_t *wrapper; +# if JEMALLOC_WIN32_TLSGETVALUE2 + if (tls_get_value2 != NULL) { + wrapper = (tsd_wrapper_t *)tls_get_value2(tsd_tsd); + } else +# endif + { + DWORD error = GetLastError(); + wrapper = (tsd_wrapper_t *)TlsGetValue(tsd_tsd); + SetLastError(error); + } if (init && unlikely(wrapper == NULL)) { - wrapper = (tsd_wrapper_t *) - malloc_tsd_malloc(sizeof(tsd_wrapper_t)); + wrapper = (tsd_wrapper_t *)malloc_tsd_malloc( + sizeof(tsd_wrapper_t)); if (wrapper == NULL) { malloc_write(": Error allocating TSD\n"); abort(); @@ -74,6 +102,12 @@ tsd_boot0(void) { } _malloc_tsd_cleanup_register(&tsd_cleanup_wrapper); tsd_wrapper_set(&tsd_boot_wrapper); +# if JEMALLOC_WIN32_TLSGETVALUE2 + tgv2_mod = LoadLibraryA("api-ms-win-core-processthreads-l1-1-8.dll"); + if (tgv2_mod != NULL) { + tls_get_value2 = (TGV2)GetProcAddress(tgv2_mod, "TlsGetValue2"); + } +# endif tsd_booted = true; return false; } @@ -81,8 +115,7 @@ tsd_boot0(void) { JEMALLOC_ALWAYS_INLINE void tsd_boot1(void) { tsd_wrapper_t *wrapper; - wrapper = (tsd_wrapper_t *) - malloc_tsd_malloc(sizeof(tsd_wrapper_t)); + wrapper = (tsd_wrapper_t *)malloc_tsd_malloc(sizeof(tsd_wrapper_t)); if (wrapper == NULL) { malloc_write(": Error allocating TSD\n"); abort(); @@ -137,3 +170,69 @@ tsd_set(tsd_t *val) { } wrapper->initialized = true; } + +#else // defined(JEMALLOC_LEGACY_WINDOWS_SUPPORT) || !defined(_MSC_VER) + +# define JEMALLOC_TSD_TYPE_ATTR(type) __declspec(thread) type + +extern JEMALLOC_TSD_TYPE_ATTR(tsd_wrapper_t) tsd_wrapper_tls; +extern bool tsd_booted; + +/* Initialization/cleanup. */ +JEMALLOC_ALWAYS_INLINE bool +tsd_cleanup_wrapper(void) { + if (tsd_wrapper_tls.initialized) { + tsd_wrapper_tls.initialized = false; + tsd_cleanup(&tsd_wrapper_tls.val); + if (tsd_wrapper_tls.initialized) { + /* Trigger another cleanup round. */ + return true; + } + } + return false; +} + +/* Initialization/cleanup. */ +JEMALLOC_ALWAYS_INLINE bool +tsd_boot0(void) { + _malloc_tsd_cleanup_register(tsd_cleanup_wrapper); + tsd_booted = true; + return false; +} + +JEMALLOC_ALWAYS_INLINE void +tsd_boot1(void) { + /* Do nothing. */ +} + +JEMALLOC_ALWAYS_INLINE bool +tsd_boot(void) { + return tsd_boot0(); +} + +JEMALLOC_ALWAYS_INLINE bool +tsd_booted_get(void) { + return tsd_booted; +} + +JEMALLOC_ALWAYS_INLINE bool +tsd_get_allocates(void) { + return false; +} + +/* Get/set. */ +JEMALLOC_ALWAYS_INLINE tsd_t * +tsd_get(bool init) { + return &(tsd_wrapper_tls.val); +} + +JEMALLOC_ALWAYS_INLINE void +tsd_set(tsd_t *val) { + assert(tsd_booted); + if (likely(&(tsd_wrapper_tls.val) != val)) { + tsd_wrapper_tls.val = (*val); + } + tsd_wrapper_tls.initialized = true; +} + +#endif // defined(JEMALLOC_LEGACY_WINDOWS_SUPPORT) || !defined(_MSC_VER) diff --git a/dep/jemalloc/include/jemalloc/internal/typed_list.h b/dep/jemalloc/include/jemalloc/internal/typed_list.h index 6535055a1..78704e485 100644 --- a/dep/jemalloc/include/jemalloc/internal/typed_list.h +++ b/dep/jemalloc/include/jemalloc/internal/typed_list.h @@ -6,50 +6,49 @@ * bit easier to use; it handles ql_elm_new calls and provides type safety. */ -#define TYPED_LIST(list_type, el_type, linkage) \ -typedef struct { \ - ql_head(el_type) head; \ -} list_type##_t; \ -static inline void \ -list_type##_init(list_type##_t *list) { \ - ql_new(&list->head); \ -} \ -static inline el_type * \ -list_type##_first(const list_type##_t *list) { \ - return ql_first(&list->head); \ -} \ -static inline el_type * \ -list_type##_last(const list_type##_t *list) { \ - return ql_last(&list->head, linkage); \ -} \ -static inline void \ -list_type##_append(list_type##_t *list, el_type *item) { \ - ql_elm_new(item, linkage); \ - ql_tail_insert(&list->head, item, linkage); \ -} \ -static inline void \ -list_type##_prepend(list_type##_t *list, el_type *item) { \ - ql_elm_new(item, linkage); \ - ql_head_insert(&list->head, item, linkage); \ -} \ -static inline void \ -list_type##_replace(list_type##_t *list, el_type *to_remove, \ - el_type *to_insert) { \ - ql_elm_new(to_insert, linkage); \ - ql_after_insert(to_remove, to_insert, linkage); \ - ql_remove(&list->head, to_remove, linkage); \ -} \ -static inline void \ -list_type##_remove(list_type##_t *list, el_type *item) { \ - ql_remove(&list->head, item, linkage); \ -} \ -static inline bool \ -list_type##_empty(list_type##_t *list) { \ - return ql_empty(&list->head); \ -} \ -static inline void \ -list_type##_concat(list_type##_t *list_a, list_type##_t *list_b) { \ - ql_concat(&list_a->head, &list_b->head, linkage); \ -} +#define TYPED_LIST(list_type, el_type, linkage) \ + typedef struct { \ + ql_head(el_type) head; \ + } list_type##_t; \ + static inline void list_type##_init(list_type##_t *list) { \ + ql_new(&list->head); \ + } \ + static inline el_type *list_type##_first(const list_type##_t *list) { \ + return ql_first(&list->head); \ + } \ + static inline el_type *list_type##_last(const list_type##_t *list) { \ + return ql_last(&list->head, linkage); \ + } \ + static inline el_type *list_type##_next( \ + const list_type##_t *list, el_type *item) { \ + return ql_next(&list->head, item, linkage); \ + } \ + static inline void list_type##_append( \ + list_type##_t *list, el_type *item) { \ + ql_elm_new(item, linkage); \ + ql_tail_insert(&list->head, item, linkage); \ + } \ + static inline void list_type##_prepend( \ + list_type##_t *list, el_type *item) { \ + ql_elm_new(item, linkage); \ + ql_head_insert(&list->head, item, linkage); \ + } \ + static inline void list_type##_replace( \ + list_type##_t *list, el_type *to_remove, el_type *to_insert) { \ + ql_elm_new(to_insert, linkage); \ + ql_after_insert(to_remove, to_insert, linkage); \ + ql_remove(&list->head, to_remove, linkage); \ + } \ + static inline void list_type##_remove( \ + list_type##_t *list, el_type *item) { \ + ql_remove(&list->head, item, linkage); \ + } \ + static inline bool list_type##_empty(list_type##_t *list) { \ + return ql_empty(&list->head); \ + } \ + static inline void list_type##_concat( \ + list_type##_t *list_a, list_type##_t *list_b) { \ + ql_concat(&list_a->head, &list_b->head, linkage); \ + } #endif /* JEMALLOC_INTERNAL_TYPED_LIST_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/util.h b/dep/jemalloc/include/jemalloc/internal/util.h index dcb1c0a5d..ecfa76b82 100644 --- a/dep/jemalloc/include/jemalloc/internal/util.h +++ b/dep/jemalloc/include/jemalloc/internal/util.h @@ -1,14 +1,17 @@ #ifndef JEMALLOC_INTERNAL_UTIL_H #define JEMALLOC_INTERNAL_UTIL_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_types.h" + #define UTIL_INLINE static inline /* Junk fill patterns. */ #ifndef JEMALLOC_ALLOC_JUNK -# define JEMALLOC_ALLOC_JUNK ((uint8_t)0xa5) +# define JEMALLOC_ALLOC_JUNK ((uint8_t)0xa5) #endif #ifndef JEMALLOC_FREE_JUNK -# define JEMALLOC_FREE_JUNK ((uint8_t)0x5a) +# define JEMALLOC_FREE_JUNK ((uint8_t)0x5a) #endif /* @@ -17,6 +20,9 @@ */ #define JEMALLOC_ARG_CONCAT(...) __VA_ARGS__ +/* Number of elements in a fixed-size array. */ +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) + /* cpp macro definition stringification. */ #define STRINGIFY_HELPER(x) #x #define STRINGIFY(x) STRINGIFY_HELPER(x) @@ -26,22 +32,25 @@ * wherever the compiler fails to recognize that the variable is never used * uninitialized. */ -#define JEMALLOC_CC_SILENCE_INIT(v) = v +#define JEMALLOC_CC_SILENCE_INIT(...) = __VA_ARGS__ #ifdef __GNUC__ -# define likely(x) __builtin_expect(!!(x), 1) -# define unlikely(x) __builtin_expect(!!(x), 0) +# define likely(x) __builtin_expect(!!(x), 1) +# define unlikely(x) __builtin_expect(!!(x), 0) #else -# define likely(x) !!(x) -# define unlikely(x) !!(x) +# define likely(x) !!(x) +# define unlikely(x) !!(x) #endif -#if !defined(JEMALLOC_INTERNAL_UNREACHABLE) -# error JEMALLOC_INTERNAL_UNREACHABLE should have been defined by configure +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +# include +#else +# if !defined(JEMALLOC_INTERNAL_UNREACHABLE) +# error JEMALLOC_INTERNAL_UNREACHABLE should have been defined by configure +# endif +# define unreachable() JEMALLOC_INTERNAL_UNREACHABLE() #endif -#define unreachable() JEMALLOC_INTERNAL_UNREACHABLE() - /* Set error code. */ UTIL_INLINE void set_errno(int errnum) { @@ -62,12 +71,29 @@ get_errno(void) { #endif } -JEMALLOC_ALWAYS_INLINE void -util_assume(bool b) { - if (!b) { - unreachable(); - } -} +#ifdef _MSC_VER +# define util_assume __assume +#elif defined(__clang__) \ + && (__clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 6)) +# define util_assume __builtin_assume +#else +# define util_assume(expr) \ + do { \ + if (!(expr)) { \ + unreachable(); \ + } \ + } while (0) +#endif + +/* Allows compiler constant folding on inlined paths. */ +#if defined(__has_builtin) +# if __has_builtin(__builtin_constant_p) +# define util_compile_time_const(x) __builtin_constant_p(x) +# endif +#endif +#ifndef util_compile_time_const +# define util_compile_time_const(x) (false) +#endif /* ptr should be valid. */ JEMALLOC_ALWAYS_INLINE void @@ -107,17 +133,24 @@ util_prefetch_write(void *ptr) { JEMALLOC_ALWAYS_INLINE void util_prefetch_read_range(void *ptr, size_t sz) { for (size_t i = 0; i < sz; i += CACHELINE) { - util_prefetch_read((void *)((uintptr_t)ptr + i)); + util_prefetch_read((void *)((byte_t *)ptr + i)); } } JEMALLOC_ALWAYS_INLINE void util_prefetch_write_range(void *ptr, size_t sz) { for (size_t i = 0; i < sz; i += CACHELINE) { - util_prefetch_write((void *)((uintptr_t)ptr + i)); + util_prefetch_write((void *)((byte_t *)ptr + i)); } } #undef UTIL_INLINE +/* + * Reads the settings in the following format: + * key1-key2:value|key3-key4:value|... + * Note it does not handle the ending '\0'. + */ +bool multi_setting_parse_next(const char **setting_segment_cur, + size_t *len_left, size_t *key_start, size_t *key_end, size_t *value); #endif /* JEMALLOC_INTERNAL_UTIL_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/witness.h b/dep/jemalloc/include/jemalloc/internal/witness.h index e81b9a006..0a426ff56 100644 --- a/dep/jemalloc/include/jemalloc/internal/witness.h +++ b/dep/jemalloc/include/jemalloc/internal/witness.h @@ -1,6 +1,8 @@ #ifndef JEMALLOC_INTERNAL_WITNESS_H #define JEMALLOC_INTERNAL_WITNESS_H +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/assert.h" #include "jemalloc/internal/ql.h" /******************************************************************************/ @@ -44,7 +46,7 @@ enum witness_rank_e { WITNESS_RANK_DECAY = WITNESS_RANK_CORE, WITNESS_RANK_TCACHE_QL, - WITNESS_RANK_SEC_SHARD, + WITNESS_RANK_SEC_BIN, WITNESS_RANK_EXTENT_GROW, WITNESS_RANK_HPA_SHARD_GROW = WITNESS_RANK_EXTENT_GROW, @@ -63,7 +65,7 @@ enum witness_rank_e { WITNESS_RANK_ARENA_LARGE, WITNESS_RANK_HOOK, - WITNESS_RANK_LEAF=0x1000, + WITNESS_RANK_LEAF = 0x1000, WITNESS_RANK_BIN = WITNESS_RANK_LEAF, WITNESS_RANK_ARENA_STATS = WITNESS_RANK_LEAF, WITNESS_RANK_COUNTER_ACCUM = WITNESS_RANK_LEAF, @@ -75,6 +77,7 @@ enum witness_rank_e { WITNESS_RANK_PROF_RECENT_ALLOC = WITNESS_RANK_LEAF, WITNESS_RANK_PROF_STATS = WITNESS_RANK_LEAF, WITNESS_RANK_PROF_THREAD_ACTIVE_INIT = WITNESS_RANK_LEAF, + WITNESS_RANK_THREAD_EVENTS_USER = WITNESS_RANK_LEAF, }; typedef enum witness_rank_e witness_rank_t; @@ -82,38 +85,43 @@ typedef enum witness_rank_e witness_rank_t; /* PER-WITNESS DATA */ /******************************************************************************/ #if defined(JEMALLOC_DEBUG) -# define WITNESS_INITIALIZER(name, rank) {name, rank, NULL, NULL, {NULL, NULL}} +# define WITNESS_INITIALIZER(name, rank) \ + { \ + name, rank, NULL, NULL, { \ + NULL, NULL \ + } \ + } #else -# define WITNESS_INITIALIZER(name, rank) +# define WITNESS_INITIALIZER(name, rank) #endif typedef struct witness_s witness_t; typedef ql_head(witness_t) witness_list_t; -typedef int witness_comp_t (const witness_t *, void *, const witness_t *, - void *); +typedef int witness_comp_t( + const witness_t *, void *, const witness_t *, void *); struct witness_s { /* Name, used for printing lock order reversal messages. */ - const char *name; + const char *name; /* * Witness rank, where 0 is lowest and WITNESS_RANK_LEAF is highest. * Witnesses must be acquired in order of increasing rank. */ - witness_rank_t rank; + witness_rank_t rank; /* * If two witnesses are of equal rank and they have the samp comp * function pointer, it is called as a last attempt to differentiate * between witnesses of equal rank. */ - witness_comp_t *comp; + witness_comp_t *comp; /* Opaque data, passed to comp(). */ - void *opaque; + void *opaque; /* Linkage for thread's currently owned locks. */ - ql_elm(witness_t) link; + ql_elm(witness_t) link; }; /******************************************************************************/ @@ -122,10 +130,11 @@ struct witness_s { typedef struct witness_tsd_s witness_tsd_t; struct witness_tsd_s { witness_list_t witnesses; - bool forking; + bool forking; }; -#define WITNESS_TSD_INITIALIZER { ql_head_initializer(witnesses), false } +#define WITNESS_TSD_INITIALIZER \ + { ql_head_initializer(witnesses), false } #define WITNESS_TSDN_NULL ((witness_tsdn_t *)0) /******************************************************************************/ @@ -158,17 +167,17 @@ witness_tsdn_tsd(witness_tsdn_t *witness_tsdn) { void witness_init(witness_t *witness, const char *name, witness_rank_t rank, witness_comp_t *comp, void *opaque); -typedef void (witness_lock_error_t)(const witness_list_t *, const witness_t *); +typedef void(witness_lock_error_t)(const witness_list_t *, const witness_t *); extern witness_lock_error_t *JET_MUTABLE witness_lock_error; -typedef void (witness_owner_error_t)(const witness_t *); +typedef void(witness_owner_error_t)(const witness_t *); extern witness_owner_error_t *JET_MUTABLE witness_owner_error; -typedef void (witness_not_owner_error_t)(const witness_t *); +typedef void(witness_not_owner_error_t)(const witness_t *); extern witness_not_owner_error_t *JET_MUTABLE witness_not_owner_error; -typedef void (witness_depth_error_t)(const witness_list_t *, - witness_rank_t rank_inclusive, unsigned depth); +typedef void(witness_depth_error_t)( + const witness_list_t *, witness_rank_t rank_inclusive, unsigned depth); extern witness_depth_error_t *JET_MUTABLE witness_depth_error; void witnesses_cleanup(witness_tsd_t *witness_tsd); @@ -180,12 +189,12 @@ void witness_postfork_child(witness_tsd_t *witness_tsd); static inline bool witness_owner(witness_tsd_t *witness_tsd, const witness_t *witness) { witness_list_t *witnesses; - witness_t *w; + witness_t *w; cassert(config_debug); witnesses = &witness_tsd->witnesses; - ql_foreach(w, witnesses, link) { + ql_foreach (w, witnesses, link) { if (w == witness) { return true; } @@ -217,11 +226,11 @@ witness_assert_owner(witness_tsdn_t *witness_tsdn, const witness_t *witness) { } static inline void -witness_assert_not_owner(witness_tsdn_t *witness_tsdn, - const witness_t *witness) { - witness_tsd_t *witness_tsd; +witness_assert_not_owner( + witness_tsdn_t *witness_tsdn, const witness_t *witness) { + witness_tsd_t *witness_tsd; witness_list_t *witnesses; - witness_t *w; + witness_t *w; if (!config_debug) { return; @@ -236,7 +245,7 @@ witness_assert_not_owner(witness_tsdn_t *witness_tsdn, } witnesses = &witness_tsd->witnesses; - ql_foreach(w, witnesses, link) { + ql_foreach (w, witnesses, link) { if (w == witness) { witness_not_owner_error(witness); } @@ -245,9 +254,9 @@ witness_assert_not_owner(witness_tsdn_t *witness_tsdn, /* Returns depth. Not intended for direct use. */ static inline unsigned -witness_depth_to_rank(witness_list_t *witnesses, witness_rank_t rank_inclusive) -{ - unsigned d = 0; +witness_depth_to_rank( + witness_list_t *witnesses, witness_rank_t rank_inclusive) { + unsigned d = 0; witness_t *w = ql_last(witnesses, link); if (w != NULL) { @@ -270,7 +279,7 @@ witness_assert_depth_to_rank(witness_tsdn_t *witness_tsdn, } witness_list_t *witnesses = &witness_tsdn_tsd(witness_tsdn)->witnesses; - unsigned d = witness_depth_to_rank(witnesses, rank_inclusive); + unsigned d = witness_depth_to_rank(witnesses, rank_inclusive); if (d != depth) { witness_depth_error(witnesses, rank_inclusive, depth); @@ -288,14 +297,14 @@ witness_assert_lockless(witness_tsdn_t *witness_tsdn) { } static inline void -witness_assert_positive_depth_to_rank(witness_tsdn_t *witness_tsdn, - witness_rank_t rank_inclusive) { +witness_assert_positive_depth_to_rank( + witness_tsdn_t *witness_tsdn, witness_rank_t rank_inclusive) { if (!config_debug || witness_tsdn_null(witness_tsdn)) { return; } witness_list_t *witnesses = &witness_tsdn_tsd(witness_tsdn)->witnesses; - unsigned d = witness_depth_to_rank(witnesses, rank_inclusive); + unsigned d = witness_depth_to_rank(witnesses, rank_inclusive); if (d == 0) { witness_depth_error(witnesses, rank_inclusive, 1); @@ -304,9 +313,9 @@ witness_assert_positive_depth_to_rank(witness_tsdn_t *witness_tsdn, static inline void witness_lock(witness_tsdn_t *witness_tsdn, witness_t *witness) { - witness_tsd_t *witness_tsd; + witness_tsd_t *witness_tsd; witness_list_t *witnesses; - witness_t *w; + witness_t *w; if (!config_debug) { return; @@ -331,9 +340,9 @@ witness_lock(witness_tsdn_t *witness_tsdn, witness_t *witness) { } else if (w->rank > witness->rank) { /* Not forking, rank order reversal. */ witness_lock_error(witnesses, witness); - } else if (w->rank == witness->rank && (w->comp == NULL || w->comp != - witness->comp || w->comp(w, w->opaque, witness, witness->opaque) > - 0)) { + } else if (w->rank == witness->rank + && (w->comp == NULL || w->comp != witness->comp + || w->comp(w, w->opaque, witness, witness->opaque) > 0)) { /* * Missing/incompatible comparison function, or comparison * function indicates rank order reversal. @@ -341,13 +350,16 @@ witness_lock(witness_tsdn_t *witness_tsdn, witness_t *witness) { witness_lock_error(witnesses, witness); } + /* Suppress spurious warning from static analysis */ + assert( + ql_empty(witnesses) || qr_prev(ql_first(witnesses), link) != NULL); ql_elm_new(witness, link); ql_tail_insert(witnesses, witness, link); } static inline void witness_unlock(witness_tsdn_t *witness_tsdn, witness_t *witness) { - witness_tsd_t *witness_tsd; + witness_tsd_t *witness_tsd; witness_list_t *witnesses; if (!config_debug) { diff --git a/dep/jemalloc/include/jemalloc/jemalloc.h.in b/dep/jemalloc/include/jemalloc/jemalloc.h.in index 6139d3e79..29f9693ad 100644 --- a/dep/jemalloc/include/jemalloc/jemalloc.h.in +++ b/dep/jemalloc/include/jemalloc/jemalloc.h.in @@ -1,8 +1,6 @@ #ifndef JEMALLOC_H_ #define JEMALLOC_H_ -#ifdef __cplusplus -extern "C" { -#endif +#pragma GCC system_header /* jemalloc_defs.h */ @@ -27,12 +25,16 @@ extern "C" { /* Defined if cold attribute is supported. */ #cmakedefine JEMALLOC_HAVE_ATTR_COLD +/* Defined if deprecated attribute is supported. */ +#cmakedefine JEMALLOC_HAVE_ATTR_DEPRECATED + /* * Define overrides for non-standard allocator-related functions if they are * present on the system. */ #cmakedefine JEMALLOC_OVERRIDE_MEMALIGN #cmakedefine JEMALLOC_OVERRIDE_VALLOC +#cmakedefine JEMALLOC_OVERRIDE_PVALLOC /* * At least Linux omits the "const" in: @@ -50,6 +52,12 @@ extern "C" { */ #cmakedefine JEMALLOC_USE_CXX_THROW +/* + * If undefined, disables reading configuration from environment variable or file + */ +#cmakedefine JEMALLOC_CONFIG_ENV +#cmakedefine JEMALLOC_CONFIG_FILE + #ifdef _MSC_VER # ifdef _WIN64 # define LG_SIZEOF_PTR_WIN 3 @@ -73,6 +81,8 @@ extern "C" { # define je_calloc @JEMALLOC_PREFIX@calloc # define je_dallocx @JEMALLOC_PREFIX@dallocx # define je_free @JEMALLOC_PREFIX@free +# define je_free_sized @JEMALLOC_PREFIX@free_sized +# define je_free_aligned_sized @JEMALLOC_PREFIX@free_aligned_sized # define je_mallctl @JEMALLOC_PREFIX@mallctl # define je_mallctlbymib @JEMALLOC_PREFIX@mallctlbymib # define je_mallctlnametomib @JEMALLOC_PREFIX@mallctlnametomib @@ -93,6 +103,7 @@ extern "C" { # define je_xallocx @JEMALLOC_PREFIX@xallocx # define je_memalign @JEMALLOC_PREFIX@memalign # define je_valloc @JEMALLOC_PREFIX@valloc +# define je_pvalloc @JEMALLOC_PREFIX@pvalloc # define je_malloc_size @JEMALLOC_PREFIX@malloc_size #endif @@ -152,7 +163,7 @@ extern "C" { #define MALLCTL_ARENAS_DESTROYED 4097 #if defined(__cplusplus) && defined(JEMALLOC_USE_CXX_THROW) -# define JEMALLOC_CXX_THROW throw() +# define JEMALLOC_CXX_THROW noexcept (true) #else # define JEMALLOC_CXX_THROW #endif @@ -186,6 +197,7 @@ extern "C" { # define JEMALLOC_ALLOCATOR # endif # define JEMALLOC_COLD +# define JEMALLOC_WARN_ON_USAGE(warning_message) #elif defined(JEMALLOC_HAVE_ATTR) # define JEMALLOC_ATTR(s) __attribute__((s)) # define JEMALLOC_ALIGNED(s) JEMALLOC_ATTR(aligned(s)) @@ -226,6 +238,11 @@ extern "C" { # else # define JEMALLOC_COLD # endif +# ifdef JEMALLOC_HAVE_ATTR_DEPRECATED +# define JEMALLOC_WARN_ON_USAGE(warning_message) JEMALLOC_ATTR(deprecated(warning_message)) +# else +# define JEMALLOC_WARN_ON_USAGE(warning_message) +# endif #else # define JEMALLOC_ATTR(s) # define JEMALLOC_ALIGNED(s) @@ -240,9 +257,10 @@ extern "C" { # define JEMALLOC_RESTRICT_RETURN # define JEMALLOC_ALLOCATOR # define JEMALLOC_COLD +# define JEMALLOC_WARN_ON_USAGE(warning_message) #endif -#if (defined(__APPLE__) || defined(__FreeBSD__)) && !defined(JEMALLOC_NO_RENAME) +#if (defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) || (defined(__linux__) && !defined(__GLIBC__))) && !defined(JEMALLOC_NO_RENAME) # define JEMALLOC_SYS_NOTHROW #else # define JEMALLOC_SYS_NOTHROW JEMALLOC_NOTHROW @@ -255,7 +273,12 @@ extern "C" { * of namespace management, and should be omitted in application code unless * JEMALLOC_NO_DEMANGLE is defined (see jemalloc_mangle@install_suffix@.h). */ +#ifdef __cplusplus +extern "C" { +#endif + extern JEMALLOC_EXPORT const char *@je_@malloc_conf; +extern JEMALLOC_EXPORT const char *@je_@malloc_conf_2_conf_harder; extern JEMALLOC_EXPORT void (*@je_@malloc_message)(void *cbopaque, const char *s); @@ -277,6 +300,9 @@ JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN JEMALLOC_CXX_THROW JEMALLOC_ALLOC_SIZE(2); JEMALLOC_EXPORT void JEMALLOC_SYS_NOTHROW @je_@free(void *ptr) JEMALLOC_CXX_THROW; +JEMALLOC_EXPORT void JEMALLOC_NOTHROW @je_@free_sized(void *ptr, size_t size); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW @je_@free_aligned_sized( + void *ptr, size_t alignment, size_t size); JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN void JEMALLOC_NOTHROW *@je_@mallocx(size_t size, int flags) @@ -322,8 +348,22 @@ JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN JEMALLOC_ATTR(malloc); #endif +#ifdef JEMALLOC_OVERRIDE_PVALLOC +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *@je_@pvalloc(size_t size) JEMALLOC_CXX_THROW + JEMALLOC_ATTR(malloc); +#endif + +#ifdef __cplusplus +} +#endif + /* jemalloc_typedefs.h */ +#ifdef __cplusplus +extern "C" { +#endif + typedef struct extent_hooks_s extent_hooks_t; /* @@ -402,6 +442,10 @@ struct extent_hooks_s { extent_merge_t *merge; }; +#ifdef __cplusplus +} +#endif + /* jemalloc_mangle.h */ /* @@ -419,6 +463,8 @@ struct extent_hooks_s { # define calloc je_calloc # define dallocx je_dallocx # define free je_free +# define free_sized je_free_sized +# define free_aligned_sized je_free_aligned_sized # define mallctl je_mallctl # define mallctlbymib je_mallctlbymib # define mallctlnametomib je_mallctlnametomib @@ -439,6 +485,7 @@ struct extent_hooks_s { # define xallocx je_xallocx # define memalign je_memalign # define valloc je_valloc +# define pvalloc je_pvalloc # define malloc_size je_malloc_size #endif @@ -454,12 +501,14 @@ struct extent_hooks_s { # undef je_calloc # undef je_dallocx # undef je_free +# undef je_free_sized +# undef je_free_aligned_sized # undef je_mallctl # undef je_mallctlbymib # undef je_mallctlnametomib # undef je_malloc # undef je_malloc_conf -# undef malloc_conf_2_conf_harder +# undef je_malloc_conf_2_conf_harder # undef je_malloc_message # undef je_malloc_stats_print # undef je_malloc_usable_size @@ -474,10 +523,8 @@ struct extent_hooks_s { # undef je_xallocx # undef je_memalign # undef je_valloc +# undef je_pvalloc # undef je_malloc_size #endif -#ifdef __cplusplus -} -#endif #endif /* JEMALLOC_H_ */ diff --git a/dep/jemalloc/src/arena.c b/dep/jemalloc/src/arena.c index 857b27c52..e43aabe4c 100644 --- a/dep/jemalloc/src/arena.c +++ b/dep/jemalloc/src/arena.c @@ -18,16 +18,11 @@ JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS /* Data. */ /* - * Define names for both unininitialized and initialized phases, so that + * Define names for both uninitialized and initialized phases, so that * options and mallctl processing are straightforward. */ -const char *percpu_arena_mode_names[] = { - "percpu", - "phycpu", - "disabled", - "percpu", - "phycpu" -}; +const char *const percpu_arena_mode_names[] = { + "percpu", "phycpu", "disabled", "percpu", "phycpu"}; percpu_arena_mode_t opt_percpu_arena = PERCPU_ARENA_DEFAULT; ssize_t opt_dirty_decay_ms = DIRTY_DECAY_MS_DEFAULT; @@ -36,22 +31,31 @@ ssize_t opt_muzzy_decay_ms = MUZZY_DECAY_MS_DEFAULT; static atomic_zd_t dirty_decay_ms_default; static atomic_zd_t muzzy_decay_ms_default; -emap_t arena_emap_global; -pa_central_t arena_pa_central_global; +emap_t arena_emap_global; +static pa_central_t arena_pa_central_global; div_info_t arena_binind_div_info[SC_NBINS]; size_t opt_oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT; size_t oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT; -uint32_t arena_bin_offsets[SC_NBINS]; +uint32_t arena_bin_offsets[SC_NBINS]; static unsigned nbins_total; -static unsigned huge_arena_ind; +/* + * a0 is used to handle huge requests before malloc init completes. After + * that, the huge_arena_ind is updated to point to the actual huge arena, + * which is the last one of the auto arenas. + */ +unsigned huge_arena_ind = 0; +bool opt_huge_arena_pac_thp = false; +pac_thp_t huge_arena_pac_thp = {.thp_madvise = false, + .auto_thp_switched = false, + .n_thp_lazy = ATOMIC_INIT(0)}; const arena_config_t arena_config_default = { - /* .extent_hooks = */ (extent_hooks_t *)&ehooks_default_extent_hooks, - /* .metadata_use_hooks = */ true, + /* .extent_hooks = */ (extent_hooks_t *)&ehooks_default_extent_hooks, + /* .metadata_use_hooks = */ true, }; /******************************************************************************/ @@ -60,13 +64,10 @@ const arena_config_t arena_config_default = { * definition. */ -static bool arena_decay_dirty(tsdn_t *tsdn, arena_t *arena, - bool is_background_thread, bool all); -static void arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, edata_t *slab, - bin_t *bin); -static void -arena_maybe_do_deferred_work(tsdn_t *tsdn, arena_t *arena, decay_t *decay, - size_t npages_new); +static bool arena_decay_dirty( + tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all); +static void arena_maybe_do_deferred_work( + tsdn_t *tsdn, arena_t *arena, decay_t *decay, size_t npages_new); /******************************************************************************/ @@ -85,15 +86,17 @@ void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats, - bin_stats_data_t *bstats, arena_stats_large_t *lstats, - pac_estats_t *estats, hpa_shard_stats_t *hpastats, sec_stats_t *secstats) { + bin_stats_data_t *bstats, arena_stats_large_t *lstats, pac_estats_t *estats, + hpa_shard_stats_t *hpastats) { cassert(config_stats); arena_basic_stats_merge(tsdn, arena, nthreads, dss, dirty_decay_ms, muzzy_decay_ms, nactive, ndirty, nmuzzy); - size_t base_allocated, base_resident, base_mapped, metadata_thp; - base_stats_get(tsdn, arena->base, &base_allocated, &base_resident, + size_t base_allocated, base_edata_allocated, base_rtree_allocated, + base_resident, base_mapped, metadata_thp; + base_stats_get(tsdn, arena->base, &base_allocated, + &base_edata_allocated, &base_rtree_allocated, &base_resident, &base_mapped, &metadata_thp); size_t pac_mapped_sz = pac_mapped(&arena->pa_shard.pac); astats->mapped += base_mapped + pac_mapped_sz; @@ -102,27 +105,32 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, LOCKEDINT_MTX_LOCK(tsdn, arena->stats.mtx); astats->base += base_allocated; + astats->metadata_edata += base_edata_allocated; + astats->metadata_rtree += base_rtree_allocated; atomic_load_add_store_zu(&astats->internal, arena_internal_get(arena)); astats->metadata_thp += metadata_thp; for (szind_t i = 0; i < SC_NSIZES - SC_NBINS; i++) { - uint64_t nmalloc = locked_read_u64(tsdn, - LOCKEDINT_MTX(arena->stats.mtx), - &arena->stats.lstats[i].nmalloc); - locked_inc_u64_unsynchronized(&lstats[i].nmalloc, nmalloc); - astats->nmalloc_large += nmalloc; - + /* ndalloc should be read before nmalloc, + * since otherwise it is possible for ndalloc to be incremented, + * and the following can become true: ndalloc > nmalloc */ uint64_t ndalloc = locked_read_u64(tsdn, LOCKEDINT_MTX(arena->stats.mtx), &arena->stats.lstats[i].ndalloc); locked_inc_u64_unsynchronized(&lstats[i].ndalloc, ndalloc); astats->ndalloc_large += ndalloc; + uint64_t nmalloc = locked_read_u64(tsdn, + LOCKEDINT_MTX(arena->stats.mtx), + &arena->stats.lstats[i].nmalloc); + locked_inc_u64_unsynchronized(&lstats[i].nmalloc, nmalloc); + astats->nmalloc_large += nmalloc; + uint64_t nrequests = locked_read_u64(tsdn, LOCKEDINT_MTX(arena->stats.mtx), &arena->stats.lstats[i].nrequests); - locked_inc_u64_unsynchronized(&lstats[i].nrequests, - nmalloc + nrequests); + locked_inc_u64_unsynchronized( + &lstats[i].nrequests, nmalloc + nrequests); astats->nrequests_large += nmalloc + nrequests; /* nfill == nmalloc for large currently. */ @@ -139,12 +147,17 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, assert(nmalloc - ndalloc <= SIZE_T_MAX); size_t curlextents = (size_t)(nmalloc - ndalloc); lstats[i].curlextents += curlextents; - astats->allocated_large += - curlextents * sz_index2size(SC_NBINS + i); + + uint64_t active_bytes = locked_read_u64(tsdn, + LOCKEDINT_MTX(arena->stats.mtx), + &arena->stats.lstats[i].active_bytes); + locked_inc_u64_unsynchronized( + &lstats[i].active_bytes, active_bytes); + astats->allocated_large += active_bytes; } pa_shard_stats_merge(tsdn, &arena->pa_shard, &astats->pa_shard_stats, - estats, hpastats, secstats, &astats->resident); + estats, hpastats, &astats->resident); LOCKEDINT_MTX_UNLOCK(tsdn, arena->stats.mtx); @@ -153,16 +166,19 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, astats->tcache_stashed_bytes = 0; malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); cache_bin_array_descriptor_t *descriptor; - ql_foreach(descriptor, &arena->cache_bin_array_descriptor_ql, link) { - for (szind_t i = 0; i < nhbins; i++) { + ql_foreach (descriptor, &arena->cache_bin_array_descriptor_ql, link) { + for (szind_t i = 0; i < TCACHE_NBINS_MAX; i++) { cache_bin_t *cache_bin = &descriptor->bins[i]; - cache_bin_sz_t ncached, nstashed; - cache_bin_nitems_get_remote(cache_bin, - &tcache_bin_info[i], &ncached, &nstashed); + if (cache_bin_disabled(cache_bin)) { + continue; + } + cache_bin_sz_t ncached, nstashed; + cache_bin_nitems_get_remote( + cache_bin, &ncached, &nstashed); astats->tcache_bytes += ncached * sz_index2size(i); - astats->tcache_stashed_bytes += nstashed * - sz_index2size(i); + astats->tcache_stashed_bytes += nstashed + * sz_index2size(i); } } malloc_mutex_prof_read(tsdn, @@ -170,19 +186,18 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, &arena->tcache_ql_mtx); malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx); -#define READ_ARENA_MUTEX_PROF_DATA(mtx, ind) \ - malloc_mutex_lock(tsdn, &arena->mtx); \ - malloc_mutex_prof_read(tsdn, &astats->mutex_prof_data[ind], \ - &arena->mtx); \ - malloc_mutex_unlock(tsdn, &arena->mtx); +#define READ_ARENA_MUTEX_PROF_DATA(mtx, ind) \ + malloc_mutex_lock(tsdn, &arena->mtx); \ + malloc_mutex_prof_read( \ + tsdn, &astats->mutex_prof_data[ind], &arena->mtx); \ + malloc_mutex_unlock(tsdn, &arena->mtx); /* Gather per arena mutex profiling data. */ READ_ARENA_MUTEX_PROF_DATA(large_mtx, arena_prof_mutex_large); - READ_ARENA_MUTEX_PROF_DATA(base->mtx, - arena_prof_mutex_base); + READ_ARENA_MUTEX_PROF_DATA(base->mtx, arena_prof_mutex_base); #undef READ_ARENA_MUTEX_PROF_DATA - pa_shard_mtx_stats_read(tsdn, &arena->pa_shard, - astats->mutex_prof_data); + pa_shard_mtx_stats_read( + tsdn, &arena->pa_shard, astats->mutex_prof_data); nstime_copy(&astats->uptime, &arena->create_time); nstime_update(&astats->uptime); @@ -190,32 +205,33 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, for (szind_t i = 0; i < SC_NBINS; i++) { for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { - bin_stats_merge(tsdn, &bstats[i], - arena_get_bin(arena, i, j)); + bin_stats_merge( + tsdn, &bstats[i], arena_get_bin(arena, i, j)); } } } static void -arena_background_thread_inactivity_check(tsdn_t *tsdn, arena_t *arena, - bool is_background_thread) { +arena_background_thread_inactivity_check( + tsdn_t *tsdn, arena_t *arena, bool is_background_thread) { if (!background_thread_enabled() || is_background_thread) { return; } - background_thread_info_t *info = - arena_background_thread_info_get(arena); + background_thread_info_t *info = arena_background_thread_info_get( + arena); if (background_thread_indefinite_sleep(info)) { - arena_maybe_do_deferred_work(tsdn, arena, - &arena->pa_shard.pac.decay_dirty, 0); + arena_maybe_do_deferred_work( + tsdn, arena, &arena->pa_shard.pac.decay_dirty, 0); } } /* * React to deferred work generated by a PAI function. */ -void arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena) { - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); +void +arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena) { + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); if (decay_immediately(&arena->pa_shard.pac.decay_dirty)) { arena_decay_dirty(tsdn, arena, false, true); @@ -223,133 +239,105 @@ void arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena) { arena_background_thread_inactivity_check(tsdn, arena, false); } -static void * -arena_slab_reg_alloc(edata_t *slab, const bin_info_t *bin_info) { - void *ret; - slab_data_t *slab_data = edata_slab_data_get(slab); - size_t regind; - - assert(edata_nfree_get(slab) > 0); - assert(!bitmap_full(slab_data->bitmap, &bin_info->bitmap_info)); - - regind = bitmap_sfu(slab_data->bitmap, &bin_info->bitmap_info); - ret = (void *)((uintptr_t)edata_addr_get(slab) + - (uintptr_t)(bin_info->reg_size * regind)); - edata_nfree_dec(slab); - return ret; -} - -static void -arena_slab_reg_alloc_batch(edata_t *slab, const bin_info_t *bin_info, - unsigned cnt, void** ptrs) { - slab_data_t *slab_data = edata_slab_data_get(slab); - - assert(edata_nfree_get(slab) >= cnt); - assert(!bitmap_full(slab_data->bitmap, &bin_info->bitmap_info)); - -#if (! defined JEMALLOC_INTERNAL_POPCOUNTL) || (defined BITMAP_USE_TREE) - for (unsigned i = 0; i < cnt; i++) { - size_t regind = bitmap_sfu(slab_data->bitmap, - &bin_info->bitmap_info); - *(ptrs + i) = (void *)((uintptr_t)edata_addr_get(slab) + - (uintptr_t)(bin_info->reg_size * regind)); - } -#else - unsigned group = 0; - bitmap_t g = slab_data->bitmap[group]; - unsigned i = 0; - while (i < cnt) { - while (g == 0) { - g = slab_data->bitmap[++group]; - } - size_t shift = group << LG_BITMAP_GROUP_NBITS; - size_t pop = popcount_lu(g); - if (pop > (cnt - i)) { - pop = cnt - i; - } - - /* - * Load from memory locations only once, outside the - * hot loop below. - */ - uintptr_t base = (uintptr_t)edata_addr_get(slab); - uintptr_t regsize = (uintptr_t)bin_info->reg_size; - while (pop--) { - size_t bit = cfs_lu(&g); - size_t regind = shift + bit; - *(ptrs + i) = (void *)(base + regsize * regind); - - i++; - } - slab_data->bitmap[group] = g; - } -#endif - edata_nfree_sub(slab, cnt); -} - static void arena_large_malloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t usize) { - szind_t index, hindex; - cassert(config_stats); + szind_t index = sz_size2index(usize); + /* This only occurs when we have a sampled small allocation */ if (usize < SC_LARGE_MINCLASS) { - usize = SC_LARGE_MINCLASS; + assert(index < SC_NBINS); + assert(usize >= PAGE && usize % PAGE == 0); + bin_t *bin = arena_get_bin(arena, index, /* binshard */ 0); + malloc_mutex_lock(tsdn, &bin->lock); + bin->stats.nmalloc++; + malloc_mutex_unlock(tsdn, &bin->lock); + } else { + assert(index >= SC_NBINS); + szind_t hindex = index - SC_NBINS; + LOCKEDINT_MTX_LOCK(tsdn, arena->stats.mtx); + locked_inc_u64(tsdn, LOCKEDINT_MTX(arena->stats.mtx), + &arena->stats.lstats[hindex].nmalloc, 1); + locked_inc_u64(tsdn, LOCKEDINT_MTX(arena->stats.mtx), + &arena->stats.lstats[hindex].active_bytes, usize); + LOCKEDINT_MTX_UNLOCK(tsdn, arena->stats.mtx); } - index = sz_size2index(usize); - hindex = (index >= SC_NBINS) ? index - SC_NBINS : 0; - - locked_inc_u64(tsdn, LOCKEDINT_MTX(arena->stats.mtx), - &arena->stats.lstats[hindex].nmalloc, 1); } static void arena_large_dalloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t usize) { - szind_t index, hindex; - cassert(config_stats); + szind_t index = sz_size2index(usize); + /* This only occurs when we have a sampled small allocation */ if (usize < SC_LARGE_MINCLASS) { - usize = SC_LARGE_MINCLASS; + assert(index < SC_NBINS); + assert(usize >= PAGE && usize % PAGE == 0); + bin_t *bin = arena_get_bin(arena, index, /* binshard */ 0); + malloc_mutex_lock(tsdn, &bin->lock); + bin->stats.ndalloc++; + malloc_mutex_unlock(tsdn, &bin->lock); + } else { + assert(index >= SC_NBINS); + szind_t hindex = index - SC_NBINS; + LOCKEDINT_MTX_LOCK(tsdn, arena->stats.mtx); + locked_inc_u64(tsdn, LOCKEDINT_MTX(arena->stats.mtx), + &arena->stats.lstats[hindex].ndalloc, 1); + locked_dec_u64(tsdn, LOCKEDINT_MTX(arena->stats.mtx), + &arena->stats.lstats[hindex].active_bytes, usize); + LOCKEDINT_MTX_UNLOCK(tsdn, arena->stats.mtx); } - index = sz_size2index(usize); - hindex = (index >= SC_NBINS) ? index - SC_NBINS : 0; - - locked_inc_u64(tsdn, LOCKEDINT_MTX(arena->stats.mtx), - &arena->stats.lstats[hindex].ndalloc, 1); } static void -arena_large_ralloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t oldusize, - size_t usize) { +arena_large_ralloc_stats_update( + tsdn_t *tsdn, arena_t *arena, size_t oldusize, size_t usize) { arena_large_malloc_stats_update(tsdn, arena, usize); arena_large_dalloc_stats_update(tsdn, arena, oldusize); } edata_t * -arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, - size_t alignment, bool zero) { - bool deferred_work_generated = false; +arena_extent_alloc_large( + tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero) { + bool deferred_work_generated = false; szind_t szind = sz_size2index(usize); - size_t esize = usize + sz_large_pad; + size_t esize = usize + sz_large_pad; - bool guarded = san_large_extent_decide_guard(tsdn, - arena_get_ehooks(arena), esize, alignment); + bool guarded = san_large_extent_decide_guard( + tsdn, arena_get_ehooks(arena), esize, alignment); + + /* + * - if usize >= opt_calloc_madvise_threshold, + * - pa_alloc(..., zero_override = zero, ...) + * - otherwise, + * - pa_alloc(..., zero_override = false, ...) + * - use memset() to zero out memory if zero == true. + */ + bool zero_override = zero && (usize >= opt_calloc_madvise_threshold); edata_t *edata = pa_alloc(tsdn, &arena->pa_shard, esize, alignment, - /* slab */ false, szind, zero, guarded, &deferred_work_generated); - assert(deferred_work_generated == false); + /* slab */ false, szind, zero_override, guarded, + &deferred_work_generated); - if (edata != NULL) { - if (config_stats) { - LOCKEDINT_MTX_LOCK(tsdn, arena->stats.mtx); - arena_large_malloc_stats_update(tsdn, arena, usize); - LOCKEDINT_MTX_UNLOCK(tsdn, arena->stats.mtx); - } + if (edata == NULL) { + return NULL; } - if (edata != NULL && sz_large_pad != 0) { + if (config_stats) { + arena_large_malloc_stats_update(tsdn, arena, usize); + } + if (sz_large_pad != 0) { arena_cache_oblivious_randomize(tsdn, arena, edata, alignment); } + /* + * This branch should be put after the randomization so that the addr + * returned by edata_addr_get() has already be randomized, + * if cache_oblivious is enabled. + */ + if (zero && !zero_override && !edata_zeroed_get(edata)) { + void *addr = edata_addr_get(edata); + size_t usize = edata_usize_get(edata); + memset(addr, 0, usize); + } return edata; } @@ -357,34 +345,28 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, void arena_extent_dalloc_large_prep(tsdn_t *tsdn, arena_t *arena, edata_t *edata) { if (config_stats) { - LOCKEDINT_MTX_LOCK(tsdn, arena->stats.mtx); - arena_large_dalloc_stats_update(tsdn, arena, - edata_usize_get(edata)); - LOCKEDINT_MTX_UNLOCK(tsdn, arena->stats.mtx); + arena_large_dalloc_stats_update( + tsdn, arena, edata_usize_get(edata)); } } void -arena_extent_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, edata_t *edata, - size_t oldusize) { +arena_extent_ralloc_large_shrink( + tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t oldusize) { size_t usize = edata_usize_get(edata); if (config_stats) { - LOCKEDINT_MTX_LOCK(tsdn, arena->stats.mtx); arena_large_ralloc_stats_update(tsdn, arena, oldusize, usize); - LOCKEDINT_MTX_UNLOCK(tsdn, arena->stats.mtx); } } void -arena_extent_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena, edata_t *edata, - size_t oldusize) { +arena_extent_ralloc_large_expand( + tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t oldusize) { size_t usize = edata_usize_get(edata); if (config_stats) { - LOCKEDINT_MTX_LOCK(tsdn, arena->stats.mtx); arena_large_ralloc_stats_update(tsdn, arena, oldusize, usize); - LOCKEDINT_MTX_UNLOCK(tsdn, arena->stats.mtx); } } @@ -405,12 +387,12 @@ arena_decide_unforced_purge_eagerness(bool is_background_thread) { } bool -arena_decay_ms_set(tsdn_t *tsdn, arena_t *arena, extent_state_t state, - ssize_t decay_ms) { +arena_decay_ms_set( + tsdn_t *tsdn, arena_t *arena, extent_state_t state, ssize_t decay_ms) { pac_purge_eagerness_t eagerness = arena_decide_unforced_purge_eagerness( /* is_background_thread */ false); - return pa_decay_ms_set(tsdn, &arena->pa_shard, state, decay_ms, - eagerness); + return pa_decay_ms_set( + tsdn, &arena->pa_shard, state, decay_ms, eagerness); } ssize_t @@ -420,8 +402,8 @@ arena_decay_ms_get(arena_t *arena, extent_state_t state) { static bool arena_decay_impl(tsdn_t *tsdn, arena_t *arena, decay_t *decay, - pac_decay_stats_t *decay_stats, ecache_t *ecache, - bool is_background_thread, bool all) { + pac_decay_stats_t *decay_stats, ecache_t *ecache, bool is_background_thread, + bool all) { if (all) { malloc_mutex_lock(tsdn, &decay->mtx); pac_decay_all(tsdn, &arena->pa_shard.pac, decay, decay_stats, @@ -434,19 +416,19 @@ arena_decay_impl(tsdn_t *tsdn, arena_t *arena, decay_t *decay, /* No need to wait if another thread is in progress. */ return true; } - pac_purge_eagerness_t eagerness = - arena_decide_unforced_purge_eagerness(is_background_thread); - bool epoch_advanced = pac_maybe_decay_purge(tsdn, &arena->pa_shard.pac, - decay, decay_stats, ecache, eagerness); - size_t npages_new; + pac_purge_eagerness_t eagerness = arena_decide_unforced_purge_eagerness( + is_background_thread); + bool epoch_advanced = pac_maybe_decay_purge( + tsdn, &arena->pa_shard.pac, decay, decay_stats, ecache, eagerness); + size_t npages_new JEMALLOC_CLANG_ANALYZER_SILENCE_INIT(0); if (epoch_advanced) { /* Backlog is updated on epoch advance. */ npages_new = decay_epoch_npages_delta(decay); } malloc_mutex_unlock(tsdn, &decay->mtx); - if (have_background_thread && background_thread_enabled() && - epoch_advanced && !is_background_thread) { + if (have_background_thread && background_thread_enabled() + && epoch_advanced && !is_background_thread) { arena_maybe_do_deferred_work(tsdn, arena, decay, npages_new); } @@ -454,16 +436,16 @@ arena_decay_impl(tsdn_t *tsdn, arena_t *arena, decay_t *decay, } static bool -arena_decay_dirty(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, - bool all) { +arena_decay_dirty( + tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) { return arena_decay_impl(tsdn, arena, &arena->pa_shard.pac.decay_dirty, &arena->pa_shard.pac.stats->decay_dirty, &arena->pa_shard.pac.ecache_dirty, is_background_thread, all); } static bool -arena_decay_muzzy(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, - bool all) { +arena_decay_muzzy( + tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) { if (pa_shard_dont_decay_muzzy(&arena->pa_shard)) { return false; } @@ -480,7 +462,7 @@ arena_decay(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) { * as possible", including flushing any caches (for situations * like thread death, or manual purge calls). */ - sec_flush(tsdn, &arena->pa_shard.hpa_sec); + pa_shard_flush(tsdn, &arena->pa_shard); } if (arena_decay_dirty(tsdn, arena, is_background_thread, all)) { return; @@ -510,13 +492,13 @@ arena_should_decay_early(tsdn_t *tsdn, arena_t *arena, decay_t *decay, } nstime_subtract(remaining_sleep, &decay->epoch); if (npages_new > 0) { - uint64_t npurge_new = decay_npages_purge_in(decay, - remaining_sleep, npages_new); + uint64_t npurge_new = decay_npages_purge_in( + decay, remaining_sleep, npages_new); info->npages_to_purge_new += npurge_new; } malloc_mutex_unlock(tsdn, &decay->mtx); - return info->npages_to_purge_new > - ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD; + return info->npages_to_purge_new + > ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD; } /* @@ -528,8 +510,8 @@ arena_should_decay_early(tsdn_t *tsdn, arena_t *arena, decay_t *decay, * deferred work has been generated. */ static void -arena_maybe_do_deferred_work(tsdn_t *tsdn, arena_t *arena, decay_t *decay, - size_t npages_new) { +arena_maybe_do_deferred_work( + tsdn_t *tsdn, arena_t *arena, decay_t *decay, size_t npages_new) { background_thread_info_t *info = arena_background_thread_info_get( arena); if (malloc_mutex_trylock(tsdn, &info->mtx)) { @@ -549,7 +531,7 @@ arena_maybe_do_deferred_work(tsdn_t *tsdn, arena_t *arena, decay_t *decay, if (background_thread_indefinite_sleep(info)) { background_thread_wakeup_early(info, NULL); } else if (arena_should_decay_early(tsdn, arena, decay, info, - &remaining_sleep, npages_new)) { + &remaining_sleep, npages_new)) { info->npages_to_purge_new = 0; background_thread_wakeup_early(info, &remaining_sleep); } @@ -573,63 +555,12 @@ arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *slab) { } } -static void -arena_bin_slabs_nonfull_insert(bin_t *bin, edata_t *slab) { - assert(edata_nfree_get(slab) > 0); - edata_heap_insert(&bin->slabs_nonfull, slab); - if (config_stats) { - bin->stats.nonfull_slabs++; - } -} - -static void -arena_bin_slabs_nonfull_remove(bin_t *bin, edata_t *slab) { - edata_heap_remove(&bin->slabs_nonfull, slab); - if (config_stats) { - bin->stats.nonfull_slabs--; - } -} - -static edata_t * -arena_bin_slabs_nonfull_tryget(bin_t *bin) { - edata_t *slab = edata_heap_remove_first(&bin->slabs_nonfull); - if (slab == NULL) { - return NULL; - } - if (config_stats) { - bin->stats.reslabs++; - bin->stats.nonfull_slabs--; - } - return slab; -} - -static void -arena_bin_slabs_full_insert(arena_t *arena, bin_t *bin, edata_t *slab) { - assert(edata_nfree_get(slab) == 0); - /* - * Tracking extents is required by arena_reset, which is not allowed - * for auto arenas. Bypass this step to avoid touching the edata - * linkage (often results in cache misses) for auto arenas. - */ - if (arena_is_auto(arena)) { - return; - } - edata_list_active_append(&bin->slabs_full, slab); -} - -static void -arena_bin_slabs_full_remove(arena_t *arena, bin_t *bin, edata_t *slab) { - if (arena_is_auto(arena)) { - return; - } - edata_list_active_remove(&bin->slabs_full, slab); -} - static void arena_bin_reset(tsd_t *tsd, arena_t *arena, bin_t *bin) { edata_t *slab; malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + if (bin->slabcur != NULL) { slab = bin->slabcur; bin->slabcur = NULL; @@ -643,8 +574,8 @@ arena_bin_reset(tsd_t *tsd, arena_t *arena, bin_t *bin) { malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); } for (slab = edata_list_active_first(&bin->slabs_full); slab != NULL; - slab = edata_list_active_first(&bin->slabs_full)) { - arena_bin_slabs_full_remove(arena, bin, slab); + slab = edata_list_active_first(&bin->slabs_full)) { + bin_slabs_full_remove(false, bin, slab); malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); @@ -656,6 +587,79 @@ arena_bin_reset(tsd_t *tsd, arena_t *arena, bin_t *bin) { malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); } +void +arena_prof_promote(tsdn_t *tsdn, void *ptr, size_t usize, size_t bumped_usize) { + cassert(config_prof); + assert(ptr != NULL); + assert(isalloc(tsdn, ptr) == bumped_usize); + assert(sz_can_use_slab(usize)); + + if (config_opt_safety_checks) { + safety_check_set_redzone(ptr, usize, bumped_usize); + } + + edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr); + + szind_t szind = sz_size2index(usize); + edata_szind_set(edata, szind); + emap_remap(tsdn, &arena_emap_global, edata, szind, /* slab */ false); + + assert(isalloc(tsdn, ptr) == usize); +} + +static size_t +arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr) { + cassert(config_prof); + assert(ptr != NULL); + size_t usize = isalloc(tsdn, ptr); + size_t bumped_usize = sz_sa2u(usize, PROF_SAMPLE_ALIGNMENT); + assert(bumped_usize <= SC_LARGE_MINCLASS + && PAGE_CEILING(bumped_usize) == bumped_usize); + assert(edata_size_get(edata) - bumped_usize <= sz_large_pad); + szind_t szind = sz_size2index(bumped_usize); + + edata_szind_set(edata, szind); + emap_remap(tsdn, &arena_emap_global, edata, szind, /* slab */ false); + + assert(isalloc(tsdn, ptr) == bumped_usize); + + return bumped_usize; +} + +static void +arena_dalloc_promoted_impl( + tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path, edata_t *edata) { + cassert(config_prof); + assert(opt_prof); + + size_t usize = edata_usize_get(edata); + size_t bumped_usize = arena_prof_demote(tsdn, edata, ptr); + if (config_opt_safety_checks && usize < SC_LARGE_MINCLASS) { + /* + * Currently, we only do redzoning for small sampled + * allocations. + */ + safety_check_verify_redzone(ptr, usize, bumped_usize); + } + szind_t bumped_ind = sz_size2index(bumped_usize); + if (bumped_usize >= SC_LARGE_MINCLASS && tcache != NULL + && bumped_ind < TCACHE_NBINS_MAX + && !tcache_bin_disabled( + bumped_ind, &tcache->bins[bumped_ind], tcache->tcache_slow)) { + tcache_dalloc_large( + tsdn_tsd(tsdn), tcache, ptr, bumped_ind, slow_path); + } else { + large_dalloc(tsdn, edata); + } +} + +void +arena_dalloc_promoted( + tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path) { + edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr); + arena_dalloc_promoted_impl(tsdn, ptr, tcache, slow_path, edata); +} + void arena_reset(tsd_t *tsd, arena_t *arena) { /* @@ -677,24 +681,29 @@ arena_reset(tsd_t *tsd, arena_t *arena) { for (edata_t *edata = edata_list_active_first(&arena->large); edata != NULL; edata = edata_list_active_first(&arena->large)) { - void *ptr = edata_base_get(edata); + void *ptr = edata_base_get(edata); size_t usize; malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx); emap_alloc_ctx_t alloc_ctx; - emap_alloc_ctx_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr, - &alloc_ctx); + emap_alloc_ctx_lookup( + tsd_tsdn(tsd), &arena_emap_global, ptr, &alloc_ctx); assert(alloc_ctx.szind != SC_NSIZES); if (config_stats || (config_prof && opt_prof)) { - usize = sz_index2size(alloc_ctx.szind); + usize = emap_alloc_ctx_usize_get(&alloc_ctx); assert(usize == isalloc(tsd_tsdn(tsd), ptr)); } /* Remove large allocation from prof sample set. */ if (config_prof && opt_prof) { prof_free(tsd, ptr, usize, &alloc_ctx); } - large_dalloc(tsd_tsdn(tsd), edata); + if (config_prof && opt_prof && alloc_ctx.szind < SC_NBINS) { + arena_dalloc_promoted_impl(tsd_tsdn(tsd), ptr, + /* tcache */ NULL, /* slow_path */ true, edata); + } else { + large_dalloc(tsd_tsdn(tsd), edata); + } malloc_mutex_lock(tsd_tsdn(tsd), &arena->large_mtx); } malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx); @@ -709,8 +718,8 @@ arena_reset(tsd_t *tsd, arena_t *arena) { } static void -arena_prepare_base_deletion_sync_finish(tsd_t *tsd, malloc_mutex_t **mutexes, - unsigned n_mtx) { +arena_prepare_base_deletion_sync_finish( + tsd_t *tsd, malloc_mutex_t **mutexes, unsigned n_mtx) { for (unsigned i = 0; i < n_mtx; i++) { malloc_mutex_lock(tsd_tsdn(tsd), mutexes[i]); malloc_mutex_unlock(tsd_tsdn(tsd), mutexes[i]); @@ -769,9 +778,9 @@ arena_prepare_base_deletion(tsd_t *tsd, base_t *base_to_destroy) { unsigned destroy_ind = base_ind_get(base_to_destroy); assert(destroy_ind >= manual_arena_base); - tsdn_t *tsdn = tsd_tsdn(tsd); + tsdn_t *tsdn = tsd_tsdn(tsd); malloc_mutex_t *delayed_mtx[ARENA_DESTROY_MAX_DELAYED_MTX]; - unsigned n_delayed = 0, total = narenas_total_get(); + unsigned n_delayed = 0, total = narenas_total_get(); for (unsigned i = 0; i < total; i++) { if (i == destroy_ind) { continue; @@ -781,12 +790,12 @@ arena_prepare_base_deletion(tsd_t *tsd, base_t *base_to_destroy) { continue; } pac_t *pac = &arena->pa_shard.pac; - arena_prepare_base_deletion_sync(tsd, &pac->ecache_dirty.mtx, - delayed_mtx, &n_delayed); - arena_prepare_base_deletion_sync(tsd, &pac->ecache_muzzy.mtx, - delayed_mtx, &n_delayed); - arena_prepare_base_deletion_sync(tsd, &pac->ecache_retained.mtx, - delayed_mtx, &n_delayed); + arena_prepare_base_deletion_sync( + tsd, &pac->ecache_dirty.mtx, delayed_mtx, &n_delayed); + arena_prepare_base_deletion_sync( + tsd, &pac->ecache_muzzy.mtx, delayed_mtx, &n_delayed); + arena_prepare_base_deletion_sync( + tsd, &pac->ecache_retained.mtx, delayed_mtx, &n_delayed); } arena_prepare_base_deletion_sync_finish(tsd, delayed_mtx, n_delayed); } @@ -828,17 +837,17 @@ arena_destroy(tsd_t *tsd, arena_t *arena) { } static edata_t * -arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned binshard, - const bin_info_t *bin_info) { +arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, + unsigned binshard, const bin_info_t *bin_info) { bool deferred_work_generated = false; - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); - bool guarded = san_slab_extent_decide_guard(tsdn, - arena_get_ehooks(arena)); + bool guarded = san_slab_extent_decide_guard( + tsdn, arena_get_ehooks(arena)); edata_t *slab = pa_alloc(tsdn, &arena->pa_shard, bin_info->slab_size, /* alignment */ PAGE, /* slab */ true, /* szind */ binind, - /* zero */ false, guarded, &deferred_work_generated); + /* zero */ false, guarded, &deferred_work_generated); if (deferred_work_generated) { arena_handle_deferred_work(tsdn, arena); @@ -857,84 +866,13 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned binshard return slab; } -/* - * Before attempting the _with_fresh_slab approaches below, the _no_fresh_slab - * variants (i.e. through slabcur and nonfull) must be tried first. - */ -static void -arena_bin_refill_slabcur_with_fresh_slab(tsdn_t *tsdn, arena_t *arena, - bin_t *bin, szind_t binind, edata_t *fresh_slab) { - malloc_mutex_assert_owner(tsdn, &bin->lock); - /* Only called after slabcur and nonfull both failed. */ - assert(bin->slabcur == NULL); - assert(edata_heap_first(&bin->slabs_nonfull) == NULL); - assert(fresh_slab != NULL); - - /* A new slab from arena_slab_alloc() */ - assert(edata_nfree_get(fresh_slab) == bin_infos[binind].nregs); - if (config_stats) { - bin->stats.nslabs++; - bin->stats.curslabs++; - } - bin->slabcur = fresh_slab; -} - -/* Refill slabcur and then alloc using the fresh slab */ -static void * -arena_bin_malloc_with_fresh_slab(tsdn_t *tsdn, arena_t *arena, bin_t *bin, - szind_t binind, edata_t *fresh_slab) { - malloc_mutex_assert_owner(tsdn, &bin->lock); - arena_bin_refill_slabcur_with_fresh_slab(tsdn, arena, bin, binind, - fresh_slab); - - return arena_slab_reg_alloc(bin->slabcur, &bin_infos[binind]); -} - -static bool -arena_bin_refill_slabcur_no_fresh_slab(tsdn_t *tsdn, arena_t *arena, - bin_t *bin) { - malloc_mutex_assert_owner(tsdn, &bin->lock); - /* Only called after arena_slab_reg_alloc[_batch] failed. */ - assert(bin->slabcur == NULL || edata_nfree_get(bin->slabcur) == 0); - - if (bin->slabcur != NULL) { - arena_bin_slabs_full_insert(arena, bin, bin->slabcur); - } - - /* Look for a usable slab. */ - bin->slabcur = arena_bin_slabs_nonfull_tryget(bin); - assert(bin->slabcur == NULL || edata_nfree_get(bin->slabcur) > 0); - - return (bin->slabcur == NULL); -} - -bin_t * -arena_bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind, - unsigned *binshard_p) { - unsigned binshard; - if (tsdn_null(tsdn) || tsd_arena_get(tsdn_tsd(tsdn)) == NULL) { - binshard = 0; - } else { - binshard = tsd_binshardsp_get(tsdn_tsd(tsdn))->binshard[binind]; - } - assert(binshard < bin_infos[binind].n_shards); - if (binshard_p != NULL) { - *binshard_p = binshard; - } - return arena_get_bin(arena, binind, binshard); -} - -void -arena_cache_bin_fill_small(tsdn_t *tsdn, arena_t *arena, - cache_bin_t *cache_bin, cache_bin_info_t *cache_bin_info, szind_t binind, - const unsigned nfill) { - assert(cache_bin_ncached_get_local(cache_bin, cache_bin_info) == 0); +cache_bin_sz_t +arena_ptr_array_fill_small(tsdn_t *tsdn, arena_t *arena, szind_t binind, + cache_bin_ptr_array_t *arr, const cache_bin_sz_t nfill_min, + const cache_bin_sz_t nfill_max, cache_bin_stats_t merge_stats) { + assert(nfill_min > 0 && nfill_min <= nfill_max); const bin_info_t *bin_info = &bin_infos[binind]; - - CACHE_BIN_PTR_ARRAY_DECLARE(ptrs, nfill); - cache_bin_init_ptr_array_for_fill(cache_bin, cache_bin_info, &ptrs, - nfill); /* * Bin-local resources are used first: 1) bin->slabcur, and 2) nonfull * slabs. After both are exhausted, new slabs will be allocated through @@ -961,40 +899,47 @@ arena_cache_bin_fill_small(tsdn_t *tsdn, arena_t *arena, * local exhausted, b) unlock and slab_alloc returns null, c) re-lock * and bin local fails again. */ - bool made_progress = true; - edata_t *fresh_slab = NULL; - bool alloc_and_retry = false; - unsigned filled = 0; - unsigned binshard; - bin_t *bin = arena_bin_choose(tsdn, arena, binind, &binshard); + bool made_progress = true; + edata_t *fresh_slab = NULL; + bool alloc_and_retry = false; + bool is_auto = arena_is_auto(arena); + cache_bin_sz_t filled = 0; + unsigned binshard; + bin_t *bin = bin_choose(tsdn, arena, binind, &binshard); label_refill: malloc_mutex_lock(tsdn, &bin->lock); - while (filled < nfill) { + while (filled < nfill_min) { /* Try batch-fill from slabcur first. */ edata_t *slabcur = bin->slabcur; if (slabcur != NULL && edata_nfree_get(slabcur) > 0) { - unsigned tofill = nfill - filled; - unsigned nfree = edata_nfree_get(slabcur); - unsigned cnt = tofill < nfree ? tofill : nfree; + /* + * Use up the free slots if the total filled <= nfill_max. + * Otherwise, fallback to nfill_min for a more conservative + * memory usage. + */ + unsigned cnt = edata_nfree_get(slabcur); + if (cnt + filled > nfill_max) { + cnt = nfill_min - filled; + } - arena_slab_reg_alloc_batch(slabcur, bin_info, cnt, - &ptrs.ptr[filled]); + bin_slab_reg_alloc_batch( + slabcur, bin_info, cnt, &arr->ptr[filled]); made_progress = true; filled += cnt; continue; } /* Next try refilling slabcur from nonfull slabs. */ - if (!arena_bin_refill_slabcur_no_fresh_slab(tsdn, arena, bin)) { + if (!bin_refill_slabcur_no_fresh_slab(tsdn, is_auto, bin)) { assert(bin->slabcur != NULL); continue; } /* Then see if a new slab was reserved already. */ if (fresh_slab != NULL) { - arena_bin_refill_slabcur_with_fresh_slab(tsdn, arena, - bin, binind, fresh_slab); + bin_refill_slabcur_with_fresh_slab( + tsdn, bin, binind, fresh_slab); assert(bin->slabcur != NULL); fresh_slab = NULL; continue; @@ -1014,32 +959,32 @@ label_refill: assert(fresh_slab == NULL); assert(!alloc_and_retry); break; - } /* while (filled < nfill) loop. */ + } /* while (filled < nfill_min) loop. */ if (config_stats && !alloc_and_retry) { bin->stats.nmalloc += filled; - bin->stats.nrequests += cache_bin->tstats.nrequests; + bin->stats.nrequests += merge_stats.nrequests; bin->stats.curregs += filled; bin->stats.nfills++; - cache_bin->tstats.nrequests = 0; } malloc_mutex_unlock(tsdn, &bin->lock); if (alloc_and_retry) { assert(fresh_slab == NULL); - assert(filled < nfill); + assert(filled < nfill_min); assert(made_progress); - fresh_slab = arena_slab_alloc(tsdn, arena, binind, binshard, - bin_info); + fresh_slab = arena_slab_alloc( + tsdn, arena, binind, binshard, bin_info); /* fresh_slab NULL case handled in the for loop. */ alloc_and_retry = false; made_progress = false; goto label_refill; } - assert(filled == nfill || (fresh_slab == NULL && !made_progress)); + assert((filled >= nfill_min && filled <= nfill_max) + || (fresh_slab == NULL && !made_progress)); /* Release if allocated but not used. */ if (fresh_slab != NULL) { @@ -1048,8 +993,8 @@ label_refill: fresh_slab = NULL; } - cache_bin_finish_fill(cache_bin, cache_bin_info, &ptrs, filled); arena_decay_tick(tsdn, arena); + return filled; } size_t @@ -1057,22 +1002,24 @@ arena_fill_small_fresh(tsdn_t *tsdn, arena_t *arena, szind_t binind, void **ptrs, size_t nfill, bool zero) { assert(binind < SC_NBINS); const bin_info_t *bin_info = &bin_infos[binind]; - const size_t nregs = bin_info->nregs; + const size_t nregs = bin_info->nregs; assert(nregs > 0); const size_t usize = bin_info->reg_size; const bool manual_arena = !arena_is_auto(arena); - unsigned binshard; - bin_t *bin = arena_bin_choose(tsdn, arena, binind, &binshard); + unsigned binshard; + bin_t *bin = bin_choose(tsdn, arena, binind, &binshard); - size_t nslab = 0; - size_t filled = 0; - edata_t *slab = NULL; + size_t nslab = 0; + size_t filled = 0; + edata_t *slab = NULL; edata_list_active_t fulls; edata_list_active_init(&fulls); - while (filled < nfill && (slab = arena_slab_alloc(tsdn, arena, binind, - binshard, bin_info)) != NULL) { + while (filled < nfill + && (slab = arena_slab_alloc( + tsdn, arena, binind, binshard, bin_info)) + != NULL) { assert((size_t)edata_nfree_get(slab) == nregs); ++nslab; size_t batch = nfill - filled; @@ -1080,8 +1027,8 @@ arena_fill_small_fresh(tsdn_t *tsdn, arena_t *arena, szind_t binind, batch = nregs; } assert(batch > 0); - arena_slab_reg_alloc_batch(slab, bin_info, (unsigned)batch, - &ptrs[filled]); + bin_slab_reg_alloc_batch( + slab, bin_info, (unsigned)batch, &ptrs[filled]); assert(edata_addr_get(slab) == ptrs[filled]); if (zero) { memset(ptrs[filled], 0, batch * usize); @@ -1101,7 +1048,7 @@ arena_fill_small_fresh(tsdn_t *tsdn, arena_t *arena, szind_t binind, * iff slab != NULL. */ if (slab != NULL) { - arena_bin_lower_slab(tsdn, arena, slab, bin); + bin_lower_slab(tsdn, !manual_arena, slab, bin); } if (manual_arena) { edata_list_active_concat(&bin->slabs_full, &fulls); @@ -1120,52 +1067,35 @@ arena_fill_small_fresh(tsdn_t *tsdn, arena_t *arena, szind_t binind, return filled; } -/* - * Without allocating a new slab, try arena_slab_reg_alloc() and re-fill - * bin->slabcur if necessary. - */ -static void * -arena_bin_malloc_no_fresh_slab(tsdn_t *tsdn, arena_t *arena, bin_t *bin, - szind_t binind) { - malloc_mutex_assert_owner(tsdn, &bin->lock); - if (bin->slabcur == NULL || edata_nfree_get(bin->slabcur) == 0) { - if (arena_bin_refill_slabcur_no_fresh_slab(tsdn, arena, bin)) { - return NULL; - } - } - - assert(bin->slabcur != NULL && edata_nfree_get(bin->slabcur) > 0); - return arena_slab_reg_alloc(bin->slabcur, &bin_infos[binind]); -} - static void * arena_malloc_small(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) { assert(binind < SC_NBINS); const bin_info_t *bin_info = &bin_infos[binind]; - size_t usize = sz_index2size(binind); - unsigned binshard; - bin_t *bin = arena_bin_choose(tsdn, arena, binind, &binshard); + size_t usize = sz_index2size(binind); + bool is_auto = arena_is_auto(arena); + unsigned binshard; + bin_t *bin = bin_choose(tsdn, arena, binind, &binshard); malloc_mutex_lock(tsdn, &bin->lock); edata_t *fresh_slab = NULL; - void *ret = arena_bin_malloc_no_fresh_slab(tsdn, arena, bin, binind); + void *ret = bin_malloc_no_fresh_slab(tsdn, is_auto, bin, binind); if (ret == NULL) { malloc_mutex_unlock(tsdn, &bin->lock); /******************************/ - fresh_slab = arena_slab_alloc(tsdn, arena, binind, binshard, - bin_info); + fresh_slab = arena_slab_alloc( + tsdn, arena, binind, binshard, bin_info); /********************************/ malloc_mutex_lock(tsdn, &bin->lock); /* Retry since the lock was dropped. */ - ret = arena_bin_malloc_no_fresh_slab(tsdn, arena, bin, binind); + ret = bin_malloc_no_fresh_slab(tsdn, is_auto, bin, binind); if (ret == NULL) { if (fresh_slab == NULL) { /* OOM */ malloc_mutex_unlock(tsdn, &bin->lock); return NULL; } - ret = arena_bin_malloc_with_fresh_slab(tsdn, arena, bin, - binind, fresh_slab); + ret = bin_malloc_with_fresh_slab( + tsdn, bin, binind, fresh_slab); fresh_slab = NULL; } } @@ -1189,7 +1119,7 @@ arena_malloc_small(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) { void * arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, - bool zero) { + bool zero, bool slab) { assert(!tsdn_null(tsdn) || arena != NULL); if (likely(!tsdn_null(tsdn))) { @@ -1199,18 +1129,19 @@ arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, return NULL; } - if (likely(size <= SC_SMALL_MAXCLASS)) { + if (likely(slab)) { + assert(sz_can_use_slab(size)); return arena_malloc_small(tsdn, arena, ind, zero); + } else { + return large_malloc(tsdn, arena, sz_s2u(size), zero); } - return large_malloc(tsdn, arena, sz_index2size(ind), zero); } void * arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, - bool zero, tcache_t *tcache) { - void *ret; - - if (usize <= SC_SMALL_MAXCLASS) { + bool zero, bool slab, tcache_t *tcache) { + if (slab) { + assert(sz_can_use_slab(usize)); /* Small; alignment doesn't require special slab placement. */ /* usize should be a result of sz_sa2u() */ @@ -1221,161 +1152,30 @@ arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, */ assert(alignment <= PAGE); - ret = arena_malloc(tsdn, arena, usize, sz_size2index(usize), - zero, tcache, true); + return arena_malloc(tsdn, arena, usize, sz_size2index(usize), + zero, slab, tcache, true); } else { if (likely(alignment <= CACHELINE)) { - ret = large_malloc(tsdn, arena, usize, zero); + return large_malloc(tsdn, arena, usize, zero); } else { - ret = large_palloc(tsdn, arena, usize, alignment, zero); + return large_palloc( + tsdn, arena, usize, alignment, zero); } } - return ret; -} - -void -arena_prof_promote(tsdn_t *tsdn, void *ptr, size_t usize) { - cassert(config_prof); - assert(ptr != NULL); - assert(isalloc(tsdn, ptr) == SC_LARGE_MINCLASS); - assert(usize <= SC_SMALL_MAXCLASS); - - if (config_opt_safety_checks) { - safety_check_set_redzone(ptr, usize, SC_LARGE_MINCLASS); - } - - edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr); - - szind_t szind = sz_size2index(usize); - edata_szind_set(edata, szind); - emap_remap(tsdn, &arena_emap_global, edata, szind, /* slab */ false); - - assert(isalloc(tsdn, ptr) == usize); -} - -static size_t -arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr) { - cassert(config_prof); - assert(ptr != NULL); - - edata_szind_set(edata, SC_NBINS); - emap_remap(tsdn, &arena_emap_global, edata, SC_NBINS, /* slab */ false); - - assert(isalloc(tsdn, ptr) == SC_LARGE_MINCLASS); - - return SC_LARGE_MINCLASS; -} - -void -arena_dalloc_promoted(tsdn_t *tsdn, void *ptr, tcache_t *tcache, - bool slow_path) { - cassert(config_prof); - assert(opt_prof); - - edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr); - size_t usize = edata_usize_get(edata); - size_t bumped_usize = arena_prof_demote(tsdn, edata, ptr); - if (config_opt_safety_checks && usize < SC_LARGE_MINCLASS) { - /* - * Currently, we only do redzoning for small sampled - * allocations. - */ - assert(bumped_usize == SC_LARGE_MINCLASS); - safety_check_verify_redzone(ptr, usize, bumped_usize); - } - if (bumped_usize <= tcache_maxclass && tcache != NULL) { - tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, - sz_size2index(bumped_usize), slow_path); - } else { - large_dalloc(tsdn, edata); - } -} - -static void -arena_dissociate_bin_slab(arena_t *arena, edata_t *slab, bin_t *bin) { - /* Dissociate slab from bin. */ - if (slab == bin->slabcur) { - bin->slabcur = NULL; - } else { - szind_t binind = edata_szind_get(slab); - const bin_info_t *bin_info = &bin_infos[binind]; - - /* - * The following block's conditional is necessary because if the - * slab only contains one region, then it never gets inserted - * into the non-full slabs heap. - */ - if (bin_info->nregs == 1) { - arena_bin_slabs_full_remove(arena, bin, slab); - } else { - arena_bin_slabs_nonfull_remove(bin, slab); - } - } -} - -static void -arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, edata_t *slab, - bin_t *bin) { - assert(edata_nfree_get(slab) > 0); - - /* - * Make sure that if bin->slabcur is non-NULL, it refers to the - * oldest/lowest non-full slab. It is okay to NULL slabcur out rather - * than proactively keeping it pointing at the oldest/lowest non-full - * slab. - */ - if (bin->slabcur != NULL && edata_snad_comp(bin->slabcur, slab) > 0) { - /* Switch slabcur. */ - if (edata_nfree_get(bin->slabcur) > 0) { - arena_bin_slabs_nonfull_insert(bin, bin->slabcur); - } else { - arena_bin_slabs_full_insert(arena, bin, bin->slabcur); - } - bin->slabcur = slab; - if (config_stats) { - bin->stats.reslabs++; - } - } else { - arena_bin_slabs_nonfull_insert(bin, slab); - } -} - -static void -arena_dalloc_bin_slab_prepare(tsdn_t *tsdn, edata_t *slab, bin_t *bin) { - malloc_mutex_assert_owner(tsdn, &bin->lock); - - assert(slab != bin->slabcur); - if (config_stats) { - bin->stats.curslabs--; - } -} - -void -arena_dalloc_bin_locked_handle_newly_empty(tsdn_t *tsdn, arena_t *arena, - edata_t *slab, bin_t *bin) { - arena_dissociate_bin_slab(arena, slab, bin); - arena_dalloc_bin_slab_prepare(tsdn, slab, bin); -} - -void -arena_dalloc_bin_locked_handle_newly_nonempty(tsdn_t *tsdn, arena_t *arena, - edata_t *slab, bin_t *bin) { - arena_bin_slabs_full_remove(arena, bin, slab); - arena_bin_lower_slab(tsdn, arena, slab, bin); } static void arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, edata_t *edata, void *ptr) { - szind_t binind = edata_szind_get(edata); + szind_t binind = edata_szind_get(edata); unsigned binshard = edata_binshard_get(edata); - bin_t *bin = arena_get_bin(arena, binind, binshard); + bin_t *bin = arena_get_bin(arena, binind, binshard); malloc_mutex_lock(tsdn, &bin->lock); - arena_dalloc_bin_locked_info_t info; - arena_dalloc_bin_locked_begin(&info, binind); - bool ret = arena_dalloc_bin_locked_step(tsdn, arena, bin, - &info, binind, edata, ptr); - arena_dalloc_bin_locked_finish(tsdn, arena, bin, &info); + bin_dalloc_locked_info_t info; + bin_dalloc_locked_begin(&info, binind); + bool ret = bin_dalloc_locked_step( + tsdn, arena_is_auto(arena), bin, &info, binind, edata, ptr); + bin_dalloc_locked_finish(tsdn, bin, &info); malloc_mutex_unlock(tsdn, &bin->lock); if (ret) { @@ -1392,6 +1192,358 @@ arena_dalloc_small(tsdn_t *tsdn, void *ptr) { arena_decay_tick(tsdn, arena); } +static const void * +arena_ptr_array_flush_ptr_getter(void *arr_ctx, size_t ind) { + cache_bin_ptr_array_t *arr = (cache_bin_ptr_array_t *)arr_ctx; + return arr->ptr[ind]; +} + +static void +arena_ptr_array_flush_metadata_visitor( + void *szind_sum_ctx, emap_full_alloc_ctx_t *alloc_ctx) { + size_t *szind_sum = (size_t *)szind_sum_ctx; + *szind_sum -= alloc_ctx->szind; + util_prefetch_write_range(alloc_ctx->edata, sizeof(edata_t)); +} + +JEMALLOC_NOINLINE static void +arena_ptr_array_flush_size_check_fail(cache_bin_ptr_array_t *arr, szind_t szind, + size_t nptrs, emap_batch_lookup_result_t *edatas) { + bool found_mismatch = false; + for (size_t i = 0; i < nptrs; i++) { + szind_t true_szind = edata_szind_get(edatas[i].edata); + if (true_szind != szind) { + found_mismatch = true; + safety_check_fail_sized_dealloc( + /* current_dealloc */ false, + /* ptr */ arena_ptr_array_flush_ptr_getter(arr, i), + /* true_size */ sz_index2size(true_szind), + /* input_size */ sz_index2size(szind)); + } + } + assert(found_mismatch); +} + +JEMALLOC_ALWAYS_INLINE void +arena_ptr_array_flush_impl_small(tsdn_t *tsdn, szind_t binind, + cache_bin_ptr_array_t *arr, emap_batch_lookup_result_t *item_edata, + cache_bin_sz_t nflush, arena_t *stats_arena, + cache_bin_stats_t **merge_stats) { + /* + * The slabs where we freed the last remaining object in the slab (and + * so need to free the slab itself). + * Used only if small == true. + */ + unsigned dalloc_count = 0; + VARIABLE_ARRAY(edata_t *, dalloc_slabs, nflush + 1); + /* + * We're about to grab a bunch of locks. If one of them happens to be + * the one guarding the arena-level stats counters we flush our + * thread-local ones to, we do so under one critical section. + */ + /* + * We maintain the invariant that all edatas yet to be flushed are + * contained in the half-open range [flush_start, flush_end). We'll + * repeatedly partition the array so that the unflushed items are at the + * end. + */ + unsigned flush_start = 0; + + while (flush_start < nflush) { + /* + * After our partitioning step, all objects to flush will be in + * the half-open range [prev_flush_start, flush_start), and + * flush_start will be updated to correspond to the next loop + * iteration. + */ + unsigned prev_flush_start = flush_start; + + edata_t *cur_edata = item_edata[flush_start].edata; + unsigned cur_arena_ind = edata_arena_ind_get(cur_edata); + arena_t *cur_arena = arena_get(tsdn, cur_arena_ind, false); + + unsigned cur_binshard = edata_binshard_get(cur_edata); + bin_t *cur_bin = arena_get_bin(cur_arena, binind, cur_binshard); + assert(cur_binshard < bin_infos[binind].n_shards); + /* + * Start off the partition; item_edata[i] always matches itself + * of course. + */ + flush_start++; + for (unsigned i = flush_start; i < nflush; i++) { + void *ptr = arr->ptr[i]; + edata_t *edata = item_edata[i].edata; + assert(ptr != NULL && edata != NULL); + assert( + (uintptr_t)ptr >= (uintptr_t)edata_addr_get(edata)); + assert( + (uintptr_t)ptr < (uintptr_t)edata_past_get(edata)); + if (edata_arena_ind_get(edata) == cur_arena_ind + && edata_binshard_get(edata) == cur_binshard) { + /* Swap the edatas. */ + emap_batch_lookup_result_t temp_edata = + item_edata[flush_start]; + item_edata[flush_start] = item_edata[i]; + item_edata[i] = temp_edata; + /* Swap the pointers */ + void *temp_ptr = arr->ptr[flush_start]; + arr->ptr[flush_start] = arr->ptr[i]; + arr->ptr[i] = temp_ptr; + flush_start++; + } + } + /* Make sure we implemented partitioning correctly. */ + if (config_debug) { + for (unsigned i = prev_flush_start; i < flush_start; + i++) { + edata_t *edata = item_edata[i].edata; + unsigned arena_ind = edata_arena_ind_get(edata); + assert(arena_ind == cur_arena_ind); + unsigned binshard = edata_binshard_get(edata); + assert(binshard == cur_binshard); + } + for (unsigned i = flush_start; i < nflush; i++) { + edata_t *edata = item_edata[i].edata; + assert( + edata_arena_ind_get(edata) != cur_arena_ind + || edata_binshard_get(edata) + != cur_binshard); + } + } + + /* Actually do the flushing. */ + malloc_mutex_lock(tsdn, &cur_bin->lock); + + /* + * Flush stats first, if that was the right lock. Note that we + * don't actually have to flush stats into the current thread's + * binshard. Flushing into any binshard in the same arena is + * enough; we don't expose stats on per-binshard basis (just + * per-bin). + */ + if (config_stats && stats_arena == cur_arena + && *merge_stats != NULL) { + cur_bin->stats.nflushes++; + cur_bin->stats.nrequests += (*merge_stats)->nrequests; + *merge_stats = NULL; + } + + /* Next flush objects. */ + /* Init only to avoid used-uninitialized warning. */ + bin_dalloc_locked_info_t dalloc_bin_info = {0}; + bin_dalloc_locked_begin(&dalloc_bin_info, binind); + for (unsigned i = prev_flush_start; i < flush_start; i++) { + void *ptr = arr->ptr[i]; + edata_t *edata = item_edata[i].edata; + if (bin_dalloc_locked_step(tsdn, + arena_is_auto(cur_arena), + cur_bin, &dalloc_bin_info, binind, edata, + ptr)) { + dalloc_slabs[dalloc_count] = edata; + dalloc_count++; + } + } + + bin_dalloc_locked_finish( + tsdn, cur_bin, &dalloc_bin_info); + malloc_mutex_unlock(tsdn, &cur_bin->lock); + + arena_decay_ticks( + tsdn, cur_arena, flush_start - prev_flush_start); + } + + /* Handle all deferred slab dalloc. */ + for (unsigned i = 0; i < dalloc_count; i++) { + edata_t *slab = dalloc_slabs[i]; + arena_slab_dalloc(tsdn, arena_get_from_edata(slab), slab); + } + + if (config_stats && *merge_stats != NULL) { + /* + * The flush loop didn't happen to flush to this + * thread's arena, so the stats didn't get merged. + * Manually do so now. + */ + bin_t *bin = bin_choose(tsdn, stats_arena, binind, NULL); + malloc_mutex_lock(tsdn, &bin->lock); + bin->stats.nflushes++; + bin->stats.nrequests += (*merge_stats)->nrequests; + *merge_stats = NULL; + malloc_mutex_unlock(tsdn, &bin->lock); + } +} + +JEMALLOC_ALWAYS_INLINE void +arena_ptr_array_flush_impl_large(tsdn_t *tsdn, szind_t binind, + cache_bin_ptr_array_t *arr, emap_batch_lookup_result_t *item_edata, + cache_bin_sz_t nflush, arena_t *stats_arena, + cache_bin_stats_t **merge_stats) { + /* + * We're about to grab a bunch of locks. If one of them happens to be + * the one guarding the arena-level stats counters we flush our + * thread-local ones to, we do so under one critical section. + */ + while (nflush > 0) { + /* Lock the arena, or bin, associated with the first object. */ + edata_t *edata = item_edata[0].edata; + unsigned cur_arena_ind = edata_arena_ind_get(edata); + arena_t *cur_arena = arena_get(tsdn, cur_arena_ind, false); + + if (!arena_is_auto(cur_arena)) { + malloc_mutex_lock(tsdn, &cur_arena->large_mtx); + } + + /* + * If we acquired the right lock and have some stats to flush, + * flush them. + */ + if (config_stats && stats_arena == cur_arena + && *merge_stats != NULL) { + arena_stats_large_flush_nrequests_add(tsdn, + &stats_arena->stats, binind, + (*merge_stats)->nrequests); + *merge_stats = NULL; + } + + /* + * Large allocations need special prep done. Afterwards, we can + * drop the large lock. + */ + for (unsigned i = 0; i < nflush; i++) { + void *ptr = arr->ptr[i]; + edata = item_edata[i].edata; + assert(ptr != NULL && edata != NULL); + + if (edata_arena_ind_get(edata) == cur_arena_ind) { + large_dalloc_prep_locked(tsdn, edata); + } + } + if (!arena_is_auto(cur_arena)) { + malloc_mutex_unlock(tsdn, &cur_arena->large_mtx); + } + + /* Deallocate whatever we can. */ + unsigned ndeferred = 0; + for (unsigned i = 0; i < nflush; i++) { + void *ptr = arr->ptr[i]; + edata = item_edata[i].edata; + assert(ptr != NULL && edata != NULL); + if (edata_arena_ind_get(edata) != cur_arena_ind) { + /* + * The object was allocated either via a + * different arena, or a different bin in this + * arena. Either way, stash the object so that + * it can be handled in a future pass. + */ + arr->ptr[ndeferred] = ptr; + item_edata[ndeferred].edata = edata; + ndeferred++; + continue; + } + if (large_dalloc_safety_checks( + edata, ptr, sz_index2size(binind))) { + /* See the comment in isfree. */ + continue; + } + large_dalloc_finish(tsdn, edata); + } + arena_decay_ticks(tsdn, cur_arena, nflush - ndeferred); + nflush = ndeferred; + } + + if (config_stats && *merge_stats != NULL) { + arena_stats_large_flush_nrequests_add(tsdn, &stats_arena->stats, + binind, (*merge_stats)->nrequests); + *merge_stats = NULL; + } +} + +JEMALLOC_ALWAYS_INLINE void +arena_ptr_array_flush_impl(tsd_t *tsd, szind_t binind, + cache_bin_ptr_array_t *arr, unsigned nflush, bool small, + arena_t *stats_arena, cache_bin_stats_t **merge_stats) { + /* + * A couple lookup calls take tsdn; declare it once for convenience + * instead of calling tsd_tsdn(tsd) all the time. + */ + tsdn_t *tsdn = tsd_tsdn(tsd); + /* + * Variable length array must have > 0 length; the last element is never + * touched (it's just included to satisfy the no-zero-length rule). + */ + VARIABLE_ARRAY(emap_batch_lookup_result_t, item_edata, nflush + 1); + /* + * This gets compiled away when config_opt_safety_checks is false. + * Checks for sized deallocation bugs, failing early rather than + * corrupting metadata. + */ + size_t szind_sum = binind * nflush; + emap_edata_lookup_batch(tsd, &arena_emap_global, nflush, + &arena_ptr_array_flush_ptr_getter, (void *)arr, + &arena_ptr_array_flush_metadata_visitor, (void *)&szind_sum, + item_edata); + if (config_opt_safety_checks && unlikely(szind_sum != 0)) { + arena_ptr_array_flush_size_check_fail( + arr, binind, nflush, item_edata); + } + + /* + * The small/large flush logic is very similar; you might conclude that + * it's a good opportunity to share code. We've tried this, and by and + * large found this to obscure more than it helps; there are so many + * fiddly bits around things like stats handling, precisely when and + * which mutexes are acquired, etc., that almost all code ends up being + * gated behind 'if (small) { ... } else { ... }'. Even though the + * '...' is morally equivalent, the code itself needs slight tweaks. + */ + if (small) { + return arena_ptr_array_flush_impl_small(tsdn, binind, arr, + item_edata, nflush, stats_arena, merge_stats); + } else { + return arena_ptr_array_flush_impl_large(tsdn, binind, arr, + item_edata, nflush, stats_arena, merge_stats); + } +} + +/* + * In practice, pointers are flushed back to their original allocation arenas, + * so multiple arenas may be involved here. The input stats_arena simply + * indicates where the cache stats should be merged into. + */ +void +arena_ptr_array_flush(tsd_t *tsd, szind_t binind, cache_bin_ptr_array_t *arr, + unsigned nflush, bool small, arena_t *stats_arena, + cache_bin_stats_t merge_stats) { + assert(arr != NULL && arr->ptr != NULL); + /* + * The input cache bin stats represent a snapshot taken when the pointer + * array is set up, and will be merged into the next-level bin stats. + * The original bin stats will be reset by the caller itself. + * This separation ensures that each layer operates independently and + * does not modify another layer's data directly. + */ + cache_bin_stats_t *stats = &merge_stats; + unsigned nflush_batch, nflushed = 0; + cache_bin_ptr_array_t ptrs_batch; + do { + nflush_batch = nflush - nflushed; + if (nflush_batch > CACHE_BIN_NFLUSH_BATCH_MAX) { + nflush_batch = CACHE_BIN_NFLUSH_BATCH_MAX; + } + assert(nflush_batch <= CACHE_BIN_NFLUSH_BATCH_MAX); + (&ptrs_batch)->n = (cache_bin_sz_t)nflush_batch; + (&ptrs_batch)->ptr = arr->ptr + nflushed; + arena_ptr_array_flush_impl(tsd, binind, &ptrs_batch, + nflush_batch, small, stats_arena, &stats); + nflushed += nflush_batch; + } while (nflushed < nflush); + assert(nflush == nflushed); + assert((arr->ptr + nflush) == ((&ptrs_batch)->ptr + nflush_batch)); + if (config_stats) { + assert(stats == NULL); + } +} + bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, bool zero, size_t *newsize) { @@ -1407,16 +1559,15 @@ arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t usize_min = sz_s2u(size); size_t usize_max = sz_s2u(size + extra); - if (likely(oldsize <= SC_SMALL_MAXCLASS && usize_min - <= SC_SMALL_MAXCLASS)) { + if (likely(oldsize <= SC_SMALL_MAXCLASS + && usize_min <= SC_SMALL_MAXCLASS)) { /* * Avoid moving the allocation if the size class can be left the * same. */ - assert(bin_infos[sz_size2index(oldsize)].reg_size == - oldsize); + assert(bin_infos[sz_size2index(oldsize)].reg_size == oldsize); if ((usize_max > SC_SMALL_MAXCLASS - || sz_size2index(usize_max) != sz_size2index(oldsize)) + || sz_size2index(usize_max) != sz_size2index(oldsize)) && (size > oldsize || usize_max < oldsize)) { ret = true; goto done; @@ -1427,8 +1578,8 @@ arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, ret = false; } else if (oldsize >= SC_LARGE_MINCLASS && usize_max >= SC_LARGE_MINCLASS) { - ret = large_ralloc_no_move(tsdn, edata, usize_min, usize_max, - zero); + ret = large_ralloc_no_move( + tsdn, edata, usize_min, usize_max, zero); } else { ret = true; } @@ -1441,61 +1592,64 @@ done: static void * arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, - size_t alignment, bool zero, tcache_t *tcache) { + size_t alignment, bool zero, bool slab, tcache_t *tcache) { if (alignment == 0) { return arena_malloc(tsdn, arena, usize, sz_size2index(usize), - zero, tcache, true); + zero, slab, tcache, true); } usize = sz_sa2u(usize, alignment); if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { return NULL; } - return ipalloct(tsdn, usize, alignment, zero, tcache, arena); + return ipalloct_explicit_slab( + tsdn, usize, alignment, zero, slab, tcache, arena); } void * arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, - size_t size, size_t alignment, bool zero, tcache_t *tcache, + size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache, hook_ralloc_args_t *hook_args) { size_t usize = alignment == 0 ? sz_s2u(size) : sz_sa2u(size, alignment); if (unlikely(usize == 0 || size > SC_LARGE_MAXCLASS)) { return NULL; } - if (likely(usize <= SC_SMALL_MAXCLASS)) { + if (likely(slab)) { + assert(sz_can_use_slab(usize)); /* Try to avoid moving the allocation. */ UNUSED size_t newsize; - if (!arena_ralloc_no_move(tsdn, ptr, oldsize, usize, 0, zero, - &newsize)) { + if (!arena_ralloc_no_move( + tsdn, ptr, oldsize, usize, 0, zero, &newsize)) { hook_invoke_expand(hook_args->is_realloc - ? hook_expand_realloc : hook_expand_rallocx, + ? hook_expand_realloc + : hook_expand_rallocx, ptr, oldsize, usize, (uintptr_t)ptr, hook_args->args); return ptr; } } - if (oldsize >= SC_LARGE_MINCLASS - && usize >= SC_LARGE_MINCLASS) { - return large_ralloc(tsdn, arena, ptr, usize, - alignment, zero, tcache, hook_args); + if (oldsize >= SC_LARGE_MINCLASS && usize >= SC_LARGE_MINCLASS) { + return large_ralloc(tsdn, arena, ptr, usize, alignment, zero, + tcache, hook_args); } /* * size and oldsize are different enough that we need to move the * object. In that case, fall back to allocating new space and copying. */ - void *ret = arena_ralloc_move_helper(tsdn, arena, usize, alignment, - zero, tcache); + void *ret = arena_ralloc_move_helper( + tsdn, arena, usize, alignment, zero, slab, tcache); if (ret == NULL) { return NULL; } - hook_invoke_alloc(hook_args->is_realloc - ? hook_alloc_realloc : hook_alloc_rallocx, ret, (uintptr_t)ret, - hook_args->args); - hook_invoke_dalloc(hook_args->is_realloc - ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args); + hook_invoke_alloc( + hook_args->is_realloc ? hook_alloc_realloc : hook_alloc_rallocx, + ret, (uintptr_t)ret, hook_args->args); + hook_invoke_dalloc( + hook_args->is_realloc ? hook_dalloc_realloc : hook_dalloc_rallocx, + ptr, hook_args->args); /* * Junk/zero-filling were already done by @@ -1513,8 +1667,8 @@ arena_get_ehooks(arena_t *arena) { } extent_hooks_t * -arena_set_extent_hooks(tsd_t *tsd, arena_t *arena, - extent_hooks_t *extent_hooks) { +arena_set_extent_hooks( + tsd_t *tsd, arena_t *arena, extent_hooks_t *extent_hooks) { background_thread_info_t *info; if (have_background_thread) { info = arena_background_thread_info_get(arena); @@ -1544,6 +1698,22 @@ arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) { return false; } +void +arena_name_get(arena_t *arena, char *name) { + char *end = (char *)memchr((void *)arena->name, '\0', ARENA_NAME_LEN); + assert(end != NULL); + size_t len = (uintptr_t)end - (uintptr_t)arena->name + 1; + assert(len > 0 && len <= ARENA_NAME_LEN); + + strncpy(name, arena->name, len); +} + +void +arena_name_set(arena_t *arena, const char *name) { + strncpy(arena->name, name, ARENA_NAME_LEN); + arena->name[ARENA_NAME_LEN - 1] = '\0'; +} + ssize_t arena_dirty_decay_ms_default_get(void) { return atomic_load_zd(&dirty_decay_ms_default, ATOMIC_RELAXED); @@ -1573,11 +1743,11 @@ arena_muzzy_decay_ms_default_set(ssize_t decay_ms) { } bool -arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena, size_t *old_limit, - size_t *new_limit) { +arena_retain_grow_limit_get_set( + tsd_t *tsd, arena_t *arena, size_t *old_limit, size_t *new_limit) { assert(opt_retain); - return pac_retain_grow_limit_get_set(tsd_tsdn(tsd), - &arena->pa_shard.pac, old_limit, new_limit); + return pac_retain_grow_limit_get_set( + tsd_tsdn(tsd), &arena->pa_shard.pac, old_limit, new_limit); } unsigned @@ -1598,7 +1768,7 @@ arena_nthreads_dec(arena_t *arena, bool internal) { arena_t * arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) { arena_t *arena; - base_t *base; + base_t *base; unsigned i; if (ind == 0) { @@ -1611,12 +1781,15 @@ arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) { } } - size_t arena_size = sizeof(arena_t) + sizeof(bin_t) * nbins_total; + size_t arena_size = ALIGNMENT_CEILING(sizeof(arena_t), CACHELINE) + + sizeof(bin_t) * nbins_total; arena = (arena_t *)base_alloc(tsdn, base, arena_size, CACHELINE); if (arena == NULL) { goto label_error; } - + JEMALLOC_SUPPRESS_WARN_ON_USAGE( + assert((uintptr_t)&arena->all_bins[nbins_total - 1] + sizeof(bin_t) + <= (uintptr_t)arena + arena_size);) atomic_store_u(&arena->nthreads[0], 0, ATOMIC_RELAXED); atomic_store_u(&arena->nthreads[1], 0, ATOMIC_RELAXED); arena->last_thd = NULL; @@ -1629,34 +1802,35 @@ arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) { ql_new(&arena->tcache_ql); ql_new(&arena->cache_bin_array_descriptor_ql); if (malloc_mutex_init(&arena->tcache_ql_mtx, "tcache_ql", - WITNESS_RANK_TCACHE_QL, malloc_mutex_rank_exclusive)) { + WITNESS_RANK_TCACHE_QL, malloc_mutex_rank_exclusive)) { goto label_error; } } - atomic_store_u(&arena->dss_prec, (unsigned)extent_dss_prec_get(), - ATOMIC_RELAXED); + atomic_store_u( + &arena->dss_prec, (unsigned)extent_dss_prec_get(), ATOMIC_RELAXED); edata_list_active_init(&arena->large); if (malloc_mutex_init(&arena->large_mtx, "arena_large", - WITNESS_RANK_ARENA_LARGE, malloc_mutex_rank_exclusive)) { + WITNESS_RANK_ARENA_LARGE, malloc_mutex_rank_exclusive)) { goto label_error; } nstime_t cur_time; nstime_init_update(&cur_time); if (pa_shard_init(tsdn, &arena->pa_shard, &arena_pa_central_global, - &arena_emap_global, base, ind, &arena->stats.pa_shard_stats, - LOCKEDINT_MTX(arena->stats.mtx), &cur_time, oversize_threshold, - arena_dirty_decay_ms_default_get(), - arena_muzzy_decay_ms_default_get())) { + &arena_emap_global, base, ind, &arena->stats.pa_shard_stats, + LOCKEDINT_MTX(arena->stats.mtx), &cur_time, oversize_threshold, + arena_dirty_decay_ms_default_get(), + arena_muzzy_decay_ms_default_get())) { goto label_error; } /* Initialize bins. */ atomic_store_u(&arena->binshard_next, 0, ATOMIC_RELEASE); for (i = 0; i < nbins_total; i++) { - bool err = bin_init(&arena->bins[i]); + JEMALLOC_SUPPRESS_WARN_ON_USAGE( + bool err = bin_init(&arena->all_bins[i]);) if (err) { goto label_error; } @@ -1667,20 +1841,25 @@ arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) { arena_set(ind, arena); arena->ind = ind; + /* Init the name. */ + malloc_snprintf(arena->name, sizeof(arena->name), "%s_%u", + arena_is_auto(arena) ? "auto" : "manual", arena->ind); + arena->name[ARENA_NAME_LEN - 1] = '\0'; + nstime_init_update(&arena->create_time); /* * We turn on the HPA if set to. There are two exceptions: * - Custom extent hooks (we should only return memory allocated from * them in that case). - * - Arena 0 initialization. In this case, we're mid-bootstrapping, and - * so arena_hpa_global is not yet initialized. + * - Arena 0 initialization. In this case, we're mid-bootstrapping, + * and so background_thread_enabled is not yet initialized. */ if (opt_hpa && ehooks_are_default(base_ehooks_get(base)) && ind != 0) { hpa_shard_opts_t hpa_shard_opts = opt_hpa_opts; hpa_shard_opts.deferral_allowed = background_thread_enabled(); - if (pa_shard_enable_hpa(tsdn, &arena->pa_shard, - &hpa_shard_opts, &opt_hpa_sec_opts)) { + if (pa_shard_enable_hpa(tsdn, &arena->pa_shard, &hpa_shard_opts, + &opt_hpa_sec_opts)) { goto label_error; } } @@ -1707,6 +1886,42 @@ label_error: return NULL; } +static arena_t * +arena_create_huge_arena(tsd_t *tsd, unsigned ind) { + assert(ind != 0); + + arena_t *huge_arena = arena_get(tsd_tsdn(tsd), ind, true); + if (huge_arena == NULL) { + return NULL; + } + + char *huge_arena_name = "auto_oversize"; + strncpy(huge_arena->name, huge_arena_name, ARENA_NAME_LEN); + huge_arena->name[ARENA_NAME_LEN - 1] = '\0'; + + /* + * Purge eagerly for huge allocations, because: 1) number of huge + * allocations is usually small, which means ticker based decay is not + * reliable; and 2) less immediate reuse is expected for huge + * allocations. + * + * However, with background threads enabled, keep normal purging since + * the purging delay is bounded. + */ + if (!background_thread_enabled() + && arena_dirty_decay_ms_default_get() > 0) { + arena_decay_ms_set( + tsd_tsdn(tsd), huge_arena, extent_state_dirty, 0); + } + if (!background_thread_enabled() + && arena_muzzy_decay_ms_default_get() > 0) { + arena_decay_ms_set( + tsd_tsdn(tsd), huge_arena, extent_state_muzzy, 0); + } + + return huge_arena; +} + arena_t * arena_choose_huge(tsd_t *tsd) { /* huge_arena_ind can be 0 during init (will use a0). */ @@ -1717,58 +1932,51 @@ arena_choose_huge(tsd_t *tsd) { arena_t *huge_arena = arena_get(tsd_tsdn(tsd), huge_arena_ind, false); if (huge_arena == NULL) { /* Create the huge arena on demand. */ - assert(huge_arena_ind != 0); - huge_arena = arena_get(tsd_tsdn(tsd), huge_arena_ind, true); - if (huge_arena == NULL) { - return NULL; - } - /* - * Purge eagerly for huge allocations, because: 1) number of - * huge allocations is usually small, which means ticker based - * decay is not reliable; and 2) less immediate reuse is - * expected for huge allocations. - */ - if (arena_dirty_decay_ms_default_get() > 0) { - arena_decay_ms_set(tsd_tsdn(tsd), huge_arena, - extent_state_dirty, 0); - } - if (arena_muzzy_decay_ms_default_get() > 0) { - arena_decay_ms_set(tsd_tsdn(tsd), huge_arena, - extent_state_muzzy, 0); - } + huge_arena = arena_create_huge_arena(tsd, huge_arena_ind); } return huge_arena; } bool -arena_init_huge(void) { +arena_init_huge(tsdn_t *tsdn, arena_t *a0) { bool huge_enabled; + assert(huge_arena_ind == 0); /* The threshold should be large size class. */ - if (opt_oversize_threshold > SC_LARGE_MAXCLASS || - opt_oversize_threshold < SC_LARGE_MINCLASS) { + if (opt_oversize_threshold > SC_LARGE_MAXCLASS + || opt_oversize_threshold < SC_LARGE_MINCLASS) { opt_oversize_threshold = 0; oversize_threshold = SC_LARGE_MAXCLASS + PAGE; huge_enabled = false; } else { /* Reserve the index for the huge arena. */ huge_arena_ind = narenas_total_get(); + assert(huge_arena_ind != 0); oversize_threshold = opt_oversize_threshold; + /* a0 init happened before malloc_conf_init. */ + atomic_store_zu(&a0->pa_shard.pac.oversize_threshold, + oversize_threshold, ATOMIC_RELAXED); + /* Initialize huge_arena_pac_thp fields. */ + base_t *b0 = a0->base; + /* Make sure that b0 thp auto-switch won't happen concurrently here. */ + malloc_mutex_lock(tsdn, &b0->mtx); + (&huge_arena_pac_thp)->thp_madvise = opt_huge_arena_pac_thp + && metadata_thp_enabled() + && (opt_thp == thp_mode_do_nothing) + && (init_system_thp_mode == system_thp_mode_madvise); + (&huge_arena_pac_thp)->auto_thp_switched = + b0->auto_thp_switched; + malloc_mutex_init(&(&huge_arena_pac_thp)->lock, "pac_thp", + WITNESS_RANK_LEAF, malloc_mutex_rank_exclusive); + edata_list_active_init(&(&huge_arena_pac_thp)->thp_lazy_list); + malloc_mutex_unlock(tsdn, &b0->mtx); huge_enabled = true; } return huge_enabled; } -bool -arena_is_huge(unsigned arena_ind) { - if (huge_arena_ind == 0) { - return false; - } - return (arena_ind == huge_arena_ind); -} - bool arena_boot(sc_data_t *sc_data, base_t *base, bool hpa) { arena_dirty_decay_ms_default_set(opt_dirty_decay_ms); @@ -1779,14 +1987,15 @@ arena_boot(sc_data_t *sc_data, base_t *base, bool hpa) { (1U << sc->lg_base) + (sc->ndelta << sc->lg_delta)); } - uint32_t cur_offset = (uint32_t)offsetof(arena_t, bins); + JEMALLOC_SUPPRESS_WARN_ON_USAGE( + uint32_t cur_offset = (uint32_t)offsetof(arena_t, all_bins);) for (szind_t i = 0; i < SC_NBINS; i++) { arena_bin_offsets[i] = cur_offset; nbins_total += bin_infos[i].n_shards; cur_offset += (uint32_t)(bin_infos[i].n_shards * sizeof(bin_t)); } - return pa_central_init(&arena_pa_central_global, base, hpa, - &hpa_hooks_default); + return pa_central_init( + &arena_pa_central_global, base, hpa, &hpa_hooks_default); } void @@ -1834,14 +2043,16 @@ arena_prefork7(tsdn_t *tsdn, arena_t *arena) { void arena_prefork8(tsdn_t *tsdn, arena_t *arena) { for (unsigned i = 0; i < nbins_total; i++) { - bin_prefork(tsdn, &arena->bins[i]); + JEMALLOC_SUPPRESS_WARN_ON_USAGE( + bin_prefork(tsdn, &arena->all_bins[i]);) } } void arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) { for (unsigned i = 0; i < nbins_total; i++) { - bin_postfork_parent(tsdn, &arena->bins[i]); + JEMALLOC_SUPPRESS_WARN_ON_USAGE( + bin_postfork_parent(tsdn, &arena->all_bins[i]);) } malloc_mutex_postfork_parent(tsdn, &arena->large_mtx); @@ -1879,7 +2090,8 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) { } for (unsigned i = 0; i < nbins_total; i++) { - bin_postfork_child(tsdn, &arena->bins[i]); + JEMALLOC_SUPPRESS_WARN_ON_USAGE( + bin_postfork_child(tsdn, &arena->all_bins[i]);) } malloc_mutex_postfork_child(tsdn, &arena->large_mtx); diff --git a/dep/jemalloc/src/background_thread.c b/dep/jemalloc/src/background_thread.c index 3bb8d26cd..4901856a8 100644 --- a/dep/jemalloc/src/background_thread.c +++ b/dep/jemalloc/src/background_thread.c @@ -11,15 +11,15 @@ JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS /* This option should be opt-in only. */ #define BACKGROUND_THREAD_DEFAULT false /* Read-only after initialization. */ -bool opt_background_thread = BACKGROUND_THREAD_DEFAULT; +bool opt_background_thread = BACKGROUND_THREAD_DEFAULT; size_t opt_max_background_threads = MAX_BACKGROUND_THREAD_LIMIT + 1; /* Used for thread creation, termination and stats. */ malloc_mutex_t background_thread_lock; /* Indicates global state. Atomic because decay reads this w/o locking. */ atomic_b_t background_thread_enabled_state; -size_t n_background_threads; -size_t max_background_threads; +size_t n_background_threads; +size_t max_background_threads; /* Thread info per-index. */ background_thread_info_t *background_thread_info; @@ -32,11 +32,11 @@ static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *, static void pthread_create_wrapper_init(void) { -#ifdef JEMALLOC_LAZY_LOCK +# ifdef JEMALLOC_LAZY_LOCK if (!isthreaded) { isthreaded = true; } -#endif +# endif } int @@ -46,24 +46,64 @@ pthread_create_wrapper(pthread_t *__restrict thread, const pthread_attr_t *attr, return pthread_create_fptr(thread, attr, start_routine, arg); } + +# ifdef JEMALLOC_HAVE_DLSYM +# include +# endif + +static bool +pthread_create_fptr_init(void) { + if (pthread_create_fptr != NULL) { + return false; + } + /* + * Try the next symbol first, because 1) when use lazy_lock we have a + * wrapper for pthread_create; and 2) application may define its own + * wrapper as well (and can call malloc within the wrapper). + */ +# ifdef JEMALLOC_HAVE_DLSYM + pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create"); + if (pthread_create_fptr == NULL) { + pthread_create_fptr = dlsym(RTLD_DEFAULT, "pthread_create"); + } +# else + pthread_create_fptr = NULL; +# endif + if (pthread_create_fptr == NULL) { + if (config_lazy_lock) { + malloc_write( + ": Error in dlsym(RTLD_NEXT, " + "\"pthread_create\")\n"); + abort(); + } else { + /* Fall back to the default symbol. */ + pthread_create_fptr = pthread_create; + } + } + + return false; +} #endif /* JEMALLOC_PTHREAD_CREATE_WRAPPER */ #ifndef JEMALLOC_BACKGROUND_THREAD -#define NOT_REACHED { not_reached(); } -bool background_thread_create(tsd_t *tsd, unsigned arena_ind) NOT_REACHED -bool background_threads_enable(tsd_t *tsd) NOT_REACHED -bool background_threads_disable(tsd_t *tsd) NOT_REACHED -bool background_thread_is_started(background_thread_info_t *info) NOT_REACHED -void background_thread_wakeup_early(background_thread_info_t *info, - nstime_t *remaining_sleep) NOT_REACHED -void background_thread_prefork0(tsdn_t *tsdn) NOT_REACHED -void background_thread_prefork1(tsdn_t *tsdn) NOT_REACHED -void background_thread_postfork_parent(tsdn_t *tsdn) NOT_REACHED -void background_thread_postfork_child(tsdn_t *tsdn) NOT_REACHED -bool background_thread_stats_read(tsdn_t *tsdn, - background_thread_stats_t *stats) NOT_REACHED -void background_thread_ctl_init(tsdn_t *tsdn) NOT_REACHED -#undef NOT_REACHED +# define NOT_REACHED \ + { not_reached(); } +bool +background_thread_create(tsd_t *tsd, unsigned arena_ind) NOT_REACHED + bool background_threads_enable(tsd_t *tsd) NOT_REACHED + bool background_threads_disable(tsd_t *tsd) NOT_REACHED + bool background_thread_is_started( + background_thread_info_t *info) NOT_REACHED + void background_thread_wakeup_early( + background_thread_info_t *info, nstime_t *remaining_sleep) NOT_REACHED + void background_thread_prefork0(tsdn_t *tsdn) NOT_REACHED + void background_thread_prefork1(tsdn_t *tsdn) NOT_REACHED + void background_thread_postfork_parent(tsdn_t *tsdn) NOT_REACHED + void background_thread_postfork_child(tsdn_t *tsdn) NOT_REACHED + bool background_thread_stats_read( + tsdn_t *tsdn, background_thread_stats_t *stats) NOT_REACHED + void background_thread_ctl_init(tsdn_t *tsdn) NOT_REACHED +# undef NOT_REACHED #else static bool background_thread_enabled_at_fork; @@ -80,45 +120,70 @@ background_thread_info_init(tsdn_t *tsdn, background_thread_info_t *info) { static inline bool set_current_thread_affinity(int cpu) { -#if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) +# if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) \ + || defined(JEMALLOC_HAVE_PTHREAD_SETAFFINITY_NP) +# if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) cpu_set_t cpuset; -#else -# ifndef __NetBSD__ +# else +# ifndef __NetBSD__ cpuset_t cpuset; -# else +# else cpuset_t *cpuset; -# endif -#endif +# endif +# endif -#ifndef __NetBSD__ +# ifndef __NetBSD__ CPU_ZERO(&cpuset); CPU_SET(cpu, &cpuset); -#else +# else cpuset = cpuset_create(); -#endif +# endif -#if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) +# if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) return (sched_setaffinity(0, sizeof(cpu_set_t), &cpuset) != 0); -#else -# ifndef __NetBSD__ - int ret = pthread_setaffinity_np(pthread_self(), sizeof(cpuset_t), - &cpuset); -# else - int ret = pthread_setaffinity_np(pthread_self(), cpuset_size(cpuset), - cpuset); +# else +# ifndef __NetBSD__ + int ret = pthread_setaffinity_np( + pthread_self(), sizeof(cpuset_t), &cpuset); +# else + int ret = pthread_setaffinity_np( + pthread_self(), cpuset_size(cpuset), cpuset); cpuset_destroy(cpuset); -# endif +# endif return ret != 0; -#endif +# endif +# else + return false; +# endif } -#define BILLION UINT64_C(1000000000) +# define BILLION UINT64_C(1000000000) /* Minimal sleep interval 100 ms. */ -#define BACKGROUND_THREAD_MIN_INTERVAL_NS (BILLION / 10) +# define BACKGROUND_THREAD_MIN_INTERVAL_NS (BILLION / 10) + +static int +background_thread_cond_wait( + background_thread_info_t *info, struct timespec *ts) { + int ret; + + /* + * pthread_cond_wait drops and re-acquires the mutex internally, w/o + * going through our wrapper. Update the locked state explicitly. + */ + atomic_store_b(&info->mtx.locked, false, ATOMIC_RELAXED); + if (ts == NULL) { + ret = pthread_cond_wait(&info->cond, &info->mtx.lock); + } else { + ret = pthread_cond_timedwait(&info->cond, &info->mtx.lock, ts); + } + atomic_store_b(&info->mtx.locked, true, ATOMIC_RELAXED); + + return ret; +} static void -background_thread_sleep(tsdn_t *tsdn, background_thread_info_t *info, - uint64_t interval) { +background_thread_sleep( + tsdn_t *tsdn, background_thread_info_t *info, uint64_t interval) { if (config_stats) { info->tot_n_runs++; } @@ -132,21 +197,21 @@ background_thread_sleep(tsdn_t *tsdn, background_thread_info_t *info, int ret; if (interval == BACKGROUND_THREAD_INDEFINITE_SLEEP) { - background_thread_wakeup_time_set(tsdn, info, - BACKGROUND_THREAD_INDEFINITE_SLEEP); - ret = pthread_cond_wait(&info->cond, &info->mtx.lock); + background_thread_wakeup_time_set( + tsdn, info, BACKGROUND_THREAD_INDEFINITE_SLEEP); + ret = background_thread_cond_wait(info, NULL); assert(ret == 0); } else { - assert(interval >= BACKGROUND_THREAD_MIN_INTERVAL_NS && - interval <= BACKGROUND_THREAD_INDEFINITE_SLEEP); + assert(interval >= BACKGROUND_THREAD_MIN_INTERVAL_NS + && interval <= BACKGROUND_THREAD_INDEFINITE_SLEEP); /* We need malloc clock (can be different from tv). */ nstime_t next_wakeup; nstime_init_update(&next_wakeup); nstime_iadd(&next_wakeup, interval); - assert(nstime_ns(&next_wakeup) < - BACKGROUND_THREAD_INDEFINITE_SLEEP); - background_thread_wakeup_time_set(tsdn, info, - nstime_ns(&next_wakeup)); + assert(nstime_ns(&next_wakeup) + < BACKGROUND_THREAD_INDEFINITE_SLEEP); + background_thread_wakeup_time_set( + tsdn, info, nstime_ns(&next_wakeup)); nstime_t ts_wakeup; nstime_copy(&ts_wakeup, &before_sleep); @@ -156,7 +221,7 @@ background_thread_sleep(tsdn_t *tsdn, background_thread_info_t *info, ts.tv_nsec = (size_t)nstime_nsec(&ts_wakeup); assert(!background_thread_indefinite_sleep(info)); - ret = pthread_cond_timedwait(&info->cond, &info->mtx.lock, &ts); + ret = background_thread_cond_wait(info, &ts); assert(ret == ETIMEDOUT || ret == 0); } if (config_stats) { @@ -185,11 +250,11 @@ background_thread_pause_check(tsdn_t *tsdn, background_thread_info_t *info) { } static inline void -background_work_sleep_once(tsdn_t *tsdn, background_thread_info_t *info, - unsigned ind) { +background_work_sleep_once( + tsdn_t *tsdn, background_thread_info_t *info, unsigned ind) { uint64_t ns_until_deferred = BACKGROUND_THREAD_DEFERRED_MAX; unsigned narenas = narenas_total_get(); - bool slept_indefinitely = background_thread_indefinite_sleep(info); + bool slept_indefinitely = background_thread_indefinite_sleep(info); for (unsigned i = ind; i < narenas; i += max_background_threads) { arena_t *arena = arena_get(tsdn, i, false); @@ -219,11 +284,10 @@ background_work_sleep_once(tsdn_t *tsdn, background_thread_info_t *info, if (ns_until_deferred == BACKGROUND_THREAD_DEFERRED_MAX) { sleep_ns = BACKGROUND_THREAD_INDEFINITE_SLEEP; } else { - sleep_ns = - (ns_until_deferred < BACKGROUND_THREAD_MIN_INTERVAL_NS) + sleep_ns = (ns_until_deferred + < BACKGROUND_THREAD_MIN_INTERVAL_NS) ? BACKGROUND_THREAD_MIN_INTERVAL_NS : ns_until_deferred; - } background_thread_sleep(tsdn, info, sleep_ns); @@ -232,11 +296,11 @@ background_work_sleep_once(tsdn_t *tsdn, background_thread_info_t *info, static bool background_threads_disable_single(tsd_t *tsd, background_thread_info_t *info) { if (info == &background_thread_info[0]) { - malloc_mutex_assert_owner(tsd_tsdn(tsd), - &background_thread_lock); + malloc_mutex_assert_owner( + tsd_tsdn(tsd), &background_thread_lock); } else { - malloc_mutex_assert_not_owner(tsd_tsdn(tsd), - &background_thread_lock); + malloc_mutex_assert_not_owner( + tsd_tsdn(tsd), &background_thread_lock); } pre_reentrancy(tsd, NULL); @@ -280,21 +344,23 @@ background_thread_create_signals_masked(pthread_t *thread, sigset_t set; sigfillset(&set); sigset_t oldset; - int mask_err = pthread_sigmask(SIG_SETMASK, &set, &oldset); + int mask_err = pthread_sigmask(SIG_SETMASK, &set, &oldset); if (mask_err != 0) { return mask_err; } - int create_err = pthread_create_wrapper(thread, attr, start_routine, - arg); + int create_err = pthread_create_wrapper( + thread, attr, start_routine, arg); /* * Restore the signal mask. Failure to restore the signal mask here * changes program behavior. */ int restore_err = pthread_sigmask(SIG_SETMASK, &oldset, NULL); if (restore_err != 0) { - malloc_printf(": background thread creation " + malloc_printf( + ": background thread creation " "failed (%d), and signal mask restoration failed " - "(%d)\n", create_err, restore_err); + "(%d)\n", + create_err, restore_err); if (opt_abort) { abort(); } @@ -303,7 +369,8 @@ background_thread_create_signals_masked(pthread_t *thread, } static bool -check_background_thread_creation(tsd_t *tsd, unsigned *n_created, +check_background_thread_creation(tsd_t *tsd, + const size_t const_max_background_threads, unsigned *n_created, bool *created_threads) { bool ret = false; if (likely(*n_created == n_background_threads)) { @@ -312,7 +379,7 @@ check_background_thread_creation(tsd_t *tsd, unsigned *n_created, tsdn_t *tsdn = tsd_tsdn(tsd); malloc_mutex_unlock(tsdn, &background_thread_info[0].mtx); - for (unsigned i = 1; i < max_background_threads; i++) { + for (unsigned i = 1; i < const_max_background_threads; i++) { if (created_threads[i]) { continue; } @@ -330,6 +397,7 @@ check_background_thread_creation(tsd_t *tsd, unsigned *n_created, pre_reentrancy(tsd, NULL); int err = background_thread_create_signals_masked(&info->thread, + /* NOLINTNEXTLINE(performance-no-int-to-ptr) */ NULL, background_thread_entry, (void *)(uintptr_t)i); post_reentrancy(tsd); @@ -337,8 +405,10 @@ check_background_thread_creation(tsd_t *tsd, unsigned *n_created, (*n_created)++; created_threads[i] = true; } else { - malloc_printf(": background thread " - "creation failed (%d)\n", err); + malloc_printf( + ": background thread " + "creation failed (%d)\n", + err); if (opt_abort) { abort(); } @@ -354,25 +424,35 @@ check_background_thread_creation(tsd_t *tsd, unsigned *n_created, static void background_thread0_work(tsd_t *tsd) { - /* Thread0 is also responsible for launching / terminating threads. */ - VARIABLE_ARRAY(bool, created_threads, max_background_threads); + /* + * Thread0 is also responsible for launching / terminating threads. + * We are guaranteed that `max_background_threads` will not change + * underneath us. Unfortunately static analysis tools do not understand + * this, so we are extracting `max_background_threads` into a local + * variable solely for the sake of exposing this information to such + * tools. + */ + const size_t const_max_background_threads = max_background_threads; + assert(const_max_background_threads > 0); + VARIABLE_ARRAY(bool, created_threads, const_max_background_threads); unsigned i; - for (i = 1; i < max_background_threads; i++) { + for (i = 1; i < const_max_background_threads; i++) { created_threads[i] = false; } /* Start working, and create more threads when asked. */ unsigned n_created = 1; while (background_thread_info[0].state != background_thread_stopped) { - if (background_thread_pause_check(tsd_tsdn(tsd), - &background_thread_info[0])) { + if (background_thread_pause_check( + tsd_tsdn(tsd), &background_thread_info[0])) { continue; } - if (check_background_thread_creation(tsd, &n_created, - (bool *)&created_threads)) { + if (check_background_thread_creation(tsd, + const_max_background_threads, &n_created, + created_threads)) { continue; } - background_work_sleep_once(tsd_tsdn(tsd), - &background_thread_info[0], 0); + background_work_sleep_once( + tsd_tsdn(tsd), &background_thread_info[0], 0); } /* @@ -380,7 +460,7 @@ background_thread0_work(tsd_t *tsd) { * the global background_thread mutex (and is waiting) for us. */ assert(!background_thread_enabled()); - for (i = 1; i < max_background_threads; i++) { + for (i = 1; i < const_max_background_threads; i++) { background_thread_info_t *info = &background_thread_info[i]; assert(info->state != background_thread_paused); if (created_threads[i]) { @@ -389,8 +469,8 @@ background_thread0_work(tsd_t *tsd) { malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); if (info->state != background_thread_stopped) { /* The thread was not created. */ - assert(info->state == - background_thread_started); + assert( + info->state == background_thread_started); n_background_threads--; info->state = background_thread_stopped; } @@ -406,14 +486,14 @@ background_work(tsd_t *tsd, unsigned ind) { background_thread_info_t *info = &background_thread_info[ind]; malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); - background_thread_wakeup_time_set(tsd_tsdn(tsd), info, - BACKGROUND_THREAD_INDEFINITE_SLEEP); + background_thread_wakeup_time_set( + tsd_tsdn(tsd), info, BACKGROUND_THREAD_INDEFINITE_SLEEP); if (ind == 0) { background_thread0_work(tsd); } else { while (info->state != background_thread_stopped) { - if (background_thread_pause_check(tsd_tsdn(tsd), - info)) { + if (background_thread_pause_check( + tsd_tsdn(tsd), info)) { continue; } background_work_sleep_once(tsd_tsdn(tsd), info, ind); @@ -428,11 +508,11 @@ static void * background_thread_entry(void *ind_arg) { unsigned thread_ind = (unsigned)(uintptr_t)ind_arg; assert(thread_ind < max_background_threads); -#ifdef JEMALLOC_HAVE_PTHREAD_SETNAME_NP +# ifdef JEMALLOC_HAVE_PTHREAD_SETNAME_NP pthread_setname_np(pthread_self(), "jemalloc_bg_thd"); -#elif defined(__FreeBSD__) || defined(__DragonFly__) +# elif defined(JEMALLOC_HAVE_PTHREAD_SET_NAME_NP) pthread_set_name_np(pthread_self(), "jemalloc_bg_thd"); -#endif +# endif if (opt_percpu_arena != percpu_arena_disabled) { set_current_thread_affinity((int)thread_ind); } @@ -442,8 +522,8 @@ background_thread_entry(void *ind_arg) { * turn triggers another background thread creation). */ background_work(tsd_internal_fetch(), thread_ind); - assert(pthread_equal(pthread_self(), - background_thread_info[thread_ind].thread)); + assert(pthread_equal( + pthread_self(), background_thread_info[thread_ind].thread)); return NULL; } @@ -467,8 +547,13 @@ background_thread_create_locked(tsd_t *tsd, unsigned arena_ind) { bool need_new_thread; malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); - need_new_thread = background_thread_enabled() && - (info->state == background_thread_stopped); + /* + * The last check is there to leave Thread 0 creation entirely + * to the initializing thread (arena 0). + */ + need_new_thread = background_thread_enabled() + && (info->state == background_thread_stopped) + && (thread_ind != 0 || arena_ind == 0); if (need_new_thread) { background_thread_init(tsd, info); } @@ -480,7 +565,6 @@ background_thread_create_locked(tsd_t *tsd, unsigned arena_ind) { /* Threads are created asynchronously by Thread 0. */ background_thread_info_t *t0 = &background_thread_info[0]; malloc_mutex_lock(tsd_tsdn(tsd), &t0->mtx); - assert(t0->state == background_thread_started); pthread_cond_signal(&t0->cond); malloc_mutex_unlock(tsd_tsdn(tsd), &t0->mtx); @@ -493,12 +577,15 @@ background_thread_create_locked(tsd_t *tsd, unsigned arena_ind) { * background threads with the underlying pthread_create. */ int err = background_thread_create_signals_masked(&info->thread, NULL, + /* NOLINTNEXTLINE(performance-no-int-to-ptr) */ background_thread_entry, (void *)thread_ind); post_reentrancy(tsd); if (err != 0) { - malloc_printf(": arena 0 background thread creation " - "failed (%d)\n", err); + malloc_printf( + ": arena 0 background thread creation " + "failed (%d)\n", + err); malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); info->state = background_thread_stopped; n_background_threads--; @@ -531,7 +618,7 @@ background_threads_enable(tsd_t *tsd) { VARIABLE_ARRAY(bool, marked, max_background_threads); unsigned nmarked; - for (unsigned i = 0; i < max_background_threads; i++) { + for (size_t i = 0; i < max_background_threads; i++) { marked[i] = false; } nmarked = 0; @@ -540,12 +627,12 @@ background_threads_enable(tsd_t *tsd) { /* Mark the threads we need to create for thread 0. */ unsigned narenas = narenas_total_get(); for (unsigned i = 1; i < narenas; i++) { - if (marked[i % max_background_threads] || - arena_get(tsd_tsdn(tsd), i, false) == NULL) { + if (marked[i % max_background_threads] + || arena_get(tsd_tsdn(tsd), i, false) == NULL) { continue; } - background_thread_info_t *info = &background_thread_info[ - i % max_background_threads]; + background_thread_info_t *info = + &background_thread_info[i % max_background_threads]; malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); assert(info->state == background_thread_stopped); background_thread_init(tsd, info); @@ -563,8 +650,8 @@ background_threads_enable(tsd_t *tsd) { for (unsigned i = 0; i < narenas; i++) { arena_t *arena = arena_get(tsd_tsdn(tsd), i, false); if (arena != NULL) { - pa_shard_set_deferral_allowed(tsd_tsdn(tsd), - &arena->pa_shard, true); + pa_shard_set_deferral_allowed( + tsd_tsdn(tsd), &arena->pa_shard, true); } } return false; @@ -576,8 +663,8 @@ background_threads_disable(tsd_t *tsd) { malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock); /* Thread 0 will be responsible for terminating other threads. */ - if (background_threads_disable_single(tsd, - &background_thread_info[0])) { + if (background_threads_disable_single( + tsd, &background_thread_info[0])) { return true; } assert(n_background_threads == 0); @@ -585,8 +672,8 @@ background_threads_disable(tsd_t *tsd) { for (unsigned i = 0; i < narenas; i++) { arena_t *arena = arena_get(tsd_tsdn(tsd), i, false); if (arena != NULL) { - pa_shard_set_deferral_allowed(tsd_tsdn(tsd), - &arena->pa_shard, false); + pa_shard_set_deferral_allowed( + tsd_tsdn(tsd), &arena->pa_shard, false); } } @@ -599,15 +686,15 @@ background_thread_is_started(background_thread_info_t *info) { } void -background_thread_wakeup_early(background_thread_info_t *info, - nstime_t *remaining_sleep) { +background_thread_wakeup_early( + background_thread_info_t *info, nstime_t *remaining_sleep) { /* * This is an optimization to increase batching. At this point * we know that background thread wakes up soon, so the time to cache * the just freed memory is bounded and low. */ - if (remaining_sleep != NULL && nstime_ns(remaining_sleep) < - BACKGROUND_THREAD_MIN_INTERVAL_NS) { + if (remaining_sleep != NULL + && nstime_ns(remaining_sleep) < BACKGROUND_THREAD_MIN_INTERVAL_NS) { return; } pthread_cond_signal(&info->cond); @@ -629,8 +716,8 @@ background_thread_prefork1(tsdn_t *tsdn) { void background_thread_postfork_parent(tsdn_t *tsdn) { for (unsigned i = 0; i < max_background_threads; i++) { - malloc_mutex_postfork_parent(tsdn, - &background_thread_info[i].mtx); + malloc_mutex_postfork_parent( + tsdn, &background_thread_info[i].mtx); } malloc_mutex_postfork_parent(tsdn, &background_thread_lock); } @@ -638,8 +725,8 @@ background_thread_postfork_parent(tsdn_t *tsdn) { void background_thread_postfork_child(tsdn_t *tsdn) { for (unsigned i = 0; i < max_background_threads; i++) { - malloc_mutex_postfork_child(tsdn, - &background_thread_info[i].mtx); + malloc_mutex_postfork_child( + tsdn, &background_thread_info[i].mtx); } malloc_mutex_postfork_child(tsdn, &background_thread_lock); if (!background_thread_enabled_at_fork) { @@ -688,8 +775,8 @@ background_thread_stats_read(tsdn_t *tsdn, background_thread_stats_t *stats) { if (info->state != background_thread_stopped) { num_runs += info->tot_n_runs; nstime_add(&stats->run_interval, &info->tot_sleep_time); - malloc_mutex_prof_max_update(tsdn, - &stats->max_counter_per_bg_thd, &info->mtx); + malloc_mutex_prof_max_update( + tsdn, &stats->max_counter_per_bg_thd, &info->mtx); } malloc_mutex_unlock(tsdn, &info->mtx); } @@ -702,42 +789,9 @@ background_thread_stats_read(tsdn_t *tsdn, background_thread_stats_t *stats) { return false; } -#undef BACKGROUND_THREAD_NPAGES_THRESHOLD -#undef BILLION -#undef BACKGROUND_THREAD_MIN_INTERVAL_NS - -#ifdef JEMALLOC_HAVE_DLSYM -#include -#endif - -static bool -pthread_create_fptr_init(void) { - if (pthread_create_fptr != NULL) { - return false; - } - /* - * Try the next symbol first, because 1) when use lazy_lock we have a - * wrapper for pthread_create; and 2) application may define its own - * wrapper as well (and can call malloc within the wrapper). - */ -#ifdef JEMALLOC_HAVE_DLSYM - pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create"); -#else - pthread_create_fptr = NULL; -#endif - if (pthread_create_fptr == NULL) { - if (config_lazy_lock) { - malloc_write(": Error in dlsym(RTLD_NEXT, " - "\"pthread_create\")\n"); - abort(); - } else { - /* Fall back to the default symbol. */ - pthread_create_fptr = pthread_create; - } - } - - return false; -} +# undef BACKGROUND_THREAD_NPAGES_THRESHOLD +# undef BILLION +# undef BACKGROUND_THREAD_MIN_INTERVAL_NS /* * When lazy lock is enabled, we need to make sure setting isthreaded before @@ -748,24 +802,24 @@ pthread_create_fptr_init(void) { void background_thread_ctl_init(tsdn_t *tsdn) { malloc_mutex_assert_not_owner(tsdn, &background_thread_lock); -#ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER +# ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER pthread_create_fptr_init(); pthread_create_wrapper_init(); -#endif +# endif } #endif /* defined(JEMALLOC_BACKGROUND_THREAD) */ -bool -background_thread_boot0(void) { + bool background_thread_boot0(void) { if (!have_background_thread && opt_background_thread) { - malloc_printf(": option background_thread currently " + malloc_printf( + ": option background_thread currently " "supports pthread only\n"); return true; } #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER - if ((config_lazy_lock || opt_background_thread) && - pthread_create_fptr_init()) { + if ((config_lazy_lock || opt_background_thread) + && pthread_create_fptr_init()) { return true; } #endif @@ -783,17 +837,16 @@ background_thread_boot1(tsdn_t *tsdn, base_t *base) { } max_background_threads = opt_max_background_threads; - background_thread_enabled_set(tsdn, opt_background_thread); if (malloc_mutex_init(&background_thread_lock, - "background_thread_global", - WITNESS_RANK_BACKGROUND_THREAD_GLOBAL, - malloc_mutex_rank_exclusive)) { + "background_thread_global", + WITNESS_RANK_BACKGROUND_THREAD_GLOBAL, + malloc_mutex_rank_exclusive)) { return true; } background_thread_info = (background_thread_info_t *)base_alloc(tsdn, - base, opt_max_background_threads * - sizeof(background_thread_info_t), CACHELINE); + base, opt_max_background_threads * sizeof(background_thread_info_t), + CACHELINE); if (background_thread_info == NULL) { return true; } @@ -802,8 +855,8 @@ background_thread_boot1(tsdn_t *tsdn, base_t *base) { background_thread_info_t *info = &background_thread_info[i]; /* Thread mutex is rank_inclusive because of thread0. */ if (malloc_mutex_init(&info->mtx, "background_thread", - WITNESS_RANK_BACKGROUND_THREAD, - malloc_mutex_address_ordered)) { + WITNESS_RANK_BACKGROUND_THREAD, + malloc_mutex_address_ordered)) { return true; } if (pthread_cond_init(&info->cond, NULL)) { @@ -814,7 +867,8 @@ background_thread_boot1(tsdn_t *tsdn, base_t *base) { background_thread_info_init(tsdn, info); malloc_mutex_unlock(tsdn, &info->mtx); } + /* Using _impl to bypass the locking check during init. */ + background_thread_enabled_set_impl(opt_background_thread); #endif - return false; } diff --git a/dep/jemalloc/src/base.c b/dep/jemalloc/src/base.c index 7f4d67564..ef7f0dd45 100644 --- a/dep/jemalloc/src/base.c +++ b/dep/jemalloc/src/base.c @@ -12,7 +12,7 @@ * of metadata), since more metadata (e.g. rtree nodes) come from a0's base. */ -#define BASE_AUTO_THP_THRESHOLD 2 +#define BASE_AUTO_THP_THRESHOLD 2 #define BASE_AUTO_THP_THRESHOLD_A0 5 /******************************************************************************/ @@ -22,45 +22,46 @@ static base_t *b0; metadata_thp_mode_t opt_metadata_thp = METADATA_THP_DEFAULT; -const char *metadata_thp_mode_names[] = { - "disabled", - "auto", - "always" -}; +const char *const metadata_thp_mode_names[] = {"disabled", "auto", "always"}; /******************************************************************************/ static inline bool metadata_thp_madvise(void) { return (metadata_thp_enabled() && - (init_system_thp_mode == thp_mode_default)); + (init_system_thp_mode == system_thp_mode_madvise)); } static void * base_map(tsdn_t *tsdn, ehooks_t *ehooks, unsigned ind, size_t size) { void *addr; - bool zero = true; - bool commit = true; + bool zero = true; + bool commit = true; - /* Use huge page sizes and alignment regardless of opt_metadata_thp. */ - assert(size == HUGEPAGE_CEILING(size)); - size_t alignment = HUGEPAGE; + /* + * Use huge page sizes and alignment when opt_metadata_thp is enabled + * or auto. + */ + size_t alignment; + if (opt_metadata_thp == metadata_thp_disabled) { + alignment = BASE_BLOCK_MIN_ALIGN; + } else { + assert(size == HUGEPAGE_CEILING(size)); + alignment = HUGEPAGE; + } if (ehooks_are_default(ehooks)) { addr = extent_alloc_mmap(NULL, size, alignment, &zero, &commit); - if (have_madvise_huge && addr) { - pages_set_thp_state(addr, size); - } } else { - addr = ehooks_alloc(tsdn, ehooks, NULL, size, alignment, &zero, - &commit); + addr = ehooks_alloc( + tsdn, ehooks, NULL, size, alignment, &zero, &commit); } return addr; } static void -base_unmap(tsdn_t *tsdn, ehooks_t *ehooks, unsigned ind, void *addr, - size_t size) { +base_unmap( + tsdn_t *tsdn, ehooks_t *ehooks, unsigned ind, void *addr, size_t size) { /* * Cascade through dalloc, decommit, purge_forced, and purge_lazy, * stopping at first success. This cascade is performed for consistency @@ -104,21 +105,31 @@ base_unmap(tsdn_t *tsdn, ehooks_t *ehooks, unsigned ind, void *addr, label_done: if (metadata_thp_madvise()) { /* Set NOHUGEPAGE after unmap to avoid kernel defrag. */ - assert(((uintptr_t)addr & HUGEPAGE_MASK) == 0 && - (size & HUGEPAGE_MASK) == 0); + assert(((uintptr_t)addr & HUGEPAGE_MASK) == 0 + && (size & HUGEPAGE_MASK) == 0); pages_nohuge(addr, size); } } +static inline bool +base_edata_is_reused(edata_t *edata) { + /* + * Borrow the guarded bit to indicate if the extent is a recycled one, + * i.e. the ones returned to base for reuse; currently only tcache bin + * stacks. Skips stats updating if so (needed for this purpose only). + */ + return edata_guarded_get(edata); +} + static void -base_edata_init(size_t *extent_sn_next, edata_t *edata, void *addr, - size_t size) { +base_edata_init( + size_t *extent_sn_next, edata_t *edata, void *addr, size_t size) { size_t sn; sn = *extent_sn_next; (*extent_sn_next)++; - edata_binit(edata, addr, size, sn); + edata_binit(edata, addr, size, sn, false /* is_reused */); } static size_t @@ -135,6 +146,42 @@ base_get_num_blocks(base_t *base, bool with_new_block) { return n_blocks; } +static void +huge_arena_auto_thp_switch(tsdn_t *tsdn, pac_thp_t *pac_thp) { + assert(opt_huge_arena_pac_thp); +#ifdef JEMALLOC_JET + if (pac_thp->auto_thp_switched) { + return; + } +#else + /* + * The switch should be turned on only once when the b0 auto thp switch is + * turned on, unless it's a unit test where b0 gets deleted and then + * recreated. + */ + assert(!pac_thp->auto_thp_switched); +#endif + + edata_list_active_t *pending_list; + malloc_mutex_lock(tsdn, &pac_thp->lock); + pending_list = &pac_thp->thp_lazy_list; + pac_thp->auto_thp_switched = true; + malloc_mutex_unlock(tsdn, &pac_thp->lock); + + unsigned cnt = 0; + edata_t *edata; + ql_foreach (edata, &pending_list->head, ql_link_active) { + assert(edata != NULL); + void *addr = edata_addr_get(edata); + size_t size = edata_size_get(edata); + assert(HUGEPAGE_ADDR2BASE(addr) == addr); + assert(HUGEPAGE_CEILING(size) == size && size != 0); + pages_huge(addr, size); + cnt++; + } + assert(cnt == atomic_load_u(&pac_thp->n_thp_lazy, ATOMIC_RELAXED)); +} + static void base_auto_thp_switch(tsdn_t *tsdn, base_t *base) { assert(opt_metadata_thp == metadata_thp_auto); @@ -145,11 +192,11 @@ base_auto_thp_switch(tsdn_t *tsdn, base_t *base) { /* Called when adding a new block. */ bool should_switch; if (base_ind_get(base) != 0) { - should_switch = (base_get_num_blocks(base, true) == - BASE_AUTO_THP_THRESHOLD); + should_switch = (base_get_num_blocks(base, true) + == BASE_AUTO_THP_THRESHOLD); } else { - should_switch = (base_get_num_blocks(base, true) == - BASE_AUTO_THP_THRESHOLD_A0); + should_switch = (base_get_num_blocks(base, true) + == BASE_AUTO_THP_THRESHOLD_A0); } if (!should_switch) { return; @@ -163,77 +210,141 @@ base_auto_thp_switch(tsdn_t *tsdn, base_t *base) { assert((block->size & HUGEPAGE_MASK) == 0); pages_huge(block, block->size); if (config_stats) { - base->n_thp += HUGEPAGE_CEILING(block->size - - edata_bsize_get(&block->edata)) >> LG_HUGEPAGE; + base->n_thp += HUGEPAGE_CEILING(block->size + - edata_bsize_get(&block->edata)) + >> LG_HUGEPAGE; } block = block->next; assert(block == NULL || (base_ind_get(base) == 0)); } + + /* Handle the THP auto switch for the huge arena. */ + if (!huge_arena_pac_thp.thp_madvise || base_ind_get(base) != 0) { + /* + * The huge arena THP auto-switch is triggered only by b0 switch, + * provided that the huge arena is initialized. If b0 switch is enabled + * before huge arena is ready, the huge arena switch will be enabled + * during huge_arena_pac_thp initialization. + */ + return; + } + /* + * thp_madvise above is by default false and set in arena_init_huge() with + * b0 mtx held. So if we reach here, it means the entire huge_arena_pac_thp + * is initialized and we can safely switch the THP. + */ + malloc_mutex_unlock(tsdn, &base->mtx); + huge_arena_auto_thp_switch(tsdn, &huge_arena_pac_thp); + malloc_mutex_lock(tsdn, &base->mtx); } static void * -base_extent_bump_alloc_helper(edata_t *edata, size_t *gap_size, size_t size, - size_t alignment) { +base_extent_bump_alloc_helper( + edata_t *edata, size_t *gap_size, size_t size, size_t alignment) { void *ret; assert(alignment == ALIGNMENT_CEILING(alignment, QUANTUM)); assert(size == ALIGNMENT_CEILING(size, alignment)); - *gap_size = ALIGNMENT_CEILING((uintptr_t)edata_addr_get(edata), - alignment) - (uintptr_t)edata_addr_get(edata); - ret = (void *)((uintptr_t)edata_addr_get(edata) + *gap_size); + *gap_size = ALIGNMENT_CEILING( + (uintptr_t)edata_addr_get(edata), alignment) + - (uintptr_t)edata_addr_get(edata); + ret = (void *)((byte_t *)edata_addr_get(edata) + *gap_size); assert(edata_bsize_get(edata) >= *gap_size + size); - edata_binit(edata, (void *)((uintptr_t)edata_addr_get(edata) + - *gap_size + size), edata_bsize_get(edata) - *gap_size - size, - edata_sn_get(edata)); + edata_binit(edata, + (void *)((byte_t *)edata_addr_get(edata) + *gap_size + size), + edata_bsize_get(edata) - *gap_size - size, edata_sn_get(edata), + base_edata_is_reused(edata)); return ret; } static void -base_extent_bump_alloc_post(base_t *base, edata_t *edata, size_t gap_size, - void *addr, size_t size) { - if (edata_bsize_get(edata) > 0) { - /* - * Compute the index for the largest size class that does not - * exceed extent's size. - */ - szind_t index_floor = - sz_size2index(edata_bsize_get(edata) + 1) - 1; - edata_heap_insert(&base->avail[index_floor], edata); +base_edata_heap_insert(tsdn_t *tsdn, base_t *base, edata_t *edata) { + malloc_mutex_assert_owner(tsdn, &base->mtx); + + size_t bsize = edata_bsize_get(edata); + assert(bsize > 0); + /* + * Compute the index for the largest size class that does not exceed + * extent's size. + */ + szind_t index_floor = sz_size2index(bsize + 1) - 1; + edata_heap_insert(&base->avail[index_floor], edata); +} + +/* + * Only can be called by top-level functions, since it may call base_alloc + * internally when cache is empty. + */ +static edata_t * +base_alloc_base_edata(tsdn_t *tsdn, base_t *base) { + edata_t *edata; + + malloc_mutex_lock(tsdn, &base->mtx); + edata = edata_avail_first(&base->edata_avail); + if (edata != NULL) { + edata_avail_remove(&base->edata_avail, edata); + } + malloc_mutex_unlock(tsdn, &base->mtx); + + if (edata == NULL) { + edata = base_alloc_edata(tsdn, base); } - if (config_stats) { + return edata; +} + +static void +base_extent_bump_alloc_post(tsdn_t *tsdn, base_t *base, edata_t *edata, + size_t gap_size, void *addr, size_t size) { + if (edata_bsize_get(edata) > 0) { + base_edata_heap_insert(tsdn, base, edata); + } else { + /* Freed base edata_t stored in edata_avail. */ + edata_avail_insert(&base->edata_avail, edata); + } + + if (config_stats && !base_edata_is_reused(edata)) { base->allocated += size; /* * Add one PAGE to base_resident for every page boundary that is * crossed by the new allocation. Adjust n_thp similarly when * metadata_thp is enabled. */ - base->resident += PAGE_CEILING((uintptr_t)addr + size) - - PAGE_CEILING((uintptr_t)addr - gap_size); + base->resident += PAGE_CEILING((uintptr_t)addr + size) + - PAGE_CEILING((uintptr_t)addr - gap_size); assert(base->allocated <= base->resident); assert(base->resident <= base->mapped); - if (metadata_thp_madvise() && (opt_metadata_thp == - metadata_thp_always || base->auto_thp_switched)) { + if (metadata_thp_madvise() + && (opt_metadata_thp == metadata_thp_always + || base->auto_thp_switched)) { base->n_thp += (HUGEPAGE_CEILING((uintptr_t)addr + size) - - HUGEPAGE_CEILING((uintptr_t)addr - gap_size)) >> - LG_HUGEPAGE; + - HUGEPAGE_CEILING( + (uintptr_t)addr - gap_size)) + >> LG_HUGEPAGE; assert(base->mapped >= base->n_thp << LG_HUGEPAGE); } } } static void * -base_extent_bump_alloc(base_t *base, edata_t *edata, size_t size, - size_t alignment) { - void *ret; +base_extent_bump_alloc( + tsdn_t *tsdn, base_t *base, edata_t *edata, size_t size, size_t alignment) { + void *ret; size_t gap_size; ret = base_extent_bump_alloc_helper(edata, &gap_size, size, alignment); - base_extent_bump_alloc_post(base, edata, gap_size, ret, size); + base_extent_bump_alloc_post(tsdn, base, edata, gap_size, ret, size); return ret; } +static size_t +base_block_size_ceil(size_t block_size) { + return opt_metadata_thp == metadata_thp_disabled + ? ALIGNMENT_CEILING(block_size, BASE_BLOCK_MIN_ALIGN) + : HUGEPAGE_CEILING(block_size); +} + /* * Allocate a block of virtual memory that is large enough to start with a * base_block_t header, followed by an object of specified size and alignment. @@ -246,36 +357,38 @@ base_block_alloc(tsdn_t *tsdn, base_t *base, ehooks_t *ehooks, unsigned ind, alignment = ALIGNMENT_CEILING(alignment, QUANTUM); size_t usize = ALIGNMENT_CEILING(size, alignment); size_t header_size = sizeof(base_block_t); - size_t gap_size = ALIGNMENT_CEILING(header_size, alignment) - - header_size; + size_t gap_size = ALIGNMENT_CEILING(header_size, alignment) + - header_size; /* * Create increasingly larger blocks in order to limit the total number * of disjoint virtual memory ranges. Choose the next size in the page * size class series (skipping size classes that are not a multiple of - * HUGEPAGE), or a size large enough to satisfy the requested size and - * alignment, whichever is larger. + * HUGEPAGE when using metadata_thp), or a size large enough to satisfy + * the requested size and alignment, whichever is larger. */ - size_t min_block_size = HUGEPAGE_CEILING(sz_psz2u(header_size + gap_size - + usize)); - pszind_t pind_next = (*pind_last + 1 < sz_psz2ind(SC_LARGE_MAXCLASS)) ? - *pind_last + 1 : *pind_last; - size_t next_block_size = HUGEPAGE_CEILING(sz_pind2sz(pind_next)); - size_t block_size = (min_block_size > next_block_size) ? min_block_size - : next_block_size; - base_block_t *block = (base_block_t *)base_map(tsdn, ehooks, ind, - block_size); + size_t min_block_size = base_block_size_ceil( + sz_psz2u(header_size + gap_size + usize)); + pszind_t pind_next = (*pind_last + 1 < sz_psz2ind(SC_LARGE_MAXCLASS)) + ? *pind_last + 1 + : *pind_last; + size_t next_block_size = base_block_size_ceil(sz_pind2sz(pind_next)); + size_t block_size = (min_block_size > next_block_size) + ? min_block_size + : next_block_size; + base_block_t *block = (base_block_t *)base_map( + tsdn, ehooks, ind, block_size); if (block == NULL) { return NULL; } if (metadata_thp_madvise()) { void *addr = (void *)block; - assert(((uintptr_t)addr & HUGEPAGE_MASK) == 0 && - (block_size & HUGEPAGE_MASK) == 0); + assert(((uintptr_t)addr & HUGEPAGE_MASK) == 0 + && (block_size & HUGEPAGE_MASK) == 0); if (opt_metadata_thp == metadata_thp_always) { pages_huge(addr, block_size); - } else if (opt_metadata_thp == metadata_thp_auto && - base != NULL) { + } else if (opt_metadata_thp == metadata_thp_auto + && base != NULL) { /* base != NULL indicates this is not a new base. */ malloc_mutex_lock(tsdn, &base->mtx); base_auto_thp_switch(tsdn, base); @@ -291,7 +404,7 @@ base_block_alloc(tsdn_t *tsdn, base_t *base, ehooks_t *ehooks, unsigned ind, block->next = NULL; assert(block_size >= header_size); base_edata_init(extent_sn_next, &block->edata, - (void *)((uintptr_t)block + header_size), block_size - header_size); + (void *)((byte_t *)block + header_size), block_size - header_size); return block; } @@ -322,12 +435,12 @@ base_extent_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment) { base->allocated += sizeof(base_block_t); base->resident += PAGE_CEILING(sizeof(base_block_t)); base->mapped += block->size; - if (metadata_thp_madvise() && - !(opt_metadata_thp == metadata_thp_auto - && !base->auto_thp_switched)) { + if (metadata_thp_madvise() + && !(opt_metadata_thp == metadata_thp_auto + && !base->auto_thp_switched)) { assert(base->n_thp > 0); - base->n_thp += HUGEPAGE_CEILING(sizeof(base_block_t)) >> - LG_HUGEPAGE; + base->n_thp += HUGEPAGE_CEILING(sizeof(base_block_t)) + >> LG_HUGEPAGE; } assert(base->allocated <= base->resident); assert(base->resident <= base->mapped); @@ -345,7 +458,7 @@ base_t * base_new(tsdn_t *tsdn, unsigned ind, const extent_hooks_t *extent_hooks, bool metadata_use_hooks) { pszind_t pind_last = 0; - size_t extent_sn_next = 0; + size_t extent_sn_next = 0; /* * The base will contain the ehooks eventually, but it itself is @@ -353,9 +466,10 @@ base_new(tsdn_t *tsdn, unsigned ind, const extent_hooks_t *extent_hooks, * memory, and then initialize the ehooks within the base_t. */ ehooks_t fake_ehooks; - ehooks_init(&fake_ehooks, metadata_use_hooks ? - (extent_hooks_t *)extent_hooks : - (extent_hooks_t *)&ehooks_default_extent_hooks, ind); + ehooks_init(&fake_ehooks, + metadata_use_hooks ? (extent_hooks_t *)extent_hooks + : (extent_hooks_t *)&ehooks_default_extent_hooks, + ind); base_block_t *block = base_block_alloc(tsdn, NULL, &fake_ehooks, ind, &pind_last, &extent_sn_next, sizeof(base_t), QUANTUM); @@ -363,17 +477,18 @@ base_new(tsdn_t *tsdn, unsigned ind, const extent_hooks_t *extent_hooks, return NULL; } - size_t gap_size; - size_t base_alignment = CACHELINE; - size_t base_size = ALIGNMENT_CEILING(sizeof(base_t), base_alignment); - base_t *base = (base_t *)base_extent_bump_alloc_helper(&block->edata, - &gap_size, base_size, base_alignment); + size_t gap_size; + size_t base_alignment = CACHELINE; + size_t base_size = ALIGNMENT_CEILING(sizeof(base_t), base_alignment); + base_t *base = (base_t *)base_extent_bump_alloc_helper( + &block->edata, &gap_size, base_size, base_alignment); ehooks_init(&base->ehooks, (extent_hooks_t *)extent_hooks, ind); - ehooks_init(&base->ehooks_base, metadata_use_hooks ? - (extent_hooks_t *)extent_hooks : - (extent_hooks_t *)&ehooks_default_extent_hooks, ind); + ehooks_init(&base->ehooks_base, + metadata_use_hooks ? (extent_hooks_t *)extent_hooks + : (extent_hooks_t *)&ehooks_default_extent_hooks, + ind); if (malloc_mutex_init(&base->mtx, "base", WITNESS_RANK_BASE, - malloc_mutex_rank_exclusive)) { + malloc_mutex_rank_exclusive)) { base_unmap(tsdn, &fake_ehooks, ind, block, block->size); return NULL; } @@ -384,32 +499,41 @@ base_new(tsdn_t *tsdn, unsigned ind, const extent_hooks_t *extent_hooks, for (szind_t i = 0; i < SC_NSIZES; i++) { edata_heap_new(&base->avail[i]); } + edata_avail_new(&base->edata_avail); + if (config_stats) { + base->edata_allocated = 0; + base->rtree_allocated = 0; base->allocated = sizeof(base_block_t); base->resident = PAGE_CEILING(sizeof(base_block_t)); base->mapped = block->size; - base->n_thp = (opt_metadata_thp == metadata_thp_always) && - metadata_thp_madvise() ? HUGEPAGE_CEILING(sizeof(base_block_t)) - >> LG_HUGEPAGE : 0; + base->n_thp = (opt_metadata_thp == metadata_thp_always) + && metadata_thp_madvise() + ? HUGEPAGE_CEILING(sizeof(base_block_t)) >> LG_HUGEPAGE + : 0; assert(base->allocated <= base->resident); assert(base->resident <= base->mapped); assert(base->n_thp << LG_HUGEPAGE <= base->mapped); } - base_extent_bump_alloc_post(base, &block->edata, gap_size, base, - base_size); + + /* Locking here is only necessary because of assertions. */ + malloc_mutex_lock(tsdn, &base->mtx); + base_extent_bump_alloc_post( + tsdn, base, &block->edata, gap_size, base, base_size); + malloc_mutex_unlock(tsdn, &base->mtx); return base; } void base_delete(tsdn_t *tsdn, base_t *base) { - ehooks_t *ehooks = base_ehooks_get_for_metadata(base); + ehooks_t *ehooks = base_ehooks_get_for_metadata(base); base_block_t *next = base->blocks; do { base_block_t *block = next; next = block->next; - base_unmap(tsdn, ehooks, base_ind_get(base), block, - block->size); + base_unmap( + tsdn, ehooks, base_ind_get(base), block, block->size); } while (next != NULL); } @@ -425,15 +549,15 @@ base_ehooks_get_for_metadata(base_t *base) { extent_hooks_t * base_extent_hooks_set(base_t *base, extent_hooks_t *extent_hooks) { - extent_hooks_t *old_extent_hooks = - ehooks_get_extent_hooks_ptr(&base->ehooks); + extent_hooks_t *old_extent_hooks = ehooks_get_extent_hooks_ptr( + &base->ehooks); ehooks_init(&base->ehooks, extent_hooks, ehooks_ind_get(&base->ehooks)); return old_extent_hooks; } static void * base_alloc_impl(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment, - size_t *esn) { + size_t *esn, size_t *ret_usize) { alignment = QUANTUM_CEILING(alignment); size_t usize = ALIGNMENT_CEILING(size, alignment); size_t asize = usize + alignment - QUANTUM; @@ -457,10 +581,13 @@ base_alloc_impl(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment, goto label_return; } - ret = base_extent_bump_alloc(base, edata, usize, alignment); + ret = base_extent_bump_alloc(tsdn, base, edata, usize, alignment); if (esn != NULL) { *esn = (size_t)edata_sn_get(edata); } + if (ret_usize != NULL) { + *ret_usize = usize; + } label_return: malloc_mutex_unlock(tsdn, &base->mtx); return ret; @@ -476,30 +603,121 @@ label_return: */ void * base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment) { - return base_alloc_impl(tsdn, base, size, alignment, NULL); + return base_alloc_impl(tsdn, base, size, alignment, NULL, NULL); } edata_t * base_alloc_edata(tsdn_t *tsdn, base_t *base) { - size_t esn; - edata_t *edata = base_alloc_impl(tsdn, base, sizeof(edata_t), - EDATA_ALIGNMENT, &esn); + size_t esn, usize; + edata_t *edata = base_alloc_impl( + tsdn, base, sizeof(edata_t), EDATA_ALIGNMENT, &esn, &usize); if (edata == NULL) { return NULL; } + if (config_stats) { + base->edata_allocated += usize; + } edata_esn_set(edata, esn); return edata; } +void * +base_alloc_rtree(tsdn_t *tsdn, base_t *base, size_t size) { + size_t usize; + void *rtree = base_alloc_impl( + tsdn, base, size, CACHELINE, NULL, &usize); + if (rtree == NULL) { + return NULL; + } + if (config_stats) { + base->rtree_allocated += usize; + } + return rtree; +} + +static inline void +b0_alloc_header_size(size_t *header_size, size_t *alignment) { + *alignment = QUANTUM; + *header_size = QUANTUM > sizeof(edata_t *) ? QUANTUM + : sizeof(edata_t *); +} + +/* + * Each piece allocated here is managed by a separate edata, because it was bump + * allocated and cannot be merged back into the original base_block. This means + * it's not for general purpose: 1) they are not page aligned, nor page sized, + * and 2) the requested size should not be too small (as each piece comes with + * an edata_t). Only used for tcache bin stack allocation now. + */ +void * +b0_alloc_tcache_stack(tsdn_t *tsdn, size_t stack_size) { + base_t *base = b0get(); + edata_t *edata = base_alloc_base_edata(tsdn, base); + if (edata == NULL) { + return NULL; + } + + /* + * Reserve room for the header, which stores a pointer to the managing + * edata_t. The header itself is located right before the return + * address, so that edata can be retrieved on dalloc. Bump up to usize + * to improve reusability -- otherwise the freed stacks will be put back + * into the previous size class. + */ + size_t esn, alignment, header_size; + b0_alloc_header_size(&header_size, &alignment); + + size_t alloc_size = sz_s2u(stack_size + header_size); + void *addr = base_alloc_impl( + tsdn, base, alloc_size, alignment, &esn, NULL); + if (addr == NULL) { + edata_avail_insert(&base->edata_avail, edata); + return NULL; + } + + /* Set is_reused: see comments in base_edata_is_reused. */ + edata_binit(edata, addr, alloc_size, esn, true /* is_reused */); + *(edata_t **)addr = edata; + + return (byte_t *)addr + header_size; +} + void -base_stats_get(tsdn_t *tsdn, base_t *base, size_t *allocated, size_t *resident, +b0_dalloc_tcache_stack(tsdn_t *tsdn, void *tcache_stack) { + /* edata_t pointer stored in header. */ + size_t alignment, header_size; + b0_alloc_header_size(&header_size, &alignment); + + edata_t *edata = *(edata_t **)((byte_t *)tcache_stack - header_size); + void *addr = edata_addr_get(edata); + size_t bsize = edata_bsize_get(edata); + /* Marked as "reused" to avoid double counting stats. */ + assert(base_edata_is_reused(edata)); + assert(addr != NULL && bsize > 0); + + /* Zero out since base_alloc returns zeroed memory. */ + memset(addr, 0, bsize); + + base_t *base = b0get(); + malloc_mutex_lock(tsdn, &base->mtx); + base_edata_heap_insert(tsdn, base, edata); + malloc_mutex_unlock(tsdn, &base->mtx); +} + +void +base_stats_get(tsdn_t *tsdn, base_t *base, size_t *allocated, + size_t *edata_allocated, size_t *rtree_allocated, size_t *resident, size_t *mapped, size_t *n_thp) { cassert(config_stats); malloc_mutex_lock(tsdn, &base->mtx); assert(base->allocated <= base->resident); assert(base->resident <= base->mapped); + assert( + base->edata_allocated + base->rtree_allocated <= base->allocated); *allocated = base->allocated; + *edata_allocated = base->edata_allocated; + *rtree_allocated = base->rtree_allocated; *resident = base->resident; *mapped = base->mapped; *n_thp = base->n_thp; diff --git a/dep/jemalloc/src/bin.c b/dep/jemalloc/src/bin.c index fa2045870..6bab4b227 100644 --- a/dep/jemalloc/src/bin.c +++ b/dep/jemalloc/src/bin.c @@ -41,7 +41,7 @@ bin_shard_sizes_boot(unsigned bin_shard_sizes[SC_NBINS]) { bool bin_init(bin_t *bin) { if (malloc_mutex_init(&bin->lock, "bin", WITNESS_RANK_BIN, - malloc_mutex_rank_exclusive)) { + malloc_mutex_rank_exclusive)) { return true; } bin->slabcur = NULL; @@ -67,3 +67,266 @@ void bin_postfork_child(tsdn_t *tsdn, bin_t *bin) { malloc_mutex_postfork_child(tsdn, &bin->lock); } + +void * +bin_slab_reg_alloc(edata_t *slab, const bin_info_t *bin_info) { + void *ret; + slab_data_t *slab_data = edata_slab_data_get(slab); + size_t regind; + + assert(edata_nfree_get(slab) > 0); + assert(!bitmap_full(slab_data->bitmap, &bin_info->bitmap_info)); + + regind = bitmap_sfu(slab_data->bitmap, &bin_info->bitmap_info); + ret = (void *)((byte_t *)edata_addr_get(slab) + + (uintptr_t)(bin_info->reg_size * regind)); + edata_nfree_dec(slab); + return ret; +} + +void +bin_slab_reg_alloc_batch( + edata_t *slab, const bin_info_t *bin_info, unsigned cnt, void **ptrs) { + slab_data_t *slab_data = edata_slab_data_get(slab); + + assert(edata_nfree_get(slab) >= cnt); + assert(!bitmap_full(slab_data->bitmap, &bin_info->bitmap_info)); + +#if (!defined JEMALLOC_INTERNAL_POPCOUNTL) || (defined BITMAP_USE_TREE) + for (unsigned i = 0; i < cnt; i++) { + size_t regind = bitmap_sfu( + slab_data->bitmap, &bin_info->bitmap_info); + *(ptrs + i) = (void *)((uintptr_t)edata_addr_get(slab) + + (uintptr_t)(bin_info->reg_size * regind)); + } +#else + unsigned group = 0; + bitmap_t g = slab_data->bitmap[group]; + unsigned i = 0; + while (i < cnt) { + while (g == 0) { + g = slab_data->bitmap[++group]; + } + size_t shift = group << LG_BITMAP_GROUP_NBITS; + size_t pop = popcount_lu(g); + if (pop > (cnt - i)) { + pop = cnt - i; + } + + /* + * Load from memory locations only once, outside the + * hot loop below. + */ + uintptr_t base = (uintptr_t)edata_addr_get(slab); + uintptr_t regsize = (uintptr_t)bin_info->reg_size; + while (pop--) { + size_t bit = cfs_lu(&g); + size_t regind = shift + bit; + /* NOLINTNEXTLINE(performance-no-int-to-ptr) */ + *(ptrs + i) = (void *)(base + regsize * regind); + + i++; + } + slab_data->bitmap[group] = g; + } +#endif + edata_nfree_sub(slab, cnt); +} + +void +bin_slabs_nonfull_insert(bin_t *bin, edata_t *slab) { + assert(edata_nfree_get(slab) > 0); + edata_heap_insert(&bin->slabs_nonfull, slab); + if (config_stats) { + bin->stats.nonfull_slabs++; + } +} + +void +bin_slabs_nonfull_remove(bin_t *bin, edata_t *slab) { + edata_heap_remove(&bin->slabs_nonfull, slab); + if (config_stats) { + bin->stats.nonfull_slabs--; + } +} + +edata_t * +bin_slabs_nonfull_tryget(bin_t *bin) { + edata_t *slab = edata_heap_remove_first(&bin->slabs_nonfull); + if (slab == NULL) { + return NULL; + } + if (config_stats) { + bin->stats.reslabs++; + bin->stats.nonfull_slabs--; + } + return slab; +} + +void +bin_slabs_full_insert(bool is_auto, bin_t *bin, edata_t *slab) { + assert(edata_nfree_get(slab) == 0); + /* + * Tracking extents is required by arena_reset, which is not allowed + * for auto arenas. Bypass this step to avoid touching the edata + * linkage (often results in cache misses) for auto arenas. + */ + if (is_auto) { + return; + } + edata_list_active_append(&bin->slabs_full, slab); +} + +void +bin_slabs_full_remove(bool is_auto, bin_t *bin, edata_t *slab) { + if (is_auto) { + return; + } + edata_list_active_remove(&bin->slabs_full, slab); +} + +void +bin_dissociate_slab(bool is_auto, edata_t *slab, bin_t *bin) { + /* Dissociate slab from bin. */ + if (slab == bin->slabcur) { + bin->slabcur = NULL; + } else { + szind_t binind = edata_szind_get(slab); + const bin_info_t *bin_info = &bin_infos[binind]; + + /* + * The following block's conditional is necessary because if the + * slab only contains one region, then it never gets inserted + * into the non-full slabs heap. + */ + if (bin_info->nregs == 1) { + bin_slabs_full_remove(is_auto, bin, slab); + } else { + bin_slabs_nonfull_remove(bin, slab); + } + } +} + +void +bin_lower_slab(tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin) { + assert(edata_nfree_get(slab) > 0); + + /* + * Make sure that if bin->slabcur is non-NULL, it refers to the + * oldest/lowest non-full slab. It is okay to NULL slabcur out rather + * than proactively keeping it pointing at the oldest/lowest non-full + * slab. + */ + if (bin->slabcur != NULL && edata_snad_comp(bin->slabcur, slab) > 0) { + /* Switch slabcur. */ + if (edata_nfree_get(bin->slabcur) > 0) { + bin_slabs_nonfull_insert(bin, bin->slabcur); + } else { + bin_slabs_full_insert(is_auto, bin, bin->slabcur); + } + bin->slabcur = slab; + if (config_stats) { + bin->stats.reslabs++; + } + } else { + bin_slabs_nonfull_insert(bin, slab); + } +} + +void +bin_dalloc_slab_prepare(tsdn_t *tsdn, edata_t *slab, bin_t *bin) { + malloc_mutex_assert_owner(tsdn, &bin->lock); + + assert(slab != bin->slabcur); + if (config_stats) { + bin->stats.curslabs--; + } +} + +void +bin_dalloc_locked_handle_newly_empty( + tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin) { + bin_dissociate_slab(is_auto, slab, bin); + bin_dalloc_slab_prepare(tsdn, slab, bin); +} + +void +bin_dalloc_locked_handle_newly_nonempty( + tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin) { + bin_slabs_full_remove(is_auto, bin, slab); + bin_lower_slab(tsdn, is_auto, slab, bin); +} + +void +bin_refill_slabcur_with_fresh_slab(tsdn_t *tsdn, bin_t *bin, + szind_t binind, edata_t *fresh_slab) { + malloc_mutex_assert_owner(tsdn, &bin->lock); + /* Only called after slabcur and nonfull both failed. */ + assert(bin->slabcur == NULL); + assert(edata_heap_first(&bin->slabs_nonfull) == NULL); + assert(fresh_slab != NULL); + + /* A new slab from arena_slab_alloc() */ + assert(edata_nfree_get(fresh_slab) == bin_infos[binind].nregs); + if (config_stats) { + bin->stats.nslabs++; + bin->stats.curslabs++; + } + bin->slabcur = fresh_slab; +} + +void * +bin_malloc_with_fresh_slab(tsdn_t *tsdn, bin_t *bin, + szind_t binind, edata_t *fresh_slab) { + malloc_mutex_assert_owner(tsdn, &bin->lock); + bin_refill_slabcur_with_fresh_slab(tsdn, bin, binind, fresh_slab); + + return bin_slab_reg_alloc(bin->slabcur, &bin_infos[binind]); +} + +bool +bin_refill_slabcur_no_fresh_slab(tsdn_t *tsdn, bool is_auto, bin_t *bin) { + malloc_mutex_assert_owner(tsdn, &bin->lock); + /* Only called after bin_slab_reg_alloc[_batch] failed. */ + assert(bin->slabcur == NULL || edata_nfree_get(bin->slabcur) == 0); + + if (bin->slabcur != NULL) { + bin_slabs_full_insert(is_auto, bin, bin->slabcur); + } + + /* Look for a usable slab. */ + bin->slabcur = bin_slabs_nonfull_tryget(bin); + assert(bin->slabcur == NULL || edata_nfree_get(bin->slabcur) > 0); + + return (bin->slabcur == NULL); +} + +void * +bin_malloc_no_fresh_slab(tsdn_t *tsdn, bool is_auto, bin_t *bin, + szind_t binind) { + malloc_mutex_assert_owner(tsdn, &bin->lock); + if (bin->slabcur == NULL || edata_nfree_get(bin->slabcur) == 0) { + if (bin_refill_slabcur_no_fresh_slab(tsdn, is_auto, bin)) { + return NULL; + } + } + + assert(bin->slabcur != NULL && edata_nfree_get(bin->slabcur) > 0); + return bin_slab_reg_alloc(bin->slabcur, &bin_infos[binind]); +} + +bin_t * +bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind, + unsigned *binshard_p) { + unsigned binshard; + if (tsdn_null(tsdn) || tsd_arena_get(tsdn_tsd(tsdn)) == NULL) { + binshard = 0; + } else { + binshard = tsd_binshardsp_get(tsdn_tsd(tsdn))->binshard[binind]; + } + assert(binshard < bin_infos[binind].n_shards); + if (binshard_p != NULL) { + *binshard_p = binshard; + } + return arena_get_bin(arena, binind, binshard); +} diff --git a/dep/jemalloc/src/bin_info.c b/dep/jemalloc/src/bin_info.c index 8629ef881..e10042fdd 100644 --- a/dep/jemalloc/src/bin_info.c +++ b/dep/jemalloc/src/bin_info.c @@ -10,12 +10,12 @@ bin_infos_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], bin_info_t infos[SC_NBINS]) { for (unsigned i = 0; i < SC_NBINS; i++) { bin_info_t *bin_info = &infos[i]; - sc_t *sc = &sc_data->sc[i]; + sc_t *sc = &sc_data->sc[i]; bin_info->reg_size = ((size_t)1U << sc->lg_base) + ((size_t)sc->ndelta << sc->lg_delta); bin_info->slab_size = (sc->pgs << LG_PAGE); - bin_info->nregs = - (uint32_t)(bin_info->slab_size / bin_info->reg_size); + bin_info->nregs = (uint32_t)(bin_info->slab_size + / bin_info->reg_size); bin_info->n_shards = bin_shard_sizes[i]; bitmap_info_t bitmap_info = BITMAP_INFO_INITIALIZER( bin_info->nregs); diff --git a/dep/jemalloc/src/bitmap.c b/dep/jemalloc/src/bitmap.c index 0ccedc5db..8ac81a678 100644 --- a/dep/jemalloc/src/bitmap.c +++ b/dep/jemalloc/src/bitmap.c @@ -10,7 +10,7 @@ void bitmap_info_init(bitmap_info_t *binfo, size_t nbits) { unsigned i; - size_t group_count; + size_t group_count; assert(nbits > 0); assert(nbits <= (ZU(1) << LG_BITMAP_MAXBITS)); @@ -24,11 +24,11 @@ bitmap_info_init(bitmap_info_t *binfo, size_t nbits) { group_count = BITMAP_BITS2GROUPS(nbits); for (i = 1; group_count > 1; i++) { assert(i < BITMAP_MAX_LEVELS); - binfo->levels[i].group_offset = binfo->levels[i-1].group_offset - + group_count; + binfo->levels[i].group_offset = + binfo->levels[i - 1].group_offset + group_count; group_count = BITMAP_BITS2GROUPS(group_count); } - binfo->levels[i].group_offset = binfo->levels[i-1].group_offset + binfo->levels[i].group_offset = binfo->levels[i - 1].group_offset + group_count; assert(binfo->levels[i].group_offset <= BITMAP_GROUPS_MAX); binfo->nlevels = i; @@ -42,7 +42,7 @@ bitmap_info_ngroups(const bitmap_info_t *binfo) { void bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo, bool fill) { - size_t extra; + size_t extra; unsigned i; /* @@ -69,12 +69,13 @@ bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo, bool fill) { bitmap[binfo->levels[1].group_offset - 1] >>= extra; } for (i = 1; i < binfo->nlevels; i++) { - size_t group_count = binfo->levels[i].group_offset - - binfo->levels[i-1].group_offset; - extra = (BITMAP_GROUP_NBITS - (group_count & - BITMAP_GROUP_NBITS_MASK)) & BITMAP_GROUP_NBITS_MASK; + size_t group_count = binfo->levels[i].group_offset + - binfo->levels[i - 1].group_offset; + extra = (BITMAP_GROUP_NBITS + - (group_count & BITMAP_GROUP_NBITS_MASK)) + & BITMAP_GROUP_NBITS_MASK; if (extra != 0) { - bitmap[binfo->levels[i+1].group_offset - 1] >>= extra; + bitmap[binfo->levels[i + 1].group_offset - 1] >>= extra; } } } diff --git a/dep/jemalloc/src/buf_writer.c b/dep/jemalloc/src/buf_writer.c index 7c6f79403..3c298502e 100644 --- a/dep/jemalloc/src/buf_writer.c +++ b/dep/jemalloc/src/buf_writer.c @@ -43,8 +43,9 @@ buf_writer_init(tsdn_t *tsdn, buf_writer_t *buf_writer, write_cb_t *write_cb, if (write_cb != NULL) { buf_writer->write_cb = write_cb; } else { - buf_writer->write_cb = je_malloc_message != NULL ? - je_malloc_message : wrtmessage; + buf_writer->write_cb = je_malloc_message != NULL + ? je_malloc_message + : wrtmessage; } buf_writer->cbopaque = cbopaque; assert(buf_len >= 2); @@ -52,8 +53,8 @@ buf_writer_init(tsdn_t *tsdn, buf_writer_t *buf_writer, write_cb_t *write_cb, buf_writer->buf = buf; buf_writer->internal_buf = false; } else { - buf_writer->buf = buf_writer_allocate_internal_buf(tsdn, - buf_len); + buf_writer->buf = buf_writer_allocate_internal_buf( + tsdn, buf_len); buf_writer->internal_buf = true; } if (buf_writer->buf != NULL) { @@ -111,13 +112,13 @@ buf_writer_terminate(tsdn_t *tsdn, buf_writer_t *buf_writer) { } void -buf_writer_pipe(buf_writer_t *buf_writer, read_cb_t *read_cb, - void *read_cbopaque) { +buf_writer_pipe( + buf_writer_t *buf_writer, read_cb_t *read_cb, void *read_cbopaque) { /* * A tiny local buffer in case the buffered writer failed to allocate * at init. */ - static char backup_buf[16]; + static char backup_buf[16]; static buf_writer_t backup_buf_writer; buf_writer_assert(buf_writer); diff --git a/dep/jemalloc/src/cache_bin.c b/dep/jemalloc/src/cache_bin.c index 9ae072a0e..ec6779484 100644 --- a/dep/jemalloc/src/cache_bin.c +++ b/dep/jemalloc/src/cache_bin.c @@ -5,17 +5,29 @@ #include "jemalloc/internal/cache_bin.h" #include "jemalloc/internal/safety_check.h" +const uintptr_t disabled_bin = JUNK_ADDR; + void -cache_bin_info_init(cache_bin_info_t *info, - cache_bin_sz_t ncached_max) { +cache_bin_info_init(cache_bin_info_t *info, cache_bin_sz_t ncached_max) { assert(ncached_max <= CACHE_BIN_NCACHED_MAX); size_t stack_size = (size_t)ncached_max * sizeof(void *); assert(stack_size < ((size_t)1 << (sizeof(cache_bin_sz_t) * 8))); info->ncached_max = (cache_bin_sz_t)ncached_max; } +bool +cache_bin_stack_use_thp(void) { + /* + * If metadata_thp is enabled, allocating tcache stack from the base + * allocator for efficiency gains. The downside, however, is that base + * allocator never purges freed memory, and may cache a fair amount of + * memory after many threads are terminated and not reused. + */ + return metadata_thp_enabled(); +} + void -cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos, +cache_bin_info_compute_alloc(const cache_bin_info_t *infos, szind_t ninfos, size_t *size, size_t *alignment) { /* For the total bin stack region (per tcache), reserve 2 more slots so * that @@ -26,74 +38,80 @@ cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos, */ *size = sizeof(void *) * 2; for (szind_t i = 0; i < ninfos; i++) { - assert(infos[i].ncached_max > 0); *size += infos[i].ncached_max * sizeof(void *); } /* - * Align to at least PAGE, to minimize the # of TLBs needed by the - * smaller sizes; also helps if the larger sizes don't get used at all. + * When not using THP, align to at least PAGE, to minimize the # of TLBs + * needed by the smaller sizes; also helps if the larger sizes don't get + * used at all. */ - *alignment = PAGE; + *alignment = cache_bin_stack_use_thp() ? QUANTUM : PAGE; } void -cache_bin_preincrement(cache_bin_info_t *infos, szind_t ninfos, void *alloc, - size_t *cur_offset) { +cache_bin_preincrement(const cache_bin_info_t *infos, szind_t ninfos, + void *alloc, size_t *cur_offset) { if (config_debug) { size_t computed_size; size_t computed_alignment; /* Pointer should be as aligned as we asked for. */ - cache_bin_info_compute_alloc(infos, ninfos, &computed_size, - &computed_alignment); + cache_bin_info_compute_alloc( + infos, ninfos, &computed_size, &computed_alignment); assert(((uintptr_t)alloc & (computed_alignment - 1)) == 0); } - *(uintptr_t *)((uintptr_t)alloc + *cur_offset) = - cache_bin_preceding_junk; + *(uintptr_t *)((byte_t *)alloc + + *cur_offset) = cache_bin_preceding_junk; *cur_offset += sizeof(void *); } void -cache_bin_postincrement(cache_bin_info_t *infos, szind_t ninfos, void *alloc, - size_t *cur_offset) { - *(uintptr_t *)((uintptr_t)alloc + *cur_offset) = - cache_bin_trailing_junk; +cache_bin_postincrement(void *alloc, size_t *cur_offset) { + *(uintptr_t *)((byte_t *)alloc + *cur_offset) = cache_bin_trailing_junk; *cur_offset += sizeof(void *); } void -cache_bin_init(cache_bin_t *bin, cache_bin_info_t *info, void *alloc, +cache_bin_init(cache_bin_t *bin, const cache_bin_info_t *info, void *alloc, size_t *cur_offset) { /* * The full_position points to the lowest available space. Allocations * will access the slots toward higher addresses (for the benefit of * adjacent prefetch). */ - void *stack_cur = (void *)((uintptr_t)alloc + *cur_offset); - void *full_position = stack_cur; - uint16_t bin_stack_size = info->ncached_max * sizeof(void *); + void *stack_cur = (void *)((byte_t *)alloc + *cur_offset); + void *full_position = stack_cur; + cache_bin_sz_t bin_stack_size = info->ncached_max * sizeof(void *); *cur_offset += bin_stack_size; - void *empty_position = (void *)((uintptr_t)alloc + *cur_offset); + void *empty_position = (void *)((byte_t *)alloc + *cur_offset); /* Init to the empty position. */ bin->stack_head = (void **)empty_position; - bin->low_bits_low_water = (uint16_t)(uintptr_t)bin->stack_head; - bin->low_bits_full = (uint16_t)(uintptr_t)full_position; - bin->low_bits_empty = (uint16_t)(uintptr_t)empty_position; - cache_bin_sz_t free_spots = cache_bin_diff(bin, - bin->low_bits_full, (uint16_t)(uintptr_t)bin->stack_head, - /* racy */ false); + bin->low_bits_low_water = (cache_bin_sz_t)(uintptr_t)bin->stack_head; + bin->low_bits_full = (cache_bin_sz_t)(uintptr_t)full_position; + bin->low_bits_empty = (cache_bin_sz_t)(uintptr_t)empty_position; + cache_bin_info_init(&bin->bin_info, info->ncached_max); + cache_bin_sz_t free_spots = cache_bin_diff(bin, bin->low_bits_full, + (cache_bin_sz_t)(uintptr_t)bin->stack_head); assert(free_spots == bin_stack_size); - assert(cache_bin_ncached_get_local(bin, info) == 0); + if (!cache_bin_disabled(bin)) { + assert(cache_bin_ncached_get_local(bin) == 0); + } assert(cache_bin_empty_position_get(bin) == empty_position); assert(bin_stack_size > 0 || empty_position == full_position); } -bool -cache_bin_still_zero_initialized(cache_bin_t *bin) { - return bin->stack_head == NULL; +void +cache_bin_init_disabled(cache_bin_t *bin, cache_bin_sz_t ncached_max) { + const void *fake_stack = cache_bin_disabled_bin_stack(); + size_t fake_offset = 0; + cache_bin_info_t fake_info; + cache_bin_info_init(&fake_info, 0); + cache_bin_init(bin, &fake_info, (void *)fake_stack, &fake_offset); + cache_bin_info_init(&bin->bin_info, ncached_max); + assert(fake_offset == 0); } diff --git a/dep/jemalloc/src/ckh.c b/dep/jemalloc/src/ckh.c index 8db4319c5..80688162a 100644 --- a/dep/jemalloc/src/ckh.c +++ b/dep/jemalloc/src/ckh.c @@ -49,8 +49,8 @@ /******************************************************************************/ /* Function prototypes for non-inline static functions. */ -static bool ckh_grow(tsd_t *tsd, ckh_t *ckh); -static void ckh_shrink(tsd_t *tsd, ckh_t *ckh); +static bool ckh_grow(tsd_t *tsd, ckh_t *ckh); +static void ckh_shrink(tsd_t *tsd, ckh_t *ckh); /******************************************************************************/ @@ -60,7 +60,7 @@ static void ckh_shrink(tsd_t *tsd, ckh_t *ckh); */ static size_t ckh_bucket_search(ckh_t *ckh, size_t bucket, const void *key) { - ckhc_t *cell; + ckhc_t *cell; unsigned i; for (i = 0; i < (ZU(1) << LG_CKH_BUCKET_CELLS); i++) { @@ -98,20 +98,20 @@ ckh_isearch(ckh_t *ckh, const void *key) { } static bool -ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key, - const void *data) { - ckhc_t *cell; +ckh_try_bucket_insert( + ckh_t *ckh, size_t bucket, const void *key, const void *data) { + ckhc_t *cell; unsigned offset, i; /* * Cycle through the cells in the bucket, starting at a random position. * The randomness avoids worst-case search overhead as buckets fill up. */ - offset = (unsigned)prng_lg_range_u64(&ckh->prng_state, - LG_CKH_BUCKET_CELLS); + offset = (unsigned)prng_lg_range_u64( + &ckh->prng_state, LG_CKH_BUCKET_CELLS); for (i = 0; i < (ZU(1) << LG_CKH_BUCKET_CELLS); i++) { - cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + - ((i + offset) & ((ZU(1) << LG_CKH_BUCKET_CELLS) - 1))]; + cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + + ((i + offset) & ((ZU(1) << LG_CKH_BUCKET_CELLS) - 1))]; if (cell->key == NULL) { cell->key = key; cell->data = data; @@ -130,12 +130,12 @@ ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key, * eviction/relocation bucket cycle. */ static bool -ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey, - void const **argdata) { +ckh_evict_reloc_insert( + ckh_t *ckh, size_t argbucket, void const **argkey, void const **argdata) { const void *key, *data, *tkey, *tdata; - ckhc_t *cell; - size_t hashes[2], bucket, tbucket; - unsigned i; + ckhc_t *cell; + size_t hashes[2], bucket, tbucket; + unsigned i; bucket = argbucket; key = *argkey; @@ -149,15 +149,18 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey, * were an item for which both hashes indicated the same * bucket. */ - i = (unsigned)prng_lg_range_u64(&ckh->prng_state, - LG_CKH_BUCKET_CELLS); + i = (unsigned)prng_lg_range_u64( + &ckh->prng_state, LG_CKH_BUCKET_CELLS); cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + i]; assert(cell->key != NULL); /* Swap cell->{key,data} and {key,data} (evict). */ - tkey = cell->key; tdata = cell->data; - cell->key = key; cell->data = data; - key = tkey; data = tdata; + tkey = cell->key; + tdata = cell->data; + cell->key = key; + cell->data = data; + key = tkey; + data = tdata; #ifdef CKH_COUNT ckh->nrelocs++; @@ -167,8 +170,8 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey, ckh->hash(key, hashes); tbucket = hashes[1] & ((ZU(1) << ckh->lg_curbuckets) - 1); if (tbucket == bucket) { - tbucket = hashes[0] & ((ZU(1) << ckh->lg_curbuckets) - - 1); + tbucket = hashes[0] + & ((ZU(1) << ckh->lg_curbuckets) - 1); /* * It may be that (tbucket == bucket) still, if the * item's hashes both indicate this bucket. However, @@ -201,8 +204,8 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey, } static bool -ckh_try_insert(ckh_t *ckh, void const**argkey, void const**argdata) { - size_t hashes[2], bucket; +ckh_try_insert(ckh_t *ckh, void const **argkey, void const **argdata) { + size_t hashes[2], bucket; const void *key = *argkey; const void *data = *argdata; @@ -232,7 +235,7 @@ ckh_try_insert(ckh_t *ckh, void const**argkey, void const**argdata) { */ static bool ckh_rebuild(ckh_t *ckh, ckhc_t *aTab) { - size_t count, i, nins; + size_t count, i, nins; const void *key, *data; count = ckh->count; @@ -254,8 +257,8 @@ ckh_rebuild(ckh_t *ckh, ckhc_t *aTab) { static bool ckh_grow(tsd_t *tsd, ckh_t *ckh) { - bool ret; - ckhc_t *tab, *ttab; + bool ret; + ckhc_t *tab, *ttab; unsigned lg_prevbuckets, lg_curcells; #ifdef CKH_COUNT @@ -274,8 +277,7 @@ ckh_grow(tsd_t *tsd, ckh_t *ckh) { lg_curcells++; usize = sz_sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE); - if (unlikely(usize == 0 - || usize > SC_LARGE_MAXCLASS)) { + if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { ret = true; goto label_return; } @@ -309,8 +311,8 @@ label_return: static void ckh_shrink(tsd_t *tsd, ckh_t *ckh) { - ckhc_t *tab, *ttab; - size_t usize; + ckhc_t *tab, *ttab; + size_t usize; unsigned lg_prevbuckets, lg_curcells; /* @@ -358,8 +360,8 @@ ckh_shrink(tsd_t *tsd, ckh_t *ckh) { bool ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *ckh_hash, ckh_keycomp_t *keycomp) { - bool ret; - size_t mincells, usize; + bool ret; + size_t mincells, usize; unsigned lg_mincells; assert(minitems > 0); @@ -386,8 +388,7 @@ ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *ckh_hash, assert(LG_CKH_BUCKET_CELLS > 0); mincells = ((minitems + (3 - (minitems % 3))) / 3) << 2; for (lg_mincells = LG_CKH_BUCKET_CELLS; - (ZU(1) << lg_mincells) < mincells; - lg_mincells++) { + (ZU(1) << lg_mincells) < mincells; lg_mincells++) { /* Do nothing. */ } ckh->lg_minbuckets = lg_mincells - LG_CKH_BUCKET_CELLS; @@ -417,11 +418,12 @@ ckh_delete(tsd_t *tsd, ckh_t *ckh) { assert(ckh != NULL); #ifdef CKH_VERBOSE - malloc_printf( - "%s(%p): ngrows: %"FMTu64", nshrinks: %"FMTu64"," - " nshrinkfails: %"FMTu64", ninserts: %"FMTu64"," - " nrelocs: %"FMTu64"\n", __func__, ckh, - (unsigned long long)ckh->ngrows, + malloc_printf("%s(%p): ngrows: %" FMTu64 ", nshrinks: %" FMTu64 + "," + " nshrinkfails: %" FMTu64 ", ninserts: %" FMTu64 + "," + " nrelocs: %" FMTu64 "\n", + __func__, ckh, (unsigned long long)ckh->ngrows, (unsigned long long)ckh->nshrinks, (unsigned long long)ckh->nshrinkfails, (unsigned long long)ckh->ninserts, @@ -445,8 +447,9 @@ bool ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data) { size_t i, ncells; - for (i = *tabind, ncells = (ZU(1) << (ckh->lg_curbuckets + - LG_CKH_BUCKET_CELLS)); i < ncells; i++) { + for (i = *tabind, + ncells = (ZU(1) << (ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS)); + i < ncells; i++) { if (ckh->tab[i].key != NULL) { if (key != NULL) { *key = (void *)ckh->tab[i].key; @@ -486,8 +489,8 @@ label_return: } bool -ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key, - void **data) { +ckh_remove( + tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key, void **data) { size_t cell; assert(ckh != NULL); @@ -505,9 +508,9 @@ ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key, ckh->count--; /* Try to halve the table if it is less than 1/4 full. */ - if (ckh->count < (ZU(1) << (ckh->lg_curbuckets - + LG_CKH_BUCKET_CELLS - 2)) && ckh->lg_curbuckets - > ckh->lg_minbuckets) { + if (ckh->count < (ZU(1) + << (ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS - 2)) + && ckh->lg_curbuckets > ckh->lg_minbuckets) { /* Ignore error due to OOM. */ ckh_shrink(tsd, ckh); } @@ -554,8 +557,8 @@ ckh_string_keycomp(const void *k1, const void *k2) { void ckh_pointer_hash(const void *key, size_t r_hash[2]) { union { - const void *v; - size_t i; + const void *v; + size_t i; } u; assert(sizeof(u.v) == sizeof(u.i)); diff --git a/dep/jemalloc/src/conf.c b/dep/jemalloc/src/conf.c new file mode 100644 index 000000000..65abcd259 --- /dev/null +++ b/dep/jemalloc/src/conf.c @@ -0,0 +1,1203 @@ +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" + +#include "jemalloc/internal/assert.h" +#include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/extent_dss.h" +#include "jemalloc/internal/extent_mmap.h" +#include "jemalloc/internal/fxp.h" +#include "jemalloc/internal/log.h" +#include "jemalloc/internal/malloc_io.h" +#include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/nstime.h" +#include "jemalloc/internal/safety_check.h" +#include "jemalloc/internal/san.h" +#include "jemalloc/internal/sc.h" +#include "jemalloc/internal/util.h" + +#include "jemalloc/internal/conf.h" + +/* Whether encountered any invalid config options. */ +bool had_conf_error; + +static char * +jemalloc_getenv(const char *name) { +#ifdef JEMALLOC_FORCE_GETENV + return getenv(name); +#else +# ifdef JEMALLOC_HAVE_SECURE_GETENV + return secure_getenv(name); +# else +# ifdef JEMALLOC_HAVE_ISSETUGID + if (issetugid() != 0) { + return NULL; + } +# endif + return getenv(name); +# endif +#endif +} + +static void +init_opt_stats_opts(const char *v, size_t vlen, char *dest) { + size_t opts_len = strlen(dest); + assert(opts_len <= stats_print_tot_num_options); + + for (size_t i = 0; i < vlen; i++) { + switch (v[i]) { +#define OPTION(o, v, d, s) \ + case o: \ + break; + STATS_PRINT_OPTIONS +#undef OPTION + default: + continue; + } + + if (strchr(dest, v[i]) != NULL) { + /* Ignore repeated. */ + continue; + } + + dest[opts_len++] = v[i]; + dest[opts_len] = '\0'; + assert(opts_len <= stats_print_tot_num_options); + } + assert(opts_len == strlen(dest)); +} + +static void +malloc_conf_format_error(const char *msg, const char *begin, const char *end) { + size_t len = end - begin + 1; + len = len > BUFERROR_BUF ? BUFERROR_BUF : len; + + malloc_printf(": %s -- %.*s\n", msg, (int)len, begin); +} + +JET_EXTERN bool +conf_next(char const **opts_p, char const **k_p, size_t *klen_p, + char const **v_p, size_t *vlen_p) { + bool accept; + const char *opts = *opts_p; + + *k_p = opts; + + for (accept = false; !accept;) { + switch (*opts) { + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': + case 'F': + case 'G': + case 'H': + case 'I': + case 'J': + case 'K': + case 'L': + case 'M': + case 'N': + case 'O': + case 'P': + case 'Q': + case 'R': + case 'S': + case 'T': + case 'U': + case 'V': + case 'W': + case 'X': + case 'Y': + case 'Z': + case 'a': + case 'b': + case 'c': + case 'd': + case 'e': + case 'f': + case 'g': + case 'h': + case 'i': + case 'j': + case 'k': + case 'l': + case 'm': + case 'n': + case 'o': + case 'p': + case 'q': + case 'r': + case 's': + case 't': + case 'u': + case 'v': + case 'w': + case 'x': + case 'y': + case 'z': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case '_': + opts++; + break; + case ':': + opts++; + *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; + *v_p = opts; + accept = true; + break; + case '\0': + if (opts != *opts_p) { + malloc_conf_format_error( + "Conf string ends with key", *opts_p, + opts - 1); + had_conf_error = true; + } + return true; + default: + malloc_conf_format_error( + "Malformed conf string", *opts_p, opts); + had_conf_error = true; + return true; + } + } + + for (accept = false; !accept;) { + switch (*opts) { + case ',': + opts++; + /* + * Look ahead one character here, because the next time + * this function is called, it will assume that end of + * input has been cleanly reached if no input remains, + * but we have optimistically already consumed the + * comma if one exists. + */ + if (*opts == '\0') { + malloc_conf_format_error( + "Conf string ends with comma", *opts_p, + opts - 1); + had_conf_error = true; + } + *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; + accept = true; + break; + case '\0': + *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; + accept = true; + break; + default: + opts++; + break; + } + } + + *opts_p = opts; + return false; +} + +void +malloc_abort_invalid_conf(void) { + assert(opt_abort_conf); + malloc_printf( + ": Abort (abort_conf:true) on invalid conf " + "value (see above).\n"); + invalid_conf_abort(); +} + +JET_EXTERN void +conf_error( + const char *msg, const char *k, size_t klen, const char *v, size_t vlen) { + malloc_printf( + ": %s: %.*s:%.*s\n", msg, (int)klen, k, (int)vlen, v); + /* If abort_conf is set, error out after processing all options. */ + const char *experimental = "experimental_"; + if (strncmp(k, experimental, strlen(experimental)) == 0) { + /* However, tolerate experimental features. */ + return; + } + const char *deprecated[] = {"hpa_sec_bytes_after_flush"}; + const size_t deprecated_cnt = (sizeof(deprecated) + / sizeof(deprecated[0])); + for (size_t i = 0; i < deprecated_cnt; ++i) { + if (strncmp(k, deprecated[i], strlen(deprecated[i])) == 0) { + /* Tolerate deprecated features. */ + return; + } + } + had_conf_error = true; +} + +JET_EXTERN bool +conf_handle_bool(const char *v, size_t vlen, bool *result) { + if (sizeof("true") - 1 == vlen && strncmp("true", v, vlen) == 0) { + *result = true; + } else if (sizeof("false") - 1 == vlen + && strncmp("false", v, vlen) == 0) { + *result = false; + } else { + return true; + } + return false; +} + +JEMALLOC_DIAGNOSTIC_PUSH +JEMALLOC_DIAGNOSTIC_IGNORE("-Wunused-function") + +JET_EXTERN bool +conf_handle_signed(const char *v, size_t vlen, intmax_t min, intmax_t max, + bool check_min, bool check_max, bool clip, intmax_t *result) { + char *end; + set_errno(0); + intmax_t mv = (intmax_t)malloc_strtoumax(v, &end, 0); + if (get_errno() != 0 || (uintptr_t)end - (uintptr_t)v != vlen) { + return true; + } + if (clip) { + if (check_min && mv < min) { + *result = min; + } else if (check_max && mv > max) { + *result = max; + } else { + *result = mv; + } + } else { + if ((check_min && mv < min) || (check_max && mv > max)) { + return true; + } + *result = mv; + } + return false; +} + +JET_EXTERN bool +conf_handle_char_p(const char *v, size_t vlen, char *dest, size_t dest_sz) { + if (dest_sz == 0) { + return false; + } + size_t cpylen = (vlen <= dest_sz - 1) ? vlen : dest_sz - 1; + strncpy(dest, v, cpylen); + dest[cpylen] = '\0'; + return false; +} + +JEMALLOC_DIAGNOSTIC_POP + +/* Number of sources for initializing malloc_conf */ +#define MALLOC_CONF_NSOURCES 5 + +static const char * +obtain_malloc_conf(unsigned which_source, char readlink_buf[PATH_MAX + 1]) { + if (config_debug) { + static unsigned read_source = 0; + /* + * Each source should only be read once, to minimize # of + * syscalls on init. + */ + assert(read_source == which_source); + read_source++; + } + assert(which_source < MALLOC_CONF_NSOURCES); + + const char *ret; + switch (which_source) { + case 0: + ret = config_malloc_conf; + break; + case 1: + if (je_malloc_conf != NULL) { + /* Use options that were compiled into the program. */ + ret = je_malloc_conf; + } else { + /* No configuration specified. */ + ret = NULL; + } + break; + case 2: { +#ifndef JEMALLOC_CONFIG_FILE + ret = NULL; + break; +#else + ssize_t linklen = 0; +# ifndef _WIN32 + int saved_errno = errno; + const char *linkname = +# ifdef JEMALLOC_PREFIX + "/etc/" JEMALLOC_PREFIX "malloc.conf" +# else + "/etc/malloc.conf" +# endif + ; + + /* + * Try to use the contents of the "/etc/malloc.conf" symbolic + * link's name. + */ +# ifndef JEMALLOC_READLINKAT + linklen = readlink(linkname, readlink_buf, PATH_MAX); +# else + linklen = readlinkat( + AT_FDCWD, linkname, readlink_buf, PATH_MAX); +# endif + if (linklen == -1) { + /* No configuration specified. */ + linklen = 0; + /* Restore errno. */ + set_errno(saved_errno); + } +# endif + readlink_buf[linklen] = '\0'; + ret = readlink_buf; + break; +#endif + } + case 3: { +#ifndef JEMALLOC_CONFIG_ENV + ret = NULL; + break; +#else + const char *envname = +# ifdef JEMALLOC_PREFIX + JEMALLOC_CPREFIX "MALLOC_CONF" +# else + "MALLOC_CONF" +# endif + ; + + if ((ret = jemalloc_getenv(envname)) != NULL) { + opt_malloc_conf_env_var = ret; + } else { + /* No configuration specified. */ + ret = NULL; + } + break; +#endif + } + case 4: { + ret = je_malloc_conf_2_conf_harder; + break; + } + default: + not_reached(); + ret = NULL; + } + return ret; +} + +static void +validate_hpa_settings(void) { + if (!hpa_supported() || !opt_hpa) { + return; + } + if (HUGEPAGE > HUGEPAGE_MAX_EXPECTED_SIZE) { + had_conf_error = true; + malloc_printf( + ": huge page size (%zu) greater than expected." + "May not be supported or behave as expected.", + HUGEPAGE); + } +#ifndef JEMALLOC_HAVE_MADVISE_COLLAPSE + if (opt_hpa_opts.hugify_sync) { + had_conf_error = true; + malloc_printf( + ": hpa_hugify_sync config option is enabled, " + "but MADV_COLLAPSE support was not detected at build " + "time."); + } +#endif +} + +static void +malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], + bool initial_call, const char *opts_cache[MALLOC_CONF_NSOURCES], + char readlink_buf[PATH_MAX + 1]) { + static const char *opts_explain[MALLOC_CONF_NSOURCES] = { + "string specified via --with-malloc-conf", + "string pointed to by the global variable malloc_conf", + "\"name\" of the file referenced by the symbolic link named " + "/etc/malloc.conf", + "value of the environment variable MALLOC_CONF", + "string pointed to by the global variable " + "malloc_conf_2_conf_harder", + }; + unsigned i; + const char *opts, *k, *v; + size_t klen, vlen; + + for (i = 0; i < MALLOC_CONF_NSOURCES; i++) { + /* Get runtime configuration. */ + if (initial_call) { + opts_cache[i] = obtain_malloc_conf(i, readlink_buf); + } + opts = opts_cache[i]; + if (!initial_call && opt_confirm_conf) { + malloc_printf( + ": malloc_conf #%u (%s): \"%s\"\n", i + 1, + opts_explain[i], opts != NULL ? opts : ""); + } + if (opts == NULL) { + continue; + } + + while ( + *opts != '\0' && !conf_next(&opts, &k, &klen, &v, &vlen)) { +#define CONF_ERROR(msg, k, klen, v, vlen) \ + if (!initial_call) { \ + conf_error(msg, k, klen, v, vlen); \ + cur_opt_valid = false; \ + } +#define CONF_CONTINUE \ + { \ + if (!initial_call && opt_confirm_conf && cur_opt_valid) { \ + malloc_printf( \ + ": -- " \ + "Set conf value: %.*s:%.*s" \ + "\n", \ + (int)klen, k, (int)vlen, v); \ + } \ + continue; \ + } +#define CONF_MATCH(n) (sizeof(n) - 1 == klen && strncmp(n, k, klen) == 0) +#define CONF_MATCH_VALUE(n) (sizeof(n) - 1 == vlen && strncmp(n, v, vlen) == 0) +#define CONF_HANDLE_BOOL(o, n) \ + if (CONF_MATCH(n)) { \ + if (conf_handle_bool(v, vlen, &o)) { \ + CONF_ERROR("Invalid conf value", k, klen, v, vlen); \ + } \ + CONF_CONTINUE; \ + } + /* + * One of the CONF_MIN macros below expands, in one of the use points, + * to "unsigned integer < 0", which is always false, triggering the + * GCC -Wtype-limits warning, which we disable here and re-enable below. + */ + JEMALLOC_DIAGNOSTIC_PUSH + JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS + +#define CONF_DONT_CHECK_MIN(um, min) false +#define CONF_CHECK_MIN(um, min) ((um) < (min)) +#define CONF_DONT_CHECK_MAX(um, max) false +#define CONF_CHECK_MAX(um, max) ((um) > (max)) + +#define CONF_VALUE_READ(max_t, result) \ + char *end; \ + set_errno(0); \ + result = (max_t)malloc_strtoumax(v, &end, 0); +#define CONF_VALUE_READ_FAIL() \ + (get_errno() != 0 || (uintptr_t)end - (uintptr_t)v != vlen) + +#define CONF_HANDLE_T(t, max_t, o, n, min, max, check_min, check_max, clip) \ + if (CONF_MATCH(n)) { \ + max_t mv; \ + CONF_VALUE_READ(max_t, mv) \ + if (CONF_VALUE_READ_FAIL()) { \ + CONF_ERROR("Invalid conf value", k, klen, v, vlen); \ + } else if (clip) { \ + if (check_min(mv, (t)(min))) { \ + o = (t)(min); \ + } else if (check_max(mv, (t)(max))) { \ + o = (t)(max); \ + } else { \ + o = (t)mv; \ + } \ + } else { \ + if (check_min(mv, (t)(min)) \ + || check_max(mv, (t)(max))) { \ + CONF_ERROR( \ + "Out-of-range " \ + "conf value", \ + k, klen, v, vlen); \ + } else { \ + o = (t)mv; \ + } \ + } \ + CONF_CONTINUE; \ + } +#define CONF_HANDLE_T_U(t, o, n, min, max, check_min, check_max, clip) \ + CONF_HANDLE_T(t, uintmax_t, o, n, min, max, check_min, check_max, clip) +#define CONF_HANDLE_T_SIGNED(t, o, n, min, max, check_min, check_max, clip) \ + CONF_HANDLE_T(t, intmax_t, o, n, min, max, check_min, check_max, clip) + +#define CONF_HANDLE_UNSIGNED(o, n, min, max, check_min, check_max, clip) \ + CONF_HANDLE_T_U(unsigned, o, n, min, max, check_min, check_max, clip) +#define CONF_HANDLE_SIZE_T(o, n, min, max, check_min, check_max, clip) \ + CONF_HANDLE_T_U(size_t, o, n, min, max, check_min, check_max, clip) +#define CONF_HANDLE_INT64_T(o, n, min, max, check_min, check_max, clip) \ + CONF_HANDLE_T_SIGNED( \ + int64_t, o, n, min, max, check_min, check_max, clip) +#define CONF_HANDLE_UINT64_T(o, n, min, max, check_min, check_max, clip) \ + CONF_HANDLE_T_U(uint64_t, o, n, min, max, check_min, check_max, clip) +#define CONF_HANDLE_SSIZE_T(o, n, min, max) \ + CONF_HANDLE_T_SIGNED( \ + ssize_t, o, n, min, max, CONF_CHECK_MIN, CONF_CHECK_MAX, false) +#define CONF_HANDLE_CHAR_P(o, n, d) \ + if (CONF_MATCH(n)) { \ + size_t cpylen = (vlen <= sizeof(o) - 1) ? vlen \ + : sizeof(o) - 1; \ + strncpy(o, v, cpylen); \ + o[cpylen] = '\0'; \ + CONF_CONTINUE; \ + } + + bool cur_opt_valid = true; + + CONF_HANDLE_BOOL(opt_confirm_conf, "confirm_conf") + if (initial_call) { + continue; + } + + CONF_HANDLE_BOOL(opt_abort, "abort") + CONF_HANDLE_BOOL(opt_abort_conf, "abort_conf") + CONF_HANDLE_BOOL(opt_cache_oblivious, "cache_oblivious") + CONF_HANDLE_BOOL(opt_trust_madvise, "trust_madvise") + CONF_HANDLE_BOOL( + opt_experimental_hpa_start_huge_if_thp_always, + "experimental_hpa_start_huge_if_thp_always") + CONF_HANDLE_BOOL(opt_experimental_hpa_enforce_hugify, + "experimental_hpa_enforce_hugify") + CONF_HANDLE_BOOL( + opt_huge_arena_pac_thp, "huge_arena_pac_thp") + if (strncmp("metadata_thp", k, klen) == 0) { + int m; + bool match = false; + for (m = 0; m < metadata_thp_mode_limit; m++) { + if (strncmp(metadata_thp_mode_names[m], + v, vlen) + == 0) { + opt_metadata_thp = m; + match = true; + break; + } + } + if (!match) { + CONF_ERROR("Invalid conf value", k, + klen, v, vlen); + } + CONF_CONTINUE; + } + CONF_HANDLE_BOOL(opt_retain, "retain") + if (strncmp("dss", k, klen) == 0) { + int m; + bool match = false; + for (m = 0; m < dss_prec_limit; m++) { + if (strncmp(dss_prec_names[m], v, vlen) + == 0) { + if (extent_dss_prec_set(m)) { + CONF_ERROR( + "Error setting dss", + k, klen, v, vlen); + } else { + opt_dss = + dss_prec_names[m]; + match = true; + break; + } + } + } + if (!match) { + CONF_ERROR("Invalid conf value", k, + klen, v, vlen); + } + CONF_CONTINUE; + } + if (CONF_MATCH("narenas")) { + if (CONF_MATCH_VALUE("default")) { + opt_narenas = 0; + CONF_CONTINUE; + } else { + CONF_HANDLE_UNSIGNED(opt_narenas, + "narenas", 1, UINT_MAX, + CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, + /* clip */ false) + } + } + if (CONF_MATCH("narenas_ratio")) { + char *end; + bool err = fxp_parse( + &opt_narenas_ratio, v, &end); + if (err || (size_t)(end - v) != vlen) { + CONF_ERROR("Invalid conf value", k, + klen, v, vlen); + } + CONF_CONTINUE; + } + if (CONF_MATCH("bin_shards")) { + const char *bin_shards_segment_cur = v; + size_t vlen_left = vlen; + do { + size_t size_start; + size_t size_end; + size_t nshards; + bool err = multi_setting_parse_next( + &bin_shards_segment_cur, &vlen_left, + &size_start, &size_end, &nshards); + if (err + || bin_update_shard_size( + bin_shard_sizes, size_start, + size_end, nshards)) { + CONF_ERROR( + "Invalid settings for " + "bin_shards", + k, klen, v, vlen); + break; + } + } while (vlen_left > 0); + CONF_CONTINUE; + } + if (CONF_MATCH("tcache_ncached_max")) { + bool err = tcache_bin_info_default_init( + v, vlen); + if (err) { + CONF_ERROR( + "Invalid settings for " + "tcache_ncached_max", + k, klen, v, vlen); + } + CONF_CONTINUE; + } + CONF_HANDLE_INT64_T(opt_mutex_max_spin, + "mutex_max_spin", -1, INT64_MAX, CONF_CHECK_MIN, + CONF_DONT_CHECK_MAX, false); + CONF_HANDLE_SSIZE_T(opt_dirty_decay_ms, + "dirty_decay_ms", -1, + NSTIME_SEC_MAX * KQU(1000) < QU(SSIZE_MAX) + ? NSTIME_SEC_MAX * KQU(1000) + : SSIZE_MAX); + CONF_HANDLE_SSIZE_T(opt_muzzy_decay_ms, + "muzzy_decay_ms", -1, + NSTIME_SEC_MAX * KQU(1000) < QU(SSIZE_MAX) + ? NSTIME_SEC_MAX * KQU(1000) + : SSIZE_MAX); + CONF_HANDLE_SIZE_T(opt_process_madvise_max_batch, + "process_madvise_max_batch", 0, + PROCESS_MADVISE_MAX_BATCH_LIMIT, + CONF_DONT_CHECK_MIN, CONF_CHECK_MAX, + /* clip */ true) + CONF_HANDLE_BOOL(opt_stats_print, "stats_print") + if (CONF_MATCH("stats_print_opts")) { + init_opt_stats_opts( + v, vlen, opt_stats_print_opts); + CONF_CONTINUE; + } + CONF_HANDLE_INT64_T(opt_stats_interval, + "stats_interval", -1, INT64_MAX, CONF_CHECK_MIN, + CONF_DONT_CHECK_MAX, false) + if (CONF_MATCH("stats_interval_opts")) { + init_opt_stats_opts( + v, vlen, opt_stats_interval_opts); + CONF_CONTINUE; + } + if (config_fill) { + if (CONF_MATCH("junk")) { + if (CONF_MATCH_VALUE("true")) { + opt_junk = "true"; + opt_junk_alloc = opt_junk_free = + true; + } else if (CONF_MATCH_VALUE("false")) { + opt_junk = "false"; + opt_junk_alloc = opt_junk_free = + false; + } else if (CONF_MATCH_VALUE("alloc")) { + opt_junk = "alloc"; + opt_junk_alloc = true; + opt_junk_free = false; + } else if (CONF_MATCH_VALUE("free")) { + opt_junk = "free"; + opt_junk_alloc = false; + opt_junk_free = true; + } else { + CONF_ERROR("Invalid conf value", + k, klen, v, vlen); + } + CONF_CONTINUE; + } + CONF_HANDLE_BOOL(opt_zero, "zero") + } + if (config_utrace) { + CONF_HANDLE_BOOL(opt_utrace, "utrace") + } + if (config_xmalloc) { + CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc") + } + if (config_enable_cxx) { + CONF_HANDLE_BOOL( + opt_experimental_infallible_new, + "experimental_infallible_new") + } + + CONF_HANDLE_BOOL(opt_experimental_tcache_gc, + "experimental_tcache_gc") + CONF_HANDLE_BOOL(opt_tcache, "tcache") + CONF_HANDLE_SIZE_T(opt_tcache_max, "tcache_max", 0, + TCACHE_MAXCLASS_LIMIT, CONF_DONT_CHECK_MIN, + CONF_CHECK_MAX, /* clip */ true) + if (CONF_MATCH("lg_tcache_max")) { + size_t m; + CONF_VALUE_READ(size_t, m) + if (CONF_VALUE_READ_FAIL()) { + CONF_ERROR("Invalid conf value", k, + klen, v, vlen); + } else { + /* clip if necessary */ + if (m > TCACHE_LG_MAXCLASS_LIMIT) { + m = TCACHE_LG_MAXCLASS_LIMIT; + } + opt_tcache_max = (size_t)1 << m; + } + CONF_CONTINUE; + } + /* + * Anyone trying to set a value outside -16 to 16 is + * deeply confused. + */ + CONF_HANDLE_SSIZE_T(opt_lg_tcache_nslots_mul, + "lg_tcache_nslots_mul", -16, 16) + /* Ditto with values past 2048. */ + CONF_HANDLE_UNSIGNED(opt_tcache_nslots_small_min, + "tcache_nslots_small_min", 1, 2048, CONF_CHECK_MIN, + CONF_CHECK_MAX, /* clip */ true) + CONF_HANDLE_UNSIGNED(opt_tcache_nslots_small_max, + "tcache_nslots_small_max", 1, 2048, CONF_CHECK_MIN, + CONF_CHECK_MAX, /* clip */ true) + CONF_HANDLE_UNSIGNED(opt_tcache_nslots_large, + "tcache_nslots_large", 1, 2048, CONF_CHECK_MIN, + CONF_CHECK_MAX, /* clip */ true) + CONF_HANDLE_SIZE_T(opt_tcache_gc_incr_bytes, + "tcache_gc_incr_bytes", 1024, SIZE_T_MAX, + CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, + /* clip */ true) + CONF_HANDLE_SIZE_T(opt_tcache_gc_delay_bytes, + "tcache_gc_delay_bytes", 0, SIZE_T_MAX, + CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, + /* clip */ false) + CONF_HANDLE_UNSIGNED(opt_lg_tcache_flush_small_div, + "lg_tcache_flush_small_div", 1, 16, CONF_CHECK_MIN, + CONF_CHECK_MAX, /* clip */ true) + CONF_HANDLE_UNSIGNED(opt_lg_tcache_flush_large_div, + "lg_tcache_flush_large_div", 1, 16, CONF_CHECK_MIN, + CONF_CHECK_MAX, /* clip */ true) + CONF_HANDLE_UNSIGNED(opt_debug_double_free_max_scan, + "debug_double_free_max_scan", 0, UINT_MAX, + CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, + /* clip */ false) + CONF_HANDLE_SIZE_T(opt_calloc_madvise_threshold, + "calloc_madvise_threshold", 0, SC_LARGE_MAXCLASS, + CONF_DONT_CHECK_MIN, CONF_CHECK_MAX, + /* clip */ false) + + /* + * The runtime option of oversize_threshold remains + * undocumented. It may be tweaked in the next major + * release (6.0). The default value 8M is rather + * conservative / safe. Tuning it further down may + * improve fragmentation a bit more, but may also cause + * contention on the huge arena. + */ + CONF_HANDLE_SIZE_T(opt_oversize_threshold, + "oversize_threshold", 0, SC_LARGE_MAXCLASS, + CONF_DONT_CHECK_MIN, CONF_CHECK_MAX, false) + CONF_HANDLE_SIZE_T(opt_lg_extent_max_active_fit, + "lg_extent_max_active_fit", 0, + (sizeof(size_t) << 3), CONF_DONT_CHECK_MIN, + CONF_CHECK_MAX, false) + + if (strncmp("percpu_arena", k, klen) == 0) { + bool match = false; + for (int m = percpu_arena_mode_names_base; + m < percpu_arena_mode_names_limit; m++) { + if (strncmp(percpu_arena_mode_names[m], + v, vlen) + == 0) { + if (!have_percpu_arena) { + CONF_ERROR( + "No getcpu support", + k, klen, v, vlen); + } + opt_percpu_arena = m; + match = true; + break; + } + } + if (!match) { + CONF_ERROR("Invalid conf value", k, + klen, v, vlen); + } + CONF_CONTINUE; + } + CONF_HANDLE_BOOL( + opt_background_thread, "background_thread"); + CONF_HANDLE_SIZE_T(opt_max_background_threads, + "max_background_threads", 1, + opt_max_background_threads, CONF_CHECK_MIN, + CONF_CHECK_MAX, true); + CONF_HANDLE_BOOL(opt_hpa, "hpa") + CONF_HANDLE_SIZE_T(opt_hpa_opts.slab_max_alloc, + "hpa_slab_max_alloc", PAGE, HUGEPAGE, + CONF_CHECK_MIN, CONF_CHECK_MAX, true); + + /* + * Accept either a ratio-based or an exact hugification + * threshold. + */ + CONF_HANDLE_SIZE_T(opt_hpa_opts.hugification_threshold, + "hpa_hugification_threshold", PAGE, HUGEPAGE, + CONF_CHECK_MIN, CONF_CHECK_MAX, true); + if (CONF_MATCH("hpa_hugification_threshold_ratio")) { + fxp_t ratio; + char *end; + bool err = fxp_parse(&ratio, v, &end); + if (err || (size_t)(end - v) != vlen + || ratio > FXP_INIT_INT(1)) { + CONF_ERROR("Invalid conf value", k, + klen, v, vlen); + } else { + opt_hpa_opts.hugification_threshold = + fxp_mul_frac(HUGEPAGE, ratio); + } + CONF_CONTINUE; + } + + CONF_HANDLE_UINT64_T(opt_hpa_opts.hugify_delay_ms, + "hpa_hugify_delay_ms", 0, 0, CONF_DONT_CHECK_MIN, + CONF_DONT_CHECK_MAX, false); + + CONF_HANDLE_BOOL( + opt_hpa_opts.hugify_sync, "hpa_hugify_sync"); + + CONF_HANDLE_UINT64_T(opt_hpa_opts.min_purge_interval_ms, + "hpa_min_purge_interval_ms", 0, 0, + CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, false); + + CONF_HANDLE_SSIZE_T( + opt_hpa_opts.experimental_max_purge_nhp, + "experimental_hpa_max_purge_nhp", -1, SSIZE_MAX); + + /* + * Accept either a ratio-based or an exact purge + * threshold. + */ + CONF_HANDLE_SIZE_T(opt_hpa_opts.purge_threshold, + "hpa_purge_threshold", PAGE, HUGEPAGE, + CONF_CHECK_MIN, CONF_CHECK_MAX, true); + if (CONF_MATCH("hpa_purge_threshold_ratio")) { + fxp_t ratio; + char *end; + bool err = fxp_parse(&ratio, v, &end); + if (err || (size_t)(end - v) != vlen + || ratio > FXP_INIT_INT(1)) { + CONF_ERROR("Invalid conf value", k, + klen, v, vlen); + } else { + opt_hpa_opts.purge_threshold = + fxp_mul_frac(HUGEPAGE, ratio); + } + CONF_CONTINUE; + } + + CONF_HANDLE_UINT64_T(opt_hpa_opts.min_purge_delay_ms, + "hpa_min_purge_delay_ms", 0, UINT64_MAX, + CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, false); + + if (strncmp("hpa_hugify_style", k, klen) == 0) { + bool match = false; + for (int m = 0; m < hpa_hugify_style_limit; + m++) { + if (strncmp(hpa_hugify_style_names[m], + v, vlen) + == 0) { + opt_hpa_opts.hugify_style = m; + match = true; + break; + } + } + if (!match) { + CONF_ERROR("Invalid conf value", k, + klen, v, vlen); + } + CONF_CONTINUE; + } + + if (CONF_MATCH("hpa_dirty_mult")) { + if (CONF_MATCH_VALUE("-1")) { + opt_hpa_opts.dirty_mult = (fxp_t)-1; + CONF_CONTINUE; + } + fxp_t ratio; + char *end; + bool err = fxp_parse(&ratio, v, &end); + if (err || (size_t)(end - v) != vlen) { + CONF_ERROR("Invalid conf value", k, + klen, v, vlen); + } else { + opt_hpa_opts.dirty_mult = ratio; + } + CONF_CONTINUE; + } + CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.nshards, + "hpa_sec_nshards", 0, 0, CONF_CHECK_MIN, + CONF_DONT_CHECK_MAX, true); + CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_alloc, + "hpa_sec_max_alloc", PAGE, + USIZE_GROW_SLOW_THRESHOLD, CONF_CHECK_MIN, + CONF_CHECK_MAX, true); + CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_bytes, + "hpa_sec_max_bytes", SEC_OPTS_MAX_BYTES_DEFAULT, 0, + CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true); + CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.batch_fill_extra, + "hpa_sec_batch_fill_extra", 1, HUGEPAGE_PAGES, + CONF_CHECK_MIN, CONF_CHECK_MAX, true); + + if (CONF_MATCH("slab_sizes")) { + if (CONF_MATCH_VALUE("default")) { + sc_data_init(sc_data); + CONF_CONTINUE; + } + bool err; + const char *slab_size_segment_cur = v; + size_t vlen_left = vlen; + do { + size_t slab_start; + size_t slab_end; + size_t pgs; + err = multi_setting_parse_next( + &slab_size_segment_cur, &vlen_left, + &slab_start, &slab_end, &pgs); + if (!err) { + sc_data_update_slab_size( + sc_data, slab_start, + slab_end, (int)pgs); + } else { + CONF_ERROR( + "Invalid settings " + "for slab_sizes", + k, klen, v, vlen); + } + } while (!err && vlen_left > 0); + CONF_CONTINUE; + } + if (config_prof) { + CONF_HANDLE_BOOL(opt_prof, "prof") + CONF_HANDLE_CHAR_P( + opt_prof_prefix, "prof_prefix", "jeprof") + CONF_HANDLE_BOOL(opt_prof_active, "prof_active") + CONF_HANDLE_BOOL(opt_prof_thread_active_init, + "prof_thread_active_init") + CONF_HANDLE_SIZE_T(opt_lg_prof_sample, + "lg_prof_sample", 0, + (sizeof(uint64_t) << 3) - 1, + CONF_DONT_CHECK_MIN, CONF_CHECK_MAX, true) + CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum") + CONF_HANDLE_UNSIGNED(opt_prof_bt_max, + "prof_bt_max", 1, PROF_BT_MAX_LIMIT, + CONF_CHECK_MIN, CONF_CHECK_MAX, + /* clip */ true) + CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, + "lg_prof_interval", -1, + (sizeof(uint64_t) << 3) - 1) + CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump") + CONF_HANDLE_BOOL(opt_prof_final, "prof_final") + CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak") + CONF_HANDLE_BOOL( + opt_prof_leak_error, "prof_leak_error") + CONF_HANDLE_BOOL(opt_prof_log, "prof_log") + CONF_HANDLE_BOOL(opt_prof_pid_namespace, + "prof_pid_namespace") + CONF_HANDLE_SSIZE_T(opt_prof_recent_alloc_max, + "prof_recent_alloc_max", -1, SSIZE_MAX) + CONF_HANDLE_BOOL(opt_prof_stats, "prof_stats") + CONF_HANDLE_BOOL(opt_prof_sys_thread_name, + "prof_sys_thread_name") + if (CONF_MATCH("prof_time_resolution")) { + if (CONF_MATCH_VALUE("default")) { + opt_prof_time_res = + prof_time_res_default; + } else if (CONF_MATCH_VALUE("high")) { + if (!config_high_res_timer) { + CONF_ERROR( + "No high resolution" + " timer support", + k, klen, v, vlen); + } else { + opt_prof_time_res = + prof_time_res_high; + } + } else { + CONF_ERROR("Invalid conf value", + k, klen, v, vlen); + } + CONF_CONTINUE; + } + /* + * Undocumented. When set to false, don't + * correct for an unbiasing bug in jeprof + * attribution. This can be handy if you want + * to get consistent numbers from your binary + * across different jemalloc versions, even if + * those numbers are incorrect. The default is + * true. + */ + CONF_HANDLE_BOOL(opt_prof_unbias, "prof_unbias") + } + if (config_log) { + if (CONF_MATCH("log")) { + size_t cpylen = (vlen + <= sizeof(log_var_names) + ? vlen + : sizeof(log_var_names) - 1); + strncpy(log_var_names, v, cpylen); + log_var_names[cpylen] = '\0'; + CONF_CONTINUE; + } + } + if (CONF_MATCH("thp")) { + bool match = false; + for (int m = 0; m < thp_mode_names_limit; m++) { + if (strncmp(thp_mode_names[m], v, vlen) + == 0) { + if (!have_madvise_huge + && !have_memcntl) { + CONF_ERROR( + "No THP support", k, + klen, v, vlen); + } + opt_thp = m; + match = true; + break; + } + } + if (!match) { + CONF_ERROR("Invalid conf value", k, + klen, v, vlen); + } + CONF_CONTINUE; + } + if (CONF_MATCH("zero_realloc")) { + if (CONF_MATCH_VALUE("alloc")) { + opt_zero_realloc_action = + zero_realloc_action_alloc; + } else if (CONF_MATCH_VALUE("free")) { + opt_zero_realloc_action = + zero_realloc_action_free; + } else if (CONF_MATCH_VALUE("abort")) { + opt_zero_realloc_action = + zero_realloc_action_abort; + } else { + CONF_ERROR("Invalid conf value", k, + klen, v, vlen); + } + CONF_CONTINUE; + } + if (config_uaf_detection + && CONF_MATCH("lg_san_uaf_align")) { + ssize_t a; + CONF_VALUE_READ(ssize_t, a) + if (CONF_VALUE_READ_FAIL() || a < -1) { + CONF_ERROR("Invalid conf value", k, + klen, v, vlen); + } + if (a == -1) { + opt_lg_san_uaf_align = -1; + CONF_CONTINUE; + } + + /* clip if necessary */ + ssize_t max_allowed = (sizeof(size_t) << 3) - 1; + ssize_t min_allowed = LG_PAGE; + if (a > max_allowed) { + a = max_allowed; + } else if (a < min_allowed) { + a = min_allowed; + } + + opt_lg_san_uaf_align = a; + CONF_CONTINUE; + } + + CONF_HANDLE_SIZE_T(opt_san_guard_small, + "san_guard_small", 0, SIZE_T_MAX, + CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, false) + CONF_HANDLE_SIZE_T(opt_san_guard_large, + "san_guard_large", 0, SIZE_T_MAX, + CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, false) + + /* + * Disable large size classes is now the default + * behavior in jemalloc. Although it is configurable + * in MALLOC_CONF, this is mainly for debugging + * purposes and should not be tuned. + */ + CONF_HANDLE_BOOL(opt_disable_large_size_classes, + "disable_large_size_classes"); + + CONF_ERROR("Invalid conf pair", k, klen, v, vlen); +#undef CONF_ERROR +#undef CONF_CONTINUE +#undef CONF_MATCH +#undef CONF_MATCH_VALUE +#undef CONF_HANDLE_BOOL +#undef CONF_DONT_CHECK_MIN +#undef CONF_CHECK_MIN +#undef CONF_DONT_CHECK_MAX +#undef CONF_CHECK_MAX +#undef CONF_HANDLE_T +#undef CONF_HANDLE_T_U +#undef CONF_HANDLE_T_SIGNED +#undef CONF_HANDLE_UNSIGNED +#undef CONF_HANDLE_SIZE_T +#undef CONF_HANDLE_SSIZE_T +#undef CONF_HANDLE_CHAR_P + /* Re-enable diagnostic "-Wtype-limits" */ + JEMALLOC_DIAGNOSTIC_POP + } + validate_hpa_settings(); + if (opt_abort_conf && had_conf_error) { + malloc_abort_invalid_conf(); + } + } + atomic_store_b(&log_init_done, true, ATOMIC_RELEASE); +} + +static bool +malloc_conf_init_check_deps(void) { + if (opt_prof_leak_error && !opt_prof_final) { + malloc_printf( + ": prof_leak_error is set w/o " + "prof_final.\n"); + return true; + } + /* To emphasize in the stats output that opt is disabled when !debug. */ + if (!config_debug) { + opt_debug_double_free_max_scan = 0; + } + + return false; +} + +void +malloc_conf_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], + char readlink_buf[PATH_MAX + 1]) { + const char *opts_cache[MALLOC_CONF_NSOURCES] = { + NULL, NULL, NULL, NULL, NULL}; + + /* The first call only set the confirm_conf option and opts_cache */ + malloc_conf_init_helper(NULL, NULL, true, opts_cache, readlink_buf); + malloc_conf_init_helper( + sc_data, bin_shard_sizes, false, opts_cache, NULL); + if (malloc_conf_init_check_deps()) { + /* check_deps does warning msg only; abort below if needed. */ + if (opt_abort_conf) { + malloc_abort_invalid_conf(); + } + } +} + +#undef MALLOC_CONF_NSOURCES diff --git a/dep/jemalloc/src/counter.c b/dep/jemalloc/src/counter.c index 8f1ae3af4..8257a0627 100644 --- a/dep/jemalloc/src/counter.c +++ b/dep/jemalloc/src/counter.c @@ -6,7 +6,7 @@ bool counter_accum_init(counter_accum_t *counter, uint64_t interval) { if (LOCKEDINT_MTX_INIT(counter->mtx, "counter_accum", - WITNESS_RANK_COUNTER_ACCUM, malloc_mutex_rank_exclusive)) { + WITNESS_RANK_COUNTER_ACCUM, malloc_mutex_rank_exclusive)) { return true; } locked_init_u64_unsynchronized(&counter->accumbytes, 0); diff --git a/dep/jemalloc/src/ctl.c b/dep/jemalloc/src/ctl.c index 135271baf..0b72086cf 100644 --- a/dep/jemalloc/src/ctl.c +++ b/dep/jemalloc/src/ctl.c @@ -25,10 +25,10 @@ * ctl_mtx protects the following: * - ctl_stats->* */ -static malloc_mutex_t ctl_mtx; -static bool ctl_initialized; -static ctl_stats_t *ctl_stats; -static ctl_arenas_t *ctl_arenas; +static malloc_mutex_t ctl_mtx; +static bool ctl_initialized; +static ctl_stats_t *ctl_stats; +static ctl_arenas_t *ctl_arenas; /******************************************************************************/ /* Helpers for named and indexed nodes. */ @@ -53,20 +53,23 @@ ctl_indexed_node(const ctl_node_t *node) { /******************************************************************************/ /* Function prototypes for non-inline static functions. */ -#define CTL_PROTO(n) \ -static int n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ - void *oldp, size_t *oldlenp, void *newp, size_t newlen); +#define CTL_PROTO(n) \ + static int n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ + void *oldp, size_t *oldlenp, void *newp, size_t newlen); -#define INDEX_PROTO(n) \ -static const ctl_named_node_t *n##_index(tsdn_t *tsdn, \ - const size_t *mib, size_t miblen, size_t i); +#define INDEX_PROTO(n) \ + static const ctl_named_node_t *n##_index( \ + tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i); CTL_PROTO(version) CTL_PROTO(epoch) CTL_PROTO(background_thread) CTL_PROTO(max_background_threads) CTL_PROTO(thread_tcache_enabled) +CTL_PROTO(thread_tcache_max) CTL_PROTO(thread_tcache_flush) +CTL_PROTO(thread_tcache_ncached_max_write) +CTL_PROTO(thread_tcache_ncached_max_read_sizeclass) CTL_PROTO(thread_peak_read) CTL_PROTO(thread_peak_reset) CTL_PROTO(thread_prof_name) @@ -86,25 +89,34 @@ CTL_PROTO(config_opt_safety_checks) CTL_PROTO(config_prof) CTL_PROTO(config_prof_libgcc) CTL_PROTO(config_prof_libunwind) +CTL_PROTO(config_prof_frameptr) CTL_PROTO(config_stats) CTL_PROTO(config_utrace) CTL_PROTO(config_xmalloc) CTL_PROTO(opt_abort) CTL_PROTO(opt_abort_conf) CTL_PROTO(opt_cache_oblivious) +CTL_PROTO(opt_debug_double_free_max_scan) CTL_PROTO(opt_trust_madvise) +CTL_PROTO(opt_experimental_hpa_start_huge_if_thp_always) +CTL_PROTO(opt_experimental_hpa_enforce_hugify) CTL_PROTO(opt_confirm_conf) CTL_PROTO(opt_hpa) CTL_PROTO(opt_hpa_slab_max_alloc) CTL_PROTO(opt_hpa_hugification_threshold) CTL_PROTO(opt_hpa_hugify_delay_ms) +CTL_PROTO(opt_hpa_hugify_sync) CTL_PROTO(opt_hpa_min_purge_interval_ms) +CTL_PROTO(opt_experimental_hpa_max_purge_nhp) +CTL_PROTO(opt_hpa_purge_threshold) +CTL_PROTO(opt_hpa_min_purge_delay_ms) +CTL_PROTO(opt_hpa_hugify_style) CTL_PROTO(opt_hpa_dirty_mult) CTL_PROTO(opt_hpa_sec_nshards) CTL_PROTO(opt_hpa_sec_max_alloc) CTL_PROTO(opt_hpa_sec_max_bytes) -CTL_PROTO(opt_hpa_sec_bytes_after_flush) CTL_PROTO(opt_hpa_sec_batch_fill_extra) +CTL_PROTO(opt_huge_arena_pac_thp) CTL_PROTO(opt_metadata_thp) CTL_PROTO(opt_retain) CTL_PROTO(opt_dss) @@ -125,6 +137,7 @@ CTL_PROTO(opt_zero) CTL_PROTO(opt_utrace) CTL_PROTO(opt_xmalloc) CTL_PROTO(opt_experimental_infallible_new) +CTL_PROTO(opt_experimental_tcache_gc) CTL_PROTO(opt_tcache) CTL_PROTO(opt_tcache_max) CTL_PROTO(opt_tcache_nslots_small_min) @@ -141,6 +154,7 @@ CTL_PROTO(opt_prof) CTL_PROTO(opt_prof_prefix) CTL_PROTO(opt_prof_active) CTL_PROTO(opt_prof_thread_active_init) +CTL_PROTO(opt_prof_bt_max) CTL_PROTO(opt_lg_prof_sample) CTL_PROTO(opt_lg_prof_interval) CTL_PROTO(opt_prof_gdump) @@ -148,12 +162,19 @@ CTL_PROTO(opt_prof_final) CTL_PROTO(opt_prof_leak) CTL_PROTO(opt_prof_leak_error) CTL_PROTO(opt_prof_accum) +CTL_PROTO(opt_prof_pid_namespace) CTL_PROTO(opt_prof_recent_alloc_max) CTL_PROTO(opt_prof_stats) CTL_PROTO(opt_prof_sys_thread_name) CTL_PROTO(opt_prof_time_res) CTL_PROTO(opt_lg_san_uaf_align) CTL_PROTO(opt_zero_realloc) +CTL_PROTO(opt_disable_large_size_classes) +CTL_PROTO(opt_process_madvise_max_batch) +CTL_PROTO(opt_malloc_conf_symlink) +CTL_PROTO(opt_malloc_conf_env_var) +CTL_PROTO(opt_malloc_conf_global_var) +CTL_PROTO(opt_malloc_conf_global_var_2_conf_harder) CTL_PROTO(tcache_create) CTL_PROTO(tcache_flush) CTL_PROTO(tcache_destroy) @@ -168,6 +189,7 @@ CTL_PROTO(arena_i_dirty_decay_ms) CTL_PROTO(arena_i_muzzy_decay_ms) CTL_PROTO(arena_i_extent_hooks) CTL_PROTO(arena_i_retain_grow_limit) +CTL_PROTO(arena_i_name) INDEX_PROTO(arena_i) CTL_PROTO(arenas_bin_i_size) CTL_PROTO(arenas_bin_i_nregs) @@ -181,6 +203,7 @@ CTL_PROTO(arenas_dirty_decay_ms) CTL_PROTO(arenas_muzzy_decay_ms) CTL_PROTO(arenas_quantum) CTL_PROTO(arenas_page) +CTL_PROTO(arenas_hugepage) CTL_PROTO(arenas_tcache_max) CTL_PROTO(arenas_nbins) CTL_PROTO(arenas_nhbins) @@ -238,12 +261,27 @@ CTL_PROTO(stats_arenas_i_extents_j_dirty_bytes) CTL_PROTO(stats_arenas_i_extents_j_muzzy_bytes) CTL_PROTO(stats_arenas_i_extents_j_retained_bytes) INDEX_PROTO(stats_arenas_i_extents_j) + +/* Merged set of stats for HPA shard. */ +CTL_PROTO(stats_arenas_i_hpa_shard_npageslabs) +CTL_PROTO(stats_arenas_i_hpa_shard_nactive) +CTL_PROTO(stats_arenas_i_hpa_shard_ndirty) + CTL_PROTO(stats_arenas_i_hpa_shard_npurge_passes) CTL_PROTO(stats_arenas_i_hpa_shard_npurges) CTL_PROTO(stats_arenas_i_hpa_shard_nhugifies) +CTL_PROTO(stats_arenas_i_hpa_shard_nhugify_failures) CTL_PROTO(stats_arenas_i_hpa_shard_ndehugifies) -/* We have a set of stats for full slabs. */ +/* Set of stats for non-hugified and hugified slabs. */ +CTL_PROTO(stats_arenas_i_hpa_shard_slabs_npageslabs_nonhuge) +CTL_PROTO(stats_arenas_i_hpa_shard_slabs_npageslabs_huge) +CTL_PROTO(stats_arenas_i_hpa_shard_slabs_nactive_nonhuge) +CTL_PROTO(stats_arenas_i_hpa_shard_slabs_nactive_huge) +CTL_PROTO(stats_arenas_i_hpa_shard_slabs_ndirty_nonhuge) +CTL_PROTO(stats_arenas_i_hpa_shard_slabs_ndirty_huge) + +/* A parallel set of stats for full slabs. */ CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_npageslabs_nonhuge) CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_npageslabs_huge) CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_nactive_nonhuge) @@ -271,6 +309,7 @@ CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_nonhuge) CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_huge) INDEX_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j) + CTL_PROTO(stats_arenas_i_nthreads) CTL_PROTO(stats_arenas_i_uptime) CTL_PROTO(stats_arenas_i_dss) @@ -290,12 +329,19 @@ CTL_PROTO(stats_arenas_i_muzzy_nmadvise) CTL_PROTO(stats_arenas_i_muzzy_purged) CTL_PROTO(stats_arenas_i_base) CTL_PROTO(stats_arenas_i_internal) +CTL_PROTO(stats_arenas_i_metadata_edata) +CTL_PROTO(stats_arenas_i_metadata_rtree) CTL_PROTO(stats_arenas_i_metadata_thp) CTL_PROTO(stats_arenas_i_tcache_bytes) CTL_PROTO(stats_arenas_i_tcache_stashed_bytes) CTL_PROTO(stats_arenas_i_resident) CTL_PROTO(stats_arenas_i_abandoned_vm) CTL_PROTO(stats_arenas_i_hpa_sec_bytes) +CTL_PROTO(stats_arenas_i_hpa_sec_hits) +CTL_PROTO(stats_arenas_i_hpa_sec_misses) +CTL_PROTO(stats_arenas_i_hpa_sec_dalloc_flush) +CTL_PROTO(stats_arenas_i_hpa_sec_dalloc_noflush) +CTL_PROTO(stats_arenas_i_hpa_sec_overfills) INDEX_PROTO(stats_arenas_i) CTL_PROTO(stats_allocated) CTL_PROTO(stats_active) @@ -303,17 +349,22 @@ CTL_PROTO(stats_background_thread_num_threads) CTL_PROTO(stats_background_thread_num_runs) CTL_PROTO(stats_background_thread_run_interval) CTL_PROTO(stats_metadata) +CTL_PROTO(stats_metadata_edata) +CTL_PROTO(stats_metadata_rtree) CTL_PROTO(stats_metadata_thp) CTL_PROTO(stats_resident) CTL_PROTO(stats_mapped) CTL_PROTO(stats_retained) CTL_PROTO(stats_zero_reallocs) +CTL_PROTO(approximate_stats_active) CTL_PROTO(experimental_hooks_install) CTL_PROTO(experimental_hooks_remove) CTL_PROTO(experimental_hooks_prof_backtrace) CTL_PROTO(experimental_hooks_prof_dump) +CTL_PROTO(experimental_hooks_prof_sample) +CTL_PROTO(experimental_hooks_prof_sample_free) +CTL_PROTO(experimental_hooks_thread_event) CTL_PROTO(experimental_hooks_safety_check_abort) -CTL_PROTO(experimental_thread_activity_callback) CTL_PROTO(experimental_utilization_query) CTL_PROTO(experimental_utilization_batch_query) CTL_PROTO(experimental_arenas_i_pactivep) @@ -323,14 +374,14 @@ CTL_PROTO(experimental_prof_recent_alloc_dump) CTL_PROTO(experimental_batch_alloc) CTL_PROTO(experimental_arenas_create_ext) -#define MUTEX_STATS_CTL_PROTO_GEN(n) \ -CTL_PROTO(stats_##n##_num_ops) \ -CTL_PROTO(stats_##n##_num_wait) \ -CTL_PROTO(stats_##n##_num_spin_acq) \ -CTL_PROTO(stats_##n##_num_owner_switch) \ -CTL_PROTO(stats_##n##_total_wait_time) \ -CTL_PROTO(stats_##n##_max_wait_time) \ -CTL_PROTO(stats_##n##_max_num_thds) +#define MUTEX_STATS_CTL_PROTO_GEN(n) \ + CTL_PROTO(stats_##n##_num_ops) \ + CTL_PROTO(stats_##n##_num_wait) \ + CTL_PROTO(stats_##n##_num_spin_acq) \ + CTL_PROTO(stats_##n##_num_owner_switch) \ + CTL_PROTO(stats_##n##_total_wait_time) \ + CTL_PROTO(stats_##n##_max_wait_time) \ + CTL_PROTO(stats_##n##_max_num_thds) /* Global mutexes. */ #define OP(mtx) MUTEX_STATS_CTL_PROTO_GEN(mutexes_##mtx) @@ -351,473 +402,450 @@ CTL_PROTO(stats_mutexes_reset) /******************************************************************************/ /* mallctl tree. */ -#define NAME(n) {true}, n -#define CHILD(t, c) \ - sizeof(c##_node) / sizeof(ctl_##t##_node_t), \ - (ctl_node_t *)c##_node, \ - NULL -#define CTL(c) 0, NULL, c##_ctl +#define NAME(n) {true}, n +#define CHILD(t, c) \ + sizeof(c##_node) / sizeof(ctl_##t##_node_t), (ctl_node_t *)c##_node, \ + NULL +#define CTL(c) 0, NULL, c##_ctl /* * Only handles internal indexed nodes, since there are currently no external * ones. */ -#define INDEX(i) {false}, i##_index +#define INDEX(i) {false}, i##_index -static const ctl_named_node_t thread_tcache_node[] = { - {NAME("enabled"), CTL(thread_tcache_enabled)}, - {NAME("flush"), CTL(thread_tcache_flush)} +static const ctl_named_node_t thread_tcache_ncached_max_node[] = { + {NAME("read_sizeclass"), CTL(thread_tcache_ncached_max_read_sizeclass)}, + {NAME("write"), CTL(thread_tcache_ncached_max_write)}}; + +static const ctl_named_node_t thread_tcache_node[] = { + {NAME("enabled"), CTL(thread_tcache_enabled)}, + {NAME("max"), CTL(thread_tcache_max)}, + {NAME("flush"), CTL(thread_tcache_flush)}, + {NAME("ncached_max"), CHILD(named, thread_tcache_ncached_max)}}; + +static const ctl_named_node_t thread_peak_node[] = { + {NAME("read"), CTL(thread_peak_read)}, + {NAME("reset"), CTL(thread_peak_reset)}, }; -static const ctl_named_node_t thread_peak_node[] = { - {NAME("read"), CTL(thread_peak_read)}, - {NAME("reset"), CTL(thread_peak_reset)}, -}; +static const ctl_named_node_t thread_prof_node[] = { + {NAME("name"), CTL(thread_prof_name)}, + {NAME("active"), CTL(thread_prof_active)}}; -static const ctl_named_node_t thread_prof_node[] = { - {NAME("name"), CTL(thread_prof_name)}, - {NAME("active"), CTL(thread_prof_active)} -}; +static const ctl_named_node_t thread_node[] = { + {NAME("arena"), CTL(thread_arena)}, + {NAME("allocated"), CTL(thread_allocated)}, + {NAME("allocatedp"), CTL(thread_allocatedp)}, + {NAME("deallocated"), CTL(thread_deallocated)}, + {NAME("deallocatedp"), CTL(thread_deallocatedp)}, + {NAME("tcache"), CHILD(named, thread_tcache)}, + {NAME("peak"), CHILD(named, thread_peak)}, + {NAME("prof"), CHILD(named, thread_prof)}, + {NAME("idle"), CTL(thread_idle)}}; -static const ctl_named_node_t thread_node[] = { - {NAME("arena"), CTL(thread_arena)}, - {NAME("allocated"), CTL(thread_allocated)}, - {NAME("allocatedp"), CTL(thread_allocatedp)}, - {NAME("deallocated"), CTL(thread_deallocated)}, - {NAME("deallocatedp"), CTL(thread_deallocatedp)}, - {NAME("tcache"), CHILD(named, thread_tcache)}, - {NAME("peak"), CHILD(named, thread_peak)}, - {NAME("prof"), CHILD(named, thread_prof)}, - {NAME("idle"), CTL(thread_idle)} -}; +static const ctl_named_node_t config_node[] = { + {NAME("cache_oblivious"), CTL(config_cache_oblivious)}, + {NAME("debug"), CTL(config_debug)}, {NAME("fill"), CTL(config_fill)}, + {NAME("lazy_lock"), CTL(config_lazy_lock)}, + {NAME("malloc_conf"), CTL(config_malloc_conf)}, + {NAME("opt_safety_checks"), CTL(config_opt_safety_checks)}, + {NAME("prof"), CTL(config_prof)}, + {NAME("prof_libgcc"), CTL(config_prof_libgcc)}, + {NAME("prof_libunwind"), CTL(config_prof_libunwind)}, + {NAME("prof_frameptr"), CTL(config_prof_frameptr)}, + {NAME("stats"), CTL(config_stats)}, {NAME("utrace"), CTL(config_utrace)}, + {NAME("xmalloc"), CTL(config_xmalloc)}}; -static const ctl_named_node_t config_node[] = { - {NAME("cache_oblivious"), CTL(config_cache_oblivious)}, - {NAME("debug"), CTL(config_debug)}, - {NAME("fill"), CTL(config_fill)}, - {NAME("lazy_lock"), CTL(config_lazy_lock)}, - {NAME("malloc_conf"), CTL(config_malloc_conf)}, - {NAME("opt_safety_checks"), CTL(config_opt_safety_checks)}, - {NAME("prof"), CTL(config_prof)}, - {NAME("prof_libgcc"), CTL(config_prof_libgcc)}, - {NAME("prof_libunwind"), CTL(config_prof_libunwind)}, - {NAME("stats"), CTL(config_stats)}, - {NAME("utrace"), CTL(config_utrace)}, - {NAME("xmalloc"), CTL(config_xmalloc)} -}; +static const ctl_named_node_t opt_malloc_conf_node[] = { + {NAME("symlink"), CTL(opt_malloc_conf_symlink)}, + {NAME("env_var"), CTL(opt_malloc_conf_env_var)}, + {NAME("global_var"), CTL(opt_malloc_conf_global_var)}, + {NAME("global_var_2_conf_harder"), + CTL(opt_malloc_conf_global_var_2_conf_harder)}}; -static const ctl_named_node_t opt_node[] = { - {NAME("abort"), CTL(opt_abort)}, - {NAME("abort_conf"), CTL(opt_abort_conf)}, - {NAME("cache_oblivious"), CTL(opt_cache_oblivious)}, - {NAME("trust_madvise"), CTL(opt_trust_madvise)}, - {NAME("confirm_conf"), CTL(opt_confirm_conf)}, - {NAME("hpa"), CTL(opt_hpa)}, - {NAME("hpa_slab_max_alloc"), CTL(opt_hpa_slab_max_alloc)}, - {NAME("hpa_hugification_threshold"), - CTL(opt_hpa_hugification_threshold)}, - {NAME("hpa_hugify_delay_ms"), CTL(opt_hpa_hugify_delay_ms)}, - {NAME("hpa_min_purge_interval_ms"), CTL(opt_hpa_min_purge_interval_ms)}, - {NAME("hpa_dirty_mult"), CTL(opt_hpa_dirty_mult)}, - {NAME("hpa_sec_nshards"), CTL(opt_hpa_sec_nshards)}, - {NAME("hpa_sec_max_alloc"), CTL(opt_hpa_sec_max_alloc)}, - {NAME("hpa_sec_max_bytes"), CTL(opt_hpa_sec_max_bytes)}, - {NAME("hpa_sec_bytes_after_flush"), - CTL(opt_hpa_sec_bytes_after_flush)}, - {NAME("hpa_sec_batch_fill_extra"), - CTL(opt_hpa_sec_batch_fill_extra)}, - {NAME("metadata_thp"), CTL(opt_metadata_thp)}, - {NAME("retain"), CTL(opt_retain)}, - {NAME("dss"), CTL(opt_dss)}, - {NAME("narenas"), CTL(opt_narenas)}, - {NAME("percpu_arena"), CTL(opt_percpu_arena)}, - {NAME("oversize_threshold"), CTL(opt_oversize_threshold)}, - {NAME("mutex_max_spin"), CTL(opt_mutex_max_spin)}, - {NAME("background_thread"), CTL(opt_background_thread)}, - {NAME("max_background_threads"), CTL(opt_max_background_threads)}, - {NAME("dirty_decay_ms"), CTL(opt_dirty_decay_ms)}, - {NAME("muzzy_decay_ms"), CTL(opt_muzzy_decay_ms)}, - {NAME("stats_print"), CTL(opt_stats_print)}, - {NAME("stats_print_opts"), CTL(opt_stats_print_opts)}, - {NAME("stats_interval"), CTL(opt_stats_interval)}, - {NAME("stats_interval_opts"), CTL(opt_stats_interval_opts)}, - {NAME("junk"), CTL(opt_junk)}, - {NAME("zero"), CTL(opt_zero)}, - {NAME("utrace"), CTL(opt_utrace)}, - {NAME("xmalloc"), CTL(opt_xmalloc)}, - {NAME("experimental_infallible_new"), - CTL(opt_experimental_infallible_new)}, - {NAME("tcache"), CTL(opt_tcache)}, - {NAME("tcache_max"), CTL(opt_tcache_max)}, - {NAME("tcache_nslots_small_min"), - CTL(opt_tcache_nslots_small_min)}, - {NAME("tcache_nslots_small_max"), - CTL(opt_tcache_nslots_small_max)}, - {NAME("tcache_nslots_large"), CTL(opt_tcache_nslots_large)}, - {NAME("lg_tcache_nslots_mul"), CTL(opt_lg_tcache_nslots_mul)}, - {NAME("tcache_gc_incr_bytes"), CTL(opt_tcache_gc_incr_bytes)}, - {NAME("tcache_gc_delay_bytes"), CTL(opt_tcache_gc_delay_bytes)}, - {NAME("lg_tcache_flush_small_div"), - CTL(opt_lg_tcache_flush_small_div)}, - {NAME("lg_tcache_flush_large_div"), - CTL(opt_lg_tcache_flush_large_div)}, - {NAME("thp"), CTL(opt_thp)}, - {NAME("lg_extent_max_active_fit"), CTL(opt_lg_extent_max_active_fit)}, - {NAME("prof"), CTL(opt_prof)}, - {NAME("prof_prefix"), CTL(opt_prof_prefix)}, - {NAME("prof_active"), CTL(opt_prof_active)}, - {NAME("prof_thread_active_init"), CTL(opt_prof_thread_active_init)}, - {NAME("lg_prof_sample"), CTL(opt_lg_prof_sample)}, - {NAME("lg_prof_interval"), CTL(opt_lg_prof_interval)}, - {NAME("prof_gdump"), CTL(opt_prof_gdump)}, - {NAME("prof_final"), CTL(opt_prof_final)}, - {NAME("prof_leak"), CTL(opt_prof_leak)}, - {NAME("prof_leak_error"), CTL(opt_prof_leak_error)}, - {NAME("prof_accum"), CTL(opt_prof_accum)}, - {NAME("prof_recent_alloc_max"), CTL(opt_prof_recent_alloc_max)}, - {NAME("prof_stats"), CTL(opt_prof_stats)}, - {NAME("prof_sys_thread_name"), CTL(opt_prof_sys_thread_name)}, - {NAME("prof_time_resolution"), CTL(opt_prof_time_res)}, - {NAME("lg_san_uaf_align"), CTL(opt_lg_san_uaf_align)}, - {NAME("zero_realloc"), CTL(opt_zero_realloc)} -}; +static const ctl_named_node_t opt_node[] = {{NAME("abort"), CTL(opt_abort)}, + {NAME("abort_conf"), CTL(opt_abort_conf)}, + {NAME("cache_oblivious"), CTL(opt_cache_oblivious)}, + {NAME("trust_madvise"), CTL(opt_trust_madvise)}, + {NAME("experimental_hpa_start_huge_if_thp_always"), + CTL(opt_experimental_hpa_start_huge_if_thp_always)}, + {NAME("experimental_hpa_enforce_hugify"), + CTL(opt_experimental_hpa_enforce_hugify)}, + {NAME("confirm_conf"), CTL(opt_confirm_conf)}, {NAME("hpa"), CTL(opt_hpa)}, + {NAME("hpa_slab_max_alloc"), CTL(opt_hpa_slab_max_alloc)}, + {NAME("hpa_hugification_threshold"), CTL(opt_hpa_hugification_threshold)}, + {NAME("hpa_hugify_delay_ms"), CTL(opt_hpa_hugify_delay_ms)}, + {NAME("hpa_hugify_sync"), CTL(opt_hpa_hugify_sync)}, + {NAME("hpa_min_purge_interval_ms"), CTL(opt_hpa_min_purge_interval_ms)}, + {NAME("experimental_hpa_max_purge_nhp"), + CTL(opt_experimental_hpa_max_purge_nhp)}, + {NAME("hpa_purge_threshold"), CTL(opt_hpa_purge_threshold)}, + {NAME("hpa_min_purge_delay_ms"), CTL(opt_hpa_min_purge_delay_ms)}, + {NAME("hpa_hugify_style"), CTL(opt_hpa_hugify_style)}, + {NAME("hpa_dirty_mult"), CTL(opt_hpa_dirty_mult)}, + {NAME("hpa_sec_nshards"), CTL(opt_hpa_sec_nshards)}, + {NAME("hpa_sec_max_alloc"), CTL(opt_hpa_sec_max_alloc)}, + {NAME("hpa_sec_max_bytes"), CTL(opt_hpa_sec_max_bytes)}, + {NAME("hpa_sec_batch_fill_extra"), CTL(opt_hpa_sec_batch_fill_extra)}, + {NAME("huge_arena_pac_thp"), CTL(opt_huge_arena_pac_thp)}, + {NAME("metadata_thp"), CTL(opt_metadata_thp)}, + {NAME("retain"), CTL(opt_retain)}, {NAME("dss"), CTL(opt_dss)}, + {NAME("narenas"), CTL(opt_narenas)}, + {NAME("percpu_arena"), CTL(opt_percpu_arena)}, + {NAME("oversize_threshold"), CTL(opt_oversize_threshold)}, + {NAME("mutex_max_spin"), CTL(opt_mutex_max_spin)}, + {NAME("background_thread"), CTL(opt_background_thread)}, + {NAME("max_background_threads"), CTL(opt_max_background_threads)}, + {NAME("dirty_decay_ms"), CTL(opt_dirty_decay_ms)}, + {NAME("muzzy_decay_ms"), CTL(opt_muzzy_decay_ms)}, + {NAME("stats_print"), CTL(opt_stats_print)}, + {NAME("stats_print_opts"), CTL(opt_stats_print_opts)}, + {NAME("stats_interval"), CTL(opt_stats_interval)}, + {NAME("stats_interval_opts"), CTL(opt_stats_interval_opts)}, + {NAME("junk"), CTL(opt_junk)}, {NAME("zero"), CTL(opt_zero)}, + {NAME("utrace"), CTL(opt_utrace)}, {NAME("xmalloc"), CTL(opt_xmalloc)}, + {NAME("experimental_infallible_new"), CTL(opt_experimental_infallible_new)}, + {NAME("experimental_tcache_gc"), CTL(opt_experimental_tcache_gc)}, + {NAME("tcache"), CTL(opt_tcache)}, + {NAME("tcache_max"), CTL(opt_tcache_max)}, + {NAME("tcache_nslots_small_min"), CTL(opt_tcache_nslots_small_min)}, + {NAME("tcache_nslots_small_max"), CTL(opt_tcache_nslots_small_max)}, + {NAME("tcache_nslots_large"), CTL(opt_tcache_nslots_large)}, + {NAME("lg_tcache_nslots_mul"), CTL(opt_lg_tcache_nslots_mul)}, + {NAME("tcache_gc_incr_bytes"), CTL(opt_tcache_gc_incr_bytes)}, + {NAME("tcache_gc_delay_bytes"), CTL(opt_tcache_gc_delay_bytes)}, + {NAME("lg_tcache_flush_small_div"), CTL(opt_lg_tcache_flush_small_div)}, + {NAME("lg_tcache_flush_large_div"), CTL(opt_lg_tcache_flush_large_div)}, + {NAME("thp"), CTL(opt_thp)}, + {NAME("lg_extent_max_active_fit"), CTL(opt_lg_extent_max_active_fit)}, + {NAME("prof"), CTL(opt_prof)}, {NAME("prof_prefix"), CTL(opt_prof_prefix)}, + {NAME("prof_active"), CTL(opt_prof_active)}, + {NAME("prof_thread_active_init"), CTL(opt_prof_thread_active_init)}, + {NAME("prof_bt_max"), CTL(opt_prof_bt_max)}, + {NAME("lg_prof_sample"), CTL(opt_lg_prof_sample)}, + {NAME("lg_prof_interval"), CTL(opt_lg_prof_interval)}, + {NAME("prof_gdump"), CTL(opt_prof_gdump)}, + {NAME("prof_final"), CTL(opt_prof_final)}, + {NAME("prof_leak"), CTL(opt_prof_leak)}, + {NAME("prof_leak_error"), CTL(opt_prof_leak_error)}, + {NAME("prof_accum"), CTL(opt_prof_accum)}, + {NAME("prof_pid_namespace"), CTL(opt_prof_pid_namespace)}, + {NAME("prof_recent_alloc_max"), CTL(opt_prof_recent_alloc_max)}, + {NAME("prof_stats"), CTL(opt_prof_stats)}, + {NAME("prof_sys_thread_name"), CTL(opt_prof_sys_thread_name)}, + {NAME("prof_time_resolution"), CTL(opt_prof_time_res)}, + {NAME("lg_san_uaf_align"), CTL(opt_lg_san_uaf_align)}, + {NAME("zero_realloc"), CTL(opt_zero_realloc)}, + {NAME("debug_double_free_max_scan"), CTL(opt_debug_double_free_max_scan)}, + {NAME("disable_large_size_classes"), CTL(opt_disable_large_size_classes)}, + {NAME("process_madvise_max_batch"), CTL(opt_process_madvise_max_batch)}, + {NAME("malloc_conf"), CHILD(named, opt_malloc_conf)}}; -static const ctl_named_node_t tcache_node[] = { - {NAME("create"), CTL(tcache_create)}, - {NAME("flush"), CTL(tcache_flush)}, - {NAME("destroy"), CTL(tcache_destroy)} -}; +static const ctl_named_node_t tcache_node[] = { + {NAME("create"), CTL(tcache_create)}, {NAME("flush"), CTL(tcache_flush)}, + {NAME("destroy"), CTL(tcache_destroy)}}; static const ctl_named_node_t arena_i_node[] = { - {NAME("initialized"), CTL(arena_i_initialized)}, - {NAME("decay"), CTL(arena_i_decay)}, - {NAME("purge"), CTL(arena_i_purge)}, - {NAME("reset"), CTL(arena_i_reset)}, - {NAME("destroy"), CTL(arena_i_destroy)}, - {NAME("dss"), CTL(arena_i_dss)}, - /* + {NAME("initialized"), CTL(arena_i_initialized)}, + {NAME("decay"), CTL(arena_i_decay)}, {NAME("purge"), CTL(arena_i_purge)}, + {NAME("reset"), CTL(arena_i_reset)}, + {NAME("destroy"), CTL(arena_i_destroy)}, {NAME("dss"), CTL(arena_i_dss)}, + /* * Undocumented for now, since we anticipate an arena API in flux after * we cut the last 5-series release. */ - {NAME("oversize_threshold"), CTL(arena_i_oversize_threshold)}, - {NAME("dirty_decay_ms"), CTL(arena_i_dirty_decay_ms)}, - {NAME("muzzy_decay_ms"), CTL(arena_i_muzzy_decay_ms)}, - {NAME("extent_hooks"), CTL(arena_i_extent_hooks)}, - {NAME("retain_grow_limit"), CTL(arena_i_retain_grow_limit)} -}; + {NAME("oversize_threshold"), CTL(arena_i_oversize_threshold)}, + {NAME("dirty_decay_ms"), CTL(arena_i_dirty_decay_ms)}, + {NAME("muzzy_decay_ms"), CTL(arena_i_muzzy_decay_ms)}, + {NAME("extent_hooks"), CTL(arena_i_extent_hooks)}, + {NAME("retain_grow_limit"), CTL(arena_i_retain_grow_limit)}, + {NAME("name"), CTL(arena_i_name)}}; static const ctl_named_node_t super_arena_i_node[] = { - {NAME(""), CHILD(named, arena_i)} -}; + {NAME(""), CHILD(named, arena_i)}}; -static const ctl_indexed_node_t arena_node[] = { - {INDEX(arena_i)} -}; +static const ctl_indexed_node_t arena_node[] = {{INDEX(arena_i)}}; static const ctl_named_node_t arenas_bin_i_node[] = { - {NAME("size"), CTL(arenas_bin_i_size)}, - {NAME("nregs"), CTL(arenas_bin_i_nregs)}, - {NAME("slab_size"), CTL(arenas_bin_i_slab_size)}, - {NAME("nshards"), CTL(arenas_bin_i_nshards)} -}; + {NAME("size"), CTL(arenas_bin_i_size)}, + {NAME("nregs"), CTL(arenas_bin_i_nregs)}, + {NAME("slab_size"), CTL(arenas_bin_i_slab_size)}, + {NAME("nshards"), CTL(arenas_bin_i_nshards)}}; static const ctl_named_node_t super_arenas_bin_i_node[] = { - {NAME(""), CHILD(named, arenas_bin_i)} -}; + {NAME(""), CHILD(named, arenas_bin_i)}}; -static const ctl_indexed_node_t arenas_bin_node[] = { - {INDEX(arenas_bin_i)} -}; +static const ctl_indexed_node_t arenas_bin_node[] = {{INDEX(arenas_bin_i)}}; static const ctl_named_node_t arenas_lextent_i_node[] = { - {NAME("size"), CTL(arenas_lextent_i_size)} -}; + {NAME("size"), CTL(arenas_lextent_i_size)}}; static const ctl_named_node_t super_arenas_lextent_i_node[] = { - {NAME(""), CHILD(named, arenas_lextent_i)} -}; + {NAME(""), CHILD(named, arenas_lextent_i)}}; static const ctl_indexed_node_t arenas_lextent_node[] = { - {INDEX(arenas_lextent_i)} -}; + {INDEX(arenas_lextent_i)}}; static const ctl_named_node_t arenas_node[] = { - {NAME("narenas"), CTL(arenas_narenas)}, - {NAME("dirty_decay_ms"), CTL(arenas_dirty_decay_ms)}, - {NAME("muzzy_decay_ms"), CTL(arenas_muzzy_decay_ms)}, - {NAME("quantum"), CTL(arenas_quantum)}, - {NAME("page"), CTL(arenas_page)}, - {NAME("tcache_max"), CTL(arenas_tcache_max)}, - {NAME("nbins"), CTL(arenas_nbins)}, - {NAME("nhbins"), CTL(arenas_nhbins)}, - {NAME("bin"), CHILD(indexed, arenas_bin)}, - {NAME("nlextents"), CTL(arenas_nlextents)}, - {NAME("lextent"), CHILD(indexed, arenas_lextent)}, - {NAME("create"), CTL(arenas_create)}, - {NAME("lookup"), CTL(arenas_lookup)} -}; + {NAME("narenas"), CTL(arenas_narenas)}, + {NAME("dirty_decay_ms"), CTL(arenas_dirty_decay_ms)}, + {NAME("muzzy_decay_ms"), CTL(arenas_muzzy_decay_ms)}, + {NAME("quantum"), CTL(arenas_quantum)}, {NAME("page"), CTL(arenas_page)}, + {NAME("hugepage"), CTL(arenas_hugepage)}, + {NAME("tcache_max"), CTL(arenas_tcache_max)}, + {NAME("nbins"), CTL(arenas_nbins)}, {NAME("nhbins"), CTL(arenas_nhbins)}, + {NAME("bin"), CHILD(indexed, arenas_bin)}, + {NAME("nlextents"), CTL(arenas_nlextents)}, + {NAME("lextent"), CHILD(indexed, arenas_lextent)}, + {NAME("create"), CTL(arenas_create)}, {NAME("lookup"), CTL(arenas_lookup)}}; static const ctl_named_node_t prof_stats_bins_i_node[] = { - {NAME("live"), CTL(prof_stats_bins_i_live)}, - {NAME("accum"), CTL(prof_stats_bins_i_accum)} -}; + {NAME("live"), CTL(prof_stats_bins_i_live)}, + {NAME("accum"), CTL(prof_stats_bins_i_accum)}}; static const ctl_named_node_t super_prof_stats_bins_i_node[] = { - {NAME(""), CHILD(named, prof_stats_bins_i)} -}; + {NAME(""), CHILD(named, prof_stats_bins_i)}}; static const ctl_indexed_node_t prof_stats_bins_node[] = { - {INDEX(prof_stats_bins_i)} -}; + {INDEX(prof_stats_bins_i)}}; static const ctl_named_node_t prof_stats_lextents_i_node[] = { - {NAME("live"), CTL(prof_stats_lextents_i_live)}, - {NAME("accum"), CTL(prof_stats_lextents_i_accum)} -}; + {NAME("live"), CTL(prof_stats_lextents_i_live)}, + {NAME("accum"), CTL(prof_stats_lextents_i_accum)}}; static const ctl_named_node_t super_prof_stats_lextents_i_node[] = { - {NAME(""), CHILD(named, prof_stats_lextents_i)} -}; + {NAME(""), CHILD(named, prof_stats_lextents_i)}}; static const ctl_indexed_node_t prof_stats_lextents_node[] = { - {INDEX(prof_stats_lextents_i)} + {INDEX(prof_stats_lextents_i)}}; + +static const ctl_named_node_t prof_stats_node[] = { + {NAME("bins"), CHILD(indexed, prof_stats_bins)}, + {NAME("lextents"), CHILD(indexed, prof_stats_lextents)}, }; -static const ctl_named_node_t prof_stats_node[] = { - {NAME("bins"), CHILD(indexed, prof_stats_bins)}, - {NAME("lextents"), CHILD(indexed, prof_stats_lextents)}, -}; - -static const ctl_named_node_t prof_node[] = { - {NAME("thread_active_init"), CTL(prof_thread_active_init)}, - {NAME("active"), CTL(prof_active)}, - {NAME("dump"), CTL(prof_dump)}, - {NAME("gdump"), CTL(prof_gdump)}, - {NAME("prefix"), CTL(prof_prefix)}, - {NAME("reset"), CTL(prof_reset)}, - {NAME("interval"), CTL(prof_interval)}, - {NAME("lg_sample"), CTL(lg_prof_sample)}, - {NAME("log_start"), CTL(prof_log_start)}, - {NAME("log_stop"), CTL(prof_log_stop)}, - {NAME("stats"), CHILD(named, prof_stats)} -}; +static const ctl_named_node_t prof_node[] = { + {NAME("thread_active_init"), CTL(prof_thread_active_init)}, + {NAME("active"), CTL(prof_active)}, {NAME("dump"), CTL(prof_dump)}, + {NAME("gdump"), CTL(prof_gdump)}, {NAME("prefix"), CTL(prof_prefix)}, + {NAME("reset"), CTL(prof_reset)}, {NAME("interval"), CTL(prof_interval)}, + {NAME("lg_sample"), CTL(lg_prof_sample)}, + {NAME("log_start"), CTL(prof_log_start)}, + {NAME("log_stop"), CTL(prof_log_stop)}, + {NAME("stats"), CHILD(named, prof_stats)}}; static const ctl_named_node_t stats_arenas_i_small_node[] = { - {NAME("allocated"), CTL(stats_arenas_i_small_allocated)}, - {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)}, - {NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)}, - {NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)}, - {NAME("nfills"), CTL(stats_arenas_i_small_nfills)}, - {NAME("nflushes"), CTL(stats_arenas_i_small_nflushes)} -}; + {NAME("allocated"), CTL(stats_arenas_i_small_allocated)}, + {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)}, + {NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)}, + {NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)}, + {NAME("nfills"), CTL(stats_arenas_i_small_nfills)}, + {NAME("nflushes"), CTL(stats_arenas_i_small_nflushes)}}; static const ctl_named_node_t stats_arenas_i_large_node[] = { - {NAME("allocated"), CTL(stats_arenas_i_large_allocated)}, - {NAME("nmalloc"), CTL(stats_arenas_i_large_nmalloc)}, - {NAME("ndalloc"), CTL(stats_arenas_i_large_ndalloc)}, - {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)}, - {NAME("nfills"), CTL(stats_arenas_i_large_nfills)}, - {NAME("nflushes"), CTL(stats_arenas_i_large_nflushes)} -}; + {NAME("allocated"), CTL(stats_arenas_i_large_allocated)}, + {NAME("nmalloc"), CTL(stats_arenas_i_large_nmalloc)}, + {NAME("ndalloc"), CTL(stats_arenas_i_large_ndalloc)}, + {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)}, + {NAME("nfills"), CTL(stats_arenas_i_large_nfills)}, + {NAME("nflushes"), CTL(stats_arenas_i_large_nflushes)}}; -#define MUTEX_PROF_DATA_NODE(prefix) \ -static const ctl_named_node_t stats_##prefix##_node[] = { \ - {NAME("num_ops"), \ - CTL(stats_##prefix##_num_ops)}, \ - {NAME("num_wait"), \ - CTL(stats_##prefix##_num_wait)}, \ - {NAME("num_spin_acq"), \ - CTL(stats_##prefix##_num_spin_acq)}, \ - {NAME("num_owner_switch"), \ - CTL(stats_##prefix##_num_owner_switch)}, \ - {NAME("total_wait_time"), \ - CTL(stats_##prefix##_total_wait_time)}, \ - {NAME("max_wait_time"), \ - CTL(stats_##prefix##_max_wait_time)}, \ - {NAME("max_num_thds"), \ - CTL(stats_##prefix##_max_num_thds)} \ - /* Note that # of current waiting thread not provided. */ \ -}; +#define MUTEX_PROF_DATA_NODE(prefix) \ + static const ctl_named_node_t stats_##prefix##_node[] = { \ + {NAME("num_ops"), CTL(stats_##prefix##_num_ops)}, \ + {NAME("num_wait"), CTL(stats_##prefix##_num_wait)}, \ + {NAME("num_spin_acq"), CTL(stats_##prefix##_num_spin_acq)}, \ + {NAME("num_owner_switch"), \ + CTL(stats_##prefix##_num_owner_switch)}, \ + {NAME("total_wait_time"), CTL(stats_##prefix##_total_wait_time)}, \ + {NAME("max_wait_time"), CTL(stats_##prefix##_max_wait_time)}, \ + {NAME("max_num_thds"), \ + CTL(stats_##prefix##_max_num_thds)} /* Note that # of current waiting thread not provided. */ \ + }; MUTEX_PROF_DATA_NODE(arenas_i_bins_j_mutex) static const ctl_named_node_t stats_arenas_i_bins_j_node[] = { - {NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)}, - {NAME("ndalloc"), CTL(stats_arenas_i_bins_j_ndalloc)}, - {NAME("nrequests"), CTL(stats_arenas_i_bins_j_nrequests)}, - {NAME("curregs"), CTL(stats_arenas_i_bins_j_curregs)}, - {NAME("nfills"), CTL(stats_arenas_i_bins_j_nfills)}, - {NAME("nflushes"), CTL(stats_arenas_i_bins_j_nflushes)}, - {NAME("nslabs"), CTL(stats_arenas_i_bins_j_nslabs)}, - {NAME("nreslabs"), CTL(stats_arenas_i_bins_j_nreslabs)}, - {NAME("curslabs"), CTL(stats_arenas_i_bins_j_curslabs)}, - {NAME("nonfull_slabs"), CTL(stats_arenas_i_bins_j_nonfull_slabs)}, - {NAME("mutex"), CHILD(named, stats_arenas_i_bins_j_mutex)} -}; + {NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)}, + {NAME("ndalloc"), CTL(stats_arenas_i_bins_j_ndalloc)}, + {NAME("nrequests"), CTL(stats_arenas_i_bins_j_nrequests)}, + {NAME("curregs"), CTL(stats_arenas_i_bins_j_curregs)}, + {NAME("nfills"), CTL(stats_arenas_i_bins_j_nfills)}, + {NAME("nflushes"), CTL(stats_arenas_i_bins_j_nflushes)}, + {NAME("nslabs"), CTL(stats_arenas_i_bins_j_nslabs)}, + {NAME("nreslabs"), CTL(stats_arenas_i_bins_j_nreslabs)}, + {NAME("curslabs"), CTL(stats_arenas_i_bins_j_curslabs)}, + {NAME("nonfull_slabs"), CTL(stats_arenas_i_bins_j_nonfull_slabs)}, + {NAME("mutex"), CHILD(named, stats_arenas_i_bins_j_mutex)}}; static const ctl_named_node_t super_stats_arenas_i_bins_j_node[] = { - {NAME(""), CHILD(named, stats_arenas_i_bins_j)} -}; + {NAME(""), CHILD(named, stats_arenas_i_bins_j)}}; static const ctl_indexed_node_t stats_arenas_i_bins_node[] = { - {INDEX(stats_arenas_i_bins_j)} -}; + {INDEX(stats_arenas_i_bins_j)}}; static const ctl_named_node_t stats_arenas_i_lextents_j_node[] = { - {NAME("nmalloc"), CTL(stats_arenas_i_lextents_j_nmalloc)}, - {NAME("ndalloc"), CTL(stats_arenas_i_lextents_j_ndalloc)}, - {NAME("nrequests"), CTL(stats_arenas_i_lextents_j_nrequests)}, - {NAME("curlextents"), CTL(stats_arenas_i_lextents_j_curlextents)} -}; + {NAME("nmalloc"), CTL(stats_arenas_i_lextents_j_nmalloc)}, + {NAME("ndalloc"), CTL(stats_arenas_i_lextents_j_ndalloc)}, + {NAME("nrequests"), CTL(stats_arenas_i_lextents_j_nrequests)}, + {NAME("curlextents"), CTL(stats_arenas_i_lextents_j_curlextents)}}; static const ctl_named_node_t super_stats_arenas_i_lextents_j_node[] = { - {NAME(""), CHILD(named, stats_arenas_i_lextents_j)} -}; + {NAME(""), CHILD(named, stats_arenas_i_lextents_j)}}; static const ctl_indexed_node_t stats_arenas_i_lextents_node[] = { - {INDEX(stats_arenas_i_lextents_j)} -}; + {INDEX(stats_arenas_i_lextents_j)}}; static const ctl_named_node_t stats_arenas_i_extents_j_node[] = { - {NAME("ndirty"), CTL(stats_arenas_i_extents_j_ndirty)}, - {NAME("nmuzzy"), CTL(stats_arenas_i_extents_j_nmuzzy)}, - {NAME("nretained"), CTL(stats_arenas_i_extents_j_nretained)}, - {NAME("dirty_bytes"), CTL(stats_arenas_i_extents_j_dirty_bytes)}, - {NAME("muzzy_bytes"), CTL(stats_arenas_i_extents_j_muzzy_bytes)}, - {NAME("retained_bytes"), CTL(stats_arenas_i_extents_j_retained_bytes)} -}; + {NAME("ndirty"), CTL(stats_arenas_i_extents_j_ndirty)}, + {NAME("nmuzzy"), CTL(stats_arenas_i_extents_j_nmuzzy)}, + {NAME("nretained"), CTL(stats_arenas_i_extents_j_nretained)}, + {NAME("dirty_bytes"), CTL(stats_arenas_i_extents_j_dirty_bytes)}, + {NAME("muzzy_bytes"), CTL(stats_arenas_i_extents_j_muzzy_bytes)}, + {NAME("retained_bytes"), CTL(stats_arenas_i_extents_j_retained_bytes)}}; static const ctl_named_node_t super_stats_arenas_i_extents_j_node[] = { - {NAME(""), CHILD(named, stats_arenas_i_extents_j)} -}; + {NAME(""), CHILD(named, stats_arenas_i_extents_j)}}; static const ctl_indexed_node_t stats_arenas_i_extents_node[] = { - {INDEX(stats_arenas_i_extents_j)} -}; + {INDEX(stats_arenas_i_extents_j)}}; -#define OP(mtx) MUTEX_PROF_DATA_NODE(arenas_i_mutexes_##mtx) +#define OP(mtx) MUTEX_PROF_DATA_NODE(arenas_i_mutexes_##mtx) MUTEX_PROF_ARENA_MUTEXES #undef OP static const ctl_named_node_t stats_arenas_i_mutexes_node[] = { #define OP(mtx) {NAME(#mtx), CHILD(named, stats_arenas_i_mutexes_##mtx)}, -MUTEX_PROF_ARENA_MUTEXES + MUTEX_PROF_ARENA_MUTEXES #undef OP }; +static const ctl_named_node_t stats_arenas_i_hpa_shard_slabs_node[] = { + {NAME("npageslabs_nonhuge"), + CTL(stats_arenas_i_hpa_shard_slabs_npageslabs_nonhuge)}, + {NAME("npageslabs_huge"), + CTL(stats_arenas_i_hpa_shard_slabs_npageslabs_huge)}, + {NAME("nactive_nonhuge"), + CTL(stats_arenas_i_hpa_shard_slabs_nactive_nonhuge)}, + {NAME("nactive_huge"), CTL(stats_arenas_i_hpa_shard_slabs_nactive_huge)}, + {NAME("ndirty_nonhuge"), + CTL(stats_arenas_i_hpa_shard_slabs_ndirty_nonhuge)}, + {NAME("ndirty_huge"), CTL(stats_arenas_i_hpa_shard_slabs_ndirty_huge)}}; + static const ctl_named_node_t stats_arenas_i_hpa_shard_full_slabs_node[] = { - {NAME("npageslabs_nonhuge"), - CTL(stats_arenas_i_hpa_shard_full_slabs_npageslabs_nonhuge)}, - {NAME("npageslabs_huge"), - CTL(stats_arenas_i_hpa_shard_full_slabs_npageslabs_huge)}, - {NAME("nactive_nonhuge"), - CTL(stats_arenas_i_hpa_shard_full_slabs_nactive_nonhuge)}, - {NAME("nactive_huge"), - CTL(stats_arenas_i_hpa_shard_full_slabs_nactive_huge)}, - {NAME("ndirty_nonhuge"), - CTL(stats_arenas_i_hpa_shard_full_slabs_ndirty_nonhuge)}, - {NAME("ndirty_huge"), - CTL(stats_arenas_i_hpa_shard_full_slabs_ndirty_huge)} -}; + {NAME("npageslabs_nonhuge"), + CTL(stats_arenas_i_hpa_shard_full_slabs_npageslabs_nonhuge)}, + {NAME("npageslabs_huge"), + CTL(stats_arenas_i_hpa_shard_full_slabs_npageslabs_huge)}, + {NAME("nactive_nonhuge"), + CTL(stats_arenas_i_hpa_shard_full_slabs_nactive_nonhuge)}, + {NAME("nactive_huge"), + CTL(stats_arenas_i_hpa_shard_full_slabs_nactive_huge)}, + {NAME("ndirty_nonhuge"), + CTL(stats_arenas_i_hpa_shard_full_slabs_ndirty_nonhuge)}, + {NAME("ndirty_huge"), + CTL(stats_arenas_i_hpa_shard_full_slabs_ndirty_huge)}}; static const ctl_named_node_t stats_arenas_i_hpa_shard_empty_slabs_node[] = { - {NAME("npageslabs_nonhuge"), - CTL(stats_arenas_i_hpa_shard_empty_slabs_npageslabs_nonhuge)}, - {NAME("npageslabs_huge"), - CTL(stats_arenas_i_hpa_shard_empty_slabs_npageslabs_huge)}, - {NAME("nactive_nonhuge"), - CTL(stats_arenas_i_hpa_shard_empty_slabs_nactive_nonhuge)}, - {NAME("nactive_huge"), - CTL(stats_arenas_i_hpa_shard_empty_slabs_nactive_huge)}, - {NAME("ndirty_nonhuge"), - CTL(stats_arenas_i_hpa_shard_empty_slabs_ndirty_nonhuge)}, - {NAME("ndirty_huge"), - CTL(stats_arenas_i_hpa_shard_empty_slabs_ndirty_huge)} -}; + {NAME("npageslabs_nonhuge"), + CTL(stats_arenas_i_hpa_shard_empty_slabs_npageslabs_nonhuge)}, + {NAME("npageslabs_huge"), + CTL(stats_arenas_i_hpa_shard_empty_slabs_npageslabs_huge)}, + {NAME("nactive_nonhuge"), + CTL(stats_arenas_i_hpa_shard_empty_slabs_nactive_nonhuge)}, + {NAME("nactive_huge"), + CTL(stats_arenas_i_hpa_shard_empty_slabs_nactive_huge)}, + {NAME("ndirty_nonhuge"), + CTL(stats_arenas_i_hpa_shard_empty_slabs_ndirty_nonhuge)}, + {NAME("ndirty_huge"), + CTL(stats_arenas_i_hpa_shard_empty_slabs_ndirty_huge)}}; -static const ctl_named_node_t stats_arenas_i_hpa_shard_nonfull_slabs_j_node[] = { - {NAME("npageslabs_nonhuge"), - CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_nonhuge)}, - {NAME("npageslabs_huge"), - CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_huge)}, - {NAME("nactive_nonhuge"), - CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_nonhuge)}, - {NAME("nactive_huge"), - CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_huge)}, - {NAME("ndirty_nonhuge"), - CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_nonhuge)}, - {NAME("ndirty_huge"), - CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_huge)} -}; +static const ctl_named_node_t stats_arenas_i_hpa_shard_nonfull_slabs_j_node[] = + {{NAME("npageslabs_nonhuge"), + CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_nonhuge)}, + {NAME("npageslabs_huge"), + CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_huge)}, + {NAME("nactive_nonhuge"), + CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_nonhuge)}, + {NAME("nactive_huge"), + CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_huge)}, + {NAME("ndirty_nonhuge"), + CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_nonhuge)}, + {NAME("ndirty_huge"), + CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_huge)}}; -static const ctl_named_node_t super_stats_arenas_i_hpa_shard_nonfull_slabs_j_node[] = { - {NAME(""), - CHILD(named, stats_arenas_i_hpa_shard_nonfull_slabs_j)} -}; +static const ctl_named_node_t + super_stats_arenas_i_hpa_shard_nonfull_slabs_j_node[] = { + {NAME(""), CHILD(named, stats_arenas_i_hpa_shard_nonfull_slabs_j)}}; static const ctl_indexed_node_t stats_arenas_i_hpa_shard_nonfull_slabs_node[] = -{ - {INDEX(stats_arenas_i_hpa_shard_nonfull_slabs_j)} -}; + {{INDEX(stats_arenas_i_hpa_shard_nonfull_slabs_j)}}; static const ctl_named_node_t stats_arenas_i_hpa_shard_node[] = { - {NAME("full_slabs"), CHILD(named, - stats_arenas_i_hpa_shard_full_slabs)}, - {NAME("empty_slabs"), CHILD(named, - stats_arenas_i_hpa_shard_empty_slabs)}, - {NAME("nonfull_slabs"), CHILD(indexed, - stats_arenas_i_hpa_shard_nonfull_slabs)}, + {NAME("npageslabs"), CTL(stats_arenas_i_hpa_shard_npageslabs)}, + {NAME("nactive"), CTL(stats_arenas_i_hpa_shard_nactive)}, + {NAME("ndirty"), CTL(stats_arenas_i_hpa_shard_ndirty)}, - {NAME("npurge_passes"), CTL(stats_arenas_i_hpa_shard_npurge_passes)}, - {NAME("npurges"), CTL(stats_arenas_i_hpa_shard_npurges)}, - {NAME("nhugifies"), CTL(stats_arenas_i_hpa_shard_nhugifies)}, - {NAME("ndehugifies"), CTL(stats_arenas_i_hpa_shard_ndehugifies)} -}; + {NAME("slabs"), CHILD(named, stats_arenas_i_hpa_shard_slabs)}, + + {NAME("npurge_passes"), CTL(stats_arenas_i_hpa_shard_npurge_passes)}, + {NAME("npurges"), CTL(stats_arenas_i_hpa_shard_npurges)}, + {NAME("nhugifies"), CTL(stats_arenas_i_hpa_shard_nhugifies)}, + {NAME("nhugify_failures"), CTL(stats_arenas_i_hpa_shard_nhugify_failures)}, + {NAME("ndehugifies"), CTL(stats_arenas_i_hpa_shard_ndehugifies)}, + + {NAME("full_slabs"), CHILD(named, stats_arenas_i_hpa_shard_full_slabs)}, + {NAME("empty_slabs"), CHILD(named, stats_arenas_i_hpa_shard_empty_slabs)}, + {NAME("nonfull_slabs"), + CHILD(indexed, stats_arenas_i_hpa_shard_nonfull_slabs)}}; static const ctl_named_node_t stats_arenas_i_node[] = { - {NAME("nthreads"), CTL(stats_arenas_i_nthreads)}, - {NAME("uptime"), CTL(stats_arenas_i_uptime)}, - {NAME("dss"), CTL(stats_arenas_i_dss)}, - {NAME("dirty_decay_ms"), CTL(stats_arenas_i_dirty_decay_ms)}, - {NAME("muzzy_decay_ms"), CTL(stats_arenas_i_muzzy_decay_ms)}, - {NAME("pactive"), CTL(stats_arenas_i_pactive)}, - {NAME("pdirty"), CTL(stats_arenas_i_pdirty)}, - {NAME("pmuzzy"), CTL(stats_arenas_i_pmuzzy)}, - {NAME("mapped"), CTL(stats_arenas_i_mapped)}, - {NAME("retained"), CTL(stats_arenas_i_retained)}, - {NAME("extent_avail"), CTL(stats_arenas_i_extent_avail)}, - {NAME("dirty_npurge"), CTL(stats_arenas_i_dirty_npurge)}, - {NAME("dirty_nmadvise"), CTL(stats_arenas_i_dirty_nmadvise)}, - {NAME("dirty_purged"), CTL(stats_arenas_i_dirty_purged)}, - {NAME("muzzy_npurge"), CTL(stats_arenas_i_muzzy_npurge)}, - {NAME("muzzy_nmadvise"), CTL(stats_arenas_i_muzzy_nmadvise)}, - {NAME("muzzy_purged"), CTL(stats_arenas_i_muzzy_purged)}, - {NAME("base"), CTL(stats_arenas_i_base)}, - {NAME("internal"), CTL(stats_arenas_i_internal)}, - {NAME("metadata_thp"), CTL(stats_arenas_i_metadata_thp)}, - {NAME("tcache_bytes"), CTL(stats_arenas_i_tcache_bytes)}, - {NAME("tcache_stashed_bytes"), - CTL(stats_arenas_i_tcache_stashed_bytes)}, - {NAME("resident"), CTL(stats_arenas_i_resident)}, - {NAME("abandoned_vm"), CTL(stats_arenas_i_abandoned_vm)}, - {NAME("hpa_sec_bytes"), CTL(stats_arenas_i_hpa_sec_bytes)}, - {NAME("small"), CHILD(named, stats_arenas_i_small)}, - {NAME("large"), CHILD(named, stats_arenas_i_large)}, - {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)}, - {NAME("lextents"), CHILD(indexed, stats_arenas_i_lextents)}, - {NAME("extents"), CHILD(indexed, stats_arenas_i_extents)}, - {NAME("mutexes"), CHILD(named, stats_arenas_i_mutexes)}, - {NAME("hpa_shard"), CHILD(named, stats_arenas_i_hpa_shard)} -}; + {NAME("nthreads"), CTL(stats_arenas_i_nthreads)}, + {NAME("uptime"), CTL(stats_arenas_i_uptime)}, + {NAME("dss"), CTL(stats_arenas_i_dss)}, + {NAME("dirty_decay_ms"), CTL(stats_arenas_i_dirty_decay_ms)}, + {NAME("muzzy_decay_ms"), CTL(stats_arenas_i_muzzy_decay_ms)}, + {NAME("pactive"), CTL(stats_arenas_i_pactive)}, + {NAME("pdirty"), CTL(stats_arenas_i_pdirty)}, + {NAME("pmuzzy"), CTL(stats_arenas_i_pmuzzy)}, + {NAME("mapped"), CTL(stats_arenas_i_mapped)}, + {NAME("retained"), CTL(stats_arenas_i_retained)}, + {NAME("extent_avail"), CTL(stats_arenas_i_extent_avail)}, + {NAME("dirty_npurge"), CTL(stats_arenas_i_dirty_npurge)}, + {NAME("dirty_nmadvise"), CTL(stats_arenas_i_dirty_nmadvise)}, + {NAME("dirty_purged"), CTL(stats_arenas_i_dirty_purged)}, + {NAME("muzzy_npurge"), CTL(stats_arenas_i_muzzy_npurge)}, + {NAME("muzzy_nmadvise"), CTL(stats_arenas_i_muzzy_nmadvise)}, + {NAME("muzzy_purged"), CTL(stats_arenas_i_muzzy_purged)}, + {NAME("base"), CTL(stats_arenas_i_base)}, + {NAME("internal"), CTL(stats_arenas_i_internal)}, + {NAME("metadata_edata"), CTL(stats_arenas_i_metadata_edata)}, + {NAME("metadata_rtree"), CTL(stats_arenas_i_metadata_rtree)}, + {NAME("metadata_thp"), CTL(stats_arenas_i_metadata_thp)}, + {NAME("tcache_bytes"), CTL(stats_arenas_i_tcache_bytes)}, + {NAME("tcache_stashed_bytes"), CTL(stats_arenas_i_tcache_stashed_bytes)}, + {NAME("resident"), CTL(stats_arenas_i_resident)}, + {NAME("abandoned_vm"), CTL(stats_arenas_i_abandoned_vm)}, + {NAME("hpa_sec_bytes"), CTL(stats_arenas_i_hpa_sec_bytes)}, + {NAME("hpa_sec_hits"), CTL(stats_arenas_i_hpa_sec_hits)}, + {NAME("hpa_sec_misses"), CTL(stats_arenas_i_hpa_sec_misses)}, + {NAME("hpa_sec_dalloc_noflush"), + CTL(stats_arenas_i_hpa_sec_dalloc_noflush)}, + {NAME("hpa_sec_dalloc_flush"), CTL(stats_arenas_i_hpa_sec_dalloc_flush)}, + {NAME("hpa_sec_overfills"), CTL(stats_arenas_i_hpa_sec_overfills)}, + {NAME("small"), CHILD(named, stats_arenas_i_small)}, + {NAME("large"), CHILD(named, stats_arenas_i_large)}, + {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)}, + {NAME("lextents"), CHILD(indexed, stats_arenas_i_lextents)}, + {NAME("extents"), CHILD(indexed, stats_arenas_i_extents)}, + {NAME("mutexes"), CHILD(named, stats_arenas_i_mutexes)}, + {NAME("hpa_shard"), CHILD(named, stats_arenas_i_hpa_shard)}}; static const ctl_named_node_t super_stats_arenas_i_node[] = { - {NAME(""), CHILD(named, stats_arenas_i)} -}; + {NAME(""), CHILD(named, stats_arenas_i)}}; -static const ctl_indexed_node_t stats_arenas_node[] = { - {INDEX(stats_arenas_i)} -}; +static const ctl_indexed_node_t stats_arenas_node[] = {{INDEX(stats_arenas_i)}}; static const ctl_named_node_t stats_background_thread_node[] = { - {NAME("num_threads"), CTL(stats_background_thread_num_threads)}, - {NAME("num_runs"), CTL(stats_background_thread_num_runs)}, - {NAME("run_interval"), CTL(stats_background_thread_run_interval)} -}; + {NAME("num_threads"), CTL(stats_background_thread_num_threads)}, + {NAME("num_runs"), CTL(stats_background_thread_num_runs)}, + {NAME("run_interval"), CTL(stats_background_thread_run_interval)}}; #define OP(mtx) MUTEX_PROF_DATA_NODE(mutexes_##mtx) MUTEX_PROF_GLOBAL_MUTEXES @@ -825,89 +853,81 @@ MUTEX_PROF_GLOBAL_MUTEXES static const ctl_named_node_t stats_mutexes_node[] = { #define OP(mtx) {NAME(#mtx), CHILD(named, stats_mutexes_##mtx)}, -MUTEX_PROF_GLOBAL_MUTEXES + MUTEX_PROF_GLOBAL_MUTEXES #undef OP - {NAME("reset"), CTL(stats_mutexes_reset)} -}; + {NAME("reset"), CTL(stats_mutexes_reset)}}; #undef MUTEX_PROF_DATA_NODE +static const ctl_named_node_t approximate_stats_node[] = { + {NAME("active"), CTL(approximate_stats_active)}, +}; + static const ctl_named_node_t stats_node[] = { - {NAME("allocated"), CTL(stats_allocated)}, - {NAME("active"), CTL(stats_active)}, - {NAME("metadata"), CTL(stats_metadata)}, - {NAME("metadata_thp"), CTL(stats_metadata_thp)}, - {NAME("resident"), CTL(stats_resident)}, - {NAME("mapped"), CTL(stats_mapped)}, - {NAME("retained"), CTL(stats_retained)}, - {NAME("background_thread"), - CHILD(named, stats_background_thread)}, - {NAME("mutexes"), CHILD(named, stats_mutexes)}, - {NAME("arenas"), CHILD(indexed, stats_arenas)}, - {NAME("zero_reallocs"), CTL(stats_zero_reallocs)}, + {NAME("allocated"), CTL(stats_allocated)}, + {NAME("active"), CTL(stats_active)}, + {NAME("metadata"), CTL(stats_metadata)}, + {NAME("metadata_edata"), CTL(stats_metadata_edata)}, + {NAME("metadata_rtree"), CTL(stats_metadata_rtree)}, + {NAME("metadata_thp"), CTL(stats_metadata_thp)}, + {NAME("resident"), CTL(stats_resident)}, + {NAME("mapped"), CTL(stats_mapped)}, + {NAME("retained"), CTL(stats_retained)}, + {NAME("background_thread"), CHILD(named, stats_background_thread)}, + {NAME("mutexes"), CHILD(named, stats_mutexes)}, + {NAME("arenas"), CHILD(indexed, stats_arenas)}, + {NAME("zero_reallocs"), CTL(stats_zero_reallocs)}, }; static const ctl_named_node_t experimental_hooks_node[] = { - {NAME("install"), CTL(experimental_hooks_install)}, - {NAME("remove"), CTL(experimental_hooks_remove)}, - {NAME("prof_backtrace"), CTL(experimental_hooks_prof_backtrace)}, - {NAME("prof_dump"), CTL(experimental_hooks_prof_dump)}, - {NAME("safety_check_abort"), CTL(experimental_hooks_safety_check_abort)}, -}; - -static const ctl_named_node_t experimental_thread_node[] = { - {NAME("activity_callback"), - CTL(experimental_thread_activity_callback)} + {NAME("install"), CTL(experimental_hooks_install)}, + {NAME("remove"), CTL(experimental_hooks_remove)}, + {NAME("prof_backtrace"), CTL(experimental_hooks_prof_backtrace)}, + {NAME("prof_dump"), CTL(experimental_hooks_prof_dump)}, + {NAME("prof_sample"), CTL(experimental_hooks_prof_sample)}, + {NAME("prof_sample_free"), CTL(experimental_hooks_prof_sample_free)}, + {NAME("safety_check_abort"), CTL(experimental_hooks_safety_check_abort)}, + {NAME("thread_event"), CTL(experimental_hooks_thread_event)}, }; static const ctl_named_node_t experimental_utilization_node[] = { - {NAME("query"), CTL(experimental_utilization_query)}, - {NAME("batch_query"), CTL(experimental_utilization_batch_query)} -}; + {NAME("query"), CTL(experimental_utilization_query)}, + {NAME("batch_query"), CTL(experimental_utilization_batch_query)}}; static const ctl_named_node_t experimental_arenas_i_node[] = { - {NAME("pactivep"), CTL(experimental_arenas_i_pactivep)} -}; + {NAME("pactivep"), CTL(experimental_arenas_i_pactivep)}}; static const ctl_named_node_t super_experimental_arenas_i_node[] = { - {NAME(""), CHILD(named, experimental_arenas_i)} -}; + {NAME(""), CHILD(named, experimental_arenas_i)}}; static const ctl_indexed_node_t experimental_arenas_node[] = { - {INDEX(experimental_arenas_i)} -}; + {INDEX(experimental_arenas_i)}}; static const ctl_named_node_t experimental_prof_recent_node[] = { - {NAME("alloc_max"), CTL(experimental_prof_recent_alloc_max)}, - {NAME("alloc_dump"), CTL(experimental_prof_recent_alloc_dump)}, + {NAME("alloc_max"), CTL(experimental_prof_recent_alloc_max)}, + {NAME("alloc_dump"), CTL(experimental_prof_recent_alloc_dump)}, }; static const ctl_named_node_t experimental_node[] = { - {NAME("hooks"), CHILD(named, experimental_hooks)}, - {NAME("utilization"), CHILD(named, experimental_utilization)}, - {NAME("arenas"), CHILD(indexed, experimental_arenas)}, - {NAME("arenas_create_ext"), CTL(experimental_arenas_create_ext)}, - {NAME("prof_recent"), CHILD(named, experimental_prof_recent)}, - {NAME("batch_alloc"), CTL(experimental_batch_alloc)}, - {NAME("thread"), CHILD(named, experimental_thread)} -}; + {NAME("hooks"), CHILD(named, experimental_hooks)}, + {NAME("utilization"), CHILD(named, experimental_utilization)}, + {NAME("arenas"), CHILD(indexed, experimental_arenas)}, + {NAME("arenas_create_ext"), CTL(experimental_arenas_create_ext)}, + {NAME("prof_recent"), CHILD(named, experimental_prof_recent)}, + {NAME("batch_alloc"), CTL(experimental_batch_alloc)}}; -static const ctl_named_node_t root_node[] = { - {NAME("version"), CTL(version)}, - {NAME("epoch"), CTL(epoch)}, - {NAME("background_thread"), CTL(background_thread)}, - {NAME("max_background_threads"), CTL(max_background_threads)}, - {NAME("thread"), CHILD(named, thread)}, - {NAME("config"), CHILD(named, config)}, - {NAME("opt"), CHILD(named, opt)}, - {NAME("tcache"), CHILD(named, tcache)}, - {NAME("arena"), CHILD(indexed, arena)}, - {NAME("arenas"), CHILD(named, arenas)}, - {NAME("prof"), CHILD(named, prof)}, - {NAME("stats"), CHILD(named, stats)}, - {NAME("experimental"), CHILD(named, experimental)} -}; +static const ctl_named_node_t root_node[] = {{NAME("version"), CTL(version)}, + {NAME("epoch"), CTL(epoch)}, + {NAME("background_thread"), CTL(background_thread)}, + {NAME("max_background_threads"), CTL(max_background_threads)}, + {NAME("thread"), CHILD(named, thread)}, + {NAME("config"), CHILD(named, config)}, {NAME("opt"), CHILD(named, opt)}, + {NAME("tcache"), CHILD(named, tcache)}, + {NAME("arena"), CHILD(indexed, arena)}, + {NAME("arenas"), CHILD(named, arenas)}, {NAME("prof"), CHILD(named, prof)}, + {NAME("stats"), CHILD(named, stats)}, + {NAME("approximate_stats"), CHILD(named, approximate_stats)}, + {NAME("experimental"), CHILD(named, experimental)}}; static const ctl_named_node_t super_root_node[] = { - {NAME(""), CHILD(named, root)} -}; + {NAME(""), CHILD(named, root)}}; #undef NAME #undef CHILD @@ -922,8 +942,7 @@ static const ctl_named_node_t super_root_node[] = { */ static void ctl_accum_locked_u64(locked_u64_t *dst, locked_u64_t *src) { - locked_inc_u64_unsynchronized(dst, - locked_read_u64_unsynchronized(src)); + locked_inc_u64_unsynchronized(dst, locked_read_u64_unsynchronized(src)); } static void @@ -963,8 +982,8 @@ arenas_i2a_impl(size_t i, bool compat, bool validate) { * more than one past the range of indices that have * initialized ctl data. */ - assert(i < ctl_arenas->narenas || (!validate && i == - ctl_arenas->narenas)); + assert(i < ctl_arenas->narenas + || (!validate && i == ctl_arenas->narenas)); a = (unsigned)i + 2; } break; @@ -988,12 +1007,12 @@ arenas_i_impl(tsd_t *tsd, size_t i, bool compat, bool init) { if (init && ret == NULL) { if (config_stats) { struct container_s { - ctl_arena_t ctl_arena; - ctl_arena_stats_t astats; + ctl_arena_t ctl_arena; + ctl_arena_stats_t astats; }; - struct container_s *cont = - (struct container_s *)base_alloc(tsd_tsdn(tsd), - b0get(), sizeof(struct container_s), QUANTUM); + struct container_s *cont = (struct container_s *) + base_alloc(tsd_tsdn(tsd), b0get(), + sizeof(struct container_s), QUANTUM); if (cont == NULL) { return NULL; } @@ -1031,23 +1050,7 @@ ctl_arena_clear(ctl_arena_t *ctl_arena) { ctl_arena->pdirty = 0; ctl_arena->pmuzzy = 0; if (config_stats) { - memset(&ctl_arena->astats->astats, 0, sizeof(arena_stats_t)); - ctl_arena->astats->allocated_small = 0; - ctl_arena->astats->nmalloc_small = 0; - ctl_arena->astats->ndalloc_small = 0; - ctl_arena->astats->nrequests_small = 0; - ctl_arena->astats->nfills_small = 0; - ctl_arena->astats->nflushes_small = 0; - memset(ctl_arena->astats->bstats, 0, SC_NBINS * - sizeof(bin_stats_data_t)); - memset(ctl_arena->astats->lstats, 0, (SC_NSIZES - SC_NBINS) * - sizeof(arena_stats_large_t)); - memset(ctl_arena->astats->estats, 0, SC_NPSIZES * - sizeof(pac_estats_t)); - memset(&ctl_arena->astats->hpastats, 0, - sizeof(hpa_shard_stats_t)); - memset(&ctl_arena->astats->secstats, 0, - sizeof(sec_stats_t)); + memset(ctl_arena->astats, 0, sizeof(*(ctl_arena->astats))); } } @@ -1062,13 +1065,13 @@ ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) { &ctl_arena->pdirty, &ctl_arena->pmuzzy, &ctl_arena->astats->astats, ctl_arena->astats->bstats, ctl_arena->astats->lstats, ctl_arena->astats->estats, - &ctl_arena->astats->hpastats, &ctl_arena->astats->secstats); + &ctl_arena->astats->hpastats); for (i = 0; i < SC_NBINS; i++) { bin_stats_t *bstats = &ctl_arena->astats->bstats[i].stats_data; - ctl_arena->astats->allocated_small += bstats->curregs * - sz_index2size(i); + ctl_arena->astats->allocated_small += bstats->curregs + * sz_index2size(i); ctl_arena->astats->nmalloc_small += bstats->nmalloc; ctl_arena->astats->ndalloc_small += bstats->ndalloc; ctl_arena->astats->nrequests_small += bstats->nrequests; @@ -1084,8 +1087,8 @@ ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) { } static void -ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena, - bool destroyed) { +ctl_arena_stats_sdmerge( + ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena, bool destroyed) { unsigned i; if (!destroyed) { @@ -1106,48 +1109,59 @@ ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena, if (!destroyed) { sdstats->astats.mapped += astats->astats.mapped; - sdstats->astats.pa_shard_stats.pac_stats.retained - += astats->astats.pa_shard_stats.pac_stats.retained; - sdstats->astats.pa_shard_stats.edata_avail - += astats->astats.pa_shard_stats.edata_avail; + sdstats->astats.pa_shard_stats.pac_stats.retained += + astats->astats.pa_shard_stats.pac_stats.retained; + sdstats->astats.pa_shard_stats.edata_avail += + astats->astats.pa_shard_stats.edata_avail; } - ctl_accum_locked_u64( - &sdstats->astats.pa_shard_stats.pac_stats.decay_dirty.npurge, - &astats->astats.pa_shard_stats.pac_stats.decay_dirty.npurge); - ctl_accum_locked_u64( - &sdstats->astats.pa_shard_stats.pac_stats.decay_dirty.nmadvise, - &astats->astats.pa_shard_stats.pac_stats.decay_dirty.nmadvise); - ctl_accum_locked_u64( - &sdstats->astats.pa_shard_stats.pac_stats.decay_dirty.purged, - &astats->astats.pa_shard_stats.pac_stats.decay_dirty.purged); + ctl_accum_locked_u64(&sdstats->astats.pa_shard_stats.pac_stats + .decay_dirty.npurge, + &astats->astats.pa_shard_stats.pac_stats.decay_dirty + .npurge); + ctl_accum_locked_u64(&sdstats->astats.pa_shard_stats.pac_stats + .decay_dirty.nmadvise, + &astats->astats.pa_shard_stats.pac_stats.decay_dirty + .nmadvise); + ctl_accum_locked_u64(&sdstats->astats.pa_shard_stats.pac_stats + .decay_dirty.purged, + &astats->astats.pa_shard_stats.pac_stats.decay_dirty + .purged); - ctl_accum_locked_u64( - &sdstats->astats.pa_shard_stats.pac_stats.decay_muzzy.npurge, - &astats->astats.pa_shard_stats.pac_stats.decay_muzzy.npurge); - ctl_accum_locked_u64( - &sdstats->astats.pa_shard_stats.pac_stats.decay_muzzy.nmadvise, - &astats->astats.pa_shard_stats.pac_stats.decay_muzzy.nmadvise); - ctl_accum_locked_u64( - &sdstats->astats.pa_shard_stats.pac_stats.decay_muzzy.purged, - &astats->astats.pa_shard_stats.pac_stats.decay_muzzy.purged); + ctl_accum_locked_u64(&sdstats->astats.pa_shard_stats.pac_stats + .decay_muzzy.npurge, + &astats->astats.pa_shard_stats.pac_stats.decay_muzzy + .npurge); + ctl_accum_locked_u64(&sdstats->astats.pa_shard_stats.pac_stats + .decay_muzzy.nmadvise, + &astats->astats.pa_shard_stats.pac_stats.decay_muzzy + .nmadvise); + ctl_accum_locked_u64(&sdstats->astats.pa_shard_stats.pac_stats + .decay_muzzy.purged, + &astats->astats.pa_shard_stats.pac_stats.decay_muzzy + .purged); -#define OP(mtx) malloc_mutex_prof_merge( \ - &(sdstats->astats.mutex_prof_data[ \ - arena_prof_mutex_##mtx]), \ - &(astats->astats.mutex_prof_data[ \ - arena_prof_mutex_##mtx])); -MUTEX_PROF_ARENA_MUTEXES +#define OP(mtx) \ + malloc_mutex_prof_merge( \ + &(sdstats->astats.mutex_prof_data[arena_prof_mutex_##mtx]), \ + &(astats->astats.mutex_prof_data[arena_prof_mutex_##mtx])); + MUTEX_PROF_ARENA_MUTEXES #undef OP if (!destroyed) { sdstats->astats.base += astats->astats.base; + sdstats->astats.metadata_edata += + astats->astats.metadata_edata; + sdstats->astats.metadata_rtree += + astats->astats.metadata_rtree; sdstats->astats.resident += astats->astats.resident; - sdstats->astats.metadata_thp += astats->astats.metadata_thp; + sdstats->astats.metadata_thp += + astats->astats.metadata_thp; ctl_accum_atomic_zu(&sdstats->astats.internal, &astats->astats.internal); } else { assert(atomic_load_zu( - &astats->astats.internal, ATOMIC_RELAXED) == 0); + &astats->astats.internal, ATOMIC_RELAXED) + == 0); } if (!destroyed) { @@ -1169,8 +1183,8 @@ MUTEX_PROF_ARENA_MUTEXES } sdstats->astats.nmalloc_large += astats->astats.nmalloc_large; sdstats->astats.ndalloc_large += astats->astats.ndalloc_large; - sdstats->astats.nrequests_large - += astats->astats.nrequests_large; + sdstats->astats.nrequests_large += + astats->astats.nrequests_large; sdstats->astats.nflushes_large += astats->astats.nflushes_large; ctl_accum_atomic_zu( &sdstats->astats.pa_shard_stats.pac_stats.abandoned_vm, @@ -1231,19 +1245,18 @@ MUTEX_PROF_ARENA_MUTEXES for (i = 0; i < SC_NPSIZES; i++) { sdstats->estats[i].ndirty += astats->estats[i].ndirty; sdstats->estats[i].nmuzzy += astats->estats[i].nmuzzy; - sdstats->estats[i].nretained - += astats->estats[i].nretained; - sdstats->estats[i].dirty_bytes - += astats->estats[i].dirty_bytes; - sdstats->estats[i].muzzy_bytes - += astats->estats[i].muzzy_bytes; - sdstats->estats[i].retained_bytes - += astats->estats[i].retained_bytes; + sdstats->estats[i].nretained += + astats->estats[i].nretained; + sdstats->estats[i].dirty_bytes += + astats->estats[i].dirty_bytes; + sdstats->estats[i].muzzy_bytes += + astats->estats[i].muzzy_bytes; + sdstats->estats[i].retained_bytes += + astats->estats[i].retained_bytes; } /* Merge HPA stats. */ hpa_shard_stats_accum(&sdstats->hpastats, &astats->hpastats); - sec_stats_accum(&sdstats->secstats, &astats->secstats); } } @@ -1260,11 +1273,11 @@ ctl_arena_refresh(tsdn_t *tsdn, arena_t *arena, ctl_arena_t *ctl_sdarena, static unsigned ctl_arena_init(tsd_t *tsd, const arena_config_t *config) { - unsigned arena_ind; + unsigned arena_ind; ctl_arena_t *ctl_arena; - if ((ctl_arena = ql_last(&ctl_arenas->destroyed, destroyed_link)) != - NULL) { + if ((ctl_arena = ql_last(&ctl_arenas->destroyed, destroyed_link)) + != NULL) { ql_remove(&ctl_arenas->destroyed, ctl_arena, destroyed_link); arena_ind = ctl_arena->arena_ind; } else { @@ -1291,8 +1304,8 @@ ctl_arena_init(tsd_t *tsd, const arena_config_t *config) { static void ctl_background_thread_stats_read(tsdn_t *tsdn) { background_thread_stats_t *stats = &ctl_stats->background_thread; - if (!have_background_thread || - background_thread_stats_read(tsdn, stats)) { + if (!have_background_thread + || background_thread_stats_read(tsdn, stats)) { memset(stats, 0, sizeof(background_thread_stats_t)); nstime_init_zero(&stats->run_interval); } @@ -1303,9 +1316,18 @@ ctl_background_thread_stats_read(tsdn_t *tsdn) { static void ctl_refresh(tsdn_t *tsdn) { - unsigned i; + malloc_mutex_assert_owner(tsdn, &ctl_mtx); + /* + * We are guaranteed that `ctl_arenas->narenas` will not change + * underneath us since we hold `ctl_mtx` for the duration of this + * function. Unfortunately static analysis tools do not understand this, + * so we are extracting `narenas` into a local variable solely for the + * sake of exposing this information to such tools. + */ + const unsigned narenas = ctl_arenas->narenas; + assert(narenas > 0); ctl_arena_t *ctl_sarena = arenas_i(MALLCTL_ARENAS_ALL); - VARIABLE_ARRAY(arena_t *, tarenas, ctl_arenas->narenas); + VARIABLE_ARRAY_UNSAFE(arena_t *, tarenas, narenas); /* * Clear sum stats, since they will be merged into by @@ -1313,41 +1335,45 @@ ctl_refresh(tsdn_t *tsdn) { */ ctl_arena_clear(ctl_sarena); - for (i = 0; i < ctl_arenas->narenas; i++) { + for (unsigned i = 0; i < narenas; i++) { tarenas[i] = arena_get(tsdn, i, false); } - for (i = 0; i < ctl_arenas->narenas; i++) { + for (unsigned i = 0; i < narenas; i++) { ctl_arena_t *ctl_arena = arenas_i(i); - bool initialized = (tarenas[i] != NULL); + bool initialized = (tarenas[i] != NULL); ctl_arena->initialized = initialized; if (initialized) { - ctl_arena_refresh(tsdn, tarenas[i], ctl_sarena, i, - false); + ctl_arena_refresh( + tsdn, tarenas[i], ctl_sarena, i, false); } } if (config_stats) { - ctl_stats->allocated = ctl_sarena->astats->allocated_small + - ctl_sarena->astats->astats.allocated_large; + ctl_stats->allocated = ctl_sarena->astats->allocated_small + + ctl_sarena->astats->astats.allocated_large; ctl_stats->active = (ctl_sarena->pactive << LG_PAGE); - ctl_stats->metadata = ctl_sarena->astats->astats.base + - atomic_load_zu(&ctl_sarena->astats->astats.internal, - ATOMIC_RELAXED); + ctl_stats->metadata = ctl_sarena->astats->astats.base + + atomic_load_zu( + &ctl_sarena->astats->astats.internal, ATOMIC_RELAXED); + ctl_stats->metadata_edata = + ctl_sarena->astats->astats.metadata_edata; + ctl_stats->metadata_rtree = + ctl_sarena->astats->astats.metadata_rtree; ctl_stats->resident = ctl_sarena->astats->astats.resident; ctl_stats->metadata_thp = ctl_sarena->astats->astats.metadata_thp; ctl_stats->mapped = ctl_sarena->astats->astats.mapped; - ctl_stats->retained = ctl_sarena->astats->astats - .pa_shard_stats.pac_stats.retained; + ctl_stats->retained = ctl_sarena->astats->astats.pa_shard_stats + .pac_stats.retained; ctl_background_thread_stats_read(tsdn); -#define READ_GLOBAL_MUTEX_PROF_DATA(i, mtx) \ - malloc_mutex_lock(tsdn, &mtx); \ - malloc_mutex_prof_read(tsdn, &ctl_stats->mutex_prof_data[i], &mtx); \ - malloc_mutex_unlock(tsdn, &mtx); +#define READ_GLOBAL_MUTEX_PROF_DATA(i, mtx) \ + malloc_mutex_lock(tsdn, &mtx); \ + malloc_mutex_prof_read(tsdn, &ctl_stats->mutex_prof_data[i], &mtx); \ + malloc_mutex_unlock(tsdn, &mtx); if (config_prof && opt_prof) { READ_GLOBAL_MUTEX_PROF_DATA( @@ -1370,9 +1396,9 @@ ctl_refresh(tsdn_t *tsdn) { global_prof_mutex_background_thread, background_thread_lock); } else { - memset(&ctl_stats->mutex_prof_data[ - global_prof_mutex_background_thread], 0, - sizeof(mutex_prof_data_t)); + memset(&ctl_stats->mutex_prof_data + [global_prof_mutex_background_thread], + 0, sizeof(mutex_prof_data_t)); } /* We own ctl mutex already. */ malloc_mutex_prof_read(tsdn, @@ -1385,21 +1411,21 @@ ctl_refresh(tsdn_t *tsdn) { static bool ctl_init(tsd_t *tsd) { - bool ret; + bool ret; tsdn_t *tsdn = tsd_tsdn(tsd); malloc_mutex_lock(tsdn, &ctl_mtx); if (!ctl_initialized) { ctl_arena_t *ctl_sarena, *ctl_darena; - unsigned i; + unsigned i; /* * Allocate demand-zeroed space for pointers to the full * range of supported arena indices. */ if (ctl_arenas == NULL) { - ctl_arenas = (ctl_arenas_t *)base_alloc(tsdn, - b0get(), sizeof(ctl_arenas_t), QUANTUM); + ctl_arenas = (ctl_arenas_t *)base_alloc( + tsdn, b0get(), sizeof(ctl_arenas_t), QUANTUM); if (ctl_arenas == NULL) { ret = true; goto label_return; @@ -1407,8 +1433,8 @@ ctl_init(tsd_t *tsd) { } if (config_stats && ctl_stats == NULL) { - ctl_stats = (ctl_stats_t *)base_alloc(tsdn, b0get(), - sizeof(ctl_stats_t), QUANTUM); + ctl_stats = (ctl_stats_t *)base_alloc( + tsdn, b0get(), sizeof(ctl_stats_t), QUANTUM); if (ctl_stats == NULL) { ret = true; goto label_return; @@ -1420,15 +1446,17 @@ ctl_init(tsd_t *tsd) { * here rather than doing it lazily elsewhere, in order * to limit when OOM-caused errors can occur. */ - if ((ctl_sarena = arenas_i_impl(tsd, MALLCTL_ARENAS_ALL, false, - true)) == NULL) { + if ((ctl_sarena = arenas_i_impl( + tsd, MALLCTL_ARENAS_ALL, false, true)) + == NULL) { ret = true; goto label_return; } ctl_sarena->initialized = true; - if ((ctl_darena = arenas_i_impl(tsd, MALLCTL_ARENAS_DESTROYED, - false, true)) == NULL) { + if ((ctl_darena = arenas_i_impl( + tsd, MALLCTL_ARENAS_DESTROYED, false, true)) + == NULL) { ret = true; goto label_return; } @@ -1463,9 +1491,9 @@ static int ctl_lookup(tsdn_t *tsdn, const ctl_named_node_t *starting_node, const char *name, const ctl_named_node_t **ending_nodep, size_t *mibp, size_t *depthp) { - int ret; - const char *elm, *tdot, *dot; - size_t elen, i, j; + int ret; + const char *elm, *tdot, *dot; + size_t elen, i, j; const ctl_named_node_t *node; elm = name; @@ -1487,8 +1515,8 @@ ctl_lookup(tsdn_t *tsdn, const ctl_named_node_t *starting_node, for (j = 0; j < node->nchildren; j++) { const ctl_named_node_t *child = ctl_named_children(node, j); - if (strlen(child->name) == elen && - strncmp(elm, child->name, elen) == 0) { + if (strlen(child->name) == elen + && strncmp(elm, child->name, elen) == 0) { node = child; mibp[i] = j; break; @@ -1499,7 +1527,7 @@ ctl_lookup(tsdn_t *tsdn, const ctl_named_node_t *starting_node, goto label_return; } } else { - uintmax_t index; + uintmax_t index; const ctl_indexed_node_t *inode; /* Children are indexed. */ @@ -1537,8 +1565,8 @@ ctl_lookup(tsdn_t *tsdn, const ctl_named_node_t *starting_node, /* Update elm. */ elm = &dot[1]; - dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot : - strchr(elm, '\0'); + dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot + : strchr(elm, '\0'); elen = (size_t)((uintptr_t)dot - (uintptr_t)elm); } if (ending_nodep != NULL) { @@ -1553,9 +1581,9 @@ label_return: int ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; - size_t depth; - size_t mib[CTL_MAX_DEPTH]; + int ret; + size_t depth; + size_t mib[CTL_MAX_DEPTH]; const ctl_named_node_t *node; if (!ctl_initialized && ctl_init(tsd)) { @@ -1564,8 +1592,8 @@ ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp, } depth = CTL_MAX_DEPTH; - ret = ctl_lookup(tsd_tsdn(tsd), super_root_node, name, &node, mib, - &depth); + ret = ctl_lookup( + tsd_tsdn(tsd), super_root_node, name, &node, mib, &depth); if (ret != 0) { goto label_return; } @@ -1578,7 +1606,7 @@ ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp, } label_return: - return(ret); + return (ret); } int @@ -1590,10 +1618,10 @@ ctl_nametomib(tsd_t *tsd, const char *name, size_t *mibp, size_t *miblenp) { goto label_return; } - ret = ctl_lookup(tsd_tsdn(tsd), super_root_node, name, NULL, mibp, - miblenp); + ret = ctl_lookup( + tsd_tsdn(tsd), super_root_node, name, NULL, mibp, miblenp); label_return: - return(ret); + return (ret); } static int @@ -1629,13 +1657,13 @@ ctl_lookupbymib(tsdn_t *tsdn, const ctl_named_node_t **ending_nodep, ret = 0; label_return: - return(ret); + return (ret); } int ctl_bymib(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; + int ret; const ctl_named_node_t *node; if (!ctl_initialized && ctl_init(tsd)) { @@ -1657,13 +1685,13 @@ ctl_bymib(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, } label_return: - return(ret); + return (ret); } int -ctl_mibnametomib(tsd_t *tsd, size_t *mib, size_t miblen, const char *name, - size_t *miblenp) { - int ret; +ctl_mibnametomib( + tsd_t *tsd, size_t *mib, size_t miblen, const char *name, size_t *miblenp) { + int ret; const ctl_named_node_t *node; if (!ctl_initialized && ctl_init(tsd)) { @@ -1683,17 +1711,17 @@ ctl_mibnametomib(tsd_t *tsd, size_t *mib, size_t miblen, const char *name, assert(miblenp != NULL); assert(*miblenp >= miblen); *miblenp -= miblen; - ret = ctl_lookup(tsd_tsdn(tsd), node, name, NULL, mib + miblen, - miblenp); + ret = ctl_lookup( + tsd_tsdn(tsd), node, name, NULL, mib + miblen, miblenp); *miblenp += miblen; label_return: - return(ret); + return (ret); } int ctl_bymibname(tsd_t *tsd, size_t *mib, size_t miblen, const char *name, size_t *miblenp, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; + int ret; const ctl_named_node_t *node; if (!ctl_initialized && ctl_init(tsd)) { @@ -1716,29 +1744,29 @@ ctl_bymibname(tsd_t *tsd, size_t *mib, size_t miblen, const char *name, /* * The same node supplies the starting node and stores the ending node. */ - ret = ctl_lookup(tsd_tsdn(tsd), node, name, &node, mib + miblen, - miblenp); + ret = ctl_lookup( + tsd_tsdn(tsd), node, name, &node, mib + miblen, miblenp); *miblenp += miblen; if (ret != 0) { goto label_return; } if (node != NULL && node->ctl) { - ret = node->ctl(tsd, mib, *miblenp, oldp, oldlenp, newp, - newlen); + ret = node->ctl( + tsd, mib, *miblenp, oldp, oldlenp, newp, newlen); } else { /* The name refers to a partial path through the ctl tree. */ ret = ENOENT; } label_return: - return(ret); + return (ret); } bool ctl_boot(void) { if (malloc_mutex_init(&ctl_mtx, "ctl", WITNESS_RANK_CTL, - malloc_mutex_rank_exclusive)) { + malloc_mutex_rank_exclusive)) { return true; } @@ -1770,218 +1798,201 @@ ctl_mtx_assert_held(tsdn_t *tsdn) { /******************************************************************************/ /* *_ctl() functions. */ -#define READONLY() do { \ - if (newp != NULL || newlen != 0) { \ - ret = EPERM; \ - goto label_return; \ - } \ -} while (0) +#define READONLY() \ + do { \ + if (newp != NULL || newlen != 0) { \ + ret = EPERM; \ + goto label_return; \ + } \ + } while (0) -#define WRITEONLY() do { \ - if (oldp != NULL || oldlenp != NULL) { \ - ret = EPERM; \ - goto label_return; \ - } \ -} while (0) +#define WRITEONLY() \ + do { \ + if (oldp != NULL || oldlenp != NULL) { \ + ret = EPERM; \ + goto label_return; \ + } \ + } while (0) /* Can read or write, but not both. */ -#define READ_XOR_WRITE() do { \ - if ((oldp != NULL && oldlenp != NULL) && (newp != NULL || \ - newlen != 0)) { \ - ret = EPERM; \ - goto label_return; \ - } \ -} while (0) +#define READ_XOR_WRITE() \ + do { \ + if ((oldp != NULL && oldlenp != NULL) \ + && (newp != NULL || newlen != 0)) { \ + ret = EPERM; \ + goto label_return; \ + } \ + } while (0) /* Can neither read nor write. */ -#define NEITHER_READ_NOR_WRITE() do { \ - if (oldp != NULL || oldlenp != NULL || newp != NULL || \ - newlen != 0) { \ - ret = EPERM; \ - goto label_return; \ - } \ -} while (0) +#define NEITHER_READ_NOR_WRITE() \ + do { \ + if (oldp != NULL || oldlenp != NULL || newp != NULL \ + || newlen != 0) { \ + ret = EPERM; \ + goto label_return; \ + } \ + } while (0) /* Verify that the space provided is enough. */ -#define VERIFY_READ(t) do { \ - if (oldp == NULL || oldlenp == NULL || *oldlenp != sizeof(t)) { \ - *oldlenp = 0; \ - ret = EINVAL; \ - goto label_return; \ - } \ -} while (0) +#define VERIFY_READ(t) \ + do { \ + if (oldp == NULL || oldlenp == NULL \ + || *oldlenp != sizeof(t)) { \ + if (oldlenp != NULL) { \ + *oldlenp = 0; \ + } \ + ret = EINVAL; \ + goto label_return; \ + } \ + } while (0) -#define READ(v, t) do { \ - if (oldp != NULL && oldlenp != NULL) { \ - if (*oldlenp != sizeof(t)) { \ - size_t copylen = (sizeof(t) <= *oldlenp) \ - ? sizeof(t) : *oldlenp; \ - memcpy(oldp, (void *)&(v), copylen); \ - *oldlenp = copylen; \ - ret = EINVAL; \ - goto label_return; \ - } \ - *(t *)oldp = (v); \ - } \ -} while (0) +#define READ(v, t) \ + do { \ + if (oldp != NULL && oldlenp != NULL) { \ + if (*oldlenp != sizeof(t)) { \ + size_t copylen = (sizeof(t) <= *oldlenp) \ + ? sizeof(t) \ + : *oldlenp; \ + memcpy(oldp, (void *)&(v), copylen); \ + *oldlenp = copylen; \ + ret = EINVAL; \ + goto label_return; \ + } \ + *(t *)oldp = (v); \ + } \ + } while (0) -#define WRITE(v, t) do { \ - if (newp != NULL) { \ - if (newlen != sizeof(t)) { \ - ret = EINVAL; \ - goto label_return; \ - } \ - (v) = *(t *)newp; \ - } \ -} while (0) +#define WRITE(v, t) \ + do { \ + if (newp != NULL) { \ + if (newlen != sizeof(t)) { \ + ret = EINVAL; \ + goto label_return; \ + } \ + (v) = *(t *)newp; \ + } \ + } while (0) -#define ASSURED_WRITE(v, t) do { \ - if (newp == NULL || newlen != sizeof(t)) { \ - ret = EINVAL; \ - goto label_return; \ - } \ - (v) = *(t *)newp; \ -} while (0) +#define ASSURED_WRITE(v, t) \ + do { \ + if (newp == NULL || newlen != sizeof(t)) { \ + ret = EINVAL; \ + goto label_return; \ + } \ + (v) = *(t *)newp; \ + } while (0) -#define MIB_UNSIGNED(v, i) do { \ - if (mib[i] > UINT_MAX) { \ - ret = EFAULT; \ - goto label_return; \ - } \ - v = (unsigned)mib[i]; \ -} while (0) +#define MIB_UNSIGNED(v, i) \ + do { \ + if (mib[i] > UINT_MAX) { \ + ret = EFAULT; \ + goto label_return; \ + } \ + v = (unsigned)mib[i]; \ + } while (0) /* * There's a lot of code duplication in the following macros due to limitations * in how nested cpp macros are expanded. */ -#define CTL_RO_CLGEN(c, l, n, v, t) \ -static int \ -n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ - size_t *oldlenp, void *newp, size_t newlen) { \ - int ret; \ - t oldval; \ - \ - if (!(c)) { \ - return ENOENT; \ - } \ - if (l) { \ - malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \ - } \ - READONLY(); \ - oldval = (v); \ - READ(oldval, t); \ - \ - ret = 0; \ -label_return: \ - if (l) { \ - malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \ - } \ - return ret; \ -} +#define CTL_RO_CGEN(c, n, v, t) \ + static int n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ + int ret; \ + t oldval; \ + \ + if (!(c)) { \ + return ENOENT; \ + } \ + malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \ + READONLY(); \ + oldval = (v); \ + READ(oldval, t); \ + \ + ret = 0; \ + label_return: \ + malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \ + return ret; \ + } -#define CTL_RO_CGEN(c, n, v, t) \ -static int \ -n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ - int ret; \ - t oldval; \ - \ - if (!(c)) { \ - return ENOENT; \ - } \ - malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \ - READONLY(); \ - oldval = (v); \ - READ(oldval, t); \ - \ - ret = 0; \ -label_return: \ - malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \ - return ret; \ -} - -#define CTL_RO_GEN(n, v, t) \ -static int \ -n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ - size_t *oldlenp, void *newp, size_t newlen) { \ - int ret; \ - t oldval; \ - \ - malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \ - READONLY(); \ - oldval = (v); \ - READ(oldval, t); \ - \ - ret = 0; \ -label_return: \ - malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \ - return ret; \ -} +#define CTL_RO_GEN(n, v, t) \ + static int n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ + int ret; \ + t oldval; \ + \ + malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \ + READONLY(); \ + oldval = (v); \ + READ(oldval, t); \ + \ + ret = 0; \ + label_return: \ + malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \ + return ret; \ + } /* * ctl_mtx is not acquired, under the assumption that no pertinent data will * mutate during the call. */ -#define CTL_RO_NL_CGEN(c, n, v, t) \ -static int \ -n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ - int ret; \ - t oldval; \ - \ - if (!(c)) { \ - return ENOENT; \ - } \ - READONLY(); \ - oldval = (v); \ - READ(oldval, t); \ - \ - ret = 0; \ -label_return: \ - return ret; \ -} +#define CTL_RO_NL_CGEN(c, n, v, t) \ + static int n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ + int ret; \ + t oldval; \ + \ + if (!(c)) { \ + return ENOENT; \ + } \ + READONLY(); \ + oldval = (v); \ + READ(oldval, t); \ + \ + ret = 0; \ + label_return: \ + return ret; \ + } -#define CTL_RO_NL_GEN(n, v, t) \ -static int \ -n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ - int ret; \ - t oldval; \ - \ - READONLY(); \ - oldval = (v); \ - READ(oldval, t); \ - \ - ret = 0; \ -label_return: \ - return ret; \ -} +#define CTL_RO_NL_GEN(n, v, t) \ + static int n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ + int ret; \ + t oldval; \ + \ + READONLY(); \ + oldval = (v); \ + READ(oldval, t); \ + \ + ret = 0; \ + label_return: \ + return ret; \ + } -#define CTL_RO_CONFIG_GEN(n, t) \ -static int \ -n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ - int ret; \ - t oldval; \ - \ - READONLY(); \ - oldval = n; \ - READ(oldval, t); \ - \ - ret = 0; \ -label_return: \ - return ret; \ -} +#define CTL_RO_CONFIG_GEN(n, t) \ + static int n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ + int ret; \ + t oldval; \ + \ + READONLY(); \ + oldval = n; \ + READ(oldval, t); \ + \ + ret = 0; \ + label_return: \ + return ret; \ + } /******************************************************************************/ CTL_RO_NL_GEN(version, JEMALLOC_VERSION, const char *) static int -epoch_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; +epoch_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + int ret; UNUSED uint64_t newval; malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); @@ -1998,10 +2009,9 @@ label_return: } static int -background_thread_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen) { - int ret; +background_thread_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + int ret; bool oldval; if (!have_background_thread) { @@ -2050,10 +2060,9 @@ label_return: } static int -max_background_threads_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) { - int ret; +max_background_threads_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + int ret; size_t oldval; if (!have_background_thread) { @@ -2079,7 +2088,7 @@ max_background_threads_ctl(tsd_t *tsd, const size_t *mib, ret = 0; goto label_return; } - if (newval > opt_max_background_threads) { + if (newval > opt_max_background_threads || newval == 0) { ret = EINVAL; goto label_return; } @@ -2119,6 +2128,7 @@ CTL_RO_CONFIG_GEN(config_opt_safety_checks, bool) CTL_RO_CONFIG_GEN(config_prof, bool) CTL_RO_CONFIG_GEN(config_prof_libgcc, bool) CTL_RO_CONFIG_GEN(config_prof_libunwind, bool) +CTL_RO_CONFIG_GEN(config_prof_frameptr, bool) CTL_RO_CONFIG_GEN(config_stats, bool) CTL_RO_CONFIG_GEN(config_utrace, bool) CTL_RO_CONFIG_GEN(config_xmalloc, bool) @@ -2128,17 +2138,30 @@ CTL_RO_CONFIG_GEN(config_xmalloc, bool) CTL_RO_NL_GEN(opt_abort, opt_abort, bool) CTL_RO_NL_GEN(opt_abort_conf, opt_abort_conf, bool) CTL_RO_NL_GEN(opt_cache_oblivious, opt_cache_oblivious, bool) +CTL_RO_NL_GEN( + opt_debug_double_free_max_scan, opt_debug_double_free_max_scan, unsigned) CTL_RO_NL_GEN(opt_trust_madvise, opt_trust_madvise, bool) +CTL_RO_NL_GEN(opt_experimental_hpa_start_huge_if_thp_always, + opt_experimental_hpa_start_huge_if_thp_always, bool) +CTL_RO_NL_GEN(opt_experimental_hpa_enforce_hugify, + opt_experimental_hpa_enforce_hugify, bool) CTL_RO_NL_GEN(opt_confirm_conf, opt_confirm_conf, bool) /* HPA options. */ CTL_RO_NL_GEN(opt_hpa, opt_hpa, bool) -CTL_RO_NL_GEN(opt_hpa_hugification_threshold, - opt_hpa_opts.hugification_threshold, size_t) +CTL_RO_NL_GEN( + opt_hpa_hugification_threshold, opt_hpa_opts.hugification_threshold, size_t) CTL_RO_NL_GEN(opt_hpa_hugify_delay_ms, opt_hpa_opts.hugify_delay_ms, uint64_t) -CTL_RO_NL_GEN(opt_hpa_min_purge_interval_ms, opt_hpa_opts.min_purge_interval_ms, - uint64_t) - +CTL_RO_NL_GEN(opt_hpa_hugify_sync, opt_hpa_opts.hugify_sync, bool) +CTL_RO_NL_GEN( + opt_hpa_min_purge_interval_ms, opt_hpa_opts.min_purge_interval_ms, uint64_t) +CTL_RO_NL_GEN(opt_experimental_hpa_max_purge_nhp, + opt_hpa_opts.experimental_max_purge_nhp, ssize_t) +CTL_RO_NL_GEN(opt_hpa_purge_threshold, opt_hpa_opts.purge_threshold, size_t) +CTL_RO_NL_GEN( + opt_hpa_min_purge_delay_ms, opt_hpa_opts.min_purge_delay_ms, uint64_t) +CTL_RO_NL_GEN(opt_hpa_hugify_style, + hpa_hugify_style_names[opt_hpa_opts.hugify_style], const char *) /* * This will have to change before we publicly document this option; fxp_t and * its representation are internal implementation details. @@ -2150,18 +2173,16 @@ CTL_RO_NL_GEN(opt_hpa_slab_max_alloc, opt_hpa_opts.slab_max_alloc, size_t) CTL_RO_NL_GEN(opt_hpa_sec_nshards, opt_hpa_sec_opts.nshards, size_t) CTL_RO_NL_GEN(opt_hpa_sec_max_alloc, opt_hpa_sec_opts.max_alloc, size_t) CTL_RO_NL_GEN(opt_hpa_sec_max_bytes, opt_hpa_sec_opts.max_bytes, size_t) -CTL_RO_NL_GEN(opt_hpa_sec_bytes_after_flush, opt_hpa_sec_opts.bytes_after_flush, - size_t) -CTL_RO_NL_GEN(opt_hpa_sec_batch_fill_extra, opt_hpa_sec_opts.batch_fill_extra, - size_t) - -CTL_RO_NL_GEN(opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp], - const char *) +CTL_RO_NL_GEN( + opt_hpa_sec_batch_fill_extra, opt_hpa_sec_opts.batch_fill_extra, size_t) +CTL_RO_NL_GEN(opt_huge_arena_pac_thp, opt_huge_arena_pac_thp, bool) +CTL_RO_NL_GEN( + opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp], const char *) CTL_RO_NL_GEN(opt_retain, opt_retain, bool) CTL_RO_NL_GEN(opt_dss, opt_dss, const char *) CTL_RO_NL_GEN(opt_narenas, opt_narenas, unsigned) -CTL_RO_NL_GEN(opt_percpu_arena, percpu_arena_mode_names[opt_percpu_arena], - const char *) +CTL_RO_NL_GEN( + opt_percpu_arena, percpu_arena_mode_names[opt_percpu_arena], const char *) CTL_RO_NL_GEN(opt_mutex_max_spin, opt_mutex_max_spin, int64_t) CTL_RO_NL_GEN(opt_oversize_threshold, opt_oversize_threshold, size_t) CTL_RO_NL_GEN(opt_background_thread, opt_background_thread, bool) @@ -2178,53 +2199,72 @@ CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool) CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool) CTL_RO_NL_CGEN(config_enable_cxx, opt_experimental_infallible_new, opt_experimental_infallible_new, bool) +CTL_RO_NL_GEN(opt_experimental_tcache_gc, opt_experimental_tcache_gc, bool) CTL_RO_NL_GEN(opt_tcache, opt_tcache, bool) CTL_RO_NL_GEN(opt_tcache_max, opt_tcache_max, size_t) -CTL_RO_NL_GEN(opt_tcache_nslots_small_min, opt_tcache_nslots_small_min, - unsigned) -CTL_RO_NL_GEN(opt_tcache_nslots_small_max, opt_tcache_nslots_small_max, - unsigned) +CTL_RO_NL_GEN( + opt_tcache_nslots_small_min, opt_tcache_nslots_small_min, unsigned) +CTL_RO_NL_GEN( + opt_tcache_nslots_small_max, opt_tcache_nslots_small_max, unsigned) CTL_RO_NL_GEN(opt_tcache_nslots_large, opt_tcache_nslots_large, unsigned) CTL_RO_NL_GEN(opt_lg_tcache_nslots_mul, opt_lg_tcache_nslots_mul, ssize_t) CTL_RO_NL_GEN(opt_tcache_gc_incr_bytes, opt_tcache_gc_incr_bytes, size_t) CTL_RO_NL_GEN(opt_tcache_gc_delay_bytes, opt_tcache_gc_delay_bytes, size_t) -CTL_RO_NL_GEN(opt_lg_tcache_flush_small_div, opt_lg_tcache_flush_small_div, - unsigned) -CTL_RO_NL_GEN(opt_lg_tcache_flush_large_div, opt_lg_tcache_flush_large_div, - unsigned) +CTL_RO_NL_GEN( + opt_lg_tcache_flush_small_div, opt_lg_tcache_flush_small_div, unsigned) +CTL_RO_NL_GEN( + opt_lg_tcache_flush_large_div, opt_lg_tcache_flush_large_div, unsigned) CTL_RO_NL_GEN(opt_thp, thp_mode_names[opt_thp], const char *) -CTL_RO_NL_GEN(opt_lg_extent_max_active_fit, opt_lg_extent_max_active_fit, - size_t) +CTL_RO_NL_GEN( + opt_lg_extent_max_active_fit, opt_lg_extent_max_active_fit, size_t) +CTL_RO_NL_GEN( + opt_process_madvise_max_batch, opt_process_madvise_max_batch, size_t) CTL_RO_NL_CGEN(config_prof, opt_prof, opt_prof, bool) CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *) CTL_RO_NL_CGEN(config_prof, opt_prof_active, opt_prof_active, bool) -CTL_RO_NL_CGEN(config_prof, opt_prof_thread_active_init, - opt_prof_thread_active_init, bool) +CTL_RO_NL_CGEN( + config_prof, opt_prof_thread_active_init, opt_prof_thread_active_init, bool) +CTL_RO_NL_CGEN(config_prof, opt_prof_bt_max, opt_prof_bt_max, unsigned) CTL_RO_NL_CGEN(config_prof, opt_lg_prof_sample, opt_lg_prof_sample, size_t) CTL_RO_NL_CGEN(config_prof, opt_prof_accum, opt_prof_accum, bool) +CTL_RO_NL_CGEN( + config_prof, opt_prof_pid_namespace, opt_prof_pid_namespace, bool) CTL_RO_NL_CGEN(config_prof, opt_lg_prof_interval, opt_lg_prof_interval, ssize_t) CTL_RO_NL_CGEN(config_prof, opt_prof_gdump, opt_prof_gdump, bool) CTL_RO_NL_CGEN(config_prof, opt_prof_final, opt_prof_final, bool) CTL_RO_NL_CGEN(config_prof, opt_prof_leak, opt_prof_leak, bool) CTL_RO_NL_CGEN(config_prof, opt_prof_leak_error, opt_prof_leak_error, bool) -CTL_RO_NL_CGEN(config_prof, opt_prof_recent_alloc_max, - opt_prof_recent_alloc_max, ssize_t) +CTL_RO_NL_CGEN( + config_prof, opt_prof_recent_alloc_max, opt_prof_recent_alloc_max, ssize_t) CTL_RO_NL_CGEN(config_prof, opt_prof_stats, opt_prof_stats, bool) -CTL_RO_NL_CGEN(config_prof, opt_prof_sys_thread_name, opt_prof_sys_thread_name, - bool) +CTL_RO_NL_CGEN( + config_prof, opt_prof_sys_thread_name, opt_prof_sys_thread_name, bool) CTL_RO_NL_CGEN(config_prof, opt_prof_time_res, prof_time_res_mode_names[opt_prof_time_res], const char *) -CTL_RO_NL_CGEN(config_uaf_detection, opt_lg_san_uaf_align, - opt_lg_san_uaf_align, ssize_t) +CTL_RO_NL_CGEN( + config_uaf_detection, opt_lg_san_uaf_align, opt_lg_san_uaf_align, ssize_t) CTL_RO_NL_GEN(opt_zero_realloc, zero_realloc_mode_names[opt_zero_realloc_action], const char *) +CTL_RO_NL_GEN( + opt_disable_large_size_classes, opt_disable_large_size_classes, bool) + +/* malloc_conf options */ +CTL_RO_NL_CGEN(opt_malloc_conf_symlink, opt_malloc_conf_symlink, + opt_malloc_conf_symlink, const char *) +CTL_RO_NL_CGEN(opt_malloc_conf_env_var, opt_malloc_conf_env_var, + opt_malloc_conf_env_var, const char *) +CTL_RO_NL_CGEN( + je_malloc_conf, opt_malloc_conf_global_var, je_malloc_conf, const char *) +CTL_RO_NL_CGEN(je_malloc_conf_2_conf_harder, + opt_malloc_conf_global_var_2_conf_harder, je_malloc_conf_2_conf_harder, + const char *) /******************************************************************************/ static int -thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; +thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + int ret; arena_t *oldarena; unsigned newind, oldind; @@ -2245,8 +2285,8 @@ thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, goto label_return; } - if (have_percpu_arena && - PERCPU_ARENA_ENABLED(opt_percpu_arena)) { + if (have_percpu_arena + && PERCPU_ARENA_ENABLED(opt_percpu_arena)) { if (newind < percpu_arena_ind_limit(opt_percpu_arena)) { /* * If perCPU arena is enabled, thread_arena @@ -2280,14 +2320,83 @@ label_return: CTL_RO_NL_GEN(thread_allocated, tsd_thread_allocated_get(tsd), uint64_t) CTL_RO_NL_GEN(thread_allocatedp, tsd_thread_allocatedp_get(tsd), uint64_t *) + +static int +thread_tcache_ncached_max_read_sizeclass_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + int ret; + size_t bin_size = 0; + + /* Read the bin size from newp. */ + if (newp == NULL) { + ret = EINVAL; + goto label_return; + } + WRITE(bin_size, size_t); + + cache_bin_sz_t ncached_max = 0; + if (tcache_bin_ncached_max_read(tsd, bin_size, &ncached_max)) { + ret = EINVAL; + goto label_return; + } + size_t result = (size_t)ncached_max; + READ(result, size_t); + ret = 0; +label_return: + return ret; +} + +static int +thread_tcache_ncached_max_write_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + int ret; + WRITEONLY(); + if (newp != NULL) { + if (!tcache_available(tsd)) { + ret = ENOENT; + goto label_return; + } + char *settings = NULL; + WRITE(settings, char *); + if (settings == NULL) { + ret = EINVAL; + goto label_return; + } + /* Get the length of the setting string safely. */ + char *end = (char *)memchr( + settings, '\0', CTL_MULTI_SETTING_MAX_LEN); + if (end == NULL) { + ret = EINVAL; + goto label_return; + } + /* + * Exclude the last '\0' for len since it is not handled by + * multi_setting_parse_next. + */ + size_t len = (uintptr_t)end - (uintptr_t)settings; + if (len == 0) { + ret = 0; + goto label_return; + } + + if (tcache_bins_ncached_max_write(tsd, settings, len)) { + ret = EINVAL; + goto label_return; + } + } + + ret = 0; +label_return: + return ret; +} + CTL_RO_NL_GEN(thread_deallocated, tsd_thread_deallocated_get(tsd), uint64_t) CTL_RO_NL_GEN(thread_deallocatedp, tsd_thread_deallocatedp_get(tsd), uint64_t *) static int -thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) { - int ret; +thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + int ret; bool oldval; oldval = tcache_enabled_get(tsd); @@ -2306,9 +2415,41 @@ label_return: } static int -thread_tcache_flush_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) { +thread_tcache_max_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + int ret; + size_t oldval; + + /* pointer to tcache_t always exists even with tcache disabled. */ + tcache_t *tcache = tsd_tcachep_get(tsd); + assert(tcache != NULL); + oldval = tcache_max_get(tcache->tcache_slow); + READ(oldval, size_t); + + if (newp != NULL) { + if (newlen != sizeof(size_t)) { + ret = EINVAL; + goto label_return; + } + size_t new_tcache_max = oldval; + WRITE(new_tcache_max, size_t); + if (new_tcache_max > TCACHE_MAXCLASS_LIMIT) { + new_tcache_max = TCACHE_MAXCLASS_LIMIT; + } + new_tcache_max = sz_s2u(new_tcache_max); + if (new_tcache_max != oldval) { + thread_tcache_max_set(tsd, new_tcache_max); + } + } + + ret = 0; +label_return: + return ret; +} + +static int +thread_tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; if (!tcache_available(tsd)) { @@ -2326,9 +2467,8 @@ label_return: } static int -thread_peak_read_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) { +thread_peak_read_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { int ret; if (!config_stats) { return ENOENT; @@ -2343,9 +2483,8 @@ label_return: } static int -thread_peak_reset_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) { +thread_peak_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { int ret; if (!config_stats) { return ENOENT; @@ -2358,9 +2497,8 @@ label_return: } static int -thread_prof_name_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) { +thread_prof_name_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { int ret; if (!config_prof || !opt_prof) { @@ -2370,13 +2508,13 @@ thread_prof_name_ctl(tsd_t *tsd, const size_t *mib, READ_XOR_WRITE(); if (newp != NULL) { - if (newlen != sizeof(const char *)) { + const char *newval = *(const char **)newp; + if (newlen != sizeof(const char *) || newval == NULL) { ret = EINVAL; goto label_return; } - if ((ret = prof_thread_name_set(tsd, *(const char **)newp)) != - 0) { + if ((ret = prof_thread_name_set(tsd, newval)) != 0) { goto label_return; } } else { @@ -2390,10 +2528,9 @@ label_return: } static int -thread_prof_active_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) { - int ret; +thread_prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + int ret; bool oldval; if (!config_prof) { @@ -2423,9 +2560,8 @@ label_return: } static int -thread_idle_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) { +thread_idle_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { int ret; NEITHER_READ_NOR_WRITE(); @@ -2458,9 +2594,9 @@ label_return: /******************************************************************************/ static int -tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; +tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + int ret; unsigned tcache_ind; READONLY(); @@ -2477,9 +2613,9 @@ label_return: } static int -tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; +tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + int ret; unsigned tcache_ind; WRITEONLY(); @@ -2492,9 +2628,9 @@ label_return: } static int -tcache_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; +tcache_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + int ret; unsigned tcache_ind; WRITEONLY(); @@ -2511,10 +2647,10 @@ label_return: static int arena_i_initialized_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; - tsdn_t *tsdn = tsd_tsdn(tsd); + int ret; + tsdn_t *tsdn = tsd_tsdn(tsd); unsigned arena_ind; - bool initialized; + bool initialized; READONLY(); MIB_UNSIGNED(arena_ind, 1); @@ -2542,7 +2678,7 @@ arena_i_decay(tsdn_t *tsdn, unsigned arena_ind, bool all) { */ if (arena_ind == MALLCTL_ARENAS_ALL || arena_ind == narenas) { unsigned i; - VARIABLE_ARRAY(arena_t *, tarenas, narenas); + VARIABLE_ARRAY_UNSAFE(arena_t *, tarenas, narenas); for (i = 0; i < narenas; i++) { tarenas[i] = arena_get(tsdn, i, false); @@ -2556,8 +2692,8 @@ arena_i_decay(tsdn_t *tsdn, unsigned arena_ind, bool all) { for (i = 0; i < narenas; i++) { if (tarenas[i] != NULL) { - arena_decay(tsdn, tarenas[i], false, - all); + arena_decay( + tsdn, tarenas[i], false, all); } } } else { @@ -2580,7 +2716,7 @@ arena_i_decay(tsdn_t *tsdn, unsigned arena_ind, bool all) { static int arena_i_decay_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; + int ret; unsigned arena_ind; NEITHER_READ_NOR_WRITE(); @@ -2595,7 +2731,7 @@ label_return: static int arena_i_purge_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; + int ret; unsigned arena_ind; NEITHER_READ_NOR_WRITE(); @@ -2661,12 +2797,12 @@ arena_reset_finish_background_thread(tsd_t *tsd, unsigned arena_ind) { static int arena_i_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; + int ret; unsigned arena_ind; arena_t *arena; - ret = arena_i_reset_destroy_helper(tsd, mib, miblen, oldp, oldlenp, - newp, newlen, &arena_ind, &arena); + ret = arena_i_reset_destroy_helper( + tsd, mib, miblen, oldp, oldlenp, newp, newlen, &arena_ind, &arena); if (ret != 0) { return ret; } @@ -2681,21 +2817,21 @@ arena_i_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, static int arena_i_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; - unsigned arena_ind; - arena_t *arena; + int ret; + unsigned arena_ind; + arena_t *arena; ctl_arena_t *ctl_darena, *ctl_arena; malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); - ret = arena_i_reset_destroy_helper(tsd, mib, miblen, oldp, oldlenp, - newp, newlen, &arena_ind, &arena); + ret = arena_i_reset_destroy_helper( + tsd, mib, miblen, oldp, oldlenp, newp, newlen, &arena_ind, &arena); if (ret != 0) { goto label_return; } - if (arena_nthreads_get(arena, false) != 0 || arena_nthreads_get(arena, - true) != 0) { + if (arena_nthreads_get(arena, false) != 0 + || arena_nthreads_get(arena, true) != 0) { ret = EFAULT; goto label_return; } @@ -2726,17 +2862,16 @@ label_return: static int arena_i_dss_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; + int ret; const char *dss = NULL; - unsigned arena_ind; - dss_prec_t dss_prec_old = dss_prec_limit; - dss_prec_t dss_prec = dss_prec_limit; + unsigned arena_ind; + dss_prec_t dss_prec = dss_prec_limit; malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); WRITE(dss, const char *); MIB_UNSIGNED(arena_ind, 1); if (dss != NULL) { - int i; + int i; bool match = false; for (i = 0; i < dss_prec_limit; i++) { @@ -2757,18 +2892,20 @@ arena_i_dss_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, * Access via index narenas is deprecated, and scheduled for removal in * 6.0.0. */ - if (arena_ind == MALLCTL_ARENAS_ALL || arena_ind == - ctl_arenas->narenas) { - if (dss_prec != dss_prec_limit && - extent_dss_prec_set(dss_prec)) { + dss_prec_t dss_prec_old; + if (arena_ind == MALLCTL_ARENAS_ALL + || arena_ind == ctl_arenas->narenas) { + if (dss_prec != dss_prec_limit + && extent_dss_prec_set(dss_prec)) { ret = EFAULT; goto label_return; } dss_prec_old = extent_dss_prec_get(); } else { arena_t *arena = arena_get(tsd_tsdn(tsd), arena_ind, false); - if (arena == NULL || (dss_prec != dss_prec_limit && - arena_dss_prec_set(arena, dss_prec))) { + if (arena == NULL + || (dss_prec != dss_prec_limit + && arena_dss_prec_set(arena, dss_prec))) { ret = EFAULT; goto label_return; } @@ -2819,7 +2956,7 @@ label_return: static int arena_i_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) { - int ret; + int ret; unsigned arena_ind; arena_t *arena; @@ -2840,20 +2977,9 @@ arena_i_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, ret = EINVAL; goto label_return; } - if (arena_is_huge(arena_ind) && *(ssize_t *)newp > 0) { - /* - * By default the huge arena purges eagerly. If it is - * set to non-zero decay time afterwards, background - * thread might be needed. - */ - if (background_thread_create(tsd, arena_ind)) { - ret = EFAULT; - goto label_return; - } - } - if (arena_decay_ms_set(tsd_tsdn(tsd), arena, state, - *(ssize_t *)newp)) { + if (arena_decay_ms_set( + tsd_tsdn(tsd), arena, state, *(ssize_t *)newp)) { ret = EFAULT; goto label_return; } @@ -2867,21 +2993,21 @@ label_return: static int arena_i_dirty_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - return arena_i_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp, - newlen, true); + return arena_i_decay_ms_ctl_impl( + tsd, mib, miblen, oldp, oldlenp, newp, newlen, true); } static int arena_i_muzzy_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - return arena_i_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp, - newlen, false); + return arena_i_decay_ms_ctl_impl( + tsd, mib, miblen, oldp, oldlenp, newp, newlen, false); } static int arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; + int ret; unsigned arena_ind; arena_t *arena; @@ -2906,8 +3032,8 @@ arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, arena_config_t config = arena_config_default; config.extent_hooks = new_extent_hooks; - arena = arena_init(tsd_tsdn(tsd), arena_ind, - &config); + arena = arena_init( + tsd_tsdn(tsd), arena_ind, &config); if (arena == NULL) { ret = EFAULT; goto label_return; @@ -2918,13 +3044,12 @@ arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, extent_hooks_t *new_extent_hooks JEMALLOC_CC_SILENCE_INIT(NULL); WRITE(new_extent_hooks, extent_hooks_t *); - old_extent_hooks = arena_set_extent_hooks(tsd, - arena, new_extent_hooks); + old_extent_hooks = arena_set_extent_hooks( + tsd, arena, new_extent_hooks); READ(old_extent_hooks, extent_hooks_t *); } else { - old_extent_hooks = - ehooks_get_extent_hooks_ptr( - arena_get_ehooks(arena)); + old_extent_hooks = ehooks_get_extent_hooks_ptr( + arena_get_ehooks(arena)); READ(old_extent_hooks, extent_hooks_t *); } } @@ -2939,10 +3064,9 @@ label_return: } static int -arena_i_retain_grow_limit_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) { - int ret; +arena_i_retain_grow_limit_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + int ret; unsigned arena_ind; arena_t *arena; @@ -2953,14 +3077,14 @@ arena_i_retain_grow_limit_ctl(tsd_t *tsd, const size_t *mib, malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); MIB_UNSIGNED(arena_ind, 1); - if (arena_ind < narenas_total_get() && (arena = - arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) { + if (arena_ind < narenas_total_get() + && (arena = arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) { size_t old_limit, new_limit; if (newp != NULL) { WRITE(new_limit, size_t); } - bool err = arena_retain_grow_limit_get_set(tsd, arena, - &old_limit, newp != NULL ? &new_limit : NULL); + bool err = arena_retain_grow_limit_get_set( + tsd, arena, &old_limit, newp != NULL ? &new_limit : NULL); if (!err) { READ(old_limit, size_t); ret = 0; @@ -2975,9 +3099,63 @@ label_return: return ret; } +/* + * When writing, newp should point to a char array storing the name to be set. + * A name longer than ARENA_NAME_LEN will be arbitrarily cut. When reading, + * oldp should point to a char array whose length is no shorter than + * ARENA_NAME_LEN or the length of the name when it was set. + */ +static int +arena_i_name_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + int ret; + unsigned arena_ind; + char *name JEMALLOC_CLANG_ANALYZER_SILENCE_INIT(NULL); + + malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); + MIB_UNSIGNED(arena_ind, 1); + if (arena_ind == MALLCTL_ARENAS_ALL + || arena_ind >= ctl_arenas->narenas) { + ret = EINVAL; + goto label_return; + } + arena_t *arena = arena_get(tsd_tsdn(tsd), arena_ind, false); + if (arena == NULL) { + ret = EFAULT; + goto label_return; + } + + if (oldp != NULL && oldlenp != NULL) { + /* + * Read the arena name. When reading, the input oldp should + * point to an array with a length no shorter than + * ARENA_NAME_LEN or the length when it was set. + */ + if (*oldlenp != sizeof(char *)) { + ret = EINVAL; + goto label_return; + } + name = *(char **)oldp; + arena_name_get(arena, name); + } + + if (newp != NULL) { + /* Write the arena name. */ + WRITE(name, char *); + if (name == NULL) { + ret = EINVAL; + goto label_return; + } + arena_name_set(arena, name); + } + ret = 0; +label_return: + malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); + return ret; +} + static const ctl_named_node_t * -arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, - size_t i) { +arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { const ctl_named_node_t *ret; malloc_mutex_lock(tsdn, &ctl_mtx); @@ -3002,9 +3180,9 @@ label_return: /******************************************************************************/ static int -arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; +arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + int ret; unsigned narenas; malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); @@ -3019,14 +3197,13 @@ label_return: } static int -arenas_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, - size_t newlen, bool dirty) { +arenas_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) { int ret; if (oldp != NULL && oldlenp != NULL) { - size_t oldval = (dirty ? arena_dirty_decay_ms_default_get() : - arena_muzzy_decay_ms_default_get()); + size_t oldval = (dirty ? arena_dirty_decay_ms_default_get() + : arena_muzzy_decay_ms_default_get()); READ(oldval, ssize_t); } if (newp != NULL) { @@ -3034,8 +3211,9 @@ arenas_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, ret = EINVAL; goto label_return; } - if (dirty ? arena_dirty_decay_ms_default_set(*(ssize_t *)newp) - : arena_muzzy_decay_ms_default_set(*(ssize_t *)newp)) { + if (dirty + ? arena_dirty_decay_ms_default_set(*(ssize_t *)newp) + : arena_muzzy_decay_ms_default_set(*(ssize_t *)newp)) { ret = EFAULT; goto label_return; } @@ -3049,51 +3227,51 @@ label_return: static int arenas_dirty_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - return arenas_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp, - newlen, true); + return arenas_decay_ms_ctl_impl( + tsd, mib, miblen, oldp, oldlenp, newp, newlen, true); } static int arenas_muzzy_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - return arenas_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp, - newlen, false); + return arenas_decay_ms_ctl_impl( + tsd, mib, miblen, oldp, oldlenp, newp, newlen, false); } CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t) CTL_RO_NL_GEN(arenas_page, PAGE, size_t) -CTL_RO_NL_GEN(arenas_tcache_max, tcache_maxclass, size_t) +CTL_RO_NL_GEN(arenas_hugepage, HUGEPAGE, size_t) +CTL_RO_NL_GEN(arenas_tcache_max, global_do_not_change_tcache_maxclass, size_t) CTL_RO_NL_GEN(arenas_nbins, SC_NBINS, unsigned) -CTL_RO_NL_GEN(arenas_nhbins, nhbins, unsigned) +CTL_RO_NL_GEN(arenas_nhbins, global_do_not_change_tcache_nbins, unsigned) CTL_RO_NL_GEN(arenas_bin_i_size, bin_infos[mib[2]].reg_size, size_t) CTL_RO_NL_GEN(arenas_bin_i_nregs, bin_infos[mib[2]].nregs, uint32_t) CTL_RO_NL_GEN(arenas_bin_i_slab_size, bin_infos[mib[2]].slab_size, size_t) CTL_RO_NL_GEN(arenas_bin_i_nshards, bin_infos[mib[2]].n_shards, uint32_t) static const ctl_named_node_t * -arenas_bin_i_index(tsdn_t *tsdn, const size_t *mib, - size_t miblen, size_t i) { - if (i > SC_NBINS) { +arenas_bin_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { + if (i >= SC_NBINS) { return NULL; } return super_arenas_bin_i_node; } CTL_RO_NL_GEN(arenas_nlextents, SC_NSIZES - SC_NBINS, unsigned) -CTL_RO_NL_GEN(arenas_lextent_i_size, sz_index2size(SC_NBINS+(szind_t)mib[2]), - size_t) +CTL_RO_NL_GEN(arenas_lextent_i_size, + sz_index2size_unsafe(SC_NBINS + (szind_t)mib[2]), size_t) static const ctl_named_node_t * -arenas_lextent_i_index(tsdn_t *tsdn, const size_t *mib, - size_t miblen, size_t i) { - if (i > SC_NSIZES - SC_NBINS) { +arenas_lextent_i_index( + tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { + if (i >= SC_NSIZES - SC_NBINS) { return NULL; } return super_arenas_lextent_i_node; } static int -arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; +arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + int ret; unsigned arena_ind; malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); @@ -3114,10 +3292,9 @@ label_return: } static int -experimental_arenas_create_ext_ctl(tsd_t *tsd, - const size_t *mib, size_t miblen, +experimental_arenas_create_ext_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; + int ret; unsigned arena_ind; malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); @@ -3138,25 +3315,26 @@ label_return: } static int -arenas_lookup_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) { - int ret; - unsigned arena_ind; - void *ptr; - edata_t *edata; - arena_t *arena; +arenas_lookup_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + int ret; + unsigned arena_ind; + void *ptr; + emap_full_alloc_ctx_t alloc_ctx; + bool ptr_not_present; + arena_t *arena; ptr = NULL; ret = EINVAL; malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); WRITE(ptr, void *); - edata = emap_edata_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr); - if (edata == NULL) { + ptr_not_present = emap_full_alloc_ctx_try_lookup( + tsd_tsdn(tsd), &arena_emap_global, ptr, &alloc_ctx); + if (ptr_not_present || alloc_ctx.edata == NULL) { goto label_return; } - arena = arena_get_from_edata(edata); + arena = arena_get_from_edata(alloc_ctx.edata); if (arena == NULL) { goto label_return; } @@ -3173,10 +3351,9 @@ label_return: /******************************************************************************/ static int -prof_thread_active_init_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) { - int ret; +prof_thread_active_init_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + int ret; bool oldval; if (!config_prof) { @@ -3192,11 +3369,11 @@ prof_thread_active_init_ctl(tsd_t *tsd, const size_t *mib, ret = EINVAL; goto label_return; } - oldval = prof_thread_active_init_set(tsd_tsdn(tsd), - *(bool *)newp); + oldval = prof_thread_active_init_set( + tsd_tsdn(tsd), *(bool *)newp); } else { - oldval = opt_prof ? prof_thread_active_init_get(tsd_tsdn(tsd)) : - false; + oldval = opt_prof ? prof_thread_active_init_get(tsd_tsdn(tsd)) + : false; } READ(oldval, bool); @@ -3206,9 +3383,9 @@ label_return: } static int -prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; +prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + int ret; bool oldval; if (!config_prof) { @@ -3244,9 +3421,9 @@ label_return: } static int -prof_dump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; +prof_dump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + int ret; const char *filename = NULL; if (!config_prof || !opt_prof) { @@ -3267,9 +3444,9 @@ label_return: } static int -prof_gdump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; +prof_gdump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + int ret; bool oldval; if (!config_prof) { @@ -3297,9 +3474,9 @@ label_return: } static int -prof_prefix_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; +prof_prefix_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + int ret; const char *prefix = NULL; if (!config_prof || !opt_prof) { @@ -3317,9 +3494,9 @@ label_return: } static int -prof_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; +prof_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + int ret; size_t lg_sample = lg_prof_sample; if (!config_prof || !opt_prof) { @@ -3390,8 +3567,7 @@ experimental_hooks_prof_backtrace_ctl(tsd_t *tsd, const size_t *mib, goto label_return; } if (oldp != NULL) { - prof_backtrace_hook_t old_hook = - prof_backtrace_hook_get(); + prof_backtrace_hook_t old_hook = prof_backtrace_hook_get(); READ(old_hook, prof_backtrace_hook_t); } if (newp != NULL) { @@ -3413,8 +3589,8 @@ label_return: } static int -experimental_hooks_prof_dump_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +experimental_hooks_prof_dump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; if (oldp == NULL && newp == NULL) { @@ -3422,8 +3598,7 @@ experimental_hooks_prof_dump_ctl(tsd_t *tsd, const size_t *mib, goto label_return; } if (oldp != NULL) { - prof_dump_hook_t old_hook = - prof_dump_hook_get(); + prof_dump_hook_t old_hook = prof_dump_hook_get(); READ(old_hook, prof_dump_hook_t); } if (newp != NULL) { @@ -3440,6 +3615,78 @@ label_return: return ret; } +static int +experimental_hooks_prof_sample_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + int ret; + + if (oldp == NULL && newp == NULL) { + ret = EINVAL; + goto label_return; + } + if (oldp != NULL) { + prof_sample_hook_t old_hook = prof_sample_hook_get(); + READ(old_hook, prof_sample_hook_t); + } + if (newp != NULL) { + if (!opt_prof) { + ret = ENOENT; + goto label_return; + } + prof_sample_hook_t new_hook JEMALLOC_CC_SILENCE_INIT(NULL); + WRITE(new_hook, prof_sample_hook_t); + prof_sample_hook_set(new_hook); + } + ret = 0; +label_return: + return ret; +} + +static int +experimental_hooks_prof_sample_free_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + int ret; + + if (oldp == NULL && newp == NULL) { + ret = EINVAL; + goto label_return; + } + if (oldp != NULL) { + prof_sample_free_hook_t old_hook = prof_sample_free_hook_get(); + READ(old_hook, prof_sample_free_hook_t); + } + if (newp != NULL) { + if (!opt_prof) { + ret = ENOENT; + goto label_return; + } + prof_sample_free_hook_t new_hook JEMALLOC_CC_SILENCE_INIT(NULL); + WRITE(new_hook, prof_sample_free_hook_t); + prof_sample_free_hook_set(new_hook); + } + ret = 0; +label_return: + return ret; +} + +static int +experimental_hooks_thread_event_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + int ret; + + if (newp == NULL) { + ret = EINVAL; + goto label_return; + } + + user_hook_object_t t_new = {NULL, 0, false}; + WRITE(t_new, user_hook_object_t); + ret = te_register_user_handler(tsd_tsdn(tsd), &t_new); + +label_return: + return ret; +} + /* For integration test purpose only. No plan to move out of experimental. */ static int experimental_hooks_safety_check_abort_ctl(tsd_t *tsd, const size_t *mib, @@ -3466,6 +3713,10 @@ label_return: CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats->allocated, size_t) CTL_RO_CGEN(config_stats, stats_active, ctl_stats->active, size_t) CTL_RO_CGEN(config_stats, stats_metadata, ctl_stats->metadata, size_t) +CTL_RO_CGEN( + config_stats, stats_metadata_edata, ctl_stats->metadata_edata, size_t) +CTL_RO_CGEN( + config_stats, stats_metadata_rtree, ctl_stats->metadata_rtree, size_t) CTL_RO_CGEN(config_stats, stats_metadata_thp, ctl_stats->metadata_thp, size_t) CTL_RO_CGEN(config_stats, stats_resident, ctl_stats->resident, size_t) CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats->mapped, size_t) @@ -3481,11 +3732,50 @@ CTL_RO_CGEN(config_stats, stats_background_thread_run_interval, CTL_RO_CGEN(config_stats, stats_zero_reallocs, atomic_load_zu(&zero_realloc_count, ATOMIC_RELAXED), size_t) +/* + * approximate_stats.active returns a result that is informative itself, + * but the returned value SHOULD NOT be compared against other stats retrieved. + * For instance, approximate_stats.active should not be compared against + * any stats, e.g., stats.active or stats.resident, because there is no + * guarantee in the comparison results. Results returned by stats.*, on the + * other hand, provides such guarantees, i.e., stats.active <= stats.resident, + * as long as epoch is called right before the queries. + */ + +static int +approximate_stats_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + int ret; + size_t approximate_nactive = 0; + size_t approximate_active_bytes = 0; + + READONLY(); + + tsdn_t *tsdn = tsd_tsdn(tsd); + unsigned n = narenas_total_get(); + + for (unsigned i = 0; i < n; i++) { + arena_t *arena = arena_get(tsdn, i, false); + if (!arena) { + continue; + } + /* Accumulate nactive pages from each arena's pa_shard */ + approximate_nactive += pa_shard_nactive(&arena->pa_shard); + } + + approximate_active_bytes = approximate_nactive << LG_PAGE; + READ(approximate_active_bytes, size_t); + + ret = 0; +label_return: + return ret; +} + CTL_RO_GEN(stats_arenas_i_dss, arenas_i(mib[2])->dss, const char *) -CTL_RO_GEN(stats_arenas_i_dirty_decay_ms, arenas_i(mib[2])->dirty_decay_ms, - ssize_t) -CTL_RO_GEN(stats_arenas_i_muzzy_decay_ms, arenas_i(mib[2])->muzzy_decay_ms, - ssize_t) +CTL_RO_GEN( + stats_arenas_i_dirty_decay_ms, arenas_i(mib[2])->dirty_decay_ms, ssize_t) +CTL_RO_GEN( + stats_arenas_i_muzzy_decay_ms, arenas_i(mib[2])->muzzy_decay_ms, ssize_t) CTL_RO_GEN(stats_arenas_i_nthreads, arenas_i(mib[2])->nthreads, unsigned) CTL_RO_GEN(stats_arenas_i_uptime, nstime_ns(&arenas_i(mib[2])->astats->astats.uptime), uint64_t) @@ -3500,37 +3790,40 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_extent_avail, arenas_i(mib[2])->astats->astats.pa_shard_stats.edata_avail, size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_npurge, - locked_read_u64_unsynchronized( - &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_dirty.npurge), + locked_read_u64_unsynchronized(&arenas_i(mib[2]) + ->astats->astats.pa_shard_stats.pac_stats.decay_dirty.npurge), uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_nmadvise, - locked_read_u64_unsynchronized( - &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_dirty.nmadvise), + locked_read_u64_unsynchronized(&arenas_i(mib[2]) + ->astats->astats.pa_shard_stats.pac_stats.decay_dirty.nmadvise), uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_purged, - locked_read_u64_unsynchronized( - &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_dirty.purged), + locked_read_u64_unsynchronized(&arenas_i(mib[2]) + ->astats->astats.pa_shard_stats.pac_stats.decay_dirty.purged), uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_npurge, - locked_read_u64_unsynchronized( - &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_muzzy.npurge), + locked_read_u64_unsynchronized(&arenas_i(mib[2]) + ->astats->astats.pa_shard_stats.pac_stats.decay_muzzy.npurge), uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_nmadvise, - locked_read_u64_unsynchronized( - &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_muzzy.nmadvise), + locked_read_u64_unsynchronized(&arenas_i(mib[2]) + ->astats->astats.pa_shard_stats.pac_stats.decay_muzzy.nmadvise), uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_purged, - locked_read_u64_unsynchronized( - &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.decay_muzzy.purged), + locked_read_u64_unsynchronized(&arenas_i(mib[2]) + ->astats->astats.pa_shard_stats.pac_stats.decay_muzzy.purged), uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_base, - arenas_i(mib[2])->astats->astats.base, - size_t) + arenas_i(mib[2])->astats->astats.base, size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_internal, atomic_load_zu(&arenas_i(mib[2])->astats->astats.internal, ATOMIC_RELAXED), size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_metadata_edata, + arenas_i(mib[2])->astats->astats.metadata_edata, size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_metadata_rtree, + arenas_i(mib[2])->astats->astats.metadata_rtree, size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_metadata_thp, arenas_i(mib[2])->astats->astats.metadata_thp, size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_tcache_bytes, @@ -3538,15 +3831,25 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_tcache_bytes, CTL_RO_CGEN(config_stats, stats_arenas_i_tcache_stashed_bytes, arenas_i(mib[2])->astats->astats.tcache_stashed_bytes, size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_resident, - arenas_i(mib[2])->astats->astats.resident, - size_t) + arenas_i(mib[2])->astats->astats.resident, size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_abandoned_vm, atomic_load_zu( - &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.abandoned_vm, - ATOMIC_RELAXED), size_t) + &arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.abandoned_vm, + ATOMIC_RELAXED), + size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_bytes, - arenas_i(mib[2])->astats->secstats.bytes, size_t) + arenas_i(mib[2])->astats->hpastats.secstats.bytes, size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_hits, + arenas_i(mib[2])->astats->hpastats.secstats.total.nhits, size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_misses, + arenas_i(mib[2])->astats->hpastats.secstats.total.nmisses, size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_dalloc_flush, + arenas_i(mib[2])->astats->hpastats.secstats.total.ndalloc_flush, size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_dalloc_noflush, + arenas_i(mib[2])->astats->hpastats.secstats.total.ndalloc_noflush, size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_overfills, + arenas_i(mib[2])->astats->hpastats.secstats.total.noverfills, size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated, arenas_i(mib[2])->astats->allocated_small, size_t) @@ -3578,55 +3881,55 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_large_nflushes, arenas_i(mib[2])->astats->astats.nflushes_large, uint64_t) /* Lock profiling related APIs below. */ -#define RO_MUTEX_CTL_GEN(n, l) \ -CTL_RO_CGEN(config_stats, stats_##n##_num_ops, \ - l.n_lock_ops, uint64_t) \ -CTL_RO_CGEN(config_stats, stats_##n##_num_wait, \ - l.n_wait_times, uint64_t) \ -CTL_RO_CGEN(config_stats, stats_##n##_num_spin_acq, \ - l.n_spin_acquired, uint64_t) \ -CTL_RO_CGEN(config_stats, stats_##n##_num_owner_switch, \ - l.n_owner_switches, uint64_t) \ -CTL_RO_CGEN(config_stats, stats_##n##_total_wait_time, \ - nstime_ns(&l.tot_wait_time), uint64_t) \ -CTL_RO_CGEN(config_stats, stats_##n##_max_wait_time, \ - nstime_ns(&l.max_wait_time), uint64_t) \ -CTL_RO_CGEN(config_stats, stats_##n##_max_num_thds, \ - l.max_n_thds, uint32_t) +#define RO_MUTEX_CTL_GEN(n, l) \ + CTL_RO_CGEN(config_stats, stats_##n##_num_ops, l.n_lock_ops, uint64_t) \ + CTL_RO_CGEN( \ + config_stats, stats_##n##_num_wait, l.n_wait_times, uint64_t) \ + CTL_RO_CGEN(config_stats, stats_##n##_num_spin_acq, l.n_spin_acquired, \ + uint64_t) \ + CTL_RO_CGEN(config_stats, stats_##n##_num_owner_switch, \ + l.n_owner_switches, uint64_t) \ + CTL_RO_CGEN(config_stats, stats_##n##_total_wait_time, \ + nstime_ns(&l.tot_wait_time), uint64_t) \ + CTL_RO_CGEN(config_stats, stats_##n##_max_wait_time, \ + nstime_ns(&l.max_wait_time), uint64_t) \ + CTL_RO_CGEN( \ + config_stats, stats_##n##_max_num_thds, l.max_n_thds, uint32_t) /* Global mutexes. */ -#define OP(mtx) \ - RO_MUTEX_CTL_GEN(mutexes_##mtx, \ - ctl_stats->mutex_prof_data[global_prof_mutex_##mtx]) +#define OP(mtx) \ + RO_MUTEX_CTL_GEN(mutexes_##mtx, \ + ctl_stats->mutex_prof_data[global_prof_mutex_##mtx]) MUTEX_PROF_GLOBAL_MUTEXES #undef OP /* Per arena mutexes */ -#define OP(mtx) RO_MUTEX_CTL_GEN(arenas_i_mutexes_##mtx, \ - arenas_i(mib[2])->astats->astats.mutex_prof_data[arena_prof_mutex_##mtx]) +#define OP(mtx) \ + RO_MUTEX_CTL_GEN(arenas_i_mutexes_##mtx, \ + arenas_i(mib[2]) \ + ->astats->astats.mutex_prof_data[arena_prof_mutex_##mtx]) MUTEX_PROF_ARENA_MUTEXES #undef OP /* tcache bin mutex */ -RO_MUTEX_CTL_GEN(arenas_i_bins_j_mutex, - arenas_i(mib[2])->astats->bstats[mib[4]].mutex_data) +RO_MUTEX_CTL_GEN( + arenas_i_bins_j_mutex, arenas_i(mib[2])->astats->bstats[mib[4]].mutex_data) #undef RO_MUTEX_CTL_GEN /* Resets all mutex stats, including global, arena and bin mutexes. */ static int -stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen) { +stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { if (!config_stats) { return ENOENT; } tsdn_t *tsdn = tsd_tsdn(tsd); -#define MUTEX_PROF_RESET(mtx) \ - malloc_mutex_lock(tsdn, &mtx); \ - malloc_mutex_prof_data_reset(tsdn, &mtx); \ - malloc_mutex_unlock(tsdn, &mtx); +#define MUTEX_PROF_RESET(mtx) \ + malloc_mutex_lock(tsdn, &mtx); \ + malloc_mutex_prof_data_reset(tsdn, &mtx); \ + malloc_mutex_unlock(tsdn, &mtx); /* Global mutexes: ctl and prof. */ MUTEX_PROF_RESET(ctl_mtx); @@ -3693,9 +3996,9 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nonfull_slabs, arenas_i(mib[2])->astats->bstats[mib[4]].stats_data.nonfull_slabs, size_t) static const ctl_named_node_t * -stats_arenas_i_bins_j_index(tsdn_t *tsdn, const size_t *mib, - size_t miblen, size_t j) { - if (j > SC_NBINS) { +stats_arenas_i_bins_j_index( + tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t j) { + if (j >= SC_NBINS) { return NULL; } return super_stats_arenas_i_bins_j_node; @@ -3703,117 +4006,173 @@ stats_arenas_i_bins_j_index(tsdn_t *tsdn, const size_t *mib, CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nmalloc, locked_read_u64_unsynchronized( - &arenas_i(mib[2])->astats->lstats[mib[4]].nmalloc), uint64_t) + &arenas_i(mib[2])->astats->lstats[mib[4]].nmalloc), + uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_ndalloc, locked_read_u64_unsynchronized( - &arenas_i(mib[2])->astats->lstats[mib[4]].ndalloc), uint64_t) + &arenas_i(mib[2])->astats->lstats[mib[4]].ndalloc), + uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nrequests, locked_read_u64_unsynchronized( - &arenas_i(mib[2])->astats->lstats[mib[4]].nrequests), uint64_t) + &arenas_i(mib[2])->astats->lstats[mib[4]].nrequests), + uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_curlextents, arenas_i(mib[2])->astats->lstats[mib[4]].curlextents, size_t) static const ctl_named_node_t * -stats_arenas_i_lextents_j_index(tsdn_t *tsdn, const size_t *mib, - size_t miblen, size_t j) { - if (j > SC_NSIZES - SC_NBINS) { +stats_arenas_i_lextents_j_index( + tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t j) { + if (j >= SC_NSIZES - SC_NBINS) { return NULL; } return super_stats_arenas_i_lextents_j_node; } CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_ndirty, - arenas_i(mib[2])->astats->estats[mib[4]].ndirty, size_t); + arenas_i(mib[2])->astats->estats[mib[4]].ndirty, size_t); CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nmuzzy, - arenas_i(mib[2])->astats->estats[mib[4]].nmuzzy, size_t); + arenas_i(mib[2])->astats->estats[mib[4]].nmuzzy, size_t); CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nretained, - arenas_i(mib[2])->astats->estats[mib[4]].nretained, size_t); + arenas_i(mib[2])->astats->estats[mib[4]].nretained, size_t); CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_dirty_bytes, - arenas_i(mib[2])->astats->estats[mib[4]].dirty_bytes, size_t); + arenas_i(mib[2])->astats->estats[mib[4]].dirty_bytes, size_t); CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_muzzy_bytes, - arenas_i(mib[2])->astats->estats[mib[4]].muzzy_bytes, size_t); + arenas_i(mib[2])->astats->estats[mib[4]].muzzy_bytes, size_t); CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_retained_bytes, - arenas_i(mib[2])->astats->estats[mib[4]].retained_bytes, size_t); + arenas_i(mib[2])->astats->estats[mib[4]].retained_bytes, size_t); static const ctl_named_node_t * -stats_arenas_i_extents_j_index(tsdn_t *tsdn, const size_t *mib, - size_t miblen, size_t j) { +stats_arenas_i_extents_j_index( + tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t j) { if (j >= SC_NPSIZES) { return NULL; } return super_stats_arenas_i_extents_j_node; } +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_npageslabs, + arenas_i(mib[2])->astats->hpastats.psset_stats.merged.npageslabs, size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nactive, + arenas_i(mib[2])->astats->hpastats.psset_stats.merged.nactive, size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_ndirty, + arenas_i(mib[2])->astats->hpastats.psset_stats.merged.ndirty, size_t); + +/* Nonhuge slabs */ +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_slabs_npageslabs_nonhuge, + arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[0].npageslabs, size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_slabs_nactive_nonhuge, + arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[0].nactive, size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_slabs_ndirty_nonhuge, + arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[0].ndirty, size_t); + +/* Huge slabs */ +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_slabs_npageslabs_huge, + arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[1].npageslabs, size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_slabs_nactive_huge, + arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[1].nactive, size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_slabs_ndirty_huge, + arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[1].ndirty, size_t); + CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_npurge_passes, - arenas_i(mib[2])->astats->hpastats.nonderived_stats.npurge_passes, uint64_t); + arenas_i(mib[2])->astats->hpastats.nonderived_stats.npurge_passes, + uint64_t); CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_npurges, arenas_i(mib[2])->astats->hpastats.nonderived_stats.npurges, uint64_t); CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nhugifies, arenas_i(mib[2])->astats->hpastats.nonderived_stats.nhugifies, uint64_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nhugify_failures, + arenas_i(mib[2])->astats->hpastats.nonderived_stats.nhugify_failures, + uint64_t); CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_ndehugifies, arenas_i(mib[2])->astats->hpastats.nonderived_stats.ndehugifies, uint64_t); /* Full, nonhuge */ -CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_npageslabs_nonhuge, +CTL_RO_CGEN(config_stats, + stats_arenas_i_hpa_shard_full_slabs_npageslabs_nonhuge, arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].npageslabs, size_t); CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive_nonhuge, - arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].nactive, size_t); + arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].nactive, + size_t); CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ndirty_nonhuge, - arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].ndirty, size_t); + arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].ndirty, + size_t); /* Full, huge */ CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_npageslabs_huge, arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].npageslabs, size_t); CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive_huge, - arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].nactive, size_t); + arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].nactive, + size_t); CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ndirty_huge, - arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].ndirty, size_t); + arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].ndirty, + size_t); /* Empty, nonhuge */ -CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_npageslabs_nonhuge, +CTL_RO_CGEN(config_stats, + stats_arenas_i_hpa_shard_empty_slabs_npageslabs_nonhuge, arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[0].npageslabs, size_t); CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_nactive_nonhuge, - arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[0].nactive, size_t); + arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[0].nactive, + size_t); CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_ndirty_nonhuge, - arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[0].ndirty, size_t); + arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[0].ndirty, + size_t); /* Empty, huge */ CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_npageslabs_huge, arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[1].npageslabs, size_t); CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_nactive_huge, - arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[1].nactive, size_t); + arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[1].nactive, + size_t); CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_ndirty_huge, - arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[1].ndirty, size_t); + arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[1].ndirty, + size_t); /* Nonfull, nonhuge */ -CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_nonhuge, - arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0].npageslabs, +CTL_RO_CGEN(config_stats, + stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_nonhuge, + arenas_i(mib[2]) + ->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0] + .npageslabs, size_t); -CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_nonhuge, - arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0].nactive, +CTL_RO_CGEN(config_stats, + stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_nonhuge, + arenas_i(mib[2]) + ->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0] + .nactive, size_t); -CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_nonhuge, - arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0].ndirty, +CTL_RO_CGEN(config_stats, + stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_nonhuge, + arenas_i(mib[2]) + ->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0] + .ndirty, size_t); /* Nonfull, huge */ -CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_huge, - arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1].npageslabs, +CTL_RO_CGEN(config_stats, + stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_huge, + arenas_i(mib[2]) + ->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1] + .npageslabs, size_t); CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_huge, - arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1].nactive, + arenas_i(mib[2]) + ->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1] + .nactive, size_t); CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_huge, - arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1].ndirty, + arenas_i(mib[2]) + ->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1] + .ndirty, size_t); static const ctl_named_node_t * -stats_arenas_i_hpa_shard_nonfull_slabs_j_index(tsdn_t *tsdn, const size_t *mib, - size_t miblen, size_t j) { +stats_arenas_i_hpa_shard_nonfull_slabs_j_index( + tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t j) { if (j >= PSSET_NPSIZES) { return NULL; } @@ -3831,8 +4190,7 @@ ctl_arenas_i_verify(size_t i) { } static const ctl_named_node_t * -stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib, - size_t miblen, size_t i) { +stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { const ctl_named_node_t *ret; malloc_mutex_lock(tsdn, &ctl_mtx); @@ -3851,7 +4209,7 @@ static int experimental_hooks_install_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; - if (oldp == NULL || oldlenp == NULL|| newp == NULL) { + if (oldp == NULL || oldlenp == NULL || newp == NULL) { ret = EINVAL; goto label_return; } @@ -3892,32 +4250,6 @@ label_return: return ret; } -static int -experimental_thread_activity_callback_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; - - if (!config_stats) { - return ENOENT; - } - - activity_callback_thunk_t t_old = tsd_activity_callback_thunk_get(tsd); - READ(t_old, activity_callback_thunk_t); - - if (newp != NULL) { - /* - * This initialization is unnecessary. If it's omitted, though, - * clang gets confused and warns on the subsequent use of t_new. - */ - activity_callback_thunk_t t_new = {NULL, NULL}; - WRITE(t_new, activity_callback_thunk_t); - tsd_activity_callback_thunk_set(tsd, t_new); - } - ret = 0; -label_return: - return ret; -} - /* * Output six memory utilization entries for an input pointer, the first one of * type (void *) and the remaining five of type size_t, describing the following @@ -3986,8 +4318,8 @@ label_return: * motivation from C++. */ static int -experimental_utilization_query_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +experimental_utilization_query_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; assert(sizeof(inspect_extent_util_stats_verbose_t) @@ -4002,8 +4334,8 @@ experimental_utilization_query_ctl(tsd_t *tsd, const size_t *mib, void *ptr = NULL; WRITE(ptr, void *); - inspect_extent_util_stats_verbose_t *util_stats - = (inspect_extent_util_stats_verbose_t *)oldp; + inspect_extent_util_stats_verbose_t *util_stats = + (inspect_extent_util_stats_verbose_t *)oldp; inspect_extent_util_stats_verbose_get(tsd_tsdn(tsd), ptr, &util_stats->nfree, &util_stats->nregs, &util_stats->size, &util_stats->bin_nfree, &util_stats->bin_nregs, @@ -4125,7 +4457,7 @@ experimental_utilization_batch_query_ctl(tsd_t *tsd, const size_t *mib, goto label_return; } - void **ptrs = (void **)newp; + void **ptrs = (void **)newp; inspect_extent_util_stats_t *util_stats = (inspect_extent_util_stats_t *)oldp; size_t i; @@ -4141,8 +4473,8 @@ label_return: } static const ctl_named_node_t * -experimental_arenas_i_index(tsdn_t *tsdn, const size_t *mib, - size_t miblen, size_t i) { +experimental_arenas_i_index( + tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { const ctl_named_node_t *ret; malloc_mutex_lock(tsdn, &ctl_mtx); @@ -4157,8 +4489,8 @@ label_return: } static int -experimental_arenas_i_pactivep_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +experimental_arenas_i_pactivep_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { if (!config_stats) { return ENOENT; } @@ -4168,16 +4500,16 @@ experimental_arenas_i_pactivep_ctl(tsd_t *tsd, const size_t *mib, unsigned arena_ind; arena_t *arena; - int ret; - size_t *pactivep; + int ret; + size_t *pactivep; malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); READONLY(); MIB_UNSIGNED(arena_ind, 2); - if (arena_ind < narenas_total_get() && (arena = - arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) { -#if defined(JEMALLOC_GCC_ATOMIC_ATOMICS) || \ - defined(JEMALLOC_GCC_SYNC_ATOMICS) || defined(_MSC_VER) + if (arena_ind < narenas_total_get() + && (arena = arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) { +#if defined(JEMALLOC_GCC_ATOMIC_ATOMICS) || defined(JEMALLOC_GCC_SYNC_ATOMICS) \ + || defined(_MSC_VER) /* Expose the underlying counter for fast read. */ pactivep = (size_t *)&(arena->pa_shard.nactive.repr); READ(pactivep, size_t *); @@ -4229,7 +4561,7 @@ label_return: typedef struct write_cb_packet_s write_cb_packet_t; struct write_cb_packet_s { write_cb_t *write_cb; - void *cbopaque; + void *cbopaque; }; static int @@ -4248,8 +4580,8 @@ experimental_prof_recent_alloc_dump_ctl(tsd_t *tsd, const size_t *mib, write_cb_packet_t write_cb_packet; ASSURED_WRITE(write_cb_packet, write_cb_packet_t); - prof_recent_alloc_dump(tsd, write_cb_packet.write_cb, - write_cb_packet.cbopaque); + prof_recent_alloc_dump( + tsd, write_cb_packet.write_cb, write_cb_packet.cbopaque); ret = 0; @@ -4262,12 +4594,12 @@ struct batch_alloc_packet_s { void **ptrs; size_t num; size_t size; - int flags; + int flags; }; static int -experimental_batch_alloc_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +experimental_batch_alloc_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; VERIFY_READ(size_t); @@ -4288,8 +4620,8 @@ label_return: static int prof_stats_bins_i_live_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; - unsigned binind; + int ret; + unsigned binind; prof_stats_t stats; if (!(config_prof && opt_prof && opt_prof_stats)) { @@ -4314,8 +4646,8 @@ label_return: static int prof_stats_bins_i_accum_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; - unsigned binind; + int ret; + unsigned binind; prof_stats_t stats; if (!(config_prof && opt_prof && opt_prof_stats)) { @@ -4338,8 +4670,8 @@ label_return: } static const ctl_named_node_t * -prof_stats_bins_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, - size_t i) { +prof_stats_bins_i_index( + tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { if (!(config_prof && opt_prof && opt_prof_stats)) { return NULL; } @@ -4352,8 +4684,8 @@ prof_stats_bins_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, static int prof_stats_lextents_i_live_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; - unsigned lextent_ind; + int ret; + unsigned lextent_ind; prof_stats_t stats; if (!(config_prof && opt_prof && opt_prof_stats)) { @@ -4378,8 +4710,8 @@ label_return: static int prof_stats_lextents_i_accum_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; - unsigned lextent_ind; + int ret; + unsigned lextent_ind; prof_stats_t stats; if (!(config_prof && opt_prof && opt_prof_stats)) { @@ -4402,8 +4734,8 @@ label_return: } static const ctl_named_node_t * -prof_stats_lextents_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, - size_t i) { +prof_stats_lextents_i_index( + tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { if (!(config_prof && opt_prof && opt_prof_stats)) { return NULL; } diff --git a/dep/jemalloc/src/decay.c b/dep/jemalloc/src/decay.c index d801b2bc0..7bbce2a6c 100644 --- a/dep/jemalloc/src/decay.c +++ b/dep/jemalloc/src/decay.c @@ -4,9 +4,8 @@ #include "jemalloc/internal/decay.h" static const uint64_t h_steps[SMOOTHSTEP_NSTEPS] = { -#define STEP(step, h, x, y) \ - h, - SMOOTHSTEP +#define STEP(step, h, x, y) h, + SMOOTHSTEP #undef STEP }; @@ -14,15 +13,16 @@ static const uint64_t h_steps[SMOOTHSTEP_NSTEPS] = { * Generate a new deadline that is uniformly random within the next epoch after * the current one. */ -void +static void decay_deadline_init(decay_t *decay) { nstime_copy(&decay->deadline, &decay->epoch); nstime_add(&decay->deadline, &decay->interval); if (decay_ms_read(decay) > 0) { nstime_t jitter; - nstime_init(&jitter, prng_range_u64(&decay->jitter_state, - nstime_ns(&decay->interval))); + nstime_init(&jitter, + prng_range_u64( + &decay->jitter_state, nstime_ns(&decay->interval))); nstime_add(&decay->deadline, &jitter); } } @@ -31,8 +31,8 @@ void decay_reinit(decay_t *decay, nstime_t *cur_time, ssize_t decay_ms) { atomic_store_zd(&decay->time_ms, decay_ms, ATOMIC_RELAXED); if (decay_ms > 0) { - nstime_init(&decay->interval, (uint64_t)decay_ms * - KQU(1000000)); + nstime_init( + &decay->interval, (uint64_t)decay_ms * KQU(1000000)); nstime_idivide(&decay->interval, SMOOTHSTEP_NSTEPS); } @@ -52,7 +52,7 @@ decay_init(decay_t *decay, nstime_t *cur_time, ssize_t decay_ms) { decay->ceil_npages = 0; } if (malloc_mutex_init(&decay->mtx, "decay", WITNESS_RANK_DECAY, - malloc_mutex_rank_exclusive)) { + malloc_mutex_rank_exclusive)) { return true; } decay->purging = false; @@ -65,8 +65,8 @@ decay_ms_valid(ssize_t decay_ms) { if (decay_ms < -1) { return false; } - if (decay_ms == -1 || (uint64_t)decay_ms <= NSTIME_SEC_MAX * - KQU(1000)) { + if (decay_ms == -1 + || (uint64_t)decay_ms <= NSTIME_SEC_MAX * KQU(1000)) { return true; } return false; @@ -74,8 +74,8 @@ decay_ms_valid(ssize_t decay_ms) { static void decay_maybe_update_time(decay_t *decay, nstime_t *new_time) { - if (unlikely(!nstime_monotonic() && nstime_compare(&decay->epoch, - new_time) > 0)) { + if (unlikely(!nstime_monotonic() + && nstime_compare(&decay->epoch, new_time) > 0)) { /* * Time went backwards. Move the epoch back in time and * generate a new deadline, with the expectation that time @@ -115,11 +115,11 @@ decay_backlog_npages_limit(const decay_t *decay) { * placed as the newest record. */ static void -decay_backlog_update(decay_t *decay, uint64_t nadvance_u64, - size_t current_npages) { +decay_backlog_update( + decay_t *decay, uint64_t nadvance_u64, size_t current_npages) { if (nadvance_u64 >= SMOOTHSTEP_NSTEPS) { - memset(decay->backlog, 0, (SMOOTHSTEP_NSTEPS-1) * - sizeof(size_t)); + memset(decay->backlog, 0, + (SMOOTHSTEP_NSTEPS - 1) * sizeof(size_t)); } else { size_t nadvance_z = (size_t)nadvance_u64; @@ -128,14 +128,15 @@ decay_backlog_update(decay_t *decay, uint64_t nadvance_u64, memmove(decay->backlog, &decay->backlog[nadvance_z], (SMOOTHSTEP_NSTEPS - nadvance_z) * sizeof(size_t)); if (nadvance_z > 1) { - memset(&decay->backlog[SMOOTHSTEP_NSTEPS - - nadvance_z], 0, (nadvance_z-1) * sizeof(size_t)); + memset(&decay->backlog[SMOOTHSTEP_NSTEPS - nadvance_z], + 0, (nadvance_z - 1) * sizeof(size_t)); } } - size_t npages_delta = (current_npages > decay->nunpurged) ? - current_npages - decay->nunpurged : 0; - decay->backlog[SMOOTHSTEP_NSTEPS-1] = npages_delta; + size_t npages_delta = (current_npages > decay->nunpurged) + ? current_npages - decay->nunpurged + : 0; + decay->backlog[SMOOTHSTEP_NSTEPS - 1] = npages_delta; if (config_debug) { if (current_npages > decay->ceil_npages) { @@ -157,6 +158,7 @@ decay_deadline_reached(const decay_t *decay, const nstime_t *time) { uint64_t decay_npages_purge_in(decay_t *decay, nstime_t *time, size_t npages_new) { uint64_t decay_interval_ns = decay_epoch_duration_ns(decay); + assert(decay_interval_ns != 0); size_t n_epoch = (size_t)(nstime_ns(time) / decay_interval_ns); uint64_t npages_purge; @@ -164,18 +166,17 @@ decay_npages_purge_in(decay_t *decay, nstime_t *time, size_t npages_new) { npages_purge = npages_new; } else { uint64_t h_steps_max = h_steps[SMOOTHSTEP_NSTEPS - 1]; - assert(h_steps_max >= - h_steps[SMOOTHSTEP_NSTEPS - 1 - n_epoch]); - npages_purge = npages_new * (h_steps_max - - h_steps[SMOOTHSTEP_NSTEPS - 1 - n_epoch]); + assert(h_steps_max >= h_steps[SMOOTHSTEP_NSTEPS - 1 - n_epoch]); + npages_purge = npages_new + * (h_steps_max - h_steps[SMOOTHSTEP_NSTEPS - 1 - n_epoch]); npages_purge >>= SMOOTHSTEP_BFP; } return npages_purge; } bool -decay_maybe_advance_epoch(decay_t *decay, nstime_t *new_time, - size_t npages_current) { +decay_maybe_advance_epoch( + decay_t *decay, nstime_t *new_time, size_t npages_current) { /* Handle possible non-monotonicity of time. */ decay_maybe_update_time(decay, new_time); @@ -201,8 +202,9 @@ decay_maybe_advance_epoch(decay_t *decay, nstime_t *new_time, decay_backlog_update(decay, nadvance_u64, npages_current); decay->npages_limit = decay_backlog_npages_limit(decay); - decay->nunpurged = (decay->npages_limit > npages_current) ? - decay->npages_limit : npages_current; + decay->nunpurged = (decay->npages_limit > npages_current) + ? decay->npages_limit + : npages_current; return true; } @@ -225,21 +227,21 @@ decay_maybe_advance_epoch(decay_t *decay, nstime_t *new_time, */ static inline size_t decay_npurge_after_interval(decay_t *decay, size_t interval) { - size_t i; + size_t i; uint64_t sum = 0; for (i = 0; i < interval; i++) { sum += decay->backlog[i] * h_steps[i]; } for (; i < SMOOTHSTEP_NSTEPS; i++) { - sum += decay->backlog[i] * - (h_steps[i] - h_steps[i - interval]); + sum += decay->backlog[i] * (h_steps[i] - h_steps[i - interval]); } return (size_t)(sum >> SMOOTHSTEP_BFP); } -uint64_t decay_ns_until_purge(decay_t *decay, size_t npages_current, - uint64_t npages_threshold) { +uint64_t +decay_ns_until_purge( + decay_t *decay, size_t npages_current, uint64_t npages_threshold) { if (!decay_gradually(decay)) { return DECAY_UNBOUNDED_TIME_TO_PURGE; } @@ -277,7 +279,7 @@ uint64_t decay_ns_until_purge(decay_t *decay, size_t npages_current, } unsigned n_search = 0; - size_t target, npurge; + size_t target, npurge; while ((npurge_lb + npages_threshold < npurge_ub) && (lb + 2 < ub)) { target = (lb + ub) / 2; npurge = decay_npurge_after_interval(decay, target); diff --git a/dep/jemalloc/src/ecache.c b/dep/jemalloc/src/ecache.c index a242227d3..20fcee9e6 100644 --- a/dep/jemalloc/src/ecache.c +++ b/dep/jemalloc/src/ecache.c @@ -7,7 +7,7 @@ bool ecache_init(tsdn_t *tsdn, ecache_t *ecache, extent_state_t state, unsigned ind, bool delay_coalesce) { if (malloc_mutex_init(&ecache->mtx, "extents", WITNESS_RANK_EXTENTS, - malloc_mutex_rank_exclusive)) { + malloc_mutex_rank_exclusive)) { return true; } ecache->state = state; diff --git a/dep/jemalloc/src/edata.c b/dep/jemalloc/src/edata.c index 82b6f5654..d71d1679d 100644 --- a/dep/jemalloc/src/edata.c +++ b/dep/jemalloc/src/edata.c @@ -1,6 +1,5 @@ #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/jemalloc_internal_includes.h" -ph_gen(, edata_avail, edata_t, avail_link, - edata_esnead_comp) -ph_gen(, edata_heap, edata_t, heap_link, edata_snad_comp) +ph_gen(, edata_avail, edata_t, avail_link, edata_esnead_comp) + ph_gen(, edata_heap, edata_t, heap_link, edata_snad_comp) diff --git a/dep/jemalloc/src/edata_cache.c b/dep/jemalloc/src/edata_cache.c index 6bc1848cb..3ac8273a8 100644 --- a/dep/jemalloc/src/edata_cache.c +++ b/dep/jemalloc/src/edata_cache.c @@ -11,7 +11,7 @@ edata_cache_init(edata_cache_t *edata_cache, base_t *base) { */ atomic_store_zu(&edata_cache->count, 0, ATOMIC_RELAXED); if (malloc_mutex_init(&edata_cache->mtx, "edata_cache", - WITNESS_RANK_EDATA_CACHE, malloc_mutex_rank_exclusive)) { + WITNESS_RANK_EDATA_CACHE, malloc_mutex_rank_exclusive)) { return true; } edata_cache->base = base; @@ -63,8 +63,7 @@ edata_cache_fast_init(edata_cache_fast_t *ecs, edata_cache_t *fallback) { } static void -edata_cache_fast_try_fill_from_fallback(tsdn_t *tsdn, - edata_cache_fast_t *ecs) { +edata_cache_fast_try_fill_from_fallback(tsdn_t *tsdn, edata_cache_fast_t *ecs) { edata_t *edata; malloc_mutex_lock(tsdn, &ecs->fallback->mtx); for (int i = 0; i < EDATA_CACHE_FAST_FILL; i++) { @@ -80,8 +79,8 @@ edata_cache_fast_try_fill_from_fallback(tsdn_t *tsdn, edata_t * edata_cache_fast_get(tsdn_t *tsdn, edata_cache_fast_t *ecs) { - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_EDATA_CACHE, 0); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_EDATA_CACHE, 0); if (ecs->disabled) { assert(edata_list_inactive_first(&ecs->list) == NULL); @@ -118,7 +117,7 @@ edata_cache_fast_flush_all(tsdn_t *tsdn, edata_cache_fast_t *ecs) { * flush and disable pathways. */ edata_t *edata; - size_t nflushed = 0; + size_t nflushed = 0; malloc_mutex_lock(tsdn, &ecs->fallback->mtx); while ((edata = edata_list_inactive_first(&ecs->list)) != NULL) { edata_list_inactive_remove(&ecs->list, edata); @@ -131,8 +130,8 @@ edata_cache_fast_flush_all(tsdn_t *tsdn, edata_cache_fast_t *ecs) { void edata_cache_fast_put(tsdn_t *tsdn, edata_cache_fast_t *ecs, edata_t *edata) { - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_EDATA_CACHE, 0); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_EDATA_CACHE, 0); if (ecs->disabled) { assert(edata_list_inactive_first(&ecs->list) == NULL); diff --git a/dep/jemalloc/src/ehooks.c b/dep/jemalloc/src/ehooks.c index 383e9de6a..d7abb960b 100644 --- a/dep/jemalloc/src/ehooks.c +++ b/dep/jemalloc/src/ehooks.c @@ -27,9 +27,10 @@ extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, assert(alignment != 0); /* "primary" dss. */ - if (have_dss && dss_prec == dss_prec_primary && (ret = - extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero, - commit)) != NULL) { + if (have_dss && dss_prec == dss_prec_primary + && (ret = extent_alloc_dss( + tsdn, arena, new_addr, size, alignment, zero, commit)) + != NULL) { return ret; } /* mmap. */ @@ -38,9 +39,10 @@ extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, return ret; } /* "secondary" dss. */ - if (have_dss && dss_prec == dss_prec_secondary && (ret = - extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero, - commit)) != NULL) { + if (have_dss && dss_prec == dss_prec_secondary + && (ret = extent_alloc_dss( + tsdn, arena, new_addr, size, alignment, zero, commit)) + != NULL) { return ret; } @@ -53,11 +55,12 @@ ehooks_default_alloc_impl(tsdn_t *tsdn, void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit, unsigned arena_ind) { arena_t *arena = arena_get(tsdn, arena_ind, false); /* NULL arena indicates arena_create. */ - assert(arena != NULL || alignment == HUGEPAGE); - dss_prec_t dss = (arena == NULL) ? dss_prec_disabled : - (dss_prec_t)atomic_load_u(&arena->dss_prec, ATOMIC_RELAXED); - void *ret = extent_alloc_core(tsdn, arena, new_addr, size, alignment, - zero, commit, dss); + assert(arena != NULL || alignment == BASE_BLOCK_MIN_ALIGN); + dss_prec_t dss = (arena == NULL) + ? dss_prec_disabled + : (dss_prec_t)atomic_load_u(&arena->dss_prec, ATOMIC_RELAXED); + void *ret = extent_alloc_core( + tsdn, arena, new_addr, size, alignment, zero, commit, dss); if (have_madvise_huge && ret) { pages_set_thp_state(ret, size); } @@ -100,8 +103,8 @@ ehooks_default_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size, bool ehooks_default_commit_impl(void *addr, size_t offset, size_t length) { - return pages_commit((void *)((uintptr_t)addr + (uintptr_t)offset), - length); + return pages_commit( + (void *)((byte_t *)addr + (uintptr_t)offset), length); } static bool @@ -112,8 +115,8 @@ ehooks_default_commit(extent_hooks_t *extent_hooks, void *addr, size_t size, bool ehooks_default_decommit_impl(void *addr, size_t offset, size_t length) { - return pages_decommit((void *)((uintptr_t)addr + (uintptr_t)offset), - length); + return pages_decommit( + (void *)((byte_t *)addr + (uintptr_t)offset), length); } static bool @@ -125,8 +128,8 @@ ehooks_default_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size, #ifdef PAGES_CAN_PURGE_LAZY bool ehooks_default_purge_lazy_impl(void *addr, size_t offset, size_t length) { - return pages_purge_lazy((void *)((uintptr_t)addr + (uintptr_t)offset), - length); + return pages_purge_lazy( + (void *)((byte_t *)addr + (uintptr_t)offset), length); } static bool @@ -143,8 +146,8 @@ ehooks_default_purge_lazy(extent_hooks_t *extent_hooks, void *addr, size_t size, #ifdef PAGES_CAN_PURGE_FORCED bool ehooks_default_purge_forced_impl(void *addr, size_t offset, size_t length) { - return pages_purge_forced((void *)((uintptr_t)addr + - (uintptr_t)offset), length); + return pages_purge_forced( + (void *)((byte_t *)addr + (uintptr_t)offset), length); } static bool @@ -159,7 +162,7 @@ ehooks_default_purge_forced(extent_hooks_t *extent_hooks, void *addr, #endif bool -ehooks_default_split_impl() { +ehooks_default_split_impl(void) { if (!maps_coalesce) { /* * Without retain, only whole regions can be purged (required by @@ -201,11 +204,11 @@ ehooks_default_merge_impl(tsdn_t *tsdn, void *addr_a, void *addr_b) { return true; } if (config_debug) { - edata_t *a = emap_edata_lookup(tsdn, &arena_emap_global, - addr_a); - bool head_a = edata_is_head_get(a); - edata_t *b = emap_edata_lookup(tsdn, &arena_emap_global, - addr_b); + edata_t *a = emap_edata_lookup( + tsdn, &arena_emap_global, addr_a); + bool head_a = edata_is_head_get(a); + edata_t *b = emap_edata_lookup( + tsdn, &arena_emap_global, addr_b); bool head_b = edata_is_head_get(b); emap_assert_mapped(tsdn, &arena_emap_global, a); emap_assert_mapped(tsdn, &arena_emap_global, b); @@ -254,22 +257,17 @@ ehooks_default_unguard_impl(void *guard1, void *guard2) { pages_unmark_guards(guard1, guard2); } -const extent_hooks_t ehooks_default_extent_hooks = { - ehooks_default_alloc, - ehooks_default_dalloc, - ehooks_default_destroy, - ehooks_default_commit, - ehooks_default_decommit, +const extent_hooks_t ehooks_default_extent_hooks = {ehooks_default_alloc, + ehooks_default_dalloc, ehooks_default_destroy, ehooks_default_commit, + ehooks_default_decommit, #ifdef PAGES_CAN_PURGE_LAZY - ehooks_default_purge_lazy, + ehooks_default_purge_lazy, #else - NULL, + NULL, #endif #ifdef PAGES_CAN_PURGE_FORCED - ehooks_default_purge_forced, + ehooks_default_purge_forced, #else - NULL, + NULL, #endif - ehooks_default_split, - ehooks_default_merge -}; + ehooks_default_split, ehooks_default_merge}; diff --git a/dep/jemalloc/src/emap.c b/dep/jemalloc/src/emap.c index 9cc95a724..54bfabab0 100644 --- a/dep/jemalloc/src/emap.c +++ b/dep/jemalloc/src/emap.c @@ -16,10 +16,10 @@ emap_init(emap_t *emap, base_t *base, bool zeroed) { } void -emap_update_edata_state(tsdn_t *tsdn, emap_t *emap, edata_t *edata, - extent_state_t state) { - witness_assert_positive_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE); +emap_update_edata_state( + tsdn_t *tsdn, emap_t *emap, edata_t *edata, extent_state_t state) { + witness_assert_positive_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE); edata_state_set(edata, state); @@ -28,10 +28,11 @@ emap_update_edata_state(tsdn_t *tsdn, emap_t *emap, edata_t *edata, rtree_ctx, (uintptr_t)edata_base_get(edata), /* dependent */ true, /* init_missing */ false); assert(elm1 != NULL); - rtree_leaf_elm_t *elm2 = edata_size_get(edata) == PAGE ? NULL : - rtree_leaf_elm_lookup(tsdn, &emap->rtree, rtree_ctx, - (uintptr_t)edata_last_get(edata), /* dependent */ true, - /* init_missing */ false); + rtree_leaf_elm_t *elm2 = edata_size_get(edata) == PAGE + ? NULL + : rtree_leaf_elm_lookup(tsdn, &emap->rtree, rtree_ctx, + (uintptr_t)edata_last_get(edata), /* dependent */ true, + /* init_missing */ false); rtree_leaf_elm_state_update(tsdn, &emap->rtree, elm1, elm2, state); @@ -42,17 +43,17 @@ static inline edata_t * emap_try_acquire_edata_neighbor_impl(tsdn_t *tsdn, emap_t *emap, edata_t *edata, extent_pai_t pai, extent_state_t expected_state, bool forward, bool expanding) { - witness_assert_positive_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE); + witness_assert_positive_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE); assert(!edata_guarded_get(edata)); assert(!expanding || forward); assert(!edata_state_in_transition(expected_state)); - assert(expected_state == extent_state_dirty || - expected_state == extent_state_muzzy || - expected_state == extent_state_retained); + assert(expected_state == extent_state_dirty + || expected_state == extent_state_muzzy + || expected_state == extent_state_retained); - void *neighbor_addr = forward ? edata_past_get(edata) : - edata_before_get(edata); + void *neighbor_addr = forward ? edata_past_get(edata) + : edata_before_get(edata); /* * This is subtle; the rtree code asserts that its input pointer is * non-NULL, and this is a useful thing to check. But it's possible @@ -73,10 +74,10 @@ emap_try_acquire_edata_neighbor_impl(tsdn_t *tsdn, emap_t *emap, edata_t *edata, return NULL; } - rtree_contents_t neighbor_contents = rtree_leaf_elm_read(tsdn, - &emap->rtree, elm, /* dependent */ true); + rtree_contents_t neighbor_contents = rtree_leaf_elm_read( + tsdn, &emap->rtree, elm, /* dependent */ false); if (!extent_can_acquire_neighbor(edata, neighbor_contents, pai, - expected_state, forward, expanding)) { + expected_state, forward, expanding)) { return NULL; } @@ -109,8 +110,8 @@ emap_try_acquire_edata_neighbor_expand(tsdn_t *tsdn, emap_t *emap, } void -emap_release_edata(tsdn_t *tsdn, emap_t *emap, edata_t *edata, - extent_state_t new_state) { +emap_release_edata( + tsdn_t *tsdn, emap_t *emap, edata_t *edata, extent_state_t new_state) { assert(emap_edata_in_transition(tsdn, emap, edata)); assert(emap_edata_is_acquired(tsdn, emap, edata)); @@ -145,8 +146,8 @@ emap_rtree_write_acquired(tsdn_t *tsdn, emap_t *emap, rtree_leaf_elm_t *elm_a, contents.edata = edata; contents.metadata.szind = szind; contents.metadata.slab = slab; - contents.metadata.is_head = (edata == NULL) ? false : - edata_is_head_get(edata); + contents.metadata.is_head = (edata == NULL) ? false + : edata_is_head_get(edata); contents.metadata.state = (edata == NULL) ? 0 : edata_state_get(edata); rtree_leaf_elm_write(tsdn, &emap->rtree, elm_a, contents); if (elm_b != NULL) { @@ -155,29 +156,33 @@ emap_rtree_write_acquired(tsdn_t *tsdn, emap_t *emap, rtree_leaf_elm_t *elm_a, } bool -emap_register_boundary(tsdn_t *tsdn, emap_t *emap, edata_t *edata, - szind_t szind, bool slab) { +emap_register_boundary( + tsdn_t *tsdn, emap_t *emap, edata_t *edata, szind_t szind, bool slab) { assert(edata_state_get(edata) == extent_state_active); EMAP_DECLARE_RTREE_CTX; rtree_leaf_elm_t *elm_a, *elm_b; - bool err = emap_rtree_leaf_elms_lookup(tsdn, emap, rtree_ctx, edata, - false, true, &elm_a, &elm_b); + bool err = emap_rtree_leaf_elms_lookup( + tsdn, emap, rtree_ctx, edata, false, true, &elm_a, &elm_b); if (err) { return true; } assert(rtree_leaf_elm_read(tsdn, &emap->rtree, elm_a, - /* dependent */ false).edata == NULL); + /* dependent */ false) + .edata + == NULL); assert(rtree_leaf_elm_read(tsdn, &emap->rtree, elm_b, - /* dependent */ false).edata == NULL); + /* dependent */ false) + .edata + == NULL); emap_rtree_write_acquired(tsdn, emap, elm_a, elm_b, edata, szind, slab); return false; } /* Invoked *after* emap_register_boundary. */ void -emap_register_interior(tsdn_t *tsdn, emap_t *emap, edata_t *edata, - szind_t szind) { +emap_register_interior( + tsdn_t *tsdn, emap_t *emap, edata_t *edata, szind_t szind) { EMAP_DECLARE_RTREE_CTX; assert(edata_slab_get(edata)); @@ -226,10 +231,10 @@ emap_deregister_boundary(tsdn_t *tsdn, emap_t *emap, edata_t *edata) { EMAP_DECLARE_RTREE_CTX; rtree_leaf_elm_t *elm_a, *elm_b; - emap_rtree_leaf_elms_lookup(tsdn, emap, rtree_ctx, edata, - true, false, &elm_a, &elm_b); - emap_rtree_write_acquired(tsdn, emap, elm_a, elm_b, NULL, SC_NSIZES, - false); + emap_rtree_leaf_elms_lookup( + tsdn, emap, rtree_ctx, edata, true, false, &elm_a, &elm_b); + emap_rtree_write_acquired( + tsdn, emap, elm_a, elm_b, NULL, SC_NSIZES, false); } void @@ -245,8 +250,8 @@ emap_deregister_interior(tsdn_t *tsdn, emap_t *emap, edata_t *edata) { } void -emap_remap(tsdn_t *tsdn, emap_t *emap, edata_t *edata, szind_t szind, - bool slab) { +emap_remap( + tsdn_t *tsdn, emap_t *emap, edata_t *edata, szind_t szind, bool slab) { EMAP_DECLARE_RTREE_CTX; if (szind != SC_NSIZES) { @@ -274,8 +279,8 @@ emap_remap(tsdn_t *tsdn, emap_t *emap, edata_t *edata, szind_t szind, if (slab && edata_size_get(edata) > PAGE) { uintptr_t key = (uintptr_t)edata_past_get(edata) - (uintptr_t)PAGE; - rtree_write(tsdn, &emap->rtree, rtree_ctx, key, - contents); + rtree_write( + tsdn, &emap->rtree, rtree_ctx, key, contents); } } } @@ -344,29 +349,29 @@ emap_merge_commit(tsdn_t *tsdn, emap_t *emap, emap_prepare_t *prepare, clear_contents.metadata.state = (extent_state_t)0; if (prepare->lead_elm_b != NULL) { - rtree_leaf_elm_write(tsdn, &emap->rtree, - prepare->lead_elm_b, clear_contents); + rtree_leaf_elm_write( + tsdn, &emap->rtree, prepare->lead_elm_b, clear_contents); } rtree_leaf_elm_t *merged_b; if (prepare->trail_elm_b != NULL) { - rtree_leaf_elm_write(tsdn, &emap->rtree, - prepare->trail_elm_a, clear_contents); + rtree_leaf_elm_write( + tsdn, &emap->rtree, prepare->trail_elm_a, clear_contents); merged_b = prepare->trail_elm_b; } else { merged_b = prepare->trail_elm_a; } - emap_rtree_write_acquired(tsdn, emap, prepare->lead_elm_a, merged_b, - lead, SC_NSIZES, false); + emap_rtree_write_acquired( + tsdn, emap, prepare->lead_elm_a, merged_b, lead, SC_NSIZES, false); } void emap_do_assert_mapped(tsdn_t *tsdn, emap_t *emap, edata_t *edata) { EMAP_DECLARE_RTREE_CTX; - rtree_contents_t contents = rtree_read(tsdn, &emap->rtree, rtree_ctx, - (uintptr_t)edata_base_get(edata)); + rtree_contents_t contents = rtree_read( + tsdn, &emap->rtree, rtree_ctx, (uintptr_t)edata_base_get(edata)); assert(contents.edata == edata); assert(contents.metadata.is_head == edata_is_head_get(edata)); assert(contents.metadata.state == edata_state_get(edata)); @@ -375,12 +380,12 @@ emap_do_assert_mapped(tsdn_t *tsdn, emap_t *emap, edata_t *edata) { void emap_do_assert_not_mapped(tsdn_t *tsdn, emap_t *emap, edata_t *edata) { emap_full_alloc_ctx_t context1 = {0}; - emap_full_alloc_ctx_try_lookup(tsdn, emap, edata_base_get(edata), - &context1); + emap_full_alloc_ctx_try_lookup( + tsdn, emap, edata_base_get(edata), &context1); assert(context1.edata == NULL); emap_full_alloc_ctx_t context2 = {0}; - emap_full_alloc_ctx_try_lookup(tsdn, emap, edata_last_get(edata), - &context2); + emap_full_alloc_ctx_try_lookup( + tsdn, emap, edata_last_get(edata), &context2); assert(context2.edata == NULL); } diff --git a/dep/jemalloc/src/eset.c b/dep/jemalloc/src/eset.c index 6f8f335e1..4a427d78d 100644 --- a/dep/jemalloc/src/eset.c +++ b/dep/jemalloc/src/eset.c @@ -48,32 +48,32 @@ eset_nbytes_get(eset_t *eset, pszind_t pind) { static void eset_stats_add(eset_t *eset, pszind_t pind, size_t sz) { - size_t cur = atomic_load_zu(&eset->bin_stats[pind].nextents, - ATOMIC_RELAXED); - atomic_store_zu(&eset->bin_stats[pind].nextents, cur + 1, - ATOMIC_RELAXED); + size_t cur = atomic_load_zu( + &eset->bin_stats[pind].nextents, ATOMIC_RELAXED); + atomic_store_zu( + &eset->bin_stats[pind].nextents, cur + 1, ATOMIC_RELAXED); cur = atomic_load_zu(&eset->bin_stats[pind].nbytes, ATOMIC_RELAXED); - atomic_store_zu(&eset->bin_stats[pind].nbytes, cur + sz, - ATOMIC_RELAXED); + atomic_store_zu( + &eset->bin_stats[pind].nbytes, cur + sz, ATOMIC_RELAXED); } static void eset_stats_sub(eset_t *eset, pszind_t pind, size_t sz) { - size_t cur = atomic_load_zu(&eset->bin_stats[pind].nextents, - ATOMIC_RELAXED); - atomic_store_zu(&eset->bin_stats[pind].nextents, cur - 1, - ATOMIC_RELAXED); + size_t cur = atomic_load_zu( + &eset->bin_stats[pind].nextents, ATOMIC_RELAXED); + atomic_store_zu( + &eset->bin_stats[pind].nextents, cur - 1, ATOMIC_RELAXED); cur = atomic_load_zu(&eset->bin_stats[pind].nbytes, ATOMIC_RELAXED); - atomic_store_zu(&eset->bin_stats[pind].nbytes, cur - sz, - ATOMIC_RELAXED); + atomic_store_zu( + &eset->bin_stats[pind].nbytes, cur - sz, ATOMIC_RELAXED); } void eset_insert(eset_t *eset, edata_t *edata) { assert(edata_state_get(edata) == eset->state); - size_t size = edata_size_get(edata); - size_t psz = sz_psz_quantize_floor(size); + size_t size = edata_size_get(edata); + size_t psz = sz_psz_quantize_floor(size); pszind_t pind = sz_psz2ind(psz); edata_cmp_summary_t edata_cmp_summary = edata_cmp_summary_get(edata); @@ -86,8 +86,9 @@ eset_insert(eset_t *eset, edata_t *edata) { * There's already a min element; update the summary if we're * about to insert a lower one. */ - if (edata_cmp_summary_comp(edata_cmp_summary, - eset->bins[pind].heap_min) < 0) { + if (edata_cmp_summary_comp( + edata_cmp_summary, eset->bins[pind].heap_min) + < 0) { eset->bins[pind].heap_min = edata_cmp_summary; } } @@ -104,19 +105,18 @@ eset_insert(eset_t *eset, edata_t *edata) { * don't need an atomic fetch-add; we can get by with a load followed by * a store. */ - size_t cur_eset_npages = - atomic_load_zu(&eset->npages, ATOMIC_RELAXED); - atomic_store_zu(&eset->npages, cur_eset_npages + npages, - ATOMIC_RELAXED); + size_t cur_eset_npages = atomic_load_zu(&eset->npages, ATOMIC_RELAXED); + atomic_store_zu( + &eset->npages, cur_eset_npages + npages, ATOMIC_RELAXED); } void eset_remove(eset_t *eset, edata_t *edata) { - assert(edata_state_get(edata) == eset->state || - edata_state_in_transition(edata_state_get(edata))); + assert(edata_state_get(edata) == eset->state + || edata_state_in_transition(edata_state_get(edata))); - size_t size = edata_size_get(edata); - size_t psz = sz_psz_quantize_floor(size); + size_t size = edata_size_get(edata); + size_t psz = sz_psz_quantize_floor(size); pszind_t pind = sz_psz2ind(psz); if (config_stats) { eset_stats_sub(eset, pind, size); @@ -136,8 +136,9 @@ eset_remove(eset_t *eset, edata_t *edata) { * summaries of the removed element and the min element should * compare equal. */ - if (edata_cmp_summary_comp(edata_cmp_summary, - eset->bins[pind].heap_min) == 0) { + if (edata_cmp_summary_comp( + edata_cmp_summary, eset->bins[pind].heap_min) + == 0) { eset->bins[pind].heap_min = edata_cmp_summary_get( edata_heap_first(&eset->bins[pind].heap)); } @@ -148,11 +149,77 @@ eset_remove(eset_t *eset, edata_t *edata) { * As in eset_insert, we hold eset->mtx and so don't need atomic * operations for updating eset->npages. */ - size_t cur_extents_npages = - atomic_load_zu(&eset->npages, ATOMIC_RELAXED); + size_t cur_extents_npages = atomic_load_zu( + &eset->npages, ATOMIC_RELAXED); assert(cur_extents_npages >= npages); - atomic_store_zu(&eset->npages, - cur_extents_npages - (size >> LG_PAGE), ATOMIC_RELAXED); + atomic_store_zu(&eset->npages, cur_extents_npages - (size >> LG_PAGE), + ATOMIC_RELAXED); +} + +static edata_t * +eset_enumerate_alignment_search( + eset_t *eset, size_t size, pszind_t bin_ind, size_t alignment) { + if (edata_heap_empty(&eset->bins[bin_ind].heap)) { + return NULL; + } + + edata_t *edata = NULL; + edata_heap_enumerate_helper_t helper; + edata_heap_enumerate_prepare(&eset->bins[bin_ind].heap, &helper, + ESET_ENUMERATE_MAX_NUM, sizeof(helper.bfs_queue) / sizeof(void *)); + while ((edata = edata_heap_enumerate_next( + &eset->bins[bin_ind].heap, &helper)) + != NULL) { + uintptr_t base = (uintptr_t)edata_base_get(edata); + size_t candidate_size = edata_size_get(edata); + if (candidate_size < size) { + continue; + } + + uintptr_t next_align = ALIGNMENT_CEILING( + (uintptr_t)base, PAGE_CEILING(alignment)); + if (base > next_align || base + candidate_size <= next_align) { + /* Overflow or not crossing the next alignment. */ + continue; + } + + size_t leadsize = next_align - base; + if (candidate_size - leadsize >= size) { + return edata; + } + } + + return NULL; +} + +static edata_t * +eset_enumerate_search(eset_t *eset, size_t size, pszind_t bin_ind, + bool exact_only, edata_cmp_summary_t *ret_summ) { + if (edata_heap_empty(&eset->bins[bin_ind].heap)) { + return NULL; + } + + edata_t *ret = NULL, *edata = NULL; + edata_heap_enumerate_helper_t helper; + edata_heap_enumerate_prepare(&eset->bins[bin_ind].heap, &helper, + ESET_ENUMERATE_MAX_NUM, sizeof(helper.bfs_queue) / sizeof(void *)); + while ((edata = edata_heap_enumerate_next( + &eset->bins[bin_ind].heap, &helper)) + != NULL) { + if ((!exact_only && edata_size_get(edata) >= size) + || (exact_only && edata_size_get(edata) == size)) { + edata_cmp_summary_t temp_summ = edata_cmp_summary_get( + edata); + if (ret == NULL + || edata_cmp_summary_comp(temp_summ, *ret_summ) + < 0) { + ret = edata; + *ret_summ = temp_summ; + } + } + } + + return ret; } /* @@ -160,24 +227,35 @@ eset_remove(eset_t *eset, edata_t *edata) { * requirement. For each size, try only the first extent in the heap. */ static edata_t * -eset_fit_alignment(eset_t *eset, size_t min_size, size_t max_size, - size_t alignment) { - pszind_t pind = sz_psz2ind(sz_psz_quantize_ceil(min_size)); - pszind_t pind_max = sz_psz2ind(sz_psz_quantize_ceil(max_size)); +eset_fit_alignment( + eset_t *eset, size_t min_size, size_t max_size, size_t alignment) { + pszind_t pind = sz_psz2ind(sz_psz_quantize_ceil(min_size)); + pszind_t pind_max = sz_psz2ind(sz_psz_quantize_ceil(max_size)); + + /* See comments in eset_first_fit for why we enumerate search below. */ + pszind_t pind_prev = sz_psz2ind(sz_psz_quantize_floor(min_size)); + if (sz_large_size_classes_disabled() && pind != pind_prev) { + edata_t *ret = NULL; + ret = eset_enumerate_alignment_search( + eset, min_size, pind_prev, alignment); + if (ret != NULL) { + return ret; + } + } for (pszind_t i = - (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)pind); - i < pind_max; - i = (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)i + 1)) { + (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)pind); + i < pind_max; + i = (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)i + 1)) { assert(i < SC_NPSIZES); assert(!edata_heap_empty(&eset->bins[i].heap)); - edata_t *edata = edata_heap_first(&eset->bins[i].heap); + edata_t *edata = edata_heap_first(&eset->bins[i].heap); uintptr_t base = (uintptr_t)edata_base_get(edata); - size_t candidate_size = edata_size_get(edata); + size_t candidate_size = edata_size_get(edata); assert(candidate_size >= min_size); - uintptr_t next_align = ALIGNMENT_CEILING((uintptr_t)base, - PAGE_CEILING(alignment)); + uintptr_t next_align = ALIGNMENT_CEILING( + (uintptr_t)base, PAGE_CEILING(alignment)); if (base > next_align || base + candidate_size <= next_align) { /* Overflow or not crossing the next alignment. */ continue; @@ -203,22 +281,58 @@ eset_fit_alignment(eset_t *eset, size_t min_size, size_t max_size, * for others. */ static edata_t * -eset_first_fit(eset_t *eset, size_t size, bool exact_only, - unsigned lg_max_fit) { - edata_t *ret = NULL; +eset_first_fit( + eset_t *eset, size_t size, bool exact_only, unsigned lg_max_fit) { + edata_t *ret = NULL; edata_cmp_summary_t ret_summ JEMALLOC_CC_SILENCE_INIT({0}); pszind_t pind = sz_psz2ind(sz_psz_quantize_ceil(size)); if (exact_only) { - return edata_heap_empty(&eset->bins[pind].heap) ? NULL : - edata_heap_first(&eset->bins[pind].heap); + if (sz_large_size_classes_disabled()) { + pszind_t pind_prev = sz_psz2ind( + sz_psz_quantize_floor(size)); + return eset_enumerate_search(eset, size, pind_prev, + /* exact_only */ true, &ret_summ); + } else { + return edata_heap_empty(&eset->bins[pind].heap) + ? NULL + : edata_heap_first(&eset->bins[pind].heap); + } + } + + /* + * Each element in the eset->bins is a heap corresponding to a size + * class. When sz_large_size_classes_disabled() is false, all heaps after + * pind (including pind itself) will surely satisfy the rquests while + * heaps before pind cannot satisfy the request because usize is + * calculated based on size classes then. However, when + * sz_large_size_classes_disabled() is true, usize is calculated by + * ceiling user requested size to the closest multiple of PAGE. This + * means in the heap before pind, i.e., pind_prev, there may exist + * extents able to satisfy the request and we should enumerate the heap + * when pind_prev != pind. + * + * For example, when PAGE=4KB and the user requested size is 1MB + 4KB, + * usize would be 1.25MB when sz_large_size_classes_disabled() is false. + * pind points to the heap containing extents ranging in + * [1.25MB, 1.5MB). Thus, searching starting from pind will not miss + * any candidates. When sz_large_size_classes_disabled() is true, the + * usize would be 1MB + 4KB and pind still points to the same heap. + * In this case, the heap pind_prev points to, which contains extents + * in the range [1MB, 1.25MB), may contain candidates satisfying the + * usize and thus should be enumerated. + */ + pszind_t pind_prev = sz_psz2ind(sz_psz_quantize_floor(size)); + if (sz_large_size_classes_disabled() && pind != pind_prev) { + ret = eset_enumerate_search(eset, size, pind_prev, + /* exact_only */ false, &ret_summ); } for (pszind_t i = - (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)pind); - i < ESET_NPSIZES; - i = (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)i + 1)) { + (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)pind); + i < ESET_NPSIZES; + i = (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)i + 1)) { assert(!edata_heap_empty(&eset->bins[i].heap)); if (lg_max_fit == SC_PTR_BITS) { /* @@ -231,8 +345,9 @@ eset_first_fit(eset_t *eset, size_t size, bool exact_only, if ((sz_pind2sz(i) >> lg_max_fit) > size) { break; } - if (ret == NULL || edata_cmp_summary_comp( - eset->bins[i].heap_min, ret_summ) < 0) { + if (ret == NULL + || edata_cmp_summary_comp(eset->bins[i].heap_min, ret_summ) + < 0) { /* * We grab the edata as early as possible, even though * we might change it later. Practically, a large @@ -243,9 +358,10 @@ eset_first_fit(eset_t *eset, size_t size, bool exact_only, edata_t *edata = edata_heap_first(&eset->bins[i].heap); assert(edata_size_get(edata) >= size); assert(ret == NULL || edata_snad_comp(edata, ret) < 0); - assert(ret == NULL || edata_cmp_summary_comp( - eset->bins[i].heap_min, - edata_cmp_summary_get(edata)) == 0); + assert(ret == NULL + || edata_cmp_summary_comp(eset->bins[i].heap_min, + edata_cmp_summary_get(edata)) + == 0); ret = edata; ret_summ = eset->bins[i].heap_min; } diff --git a/dep/jemalloc/src/exp_grow.c b/dep/jemalloc/src/exp_grow.c index 386471f49..955823a1d 100644 --- a/dep/jemalloc/src/exp_grow.c +++ b/dep/jemalloc/src/exp_grow.c @@ -3,6 +3,12 @@ void exp_grow_init(exp_grow_t *exp_grow) { - exp_grow->next = sz_psz2ind(HUGEPAGE); + /* + * Enforce a minimal of 2M grow, which is convenient for the huge page + * use cases. Avoid using HUGEPAGE as the value though, because on some + * platforms it can be very large (e.g. 512M on aarch64 w/ 64K pages). + */ + const size_t min_grow = (size_t)2 << 20; + exp_grow->next = sz_psz2ind(min_grow); exp_grow->limit = sz_psz2ind(SC_LARGE_MAXCLASS); } diff --git a/dep/jemalloc/src/extent.c b/dep/jemalloc/src/extent.c index cf3d1f311..118c8785f 100644 --- a/dep/jemalloc/src/extent.c +++ b/dep/jemalloc/src/extent.c @@ -12,6 +12,14 @@ /* Data. */ size_t opt_lg_extent_max_active_fit = LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT; +/* This option is intended for kernel tuning, not app tuning. */ +size_t opt_process_madvise_max_batch = +#ifdef JEMALLOC_HAVE_PROCESS_MADVISE + PROCESS_MADVISE_MAX_BATCH_DEFAULT; +#else + 0 +#endif +; static bool extent_commit_impl(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, size_t offset, size_t length, bool growing_retained); @@ -21,8 +29,8 @@ static bool extent_purge_forced_impl(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, size_t offset, size_t length, bool growing_retained); static edata_t *extent_split_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata, size_t size_a, size_t size_b, bool holding_core_locks); -static bool extent_merge_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, - edata_t *a, edata_t *b, bool holding_core_locks); +static bool extent_merge_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, + edata_t *a, edata_t *b, bool holding_core_locks); /* Used exclusively for gdump triggering. */ static atomic_zu_t curpages; @@ -34,7 +42,7 @@ static atomic_zu_t highpages; * definition. */ -static void extent_deregister(tsdn_t *tsdn, pac_t *pac, edata_t *edata); +static void extent_deregister(tsdn_t *tsdn, pac_t *pac, edata_t *edata); static edata_t *extent_recycle(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, edata_t *expand_edata, size_t usize, size_t alignment, bool zero, bool *commit, bool growing_retained, bool guarded); @@ -43,6 +51,8 @@ static edata_t *extent_try_coalesce(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, static edata_t *extent_alloc_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *expand_edata, size_t size, size_t alignment, bool zero, bool *commit, bool guarded); +static bool extent_decommit_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, + edata_t *edata, size_t offset, size_t length); /******************************************************************************/ @@ -63,8 +73,8 @@ extent_try_delayed_coalesce(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, emap_update_edata_state(tsdn, pac->emap, edata, extent_state_active); bool coalesced; - edata = extent_try_coalesce(tsdn, pac, ehooks, ecache, - edata, &coalesced); + edata = extent_try_coalesce( + tsdn, pac, ehooks, ecache, edata, &coalesced); emap_update_edata_state(tsdn, pac->emap, edata, ecache->state); if (!coalesced) { @@ -80,10 +90,10 @@ ecache_alloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, bool guarded) { assert(size != 0); assert(alignment != 0); - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); - bool commit = true; + bool commit = true; edata_t *edata = extent_recycle(tsdn, pac, ehooks, ecache, expand_edata, size, alignment, zero, &commit, false, guarded); assert(edata == NULL || edata_pai_get(edata) == EXTENT_PAI_PAC); @@ -97,10 +107,10 @@ ecache_alloc_grow(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, bool guarded) { assert(size != 0); assert(alignment != 0); - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); - bool commit = true; + bool commit = true; edata_t *edata = extent_alloc_retained(tsdn, pac, ehooks, expand_edata, size, alignment, zero, &commit, guarded); if (edata == NULL) { @@ -121,10 +131,11 @@ ecache_alloc_grow(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, */ return NULL; } - void *new_addr = (expand_edata == NULL) ? NULL : - edata_past_get(expand_edata); - edata = extent_alloc_wrapper(tsdn, pac, ehooks, new_addr, - size, alignment, zero, &commit, + void *new_addr = (expand_edata == NULL) + ? NULL + : edata_past_get(expand_edata); + edata = extent_alloc_wrapper(tsdn, pac, ehooks, new_addr, size, + alignment, zero, &commit, /* growing_retained */ false); } @@ -138,8 +149,8 @@ ecache_dalloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, assert(edata_base_get(edata) != NULL); assert(edata_size_get(edata) != 0); assert(edata_pai_get(edata) == EXTENT_PAI_PAC); - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); edata_addr_set(edata, edata_base_get(edata)); edata_zeroed_set(edata, false); @@ -148,8 +159,8 @@ ecache_dalloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, } edata_t * -ecache_evict(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, - ecache_t *ecache, size_t npages_min) { +ecache_evict(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, + size_t npages_min) { malloc_mutex_lock(tsdn, &ecache->mtx); /* @@ -184,8 +195,8 @@ ecache_evict(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, break; } /* Try to coalesce. */ - if (extent_try_delayed_coalesce(tsdn, pac, ehooks, ecache, - edata)) { + if (extent_try_delayed_coalesce( + tsdn, pac, ehooks, ecache, edata)) { break; } /* @@ -199,16 +210,17 @@ ecache_evict(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, * concurrent operations. */ switch (ecache->state) { - case extent_state_active: - not_reached(); case extent_state_dirty: case extent_state_muzzy: - emap_update_edata_state(tsdn, pac->emap, edata, - extent_state_active); + emap_update_edata_state( + tsdn, pac->emap, edata, extent_state_active); break; case extent_state_retained: extent_deregister(tsdn, pac, edata); break; + case extent_state_active: + case extent_state_transition: + case extent_state_merging: default: not_reached(); } @@ -227,16 +239,16 @@ extents_abandon_vm(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, edata_t *edata, bool growing_retained) { size_t sz = edata_size_get(edata); if (config_stats) { - atomic_fetch_add_zu(&pac->stats->abandoned_vm, sz, - ATOMIC_RELAXED); + atomic_fetch_add_zu( + &pac->stats->abandoned_vm, sz, ATOMIC_RELAXED); } /* * Leak extent after making sure its pages have already been purged, so * that this is only a virtual memory leak. */ if (ecache->state == extent_state_dirty) { - if (extent_purge_lazy_impl(tsdn, ehooks, edata, 0, sz, - growing_retained)) { + if (extent_purge_lazy_impl( + tsdn, ehooks, edata, 0, sz, growing_retained)) { extent_purge_forced_impl(tsdn, ehooks, edata, 0, edata_size_get(edata), growing_retained); } @@ -245,20 +257,20 @@ extents_abandon_vm(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, } static void -extent_deactivate_locked_impl(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache, - edata_t *edata) { +extent_deactivate_locked_impl( + tsdn_t *tsdn, pac_t *pac, ecache_t *ecache, edata_t *edata) { malloc_mutex_assert_owner(tsdn, &ecache->mtx); assert(edata_arena_ind_get(edata) == ecache_ind_get(ecache)); emap_update_edata_state(tsdn, pac->emap, edata, ecache->state); - eset_t *eset = edata_guarded_get(edata) ? &ecache->guarded_eset : - &ecache->eset; + eset_t *eset = edata_guarded_get(edata) ? &ecache->guarded_eset + : &ecache->eset; eset_insert(eset, edata); } static void -extent_deactivate_locked(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache, - edata_t *edata) { +extent_deactivate_locked( + tsdn_t *tsdn, pac_t *pac, ecache_t *ecache, edata_t *edata) { assert(edata_state_get(edata) == extent_state_active); extent_deactivate_locked_impl(tsdn, pac, ecache, edata); } @@ -271,11 +283,11 @@ extent_deactivate_check_state_locked(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache, } static void -extent_activate_locked(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache, eset_t *eset, - edata_t *edata) { +extent_activate_locked( + tsdn_t *tsdn, pac_t *pac, ecache_t *ecache, eset_t *eset, edata_t *edata) { assert(edata_arena_ind_get(edata) == ecache_ind_get(ecache)); - assert(edata_state_get(edata) == ecache->state || - edata_state_get(edata) == extent_state_merging); + assert(edata_state_get(edata) == ecache->state + || edata_state_get(edata) == extent_state_merging); eset_remove(eset, edata); emap_update_edata_state(tsdn, pac->emap, edata, extent_state_active); @@ -285,16 +297,18 @@ void extent_gdump_add(tsdn_t *tsdn, const edata_t *edata) { cassert(config_prof); /* prof_gdump() requirement. */ - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); if (opt_prof && edata_state_get(edata) == extent_state_active) { size_t nadd = edata_size_get(edata) >> LG_PAGE; - size_t cur = atomic_fetch_add_zu(&curpages, nadd, - ATOMIC_RELAXED) + nadd; + size_t cur = atomic_fetch_add_zu( + &curpages, nadd, ATOMIC_RELAXED) + + nadd; size_t high = atomic_load_zu(&highpages, ATOMIC_RELAXED); - while (cur > high && !atomic_compare_exchange_weak_zu( - &highpages, &high, cur, ATOMIC_RELAXED, ATOMIC_RELAXED)) { + while (cur > high + && !atomic_compare_exchange_weak_zu(&highpages, &high, cur, + ATOMIC_RELAXED, ATOMIC_RELAXED)) { /* * Don't refresh cur, because it may have decreased * since this thread lost the highpages update race. @@ -326,7 +340,7 @@ extent_register_impl(tsdn_t *tsdn, pac_t *pac, edata_t *edata, bool gdump_add) { * prevents other threads from accessing the edata. */ if (emap_register_boundary(tsdn, pac->emap, edata, SC_NSIZES, - /* slab */ false)) { + /* slab */ false)) { return true; } @@ -357,8 +371,7 @@ extent_reregister(tsdn_t *tsdn, pac_t *pac, edata_t *edata) { * Removes all pointers to the given extent from the global rtree. */ static void -extent_deregister_impl(tsdn_t *tsdn, pac_t *pac, edata_t *edata, - bool gdump) { +extent_deregister_impl(tsdn_t *tsdn, pac_t *pac, edata_t *edata, bool gdump) { emap_deregister_boundary(tsdn, pac->emap, edata); if (config_prof && gdump) { @@ -372,8 +385,7 @@ extent_deregister(tsdn_t *tsdn, pac_t *pac, edata_t *edata) { } static void -extent_deregister_no_gdump_sub(tsdn_t *tsdn, pac_t *pac, - edata_t *edata) { +extent_deregister_no_gdump_sub(tsdn_t *tsdn, pac_t *pac, edata_t *edata) { extent_deregister_impl(tsdn, pac, edata, false); } @@ -400,15 +412,16 @@ extent_recycle_extract(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, } edata_t *edata; - eset_t *eset = guarded ? &ecache->guarded_eset : &ecache->eset; + eset_t *eset = guarded ? &ecache->guarded_eset : &ecache->eset; if (expand_edata != NULL) { edata = emap_try_acquire_edata_neighbor_expand(tsdn, pac->emap, expand_edata, EXTENT_PAI_PAC, ecache->state); if (edata != NULL) { + /* NOLINTNEXTLINE(readability-suspicious-call-argument) */ extent_assert_can_expand(expand_edata, edata); if (edata_size_get(edata) < size) { - emap_release_edata(tsdn, pac->emap, edata, - ecache->state); + emap_release_edata( + tsdn, pac->emap, edata, ecache->state); edata = NULL; } } @@ -423,7 +436,8 @@ extent_recycle_extract(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, * put a cap on how big an extent we can split for a request. */ unsigned lg_max_fit = ecache->delay_coalesce - ? (unsigned)opt_lg_extent_max_active_fit : SC_PTR_BITS; + ? (unsigned)opt_lg_extent_max_active_fit + : SC_PTR_BITS; /* * If split and merge are not allowed (Windows w/o retain), try @@ -434,8 +448,7 @@ extent_recycle_extract(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, * allocations. */ bool exact_only = (!maps_coalesce && !opt_retain) || guarded; - edata = eset_fit(eset, size, alignment, exact_only, - lg_max_fit); + edata = eset_fit(eset, size, alignment, exact_only, lg_max_fit); } if (edata == NULL) { return NULL; @@ -477,10 +490,11 @@ extent_split_interior(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, /* The result of splitting, in case of success. */ edata_t **edata, edata_t **lead, edata_t **trail, /* The mess to clean up, in case of error. */ - edata_t **to_leak, edata_t **to_salvage, - edata_t *expand_edata, size_t size, size_t alignment) { + edata_t **to_leak, edata_t **to_salvage, edata_t *expand_edata, size_t size, + size_t alignment) { size_t leadsize = ALIGNMENT_CEILING((uintptr_t)edata_base_get(*edata), - PAGE_CEILING(alignment)) - (uintptr_t)edata_base_get(*edata); + PAGE_CEILING(alignment)) + - (uintptr_t)edata_base_get(*edata); assert(expand_edata == NULL || leadsize == 0); if (edata_size_get(*edata) < leadsize + size) { return extent_split_interior_cant_alloc; @@ -535,14 +549,14 @@ extent_recycle_split(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, assert(!edata_guarded_get(edata) || size == edata_size_get(edata)); malloc_mutex_assert_owner(tsdn, &ecache->mtx); - edata_t *lead; - edata_t *trail; - edata_t *to_leak JEMALLOC_CC_SILENCE_INIT(NULL); + edata_t *lead; + edata_t *trail; + edata_t *to_leak JEMALLOC_CC_SILENCE_INIT(NULL); edata_t *to_salvage JEMALLOC_CC_SILENCE_INIT(NULL); - extent_split_interior_result_t result = extent_split_interior( - tsdn, pac, ehooks, &edata, &lead, &trail, &to_leak, &to_salvage, - expand_edata, size, alignment); + extent_split_interior_result_t result = extent_split_interior(tsdn, pac, + ehooks, &edata, &lead, &trail, &to_leak, &to_salvage, expand_edata, + size, alignment); if (!maps_coalesce && result != extent_split_interior_ok && !opt_retain) { @@ -603,8 +617,8 @@ extent_recycle(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, malloc_mutex_lock(tsdn, &ecache->mtx); - edata_t *edata = extent_recycle_extract(tsdn, pac, ehooks, ecache, - expand_edata, size, alignment, guarded); + edata_t *edata = extent_recycle_extract( + tsdn, pac, ehooks, ecache, expand_edata, size, alignment, guarded); if (edata == NULL) { malloc_mutex_unlock(tsdn, &ecache->mtx); return NULL; @@ -618,8 +632,8 @@ extent_recycle(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, } assert(edata_state_get(edata) == extent_state_active); - if (extent_commit_zero(tsdn, ehooks, edata, *commit, zero, - growing_retained)) { + if (extent_commit_zero( + tsdn, ehooks, edata, *commit, zero, growing_retained)) { extent_record(tsdn, pac, ehooks, ecache, edata); return NULL; } @@ -635,14 +649,65 @@ extent_recycle(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, return edata; } +static void +extent_handle_huge_arena_thp(tsdn_t *tsdn, pac_thp_t *pac_thp, + edata_cache_t *edata_cache, void *addr, size_t size) { + assert(opt_huge_arena_pac_thp); + assert(opt_metadata_thp != metadata_thp_disabled); + /* + * With rounding up the given memory region [addr, addr + size) to + * the huge page region that it crosses boundaries with, + * essentially we're aligning the start addr down and the end addr + * up to the nearest HUGEPAGE boundaries. The memory overhead can + * be within the range of [0, 2 * (HUGEPAGE - 1)]. + */ + void *huge_addr = HUGEPAGE_ADDR2BASE(addr); + void *huge_end = HUGEPAGE_ADDR2BASE( + (void *)((byte_t *)addr + (uintptr_t)(size + HUGEPAGE - 1))); + assert((uintptr_t)huge_end > (uintptr_t)huge_addr); + + size_t huge_size = (uintptr_t)huge_end - (uintptr_t)huge_addr; + assert( + huge_size <= (size + ((HUGEPAGE - 1) << 1)) && huge_size >= size); + + if (opt_metadata_thp == metadata_thp_always + || pac_thp->auto_thp_switched) { + pages_huge(huge_addr, huge_size); + } else { + assert(opt_metadata_thp == metadata_thp_auto); + edata_t *edata = edata_cache_get(tsdn, edata_cache); + + malloc_mutex_lock(tsdn, &pac_thp->lock); + /* Can happen if the switch is turned on during edata retrieval. */ + if (pac_thp->auto_thp_switched) { + malloc_mutex_unlock(tsdn, &pac_thp->lock); + pages_huge(huge_addr, huge_size); + if (edata != NULL) { + edata_cache_put(tsdn, edata_cache, edata); + } + } else { + if (edata != NULL) { + edata_addr_set(edata, huge_addr); + edata_size_set(edata, huge_size); + edata_list_active_append( + &pac_thp->thp_lazy_list, edata); + atomic_fetch_add_u( + &pac_thp->n_thp_lazy, 1, ATOMIC_RELAXED); + } + malloc_mutex_unlock(tsdn, &pac_thp->lock); + } + malloc_mutex_assert_not_owner(tsdn, &pac_thp->lock); + } +} + /* * If virtual memory is retained, create increasingly larger extents from which * to split requested extents in order to limit the total number of disjoint * virtual memory ranges retained by each shard. */ static edata_t * -extent_grow_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, - size_t size, size_t alignment, bool zero, bool *commit) { +extent_grow_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size, + size_t alignment, bool zero, bool *commit) { malloc_mutex_assert_owner(tsdn, &pac->grow_mtx); size_t alloc_size_min = size + PAGE_CEILING(alignment) - PAGE; @@ -654,10 +719,10 @@ extent_grow_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, * Find the next extent size in the series that would be large enough to * satisfy this request. */ - size_t alloc_size; + size_t alloc_size; pszind_t exp_grow_skip; - bool err = exp_grow_size_prepare(&pac->exp_grow, alloc_size_min, - &alloc_size, &exp_grow_skip); + bool err = exp_grow_size_prepare( + &pac->exp_grow, alloc_size_min, &alloc_size, &exp_grow_skip); if (err) { goto label_err; } @@ -669,18 +734,18 @@ extent_grow_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, bool zeroed = false; bool committed = false; - void *ptr = ehooks_alloc(tsdn, ehooks, NULL, alloc_size, PAGE, &zeroed, - &committed); + void *ptr = ehooks_alloc( + tsdn, ehooks, NULL, alloc_size, PAGE, &zeroed, &committed); if (ptr == NULL) { edata_cache_put(tsdn, pac->edata_cache, edata); goto label_err; } - edata_init(edata, ecache_ind_get(&pac->ecache_retained), ptr, - alloc_size, false, SC_NSIZES, extent_sn_next(pac), - extent_state_active, zeroed, committed, EXTENT_PAI_PAC, - EXTENT_IS_HEAD); + unsigned ind = ecache_ind_get(&pac->ecache_retained); + edata_init(edata, ind, ptr, alloc_size, false, SC_NSIZES, + extent_sn_next(pac), extent_state_active, zeroed, committed, + EXTENT_PAI_PAC, EXTENT_IS_HEAD); if (extent_register_no_gdump_add(tsdn, pac, edata)) { edata_cache_put(tsdn, pac->edata_cache, edata); @@ -691,23 +756,23 @@ extent_grow_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, *commit = true; } - edata_t *lead; - edata_t *trail; - edata_t *to_leak JEMALLOC_CC_SILENCE_INIT(NULL); + edata_t *lead; + edata_t *trail; + edata_t *to_leak JEMALLOC_CC_SILENCE_INIT(NULL); edata_t *to_salvage JEMALLOC_CC_SILENCE_INIT(NULL); - extent_split_interior_result_t result = extent_split_interior(tsdn, - pac, ehooks, &edata, &lead, &trail, &to_leak, &to_salvage, NULL, - size, alignment); + extent_split_interior_result_t result = extent_split_interior(tsdn, pac, + ehooks, &edata, &lead, &trail, &to_leak, &to_salvage, NULL, size, + alignment); if (result == extent_split_interior_ok) { if (lead != NULL) { - extent_record(tsdn, pac, ehooks, &pac->ecache_retained, - lead); + extent_record( + tsdn, pac, ehooks, &pac->ecache_retained, lead); } if (trail != NULL) { - extent_record(tsdn, pac, ehooks, &pac->ecache_retained, - trail); + extent_record( + tsdn, pac, ehooks, &pac->ecache_retained, trail); } } else { /* @@ -731,16 +796,16 @@ extent_grow_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, } if (*commit && !edata_committed_get(edata)) { - if (extent_commit_impl(tsdn, ehooks, edata, 0, - edata_size_get(edata), true)) { - extent_record(tsdn, pac, ehooks, - &pac->ecache_retained, edata); + if (extent_commit_impl( + tsdn, ehooks, edata, 0, edata_size_get(edata), true)) { + extent_record( + tsdn, pac, ehooks, &pac->ecache_retained, edata); goto label_err; } /* A successful commit should return zeroed memory. */ if (config_debug) { - void *addr = edata_addr_get(edata); - size_t *p = (size_t *)(uintptr_t)addr; + void *addr = edata_addr_get(edata); + size_t *p = (size_t *)addr; /* Check the first page only. */ for (size_t i = 0; i < PAGE / sizeof(size_t); i++) { assert(p[i] == 0); @@ -756,13 +821,23 @@ extent_grow_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, exp_grow_size_commit(&pac->exp_grow, exp_grow_skip); malloc_mutex_unlock(tsdn, &pac->grow_mtx); + if (huge_arena_pac_thp.thp_madvise) { + /* Avoid using HUGEPAGE when the grow size is less than HUGEPAGE. */ + if (ind != 0 && ind == huge_arena_ind + && ehooks_are_default(ehooks) + && likely(alloc_size >= HUGEPAGE)) { + extent_handle_huge_arena_thp(tsdn, &huge_arena_pac_thp, + pac->edata_cache, ptr, alloc_size); + } + } + if (config_prof) { /* Adjust gdump stats now that extent is final size. */ extent_gdump_add(tsdn, edata); } if (zero && !edata_zeroed_get(edata)) { - ehooks_zero(tsdn, ehooks, edata_base_get(edata), - edata_size_get(edata)); + ehooks_zero( + tsdn, ehooks, edata_base_get(edata), edata_size_get(edata)); } return edata; label_err: @@ -788,8 +863,8 @@ extent_alloc_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, extent_gdump_add(tsdn, edata); } } else if (opt_retain && expand_edata == NULL && !guarded) { - edata = extent_grow_retained(tsdn, pac, ehooks, size, - alignment, zero, commit); + edata = extent_grow_retained( + tsdn, pac, ehooks, size, alignment, zero, commit); /* extent_grow_retained() always releases pac->grow_mtx. */ } else { malloc_mutex_unlock(tsdn, &pac->grow_mtx); @@ -805,12 +880,12 @@ extent_coalesce(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, extent_assert_can_coalesce(inner, outer); eset_remove(&ecache->eset, outer); - bool err = extent_merge_impl(tsdn, pac, ehooks, - forward ? inner : outer, forward ? outer : inner, + bool err = extent_merge_impl(tsdn, pac, ehooks, forward ? inner : outer, + forward ? outer : inner, /* holding_core_locks */ true); if (err) { - extent_deactivate_check_state_locked(tsdn, pac, ecache, outer, - extent_state_merging); + extent_deactivate_check_state_locked( + tsdn, pac, ecache, outer, extent_state_merging); } return err; @@ -818,8 +893,10 @@ extent_coalesce(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, static edata_t * extent_try_coalesce_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, - ecache_t *ecache, edata_t *edata, bool *coalesced) { + ecache_t *ecache, edata_t *edata, size_t max_size, bool *coalesced) { assert(!edata_guarded_get(edata)); + assert(coalesced != NULL); + *coalesced = false; /* * We avoid checking / locking inactive neighbors for large size * classes, since they are eagerly coalesced on deallocation which can @@ -836,31 +913,47 @@ extent_try_coalesce_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, /* Try to coalesce forward. */ edata_t *next = emap_try_acquire_edata_neighbor(tsdn, pac->emap, edata, EXTENT_PAI_PAC, ecache->state, /* forward */ true); + size_t max_next_neighbor = max_size > edata_size_get(edata) + ? max_size - edata_size_get(edata) + : 0; if (next != NULL) { - if (!extent_coalesce(tsdn, pac, ehooks, ecache, edata, - next, true)) { - if (ecache->delay_coalesce) { - /* Do minimal coalescing. */ - *coalesced = true; - return edata; + if (edata_size_get(next) > max_next_neighbor) { + emap_release_edata( + tsdn, pac->emap, next, ecache->state); + } else { + if (!extent_coalesce(tsdn, pac, ehooks, ecache, + edata, next, true)) { + if (ecache->delay_coalesce) { + /* Do minimal coalescing. */ + *coalesced = true; + return edata; + } + again = true; } - again = true; } } /* Try to coalesce backward. */ edata_t *prev = emap_try_acquire_edata_neighbor(tsdn, pac->emap, edata, EXTENT_PAI_PAC, ecache->state, /* forward */ false); + size_t max_prev_neighbor = max_size > edata_size_get(edata) + ? max_size - edata_size_get(edata) + : 0; if (prev != NULL) { - if (!extent_coalesce(tsdn, pac, ehooks, ecache, edata, - prev, false)) { - edata = prev; - if (ecache->delay_coalesce) { - /* Do minimal coalescing. */ - *coalesced = true; - return edata; + if (edata_size_get(prev) > max_prev_neighbor) { + emap_release_edata( + tsdn, pac->emap, prev, ecache->state); + } else { + if (!extent_coalesce(tsdn, pac, ehooks, ecache, + edata, prev, false)) { + edata = prev; + if (ecache->delay_coalesce) { + /* Do minimal coalescing. */ + *coalesced = true; + return edata; + } + again = true; } - again = true; } } } while (again); @@ -874,36 +967,33 @@ extent_try_coalesce_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, static edata_t * extent_try_coalesce(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, edata_t *edata, bool *coalesced) { - return extent_try_coalesce_impl(tsdn, pac, ehooks, ecache, edata, - coalesced); + return extent_try_coalesce_impl( + tsdn, pac, ehooks, ecache, edata, SC_LARGE_MAXCLASS, coalesced); } static edata_t * extent_try_coalesce_large(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, - ecache_t *ecache, edata_t *edata, bool *coalesced) { - return extent_try_coalesce_impl(tsdn, pac, ehooks, ecache, edata, - coalesced); + ecache_t *ecache, edata_t *edata, size_t max_size, bool *coalesced) { + return extent_try_coalesce_impl( + tsdn, pac, ehooks, ecache, edata, max_size, coalesced); } /* Purge a single extent to retained / unmapped directly. */ static void -extent_maximally_purge(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, - edata_t *edata) { +extent_maximally_purge( + tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata) { size_t extent_size = edata_size_get(edata); extent_dalloc_wrapper(tsdn, pac, ehooks, edata); if (config_stats) { /* Update stats accordingly. */ LOCKEDINT_MTX_LOCK(tsdn, *pac->stats_mtx); - locked_inc_u64(tsdn, - LOCKEDINT_MTX(*pac->stats_mtx), + locked_inc_u64(tsdn, LOCKEDINT_MTX(*pac->stats_mtx), &pac->stats->decay_dirty.nmadvise, 1); - locked_inc_u64(tsdn, - LOCKEDINT_MTX(*pac->stats_mtx), - &pac->stats->decay_dirty.purged, - extent_size >> LG_PAGE); + locked_inc_u64(tsdn, LOCKEDINT_MTX(*pac->stats_mtx), + &pac->stats->decay_dirty.purged, extent_size >> LG_PAGE); LOCKEDINT_MTX_UNLOCK(tsdn, *pac->stats_mtx); - atomic_fetch_sub_zu(&pac->stats->pac_mapped, extent_size, - ATOMIC_RELAXED); + atomic_fetch_sub_zu( + &pac->stats->pac_mapped, extent_size, ATOMIC_RELAXED); } } @@ -914,9 +1004,9 @@ extent_maximally_purge(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, void extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, edata_t *edata) { - assert((ecache->state != extent_state_dirty && - ecache->state != extent_state_muzzy) || - !edata_zeroed_get(edata)); + assert((ecache->state != extent_state_dirty + && ecache->state != extent_state_muzzy) + || !edata_zeroed_get(edata)); malloc_mutex_lock(tsdn, &ecache->mtx); @@ -926,19 +1016,49 @@ extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, goto label_skip_coalesce; } if (!ecache->delay_coalesce) { - edata = extent_try_coalesce(tsdn, pac, ehooks, ecache, edata, - NULL); + bool coalesced_unused; + edata = extent_try_coalesce( + tsdn, pac, ehooks, ecache, edata, &coalesced_unused); } else if (edata_size_get(edata) >= SC_LARGE_MINCLASS) { assert(ecache == &pac->ecache_dirty); /* Always coalesce large extents eagerly. */ - bool coalesced; + /** + * Maximum size limit (max_size) for large extents waiting to be coalesced + * in dirty ecache. + * + * When set to a non-zero value, this parameter restricts the maximum size + * of large extents after coalescing. If the combined size of two extents + * would exceed this threshold, the coalescing operation is skipped. + * + * This improves dirty ecache reuse efficiency by: + * - Maintaining appropriately sized extents that match common allocation requests + * - Limiting large extent coalescence to prevent overly large extents that are + * less likely to be reused efficiently + * - Setting lg_max_coalesce for large extent merging scenarios, similar to how + * lg_max_fit is used during extent reuse + * + * Note that during extent decay/purge operations, no coalescing restrictions + * are applied to dirty ecache despite the delay_coalesce setting. This ensures + * that while improving dirty ecache reuse efficiency, we don't compromise + * the final coalescing that happens during the transition from dirty ecache + * to muzzy/retained ecache states. + */ + unsigned lg_max_coalesce = (unsigned) + opt_lg_extent_max_active_fit; + size_t edata_size = edata_size_get(edata); + size_t max_size = (SC_LARGE_MAXCLASS >> lg_max_coalesce) + > edata_size + ? (edata_size << lg_max_coalesce) + : SC_LARGE_MAXCLASS; + bool coalesced; do { assert(edata_state_get(edata) == extent_state_active); edata = extent_try_coalesce_large(tsdn, pac, ehooks, - ecache, edata, &coalesced); + ecache, edata, max_size, &coalesced); } while (coalesced); - if (edata_size_get(edata) >= - atomic_load_zu(&pac->oversize_threshold, ATOMIC_RELAXED) + if (edata_size_get(edata) >= atomic_load_zu( + &pac->oversize_threshold, ATOMIC_RELAXED) + && !background_thread_enabled() && extent_may_force_decay(pac)) { /* Shortcut to purge the oversize extent eagerly. */ malloc_mutex_unlock(tsdn, &ecache->mtx); @@ -953,10 +1073,9 @@ label_skip_coalesce: } void -extent_dalloc_gap(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, - edata_t *edata) { - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); +extent_dalloc_gap(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata) { + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); if (extent_register(tsdn, pac, edata)) { edata_cache_put(tsdn, pac->edata_cache, edata); @@ -966,14 +1085,14 @@ extent_dalloc_gap(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, } static bool -extent_dalloc_wrapper_try(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, - edata_t *edata) { +extent_dalloc_wrapper_try( + tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata) { bool err; assert(edata_base_get(edata) != NULL); assert(edata_size_get(edata) != 0); - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); edata_addr_set(edata, edata_base_get(edata)); @@ -989,8 +1108,8 @@ extent_dalloc_wrapper_try(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, } edata_t * -extent_alloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, - void *new_addr, size_t size, size_t alignment, bool zero, bool *commit, +extent_alloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, void *new_addr, + size_t size, size_t alignment, bool zero, bool *commit, bool growing_retained) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, growing_retained ? 1 : 0); @@ -1000,14 +1119,14 @@ extent_alloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, return NULL; } size_t palignment = ALIGNMENT_CEILING(alignment, PAGE); - void *addr = ehooks_alloc(tsdn, ehooks, new_addr, size, palignment, - &zero, commit); + void *addr = ehooks_alloc( + tsdn, ehooks, new_addr, size, palignment, &zero, commit); if (addr == NULL) { edata_cache_put(tsdn, pac->edata_cache, edata); return NULL; } - edata_init(edata, ecache_ind_get(&pac->ecache_dirty), addr, - size, /* slab */ false, SC_NSIZES, extent_sn_next(pac), + edata_init(edata, ecache_ind_get(&pac->ecache_dirty), addr, size, + /* slab */ false, SC_NSIZES, extent_sn_next(pac), extent_state_active, zero, *commit, EXTENT_PAI_PAC, opt_retain ? EXTENT_IS_HEAD : EXTENT_NOT_HEAD); /* @@ -1024,20 +1143,43 @@ extent_alloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, return edata; } +static void +extent_dalloc_wrapper_finish( + tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata) { + if (config_prof) { + extent_gdump_sub(tsdn, edata); + } + extent_record(tsdn, pac, ehooks, &pac->ecache_retained, edata); +} + void -extent_dalloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, - edata_t *edata) { +extent_dalloc_wrapper_purged( + tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata) { assert(edata_pai_get(edata) == EXTENT_PAI_PAC); - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); + + /* Verify that will not go down the dalloc / munmap route. */ + assert(ehooks_dalloc_will_fail(ehooks)); + + edata_zeroed_set(edata, true); + extent_dalloc_wrapper_finish(tsdn, pac, ehooks, edata); +} + +void +extent_dalloc_wrapper( + tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata) { + assert(edata_pai_get(edata) == EXTENT_PAI_PAC); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); /* Avoid calling the default extent_dalloc unless have to. */ if (!ehooks_dalloc_will_fail(ehooks)) { /* Remove guard pages for dalloc / unmap. */ if (edata_guarded_get(edata)) { assert(ehooks_are_default(ehooks)); - san_unguard_pages_two_sided(tsdn, ehooks, edata, - pac->emap); + san_unguard_pages_two_sided( + tsdn, ehooks, edata, pac->emap); } /* * Deregister first to avoid a race with other allocating @@ -1054,38 +1196,34 @@ extent_dalloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, bool zeroed; if (!edata_committed_get(edata)) { zeroed = true; - } else if (!extent_decommit_wrapper(tsdn, ehooks, edata, 0, - edata_size_get(edata))) { + } else if (!extent_decommit_wrapper( + tsdn, ehooks, edata, 0, edata_size_get(edata))) { zeroed = true; } else if (!ehooks_purge_forced(tsdn, ehooks, edata_base_get(edata), - edata_size_get(edata), 0, edata_size_get(edata))) { + edata_size_get(edata), 0, edata_size_get(edata))) { zeroed = true; - } else if (edata_state_get(edata) == extent_state_muzzy || - !ehooks_purge_lazy(tsdn, ehooks, edata_base_get(edata), - edata_size_get(edata), 0, edata_size_get(edata))) { + } else if (edata_state_get(edata) == extent_state_muzzy + || !ehooks_purge_lazy(tsdn, ehooks, edata_base_get(edata), + edata_size_get(edata), 0, edata_size_get(edata))) { zeroed = false; } else { zeroed = false; } edata_zeroed_set(edata, zeroed); - if (config_prof) { - extent_gdump_sub(tsdn, edata); - } - - extent_record(tsdn, pac, ehooks, &pac->ecache_retained, edata); + extent_dalloc_wrapper_finish(tsdn, pac, ehooks, edata); } void -extent_destroy_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, - edata_t *edata) { +extent_destroy_wrapper( + tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata) { assert(edata_base_get(edata) != NULL); assert(edata_size_get(edata) != 0); extent_state_t state = edata_state_get(edata); assert(state == extent_state_retained || state == extent_state_active); assert(emap_edata_is_acquired(tsdn, pac->emap, edata)); - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); if (edata_guarded_get(edata)) { assert(opt_retain); @@ -1111,18 +1249,11 @@ extent_commit_impl(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, return err; } -bool -extent_commit_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, - size_t offset, size_t length) { - return extent_commit_impl(tsdn, ehooks, edata, offset, length, - /* growing_retained */ false); -} - -bool +static bool extent_decommit_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, size_t offset, size_t length) { - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); bool err = ehooks_decommit(tsdn, ehooks, edata_base_get(edata), edata_size_get(edata), offset, length); edata_committed_set(edata, edata_committed_get(edata) && err); @@ -1142,8 +1273,8 @@ extent_purge_lazy_impl(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, bool extent_purge_lazy_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, size_t offset, size_t length) { - return extent_purge_lazy_impl(tsdn, ehooks, edata, offset, - length, false); + return extent_purge_lazy_impl( + tsdn, ehooks, edata, offset, length, false); } static bool @@ -1159,8 +1290,8 @@ extent_purge_forced_impl(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, bool extent_purge_forced_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, size_t offset, size_t length) { - return extent_purge_forced_impl(tsdn, ehooks, edata, offset, length, - false); + return extent_purge_forced_impl( + tsdn, ehooks, edata, offset, length, false); } /* @@ -1171,16 +1302,16 @@ extent_purge_forced_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, * and returns the trail (except in case of error). */ static edata_t * -extent_split_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, - edata_t *edata, size_t size_a, size_t size_b, bool holding_core_locks) { +extent_split_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata, + size_t size_a, size_t size_b, bool holding_core_locks) { assert(edata_size_get(edata) == size_a + size_b); /* Only the shrink path may split w/o holding core locks. */ if (holding_core_locks) { witness_assert_positive_depth_to_rank( tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE); } else { - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); } if (ehooks_split_will_fail(ehooks)) { @@ -1193,13 +1324,13 @@ extent_split_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, } edata_init(trail, edata_arena_ind_get(edata), - (void *)((uintptr_t)edata_base_get(edata) + size_a), size_b, + (void *)((byte_t *)edata_base_get(edata) + size_a), size_b, /* slab */ false, SC_NSIZES, edata_sn_get(edata), edata_state_get(edata), edata_zeroed_get(edata), edata_committed_get(edata), EXTENT_PAI_PAC, EXTENT_NOT_HEAD); emap_prepare_t prepare; - bool err = emap_split_prepare(tsdn, pac->emap, &prepare, edata, - size_a, trail, size_b); + bool err = emap_split_prepare( + tsdn, pac->emap, &prepare, edata, size_a, trail, size_b); if (err) { goto label_error_b; } @@ -1221,8 +1352,8 @@ extent_split_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, } edata_size_set(edata, size_a); - emap_split_commit(tsdn, pac->emap, &prepare, edata, size_a, trail, - size_b); + emap_split_commit( + tsdn, pac->emap, &prepare, edata, size_a, trail, size_b); return trail; label_error_b: @@ -1234,8 +1365,8 @@ label_error_a: edata_t * extent_split_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata, size_t size_a, size_t size_b, bool holding_core_locks) { - return extent_split_impl(tsdn, pac, ehooks, edata, size_a, size_b, - holding_core_locks); + return extent_split_impl( + tsdn, pac, ehooks, edata, size_a, size_b, holding_core_locks); } static bool @@ -1246,8 +1377,8 @@ extent_merge_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *a, witness_assert_positive_depth_to_rank( tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE); } else { - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); } assert(edata_base_get(a) < edata_base_get(b)); @@ -1272,12 +1403,13 @@ extent_merge_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *a, emap_prepare_t prepare; emap_merge_prepare(tsdn, pac->emap, &prepare, a, b); - assert(edata_state_get(a) == extent_state_active || - edata_state_get(a) == extent_state_merging); + assert(edata_state_get(a) == extent_state_active + || edata_state_get(a) == extent_state_merging); edata_state_set(a, extent_state_active); edata_size_set(a, edata_size_get(a) + edata_size_get(b)); - edata_sn_set(a, (edata_sn_get(a) < edata_sn_get(b)) ? - edata_sn_get(a) : edata_sn_get(b)); + edata_sn_set(a, + (edata_sn_get(a) < edata_sn_get(b)) ? edata_sn_get(a) + : edata_sn_get(b)); edata_zeroed_set(a, edata_zeroed_get(a) && edata_zeroed_get(b)); emap_merge_commit(tsdn, pac->emap, &prepare, a, b); @@ -1288,26 +1420,26 @@ extent_merge_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *a, } bool -extent_merge_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, - edata_t *a, edata_t *b) { +extent_merge_wrapper( + tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *a, edata_t *b) { return extent_merge_impl(tsdn, pac, ehooks, a, b, /* holding_core_locks */ false); } bool -extent_commit_zero(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, - bool commit, bool zero, bool growing_retained) { +extent_commit_zero(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, bool commit, + bool zero, bool growing_retained) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, growing_retained ? 1 : 0); if (commit && !edata_committed_get(edata)) { if (extent_commit_impl(tsdn, ehooks, edata, 0, - edata_size_get(edata), growing_retained)) { + edata_size_get(edata), growing_retained)) { return true; } } if (zero && !edata_zeroed_get(edata)) { - void *addr = edata_base_get(edata); + void *addr = edata_base_get(edata); size_t size = edata_size_get(edata); ehooks_zero(tsdn, ehooks, addr, size); } diff --git a/dep/jemalloc/src/extent_dss.c b/dep/jemalloc/src/extent_dss.c index 9a35bacfb..c7c34207e 100644 --- a/dep/jemalloc/src/extent_dss.c +++ b/dep/jemalloc/src/extent_dss.c @@ -8,14 +8,13 @@ /******************************************************************************/ /* Data. */ -const char *opt_dss = DSS_DEFAULT; +/* NOLINTNEXTLINE(performance-no-int-to-ptr) */ +#define SBRK_INVALID ((void *)-1) -const char *dss_prec_names[] = { - "disabled", - "primary", - "secondary", - "N/A" -}; +const char *opt_dss = DSS_DEFAULT; + +const char *const dss_prec_names[] = { + "disabled", "primary", "secondary", "N/A"}; /* * Current dss precedence default, used when creating new arenas. NB: This is @@ -23,17 +22,16 @@ const char *dss_prec_names[] = { * guarantee that sizeof(dss_prec_t) is the same as sizeof(unsigned), and we use * atomic operations to synchronize the setting. */ -static atomic_u_t dss_prec_default = ATOMIC_INIT( - (unsigned)DSS_PREC_DEFAULT); +static atomic_u_t dss_prec_default = ATOMIC_INIT((unsigned)DSS_PREC_DEFAULT); /* Base address of the DSS. */ -static void *dss_base; +static void *dss_base; /* Atomic boolean indicating whether a thread is currently extending DSS. */ -static atomic_b_t dss_extending; +static atomic_b_t dss_extending; /* Atomic boolean indicating whether the DSS is exhausted. */ -static atomic_b_t dss_exhausted; +static atomic_b_t dss_exhausted; /* Atomic current upper limit on DSS addresses. */ -static atomic_p_t dss_max; +static atomic_p_t dss_max; /******************************************************************************/ @@ -73,7 +71,7 @@ extent_dss_extending_start(void) { while (true) { bool expected = false; if (atomic_compare_exchange_weak_b(&dss_extending, &expected, - true, ATOMIC_ACQ_REL, ATOMIC_RELAXED)) { + true, ATOMIC_ACQ_REL, ATOMIC_RELAXED)) { break; } spin_adaptive(&spinner); @@ -94,7 +92,7 @@ extent_dss_max_update(void *new_addr) { * up to date. */ void *max_cur = extent_dss_sbrk(0); - if (max_cur == (void *)-1) { + if (max_cur == SBRK_INVALID) { return NULL; } atomic_store_p(&dss_max, max_cur, ATOMIC_RELEASE); @@ -140,51 +138,54 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, goto label_oom; } - bool head_state = opt_retain ? EXTENT_IS_HEAD : - EXTENT_NOT_HEAD; + bool head_state = opt_retain ? EXTENT_IS_HEAD + : EXTENT_NOT_HEAD; /* * Compute how much page-aligned gap space (if any) is * necessary to satisfy alignment. This space can be * recycled for later use. */ - void *gap_addr_page = (void *)(PAGE_CEILING( - (uintptr_t)max_cur)); - void *ret = (void *)ALIGNMENT_CEILING( - (uintptr_t)gap_addr_page, alignment); - size_t gap_size_page = (uintptr_t)ret - - (uintptr_t)gap_addr_page; + void *gap_addr_page = ALIGNMENT_ADDR2CEILING( + max_cur, PAGE); + void *ret = ALIGNMENT_ADDR2CEILING( + gap_addr_page, alignment); + size_t gap_size_page = (uintptr_t)ret + - (uintptr_t)gap_addr_page; if (gap_size_page != 0) { edata_init(gap, arena_ind_get(arena), - gap_addr_page, gap_size_page, false, - SC_NSIZES, extent_sn_next( - &arena->pa_shard.pac), - extent_state_active, false, true, - EXTENT_PAI_PAC, head_state); + gap_addr_page, gap_size_page, + /* slab */ false, + /* szind */ SC_NSIZES, + extent_sn_next(&arena->pa_shard.pac), + extent_state_active, + /* zeroed */ false, + /* committed */ true, + /* pai */ EXTENT_PAI_PAC, head_state); } /* * Compute the address just past the end of the desired * allocation space. */ - void *dss_next = (void *)((uintptr_t)ret + size); - if ((uintptr_t)ret < (uintptr_t)max_cur || - (uintptr_t)dss_next < (uintptr_t)max_cur) { + void *dss_next = (void *)((byte_t *)ret + size); + if ((uintptr_t)ret < (uintptr_t)max_cur + || (uintptr_t)dss_next < (uintptr_t)max_cur) { goto label_oom; /* Wrap-around. */ } /* Compute the increment, including subpage bytes. */ - void *gap_addr_subpage = max_cur; - size_t gap_size_subpage = (uintptr_t)ret - - (uintptr_t)gap_addr_subpage; + void *gap_addr_subpage = max_cur; + size_t gap_size_subpage = (uintptr_t)ret + - (uintptr_t)gap_addr_subpage; intptr_t incr = gap_size_subpage + size; - assert((uintptr_t)max_cur + incr == (uintptr_t)ret + - size); + assert( + (uintptr_t)max_cur + incr == (uintptr_t)ret + size); /* Try to allocate. */ void *dss_prev = extent_dss_sbrk(incr); if (dss_prev == max_cur) { /* Success. */ - atomic_store_p(&dss_max, dss_next, - ATOMIC_RELEASE); + atomic_store_p( + &dss_max, dss_next, ATOMIC_RELEASE); extent_dss_extending_finish(); if (gap_size_page != 0) { @@ -200,17 +201,23 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, *commit = pages_decommit(ret, size); } if (*zero && *commit) { - edata_t edata = {0}; + edata_t edata = {0}; ehooks_t *ehooks = arena_get_ehooks( arena); - edata_init(&edata, - arena_ind_get(arena), ret, size, - size, false, SC_NSIZES, - extent_state_active, false, true, - EXTENT_PAI_PAC, head_state); + edata_init(&edata, arena_ind_get(arena), + ret, size, + /* slab */ false, + /* szind */ SC_NSIZES, + extent_sn_next( + &arena->pa_shard.pac), + extent_state_active, + /* zeroed */ false, + /* committed */ true, + /* pai */ EXTENT_PAI_PAC, + head_state); if (extent_purge_forced_wrapper(tsdn, - ehooks, &edata, 0, size)) { + ehooks, &edata, 0, size)) { memset(ret, 0, size); } } @@ -220,10 +227,10 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, * Failure, whether due to OOM or a race with a raw * sbrk() call from outside the allocator. */ - if (dss_prev == (void *)-1) { + if (dss_prev == SBRK_INVALID) { /* OOM. */ - atomic_store_b(&dss_exhausted, true, - ATOMIC_RELEASE); + atomic_store_b( + &dss_exhausted, true, ATOMIC_RELEASE); goto label_oom; } } @@ -236,16 +243,16 @@ label_oom: static bool extent_in_dss_helper(void *addr, void *max) { - return ((uintptr_t)addr >= (uintptr_t)dss_base && (uintptr_t)addr < - (uintptr_t)max); + return ((uintptr_t)addr >= (uintptr_t)dss_base + && (uintptr_t)addr < (uintptr_t)max); } bool extent_in_dss(void *addr) { cassert(have_dss); - return extent_in_dss_helper(addr, atomic_load_p(&dss_max, - ATOMIC_ACQUIRE)); + return extent_in_dss_helper( + addr, atomic_load_p(&dss_max, ATOMIC_ACQUIRE)); } bool @@ -254,14 +261,14 @@ extent_dss_mergeable(void *addr_a, void *addr_b) { cassert(have_dss); - if ((uintptr_t)addr_a < (uintptr_t)dss_base && (uintptr_t)addr_b < - (uintptr_t)dss_base) { + if ((uintptr_t)addr_a < (uintptr_t)dss_base + && (uintptr_t)addr_b < (uintptr_t)dss_base) { return true; } max = atomic_load_p(&dss_max, ATOMIC_ACQUIRE); - return (extent_in_dss_helper(addr_a, max) == - extent_in_dss_helper(addr_b, max)); + return (extent_in_dss_helper(addr_a, max) + == extent_in_dss_helper(addr_b, max)); } void @@ -270,7 +277,8 @@ extent_dss_boot(void) { dss_base = extent_dss_sbrk(0); atomic_store_b(&dss_extending, false, ATOMIC_RELAXED); - atomic_store_b(&dss_exhausted, dss_base == (void *)-1, ATOMIC_RELAXED); + atomic_store_b( + &dss_exhausted, dss_base == SBRK_INVALID, ATOMIC_RELAXED); atomic_store_p(&dss_max, dss_base, ATOMIC_RELAXED); } diff --git a/dep/jemalloc/src/extent_mmap.c b/dep/jemalloc/src/extent_mmap.c index 5f0ee2d24..d39bddc60 100644 --- a/dep/jemalloc/src/extent_mmap.c +++ b/dep/jemalloc/src/extent_mmap.c @@ -7,7 +7,7 @@ /******************************************************************************/ /* Data. */ -bool opt_retain = +bool opt_retain = #ifdef JEMALLOC_RETAIN true #else @@ -18,8 +18,8 @@ bool opt_retain = /******************************************************************************/ void * -extent_alloc_mmap(void *new_addr, size_t size, size_t alignment, bool *zero, - bool *commit) { +extent_alloc_mmap( + void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit) { assert(alignment == ALIGNMENT_CEILING(alignment, PAGE)); void *ret = pages_map(new_addr, size, alignment, commit); if (ret == NULL) { diff --git a/dep/jemalloc/src/fxp.c b/dep/jemalloc/src/fxp.c index 96585f0a6..faeab207a 100644 --- a/dep/jemalloc/src/fxp.c +++ b/dep/jemalloc/src/fxp.c @@ -83,8 +83,8 @@ fxp_parse(fxp_t *result, const char *str, char **end) { } assert(fractional_part < frac_div); - uint32_t fractional_repr = (uint32_t)( - (fractional_part << 16) / frac_div); + uint32_t fractional_repr = (uint32_t)((fractional_part << 16) + / frac_div); /* Success! */ *result = (integer_part << 16) + fractional_repr; @@ -99,7 +99,7 @@ fxp_print(fxp_t a, char buf[FXP_BUF_SIZE]) { uint32_t integer_part = fxp_round_down(a); uint32_t fractional_part = (a & ((1U << 16) - 1)); - int leading_fraction_zeros = 0; + int leading_fraction_zeros = 0; uint64_t fraction_digits = fractional_part; for (int i = 0; i < FXP_FRACTIONAL_PART_DIGITS; i++) { if (fraction_digits < (1U << 16) @@ -113,12 +113,12 @@ fxp_print(fxp_t a, char buf[FXP_BUF_SIZE]) { fraction_digits /= 10; } - size_t printed = malloc_snprintf(buf, FXP_BUF_SIZE, "%"FMTu32".", - integer_part); + size_t printed = malloc_snprintf( + buf, FXP_BUF_SIZE, "%" FMTu32 ".", integer_part); for (int i = 0; i < leading_fraction_zeros; i++) { buf[printed] = '0'; printed++; } - malloc_snprintf(&buf[printed], FXP_BUF_SIZE - printed, "%"FMTu64, - fraction_digits); + malloc_snprintf( + &buf[printed], FXP_BUF_SIZE - printed, "%" FMTu64, fraction_digits); } diff --git a/dep/jemalloc/src/hook.c b/dep/jemalloc/src/hook.c index 493edbbe5..4270ad607 100644 --- a/dep/jemalloc/src/hook.c +++ b/dep/jemalloc/src/hook.c @@ -9,19 +9,19 @@ typedef struct hooks_internal_s hooks_internal_t; struct hooks_internal_s { hooks_t hooks; - bool in_use; + bool in_use; }; seq_define(hooks_internal_t, hooks) -static atomic_u_t nhooks = ATOMIC_INIT(0); -static seq_hooks_t hooks[HOOK_MAX]; + static atomic_u_t nhooks = ATOMIC_INIT(0); +static seq_hooks_t hooks[HOOK_MAX]; static malloc_mutex_t hooks_mu; bool -hook_boot() { - return malloc_mutex_init(&hooks_mu, "hooks", WITNESS_RANK_HOOK, - malloc_mutex_rank_exclusive); +hook_boot(void) { + return malloc_mutex_init( + &hooks_mu, "hooks", WITNESS_RANK_HOOK, malloc_mutex_rank_exclusive); } static void * @@ -84,23 +84,21 @@ hook_remove(tsdn_t *tsdn, void *opaque) { malloc_mutex_unlock(tsdn, &hooks_mu); } -#define FOR_EACH_HOOK_BEGIN(hooks_internal_ptr) \ -for (int for_each_hook_counter = 0; \ - for_each_hook_counter < HOOK_MAX; \ - for_each_hook_counter++) { \ - bool for_each_hook_success = seq_try_load_hooks( \ - (hooks_internal_ptr), &hooks[for_each_hook_counter]); \ - if (!for_each_hook_success) { \ - continue; \ - } \ - if (!(hooks_internal_ptr)->in_use) { \ - continue; \ - } -#define FOR_EACH_HOOK_END \ -} +#define FOR_EACH_HOOK_BEGIN(hooks_internal_ptr) \ + for (int for_each_hook_counter = 0; for_each_hook_counter < HOOK_MAX; \ + for_each_hook_counter++) { \ + bool for_each_hook_success = seq_try_load_hooks( \ + (hooks_internal_ptr), &hooks[for_each_hook_counter]); \ + if (!for_each_hook_success) { \ + continue; \ + } \ + if (!(hooks_internal_ptr)->in_use) { \ + continue; \ + } +#define FOR_EACH_HOOK_END } static bool * -hook_reentrantp() { +hook_reentrantp(void) { /* * We prevent user reentrancy within hooks. This is basically just a * thread-local bool that triggers an early-exit. @@ -129,26 +127,25 @@ hook_reentrantp() { * untouched. */ static bool in_hook_global = true; - tsdn_t *tsdn = tsdn_fetch(); - bool *in_hook = tsdn_in_hookp_get(tsdn); - if (in_hook!= NULL) { + tsdn_t *tsdn = tsdn_fetch(); + bool *in_hook = tsdn_in_hookp_get(tsdn); + if (in_hook != NULL) { return in_hook; } return &in_hook_global; } -#define HOOK_PROLOGUE \ - if (likely(atomic_load_u(&nhooks, ATOMIC_RELAXED) == 0)) { \ - return; \ - } \ - bool *in_hook = hook_reentrantp(); \ - if (*in_hook) { \ - return; \ - } \ +#define HOOK_PROLOGUE \ + if (likely(atomic_load_u(&nhooks, ATOMIC_RELAXED) == 0)) { \ + return; \ + } \ + bool *in_hook = hook_reentrantp(); \ + if (*in_hook) { \ + return; \ + } \ *in_hook = true; -#define HOOK_EPILOGUE \ - *in_hook = false; +#define HOOK_EPILOGUE *in_hook = false; void hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw, @@ -157,10 +154,10 @@ hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw, hooks_internal_t hook; FOR_EACH_HOOK_BEGIN(&hook) - hook_alloc h = hook.hooks.alloc_hook; - if (h != NULL) { - h(hook.hooks.extra, type, result, result_raw, args_raw); - } + hook_alloc h = hook.hooks.alloc_hook; + if (h != NULL) { + h(hook.hooks.extra, type, result, result_raw, args_raw); + } FOR_EACH_HOOK_END HOOK_EPILOGUE @@ -171,10 +168,10 @@ hook_invoke_dalloc(hook_dalloc_t type, void *address, uintptr_t args_raw[3]) { HOOK_PROLOGUE hooks_internal_t hook; FOR_EACH_HOOK_BEGIN(&hook) - hook_dalloc h = hook.hooks.dalloc_hook; - if (h != NULL) { - h(hook.hooks.extra, type, address, args_raw); - } + hook_dalloc h = hook.hooks.dalloc_hook; + if (h != NULL) { + h(hook.hooks.extra, type, address, args_raw); + } FOR_EACH_HOOK_END HOOK_EPILOGUE } @@ -185,11 +182,11 @@ hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize, HOOK_PROLOGUE hooks_internal_t hook; FOR_EACH_HOOK_BEGIN(&hook) - hook_expand h = hook.hooks.expand_hook; - if (h != NULL) { - h(hook.hooks.extra, type, address, old_usize, new_usize, - result_raw, args_raw); - } + hook_expand h = hook.hooks.expand_hook; + if (h != NULL) { + h(hook.hooks.extra, type, address, old_usize, new_usize, + result_raw, args_raw); + } FOR_EACH_HOOK_END HOOK_EPILOGUE } diff --git a/dep/jemalloc/src/hpa.c b/dep/jemalloc/src/hpa.c index 7e2aeba0c..ce6ece2bd 100644 --- a/dep/jemalloc/src/hpa.c +++ b/dep/jemalloc/src/hpa.c @@ -2,29 +2,38 @@ #include "jemalloc/internal/jemalloc_internal_includes.h" #include "jemalloc/internal/hpa.h" +#include "jemalloc/internal/hpa_utils.h" #include "jemalloc/internal/fb.h" #include "jemalloc/internal/witness.h" - -#define HPA_EDEN_SIZE (128 * HUGEPAGE) +#include "jemalloc/internal/jemalloc_probe.h" static edata_t *hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero, bool guarded, bool frequent_reuse, bool *deferred_work_generated); -static size_t hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size, - size_t nallocs, edata_list_active_t *results, bool *deferred_work_generated); -static bool hpa_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, - size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated); -static bool hpa_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, - size_t old_size, size_t new_size, bool *deferred_work_generated); -static void hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata, - bool *deferred_work_generated); -static void hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self, - edata_list_active_t *list, bool *deferred_work_generated); +static bool hpa_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, + size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated); +static bool hpa_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, + size_t old_size, size_t new_size, bool *deferred_work_generated); +static void hpa_dalloc( + tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated); static uint64_t hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self); +static void hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self, + edata_list_active_t *list, bool *deferred_work_generated); + +const char *const hpa_hugify_style_names[] = {"auto", "none", "eager", "lazy"}; + +bool opt_experimental_hpa_start_huge_if_thp_always = true; +bool opt_experimental_hpa_enforce_hugify = false; + bool -hpa_supported() { +hpa_hugepage_size_exceeds_limit(void) { + return HUGEPAGE > HUGEPAGE_MAX_EXPECTED_SIZE; +} + +bool +hpa_supported(void) { #ifdef _WIN32 /* * At least until the API and implementation is somewhat settled, we @@ -37,7 +46,7 @@ hpa_supported() { return false; } /* - * We fundamentally rely on a address-space-hungry growth strategy for + * We fundamentally rely on an address-space-hungry growth strategy for * hugepages. */ if (LG_SIZEOF_PTR != 3) { @@ -50,6 +59,10 @@ hpa_supported() { if (HUGEPAGE_PAGES == 1) { return false; } + /* As mentioned in pages.h, do not support If HUGEPAGE is too large. */ + if (hpa_hugepage_size_exceeds_limit()) { + return false; + } return true; } @@ -59,125 +72,9 @@ hpa_do_consistency_checks(hpa_shard_t *shard) { } bool -hpa_central_init(hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks) { - /* malloc_conf processing should have filtered out these cases. */ - assert(hpa_supported()); - bool err; - err = malloc_mutex_init(¢ral->grow_mtx, "hpa_central_grow", - WITNESS_RANK_HPA_CENTRAL_GROW, malloc_mutex_rank_exclusive); - if (err) { - return true; - } - err = malloc_mutex_init(¢ral->mtx, "hpa_central", - WITNESS_RANK_HPA_CENTRAL, malloc_mutex_rank_exclusive); - if (err) { - return true; - } - central->base = base; - central->eden = NULL; - central->eden_len = 0; - central->age_counter = 0; - central->hooks = *hooks; - return false; -} - -static hpdata_t * -hpa_alloc_ps(tsdn_t *tsdn, hpa_central_t *central) { - return (hpdata_t *)base_alloc(tsdn, central->base, sizeof(hpdata_t), - CACHELINE); -} - -hpdata_t * -hpa_central_extract(tsdn_t *tsdn, hpa_central_t *central, size_t size, - bool *oom) { - /* Don't yet support big allocations; these should get filtered out. */ - assert(size <= HUGEPAGE); - /* - * Should only try to extract from the central allocator if the local - * shard is exhausted. We should hold the grow_mtx on that shard. - */ - witness_assert_positive_depth_to_rank( - tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_HPA_SHARD_GROW); - - malloc_mutex_lock(tsdn, ¢ral->grow_mtx); - *oom = false; - - hpdata_t *ps = NULL; - - /* Is eden a perfect fit? */ - if (central->eden != NULL && central->eden_len == HUGEPAGE) { - ps = hpa_alloc_ps(tsdn, central); - if (ps == NULL) { - *oom = true; - malloc_mutex_unlock(tsdn, ¢ral->grow_mtx); - return NULL; - } - hpdata_init(ps, central->eden, central->age_counter++); - central->eden = NULL; - central->eden_len = 0; - malloc_mutex_unlock(tsdn, ¢ral->grow_mtx); - return ps; - } - - /* - * We're about to try to allocate from eden by splitting. If eden is - * NULL, we have to allocate it too. Otherwise, we just have to - * allocate an edata_t for the new psset. - */ - if (central->eden == NULL) { - /* - * During development, we're primarily concerned with systems - * with overcommit. Eventually, we should be more careful here. - */ - bool commit = true; - /* Allocate address space, bailing if we fail. */ - void *new_eden = pages_map(NULL, HPA_EDEN_SIZE, HUGEPAGE, - &commit); - if (new_eden == NULL) { - *oom = true; - malloc_mutex_unlock(tsdn, ¢ral->grow_mtx); - return NULL; - } - ps = hpa_alloc_ps(tsdn, central); - if (ps == NULL) { - pages_unmap(new_eden, HPA_EDEN_SIZE); - *oom = true; - malloc_mutex_unlock(tsdn, ¢ral->grow_mtx); - return NULL; - } - central->eden = new_eden; - central->eden_len = HPA_EDEN_SIZE; - } else { - /* Eden is already nonempty; only need an edata for ps. */ - ps = hpa_alloc_ps(tsdn, central); - if (ps == NULL) { - *oom = true; - malloc_mutex_unlock(tsdn, ¢ral->grow_mtx); - return NULL; - } - } - assert(ps != NULL); - assert(central->eden != NULL); - assert(central->eden_len > HUGEPAGE); - assert(central->eden_len % HUGEPAGE == 0); - assert(HUGEPAGE_ADDR2BASE(central->eden) == central->eden); - - hpdata_init(ps, central->eden, central->age_counter++); - - char *eden_char = (char *)central->eden; - eden_char += HUGEPAGE; - central->eden = (void *)eden_char; - central->eden_len -= HUGEPAGE; - - malloc_mutex_unlock(tsdn, ¢ral->grow_mtx); - - return ps; -} - -bool -hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap, - base_t *base, edata_cache_t *edata_cache, unsigned ind, - const hpa_shard_opts_t *opts) { +hpa_shard_init(tsdn_t *tsdn, hpa_shard_t *shard, hpa_central_t *central, + emap_t *emap, base_t *base, edata_cache_t *edata_cache, unsigned ind, + const hpa_shard_opts_t *opts, const sec_opts_t *sec_opts) { /* malloc_conf processing should have filtered out these cases. */ assert(hpa_supported()); bool err; @@ -205,10 +102,12 @@ hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap, shard->npending_purge = 0; nstime_init_zero(&shard->last_purge); + nstime_init_zero(&shard->last_time_work_attempted); shard->stats.npurge_passes = 0; shard->stats.npurges = 0; shard->stats.nhugifies = 0; + shard->stats.nhugify_failures = 0; shard->stats.ndehugifies = 0; /* @@ -217,13 +116,16 @@ hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap, * operating on corrupted data. */ shard->pai.alloc = &hpa_alloc; - shard->pai.alloc_batch = &hpa_alloc_batch; shard->pai.expand = &hpa_expand; shard->pai.shrink = &hpa_shrink; shard->pai.dalloc = &hpa_dalloc; - shard->pai.dalloc_batch = &hpa_dalloc_batch; shard->pai.time_until_deferred_work = &hpa_time_until_deferred_work; + err = sec_init(tsdn, &shard->sec, base, sec_opts); + if (err) { + return true; + } + hpa_do_consistency_checks(shard); return false; @@ -232,28 +134,30 @@ hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap, /* * Note that the stats functions here follow the usual stats naming conventions; * "merge" obtains the stats from some live object of instance, while "accum" - * only combines the stats from one stats objet to another. Hence the lack of + * only combines the stats from one stats object to another. Hence the lack of * locking here. */ static void -hpa_shard_nonderived_stats_accum(hpa_shard_nonderived_stats_t *dst, - hpa_shard_nonderived_stats_t *src) { +hpa_shard_nonderived_stats_accum( + hpa_shard_nonderived_stats_t *dst, hpa_shard_nonderived_stats_t *src) { dst->npurge_passes += src->npurge_passes; dst->npurges += src->npurges; dst->nhugifies += src->nhugifies; + dst->nhugify_failures += src->nhugify_failures; dst->ndehugifies += src->ndehugifies; } void hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src) { psset_stats_accum(&dst->psset_stats, &src->psset_stats); - hpa_shard_nonderived_stats_accum(&dst->nonderived_stats, - &src->nonderived_stats); + hpa_shard_nonderived_stats_accum( + &dst->nonderived_stats, &src->nonderived_stats); + sec_stats_accum(&dst->secstats, &src->secstats); } void -hpa_shard_stats_merge(tsdn_t *tsdn, hpa_shard_t *shard, - hpa_shard_stats_t *dst) { +hpa_shard_stats_merge( + tsdn_t *tsdn, hpa_shard_t *shard, hpa_shard_stats_t *dst) { hpa_do_consistency_checks(shard); malloc_mutex_lock(tsdn, &shard->grow_mtx); @@ -262,6 +166,36 @@ hpa_shard_stats_merge(tsdn_t *tsdn, hpa_shard_t *shard, hpa_shard_nonderived_stats_accum(&dst->nonderived_stats, &shard->stats); malloc_mutex_unlock(tsdn, &shard->mtx); malloc_mutex_unlock(tsdn, &shard->grow_mtx); + + sec_stats_merge(tsdn, &shard->sec, &dst->secstats); +} + +static bool +hpa_is_hugify_eager(hpa_shard_t *shard) { + return shard->opts.hugify_style == hpa_hugify_style_eager; +} + +static bool +hpa_is_hugify_lazy(hpa_shard_t *shard) { + /* When hugify_sync==true we also set/unset HG bit manually */ + return shard->opts.hugify_style == hpa_hugify_style_lazy + || shard->opts.hugify_sync; +} + +static bool +hpa_is_hugify_none(hpa_shard_t *shard) { + return shard->opts.hugify_style == hpa_hugify_style_none; +} + +/* + * Experimentation has shown that when we are purging only HUGEPAGE ranges and + * hugifying eagerly (or thp enabled=always) we get huge pages more often. This + * helps us have more realistic accounting. + */ +static bool +hpa_should_assume_huge(hpa_shard_t *shard, const hpdata_t *ps) { + return (hpa_is_hugify_eager(shard) || hpa_is_hugify_none(shard)) + && hpdata_purged_when_empty_and_huge_get(ps); } static bool @@ -275,6 +209,20 @@ hpa_good_hugification_candidate(hpa_shard_t *shard, hpdata_t *ps) { >= shard->opts.hugification_threshold; } +static bool +hpa_good_purge_candidate(hpa_shard_t *shard, hpdata_t *ps) { + if (shard->opts.dirty_mult == (fxp_t)-1) { + /* No purging. */ + return false; + } + size_t ndirty = hpdata_ndirty_get(ps); + /* Empty pages are good candidate for purging. */ + if (ndirty > 0 && hpdata_empty(ps)) { + return true; + } + return ndirty * PAGE >= shard->opts.purge_threshold; +} + static size_t hpa_adjusted_ndirty(tsdn_t *tsdn, hpa_shard_t *shard) { malloc_mutex_assert_owner(tsdn, &shard->mtx); @@ -287,8 +235,8 @@ hpa_ndirty_max(tsdn_t *tsdn, hpa_shard_t *shard) { if (shard->opts.dirty_mult == (fxp_t)-1) { return (size_t)-1; } - return fxp_mul_frac(psset_nactive(&shard->psset), - shard->opts.dirty_mult); + return fxp_mul_frac( + psset_nactive(&shard->psset), shard->opts.dirty_mult); } static bool @@ -299,12 +247,21 @@ hpa_hugify_blocked_by_ndirty(tsdn_t *tsdn, hpa_shard_t *shard) { return false; } return hpa_adjusted_ndirty(tsdn, shard) - + hpdata_nretained_get(to_hugify) > hpa_ndirty_max(tsdn, shard); + + hpdata_nretained_get(to_hugify) + > hpa_ndirty_max(tsdn, shard); } static bool hpa_should_purge(tsdn_t *tsdn, hpa_shard_t *shard) { malloc_mutex_assert_owner(tsdn, &shard->mtx); + /* + * The page that is purgable may be delayed, but we just want to know + * if there is a need for bg thread to wake up in the future. + */ + hpdata_t *ps = psset_pick_purge(&shard->psset, NULL); + if (ps == NULL) { + return false; + } if (hpa_adjusted_ndirty(tsdn, shard) > hpa_ndirty_max(tsdn, shard)) { return true; } @@ -315,8 +272,22 @@ hpa_should_purge(tsdn_t *tsdn, hpa_shard_t *shard) { } static void -hpa_update_purge_hugify_eligibility(tsdn_t *tsdn, hpa_shard_t *shard, - hpdata_t *ps) { +hpa_assume_huge(tsdn_t *tsdn, hpa_shard_t *shard, hpdata_t *ps) { + malloc_mutex_assert_owner(tsdn, &shard->mtx); + + assert(hpa_should_assume_huge(shard, ps)); + if (hpdata_huge_get(ps) || hpdata_empty(ps)) { + return; + } + + if (hpdata_ntouched_get(ps) != HUGEPAGE_PAGES) { + hpdata_hugify(ps); + } +} + +static void +hpa_update_purge_hugify_eligibility( + tsdn_t *tsdn, hpa_shard_t *shard, hpdata_t *ps) { malloc_mutex_assert_owner(tsdn, &shard->mtx); if (hpdata_changing_state_get(ps)) { hpdata_purge_allowed_set(ps, false); @@ -345,26 +316,41 @@ hpa_update_purge_hugify_eligibility(tsdn_t *tsdn, hpa_shard_t *shard, * allocator's end at all; we just try to pack allocations in a * hugepage-friendly manner and let the OS hugify in the background. */ - hpdata_purge_allowed_set(ps, hpdata_ndirty_get(ps) > 0); - if (hpa_good_hugification_candidate(shard, ps) + if (hpa_should_assume_huge(shard, ps)) { + /* Assume it is huge without the need to madvise */ + hpa_assume_huge(tsdn, shard, ps); + } + if ((hpa_is_hugify_lazy(shard) || opt_experimental_hpa_enforce_hugify) + && hpa_good_hugification_candidate(shard, ps) && !hpdata_huge_get(ps)) { nstime_t now; shard->central->hooks.curtime(&now, /* first_reading */ true); hpdata_allow_hugify(ps, now); } + bool purgable = hpa_good_purge_candidate(shard, ps); + if (purgable && !hpdata_purge_allowed_get(ps) + && (shard->opts.min_purge_delay_ms > 0)) { + nstime_t now; + uint64_t delayns = shard->opts.min_purge_delay_ms * 1000 * 1000; + shard->central->hooks.curtime(&now, /* first_reading */ true); + nstime_iadd(&now, delayns); + hpdata_time_purge_allowed_set(ps, &now); + } + hpdata_purge_allowed_set(ps, purgable); + /* * Once a hugepage has become eligible for hugification, we don't mark * it as ineligible just because it stops meeting the criteria (this * could lead to situations where a hugepage that spends most of its * time meeting the criteria never quite getting hugified if there are * intervening deallocations). The idea is that the hugification delay - * will allow them to get purged, reseting their "hugify-allowed" bit. + * will allow them to get purged, resetting their "hugify-allowed" bit. * If they don't get purged, then the hugification isn't hurting and * might help. As an exception, we don't hugify hugepages that are now * empty; it definitely doesn't help there until the hugepage gets * reused, which is likely not for a while. */ - if (hpdata_nactive_get(ps) == 0) { + if (hpdata_nactive_get(ps) == 0 && !hpa_should_assume_huge(shard, ps)) { hpdata_disallow_hugify(ps); } } @@ -376,14 +362,26 @@ hpa_shard_has_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) { return to_hugify != NULL || hpa_should_purge(tsdn, shard); } -/* Returns whether or not we purged anything. */ -static bool -hpa_try_purge(tsdn_t *tsdn, hpa_shard_t *shard) { - malloc_mutex_assert_owner(tsdn, &shard->mtx); +static inline bool +hpa_needs_dehugify(hpa_shard_t *shard, const hpdata_t *ps) { + return (hpa_is_hugify_lazy(shard) + || opt_experimental_hpa_enforce_hugify) + && hpdata_huge_get(ps) && !hpdata_empty(ps); +} - hpdata_t *to_purge = psset_pick_purge(&shard->psset); +/* Prepare purge of one page. Return number of dirty regular pages on it + * Return 0 if no purgable huge page is found + * + * If there was a page to purge its purge state is initialized + */ +static inline size_t +hpa_purge_start_hp(hpa_purge_batch_t *b, hpa_shard_t *shard) { + psset_t *psset = &shard->psset; + hpdata_t *to_purge = (shard->opts.min_purge_delay_ms > 0) + ? psset_pick_purge(psset, &shard->last_time_work_attempted) + : psset_pick_purge(psset, NULL); if (to_purge == NULL) { - return false; + return 0; } assert(hpdata_purge_allowed_get(to_purge)); assert(!hpdata_changing_state_get(to_purge)); @@ -393,7 +391,7 @@ hpa_try_purge(tsdn_t *tsdn, hpa_shard_t *shard) { * we're purging it (allocations and deallocations are * OK). */ - psset_update_begin(&shard->psset, to_purge); + psset_update_begin(psset, to_purge); assert(hpdata_alloc_allowed_get(to_purge)); hpdata_mid_purge_set(to_purge, true); hpdata_purge_allowed_set(to_purge, false); @@ -406,59 +404,103 @@ hpa_try_purge(tsdn_t *tsdn, hpa_shard_t *shard) { * (clearing out user data). */ hpdata_alloc_allowed_set(to_purge, false); - psset_update_end(&shard->psset, to_purge); + psset_update_end(psset, to_purge); + assert(b->item_cnt < b->items_capacity); + hpa_purge_item_t *hp_item = &b->items[b->item_cnt]; + b->item_cnt++; + hp_item->hp = to_purge; /* Gather all the metadata we'll need during the purge. */ - bool dehugify = hpdata_huge_get(to_purge); - hpdata_purge_state_t purge_state; - size_t num_to_purge = hpdata_purge_begin(to_purge, &purge_state); + hp_item->dehugify = hpa_needs_dehugify(shard, hp_item->hp); + hpdata_purged_when_empty_and_huge_set(hp_item->hp, + hpdata_huge_get(hp_item->hp) && hpdata_empty(hp_item->hp)); + size_t nranges; + size_t ndirty = hpdata_purge_begin( + hp_item->hp, &hp_item->state, &nranges); + /* We picked hp to purge, so it should have some dirty ranges */ + assert(ndirty > 0 && nranges > 0); + b->ndirty_in_batch += ndirty; + b->nranges += nranges; + return ndirty; +} - shard->npending_purge += num_to_purge; - - malloc_mutex_unlock(tsdn, &shard->mtx); - - /* Actually do the purging, now that the lock is dropped. */ - if (dehugify) { - shard->central->hooks.dehugify(hpdata_addr_get(to_purge), - HUGEPAGE); - } - size_t total_purged = 0; - uint64_t purges_this_pass = 0; - void *purge_addr; - size_t purge_size; - while (hpdata_purge_next(to_purge, &purge_state, &purge_addr, - &purge_size)) { - total_purged += purge_size; - assert(total_purged <= HUGEPAGE); - purges_this_pass++; - shard->central->hooks.purge(purge_addr, purge_size); - } - - malloc_mutex_lock(tsdn, &shard->mtx); - /* The shard updates */ - shard->npending_purge -= num_to_purge; - shard->stats.npurge_passes++; - shard->stats.npurges += purges_this_pass; - shard->central->hooks.curtime(&shard->last_purge, - /* first_reading */ false); - if (dehugify) { +/* Finish purge of one huge page. */ +static inline void +hpa_purge_finish_hp( + tsdn_t *tsdn, hpa_shard_t *shard, hpa_purge_item_t *hp_item) { + if (hp_item->dehugify) { shard->stats.ndehugifies++; } - /* The hpdata updates. */ - psset_update_begin(&shard->psset, to_purge); - if (dehugify) { - hpdata_dehugify(to_purge); + psset_update_begin(&shard->psset, hp_item->hp); + if (hpdata_huge_get(hp_item->hp)) { + /* + * Even when dehugify is not explicitly called, the page is + * assumed to be non-huge after purge. + */ + hpdata_dehugify(hp_item->hp); } - hpdata_purge_end(to_purge, &purge_state); - hpdata_mid_purge_set(to_purge, false); + hpdata_purge_end(hp_item->hp, &hp_item->state); + hpdata_mid_purge_set(hp_item->hp, false); - hpdata_alloc_allowed_set(to_purge, true); - hpa_update_purge_hugify_eligibility(tsdn, shard, to_purge); + hpdata_alloc_allowed_set(hp_item->hp, true); + hpa_update_purge_hugify_eligibility(tsdn, shard, hp_item->hp); - psset_update_end(&shard->psset, to_purge); + psset_update_end(&shard->psset, hp_item->hp); +} - return true; +/* Returns number of huge pages purged. */ +static inline size_t +hpa_purge(tsdn_t *tsdn, hpa_shard_t *shard, size_t max_hp) { + malloc_mutex_assert_owner(tsdn, &shard->mtx); + assert(max_hp > 0); + + assert(HPA_PURGE_BATCH_MAX > 0); + assert(HPA_PURGE_BATCH_MAX + < (VARIABLE_ARRAY_SIZE_MAX / sizeof(hpa_purge_item_t))); + VARIABLE_ARRAY(hpa_purge_item_t, items, HPA_PURGE_BATCH_MAX); + hpa_purge_batch_t batch = { + .max_hp = max_hp, + .npurged_hp_total = 0, + .items = &items[0], + .items_capacity = HPA_PURGE_BATCH_MAX, + .range_watermark = hpa_process_madvise_max_iovec_len(), + }; + assert(batch.range_watermark > 0); + + while (1) { + hpa_batch_pass_start(&batch); + assert(hpa_batch_empty(&batch)); + while ( + !hpa_batch_full(&batch) && hpa_should_purge(tsdn, shard)) { + size_t ndirty = hpa_purge_start_hp(&batch, shard); + if (ndirty == 0) { + break; + } + shard->npending_purge += ndirty; + batch.npurged_hp_total++; + } + + if (hpa_batch_empty(&batch)) { + break; + } + hpa_hooks_t *hooks = &shard->central->hooks; + malloc_mutex_unlock(tsdn, &shard->mtx); + hpa_purge_batch(hooks, batch.items, batch.item_cnt); + malloc_mutex_lock(tsdn, &shard->mtx); + + /* The shard updates */ + shard->npending_purge -= batch.ndirty_in_batch; + shard->stats.npurges += batch.ndirty_in_batch; + shard->central->hooks.curtime(&shard->last_purge, + /* first_reading */ false); + for (size_t i = 0; i < batch.item_cnt; ++i) { + hpa_purge_finish_hp(tsdn, shard, &batch.items[i]); + } + } + malloc_mutex_assert_owner(tsdn, &shard->mtx); + shard->stats.npurge_passes++; + return batch.npurged_hp_total; } /* Returns whether or not we hugified anything. */ @@ -495,13 +537,34 @@ hpa_try_hugify(tsdn_t *tsdn, hpa_shard_t *shard) { hpdata_disallow_hugify(to_hugify); assert(hpdata_alloc_allowed_get(to_hugify)); psset_update_end(&shard->psset, to_hugify); - - malloc_mutex_unlock(tsdn, &shard->mtx); - - shard->central->hooks.hugify(hpdata_addr_get(to_hugify), HUGEPAGE); - - malloc_mutex_lock(tsdn, &shard->mtx); - shard->stats.nhugifies++; + /* + * Without lazy hugification, user relies on eagerly setting HG bit, or + * leaving everything up to the kernel (ex: thp enabled=always). We + * will still pretend that call succeeds to keep our accounting close to + * what user believes is the truth on the target system, but we won't + * update nhugifies stat as system call is not being made. + */ + if (hpa_is_hugify_lazy(shard) || opt_experimental_hpa_enforce_hugify) { + malloc_mutex_unlock(tsdn, &shard->mtx); + bool err = shard->central->hooks.hugify( + hpdata_addr_get(to_hugify), HUGEPAGE, + shard->opts.hugify_sync); + malloc_mutex_lock(tsdn, &shard->mtx); + shard->stats.nhugifies++; + if (err) { + /* + * When asynchronous hugification is used + * (shard->opts.hugify_sync option is false), we are not + * expecting to get here, unless something went terrible + * wrong. Because underlying syscall is only setting + * kernel flag for memory range (actual hugification + * happens asynchronously and we are not getting any + * feedback about its outcome), we expect syscall to be + * successful all the time. + */ + shard->stats.nhugify_failures++; + } + } psset_update_begin(&shard->psset, to_hugify); hpdata_hugify(to_hugify); @@ -512,51 +575,87 @@ hpa_try_hugify(tsdn_t *tsdn, hpa_shard_t *shard) { return true; } +static bool +hpa_min_purge_interval_passed(tsdn_t *tsdn, hpa_shard_t *shard) { + malloc_mutex_assert_owner(tsdn, &shard->mtx); + uint64_t since_last_purge_ms = nstime_ms_between( + &shard->last_purge, &shard->last_time_work_attempted); + return since_last_purge_ms >= shard->opts.min_purge_interval_ms; +} + +static inline void +hpa_update_time_work_attempted(tsdn_t *tsdn, hpa_shard_t *shard) { + malloc_mutex_assert_owner(tsdn, &shard->mtx); + shard->central->hooks.curtime(&shard->last_time_work_attempted, + /* first_reading */ false); +} + /* * Execution of deferred work is forced if it's triggered by an explicit * hpa_shard_do_deferred_work() call. */ static void -hpa_shard_maybe_do_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard, - bool forced) { +hpa_shard_maybe_do_deferred_work( + tsdn_t *tsdn, hpa_shard_t *shard, bool forced) { malloc_mutex_assert_owner(tsdn, &shard->mtx); if (!forced && shard->opts.deferral_allowed) { return; } + hpa_update_time_work_attempted(tsdn, shard); + /* * If we're on a background thread, do work so long as there's work to * be done. Otherwise, bound latency to not be *too* bad by doing at * most a small fixed number of operations. */ - bool hugified = false; - bool purged = false; size_t max_ops = (forced ? (size_t)-1 : 16); size_t nops = 0; - do { + + /* + * Always purge before hugifying, to make sure we get some + * ability to hit our quiescence targets. + */ + + /* + * Make sure we respect purge interval setting and don't purge + * too frequently. + */ + if (hpa_min_purge_interval_passed(tsdn, shard)) { + size_t max_purges = max_ops; /* - * Always purge before hugifying, to make sure we get some - * ability to hit our quiescence targets. + * Limit number of hugepages (slabs) to purge. + * When experimental_max_purge_nhp option is used, there is no + * guarantee we'll always respect dirty_mult option. Option + * experimental_max_purge_nhp provides a way to configure same + * behavior as was possible before, with buggy implementation + * of purging algorithm. */ - purged = false; - while (hpa_should_purge(tsdn, shard) && nops < max_ops) { - purged = hpa_try_purge(tsdn, shard); - if (purged) { - nops++; - } - } - hugified = hpa_try_hugify(tsdn, shard); - if (hugified) { - nops++; + ssize_t max_purge_nhp = shard->opts.experimental_max_purge_nhp; + if (max_purge_nhp != -1 && max_purges > (size_t)max_purge_nhp) { + max_purges = max_purge_nhp; } + malloc_mutex_assert_owner(tsdn, &shard->mtx); + nops += hpa_purge(tsdn, shard, max_purges); malloc_mutex_assert_owner(tsdn, &shard->mtx); - } while ((hugified || purged) && nops < max_ops); + } + + /* + * Try to hugify at least once, even if we out of operations to make at + * least some progress on hugification even at worst case. + */ + while (hpa_try_hugify(tsdn, shard) && nops < max_ops) { + malloc_mutex_assert_owner(tsdn, &shard->mtx); + nops++; + } } static edata_t * -hpa_try_alloc_one_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, - bool *oom) { - bool err; +hpa_try_alloc_one_no_grow( + tsdn_t *tsdn, hpa_shard_t *shard, size_t size, bool *oom) { + malloc_mutex_assert_owner(tsdn, &shard->mtx); + + bool err; edata_t *edata = edata_cache_fast_get(tsdn, &shard->ecf); if (edata == NULL) { *oom = true; @@ -576,15 +675,17 @@ hpa_try_alloc_one_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, * If the pageslab used to be empty, treat it as though it's * brand new for fragmentation-avoidance purposes; what we're * trying to approximate is the age of the allocations *in* that - * pageslab, and the allocations in the new pageslab are - * definitionally the youngest in this hpa shard. + * pageslab, and the allocations in the new pageslab are by + * definition the youngest in this hpa shard. */ hpdata_age_set(ps, shard->age_counter++); } void *addr = hpdata_reserve_alloc(ps, size); - edata_init(edata, shard->ind, addr, size, /* slab */ false, - SC_NSIZES, /* sn */ hpdata_age_get(ps), extent_state_active, + JE_USDT(hpa_alloc, 5, shard->ind, addr, size, hpdata_nactive_get(ps), + hpdata_age_get(ps)); + edata_init(edata, shard->ind, addr, size, /* slab */ false, SC_NSIZES, + /* sn */ hpdata_age_get(ps), extent_state_active, /* zeroed */ false, /* committed */ true, EXTENT_PAI_HPA, EXTENT_NOT_HEAD); edata_ps_set(edata, ps); @@ -597,11 +698,14 @@ hpa_try_alloc_one_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, * dropped. This would force us to deal with a pageslab eviction down * the error pathway, which is a pain. */ - err = emap_register_boundary(tsdn, shard->emap, edata, - SC_NSIZES, /* slab */ false); + err = emap_register_boundary( + tsdn, shard->emap, edata, SC_NSIZES, /* slab */ false); if (err) { - hpdata_unreserve(ps, edata_addr_get(edata), - edata_size_get(edata)); + hpdata_unreserve( + ps, edata_addr_get(edata), edata_size_get(edata)); + JE_USDT(hpa_dalloc_err, 5, shard->ind, edata_addr_get(edata), + edata_size_get(edata), hpdata_nactive_get(ps), + hpdata_age_get(ps)); /* * We should arguably reset dirty state here, but this would * require some sort of prepare + commit functionality that's a @@ -623,14 +727,14 @@ hpa_try_alloc_one_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, } static size_t -hpa_try_alloc_batch_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, - bool *oom, size_t nallocs, edata_list_active_t *results, +hpa_try_alloc_batch_no_grow_locked(tsdn_t *tsdn, hpa_shard_t *shard, + size_t size, bool *oom, size_t nallocs, edata_list_active_t *results, bool *deferred_work_generated) { - malloc_mutex_lock(tsdn, &shard->mtx); + malloc_mutex_assert_owner(tsdn, &shard->mtx); size_t nsuccess = 0; for (; nsuccess < nallocs; nsuccess++) { - edata_t *edata = hpa_try_alloc_one_no_grow(tsdn, shard, size, - oom); + edata_t *edata = hpa_try_alloc_one_no_grow( + tsdn, shard, size, oom); if (edata == NULL) { break; } @@ -639,6 +743,16 @@ hpa_try_alloc_batch_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, hpa_shard_maybe_do_deferred_work(tsdn, shard, /* forced */ false); *deferred_work_generated = hpa_shard_has_deferred_work(tsdn, shard); + return nsuccess; +} + +static size_t +hpa_try_alloc_batch_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, + bool *oom, size_t nallocs, edata_list_active_t *results, + bool *deferred_work_generated) { + malloc_mutex_lock(tsdn, &shard->mtx); + size_t nsuccess = hpa_try_alloc_batch_no_grow_locked( + tsdn, shard, size, oom, nallocs, results, deferred_work_generated); malloc_mutex_unlock(tsdn, &shard->mtx); return nsuccess; } @@ -647,11 +761,12 @@ static size_t hpa_alloc_batch_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, size_t nallocs, edata_list_active_t *results, bool *deferred_work_generated) { - assert(size <= shard->opts.slab_max_alloc); + assert(size <= HUGEPAGE); + assert(size <= shard->opts.slab_max_alloc || size == sz_s2u(size)); bool oom = false; - size_t nsuccess = hpa_try_alloc_batch_no_grow(tsdn, shard, size, &oom, - nallocs, results, deferred_work_generated); + size_t nsuccess = hpa_try_alloc_batch_no_grow( + tsdn, shard, size, &oom, nallocs, results, deferred_work_generated); if (nsuccess == nallocs || oom) { return nsuccess; @@ -678,24 +793,24 @@ hpa_alloc_batch_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, * deallocations (and allocations of smaller sizes) may still succeed * while we're doing this potentially expensive system call. */ - hpdata_t *ps = hpa_central_extract(tsdn, shard->central, size, &oom); + hpdata_t *ps = hpa_central_extract(tsdn, shard->central, size, + shard->age_counter++, hpa_is_hugify_eager(shard), &oom); if (ps == NULL) { malloc_mutex_unlock(tsdn, &shard->grow_mtx); return nsuccess; } /* - * We got the pageslab; allocate from it. This does an unlock followed - * by a lock on the same mutex, and holds the grow mutex while doing - * deferred work, but this is an uncommon path; the simplicity is worth - * it. + * We got the pageslab; allocate from it. This holds the grow mutex + * while doing deferred work, but this is an uncommon path; the + * simplicity is worth it. */ malloc_mutex_lock(tsdn, &shard->mtx); psset_insert(&shard->psset, ps); + nsuccess += hpa_try_alloc_batch_no_grow_locked(tsdn, shard, size, &oom, + nallocs - nsuccess, results, deferred_work_generated); malloc_mutex_unlock(tsdn, &shard->mtx); - nsuccess += hpa_try_alloc_batch_no_grow(tsdn, shard, size, &oom, - nallocs - nsuccess, results, deferred_work_generated); /* * Drop grow_mtx before doing deferred work; other threads blocked on it * should be allowed to proceed while we're working. @@ -707,32 +822,16 @@ hpa_alloc_batch_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, static hpa_shard_t * hpa_from_pai(pai_t *self) { - assert(self->alloc = &hpa_alloc); - assert(self->expand = &hpa_expand); - assert(self->shrink = &hpa_shrink); - assert(self->dalloc = &hpa_dalloc); + assert(self->alloc == &hpa_alloc); + assert(self->expand == &hpa_expand); + assert(self->shrink == &hpa_shrink); + assert(self->dalloc == &hpa_dalloc); return (hpa_shard_t *)self; } -static size_t -hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs, - edata_list_active_t *results, bool *deferred_work_generated) { - assert(nallocs > 0); - assert((size & PAGE_MASK) == 0); - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); - hpa_shard_t *shard = hpa_from_pai(self); - - if (size > shard->opts.slab_max_alloc) { - return 0; - } - - size_t nsuccess = hpa_alloc_batch_psset(tsdn, shard, size, nallocs, - results, deferred_work_generated); - - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); - +static void +hpa_assert_results( + tsdn_t *tsdn, hpa_shard_t *shard, edata_list_active_t *results) { /* * Guard the sanity checks with config_debug because the loop cannot be * proven non-circular by the compiler, even if everything within the @@ -740,20 +839,19 @@ hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs, */ if (config_debug) { edata_t *edata; - ql_foreach(edata, &results->head, ql_link_active) { + ql_foreach (edata, &results->head, ql_link_active) { emap_assert_mapped(tsdn, shard->emap, edata); assert(edata_pai_get(edata) == EXTENT_PAI_HPA); assert(edata_state_get(edata) == extent_state_active); assert(edata_arena_ind_get(edata) == shard->ind); - assert(edata_szind_get_maybe_invalid(edata) == - SC_NSIZES); + assert( + edata_szind_get_maybe_invalid(edata) == SC_NSIZES); assert(!edata_slab_get(edata)); assert(edata_committed_get(edata)); assert(edata_base_get(edata) == edata_addr_get(edata)); assert(edata_base_get(edata) != NULL); } } - return nsuccess; } static edata_t * @@ -761,23 +859,59 @@ hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero, bool guarded, bool frequent_reuse, bool *deferred_work_generated) { assert((size & PAGE_MASK) == 0); assert(!guarded); - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); /* We don't handle alignment or zeroing for now. */ if (alignment > PAGE || zero) { return NULL; } + hpa_shard_t *shard = hpa_from_pai(self); + /* - * An alloc with alignment == PAGE and zero == false is equivalent to a - * batch alloc of 1. Just do that, so we can share code. + * frequent_use here indicates this request comes from the arena bins, + * in which case it will be split into slabs, and therefore there is no + * intrinsic slack in the allocation (the entire range of allocated size + * will be accessed). + * + * In this case bypass the slab_max_alloc limit (if still within the + * huge page size). These requests do not concern internal + * fragmentation with huge pages (again, the full size will be used). */ + if (!(frequent_reuse && size <= HUGEPAGE) + && (size > shard->opts.slab_max_alloc)) { + return NULL; + } + edata_t *edata = sec_alloc(tsdn, &shard->sec, size); + if (edata != NULL) { + return edata; + } + size_t nallocs = sec_size_supported(&shard->sec, size) + ? shard->sec.opts.batch_fill_extra + 1 + : 1; edata_list_active_t results; edata_list_active_init(&results); - size_t nallocs = hpa_alloc_batch(tsdn, self, size, /* nallocs */ 1, - &results, deferred_work_generated); - assert(nallocs == 0 || nallocs == 1); - edata_t *edata = edata_list_active_first(&results); + size_t nsuccess = hpa_alloc_batch_psset( + tsdn, shard, size, nallocs, &results, deferred_work_generated); + hpa_assert_results(tsdn, shard, &results); + edata = edata_list_active_first(&results); + + if (edata != NULL) { + edata_list_active_remove(&results, edata); + assert(nsuccess > 0); + nsuccess--; + } + if (nsuccess > 0) { + assert(sec_size_supported(&shard->sec, size)); + sec_fill(tsdn, &shard->sec, size, &results, nsuccess); + /* Unlikely rollback in case of overfill */ + if (!edata_list_active_empty(&results)) { + hpa_dalloc_batch( + tsdn, self, &results, deferred_work_generated); + } + } + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); return edata; } @@ -789,8 +923,8 @@ hpa_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, } static bool -hpa_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, - size_t old_size, size_t new_size, bool *deferred_work_generated) { +hpa_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, + size_t new_size, bool *deferred_work_generated) { /* Shrink not yet supported. */ return true; } @@ -835,12 +969,14 @@ hpa_dalloc_locked(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata) { hpdata_t *ps = edata_ps_get(edata); /* Currently, all edatas come from pageslabs. */ assert(ps != NULL); - void *unreserve_addr = edata_addr_get(edata); + void *unreserve_addr = edata_addr_get(edata); size_t unreserve_size = edata_size_get(edata); edata_cache_fast_put(tsdn, &shard->ecf, edata); psset_update_begin(&shard->psset, ps); hpdata_unreserve(ps, unreserve_addr, unreserve_size); + JE_USDT(hpa_dalloc, 5, shard->ind, unreserve_addr, unreserve_size, + hpdata_nactive_get(ps), hpdata_age_get(ps)); hpa_update_purge_hugify_eligibility(tsdn, shard, ps); psset_update_end(&shard->psset, ps); } @@ -851,7 +987,7 @@ hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self, edata_list_active_t *list, hpa_shard_t *shard = hpa_from_pai(self); edata_t *edata; - ql_foreach(edata, &list->head, ql_link_active) { + ql_foreach (edata, &list->head, ql_link_active) { hpa_dalloc_prepare_unlocked(tsdn, shard, edata); } @@ -862,20 +998,28 @@ hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self, edata_list_active_t *list, hpa_dalloc_locked(tsdn, shard, edata); } hpa_shard_maybe_do_deferred_work(tsdn, shard, /* forced */ false); - *deferred_work_generated = - hpa_shard_has_deferred_work(tsdn, shard); + *deferred_work_generated = hpa_shard_has_deferred_work(tsdn, shard); malloc_mutex_unlock(tsdn, &shard->mtx); } static void -hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata, - bool *deferred_work_generated) { +hpa_dalloc( + tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) { assert(!edata_guarded_get(edata)); - /* Just a dalloc_batch of size 1; this lets us share logic. */ + edata_list_active_t dalloc_list; edata_list_active_init(&dalloc_list); edata_list_active_append(&dalloc_list, edata); + + hpa_shard_t *shard = hpa_from_pai(self); + sec_dalloc(tsdn, &shard->sec, &dalloc_list); + if (edata_list_active_empty(&dalloc_list)) { + /* sec consumed the pointer */ + *deferred_work_generated = false; + return; + } + /* We may have more than one pointer to flush now */ hpa_dalloc_batch(tsdn, self, &dalloc_list, deferred_work_generated); } @@ -886,14 +1030,14 @@ hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata, static uint64_t hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) { hpa_shard_t *shard = hpa_from_pai(self); - uint64_t time_ns = BACKGROUND_THREAD_DEFERRED_MAX; + uint64_t time_ns = BACKGROUND_THREAD_DEFERRED_MAX; malloc_mutex_lock(tsdn, &shard->mtx); hpdata_t *to_hugify = psset_pick_hugify(&shard->psset); if (to_hugify != NULL) { - nstime_t time_hugify_allowed = - hpdata_time_hugify_allowed(to_hugify); + nstime_t time_hugify_allowed = hpdata_time_hugify_allowed( + to_hugify); uint64_t since_hugify_allowed_ms = shard->central->hooks.ms_since(&time_hugify_allowed); /* @@ -901,8 +1045,8 @@ hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) { * sleep for the rest. */ if (since_hugify_allowed_ms < shard->opts.hugify_delay_ms) { - time_ns = shard->opts.hugify_delay_ms - - since_hugify_allowed_ms; + time_ns = shard->opts.hugify_delay_ms + - since_hugify_allowed_ms; time_ns *= 1000 * 1000; } else { malloc_mutex_unlock(tsdn, &shard->mtx); @@ -924,8 +1068,8 @@ hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) { if (since_last_purge_ms < shard->opts.min_purge_interval_ms) { uint64_t until_purge_ns; - until_purge_ns = shard->opts.min_purge_interval_ms - - since_last_purge_ms; + until_purge_ns = shard->opts.min_purge_interval_ms + - since_last_purge_ms; until_purge_ns *= 1000 * 1000; if (until_purge_ns < time_ns) { @@ -939,15 +1083,32 @@ hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) { return time_ns; } +static void +hpa_sec_flush_impl(tsdn_t *tsdn, hpa_shard_t *shard) { + edata_list_active_t to_flush; + edata_list_active_init(&to_flush); + + sec_flush(tsdn, &shard->sec, &to_flush); + bool deferred_work_generated; + hpa_dalloc_batch( + tsdn, (pai_t *)shard, &to_flush, &deferred_work_generated); +} + void hpa_shard_disable(tsdn_t *tsdn, hpa_shard_t *shard) { hpa_do_consistency_checks(shard); + hpa_sec_flush_impl(tsdn, shard); malloc_mutex_lock(tsdn, &shard->mtx); edata_cache_fast_disable(tsdn, &shard->ecf); malloc_mutex_unlock(tsdn, &shard->mtx); } +void +hpa_shard_flush(tsdn_t *tsdn, hpa_shard_t *shard) { + hpa_sec_flush_impl(tsdn, shard); +} + static void hpa_shard_assert_stats_empty(psset_bin_stats_t *bin_stats) { assert(bin_stats->npageslabs == 0); @@ -969,6 +1130,7 @@ hpa_assert_empty(tsdn_t *tsdn, hpa_shard_t *shard, psset_t *psset) { void hpa_shard_destroy(tsdn_t *tsdn, hpa_shard_t *shard) { hpa_do_consistency_checks(shard); + hpa_shard_flush(tsdn, shard); /* * By the time we're here, the arena code should have dalloc'd all the * active extents, which means we should have eventually evicted @@ -990,8 +1152,8 @@ hpa_shard_destroy(tsdn_t *tsdn, hpa_shard_t *shard) { } void -hpa_shard_set_deferral_allowed(tsdn_t *tsdn, hpa_shard_t *shard, - bool deferral_allowed) { +hpa_shard_set_deferral_allowed( + tsdn_t *tsdn, hpa_shard_t *shard, bool deferral_allowed) { hpa_do_consistency_checks(shard); malloc_mutex_lock(tsdn, &shard->mtx); @@ -1013,6 +1175,12 @@ hpa_shard_do_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) { malloc_mutex_unlock(tsdn, &shard->mtx); } +void +hpa_shard_prefork2(tsdn_t *tsdn, hpa_shard_t *shard) { + hpa_do_consistency_checks(shard); + sec_prefork2(tsdn, &shard->sec); +} + void hpa_shard_prefork3(tsdn_t *tsdn, hpa_shard_t *shard) { hpa_do_consistency_checks(shard); @@ -1031,6 +1199,7 @@ void hpa_shard_postfork_parent(tsdn_t *tsdn, hpa_shard_t *shard) { hpa_do_consistency_checks(shard); + sec_postfork_parent(tsdn, &shard->sec); malloc_mutex_postfork_parent(tsdn, &shard->grow_mtx); malloc_mutex_postfork_parent(tsdn, &shard->mtx); } @@ -1039,6 +1208,7 @@ void hpa_shard_postfork_child(tsdn_t *tsdn, hpa_shard_t *shard) { hpa_do_consistency_checks(shard); + sec_postfork_child(tsdn, &shard->sec); malloc_mutex_postfork_child(tsdn, &shard->grow_mtx); malloc_mutex_postfork_child(tsdn, &shard->mtx); } diff --git a/dep/jemalloc/src/hpa_central.c b/dep/jemalloc/src/hpa_central.c new file mode 100644 index 000000000..b4f770c2c --- /dev/null +++ b/dep/jemalloc/src/hpa_central.c @@ -0,0 +1,121 @@ +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" + +#include "jemalloc/internal/hpa_central.h" +#include "jemalloc/internal/tsd.h" +#include "jemalloc/internal/witness.h" + +#define HPA_EDEN_SIZE (128 * HUGEPAGE) + +bool +hpa_central_init( + hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks) { + /* malloc_conf processing should have filtered out these cases. */ + assert(hpa_supported()); + bool err; + err = malloc_mutex_init(¢ral->grow_mtx, "hpa_central_grow", + WITNESS_RANK_HPA_CENTRAL_GROW, malloc_mutex_rank_exclusive); + if (err) { + return true; + } + + central->base = base; + central->eden = NULL; + central->eden_len = 0; + central->hooks = *hooks; + return false; +} + +static hpdata_t * +hpa_alloc_ps(tsdn_t *tsdn, hpa_central_t *central) { + return (hpdata_t *)base_alloc( + tsdn, central->base, sizeof(hpdata_t), CACHELINE); +} + +hpdata_t * +hpa_central_extract(tsdn_t *tsdn, hpa_central_t *central, size_t size, + uint64_t age, bool hugify_eager, bool *oom) { + /* Don't yet support big allocations; these should get filtered out. */ + assert(size <= HUGEPAGE); + /* + * Should only try to extract from the central allocator if the local + * shard is exhausted. We should hold the grow_mtx on that shard. + */ + witness_assert_positive_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_HPA_SHARD_GROW); + + malloc_mutex_lock(tsdn, ¢ral->grow_mtx); + *oom = false; + + hpdata_t *ps = NULL; + bool start_as_huge = hugify_eager + || (init_system_thp_mode == system_thp_mode_always + && opt_experimental_hpa_start_huge_if_thp_always); + + /* Is eden a perfect fit? */ + if (central->eden != NULL && central->eden_len == HUGEPAGE) { + ps = hpa_alloc_ps(tsdn, central); + if (ps == NULL) { + *oom = true; + malloc_mutex_unlock(tsdn, ¢ral->grow_mtx); + return NULL; + } + hpdata_init(ps, central->eden, age, start_as_huge); + central->eden = NULL; + central->eden_len = 0; + malloc_mutex_unlock(tsdn, ¢ral->grow_mtx); + return ps; + } + + /* + * We're about to try to allocate from eden by splitting. If eden is + * NULL, we have to allocate it too. Otherwise, we just have to + * allocate an edata_t for the new psset. + */ + if (central->eden == NULL) { + /* Allocate address space, bailing if we fail. */ + void *new_eden = central->hooks.map(HPA_EDEN_SIZE); + if (new_eden == NULL) { + *oom = true; + malloc_mutex_unlock(tsdn, ¢ral->grow_mtx); + return NULL; + } + if (hugify_eager) { + central->hooks.hugify( + new_eden, HPA_EDEN_SIZE, /* sync */ false); + } + ps = hpa_alloc_ps(tsdn, central); + if (ps == NULL) { + central->hooks.unmap(new_eden, HPA_EDEN_SIZE); + *oom = true; + malloc_mutex_unlock(tsdn, ¢ral->grow_mtx); + return NULL; + } + central->eden = new_eden; + central->eden_len = HPA_EDEN_SIZE; + } else { + /* Eden is already nonempty; only need an edata for ps. */ + ps = hpa_alloc_ps(tsdn, central); + if (ps == NULL) { + *oom = true; + malloc_mutex_unlock(tsdn, ¢ral->grow_mtx); + return NULL; + } + } + assert(ps != NULL); + assert(central->eden != NULL); + assert(central->eden_len > HUGEPAGE); + assert(central->eden_len % HUGEPAGE == 0); + assert(HUGEPAGE_ADDR2BASE(central->eden) == central->eden); + + hpdata_init(ps, central->eden, age, start_as_huge); + + char *eden_char = (char *)central->eden; + eden_char += HUGEPAGE; + central->eden = (void *)eden_char; + central->eden_len -= HUGEPAGE; + + malloc_mutex_unlock(tsdn, ¢ral->grow_mtx); + + return ps; +} diff --git a/dep/jemalloc/src/hpa_hooks.c b/dep/jemalloc/src/hpa_hooks.c index ade581e8d..2ec7029da 100644 --- a/dep/jemalloc/src/hpa_hooks.c +++ b/dep/jemalloc/src/hpa_hooks.c @@ -2,50 +2,75 @@ #include "jemalloc/internal/jemalloc_internal_includes.h" #include "jemalloc/internal/hpa_hooks.h" +#include "jemalloc/internal/jemalloc_probe.h" -static void *hpa_hooks_map(size_t size); -static void hpa_hooks_unmap(void *ptr, size_t size); -static void hpa_hooks_purge(void *ptr, size_t size); -static void hpa_hooks_hugify(void *ptr, size_t size); -static void hpa_hooks_dehugify(void *ptr, size_t size); -static void hpa_hooks_curtime(nstime_t *r_nstime, bool first_reading); +static void *hpa_hooks_map(size_t size); +static void hpa_hooks_unmap(void *ptr, size_t size); +static void hpa_hooks_purge(void *ptr, size_t size); +static bool hpa_hooks_hugify(void *ptr, size_t size, bool sync); +static void hpa_hooks_dehugify(void *ptr, size_t size); +static void hpa_hooks_curtime(nstime_t *r_nstime, bool first_reading); static uint64_t hpa_hooks_ms_since(nstime_t *past_nstime); +static bool hpa_hooks_vectorized_purge(void *vec, size_t vlen, size_t nbytes); -hpa_hooks_t hpa_hooks_default = { - &hpa_hooks_map, - &hpa_hooks_unmap, - &hpa_hooks_purge, - &hpa_hooks_hugify, - &hpa_hooks_dehugify, - &hpa_hooks_curtime, - &hpa_hooks_ms_since -}; +const hpa_hooks_t hpa_hooks_default = {&hpa_hooks_map, &hpa_hooks_unmap, + &hpa_hooks_purge, &hpa_hooks_hugify, &hpa_hooks_dehugify, + &hpa_hooks_curtime, &hpa_hooks_ms_since, &hpa_hooks_vectorized_purge}; static void * hpa_hooks_map(size_t size) { + /* + * During development, we're primarily concerned with systems + * that overcommit. Eventually, we should be more careful here. + */ + bool commit = true; - return pages_map(NULL, size, HUGEPAGE, &commit); + assert((size & HUGEPAGE_MASK) == 0); + void *ret = pages_map(NULL, size, HUGEPAGE, &commit); + JE_USDT(hpa_map, 2, size, ret); + return ret; } static void hpa_hooks_unmap(void *ptr, size_t size) { + JE_USDT(hpa_unmap, 2, size, ptr); pages_unmap(ptr, size); } static void hpa_hooks_purge(void *ptr, size_t size) { + JE_USDT(hpa_purge, 2, size, ptr); pages_purge_forced(ptr, size); } -static void -hpa_hooks_hugify(void *ptr, size_t size) { +static bool +hpa_hooks_hugify(void *ptr, size_t size, bool sync) { + /* + * We mark memory range as huge independently on which hugification + * technique is used (synchronous or asynchronous) to have correct + * VmFlags set for introspection and accounting purposes. If + * synchronous hugification is enabled and pages_collapse call fails, + * then we hope memory range will be hugified asynchronously by + * khugepaged eventually. Right now, 3 out of 4 error return codes of + * madvise(..., MADV_COLLAPSE) are retryable. Instead of retrying, we + * just fallback to asynchronous khugepaged hugification to simplify + * implementation, even if we might know khugepaged fallback will not + * be successful (current madvise(..., MADV_COLLAPSE) implementation + * hints, when EINVAL is returned it is likely that khugepaged won't be + * able to collapse memory range into hugepage either). + */ bool err = pages_huge(ptr, size); - (void)err; + if (sync) { + err = pages_collapse(ptr, size); + } + JE_USDT(hpa_hugify, 4, size, ptr, err, sync); + return err; } static void hpa_hooks_dehugify(void *ptr, size_t size) { bool err = pages_nohuge(ptr, size); + JE_USDT(hpa_dehugify, 3, size, ptr, err); (void)err; } @@ -59,5 +84,17 @@ hpa_hooks_curtime(nstime_t *r_nstime, bool first_reading) { static uint64_t hpa_hooks_ms_since(nstime_t *past_nstime) { - return nstime_ns_since(past_nstime) / 1000 / 1000; + return nstime_ms_since(past_nstime); +} + +/* Return true if we did not purge all nbytes, or on some error */ +static bool +hpa_hooks_vectorized_purge(void *vec, size_t vlen, size_t nbytes) { +#ifdef JEMALLOC_HAVE_PROCESS_MADVISE + bool err = pages_purge_process_madvise(vec, vlen, nbytes); + JE_USDT(hpa_vectorized_purge, 3, nbytes, vlen, err); + return err; +#else + return true; +#endif } diff --git a/dep/jemalloc/src/hpa_utils.c b/dep/jemalloc/src/hpa_utils.c new file mode 100644 index 000000000..59bb0d1f4 --- /dev/null +++ b/dep/jemalloc/src/hpa_utils.c @@ -0,0 +1,33 @@ +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" + +#include "jemalloc/internal/hpa_utils.h" + +void +hpa_purge_batch(hpa_hooks_t *hooks, hpa_purge_item_t *batch, size_t batch_sz) { + assert(batch_sz > 0); + + size_t len = hpa_process_madvise_max_iovec_len(); + VARIABLE_ARRAY(hpa_io_vector_t, vec, len); + + hpa_range_accum_t accum; + hpa_range_accum_init(&accum, vec, len); + + for (size_t i = 0; i < batch_sz; ++i) { + /* Actually do the purging, now that the lock is dropped. */ + if (batch[i].dehugify) { + hooks->dehugify(hpdata_addr_get(batch[i].hp), HUGEPAGE); + } + void *purge_addr; + size_t purge_size; + size_t total_purged_on_one_hp = 0; + while (hpdata_purge_next( + batch[i].hp, &batch[i].state, &purge_addr, &purge_size)) { + total_purged_on_one_hp += purge_size; + assert(total_purged_on_one_hp <= HUGEPAGE); + hpa_range_accum_add( + &accum, purge_addr, purge_size, hooks); + } + } + hpa_range_accum_finish(&accum, hooks); +} diff --git a/dep/jemalloc/src/hpdata.c b/dep/jemalloc/src/hpdata.c index e7d7294c7..e17d9ecff 100644 --- a/dep/jemalloc/src/hpdata.c +++ b/dep/jemalloc/src/hpdata.c @@ -17,11 +17,10 @@ hpdata_age_comp(const hpdata_t *a, const hpdata_t *b) { ph_gen(, hpdata_age_heap, hpdata_t, age_link, hpdata_age_comp) -void -hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age) { + void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age, bool is_huge) { hpdata_addr_set(hpdata, addr); hpdata_age_set(hpdata, age); - hpdata->h_huge = false; + hpdata->h_huge = is_huge; hpdata->h_alloc_allowed = true; hpdata->h_in_psset_alloc_container = false; hpdata->h_purge_allowed = false; @@ -34,8 +33,16 @@ hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age) { hpdata_longest_free_range_set(hpdata, HUGEPAGE_PAGES); hpdata->h_nactive = 0; fb_init(hpdata->active_pages, HUGEPAGE_PAGES); - hpdata->h_ntouched = 0; - fb_init(hpdata->touched_pages, HUGEPAGE_PAGES); + if (is_huge) { + fb_set_range( + hpdata->touched_pages, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES); + hpdata->h_ntouched = HUGEPAGE_PAGES; + } else { + fb_init(hpdata->touched_pages, HUGEPAGE_PAGES); + hpdata->h_ntouched = 0; + } + nstime_init_zero(&hpdata->h_time_purge_allowed); + hpdata->h_purged_when_empty_and_huge = false; hpdata_assert_consistent(hpdata); } @@ -66,8 +73,8 @@ hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz) { size_t largest_unchosen_range = 0; while (true) { - bool found = fb_urange_iter(hpdata->active_pages, - HUGEPAGE_PAGES, start, &begin, &len); + bool found = fb_urange_iter( + hpdata->active_pages, HUGEPAGE_PAGES, start, &begin, &len); /* * A precondition to this function is that hpdata must be able * to serve the allocation. @@ -97,8 +104,8 @@ hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz) { * We might be about to dirty some memory for the first time; update our * count if so. */ - size_t new_dirty = fb_ucount(hpdata->touched_pages, HUGEPAGE_PAGES, - result, npages); + size_t new_dirty = fb_ucount( + hpdata->touched_pages, HUGEPAGE_PAGES, result, npages); fb_set_range(hpdata->touched_pages, HUGEPAGE_PAGES, result, npages); hpdata->h_ntouched += new_dirty; @@ -129,8 +136,8 @@ hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz) { } hpdata_assert_consistent(hpdata); - return (void *)( - (uintptr_t)hpdata_addr_get(hpdata) + (result << LG_PAGE)); + return ( + void *)((byte_t *)hpdata_addr_get(hpdata) + (result << LG_PAGE)); } void @@ -148,10 +155,10 @@ hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz) { fb_unset_range(hpdata->active_pages, HUGEPAGE_PAGES, begin, npages); /* We might have just created a new, larger range. */ - size_t new_begin = (fb_fls(hpdata->active_pages, HUGEPAGE_PAGES, - begin) + 1); - size_t new_end = fb_ffs(hpdata->active_pages, HUGEPAGE_PAGES, - begin + npages - 1); + size_t new_begin = (fb_fls(hpdata->active_pages, HUGEPAGE_PAGES, begin) + + 1); + size_t new_end = fb_ffs( + hpdata->active_pages, HUGEPAGE_PAGES, begin + npages - 1); size_t new_range_len = new_end - new_begin; if (new_range_len > old_longest_range) { @@ -164,7 +171,8 @@ hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz) { } size_t -hpdata_purge_begin(hpdata_t *hpdata, hpdata_purge_state_t *purge_state) { +hpdata_purge_begin( + hpdata_t *hpdata, hpdata_purge_state_t *purge_state, size_t *nranges) { hpdata_assert_consistent(hpdata); /* * See the comment below; we might purge any inactive extent, so it's @@ -211,34 +219,36 @@ hpdata_purge_begin(hpdata_t *hpdata, hpdata_purge_state_t *purge_state) { fb_group_t dirty_pages[FB_NGROUPS(HUGEPAGE_PAGES)]; fb_init(dirty_pages, HUGEPAGE_PAGES); fb_bit_not(dirty_pages, hpdata->active_pages, HUGEPAGE_PAGES); - fb_bit_and(dirty_pages, dirty_pages, hpdata->touched_pages, - HUGEPAGE_PAGES); + fb_bit_and( + dirty_pages, dirty_pages, hpdata->touched_pages, HUGEPAGE_PAGES); fb_init(purge_state->to_purge, HUGEPAGE_PAGES); size_t next_bit = 0; + *nranges = 0; while (next_bit < HUGEPAGE_PAGES) { - size_t next_dirty = fb_ffs(dirty_pages, HUGEPAGE_PAGES, - next_bit); + size_t next_dirty = fb_ffs( + dirty_pages, HUGEPAGE_PAGES, next_bit); /* Recall that fb_ffs returns nbits if no set bit is found. */ if (next_dirty == HUGEPAGE_PAGES) { break; } - size_t next_active = fb_ffs(hpdata->active_pages, - HUGEPAGE_PAGES, next_dirty); + size_t next_active = fb_ffs( + hpdata->active_pages, HUGEPAGE_PAGES, next_dirty); /* * Don't purge past the end of the dirty extent, into retained * pages. This helps the kernel a tiny bit, but honestly it's * mostly helpful for testing (where we tend to write test cases * that think in terms of the dirty ranges). */ - ssize_t last_dirty = fb_fls(dirty_pages, HUGEPAGE_PAGES, - next_active - 1); + ssize_t last_dirty = fb_fls( + dirty_pages, HUGEPAGE_PAGES, next_active - 1); assert(last_dirty >= 0); assert((size_t)last_dirty >= next_dirty); assert((size_t)last_dirty - next_dirty + 1 <= HUGEPAGE_PAGES); fb_set_range(purge_state->to_purge, HUGEPAGE_PAGES, next_dirty, last_dirty - next_dirty + 1); + (*nranges)++; next_bit = next_active + 1; } @@ -246,9 +256,11 @@ hpdata_purge_begin(hpdata_t *hpdata, hpdata_purge_state_t *purge_state) { size_t ndirty = hpdata->h_ntouched - hpdata->h_nactive; purge_state->ndirty_to_purge = ndirty; assert(ndirty <= fb_scount( - purge_state->to_purge, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES)); - assert(ndirty == fb_scount(dirty_pages, HUGEPAGE_PAGES, 0, - HUGEPAGE_PAGES)); + purge_state->to_purge, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES)); + assert(ndirty + == fb_scount(dirty_pages, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES)); + assert(*nranges <= ndirty); + assert(ndirty == 0 || *nranges > 0); hpdata_assert_consistent(hpdata); @@ -276,8 +288,8 @@ hpdata_purge_next(hpdata_t *hpdata, hpdata_purge_state_t *purge_state, return false; } - *r_purge_addr = (void *)( - (uintptr_t)hpdata_addr_get(hpdata) + purge_begin * PAGE); + *r_purge_addr = (void *)((byte_t *)hpdata_addr_get(hpdata) + + purge_begin * PAGE); *r_purge_size = purge_len * PAGE; purge_state->next_purge_search_begin = purge_begin + purge_len; @@ -294,12 +306,13 @@ hpdata_purge_end(hpdata_t *hpdata, hpdata_purge_state_t *purge_state) { /* See the comment in reserve. */ assert(!hpdata->h_in_psset || hpdata->h_updating); - assert(purge_state->npurged == fb_scount(purge_state->to_purge, - HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES)); + assert(purge_state->npurged + == fb_scount( + purge_state->to_purge, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES)); assert(purge_state->npurged >= purge_state->ndirty_to_purge); - fb_bit_not(purge_state->to_purge, purge_state->to_purge, - HUGEPAGE_PAGES); + fb_bit_not( + purge_state->to_purge, purge_state->to_purge, HUGEPAGE_PAGES); fb_bit_and(hpdata->touched_pages, hpdata->touched_pages, purge_state->to_purge, HUGEPAGE_PAGES); assert(hpdata->h_ntouched >= purge_state->ndirty_to_purge); diff --git a/dep/jemalloc/src/inspect.c b/dep/jemalloc/src/inspect.c index 911b5d524..116e77a18 100644 --- a/dep/jemalloc/src/inspect.c +++ b/dep/jemalloc/src/inspect.c @@ -1,9 +1,10 @@ #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/jemalloc_internal_includes.h" +#include "jemalloc/internal/inspect.h" void -inspect_extent_util_stats_get(tsdn_t *tsdn, const void *ptr, size_t *nfree, - size_t *nregs, size_t *size) { +inspect_extent_util_stats_get( + tsdn_t *tsdn, const void *ptr, size_t *nfree, size_t *nregs, size_t *size) { assert(ptr != NULL && nfree != NULL && nregs != NULL && size != NULL); const edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr); @@ -56,7 +57,7 @@ inspect_extent_util_stats_verbose_get(tsdn_t *tsdn, const void *ptr, &arenas[edata_arena_ind_get(edata)], ATOMIC_RELAXED); assert(arena != NULL); const unsigned binshard = edata_binshard_get(edata); - bin_t *bin = arena_get_bin(arena, szind, binshard); + bin_t *bin = arena_get_bin(arena, szind, binshard); malloc_mutex_lock(tsdn, &bin->lock); if (config_stats) { diff --git a/dep/jemalloc/src/jemalloc.c b/dep/jemalloc/src/jemalloc.c index 7655de4e2..1ca397262 100644 --- a/dep/jemalloc/src/jemalloc.c +++ b/dep/jemalloc/src/jemalloc.c @@ -1,4 +1,3 @@ -#define JEMALLOC_C_ #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/jemalloc_internal_includes.h" @@ -26,15 +25,17 @@ #include "jemalloc/internal/thread_event.h" #include "jemalloc/internal/util.h" +#include "jemalloc/internal/conf.h" + /******************************************************************************/ /* Data. */ /* Runtime configuration options. */ -const char *je_malloc_conf +const char *je_malloc_conf #ifndef _WIN32 JEMALLOC_ATTR(weak) #endif - ; + ; /* * The usual rule is that the closer to runtime you are, the higher priority * your configuration settings are (so the jemalloc config options get lower @@ -42,7 +43,7 @@ const char *je_malloc_conf * setting, which gets lower priority than the environment settings). * * But it's a fairly common use case in some testing environments for a user to - * be able to control the binary, but nothing else (e.g. a performancy canary + * be able to control the binary, but nothing else (e.g. a performance canary * uses the production OS and environment variables, but can run any binary in * those circumstances). For these use cases, it's handy to have an in-binary * mechanism for overriding environment variable settings, with the idea that if @@ -52,20 +53,23 @@ const char *je_malloc_conf * We don't actually want this to be widespread, so we'll give it a silly name * and not mention it in headers or documentation. */ -const char *je_malloc_conf_2_conf_harder +const char *je_malloc_conf_2_conf_harder #ifndef _WIN32 JEMALLOC_ATTR(weak) #endif - ; + ; -bool opt_abort = +const char *opt_malloc_conf_symlink = NULL; +const char *opt_malloc_conf_env_var = NULL; + +bool opt_abort = #ifdef JEMALLOC_DEBUG true #else false #endif ; -bool opt_abort_conf = +bool opt_abort_conf = #ifdef JEMALLOC_DEBUG true #else @@ -73,29 +77,29 @@ bool opt_abort_conf = #endif ; /* Intentionally default off, even with debug builds. */ -bool opt_confirm_conf = false; -const char *opt_junk = +bool opt_confirm_conf = false; +const char *opt_junk = #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) "true" #else "false" #endif ; -bool opt_junk_alloc = +bool opt_junk_alloc = #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) true #else false #endif ; -bool opt_junk_free = +bool opt_junk_free = #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) true #else false #endif ; -bool opt_trust_madvise = +bool opt_trust_madvise = #ifdef JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS false #else @@ -121,10 +125,17 @@ zero_realloc_action_t opt_zero_realloc_action = atomic_zu_t zero_realloc_count = ATOMIC_INIT(0); -const char *zero_realloc_mode_names[] = { - "alloc", - "free", - "abort", +/* + * Disable large size classes is now the default behavior in jemalloc. + * Although it is configurable in MALLOC_CONF, this is mainly for debugging + * purposes and should not be tuned. + */ +bool opt_disable_large_size_classes = true; + +const char *const zero_realloc_mode_names[] = { + "alloc", + "free", + "abort", }; /* @@ -134,33 +145,44 @@ const char *zero_realloc_mode_names[] = { static const uint8_t junk_alloc_byte = 0xa5; static const uint8_t junk_free_byte = 0x5a; -static void default_junk_alloc(void *ptr, size_t usize) { +static void +default_junk_alloc(void *ptr, size_t usize) { memset(ptr, junk_alloc_byte, usize); } -static void default_junk_free(void *ptr, size_t usize) { +static void +default_junk_free(void *ptr, size_t usize) { memset(ptr, junk_free_byte, usize); } -void (*junk_alloc_callback)(void *ptr, size_t size) = &default_junk_alloc; -void (*junk_free_callback)(void *ptr, size_t size) = &default_junk_free; +void (*JET_MUTABLE junk_alloc_callback)( + void *ptr, size_t size) = &default_junk_alloc; +void (*JET_MUTABLE junk_free_callback)( + void *ptr, size_t size) = &default_junk_free; +void (*JET_MUTABLE invalid_conf_abort)(void) = &abort; -bool opt_utrace = false; -bool opt_xmalloc = false; -bool opt_experimental_infallible_new = false; -bool opt_zero = false; -unsigned opt_narenas = 0; -fxp_t opt_narenas_ratio = FXP_INIT_INT(4); +bool opt_utrace = false; +bool opt_xmalloc = false; +bool opt_experimental_infallible_new = false; +bool opt_experimental_tcache_gc = true; +bool opt_zero = false; +unsigned opt_narenas = 0; +fxp_t opt_narenas_ratio = FXP_INIT_INT(4); -unsigned ncpus; +unsigned ncpus; + +unsigned opt_debug_double_free_max_scan = + SAFETY_CHECK_DOUBLE_FREE_MAX_SCAN_DEFAULT; + +size_t opt_calloc_madvise_threshold = CALLOC_MADVISE_THRESHOLD_DEFAULT; /* Protects arenas initialization. */ -malloc_mutex_t arenas_lock; +static malloc_mutex_t arenas_lock; /* The global hpa, and whether it's on. */ -bool opt_hpa = false; +bool opt_hpa = false; hpa_shard_opts_t opt_hpa_opts = HPA_SHARD_OPTS_DEFAULT; -sec_opts_t opt_hpa_sec_opts = SEC_OPTS_DEFAULT; +sec_opts_t opt_hpa_sec_opts = SEC_OPTS_DEFAULT; /* * Arenas that are used to service external requests. Not all elements of the @@ -173,48 +195,49 @@ sec_opts_t opt_hpa_sec_opts = SEC_OPTS_DEFAULT; * Points to an arena_t. */ JEMALLOC_ALIGNED(CACHELINE) -atomic_p_t arenas[MALLOCX_ARENA_LIMIT]; -static atomic_u_t narenas_total; /* Use narenas_total_*(). */ +atomic_p_t arenas[MALLOCX_ARENA_LIMIT]; +static atomic_u_t narenas_total; /* Use narenas_total_*(). */ /* Below three are read-only after initialization. */ -static arena_t *a0; /* arenas[0]. */ -unsigned narenas_auto; -unsigned manual_arena_base; +static arena_t *a0; /* arenas[0]. */ +unsigned narenas_auto; +unsigned manual_arena_base; malloc_init_t malloc_init_state = malloc_init_uninitialized; /* False should be the common case. Set to true to trigger initialization. */ -bool malloc_slow = true; +bool malloc_slow = true; /* When malloc_slow is true, set the corresponding bits for sanity check. */ enum { - flag_opt_junk_alloc = (1U), - flag_opt_junk_free = (1U << 1), - flag_opt_zero = (1U << 2), - flag_opt_utrace = (1U << 3), - flag_opt_xmalloc = (1U << 4) + flag_opt_junk_alloc = (1U), + flag_opt_junk_free = (1U << 1), + flag_opt_zero = (1U << 2), + flag_opt_utrace = (1U << 3), + flag_opt_xmalloc = (1U << 4) }; -static uint8_t malloc_slow_flags; +static uint8_t malloc_slow_flags; #ifdef JEMALLOC_THREADED_INIT /* Used to let the initializing thread recursively allocate. */ -# define NO_INITIALIZER ((unsigned long)0) -# define INITIALIZER pthread_self() -# define IS_INITIALIZER (malloc_initializer == pthread_self()) -static pthread_t malloc_initializer = NO_INITIALIZER; +# define NO_INITIALIZER ((unsigned long)0) +# define INITIALIZER pthread_self() +# define IS_INITIALIZER \ + (pthread_equal(malloc_initializer, pthread_self())) +static pthread_t malloc_initializer = NO_INITIALIZER; #else -# define NO_INITIALIZER false -# define INITIALIZER true -# define IS_INITIALIZER malloc_initializer -static bool malloc_initializer = NO_INITIALIZER; +# define NO_INITIALIZER false +# define INITIALIZER true +# define IS_INITIALIZER malloc_initializer +static bool malloc_initializer = NO_INITIALIZER; #endif /* Used to avoid initialization races. */ #ifdef _WIN32 -#if _WIN32_WINNT >= 0x0600 -static malloc_mutex_t init_lock = SRWLOCK_INIT; -#else -static malloc_mutex_t init_lock; -static bool init_lock_initialized = false; +# if _WIN32_WINNT >= 0x0600 +static malloc_mutex_t init_lock = SRWLOCK_INIT; +# else +static malloc_mutex_t init_lock; +static bool init_lock_initialized = false; JEMALLOC_ATTR(constructor) static void WINAPI @@ -236,40 +259,40 @@ _init_init_lock(void) { init_lock_initialized = true; } -#ifdef _MSC_VER -# pragma section(".CRT$XCU", read) -JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used) -static const void (WINAPI *init_init_lock)(void) = _init_init_lock; -#endif -#endif +# ifdef _MSC_VER +# pragma section(".CRT$XCU", read) +JEMALLOC_SECTION(".CRT$XCU") +JEMALLOC_ATTR(used) +static const void(WINAPI *init_init_lock)(void) = _init_init_lock; +# endif +# endif #else -static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER; +static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER; #endif typedef struct { - void *p; /* Input pointer (as in realloc(p, s)). */ - size_t s; /* Request size. */ - void *r; /* Result pointer. */ + void *p; /* Input pointer (as in realloc(p, s)). */ + size_t s; /* Request size. */ + void *r; /* Result pointer. */ } malloc_utrace_t; #ifdef JEMALLOC_UTRACE -# define UTRACE(a, b, c) do { \ - if (unlikely(opt_utrace)) { \ - int utrace_serrno = errno; \ - malloc_utrace_t ut; \ - ut.p = (a); \ - ut.s = (b); \ - ut.r = (c); \ - UTRACE_CALL(&ut, sizeof(ut)); \ - errno = utrace_serrno; \ - } \ -} while (0) +# define UTRACE(a, b, c) \ + do { \ + if (unlikely(opt_utrace)) { \ + int utrace_serrno = errno; \ + malloc_utrace_t ut; \ + ut.p = (a); \ + ut.s = (b); \ + ut.r = (c); \ + UTRACE_CALL(&ut, sizeof(ut)); \ + errno = utrace_serrno; \ + } \ + } while (0) #else -# define UTRACE(a, b, c) +# define UTRACE(a, b, c) #endif -/* Whether encountered any invalid config options. */ -static bool had_conf_error = false; /******************************************************************************/ /* @@ -277,8 +300,8 @@ static bool had_conf_error = false; * definition. */ -static bool malloc_init_hard_a0(void); -static bool malloc_init_hard(void); +static bool malloc_init_hard_a0(void); +static bool malloc_init_hard(void); /******************************************************************************/ /* @@ -422,14 +445,13 @@ arena_new_create_background_thread(tsdn_t *tsdn, unsigned ind) { if (ind == 0) { return; } - /* - * Avoid creating a new background thread just for the huge arena, which - * purges eagerly by default. - */ - if (have_background_thread && !arena_is_huge(ind)) { + + if (have_background_thread) { if (background_thread_create(tsdn_tsd(tsdn), ind)) { - malloc_printf(": error in background thread " - "creation for arena %u. Abort.\n", ind); + malloc_printf( + ": error in background thread " + "creation for arena %u. Abort.\n", + ind); abort(); } } @@ -457,12 +479,16 @@ arena_bind(tsd_t *tsd, unsigned ind, bool internal) { tsd_iarena_set(tsd, arena); } else { tsd_arena_set(tsd, arena); - unsigned shard = atomic_fetch_add_u(&arena->binshard_next, 1, - ATOMIC_RELAXED); + /* + * While shard acts as a random seed, the cast below should + * not make much difference. + */ + uint8_t shard = (uint8_t)atomic_fetch_add_u( + &arena->binshard_next, 1, ATOMIC_RELAXED); tsd_binshards_t *bins = tsd_binshardsp_get(tsd); for (unsigned i = 0; i < SC_NBINS; i++) { - assert(bin_infos[i].n_shards > 0 && - bin_infos[i].n_shards <= BIN_SHARDS_MAX); + assert(bin_infos[i].n_shards > 0 + && bin_infos[i].n_shards <= BIN_SHARDS_MAX); bins->binshard[i] = shard % bin_infos[i].n_shards; } } @@ -477,8 +503,12 @@ arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena) { arena_nthreads_inc(newarena, false); tsd_arena_set(tsd, newarena); - if (arena_nthreads_get(oldarena, false) == 0) { - /* Purge if the old arena has no associated threads anymore. */ + if (arena_nthreads_get(oldarena, false) == 0 + && !background_thread_enabled()) { + /* + * Purge if the old arena has no associated threads anymore and + * no background threads. + */ arena_decay(tsd_tsdn(tsd), oldarena, /* is_background_thread */ false, /* all */ true); } @@ -515,7 +545,7 @@ arena_choose_hard(tsd_t *tsd, bool internal) { if (narenas_auto > 1) { unsigned i, j, choose[2], first_null; - bool is_new_arena[2]; + bool is_new_arena[2]; /* * Determine binding for both non-internal and internal @@ -540,11 +570,14 @@ arena_choose_hard(tsd_t *tsd, bool internal) { * number of threads assigned to it. */ for (j = 0; j < 2; j++) { - if (arena_nthreads_get(arena_get( - tsd_tsdn(tsd), i, false), !!j) < - arena_nthreads_get(arena_get( - tsd_tsdn(tsd), choose[j], false), - !!j)) { + if (arena_nthreads_get( + arena_get( + tsd_tsdn(tsd), i, false), + !!j) + < arena_nthreads_get( + arena_get(tsd_tsdn(tsd), + choose[j], false), + !!j)) { choose[j] = i; } } @@ -563,16 +596,17 @@ arena_choose_hard(tsd_t *tsd, bool internal) { } for (j = 0; j < 2; j++) { - if (arena_nthreads_get(arena_get(tsd_tsdn(tsd), - choose[j], false), !!j) == 0 || first_null == - narenas_auto) { + if (arena_nthreads_get( + arena_get(tsd_tsdn(tsd), choose[j], false), !!j) + == 0 + || first_null == narenas_auto) { /* * Use an unloaded arena, or the least loaded * arena if all arenas are already initialized. */ if (!!j == internal) { - ret = arena_get(tsd_tsdn(tsd), - choose[j], false); + ret = arena_get( + tsd_tsdn(tsd), choose[j], false); } } else { arena_t *arena; @@ -582,8 +616,8 @@ arena_choose_hard(tsd_t *tsd, bool internal) { arena = arena_init_locked(tsd_tsdn(tsd), choose[j], &arena_config_default); if (arena == NULL) { - malloc_mutex_unlock(tsd_tsdn(tsd), - &arenas_lock); + malloc_mutex_unlock( + tsd_tsdn(tsd), &arenas_lock); return NULL; } is_new_arena[j] = true; @@ -635,7 +669,7 @@ arena_cleanup(tsd_t *tsd) { static void stats_print_atexit(void) { if (config_stats) { - tsdn_t *tsdn; + tsdn_t *tsdn; unsigned narenas, i; tsdn = tsdn_fetch(); @@ -653,13 +687,13 @@ stats_print_atexit(void) { tcache_slow_t *tcache_slow; malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); - ql_foreach(tcache_slow, &arena->tcache_ql, - link) { - tcache_stats_merge(tsdn, - tcache_slow->tcache, arena); + ql_foreach ( + tcache_slow, &arena->tcache_ql, link) { + tcache_stats_merge( + tsdn, tcache_slow->tcache, arena); } - malloc_mutex_unlock(tsdn, - &arena->tcache_ql_mtx); + malloc_mutex_unlock( + tsdn, &arena->tcache_ql_mtx); } } } @@ -699,20 +733,6 @@ check_entry_exit_locking(tsdn_t *tsdn) { * Begin initialization functions. */ -static char * -jemalloc_secure_getenv(const char *name) { -#ifdef JEMALLOC_HAVE_SECURE_GETENV - return secure_getenv(name); -#else -# ifdef JEMALLOC_HAVE_ISSETUGID - if (issetugid() != 0) { - return NULL; - } -# endif - return getenv(name); -#endif -} - static unsigned malloc_ncpus(void) { long result; @@ -733,16 +753,16 @@ malloc_ncpus(void) { * is available, to avoid using more arenas than necessary. */ { -# if defined(__FreeBSD__) || defined(__DragonFly__) +# if defined(__FreeBSD__) || defined(__DragonFly__) cpuset_t set; -# else +# else cpu_set_t set; -# endif -# if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) +# endif +# if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) sched_getaffinity(0, sizeof(set), &set); -# else +# else pthread_getaffinity_np(pthread_self(), sizeof(set), &set); -# endif +# endif result = CPU_COUNT(&set); } #else @@ -759,8 +779,7 @@ malloc_ncpus(void) { * Since otherwise tricky things is possible with percpu arenas in use. */ static bool -malloc_cpu_count_is_deterministic() -{ +malloc_cpu_count_is_deterministic(void) { #ifdef _WIN32 return true; #else @@ -769,195 +788,26 @@ malloc_cpu_count_is_deterministic() if (cpu_onln != cpu_conf) { return false; } -# if defined(CPU_COUNT) -# if defined(__FreeBSD__) || defined(__DragonFly__) +# if defined(CPU_COUNT) +# if defined(__FreeBSD__) || defined(__DragonFly__) cpuset_t set; -# else +# else cpu_set_t set; -# endif /* __FreeBSD__ */ -# if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) +# endif /* __FreeBSD__ */ +# if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) sched_getaffinity(0, sizeof(set), &set); -# else /* !JEMALLOC_HAVE_SCHED_SETAFFINITY */ +# else /* !JEMALLOC_HAVE_SCHED_SETAFFINITY */ pthread_getaffinity_np(pthread_self(), sizeof(set), &set); -# endif /* JEMALLOC_HAVE_SCHED_SETAFFINITY */ +# endif /* JEMALLOC_HAVE_SCHED_SETAFFINITY */ long cpu_affinity = CPU_COUNT(&set); if (cpu_affinity != cpu_conf) { return false; } -# endif /* CPU_COUNT */ +# endif /* CPU_COUNT */ return true; #endif } -static void -init_opt_stats_opts(const char *v, size_t vlen, char *dest) { - size_t opts_len = strlen(dest); - assert(opts_len <= stats_print_tot_num_options); - - for (size_t i = 0; i < vlen; i++) { - switch (v[i]) { -#define OPTION(o, v, d, s) case o: break; - STATS_PRINT_OPTIONS -#undef OPTION - default: continue; - } - - if (strchr(dest, v[i]) != NULL) { - /* Ignore repeated. */ - continue; - } - - dest[opts_len++] = v[i]; - dest[opts_len] = '\0'; - assert(opts_len <= stats_print_tot_num_options); - } - assert(opts_len == strlen(dest)); -} - -/* Reads the next size pair in a multi-sized option. */ -static bool -malloc_conf_multi_sizes_next(const char **slab_size_segment_cur, - size_t *vlen_left, size_t *slab_start, size_t *slab_end, size_t *new_size) { - const char *cur = *slab_size_segment_cur; - char *end; - uintmax_t um; - - set_errno(0); - - /* First number, then '-' */ - um = malloc_strtoumax(cur, &end, 0); - if (get_errno() != 0 || *end != '-') { - return true; - } - *slab_start = (size_t)um; - cur = end + 1; - - /* Second number, then ':' */ - um = malloc_strtoumax(cur, &end, 0); - if (get_errno() != 0 || *end != ':') { - return true; - } - *slab_end = (size_t)um; - cur = end + 1; - - /* Last number */ - um = malloc_strtoumax(cur, &end, 0); - if (get_errno() != 0) { - return true; - } - *new_size = (size_t)um; - - /* Consume the separator if there is one. */ - if (*end == '|') { - end++; - } - - *vlen_left -= end - *slab_size_segment_cur; - *slab_size_segment_cur = end; - - return false; -} - -static bool -malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, - char const **v_p, size_t *vlen_p) { - bool accept; - const char *opts = *opts_p; - - *k_p = opts; - - for (accept = false; !accept;) { - switch (*opts) { - case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': - case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': - case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': - case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': - case 'Y': case 'Z': - case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': - case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': - case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': - case 's': case 't': case 'u': case 'v': case 'w': case 'x': - case 'y': case 'z': - case '0': case '1': case '2': case '3': case '4': case '5': - case '6': case '7': case '8': case '9': - case '_': - opts++; - break; - case ':': - opts++; - *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; - *v_p = opts; - accept = true; - break; - case '\0': - if (opts != *opts_p) { - malloc_write(": Conf string ends " - "with key\n"); - had_conf_error = true; - } - return true; - default: - malloc_write(": Malformed conf string\n"); - had_conf_error = true; - return true; - } - } - - for (accept = false; !accept;) { - switch (*opts) { - case ',': - opts++; - /* - * Look ahead one character here, because the next time - * this function is called, it will assume that end of - * input has been cleanly reached if no input remains, - * but we have optimistically already consumed the - * comma if one exists. - */ - if (*opts == '\0') { - malloc_write(": Conf string ends " - "with comma\n"); - had_conf_error = true; - } - *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; - accept = true; - break; - case '\0': - *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; - accept = true; - break; - default: - opts++; - break; - } - } - - *opts_p = opts; - return false; -} - -static void -malloc_abort_invalid_conf(void) { - assert(opt_abort_conf); - malloc_printf(": Abort (abort_conf:true) on invalid conf " - "value (see above).\n"); - abort(); -} - -static void -malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, - size_t vlen) { - malloc_printf(": %s: %.*s:%.*s\n", msg, (int)klen, k, - (int)vlen, v); - /* If abort_conf is set, error out after processing all options. */ - const char *experimental = "experimental_"; - if (strncmp(k, experimental, strlen(experimental)) == 0) { - /* However, tolerate experimental features. */ - return; - } - had_conf_error = true; -} - static void malloc_slow_flag_init(void) { /* @@ -973,797 +823,11 @@ malloc_slow_flag_init(void) { malloc_slow = (malloc_slow_flags != 0); } -/* Number of sources for initializing malloc_conf */ -#define MALLOC_CONF_NSOURCES 5 - -static const char * -obtain_malloc_conf(unsigned which_source, char buf[PATH_MAX + 1]) { - if (config_debug) { - static unsigned read_source = 0; - /* - * Each source should only be read once, to minimize # of - * syscalls on init. - */ - assert(read_source++ == which_source); - } - assert(which_source < MALLOC_CONF_NSOURCES); - - const char *ret; - switch (which_source) { - case 0: - ret = config_malloc_conf; - break; - case 1: - if (je_malloc_conf != NULL) { - /* Use options that were compiled into the program. */ - ret = je_malloc_conf; - } else { - /* No configuration specified. */ - ret = NULL; - } - break; - case 2: { - ssize_t linklen = 0; -#ifndef _WIN32 - int saved_errno = errno; - const char *linkname = -# ifdef JEMALLOC_PREFIX - "/etc/"JEMALLOC_PREFIX"malloc.conf" -# else - "/etc/malloc.conf" -# endif - ; - - /* - * Try to use the contents of the "/etc/malloc.conf" symbolic - * link's name. - */ -#ifndef JEMALLOC_READLINKAT - linklen = readlink(linkname, buf, PATH_MAX); -#else - linklen = readlinkat(AT_FDCWD, linkname, buf, PATH_MAX); -#endif - if (linklen == -1) { - /* No configuration specified. */ - linklen = 0; - /* Restore errno. */ - set_errno(saved_errno); - } -#endif - buf[linklen] = '\0'; - ret = buf; - break; - } case 3: { - const char *envname = -#ifdef JEMALLOC_PREFIX - JEMALLOC_CPREFIX"MALLOC_CONF" -#else - "MALLOC_CONF" -#endif - ; - - if ((ret = jemalloc_secure_getenv(envname)) != NULL) { - /* - * Do nothing; opts is already initialized to the value - * of the MALLOC_CONF environment variable. - */ - } else { - /* No configuration specified. */ - ret = NULL; - } - break; - } case 4: { - ret = je_malloc_conf_2_conf_harder; - break; - } default: - not_reached(); - ret = NULL; - } - return ret; -} - -static void -malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], - bool initial_call, const char *opts_cache[MALLOC_CONF_NSOURCES], - char buf[PATH_MAX + 1]) { - static const char *opts_explain[MALLOC_CONF_NSOURCES] = { - "string specified via --with-malloc-conf", - "string pointed to by the global variable malloc_conf", - "\"name\" of the file referenced by the symbolic link named " - "/etc/malloc.conf", - "value of the environment variable MALLOC_CONF", - "string pointed to by the global variable " - "malloc_conf_2_conf_harder", - }; - unsigned i; - const char *opts, *k, *v; - size_t klen, vlen; - - for (i = 0; i < MALLOC_CONF_NSOURCES; i++) { - /* Get runtime configuration. */ - if (initial_call) { - opts_cache[i] = obtain_malloc_conf(i, buf); - } - opts = opts_cache[i]; - if (!initial_call && opt_confirm_conf) { - malloc_printf( - ": malloc_conf #%u (%s): \"%s\"\n", - i + 1, opts_explain[i], opts != NULL ? opts : ""); - } - if (opts == NULL) { - continue; - } - - while (*opts != '\0' && !malloc_conf_next(&opts, &k, &klen, &v, - &vlen)) { - -#define CONF_ERROR(msg, k, klen, v, vlen) \ - if (!initial_call) { \ - malloc_conf_error( \ - msg, k, klen, v, vlen); \ - cur_opt_valid = false; \ - } -#define CONF_CONTINUE { \ - if (!initial_call && opt_confirm_conf \ - && cur_opt_valid) { \ - malloc_printf(": -- " \ - "Set conf value: %.*s:%.*s" \ - "\n", (int)klen, k, \ - (int)vlen, v); \ - } \ - continue; \ - } -#define CONF_MATCH(n) \ - (sizeof(n)-1 == klen && strncmp(n, k, klen) == 0) -#define CONF_MATCH_VALUE(n) \ - (sizeof(n)-1 == vlen && strncmp(n, v, vlen) == 0) -#define CONF_HANDLE_BOOL(o, n) \ - if (CONF_MATCH(n)) { \ - if (CONF_MATCH_VALUE("true")) { \ - o = true; \ - } else if (CONF_MATCH_VALUE("false")) { \ - o = false; \ - } else { \ - CONF_ERROR("Invalid conf value",\ - k, klen, v, vlen); \ - } \ - CONF_CONTINUE; \ - } - /* - * One of the CONF_MIN macros below expands, in one of the use points, - * to "unsigned integer < 0", which is always false, triggering the - * GCC -Wtype-limits warning, which we disable here and re-enable below. - */ - JEMALLOC_DIAGNOSTIC_PUSH - JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS - -#define CONF_DONT_CHECK_MIN(um, min) false -#define CONF_CHECK_MIN(um, min) ((um) < (min)) -#define CONF_DONT_CHECK_MAX(um, max) false -#define CONF_CHECK_MAX(um, max) ((um) > (max)) - -#define CONF_VALUE_READ(max_t, result) \ - char *end; \ - set_errno(0); \ - result = (max_t)malloc_strtoumax(v, &end, 0); -#define CONF_VALUE_READ_FAIL() \ - (get_errno() != 0 || (uintptr_t)end - (uintptr_t)v != vlen) - -#define CONF_HANDLE_T(t, max_t, o, n, min, max, check_min, check_max, clip) \ - if (CONF_MATCH(n)) { \ - max_t mv; \ - CONF_VALUE_READ(max_t, mv) \ - if (CONF_VALUE_READ_FAIL()) { \ - CONF_ERROR("Invalid conf value",\ - k, klen, v, vlen); \ - } else if (clip) { \ - if (check_min(mv, (t)(min))) { \ - o = (t)(min); \ - } else if ( \ - check_max(mv, (t)(max))) { \ - o = (t)(max); \ - } else { \ - o = (t)mv; \ - } \ - } else { \ - if (check_min(mv, (t)(min)) || \ - check_max(mv, (t)(max))) { \ - CONF_ERROR( \ - "Out-of-range " \ - "conf value", \ - k, klen, v, vlen); \ - } else { \ - o = (t)mv; \ - } \ - } \ - CONF_CONTINUE; \ - } -#define CONF_HANDLE_T_U(t, o, n, min, max, check_min, check_max, clip) \ - CONF_HANDLE_T(t, uintmax_t, o, n, min, max, check_min, \ - check_max, clip) -#define CONF_HANDLE_T_SIGNED(t, o, n, min, max, check_min, check_max, clip)\ - CONF_HANDLE_T(t, intmax_t, o, n, min, max, check_min, \ - check_max, clip) - -#define CONF_HANDLE_UNSIGNED(o, n, min, max, check_min, check_max, \ - clip) \ - CONF_HANDLE_T_U(unsigned, o, n, min, max, \ - check_min, check_max, clip) -#define CONF_HANDLE_SIZE_T(o, n, min, max, check_min, check_max, clip) \ - CONF_HANDLE_T_U(size_t, o, n, min, max, \ - check_min, check_max, clip) -#define CONF_HANDLE_INT64_T(o, n, min, max, check_min, check_max, clip) \ - CONF_HANDLE_T_SIGNED(int64_t, o, n, min, max, \ - check_min, check_max, clip) -#define CONF_HANDLE_UINT64_T(o, n, min, max, check_min, check_max, clip)\ - CONF_HANDLE_T_U(uint64_t, o, n, min, max, \ - check_min, check_max, clip) -#define CONF_HANDLE_SSIZE_T(o, n, min, max) \ - CONF_HANDLE_T_SIGNED(ssize_t, o, n, min, max, \ - CONF_CHECK_MIN, CONF_CHECK_MAX, false) -#define CONF_HANDLE_CHAR_P(o, n, d) \ - if (CONF_MATCH(n)) { \ - size_t cpylen = (vlen <= \ - sizeof(o)-1) ? vlen : \ - sizeof(o)-1; \ - strncpy(o, v, cpylen); \ - o[cpylen] = '\0'; \ - CONF_CONTINUE; \ - } - - bool cur_opt_valid = true; - - CONF_HANDLE_BOOL(opt_confirm_conf, "confirm_conf") - if (initial_call) { - continue; - } - - CONF_HANDLE_BOOL(opt_abort, "abort") - CONF_HANDLE_BOOL(opt_abort_conf, "abort_conf") - CONF_HANDLE_BOOL(opt_trust_madvise, "trust_madvise") - if (strncmp("metadata_thp", k, klen) == 0) { - int m; - bool match = false; - for (m = 0; m < metadata_thp_mode_limit; m++) { - if (strncmp(metadata_thp_mode_names[m], - v, vlen) == 0) { - opt_metadata_thp = m; - match = true; - break; - } - } - if (!match) { - CONF_ERROR("Invalid conf value", - k, klen, v, vlen); - } - CONF_CONTINUE; - } - CONF_HANDLE_BOOL(opt_retain, "retain") - if (strncmp("dss", k, klen) == 0) { - int m; - bool match = false; - for (m = 0; m < dss_prec_limit; m++) { - if (strncmp(dss_prec_names[m], v, vlen) - == 0) { - if (extent_dss_prec_set(m)) { - CONF_ERROR( - "Error setting dss", - k, klen, v, vlen); - } else { - opt_dss = - dss_prec_names[m]; - match = true; - break; - } - } - } - if (!match) { - CONF_ERROR("Invalid conf value", - k, klen, v, vlen); - } - CONF_CONTINUE; - } - if (CONF_MATCH("narenas")) { - if (CONF_MATCH_VALUE("default")) { - opt_narenas = 0; - CONF_CONTINUE; - } else { - CONF_HANDLE_UNSIGNED(opt_narenas, - "narenas", 1, UINT_MAX, - CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, - /* clip */ false) - } - } - if (CONF_MATCH("narenas_ratio")) { - char *end; - bool err = fxp_parse(&opt_narenas_ratio, v, - &end); - if (err || (size_t)(end - v) != vlen) { - CONF_ERROR("Invalid conf value", - k, klen, v, vlen); - } - CONF_CONTINUE; - } - if (CONF_MATCH("bin_shards")) { - const char *bin_shards_segment_cur = v; - size_t vlen_left = vlen; - do { - size_t size_start; - size_t size_end; - size_t nshards; - bool err = malloc_conf_multi_sizes_next( - &bin_shards_segment_cur, &vlen_left, - &size_start, &size_end, &nshards); - if (err || bin_update_shard_size( - bin_shard_sizes, size_start, - size_end, nshards)) { - CONF_ERROR( - "Invalid settings for " - "bin_shards", k, klen, v, - vlen); - break; - } - } while (vlen_left > 0); - CONF_CONTINUE; - } - CONF_HANDLE_INT64_T(opt_mutex_max_spin, - "mutex_max_spin", -1, INT64_MAX, CONF_CHECK_MIN, - CONF_DONT_CHECK_MAX, false); - CONF_HANDLE_SSIZE_T(opt_dirty_decay_ms, - "dirty_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) < - QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) : - SSIZE_MAX); - CONF_HANDLE_SSIZE_T(opt_muzzy_decay_ms, - "muzzy_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) < - QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) : - SSIZE_MAX); - CONF_HANDLE_BOOL(opt_stats_print, "stats_print") - if (CONF_MATCH("stats_print_opts")) { - init_opt_stats_opts(v, vlen, - opt_stats_print_opts); - CONF_CONTINUE; - } - CONF_HANDLE_INT64_T(opt_stats_interval, - "stats_interval", -1, INT64_MAX, - CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, false) - if (CONF_MATCH("stats_interval_opts")) { - init_opt_stats_opts(v, vlen, - opt_stats_interval_opts); - CONF_CONTINUE; - } - if (config_fill) { - if (CONF_MATCH("junk")) { - if (CONF_MATCH_VALUE("true")) { - opt_junk = "true"; - opt_junk_alloc = opt_junk_free = - true; - } else if (CONF_MATCH_VALUE("false")) { - opt_junk = "false"; - opt_junk_alloc = opt_junk_free = - false; - } else if (CONF_MATCH_VALUE("alloc")) { - opt_junk = "alloc"; - opt_junk_alloc = true; - opt_junk_free = false; - } else if (CONF_MATCH_VALUE("free")) { - opt_junk = "free"; - opt_junk_alloc = false; - opt_junk_free = true; - } else { - CONF_ERROR( - "Invalid conf value", - k, klen, v, vlen); - } - CONF_CONTINUE; - } - CONF_HANDLE_BOOL(opt_zero, "zero") - } - if (config_utrace) { - CONF_HANDLE_BOOL(opt_utrace, "utrace") - } - if (config_xmalloc) { - CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc") - } - if (config_enable_cxx) { - CONF_HANDLE_BOOL( - opt_experimental_infallible_new, - "experimental_infallible_new") - } - - CONF_HANDLE_BOOL(opt_tcache, "tcache") - CONF_HANDLE_SIZE_T(opt_tcache_max, "tcache_max", - 0, TCACHE_MAXCLASS_LIMIT, CONF_DONT_CHECK_MIN, - CONF_CHECK_MAX, /* clip */ true) - if (CONF_MATCH("lg_tcache_max")) { - size_t m; - CONF_VALUE_READ(size_t, m) - if (CONF_VALUE_READ_FAIL()) { - CONF_ERROR("Invalid conf value", - k, klen, v, vlen); - } else { - /* clip if necessary */ - if (m > TCACHE_LG_MAXCLASS_LIMIT) { - m = TCACHE_LG_MAXCLASS_LIMIT; - } - opt_tcache_max = (size_t)1 << m; - } - CONF_CONTINUE; - } - /* - * Anyone trying to set a value outside -16 to 16 is - * deeply confused. - */ - CONF_HANDLE_SSIZE_T(opt_lg_tcache_nslots_mul, - "lg_tcache_nslots_mul", -16, 16) - /* Ditto with values past 2048. */ - CONF_HANDLE_UNSIGNED(opt_tcache_nslots_small_min, - "tcache_nslots_small_min", 1, 2048, - CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true) - CONF_HANDLE_UNSIGNED(opt_tcache_nslots_small_max, - "tcache_nslots_small_max", 1, 2048, - CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true) - CONF_HANDLE_UNSIGNED(opt_tcache_nslots_large, - "tcache_nslots_large", 1, 2048, - CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true) - CONF_HANDLE_SIZE_T(opt_tcache_gc_incr_bytes, - "tcache_gc_incr_bytes", 1024, SIZE_T_MAX, - CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, - /* clip */ true) - CONF_HANDLE_SIZE_T(opt_tcache_gc_delay_bytes, - "tcache_gc_delay_bytes", 0, SIZE_T_MAX, - CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, - /* clip */ false) - CONF_HANDLE_UNSIGNED(opt_lg_tcache_flush_small_div, - "lg_tcache_flush_small_div", 1, 16, - CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true) - CONF_HANDLE_UNSIGNED(opt_lg_tcache_flush_large_div, - "lg_tcache_flush_large_div", 1, 16, - CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true) - - /* - * The runtime option of oversize_threshold remains - * undocumented. It may be tweaked in the next major - * release (6.0). The default value 8M is rather - * conservative / safe. Tuning it further down may - * improve fragmentation a bit more, but may also cause - * contention on the huge arena. - */ - CONF_HANDLE_SIZE_T(opt_oversize_threshold, - "oversize_threshold", 0, SC_LARGE_MAXCLASS, - CONF_DONT_CHECK_MIN, CONF_CHECK_MAX, false) - CONF_HANDLE_SIZE_T(opt_lg_extent_max_active_fit, - "lg_extent_max_active_fit", 0, - (sizeof(size_t) << 3), CONF_DONT_CHECK_MIN, - CONF_CHECK_MAX, false) - - if (strncmp("percpu_arena", k, klen) == 0) { - bool match = false; - for (int m = percpu_arena_mode_names_base; m < - percpu_arena_mode_names_limit; m++) { - if (strncmp(percpu_arena_mode_names[m], - v, vlen) == 0) { - if (!have_percpu_arena) { - CONF_ERROR( - "No getcpu support", - k, klen, v, vlen); - } - opt_percpu_arena = m; - match = true; - break; - } - } - if (!match) { - CONF_ERROR("Invalid conf value", - k, klen, v, vlen); - } - CONF_CONTINUE; - } - CONF_HANDLE_BOOL(opt_background_thread, - "background_thread"); - CONF_HANDLE_SIZE_T(opt_max_background_threads, - "max_background_threads", 1, - opt_max_background_threads, - CONF_CHECK_MIN, CONF_CHECK_MAX, - true); - CONF_HANDLE_BOOL(opt_hpa, "hpa") - CONF_HANDLE_SIZE_T(opt_hpa_opts.slab_max_alloc, - "hpa_slab_max_alloc", PAGE, HUGEPAGE, - CONF_CHECK_MIN, CONF_CHECK_MAX, true); - - /* - * Accept either a ratio-based or an exact hugification - * threshold. - */ - CONF_HANDLE_SIZE_T(opt_hpa_opts.hugification_threshold, - "hpa_hugification_threshold", PAGE, HUGEPAGE, - CONF_CHECK_MIN, CONF_CHECK_MAX, true); - if (CONF_MATCH("hpa_hugification_threshold_ratio")) { - fxp_t ratio; - char *end; - bool err = fxp_parse(&ratio, v, - &end); - if (err || (size_t)(end - v) != vlen - || ratio > FXP_INIT_INT(1)) { - CONF_ERROR("Invalid conf value", - k, klen, v, vlen); - } else { - opt_hpa_opts.hugification_threshold = - fxp_mul_frac(HUGEPAGE, ratio); - } - CONF_CONTINUE; - } - - CONF_HANDLE_UINT64_T( - opt_hpa_opts.hugify_delay_ms, "hpa_hugify_delay_ms", - 0, 0, CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, - false); - - CONF_HANDLE_UINT64_T( - opt_hpa_opts.min_purge_interval_ms, - "hpa_min_purge_interval_ms", 0, 0, - CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, false); - - if (CONF_MATCH("hpa_dirty_mult")) { - if (CONF_MATCH_VALUE("-1")) { - opt_hpa_opts.dirty_mult = (fxp_t)-1; - CONF_CONTINUE; - } - fxp_t ratio; - char *end; - bool err = fxp_parse(&ratio, v, - &end); - if (err || (size_t)(end - v) != vlen) { - CONF_ERROR("Invalid conf value", - k, klen, v, vlen); - } else { - opt_hpa_opts.dirty_mult = ratio; - } - CONF_CONTINUE; - } - - CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.nshards, - "hpa_sec_nshards", 0, 0, CONF_CHECK_MIN, - CONF_DONT_CHECK_MAX, true); - CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_alloc, - "hpa_sec_max_alloc", PAGE, 0, CONF_CHECK_MIN, - CONF_DONT_CHECK_MAX, true); - CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_bytes, - "hpa_sec_max_bytes", PAGE, 0, CONF_CHECK_MIN, - CONF_DONT_CHECK_MAX, true); - CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.bytes_after_flush, - "hpa_sec_bytes_after_flush", PAGE, 0, - CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true); - CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.batch_fill_extra, - "hpa_sec_batch_fill_extra", 0, HUGEPAGE_PAGES, - CONF_CHECK_MIN, CONF_CHECK_MAX, true); - - if (CONF_MATCH("slab_sizes")) { - if (CONF_MATCH_VALUE("default")) { - sc_data_init(sc_data); - CONF_CONTINUE; - } - bool err; - const char *slab_size_segment_cur = v; - size_t vlen_left = vlen; - do { - size_t slab_start; - size_t slab_end; - size_t pgs; - err = malloc_conf_multi_sizes_next( - &slab_size_segment_cur, - &vlen_left, &slab_start, &slab_end, - &pgs); - if (!err) { - sc_data_update_slab_size( - sc_data, slab_start, - slab_end, (int)pgs); - } else { - CONF_ERROR("Invalid settings " - "for slab_sizes", - k, klen, v, vlen); - } - } while (!err && vlen_left > 0); - CONF_CONTINUE; - } - if (config_prof) { - CONF_HANDLE_BOOL(opt_prof, "prof") - CONF_HANDLE_CHAR_P(opt_prof_prefix, - "prof_prefix", "jeprof") - CONF_HANDLE_BOOL(opt_prof_active, "prof_active") - CONF_HANDLE_BOOL(opt_prof_thread_active_init, - "prof_thread_active_init") - CONF_HANDLE_SIZE_T(opt_lg_prof_sample, - "lg_prof_sample", 0, (sizeof(uint64_t) << 3) - - 1, CONF_DONT_CHECK_MIN, CONF_CHECK_MAX, - true) - CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum") - CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, - "lg_prof_interval", -1, - (sizeof(uint64_t) << 3) - 1) - CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump") - CONF_HANDLE_BOOL(opt_prof_final, "prof_final") - CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak") - CONF_HANDLE_BOOL(opt_prof_leak_error, - "prof_leak_error") - CONF_HANDLE_BOOL(opt_prof_log, "prof_log") - CONF_HANDLE_SSIZE_T(opt_prof_recent_alloc_max, - "prof_recent_alloc_max", -1, SSIZE_MAX) - CONF_HANDLE_BOOL(opt_prof_stats, "prof_stats") - CONF_HANDLE_BOOL(opt_prof_sys_thread_name, - "prof_sys_thread_name") - if (CONF_MATCH("prof_time_resolution")) { - if (CONF_MATCH_VALUE("default")) { - opt_prof_time_res = - prof_time_res_default; - } else if (CONF_MATCH_VALUE("high")) { - if (!config_high_res_timer) { - CONF_ERROR( - "No high resolution" - " timer support", - k, klen, v, vlen); - } else { - opt_prof_time_res = - prof_time_res_high; - } - } else { - CONF_ERROR("Invalid conf value", - k, klen, v, vlen); - } - CONF_CONTINUE; - } - /* - * Undocumented. When set to false, don't - * correct for an unbiasing bug in jeprof - * attribution. This can be handy if you want - * to get consistent numbers from your binary - * across different jemalloc versions, even if - * those numbers are incorrect. The default is - * true. - */ - CONF_HANDLE_BOOL(opt_prof_unbias, "prof_unbias") - } - if (config_log) { - if (CONF_MATCH("log")) { - size_t cpylen = ( - vlen <= sizeof(log_var_names) ? - vlen : sizeof(log_var_names) - 1); - strncpy(log_var_names, v, cpylen); - log_var_names[cpylen] = '\0'; - CONF_CONTINUE; - } - } - if (CONF_MATCH("thp")) { - bool match = false; - for (int m = 0; m < thp_mode_names_limit; m++) { - if (strncmp(thp_mode_names[m],v, vlen) - == 0) { - if (!have_madvise_huge && !have_memcntl) { - CONF_ERROR( - "No THP support", - k, klen, v, vlen); - } - opt_thp = m; - match = true; - break; - } - } - if (!match) { - CONF_ERROR("Invalid conf value", - k, klen, v, vlen); - } - CONF_CONTINUE; - } - if (CONF_MATCH("zero_realloc")) { - if (CONF_MATCH_VALUE("alloc")) { - opt_zero_realloc_action - = zero_realloc_action_alloc; - } else if (CONF_MATCH_VALUE("free")) { - opt_zero_realloc_action - = zero_realloc_action_free; - } else if (CONF_MATCH_VALUE("abort")) { - opt_zero_realloc_action - = zero_realloc_action_abort; - } else { - CONF_ERROR("Invalid conf value", - k, klen, v, vlen); - } - CONF_CONTINUE; - } - if (config_uaf_detection && - CONF_MATCH("lg_san_uaf_align")) { - ssize_t a; - CONF_VALUE_READ(ssize_t, a) - if (CONF_VALUE_READ_FAIL() || a < -1) { - CONF_ERROR("Invalid conf value", - k, klen, v, vlen); - } - if (a == -1) { - opt_lg_san_uaf_align = -1; - CONF_CONTINUE; - } - - /* clip if necessary */ - ssize_t max_allowed = (sizeof(size_t) << 3) - 1; - ssize_t min_allowed = LG_PAGE; - if (a > max_allowed) { - a = max_allowed; - } else if (a < min_allowed) { - a = min_allowed; - } - - opt_lg_san_uaf_align = a; - CONF_CONTINUE; - } - - CONF_HANDLE_SIZE_T(opt_san_guard_small, - "san_guard_small", 0, SIZE_T_MAX, - CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, false) - CONF_HANDLE_SIZE_T(opt_san_guard_large, - "san_guard_large", 0, SIZE_T_MAX, - CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, false) - - CONF_ERROR("Invalid conf pair", k, klen, v, vlen); -#undef CONF_ERROR -#undef CONF_CONTINUE -#undef CONF_MATCH -#undef CONF_MATCH_VALUE -#undef CONF_HANDLE_BOOL -#undef CONF_DONT_CHECK_MIN -#undef CONF_CHECK_MIN -#undef CONF_DONT_CHECK_MAX -#undef CONF_CHECK_MAX -#undef CONF_HANDLE_T -#undef CONF_HANDLE_T_U -#undef CONF_HANDLE_T_SIGNED -#undef CONF_HANDLE_UNSIGNED -#undef CONF_HANDLE_SIZE_T -#undef CONF_HANDLE_SSIZE_T -#undef CONF_HANDLE_CHAR_P - /* Re-enable diagnostic "-Wtype-limits" */ - JEMALLOC_DIAGNOSTIC_POP - } - if (opt_abort_conf && had_conf_error) { - malloc_abort_invalid_conf(); - } - } - atomic_store_b(&log_init_done, true, ATOMIC_RELEASE); -} - -static bool -malloc_conf_init_check_deps(void) { - if (opt_prof_leak_error && !opt_prof_final) { - malloc_printf(": prof_leak_error is set w/o " - "prof_final.\n"); - return true; - } - - return false; -} - -static void -malloc_conf_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]) { - const char *opts_cache[MALLOC_CONF_NSOURCES] = {NULL, NULL, NULL, NULL, - NULL}; - char buf[PATH_MAX + 1]; - - /* The first call only set the confirm_conf option and opts_cache */ - malloc_conf_init_helper(NULL, NULL, true, opts_cache, buf); - malloc_conf_init_helper(sc_data, bin_shard_sizes, false, opts_cache, - NULL); - if (malloc_conf_init_check_deps()) { - /* check_deps does warning msg only; abort below if needed. */ - if (opt_abort_conf) { - malloc_abort_invalid_conf(); - } - } -} - -#undef MALLOC_CONF_NSOURCES - static bool malloc_init_hard_needed(void) { - if (malloc_initialized() || (IS_INITIALIZER && malloc_init_state == - malloc_init_recursible)) { + if (malloc_initialized() + || (IS_INITIALIZER + && malloc_init_state == malloc_init_recursible)) { /* * Another thread initialized the allocator before this one * acquired init_lock, or this thread is the initializing @@ -1787,7 +851,7 @@ malloc_init_hard_needed(void) { } static bool -malloc_init_hard_a0_locked() { +malloc_init_hard_a0_locked(void) { malloc_initializer = INITIALIZER; JEMALLOC_DIAGNOSTIC_PUSH @@ -1813,7 +877,9 @@ malloc_init_hard_a0_locked() { if (config_prof) { prof_boot0(); } - malloc_conf_init(&sc_data, bin_shard_sizes); + char readlink_buf[PATH_MAX + 1]; + readlink_buf[0] = '\0'; + malloc_conf_init(&sc_data, bin_shard_sizes, readlink_buf); san_init(opt_lg_san_uaf_align); sz_boot(&sc_data, opt_cache_oblivious); bin_info_boot(&sc_data, bin_shard_sizes); @@ -1851,7 +917,8 @@ malloc_init_hard_a0_locked() { prof_boot1(); } if (opt_hpa && !hpa_supported()) { - malloc_printf(": HPA not supported in the current " + malloc_printf( + ": HPA not supported in the current " "configuration; %s.", opt_abort_conf ? "aborting" : "disabling"); if (opt_abort_conf) { @@ -1867,10 +934,11 @@ malloc_init_hard_a0_locked() { return true; } if (malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS, - malloc_mutex_rank_exclusive)) { + malloc_mutex_rank_exclusive)) { return true; } hook_boot(); + experimental_thread_events_boot(); /* * Create enough scaffolding to allow recursive allocation in * malloc_ncpus(). @@ -1888,7 +956,8 @@ malloc_init_hard_a0_locked() { a0 = arena_get(TSDN_NULL, 0, false); if (opt_hpa && !hpa_supported()) { - malloc_printf(": HPA not supported in the current " + malloc_printf( + ": HPA not supported in the current " "configuration; %s.", opt_abort_conf ? "aborting" : "disabling"); if (opt_abort_conf) { @@ -1896,17 +965,19 @@ malloc_init_hard_a0_locked() { } else { opt_hpa = false; } - } else if (opt_hpa) { - hpa_shard_opts_t hpa_shard_opts = opt_hpa_opts; - hpa_shard_opts.deferral_allowed = background_thread_enabled(); - if (pa_shard_enable_hpa(TSDN_NULL, &a0->pa_shard, - &hpa_shard_opts, &opt_hpa_sec_opts)) { - return true; - } } malloc_init_state = malloc_init_a0_initialized; + size_t buf_len = strlen(readlink_buf); + if (buf_len > 0) { + void *readlink_allocated = a0ialloc(buf_len + 1, false, true); + if (readlink_allocated != NULL) { + memcpy(readlink_allocated, readlink_buf, buf_len + 1); + opt_malloc_conf_symlink = readlink_allocated; + } + } + return false; } @@ -1937,7 +1008,8 @@ malloc_init_hard_recursible(void) { */ if (opt_narenas == 0) { opt_percpu_arena = percpu_arena_disabled; - malloc_write(": Number of CPUs " + malloc_write( + ": Number of CPUs " "detected is not deterministic. Per-CPU " "arena disabled.\n"); if (opt_abort_conf) { @@ -1951,11 +1023,12 @@ malloc_init_hard_recursible(void) { } #if (defined(JEMALLOC_HAVE_PTHREAD_ATFORK) && !defined(JEMALLOC_MUTEX_INIT_CB) \ - && !defined(JEMALLOC_ZONE) && !defined(_WIN32) && \ - !defined(__native_client__)) + && !defined(JEMALLOC_ZONE) && !defined(_WIN32) \ + && !defined(__native_client__)) /* LinuxThreads' pthread_atfork() allocates. */ if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, - jemalloc_postfork_child) != 0) { + jemalloc_postfork_child) + != 0) { malloc_write(": Error in pthread_atfork()\n"); if (opt_abort) { abort(); @@ -1979,8 +1052,8 @@ malloc_narenas_default(void) { * default. */ if (ncpus > 1) { - fxp_t fxp_ncpus = FXP_INIT_INT(ncpus); - fxp_t goal = fxp_mul(fxp_ncpus, opt_narenas_ratio); + fxp_t fxp_ncpus = FXP_INIT_INT(ncpus); + fxp_t goal = fxp_mul(fxp_ncpus, opt_narenas_ratio); uint32_t int_goal = fxp_round_nearest(goal); if (int_goal == 0) { return 1; @@ -2004,34 +1077,41 @@ percpu_arena_as_initialized(percpu_arena_mode_t mode) { } static bool -malloc_init_narenas(void) { +malloc_init_narenas(tsdn_t *tsdn) { assert(ncpus > 0); if (opt_percpu_arena != percpu_arena_disabled) { if (!have_percpu_arena || malloc_getcpu() < 0) { opt_percpu_arena = percpu_arena_disabled; - malloc_printf(": perCPU arena getcpu() not " - "available. Setting narenas to %u.\n", opt_narenas ? - opt_narenas : malloc_narenas_default()); + malloc_printf( + ": perCPU arena getcpu() not " + "available. Setting narenas to %u.\n", + opt_narenas ? opt_narenas + : malloc_narenas_default()); if (opt_abort) { abort(); } } else { if (ncpus >= MALLOCX_ARENA_LIMIT) { - malloc_printf(": narenas w/ percpu" - "arena beyond limit (%d)\n", ncpus); + malloc_printf( + ": narenas w/ percpu" + "arena beyond limit (%d)\n", + ncpus); if (opt_abort) { abort(); } return true; } /* NB: opt_percpu_arena isn't fully initialized yet. */ - if (percpu_arena_as_initialized(opt_percpu_arena) == - per_phycpu_arena && ncpus % 2 != 0) { - malloc_printf(": invalid " + if (percpu_arena_as_initialized(opt_percpu_arena) + == per_phycpu_arena + && ncpus % 2 != 0) { + malloc_printf( + ": invalid " "configuration -- per physical CPU arena " "with odd number (%u) of CPUs (no hyper " - "threading?).\n", ncpus); + "threading?).\n", + ncpus); if (opt_abort) abort(); } @@ -2071,7 +1151,7 @@ malloc_init_narenas(void) { narenas_auto); } narenas_total_set(narenas_auto); - if (arena_init_huge()) { + if (arena_init_huge(tsdn, a0)) { narenas_total_inc(); } manual_arena_base = narenas_total_get(); @@ -2112,21 +1192,30 @@ static bool malloc_init_hard(void) { tsd_t *tsd; + assert(TCACHE_MAXCLASS_LIMIT <= USIZE_GROW_SLOW_THRESHOLD); + assert(SC_LOOKUP_MAXCLASS <= USIZE_GROW_SLOW_THRESHOLD); + /* + * This asserts an extreme case where TINY_MAXCLASS is larger + * than LARGE_MINCLASS. It could only happen if some constants + * are configured miserably wrong. + */ + assert(SC_LG_TINY_MAXCLASS <= (size_t)1ULL << (LG_PAGE + SC_LG_NGROUP)); + #if defined(_WIN32) && _WIN32_WINNT < 0x0600 _init_init_lock(); #endif malloc_mutex_lock(TSDN_NULL, &init_lock); -#define UNLOCK_RETURN(tsdn, ret, reentrancy) \ - malloc_init_hard_cleanup(tsdn, reentrancy); \ +#define UNLOCK_RETURN(tsdn, ret, reentrancy) \ + malloc_init_hard_cleanup(tsdn, reentrancy); \ return ret; if (!malloc_init_hard_needed()) { UNLOCK_RETURN(TSDN_NULL, false, false) } - if (malloc_init_state != malloc_init_a0_initialized && - malloc_init_hard_a0_locked()) { + if (malloc_init_state != malloc_init_a0_initialized + && malloc_init_hard_a0_locked()) { UNLOCK_RETURN(TSDN_NULL, true, false) } @@ -2144,10 +1233,24 @@ malloc_init_hard(void) { /* Set reentrancy level to 1 during init. */ pre_reentrancy(tsd, NULL); /* Initialize narenas before prof_boot2 (for allocation). */ - if (malloc_init_narenas() + if (malloc_init_narenas(tsd_tsdn(tsd)) || background_thread_boot1(tsd_tsdn(tsd), b0get())) { UNLOCK_RETURN(tsd_tsdn(tsd), true, true) } + if (opt_hpa) { + /* + * We didn't initialize arena 0 hpa_shard in arena_new, because + * background_thread_enabled wasn't initialized yet, but we + * need it to set correct value for deferral_allowed. + */ + arena_t *a0 = arena_get(tsd_tsdn(tsd), 0, false); + hpa_shard_opts_t hpa_shard_opts = opt_hpa_opts; + hpa_shard_opts.deferral_allowed = background_thread_enabled(); + if (pa_shard_enable_hpa(tsd_tsdn(tsd), &a0->pa_shard, + &hpa_shard_opts, &opt_hpa_sec_opts)) { + UNLOCK_RETURN(tsd_tsdn(tsd), true, true) + } + } if (config_prof && prof_boot2(tsd, b0get())) { UNLOCK_RETURN(tsd_tsdn(tsd), true, true) } @@ -2160,8 +1263,8 @@ malloc_init_hard(void) { post_reentrancy(tsd); malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock); - witness_assert_lockless(witness_tsd_tsdn( - tsd_witness_tsdp_get_unsafe(tsd))); + witness_assert_lockless( + witness_tsd_tsdn(tsd_witness_tsdp_get_unsafe(tsd))); malloc_tsd_boot1(); /* Update TSD after tsd_boot1. */ tsd = tsd_fetch(); @@ -2254,23 +1357,14 @@ static_opts_init(static_opts_t *static_opts) { static_opts->usize = false; } -/* - * These correspond to the macros in jemalloc/jemalloc_macros.h. Broadly, we - * should have one constant here per magic value there. Note however that the - * representations need not be related. - */ -#define TCACHE_IND_NONE ((unsigned)-1) -#define TCACHE_IND_AUTOMATIC ((unsigned)-2) -#define ARENA_IND_AUTOMATIC ((unsigned)-1) - typedef struct dynamic_opts_s dynamic_opts_t; struct dynamic_opts_s { - void **result; - size_t usize; - size_t num_items; - size_t item_size; - size_t alignment; - bool zero; + void **result; + size_t usize; + size_t num_items; + size_t item_size; + size_t alignment; + bool zero; unsigned tcache_ind; unsigned arena_ind; }; @@ -2301,7 +1395,9 @@ aligned_usize_get(size_t size, size_t alignment, size_t *usize, szind_t *ind, if (unlikely(*ind >= SC_NSIZES)) { return true; } - *usize = sz_index2size(*ind); + *usize = sz_large_size_classes_disabled() + ? sz_s2u(size) + : sz_index2size(*ind); assert(*usize > 0 && *usize <= SC_LARGE_MAXCLASS); return false; } @@ -2327,36 +1423,6 @@ zero_get(bool guarantee, bool slow) { } } -JEMALLOC_ALWAYS_INLINE tcache_t * -tcache_get_from_ind(tsd_t *tsd, unsigned tcache_ind, bool slow, bool is_alloc) { - tcache_t *tcache; - if (tcache_ind == TCACHE_IND_AUTOMATIC) { - if (likely(!slow)) { - /* Getting tcache ptr unconditionally. */ - tcache = tsd_tcachep_get(tsd); - assert(tcache == tcache_get(tsd)); - } else if (is_alloc || - likely(tsd_reentrancy_level_get(tsd) == 0)) { - tcache = tcache_get(tsd); - } else { - tcache = NULL; - } - } else { - /* - * Should not specify tcache on deallocation path when being - * reentrant. - */ - assert(is_alloc || tsd_reentrancy_level_get(tsd) == 0 || - tsd_state_nocleanup(tsd)); - if (tcache_ind == TCACHE_IND_NONE) { - tcache = NULL; - } else { - tcache = tcaches_get(tsd, tcache_ind); - } - } - return tcache; -} - /* Return true if a manual arena is specified and arena_get() OOMs. */ JEMALLOC_ALWAYS_INLINE bool arena_get_from_ind(tsd_t *tsd, unsigned arena_ind, arena_t **arena_p) { @@ -2379,10 +1445,10 @@ arena_get_from_ind(tsd_t *tsd, unsigned arena_ind, arena_t **arena_p) { /* ind is ignored if dopts->alignment > 0. */ JEMALLOC_ALWAYS_INLINE void * imalloc_no_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd, - size_t size, size_t usize, szind_t ind) { + size_t size, size_t usize, szind_t ind, bool slab) { /* Fill in the tcache. */ - tcache_t *tcache = tcache_get_from_ind(tsd, dopts->tcache_ind, - sopts->slow, /* is_alloc */ true); + tcache_t *tcache = tcache_get_from_ind( + tsd, dopts->tcache_ind, sopts->slow, /* is_alloc */ true); /* Fill in the arena. */ arena_t *arena; @@ -2391,12 +1457,12 @@ imalloc_no_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd, } if (unlikely(dopts->alignment != 0)) { - return ipalloct(tsd_tsdn(tsd), usize, dopts->alignment, - dopts->zero, tcache, arena); + return ipalloct_explicit_slab(tsd_tsdn(tsd), usize, + dopts->alignment, dopts->zero, slab, tcache, arena); } - return iallocztm(tsd_tsdn(tsd), size, ind, dopts->zero, tcache, false, - arena, sopts->slow); + return iallocztm_explicit_slab(tsd_tsdn(tsd), size, ind, dopts->zero, + slab, tcache, false, arena, sopts->slow); } JEMALLOC_ALWAYS_INLINE void * @@ -2404,29 +1470,26 @@ imalloc_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd, size_t usize, szind_t ind) { void *ret; + dopts->alignment = prof_sample_align(usize, dopts->alignment); /* - * For small allocations, sampling bumps the usize. If so, we allocate - * from the ind_large bucket. + * If the allocation is small enough that it would normally be allocated + * on a slab, we need to take additional steps to ensure that it gets + * its own extent instead. */ - szind_t ind_large; - size_t bumped_usize = usize; - - dopts->alignment = prof_sample_align(dopts->alignment); - if (usize <= SC_SMALL_MAXCLASS) { - assert(((dopts->alignment == 0) ? - sz_s2u(SC_LARGE_MINCLASS) : - sz_sa2u(SC_LARGE_MINCLASS, dopts->alignment)) - == SC_LARGE_MINCLASS); - ind_large = sz_size2index(SC_LARGE_MINCLASS); - bumped_usize = sz_s2u(SC_LARGE_MINCLASS); + if (sz_can_use_slab(usize)) { + assert((dopts->alignment & PROF_SAMPLE_ALIGNMENT_MASK) == 0); + size_t bumped_usize = sz_sa2u(usize, dopts->alignment); + szind_t bumped_ind = sz_size2index(bumped_usize); + dopts->tcache_ind = TCACHE_IND_NONE; ret = imalloc_no_sample(sopts, dopts, tsd, bumped_usize, - bumped_usize, ind_large); + bumped_usize, bumped_ind, /* slab */ false); if (unlikely(ret == NULL)) { return NULL; } - arena_prof_promote(tsd_tsdn(tsd), ret, usize); + arena_prof_promote(tsd_tsdn(tsd), ret, usize, bumped_usize); } else { - ret = imalloc_no_sample(sopts, dopts, tsd, usize, usize, ind); + ret = imalloc_no_sample(sopts, dopts, tsd, usize, usize, ind, + /* slab */ false); } assert(prof_sample_aligned(ret)); @@ -2438,8 +1501,8 @@ imalloc_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd, * *size to the product either way. */ JEMALLOC_ALWAYS_INLINE bool -compute_size_with_overflow(bool may_overflow, dynamic_opts_t *dopts, - size_t *size) { +compute_size_with_overflow( + bool may_overflow, dynamic_opts_t *dopts, size_t *size) { /* * This function is just num_items * item_size, except that we may have * to check for overflow. @@ -2495,26 +1558,26 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { int8_t reentrancy_level; /* Compute the amount of memory the user wants. */ - if (unlikely(compute_size_with_overflow(sopts->may_overflow, dopts, - &size))) { + if (unlikely(compute_size_with_overflow( + sopts->may_overflow, dopts, &size))) { goto label_oom; } if (unlikely(dopts->alignment < sopts->min_alignment - || (dopts->alignment & (dopts->alignment - 1)) != 0)) { + || (dopts->alignment & (dopts->alignment - 1)) != 0)) { goto label_invalid_alignment; } /* This is the beginning of the "core" algorithm. */ dopts->zero = zero_get(dopts->zero, sopts->slow); if (aligned_usize_get(size, dopts->alignment, &usize, &ind, - sopts->bump_empty_aligned_alloc)) { + sopts->bump_empty_aligned_alloc)) { goto label_oom; } dopts->usize = usize; /* Validate the user input. */ if (sopts->assert_nonempty_alloc) { - assert (size != 0); + assert(size != 0); } check_entry_exit_locking(tsd_tsdn(tsd)); @@ -2529,8 +1592,8 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { * We should never specify particular arenas or tcaches from * within our internal allocations. */ - assert(dopts->tcache_ind == TCACHE_IND_AUTOMATIC || - dopts->tcache_ind == TCACHE_IND_NONE); + assert(dopts->tcache_ind == TCACHE_IND_AUTOMATIC + || dopts->tcache_ind == TCACHE_IND_NONE); assert(dopts->arena_ind == ARENA_IND_AUTOMATIC); dopts->tcache_ind = TCACHE_IND_NONE; /* We know that arena 0 has already been initialized. */ @@ -2547,15 +1610,15 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { if (config_prof && opt_prof) { bool prof_active = prof_active_get_unlocked(); bool sample_event = te_prof_sample_event_lookahead(tsd, usize); - prof_tctx_t *tctx = prof_alloc_prep(tsd, prof_active, - sample_event); + prof_tctx_t *tctx = prof_alloc_prep( + tsd, prof_active, sample_event); emap_alloc_ctx_t alloc_ctx; - if (likely((uintptr_t)tctx == (uintptr_t)1U)) { - alloc_ctx.slab = (usize <= SC_SMALL_MAXCLASS); - allocation = imalloc_no_sample( - sopts, dopts, tsd, usize, usize, ind); - } else if ((uintptr_t)tctx > (uintptr_t)1U) { + if (likely(tctx == PROF_TCTX_SENTINEL)) { + alloc_ctx.slab = sz_can_use_slab(usize); + allocation = imalloc_no_sample(sopts, dopts, tsd, usize, + usize, ind, alloc_ctx.slab); + } else if (tctx != NULL) { allocation = imalloc_sample( sopts, dopts, tsd, usize, ind); alloc_ctx.slab = false; @@ -2571,7 +1634,7 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { } else { assert(!opt_prof); allocation = imalloc_no_sample(sopts, dopts, tsd, size, usize, - ind); + ind, sz_can_use_slab(usize)); if (unlikely(allocation == NULL)) { goto label_oom; } @@ -2698,8 +1761,8 @@ imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) { JEMALLOC_NOINLINE void * malloc_default(size_t size) { - void *ret; - static_opts_t sopts; + void *ret; + static_opts_t sopts; dynamic_opts_t dopts; /* @@ -2729,8 +1792,6 @@ malloc_default(size_t size) { hook_invoke_alloc(hook_alloc_malloc, ret, (uintptr_t)ret, args); } - LOG("core.malloc.exit", "result: %p", ret); - return ret; } @@ -2739,22 +1800,28 @@ malloc_default(size_t size) { * Begin malloc(3)-compatible functions. */ -JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN -void JEMALLOC_NOTHROW * -JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) -je_malloc(size_t size) { - return imalloc_fastpath(size, &malloc_default); +JEMALLOC_EXPORT +JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN void JEMALLOC_NOTHROW * +JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) je_malloc(size_t size) { + LOG("core.malloc.entry", "size: %zu", size); + + void *ret = imalloc_fastpath(size, &malloc_default); + + LOG("core.malloc.exit", "result: %p", ret); + return ret; } JEMALLOC_EXPORT int JEMALLOC_NOTHROW JEMALLOC_ATTR(nonnull(1)) -je_posix_memalign(void **memptr, size_t alignment, size_t size) { - int ret; - static_opts_t sopts; + je_posix_memalign(void **memptr, size_t alignment, size_t size) { + int ret; + static_opts_t sopts; dynamic_opts_t dopts; - LOG("core.posix_memalign.entry", "mem ptr: %p, alignment: %zu, " - "size: %zu", memptr, alignment, size); + LOG("core.posix_memalign.entry", + "mem ptr: %p, alignment: %zu, " + "size: %zu", + memptr, alignment, size); static_opts_init(&sopts); dynamic_opts_init(&dopts); @@ -2773,10 +1840,10 @@ je_posix_memalign(void **memptr, size_t alignment, size_t size) { ret = imalloc(&sopts, &dopts); if (sopts.slow) { - uintptr_t args[3] = {(uintptr_t)memptr, (uintptr_t)alignment, - (uintptr_t)size}; - hook_invoke_alloc(hook_alloc_posix_memalign, *memptr, - (uintptr_t)ret, args); + uintptr_t args[3] = { + (uintptr_t)memptr, (uintptr_t)alignment, (uintptr_t)size}; + hook_invoke_alloc( + hook_alloc_posix_memalign, *memptr, (uintptr_t)ret, args); } LOG("core.posix_memalign.exit", "result: %d, alloc ptr: %p", ret, @@ -2785,13 +1852,13 @@ je_posix_memalign(void **memptr, size_t alignment, size_t size) { return ret; } -JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN -void JEMALLOC_NOTHROW * +JEMALLOC_EXPORT +JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN void JEMALLOC_NOTHROW * JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(2) -je_aligned_alloc(size_t alignment, size_t size) { + je_aligned_alloc(size_t alignment, size_t size) { void *ret; - static_opts_t sopts; + static_opts_t sopts; dynamic_opts_t dopts; LOG("core.aligned_alloc.entry", "alignment: %zu, size: %zu\n", @@ -2817,8 +1884,8 @@ je_aligned_alloc(size_t alignment, size_t size) { imalloc(&sopts, &dopts); if (sopts.slow) { uintptr_t args[3] = {(uintptr_t)alignment, (uintptr_t)size}; - hook_invoke_alloc(hook_alloc_aligned_alloc, ret, - (uintptr_t)ret, args); + hook_invoke_alloc( + hook_alloc_aligned_alloc, ret, (uintptr_t)ret, args); } LOG("core.aligned_alloc.exit", "result: %p", ret); @@ -2826,15 +1893,15 @@ je_aligned_alloc(size_t alignment, size_t size) { return ret; } -JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN -void JEMALLOC_NOTHROW * +JEMALLOC_EXPORT +JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN void JEMALLOC_NOTHROW * JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2) -je_calloc(size_t num, size_t size) { - void *ret; - static_opts_t sopts; + je_calloc(size_t num, size_t size) { + void *ret; + static_opts_t sopts; dynamic_opts_t dopts; - LOG("core.calloc.entry", "num: %zu, size: %zu\n", num, size); + LOG("core.calloc.entry", "num: %zu, size: %zu", num, size); static_opts_init(&sopts); dynamic_opts_init(&dopts); @@ -2874,51 +1941,26 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) { assert(malloc_initialized() || IS_INITIALIZER); emap_alloc_ctx_t alloc_ctx; - emap_alloc_ctx_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr, - &alloc_ctx); + emap_alloc_ctx_lookup( + tsd_tsdn(tsd), &arena_emap_global, ptr, &alloc_ctx); assert(alloc_ctx.szind != SC_NSIZES); - size_t usize = sz_index2size(alloc_ctx.szind); + size_t usize = emap_alloc_ctx_usize_get(&alloc_ctx); if (config_prof && opt_prof) { prof_free(tsd, ptr, usize, &alloc_ctx); } if (likely(!slow_path)) { - idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false, - false); + idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false, false); } else { if (config_fill && slow_path && opt_junk_free) { junk_free_callback(ptr, usize); } - idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false, - true); + idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false, true); } thread_dalloc_event(tsd, usize); } -JEMALLOC_ALWAYS_INLINE bool -maybe_check_alloc_ctx(tsd_t *tsd, void *ptr, emap_alloc_ctx_t *alloc_ctx) { - if (config_opt_size_checks) { - emap_alloc_ctx_t dbg_ctx; - emap_alloc_ctx_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr, - &dbg_ctx); - if (alloc_ctx->szind != dbg_ctx.szind) { - safety_check_fail_sized_dealloc( - /* current_dealloc */ true, ptr, - /* true_size */ sz_size2index(dbg_ctx.szind), - /* input_size */ sz_size2index(alloc_ctx->szind)); - return true; - } - if (alloc_ctx->slab != dbg_ctx.slab) { - safety_check_fail( - "Internal heap corruption detected: " - "mismatch in slab bit"); - return true; - } - } - return false; -} - JEMALLOC_ALWAYS_INLINE void isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) { if (!slow_path) { @@ -2933,35 +1975,41 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) { assert(malloc_initialized() || IS_INITIALIZER); emap_alloc_ctx_t alloc_ctx; + szind_t szind = sz_size2index(usize); if (!config_prof) { - alloc_ctx.szind = sz_size2index(usize); - alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS); + emap_alloc_ctx_init( + &alloc_ctx, szind, (szind < SC_NBINS), usize); } else { if (likely(!prof_sample_aligned(ptr))) { /* * When the ptr is not page aligned, it was not sampled. * usize can be trusted to determine szind and slab. */ - alloc_ctx.szind = sz_size2index(usize); - alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS); + emap_alloc_ctx_init( + &alloc_ctx, szind, (szind < SC_NBINS), usize); } else if (opt_prof) { - emap_alloc_ctx_lookup(tsd_tsdn(tsd), &arena_emap_global, - ptr, &alloc_ctx); + /* + * Small sampled allocs promoted can still get correct + * usize here. Check comments in edata_usize_get. + */ + emap_alloc_ctx_lookup( + tsd_tsdn(tsd), &arena_emap_global, ptr, &alloc_ctx); if (config_opt_safety_checks) { /* Small alloc may have !slab (sampled). */ - if (unlikely(alloc_ctx.szind != - sz_size2index(usize))) { + size_t true_size = emap_alloc_ctx_usize_get( + &alloc_ctx); + if (unlikely(alloc_ctx.szind + != sz_size2index(usize))) { safety_check_fail_sized_dealloc( /* current_dealloc */ true, ptr, - /* true_size */ sz_index2size( - alloc_ctx.szind), + /* true_size */ true_size, /* input_size */ usize); } } } else { - alloc_ctx.szind = sz_size2index(usize); - alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS); + emap_alloc_ctx_init( + &alloc_ctx, szind, (szind < SC_NBINS), usize); } } bool fail = maybe_check_alloc_ctx(tsd, ptr, &alloc_ctx); @@ -2979,14 +2027,12 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) { prof_free(tsd, ptr, usize, &alloc_ctx); } if (likely(!slow_path)) { - isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, &alloc_ctx, - false); + isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, &alloc_ctx, false); } else { if (config_fill && slow_path && opt_junk_free) { junk_free_callback(ptr, usize); } - isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, &alloc_ctx, - true); + isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, &alloc_ctx, true); } thread_dalloc_event(tsd, usize); } @@ -3025,146 +2071,29 @@ free_default(void *ptr) { } } -JEMALLOC_ALWAYS_INLINE bool -free_fastpath_nonfast_aligned(void *ptr, bool check_prof) { - /* - * free_fastpath do not handle two uncommon cases: 1) sampled profiled - * objects and 2) sampled junk & stash for use-after-free detection. - * Both have special alignments which are used to escape the fastpath. - * - * prof_sample is page-aligned, which covers the UAF check when both - * are enabled (the assertion below). Avoiding redundant checks since - * this is on the fastpath -- at most one runtime branch from this. - */ - if (config_debug && cache_bin_nonfast_aligned(ptr)) { - assert(prof_sample_aligned(ptr)); - } - - if (config_prof && check_prof) { - /* When prof is enabled, the prof_sample alignment is enough. */ - if (prof_sample_aligned(ptr)) { - return true; - } else { - return false; - } - } - - if (config_uaf_detection) { - if (cache_bin_nonfast_aligned(ptr)) { - return true; - } else { - return false; - } - } - - return false; -} - -/* Returns whether or not the free attempt was successful. */ -JEMALLOC_ALWAYS_INLINE -bool free_fastpath(void *ptr, size_t size, bool size_hint) { - tsd_t *tsd = tsd_get(false); - /* The branch gets optimized away unless tsd_get_allocates(). */ - if (unlikely(tsd == NULL)) { - return false; - } - /* - * The tsd_fast() / initialized checks are folded into the branch - * testing (deallocated_after >= threshold) later in this function. - * The threshold will be set to 0 when !tsd_fast. - */ - assert(tsd_fast(tsd) || - *tsd_thread_deallocated_next_event_fastp_get_unsafe(tsd) == 0); - - emap_alloc_ctx_t alloc_ctx; - if (!size_hint) { - bool err = emap_alloc_ctx_try_lookup_fast(tsd, - &arena_emap_global, ptr, &alloc_ctx); - - /* Note: profiled objects will have alloc_ctx.slab set */ - if (unlikely(err || !alloc_ctx.slab || - free_fastpath_nonfast_aligned(ptr, - /* check_prof */ false))) { - return false; - } - assert(alloc_ctx.szind != SC_NSIZES); - } else { - /* - * Check for both sizes that are too large, and for sampled / - * special aligned objects. The alignment check will also check - * for null ptr. - */ - if (unlikely(size > SC_LOOKUP_MAXCLASS || - free_fastpath_nonfast_aligned(ptr, - /* check_prof */ true))) { - return false; - } - alloc_ctx.szind = sz_size2index_lookup(size); - /* Max lookup class must be small. */ - assert(alloc_ctx.szind < SC_NBINS); - /* This is a dead store, except when opt size checking is on. */ - alloc_ctx.slab = true; - } - /* - * Currently the fastpath only handles small sizes. The branch on - * SC_LOOKUP_MAXCLASS makes sure of it. This lets us avoid checking - * tcache szind upper limit (i.e. tcache_maxclass) as well. - */ - assert(alloc_ctx.slab); - - uint64_t deallocated, threshold; - te_free_fastpath_ctx(tsd, &deallocated, &threshold); - - size_t usize = sz_index2size(alloc_ctx.szind); - uint64_t deallocated_after = deallocated + usize; - /* - * Check for events and tsd non-nominal (fast_threshold will be set to - * 0) in a single branch. Note that this handles the uninitialized case - * as well (TSD init will be triggered on the non-fastpath). Therefore - * anything depends on a functional TSD (e.g. the alloc_ctx sanity check - * below) needs to be after this branch. - */ - if (unlikely(deallocated_after >= threshold)) { - return false; - } - assert(tsd_fast(tsd)); - bool fail = maybe_check_alloc_ctx(tsd, ptr, &alloc_ctx); - if (fail) { - /* See the comment in isfree. */ - return true; - } - - tcache_t *tcache = tcache_get_from_ind(tsd, TCACHE_IND_AUTOMATIC, - /* slow */ false, /* is_alloc */ false); - cache_bin_t *bin = &tcache->bins[alloc_ctx.szind]; - - /* - * If junking were enabled, this is where we would do it. It's not - * though, since we ensured above that we're on the fast path. Assert - * that to double-check. - */ - assert(!opt_junk_free); - - if (!cache_bin_dalloc_easy(bin, ptr)) { - return false; - } - - *tsd_thread_deallocatedp_get(tsd) = deallocated_after; - - return true; -} - JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_free(void *ptr) { LOG("core.free.entry", "ptr: %p", ptr); - if (!free_fastpath(ptr, 0, false)) { - free_default(ptr); - } + je_free_impl(ptr); LOG("core.free.exit", ""); } +JEMALLOC_EXPORT void JEMALLOC_NOTHROW +je_free_sized(void *ptr, size_t size) { + LOG("core.free_sized.entry", "ptr: %p, size: %zu", ptr, size); + + je_sdallocx_noflags(ptr, size); + + LOG("core.free_sized.exit", ""); +} + +JEMALLOC_EXPORT void JEMALLOC_NOTHROW +je_free_aligned_sized(void *ptr, size_t alignment, size_t size) { + return je_sdallocx(ptr, size, /* flags */ MALLOCX_ALIGN(alignment)); +} + /* * End malloc(3)-compatible functions. */ @@ -3174,12 +2103,11 @@ je_free(void *ptr) { */ #ifdef JEMALLOC_OVERRIDE_MEMALIGN -JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN -void JEMALLOC_NOTHROW * -JEMALLOC_ATTR(malloc) -je_memalign(size_t alignment, size_t size) { - void *ret; - static_opts_t sopts; +JEMALLOC_EXPORT +JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN void JEMALLOC_NOTHROW * +JEMALLOC_ATTR(malloc) je_memalign(size_t alignment, size_t size) { + void *ret; + static_opts_t sopts; dynamic_opts_t dopts; LOG("core.memalign.entry", "alignment: %zu, size: %zu\n", alignment, @@ -3188,6 +2116,7 @@ je_memalign(size_t alignment, size_t size) { static_opts_init(&sopts); dynamic_opts_init(&dopts); + sopts.bump_empty_aligned_alloc = true; sopts.min_alignment = 1; sopts.oom_string = ": Error allocating aligned memory: out of memory\n"; @@ -3203,8 +2132,8 @@ je_memalign(size_t alignment, size_t size) { imalloc(&sopts, &dopts); if (sopts.slow) { uintptr_t args[3] = {alignment, size}; - hook_invoke_alloc(hook_alloc_memalign, ret, (uintptr_t)ret, - args); + hook_invoke_alloc( + hook_alloc_memalign, ret, (uintptr_t)ret, args); } LOG("core.memalign.exit", "result: %p", ret); @@ -3213,13 +2142,12 @@ je_memalign(size_t alignment, size_t size) { #endif #ifdef JEMALLOC_OVERRIDE_VALLOC -JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN -void JEMALLOC_NOTHROW * -JEMALLOC_ATTR(malloc) -je_valloc(size_t size) { +JEMALLOC_EXPORT +JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN void JEMALLOC_NOTHROW * +JEMALLOC_ATTR(malloc) je_valloc(size_t size) { void *ret; - static_opts_t sopts; + static_opts_t sopts; dynamic_opts_t dopts; LOG("core.valloc.entry", "size: %zu\n", size); @@ -3250,6 +2178,48 @@ je_valloc(size_t size) { } #endif +#ifdef JEMALLOC_OVERRIDE_PVALLOC +JEMALLOC_EXPORT +JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN void JEMALLOC_NOTHROW * +JEMALLOC_ATTR(malloc) je_pvalloc(size_t size) { + void *ret; + + static_opts_t sopts; + dynamic_opts_t dopts; + + LOG("core.pvalloc.entry", "size: %zu\n", size); + + static_opts_init(&sopts); + dynamic_opts_init(&dopts); + + sopts.null_out_result_on_error = true; + sopts.min_alignment = PAGE; + sopts.oom_string = + ": Error allocating aligned memory: out of memory\n"; + sopts.invalid_alignment_string = + ": Error allocating aligned memory: invalid alignment\n"; + + dopts.result = &ret; + dopts.num_items = 1; + /* + * This is the only difference from je_valloc - size is rounded up to + * a PAGE multiple. + */ + dopts.item_size = PAGE_CEILING(size); + dopts.alignment = PAGE; + + imalloc(&sopts, &dopts); + if (sopts.slow) { + uintptr_t args[3] = {size}; + hook_invoke_alloc( + hook_alloc_pvalloc, ret, (uintptr_t)ret, args); + } + + LOG("core.pvalloc.exit", "result: %p\n", ret); + return ret; +} +#endif + #if defined(JEMALLOC_IS_MALLOC) && defined(JEMALLOC_GLIBC_MALLOC_HOOK) /* * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible @@ -3260,49 +2230,59 @@ je_valloc(size_t size) { * passed an extra argument for the caller return address, which will be * ignored. */ -#include // defines __GLIBC__ if we are compiling against glibc +# include // defines __GLIBC__ if we are compiling against glibc JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free; JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc; JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc; -# ifdef JEMALLOC_GLIBC_MEMALIGN_HOOK -JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) = - je_memalign; -# endif +# ifdef JEMALLOC_GLIBC_MEMALIGN_HOOK +JEMALLOC_EXPORT void *(*__memalign_hook)( + size_t alignment, size_t size) = je_memalign; +# endif -# ifdef __GLIBC__ +# ifdef __GLIBC__ /* * To enable static linking with glibc, the libc specific malloc interface must * be implemented also, so none of glibc's malloc.o functions are added to the * link. */ -# define ALIAS(je_fn) __attribute__((alias (#je_fn), used)) +# define ALIAS(je_fn) __attribute__((alias(#je_fn), used)) /* To force macro expansion of je_ prefix before stringification. */ -# define PREALIAS(je_fn) ALIAS(je_fn) -# ifdef JEMALLOC_OVERRIDE___LIBC_CALLOC +# define PREALIAS(je_fn) ALIAS(je_fn) +# ifdef JEMALLOC_OVERRIDE___LIBC_CALLOC void *__libc_calloc(size_t n, size_t size) PREALIAS(je_calloc); -# endif -# ifdef JEMALLOC_OVERRIDE___LIBC_FREE -void __libc_free(void* ptr) PREALIAS(je_free); -# endif -# ifdef JEMALLOC_OVERRIDE___LIBC_MALLOC +# endif +# ifdef JEMALLOC_OVERRIDE___LIBC_FREE +void __libc_free(void *ptr) PREALIAS(je_free); +# endif +# ifdef JEMALLOC_OVERRIDE___LIBC_FREE_SIZED +void __libc_free_sized(void *ptr, size_t size) PREALIAS(je_free_sized); +# endif +# ifdef JEMALLOC_OVERRIDE___LIBC_FREE_ALIGNED_SIZED +void __libc_free_aligned_sized(void *ptr, size_t alignment, size_t size) + PREALIAS(je_free_aligned_sized); +# endif +# ifdef JEMALLOC_OVERRIDE___LIBC_MALLOC void *__libc_malloc(size_t size) PREALIAS(je_malloc); -# endif -# ifdef JEMALLOC_OVERRIDE___LIBC_MEMALIGN +# endif +# ifdef JEMALLOC_OVERRIDE___LIBC_MEMALIGN void *__libc_memalign(size_t align, size_t s) PREALIAS(je_memalign); -# endif -# ifdef JEMALLOC_OVERRIDE___LIBC_REALLOC -void *__libc_realloc(void* ptr, size_t size) PREALIAS(je_realloc); -# endif -# ifdef JEMALLOC_OVERRIDE___LIBC_VALLOC +# endif +# ifdef JEMALLOC_OVERRIDE___LIBC_REALLOC +void *__libc_realloc(void *ptr, size_t size) PREALIAS(je_realloc); +# endif +# ifdef JEMALLOC_OVERRIDE___LIBC_VALLOC void *__libc_valloc(size_t size) PREALIAS(je_valloc); -# endif -# ifdef JEMALLOC_OVERRIDE___POSIX_MEMALIGN -int __posix_memalign(void** r, size_t a, size_t s) PREALIAS(je_posix_memalign); -# endif -# undef PREALIAS -# undef ALIAS -# endif +# endif +# ifdef JEMALLOC_OVERRIDE___LIBC_PVALLOC +void *__libc_pvalloc(size_t size) PREALIAS(je_pvalloc); +# endif +# ifdef JEMALLOC_OVERRIDE___POSIX_MEMALIGN +int __posix_memalign(void **r, size_t a, size_t s) PREALIAS(je_posix_memalign); +# endif +# undef PREALIAS +# undef ALIAS +# endif #endif /* @@ -3335,23 +2315,23 @@ mallocx_arena_get(int flags) { #ifdef JEMALLOC_EXPERIMENTAL_SMALLOCX_API -#define JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y) x ## y -#define JEMALLOC_SMALLOCX_CONCAT_HELPER2(x, y) \ - JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y) +# define JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y) x##y +# define JEMALLOC_SMALLOCX_CONCAT_HELPER2(x, y) \ + JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y) typedef struct { - void *ptr; + void *ptr; size_t size; } smallocx_return_t; -JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN -smallocx_return_t JEMALLOC_NOTHROW -/* +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN smallocx_return_t + JEMALLOC_NOTHROW + /* * The attribute JEMALLOC_ATTR(malloc) cannot be used due to: * - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86488 */ -JEMALLOC_SMALLOCX_CONCAT_HELPER2(je_smallocx_, JEMALLOC_VERSION_GID_IDENT) - (size_t size, int flags) { + JEMALLOC_SMALLOCX_CONCAT_HELPER2(je_smallocx_, JEMALLOC_VERSION_GID_IDENT)( + size_t size, int flags) { /* * Note: the attribute JEMALLOC_ALLOC_SIZE(1) cannot be * used here because it makes writing beyond the `size` @@ -3360,8 +2340,8 @@ JEMALLOC_SMALLOCX_CONCAT_HELPER2(je_smallocx_, JEMALLOC_VERSION_GID_IDENT) * up to `smallocx_return_t::size`. */ smallocx_return_t ret; - static_opts_t sopts; - dynamic_opts_t dopts; + static_opts_t sopts; + dynamic_opts_t dopts; LOG("core.smallocx.entry", "size: %zu, flags: %d", size, flags); @@ -3390,16 +2370,16 @@ JEMALLOC_SMALLOCX_CONCAT_HELPER2(je_smallocx_, JEMALLOC_VERSION_GID_IDENT) LOG("core.smallocx.exit", "result: %p, size: %zu", ret.ptr, ret.size); return ret; } -#undef JEMALLOC_SMALLOCX_CONCAT_HELPER -#undef JEMALLOC_SMALLOCX_CONCAT_HELPER2 +# undef JEMALLOC_SMALLOCX_CONCAT_HELPER +# undef JEMALLOC_SMALLOCX_CONCAT_HELPER2 #endif -JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN -void JEMALLOC_NOTHROW * +JEMALLOC_EXPORT +JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN void JEMALLOC_NOTHROW * JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) -je_mallocx(size_t size, int flags) { - void *ret; - static_opts_t sopts; + je_mallocx(size_t size, int flags) { + void *ret; + static_opts_t sopts; dynamic_opts_t dopts; LOG("core.mallocx.entry", "size: %zu, flags: %d", size, flags); @@ -3424,8 +2404,8 @@ je_mallocx(size_t size, int flags) { imalloc(&sopts, &dopts); if (sopts.slow) { uintptr_t args[3] = {size, flags}; - hook_invoke_alloc(hook_alloc_mallocx, ret, (uintptr_t)ret, - args); + hook_invoke_alloc( + hook_alloc_mallocx, ret, (uintptr_t)ret, args); } LOG("core.mallocx.exit", "result: %p", ret); @@ -3442,18 +2422,25 @@ irallocx_prof_sample(tsdn_t *tsdn, void *old_ptr, size_t old_usize, return NULL; } - alignment = prof_sample_align(alignment); - if (usize <= SC_SMALL_MAXCLASS) { - p = iralloct(tsdn, old_ptr, old_usize, - SC_LARGE_MINCLASS, alignment, zero, tcache, + alignment = prof_sample_align(usize, alignment); + /* + * If the allocation is small enough that it would normally be allocated + * on a slab, we need to take additional steps to ensure that it gets + * its own extent instead. + */ + if (sz_can_use_slab(usize)) { + size_t bumped_usize = sz_sa2u(usize, alignment); + p = iralloct_explicit_slab(tsdn, old_ptr, old_usize, + bumped_usize, alignment, zero, /* slab */ false, tcache, arena, hook_args); if (p == NULL) { return NULL; } - arena_prof_promote(tsdn, p, usize); + arena_prof_promote(tsdn, p, usize, bumped_usize); } else { - p = iralloct(tsdn, old_ptr, old_usize, usize, alignment, zero, - tcache, arena, hook_args); + p = iralloct_explicit_slab(tsdn, old_ptr, old_usize, usize, + alignment, zero, /* slab */ false, tcache, arena, + hook_args); } assert(prof_sample_aligned(p)); @@ -3462,40 +2449,39 @@ irallocx_prof_sample(tsdn_t *tsdn, void *old_ptr, size_t old_usize, JEMALLOC_ALWAYS_INLINE void * irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size, - size_t alignment, size_t usize, bool zero, tcache_t *tcache, - arena_t *arena, emap_alloc_ctx_t *alloc_ctx, - hook_ralloc_args_t *hook_args) { + size_t alignment, size_t usize, bool zero, tcache_t *tcache, arena_t *arena, + emap_alloc_ctx_t *alloc_ctx, hook_ralloc_args_t *hook_args) { prof_info_t old_prof_info; prof_info_get_and_reset_recent(tsd, old_ptr, alloc_ctx, &old_prof_info); - bool prof_active = prof_active_get_unlocked(); - bool sample_event = te_prof_sample_event_lookahead(tsd, usize); + bool prof_active = prof_active_get_unlocked(); + bool sample_event = te_prof_sample_event_lookahead(tsd, usize); prof_tctx_t *tctx = prof_alloc_prep(tsd, prof_active, sample_event); - void *p; - if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { + void *p; + if (unlikely(tctx != PROF_TCTX_SENTINEL)) { p = irallocx_prof_sample(tsd_tsdn(tsd), old_ptr, old_usize, usize, alignment, zero, tcache, arena, tctx, hook_args); } else { p = iralloct(tsd_tsdn(tsd), old_ptr, old_usize, size, alignment, - zero, tcache, arena, hook_args); + usize, zero, tcache, arena, hook_args); } if (unlikely(p == NULL)) { prof_alloc_rollback(tsd, tctx); return NULL; } assert(usize == isalloc(tsd_tsdn(tsd), p)); - prof_realloc(tsd, p, size, usize, tctx, prof_active, old_ptr, - old_usize, &old_prof_info, sample_event); + prof_realloc(tsd, p, size, usize, tctx, prof_active, old_ptr, old_usize, + &old_prof_info, sample_event); return p; } static void * do_rallocx(void *ptr, size_t size, int flags, bool is_realloc) { - void *p; - tsd_t *tsd; - size_t usize; - size_t old_usize; - size_t alignment = MALLOCX_ALIGN_GET(flags); + void *p; + tsd_t *tsd; + size_t usize; + size_t old_usize; + size_t alignment = MALLOCX_ALIGN_GET(flags); arena_t *arena; assert(ptr != NULL); @@ -3511,22 +2497,22 @@ do_rallocx(void *ptr, size_t size, int flags, bool is_realloc) { goto label_oom; } - unsigned tcache_ind = mallocx_tcache_get(flags); + unsigned tcache_ind = mallocx_tcache_get(flags); tcache_t *tcache = tcache_get_from_ind(tsd, tcache_ind, /* slow */ true, /* is_alloc */ true); emap_alloc_ctx_t alloc_ctx; - emap_alloc_ctx_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr, - &alloc_ctx); + emap_alloc_ctx_lookup( + tsd_tsdn(tsd), &arena_emap_global, ptr, &alloc_ctx); assert(alloc_ctx.szind != SC_NSIZES); - old_usize = sz_index2size(alloc_ctx.szind); + old_usize = emap_alloc_ctx_usize_get(&alloc_ctx); assert(old_usize == isalloc(tsd_tsdn(tsd), ptr)); if (aligned_usize_get(size, alignment, &usize, NULL, false)) { goto label_oom; } - hook_ralloc_args_t hook_args = {is_realloc, {(uintptr_t)ptr, size, - flags, 0}}; + hook_ralloc_args_t hook_args = { + is_realloc, {(uintptr_t)ptr, size, flags, 0}}; if (config_prof && opt_prof) { p = irallocx_prof(tsd, ptr, old_usize, size, alignment, usize, zero, tcache, arena, &alloc_ctx, &hook_args); @@ -3535,7 +2521,7 @@ do_rallocx(void *ptr, size_t size, int flags, bool is_realloc) { } } else { p = iralloct(tsd_tsdn(tsd), ptr, old_usize, size, alignment, - zero, tcache, arena, &hook_args); + usize, zero, tcache, arena, &hook_args); if (unlikely(p == NULL)) { goto label_oom; } @@ -3551,12 +2537,15 @@ do_rallocx(void *ptr, size_t size, int flags, bool is_realloc) { if (config_fill && unlikely(opt_junk_alloc) && usize > old_usize && !zero) { size_t excess_len = usize - old_usize; - void *excess_start = (void *)((uintptr_t)p + old_usize); + void *excess_start = (void *)((byte_t *)p + old_usize); junk_alloc_callback(excess_start, excess_len); } return p; label_oom: + if (is_realloc) { + set_errno(ENOMEM); + } if (config_xmalloc && unlikely(opt_xmalloc)) { malloc_write(": Error in rallocx(): out of memory\n"); abort(); @@ -3567,12 +2556,11 @@ label_oom: return NULL; } -JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN -void JEMALLOC_NOTHROW * -JEMALLOC_ALLOC_SIZE(2) -je_rallocx(void *ptr, size_t size, int flags) { - LOG("core.rallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr, - size, flags); +JEMALLOC_EXPORT +JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN void JEMALLOC_NOTHROW * +JEMALLOC_ALLOC_SIZE(2) je_rallocx(void *ptr, size_t size, int flags) { + LOG("core.rallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr, size, + flags); void *ret = do_rallocx(ptr, size, flags, false); LOG("core.rallocx.exit", "result: %p", ret); return ret; @@ -3606,7 +2594,8 @@ do_realloc_nonnull_zero(void *ptr) { check_entry_exit_locking(tsd_tsdn(tsd)); return NULL; } else { - safety_check_fail("Called realloc(non-null-ptr, 0) with " + safety_check_fail( + "Called realloc(non-null-ptr, 0) with " "zero_realloc:abort set\n"); /* In real code, this will never run; the safety check failure * will call abort. In the unit test, we just want to bail out @@ -3617,10 +2606,9 @@ do_realloc_nonnull_zero(void *ptr) { } } -JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN -void JEMALLOC_NOTHROW * -JEMALLOC_ALLOC_SIZE(2) -je_realloc(void *ptr, size_t size) { +JEMALLOC_EXPORT +JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN void JEMALLOC_NOTHROW * +JEMALLOC_ALLOC_SIZE(2) je_realloc(void *ptr, size_t size) { LOG("core.realloc.entry", "ptr: %p, size: %zu\n", ptr, size); if (likely(ptr != NULL && size != 0)) { @@ -3635,7 +2623,7 @@ je_realloc(void *ptr, size_t size) { /* realloc(NULL, size) is equivalent to malloc(size). */ void *ret; - static_opts_t sopts; + static_opts_t sopts; dynamic_opts_t dopts; static_opts_init(&sopts); @@ -3653,8 +2641,8 @@ je_realloc(void *ptr, size_t size) { imalloc(&sopts, &dopts); if (sopts.slow) { uintptr_t args[3] = {(uintptr_t)ptr, size}; - hook_invoke_alloc(hook_alloc_realloc, ret, - (uintptr_t)ret, args); + hook_invoke_alloc( + hook_alloc_realloc, ret, (uintptr_t)ret, args); } LOG("core.realloc.exit", "result: %p", ret); return ret; @@ -3666,8 +2654,8 @@ ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size, size_t extra, size_t alignment, bool zero) { size_t newsize; - if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero, - &newsize)) { + if (ixalloc( + tsdn, ptr, old_usize, size, extra, alignment, zero, &newsize)) { return old_usize; } @@ -3682,8 +2670,8 @@ ixallocx_prof_sample(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size, return old_usize; } - return ixallocx_helper(tsdn, ptr, old_usize, size, extra, alignment, - zero); + return ixallocx_helper( + tsdn, ptr, old_usize, size, extra, alignment, zero); } JEMALLOC_ALWAYS_INLINE size_t @@ -3703,8 +2691,8 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, * prof_realloc() will use the actual usize to decide whether to sample. */ size_t usize_max; - if (aligned_usize_get(size + extra, alignment, &usize_max, NULL, - false)) { + if (aligned_usize_get( + size + extra, alignment, &usize_max, NULL, false)) { /* * usize_max is out of range, and chances are that allocation * will fail, but use the maximum possible value and carry on @@ -3717,7 +2705,7 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, prof_tctx_t *tctx = prof_alloc_prep(tsd, prof_active, sample_event); size_t usize; - if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { + if (unlikely(tctx != PROF_TCTX_SENTINEL)) { usize = ixallocx_prof_sample(tsd_tsdn(tsd), ptr, old_usize, size, extra, alignment, zero, tctx); } else { @@ -3738,7 +2726,15 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, prof_info_get(tsd, ptr, alloc_ctx, &prof_info); prof_alloc_rollback(tsd, tctx); } else { - prof_info_get_and_reset_recent(tsd, ptr, alloc_ctx, &prof_info); + /* + * Need to retrieve the new alloc_ctx since the modification + * to edata has already been done. + */ + emap_alloc_ctx_t new_alloc_ctx; + emap_alloc_ctx_lookup( + tsd_tsdn(tsd), &arena_emap_global, ptr, &new_alloc_ctx); + prof_info_get_and_reset_recent( + tsd, ptr, &new_alloc_ctx, &prof_info); assert(usize <= usize_max); sample_event = te_prof_sample_event_lookahead(tsd, usize); prof_realloc(tsd, ptr, size, usize, tctx, prof_active, ptr, @@ -3754,10 +2750,12 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) { tsd_t *tsd; size_t usize, old_usize; size_t alignment = MALLOCX_ALIGN_GET(flags); - bool zero = zero_get(MALLOCX_ZERO_GET(flags), /* slow */ true); + bool zero = zero_get(MALLOCX_ZERO_GET(flags), /* slow */ true); - LOG("core.xallocx.entry", "ptr: %p, size: %zu, extra: %zu, " - "flags: %d", ptr, size, extra, flags); + LOG("core.xallocx.entry", + "ptr: %p, size: %zu, extra: %zu, " + "flags: %d", + ptr, size, extra, flags); assert(ptr != NULL); assert(size != 0); @@ -3771,14 +2769,14 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) { * object associated with the ptr (though the content of the edata_t * object can be changed). */ - edata_t *old_edata = emap_edata_lookup(tsd_tsdn(tsd), - &arena_emap_global, ptr); + edata_t *old_edata = emap_edata_lookup( + tsd_tsdn(tsd), &arena_emap_global, ptr); emap_alloc_ctx_t alloc_ctx; - emap_alloc_ctx_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr, - &alloc_ctx); + emap_alloc_ctx_lookup( + tsd_tsdn(tsd), &arena_emap_global, ptr, &alloc_ctx); assert(alloc_ctx.szind != SC_NSIZES); - old_usize = sz_index2size(alloc_ctx.szind); + old_usize = emap_alloc_ctx_usize_get(&alloc_ctx); assert(old_usize == isalloc(tsd_tsdn(tsd), ptr)); /* * The API explicitly absolves itself of protecting against (size + @@ -3818,17 +2816,17 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) { thread_alloc_event(tsd, usize); thread_dalloc_event(tsd, old_usize); - if (config_fill && unlikely(opt_junk_alloc) && usize > old_usize && - !zero) { + if (config_fill && unlikely(opt_junk_alloc) && usize > old_usize + && !zero) { size_t excess_len = usize - old_usize; - void *excess_start = (void *)((uintptr_t)ptr + old_usize); + void *excess_start = (void *)((byte_t *)ptr + old_usize); junk_alloc_callback(excess_start, excess_len); } label_not_resized: if (unlikely(!tsd_fast(tsd))) { uintptr_t args[4] = {(uintptr_t)ptr, size, extra, flags}; - hook_invoke_expand(hook_expand_xallocx, ptr, old_usize, - usize, (uintptr_t)usize, args); + hook_invoke_expand(hook_expand_xallocx, ptr, old_usize, usize, + (uintptr_t)usize, args); } UTRACE(ptr, size, ptr); @@ -3839,9 +2837,8 @@ label_not_resized: } JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW -JEMALLOC_ATTR(pure) -je_sallocx(const void *ptr, int flags) { - size_t usize; +JEMALLOC_ATTR(pure) je_sallocx(const void *ptr, int flags) { + size_t usize; tsdn_t *tsdn; LOG("core.sallocx.entry", "ptr: %p, flags: %d", ptr, flags); @@ -3873,10 +2870,10 @@ je_dallocx(void *ptr, int flags) { assert(malloc_initialized() || IS_INITIALIZER); tsd_t *tsd = tsd_fetch_min(); - bool fast = tsd_fast(tsd); + bool fast = tsd_fast(tsd); check_entry_exit_locking(tsd_tsdn(tsd)); - unsigned tcache_ind = mallocx_tcache_get(flags); + unsigned tcache_ind = mallocx_tcache_get(flags); tcache_t *tcache = tcache_get_from_ind(tsd, tcache_ind, !fast, /* is_alloc */ false); @@ -3910,11 +2907,11 @@ sdallocx_default(void *ptr, size_t size, int flags) { assert(malloc_initialized() || IS_INITIALIZER); tsd_t *tsd = tsd_fetch_min(); - bool fast = tsd_fast(tsd); + bool fast = tsd_fast(tsd); size_t usize = inallocx(tsd_tsdn(tsd), size, flags); check_entry_exit_locking(tsd_tsdn(tsd)); - unsigned tcache_ind = mallocx_tcache_get(flags); + unsigned tcache_ind = mallocx_tcache_get(flags); tcache_t *tcache = tcache_get_from_ind(tsd, tcache_ind, !fast, /* is_alloc */ false); @@ -3932,32 +2929,17 @@ sdallocx_default(void *ptr, size_t size, int flags) { JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_sdallocx(void *ptr, size_t size, int flags) { - LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr, - size, flags); + LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr, size, + flags); - if (flags != 0 || !free_fastpath(ptr, size, true)) { - sdallocx_default(ptr, size, flags); - } - - LOG("core.sdallocx.exit", ""); -} - -void JEMALLOC_NOTHROW -je_sdallocx_noflags(void *ptr, size_t size) { - LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: 0", ptr, - size); - - if (!free_fastpath(ptr, size, true)) { - sdallocx_default(ptr, size, 0); - } + je_sdallocx_impl(ptr, size, flags); LOG("core.sdallocx.exit", ""); } JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW -JEMALLOC_ATTR(pure) -je_nallocx(size_t size, int flags) { - size_t usize; +JEMALLOC_ATTR(pure) je_nallocx(size_t size, int flags) { + size_t usize; tsdn_t *tsdn; assert(size != 0); @@ -3982,9 +2964,9 @@ je_nallocx(size_t size, int flags) { } JEMALLOC_EXPORT int JEMALLOC_NOTHROW -je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) { - int ret; +je_mallctl( + const char *name, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + int ret; tsd_t *tsd; LOG("core.mallctl.entry", "name: %s", name); @@ -4025,8 +3007,8 @@ je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) { JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen) { - int ret; + void *newp, size_t newlen) { + int ret; tsd_t *tsd; LOG("core.mallctlbymib.entry", ""); @@ -4046,8 +3028,8 @@ je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, #define STATS_PRINT_BUFSIZE 65536 JEMALLOC_EXPORT void JEMALLOC_NOTHROW -je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, - const char *opts) { +je_malloc_stats_print( + void (*write_cb)(void *, const char *), void *cbopaque, const char *opts) { tsdn_t *tsdn; LOG("core.malloc_stats_print.entry", ""); @@ -4118,12 +3100,12 @@ je_malloc_size(const void *ptr) { static void batch_alloc_prof_sample_assert(tsd_t *tsd, size_t batch, size_t usize) { assert(config_prof && opt_prof); - bool prof_sample_event = te_prof_sample_event_lookahead(tsd, - batch * usize); + bool prof_sample_event = te_prof_sample_event_lookahead( + tsd, batch * usize); assert(!prof_sample_event); size_t surplus; - prof_sample_event = te_prof_sample_event_lookahead_surplus(tsd, - (batch + 1) * usize, &surplus); + prof_sample_event = te_prof_sample_event_lookahead_surplus( + tsd, (batch + 1) * usize, &surplus); assert(prof_sample_event); assert(surplus < usize); } @@ -4148,14 +3130,14 @@ batch_alloc(void **ptrs, size_t num, size_t size, int flags) { goto label_done; } szind_t ind = sz_size2index(usize); - bool zero = zero_get(MALLOCX_ZERO_GET(flags), /* slow */ true); + bool zero = zero_get(MALLOCX_ZERO_GET(flags), /* slow */ true); /* * The cache bin and arena will be lazily initialized; it's hard to * know in advance whether each of them needs to be initialized. */ cache_bin_t *bin = NULL; - arena_t *arena = NULL; + arena_t *arena = NULL; size_t nregs = 0; if (likely(ind < SC_NBINS)) { @@ -4166,10 +3148,10 @@ batch_alloc(void **ptrs, size_t num, size_t size, int flags) { while (filled < num) { size_t batch = num - filled; size_t surplus = SIZE_MAX; /* Dead store. */ - bool prof_sample_event = config_prof && opt_prof + bool prof_sample_event = config_prof && opt_prof && prof_active_get_unlocked() - && te_prof_sample_event_lookahead_surplus(tsd, - batch * usize, &surplus); + && te_prof_sample_event_lookahead_surplus( + tsd, batch * usize, &surplus); if (prof_sample_event) { /* @@ -4185,8 +3167,8 @@ batch_alloc(void **ptrs, size_t num, size_t size, int flags) { if (likely(ind < SC_NBINS) && batch >= nregs) { if (arena == NULL) { unsigned arena_ind = mallocx_arena_get(flags); - if (arena_get_from_ind(tsd, arena_ind, - &arena)) { + if (arena_get_from_ind( + tsd, arena_ind, &arena)) { goto label_done; } if (arena == NULL) { @@ -4203,15 +3185,16 @@ batch_alloc(void **ptrs, size_t num, size_t size, int flags) { filled += n; } - if (likely(ind < nhbins) && progress < batch) { + unsigned tcache_ind = mallocx_tcache_get(flags); + tcache_t *tcache = tcache_get_from_ind(tsd, tcache_ind, + /* slow */ true, /* is_alloc */ true); + if (likely(tcache != NULL + && ind < tcache_nbins_get(tcache->tcache_slow) + && !tcache_bin_disabled( + ind, &tcache->bins[ind], tcache->tcache_slow)) + && progress < batch) { if (bin == NULL) { - unsigned tcache_ind = mallocx_tcache_get(flags); - tcache_t *tcache = tcache_get_from_ind(tsd, - tcache_ind, /* slow */ true, - /* is_alloc */ true); - if (tcache != NULL) { - bin = &tcache->bins[ind]; - } + bin = &tcache->bins[ind]; } /* * If we don't have a tcache bin, we don't want to @@ -4240,22 +3223,22 @@ batch_alloc(void **ptrs, size_t num, size_t size, int flags) { * additional benefit is that the tcache will * not be empty for the next allocation request. */ - size_t n = cache_bin_alloc_batch(bin, bin_batch, - ptrs + filled); + size_t n = cache_bin_alloc_batch( + bin, bin_batch, ptrs + filled); if (config_stats) { bin->tstats.nrequests += n; } if (zero) { for (size_t i = 0; i < n; ++i) { - memset(ptrs[filled + i], 0, - usize); + memset( + ptrs[filled + i], 0, usize); } } if (config_prof && opt_prof && unlikely(ind >= SC_NBINS)) { for (size_t i = 0; i < n; ++i) { - prof_tctx_reset_sampled(tsd, - ptrs[filled + i]); + prof_tctx_reset_sampled( + tsd, ptrs[filled + i]); } } progress += n; @@ -4331,7 +3314,7 @@ JEMALLOC_EXPORT void _malloc_prefork(void) #endif { - tsd_t *tsd; + tsd_t *tsd; unsigned i, j, narenas; arena_t *arena; @@ -4361,8 +3344,8 @@ _malloc_prefork(void) /* Break arena prefork into stages to preserve lock order. */ for (i = 0; i < 9; i++) { for (j = 0; j < narenas; j++) { - if ((arena = arena_get(tsd_tsdn(tsd), j, false)) != - NULL) { + if ((arena = arena_get(tsd_tsdn(tsd), j, false)) + != NULL) { switch (i) { case 0: arena_prefork0(tsd_tsdn(tsd), arena); @@ -4391,11 +3374,11 @@ _malloc_prefork(void) case 8: arena_prefork8(tsd_tsdn(tsd), arena); break; - default: not_reached(); + default: + not_reached(); } } } - } prof_prefork1(tsd_tsdn(tsd)); stats_prefork(tsd_tsdn(tsd)); @@ -4410,7 +3393,7 @@ JEMALLOC_EXPORT void _malloc_postfork(void) #endif { - tsd_t *tsd; + tsd_t *tsd; unsigned i, narenas; #ifdef JEMALLOC_MUTEX_INIT_CB @@ -4445,7 +3428,7 @@ _malloc_postfork(void) void jemalloc_postfork_child(void) { - tsd_t *tsd; + tsd_t *tsd; unsigned i, narenas; assert(malloc_initialized()); diff --git a/dep/jemalloc/src/jemalloc_cpp.cpp b/dep/jemalloc/src/jemalloc_cpp.cpp index 451655f1b..ac109bb28 100644 --- a/dep/jemalloc/src/jemalloc_cpp.cpp +++ b/dep/jemalloc/src/jemalloc_cpp.cpp @@ -1,7 +1,7 @@ -#include +#include #include +// NOLINTBEGIN(misc-use-anonymous-namespace) -#define JEMALLOC_CPP_CPP_ #ifdef __cplusplus extern "C" { #endif @@ -24,77 +24,90 @@ extern "C" { // // ... but it needs to work with jemalloc namespaces. -void *operator new(std::size_t size); -void *operator new[](std::size_t size); -void *operator new(std::size_t size, const std::nothrow_t &) noexcept; -void *operator new[](std::size_t size, const std::nothrow_t &) noexcept; -void operator delete(void *ptr) noexcept; -void operator delete[](void *ptr) noexcept; -void operator delete(void *ptr, const std::nothrow_t &) noexcept; -void operator delete[](void *ptr, const std::nothrow_t &) noexcept; +void *operator new(std::size_t size); +void *operator new[](std::size_t size); +void *operator new(std::size_t size, const std::nothrow_t &) noexcept; +void *operator new[](std::size_t size, const std::nothrow_t &) noexcept; +void operator delete(void *ptr) noexcept; +void operator delete[](void *ptr) noexcept; +void operator delete(void *ptr, const std::nothrow_t &) noexcept; +void operator delete[](void *ptr, const std::nothrow_t &) noexcept; #if __cpp_sized_deallocation >= 201309 /* C++14's sized-delete operators. */ -void operator delete(void *ptr, std::size_t size) noexcept; -void operator delete[](void *ptr, std::size_t size) noexcept; +void operator delete(void *ptr, std::size_t size) noexcept; +void operator delete[](void *ptr, std::size_t size) noexcept; #endif #if __cpp_aligned_new >= 201606 /* C++17's over-aligned operators. */ -void *operator new(std::size_t size, std::align_val_t); -void *operator new(std::size_t size, std::align_val_t, const std::nothrow_t &) noexcept; -void *operator new[](std::size_t size, std::align_val_t); -void *operator new[](std::size_t size, std::align_val_t, const std::nothrow_t &) noexcept; -void operator delete(void* ptr, std::align_val_t) noexcept; -void operator delete(void* ptr, std::align_val_t, const std::nothrow_t &) noexcept; -void operator delete(void* ptr, std::size_t size, std::align_val_t al) noexcept; -void operator delete[](void* ptr, std::align_val_t) noexcept; -void operator delete[](void* ptr, std::align_val_t, const std::nothrow_t &) noexcept; -void operator delete[](void* ptr, std::size_t size, std::align_val_t al) noexcept; +void *operator new(std::size_t size, std::align_val_t); +void *operator new( + std::size_t size, std::align_val_t, const std::nothrow_t &) noexcept; +void *operator new[](std::size_t size, std::align_val_t); +void *operator new[]( + std::size_t size, std::align_val_t, const std::nothrow_t &) noexcept; +void operator delete(void *ptr, std::align_val_t) noexcept; +void operator delete( + void *ptr, std::align_val_t, const std::nothrow_t &) noexcept; +void operator delete(void *ptr, std::size_t size, std::align_val_t al) noexcept; +void operator delete[](void *ptr, std::align_val_t) noexcept; +void operator delete[]( + void *ptr, std::align_val_t, const std::nothrow_t &) noexcept; +void operator delete[]( + void *ptr, std::size_t size, std::align_val_t al) noexcept; #endif JEMALLOC_NOINLINE static void * handleOOM(std::size_t size, bool nothrow) { if (opt_experimental_infallible_new) { - safety_check_fail(": Allocation failed and " - "opt.experimental_infallible_new is true. Aborting.\n"); + const char *huge_warning = (size >= ((std::size_t)1 << 30)) + ? "This may be caused by heap corruption, if the large size " + "is unexpected (suggest building with sanitizers for " + "debugging)." + : ""; + + safety_check_fail( + ": Allocation of size %zu failed. " + "%s opt.experimental_infallible_new is true. Aborting.\n", + size, huge_warning); return nullptr; } void *ptr = nullptr; while (ptr == nullptr) { - std::new_handler handler; - // GCC-4.8 and clang 4.0 do not have std::get_new_handler. - { - static std::mutex mtx; - std::lock_guard lock(mtx); - - handler = std::set_new_handler(nullptr); - std::set_new_handler(handler); - } + std::new_handler handler = std::get_new_handler(); if (handler == nullptr) break; +#ifdef JEMALLOC_HAVE_CXX_EXCEPTIONS try { handler(); } catch (const std::bad_alloc &) { break; } +#else + handler(); +#endif ptr = je_malloc(size); } - if (ptr == nullptr && !nothrow) - std::__throw_bad_alloc(); + if (ptr == nullptr && !nothrow) { +#ifdef JEMALLOC_HAVE_CXX_EXCEPTIONS + throw std::bad_alloc(); +#else + std::terminate(); +#endif + } return ptr; } template -JEMALLOC_NOINLINE -static void * -fallback_impl(std::size_t size) noexcept(IsNoExcept) { +JEMALLOC_NOINLINE static void * +fallbackNewImpl(std::size_t size) noexcept(IsNoExcept) { void *ptr = malloc_default(size); if (likely(ptr != nullptr)) { return ptr; @@ -103,10 +116,14 @@ fallback_impl(std::size_t size) noexcept(IsNoExcept) { } template -JEMALLOC_ALWAYS_INLINE -void * +JEMALLOC_ALWAYS_INLINE void * newImpl(std::size_t size) noexcept(IsNoExcept) { - return imalloc_fastpath(size, &fallback_impl); + LOG("core.operator_new.entry", "size: %zu", size); + + void *ret = imalloc_fastpath(size, &fallbackNewImpl); + + LOG("core.operator_new.exit", "result: %p", ret); + return ret; } void * @@ -132,9 +149,9 @@ operator new[](std::size_t size, const std::nothrow_t &) noexcept { #if __cpp_aligned_new >= 201606 template -JEMALLOC_ALWAYS_INLINE -void * -alignedNewImpl(std::size_t size, std::align_val_t alignment) noexcept(IsNoExcept) { +JEMALLOC_ALWAYS_INLINE void * +alignedNewImpl(std::size_t size, std::align_val_t alignment) noexcept( + IsNoExcept) { void *ptr = je_aligned_alloc(static_cast(alignment), size); if (likely(ptr != nullptr)) { return ptr; @@ -154,45 +171,68 @@ operator new[](std::size_t size, std::align_val_t alignment) { } void * -operator new(std::size_t size, std::align_val_t alignment, const std::nothrow_t &) noexcept { +operator new(std::size_t size, std::align_val_t alignment, + const std::nothrow_t &) noexcept { return alignedNewImpl(size, alignment); } void * -operator new[](std::size_t size, std::align_val_t alignment, const std::nothrow_t &) noexcept { +operator new[](std::size_t size, std::align_val_t alignment, + const std::nothrow_t &) noexcept { return alignedNewImpl(size, alignment); } -#endif // __cpp_aligned_new +#endif // __cpp_aligned_new void operator delete(void *ptr) noexcept { - je_free(ptr); + LOG("core.operator_delete.entry", "ptr: %p", ptr); + + je_free_impl(ptr); + + LOG("core.operator_delete.exit", ""); } void operator delete[](void *ptr) noexcept { - je_free(ptr); + LOG("core.operator_delete.entry", "ptr: %p", ptr); + + je_free_impl(ptr); + + LOG("core.operator_delete.exit", ""); } void operator delete(void *ptr, const std::nothrow_t &) noexcept { - je_free(ptr); + LOG("core.operator_delete.entry", "ptr: %p", ptr); + + je_free_impl(ptr); + + LOG("core.operator_delete.exit", ""); } -void operator delete[](void *ptr, const std::nothrow_t &) noexcept { - je_free(ptr); +void +operator delete[](void *ptr, const std::nothrow_t &) noexcept { + LOG("core.operator_delete.entry", "ptr: %p", ptr); + + je_free_impl(ptr); + + LOG("core.operator_delete.exit", ""); } #if __cpp_sized_deallocation >= 201309 JEMALLOC_ALWAYS_INLINE void -sizedDeleteImpl(void* ptr, std::size_t size) noexcept { +sizedDeleteImpl(void *ptr, std::size_t size) noexcept { if (unlikely(ptr == nullptr)) { return; } + LOG("core.operator_delete.entry", "ptr: %p, size: %zu", ptr, size); + je_sdallocx_noflags(ptr, size); + + LOG("core.operator_delete.exit", ""); } void @@ -205,50 +245,76 @@ operator delete[](void *ptr, std::size_t size) noexcept { sizedDeleteImpl(ptr, size); } -#endif // __cpp_sized_deallocation +#endif // __cpp_sized_deallocation #if __cpp_aligned_new >= 201606 JEMALLOC_ALWAYS_INLINE void -alignedSizedDeleteImpl(void* ptr, std::size_t size, std::align_val_t alignment) noexcept { +alignedSizedDeleteImpl( + void *ptr, std::size_t size, std::align_val_t alignment) noexcept { if (config_debug) { assert(((size_t)alignment & ((size_t)alignment - 1)) == 0); } if (unlikely(ptr == nullptr)) { return; } - je_sdallocx(ptr, size, MALLOCX_ALIGN(alignment)); + LOG("core.operator_delete.entry", "ptr: %p, size: %zu, alignment: %zu", + ptr, size, alignment); + + je_sdallocx_impl(ptr, size, MALLOCX_ALIGN(alignment)); + + LOG("core.operator_delete.exit", ""); } void -operator delete(void* ptr, std::align_val_t) noexcept { - je_free(ptr); +operator delete(void *ptr, std::align_val_t) noexcept { + LOG("core.operator_delete.entry", "ptr: %p", ptr); + + je_free_impl(ptr); + + LOG("core.operator_delete.exit", ""); } void -operator delete[](void* ptr, std::align_val_t) noexcept { - je_free(ptr); +operator delete[](void *ptr, std::align_val_t) noexcept { + LOG("core.operator_delete.entry", "ptr: %p", ptr); + + je_free_impl(ptr); + + LOG("core.operator_delete.exit", ""); } void -operator delete(void* ptr, std::align_val_t, const std::nothrow_t&) noexcept { - je_free(ptr); +operator delete(void *ptr, std::align_val_t, const std::nothrow_t &) noexcept { + LOG("core.operator_delete.entry", "ptr: %p", ptr); + + je_free_impl(ptr); + + LOG("core.operator_delete.exit", ""); } void -operator delete[](void* ptr, std::align_val_t, const std::nothrow_t&) noexcept { - je_free(ptr); +operator delete[]( + void *ptr, std::align_val_t, const std::nothrow_t &) noexcept { + LOG("core.operator_delete.entry", "ptr: %p", ptr); + + je_free_impl(ptr); + + LOG("core.operator_delete.exit", ""); } void -operator delete(void* ptr, std::size_t size, std::align_val_t alignment) noexcept { +operator delete( + void *ptr, std::size_t size, std::align_val_t alignment) noexcept { alignedSizedDeleteImpl(ptr, size, alignment); } void -operator delete[](void* ptr, std::size_t size, std::align_val_t alignment) noexcept { +operator delete[]( + void *ptr, std::size_t size, std::align_val_t alignment) noexcept { alignedSizedDeleteImpl(ptr, size, alignment); } -#endif // __cpp_aligned_new +#endif // __cpp_aligned_new +// NOLINTEND(misc-use-anonymous-namespace) diff --git a/dep/jemalloc/src/large.c b/dep/jemalloc/src/large.c index 5fc4bf584..6ccf49d76 100644 --- a/dep/jemalloc/src/large.c +++ b/dep/jemalloc/src/large.c @@ -18,10 +18,10 @@ large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero) { } void * -large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, - bool zero) { - size_t ausize; - edata_t *edata; +large_palloc( + tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero) { + size_t ausize; + edata_t *edata; UNUSED bool idump JEMALLOC_CC_SILENCE_INIT(false); assert(!tsdn_null(tsdn) || arena != NULL); @@ -34,12 +34,14 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, if (likely(!tsdn_null(tsdn))) { arena = arena_choose_maybe_huge(tsdn_tsd(tsdn), arena, usize); } - if (unlikely(arena == NULL) || (edata = arena_extent_alloc_large(tsdn, - arena, usize, alignment, zero)) == NULL) { + if (unlikely(arena == NULL) + || (edata = arena_extent_alloc_large( + tsdn, arena, usize, alignment, zero)) + == NULL) { return NULL; } - /* See comments in arena_bin_slabs_full_insert(). */ + /* See comments in bin_slabs_full_insert(). */ if (!arena_is_auto(arena)) { /* Insert edata into large. */ malloc_mutex_lock(tsdn, &arena->large_mtx); @@ -53,10 +55,10 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, static bool large_ralloc_no_move_shrink(tsdn_t *tsdn, edata_t *edata, size_t usize) { - arena_t *arena = arena_get_from_edata(edata); + arena_t *arena = arena_get_from_edata(edata); ehooks_t *ehooks = arena_get_ehooks(arena); - size_t old_size = edata_size_get(edata); - size_t old_usize = edata_usize_get(edata); + size_t old_size = edata_size_get(edata); + size_t old_usize = edata_usize_get(edata); assert(old_usize > usize); @@ -80,8 +82,8 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, edata_t *edata, size_t usize) { } static bool -large_ralloc_no_move_expand(tsdn_t *tsdn, edata_t *edata, size_t usize, - bool zero) { +large_ralloc_no_move_expand( + tsdn_t *tsdn, edata_t *edata, size_t usize, bool zero) { arena_t *arena = arena_get_from_edata(edata); size_t old_size = edata_size_get(edata); @@ -112,11 +114,11 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, edata_t *edata, size_t usize, * offset from the beginning of the extent is a multiple * of CACHELINE in [0 .. PAGE). */ - void *zbase = (void *) - ((uintptr_t)edata_addr_get(edata) + old_usize); - void *zpast = PAGE_ADDR2BASE((void *)((uintptr_t)zbase + - PAGE)); - size_t nzero = (uintptr_t)zpast - (uintptr_t)zbase; + void *zbase = (void *)((byte_t *)edata_addr_get(edata) + + old_usize); + void *zpast = PAGE_ADDR2BASE( + (void *)((byte_t *)zbase + PAGE)); + size_t nzero = (byte_t *)zpast - (byte_t *)zbase; assert(nzero > 0); memset(zbase, 0, nzero); } @@ -134,19 +136,19 @@ large_ralloc_no_move(tsdn_t *tsdn, edata_t *edata, size_t usize_min, /* The following should have been caught by callers. */ assert(usize_min > 0 && usize_max <= SC_LARGE_MAXCLASS); /* Both allocation sizes must be large to avoid a move. */ - assert(oldusize >= SC_LARGE_MINCLASS - && usize_max >= SC_LARGE_MINCLASS); + assert(oldusize >= SC_LARGE_MINCLASS && usize_max >= SC_LARGE_MINCLASS); if (usize_max > oldusize) { /* Attempt to expand the allocation in-place. */ - if (!large_ralloc_no_move_expand(tsdn, edata, usize_max, - zero)) { + if (!large_ralloc_no_move_expand( + tsdn, edata, usize_max, zero)) { arena_decay_tick(tsdn, arena_get_from_edata(edata)); return false; } /* Try again, this time with usize_min. */ - if (usize_min < usize_max && usize_min > oldusize && - large_ralloc_no_move_expand(tsdn, edata, usize_min, zero)) { + if (usize_min < usize_max && usize_min > oldusize + && !large_ralloc_no_move_expand( + tsdn, edata, usize_min, zero)) { arena_decay_tick(tsdn, arena_get_from_edata(edata)); return false; } @@ -172,8 +174,8 @@ large_ralloc_no_move(tsdn_t *tsdn, edata_t *edata, size_t usize_min, } static void * -large_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, - size_t alignment, bool zero) { +large_ralloc_move_helper( + tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero) { if (alignment <= CACHELINE) { return large_malloc(tsdn, arena, usize, zero); } @@ -190,14 +192,13 @@ large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize, /* The following should have been caught by callers. */ assert(usize > 0 && usize <= SC_LARGE_MAXCLASS); /* Both allocation sizes must be large to avoid a move. */ - assert(oldusize >= SC_LARGE_MINCLASS - && usize >= SC_LARGE_MINCLASS); + assert(oldusize >= SC_LARGE_MINCLASS && usize >= SC_LARGE_MINCLASS); /* Try to avoid moving the allocation. */ if (!large_ralloc_no_move(tsdn, edata, usize, usize, zero)) { - hook_invoke_expand(hook_args->is_realloc - ? hook_expand_realloc : hook_expand_rallocx, ptr, oldusize, - usize, (uintptr_t)ptr, hook_args->args); + hook_invoke_expand(hook_args->is_realloc ? hook_expand_realloc + : hook_expand_rallocx, + ptr, oldusize, usize, (uintptr_t)ptr, hook_args->args); return edata_addr_get(edata); } @@ -206,17 +207,18 @@ large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize, * different size class. In that case, fall back to allocating new * space and copying. */ - void *ret = large_ralloc_move_helper(tsdn, arena, usize, alignment, - zero); + void *ret = large_ralloc_move_helper( + tsdn, arena, usize, alignment, zero); if (ret == NULL) { return NULL; } - hook_invoke_alloc(hook_args->is_realloc - ? hook_alloc_realloc : hook_alloc_rallocx, ret, (uintptr_t)ret, - hook_args->args); - hook_invoke_dalloc(hook_args->is_realloc - ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args); + hook_invoke_alloc( + hook_args->is_realloc ? hook_alloc_realloc : hook_alloc_rallocx, + ret, (uintptr_t)ret, hook_args->args); + hook_invoke_dalloc( + hook_args->is_realloc ? hook_dalloc_realloc : hook_dalloc_rallocx, + ptr, hook_args->args); size_t copysize = (usize < oldusize) ? usize : oldusize; memcpy(ret, edata_addr_get(edata), copysize); @@ -228,10 +230,10 @@ large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize, * locked indicates whether the arena's large_mtx is currently held. */ static void -large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, edata_t *edata, - bool locked) { +large_dalloc_prep_impl( + tsdn_t *tsdn, arena_t *arena, edata_t *edata, bool locked) { if (!locked) { - /* See comments in arena_bin_slabs_full_insert(). */ + /* See comments in bin_slabs_full_insert(). */ if (!arena_is_auto(arena)) { malloc_mutex_lock(tsdn, &arena->large_mtx); edata_list_active_remove(&arena->large, edata); @@ -274,22 +276,17 @@ large_dalloc(tsdn_t *tsdn, edata_t *edata) { arena_decay_tick(tsdn, arena); } -size_t -large_salloc(tsdn_t *tsdn, const edata_t *edata) { - return edata_usize_get(edata); -} - void -large_prof_info_get(tsd_t *tsd, edata_t *edata, prof_info_t *prof_info, - bool reset_recent) { +large_prof_info_get( + tsd_t *tsd, edata_t *edata, prof_info_t *prof_info, bool reset_recent) { assert(prof_info != NULL); prof_tctx_t *alloc_tctx = edata_prof_tctx_get(edata); prof_info->alloc_tctx = alloc_tctx; - if ((uintptr_t)alloc_tctx > (uintptr_t)1U) { - nstime_copy(&prof_info->alloc_time, - edata_prof_alloc_time_get(edata)); + if (prof_tctx_is_valid(alloc_tctx)) { + nstime_copy( + &prof_info->alloc_time, edata_prof_alloc_time_get(edata)); prof_info->alloc_size = edata_prof_alloc_size_get(edata); if (reset_recent) { /* @@ -308,7 +305,7 @@ large_prof_tctx_set(edata_t *edata, prof_tctx_t *tctx) { void large_prof_tctx_reset(edata_t *edata) { - large_prof_tctx_set(edata, (prof_tctx_t *)(uintptr_t)1U); + large_prof_tctx_set(edata, PROF_TCTX_SENTINEL); } void diff --git a/dep/jemalloc/src/log.c b/dep/jemalloc/src/log.c index 778902fb9..9b1c6261a 100644 --- a/dep/jemalloc/src/log.c +++ b/dep/jemalloc/src/log.c @@ -3,7 +3,7 @@ #include "jemalloc/internal/log.h" -char log_var_names[JEMALLOC_LOG_VAR_BUFSIZE]; +char log_var_names[JEMALLOC_LOG_VAR_BUFSIZE]; atomic_b_t log_init_done = ATOMIC_INIT(false); /* @@ -11,7 +11,7 @@ atomic_b_t log_init_done = ATOMIC_INIT(false); * with a pointer to the first character after the end of the string. */ static const char * -log_var_extract_segment(const char* segment_begin) { +log_var_extract_segment(const char *segment_begin) { const char *end; for (end = segment_begin; *end != '\0' && *end != '|'; end++) { } @@ -30,12 +30,12 @@ log_var_matches_segment(const char *segment_begin, const char *segment_end, if (segment_len == 1 && *segment_begin == '.') { return true; } - if (segment_len == log_var_len) { + if (segment_len == log_var_len) { return strncmp(segment_begin, log_var_begin, segment_len) == 0; } else if (segment_len < log_var_len) { return strncmp(segment_begin, log_var_begin, segment_len) == 0 && log_var_begin[segment_len] == '.'; - } else { + } else { return false; } } @@ -61,9 +61,9 @@ log_var_update_state(log_var_t *log_var) { segment_begin); assert(segment_end < log_var_names + JEMALLOC_LOG_VAR_BUFSIZE); if (log_var_matches_segment(segment_begin, segment_end, - log_var_begin, log_var_end)) { - atomic_store_u(&log_var->state, LOG_ENABLED, - ATOMIC_RELAXED); + log_var_begin, log_var_end)) { + atomic_store_u( + &log_var->state, LOG_ENABLED, ATOMIC_RELAXED); return LOG_ENABLED; } if (*segment_end == '\0') { diff --git a/dep/jemalloc/src/malloc_io.c b/dep/jemalloc/src/malloc_io.c index b76885cbb..9716c6682 100644 --- a/dep/jemalloc/src/malloc_io.c +++ b/dep/jemalloc/src/malloc_io.c @@ -5,63 +5,68 @@ #include "jemalloc/internal/util.h" #ifdef assert -# undef assert +# undef assert #endif #ifdef not_reached -# undef not_reached +# undef not_reached #endif #ifdef not_implemented -# undef not_implemented +# undef not_implemented #endif #ifdef assert_not_implemented -# undef assert_not_implemented +# undef assert_not_implemented #endif /* * Define simple versions of assertion macros that won't recurse in case * of assertion failures in malloc_*printf(). */ -#define assert(e) do { \ - if (config_debug && !(e)) { \ - malloc_write(": Failed assertion\n"); \ - abort(); \ - } \ -} while (0) +#define assert(e) \ + do { \ + if (config_debug && !(e)) { \ + malloc_write(": Failed assertion\n"); \ + abort(); \ + } \ + } while (0) -#define not_reached() do { \ - if (config_debug) { \ - malloc_write(": Unreachable code reached\n"); \ - abort(); \ - } \ - unreachable(); \ -} while (0) +#define not_reached() \ + do { \ + if (config_debug) { \ + malloc_write( \ + ": Unreachable code reached\n"); \ + abort(); \ + } \ + unreachable(); \ + } while (0) -#define not_implemented() do { \ - if (config_debug) { \ - malloc_write(": Not implemented\n"); \ - abort(); \ - } \ -} while (0) +#define not_implemented() \ + do { \ + if (config_debug) { \ + malloc_write(": Not implemented\n"); \ + abort(); \ + } \ + } while (0) -#define assert_not_implemented(e) do { \ - if (unlikely(config_debug && !(e))) { \ - not_implemented(); \ - } \ -} while (0) +#define assert_not_implemented(e) \ + do { \ + if (unlikely(config_debug && !(e))) { \ + not_implemented(); \ + } \ + } while (0) /******************************************************************************/ /* Function prototypes for non-inline static functions. */ #define U2S_BUFSIZE ((1U << (LG_SIZEOF_INTMAX_T + 3)) + 1) -static char *u2s(uintmax_t x, unsigned base, bool uppercase, char *s, - size_t *slen_p); +static char *u2s( + uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p); #define D2S_BUFSIZE (1 + U2S_BUFSIZE) static char *d2s(intmax_t x, char sign, char *s, size_t *slen_p); #define O2S_BUFSIZE (1 + U2S_BUFSIZE) static char *o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p); #define X2S_BUFSIZE (2 + U2S_BUFSIZE) -static char *x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, - size_t *slen_p); +static char *x2s( + uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p); /******************************************************************************/ @@ -71,7 +76,7 @@ wrtmessage(void *cbopaque, const char *s) { malloc_write_fd(STDERR_FILENO, s, strlen(s)); } -JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s); +JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s); /* * Wrapper around malloc_message() that avoids the need for @@ -93,14 +98,15 @@ malloc_write(const char *s) { int buferror(int err, char *buf, size_t buflen) { #ifdef _WIN32 - FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, err, 0, - (LPSTR)buf, (DWORD)buflen, NULL); + FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, err, 0, (LPSTR)buf, + (DWORD)buflen, NULL); return 0; -#elif defined(JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE) && defined(_GNU_SOURCE) +#elif defined(JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE) \ + && defined(_GNU_SOURCE) char *b = strerror_r(err, buf, buflen); if (b != buf) { strncpy(buf, b, buflen); - buf[buflen-1] = '\0'; + buf[buflen - 1] = '\0'; } return 0; #else @@ -110,9 +116,9 @@ buferror(int err, char *buf, size_t buflen) { uintmax_t malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base) { - uintmax_t ret, digit; - unsigned b; - bool neg; + uintmax_t ret, digit; + unsigned b; + bool neg; const char *p, *ns; p = nptr; @@ -128,7 +134,12 @@ malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base) { neg = false; while (true) { switch (*p) { - case '\t': case '\n': case '\v': case '\f': case '\r': case ' ': + case '\t': + case '\n': + case '\v': + case '\f': + case '\r': + case ' ': p++; break; case '-': @@ -142,8 +153,8 @@ malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base) { } } - /* Get prefix, if any. */ - label_prefix: +/* Get prefix, if any. */ +label_prefix: /* * Note where the first non-whitespace/sign character is so that it is * possible to tell whether any digits are consumed (e.g., " 0" vs. @@ -152,8 +163,14 @@ malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base) { ns = p; if (*p == '0') { switch (p[1]) { - case '0': case '1': case '2': case '3': case '4': case '5': - case '6': case '7': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': if (b == 0) { b = 8; } @@ -161,13 +178,30 @@ malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base) { p++; } break; - case 'X': case 'x': + case 'X': + case 'x': switch (p[2]) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - case 'A': case 'B': case 'C': case 'D': case 'E': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': case 'F': - case 'a': case 'b': case 'c': case 'd': case 'e': + case 'a': + case 'b': + case 'c': + case 'd': + case 'e': case 'f': if (b == 0) { b = 16; @@ -244,9 +278,8 @@ u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p) { } while (x > 0); break; case 16: { - const char *digits = (uppercase) - ? "0123456789ABCDEF" - : "0123456789abcdef"; + const char *digits = (uppercase) ? "0123456789ABCDEF" + : "0123456789abcdef"; do { i--; @@ -254,7 +287,8 @@ u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p) { x >>= 4; } while (x > 0); break; - } default: { + } + default: { const char *digits = (uppercase) ? "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" : "0123456789abcdefghijklmnopqrstuvwxyz"; @@ -265,7 +299,8 @@ u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p) { s[i] = digits[x % (uint64_t)base]; x /= (uint64_t)base; } while (x > 0); - }} + } + } *slen_p = U2S_BUFSIZE - 1 - i; return &s[i]; @@ -294,7 +329,8 @@ d2s(intmax_t x, char sign, char *s, size_t *slen_p) { (*slen_p)++; *s = sign; break; - default: not_reached(); + default: + not_reached(); } return s; } @@ -316,7 +352,8 @@ x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p) { if (alt_form) { s -= 2; (*slen_p) += 2; - memcpy(s, uppercase ? "0X" : "0x", 2); + s[0] = '0'; + s[1] = uppercase ? 'X' : 'x'; } return s; } @@ -324,107 +361,112 @@ x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p) { JEMALLOC_COLD size_t malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) { - size_t i; + size_t i; const char *f; -#define APPEND_C(c) do { \ - if (i < size) { \ - str[i] = (c); \ - } \ - i++; \ -} while (0) -#define APPEND_S(s, slen) do { \ - if (i < size) { \ - size_t cpylen = (slen <= size - i) ? slen : size - i; \ - memcpy(&str[i], s, cpylen); \ - } \ - i += slen; \ -} while (0) -#define APPEND_PADDED_S(s, slen, width, left_justify) do { \ - /* Left padding. */ \ - size_t pad_len = (width == -1) ? 0 : ((slen < (size_t)width) ? \ - (size_t)width - slen : 0); \ - if (!left_justify && pad_len != 0) { \ - size_t j; \ - for (j = 0; j < pad_len; j++) { \ - if (pad_zero) { \ - APPEND_C('0'); \ - } else { \ - APPEND_C(' '); \ - } \ - } \ - } \ - /* Value. */ \ - APPEND_S(s, slen); \ - /* Right padding. */ \ - if (left_justify && pad_len != 0) { \ - size_t j; \ - for (j = 0; j < pad_len; j++) { \ - APPEND_C(' '); \ - } \ - } \ -} while (0) -#define GET_ARG_NUMERIC(val, len) do { \ - switch ((unsigned char)len) { \ - case '?': \ - val = va_arg(ap, int); \ - break; \ - case '?' | 0x80: \ - val = va_arg(ap, unsigned int); \ - break; \ - case 'l': \ - val = va_arg(ap, long); \ - break; \ - case 'l' | 0x80: \ - val = va_arg(ap, unsigned long); \ - break; \ - case 'q': \ - val = va_arg(ap, long long); \ - break; \ - case 'q' | 0x80: \ - val = va_arg(ap, unsigned long long); \ - break; \ - case 'j': \ - val = va_arg(ap, intmax_t); \ - break; \ - case 'j' | 0x80: \ - val = va_arg(ap, uintmax_t); \ - break; \ - case 't': \ - val = va_arg(ap, ptrdiff_t); \ - break; \ - case 'z': \ - val = va_arg(ap, ssize_t); \ - break; \ - case 'z' | 0x80: \ - val = va_arg(ap, size_t); \ - break; \ - case 'p': /* Synthetic; used for %p. */ \ - val = va_arg(ap, uintptr_t); \ - break; \ - default: \ - not_reached(); \ - val = 0; \ - } \ -} while (0) +#define APPEND_C(c) \ + do { \ + if (i < size) { \ + str[i] = (c); \ + } \ + i++; \ + } while (0) +#define APPEND_S(s, slen) \ + do { \ + if (i < size) { \ + size_t cpylen = (slen <= size - i) ? slen : size - i; \ + memcpy(&str[i], s, cpylen); \ + } \ + i += slen; \ + } while (0) +#define APPEND_PADDED_S(s, slen, width, left_justify) \ + do { \ + /* Left padding. */ \ + size_t pad_len = (width == -1) \ + ? 0 \ + : ((slen < (size_t)width) ? (size_t)width - slen : 0); \ + if (!left_justify && pad_len != 0) { \ + size_t j; \ + for (j = 0; j < pad_len; j++) { \ + if (pad_zero) { \ + APPEND_C('0'); \ + } else { \ + APPEND_C(' '); \ + } \ + } \ + } \ + /* Value. */ \ + APPEND_S(s, slen); \ + /* Right padding. */ \ + if (left_justify && pad_len != 0) { \ + size_t j; \ + for (j = 0; j < pad_len; j++) { \ + APPEND_C(' '); \ + } \ + } \ + } while (0) +#define GET_ARG_NUMERIC(val, len) \ + do { \ + switch ((unsigned char)len) { \ + case '?': \ + val = va_arg(ap, int); \ + break; \ + case '?' | 0x80: \ + val = va_arg(ap, unsigned int); \ + break; \ + case 'l': \ + val = va_arg(ap, long); \ + break; \ + case 'l' | 0x80: \ + val = va_arg(ap, unsigned long); \ + break; \ + case 'q': \ + val = va_arg(ap, long long); \ + break; \ + case 'q' | 0x80: \ + val = va_arg(ap, unsigned long long); \ + break; \ + case 'j': \ + val = va_arg(ap, intmax_t); \ + break; \ + case 'j' | 0x80: \ + val = va_arg(ap, uintmax_t); \ + break; \ + case 't': \ + val = va_arg(ap, ptrdiff_t); \ + break; \ + case 'z': \ + val = va_arg(ap, ssize_t); \ + break; \ + case 'z' | 0x80: \ + val = va_arg(ap, size_t); \ + break; \ + case 'p': /* Synthetic; used for %p. */ \ + val = va_arg(ap, uintptr_t); \ + break; \ + default: \ + not_reached(); \ + val = 0; \ + } \ + } while (0) i = 0; f = format; while (true) { switch (*f) { - case '\0': goto label_out; + case '\0': + goto label_out; case '%': { - bool alt_form = false; - bool left_justify = false; - bool plus_space = false; - bool plus_plus = false; - int prec = -1; - int width = -1; + bool alt_form = false; + bool left_justify = false; + bool plus_space = false; + bool plus_plus = false; + int prec = -1; + int width = -1; unsigned char len = '?'; - char *s; - size_t slen; - bool first_width_digit = true; - bool pad_zero = false; + char *s; + size_t slen; + bool pad_zero = false; f++; /* Flags. */ @@ -446,12 +488,13 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) { assert(!plus_plus); plus_plus = true; break; - default: goto label_width; + default: + goto label_width; } f++; } - /* Width. */ - label_width: + /* Width. */ + label_width: switch (*f) { case '*': width = va_arg(ap, int); @@ -462,21 +505,26 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) { } break; case '0': - if (first_width_digit) { - pad_zero = true; - } + pad_zero = true; JEMALLOC_FALLTHROUGH; - case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': { + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': { uintmax_t uwidth; set_errno(0); uwidth = malloc_strtoumax(f, (char **)&f, 10); - assert(uwidth != UINTMAX_MAX || get_errno() != - ERANGE); + assert(uwidth != UINTMAX_MAX + || get_errno() != ERANGE); width = (int)uwidth; - first_width_digit = false; break; - } default: + } + default: break; } /* Width/precision separator. */ @@ -491,20 +539,29 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) { prec = va_arg(ap, int); f++; break; - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': { uintmax_t uprec; set_errno(0); uprec = malloc_strtoumax(f, (char **)&f, 10); - assert(uprec != UINTMAX_MAX || get_errno() != - ERANGE); + assert(uprec != UINTMAX_MAX + || get_errno() != ERANGE); prec = (int)uprec; break; } - default: break; + default: + break; } - /* Length. */ - label_length: + /* Length. */ + label_length: switch (*f) { case 'l': f++; @@ -515,11 +572,15 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) { len = 'l'; } break; - case 'q': case 'j': case 't': case 'z': + case 'q': + case 'j': + case 't': + case 'z': len = *f; f++; break; - default: break; + default: + break; } /* Conversion specifier. */ switch (*f) { @@ -528,9 +589,10 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) { APPEND_C(*f); f++; break; - case 'd': case 'i': { + case 'd': + case 'i': { intmax_t val JEMALLOC_CC_SILENCE_INIT(0); - char buf[D2S_BUFSIZE]; + char buf[D2S_BUFSIZE]; /* * Outputting negative, zero-padded numbers @@ -545,41 +607,48 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) { assert(!pad_zero); GET_ARG_NUMERIC(val, len); - s = d2s(val, (plus_plus ? '+' : (plus_space ? - ' ' : '-')), buf, &slen); + s = d2s(val, + (plus_plus ? '+' + : (plus_space ? ' ' : '-')), + buf, &slen); APPEND_PADDED_S(s, slen, width, left_justify); f++; break; - } case 'o': { + } + case 'o': { uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); - char buf[O2S_BUFSIZE]; + char buf[O2S_BUFSIZE]; GET_ARG_NUMERIC(val, len | 0x80); s = o2s(val, alt_form, buf, &slen); APPEND_PADDED_S(s, slen, width, left_justify); f++; break; - } case 'u': { + } + case 'u': { uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); - char buf[U2S_BUFSIZE]; + char buf[U2S_BUFSIZE]; GET_ARG_NUMERIC(val, len | 0x80); s = u2s(val, 10, false, buf, &slen); APPEND_PADDED_S(s, slen, width, left_justify); f++; break; - } case 'x': case 'X': { + } + case 'x': + case 'X': { uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); - char buf[X2S_BUFSIZE]; + char buf[X2S_BUFSIZE]; GET_ARG_NUMERIC(val, len | 0x80); s = x2s(val, alt_form, *f == 'X', buf, &slen); APPEND_PADDED_S(s, slen, width, left_justify); f++; break; - } case 'c': { + } + case 'c': { unsigned char val; - char buf[2]; + char buf[2]; assert(len == '?' || len == 'l'); assert_not_implemented(len != 'l'); @@ -589,7 +658,8 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) { APPEND_PADDED_S(buf, 1, width, left_justify); f++; break; - } case 's': + } + case 's': assert(len == '?' || len == 'l'); assert_not_implemented(len != 'l'); s = va_arg(ap, char *); @@ -599,26 +669,30 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) { break; case 'p': { uintmax_t val; - char buf[X2S_BUFSIZE]; + char buf[X2S_BUFSIZE]; GET_ARG_NUMERIC(val, 'p'); s = x2s(val, true, false, buf, &slen); APPEND_PADDED_S(s, slen, width, left_justify); f++; break; - } default: not_reached(); + } + default: + not_reached(); } break; - } default: { + } + default: { APPEND_C(*f); f++; break; - }} + } + } } - label_out: +label_out: if (i < size) { str[i] = '\0'; - } else { + } else if (size != 0) { str[size - 1] = '\0'; } @@ -632,7 +706,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) { JEMALLOC_FORMAT_PRINTF(3, 4) size_t malloc_snprintf(char *str, size_t size, const char *format, ...) { - size_t ret; + size_t ret; va_list ap; va_start(ap, format); @@ -643,8 +717,8 @@ malloc_snprintf(char *str, size_t size, const char *format, ...) { } void -malloc_vcprintf(write_cb_t *write_cb, void *cbopaque, const char *format, - va_list ap) { +malloc_vcprintf( + write_cb_t *write_cb, void *cbopaque, const char *format, va_list ap) { char buf[MALLOC_PRINTF_BUFSIZE]; if (write_cb == NULL) { @@ -653,8 +727,8 @@ malloc_vcprintf(write_cb_t *write_cb, void *cbopaque, const char *format, * function, so use the default one. malloc_write() is an * inline function, so use malloc_message() directly here. */ - write_cb = (je_malloc_message != NULL) ? je_malloc_message : - wrtmessage; + write_cb = (je_malloc_message != NULL) ? je_malloc_message + : wrtmessage; } malloc_vsnprintf(buf, sizeof(buf), format, ap); @@ -686,6 +760,81 @@ malloc_printf(const char *format, ...) { va_end(ap); } +static ssize_t +malloc_write_fd_syscall(int fd, const void *buf, size_t count) { +#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_write) + /* + * Use syscall(2) rather than write(2) when possible in order to avoid + * the possibility of memory allocation within libc. This is necessary + * on FreeBSD; most operating systems do not have this problem though. + * + * syscall() returns long or int, depending on platform, so capture the + * result in the widest plausible type to avoid compiler warnings. + */ + return (ssize_t)syscall(SYS_write, fd, buf, count); +#else + return (ssize_t)write(fd, buf, +# ifdef _WIN32 + (unsigned int) +# endif + count); +#endif +} + +ssize_t +malloc_write_fd(int fd, const void *buf, size_t count) { + size_t bytes_written = 0; + do { + ssize_t result = malloc_write_fd_syscall(fd, + &((const byte_t *)buf)[bytes_written], + count - bytes_written); + if (result < 0) { +#ifndef _WIN32 + if (errno == EINTR) { + continue; + } +#endif + return result; + } + bytes_written += result; + } while (bytes_written < count); + return bytes_written; +} + +static ssize_t +malloc_read_fd_syscall(int fd, void *buf, size_t count) { +#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_read) + return (ssize_t)syscall(SYS_read, fd, buf, count); +#else + return (ssize_t)read(fd, buf, +# ifdef _WIN32 + (unsigned int) +# endif + count); +#endif +} + +ssize_t +malloc_read_fd(int fd, void *buf, size_t count) { + size_t bytes_read = 0; + do { + ssize_t result = malloc_read_fd_syscall( + fd, &((byte_t *)buf)[bytes_read], count - bytes_read); + if (result < 0) { +#ifndef _WIN32 + if (errno == EINTR) { + continue; + } +#endif + return result; + } else if (result == 0) { + break; + } + bytes_read += result; + } while (bytes_read < count); + return bytes_read; +} + /* * Restore normal assertion macros, in order to make it possible to compile all * C files as a single concatenation. diff --git a/dep/jemalloc/src/mutex.c b/dep/jemalloc/src/mutex.c index 0b3547a87..aa2ab6659 100644 --- a/dep/jemalloc/src/mutex.c +++ b/dep/jemalloc/src/mutex.c @@ -5,8 +5,8 @@ #include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/spin.h" -#ifndef _CRT_SPINCOUNT -#define _CRT_SPINCOUNT 4000 +#if defined(_WIN32) && !defined(_CRT_SPINCOUNT) +# define _CRT_SPINCOUNT 4000 #endif /* @@ -22,8 +22,8 @@ int64_t opt_mutex_max_spin = 600; bool isthreaded = false; #endif #ifdef JEMALLOC_MUTEX_INIT_CB -static bool postpone_init = true; -static malloc_mutex_t *postponed_mutexes = NULL; +static bool postpone_init = true; +static malloc_mutex_t *postponed_mutexes = NULL; #endif /******************************************************************************/ @@ -44,14 +44,14 @@ pthread_create(pthread_t *__restrict thread, /******************************************************************************/ #ifdef JEMALLOC_MUTEX_INIT_CB -JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, - void *(calloc_cb)(size_t, size_t)); +JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb( + pthread_mutex_t *mutex, void *(calloc_cb)(size_t, size_t)); #endif void malloc_mutex_lock_slow(malloc_mutex_t *mutex) { mutex_prof_data_t *data = &mutex->prof_data; - nstime_t before; + nstime_t before; if (ncpus == 1) { goto label_spin_done; @@ -61,7 +61,7 @@ malloc_mutex_lock_slow(malloc_mutex_t *mutex) { do { spin_cpu_spinwait(); if (!atomic_load_b(&mutex->locked, ATOMIC_RELAXED) - && !malloc_mutex_trylock_final(mutex)) { + && !malloc_mutex_trylock_final(mutex)) { data->n_spin_acquired++; return; } @@ -77,8 +77,9 @@ label_spin_done: /* Copy before to after to avoid clock skews. */ nstime_t after; nstime_copy(&after, &before); - uint32_t n_thds = atomic_fetch_add_u32(&data->n_waiting_thds, 1, - ATOMIC_RELAXED) + 1; + uint32_t n_thds = atomic_fetch_add_u32( + &data->n_waiting_thds, 1, ATOMIC_RELAXED) + + 1; /* One last try as above two calls may take quite some cycles. */ if (!malloc_mutex_trylock_final(mutex)) { atomic_fetch_sub_u32(&data->n_waiting_thds, 1, ATOMIC_RELAXED); @@ -137,27 +138,28 @@ mutex_addr_comp(const witness_t *witness1, void *mutex1, } bool -malloc_mutex_init(malloc_mutex_t *mutex, const char *name, - witness_rank_t rank, malloc_mutex_lock_order_t lock_order) { +malloc_mutex_init(malloc_mutex_t *mutex, const char *name, witness_rank_t rank, + malloc_mutex_lock_order_t lock_order) { mutex_prof_data_init(&mutex->prof_data); #ifdef _WIN32 -# if _WIN32_WINNT >= 0x0600 +# if _WIN32_WINNT >= 0x0600 InitializeSRWLock(&mutex->lock); -# else - if (!InitializeCriticalSectionAndSpinCount(&mutex->lock, - _CRT_SPINCOUNT)) { +# else + if (!InitializeCriticalSectionAndSpinCount( + &mutex->lock, _CRT_SPINCOUNT)) { return true; } -# endif +# endif #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) - mutex->lock = OS_UNFAIR_LOCK_INIT; + mutex->lock = OS_UNFAIR_LOCK_INIT; #elif (defined(JEMALLOC_MUTEX_INIT_CB)) if (postpone_init) { mutex->postponed_next = postponed_mutexes; postponed_mutexes = mutex; } else { - if (_pthread_mutex_init_calloc_cb(&mutex->lock, - bootstrap_calloc) != 0) { + if (_pthread_mutex_init_calloc_cb( + &mutex->lock, bootstrap_calloc) + != 0) { return true; } } @@ -201,9 +203,10 @@ malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex) { #ifdef JEMALLOC_MUTEX_INIT_CB malloc_mutex_unlock(tsdn, mutex); #else - if (malloc_mutex_init(mutex, mutex->witness.name, - mutex->witness.rank, mutex->lock_order)) { - malloc_printf(": Error re-initializing mutex in " + if (malloc_mutex_init(mutex, mutex->witness.name, mutex->witness.rank, + mutex->lock_order)) { + malloc_printf( + ": Error re-initializing mutex in " "child\n"); if (opt_abort) { abort(); @@ -217,8 +220,9 @@ malloc_mutex_boot(void) { #ifdef JEMALLOC_MUTEX_INIT_CB postpone_init = false; while (postponed_mutexes != NULL) { - if (_pthread_mutex_init_calloc_cb(&postponed_mutexes->lock, - bootstrap_calloc) != 0) { + if (_pthread_mutex_init_calloc_cb( + &postponed_mutexes->lock, bootstrap_calloc) + != 0) { return true; } postponed_mutexes = postponed_mutexes->postponed_next; diff --git a/dep/jemalloc/src/nstime.c b/dep/jemalloc/src/nstime.c index a1a53777f..0dfbeda15 100644 --- a/dep/jemalloc/src/nstime.c +++ b/dep/jemalloc/src/nstime.c @@ -5,8 +5,8 @@ #include "jemalloc/internal/assert.h" -#define BILLION UINT64_C(1000000000) -#define MILLION UINT64_C(1000000) +#define BILLION UINT64_C(1000000000) +#define MILLION UINT64_C(1000000) static void nstime_set_initialized(nstime_t *time) { @@ -22,8 +22,8 @@ nstime_assert_initialized(const nstime_t *time) { * Some parts (e.g. stats) rely on memset to zero initialize. Treat * these as valid initialization. */ - assert(time->magic == NSTIME_MAGIC || - (time->magic == 0 && time->ns == 0)); + assert( + time->magic == NSTIME_MAGIC || (time->magic == 0 && time->ns == 0)); #endif } @@ -63,7 +63,7 @@ nstime_ns(const nstime_t *time) { } uint64_t -nstime_msec(const nstime_t *time) { +nstime_ms(const nstime_t *time) { nstime_assert_initialized(time); return time->ns / MILLION; } @@ -133,8 +133,10 @@ nstime_isubtract(nstime_t *time, uint64_t subtrahend) { void nstime_imultiply(nstime_t *time, uint64_t multiplier) { nstime_assert_initialized(time); - assert((((time->ns | multiplier) & (UINT64_MAX << (sizeof(uint64_t) << - 2))) == 0) || ((time->ns * multiplier) / multiplier == time->ns)); + assert( + (((time->ns | multiplier) & (UINT64_MAX << (sizeof(uint64_t) << 2))) + == 0) + || ((time->ns * multiplier) / multiplier == time->ns)); nstime_initialize_operand(time); time->ns *= multiplier; @@ -158,7 +160,20 @@ nstime_divide(const nstime_t *time, const nstime_t *divisor) { return time->ns / divisor->ns; } -/* Returns time since *past, w/o updating *past. */ +uint64_t +nstime_ns_between(const nstime_t *earlier, const nstime_t *later) { + nstime_assert_initialized(earlier); + nstime_assert_initialized(later); + assert(nstime_compare(later, earlier) >= 0); + return later->ns - earlier->ns; +} + +uint64_t +nstime_ms_between(const nstime_t *earlier, const nstime_t *later) { + return nstime_ns_between(earlier, later) / MILLION; +} + +/* Returns time since *past in nanoseconds, w/o updating *past. */ uint64_t nstime_ns_since(const nstime_t *past) { nstime_assert_initialized(past); @@ -166,13 +181,17 @@ nstime_ns_since(const nstime_t *past) { nstime_t now; nstime_copy(&now, past); nstime_update(&now); + return nstime_ns_between(past, &now); +} - assert(nstime_compare(&now, past) >= 0); - return now.ns - past->ns; +/* Returns time since *past in milliseconds, w/o updating *past. */ +uint64_t +nstime_ms_since(const nstime_t *past) { + return nstime_ns_since(past) / MILLION; } #ifdef _WIN32 -# define NSTIME_MONOTONIC true +# define NSTIME_MONOTONIC false static void nstime_get(nstime_t *time) { FILETIME ft; @@ -184,7 +203,7 @@ nstime_get(nstime_t *time) { nstime_init(time, ticks_100ns * 100); } #elif defined(JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE) -# define NSTIME_MONOTONIC true +# define NSTIME_MONOTONIC true static void nstime_get(nstime_t *time) { struct timespec ts; @@ -193,7 +212,7 @@ nstime_get(nstime_t *time) { nstime_init2(time, ts.tv_sec, ts.tv_nsec); } #elif defined(JEMALLOC_HAVE_CLOCK_MONOTONIC) -# define NSTIME_MONOTONIC true +# define NSTIME_MONOTONIC true static void nstime_get(nstime_t *time) { struct timespec ts; @@ -201,14 +220,25 @@ nstime_get(nstime_t *time) { clock_gettime(CLOCK_MONOTONIC, &ts); nstime_init2(time, ts.tv_sec, ts.tv_nsec); } -#elif defined(JEMALLOC_HAVE_MACH_ABSOLUTE_TIME) -# define NSTIME_MONOTONIC true +#elif defined(JEMALLOC_HAVE_CLOCK_GETTIME_NSEC_NP) +# define NSTIME_MONOTONIC true static void nstime_get(nstime_t *time) { - nstime_init(time, mach_absolute_time()); + nstime_init(time, clock_gettime_nsec_np(CLOCK_UPTIME_RAW)); +} +#elif defined(JEMALLOC_HAVE_MACH_ABSOLUTE_TIME) +# define NSTIME_MONOTONIC true +static void +nstime_get(nstime_t *time) { + static mach_timebase_info_data_t sTimebaseInfo; + if (sTimebaseInfo.denom == 0) { + (void)mach_timebase_info(&sTimebaseInfo); + } + nstime_init(time, + mach_absolute_time() * sTimebaseInfo.numer / sTimebaseInfo.denom); } #else -# define NSTIME_MONOTONIC false +# define NSTIME_MONOTONIC false static void nstime_get(nstime_t *time) { struct timeval tv; @@ -225,15 +255,13 @@ nstime_monotonic_impl(void) { } nstime_monotonic_t *JET_MUTABLE nstime_monotonic = nstime_monotonic_impl; -prof_time_res_t opt_prof_time_res = - prof_time_res_default; +prof_time_res_t opt_prof_time_res = prof_time_res_default; -const char *prof_time_res_mode_names[] = { - "default", - "high", +const char *const prof_time_res_mode_names[] = { + "default", + "high", }; - static void nstime_get_realtime(nstime_t *time) { #if defined(JEMALLOC_HAVE_CLOCK_REALTIME) && !defined(_WIN32) @@ -285,5 +313,3 @@ nstime_prof_init_update(nstime_t *time) { nstime_init_zero(time); nstime_prof_update(time); } - - diff --git a/dep/jemalloc/src/pa.c b/dep/jemalloc/src/pa.c index eb7e4620e..a03b0c1c4 100644 --- a/dep/jemalloc/src/pa.c +++ b/dep/jemalloc/src/pa.c @@ -11,13 +11,13 @@ pa_nactive_add(pa_shard_t *shard, size_t add_pages) { static void pa_nactive_sub(pa_shard_t *shard, size_t sub_pages) { - assert(atomic_load_zu(&shard->nactive, ATOMIC_RELAXED) >= sub_pages); + assert(pa_shard_nactive(shard) >= sub_pages); atomic_fetch_sub_zu(&shard->nactive, sub_pages, ATOMIC_RELAXED); } bool pa_central_init(pa_central_t *central, base_t *base, bool hpa, - hpa_hooks_t *hpa_hooks) { + const hpa_hooks_t *hpa_hooks) { bool err; if (hpa) { err = hpa_central_init(¢ral->hpa, base, hpa_hooks); @@ -41,8 +41,8 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, pa_central_t *central, } if (pac_init(tsdn, &shard->pac, base, emap, &shard->edata_cache, - cur_time, pac_oversize_threshold, dirty_decay_ms, muzzy_decay_ms, - &stats->pac_stats, stats_mtx)) { + cur_time, pac_oversize_threshold, dirty_decay_ms, + muzzy_decay_ms, &stats->pac_stats, stats_mtx)) { return true; } @@ -67,12 +67,9 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, pa_central_t *central, bool pa_shard_enable_hpa(tsdn_t *tsdn, pa_shard_t *shard, const hpa_shard_opts_t *hpa_opts, const sec_opts_t *hpa_sec_opts) { - if (hpa_shard_init(&shard->hpa_shard, &shard->central->hpa, shard->emap, - shard->base, &shard->edata_cache, shard->ind, hpa_opts)) { - return true; - } - if (sec_init(tsdn, &shard->hpa_sec, shard->base, &shard->hpa_shard.pai, - hpa_sec_opts)) { + if (hpa_shard_init(tsdn, &shard->hpa_shard, &shard->central->hpa, + shard->emap, shard->base, &shard->edata_cache, shard->ind, + hpa_opts, hpa_sec_opts)) { return true; } shard->ever_used_hpa = true; @@ -85,7 +82,6 @@ void pa_shard_disable_hpa(tsdn_t *tsdn, pa_shard_t *shard) { atomic_store_b(&shard->use_hpa, false, ATOMIC_RELAXED); if (shard->ever_used_hpa) { - sec_disable(tsdn, &shard->hpa_sec); hpa_shard_disable(tsdn, &shard->hpa_shard); } } @@ -93,8 +89,13 @@ pa_shard_disable_hpa(tsdn_t *tsdn, pa_shard_t *shard) { void pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard) { atomic_store_zu(&shard->nactive, 0, ATOMIC_RELAXED); + pa_shard_flush(tsdn, shard); +} + +void +pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard) { if (shard->ever_used_hpa) { - sec_flush(tsdn, &shard->hpa_sec); + hpa_shard_flush(tsdn, &shard->hpa_shard); } } @@ -107,28 +108,27 @@ void pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard) { pac_destroy(tsdn, &shard->pac); if (shard->ever_used_hpa) { - sec_flush(tsdn, &shard->hpa_sec); - hpa_shard_disable(tsdn, &shard->hpa_shard); + hpa_shard_destroy(tsdn, &shard->hpa_shard); } } static pai_t * pa_get_pai(pa_shard_t *shard, edata_t *edata) { - return (edata_pai_get(edata) == EXTENT_PAI_PAC - ? &shard->pac.pai : &shard->hpa_sec.pai); + return (edata_pai_get(edata) == EXTENT_PAI_PAC ? &shard->pac.pai + : &shard->hpa_shard.pai); } edata_t * pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment, bool slab, szind_t szind, bool zero, bool guarded, bool *deferred_work_generated) { - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); assert(!guarded || alignment <= PAGE); edata_t *edata = NULL; if (!guarded && pa_shard_uses_hpa(shard)) { - edata = pai_alloc(tsdn, &shard->hpa_sec.pai, size, alignment, + edata = pai_alloc(tsdn, &shard->hpa_shard.pai, size, alignment, zero, /* guarded */ false, slab, deferred_work_generated); } /* @@ -190,8 +190,8 @@ pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size, size_t shrink_amount = old_size - new_size; pai_t *pai = pa_get_pai(shard, edata); - bool error = pai_shrink(tsdn, pai, edata, old_size, new_size, - deferred_work_generated); + bool error = pai_shrink( + tsdn, pai, edata, old_size, new_size, deferred_work_generated); if (error) { return true; } @@ -220,13 +220,6 @@ pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, pai_dalloc(tsdn, pai, edata, deferred_work_generated); } -bool -pa_shard_retain_grow_limit_get_set(tsdn_t *tsdn, pa_shard_t *shard, - size_t *old_limit, size_t *new_limit) { - return pac_retain_grow_limit_get_set(tsdn, &shard->pac, old_limit, - new_limit); -} - bool pa_decay_ms_set(tsdn_t *tsdn, pa_shard_t *shard, extent_state_t state, ssize_t decay_ms, pac_purge_eagerness_t eagerness) { @@ -239,11 +232,11 @@ pa_decay_ms_get(pa_shard_t *shard, extent_state_t state) { } void -pa_shard_set_deferral_allowed(tsdn_t *tsdn, pa_shard_t *shard, - bool deferral_allowed) { +pa_shard_set_deferral_allowed( + tsdn_t *tsdn, pa_shard_t *shard, bool deferral_allowed) { if (pa_shard_uses_hpa(shard)) { - hpa_shard_set_deferral_allowed(tsdn, &shard->hpa_shard, - deferral_allowed); + hpa_shard_set_deferral_allowed( + tsdn, &shard->hpa_shard, deferral_allowed); } } @@ -267,8 +260,8 @@ pa_shard_time_until_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) { } if (pa_shard_uses_hpa(shard)) { - uint64_t hpa = - pai_time_until_deferred_work(tsdn, &shard->hpa_shard.pai); + uint64_t hpa = pai_time_until_deferred_work( + tsdn, &shard->hpa_shard.pai); if (hpa < time) { time = hpa; } diff --git a/dep/jemalloc/src/pa_extra.c b/dep/jemalloc/src/pa_extra.c index 0f488be69..ff45674f2 100644 --- a/dep/jemalloc/src/pa_extra.c +++ b/dep/jemalloc/src/pa_extra.c @@ -17,7 +17,7 @@ pa_shard_prefork0(tsdn_t *tsdn, pa_shard_t *shard) { void pa_shard_prefork2(tsdn_t *tsdn, pa_shard_t *shard) { if (shard->ever_used_hpa) { - sec_prefork2(tsdn, &shard->hpa_sec); + hpa_shard_prefork2(tsdn, &shard->hpa_shard); } } @@ -54,7 +54,6 @@ pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard) { malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_dirty.mtx); malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_muzzy.mtx); if (shard->ever_used_hpa) { - sec_postfork_parent(tsdn, &shard->hpa_sec); hpa_shard_postfork_parent(tsdn, &shard->hpa_shard); } } @@ -69,24 +68,41 @@ pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard) { malloc_mutex_postfork_child(tsdn, &shard->pac.decay_dirty.mtx); malloc_mutex_postfork_child(tsdn, &shard->pac.decay_muzzy.mtx); if (shard->ever_used_hpa) { - sec_postfork_child(tsdn, &shard->hpa_sec); hpa_shard_postfork_child(tsdn, &shard->hpa_shard); } } +size_t +pa_shard_nactive(pa_shard_t *shard) { + return atomic_load_zu(&shard->nactive, ATOMIC_RELAXED); +} + +size_t +pa_shard_ndirty(pa_shard_t *shard) { + size_t ndirty = ecache_npages_get(&shard->pac.ecache_dirty); + if (shard->ever_used_hpa) { + ndirty += psset_ndirty(&shard->hpa_shard.psset); + } + return ndirty; +} + +size_t +pa_shard_nmuzzy(pa_shard_t *shard) { + return ecache_npages_get(&shard->pac.ecache_muzzy); +} + void -pa_shard_basic_stats_merge(pa_shard_t *shard, size_t *nactive, size_t *ndirty, - size_t *nmuzzy) { - *nactive += atomic_load_zu(&shard->nactive, ATOMIC_RELAXED); - *ndirty += ecache_npages_get(&shard->pac.ecache_dirty); - *nmuzzy += ecache_npages_get(&shard->pac.ecache_muzzy); +pa_shard_basic_stats_merge( + pa_shard_t *shard, size_t *nactive, size_t *ndirty, size_t *nmuzzy) { + *nactive += pa_shard_nactive(shard); + *ndirty += pa_shard_ndirty(shard); + *nmuzzy += pa_shard_nmuzzy(shard); } void pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard, pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out, - hpa_shard_stats_t *hpa_stats_out, sec_stats_t *sec_stats_out, - size_t *resident) { + hpa_shard_stats_t *hpa_stats_out, size_t *resident) { cassert(config_stats); pa_shard_stats_out->pac_stats.retained += @@ -95,37 +111,37 @@ pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard, &shard->edata_cache.count, ATOMIC_RELAXED); size_t resident_pgs = 0; - resident_pgs += atomic_load_zu(&shard->nactive, ATOMIC_RELAXED); - resident_pgs += ecache_npages_get(&shard->pac.ecache_dirty); + resident_pgs += pa_shard_nactive(shard); + resident_pgs += pa_shard_ndirty(shard); *resident += (resident_pgs << LG_PAGE); /* Dirty decay stats */ locked_inc_u64_unsynchronized( &pa_shard_stats_out->pac_stats.decay_dirty.npurge, locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx), - &shard->pac.stats->decay_dirty.npurge)); + &shard->pac.stats->decay_dirty.npurge)); locked_inc_u64_unsynchronized( &pa_shard_stats_out->pac_stats.decay_dirty.nmadvise, locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx), - &shard->pac.stats->decay_dirty.nmadvise)); + &shard->pac.stats->decay_dirty.nmadvise)); locked_inc_u64_unsynchronized( &pa_shard_stats_out->pac_stats.decay_dirty.purged, locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx), - &shard->pac.stats->decay_dirty.purged)); + &shard->pac.stats->decay_dirty.purged)); /* Muzzy decay stats */ locked_inc_u64_unsynchronized( &pa_shard_stats_out->pac_stats.decay_muzzy.npurge, locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx), - &shard->pac.stats->decay_muzzy.npurge)); + &shard->pac.stats->decay_muzzy.npurge)); locked_inc_u64_unsynchronized( &pa_shard_stats_out->pac_stats.decay_muzzy.nmadvise, locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx), - &shard->pac.stats->decay_muzzy.nmadvise)); + &shard->pac.stats->decay_muzzy.nmadvise)); locked_inc_u64_unsynchronized( &pa_shard_stats_out->pac_stats.decay_muzzy.purged, locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx), - &shard->pac.stats->decay_muzzy.purged)); + &shard->pac.stats->decay_muzzy.purged)); atomic_load_add_store_zu(&pa_shard_stats_out->pac_stats.abandoned_vm, atomic_load_zu(&shard->pac.stats->abandoned_vm, ATOMIC_RELAXED)); @@ -138,8 +154,8 @@ pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard, retained = ecache_nextents_get(&shard->pac.ecache_retained, i); dirty_bytes = ecache_nbytes_get(&shard->pac.ecache_dirty, i); muzzy_bytes = ecache_nbytes_get(&shard->pac.ecache_muzzy, i); - retained_bytes = ecache_nbytes_get(&shard->pac.ecache_retained, - i); + retained_bytes = ecache_nbytes_get( + &shard->pac.ecache_retained, i); estats_out[i].ndirty = dirty; estats_out[i].nmuzzy = muzzy; @@ -151,7 +167,6 @@ pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard, if (shard->ever_used_hpa) { hpa_shard_stats_merge(tsdn, &shard->hpa_shard, hpa_stats_out); - sec_stats_merge(tsdn, &shard->hpa_sec, sec_stats_out); } } @@ -185,7 +200,7 @@ pa_shard_mtx_stats_read(tsdn_t *tsdn, pa_shard_t *shard, pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data, &shard->hpa_shard.grow_mtx, arena_prof_mutex_hpa_shard_grow); - sec_mutex_stats_read(tsdn, &shard->hpa_sec, + sec_mutex_stats_read(tsdn, &shard->hpa_shard.sec, &mutex_prof_data[arena_prof_mutex_hpa_sec]); } } diff --git a/dep/jemalloc/src/pac.c b/dep/jemalloc/src/pac.c index 53e3d8237..ed0f77c2a 100644 --- a/dep/jemalloc/src/pac.c +++ b/dep/jemalloc/src/pac.c @@ -7,18 +7,18 @@ static edata_t *pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero, bool guarded, bool frequent_reuse, bool *deferred_work_generated); -static bool pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, - size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated); -static bool pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, - size_t old_size, size_t new_size, bool *deferred_work_generated); -static void pac_dalloc_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, - bool *deferred_work_generated); +static bool pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, + size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated); +static bool pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, + size_t old_size, size_t new_size, bool *deferred_work_generated); +static void pac_dalloc_impl( + tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated); static uint64_t pac_time_until_deferred_work(tsdn_t *tsdn, pai_t *self); static inline void -pac_decay_data_get(pac_t *pac, extent_state_t state, - decay_t **r_decay, pac_decay_stats_t **r_decay_stats, ecache_t **r_ecache) { - switch(state) { +pac_decay_data_get(pac_t *pac, extent_state_t state, decay_t **r_decay, + pac_decay_stats_t **r_decay_stats, ecache_t **r_ecache) { + switch (state) { case extent_state_dirty: *r_decay = &pac->decay_dirty; *r_decay_stats = &pac->stats->decay_dirty; @@ -29,6 +29,10 @@ pac_decay_data_get(pac_t *pac, extent_state_t state, *r_decay_stats = &pac->stats->decay_muzzy; *r_ecache = &pac->ecache_muzzy; return; + case extent_state_active: + case extent_state_retained: + case extent_state_transition: + case extent_state_merging: default: unreachable(); } @@ -47,7 +51,7 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap, * merging/splitting extents is non-trivial. */ if (ecache_init(tsdn, &pac->ecache_dirty, extent_state_dirty, ind, - /* delay_coalesce */ true)) { + /* delay_coalesce */ true)) { return true; } /* @@ -55,7 +59,7 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap, * the critical path much less often than for dirty extents. */ if (ecache_init(tsdn, &pac->ecache_muzzy, extent_state_muzzy, ind, - /* delay_coalesce */ false)) { + /* delay_coalesce */ false)) { return true; } /* @@ -64,17 +68,17 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap, * coalescing), but also because operations on retained extents are not * in the critical path. */ - if (ecache_init(tsdn, &pac->ecache_retained, extent_state_retained, - ind, /* delay_coalesce */ false)) { + if (ecache_init(tsdn, &pac->ecache_retained, extent_state_retained, ind, + /* delay_coalesce */ false)) { return true; } exp_grow_init(&pac->exp_grow); if (malloc_mutex_init(&pac->grow_mtx, "extent_grow", - WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) { + WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) { return true; } - atomic_store_zu(&pac->oversize_threshold, pac_oversize_threshold, - ATOMIC_RELAXED); + atomic_store_zu( + &pac->oversize_threshold, pac_oversize_threshold, ATOMIC_RELAXED); if (decay_init(&pac->decay_dirty, cur_time, dirty_decay_ms)) { return true; } @@ -93,11 +97,9 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap, atomic_store_zu(&pac->extent_sn_next, 0, ATOMIC_RELAXED); pac->pai.alloc = &pac_alloc_impl; - pac->pai.alloc_batch = &pai_alloc_batch_default; pac->pai.expand = &pac_expand_impl; pac->pai.shrink = &pac_shrink_impl; pac->pai.dalloc = &pac_dalloc_impl; - pac->pai.dalloc_batch = &pai_dalloc_batch_default; pac->pai.time_until_deferred_work = &pac_time_until_deferred_work; return false; @@ -108,10 +110,28 @@ pac_may_have_muzzy(pac_t *pac) { return pac_decay_ms_get(pac, extent_state_muzzy) != 0; } +static size_t +pac_alloc_retained_batched_size(size_t size) { + if (size > SC_LARGE_MAXCLASS) { + /* + * A valid input with usize SC_LARGE_MAXCLASS could still + * reach here because of sz_large_pad. Such a request is valid + * but we should not further increase it. Thus, directly + * return size for such cases. + */ + return size; + } + size_t batched_size = sz_s2u_compute_using_delta(size); + size_t next_hugepage_size = HUGEPAGE_CEILING(size); + return batched_size > next_hugepage_size ? next_hugepage_size + : batched_size; +} + static edata_t * pac_alloc_real(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size, size_t alignment, bool zero, bool guarded) { assert(!guarded || alignment <= PAGE); + size_t newly_mapped_size = 0; edata_t *edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty, NULL, size, alignment, zero, guarded); @@ -120,16 +140,74 @@ pac_alloc_real(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size, edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_muzzy, NULL, size, alignment, zero, guarded); } + + /* + * We batched allocate a larger extent with large size classes disabled + * because the reuse of extents in the dirty pool is worse without size + * classes for large allocs. For instance, when + * disable_large_size_classes is false, 1.1MB, 1.15MB, and 1.2MB allocs + * will all be ceiled to 1.25MB and can reuse the same buffer if they + * are alloc & dalloc sequentially. However, with + * disable_large_size_classes being true, they cannot reuse the same + * buffer and their sequential allocs & dallocs will result in three + * different extents. Thus, we cache extra mergeable extents in the + * dirty pool to improve the reuse. We skip this optimization if both + * maps_coalesce and opt_retain are disabled because VM is not cheap + * enough in such cases to be used aggressively and extents cannot be + * merged at will (only extents from the same VirtualAlloc can be + * merged). Note that it could still be risky to cache more extents + * when either mpas_coalesce or opt_retain is enabled. Yet doing + * so is still beneficial in improving the reuse of extents with some + * limits. This choice should be reevaluated if + * pac_alloc_retained_batched_size is changed to be more aggressive. + */ + if (sz_large_size_classes_disabled() && edata == NULL + && (maps_coalesce || opt_retain)) { + size_t batched_size = pac_alloc_retained_batched_size(size); + /* + * Note that ecache_alloc_grow will try to retrieve virtual + * memory from both retained pool and directly from OS through + * extent_alloc_wrapper if the retained pool has no qualified + * extents. This is also why the overcaching still works even + * with opt_retain off. + */ + edata = ecache_alloc_grow(tsdn, pac, ehooks, + &pac->ecache_retained, NULL, batched_size, alignment, zero, + guarded); + + if (edata != NULL && batched_size > size) { + edata_t *trail = extent_split_wrapper(tsdn, pac, ehooks, + edata, size, batched_size - size, + /* holding_core_locks */ false); + if (trail == NULL) { + ecache_dalloc(tsdn, pac, ehooks, + &pac->ecache_retained, edata); + edata = NULL; + } else { + ecache_dalloc(tsdn, pac, ehooks, + &pac->ecache_dirty, trail); + } + } + + if (edata != NULL) { + newly_mapped_size = batched_size; + } + } + if (edata == NULL) { edata = ecache_alloc_grow(tsdn, pac, ehooks, &pac->ecache_retained, NULL, size, alignment, zero, guarded); - if (config_stats && edata != NULL) { - atomic_fetch_add_zu(&pac->stats->pac_mapped, size, - ATOMIC_RELAXED); + if (edata != NULL) { + newly_mapped_size = size; } } + if (config_stats && newly_mapped_size != 0) { + atomic_fetch_add_zu( + &pac->stats->pac_mapped, newly_mapped_size, ATOMIC_RELAXED); + } + return edata; } @@ -140,8 +218,8 @@ pac_alloc_new_guarded(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size, edata_t *edata; if (san_bump_enabled() && frequent_reuse) { - edata = san_bump_alloc(tsdn, &pac->sba, pac, ehooks, size, - zero); + edata = san_bump_alloc( + tsdn, &pac->sba, pac, ehooks, size, zero); } else { size_t size_with_guards = san_two_side_guarded_sz(size); /* Alloc a non-guarded extent first.*/ @@ -150,12 +228,12 @@ pac_alloc_new_guarded(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size, if (edata != NULL) { /* Add guards around it. */ assert(edata_size_get(edata) == size_with_guards); - san_guard_pages_two_sided(tsdn, ehooks, edata, - pac->emap, true); + san_guard_pages_two_sided( + tsdn, ehooks, edata, pac->emap, true); } } - assert(edata == NULL || (edata_guarded_get(edata) && - edata_size_get(edata) == size)); + assert(edata == NULL + || (edata_guarded_get(edata) && edata_size_get(edata) == size)); return edata; } @@ -164,7 +242,7 @@ static edata_t * pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero, bool guarded, bool frequent_reuse, bool *deferred_work_generated) { - pac_t *pac = (pac_t *)self; + pac_t *pac = (pac_t *)self; ehooks_t *ehooks = pac_ehooks_get(pac); edata_t *edata = NULL; @@ -175,13 +253,13 @@ pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, * for such allocations would always return NULL. * */ if (!guarded || frequent_reuse) { - edata = pac_alloc_real(tsdn, pac, ehooks, size, alignment, - zero, guarded); + edata = pac_alloc_real( + tsdn, pac, ehooks, size, alignment, zero, guarded); } if (edata == NULL && guarded) { /* No cached guarded extents; creating a new one. */ - edata = pac_alloc_new_guarded(tsdn, pac, ehooks, size, - alignment, zero, frequent_reuse); + edata = pac_alloc_new_guarded( + tsdn, pac, ehooks, size, alignment, zero, frequent_reuse); } return edata; @@ -190,7 +268,7 @@ pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, static bool pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated) { - pac_t *pac = (pac_t *)self; + pac_t *pac = (pac_t *)self; ehooks_t *ehooks = pac_ehooks_get(pac); size_t mapped_add = 0; @@ -219,8 +297,8 @@ pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, return true; } if (config_stats && mapped_add > 0) { - atomic_fetch_add_zu(&pac->stats->pac_mapped, mapped_add, - ATOMIC_RELAXED); + atomic_fetch_add_zu( + &pac->stats->pac_mapped, mapped_add, ATOMIC_RELAXED); } return false; } @@ -228,7 +306,7 @@ pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, static bool pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, size_t new_size, bool *deferred_work_generated) { - pac_t *pac = (pac_t *)self; + pac_t *pac = (pac_t *)self; ehooks_t *ehooks = pac_ehooks_get(pac); size_t shrink_amount = old_size - new_size; @@ -248,9 +326,9 @@ pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, } static void -pac_dalloc_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, - bool *deferred_work_generated) { - pac_t *pac = (pac_t *)self; +pac_dalloc_impl( + tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) { + pac_t *pac = (pac_t *)self; ehooks_t *ehooks = pac_ehooks_get(pac); if (edata_guarded_get(edata)) { @@ -267,10 +345,10 @@ pac_dalloc_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, * guarded). */ if (!edata_slab_get(edata) || !maps_coalesce) { - assert(edata_size_get(edata) >= SC_LARGE_MINCLASS || - !maps_coalesce); - san_unguard_pages_two_sided(tsdn, ehooks, edata, - pac->emap); + assert(edata_size_get(edata) >= SC_LARGE_MINCLASS + || !maps_coalesce); + san_unguard_pages_two_sided( + tsdn, ehooks, edata, pac->emap); } } @@ -285,8 +363,8 @@ pac_ns_until_purge(tsdn_t *tsdn, decay_t *decay, size_t npages) { /* Use minimal interval if decay is contended. */ return BACKGROUND_THREAD_DEFERRED_MIN; } - uint64_t result = decay_ns_until_purge(decay, npages, - ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD); + uint64_t result = decay_ns_until_purge( + decay, npages, ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD); malloc_mutex_unlock(tsdn, &decay->mtx); return result; @@ -295,18 +373,16 @@ pac_ns_until_purge(tsdn_t *tsdn, decay_t *decay, size_t npages) { static uint64_t pac_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) { uint64_t time; - pac_t *pac = (pac_t *)self; + pac_t *pac = (pac_t *)self; - time = pac_ns_until_purge(tsdn, - &pac->decay_dirty, - ecache_npages_get(&pac->ecache_dirty)); + time = pac_ns_until_purge( + tsdn, &pac->decay_dirty, ecache_npages_get(&pac->ecache_dirty)); if (time == BACKGROUND_THREAD_DEFERRED_MIN) { return time; } - uint64_t muzzy = pac_ns_until_purge(tsdn, - &pac->decay_muzzy, - ecache_npages_get(&pac->ecache_muzzy)); + uint64_t muzzy = pac_ns_until_purge( + tsdn, &pac->decay_muzzy, ecache_npages_get(&pac->ecache_muzzy)); if (muzzy < time) { time = muzzy; } @@ -314,8 +390,8 @@ pac_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) { } bool -pac_retain_grow_limit_get_set(tsdn_t *tsdn, pac_t *pac, size_t *old_limit, - size_t *new_limit) { +pac_retain_grow_limit_get_set( + tsdn_t *tsdn, pac_t *pac, size_t *old_limit, size_t *new_limit) { pszind_t new_ind JEMALLOC_CC_SILENCE_INIT(0); if (new_limit != NULL) { size_t limit = *new_limit; @@ -341,15 +417,15 @@ static size_t pac_stash_decayed(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache, size_t npages_limit, size_t npages_decay_max, edata_list_inactive_t *result) { - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); ehooks_t *ehooks = pac_ehooks_get(pac); /* Stash extents according to npages_limit. */ size_t nstashed = 0; while (nstashed < npages_decay_max) { - edata_t *edata = ecache_evict(tsdn, pac, ehooks, ecache, - npages_limit); + edata_t *edata = ecache_evict( + tsdn, pac, ehooks, ecache, npages_limit); if (edata == NULL) { break; } @@ -359,6 +435,44 @@ pac_stash_decayed(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache, return nstashed; } +static bool +decay_with_process_madvise(edata_list_inactive_t *decay_extents) { + cassert(have_process_madvise); + assert(opt_process_madvise_max_batch > 0); +#ifndef JEMALLOC_HAVE_PROCESS_MADVISE + return true; +#else + assert( + opt_process_madvise_max_batch <= PROCESS_MADVISE_MAX_BATCH_LIMIT); + size_t len = opt_process_madvise_max_batch; + VARIABLE_ARRAY(struct iovec, vec, len); + + size_t cur = 0, total_bytes = 0; + for (edata_t *edata = edata_list_inactive_first(decay_extents); + edata != NULL; + edata = edata_list_inactive_next(decay_extents, edata)) { + size_t pages_bytes = edata_size_get(edata); + vec[cur].iov_base = edata_base_get(edata); + vec[cur].iov_len = pages_bytes; + total_bytes += pages_bytes; + cur++; + if (cur == len) { + bool err = pages_purge_process_madvise( + vec, len, total_bytes); + if (err) { + return true; + } + cur = 0; + total_bytes = 0; + } + } + if (cur > 0) { + return pages_purge_process_madvise(vec, cur, total_bytes); + } + return false; +#endif +} + static size_t pac_decay_stashed(tsdn_t *tsdn, pac_t *pac, decay_t *decay, pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay, @@ -374,8 +488,30 @@ pac_decay_stashed(tsdn_t *tsdn, pac_t *pac, decay_t *decay, bool try_muzzy = !fully_decay && pac_decay_ms_get(pac, extent_state_muzzy) != 0; - for (edata_t *edata = edata_list_inactive_first(decay_extents); edata != - NULL; edata = edata_list_inactive_first(decay_extents)) { + bool purge_to_retained = !try_muzzy + || ecache->state == extent_state_muzzy; + /* + * Attempt process_madvise only if 1) enabled, 2) purging to retained, + * and 3) not using custom hooks. + */ + bool try_process_madvise = (opt_process_madvise_max_batch > 0) + && purge_to_retained && ehooks_dalloc_will_fail(ehooks); + + bool already_purged; + if (try_process_madvise) { + /* + * If anything unexpected happened during process_madvise + * (e.g. not supporting MADV_DONTNEED, or partial success for + * some reason), we will consider nothing is purged and fallback + * to the regular madvise. + */ + already_purged = !decay_with_process_madvise(decay_extents); + } else { + already_purged = false; + } + + for (edata_t *edata = edata_list_inactive_first(decay_extents); + edata != NULL; edata = edata_list_inactive_first(decay_extents)) { edata_list_inactive_remove(decay_extents, edata); size_t size = edata_size_get(edata); @@ -385,12 +521,10 @@ pac_decay_stashed(tsdn_t *tsdn, pac_t *pac, decay_t *decay, npurged += npages; switch (ecache->state) { - case extent_state_active: - not_reached(); case extent_state_dirty: if (try_muzzy) { - err = extent_purge_lazy_wrapper(tsdn, ehooks, - edata, /* offset */ 0, size); + err = extent_purge_lazy_wrapper( + tsdn, ehooks, edata, /* offset */ 0, size); if (!err) { ecache_dalloc(tsdn, pac, ehooks, &pac->ecache_muzzy, edata); @@ -399,10 +533,18 @@ pac_decay_stashed(tsdn_t *tsdn, pac_t *pac, decay_t *decay, } JEMALLOC_FALLTHROUGH; case extent_state_muzzy: - extent_dalloc_wrapper(tsdn, pac, ehooks, edata); + if (already_purged) { + extent_dalloc_wrapper_purged( + tsdn, pac, ehooks, edata); + } else { + extent_dalloc_wrapper(tsdn, pac, ehooks, edata); + } nunmapped += npages; break; + case extent_state_active: case extent_state_retained: + case extent_state_transition: + case extent_state_merging: default: not_reached(); } @@ -435,8 +577,8 @@ static void pac_decay_to_limit(tsdn_t *tsdn, pac_t *pac, decay_t *decay, pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay, size_t npages_limit, size_t npages_decay_max) { - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 1); + witness_assert_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 1); if (decay->purging || npages_decay_max == 0) { return; @@ -446,8 +588,8 @@ pac_decay_to_limit(tsdn_t *tsdn, pac_t *pac, decay_t *decay, edata_list_inactive_t decay_extents; edata_list_inactive_init(&decay_extents); - size_t npurge = pac_stash_decayed(tsdn, pac, ecache, npages_limit, - npages_decay_max, &decay_extents); + size_t npurge = pac_stash_decayed( + tsdn, pac, ecache, npages_limit, npages_decay_max, &decay_extents); if (npurge != 0) { size_t npurged = pac_decay_stashed(tsdn, pac, decay, decay_stats, ecache, fully_decay, &decay_extents); @@ -468,8 +610,8 @@ pac_decay_all(tsdn_t *tsdn, pac_t *pac, decay_t *decay, static void pac_decay_try_purge(tsdn_t *tsdn, pac_t *pac, decay_t *decay, - pac_decay_stats_t *decay_stats, ecache_t *ecache, - size_t current_npages, size_t npages_limit) { + pac_decay_stats_t *decay_stats, ecache_t *ecache, size_t current_npages, + size_t npages_limit) { if (current_npages > npages_limit) { pac_decay_to_limit(tsdn, pac, decay, decay_stats, ecache, /* fully_decay */ false, npages_limit, @@ -504,8 +646,8 @@ pac_maybe_decay_purge(tsdn_t *tsdn, pac_t *pac, decay_t *decay, nstime_t time; nstime_init_update(&time); size_t npages_current = ecache_npages_get(ecache); - bool epoch_advanced = decay_maybe_advance_epoch(decay, &time, - npages_current); + bool epoch_advanced = decay_maybe_advance_epoch( + decay, &time, npages_current); if (eagerness == PAC_PURGE_ALWAYS || (epoch_advanced && eagerness == PAC_PURGE_ON_EPOCH_ADVANCE)) { size_t npages_limit = decay_npages_limit_get(decay); @@ -519,9 +661,9 @@ pac_maybe_decay_purge(tsdn_t *tsdn, pac_t *pac, decay_t *decay, bool pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state, ssize_t decay_ms, pac_purge_eagerness_t eagerness) { - decay_t *decay; + decay_t *decay; pac_decay_stats_t *decay_stats; - ecache_t *ecache; + ecache_t *ecache; pac_decay_data_get(pac, state, &decay, &decay_stats, &ecache); if (!decay_ms_valid(decay_ms)) { @@ -548,9 +690,9 @@ pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state, ssize_t pac_decay_ms_get(pac_t *pac, extent_state_t state) { - decay_t *decay; + decay_t *decay; pac_decay_stats_t *decay_stats; - ecache_t *ecache; + ecache_t *ecache; pac_decay_data_get(pac, state, &decay, &decay_stats, &ecache); return decay_ms_read(decay); } @@ -579,9 +721,10 @@ pac_destroy(tsdn_t *tsdn, pac_t *pac) { * dss-based extents for later reuse. */ ehooks_t *ehooks = pac_ehooks_get(pac); - edata_t *edata; - while ((edata = ecache_evict(tsdn, pac, ehooks, - &pac->ecache_retained, 0)) != NULL) { + edata_t *edata; + while ( + (edata = ecache_evict(tsdn, pac, ehooks, &pac->ecache_retained, 0)) + != NULL) { extent_destroy_wrapper(tsdn, pac, ehooks, edata); } } diff --git a/dep/jemalloc/src/pages.c b/dep/jemalloc/src/pages.c index 8c83a7de7..5c12ae426 100644 --- a/dep/jemalloc/src/pages.c +++ b/dep/jemalloc/src/pages.c @@ -8,41 +8,46 @@ #include "jemalloc/internal/malloc_io.h" #ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT -#include -#ifdef __FreeBSD__ -#include -#endif +# include +# ifdef __FreeBSD__ +# include +# endif #endif #ifdef __NetBSD__ -#include /* ilog2 */ +# include /* ilog2 */ #endif #ifdef JEMALLOC_HAVE_VM_MAKE_TAG -#define PAGES_FD_TAG VM_MAKE_TAG(101U) +# define PAGES_FD_TAG VM_MAKE_TAG(254U) #else -#define PAGES_FD_TAG -1 +# define PAGES_FD_TAG -1 +#endif +#if defined(JEMALLOC_HAVE_PRCTL) && defined(JEMALLOC_PAGEID) +# include +# ifndef PR_SET_VMA +# define PR_SET_VMA 0x53564d41 +# define PR_SET_VMA_ANON_NAME 0 +# endif #endif /******************************************************************************/ /* Data. */ /* Actual operating system page size, detected during bootstrap, <= PAGE. */ -static size_t os_page; +size_t os_page; #ifndef _WIN32 -# define PAGES_PROT_COMMIT (PROT_READ | PROT_WRITE) -# define PAGES_PROT_DECOMMIT (PROT_NONE) -static int mmap_flags; +# define PAGES_PROT_COMMIT (PROT_READ | PROT_WRITE) +# define PAGES_PROT_DECOMMIT (PROT_NONE) +static int mmap_flags; #endif -static bool os_overcommits; +static bool os_overcommits; -const char *thp_mode_names[] = { - "default", - "always", - "never", - "not supported" -}; -thp_mode_t opt_thp = THP_MODE_DEFAULT; -thp_mode_t init_system_thp_mode; +const char *const thp_mode_names[] = { + "default", "always", "never", "not supported"}; +const char *const system_thp_mode_names[] = { + "madvise", "always", "never", "not supported"}; +thp_mode_t opt_thp = THP_MODE_DEFAULT; +system_thp_mode_t init_system_thp_mode; /* Runtime support for lazy purge. Irrelevant when !pages_can_purge_lazy. */ static bool pages_can_purge_lazy_runtime = true; @@ -59,16 +64,16 @@ static int madvise_dont_need_zeros_is_faulty = -1; * * [1]: https://patchwork.kernel.org/patch/10576637/ */ -static int madvise_MADV_DONTNEED_zeroes_pages() -{ - int works = -1; +static int +madvise_MADV_DONTNEED_zeroes_pages(void) { size_t size = PAGE; - void * addr = mmap(NULL, size, PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + void *addr = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (addr == MAP_FAILED) { - malloc_write(": Cannot allocate memory for " + malloc_write( + ": Cannot allocate memory for " "MADV_DONTNEED check\n"); if (opt_abort) { abort(); @@ -76,6 +81,7 @@ static int madvise_MADV_DONTNEED_zeroes_pages() } memset(addr, 'A', size); + int works; if (madvise(addr, size, MADV_DONTNEED) == 0) { works = memchr(addr, 'A', size) == NULL; } else { @@ -87,7 +93,8 @@ static int madvise_MADV_DONTNEED_zeroes_pages() } if (munmap(addr, size) != 0) { - malloc_write(": Cannot deallocate memory for " + malloc_write( + ": Cannot deallocate memory for " "MADV_DONTNEED check\n"); if (opt_abort) { abort(); @@ -98,6 +105,26 @@ static int madvise_MADV_DONTNEED_zeroes_pages() } #endif +#ifdef JEMALLOC_PAGEID +static int +os_page_id(void *addr, size_t size, const char *name) { +# ifdef JEMALLOC_HAVE_PRCTL + /* + * While parsing `/proc//maps` file, the block could appear as + * 7f4836000000-7f4836800000 rw-p 00000000 00:00 0 [anon:jemalloc_pg_overcommit]` + */ + int n; + assert(addr != NULL); + n = prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, (uintptr_t)addr, size, + (uintptr_t)name); + assert(n == 0 || (n == -1 && get_errno() == EINVAL)); + return n; +# else + return 0; +# endif +} +#endif + /******************************************************************************/ /* * Function prototypes for static functions that are referenced prior to @@ -132,7 +159,8 @@ os_pages_map(void *addr, size_t size, size_t alignment, bool *commit) { * of existing mappings, and we only want to create new mappings. */ { -#ifdef __NetBSD__ + int flags = mmap_flags; +# ifdef __NetBSD__ /* * On NetBSD PAGE for a platform is defined to the * maximum page size of all machine architectures @@ -141,12 +169,12 @@ os_pages_map(void *addr, size_t size, size_t alignment, bool *commit) { */ if (alignment > os_page || PAGE > os_page) { unsigned int a = ilog2(MAX(alignment, PAGE)); - mmap_flags |= MAP_ALIGNED(a); + flags |= MAP_ALIGNED(a); } -#endif +# endif int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT; - ret = mmap(addr, size, prot, mmap_flags, PAGES_FD_TAG, 0); + ret = mmap(addr, size, prot, flags, PAGES_FD_TAG, 0); } assert(ret != NULL); @@ -160,15 +188,21 @@ os_pages_map(void *addr, size_t size, size_t alignment, bool *commit) { ret = NULL; } #endif - assert(ret == NULL || (addr == NULL && ret != addr) || (addr != NULL && - ret == addr)); + assert(ret == NULL || (addr == NULL && ret != addr) + || (addr != NULL && ret == addr)); +#ifdef JEMALLOC_PAGEID + if (ret != NULL) { + os_page_id(ret, size, + os_overcommits ? "jemalloc_pg_overcommit" : "jemalloc_pg"); + } +#endif return ret; } static void * -os_pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size, - bool *commit) { - void *ret = (void *)((uintptr_t)addr + leadsize); +os_pages_trim( + void *addr, size_t alloc_size, size_t leadsize, size_t size, bool *commit) { + void *ret = (void *)((byte_t *)addr + leadsize); assert(alloc_size >= leadsize + size); #ifdef _WIN32 @@ -188,7 +222,7 @@ os_pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size, os_pages_unmap(addr, leadsize); } if (trailsize != 0) { - os_pages_unmap((void *)((uintptr_t)ret + size), trailsize); + os_pages_unmap((void *)((byte_t *)ret + size), trailsize); } return ret; #endif @@ -208,13 +242,15 @@ os_pages_unmap(void *addr, size_t size) { char buf[BUFERROR_BUF]; buferror(get_errno(), buf, sizeof(buf)); - malloc_printf(": Error in " + malloc_printf( + ": Error in " #ifdef _WIN32 "VirtualFree" #else "munmap" #endif - "(): %s\n", buf); + "(): %s\n", + buf); if (opt_abort) { abort(); } @@ -321,13 +357,14 @@ os_pages_commit(void *addr, size_t size, bool commit) { assert(PAGE_CEILING(size) == size); #ifdef _WIN32 - return (commit ? (addr != VirtualAlloc(addr, size, MEM_COMMIT, - PAGE_READWRITE)) : (!VirtualFree(addr, size, MEM_DECOMMIT))); + return (commit + ? (addr != VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE)) + : (!VirtualFree(addr, size, MEM_DECOMMIT))); #else { - int prot = commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT; - void *result = mmap(addr, size, prot, mmap_flags | MAP_FIXED, - PAGES_FD_TAG, 0); + int prot = commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT; + void *result = mmap( + addr, size, prot, mmap_flags | MAP_FIXED, PAGES_FD_TAG, 0); if (result == MAP_FAILED) { return true; } @@ -366,8 +403,8 @@ pages_decommit(void *addr, size_t size) { void pages_mark_guards(void *head, void *tail) { assert(head != NULL || tail != NULL); - assert(head == NULL || tail == NULL || - (uintptr_t)head < (uintptr_t)tail); + assert( + head == NULL || tail == NULL || (uintptr_t)head < (uintptr_t)tail); #ifdef JEMALLOC_HAVE_MPROTECT if (head != NULL) { mprotect(head, PAGE, PROT_NONE); @@ -389,13 +426,12 @@ pages_mark_guards(void *head, void *tail) { void pages_unmark_guards(void *head, void *tail) { assert(head != NULL || tail != NULL); - assert(head == NULL || tail == NULL || - (uintptr_t)head < (uintptr_t)tail); + assert( + head == NULL || tail == NULL || (uintptr_t)head < (uintptr_t)tail); #ifdef JEMALLOC_HAVE_MPROTECT - bool head_and_tail = (head != NULL) && (tail != NULL); - size_t range = head_and_tail ? - (uintptr_t)tail - (uintptr_t)head + PAGE : - SIZE_T_MAX; + bool head_and_tail = (head != NULL) && (tail != NULL); + size_t range = head_and_tail ? (uintptr_t)tail - (uintptr_t)head + PAGE + : SIZE_T_MAX; /* * The amount of work that the kernel does in mprotect depends on the * range argument. SC_LARGE_MINCLASS is an arbitrary threshold chosen @@ -444,17 +480,18 @@ pages_purge_lazy(void *addr, size_t size) { return false; #elif defined(JEMALLOC_PURGE_MADVISE_FREE) return (madvise(addr, size, -# ifdef MADV_FREE - MADV_FREE -# else - JEMALLOC_MADV_FREE -# endif - ) != 0); -#elif defined(JEMALLOC_PURGE_MADVISE_DONTNEED) && \ - !defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS) +# ifdef MADV_FREE + MADV_FREE +# else + JEMALLOC_MADV_FREE +# endif + ) + != 0); +#elif defined(JEMALLOC_PURGE_MADVISE_DONTNEED) \ + && !defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS) return (madvise(addr, size, MADV_DONTNEED) != 0); -#elif defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED) && \ - !defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS) +#elif defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED) \ + && !defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS) return (posix_madvise(addr, size, POSIX_MADV_DONTNEED) != 0); #else not_reached(); @@ -470,14 +507,14 @@ pages_purge_forced(void *addr, size_t size) { return true; } -#if defined(JEMALLOC_PURGE_MADVISE_DONTNEED) && \ - defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS) - return (unlikely(madvise_dont_need_zeros_is_faulty) || - madvise(addr, size, MADV_DONTNEED) != 0); -#elif defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED) && \ - defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS) - return (unlikely(madvise_dont_need_zeros_is_faulty) || - posix_madvise(addr, size, POSIX_MADV_DONTNEED) != 0); +#if defined(JEMALLOC_PURGE_MADVISE_DONTNEED) \ + && defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS) + return (unlikely(madvise_dont_need_zeros_is_faulty) + || madvise(addr, size, MADV_DONTNEED) != 0); +#elif defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED) \ + && defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS) + return (unlikely(madvise_dont_need_zeros_is_faulty) + || posix_madvise(addr, size, POSIX_MADV_DONTNEED) != 0); #elif defined(JEMALLOC_MAPS_COALESCE) /* Try to overlay a new demand-zeroed mapping. */ return pages_commit(addr, size); @@ -538,6 +575,30 @@ pages_nohuge_unaligned(void *addr, size_t size) { return pages_nohuge_impl(addr, size, false); } +bool +pages_collapse(void *addr, size_t size) { + assert(PAGE_ADDR2BASE(addr) == addr); + assert(PAGE_CEILING(size) == size); + /* + * There is one more MADV_COLLAPSE precondition that is not easy to + * express with assert statement. In order to madvise(addr, size, + * MADV_COLLAPSE) call to be successful, at least one page in the range + * must currently be backed by physical memory. In particularly, this + * means we can't call pages_collapse on freshly mapped memory region. + * See madvise(2) man page for more details. + */ +#if defined(JEMALLOC_HAVE_MADVISE_COLLAPSE) \ + && (defined(MADV_COLLAPSE) || defined(JEMALLOC_MADV_COLLAPSE)) +# if defined(MADV_COLLAPSE) + return (madvise(addr, size, MADV_COLLAPSE) != 0); +# elif defined(JEMALLOC_MADV_COLLAPSE) + return (madvise(addr, size, JEMALLOC_MADV_COLLAPSE) != 0); +# endif +#else + return true; +#endif +} + bool pages_dontdump(void *addr, size_t size) { assert(PAGE_ADDR2BASE(addr) == addr); @@ -564,6 +625,83 @@ pages_dodump(void *addr, size_t size) { #endif } +#ifdef JEMALLOC_HAVE_PROCESS_MADVISE +# include +# include + +# ifndef PIDFD_SELF +# define PIDFD_SELF -10000 +# endif + +static atomic_b_t process_madvise_gate = ATOMIC_INIT(true); + +static bool +init_process_madvise(void) { + if (opt_process_madvise_max_batch == 0) { + return false; + } + + if (opt_process_madvise_max_batch > PROCESS_MADVISE_MAX_BATCH_LIMIT) { + opt_process_madvise_max_batch = PROCESS_MADVISE_MAX_BATCH_LIMIT; + } + + return false; +} + +# ifdef SYS_process_madvise +# define JE_SYS_PROCESS_MADVISE_NR SYS_process_madvise +# else +# define JE_SYS_PROCESS_MADVISE_NR \ + EXPERIMENTAL_SYS_PROCESS_MADVISE_NR +# endif + +static bool +pages_purge_process_madvise_impl( + void *vec, size_t vec_len, size_t total_bytes) { + if (!atomic_load_b(&process_madvise_gate, ATOMIC_RELAXED)) { + return true; + } + + /* + * TODO: remove this save/restore of errno after supporting errno + * preservation for free() call properly. + */ + int saved_errno = get_errno(); + size_t purged_bytes = (size_t)syscall(JE_SYS_PROCESS_MADVISE_NR, + PIDFD_SELF, (struct iovec *)vec, vec_len, MADV_DONTNEED, 0); + if (purged_bytes == (size_t)-1) { + if (errno == EPERM || errno == EINVAL || errno == ENOSYS + || errno == EBADF) { + /* Process madvise not supported the way we need it. */ + atomic_store_b( + &process_madvise_gate, false, ATOMIC_RELAXED); + } + set_errno(saved_errno); + } + + return purged_bytes != total_bytes; +} + +#else + +static bool +init_process_madvise(void) { + return false; +} + +static bool +pages_purge_process_madvise_impl( + void *vec, size_t vec_len, size_t total_bytes) { + not_reached(); + return true; +} + +#endif + +bool +pages_purge_process_madvise(void *vec, size_t vec_len, size_t total_bytes) { + return pages_purge_process_madvise_impl(vec, vec_len, total_bytes); +} static size_t os_page_detect(void) { @@ -580,7 +718,7 @@ os_page_detect(void) { #else long result = sysconf(_SC_PAGESIZE); if (result == -1) { - return LG_PAGE; + return PAGE; } return (size_t)result; #endif @@ -589,11 +727,11 @@ os_page_detect(void) { #ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT static bool os_overcommits_sysctl(void) { - int vm_overcommit; + int vm_overcommit; size_t sz; sz = sizeof(vm_overcommit); -#if defined(__FreeBSD__) && defined(VM_OVERCOMMIT) +# if defined(__FreeBSD__) && defined(VM_OVERCOMMIT) int mib[2]; mib[0] = CTL_VM; @@ -601,69 +739,38 @@ os_overcommits_sysctl(void) { if (sysctl(mib, 2, &vm_overcommit, &sz, NULL, 0) != 0) { return false; /* Error. */ } -#else +# else if (sysctlbyname("vm.overcommit", &vm_overcommit, &sz, NULL, 0) != 0) { return false; /* Error. */ } -#endif +# endif return ((vm_overcommit & 0x3) == 0); } #endif #ifdef JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY -/* - * Use syscall(2) rather than {open,read,close}(2) when possible to avoid - * reentry during bootstrapping if another library has interposed system call - * wrappers. - */ static bool os_overcommits_proc(void) { - int fd; + int fd; char buf[1]; -#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_open) - #if defined(O_CLOEXEC) - fd = (int)syscall(SYS_open, "/proc/sys/vm/overcommit_memory", O_RDONLY | - O_CLOEXEC); - #else - fd = (int)syscall(SYS_open, "/proc/sys/vm/overcommit_memory", O_RDONLY); - if (fd != -1) { - fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC); - } - #endif -#elif defined(JEMALLOC_USE_SYSCALL) && defined(SYS_openat) - #if defined(O_CLOEXEC) - fd = (int)syscall(SYS_openat, - AT_FDCWD, "/proc/sys/vm/overcommit_memory", O_RDONLY | O_CLOEXEC); - #else - fd = (int)syscall(SYS_openat, - AT_FDCWD, "/proc/sys/vm/overcommit_memory", O_RDONLY); - if (fd != -1) { - fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC); - } - #endif -#else - #if defined(O_CLOEXEC) - fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY | O_CLOEXEC); - #else - fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY); - if (fd != -1) { - fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC); - } - #endif -#endif +# if defined(O_CLOEXEC) + fd = malloc_open( + "/proc/sys/vm/overcommit_memory", O_RDONLY | O_CLOEXEC); +# else + fd = malloc_open("/proc/sys/vm/overcommit_memory", O_RDONLY); + if (fd != -1) { + fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC); + } +# endif if (fd == -1) { return false; /* Error. */ } ssize_t nread = malloc_read_fd(fd, &buf, sizeof(buf)); -#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_close) - syscall(SYS_close, fd); -#else - close(fd); -#endif + malloc_close(fd); if (nread < 1) { return false; /* Error. */ @@ -678,21 +785,31 @@ os_overcommits_proc(void) { } #endif +static bool +pages_should_skip_set_thp_state() { + if (opt_thp == thp_mode_do_nothing + || (opt_thp == thp_mode_always + && init_system_thp_mode == system_thp_mode_always) + || (opt_thp == thp_mode_never + && init_system_thp_mode == system_thp_mode_never)) { + return true; + } + return false; +} void -pages_set_thp_state (void *ptr, size_t size) { - if (opt_thp == thp_mode_default || opt_thp == init_system_thp_mode) { +pages_set_thp_state(void *ptr, size_t size) { + if (pages_should_skip_set_thp_state()) { return; } - assert(opt_thp != thp_mode_not_supported && - init_system_thp_mode != thp_mode_not_supported); + assert(opt_thp != thp_mode_not_supported + && init_system_thp_mode != system_thp_mode_not_supported); if (opt_thp == thp_mode_always - && init_system_thp_mode != thp_mode_never) { - assert(init_system_thp_mode == thp_mode_default); + && init_system_thp_mode == system_thp_mode_madvise) { pages_huge_unaligned(ptr, size); } else if (opt_thp == thp_mode_never) { - assert(init_system_thp_mode == thp_mode_default || - init_system_thp_mode == thp_mode_always); + assert(init_system_thp_mode == system_thp_mode_madvise + || init_system_thp_mode == system_thp_mode_always); pages_nohuge_unaligned(ptr, size); } } @@ -710,48 +827,47 @@ init_thp_state(void) { static const char sys_state_madvise[] = "always [madvise] never\n"; static const char sys_state_always[] = "[always] madvise never\n"; static const char sys_state_never[] = "always madvise [never]\n"; - char buf[sizeof(sys_state_madvise)]; + char buf[sizeof(sys_state_madvise)]; -#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_open) - int fd = (int)syscall(SYS_open, + int fd = malloc_open( "/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY); -#elif defined(JEMALLOC_USE_SYSCALL) && defined(SYS_openat) - int fd = (int)syscall(SYS_openat, - AT_FDCWD, "/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY); -#else - int fd = open("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY); -#endif if (fd == -1) { goto label_error; } ssize_t nread = malloc_read_fd(fd, &buf, sizeof(buf)); -#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_close) - syscall(SYS_close, fd); -#else - close(fd); -#endif - - if (nread < 0) { + malloc_close(fd); + if (nread < 0) { goto label_error; - } + } if (strncmp(buf, sys_state_madvise, (size_t)nread) == 0) { - init_system_thp_mode = thp_mode_default; + init_system_thp_mode = system_thp_mode_madvise; } else if (strncmp(buf, sys_state_always, (size_t)nread) == 0) { - init_system_thp_mode = thp_mode_always; + init_system_thp_mode = system_thp_mode_always; } else if (strncmp(buf, sys_state_never, (size_t)nread) == 0) { - init_system_thp_mode = thp_mode_never; + init_system_thp_mode = system_thp_mode_never; } else { goto label_error; } + if (opt_hpa_opts.hugify_style == hpa_hugify_style_auto) { + if (init_system_thp_mode == system_thp_mode_madvise) { + opt_hpa_opts.hugify_style = hpa_hugify_style_lazy; + } else { + opt_hpa_opts.hugify_style = hpa_hugify_style_none; + } + } return; #elif defined(JEMALLOC_HAVE_MEMCNTL) - init_system_thp_mode = thp_mode_default; + init_system_thp_mode = system_thp_mode_madvise; + if (opt_hpa_opts.hugify_style == hpa_hugify_style_auto) { + opt_hpa_opts.hugify_style = hpa_hugify_style_eager; + } return; #endif label_error: - opt_thp = init_system_thp_mode = thp_mode_not_supported; + opt_thp = thp_mode_not_supported; + init_system_thp_mode = system_thp_mode_not_supported; } bool @@ -767,10 +883,13 @@ pages_boot(void) { #ifdef JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS if (!opt_trust_madvise) { - madvise_dont_need_zeros_is_faulty = !madvise_MADV_DONTNEED_zeroes_pages(); + madvise_dont_need_zeros_is_faulty = + !madvise_MADV_DONTNEED_zeroes_pages(); if (madvise_dont_need_zeros_is_faulty) { - malloc_write(": MADV_DONTNEED does not work (memset will be used instead)\n"); - malloc_write(": (This is the expected behaviour if you are running under QEMU)\n"); + malloc_write( + ": MADV_DONTNEED does not work (memset will be used instead)\n"); + malloc_write( + ": (This is the expected behaviour if you are running under QEMU)\n"); } } else { /* In case opt_trust_madvise is disable, @@ -787,11 +906,11 @@ pages_boot(void) { os_overcommits = os_overcommits_sysctl(); #elif defined(JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY) os_overcommits = os_overcommits_proc(); -# ifdef MAP_NORESERVE +# ifdef MAP_NORESERVE if (os_overcommits) { mmap_flags |= MAP_NORESERVE; } -# endif +# endif #elif defined(__NetBSD__) os_overcommits = true; #else @@ -807,8 +926,9 @@ pages_boot(void) { #else /* Detect lazy purge runtime support. */ if (pages_can_purge_lazy) { - bool committed = false; - void *madv_free_page = os_pages_map(NULL, PAGE, PAGE, &committed); + bool committed = false; + void *madv_free_page = os_pages_map( + NULL, PAGE, PAGE, &committed); if (madv_free_page == NULL) { return true; } @@ -819,6 +939,12 @@ pages_boot(void) { os_pages_unmap(madv_free_page, PAGE); } #endif + if (init_process_madvise()) { + if (opt_abort) { + abort(); + } + return true; + } return false; } diff --git a/dep/jemalloc/src/pai.c b/dep/jemalloc/src/pai.c deleted file mode 100644 index 45c877292..000000000 --- a/dep/jemalloc/src/pai.c +++ /dev/null @@ -1,31 +0,0 @@ -#include "jemalloc/internal/jemalloc_preamble.h" -#include "jemalloc/internal/jemalloc_internal_includes.h" - -size_t -pai_alloc_batch_default(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs, - edata_list_active_t *results, bool *deferred_work_generated) { - for (size_t i = 0; i < nallocs; i++) { - bool deferred_by_alloc = false; - edata_t *edata = pai_alloc(tsdn, self, size, PAGE, - /* zero */ false, /* guarded */ false, - /* frequent_reuse */ false, &deferred_by_alloc); - *deferred_work_generated |= deferred_by_alloc; - if (edata == NULL) { - return i; - } - edata_list_active_append(results, edata); - } - return nallocs; -} - -void -pai_dalloc_batch_default(tsdn_t *tsdn, pai_t *self, - edata_list_active_t *list, bool *deferred_work_generated) { - edata_t *edata; - while ((edata = edata_list_active_first(list)) != NULL) { - bool deferred_by_dalloc = false; - edata_list_active_remove(list, edata); - pai_dalloc(tsdn, self, edata, &deferred_by_dalloc); - *deferred_work_generated |= deferred_by_dalloc; - } -} diff --git a/dep/jemalloc/src/peak_event.c b/dep/jemalloc/src/peak_event.c index 4093fbcc6..39f90b70b 100644 --- a/dep/jemalloc/src/peak_event.c +++ b/dep/jemalloc/src/peak_event.c @@ -3,43 +3,24 @@ #include "jemalloc/internal/peak_event.h" -#include "jemalloc/internal/activity_callback.h" #include "jemalloc/internal/peak.h" - -/* - * Update every 64K by default. We're not exposing this as a configuration - * option for now; we don't want to bind ourselves too tightly to any particular - * performance requirements for small values, or guarantee that we'll even be - * able to provide fine-grained accuracy. - */ -#define PEAK_EVENT_WAIT (64 * 1024) +#include "jemalloc/internal/thread_event_registry.h" /* Update the peak with current tsd state. */ void peak_event_update(tsd_t *tsd) { uint64_t alloc = tsd_thread_allocated_get(tsd); uint64_t dalloc = tsd_thread_deallocated_get(tsd); - peak_t *peak = tsd_peakp_get(tsd); + peak_t *peak = tsd_peakp_get(tsd); peak_update(peak, alloc, dalloc); } -static void -peak_event_activity_callback(tsd_t *tsd) { - activity_callback_thunk_t *thunk = tsd_activity_callback_thunkp_get( - tsd); - uint64_t alloc = tsd_thread_allocated_get(tsd); - uint64_t dalloc = tsd_thread_deallocated_get(tsd); - if (thunk->callback != NULL) { - thunk->callback(thunk->uctx, alloc, dalloc); - } -} - /* Set current state to zero. */ void peak_event_zero(tsd_t *tsd) { uint64_t alloc = tsd_thread_allocated_get(tsd); uint64_t dalloc = tsd_thread_deallocated_get(tsd); - peak_t *peak = tsd_peakp_get(tsd); + peak_t *peak = tsd_peakp_get(tsd); peak_set_zero(peak, alloc, dalloc); } @@ -49,34 +30,30 @@ peak_event_max(tsd_t *tsd) { return peak_max(peak); } -uint64_t -peak_alloc_new_event_wait(tsd_t *tsd) { +static uint64_t +peak_event_new_event_wait(tsd_t *tsd) { return PEAK_EVENT_WAIT; } -uint64_t -peak_alloc_postponed_event_wait(tsd_t *tsd) { +static uint64_t +peak_event_postponed_event_wait(tsd_t *tsd) { return TE_MIN_START_WAIT; } -void -peak_alloc_event_handler(tsd_t *tsd, uint64_t elapsed) { +static void +peak_event_handler(tsd_t *tsd) { peak_event_update(tsd); - peak_event_activity_callback(tsd); } -uint64_t -peak_dalloc_new_event_wait(tsd_t *tsd) { - return PEAK_EVENT_WAIT; +static te_enabled_t +peak_event_enabled(void) { + return config_stats ? te_enabled_yes : te_enabled_no; } -uint64_t -peak_dalloc_postponed_event_wait(tsd_t *tsd) { - return TE_MIN_START_WAIT; -} - -void -peak_dalloc_event_handler(tsd_t *tsd, uint64_t elapsed) { - peak_event_update(tsd); - peak_event_activity_callback(tsd); -} +/* Handles alloc and dalloc */ +te_base_cb_t peak_te_handler = { + .enabled = &peak_event_enabled, + .new_event_wait = &peak_event_new_event_wait, + .postponed_event_wait = &peak_event_postponed_event_wait, + .event_handler = &peak_event_handler, +}; diff --git a/dep/jemalloc/src/prof.c b/dep/jemalloc/src/prof.c index 7a6d5d569..a833fed50 100644 --- a/dep/jemalloc/src/prof.c +++ b/dep/jemalloc/src/prof.c @@ -12,6 +12,7 @@ #include "jemalloc/internal/prof_sys.h" #include "jemalloc/internal/prof_hook.h" #include "jemalloc/internal/thread_event.h" +#include "jemalloc/internal/thread_event_registry.h" /* * This file implements the profiling "APIs" needed by other parts of jemalloc, @@ -23,19 +24,21 @@ /* Data. */ -bool opt_prof = false; -bool opt_prof_active = true; -bool opt_prof_thread_active_init = true; -size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT; -ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT; -bool opt_prof_gdump = false; -bool opt_prof_final = false; -bool opt_prof_leak = false; -bool opt_prof_leak_error = false; -bool opt_prof_accum = false; -char opt_prof_prefix[PROF_DUMP_FILENAME_LEN]; -bool opt_prof_sys_thread_name = false; -bool opt_prof_unbias = true; +bool opt_prof = false; +bool opt_prof_active = true; +bool opt_prof_thread_active_init = true; +unsigned opt_prof_bt_max = PROF_BT_MAX_DEFAULT; +size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT; +ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT; +bool opt_prof_gdump = false; +bool opt_prof_final = false; +bool opt_prof_leak = false; +bool opt_prof_leak_error = false; +bool opt_prof_accum = false; +bool opt_prof_pid_namespace = false; +char opt_prof_prefix[PROF_DUMP_FILENAME_LEN]; +bool opt_prof_sys_thread_name = false; +bool opt_prof_unbias = true; /* Accessed via prof_sample_event_handler(). */ static counter_accum_t prof_idump_accumulated; @@ -44,38 +47,44 @@ static counter_accum_t prof_idump_accumulated; * Initialized as opt_prof_active, and accessed via * prof_active_[gs]et{_unlocked,}(). */ -bool prof_active_state; +bool prof_active_state; static malloc_mutex_t prof_active_mtx; /* * Initialized as opt_prof_thread_active_init, and accessed via * prof_thread_active_init_[gs]et(). */ -static bool prof_thread_active_init; +static bool prof_thread_active_init; static malloc_mutex_t prof_thread_active_init_mtx; /* * Initialized as opt_prof_gdump, and accessed via * prof_gdump_[gs]et{_unlocked,}(). */ -bool prof_gdump_val; +bool prof_gdump_val; static malloc_mutex_t prof_gdump_mtx; uint64_t prof_interval = 0; size_t lg_prof_sample; -static uint64_t next_thr_uid; +static uint64_t next_thr_uid; static malloc_mutex_t next_thr_uid_mtx; /* Do not dump any profiles until bootstrapping is complete. */ bool prof_booted = false; /* Logically a prof_backtrace_hook_t. */ -atomic_p_t prof_backtrace_hook; +static atomic_p_t prof_backtrace_hook; /* Logically a prof_dump_hook_t. */ -atomic_p_t prof_dump_hook; +static atomic_p_t prof_dump_hook; + +/* Logically a prof_sample_hook_t. */ +static atomic_p_t prof_sample_hook; + +/* Logically a prof_sample_free_hook_t. */ +static atomic_p_t prof_sample_free_hook; /******************************************************************************/ @@ -84,11 +93,19 @@ prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx) { cassert(config_prof); if (tsd_reentrancy_level_get(tsd) > 0) { - assert((uintptr_t)tctx == (uintptr_t)1U); + assert(tctx == PROF_TCTX_SENTINEL); return; } - if ((uintptr_t)tctx > (uintptr_t)1U) { + if (prof_tctx_is_valid(tctx)) { + /* + * This `assert` really shouldn't be necessary. It's here + * because there's a bug in the clang static analyzer; it + * somehow does not realize that by `prof_tctx_is_valid(tctx)` + * being true that we've already ensured that `tctx` is not + * `NULL`. + */ + assert(tctx != NULL); malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock); tctx->prepared = false; prof_tctx_try_destroy(tsd, tctx); @@ -96,16 +113,16 @@ prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx) { } void -prof_malloc_sample_object(tsd_t *tsd, const void *ptr, size_t size, - size_t usize, prof_tctx_t *tctx) { +prof_malloc_sample_object( + tsd_t *tsd, const void *ptr, size_t size, size_t usize, prof_tctx_t *tctx) { cassert(config_prof); if (opt_prof_sys_thread_name) { prof_sys_thread_name_fetch(tsd); } - edata_t *edata = emap_edata_lookup(tsd_tsdn(tsd), &arena_emap_global, - ptr); + edata_t *edata = emap_edata_lookup( + tsd_tsdn(tsd), &arena_emap_global, ptr); prof_info_set(tsd, edata, tctx, size); szind_t szind = sz_size2index(usize); @@ -144,17 +161,37 @@ prof_malloc_sample_object(tsd_t *tsd, const void *ptr, size_t size, if (opt_prof_stats) { prof_stats_inc(tsd, szind, size); } + + /* Sample hook. */ + prof_sample_hook_t prof_sample_hook = prof_sample_hook_get(); + if (prof_sample_hook != NULL) { + prof_bt_t *bt = &tctx->gctx->bt; + pre_reentrancy(tsd, NULL); + prof_sample_hook(ptr, size, bt->vec, bt->len, usize); + post_reentrancy(tsd); + } } void -prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_info_t *prof_info) { +prof_free_sampled_object( + tsd_t *tsd, const void *ptr, size_t usize, prof_info_t *prof_info) { cassert(config_prof); assert(prof_info != NULL); prof_tctx_t *tctx = prof_info->alloc_tctx; - assert((uintptr_t)tctx > (uintptr_t)1U); + assert(prof_tctx_is_valid(tctx)); szind_t szind = sz_size2index(usize); + + /* Unsample hook. */ + prof_sample_free_hook_t prof_sample_free_hook = + prof_sample_free_hook_get(); + if (prof_sample_free_hook != NULL) { + pre_reentrancy(tsd, NULL); + prof_sample_free_hook(ptr, usize); + post_reentrancy(tsd); + } + malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock); assert(tctx->cnts.curobjs > 0); @@ -242,9 +279,12 @@ prof_sample_new_event_wait(tsd_t *tsd) { * otherwise bytes_until_sample would be 0 if u is exactly 1.0. */ uint64_t r = prng_lg_range_u64(tsd_prng_statep_get(tsd), 53); - double u = (r == 0U) ? 1.0 : (double)r * (1.0/9007199254740992.0L); - return (uint64_t)(log(u) / - log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample)))) + double u = (r == 0U) + ? 1.0 + : (double)((long double)r * (1.0L / 9007199254740992.0L)); + return (uint64_t)(log(u) + / log( + 1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample)))) + (uint64_t)1U; #else not_reached(); @@ -252,30 +292,51 @@ prof_sample_new_event_wait(tsd_t *tsd) { #endif } +void +prof_sample_event_handler(tsd_t *tsd) { + cassert(config_prof); + if (prof_interval == 0 || !prof_active_get_unlocked()) { + return; + } + uint64_t last_event = thread_allocated_last_event_get(tsd); + uint64_t last_sample_event = tsd_prof_sample_last_event_get(tsd); + tsd_prof_sample_last_event_set(tsd, last_event); + uint64_t elapsed = last_event - last_sample_event; + assert(elapsed > 0 && elapsed != TE_INVALID_ELAPSED); + if (counter_accum(tsd_tsdn(tsd), &prof_idump_accumulated, elapsed)) { + prof_idump(tsd_tsdn(tsd)); + } +} + uint64_t -prof_sample_postponed_event_wait(tsd_t *tsd) { - /* +tsd_prof_sample_event_wait_get(tsd_t *tsd) { +#ifdef JEMALLOC_PROF + return tsd_te_datap_get_unsafe(tsd)->alloc_wait[te_alloc_prof_sample]; +#else + not_reached(); + return TE_MAX_START_WAIT; +#endif +} + +static te_enabled_t +prof_sample_enabled(void) { + return config_prof && opt_prof ? te_enabled_yes : te_enabled_no; +} + +te_base_cb_t prof_sample_te_handler = { + .enabled = &prof_sample_enabled, + .new_event_wait = &prof_sample_new_event_wait, + /* * The postponed wait time for prof sample event is computed as if we * want a new wait time (i.e. as if the event were triggered). If we * instead postpone to the immediate next allocation, like how we're * handling the other events, then we can have sampling bias, if e.g. * the allocation immediately following a reentrancy always comes from * the same stack trace. - */ - return prof_sample_new_event_wait(tsd); -} - -void -prof_sample_event_handler(tsd_t *tsd, uint64_t elapsed) { - cassert(config_prof); - assert(elapsed > 0 && elapsed != TE_INVALID_ELAPSED); - if (prof_interval == 0 || !prof_active_get_unlocked()) { - return; - } - if (counter_accum(tsd_tsdn(tsd), &prof_idump_accumulated, elapsed)) { - prof_idump(tsd_tsdn(tsd)); - } -} + */ + .postponed_event_wait = &prof_sample_new_event_wait, + .event_handler = &prof_sample_event_handler, +}; static void prof_fdump(void) { @@ -302,7 +363,7 @@ prof_idump_accum_init(void) { void prof_idump(tsdn_t *tsdn) { - tsd_t *tsd; + tsd_t *tsd; prof_tdata_t *tdata; cassert(config_prof); @@ -341,7 +402,7 @@ prof_mdump(tsd_t *tsd, const char *filename) { void prof_gdump(tsdn_t *tsdn) { - tsd_t *tsd; + tsd_t *tsd; prof_tdata_t *tdata; cassert(config_prof); @@ -388,13 +449,16 @@ prof_tdata_t * prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata) { uint64_t thr_uid = tdata->thr_uid; uint64_t thr_discrim = tdata->thr_discrim + 1; - char *thread_name = (tdata->thread_name != NULL) ? - prof_thread_name_alloc(tsd, tdata->thread_name) : NULL; - bool active = tdata->active; + bool active = tdata->active; + /* Keep a local copy of the thread name, before detaching. */ + prof_thread_name_assert(tdata); + char thread_name[PROF_THREAD_NAME_MAX_LEN]; + strncpy(thread_name, tdata->thread_name, PROF_THREAD_NAME_MAX_LEN); prof_tdata_detach(tsd, tdata); - return prof_tdata_init_impl(tsd, thr_uid, thr_discrim, thread_name, - active); + + return prof_tdata_init_impl( + tsd, thr_uid, thr_discrim, thread_name, active); } void @@ -437,15 +501,15 @@ prof_active_set(tsdn_t *tsdn, bool active) { const char * prof_thread_name_get(tsd_t *tsd) { + static const char *prof_thread_name_dummy = ""; + assert(tsd_reentrancy_level_get(tsd) == 0); - - prof_tdata_t *tdata; - - tdata = prof_tdata_get(tsd, true); + prof_tdata_t *tdata = prof_tdata_get(tsd, true); if (tdata == NULL) { - return ""; + return prof_thread_name_dummy; } - return (tdata->thread_name != NULL ? tdata->thread_name : ""); + + return tdata->thread_name; } int @@ -532,9 +596,9 @@ prof_backtrace_hook_set(prof_backtrace_hook_t hook) { } prof_backtrace_hook_t -prof_backtrace_hook_get() { - return (prof_backtrace_hook_t)atomic_load_p(&prof_backtrace_hook, - ATOMIC_ACQUIRE); +prof_backtrace_hook_get(void) { + return (prof_backtrace_hook_t)atomic_load_p( + &prof_backtrace_hook, ATOMIC_ACQUIRE); } void @@ -543,17 +607,38 @@ prof_dump_hook_set(prof_dump_hook_t hook) { } prof_dump_hook_t -prof_dump_hook_get() { - return (prof_dump_hook_t)atomic_load_p(&prof_dump_hook, - ATOMIC_ACQUIRE); +prof_dump_hook_get(void) { + return (prof_dump_hook_t)atomic_load_p(&prof_dump_hook, ATOMIC_ACQUIRE); +} + +void +prof_sample_hook_set(prof_sample_hook_t hook) { + atomic_store_p(&prof_sample_hook, hook, ATOMIC_RELEASE); +} + +prof_sample_hook_t +prof_sample_hook_get(void) { + return (prof_sample_hook_t)atomic_load_p( + &prof_sample_hook, ATOMIC_ACQUIRE); +} + +void +prof_sample_free_hook_set(prof_sample_free_hook_t hook) { + atomic_store_p(&prof_sample_free_hook, hook, ATOMIC_RELEASE); +} + +prof_sample_free_hook_t +prof_sample_free_hook_get(void) { + return (prof_sample_free_hook_t)atomic_load_p( + &prof_sample_free_hook, ATOMIC_ACQUIRE); } void prof_boot0(void) { cassert(config_prof); - memcpy(opt_prof_prefix, PROF_PREFIX_DEFAULT, - sizeof(PROF_PREFIX_DEFAULT)); + memcpy( + opt_prof_prefix, PROF_PREFIX_DEFAULT, sizeof(PROF_PREFIX_DEFAULT)); } void @@ -577,8 +662,8 @@ prof_boot1(void) { opt_prof_gdump = false; } else if (opt_prof) { if (opt_lg_prof_interval >= 0) { - prof_interval = (((uint64_t)1U) << - opt_lg_prof_interval); + prof_interval = (((uint64_t)1U) + << opt_lg_prof_interval); } } } @@ -592,41 +677,40 @@ prof_boot2(tsd_t *tsd, base_t *base) { * stats when opt_prof is false. */ if (malloc_mutex_init(&prof_active_mtx, "prof_active", - WITNESS_RANK_PROF_ACTIVE, malloc_mutex_rank_exclusive)) { + WITNESS_RANK_PROF_ACTIVE, malloc_mutex_rank_exclusive)) { return true; } if (malloc_mutex_init(&prof_gdump_mtx, "prof_gdump", - WITNESS_RANK_PROF_GDUMP, malloc_mutex_rank_exclusive)) { + WITNESS_RANK_PROF_GDUMP, malloc_mutex_rank_exclusive)) { return true; } if (malloc_mutex_init(&prof_thread_active_init_mtx, - "prof_thread_active_init", WITNESS_RANK_PROF_THREAD_ACTIVE_INIT, - malloc_mutex_rank_exclusive)) { + "prof_thread_active_init", WITNESS_RANK_PROF_THREAD_ACTIVE_INIT, + malloc_mutex_rank_exclusive)) { return true; } if (malloc_mutex_init(&bt2gctx_mtx, "prof_bt2gctx", - WITNESS_RANK_PROF_BT2GCTX, malloc_mutex_rank_exclusive)) { + WITNESS_RANK_PROF_BT2GCTX, malloc_mutex_rank_exclusive)) { return true; } if (malloc_mutex_init(&tdatas_mtx, "prof_tdatas", - WITNESS_RANK_PROF_TDATAS, malloc_mutex_rank_exclusive)) { + WITNESS_RANK_PROF_TDATAS, malloc_mutex_rank_exclusive)) { return true; } if (malloc_mutex_init(&next_thr_uid_mtx, "prof_next_thr_uid", - WITNESS_RANK_PROF_NEXT_THR_UID, malloc_mutex_rank_exclusive)) { + WITNESS_RANK_PROF_NEXT_THR_UID, malloc_mutex_rank_exclusive)) { return true; } if (malloc_mutex_init(&prof_stats_mtx, "prof_stats", - WITNESS_RANK_PROF_STATS, malloc_mutex_rank_exclusive)) { + WITNESS_RANK_PROF_STATS, malloc_mutex_rank_exclusive)) { return true; } - if (malloc_mutex_init(&prof_dump_filename_mtx, - "prof_dump_filename", WITNESS_RANK_PROF_DUMP_FILENAME, - malloc_mutex_rank_exclusive)) { + if (malloc_mutex_init(&prof_dump_filename_mtx, "prof_dump_filename", + WITNESS_RANK_PROF_DUMP_FILENAME, malloc_mutex_rank_exclusive)) { return true; } if (malloc_mutex_init(&prof_dump_mtx, "prof_dump", - WITNESS_RANK_PROF_DUMP, malloc_mutex_rank_exclusive)) { + WITNESS_RANK_PROF_DUMP, malloc_mutex_rank_exclusive)) { return true; } @@ -646,8 +730,8 @@ prof_boot2(tsd_t *tsd, base_t *base) { return true; } - if (opt_prof_final && opt_prof_prefix[0] != '\0' && - atexit(prof_fdump) != 0) { + if (opt_prof_final && opt_prof_prefix[0] != '\0' + && atexit(prof_fdump) != 0) { malloc_write(": Error in atexit()\n"); if (opt_abort) { abort(); @@ -671,8 +755,8 @@ prof_boot2(tsd_t *tsd, base_t *base) { } for (unsigned i = 0; i < PROF_NCTX_LOCKS; i++) { if (malloc_mutex_init(&gctx_locks[i], "prof_gctx", - WITNESS_RANK_PROF_GCTX, - malloc_mutex_rank_exclusive)) { + WITNESS_RANK_PROF_GCTX, + malloc_mutex_rank_exclusive)) { return true; } } @@ -684,8 +768,8 @@ prof_boot2(tsd_t *tsd, base_t *base) { } for (unsigned i = 0; i < PROF_NTDATA_LOCKS; i++) { if (malloc_mutex_init(&tdata_locks[i], "prof_tdata", - WITNESS_RANK_PROF_TDATA, - malloc_mutex_rank_exclusive)) { + WITNESS_RANK_PROF_TDATA, + malloc_mutex_rank_exclusive)) { return true; } } @@ -736,8 +820,8 @@ prof_postfork_parent(tsdn_t *tsdn) { if (config_prof && opt_prof) { unsigned i; - malloc_mutex_postfork_parent(tsdn, - &prof_thread_active_init_mtx); + malloc_mutex_postfork_parent( + tsdn, &prof_thread_active_init_mtx); malloc_mutex_postfork_parent(tsdn, &next_thr_uid_mtx); malloc_mutex_postfork_parent(tsdn, &prof_stats_mtx); malloc_mutex_postfork_parent(tsdn, &prof_recent_alloc_mtx); diff --git a/dep/jemalloc/src/prof_data.c b/dep/jemalloc/src/prof_data.c index bfa55be1c..7aa047ac0 100644 --- a/dep/jemalloc/src/prof_data.c +++ b/dep/jemalloc/src/prof_data.c @@ -36,7 +36,7 @@ malloc_mutex_t prof_dump_mtx; * and destroying mutexes causes complications for systems that allocate when * creating/destroying mutexes. */ -malloc_mutex_t *gctx_locks; +malloc_mutex_t *gctx_locks; static atomic_u_t cum_gctxs; /* Atomic counter. */ /* @@ -69,43 +69,45 @@ static int prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b) { uint64_t a_thr_uid = a->thr_uid; uint64_t b_thr_uid = b->thr_uid; - int ret = (a_thr_uid > b_thr_uid) - (a_thr_uid < b_thr_uid); + int ret = (a_thr_uid > b_thr_uid) - (a_thr_uid < b_thr_uid); if (ret == 0) { uint64_t a_thr_discrim = a->thr_discrim; uint64_t b_thr_discrim = b->thr_discrim; - ret = (a_thr_discrim > b_thr_discrim) - (a_thr_discrim < - b_thr_discrim); + ret = (a_thr_discrim > b_thr_discrim) + - (a_thr_discrim < b_thr_discrim); if (ret == 0) { uint64_t a_tctx_uid = a->tctx_uid; uint64_t b_tctx_uid = b->tctx_uid; - ret = (a_tctx_uid > b_tctx_uid) - (a_tctx_uid < - b_tctx_uid); + ret = (a_tctx_uid > b_tctx_uid) + - (a_tctx_uid < b_tctx_uid); } } return ret; } -rb_gen(static UNUSED, tctx_tree_, prof_tctx_tree_t, prof_tctx_t, - tctx_link, prof_tctx_comp) +/* NOLINTBEGIN(performance-no-int-to-ptr) */ +rb_gen(static UNUSED, tctx_tree_, prof_tctx_tree_t, prof_tctx_t, tctx_link, + prof_tctx_comp) + /* NOLINTEND(performance-no-int-to-ptr) */ -static int -prof_gctx_comp(const prof_gctx_t *a, const prof_gctx_t *b) { + static int prof_gctx_comp(const prof_gctx_t *a, const prof_gctx_t *b) { unsigned a_len = a->bt.len; unsigned b_len = b->bt.len; unsigned comp_len = (a_len < b_len) ? a_len : b_len; - int ret = memcmp(a->bt.vec, b->bt.vec, comp_len * sizeof(void *)); + int ret = memcmp(a->bt.vec, b->bt.vec, comp_len * sizeof(void *)); if (ret == 0) { ret = (a_len > b_len) - (a_len < b_len); } return ret; } +/* NOLINTBEGIN(performance-no-int-to-ptr) */ rb_gen(static UNUSED, gctx_tree_, prof_gctx_tree_t, prof_gctx_t, dump_link, prof_gctx_comp) + /* NOLINTEND(performance-no-int-to-ptr) */ -static int -prof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b) { - int ret; + static int prof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b) { + int ret; uint64_t a_uid = a->thr_uid; uint64_t b_uid = b->thr_uid; @@ -119,13 +121,14 @@ prof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b) { return ret; } +/* NOLINTBEGIN(performance-no-int-to-ptr) */ rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link, prof_tdata_comp) + /* NOLINTEND(performance-no-int-to-ptr) */ -/******************************************************************************/ + /******************************************************************************/ -static malloc_mutex_t * -prof_gctx_mutex_choose(void) { + static malloc_mutex_t *prof_gctx_mutex_choose(void) { unsigned ngctxs = atomic_fetch_add_u(&cum_gctxs, 1, ATOMIC_RELAXED); return &gctx_locks[(ngctxs - 1) % PROF_NCTX_LOCKS]; @@ -139,8 +142,8 @@ prof_tdata_mutex_choose(uint64_t thr_uid) { bool prof_data_init(tsd_t *tsd) { tdata_tree_new(&tdatas); - return ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS, - prof_bt_hash, prof_bt_keycomp); + return ckh_new( + tsd, &bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash, prof_bt_keycomp); } static void @@ -189,8 +192,8 @@ prof_gctx_create(tsdn_t *tsdn, prof_bt_t *bt) { */ size_t size = offsetof(prof_gctx_t, vec) + (bt->len * sizeof(void *)); prof_gctx_t *gctx = (prof_gctx_t *)iallocztm(tsdn, size, - sz_size2index(size), false, NULL, true, arena_get(TSDN_NULL, 0, true), - true); + sz_size2index(size), false, NULL, true, + arena_get(TSDN_NULL, 0, true), true); if (gctx == NULL) { return NULL; } @@ -209,8 +212,7 @@ prof_gctx_create(tsdn_t *tsdn, prof_bt_t *bt) { } static void -prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, - prof_gctx_t *gctx) { +prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx) { cassert(config_prof); /* @@ -261,12 +263,12 @@ static bool prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata, void **p_btkey, prof_gctx_t **p_gctx, bool *p_new_gctx) { union { - prof_gctx_t *p; - void *v; + prof_gctx_t *p; + void *v; } gctx, tgctx; union { - prof_bt_t *p; - void *v; + prof_bt_t *p; + void *v; } btkey; bool new_gctx; @@ -310,8 +312,8 @@ prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata, if (tgctx.v != NULL) { /* Lost race to insert. */ - idalloctm(tsd_tsdn(tsd), tgctx.v, NULL, NULL, true, - true); + idalloctm( + tsd_tsdn(tsd), tgctx.v, NULL, NULL, true, true); } } prof_leave(tsd, tdata); @@ -325,11 +327,11 @@ prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata, prof_tctx_t * prof_lookup(tsd_t *tsd, prof_bt_t *bt) { union { - prof_tctx_t *p; - void *v; + prof_tctx_t *p; + void *v; } ret; prof_tdata_t *tdata; - bool not_found; + bool not_found; cassert(config_prof); @@ -343,16 +345,16 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) { } malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock); if (not_found) { - void *btkey; + void *btkey; prof_gctx_t *gctx; - bool new_gctx, error; + bool new_gctx, error; /* * This thread's cache lacks bt. Look for it in the global * cache. */ - if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx, - &new_gctx)) { + if (prof_lookup_global( + tsd, bt, tdata, &btkey, &gctx, &new_gctx)) { return NULL; } @@ -397,8 +399,8 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) { /* Used in unit tests. */ static prof_tdata_t * -prof_tdata_count_iter(prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata, - void *arg) { +prof_tdata_count_iter( + prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata, void *arg) { size_t *tdata_count = (size_t *)arg; (*tdata_count)++; @@ -409,13 +411,13 @@ prof_tdata_count_iter(prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata, /* Used in unit tests. */ size_t prof_tdata_count(void) { - size_t tdata_count = 0; + size_t tdata_count = 0; tsdn_t *tsdn; tsdn = tsdn_fetch(); malloc_mutex_lock(tsdn, &tdatas_mtx); - tdata_tree_iter(&tdatas, NULL, prof_tdata_count_iter, - (void *)&tdata_count); + tdata_tree_iter( + &tdatas, NULL, prof_tdata_count_iter, (void *)&tdata_count); malloc_mutex_unlock(tsdn, &tdatas_mtx); return tdata_count; @@ -424,8 +426,8 @@ prof_tdata_count(void) { /* Used in unit tests. */ size_t prof_bt_count(void) { - size_t bt_count; - tsd_t *tsd; + size_t bt_count; + tsd_t *tsd; prof_tdata_t *tdata; tsd = tsd_fetch(); @@ -441,75 +443,40 @@ prof_bt_count(void) { return bt_count; } -char * -prof_thread_name_alloc(tsd_t *tsd, const char *thread_name) { - char *ret; - size_t size; - - if (thread_name == NULL) { - return NULL; - } - - size = strlen(thread_name) + 1; - if (size == 1) { - return ""; - } - - ret = iallocztm(tsd_tsdn(tsd), size, sz_size2index(size), false, NULL, - true, arena_get(TSDN_NULL, 0, true), true); - if (ret == NULL) { - return NULL; - } - memcpy(ret, thread_name, size); - return ret; +static void +prof_thread_name_write_tdata(prof_tdata_t *tdata, const char *thread_name) { + strncpy(tdata->thread_name, thread_name, PROF_THREAD_NAME_MAX_LEN); + tdata->thread_name[PROF_THREAD_NAME_MAX_LEN - 1] = '\0'; } int prof_thread_name_set_impl(tsd_t *tsd, const char *thread_name) { assert(tsd_reentrancy_level_get(tsd) == 0); + assert(thread_name != NULL); - prof_tdata_t *tdata; - unsigned i; - char *s; - - tdata = prof_tdata_get(tsd, true); - if (tdata == NULL) { - return EAGAIN; - } - - /* Validate input. */ - if (thread_name == NULL) { - return EFAULT; - } - for (i = 0; thread_name[i] != '\0'; i++) { + for (unsigned i = 0; thread_name[i] != '\0'; i++) { char c = thread_name[i]; if (!isgraph(c) && !isblank(c)) { - return EFAULT; + return EINVAL; } } - s = prof_thread_name_alloc(tsd, thread_name); - if (s == NULL) { - return EAGAIN; + prof_tdata_t *tdata = prof_tdata_get(tsd, true); + if (tdata == NULL) { + return ENOMEM; } - if (tdata->thread_name != NULL) { - idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, NULL, true, - true); - tdata->thread_name = NULL; - } - if (strlen(s) > 0) { - tdata->thread_name = s; - } + prof_thread_name_write_tdata(tdata, thread_name); + return 0; } JEMALLOC_FORMAT_PRINTF(3, 4) static void -prof_dump_printf(write_cb_t *prof_dump_write, void *cbopaque, - const char *format, ...) { +prof_dump_printf( + write_cb_t *prof_dump_write, void *cbopaque, const char *format, ...) { va_list ap; - char buf[PROF_PRINTF_BUFSIZE]; + char buf[PROF_PRINTF_BUFSIZE]; va_start(ap, format); malloc_vsnprintf(buf, sizeof(buf), format, ap); @@ -538,11 +505,18 @@ prof_double_uint64_cast(double d) { } #endif -void prof_unbias_map_init() { +void +prof_unbias_map_init(void) { /* See the comment in prof_sample_new_event_wait */ #ifdef JEMALLOC_PROF for (szind_t i = 0; i < SC_NSIZES; i++) { - double sz = (double)sz_index2size(i); + /* + * With large size classes disabled, the unbiased calculation + * here is not as accurate as it was because usize now changes + * in a finer grain while the unbiased_sz is still calculated + * using the old way. + */ + double sz = (double)sz_index2size_unsafe(i); double rate = (double)(ZU(1) << lg_prof_sample); double div_val = 1.0 - exp(-sz / rate); double unbiased_sz = sz / div_val; @@ -644,8 +618,8 @@ prof_do_unbias(uint64_t c_out_shifted_i, uint64_t s_out_i, uint64_t *r_c_in, } static void -prof_dump_print_cnts(write_cb_t *prof_dump_write, void *cbopaque, - const prof_cnt_t *cnts) { +prof_dump_print_cnts( + write_cb_t *prof_dump_write, void *cbopaque, const prof_cnt_t *cnts) { uint64_t curobjs; uint64_t curbytes; uint64_t accumobjs; @@ -662,8 +636,8 @@ prof_dump_print_cnts(write_cb_t *prof_dump_write, void *cbopaque, accumbytes = cnts->accumbytes; } prof_dump_printf(prof_dump_write, cbopaque, - "%"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]", - curobjs, curbytes, accumobjs, accumbytes); + "%" FMTu64 ": %" FMTu64 " [%" FMTu64 ": %" FMTu64 "]", curobjs, + curbytes, accumobjs, accumbytes); } static void @@ -683,11 +657,11 @@ prof_tctx_merge_tdata(tsdn_t *tsdn, prof_tctx_t *tctx, prof_tdata_t *tdata) { memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t)); tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs; - tdata->cnt_summed.curobjs_shifted_unbiased - += tctx->dump_cnts.curobjs_shifted_unbiased; + tdata->cnt_summed.curobjs_shifted_unbiased += + tctx->dump_cnts.curobjs_shifted_unbiased; tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes; - tdata->cnt_summed.curbytes_unbiased - += tctx->dump_cnts.curbytes_unbiased; + tdata->cnt_summed.curbytes_unbiased += + tctx->dump_cnts.curbytes_unbiased; if (opt_prof_accum) { tdata->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs; @@ -710,17 +684,17 @@ prof_tctx_merge_gctx(tsdn_t *tsdn, prof_tctx_t *tctx, prof_gctx_t *gctx) { malloc_mutex_assert_owner(tsdn, gctx->lock); gctx->cnt_summed.curobjs += tctx->dump_cnts.curobjs; - gctx->cnt_summed.curobjs_shifted_unbiased - += tctx->dump_cnts.curobjs_shifted_unbiased; + gctx->cnt_summed.curobjs_shifted_unbiased += + tctx->dump_cnts.curobjs_shifted_unbiased; gctx->cnt_summed.curbytes += tctx->dump_cnts.curbytes; gctx->cnt_summed.curbytes_unbiased += tctx->dump_cnts.curbytes_unbiased; if (opt_prof_accum) { gctx->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs; - gctx->cnt_summed.accumobjs_shifted_unbiased - += tctx->dump_cnts.accumobjs_shifted_unbiased; + gctx->cnt_summed.accumobjs_shifted_unbiased += + tctx->dump_cnts.accumobjs_shifted_unbiased; gctx->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes; - gctx->cnt_summed.accumbytes_unbiased - += tctx->dump_cnts.accumbytes_unbiased; + gctx->cnt_summed.accumbytes_unbiased += + tctx->dump_cnts.accumbytes_unbiased; } } @@ -738,6 +712,7 @@ prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) { case prof_tctx_state_purgatory: prof_tctx_merge_gctx(tsdn, tctx, tctx->gctx); break; + case prof_tctx_state_initializing: default: not_reached(); } @@ -747,9 +722,9 @@ prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) { typedef struct prof_dump_iter_arg_s prof_dump_iter_arg_t; struct prof_dump_iter_arg_s { - tsdn_t *tsdn; + tsdn_t *tsdn; write_cb_t *prof_dump_write; - void *cbopaque; + void *cbopaque; }; static prof_tctx_t * @@ -765,9 +740,9 @@ prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *opaque) { case prof_tctx_state_dumping: case prof_tctx_state_purgatory: prof_dump_printf(arg->prof_dump_write, arg->cbopaque, - " t%"FMTu64": ", tctx->thr_uid); - prof_dump_print_cnts(arg->prof_dump_write, arg->cbopaque, - &tctx->dump_cnts); + " t%" FMTu64 ": ", tctx->thr_uid); + prof_dump_print_cnts( + arg->prof_dump_write, arg->cbopaque, &tctx->dump_cnts); arg->prof_dump_write(arg->cbopaque, "\n"); break; default: @@ -778,7 +753,7 @@ prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *opaque) { static prof_tctx_t * prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) { - tsdn_t *tsdn = (tsdn_t *)arg; + tsdn_t *tsdn = (tsdn_t *)arg; prof_tctx_t *ret; malloc_mutex_assert_owner(tsdn, tctx->gctx->lock); @@ -793,6 +768,7 @@ prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) { case prof_tctx_state_purgatory: ret = tctx; goto label_return; + case prof_tctx_state_initializing: default: not_reached(); } @@ -832,8 +808,8 @@ prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) { prof_gctx_merge_iter_arg_t *arg = (prof_gctx_merge_iter_arg_t *)opaque; malloc_mutex_lock(arg->tsdn, gctx->lock); - tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_merge_iter, - (void *)arg->tsdn); + tctx_tree_iter( + &gctx->tctxs, NULL, prof_tctx_merge_iter, (void *)arg->tsdn); if (gctx->cnt_summed.curobjs != 0) { (*arg->leak_ngctx)++; } @@ -845,7 +821,7 @@ prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) { static void prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs) { prof_tdata_t *tdata = prof_tdata_get(tsd, false); - prof_gctx_t *gctx; + prof_gctx_t *gctx; /* * Standard tree iteration won't work here, because as soon as we @@ -861,15 +837,14 @@ prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs) { next = NULL; do { - prof_tctx_t *to_destroy = - tctx_tree_iter(&gctx->tctxs, next, - prof_tctx_finish_iter, + prof_tctx_t *to_destroy = tctx_tree_iter( + &gctx->tctxs, next, prof_tctx_finish_iter, (void *)tsd_tsdn(tsd)); if (to_destroy != NULL) { - next = tctx_tree_next(&gctx->tctxs, - to_destroy); - tctx_tree_remove(&gctx->tctxs, - to_destroy); + next = tctx_tree_next( + &gctx->tctxs, to_destroy); + tctx_tree_remove( + &gctx->tctxs, to_destroy); idalloctm(tsd_tsdn(tsd), to_destroy, NULL, NULL, true, true); } else { @@ -890,41 +865,41 @@ prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs) { typedef struct prof_tdata_merge_iter_arg_s prof_tdata_merge_iter_arg_t; struct prof_tdata_merge_iter_arg_s { - tsdn_t *tsdn; + tsdn_t *tsdn; prof_cnt_t *cnt_all; }; static prof_tdata_t * -prof_tdata_merge_iter(prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata, - void *opaque) { - prof_tdata_merge_iter_arg_t *arg = - (prof_tdata_merge_iter_arg_t *)opaque; +prof_tdata_merge_iter( + prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata, void *opaque) { + prof_tdata_merge_iter_arg_t *arg = (prof_tdata_merge_iter_arg_t *) + opaque; malloc_mutex_lock(arg->tsdn, tdata->lock); if (!tdata->expired) { size_t tabind; union { - prof_tctx_t *p; - void *v; + prof_tctx_t *p; + void *v; } tctx; tdata->dumping = true; memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t)); - for (tabind = 0; !ckh_iter(&tdata->bt2tctx, &tabind, NULL, - &tctx.v);) { + for (tabind = 0; + !ckh_iter(&tdata->bt2tctx, &tabind, NULL, &tctx.v);) { prof_tctx_merge_tdata(arg->tsdn, tctx.p, tdata); } arg->cnt_all->curobjs += tdata->cnt_summed.curobjs; - arg->cnt_all->curobjs_shifted_unbiased - += tdata->cnt_summed.curobjs_shifted_unbiased; + arg->cnt_all->curobjs_shifted_unbiased += + tdata->cnt_summed.curobjs_shifted_unbiased; arg->cnt_all->curbytes += tdata->cnt_summed.curbytes; - arg->cnt_all->curbytes_unbiased - += tdata->cnt_summed.curbytes_unbiased; + arg->cnt_all->curbytes_unbiased += + tdata->cnt_summed.curbytes_unbiased; if (opt_prof_accum) { arg->cnt_all->accumobjs += tdata->cnt_summed.accumobjs; - arg->cnt_all->accumobjs_shifted_unbiased - += tdata->cnt_summed.accumobjs_shifted_unbiased; + arg->cnt_all->accumobjs_shifted_unbiased += + tdata->cnt_summed.accumobjs_shifted_unbiased; arg->cnt_all->accumbytes += tdata->cnt_summed.accumbytes; arg->cnt_all->accumbytes_unbiased += @@ -939,18 +914,18 @@ prof_tdata_merge_iter(prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata, } static prof_tdata_t * -prof_tdata_dump_iter(prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata, - void *opaque) { +prof_tdata_dump_iter( + prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata, void *opaque) { if (!tdata->dumping) { return NULL; } prof_dump_iter_arg_t *arg = (prof_dump_iter_arg_t *)opaque; - prof_dump_printf(arg->prof_dump_write, arg->cbopaque, " t%"FMTu64": ", - tdata->thr_uid); - prof_dump_print_cnts(arg->prof_dump_write, arg->cbopaque, - &tdata->cnt_summed); - if (tdata->thread_name != NULL) { + prof_dump_printf(arg->prof_dump_write, arg->cbopaque, + " t%" FMTu64 ": ", tdata->thr_uid); + prof_dump_print_cnts( + arg->prof_dump_write, arg->cbopaque, &tdata->cnt_summed); + if (!prof_thread_name_empty(tdata)) { arg->prof_dump_write(arg->cbopaque, " "); arg->prof_dump_write(arg->cbopaque, tdata->thread_name); } @@ -961,7 +936,7 @@ prof_tdata_dump_iter(prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata, static void prof_dump_header(prof_dump_iter_arg_t *arg, const prof_cnt_t *cnt_all) { prof_dump_printf(arg->prof_dump_write, arg->cbopaque, - "heap_v2/%"FMTu64"\n t*: ", ((uint64_t)1U << lg_prof_sample)); + "heap_v2/%" FMTu64 "\n t*: ", ((uint64_t)1U << lg_prof_sample)); prof_dump_print_cnts(arg->prof_dump_write, arg->cbopaque, cnt_all); arg->prof_dump_write(arg->cbopaque, "\n"); @@ -977,8 +952,8 @@ prof_dump_gctx(prof_dump_iter_arg_t *arg, prof_gctx_t *gctx, malloc_mutex_assert_owner(arg->tsdn, gctx->lock); /* Avoid dumping such gctx's that have no useful data. */ - if ((!opt_prof_accum && gctx->cnt_summed.curobjs == 0) || - (opt_prof_accum && gctx->cnt_summed.accumobjs == 0)) { + if ((!opt_prof_accum && gctx->cnt_summed.curobjs == 0) + || (opt_prof_accum && gctx->cnt_summed.accumobjs == 0)) { assert(gctx->cnt_summed.curobjs == 0); assert(gctx->cnt_summed.curbytes == 0); /* @@ -997,12 +972,12 @@ prof_dump_gctx(prof_dump_iter_arg_t *arg, prof_gctx_t *gctx, arg->prof_dump_write(arg->cbopaque, "@"); for (unsigned i = 0; i < bt->len; i++) { prof_dump_printf(arg->prof_dump_write, arg->cbopaque, - " %#"FMTxPTR, (uintptr_t)bt->vec[i]); + " %#" FMTxPTR, (uintptr_t)bt->vec[i]); } arg->prof_dump_write(arg->cbopaque, "\n t*: "); - prof_dump_print_cnts(arg->prof_dump_write, arg->cbopaque, - &gctx->cnt_summed); + prof_dump_print_cnts( + arg->prof_dump_write, arg->cbopaque, &gctx->cnt_summed); arg->prof_dump_write(arg->cbopaque, "\n"); tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_dump_iter, arg); @@ -1023,18 +998,21 @@ prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_ngctx) { */ if (cnt_all->curbytes != 0) { double sample_period = (double)((uint64_t)1 << lg_prof_sample); - double ratio = (((double)cnt_all->curbytes) / - (double)cnt_all->curobjs) / sample_period; - double scale_factor = 1.0 / (1.0 - exp(-ratio)); - uint64_t curbytes = (uint64_t)round(((double)cnt_all->curbytes) - * scale_factor); - uint64_t curobjs = (uint64_t)round(((double)cnt_all->curobjs) * - scale_factor); + double ratio = (((double)cnt_all->curbytes) + / (double)cnt_all->curobjs) + / sample_period; + double scale_factor = 1.0 / (1.0 - exp(-ratio)); + uint64_t curbytes = (uint64_t)round( + ((double)cnt_all->curbytes) * scale_factor); + uint64_t curobjs = (uint64_t)round( + ((double)cnt_all->curobjs) * scale_factor); - malloc_printf(": Leak approximation summary: ~%"FMTu64 - " byte%s, ~%"FMTu64" object%s, >= %zu context%s\n", - curbytes, (curbytes != 1) ? "s" : "", curobjs, (curobjs != - 1) ? "s" : "", leak_ngctx, (leak_ngctx != 1) ? "s" : ""); + malloc_printf( + ": Leak approximation summary: ~%" FMTu64 + " byte%s, ~%" FMTu64 " object%s, >= %zu context%s\n", + curbytes, (curbytes != 1) ? "s" : "", curobjs, + (curobjs != 1) ? "s" : "", leak_ngctx, + (leak_ngctx != 1) ? "s" : ""); malloc_printf( ": Run jeprof on dump output for leak detail\n"); if (opt_prof_leak_error) { @@ -1065,8 +1043,8 @@ prof_dump_prep(tsd_t *tsd, prof_tdata_t *tdata, prof_cnt_t *cnt_all, size_t *leak_ngctx, prof_gctx_tree_t *gctxs) { size_t tabind; union { - prof_gctx_t *p; - void *v; + prof_gctx_t *p; + void *v; } gctx; prof_enter(tsd, tdata); @@ -1085,19 +1063,19 @@ prof_dump_prep(tsd_t *tsd, prof_tdata_t *tdata, prof_cnt_t *cnt_all, * stats and merge them into the associated gctx's. */ memset(cnt_all, 0, sizeof(prof_cnt_t)); - prof_tdata_merge_iter_arg_t prof_tdata_merge_iter_arg = {tsd_tsdn(tsd), - cnt_all}; + prof_tdata_merge_iter_arg_t prof_tdata_merge_iter_arg = { + tsd_tsdn(tsd), cnt_all}; malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx); - tdata_tree_iter(&tdatas, NULL, prof_tdata_merge_iter, - &prof_tdata_merge_iter_arg); + tdata_tree_iter( + &tdatas, NULL, prof_tdata_merge_iter, &prof_tdata_merge_iter_arg); malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx); /* Merge tctx stats into gctx's. */ *leak_ngctx = 0; - prof_gctx_merge_iter_arg_t prof_gctx_merge_iter_arg = {tsd_tsdn(tsd), - leak_ngctx}; - gctx_tree_iter(gctxs, NULL, prof_gctx_merge_iter, - &prof_gctx_merge_iter_arg); + prof_gctx_merge_iter_arg_t prof_gctx_merge_iter_arg = { + tsd_tsdn(tsd), leak_ngctx}; + gctx_tree_iter( + gctxs, NULL, prof_gctx_merge_iter, &prof_gctx_merge_iter_arg); prof_leave(tsd, tdata); } @@ -1106,12 +1084,12 @@ void prof_dump_impl(tsd_t *tsd, write_cb_t *prof_dump_write, void *cbopaque, prof_tdata_t *tdata, bool leakcheck) { malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_dump_mtx); - prof_cnt_t cnt_all; - size_t leak_ngctx; + prof_cnt_t cnt_all; + size_t leak_ngctx; prof_gctx_tree_t gctxs; prof_dump_prep(tsd, tdata, &cnt_all, &leak_ngctx, &gctxs); - prof_dump_iter_arg_t prof_dump_iter_arg = {tsd_tsdn(tsd), - prof_dump_write, cbopaque}; + prof_dump_iter_arg_t prof_dump_iter_arg = { + tsd_tsdn(tsd), prof_dump_write, cbopaque}; prof_dump_header(&prof_dump_iter_arg, &cnt_all); gctx_tree_iter(&gctxs, NULL, prof_gctx_dump_iter, &prof_dump_iter_arg); prof_gctx_finish(tsd, &gctxs); @@ -1123,12 +1101,12 @@ prof_dump_impl(tsd_t *tsd, write_cb_t *prof_dump_write, void *cbopaque, /* Used in unit tests. */ void prof_cnt_all(prof_cnt_t *cnt_all) { - tsd_t *tsd = tsd_fetch(); + tsd_t *tsd = tsd_fetch(); prof_tdata_t *tdata = prof_tdata_get(tsd, false); if (tdata == NULL) { memset(cnt_all, 0, sizeof(prof_cnt_t)); } else { - size_t leak_ngctx; + size_t leak_ngctx; prof_gctx_tree_t gctxs; prof_dump_prep(tsd, tdata, cnt_all, &leak_ngctx, &gctxs); prof_gctx_finish(tsd, &gctxs); @@ -1167,23 +1145,31 @@ prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim, cassert(config_prof); /* Initialize an empty cache for this thread. */ - tdata = (prof_tdata_t *)iallocztm(tsd_tsdn(tsd), sizeof(prof_tdata_t), - sz_size2index(sizeof(prof_tdata_t)), false, NULL, true, + size_t tdata_sz = ALIGNMENT_CEILING(sizeof(prof_tdata_t), QUANTUM); + size_t total_sz = tdata_sz + sizeof(void *) * opt_prof_bt_max; + tdata = (prof_tdata_t *)iallocztm(tsd_tsdn(tsd), total_sz, + sz_size2index(total_sz), false, NULL, true, arena_get(TSDN_NULL, 0, true), true); if (tdata == NULL) { return NULL; } + tdata->vec = (void **)((byte_t *)tdata + tdata_sz); tdata->lock = prof_tdata_mutex_choose(thr_uid); tdata->thr_uid = thr_uid; tdata->thr_discrim = thr_discrim; - tdata->thread_name = thread_name; tdata->attached = true; tdata->expired = false; tdata->tctx_uid_next = 0; + if (thread_name == NULL) { + prof_thread_name_clear(tdata); + } else { + prof_thread_name_write_tdata(tdata, thread_name); + } + prof_thread_name_assert(tdata); if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS, prof_bt_hash, - prof_bt_keycomp)) { + prof_bt_keycomp)) { idalloctm(tsd_tsdn(tsd), tdata, NULL, NULL, true, true); return NULL; } @@ -1214,27 +1200,22 @@ prof_tdata_should_destroy_unlocked(prof_tdata_t *tdata, bool even_if_attached) { } static bool -prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata, - bool even_if_attached) { +prof_tdata_should_destroy( + tsdn_t *tsdn, prof_tdata_t *tdata, bool even_if_attached) { malloc_mutex_assert_owner(tsdn, tdata->lock); return prof_tdata_should_destroy_unlocked(tdata, even_if_attached); } static void -prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata, - bool even_if_attached) { +prof_tdata_destroy_locked( + tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached) { malloc_mutex_assert_owner(tsd_tsdn(tsd), &tdatas_mtx); malloc_mutex_assert_not_owner(tsd_tsdn(tsd), tdata->lock); tdata_tree_remove(&tdatas, tdata); - assert(prof_tdata_should_destroy_unlocked(tdata, even_if_attached)); - if (tdata->thread_name != NULL) { - idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, NULL, true, - true); - } ckh_delete(tsd, &tdata->bt2tctx); idalloctm(tsd_tsdn(tsd), tdata, NULL, NULL, true, true); } @@ -1252,8 +1233,8 @@ prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata) { malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock); if (tdata->attached) { - destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata, - true); + destroy_tdata = prof_tdata_should_destroy( + tsd_tsdn(tsd), tdata, true); /* * Only detach if !destroy_tdata, because detaching would allow * another thread to win the race to destroy tdata. @@ -1288,8 +1269,8 @@ prof_tdata_expire(tsdn_t *tsdn, prof_tdata_t *tdata) { } static prof_tdata_t * -prof_tdata_reset_iter(prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata, - void *arg) { +prof_tdata_reset_iter( + prof_tdata_tree_t *tdatas_ptr, prof_tdata_t *tdata, void *arg) { tsdn_t *tsdn = (tsdn_t *)arg; return (prof_tdata_expire(tsdn, tdata) ? tdata : NULL); @@ -1309,8 +1290,8 @@ prof_reset(tsd_t *tsd, size_t lg_sample) { next = NULL; do { - prof_tdata_t *to_destroy = tdata_tree_iter(&tdatas, next, - prof_tdata_reset_iter, (void *)tsd); + prof_tdata_t *to_destroy = tdata_tree_iter( + &tdatas, next, prof_tdata_reset_iter, (void *)tsd); if (to_destroy != NULL) { next = tdata_tree_next(&tdatas, to_destroy); prof_tdata_destroy_locked(tsd, to_destroy, false); @@ -1373,8 +1354,8 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) { prof_tdata_t *tdata = tctx->tdata; tctx->tdata = NULL; ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL); - bool destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), - tdata, false); + bool destroy_tdata = prof_tdata_should_destroy( + tsd_tsdn(tsd), tdata, false); malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock); if (destroy_tdata) { prof_tdata_destroy(tsd, tdata, false); @@ -1419,6 +1400,8 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) { destroy_tctx = false; destroy_gctx = false; break; + case prof_tctx_state_initializing: + case prof_tctx_state_purgatory: default: not_reached(); destroy_tctx = false; diff --git a/dep/jemalloc/src/prof_log.c b/dep/jemalloc/src/prof_log.c index 0632c3b37..74f1372f5 100644 --- a/dep/jemalloc/src/prof_log.c +++ b/dep/jemalloc/src/prof_log.c @@ -12,7 +12,7 @@ #include "jemalloc/internal/prof_log.h" #include "jemalloc/internal/prof_sys.h" -bool opt_prof_log = false; +bool opt_prof_log = false; typedef enum prof_logging_state_e prof_logging_state_t; enum prof_logging_state_e { prof_logging_state_stopped, @@ -25,15 +25,15 @@ enum prof_logging_state_e { * - started: log_start called, log_stop not called yet. Allocations are logged. * - dumping: log_stop called but not finished; samples are not logged anymore. */ -prof_logging_state_t prof_logging_state = prof_logging_state_stopped; +static prof_logging_state_t prof_logging_state = prof_logging_state_stopped; /* Used in unit tests. */ static bool prof_log_dummy = false; /* Incremented for every log file that is output. */ static uint64_t log_seq = 0; -static char log_filename[ - /* Minimize memory bloat for non-prof builds. */ +static char log_filename[ +/* Minimize memory bloat for non-prof builds. */ #ifdef JEMALLOC_PROF PATH_MAX + #endif @@ -51,8 +51,8 @@ typedef struct prof_bt_node_s prof_bt_node_t; struct prof_bt_node_s { prof_bt_node_t *next; - size_t index; - prof_bt_t bt; + size_t index; + prof_bt_t bt; /* Variable size backtrace vector pointed to by bt. */ void *vec[1]; }; @@ -61,8 +61,8 @@ typedef struct prof_thr_node_s prof_thr_node_t; struct prof_thr_node_s { prof_thr_node_t *next; - size_t index; - uint64_t thr_uid; + size_t index; + uint64_t thr_uid; /* Variable size based on thr_name_sz. */ char name[1]; }; @@ -91,15 +91,15 @@ struct prof_alloc_node_s { * These are the backtraces and threads that have already been logged by an * allocation. */ -static bool log_tables_initialized = false; +static bool log_tables_initialized = false; static ckh_t log_bt_node_set; static ckh_t log_thr_node_set; /* Store linked lists for logged data. */ -static prof_bt_node_t *log_bt_first = NULL; -static prof_bt_node_t *log_bt_last = NULL; -static prof_thr_node_t *log_thr_first = NULL; -static prof_thr_node_t *log_thr_last = NULL; +static prof_bt_node_t *log_bt_first = NULL; +static prof_bt_node_t *log_bt_last = NULL; +static prof_thr_node_t *log_thr_first = NULL; +static prof_thr_node_t *log_thr_last = NULL; static prof_alloc_node_t *log_alloc_first = NULL; static prof_alloc_node_t *log_alloc_last = NULL; @@ -131,12 +131,12 @@ prof_log_bt_index(tsd_t *tsd, prof_bt_t *bt) { /* See if this backtrace is already cached in the table. */ if (ckh_search(&log_bt_node_set, (void *)(&dummy_node), - (void **)(&node), NULL)) { - size_t sz = offsetof(prof_bt_node_t, vec) + - (bt->len * sizeof(void *)); - prof_bt_node_t *new_node = (prof_bt_node_t *) - iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, - true, arena_get(TSDN_NULL, 0, true), true); + (void **)(&node), NULL)) { + size_t sz = offsetof(prof_bt_node_t, vec) + + (bt->len * sizeof(void *)); + prof_bt_node_t *new_node = (prof_bt_node_t *)iallocztm( + tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, true, + arena_get(TSDN_NULL, 0, true), true); if (log_bt_first == NULL) { log_bt_first = new_node; log_bt_last = new_node; @@ -174,11 +174,11 @@ prof_log_thr_index(tsd_t *tsd, uint64_t thr_uid, const char *name) { /* See if this thread is already cached in the table. */ if (ckh_search(&log_thr_node_set, (void *)(&dummy_node), - (void **)(&node), NULL)) { + (void **)(&node), NULL)) { size_t sz = offsetof(prof_thr_node_t, name) + strlen(name) + 1; - prof_thr_node_t *new_node = (prof_thr_node_t *) - iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, - true, arena_get(TSDN_NULL, 0, true), true); + prof_thr_node_t *new_node = (prof_thr_node_t *)iallocztm( + tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, true, + arena_get(TSDN_NULL, 0, true), true); if (log_thr_first == NULL) { log_thr_first = new_node; log_thr_last = new_node; @@ -225,9 +225,9 @@ prof_try_log(tsd_t *tsd, size_t usize, prof_info_t *prof_info) { if (!log_tables_initialized) { bool err1 = ckh_new(tsd, &log_bt_node_set, PROF_CKH_MINITEMS, - prof_bt_node_hash, prof_bt_node_keycomp); + prof_bt_node_hash, prof_bt_node_keycomp); bool err2 = ckh_new(tsd, &log_thr_node_set, PROF_CKH_MINITEMS, - prof_thr_node_hash, prof_thr_node_keycomp); + prof_thr_node_hash, prof_thr_node_keycomp); if (err1 || err2) { goto label_done; } @@ -238,13 +238,12 @@ prof_try_log(tsd_t *tsd, size_t usize, prof_info_t *prof_info) { nstime_t free_time; nstime_prof_init_update(&free_time); - size_t sz = sizeof(prof_alloc_node_t); - prof_alloc_node_t *new_node = (prof_alloc_node_t *) - iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, true, + size_t sz = sizeof(prof_alloc_node_t); + prof_alloc_node_t *new_node = (prof_alloc_node_t *)iallocztm( + tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, true, arena_get(TSDN_NULL, 0, true), true); - const char *prod_thr_name = (tctx->tdata->thread_name == NULL)? - "" : tctx->tdata->thread_name; + const char *prod_thr_name = tctx->tdata->thread_name; const char *cons_thr_name = prof_thread_name_get(tsd); prof_bt_t bt; @@ -257,10 +256,10 @@ prof_try_log(tsd_t *tsd, size_t usize, prof_info_t *prof_info) { prof_bt_t *prod_bt = &tctx->gctx->bt; new_node->next = NULL; - new_node->alloc_thr_ind = prof_log_thr_index(tsd, tctx->tdata->thr_uid, - prod_thr_name); - new_node->free_thr_ind = prof_log_thr_index(tsd, cons_tdata->thr_uid, - cons_thr_name); + new_node->alloc_thr_ind = prof_log_thr_index( + tsd, tctx->tdata->thr_uid, prod_thr_name); + new_node->free_thr_ind = prof_log_thr_index( + tsd, cons_tdata->thr_uid, cons_thr_name); new_node->alloc_bt_ind = prof_log_bt_index(tsd, prod_bt); new_node->free_bt_ind = prof_log_bt_index(tsd, cons_bt); new_node->alloc_time_ns = nstime_ns(&alloc_time); @@ -289,8 +288,8 @@ static bool prof_bt_node_keycomp(const void *k1, const void *k2) { const prof_bt_node_t *bt_node1 = (prof_bt_node_t *)k1; const prof_bt_node_t *bt_node2 = (prof_bt_node_t *)k2; - return prof_bt_keycomp((void *)(&bt_node1->bt), - (void *)(&bt_node2->bt)); + return prof_bt_keycomp( + (void *)(&bt_node1->bt), (void *)(&bt_node2->bt)); } static void @@ -310,7 +309,7 @@ prof_thr_node_keycomp(const void *k1, const void *k2) { size_t prof_log_bt_count(void) { cassert(config_prof); - size_t cnt = 0; + size_t cnt = 0; prof_bt_node_t *node = log_bt_first; while (node != NULL) { cnt++; @@ -323,7 +322,7 @@ prof_log_bt_count(void) { size_t prof_log_alloc_count(void) { cassert(config_prof); - size_t cnt = 0; + size_t cnt = 0; prof_alloc_node_t *node = log_alloc_first; while (node != NULL) { cnt++; @@ -336,7 +335,7 @@ prof_log_alloc_count(void) { size_t prof_log_thr_count(void) { cassert(config_prof); - size_t cnt = 0; + size_t cnt = 0; prof_thr_node_t *node = log_thr_first; while (node != NULL) { cnt++; @@ -375,9 +374,8 @@ prof_log_rep_check(void) { size_t thr_count = prof_log_thr_count(); size_t alloc_count = prof_log_alloc_count(); - if (prof_logging_state == prof_logging_state_stopped) { - if (bt_count != 0 || thr_count != 0 || alloc_count || 0) { + if (bt_count != 0 || thr_count != 0 || alloc_count != 0) { return true; } } @@ -436,7 +434,8 @@ prof_log_start(tsdn_t *tsdn, const char *filename) { if (!prof_log_atexit_called) { prof_log_atexit_called = true; if (atexit(prof_log_stop_final) != 0) { - malloc_write(": Error in atexit() " + malloc_write( + ": Error in atexit() " "for logging\n"); if (opt_abort) { abort(); @@ -470,14 +469,14 @@ label_done: } struct prof_emitter_cb_arg_s { - int fd; + int fd; ssize_t ret; }; static void prof_emitter_write_cb(void *opaque, const char *to_write) { - struct prof_emitter_cb_arg_s *arg = - (struct prof_emitter_cb_arg_s *)opaque; + struct prof_emitter_cb_arg_s *arg = (struct prof_emitter_cb_arg_s *) + opaque; size_t bytes = strlen(to_write); if (prof_log_dummy) { return; @@ -502,8 +501,8 @@ prof_log_emit_threads(tsd_t *tsd, emitter_t *emitter) { char *thr_name = thr_node->name; - emitter_json_kv(emitter, "thr_name", emitter_type_string, - &thr_name); + emitter_json_kv( + emitter, "thr_name", emitter_type_string, &thr_name); emitter_json_object_end(emitter); thr_old_node = thr_node; @@ -522,7 +521,7 @@ prof_log_emit_traces(tsd_t *tsd, emitter_t *emitter) { * Calculate how many hex digits we need: twice number of bytes, two for * "0x", and then one more for terminating '\0'. */ - char buf[2 * sizeof(intptr_t) + 3]; + char buf[2 * sizeof(intptr_t) + 3]; size_t buf_sz = sizeof(buf); while (bt_node != NULL) { emitter_json_array_begin(emitter); @@ -530,8 +529,8 @@ prof_log_emit_traces(tsd_t *tsd, emitter_t *emitter) { for (i = 0; i < bt_node->bt.len; i++) { malloc_snprintf(buf, buf_sz, "%p", bt_node->bt.vec[i]); char *trace_str = buf; - emitter_json_value(emitter, emitter_type_string, - &trace_str); + emitter_json_value( + emitter, emitter_type_string, &trace_str); } emitter_json_array_end(emitter); @@ -562,21 +561,21 @@ prof_log_emit_allocs(tsd_t *tsd, emitter_t *emitter) { emitter_json_kv(emitter, "free_trace", emitter_type_size, &alloc_node->free_bt_ind); - emitter_json_kv(emitter, "alloc_timestamp", - emitter_type_uint64, &alloc_node->alloc_time_ns); + emitter_json_kv(emitter, "alloc_timestamp", emitter_type_uint64, + &alloc_node->alloc_time_ns); emitter_json_kv(emitter, "free_timestamp", emitter_type_uint64, &alloc_node->free_time_ns); - emitter_json_kv(emitter, "usize", emitter_type_uint64, - &alloc_node->usize); + emitter_json_kv( + emitter, "usize", emitter_type_uint64, &alloc_node->usize); emitter_json_object_end(emitter); alloc_old_node = alloc_node; alloc_node = alloc_node->next; - idalloctm(tsd_tsdn(tsd), alloc_old_node, NULL, NULL, true, - true); + idalloctm( + tsd_tsdn(tsd), alloc_old_node, NULL, NULL, true, true); } emitter_json_array_end(emitter); } @@ -592,15 +591,14 @@ prof_log_emit_metadata(emitter_t *emitter) { emitter_json_kv(emitter, "duration", emitter_type_uint64, &ns); char *vers = JEMALLOC_VERSION; - emitter_json_kv(emitter, "version", - emitter_type_string, &vers); + emitter_json_kv(emitter, "version", emitter_type_string, &vers); - emitter_json_kv(emitter, "lg_sample_rate", - emitter_type_int, &lg_prof_sample); + emitter_json_kv( + emitter, "lg_sample_rate", emitter_type_int, &lg_prof_sample); const char *res_type = prof_time_res_mode_names[opt_prof_time_res]; - emitter_json_kv(emitter, "prof_time_resolution", emitter_type_string, - &res_type); + emitter_json_kv( + emitter, "prof_time_resolution", emitter_type_string, &res_type); int pid = prof_getpid(); emitter_json_kv(emitter, "pid", emitter_type_int, &pid); @@ -633,7 +631,6 @@ prof_log_stop(tsdn_t *tsdn) { prof_logging_state = prof_logging_state_dumping; malloc_mutex_unlock(tsdn, &log_mtx); - emitter_t emitter; /* Create a file. */ @@ -646,8 +643,10 @@ prof_log_stop(tsdn_t *tsdn) { } if (fd == -1) { - malloc_printf(": creat() for log file \"%s\" " - " failed with %d\n", log_filename, errno); + malloc_printf( + ": creat() for log file \"%s\" " + " failed with %d\n", + log_filename, errno); if (opt_abort) { abort(); } @@ -660,8 +659,8 @@ prof_log_stop(tsdn_t *tsdn) { buf_writer_t buf_writer; buf_writer_init(tsdn, &buf_writer, prof_emitter_write_cb, &arg, NULL, PROF_LOG_STOP_BUFSIZE); - emitter_init(&emitter, emitter_output_json_compact, buf_writer_cb, - &buf_writer); + emitter_init( + &emitter, emitter_output_json_compact, buf_writer_cb, &buf_writer); emitter_begin(&emitter); prof_log_emit_metadata(&emitter); @@ -702,8 +701,8 @@ JEMALLOC_COLD bool prof_log_init(tsd_t *tsd) { cassert(config_prof); - if (malloc_mutex_init(&log_mtx, "prof_log", - WITNESS_RANK_PROF_LOG, malloc_mutex_rank_exclusive)) { + if (malloc_mutex_init(&log_mtx, "prof_log", WITNESS_RANK_PROF_LOG, + malloc_mutex_rank_exclusive)) { return true; } diff --git a/dep/jemalloc/src/prof_recent.c b/dep/jemalloc/src/prof_recent.c index 834a9446c..f7108bee8 100644 --- a/dep/jemalloc/src/prof_recent.c +++ b/dep/jemalloc/src/prof_recent.c @@ -7,22 +7,22 @@ #include "jemalloc/internal/prof_data.h" #include "jemalloc/internal/prof_recent.h" -ssize_t opt_prof_recent_alloc_max = PROF_RECENT_ALLOC_MAX_DEFAULT; -malloc_mutex_t prof_recent_alloc_mtx; /* Protects the fields below */ +ssize_t opt_prof_recent_alloc_max = PROF_RECENT_ALLOC_MAX_DEFAULT; +malloc_mutex_t prof_recent_alloc_mtx; /* Protects the fields below */ static atomic_zd_t prof_recent_alloc_max; -static ssize_t prof_recent_alloc_count = 0; +static ssize_t prof_recent_alloc_count = 0; prof_recent_list_t prof_recent_alloc_list; malloc_mutex_t prof_recent_dump_mtx; /* Protects dumping. */ static void -prof_recent_alloc_max_init() { - atomic_store_zd(&prof_recent_alloc_max, opt_prof_recent_alloc_max, - ATOMIC_RELAXED); +prof_recent_alloc_max_init(void) { + atomic_store_zd( + &prof_recent_alloc_max, opt_prof_recent_alloc_max, ATOMIC_RELAXED); } static inline ssize_t -prof_recent_alloc_max_get_no_lock() { +prof_recent_alloc_max_get_no_lock(void) { return atomic_load_zd(&prof_recent_alloc_max, ATOMIC_RELAXED); } @@ -144,26 +144,26 @@ edata_prof_recent_alloc_get_no_lock_test(const edata_t *edata) { static inline prof_recent_t * edata_prof_recent_alloc_get(tsd_t *tsd, const edata_t *edata) { malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx); - prof_recent_t *recent_alloc = - edata_prof_recent_alloc_get_no_lock(edata); - assert(recent_alloc == NULL || - prof_recent_alloc_edata_get(tsd, recent_alloc) == edata); + prof_recent_t *recent_alloc = edata_prof_recent_alloc_get_no_lock( + edata); + assert(recent_alloc == NULL + || prof_recent_alloc_edata_get(tsd, recent_alloc) == edata); return recent_alloc; } static prof_recent_t * -edata_prof_recent_alloc_update_internal(tsd_t *tsd, edata_t *edata, - prof_recent_t *recent_alloc) { +edata_prof_recent_alloc_update_internal( + tsd_t *tsd, edata_t *edata, prof_recent_t *recent_alloc) { malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx); - prof_recent_t *old_recent_alloc = - edata_prof_recent_alloc_get(tsd, edata); + prof_recent_t *old_recent_alloc = edata_prof_recent_alloc_get( + tsd, edata); edata_prof_recent_alloc_set_dont_call_directly(edata, recent_alloc); return old_recent_alloc; } static void -edata_prof_recent_alloc_set(tsd_t *tsd, edata_t *edata, - prof_recent_t *recent_alloc) { +edata_prof_recent_alloc_set( + tsd_t *tsd, edata_t *edata, prof_recent_t *recent_alloc) { malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx); assert(recent_alloc != NULL); prof_recent_t *old_recent_alloc = @@ -173,8 +173,8 @@ edata_prof_recent_alloc_set(tsd_t *tsd, edata_t *edata, } static void -edata_prof_recent_alloc_reset(tsd_t *tsd, edata_t *edata, - prof_recent_t *recent_alloc) { +edata_prof_recent_alloc_reset( + tsd_t *tsd, edata_t *edata, prof_recent_t *recent_alloc) { malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx); assert(recent_alloc != NULL); prof_recent_t *old_recent_alloc = @@ -265,14 +265,14 @@ prof_recent_alloc_assert_count(tsd_t *tsd) { if (!config_debug) { return; } - ssize_t count = 0; + ssize_t count = 0; prof_recent_t *n; - ql_foreach(n, &prof_recent_alloc_list, link) { + ql_foreach (n, &prof_recent_alloc_list, link) { ++count; } assert(count == prof_recent_alloc_count); - assert(prof_recent_alloc_max_get(tsd) == -1 || - count <= prof_recent_alloc_max_get(tsd)); + assert(prof_recent_alloc_max_get(tsd) == -1 + || count <= prof_recent_alloc_max_get(tsd)); } void @@ -319,8 +319,8 @@ prof_recent_alloc(tsd_t *tsd, edata_t *edata, size_t size, size_t usize) { * the allocation locks. */ prof_recent_t *reserve = NULL; - if (prof_recent_alloc_max_get(tsd) == -1 || - prof_recent_alloc_count < prof_recent_alloc_max_get(tsd)) { + if (prof_recent_alloc_max_get(tsd) == -1 + || prof_recent_alloc_count < prof_recent_alloc_max_get(tsd)) { assert(prof_recent_alloc_max_get(tsd) != 0); malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_alloc_mtx); reserve = prof_recent_allocate_node(tsd_tsdn(tsd)); @@ -346,8 +346,9 @@ prof_recent_alloc(tsd_t *tsd, edata_t *edata, size_t size, size_t usize) { ql_rotate(&prof_recent_alloc_list, link); } else { /* Otherwise make use of the new node. */ - assert(prof_recent_alloc_max_get(tsd) == -1 || - prof_recent_alloc_count < prof_recent_alloc_max_get(tsd)); + assert(prof_recent_alloc_max_get(tsd) == -1 + || prof_recent_alloc_count + < prof_recent_alloc_max_get(tsd)); if (reserve == NULL) { goto label_rollback; } @@ -403,7 +404,7 @@ label_rollback: } ssize_t -prof_recent_alloc_max_ctl_read() { +prof_recent_alloc_max_ctl_read(void) { cassert(config_prof); /* Don't bother to acquire the lock. */ return prof_recent_alloc_max_get_no_lock(); @@ -421,7 +422,7 @@ prof_recent_alloc_restore_locked(tsd_t *tsd, prof_recent_list_t *to_delete) { } prof_recent_t *node; - ql_foreach(node, &prof_recent_alloc_list, link) { + ql_foreach (node, &prof_recent_alloc_list, link) { if (prof_recent_alloc_count == max) { break; } @@ -462,7 +463,7 @@ prof_recent_alloc_max_ctl_write(tsd_t *tsd, ssize_t max) { assert(max >= -1); malloc_mutex_lock(tsd_tsdn(tsd), &prof_recent_alloc_mtx); prof_recent_alloc_assert_count(tsd); - const ssize_t old_max = prof_recent_alloc_max_update(tsd, max); + const ssize_t old_max = prof_recent_alloc_max_update(tsd, max); prof_recent_list_t to_delete; prof_recent_alloc_restore_locked(tsd, &to_delete); malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_alloc_mtx); @@ -472,7 +473,7 @@ prof_recent_alloc_max_ctl_write(tsd_t *tsd, ssize_t max) { static void prof_recent_alloc_dump_bt(emitter_t *emitter, prof_tctx_t *tctx) { - char bt_buf[2 * sizeof(intptr_t) + 3]; + char bt_buf[2 * sizeof(intptr_t) + 3]; char *s = bt_buf; assert(tctx != NULL); prof_bt_t *bt = &tctx->gctx->bt; @@ -495,13 +496,14 @@ prof_recent_alloc_dump_node(emitter_t *emitter, prof_recent_t *node) { &node->alloc_tctx->thr_uid); prof_tdata_t *alloc_tdata = node->alloc_tctx->tdata; assert(alloc_tdata != NULL); - if (alloc_tdata->thread_name != NULL) { + if (!prof_thread_name_empty(alloc_tdata)) { + const char *thread_name = alloc_tdata->thread_name; emitter_json_kv(emitter, "alloc_thread_name", - emitter_type_string, &alloc_tdata->thread_name); + emitter_type_string, &thread_name); } uint64_t alloc_time_ns = nstime_ns(&node->alloc_time); - emitter_json_kv(emitter, "alloc_time", emitter_type_uint64, - &alloc_time_ns); + emitter_json_kv( + emitter, "alloc_time", emitter_type_uint64, &alloc_time_ns); emitter_json_array_kv_begin(emitter, "alloc_trace"); prof_recent_alloc_dump_bt(emitter, node->alloc_tctx); emitter_json_array_end(emitter); @@ -511,9 +513,10 @@ prof_recent_alloc_dump_node(emitter_t *emitter, prof_recent_t *node) { emitter_type_uint64, &node->dalloc_tctx->thr_uid); prof_tdata_t *dalloc_tdata = node->dalloc_tctx->tdata; assert(dalloc_tdata != NULL); - if (dalloc_tdata->thread_name != NULL) { + if (!prof_thread_name_empty(dalloc_tdata)) { + const char *thread_name = dalloc_tdata->thread_name; emitter_json_kv(emitter, "dalloc_thread_name", - emitter_type_string, &dalloc_tdata->thread_name); + emitter_type_string, &thread_name); } assert(!nstime_equals_zero(&node->dalloc_time)); uint64_t dalloc_time_ns = nstime_ns(&node->dalloc_time); @@ -537,8 +540,8 @@ prof_recent_alloc_dump(tsd_t *tsd, write_cb_t *write_cb, void *cbopaque) { buf_writer_init(tsd_tsdn(tsd), &buf_writer, write_cb, cbopaque, NULL, PROF_RECENT_PRINT_BUFSIZE); emitter_t emitter; - emitter_init(&emitter, emitter_output_json_compact, buf_writer_cb, - &buf_writer); + emitter_init( + &emitter, emitter_output_json_compact, buf_writer_cb, &buf_writer); prof_recent_list_t temp_list; malloc_mutex_lock(tsd_tsdn(tsd), &prof_recent_alloc_mtx); @@ -552,13 +555,13 @@ prof_recent_alloc_dump(tsd_t *tsd, write_cb_t *write_cb, void *cbopaque) { emitter_begin(&emitter); uint64_t sample_interval = (uint64_t)1U << lg_prof_sample; - emitter_json_kv(&emitter, "sample_interval", emitter_type_uint64, - &sample_interval); - emitter_json_kv(&emitter, "recent_alloc_max", emitter_type_ssize, - &dump_max); + emitter_json_kv( + &emitter, "sample_interval", emitter_type_uint64, &sample_interval); + emitter_json_kv( + &emitter, "recent_alloc_max", emitter_type_ssize, &dump_max); emitter_json_array_kv_begin(&emitter, "recent_alloc"); prof_recent_t *node; - ql_foreach(node, &temp_list, link) { + ql_foreach (node, &temp_list, link) { prof_recent_alloc_dump_node(&emitter, node); } emitter_json_array_end(&emitter); @@ -580,17 +583,17 @@ prof_recent_alloc_dump(tsd_t *tsd, write_cb_t *write_cb, void *cbopaque) { #undef PROF_RECENT_PRINT_BUFSIZE bool -prof_recent_init() { +prof_recent_init(void) { cassert(config_prof); prof_recent_alloc_max_init(); if (malloc_mutex_init(&prof_recent_alloc_mtx, "prof_recent_alloc", - WITNESS_RANK_PROF_RECENT_ALLOC, malloc_mutex_rank_exclusive)) { + WITNESS_RANK_PROF_RECENT_ALLOC, malloc_mutex_rank_exclusive)) { return true; } if (malloc_mutex_init(&prof_recent_dump_mtx, "prof_recent_dump", - WITNESS_RANK_PROF_RECENT_DUMP, malloc_mutex_rank_exclusive)) { + WITNESS_RANK_PROF_RECENT_DUMP, malloc_mutex_rank_exclusive)) { return true; } diff --git a/dep/jemalloc/src/prof_stack_range.c b/dep/jemalloc/src/prof_stack_range.c new file mode 100644 index 000000000..8ebcab8ee --- /dev/null +++ b/dep/jemalloc/src/prof_stack_range.c @@ -0,0 +1,175 @@ +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" + +#include "jemalloc/internal/malloc_io.h" +#include "jemalloc/internal/prof_sys.h" + +#if defined(__linux__) && defined(JEMALLOC_HAVE_GETTID) + +# include +# include +# include +# include // strtoul +# include +# include + +/* + * Converts a string representing a hexadecimal number to an unsigned long long + * integer. Functionally equivalent to strtoull() (for base 16) but faster for + * that case. + * + * @param nptr Pointer to the string to be converted. + * @param endptr Pointer to a pointer to character, which will be set to the + * character in `nptr` where parsing stopped. Can be NULL. + * @return The converted unsigned long long integer value. + */ +static inline unsigned long long int +strtoull_hex(const char *nptr, char **endptr) { + unsigned long long int val = 0; + int ii = 0; + for (; ii < 16; ++ii) { + char c = nptr[ii]; + if (c >= '0' && c <= '9') { + val = (val << 4) + (c - '0'); + } else if (c >= 'a' && c <= 'f') { + val = (val << 4) + (c - 'a' + 10); + } else { + break; + } + } + if (endptr) { + *endptr = (char *)(nptr + ii); + } + return val; +} + +static int +prof_mapping_containing_addr(uintptr_t addr, const char *maps_path, + uintptr_t *mm_start, uintptr_t *mm_end) { + int ret = ENOENT; /* not found */ + *mm_start = *mm_end = 0; + + /* + * Each line of /proc//maps is: + * - + * + * The fields we care about are always within the first 34 characters so + * as long as `buf` contains the start of a mapping line it can always be + * parsed. + */ + static const int kMappingFieldsWidth = 34; + + int fd = -1; + char buf[4096]; + ssize_t remaining = 0; /* actual number of bytes read to buf */ + char *line = NULL; + + while (1) { + if (fd < 0) { + /* case 0: initial open of maps file */ + fd = malloc_open(maps_path, O_RDONLY); + if (fd < 0) { + return errno; + } + + remaining = malloc_read_fd(fd, buf, sizeof(buf)); + if (remaining < 0) { + ret = errno; + break; + } else if (remaining == 0) { + break; + } + line = buf; + } else if (line == NULL) { + /* case 1: no newline found in buf */ + remaining = malloc_read_fd(fd, buf, sizeof(buf)); + if (remaining < 0) { + ret = errno; + break; + } else if (remaining == 0) { + break; + } + line = memchr(buf, '\n', remaining); + if (line != NULL) { + line++; /* advance to character after newline */ + remaining -= (line - buf); + } + } else if (line != NULL && remaining < kMappingFieldsWidth) { + /* + * case 2: found newline but insufficient characters remaining in + * buf + */ + memcpy(buf, line, + remaining); /* copy remaining characters to start of buf */ + line = buf; + + ssize_t count = malloc_read_fd( + fd, buf + remaining, sizeof(buf) - remaining); + if (count < 0) { + ret = errno; + break; + } else if (count == 0) { + break; + } + + remaining += + count; /* actual number of bytes read to buf */ + } else { + /* case 3: found newline and sufficient characters to parse */ + + /* parse - */ + char *tmp = line; + uintptr_t start_addr = (uintptr_t)strtoull_hex( + tmp, &tmp); + if (addr >= start_addr) { + tmp++; /* advance to character after '-' */ + uintptr_t end_addr = (uintptr_t)strtoull_hex( + tmp, NULL); + if (addr < end_addr) { + *mm_start = start_addr; + *mm_end = end_addr; + ret = 0; + break; + } + } + + /* Advance to character after next newline in the current buf. */ + char *prev_line = line; + line = memchr(line, '\n', remaining); + if (line != NULL) { + line++; /* advance to character after newline */ + remaining -= (line - prev_line); + } + } + } + + malloc_close(fd); + return ret; +} + +int +prof_thread_stack_range(uintptr_t fp, uintptr_t *low, uintptr_t *high) { + /* + * NOTE: Prior to kernel 4.5 an entry for every thread stack was included in + * /proc//maps as [STACK:]. Starting with kernel 4.5 only the main + * thread stack remains as the [stack] mapping. For other thread stacks the + * mapping is still visible in /proc//task//maps (though not + * labeled as [STACK:tid]). + * https://lists.ubuntu.com/archives/kernel-team/2016-March/074681.html + */ + char maps_path[64]; // "/proc//task//maps" + malloc_snprintf(maps_path, sizeof(maps_path), "/proc/%d/task/%d/maps", + getpid(), gettid()); + return prof_mapping_containing_addr(fp, maps_path, low, high); +} + +#else + +int +prof_thread_stack_range( + UNUSED uintptr_t addr, uintptr_t *stack_start, uintptr_t *stack_end) { + *stack_start = *stack_end = 0; + return ENOENT; +} + +#endif // __linux__ diff --git a/dep/jemalloc/src/prof_stats.c b/dep/jemalloc/src/prof_stats.c index 5d1a506bb..db248be76 100644 --- a/dep/jemalloc/src/prof_stats.c +++ b/dep/jemalloc/src/prof_stats.c @@ -3,8 +3,8 @@ #include "jemalloc/internal/prof_stats.h" -bool opt_prof_stats = false; -malloc_mutex_t prof_stats_mtx; +bool opt_prof_stats = false; +malloc_mutex_t prof_stats_mtx; static prof_stats_t prof_stats_live[PROF_SC_NSIZES]; static prof_stats_t prof_stats_accum[PROF_SC_NSIZES]; diff --git a/dep/jemalloc/src/prof_sys.c b/dep/jemalloc/src/prof_sys.c index b5f1f5b22..be50c0bed 100644 --- a/dep/jemalloc/src/prof_sys.c +++ b/dep/jemalloc/src/prof_sys.c @@ -1,15 +1,15 @@ -#define JEMALLOC_PROF_SYS_C_ #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/jemalloc_internal_includes.h" #include "jemalloc/internal/buf_writer.h" #include "jemalloc/internal/ctl.h" +#include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/prof_data.h" #include "jemalloc/internal/prof_sys.h" #ifdef JEMALLOC_PROF_LIBUNWIND -#define UNW_LOCAL_ONLY -#include +# define UNW_LOCAL_ONLY +# include #endif #ifdef JEMALLOC_PROF_LIBGCC @@ -18,17 +18,21 @@ * use libgcc's unwinding functionality, but after we've included that, we've * already hooked _Unwind_Backtrace. We'll temporarily disable hooking. */ -#undef _Unwind_Backtrace -#include -#define _Unwind_Backtrace JEMALLOC_TEST_HOOK(_Unwind_Backtrace, test_hooks_libc_hook) +# undef _Unwind_Backtrace +# include +# define _Unwind_Backtrace \ + JEMALLOC_TEST_HOOK(_Unwind_Backtrace, test_hooks_libc_hook) +#endif + +#ifdef JEMALLOC_PROF_FRAME_POINTER +// execinfo backtrace() as fallback unwinder +# include #endif /******************************************************************************/ malloc_mutex_t prof_dump_filename_mtx; -bool prof_do_mock = false; - static uint64_t prof_dump_seq; static uint64_t prof_dump_iseq; static uint64_t prof_dump_mseq; @@ -55,9 +59,9 @@ prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) { cassert(config_prof); assert(*len == 0); assert(vec != NULL); - assert(max_len == PROF_BT_MAX); + assert(max_len <= PROF_BT_MAX_LIMIT); - nframes = unw_backtrace(vec, PROF_BT_MAX); + nframes = unw_backtrace(vec, max_len); if (nframes <= 0) { return; } @@ -74,7 +78,7 @@ prof_unwind_init_callback(struct _Unwind_Context *context, void *arg) { static _Unwind_Reason_Code prof_unwind_callback(struct _Unwind_Context *context, void *arg) { prof_unwind_data_t *data = (prof_unwind_data_t *)arg; - void *ip; + void *ip; cassert(config_prof); @@ -97,32 +101,132 @@ prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) { cassert(config_prof); assert(vec != NULL); - assert(max_len == PROF_BT_MAX); + assert(max_len <= PROF_BT_MAX_LIMIT); _Unwind_Backtrace(prof_unwind_callback, &data); } -#elif (defined(JEMALLOC_PROF_GCC)) +#elif (defined(JEMALLOC_PROF_FRAME_POINTER)) +JEMALLOC_DIAGNOSTIC_PUSH +JEMALLOC_DIAGNOSTIC_IGNORE_FRAME_ADDRESS + +struct stack_range { + uintptr_t start; + uintptr_t end; +}; + +struct thread_unwind_info { + struct stack_range stack_range; + bool fallback; +}; +static __thread struct thread_unwind_info unwind_info = { + .stack_range = + { + .start = 0, + .end = 0, + }, + .fallback = false, +}; /* thread local */ + static void prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) { -#define BT_FRAME(i) \ - if ((i) < max_len) { \ - void *p; \ - if (__builtin_frame_address(i) == 0) { \ - return; \ - } \ - p = __builtin_return_address(i); \ - if (p == NULL) { \ - return; \ - } \ - vec[(i)] = p; \ - *len = (i) + 1; \ - } else { \ - return; \ + /* fp: current stack frame pointer + * + * stack_range: readable stack memory range for the current thread. + * Used to validate frame addresses during stack unwinding. + * For most threads there is a single valid stack range + * that is fixed at thread creation time. This may not be + * the case when folly fibers or boost contexts are used. + * In those cases fall back to using execinfo backtrace() + * (DWARF unwind). + */ + + /* always safe to get the current stack frame address */ + uintptr_t fp = (uintptr_t)__builtin_frame_address(0); + + /* new thread - get the stack range */ + if (!unwind_info.fallback + && unwind_info.stack_range.start == unwind_info.stack_range.end) { + if (prof_thread_stack_range(fp, &unwind_info.stack_range.start, + &unwind_info.stack_range.end) + != 0) { + unwind_info.fallback = true; + } else { + assert(fp >= unwind_info.stack_range.start + && fp < unwind_info.stack_range.end); + } } + if (unwind_info.fallback) { + goto label_fallback; + } + + unsigned ii = 0; + while (ii < max_len && fp != 0) { + if (fp < unwind_info.stack_range.start + || fp >= unwind_info.stack_range.end) { + /* + * Determining the stack range from procfs can be + * relatively expensive especially for programs with + * many threads / shared libraries. If the stack + * range has changed, it is likely to change again + * in the future (fibers or some other stack + * manipulation). So fall back to backtrace for this + * thread. + */ + unwind_info.fallback = true; + goto label_fallback; + } + void *ip = ((void **)fp)[1]; + if (ip == 0) { + break; + } + vec[ii++] = ip; + fp = ((uintptr_t *)fp)[0]; + } + *len = ii; + return; + +label_fallback: + /* + * Using the backtrace from execinfo.h here. Note that it may get + * redirected to libunwind when a libunwind not built with build-time + * flag --disable-weak-backtrace is linked. + */ + assert(unwind_info.fallback); + int nframes = backtrace(vec, max_len); + if (nframes > 0) { + *len = nframes; + } else { + *len = 0; + } +} + +JEMALLOC_DIAGNOSTIC_POP +#elif (defined(JEMALLOC_PROF_GCC)) +JEMALLOC_DIAGNOSTIC_PUSH +JEMALLOC_DIAGNOSTIC_IGNORE_FRAME_ADDRESS +static void +prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) { +/* The input arg must be a constant for __builtin_return_address. */ +# define BT_FRAME(i) \ + if ((i) < max_len) { \ + void *p; \ + if (__builtin_frame_address(i) == 0) { \ + return; \ + } \ + p = __builtin_return_address(i); \ + if (p == NULL) { \ + return; \ + } \ + vec[(i)] = p; \ + *len = (i) + 1; \ + } else { \ + return; \ + } + cassert(config_prof); assert(vec != NULL); - assert(max_len == PROF_BT_MAX); + assert(max_len <= PROF_BT_MAX_LIMIT); BT_FRAME(0) BT_FRAME(1) @@ -264,7 +368,149 @@ prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) { BT_FRAME(125) BT_FRAME(126) BT_FRAME(127) -#undef BT_FRAME + BT_FRAME(128) + BT_FRAME(129) + + BT_FRAME(130) + BT_FRAME(131) + BT_FRAME(132) + BT_FRAME(133) + BT_FRAME(134) + BT_FRAME(135) + BT_FRAME(136) + BT_FRAME(137) + BT_FRAME(138) + BT_FRAME(139) + + BT_FRAME(140) + BT_FRAME(141) + BT_FRAME(142) + BT_FRAME(143) + BT_FRAME(144) + BT_FRAME(145) + BT_FRAME(146) + BT_FRAME(147) + BT_FRAME(148) + BT_FRAME(149) + + BT_FRAME(150) + BT_FRAME(151) + BT_FRAME(152) + BT_FRAME(153) + BT_FRAME(154) + BT_FRAME(155) + BT_FRAME(156) + BT_FRAME(157) + BT_FRAME(158) + BT_FRAME(159) + + BT_FRAME(160) + BT_FRAME(161) + BT_FRAME(162) + BT_FRAME(163) + BT_FRAME(164) + BT_FRAME(165) + BT_FRAME(166) + BT_FRAME(167) + BT_FRAME(168) + BT_FRAME(169) + + BT_FRAME(170) + BT_FRAME(171) + BT_FRAME(172) + BT_FRAME(173) + BT_FRAME(174) + BT_FRAME(175) + BT_FRAME(176) + BT_FRAME(177) + BT_FRAME(178) + BT_FRAME(179) + + BT_FRAME(180) + BT_FRAME(181) + BT_FRAME(182) + BT_FRAME(183) + BT_FRAME(184) + BT_FRAME(185) + BT_FRAME(186) + BT_FRAME(187) + BT_FRAME(188) + BT_FRAME(189) + + BT_FRAME(190) + BT_FRAME(191) + BT_FRAME(192) + BT_FRAME(193) + BT_FRAME(194) + BT_FRAME(195) + BT_FRAME(196) + BT_FRAME(197) + BT_FRAME(198) + BT_FRAME(199) + + BT_FRAME(200) + BT_FRAME(201) + BT_FRAME(202) + BT_FRAME(203) + BT_FRAME(204) + BT_FRAME(205) + BT_FRAME(206) + BT_FRAME(207) + BT_FRAME(208) + BT_FRAME(209) + + BT_FRAME(210) + BT_FRAME(211) + BT_FRAME(212) + BT_FRAME(213) + BT_FRAME(214) + BT_FRAME(215) + BT_FRAME(216) + BT_FRAME(217) + BT_FRAME(218) + BT_FRAME(219) + + BT_FRAME(220) + BT_FRAME(221) + BT_FRAME(222) + BT_FRAME(223) + BT_FRAME(224) + BT_FRAME(225) + BT_FRAME(226) + BT_FRAME(227) + BT_FRAME(228) + BT_FRAME(229) + + BT_FRAME(230) + BT_FRAME(231) + BT_FRAME(232) + BT_FRAME(233) + BT_FRAME(234) + BT_FRAME(235) + BT_FRAME(236) + BT_FRAME(237) + BT_FRAME(238) + BT_FRAME(239) + + BT_FRAME(240) + BT_FRAME(241) + BT_FRAME(242) + BT_FRAME(243) + BT_FRAME(244) + BT_FRAME(245) + BT_FRAME(246) + BT_FRAME(247) + BT_FRAME(248) + BT_FRAME(249) + + BT_FRAME(250) + BT_FRAME(251) + BT_FRAME(252) + BT_FRAME(253) + BT_FRAME(254) + BT_FRAME(255) +# undef BT_FRAME + JEMALLOC_DIAGNOSTIC_POP } #else static void @@ -281,18 +527,20 @@ prof_backtrace(tsd_t *tsd, prof_bt_t *bt) { assert(prof_backtrace_hook != NULL); pre_reentrancy(tsd, NULL); - prof_backtrace_hook(bt->vec, &bt->len, PROF_BT_MAX); + prof_backtrace_hook(bt->vec, &bt->len, opt_prof_bt_max); post_reentrancy(tsd); } void -prof_hooks_init() { +prof_hooks_init(void) { prof_backtrace_hook_set(&prof_backtrace_impl); prof_dump_hook_set(NULL); + prof_sample_hook_set(NULL); + prof_sample_free_hook_set(NULL); } void -prof_unwind_init() { +prof_unwind_init(void) { #ifdef JEMALLOC_PROF_LIBGCC /* * Cause the backtracing machinery to allocate its internal @@ -318,12 +566,18 @@ prof_sys_thread_name_read_t *JET_MUTABLE prof_sys_thread_name_read = void prof_sys_thread_name_fetch(tsd_t *tsd) { -#define THREAD_NAME_MAX_LEN 16 - char buf[THREAD_NAME_MAX_LEN]; - if (!prof_sys_thread_name_read(buf, THREAD_NAME_MAX_LEN)) { - prof_thread_name_set_impl(tsd, buf); + prof_tdata_t *tdata = prof_tdata_get(tsd, true); + if (tdata == NULL) { + return; } -#undef THREAD_NAME_MAX_LEN + + if (prof_sys_thread_name_read( + tdata->thread_name, PROF_THREAD_NAME_MAX_LEN) + != 0) { + prof_thread_name_clear(tdata); + } + + tdata->thread_name[PROF_THREAD_NAME_MAX_LEN - 1] = '\0'; } int @@ -335,6 +589,41 @@ prof_getpid(void) { #endif } +static long +prof_get_pid_namespace(void) { + long ret = 0; + +#if defined(_WIN32) || defined(__APPLE__) + // Not supported, do nothing. +#else + char buf[PATH_MAX]; + const char *linkname = +# if defined(__FreeBSD__) || defined(__DragonFly__) + "/proc/curproc/ns/pid" +# else + "/proc/self/ns/pid" +# endif + ; + ssize_t linklen = +# ifndef JEMALLOC_READLINKAT + readlink(linkname, buf, PATH_MAX) +# else + readlinkat(AT_FDCWD, linkname, buf, PATH_MAX) +# endif + ; + + // namespace string is expected to be like pid:[4026531836] + if (linklen > 0) { + // Trim the trailing "]" + buf[linklen - 1] = '\0'; + char *index = strtok(buf, "pid:["); + ret = atol(index); + } +#endif + + return ret; +} + /* * This buffer is rather large for stack allocation, so use a single buffer for * all profile dumps; protected by prof_dump_mtx. @@ -362,8 +651,8 @@ struct prof_dump_arg_s { }; static void -prof_dump_check_possible_error(prof_dump_arg_t *arg, bool err_cond, - const char *format, ...) { +prof_dump_check_possible_error( + prof_dump_arg_t *arg, bool err_cond, const char *format, ...) { assert(!arg->error); if (!err_cond) { return; @@ -375,7 +664,7 @@ prof_dump_check_possible_error(prof_dump_arg_t *arg, bool err_cond, } va_list ap; - char buf[PROF_PRINTF_BUFSIZE]; + char buf[PROF_PRINTF_BUFSIZE]; va_start(ap, format); malloc_vsnprintf(buf, sizeof(buf), format, ap); va_end(ap); @@ -407,8 +696,8 @@ prof_dump_flush(void *opaque, const char *s) { cassert(config_prof); prof_dump_arg_t *arg = (prof_dump_arg_t *)opaque; if (!arg->error) { - ssize_t err = prof_dump_write_file(arg->prof_dump_fd, s, - strlen(s)); + ssize_t err = prof_dump_write_file( + arg->prof_dump_fd, s, strlen(s)); prof_dump_check_possible_error(arg, err == -1, ": failed to write during heap profile flush\n"); } @@ -421,48 +710,118 @@ prof_dump_close(prof_dump_arg_t *arg) { } } -#ifndef _WIN32 +#ifdef __APPLE__ +# include + +# ifdef __LP64__ +typedef struct mach_header_64 mach_header_t; +typedef struct segment_command_64 segment_command_t; +# define MH_MAGIC_VALUE MH_MAGIC_64 +# define MH_CIGAM_VALUE MH_CIGAM_64 +# define LC_SEGMENT_VALUE LC_SEGMENT_64 +# else +typedef struct mach_header mach_header_t; +typedef struct segment_command segment_command_t; +# define MH_MAGIC_VALUE MH_MAGIC +# define MH_CIGAM_VALUE MH_CIGAM +# define LC_SEGMENT_VALUE LC_SEGMENT +# endif + +static void +prof_dump_dyld_image_vmaddr(buf_writer_t *buf_writer, uint32_t image_index) { + const mach_header_t *header = (const mach_header_t *) + _dyld_get_image_header(image_index); + if (header == NULL + || (header->magic != MH_MAGIC_VALUE + && header->magic != MH_CIGAM_VALUE)) { + // Invalid header + return; + } + + intptr_t slide = _dyld_get_image_vmaddr_slide(image_index); + const char *name = _dyld_get_image_name(image_index); + struct load_command *load_cmd = (struct load_command *)((char *)header + + sizeof(mach_header_t)); + for (uint32_t i = 0; load_cmd && (i < header->ncmds); i++) { + if (load_cmd->cmd == LC_SEGMENT_VALUE) { + const segment_command_t *segment_cmd = + (const segment_command_t *)load_cmd; + if (!strcmp(segment_cmd->segname, "__TEXT")) { + char buffer[PATH_MAX + 1]; + malloc_snprintf(buffer, sizeof(buffer), + "%016llx-%016llx: %s\n", + segment_cmd->vmaddr + slide, + segment_cmd->vmaddr + slide + + segment_cmd->vmsize, + name); + buf_writer_cb(buf_writer, buffer); + return; + } + } + load_cmd = (struct load_command *)((char *)load_cmd + + load_cmd->cmdsize); + } +} + +static void +prof_dump_dyld_maps(buf_writer_t *buf_writer) { + uint32_t image_count = _dyld_image_count(); + for (uint32_t i = 0; i < image_count; i++) { + prof_dump_dyld_image_vmaddr(buf_writer, i); + } +} + +prof_dump_open_maps_t *JET_MUTABLE prof_dump_open_maps = NULL; + +static void +prof_dump_maps(buf_writer_t *buf_writer) { + buf_writer_cb(buf_writer, "\nMAPPED_LIBRARIES:\n"); + /* No proc map file to read on MacOS, dump dyld maps for backtrace. */ + prof_dump_dyld_maps(buf_writer); +} +#else /* !__APPLE__ */ +# ifndef _WIN32 JEMALLOC_FORMAT_PRINTF(1, 2) static int prof_open_maps_internal(const char *format, ...) { - int mfd; + int mfd; va_list ap; - char filename[PATH_MAX + 1]; + char filename[PATH_MAX + 1]; va_start(ap, format); malloc_vsnprintf(filename, sizeof(filename), format, ap); va_end(ap); -#if defined(O_CLOEXEC) +# if defined(O_CLOEXEC) mfd = open(filename, O_RDONLY | O_CLOEXEC); -#else +# else mfd = open(filename, O_RDONLY); if (mfd != -1) { fcntl(mfd, F_SETFD, fcntl(mfd, F_GETFD) | FD_CLOEXEC); } -#endif +# endif return mfd; } -#endif +# endif static int -prof_dump_open_maps_impl() { +prof_dump_open_maps_impl(void) { int mfd; cassert(config_prof); -#if defined(__FreeBSD__) || defined(__DragonFly__) +# if defined(__FreeBSD__) || defined(__DragonFly__) mfd = prof_open_maps_internal("/proc/curproc/map"); -#elif defined(_WIN32) +# elif defined(_WIN32) mfd = -1; // Not implemented -#else +# else int pid = prof_getpid(); mfd = prof_open_maps_internal("/proc/%d/task/%d/maps", pid, pid); if (mfd == -1) { mfd = prof_open_maps_internal("/proc/%d/maps", pid); } -#endif +# endif return mfd; } prof_dump_open_maps_t *JET_MUTABLE prof_dump_open_maps = @@ -486,14 +845,15 @@ prof_dump_maps(buf_writer_t *buf_writer) { buf_writer_pipe(buf_writer, prof_dump_read_maps_cb, &mfd); close(mfd); } +#endif /* __APPLE__ */ static bool -prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, - bool leakcheck) { +prof_dump( + tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck) { cassert(config_prof); assert(tsd_reentrancy_level_get(tsd) == 0); - prof_tdata_t * tdata = prof_tdata_get(tsd, true); + prof_tdata_t *tdata = prof_tdata_get(tsd, true); if (tdata == NULL) { return true; } @@ -540,7 +900,7 @@ prof_strncpy(char *UNUSED dest, const char *UNUSED src, size_t UNUSED size) { } static const char * -prof_prefix_get(tsdn_t* tsdn) { +prof_prefix_get(tsdn_t *tsdn) { malloc_mutex_assert_owner(tsdn, &prof_dump_filename_mtx); return prof_prefix == NULL ? opt_prof_prefix : prof_prefix; @@ -564,15 +924,31 @@ prof_dump_filename(tsd_t *tsd, char *filename, char v, uint64_t vseq) { const char *prefix = prof_prefix_get(tsd_tsdn(tsd)); if (vseq != VSEQ_INVALID) { - /* "...v.heap" */ - malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, - "%s.%d.%"FMTu64".%c%"FMTu64".heap", prefix, prof_getpid(), - prof_dump_seq, v, vseq); + if (opt_prof_pid_namespace) { + /* "....v.heap" */ + malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, + "%s.%ld.%d.%" FMTu64 ".%c%" FMTu64 ".heap", prefix, + prof_get_pid_namespace(), prof_getpid(), + prof_dump_seq, v, vseq); + } else { + /* "...v.heap" */ + malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, + "%s.%d.%" FMTu64 ".%c%" FMTu64 ".heap", prefix, + prof_getpid(), prof_dump_seq, v, vseq); + } } else { - /* "....heap" */ - malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, - "%s.%d.%"FMTu64".%c.heap", prefix, prof_getpid(), - prof_dump_seq, v); + if (opt_prof_pid_namespace) { + /* ".....heap" */ + malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, + "%s.%ld.%d.%" FMTu64 ".%c.heap", prefix, + prof_get_pid_namespace(), prof_getpid(), + prof_dump_seq, v); + } else { + /* "....heap" */ + malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, + "%s.%d.%" FMTu64 ".%c.heap", prefix, prof_getpid(), + prof_dump_seq, v); + } } prof_dump_seq++; } @@ -580,8 +956,15 @@ prof_dump_filename(tsd_t *tsd, char *filename, char v, uint64_t vseq) { void prof_get_default_filename(tsdn_t *tsdn, char *filename, uint64_t ind) { malloc_mutex_lock(tsdn, &prof_dump_filename_mtx); - malloc_snprintf(filename, PROF_DUMP_FILENAME_LEN, - "%s.%d.%"FMTu64".json", prof_prefix_get(tsdn), prof_getpid(), ind); + if (opt_prof_pid_namespace) { + malloc_snprintf(filename, PROF_DUMP_FILENAME_LEN, + "%s.%ld.%d.%" FMTu64 ".json", prof_prefix_get(tsdn), + prof_get_pid_namespace(), prof_getpid(), ind); + } else { + malloc_snprintf(filename, PROF_DUMP_FILENAME_LEN, + "%s.%d.%" FMTu64 ".json", prof_prefix_get(tsdn), + prof_getpid(), ind); + } malloc_mutex_unlock(tsdn, &prof_dump_filename_mtx); } @@ -600,12 +983,15 @@ bool prof_prefix_set(tsdn_t *tsdn, const char *prefix) { cassert(config_prof); ctl_mtx_assert_held(tsdn); + if (prefix == NULL) { + return true; + } malloc_mutex_lock(tsdn, &prof_dump_filename_mtx); if (prof_prefix == NULL) { malloc_mutex_unlock(tsdn, &prof_dump_filename_mtx); /* Everything is still guarded by ctl_mtx. */ - char *buffer = base_alloc(tsdn, prof_base, - PROF_DUMP_FILENAME_LEN, QUANTUM); + char *buffer = base_alloc( + tsdn, prof_base, PROF_DUMP_FILENAME_LEN, QUANTUM); if (buffer == NULL) { return true; } @@ -642,7 +1028,8 @@ prof_mdump_impl(tsd_t *tsd, const char *filename) { /* No filename specified, so automatically generate one. */ malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_filename_mtx); if (prof_prefix_get(tsd_tsdn(tsd))[0] == '\0') { - malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_filename_mtx); + malloc_mutex_unlock( + tsd_tsdn(tsd), &prof_dump_filename_mtx); return true; } prof_dump_filename(tsd, filename_buf, 'm', prof_dump_mseq); diff --git a/dep/jemalloc/src/psset.c b/dep/jemalloc/src/psset.c index 9a8f054f1..4e904feb8 100644 --- a/dep/jemalloc/src/psset.c +++ b/dep/jemalloc/src/psset.c @@ -11,7 +11,6 @@ psset_init(psset_t *psset) { hpdata_age_heap_new(&psset->pageslabs[i]); } fb_init(psset->pageslab_bitmap, PSSET_NPSIZES); - memset(&psset->merged_stats, 0, sizeof(psset->merged_stats)); memset(&psset->stats, 0, sizeof(psset->stats)); hpdata_empty_list_init(&psset->empty); for (int i = 0; i < PSSET_NPURGE_LISTS; i++) { @@ -30,15 +29,19 @@ psset_bin_stats_accum(psset_bin_stats_t *dst, psset_bin_stats_t *src) { void psset_stats_accum(psset_stats_t *dst, psset_stats_t *src) { - psset_bin_stats_accum(&dst->full_slabs[0], &src->full_slabs[0]); - psset_bin_stats_accum(&dst->full_slabs[1], &src->full_slabs[1]); - psset_bin_stats_accum(&dst->empty_slabs[0], &src->empty_slabs[0]); - psset_bin_stats_accum(&dst->empty_slabs[1], &src->empty_slabs[1]); + psset_bin_stats_accum(&dst->merged, &src->merged); + for (int huge = 0; huge < PSSET_NHUGE; huge++) { + psset_bin_stats_accum(&dst->slabs[huge], &src->slabs[huge]); + psset_bin_stats_accum( + &dst->full_slabs[huge], &src->full_slabs[huge]); + psset_bin_stats_accum( + &dst->empty_slabs[huge], &src->empty_slabs[huge]); + } for (pszind_t i = 0; i < PSSET_NPSIZES; i++) { - psset_bin_stats_accum(&dst->nonfull_slabs[i][0], - &src->nonfull_slabs[i][0]); - psset_bin_stats_accum(&dst->nonfull_slabs[i][1], - &src->nonfull_slabs[i][1]); + psset_bin_stats_accum( + &dst->nonfull_slabs[i][0], &src->nonfull_slabs[i][0]); + psset_bin_stats_accum( + &dst->nonfull_slabs[i][1], &src->nonfull_slabs[i][1]); } } @@ -48,52 +51,92 @@ psset_stats_accum(psset_stats_t *dst, psset_stats_t *src) { * bin) when we call psset_update_end. */ JEMALLOC_ALWAYS_INLINE void -psset_bin_stats_insert_remove(psset_t *psset, psset_bin_stats_t *binstats, - hpdata_t *ps, bool insert) { +psset_slab_stats_insert_remove(psset_stats_t *stats, + psset_bin_stats_t *binstats, hpdata_t *ps, bool insert) { size_t mul = insert ? (size_t)1 : (size_t)-1; + size_t nactive = hpdata_nactive_get(ps); + size_t ndirty = hpdata_ndirty_get(ps); + + stats->merged.npageslabs += mul * 1; + stats->merged.nactive += mul * nactive; + stats->merged.ndirty += mul * ndirty; + + /* + * Stats above are necessary for purging logic to work, everything + * below is to improve observability, thense is optional, so we don't + * update it, when stats disabled. + */ + if (!config_stats) { + return; + } + size_t huge_idx = (size_t)hpdata_huge_get(ps); - binstats[huge_idx].npageslabs += mul * 1; - binstats[huge_idx].nactive += mul * hpdata_nactive_get(ps); - binstats[huge_idx].ndirty += mul * hpdata_ndirty_get(ps); + stats->slabs[huge_idx].npageslabs += mul * 1; + stats->slabs[huge_idx].nactive += mul * nactive; + stats->slabs[huge_idx].ndirty += mul * ndirty; - psset->merged_stats.npageslabs += mul * 1; - psset->merged_stats.nactive += mul * hpdata_nactive_get(ps); - psset->merged_stats.ndirty += mul * hpdata_ndirty_get(ps); + binstats[huge_idx].npageslabs += mul * 1; + binstats[huge_idx].nactive += mul * nactive; + binstats[huge_idx].ndirty += mul * ndirty; if (config_debug) { - psset_bin_stats_t check_stats = {0}; - for (size_t huge = 0; huge <= 1; huge++) { - psset_bin_stats_accum(&check_stats, - &psset->stats.full_slabs[huge]); - psset_bin_stats_accum(&check_stats, - &psset->stats.empty_slabs[huge]); + psset_bin_stats_t check_stats[PSSET_NHUGE] = {{0}}; + for (int huge = 0; huge < PSSET_NHUGE; huge++) { + psset_bin_stats_accum( + &check_stats[huge], &stats->full_slabs[huge]); + psset_bin_stats_accum( + &check_stats[huge], &stats->empty_slabs[huge]); for (pszind_t pind = 0; pind < PSSET_NPSIZES; pind++) { - psset_bin_stats_accum(&check_stats, - &psset->stats.nonfull_slabs[pind][huge]); + psset_bin_stats_accum(&check_stats[huge], + &stats->nonfull_slabs[pind][huge]); } } - assert(psset->merged_stats.npageslabs - == check_stats.npageslabs); - assert(psset->merged_stats.nactive == check_stats.nactive); - assert(psset->merged_stats.ndirty == check_stats.ndirty); + + assert(stats->merged.npageslabs + == check_stats[0].npageslabs + check_stats[1].npageslabs); + assert(stats->merged.nactive + == check_stats[0].nactive + check_stats[1].nactive); + assert(stats->merged.ndirty + == check_stats[0].ndirty + check_stats[1].ndirty); + + for (int huge = 0; huge < PSSET_NHUGE; huge++) { + assert(stats->slabs[huge].npageslabs + == check_stats[huge].npageslabs); + assert(stats->slabs[huge].nactive + == check_stats[huge].nactive); + assert(stats->slabs[huge].ndirty + == check_stats[huge].ndirty); + } } } static void -psset_bin_stats_insert(psset_t *psset, psset_bin_stats_t *binstats, - hpdata_t *ps) { - psset_bin_stats_insert_remove(psset, binstats, ps, true); +psset_slab_stats_insert( + psset_stats_t *stats, psset_bin_stats_t *binstats, hpdata_t *ps) { + psset_slab_stats_insert_remove(stats, binstats, ps, true); } static void -psset_bin_stats_remove(psset_t *psset, psset_bin_stats_t *binstats, - hpdata_t *ps) { - psset_bin_stats_insert_remove(psset, binstats, ps, false); +psset_slab_stats_remove( + psset_stats_t *stats, psset_bin_stats_t *binstats, hpdata_t *ps) { + psset_slab_stats_insert_remove(stats, binstats, ps, false); +} + +static pszind_t +psset_hpdata_heap_index(const hpdata_t *ps) { + assert(!hpdata_full(ps)); + assert(!hpdata_empty(ps)); + size_t longest_free_range = hpdata_longest_free_range_get(ps); + pszind_t pind = sz_psz2ind( + sz_psz_quantize_floor(longest_free_range << LG_PAGE)); + assert(pind < PSSET_NPSIZES); + return pind; } static void -psset_hpdata_heap_remove(psset_t *psset, pszind_t pind, hpdata_t *ps) { +psset_hpdata_heap_remove(psset_t *psset, hpdata_t *ps) { + pszind_t pind = psset_hpdata_heap_index(ps); hpdata_age_heap_remove(&psset->pageslabs[pind], ps); if (hpdata_age_heap_empty(&psset->pageslabs[pind])) { fb_unset(psset->pageslab_bitmap, PSSET_NPSIZES, (size_t)pind); @@ -101,7 +144,8 @@ psset_hpdata_heap_remove(psset_t *psset, pszind_t pind, hpdata_t *ps) { } static void -psset_hpdata_heap_insert(psset_t *psset, pszind_t pind, hpdata_t *ps) { +psset_hpdata_heap_insert(psset_t *psset, hpdata_t *ps) { + pszind_t pind = psset_hpdata_heap_index(ps); if (hpdata_age_heap_empty(&psset->pageslabs[pind])) { fb_set(psset->pageslab_bitmap, PSSET_NPSIZES, (size_t)pind); } @@ -109,38 +153,30 @@ psset_hpdata_heap_insert(psset_t *psset, pszind_t pind, hpdata_t *ps) { } static void -psset_stats_insert(psset_t* psset, hpdata_t *ps) { +psset_stats_insert(psset_t *psset, hpdata_t *ps) { + psset_stats_t *stats = &psset->stats; if (hpdata_empty(ps)) { - psset_bin_stats_insert(psset, psset->stats.empty_slabs, ps); + psset_slab_stats_insert(stats, psset->stats.empty_slabs, ps); } else if (hpdata_full(ps)) { - psset_bin_stats_insert(psset, psset->stats.full_slabs, ps); + psset_slab_stats_insert(stats, psset->stats.full_slabs, ps); } else { - size_t longest_free_range = hpdata_longest_free_range_get(ps); - - pszind_t pind = sz_psz2ind(sz_psz_quantize_floor( - longest_free_range << LG_PAGE)); - assert(pind < PSSET_NPSIZES); - - psset_bin_stats_insert(psset, psset->stats.nonfull_slabs[pind], - ps); + pszind_t pind = psset_hpdata_heap_index(ps); + psset_slab_stats_insert( + stats, psset->stats.nonfull_slabs[pind], ps); } } static void psset_stats_remove(psset_t *psset, hpdata_t *ps) { + psset_stats_t *stats = &psset->stats; if (hpdata_empty(ps)) { - psset_bin_stats_remove(psset, psset->stats.empty_slabs, ps); + psset_slab_stats_remove(stats, psset->stats.empty_slabs, ps); } else if (hpdata_full(ps)) { - psset_bin_stats_remove(psset, psset->stats.full_slabs, ps); + psset_slab_stats_remove(stats, psset->stats.full_slabs, ps); } else { - size_t longest_free_range = hpdata_longest_free_range_get(ps); - - pszind_t pind = sz_psz2ind(sz_psz_quantize_floor( - longest_free_range << LG_PAGE)); - assert(pind < PSSET_NPSIZES); - - psset_bin_stats_remove(psset, psset->stats.nonfull_slabs[pind], - ps); + pszind_t pind = psset_hpdata_heap_index(ps); + psset_slab_stats_remove( + stats, psset->stats.nonfull_slabs[pind], ps); } } @@ -165,13 +201,7 @@ psset_alloc_container_insert(psset_t *psset, hpdata_t *ps) { * going to return them from a psset_pick_alloc call. */ } else { - size_t longest_free_range = hpdata_longest_free_range_get(ps); - - pszind_t pind = sz_psz2ind(sz_psz_quantize_floor( - longest_free_range << LG_PAGE)); - assert(pind < PSSET_NPSIZES); - - psset_hpdata_heap_insert(psset, pind, ps); + psset_hpdata_heap_insert(psset, ps); } } @@ -186,13 +216,7 @@ psset_alloc_container_remove(psset_t *psset, hpdata_t *ps) { } else if (hpdata_full(ps)) { /* Same as above -- do nothing in this case. */ } else { - size_t longest_free_range = hpdata_longest_free_range_get(ps); - - pszind_t pind = sz_psz2ind(sz_psz_quantize_floor( - longest_free_range << LG_PAGE)); - assert(pind < PSSET_NPSIZES); - - psset_hpdata_heap_remove(psset, pind, ps); + psset_hpdata_heap_remove(psset, ps); } } @@ -240,7 +264,7 @@ psset_maybe_remove_purge_list(psset_t *psset, hpdata_t *ps) { * purge LRU within a given dirtiness bucket. */ if (hpdata_purge_allowed_get(ps)) { - size_t ind = psset_purge_list_ind(ps); + size_t ind = psset_purge_list_ind(ps); hpdata_purge_list_t *purge_list = &psset->to_purge[ind]; hpdata_purge_list_remove(purge_list, ps); if (hpdata_purge_list_empty(purge_list)) { @@ -252,14 +276,13 @@ psset_maybe_remove_purge_list(psset_t *psset, hpdata_t *ps) { static void psset_maybe_insert_purge_list(psset_t *psset, hpdata_t *ps) { if (hpdata_purge_allowed_get(ps)) { - size_t ind = psset_purge_list_ind(ps); + size_t ind = psset_purge_list_ind(ps); hpdata_purge_list_t *purge_list = &psset->to_purge[ind]; if (hpdata_purge_list_empty(purge_list)) { fb_set(psset->purge_bitmap, PSSET_NPURGE_LISTS, ind); } hpdata_purge_list_append(purge_list, ps); } - } void @@ -313,18 +336,50 @@ psset_update_end(psset_t *psset, hpdata_t *ps) { hpdata_assert_consistent(ps); } +static hpdata_t * +psset_enumerate_search(psset_t *psset, pszind_t pind, size_t size) { + if (hpdata_age_heap_empty(&psset->pageslabs[pind])) { + return NULL; + } + + hpdata_t *ps = NULL; + hpdata_age_heap_enumerate_helper_t helper; + hpdata_age_heap_enumerate_prepare(&psset->pageslabs[pind], &helper, + PSSET_ENUMERATE_MAX_NUM, sizeof(helper.bfs_queue) / sizeof(void *)); + + while ((ps = hpdata_age_heap_enumerate_next( + &psset->pageslabs[pind], &helper))) { + if (hpdata_longest_free_range_get(ps) >= size) { + return ps; + } + } + + return NULL; +} + hpdata_t * psset_pick_alloc(psset_t *psset, size_t size) { assert((size & PAGE_MASK) == 0); assert(size <= HUGEPAGE); - pszind_t min_pind = sz_psz2ind(sz_psz_quantize_ceil(size)); - pszind_t pind = (pszind_t)fb_ffs(psset->pageslab_bitmap, PSSET_NPSIZES, - (size_t)min_pind); + pszind_t min_pind = sz_psz2ind(sz_psz_quantize_ceil(size)); + hpdata_t *ps = NULL; + + /* See comments in eset_first_fit for why we enumerate search below. */ + pszind_t pind_prev = sz_psz2ind(sz_psz_quantize_floor(size)); + if (sz_large_size_classes_disabled() && pind_prev < min_pind) { + ps = psset_enumerate_search(psset, pind_prev, size); + if (ps != NULL) { + return ps; + } + } + + pszind_t pind = (pszind_t)fb_ffs( + psset->pageslab_bitmap, PSSET_NPSIZES, (size_t)min_pind); if (pind == PSSET_NPSIZES) { return hpdata_empty_list_first(&psset->empty); } - hpdata_t *ps = hpdata_age_heap_first(&psset->pageslabs[pind]); + ps = hpdata_age_heap_first(&psset->pageslabs[pind]); if (ps == NULL) { return NULL; } @@ -335,17 +390,40 @@ psset_pick_alloc(psset_t *psset, size_t size) { } hpdata_t * -psset_pick_purge(psset_t *psset) { - ssize_t ind_ssz = fb_fls(psset->purge_bitmap, PSSET_NPURGE_LISTS, - PSSET_NPURGE_LISTS - 1); - if (ind_ssz < 0) { - return NULL; +psset_pick_purge(psset_t *psset, const nstime_t *now) { + size_t max_bit = PSSET_NPURGE_LISTS - 1; + while (1) { + ssize_t ind_ssz = fb_fls( + psset->purge_bitmap, PSSET_NPURGE_LISTS, max_bit); + if (ind_ssz < 0) { + break; + } + pszind_t ind = (pszind_t)ind_ssz; + assert(ind < PSSET_NPURGE_LISTS); + hpdata_t *ps = hpdata_purge_list_first(&psset->to_purge[ind]); + assert(ps != NULL); + if (now == NULL) { + return ps; + } + /* + * We only check the first page (it had least recent hpa_alloc + * or hpa_dalloc). It is possible that some page in the list + * would meet the time, but we only guarantee the min delay. If + * we want to get the one that changed the state to purgable + * the earliest, we would change the list into a heap ordered by + * time. We will use benchmark to make a decision. + */ + const nstime_t *tm_allowed = hpdata_time_purge_allowed_get(ps); + if (nstime_compare(tm_allowed, now) <= 0) { + return ps; + } + if (ind == 0) { + break; + } + max_bit = ind - 1; } - pszind_t ind = (pszind_t)ind_ssz; - assert(ind < PSSET_NPURGE_LISTS); - hpdata_t *ps = hpdata_purge_list_first(&psset->to_purge[ind]); - assert(ps != NULL); - return ps; + /* No page is ready yet */ + return NULL; } hpdata_t * diff --git a/dep/jemalloc/src/rtree.c b/dep/jemalloc/src/rtree.c index 6496b5afd..ac27f8299 100644 --- a/dep/jemalloc/src/rtree.c +++ b/dep/jemalloc/src/rtree.c @@ -20,7 +20,7 @@ rtree_new(rtree_t *rtree, base_t *base, bool zeroed) { rtree->base = base; if (malloc_mutex_init(&rtree->init_lock, "rtree", WITNESS_RANK_RTREE, - malloc_mutex_rank_exclusive)) { + malloc_mutex_rank_exclusive)) { return true; } @@ -29,19 +29,19 @@ rtree_new(rtree_t *rtree, base_t *base, bool zeroed) { static rtree_node_elm_t * rtree_node_alloc(tsdn_t *tsdn, rtree_t *rtree, size_t nelms) { - return (rtree_node_elm_t *)base_alloc(tsdn, rtree->base, - nelms * sizeof(rtree_node_elm_t), CACHELINE); + return (rtree_node_elm_t *)base_alloc_rtree( + tsdn, rtree->base, nelms * sizeof(rtree_node_elm_t)); } static rtree_leaf_elm_t * rtree_leaf_alloc(tsdn_t *tsdn, rtree_t *rtree, size_t nelms) { - return (rtree_leaf_elm_t *)base_alloc(tsdn, rtree->base, - nelms * sizeof(rtree_leaf_elm_t), CACHELINE); + return (rtree_leaf_elm_t *)base_alloc_rtree( + tsdn, rtree->base, nelms * sizeof(rtree_leaf_elm_t)); } static rtree_node_elm_t * -rtree_node_init(tsdn_t *tsdn, rtree_t *rtree, unsigned level, - atomic_p_t *elmp) { +rtree_node_init( + tsdn_t *tsdn, rtree_t *rtree, unsigned level, atomic_p_t *elmp) { malloc_mutex_lock(tsdn, &rtree->init_lock); /* * If *elmp is non-null, then it was initialized with the init lock @@ -49,8 +49,8 @@ rtree_node_init(tsdn_t *tsdn, rtree_t *rtree, unsigned level, */ rtree_node_elm_t *node = atomic_load_p(elmp, ATOMIC_RELAXED); if (node == NULL) { - node = rtree_node_alloc(tsdn, rtree, ZU(1) << - rtree_levels[level].bits); + node = rtree_node_alloc( + tsdn, rtree, ZU(1) << rtree_levels[level].bits); if (node == NULL) { malloc_mutex_unlock(tsdn, &rtree->init_lock); return NULL; @@ -75,8 +75,8 @@ rtree_leaf_init(tsdn_t *tsdn, rtree_t *rtree, atomic_p_t *elmp) { */ rtree_leaf_elm_t *leaf = atomic_load_p(elmp, ATOMIC_RELAXED); if (leaf == NULL) { - leaf = rtree_leaf_alloc(tsdn, rtree, ZU(1) << - rtree_levels[RTREE_HEIGHT-1].bits); + leaf = rtree_leaf_alloc( + tsdn, rtree, ZU(1) << rtree_levels[RTREE_HEIGHT - 1].bits); if (leaf == NULL) { malloc_mutex_unlock(tsdn, &rtree->init_lock); return NULL; @@ -107,11 +107,11 @@ rtree_child_node_tryread(rtree_node_elm_t *elm, bool dependent) { rtree_node_elm_t *node; if (dependent) { - node = (rtree_node_elm_t *)atomic_load_p(&elm->child, - ATOMIC_RELAXED); + node = (rtree_node_elm_t *)atomic_load_p( + &elm->child, ATOMIC_RELAXED); } else { - node = (rtree_node_elm_t *)atomic_load_p(&elm->child, - ATOMIC_ACQUIRE); + node = (rtree_node_elm_t *)atomic_load_p( + &elm->child, ATOMIC_ACQUIRE); } assert(!dependent || node != NULL); @@ -136,11 +136,11 @@ rtree_child_leaf_tryread(rtree_node_elm_t *elm, bool dependent) { rtree_leaf_elm_t *leaf; if (dependent) { - leaf = (rtree_leaf_elm_t *)atomic_load_p(&elm->child, - ATOMIC_RELAXED); + leaf = (rtree_leaf_elm_t *)atomic_load_p( + &elm->child, ATOMIC_RELAXED); } else { - leaf = (rtree_leaf_elm_t *)atomic_load_p(&elm->child, - ATOMIC_ACQUIRE); + leaf = (rtree_leaf_elm_t *)atomic_load_p( + &elm->child, ATOMIC_ACQUIRE); } assert(!dependent || leaf != NULL); @@ -181,53 +181,54 @@ rtree_leaf_elm_lookup_hard(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, } } -#define RTREE_GET_CHILD(level) { \ - assert(level < RTREE_HEIGHT-1); \ - if (level != 0 && !dependent && \ - unlikely(!rtree_node_valid(node))) { \ - return NULL; \ - } \ - uintptr_t subkey = rtree_subkey(key, level); \ - if (level + 2 < RTREE_HEIGHT) { \ - node = init_missing ? \ - rtree_child_node_read(tsdn, rtree, \ - &node[subkey], level, dependent) : \ - rtree_child_node_tryread(&node[subkey], \ - dependent); \ - } else { \ - leaf = init_missing ? \ - rtree_child_leaf_read(tsdn, rtree, \ - &node[subkey], level, dependent) : \ - rtree_child_leaf_tryread(&node[subkey], \ - dependent); \ - } \ +#define RTREE_GET_CHILD(level) \ + { \ + assert(level < RTREE_HEIGHT - 1); \ + if (level != 0 && !dependent \ + && unlikely(!rtree_node_valid(node))) { \ + return NULL; \ + } \ + uintptr_t subkey = rtree_subkey(key, level); \ + if (level + 2 < RTREE_HEIGHT) { \ + node = init_missing \ + ? rtree_child_node_read(tsdn, rtree, \ + &node[subkey], level, dependent) \ + : rtree_child_node_tryread( \ + &node[subkey], dependent); \ + } else { \ + leaf = init_missing \ + ? rtree_child_leaf_read(tsdn, rtree, \ + &node[subkey], level, dependent) \ + : rtree_child_leaf_tryread( \ + &node[subkey], dependent); \ + } \ } /* * Cache replacement upon hard lookup (i.e. L1 & L2 rtree cache miss): * (1) evict last entry in L2 cache; (2) move the collision slot from L1 * cache down to L2; and 3) fill L1. */ -#define RTREE_GET_LEAF(level) { \ - assert(level == RTREE_HEIGHT-1); \ - if (!dependent && unlikely(!rtree_leaf_valid(leaf))) { \ - return NULL; \ - } \ - if (RTREE_CTX_NCACHE_L2 > 1) { \ - memmove(&rtree_ctx->l2_cache[1], \ - &rtree_ctx->l2_cache[0], \ - sizeof(rtree_ctx_cache_elm_t) * \ - (RTREE_CTX_NCACHE_L2 - 1)); \ - } \ - size_t slot = rtree_cache_direct_map(key); \ - rtree_ctx->l2_cache[0].leafkey = \ - rtree_ctx->cache[slot].leafkey; \ - rtree_ctx->l2_cache[0].leaf = \ - rtree_ctx->cache[slot].leaf; \ - uintptr_t leafkey = rtree_leafkey(key); \ - rtree_ctx->cache[slot].leafkey = leafkey; \ - rtree_ctx->cache[slot].leaf = leaf; \ - uintptr_t subkey = rtree_subkey(key, level); \ - return &leaf[subkey]; \ +#define RTREE_GET_LEAF(level) \ + { \ + assert(level == RTREE_HEIGHT - 1); \ + if (!dependent && unlikely(!rtree_leaf_valid(leaf))) { \ + return NULL; \ + } \ + if (RTREE_CTX_NCACHE_L2 > 1) { \ + memmove(&rtree_ctx->l2_cache[1], \ + &rtree_ctx->l2_cache[0], \ + sizeof(rtree_ctx_cache_elm_t) \ + * (RTREE_CTX_NCACHE_L2 - 1)); \ + } \ + size_t slot = rtree_cache_direct_map(key); \ + rtree_ctx->l2_cache[0].leafkey = \ + rtree_ctx->cache[slot].leafkey; \ + rtree_ctx->l2_cache[0].leaf = rtree_ctx->cache[slot].leaf; \ + uintptr_t leafkey = rtree_leafkey(key); \ + rtree_ctx->cache[slot].leafkey = leafkey; \ + rtree_ctx->cache[slot].leaf = leaf; \ + uintptr_t subkey = rtree_subkey(key, level); \ + return &leaf[subkey]; \ } if (RTREE_HEIGHT > 1) { RTREE_GET_CHILD(0) @@ -236,11 +237,11 @@ rtree_leaf_elm_lookup_hard(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, RTREE_GET_CHILD(1) } if (RTREE_HEIGHT > 3) { - for (unsigned i = 2; i < RTREE_HEIGHT-1; i++) { + for (unsigned i = 2; i < RTREE_HEIGHT - 1; i++) { RTREE_GET_CHILD(i) } } - RTREE_GET_LEAF(RTREE_HEIGHT-1) + RTREE_GET_LEAF(RTREE_HEIGHT - 1) #undef RTREE_GET_CHILD #undef RTREE_GET_LEAF not_reached(); diff --git a/dep/jemalloc/src/safety_check.c b/dep/jemalloc/src/safety_check.c index 209fdda92..d052718da 100644 --- a/dep/jemalloc/src/safety_check.c +++ b/dep/jemalloc/src/safety_check.c @@ -3,30 +3,34 @@ static safety_check_abort_hook_t safety_check_abort; -void safety_check_fail_sized_dealloc(bool current_dealloc, const void *ptr, +void +safety_check_fail_sized_dealloc(bool current_dealloc, const void *ptr, size_t true_size, size_t input_size) { - char *src = current_dealloc ? "the current pointer being freed" : - "in thread cache, possibly from previous deallocations"; + char *src = current_dealloc + ? "the current pointer being freed" + : "in thread cache, possibly from previous deallocations"; + char *suggest_debug_build = config_debug ? "" : " --enable-debug or"; - safety_check_fail(": size mismatch detected (true size %zu " + safety_check_fail( + ": size mismatch detected (true size %zu " "vs input size %zu), likely caused by application sized " - "deallocation bugs (source address: %p, %s). Suggest building with " - "--enable-debug or address sanitizer for debugging. Abort.\n", - true_size, input_size, ptr, src); + "deallocation bugs (source address: %p, %s). Suggest building with" + "%s address sanitizer for debugging. Abort.\n", + true_size, input_size, ptr, src, suggest_debug_build); } -void safety_check_set_abort(safety_check_abort_hook_t abort_fn) { +void +safety_check_set_abort(safety_check_abort_hook_t abort_fn) { safety_check_abort = abort_fn; } -void safety_check_fail(const char *format, ...) { - char buf[MALLOC_PRINTF_BUFSIZE]; - - va_list ap; - va_start(ap, format); - malloc_vsnprintf(buf, MALLOC_PRINTF_BUFSIZE, format, ap); - va_end(ap); - +/* + * In addition to malloc_write, also embed hint msg in the abort function name + * because there are cases only logging crash stack traces. + */ +static void +safety_check_detected_heap_corruption___run_address_sanitizer_build_to_debug( + const char *buf) { if (safety_check_abort == NULL) { malloc_write(buf); abort(); @@ -34,3 +38,16 @@ void safety_check_fail(const char *format, ...) { safety_check_abort(buf); } } + +void +safety_check_fail(const char *format, ...) { + char buf[MALLOC_PRINTF_BUFSIZE]; + + va_list ap; + va_start(ap, format); + malloc_vsnprintf(buf, MALLOC_PRINTF_BUFSIZE, format, ap); + va_end(ap); + + safety_check_detected_heap_corruption___run_address_sanitizer_build_to_debug( + buf); +} diff --git a/dep/jemalloc/src/san.c b/dep/jemalloc/src/san.c index 6e5129113..5448c67fb 100644 --- a/dep/jemalloc/src/san.c +++ b/dep/jemalloc/src/san.c @@ -20,43 +20,43 @@ ssize_t opt_lg_san_uaf_align = SAN_LG_UAF_ALIGN_DEFAULT; uintptr_t san_cache_bin_nonfast_mask = SAN_CACHE_BIN_NONFAST_MASK_DEFAULT; static inline void -san_find_guarded_addr(edata_t *edata, uintptr_t *guard1, uintptr_t *guard2, - uintptr_t *addr, size_t size, bool left, bool right) { +san_find_guarded_addr(edata_t *edata, void **guard1, void **guard2, void **addr, + size_t size, bool left, bool right) { assert(!edata_guarded_get(edata)); assert(size % PAGE == 0); - *addr = (uintptr_t)edata_base_get(edata); + *addr = edata_base_get(edata); if (left) { *guard1 = *addr; - *addr += SAN_PAGE_GUARD; + *addr = ((byte_t *)*addr) + SAN_PAGE_GUARD; } else { - *guard1 = 0; + *guard1 = NULL; } if (right) { - *guard2 = *addr + size; + *guard2 = ((byte_t *)*addr) + size; } else { - *guard2 = 0; + *guard2 = NULL; } } static inline void -san_find_unguarded_addr(edata_t *edata, uintptr_t *guard1, uintptr_t *guard2, - uintptr_t *addr, size_t size, bool left, bool right) { +san_find_unguarded_addr(edata_t *edata, void **guard1, void **guard2, + void **addr, size_t size, bool left, bool right) { assert(edata_guarded_get(edata)); assert(size % PAGE == 0); - *addr = (uintptr_t)edata_base_get(edata); + *addr = edata_base_get(edata); if (right) { - *guard2 = *addr + size; + *guard2 = ((byte_t *)*addr) + size; } else { - *guard2 = 0; + *guard2 = NULL; } if (left) { - *guard1 = *addr - SAN_PAGE_GUARD; - assert(*guard1 != 0); + *guard1 = ((byte_t *)*addr) - SAN_PAGE_GUARD; + assert(*guard1 != NULL); *addr = *guard1; } else { - *guard1 = 0; + *guard1 = NULL; } } @@ -73,16 +73,16 @@ san_guard_pages(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, emap_t *emap, ? san_two_side_unguarded_sz(size_with_guards) : san_one_side_unguarded_sz(size_with_guards); - uintptr_t guard1, guard2, addr; - san_find_guarded_addr(edata, &guard1, &guard2, &addr, usize, left, - right); + void *guard1, *guard2, *addr; + san_find_guarded_addr( + edata, &guard1, &guard2, &addr, usize, left, right); assert(edata_state_get(edata) == extent_state_active); - ehooks_guard(tsdn, ehooks, (void *)guard1, (void *)guard2); + ehooks_guard(tsdn, ehooks, guard1, guard2); /* Update the guarded addr and usable size of the edata. */ edata_size_set(edata, usize); - edata_addr_set(edata, (void *)addr); + edata_addr_set(edata, addr); edata_guarded_set(edata, true); if (remap) { @@ -108,9 +108,9 @@ san_unguard_pages_impl(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, ? san_two_side_guarded_sz(size) : san_one_side_guarded_sz(size); - uintptr_t guard1, guard2, addr; - san_find_unguarded_addr(edata, &guard1, &guard2, &addr, size, left, - right); + void *guard1, *guard2, *addr; + san_find_unguarded_addr( + edata, &guard1, &guard2, &addr, size, left, right); ehooks_unguard(tsdn, ehooks, (void *)guard1, (void *)guard2); @@ -130,15 +130,15 @@ san_unguard_pages_impl(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, } void -san_unguard_pages(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, - emap_t *emap, bool left, bool right) { +san_unguard_pages(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, emap_t *emap, + bool left, bool right) { san_unguard_pages_impl(tsdn, ehooks, edata, emap, left, right, /* remap */ true); } void -san_unguard_pages_pre_destroy(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, - emap_t *emap) { +san_unguard_pages_pre_destroy( + tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, emap_t *emap) { emap_assert_not_mapped(tsdn, emap, edata); /* * We don't want to touch the emap of about to be destroyed extents, as @@ -146,7 +146,7 @@ san_unguard_pages_pre_destroy(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, * we unguard the extents to the right, because retained extents only * own their right guard page per san_bump_alloc's logic. */ - san_unguard_pages_impl(tsdn, ehooks, edata, emap, /* left */ false, + san_unguard_pages_impl(tsdn, ehooks, edata, emap, /* left */ false, /* right */ true, /* remap */ false); } @@ -163,9 +163,9 @@ san_stashed_corrupted(void *ptr, size_t size) { void *first, *mid, *last; san_junk_ptr_locations(ptr, size, &first, &mid, &last); - if (*(uintptr_t *)first != uaf_detect_junk || - *(uintptr_t *)mid != uaf_detect_junk || - *(uintptr_t *)last != uaf_detect_junk) { + if (*(uintptr_t *)first != uaf_detect_junk + || *(uintptr_t *)mid != uaf_detect_junk + || *(uintptr_t *)last != uaf_detect_junk) { return true; } @@ -183,7 +183,8 @@ san_check_stashed_ptrs(void **ptrs, size_t nstashed, size_t usize) { assert(stashed != NULL); assert(cache_bin_nonfast_aligned(stashed)); if (unlikely(san_stashed_corrupted(stashed, usize))) { - safety_check_fail(": Write-after-free " + safety_check_fail( + ": Write-after-free " "detected on deallocated pointer %p (size %zu).\n", stashed, usize); } diff --git a/dep/jemalloc/src/san_bump.c b/dep/jemalloc/src/san_bump.c index 888974555..110312902 100644 --- a/dep/jemalloc/src/san_bump.c +++ b/dep/jemalloc/src/san_bump.c @@ -7,31 +7,31 @@ #include "jemalloc/internal/ehooks.h" #include "jemalloc/internal/edata_cache.h" -static bool -san_bump_grow_locked(tsdn_t *tsdn, san_bump_alloc_t *sba, pac_t *pac, - ehooks_t *ehooks, size_t size); +static bool san_bump_grow_locked(tsdn_t *tsdn, san_bump_alloc_t *sba, + pac_t *pac, ehooks_t *ehooks, size_t size); edata_t * -san_bump_alloc(tsdn_t *tsdn, san_bump_alloc_t* sba, pac_t *pac, +san_bump_alloc(tsdn_t *tsdn, san_bump_alloc_t *sba, pac_t *pac, ehooks_t *ehooks, size_t size, bool zero) { assert(san_bump_enabled()); - edata_t* to_destroy; - size_t guarded_size = san_one_side_guarded_sz(size); + edata_t *to_destroy; + size_t guarded_size = san_one_side_guarded_sz(size); malloc_mutex_lock(tsdn, &sba->mtx); - if (sba->curr_reg == NULL || - edata_size_get(sba->curr_reg) < guarded_size) { + if (sba->curr_reg == NULL + || edata_size_get(sba->curr_reg) < guarded_size) { /* * If the current region can't accommodate the allocation, * try replacing it with a larger one and destroy current if the * replacement succeeds. */ to_destroy = sba->curr_reg; - bool err = san_bump_grow_locked(tsdn, sba, pac, ehooks, - guarded_size); + bool err = san_bump_grow_locked( + tsdn, sba, pac, ehooks, guarded_size); if (err) { + sba->curr_reg = to_destroy; goto label_err; } } else { @@ -40,9 +40,9 @@ san_bump_alloc(tsdn_t *tsdn, san_bump_alloc_t* sba, pac_t *pac, assert(guarded_size <= edata_size_get(sba->curr_reg)); size_t trail_size = edata_size_get(sba->curr_reg) - guarded_size; - edata_t* edata; + edata_t *edata; if (trail_size != 0) { - edata_t* curr_reg_trail = extent_split_wrapper(tsdn, pac, + edata_t *curr_reg_trail = extent_split_wrapper(tsdn, pac, ehooks, sba->curr_reg, guarded_size, trail_size, /* holding_core_locks */ true); if (curr_reg_trail == NULL) { @@ -69,9 +69,8 @@ san_bump_alloc(tsdn_t *tsdn, san_bump_alloc_t* sba, pac_t *pac, /* right */ true, /* remap */ true); if (extent_commit_zero(tsdn, ehooks, edata, /* commit */ true, zero, - /* growing_retained */ false)) { - extent_record(tsdn, pac, ehooks, &pac->ecache_retained, - edata); + /* growing_retained */ false)) { + extent_record(tsdn, pac, ehooks, &pac->ecache_retained, edata); return NULL; } @@ -90,9 +89,10 @@ san_bump_grow_locked(tsdn_t *tsdn, san_bump_alloc_t *sba, pac_t *pac, ehooks_t *ehooks, size_t size) { malloc_mutex_assert_owner(tsdn, &sba->mtx); - bool committed = false, zeroed = false; - size_t alloc_size = size > SBA_RETAINED_ALLOC_SIZE ? size : - SBA_RETAINED_ALLOC_SIZE; + bool committed = false, zeroed = false; + size_t alloc_size = size > SBA_RETAINED_ALLOC_SIZE + ? size + : SBA_RETAINED_ALLOC_SIZE; assert((alloc_size & PAGE_MASK) == 0); sba->curr_reg = extent_alloc_wrapper(tsdn, pac, ehooks, NULL, alloc_size, PAGE, zeroed, &committed, diff --git a/dep/jemalloc/src/sc.c b/dep/jemalloc/src/sc.c index e4a94d89f..014ab95d8 100644 --- a/dep/jemalloc/src/sc.c +++ b/dep/jemalloc/src/sc.c @@ -27,7 +27,7 @@ slab_size(int lg_page, int lg_base, int lg_delta, int ndelta) { size_t try_slab_size = page; size_t try_nregs = try_slab_size / reg_size; size_t perfect_slab_size = 0; - bool perfect = false; + bool perfect = false; /* * This loop continues until we find the least common multiple of the * page size and size class size. Size classes are all of the form @@ -106,7 +106,7 @@ size_classes( /* Outputs that we update as we go. */ size_t lookup_maxclass = 0; size_t small_maxclass = 0; - int lg_large_minclass = 0; + int lg_large_minclass = 0; size_t large_maxclass = 0; /* Tiny size classes. */ @@ -209,7 +209,7 @@ size_classes( lg_delta++; } /* Additional outputs. */ - int nsizes = index; + int nsizes = index; unsigned lg_ceil_nsizes = lg_ceil(nsizes); /* Fill in the output data. */ @@ -292,8 +292,8 @@ sc_data_update_slab_size(sc_data_t *data, size_t begin, size_t end, int pgs) { if (!sc->bin) { break; } - size_t reg_size = reg_size_compute(sc->lg_base, sc->lg_delta, - sc->ndelta); + size_t reg_size = reg_size_compute( + sc->lg_base, sc->lg_delta, sc->ndelta); if (begin <= reg_size && reg_size <= end) { sc_data_update_sc_slab_size(sc, reg_size, pgs); } diff --git a/dep/jemalloc/src/sec.c b/dep/jemalloc/src/sec.c index df6755904..a32545370 100644 --- a/dep/jemalloc/src/sec.c +++ b/dep/jemalloc/src/sec.c @@ -2,90 +2,58 @@ #include "jemalloc/internal/jemalloc_internal_includes.h" #include "jemalloc/internal/sec.h" +#include "jemalloc/internal/jemalloc_probe.h" -static edata_t *sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, - size_t alignment, bool zero, bool guarded, bool frequent_reuse, - bool *deferred_work_generated); -static bool sec_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, - size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated); -static bool sec_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, - size_t old_size, size_t new_size, bool *deferred_work_generated); -static void sec_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata, - bool *deferred_work_generated); - -static void +static bool sec_bin_init(sec_bin_t *bin) { - bin->being_batch_filled = false; bin->bytes_cur = 0; + sec_bin_stats_init(&bin->stats); edata_list_active_init(&bin->freelist); -} - -bool -sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, pai_t *fallback, - const sec_opts_t *opts) { - assert(opts->max_alloc >= PAGE); - - size_t max_alloc = PAGE_FLOOR(opts->max_alloc); - pszind_t npsizes = sz_psz2ind(max_alloc) + 1; - - size_t sz_shards = opts->nshards * sizeof(sec_shard_t); - size_t sz_bins = opts->nshards * (size_t)npsizes * sizeof(sec_bin_t); - size_t sz_alloc = sz_shards + sz_bins; - void *dynalloc = base_alloc(tsdn, base, sz_alloc, CACHELINE); - if (dynalloc == NULL) { + bool err = malloc_mutex_init(&bin->mtx, "sec_bin", WITNESS_RANK_SEC_BIN, + malloc_mutex_rank_exclusive); + if (err) { return true; } - sec_shard_t *shard_cur = (sec_shard_t *)dynalloc; - sec->shards = shard_cur; - sec_bin_t *bin_cur = (sec_bin_t *)&shard_cur[opts->nshards]; - /* Just for asserts, below. */ - sec_bin_t *bin_start = bin_cur; - - for (size_t i = 0; i < opts->nshards; i++) { - sec_shard_t *shard = shard_cur; - shard_cur++; - bool err = malloc_mutex_init(&shard->mtx, "sec_shard", - WITNESS_RANK_SEC_SHARD, malloc_mutex_rank_exclusive); - if (err) { - return true; - } - shard->enabled = true; - shard->bins = bin_cur; - for (pszind_t j = 0; j < npsizes; j++) { - sec_bin_init(&shard->bins[j]); - bin_cur++; - } - shard->bytes_cur = 0; - shard->to_flush_next = 0; - } - /* - * Should have exactly matched the bin_start to the first unused byte - * after the shards. - */ - assert((void *)shard_cur == (void *)bin_start); - /* And the last bin to use up the last bytes of the allocation. */ - assert((char *)bin_cur == ((char *)dynalloc + sz_alloc)); - sec->fallback = fallback; - - - sec->opts = *opts; - sec->npsizes = npsizes; - - /* - * Initialize these last so that an improper use of an SEC whose - * initialization failed will segfault in an easy-to-spot way. - */ - sec->pai.alloc = &sec_alloc; - sec->pai.alloc_batch = &pai_alloc_batch_default; - sec->pai.expand = &sec_expand; - sec->pai.shrink = &sec_shrink; - sec->pai.dalloc = &sec_dalloc; - sec->pai.dalloc_batch = &pai_dalloc_batch_default; return false; } -static sec_shard_t * +bool +sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts) { + sec->opts = *opts; + if (opts->nshards == 0) { + return false; + } + assert(opts->max_alloc >= PAGE); + + /* + * Same as tcache, sec do not cache allocs/dallocs larger than + * USIZE_GROW_SLOW_THRESHOLD because the usize above this increases + * by PAGE and the number of usizes is too large. + */ + assert(opts->max_alloc <= USIZE_GROW_SLOW_THRESHOLD); + + size_t max_alloc = PAGE_FLOOR(opts->max_alloc); + pszind_t npsizes = sz_psz2ind(max_alloc) + 1; + + size_t ntotal_bins = opts->nshards * (size_t)npsizes; + size_t sz_bins = sizeof(sec_bin_t) * ntotal_bins; + void *dynalloc = base_alloc(tsdn, base, sz_bins, CACHELINE); + if (dynalloc == NULL) { + return true; + } + sec->bins = (sec_bin_t *)dynalloc; + for (pszind_t j = 0; j < ntotal_bins; j++) { + if (sec_bin_init(&sec->bins[j])) { + return true; + } + } + sec->npsizes = npsizes; + + return false; +} + +static uint8_t sec_shard_pick(tsdn_t *tsdn, sec_t *sec) { /* * Eventually, we should implement affinity, tracking source shard using @@ -93,9 +61,9 @@ sec_shard_pick(tsdn_t *tsdn, sec_t *sec) { * distribute across all shards. */ if (tsdn_null(tsdn)) { - return &sec->shards[0]; + return 0; } - tsd_t *tsd = tsdn_tsd(tsdn); + tsd_t *tsd = tsdn_tsd(tsdn); uint8_t *idxp = tsd_sec_shardp_get(tsd); if (*idxp == (uint8_t)-1) { /* @@ -104,319 +72,314 @@ sec_shard_pick(tsdn_t *tsdn, sec_t *sec) { * number to store 32 bits, since we'll deliberately overflow * when we multiply by the number of shards. */ - uint64_t rand32 = prng_lg_range_u64(tsd_prng_statep_get(tsd), 32); - uint32_t idx = - (uint32_t)((rand32 * (uint64_t)sec->opts.nshards) >> 32); + uint64_t rand32 = prng_lg_range_u64( + tsd_prng_statep_get(tsd), 32); + uint32_t idx = (uint32_t)((rand32 * (uint64_t)sec->opts.nshards) + >> 32); assert(idx < (uint32_t)sec->opts.nshards); *idxp = (uint8_t)idx; } - return &sec->shards[*idxp]; + return *idxp; } -/* - * Perhaps surprisingly, this can be called on the alloc pathways; if we hit an - * empty cache, we'll try to fill it, which can push the shard over it's limit. - */ -static void -sec_flush_some_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard) { - malloc_mutex_assert_owner(tsdn, &shard->mtx); - edata_list_active_t to_flush; - edata_list_active_init(&to_flush); - while (shard->bytes_cur > sec->opts.bytes_after_flush) { - /* Pick a victim. */ - sec_bin_t *bin = &shard->bins[shard->to_flush_next]; - - /* Update our victim-picking state. */ - shard->to_flush_next++; - if (shard->to_flush_next == sec->npsizes) { - shard->to_flush_next = 0; - } - - assert(shard->bytes_cur >= bin->bytes_cur); - if (bin->bytes_cur != 0) { - shard->bytes_cur -= bin->bytes_cur; - bin->bytes_cur = 0; - edata_list_active_concat(&to_flush, &bin->freelist); - } - /* - * Either bin->bytes_cur was 0, in which case we didn't touch - * the bin list but it should be empty anyways (or else we - * missed a bytes_cur update on a list modification), or it - * *was* 0 and we emptied it ourselves. Either way, it should - * be empty now. - */ - assert(edata_list_active_empty(&bin->freelist)); - } - - malloc_mutex_unlock(tsdn, &shard->mtx); - bool deferred_work_generated = false; - pai_dalloc_batch(tsdn, sec->fallback, &to_flush, - &deferred_work_generated); +static sec_bin_t * +sec_bin_pick(sec_t *sec, uint8_t shard, pszind_t pszind) { + assert(shard < sec->opts.nshards); + size_t ind = (size_t)shard * sec->npsizes + pszind; + assert(ind < sec->npsizes * sec->opts.nshards); + return &sec->bins[ind]; } static edata_t * -sec_shard_alloc_locked(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, - sec_bin_t *bin) { - malloc_mutex_assert_owner(tsdn, &shard->mtx); - if (!shard->enabled) { - return NULL; - } +sec_bin_alloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size) { + malloc_mutex_assert_owner(tsdn, &bin->mtx); + edata_t *edata = edata_list_active_first(&bin->freelist); if (edata != NULL) { + assert(!edata_list_active_empty(&bin->freelist)); edata_list_active_remove(&bin->freelist, edata); - assert(edata_size_get(edata) <= bin->bytes_cur); - bin->bytes_cur -= edata_size_get(edata); - assert(edata_size_get(edata) <= shard->bytes_cur); - shard->bytes_cur -= edata_size_get(edata); + size_t sz = edata_size_get(edata); + assert(sz <= bin->bytes_cur && sz > 0); + bin->bytes_cur -= sz; + bin->stats.nhits++; } return edata; } static edata_t * -sec_batch_fill_and_alloc(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, - sec_bin_t *bin, size_t size) { - malloc_mutex_assert_not_owner(tsdn, &shard->mtx); +sec_multishard_trylock_alloc( + tsdn_t *tsdn, sec_t *sec, size_t size, pszind_t pszind) { + assert(sec->opts.nshards > 0); - edata_list_active_t result; - edata_list_active_init(&result); - bool deferred_work_generated = false; - size_t nalloc = pai_alloc_batch(tsdn, sec->fallback, size, - 1 + sec->opts.batch_fill_extra, &result, &deferred_work_generated); - - edata_t *ret = edata_list_active_first(&result); - if (ret != NULL) { - edata_list_active_remove(&result, ret); + uint8_t cur_shard = sec_shard_pick(tsdn, sec); + sec_bin_t *bin; + for (size_t i = 0; i < sec->opts.nshards; ++i) { + bin = sec_bin_pick(sec, cur_shard, pszind); + if (!malloc_mutex_trylock(tsdn, &bin->mtx)) { + edata_t *edata = sec_bin_alloc_locked( + tsdn, sec, bin, size); + malloc_mutex_unlock(tsdn, &bin->mtx); + if (edata != NULL) { + JE_USDT(sec_alloc, 5, sec, bin, edata, size, + /* frequent_reuse */ 1); + return edata; + } + } + cur_shard++; + if (cur_shard == sec->opts.nshards) { + cur_shard = 0; + } } - - malloc_mutex_lock(tsdn, &shard->mtx); - bin->being_batch_filled = false; /* - * Handle the easy case first: nothing to cache. Note that this can - * only happen in case of OOM, since sec_alloc checks the expected - * number of allocs, and doesn't bother going down the batch_fill - * pathway if there won't be anything left to cache. So to be in this - * code path, we must have asked for > 1 alloc, but only gotten 1 back. + * TODO: Benchmark whether it is worth blocking on all shards here before + * declaring a miss. That could recover more remote-shard hits under + * contention, but it also changes the allocation latency policy. */ - if (nalloc <= 1) { - malloc_mutex_unlock(tsdn, &shard->mtx); - return ret; + assert(cur_shard == sec_shard_pick(tsdn, sec)); + bin = sec_bin_pick(sec, cur_shard, pszind); + malloc_mutex_lock(tsdn, &bin->mtx); + edata_t *edata = sec_bin_alloc_locked(tsdn, sec, bin, size); + if (edata == NULL) { + /* Only now we know it is a miss. */ + bin->stats.nmisses++; } - - size_t new_cached_bytes = (nalloc - 1) * size; - - edata_list_active_concat(&bin->freelist, &result); - bin->bytes_cur += new_cached_bytes; - shard->bytes_cur += new_cached_bytes; - - if (shard->bytes_cur > sec->opts.max_bytes) { - sec_flush_some_and_unlock(tsdn, sec, shard); - } else { - malloc_mutex_unlock(tsdn, &shard->mtx); - } - - return ret; + malloc_mutex_unlock(tsdn, &bin->mtx); + JE_USDT(sec_alloc, 5, sec, bin, edata, size, /* frequent_reuse */ 1); + return edata; } -static edata_t * -sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero, - bool guarded, bool frequent_reuse, bool *deferred_work_generated) { +edata_t * +sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size) { + if (!sec_size_supported(sec, size)) { + return NULL; + } assert((size & PAGE_MASK) == 0); - assert(!guarded); - - sec_t *sec = (sec_t *)self; - - if (zero || alignment > PAGE || sec->opts.nshards == 0 - || size > sec->opts.max_alloc) { - return pai_alloc(tsdn, sec->fallback, size, alignment, zero, - /* guarded */ false, frequent_reuse, - deferred_work_generated); - } pszind_t pszind = sz_psz2ind(size); assert(pszind < sec->npsizes); - sec_shard_t *shard = sec_shard_pick(tsdn, sec); - sec_bin_t *bin = &shard->bins[pszind]; - bool do_batch_fill = false; - - malloc_mutex_lock(tsdn, &shard->mtx); - edata_t *edata = sec_shard_alloc_locked(tsdn, sec, shard, bin); - if (edata == NULL) { - if (!bin->being_batch_filled - && sec->opts.batch_fill_extra > 0) { - bin->being_batch_filled = true; - do_batch_fill = true; + /* + * If there's only one shard, skip the trylock optimization and + * go straight to the blocking lock. + */ + if (sec->opts.nshards == 1) { + sec_bin_t *bin = sec_bin_pick(sec, /* shard */ 0, pszind); + malloc_mutex_lock(tsdn, &bin->mtx); + edata_t *edata = sec_bin_alloc_locked(tsdn, sec, bin, size); + if (edata == NULL) { + bin->stats.nmisses++; } + malloc_mutex_unlock(tsdn, &bin->mtx); + JE_USDT(sec_alloc, 5, sec, bin, edata, size, + /* frequent_reuse */ 1); + return edata; } - malloc_mutex_unlock(tsdn, &shard->mtx); - if (edata == NULL) { - if (do_batch_fill) { - edata = sec_batch_fill_and_alloc(tsdn, sec, shard, bin, - size); - } else { - edata = pai_alloc(tsdn, sec->fallback, size, alignment, - zero, /* guarded */ false, frequent_reuse, - deferred_work_generated); - } - } - return edata; -} - -static bool -sec_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, - size_t new_size, bool zero, bool *deferred_work_generated) { - sec_t *sec = (sec_t *)self; - return pai_expand(tsdn, sec->fallback, edata, old_size, new_size, zero, - deferred_work_generated); -} - -static bool -sec_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, - size_t new_size, bool *deferred_work_generated) { - sec_t *sec = (sec_t *)self; - return pai_shrink(tsdn, sec->fallback, edata, old_size, new_size, - deferred_work_generated); + return sec_multishard_trylock_alloc(tsdn, sec, size, pszind); } static void -sec_flush_all_locked(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard) { - malloc_mutex_assert_owner(tsdn, &shard->mtx); - shard->bytes_cur = 0; - edata_list_active_t to_flush; - edata_list_active_init(&to_flush); - for (pszind_t i = 0; i < sec->npsizes; i++) { - sec_bin_t *bin = &shard->bins[i]; - bin->bytes_cur = 0; - edata_list_active_concat(&to_flush, &bin->freelist); - } +sec_bin_dalloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size, + edata_list_active_t *dalloc_list) { + malloc_mutex_assert_owner(tsdn, &bin->mtx); - /* - * Ordinarily we would try to avoid doing the batch deallocation while - * holding the shard mutex, but the flush_all pathways only happen when - * we're disabling the HPA or resetting the arena, both of which are - * rare pathways. - */ - bool deferred_work_generated = false; - pai_dalloc_batch(tsdn, sec->fallback, &to_flush, - &deferred_work_generated); -} - -static void -sec_shard_dalloc_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, - edata_t *edata) { - malloc_mutex_assert_owner(tsdn, &shard->mtx); - assert(shard->bytes_cur <= sec->opts.max_bytes); - size_t size = edata_size_get(edata); - pszind_t pszind = sz_psz2ind(size); - assert(pszind < sec->npsizes); - /* - * Prepending here results in LIFO allocation per bin, which seems - * reasonable. - */ - sec_bin_t *bin = &shard->bins[pszind]; - edata_list_active_prepend(&bin->freelist, edata); bin->bytes_cur += size; - shard->bytes_cur += size; - if (shard->bytes_cur > sec->opts.max_bytes) { - /* - * We've exceeded the shard limit. We make two nods in the - * direction of fragmentation avoidance: we flush everything in - * the shard, rather than one particular bin, and we hold the - * lock while flushing (in case one of the extents we flush is - * highly preferred from a fragmentation-avoidance perspective - * in the backing allocator). This has the extra advantage of - * not requiring advanced cache balancing strategies. - */ - sec_flush_some_and_unlock(tsdn, sec, shard); - malloc_mutex_assert_not_owner(tsdn, &shard->mtx); - } else { - malloc_mutex_unlock(tsdn, &shard->mtx); - } -} + edata_t *edata = edata_list_active_first(dalloc_list); + assert(edata != NULL); + edata_list_active_remove(dalloc_list, edata); + JE_USDT(sec_dalloc, 3, sec, bin, edata); + edata_list_active_prepend(&bin->freelist, edata); + /* Single extent can be returned to SEC */ + assert(edata_list_active_empty(dalloc_list)); -static void -sec_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata, - bool *deferred_work_generated) { - sec_t *sec = (sec_t *)self; - if (sec->opts.nshards == 0 - || edata_size_get(edata) > sec->opts.max_alloc) { - pai_dalloc(tsdn, sec->fallback, edata, - deferred_work_generated); + if (bin->bytes_cur <= sec->opts.max_bytes) { + bin->stats.ndalloc_noflush++; return; } - sec_shard_t *shard = sec_shard_pick(tsdn, sec); - malloc_mutex_lock(tsdn, &shard->mtx); - if (shard->enabled) { - sec_shard_dalloc_and_unlock(tsdn, sec, shard, edata); + bin->stats.ndalloc_flush++; + /* we want to flush 1/4 of max_bytes */ + size_t bytes_target = sec->opts.max_bytes - (sec->opts.max_bytes >> 2); + while (bin->bytes_cur > bytes_target + && !edata_list_active_empty(&bin->freelist)) { + edata_t *cur = edata_list_active_last(&bin->freelist); + size_t sz = edata_size_get(cur); + assert(sz <= bin->bytes_cur && sz > 0); + bin->bytes_cur -= sz; + edata_list_active_remove(&bin->freelist, cur); + edata_list_active_append(dalloc_list, cur); + } +} + +static void +sec_multishard_trylock_dalloc(tsdn_t *tsdn, sec_t *sec, size_t size, + pszind_t pszind, edata_list_active_t *dalloc_list) { + assert(sec->opts.nshards > 0); + + /* Try to dalloc in this threads bin first */ + uint8_t cur_shard = sec_shard_pick(tsdn, sec); + for (size_t i = 0; i < sec->opts.nshards; ++i) { + sec_bin_t *bin = sec_bin_pick(sec, cur_shard, pszind); + if (!malloc_mutex_trylock(tsdn, &bin->mtx)) { + sec_bin_dalloc_locked( + tsdn, sec, bin, size, dalloc_list); + malloc_mutex_unlock(tsdn, &bin->mtx); + return; + } + cur_shard++; + if (cur_shard == sec->opts.nshards) { + cur_shard = 0; + } + } + /* No bin had alloc or had the extent */ + assert(cur_shard == sec_shard_pick(tsdn, sec)); + sec_bin_t *bin = sec_bin_pick(sec, cur_shard, pszind); + malloc_mutex_lock(tsdn, &bin->mtx); + sec_bin_dalloc_locked(tsdn, sec, bin, size, dalloc_list); + malloc_mutex_unlock(tsdn, &bin->mtx); +} + +void +sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list) { + if (!sec_is_used(sec)) { + return; + } + edata_t *edata = edata_list_active_first(dalloc_list); + size_t size = edata_size_get(edata); + if (size > sec->opts.max_alloc) { + return; + } + pszind_t pszind = sz_psz2ind(size); + assert(pszind < sec->npsizes); + + /* + * If there's only one shard, skip the trylock optimization and + * go straight to the blocking lock. + */ + if (sec->opts.nshards == 1) { + sec_bin_t *bin = sec_bin_pick(sec, /* shard */ 0, pszind); + malloc_mutex_lock(tsdn, &bin->mtx); + sec_bin_dalloc_locked(tsdn, sec, bin, size, dalloc_list); + malloc_mutex_unlock(tsdn, &bin->mtx); + return; + } + sec_multishard_trylock_dalloc(tsdn, sec, size, pszind, dalloc_list); +} + +void +sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size, edata_list_active_t *result, + size_t nallocs) { + assert((size & PAGE_MASK) == 0); + assert(sec->opts.nshards != 0 && size <= sec->opts.max_alloc); + assert(nallocs > 0); + + pszind_t pszind = sz_psz2ind(size); + assert(pszind < sec->npsizes); + + sec_bin_t *bin = sec_bin_pick(sec, sec_shard_pick(tsdn, sec), pszind); + malloc_mutex_assert_not_owner(tsdn, &bin->mtx); + malloc_mutex_lock(tsdn, &bin->mtx); + size_t new_cached_bytes = nallocs * size; + if (bin->bytes_cur + new_cached_bytes <= sec->opts.max_bytes) { + assert(!edata_list_active_empty(result)); + edata_list_active_concat(&bin->freelist, result); + bin->bytes_cur += new_cached_bytes; } else { - malloc_mutex_unlock(tsdn, &shard->mtx); - pai_dalloc(tsdn, sec->fallback, edata, - deferred_work_generated); + /* + * Unlikely case of many threads filling at the same time and + * going above max. + */ + bin->stats.noverfills++; + while (bin->bytes_cur + size <= sec->opts.max_bytes) { + edata_t *edata = edata_list_active_first(result); + if (edata == NULL) { + break; + } + edata_list_active_remove(result, edata); + assert(size == edata_size_get(edata)); + edata_list_active_append(&bin->freelist, edata); + bin->bytes_cur += size; + } } + malloc_mutex_unlock(tsdn, &bin->mtx); } void -sec_flush(tsdn_t *tsdn, sec_t *sec) { - for (size_t i = 0; i < sec->opts.nshards; i++) { - malloc_mutex_lock(tsdn, &sec->shards[i].mtx); - sec_flush_all_locked(tsdn, sec, &sec->shards[i]); - malloc_mutex_unlock(tsdn, &sec->shards[i].mtx); +sec_flush(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *to_flush) { + if (!sec_is_used(sec)) { + return; } -} - -void -sec_disable(tsdn_t *tsdn, sec_t *sec) { - for (size_t i = 0; i < sec->opts.nshards; i++) { - malloc_mutex_lock(tsdn, &sec->shards[i].mtx); - sec->shards[i].enabled = false; - sec_flush_all_locked(tsdn, sec, &sec->shards[i]); - malloc_mutex_unlock(tsdn, &sec->shards[i].mtx); + size_t ntotal_bins = sec->opts.nshards * sec->npsizes; + for (pszind_t i = 0; i < ntotal_bins; i++) { + sec_bin_t *bin = &sec->bins[i]; + malloc_mutex_lock(tsdn, &bin->mtx); + bin->bytes_cur = 0; + edata_list_active_concat(to_flush, &bin->freelist); + malloc_mutex_unlock(tsdn, &bin->mtx); } } void sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats) { + if (!sec_is_used(sec)) { + return; + } size_t sum = 0; - for (size_t i = 0; i < sec->opts.nshards; i++) { - /* - * We could save these lock acquisitions by making bytes_cur - * atomic, but stats collection is rare anyways and we expect - * the number and type of stats to get more interesting. - */ - malloc_mutex_lock(tsdn, &sec->shards[i].mtx); - sum += sec->shards[i].bytes_cur; - malloc_mutex_unlock(tsdn, &sec->shards[i].mtx); + size_t ntotal_bins = sec->opts.nshards * sec->npsizes; + for (pszind_t i = 0; i < ntotal_bins; i++) { + sec_bin_t *bin = &sec->bins[i]; + malloc_mutex_lock(tsdn, &bin->mtx); + sum += bin->bytes_cur; + sec_bin_stats_accum(&stats->total, &bin->stats); + malloc_mutex_unlock(tsdn, &bin->mtx); } stats->bytes += sum; } void -sec_mutex_stats_read(tsdn_t *tsdn, sec_t *sec, - mutex_prof_data_t *mutex_prof_data) { - for (size_t i = 0; i < sec->opts.nshards; i++) { - malloc_mutex_lock(tsdn, &sec->shards[i].mtx); - malloc_mutex_prof_accum(tsdn, mutex_prof_data, - &sec->shards[i].mtx); - malloc_mutex_unlock(tsdn, &sec->shards[i].mtx); +sec_mutex_stats_read( + tsdn_t *tsdn, sec_t *sec, mutex_prof_data_t *mutex_prof_data) { + if (!sec_is_used(sec)) { + return; + } + size_t ntotal_bins = sec->opts.nshards * sec->npsizes; + for (pszind_t i = 0; i < ntotal_bins; i++) { + sec_bin_t *bin = &sec->bins[i]; + malloc_mutex_lock(tsdn, &bin->mtx); + malloc_mutex_prof_accum(tsdn, mutex_prof_data, &bin->mtx); + malloc_mutex_unlock(tsdn, &bin->mtx); } } void sec_prefork2(tsdn_t *tsdn, sec_t *sec) { - for (size_t i = 0; i < sec->opts.nshards; i++) { - malloc_mutex_prefork(tsdn, &sec->shards[i].mtx); + if (!sec_is_used(sec)) { + return; + } + size_t ntotal_bins = sec->opts.nshards * sec->npsizes; + for (pszind_t i = 0; i < ntotal_bins; i++) { + sec_bin_t *bin = &sec->bins[i]; + malloc_mutex_prefork(tsdn, &bin->mtx); } } void sec_postfork_parent(tsdn_t *tsdn, sec_t *sec) { - for (size_t i = 0; i < sec->opts.nshards; i++) { - malloc_mutex_postfork_parent(tsdn, &sec->shards[i].mtx); + if (!sec_is_used(sec)) { + return; + } + size_t ntotal_bins = sec->opts.nshards * sec->npsizes; + for (pszind_t i = 0; i < ntotal_bins; i++) { + sec_bin_t *bin = &sec->bins[i]; + malloc_mutex_postfork_parent(tsdn, &bin->mtx); } } void sec_postfork_child(tsdn_t *tsdn, sec_t *sec) { - for (size_t i = 0; i < sec->opts.nshards; i++) { - malloc_mutex_postfork_child(tsdn, &sec->shards[i].mtx); + if (!sec_is_used(sec)) { + return; + } + size_t ntotal_bins = sec->opts.nshards * sec->npsizes; + for (pszind_t i = 0; i < ntotal_bins; i++) { + sec_bin_t *bin = &sec->bins[i]; + malloc_mutex_postfork_child(tsdn, &bin->mtx); } } diff --git a/dep/jemalloc/src/stats.c b/dep/jemalloc/src/stats.c index efc70fd3c..82458fec4 100644 --- a/dep/jemalloc/src/stats.c +++ b/dep/jemalloc/src/stats.c @@ -9,60 +9,67 @@ #include "jemalloc/internal/mutex_prof.h" #include "jemalloc/internal/prof_stats.h" -const char *global_mutex_names[mutex_prof_num_global_mutexes] = { +static const char *const global_mutex_names[mutex_prof_num_global_mutexes] = { #define OP(mtx) #mtx, - MUTEX_PROF_GLOBAL_MUTEXES + MUTEX_PROF_GLOBAL_MUTEXES #undef OP }; -const char *arena_mutex_names[mutex_prof_num_arena_mutexes] = { +static const char *const arena_mutex_names[mutex_prof_num_arena_mutexes] = { #define OP(mtx) #mtx, - MUTEX_PROF_ARENA_MUTEXES + MUTEX_PROF_ARENA_MUTEXES #undef OP }; -#define CTL_GET(n, v, t) do { \ - size_t sz = sizeof(t); \ - xmallctl(n, (void *)v, &sz, NULL, 0); \ -} while (0) +#define CTL_GET(n, v, t) \ + do { \ + size_t sz = sizeof(t); \ + xmallctl(n, (void *)v, &sz, NULL, 0); \ + } while (0) -#define CTL_LEAF_PREPARE(mib, miblen, name) do { \ - assert(miblen < CTL_MAX_DEPTH); \ - size_t miblen_new = CTL_MAX_DEPTH; \ - xmallctlmibnametomib(mib, miblen, name, &miblen_new); \ - assert(miblen_new > miblen); \ -} while (0) +#define CTL_LEAF_PREPARE(mib, miblen, name) \ + do { \ + assert(miblen < CTL_MAX_DEPTH); \ + size_t miblen_new = CTL_MAX_DEPTH; \ + xmallctlmibnametomib(mib, miblen, name, &miblen_new); \ + assert(miblen_new > miblen); \ + } while (0) -#define CTL_LEAF(mib, miblen, leaf, v, t) do { \ - assert(miblen < CTL_MAX_DEPTH); \ - size_t miblen_new = CTL_MAX_DEPTH; \ - size_t sz = sizeof(t); \ - xmallctlbymibname(mib, miblen, leaf, &miblen_new, (void *)v, \ - &sz, NULL, 0); \ - assert(miblen_new == miblen + 1); \ -} while (0) +#define CTL_LEAF(mib, miblen, leaf, v, t) \ + do { \ + assert(miblen < CTL_MAX_DEPTH); \ + size_t miblen_new = CTL_MAX_DEPTH; \ + size_t sz = sizeof(t); \ + xmallctlbymibname( \ + mib, miblen, leaf, &miblen_new, (void *)v, &sz, NULL, 0); \ + assert(miblen_new == miblen + 1); \ + } while (0) -#define CTL_M2_GET(n, i, v, t) do { \ - size_t mib[CTL_MAX_DEPTH]; \ - size_t miblen = sizeof(mib) / sizeof(size_t); \ - size_t sz = sizeof(t); \ - xmallctlnametomib(n, mib, &miblen); \ - mib[2] = (i); \ - xmallctlbymib(mib, miblen, (void *)v, &sz, NULL, 0); \ -} while (0) +#define CTL_MIB_GET(n, i, v, t, ind) \ + do { \ + size_t mib[CTL_MAX_DEPTH]; \ + size_t miblen = sizeof(mib) / sizeof(size_t); \ + size_t sz = sizeof(t); \ + xmallctlnametomib(n, mib, &miblen); \ + mib[(ind)] = (i); \ + xmallctlbymib(mib, miblen, (void *)v, &sz, NULL, 0); \ + } while (0) + +#define CTL_M1_GET(n, i, v, t) CTL_MIB_GET(n, i, v, t, 1) +#define CTL_M2_GET(n, i, v, t) CTL_MIB_GET(n, i, v, t, 2) /******************************************************************************/ /* Data. */ bool opt_stats_print = false; -char opt_stats_print_opts[stats_print_tot_num_options+1] = ""; +char opt_stats_print_opts[stats_print_tot_num_options + 1] = ""; int64_t opt_stats_interval = STATS_INTERVAL_DEFAULT; -char opt_stats_interval_opts[stats_print_tot_num_options+1] = ""; +char opt_stats_interval_opts[stats_print_tot_num_options + 1] = ""; static counter_accum_t stats_interval_accumulated; /* Per thread batch accum size for stats_interval. */ -static uint64_t stats_interval_accum_batch; +uint64_t stats_interval_accum_batch; /******************************************************************************/ @@ -108,8 +115,8 @@ get_rate_str(uint64_t dividend, uint64_t divisor, char str[6]) { static void mutex_stats_init_cols(emitter_row_t *row, const char *table_name, emitter_col_t *name, - emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], - emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters]) { + emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], + emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters]) { mutex_prof_uint64_t_counter_ind_t k_uint64_t = 0; mutex_prof_uint32_t_counter_ind_t k_uint32_t = 0; @@ -125,13 +132,13 @@ mutex_stats_init_cols(emitter_row_t *row, const char *table_name, #define WIDTH_uint32_t 12 #define WIDTH_uint64_t 16 -#define OP(counter, counter_type, human, derived, base_counter) \ - col = &col_##counter_type[k_##counter_type]; \ - ++k_##counter_type; \ - emitter_col_init(col, row); \ - col->justify = emitter_justify_right; \ - col->width = derived ? 8 : WIDTH_##counter_type; \ - col->type = emitter_type_title; \ +#define OP(counter, counter_type, human, derived, base_counter) \ + col = &col_##counter_type[k_##counter_type]; \ + ++k_##counter_type; \ + emitter_col_init(col, row); \ + col->justify = emitter_justify_right; \ + col->width = derived ? 8 : WIDTH_##counter_type; \ + col->type = emitter_type_title; \ col->str_val = human; MUTEX_PROF_COUNTERS #undef OP @@ -143,9 +150,9 @@ mutex_stats_init_cols(emitter_row_t *row, const char *table_name, static void mutex_stats_read_global(size_t mib[], size_t miblen, const char *name, emitter_col_t *col_name, - emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], - emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters], - uint64_t uptime) { + emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], + emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters], + uint64_t uptime) { CTL_LEAF_PREPARE(mib, miblen, name); size_t miblen_name = miblen + 1; @@ -154,18 +161,17 @@ mutex_stats_read_global(size_t mib[], size_t miblen, const char *name, emitter_col_t *dst; #define EMITTER_TYPE_uint32_t emitter_type_uint32 #define EMITTER_TYPE_uint64_t emitter_type_uint64 -#define OP(counter, counter_type, human, derived, base_counter) \ - dst = &col_##counter_type[mutex_counter_##counter]; \ - dst->type = EMITTER_TYPE_##counter_type; \ - if (!derived) { \ - CTL_LEAF(mib, miblen_name, #counter, \ - (counter_type *)&dst->bool_val, counter_type); \ - } else { \ - emitter_col_t *base = \ - &col_##counter_type[mutex_counter_##base_counter]; \ - dst->counter_type##_val = \ - (counter_type)rate_per_second( \ - base->counter_type##_val, uptime); \ +#define OP(counter, counter_type, human, derived, base_counter) \ + dst = &col_##counter_type[mutex_counter_##counter]; \ + dst->type = EMITTER_TYPE_##counter_type; \ + if (!derived) { \ + CTL_LEAF(mib, miblen_name, #counter, \ + (counter_type *)&dst->bool_val, counter_type); \ + } else { \ + emitter_col_t *base = \ + &col_##counter_type[mutex_counter_##base_counter]; \ + dst->counter_type##_val = (counter_type)rate_per_second( \ + base->counter_type##_val, uptime); \ } MUTEX_PROF_COUNTERS #undef OP @@ -176,9 +182,9 @@ mutex_stats_read_global(size_t mib[], size_t miblen, const char *name, static void mutex_stats_read_arena(size_t mib[], size_t miblen, const char *name, emitter_col_t *col_name, - emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], - emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters], - uint64_t uptime) { + emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], + emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters], + uint64_t uptime) { CTL_LEAF_PREPARE(mib, miblen, name); size_t miblen_name = miblen + 1; @@ -187,18 +193,17 @@ mutex_stats_read_arena(size_t mib[], size_t miblen, const char *name, emitter_col_t *dst; #define EMITTER_TYPE_uint32_t emitter_type_uint32 #define EMITTER_TYPE_uint64_t emitter_type_uint64 -#define OP(counter, counter_type, human, derived, base_counter) \ - dst = &col_##counter_type[mutex_counter_##counter]; \ - dst->type = EMITTER_TYPE_##counter_type; \ - if (!derived) { \ - CTL_LEAF(mib, miblen_name, #counter, \ - (counter_type *)&dst->bool_val, counter_type); \ - } else { \ - emitter_col_t *base = \ - &col_##counter_type[mutex_counter_##base_counter]; \ - dst->counter_type##_val = \ - (counter_type)rate_per_second( \ - base->counter_type##_val, uptime); \ +#define OP(counter, counter_type, human, derived, base_counter) \ + dst = &col_##counter_type[mutex_counter_##counter]; \ + dst->type = EMITTER_TYPE_##counter_type; \ + if (!derived) { \ + CTL_LEAF(mib, miblen_name, #counter, \ + (counter_type *)&dst->bool_val, counter_type); \ + } else { \ + emitter_col_t *base = \ + &col_##counter_type[mutex_counter_##base_counter]; \ + dst->counter_type##_val = (counter_type)rate_per_second( \ + base->counter_type##_val, uptime); \ } MUTEX_PROF_COUNTERS #undef OP @@ -210,7 +215,7 @@ static void mutex_stats_read_arena_bin(size_t mib[], size_t miblen, emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters], - uint64_t uptime) { + uint64_t uptime) { CTL_LEAF_PREPARE(mib, miblen, "mutex"); size_t miblen_mutex = miblen + 1; @@ -218,18 +223,17 @@ mutex_stats_read_arena_bin(size_t mib[], size_t miblen, #define EMITTER_TYPE_uint32_t emitter_type_uint32 #define EMITTER_TYPE_uint64_t emitter_type_uint64 -#define OP(counter, counter_type, human, derived, base_counter) \ - dst = &col_##counter_type[mutex_counter_##counter]; \ - dst->type = EMITTER_TYPE_##counter_type; \ - if (!derived) { \ - CTL_LEAF(mib, miblen_mutex, #counter, \ - (counter_type *)&dst->bool_val, counter_type); \ - } else { \ - emitter_col_t *base = \ - &col_##counter_type[mutex_counter_##base_counter]; \ - dst->counter_type##_val = \ - (counter_type)rate_per_second( \ - base->counter_type##_val, uptime); \ +#define OP(counter, counter_type, human, derived, base_counter) \ + dst = &col_##counter_type[mutex_counter_##counter]; \ + dst->type = EMITTER_TYPE_##counter_type; \ + if (!derived) { \ + CTL_LEAF(mib, miblen_mutex, #counter, \ + (counter_type *)&dst->bool_val, counter_type); \ + } else { \ + emitter_col_t *base = \ + &col_##counter_type[mutex_counter_##base_counter]; \ + dst->counter_type##_val = (counter_type)rate_per_second( \ + base->counter_type##_val, uptime); \ } MUTEX_PROF_COUNTERS #undef OP @@ -253,57 +257,55 @@ mutex_stats_emit(emitter_t *emitter, emitter_row_t *row, #define EMITTER_TYPE_uint32_t emitter_type_uint32 #define EMITTER_TYPE_uint64_t emitter_type_uint64 -#define OP(counter, type, human, derived, base_counter) \ - if (!derived) { \ - col = &col_##type[k_##type]; \ - ++k_##type; \ +#define OP(counter, type, human, derived, base_counter) \ + if (!derived) { \ + col = &col_##type[k_##type]; \ emitter_json_kv(emitter, #counter, EMITTER_TYPE_##type, \ - (const void *)&col->bool_val); \ - } + (const void *)&col->bool_val); \ + } \ + ++k_##type; MUTEX_PROF_COUNTERS; #undef OP #undef EMITTER_TYPE_uint32_t #undef EMITTER_TYPE_uint64_t } -#define COL_DECLARE(column_name) \ - emitter_col_t col_##column_name; +#define COL_DECLARE(column_name) emitter_col_t col_##column_name; -#define COL_INIT(row_name, column_name, left_or_right, col_width, etype)\ - emitter_col_init(&col_##column_name, &row_name); \ - col_##column_name.justify = emitter_justify_##left_or_right; \ - col_##column_name.width = col_width; \ +#define COL_INIT(row_name, column_name, left_or_right, col_width, etype) \ + emitter_col_init(&col_##column_name, &row_name); \ + col_##column_name.justify = emitter_justify_##left_or_right; \ + col_##column_name.width = col_width; \ col_##column_name.type = emitter_type_##etype; -#define COL(row_name, column_name, left_or_right, col_width, etype) \ - COL_DECLARE(column_name); \ +#define COL(row_name, column_name, left_or_right, col_width, etype) \ + COL_DECLARE(column_name); \ COL_INIT(row_name, column_name, left_or_right, col_width, etype) -#define COL_HDR_DECLARE(column_name) \ - COL_DECLARE(column_name); \ +#define COL_HDR_DECLARE(column_name) \ + COL_DECLARE(column_name); \ emitter_col_t header_##column_name; -#define COL_HDR_INIT(row_name, column_name, human, left_or_right, \ - col_width, etype) \ - COL_INIT(row_name, column_name, left_or_right, col_width, etype)\ - emitter_col_init(&header_##column_name, &header_##row_name); \ - header_##column_name.justify = emitter_justify_##left_or_right; \ - header_##column_name.width = col_width; \ - header_##column_name.type = emitter_type_title; \ +#define COL_HDR_INIT( \ + row_name, column_name, human, left_or_right, col_width, etype) \ + COL_INIT(row_name, column_name, left_or_right, col_width, etype) \ + emitter_col_init(&header_##column_name, &header_##row_name); \ + header_##column_name.justify = emitter_justify_##left_or_right; \ + header_##column_name.width = col_width; \ + header_##column_name.type = emitter_type_title; \ header_##column_name.str_val = human ? human : #column_name; -#define COL_HDR(row_name, column_name, human, left_or_right, col_width, \ - etype) \ - COL_HDR_DECLARE(column_name) \ - COL_HDR_INIT(row_name, column_name, human, left_or_right, \ - col_width, etype) +#define COL_HDR(row_name, column_name, human, left_or_right, col_width, etype) \ + COL_HDR_DECLARE(column_name) \ + COL_HDR_INIT( \ + row_name, column_name, human, left_or_right, col_width, etype) JEMALLOC_COLD static void -stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, - uint64_t uptime) { - size_t page; - bool in_gap, in_gap_prev; +stats_arena_bins_print( + emitter_t *emitter, bool mutex, unsigned i, uint64_t uptime) { + size_t page; + bool in_gap, in_gap_prev; unsigned nbins, j; CTL_GET("arenas.page", &page, size_t); @@ -321,12 +323,12 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, COL_HDR(row, size, NULL, right, 20, size) COL_HDR(row, ind, NULL, right, 4, unsigned) - COL_HDR(row, allocated, NULL, right, 13, uint64) - COL_HDR(row, nmalloc, NULL, right, 13, uint64) + COL_HDR(row, allocated, NULL, right, 14, size) + COL_HDR(row, nmalloc, NULL, right, 14, uint64) COL_HDR(row, nmalloc_ps, "(#/sec)", right, 8, uint64) - COL_HDR(row, ndalloc, NULL, right, 13, uint64) + COL_HDR(row, ndalloc, NULL, right, 14, uint64) COL_HDR(row, ndalloc_ps, "(#/sec)", right, 8, uint64) - COL_HDR(row, nrequests, NULL, right, 13, uint64) + COL_HDR(row, nrequests, NULL, right, 15, uint64) COL_HDR(row, nrequests_ps, "(#/sec)", right, 10, uint64) COL_HDR_DECLARE(prof_live_requested); COL_HDR_DECLARE(prof_live_count); @@ -366,17 +368,17 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, emitter_col_t header_mutex32[mutex_prof_num_uint32_t_counters]; if (mutex) { - mutex_stats_init_cols(&row, NULL, NULL, col_mutex64, - col_mutex32); - mutex_stats_init_cols(&header_row, NULL, NULL, header_mutex64, - header_mutex32); + mutex_stats_init_cols( + &row, NULL, NULL, col_mutex64, col_mutex32); + mutex_stats_init_cols( + &header_row, NULL, NULL, header_mutex64, header_mutex32); } /* * We print a "bins:" header as part of the table row; we need to adjust * the header size column to compensate. */ - header_size.width -=5; + header_size.width -= 5; emitter_table_printf(emitter, "bins:"); emitter_table_row(emitter, &header_row); emitter_json_array_kv_begin(emitter, "bins"); @@ -395,13 +397,13 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, } for (j = 0, in_gap = false; j < nbins; j++) { - uint64_t nslabs; - size_t reg_size, slab_size, curregs; - size_t curslabs; - size_t nonfull_slabs; - uint32_t nregs, nshards; - uint64_t nmalloc, ndalloc, nrequests, nfills, nflushes; - uint64_t nreslabs; + uint64_t nslabs; + size_t reg_size, slab_size, curregs; + size_t curslabs; + size_t nonfull_slabs; + uint32_t nregs, nshards; + uint64_t nmalloc, ndalloc, nrequests, nfills, nflushes; + uint64_t nreslabs; prof_stats_t prof_live; prof_stats_t prof_accum; @@ -426,8 +428,8 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, } if (in_gap_prev && !in_gap) { - emitter_table_printf(emitter, - " ---\n"); + emitter_table_printf( + emitter, " ---\n"); } if (in_gap && !emitter_outputs_json(emitter)) { @@ -441,8 +443,8 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, CTL_LEAF(stats_arenas_mib, 5, "nmalloc", &nmalloc, uint64_t); CTL_LEAF(stats_arenas_mib, 5, "ndalloc", &ndalloc, uint64_t); CTL_LEAF(stats_arenas_mib, 5, "curregs", &curregs, size_t); - CTL_LEAF(stats_arenas_mib, 5, "nrequests", &nrequests, - uint64_t); + CTL_LEAF( + stats_arenas_mib, 5, "nrequests", &nrequests, uint64_t); CTL_LEAF(stats_arenas_mib, 5, "nfills", &nfills, uint64_t); CTL_LEAF(stats_arenas_mib, 5, "nflushes", &nflushes, uint64_t); CTL_LEAF(stats_arenas_mib, 5, "nreslabs", &nreslabs, uint64_t); @@ -456,14 +458,14 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, } emitter_json_object_begin(emitter); - emitter_json_kv(emitter, "nmalloc", emitter_type_uint64, - &nmalloc); - emitter_json_kv(emitter, "ndalloc", emitter_type_uint64, - &ndalloc); - emitter_json_kv(emitter, "curregs", emitter_type_size, - &curregs); - emitter_json_kv(emitter, "nrequests", emitter_type_uint64, - &nrequests); + emitter_json_kv( + emitter, "nmalloc", emitter_type_uint64, &nmalloc); + emitter_json_kv( + emitter, "ndalloc", emitter_type_uint64, &ndalloc); + emitter_json_kv( + emitter, "curregs", emitter_type_size, &curregs); + emitter_json_kv( + emitter, "nrequests", emitter_type_uint64, &nrequests); if (prof_stats_on) { emitter_json_kv(emitter, "prof_live_requested", emitter_type_uint64, &prof_live.req_sum); @@ -474,28 +476,28 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, emitter_json_kv(emitter, "prof_accum_count", emitter_type_uint64, &prof_accum.count); } - emitter_json_kv(emitter, "nfills", emitter_type_uint64, - &nfills); - emitter_json_kv(emitter, "nflushes", emitter_type_uint64, - &nflushes); - emitter_json_kv(emitter, "nreslabs", emitter_type_uint64, - &nreslabs); - emitter_json_kv(emitter, "curslabs", emitter_type_size, - &curslabs); + emitter_json_kv( + emitter, "nfills", emitter_type_uint64, &nfills); + emitter_json_kv( + emitter, "nflushes", emitter_type_uint64, &nflushes); + emitter_json_kv( + emitter, "nreslabs", emitter_type_uint64, &nreslabs); + emitter_json_kv( + emitter, "curslabs", emitter_type_size, &curslabs); emitter_json_kv(emitter, "nonfull_slabs", emitter_type_size, &nonfull_slabs); if (mutex) { emitter_json_object_kv_begin(emitter, "mutex"); - mutex_stats_emit(emitter, NULL, col_mutex64, - col_mutex32); + mutex_stats_emit( + emitter, NULL, col_mutex64, col_mutex32); emitter_json_object_end(emitter); } emitter_json_object_end(emitter); size_t availregs = nregs * curslabs; - char util[6]; - if (get_rate_str((uint64_t)curregs, (uint64_t)availregs, util)) - { + char util[6]; + if (get_rate_str( + (uint64_t)curregs, (uint64_t)availregs, util)) { if (availregs == 0) { malloc_snprintf(util, sizeof(util), "1"); } else if (curregs > availregs) { @@ -519,7 +521,8 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, col_ndalloc.uint64_val = ndalloc; col_ndalloc_ps.uint64_val = rate_per_second(ndalloc, uptime); col_nrequests.uint64_val = nrequests; - col_nrequests_ps.uint64_val = rate_per_second(nrequests, uptime); + col_nrequests_ps.uint64_val = rate_per_second( + nrequests, uptime); if (prof_stats_on) { col_prof_live_requested.uint64_val = prof_live.req_sum; col_prof_live_count.uint64_val = prof_live.count; @@ -560,7 +563,7 @@ JEMALLOC_COLD static void stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) { unsigned nbins, nlextents, j; - bool in_gap, in_gap_prev; + bool in_gap, in_gap_prev; CTL_GET("arenas.nbins", &nbins, unsigned); CTL_GET("arenas.nlextents", &nlextents, unsigned); @@ -614,8 +617,8 @@ stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) { } for (j = 0, in_gap = false; j < nlextents; j++) { - uint64_t nmalloc, ndalloc, nrequests; - size_t lextent_size, curlextents; + uint64_t nmalloc, ndalloc, nrequests; + size_t lextent_size, curlextents; prof_stats_t prof_live; prof_stats_t prof_accum; @@ -624,20 +627,20 @@ stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) { CTL_LEAF(stats_arenas_mib, 5, "nmalloc", &nmalloc, uint64_t); CTL_LEAF(stats_arenas_mib, 5, "ndalloc", &ndalloc, uint64_t); - CTL_LEAF(stats_arenas_mib, 5, "nrequests", &nrequests, - uint64_t); + CTL_LEAF( + stats_arenas_mib, 5, "nrequests", &nrequests, uint64_t); in_gap_prev = in_gap; in_gap = (nrequests == 0); if (in_gap_prev && !in_gap) { - emitter_table_printf(emitter, - " ---\n"); + emitter_table_printf( + emitter, " ---\n"); } CTL_LEAF(arenas_lextent_mib, 3, "size", &lextent_size, size_t); - CTL_LEAF(stats_arenas_mib, 5, "curlextents", &curlextents, - size_t); + CTL_LEAF( + stats_arenas_mib, 5, "curlextents", &curlextents, size_t); if (prof_stats_on) { prof_stats_mib[3] = j; @@ -658,8 +661,8 @@ stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) { emitter_json_kv(emitter, "prof_accum_count", emitter_type_uint64, &prof_accum.count); } - emitter_json_kv(emitter, "curlextents", emitter_type_size, - &curlextents); + emitter_json_kv( + emitter, "curlextents", emitter_type_size, &curlextents); emitter_json_object_end(emitter); col_size.size_val = lextent_size; @@ -670,7 +673,8 @@ stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) { col_ndalloc.uint64_val = ndalloc; col_ndalloc_ps.uint64_val = rate_per_second(ndalloc, uptime); col_nrequests.uint64_val = nrequests; - col_nrequests_ps.uint64_val = rate_per_second(nrequests, uptime); + col_nrequests_ps.uint64_val = rate_per_second( + nrequests, uptime); if (prof_stats_on) { col_prof_live_requested.uint64_val = prof_live.req_sum; col_prof_live_count.uint64_val = prof_live.count; @@ -693,8 +697,8 @@ stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) { JEMALLOC_COLD static void stats_arena_extents_print(emitter_t *emitter, unsigned i) { - unsigned j; - bool in_gap, in_gap_prev; + unsigned j; + bool in_gap, in_gap_prev; emitter_row_t header_row; emitter_row_init(&header_row); emitter_row_t row; @@ -731,12 +735,12 @@ stats_arena_extents_print(emitter_t *emitter, unsigned i) { CTL_LEAF(stats_arenas_mib, 5, "ndirty", &ndirty, size_t); CTL_LEAF(stats_arenas_mib, 5, "nmuzzy", &nmuzzy, size_t); CTL_LEAF(stats_arenas_mib, 5, "nretained", &nretained, size_t); - CTL_LEAF(stats_arenas_mib, 5, "dirty_bytes", &dirty_bytes, + CTL_LEAF( + stats_arenas_mib, 5, "dirty_bytes", &dirty_bytes, size_t); + CTL_LEAF( + stats_arenas_mib, 5, "muzzy_bytes", &muzzy_bytes, size_t); + CTL_LEAF(stats_arenas_mib, 5, "retained_bytes", &retained_bytes, size_t); - CTL_LEAF(stats_arenas_mib, 5, "muzzy_bytes", &muzzy_bytes, - size_t); - CTL_LEAF(stats_arenas_mib, 5, "retained_bytes", - &retained_bytes, size_t); total = ndirty + nmuzzy + nretained; total_bytes = dirty_bytes + muzzy_bytes + retained_bytes; @@ -745,20 +749,20 @@ stats_arena_extents_print(emitter_t *emitter, unsigned i) { in_gap = (total == 0); if (in_gap_prev && !in_gap) { - emitter_table_printf(emitter, - " ---\n"); + emitter_table_printf( + emitter, " ---\n"); } emitter_json_object_begin(emitter); emitter_json_kv(emitter, "ndirty", emitter_type_size, &ndirty); emitter_json_kv(emitter, "nmuzzy", emitter_type_size, &nmuzzy); - emitter_json_kv(emitter, "nretained", emitter_type_size, - &nretained); + emitter_json_kv( + emitter, "nretained", emitter_type_size, &nretained); - emitter_json_kv(emitter, "dirty_bytes", emitter_type_size, - &dirty_bytes); - emitter_json_kv(emitter, "muzzy_bytes", emitter_type_size, - &muzzy_bytes); + emitter_json_kv( + emitter, "dirty_bytes", emitter_type_size, &dirty_bytes); + emitter_json_kv( + emitter, "muzzy_bytes", emitter_type_size, &muzzy_bytes); emitter_json_kv(emitter, "retained_bytes", emitter_type_size, &retained_bytes); emitter_json_object_end(emitter); @@ -785,25 +789,156 @@ stats_arena_extents_print(emitter_t *emitter, unsigned i) { } static void -stats_arena_hpa_shard_print(emitter_t *emitter, unsigned i, uint64_t uptime) { - emitter_row_t header_row; - emitter_row_init(&header_row); - emitter_row_t row; - emitter_row_init(&row); +stats_arena_hpa_shard_sec_print(emitter_t *emitter, unsigned i) { + size_t sec_bytes; + size_t sec_hits; + size_t sec_misses; + size_t sec_dalloc_flush; + size_t sec_dalloc_noflush; + size_t sec_overfills; + CTL_M2_GET("stats.arenas.0.hpa_sec_bytes", i, &sec_bytes, size_t); + emitter_kv(emitter, "sec_bytes", "Bytes in small extent cache", + emitter_type_size, &sec_bytes); + CTL_M2_GET("stats.arenas.0.hpa_sec_hits", i, &sec_hits, size_t); + emitter_kv(emitter, "sec_hits", "Total hits in small extent cache", + emitter_type_size, &sec_hits); + CTL_M2_GET("stats.arenas.0.hpa_sec_misses", i, &sec_misses, size_t); + emitter_kv(emitter, "sec_misses", "Total misses in small extent cache", + emitter_type_size, &sec_misses); + CTL_M2_GET("stats.arenas.0.hpa_sec_dalloc_noflush", i, + &sec_dalloc_noflush, size_t); + emitter_kv(emitter, "sec_dalloc_noflush", + "Dalloc calls without flush in small extent cache", + emitter_type_size, &sec_dalloc_noflush); + CTL_M2_GET("stats.arenas.0.hpa_sec_dalloc_flush", i, &sec_dalloc_flush, + size_t); + emitter_kv(emitter, "sec_dalloc_flush", + "Dalloc calls with flush in small extent cache", emitter_type_size, + &sec_dalloc_flush); + CTL_M2_GET( + "stats.arenas.0.hpa_sec_overfills", i, &sec_overfills, size_t); + emitter_kv(emitter, "sec_overfills", + "sec_fill calls that went over max_bytes", emitter_type_size, + &sec_overfills); +} + +static void +stats_arena_hpa_shard_counters_print( + emitter_t *emitter, unsigned i, uint64_t uptime) { + size_t npageslabs; + size_t nactive; + size_t ndirty; + + size_t npageslabs_nonhuge; + size_t nactive_nonhuge; + size_t ndirty_nonhuge; + size_t nretained_nonhuge; + + size_t npageslabs_huge; + size_t nactive_huge; + size_t ndirty_huge; uint64_t npurge_passes; uint64_t npurges; uint64_t nhugifies; + uint64_t nhugify_failures; uint64_t ndehugifies; - CTL_M2_GET("stats.arenas.0.hpa_shard.npurge_passes", - i, &npurge_passes, uint64_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.npurges", - i, &npurges, uint64_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.nhugifies", - i, &nhugifies, uint64_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.ndehugifies", - i, &ndehugifies, uint64_t); + CTL_M2_GET( + "stats.arenas.0.hpa_shard.npageslabs", i, &npageslabs, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.nactive", i, &nactive, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.ndirty", i, &ndirty, size_t); + + CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.npageslabs_nonhuge", i, + &npageslabs_nonhuge, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.nactive_nonhuge", i, + &nactive_nonhuge, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.ndirty_nonhuge", i, + &ndirty_nonhuge, size_t); + nretained_nonhuge = npageslabs_nonhuge * HUGEPAGE_PAGES + - nactive_nonhuge - ndirty_nonhuge; + + CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.npageslabs_huge", i, + &npageslabs_huge, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.nactive_huge", i, + &nactive_huge, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.ndirty_huge", i, + &ndirty_huge, size_t); + + CTL_M2_GET("stats.arenas.0.hpa_shard.npurge_passes", i, &npurge_passes, + uint64_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.npurges", i, &npurges, uint64_t); + CTL_M2_GET( + "stats.arenas.0.hpa_shard.nhugifies", i, &nhugifies, uint64_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.nhugify_failures", i, + &nhugify_failures, uint64_t); + CTL_M2_GET( + "stats.arenas.0.hpa_shard.ndehugifies", i, &ndehugifies, uint64_t); + + emitter_table_printf(emitter, + "HPA shard stats:\n" + " Pageslabs: %zu (%zu huge, %zu nonhuge)\n" + " Active pages: %zu (%zu huge, %zu nonhuge)\n" + " Dirty pages: %zu (%zu huge, %zu nonhuge)\n" + " Retained pages: %zu\n" + " Purge passes: %" FMTu64 " (%" FMTu64 + " / sec)\n" + " Purges: %" FMTu64 " (%" FMTu64 + " / sec)\n" + " Hugeifies: %" FMTu64 " (%" FMTu64 + " / sec)\n" + " Hugify failures: %" FMTu64 " (%" FMTu64 + " / sec)\n" + " Dehugifies: %" FMTu64 " (%" FMTu64 + " / sec)\n" + "\n", + npageslabs, npageslabs_huge, npageslabs_nonhuge, nactive, + nactive_huge, nactive_nonhuge, ndirty, ndirty_huge, ndirty_nonhuge, + nretained_nonhuge, npurge_passes, + rate_per_second(npurge_passes, uptime), npurges, + rate_per_second(npurges, uptime), nhugifies, + rate_per_second(nhugifies, uptime), nhugify_failures, + rate_per_second(nhugify_failures, uptime), ndehugifies, + rate_per_second(ndehugifies, uptime)); + + emitter_json_kv(emitter, "npageslabs", emitter_type_size, &npageslabs); + emitter_json_kv(emitter, "nactive", emitter_type_size, &nactive); + emitter_json_kv(emitter, "ndirty", emitter_type_size, &ndirty); + + emitter_json_kv( + emitter, "npurge_passes", emitter_type_uint64, &npurge_passes); + emitter_json_kv(emitter, "npurges", emitter_type_uint64, &npurges); + emitter_json_kv(emitter, "nhugifies", emitter_type_uint64, &nhugifies); + emitter_json_kv(emitter, "nhugify_failures", emitter_type_uint64, + &nhugify_failures); + emitter_json_kv( + emitter, "ndehugifies", emitter_type_uint64, &ndehugifies); + + emitter_json_object_kv_begin(emitter, "slabs"); + emitter_json_kv(emitter, "npageslabs_nonhuge", emitter_type_size, + &npageslabs_nonhuge); + emitter_json_kv( + emitter, "nactive_nonhuge", emitter_type_size, &nactive_nonhuge); + emitter_json_kv( + emitter, "ndirty_nonhuge", emitter_type_size, &ndirty_nonhuge); + emitter_json_kv(emitter, "nretained_nonhuge", emitter_type_size, + &nretained_nonhuge); + + emitter_json_kv( + emitter, "npageslabs_huge", emitter_type_size, &npageslabs_huge); + emitter_json_kv( + emitter, "nactive_huge", emitter_type_size, &nactive_huge); + emitter_json_kv( + emitter, "ndirty_huge", emitter_type_size, &ndirty_huge); + emitter_json_object_end(emitter); /* End "slabs" */ +} + +static void +stats_arena_hpa_shard_slabs_print(emitter_t *emitter, unsigned i) { + emitter_row_t header_row; + emitter_row_init(&header_row); + emitter_row_t row; + emitter_row_init(&row); size_t npageslabs_huge; size_t nactive_huge; @@ -814,48 +949,20 @@ stats_arena_hpa_shard_print(emitter_t *emitter, unsigned i, uint64_t uptime) { size_t ndirty_nonhuge; size_t nretained_nonhuge; - size_t sec_bytes; - CTL_M2_GET("stats.arenas.0.hpa_sec_bytes", i, &sec_bytes, size_t); - emitter_kv(emitter, "sec_bytes", "Bytes in small extent cache", - emitter_type_size, &sec_bytes); + /* Full slab stats. */ + CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.npageslabs_huge", i, + &npageslabs_huge, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.nactive_huge", i, + &nactive_huge, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.ndirty_huge", i, + &ndirty_huge, size_t); - /* First, global stats. */ - emitter_table_printf(emitter, - "HPA shard stats:\n" - " Purge passes: %" FMTu64 " (%" FMTu64 " / sec)\n" - " Purges: %" FMTu64 " (%" FMTu64 " / sec)\n" - " Hugeifies: %" FMTu64 " (%" FMTu64 " / sec)\n" - " Dehugifies: %" FMTu64 " (%" FMTu64 " / sec)\n" - "\n", - npurge_passes, rate_per_second(npurge_passes, uptime), - npurges, rate_per_second(npurges, uptime), - nhugifies, rate_per_second(nhugifies, uptime), - ndehugifies, rate_per_second(ndehugifies, uptime)); - - emitter_json_object_kv_begin(emitter, "hpa_shard"); - emitter_json_kv(emitter, "npurge_passes", emitter_type_uint64, - &npurge_passes); - emitter_json_kv(emitter, "npurges", emitter_type_uint64, - &npurges); - emitter_json_kv(emitter, "nhugifies", emitter_type_uint64, - &nhugifies); - emitter_json_kv(emitter, "ndehugifies", emitter_type_uint64, - &ndehugifies); - - /* Next, full slab stats. */ - CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.npageslabs_huge", - i, &npageslabs_huge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.nactive_huge", - i, &nactive_huge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.ndirty_huge", - i, &ndirty_huge, size_t); - - CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.npageslabs_nonhuge", - i, &npageslabs_nonhuge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.nactive_nonhuge", - i, &nactive_nonhuge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.ndirty_nonhuge", - i, &ndirty_nonhuge, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.npageslabs_nonhuge", i, + &npageslabs_nonhuge, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.nactive_nonhuge", i, + &nactive_nonhuge, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.ndirty_nonhuge", i, + &ndirty_nonhuge, size_t); nretained_nonhuge = npageslabs_nonhuge * HUGEPAGE_PAGES - nactive_nonhuge - ndirty_nonhuge; @@ -865,40 +972,40 @@ stats_arena_hpa_shard_print(emitter_t *emitter, unsigned i, uint64_t uptime) { " nactive: %zu huge, %zu nonhuge \n" " ndirty: %zu huge, %zu nonhuge \n" " nretained: 0 huge, %zu nonhuge \n", - npageslabs_huge, npageslabs_nonhuge, - nactive_huge, nactive_nonhuge, - ndirty_huge, ndirty_nonhuge, - nretained_nonhuge); + npageslabs_huge, npageslabs_nonhuge, nactive_huge, nactive_nonhuge, + ndirty_huge, ndirty_nonhuge, nretained_nonhuge); emitter_json_object_kv_begin(emitter, "full_slabs"); - emitter_json_kv(emitter, "npageslabs_huge", emitter_type_size, - &npageslabs_huge); - emitter_json_kv(emitter, "nactive_huge", emitter_type_size, - &nactive_huge); - emitter_json_kv(emitter, "nactive_huge", emitter_type_size, - &nactive_huge); + emitter_json_kv( + emitter, "npageslabs_huge", emitter_type_size, &npageslabs_huge); + emitter_json_kv( + emitter, "nactive_huge", emitter_type_size, &nactive_huge); + emitter_json_kv( + emitter, "ndirty_huge", emitter_type_size, &ndirty_huge); emitter_json_kv(emitter, "npageslabs_nonhuge", emitter_type_size, &npageslabs_nonhuge); - emitter_json_kv(emitter, "nactive_nonhuge", emitter_type_size, - &nactive_nonhuge); - emitter_json_kv(emitter, "ndirty_nonhuge", emitter_type_size, - &ndirty_nonhuge); + emitter_json_kv( + emitter, "nactive_nonhuge", emitter_type_size, &nactive_nonhuge); + emitter_json_kv( + emitter, "ndirty_nonhuge", emitter_type_size, &ndirty_nonhuge); + emitter_json_kv(emitter, "nretained_nonhuge", emitter_type_size, + &nretained_nonhuge); emitter_json_object_end(emitter); /* End "full_slabs" */ /* Next, empty slab stats. */ - CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.npageslabs_huge", - i, &npageslabs_huge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.nactive_huge", - i, &nactive_huge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.ndirty_huge", - i, &ndirty_huge, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.npageslabs_huge", i, + &npageslabs_huge, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.nactive_huge", i, + &nactive_huge, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.ndirty_huge", i, + &ndirty_huge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.npageslabs_nonhuge", - i, &npageslabs_nonhuge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.nactive_nonhuge", - i, &nactive_nonhuge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.ndirty_nonhuge", - i, &ndirty_nonhuge, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.npageslabs_nonhuge", i, + &npageslabs_nonhuge, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.nactive_nonhuge", i, + &nactive_nonhuge, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.ndirty_nonhuge", i, + &ndirty_nonhuge, size_t); nretained_nonhuge = npageslabs_nonhuge * HUGEPAGE_PAGES - nactive_nonhuge - ndirty_nonhuge; @@ -907,28 +1014,28 @@ stats_arena_hpa_shard_print(emitter_t *emitter, unsigned i, uint64_t uptime) { " npageslabs: %zu huge, %zu nonhuge\n" " nactive: %zu huge, %zu nonhuge \n" " ndirty: %zu huge, %zu nonhuge \n" - " nretained: 0 huge, %zu nonhuge \n" - "\n", - npageslabs_huge, npageslabs_nonhuge, - nactive_huge, nactive_nonhuge, - ndirty_huge, ndirty_nonhuge, - nretained_nonhuge); + " nretained: 0 huge, %zu nonhuge \n", + npageslabs_huge, npageslabs_nonhuge, nactive_huge, nactive_nonhuge, + ndirty_huge, ndirty_nonhuge, nretained_nonhuge); emitter_json_object_kv_begin(emitter, "empty_slabs"); - emitter_json_kv(emitter, "npageslabs_huge", emitter_type_size, - &npageslabs_huge); - emitter_json_kv(emitter, "nactive_huge", emitter_type_size, - &nactive_huge); - emitter_json_kv(emitter, "nactive_huge", emitter_type_size, - &nactive_huge); + emitter_json_kv( + emitter, "npageslabs_huge", emitter_type_size, &npageslabs_huge); + emitter_json_kv( + emitter, "nactive_huge", emitter_type_size, &nactive_huge); + emitter_json_kv( + emitter, "ndirty_huge", emitter_type_size, &ndirty_huge); emitter_json_kv(emitter, "npageslabs_nonhuge", emitter_type_size, &npageslabs_nonhuge); - emitter_json_kv(emitter, "nactive_nonhuge", emitter_type_size, - &nactive_nonhuge); - emitter_json_kv(emitter, "ndirty_nonhuge", emitter_type_size, - &ndirty_nonhuge); + emitter_json_kv( + emitter, "nactive_nonhuge", emitter_type_size, &nactive_nonhuge); + emitter_json_kv( + emitter, "ndirty_nonhuge", emitter_type_size, &ndirty_nonhuge); + emitter_json_kv(emitter, "nretained_nonhuge", emitter_type_size, + &nretained_nonhuge); emitter_json_object_end(emitter); /* End "empty_slabs" */ + /* Last, nonfull slab stats. */ COL_HDR(row, size, NULL, right, 20, size) COL_HDR(row, ind, NULL, right, 4, unsigned) COL_HDR(row, npageslabs_huge, NULL, right, 16, size) @@ -944,6 +1051,7 @@ stats_arena_hpa_shard_print(emitter_t *emitter, unsigned i, uint64_t uptime) { stats_arenas_mib[2] = i; CTL_LEAF_PREPARE(stats_arenas_mib, 3, "hpa_shard.nonfull_slabs"); + emitter_table_printf(emitter, " In nonfull slabs:\n"); emitter_table_row(emitter, &header_row); emitter_json_array_kv_begin(emitter, "nonfull_slabs"); bool in_gap = false; @@ -952,25 +1060,25 @@ stats_arena_hpa_shard_print(emitter_t *emitter, unsigned i, uint64_t uptime) { CTL_LEAF(stats_arenas_mib, 6, "npageslabs_huge", &npageslabs_huge, size_t); - CTL_LEAF(stats_arenas_mib, 6, "nactive_huge", - &nactive_huge, size_t); - CTL_LEAF(stats_arenas_mib, 6, "ndirty_huge", - &ndirty_huge, size_t); + CTL_LEAF( + stats_arenas_mib, 6, "nactive_huge", &nactive_huge, size_t); + CTL_LEAF( + stats_arenas_mib, 6, "ndirty_huge", &ndirty_huge, size_t); CTL_LEAF(stats_arenas_mib, 6, "npageslabs_nonhuge", &npageslabs_nonhuge, size_t); CTL_LEAF(stats_arenas_mib, 6, "nactive_nonhuge", &nactive_nonhuge, size_t); - CTL_LEAF(stats_arenas_mib, 6, "ndirty_nonhuge", - &ndirty_nonhuge, size_t); + CTL_LEAF(stats_arenas_mib, 6, "ndirty_nonhuge", &ndirty_nonhuge, + size_t); nretained_nonhuge = npageslabs_nonhuge * HUGEPAGE_PAGES - nactive_nonhuge - ndirty_nonhuge; bool in_gap_prev = in_gap; in_gap = (npageslabs_huge == 0 && npageslabs_nonhuge == 0); if (in_gap_prev && !in_gap) { - emitter_table_printf(emitter, - " ---\n"); + emitter_table_printf( + emitter, " ---\n"); } col_size.size_val = sz_pind2sz(j); @@ -989,27 +1097,38 @@ stats_arena_hpa_shard_print(emitter_t *emitter, unsigned i, uint64_t uptime) { emitter_json_object_begin(emitter); emitter_json_kv(emitter, "npageslabs_huge", emitter_type_size, &npageslabs_huge); - emitter_json_kv(emitter, "nactive_huge", emitter_type_size, - &nactive_huge); - emitter_json_kv(emitter, "ndirty_huge", emitter_type_size, - &ndirty_huge); - emitter_json_kv(emitter, "npageslabs_nonhuge", emitter_type_size, - &npageslabs_nonhuge); + emitter_json_kv( + emitter, "nactive_huge", emitter_type_size, &nactive_huge); + emitter_json_kv( + emitter, "ndirty_huge", emitter_type_size, &ndirty_huge); + emitter_json_kv(emitter, "npageslabs_nonhuge", + emitter_type_size, &npageslabs_nonhuge); emitter_json_kv(emitter, "nactive_nonhuge", emitter_type_size, &nactive_nonhuge); emitter_json_kv(emitter, "ndirty_nonhuge", emitter_type_size, &ndirty_nonhuge); + emitter_json_kv(emitter, "nretained_nonhuge", emitter_type_size, + &nretained_nonhuge); emitter_json_object_end(emitter); } emitter_json_array_end(emitter); /* End "nonfull_slabs" */ - emitter_json_object_end(emitter); /* End "hpa_shard" */ if (in_gap) { emitter_table_printf(emitter, " ---\n"); } } static void -stats_arena_mutexes_print(emitter_t *emitter, unsigned arena_ind, uint64_t uptime) { +stats_arena_hpa_shard_print(emitter_t *emitter, unsigned i, uint64_t uptime) { + emitter_json_object_kv_begin(emitter, "hpa_shard"); + stats_arena_hpa_shard_sec_print(emitter, i); + stats_arena_hpa_shard_counters_print(emitter, i, uptime); + stats_arena_hpa_shard_slabs_print(emitter, i); + emitter_json_object_end(emitter); /* End "hpa_shard" */ +} + +static void +stats_arena_mutexes_print( + emitter_t *emitter, unsigned arena_ind, uint64_t uptime) { emitter_row_t row; emitter_col_t col_name; emitter_col_t col64[mutex_prof_num_uint64_t_counters]; @@ -1030,8 +1149,8 @@ stats_arena_mutexes_print(emitter_t *emitter, unsigned arena_ind, uint64_t uptim i++) { const char *name = arena_mutex_names[i]; emitter_json_object_kv_begin(emitter, name); - mutex_stats_read_arena(stats_arenas_mib, 4, name, &col_name, - col64, col32, uptime); + mutex_stats_read_arena( + stats_arenas_mib, 4, name, &col_name, col64, col32, uptime); mutex_stats_emit(emitter, &row, col64, col32); emitter_json_object_end(emitter); /* Close the mutex dict. */ } @@ -1042,78 +1161,86 @@ JEMALLOC_COLD static void stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, bool mutex, bool extents, bool hpa) { - unsigned nthreads; + char name[ARENA_NAME_LEN]; + char *namep = name; + unsigned nthreads; const char *dss; - ssize_t dirty_decay_ms, muzzy_decay_ms; - size_t page, pactive, pdirty, pmuzzy, mapped, retained; - size_t base, internal, resident, metadata_thp, extent_avail; + ssize_t dirty_decay_ms, muzzy_decay_ms; + size_t page, pactive, pdirty, pmuzzy, mapped, retained; + size_t base, internal, resident, metadata_edata, metadata_rtree, + metadata_thp, extent_avail; uint64_t dirty_npurge, dirty_nmadvise, dirty_purged; uint64_t muzzy_npurge, muzzy_nmadvise, muzzy_purged; - size_t small_allocated; + size_t small_allocated; uint64_t small_nmalloc, small_ndalloc, small_nrequests, small_nfills, small_nflushes; - size_t large_allocated; + size_t large_allocated; uint64_t large_nmalloc, large_ndalloc, large_nrequests, large_nfills, large_nflushes; - size_t tcache_bytes, tcache_stashed_bytes, abandoned_vm; + size_t tcache_bytes, tcache_stashed_bytes, abandoned_vm; uint64_t uptime; CTL_GET("arenas.page", &page, size_t); + if (i != MALLCTL_ARENAS_ALL && i != MALLCTL_ARENAS_DESTROYED) { + CTL_M1_GET("arena.0.name", i, (void *)&namep, const char *); + emitter_kv( + emitter, "name", "name", emitter_type_string, &namep); + } CTL_M2_GET("stats.arenas.0.nthreads", i, &nthreads, unsigned); emitter_kv(emitter, "nthreads", "assigned threads", emitter_type_unsigned, &nthreads); CTL_M2_GET("stats.arenas.0.uptime", i, &uptime, uint64_t); - emitter_kv(emitter, "uptime_ns", "uptime", emitter_type_uint64, - &uptime); + emitter_kv( + emitter, "uptime_ns", "uptime", emitter_type_uint64, &uptime); CTL_M2_GET("stats.arenas.0.dss", i, &dss, const char *); emitter_kv(emitter, "dss", "dss allocation precedence", emitter_type_string, &dss); - CTL_M2_GET("stats.arenas.0.dirty_decay_ms", i, &dirty_decay_ms, - ssize_t); - CTL_M2_GET("stats.arenas.0.muzzy_decay_ms", i, &muzzy_decay_ms, - ssize_t); + CTL_M2_GET( + "stats.arenas.0.dirty_decay_ms", i, &dirty_decay_ms, ssize_t); + CTL_M2_GET( + "stats.arenas.0.muzzy_decay_ms", i, &muzzy_decay_ms, ssize_t); CTL_M2_GET("stats.arenas.0.pactive", i, &pactive, size_t); CTL_M2_GET("stats.arenas.0.pdirty", i, &pdirty, size_t); CTL_M2_GET("stats.arenas.0.pmuzzy", i, &pmuzzy, size_t); CTL_M2_GET("stats.arenas.0.dirty_npurge", i, &dirty_npurge, uint64_t); - CTL_M2_GET("stats.arenas.0.dirty_nmadvise", i, &dirty_nmadvise, - uint64_t); + CTL_M2_GET( + "stats.arenas.0.dirty_nmadvise", i, &dirty_nmadvise, uint64_t); CTL_M2_GET("stats.arenas.0.dirty_purged", i, &dirty_purged, uint64_t); CTL_M2_GET("stats.arenas.0.muzzy_npurge", i, &muzzy_npurge, uint64_t); - CTL_M2_GET("stats.arenas.0.muzzy_nmadvise", i, &muzzy_nmadvise, - uint64_t); + CTL_M2_GET( + "stats.arenas.0.muzzy_nmadvise", i, &muzzy_nmadvise, uint64_t); CTL_M2_GET("stats.arenas.0.muzzy_purged", i, &muzzy_purged, uint64_t); emitter_row_t decay_row; emitter_row_init(&decay_row); /* JSON-style emission. */ - emitter_json_kv(emitter, "dirty_decay_ms", emitter_type_ssize, - &dirty_decay_ms); - emitter_json_kv(emitter, "muzzy_decay_ms", emitter_type_ssize, - &muzzy_decay_ms); + emitter_json_kv( + emitter, "dirty_decay_ms", emitter_type_ssize, &dirty_decay_ms); + emitter_json_kv( + emitter, "muzzy_decay_ms", emitter_type_ssize, &muzzy_decay_ms); emitter_json_kv(emitter, "pactive", emitter_type_size, &pactive); emitter_json_kv(emitter, "pdirty", emitter_type_size, &pdirty); emitter_json_kv(emitter, "pmuzzy", emitter_type_size, &pmuzzy); - emitter_json_kv(emitter, "dirty_npurge", emitter_type_uint64, - &dirty_npurge); - emitter_json_kv(emitter, "dirty_nmadvise", emitter_type_uint64, - &dirty_nmadvise); - emitter_json_kv(emitter, "dirty_purged", emitter_type_uint64, - &dirty_purged); + emitter_json_kv( + emitter, "dirty_npurge", emitter_type_uint64, &dirty_npurge); + emitter_json_kv( + emitter, "dirty_nmadvise", emitter_type_uint64, &dirty_nmadvise); + emitter_json_kv( + emitter, "dirty_purged", emitter_type_uint64, &dirty_purged); - emitter_json_kv(emitter, "muzzy_npurge", emitter_type_uint64, - &muzzy_npurge); - emitter_json_kv(emitter, "muzzy_nmadvise", emitter_type_uint64, - &muzzy_nmadvise); - emitter_json_kv(emitter, "muzzy_purged", emitter_type_uint64, - &muzzy_purged); + emitter_json_kv( + emitter, "muzzy_npurge", emitter_type_uint64, &muzzy_npurge); + emitter_json_kv( + emitter, "muzzy_nmadvise", emitter_type_uint64, &muzzy_nmadvise); + emitter_json_kv( + emitter, "muzzy_purged", emitter_type_uint64, &muzzy_purged); /* Table-style emission. */ COL(decay_row, decay_type, right, 9, title); @@ -1230,12 +1357,12 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, col_count_nfills_ps.type = emitter_type_uint64; col_count_nflushes_ps.type = emitter_type_uint64; -#define GET_AND_EMIT_ALLOC_STAT(small_or_large, name, valtype) \ - CTL_M2_GET("stats.arenas.0." #small_or_large "." #name, i, \ - &small_or_large##_##name, valtype##_t); \ - emitter_json_kv(emitter, #name, emitter_type_##valtype, \ - &small_or_large##_##name); \ - col_count_##name.type = emitter_type_##valtype; \ +#define GET_AND_EMIT_ALLOC_STAT(small_or_large, name, valtype) \ + CTL_M2_GET("stats.arenas.0." #small_or_large "." #name, i, \ + &small_or_large##_##name, valtype##_t); \ + emitter_json_kv( \ + emitter, #name, emitter_type_##valtype, &small_or_large##_##name); \ + col_count_##name.type = emitter_type_##valtype; \ col_count_##name.valtype##_val = small_or_large##_##name; emitter_json_object_kv_begin(emitter, "small"); @@ -1243,20 +1370,20 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, GET_AND_EMIT_ALLOC_STAT(small, allocated, size) GET_AND_EMIT_ALLOC_STAT(small, nmalloc, uint64) - col_count_nmalloc_ps.uint64_val = - rate_per_second(col_count_nmalloc.uint64_val, uptime); + col_count_nmalloc_ps.uint64_val = rate_per_second( + col_count_nmalloc.uint64_val, uptime); GET_AND_EMIT_ALLOC_STAT(small, ndalloc, uint64) - col_count_ndalloc_ps.uint64_val = - rate_per_second(col_count_ndalloc.uint64_val, uptime); + col_count_ndalloc_ps.uint64_val = rate_per_second( + col_count_ndalloc.uint64_val, uptime); GET_AND_EMIT_ALLOC_STAT(small, nrequests, uint64) - col_count_nrequests_ps.uint64_val = - rate_per_second(col_count_nrequests.uint64_val, uptime); + col_count_nrequests_ps.uint64_val = rate_per_second( + col_count_nrequests.uint64_val, uptime); GET_AND_EMIT_ALLOC_STAT(small, nfills, uint64) - col_count_nfills_ps.uint64_val = - rate_per_second(col_count_nfills.uint64_val, uptime); + col_count_nfills_ps.uint64_val = rate_per_second( + col_count_nfills.uint64_val, uptime); GET_AND_EMIT_ALLOC_STAT(small, nflushes, uint64) - col_count_nflushes_ps.uint64_val = - rate_per_second(col_count_nflushes.uint64_val, uptime); + col_count_nflushes_ps.uint64_val = rate_per_second( + col_count_nflushes.uint64_val, uptime); emitter_table_row(emitter, &alloc_count_row); emitter_json_object_end(emitter); /* Close "small". */ @@ -1266,20 +1393,20 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, GET_AND_EMIT_ALLOC_STAT(large, allocated, size) GET_AND_EMIT_ALLOC_STAT(large, nmalloc, uint64) - col_count_nmalloc_ps.uint64_val = - rate_per_second(col_count_nmalloc.uint64_val, uptime); + col_count_nmalloc_ps.uint64_val = rate_per_second( + col_count_nmalloc.uint64_val, uptime); GET_AND_EMIT_ALLOC_STAT(large, ndalloc, uint64) - col_count_ndalloc_ps.uint64_val = - rate_per_second(col_count_ndalloc.uint64_val, uptime); + col_count_ndalloc_ps.uint64_val = rate_per_second( + col_count_ndalloc.uint64_val, uptime); GET_AND_EMIT_ALLOC_STAT(large, nrequests, uint64) - col_count_nrequests_ps.uint64_val = - rate_per_second(col_count_nrequests.uint64_val, uptime); + col_count_nrequests_ps.uint64_val = rate_per_second( + col_count_nrequests.uint64_val, uptime); GET_AND_EMIT_ALLOC_STAT(large, nfills, uint64) - col_count_nfills_ps.uint64_val = - rate_per_second(col_count_nfills.uint64_val, uptime); + col_count_nfills_ps.uint64_val = rate_per_second( + col_count_nfills.uint64_val, uptime); GET_AND_EMIT_ALLOC_STAT(large, nflushes, uint64) - col_count_nflushes_ps.uint64_val = - rate_per_second(col_count_nflushes.uint64_val, uptime); + col_count_nflushes_ps.uint64_val = rate_per_second( + col_count_nflushes.uint64_val, uptime); emitter_table_row(emitter, &alloc_count_row); emitter_json_object_end(emitter); /* Close "large". */ @@ -1294,16 +1421,16 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, col_count_nrequests.uint64_val = small_nrequests + large_nrequests; col_count_nfills.uint64_val = small_nfills + large_nfills; col_count_nflushes.uint64_val = small_nflushes + large_nflushes; - col_count_nmalloc_ps.uint64_val = - rate_per_second(col_count_nmalloc.uint64_val, uptime); - col_count_ndalloc_ps.uint64_val = - rate_per_second(col_count_ndalloc.uint64_val, uptime); - col_count_nrequests_ps.uint64_val = - rate_per_second(col_count_nrequests.uint64_val, uptime); - col_count_nfills_ps.uint64_val = - rate_per_second(col_count_nfills.uint64_val, uptime); - col_count_nflushes_ps.uint64_val = - rate_per_second(col_count_nflushes.uint64_val, uptime); + col_count_nmalloc_ps.uint64_val = rate_per_second( + col_count_nmalloc.uint64_val, uptime); + col_count_ndalloc_ps.uint64_val = rate_per_second( + col_count_ndalloc.uint64_val, uptime); + col_count_nrequests_ps.uint64_val = rate_per_second( + col_count_nrequests.uint64_val, uptime); + col_count_nfills_ps.uint64_val = rate_per_second( + col_count_nfills.uint64_val, uptime); + col_count_nflushes_ps.uint64_val = rate_per_second( + col_count_nflushes.uint64_val, uptime); emitter_table_row(emitter, &alloc_count_row); emitter_row_t mem_count_row; @@ -1331,17 +1458,19 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, mem_count_val.size_val = pactive * page; emitter_table_row(emitter, &mem_count_row); -#define GET_AND_EMIT_MEM_STAT(stat) \ - CTL_M2_GET("stats.arenas.0."#stat, i, &stat, size_t); \ - emitter_json_kv(emitter, #stat, emitter_type_size, &stat); \ - mem_count_title.str_val = #stat":"; \ - mem_count_val.size_val = stat; \ +#define GET_AND_EMIT_MEM_STAT(stat) \ + CTL_M2_GET("stats.arenas.0." #stat, i, &stat, size_t); \ + emitter_json_kv(emitter, #stat, emitter_type_size, &stat); \ + mem_count_title.str_val = #stat ":"; \ + mem_count_val.size_val = stat; \ emitter_table_row(emitter, &mem_count_row); GET_AND_EMIT_MEM_STAT(mapped) GET_AND_EMIT_MEM_STAT(retained) GET_AND_EMIT_MEM_STAT(base) GET_AND_EMIT_MEM_STAT(internal) + GET_AND_EMIT_MEM_STAT(metadata_edata) + GET_AND_EMIT_MEM_STAT(metadata_rtree) GET_AND_EMIT_MEM_STAT(metadata_thp) GET_AND_EMIT_MEM_STAT(tcache_bytes) GET_AND_EMIT_MEM_STAT(tcache_stashed_bytes) @@ -1371,13 +1500,13 @@ JEMALLOC_COLD static void stats_general_print(emitter_t *emitter) { const char *cpv; - bool bv, bv2; - unsigned uv; - uint32_t u32v; - uint64_t u64v; - int64_t i64v; - ssize_t ssv, ssv2; - size_t sv, bsz, usz, u32sz, u64sz, i64sz, ssz, sssz, cpsz; + bool bv, bv2; + unsigned uv; + uint32_t u32v; + uint64_t u64v; + int64_t i64v; + ssize_t ssv, ssv2; + size_t sv, bsz, usz, u32sz, u64sz, i64sz, ssz, sssz, cpsz; bsz = sizeof(bool); usz = sizeof(unsigned); @@ -1393,11 +1522,11 @@ stats_general_print(emitter_t *emitter) { /* config. */ emitter_dict_begin(emitter, "config", "Build-time option settings"); -#define CONFIG_WRITE_BOOL(name) \ - do { \ - CTL_GET("config."#name, &bv, bool); \ - emitter_kv(emitter, #name, "config."#name, \ - emitter_type_bool, &bv); \ +#define CONFIG_WRITE_BOOL(name) \ + do { \ + CTL_GET("config." #name, &bv, bool); \ + emitter_kv( \ + emitter, #name, "config." #name, emitter_type_bool, &bv); \ } while (0) CONFIG_WRITE_BOOL(cache_oblivious); @@ -1411,59 +1540,103 @@ stats_general_print(emitter_t *emitter) { CONFIG_WRITE_BOOL(prof); CONFIG_WRITE_BOOL(prof_libgcc); CONFIG_WRITE_BOOL(prof_libunwind); + CONFIG_WRITE_BOOL(prof_frameptr); CONFIG_WRITE_BOOL(stats); CONFIG_WRITE_BOOL(utrace); CONFIG_WRITE_BOOL(xmalloc); #undef CONFIG_WRITE_BOOL emitter_dict_end(emitter); /* Close "config" dict. */ + /* system. */ + emitter_dict_begin(emitter, "system", "System configuration"); + + /* + * This shows system's THP mode detected at jemalloc's init time. + * jemalloc does not re-detect the mode even if it changes after + * jemalloc's init. It is assumed that system's THP mode is stable + * during the process's lifetime and a violation could lead to + * undefined behavior. + */ + const char *thp_mode_name = system_thp_mode_names[init_system_thp_mode]; + emitter_kv(emitter, "thp_mode", "system.thp_mode", emitter_type_string, + &thp_mode_name); + + emitter_dict_end(emitter); /* Close "system". */ + /* opt. */ -#define OPT_WRITE(name, var, size, emitter_type) \ - if (je_mallctl("opt."name, (void *)&var, &size, NULL, 0) == \ - 0) { \ - emitter_kv(emitter, name, "opt."name, emitter_type, \ - &var); \ +#define OPT_WRITE(name, var, size, emitter_type) \ + if (je_mallctl("opt." name, (void *)&var, &size, NULL, 0) == 0) { \ + emitter_kv(emitter, name, "opt." name, emitter_type, &var); \ } -#define OPT_WRITE_MUTABLE(name, var1, var2, size, emitter_type, \ - altname) \ - if (je_mallctl("opt."name, (void *)&var1, &size, NULL, 0) == \ - 0 && je_mallctl(altname, (void *)&var2, &size, NULL, 0) \ - == 0) { \ - emitter_kv_note(emitter, name, "opt."name, \ - emitter_type, &var1, altname, emitter_type, \ - &var2); \ +#define OPT_WRITE_MUTABLE(name, var1, var2, size, emitter_type, altname) \ + if (je_mallctl("opt." name, (void *)&var1, &size, NULL, 0) == 0 \ + && je_mallctl(altname, (void *)&var2, &size, NULL, 0) == 0) { \ + emitter_kv_note(emitter, name, "opt." name, emitter_type, \ + &var1, altname, emitter_type, &var2); \ } #define OPT_WRITE_BOOL(name) OPT_WRITE(name, bv, bsz, emitter_type_bool) -#define OPT_WRITE_BOOL_MUTABLE(name, altname) \ +#define OPT_WRITE_BOOL_MUTABLE(name, altname) \ OPT_WRITE_MUTABLE(name, bv, bv2, bsz, emitter_type_bool, altname) -#define OPT_WRITE_UNSIGNED(name) \ - OPT_WRITE(name, uv, usz, emitter_type_unsigned) +#define OPT_WRITE_UNSIGNED(name) OPT_WRITE(name, uv, usz, emitter_type_unsigned) -#define OPT_WRITE_INT64(name) \ - OPT_WRITE(name, i64v, i64sz, emitter_type_int64) -#define OPT_WRITE_UINT64(name) \ - OPT_WRITE(name, u64v, u64sz, emitter_type_uint64) +#define OPT_WRITE_INT64(name) OPT_WRITE(name, i64v, i64sz, emitter_type_int64) +#define OPT_WRITE_UINT64(name) OPT_WRITE(name, u64v, u64sz, emitter_type_uint64) -#define OPT_WRITE_SIZE_T(name) \ - OPT_WRITE(name, sv, ssz, emitter_type_size) -#define OPT_WRITE_SSIZE_T(name) \ - OPT_WRITE(name, ssv, sssz, emitter_type_ssize) -#define OPT_WRITE_SSIZE_T_MUTABLE(name, altname) \ - OPT_WRITE_MUTABLE(name, ssv, ssv2, sssz, emitter_type_ssize, \ - altname) +#define OPT_WRITE_SIZE_T(name) OPT_WRITE(name, sv, ssz, emitter_type_size) +#define OPT_WRITE_SSIZE_T(name) OPT_WRITE(name, ssv, sssz, emitter_type_ssize) +#define OPT_WRITE_SSIZE_T_MUTABLE(name, altname) \ + OPT_WRITE_MUTABLE(name, ssv, ssv2, sssz, emitter_type_ssize, altname) -#define OPT_WRITE_CHAR_P(name) \ - OPT_WRITE(name, cpv, cpsz, emitter_type_string) +#define OPT_WRITE_CHAR_P(name) OPT_WRITE(name, cpv, cpsz, emitter_type_string) emitter_dict_begin(emitter, "opt", "Run-time option settings"); + /* + * opt.malloc_conf. + * + * Sources are documented in https://jemalloc.net/jemalloc.3.html#tuning + * - (Not Included Here) The string specified via --with-malloc-conf, + * which is already printed out above as config.malloc_conf + * - (Included) The string pointed to by the global variable malloc_conf + * - (Included) The “name” of the file referenced by the symbolic link + * named /etc/malloc.conf + * - (Included) The value of the environment variable MALLOC_CONF + * - (Optional, Unofficial) The string pointed to by the global variable + * malloc_conf_2_conf_harder, which is hidden from the public. + * + * Note: The outputs are strictly ordered by priorities (low -> high). + * + */ +#define MALLOC_CONF_WRITE(name, message) \ + if (je_mallctl("opt.malloc_conf." name, (void *)&cpv, &cpsz, NULL, 0) \ + != 0) { \ + cpv = ""; \ + } \ + emitter_kv(emitter, name, message, emitter_type_string, &cpv); + + MALLOC_CONF_WRITE("global_var", "Global variable malloc_conf"); + MALLOC_CONF_WRITE("symlink", "Symbolic link malloc.conf"); + MALLOC_CONF_WRITE("env_var", "Environment variable MALLOC_CONF"); + /* As this config is unofficial, skip the output if it's NULL */ + if (je_mallctl("opt.malloc_conf.global_var_2_conf_harder", (void *)&cpv, + &cpsz, NULL, 0) + == 0) { + emitter_kv(emitter, "global_var_2_conf_harder", + "Global " + "variable malloc_conf_2_conf_harder", + emitter_type_string, &cpv); + } +#undef MALLOC_CONF_WRITE + OPT_WRITE_BOOL("abort") OPT_WRITE_BOOL("abort_conf") OPT_WRITE_BOOL("cache_oblivious") OPT_WRITE_BOOL("confirm_conf") + OPT_WRITE_BOOL("experimental_hpa_start_huge_if_thp_always") + OPT_WRITE_BOOL("experimental_hpa_enforce_hugify") OPT_WRITE_BOOL("retain") OPT_WRITE_CHAR_P("dss") OPT_WRITE_UNSIGNED("narenas") @@ -1473,7 +1646,9 @@ stats_general_print(emitter_t *emitter) { OPT_WRITE_SIZE_T("hpa_slab_max_alloc") OPT_WRITE_SIZE_T("hpa_hugification_threshold") OPT_WRITE_UINT64("hpa_hugify_delay_ms") + OPT_WRITE_BOOL("hpa_hugify_sync") OPT_WRITE_UINT64("hpa_min_purge_interval_ms") + OPT_WRITE_SSIZE_T("experimental_hpa_max_purge_nhp") if (je_mallctl("opt.hpa_dirty_mult", (void *)&u32v, &u32sz, NULL, 0) == 0) { /* @@ -1492,11 +1667,14 @@ stats_general_print(emitter_t *emitter) { "opt.hpa_dirty_mult", emitter_type_string, &bufp); } } + OPT_WRITE_SIZE_T("hpa_purge_threshold") + OPT_WRITE_UINT64("hpa_min_purge_delay_ms") + OPT_WRITE_CHAR_P("hpa_hugify_style") OPT_WRITE_SIZE_T("hpa_sec_nshards") OPT_WRITE_SIZE_T("hpa_sec_max_alloc") OPT_WRITE_SIZE_T("hpa_sec_max_bytes") - OPT_WRITE_SIZE_T("hpa_sec_bytes_after_flush") OPT_WRITE_SIZE_T("hpa_sec_batch_fill_extra") + OPT_WRITE_BOOL("huge_arena_pac_thp") OPT_WRITE_CHAR_P("metadata_thp") OPT_WRITE_INT64("mutex_max_spin") OPT_WRITE_BOOL_MUTABLE("background_thread", "background_thread") @@ -1508,6 +1686,7 @@ stats_general_print(emitter_t *emitter) { OPT_WRITE_BOOL("utrace") OPT_WRITE_BOOL("xmalloc") OPT_WRITE_BOOL("experimental_infallible_new") + OPT_WRITE_BOOL("experimental_tcache_gc") OPT_WRITE_BOOL("tcache") OPT_WRITE_SIZE_T("tcache_max") OPT_WRITE_UNSIGNED("tcache_nslots_small_min") @@ -1518,12 +1697,14 @@ stats_general_print(emitter_t *emitter) { OPT_WRITE_SIZE_T("tcache_gc_delay_bytes") OPT_WRITE_UNSIGNED("lg_tcache_flush_small_div") OPT_WRITE_UNSIGNED("lg_tcache_flush_large_div") + OPT_WRITE_UNSIGNED("debug_double_free_max_scan") OPT_WRITE_CHAR_P("thp") OPT_WRITE_BOOL("prof") + OPT_WRITE_UNSIGNED("prof_bt_max") OPT_WRITE_CHAR_P("prof_prefix") OPT_WRITE_BOOL_MUTABLE("prof_active", "prof.active") - OPT_WRITE_BOOL_MUTABLE("prof_thread_active_init", - "prof.thread_active_init") + OPT_WRITE_BOOL_MUTABLE( + "prof_thread_active_init", "prof.thread_active_init") OPT_WRITE_SSIZE_T_MUTABLE("lg_prof_sample", "prof.lg_sample") OPT_WRITE_BOOL("prof_accum") OPT_WRITE_SSIZE_T("lg_prof_interval") @@ -1538,8 +1719,10 @@ stats_general_print(emitter_t *emitter) { OPT_WRITE_INT64("stats_interval") OPT_WRITE_CHAR_P("stats_interval_opts") OPT_WRITE_CHAR_P("zero_realloc") + OPT_WRITE_SIZE_T("process_madvise_max_batch") + OPT_WRITE_BOOL("disable_large_size_classes") - emitter_dict_end(emitter); + emitter_dict_end(emitter); /* Close "opt". */ #undef OPT_WRITE #undef OPT_WRITE_MUTABLE @@ -1559,12 +1742,12 @@ stats_general_print(emitter_t *emitter) { "prof.thread_active_init", emitter_type_bool, &bv); CTL_GET("prof.active", &bv, bool); - emitter_kv(emitter, "active", "prof.active", emitter_type_bool, - &bv); + emitter_kv( + emitter, "active", "prof.active", emitter_type_bool, &bv); CTL_GET("prof.gdump", &bv, bool); - emitter_kv(emitter, "gdump", "prof.gdump", emitter_type_bool, - &bv); + emitter_kv( + emitter, "gdump", "prof.gdump", emitter_type_bool, &bv); CTL_GET("prof.interval", &u64v, uint64_t); emitter_kv(emitter, "interval", "prof.interval", @@ -1603,6 +1786,10 @@ stats_general_print(emitter_t *emitter) { CTL_GET("arenas.page", &sv, size_t); emitter_kv(emitter, "page", "Page size", emitter_type_size, &sv); + CTL_GET("arenas.hugepage", &sv, size_t); + emitter_kv( + emitter, "hugepage", "Hugepage size", emitter_type_size, &sv); + if (je_mallctl("arenas.tcache_max", (void *)&sv, &ssz, NULL, 0) == 0) { emitter_kv(emitter, "tcache_max", "Maximum thread-cached size class", emitter_type_size, &sv); @@ -1631,20 +1818,20 @@ stats_general_print(emitter_t *emitter) { emitter_json_object_begin(emitter); CTL_LEAF(arenas_bin_mib, 3, "size", &sv, size_t); - emitter_json_kv(emitter, "size", emitter_type_size, - &sv); + emitter_json_kv( + emitter, "size", emitter_type_size, &sv); CTL_LEAF(arenas_bin_mib, 3, "nregs", &u32v, uint32_t); - emitter_json_kv(emitter, "nregs", emitter_type_uint32, - &u32v); + emitter_json_kv( + emitter, "nregs", emitter_type_uint32, &u32v); CTL_LEAF(arenas_bin_mib, 3, "slab_size", &sv, size_t); - emitter_json_kv(emitter, "slab_size", emitter_type_size, - &sv); + emitter_json_kv( + emitter, "slab_size", emitter_type_size, &sv); CTL_LEAF(arenas_bin_mib, 3, "nshards", &u32v, uint32_t); - emitter_json_kv(emitter, "nshards", emitter_type_uint32, - &u32v); + emitter_json_kv( + emitter, "nshards", emitter_type_uint32, &u32v); emitter_json_object_end(emitter); } @@ -1665,8 +1852,8 @@ stats_general_print(emitter_t *emitter) { emitter_json_object_begin(emitter); CTL_LEAF(arenas_lextent_mib, 3, "size", &sv, size_t); - emitter_json_kv(emitter, "size", emitter_type_size, - &sv); + emitter_json_kv( + emitter, "size", emitter_type_size, &sv); emitter_json_object_end(emitter); } @@ -1684,15 +1871,17 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, * These should be deleted. We keep them around for a while, to aid in * the transition to the emitter code. */ - size_t allocated, active, metadata, metadata_thp, resident, mapped, - retained; - size_t num_background_threads; - size_t zero_reallocs; + size_t allocated, active, metadata, metadata_edata, metadata_rtree, + metadata_thp, resident, mapped, retained; + size_t num_background_threads; + size_t zero_reallocs; uint64_t background_thread_num_runs, background_thread_run_interval; CTL_GET("stats.allocated", &allocated, size_t); CTL_GET("stats.active", &active, size_t); CTL_GET("stats.metadata", &metadata, size_t); + CTL_GET("stats.metadata_edata", &metadata_edata, size_t); + CTL_GET("stats.metadata_rtree", &metadata_rtree, size_t); CTL_GET("stats.metadata_thp", &metadata_thp, size_t); CTL_GET("stats.resident", &resident, size_t); CTL_GET("stats.mapped", &mapped, size_t); @@ -1718,18 +1907,24 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, emitter_json_kv(emitter, "allocated", emitter_type_size, &allocated); emitter_json_kv(emitter, "active", emitter_type_size, &active); emitter_json_kv(emitter, "metadata", emitter_type_size, &metadata); - emitter_json_kv(emitter, "metadata_thp", emitter_type_size, - &metadata_thp); + emitter_json_kv( + emitter, "metadata_edata", emitter_type_size, &metadata_edata); + emitter_json_kv( + emitter, "metadata_rtree", emitter_type_size, &metadata_rtree); + emitter_json_kv( + emitter, "metadata_thp", emitter_type_size, &metadata_thp); emitter_json_kv(emitter, "resident", emitter_type_size, &resident); emitter_json_kv(emitter, "mapped", emitter_type_size, &mapped); emitter_json_kv(emitter, "retained", emitter_type_size, &retained); - emitter_json_kv(emitter, "zero_reallocs", emitter_type_size, - &zero_reallocs); + emitter_json_kv( + emitter, "zero_reallocs", emitter_type_size, &zero_reallocs); - emitter_table_printf(emitter, "Allocated: %zu, active: %zu, " - "metadata: %zu (n_thp %zu), resident: %zu, mapped: %zu, " - "retained: %zu\n", allocated, active, metadata, metadata_thp, - resident, mapped, retained); + emitter_table_printf(emitter, + "Allocated: %zu, active: %zu, " + "metadata: %zu (n_thp %zu, edata %zu, rtree %zu), resident: %zu, " + "mapped: %zu, retained: %zu\n", + allocated, active, metadata, metadata_thp, metadata_edata, + metadata_rtree, resident, mapped, retained); /* Strange behaviors */ emitter_table_printf(emitter, @@ -1737,16 +1932,17 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, /* Background thread stats. */ emitter_json_object_kv_begin(emitter, "background_thread"); - emitter_json_kv(emitter, "num_threads", emitter_type_size, - &num_background_threads); + emitter_json_kv( + emitter, "num_threads", emitter_type_size, &num_background_threads); emitter_json_kv(emitter, "num_runs", emitter_type_uint64, &background_thread_num_runs); emitter_json_kv(emitter, "run_interval", emitter_type_uint64, &background_thread_run_interval); emitter_json_object_end(emitter); /* Close "background_thread". */ - emitter_table_printf(emitter, "Background threads: %zu, " - "num_runs: %"FMTu64", run_interval: %"FMTu64" ns\n", + emitter_table_printf(emitter, + "Background threads: %zu, " + "num_runs: %" FMTu64 ", run_interval: %" FMTu64 " ns\n", num_background_threads, background_thread_num_runs, background_thread_run_interval); @@ -1755,7 +1951,7 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, emitter_col_t name; emitter_col_t col64[mutex_prof_num_uint64_t_counters]; emitter_col_t col32[mutex_prof_num_uint32_t_counters]; - uint64_t uptime; + uint64_t uptime; emitter_row_init(&row); mutex_stats_init_cols(&row, "", &name, col64, col32); @@ -1770,7 +1966,8 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, for (int i = 0; i < mutex_prof_num_global_mutexes; i++) { mutex_stats_read_global(stats_mutexes_mib, 2, global_mutex_names[i], &name, col64, col32, uptime); - emitter_json_object_kv_begin(emitter, global_mutex_names[i]); + emitter_json_object_kv_begin( + emitter, global_mutex_names[i]); mutex_stats_emit(emitter, &row, col64, col32); emitter_json_object_end(emitter); } @@ -1789,24 +1986,24 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, size_t mib[3]; size_t miblen = sizeof(mib) / sizeof(size_t); size_t sz; - VARIABLE_ARRAY(bool, initialized, narenas); - bool destroyed_initialized; - unsigned i, j, ninitialized; + VARIABLE_ARRAY_UNSAFE(bool, initialized, narenas); + bool destroyed_initialized; + unsigned i, ninitialized; xmallctlnametomib("arena.0.initialized", mib, &miblen); for (i = ninitialized = 0; i < narenas; i++) { mib[1] = i; sz = sizeof(bool); - xmallctlbymib(mib, miblen, &initialized[i], &sz, - NULL, 0); + xmallctlbymib( + mib, miblen, &initialized[i], &sz, NULL, 0); if (initialized[i]) { ninitialized++; } } mib[1] = MALLCTL_ARENAS_DESTROYED; sz = sizeof(bool); - xmallctlbymib(mib, miblen, &destroyed_initialized, &sz, - NULL, 0); + xmallctlbymib( + mib, miblen, &destroyed_initialized, &sz, NULL, 0); /* Merged stats. */ if (merged && (ninitialized > 1 || !unmerged)) { @@ -1821,23 +2018,24 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, /* Destroyed stats. */ if (destroyed_initialized && destroyed) { /* Print destroyed arena stats. */ - emitter_table_printf(emitter, - "Destroyed arenas stats:\n"); + emitter_table_printf( + emitter, "Destroyed arenas stats:\n"); emitter_json_object_kv_begin(emitter, "destroyed"); stats_arena_print(emitter, MALLCTL_ARENAS_DESTROYED, bins, large, mutex, extents, hpa); - emitter_json_object_end(emitter); /* Close "destroyed". */ + emitter_json_object_end( + emitter); /* Close "destroyed". */ } /* Unmerged stats. */ if (unmerged) { - for (i = j = 0; i < narenas; i++) { + for (i = 0; i < narenas; i++) { if (initialized[i]) { char arena_ind_str[20]; malloc_snprintf(arena_ind_str, sizeof(arena_ind_str), "%u", i); - emitter_json_object_kv_begin(emitter, - arena_ind_str); + emitter_json_object_kv_begin( + emitter, arena_ind_str); emitter_table_printf(emitter, "arenas[%s]:\n", arena_ind_str); stats_arena_print(emitter, i, bins, @@ -1853,9 +2051,9 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, void stats_print(write_cb_t *write_cb, void *cbopaque, const char *opts) { - int err; + int err; uint64_t epoch; - size_t u64sz; + size_t u64sz; #define OPTION(o, v, d, s) bool v = d; STATS_PRINT_OPTIONS #undef OPTION @@ -1869,15 +2067,17 @@ stats_print(write_cb_t *write_cb, void *cbopaque, const char *opts) { * */ epoch = 1; u64sz = sizeof(uint64_t); - err = je_mallctl("epoch", (void *)&epoch, &u64sz, (void *)&epoch, - sizeof(uint64_t)); + err = je_mallctl( + "epoch", (void *)&epoch, &u64sz, (void *)&epoch, sizeof(uint64_t)); if (err != 0) { if (err == EAGAIN) { - malloc_write(": Memory allocation failure in " + malloc_write( + ": Memory allocation failure in " "mallctl(\"epoch\", ...)\n"); return; } - malloc_write(": Failure in mallctl(\"epoch\", " + malloc_write( + ": Failure in mallctl(\"epoch\", " "...)\n"); abort(); } @@ -1885,7 +2085,10 @@ stats_print(write_cb_t *write_cb, void *cbopaque, const char *opts) { if (opts != NULL) { for (unsigned i = 0; opts[i] != '\0'; i++) { switch (opts[i]) { -#define OPTION(o, v, d, s) case o: v = s; break; +#define OPTION(o, v, d, s) \ + case o: \ + v = s; \ + break; STATS_PRINT_OPTIONS #undef OPTION default:; @@ -1895,8 +2098,8 @@ stats_print(write_cb_t *write_cb, void *cbopaque, const char *opts) { emitter_t emitter; emitter_init(&emitter, - json ? emitter_output_json_compact : emitter_output_table, - write_cb, cbopaque); + json ? emitter_output_json_compact : emitter_output_table, write_cb, + cbopaque); emitter_begin(&emitter); emitter_table_printf(&emitter, "___ Begin jemalloc statistics ___\n"); emitter_json_object_kv_begin(&emitter, "jemalloc"); @@ -1905,8 +2108,8 @@ stats_print(write_cb_t *write_cb, void *cbopaque, const char *opts) { stats_general_print(&emitter); } if (config_stats) { - stats_print_helper(&emitter, merged, destroyed, unmerged, - bins, large, mutex, extents, hpa); + stats_print_helper(&emitter, merged, destroyed, unmerged, bins, + large, mutex, extents, hpa); } emitter_json_object_end(&emitter); /* Closes the "jemalloc" dict. */ @@ -1925,14 +2128,31 @@ stats_interval_postponed_event_wait(tsd_t *tsd) { } void -stats_interval_event_handler(tsd_t *tsd, uint64_t elapsed) { +stats_interval_event_handler(tsd_t *tsd) { + uint64_t last_event = thread_allocated_last_event_get(tsd); + uint64_t last_sample_event = tsd_stats_interval_last_event_get(tsd); + tsd_stats_interval_last_event_set(tsd, last_event); + uint64_t elapsed = last_event - last_sample_event; + assert(elapsed > 0 && elapsed != TE_INVALID_ELAPSED); - if (counter_accum(tsd_tsdn(tsd), &stats_interval_accumulated, - elapsed)) { + if (counter_accum( + tsd_tsdn(tsd), &stats_interval_accumulated, elapsed)) { je_malloc_stats_print(NULL, NULL, opt_stats_interval_opts); } } +static te_enabled_t +stats_interval_enabled(void) { + return opt_stats_interval >= 0 ? te_enabled_yes : te_enabled_no; +} + +te_base_cb_t stats_interval_te_handler = { + .enabled = &stats_interval_enabled, + .new_event_wait = &stats_interval_new_event_wait, + .postponed_event_wait = &stats_interval_postponed_event_wait, + .event_handler = &stats_interval_event_handler, +}; + bool stats_boot(void) { uint64_t stats_interval; @@ -1940,12 +2160,12 @@ stats_boot(void) { assert(opt_stats_interval == -1); stats_interval = 0; stats_interval_accum_batch = 0; - } else{ + } else { /* See comments in stats.h */ - stats_interval = (opt_stats_interval > 0) ? - opt_stats_interval : 1; - uint64_t batch = stats_interval >> - STATS_INTERVAL_ACCUM_LG_BATCH_SIZE; + stats_interval = (opt_stats_interval > 0) ? opt_stats_interval + : 1; + uint64_t batch = stats_interval + >> STATS_INTERVAL_ACCUM_LG_BATCH_SIZE; if (batch > STATS_INTERVAL_ACCUM_BATCH_MAX) { batch = STATS_INTERVAL_ACCUM_BATCH_MAX; } else if (batch == 0) { diff --git a/dep/jemalloc/src/sz.c b/dep/jemalloc/src/sz.c index d3115dda7..da92f2b41 100644 --- a/dep/jemalloc/src/sz.c +++ b/dep/jemalloc/src/sz.c @@ -3,12 +3,12 @@ #include "jemalloc/internal/sz.h" JEMALLOC_ALIGNED(CACHELINE) -size_t sz_pind2sz_tab[SC_NPSIZES+1]; +size_t sz_pind2sz_tab[SC_NPSIZES + 1]; size_t sz_large_pad; size_t sz_psz_quantize_floor(size_t size) { - size_t ret; + size_t ret; pszind_t pind; assert(size > 0); @@ -47,8 +47,8 @@ sz_psz_quantize_ceil(size_t size) { * search would potentially find sufficiently aligned available * memory somewhere lower. */ - ret = sz_pind2sz(sz_psz2ind(ret - sz_large_pad + 1)) + - sz_large_pad; + ret = sz_pind2sz(sz_psz2ind(ret - sz_large_pad + 1)) + + sz_large_pad; } return ret; } @@ -65,7 +65,7 @@ sz_boot_pind2sz_tab(const sc_data_t *sc_data) { } } for (int i = pind; i <= (int)SC_NPSIZES; i++) { - sz_pind2sz_tab[pind] = sc_data->large_maxclass + PAGE; + sz_pind2sz_tab[i] = sc_data->large_maxclass + PAGE; } } @@ -95,12 +95,13 @@ sz_boot_size2index_tab(const sc_data_t *sc_data) { for (unsigned sc_ind = 0; sc_ind < SC_NSIZES && dst_ind < dst_max; sc_ind++) { const sc_t *sc = &sc_data->sc[sc_ind]; - size_t sz = (ZU(1) << sc->lg_base) + size_t sz = (ZU(1) << sc->lg_base) + (ZU(sc->ndelta) << sc->lg_delta); size_t max_ind = ((sz + (ZU(1) << SC_LG_TINY_MIN) - 1) - >> SC_LG_TINY_MIN); + >> SC_LG_TINY_MIN); for (; dst_ind <= max_ind && dst_ind < dst_max; dst_ind++) { - sz_size2index_tab[dst_ind] = sc_ind; + assert(sc_ind < 1 << (sizeof(uint8_t) * 8)); + sz_size2index_tab[dst_ind] = (uint8_t)sc_ind; } } } diff --git a/dep/jemalloc/src/tcache.c b/dep/jemalloc/src/tcache.c index fa16732e4..fe210d27b 100644 --- a/dep/jemalloc/src/tcache.c +++ b/dep/jemalloc/src/tcache.c @@ -2,6 +2,7 @@ #include "jemalloc/internal/jemalloc_internal_includes.h" #include "jemalloc/internal/assert.h" +#include "jemalloc/internal/base.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/safety_check.h" #include "jemalloc/internal/san.h" @@ -12,7 +13,7 @@ bool opt_tcache = true; -/* tcache_maxclass is set to 32KB by default. */ +/* global_do_not_change_tcache_maxclass is set to 32KB by default. */ size_t opt_tcache_max = ((size_t)1) << 15; /* Reasonable defaults for min and max values. */ @@ -28,7 +29,7 @@ unsigned opt_tcache_nslots_large = 20; * This is bounded by some other constraints as well, like the fact that it * must be even, must be less than opt_tcache_nslots_small_max, etc.. */ -ssize_t opt_lg_tcache_nslots_mul = 1; +ssize_t opt_lg_tcache_nslots_mul = 1; /* * Number of allocation bytes between tcache incremental GCs. Again, this @@ -57,27 +58,41 @@ size_t opt_tcache_gc_delay_bytes = 0; unsigned opt_lg_tcache_flush_small_div = 1; unsigned opt_lg_tcache_flush_large_div = 1; -cache_bin_info_t *tcache_bin_info; +/* + * Number of cache bins enabled, including both large and small. This value + * is only used to initialize tcache_nbins in the per-thread tcache. + * Directly modifying it will not affect threads already launched. + */ +unsigned global_do_not_change_tcache_nbins; +/* + * Max size class to be cached (can be small or large). This value is only used + * to initialize tcache_max in the per-thread tcache. Directly modifying it + * will not affect threads already launched. + */ +size_t global_do_not_change_tcache_maxclass; -/* Total stack size required (per tcache). Include the padding above. */ -static size_t tcache_bin_alloc_size; -static size_t tcache_bin_alloc_alignment; +/* + * Default bin info for each bin. Will be initialized in malloc_conf_init + * and tcache_boot and should not be modified after that. + */ +static cache_bin_info_t opt_tcache_ncached_max[TCACHE_NBINS_MAX] = {{0}}; +/* + * Marks whether a bin's info is set already. This is used in + * tcache_bin_info_compute to avoid overwriting ncached_max specified by + * malloc_conf. It should be set only when parsing malloc_conf. + */ +static bool opt_tcache_ncached_max_set[TCACHE_NBINS_MAX] = {0}; -/* Number of cache bins enabled, including both large and small. */ -unsigned nhbins; -/* Max size class to be cached (can be small or large). */ -size_t tcache_maxclass; - -tcaches_t *tcaches; +tcaches_t *tcaches; /* Index of first element within tcaches that has never been used. */ -static unsigned tcaches_past; +static unsigned tcaches_past; /* Head of singly linked list tracking available tcaches elements. */ -static tcaches_t *tcaches_avail; +static tcaches_t *tcaches_avail; /* Protects tcaches{,_past,_avail}. */ -static malloc_mutex_t tcaches_mtx; +static malloc_mutex_t tcaches_mtx; /******************************************************************************/ @@ -96,14 +111,83 @@ tcache_gc_postponed_event_wait(tsd_t *tsd) { return TE_MIN_START_WAIT; } -uint64_t -tcache_gc_dalloc_new_event_wait(tsd_t *tsd) { - return opt_tcache_gc_incr_bytes; +static inline void +tcache_bin_fill_ctl_init(tcache_slow_t *tcache_slow, szind_t szind) { + assert(szind < SC_NBINS); + cache_bin_fill_ctl_t *ctl = + &tcache_slow->bin_fill_ctl_do_not_access_directly[szind]; + ctl->base = 1; + ctl->offset = 0; } -uint64_t -tcache_gc_dalloc_postponed_event_wait(tsd_t *tsd) { - return TE_MIN_START_WAIT; +static inline cache_bin_fill_ctl_t * +tcache_bin_fill_ctl_get(tcache_slow_t *tcache_slow, szind_t szind) { + assert(szind < SC_NBINS); + cache_bin_fill_ctl_t *ctl = + &tcache_slow->bin_fill_ctl_do_not_access_directly[szind]; + assert(ctl->base > ctl->offset); + return ctl; +} + +/* + * The number of items to be filled at a time for a given small bin is + * calculated by (ncached_max >> lg_fill_div). + * The actual ctl struct consists of two fields, i.e. base and offset, + * and the difference between the two(base - offset) is the final lg_fill_div. + * The base is adjusted during GC based on the traffic within a period of time, + * while the offset is updated in real time to handle the immediate traffic. + */ +static inline uint8_t +tcache_nfill_small_lg_div_get(tcache_slow_t *tcache_slow, szind_t szind) { + cache_bin_fill_ctl_t *ctl = tcache_bin_fill_ctl_get(tcache_slow, szind); + return (ctl->base - (opt_experimental_tcache_gc ? ctl->offset : 0)); +} + +/* + * When we want to fill more items to respond to burst load, + * offset is increased so that (base - offset) is decreased, + * which in return increases the number of items to be filled. + */ +static inline void +tcache_nfill_small_burst_prepare(tcache_slow_t *tcache_slow, szind_t szind) { + cache_bin_fill_ctl_t *ctl = tcache_bin_fill_ctl_get(tcache_slow, szind); + if (ctl->offset + 1 < ctl->base) { + ctl->offset++; + } +} + +static inline void +tcache_nfill_small_burst_reset(tcache_slow_t *tcache_slow, szind_t szind) { + cache_bin_fill_ctl_t *ctl = tcache_bin_fill_ctl_get(tcache_slow, szind); + ctl->offset = 0; +} + +/* + * limit == 0: indicating that the fill count should be increased, + * i.e. lg_div(base) should be decreased. + * + * limit != 0: limit is set to ncached_max, indicating that the fill + * count should be decreased, i.e. lg_div(base) should be increased. + */ +static inline void +tcache_nfill_small_gc_update( + tcache_slow_t *tcache_slow, szind_t szind, cache_bin_sz_t limit) { + cache_bin_fill_ctl_t *ctl = tcache_bin_fill_ctl_get(tcache_slow, szind); + if (!limit && ctl->base > 1) { + /* + * Increase fill count by 2X for small bins. Make sure + * lg_fill_div stays greater than 1. + */ + ctl->base--; + } else if (limit && (limit >> ctl->base) > 1) { + /* + * Reduce fill count by 2X. Limit lg_fill_div such that + * the fill count is always at least 1. + */ + ctl->base++; + } + /* Reset the offset for the next GC period. */ + ctl->offset = 0; } static uint8_t @@ -119,420 +203,467 @@ tcache_gc_item_delay_compute(szind_t szind) { return (uint8_t)item_delay; } -static void -tcache_gc_small(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, - szind_t szind) { - /* Aim to flush 3/4 of items below low-water. */ +static inline void * +tcache_gc_small_heuristic_addr_get( + tsd_t *tsd, tcache_slow_t *tcache_slow, szind_t szind) { + assert(szind < SC_NBINS); + tsdn_t *tsdn = tsd_tsdn(tsd); + bin_t *bin = bin_choose(tsdn, tcache_slow->arena, szind, NULL); + assert(bin != NULL); + + malloc_mutex_lock(tsdn, &bin->lock); + edata_t *slab = (bin->slabcur == NULL) + ? edata_heap_first(&bin->slabs_nonfull) + : bin->slabcur; + assert(slab != NULL || edata_heap_empty(&bin->slabs_nonfull)); + void *ret = (slab != NULL) ? edata_addr_get(slab) : NULL; + assert(ret != NULL || slab == NULL); + malloc_mutex_unlock(tsdn, &bin->lock); + + return ret; +} + +static inline bool +tcache_gc_is_addr_remote(void *addr, uintptr_t min, uintptr_t max) { + assert(addr != NULL); + return ((uintptr_t)addr < min || (uintptr_t)addr >= max); +} + +static inline cache_bin_sz_t +tcache_gc_small_nremote_get(cache_bin_t *cache_bin, void *addr, + uintptr_t *addr_min, uintptr_t *addr_max, szind_t szind, size_t nflush) { + assert(addr != NULL && addr_min != NULL && addr_max != NULL); + /* The slab address range that the provided addr belongs to. */ + uintptr_t slab_min = (uintptr_t)addr; + uintptr_t slab_max = slab_min + bin_infos[szind].slab_size; + /* + * When growing retained virtual memory, it's increased exponentially, + * starting from 2M, so that the total number of disjoint virtual + * memory ranges retained by each shard is limited. + */ + uintptr_t neighbor_min = ((uintptr_t)addr > TCACHE_GC_NEIGHBOR_LIMIT) + ? ((uintptr_t)addr - TCACHE_GC_NEIGHBOR_LIMIT) + : 0; + uintptr_t neighbor_max = ((uintptr_t)addr + < (UINTPTR_MAX - TCACHE_GC_NEIGHBOR_LIMIT)) + ? ((uintptr_t)addr + TCACHE_GC_NEIGHBOR_LIMIT) + : UINTPTR_MAX; + + /* Scan the entire bin to count the number of remote pointers. */ + void **head = cache_bin->stack_head; + cache_bin_sz_t n_remote_slab = 0, n_remote_neighbor = 0; + cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin); + for (void **cur = head; cur < head + ncached; cur++) { + n_remote_slab += (cache_bin_sz_t)tcache_gc_is_addr_remote( + *cur, slab_min, slab_max); + n_remote_neighbor += (cache_bin_sz_t)tcache_gc_is_addr_remote( + *cur, neighbor_min, neighbor_max); + } + /* + * Note: since slab size is dynamic and can be larger than 2M, i.e. + * TCACHE_GC_NEIGHBOR_LIMIT, there is no guarantee as to which of + * n_remote_slab and n_remote_neighbor is greater. + */ + assert(n_remote_slab <= ncached && n_remote_neighbor <= ncached); + /* + * We first consider keeping ptrs from the neighboring addr range, + * since in most cases the range is greater than the slab range. + * So if the number of non-neighbor ptrs is more than the intended + * flush amount, we use it as the anchor for flushing. + */ + if (n_remote_neighbor >= nflush) { + *addr_min = neighbor_min; + *addr_max = neighbor_max; + return n_remote_neighbor; + } + /* + * We then consider only keeping ptrs from the local slab, and in most + * cases this is stricter, assuming that slab < 2M is the common case. + */ + *addr_min = slab_min; + *addr_max = slab_max; + return n_remote_slab; +} + +/* Shuffle the ptrs in the bin to put the remote pointers at the bottom. */ +static inline void +tcache_gc_small_bin_shuffle(cache_bin_t *cache_bin, cache_bin_sz_t nremote, + uintptr_t addr_min, uintptr_t addr_max) { + void **swap = NULL; + cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin); + cache_bin_sz_t ntop = ncached - nremote, cnt = 0; + assert(ntop > 0 && ntop < ncached); + /* + * Scan the [head, head + ntop) part of the cache bin, during which + * bubbling the non-remote ptrs to the top of the bin. + * After this, the [head, head + cnt) part of the bin contains only + * non-remote ptrs, and they're in the same relative order as before. + * While the [head + cnt, head + ntop) part contains only remote ptrs. + */ + void **head = cache_bin->stack_head; + for (void **cur = head; cur < head + ntop; cur++) { + if (!tcache_gc_is_addr_remote(*cur, addr_min, addr_max)) { + /* Tracks the number of non-remote ptrs seen so far. */ + cnt++; + /* + * There is remote ptr before the current non-remote ptr, + * swap the current non-remote ptr with the remote ptr, + * and increment the swap pointer so that it's still + * pointing to the top remote ptr in the bin. + */ + if (swap != NULL) { + assert(swap < cur); + assert(tcache_gc_is_addr_remote( + *swap, addr_min, addr_max)); + void *tmp = *cur; + *cur = *swap; + *swap = tmp; + swap++; + assert(swap <= cur); + assert(tcache_gc_is_addr_remote( + *swap, addr_min, addr_max)); + } + continue; + } else if (swap == NULL) { + /* Swap always points to the top remote ptr in the bin. */ + swap = cur; + } + } + /* + * Scan the [head + ntop, head + ncached) part of the cache bin, + * after which it should only contain remote ptrs. + */ + for (void **cur = head + ntop; cur < head + ncached; cur++) { + /* Early break if all non-remote ptrs have been moved. */ + if (cnt == ntop) { + break; + } + if (!tcache_gc_is_addr_remote(*cur, addr_min, addr_max)) { + assert(tcache_gc_is_addr_remote( + *(head + cnt), addr_min, addr_max)); + void *tmp = *cur; + *cur = *(head + cnt); + *(head + cnt) = tmp; + cnt++; + } + } + assert(cnt == ntop); + /* Sanity check to make sure the shuffle is done correctly. */ + for (void **cur = head; cur < head + ncached; cur++) { + assert(*cur != NULL); + assert( + ((cur < head + ntop) + && !tcache_gc_is_addr_remote(*cur, addr_min, addr_max)) + || ((cur >= head + ntop) + && tcache_gc_is_addr_remote(*cur, addr_min, addr_max))); + } +} + +static bool +tcache_gc_small( + tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, szind_t szind) { + /* + * Aim to flush 3/4 of items below low-water, with remote pointers being + * prioritized for flushing. + */ assert(szind < SC_NBINS); cache_bin_t *cache_bin = &tcache->bins[szind]; - cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin, - &tcache_bin_info[szind]); - cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin, - &tcache_bin_info[szind]); + assert(!tcache_bin_disabled(szind, cache_bin, tcache->tcache_slow)); + cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin); + cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin); + if (low_water > 0) { + /* + * There is unused items within the GC period => reduce fill count. + * limit field != 0 is borrowed to indicate that the fill count + * should be reduced. + */ + tcache_nfill_small_gc_update(tcache_slow, szind, + /* limit */ cache_bin_ncached_max_get(cache_bin)); + } else if (tcache_slow->bin_refilled[szind]) { + /* + * There has been refills within the GC period => increase fill count. + * limit field set to 0 is borrowed to indicate that the fill count + * should be increased. + */ + tcache_nfill_small_gc_update(tcache_slow, szind, /* limit */ 0); + tcache_slow->bin_refilled[szind] = false; + } assert(!tcache_slow->bin_refilled[szind]); - size_t nflush = low_water - (low_water >> 2); - if (nflush < tcache_slow->bin_flush_delay_items[szind]) { - /* Workaround for a conversion warning. */ - uint8_t nflush_uint8 = (uint8_t)nflush; - assert(sizeof(tcache_slow->bin_flush_delay_items[0]) == - sizeof(nflush_uint8)); - tcache_slow->bin_flush_delay_items[szind] -= nflush_uint8; - return; - } else { - tcache_slow->bin_flush_delay_items[szind] - = tcache_gc_item_delay_compute(szind); + cache_bin_sz_t nflush = low_water - (low_water >> 2); + /* + * When the new tcache gc is not enabled, keep the flush delay logic, + * and directly flush the bottom nflush items if needed. + */ + if (!opt_experimental_tcache_gc) { + if (nflush < tcache_slow->bin_flush_delay_items[szind]) { + /* Workaround for a conversion warning. */ + uint8_t nflush_uint8 = (uint8_t)nflush; + assert(sizeof(tcache_slow->bin_flush_delay_items[0]) + == sizeof(nflush_uint8)); + tcache_slow->bin_flush_delay_items[szind] -= + nflush_uint8; + return false; + } + + tcache_slow->bin_flush_delay_items[szind] = + tcache_gc_item_delay_compute(szind); + goto label_flush; } - tcache_bin_flush_small(tsd, tcache, cache_bin, szind, - (unsigned)(ncached - nflush)); + /* Directly goto the flush path when the entire bin needs to be flushed. */ + if (nflush == ncached) { + goto label_flush; + } + + /* Query arena binshard to get heuristic locality info. */ + void *addr = tcache_gc_small_heuristic_addr_get( + tsd, tcache_slow, szind); + if (addr == NULL) { + goto label_flush; + } /* - * Reduce fill count by 2X. Limit lg_fill_div such that - * the fill count is always at least 1. + * Use the queried addr above to get the number of remote ptrs in the + * bin, and the min/max of the local addr range. */ - if ((cache_bin_info_ncached_max(&tcache_bin_info[szind]) - >> (tcache_slow->lg_fill_div[szind] + 1)) >= 1) { - tcache_slow->lg_fill_div[szind]++; + uintptr_t addr_min, addr_max; + cache_bin_sz_t nremote = tcache_gc_small_nremote_get( + cache_bin, addr, &addr_min, &addr_max, szind, nflush); + + /* + * Update the nflush to the larger value between the intended flush count + * and the number of remote ptrs. + */ + if (nremote > nflush) { + nflush = nremote; } + /* + * When entering the locality check, nflush should be less than ncached, + * otherwise the entire bin should be flushed regardless. The only case + * when nflush gets updated to ncached after locality check is, when all + * the items in the bin are remote, in which case the entire bin should + * also be flushed. + */ + assert(nflush < ncached || nremote == ncached); + if (nremote == 0 || nremote == ncached) { + goto label_flush; + } + + /* + * Move the remote points to the bottom of the bin for flushing. + * As long as moved to the bottom, the order of these nremote ptrs + * does not matter, since they are going to be flushed anyway. + * The rest of the ptrs are moved to the top of the bin, and their + * relative order is maintained. + */ + tcache_gc_small_bin_shuffle(cache_bin, nremote, addr_min, addr_max); + +label_flush: + if (nflush == 0) { + assert(low_water == 0); + return false; + } + assert(nflush <= ncached); + tcache_bin_flush_small( + tsd, tcache, cache_bin, szind, (unsigned)(ncached - nflush)); + return true; } -static void -tcache_gc_large(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, - szind_t szind) { - /* Like the small GC; flush 3/4 of untouched items. */ +static bool +tcache_gc_large( + tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, szind_t szind) { + /* + * Like the small GC, flush 3/4 of untouched items. However, simply flush + * the bottom nflush items, without any locality check. + */ assert(szind >= SC_NBINS); cache_bin_t *cache_bin = &tcache->bins[szind]; - cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin, - &tcache_bin_info[szind]); - cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin, - &tcache_bin_info[szind]); - tcache_bin_flush_large(tsd, tcache, cache_bin, szind, - (unsigned)(ncached - low_water + (low_water >> 2))); + assert(!tcache_bin_disabled(szind, cache_bin, tcache->tcache_slow)); + cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin); + if (low_water == 0) { + return false; + } + unsigned nrem = (unsigned)(cache_bin_ncached_get_local(cache_bin) + - low_water + (low_water >> 2)); + tcache_bin_flush_large(tsd, tcache, cache_bin, szind, nrem); + return true; +} + +/* Try to gc one bin by szind, return true if there is item flushed. */ +static bool +tcache_try_gc_bin( + tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, szind_t szind) { + assert(tcache != NULL); + cache_bin_t *cache_bin = &tcache->bins[szind]; + if (tcache_bin_disabled(szind, cache_bin, tcache_slow)) { + return false; + } + + bool is_small = (szind < SC_NBINS); + tcache_bin_flush_stashed(tsd, tcache, cache_bin, szind, is_small); + bool ret = is_small ? tcache_gc_small(tsd, tcache_slow, tcache, szind) + : tcache_gc_large(tsd, tcache_slow, tcache, szind); + cache_bin_low_water_set(cache_bin); + return ret; } static void -tcache_event(tsd_t *tsd) { +tcache_gc_event(tsd_t *tsd) { tcache_t *tcache = tcache_get(tsd); if (tcache == NULL) { return; } tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd); - szind_t szind = tcache_slow->next_gc_bin; - bool is_small = (szind < SC_NBINS); - cache_bin_t *cache_bin = &tcache->bins[szind]; + assert(tcache_slow != NULL); - tcache_bin_flush_stashed(tsd, tcache, cache_bin, szind, is_small); - - cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin, - &tcache_bin_info[szind]); - if (low_water > 0) { - if (is_small) { - tcache_gc_small(tsd, tcache_slow, tcache, szind); - } else { - tcache_gc_large(tsd, tcache_slow, tcache, szind); + /* When the new tcache gc is not enabled, GC one bin at a time. */ + if (!opt_experimental_tcache_gc) { + szind_t szind = tcache_slow->next_gc_bin; + tcache_try_gc_bin(tsd, tcache_slow, tcache, szind); + tcache_slow->next_gc_bin++; + if (tcache_slow->next_gc_bin == tcache_nbins_get(tcache_slow)) { + tcache_slow->next_gc_bin = 0; } - } else if (is_small && tcache_slow->bin_refilled[szind]) { - assert(low_water == 0); - /* - * Increase fill count by 2X for small bins. Make sure - * lg_fill_div stays greater than 0. - */ - if (tcache_slow->lg_fill_div[szind] > 1) { - tcache_slow->lg_fill_div[szind]--; + return; + } + + nstime_t now; + nstime_copy(&now, &tcache_slow->last_gc_time); + nstime_update(&now); + assert(nstime_compare(&now, &tcache_slow->last_gc_time) >= 0); + + if (nstime_ns(&now) - nstime_ns(&tcache_slow->last_gc_time) + < TCACHE_GC_INTERVAL_NS) { + // time interval is too short, skip this event. + return; + } + /* Update last_gc_time to now. */ + nstime_copy(&tcache_slow->last_gc_time, &now); + + unsigned gc_small_nbins = 0, gc_large_nbins = 0; + unsigned tcache_nbins = tcache_nbins_get(tcache_slow); + unsigned small_nbins = tcache_nbins > SC_NBINS ? SC_NBINS + : tcache_nbins; + szind_t szind_small = tcache_slow->next_gc_bin_small; + szind_t szind_large = tcache_slow->next_gc_bin_large; + + /* Flush at most TCACHE_GC_SMALL_NBINS_MAX small bins at a time. */ + for (unsigned i = 0; + i < small_nbins && gc_small_nbins < TCACHE_GC_SMALL_NBINS_MAX; + i++) { + assert(szind_small < SC_NBINS); + if (tcache_try_gc_bin(tsd, tcache_slow, tcache, szind_small)) { + gc_small_nbins++; + } + if (++szind_small == small_nbins) { + szind_small = 0; } - tcache_slow->bin_refilled[szind] = false; } - cache_bin_low_water_set(cache_bin); + tcache_slow->next_gc_bin_small = szind_small; - tcache_slow->next_gc_bin++; - if (tcache_slow->next_gc_bin == nhbins) { - tcache_slow->next_gc_bin = 0; + if (tcache_nbins <= SC_NBINS) { + return; } -} -void -tcache_gc_event_handler(tsd_t *tsd, uint64_t elapsed) { - assert(elapsed == TE_INVALID_ELAPSED); - tcache_event(tsd); -} - -void -tcache_gc_dalloc_event_handler(tsd_t *tsd, uint64_t elapsed) { - assert(elapsed == TE_INVALID_ELAPSED); - tcache_event(tsd); + /* Flush at most TCACHE_GC_LARGE_NBINS_MAX large bins at a time. */ + for (unsigned i = SC_NBINS; + i < tcache_nbins && gc_large_nbins < TCACHE_GC_LARGE_NBINS_MAX; + i++) { + assert(szind_large >= SC_NBINS && szind_large < tcache_nbins); + if (tcache_try_gc_bin(tsd, tcache_slow, tcache, szind_large)) { + gc_large_nbins++; + } + if (++szind_large == tcache_nbins) { + szind_large = SC_NBINS; + } + } + tcache_slow->next_gc_bin_large = szind_large; } void * -tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, - tcache_t *tcache, cache_bin_t *cache_bin, szind_t binind, - bool *tcache_success) { +tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, + cache_bin_t *cache_bin, szind_t binind, bool *tcache_success) { tcache_slow_t *tcache_slow = tcache->tcache_slow; - void *ret; + void *ret; assert(tcache_slow->arena != NULL); - unsigned nfill = cache_bin_info_ncached_max(&tcache_bin_info[binind]) - >> tcache_slow->lg_fill_div[binind]; - arena_cache_bin_fill_small(tsdn, arena, cache_bin, - &tcache_bin_info[binind], binind, nfill); + assert(!tcache_bin_disabled(binind, cache_bin, tcache_slow)); + assert(cache_bin_ncached_get_local(cache_bin) == 0); + cache_bin_sz_t nfill = cache_bin_ncached_max_get(cache_bin) + >> tcache_nfill_small_lg_div_get(tcache_slow, binind); + if (nfill == 0) { + nfill = 1; + } + cache_bin_sz_t nfill_min = opt_experimental_tcache_gc + ? ((nfill >> 1) + 1) + : nfill; + cache_bin_sz_t nfill_max = nfill; + CACHE_BIN_PTR_ARRAY_DECLARE(ptrs, nfill_max); + cache_bin_init_ptr_array_for_fill(cache_bin, &ptrs, nfill_max); + + cache_bin_sz_t filled = arena_ptr_array_fill_small(tsdn, arena, binind, + &ptrs, /* nfill_min */ nfill_min, /* nfill_max */ nfill_max, + cache_bin->tstats); + cache_bin_finish_fill(cache_bin, &ptrs, filled); + assert(filled >= nfill_min && filled <= nfill_max); + assert(cache_bin_ncached_get_local(cache_bin) == filled); + tcache_slow->bin_refilled[binind] = true; + tcache_nfill_small_burst_prepare(tcache_slow, binind); ret = cache_bin_alloc(cache_bin, tcache_success); return ret; } -static const void * -tcache_bin_flush_ptr_getter(void *arr_ctx, size_t ind) { - cache_bin_ptr_array_t *arr = (cache_bin_ptr_array_t *)arr_ctx; - return arr->ptr[ind]; -} - -static void -tcache_bin_flush_metadata_visitor(void *szind_sum_ctx, - emap_full_alloc_ctx_t *alloc_ctx) { - size_t *szind_sum = (size_t *)szind_sum_ctx; - *szind_sum -= alloc_ctx->szind; - util_prefetch_write_range(alloc_ctx->edata, sizeof(edata_t)); -} - -JEMALLOC_NOINLINE static void -tcache_bin_flush_size_check_fail(cache_bin_ptr_array_t *arr, szind_t szind, - size_t nptrs, emap_batch_lookup_result_t *edatas) { - bool found_mismatch = false; - for (size_t i = 0; i < nptrs; i++) { - szind_t true_szind = edata_szind_get(edatas[i].edata); - if (true_szind != szind) { - found_mismatch = true; - safety_check_fail_sized_dealloc( - /* current_dealloc */ false, - /* ptr */ tcache_bin_flush_ptr_getter(arr, i), - /* true_size */ sz_index2size(true_szind), - /* input_size */ sz_index2size(szind)); - } - } - assert(found_mismatch); -} - -static void -tcache_bin_flush_edatas_lookup(tsd_t *tsd, cache_bin_ptr_array_t *arr, - szind_t binind, size_t nflush, emap_batch_lookup_result_t *edatas) { - - /* - * This gets compiled away when config_opt_safety_checks is false. - * Checks for sized deallocation bugs, failing early rather than - * corrupting metadata. - */ - size_t szind_sum = binind * nflush; - emap_edata_lookup_batch(tsd, &arena_emap_global, nflush, - &tcache_bin_flush_ptr_getter, (void *)arr, - &tcache_bin_flush_metadata_visitor, (void *)&szind_sum, - edatas); - if (config_opt_safety_checks && unlikely(szind_sum != 0)) { - tcache_bin_flush_size_check_fail(arr, binind, nflush, edatas); - } -} - -JEMALLOC_ALWAYS_INLINE bool -tcache_bin_flush_match(edata_t *edata, unsigned cur_arena_ind, - unsigned cur_binshard, bool small) { - if (small) { - return edata_arena_ind_get(edata) == cur_arena_ind - && edata_binshard_get(edata) == cur_binshard; - } else { - return edata_arena_ind_get(edata) == cur_arena_ind; - } -} - -JEMALLOC_ALWAYS_INLINE void -tcache_bin_flush_impl(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin, - szind_t binind, cache_bin_ptr_array_t *ptrs, unsigned nflush, bool small) { - tcache_slow_t *tcache_slow = tcache->tcache_slow; - /* - * A couple lookup calls take tsdn; declare it once for convenience - * instead of calling tsd_tsdn(tsd) all the time. - */ - tsdn_t *tsdn = tsd_tsdn(tsd); - - if (small) { - assert(binind < SC_NBINS); - } else { - assert(binind < nhbins); - } - arena_t *tcache_arena = tcache_slow->arena; - assert(tcache_arena != NULL); - - /* - * Variable length array must have > 0 length; the last element is never - * touched (it's just included to satisfy the no-zero-length rule). - */ - VARIABLE_ARRAY(emap_batch_lookup_result_t, item_edata, nflush + 1); - tcache_bin_flush_edatas_lookup(tsd, ptrs, binind, nflush, item_edata); - - /* - * The slabs where we freed the last remaining object in the slab (and - * so need to free the slab itself). - * Used only if small == true. - */ - unsigned dalloc_count = 0; - VARIABLE_ARRAY(edata_t *, dalloc_slabs, nflush + 1); - - /* - * We're about to grab a bunch of locks. If one of them happens to be - * the one guarding the arena-level stats counters we flush our - * thread-local ones to, we do so under one critical section. - */ - bool merged_stats = false; - while (nflush > 0) { - /* Lock the arena, or bin, associated with the first object. */ - edata_t *edata = item_edata[0].edata; - unsigned cur_arena_ind = edata_arena_ind_get(edata); - arena_t *cur_arena = arena_get(tsdn, cur_arena_ind, false); - - /* - * These assignments are always overwritten when small is true, - * and their values are always ignored when small is false, but - * to avoid the technical UB when we pass them as parameters, we - * need to intialize them. - */ - unsigned cur_binshard = 0; - bin_t *cur_bin = NULL; - if (small) { - cur_binshard = edata_binshard_get(edata); - cur_bin = arena_get_bin(cur_arena, binind, - cur_binshard); - assert(cur_binshard < bin_infos[binind].n_shards); - /* - * If you're looking at profiles, you might think this - * is a good place to prefetch the bin stats, which are - * often a cache miss. This turns out not to be - * helpful on the workloads we've looked at, with moving - * the bin stats next to the lock seeming to do better. - */ - } - - if (small) { - malloc_mutex_lock(tsdn, &cur_bin->lock); - } - if (!small && !arena_is_auto(cur_arena)) { - malloc_mutex_lock(tsdn, &cur_arena->large_mtx); - } - - /* - * If we acquired the right lock and have some stats to flush, - * flush them. - */ - if (config_stats && tcache_arena == cur_arena - && !merged_stats) { - merged_stats = true; - if (small) { - cur_bin->stats.nflushes++; - cur_bin->stats.nrequests += - cache_bin->tstats.nrequests; - cache_bin->tstats.nrequests = 0; - } else { - arena_stats_large_flush_nrequests_add(tsdn, - &tcache_arena->stats, binind, - cache_bin->tstats.nrequests); - cache_bin->tstats.nrequests = 0; - } - } - - /* - * Large allocations need special prep done. Afterwards, we can - * drop the large lock. - */ - if (!small) { - for (unsigned i = 0; i < nflush; i++) { - void *ptr = ptrs->ptr[i]; - edata = item_edata[i].edata; - assert(ptr != NULL && edata != NULL); - - if (tcache_bin_flush_match(edata, cur_arena_ind, - cur_binshard, small)) { - large_dalloc_prep_locked(tsdn, - edata); - } - } - } - if (!small && !arena_is_auto(cur_arena)) { - malloc_mutex_unlock(tsdn, &cur_arena->large_mtx); - } - - /* Deallocate whatever we can. */ - unsigned ndeferred = 0; - /* Init only to avoid used-uninitialized warning. */ - arena_dalloc_bin_locked_info_t dalloc_bin_info = {0}; - if (small) { - arena_dalloc_bin_locked_begin(&dalloc_bin_info, binind); - } - for (unsigned i = 0; i < nflush; i++) { - void *ptr = ptrs->ptr[i]; - edata = item_edata[i].edata; - assert(ptr != NULL && edata != NULL); - if (!tcache_bin_flush_match(edata, cur_arena_ind, - cur_binshard, small)) { - /* - * The object was allocated either via a - * different arena, or a different bin in this - * arena. Either way, stash the object so that - * it can be handled in a future pass. - */ - ptrs->ptr[ndeferred] = ptr; - item_edata[ndeferred].edata = edata; - ndeferred++; - continue; - } - if (small) { - if (arena_dalloc_bin_locked_step(tsdn, - cur_arena, cur_bin, &dalloc_bin_info, - binind, edata, ptr)) { - dalloc_slabs[dalloc_count] = edata; - dalloc_count++; - } - } else { - if (large_dalloc_safety_checks(edata, ptr, - binind)) { - /* See the comment in isfree. */ - continue; - } - large_dalloc_finish(tsdn, edata); - } - } - - if (small) { - arena_dalloc_bin_locked_finish(tsdn, cur_arena, cur_bin, - &dalloc_bin_info); - malloc_mutex_unlock(tsdn, &cur_bin->lock); - } - arena_decay_ticks(tsdn, cur_arena, nflush - ndeferred); - nflush = ndeferred; - } - - /* Handle all deferred slab dalloc. */ - assert(small || dalloc_count == 0); - for (unsigned i = 0; i < dalloc_count; i++) { - edata_t *slab = dalloc_slabs[i]; - arena_slab_dalloc(tsdn, arena_get_from_edata(slab), slab); - - } - - if (config_stats && !merged_stats) { - if (small) { - /* - * The flush loop didn't happen to flush to this - * thread's arena, so the stats didn't get merged. - * Manually do so now. - */ - bin_t *bin = arena_bin_choose(tsdn, tcache_arena, - binind, NULL); - malloc_mutex_lock(tsdn, &bin->lock); - bin->stats.nflushes++; - bin->stats.nrequests += cache_bin->tstats.nrequests; - cache_bin->tstats.nrequests = 0; - malloc_mutex_unlock(tsdn, &bin->lock); - } else { - arena_stats_large_flush_nrequests_add(tsdn, - &tcache_arena->stats, binind, - cache_bin->tstats.nrequests); - cache_bin->tstats.nrequests = 0; - } - } - -} - JEMALLOC_ALWAYS_INLINE void tcache_bin_flush_bottom(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin, szind_t binind, unsigned rem, bool small) { + assert(rem <= cache_bin_ncached_max_get(cache_bin)); + assert(!tcache_bin_disabled(binind, cache_bin, tcache->tcache_slow)); + cache_bin_sz_t orig_nstashed = cache_bin_nstashed_get_local(cache_bin); tcache_bin_flush_stashed(tsd, tcache, cache_bin, binind, small); - cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin, - &tcache_bin_info[binind]); - assert((cache_bin_sz_t)rem <= ncached); - unsigned nflush = ncached - rem; + cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin); + assert((cache_bin_sz_t)rem <= ncached + orig_nstashed); + if ((cache_bin_sz_t)rem > ncached) { + /* + * The flush_stashed above could have done enough flushing, if + * there were many items stashed. Validate that: 1) non zero + * stashed, and 2) bin stack has available space now. + */ + assert(orig_nstashed > 0); + assert(ncached + cache_bin_nstashed_get_local(cache_bin) + < cache_bin_ncached_max_get(cache_bin)); + /* Still go through the flush logic for stats purpose only. */ + rem = ncached; + } + cache_bin_sz_t nflush = ncached - (cache_bin_sz_t)rem; CACHE_BIN_PTR_ARRAY_DECLARE(ptrs, nflush); - cache_bin_init_ptr_array_for_flush(cache_bin, &tcache_bin_info[binind], - &ptrs, nflush); + cache_bin_init_ptr_array_for_flush(cache_bin, &ptrs, nflush); - tcache_bin_flush_impl(tsd, tcache, cache_bin, binind, &ptrs, nflush, - small); + arena_ptr_array_flush(tsd, binind, &ptrs, nflush, small, + tcache->tcache_slow->arena, cache_bin->tstats); - cache_bin_finish_flush(cache_bin, &tcache_bin_info[binind], &ptrs, - ncached - rem); + cache_bin_finish_flush(cache_bin, &ptrs, nflush); } void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin, szind_t binind, unsigned rem) { - tcache_bin_flush_bottom(tsd, tcache, cache_bin, binind, rem, true); + tcache_nfill_small_burst_reset(tcache->tcache_slow, binind); + tcache_bin_flush_bottom(tsd, tcache, cache_bin, binind, rem, + /* small */ true); } void tcache_bin_flush_large(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin, szind_t binind, unsigned rem) { - tcache_bin_flush_bottom(tsd, tcache, cache_bin, binind, rem, false); + tcache_bin_flush_bottom(tsd, tcache, cache_bin, binind, rem, + /* small */ false); } /* @@ -548,35 +679,67 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin, void tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin, szind_t binind, bool is_small) { - cache_bin_info_t *info = &tcache_bin_info[binind]; + assert(!tcache_bin_disabled(binind, cache_bin, tcache->tcache_slow)); /* * The two below are for assertion only. The content of original cached * items remain unchanged -- the stashed items reside on the other end * of the stack. Checking the stack head and ncached to verify. */ - void *head_content = *cache_bin->stack_head; - cache_bin_sz_t orig_cached = cache_bin_ncached_get_local(cache_bin, - info); + void *head_content = *cache_bin->stack_head; + cache_bin_sz_t orig_cached = cache_bin_ncached_get_local(cache_bin); - cache_bin_sz_t nstashed = cache_bin_nstashed_get_local(cache_bin, info); - assert(orig_cached + nstashed <= cache_bin_info_ncached_max(info)); + cache_bin_sz_t nstashed = cache_bin_nstashed_get_local(cache_bin); + assert(orig_cached + nstashed <= cache_bin_ncached_max_get(cache_bin)); if (nstashed == 0) { return; } CACHE_BIN_PTR_ARRAY_DECLARE(ptrs, nstashed); - cache_bin_init_ptr_array_for_stashed(cache_bin, binind, info, &ptrs, - nstashed); + cache_bin_init_ptr_array_for_stashed( + cache_bin, binind, &ptrs, nstashed); san_check_stashed_ptrs(ptrs.ptr, nstashed, sz_index2size(binind)); - tcache_bin_flush_impl(tsd, tcache, cache_bin, binind, &ptrs, nstashed, - is_small); - cache_bin_finish_flush_stashed(cache_bin, info); + arena_ptr_array_flush(tsd, binind, &ptrs, nstashed, is_small, + tcache->tcache_slow->arena, cache_bin->tstats); + cache_bin_finish_flush_stashed(cache_bin); - assert(cache_bin_nstashed_get_local(cache_bin, info) == 0); - assert(cache_bin_ncached_get_local(cache_bin, info) == orig_cached); + assert(cache_bin_nstashed_get_local(cache_bin) == 0); + assert(cache_bin_ncached_get_local(cache_bin) == orig_cached); assert(head_content == *cache_bin->stack_head); } +JET_EXTERN bool +tcache_get_default_ncached_max_set(szind_t ind) { + return opt_tcache_ncached_max_set[ind]; +} + +JET_EXTERN const cache_bin_info_t * +tcache_get_default_ncached_max(void) { + return opt_tcache_ncached_max; +} + +bool +tcache_bin_ncached_max_read( + tsd_t *tsd, size_t bin_size, cache_bin_sz_t *ncached_max) { + if (bin_size > TCACHE_MAXCLASS_LIMIT) { + return true; + } + + if (!tcache_available(tsd)) { + *ncached_max = 0; + return false; + } + + tcache_t *tcache = tsd_tcachep_get(tsd); + assert(tcache != NULL); + szind_t bin_ind = sz_size2index(bin_size); + + cache_bin_t *bin = &tcache->bins[bin_ind]; + *ncached_max = tcache_bin_disabled(bin_ind, bin, tcache->tcache_slow) + ? 0 + : cache_bin_ncached_max_get(bin); + return false; +} + void tcache_arena_associate(tsdn_t *tsdn, tcache_slow_t *tcache_slow, tcache_t *tcache, arena_t *arena) { @@ -599,17 +762,17 @@ tcache_arena_associate(tsdn_t *tsdn, tcache_slow_t *tcache_slow, } static void -tcache_arena_dissociate(tsdn_t *tsdn, tcache_slow_t *tcache_slow, - tcache_t *tcache) { +tcache_arena_dissociate( + tsdn_t *tsdn, tcache_slow_t *tcache_slow, tcache_t *tcache) { arena_t *arena = tcache_slow->arena; assert(arena != NULL); if (config_stats) { /* Unlink from list of extant tcaches. */ malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); if (config_debug) { - bool in_ql = false; + bool in_ql = false; tcache_slow_t *iter; - ql_foreach(iter, &arena->tcache_ql, link) { + ql_foreach (iter, &arena->tcache_ql, link) { if (iter == tcache_slow) { in_ql = true; break; @@ -633,94 +796,182 @@ tcache_arena_reassociate(tsdn_t *tsdn, tcache_slow_t *tcache_slow, tcache_arena_associate(tsdn, tcache_slow, tcache, arena); } -bool -tsd_tcache_enabled_data_init(tsd_t *tsd) { - /* Called upon tsd initialization. */ - tsd_tcache_enabled_set(tsd, opt_tcache); - tsd_slow_update(tsd); - - if (opt_tcache) { - /* Trigger tcache init. */ - tsd_tcache_data_init(tsd); - } - - return false; +static void +tcache_default_settings_init(tcache_slow_t *tcache_slow) { + assert(tcache_slow != NULL); + assert(global_do_not_change_tcache_maxclass != 0); + assert(global_do_not_change_tcache_nbins != 0); + tcache_slow->tcache_nbins = global_do_not_change_tcache_nbins; } static void -tcache_init(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, - void *mem) { +tcache_init(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, void *mem, + const cache_bin_info_t *tcache_bin_info) { tcache->tcache_slow = tcache_slow; tcache_slow->tcache = tcache; memset(&tcache_slow->link, 0, sizeof(ql_elm(tcache_t))); + nstime_init_zero(&tcache_slow->last_gc_time); tcache_slow->next_gc_bin = 0; + tcache_slow->next_gc_bin_small = 0; + tcache_slow->next_gc_bin_large = SC_NBINS; tcache_slow->arena = NULL; tcache_slow->dyn_alloc = mem; /* * We reserve cache bins for all small size classes, even if some may - * not get used (i.e. bins higher than nhbins). This allows the fast - * and common paths to access cache bin metadata safely w/o worrying - * about which ones are disabled. + * not get used (i.e. bins higher than tcache_nbins). This allows + * the fast and common paths to access cache bin metadata safely w/o + * worrying about which ones are disabled. */ - unsigned n_reserved_bins = nhbins < SC_NBINS ? SC_NBINS : nhbins; - memset(tcache->bins, 0, sizeof(cache_bin_t) * n_reserved_bins); - - size_t cur_offset = 0; - cache_bin_preincrement(tcache_bin_info, nhbins, mem, - &cur_offset); - for (unsigned i = 0; i < nhbins; i++) { + unsigned tcache_nbins = tcache_nbins_get(tcache_slow); + size_t cur_offset = 0; + cache_bin_preincrement(tcache_bin_info, tcache_nbins, mem, &cur_offset); + for (unsigned i = 0; i < tcache_nbins; i++) { if (i < SC_NBINS) { - tcache_slow->lg_fill_div[i] = 1; + tcache_bin_fill_ctl_init(tcache_slow, i); tcache_slow->bin_refilled[i] = false; - tcache_slow->bin_flush_delay_items[i] - = tcache_gc_item_delay_compute(i); + tcache_slow->bin_flush_delay_items[i] = + tcache_gc_item_delay_compute(i); } cache_bin_t *cache_bin = &tcache->bins[i]; - cache_bin_init(cache_bin, &tcache_bin_info[i], mem, - &cur_offset); + if (tcache_bin_info[i].ncached_max > 0) { + cache_bin_init( + cache_bin, &tcache_bin_info[i], mem, &cur_offset); + } else { + cache_bin_init_disabled( + cache_bin, tcache_bin_info[i].ncached_max); + } } /* - * For small size classes beyond tcache_maxclass (i.e. nhbins < NBINS), - * their cache bins are initialized to a state to safely and efficiently - * fail all fastpath alloc / free, so that no additional check around - * nhbins is needed on fastpath. + * Initialize all disabled bins to a state that can safely and + * efficiently fail all fastpath alloc / free, so that no additional + * check around tcache_nbins is needed on fastpath. Yet we still + * store the ncached_max in the bin_info for future usage. */ - for (unsigned i = nhbins; i < SC_NBINS; i++) { - /* Disabled small bins. */ + for (unsigned i = tcache_nbins; i < TCACHE_NBINS_MAX; i++) { cache_bin_t *cache_bin = &tcache->bins[i]; - void *fake_stack = mem; - size_t fake_offset = 0; - - cache_bin_init(cache_bin, &tcache_bin_info[i], fake_stack, - &fake_offset); - assert(tcache_small_bin_disabled(i, cache_bin)); + cache_bin_init_disabled( + cache_bin, tcache_bin_info[i].ncached_max); + assert(tcache_bin_disabled(i, cache_bin, tcache->tcache_slow)); } - cache_bin_postincrement(tcache_bin_info, nhbins, mem, - &cur_offset); - /* Sanity check that the whole stack is used. */ - assert(cur_offset == tcache_bin_alloc_size); + cache_bin_postincrement(mem, &cur_offset); + if (config_debug) { + /* Sanity check that the whole stack is used. */ + size_t size, alignment; + cache_bin_info_compute_alloc( + tcache_bin_info, tcache_nbins, &size, &alignment); + assert(cur_offset == size); + } } -/* Initialize auto tcache (embedded in TSD). */ -bool -tsd_tcache_data_init(tsd_t *tsd) { +static inline unsigned +tcache_ncached_max_compute(szind_t szind) { + if (szind >= SC_NBINS) { + return opt_tcache_nslots_large; + } + unsigned slab_nregs = bin_infos[szind].nregs; + + /* We may modify these values; start with the opt versions. */ + unsigned nslots_small_min = opt_tcache_nslots_small_min; + unsigned nslots_small_max = opt_tcache_nslots_small_max; + + /* + * Clamp values to meet our constraints -- even, nonzero, min < max, and + * suitable for a cache bin size. + */ + if (opt_tcache_nslots_small_max > CACHE_BIN_NCACHED_MAX) { + nslots_small_max = CACHE_BIN_NCACHED_MAX; + } + if (nslots_small_min % 2 != 0) { + nslots_small_min++; + } + if (nslots_small_max % 2 != 0) { + nslots_small_max--; + } + if (nslots_small_min < 2) { + nslots_small_min = 2; + } + if (nslots_small_max < 2) { + nslots_small_max = 2; + } + if (nslots_small_min > nslots_small_max) { + nslots_small_min = nslots_small_max; + } + + unsigned candidate; + if (opt_lg_tcache_nslots_mul < 0) { + candidate = slab_nregs >> (-opt_lg_tcache_nslots_mul); + } else { + candidate = slab_nregs << opt_lg_tcache_nslots_mul; + } + if (candidate % 2 != 0) { + /* + * We need the candidate size to be even -- we assume that we + * can divide by two and get a positive number (e.g. when + * flushing). + */ + ++candidate; + } + if (candidate <= nslots_small_min) { + return nslots_small_min; + } else if (candidate <= nslots_small_max) { + return candidate; + } else { + return nslots_small_max; + } +} + +JET_EXTERN void +tcache_bin_info_compute(cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) { + /* + * Compute the values for each bin, but for bins with indices larger + * than tcache_nbins, no items will be cached. + */ + for (szind_t i = 0; i < TCACHE_NBINS_MAX; i++) { + unsigned ncached_max = tcache_get_default_ncached_max_set(i) + ? (unsigned)tcache_get_default_ncached_max()[i].ncached_max + : tcache_ncached_max_compute(i); + assert(ncached_max <= CACHE_BIN_NCACHED_MAX); + cache_bin_info_init( + &tcache_bin_info[i], (cache_bin_sz_t)ncached_max); + } +} + +static void * +tcache_stack_alloc_impl(tsdn_t *tsdn, size_t size, size_t alignment) { + if (cache_bin_stack_use_thp()) { + /* Alignment is ignored since it comes from THP. */ + assert(alignment == QUANTUM); + return b0_alloc_tcache_stack(tsdn, size); + } + size = sz_sa2u(size, alignment); + return ipallocztm(tsdn, size, alignment, true, NULL, + true, arena_get(TSDN_NULL, 0, true)); +} + +void *(*JET_MUTABLE tcache_stack_alloc)(tsdn_t *tsdn, size_t size, + size_t alignment) = tcache_stack_alloc_impl; + +static bool +tsd_tcache_data_init_impl( + tsd_t *tsd, arena_t *arena, const cache_bin_info_t *tcache_bin_info) { tcache_slow_t *tcache_slow = tsd_tcache_slowp_get_unsafe(tsd); - tcache_t *tcache = tsd_tcachep_get_unsafe(tsd); + tcache_t *tcache = tsd_tcachep_get_unsafe(tsd); assert(cache_bin_still_zero_initialized(&tcache->bins[0])); - size_t alignment = tcache_bin_alloc_alignment; - size_t size = sz_sa2u(tcache_bin_alloc_size, alignment); + unsigned tcache_nbins = tcache_nbins_get(tcache_slow); + size_t size, alignment; + cache_bin_info_compute_alloc( + tcache_bin_info, tcache_nbins, &size, &alignment); - void *mem = ipallocztm(tsd_tsdn(tsd), size, alignment, true, NULL, - true, arena_get(TSDN_NULL, 0, true)); + void *mem = tcache_stack_alloc(tsd_tsdn(tsd), size, alignment); if (mem == NULL) { return true; } - tcache_init(tsd, tcache_slow, tcache, mem); + tcache_init(tsd, tcache_slow, tcache, mem, tcache_bin_info); /* * Initialization is a bit tricky here. After malloc init is done, all * threads can rely on arena_choose and associate tcache accordingly. @@ -730,18 +981,19 @@ tsd_tcache_data_init(tsd_t *tsd) { * arena_choose_hard() will re-associate properly. */ tcache_slow->arena = NULL; - arena_t *arena; if (!malloc_initialized()) { /* If in initialization, assign to a0. */ arena = arena_get(tsd_tsdn(tsd), 0, false); - tcache_arena_associate(tsd_tsdn(tsd), tcache_slow, tcache, - arena); + tcache_arena_associate( + tsd_tsdn(tsd), tcache_slow, tcache, arena); } else { - arena = arena_choose(tsd, NULL); + if (arena == NULL) { + arena = arena_choose(tsd, NULL); + } /* This may happen if thread.tcache.enabled is used. */ if (tcache_slow->arena == NULL) { - tcache_arena_associate(tsd_tsdn(tsd), tcache_slow, - tcache, arena); + tcache_arena_associate( + tsd_tsdn(tsd), tcache_slow, tcache, arena); } } assert(arena == tcache_slow->arena); @@ -749,6 +1001,27 @@ tsd_tcache_data_init(tsd_t *tsd) { return false; } +/* Initialize auto tcache (embedded in TSD). */ +static bool +tsd_tcache_data_init(tsd_t *tsd, arena_t *arena, + const cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) { + assert(tcache_bin_info != NULL); + bool err = tsd_tcache_data_init_impl(tsd, arena, tcache_bin_info); + if (unlikely(err)) { + /* + * Disable the tcache before calling malloc_write to + * avoid recursive allocations through libc hooks. + */ + tsd_tcache_enabled_set(tsd, false); + tsd_slow_update(tsd); + malloc_write(": Failed to allocate tcache data\n"); + if (opt_abort) { + abort(); + } + } + return err; +} + /* Created manual tcache for tcache.create mallctl. */ tcache_t * tcache_create_explicit(tsd_t *tsd) { @@ -757,35 +1030,179 @@ tcache_create_explicit(tsd_t *tsd) { * the beginning of the whole allocation (for freeing). The makes sure * the cache bins have the requested alignment. */ - size_t size = tcache_bin_alloc_size + sizeof(tcache_t) - + sizeof(tcache_slow_t); + unsigned tcache_nbins = global_do_not_change_tcache_nbins; + size_t tcache_size, alignment; + cache_bin_info_compute_alloc(tcache_get_default_ncached_max(), + tcache_nbins, &tcache_size, &alignment); + + size_t size = tcache_size + sizeof(tcache_t) + sizeof(tcache_slow_t); /* Naturally align the pointer stacks. */ size = PTR_CEILING(size); - size = sz_sa2u(size, tcache_bin_alloc_alignment); + size = sz_sa2u(size, alignment); - void *mem = ipallocztm(tsd_tsdn(tsd), size, tcache_bin_alloc_alignment, - true, NULL, true, arena_get(TSDN_NULL, 0, true)); + void *mem = ipallocztm(tsd_tsdn(tsd), size, alignment, true, NULL, true, + arena_get(TSDN_NULL, 0, true)); if (mem == NULL) { return NULL; } - tcache_t *tcache = (void *)((uintptr_t)mem + tcache_bin_alloc_size); - tcache_slow_t *tcache_slow = - (void *)((uintptr_t)mem + tcache_bin_alloc_size + sizeof(tcache_t)); - tcache_init(tsd, tcache_slow, tcache, mem); + tcache_t *tcache = (void *)((byte_t *)mem + tcache_size); + tcache_slow_t *tcache_slow = (void *)((byte_t *)mem + tcache_size + + sizeof(tcache_t)); + tcache_default_settings_init(tcache_slow); + tcache_init( + tsd, tcache_slow, tcache, mem, tcache_get_default_ncached_max()); - tcache_arena_associate(tsd_tsdn(tsd), tcache_slow, tcache, - arena_ichoose(tsd, NULL)); + tcache_arena_associate( + tsd_tsdn(tsd), tcache_slow, tcache, arena_ichoose(tsd, NULL)); return tcache; } +bool +tsd_tcache_enabled_data_init(tsd_t *tsd) { + /* Called upon tsd initialization. */ + tsd_tcache_enabled_set(tsd, opt_tcache); + /* + * tcache is not available yet, but we need to set up its tcache_nbins + * in advance. + */ + tcache_default_settings_init(tsd_tcache_slowp_get(tsd)); + tsd_slow_update(tsd); + + if (opt_tcache) { + /* Trigger tcache init. */ + return tsd_tcache_data_init( + tsd, NULL, tcache_get_default_ncached_max()); + } + + return false; +} + +void +tcache_enabled_set(tsd_t *tsd, bool enabled) { + bool was_enabled = tsd_tcache_enabled_get(tsd); + + if (!was_enabled && enabled) { + if (tsd_tcache_data_init( + tsd, NULL, tcache_get_default_ncached_max())) { + return; + } + } else if (was_enabled && !enabled) { + tcache_cleanup(tsd); + } + /* Commit the state last. Above calls check current state. */ + tsd_tcache_enabled_set(tsd, enabled); + tsd_slow_update(tsd); +} + +bool +thread_tcache_max_set(tsd_t *tsd, size_t tcache_max) { + assert(tcache_max <= TCACHE_MAXCLASS_LIMIT); + assert(tcache_max == sz_s2u(tcache_max)); + tcache_t *tcache = tsd_tcachep_get(tsd); + tcache_slow_t *tcache_slow = tcache->tcache_slow; + cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX] = {{0}}; + bool ret = false; + assert(tcache != NULL && tcache_slow != NULL); + + bool enabled = tcache_available(tsd); + arena_t *assigned_arena JEMALLOC_CLANG_ANALYZER_SILENCE_INIT(NULL); + if (enabled) { + assigned_arena = tcache_slow->arena; + /* Carry over the bin settings during the reboot. */ + tcache_bin_settings_backup(tcache, tcache_bin_info); + /* Shutdown and reboot the tcache for a clean slate. */ + tcache_cleanup(tsd); + } + + /* + * Still set tcache_nbins of the tcache even if the tcache is not + * available yet because the values are stored in tsd_t and are + * always available for changing. + */ + tcache_max_set(tcache_slow, tcache_max); + + if (enabled) { + ret = tsd_tcache_data_init(tsd, assigned_arena, tcache_bin_info); + } + + assert(tcache_nbins_get(tcache_slow) == sz_size2index(tcache_max) + 1); + return ret; +} + +static bool +tcache_bin_info_settings_parse(const char *bin_settings_segment_cur, + size_t len_left, cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX], + bool bin_info_is_set[TCACHE_NBINS_MAX]) { + do { + size_t size_start, size_end; + size_t ncached_max; + bool err = multi_setting_parse_next(&bin_settings_segment_cur, + &len_left, &size_start, &size_end, &ncached_max); + if (err) { + return true; + } + if (size_end > TCACHE_MAXCLASS_LIMIT) { + size_end = TCACHE_MAXCLASS_LIMIT; + } + if (size_start > TCACHE_MAXCLASS_LIMIT + || size_start > size_end) { + continue; + } + /* May get called before sz_init (during malloc_conf_init). */ + szind_t bin_start = sz_size2index_compute(size_start); + szind_t bin_end = sz_size2index_compute(size_end); + if (ncached_max > CACHE_BIN_NCACHED_MAX) { + ncached_max = (size_t)CACHE_BIN_NCACHED_MAX; + } + for (szind_t i = bin_start; i <= bin_end; i++) { + cache_bin_info_init( + &tcache_bin_info[i], (cache_bin_sz_t)ncached_max); + if (bin_info_is_set != NULL) { + bin_info_is_set[i] = true; + } + } + } while (len_left > 0); + + return false; +} + +bool +tcache_bin_info_default_init( + const char *bin_settings_segment_cur, size_t len_left) { + return tcache_bin_info_settings_parse(bin_settings_segment_cur, + len_left, opt_tcache_ncached_max, opt_tcache_ncached_max_set); +} + +bool +tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len) { + assert(tcache_available(tsd)); + assert(len != 0); + tcache_t *tcache = tsd_tcachep_get(tsd); + assert(tcache != NULL); + cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]; + tcache_bin_settings_backup(tcache, tcache_bin_info); + + if (tcache_bin_info_settings_parse( + settings, len, tcache_bin_info, NULL)) { + return true; + } + + arena_t *assigned_arena = tcache->tcache_slow->arena; + tcache_cleanup(tsd); + return tsd_tcache_data_init(tsd, assigned_arena, tcache_bin_info); +} + static void tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) { tcache_slow_t *tcache_slow = tcache->tcache_slow; assert(tcache_slow->arena != NULL); - for (unsigned i = 0; i < nhbins; i++) { + for (unsigned i = 0; i < tcache_nbins_get(tcache_slow); i++) { cache_bin_t *cache_bin = &tcache->bins[i]; + if (tcache_bin_disabled(i, cache_bin, tcache_slow)) { + continue; + } if (i < SC_NBINS) { tcache_bin_flush_small(tsd, tcache, cache_bin, i, 0); } else { @@ -812,10 +1229,14 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) { if (tsd_tcache) { cache_bin_t *cache_bin = &tcache->bins[0]; - cache_bin_assert_empty(cache_bin, &tcache_bin_info[0]); + cache_bin_assert_empty(cache_bin); + } + if (tsd_tcache && cache_bin_stack_use_thp()) { + b0_dalloc_tcache_stack(tsd_tsdn(tsd), tcache_slow->dyn_alloc); + } else { + idalloctm(tsd_tsdn(tsd), tcache_slow->dyn_alloc, NULL, NULL, + true, true); } - idalloctm(tsd_tsdn(tsd), tcache_slow->dyn_alloc, NULL, NULL, true, - true); /* * The deallocation and tcache flush above may not trigger decay since @@ -823,11 +1244,11 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) { * tsd). Manually trigger decay to avoid pathological cases. Also * include arena 0 because the tcache array is allocated from it. */ - arena_decay(tsd_tsdn(tsd), arena_get(tsd_tsdn(tsd), 0, false), - false, false); + arena_decay( + tsd_tsdn(tsd), arena_get(tsd_tsdn(tsd), 0, false), false, false); - if (arena_nthreads_get(arena, false) == 0 && - !background_thread_enabled()) { + if (arena_nthreads_get(arena, false) == 0 + && !background_thread_enabled()) { /* Force purging when no threads assigned to the arena anymore. */ arena_decay(tsd_tsdn(tsd), arena, /* is_background_thread */ false, /* all */ true); @@ -850,13 +1271,8 @@ tcache_cleanup(tsd_t *tsd) { assert(!cache_bin_still_zero_initialized(&tcache->bins[0])); tcache_destroy(tsd, tcache, true); - if (config_debug) { - /* - * For debug testing only, we want to pretend we're still in the - * zero-initialized state. - */ - memset(tcache->bins, 0, sizeof(cache_bin_t) * nhbins); - } + /* Make sure all bins used are reinitialized to the clean state. */ + memset(tcache->bins, 0, sizeof(cache_bin_t) * TCACHE_NBINS_MAX); } void @@ -864,10 +1280,13 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { cassert(config_stats); /* Merge and reset tcache stats. */ - for (unsigned i = 0; i < nhbins; i++) { + for (unsigned i = 0; i < tcache_nbins_get(tcache->tcache_slow); i++) { cache_bin_t *cache_bin = &tcache->bins[i]; + if (tcache_bin_disabled(i, cache_bin, tcache->tcache_slow)) { + continue; + } if (i < SC_NBINS) { - bin_t *bin = arena_bin_choose(tsdn, arena, i, NULL); + bin_t *bin = bin_choose(tsdn, arena, i, NULL); malloc_mutex_lock(tsdn, &bin->lock); bin->stats.nrequests += cache_bin->tstats.nrequests; malloc_mutex_unlock(tsdn, &bin->lock); @@ -887,7 +1306,7 @@ tcaches_create_prep(tsd_t *tsd, base_t *base) { if (tcaches == NULL) { tcaches = base_alloc(tsd_tsdn(tsd), base, - sizeof(tcache_t *) * (MALLOCX_TCACHE_MAX+1), CACHELINE); + sizeof(tcache_t *) * (MALLOCX_TCACHE_MAX + 1), CACHELINE); if (tcaches == NULL) { err = true; goto label_return; @@ -978,7 +1397,7 @@ void tcaches_destroy(tsd_t *tsd, unsigned ind) { malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); tcaches_t *elm = &tcaches[ind]; - tcache_t *tcache = tcaches_elm_remove(tsd, elm, false); + tcache_t *tcache = tcaches_elm_remove(tsd, elm, false); elm->next = tcaches_avail; tcaches_avail = elm; malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); @@ -987,97 +1406,25 @@ tcaches_destroy(tsd_t *tsd, unsigned ind) { } } -static unsigned -tcache_ncached_max_compute(szind_t szind) { - if (szind >= SC_NBINS) { - assert(szind < nhbins); - return opt_tcache_nslots_large; - } - unsigned slab_nregs = bin_infos[szind].nregs; - - /* We may modify these values; start with the opt versions. */ - unsigned nslots_small_min = opt_tcache_nslots_small_min; - unsigned nslots_small_max = opt_tcache_nslots_small_max; - - /* - * Clamp values to meet our constraints -- even, nonzero, min < max, and - * suitable for a cache bin size. - */ - if (opt_tcache_nslots_small_max > CACHE_BIN_NCACHED_MAX) { - nslots_small_max = CACHE_BIN_NCACHED_MAX; - } - if (nslots_small_min % 2 != 0) { - nslots_small_min++; - } - if (nslots_small_max % 2 != 0) { - nslots_small_max--; - } - if (nslots_small_min < 2) { - nslots_small_min = 2; - } - if (nslots_small_max < 2) { - nslots_small_max = 2; - } - if (nslots_small_min > nslots_small_max) { - nslots_small_min = nslots_small_max; - } - - unsigned candidate; - if (opt_lg_tcache_nslots_mul < 0) { - candidate = slab_nregs >> (-opt_lg_tcache_nslots_mul); - } else { - candidate = slab_nregs << opt_lg_tcache_nslots_mul; - } - if (candidate % 2 != 0) { - /* - * We need the candidate size to be even -- we assume that we - * can divide by two and get a positive number (e.g. when - * flushing). - */ - ++candidate; - } - if (candidate <= nslots_small_min) { - return nslots_small_min; - } else if (candidate <= nslots_small_max) { - return candidate; - } else { - return nslots_small_max; - } -} - bool tcache_boot(tsdn_t *tsdn, base_t *base) { - tcache_maxclass = sz_s2u(opt_tcache_max); - assert(tcache_maxclass <= TCACHE_MAXCLASS_LIMIT); - nhbins = sz_size2index(tcache_maxclass) + 1; + global_do_not_change_tcache_maxclass = sz_s2u(opt_tcache_max); + assert(global_do_not_change_tcache_maxclass <= TCACHE_MAXCLASS_LIMIT); + global_do_not_change_tcache_nbins = + sz_size2index(global_do_not_change_tcache_maxclass) + 1; + /* + * Pre-compute default bin info and store the results in + * opt_tcache_ncached_max. After the changes here, + * opt_tcache_ncached_max should not be modified and should always be + * accessed using tcache_get_default_ncached_max. + */ + tcache_bin_info_compute(opt_tcache_ncached_max); if (malloc_mutex_init(&tcaches_mtx, "tcaches", WITNESS_RANK_TCACHES, - malloc_mutex_rank_exclusive)) { + malloc_mutex_rank_exclusive)) { return true; } - /* Initialize tcache_bin_info. See comments in tcache_init(). */ - unsigned n_reserved_bins = nhbins < SC_NBINS ? SC_NBINS : nhbins; - size_t size = n_reserved_bins * sizeof(cache_bin_info_t); - tcache_bin_info = (cache_bin_info_t *)base_alloc(tsdn, base, size, - CACHELINE); - if (tcache_bin_info == NULL) { - return true; - } - - for (szind_t i = 0; i < nhbins; i++) { - unsigned ncached_max = tcache_ncached_max_compute(i); - cache_bin_info_init(&tcache_bin_info[i], ncached_max); - } - for (szind_t i = nhbins; i < SC_NBINS; i++) { - /* Disabled small bins. */ - cache_bin_info_init(&tcache_bin_info[i], 0); - assert(tcache_small_bin_disabled(i, NULL)); - } - - cache_bin_info_compute_alloc(tcache_bin_info, nhbins, - &tcache_bin_alloc_size, &tcache_bin_alloc_alignment); - return false; } @@ -1096,6 +1443,20 @@ tcache_postfork_child(tsdn_t *tsdn) { malloc_mutex_postfork_child(tsdn, &tcaches_mtx); } -void tcache_assert_initialized(tcache_t *tcache) { +void +tcache_assert_initialized(tcache_t *tcache) { assert(!cache_bin_still_zero_initialized(&tcache->bins[0])); } + +static te_enabled_t +tcache_gc_enabled(void) { + return (opt_tcache_gc_incr_bytes > 0) ? te_enabled_yes : te_enabled_no; +} + +/* Handles alloc and dalloc the same way */ +te_base_cb_t tcache_gc_te_handler = { + .enabled = &tcache_gc_enabled, + .new_event_wait = &tcache_gc_new_event_wait, + .postponed_event_wait = &tcache_gc_postponed_event_wait, + .event_handler = &tcache_gc_event, +}; diff --git a/dep/jemalloc/src/test_hooks.c b/dep/jemalloc/src/test_hooks.c index ace00d9c4..40621199d 100644 --- a/dep/jemalloc/src/test_hooks.c +++ b/dep/jemalloc/src/test_hooks.c @@ -6,7 +6,7 @@ * from outside the generated library, so that we can use them in test code. */ JEMALLOC_EXPORT -void (*test_hooks_arena_new_hook)() = NULL; +void (*test_hooks_arena_new_hook)(void) = NULL; JEMALLOC_EXPORT -void (*test_hooks_libc_hook)() = NULL; +void (*test_hooks_libc_hook)(void) = NULL; diff --git a/dep/jemalloc/src/thread_event.c b/dep/jemalloc/src/thread_event.c index 37eb5827d..82776342e 100644 --- a/dep/jemalloc/src/thread_event.c +++ b/dep/jemalloc/src/thread_event.c @@ -2,103 +2,48 @@ #include "jemalloc/internal/jemalloc_internal_includes.h" #include "jemalloc/internal/thread_event.h" - -/* - * Signatures for event specific functions. These functions should be defined - * by the modules owning each event. The signatures here verify that the - * definitions follow the right format. - * - * The first two are functions computing new / postponed event wait time. New - * event wait time is the time till the next event if an event is currently - * being triggered; postponed event wait time is the time till the next event - * if an event should be triggered but needs to be postponed, e.g. when the TSD - * is not nominal or during reentrancy. - * - * The third is the event handler function, which is called whenever an event - * is triggered. The parameter is the elapsed time since the last time an - * event of the same type was triggered. - */ -#define E(event, condition_unused, is_alloc_event_unused) \ -uint64_t event##_new_event_wait(tsd_t *tsd); \ -uint64_t event##_postponed_event_wait(tsd_t *tsd); \ -void event##_event_handler(tsd_t *tsd, uint64_t elapsed); - -ITERATE_OVER_ALL_EVENTS -#undef E - -/* Signatures for internal functions fetching elapsed time. */ -#define E(event, condition_unused, is_alloc_event_unused) \ -static uint64_t event##_fetch_elapsed(tsd_t *tsd); - -ITERATE_OVER_ALL_EVENTS -#undef E - -static uint64_t -tcache_gc_fetch_elapsed(tsd_t *tsd) { - return TE_INVALID_ELAPSED; -} - -static uint64_t -tcache_gc_dalloc_fetch_elapsed(tsd_t *tsd) { - return TE_INVALID_ELAPSED; -} - -static uint64_t -prof_sample_fetch_elapsed(tsd_t *tsd) { - uint64_t last_event = thread_allocated_last_event_get(tsd); - uint64_t last_sample_event = prof_sample_last_event_get(tsd); - prof_sample_last_event_set(tsd, last_event); - return last_event - last_sample_event; -} - -static uint64_t -stats_interval_fetch_elapsed(tsd_t *tsd) { - uint64_t last_event = thread_allocated_last_event_get(tsd); - uint64_t last_stats_event = stats_interval_last_event_get(tsd); - stats_interval_last_event_set(tsd, last_event); - return last_event - last_stats_event; -} - -static uint64_t -peak_alloc_fetch_elapsed(tsd_t *tsd) { - return TE_INVALID_ELAPSED; -} - -static uint64_t -peak_dalloc_fetch_elapsed(tsd_t *tsd) { - return TE_INVALID_ELAPSED; -} - -/* Per event facilities done. */ +#include "jemalloc/internal/thread_event_registry.h" +#include "jemalloc/internal/peak_event.h" static bool te_ctx_has_active_events(te_ctx_t *ctx) { assert(config_debug); -#define E(event, condition, alloc_event) \ - if (condition && alloc_event == ctx->is_alloc) { \ - return true; \ + if (ctx->is_alloc) { + for (int i = 0; i < te_alloc_count; ++i) { + if (te_enabled_yes == te_alloc_handlers[i]->enabled()) { + return true; + } + } + } else { + for (int i = 0; i < te_dalloc_count; ++i) { + if (te_enabled_yes + == te_dalloc_handlers[i]->enabled()) { + return true; + } + } } - ITERATE_OVER_ALL_EVENTS -#undef E return false; } static uint64_t te_next_event_compute(tsd_t *tsd, bool is_alloc) { - uint64_t wait = TE_MAX_START_WAIT; -#define E(event, condition, alloc_event) \ - if (is_alloc == alloc_event && condition) { \ - uint64_t event_wait = \ - event##_event_wait_get(tsd); \ - assert(event_wait <= TE_MAX_START_WAIT); \ - if (event_wait > 0U && event_wait < wait) { \ - wait = event_wait; \ - } \ - } + te_base_cb_t **handlers = is_alloc ? te_alloc_handlers + : te_dalloc_handlers; + uint64_t *waits = is_alloc ? tsd_te_datap_get_unsafe(tsd)->alloc_wait + : tsd_te_datap_get_unsafe(tsd)->dalloc_wait; + int count = is_alloc ? te_alloc_count : te_dalloc_count; - ITERATE_OVER_ALL_EVENTS -#undef E - assert(wait <= TE_MAX_START_WAIT); + uint64_t wait = TE_MAX_START_WAIT; + + for (int i = 0; i < count; i++) { + if (te_enabled_yes == handlers[i]->enabled()) { + uint64_t ev_wait = waits[i]; + assert(ev_wait <= TE_MAX_START_WAIT); + if (ev_wait > 0U && ev_wait < wait) { + wait = ev_wait; + } + } + } return wait; } @@ -121,6 +66,19 @@ te_assert_invariants_impl(tsd_t *tsd, te_ctx_t *ctx) { /* The subtraction is intentionally susceptible to underflow. */ assert(current_bytes - last_event < interval); + + /* This computation assumes that event did not become active in the + * time since the last trigger. This works fine if waits for inactive + * events are initialized with 0 as those are ignored + * If we wanted to initialize user events to anything other than + * zero, computation would take it into account and min_wait could + * be smaller than interval (as it was not part of the calc setting + * next_event). + * + * If we ever wanted to unregister the events assert would also + * need to account for the possibility that next_event was set, by + * event that is now gone + */ uint64_t min_wait = te_next_event_compute(tsd, te_ctx_is_alloc(ctx)); /* * next_event should have been pushed up only except when no event is @@ -128,9 +86,9 @@ te_assert_invariants_impl(tsd_t *tsd, te_ctx_t *ctx) { * below is stronger than needed, but having an exactly accurate guard * is more complicated to implement. */ - assert((!te_ctx_has_active_events(ctx) && last_event == 0U) || - interval == min_wait || - (interval < min_wait && interval == TE_MAX_INTERVAL)); + assert((!te_ctx_has_active_events(ctx) && last_event == 0U) + || interval == min_wait + || (interval < min_wait && interval == TE_MAX_INTERVAL)); } void @@ -193,8 +151,9 @@ te_assert_invariants_debug(tsd_t *tsd) { static void te_ctx_next_event_fast_update(te_ctx_t *ctx) { uint64_t next_event = te_ctx_next_event_get(ctx); - uint64_t next_event_fast = (next_event <= TE_NEXT_EVENT_FAST_MAX) ? - next_event : 0U; + uint64_t next_event_fast = (next_event <= TE_NEXT_EVENT_FAST_MAX) + ? next_event + : 0U; te_ctx_next_event_fast_set(ctx, next_event_fast); } @@ -218,9 +177,8 @@ te_recompute_fast_threshold(tsd_t *tsd) { } } -static void -te_adjust_thresholds_helper(tsd_t *tsd, te_ctx_t *ctx, - uint64_t wait) { +static inline void +te_adjust_thresholds_impl(tsd_t *tsd, te_ctx_t *ctx, uint64_t wait) { /* * The next threshold based on future events can only be adjusted after * progressing the last_event counter (which is set to current). @@ -228,23 +186,165 @@ te_adjust_thresholds_helper(tsd_t *tsd, te_ctx_t *ctx, assert(te_ctx_current_bytes_get(ctx) == te_ctx_last_event_get(ctx)); assert(wait <= TE_MAX_START_WAIT); - uint64_t next_event = te_ctx_last_event_get(ctx) + (wait <= - TE_MAX_INTERVAL ? wait : TE_MAX_INTERVAL); + uint64_t next_event = te_ctx_last_event_get(ctx) + + (wait <= TE_MAX_INTERVAL ? wait : TE_MAX_INTERVAL); te_ctx_next_event_set(tsd, ctx, next_event); } +void +te_adjust_thresholds_helper(tsd_t *tsd, te_ctx_t *ctx, uint64_t wait) { + te_adjust_thresholds_impl(tsd, ctx, wait); +} -static uint64_t -te_clip_event_wait(uint64_t event_wait) { - assert(event_wait > 0U); - if (TE_MIN_START_WAIT > 1U && - unlikely(event_wait < TE_MIN_START_WAIT)) { - event_wait = TE_MIN_START_WAIT; +static void +te_init_waits(tsd_t *tsd, uint64_t *wait, bool is_alloc) { + te_base_cb_t **handlers = is_alloc ? te_alloc_handlers + : te_dalloc_handlers; + uint64_t *waits = is_alloc ? tsd_te_datap_get_unsafe(tsd)->alloc_wait + : tsd_te_datap_get_unsafe(tsd)->dalloc_wait; + int count = is_alloc ? te_alloc_count : te_dalloc_count; + for (int i = 0; i < count; i++) { + if (te_enabled_yes == handlers[i]->enabled()) { + uint64_t ev_wait = handlers[i]->new_event_wait(tsd); + assert(ev_wait > 0); + waits[i] = ev_wait; + if (ev_wait < *wait) { + *wait = ev_wait; + } + } } - if (TE_MAX_START_WAIT < UINT64_MAX && - unlikely(event_wait > TE_MAX_START_WAIT)) { - event_wait = TE_MAX_START_WAIT; +} + +static inline bool +te_update_wait(tsd_t *tsd, uint64_t accumbytes, bool allow, uint64_t *ev_wait, + uint64_t *wait, te_base_cb_t *handler, uint64_t new_wait) { + bool ret = false; + if (*ev_wait > accumbytes) { + *ev_wait -= accumbytes; + } else if (!allow) { + *ev_wait = handler->postponed_event_wait(tsd); + } else { + ret = true; + *ev_wait = new_wait == 0 ? handler->new_event_wait(tsd) + : new_wait; } - return event_wait; + + assert(*ev_wait > 0); + if (*ev_wait < *wait) { + *wait = *ev_wait; + } + return ret; +} + +extern uint64_t stats_interval_accum_batch; +/* Return number of handlers enqueued into to_trigger array */ +static inline size_t +te_update_alloc_events(tsd_t *tsd, te_base_cb_t **to_trigger, + uint64_t accumbytes, bool allow, uint64_t *wait) { + /* + * We do not loop and invoke the functions via interface because + * of the perf cost. This path is relatively hot, so we sacrifice + * elegance for perf. + */ + size_t nto_trigger = 0; + uint64_t *waits = tsd_te_datap_get_unsafe(tsd)->alloc_wait; + if (opt_tcache_gc_incr_bytes > 0) { + assert(te_enabled_yes + == te_alloc_handlers[te_alloc_tcache_gc]->enabled()); + if (te_update_wait(tsd, accumbytes, allow, + &waits[te_alloc_tcache_gc], wait, + te_alloc_handlers[te_alloc_tcache_gc], + opt_tcache_gc_incr_bytes)) { + to_trigger[nto_trigger++] = + te_alloc_handlers[te_alloc_tcache_gc]; + } + } +#ifdef JEMALLOC_PROF + if (opt_prof) { + assert(te_enabled_yes + == te_alloc_handlers[te_alloc_prof_sample]->enabled()); + if (te_update_wait(tsd, accumbytes, allow, + &waits[te_alloc_prof_sample], wait, + te_alloc_handlers[te_alloc_prof_sample], 0)) { + to_trigger[nto_trigger++] = + te_alloc_handlers[te_alloc_prof_sample]; + } + } +#endif + if (opt_stats_interval >= 0) { + if (te_update_wait(tsd, accumbytes, allow, + &waits[te_alloc_stats_interval], wait, + te_alloc_handlers[te_alloc_stats_interval], + stats_interval_accum_batch)) { + assert(te_enabled_yes + == te_alloc_handlers[te_alloc_stats_interval] + ->enabled()); + to_trigger[nto_trigger++] = + te_alloc_handlers[te_alloc_stats_interval]; + } + } + +#ifdef JEMALLOC_STATS + assert(te_enabled_yes == te_alloc_handlers[te_alloc_peak]->enabled()); + if (te_update_wait(tsd, accumbytes, allow, &waits[te_alloc_peak], wait, + te_alloc_handlers[te_alloc_peak], PEAK_EVENT_WAIT)) { + to_trigger[nto_trigger++] = te_alloc_handlers[te_alloc_peak]; + } + +#endif + + for (te_alloc_t ue = te_alloc_user0; ue <= te_alloc_user3; ue++) { + te_enabled_t status = te_user_event_enabled( + ue - te_alloc_user0, true); + if (status == te_enabled_not_installed) { + break; + } else if (status == te_enabled_yes) { + if (te_update_wait(tsd, accumbytes, allow, &waits[ue], + wait, te_alloc_handlers[ue], 0)) { + to_trigger[nto_trigger++] = + te_alloc_handlers[ue]; + } + } + } + return nto_trigger; +} + +static inline size_t +te_update_dalloc_events(tsd_t *tsd, te_base_cb_t **to_trigger, + uint64_t accumbytes, bool allow, uint64_t *wait) { + size_t nto_trigger = 0; + uint64_t *waits = tsd_te_datap_get_unsafe(tsd)->dalloc_wait; + if (opt_tcache_gc_incr_bytes > 0) { + assert(te_enabled_yes + == te_dalloc_handlers[te_dalloc_tcache_gc]->enabled()); + if (te_update_wait(tsd, accumbytes, allow, + &waits[te_dalloc_tcache_gc], wait, + te_dalloc_handlers[te_dalloc_tcache_gc], + opt_tcache_gc_incr_bytes)) { + to_trigger[nto_trigger++] = + te_dalloc_handlers[te_dalloc_tcache_gc]; + } + } +#ifdef JEMALLOC_STATS + assert(te_enabled_yes == te_dalloc_handlers[te_dalloc_peak]->enabled()); + if (te_update_wait(tsd, accumbytes, allow, &waits[te_dalloc_peak], wait, + te_dalloc_handlers[te_dalloc_peak], PEAK_EVENT_WAIT)) { + to_trigger[nto_trigger++] = te_dalloc_handlers[te_dalloc_peak]; + } +#endif + for (te_dalloc_t ue = te_dalloc_user0; ue <= te_dalloc_user3; ue++) { + te_enabled_t status = te_user_event_enabled( + ue - te_dalloc_user0, false); + if (status == te_enabled_not_installed) { + break; + } else if (status == te_enabled_yes) { + if (te_update_wait(tsd, accumbytes, allow, &waits[ue], + wait, te_dalloc_handlers[ue], 0)) { + to_trigger[nto_trigger++] = + te_dalloc_handlers[ue]; + } + } + } + return nto_trigger; } void @@ -256,49 +356,30 @@ te_event_trigger(tsd_t *tsd, te_ctx_t *ctx) { te_ctx_last_event_set(ctx, bytes_after); - bool allow_event_trigger = tsd_nominal(tsd) && - tsd_reentrancy_level_get(tsd) == 0; - bool is_alloc = ctx->is_alloc; + bool allow_event_trigger = tsd_nominal(tsd) + && tsd_reentrancy_level_get(tsd) == 0; uint64_t wait = TE_MAX_START_WAIT; -#define E(event, condition, alloc_event) \ - bool is_##event##_triggered = false; \ - if (is_alloc == alloc_event && condition) { \ - uint64_t event_wait = event##_event_wait_get(tsd); \ - assert(event_wait <= TE_MAX_START_WAIT); \ - if (event_wait > accumbytes) { \ - event_wait -= accumbytes; \ - } else if (!allow_event_trigger) { \ - event_wait = event##_postponed_event_wait(tsd); \ - } else { \ - is_##event##_triggered = true; \ - event_wait = event##_new_event_wait(tsd); \ - } \ - event_wait = te_clip_event_wait(event_wait); \ - event##_event_wait_set(tsd, event_wait); \ - if (event_wait < wait) { \ - wait = event_wait; \ - } \ + assert((int)te_alloc_count >= (int)te_dalloc_count); + te_base_cb_t *to_trigger[te_alloc_count]; + size_t nto_trigger; + if (ctx->is_alloc) { + nto_trigger = te_update_alloc_events( + tsd, to_trigger, accumbytes, allow_event_trigger, &wait); + } else { + nto_trigger = te_update_dalloc_events( + tsd, to_trigger, accumbytes, allow_event_trigger, &wait); } - ITERATE_OVER_ALL_EVENTS -#undef E - assert(wait <= TE_MAX_START_WAIT); te_adjust_thresholds_helper(tsd, ctx, wait); te_assert_invariants(tsd); -#define E(event, condition, alloc_event) \ - if (is_alloc == alloc_event && condition && \ - is_##event##_triggered) { \ - assert(allow_event_trigger); \ - uint64_t elapsed = event##_fetch_elapsed(tsd); \ - event##_event_handler(tsd, elapsed); \ + for (size_t i = 0; i < nto_trigger; i++) { + assert(allow_event_trigger); + to_trigger[i]->event_handler(tsd); } - ITERATE_OVER_ALL_EVENTS -#undef E - te_assert_invariants(tsd); } @@ -318,19 +399,9 @@ te_init(tsd_t *tsd, bool is_alloc) { te_ctx_last_event_set(&ctx, te_ctx_current_bytes_get(&ctx)); uint64_t wait = TE_MAX_START_WAIT; -#define E(event, condition, alloc_event) \ - if (is_alloc == alloc_event && condition) { \ - uint64_t event_wait = event##_new_event_wait(tsd); \ - event_wait = te_clip_event_wait(event_wait); \ - event##_event_wait_set(tsd, event_wait); \ - if (event_wait < wait) { \ - wait = event_wait; \ - } \ - } + te_init_waits(tsd, &wait, is_alloc); - ITERATE_OVER_ALL_EVENTS -#undef E - te_adjust_thresholds_helper(tsd, &ctx, wait); + te_adjust_thresholds_impl(tsd, &ctx, wait); } void diff --git a/dep/jemalloc/src/thread_event_registry.c b/dep/jemalloc/src/thread_event_registry.c new file mode 100644 index 000000000..b8307df06 --- /dev/null +++ b/dep/jemalloc/src/thread_event_registry.c @@ -0,0 +1,238 @@ +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" + +#include "jemalloc/internal/thread_event.h" +#include "jemalloc/internal/thread_event_registry.h" +#include "jemalloc/internal/tcache_externs.h" +#include "jemalloc/internal/peak_event.h" +#include "jemalloc/internal/prof_externs.h" +#include "jemalloc/internal/stats.h" + +static malloc_mutex_t uevents_mu; + +bool +experimental_thread_events_boot(void) { + return malloc_mutex_init(&uevents_mu, "thread_events", + WITNESS_RANK_THREAD_EVENTS_USER, malloc_mutex_rank_exclusive); +} + +#define TE_REGISTER_ERRCODE_FULL_SLOTS -1 +#define TE_REGISTER_ERRCODE_ALREADY_REGISTERED -2 + +static user_hook_object_t uevents_storage[TE_MAX_USER_EVENTS] = { + {NULL, 0, false}, +}; + +static atomic_p_t uevent_obj_p[TE_MAX_USER_EVENTS] = { + NULL, +}; + +static inline bool +user_object_eq(user_hook_object_t *lhs, user_hook_object_t *rhs) { + assert(lhs != NULL && rhs != NULL); + + return lhs->callback == rhs->callback && lhs->interval == rhs->interval + && lhs->is_alloc_only == rhs->is_alloc_only; +} + +/* + * Return slot number that event is registered at on success + * it will be [0, TE_MAX_USER_EVENTS) + * Return negative value on some error + */ +static inline int +te_register_user_handler_locked(user_hook_object_t *new_obj) { + /* Attempt to find the free slot in global register */ + for (int i = 0; i < TE_MAX_USER_EVENTS; ++i) { + user_hook_object_t *p = (user_hook_object_t *)atomic_load_p( + &uevent_obj_p[i], ATOMIC_ACQUIRE); + + if (p && user_object_eq(p, new_obj)) { + /* Same callback and interval are registered - no error. */ + return TE_REGISTER_ERRCODE_ALREADY_REGISTERED; + } else if (p == NULL) { + /* Empty slot */ + uevents_storage[i] = *new_obj; + atomic_fence(ATOMIC_SEQ_CST); + atomic_store_p(&uevent_obj_p[i], &uevents_storage[i], + ATOMIC_RELEASE); + return i; + } + } + + return TE_REGISTER_ERRCODE_FULL_SLOTS; +} + +static inline user_hook_object_t * +uobj_get(size_t cb_idx) { + assert(cb_idx < TE_MAX_USER_EVENTS); + return (user_hook_object_t *)atomic_load_p( + &uevent_obj_p[cb_idx], ATOMIC_ACQUIRE); +} + +te_enabled_t +te_user_event_enabled(size_t ue_idx, bool is_alloc) { + assert(ue_idx < TE_MAX_USER_EVENTS); + user_hook_object_t *obj = uobj_get(ue_idx); + if (!obj) { + return te_enabled_not_installed; + } + if (is_alloc || !obj->is_alloc_only) { + return te_enabled_yes; + } + return te_enabled_no; +} + +static inline uint64_t +new_event_wait(size_t cb_idx) { + user_hook_object_t *obj = uobj_get(cb_idx); + /* Enabled should have guarded it */ + assert(obj); + return obj->interval; +} + +static uint64_t +postponed_event_wait(tsd_t *tsd) { + return TE_MIN_START_WAIT; +} + +static inline void +handler_wrapper(tsd_t *tsd, bool is_alloc, size_t cb_idx) { + user_hook_object_t *obj = uobj_get(cb_idx); + /* Enabled should have guarded it */ + assert(obj); + uint64_t alloc = tsd_thread_allocated_get(tsd); + uint64_t dalloc = tsd_thread_deallocated_get(tsd); + + pre_reentrancy(tsd, NULL); + obj->callback(is_alloc, alloc, dalloc); + post_reentrancy(tsd); +} + +#define TE_USER_HANDLER_BINDING_IDX(i) \ + static te_enabled_t te_user_alloc_enabled##i(void) { \ + return te_user_event_enabled(i, true); \ + } \ + static te_enabled_t te_user_dalloc_enabled##i(void) { \ + return te_user_event_enabled(i, false); \ + } \ + static uint64_t te_user_new_event_wait_##i(tsd_t *tsd) { \ + return new_event_wait(i); \ + } \ + static void te_user_alloc_handler_call##i(tsd_t *tsd) { \ + handler_wrapper(tsd, true, i); \ + } \ + static void te_user_dalloc_handler_call##i(tsd_t *tsd) { \ + handler_wrapper(tsd, false, i); \ + } \ + static te_base_cb_t user_alloc_handler##i = { \ + .enabled = &te_user_alloc_enabled##i, \ + .new_event_wait = &te_user_new_event_wait_##i, \ + .postponed_event_wait = &postponed_event_wait, \ + .event_handler = &te_user_alloc_handler_call##i}; \ + static te_base_cb_t user_dalloc_handler##i = { \ + .enabled = &te_user_dalloc_enabled##i, \ + .new_event_wait = &te_user_new_event_wait_##i, \ + .postponed_event_wait = &postponed_event_wait, \ + .event_handler = &te_user_dalloc_handler_call##i} + +TE_USER_HANDLER_BINDING_IDX(0); +TE_USER_HANDLER_BINDING_IDX(1); +TE_USER_HANDLER_BINDING_IDX(2); +TE_USER_HANDLER_BINDING_IDX(3); + +/* Table of all the thread events. */ +te_base_cb_t *te_alloc_handlers[te_alloc_count] = { +#ifdef JEMALLOC_PROF + &prof_sample_te_handler, +#endif + &stats_interval_te_handler, &tcache_gc_te_handler, +#ifdef JEMALLOC_STATS + &peak_te_handler, +#endif + &user_alloc_handler0, &user_alloc_handler1, &user_alloc_handler2, + &user_alloc_handler3}; + +te_base_cb_t *te_dalloc_handlers[te_dalloc_count] = {&tcache_gc_te_handler, +#ifdef JEMALLOC_STATS + &peak_te_handler, +#endif + &user_dalloc_handler0, &user_dalloc_handler1, &user_dalloc_handler2, + &user_dalloc_handler3}; + +static inline bool +te_update_tsd(tsd_t *tsd, uint64_t new_wait, size_t ue_idx, bool is_alloc) { + bool needs_recompute = false; + te_ctx_t ctx; + uint64_t next, current, cur_wait; + + if (is_alloc) { + tsd_te_datap_get_unsafe(tsd) + ->alloc_wait[te_alloc_user0 + ue_idx] = new_wait; + } else { + tsd_te_datap_get_unsafe(tsd) + ->dalloc_wait[te_dalloc_user0 + ue_idx] = new_wait; + } + te_ctx_get(tsd, &ctx, is_alloc); + + next = te_ctx_next_event_get(&ctx); + current = te_ctx_current_bytes_get(&ctx); + cur_wait = next - current; + + if (new_wait < cur_wait) { + /* + * Set last event to current (same as when te inits). This + * will make sure that all the invariants are correct, before + * we adjust next_event and next_event fast. + */ + te_ctx_last_event_set(&ctx, te_ctx_current_bytes_get(&ctx)); + te_adjust_thresholds_helper(tsd, &ctx, new_wait); + needs_recompute = true; + } + return needs_recompute; +} + +static inline void +te_recalculate_current_thread_data(tsdn_t *tsdn, int ue_idx, bool alloc_only) { + bool recompute = false; + /* we do not need lock to recalculate the events on the current thread */ + assert(ue_idx < TE_MAX_USER_EVENTS); + tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn); + if (tsd) { + uint64_t new_wait = new_event_wait(ue_idx); + recompute = te_update_tsd(tsd, new_wait, ue_idx, true); + if (!alloc_only) { + recompute = te_update_tsd(tsd, new_wait, ue_idx, false) + || recompute; + } + + if (recompute) { + te_recompute_fast_threshold(tsd); + } + } +} + +int +te_register_user_handler(tsdn_t *tsdn, user_hook_object_t *te_uobj) { + int ret; + int reg_retcode; + if (!te_uobj || !te_uobj->callback || te_uobj->interval == 0) { + return EINVAL; + } + + malloc_mutex_lock(tsdn, &uevents_mu); + reg_retcode = te_register_user_handler_locked(te_uobj); + malloc_mutex_unlock(tsdn, &uevents_mu); + + if (reg_retcode >= 0) { + te_recalculate_current_thread_data( + tsdn, reg_retcode, te_uobj->is_alloc_only); + ret = 0; + } else if (reg_retcode == TE_REGISTER_ERRCODE_ALREADY_REGISTERED) { + ret = 0; + } else { + ret = EINVAL; + } + + return ret; +} diff --git a/dep/jemalloc/src/ticker.c b/dep/jemalloc/src/ticker.c index 790b5c200..1fd6ac968 100644 --- a/dep/jemalloc/src/ticker.c +++ b/dep/jemalloc/src/ticker.c @@ -20,13 +20,8 @@ * The values here are computed in src/ticker.py */ -const uint8_t ticker_geom_table[1 << TICKER_GEOM_NBITS] = { - 254, 211, 187, 169, 156, 144, 135, 127, - 120, 113, 107, 102, 97, 93, 89, 85, - 81, 77, 74, 71, 68, 65, 62, 60, - 57, 55, 53, 50, 48, 46, 44, 42, - 40, 39, 37, 35, 33, 32, 30, 29, - 27, 26, 24, 23, 21, 20, 19, 18, - 16, 15, 14, 13, 12, 10, 9, 8, - 7, 6, 5, 4, 3, 2, 1, 0 -}; +const uint8_t ticker_geom_table[1 << TICKER_GEOM_NBITS] = {254, 211, 187, 169, + 156, 144, 135, 127, 120, 113, 107, 102, 97, 93, 89, 85, 81, 77, 74, 71, 68, + 65, 62, 60, 57, 55, 53, 50, 48, 46, 44, 42, 40, 39, 37, 35, 33, 32, 30, 29, + 27, 26, 24, 23, 21, 20, 19, 18, 16, 15, 14, 13, 12, 10, 9, 8, 7, 6, 5, 4, 3, + 2, 1, 0}; diff --git a/dep/jemalloc/src/tsd.c b/dep/jemalloc/src/tsd.c index e8e4f3a33..30acad93e 100644 --- a/dep/jemalloc/src/tsd.c +++ b/dep/jemalloc/src/tsd.c @@ -20,11 +20,20 @@ bool tsd_booted = false; #elif (defined(JEMALLOC_TLS)) JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls = TSD_INITIALIZER; pthread_key_t tsd_tsd; -bool tsd_booted = false; +bool tsd_booted = false; #elif (defined(_WIN32)) -DWORD tsd_tsd; -tsd_wrapper_t tsd_boot_wrapper = {false, TSD_INITIALIZER}; +# if defined(JEMALLOC_LEGACY_WINDOWS_SUPPORT) || !defined(_MSC_VER) +DWORD tsd_tsd; +tsd_wrapper_t tsd_boot_wrapper = {TSD_INITIALIZER, false}; +# else +JEMALLOC_TSD_TYPE_ATTR(tsd_wrapper_t) +tsd_wrapper_tls = {TSD_INITIALIZER, false}; +# endif bool tsd_booted = false; +# if JEMALLOC_WIN32_TLSGETVALUE2 +TGV2 tls_get_value2 = NULL; +HMODULE tgv2_mod = NULL; +# endif #else /* @@ -37,17 +46,12 @@ struct tsd_init_head_s { malloc_mutex_t lock; }; -pthread_key_t tsd_tsd; -tsd_init_head_t tsd_init_head = { - ql_head_initializer(blocks), - MALLOC_MUTEX_INITIALIZER -}; +pthread_key_t tsd_tsd; +tsd_init_head_t tsd_init_head = { + ql_head_initializer(blocks), MALLOC_MUTEX_INITIALIZER}; -tsd_wrapper_t tsd_boot_wrapper = { - false, - TSD_INITIALIZER -}; -bool tsd_booted = false; +tsd_wrapper_t tsd_boot_wrapper = {false, TSD_INITIALIZER}; +bool tsd_booted = false; #endif JEMALLOC_DIAGNOSTIC_POP @@ -56,7 +60,7 @@ JEMALLOC_DIAGNOSTIC_POP /* A list of all the tsds in the nominal state. */ typedef ql_head(tsd_t) tsd_list_t; -static tsd_list_t tsd_nominal_tsds = ql_head_initializer(tsd_nominal_tsds); +static tsd_list_t tsd_nominal_tsds = ql_head_initializer(tsd_nominal_tsds); static malloc_mutex_t tsd_nominal_tsds_lock; /* How many slow-path-enabling features are turned on. */ @@ -65,13 +69,13 @@ static atomic_u32_t tsd_global_slow_count = ATOMIC_INIT(0); static bool tsd_in_nominal_list(tsd_t *tsd) { tsd_t *tsd_list; - bool found = false; + bool found = false; /* * We don't know that tsd is nominal; it might not be safe to get data * out of it here. */ malloc_mutex_lock(TSDN_NULL, &tsd_nominal_tsds_lock); - ql_foreach(tsd_list, &tsd_nominal_tsds, TSD_MANGLE(tsd_link)) { + ql_foreach (tsd_list, &tsd_nominal_tsds, TSD_MANGLE(tsd_link)) { if (tsd == tsd_list) { found = true; break; @@ -109,7 +113,7 @@ tsd_force_recompute(tsdn_t *tsdn) { atomic_fence(ATOMIC_RELEASE); malloc_mutex_lock(tsdn, &tsd_nominal_tsds_lock); tsd_t *remote_tsd; - ql_foreach(remote_tsd, &tsd_nominal_tsds, TSD_MANGLE(tsd_link)) { + ql_foreach (remote_tsd, &tsd_nominal_tsds, TSD_MANGLE(tsd_link)) { assert(tsd_atomic_load(&remote_tsd->state, ATOMIC_RELAXED) <= tsd_state_nominal_max); tsd_atomic_store(&remote_tsd->state, @@ -135,7 +139,8 @@ tsd_global_slow_inc(tsdn_t *tsdn) { tsd_force_recompute(tsdn); } -void tsd_global_slow_dec(tsdn_t *tsdn) { +void +tsd_global_slow_dec(tsdn_t *tsdn) { atomic_fetch_sub_u32(&tsd_global_slow_count, 1, ATOMIC_RELAXED); /* See the note in ..._inc(). */ tsd_force_recompute(tsdn); @@ -148,7 +153,7 @@ tsd_local_slow(tsd_t *tsd) { } bool -tsd_global_slow() { +tsd_global_slow(void) { return atomic_load_u32(&tsd_global_slow_count, ATOMIC_RELAXED) > 0; } @@ -172,8 +177,8 @@ tsd_slow_update(tsd_t *tsd) { uint8_t old_state; do { uint8_t new_state = tsd_state_compute(tsd); - old_state = tsd_atomic_exchange(&tsd->state, new_state, - ATOMIC_ACQUIRE); + old_state = tsd_atomic_exchange( + &tsd->state, new_state, ATOMIC_ACQUIRE); } while (old_state == tsd_state_nominal_recompute); te_recompute_fast_threshold(tsd); @@ -203,8 +208,8 @@ tsd_state_set(tsd_t *tsd, uint8_t new_state) { assert(tsd_in_nominal_list(tsd)); if (new_state > tsd_state_nominal_max) { tsd_remove_nominal(tsd); - tsd_atomic_store(&tsd->state, new_state, - ATOMIC_RELAXED); + tsd_atomic_store( + &tsd->state, new_state, ATOMIC_RELAXED); } else { /* * This is the tricky case. We're transitioning from @@ -227,8 +232,7 @@ tsd_prng_state_init(tsd_t *tsd) { * cost of test repeatability. For debug builds, instead use a * deterministic seed. */ - *tsd_prng_statep_get(tsd) = config_debug ? 0 : - (uint64_t)(uintptr_t)tsd; + *tsd_prng_statep_get(tsd) = config_debug ? 0 : (uint64_t)(uintptr_t)tsd; } static bool @@ -256,8 +260,8 @@ assert_tsd_data_cleanup_done(tsd_t *tsd) { static bool tsd_data_init_nocleanup(tsd_t *tsd) { - assert(tsd_state_get(tsd) == tsd_state_reincarnated || - tsd_state_get(tsd) == tsd_state_minimal_initialized); + assert(tsd_state_get(tsd) == tsd_state_reincarnated + || tsd_state_get(tsd) == tsd_state_minimal_initialized); /* * During reincarnation, there is no guarantee that the cleanup function * will be called (deallocation may happen after all tsd destructors). @@ -300,9 +304,25 @@ tsd_fetch_slow(tsd_t *tsd, bool minimal) { tsd_state_set(tsd, tsd_state_minimal_initialized); tsd_set(tsd); tsd_data_init_nocleanup(tsd); + *tsd_min_init_state_nfetchedp_get(tsd) = 1; } } else if (tsd_state_get(tsd) == tsd_state_minimal_initialized) { - if (!minimal) { + /* + * If a thread only ever deallocates (e.g. dedicated reclamation + * threads), we want to help it to eventually escape the slow + * path (caused by the minimal initialized state). The nfetched + * counter tracks the number of times the tsd has been accessed + * under the min init state, and triggers the switch to nominal + * once reached the max allowed count. + * + * This means at most 128 deallocations stay on the slow path. + * + * Also see comments in free_default(). + */ + uint8_t *nfetched = tsd_min_init_state_nfetchedp_get(tsd); + assert(*nfetched >= 1); + (*nfetched)++; + if (!minimal || *nfetched == TSD_MIN_INIT_STATE_MAX_FETCHED) { /* Switch to fully initialized. */ tsd_state_set(tsd, tsd_state_nominal); assert(*tsd_reentrancy_levelp_get(tsd) >= 1); @@ -334,15 +354,15 @@ malloc_tsd_dalloc(void *wrapper) { } #if defined(JEMALLOC_MALLOC_THREAD_CLEANUP) || defined(_WIN32) -static unsigned ncleanups; +static unsigned ncleanups; static malloc_tsd_cleanup_t cleanups[MALLOC_TSD_CLEANUPS_MAX]; -#ifndef _WIN32 +# ifndef _WIN32 JEMALLOC_EXPORT -#endif +# endif void _malloc_thread_cleanup(void) { - bool pending[MALLOC_TSD_CLEANUPS_MAX], again; + bool pending[MALLOC_TSD_CLEANUPS_MAX], again; unsigned i; for (i = 0; i < ncleanups; i++) { @@ -362,9 +382,9 @@ _malloc_thread_cleanup(void) { } while (again); } -#ifndef _WIN32 +# ifndef _WIN32 JEMALLOC_EXPORT -#endif +# endif void _malloc_tsd_cleanup_register(bool (*f)(void)) { assert(ncleanups < MALLOC_TSD_CLEANUPS_MAX); @@ -422,7 +442,7 @@ tsd_cleanup(void *arg) { } #ifdef JEMALLOC_JET test_callback_t test_callback = *tsd_test_callbackp_get_unsafe(tsd); - int *data = tsd_test_datap_get_unsafe(tsd); + int *data = tsd_test_datap_get_unsafe(tsd); if (test_callback != NULL) { test_callback(data); } @@ -437,7 +457,7 @@ malloc_tsd_boot0(void) { ncleanups = 0; #endif if (malloc_mutex_init(&tsd_nominal_tsds_lock, "tsd_nominal_tsds_lock", - WITNESS_RANK_OMIT, malloc_mutex_rank_exclusive)) { + WITNESS_RANK_OMIT, malloc_mutex_rank_exclusive)) { return NULL; } if (tsd_boot0()) { @@ -459,11 +479,11 @@ malloc_tsd_boot1(void) { static BOOL WINAPI _tls_callback(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) { switch (fdwReason) { -#ifdef JEMALLOC_LAZY_LOCK +# ifdef JEMALLOC_LAZY_LOCK case DLL_THREAD_ATTACH: isthreaded = true; break; -#endif +# endif case DLL_THREAD_DETACH: _malloc_thread_cleanup(); break; @@ -478,36 +498,37 @@ _tls_callback(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) { * hooked "read". We won't read for the rest of the file, so we can get away * with unhooking. */ -#ifdef read -# undef read +# ifdef read +# undef read +# endif + +# ifdef _MSC_VER +# ifdef _M_IX86 +# pragma comment(linker, "/INCLUDE:__tls_used") +# pragma comment(linker, "/INCLUDE:_tls_callback") +# else +# pragma comment(linker, "/INCLUDE:_tls_used") +# pragma comment( \ + linker, "/INCLUDE:" STRINGIFY(tls_callback)) +# endif +# pragma section(".CRT$XLY", long, read) +# endif +JEMALLOC_SECTION(".CRT$XLY") +JEMALLOC_ATTR(used) BOOL(WINAPI *const tls_callback)( + HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) = _tls_callback; #endif -#ifdef _MSC_VER -# ifdef _M_IX86 -# pragma comment(linker, "/INCLUDE:__tls_used") -# pragma comment(linker, "/INCLUDE:_tls_callback") -# else -# pragma comment(linker, "/INCLUDE:_tls_used") -# pragma comment(linker, "/INCLUDE:" STRINGIFY(tls_callback) ) -# endif -# pragma section(".CRT$XLY",long,read) -#endif -JEMALLOC_SECTION(".CRT$XLY") JEMALLOC_ATTR(used) -BOOL (WINAPI *const tls_callback)(HINSTANCE hinstDLL, - DWORD fdwReason, LPVOID lpvReserved) = _tls_callback; -#endif - -#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ - !defined(_WIN32)) +#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) \ + && !defined(_WIN32)) void * tsd_init_check_recursion(tsd_init_head_t *head, tsd_init_block_t *block) { - pthread_t self = pthread_self(); + pthread_t self = pthread_self(); tsd_init_block_t *iter; /* Check whether this thread has already inserted into the list. */ malloc_mutex_lock(TSDN_NULL, &head->lock); - ql_foreach(iter, &head->blocks, link) { - if (iter->thread == self) { + ql_foreach (iter, &head->blocks, link) { + if (pthread_equal(iter->thread, self)) { malloc_mutex_unlock(TSDN_NULL, &head->lock); return iter->data; } diff --git a/dep/jemalloc/src/util.c b/dep/jemalloc/src/util.c new file mode 100644 index 000000000..1bcf4fee4 --- /dev/null +++ b/dep/jemalloc/src/util.c @@ -0,0 +1,48 @@ +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" + +#include "jemalloc/internal/util.h" + +/* Reads the next size pair in a multi-sized option. */ +bool +multi_setting_parse_next(const char **setting_segment_cur, size_t *len_left, + size_t *key_start, size_t *key_end, size_t *value) { + const char *cur = *setting_segment_cur; + char *end; + uintmax_t um; + + set_errno(0); + + /* First number, then '-' */ + um = malloc_strtoumax(cur, &end, 0); + if (get_errno() != 0 || *end != '-') { + return true; + } + *key_start = (size_t)um; + cur = end + 1; + + /* Second number, then ':' */ + um = malloc_strtoumax(cur, &end, 0); + if (get_errno() != 0 || *end != ':') { + return true; + } + *key_end = (size_t)um; + cur = end + 1; + + /* Last number */ + um = malloc_strtoumax(cur, &end, 0); + if (get_errno() != 0) { + return true; + } + *value = (size_t)um; + + /* Consume the separator if there is one. */ + if (*end == '|') { + end++; + } + + *len_left -= end - *setting_segment_cur; + *setting_segment_cur = end; + + return false; +} diff --git a/dep/jemalloc/src/witness.c b/dep/jemalloc/src/witness.c index 4474af04c..940b1eae8 100644 --- a/dep/jemalloc/src/witness.c +++ b/dep/jemalloc/src/witness.c @@ -26,8 +26,8 @@ witness_print_witness(witness_t *w, unsigned n) { static void witness_print_witnesses(const witness_list_t *witnesses) { witness_t *w, *last = NULL; - unsigned n = 0; - ql_foreach(w, witnesses, link) { + unsigned n = 0; + ql_foreach (w, witnesses, link) { if (last != NULL && w->rank > last->rank) { assert(w->name != last->name); witness_print_witness(last, n); @@ -45,8 +45,8 @@ witness_print_witnesses(const witness_list_t *witnesses) { } static void -witness_lock_error_impl(const witness_list_t *witnesses, - const witness_t *witness) { +witness_lock_error_impl( + const witness_list_t *witnesses, const witness_t *witness) { malloc_printf(": Lock rank order reversal:"); witness_print_witnesses(witnesses); malloc_printf(" %s(%u)\n", witness->name, witness->rank); @@ -56,8 +56,8 @@ witness_lock_error_t *JET_MUTABLE witness_lock_error = witness_lock_error_impl; static void witness_owner_error_impl(const witness_t *witness) { - malloc_printf(": Should own %s(%u)\n", witness->name, - witness->rank); + malloc_printf( + ": Should own %s(%u)\n", witness->name, witness->rank); abort(); } witness_owner_error_t *JET_MUTABLE witness_owner_error = @@ -76,7 +76,7 @@ static void witness_depth_error_impl(const witness_list_t *witnesses, witness_rank_t rank_inclusive, unsigned depth) { malloc_printf(": Should own %u lock%s of rank >= %u:", depth, - (depth != 1) ? "s" : "", rank_inclusive); + (depth != 1) ? "s" : "", rank_inclusive); witness_print_witnesses(witnesses); malloc_printf("\n"); abort(); diff --git a/dep/jemalloc/src/zone.c b/dep/jemalloc/src/zone.c index 23dfdd04a..e09de4b89 100644 --- a/dep/jemalloc/src/zone.c +++ b/dep/jemalloc/src/zone.c @@ -4,7 +4,7 @@ #include "jemalloc/internal/assert.h" #ifndef JEMALLOC_ZONE -# error "This source file is for zones on Darwin (OS X)." +# error "This source file is for zones on Darwin (OS X)." #endif /* Definitions of the following structs in malloc/malloc.h might be too old @@ -22,10 +22,11 @@ typedef struct _malloc_zone_t { void *(*realloc)(struct _malloc_zone_t *, void *, size_t); void (*destroy)(struct _malloc_zone_t *); const char *zone_name; - unsigned (*batch_malloc)(struct _malloc_zone_t *, size_t, void **, unsigned); + unsigned (*batch_malloc)( + struct _malloc_zone_t *, size_t, void **, unsigned); void (*batch_free)(struct _malloc_zone_t *, void **, unsigned); struct malloc_introspection_t *introspect; - unsigned version; + unsigned version; void *(*memalign)(struct _malloc_zone_t *, size_t, size_t); void (*free_definite_size)(struct _malloc_zone_t *, void *, size_t); size_t (*pressure_relief)(struct _malloc_zone_t *, size_t); @@ -33,22 +34,24 @@ typedef struct _malloc_zone_t { typedef struct { vm_address_t address; - vm_size_t size; + vm_size_t size; } vm_range_t; typedef struct malloc_statistics_t { unsigned blocks_in_use; - size_t size_in_use; - size_t max_size_in_use; - size_t size_allocated; + size_t size_in_use; + size_t max_size_in_use; + size_t size_allocated; } malloc_statistics_t; typedef kern_return_t memory_reader_t(task_t, vm_address_t, vm_size_t, void **); -typedef void vm_range_recorder_t(task_t, void *, unsigned type, vm_range_t *, unsigned); +typedef void vm_range_recorder_t( + task_t, void *, unsigned type, vm_range_t *, unsigned); typedef struct malloc_introspection_t { - kern_return_t (*enumerator)(task_t, void *, unsigned, vm_address_t, memory_reader_t, vm_range_recorder_t); + kern_return_t (*enumerator)(task_t, void *, unsigned, vm_address_t, + memory_reader_t, vm_range_recorder_t); size_t (*good_size)(malloc_zone_t *, size_t); boolean_t (*check)(malloc_zone_t *); void (*print)(malloc_zone_t *, boolean_t); @@ -61,14 +64,16 @@ typedef struct malloc_introspection_t { boolean_t (*disable_discharge_checking)(malloc_zone_t *); void (*discharge)(malloc_zone_t *, void *); #ifdef __BLOCKS__ - void (*enumerate_discharged_pointers)(malloc_zone_t *, void (^)(void *, void *)); + void (*enumerate_discharged_pointers)( + malloc_zone_t *, void (^)(void *, void *)); #else void *enumerate_unavailable_without_blocks; #endif void (*reinit_lock)(malloc_zone_t *); } malloc_introspection_t; -extern kern_return_t malloc_get_all_zones(task_t, memory_reader_t, vm_address_t **, unsigned *); +extern kern_return_t malloc_get_all_zones( + task_t, memory_reader_t, vm_address_t **, unsigned *); extern malloc_zone_t *malloc_default_zone(void); @@ -81,48 +86,46 @@ extern void malloc_zone_unregister(malloc_zone_t *zone); * We need to check whether it is present at runtime, thus the weak_import. */ extern malloc_zone_t *malloc_default_purgeable_zone(void) -JEMALLOC_ATTR(weak_import); + JEMALLOC_ATTR(weak_import); /******************************************************************************/ /* Data. */ -static malloc_zone_t *default_zone, *purgeable_zone; -static malloc_zone_t jemalloc_zone; +static malloc_zone_t *default_zone, *purgeable_zone; +static malloc_zone_t jemalloc_zone; static struct malloc_introspection_t jemalloc_zone_introspect; -static pid_t zone_force_lock_pid = -1; +static pid_t zone_force_lock_pid = -1; /******************************************************************************/ /* Function prototypes for non-inline static functions. */ -static size_t zone_size(malloc_zone_t *zone, const void *ptr); -static void *zone_malloc(malloc_zone_t *zone, size_t size); -static void *zone_calloc(malloc_zone_t *zone, size_t num, size_t size); -static void *zone_valloc(malloc_zone_t *zone, size_t size); -static void zone_free(malloc_zone_t *zone, void *ptr); -static void *zone_realloc(malloc_zone_t *zone, void *ptr, size_t size); -static void *zone_memalign(malloc_zone_t *zone, size_t alignment, - size_t size); -static void zone_free_definite_size(malloc_zone_t *zone, void *ptr, - size_t size); -static void zone_destroy(malloc_zone_t *zone); -static unsigned zone_batch_malloc(struct _malloc_zone_t *zone, size_t size, +static size_t zone_size(malloc_zone_t *zone, const void *ptr); +static void *zone_malloc(malloc_zone_t *zone, size_t size); +static void *zone_calloc(malloc_zone_t *zone, size_t num, size_t size); +static void *zone_valloc(malloc_zone_t *zone, size_t size); +static void zone_free(malloc_zone_t *zone, void *ptr); +static void *zone_realloc(malloc_zone_t *zone, void *ptr, size_t size); +static void *zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size); +static void zone_free_definite_size( + malloc_zone_t *zone, void *ptr, size_t size); +static void zone_destroy(malloc_zone_t *zone); +static unsigned zone_batch_malloc(struct _malloc_zone_t *zone, size_t size, void **results, unsigned num_requested); -static void zone_batch_free(struct _malloc_zone_t *zone, - void **to_be_freed, unsigned num_to_be_freed); -static size_t zone_pressure_relief(struct _malloc_zone_t *zone, size_t goal); -static size_t zone_good_size(malloc_zone_t *zone, size_t size); -static kern_return_t zone_enumerator(task_t task, void *data, unsigned type_mask, - vm_address_t zone_address, memory_reader_t reader, +static void zone_batch_free( + struct _malloc_zone_t *zone, void **to_be_freed, unsigned num_to_be_freed); +static size_t zone_pressure_relief(struct _malloc_zone_t *zone, size_t goal); +static size_t zone_good_size(malloc_zone_t *zone, size_t size); +static kern_return_t zone_enumerator(task_t task, void *data, + unsigned type_mask, vm_address_t zone_address, memory_reader_t reader, vm_range_recorder_t recorder); -static boolean_t zone_check(malloc_zone_t *zone); -static void zone_print(malloc_zone_t *zone, boolean_t verbose); -static void zone_log(malloc_zone_t *zone, void *address); -static void zone_force_lock(malloc_zone_t *zone); -static void zone_force_unlock(malloc_zone_t *zone); -static void zone_statistics(malloc_zone_t *zone, - malloc_statistics_t *stats); -static boolean_t zone_locked(malloc_zone_t *zone); -static void zone_reinit_lock(malloc_zone_t *zone); +static boolean_t zone_check(malloc_zone_t *zone); +static void zone_print(malloc_zone_t *zone, boolean_t verbose); +static void zone_log(malloc_zone_t *zone, void *address); +static void zone_force_lock(malloc_zone_t *zone); +static void zone_force_unlock(malloc_zone_t *zone); +static void zone_statistics(malloc_zone_t *zone, malloc_statistics_t *stats); +static boolean_t zone_locked(malloc_zone_t *zone); +static void zone_reinit_lock(malloc_zone_t *zone); /******************************************************************************/ /* @@ -225,8 +228,8 @@ zone_batch_malloc(struct _malloc_zone_t *zone, size_t size, void **results, } static void -zone_batch_free(struct _malloc_zone_t *zone, void **to_be_freed, - unsigned num_to_be_freed) { +zone_batch_free( + struct _malloc_zone_t *zone, void **to_be_freed, unsigned num_to_be_freed) { unsigned i; for (i = 0; i < num_to_be_freed; i++) { @@ -261,12 +264,10 @@ zone_check(malloc_zone_t *zone) { } static void -zone_print(malloc_zone_t *zone, boolean_t verbose) { -} +zone_print(malloc_zone_t *zone, boolean_t verbose) {} static void -zone_log(malloc_zone_t *zone, void *address) { -} +zone_log(malloc_zone_t *zone, void *address) {} static void zone_force_lock(malloc_zone_t *zone) { @@ -369,7 +370,7 @@ zone_init(void) { static malloc_zone_t * zone_default_get(void) { malloc_zone_t **zones = NULL; - unsigned int num_zones = 0; + unsigned int num_zones = 0; /* * On OSX 10.12, malloc_default_zone returns a special zone that is not @@ -380,8 +381,9 @@ zone_default_get(void) { * zone is the default. So get the list of zones to get the first one, * instead of relying on malloc_default_zone. */ - if (KERN_SUCCESS != malloc_get_all_zones(0, NULL, - (vm_address_t**)&zones, &num_zones)) { + if (KERN_SUCCESS + != malloc_get_all_zones( + 0, NULL, (vm_address_t **)&zones, &num_zones)) { /* * Reset the value in case the failure happened after it was * set. @@ -441,8 +443,8 @@ zone_register(void) { * register jemalloc's. */ default_zone = zone_default_get(); - if (!default_zone->zone_name || strcmp(default_zone->zone_name, - "DefaultMallocZone") != 0) { + if (!default_zone->zone_name + || strcmp(default_zone->zone_name, "DefaultMallocZone") != 0) { return; } @@ -457,8 +459,9 @@ zone_register(void) { * to check for the existence of malloc_default_purgeable_zone() at * run time. */ - purgeable_zone = (malloc_default_purgeable_zone == NULL) ? NULL : - malloc_default_purgeable_zone(); + purgeable_zone = (malloc_default_purgeable_zone == NULL) + ? NULL + : malloc_default_purgeable_zone(); /* Register the custom zone. At this point it won't be the default. */ zone_init();